05 - Top 10 pandas tips and tricks.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "edf11b90",
   "metadata": {},
   "source": [
    "# Top 10 pandas tips and tricks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b3cf8f15",
   "metadata": {
    "hide_input": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from seaborn import load_dataset\n",
    "\n",
    "pd.set_option('display.max_columns', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "bc0bdb6c",
   "metadata": {
    "hide_input": true
   },
   "outputs": [],
   "source": [
    "diamonds = load_dataset(\"diamonds\")\n",
    "tips = load_dataset(\"tips\")\n",
    "dow = load_dataset(\"dowjones\")\n",
    "crashes = load_dataset(\"car_crashes\")\n",
    "glue = load_dataset(\"glue\")\n",
    "taxis = load_dataset(\"taxis\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7299af83",
   "metadata": {
    "hide_input": true
   },
   "source": [
    "# 1. Convert all columns to lowercase and remove whitespace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9e4b0111",
   "metadata": {
    "hide_input": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>User  ID</th>\n",
       "      <th>product/description</th>\n",
       "      <th>price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>iphone 11</td>\n",
       "      <td>11.756733</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>iphone 11</td>\n",
       "      <td>11.022684</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>iphone 11</td>\n",
       "      <td>9.753071</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>iphone 11</td>\n",
       "      <td>9.855838</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>samsung s20</td>\n",
       "      <td>9.757330</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2</td>\n",
       "      <td>samsung s20</td>\n",
       "      <td>8.069593</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2</td>\n",
       "      <td>samsung s20</td>\n",
       "      <td>11.352987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2</td>\n",
       "      <td>samsung s20</td>\n",
       "      <td>9.816936</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   User  ID product/description      price\n",
       "0         1           iphone 11  11.756733\n",
       "1         1           iphone 11  11.022684\n",
       "2         1           iphone 11   9.753071\n",
       "3         1           iphone 11   9.855838\n",
       "4         2         samsung s20   9.757330\n",
       "5         2         samsung s20   8.069593\n",
       "6         2         samsung s20  11.352987\n",
       "7         2         samsung s20   9.816936"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame({\n",
    "    \"User  ID\": [1, 1, 1, 1, 2, 2, 2, 2],\n",
    "    \"product/description\": [\"iphone 11\"] * 4 + [\"samsung s20\"] * 4,\n",
    "    \"price\": np.random.randn(8) + 10,\n",
    "})\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b6d0a64c",
   "metadata": {
    "hide_input": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['User  ID', 'product/description', 'price'], dtype='object')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "58eb651d",
   "metadata": {
    "hide_input": true
   },
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "def clean_column(s):\n",
    "    s = str(s)                          # convert to string just in case\n",
    "    s = re.sub(\"/\", \" \", s)             # replace / for whitespace\n",
    "    s = re.sub(\" {1,}\", \" \", s)         # replace multiple whitespaces to one whitespace  \n",
    "    s = re.sub(\" \", \"_\", s)             # replace a whitespace for _\n",
    "    return s.lower()    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e8353546",
   "metadata": {
    "hide_input": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'user_id'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean_column('User  ID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "363c3be9",
   "metadata": {
    "direction": "rtl",
    "hide_input": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'product_description'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean_column('product/description')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "92a0dfee",
   "metadata": {
    "hide_input": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['user_id', 'product_description', 'price'], dtype='object')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns = [clean_column(col) for col in df.columns]\n",
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ce137cf9",
   "metadata": {
    "hide_input": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>product_description</th>\n",
       "      <th>price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>iphone 11</td>\n",
       "      <td>11.756733</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>iphone 11</td>\n",
       "      <td>11.022684</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>iphone 11</td>\n",
       "      <td>9.753071</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>iphone 11</td>\n",
       "      <td>9.855838</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>samsung s20</td>\n",
       "      <td>9.757330</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id product_description      price\n",
       "0        1           iphone 11  11.756733\n",
       "1        1           iphone 11  11.022684\n",
       "2        1           iphone 11   9.753071\n",
       "3        1           iphone 11   9.855838\n",
       "4        2         samsung s20   9.757330"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Cleaned data, now we can use df.user_id syntax!\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "607e7ce2",
   "metadata": {},
   "source": [
    "# 2. Select columns by dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "e3bc78dc",
   "metadata": {
    "hide_input": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "carat       float64\n",
       "cut        category\n",
       "color      category\n",
       "clarity    category\n",
       "depth       float64\n",
       "table       float64\n",
       "price         int64\n",
       "x           float64\n",
       "y           float64\n",
       "z           float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "diamonds.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "aabc8828",
   "metadata": {
    "hide_input": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cut</th>\n",
       "      <th>color</th>\n",
       "      <th>clarity</th>\n",
       "      <th>price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Ideal</td>\n",
       "      <td>E</td>\n",
       "      <td>SI2</td>\n",
       "      <td>326</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Premium</td>\n",
       "      <td>E</td>\n",
       "      <td>SI1</td>\n",
       "      <td>326</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Good</td>\n",
       "      <td>E</td>\n",
       "      <td>VS1</td>\n",
       "      <td>327</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Premium</td>\n",
       "      <td>I</td>\n",
       "      <td>VS2</td>\n",
       "      <td>334</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Good</td>\n",
       "      <td>J</td>\n",
       "      <td>SI2</td>\n",
       "      <td>335</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53935</th>\n",
       "      <td>Ideal</td>\n",
       "      <td>D</td>\n",
       "      <td>SI1</td>\n",
       "      <td>2757</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53936</th>\n",
       "      <td>Good</td>\n",
       "      <td>D</td>\n",
       "      <td>SI1</td>\n",
       "      <td>2757</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53937</th>\n",
       "      <td>Very Good</td>\n",
       "      <td>D</td>\n",
       "      <td>SI1</td>\n",
       "      <td>2757</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53938</th>\n",
       "      <td>Premium</td>\n",
       "      <td>H</td>\n",
       "      <td>SI2</td>\n",
       "      <td>2757</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53939</th>\n",
       "      <td>Ideal</td>\n",
       "      <td>D</td>\n",
       "      <td>SI2</td>\n",
       "      <td>2757</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>53940 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             cut color clarity  price\n",
       "0          Ideal     E     SI2    326\n",
       "1        Premium     E     SI1    326\n",
       "2           Good     E     VS1    327\n",
       "3        Premium     I     VS2    334\n",
       "4           Good     J     SI2    335\n",
       "...          ...   ...     ...    ...\n",
       "53935      Ideal     D     SI1   2757\n",
       "53936       Good     D     SI1   2757\n",
       "53937  Very Good     D     SI1   2757\n",
       "53938    Premium     H     SI2   2757\n",
       "53939      Ideal     D     SI2   2757\n",
       "\n",
       "[53940 rows x 4 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "diamonds.select_dtypes([\"category\", \"int\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "006b0a04",
   "metadata": {},
   "source": [
    "# 3. Group by multiple columns - Avoid MultiIndex output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "b40b91a9",
   "metadata": {
    "hide_input": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cut</th>\n",
       "      <th>color</th>\n",
       "      <th>clarity</th>\n",
       "      <th>price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Ideal</td>\n",
       "      <td>D</td>\n",
       "      <td>IF</td>\n",
       "      <td>6567.178571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Ideal</td>\n",
       "      <td>D</td>\n",
       "      <td>VVS1</td>\n",
       "      <td>2705.777778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Ideal</td>\n",
       "      <td>D</td>\n",
       "      <td>VVS2</td>\n",
       "      <td>3619.014085</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Ideal</td>\n",
       "      <td>D</td>\n",
       "      <td>VS1</td>\n",
       "      <td>2576.039886</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Ideal</td>\n",
       "      <td>D</td>\n",
       "      <td>VS2</td>\n",
       "      <td>2111.927174</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     cut color clarity        price\n",
       "0  Ideal     D      IF  6567.178571\n",
       "1  Ideal     D    VVS1  2705.777778\n",
       "2  Ideal     D    VVS2  3619.014085\n",
       "3  Ideal     D     VS1  2576.039886\n",
       "4  Ideal     D     VS2  2111.927174"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cols = ['cut', 'color', 'clarity', 'price']\n",
    "\n",
    "agg_price = (\n",
    "    diamonds\n",
    "    [cols]                                  # select columns using a list!\n",
    "    .groupby(['cut', 'color', 'clarity'],   # group by 3 columns\n",
    "             as_index=False)                # avoid the multi-index output\n",
    "    .mean()                                 # aggregate the data\n",
    ")\n",
    "agg_price.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a26324e7",
   "metadata": {},
   "source": [
    "# 4. Select the top-n rows (nlargest, nsmallest)\n",
    "\n",
    "This is generally useful after an aggregation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "3eb0a11e",
   "metadata": {
    "hide_input": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cut</th>\n",
       "      <th>color</th>\n",
       "      <th>clarity</th>\n",
       "      <th>price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>112</th>\n",
       "      <td>Very Good</td>\n",
       "      <td>D</td>\n",
       "      <td>IF</td>\n",
       "      <td>10298.260870</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>168</th>\n",
       "      <td>Good</td>\n",
       "      <td>D</td>\n",
       "      <td>IF</td>\n",
       "      <td>10030.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>Ideal</td>\n",
       "      <td>J</td>\n",
       "      <td>I1</td>\n",
       "      <td>9454.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>Premium</td>\n",
       "      <td>D</td>\n",
       "      <td>IF</td>\n",
       "      <td>9056.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>110</th>\n",
       "      <td>Premium</td>\n",
       "      <td>J</td>\n",
       "      <td>SI2</td>\n",
       "      <td>7550.285714</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           cut color clarity         price\n",
       "112  Very Good     D      IF  10298.260870\n",
       "168       Good     D      IF  10030.333333\n",
       "55       Ideal     J      I1   9454.000000\n",
       "56     Premium     D      IF   9056.500000\n",
       "110    Premium     J     SI2   7550.285714"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg_price.nlargest(5, 'price')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "27ddd141",
   "metadata": {
    "hide_input": true,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cut</th>\n",
       "      <th>color</th>\n",
       "      <th>clarity</th>\n",
       "      <th>price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>160</th>\n",
       "      <td>Very Good</td>\n",
       "      <td>J</td>\n",
       "      <td>IF</td>\n",
       "      <td>1074.125000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>Premium</td>\n",
       "      <td>H</td>\n",
       "      <td>VVS1</td>\n",
       "      <td>1453.758929</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>248</th>\n",
       "      <td>Fair</td>\n",
       "      <td>G</td>\n",
       "      <td>IF</td>\n",
       "      <td>1488.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>Ideal</td>\n",
       "      <td>I</td>\n",
       "      <td>IF</td>\n",
       "      <td>1502.621053</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>176</th>\n",
       "      <td>Good</td>\n",
       "      <td>E</td>\n",
       "      <td>IF</td>\n",
       "      <td>1519.222222</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           cut color clarity        price\n",
       "160  Very Good     J      IF  1074.125000\n",
       "89     Premium     H    VVS1  1453.758929\n",
       "248       Fair     G      IF  1488.000000\n",
       "40       Ideal     I      IF  1502.621053\n",
       "176       Good     E      IF  1519.222222"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agg_price.nsmallest(5, 'price')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "97bfac0c",
   "metadata": {},
   "source": [
    "# 5. Use melt to convert \"wide\" data to \"long\" format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "6397f010",
   "metadata": {
    "hide_input": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Country code</th>\n",
       "      <th>Country name</th>\n",
       "      <th>Series code</th>\n",
       "      <th>Series name</th>\n",
       "      <th>SCALE</th>\n",
       "      <th>Decimals</th>\n",
       "      <th>1990</th>\n",
       "      <th>1991</th>\n",
       "      <th>1992</th>\n",
       "      <th>1993</th>\n",
       "      <th>1994</th>\n",
       "      <th>1995</th>\n",
       "      <th>1996</th>\n",
       "      <th>1997</th>\n",
       "      <th>1998</th>\n",
       "      <th>1999</th>\n",
       "      <th>2000</th>\n",
       "      <th>2001</th>\n",
       "      <th>2002</th>\n",
       "      <th>2003</th>\n",
       "      <th>2004</th>\n",
       "      <th>2005</th>\n",
       "      <th>2006</th>\n",
       "      <th>2007</th>\n",
       "      <th>2008</th>\n",
       "      <th>2009</th>\n",
       "      <th>2010</th>\n",
       "      <th>2011</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ABW</td>\n",
       "      <td>Aruba</td>\n",
       "      <td>AG.LND.EL5M.ZS</td>\n",
       "      <td>Land area below 5m (% of land area)</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>29.574810</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>29.574810</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ADO</td>\n",
       "      <td>Andorra</td>\n",
       "      <td>AG.LND.EL5M.ZS</td>\n",
       "      <td>Land area below 5m (% of land area)</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AFG</td>\n",
       "      <td>Afghanistan</td>\n",
       "      <td>AG.LND.EL5M.ZS</td>\n",
       "      <td>Land area below 5m (% of land area)</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AGO</td>\n",
       "      <td>Angola</td>\n",
       "      <td>AG.LND.EL5M.ZS</td>\n",
       "      <td>Land area below 5m (% of land area)</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.208235</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.208235</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ALB</td>\n",
       "      <td>Albania</td>\n",
       "      <td>AG.LND.EL5M.ZS</td>\n",
       "      <td>Land area below 5m (% of land area)</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4.967875</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.967875</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Country code Country name     Series code  \\\n",
       "0          ABW        Aruba  AG.LND.EL5M.ZS   \n",
       "1          ADO      Andorra  AG.LND.EL5M.ZS   \n",
       "2          AFG  Afghanistan  AG.LND.EL5M.ZS   \n",
       "3          AGO       Angola  AG.LND.EL5M.ZS   \n",
       "4          ALB      Albania  AG.LND.EL5M.ZS   \n",
       "\n",
       "                           Series name SCALE Decimals       1990  1991  1992  \\\n",
       "0  Land area below 5m (% of land area)     0        1  29.574810   NaN   NaN   \n",
       "1  Land area below 5m (% of land area)     0        1   0.000000   NaN   NaN   \n",
       "2  Land area below 5m (% of land area)     0        1   0.000000   NaN   NaN   \n",
       "3  Land area below 5m (% of land area)     0        1   0.208235   NaN   NaN   \n",
       "4  Land area below 5m (% of land area)     0        1   4.967875   NaN   NaN   \n",
       "\n",
       "   1993  1994  1995  1996  1997  1998  1999       2000  2001  2002  2003  \\\n",
       "0   NaN   NaN   NaN   NaN   NaN   NaN   NaN  29.574810   NaN   NaN   NaN   \n",
       "1   NaN   NaN   NaN   NaN   NaN   NaN   NaN   0.000000   NaN   NaN   NaN   \n",
       "2   NaN   NaN   NaN   NaN   NaN   NaN   NaN   0.000000   NaN   NaN   NaN   \n",
       "3   NaN   NaN   NaN   NaN   NaN   NaN   NaN   0.208235   NaN   NaN   NaN   \n",
       "4   NaN   NaN   NaN   NaN   NaN   NaN   NaN   4.967875   NaN   NaN   NaN   \n",
       "\n",
       "   2004  2005  2006  2007  2008  2009  2010 2011  \n",
       "0   NaN   NaN   NaN   NaN   NaN   NaN   NaN  NaN  \n",
       "1   NaN   NaN   NaN   NaN   NaN   NaN   NaN  NaN  \n",
       "2   NaN   NaN   NaN   NaN   NaN   NaN   NaN  NaN  \n",
       "3   NaN   NaN   NaN   NaN   NaN   NaN   NaN  NaN  \n",
       "4   NaN   NaN   NaN   NaN   NaN   NaN   NaN  NaN  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Download data: https://data.world/worldbank/climate-change-data\n",
    "\n",
    "df = pd.read_excel('https://query.data.world/s/6t2guxfsulbqf5igqrd64hj7f6my3u?dws=00000', \n",
    "                  na_values=['..'])\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "7d96872a",
   "metadata": {
    "hide_input": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Country code', 'Country name',  'Series code',  'Series name',\n",
       "              'SCALE',     'Decimals',           1990,           1991,\n",
       "                 1992,           1993,           1994,           1995,\n",
       "                 1996,           1997,           1998,           1999,\n",
       "                 2000,           2001,           2002,           2003,\n",
       "                 2004,           2005,           2006,           2007,\n",
       "                 2008,           2009,           2010,           2011],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "8aa70817",
   "metadata": {
    "hide_input": true
   },
   "outputs": [],
   "source": [
    "# clean up columns\n",
    "df.columns = [clean_column(col) for col in df.columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "334f4962",
   "metadata": {
    "hide_input": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>country_code</th>\n",
       "      <th>country_name</th>\n",
       "      <th>series_code</th>\n",
       "      <th>series_name</th>\n",
       "      <th>scale</th>\n",
       "      <th>decimals</th>\n",
       "      <th>1990</th>\n",
       "      <th>1991</th>\n",
       "      <th>1992</th>\n",
       "      <th>1993</th>\n",
       "      <th>1994</th>\n",
       "      <th>1995</th>\n",
       "      <th>1996</th>\n",
       "      <th>1997</th>\n",
       "      <th>1998</th>\n",
       "      <th>1999</th>\n",
       "      <th>2000</th>\n",
       "      <th>2001</th>\n",
       "      <th>2002</th>\n",
       "      <th>2003</th>\n",
       "      <th>2004</th>\n",
       "      <th>2005</th>\n",
       "      <th>2006</th>\n",
       "      <th>2007</th>\n",
       "      <th>2008</th>\n",
       "      <th>2009</th>\n",
       "      <th>2010</th>\n",
       "      <th>2011</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ABW</td>\n",
       "      <td>Aruba</td>\n",
       "      <td>AG.LND.EL5M.ZS</td>\n",
       "      <td>Land area below 5m (% of land area)</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>29.57481</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>29.57481</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ADO</td>\n",
       "      <td>Andorra</td>\n",
       "      <td>AG.LND.EL5M.ZS</td>\n",
       "      <td>Land area below 5m (% of land area)</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AFG</td>\n",
       "      <td>Afghanistan</td>\n",
       "      <td>AG.LND.EL5M.ZS</td>\n",
       "      <td>Land area below 5m (% of land area)</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  country_code country_name     series_code  \\\n",
       "0          ABW        Aruba  AG.LND.EL5M.ZS   \n",
       "1          ADO      Andorra  AG.LND.EL5M.ZS   \n",
       "2          AFG  Afghanistan  AG.LND.EL5M.ZS   \n",
       "\n",
       "                           series_name scale decimals      1990  1991  1992  \\\n",
       "0  Land area below 5m (% of land area)     0        1  29.57481   NaN   NaN   \n",
       "1  Land area below 5m (% of land area)     0        1   0.00000   NaN   NaN   \n",
       "2  Land area below 5m (% of land area)     0        1   0.00000   NaN   NaN   \n",
       "\n",
       "   1993  1994  1995  1996  1997  1998  1999      2000  2001  2002  2003  2004  \\\n",
       "0   NaN   NaN   NaN   NaN   NaN   NaN   NaN  29.57481   NaN   NaN   NaN   NaN   \n",
       "1   NaN   NaN   NaN   NaN   NaN   NaN   NaN   0.00000   NaN   NaN   NaN   NaN   \n",
       "2   NaN   NaN   NaN   NaN   NaN   NaN   NaN   0.00000   NaN   NaN   NaN   NaN   \n",
       "\n",
       "   2005  2006  2007  2008  2009  2010 2011  \n",
       "0   NaN   NaN   NaN   NaN   NaN   NaN  NaN  \n",
       "1   NaN   NaN   NaN   NaN   NaN   NaN  NaN  \n",
       "2   NaN   NaN   NaN   NaN   NaN   NaN  NaN  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7a544d0d",
   "metadata": {},
   "source": [
    "### Melt arguments: You can access this with ?pd.melt\n",
    "\n",
    "```\n",
    "df.melt(\n",
    "    id_vars=None,\n",
    "    value_vars=None,\n",
    "    var_name=None,\n",
    "    value_name='value',\n",
    "    col_level: 'Level' = None,\n",
    "    ignore_index: 'bool' = True,\n",
    ") -\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "2f8a8f07",
   "metadata": {
    "hide_input": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>country_code</th>\n",
       "      <th>country_name</th>\n",
       "      <th>series_code</th>\n",
       "      <th>series_name</th>\n",
       "      <th>scale</th>\n",
       "      <th>decimals</th>\n",
       "      <th>variable</th>\n",
       "      <th>value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ABW</td>\n",
       "      <td>Aruba</td>\n",
       "      <td>AG.LND.EL5M.ZS</td>\n",
       "      <td>Land area below 5m (% of land area)</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1990</td>\n",
       "      <td>29.57481</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ADO</td>\n",
       "      <td>Andorra</td>\n",
       "      <td>AG.LND.EL5M.ZS</td>\n",
       "      <td>Land area below 5m (% of land area)</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1990</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AFG</td>\n",
       "      <td>Afghanistan</td>\n",
       "      <td>AG.LND.EL5M.ZS</td>\n",
       "      <td>Land area below 5m (% of land area)</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1990</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AGO</td>\n",
       "      <td>Angola</td>\n",
       "      <td>AG.LND.EL5M.ZS</td>\n",
       "      <td>Land area below 5m (% of land area)</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1990</td>\n",
       "      <td>0.208235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ALB</td>\n",
       "      <td>Albania</td>\n",
       "      <td>AG.LND.EL5M.ZS</td>\n",
       "      <td>Land area below 5m (% of land area)</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1990</td>\n",
       "      <td>4.967875</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  country_code country_name     series_code  \\\n",
       "0          ABW        Aruba  AG.LND.EL5M.ZS   \n",
       "1          ADO      Andorra  AG.LND.EL5M.ZS   \n",
       "2          AFG  Afghanistan  AG.LND.EL5M.ZS   \n",
       "3          AGO       Angola  AG.LND.EL5M.ZS   \n",
       "4          ALB      Albania  AG.LND.EL5M.ZS   \n",
       "\n",
       "                           series_name scale decimals variable     value  \n",
       "0  Land area below 5m (% of land area)     0        1     1990  29.57481  \n",
       "1  Land area below 5m (% of land area)     0        1     1990       0.0  \n",
       "2  Land area below 5m (% of land area)     0        1     1990       0.0  \n",
       "3  Land area below 5m (% of land area)     0        1     1990  0.208235  \n",
       "4  Land area below 5m (% of land area)     0        1     1990  4.967875  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_long = df.melt(\n",
    "    id_vars = [\"country_code\", \"country_name\", \"series_code\", \n",
    "               \"series_name\", \"scale\", \"decimals\"]\n",
    "    )\n",
    "df_long.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "8802fd11",
   "metadata": {
    "hide_input": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>country_code</th>\n",
       "      <th>country_name</th>\n",
       "      <th>series_code</th>\n",
       "      <th>series_name</th>\n",
       "      <th>scale</th>\n",
       "      <th>decimals</th>\n",
       "      <th>variable</th>\n",
       "      <th>value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ABW</td>\n",
       "      <td>Aruba</td>\n",
       "      <td>AG.LND.EL5M.ZS</td>\n",
       "      <td>Land area below 5m (% of land area)</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1990</td>\n",
       "      <td>29.574810</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ADO</td>\n",
       "      <td>Andorra</td>\n",
       "      <td>AG.LND.EL5M.ZS</td>\n",
       "      <td>Land area below 5m (% of land area)</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1990</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AFG</td>\n",
       "      <td>Afghanistan</td>\n",
       "      <td>AG.LND.EL5M.ZS</td>\n",
       "      <td>Land area below 5m (% of land area)</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1990</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AGO</td>\n",
       "      <td>Angola</td>\n",
       "      <td>AG.LND.EL5M.ZS</td>\n",
       "      <td>Land area below 5m (% of land area)</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1990</td>\n",
       "      <td>0.208235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ALB</td>\n",
       "      <td>Albania</td>\n",
       "      <td>AG.LND.EL5M.ZS</td>\n",
       "      <td>Land area below 5m (% of land area)</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1990</td>\n",
       "      <td>4.967875</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  country_code country_name     series_code  \\\n",
       "0          ABW        Aruba  AG.LND.EL5M.ZS   \n",
       "1          ADO      Andorra  AG.LND.EL5M.ZS   \n",
       "2          AFG  Afghanistan  AG.LND.EL5M.ZS   \n",
       "3          AGO       Angola  AG.LND.EL5M.ZS   \n",
       "4          ALB      Albania  AG.LND.EL5M.ZS   \n",
       "\n",
       "                           series_name scale decimals variable      value  \n",
       "0  Land area below 5m (% of land area)     0        1     1990  29.574810  \n",
       "1  Land area below 5m (% of land area)     0        1     1990   0.000000  \n",
       "2  Land area below 5m (% of land area)     0        1     1990   0.000000  \n",
       "3  Land area below 5m (% of land area)     0        1     1990   0.208235  \n",
       "4  Land area below 5m (% of land area)     0        1     1990   4.967875  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# you can pass a value vars list - same result but safer if you have more columns\n",
    "df_long = df.melt(\n",
    "    id_vars = [\"country_code\", \"country_name\", \"series_code\", \n",
    "               \"series_name\", \"scale\", \"decimals\"], \n",
    "    value_vars = [str(x) for x in list(np.arange(1990, 2011, 1))]\n",
    "    )\n",
    "df_long.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "ab2fd633",
   "metadata": {
    "hide_input": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>country_code</th>\n",
       "      <th>country_name</th>\n",
       "      <th>series_code</th>\n",
       "      <th>series_name</th>\n",
       "      <th>scale</th>\n",
       "      <th>decimals</th>\n",
       "      <th>year</th>\n",
       "      <th>value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ABW</td>\n",
       "      <td>Aruba</td>\n",
       "      <td>AG.LND.EL5M.ZS</td>\n",
       "      <td>Land area below 5m (% of land area)</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1990</td>\n",
       "      <td>29.574810</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ADO</td>\n",
       "      <td>Andorra</td>\n",
       "      <td>AG.LND.EL5M.ZS</td>\n",
       "      <td>Land area below 5m (% of land area)</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1990</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AFG</td>\n",
       "      <td>Afghanistan</td>\n",
       "      <td>AG.LND.EL5M.ZS</td>\n",
       "      <td>Land area below 5m (% of land area)</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1990</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AGO</td>\n",
       "      <td>Angola</td>\n",
       "      <td>AG.LND.EL5M.ZS</td>\n",
       "      <td>Land area below 5m (% of land area)</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1990</td>\n",
       "      <td>0.208235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ALB</td>\n",
       "      <td>Albania</td>\n",
       "      <td>AG.LND.EL5M.ZS</td>\n",
       "      <td>Land area below 5m (% of land area)</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1990</td>\n",
       "      <td>4.967875</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  country_code country_name     series_code  \\\n",
       "0          ABW        Aruba  AG.LND.EL5M.ZS   \n",
       "1          ADO      Andorra  AG.LND.EL5M.ZS   \n",
       "2          AFG  Afghanistan  AG.LND.EL5M.ZS   \n",
       "3          AGO       Angola  AG.LND.EL5M.ZS   \n",
       "4          ALB      Albania  AG.LND.EL5M.ZS   \n",
       "\n",
       "                           series_name scale decimals  year      value  \n",
       "0  Land area below 5m (% of land area)     0        1  1990  29.574810  \n",
       "1  Land area below 5m (% of land area)     0        1  1990   0.000000  \n",
       "2  Land area below 5m (% of land area)     0        1  1990   0.000000  \n",
       "3  Land area below 5m (% of land area)     0        1  1990   0.208235  \n",
       "4  Land area below 5m (% of land area)     0        1  1990   4.967875  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Using all the arguments\n",
    "df_long = df.melt(\n",
    "    id_vars = [\"country_code\", \"country_name\", \"series_code\", \n",
    "               \"series_name\", \"scale\", \"decimals\"], \n",
    "    value_vars = [str(x) for x in list(np.arange(1990, 2011, 1))], \n",
    "    var_name='year',\n",
    "    value_name='value')\n",
    "\n",
    "df_long.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "caa815b2",
   "metadata": {},
   "source": [
    "# 6. Avoid bugs: Restart and Run the notebook often\n",
    "\n",
    "- This is generally a good practice not much people do! \n",
    "- The biggest drawback of notebooks is you can go up and run a cell, then down. It's a mess. \n",
    "- Restarting and running all the cell helps make sure you are "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fc8efdd7",
   "metadata": {},
   "source": [
    "# 7. Avoid looping over a dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "c682184e",
   "metadata": {
    "hide_input": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>carat</th>\n",
       "      <th>cut</th>\n",
       "      <th>color</th>\n",
       "      <th>clarity</th>\n",
       "      <th>depth</th>\n",
       "      <th>table</th>\n",
       "      <th>price</th>\n",
       "      <th>x</th>\n",
       "      <th>y</th>\n",
       "      <th>z</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.23</td>\n",
       "      <td>Ideal</td>\n",
       "      <td>E</td>\n",
       "      <td>SI2</td>\n",
       "      <td>61.5</td>\n",
       "      <td>55.0</td>\n",
       "      <td>326</td>\n",
       "      <td>3.95</td>\n",
       "      <td>3.98</td>\n",
       "      <td>2.43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.21</td>\n",
       "      <td>Premium</td>\n",
       "      <td>E</td>\n",
       "      <td>SI1</td>\n",
       "      <td>59.8</td>\n",
       "      <td>61.0</td>\n",
       "      <td>326</td>\n",
       "      <td>3.89</td>\n",
       "      <td>3.84</td>\n",
       "      <td>2.31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.23</td>\n",
       "      <td>Good</td>\n",
       "      <td>E</td>\n",
       "      <td>VS1</td>\n",
       "      <td>56.9</td>\n",
       "      <td>65.0</td>\n",
       "      <td>327</td>\n",
       "      <td>4.05</td>\n",
       "      <td>4.07</td>\n",
       "      <td>2.31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.29</td>\n",
       "      <td>Premium</td>\n",
       "      <td>I</td>\n",
       "      <td>VS2</td>\n",
       "      <td>62.4</td>\n",
       "      <td>58.0</td>\n",
       "      <td>334</td>\n",
       "      <td>4.20</td>\n",
       "      <td>4.23</td>\n",
       "      <td>2.63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.31</td>\n",
       "      <td>Good</td>\n",
       "      <td>J</td>\n",
       "      <td>SI2</td>\n",
       "      <td>63.3</td>\n",
       "      <td>58.0</td>\n",
       "      <td>335</td>\n",
       "      <td>4.34</td>\n",
       "      <td>4.35</td>\n",
       "      <td>2.75</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   carat      cut color clarity  depth  table  price     x     y     z\n",
       "0   0.23    Ideal     E     SI2   61.5   55.0    326  3.95  3.98  2.43\n",
       "1   0.21  Premium     E     SI1   59.8   61.0    326  3.89  3.84  2.31\n",
       "2   0.23     Good     E     VS1   56.9   65.0    327  4.05  4.07  2.31\n",
       "3   0.29  Premium     I     VS2   62.4   58.0    334  4.20  4.23  2.63\n",
       "4   0.31     Good     J     SI2   63.3   58.0    335  4.34  4.35  2.75"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "diamonds.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "db92521a",
   "metadata": {
    "hide_input": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(53940, 10)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "diamonds.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "db7ad489",
   "metadata": {
    "hide_input": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "49.3 ms ± 3.84 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "dims = []\n",
    "for row in diamonds.itertuples():\n",
    "    dims.append( row.x * row.y * row.z )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "152cfcc9",
   "metadata": {
    "hide_input": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "823 ms ± 31.6 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "diamonds.apply(lambda x: x.x * x.y * x.z, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "6ed65cf3",
   "metadata": {
    "hide_input": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "397 µs ± 11 µs per loop (mean ± std. dev. of 7 runs, 1,000 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "diamonds['dims'] = diamonds['x'] * diamonds['y'] * diamonds['z']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "38e24767",
   "metadata": {
    "hide_input": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2 ms ± 94.6 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "diamonds.assign(\n",
    "    dims=lambda df_: df_.x * df_.y * df_.z\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1d2180ba",
   "metadata": {},
   "source": [
    "# 8. Use np.where to flag outliers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "bb69000b",
   "metadata": {
    "hide_input": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAGdCAYAAAAMm0nCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAkT0lEQVR4nO3da3CU5d3H8V8SshtQkhAw2aSGGKVyUEAMGnZUiiTkIKUovBClijWFkQanGEXBUQzoNIpnbSrjeKCdAUU6HipYzAoCHgJISoaDmgEmNjqwoYIkQGRZkvt54WQfVwIksMnulfv7mcmU3b323uvPzbTf7ilRlmVZAgAAMEh0uDcAAADQUQQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOP0CPcGOktLS4v27t2r3r17KyoqKtzbAQAA7WBZlg4fPqy0tDRFR5/6eZZuGzB79+5Venp6uLcBAADOwrfffqsLL7zwlLd324Dp3bu3pJ/+AuLj40N2XL/fr4qKCuXl5Sk2NjZkx410dpzbjjNL9pzbjjNL9pybmSN/5sbGRqWnpwf+d/xUum3AtL5sFB8fH/KA6dWrl+Lj4434hxAqdpzbjjNL9pzbjjNL9pybmc2Z+Uxv/+BNvAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAME6PcG/AVJeXfihf8+l/1Xdbvnl8fCfsBgAAe+EZGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADG6VDAlJWV6aqrrlLv3r2VnJysG2+8UTU1NUFrjh07puLiYvXt21fnn3++Jk+erPr6+qA1dXV1Gj9+vHr16qXk5GTNmTNHJ06cCFqzbt06XXnllXI6nRowYICWLFlydhMCAIBup0MBs379ehUXF2vjxo3yeDzy+/3Ky8vT0aNHA2vuuecevf/++1qxYoXWr1+vvXv3atKkSYHbm5ubNX78eB0/flyff/65/v73v2vJkiWaP39+YE1tba3Gjx+v66+/XtXV1Zo9e7b++Mc/6sMPPwzByAAAwHQ9OrJ49erVQZeXLFmi5ORkVVVVafTo0WpoaNCrr76qZcuWaezYsZKk119/XYMHD9bGjRs1atQoVVRU6Msvv9RHH32klJQUXXHFFXr00Uf1wAMPqLS0VA6HQ4sXL1ZmZqaefvppSdLgwYP16aef6tlnn1V+fn6IRgcAAKbqUMD8UkNDgyQpKSlJklRVVSW/36/c3NzAmkGDBql///6qrKzUqFGjVFlZqaFDhyolJSWwJj8/XzNnztTOnTs1YsQIVVZWBh2jdc3s2bNPuRefzyefzxe43NjYKEny+/3y+/3nMmaQ1mM5o61zur9pWvdt6v7Phh1nluw5tx1nluw5NzNHvvbu86wDpqWlRbNnz9Y111yjyy+/XJLk9XrlcDiUmJgYtDYlJUVerzew5ufx0np7622nW9PY2Kgff/xRPXv2PGk/ZWVlWrBgwUnXV1RUqFevXmc35Gk8OrLlrO73wQcfhHgnXcvj8YR7C13OjjNL9pzbjjNL9pybmSNXU1NTu9addcAUFxdrx44d+vTTT8/2ECE1b948lZSUBC43NjYqPT1deXl5io+PD9nj+P1+eTwePbwlWr6WqA7ff0epmS+Btc49btw4xcbGhns7XcKOM0v2nNuOM0v2nJuZI3/m1ldQzuSsAmbWrFlauXKlNmzYoAsvvDBwvcvl0vHjx3Xo0KGgZ2Hq6+vlcrkCazZv3hx0vNZPKf18zS8/uVRfX6/4+Pg2n32RJKfTKafTedL1sbGxnXLCfC1R8jV3PGBM+MdzOp319xnJ7DizZM+57TizZM+5mTlytXePHfoUkmVZmjVrlt555x2tXbtWmZmZQbdnZWUpNjZWa9asCVxXU1Ojuro6ud1uSZLb7db27du1f//+wBqPx6P4+HgNGTIksObnx2hd03oMAABgbx16Bqa4uFjLli3Te++9p969ewfes5KQkKCePXsqISFBRUVFKikpUVJSkuLj43X33XfL7XZr1KhRkqS8vDwNGTJEt912mxYtWiSv16uHHnpIxcXFgWdQ7rrrLv31r3/V/fffrzvvvFNr167VW2+9pVWrVoV4fAAAYKIOPQPz0ksvqaGhQWPGjFFqamrgZ/ny5YE1zz77rH77299q8uTJGj16tFwul95+++3A7TExMVq5cqViYmLkdrv1+9//XrfffrsWLlwYWJOZmalVq1bJ4/Fo+PDhevrpp/XKK6/wEWoAACCpg8/AWNaZPzocFxen8vJylZeXn3JNRkbGGT+NM2bMGG3durUj2wMAADbB70ICAADGOacvskPHXTT37N/H883j40O4EwAAzMUzMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjNPhgNmwYYMmTJigtLQ0RUVF6d133w26/Y477lBUVFTQT0FBQdCagwcPaurUqYqPj1diYqKKiop05MiRoDXbtm3Tddddp7i4OKWnp2vRokUdnw4AAHRLHQ6Yo0ePavjw4SovLz/lmoKCAu3bty/w88YbbwTdPnXqVO3cuVMej0crV67Uhg0bNGPGjMDtjY2NysvLU0ZGhqqqqvTkk0+qtLRUL7/8cke3CwAAuqEeHb1DYWGhCgsLT7vG6XTK5XK1edtXX32l1atX64svvtDIkSMlSS+++KJuuOEGPfXUU0pLS9PSpUt1/Phxvfbaa3I4HLrssstUXV2tZ555Jih0AACAPXU4YNpj3bp1Sk5OVp8+fTR27Fg99thj6tu3rySpsrJSiYmJgXiRpNzcXEVHR2vTpk266aabVFlZqdGjR8vhcATW5Ofn64knntAPP/ygPn36nPSYPp9PPp8vcLmxsVGS5Pf75ff7QzZb67Gc0VbIjtnRxw6H1scO5x66mh1nluw5tx1nluw5NzNHvvbuM+QBU1BQoEmTJikzM1N79uzRgw8+qMLCQlVWViomJkZer1fJycnBm+jRQ0lJSfJ6vZIkr9erzMzMoDUpKSmB29oKmLKyMi1YsOCk6ysqKtSrV69QjRfw6MiWkB/zTD744IMuf8xf8ng84d5Cl7PjzJI957bjzJI952bmyNXU1NSudSEPmClTpgT+PHToUA0bNkyXXHKJ1q1bp5ycnFA/XMC8efNUUlISuNzY2Kj09HTl5eUpPj4+ZI/j9/vl8Xj08JZo+VqiQnbc9thRmt+lj/dzrXOPGzdOsbGxYdtHV7LjzJI957bjzJI952bmyJ+59RWUM+mUl5B+7uKLL1a/fv20e/du5eTkyOVyaf/+/UFrTpw4oYMHDwbeN+NyuVRfXx+0pvXyqd5b43Q65XQ6T7o+Nja2U06YryVKvuauDZhI+IfXWX+fkcyOM0v2nNuOM0v2nJuZI1d799jp3wPz3Xff6cCBA0pNTZUkud1uHTp0SFVVVYE1a9euVUtLi7KzswNrNmzYEPQ6mMfj0cCBA9t8+QgAANhLhwPmyJEjqq6uVnV1tSSptrZW1dXVqqur05EjRzRnzhxt3LhR33zzjdasWaOJEydqwIABys//6eWPwYMHq6CgQNOnT9fmzZv12WefadasWZoyZYrS0tIkSbfeeqscDoeKioq0c+dOLV++XM8//3zQS0QAAMC+OhwwW7Zs0YgRIzRixAhJUklJiUaMGKH58+crJiZG27Zt0+9+9ztdeumlKioqUlZWlj755JOgl3eWLl2qQYMGKScnRzfccIOuvfbaoO94SUhIUEVFhWpra5WVlaV7771X8+fP5yPUAABA0lm8B2bMmDGyrFN/hPjDDz884zGSkpK0bNmy064ZNmyYPvnkk45uDwAA2AC/CwkAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGKfDAbNhwwZNmDBBaWlpioqK0rvvvht0u2VZmj9/vlJTU9WzZ0/l5uZq165dQWsOHjyoqVOnKj4+XomJiSoqKtKRI0eC1mzbtk3XXXed4uLilJ6erkWLFnV8OgAA0C11OGCOHj2q4cOHq7y8vM3bFy1apBdeeEGLFy/Wpk2bdN555yk/P1/Hjh0LrJk6dap27twpj8ejlStXasOGDZoxY0bg9sbGRuXl5SkjI0NVVVV68sknVVpaqpdffvksRgQAAN1Nj47eobCwUIWFhW3eZlmWnnvuOT300EOaOHGiJOkf//iHUlJS9O6772rKlCn66quvtHr1an3xxRcaOXKkJOnFF1/UDTfcoKeeekppaWlaunSpjh8/rtdee00Oh0OXXXaZqqur9cwzzwSFDgAAsKcOB8zp1NbWyuv1Kjc3N3BdQkKCsrOzVVlZqSlTpqiyslKJiYmBeJGk3NxcRUdHa9OmTbrppptUWVmp0aNHy+FwBNbk5+friSee0A8//KA+ffqc9Ng+n08+ny9wubGxUZLk9/vl9/tDNmPrsZzRVsiO2dHHDofWxw7nHrqaHWeW7Dm3HWeW7Dk3M0e+9u4zpAHj9XolSSkpKUHXp6SkBG7zer1KTk4O3kSPHkpKSgpak5mZedIxWm9rK2DKysq0YMGCk66vqKhQr169znKiU3t0ZEvIj3kmH3zwQZc/5i95PJ5wb6HL2XFmyZ5z23FmyZ5zM3Pkampqate6kAZMOM2bN08lJSWBy42NjUpPT1deXp7i4+ND9jh+v18ej0cPb4mWryUqZMdtjx2l+V36eD/XOve4ceMUGxsbtn10JTvOLNlzbjvOLNlzbmaO/JlbX0E5k5AGjMvlkiTV19crNTU1cH19fb2uuOKKwJr9+/cH3e/EiRM6ePBg4P4ul0v19fVBa1ovt675JafTKafTedL1sbGxnXLCfC1R8jV3bcBEwj+8zvr7jGR2nFmy59x2nFmy59zMHLnau8eQfg9MZmamXC6X1qxZE7iusbFRmzZtktvtliS53W4dOnRIVVVVgTVr165VS0uLsrOzA2s2bNgQ9DqYx+PRwIED23z5CAAA2EuHA+bIkSOqrq5WdXW1pJ/euFtdXa26ujpFRUVp9uzZeuyxx/Svf/1L27dv1+233660tDTdeOONkqTBgweroKBA06dP1+bNm/XZZ59p1qxZmjJlitLS0iRJt956qxwOh4qKirRz504tX75czz//fNBLRAAAwL46/BLSli1bdP311wcut0bFtGnTtGTJEt1///06evSoZsyYoUOHDunaa6/V6tWrFRcXF7jP0qVLNWvWLOXk5Cg6OlqTJ0/WCy+8ELg9ISFBFRUVKi4uVlZWlvr166f58+fzEWoAACDpLAJmzJgxsqxTf4Q4KipKCxcu1MKFC0+5JikpScuWLTvt4wwbNkyffPJJR7cHAABsgN+FBAAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjNMj3BtA+100d9VZ3/ebx8eHcCcAAIQXz8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADBOyAOmtLRUUVFRQT+DBg0K3H7s2DEVFxerb9++Ov/88zV58mTV19cHHaOurk7jx49Xr169lJycrDlz5ujEiROh3ioAADBUj8446GWXXaaPPvro/x+kx/8/zD333KNVq1ZpxYoVSkhI0KxZszRp0iR99tlnkqTm5maNHz9eLpdLn3/+ufbt26fbb79dsbGx+stf/tIZ2wUAAIbplIDp0aOHXC7XSdc3NDTo1Vdf1bJlyzR27FhJ0uuvv67Bgwdr48aNGjVqlCoqKvTll1/qo48+UkpKiq644go9+uijeuCBB1RaWiqHw9EZWwYAAAbplIDZtWuX0tLSFBcXJ7fbrbKyMvXv319VVVXy+/3Kzc0NrB00aJD69++vyspKjRo1SpWVlRo6dKhSUlICa/Lz8zVz5kzt3LlTI0aMaPMxfT6ffD5f4HJjY6Mkye/3y+/3h2y21mM5o62QHbMrnOvfQev9Q/l3GensOLNkz7ntOLNkz7mZOfK1d58hD5js7GwtWbJEAwcO1L59+7RgwQJdd9112rFjh7xerxwOhxITE4Puk5KSIq/XK0nyer1B8dJ6e+ttp1JWVqYFCxacdH1FRYV69ep1jlOd7NGRLSE/Zmf64IMPQnIcj8cTkuOYxI4zS/ac244zS/acm5kjV1NTU7vWhTxgCgsLA38eNmyYsrOzlZGRobfeeks9e/YM9cMFzJs3TyUlJYHLjY2NSk9PV15enuLj40P2OH6/Xx6PRw9viZavJSpkx+1sO0rzz+n+rXOPGzdOsbGxIdpVZLPjzJI957bjzJI952bmyJ+59RWUM+mUl5B+LjExUZdeeql2796tcePG6fjx4zp06FDQszD19fWB98y4XC5t3rw56Bitn1Jq6301rZxOp5xO50nXx8bGdsoJ87VEyddsTsCE6u+gs/4+I5kdZ5bsObcdZ5bsOTczR6727rHTvwfmyJEj2rNnj1JTU5WVlaXY2FitWbMmcHtNTY3q6urkdrslSW63W9u3b9f+/fsDazwej+Lj4zVkyJDO3i4AADBAyJ+Bue+++zRhwgRlZGRo7969euSRRxQTE6NbbrlFCQkJKioqUklJiZKSkhQfH6+7775bbrdbo0aNkiTl5eVpyJAhuu2227Ro0SJ5vV499NBDKi4ubvMZFgAAYD8hD5jvvvtOt9xyiw4cOKALLrhA1157rTZu3KgLLrhAkvTss88qOjpakydPls/nU35+vv72t78F7h8TE6OVK1dq5syZcrvdOu+88zRt2jQtXLgw1FsFAACGCnnAvPnmm6e9PS4uTuXl5SovLz/lmoyMjJB9agYAAHQ//C4kAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMYhYAAAgHEIGAAAYBwCBgAAGCfkv0oAkemiuavO6f67Hs0L0U4AADh3PAMDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDgEDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADjEDAAAMA4BAwAADBOj3BvAGa4vPRDLbr6p//0NUd16L7fPD6+k3YFALArnoEBAADGIWAAAIBxCBgAAGAcAgYAABiHgAEAAMbhU0jodBfNXXXW9+UTTACAtvAMDAAAMA4BAwAAjEPAAAAA4xAwAADAOAQMAAAwDgEDAACMQ8AAAADj8D0wiGjn8h0y54LvnwGAyEbAAG24aO4qOWMsLbpaurz0Q/mao9p9X+IHADofLyEBAADjEDAAAMA4BAwAADAOAQMAAIxDwAAAAOMQMAAAwDh8jBoIsXP57ho+gg0A7cMzMAAAwDg8AwNEEBOfvQnXtyWfC57pAsxHwADdhInxAwBnK6JfQiovL9dFF12kuLg4ZWdna/PmzeHeEgAAiAAR+wzM8uXLVVJSosWLFys7O1vPPfec8vPzVVNTo+Tk5HBvD+hWfv7szdn+DigA6EoRGzDPPPOMpk+frj/84Q+SpMWLF2vVqlV67bXXNHfu3DDvDoDJfvlyW0eijZfbgMgQkQFz/PhxVVVVad68eYHroqOjlZubq8rKyjbv4/P55PP5ApcbGhokSQcPHpTf7w/Z3vx+v5qamtTDH63mFvv8v9MeLZaamlpsNbcdZ5bsOXdHZh5w31tdtKtgm+blhPyYrf99duDAAcXGxob8+JGImSN/5sOHD0uSLMs67bqIDJjvv/9ezc3NSklJCbo+JSVFX3/9dZv3KSsr04IFC066PjMzs1P2aEe3hnsDYWDHmSV7zh3pM/d7Otw7ALrW4cOHlZCQcMrbIzJgzsa8efNUUlISuNzS0qKDBw+qb9++iooK3f+LbGxsVHp6ur799lvFx8eH7LiRzo5z23FmyZ5z23FmyZ5zM3Pkz2xZlg4fPqy0tLTTrovIgOnXr59iYmJUX18fdH19fb1cLleb93E6nXI6nUHXJSYmdtYWFR8fb8Q/hFCz49x2nFmy59x2nFmy59zMHNlO98xLq4j8GLXD4VBWVpbWrFkTuK6lpUVr1qyR2+0O484AAEAkiMhnYCSppKRE06ZN08iRI3X11Vfrueee09GjRwOfSgIAAPYVsQFz880363//+5/mz58vr9erK664QqtXrz7pjb1dzel06pFHHjnp5aruzo5z23FmyZ5z23FmyZ5zM3P3EWWd6XNKAAAAESYi3wMDAABwOgQMAAAwDgEDAACMQ8AAAADjEDAdVF5erosuukhxcXHKzs7W5s2bw72lTlVaWqqoqKign0GDBoV7WyG1YcMGTZgwQWlpaYqKitK7774bdLtlWZo/f75SU1PVs2dP5ebmateuXeHZbIicaeY77rjjpPNeUFAQns2GSFlZma666ir17t1bycnJuvHGG1VTUxO05tixYyouLlbfvn11/vnna/LkySd9oaZp2jP3mDFjTjrfd911V5h2fO5eeuklDRs2LPDFbW63W//+978Dt3fH8yydee7udp4JmA5Yvny5SkpK9Mgjj+g///mPhg8frvz8fO3fvz/cW+tUl112mfbt2xf4+fTTT8O9pZA6evSohg8frvLy8jZvX7RokV544QUtXrxYmzZt0nnnnaf8/HwdO3asi3caOmeaWZIKCgqCzvsbb7zRhTsMvfXr16u4uFgbN26Ux+OR3+9XXl6ejh49Glhzzz336P3339eKFSu0fv167d27V5MmTQrjrs9de+aWpOnTpwed70WLFoVpx+fuwgsv1OOPP66qqipt2bJFY8eO1cSJE7Vz505J3fM8S2eeW+pe51kW2u3qq6+2iouLA5ebm5uttLQ0q6ysLIy76lyPPPKINXz48HBvo8tIst55553A5ZaWFsvlcllPPvlk4LpDhw5ZTqfTeuONN8Kww9D75cyWZVnTpk2zJk6cGJb9dJX9+/dbkqz169dblvXTeY2NjbVWrFgRWPPVV19ZkqzKyspwbTPkfjm3ZVnWb37zG+vPf/5z+DbVBfr06WO98sortjnPrVrntqzud555Bqadjh8/rqqqKuXm5gaui46OVm5uriorK8O4s863a9cupaWl6eKLL9bUqVNVV1cX7i11mdraWnm93qDznpCQoOzs7G5/3tetW6fk5GQNHDhQM2fO1IEDB8K9pZBqaGiQJCUlJUmSqqqq5Pf7g871oEGD1L9//251rn85d6ulS5eqX79+uvzyyzVv3jw1NTWFY3sh19zcrDfffFNHjx6V2+22zXn+5dytutN5jthv4o0033//vZqbm0/6JuCUlBR9/fXXYdpV58vOztaSJUs0cOBA7du3TwsWLNB1112nHTt2qHfv3uHeXqfzer2S1OZ5b72tOyooKNCkSZOUmZmpPXv26MEHH1RhYaEqKysVExMT7u2ds5aWFs2ePVvXXHONLr/8ckk/nWuHw3HSL4HtTue6rbkl6dZbb1VGRobS0tK0bds2PfDAA6qpqdHbb78dxt2em+3bt8vtduvYsWM6//zz9c4772jIkCGqrq7u1uf5VHNL3e88EzA4rcLCwsCfhw0bpuzsbGVkZOitt95SUVFRGHeGzjRlypTAn4cOHaphw4bpkksu0bp165STkxPGnYVGcXGxduzY0e3ez3Ump5p7xowZgT8PHTpUqampysnJ0Z49e3TJJZd09TZDYuDAgaqurlZDQ4P++c9/atq0aVq/fn24t9XpTjX3kCFDut155iWkdurXr59iYmJOeqd6fX29XC5XmHbV9RITE3XppZdq9+7d4d5Kl2g9t3Y/7xdffLH69evXLc77rFmztHLlSn388ce68MILA9e7XC4dP35chw4dClrfXc71qeZuS3Z2tiQZfb4dDocGDBigrKwslZWVafjw4Xr++ee7/Xk+1dxtMf08EzDt5HA4lJWVpTVr1gSua2lp0Zo1a4JeX+zujhw5oj179ig1NTXcW+kSmZmZcrlcQee9sbFRmzZtstV5/+6773TgwAGjz7tlWZo1a5beeecdrV27VpmZmUG3Z2VlKTY2Nuhc19TUqK6uzuhzfaa521JdXS1JRp/vX2ppaZHP5+u25/lUWudui/HnOdzvIjbJm2++aTmdTmvJkiXWl19+ac2YMcNKTEy0vF5vuLfWae69915r3bp1Vm1trfXZZ59Zubm5Vr9+/az9+/eHe2shc/jwYWvr1q3W1q1bLUnWM888Y23dutX673//a1mWZT3++ONWYmKi9d5771nbtm2zJk6caGVmZlo//vhjmHd+9k438+HDh6377rvPqqystGpra62PPvrIuvLKK61f//rX1rFjx8K99bM2c+ZMKyEhwVq3bp21b9++wE9TU1NgzV133WX179/fWrt2rbVlyxbL7XZbbrc7jLs+d2eae/fu3dbChQutLVu2WLW1tdZ7771nXXzxxdbo0aPDvPOzN3fuXGv9+vVWbW2ttW3bNmvu3LlWVFSUVVFRYVlW9zzPlnX6ubvjeSZgOujFF1+0+vfvbzkcDuvqq6+2Nm7cGO4tdaqbb77ZSk1NtRwOh/WrX/3Kuvnmm63du3eHe1sh9fHHH1uSTvqZNm2aZVk/fZT64YcftlJSUiyn02nl5ORYNTU14d30OTrdzE1NTVZeXp51wQUXWLGxsVZGRoY1ffp040O9rXklWa+//npgzY8//mj96U9/svr06WP16tXLuummm6x9+/aFb9MhcKa56+rqrNGjR1tJSUmW0+m0BgwYYM2ZM8dqaGgI78bPwZ133mllZGRYDofDuuCCC6ycnJxAvFhW9zzPlnX6ubvjeY6yLMvquud7AAAAzh3vgQEAAMYhYAAAgHEIGAAAYBwCBgAAGIeAAQAAxiFgAACAcQgYAABgHAIGAAAYh4ABAADGIWAAAIBxCBgAAGAcAgYAABjn/wBGWYbpIHF0EQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dist = taxis.distance.copy()\n",
    "dist.hist(bins=30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "937d54bc",
   "metadata": {
    "hide_input": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1.64, 2.056104461371055)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# median\n",
    "dist_median = dist.median()\n",
    "\n",
    "# Median Absolute Deviation\n",
    "dist_mad = (dist - dist.median()).abs().mean()\n",
    "\n",
    "dist_median, dist_mad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "c57ad975",
   "metadata": {
    "hide_input": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7.808313384113165"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# If a distance is larger than: \n",
    "# Median + 3 * MAD then it's an outlier\n",
    "sup_limit = dist_median + 3 * dist_mad\n",
    "sup_limit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "8e64289e",
   "metadata": {
    "hide_input": true
   },
   "outputs": [],
   "source": [
    "taxis['fl_outlier'] = np.where(taxis.distance >= sup_limit, \"outlier\", \"ok\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "033f27ef",
   "metadata": {
    "hide_input": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ok         5841\n",
       "outlier     592\n",
       "Name: fl_outlier, dtype: int64"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "taxis['fl_outlier'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f80c90c8",
   "metadata": {},
   "source": [
    "# 9. Use pd.cut to discretize variables\n",
    "\n",
    "`pd.cut`: Bin values into discrete intervals.\n",
    "\n",
    "- Use cut when you need to segment and sort data values into bins. \n",
    "- This function is also useful for going from a continuous variable to a categorical variable. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "6eb56c96",
   "metadata": {
    "hide_input": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pickup</th>\n",
       "      <th>dropoff</th>\n",
       "      <th>passengers</th>\n",
       "      <th>distance</th>\n",
       "      <th>fare</th>\n",
       "      <th>tip</th>\n",
       "      <th>tolls</th>\n",
       "      <th>total</th>\n",
       "      <th>color</th>\n",
       "      <th>payment</th>\n",
       "      <th>pickup_zone</th>\n",
       "      <th>dropoff_zone</th>\n",
       "      <th>pickup_borough</th>\n",
       "      <th>dropoff_borough</th>\n",
       "      <th>fl_outlier</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2019-03-23 20:21:09</td>\n",
       "      <td>2019-03-23 20:27:24</td>\n",
       "      <td>1</td>\n",
       "      <td>1.60</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2.15</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12.95</td>\n",
       "      <td>yellow</td>\n",
       "      <td>credit card</td>\n",
       "      <td>Lenox Hill West</td>\n",
       "      <td>UN/Turtle Bay South</td>\n",
       "      <td>Manhattan</td>\n",
       "      <td>Manhattan</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2019-03-04 16:11:55</td>\n",
       "      <td>2019-03-04 16:19:00</td>\n",
       "      <td>1</td>\n",
       "      <td>0.79</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.30</td>\n",
       "      <td>yellow</td>\n",
       "      <td>cash</td>\n",
       "      <td>Upper West Side South</td>\n",
       "      <td>Upper West Side South</td>\n",
       "      <td>Manhattan</td>\n",
       "      <td>Manhattan</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2019-03-27 17:53:01</td>\n",
       "      <td>2019-03-27 18:00:25</td>\n",
       "      <td>1</td>\n",
       "      <td>1.37</td>\n",
       "      <td>7.5</td>\n",
       "      <td>2.36</td>\n",
       "      <td>0.0</td>\n",
       "      <td>14.16</td>\n",
       "      <td>yellow</td>\n",
       "      <td>credit card</td>\n",
       "      <td>Alphabet City</td>\n",
       "      <td>West Village</td>\n",
       "      <td>Manhattan</td>\n",
       "      <td>Manhattan</td>\n",
       "      <td>ok</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               pickup             dropoff  passengers  distance  fare   tip  \\\n",
       "0 2019-03-23 20:21:09 2019-03-23 20:27:24           1      1.60   7.0  2.15   \n",
       "1 2019-03-04 16:11:55 2019-03-04 16:19:00           1      0.79   5.0  0.00   \n",
       "2 2019-03-27 17:53:01 2019-03-27 18:00:25           1      1.37   7.5  2.36   \n",
       "\n",
       "   tolls  total   color      payment            pickup_zone  \\\n",
       "0    0.0  12.95  yellow  credit card        Lenox Hill West   \n",
       "1    0.0   9.30  yellow         cash  Upper West Side South   \n",
       "2    0.0  14.16  yellow  credit card          Alphabet City   \n",
       "\n",
       "            dropoff_zone pickup_borough dropoff_borough fl_outlier  \n",
       "0    UN/Turtle Bay South      Manhattan       Manhattan         ok  \n",
       "1  Upper West Side South      Manhattan       Manhattan         ok  \n",
       "2           West Village      Manhattan       Manhattan         ok  "
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "taxis.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "19e985fe",
   "metadata": {
    "hide_input": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAGdCAYAAAAMm0nCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAp/klEQVR4nO3de3SU5YHH8V+uA0EmIdBkkhJiqq3cL4JAFktRQgKmqIWzuyACa6kc2NAa4iLSKgZoG4lFVEph3VbpHqGi54iVgJAhCIiGi4GUm6ZoUVSYZAuSAMFkSN79oydvHa65DIRn5vs5J0fmfZ/3meeX4fLznfedhFiWZQkAAMAgoa29AAAAgKaiwAAAAONQYAAAgHEoMAAAwDgUGAAAYBwKDAAAMA4FBgAAGIcCAwAAjBPe2gu4Vurr63Xs2DG1b99eISEhrb0cAADQCJZl6fTp00pMTFRo6OXPswRsgTl27JiSkpJaexkAAKAZPv/8c3Xu3Pmy+wO2wLRv317SP74BTqezRXN5vV4VFhYqPT1dERER/ljeDS/YMgdbXin4MgdbXonMwZA5EPNWVVUpKSnJ/nf8cgK2wDS8beR0Ov1SYKKiouR0OgPmN8jVBFvmYMsrBV/mYMsrkTkYMgdy3qtd/sFFvAAAwDgUGAAAYBwKDAAAMA4FBgAAGIcCAwAAjEOBAQAAxqHAAAAA41BgAACAcSgwAADAOBQYAABgHAoMAAAwDgUGAAAYhwIDAACMQ4EBAADGCW/tBQSbmx9f1+xjP306048rAQDAXJyBAQAAxqHAAAAA41BgAACAcSgwAADAOBQYAABgHAoMAAAwDgUGAAAYhwIDAACMQ4EBAADGocAAAADjUGAAAIBxKDAAAMA4FBgAAGCcJhWYvLw83XHHHWrfvr3i4uJ0//33q6yszGfMsGHDFBIS4vM1bdo0nzFHjx5VZmamoqKiFBcXp1mzZun8+fM+Y7Zs2aLbb79dDodDt956q1asWNG8hAAAIOA0qcBs3bpVWVlZ2rFjh9xut7xer9LT03X27FmfcQ8//LCOHz9uf+Xn59v76urqlJmZqdraWr3//vv64x//qBUrVmju3Ln2mCNHjigzM1N33XWXSktLlZ2drZ/85CfauHFjC+MCAIBAEN6UwRs2bPB5vGLFCsXFxamkpERDhw61t0dFRcnlcl1yjsLCQh06dEibNm1SfHy8+vbtqwULFmj27NnKzc1VZGSkli9frpSUFC1atEiS1K1bN23fvl2LFy9WRkZGUzMCAIAA06QCc6HKykpJUmxsrM/2lStX6pVXXpHL5dLo0aP15JNPKioqSpJUXFysXr16KT4+3h6fkZGh6dOn6+DBg+rXr5+Ki4uVlpbmM2dGRoays7Mvu5aamhrV1NTYj6uqqiRJXq9XXq+3JTHt41s6jyQ5wqwWr+N68GdmEwRbXin4MgdbXonMwSAQ8zY2S7MLTH19vbKzszVkyBD17NnT3v7AAw8oOTlZiYmJ2rdvn2bPnq2ysjK98cYbkiSPx+NTXiTZjz0ezxXHVFVV6dy5c2rbtu1F68nLy9O8efMu2l5YWGiXp5Zyu90tniN/YPOPXb9+fYufv6n8kdkkwZZXCr7MwZZXInMwCKS81dXVjRrX7AKTlZWlAwcOaPv27T7bp06dav+6V69eSkhI0PDhw/XJJ5/olltuae7TXdWcOXOUk5NjP66qqlJSUpLS09PldDpbNLfX65Xb7daIESMUERHRorl65jb/Op4Dudfv7TN/ZjZBsOWVgi9zsOWVyBwMmQMxb8M7KFfTrAIzY8YMFRQUaNu2bercufMVxw4aNEiS9PHHH+uWW26Ry+XSrl27fMaUl5dLkn3djMvlsrd9c4zT6bzk2RdJcjgccjgcF22PiIjw24vqj7lq6kJa9PzXmz+/fyYItrxS8GUOtrwSmYNBIOVtbI4m3YVkWZZmzJihNWvWaPPmzUpJSbnqMaWlpZKkhIQESVJqaqr279+viooKe4zb7ZbT6VT37t3tMUVFRT7zuN1upaamNmW5AAAgQDWpwGRlZemVV17RqlWr1L59e3k8Hnk8Hp07d06S9Mknn2jBggUqKSnRp59+qrfeekuTJk3S0KFD1bt3b0lSenq6unfvrokTJ+ovf/mLNm7cqCeeeEJZWVn2GZRp06bpb3/7mx577DF99NFH+t3vfqfXXntNM2fO9HN8AABgoiYVmGXLlqmyslLDhg1TQkKC/bV69WpJUmRkpDZt2qT09HR17dpVjz76qMaOHau1a9fac4SFhamgoEBhYWFKTU3Vgw8+qEmTJmn+/Pn2mJSUFK1bt05ut1t9+vTRokWL9Pvf/55bqAEAgKQmXgNjWVe+BTgpKUlbt2696jzJyclXvaNm2LBh2rt3b1OWBwAAggQ/CwkAABiHAgMAAIxDgQEAAMahwAAAAONQYAAAgHEoMAAAwDgUGAAAYBwKDAAAMA4FBgAAGIcCAwAAjEOBAQAAxqHAAAAA41BgAACAcSgwAADAOBQYAABgHAoMAAAwDgUGAAAYhwIDAACMQ4EBAADGocAAAADjUGAAAIBxKDAAAMA4FBgAAGAcCgwAADAOBQYAABiHAgMAAIxDgQEAAMahwAAAAONQYAAAgHEoMAAAwDgUGAAAYBwKDAAAMA4FBgAAGIcCAwAAjEOBAQAAxqHAAAAA41BgAACAcSgwAADAOBQYAABgHAoMAAAwDgUGAAAYhwIDAACMQ4EBAADGocAAAADjUGAAAIBxKDAAAMA4FBgAAGAcCgwAADAOBQYAABiHAgMAAIxDgQEAAMahwAAAAONQYAAAgHEoMAAAwDgUGAAAYBwKDAAAME6TCkxeXp7uuOMOtW/fXnFxcbr//vtVVlbmM+brr79WVlaWOnbsqJtuukljx45VeXm5z5ijR48qMzNTUVFRiouL06xZs3T+/HmfMVu2bNHtt98uh8OhW2+9VStWrGheQgAAEHCaVGC2bt2qrKws7dixQ263W16vV+np6Tp79qw9ZubMmVq7dq1ef/11bd26VceOHdOYMWPs/XV1dcrMzFRtba3ef/99/fGPf9SKFSs0d+5ce8yRI0eUmZmpu+66S6WlpcrOztZPfvITbdy40Q+RAQCA6cKbMnjDhg0+j1esWKG4uDiVlJRo6NChqqys1B/+8AetWrVKd999tyTp5ZdfVrdu3bRjxw4NHjxYhYWFOnTokDZt2qT4+Hj17dtXCxYs0OzZs5Wbm6vIyEgtX75cKSkpWrRokSSpW7du2r59uxYvXqyMjAw/RQcAAKZqUoG5UGVlpSQpNjZWklRSUiKv16u0tDR7TNeuXdWlSxcVFxdr8ODBKi4uVq9evRQfH2+PycjI0PTp03Xw4EH169dPxcXFPnM0jMnOzr7sWmpqalRTU2M/rqqqkiR5vV55vd6WxLSPb+k8kuQIs1q8juvBn5lNEGx5peDLHGx5JTIHg0DM29gszS4w9fX1ys7O1pAhQ9SzZ09JksfjUWRkpGJiYnzGxsfHy+Px2GO+WV4a9jfsu9KYqqoqnTt3Tm3btr1oPXl5eZo3b95F2wsLCxUVFdW8kBdwu90tniN/YPOPXb9+fYufv6n8kdkkwZZXCr7MwZZXInMwCKS81dXVjRrX7AKTlZWlAwcOaPv27c2dwq/mzJmjnJwc+3FVVZWSkpKUnp4up9PZorm9Xq/cbrdGjBihiIiIFs3VM7f51/EcyL1+b5/5M7MJgi2vFHyZgy2vROZgyByIeRveQbmaZhWYGTNmqKCgQNu2bVPnzp3t7S6XS7W1tTp16pTPWZjy8nK5XC57zK5du3zma7hL6ZtjLrxzqby8XE6n85JnXyTJ4XDI4XBctD0iIsJvL6o/5qqpC2nR819v/vz+mSDY8krBlznY8kpkDgaBlLexOZp0F5JlWZoxY4bWrFmjzZs3KyUlxWd///79FRERoaKiIntbWVmZjh49qtTUVElSamqq9u/fr4qKCnuM2+2W0+lU9+7d7THfnKNhTMMcAAAguDXpDExWVpZWrVqlP//5z2rfvr19zUp0dLTatm2r6OhoTZkyRTk5OYqNjZXT6dRPf/pTpaamavDgwZKk9PR0de/eXRMnTlR+fr48Ho+eeOIJZWVl2WdQpk2bpt/+9rd67LHH9OMf/1ibN2/Wa6+9pnXr1vk5PgAAMFGTzsAsW7ZMlZWVGjZsmBISEuyv1atX22MWL16sH/7whxo7dqyGDh0ql8ulN954w94fFhamgoIChYWFKTU1VQ8++KAmTZqk+fPn22NSUlK0bt06ud1u9enTR4sWLdLvf/97bqEGAACSmngGxrKufgtwmzZttHTpUi1duvSyY5KTk696R82wYcO0d+/epiwPAAAECX4WEgAAMA4FBgAAGIcCAwAAjEOBAQAAxqHAAAAA41BgAACAcSgwAADAOBQYAABgHAoMAAAwDgUGAAAYhwIDAACMQ4EBAADGocAAAADjUGAAAIBxKDAAAMA4FBgAAGAcCgwAADAOBQYAABiHAgMAAIxDgQEAAMahwAAAAONQYAAAgHEoMAAAwDgUGAAAYBwKDAAAMA4FBgAAGIcCAwAAjEOBAQAAxqHAAAAA41BgAACAcSgwAADAOBQYAABgHAoMAAAwDgUGAAAYhwIDAACMQ4EBAADGocAAAADjUGAAAIBxKDAAAMA4FBgAAGAcCgwAADAOBQYAABiHAgMAAIxDgQEAAMahwAAAAONQYAAAgHEoMAAAwDgUGAAAYBwKDAAAMA4FBgAAGIcCAwAAjEOBAQAAxqHAAAAA41BgAACAcSgwAADAOBQYAABgnCYXmG3btmn06NFKTExUSEiI3nzzTZ/9//Ef/6GQkBCfr5EjR/qMOXnypCZMmCCn06mYmBhNmTJFZ86c8Rmzb98+ff/731ebNm2UlJSk/Pz8pqcDAAABqckF5uzZs+rTp4+WLl162TEjR47U8ePH7a8//elPPvsnTJiggwcPyu12q6CgQNu2bdPUqVPt/VVVVUpPT1dycrJKSkr0zDPPKDc3Vy+++GJTlwsAAAJQeFMPGDVqlEaNGnXFMQ6HQy6X65L7PvzwQ23YsEG7d+/WgAEDJElLlizRPffco9/85jdKTEzUypUrVVtbq5deekmRkZHq0aOHSktL9eyzz/oUHQAAEJyaXGAaY8uWLYqLi1OHDh10991365e//KU6duwoSSouLlZMTIxdXiQpLS1NoaGh2rlzp370ox+puLhYQ4cOVWRkpD0mIyNDCxcu1FdffaUOHTpc9Jw1NTWqqamxH1dVVUmSvF6vvF5vi/I0HN/SeSTJEWa1eB3Xgz8zmyDY8krBlznY8kpkDgaBmLexWfxeYEaOHKkxY8YoJSVFn3zyiX7+859r1KhRKi4uVlhYmDwej+Li4nwXER6u2NhYeTweSZLH41FKSorPmPj4eHvfpQpMXl6e5s2bd9H2wsJCRUVF+SWb2+1u8Rz5A5t/7Pr161v8/E3lj8wmCba8UvBlDra8EpmDQSDlra6ubtQ4vxeYcePG2b/u1auXevfurVtuuUVbtmzR8OHD/f10tjlz5ignJ8d+XFVVpaSkJKWnp8vpdLZobq/XK7fbrREjRigiIqJFc/XM3djsYw/kZrTouZvCn5lNEGx5peDLHGx5JTIHQ+ZAzNvwDsrVXJO3kL7pO9/5jjp16qSPP/5Yw4cPl8vlUkVFhc+Y8+fP6+TJk/Z1My6XS+Xl5T5jGh5f7toah8Mhh8Nx0faIiAi/vaj+mKumLqRFz3+9+fP7Z4JgyysFX+ZgyyuRORgEUt7G5rjmnwPzxRdf6MSJE0pISJAkpaam6tSpUyopKbHHbN68WfX19Ro0aJA9Ztu2bT7vg7ndbt12222XfPsIAAAElyYXmDNnzqi0tFSlpaWSpCNHjqi0tFRHjx7VmTNnNGvWLO3YsUOffvqpioqKdN999+nWW29VRsY/3v7o1q2bRo4cqYcffli7du3Se++9pxkzZmjcuHFKTEyUJD3wwAOKjIzUlClTdPDgQa1evVrPP/+8z1tEAAAgeDW5wHzwwQfq16+f+vXrJ0nKyclRv379NHfuXIWFhWnfvn2699579b3vfU9TpkxR//799e677/q8vbNy5Up17dpVw4cP1z333KM777zT5zNeoqOjVVhYqCNHjqh///569NFHNXfuXG6hBgAAkppxDcywYcNkWZe/FXjjxqtfpBobG6tVq1ZdcUzv3r317rvvNnV5AAAgCPCzkAAAgHEoMAAAwDgUGAAAYBwKDAAAMA4FBgAAGIcCAwAAjEOBAQAAxqHAAAAA41BgAACAcSgwAADAOBQYAABgHAoMAAAwDgUGAAAYhwIDAACMQ4EBAADGocAAAADjUGAAAIBxKDAAAMA4FBgAAGAcCgwAADAOBQYAABiHAgMAAIxDgQEAAMahwAAAAONQYAAAgHEoMAAAwDgUGAAAYBwKDAAAMA4FBgAAGIcCAwAAjEOBAQAAxqHAAAAA41BgAACAcSgwAADAOBQYAABgHAoMAAAwDgUGAAAYJ7y1F4DGu/nxdc0+9tOnM/24EgAAWhcFphlaUiQAAEDL8RYSAAAwDgUGAAAYhwIDAACMQ4EBAADGocAAAADjUGAAAIBxKDAAAMA4FBgAAGAcCgwAADAOBQYAABiHAgMAAIxDgQEAAMahwAAAAONQYAAAgHEoMAAAwDgUGAAAYJwmF5ht27Zp9OjRSkxMVEhIiN58802f/ZZlae7cuUpISFDbtm2Vlpamw4cP+4w5efKkJkyYIKfTqZiYGE2ZMkVnzpzxGbNv3z59//vfV5s2bZSUlKT8/PympwMAAAGpyQXm7Nmz6tOnj5YuXXrJ/fn5+XrhhRe0fPly7dy5U+3atVNGRoa+/vpre8yECRN08OBBud1uFRQUaNu2bZo6daq9v6qqSunp6UpOTlZJSYmeeeYZ5ebm6sUXX2xGRAAAEGjCm3rAqFGjNGrUqEvusyxLzz33nJ544gndd999kqT//d//VXx8vN58802NGzdOH374oTZs2KDdu3drwIABkqQlS5bonnvu0W9+8xslJiZq5cqVqq2t1UsvvaTIyEj16NFDpaWlevbZZ32KDgAACE5+vQbmyJEj8ng8SktLs7dFR0dr0KBBKi4uliQVFxcrJibGLi+SlJaWptDQUO3cudMeM3ToUEVGRtpjMjIyVFZWpq+++sqfSwYAAAZq8hmYK/F4PJKk+Ph4n+3x8fH2Po/Ho7i4ON9FhIcrNjbWZ0xKSspFczTs69Chw0XPXVNTo5qaGvtxVVWVJMnr9crr9bYkln18w38dYVaL5msNTf0eXJg50AVbXin4MgdbXonMwSAQ8zY2i18LTGvKy8vTvHnzLtpeWFioqKgovzyH2+2WJOUP9Mt019X69eubdVxD5mARbHml4MscbHklMgeDQMpbXV3dqHF+LTAul0uSVF5eroSEBHt7eXm5+vbta4+pqKjwOe78+fM6efKkfbzL5VJ5ebnPmIbHDWMuNGfOHOXk5NiPq6qqlJSUpPT0dDmdzhbl8nq9crvdGjFihCIiItQzd2OL5msNB3IzmjT+wsyBLtjySsGXOdjySmQOhsyBmLfhHZSr8WuBSUlJkcvlUlFRkV1YqqqqtHPnTk2fPl2SlJqaqlOnTqmkpET9+/eXJG3evFn19fUaNGiQPeYXv/iFvF6v/YK43W7ddtttl3z7SJIcDoccDsdF2yMiIvz2ojbMVVMX4pf5rqfmfg/8+f0zQbDllYIvc7DllcgcDAIpb2NzNPki3jNnzqi0tFSlpaWS/nHhbmlpqY4ePaqQkBBlZ2frl7/8pd566y3t379fkyZNUmJiou6//35JUrdu3TRy5Eg9/PDD2rVrl9577z3NmDFD48aNU2JioiTpgQceUGRkpKZMmaKDBw9q9erVev75533OsAAAgODV5DMwH3zwge666y77cUOpmDx5slasWKHHHntMZ8+e1dSpU3Xq1Cndeeed2rBhg9q0aWMfs3LlSs2YMUPDhw9XaGioxo4dqxdeeMHeHx0drcLCQmVlZal///7q1KmT5s6dyy3UAABAUjMKzLBhw2RZl78LJyQkRPPnz9f8+fMvOyY2NlarVq264vP07t1b7777blOXBwAAggA/CwkAABiHAgMAAIxDgQEAAMahwAAAAONQYAAAgHEoMAAAwDgUGAAAYBwKDAAAMA4FBgAAGIcCAwAAjEOBAQAAxqHAAAAA41BgAACAcSgwAADAOBQYAABgHAoMAAAwDgUGAAAYhwIDAACMQ4EBAADGocAAAADjUGAAAIBxKDAAAMA4FBgAAGAcCgwAADAOBQYAABiHAgMAAIxDgQEAAMahwAAAAONQYAAAgHEoMAAAwDgUGAAAYBwKDAAAMA4FBgAAGIcCAwAAjEOBAQAAxqHAAAAA41BgAACAcSgwAADAOBQYAABgHAoMAAAwDgUGAAAYhwIDAACMQ4EBAADGocAAAADjUGAAAIBxKDAAAMA4FBgAAGAcCgwAADAOBQYAABiHAgMAAIxDgQEAAMahwAAAAONQYAAAgHEoMAAAwDgUGAAAYJzw1l4Aro+bH1/XpPGOMEv5A6WeuRtVUxeiT5/OvEYrAwCg6fx+BiY3N1chISE+X127drX3f/3118rKylLHjh110003aezYsSovL/eZ4+jRo8rMzFRUVJTi4uI0a9YsnT9/3t9LBQAAhromZ2B69OihTZs2/fNJwv/5NDNnztS6dev0+uuvKzo6WjNmzNCYMWP03nvvSZLq6uqUmZkpl8ul999/X8ePH9ekSZMUERGhX//619diuQAAwDDXpMCEh4fL5XJdtL2yslJ/+MMftGrVKt19992SpJdfflndunXTjh07NHjwYBUWFurQoUPatGmT4uPj1bdvXy1YsECzZ89Wbm6uIiMjr8WSAQCAQa5JgTl8+LASExPVpk0bpaamKi8vT126dFFJSYm8Xq/S0tLssV27dlWXLl1UXFyswYMHq7i4WL169VJ8fLw9JiMjQ9OnT9fBgwfVr1+/Sz5nTU2Nampq7MdVVVWSJK/XK6/X26I8Dcc3/NcRZrVoPhM4Qi2f/7b0e3iju/A1DgbBljnY8kpkDgaBmLexWUIsy/Lrv8Zvv/22zpw5o9tuu03Hjx/XvHnz9OWXX+rAgQNau3atHnroIZ+iIUkDBw7UXXfdpYULF2rq1Kn67LPPtHHjRnt/dXW12rVrp/Xr12vUqFGXfN7c3FzNmzfvou2rVq1SVFSUPyMCAIBrpLq6Wg888IAqKyvldDovO87vZ2C+WTB69+6tQYMGKTk5Wa+99pratm3r76ezzZkzRzk5OfbjqqoqJSUlKT09/YrfgMbwer1yu90aMWKEIiIi1DN349UPMpwj1NKCAfV68oNQ1dSH6EBuRmsv6Zq68DUOBsGWOdjySmQOhsyBmLfhHZSruea3UcfExOh73/uePv74Y40YMUK1tbU6deqUYmJi7DHl5eX2NTMul0u7du3ymaPhLqVLXVfTwOFwyOFwXLQ9IiLCby9qw1w1dSF+mc8ENfUhqqkLCZg/GFfjz98vpgi2zMGWVyJzMAikvI3Ncc0/yO7MmTP65JNPlJCQoP79+ysiIkJFRUX2/rKyMh09elSpqamSpNTUVO3fv18VFRX2GLfbLafTqe7du1/r5QIAAAP4/QzMf/3Xf2n06NFKTk7WsWPH9NRTTyksLEzjx49XdHS0pkyZopycHMXGxsrpdOqnP/2pUlNTNXjwYElSenq6unfvrokTJyo/P18ej0dPPPGEsrKyLnmGBQAABB+/F5gvvvhC48eP14kTJ/Stb31Ld955p3bs2KFvfetbkqTFixcrNDRUY8eOVU1NjTIyMvS73/3OPj4sLEwFBQWaPn26UlNT1a5dO02ePFnz58/391IBAICh/F5gXn311Svub9OmjZYuXaqlS5dedkxycrLWr1/v76UBAIAAwQ9zBAAAxqHAAAAA41BgAACAcSgwAADAOBQYAABgHAoMAAAwDgUGAAAYhwIDAACMQ4EBAADGocAAAADjUGAAAIBxKDAAAMA4FBgAAGAcCgwAADAOBQYAABiHAgMAAIxDgQEAAMahwAAAAONQYAAAgHEoMAAAwDgUGAAAYBwKDAAAMA4FBgAAGIcCAwAAjEOBAQAAxqHAAAAA44S39gJghpsfX9fsYz99OtOPKwEAgDMwAADAQBQYAABgHAoMAAAwDgUGAAAYhwIDAACMQ4EBAADGocAAAADjUGAAAIBxKDAAAMA4fBIvrjk+xRcA4G+cgQEAAMahwAAAAONQYAAAgHEoMAAAwDgUGAAAYBzuQgJuINyxBQCNQ4HBDY1/0AEAl8JbSAAAwDgUGAAAYBwKDAAAMA4FBgAAGIeLeBGwmnIBsCPMUv5AqWfuRtXUhXABMADc4DgDAwAAjEOBAQAAxqHAAAAA41BgAACAcSgwAADAONyFBFwCP8IAAG5snIEBAADGuaHPwCxdulTPPPOMPB6P+vTpoyVLlmjgwIGtvSzgilpy9gYA0Dg37BmY1atXKycnR0899ZT27NmjPn36KCMjQxUVFa29NAAA0Mpu2DMwzz77rB5++GE99NBDkqTly5dr3bp1eumll/T444+38uqAG09Tz/x889OHy371w2u0KgC4Nm7IAlNbW6uSkhLNmTPH3hYaGqq0tDQVFxdf8piamhrV1NTYjysrKyVJJ0+elNfrbdF6vF6vqqurdeLECUVERCj8/NkWzWeC8HpL1dX1CveGqq4+pLWXc80FW17JN/OJEyeaPc+gvKJmH7tzzvBmH9tUF/45DgZkDvzMgZj39OnTkiTLsq447oYsMH//+99VV1en+Ph4n+3x8fH66KOPLnlMXl6e5s2bd9H2lJSUa7LGYPBAay/gOgu2vNI/M3d6pnWev9Oi1nleADe+06dPKzo6+rL7b8gC0xxz5sxRTk6O/bi+vl4nT55Ux44dFRLSsv+jrqqqUlJSkj7//HM5nc6WLtUIwZY52PJKwZc52PJKZA6GzIGY17IsnT59WomJiVccd0MWmE6dOiksLEzl5eU+28vLy+VyuS55jMPhkMPh8NkWExPj13U5nc6A+Q3SWMGWOdjySsGXOdjySmQOBoGW90pnXhrckHchRUZGqn///ioq+ud76/X19SoqKlJqamorrgwAANwIbsgzMJKUk5OjyZMna8CAARo4cKCee+45nT171r4rCQAABK8btsD8+7//u/7v//5Pc+fOlcfjUd++fbVhw4aLLuy9HhwOh5566qmL3qIKZMGWOdjySsGXOdjySmQOBsGW95tCrKvdpwQAAHCDuSGvgQEAALgSCgwAADAOBQYAABiHAgMAAIxDgWmEpUuX6uabb1abNm00aNAg7dq1q7WX5Bd5eXm644471L59e8XFxen+++9XWVmZz5ivv/5aWVlZ6tixo2666SaNHTv2og8YNNXTTz+tkJAQZWdn29sCMe+XX36pBx98UB07dlTbtm3Vq1cvffDBB/Z+y7I0d+5cJSQkqG3btkpLS9Phw4dbccUtU1dXpyeffFIpKSlq27atbrnlFi1YsMDn56qYnHnbtm0aPXq0EhMTFRISojfffNNnf2OynTx5UhMmTJDT6VRMTIymTJmiM2fOXMcUTXOlzF6vV7Nnz1avXr3Url07JSYmatKkSTp27JjPHCZlvtpr/E3Tpk1TSEiInnvuOZ/tJuVtLgrMVaxevVo5OTl66qmntGfPHvXp00cZGRmqqKho7aW12NatW5WVlaUdO3bI7XbL6/UqPT1dZ8/+84dVzpw5U2vXrtXrr7+urVu36tixYxozZkwrrto/du/erf/+7/9W7969fbYHWt6vvvpKQ4YMUUREhN5++20dOnRIixYtUocOHewx+fn5euGFF7R8+XLt3LlT7dq1U0ZGhr7++utWXHnzLVy4UMuWLdNvf/tbffjhh1q4cKHy8/O1ZMkSe4zJmc+ePas+ffpo6dKll9zfmGwTJkzQwYMH5Xa7VVBQoG3btmnq1KnXK0KTXSlzdXW19uzZoyeffFJ79uzRG2+8obKyMt17770+40zKfLXXuMGaNWu0Y8eOS37kvkl5m83CFQ0cONDKysqyH9fV1VmJiYlWXl5eK67q2qioqLAkWVu3brUsy7JOnTplRUREWK+//ro95sMPP7QkWcXFxa21zBY7ffq09d3vftdyu93WD37wA+uRRx6xLCsw886ePdu68847L7u/vr7ecrlc1jPPPGNvO3XqlOVwOKw//elP12OJfpeZmWn9+Mc/9tk2ZswYa8KECZZlBVZmSdaaNWvsx43JdujQIUuStXv3bnvM22+/bYWEhFhffvnldVt7c12Y+VJ27dplSbI+++wzy7LMzny5vF988YX17W9/2zpw4ICVnJxsLV682N5nct6m4AzMFdTW1qqkpERpaWn2ttDQUKWlpam4uLgVV3ZtVFZWSpJiY2MlSSUlJfJ6vT75u3btqi5duhidPysrS5mZmT65pMDM+9Zbb2nAgAH613/9V8XFxalfv376n//5H3v/kSNH5PF4fDJHR0dr0KBBxmb+l3/5FxUVFemvf/2rJOkvf/mLtm/frlGjRkkKzMwNGpOtuLhYMTExGjBggD0mLS1NoaGh2rlz53Vf87VQWVmpkJAQ++fhBVrm+vp6TZw4UbNmzVKPHj0u2h9oeS/nhv0k3hvB3//+d9XV1V306b/x8fH66KOPWmlV10Z9fb2ys7M1ZMgQ9ezZU5Lk8XgUGRl50Q/FjI+Pl8fjaYVVttyrr76qPXv2aPfu3RftC8S8f/vb37Rs2TLl5OTo5z//uXbv3q2f/exnioyM1OTJk+1cl/o9bmrmxx9/XFVVVeratavCwsJUV1enX/3qV5owYYIkBWTmBo3J5vF4FBcX57M/PDxcsbGxxueX/nEd2+zZszV+/Hj7hxsGWuaFCxcqPDxcP/vZzy65P9DyXg4FBpL+cVbiwIED2r59e2sv5Zr5/PPP9cgjj8jtdqtNmzatvZzror6+XgMGDNCvf/1rSVK/fv104MABLV++XJMnT27l1V0br732mlauXKlVq1apR48eKi0tVXZ2thITEwM2M/7B6/Xq3/7t32RZlpYtW9bay7kmSkpK9Pzzz2vPnj0KCQlp7eW0Kt5CuoJOnTopLCzsortQysvL5XK5WmlV/jdjxgwVFBTonXfeUefOne3tLpdLtbW1OnXqlM94U/OXlJSooqJCt99+u8LDwxUeHq6tW7fqhRdeUHh4uOLj4wMqryQlJCSoe/fuPtu6deumo0ePSpKdK5B+j8+aNUuPP/64xo0bp169emnixImaOXOm8vLyJAVm5gaNyeZyuS66CeH8+fM6efKk0fkbystnn30mt9ttn32RAivzu+++q4qKCnXp0sX+e+yzzz7To48+qptvvllSYOW9EgrMFURGRqp///4qKiqyt9XX16uoqEipqamtuDL/sCxLM2bM0Jo1a7R582alpKT47O/fv78iIiJ88peVleno0aNG5h8+fLj279+v0tJS+2vAgAGaMGGC/etAyitJQ4YMuejW+L/+9a9KTk6WJKWkpMjlcvlkrqqq0s6dO43NXF1drdBQ37/awsLCVF9fLykwMzdoTLbU1FSdOnVKJSUl9pjNmzervr5egwYNuu5r9oeG8nL48GFt2rRJHTt29NkfSJknTpyoffv2+fw9lpiYqFmzZmnjxo2SAivvFbX2VcQ3uldffdVyOBzWihUrrEOHDllTp061YmJiLI/H09pLa7Hp06db0dHR1pYtW6zjx4/bX9XV1faYadOmWV26dLE2b95sffDBB1ZqaqqVmpraiqv2r2/ehWRZgZd3165dVnh4uPWrX/3KOnz4sLVy5UorKirKeuWVV+wxTz/9tBUTE2P9+c9/tvbt22fdd999VkpKinXu3LlWXHnzTZ482fr2t79tFRQUWEeOHLHeeOMNq1OnTtZjjz1mjzE58+nTp629e/dae/futSRZzz77rLV37177jpvGZBs5cqTVr18/a+fOndb27dut7373u9b48eNbK9JVXSlzbW2tde+991qdO3e2SktLff4uq6mpsecwKfPVXuMLXXgXkmWZlbe5KDCNsGTJEqtLly5WZGSkNXDgQGvHjh2tvSS/kHTJr5dfftkec+7cOes///M/rQ4dOlhRUVHWj370I+v48eOtt2g/u7DABGLetWvXWj179rQcDofVtWtX68UXX/TZX19fbz355JNWfHy85XA4rOHDh1tlZWWttNqWq6qqsh555BGrS5cuVps2bazvfOc71i9+8Quff8xMzvzOO+9c8s/t5MmTLctqXLYTJ05Y48ePt2666SbL6XRaDz30kHX69OlWSNM4V8p85MiRy/5d9s4779hzmJT5aq/xhS5VYEzK21whlvWNj6cEAAAwANfAAAAA41BgAACAcSgwAADAOBQYAABgHAoMAAAwDgUGAAAYhwIDAACMQ4EBAADGocAAAADjUGAAAIBxKDAAAMA4FBgAAGCc/wfkCsn8BDLYuAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "taxis.fare.hist(bins=30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "487dcb1b",
   "metadata": {
    "hide_input": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0      1.0\n",
       "0.1      5.0\n",
       "0.2      6.0\n",
       "0.3      7.0\n",
       "0.4      8.0\n",
       "0.5      9.5\n",
       "0.6     11.0\n",
       "0.7     13.5\n",
       "0.8     17.0\n",
       "0.9     26.0\n",
       "1.0    150.0\n",
       "Name: fare, dtype: float64"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "taxis.fare.quantile(np.arange(0, 1.1, 0.1))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d3a900cc",
   "metadata": {},
   "source": [
    "```\n",
    "pd.cut(\n",
    "    x,\n",
    "    bins,\n",
    "    right: 'bool' = True,\n",
    "    labels=None,\n",
    "    retbins: 'bool' = False,\n",
    "    precision: 'int' = 3,\n",
    "    include_lowest: 'bool' = False,\n",
    "    duplicates: 'str' = 'raise',\n",
    "    ordered: 'bool' = True,\n",
    ")\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "dbdd9961",
   "metadata": {
    "hide_input": true
   },
   "outputs": [],
   "source": [
    "# conver a numeric variable in \"user defined\" ranges\n",
    "taxis[\"fare_levels\"] = pd.cut(taxis.fare,                # pass a series\n",
    "                              [0, 10, 30, np.inf],       # define the values to \"break\" the data into groups\n",
    "                              #labels = ['low', \"medium\", \"high\"],\n",
    "                              right=False                # not the default but it's better IMO\n",
    "                             )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "3b72bb05",
   "metadata": {
    "hide_input": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.0, 10.0)     3420\n",
       "[10.0, 30.0)    2501\n",
       "[30.0, inf)      512\n",
       "Name: fare_levels, dtype: int64"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "taxis[\"fare_levels\"].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3e7f4c0c",
   "metadata": {},
   "source": [
    "# 10. Use pd.qcut to discretize variables using quantiles\n",
    "\n",
    "`pd.qcut`: Discretize variable into equal-sized buckets based on rank or based on sample quantiles. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "d9b7c5fd",
   "metadata": {
    "hide_input": true
   },
   "outputs": [],
   "source": [
    "ranked_fares = taxis.fare.rank(method='first')\n",
    "ranked_tips = taxis.tip.rank(method='first')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "cbc246ce",
   "metadata": {
    "hide_input": true
   },
   "outputs": [],
   "source": [
    "taxis[\"fare_deciles\"] = pd.qcut(ranked_fares, q=10, labels=False, duplicates=\"drop\")\n",
    "taxis[\"tip_deciles\"] = pd.qcut(ranked_tips, q=10, labels=False, duplicates=\"drop\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "397bb567",
   "metadata": {
    "hide_input": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pickup</th>\n",
       "      <th>dropoff</th>\n",
       "      <th>passengers</th>\n",
       "      <th>distance</th>\n",
       "      <th>fare</th>\n",
       "      <th>tip</th>\n",
       "      <th>tolls</th>\n",
       "      <th>total</th>\n",
       "      <th>color</th>\n",
       "      <th>payment</th>\n",
       "      <th>pickup_zone</th>\n",
       "      <th>dropoff_zone</th>\n",
       "      <th>pickup_borough</th>\n",
       "      <th>dropoff_borough</th>\n",
       "      <th>fl_outlier</th>\n",
       "      <th>fare_levels</th>\n",
       "      <th>fare_deciles</th>\n",
       "      <th>tip_deciles</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2019-03-23 20:21:09</td>\n",
       "      <td>2019-03-23 20:27:24</td>\n",
       "      <td>1</td>\n",
       "      <td>1.60</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2.15</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12.95</td>\n",
       "      <td>yellow</td>\n",
       "      <td>credit card</td>\n",
       "      <td>Lenox Hill West</td>\n",
       "      <td>UN/Turtle Bay South</td>\n",
       "      <td>Manhattan</td>\n",
       "      <td>Manhattan</td>\n",
       "      <td>ok</td>\n",
       "      <td>[0.0, 10.0)</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2019-03-04 16:11:55</td>\n",
       "      <td>2019-03-04 16:19:00</td>\n",
       "      <td>1</td>\n",
       "      <td>0.79</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.30</td>\n",
       "      <td>yellow</td>\n",
       "      <td>cash</td>\n",
       "      <td>Upper West Side South</td>\n",
       "      <td>Upper West Side South</td>\n",
       "      <td>Manhattan</td>\n",
       "      <td>Manhattan</td>\n",
       "      <td>ok</td>\n",
       "      <td>[0.0, 10.0)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2019-03-27 17:53:01</td>\n",
       "      <td>2019-03-27 18:00:25</td>\n",
       "      <td>1</td>\n",
       "      <td>1.37</td>\n",
       "      <td>7.5</td>\n",
       "      <td>2.36</td>\n",
       "      <td>0.0</td>\n",
       "      <td>14.16</td>\n",
       "      <td>yellow</td>\n",
       "      <td>credit card</td>\n",
       "      <td>Alphabet City</td>\n",
       "      <td>West Village</td>\n",
       "      <td>Manhattan</td>\n",
       "      <td>Manhattan</td>\n",
       "      <td>ok</td>\n",
       "      <td>[0.0, 10.0)</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2019-03-10 01:23:59</td>\n",
       "      <td>2019-03-10 01:49:51</td>\n",
       "      <td>1</td>\n",
       "      <td>7.70</td>\n",
       "      <td>27.0</td>\n",
       "      <td>6.15</td>\n",
       "      <td>0.0</td>\n",
       "      <td>36.95</td>\n",
       "      <td>yellow</td>\n",
       "      <td>credit card</td>\n",
       "      <td>Hudson Sq</td>\n",
       "      <td>Yorkville West</td>\n",
       "      <td>Manhattan</td>\n",
       "      <td>Manhattan</td>\n",
       "      <td>ok</td>\n",
       "      <td>[10.0, 30.0)</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2019-03-30 13:27:42</td>\n",
       "      <td>2019-03-30 13:37:14</td>\n",
       "      <td>3</td>\n",
       "      <td>2.16</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1.10</td>\n",
       "      <td>0.0</td>\n",
       "      <td>13.40</td>\n",
       "      <td>yellow</td>\n",
       "      <td>credit card</td>\n",
       "      <td>Midtown East</td>\n",
       "      <td>Yorkville West</td>\n",
       "      <td>Manhattan</td>\n",
       "      <td>Manhattan</td>\n",
       "      <td>ok</td>\n",
       "      <td>[0.0, 10.0)</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               pickup             dropoff  passengers  distance  fare   tip  \\\n",
       "0 2019-03-23 20:21:09 2019-03-23 20:27:24           1      1.60   7.0  2.15   \n",
       "1 2019-03-04 16:11:55 2019-03-04 16:19:00           1      0.79   5.0  0.00   \n",
       "2 2019-03-27 17:53:01 2019-03-27 18:00:25           1      1.37   7.5  2.36   \n",
       "3 2019-03-10 01:23:59 2019-03-10 01:49:51           1      7.70  27.0  6.15   \n",
       "4 2019-03-30 13:27:42 2019-03-30 13:37:14           3      2.16   9.0  1.10   \n",
       "\n",
       "   tolls  total   color      payment            pickup_zone  \\\n",
       "0    0.0  12.95  yellow  credit card        Lenox Hill West   \n",
       "1    0.0   9.30  yellow         cash  Upper West Side South   \n",
       "2    0.0  14.16  yellow  credit card          Alphabet City   \n",
       "3    0.0  36.95  yellow  credit card              Hudson Sq   \n",
       "4    0.0  13.40  yellow  credit card           Midtown East   \n",
       "\n",
       "            dropoff_zone pickup_borough dropoff_borough fl_outlier  \\\n",
       "0    UN/Turtle Bay South      Manhattan       Manhattan         ok   \n",
       "1  Upper West Side South      Manhattan       Manhattan         ok   \n",
       "2           West Village      Manhattan       Manhattan         ok   \n",
       "3         Yorkville West      Manhattan       Manhattan         ok   \n",
       "4         Yorkville West      Manhattan       Manhattan         ok   \n",
       "\n",
       "    fare_levels  fare_deciles  tip_deciles  \n",
       "0   [0.0, 10.0)             2            6  \n",
       "1   [0.0, 10.0)             0            0  \n",
       "2   [0.0, 10.0)             3            6  \n",
       "3  [10.0, 30.0)             9            9  \n",
       "4   [0.0, 10.0)             4            4  "
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "taxis.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "e53419d2",
   "metadata": {
    "hide_input": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>tip_deciles</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fare_deciles</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.627321</td>\n",
       "      <td>0.457681</td>\n",
       "      <td>0.412051</td>\n",
       "      <td>0.523556</td>\n",
       "      <td>0.538109</td>\n",
       "      <td>0.644918</td>\n",
       "      <td>0.598261</td>\n",
       "      <td>0.517778</td>\n",
       "      <td>0.560000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.853210</td>\n",
       "      <td>0.781250</td>\n",
       "      <td>0.743585</td>\n",
       "      <td>0.830781</td>\n",
       "      <td>0.777107</td>\n",
       "      <td>0.843386</td>\n",
       "      <td>0.756087</td>\n",
       "      <td>0.873636</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.585000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.119259</td>\n",
       "      <td>0.962581</td>\n",
       "      <td>0.986563</td>\n",
       "      <td>1.026500</td>\n",
       "      <td>1.061458</td>\n",
       "      <td>1.040876</td>\n",
       "      <td>1.079908</td>\n",
       "      <td>1.007879</td>\n",
       "      <td>1.291250</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.297759</td>\n",
       "      <td>1.232281</td>\n",
       "      <td>1.228039</td>\n",
       "      <td>1.269048</td>\n",
       "      <td>1.250000</td>\n",
       "      <td>1.310571</td>\n",
       "      <td>1.241270</td>\n",
       "      <td>1.181842</td>\n",
       "      <td>1.289286</td>\n",
       "      <td>1.600000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.576889</td>\n",
       "      <td>1.475789</td>\n",
       "      <td>1.563182</td>\n",
       "      <td>1.540141</td>\n",
       "      <td>1.515370</td>\n",
       "      <td>1.521864</td>\n",
       "      <td>1.458452</td>\n",
       "      <td>1.632500</td>\n",
       "      <td>1.629722</td>\n",
       "      <td>1.887500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1.860290</td>\n",
       "      <td>1.876296</td>\n",
       "      <td>1.797037</td>\n",
       "      <td>1.906038</td>\n",
       "      <td>1.802857</td>\n",
       "      <td>1.918776</td>\n",
       "      <td>1.923913</td>\n",
       "      <td>1.918193</td>\n",
       "      <td>1.961698</td>\n",
       "      <td>2.220000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2.286000</td>\n",
       "      <td>2.217797</td>\n",
       "      <td>2.527679</td>\n",
       "      <td>2.546038</td>\n",
       "      <td>2.292000</td>\n",
       "      <td>2.490263</td>\n",
       "      <td>2.464375</td>\n",
       "      <td>2.288268</td>\n",
       "      <td>2.431594</td>\n",
       "      <td>2.627143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>3.256875</td>\n",
       "      <td>3.063235</td>\n",
       "      <td>3.381045</td>\n",
       "      <td>3.552833</td>\n",
       "      <td>2.943077</td>\n",
       "      <td>3.245429</td>\n",
       "      <td>3.096000</td>\n",
       "      <td>3.183125</td>\n",
       "      <td>3.196913</td>\n",
       "      <td>3.300189</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>4.957833</td>\n",
       "      <td>5.078116</td>\n",
       "      <td>4.949028</td>\n",
       "      <td>5.228936</td>\n",
       "      <td>4.381111</td>\n",
       "      <td>5.166875</td>\n",
       "      <td>4.601429</td>\n",
       "      <td>5.166957</td>\n",
       "      <td>4.365679</td>\n",
       "      <td>5.558025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>12.220196</td>\n",
       "      <td>12.410238</td>\n",
       "      <td>13.488171</td>\n",
       "      <td>11.518065</td>\n",
       "      <td>8.853333</td>\n",
       "      <td>10.385000</td>\n",
       "      <td>9.345000</td>\n",
       "      <td>9.522000</td>\n",
       "      <td>10.645000</td>\n",
       "      <td>12.532758</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "tip_deciles           0          1          2          3         4          5  \\\n",
       "fare_deciles                                                                    \n",
       "0              0.627321   0.457681   0.412051   0.523556  0.538109   0.644918   \n",
       "1              0.853210   0.781250   0.743585   0.830781  0.777107   0.843386   \n",
       "2              1.119259   0.962581   0.986563   1.026500  1.061458   1.040876   \n",
       "3              1.297759   1.232281   1.228039   1.269048  1.250000   1.310571   \n",
       "4              1.576889   1.475789   1.563182   1.540141  1.515370   1.521864   \n",
       "5              1.860290   1.876296   1.797037   1.906038  1.802857   1.918776   \n",
       "6              2.286000   2.217797   2.527679   2.546038  2.292000   2.490263   \n",
       "7              3.256875   3.063235   3.381045   3.552833  2.943077   3.245429   \n",
       "8              4.957833   5.078116   4.949028   5.228936  4.381111   5.166875   \n",
       "9             12.220196  12.410238  13.488171  11.518065  8.853333  10.385000   \n",
       "\n",
       "tip_deciles          6         7          8          9  \n",
       "fare_deciles                                            \n",
       "0             0.598261  0.517778   0.560000   0.000000  \n",
       "1             0.756087  0.873636   0.750000   0.585000  \n",
       "2             1.079908  1.007879   1.291250   0.000000  \n",
       "3             1.241270  1.181842   1.289286   1.600000  \n",
       "4             1.458452  1.632500   1.629722   1.887500  \n",
       "5             1.923913  1.918193   1.961698   2.220000  \n",
       "6             2.464375  2.288268   2.431594   2.627143  \n",
       "7             3.096000  3.183125   3.196913   3.300189  \n",
       "8             4.601429  5.166957   4.365679   5.558025  \n",
       "9             9.345000  9.522000  10.645000  12.532758  "
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mat = (taxis\n",
    " .groupby([\"fare_deciles\", \"tip_deciles\"])\n",
    " ['distance']\n",
    " .mean()\n",
    " .unstack(1)\n",
    " .fillna(0)\n",
    ")\n",
    "mat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca3e0aba",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe199872",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a617a26a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d6a7136b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "288795f1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "632fc924",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": true,
  "kernelspec": {
   "display_name": "Python [conda env:tabular]",
   "language": "python",
   "name": "conda-env-tabular-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}