data_preprocessing_practice_7.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 99,
      "metadata": {
        "id": "79NXekfmW7MK"
      },
      "outputs": [],
      "source": [
        "import pandas as pd"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 1\n",
        "cols_to_load = ['trip_distance', 'passenger_count', 'fare_amount']\n",
        "dtypes = {'trip_distance':'float32', 'passenger_count':'int8', 'fare_amount':'float32'}\n",
        "\n",
        "nyc = pd.read_parquet('/content/drive/MyDrive/Datasets/pandas 4/yellow_tripdata_2024-01.parquet', columns=cols_to_load)\n",
        "nyc"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "KuXNtZiahz2z",
        "outputId": "609160a0-456d-4acd-9d7e-f2a885018142"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "         trip_distance  passenger_count  fare_amount\n",
              "0                 1.72              1.0        17.70\n",
              "1                 1.80              1.0        10.00\n",
              "2                 4.70              1.0        23.30\n",
              "3                 1.40              1.0        10.00\n",
              "4                 0.80              1.0         7.90\n",
              "...                ...              ...          ...\n",
              "2964619           3.18              NaN        15.77\n",
              "2964620           4.00              NaN        18.40\n",
              "2964621           3.33              NaN        19.97\n",
              "2964622           3.06              NaN        23.88\n",
              "2964623           8.10              NaN        32.40\n",
              "\n",
              "[2964624 rows x 3 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-229e22ee-9e9c-49a9-b739-3b679394916d\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>trip_distance</th>\n",
              "      <th>passenger_count</th>\n",
              "      <th>fare_amount</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1.72</td>\n",
              "      <td>1.0</td>\n",
              "      <td>17.70</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1.80</td>\n",
              "      <td>1.0</td>\n",
              "      <td>10.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>4.70</td>\n",
              "      <td>1.0</td>\n",
              "      <td>23.30</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1.40</td>\n",
              "      <td>1.0</td>\n",
              "      <td>10.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.80</td>\n",
              "      <td>1.0</td>\n",
              "      <td>7.90</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2964619</th>\n",
              "      <td>3.18</td>\n",
              "      <td>NaN</td>\n",
              "      <td>15.77</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2964620</th>\n",
              "      <td>4.00</td>\n",
              "      <td>NaN</td>\n",
              "      <td>18.40</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2964621</th>\n",
              "      <td>3.33</td>\n",
              "      <td>NaN</td>\n",
              "      <td>19.97</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2964622</th>\n",
              "      <td>3.06</td>\n",
              "      <td>NaN</td>\n",
              "      <td>23.88</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2964623</th>\n",
              "      <td>8.10</td>\n",
              "      <td>NaN</td>\n",
              "      <td>32.40</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>2964624 rows × 3 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-229e22ee-9e9c-49a9-b739-3b679394916d')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-229e22ee-9e9c-49a9-b739-3b679394916d button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-229e22ee-9e9c-49a9-b739-3b679394916d');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-2303a53d-5aba-4b60-bba7-099d75a820ec\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-2303a53d-5aba-4b60-bba7-099d75a820ec')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-2303a53d-5aba-4b60-bba7-099d75a820ec button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "nyc"
            }
          },
          "metadata": {},
          "execution_count": 79
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 2\n",
        "fli = pd.read_csv('/content/drive/MyDrive/Datasets/pandas 4/flights.csv')\n",
        "\n",
        "def average_delay_per_airline(df):\n",
        "  return df.groupby('AIRLINE')['AIRLINE_DELAY'].mean()\n",
        "\n",
        "average_delay_per_airline(fli)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kQZUnzV97mz8",
        "outputId": "3ec58776-1efc-459a-a029-a104295481b5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-44-51aa24d93e8d>:2: DtypeWarning: Columns (7,8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
            "  fli = pd.read_csv('/content/drive/MyDrive/Datasets/pandas 4/flights.csv')\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "AIRLINE\n",
              "AA    21.139201\n",
              "AS    15.543352\n",
              "B6    18.150503\n",
              "DL    22.941028\n",
              "EV    21.651277\n",
              "F9    14.719983\n",
              "HA    22.792063\n",
              "MQ    17.425025\n",
              "NK    13.766839\n",
              "OO    18.959163\n",
              "UA    21.144678\n",
              "US    17.410107\n",
              "VX    11.097809\n",
              "WN    16.191674\n",
              "Name: AIRLINE_DELAY, dtype: float64"
            ]
          },
          "metadata": {},
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 3\n",
        "fli['AIRLINE'].astype('category') # reduce load time if repetitive string values"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7Y3u7Ipt8gQr",
        "outputId": "1f7eb0bc-ae8c-4ab8-e3ca-7b597fd2a2fc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0          AS\n",
              "1          AA\n",
              "2          US\n",
              "3          AA\n",
              "4          AS\n",
              "           ..\n",
              "5819074    B6\n",
              "5819075    B6\n",
              "5819076    B6\n",
              "5819077    B6\n",
              "5819078    B6\n",
              "Name: AIRLINE, Length: 5819079, dtype: category\n",
              "Categories (14, object): ['AA', 'AS', 'B6', 'DL', ..., 'UA', 'US', 'VX', 'WN']"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 4\n",
        "def replace_symbols(value):\n",
        "  if not isinstance(value, str):\n",
        "    return value\n",
        "  else:\n",
        "    for old_value, new_value in symbols.items():\n",
        "      value = value.replace(old_value, new_value)\n",
        "    return value\n",
        "\n",
        "def str_to_float(value):\n",
        "  if not isinstance(value, str):\n",
        "    return value\n",
        "  else:\n",
        "    for old_value, multiplier in numeric.items():\n",
        "      return float(value.replace(old_value, '')) * multiplier\n",
        "\n",
        "def data_clean(value):\n",
        "  x = replace_symbols(value)\n",
        "  return str_to_float(x)\n",
        "\n",
        "symbols = {'<=': '', '<': '', '>': '', '>=': ''}\n",
        "numeric = {'K': 1000}\n",
        "\n",
        "adult = pd.read_csv('/content/drive/MyDrive/Datasets/pandas 4/adult.data', header=None)\n",
        "print(adult.iloc[:, -1].apply(data_clean).mean()) # last column is about salary. values are not accurate."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1tnmILYV857L",
        "outputId": "dc42dec9-b2d6-4e11-fd27-b4ab055bb9da"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "50000.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 5\n",
        "nyc.set_index('trip_distance') # can use inplace=True"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 455
        },
        "id": "TemVO2RdwtUQ",
        "outputId": "35749c8a-c8ca-405c-8c65-77f44ead64f2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "               passenger_count  fare_amount\n",
              "trip_distance                              \n",
              "1.72                       1.0        17.70\n",
              "1.80                       1.0        10.00\n",
              "4.70                       1.0        23.30\n",
              "1.40                       1.0        10.00\n",
              "0.80                       1.0         7.90\n",
              "...                        ...          ...\n",
              "3.18                       NaN        15.77\n",
              "4.00                       NaN        18.40\n",
              "3.33                       NaN        19.97\n",
              "3.06                       NaN        23.88\n",
              "8.10                       NaN        32.40\n",
              "\n",
              "[2964624 rows x 2 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e44e6994-6ec9-48f5-880c-853cba6646bb\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>passenger_count</th>\n",
              "      <th>fare_amount</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>trip_distance</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1.72</th>\n",
              "      <td>1.0</td>\n",
              "      <td>17.70</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1.80</th>\n",
              "      <td>1.0</td>\n",
              "      <td>10.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4.70</th>\n",
              "      <td>1.0</td>\n",
              "      <td>23.30</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1.40</th>\n",
              "      <td>1.0</td>\n",
              "      <td>10.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0.80</th>\n",
              "      <td>1.0</td>\n",
              "      <td>7.90</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3.18</th>\n",
              "      <td>NaN</td>\n",
              "      <td>15.77</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4.00</th>\n",
              "      <td>NaN</td>\n",
              "      <td>18.40</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3.33</th>\n",
              "      <td>NaN</td>\n",
              "      <td>19.97</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3.06</th>\n",
              "      <td>NaN</td>\n",
              "      <td>23.88</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8.10</th>\n",
              "      <td>NaN</td>\n",
              "      <td>32.40</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>2964624 rows × 2 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e44e6994-6ec9-48f5-880c-853cba6646bb')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-e44e6994-6ec9-48f5-880c-853cba6646bb button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-e44e6994-6ec9-48f5-880c-853cba6646bb');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-b1f90d01-862b-4807-aec9-5eda42e0390e\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-b1f90d01-862b-4807-aec9-5eda42e0390e')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-b1f90d01-862b-4807-aec9-5eda42e0390e button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe"
            }
          },
          "metadata": {},
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 6\n",
        "dates = pd.to_datetime(fli['YEAR'], format='%Y').dt.year\n",
        "dates.astype('int16')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "64NgIIyixnSz",
        "outputId": "b71ffcb8-74d9-42f8-d379-f8f9adf14d5c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0          2015\n",
              "1          2015\n",
              "2          2015\n",
              "3          2015\n",
              "4          2015\n",
              "           ... \n",
              "5819074    2015\n",
              "5819075    2015\n",
              "5819076    2015\n",
              "5819077    2015\n",
              "5819078    2015\n",
              "Name: YEAR, Length: 5819079, dtype: int16"
            ]
          },
          "metadata": {},
          "execution_count": 61
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 7\n",
        "print(fli['ARRIVAL_TIME'] / 2)\n",
        "print(fli['ARRIVAL_TIME'].apply(lambda x: x / 2)) # apply operation appears take longer"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "F6SVjXjd0rBX",
        "outputId": "8af623b4-8d71-4176-9026-cfffe805583d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0          204.0\n",
            "1          370.5\n",
            "2          405.5\n",
            "3          378.0\n",
            "4          129.5\n",
            "           ...  \n",
            "5819074    376.5\n",
            "5819075    215.0\n",
            "5819076    216.0\n",
            "5819077    165.0\n",
            "5819078    221.0\n",
            "Name: ARRIVAL_TIME, Length: 5819079, dtype: float64\n",
            "0          204.0\n",
            "1          370.5\n",
            "2          405.5\n",
            "3          378.0\n",
            "4          129.5\n",
            "           ...  \n",
            "5819074    376.5\n",
            "5819075    215.0\n",
            "5819076    216.0\n",
            "5819077    165.0\n",
            "5819078    221.0\n",
            "Name: ARRIVAL_TIME, Length: 5819079, dtype: float64\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 8\n",
        "path = '/content/drive/MyDrive/Datasets/pandas 4/adult.data'\n",
        "chunk_size = 16280 # loading half at a time\n",
        "\n",
        "for chunck in pd.read_csv(path, header=None, chunksize=chunk_size):\n",
        "  print(chunck)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "U4exBptH1O3S",
        "outputId": "b29426ff-2862-425d-a188-e999701c7b3e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "       0                  1       2              3   4                    5   \\\n",
            "0      39          State-gov   77516      Bachelors  13        Never-married   \n",
            "1      50   Self-emp-not-inc   83311      Bachelors  13   Married-civ-spouse   \n",
            "2      38            Private  215646        HS-grad   9             Divorced   \n",
            "3      53            Private  234721           11th   7   Married-civ-spouse   \n",
            "4      28            Private  338409      Bachelors  13   Married-civ-spouse   \n",
            "...    ..                ...     ...            ...  ..                  ...   \n",
            "16275  45            Private  162606        HS-grad   9             Divorced   \n",
            "16276  24            Private  179627        HS-grad   9   Married-civ-spouse   \n",
            "16277  34            Private  103408   Some-college  10             Divorced   \n",
            "16278  27            Private   36440      Bachelors  13        Never-married   \n",
            "16279  26            Private   57512      Assoc-voc  11        Never-married   \n",
            "\n",
            "                       6               7       8        9     10  11  12  \\\n",
            "0            Adm-clerical   Not-in-family   White     Male  2174   0  40   \n",
            "1         Exec-managerial         Husband   White     Male     0   0  13   \n",
            "2       Handlers-cleaners   Not-in-family   White     Male     0   0  40   \n",
            "3       Handlers-cleaners         Husband   Black     Male     0   0  40   \n",
            "4          Prof-specialty            Wife   Black   Female     0   0  40   \n",
            "...                   ...             ...     ...      ...   ...  ..  ..   \n",
            "16275               Sales   Not-in-family   White   Female     0   0  35   \n",
            "16276   Machine-op-inspct         Husband   White     Male     0   0  40   \n",
            "16277      Prof-specialty   Not-in-family   Black     Male     0   0  40   \n",
            "16278               Sales   Not-in-family   White   Female     0   0  40   \n",
            "16279        Craft-repair       Own-child   White     Male     0   0  48   \n",
            "\n",
            "                   13      14  \n",
            "0       United-States   <=50K  \n",
            "1       United-States   <=50K  \n",
            "2       United-States   <=50K  \n",
            "3       United-States   <=50K  \n",
            "4                Cuba   <=50K  \n",
            "...               ...     ...  \n",
            "16275   United-States   <=50K  \n",
            "16276   United-States   <=50K  \n",
            "16277         Germany    >50K  \n",
            "16278   United-States   <=50K  \n",
            "16279   United-States   <=50K  \n",
            "\n",
            "[16280 rows x 15 columns]\n",
            "       0                  1       2              3   4                    5   \\\n",
            "16280  27            Private  187981        HS-grad   9        Never-married   \n",
            "16281  55            Private  393768      Assoc-voc  11   Married-civ-spouse   \n",
            "16282  38            Private  108726        HS-grad   9   Married-civ-spouse   \n",
            "16283  31            Private  180551        HS-grad   9   Married-civ-spouse   \n",
            "16284  51   Self-emp-not-inc  176240     Assoc-acdm  12   Married-civ-spouse   \n",
            "...    ..                ...     ...            ...  ..                  ...   \n",
            "32555  22            Private  310152   Some-college  10        Never-married   \n",
            "32556  27            Private  257302     Assoc-acdm  12   Married-civ-spouse   \n",
            "32557  40            Private  154374        HS-grad   9   Married-civ-spouse   \n",
            "32558  58            Private  151910        HS-grad   9              Widowed   \n",
            "32559  22            Private  201490        HS-grad   9        Never-married   \n",
            "\n",
            "                       6               7       8        9   10  11  12  \\\n",
            "16280   Handlers-cleaners       Own-child   White     Male   0   0  40   \n",
            "16281     Exec-managerial         Husband   White     Male   0   0  40   \n",
            "16282     Exec-managerial         Husband   White     Male   0   0  40   \n",
            "16283        Adm-clerical            Wife   White   Female   0   0  40   \n",
            "16284               Sales         Husband   White     Male   0   0  40   \n",
            "...                   ...             ...     ...      ...  ..  ..  ..   \n",
            "32555     Protective-serv   Not-in-family   White     Male   0   0  40   \n",
            "32556        Tech-support            Wife   White   Female   0   0  38   \n",
            "32557   Machine-op-inspct         Husband   White     Male   0   0  40   \n",
            "32558        Adm-clerical       Unmarried   White   Female   0   0  40   \n",
            "32559        Adm-clerical       Own-child   White     Male   0   0  20   \n",
            "\n",
            "                   13      14  \n",
            "16280   United-States   <=50K  \n",
            "16281   United-States   <=50K  \n",
            "16282   United-States    >50K  \n",
            "16283   United-States    >50K  \n",
            "16284   United-States    >50K  \n",
            "...               ...     ...  \n",
            "32555   United-States   <=50K  \n",
            "32556   United-States   <=50K  \n",
            "32557   United-States    >50K  \n",
            "32558   United-States   <=50K  \n",
            "32559   United-States   <=50K  \n",
            "\n",
            "[16280 rows x 15 columns]\n",
            "       0              1       2         3   4                    5   \\\n",
            "32560  52   Self-emp-inc  287927   HS-grad   9   Married-civ-spouse   \n",
            "\n",
            "                     6      7       8        9      10  11  12  \\\n",
            "32560   Exec-managerial   Wife   White   Female  15024   0  40   \n",
            "\n",
            "                   13     14  \n",
            "32560   United-States   >50K  \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 9\n",
        "nyc = pd.read_parquet('/content/drive/MyDrive/Datasets/pandas 4/yellow_tripdata_2024-01.parquet')\n",
        "january = pd.to_datetime(nyc['tpep_pickup_datetime']).dt.month == 1"
      ],
      "metadata": {
        "id": "Ls_oM0zA3Hrv"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}