COVID-19 Time Series Data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import libraries and read data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import plotly\n",
    "from plotly.offline import init_notebook_mode, iplot\n",
    "import plotly.express as px\n",
    "\n",
    "import plotly.graph_objects as go\n",
    "from plotly.graph_objects import Bar\n",
    "\n",
    "init_notebook_mode(connected=True)\n",
    "plotly.offline.init_notebook_mode(connected=True)\n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "import seaborn as sns\n",
    "\n",
    "sns.set()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# time series data folder path\n",
    "time_series_data_path = \"COVID-19/csse_covid_19_data/csse_covid_19_time_series\"\n",
    "\n",
    "# global data\n",
    "confirmed_global_path = (\n",
    "    f\"{time_series_data_path}/time_series_covid19_confirmed_global.csv\"\n",
    ")\n",
    "deaths_global_path = f\"{time_series_data_path}/time_series_covid19_deaths_global.csv\"\n",
    "recovered_global_path = (\n",
    "    f\"{time_series_data_path}/time_series_covid19_recovered_global.csv\"\n",
    ")\n",
    "\n",
    "# U.S. data\n",
    "confirmed_us_path = f\"{time_series_data_path}/time_series_covid19_confirmed_US.csv\"\n",
    "deaths_us_path = f\"{time_series_data_path}/time_series_covid19_deaths_US.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# global data\n",
    "confirmed_global_df = pd.read_csv(confirmed_global_path)\n",
    "deaths_global_df = pd.read_csv(deaths_global_path)\n",
    "recovered_global_df = pd.read_csv(recovered_global_path)\n",
    "\n",
    "# U.S data\n",
    "confirmed_us_df = pd.read_csv(confirmed_us_path)\n",
    "deaths_us_df = pd.read_csv(deaths_us_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_fig_as_div(fig_obj, file_name, fig_config=None):\n",
    "    with open(f\"charts/{file_name}\", \"w\") as fig_file:\n",
    "        fig_div_string = plotly.offline.plot(\n",
    "            figure_or_data=fig_obj,\n",
    "            output_type=\"div\",\n",
    "            include_plotlyjs=\"cdn\",\n",
    "            auto_play=False,\n",
    "            config=fig_config,\n",
    "        )\n",
    "        fig_file.write(fig_div_string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_n_color_palette(palette_name, n_colors, as_hex=False):\n",
    "    palette = sns.color_palette(palette=palette_name, n_colors=n_colors)\n",
    "    if as_hex:\n",
    "        palette = palette.as_hex()\n",
    "    palette.reverse()\n",
    "    return palette"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get to know the data:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Globally confirmed cases:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "confirmed_global_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "confirmed_global_df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Globally deaths:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "deaths_global_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "deaths_global_df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Globally recovered cases:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "recovered_global_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "recovered_global_df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## U.S. confirmed cases:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "confirmed_us_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "confirmed_us_df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## U.S. deaths:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "deaths_us_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "deaths_us_df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Summary:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The three datasets share the same structure, and have the same fields, one for **confirmed** cases, one for **deaths** cases, and one for **recovered** cases.\n",
    "Fields descriptioin\n",
    "* Province/State: China - province name; US/Canada/Australia/ - city name, state/province name; Others - name of the event (e.g., \"Diamond Princess\" cruise ship); other countries - blank.\n",
    "* Country/Region: country/region name conforming to WHO (will be updated).\n",
    "* Lat and Long: a coordinates reference for the user.\n",
    "* Date fields: M/DD/YYYY (UTC), the *cumulative* number of cases up until this date.\n",
    "\n",
    "It can be seen that the `U.S.` dataset differs from the `global` one, as it has these additional columns: UID, iso2, iso3, code3, FIPS (we'll get to them later)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Preprocessing:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# global data\n",
    "print(f\"Globally confirmed cases: {confirmed_global_df.shape}\")\n",
    "print(f\"Globally deaths: {deaths_global_df.shape}\")\n",
    "print(f\"Globally recovered cases: {recovered_global_df.shape}\")\n",
    "\n",
    "# U.S. data\n",
    "print(f\"U.S. Confirmed cases: {confirmed_us_df.shape}\")\n",
    "print(f\"U.S. deaths: {deaths_us_df.shape}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "although the three datasets have the same structure, they are inconsistent, the number of **recovered** cases is different, so I think we should drop it."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1 - Drop irrelevant columns:\n",
    "`Lat` and `Long` columns specify the coordinates of the case, we won't be needing this inofrmation right now."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def drop_irrelevant_columns(df, irrelevant_columns):\n",
    "    new_df = df.drop(columns=irrelevant_columns)\n",
    "    return new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "irrelevant_columns = [\"Lat\", \"Long\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "confirmed_global_df = drop_irrelevant_columns(confirmed_global_df, irrelevant_columns)\n",
    "deaths_global_df = drop_irrelevant_columns(deaths_global_df, irrelevant_columns)\n",
    "recovered_global_df = drop_irrelevant_columns(recovered_global_df, irrelevant_columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "confirmed_global_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "deaths_global_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "recovered_global_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Also, drop columns `UID`, `iso2`, `iso3`, `code3`, `FIPS`, `Admin2`, `Combined_Key` from the `U.S.` data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "irrelevant_columns = [\n",
    "    \"UID\",\n",
    "    \"iso2\",\n",
    "    \"iso3\",\n",
    "    \"code3\",\n",
    "    \"FIPS\",\n",
    "    \"Admin2\",\n",
    "    \"Lat\",\n",
    "    \"Long_\",\n",
    "    \"Combined_Key\",\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "confirmed_us_df = drop_irrelevant_columns(confirmed_us_df, irrelevant_columns)\n",
    "deaths_us_df = drop_irrelevant_columns(deaths_us_df, irrelevant_columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "confirmed_us_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "deaths_us_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "only deaths data has the column `Population`, but it should the same for the confirmed data also."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "confirmed_us_df.insert(2, \"Population\", deaths_us_df[\"Population\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "confirmed_us_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2 - Normalize columns' names:\n",
    "throughout this notebook, we will be quering columns `Province/State`, `Country/Region` a lot! so it would be better to rename them so some thing like `State` and `Country` respectively."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rename_columns(df, columns_mapping):\n",
    "    new_df = df.rename(columns=columns_mapping)\n",
    "    return new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns_mapping = {\n",
    "    \"Province/State\": \"State\",\n",
    "    \"Province_State\": \"State\",\n",
    "    \"Country/Region\": \"Country\",\n",
    "    \"Country_Region\": \"Country\",\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# global data\n",
    "confirmed_global_df = rename_columns(confirmed_global_df, columns_mapping)\n",
    "deaths_global_df = rename_columns(deaths_global_df, columns_mapping)\n",
    "recovered_global_df = rename_columns(recovered_global_df, columns_mapping)\n",
    "\n",
    "# U.S. data\n",
    "confirmed_us_df = rename_columns(confirmed_us_df, columns_mapping)\n",
    "deaths_us_df = rename_columns(deaths_us_df, columns_mapping)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Global data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "confirmed_global_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "deaths_global_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "recovered_global_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### U.S. data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "confirmed_us_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "deaths_us_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3 - Drop `State` column from the global data:\n",
    "most records of the global data doesn't have `State` value, so it's better to drop it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "confirmed_global_df = drop_irrelevant_columns(confirmed_global_df, \"State\")\n",
    "deaths_global_df = drop_irrelevant_columns(deaths_global_df, \"State\")\n",
    "recovered_global_df = drop_irrelevant_columns(recovered_global_df, \"State\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "confirmed_global_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "deaths_global_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "recovered_global_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4 - Aggregate data:\n",
    "aggregate the number of confirmed cases, number of deaths, and number of reovered cases at each day for each country."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "aggregate global data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def apply_aggregation(df, by_column):\n",
    "    aggregate_df = df.groupby(by=by_column).sum().reset_index()\n",
    "    return aggregate_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "by_column = \"Country\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# global\n",
    "confirmed_global_agg_df = apply_aggregation(confirmed_global_df, by_column)\n",
    "deaths_global_agg_df = apply_aggregation(deaths_global_df, by_column)\n",
    "recovered_global_agg_df = apply_aggregation(recovered_global_df, by_column)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "confirmed_global_agg_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "deaths_global_agg_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "recovered_global_agg_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "aggregate U.S. data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "by_columns = [\"Country\", \"State\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "confirmed_us_agg_df = apply_aggregation(confirmed_us_df, by_columns)\n",
    "deaths_us_agg_df = apply_aggregation(deaths_us_df, by_columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "confirmed_us_agg_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "deaths_us_agg_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5 - create `Population` dataframe for the `U.S.` data:\n",
    "\n",
    "Later, we will convert the data to time series structure, and the value of `Population` shouldn't be in a time series data, as it doesn't change over time, it's rather a characteristic of the state."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "us_population_df = confirmed_us_agg_df[[\"Country\", \"State\", \"Population\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "confirmed_us_agg_df = drop_irrelevant_columns(confirmed_us_agg_df, [\"Population\"])\n",
    "deaths_us_agg_df = drop_irrelevant_columns(deaths_us_agg_df, [\"Population\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "confirmed_us_agg_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "deaths_us_agg_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6 - Restructe the Data:\n",
    "convert the data into an easier structure, add column `date`, and change the columns to rows (for easier manipulation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "date_columns = confirmed_global_agg_df.filter(\n",
    "    regex=\"\\d{1,2}/\\d{1,2}/\\d{1,4}\"\n",
    ").columns.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def construct_date_df(df, date_column_name, copy_columns, value_column_name):\n",
    "\n",
    "    data = {}\n",
    "\n",
    "    data[\"date\"] = date_column_name\n",
    "\n",
    "    for column in copy_columns:\n",
    "        data[column] = df[column]\n",
    "\n",
    "    data[value_column_name] = df[date_column_name]\n",
    "\n",
    "    date_df = pd.DataFrame(data=data)\n",
    "\n",
    "    return date_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# global data\n",
    "confirmed_global_date_frames = [\n",
    "    construct_date_df(confirmed_global_agg_df, date_column, [\"Country\"], \"confirmed\")\n",
    "    for date_column in date_columns\n",
    "]\n",
    "deaths_global_date_frames = [\n",
    "    construct_date_df(deaths_global_agg_df, date_column, [\"Country\"], \"deaths\")\n",
    "    for date_column in date_columns\n",
    "]\n",
    "recovered_global_date_frames = [\n",
    "    construct_date_df(recovered_global_agg_df, date_column, [\"Country\"], \"recovered\")\n",
    "    for date_column in date_columns\n",
    "]\n",
    "\n",
    "# U.S. data\n",
    "confirmed_us_date_frames = [\n",
    "    construct_date_df(\n",
    "        confirmed_us_agg_df, date_column, [\"Country\", \"State\"], \"confirmed\"\n",
    "    )\n",
    "    for date_column in date_columns\n",
    "]\n",
    "deaths_us_date_frames = [\n",
    "    construct_date_df(deaths_us_agg_df, date_column, [\"Country\", \"State\"], \"deaths\")\n",
    "    for date_column in date_columns\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# global data\n",
    "confirmed_global_time_series = pd.concat(confirmed_global_date_frames)\n",
    "deaths_global_time_series = pd.concat(deaths_global_date_frames)\n",
    "recovered_global_time_series = pd.concat(recovered_global_date_frames)\n",
    "\n",
    "# U.S. data\n",
    "confirmed_us_time_series = pd.concat(confirmed_us_date_frames)\n",
    "deaths_us_time_series = pd.concat(deaths_us_date_frames)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 7 - Convert `date` column:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_col_to_datetime(df, col_name):\n",
    "\n",
    "    df[col_name] = pd.to_datetime(df[col_name], utc=False)\n",
    "\n",
    "    df[col_name] = df[col_name].dt.strftime(\"%Y-%m-%d\")\n",
    "\n",
    "    df.sort_values(by=\"date\", inplace=True)\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# global data\n",
    "confirmed_global_time_series = convert_col_to_datetime(\n",
    "    confirmed_global_time_series, \"date\"\n",
    ")\n",
    "deaths_global_time_series = convert_col_to_datetime(deaths_global_time_series, \"date\")\n",
    "recovered_global_time_series = convert_col_to_datetime(\n",
    "    recovered_global_time_series, \"date\"\n",
    ")\n",
    "\n",
    "# U.S. data\n",
    "confirmed_us_time_series = convert_col_to_datetime(confirmed_us_time_series, \"date\")\n",
    "deaths_us_time_series = convert_col_to_datetime(deaths_us_time_series, \"date\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "confirmed_global_time_series.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "deaths_global_time_series.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "recovered_global_time_series.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Active cases: the number of active cases on a particular day is:\n",
    "\n",
    "    active = confirmed - (deaths + recovered)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "active_cases = confirmed_global_time_series[\"confirmed\"] - (\n",
    "    deaths_global_time_series[\"deaths\"] + recovered_global_time_series[\"recovered\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "active_global_time_series = pd.DataFrame(\n",
    "    data={\n",
    "        \"date\": confirmed_global_time_series.date,\n",
    "        \"Country\": confirmed_global_time_series.Country,\n",
    "        \"active\": active_cases,\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "active_global_time_series.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "confirmed_us_time_series.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "deaths_us_time_series.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "in the rest of the notebook, we will focus on the following five dataframes:\n",
    "- `confirmed_global_time_series`: time series of the global confirmed cases.\n",
    "- `deaths_global_time_series`: time series of the global deaths.\n",
    "- `recovered_global_time_series`: time series of the global recovered cases.\n",
    "- `confirmed_us_time_series`: time series of U.S. confirmed cases.\n",
    "- `deaths_us_time_series`: time series of U.S. deaths.\n",
    "\n",
    "A *time series* data shows the change in a *statiscal variable* according to *time*.\n",
    "\n",
    "\n",
    "Each row in the previous time series is a tuple of date, country, and measure, the measure represents the cumulative sum in the country, up until the date."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 8 - Focus on *Outbreaks*:\n",
    "We will focus in this notebook only on countries with high **death tolls**, countires that are nearly affected by the pandemic will be discarded."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "last_day = deaths_global_time_series[\"date\"].values[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "most_affected_10_countries = deaths_global_time_series[\n",
    "    deaths_global_time_series[\"date\"] == last_day\n",
    "].sort_values(by=\"deaths\", ascending=False)[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "most_affected_20_countries = deaths_global_time_series[\n",
    "    deaths_global_time_series[\"date\"] == last_day\n",
    "].sort_values(by=\"deaths\", ascending=False)[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "most_affected_10_countries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "most_affected_10_countries_names = most_affected_10_countries[\"Country\"].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "most_affected_20_countries_names = most_affected_20_countries[\"Country\"].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create time series for most affected countries:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "most_10_deaths_time_series = deaths_global_time_series[\n",
    "    deaths_global_time_series[\"Country\"].isin(most_affected_10_countries_names)\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "most_10_deaths_time_series.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "most_20_deaths_time_series = deaths_global_time_series[\n",
    "    deaths_global_time_series[\"Country\"].isin(most_affected_20_countries_names)\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "most_20_deaths_time_series.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "most_10_confirmed_time_series = confirmed_global_time_series[\n",
    "    confirmed_global_time_series[\"Country\"].isin(most_affected_10_countries_names)\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "most_10_confirmed_time_series.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "most_10_recovered_time_series = recovered_global_time_series[\n",
    "    recovered_global_time_series[\"Country\"].isin(most_affected_10_countries_names)\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "most_10_recovered_time_series.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "most_10_active_time_series = active_global_time_series[\n",
    "    active_global_time_series[\"Country\"].isin(most_affected_10_countries_names)\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "most_10_active_time_series.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Visualization:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "white_color_hex = \"#FFFFFF\"\n",
    "blue_color_hex = \"#04668C\"\n",
    "green_color_hex = \"#158A24\"\n",
    "orange_color_hex = \"#F77300\"\n",
    "coquelicot_color_hex = \"#EC4414\"\n",
    "red_color_hex = \"#E01427\"\n",
    "purple_color_hex = \"#613266\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Line Chart:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Deaths trajectory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.line(\n",
    "    data_frame=most_10_deaths_time_series,\n",
    "    x=\"date\",\n",
    "    y=\"deaths\",\n",
    "    color=\"Country\",\n",
    "    line_group=\"Country\",\n",
    ")\n",
    "\n",
    "fig.update_layout(\n",
    "    autosize=False,\n",
    "    width=1000,\n",
    "    height=600,\n",
    "    title_text=\"<i><b>Deaths</b></i>\",\n",
    "    xaxis_title=\"Date\",\n",
    "    yaxis_title=\"Death tolls\",\n",
    "    plot_bgcolor=\"rgba(0, 0, 0, 0)\",\n",
    "    paper_bgcolor=\"rgba(0, 0, 0, 0)\",\n",
    "    font={\"family\": \"Courier New, monospace\", \"size\": 14, \"color\": \"#eaeaea\"},\n",
    ")\n",
    "\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_fig_as_div(fig_obj=fig, file_name=\"death_tolls_line_chart.html\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Active cases trajectory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.line(\n",
    "    data_frame=most_10_active_time_series,\n",
    "    x=\"date\",\n",
    "    y=\"active\",\n",
    "    color=\"Country\",\n",
    "    line_group=\"Country\",\n",
    ")\n",
    "\n",
    "fig.update_layout(\n",
    "    autosize=False,\n",
    "    width=1000,\n",
    "    height=600,\n",
    "    title_text=\"<i><b>Active cases</b></i>\",\n",
    "    xaxis_title=\"Date\",\n",
    "    yaxis_title=\"Number of active cases\",\n",
    "    plot_bgcolor=\"rgba(0, 0, 0, 0)\",\n",
    "    paper_bgcolor=\"rgba(0, 0, 0, 0)\",\n",
    "    font={\"family\": \"Courier New, monospace\", \"size\": 14, \"color\": \"#eaeaea\"},\n",
    ")\n",
    "\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_fig_as_div(fig_obj=fig, file_name=\"active_cases_line_chart.html\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Recovery trajectory:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.line(\n",
    "    data_frame=most_10_recovered_time_series,\n",
    "    x=\"date\",\n",
    "    y=\"recovered\",\n",
    "    color=\"Country\",\n",
    "    line_group=\"Country\",\n",
    ")\n",
    "\n",
    "fig.update_layout(\n",
    "    autosize=False,\n",
    "    width=1000,\n",
    "    height=600,\n",
    "    title_text=\"<i><b>Recovered cases</b></i>\",\n",
    "    xaxis_title=\"Date\",\n",
    "    yaxis_title=\"Number of recovered cases\",\n",
    "    plot_bgcolor=\"rgba(0, 0, 0, 0)\",\n",
    "    paper_bgcolor=\"rgba(0, 0, 0, 0)\",\n",
    "    font={\"family\": \"Courier New, monospace\", \"size\": 14, \"color\": \"#eaeaea\"},\n",
    ")\n",
    "\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_fig_as_div(fig_obj=fig, file_name=\"recovered_cases_line_chart.html\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Global Choropleth Map:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.choropleth(\n",
    "    data_frame=deaths_global_time_series,\n",
    "    locations=\"Country\",\n",
    "    locationmode=\"country names\",\n",
    "    color=\"deaths\",\n",
    "    hover_name=\"deaths\",\n",
    "    animation_frame=deaths_global_time_series.date,\n",
    "    color_continuous_scale=[white_color_hex, red_color_hex],\n",
    ")\n",
    "\n",
    "# update figure layout\n",
    "fig.update_layout(\n",
    "    autosize=False,\n",
    "    width=1000,\n",
    "    height=600,\n",
    "    title_text=\"<i><b>Deaths over time</b></i>\",\n",
    "    xaxis_title=\"Measures\",\n",
    "    yaxis_title=\"Class\",\n",
    "    plot_bgcolor=\"rgba(0, 0, 0, 0)\",\n",
    "    paper_bgcolor=\"rgba(0, 0, 0, 0)\",\n",
    "    font={\"family\": \"Courier New, monospace\", \"size\": 14, \"color\": \"#eaeaea\"},\n",
    ")\n",
    "\n",
    "# disable mouse scroll zoom\n",
    "fig.show(config={\"scrollZoom\": False})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_fig_as_div(\n",
    "    fig_obj=fig,\n",
    "    file_name=\"global_deaths_choropleth_map.html\",\n",
    "    fig_config={\"scrollZoom\": False},\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## U.S. Choropleth:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dataset that contains U.S. states codes\n",
    "# Plotly only accept U.S. states as codes\n",
    "us_states_codes_df = pd.read_csv(\"other-data/us-states-codes.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "us_states_codes_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# join U.S. deaths dataset with U.S. codes dataset on `State` name\n",
    "temp_df = deaths_us_time_series.merge(\n",
    "    right=us_states_codes_df, left_on=\"State\", right_on=\"State\", how=\"left\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = px.choropleth(\n",
    "    data_frame=temp_df,\n",
    "    locations=\"Code\",\n",
    "    locationmode=\"USA-states\",\n",
    "    color=\"deaths\",\n",
    "    animation_frame=temp_df.date,\n",
    "    scope=\"usa\",\n",
    "    color_continuous_scale=[white_color_hex, red_color_hex],\n",
    "    hover_data=[\"State\"],\n",
    ")\n",
    "\n",
    "# update figure layout\n",
    "fig.update_layout(\n",
    "    autosize=False,\n",
    "    width=1000,\n",
    "    height=600,\n",
    "    title_text=\"<i><b>U.S. Deaths over time</b></i>\",\n",
    "    plot_bgcolor=\"rgba(0, 0, 0, 0)\",\n",
    "    paper_bgcolor=\"rgba(0, 0, 0, 0)\",\n",
    "    font={\"family\": \"Courier New, monospace\", \"size\": 14, \"color\": \"#eaeaea\"},\n",
    ")\n",
    "\n",
    "# disable mouse scroll zoom\n",
    "fig.show(config={\"scrollZoom\": False})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_fig_as_div(\n",
    "    fig_obj=fig,\n",
    "    file_name=\"us_deaths_choropleth_map.html\",\n",
    "    fig_config={\"scrollZoom\": False},\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Daily deaths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "most_affected_13_countries = deaths_global_time_series[\n",
    "    deaths_global_time_series[\"date\"] == last_day\n",
    "].sort_values(by=\"deaths\", ascending=False)[:13]\n",
    "\n",
    "most_affected_13_countries_names = most_affected_13_countries[\"Country\"].values\n",
    "\n",
    "most_13_deaths_time_series = deaths_global_time_series[\n",
    "    deaths_global_time_series[\"Country\"].isin(most_affected_13_countries_names)\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_daily_deaths(deaths_columns):\n",
    "    \"\"\"\n",
    "    this function calculates the daily deaths from the time series of accumulated deaths\n",
    "    it takes the accumulated deaths array, and returns the original array which corresponds to deaths at each day\n",
    "    \"\"\"\n",
    "    number_of_days = len(deaths_columns)\n",
    "    daily_deaths = deaths_columns[0]\n",
    "    # to calculate the original array subtract deaths[0..yesterday] from deaths[1..today]\n",
    "    diff = np.absolute(\n",
    "        deaths_columns[1:number_of_days] - deaths_columns[0 : number_of_days - 1]\n",
    "    )\n",
    "    daily_deaths = np.append(daily_deaths, diff)\n",
    "    return daily_deaths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_daily_deaths_df(df, country_name):\n",
    "    \"\"\"\n",
    "    this function constructs a new Series called `daily deaths` for the passed dataframe\n",
    "    the `daily deaths` Series represents the daily deaths values at each day\n",
    "    \"\"\"\n",
    "    new_df = df[df[\"Country\"] == country_name]\n",
    "    country_daily_deaths = get_daily_deaths(new_df[\"deaths\"].values)\n",
    "\n",
    "    # add new column for daily deaths\n",
    "    new_df = new_df.assign(daily_deaths=country_daily_deaths)\n",
    "    return new_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Add daily deaths column for most affected countries:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "countries_daily_deaths = pd.concat(\n",
    "    [\n",
    "        get_daily_deaths_df(most_13_deaths_time_series, country_name)\n",
    "        for country_name in most_affected_13_countries_names\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "countries_daily_deaths.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create figure object\n",
    "fig = go.Figure()\n",
    "\n",
    "# list of drop-down menu buttons\n",
    "buttons_list = []\n",
    "\n",
    "# number of countries\n",
    "number_of_countries = len(countries_daily_deaths.Country.unique())\n",
    "\n",
    "# add bar chart for each country\n",
    "for idx, country_name in enumerate(countries_daily_deaths.Country.unique()):\n",
    "\n",
    "    # get country daily deaths\n",
    "    country_daily_deaths_df = countries_daily_deaths[\n",
    "        countries_daily_deaths.Country == country_name\n",
    "    ]\n",
    "\n",
    "    bar_visibility = None if idx == 0 else \"legendonly\"\n",
    "\n",
    "    # add bar chart for the figure\n",
    "    fig.add_trace(\n",
    "        go.Bar(\n",
    "            x=country_daily_deaths_df.date,\n",
    "            y=country_daily_deaths_df.daily_deaths,\n",
    "            name=country_name,\n",
    "            marker_color=red_color_hex,\n",
    "            visible=bar_visibility,\n",
    "        )\n",
    "    )\n",
    "\n",
    "    button_visibility_settings = [False] * number_of_countries\n",
    "    button_visibility_settings[idx] = True\n",
    "\n",
    "    button = dict(\n",
    "        label=country_name,\n",
    "        method=\"update\",\n",
    "        args=[\n",
    "            {\"visible\": button_visibility_settings},\n",
    "            {\"title\": country_name, \"showlegend\": True},\n",
    "        ],\n",
    "    )\n",
    "    # add corresponding drop-down button for this chart\n",
    "    buttons_list.append(button)\n",
    "\n",
    "# update figure layout to set menu settings\n",
    "fig.update_layout(\n",
    "    updatemenus=[\n",
    "        go.layout.Updatemenu(\n",
    "            active=0,\n",
    "            bgcolor=\"#AAAAAA\",\n",
    "            bordercolor=\"#FFFFFF\",\n",
    "            font=dict(size=14, color=\"#000000\", family=\"Courier New, monospace\"),\n",
    "            buttons=buttons_list,\n",
    "            showactive=True,\n",
    "        )\n",
    "    ],\n",
    "    autosize=False,\n",
    "    width=1000,\n",
    "    height=600,\n",
    "    title_text=\"<i><b>Daily deaths</b></i>\",\n",
    "    xaxis_title=\"Date\",\n",
    "    yaxis_title=\"Number of deaths\",\n",
    "    plot_bgcolor=\"rgba(0, 0, 0, 0)\",\n",
    "    paper_bgcolor=\"rgba(0, 0, 0, 0)\",\n",
    "    font={\"family\": \"Courier New, monospace\", \"size\": 14, \"color\": \"#eaeaea\"},\n",
    ")\n",
    "\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_fig_as_div(\n",
    "    fig_obj=fig, file_name=\"most_affected_countries_daily_deaths_bar_chart.html\"\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}