Common Agenda Analysis Framework.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import csv\n",
    "\n",
    "def read_csv(csv_raw_clock_file_path: str, data_types: dict, separator: str, quote_column: str) -> pd.DataFrame:\n",
    "    \"\"\"Reads the csv file and returns a pandas DataFrame. \n",
    "    The csv file should have the following columns:\n",
    "    - date: the date of the clock-in\n",
    "    - time: the time of the clock-in\"\"\"\n",
    "    dframe = pd.read_csv(csv_raw_clock_file_path, dtype=data_types, sep=separator, quoting=csv.QUOTE_NONE, quotechar='\"', converters={quote_column: lambda x: x})\n",
    "    dframe[quote_column] = dframe[quote_column].str.findall(r'\"([^\"]*)\"')    \n",
    "    return dframe\n",
    "\n",
    "df = read_csv(\"...\", {\"tzone\": str}, \",\", \"outline\")\n",
    "df.name = \"Complete Clocks\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "def filter_tag(tags: str, excluded_tags_re: list) -> frozenset:\n",
    "    \"\"\"Splits a string of tags separated by \":\" into a set of tags, \n",
    "    removing the one that match any of the regexp patterns in the useless_tags_re list.\n",
    "    Returns the result as a frozenset.\"\"\"\n",
    "    set_tags = {i for i in tags.split(':') if i}\n",
    "    # skips the set of tags if it contains any of the useless tags \n",
    "    set_tags = {i for i in set_tags if not any([re.match(f, i) for f in excluded_tags_re])}\n",
    "    return frozenset(set_tags)\n",
    "\n",
    "def add_frozen_tag_column(dframe: pd.DataFrame, single_string_tags_column: str =\"tags\", new_tags_column: str=\"frozen_tags\", excluded_tags_re: list=[\"u\", \"e\", \"@[A-z]\", \"travel\", \"nil\"], inPlace=False) -> pd.DataFrame:\n",
    "    \"\"\"Adds a column to the DataFrame containing the tags as a frozenset.\n",
    "    Expects dframe[single_string_tags_column] to have strings of tags separated by \":\" (where \":\" is at the beginning and end).\n",
    "    Adds a new column with the tags as a frozenset, removing the tags that match any of the regexp patterns in the excluded_tags_re list.\n",
    "    Calls the function filter_tag using apply on the tags column.\n",
    "    Returns the DataFrame with the new column only if inPlace is False.\"\"\"\n",
    "    if not inPlace:\n",
    "        dframe_copy = dframe.copy()\n",
    "        dframe_copy[new_tags_column] = dframe_copy[single_string_tags_column].apply(lambda x: filter_tag(x, excluded_tags_re))\n",
    "        return dframe_copy\n",
    "    \n",
    "    dframe[new_tags_column] = dframe[single_string_tags_column].apply(lambda x: filter_tag(x, excluded_tags_re))\n",
    "    \n",
    "df = add_frozen_tag_column(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"frozen_tags_outline\"] = df[\"outline\"].apply(lambda x: tuple(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_timestamp_string(timestamp: str, format=\"%Y-%m-%d %H:%M %z\") -> pd.Timestamp:\n",
    "    \"\"\"Parses the timestamp using the format, then converts it and returns a pd.Timestamp object with uct time.\"\"\"\n",
    "    return pd.to_datetime(timestamp, format=format, utc=True)\n",
    "\n",
    "def datetime_to_minute_of_day(series: pd.Series, offset:int=0) -> pd.Series:\n",
    "    \"\"\"Converts the datetime series to the minute of the day.\n",
    "    Adds an offset to the result and returns the result modulo 24*60.\"\"\"\n",
    "    return series.dt.time.apply(lambda x: (x.hour*60 + x.minute + offset)%(24*60)) \n",
    "\n",
    "def datetime_to_day_of_week(series: pd.Series) -> pd.Series:\n",
    "    \"\"\"Converts the datetime series to the day of the week.\"\"\"\n",
    "    return series.dt.dayofweek\n",
    "\n",
    "def datetime_to_week_of_year(series: pd.Series) -> pd.Series:\n",
    "    \"\"\"Converts the datetime series to the week of the year.\"\"\"\n",
    "    return series.dt.isocalendar().week\n",
    "\n",
    "def datetime_to_month(series: pd.Series) -> pd.Series:\n",
    "    \"\"\"Converts the datetime series to the month.\"\"\"\n",
    "    return series.dt.month\n",
    "\n",
    "def datetime_to_year(series: pd.Series) -> pd.Series:\n",
    "    \"\"\"Converts the datetime series to the year.\"\"\"\n",
    "    return series.dt.year\n",
    "\n",
    "from datetime import timedelta\n",
    "\n",
    "def add_start_and_end_timestamp_columns(dframe: pd.DataFrame, ignore_tzone: bool=True, inPlace=True) -> pd.DataFrame:\n",
    "    \"\"\"Adds two columns to the DataFrame containing the start and end timestamps:\n",
    "    - start: the start timestamp, obtained from dframe[\"date\"] + \" \" + dframe[\"start\"] + \" \" + dframe[\"tzone\"]\n",
    "    - end: the end timestamp, obtained from start + dframe[\"duration\"]\n",
    "    Returns the DataFrame with the new column only if inPlace is False.\n",
    "    - begin_datetime: the start timestamp as a pd.Timestamp object\n",
    "    - duration_timedelta: the duration as a pd.Timedelta object\n",
    "    - end_datetime: the end timestamp as a pd.Timestamp object\n",
    "    - ignore_tzone: if True, ignores the timezone column\"\"\"\n",
    "    if not inPlace:\n",
    "        dframe = dframe.copy()\n",
    "    \n",
    "    if ignore_tzone:\n",
    "        dframe[\"begin_datetime\"] = parse_timestamp_string(df[\"date\"] + \" \" + df[\"start\"], format=\"%Y-%m-%d %H:%M\")\n",
    "    else:\n",
    "        dframe[\"begin_datetime\"] = parse_timestamp_string(df[\"date\"] + \" \" + df[\"start\"] + \" \" + df[\"tzone\"])\n",
    "        \n",
    "    dframe[\"duration_timedelta\"] = pd.to_timedelta(df[\"duration\"], unit=\"m\")\n",
    "    dframe[\"end_datetime\"] = dframe[\"begin_datetime\"] + dframe[\"duration_timedelta\"]\n",
    "        \n",
    "    if not inPlace:\n",
    "        return dframe\n",
    "\n",
    "add_start_and_end_timestamp_columns(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_diff_outliers_description(cl: pd.Series) -> str:\n",
    "    \"\"\"Returns a string with the mean and the standard deviation of the differences in sorted values.\"\"\"\n",
    "    diff = cl.sort_values().diff()\n",
    "    return f\"Mean: {diff.mean()}, Std: {diff.std()}\"\n",
    "\n",
    "def extract_outliers_mask(cl: pd.Series, threshold_sd: int) -> pd.Series:\n",
    "    \"\"\"Extracts the outliers of the Series.\n",
    "    Returns a DataFrame with the outliers.\"\"\"\n",
    "    return (cl - cl.mean()).abs() > threshold_sd * cl.std()    \n",
    "\n",
    "def extract_outliers_by_group_mask(dframe: pd.DataFrame, group_column: str, value_column: str, threshold_sd: int) -> pd.Series:\n",
    "    \"\"\"Extracts the outliers of the DataFrame by group.\n",
    "    Returns a DataFrame with the outliers.\"\"\"\n",
    "    return dframe.groupby(group_column)[value_column].transform(lambda x: (x - x.mean()).abs() > threshold_sd * x.std())\n",
    "\n",
    "outliers_mask = extract_outliers_by_group_mask(df, \"tags\", \"duration_timedelta\", 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[outliers_mask].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_problematic_clocks_mask(dframe: pd.DataFrame) -> pd.DataFrame:\n",
    "    \"\"\"Returns the DataFrame, restricted to the rows where the end_datetime is before the begin_datetime.\"\"\"\n",
    "    negative_duration_mask = dframe[\"end_datetime\"] < dframe[\"begin_datetime\"]\n",
    "    zero_duration_mask = dframe[\"end_datetime\"] == dframe[\"begin_datetime\"]\n",
    "    \n",
    "    problematic = pd.DataFrame()\n",
    "    \n",
    "    # append one column per mask, with the value equal to that column's mask\n",
    "    problematic[\"negative_duration_mask\"] = negative_duration_mask\n",
    "    problematic[\"zero_duration_mask\"] = zero_duration_mask\n",
    "    \n",
    "    # check outliers beyond 3 sd, using groupby for the outline column\n",
    "    problematic[\"outliers\"] = extract_outliers_by_group_mask(df, \"tags\", \"duration_timedelta\", 4)\n",
    "    \n",
    "    return problematic\n",
    "\n",
    "problematic_indices = extract_problematic_clocks_mask(df)\n",
    "\n",
    "def print_problematic_indices_statistics(dframe: pd.DataFrame, problematic_indices: pd.DataFrame) -> None:\n",
    "    \"\"\"Prints the number of problematic indices and the number of problematic indices per column.\n",
    "    Obtains the problematic indices using extract_problematic_clocks(dframe).\"\"\"\n",
    "        \n",
    "    # count the rows with any true value in the columns\n",
    "    excluded = problematic_indices.any(axis=1)\n",
    "    total_excluded = excluded.sum()\n",
    "    \n",
    "    column_justification = 25\n",
    "    value_justification = 5\n",
    "    \n",
    "    print(f\"Problematic from {len(dframe)}\".rjust(column_justification) + \":  \" + f\"{total_excluded}, \".rjust(value_justification) + f\"{total_excluded/len(df):.2%}\")\n",
    "    for col in problematic_indices.columns:\n",
    "        # padding number of problematic indices with spaces\n",
    "        print(f\"{col}\".rjust(column_justification) + \":\" + f\"{problematic_indices[col].sum()}\".rjust(value_justification) + f\", {problematic_indices[col].mean():.2%}\")\n",
    "\n",
    "print_problematic_indices_statistics(df, problematic_indices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_problematic_clocks(dframe: pd.DataFrame, problematic_indices: pd.DataFrame, problematic_column: str) -> pd.DataFrame:\n",
    "    \"\"\"Returns the DataFrame, restricted to the rows matching one of the problematic_columns.\"\"\"\n",
    "    return dframe.loc[problematic_indices[problematic_column]]\n",
    "\n",
    "problematic_clocks = extract_problematic_clocks(df, problematic_indices, \"zero_duration_mask\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_problematic_indices(dframe: pd.DataFrame, inPlace=True) -> pd.DataFrame:\n",
    "    \"\"\"Removes the problematic indices from the DataFrame, following the rules in extract_problematic_clocks.\n",
    "    Returns the DataFrame with the problematic indices removed, only if inPlace is False.\n",
    "    Otherwise modifies the DataFrame in place.\"\"\"\n",
    "    \n",
    "    # get the mask of the rows that are not problematic in any of the columns\n",
    "    mask = ~extract_problematic_clocks_mask(dframe).any(axis=1)\n",
    "    if inPlace:\n",
    "        dframe.drop(index=dframe.index[~mask], inplace=True)\n",
    "    else:\n",
    "        dframe_extract = dframe[mask]\n",
    "        return dframe_extract\n",
    "\n",
    "remove_problematic_indices(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "        \n",
    "def plot_histogram(series: pd.Series, bins: int=100, disable_xticks=False, title:str = \"\") -> plt.Figure:\n",
    "    \"\"\"Plots the histogram of the duration of the events.\n",
    "    - If the series is of type timedelta64[ns], it converts it to nearest minute, rounded down, before plotting.\n",
    "    - If the series is of type float64 or int64, it plots the histogram.\n",
    "    Otherwise, it plots the value counts of the series.\n",
    "    The number of bins can be set using the bins parameter.\n",
    "    If disable_xticks is True, it removes the xticks. This is useful when only the shape of the histogram is needed.\n",
    "    The name of the series is used as the xticks if xticks are enabled.\"\"\"\n",
    "    \n",
    "    fig, ax = plt.subplots()\n",
    "\n",
    "    if series.dtype == pd.Timedelta:\n",
    "        (series.dt.total_seconds()/60).plot.hist(bins=bins, edgecolor=\"black\", density=True, ax=ax)\n",
    "    elif series.dtype == \"float64\" or series.dtype == \"int64\":\n",
    "        series.plot.hist(bins=bins, edgecolor=\"black\", density=True, ax=ax)\n",
    "    else:  # plot value counts\n",
    "        if disable_xticks:\n",
    "            series.value_counts().plot(kind=\"bar\", edgecolor=\"black\", ylabel=\"Occurrences\", ax=ax)\n",
    "            ax.set_xticks([])\n",
    "        else:\n",
    "            series.value_counts().plot(kind=\"bar\", edgecolor=\"black\", ylabel=\"Occurrences\", ax=ax)\n",
    "\n",
    "    if title:\n",
    "        ax.set_title(title)\n",
    "\n",
    "    plt.close(fig)\n",
    "    return fig\n",
    "    \n",
    "plot_histogram(df[\"duration\"], title=\"Duration histogram\", bins=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_formatted_time(df: pd.DataFrame, time_column: str, format: str, inPlace=False) -> pd.DataFrame:\n",
    "    \"\"\"Returns the DataFrame with the time column formatted using the format string.\"\"\"\n",
    "    if not inPlace:\n",
    "        dframe = df.copy()\n",
    "        dframe[time_column] = pd.to_datetime(dframe[time_column], format=format)\n",
    "        return dframe\n",
    "    df[time_column] = pd.to_datetime(df[time_column], format=format)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# outlines are list of nested (sub)headings, used to specify the hierarchy of the log\n",
    "\n",
    "def get_exact_outline_mask(df: pd.DataFrame, outline: list) -> pd.DataFrame:\n",
    "    \"\"\"Returns a mask for the DataFrame where the outline column equals the given outline.\n",
    "    - df: the DataFrame\n",
    "    - outline: the outline to check for, given as a list\"\"\"\n",
    "    return df['outline'].apply(lambda x: x == outline)\n",
    "\n",
    "def get_any_outline_mask(df: pd.DataFrame, outline: str) -> pd.DataFrame:\n",
    "    \"\"\"Returns a mask for the DataFrame where the outline column contains the given outline.\n",
    "    - df: the DataFrame\n",
    "    - outline: the outline to check for, given as a string\"\"\"\n",
    "    return df['outline'].apply(lambda x: outline in x)\n",
    "\n",
    "def get_index_outline(df: pd.DataFrame, outline: str, index: int) -> pd.DataFrame:\n",
    "    \"\"\"Returns a mask for the DataFrame where the outline column contains the given outline at the given index.\n",
    "    - df: the DataFrame\n",
    "    - outline: the outline to check for, given as a string\n",
    "    - index: the index of the outline to check for, which can be negative, referring to the element from last\"\"\"\n",
    "    return df['outline'].apply(lambda x: x[index] == outline if (0 <= index and index < len(x)) or (-index <= len(x) and index < 0) else False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_subset_match_tags_mask(cl: pd.Series, tags) -> pd.Series:\n",
    "    \"\"\"Returns the DataFrame with only the rows where the tags match the mask.\"\"\"\n",
    "    if isinstance(tags, str):\n",
    "        tags = {tags}\n",
    "    elif isinstance(tags, list):\n",
    "        tags = set(tags)\n",
    "    return cl.apply(lambda x: tags.issubset(x))\n",
    "\n",
    "def remove_tags_from_column(cl: pd.Series, tags: set) -> pd.DataFrame:\n",
    "    \"\"\"Returns the Series with the tags removed.\"\"\"\n",
    "    if isinstance(tags, str):\n",
    "        tags = {tags}\n",
    "    elif isinstance(tags, list):\n",
    "        tags = set(tags)\n",
    "    return cl.apply(lambda x: x - tags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO automate recognising periods of time\n",
    "\n",
    "# university lessons period\n",
    "uni = [(\"2022-09-06\", \"2022-10-15\"), \n",
    "        (\"2022-10-26\", \"2022-12-07\"),\n",
    "        (\"2023-02-06\", \"2023-03-11\"), \n",
    "        (\"2023-03-21\", \"2023-04-08\"), # easter break\n",
    "        (\"2023-04-17\", \"2023-06-13\"),\n",
    "        \n",
    "        (\"2023-09-06\", \"2023-10-14\"),\n",
    "        (\"2023-11-03\", \"2023-12-07\"),\n",
    "        (\"2024-02-05\", \"2024-03-09\"), \n",
    "        (\"2024-03-20\", \"2024-03-29\"), # easter break\n",
    "        (\"2024-04-11\", \"2024-05-11\")] \n",
    "\n",
    "# university exam session\n",
    "exa = [(\"2022-10-15\", \"2022-10-26\"), \n",
    "        (\"2022-12-07\", \"2022-12-21\"), \n",
    "        (\"2023-01-02\", \"2023-01-18\"), \n",
    "        \n",
    "        (\"2023-03-11\", \"2023-03-21\"), \n",
    "        (\"2023-06-13\", \"2023-06-19\"), \n",
    "        \n",
    "        (\"2023-10-14\", \"2023-10-31\"),\n",
    "        (\"2023-12-07\", \"2023-12-21\"),\n",
    "        (\"2023-12-27\", \"2024-01-30\"),\n",
    "        (\"2024-03-09\", \"2023-03-20\"),\n",
    "        (\"2024-05-11\", \"2024-06-14\"),\n",
    "        (\"2024-06-23\", \"2024-07-10\")]\n",
    "\n",
    "\n",
    "def get_period_mask(dframe: pd.DataFrame, periods: list, start_column: str, end_column: str) -> pd.DataFrame:\n",
    "    \"\"\"Returns the DataFrame with the mask of the periods.\"\"\"\n",
    "    mask = pd.Series(False, index=dframe.index)\n",
    "    for start, end in periods:\n",
    "        mask |= (dframe[start_column] >= start) & (dframe[end_column] < end)\n",
    "    return mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# shortcuts in simplifying the summary table\n",
    "# note that the tag values should be exclusive to avoid double counting\n",
    "tag_tree = {\n",
    "    \"standard\": {\n",
    "        \"Sleep\": [\"SWO\", \"SFR\"],\n",
    "        \"Lessons\": [\"LES\"],\n",
    "        \"Revision\": [\"REV\", \"EXM\"],\n",
    "        \"Repetitive\": [\"BUR\", \"WRK\", \"TDY\", \"ORG\", \"REP\"],\n",
    "        \"Projects\": [\"PRJ\"],\n",
    "        \"Media\": [\"MDI\"],\n",
    "        \"Social\": [\"CAL\", \"OUT\", \"EVE\", \"DOG\"],\n",
    "    },\n",
    "    \"censored\": {\n",
    "        \"Sleep\": [\"SWO\", \"SFR\"],\n",
    "        \"Lessons\": [\"LES\"],\n",
    "        \"Revision\": [\"REV\", \"EXM\"],\n",
    "        \"Repetitive\": [\"BUR\", \"WRK\", \"TDY\", \"ORG\", \"REP\"],\n",
    "        \"Projects\": [\"PRJ\"],\n",
    "        \"Media\": [\"MDI\"],\n",
    "        \"Social\": [\"CAL\", \"OUT\", \"EVE\", \"DOG\"],\n",
    "    },\n",
    "    \"uncensored\": {\n",
    "        \"Sleep\": [\"SWO\", \"SFR\"],\n",
    "        \"Lessons\": [\"LES\"],\n",
    "        \"Revision\": [\"REV\", \"EXM\"],\n",
    "        \"Repetitive\": [\"BUR\", \"WRK\", \"TDY\", \"ORG\"],\n",
    "        \"Projects\": [\"PRJ\"],\n",
    "        \"Media\": [\"MDI\"],\n",
    "        \"Social\": [\"CAL\", \"OUT\", \"EVE\", \"DOG\"],\n",
    "    },\n",
    "    \"study\": {\n",
    "        \"Theory\": [\"R\"],\n",
    "        \"Exercise\": [\"E\"],\n",
    "        \"Projects\": [\"P\"],\n",
    "        \"Exams\": [\"EXM\"],\n",
    "        \"Lessons\": [\"LES\"],\n",
    "    },\n",
    "    \"extended\": {\n",
    "        \"Sleep\": [\"SWO\", \"SFR\"],\n",
    "        \"Lessons\": [\"LES\"],\n",
    "        \"Revision\": [\"REV\"],\n",
    "        \"Exams\": [\"EXM\"],\n",
    "        \"Repetitive\": [\"REP\"],\n",
    "        \"Bureaucracy\": [\"BUR\"],\n",
    "        \"Work\": [\"WRK\"],\n",
    "        \"Tidying\": [\"TDY\"],\n",
    "        \"Organization\": [\"ORG\"],\n",
    "        \"Projects\": [\"PRJ\"],\n",
    "        \"Media\": [\"MDI\"],\n",
    "        \"Times\": [\"TM\"],\n",
    "        \"Calls\": [\"CAL\"],\n",
    "        \"Going Out\": [\"OUT\", \"EVE\", \"DOG\"],\n",
    "    },\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# use to merge the tags into a smaller set of tags in events\n",
    "def merge_tags(rule: dict, fset: frozenset) -> frozenset:\n",
    "    \"\"\"For each original tag, appends to a set the new tag associated with it.\n",
    "    - rule: entry of the tag_tree, associates the new value of a tag to a list of original tags\"\"\"\n",
    "    new_tag_description = set()\n",
    "    for k, v in rule.items():\n",
    "        for tag in v:\n",
    "            if tag in fset:\n",
    "                new_tag_description.add(k)\n",
    "    return frozenset(new_tag_description)\n",
    "\n",
    "def merge_tags_column(dframe: pd.DataFrame, tags_matching: dict) -> pd.Series:\n",
    "    \"\"\"Merges the tags in the DataFrame using the tag_tree.\n",
    "    - df: the DataFrame, with a column \"frozen_tags\" containing the tags as a frozenset\n",
    "    - tag_tree: the dictionary containing the tag_tree\"\"\"\n",
    "    return dframe[\"frozen_tags\"].apply(lambda x: merge_tags(tags_matching, x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "def plot_2D_histogram(series_x: pd.Series, series_y: pd.Series, bins: int=100, formatter_x: callable=None, formatter_y: callable=None, title: str=\"\") -> plt.Figure:\n",
    "    \"\"\"Plots the 2D histogram of the duration of the events.\n",
    "    - If the series is of type timedelta64[ns], it converts it to nearest minute, rounded down, before plotting.\n",
    "    - If the series is of type float64 or int64, it plots the histogram.\n",
    "    Otherwise, it plots the value counts of the series.\"\"\"\n",
    "    \n",
    "    le = LabelEncoder()\n",
    "    fig, ax = plt.subplots()\n",
    "    \n",
    "    if series_x.dtype == \"timedelta64[ns]\":\n",
    "        x = (series_x.dt.total_seconds()/60)\n",
    "    elif series_x.dtype == \"float64\" or series_x.dtype == \"int64\":\n",
    "        x = series_x\n",
    "    else: # plot value counts\n",
    "        x = le.fit_transform(series_x)\n",
    "    \n",
    "    if series_y.dtype == \"timedelta64[ns]\":\n",
    "        y = (series_y.dt.total_seconds()/60)\n",
    "    elif series_y.dtype == \"float64\" or series_y.dtype == \"int64\":\n",
    "        y = series_y\n",
    "    else: # plot value counts\n",
    "        y = le.fit_transform(series_y)\n",
    "        # set y ticks to go from min to max of the y variable in 15 steps\n",
    "        # associate the labels with the values\n",
    "        ax.set_yticks(range(len(le.classes_)))\n",
    "        ax.set_yticklabels(le.classes_)\n",
    "        \n",
    "    ax.hist2d(x, y, bins=bins, cmap=\"Blues\")\n",
    "    \n",
    "    title = title if title else f\"2D histogram of {series_x.name} and {series_y.name}\"\n",
    "    ax.set_title(title)\n",
    "    ax.set_xlabel(series_x.name)\n",
    "    ax.set_ylabel(series_y.name)\n",
    "\n",
    "    if formatter_x:\n",
    "        ax.xaxis.set_major_formatter(plt.FuncFormatter(formatter_x))\n",
    "        \n",
    "    if formatter_y:\n",
    "        ax.yaxis.set_major_formatter(plt.FuncFormatter(formatter_y))\n",
    "        \n",
    "    # color the 0 value white\n",
    "    ax.set_facecolor(\"white\")\n",
    "    \n",
    "    plt.close(fig)\n",
    "    return fig\n",
    "\n",
    "def minute_to_string_formatter(x: int, pos) -> str:\n",
    "    \"\"\"Converts the minute to a string in the format HH:MM.\"\"\"\n",
    "    return f\"{int((x%(24*60))/60):02d}:{int(x%60):02d}\"\n",
    "\n",
    "def minute_to_string_offset_formatter(offset: int) -> str:\n",
    "    \"\"\"Returns a function that converts the minute to a string in the format HH:MM, with an offset.\n",
    "    - offset: the offset to add to the time, in minutes\"\"\"\n",
    "    return lambda x, pos: minute_to_string_formatter(x-offset, pos)\n",
    "   \n",
    "# select some events from the whole dataset\n",
    "selection = df[get_subset_match_tags_mask(df[\"frozen_tags\"], \"SLP\")]\n",
    "\n",
    "# hour of the day when the graph should start *60\n",
    "offset = 6*60\n",
    "plot_2D_histogram(datetime_to_minute_of_day(selection[\"begin_datetime\"], 24*60 - offset), selection[\"duration\"], formatter_x=minute_to_string_offset_formatter(24*60 - offset), formatter_y=minute_to_string_offset_formatter(0), bins=20, title=\"Sleep duration by time of day\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cast_to_list_str(df: pd.DataFrame) -> pd.DataFrame:\n",
    "    return df.apply(lambda x: str(list(x)))\n",
    "\n",
    "plot_2D_histogram(selection[\"duration\"], cast_to_list_str(selection[\"frozen_tags\"]), bins=15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_2D_scatterplot(series_x: pd.Series, series_y: pd.Series, type_identifiers: pd.Series, formatter_x: callable=None, formatter_y: callable=None) -> plt.Figure:\n",
    "    \"\"\"Plots the 2D scatterplot of the duration of the events.\n",
    "    - If the series is of type timedelta64[ns], it converts it to nearest minute, rounded down, before plotting.\n",
    "    - If the series is of type float64 or int64, it plots the histogram.\n",
    "    - type_identifiers: the series containing the type of the event, used to color the points:\n",
    "        - if the series is of type timedelta64[ns], it converts it to nearest minute, rounded down, before plotting.\n",
    "        - if the series is of type float64 or int64, it plots the histogram.\n",
    "        - if the series is of type object, it uses the LabelEncoder to transform it to number.\n",
    "    Otherwise, it plots the value counts of the series.\n",
    "    \"\"\"\n",
    "    \n",
    "    le = LabelEncoder()\n",
    "    fig, ax = plt.subplots()\n",
    "    \n",
    "    if series_x.dtype == \"timedelta64[ns]\":\n",
    "        x = (series_x.dt.total_seconds()/60)\n",
    "    elif series_x.dtype == \"float64\" or series_x.dtype == \"int64\":\n",
    "        x = series_x\n",
    "    else: # plot value counts\n",
    "        x = le.fit_transform(series_x)\n",
    "    \n",
    "    if series_y.dtype == \"timedelta64[ns]\":\n",
    "        y = (series_y.dt.total_seconds()/60)\n",
    "    elif series_y.dtype == \"float64\" or series_y.dtype == \"int64\":\n",
    "        y = series_y\n",
    "    else: # plot value counts\n",
    "        y = le.fit_transform(series_y)\n",
    "        # set y ticks to go from min to max of the y variable in 15 steps\n",
    "        # associate the labels with the values\n",
    "        ax.set_yticks(range(len(le.classes_)))\n",
    "        ax.set_yticklabels(le.classes_)\n",
    "\n",
    "    # ensures the colors don't repeat\n",
    "    scatter = ax.scatter(x, y, c=le.fit_transform(type_identifiers.apply(lambda x: str(x))), cmap=\"tab20\", s=20)\n",
    "    ax.set_facecolor(\"white\")\n",
    "\n",
    "    handles, _ = scatter.legend_elements(prop=\"colors\", num=len(le.classes_))\n",
    "    labels = le.classes_\n",
    "\n",
    "    # allows the legend to grow vertically as much as needed to accomodate all lables\n",
    "    plt.legend(handles, labels, title=\"Type\", bbox_to_anchor=(1.05, 1), loc='upper left', fontsize='small')\n",
    "    # ensure the whole legend is displayed\n",
    "    plt.subplots_adjust(right=0.7)\n",
    "    \n",
    "    ax.set_title(f\"2D scatterplot of {series_x.name} and {series_y.name}\")\n",
    "    ax.set_xlabel(series_x.name)\n",
    "    ax.set_ylabel(series_y.name)\n",
    "\n",
    "    if formatter_x:\n",
    "        ax.xaxis.set_major_formatter(plt.FuncFormatter(formatter_x))\n",
    "        \n",
    "    if formatter_y:\n",
    "        ax.yaxis.set_major_formatter(plt.FuncFormatter(formatter_y))\n",
    "        \n",
    "    plt.close(fig)\n",
    "    return fig\n",
    "\n",
    "selection = df[get_any_outline_mask(df, \"Series\") & get_subset_match_tags_mask(df[\"frozen_tags\"], \"2024\")]\n",
    "dropped_tags_column = selection[\"frozen_tags\"]\n",
    "selection_restricted_tags_column = merge_tags_column(selection, tag_tree[\"standard\"])\n",
    "\n",
    "plot_2D_scatterplot(datetime_to_minute_of_day(selection[\"begin_datetime\"], offset), selection[\"duration\"], selection[\"outline\"], formatter_x=minute_to_string_offset_formatter(offset), formatter_y=minute_to_string_offset_formatter(0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_histogram_deltas(cl: pd.Series, exclude_outliers_sdev:int = 1000, bins: int=100, title: str=\"\") -> plt.Figure:\n",
    "    \"\"\"Plots the histogram of the deltas between the beginning of each event.\n",
    "    - cl: the column containing the datetime objects\n",
    "    - exclude_outliers_sdev: the number of standard deviations to exclude\n",
    "    - bins: the number of bins for the histogram\n",
    "    - title: the title of the histogram\"\"\"\n",
    "    diff = cl.sort_values().diff()\n",
    "    diff = diff[~diff.isna()]\n",
    "    regular_values = diff[~extract_outliers_mask(diff, exclude_outliers_sdev)]\n",
    "    if regular_values.empty:\n",
    "        print(\"No regular values\")\n",
    "        return\n",
    "    elif regular_values.dtype == \"timedelta64[ns]\":\n",
    "        regular_values = regular_values.dt.total_seconds()/(60*60*24)\n",
    "    \n",
    "    return plot_histogram(regular_values, bins=bins, title=title)\n",
    "\n",
    "plot_histogram_deltas(selection[\"begin_datetime\"], title=\"Distribution of the difference between the selection's start times\", bins=20, exclude_outliers_sdev=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_extreme_difference_values(dframe: pd.DataFrame, column: str, threshold_sd: int) -> pd.DataFrame:\n",
    "    \"\"\"Returns the DataFrame with the values that are above the threshold.\n",
    "    Sorts the DataFrame by the column and returns the values that are above the threshold in standard deviations.\n",
    "    Uses extract_outliers_mask to get the mask of the outliers.\"\"\"\n",
    "    sorted_df = dframe.sort_values(column)\n",
    "    return sorted_df[extract_outliers_mask(sorted_df[column], threshold_sd)]\n",
    "\n",
    "print(get_diff_outliers_description(selection[\"begin_datetime\"]))\n",
    "get_extreme_difference_values(selection, \"begin_datetime\", 1.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sort_events(df: pd.DataFrame, sort_column: str, ascending: bool=True) -> pd.DataFrame:\n",
    "    \"\"\"Sorts the events by the sort_column.\n",
    "    Returns the DataFrame sorted.\"\"\"\n",
    "    return df.sort_values(by=sort_column, ascending=ascending)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_unique_tags(dframe: pd.DataFrame, frozen_tags_column:str) -> pd.Series:\n",
    "    \"\"\"Returns the unique tags in the DataFrame as a Series.\"\"\"\n",
    "    return dframe[frozen_tags_column].unique()\n",
    "\n",
    "def produce_unique_tags_strings(unique_tags:pd.Series) -> list:\n",
    "    \"\"\"Returns the unique tags in the DataFrame, as a list of strings.\"\"\"\n",
    "    return list(map(lambda x: \" \".join(list(x)), unique_tags))\n",
    "\n",
    "def project_same_step(row: pd.Series, summary: pd.DataFrame, beginning_indices: pd.Series, matching_columns: pd.DataFrame):\n",
    "    \"\"\"Projects the duration of the event on the summary table, for events starting and ending on the same step.\"\"\"\n",
    "    destination_index = beginning_indices.loc[row.name]\n",
    "    \n",
    "    # if the destination index is within the range of the summary table\n",
    "    if destination_index >= 0 and destination_index < len(summary):\n",
    "        columns_with_matching_tags = matching_columns.loc[row.name][\"relevant_tags_strings\"]\n",
    "        summary.loc[destination_index, columns_with_matching_tags] += row[\"duration_timedelta\"]\n",
    "\n",
    "def project_different_steps(row: pd.Series, from_time: pd.Series, step_size: pd.Timedelta, summary: pd.DataFrame, beginning_indices: pd.Series, ending_indices: pd.Series, matching_columns: pd.DataFrame):\n",
    "    \"\"\"Projects the duration of the event on the summary table, for events starting and ending on different steps.\n",
    "    Since the event may span multiple steps, the duration is assigned to the indices depending on the split.\"\"\"\n",
    "    beginning_index = beginning_indices.loc[row.name]\n",
    "    ending_index = ending_indices.loc[row.name]\n",
    "    \n",
    "    columns_with_matching_tags = matching_columns.loc[row.name][\"relevant_tags_strings\"]\n",
    "\n",
    "    if beginning_index >= -1 and beginning_index + 1 < len(summary):\n",
    "        # adds the time between start and T1 \n",
    "        summary.loc[beginning_index+1, columns_with_matching_tags] += from_time[beginning_index+1] - row[\"begin_datetime\"]\n",
    "\n",
    "        if ending_index-1 < len(summary):\n",
    "            # adds to all blocks covered fully the full duration\n",
    "            summary.loc[beginning_index+1:ending_index-1, columns_with_matching_tags] += step_size\n",
    "\n",
    "        if ending_index >= 0 and ending_index < len(summary):\n",
    "            # adds the time between T3 and end\n",
    "            summary.loc[ending_index, columns_with_matching_tags] += row[\"end_datetime\"] - from_time[ending_index]\n",
    "\n",
    "\n",
    "def produce_summary_table(dframe: pd.DataFrame, start: str, end: str, step: str, frozen_tags_column:str = \"frozen_tags\", formatter=lambda x: x) -> pd.DataFrame:\n",
    "    range_min = parse_timestamp_string(start)\n",
    "    range_max = parse_timestamp_string(end)\n",
    "    range_delta = range_max - range_min\n",
    "\n",
    "    step_size = pd.to_timedelta(step)\n",
    "    \n",
    "    steps = range_delta // step_size\n",
    "    \n",
    "    unique_tags = extract_unique_tags(dframe, frozen_tags_column)\n",
    "    unique_tags_strings = produce_unique_tags_strings(map(formatter, unique_tags))\n",
    "\n",
    "    # initialises the summary table as empty\n",
    "    summary = pd.DataFrame(timedelta(0), index=range(steps), columns= [\"from_time\"] + unique_tags_strings)\n",
    "    \n",
    "    # sets the first two columns to be the time range\n",
    "    summary[\"from_time\"] = range_min + summary.index * step_size\n",
    "    \n",
    "    # exclude events starting after the end and ending before the start \n",
    "    filtered = dframe[~((dframe[\"begin_datetime\"] > range_max) | (dframe[\"end_datetime\"] < range_min))]\n",
    "    \n",
    "    # construct a dataframe with the tags and the tags as strings\n",
    "    # - \"relevant_tags\": extract_unique_tags(dframe)\n",
    "    # - \"relevant_tags_strings\": produce_unique_tags_strings(dframe, frozen_tags_column)\n",
    "    tags_dataframe = pd.DataFrame(data=[unique_tags, unique_tags_strings]).T.rename(columns={0: \"relevant_tags\", 1: \"relevant_tags_strings\"})\n",
    "    # for each row, selects the columns with the same tags\n",
    "    matching_columns = filtered.apply(lambda row: tags_dataframe[tags_dataframe[\"relevant_tags\"] == row[frozen_tags_column]], axis=1)\n",
    "\n",
    "    # produces the indices for the beginning and ending of the events\n",
    "    beginning_indices = (filtered[\"begin_datetime\"] - range_min).div(range_delta).mul(steps).astype(int)\n",
    "    ending_indices = (filtered[\"end_datetime\"] - range_min).div(range_delta).mul(steps).astype(int)\n",
    "    \n",
    "    # events starting and ending on the same step    \n",
    "    same_step = filtered[beginning_indices == ending_indices]\n",
    "    same_step.apply(lambda row: project_same_step(row, summary, beginning_indices, matching_columns), axis=1)\n",
    "        \n",
    "    # events not starting and ending on the same step\n",
    "    spanning_step = filtered[beginning_indices != ending_indices]\n",
    "    spanning_step.apply(lambda row: project_different_steps(row, summary[\"from_time\"], step_size, summary, beginning_indices, ending_indices, matching_columns), axis=1)\n",
    "\n",
    "    return summary\n",
    "\n",
    "time_step_description = \"7 days\"\n",
    "beginning_summary_str = \"2023-01-01 00:00 +0200\"\n",
    "ending_summary_str = \"2024-07-20 00:00 +0200\"\n",
    "\n",
    "\n",
    "df[\"note_tag\"] = df[\"note\"].apply(lambda x: tuple([x]))\n",
    "\n",
    "filtered_df = df[get_any_outline_mask(df, \"Series\")]\n",
    "summary = produce_summary_table(filtered_df, beginning_summary_str, ending_summary_str, time_step_description, frozen_tags_column=\"frozen_tags_outline\", formatter=lambda x: tuple([x[-1]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def restrict_summary_to_non_zero_columns(summary: pd.DataFrame) -> pd.DataFrame:\n",
    "    \"\"\"Returns the summary table, restricted to the columns where the sum is not zero.\"\"\"\n",
    "    return summary.loc[:, (summary != timedelta(0)).any(axis=0)]\n",
    "\n",
    "summary = restrict_summary_to_non_zero_columns(summary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "summary.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sum_columns_by_tag(raw: pd.DataFrame, columns_dict: dict, separator = \" \", keep_columns = [\"from_time\"]):\n",
    "    \"\"\"Adds some columns of the dataset using the columns_dict as reference.\n",
    "    - raw: the raw dataset, containing the columns to be summed and the columns to be kept\n",
    "    - columns_dict: key-list pairs where the key is the name of the new column and the list contains tag strings. \n",
    "    Each new columns is obtained by summing the columns whose names contain any of the strings in the list.\"\"\"\n",
    "    \n",
    "    if not all([kept_column in raw.columns for kept_column in keep_columns]):\n",
    "        missing_columns = [kept_column for kept_column in keep_columns if kept_column not in raw.columns]\n",
    "        raise KeyError(\"keep_columns is defined but some of its elements are not in raw.columns\", missing_columns)\n",
    "\n",
    "    original_columns = raw.columns.to_series()\n",
    "\n",
    "    # creates a new dataframe from the keys of the dictionary\n",
    "    dframe = pd.DataFrame(columns= keep_columns + list(columns_dict.keys()))\n",
    "    \n",
    "    for col in keep_columns:\n",
    "        if col in raw.columns:\n",
    "            dframe[col] = raw[col]\n",
    "    \n",
    "    for new_column_name in columns_dict.keys():\n",
    "        selection = original_columns.str.split(separator).map(lambda x: any([f in x for f in columns_dict[new_column_name]]))\n",
    "        dframe[new_column_name] = raw.loc[:, selection].sum(axis=1)\n",
    "\n",
    "    return dframe\n",
    "    \n",
    "column_grouping_selection = \"censored\"\n",
    "summary = sum_columns_by_tag(summary, tag_tree[column_grouping_selection])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_timedelta_columns_to_minutes(dframe: pd.DataFrame, inPlace=True) -> pd.DataFrame:\n",
    "    \"\"\"Converts all columns after the first to minutes in float format and removes the columns that are empty.\"\"\"\n",
    "    \n",
    "    if not inPlace:\n",
    "        dframe = dframe.copy()\n",
    "\n",
    "    for col in dframe.columns[1:]:\n",
    "        if pd.api.types.is_timedelta64_dtype(dframe[col]):\n",
    "            # Convert timedelta to minutes\n",
    "            dframe[col] = dframe[col].dt.total_seconds() / 60.0\n",
    "    \n",
    "    dframe = dframe.dropna(axis=1, how='all')\n",
    "    \n",
    "    if not inPlace:\n",
    "        return dframe\n",
    "           \n",
    "summary_minutes = convert_timedelta_columns_to_minutes(summary, False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_data(dframe: pd.DataFrame) -> pd.DataFrame:\n",
    "    \"\"\"Extracts the data from the DataFrame, by removing the columns that are not of type number.\"\"\"\n",
    "    return dframe.select_dtypes(include=[\"number\"])\n",
    "\n",
    "data = extract_data(summary_minutes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " Using the timeline series, 10 labels are sampled at uniform distance, \n",
    "    formatted to be YYYY-MM-DD strings and used as x axis labels.\n",
    "    The y axis labels are given by the names of the columns of the DataFrame."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def produce_summary_graph(dframe: pd.DataFrame, timeline: pd.Series, title: str=\"\") -> plt.Figure:\n",
    "    \"\"\"Draws a graph of the summary data.\n",
    "    Each column of dframe is plotted on a separate row of the graph, with its own line. The columns are kept separate.\n",
    "    The line for each is horizontal, varying in thickness, with the height given by the value of the first column.\n",
    "    The graph uses the timeline as the x-axis. \n",
    "    \"\"\"\n",
    "    fig, ax = plt.subplots()\n",
    "    \n",
    "    for i, col in enumerate(dframe.columns):\n",
    "        ax.plot(timeline, dframe[col] + i, label=col)\n",
    "    \n",
    "    ax.set_yticks(range(len(dframe.columns)))\n",
    "    ax.set_yticklabels(dframe.columns)\n",
    "    \n",
    "    ax.set_title(title)\n",
    "    ax.set_xlabel(\"Time\")\n",
    "    ax.set_ylabel(\"Tags\")\n",
    "    \n",
    "    plt.close(fig)\n",
    "    return fig\n",
    "\n",
    "produce_summary_graph(data, summary_minutes[\"from_time\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def produce_feature_histogram(data: pd.DataFrame, column_selection: str, time_step: str, bins: int=50, exclude_zero_values:bool=True) -> plt.Figure:\n",
    "    \"\"\"Produces a histogram of the feature in the DataFrame. Excludes the zero values.\n",
    "    - data: the DataFrame containing the data\n",
    "    - column_selection: string description of how the columns were grouped in making the data dataframe\n",
    "    - time_step: the time step to use for the histogram\n",
    "    - bins: the number of bins to use in the histogram\"\"\"\n",
    "    fig = plt.figure()\n",
    "    \n",
    "    # set the title of the figure and make it visible\n",
    "    fig.suptitle(f\"Feature histogram for {column_selection} in steps of {time_step}\")\n",
    "    \n",
    "    lam = lambda x: x[x != 0] if exclude_zero_values else x\n",
    "    \n",
    "    axes = data.apply(lam).hist(figsize=(15, 7), bins=bins, edgecolor=\"black\", density=True)\n",
    "    plt.subplots_adjust(hspace=0.8, wspace=0.2)\n",
    "    for ax in axes.flatten():\n",
    "        ax.set_xlabel(f\"Total duration in steps of {time_step}\")\n",
    "        ax.set_ylabel(\"Occurrences\")\n",
    "    \n",
    "    plt.close(fig)\n",
    "    return fig\n",
    "        \n",
    "produce_feature_histogram(data, column_grouping_selection, time_step_description)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_steps_without_activity(data: pd.DataFrame, time_step: str, selection: str) -> plt.Figure:\n",
    "    \"\"\"Plots the number of steps without any activity. Shows for each column the fraction of steps without any activity.\n",
    "    - data: the DataFrame containing the data\n",
    "    - time_step: the time step to use for the histogram\n",
    "    - selection: the selection of columns used in the data DataFrame\n",
    "    Returns the figure.\"\"\"\n",
    "    sparsity = data.apply(lambda x: x[x==0]).count()/len(data)\n",
    "    fig = plt.figure(figsize=(7, 4))\n",
    "    fig.suptitle(f\"Sparsity of {selection} in steps of {time_step}\")\n",
    "    ax = fig.add_subplot(111)\n",
    "    ax.grid(False)\n",
    "    sparsity.sort_values(ascending=False).plot(kind='bar', ax=ax, title=f\"{time_step} steps without any activity\", grid=False, ylabel=\"Fraction of steps without activity\", ylim=(0, 1), edgecolor=\"black\")\n",
    "    plt.close(fig)\n",
    "    return fig\n",
    "\n",
    "plot_steps_without_activity(data, time_step_description, column_grouping_selection)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "def to_normalized_df(data: pd.DataFrame) -> pd.DataFrame:\n",
    "    \"Takes a numeric DataFrame and returns a DataFrame with the data normalised using StandardScaler.\"\n",
    "    return pd.DataFrame(StandardScaler().fit_transform(data), columns=data.columns, index=data.index) \n",
    "\n",
    "data_norm = to_normalized_df(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from scipy.cluster.hierarchy import linkage, leaves_list\n",
    "import seaborn as sns\n",
    "\n",
    "def fill_diagonal_na(dframe: pd.DataFrame, inPlace=True) -> pd.DataFrame:\n",
    "    \"\"\"Fills the diagonal of the DataFrame with NaN values.\n",
    "    - dframe: the DataFrame to fill\n",
    "    - inPlace: if True, modifies the DataFrame in place, otherwise returns a copy of the DataFrame\"\"\"\n",
    "    if not inPlace:\n",
    "        dframe = dframe.copy()\n",
    "    \n",
    "    np.fill_diagonal(dframe.values, np.nan)\n",
    "    \n",
    "    if not inPlace:\n",
    "        return dframe\n",
    "    \n",
    "def produce_correlation_heatmap(data: pd.DataFrame, method_name: str=\"pearson\") -> plt.Figure:\n",
    "    \"\"\"Produces a heatmap of the correlation matrix of the data.\"\"\"\n",
    "    fig, ax = plt.subplots(figsize=(5, 5))\n",
    "    # only show the strictly upper triangle of the matrix, without the main diagonal\n",
    "    display_data = data.corr(method=method_name)\n",
    "    display_data = display_data.where(np.triu(np.ones(display_data.shape), k=1).astype(bool))\n",
    "    sns.heatmap(display_data, cmap=\"coolwarm\", center=0, linewidths=2, annot=True, fmt=\".2f\", ax=ax)\n",
    "    plt.close(fig)\n",
    "    return fig\n",
    "\n",
    "produce_correlation_heatmap(data_norm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_test_and_training_data(data_norm: pd.DataFrame) -> tuple:\n",
    "    \"\"\"Splits the data into 80% training and 20% test data.\"\"\"\n",
    "    return data_norm.sample(frac=0.8), data_norm.sample(frac=0.2)\n",
    "\n",
    "train_data, test_data = get_test_and_training_data(data_norm)\n",
    "\n",
    "from sklearn.linear_model import LinearRegression\n",
    "\n",
    "def model_linear_regression(train_data: pd.DataFrame, test_data: pd.DataFrame, column: str) -> tuple:\n",
    "    \"\"\"Trains a linear regression model on the training data and tests it on the test data.\n",
    "    Returns the model and the score.\"\"\"\n",
    "    model = LinearRegression(fit_intercept=False)\n",
    "    model.fit(train_data.drop(columns=[column]), train_data[column])\n",
    "    return model, model.score(test_data.drop(columns=[column]), test_data[column])\n",
    "\n",
    "def model_regression(train_data: pd.DataFrame, test_data: pd.DataFrame) -> str:\n",
    "    \"\"\"Trains a linear regression model for each column in the training data and tests it on the test data.\n",
    "    - train_data: the training data, containing the features and the target columns\n",
    "    - test_data: the test data, containing the features and the target columns\n",
    "    - Returns the string with the name of the column and the score for that column.\"\"\"\n",
    "    string = \"\"\n",
    "    for column in data.columns:\n",
    "        model, score = model_linear_regression(train_data, test_data, column)\n",
    "        string += f\"Column {column} has a score of {score:.2f}\\n\"\n",
    "    return string\n",
    "    \n",
    "print(model_regression(train_data, test_data))    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neural_network import MLPRegressor\n",
    "\n",
    "def model_multilayer_perceptron(train_data: pd.DataFrame, test_data: pd.DataFrame, column: str) -> tuple:\n",
    "    \"\"\"Trains a perceptron model on the training data and tests it on the test data.\n",
    "    - shape: the shape of the model, with the number of hidden layers and the number of neurons per layer.\n",
    "    Returns the model and the score.\"\"\"\n",
    "    model = MLPRegressor(max_iter=10000, random_state=0, hidden_layer_sizes=(100, 100))\n",
    "    model.fit(train_data.drop(columns=[column]), train_data[column])\n",
    "    return model, model.score(test_data.drop(columns=[column]), test_data[column])\n",
    "\n",
    "\n",
    "def model_perceptron(train_data: pd.DataFrame, test_data: pd.DataFrame) -> str:\n",
    "    \"\"\"Trains a perceptron model for each column in the training data and tests it on the test data.\n",
    "    - train_data: the training data, containing the features and the target columns\n",
    "    - test_data: the test data, containing the features and the target columns\n",
    "    - Returns the string with the name of the column and the score for that column.\"\"\"\n",
    "    string = \"\"\n",
    "    for column in data.columns:\n",
    "        model, score = model_multilayer_perceptron(train_data, test_data, column)\n",
    "        string += f\"{column} has a score of {score:.2f} with shape {model.hidden_layer_sizes}\\n\"\n",
    "    return string\n",
    "\n",
    "print(model_perceptron(train_data, test_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_umap_norm = to_normalized_df(data_norm.div(data_norm.sum(axis=1), axis=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import umap\n",
    "\n",
    "def reduce_dimensions(data: pd.DataFrame, n_components: int=3, n_neighbors: int=20, min_dist: float=0.1) -> pd.DataFrame:\n",
    "    \"\"\"Reduces the dimensions of the data using UMAP.\"\"\"\n",
    "    reducer = umap.UMAP(\n",
    "        n_neighbors=n_neighbors,\n",
    "        min_dist=min_dist,\n",
    "        n_components = n_components,\n",
    "        metric='correlation'\n",
    "    )\n",
    "    return pd.DataFrame(reducer.fit_transform(data), index=data.index)\n",
    "\n",
    "# promising distance metric for the data\n",
    "# cosine and correlation\n",
    "\n",
    "n_dim_umap = 2\n",
    "n_neighbors_umap = 20\n",
    "min_dist_umap = 0.1\n",
    "umap_title =  f\"UMAP {n_dim_umap}, {n_neighbors_umap}, {min_dist_umap} from {beginning_summary_str} to {ending_summary_str} in {time_step_description}, {column_grouping_selection}\"\n",
    "\n",
    "umap_reduced_data = reduce_dimensions(data_norm, n_dim_umap, n_neighbors_umap, min_dist_umap)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_scatterplot_umap(data_embedding: pd.DataFrame, labelling_values: pd.Series, title:str = \"UMAP Reduction\", cbar:bool =True) -> plt.Figure:\n",
    "    \"\"\"Plots the scatterplot of the UMAP reduction of the data.\n",
    "    - data_embedding: the DataFrame containing the UMAP reduction\n",
    "    - labelling_values: the Series containing the values to use for labelling the points\n",
    "    - title: the title of the plot\n",
    "    - cbar: if True, shows the colorbar\"\"\"\n",
    "    \n",
    "    if labelling_values.dtype != \"int64\" or labelling_values.dtype != \"float64\":\n",
    "        le = LabelEncoder()\n",
    "        labelling_values = le.fit_transform(labelling_values)\n",
    "    \n",
    "    if data_embedding.shape[1] == 2:\n",
    "        plt.figure()\n",
    "        labels = le.fit_transform(labelling_values)\n",
    "        scatter = plt.scatter(data_embedding.iloc[:, 0], data_embedding.iloc[:, 1], c=labelling_values, s=5)\n",
    "        if cbar:\n",
    "            plt.colorbar(label='Label')\n",
    "        plt.xlabel('UMAP 1')\n",
    "        plt.ylabel('UMAP 2')\n",
    "        plt.title(title)\n",
    "        # close the figure to avoid displaying it \n",
    "        return scatter\n",
    "    else:\n",
    "        fig = plt.figure()\n",
    "        ax = fig.add_subplot(111, projection='3d')\n",
    "        # add str labels to the points\n",
    "        ax.scatter(data_embedding.iloc[:, 0], data_embedding.iloc[:, 1], data_embedding.iloc[:, 2], c=labelling_values, s=5)\n",
    "        ax.set_xlabel('UMAP 1')\n",
    "        ax.set_ylabel('UMAP 2')\n",
    "        ax.set_zlabel('UMAP 3')\n",
    "        ax.set_title(title)\n",
    "\n",
    "        return ax\n",
    "\n",
    "plot_scatterplot_umap(umap_reduced_data, summary[\"from_time\"].dt.strftime(\"%m\"), umap_title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import percentileofscore\n",
    "\n",
    "def get_last_event(group: pd.DataFrame, datetime: pd.DatetimeTZDtype, compare_column:str) -> pd.Series:\n",
    "    \"\"\"Returns the last event in the group.\"\"\"\n",
    "    id_last_event_before_row = group[group[compare_column] < datetime][compare_column].idxmax()\n",
    "    return group.loc[id_last_event_before_row]\n",
    "\n",
    "\n",
    "def compare_event_to_group(row: pd.Series, group: pd.DataFrame) -> pd.DataFrame:\n",
    "    \"\"\"Compares the event to the group, returning a series of summary statistics.\n",
    "    - row is the event to compare to the group\n",
    "    - row and group should both have beginning_datetime, duration\n",
    "    - assumes that row[\"begin_datetime\"] is greater than all group[\"begin_datetime\"] \"\"\"\n",
    "    \n",
    "    previous_event = get_last_event(group, row[\"begin_datetime\"], \"begin_datetime\")\n",
    "    \n",
    "    # calculate the deltas between the events in minutes\n",
    "    deltas_minute = group[\"begin_datetime\"].sort_values().diff().dt.total_seconds()/60\n",
    "    deltas_minute = deltas_minute.dropna()\n",
    "    delta_value = (row[\"begin_datetime\"] - previous_event[\"begin_datetime\"]).total_seconds()/60\n",
    "\n",
    "    statistics = pd.DataFrame(index=[0])\n",
    "    statistics[\"delta\"] = delta_value\n",
    "    \n",
    "    # computes the inverse cumulative function evaluated at delta_row using the distribution of deltas\n",
    "    statistics[\"delta_quantile\"] = percentileofscore(deltas_minute, delta_value)/100\n",
    "    statistics[\"delta_mean\"] = deltas_minute.mean()\n",
    "    # writes the spread of the row in the group, in standard deviations\n",
    "    statistics[\"delta_std\"] = (delta_value - deltas_minute.mean())/deltas_minute.std()\n",
    "    \n",
    "    statistics[\"duration_quantile\"] = percentileofscore(group[\"duration\"], row[\"duration\"])/100\n",
    "    statistics[\"duration_mean\"] = group[\"duration\"].mean()\n",
    "    statistics[\"duration_std\"] = (row[\"duration\"] - group[\"duration\"].mean())/group[\"duration\"].std()\n",
    "    \n",
    "    return statistics\n",
    "\n",
    "\n",
    "sel = df[get_subset_match_tags_mask(df[\"frozen_tags\"], \"SLP\")]\n",
    "sel = sel[sel[\"duration\"] > 4*60]\n",
    "\n",
    "sel.sort_values(\"begin_datetime\", inplace=True)\n",
    "last_sel = sel.iloc[-5]\n",
    "\n",
    "compare_event_to_group(last_sel, sel)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}