fair_model.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/rahmanidashti/CPFairRecSys/blob/main/fair_model.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fvyNDFQm8Tr5"
      },
      "source": [
        "## Install packages"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VnNvYkBmRz-F",
        "outputId": "f90d7bc2-ee33-4e90-c30d-685cb6cf7f24"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting mip\n",
            "  Downloading mip-1.13.0-py3-none-any.whl (48.0 MB)\n",
            "\u001b[K     |████████████████████████████████| 48.0 MB 2.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: cffi in /usr/local/lib/python3.7/dist-packages (from mip) (1.15.0)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.7/dist-packages (from cffi->mip) (2.21)\n",
            "Installing collected packages: mip\n",
            "Successfully installed mip-1.13.0\n",
            "Collecting gurobipy==9.1.2\n",
            "  Downloading gurobipy-9.1.2-cp37-cp37m-manylinux1_x86_64.whl (11.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 11.1 MB 5.2 MB/s \n",
            "\u001b[?25hInstalling collected packages: gurobipy\n",
            "Successfully installed gurobipy-9.1.2\n",
            "Collecting cornac\n",
            "  Downloading cornac-1.14.2-cp37-cp37m-manylinux1_x86_64.whl (12.4 MB)\n",
            "\u001b[K     |████████████████████████████████| 12.4 MB 5.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.19 in /usr/local/lib/python3.7/dist-packages (from cornac) (4.63.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from cornac) (1.21.5)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from cornac) (1.4.1)\n",
            "Collecting powerlaw\n",
            "  Downloading powerlaw-1.5-py3-none-any.whl (24 kB)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (from powerlaw->cornac) (3.2.2)\n",
            "Requirement already satisfied: mpmath in /usr/local/lib/python3.7/dist-packages (from powerlaw->cornac) (1.2.1)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib->powerlaw->cornac) (0.11.0)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->powerlaw->cornac) (2.8.2)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->powerlaw->cornac) (1.4.0)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->powerlaw->cornac) (3.0.7)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from kiwisolver>=1.0.1->matplotlib->powerlaw->cornac) (3.10.0.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.1->matplotlib->powerlaw->cornac) (1.15.0)\n",
            "Installing collected packages: powerlaw, cornac\n",
            "Successfully installed cornac-1.14.2 powerlaw-1.5\n"
          ]
        }
      ],
      "source": [
        "# Install MIP and Gurobipy for optimization purpose\n",
        "! pip install mip\n",
        "! python -m pip install gurobipy==9.1.2\n",
        "# install Cornac framework for RecSys\n",
        "! pip install cornac"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4gT5oYyYy-X8"
      },
      "source": [
        "## Config"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        },
        "id": "8zmyV4mG7Ea1",
        "outputId": "732023f9-b4a5-43a1-ee57-17221a1239cf"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "TensorFlow 1.x selected.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'1.15.2'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 2
        }
      ],
      "source": [
        "# Swtich to tensoflow 1 to use Cornac\n",
        "%tensorflow_version 1.x\n",
        "import tensorflow as tf\n",
        "tf.__version__"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "N3KNp45j7IGM"
      },
      "outputs": [],
      "source": [
        "# import packages\n",
        "import os\n",
        "import numpy as np\n",
        "from collections import defaultdict\n",
        "from tqdm import tqdm\n",
        "\n",
        "from itertools import product\n",
        "from sys import stdout as out\n",
        "from mip import Model, xsum, maximize, BINARY\n",
        "\n",
        "import cornac\n",
        "from cornac.eval_methods import BaseMethod, RatioSplit\n",
        "from cornac.models import MostPop, UserKNN, ItemKNN, MF, PMF, BPR, NeuMF, WMF, HPF, CVAE, VAECF, NMF\n",
        "from cornac.metrics import Precision, Recall, NDCG, AUC, MAP, FMeasure, MRR\n",
        "from cornac.data import Reader\n",
        "from cornac.utils import cache"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jYlI50GuyV8l"
      },
      "source": [
        "## Download datasets, user, and item groups"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HWIh5ZiT7NwE"
      },
      "outputs": [],
      "source": [
        "# download datasets: train, test, tune\n",
        "def download_dataset():\n",
        "  ds_root_path = \"datasets/\"\n",
        "  for dataset in ds_names:\n",
        "    dataset_path = os.path.join(ds_root_path, dataset)\n",
        "\n",
        "    if not os.path.isdir(dataset_path):\n",
        "      os.makedirs(dataset_path)\n",
        "      print(\"Directory '%s' is created.\" % dataset_path)\n",
        "    else:\n",
        "      print(\"Directory '%s' is exist.\" % dataset_path)\n",
        "\n",
        "    # -nc: skip downloads that would download to existing files.\n",
        "\n",
        "    try:\n",
        "      os.system(f\"wget -P {dataset_path} -nc https://raw.githubusercontent.com/rahmanidashti/CPFairRecSys/main/datasets/{dataset}/{dataset}_train.txt\")\n",
        "      os.system(f\"wget -P {dataset_path} -nc https://raw.githubusercontent.com/rahmanidashti/CPFairRecSys/main/datasets/{dataset}/{dataset}_test.txt\")\n",
        "      os.system(f\"wget -P {dataset_path} -nc https://raw.githubusercontent.com/rahmanidashti/CPFairRecSys/main/datasets/{dataset}/{dataset}_tune.txt\")\n",
        "      print(f\"{dataset}: The train, tune, and test sets downloaded.\")\n",
        "    except Expception as e:\n",
        "      print(e)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GjL34huY7f_I"
      },
      "outputs": [],
      "source": [
        "# dowanload user groups: active and inactive\n",
        "def download_user_groups():\n",
        "  user_root_path = \"user_groups/\"\n",
        "  for dataset in ds_names:\n",
        "    for ugroup in ds_users:\n",
        "      user_groups_path = os.path.join(user_root_path, dataset, ugroup)\n",
        "\n",
        "      if not os.path.isdir(user_groups_path):\n",
        "        os.makedirs(user_groups_path)\n",
        "        print(\"Directory '%s' is created.\" % user_groups_path)\n",
        "      else:\n",
        "        print(\"Directory '%s' is exist.\" % user_groups_path)\n",
        "\n",
        "      # -nc: skip downloads that would download to existing files.\n",
        "\n",
        "      try:\n",
        "        os.system(f\"wget -P {user_groups_path} https://raw.githubusercontent.com/rahmanidashti/CPFairRecSys/main/datasets/{dataset}/groups/users/{ugroup}/active_ids.txt\")\n",
        "        os.system(f\"wget -P {user_groups_path} https://raw.githubusercontent.com/rahmanidashti/CPFairRecSys/main/datasets/{dataset}/groups/users/{ugroup}/inactive_ids.txt\")\n",
        "        print(f\"{dataset}: User groups on '{ugroup}' downloaded.\")\n",
        "      except Exception as e:\n",
        "        print(e)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gDwRmOSjJHKK"
      },
      "outputs": [],
      "source": [
        "# dowanload item groups: short-head and long-tail\n",
        "def download_item_groups():\n",
        "  item_root_path = \"item_groups/\"\n",
        "  for dataset in ds_names:\n",
        "    for igroup in ds_items:\n",
        "      item_groups_path = os.path.join(item_root_path, dataset, igroup)\n",
        "\n",
        "      if not os.path.isdir(item_groups_path):\n",
        "        os.makedirs(item_groups_path)\n",
        "        print(\"Directory '%s' is created.\" % item_groups_path)\n",
        "      else:\n",
        "        print(\"Directory '%s' is exist.\" % item_groups_path)\n",
        "      \n",
        "      # -nc: skip downloads that would download to existing files.\n",
        "    try:\n",
        "      os.system(f\"wget -P {item_groups_path} -nc https://raw.githubusercontent.com/rahmanidashti/CPFairRecSys/main/datasets/{dataset}/groups/items/{igroup}/shorthead_items.txt\")\n",
        "      os.system(f\"wget -P {item_groups_path} -nc https://raw.githubusercontent.com/rahmanidashti/CPFairRecSys/main/datasets/{dataset}/groups/items/{igroup}/longtail_items.txt\")\n",
        "      print(f\"{dataset}: Item groups on '{igroup}' downloaded.\")\n",
        "    except Exception as e:\n",
        "      print(e)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1OEbK4PzX9ul"
      },
      "source": [
        "## Run Config"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6xGcL4Gm7KhT"
      },
      "outputs": [],
      "source": [
        "# dataset congfig\n",
        "ds_names = [\"Gowalla\"]\n",
        "ds_users = ['005']\n",
        "ds_items = ['020']\n",
        "\n",
        "###\n",
        "no_user_groups = 2\n",
        "no_item_groups = 2\n",
        "topk = 50 # this is not a length of recommendation ist, it is only the first topk items for the optimisation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UVGAHeGSl7Gc",
        "outputId": "e1fbbc6c-c667-48c4-af71-af720c1ec4a2"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Directory 'datasets/Gowalla' is created.\n",
            "Gowalla: The train, tune, and test sets downloaded.\n",
            "Directory 'user_groups/Gowalla/005' is created.\n",
            "Gowalla: User groups on '005' downloaded.\n",
            "Directory 'item_groups/Gowalla/020' is created.\n",
            "Gowalla: Item groups on '020' downloaded.\n"
          ]
        }
      ],
      "source": [
        "download_dataset()\n",
        "download_user_groups()\n",
        "download_item_groups()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iVnSrbN9ygDF"
      },
      "source": [
        "## Load `Cornac` data and model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dcckaEz-7SQr"
      },
      "outputs": [],
      "source": [
        "# reading the train, test, and val sets\n",
        "def read_data(dataset):\n",
        "  \"\"\"\n",
        "  Read the train, test, and tune file using Cornac reader class\n",
        "\n",
        "  Parameters\n",
        "  ----------\n",
        "  dataset : the name of the dataset\n",
        "    example: 'MovieLens100K'\n",
        "\n",
        "  Returns\n",
        "  ----------\n",
        "  train_data:\n",
        "    The train set that is 70% of interactions\n",
        "  tune_data:\n",
        "    The tune set that is 10% of interactions\n",
        "  test_data:\n",
        "    The test set that is 20% of interactions\n",
        "  \"\"\"\n",
        "  reader = Reader()\n",
        "  train_data = reader.read(fpath=f\"datasets/{dataset}/{dataset}_train.txt\", fmt='UIR', sep='\\t')\n",
        "  tune_data = reader.read(fpath=f\"datasets/{dataset}/{dataset}_tune.txt\", fmt='UIR', sep='\\t')\n",
        "  test_data = reader.read(fpath=f\"datasets/{dataset}/{dataset}_test.txt\", fmt='UIR', sep='\\t')\n",
        "  return train_data, tune_data, test_data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JxijedzS7VHq"
      },
      "outputs": [],
      "source": [
        "# load data into Cornac evaluation method\n",
        "def load_data(train_data, test_data):\n",
        "  \"\"\"\n",
        "  load data into Cornac evaluation method\n",
        "\n",
        "  Parameters\n",
        "  ----------\n",
        "  train_data: \n",
        "    train_data from Reader Class\n",
        "  test_data:\n",
        "    test_data from Reader Class\n",
        "\n",
        "  Returns\n",
        "  ----------\n",
        "  eval_method:\n",
        "    Instantiation of a Base evaluation method using the provided train and test sets\n",
        "  \"\"\"\n",
        "  # exclude_unknowns (bool, default: False) – Whether to exclude unknown users/items in evaluation.\n",
        "  # Instantiate a Base evaluation method using the provided train and test sets\n",
        "  eval_method = BaseMethod.from_splits(\n",
        "      train_data=train_data, test_data=test_data, rating_threshold=1.0, exclude_unknowns=True, verbose=True\n",
        "  )\n",
        "\n",
        "  return eval_method"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "code",
        "id": "gtOdLiJ67XiZ"
      },
      "outputs": [],
      "source": [
        "# running the cornac\n",
        "def run_model(eval_method):\n",
        "  \"\"\"\n",
        "  running the cornac\n",
        "\n",
        "  Parameters\n",
        "  ----------\n",
        "  eval_method: \n",
        "    Cornac's evaluation protocol\n",
        "\n",
        "  Returns\n",
        "  ----------\n",
        "  exp:\n",
        "    Cornac's Experiment\n",
        "  \"\"\"\n",
        "  \n",
        "  models = [\n",
        "            # MostPop(),\n",
        "            # UserKNN(k=20, similarity=\"cosine\", weighting=\"bm25\", name=\"UserKNN-BM25\"),\n",
        "            # ItemKNN(k=20, similarity=\"cosine\", name=\"ItemKNN-Cosine\"),\n",
        "            # BPR(k=50, max_iter=200, learning_rate=0.001, lambda_reg=0.001, verbose=True),\n",
        "            # WMF(k=50, max_iter=50, learning_rate=0.001, lambda_u=0.01, lambda_v=0.01, verbose=True, seed=123),\n",
        "            # HPF(k=50, seed=123, hierarchical=False, name=\"PF\"),\n",
        "            VAECF(k=10, autoencoder_structure=[20], act_fn=\"tanh\", likelihood=\"mult\", n_epochs=100, batch_size=100, learning_rate=0.001, beta=1.0, seed=123, use_gpu=True, verbose=True),\n",
        "            # NeuMF(num_factors=9, layers=[32, 16, 8], act_fn=\"tanh\", num_epochs=5, num_neg=3, batch_size=256, lr=0.001, seed=42, verbose=True)\n",
        "            ]\n",
        "\n",
        "  # define metrics to evaluate the models\n",
        "  metrics = [\n",
        "            AUC(), MAP(), MRR(), NDCG(k=10), Recall(k=10)\n",
        "            # Precision(k=5), Precision(k=10), Precision(k=20),  Precision(k=50), \n",
        "            # Recall(k=5), Recall(k=10), Recall(k=20), Recall(k=50),\n",
        "            # FMeasure(k=5), FMeasure(k=10), FMeasure(k=20), FMeasure(k=50), \n",
        "            # NDCG(k=5), NDCG(k=10), NDCG(k=20), NDCG(k=50)\n",
        "            ]\n",
        "\n",
        "  # put it together in an experiment, voilà!\n",
        "  exp = cornac.Experiment(eval_method=eval_method, models=models, metrics=metrics)\n",
        "  exp.run()\n",
        "\n",
        "  return exp"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xKk0B-6wzEhj"
      },
      "source": [
        "## Load user and item groups"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "TmAJmsuu9nIY"
      },
      "outputs": [],
      "source": [
        "# Create a set of IDs for each users group\n",
        "# Creat a matrix U which shows the user and the groups of the user\n",
        "def read_user_groups(user_group_fpath: str, gid) -> set:\n",
        "  \"\"\"\n",
        "  Read the user groups lists\n",
        "\n",
        "  Parameters\n",
        "  ----------\n",
        "  user_group_fpath:\n",
        "    The path of the user group file\n",
        "\n",
        "  U (global variabvle):\n",
        "    The global matrix of users and their group\n",
        "\n",
        "  Returns\n",
        "  ----------\n",
        "  user_ids:\n",
        "    The set of user ids corresponding to the group\n",
        "  \"\"\"\n",
        "\n",
        "  user_group = open(user_group_fpath, 'r').readlines()\n",
        "  user_ids = set()\n",
        "  for eachline in user_group:\n",
        "    uid = eachline.strip()\n",
        "    # convert uids to uidx\n",
        "    uid = eval_method.train_set.uid_map[uid]\n",
        "    uid = int(uid)\n",
        "    user_ids.add(uid)\n",
        "    U[uid][gid] = 1\n",
        "  return user_ids"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wQIgU-RFN9a0"
      },
      "outputs": [],
      "source": [
        "# read test data\n",
        "def read_ground_truth(test_file):\n",
        "  \"\"\"\n",
        "  Read test set data\n",
        "\n",
        "  Parameters\n",
        "  ----------\n",
        "  test_file:\n",
        "    The test set data\n",
        "\n",
        "  Returns\n",
        "  ----------\n",
        "  ground_truth:\n",
        "    A dictionary includes user with actual items in test data\n",
        "  \"\"\"\n",
        "  ground_truth = defaultdict(set)\n",
        "  truth_data = open(test_file, 'r').readlines()\n",
        "  for eachline in truth_data:\n",
        "    uid, iid, _ = eachline.strip().split()\n",
        "\n",
        "    # convert uids to uidx\n",
        "    uid = eval_method.train_set.uid_map[uid]\n",
        "    # convert iids to iidx\n",
        "    iid = eval_method.train_set.iid_map[iid]\n",
        "\n",
        "    uid, iid = int(uid), int(iid)\n",
        "    ground_truth[uid].add(iid)\n",
        "  return ground_truth"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YuxZe64gNejy"
      },
      "outputs": [],
      "source": [
        "# read train data\n",
        "def read_train_data(train_file):\n",
        "  \"\"\"\n",
        "  Read test set data\n",
        "\n",
        "  Parameters\n",
        "  ----------\n",
        "  train_file:\n",
        "    The train_file set data\n",
        "\n",
        "  Returns\n",
        "  ----------\n",
        "  train_checkins:\n",
        "    A dictionary includes user with items in train data\n",
        "  pop: dictionary\n",
        "    A dictionary of all items alongside of its occurrences counter in the training data\n",
        "    example: {1198: 893, 1270: 876, 593: 876, 2762: 867}\n",
        "  \"\"\"\n",
        "  train_checkins = defaultdict(set)\n",
        "  pop_items = dict()\n",
        "  train_data = open(train_file, 'r').readlines()\n",
        "\n",
        "  for eachline in train_data:\n",
        "    uid, iid, _ = eachline.strip().split()\n",
        "\n",
        "    # convert uids to uidx\n",
        "    uid = eval_method.train_set.uid_map[uid]\n",
        "    # convert iids to iidx\n",
        "    iid = eval_method.train_set.iid_map[iid]\n",
        "\n",
        "    uid, iid = int(uid), int(iid)\n",
        "    # a dictionary of popularity of items\n",
        "    if iid in pop_items.keys():\n",
        "      pop_items[iid] += 1\n",
        "    else:\n",
        "      pop_items[iid] = 1\n",
        "    train_checkins[uid].add(iid)\n",
        "  return train_checkins, pop_items"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3vU5YS_gv6SO"
      },
      "source": [
        "## Metrics"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uI3kZR7Bn4EJ"
      },
      "outputs": [],
      "source": [
        "def catalog_coverage(predicted: list, catalog: list) -> float:\n",
        "  \"\"\"\n",
        "  Computes the catalog coverage for k lists of recommendations\n",
        "  Parameters\n",
        "  ----------\n",
        "  predicted : a list of lists\n",
        "      Ordered predictions\n",
        "      example: [['X', 'Y', 'Z'], ['X', 'Y', 'Z']]\n",
        "  catalog: list\n",
        "      A list of all unique items in the training data\n",
        "      example: ['A', 'B', 'C', 'X', 'Y', Z]\n",
        "  k: integer\n",
        "      The number of observed recommendation lists\n",
        "      which randomly choosed in our offline setup\n",
        "  Returns\n",
        "  ----------\n",
        "  catalog_coverage:\n",
        "      The catalog coverage of the recommendations as a percent rounded to 2 decimal places\n",
        "  ----------\n",
        "  Metric Defintion:\n",
        "  Ge, M., Delgado-Battenfeld, C., & Jannach, D. (2010, September).\n",
        "  Beyond accuracy: evaluating recommender systems by coverage and serendipity.\n",
        "  In Proceedings of the fourth ACM conference on Recommender systems (pp. 257-260). ACM.\n",
        "  \"\"\"\n",
        "  predicted_flattened = [p for sublist in predicted for p in sublist]\n",
        "  L_predictions = len(set(predicted_flattened))\n",
        "  catalog_coverage = round(L_predictions / (len(catalog) * 1.0) * 100, 2)\n",
        "  # output: precent (%)\n",
        "  return catalog_coverage"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "T451FOM3m4qx"
      },
      "outputs": [],
      "source": [
        "def novelty(predicted: list, pop: dict, u: int, k: int) -> float:\n",
        "  \"\"\"\n",
        "  Computes the novelty for a list of recommended items for a user\n",
        "  Parameters\n",
        "  ----------\n",
        "  predicted : a list of recommedned items\n",
        "      Ordered predictions\n",
        "      example: ['X', 'Y', 'Z']\n",
        "  pop: dictionary\n",
        "      A dictionary of all items alongside of its occurrences counter in the training data\n",
        "      example: {1198: 893, 1270: 876, 593: 876, 2762: 867}\n",
        "  u: integer\n",
        "      The number of users in the training data\n",
        "  k: integer\n",
        "      The length of recommended lists per user\n",
        "  Returns\n",
        "  ----------\n",
        "  novelty:\n",
        "      The novelty of the recommendations in system level\n",
        "  mean_self_information:\n",
        "      The novelty of the recommendations in recommended top-N list level\n",
        "  ----------\n",
        "  Metric Defintion:\n",
        "  Zhou, T., Kuscsik, Z., Liu, J. G., Medo, M., Wakeling, J. R., & Zhang, Y. C. (2010).\n",
        "  Solving the apparent diversity-accuracy dilemma of recommender systems.\n",
        "  Proceedings of the National Academy of Sciences, 107(10), 4511-4515.\n",
        "  \"\"\"\n",
        "  self_information = 0\n",
        "  for item in predicted:\n",
        "    if item in pop.keys():\n",
        "      item_popularity = pop[item] / u\n",
        "      item_novelty_value = np.sum(-np.log2(item_popularity))\n",
        "    else:\n",
        "      item_novelty_value = 0\n",
        "    self_information += item_novelty_value\n",
        "  novelty_score = self_information / k\n",
        "  return novelty_score"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "f7Ymn9W4Qdht"
      },
      "outputs": [],
      "source": [
        "def precisionk(actual, predicted):\n",
        "  return 1.0 * len(set(actual) & set(predicted)) / len(predicted)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WR8y_5vrKKaw"
      },
      "outputs": [],
      "source": [
        "def recallk(actual, predicted):\n",
        "  return 1.0 * len(set(actual) & set(predicted)) / len(actual)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VTs77lfzLMBx"
      },
      "outputs": [],
      "source": [
        "def ndcgk(actual, predicted):\n",
        "  idcg = 1.0\n",
        "  dcg = 1.0 if predicted[0] in actual else 0.0\n",
        "  for i, p in enumerate(predicted[1:]):\n",
        "    if p in actual:\n",
        "      dcg += 1.0 / np.log(i+2)\n",
        "    idcg += 1.0 / np.log(i+2)\n",
        "  return dcg / idcg"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TbchhQqfwEBg"
      },
      "source": [
        "## Load User and Item Matrices"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_YBJA3fayetN"
      },
      "outputs": [],
      "source": [
        "# Here we saved the results of scores and you can read them from repo to do your experiments.\n",
        "# S is a matrix to store user's scores on each item\n",
        "# P incldues the indecies of topk ranked items \n",
        "# Sprime saves the scores of topk ranked items\n",
        "\n",
        "def load_ranking_matrices(model, total_users, total_items, topk):\n",
        "  S = np.zeros((total_users, total_items))\n",
        "  P = np.zeros((total_users, topk))\n",
        "\n",
        "  # for model in exp.models:\n",
        "  print(model.name)\n",
        "  for uid in tqdm(range(total_users)):\n",
        "    S[uid] = model.score(uid)\n",
        "    P[uid] = np.array(list(reversed(model.score(uid).argsort()))[:topk])\n",
        "\n",
        "  return S, P"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6purGS8lSc0W"
      },
      "outputs": [],
      "source": [
        "# Ahelp is a binary matrix in which an element of its is 1 if the corresponding element in P (which is an item index) is in ground truth.\n",
        "# Actually is shows whether the rankied item in P is included in ground truth or not.\n",
        "\n",
        "def load_ground_truth_index(total_users, topk, P, train_checkins):\n",
        "  Ahelp = np.zeros((total_users, topk))\n",
        "  for uid in tqdm(range(total_users)):\n",
        "    for j in range(topk):\n",
        "      # convert user_ids to user_idx\n",
        "      # convert item_ids to item_idx\n",
        "      if P[uid][j] in train_checkins[uid]:\n",
        "        Ahelp[uid][j] = 1\n",
        "  return Ahelp"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VuOUY4lCJ57A"
      },
      "outputs": [],
      "source": [
        "# create a set of IDs for each users group\n",
        "def read_item_groups(item_group_fpath: str, gid) -> set:\n",
        "  item_group = open(item_group_fpath, 'r').readlines()\n",
        "  item_ids = set()\n",
        "  for eachline in item_group:\n",
        "    iid = eachline.strip()\n",
        "    # convert iids to iidx\n",
        "    iid = eval_method.train_set.iid_map[iid]\n",
        "    iid = int(iid)\n",
        "    item_ids.add(iid)\n",
        "    I[iid][gid] = 1\n",
        "  return item_ids"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HRlw5kZLPzQ4"
      },
      "outputs": [],
      "source": [
        "def read_item_index(total_users, topk, no_item_groups):\n",
        "  Ihelp = np.zeros((total_users, topk, no_item_groups))\n",
        "  for uid in range(total_users):\n",
        "    for lid in range(topk):\n",
        "      # convert item_ids to item_idx\n",
        "      if P[uid][lid] in shorthead_item_ids:\n",
        "        Ihelp[uid][lid][0] = 1\n",
        "      elif P[uid][lid] in longtail_item_ids:\n",
        "        Ihelp[uid][lid][1] = 1\n",
        "  return Ihelp"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kkl6XrhAwQG2"
      },
      "source": [
        "## Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "48cabQBvwPMN"
      },
      "outputs": [],
      "source": [
        "def metric_per_group(group, W):\n",
        "  NDCG10 = list()\n",
        "  Pre10 = list()\n",
        "  Rec10 = list()\n",
        "  Novelty10 = list()\n",
        "  predicted = list()\n",
        "  All_Predicted = list()\n",
        "\n",
        "  for uid in tqdm(group):\n",
        "    if uid in ground_truth.keys():\n",
        "      for j in range(50):\n",
        "        if W[uid][j].x == 1:\n",
        "          predicted.append(P[uid][j])\n",
        "      copy_predicted = predicted[:]\n",
        "      All_Predicted.append(copy_predicted)\n",
        "      NDCG = ndcgk(actual=ground_truth[uid], predicted=predicted)\n",
        "      Pre = precisionk(actual=ground_truth[uid], predicted=predicted)\n",
        "      Rec = recallk(actual=ground_truth[uid], predicted=predicted)\n",
        "      Novelty = novelty(predicted=predicted, pop=pop_items, u=eval_method.total_users, k=10)\n",
        "\n",
        "      NDCG10.append(NDCG)\n",
        "      Pre10.append(Pre)\n",
        "      Rec10.append(Rec)\n",
        "      Novelty10.append(Novelty)\n",
        "\n",
        "      # cleaning the predicted list for a new user\n",
        "      predicted.clear()\n",
        "\n",
        "  catalog = catalog_coverage(predicted=All_Predicted, catalog=pop_items.keys())\n",
        "  return round(np.mean(NDCG10), 5), round(np.mean(Pre10), 5), round(np.mean(Rec10), 5), round(np.mean(Novelty10), 5), catalog"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JorbXFzomfmv"
      },
      "outputs": [],
      "source": [
        "def metric_on_all(W):\n",
        "  \"\"\"\n",
        "  \"\"\"\n",
        "  predicted_user = list()\n",
        "  NDCG_all = list()\n",
        "  PRE_all = list()\n",
        "  REC_all = list()\n",
        "  Novelty_all = list()\n",
        "  All_Predicted = list()\n",
        "\n",
        "\n",
        "  for uid in tqdm(range(eval_method.total_users)):\n",
        "    if uid in ground_truth.keys():\n",
        "      for j in range(50):\n",
        "        if W[uid][j].x == 1:\n",
        "          predicted_user.append(P[uid][j])\n",
        "\n",
        "      copy_predicted = predicted_user[:]\n",
        "      All_Predicted.append(copy_predicted)\n",
        "\n",
        "      NDCG_user = ndcgk(actual=ground_truth[uid], predicted=predicted_user)\n",
        "      PRE_user = precisionk(actual=ground_truth[uid], predicted=predicted_user)\n",
        "      REC_user = recallk(actual=ground_truth[uid], predicted=predicted_user)\n",
        "      Novelty_user = novelty(predicted=predicted_user, pop=pop_items, u=eval_method.total_users, k=10)\n",
        "\n",
        "      NDCG_all.append(NDCG_user)\n",
        "      PRE_all.append(PRE_user)\n",
        "      REC_all.append(REC_user)\n",
        "      Novelty_all.append(Novelty_user)\n",
        "\n",
        "      # cleaning the predicted list for a new user\n",
        "      predicted_user.clear()\n",
        "\n",
        "  catalog = catalog_coverage(predicted=All_Predicted, catalog=pop_items.keys())\n",
        "  return round(np.mean(NDCG_all), 5), round(np.mean(PRE_all), 5), round(np.mean(REC_all), 5), round(np.mean(Novelty_all), 5), catalog"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PSBVs9wiwXGF"
      },
      "source": [
        "## Fairness"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4ERUinxWfALw"
      },
      "outputs": [],
      "source": [
        "def fairness_optimisation(fairness='N', uepsilon=0.000005, iepsilon = 0.0000005):\n",
        "  print(f\"Runing fairness optimisation on '{fairness}', {format(uepsilon, 'f')}, {format(iepsilon, 'f')}\")\n",
        "  # V1: No. of users\n",
        "  # V2: No. of top items (topk)\n",
        "  # V3: No. of user groups\n",
        "  # V4: no. og item groups\n",
        "  V1, V2, V3, V4 = set(range(eval_method.total_users)), set(range(topk)), set(range(no_user_groups)), set(range(no_item_groups))\n",
        "\n",
        "  # initiate model\n",
        "  model = Model()\n",
        "\n",
        "  # W is a matrix (size: user * top items) to be learned by model\n",
        "  #W = [[model.add_var(var_type=BINARY) for j in V2] for i in V1]\n",
        "  W = [[model.add_var() for j in V2] for i in V1]\n",
        "  user_dcg = [model.add_var() for i in V1] \n",
        "  user_ndcg = [model.add_var() for i in V1] \n",
        "  group_ndcg_v = [model.add_var() for k in V3]\n",
        "  item_group = [model.add_var() for k in V4]\n",
        "\n",
        "  user_precision=[model.add_var() for i in V1] \n",
        "  group_precision=[model.add_var() for k in V3]\n",
        "\n",
        "  user_recall=[model.add_var() for i in V1] \n",
        "  group_recall= [model.add_var() for k in V3]\n",
        "\n",
        "  if fairness == 'N':\n",
        "    ### No Fairness ###\n",
        "    model.objective = maximize(xsum((S[i][j] * W[i][j]) for i in V1 for j in V2))\n",
        "  elif fairness == 'C':\n",
        "    ### C-Fairness: NDCG_Best: group_ndcg_v[1] - group_ndcg_v[0] ###\n",
        "    model.objective = maximize(xsum((S[i][j] * W[i][j]) for i in V1 for j in V2) - uepsilon * (group_ndcg_v[1] - group_ndcg_v[0]))\n",
        "  elif fairness == 'P':\n",
        "    model.objective = maximize(xsum((S[i][j] * W[i][j]) for i in V1 for j in V2) - iepsilon * (item_group[0] - item_group[1]))\n",
        "  elif fairness == 'CP':\n",
        "    model.objective = maximize(xsum((S[i][j] * W[i][j]) for i in V1 for j in V2) - uepsilon * (group_ndcg_v[1] - group_ndcg_v[0]) - iepsilon * (item_group[0] - item_group[1]))\n",
        "\n",
        "  # first constraint: the number of 1 in W should be equal to top-k, recommending top-k best items\n",
        "  k = 10\n",
        "  for i in V1:\n",
        "      model += xsum(W[i][j] for j in V2) == k\n",
        "\n",
        "  for i in V1:\n",
        "    user_idcg_i = 7.137938133620551\n",
        "      \n",
        "    model += user_dcg[i] == xsum((W[i][j] * Ahelp[i][j]) for j in V2) \n",
        "    model += user_ndcg[i] == user_dcg[i] / user_idcg_i\n",
        "    \n",
        "    model += user_precision[i]==xsum((W[i][j] * Ahelp[i][j]) for j in V2) / k\n",
        "    model += user_recall[i]==xsum((W[i][j] * Ahelp[i][j]) for j in V2) / len(train_checkins[i])\n",
        "\n",
        "  for k in V3:\n",
        "    model += group_ndcg_v[k] == xsum(user_dcg[i] * U[i][k] for i in V1)\n",
        "    model += group_precision[k] == xsum(user_precision[i] * U[i][k] for i in V1)\n",
        "    model += group_recall[k] == xsum(user_recall[i] * U[i][k] for i in V1)\n",
        "\n",
        "  for k in V4:\n",
        "    model += item_group[k]== xsum(W[i][j] * Ihelp[i][j][k] for i in V1 for j in V2)\n",
        "\n",
        "  for i in V1:\n",
        "    for j in V2:\n",
        "      model += W[i][j] <= 1\n",
        "  # optimizing\n",
        "  model.optimize()\n",
        "\n",
        "  return W, item_group"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z9NhMbgRyIXg"
      },
      "source": [
        "## Run"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iPtl1pdcG7i8"
      },
      "outputs": [],
      "source": [
        "def write_results():\n",
        "  ndcg_ac, pre_ac, rec_ac, novelty_ac, coverage_ac = metric_per_group(group=active_user_ids, W=W)\n",
        "  ndcg_iac, pre_iac, rec_iac, novelty_iac, coverage_iac = metric_per_group(group=inactive_user_ids, W=W)\n",
        "  ndcg_all, pre_all, rec_all, novelty_all, coverage_all = metric_on_all(W=W)\n",
        "  if fair_mode == 'N':\n",
        "    results.write(f\"{dataset},{model.name},{u_group}%,{i_group}%,{fair_mode},-,-,{ndcg_all},{ndcg_ac},{ndcg_iac},{pre_all},{pre_ac},{pre_iac},{rec_all},{rec_ac},{rec_iac},{novelty_all},{novelty_ac},{novelty_iac},{coverage_all},{coverage_ac},{coverage_iac},{item_group[0].x},{item_group[1].x},{eval_method.total_users * 10}=={item_group[0].x + item_group[1].x}\") \n",
        "  elif fair_mode == 'C':\n",
        "    results.write(f\"{dataset},{model.name},{u_group}%,{i_group}%,{fair_mode},{format(user_eps, '.7f')},-,{ndcg_all},{ndcg_ac},{ndcg_iac},{pre_all},{pre_ac},{pre_iac},{rec_all},{rec_ac},{rec_iac},{novelty_all},{novelty_ac},{novelty_iac},{coverage_all},{coverage_ac},{coverage_iac},{item_group[0].x},{item_group[1].x},{eval_method.total_users * 10}=={item_group[0].x + item_group[1].x}\") \n",
        "  elif fair_mode == 'P':\n",
        "    results.write(f\"{dataset},{model.name},{u_group}%,{i_group}%,{fair_mode},-,{format(item_eps, '.7f')},{ndcg_all},{ndcg_ac},{ndcg_iac},{pre_all},{pre_ac},{pre_iac},{rec_all},{rec_ac},{rec_iac},{novelty_all},{novelty_ac},{novelty_iac},{coverage_all},{coverage_ac},{coverage_iac},{item_group[0].x},{item_group[1].x},{eval_method.total_users * 10}=={item_group[0].x + item_group[1].x}\") \n",
        "  elif fair_mode == 'CP':\n",
        "    results.write(f\"{dataset},{model.name},{u_group}%,{i_group}%,{fair_mode},{format(user_eps, '.7f')},{format(item_eps, '.7f')},{ndcg_all},{ndcg_ac},{ndcg_iac},{pre_all},{pre_ac},{pre_iac},{rec_all},{rec_ac},{rec_iac},{novelty_all},{novelty_ac},{novelty_iac},{coverage_all},{coverage_ac},{coverage_iac},{item_group[0].x},{item_group[1].x},{eval_method.total_users * 10}=={item_group[0].x + item_group[1].x}\") \n",
        "  results.write('\\n')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "4423d919e0f241edb68988b04f5d69a0",
            "570dd248ee7a45198e7bae5b2a5cd056",
            "5860eb88a52949fcb040e6676ff9d86a",
            "743faee1bf6f45e5b466025a352cabff",
            "c4aa68528dda43a7a95f042990a32e2e",
            "a70686baa50643399420fc10110294c7",
            "3171d1ca6bf44aa5a3050a3fdf377d45",
            "5ca1c72fd22747848f9a4313e9f037b1",
            "1f0a41cccba442ab934cb5092a21e0a0",
            "bc6a1a8b7a7d4bd38e638389d4a990f3",
            "fe7c353218144be88fb20ac5271c99a9",
            "cb9d9b037cde4b07a38d5116f2822a80",
            "ca4462e497e645588209897df98b6f87",
            "220e469f721c46ba9c25dd8b5a3cc40c",
            "8cc16cdf0d9b444692e37bda185baf59",
            "ce5cc0f1a5164034b583eee378e5ed7a",
            "9e2bcefb387d46119c00a2d8cea50d2f",
            "3fc687052c9b469f8f348e00484bfd0b",
            "186e039e08054005af2f49848258c8b0",
            "b834a6fc50ec4b1c85ec167d04a09076",
            "f94aabc1fa2745a3a3cef1e4655b13a8",
            "5a4d64d669fd40a3ad0556ea809975de"
          ]
        },
        "id": "jXev5h4_XGbf",
        "outputId": "38f23948-69c5-4ead-f7d6-d11db400d714"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Datasets: Gowalla\n",
            "rating_threshold = 1.0\n",
            "exclude_unknowns = True\n",
            "---\n",
            "Training data:\n",
            "Number of users = 1130\n",
            "Number of items = 1189\n",
            "Number of ratings = 46371\n",
            "Max rating = 215.0\n",
            "Min rating = 1.0\n",
            "Global mean = 1.8\n",
            "---\n",
            "Test data:\n",
            "Number of users = 1130\n",
            "Number of items = 1188\n",
            "Number of ratings = 13249\n",
            "Number of unknown users = 0\n",
            "Number of unknown items = 0\n",
            "---\n",
            "Total users = 1130\n",
            "Total items = 1189\n",
            "\n",
            "[VAECF] Training started!\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/100 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "4423d919e0f241edb68988b04f5d69a0"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "[VAECF] Evaluation started!\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Ranking:   0%|          | 0/1130 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "cb9d9b037cde4b07a38d5116f2822a80"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "TEST:\n",
            "...\n",
            "      |    AUC |    MAP |    MRR | NDCG@10 | Recall@10 | Train (s) | Test (s)\n",
            "----- + ------ + ------ + ------ + ------- + --------- + --------- + --------\n",
            "VAECF | 0.8454 | 0.0898 | 0.2695 |  0.1115 |    0.0965 |   18.0010 |   2.2091\n",
            "\n",
            "ActiveU: 56, InActive: 1074, All: 1130\n",
            "No. of Shorthead Items: 238 and No. of Longtaill Items: 951\n",
            "> Model: VAECF\n",
            "VAECF\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1130/1130 [00:01<00:00, 796.71it/s]\n",
            "100%|██████████| 1130/1130 [00:00<00:00, 35244.42it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Runing fairness optimisation on 'N', 0.000005, 0.000000\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 56/56 [00:00<00:00, 3784.31it/s]\n",
            "100%|██████████| 1074/1074 [00:00<00:00, 5042.78it/s]\n",
            "100%|██████████| 1130/1130 [00:00<00:00, 4802.39it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Runing fairness optimisation on 'C', 0.003000, 0.000000\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 56/56 [00:00<00:00, 3644.67it/s]\n",
            "100%|██████████| 1074/1074 [00:00<00:00, 4802.83it/s]\n",
            "100%|██████████| 1130/1130 [00:00<00:00, 5022.28it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Runing fairness optimisation on 'C', 0.000500, 0.000000\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 56/56 [00:00<00:00, 5253.32it/s]\n",
            "100%|██████████| 1074/1074 [00:00<00:00, 4919.44it/s]\n",
            "100%|██████████| 1130/1130 [00:00<00:00, 5087.69it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Runing fairness optimisation on 'C', 0.000100, 0.000000\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 56/56 [00:00<00:00, 5329.85it/s]\n",
            "100%|██████████| 1074/1074 [00:00<00:00, 4738.70it/s]\n",
            "100%|██████████| 1130/1130 [00:00<00:00, 5159.68it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Runing fairness optimisation on 'C', 0.000050, 0.000000\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 56/56 [00:00<00:00, 5096.25it/s]\n",
            "100%|██████████| 1074/1074 [00:00<00:00, 4494.91it/s]\n",
            "100%|██████████| 1130/1130 [00:00<00:00, 4591.78it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Runing fairness optimisation on 'C', 0.000005, 0.000000\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 56/56 [00:00<00:00, 5594.40it/s]\n",
            "100%|██████████| 1074/1074 [00:00<00:00, 4732.90it/s]\n",
            "100%|██████████| 1130/1130 [00:00<00:00, 5287.04it/s]\n"
          ]
        }
      ],
      "source": [
        "# 1: Iterate over the datasets\n",
        "for dataset in ds_names:\n",
        "  print(f\"Datasets: {dataset}\")\n",
        "  # read train, tune, test datasets\n",
        "  train_data, tune_data, test_data = read_data(dataset=dataset)\n",
        "  # load data into Cornac and create eval_method\n",
        "  eval_method = load_data(train_data=train_data, test_data=test_data)\n",
        "  total_users = eval_method.total_users\n",
        "  total_items = eval_method.total_items\n",
        "  # load train_checkins and pop_items dictionary\n",
        "  train_checkins, pop_items = read_train_data(train_file = f\"datasets/{dataset}/{dataset}_train.txt\")\n",
        "  # load ground truth dict\n",
        "  ground_truth = read_ground_truth(test_file = f\"datasets/{dataset}/{dataset}_test.txt\")\n",
        "  # run Cornac models and create experiment object including models' results\n",
        "  exp = run_model(eval_method=eval_method)\n",
        "  # 4: read user groups\n",
        "  for u_group in ds_users:\n",
        "    # read matrix U for users and their groups\n",
        "    U = np.zeros((total_users, no_user_groups))\n",
        "    # load active and inactive users\n",
        "    active_user_ids = read_user_groups(user_group_fpath = f\"user_groups/{dataset}/{u_group}/active_ids.txt\", gid = 0)\n",
        "    inactive_user_ids = read_user_groups(user_group_fpath = f\"user_groups/{dataset}/{u_group}/inactive_ids.txt\", gid = 1)\n",
        "    print(f\"ActiveU: {len(active_user_ids)}, InActive: {len(inactive_user_ids)}, All: {len(active_user_ids) + len(inactive_user_ids)}\")\n",
        "    len_sizes = [len(active_user_ids), len(inactive_user_ids)]\n",
        "    # 5: read item groups\n",
        "    for i_group in ds_items:\n",
        "      # read matrix I for items and their groups\n",
        "      I = np.zeros((total_items, no_item_groups))\n",
        "      # read item groups\n",
        "      shorthead_item_ids = read_item_groups(item_group_fpath = f\"item_groups/{dataset}/{i_group}/shorthead_items.txt\", gid = 0)\n",
        "      longtail_item_ids = read_item_groups(item_group_fpath = f\"item_groups/{dataset}/{i_group}/longtail_items.txt\", gid = 1)\n",
        "      print(f\"No. of Shorthead Items: {len(shorthead_item_ids)} and No. of Longtaill Items: {len(longtail_item_ids)}\")\n",
        "      # 2: iterate over the models\n",
        "      for model in exp.models:\n",
        "        results = open(f\"results_{dataset}_{model.name}.csv\", 'w')\n",
        "        results.write(\"Dataset,Model,GUser,GItem,Type,User_EPS,Item_EPS,ndcg_ALL,ndcg_ACT,ndcg_INACT,Pre_ALL,Pre_ACT,Pre_INACT,Rec_ALL,Rec_ACT,Rec_INACT,Nov_ALL,Nov_ACT,Nov_INACT,Cov_ALL,Cov_ACT,Cov_INACT,Short_Items,Long_Items,All_Items\\n\")\n",
        "        print(f\"> Model: {model.name}\")\n",
        "        # load matrix S and P\n",
        "        S, P = load_ranking_matrices(model=model, total_users=total_users, total_items=total_items, topk=topk)\n",
        "        # load matrix Ahelp\n",
        "        Ahelp = load_ground_truth_index(total_users=total_users, topk=topk, P=P, train_checkins=train_checkins)\n",
        "        # load matrix Ihelp\n",
        "        Ihelp = read_item_index(total_users=total_users, topk=50, no_item_groups=no_item_groups)\n",
        "        # iterate on fairness mode: user, item, user-item\n",
        "        for fair_mode in ['N', 'C']:\n",
        "          if fair_mode == 'N':\n",
        "            W, item_group = fairness_optimisation(fairness=fair_mode)\n",
        "            write_results()\n",
        "          elif fair_mode == 'C':\n",
        "            for user_eps in [0.003, 0.0005, 0.0001, 0.00005, 0.000005]:\n",
        "              W, item_group = fairness_optimisation(fairness=fair_mode, uepsilon=user_eps)\n",
        "              write_results()\n",
        "          elif fair_mode == 'P':\n",
        "            for item_eps in [0.003, 0.0005, 0.0001, 0.00005, 0.000005]:\n",
        "              W, item_group = fairness_optimisation(fairness=fair_mode, iepsilon=item_eps)\n",
        "              write_results()\n",
        "          elif fair_mode == 'CP':\n",
        "            for user_eps in [0.003, 0.0005, 0.0001, 0.00005, 0.000005]:\n",
        "              for item_eps in [0.003, 0.0005, 0.0001, 0.00005, 0.000005]:\n",
        "                W, item_group = fairness_optimisation(fairness=fair_mode, uepsilon=user_eps, iepsilon=item_eps)\n",
        "                write_results()\n",
        "        results.close()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "9AfYnmYPZTRL"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [],
      "name": "fair_model.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "4423d919e0f241edb68988b04f5d69a0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_570dd248ee7a45198e7bae5b2a5cd056",
              "IPY_MODEL_5860eb88a52949fcb040e6676ff9d86a",
              "IPY_MODEL_743faee1bf6f45e5b466025a352cabff"
            ],
            "layout": "IPY_MODEL_c4aa68528dda43a7a95f042990a32e2e"
          }
        },
        "570dd248ee7a45198e7bae5b2a5cd056": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a70686baa50643399420fc10110294c7",
            "placeholder": "​",
            "style": "IPY_MODEL_3171d1ca6bf44aa5a3050a3fdf377d45",
            "value": "100%"
          }
        },
        "5860eb88a52949fcb040e6676ff9d86a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5ca1c72fd22747848f9a4313e9f037b1",
            "max": 100,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_1f0a41cccba442ab934cb5092a21e0a0",
            "value": 100
          }
        },
        "743faee1bf6f45e5b466025a352cabff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bc6a1a8b7a7d4bd38e638389d4a990f3",
            "placeholder": "​",
            "style": "IPY_MODEL_fe7c353218144be88fb20ac5271c99a9",
            "value": " 100/100 [00:11&lt;00:00,  8.92it/s, loss=2.63]"
          }
        },
        "c4aa68528dda43a7a95f042990a32e2e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a70686baa50643399420fc10110294c7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3171d1ca6bf44aa5a3050a3fdf377d45": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5ca1c72fd22747848f9a4313e9f037b1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1f0a41cccba442ab934cb5092a21e0a0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "bc6a1a8b7a7d4bd38e638389d4a990f3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fe7c353218144be88fb20ac5271c99a9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cb9d9b037cde4b07a38d5116f2822a80": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ca4462e497e645588209897df98b6f87",
              "IPY_MODEL_220e469f721c46ba9c25dd8b5a3cc40c",
              "IPY_MODEL_8cc16cdf0d9b444692e37bda185baf59"
            ],
            "layout": "IPY_MODEL_ce5cc0f1a5164034b583eee378e5ed7a"
          }
        },
        "ca4462e497e645588209897df98b6f87": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9e2bcefb387d46119c00a2d8cea50d2f",
            "placeholder": "​",
            "style": "IPY_MODEL_3fc687052c9b469f8f348e00484bfd0b",
            "value": "Ranking: 100%"
          }
        },
        "220e469f721c46ba9c25dd8b5a3cc40c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_186e039e08054005af2f49848258c8b0",
            "max": 1130,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b834a6fc50ec4b1c85ec167d04a09076",
            "value": 1130
          }
        },
        "8cc16cdf0d9b444692e37bda185baf59": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f94aabc1fa2745a3a3cef1e4655b13a8",
            "placeholder": "​",
            "style": "IPY_MODEL_5a4d64d669fd40a3ad0556ea809975de",
            "value": " 1130/1130 [00:02&lt;00:00, 546.35it/s]"
          }
        },
        "ce5cc0f1a5164034b583eee378e5ed7a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9e2bcefb387d46119c00a2d8cea50d2f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3fc687052c9b469f8f348e00484bfd0b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "186e039e08054005af2f49848258c8b0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b834a6fc50ec4b1c85ec167d04a09076": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f94aabc1fa2745a3a3cef1e4655b13a8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5a4d64d669fd40a3ad0556ea809975de": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    },
    "accelerator": "TPU"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}