ipynb/client_stat.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "large-uruguay",
   "metadata": {},
   "source": [
    "NOTE: You need to run `FLOPs Acc trade off.ipynb` with all `data` in `['Cifar10_cniid', 'Digits', 'DomainNet']` first."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "brazilian-visibility",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, sys\n",
    "import pandas as pd\n",
    "import wandb\n",
    "import numpy as np\n",
    "from tqdm.notebook import tqdm\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from collections import defaultdict\n",
    "from IPython.display import display"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "diagnostic-quick",
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.set_style(\"ticks\")\n",
    "cmap = sns.color_palette()\n",
    "sns.set_palette(sns.color_palette())\n",
    "plt.rc('axes', labelsize=12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "veterinary-supervision",
   "metadata": {},
   "outputs": [],
   "source": [
    "cache_path = './fig/flops_acc_curve'\n",
    "if not os.path.exists(cache_path):\n",
    "    os.makedirs(cache_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bizarre-address",
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets = ['Cifar10_cniid', 'Digits', 'DomainNet']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "instrumental-tribe",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dfs = {}\n",
    "for data in datasets:\n",
    "    csv_file = os.path.join(cache_path, f'{data}_res_df.csv')\n",
    "    print(f'read csv from {csv_file}')\n",
    "    data_dfs[data] = pd.read_csv(csv_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "recorded-parameter",
   "metadata": {},
   "outputs": [],
   "source": [
    "agg = pd.concat([data_dfs[k] for k in data_dfs])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "surgical-closure",
   "metadata": {},
   "outputs": [],
   "source": [
    "agg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "transsexual-restriction",
   "metadata": {},
   "outputs": [],
   "source": [
    "agg['mode'] = agg['mode'].apply(lambda n: n if n!='RT' else 'FedAvg')\n",
    "agg['mode'] = agg['mode'].apply(lambda n: n if n!='SplitMix' else 'Split-Mix')\n",
    "agg['data'] = agg['data'].apply(lambda n: n if n!='Cifar10' else 'CIFAR10')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "simplified-assistant",
   "metadata": {},
   "outputs": [],
   "source": [
    "agg['GFLOPs/batch'] = agg['MFLOPs/batch'] / 1e3\n",
    "agg['FLOPs/batch'] = agg['MFLOPs/batch'] * 1e6\n",
    "agg['normal params/rnd'] = agg['params/rnd'] * 1e6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "banner-mistress",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 1, figsize=(4,3))\n",
    "sns.boxplot(data=agg, x='data', y='normal params/rnd', hue='mode', )\n",
    "ax.set(yscale='log', ylabel=r'#parameters/round', xlabel='')\n",
    "ax.grid(True)\n",
    "ax.get_legend().remove()\n",
    "\n",
    "plt.tight_layout()\n",
    "out_file = os.path.join(cache_path, f'all_data_param_rnd.pdf')\n",
    "print(f\"save fig => {out_file}\")\n",
    "plt.savefig(out_file)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "supreme-timer",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 1, figsize=(4,3))\n",
    "sns.boxplot(data=agg, x='data', y='FLOPs/batch', hue='mode', )\n",
    "ax.set(yscale='log', xlabel='', ylabel=r'MACs/batch')\n",
    "ax.grid(True)\n",
    "ax.get_legend().remove()\n",
    "\n",
    "plt.tight_layout()\n",
    "out_file = os.path.join(cache_path, f'all_data_flops_batch.pdf')\n",
    "print(f\"save fig => {out_file}\")\n",
    "plt.savefig(out_file)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "persistent-ballet",
   "metadata": {},
   "source": [
    "## Per-user test acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "subtle-guard",
   "metadata": {},
   "outputs": [],
   "source": [
    "def fetch_config_summary(runs, config_keys, summary_keys):\n",
    "    df_dict = defaultdict(list)\n",
    "    for run in runs:\n",
    "        if run.state != 'finished':\n",
    "            print(\"WARN: run not finished yet\")\n",
    "        history_len = 0\n",
    "        missing_sum_key = []\n",
    "        # print(run.summary['all_domains'])\n",
    "        # print(run.config.keys())\n",
    "        for k in summary_keys:\n",
    "            if k in run.summary:\n",
    "                h = run.summary[k]\n",
    "                df_dict[k].append(h)\n",
    "            else:\n",
    "                missing_sum_key.append(k)\n",
    "                break\n",
    "        if len(missing_sum_key) > 0:\n",
    "            print(f\"missing key: {missing_sum_key}\")\n",
    "            continue\n",
    "        for k in run.summary.keys():\n",
    "            if k.endswith('clean test acc'):\n",
    "                df_dict[k].append(run.summary[k])\n",
    "        for k in config_keys:\n",
    "            df_dict[k].append(run.config[k])\n",
    "    return df_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "primary-vulnerability",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_df = {}\n",
    "data = 'DomainNet'\n",
    "\n",
    "for data in datasets:\n",
    "    if data == 'Digits':\n",
    "        sweep_dict = {\n",
    "            'FedAvg': \"jyhong/SplitMix_release/sweeps/8g8s7kp4\",\n",
    "            'SHeteroFL': \"jyhong/SplitMix_release/sweeps/0lh7d73x\",\n",
    "            'SplitMix': \"jyhong/SplitMix_release/sweeps/3wr7bsxb\",\n",
    "        }\n",
    "    elif data == 'DomainNet':\n",
    "        sweep_dict = {\n",
    "            'FedAvg': \"jyhong/SplitMix_release/sweeps/y489wn02\",\n",
    "            'SHeteroFL': \"jyhong/SplitMix_release/sweeps/shs7yw8p\",\n",
    "            'SplitMix': \"jyhong/SplitMix_release/sweeps/2kxrau5h\",\n",
    "        }\n",
    "    elif data == 'Cifar10_cniid':\n",
    "        sweep_dict = {\n",
    "            'FedAvg': \"jyhong/SplitMix_release/sweeps/6ua8jh9x\",\n",
    "            'SHeteroFL': \"jyhong/SplitMix_release/sweeps/fvg0045z\",\n",
    "            'SplitMix': \"jyhong/SplitMix_release/sweeps/g71nb2yv\",\n",
    "        }\n",
    "    else:\n",
    "        raise ValueError()\n",
    "\n",
    "    # get FedAvg\n",
    "    mode = 'FedAvg'\n",
    "    api = wandb.Api()\n",
    "    sweep = api.sweep(sweep_dict[mode])\n",
    "\n",
    "    df_dict = fetch_config_summary(\n",
    "        sweep.runs,\n",
    "        config_keys = ['width_scale'], \n",
    "        summary_keys = ['avg test acc']\n",
    "    )\n",
    "    df = pd.DataFrame(df_dict)\n",
    "    df['mode'] = mode\n",
    "    df['data'] = data\n",
    "    df['width'] = df['width_scale']\n",
    "\n",
    "    all_df[mode] = df\n",
    "\n",
    "    df = df[df['width'] == 0.125]\n",
    "\n",
    "    _df = df.drop(['avg test acc', 'width_scale', 'width'], axis=1).set_index(['mode', 'data'])\n",
    "    _df = _df.stack().reset_index().rename(columns={'level_2': 'user', 0: 'Acc'})\n",
    "    all_df[mode+'@'+data] = _df\n",
    "    print(mode+'@'+data, f\": {len(_df)}\")\n",
    "\n",
    "    for mode in ['SHeteroFL', 'SplitMix']:\n",
    "        api = wandb.Api()\n",
    "        sweep = api.sweep(sweep_dict[mode])\n",
    "\n",
    "        df_dict = fetch_config_summary(\n",
    "            sweep.runs,\n",
    "            config_keys = ['test_slim_ratio'], \n",
    "            summary_keys = ['avg test acc']\n",
    "        )\n",
    "        df = pd.DataFrame(df_dict)\n",
    "        df['mode'] = mode\n",
    "        df['data'] = data\n",
    "        df['width'] = df['test_slim_ratio']\n",
    "\n",
    "        df = df[df['width'] == 1.]\n",
    "\n",
    "        df = df.drop(['avg test acc', 'test_slim_ratio', 'width'], axis=1).set_index(['mode', 'data'])\n",
    "        df = df.stack().reset_index().rename(columns={'level_2': 'user', 0: 'Acc'})\n",
    "        all_df[mode+'@'+data] = df\n",
    "        print(mode+'@'+data, f\": {len(df)}\")\n",
    "        # df\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bridal-migration",
   "metadata": {},
   "source": [
    "## Aggregate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dynamic-organic",
   "metadata": {},
   "outputs": [],
   "source": [
    "keys = []\n",
    "for data in datasets:\n",
    "    keys += [k for k in all_df.keys() if data in k]\n",
    "keys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fabulous-textbook",
   "metadata": {},
   "outputs": [],
   "source": [
    "agg = pd.concat([all_df[k] for k in keys])\n",
    "agg['mode'] = agg['mode'].apply(lambda n: n if n != 'RT' else 'FedAvg')\n",
    "agg['mode'] = agg['mode'].apply(lambda n: n if n != 'SplitMix' else 'Split-Mix')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "weighted-covering",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 1, figsize=(4,3))\n",
    "sns.boxplot(data=agg, x='data', y='Acc', hue='mode', )\n",
    "ax.set(ylabel=r'accuracy', xlabel='')\n",
    "ax.grid(True)\n",
    "\n",
    "plt.tight_layout()\n",
    "out_file = os.path.join(cache_path, f'all_data_acc_user.pdf')\n",
    "print(f\"save fig => {out_file}\")\n",
    "plt.savefig(out_file)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "opponent-grain",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}