train_test_plot_superviesd_unsupervised.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7db72646",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 24167,
     "status": "ok",
     "timestamp": 1633278692933,
     "user": {
      "displayName": "Navid NaderiAlizadeh",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ghjujt3S2rcWpfcLut6MmW6YZc9vrkQL0oal1FrBT4=s64",
      "userId": "14430719405503000266"
     },
     "user_tz": 420
    },
    "id": "7db72646",
    "outputId": "037d535b-bf3b-4ca9-90c6-ad06e16d47a9"
   },
   "outputs": [],
   "source": [
    "import torch_geometric\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm\n",
    "from collections import defaultdict\n",
    "import os\n",
    "import itertools\n",
    "import sys\n",
    "folder_path = './'\n",
    "sys.path.insert(1, folder_path)\n",
    "from train_pipeline import train\n",
    "from eval_pipeline import eval\n",
    "import seaborn\n",
    "seaborn.set_style('darkgrid')\n",
    "plt.rcParams.update({'font.size': 14, 'lines.markersize': 8, 'lines.linewidth': 2})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "78a9fab4",
   "metadata": {
    "executionInfo": {
     "elapsed": 9,
     "status": "ok",
     "timestamp": 1633278692934,
     "user": {
      "displayName": "Navid NaderiAlizadeh",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ghjujt3S2rcWpfcLut6MmW6YZc9vrkQL0oal1FrBT4=s64",
      "userId": "14430719405503000266"
     },
     "user_tz": 420
    },
    "id": "78a9fab4"
   },
   "outputs": [],
   "source": [
    "random_seeds = [13531, 24642, 35753]\n",
    "k_range = [4, 6, 8, 10] # number of transmitter-receiver pairs for training\n",
    "num_train_samples_range = [32, 64, 128, 256] # number of training samples\n",
    "num_test_samples = 256 # number of test samples\n",
    "modes = ['Supervised', 'Unsupervised'] # modes of training\n",
    "figsize = (8, 6)\n",
    "os.makedirs(folder_path + 'saved_training_data/results', exist_ok=True)\n",
    "os.makedirs(folder_path + 'saved_training_data/models', exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "gF3ujWbv2Kh8",
   "metadata": {
    "executionInfo": {
     "elapsed": 9,
     "status": "ok",
     "timestamp": 1633278692935,
     "user": {
      "displayName": "Navid NaderiAlizadeh",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ghjujt3S2rcWpfcLut6MmW6YZc9vrkQL0oal1FrBT4=s64",
      "userId": "14430719405503000266"
     },
     "user_tz": 420
    },
    "id": "gF3ujWbv2Kh8"
   },
   "outputs": [],
   "source": [
    "# perform training\n",
    "for k, num_train_samples, mode, seed in itertools.product(k_range, num_train_samples_range, modes, random_seeds):\n",
    "    num_samples = {'train': num_train_samples, 'test': num_test_samples}\n",
    "    experiment_name = 'k_train_{}_num_train_samples_{}_mode_{}_seed_{}'.format(k, str(num_samples), mode, seed)\n",
    "    print('Now running experiment \\\"{}\\\" ...'.format(experiment_name))\n",
    "    all_epoch_results, best_model = train(k, num_samples, mode, seed)\n",
    "    torch.save(all_epoch_results, folder_path + 'saved_training_data/results/{}.json'.format(experiment_name))\n",
    "    torch.save(best_model, folder_path + 'saved_training_data/models/{}.pt'.format(experiment_name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5bBrvxrFGS-S",
   "metadata": {
    "executionInfo": {
     "elapsed": 29918,
     "status": "ok",
     "timestamp": 1633278722845,
     "user": {
      "displayName": "Navid NaderiAlizadeh",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ghjujt3S2rcWpfcLut6MmW6YZc9vrkQL0oal1FrBT4=s64",
      "userId": "14430719405503000266"
     },
     "user_tz": 420
    },
    "id": "5bBrvxrFGS-S"
   },
   "outputs": [],
   "source": [
    "# load all results\n",
    "all_epoch_results = dict()\n",
    "for k, num_train_samples, mode, seed in itertools.product(k_range, num_train_samples_range, modes, random_seeds):\n",
    "    num_samples = {'train': num_train_samples, 'test': num_test_samples}\n",
    "    experiment_name = 'k_train_{}_num_train_samples_{}_mode_{}_seed_{}'.format(k, str(num_samples), mode, seed)\n",
    "    all_epoch_results[k, str(num_samples), mode, seed] = torch.load(folder_path + 'saved_training_data/results/{}.json'.format(experiment_name))\n",
    "os.makedirs(folder_path + 'saved_training_data/plots', exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "K8As6HqSGjnL",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 396
    },
    "executionInfo": {
     "elapsed": 563,
     "status": "ok",
     "timestamp": 1633278723397,
     "user": {
      "displayName": "Navid NaderiAlizadeh",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ghjujt3S2rcWpfcLut6MmW6YZc9vrkQL0oal1FrBT4=s64",
      "userId": "14430719405503000266"
     },
     "user_tz": 420
    },
    "id": "K8As6HqSGjnL",
    "outputId": "55790745-e6ad-40b1-80db-33c3b62330e3"
   },
   "outputs": [],
   "source": [
    "# performance comparison\n",
    "plt.figure(figsize=figsize)\n",
    "markers = ['o', 'x']\n",
    "num_samples = {'train': np.max(num_train_samples_range), 'test': num_test_samples}\n",
    "for i, mode in enumerate(modes):\n",
    "    test_sum_rates_all_seeds = []\n",
    "    for seed in random_seeds:\n",
    "        test_sum_rates = []\n",
    "        for k in k_range:\n",
    "            res = all_epoch_results[k, str(num_samples), mode, seed]\n",
    "            corresponding_test_sumrate = np.max(res['test', k, 'normalized_sum_rate'])\n",
    "            test_sum_rates.append(corresponding_test_sumrate)\n",
    "        test_sum_rates_all_seeds.append(test_sum_rates)\n",
    "    plt.plot(k_range, np.mean(test_sum_rates_all_seeds, axis=0), label=mode, marker=markers[i])\n",
    "plt.xlabel(r'$K$')\n",
    "plt.ylabel('Normalized sum-rate')\n",
    "plt.xticks(k_range)\n",
    "plt.legend()\n",
    "plt.grid(True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "HAlzlqkmHOxS",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 396
    },
    "executionInfo": {
     "elapsed": 599,
     "status": "ok",
     "timestamp": 1633278724726,
     "user": {
      "displayName": "Navid NaderiAlizadeh",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ghjujt3S2rcWpfcLut6MmW6YZc9vrkQL0oal1FrBT4=s64",
      "userId": "14430719405503000266"
     },
     "user_tz": 420
    },
    "id": "HAlzlqkmHOxS",
    "outputId": "a13a6a5e-ea3b-404a-cfe4-e8981153f954"
   },
   "outputs": [],
   "source": [
    "# sample complexity\n",
    "plt.figure(figsize=figsize)\n",
    "k_range_to_plot = [k_range[0], k_range[-1]]\n",
    "markers = ['o', 'x']\n",
    "linestyles = [':', '-']\n",
    "for i, mode in enumerate(modes):\n",
    "    for j, k in enumerate(k_range_to_plot):\n",
    "        test_sum_rates_all_seeds = []\n",
    "        for seed in random_seeds:\n",
    "            test_sum_rates = []\n",
    "            for num_train_samples in num_train_samples_range:\n",
    "                num_samples = {'train': num_train_samples, 'test': num_test_samples}\n",
    "                res = all_epoch_results[k, str(num_samples), mode, seed]\n",
    "                corresponding_test_sumrate = np.max(res['test', k, 'normalized_sum_rate'])\n",
    "                test_sum_rates.append(corresponding_test_sumrate)\n",
    "            test_sum_rates_all_seeds.append(test_sum_rates)\n",
    "        if j == 0:\n",
    "            plt.plot(range(len(num_train_samples_range)), np.mean(test_sum_rates_all_seeds, axis=0), linestyles[j], label=mode+r' ($K=${})'.format(k), marker=markers[i])\n",
    "        elif j == 1:\n",
    "            plt.plot(range(len(num_train_samples_range)), np.mean(test_sum_rates_all_seeds, axis=0), linestyles[j], label=mode+r' ($K=${})'.format(k), marker=markers[i], color=plt.gca().lines[-1].get_color())\n",
    "plt.xlabel('# Training samples')\n",
    "plt.ylabel('Normalized sum-rate')\n",
    "plt.xticks(range(len(num_train_samples_range)))\n",
    "plt.gca().set_xticklabels(num_train_samples_range)\n",
    "plt.legend()\n",
    "plt.grid(True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "Zk_1irUBXAyN",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 606
    },
    "executionInfo": {
     "elapsed": 10871,
     "status": "ok",
     "timestamp": 1633278735590,
     "user": {
      "displayName": "Navid NaderiAlizadeh",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ghjujt3S2rcWpfcLut6MmW6YZc9vrkQL0oal1FrBT4=s64",
      "userId": "14430719405503000266"
     },
     "user_tz": 420
    },
    "id": "Zk_1irUBXAyN",
    "outputId": "cad50c2e-3cd3-4f17-933f-47c282a635d3"
   },
   "outputs": [],
   "source": [
    "# generalization capability\n",
    "test_seed = 12345321\n",
    "plt.figure(figsize=figsize)\n",
    "k_range_to_plot = [k_range[0], k_range[-1]]\n",
    "k_range_to_eval = k_range#range(2, 13)\n",
    "markers = ['o', 'x']\n",
    "linestyles = [':', '-']\n",
    "num_samples = {'train': np.max(num_train_samples_range), 'test': num_test_samples}\n",
    "for i, mode in enumerate(modes):\n",
    "    for j, k_train in enumerate(k_range_to_plot):\n",
    "        test_sum_rates_all_seeds = []\n",
    "        for seed in random_seeds:\n",
    "            test_sum_rates = []\n",
    "            for k_eval in tqdm(k_range_to_eval):\n",
    "                if k_eval == k_train:\n",
    "                    res = all_epoch_results[k_train, str(num_samples), mode, seed]\n",
    "                    test_sum_rate_generalization = np.max(res['test', k_train, 'normalized_sum_rate'])\n",
    "                else:\n",
    "                    # load model\n",
    "                    experiment_name = 'k_train_{}_num_train_samples_{}_mode_{}_seed_{}'.format(k_train, str(num_samples), mode, seed)\n",
    "                    model_state_dict = torch.load(folder_path + 'saved_training_data/models/{}.pt'.format(experiment_name))\n",
    "                    test_sum_rate_generalization = eval(model_state_dict, k_eval, num_test_samples, test_seed)\n",
    "                test_sum_rates.append(test_sum_rate_generalization)\n",
    "            test_sum_rates_all_seeds.append(test_sum_rates)\n",
    "        if j == 0:\n",
    "            plt.plot(k_range_to_eval, np.mean(test_sum_rates_all_seeds, axis=0), linestyles[j], label=mode+r' ($K_{train}=$' + '{})'.format(k_train), marker=markers[i])\n",
    "        elif j == 1:\n",
    "            plt.plot(k_range_to_eval, np.mean(test_sum_rates_all_seeds, axis=0), linestyles[j], label=mode+r' ($K_{train}=$' + '{})'.format(k_train), marker=markers[i], color=plt.gca().lines[-1].get_color())\n",
    "plt.xlabel(r'$K_{test}$')\n",
    "plt.ylabel('Normalized sum-rate')\n",
    "plt.xticks(k_range_to_eval)\n",
    "plt.legend()\n",
    "plt.grid(True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "yTq3uT32SvfB",
   "metadata": {
    "executionInfo": {
     "elapsed": 12,
     "status": "ok",
     "timestamp": 1633278735591,
     "user": {
      "displayName": "Navid NaderiAlizadeh",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Ghjujt3S2rcWpfcLut6MmW6YZc9vrkQL0oal1FrBT4=s64",
      "userId": "14430719405503000266"
     },
     "user_tz": 420
    },
    "id": "yTq3uT32SvfB"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "run_pipeline_and_plot_all_params_train_test.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}