DNN_Covid19.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "DNN_Covid19.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Sprivacy/DP-Utility-in-ML/blob/main/DNN_Covid19.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SxZcrT0wHsvV",
        "outputId": "4940c12d-bedf-4cc3-c38e-d80f8861da2f"
      },
      "source": [
        "#Mount Data on your google Drive\n",
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/gdrive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fit3UwlnISQ_",
        "outputId": "87b9d924-bc3a-4b79-f4f1-4613cae4b2e2"
      },
      "source": [
        "#Neccessary Libraries for DP-SGD\n",
        "try:\n",
        "  # %tensorflow_version only exists in Colab.\n",
        "  %tensorflow_version 1.x\n",
        "except Exception:\n",
        "  pass\n",
        "\n",
        "import tensorflow as tf\n",
        "\n",
        "import numpy as np\n",
        "\n",
        "tf.compat.v1.logging.set_verbosity(tf.logging.ERROR)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "TensorFlow 1.x selected.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I38zAxhoIb1s",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8aefb392-f491-4a50-8ad4-d19bfe611dc5"
      },
      "source": [
        "#importing libraries\n",
        "#from tensorflow_privacy.privacy.analysis import compute_dp_sgd_privacy\n",
        "#from tensorflow_privacy.privacy.optimizers.dp_optimizer import DPGradientDescentGaussianOptimizer,DPAdamGaussianOptimizer\n",
        "from tensorflow.keras.datasets import cifar10\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import Dense, Flatten, Conv2D, MaxPooling2D\n",
        "from tensorflow.keras.losses import sparse_categorical_crossentropy\n",
        "from tensorflow.keras.optimizers import Adam\n",
        "import matplotlib.pyplot as plt\n",
        "import tensorflow as tf\n",
        "from tensorflow.keras import datasets, layers, models\n",
        "import matplotlib.pyplot as plt\n",
        "import sys\n",
        "from matplotlib import pyplot\n",
        "from keras.datasets import cifar10\n",
        "from keras.utils import to_categorical\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Conv2D\n",
        "from keras.layers import MaxPooling2D\n",
        "from keras.layers import Dense\n",
        "from keras.layers import Flatten\n",
        "from keras.layers import Dropout\n",
        "from keras.optimizers import SGD\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import seaborn as sns\n",
        "import tensorflow as tf\n",
        "import matplotlib.pyplot as plt\n",
        "from sklearn.preprocessing import LabelEncoder, StandardScaler\n",
        "from sklearn.model_selection import train_test_split\n",
        "import tensorflow as tf"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Using TensorFlow backend.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CNXP4inNIhGx"
      },
      "source": [
        "!pip install tensorflow_privacy"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "okLn3XLQIls1"
      },
      "source": [
        "#!pip install tensorflow_privacy\n",
        "\n",
        "from tensorflow_privacy.privacy.analysis import compute_dp_sgd_privacy\n",
        "from tensorflow_privacy.privacy.optimizers.dp_optimizer import DPGradientDescentGaussianOptimizer,DPAdamGaussianOptimizer"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mlsZ0O3uI0Ra"
      },
      "source": [
        "df = pd.read_csv('...../covid.csv', low_memory=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 268
        },
        "id": "6LTpcrCWLPBN",
        "outputId": "69900e66-6df4-4621-a6e1-15a7ec8de388"
      },
      "source": [
        "#visualize the dataset\n",
        "#dataset.head(10)\n",
        "#data=dataset\n",
        "df.isnull().sum()\n",
        "df.drop(columns={'entry_date','date_died','date_symptoms','id'},axis=1,inplace=True)\n",
        "df['covid_res'].value_counts().to_frame()\n",
        "\n",
        "df['covid_res'].replace([1,2,3],[1,0,2],inplace=True)\n",
        "df['covid_res'].value_counts().to_frame()\n",
        "\n",
        "df.rename(columns={'covid_res':'Chance'},inplace=True)\n",
        "#only take positive or negative patient, ignore awaiting patient\n",
        "df=df[df['Chance']!=2]\n",
        "\n",
        "df=df[df['intubed']!=97]\n",
        "df['intubed'].value_counts().sort_index().plot.bar()\n",
        "df=df[df['intubed']!=99]\n",
        "df['intubed'].value_counts().to_frame()\n",
        "df=df[df['pneumonia']!=99]\n",
        "df=df[df['pregnancy']!=97]\n",
        "df=df[df['pregnancy']!=98]\n",
        "df=df[df['diabetes']!=98]\n",
        "df=df[df['copd']!=98]\n",
        "df=df[df['asthma']!=98]\n",
        "df=df[df['inmsupr']!=98]\n",
        "df=df[df['hypertension']!=98]\n",
        "df=df[df['other_disease']!=98]\n",
        "df=df[df['cardiovascular']!=98]\n",
        "df=df[df['obesity']!=98]\n",
        "df=df[df['renal_chronic']!=98]\n",
        "df=df[df['tobacco']!=98]\n",
        "df=df[df['contact_other_covid']!=99]\n",
        "df=df[df['icu']!=99]\n",
        "df=df[df['icu']!=97]\n",
        "df.reset_index(drop=True,inplace=True)\n",
        "df.head(10)\n",
        "\n",
        "features=['sex', 'patient_type', 'intubed', 'pneumonia', 'age', 'pregnancy',\n",
        "       'diabetes', 'copd', 'asthma', 'inmsupr', 'hypertension',\n",
        "       'other_disease', 'cardiovascular', 'obesity', 'renal_chronic',\n",
        "       'tobacco', 'contact_other_covid', 'icu']\n",
        "list(enumerate(features))\n",
        "\n",
        "X=df[['sex', 'patient_type', 'intubed', 'pneumonia', 'age', 'pregnancy',\n",
        "       'diabetes', 'copd', 'asthma', 'inmsupr', 'hypertension',\n",
        "       'other_disease', 'cardiovascular', 'obesity', 'renal_chronic',\n",
        "       'tobacco', 'contact_other_covid', 'icu']]\n",
        "y=df['Chance']\n",
        "\n",
        "from sklearn.preprocessing import LabelEncoder, StandardScaler\n",
        "scaler = StandardScaler()\n",
        "\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAD7CAYAAACfQGjDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAP/klEQVR4nO3df6xfdX3H8efLVhhKFJAb4tq6ktnNFRYVO6hxWwxdoKhZyaIEsqwdIzaZ+GPLkq2aJU1QFkiWMUmUrJOOYpxImEsbrWs61CxmKfYiDijIuEGBNvy42gpTo6z63h/3U/3ucj8tvV/4fgt9PpJvvue8z+ec8/7mJvfVc87ne5uqQpKkubxs3A1Iko5dhoQkqcuQkCR1GRKSpC5DQpLUZUhIkroWjruB59vpp59eS5cuHXcbkvSicuedd363qiZm119yIbF06VImJyfH3YYkvagkeXiuurebJEldRwyJJJuTPJnk3oHaaUl2JnmwvZ/a6klyfZKpJHcnOWdgn3Vt/INJ1g3U35LknrbP9UlyuHNIkkbnuVxJ3ASsnlXbANxeVcuA29s6wEXAsvZaD9wAM7/wgY3AecC5wMaBX/o3AO8d2G/1Ec4hSRqRI4ZEVf0HsH9WeQ2wpS1vAS4eqN9cM3YBpyR5LXAhsLOq9lfVAWAnsLpte1VV7aqZPyJ186xjzXUOSdKIzPeZxBlV9Vhbfhw4oy0vAh4dGLe31Q5X3ztH/XDneJYk65NMJpmcnp6ex8eRJM1l6AfX7QrgBf1Tskc6R1VtqqoVVbViYuJZM7gkSfM035B4ot0qor0/2er7gCUD4xa32uHqi+eoH+4ckqQRmW9IbAMOzVBaB2wdqK9ts5xWAk+1W0Y7gAuSnNoeWF8A7Gjbnk6yss1qWjvrWHOdQ5I0Ikf8Ml2SzwJvB05PspeZWUrXALcmuQJ4GLikDd8OvAOYAn4EXA5QVfuTfBTY3cZdVVWHHoa/j5kZVCcBX2ovDnMO6XmxdMMXx93CC+o717xz3C3oJeCIIVFVl3U2rZpjbAFXdo6zGdg8R30SOHuO+vfmOockaXT8xrUkqcuQkCR1GRKSpC5DQpLUZUhIkroMCUlSlyEhSeoyJCRJXYaEJKnLkJAkdRkSkqQuQ0KS1GVISJK6DAlJUpchIUnqMiQkSV2GhCSpy5CQJHUZEpKkLkNCktRlSEiSugwJSVKXISFJ6jIkJEldhoQkqcuQkCR1GRKSpC5DQpLUZUhIkroMCUlSlyEhSeoyJCRJXYaEJKlrqJBI8udJ9iS5N8lnk/xSkjOT3JFkKsnnkpzQxp7Y1qfa9qUDx/lwqz+Q5MKB+upWm0qyYZheJUlHb94hkWQR8EFgRVWdDSwALgWuBa6rqtcDB4Ar2i5XAAda/bo2jiTL235nAauBTyZZkGQB8AngImA5cFkbK0kakWFvNy0ETkqyEHgF8BhwPnBb274FuLgtr2nrtO2rkqTVb6mqn1TVt4Ep4Nz2mqqqh6rqGeCWNlaSNCLzDomq2gf8LfAIM+HwFHAn8P2qOtiG7QUWteVFwKNt34Nt/GsG67P26dUlSSMyzO2mU5n5l/2ZwC8Dr2TmdtHIJVmfZDLJ5PT09DhakKSXpGFuN/0e8O2qmq6q/wU+D7wNOKXdfgJYDOxry/uAJQBt+6uB7w3WZ+3Tqz9LVW2qqhVVtWJiYmKIjyRJGjRMSDwCrEzyivZsYRVwH/AV4N1tzDpga1ve1tZp279cVdXql7bZT2cCy4CvA7uBZW221AnMPNzeNkS/kqSjtPDIQ+ZWVXckuQ34BnAQuAvYBHwRuCXJx1rtxrbLjcCnk0wB+5n5pU9V7UlyKzMBcxC4sqp+CpDk/cAOZmZOba6qPfPtV5J09OYdEgBVtRHYOKv8EDMzk2aP/THwns5xrgaunqO+Hdg+TI+SpPnzG9eSpC5DQpLUZUhIkroMCUlSlyEhSeoyJCRJXYaEJKnLkJAkdRkSkqQuQ0KS1GVISJK6DAlJUpchIUnqMiQkSV2GhCSpy5CQJHUZEpKkLkNCktRlSEiSugwJSVKXISFJ6jIkJEldhoQkqcuQkCR1GRKSpC5DQpLUZUhIkroMCUlSlyEhSeoyJCRJXYaEJKnLkJAkdRkSkqSuoUIiySlJbkvyrST3J3lrktOS7EzyYHs/tY1NkuuTTCW5O8k5A8dZ18Y/mGTdQP0tSe5p+1yfJMP0K0k6OsNeSXwc+LeqegPwRuB+YANwe1UtA25v6wAXAcvaaz1wA0CS04CNwHnAucDGQ8HSxrx3YL/VQ/YrSToK8w6JJK8Gfhe4EaCqnqmq7wNrgC1t2Bbg4ra8Bri5ZuwCTknyWuBCYGdV7a+qA8BOYHXb9qqq2lVVBdw8cCxJ0ggMcyVxJjAN/FOSu5J8KskrgTOq6rE25nHgjLa8CHh0YP+9rXa4+t456pKkERkmJBYC5wA3VNWbgR/yi1tLALQrgBriHM9JkvVJJpNMTk9Pv9Cnk6TjxjAhsRfYW1V3tPXbmAmNJ9qtItr7k237PmDJwP6LW+1w9cVz1J+lqjZV1YqqWjExMTHER5IkDZp3SFTV48CjSX69lVYB9wHbgEMzlNYBW9vyNmBtm+W0Eniq3ZbaAVyQ5NT2wPoCYEfb9nSSlW1W09qBY0mSRmDhkPt/APhMkhOAh4DLmQmeW5NcATwMXNLGbgfeAUwBP2pjqar9ST4K7G7jrqqq/W35fcBNwEnAl9pLkjQiQ4VEVX0TWDHHplVzjC3gys5xNgOb56hPAmcP06Mkaf78xrUkqcuQkCR1GRKSpC5DQpLUZUhIkroMCUlSlyEhSeoyJCRJXYaEJKnLkJAkdRkSkqQuQ0KS1GVISJK6DAlJUpchIUnqMiQkSV2GhCSpy5CQJHUZEpKkLkNCktRlSEiSugwJSVKXISFJ6jIkJEldhoQkqcuQkCR1GRKSpC5DQpLUZUhIkroMCUlSlyEhSeoyJCRJXYaEJKlr6JBIsiDJXUm+0NbPTHJHkqkkn0tyQquf2Nan2valA8f4cKs/kOTCgfrqVptKsmHYXiVJR+f5uJL4EHD/wPq1wHVV9XrgAHBFq18BHGj169o4kiwHLgXOAlYDn2zBswD4BHARsBy4rI2VJI3IUCGRZDHwTuBTbT3A+cBtbcgW4OK2vKat07avauPXALdU1U+q6tvAFHBue01V1UNV9QxwSxsrSRqRYa8k/h74S+Bnbf01wPer6mBb3wssasuLgEcB2van2vif12ft06tLkkZk3iGR5F3Ak1V15/PYz3x7WZ9kMsnk9PT0uNuRpJeMYa4k3gb8fpLvMHMr6Hzg48ApSRa2MYuBfW15H7AEoG1/NfC9wfqsfXr1Z6mqTVW1oqpWTExMDPGRJEmD5h0SVfXhqlpcVUuZefD85ar6Q+ArwLvbsHXA1ra8ra3Ttn+5qqrVL22zn84ElgFfB3YDy9psqRPaObbNt19J0tFbeOQhR+2vgFuSfAy4C7ix1W8EPp1kCtjPzC99qmpPkluB+4CDwJVV9VOAJO8HdgALgM1VtecF6FeS1PG8hERVfRX4alt+iJmZSbPH/Bh4T2f/q4Gr56hvB7Y/Hz1Kko6e37iWJHUZEpKkLkNCktRlSEiSugwJSVKXISFJ6jIkJEldhoQkqcuQkCR1GRKSpC5DQpLUZUhIkroMCUlSlyEhSeoyJCRJXYaEJKnLkJAkdRkSkqQuQ0KS1GVISJK6DAlJUpchIUnqMiQkSV2GhCSpy5CQJHUZEpKkLkNCktRlSEiSugwJSVKXISFJ6jIkJEldhoQkqcuQkCR1GRKSpK55h0SSJUm+kuS+JHuSfKjVT0uyM8mD7f3UVk+S65NMJbk7yTkDx1rXxj+YZN1A/S1J7mn7XJ8kw3xYSdLRGeZK4iDwF1W1HFgJXJlkObABuL2qlgG3t3WAi4Bl7bUeuAFmQgXYCJwHnAtsPBQsbcx7B/ZbPUS/kqSjNO+QqKrHquobbfl/gPuBRcAaYEsbtgW4uC2vAW6uGbuAU5K8FrgQ2FlV+6vqALATWN22vaqqdlVVATcPHEuSNALPyzOJJEuBNwN3AGdU1WNt0+PAGW15EfDowG57W+1w9b1z1Oc6//okk0kmp6enh/oskqRfGDokkpwM/AvwZ1X19OC2dgVQw57jSKpqU1WtqKoVExMTL/TpJOm4MVRIJHk5MwHxmar6fCs/0W4V0d6fbPV9wJKB3Re32uHqi+eoS5JGZJjZTQFuBO6vqr8b2LQNODRDaR2wdaC+ts1yWgk81W5L7QAuSHJqe2B9AbCjbXs6ycp2rrUDx5IkjcDCIfZ9G/BHwD1JvtlqHwGuAW5NcgXwMHBJ27YdeAcwBfwIuBygqvYn+Siwu427qqr2t+X3ATcBJwFfai9J0ojMOySq6mtA73sLq+YYX8CVnWNtBjbPUZ8Ezp5vj5Kk4fiNa0lSlyEhSeoyJCRJXYaEJKnLkJAkdRkSkqQuQ0KS1GVISJK6DAlJUpchIUnqMiQkSV2GhCSpy5CQJHUZEpKkLkNCktRlSEiSugwJSVLXMP99qYClG7447hZeUN+55p3jbkHSGHklIUnqMiQkSV2GhCSpy5CQJHUZEpKkLkNCktRlSEiSugwJSVKXISFJ6jIkJEldhoQkqcuQkCR1GRKSpC5DQpLUZUhIkrqO+ZBIsjrJA0mmkmwYdz+SdDw5pkMiyQLgE8BFwHLgsiTLx9uVJB0/jumQAM4Fpqrqoap6BrgFWDPmniTpuHGs//eli4BHB9b3AufNHpRkPbC+rf4gyQMj6G1cTge+O6qT5dpRnem44M/uxW2kP78x+JW5isd6SDwnVbUJ2DTuPkYhyWRVrRh3Hzp6/uxe3I7Xn9+xfrtpH7BkYH1xq0mSRuBYD4ndwLIkZyY5AbgU2DbmniTpuHFM326qqoNJ3g/sABYAm6tqz5jbGrfj4rbaS5Q/uxe34/Lnl6oadw+SpGPUsX67SZI0RoaEJKnLkJAkdRkS0gskyRuSrEpy8qz66nH1JB0tQ+JFKsnl4+5BfUk+CGwFPgDcm2Twz8n8zXi60nOR5OQkVyXZk+SpJNNJdiX543H3Ng7ObnqRSvJIVb1u3H1obknuAd5aVT9IshS4Dfh0VX08yV1V9eaxNqiuJFuBfwX+HbgEeCUzfzfur4F9VfWRMbY3cobEMSzJ3b1NwK9V1Ymj7EfPXZI9VXXWwPrJzATFfcD5VfWmsTWnw0ryX1X1xoH13VX1W0leBtxXVW8YY3sjd0x/mU6cAVwIHJhVD/Cfo29HR+GJJG+qqm8CtCuKdwGbgd8cb2s6gh8m+e2q+lq7TbgfoKp+liRj7m3kDIlj2xeAkw/9ohmU5Kujb0dHYS1wcLBQVQeBtUn+YTwt6Tn6U+AfkywD9gB/ApBkgpn/3+a44u0mSZolya8Cf8DMHxg9CPw38M9V9fRYGxsDZzdJ0oA2M+0G4ERgRXtfAuxK8vYxtjYWXklI0oA2M+1NVfXTJK8AtlfV25O8Dth6vM1M80pCkp7t0PPaE4GTAarqEeDlY+toTHxwLUn/36eA3UnuAH4HuBZ+/uB6/zgbGwdvN0nSLEnOAn4DuLeqvjXufsbJkJAkdflMQpLUZUhIkroMCUlSlyEhSeoyJCRJXf8HSk5eT+r4A4QAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J0wBk73E2tnw"
      },
      "source": [
        "X = scaler.fit_transform(X)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8Oki3aVWLpDA"
      },
      "source": [
        "X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.5, random_state=0)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "geP3MNnuLqn3"
      },
      "source": [
        "x_train=X_train.reshape(X_train.shape[0],len(X_train[0]),1)\n",
        "x_test=X_test.reshape(X_test.shape[0],len(X_test[0]),1)\n",
        "\n",
        "y_train=to_categorical(y_train)\n",
        "y_test=to_categorical(y_test)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eh5jTGehL0dk"
      },
      "source": [
        "def label(my_list):\n",
        "  import numpy as np\n",
        "  my_array=np.array(my_list)\n",
        "  p=np.zeros(my_array.shape)\n",
        "  b=my_array.max(-1)\n",
        "  condition = my_array == b[..., np.newaxis]\n",
        "  c = np.where(condition, 1, 0)\n",
        "  final=np.multiply(c, my_array)\n",
        "  #my_sum=np.sum(final,axis=0)\n",
        "  labels=np.argmax(final, axis=1)\n",
        "  return labels"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DBGGK67_qpHD"
      },
      "source": [
        "Input Perturbation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DswdkVYLqtl2"
      },
      "source": [
        "n=5000\n",
        "xtrain=x_train[0:n]\n",
        "ytrain=y_train[0:n]\n",
        "xtest=x_test[0:n]\n",
        "ytest=y_test[0:n]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dzmET5G8rujv"
      },
      "source": [
        "#define DNN model\n",
        "def Model(xtrain_agg,ytrain_agg,x_test,y_test):\n",
        "  #For non-private Settings\n",
        "   from keras.regularizers import l2\n",
        "   opt = tf.keras.optimizers.Adam(learning_rate=.01)\n",
        "   model_np = tf.keras.Sequential([\n",
        "   #tf.keras.layers.Flatten(),\n",
        "   tf.keras.layers.Dense(64,\n",
        "                           activation='relu',\n",
        "                           input_shape=(18,1)),\n",
        "   #tf.keras.layers.MaxPool2D(2, 1),\n",
        "   tf.keras.layers.Dense(64,\n",
        "                           activation='relu'),\n",
        "   #tf.keras.layers.MaxPool2D(2, 1),\n",
        "   tf.keras.layers.Flatten(),\n",
        "   #tf.keras.layers.Dense(50,activation='relu'),\n",
        "   #tf.keras.layers.Dense(25, activation='relu'),\n",
        "   tf.keras.layers.Dense(2, activation='softmax')\n",
        "       ])\n",
        "   loss=tf.keras.losses.CategoricalCrossentropy(\n",
        "    from_logits=False,\n",
        "    label_smoothing=0,\n",
        "    axis=-1,\n",
        "    reduction=\"auto\",\n",
        "    name=\"categorical_crossentropy\",\n",
        ")\n",
        "   model_np.compile(optimizer=opt, loss=loss, metrics=['accuracy'])\n",
        "   model_np.fit(xtrain_agg, ytrain_agg,\n",
        "          epochs=300,\n",
        "          validation_data=(x_test[0:500], y_test[0:500]),\n",
        "          batch_size=250)\n",
        "   return model_np"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mq7Feh8or3Si"
      },
      "source": [
        "def calc_labelAcc(Model,xtest,ytest):\n",
        "  from sklearn.metrics import accuracy_score\n",
        "  ypred=Model.predict(xtest)\n",
        "  acc=accuracy_score(label(ytest), label(ypred))\n",
        "  return acc"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fQaI9AVGr6r-"
      },
      "source": [
        "model_np=Model(xtrain,ytrain,xtest,ytest)\n",
        "acc_np=calc_labelAcc(model_np,xtest,ytest)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_fhVi6uasH_G"
      },
      "source": [
        "def steps(L,E,n):\n",
        "  q=L/n\n",
        "  T=E/q\n",
        "  return T"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eiiFe3hesJQE"
      },
      "source": [
        "ep=[0.01, 0.1,1,10,100,1000,10000]\n",
        "ni=len(ep)\n",
        "acc_in=np.zeros(ni)\n",
        "for i in range(ni):\n",
        "  epi=ep[i]\n",
        "  delta=.0001\n",
        "  c=5\n",
        "  L=250\n",
        "  E=300\n",
        "  T=steps(L,E,n)\n",
        "  print(T)\n",
        "  G=3*np.log(n)\n",
        "  z1=c*(G**2)*T*np.log(1/delta)\n",
        "  z2=n*(n-1)*(epi**2)\n",
        "  sigma=z1/z2\n",
        "  nsi=np.random.normal(loc=0.0, scale=sigma)\n",
        "  print(nsi)\n",
        "  X_in=xtrain+nsi\n",
        "  model_in=Model(X_in,ytrain,xtest,ytest)\n",
        "  acc_in[i]=calc_labelAcc(model_in,xtest,ytest)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Uw8ck9l5vMHl"
      },
      "source": [
        "util_in=acc_np-acc_in\n",
        "util_in"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bwEg1popqq4X"
      },
      "source": [
        "Gradient Perturbation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8eyhzmaWL34i"
      },
      "source": [
        "from keras.models import Sequential\n",
        "from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D\n",
        "from keras.optimizers import RMSprop\n",
        "from keras.preprocessing.image import ImageDataGenerator\n",
        "from keras.callbacks import ReduceLROnPlateau\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_C6CCoNBL7AR"
      },
      "source": [
        "def DP_SGD(noise_multiplier,x_train,y_train,x_test,y_test):\n",
        "   epochs = 300\n",
        "   batch_size = 250\n",
        "   l2_norm_clip = 1.5\n",
        "   #noise_multiplier = .88\n",
        "   num_microbatches = 125\n",
        "   learning_rate = 0.01\n",
        "   x_train_s=x_train\n",
        "   y_train_s=y_train\n",
        "   x_test=x_test\n",
        "   y_test_s=y_test\n",
        "   n=len(y_train)\n",
        "   #noise_multiplier=.5\n",
        "   if batch_size % num_microbatches != 0:\n",
        "     raise ValueError('Batch size should be an integer multiple of the number of microbatches')\n",
        "   \n",
        "   eps=compute_dp_sgd_privacy.compute_dp_sgd_privacy(n=n, batch_size=250, noise_multiplier=noise_multiplier, epochs=epochs, delta=1e-4)\n",
        "   print(eps)\n",
        "\n",
        "   model = tf.keras.Sequential([\n",
        "   tf.keras.layers.Dense(64,\n",
        "                           activation='relu',\n",
        "                           input_shape=(18,1)),\n",
        "   tf.keras.layers.Dense(64,\n",
        "                           activation='relu'),\n",
        "   #tf.keras.layers.MaxPool2D(2, 1),\n",
        "   tf.keras.layers.Flatten(),\n",
        "   tf.keras.layers.Dense(2, activation='softmax')\n",
        "       ])\n",
        "   print(\"compplete\")\n",
        "   optimizer = DPAdamGaussianOptimizer(\n",
        "            l2_norm_clip=l2_norm_clip,\n",
        "            noise_multiplier=noise_multiplier,\n",
        "            num_microbatches=num_microbatches,\n",
        "            learning_rate=learning_rate)\n",
        "   \n",
        "   loss = tf.keras.losses.CategoricalCrossentropy(\n",
        "    from_logits=True, reduction=tf.losses.Reduction.NONE)\n",
        "   model.compile(optimizer=optimizer, loss=loss, metrics=['accuracy'])\n",
        "   model.fit(x_train_s, y_train_s,\n",
        "          epochs=epochs,\n",
        "          validation_data=(x_test[0:1000], y_test_s[0:1000]),\n",
        "          batch_size=batch_size)\n",
        "   return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wang8EkKMGTh"
      },
      "source": [
        "#model=SGD_lg(1,x_train[0:15000],trainY[0:15000],x_test,testY)\n",
        "def calc_labelAcc(Model,xtest,ytest):\n",
        "  from sklearn.metrics import accuracy_score\n",
        "  ypred=Model.predict_classes(xtest)\n",
        "  acc=accuracy_score(label(ytest), ypred)\n",
        "  return acc"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tbJuP6O0MQ1H"
      },
      "source": [
        "###training and testing data\n",
        "n=5000\n",
        "xtrain_agg=x_train[0:n]\n",
        "ytrain_agg=y_train[0:n]\n",
        "#ytrain_agg=ytrain_agg.to_numpy()\n",
        "#ytrain_pred=model_np.predict_proba(xtrain_agg)\n",
        "xtest_agg=x_test[0:n]\n",
        "ytest_agg=y_test[0:n]\n",
        "#ytest_agg=ytest_agg.to_numpy()\n",
        "#ytest_pred=model_np.predict_proba(xtest_agg)\n",
        "target_train_agg = (xtrain_agg,ytrain_agg)\n",
        "target_test_agg = (xtest_agg,ytest_agg)\n",
        "#target_train_data_agg, target_test_data_agg = sample_data(target_train_agg, target_test_agg, NUM_TARGET)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s2JrXM5TMWX8"
      },
      "source": [
        "noise_mul=[790,114,13.6,2.015,.64,.344,.2213]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IGeMPKtVMarz"
      },
      "source": [
        "!pip install -U -q PyDrive\n",
        "ep1=np.zeros(len(noise_mul))  #calculate epsilon\n",
        "acc_sgd= np.zeros(len(noise_mul))\n",
        "model_index =0\n",
        "for i in range(7):\n",
        "  model=DP_SGD(noise_mul[i],xtrain_agg,ytrain_agg,xtest_agg,ytest_agg)\n",
        "  acc_sgd[i]=calc_labelAcc(model,xtest_agg,ytest_agg)\n",
        "   globals()['model_sgd%s' % i]=model\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mAV-zzXW_oOH"
      },
      "source": [
        "utility_sgd=np_sgd-acc_sgd\n",
        "utility_sgd"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zWnuNPLWBZvZ"
      },
      "source": [
        "array([0.116 , 0.0972, 0.0698, 0.0386, 0.031 , 0.034 , 0.0352])"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dq0FSKsLM1ke"
      },
      "source": [
        "#For non-private Settings\n",
        "from keras.regularizers import l2\n",
        "opt = tf.keras.optimizers.Adam(learning_rate=.01)\n",
        "model_np = tf.keras.Sequential([\n",
        "   #tf.keras.layers.Flatten(),\n",
        "   tf.keras.layers.Dense(64,\n",
        "                           activation='relu',\n",
        "                           input_shape=(18,1)),\n",
        "   #tf.keras.layers.MaxPool2D(2, 1),\n",
        "   tf.keras.layers.Dense(64,\n",
        "                           activation='relu'),\n",
        "   #tf.keras.layers.MaxPool2D(2, 1),\n",
        "   tf.keras.layers.Flatten(),\n",
        "   #tf.keras.layers.Dense(50,activation='relu'),\n",
        "   #tf.keras.layers.Dense(25, activation='relu'),\n",
        "   tf.keras.layers.Dense(2, activation='softmax')\n",
        "       ])\n",
        "loss = tf.keras.losses.CategoricalCrossentropy(\n",
        "    from_logits=True, reduction=tf.losses.Reduction.NONE)\n",
        "model_np.compile(optimizer=opt, loss=loss, metrics=['accuracy'])\n",
        "model_np.fit(xtrain_agg, ytrain_agg,\n",
        "          epochs=300,\n",
        "          validation_data=(x_test[0:500], y_test[0:500]),\n",
        "          batch_size=250)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pMM14wWNNYTu"
      },
      "source": [
        "np_sgd=calc_labelAcc(model_np,xtest_agg,ytest_agg)\n",
        "np_sgd"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CbVF351yiRyQ"
      },
      "source": [
        "PATE"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UEFkJmFkiPdB"
      },
      "source": [
        "def build_cnn_model():\n",
        "    import keras\n",
        "    from keras.models import Sequential\n",
        "    from keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout\n",
        "    num_class=2\n",
        "    # build the model\n",
        "    model = Sequential()\n",
        "    model.add(Dense(64, activation='relu', input_shape=(len(x_train[0]),1)))\n",
        "    model.add(Dense(64, activation='relu'))\n",
        "    model.add(Flatten())\n",
        "    #model.add(Dense(50, activation='relu'))\n",
        "    #model.add(Dense(64, activation='tanh'))\n",
        "    if num_class==1:\n",
        "        model.add(Dense(num_class, activation='sigmoid'))    \n",
        "    else:\n",
        "        model.add(Dense(num_class, activation='softmax')) \n",
        "    model.summary()\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cyCfMZQfii5i",
        "outputId": "b5445f31-b0ff-4225-a2a7-263246703598"
      },
      "source": [
        "#split datasets into multiple teacher\n",
        "#Divide the images into 5 \n",
        "# split x_train to 10 disjoint datasets\n",
        "#store each dataset variable name is the list Xtrain\n",
        "import numpy as np\n",
        "M=30\n",
        "j=0\n",
        "k=len(xtrain_agg)/M\n",
        "#print(x_train)\n",
        "\n",
        "for x in range(0,M):\n",
        "             globals()['x_train_split%s' % x]=xtrain_agg[int(j):int(k+j)]\n",
        "             globals()['y_train_split%s' % x]=ytrain_agg[int(j):int(k+j)]\n",
        "             j=k+j\n",
        "             #print(j)\n",
        "             \n",
        "print(x_train_split29.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(167, 18, 1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "HjozWOAzioUL"
      },
      "source": [
        "for iter in range(M):\n",
        "  model=build_cnn_model()\n",
        "  model.compile(optimizer='Adam', loss='categorical_crossentropy', metrics=['accuracy'])\n",
        "  model.fit(globals()['x_train_split%s' % iter], globals()['y_train_split%s' % iter], batch_size=250, epochs=300, verbose=1, shuffle=True)\n",
        "  globals()['model_pate%s' % iter]=model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q8qOYLAhjSms"
      },
      "source": [
        "#compute label\n",
        "def label(my_list):\n",
        "  import numpy as np\n",
        "  my_array=np.array(my_list)\n",
        "  p=np.zeros(my_array.shape)\n",
        "  b=my_array.max(-1)\n",
        "  condition = my_array == b[..., np.newaxis]\n",
        "  c = np.where(condition, 1, 0)\n",
        "  final=np.multiply(c, my_array)\n",
        "  #my_sum=np.sum(final,axis=0)\n",
        "  labels=np.argmax(final, axis=1)\n",
        "  return labels"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UsY0DvOIkl0_"
      },
      "source": [
        "#function for vote counting\n",
        "def vote(my_list):\n",
        "  import numpy as np\n",
        "  my_array=np.array(my_list)\n",
        "  p=np.zeros(my_array.shape)\n",
        "  b=my_array.max(-1)\n",
        "  condition = my_array == b[..., np.newaxis]\n",
        "  c = np.where(condition, 1, 0)\n",
        "  final=np.multiply(c, my_array)\n",
        "  #my_sum=np.sum(final,axis=0)\n",
        "  labels=np.argmax(final, axis=1)\n",
        "  return c"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zFbvkovikqBC"
      },
      "source": [
        "sum=np.zeros([len(y_test),2])\n",
        "for i in range(M):\n",
        "  New_model=globals()['model_pate%s' % i]\n",
        "  y=np.zeros(y_test.shape)\n",
        "  y=New_model.predict_proba(x_test)\n",
        "  y=vote(y)\n",
        "  sum=y+sum\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eHKLXvgck0C0"
      },
      "source": [
        "#create the labels\n",
        "from sklearn.metrics import accuracy_score\n",
        "lab=label(sum)\n",
        "#clear accuracy\n",
        "y_true=label(y_test)\n",
        "np_pate=accuracy_score(y_true,lab)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BfNe102Dli92"
      },
      "source": [
        "ep=[.01,.1,1,10,100,1000,10000]\n",
        "client_acc=np.zeros(len(ep))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bh7icXnolkQE"
      },
      "source": [
        "#add noise \n",
        "def add_noise_sum(noise,sum,experiment,ypred):\n",
        "  sum_s=sum\n",
        "  predt=np.zeros(experiment)\n",
        "  for i1 in range(experiment):\n",
        "    sum_s=np.zeros(sum.shape)\n",
        "    sum_f=sum+np.random.laplace(loc=0.0, scale=1/noise)\n",
        "    sum2=label(sum_f)\n",
        "    predt[i1]=accuracy_score(ypred,sum2)\n",
        "    #print(i)\n",
        "  pred=np.average(predt)\n",
        "  #print(predt)\n",
        "  #print(pred)\n",
        "  return pred"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M0wRiemmlqNH"
      },
      "source": [
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.metrics import average_precision_score\n",
        "#Noise after Aggregation Method\n",
        "for i in range(0,len(ep)):\n",
        "                          sum_s=np.zeros(sum.shape)\n",
        "                          sum_n=sum\n",
        "                          #sum_s=sum_n+np.random.laplace(loc=0.0, scale=1/noise[i])\n",
        "                          predF=add_noise_sum(ep[i],sum_n,100,y_true)\n",
        "                          client_acc[i]=predF\n",
        "print(client_acc)\n",
        "print(ep)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XlrX3Ue-EId6"
      },
      "source": [
        "utility_pate=np_pate-client_acc\n",
        "utility_pate"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sk4rVS1VEOWZ"
      },
      "source": [
        "array([0.13756093, 0.01519654, 0.        , 0.        , 0.        ,\n",
        "       0.        , 0.        ])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TSFpzhe0lxC7"
      },
      "source": [
        "Graph"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zlE6kIB4lycO"
      },
      "source": [
        "import pandas as pd \n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns #grafikleştirme için\n",
        "import matplotlib.pyplot as plt \n",
        "from google.colab import files\n",
        "test1 = plt.figure()\n",
        "#plt.semilogx(ep,non_p-acc_obj,color=\"black\",marker='3',label='Objective Perturbation',linewidth=1.5)\n",
        "plt.semilogx(ep,util_in,color=\"black\",marker='^',label='Input Perturbation',linewidth=1.5)\n",
        "plt.semilogx(ep,util_sgd,color=\"red\",marker='*',label='Gradient Perturbation',linewidth=1.5)\n",
        "#plt.semilogx(ep,np_out-acc_out,color=\"black\",marker='+',linestyle=\"--\",label='Output Perturbation',linewidth=1.5)\n",
        "plt.semilogx(ep,util_pate,color=\"brown\",marker='o',label='Prediction Perturbation',linewidth=1.5)\n",
        "#plt.semilogx(ep,non_p-acc_in,color=\"orange\",marker='.',linestyle=\"--\",label='Input',linewidth=1.5)\n",
        "#plt.plot(ep,non_p,color=\"red\",marker='*',linestyle=\"--\",label='Non-Private Model',linewidth=2)\n",
        "plt.legend(loc=1,fontsize=12)\n",
        "plt.xlabel(\"Privacy Budget($\\epsilon$)\",fontsize=13)\n",
        "plt.ylabel(\"Utility Loss\",fontsize=15)\n",
        "#plt.xscale('symlog', linthreshy=0.1)\n",
        "#plt.ylim([-.1,1])\n",
        "plt.xticks(size = 10)\n",
        "plt.yticks(size = 8)\n",
        "plt.ylim([-.05,1])\n",
        "#y.set_color(\"black\")\n",
        "plt.rcParams[\"axes.edgecolor\"] = \"black\"\n",
        "plt.rcParams[\"axes.linewidth\"] = 1\n",
        "plt.rcParams['axes.facecolor'] = 'white'\n",
        "#test1.set_facecolor('white')\n",
        "test1.show()\n",
        "test1.savefig('DNN_perturb_acc_Covid.pdf')\n",
        "files.download('DNN_perturb_acc_Covid.pdf')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Eo5Q3lyim3LN"
      },
      "source": [
        "Attack The dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hjcEwIw_m4_A"
      },
      "source": [
        "#Assign necessary variables for attacking the model\n",
        "import argparse\n",
        "import csv\n",
        "import numpy as np\n",
        "from sklearn.utils import resample, shuffle\n",
        "\n",
        "import keras\n",
        "import numpy as np\n",
        "from sklearn.utils import resample\n",
        "\n",
        "LEARNING_RATE = 0.01\n",
        "EPOCH = 300\n",
        "#EPOCH = 30\n",
        "DATA_SIZE = 10000\n",
        "TRAINING_SIZE = 5000\n",
        "TEST_SIZE = 5000\n",
        "NUM_TARGET = 1\n",
        "#NUM_SHADOW = 100\n",
        "NUM_SHADOW = 10\n",
        "IN = 1\n",
        "OUT = 0\n",
        "VERBOSE = 1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nE5QyBMcnGpM"
      },
      "source": [
        "#call required libraries\n",
        "from tensorflow.keras.datasets import cifar10\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import Dense, Flatten, Conv2D, MaxPooling2D\n",
        "from tensorflow.keras.losses import sparse_categorical_crossentropy\n",
        "from tensorflow.keras.optimizers import Adam\n",
        "import matplotlib.pyplot as plt\n",
        "import tensorflow as tf\n",
        "from tensorflow.keras import datasets, layers, models\n",
        "import matplotlib.pyplot as plt\n",
        "from keras.utils import to_categorical \n",
        "import sys\n",
        "from matplotlib import pyplot\n",
        "from keras.datasets import cifar10\n",
        "from keras.utils import to_categorical\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Conv2D\n",
        "from keras.layers import MaxPooling2D\n",
        "from keras.layers import Dense\n",
        "from keras.layers import Flatten\n",
        "from keras.layers import Dropout\n",
        "from keras.optimizers import SGD"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ifLAWqaonKm0"
      },
      "source": [
        "#Define model configuaration\n",
        "# Model configuration\n",
        "batch_size = 250\n",
        "#img_width, img_height, img_num_channels = 32, 32, 3\n",
        "#loss_function = sparse_categorical_crossentropy\n",
        "no_classes = 2\n",
        "no_epochs = 250\n",
        "optimizer = Adam()\n",
        "validation_split = 0.2\n",
        "verbosity = 0"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "je8Tkv4InY6l"
      },
      "source": [
        "def build_cnn_model():\n",
        "    import keras\n",
        "    from keras.models import Sequential\n",
        "    from keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout\n",
        "    num_class=2\n",
        "    # build the model\n",
        "    model = Sequential()\n",
        "    model.add(Dense(64, activation='relu', input_shape=(len(x_train[0]),1)))\n",
        "    model.add(Dense(64, activation='relu'))\n",
        "    model.add(Flatten())\n",
        "    #model.add(Dense(50, activation='relu'))\n",
        "    #model.add(Dense(64, activation='tanh'))\n",
        "    if num_class==1:\n",
        "        model.add(Dense(num_class, activation='sigmoid'))    \n",
        "    else:\n",
        "        model.add(Dense(num_class, activation='softmax')) \n",
        "    model.summary()\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sW-YO3SpnaWF"
      },
      "source": [
        "#data sampling\n",
        "def sample_data(train_data,test_data,num_sets):\n",
        "    (x_train, y_train), (x_test, y_test) = train_data, test_data\n",
        "    new_x_train, new_y_train = [], []\n",
        "    new_x_test, new_y_test = [], []\n",
        "    for i in range(num_sets):\n",
        "        x_temp, y_temp = resample(x_train, y_train, n_samples=n, random_state=0)\n",
        "        new_x_train.append(x_temp)\n",
        "        new_y_train.append(y_temp)\n",
        "        x_temp, y_temp = resample(x_test, y_test, n_samples=n, random_state=0)\n",
        "        new_x_test.append(x_temp)\n",
        "        new_y_test.append(y_temp)\n",
        "    return (new_x_train, new_y_train), (new_x_test, new_y_test)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m9d5OeN5nfd8"
      },
      "source": [
        "def get_trained_keras_models(keras_model, train_data, test_data, num_models):\n",
        "    (x_train, y_train), (x_test, y_test) = train_data, test_data\n",
        "    models = []\n",
        "    for i in range(num_models):\n",
        "        models.append(keras.models.clone_model(keras_model))\n",
        "        models[i].compile(optimizer='sgd', loss='binary_crossentropy', metrics=['binary_accuracy'])\n",
        "        models[i].fit(x_train[i], y_train[i], batch_size=32, epochs=EPOCH, verbose=VERBOSE, shuffle=True)\n",
        "        score = models[i].evaluate(x_test[i], y_test[i], verbose=VERBOSE)\n",
        "        print('\\n', 'Model ', i, ' test accuracy:', score[1])\n",
        "    return models"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8Sz9ZMuhnoc9"
      },
      "source": [
        "def get_trained_keras_models_target(keras_model, train_data, test_data, num_models):\n",
        "    (x_train, y_train), (x_test, y_test) = train_data, test_data\n",
        "    models = []\n",
        "    for i in range(num_models):\n",
        "        models.append(keras.models.clone_model(keras_model))\n",
        "        models[i].compile(optimizer='sgd', loss='binary_crossentropy', metrics=['binary_accuracy'])\n",
        "        models[i].fit(x_train[i], y_train[i], batch_size=32, epochs=EPOCH, verbose=VERBOSE, shuffle=True)\n",
        "        score = models[i].evaluate(x_test[i], y_test[i], verbose=VERBOSE)\n",
        "        print('\\n', 'Model ', i, ' test accuracy:', score[1])\n",
        "    return models"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6IwEk2wRny7P"
      },
      "source": [
        "#collect the attack dataset from shadow models\n",
        "def get_attack_dataset(models, train_data, test_data, num_models, data_size):\n",
        "    # generate dataset for the attack model\n",
        "    (x_train, y_train), (x_test, y_test) = train_data, test_data\n",
        "    num_classes = len(y_train[0][0])\n",
        "    x_data, y_data = [[] for i in range(num_classes)], [[] for i in range(num_classes)]\n",
        "    for i in range(num_models):\n",
        "        # IN data\n",
        "        x_temp, y_temp = resample(x_train[i], y_train[i], n_samples=data_size, random_state=0)\n",
        "        for j in range(data_size):\n",
        "            y_idx = np.argmax(y_temp[j])\n",
        "            x_data[y_idx].append(models[i].predict(x_temp[j:j+1])[0])\n",
        "            #print(y_idx)\n",
        "            y_data[y_idx].append(IN)\n",
        "            print(\"starts1\",j)\n",
        "        # OUT data\n",
        "        x_temp, y_temp = resample(x_test[i], y_test[i], n_samples=data_size, random_state=0)\n",
        "        for j in range(data_size):\n",
        "            y_idx = np.argmax(y_temp[j])\n",
        "            p=models[i].predict(x_temp[j:j+1])[0]\n",
        "            x_data[y_idx].append(p)\n",
        "            y_data[y_idx].append(OUT)\n",
        "            print(\"starts2\",j)\n",
        "    return x_data, y_data"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Rr-Ou3P6zU2i"
      },
      "source": [
        "#Get the prediction vectors from the DP model\n",
        "def get_attack_dataset_combined(x_train, x_test, train_pred, y_train, y_test, test_pred):\n",
        "    # generate dataset for the attack model\n",
        "    #(x_train, y_train), (x_test, y_test) = train_data, test_data\n",
        "    data_size=len(y_train)\n",
        "    num_class = 2\n",
        "    x_data, y_data = [[] for _ in range(num_class)], [[] for _ in range(num_class)]\n",
        "    #for i in range(num_models):\n",
        "        # IN data\n",
        "    #x_temp, y_temp = resample(x_train, y_train, n_samples=data_size, random_state=0)\n",
        "    for j in range(data_size):\n",
        "            y_idx = np.argmax(y_train[j])\n",
        "            x_data[y_idx].append(train_pred[j])\n",
        "            #print(train_pred[j])\n",
        "            #x_data[y_idx].append(models.predict(x_temp[j:j+1])[0])\n",
        "            y_data[y_idx].append(IN)\n",
        "        # OUT data\n",
        "    #x_temp, y_temp = resample(x_test, y_test, n_samples=data_size, random_state=0)\n",
        "    for j in range(data_size):\n",
        "            y_idx = np.argmax(y_train[j])\n",
        "            #x_data[y_idx].append(models.predict(x_temp[j:j+1])[0])\n",
        "            x_data[y_idx].append(test_pred[j])\n",
        "            y_data[y_idx].append(OUT)\n",
        "    return x_data, y_data"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pt0X6bBrn0lj"
      },
      "source": [
        "#generate the report\n",
        "def get_leakage(models, test_data):\n",
        "    from sklearn.metrics import accuracy_score, precision_score, recall_score\n",
        "    from sklearn.metrics import average_precision_score\n",
        "    from sklearn import metrics\n",
        "    from sklearn.metrics import confusion_matrix\n",
        "    (x_test, y_true) = test_data\n",
        "    acc_scores = []\n",
        "    pre_scores = []\n",
        "    rec_scores = []\n",
        "    fp=np.zeros(len(models))\n",
        "    tp=np.zeros(len(models))\n",
        "    tn=np.zeros(len(models))\n",
        "    fn=np.zeros(len(models))\n",
        "    for i in range(len(models)):\n",
        "        y_pred = models[i].predict(x_test[i])\n",
        "        # _LOG_PRINT(y_pred)\n",
        "        acc_scores.append(accuracy_score(y_true[i], y_pred))\n",
        "        pre_scores.append(average_precision_score(y_true[i], y_pred))\n",
        "        rec_scores.append(recall_score(y_true[i], y_pred))\n",
        "        tn[i], fp[i], fn[i], tp[i] = confusion_matrix(y_true[i], y_pred).ravel()\n",
        "    return np.sum(tn),np.sum(tp),np.sum(fn),np.sum(fp)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WEa3Sc_cn6db"
      },
      "source": [
        "#attack classifier:SVM\n",
        "def get_trained_svm_models(train_data, test_data, num_models):\n",
        "    from sklearn import svm\n",
        "    (x_train, y_train), (x_test, y_test) = train_data, test_data\n",
        "    models = []\n",
        "    for i in range(num_models):\n",
        "        print('Training svm model : ', i)\n",
        "        models.append(svm.SVC(gamma='scale',kernel='linear',verbose=VERBOSE))\n",
        "        models[i].fit(x_train[i], y_train[i])\n",
        "        score = models[i].score(x_test[i],y_test[i])\n",
        "        #print(i)\n",
        "        print('SVM model ', i, 'score : ',score)\n",
        "    return models,score"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ivn_FBGpoDzF"
      },
      "source": [
        "shadow_train = (x_train[n:n*2],y_train[n:n*2])\n",
        "shadow_test = (x_test[n:n*2],y_test[n:n*2])\n",
        "shadow_train_data, shadow_test_data = sample_data(shadow_train, shadow_test, NUM_SHADOW)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Od7JdjkXoHfD"
      },
      "source": [
        "#when we use logistic regression model from sklearn\n",
        "cnn_model=build_cnn_model()\n",
        "shadow_models = get_trained_keras_models(cnn_model,shadow_train_data, shadow_test_data, NUM_SHADOW)\n",
        "attack_train = get_attack_dataset(shadow_models, shadow_train_data, shadow_test_data, NUM_SHADOW, TEST_SIZE)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t5v3TI9ErzCt"
      },
      "source": [
        "def lk(tn,tp,fn,fp):\n",
        "  tpr=(tp/(tp+fn))\n",
        "  fpr=(fp/(fp+tn))\n",
        "  #print(fpr)\n",
        "  return tpr-fpr"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8PFPJvCir2FO"
      },
      "source": [
        "##attack classifier:RF\n",
        "def get_trained_RF_models(train_data, test_data, num_models):\n",
        "    from sklearn.ensemble import RandomForestClassifier\n",
        "    (x_train, y_train), (x_test, y_test) = train_data, test_data\n",
        "    models = []\n",
        "    score=np.zeros(num_models)\n",
        "    #RF=RandomForestClassifier(random_state=0)\n",
        "    for i in range(num_models):\n",
        "        print('Training RF model : ', i)\n",
        "        models.append(RandomForestClassifier(random_state=0))\n",
        "        models[i].fit(x_train[i], y_train[i])\n",
        "        score[i] = models[i].score(x_test[i],y_test[i])\n",
        "        print('Random Forest model ', i, 'score : ',score)\n",
        "    return models, score"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tMVzjnp4sAUb"
      },
      "source": [
        "n=5000\n",
        "xtrain=x_train[0:n]\n",
        "ytrain=y_train[0:n]\n",
        "#ytrain_agg=ytrain_agg.to_numpy()\n",
        "#ytrain_pred=model_np.predict_proba(xtrain_agg)\n",
        "xtest=x_test[0:n]\n",
        "ytest=y_test[0:n]\n",
        "#ytest_agg=ytest_agg.to_numpy()\n",
        "#ytest_pred=model_np.predict_proba(xtest_agg)\n",
        "target_train_agg = (xtrain,ytrain)\n",
        "target_test_agg = (xtest,ytest)\n",
        "target_train_data, target_test_data = sample_data(target_train_agg, target_test_agg, 1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t6F6yzrxtOR7"
      },
      "source": [
        "#non_private\n",
        "target_model = get_trained_keras_models(cnn_model,target_train_data, target_test_data, 1)\n",
        "attack_test_agg=get_attack_dataset(target_model, target_train_data, target_test_data, 1, TEST_SIZE)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kHYMkYacoey5"
      },
      "source": [
        "def get_trained_svm_models(train_data, test_data, num_models):\n",
        "    from sklearn import svm\n",
        "    (x_train, y_train), (x_test, y_test) = train_data, test_data\n",
        "    models = []\n",
        "    k=0\n",
        "    for i in range(num_models):\n",
        "        print('Training svm model : ', i)\n",
        "        models.append(svm.SVC(gamma='scale',kernel='linear',verbose=VERBOSE))\n",
        "        models[i].fit(x_train[i], y_train[i])\n",
        "        score = models[i].score(x_test[i],y_test[i])\n",
        "        print('SVM model ', i, 'score : ',score)\n",
        "        k=k+1\n",
        "        print('this is executable',k)\n",
        "    return models,score"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ESBTM5iotxzb",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3547e338-42e9-41ee-a645-8ab79cec5148"
      },
      "source": [
        "#non_private\n",
        "attack_model,scores = get_trained_RF_models(attack_train,attack_test_agg, 2)\n",
        "#scores=get_score_svm_models(attack_model,attack_train)\n",
        "print(scores)\n",
        "tn,tp,fn,fp=get_leakage(attack_model, attack_test_agg)\n",
        "l1=lk(tn,tp,fn,fp)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training RF model :  0\n",
            "Random Forest model  0 score :  [0.48741754 0.        ]\n",
            "Training RF model :  1\n",
            "Random Forest model  1 score :  [0.48741754 0.5117657 ]\n",
            "[0.48741754 0.5117657 ]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a-03l_IpxHVJ"
      },
      "source": [
        "Privacy Leakage: Input Perturbation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HCVpOMePxS8r"
      },
      "source": [
        "def Input_pert_train(epi,xtrain,ytrain,xtest,ytest):\n",
        "  delta=.0001\n",
        "  c=5\n",
        "  L=250\n",
        "  E=300\n",
        "  T=steps(L,E,n)\n",
        "  print(T)\n",
        "  G=3*np.log(n)\n",
        "  z1=c*(G**2)*T*np.log(1/delta)\n",
        "  z2=n*(n-1)*(epi**2)\n",
        "  sigma=z1/z2\n",
        "  nsi=np.random.normal(loc=0.0, scale=sigma)\n",
        "  #print(nsi)\n",
        "  X_in=xtrain+nsi\n",
        "  target_train_aggin = (X_in,ytrain)\n",
        "  target_test_aggin = (xtest,ytest)\n",
        "  target_train_datain, target_test_datain = sample_data(target_train_aggin, target_test_aggin, 1)\n",
        "  return X_in,target_train_datain,target_test_datain"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RamN_VbCxarf"
      },
      "source": [
        "p=len(ep)\n",
        "lin=np.zeros(p)\n",
        "tn0=np.zeros(p)\n",
        "tp0=np.zeros(p)\n",
        "fn0=np.zeros(p)\n",
        "fp0=np.zeros(p)\n",
        "\n",
        "for i in range(len(ep)):\n",
        "     target_model=[]\n",
        "     X_in,target_train_datain,target_test_datain=Input_pert_train(ep[i],xtrain,ytrain,xtest,ytest)\n",
        "     in_model=Model(X_in,ytrain,xtest,ytest)\n",
        "     target_model.append(in_model)\n",
        "     attack_test_agg=get_attack_dataset(target_model, target_train_datain, target_test_datain, 1, TEST_SIZE)\n",
        "     tn0[i],tp0[i],fn0[i],fp0[i]=get_leakage(attack_model, attack_test_agg)\n",
        "     lin[i]=lk(tn0[i],tp0[i],fn0[i],fp0[i])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3b8ShcGlvRrE"
      },
      "source": [
        "Privacy Leakage:gradient Perturbation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MQAGjlZkvPXQ"
      },
      "source": [
        "p=len(ep)\n",
        "lgd=np.zeros(p)\n",
        "tn2=np.zeros(p)\n",
        "tp2=np.zeros(p)\n",
        "fn2=np.zeros(p)\n",
        "fp2=np.zeros(p)\n",
        "for i in range(len(ep)):\n",
        "     target_model=[]\n",
        "     modelSGD=globals()['model_sgd%s' % i]\n",
        "     target_model.append(modelSGD)\n",
        "     attack_test_agg=get_attack_dataset(target_model, target_train_data, target_test_data, 1, TEST_SIZE)\n",
        "     tn2[i],tp2[i],fn2[i],fp2[i]=get_leakage(attack_model, attack_test_agg)\n",
        "     lgd[i]=max(0,lk(tn2[i],tp2[i],fn2[i],fp2[i]))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3D7zE6Re7hSl"
      },
      "source": [
        "print((lgd))\n",
        "print(tp2)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8m7EEG5-yg3-"
      },
      "source": [
        "PATE"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8fCVBqRwyY9f"
      },
      "source": [
        "def PATE(ep,xtest1,ytest1):\n",
        "            sum_t=np.zeros([len(ytest1),2])\n",
        "            for k1 in range(10):\n",
        "               sum=PATE_sum(xtest1,ytest1)\n",
        "               sum_s=np.zeros(sum.shape)\n",
        "               sum_n=sum\n",
        "               sum_s=sum_n+np.random.laplace(loc=0.0, scale=1/ep)\n",
        "               sum_t=sum_s+sum_t\n",
        "\n",
        "            return sum_t/10\n",
        "          \n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9Z6A0TW-yoH4"
      },
      "source": [
        "def PATE_sum(xtest1,ytest1):\n",
        "  sum=np.zeros([len(ytest1),2])\n",
        "  for iter in range(M):\n",
        "    New_model=globals()['model_pate%s' % iter]\n",
        "    y=np.zeros([len(ytest1),2])\n",
        "    y=New_model.predict_proba(xtest1)\n",
        "  #print(y.shape)\n",
        "    y=vote(y)\n",
        "    sum=y+sum\n",
        "  return sum"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6IKcSwgXyreH"
      },
      "source": [
        "p=len(ep)\n",
        "lpate=np.zeros(p)\n",
        "tn4=np.zeros(p)\n",
        "tp4=np.zeros(p)\n",
        "fn4=np.zeros(p)\n",
        "fp4=np.zeros(p)\n",
        "for i in range(len(ep)):\n",
        "     ytrain_pred=PATE(ep[i],xtrain,ytrain)\n",
        "     ytest_pred=PATE(ep[i],xtest,ytest)\n",
        "     attack_test_agg=get_attack_dataset_combined(xtrain,xtest,ytrain_pred,ytrain,ytest,ytest_pred)\n",
        "     print(\"complete\")\n",
        "     tn4[i],tp4[i],fn4[i],fp4[i]=get_leakage(attack_model, attack_test_agg)\n",
        "     lpate[i]=max(0,lk(tn2[i],tp2[i],fn2[i],fp2[i]))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BlR6zBZHULIE"
      },
      "source": [
        "print((lpate))\n",
        "print(tp4)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i4XHz5Qtz-ng"
      },
      "source": [
        "Produce Graph"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EZGAAZHy0AxW"
      },
      "source": [
        "import pandas as pd \n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns #grafikleştirme için\n",
        "import matplotlib.pyplot as plt \n",
        "from google.colab import files\n",
        "test1 = plt.figure()\n",
        "#plt.semilogx(ep,lobj,color=\"black\",marker='3',label='Objective Perturbation',linewidth=1.5)\n",
        "plt.semilogx(ep,lin,color=\"black\",marker='^',label='Input Perturbation',linewidth=1.5)\n",
        "plt.semilogx(ep,lgd,color=\"red\",marker='*',label='Gradient Perturbation',linewidth=1.5)\n",
        "#plt.semilogx(ep,lout,color=\"black\",marker='+',linestyle=\"--\",label='Output Perturbation',linewidth=1.5)\n",
        "plt.semilogx(ep,lpate,color=\"brown\",marker='o',label='Prediction Perturbation',linewidth=1.5)\n",
        "#plt.semilogx(ep,non_p-acc_in,color=\"orange\",marker='.',linestyle=\"--\",label='Input',linewidth=1.5)\n",
        "#plt.plot(ep,non_p,color=\"red\",marker='*',linestyle=\"--\",label='Non-Private Model',linewidth=2)\n",
        "plt.legend(loc=1,fontsize=12)\n",
        "plt.xlabel(\"Privacy Budget($\\epsilon$)\",fontsize=13)\n",
        "plt.ylabel(\"Privacy Leakage\",fontsize=15)\n",
        "#plt.xscale('symlog', linthreshy=0.1)\n",
        "#plt.ylim([-.1,1])\n",
        "plt.xticks(size = 10)\n",
        "plt.yticks(size = 8)\n",
        "plt.ylim([-.03,.25])\n",
        "#y.set_color(\"black\")\n",
        "plt.rcParams[\"axes.edgecolor\"] = \"black\"\n",
        "plt.rcParams[\"axes.linewidth\"] = 1\n",
        "plt.rcParams['axes.facecolor'] = 'white'\n",
        "#test1.set_facecolor('white')\n",
        "test1.show()\n",
        "test1.savefig('DNN_perturb_leakage_medical.pdf')\n",
        "files.download('DNN_perturb_leakage_medical.pdf')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cq-nn08m4Buf"
      },
      "source": [
        "import pandas as pd \n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns #grafikleştirme için\n",
        "import matplotlib.pyplot as plt \n",
        "from google.colab import files\n",
        "test1 = plt.figure()\n",
        "#plt.semilogx(ep,tp1,color=\"black\",marker='3',label='Objective Perturbation',linewidth=1.5)\n",
        "plt.semilogx(ep,tp0,color=\"black\",marker='^',label='Input Perturbation',linewidth=1.5)\n",
        "plt.semilogx(ep,tp2,color=\"red\",marker='*',label='Gradient Perturbation',linewidth=1.5)\n",
        "#plt.semilogx(ep,tp3,color=\"black\",marker='+',linestyle=\"--\",label='Output Perturbation',linewidth=1.5)\n",
        "plt.semilogx(ep,tp4,color=\"brown\",marker='o',label='Prediction Perturbation',linewidth=1.5)\n",
        "#plt.semilogx(ep,non_p-acc_in,color=\"orange\",marker='.',linestyle=\"--\",label='Input',linewidth=1.5)\n",
        "#plt.plot(ep,non_p,color=\"red\",marker='*',linestyle=\"--\",label='Non-Private Model',linewidth=2)\n",
        "plt.legend(loc=1,fontsize=10)\n",
        "plt.xlabel(\"Privacy Budget($\\epsilon$)\",fontsize=13)\n",
        "plt.ylabel(\"True Revealed Data (True Positive)\",fontsize=12)\n",
        "#plt.xscale('symlog', linthreshy=0.1)\n",
        "#plt.ylim([-.1,1])\n",
        "plt.xticks(size = 10)\n",
        "plt.yticks(size = 8)\n",
        "plt.ylim([0,8000])\n",
        "#y.set_color(\"black\")\n",
        "plt.rcParams[\"axes.edgecolor\"] = \"black\"\n",
        "plt.rcParams[\"axes.linewidth\"] = 1\n",
        "plt.rcParams['axes.facecolor'] = 'white'\n",
        "#test1.set_facecolor('white')\n",
        "test1.show()\n",
        "test1.savefig('DNN_perturb_tp_medical.pdf')\n",
        "files.download('DNN_perturb_tp_medical.pdf')"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}