Lachlan_SMOTE_GAN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d7973263",
   "metadata": {},
   "source": [
    "# Data augmentation using SMOTE-GAN\n",
    "\n",
    "### Ehsan Farahbakhsh, Nathan Wake, R. Ditmar M&uuml;ller\n",
    "\n",
    "*EarthByte Group, School of Geosciences, University of Sydney, NSW 2006, Australia*\n",
    "\n",
    "This notebook is crafted to augment positive data, specifically known mineral occurrences, within the framework of mineral prospectivity mapping. It utilizes a hybrid approach that integrates the Synthetic Minority Oversampling Technique (SMOTE) with Generative Adversarial Networks (GAN) to enhance the dataset.\n",
    "\n",
    "### Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bf4a1a16",
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.over_sampling import SMOTE\n",
    "import numpy as np\n",
    "from numpy import ones\n",
    "from numpy import zeros\n",
    "from numpy.random import randint\n",
    "from numpy.random import randn\n",
    "import pandas as pd\n",
    "from scipy import stats\n",
    "\n",
    "from tensorflow.keras.layers import Dense\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.regularizers import L1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "faefd808",
   "metadata": {},
   "source": [
    "### SMOTE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2391596a",
   "metadata": {},
   "outputs": [],
   "source": [
    "Xy_train_file = './Datasets/Outputs/Xy_train.csv'\n",
    "# Xy_train_file = './Datasets/Outputs/Xy_train_important.csv'\n",
    "Xy_train = pd.read_csv(Xy_train_file, index_col=False)\n",
    "\n",
    "Xy_train_positive = Xy_train.loc[Xy_train['label']==1]\n",
    "Xy_train_unlabelled = Xy_train.loc[Xy_train['label']==0]\n",
    "\n",
    "features_label_lst = Xy_train.columns.tolist()\n",
    "num_features_lst = []\n",
    "cat_features_lst = []\n",
    "\n",
    "for column in features_label_lst:\n",
    "    if column.startswith('Intrusions_') or column.startswith('MetamorphicFacies') or column.startswith('RockUnits'):\n",
    "        cat_features_lst.append(column)\n",
    "    else:\n",
    "        num_features_lst.append(column)\n",
    "\n",
    "num_features_labels = Xy_train[num_features_lst]\n",
    "cat_features = Xy_train[cat_features_lst]\n",
    "\n",
    "Xy_train_num = num_features_labels.to_numpy()\n",
    "num_features = Xy_train_num[:, :-1]\n",
    "num_labels = Xy_train_num[:, -1]\n",
    "\n",
    "# SMOTE\n",
    "smote = SMOTE(random_state=42)\n",
    "X_sm, y_sm = smote.fit_resample(num_features, num_labels)\n",
    "smote_samples = np.concatenate((X_sm, y_sm.reshape(-1, 1)), axis=1)\n",
    "X_positive = smote_samples[np.where(smote_samples[:, -1]==1)]\n",
    "X_positive = X_positive[:, 0:-1]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1ab3903a",
   "metadata": {},
   "source": [
    "### GAN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2072e02d",
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 1e-2\n",
    "\n",
    "# define the standalone discriminator model\n",
    "def define_discriminator(n_inputs):\n",
    "    model = Sequential()\n",
    "    model.add(Dense(int(n_inputs*10), activation='LeakyReLU', activity_regularizer=L1(learning_rate), kernel_initializer='he_uniform', input_dim=n_inputs))\n",
    "    model.add(Dense(1, activation='sigmoid', activity_regularizer=L1(learning_rate)))\n",
    "    # compile model\n",
    "    model.compile(loss='binary_crossentropy', optimizer='Adam', metrics=['accuracy'])\n",
    "    return model\n",
    "\n",
    "# define the standalone generator model\n",
    "def define_generator(latent_dim, n_outputs):\n",
    "    model = Sequential()\n",
    "    model.add(Dense(int(n_outputs*5), activation='LeakyReLU', activity_regularizer=L1(learning_rate), kernel_initializer='he_uniform', input_dim=latent_dim))\n",
    "    model.add(Dense(n_outputs, activation='linear', activity_regularizer=L1(learning_rate)))\n",
    "    return model\n",
    "\n",
    "# define the combined generator and discriminator model for updating the generator\n",
    "def define_gan(generator, discriminator):\n",
    "    # make weights in the discriminator not trainable\n",
    "    discriminator.trainable = False\n",
    "    # connect them\n",
    "    model = Sequential()\n",
    "    # add generator\n",
    "    model.add(generator)\n",
    "    # add the discriminator\n",
    "    model.add(discriminator)\n",
    "    # compile model\n",
    "    model.compile(loss='binary_crossentropy', optimizer='adam')\n",
    "    return model\n",
    "\n",
    "n_samples = int(num_features.shape[0]-(2*num_labels.sum()))\n",
    "\n",
    "# sample real data\n",
    "def sample_real_data(n=n_samples):\n",
    "    X_rand = X_positive[randint(X_positive.shape[0], size=n), :]\n",
    "    y_rand = ones((n, 1))\n",
    "    return X_rand, y_rand\n",
    "\n",
    "# generate points in the latent space as input for the generator\n",
    "def generate_latent_points(latent_dim, n=n_samples):\n",
    "    # generate points in the latent space\n",
    "    x_input = randn(latent_dim * n)\n",
    "    # reshape into a batch of inputs for the network\n",
    "    x_input = x_input.reshape(n, latent_dim)\n",
    "    return x_input\n",
    "\n",
    "# use the generator to generate n fake examples with class labels\n",
    "def generate_fake_samples(generator, latent_dim, n=n_samples):\n",
    "    # generate points in the latent space\n",
    "    x_input = generate_latent_points(latent_dim, n)\n",
    "    # predict outputs\n",
    "    X_fake = generator.predict(x_input, verbose=0)\n",
    "    # create class labels\n",
    "    y_fake = zeros((n, 1))\n",
    "    return X_fake, y_fake\n",
    "\n",
    "# evaluate the discriminator\n",
    "def summarize_performance(epoch, generator, discriminator, latent_dim):\n",
    "    # sample real data\n",
    "    x_real, y_real = sample_real_data()\n",
    "    # evaluate the discriminator on real examples\n",
    "    _, acc_real = discriminator.evaluate(X_sm, y_sm, verbose=0)\n",
    "    # prepare fake examples\n",
    "    x_fake, y_fake = generate_fake_samples(generator, latent_dim)\n",
    "    # evaluate the discriminator on fake examples\n",
    "    _, acc_fake = discriminator.evaluate(x_fake, y_fake, verbose=0)\n",
    "    return x_real, x_fake, acc_real, acc_fake\n",
    "\n",
    "# train the generator and discriminator\n",
    "def train(g_model, d_model, gan_model, latent_dim, n_epochs, n_batch=128, n_eval=100):\n",
    "    # determine half the size of one batch, for updating the discriminator\n",
    "    half_batch = int(n_batch / 2)\n",
    "    acc_real_all = []\n",
    "    acc_fake_all = []\n",
    "    # manually enumerate epochs\n",
    "    for i in range(n_epochs):\n",
    "        # prepare real samples\n",
    "        x_real, y_real = sample_real_data(half_batch)\n",
    "        # prepare fake examples\n",
    "        x_fake, y_fake = generate_fake_samples(g_model, latent_dim, half_batch)\n",
    "        # update discriminator\n",
    "        d_model.train_on_batch(x_real, y_real)\n",
    "        d_model.train_on_batch(x_fake, y_fake)\n",
    "        # prepare points in the latent space as input for the generator\n",
    "        x_gan = generate_latent_points(latent_dim, n_batch)\n",
    "        # create inverted labels for the fake samples\n",
    "        y_gan = ones((n_batch, 1))\n",
    "        # update the generator via the discriminator's error\n",
    "        gan_model.train_on_batch(x_gan, y_gan)\n",
    "        \n",
    "        # plot data points\n",
    "        # ----------------\n",
    "        # evaluate the model every n_eval epochs\n",
    "        if (i+1) % n_eval == 0:\n",
    "            # summarize discriminator performance\n",
    "            x_real, x_fake, acc_real, acc_fake = summarize_performance(i, g_model, d_model, latent_dim)\n",
    "            print(i+1, acc_real, acc_fake)\n",
    "    \n",
    "        if i == n_epochs-1:\n",
    "            # remove outliers\n",
    "            x_fake = x_fake[(np.abs(stats.zscore(x_fake)) < 3).all(axis=1)]\n",
    "            x_fake_ones = np.full((x_fake.shape[0], 1), 2)\n",
    "            x_fake = np.concatenate((x_fake, x_fake_ones), axis=1)\n",
    "            x_fake_num = x_fake.shape[0]\n",
    "            smote_gan_samples = np.concatenate((Xy_train_num, x_fake), axis=0)\n",
    "            smote_gan_samples = pd.DataFrame(smote_gan_samples, columns=num_features_lst)\n",
    "    \n",
    "    return smote_gan_samples, x_fake_num"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b35ce9a6",
   "metadata": {},
   "source": [
    "### Multiple Iterations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b2bf9d3d",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# size of the latent space\n",
    "latent_dim = int(num_features.shape[1]*0.5)\n",
    "\n",
    "n_iter = 10\n",
    "\n",
    "Xy_train_cat = Xy_train[cat_features_lst]\n",
    "Xy_train_cat_positive = Xy_train_positive[cat_features_lst]\n",
    "Xy_train_cat_positive_mode = Xy_train_cat_positive.mode()\n",
    "\n",
    "for i in range(n_iter):\n",
    "    print('--------------------')\n",
    "    print(f'Iteration {i+1}')\n",
    "    print('--------------------')\n",
    "\n",
    "    # create the discriminator\n",
    "    discriminator = define_discriminator(num_features.shape[1])\n",
    "    # create the generator\n",
    "    generator = define_generator(latent_dim, num_features.shape[1])\n",
    "    # create the gan\n",
    "    gan_model = define_gan(generator, discriminator)\n",
    "    # train model\n",
    "    smote_gan_samples, x_fake_num = train(generator, discriminator, gan_model, latent_dim, n_epochs=5000)\n",
    "    \n",
    "    Xy_train_cat_positive_mode_ = Xy_train_cat_positive_mode.loc[Xy_train_cat_positive_mode.index.repeat(x_fake_num)].reset_index(drop=True)\n",
    "    Xy_train_cat_ = pd.concat([Xy_train_cat, Xy_train_cat_positive_mode_]).reset_index(drop=True)\n",
    "    smote_gan_samples = pd.concat([smote_gan_samples, Xy_train_cat_], axis=1).reset_index(drop=True)\n",
    "    smote_gan_samples = smote_gan_samples[features_label_lst]\n",
    "    smote_gan_samples.to_csv(f'./Datasets/Outputs/smote_gan_important_{i+1}.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "39e920a6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "autoenc_pip",
   "language": "python",
   "name": "autoenc_pip"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}