create_comp_noisy_emnist_letters_training_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create Compressed & Noisy EMNIST Letter Training Data\n",
    "Creates compressed ***EMNIST*** images with imposed Poisson noise.\n",
    "\n",
    "**Author:** Fabian Santiago  \n",
    "**Update:** September 19, 2024\n",
    "\n",
    "***NOTE:*** *Code creates compressed noisy images of dimensions:*\n",
    "\n",
    "<ul>\n",
    "  <li><b><em>4x4</em></b>: Compressed and imposed Poisson noise<br></li>\n",
    "  <li><b><em>7x7</em></b>: Compressed and imposed Poisson noise<br></li>\n",
    "  <li><b><em>14x14</em></b>: Compressed and imposed Poisson noise<br></li>\n",
    "  <li><b><em>28x28</em></b>: Only imposed Poisson noise</li>\n",
    "</ul>\n",
    "\n",
    "Jupyer Notebook Version: 6.5.4  \n",
    "Python Version: 3.11.5  \n",
    "TensorFlow Version: 2.16.2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import Modules and Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import necessary modules and libraries \n",
    "import os\n",
    "import h5py\n",
    "import numpy as np\n",
    "from skimage.util.shape import view_as_windows\n",
    "from emnist import extract_training_samples\n",
    "from emnist import extract_test_samples"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set Values: Seed, Compression, and Output Dimension"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dimension of output: original are 28 x 28\n",
    "out_dim  = 28 "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper Function Definitions\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define sliding window compression\n",
    "def median_downsampling(in_image, cmp_dim):\n",
    "    if cmp_dim < 15:\n",
    "        # Compression dimension to window size\n",
    "        window_size = 28//cmp_dim\n",
    "        # Create sliding windows\n",
    "        windows = view_as_windows(in_image, (window_size, window_size), step=window_size)\n",
    "        # Calculate the median over each window\n",
    "        return np.median(windows, axis=(2, 3))\n",
    "    else:\n",
    "        return in_image\n",
    "\n",
    "# Define compression of entries in a list\n",
    "def down_sample_list(in_array,cmp_dim):\n",
    "    out_array = np.empty((len(in_array), cmp_dim, cmp_dim))\n",
    "    \n",
    "    # Iterate over array\n",
    "    for idx, image in enumerate(in_array):\n",
    "        out_array[idx] = median_downsampling(image,cmp_dim)\n",
    "        \n",
    "    # Return array containing compressed entries \n",
    "    return out_array \n",
    "\n",
    "# Create compressed noisy data\n",
    "def create_training_data(clean_train, clean_test, cmp_dim, out_dim):\n",
    "    \n",
    "    # Compress images using median in sliding window\n",
    "    cmp_train = down_sample_list(clean_train, cmp_dim)\n",
    "    cmp_test  = down_sample_list(clean_test, cmp_dim)\n",
    "\n",
    "    # Add Poisson noise to compressed images\n",
    "    noisy_train_ = np.random.poisson(lam=cmp_train)\n",
    "    noisy_test_  = np.random.poisson(lam=cmp_test)\n",
    "    \n",
    "    # Scale to [0,255]\n",
    "    noisy_train  = np.clip(noisy_train_,0,255)\n",
    "    noisy_test   = np.clip(noisy_test_,0,255)\n",
    "    \n",
    "    # Reshape Compressed/Noisy Arrays and scale to [0,1]\n",
    "    noisy_train = np.array([matrix.reshape(cmp_dim**2,) for matrix in noisy_train/255])\n",
    "    noisy_test  = np.array([matrix.reshape(cmp_dim**2,) for matrix in noisy_test/255])\n",
    "    \n",
    "    # Return training data\n",
    "    return noisy_train, noisy_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Compressed & Noisy Training Data\n",
    "Load EMNIST data and create training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "directory = 'training_data'\n",
    "\n",
    "# Create training data directory if nonexistent\n",
    "if not os.path.exists(directory):\n",
    "    os.makedirs(directory)\n",
    "    \n",
    "# Load EMNIST (og -> original) \n",
    "clean_train_og, _ = extract_training_samples('letters')    \n",
    "clean_test_og, _  = extract_test_samples('letters')\n",
    "    \n",
    "for cmp_dim in [4,7,14,28]:\n",
    "    # Set the training data file name\n",
    "    dat_file = f'{directory}/emnist_{cmp_dim}x{cmp_dim}_train.h5'\n",
    "    \n",
    "    # Create compressed/noisy training/test data\n",
    "    noisy_train, noisy_test  = create_training_data(clean_train_og, clean_test_og, cmp_dim, out_dim)\n",
    "        \n",
    "    # Save compressed/noisy training/test data\n",
    "    with h5py.File(dat_file, 'w') as f:\n",
    "        # Save compressed noisy images\n",
    "        f.create_dataset('noisy_train', data=noisy_train, compression='gzip')\n",
    "        f.create_dataset('noisy_test', data=noisy_test, compression='gzip')"
   ]
  }
 ],
 "metadata": {
  "@webio": {
   "lastCommId": null,
   "lastKernelId": null
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}