0_create_labels_df.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e8047d4d",
   "metadata": {},
   "source": [
    "# This notebook creates csv files that link absolute paths of cifar10 png image files to their correct labels."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "cd4936b1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/datasets/uly/multiannotator_benchmarks\r\n"
     ]
    }
   ],
   "source": [
    "# Initial steps for running on server if cifar2png is not installed\n",
    "# !pip install cifar2png #install png installer for cifar10 images\n",
    "# !cifar2png cifar10 ./data/cifar10 --name-with-batch-index #download the images\n",
    "\n",
    "import sys\n",
    "import os\n",
    "import pandas as pd\n",
    "sys.path.insert(0, \"../\")\n",
    "!pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c66db4f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "classes = {\"airplane\": 0, \n",
    "           \"automobile\": 1, \n",
    "           \"bird\": 2, \n",
    "           \"cat\": 3, \n",
    "           \"deer\": 4, \n",
    "           \"dog\": 5, \n",
    "           \"frog\": 6, \n",
    "           \"horse\": 7, \n",
    "           \"ship\": 8, \n",
    "           \"truck\": 9}\n",
    "\n",
    "path = os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d518f29f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create png->label mapping for train\n",
    "\n",
    "directory = './data/cifar10/train'\n",
    "model_folder = 'benchmark_data'\n",
    "\n",
    "fname = []\n",
    "labels = []\n",
    "\n",
    "for root,d_names,f_names in os.walk(directory):\n",
    "    f_names = [f for f in f_names if not f.startswith('.')] #remove hidden files\n",
    "    for f in f_names:\n",
    "        path_name = os.path.join(root, f).split('/')[1:]\n",
    "        label = int(classes[path_name[-2]])\n",
    "        path_name = '/'.join(path_name)\n",
    "        absolute_dir_name = path + '/' + path_name\n",
    "        fname.append(absolute_dir_name)\n",
    "        labels.append(label)\n",
    "\n",
    "train_df = pd.DataFrame(zip(fname, labels), columns=['image', 'label'])\n",
    "train_df.to_csv('./data/' + model_folder + '/cifar10_train_dataset.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e54ea635",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create png->label mapping for test\n",
    "\n",
    "directory = './data/cifar10/test'\n",
    "model_folder = 'benchmark_data'\n",
    "\n",
    "fname = []\n",
    "labels = []\n",
    "\n",
    "for root, d_names,f_names in os.walk(directory):\n",
    "    f_names = [f for f in f_names if not f.startswith('.')] #remove hidden files\n",
    "    for f in f_names:\n",
    "        path_name = os.path.join(root, f).split('/')[1:]\n",
    "        label = int(classes[path_name[-2]])\n",
    "        path_name = '/'.join(path_name)\n",
    "        absolute_dir_name = path + '/' + path_name\n",
    "        fname.append(absolute_dir_name)\n",
    "        labels.append(label)\n",
    "\n",
    "test_df = pd.DataFrame(zip(fname, labels), columns=['image', 'label'])\n",
    "test_df.to_csv('./data/' + model_folder + '/cifar10_test_dataset.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "97b34484",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Update consensus dataset with correct absolute paths and resave\n",
    "\n",
    "consensus_data = './data/' + model_folder + '/cifar10_test_consensus_dataset_worst_25_coin20.csv'\n",
    "consensus_df = pd.read_csv(consensus_data)\n",
    "consensus_df['image'] = \\\n",
    "consensus_df['image'].apply(lambda x: path + '/' + '/'.join(x.split('/')[-5:]))\n",
    "consensus_df.to_csv('./data/' + model_folder + '/cifar10_test_consensus_dataset_worst_25_coin20.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4fb6d0e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Update sparse noise dataset with correct absolute paths and resave\n",
    "\n",
    "train_data = './data/' + model_folder + '/cifar10_train_dataset_noise_amount_0.2_sparsity_0.4.csv'\n",
    "train_dataset_noisy_labels_loaded = pd.read_csv(train_data)\n",
    "train_dataset_noisy_labels_loaded['image'] = \\\n",
    "train_dataset_noisy_labels_loaded['image'].apply(lambda x: path + '/'.join(x.split('/')[-3:]))\n",
    "train_df.to_csv('./data/' + model_folder + '/cifar10_train_dataset_noise_amount_0.2_sparsity_0.4.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.8 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8 (main, Oct 21 2022, 22:22:30) [Clang 14.0.0 (clang-1400.0.29.202)]"
  },
  "vscode": {
   "interpreter": {
    "hash": "b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}