010_Create_Img_Clf_GT_Lst.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1fd5b9fa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 1;\n",
       "                var nbb_unformatted_code = \"%load_ext nb_black\";\n",
       "                var nbb_formatted_code = \"%load_ext nb_black\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%load_ext nb_black"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0b5c37e1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 2;\n",
       "                var nbb_unformatted_code = \"import os\\nimport json\\nimport random\\nimport pandas as pd\\nimport shutil\";\n",
       "                var nbb_formatted_code = \"import os\\nimport json\\nimport random\\nimport pandas as pd\\nimport shutil\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import os\n",
    "import json\n",
    "import random\n",
    "import pandas as pd\n",
    "import shutil"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9d208225",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mkdir: prepare_data/img_clf_lst/train_imgs: File exists\n",
      "mkdir: prepare_data/img_clf_lst/valid_imgs: File exists\n",
      "mkdir: prepare_data/img_clf_lst/train_annots: File exists\n",
      "mkdir: prepare_data/img_clf_lst/valid_annots: File exists\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 3;\n",
       "                var nbb_unformatted_code = \"!mkdir prepare_data/img_clf_lst/train_imgs\\n!mkdir prepare_data/img_clf_lst/valid_imgs\\n!mkdir prepare_data/img_clf_lst/train_annots\\n!mkdir prepare_data/img_clf_lst/valid_annots\";\n",
       "                var nbb_formatted_code = \"!mkdir prepare_data/img_clf_lst/train_imgs\\n!mkdir prepare_data/img_clf_lst/valid_imgs\\n!mkdir prepare_data/img_clf_lst/train_annots\\n!mkdir prepare_data/img_clf_lst/valid_annots\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "!mkdir prepare_data/img_clf_lst/train_imgs\n",
    "!mkdir prepare_data/img_clf_lst/valid_imgs\n",
    "!mkdir prepare_data/img_clf_lst/train_annots\n",
    "!mkdir prepare_data/img_clf_lst/valid_annots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4d73f970",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "45bceb20",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><div id=2cee66de-c2a0-452a-a7c5-efd2ff321edb style=\"display:none; background-color:#9D6CFF; color:white; width:200px; height:30px; padding-left:5px; border-radius:4px; flex-direction:row; justify-content:space-around; align-items:center;\" onmouseover=\"this.style.backgroundColor='#BA9BF8'\" onmouseout=\"this.style.backgroundColor='#9D6CFF'\" onclick=\"window.commands?.execute('create-mitosheet-from-dataframe-output');\">See Full Dataframe in Mito</div> <script> if (window.commands?.hasCommand('create-mitosheet-from-dataframe-output')) document.getElementById('2cee66de-c2a0-452a-a7c5-efd2ff321edb').style.display = 'flex' </script> <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3383430.jpg</td>\n",
       "      <td>[banana]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9071244.jpg</td>\n",
       "      <td>[banana]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1715218.jpg</td>\n",
       "      <td>[orange]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2584986.jpg</td>\n",
       "      <td>[orange]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3025786.jpg</td>\n",
       "      <td>[orange]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table></div>"
      ],
      "text/plain": [
       "         image     class\n",
       "0  3383430.jpg  [banana]\n",
       "1  9071244.jpg  [banana]\n",
       "2  1715218.jpg  [orange]\n",
       "3  2584986.jpg  [orange]\n",
       "4  3025786.jpg  [orange]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 4;\n",
       "                var nbb_unformatted_code = \"gt_df = pd.read_csv(\\n    \\\"prepare_data/img_clf_lst/all_images_gt/clf_labels.csv\\\", header=None\\n)\\ngt_df.columns = [\\\"image\\\", \\\"class\\\"]\\ngt_df.head()\";\n",
       "                var nbb_formatted_code = \"gt_df = pd.read_csv(\\n    \\\"prepare_data/img_clf_lst/all_images_gt/clf_labels.csv\\\", header=None\\n)\\ngt_df.columns = [\\\"image\\\", \\\"class\\\"]\\ngt_df.head()\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "gt_df = pd.read_csv(\n",
    "    \"prepare_data/img_clf_lst/all_images_gt/clf_labels.csv\", header=None\n",
    ")\n",
    "gt_df.columns = [\"image\", \"class\"]\n",
    "gt_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7ffde883",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100 70 30\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 5;\n",
       "                var nbb_unformatted_code = \"train_valid_split = 0.7\\nnimages = gt_df[\\\"image\\\"].nunique()\\nntrain = int(train_valid_split * nimages)\\nnvalid = nimages - ntrain\\nprint(nimages, ntrain, nvalid)\";\n",
       "                var nbb_formatted_code = \"train_valid_split = 0.7\\nnimages = gt_df[\\\"image\\\"].nunique()\\nntrain = int(train_valid_split * nimages)\\nnvalid = nimages - ntrain\\nprint(nimages, ntrain, nvalid)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_valid_split = 0.7\n",
    "nimages = gt_df[\"image\"].nunique()\n",
    "ntrain = int(train_valid_split * nimages)\n",
    "nvalid = nimages - ntrain\n",
    "print(nimages, ntrain, nvalid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ec6ebf84",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><div id=3d82eb5d-28d0-4651-8104-bea45d398c9a style=\"display:none; background-color:#9D6CFF; color:white; width:200px; height:30px; padding-left:5px; border-radius:4px; flex-direction:row; justify-content:space-around; align-items:center;\" onmouseover=\"this.style.backgroundColor='#BA9BF8'\" onmouseout=\"this.style.backgroundColor='#9D6CFF'\" onclick=\"window.commands?.execute('create-mitosheet-from-dataframe-output');\">See Full Dataframe in Mito</div> <script> if (window.commands?.hasCommand('create-mitosheet-from-dataframe-output')) document.getElementById('3d82eb5d-28d0-4651-8104-bea45d398c9a').style.display = 'flex' </script> <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3383430.jpg</td>\n",
       "      <td>banana</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9071244.jpg</td>\n",
       "      <td>banana</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1715218.jpg</td>\n",
       "      <td>orange</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2584986.jpg</td>\n",
       "      <td>orange</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3025786.jpg</td>\n",
       "      <td>orange</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table></div>"
      ],
      "text/plain": [
       "         image   class\n",
       "0  3383430.jpg  banana\n",
       "1  9071244.jpg  banana\n",
       "2  1715218.jpg  orange\n",
       "3  2584986.jpg  orange\n",
       "4  3025786.jpg  orange"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 6;\n",
       "                var nbb_unformatted_code = \"gt_df[\\\"class\\\"] = gt_df[\\\"class\\\"].apply(lambda x: x[1:-1])\\ngt_df.head()\";\n",
       "                var nbb_formatted_code = \"gt_df[\\\"class\\\"] = gt_df[\\\"class\\\"].apply(lambda x: x[1:-1])\\ngt_df.head()\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "gt_df[\"class\"] = gt_df[\"class\"].apply(lambda x: x[1:-1])\n",
    "gt_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "850284bd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'apple': 0, 'orange': 1, 'banana': 2}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 7;\n",
       "                var nbb_unformatted_code = \"labels = [\\\"apple\\\", \\\"orange\\\", \\\"banana\\\"]\\nlabels_map = {k: v for v, k in enumerate(labels)}\\nwith open(\\\"prepare_data/img_clf_lst/labels_map.json\\\", \\\"w\\\") as fp:\\n    json.dump(labels_map, fp)\\nlabels_map\";\n",
       "                var nbb_formatted_code = \"labels = [\\\"apple\\\", \\\"orange\\\", \\\"banana\\\"]\\nlabels_map = {k: v for v, k in enumerate(labels)}\\nwith open(\\\"prepare_data/img_clf_lst/labels_map.json\\\", \\\"w\\\") as fp:\\n    json.dump(labels_map, fp)\\nlabels_map\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "labels = [\"apple\", \"orange\", \"banana\"]\n",
    "labels_map = {k: v for v, k in enumerate(labels)}\n",
    "with open(\"prepare_data/img_clf_lst/labels_map.json\", \"w\") as fp:\n",
    "    json.dump(labels_map, fp)\n",
    "labels_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7c15e546",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><div id=40833604-c106-400c-8e93-9700eb0fba66 style=\"display:none; background-color:#9D6CFF; color:white; width:200px; height:30px; padding-left:5px; border-radius:4px; flex-direction:row; justify-content:space-around; align-items:center;\" onmouseover=\"this.style.backgroundColor='#BA9BF8'\" onmouseout=\"this.style.backgroundColor='#9D6CFF'\" onclick=\"window.commands?.execute('create-mitosheet-from-dataframe-output');\">See Full Dataframe in Mito</div> <script> if (window.commands?.hasCommand('create-mitosheet-from-dataframe-output')) document.getElementById('40833604-c106-400c-8e93-9700eb0fba66').style.display = 'flex' </script> <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image</th>\n",
       "      <th>class</th>\n",
       "      <th>class_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3383430.jpg</td>\n",
       "      <td>banana</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9071244.jpg</td>\n",
       "      <td>banana</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1715218.jpg</td>\n",
       "      <td>orange</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2584986.jpg</td>\n",
       "      <td>orange</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3025786.jpg</td>\n",
       "      <td>orange</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table></div>"
      ],
      "text/plain": [
       "         image   class  class_id\n",
       "0  3383430.jpg  banana         2\n",
       "1  9071244.jpg  banana         2\n",
       "2  1715218.jpg  orange         1\n",
       "3  2584986.jpg  orange         1\n",
       "4  3025786.jpg  orange         1"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 8;\n",
       "                var nbb_unformatted_code = \"gt_df[\\\"class_id\\\"] = gt_df[\\\"class\\\"].map(labels_map)\\ngt_df.head()\";\n",
       "                var nbb_formatted_code = \"gt_df[\\\"class_id\\\"] = gt_df[\\\"class\\\"].map(labels_map)\\ngt_df.head()\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "gt_df[\"class_id\"] = gt_df[\"class\"].map(labels_map)\n",
    "gt_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a45eec7c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><div id=d0a1d771-f357-460e-8f1c-4e5e3d319d48 style=\"display:none; background-color:#9D6CFF; color:white; width:200px; height:30px; padding-left:5px; border-radius:4px; flex-direction:row; justify-content:space-around; align-items:center;\" onmouseover=\"this.style.backgroundColor='#BA9BF8'\" onmouseout=\"this.style.backgroundColor='#9D6CFF'\" onclick=\"window.commands?.execute('create-mitosheet-from-dataframe-output');\">See Full Dataframe in Mito</div> <script> if (window.commands?.hasCommand('create-mitosheet-from-dataframe-output')) document.getElementById('d0a1d771-f357-460e-8f1c-4e5e3d319d48').style.display = 'flex' </script> <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image</th>\n",
       "      <th>class</th>\n",
       "      <th>class_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4985315.jpg</td>\n",
       "      <td>banana</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1168697.jpg</td>\n",
       "      <td>apple</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9071244.jpg</td>\n",
       "      <td>banana</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8159023.jpg</td>\n",
       "      <td>apple</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9638460.jpg</td>\n",
       "      <td>banana</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table></div>"
      ],
      "text/plain": [
       "         image   class  class_id\n",
       "0  4985315.jpg  banana         2\n",
       "1  1168697.jpg   apple         0\n",
       "2  9071244.jpg  banana         2\n",
       "3  8159023.jpg   apple         0\n",
       "4  9638460.jpg  banana         2"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 9;\n",
       "                var nbb_unformatted_code = \"gt_df = gt_df.sample(frac=1).reset_index(drop=True)\\ngt_df.head()\";\n",
       "                var nbb_formatted_code = \"gt_df = gt_df.sample(frac=1).reset_index(drop=True)\\ngt_df.head()\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "gt_df = gt_df.sample(frac=1).reset_index(drop=True)\n",
    "gt_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5148a4c1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><div id=66df4a1d-ef01-4645-bf10-e522bc203682 style=\"display:none; background-color:#9D6CFF; color:white; width:200px; height:30px; padding-left:5px; border-radius:4px; flex-direction:row; justify-content:space-around; align-items:center;\" onmouseover=\"this.style.backgroundColor='#BA9BF8'\" onmouseout=\"this.style.backgroundColor='#9D6CFF'\" onclick=\"window.commands?.execute('create-mitosheet-from-dataframe-output');\">See Full Dataframe in Mito</div> <script> if (window.commands?.hasCommand('create-mitosheet-from-dataframe-output')) document.getElementById('66df4a1d-ef01-4645-bf10-e522bc203682').style.display = 'flex' </script> <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image</th>\n",
       "      <th>class</th>\n",
       "      <th>class_id</th>\n",
       "      <th>index</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4985315.jpg</td>\n",
       "      <td>banana</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1168697.jpg</td>\n",
       "      <td>apple</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9071244.jpg</td>\n",
       "      <td>banana</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8159023.jpg</td>\n",
       "      <td>apple</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9638460.jpg</td>\n",
       "      <td>banana</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table></div>"
      ],
      "text/plain": [
       "         image   class  class_id  index\n",
       "0  4985315.jpg  banana         2      1\n",
       "1  1168697.jpg   apple         0      2\n",
       "2  9071244.jpg  banana         2      3\n",
       "3  8159023.jpg   apple         0      4\n",
       "4  9638460.jpg  banana         2      5"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 10;\n",
       "                var nbb_unformatted_code = \"gt_df[\\\"index\\\"] = gt_df.index + 1\\ngt_df.head()\";\n",
       "                var nbb_formatted_code = \"gt_df[\\\"index\\\"] = gt_df.index + 1\\ngt_df.head()\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "gt_df[\"index\"] = gt_df.index + 1\n",
    "gt_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "84af2739",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\t2\t4985315.jpg\r\n",
      "2\t0\t1168697.jpg\r\n",
      "3\t2\t9071244.jpg\r\n",
      "4\t0\t8159023.jpg\r\n",
      "5\t2\t9638460.jpg\r\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 17;\n",
       "                var nbb_unformatted_code = \"sel_cols = [\\\"index\\\", \\\"class_id\\\", \\\"image\\\"]\\ngt_df[sel_cols].head(ntrain).to_csv(\\n    \\\"prepare_data/img_clf_lst/train_annots/train.lst\\\",\\n    sep=\\\"\\\\t\\\",\\n    index=False,\\n    header=False,\\n)\\n!head -n 5 prepare_data/img_clf_lst/train_annots/train.lst\";\n",
       "                var nbb_formatted_code = \"sel_cols = [\\\"index\\\", \\\"class_id\\\", \\\"image\\\"]\\ngt_df[sel_cols].head(ntrain).to_csv(\\n    \\\"prepare_data/img_clf_lst/train_annots/train.lst\\\",\\n    sep=\\\"\\\\t\\\",\\n    index=False,\\n    header=False,\\n)\\n!head -n 5 prepare_data/img_clf_lst/train_annots/train.lst\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sel_cols = [\"index\", \"class_id\", \"image\"]\n",
    "gt_df[sel_cols].head(ntrain).to_csv(\n",
    "    \"prepare_data/img_clf_lst/train_annots/train.lst\",\n",
    "    sep=\"\\t\",\n",
    "    index=False,\n",
    "    header=False,\n",
    ")\n",
    "!head -n 5 prepare_data/img_clf_lst/train_annots/train.lst"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "a7450c34",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "71\t0\t8132975.jpg\r\n",
      "72\t1\t3224044.jpg\r\n",
      "73\t2\t2746994.jpg\r\n",
      "74\t0\t6361494.jpg\r\n",
      "75\t2\t1813817.jpg\r\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 18;\n",
       "                var nbb_unformatted_code = \"gt_df[sel_cols].tail(nvalid).to_csv(\\n    \\\"prepare_data/img_clf_lst/valid_annots/valid.lst\\\",\\n    sep=\\\"\\\\t\\\",\\n    index=False,\\n    header=False,\\n)\\n!head -n 5 prepare_data/img_clf_lst/valid_annots/valid.lst\";\n",
       "                var nbb_formatted_code = \"gt_df[sel_cols].tail(nvalid).to_csv(\\n    \\\"prepare_data/img_clf_lst/valid_annots/valid.lst\\\",\\n    sep=\\\"\\\\t\\\",\\n    index=False,\\n    header=False,\\n)\\n!head -n 5 prepare_data/img_clf_lst/valid_annots/valid.lst\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "gt_df[sel_cols].tail(nvalid).to_csv(\n",
    "    \"prepare_data/img_clf_lst/valid_annots/valid.lst\",\n",
    "    sep=\"\\t\",\n",
    "    index=False,\n",
    "    header=False,\n",
    ")\n",
    "!head -n 5 prepare_data/img_clf_lst/valid_annots/valid.lst"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "09377ae3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 13;\n",
       "                var nbb_unformatted_code = \"train_df = pd.read_csv(\\n    \\\"prepare_data/img_clf_lst/train_annots/train.lst\\\", sep=\\\"\\\\t\\\", header=None\\n)\\nimages = list(train_df[2].values)\\nfor image in images:\\n    shutil.copy(\\n        \\\"prepare_data/img_clf_lst/all_images_gt/\\\" + image,\\n        \\\"prepare_data/img_clf_lst/train_imgs/\\\",\\n    )\";\n",
       "                var nbb_formatted_code = \"train_df = pd.read_csv(\\n    \\\"prepare_data/img_clf_lst/train_annots/train.lst\\\", sep=\\\"\\\\t\\\", header=None\\n)\\nimages = list(train_df[2].values)\\nfor image in images:\\n    shutil.copy(\\n        \\\"prepare_data/img_clf_lst/all_images_gt/\\\" + image,\\n        \\\"prepare_data/img_clf_lst/train_imgs/\\\",\\n    )\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_df = pd.read_csv(\n",
    "    \"prepare_data/img_clf_lst/train_annots/train.lst\", sep=\"\\t\", header=None\n",
    ")\n",
    "images = list(train_df[2].values)\n",
    "for image in images:\n",
    "    shutil.copy(\n",
    "        \"prepare_data/img_clf_lst/all_images_gt/\" + image,\n",
    "        \"prepare_data/img_clf_lst/train_imgs/\",\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "16ff9807",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 14;\n",
       "                var nbb_unformatted_code = \"valid_df = pd.read_csv(\\n    \\\"prepare_data/img_clf_lst/valid_annots/valid.lst\\\", sep=\\\"\\\\t\\\", header=None\\n)\\nimages = list(valid_df[2].values)\\nfor image in images:\\n    shutil.copy(\\n        \\\"prepare_data/img_clf_lst/all_images_gt/\\\" + image,\\n        \\\"prepare_data/img_clf_lst/valid_imgs/\\\",\\n    )\";\n",
       "                var nbb_formatted_code = \"valid_df = pd.read_csv(\\n    \\\"prepare_data/img_clf_lst/valid_annots/valid.lst\\\", sep=\\\"\\\\t\\\", header=None\\n)\\nimages = list(valid_df[2].values)\\nfor image in images:\\n    shutil.copy(\\n        \\\"prepare_data/img_clf_lst/all_images_gt/\\\" + image,\\n        \\\"prepare_data/img_clf_lst/valid_imgs/\\\",\\n    )\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "valid_df = pd.read_csv(\n",
    "    \"prepare_data/img_clf_lst/valid_annots/valid.lst\", sep=\"\\t\", header=None\n",
    ")\n",
    "images = list(valid_df[2].values)\n",
    "for image in images:\n",
    "    shutil.copy(\n",
    "        \"prepare_data/img_clf_lst/all_images_gt/\" + image,\n",
    "        \"prepare_data/img_clf_lst/valid_imgs/\",\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e6909606",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cdf3edd3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc3844f2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7434f33a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a098913",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "5c01064b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 15;\n",
       "                var nbb_unformatted_code = \"# import ast\\n# labels = [\\\"apple\\\", \\\"orange\\\", \\\"banana\\\"]\\n# imgs = [str(random.randint(1000000, 9999999)) + \\\".jpg\\\" for i in range(100)]\\n# for img in imgs:\\n#     shutil.copy(\\\"images/rose.jpg\\\", \\\"prepare_date/img_clf_lst/all_images_gt/\\\" + img)\\n# clss = [random.sample(labels, 1) for i in range(100)]\\n# clss = [str(cl).replace(\\\"'\\\", \\\"\\\") for cl in clss]\\n# gt_df = pd.DataFrame({\\\"image\\\": imgs, \\\"class\\\": clss})\\n# gt_df.to_csv(\\\"prepare_date/img_clf_lst/all_images_gt/clf_labels.csv\\\", index=False)\\n# gt_df.head()\";\n",
       "                var nbb_formatted_code = \"# import ast\\n# labels = [\\\"apple\\\", \\\"orange\\\", \\\"banana\\\"]\\n# imgs = [str(random.randint(1000000, 9999999)) + \\\".jpg\\\" for i in range(100)]\\n# for img in imgs:\\n#     shutil.copy(\\\"images/rose.jpg\\\", \\\"prepare_date/img_clf_lst/all_images_gt/\\\" + img)\\n# clss = [random.sample(labels, 1) for i in range(100)]\\n# clss = [str(cl).replace(\\\"'\\\", \\\"\\\") for cl in clss]\\n# gt_df = pd.DataFrame({\\\"image\\\": imgs, \\\"class\\\": clss})\\n# gt_df.to_csv(\\\"prepare_date/img_clf_lst/all_images_gt/clf_labels.csv\\\", index=False)\\n# gt_df.head()\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "acb8f9ce",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8cd3b21e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}