export.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pds4_tools import pds4_read       # to read and inspect the data and metadata\n",
    "import matplotlib.pyplot as plt        # for plotting\n",
    "import matplotlib\n",
    "import numpy as np\n",
    "from PIL import Image\n",
    "# for plotting in Jupyter notebooks\n",
    "%matplotlib notebook\n",
    "\n",
    "from skimage import exposure\n",
    "from skimage import data, img_as_float\n",
    "import colour\n",
    "from colour.plotting import *\n",
    "import glob\n",
    "\n",
    "from colour_demosaicing import (\n",
    "    demosaicing_CFA_Bayer_bilinear,\n",
    "    demosaicing_CFA_Bayer_Malvar2004,\n",
    "    demosaicing_CFA_Bayer_Menon2007,\n",
    "    mosaicing_CFA_Bayer)\n",
    "\n",
    "cctf_encoding = colour.cctf_encoding\n",
    "colour.utilities.filter_warnings()\n",
    "# colour.utilities.describe_environment();\n",
    "\n",
    "def read_pds(path):\n",
    "    data = pds4_read(path, quiet=True)\n",
    "    img = np.array(data[0].data)\n",
    "    img = img_as_float(img)\n",
    "    return img\n",
    "def debayer_img(img, CFA='RGGB'):\n",
    "    # Menon2007 yields better edges than bilinear\n",
    "    debayered = cctf_encoding(demosaicing_CFA_Bayer_Menon2007(img, CFA))\n",
    "    return debayered\n",
    "def stretch_img(img):\n",
    "    p2, p98 = np.percentile(img, (2, 98))\n",
    "    img = exposure.rescale_intensity(img, in_range=(p2, p98))\n",
    "    return img\n",
    "def export_img(name, img):\n",
    "    pil_img = Image.fromarray(np.uint8(img*255))\n",
    "    pil_img.save(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "# from p_tqdm import p_map\n",
    "# doesn't work well"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2564/2564 [25:40<00:00,  1.66it/s] \n"
     ]
    }
   ],
   "source": [
    "for p in tqdm(glob.glob('PCAM/*.*L')):\n",
    "    try:\n",
    "        img = read_pds(p)\n",
    "        if img.shape == (1728, 2352):\n",
    "            img = debayer_img(img)\n",
    "        elif img.shape != (864, 1176):\n",
    "            print(f'{p} has an unexpected dim')\n",
    "        img = stretch_img(img)\n",
    "        export_img(f\"{p}.png\", img)\n",
    "    except:\n",
    "        print(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 240/240 [04:30<00:00,  1.13s/it]\n"
     ]
    }
   ],
   "source": [
    "for p in tqdm(glob.glob('TCAM/*.*L')):\n",
    "    try:\n",
    "        img = read_pds(p)\n",
    "        export_img(f\"{p}.png\", img)\n",
    "    except:\n",
    "        print(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "def extract_file_info(p):\n",
    "    cam,_,ser = p.split('-')  # CE4_GRAS_TCAM I 002_SCI_N_20190106033401_20190106033401_0004_A.2CL.png\n",
    "    no,_,_,ts,te,ob,ver = ser.split('_') # 002 N N 20190106033401 20190106033401 0004 A.2CL.png\n",
    "    ver = ver.split('.')[0] # A.2CL.png -> A\n",
    "    cam = cam.split('_')[2] # TCAM\n",
    "    return cam,no,ts,te,ob,ver\n",
    "    \n",
    "def make_dir_if_n_exist(d):\n",
    "    if not os.path.exists(d):\n",
    "        os.makedirs(d)\n",
    "    return d\n",
    "\n",
    "def make_new_name(no, ts, cam):\n",
    "    return f\"{no}-{ts}-{cam}.png\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "observations = {}\n",
    "CAM_DIR = 'TCAM'\n",
    "for p in glob.glob(f'{CAM_DIR}/*.png'):\n",
    "    cam,no,ts,te,ob,ver = extract_file_info(p)\n",
    "    if ob in observations:\n",
    "        observations[ob].append(p)\n",
    "    else:\n",
    "        observations[ob] = [p]\n",
    "        \n",
    "for k in observations.keys():\n",
    "    d = f\"{CAM_DIR}/{k}\"\n",
    "    make_dir_if_n_exist(d)\n",
    "    for p in observations[k]:\n",
    "        cam,no,ts,te,ob,ver = extract_file_info(p)\n",
    "        name = make_new_name(no, ts, cam)\n",
    "        os.symlink(os.path.abspath(p), f\"{d}/{name}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "observations = {}\n",
    "CAM_DIR = 'PCAM'\n",
    "for p in glob.glob(f'{CAM_DIR}/*.png'):\n",
    "    cam,no,ts,te,ob,ver = extract_file_info(p)\n",
    "    if ob in observations:\n",
    "        observations[ob].append(p)\n",
    "    else:\n",
    "        observations[ob] = [p]\n",
    "        \n",
    "for k in observations.keys():\n",
    "    d = f\"{CAM_DIR}/{k}\"\n",
    "    make_dir_if_n_exist(d)\n",
    "    l_dir = make_dir_if_n_exist(f'{d}/PCAML')\n",
    "    r_dir = make_dir_if_n_exist(f'{d}/PCAMR')\n",
    "    for p in observations[k]:\n",
    "        cam,no,ts,te,ob,ver = extract_file_info(p)\n",
    "        name = make_new_name(no, ts, cam)\n",
    "        os.symlink(f\"../../{p}\", f\"{d}/{name}\")\n",
    "        if \"PCAML\" in p:\n",
    "            os.symlink(os.path.abspath(p), f\"{l_dir}/{name}\")\n",
    "        else:\n",
    "            os.symlink(os.path.abspath(p), f\"{r_dir}/{name}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print observation time\n",
    "for k in observations.keys():\n",
    "    p = observations[k][0]\n",
    "    cam,no,ts,te,ob,ver = extract_file_info(p)\n",
    "    ob_date = datetime.strptime(ts, '%Y%m%d%H%M%S').strftime('%Y-%m-%d')\n",
    "    print(f'{k}  {ob_date}')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}