dmu1/1_Data_overview.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1 Data overview\n",
    "\n",
    "In this notebook we will catalogue all the HSC and VISTA image information required for processing. The HSC image corners will be used to find the relevant VISTA images required for swarping to the HSC pixel base."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "\n",
    "from astropy.table import Table, Column\n",
    "from astropy.io import fits\n",
    "from astropy.wcs import WCS\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import numpy as np\n",
    "import time\n",
    "\n",
    "import os\n",
    "\n",
    "import hashlib\n",
    "import yaml\n",
    "import collections"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20210331\n"
     ]
    }
   ],
   "source": [
    "SUFFIX=time.strftime(\"%Y%m%d\")\n",
    "print(SUFFIX)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/raphaelshirley/Documents/github/lsst-ir-fusion/dmu0/data_local.yml\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/raphaelshirley/Documents/github/lsst_stack/conda/miniconda3-py37_4.8.2/envs/lsst-scipipe-cb4e2dc/lib/python3.7/site-packages/ipykernel_launcher.py:20: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "def data_loc(base=None):\n",
    "    \"\"\"Return data locations depending on machine used\n",
    "    \n",
    "    Returns\n",
    "    -------\n",
    "    str\n",
    "        Full path to data repository\n",
    "    \"\"\"\n",
    "    if base == None:\n",
    "        cwd = os.getcwd()\n",
    "        base = cwd.split('lsst-ir-fusion')[0] + 'lsst-ir-fusion/'\n",
    "    \n",
    "    \n",
    "    overide = base + 'dmu0/data_local.yml'\n",
    "    standard = base + 'dmu0/data_local_template.yml'\n",
    "    print(overide)\n",
    "    if os.path.exists(overide):\n",
    "        data_loc = yaml.safe_load(open(overide, 'rb'))\n",
    "    else:\n",
    "        data_loc = yaml.safe_load(open(standard, 'rb'))\n",
    "        for k, v in data_loc.items():\n",
    "            #print(k,v)\n",
    "            data_loc[k] = v.replace('BASE/', base)\n",
    "    return data_loc\n",
    "data_loc = data_loc()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "RUN_HSC = True\n",
    "RUN_VISTA = False\n",
    "RESTART_VISTA = False\n",
    "SURVEY = 'vhs'\n",
    "RUN_PANSTARRS = False\n",
    "\n",
    "RUN_HASH=False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'vista': '/Users/raphaelshirley/Documents/github/lsst-ir-fusion/dmu0/dmu0_VISTA/',\n",
       " 'video': '/Users/raphaelshirley/Documents/github/lsst-ir-fusion/dmu0/dmu0_VISTA/dmu0_VIDEO/data/',\n",
       " 'viking': '/Users/raphaelshirley/Documents/github/lsst-ir-fusion/dmu0/dmu0_VISTA/dmu0_VIKING/data/',\n",
       " 'vhs': '/Users/raphaelshirley/Documents/github/lsst-ir-fusion/dmu0/dmu0_VISTA/dmu0_VHS/data/',\n",
       " 'hsc': '/Users/raphaelshirley/Documents/github/lsst-ir-fusion/dmu0/dmu0_HSC/data/',\n",
       " 'panstarrs': '/Users/raphaelshirley/Documents/github/lsst-ir-fusion/dmu0/dmu0_PanSTARRS/data/',\n",
       " 'gaia': '/Users/raphaelshirley/Documents/github/lsst-ir-fusion/dmu0/dmu0_GAIA/data/'}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_loc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.1 HSC images\n",
    "First lets look at the HSC images."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "images = glob.glob(data_loc['hsc'] + \"**/calexp*.fits\", recursive=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "meta = Table()\n",
    "meta.add_column(Column(data=images, name=\"file\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<i>Table length=5</i>\n",
       "<table id=\"table140383833708048\" class=\"table-striped table-bordered table-condensed\">\n",
       "<thead><tr><th>file</th></tr></thead>\n",
       "<thead><tr><th>str184</th></tr></thead>\n",
       "<tr><td>/Users/raphaelshirley/Documents/github/lsst-ir-fusion/dmu0/dmu0_HSC/data/hsc-release.mtk.nao.ac.jp/archive/filetree/pdr2_dud/deepCoadd-results/HSC-Y/8524/5,5/calexp-HSC-Y-8524-5,5.fits</td></tr>\n",
       "<tr><td>/Users/raphaelshirley/Documents/github/lsst-ir-fusion/dmu0/dmu0_HSC/data/hsc-release.mtk.nao.ac.jp/archive/filetree/pdr2_dud/deepCoadd-results/HSC-Y/8524/5,3/calexp-HSC-Y-8524-5,3.fits</td></tr>\n",
       "<tr><td>/Users/raphaelshirley/Documents/github/lsst-ir-fusion/dmu0/dmu0_HSC/data/hsc-release.mtk.nao.ac.jp/archive/filetree/pdr2_dud/deepCoadd-results/HSC-Y/8524/5,4/calexp-HSC-Y-8524-5,4.fits</td></tr>\n",
       "<tr><td>/Users/raphaelshirley/Documents/github/lsst-ir-fusion/dmu0/dmu0_HSC/data/hsc-release.mtk.nao.ac.jp/archive/filetree/pdr2_dud/deepCoadd-results/HSC-Y/8524/4,5/calexp-HSC-Y-8524-4,5.fits</td></tr>\n",
       "<tr><td>/Users/raphaelshirley/Documents/github/lsst-ir-fusion/dmu0/dmu0_HSC/data/hsc-release.mtk.nao.ac.jp/archive/filetree/pdr2_dud/deepCoadd-results/HSC-Y/8524/4,4/calexp-HSC-Y-8524-4,4.fits</td></tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<Table length=5>\n",
       "                                                                                          file                                                                                          \n",
       "                                                                                         str184                                                                                         \n",
       "----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------\n",
       "/Users/raphaelshirley/Documents/github/lsst-ir-fusion/dmu0/dmu0_HSC/data/hsc-release.mtk.nao.ac.jp/archive/filetree/pdr2_dud/deepCoadd-results/HSC-Y/8524/5,5/calexp-HSC-Y-8524-5,5.fits\n",
       "/Users/raphaelshirley/Documents/github/lsst-ir-fusion/dmu0/dmu0_HSC/data/hsc-release.mtk.nao.ac.jp/archive/filetree/pdr2_dud/deepCoadd-results/HSC-Y/8524/5,3/calexp-HSC-Y-8524-5,3.fits\n",
       "/Users/raphaelshirley/Documents/github/lsst-ir-fusion/dmu0/dmu0_HSC/data/hsc-release.mtk.nao.ac.jp/archive/filetree/pdr2_dud/deepCoadd-results/HSC-Y/8524/5,4/calexp-HSC-Y-8524-5,4.fits\n",
       "/Users/raphaelshirley/Documents/github/lsst-ir-fusion/dmu0/dmu0_HSC/data/hsc-release.mtk.nao.ac.jp/archive/filetree/pdr2_dud/deepCoadd-results/HSC-Y/8524/4,5/calexp-HSC-Y-8524-4,5.fits\n",
       "/Users/raphaelshirley/Documents/github/lsst-ir-fusion/dmu0/dmu0_HSC/data/hsc-release.mtk.nao.ac.jp/archive/filetree/pdr2_dud/deepCoadd-results/HSC-Y/8524/4,4/calexp-HSC-Y-8524-4,4.fits"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meta[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Critical imformation\n",
    "From each image we need:\n",
    "\n",
    "1. Corner pixels\n",
    "2. File sizes\n",
    "3. Checksums"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "92a2ea799e73665f538ddb95d9151c51\n"
     ]
    }
   ],
   "source": [
    "def file_as_bytes(file):\n",
    "    with file:\n",
    "        return file.read()\n",
    "\n",
    "test_hash = hashlib.md5(file_as_bytes(open(meta[0]['file'], 'rb'))).hexdigest()\n",
    "print( test_hash)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "227 ms ± 3.49 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "h = hashlib.md5(file_as_bytes(open(meta[0]['file'], 'rb'))).hexdigest()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.2.2 Run on all images\n",
    "Now lets loop over all the images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "meta.add_column(Column(data=np.full(len(meta),  np.nan), name='ra_0_0'))\n",
    "meta.add_column(Column(data=np.full(len(meta),  np.nan), name='ra_0_y'))\n",
    "meta.add_column(Column(data=np.full(len(meta),  np.nan), name='ra_x_0'))\n",
    "meta.add_column(Column(data=np.full(len(meta),  np.nan), name='ra_x_y'))\n",
    "meta.add_column(Column(data=np.full(len(meta),  np.nan), name='dec_0_0'))\n",
    "meta.add_column(Column(data=np.full(len(meta),  np.nan), name='dec_0_y'))\n",
    "meta.add_column(Column(data=np.full(len(meta),  np.nan), name='dec_x_0'))\n",
    "meta.add_column(Column(data=np.full(len(meta),  np.nan), name='dec_x_y'))\n",
    "meta.add_column(Column(data=np.full(len(meta),  0), name='size'))\n",
    "meta.add_column(Column(data=np.full(len(meta),  ''), name='hash', dtype='S{}'.format(len(test_hash))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "for im in meta:\n",
    "    if not RUN_HSC:\n",
    "        break\n",
    "    try:\n",
    "        file = im[\"file\"]\n",
    "        #print(file)\n",
    "        hdu = fits.open(file)\n",
    "        wcs = WCS(hdu[1].header)\n",
    "        ra, dec = wcs.all_pix2world([0,0,hdu[1].header['NAXIS1'],hdu[1].header['NAXIS1']], \n",
    "                            [0,hdu[1].header['NAXIS2'],0,hdu[1].header['NAXIS2']], 1) \n",
    "        im['ra_0_0'] = ra[0]\n",
    "        im['ra_0_y'] = ra[1]\n",
    "        im['ra_x_0'] = ra[2]\n",
    "        im['ra_x_y'] = ra[3]\n",
    "        im['dec_0_0'] = dec[0]\n",
    "        im['dec_0_y'] = dec[1]\n",
    "        im['dec_x_0'] = dec[2]\n",
    "        im['dec_x_y'] = dec[3]   \n",
    "        im['size'] =  os.stat(im['file']).st_size\n",
    "        if RUN_HASH:\n",
    "            im['hash'] = hashlib.md5(file_as_bytes(open(im['file'], 'rb'))).hexdigest()\n",
    "    except:\n",
    "        print(im['file'],' failed')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<i>Table length=5</i>\n",
       "<table id=\"table140383562643216-55873\" class=\"table-striped table-bordered table-condensed\">\n",
       "<thead><tr><th>idx</th><th>file</th><th>ra_0_0</th><th>ra_0_y</th><th>ra_x_0</th><th>ra_x_y</th><th>dec_0_0</th><th>dec_0_y</th><th>dec_x_0</th><th>dec_x_y</th><th>size</th><th>hash</th></tr></thead>\n",
       "<tr><td>0</td><td>/Users/raphaelshirley/Documents/github/lsst-ir-fusion/dmu0/dmu0_HSC/data/hsc-release.mtk.nao.ac.jp/archive/filetree/pdr2_dud/deepCoadd-results/HSC-Y/8524/5,5/calexp-HSC-Y-8524-5,5.fits</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>0</td><td></td></tr>\n",
       "<tr><td>1</td><td>/Users/raphaelshirley/Documents/github/lsst-ir-fusion/dmu0/dmu0_HSC/data/hsc-release.mtk.nao.ac.jp/archive/filetree/pdr2_dud/deepCoadd-results/HSC-Y/8524/5,3/calexp-HSC-Y-8524-5,3.fits</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>0</td><td></td></tr>\n",
       "<tr><td>2</td><td>/Users/raphaelshirley/Documents/github/lsst-ir-fusion/dmu0/dmu0_HSC/data/hsc-release.mtk.nao.ac.jp/archive/filetree/pdr2_dud/deepCoadd-results/HSC-Y/8524/5,4/calexp-HSC-Y-8524-5,4.fits</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>0</td><td></td></tr>\n",
       "<tr><td>3</td><td>/Users/raphaelshirley/Documents/github/lsst-ir-fusion/dmu0/dmu0_HSC/data/hsc-release.mtk.nao.ac.jp/archive/filetree/pdr2_dud/deepCoadd-results/HSC-Y/8524/4,5/calexp-HSC-Y-8524-4,5.fits</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>0</td><td></td></tr>\n",
       "<tr><td>4</td><td>/Users/raphaelshirley/Documents/github/lsst-ir-fusion/dmu0/dmu0_HSC/data/hsc-release.mtk.nao.ac.jp/archive/filetree/pdr2_dud/deepCoadd-results/HSC-Y/8524/4,4/calexp-HSC-Y-8524-4,4.fits</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>0</td><td></td></tr>\n",
       "</table><style>table.dataTable {clear: both; width: auto !important; margin: 0 !important;}\n",
       ".dataTables_info, .dataTables_length, .dataTables_filter, .dataTables_paginate{\n",
       "display: inline-block; margin-right: 1em; }\n",
       ".paginate_button { margin-right: 5px; }\n",
       "</style>\n",
       "<script>\n",
       "\n",
       "var astropy_sort_num = function(a, b) {\n",
       "    var a_num = parseFloat(a);\n",
       "    var b_num = parseFloat(b);\n",
       "\n",
       "    if (isNaN(a_num) && isNaN(b_num))\n",
       "        return ((a < b) ? -1 : ((a > b) ? 1 : 0));\n",
       "    else if (!isNaN(a_num) && !isNaN(b_num))\n",
       "        return ((a_num < b_num) ? -1 : ((a_num > b_num) ? 1 : 0));\n",
       "    else\n",
       "        return isNaN(a_num) ? -1 : 1;\n",
       "}\n",
       "\n",
       "require.config({paths: {\n",
       "    datatables: 'https://cdn.datatables.net/1.10.12/js/jquery.dataTables.min'\n",
       "}});\n",
       "require([\"datatables\"], function(){\n",
       "    console.log(\"$('#table140383562643216-55873').dataTable()\");\n",
       "    \n",
       "jQuery.extend( jQuery.fn.dataTableExt.oSort, {\n",
       "    \"optionalnum-asc\": astropy_sort_num,\n",
       "    \"optionalnum-desc\": function (a,b) { return -astropy_sort_num(a, b); }\n",
       "});\n",
       "\n",
       "    $('#table140383562643216-55873').dataTable({\n",
       "        order: [],\n",
       "        pageLength: 50,\n",
       "        lengthMenu: [[10, 25, 50, 100, 500, 1000, -1], [10, 25, 50, 100, 500, 1000, 'All']],\n",
       "        pagingType: \"full_numbers\",\n",
       "        columnDefs: [{targets: [0, 2, 3, 4, 5, 6, 7, 8, 9, 10], type: \"optionalnum\"}]\n",
       "    });\n",
       "});\n",
       "</script>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meta[:5].show_in_notebook()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "if RUN_HSC:\n",
    "    meta.write('./data/hsc_images_overview_{}.csv'.format(SUFFIX), overwrite=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.2 VISTA images\n",
    "\n",
    "Now lets investigate the VISTA images."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "v_files = glob.glob(data_loc[SURVEY] + \"/**/*.fit\", recursive=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "v_meta = Table()\n",
    "v_meta.add_column(Column(data=v_files, name=\"file\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "v_meta.add_column(Column(data=np.full(len(v_meta),  np.nan), name='ra'))\n",
    "v_meta.add_column(Column(data=np.full(len(v_meta),  np.nan), name='dec'))\n",
    "v_meta.add_column(Column(data=np.full(len(v_meta),  np.nan), name='ra_0_0'))\n",
    "v_meta.add_column(Column(data=np.full(len(v_meta),  np.nan), name='ra_0_y'))\n",
    "v_meta.add_column(Column(data=np.full(len(v_meta),  np.nan), name='ra_x_0'))\n",
    "v_meta.add_column(Column(data=np.full(len(v_meta),  np.nan), name='ra_x_y'))\n",
    "v_meta.add_column(Column(data=np.full(len(v_meta),  np.nan), name='dec_0_0'))\n",
    "v_meta.add_column(Column(data=np.full(len(v_meta),  np.nan), name='dec_0_y'))\n",
    "v_meta.add_column(Column(data=np.full(len(v_meta),  np.nan), name='dec_x_0'))\n",
    "v_meta.add_column(Column(data=np.full(len(v_meta),  np.nan), name='dec_x_y'))\n",
    "v_meta.add_column(Column(data=np.full(len(v_meta),  '        '), name='filter'))\n",
    "v_meta.add_column(Column(data=np.full(len(v_meta),  0), name='size'))\n",
    "v_meta.add_column(Column(data=np.full(len(v_meta),  0), name='visit'))\n",
    "v_meta.add_column(Column(data=np.full(len(v_meta),  0), name='exptime'))\n",
    "v_meta.add_column(Column(data=np.full(len(v_meta),  ''), name='hash', dtype='S{}'.format(len(test_hash))))\n",
    "v_meta.add_column(Column(data=np.full(len(v_meta),  np.nan), name='MAGZPT'))\n",
    "v_meta.add_column(Column(data=np.full(len(v_meta),  np.nan), name='MAGZRR'))\n",
    "v_meta.add_column(Column(data=np.full(len(v_meta),  np.nan), name='SEEING'))\n",
    "v_meta.add_column(Column(data=np.full(len(v_meta),  np.nan), name='SKYLEVEL'))\n",
    "v_meta.add_column(Column(data=np.full(len(v_meta),  np.nan), name='SKYNOISE'))\n",
    "v_meta.add_column(Column(data=np.full(len(v_meta),  '          '), name='OBSTATUS'))\n",
    "v_meta.add_column(Column(data=np.full(len(v_meta),  '          '), name='ESOGRADE'))\n",
    "v_meta.add_column(Column(data=np.full(len(v_meta),  0), name='OBSNUM'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_im = 0\n",
    "if RESTART_VISTA:\n",
    "    checkpoints = glob.glob('./data/{}_images_overview_temp_*.csv'.format(SURVEY))\n",
    "    n_im = np.max(np.array([int(t.split('_')[-1][:-4] ) for t in checkpoints] ))\n",
    "    restart_file = './data/{}_images_overview_temp_{}.csv'.format(SURVEY,n_im)\n",
    "    print(\"Restarting with: {}\".format(restart_file))\n",
    "    v_meta = Table.read(restart_file)\n",
    "    #START_VISTA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: VerifyWarning: Verification reported errors: [astropy.io.fits.verify]\n",
      "WARNING: VerifyWarning: Card 'ESO DET CHIP PXSPACE' is not FITS standard (invalid value string: '2.000e-05 / Pixel-Pixel Spacing').  Fixed 'ESO DET CHIP PXSPACE' card to meet the FITS standard. [astropy.io.fits.verify]\n",
      "WARNING: VerifyWarning: Note: astropy.io.fits uses zero-based indexing.\n",
      " [astropy.io.fits.verify]\n",
      "WARNING: FITSFixedWarning: RADECSYS= 'ICRS ' / Coordinate reference frame \n",
      "the RADECSYS keyword is deprecated, use RADESYSa. [astropy.wcs.wcs]\n",
      "WARNING: FITSFixedWarning: PROJP1 = 1. / Old style WCS \n",
      "the PROJPn keyword is deprecated, use PVi_ma. [astropy.wcs.wcs]\n",
      "WARNING: FITSFixedWarning: PROJP3 = 44. / Old style WCS \n",
      "the PROJPn keyword is deprecated, use PVi_ma. [astropy.wcs.wcs]\n",
      "WARNING: FITSFixedWarning: PROJP5 = -10300. / Old style WCS \n",
      "the PROJPn keyword is deprecated, use PVi_ma. [astropy.wcs.wcs]\n",
      "WARNING: FITSFixedWarning: RADECSYS= 'FK5 ' / Coordinate reference frame \n",
      "the RADECSYS keyword is deprecated, use RADESYSa. [astropy.wcs.wcs]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "for im in v_meta[v_meta['size']==0]:\n",
    " \n",
    "    if not RUN_VISTA:\n",
    "        break\n",
    "    file = im[\"file\"]\n",
    "    #print(file)\n",
    "    n_im += 1\n",
    "    try:\n",
    "        hdu = fits.open(file)\n",
    "        wcs = WCS(hdu[1].header)\n",
    "        ra, dec = wcs.all_pix2world([0,0,hdu[1].header['NAXIS1'],hdu[1].header['NAXIS1']], \n",
    "                            [0,hdu[1].header['NAXIS2'],0,hdu[1].header['NAXIS2']], 1) \n",
    "        this_file = v_meta['file'] == file\n",
    "        if file.endswith('_tl.fit'):\n",
    "            v_meta['ra'][this_file] = hdu[1].header['RA']\n",
    "            v_meta['dec'][this_file] = hdu[1].header['DEC'] \n",
    "            v_meta['filter'][this_file] = hdu[1].header['HIERARCH ESO INS FILT1 NAME']\n",
    "        else:\n",
    "            v_meta['ra'][this_file] = hdu[0].header['RA']\n",
    "            v_meta['dec'][this_file] = hdu[0].header['DEC']\n",
    "            v_meta['filter'][this_file] = hdu[0].header['HIERARCH ESO INS FILT1 NAME']\n",
    "        v_meta['ra_0_0'][this_file] = ra[0]\n",
    "        v_meta['ra_0_y'][this_file] = ra[1]\n",
    "        v_meta['ra_x_0'][this_file] = ra[2]\n",
    "        v_meta['ra_x_y'][this_file] = ra[3]\n",
    "        v_meta['dec_0_0'][this_file] = dec[0]\n",
    "        v_meta['dec_0_y'][this_file] = dec[1]\n",
    "        v_meta['dec_x_0'][this_file] = dec[2]\n",
    "        v_meta['dec_x_y'][this_file] = dec[3]   \n",
    "        v_meta['size'][this_file] =  os.stat(im['file']).st_size\n",
    "        if RUN_HASH:\n",
    "            v_meta['hash'][this_file] = hashlib.md5(file_as_bytes(open(im['file'], 'rb'))).hexdigest()\n",
    "        \n",
    "        v_meta['visit'][this_file] = hdu[1].header['ESO DET EXP NO']\n",
    "        v_meta['exptime'][this_file] = hdu[1].header['EXPTIME']\n",
    "        v_meta['MAGZPT'][this_file] = hdu[1].header['MAGZPT']\n",
    "        v_meta['MAGZRR'][this_file] = hdu[1].header['MAGZRR']\n",
    "        v_meta['SEEING'][this_file] = hdu[1].header['SEEING']\n",
    "        v_meta['SKYLEVEL'][this_file] = hdu[1].header['SKYLEVEL']\n",
    "        v_meta['SKYNOISE'][this_file] = hdu[1].header['SKYNOISE']\n",
    "        v_meta['OBSTATUS'][this_file] = hdu[0].header['OBSTATUS']\n",
    "        v_meta['ESOGRADE'][this_file] = hdu[0].header['ESOGRADE']\n",
    "        v_meta['OBSNUM'][this_file] = hdu[0].header['OBSNUM']\n",
    "\n",
    "\n",
    "    except:\n",
    "        pass\n",
    "    if n_im % 10000 == 0:\n",
    "        v_meta.write('./data/{}_images_overview_temp_{}.csv'.format(SURVEY,n_im), overwrite=True)\n",
    "        \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<i>Table length=5</i>\n",
       "<table id=\"table140381946686800-106850\" class=\"table-striped table-bordered table-condensed\">\n",
       "<thead><tr><th>idx</th><th>file</th><th>ra</th><th>dec</th><th>ra_0_0</th><th>ra_0_y</th><th>ra_x_0</th><th>ra_x_y</th><th>dec_0_0</th><th>dec_0_y</th><th>dec_x_0</th><th>dec_x_y</th><th>filter</th><th>size</th><th>visit</th><th>exptime</th><th>hash</th><th>MAGZPT</th><th>MAGZRR</th><th>SEEING</th><th>SKYLEVEL</th><th>SKYNOISE</th><th>OBSTATUS</th><th>ESOGRADE</th><th>OBSNUM</th></tr></thead>\n",
       "<tr><td>0</td><td>/Users/raphaelshirley/Documents/github/lsst-ir-fusion/dmu0/dmu0_VISTA/dmu0_VIDEO/data/20121122/v20121122_00164.fit</td><td>35.523512</td><td>-4.90954</td><td>36.031556697704694</td><td>36.032987518376984</td><td>35.83912465612438</td><td>35.84004103666224</td><td>-4.266153773810008</td><td>-4.4575008533645475</td><td>-4.265043153691725</td><td>-4.456717195732584</td><td>Ks</td><td>109091520</td><td>658729</td><td>60</td><td></td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td></td><td></td><td>0</td></tr>\n",
       "<tr><td>1</td><td>/Users/raphaelshirley/Documents/github/lsst-ir-fusion/dmu0/dmu0_VISTA/dmu0_VIDEO/data/20121122/v20121122_00170.fit</td><td>35.428942</td><td>-4.90593</td><td>35.93707357978604</td><td>35.93847751340113</td><td>35.744642584201834</td><td>35.74553207435302</td><td>-4.262619875795639</td><td>-4.453967153311885</td><td>-4.261482399351048</td><td>-4.453156567314746</td><td>Ks</td><td>109408320</td><td>658735</td><td>60</td><td></td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td></td><td></td><td>0</td></tr>\n",
       "<tr><td>2</td><td>/Users/raphaelshirley/Documents/github/lsst-ir-fusion/dmu0/dmu0_VISTA/dmu0_VIDEO/data/20121122/v20121122_00158.fit</td><td>35.527642</td><td>-4.91272</td><td>36.035687968877426</td><td>36.037118890422946</td><td>35.8432551302478</td><td>35.84417157367037</td><td>-4.269332535792607</td><td>-4.4606796146422285</td><td>-4.268221992414202</td><td>-4.459896034174582</td><td>Ks</td><td>109287360</td><td>658723</td><td>60</td><td></td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td></td><td></td><td>0</td></tr>\n",
       "<tr><td>3</td><td>/Users/raphaelshirley/Documents/github/lsst-ir-fusion/dmu0/dmu0_VISTA/dmu0_VIDEO/data/20121122/v20121122_00102_st.fit</td><td>35.616442</td><td>-4.72313</td><td>36.12365296903962</td><td>36.12552008187017</td><td>35.92586958047189</td><td>35.92719310164043</td><td>-4.077977314187725</td><td>-4.276795598599529</td><td>-4.077171797589524</td><td>-4.276339668965059</td><td>Ks</td><td>84542400</td><td>658667</td><td>10</td><td></td><td>23.05</td><td>0.045</td><td>2.018367</td><td>13080.05</td><td>13.89</td><td>Completed</td><td>A</td><td>45</td></tr>\n",
       "<tr><td>4</td><td>/Users/raphaelshirley/Documents/github/lsst-ir-fusion/dmu0/dmu0_VISTA/dmu0_VIDEO/data/20121122/v20121122_00159.fit</td><td>35.525487</td><td>-4.90308</td><td>36.033526927001006</td><td>36.03495754271935</td><td>35.84109650316589</td><td>35.84201275593501</td><td>-4.2596945858889335</td><td>-4.451041666875689</td><td>-4.258583809838531</td><td>-4.450257852449261</td><td>Ks</td><td>109255680</td><td>658724</td><td>60</td><td></td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td>nan</td><td></td><td></td><td>0</td></tr>\n",
       "</table><style>table.dataTable {clear: both; width: auto !important; margin: 0 !important;}\n",
       ".dataTables_info, .dataTables_length, .dataTables_filter, .dataTables_paginate{\n",
       "display: inline-block; margin-right: 1em; }\n",
       ".paginate_button { margin-right: 5px; }\n",
       "</style>\n",
       "<script>\n",
       "\n",
       "var astropy_sort_num = function(a, b) {\n",
       "    var a_num = parseFloat(a);\n",
       "    var b_num = parseFloat(b);\n",
       "\n",
       "    if (isNaN(a_num) && isNaN(b_num))\n",
       "        return ((a < b) ? -1 : ((a > b) ? 1 : 0));\n",
       "    else if (!isNaN(a_num) && !isNaN(b_num))\n",
       "        return ((a_num < b_num) ? -1 : ((a_num > b_num) ? 1 : 0));\n",
       "    else\n",
       "        return isNaN(a_num) ? -1 : 1;\n",
       "}\n",
       "\n",
       "require.config({paths: {\n",
       "    datatables: 'https://cdn.datatables.net/1.10.12/js/jquery.dataTables.min'\n",
       "}});\n",
       "require([\"datatables\"], function(){\n",
       "    console.log(\"$('#table140381946686800-106850').dataTable()\");\n",
       "    \n",
       "jQuery.extend( jQuery.fn.dataTableExt.oSort, {\n",
       "    \"optionalnum-asc\": astropy_sort_num,\n",
       "    \"optionalnum-desc\": function (a,b) { return -astropy_sort_num(a, b); }\n",
       "});\n",
       "\n",
       "    $('#table140381946686800-106850').dataTable({\n",
       "        order: [],\n",
       "        pageLength: 50,\n",
       "        lengthMenu: [[10, 25, 50, 100, 500, 1000, -1], [10, 25, 50, 100, 500, 1000, 'All']],\n",
       "        pagingType: \"full_numbers\",\n",
       "        columnDefs: [{targets: [0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 17, 18, 19, 20, 21, 24], type: \"optionalnum\"}]\n",
       "    });\n",
       "});\n",
       "</script>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v_meta[:5].show_in_notebook()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "if RUN_VISTA:\n",
    "    v_meta.write('./data/{}_images_overview_{}.csv'.format(SURVEY, SUFFIX), overwrite=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# file='/Users/raphaelshirley/Documents/github/lsst-ir-fusion/dmu0/dmu0_VISTA/dmu0_VIDEO/data/20121122/v20121122_00102_st.fit'\n",
    "# hdu = fits.open(file)\n",
    "# hdu[0].header"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "lsst",
   "language": "python",
   "name": "lsst"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}