tidal_classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# tidal_classification\n",
    "\n",
    "**Summary**: Iterates through ITEMv2 polygons, extracts Landsat observations and calculates offsets and spread of observations compared to entire OTPS tidal model. Results exported to .csv output files.\n",
    "\n",
    "**Issues**: Currently fails after polygon 220 when producing annotated plots.\n",
    "\n",
    "**Notes**: Tidal model code based on original by Claire Phillips."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 7,
        "height": 5,
        "hidden": false,
        "row": 42,
        "width": 4
       },
       "report_default": {}
      }
     }
    }
   },
   "outputs": [],
   "source": [
    "# Import all modules\n",
    "import datacube\n",
    "import xarray as xr\n",
    "import matplotlib.dates\n",
    "import fiona\n",
    "import shapely\n",
    "import shapely.geometry\n",
    "import rasterio\n",
    "import rasterio.features\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import itertools\n",
    "import otps\n",
    "import sys\n",
    "import pandas as pd\n",
    "import matplotlib as mpl\n",
    "\n",
    "from __future__ import print_function\n",
    "from shapely.geometry import Point, Polygon, MultiPolygon, mapping, shape\n",
    "from fiona import collection\n",
    "from datacube.utils import geometry\n",
    "from datacube.api.query import query_group_by\n",
    "from otps import TimePoint, predict_tide\n",
    "from dateutil.relativedelta import relativedelta\n",
    "from operator import itemgetter\n",
    "from datetime import date, datetime, timedelta\n",
    "from matplotlib import cm\n",
    "from datetime import datetime as datetime\n",
    "from IPython.display import display\n",
    "\n",
    "dc = datacube.Datacube(app='Tidal offsets')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "extensions": {
     "jupyter_dashboards": {
      "version": 1,
      "views": {
       "grid_default": {
        "col": 0,
        "height": 4,
        "hidden": false,
        "row": 71,
        "width": 4
       },
       "report_default": {}
      }
     }
    }
   },
   "source": [
    "# User to define primary variables - Location"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing polygon 221\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0520951302378 0.14892412231 0.0447338618347 0.806342015855\n",
      "Processing polygon 222\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.066000377145 0.222326984726 0.0903262304356 0.687346784839\n",
      "Processing polygon 223\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0710676995621 0.210845402492 0.0687100033681 0.720444594139\n",
      "Processing polygon 224\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0649466192171 0.139383155397 0.00948991696323 0.851126927639\n",
      "Processing polygon 225\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0149342105263 0.134473684211 0.104605263158 0.760921052632\n",
      "Processing polygon 226\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0518272425249 0.222425249169 0.11877076412 0.658803986711\n",
      "Processing polygon 227\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0884882566986 0.212371816077 0.0353953026795 0.752232881244\n",
      "Processing polygon 228\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0682689674062 0.17775951794 0.0412215831279 0.781018898932\n",
      "Processing polygon 229\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0763234227701 0.220812182741 0.0681653372009 0.711022480058\n",
      "Processing polygon 230\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.00217391304348 0.0627329192547 0.0583850931677 0.878881987578\n",
      "Processing polygon 231\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    -0.0655339805825 0.0728155339806 0.203883495146 0.723300970874\n",
      "Processing polygon 232\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0 0.0266463646745 0.0266463646745 0.946707270651\n",
      "Processing polygon 233\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0943694340707 0.228095374892 0.0393565067509 0.732548118357\n",
      "Processing polygon 234\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    -0.0982329317269 0.0128514056225 0.209317269076 0.777831325301\n",
      "Processing polygon 235\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    -0.00889110889111 0.135164835165 0.152947052947 0.711888111888\n",
      "Processing polygon 236\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0364444444444 0.120888888889 0.048 0.831111111111\n",
      "Processing polygon 237\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0321199143469 0.163169164882 0.0989293361884 0.737901498929\n",
      "Processing polygon 238\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0236426116838 0.19175257732 0.144467353952 0.663780068729\n",
      "Processing polygon 239\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    -0.0482497634816 0.0223904131189 0.118889940082 0.858719646799\n",
      "Processing polygon 240\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0327360829389 0.216137029524 0.150664863647 0.633198106829\n",
      "Processing polygon 241\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.069197630587 0.144318793753 0.00592353257943 0.849757673667\n",
      "Processing polygon 242\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    -0.00479573712256 0.0238010657194 0.0333925399645 0.942806394316\n",
      "Processing polygon 243\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.151700888753 0.316273368066 0.0128715905608 0.670855041373\n",
      "Processing polygon 244\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    -0.00285714285714 0.0991836734694 0.104897959184 0.795918367347\n",
      "Processing polygon 245\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0319693094629 0.134697357204 0.0707587382779 0.794543904518\n",
      "Processing polygon 246\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0962921970116 0.244050913116 0.0514665190924 0.704482567792\n",
      "Processing polygon 247\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0964052287582 0.216230936819 0.0234204793028 0.760348583878\n",
      "Processing polygon 248\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.00161030595813 0.0660225442834 0.0628019323671 0.871175523349\n",
      "Processing polygon 249\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0734226689001 0.163316582915 0.0164712451145 0.820212171971\n",
      "Processing polygon 250\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    -0.0578778135048 0.0144694533762 0.130225080386 0.855305466238\n",
      "Processing polygon 251\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0901029748284 0.217677345538 0.037471395881 0.744851258581\n",
      "Processing polygon 252\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0472703779477 0.208571120922 0.114030365026 0.677398514052\n",
      "Processing polygon 253\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0752332229913 0.161901895877 0.0114354498947 0.826662654228\n",
      "Processing polygon 254\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.117107393416 0.257420399352 0.0232056125202 0.719373988127\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing polygon 255\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0065180102916 0.0905660377358 0.0775300171527 0.831903945111\n",
      "Processing polygon 256\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0862469171623 0.196286087335 0.0237922530103 0.779921659655\n",
      "Processing polygon 257\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.00773036487322 0.170686456401 0.155225726654 0.674087816945\n",
      "Processing polygon 258\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.1094466524 0.243656374197 0.0247630693977 0.731580556405\n",
      "Processing polygon 259\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.108637873754 0.233554817276 0.0162790697674 0.750166112957\n",
      "Processing polygon 260\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0506153846154 0.141538461538 0.0403076923077 0.818153846154\n",
      "Processing polygon 261\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    -0.0226143141153 0.0929423459245 0.138170974155 0.76888667992\n",
      "Processing polygon 262\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    -0.0218346091103 0.128623415736 0.172292633957 0.699083950307\n",
      "Processing polygon 263\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.104470938897 0.258122205663 0.0491803278689 0.692697466468\n",
      "Processing polygon 264\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    -0.0149174214172 0.0138518913159 0.0436867341502 0.942461374534\n",
      "Processing polygon 265\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    -0.00765027322404 0.048087431694 0.0633879781421 0.888524590164\n",
      "Processing polygon 266\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0791112607305 0.187847163777 0.0296246423161 0.782528193907\n",
      "Processing polygon 267\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0271557884707 0.0828966174369 0.0285850404955 0.888518342068\n",
      "Processing polygon 268\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    -0.0425420625926 0.116729143056 0.201813268242 0.681457588702\n",
      "Processing polygon 269\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0902743142145 0.188029925187 0.0074812967581 0.804488778055\n",
      "Processing polygon 270\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0219628017412 0.189157103285 0.145231499802 0.665611396913\n",
      "Processing polygon 271\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    -0.034595034595 0.0777370777371 0.146927146927 0.775335775336\n",
      "Processing polygon 272\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0528374105508 0.226660009985 0.120985188883 0.652354801132\n",
      "Processing polygon 273\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.119509939898 0.254507628294 0.0154877484975 0.730004623209\n",
      "Processing polygon 274\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.00248175182482 0.0621897810219 0.0572262773723 0.880583941606\n",
      "Processing polygon 275\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0100572705685 0.16105601341 0.140941472273 0.698002514318\n",
      "Processing polygon 276\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0708196721311 0.16131147541 0.0196721311475 0.819016393443\n",
      "Processing polygon 277\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.073756432247 0.203259005146 0.0557461406518 0.740994854202\n",
      "Processing polygon 278\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0966963622866 0.23422420193 0.0408314773571 0.724944320713\n",
      "Processing polygon 279\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0491679273828 0.192637418053 0.0943015632879 0.713061018659\n",
      "Processing polygon 280\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0775347912525 0.207256461233 0.0521868787276 0.74055666004\n",
      "Processing polygon 281\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0 0.0223930481283 0.0223930481283 0.955213903743\n",
      "Processing polygon 282\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0366379310345 0.104525862069 0.03125 0.864224137931\n",
      "Processing polygon 283\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    -0.062816493851 0.0854432923398 0.211076280042 0.703480427618\n",
      "Processing polygon 284\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0689473684211 0.142105263158 0.00421052631579 0.853684210526\n",
      "Processing polygon 285\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0771379044684 0.191448382126 0.0371725731895 0.771379044684\n",
      "Processing polygon 286\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    -0.0202205882353 0.0186887254902 0.0591299019608 0.922181372549\n",
      "Processing polygon 287\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0976349302608 0.239539114615 0.0442692540934 0.716191631292\n",
      "Processing polygon 288\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    -0.0737100737101 0.0720720720721 0.219492219492 0.708435708436\n",
      "Processing polygon 289\n",
      "Processing modelled tide heights\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    -0.00534963698892 0.0206343141001 0.031333588078 0.948032097822\n",
      "Processing polygon 290\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    -0.0746896275531 0.046856227473 0.196235482579 0.756908289948\n",
      "Processing polygon 291\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0184753661784 0.109853528628 0.0729027962716 0.8172436751\n",
      "Processing polygon 292\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.118195564516 0.264364919355 0.0279737903226 0.707661290323\n",
      "Processing polygon 293\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    -0.0251009809579 0.0184650894403 0.068667051356 0.912867859204\n",
      "Processing polygon 294\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.00540657439446 0.0428200692042 0.0320069204152 0.925173010381\n",
      "Processing polygon 295\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.00605826243877 0.0690899716422 0.0569734467646 0.873936581593\n",
      "Processing polygon 296\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0390767824497 0.107404021938 0.0292504570384 0.863345521024\n",
      "Processing polygon 297\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0447451707956 0.197860962567 0.108370620976 0.693768416457\n",
      "Processing polygon 298\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0417075564279 0.133586849853 0.0501717369971 0.81624141315\n",
      "Processing polygon 299\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    -0.00218004616568 0.167607078738 0.17196717107 0.660425750192\n",
      "Processing polygon 300\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0890943193998 0.183279742765 0.0050911039657 0.811629153269\n",
      "Processing polygon 301\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.0171086398631 0.110350727117 0.0761334473909 0.813515825492\n",
      "Processing polygon 302\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    -0.0771848137536 0.0279369627507 0.182306590258 0.789756446991\n",
      "Processing polygon 303\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    -0.01585591133 0.0858990147783 0.117610837438 0.796490147783\n",
      "Processing polygon 304\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    -0.0303921568627 0.143137254902 0.203921568627 0.652941176471\n",
      "Processing polygon 305\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.00192023633678 0.0599704579025 0.056129985229 0.883899556869\n",
      "Processing polygon 306\n",
      "Processing modelled tide heights\n",
      "Processing observed tide heights\n",
      "Computing offsets and spread\n",
      "    Offset, low tide offset, high tide offset and spread:\n",
      "    0.05 0.135294117647 0.0352941176471 0.829411764706\n"
     ]
    }
   ],
   "source": [
    "from datetime import datetime as datetime\n",
    "\n",
    "def date_range(start_date, end_date, increment, period):\n",
    "    \n",
    "    \"\"\"Generate dates seperated by given time increment/period\"\"\"\n",
    "    \n",
    "    result = []\n",
    "    nxt = start_date\n",
    "    delta = relativedelta(**{period:increment})\n",
    "    while nxt <= end_date:\n",
    "        result.append(nxt)\n",
    "        nxt += delta\n",
    "    return result\n",
    "\n",
    "\n",
    "def datetime_slicing(input_date):\n",
    "    \n",
    "    \"\"\"\n",
    "    Faster equivelent of datetime.strptime: uses slices to extract \n",
    "    datetime object from date string in format %Y-%m-%d %H:%M:%S\n",
    "    \"\"\"\n",
    "    \n",
    "    try:\n",
    "        year = int(input_date[:4])\n",
    "        month = int(input_date[5:7])\n",
    "        day = int(input_date[8:10])\n",
    "        hour = int(input_date[11:13])\n",
    "        minute = int(input_date[14:16])\n",
    "        my_date = datetime(year, month, day, hour, minute)        \n",
    "        return(my_date)\n",
    "\n",
    "    except ValueError:\n",
    "        my_date = None\n",
    "\n",
    "filepath='/g/data/r78/intertidal/GA_native_tidal_model.shp'\n",
    "products = ['ls5_pq_albers', 'ls7_pq_albers', 'ls8_pq_albers'] \n",
    "time_period=('1986-01-01', '2018-01-01') # Global time range\n",
    "LS7_SLC_DT = datetime.strptime(\"2003-05-01\", \"%Y-%m-%d\") # Removes SLC failure\n",
    "\n",
    "# Create empty list for offset values\n",
    "offset_spread = list()\n",
    "\n",
    "# Iterate through polygons\n",
    "for polygon in range(1, 307):\n",
    "\n",
    "    print(\"Processing polygon \" + str(polygon))\n",
    "\n",
    "    try:\n",
    "\n",
    "        #########################################################\n",
    "        # modelled tide heights and dates for global time range #\n",
    "        #########################################################\n",
    "\n",
    "        print(\"Processing modelled tide heights\")\n",
    "\n",
    "        # Open ITEM polygons and identify lat long (seems awkward?)\n",
    "        with fiona.open(filepath) as Input:\n",
    "            crs = geometry.CRS(str(Input.crs_wkt))\n",
    "\n",
    "            # For each polygon, extract long/lat/ID\n",
    "            for feature in Input:\n",
    "                Id = feature['properties']['ID']            \n",
    "\n",
    "                # If polygon is selected polygon\n",
    "                if Id == polygon:\n",
    "                    lon = feature['properties']['lon']\n",
    "                    lat = feature['properties']['lat']\n",
    "\n",
    "        # Reformat time strings for start and end of period\n",
    "        start = datetime.strptime(time_period[0] + \" 00:00:00\", \"%Y-%m-%d %H:%M:%S\")\n",
    "        end = datetime.strptime(time_period[1] + \" 00:00:00\", \"%Y-%m-%d %H:%M:%S\")\n",
    "\n",
    "        # For each hour between start and end of timeperiod, convert to \n",
    "        # timepoint and add to list for model\n",
    "        tp_model = list(TimePoint(lon, lat, datetime_slicing(str(i))) \n",
    "                        for i in date_range(start, end, 1, 'hours'))\n",
    "\n",
    "        # For each timestep in list, predict tide and add to list\n",
    "        tides_model = predict_tide(tp_model)\n",
    "        tide_det = list()\n",
    "        for tt in tides_model:\n",
    "            tide_det.append([tt.timepoint.timestamp.isoformat()[0:10], tt.tide_m])\n",
    "\n",
    "        # Create pd dataframe of dates and heights\n",
    "        df2_data = {'Model_height': list(x[1] for x in sorted(tide_det))}\n",
    "        df2_index = list(x[0] for x in sorted (tide_det))\n",
    "        df2_model = pd.DataFrame(df2_data, index=df2_index) \n",
    "        \n",
    "\n",
    "        #########################################################\n",
    "        # Observed tide heights and dates for global time range #\n",
    "        #########################################################\n",
    "\n",
    "        print(\"Processing observed tide heights\")\n",
    "\n",
    "        all_times = list()\n",
    "\n",
    "        # Open ITEM polygons\n",
    "        with fiona.open(filepath) as Input:\n",
    "            crs = geometry.CRS(str(Input.crs_wkt))\n",
    "\n",
    "            # For each polygon, extract long/lat/ID\n",
    "            for feature in Input:\n",
    "                lon = feature['properties']['lon']\n",
    "                lat = feature['properties']['lat']\n",
    "                Id = feature['properties']['ID']            \n",
    "\n",
    "                # If polygon is selected polygon\n",
    "                if Id == polygon:\n",
    "\n",
    "                    # Take first geometry and and identify area covered for input to dc call\n",
    "                    first_geometry = feature['geometry']\n",
    "                    geom = geometry.Geometry(first_geometry, crs=crs)\n",
    "\n",
    "                    # For each product:                                \n",
    "                    for source in products:                 \n",
    "                        time_range = ('1986-01-01', '2003-05-01') if source == 'ls7_pq_albers' else time_period\n",
    "\n",
    "                        # Determine matching datasets for geom area grouped by solar day\n",
    "                        ds = dc.find_datasets(product=source, \n",
    "                                              time=time_range, \n",
    "                                              geopolygon=geom, \n",
    "                                              group_by='solar_day')\n",
    "                        group_by = query_group_by(group_by='solar_day')\n",
    "                        sources = dc.group_datasets(ds, group_by)\n",
    "\n",
    "                        # If data is found, add time to list then sort\n",
    "                        if len(ds) > 0 :\n",
    "                            all_times = all_times + [dd for dd in sources.time.data.astype('M8[s]').astype('O').tolist()] \n",
    "                        all_times = sorted(all_times)   \n",
    "\n",
    "                    break\n",
    "\n",
    "        # Calculate tide data from X-Y-time location \n",
    "        tp = list()\n",
    "        for dt in all_times:\n",
    "            tp.append(TimePoint(lon, lat, dt))\n",
    "        tides = predict_tide(tp)\n",
    "\n",
    "        # Add resulting data to dict and sort\n",
    "        tide_dic = dict()\n",
    "        for tt in tides:\n",
    "            tt_time = datetime_slicing(tt.timepoint.timestamp.isoformat()[0:19])\n",
    "            tide_dic[tt_time] = tt.tide_m\n",
    "        tide_data = sorted(tide_dic.items(), key=lambda x: x[1])\n",
    "\n",
    "        # Create dataframe of tide heights, indexed by date/time (UTC)\n",
    "        df1_data = {'Tide_height': list((x[1] for x in sorted(tide_data)))}\n",
    "        df1_index = list(x[0].strftime('%Y-%m-%d') for x in sorted (tide_data))\n",
    "        df1_obs = pd.DataFrame(df1_data, index=df1_index)\n",
    "\n",
    "\n",
    "        ########################################\n",
    "        # Tide observations offsets and spread #\n",
    "        ########################################\n",
    "\n",
    "        print(\"Computing offsets and spread\")\n",
    "\n",
    "        # Modelled data stats\n",
    "        model_low = df2_model.Model_height.min()\n",
    "        model_high = df2_model.Model_height.max()\n",
    "        model_range = model_high - model_low\n",
    "\n",
    "        # Observed data stats\n",
    "        obs_low = df1_obs.Tide_height.min()\n",
    "        obs_high = df1_obs.Tide_height.max()\n",
    "        obs_range = obs_high - obs_low\n",
    "\n",
    "        # Calculate offsets and spreads\n",
    "        offset = (((obs_low + obs_high) / 2) - model_low) / model_range - 0.5\n",
    "        offset_low = (obs_low - model_low) / model_range\n",
    "        offset_high = (model_high - obs_high) / model_range\n",
    "        spread = obs_range / model_range\n",
    "\n",
    "        # Append to list\n",
    "        print(\"    Offset, low tide offset, high tide offset and spread:\")\n",
    "        print(\"   \", offset, offset_low, offset_high, spread)    \n",
    "        offset_spread.append([polygon, offset, offset_low, offset_high, spread])\n",
    "\n",
    "\n",
    "#         ########\n",
    "#         # Plot #\n",
    "#         ########\n",
    "\n",
    "#         try:\n",
    "\n",
    "#             # Plot setup\n",
    "#             fig = plt.figure(figsize=(9, 3))\n",
    "#             mpl.rcParams['agg.path.chunksize'] = 10000\n",
    "#             plt.ylabel('Tide height (m)')\n",
    "#             plt.title('Available data for polygon ' + str(polygon) + \n",
    "#                       ' (offset = ' + str(round(offset, 2)) + ', spread = ' + \n",
    "#                       str(round(spread, 2)) + ')')\n",
    "\n",
    "#             # Data series, pepare x-axis\n",
    "#             obs_index = pd.DatetimeIndex(df1_index)\n",
    "#             obs_index = obs_index.to_datetime()\n",
    "#             obs_index = np.array(obs_index.to_pydatetime(), dtype = numpy.datetime64)\n",
    "\n",
    "#             # Plot observations\n",
    "#             scatter(obs_index, df1_obs.Tide_height, s=10, color='black', \n",
    "#                     marker='o', zorder=2, label = 'observations') # All observations\n",
    "\n",
    "#             ## Model values, prepare x-axis\n",
    "#             model_index = pd.DatetimeIndex(df2_index)\n",
    "#             model_index = model_index.to_datetime()\n",
    "#             model_index = np.array(model_index.to_pydatetime(), dtype = numpy.datetime64)\n",
    "\n",
    "#             # Plot model values\n",
    "#             plot(model_index, df2_model.Model_height, color='lightgray', \n",
    "#                  linewidth=0.5, zorder=1, label = 'OTPS model')\n",
    "\n",
    "#             # Create percentage lines\n",
    "#             per10_list = []\n",
    "#             for PERC in [0, offset_low, offset + 0.5, 1-offset_high, 1.0]:\n",
    "#                 lmr = PERC * model_range + model_low  # low tide max range\n",
    "#                 per10_list.append([round(PERC, 2), lmr]) \n",
    "\n",
    "#             per10_data = {'PERC_tide_height': list(x[1] for x in sorted(per10_list))}\n",
    "#             per10_index = list(x[0] for x in sorted(per10_list))\n",
    "#             df_per10 = pd.DataFrame(per10_data, per10_index)\n",
    "\n",
    "#             # Add lines to plot\n",
    "#             for i in per10_list:\n",
    "#                 plot([model_index[0], \n",
    "#                       model_index[-1]], \n",
    "#                      [df_per10.PERC_tide_height[i],                                             \n",
    "#                       df_per10.PERC_tide_height[i]], 'black', linewidth=0.4)\n",
    "#                 plt.text(model_index[-1] + 100, i[1] - 0.025, str(i[0]), fontsize=8)\n",
    "\n",
    "#             # Force space for y axis by adding data to x axis\n",
    "#             plot([(model_index[0]), \n",
    "#                   (model_index[-1] + 1000)], \n",
    "#                  [(df_per10.PERC_tide_height[0] - 0.3),\n",
    "#                   df_per10.PERC_tide_height[0]], 'none', linewidth=0)\n",
    "\n",
    "#             plt.savefig('figures/tidal_polygons/' + str(polygon) + '_tidal_data.jpg')        \n",
    "#             %matplotlib inline\n",
    "#             display(fig)\n",
    "\n",
    "#         except:\n",
    "#             print(\"Figure failed\")\n",
    "\n",
    "        # Close datasets\n",
    "        tides_model = None\n",
    "        tides = None\n",
    "        tide_data = None\n",
    "        tp_model = None\n",
    "        df2_data = None\n",
    "        df2_model = None\n",
    "        df1_data = None\n",
    "        df1_obs = None\n",
    "\n",
    "    except:\n",
    "\n",
    "        print(\"Skipping polygon \" + str(polygon))\n",
    "\n",
    "# Create dataframe\n",
    "offset_spread_index = list(x[0] for x in sorted(offset_spread))\n",
    "offset_spread_df = pd.DataFrame(offset_spread,\n",
    "                                columns=[\"ID\", \"offset\", \n",
    "                                         \"offset_low\", \"offset_high\", \"spread\"],\n",
    "                                index = offset_spread_index)\n",
    "offset_spread_df.to_csv(\"item_offset_spread_1-306.csv\", index = False)\n"
   ]
  }
 ],
 "metadata": {
  "extensions": {
   "jupyter_dashboards": {
    "activeView": "grid_default",
    "version": 1,
    "views": {
     "grid_default": {
      "cellMargin": 10,
      "defaultCellHeight": 20,
      "maxColumns": 12,
      "name": "grid",
      "type": "grid"
     },
     "report_default": {
      "name": "report",
      "type": "report"
     }
    }
   }
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {
    "height": "663px",
    "left": "0px",
    "right": "915px",
    "top": "111px",
    "width": "227px"
   },
   "toc_section_display": "block",
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}