Merge pull request #129 from nismod/feature/chunk_by_country

feature/chunk_by_country

.gitignore

@@ -27,3 +27,7 @@ tester/*
 
 # Rendered documentation
 docs/book/
+
+validation/plots/*
+validation/holland_done_ratio
+validation/wind_model_comparison

README.md

@@ -59,6 +59,10 @@ The workflow leans heavily on the GDAL toolset. To install using APT:
 jq is used to parse JSON files. To install using APT:
 `sudo apt install jq`
 
+### exactextract
+
+exactextract is used for zonal statistics. Please see installation instructions [here](https://github.com/isciences/exactextract).
+
 ## Running tests
 
 Workflow steps are tested using a small sample dataset. Run:
@@ -168,18 +172,13 @@ that uses just one OSM dataset, one hazard dataset, and one slice:
 You can remove all intermediate files by running
 
 ```
-snakemake -c1 clean
+snakemake -c1 -R clean
 ```
 
 Note that this will *not* remove the final data files
 `<output_dir>/<dataset>_filter-<filters>_hazard-<hazard>.geoparquet`,
 nor will it remove the original input files `<output_dir>/input/*`.
 
-You can remove all intermediate data _including_ input files by running
-```
-snakemake -c1 clean_all
-```
-
 Snakemake has utilities to improve the workflow code quality:
 - `snakemake --lint` suggests improvements and fixes for common problems
 - `snakefmt .` reformats files according to a code style guide, similar to `black` for Python code.

archive_results.sh

@@ -88,6 +88,9 @@ Diff:
 $(git diff)
 EOF
 
+# read only permissions
+chmod -R ugo-w $ARCHIVE_DIR
+
 echo "open-gira results directory (excl. input) archived to $ARCHIVE_DIR"
 
 ##### DONE #####

config/config.yaml

@@ -6,7 +6,6 @@ output_dir: 'results'
 ### FLOODING / TRANSPORT WORKFLOW ###
 #####################################
 
-
 ## Aqueduct Analysis ##
 # This should be a named list of files that specify hazard raster files
 # to retrieve using wget -i
@@ -87,12 +86,6 @@ direct_damages:
   # per asset type, per hazard (flood, storm, etc.)
   curves_dir: 'bundled_data/damage_curves/'
 
-
-###############################
-### STORM / ENERGY WORKFLOW ###
-###############################
-
-
 # Options for exposure tif file generation
 exposure_tifs:
   # When generating exposure summaries, filter by cells with at >= exposure_threshold m of flooding
@@ -108,77 +101,50 @@ exposure_tifs:
       # Colour mapping to use for raster data
       cmap: 'Reds'
 
-## STORM Power Network Analysis
 
-# for any given spatial domain, use multiprocessing to parallelise by storm
-# works for estimate_wind_fields.py and grid_disruption.py
-parallelise_by_storm: True
+###############################
+### STORM / ENERGY WORKFLOW ###
+###############################
+
+# sets of storm ids to process for potentially many country networks
+storm_sets:
+    # files containing empty list, indicating to process all in dataset
+    IBTrACS: 'config/storm_sets/ibtracs.json'
+    STORM-constant: 'config/storm_sets/storm-constant.json'
+    STORM-CMCC-CM2-VHR4: 'config/storm_sets/storm-cmcc-cm2-vhr4.json'
+    STORM-CNRM-CM6-1-HR: 'config/storm_sets/storm-cnrm-cm6-1-hr.json'
+    STORM-EC-Earth3P-HR: 'config/storm_sets/storm-ec-earth3p-hr.json'
+    STORM-HadGEM3-GC31-HM: 'config/storm_sets/storm-hadgem3-gc31-hm.json'
+    # files containing a list of storm id strings that constitute the storm set
+    # key should follow the pattern: <dataset>_<storm_set_reference>
+    # where dataset belongs to:
+    # {IBTrACS, STORM-constant, STORM-CMCC-CM2-VHR4, etc.}
+    IBTrACS_maria-2017: 'config/storm_sets/maria.json'
+    IBTrACS_irma-2017: 'config/storm_sets/irma.json'
+    IBTrACS_irma-and-maria: 'irma_and_maria.json'
+    IBTrACS_black-marble-validation: 'config/storm_sets/20230120_black_marble.json'
+
+# consider countries at risk of a storm if within this many degrees of any storm track point
+max_track_search_radius_deg: 2
+
+# wind speed is constant within a square area of sides wind_deg
+# note that for large domains, e.g. USA or CHN, 0.05 deg resolution requires
+# on the order of 4GB RAM per storm wind estimate
+wind_deg: 0.1  # approx 11km latitude
 
 # Failure thresholds m/s. These values are the thresholds at which the network assets
 # are expected to fail based on available literature.
-transmission_windspeed_failure: [20, 35, 50]
+transmission_windspeed_failure: [20., 22.5, 25.0, 27.5, 30., 32.5, 35., 37.5, 40, 42.5, 45]
 
-# width and height of a square cell or 'box' for power/storms workflow
-# dataset-splitting -- must be a factor of 180
-box_deg: 5
-# wind_deg defines another grid which nests inside the above
-# this value must be a factor of box_deg
-# wind speed is constant within a square area of sides wind_deg
-wind_deg: 0.1  # approx 11km latitude
+# some of the scripts and rules are parallelised with multiprocessing
+# works for annotate_targets.py and estimate_wind_fields.py
+# setting to 1 will run any parallelisable job in serial
+# N.B. snakemake will enforce:
+# n_parallelisable_jobs * processes_per_parallel_job + n_serial_jobs <= the cores execution parameter
+processes_per_parallel_job: 24
 
 # whether to plot maximum wind fields and storm track animations for each storm
-# WARNING: this is very slow.. you probably only ever want it for historic tracks
-plot_wind_fields: False
-
-# List of boxes NUMBERS to analyse (only) e.g. [1029, 1028, 1030, 957].
-# Note that this must be known for the box_width_height value entered above.
-# use an empty list to evaluate globally
-specific_boxes: []
-# this contains Puerto Rico (direct hit for Hurricane Maria of 2017)
-# specific_boxes: [1030]
-# box IDs for the Caribbean on a 5 degree grid, east to west, south to north
-# specific_boxes: [1103, 1032, 1030, 1029, 957, 1028, 956, 884, 955, 883]
-
-# STORM BASINS #
-# Tropical cyclone basins to analyse. List of strings.
-# Empty list for global analysis.
-# Options: "EP", "NA", "NI", "SI", "SP", "WP".
-# If `specific_boxes` are selected, this must be coherent with `storm_basins`
-storm_basins: []
-
-# SAMPLES #
-# storm datasets are split into 'samples', each comprising 1,000 years of storm data
-# typically there are 10 samples available for each model
-# to use all available samples, specify an empty list
-storm_files_sample_set: [0]
-
-# Direct damage cost per km (USD) for high voltage lines. 400 000USD suggestion for upper limit
-reconstruction_cost_high: 400000
-
-# Direct damage cost per km (USD) for low and medium voltage lines. XXX 000USD suggestion for upper limit
-reconstruction_cost_lowmedium: 200000
-
-# Further Configuration #
-# Storms will be analysed in batches of this value. Higher value: quicker process. Lower value: better memory. It is recommended to tweak this value dependent on the machine's available memory and/or user requirements (500 is often a good starting value). Note that at below 25, attempts are made to record which files store which data as to only load the relevant wind csv files. This may explain increases/decreases in performance.
-storm_batches: 900
-
-# If set to True then wind_extracter will overwrite existing wind files. It is recommended to keep this as False unless changes are made in the wind configuration files/scripts.
-wind_rerun: True
-
-# Analysis #
-# Percentage of storms to select for geospatial analysis and visualisation. Set to 100 for all
-top_select: 100
-
-# Set increasing severity for top_select. True means top_select is the first top_select% of storms in increasing severity (i.e. first one is the least damaging storm). False means the top top_select% worst storms are analysed (i.e. first on is the most damaging storm)
-increased_severity_sort: True
-
-# Percentile select
-percentile: 99
-
-# Spatial level to aggregate to for geospatial analyses (0 = country and up to 5 inclusive). The higher the number the longer the processing
-aggregate_level: 1
-
-# specific storm IDs to use in grid_disruption.py and subsequently
-# use an empty list to process all available storms
-#specific_storms: ['2017242N16333', '2017260N12310']  # Maria and Irma
-specific_storms: []
+plot_wind:
+    max_speed: True
+    # WARNING: plotting the animation is slow.. you probably only ever want it for historic tracks
+    animation: False

config/hazard_resource_locations/storm_tracks_CMCC-CM2-VHR4.txt

@@ -1 +1 @@
-https://data.4tu.nl/ndownloader/files/27800580
+https://data.4tu.nl/file/98900e17-8e01-4d70-b3b6-ca1a1da2f194/bdc3820c-7b7e-41de-8def-792f50b06c49

config/hazard_resource_locations/storm_tracks_CNRM-CM6-1-HR.txt

@@ -1 +1 @@
-https://data.4tu.nl/ndownloader/files/27800604
+https://data.4tu.nl/file/98900e17-8e01-4d70-b3b6-ca1a1da2f194/3b08e6a8-2824-435f-a141-9509a9922cfe

config/hazard_resource_locations/storm_tracks_EC-Earth3P-HR.txt

@@ -1 +1 @@
-https://data.4tu.nl/ndownloader/files/27800484
+https://data.4tu.nl/file/98900e17-8e01-4d70-b3b6-ca1a1da2f194/2eaeb5f6-40f3-4f94-9f9c-ca7aad3237fc

config/hazard_resource_locations/storm_tracks_HadGEM3-GC31-HM.txt

@@ -1 +1 @@
-https://data.4tu.nl/ndownloader/files/27800550
+https://data.4tu.nl/file/98900e17-8e01-4d70-b3b6-ca1a1da2f194/f35142d6-d858-4874-9a51-2dc08462d45f

config/hazard_resource_locations/storm_tracks_constant.txt

@@ -1,2 +1 @@
-https://data.4tu.nl/ndownloader/files/24876980
-https://data.4tu.nl/ndownloader/files/36391536
+https://data.4tu.nl/file/01b2ebc7-7903-42ef-b46b-f43b9175dbf4/78b72e86-b074-4478-9dc4-7280deb756ab

config/land_use_to_surface_roughness.csv

@@ -0,0 +1,29 @@
+# Data mostly from the Final Report of the Global Wind Atlas project:
+# https://energiforskning.dk/files/slutrapporter/gwa_64011-0347_finalreport.pdf
+# `water bodes` surface roughness changed from 0 to a value used in WRF
+# `wetlands` surface roughness changed from 11 to 1.1 (presumed error)
+# `no data` added with same roughness as `permanent snow and ice`
+"category_name","glob_cover_2009_id","roughness_length_m"
+"water bodies",210,0.0003
+"no data",230,0.004
+"permanent snow and ice",220,0.004
+"bare areas",200,0.005
+"grassland, savannas or lichens/mosses",140,0.03
+"sparse vegetation",150,0.05
+"croplands",11,0.1
+"croplands",14,0.1
+"shrubland",130,0.1
+"mosaic natural vegetation / cropland",20,0.3
+"mosaic natural vegetation / cropland",30,0.3
+"flooded forest",160,0.5
+"mosaic grassland / forest",120,0.5
+"flooded forest or shrubland",170,0.6
+"urban areas",190,1
+"wetlands",180,1.1
+"forest",40,1.5
+"forest",50,1.5
+"forest",60,1.5
+"forest",70,1.5
+"forest",90,1.5
+"forest",100,1.5
+"forest",110,1.5

config/storm_sets/20230120_black_marble.json

@@ -0,0 +1,29 @@
+[
+  "2013018S14138",
+  "2013350S08099",
+  "2014066S09138",
+  "2014086S10041",
+  "2014092S11159",
+  "2014234N12261",
+  "2015009S19054",
+  "2015024S15078",
+  "2015066S08170",
+  "2015293N13266",
+  "2016242N24279",
+  "2016041S14170",
+  "2016102S12074",
+  "2016253N13144",
+  "2016273N13300",
+  "2017094S10115",
+  "2017242N16333",
+  "2017260N12310",
+  "2018073S09129",
+  "2018226N11245",
+  "2018242N13343",
+  "2018269N11220",
+  "2018280N18273",
+  "2019236N10314",
+  "2019266N11341",
+  "2020092S09155",
+  "2020211N13306"
+]

config/storm_sets/20230120_black_marble_lookup.csv

@@ -0,0 +1,28 @@
+name,year,track_id
+Oswald,2013,2013018S14138
+Bruce,2013,2013350S08099
+Gillian,2014,2014066S09138
+Hellen,2014,2014086S10041
+Ita,2014,2014092S11159
+Marie,2014,2014234N12261
+Bansi,2015,2015009S19054
+Eunice,2015,2015024S15078
+Pam,2015,2015066S08170
+Patricia,2015,2015293N13266
+Hermine,2016,2016242N24279
+Winston,2016,2016041S14170
+Fantala,2016,2016102S12074
+Meranti,2016,2016253N13144
+Matthew,2016,2016273N13300
+Ernie,2017,2017094S10115
+Irma,2017,2017242N16333
+Maria,2017,2017260N12310
+Marcus,2018,2018073S09129
+Lane,2018,2018226N11245
+Florence,2018,2018242N13343
+Walaka,2018,2018269N11220
+Michael,2018,2018280N18273
+Dorian,2019,2019236N10314
+Lorenzo,2019,2019266N11341
+Harold,2020,2020092S09155
+Isaias,2020,2020211N13306

config/storm_sets/ibtracs.json

@@ -0,0 +1 @@
+[]

config/storm_sets/irma.json

@@ -0,0 +1,3 @@
+[
+  "2017242N16333"
+]

config/storm_sets/maria.json

@@ -0,0 +1,3 @@
+[
+  "2017260N12310"
+]

config/storm_sets/storm-cmcc-cm2-vhr4.json

@@ -0,0 +1 @@
+[]

config/storm_sets/storm-cnrm-cm6-1-hr.json

@@ -0,0 +1 @@
+[]

config/storm_sets/storm-constant.json

@@ -0,0 +1 @@
+[]

config/storm_sets/storm-ec-earth3p-hr.json

@@ -0,0 +1 @@
+[]

config/storm_sets/storm-hadgem3-gc31-hm.json

@@ -0,0 +1 @@
+[]

docs/src/SUMMARY.md

@@ -9,45 +9,24 @@
 - [Configuration](user-guide/configuration.md)
 - [Running](user-guide/running.md)
 - [Workflow](user-guide/workflow.md)
-  - [Download](user-guide/workflow/download.md)
-    - [Administrative boundaries](user-guide/workflow/download/boundaries.md)
-    - [Coastlines](user-guide/workflow/download/coastline.md)
-    - [OSM](user-guide/workflow/download/openstreetmap.md)
-    - [GDP](user-guide/workflow/download/dryad-gdp.md)
-    - [Gridfinder](user-guide/workflow/download/gridfinder.md)
-    - [Population](user-guide/workflow/download/population.md)
-    - [Powerplants](user-guide/workflow/download/powerplants.md)
-    - [STORM IBTrACS](user-guide/workflow/download/storm-ibtracs.md)
-    - [Aqueduct](user-guide/workflow/download/aqueduct.md)
-  - [Create Networks](user-guide/workflow/process.md)
-    - [Filter OSM](user-guide/workflow/process/filter.md)
-    - [Determine bounding box](user-guide/workflow/process/bbox.md)
-    - [Trim hazards](user-guide/workflow/process/trim-hazard.md)
-    - [Slice coordinates](user-guide/workflow/process/osm-extracts.md)
-    - [Slice OSM](user-guide/workflow/process/osm-slice.md)
-    - [Convert to GeoParquet](user-guide/workflow/process/geoparquet.md)
-    - [Split World](user-guide/workflow/process/worldsplit.md)
-    - [Missing countries](user-guide/workflow/process/excludecountries.md)
-    - [Powerplants](user-guide/workflow/process/powerplants.md)
-    - [Targets](user-guide/workflow/process/targets.md)
-    - [Gridfinder](user-guide/workflow/process/gridfinder.md)
-    - [Network](user-guide/workflow/process/network.md)
-  - [Intersect Networks with Hazards](user-guide/workflow/intersect.md)
-    - [Intersect with flooding](user-guide/workflow/intersect/flood.md)
-    - [Intersect with storms](user-guide/workflow/intersect/storms.md)
-    - [Region boxes](user-guide/workflow/intersect/boxes.md)
-    - [Generate region units](user-guide/workflow/intersect/grid.md)
-    - [Analyse storm winds](user-guide/workflow/intersect/windspeed.md)
-    - [GDP Loss](user-guide/workflow/intersect/gdploss.md)
-  - [Analyse](user-guide/workflow/analysis.md)
-    - [Join slices](user-guide/workflow/analysis/join.md)
-    - [Exposure rasters](user-guide/workflow/analysis/exposure.md)
-    - [Exposure images](user-guide/workflow/analysis/image.md)
-    - [Figures](user-guide/workflow/analysis/figures.md)
-    - [Empirical distribution](user-guide/workflow/analysis/empirical_distribution.md)
-    - [Empirical country pair matrix](user-guide/workflow/analysis/empirical_matrix.md)
-    - [Target statistics](user-guide/workflow/analysis/target_specific.md)
-    - [Statistic aggregation](user-guide/workflow/analysis/aggregate_levels.md)
-- [Further analysis](user-guide/further-analysis.md)
+  - [Transport - flooding](user-guide/transport-flooding.md)
+    - [Download](user-guide/transport-flooding/download.md)
+      - [Administrative boundaries](user-guide/transport-flooding/download/boundaries.md)
+      - [Coastlines](user-guide/transport-flooding/download/coastline.md)
+      - [OSM](user-guide/transport-flooding/download/openstreetmap.md)
+      - [Aqueduct](user-guide/transport-flooding/download/aqueduct.md)
+    - [Create Networks](user-guide/transport-flooding/process.md)
+      - [Filter OSM](user-guide/transport-flooding/process/filter.md)
+      - [Determine bounding box](user-guide/transport-flooding/process/bbox.md)
+      - [Trim hazards](user-guide/transport-flooding/process/trim-hazard.md)
+      - [Slice coordinates](user-guide/transport-flooding/process/osm-extracts.md)
+      - [Slice OSM](user-guide/transport-flooding/process/osm-slice.md)
+      - [Convert to GeoParquet](user-guide/transport-flooding/process/geoparquet.md)
+    - [Intersect Networks with Hazards](user-guide/transport-flooding/intersect.md)
+      - [Intersect with flooding](user-guide/transport-flooding/intersect/flood.md)
+    - [Analyse](user-guide/transport-flooding/analysis.md)
+      - [Join slices](user-guide/transport-flooding/analysis/join.md)
+      - [Exposure rasters](user-guide/transport-flooding/analysis/exposure.md)
+      - [Exposure images](user-guide/transport-flooding/analysis/image.md)
+  - [Electricity grids - tropical cyclones](user-guide/cyclone-grid.md)
 - [Cleaning output](user-guide/clean.md)
-- [Workflow (_advanced_)](user-guide/workflow-advanced.md)