Task/clean mess

.github/workflows/build.yml

@@ -3,7 +3,9 @@ name: Build
 on:
   push:
     branches:
-      - main
+      - '*'
+  pull_request:
+    types: [opened, synchronize, reopened]
 
 jobs:
   build:

README.md

@@ -54,12 +54,11 @@ Scripts are run in combination with hard-coded configuration files in the follow
 ## Data preparation
 1. `clip_image.py`: The goal of this script is to clip images with a AOI vector layer. In a first step, the AOI is buffered by 50 m. This vector layer is then used as an input to clip aerial imagery data.
 	* Use clip_image.yaml to specify the inputs data. 
-	* Please change the join option ("predicate") in [`functions/fct_misc.py`](./scripts/functions/fct_misc.py) in Line 83 form "within" to "intersects". 
 2. `calculate_raster.py`: compute NDVI and luminosity rasters. Watch out for the right band in functions `calculate_ndvi` and `calculate_luminosity`. 
 	* Use logReg.yaml to specify the inputs data.
 3. `roof_stats.py`: compute statistics of NDVI and luminostiy values per roofs to help define thresholds. Split the roofs into a training and a test dataset. 
 	* Use logReg.yaml to specify the inputs data.
-	* Please verifiy that the join option ("predicate") in [`functions/fct_misc.py`](./scripts/functions/fct_misc.py) in Line 83 is "intersects".
+	* Please verifiy that the join option ("predicate") in [`functions/fct_misc.py`](./scripts/functions/fct_misc.py) in Line 83 is "within".
 
 ## Logistic regression approach
 The logistic regression approach was developed inspired by Louis-Lucas et al (1) and implemented for the specific project in `functions/fct_misc.py`. 

config/clipImage.yaml

@@ -1,20 +1,22 @@
-clip_image:
-  working_directory: C:/Users/cmarmy/Documents/STDL/proj-vegroofs/data_test
-  inputs:
-    ortho_directory: 01_initial/images/scratch/tiles/
-    aoi: 01_initial/aoi/STDL_ZH_AOI.shp
-    epsg: 'epsg:2056'
-  outputs: 
-    clip_ortho_directory: 02_intermediate/images/scratch/tiles
-    extent_ortho_directory: 01_initial/images/scratch/extent
-
 # clip_image:
 #   working_directory: C:/Users/cmarmy/Documents/STDL/proj-vegroofs/data_test
 #   inputs:
-#     ortho_directory: 01_initial/images/tlm/tiles/
+#     ortho_directory: 01_initial/images/rs_tlm/tiles/
 #     aoi: 01_initial/aoi/STDL_ZH_AOI.shp
 #     epsg: 'epsg:2056'
+#     predicate_sjoin: 'intersects'
 #   outputs: 
-#     clip_ortho_directory: 02_intermediate/images/tlm/tiles
-#     extent_ortho_directory: 01_initial/images/tlm/extent
+#     clip_ortho_directory: 02_intermediate/images/rs_tlm/tiles
+#     extent_ortho_directory: 01_initial/images/rs_tlm/extent
+
+clip_image:
+  working_directory: C:/Users/cmarmy/Documents/STDL/proj-vegroofs/data
+  inputs:
+    ortho_directory: 01_initial/images/rs_tlm/ge/tiles/
+    aoi: 01_initial/aoi/STDL_GE_AOI.shp
+    epsg: 'epsg:2056'
+    predicate_sjoin: 'intersects'
+  outputs: 
+    clip_ortho_directory: 02_intermediate/images/rs_tlm/tiles
+    extent_ortho_directory: 01_initial/images/rs_tlm/extent
 

config/clipImage_dummy.yaml

@@ -0,0 +1,11 @@
+clip_image:
+  working_directory: /proj-vegroofs/data
+  inputs:
+    ortho_directory: 01_initial/images/rs_tlm/tiles/
+    aoi: 01_initial/aoi/STDL_ZH_AOI.shp
+    epsg: 'epsg:2056'
+    predicate_sjoin: 'intersects'
+  outputs: 
+    clip_ortho_directory: 02_intermediate/images/rs_tlm/tiles
+    extent_ortho_directory: 01_initial/images/rs_tlm/extent
+

config/logReg.yaml

@@ -2,65 +2,41 @@ hydra:
   run:
     dir: 02_intermediate/th/${now:%Y-%m-%d}/${now:%H-%M-%S}
 
-dev:
-  working_directory: C:/Users/cmarmy/Documents/STDL/proj-vegroofs/data_test
-  ortho_directory: 02_intermediate/images/tlm/tiles
-  tile_delimitation: 02_intermediate/images/tlm/extent/
-  ndvi_directory: 02_intermediate/images/tlm/ndvi
-  lum_directory: 02_intermediate/images/tlm/lum
-  results_directory: 03_results/tlm
-  aoi: 01_initial/aoi/STDL_ZH_AOI.shp
-  roofs_gt: 01_initial/gt/gt_test.shp
-  roofs_layer: gt_zh
-  egid_train_test: 03_results/egid_train_test.csv
-  th_ndvi: 0 # no thresholding -1
-  th_lum: 400 # no thresholding 765
-  epsg: 'epsg:2056'
-
 # dev:
 #   working_directory: C:/Users/cmarmy/Documents/STDL/proj-vegroofs/data_test
-#   ortho_directory: 02_intermediate/images/scratch/tiles
-#   tile_delimitation: 02_intermediate/images/scratch/extent/
-#   ndvi_directory: 02_intermediate/images/scratch/ndvi
-#   lum_directory: 02_intermediate/images/scratch/lum
-#   results_directory: 03_results/scratch
-#   aoi: 01_initial/aoi/STDL_ZH_AOI.shp
-#   roofs_gt: 01_initial/gt/gt_test.shp
-#   roofs_layer: gt_zh
-#   roofs_lr: 02_intermediate/th/2024-03-25/14-20-01/0_400_roof_4_lr.shp
-#   egid_train_test: 03_results/egid_train_test.csv
-#   th_ndvi: 0 # no thresholding -1
-#   th_lum: 400 # no thresholding 765
-#   epsg: 'epsg:2056'
-
-
-# prod: 
-#   working_directory: C:/Users/cmarmy/Documents/STDL/proj-vegroofs/data
 #   ortho_directory: 02_intermediate/images/rs_tlm/tiles
 #   tile_delimitation: 02_intermediate/images/rs_tlm/extent/
 #   ndvi_directory: 02_intermediate/images/rs_tlm/ndvi
 #   lum_directory: 02_intermediate/images/rs_tlm/lum
 #   results_directory: 03_results/rs_tlm
-#   aoi: 01_initial/aoi/STDL_ZH_AOI.shp
-#   roofs_gt: 02_intermediate/gt/gt_tot.gpkg
-#   roofs_layer: gt_to_label
-#   egid_train_test: 03_results/egid_train_test.csv
-#   th_ndvi: -1 # no thresholding -1
+#   roofs_gt: 01_initial/gt/gt_test.shp 
+#   roofs_layer: gt_zh
+#   roofs_lr: 02_intermediate/th/2024-06-14/10-10-21/0_765_gt_tot_green.shp
+#   chm_layer: 02_intermediate/autres/CHM_AOI.gpkg
+#   th_ndvi: 0 # no thresholding -1
 #   th_lum: 765 # no thresholding 765
+#   egid_train_test: egid_train_test_gt.csv
+#   predicate_sjoin: 'intersects'
+#   cls_lr: 'multi' # 'multi' 'multi_aggreg'
+#   model_ml: 'LR' # 'LR' 'RF'
 #   epsg: 'epsg:2056'
 
-# prod: 
-#   working_directory: C:/Users/cmarmy/Documents/STDL/proj-vegroofs/data
-#   ortho_directory: 02_intermediate/images/scratch/tiles
-#   tile_delimitation: 02_intermediate/images/scratch/extent/
-#   ndvi_directory: 02_intermediate/images/scratch/ndvi
-#   lum_directory: 02_intermediate/images/scratch/lum
-#   results_directory: 03_results/scratch
-#   aoi: 01_initial/aoi/STDL_ZH_AOI.shp
-#   roofs_gt: 02_intermediate/gt/gt_tot.gpkg
-#   roofs_layer: gt_to_label
-#   egid_train_test: 03_results/egid_train_test.csv
-#   th_ndvi: -1 # no thresholding -1
-#   th_lum: 765 # no thresholding 765
-#   epsg: 'epsg:2056'
+prod: 
+  working_directory: C:/Users/cmarmy/Documents/STDL/proj-vegroofs/data
+  ortho_directory: 02_intermediate/images/rs_tlm/tiles
+  tile_delimitation: 02_intermediate/images/rs_tlm/extent/
+  ndvi_directory: 02_intermediate/images/rs_tlm/ndvi
+  lum_directory: 02_intermediate/images/rs_tlm/lum
+  roofs_gt: 02_intermediate/gt/gt_tot.gpkg # 02_intermediate/autres/swissbuilding_aoi.gpkg
+  roofs_layer: joined_layer
+  chm_layer: 02_intermediate/autres/CHM_AOI.gpkg
+  results_directory: 03_results/rs_tlm_sb_noratio/
+  egid_train_test: egid_train_test_gt.csv
+  roofs_lr: 02_intermediate/autres/swissbuilding_aoi.gpkg # 02_intermediate/gt/gt_tot.gpkg #02_intermediate/th/2024-06-14/15-02-41/0_765_gt_tot_green.shp
+  predicate_sjoin: 'within'
+  th_ndvi: -1 # no thresholding -1
+  th_lum: 210000 # no thresholding 765 or 210000
+  cls_lr: 'multi' # 'binary' 'multi' 'multi_aggreg'
+  model_ml: 'LR' # 'LR' 'RF'
+  epsg: 'epsg:2056'
 

config/logReg_dummy.yaml

@@ -0,0 +1,25 @@
+hydra:
+  run:
+    dir: 02_intermediate/th/${now:%Y-%m-%d}/${now:%H-%M-%S}
+
+prod: 
+prod: 
+  working_directory: /proj-vegroofs/data
+  ortho_directory: 02_intermediate/images/scratch/tiles
+  tile_delimitation: 02_intermediate/images/scratch/extent/
+  ndvi_directory: 02_intermediate/images/scratch/ndvi
+  lum_directory: 02_intermediate/images/scratch/lum
+  roofs_gt: 02_intermediate/gt/gt_tot.gpkg
+  roofs_layer: gt_to_label
+  chm_layer: 02_intermediate/autres/CHM_AOI.gpkg
+  results_directory: 03_results/scratch_gt/
+  egid_train_test: egid_train_test_gt.csv
+  roofs_lr: 02_intermediate/th/2024-06-14/10-10-21/0_765_gt_tot_green.shp
+  predicate_sjoin: 'within'
+  th_ndvi: 0 # no thresholding -1
+  th_lum: 765 # no thresholding 765
+  cls_lr: 'multi' # 'multi' 'multi_aggreg'
+  model_ml: 'LR' # 'LR' 'RF'
+  epsg: 'epsg:2056'
+
+

scripts/calculate_raster.py

@@ -86,7 +86,7 @@ def calculate_lum(tile, band_nbr_red=1, band_nbr_green=2, band_nbr_blue=3, path=
 
     # load input parameters
     with open(args.config_file) as fp:
-        cfg = yaml.load(fp, Loader=yaml.FullLoader)['dev']
+        cfg = yaml.load(fp, Loader=yaml.FullLoader)['prod']
 
     logger.info('Defining constants...')
 

scripts/clip_image.py

@@ -38,6 +38,7 @@
     INPUTS=cfg['inputs']
     ORTHO_DIR=INPUTS['ortho_directory']
     AOI=INPUTS['aoi']
+    PREDICATE=INPUTS['predicate_sjoin']
     EPSG=INPUTS['epsg']
 
     OUTPUTS=cfg['outputs']
@@ -61,9 +62,7 @@
     fct_misc.generate_extent(ORTHO_DIR, TILE_DELIMITATION, EPSG)
     tiles=gpd.read_file(TILE_DELIMITATION)
 
-    aoi_clipped=fct_misc.clip_labels(labels_gdf=aoi, tiles_gdf=tiles)
-    # keep only elements that are non-empty
-    aoi_clipped=aoi_clipped[~aoi_clipped.is_empty]
+    aoi_clipped=fct_misc.clip_labels(labels_gdf=aoi, tiles_gdf=tiles, predicate_sjoin=PREDICATE)
     aoi_clipped=aoi_clipped.reset_index(drop=True)
 
     i=1