Merge pull request #79 from satsin06/extract_train

MarconiS · web-flow · commit e2858fb4c6cb · 2023-08-14T08:35:10.000-04:00
Add test for extract training data
diff --git a/neonwranglerpy/lib/extract_training_data.py b/neonwranglerpy/lib/extract_training_data.py
@@ -1,11 +1,10 @@
+"""Extract training data from NEON AOP data."""
 import geopandas as gpd
 import cv2
 import os
 import re
 import numpy as np
 import pandas as pd
-import matplotlib.pyplot as plt
-from shapely.geometry import Point
 import rasterio
 from deepforest import main
 from deepforest import utilities
@@ -14,11 +13,12 @@
 
 def extract_training_data(vst_data,
                           geo_data_frame,
-                     year, dpID='DP3.30010.001',
-                     savepath='/content',
-                     site='DELA'):
+                          year,
+                          dpID='DP3.30010.001',
+                          savepath='/content',
+                          site='DELA'):
     """
-    Extracting training data with geo_data_frame and image predictions.
+    Extract training data with geo_data_frame and image predictions.
 
     Parameters
     ------------
@@ -37,12 +37,13 @@ def extract_training_data(vst_data,
                                         int(x[:-1]))).astype(str)
     geo_data_frame = gpd.GeoDataFrame(vst_data, geometry=geometry, crs=epsg_codes.iloc[0])
 
-    extract_training_data(vst_data=vst_data, geo_data_frame=geo_data_frame, year='2018', dpID='DP3.30010.001',
-                    savepath='/content', site='DELA')
+    extract_training_data(vst_data=vst_data, geo_data_frame=geo_data_frame, year='2018',
+    dpID='DP3.30010.001', savepath='/content', site='DELA')
     """
     retrieve_aop_data(vst_data, year, dpID, savepath)
     site_level_data = vst_data[vst_data.plotID.str.contains(site)]
-    get_tiles = ((site_level_data.easting/1000).astype(int) * 1000).astype(str) + "_" + ((site_level_data.northing/1000).astype(int) * 1000).astype(str)
+    get_tiles = (((site_level_data.easting / 1000).astype(int) * 1000).astype(str) + "_" +
+                 ((site_level_data.northing / 1000).astype(int) * 1000).astype(str))
     print("get_tiles")
     print(get_tiles.unique())
 
@@ -85,46 +86,87 @@ def extract_training_data(vst_data,
                         easting = row.easting
                         northing = row.northing
 
-                        x_min = int(affine[2] + 10/affine[0] - easting)
-                        y_min = int(affine[5] + 10/affine[0] - northing)
-                        x_max = int(affine[2] - 10/affine[0] - easting)
-                        y_max = int(affine[5] - 10/affine[0] - northing)
+                        x_min = int(affine[2] + 10 / affine[0] - easting)
+                        y_min = int(affine[5] + 10 / affine[0] - northing)
+                        x_max = int(affine[2] - 10 / affine[0] - easting)
+                        y_max = int(affine[5] - 10 / affine[0] - northing)
 
-                        section_file = os.path.join(output_folder, f"section_{x_min}_{y_min}_{x_max}_{y_max}.tif")
+                        file_name = f"section_{x_min}_{y_min}_{x_max}_{y_max}.tif"
 
+                        section_file = os.path.join(output_folder, file_name)
 
                         if section_file not in section_files:
+                            section = image[y_max:y_min, x_max:x_min, :]
+                            print("Section shape:", section.shape)
 
-                          section = image[y_max:y_min, x_max:x_min, :]
-                          print("Section shape:", section.shape)
+                            section_meta = src.meta.copy()
+                            section_meta['width'] = (affine[2] + x_min) - (affine[2] +
+                                                                           x_max)
+                            section_meta['height'] = (affine[5] + y_min) - (affine[5] +
+                                                                            y_max)
+                            section_meta['transform'] = rasterio.Affine(
+                                affine[0], 0, (affine[2] - x_min), 0, affine[4],
+                                (affine[5] - y_min))
 
-                          section_meta = src.meta.copy()
-                          section_meta['width'], section_meta['height'] = (affine[2] + x_min) - (affine[2] + x_max), (affine[5] + y_min) - (affine[5] + y_max)
-                          section_meta['transform'] = rasterio.Affine(affine[0], 0, (affine[2] - x_min), 0, affine[4], (affine[5] - y_min))
+                            section_np = np.moveaxis(section, -1, 0)
 
+                            with rasterio.open(section_file, 'w', **section_meta) as dst:
+                                dst.write(section_np)
+                                section_affine = dst.transform
 
-                          section_np = np.moveaxis(section, -1, 0)
+                            section_files[section_file] = section_affine
 
-                          with rasterio.open(section_file, 'w', **section_meta) as dst:
-                              dst.write(section_np)
-                              section_affine = dst.transform
+                            print("Crop affine: ")
+                            print(section_affine)
 
-                          section_files[section_file] = section_affine
-
-                          print("Crop affine: ")
-                          print(section_affine)
-
-                          print("Expected file path:", section_file)
+                            print("Expected file path:", section_file)
 
                         prediction = model.predict_image(path=section_file)
 
-                        gdf = utilities.boxes_to_shapefile(prediction, root_dir=os.path.dirname(section_file), projected=True)
+                        gdf = utilities.boxes_to_shapefile(
+                            prediction,
+                            root_dir=os.path.dirname(section_file),
+                            projected=True)
 
                         all_predictions.append(gdf)
 
     all_predictions_df = pd.concat(all_predictions)
 
+    all_predictions_df['temp_geo'] = all_predictions_df['geometry']
+
     merged_data = gpd.sjoin(geo_data_frame, all_predictions_df, how="inner", op="within")
+    merged_data.drop(columns=['geometry'], inplace=True)
+    merged_data.rename(columns={'temp_geo': 'geometry'}, inplace=True)
+    canopy_position_mapping = {
+        np.nan: 0,
+        'Full shade': 1,
+        'Mostly shaded': 2,
+        'Partially shaded': 3,
+        'Full sun': 4,
+        'Open grown': 5
+    }
+
+    predictions = merged_data
+
+    predictions_copy = predictions.copy()
+
+    cp = 'canopyPosition'
+
+    predictions_copy[cp] = predictions_copy[cp].replace(canopy_position_mapping)
+
+    duplicate_mask = predictions_copy.duplicated(subset=['xmin', 'ymin', 'xmax', 'ymax'],
+                                                 keep=False)
+
+    duplicate_entries = predictions[duplicate_mask]
+
+    print(duplicate_entries)
+
+    predictions_sorted = predictions.sort_values(by=['height', cp, 'stemDiameter'],
+                                                 ascending=[False, False, False])
+
+    duplicates_mask = predictions_sorted.duplicated(
+        subset=['xmin', 'ymin', 'xmax', 'ymax'], keep='first')
 
+    clean_predictions = predictions_sorted[~duplicates_mask]
 
-    return merged_data
+    return clean_predictions
diff --git a/neonwranglerpy/lib/predict_aop_data.py b/neonwranglerpy/lib/predict_aop_data.py
@@ -11,7 +11,8 @@
 
 
 def predict_aop_data(vst_data,
-                     year, dpID='DP3.30010.001',
+                     year,
+                     dpID='DP3.30010.001',
                      savepath='/content',
                      site='DELA',
                      plot_crop=True):
@@ -33,14 +34,16 @@ def predict_aop_data(vst_data,
                     savepath='/content', site='DELA')
     """
     retrieve_aop_data(vst_data, year, dpID, savepath)
-    geometry = [Point(easting, northing) for easting, northing in
-                zip(vst_data['easting'], vst_data['northing'])]
-    epsg_codes = (vst_data['utmZone'].map(lambda x: (326 * 100) +
-                                          int(x[:-1]))).astype(str)
+    geometry = [
+        Point(easting, northing)
+        for easting, northing in zip(vst_data['easting'], vst_data['northing'])
+    ]
+    epsg_codes = (
+        vst_data['utmZone'].map(lambda x: (326 * 100) + int(x[:-1]))).astype(str)
     geo_data_frame = gpd.GeoDataFrame(vst_data, geometry=geometry, crs=epsg_codes.iloc[0])
     site_level_data = vst_data[vst_data.plotID.str.contains(site)]
-    get_tiles = ((site_level_data.easting/1000).astype(int) * 1000).astype(str) + "_"
-    + ((site_level_data.northing/1000).astype(int) * 1000).astype(str)
+    get_tiles = ((site_level_data.easting / 1000).astype(int) * 1000).astype(str) + "_"
+    +((site_level_data.northing / 1000).astype(int) * 1000).astype(str)
     print(get_tiles.unique())
 
     pattern = fr"{year}_{site}_.*_{get_tiles.unique()[0]}"
@@ -74,10 +77,10 @@ def predict_aop_data(vst_data,
                         easting = row.easting
                         northing = row.northing
 
-                        x_min = int(affine[2] + 10/affine[0] - easting)
-                        y_min = int(affine[5] + 10/affine[0] - northing)
-                        x_max = int(affine[2] - 10/affine[0] - easting)
-                        y_max = int(affine[5] - 10/affine[0] - northing)
+                        x_min = int(affine[2] + 10 / affine[0] - easting)
+                        y_min = int(affine[5] + 10 / affine[0] - northing)
+                        x_max = int(affine[2] - 10 / affine[0] - easting)
+                        y_max = int(affine[5] - 10 / affine[0] - northing)
 
                         print(x_min, y_min, x_max, y_max)
                         section = image[y_max:y_min, x_max:x_min, :]
@@ -94,4 +97,3 @@ def predict_aop_data(vst_data,
                         all_predictions.append(prediction)
 
     return all_predictions
-
diff --git a/neonwranglerpy/lib/retrieve_aop_data.py b/neonwranglerpy/lib/retrieve_aop_data.py
@@ -1,6 +1,7 @@
 """Download AOP data around vst data for specified year, site."""
 from neonwranglerpy.utilities.byTileAOP import by_tile_aop
 
+
 def retrieve_aop_data(data, year=2019, dpID=['DP3.30006.001'], savepath=""):
     """Download AOP data around vst data for specified year, site.
 
@@ -20,20 +21,18 @@ def retrieve_aop_data(data, year=2019, dpID=['DP3.30006.001'], savepath=""):
     savepath : str
         The full path to the folder in which the files would be placed locally.
     """
-    coords_for_tiles = data[[
-        'plotID', 'siteID', 'utmZone', 'easting', 'northing']]
+    coords_for_tiles = data[['plotID', 'siteID', 'utmZone', 'easting', 'northing']]
     # get tiles dimensions
-    coords_for_tiles['easting'] = (coords_for_tiles[['easting']] / 
+    coords_for_tiles['easting'] = (coords_for_tiles[['easting']] /
                                    1000).astype(int) * 1000
-    coords_for_tiles['northing'] = (coords_for_tiles[['northing']] / 
+    coords_for_tiles['northing'] = (coords_for_tiles[['northing']] /
                                     1000).astype(int) * 1000
-    print(coords_for_tiles.easting.shape[0] )
+    print(coords_for_tiles.easting.shape[0])
     # if there are more than 1 row, drop duplicates
     if coords_for_tiles.easting.shape[0] > 1:
-    # drop duplicates values
+        # drop duplicates values
         tiles = coords_for_tiles.drop_duplicates(
-            subset=['siteID', 'utmZone', 'easting', 'northing']).reset_index(
-                drop=True)
+            subset=['siteID', 'utmZone', 'easting', 'northing']).reset_index(drop=True)
         tiles.dropna(axis=0, how='any', inplace=True)
 
         # convert CHEQ into STEI
@@ -49,7 +48,7 @@ def retrieve_aop_data(data, year=2019, dpID=['DP3.30006.001'], savepath=""):
     else:
         tiles = coords_for_tiles
         tiles.dropna(axis=0, how='any', inplace=True)
-            # convert CHEQ into STEI
+        # convert CHEQ into STEI
         which_cheq = tiles['siteID'] == 'STEI'
         if which_cheq:
             which_easting = tiles['easting'] > 500000
@@ -64,9 +63,13 @@ def retrieve_aop_data(data, year=2019, dpID=['DP3.30006.001'], savepath=""):
             try:
                 if coords_for_tiles.easting.shape[0] > 1:
                     tile = tiles.iloc[i, :]
-                    siteID, tile_easting, tile_northing = tile['siteID'], tile['easting'], tile['northing']
+                    siteID = tile['siteID']
+                    tile_easting = tile['easting']
+                    tile_northing = tile['northing']
                 else:
-                    siteID, tile_easting, tile_northing = tiles['siteID'], tiles['easting'][0], tiles['northing'][0]
+                    siteID = tiles['siteID']
+                    tile_easting = tiles['easting'][0]
+                    tile_northing = tiles['northing'][0]
 
                 by_tile_aop(prd,
                             siteID,
diff --git a/neonwranglerpy/utilities/byTileAOP.py b/neonwranglerpy/utilities/byTileAOP.py
@@ -1,5 +1,4 @@
 """Downloads the AOP data from NEON API."""
-import math
 import os
 import re
 import numpy as np
diff --git a/neonwranglerpy/utilities/get_tile_urls.py b/neonwranglerpy/utilities/get_tile_urls.py
@@ -3,6 +3,7 @@
 from neonwranglerpy.utilities.tools import get_api
 import numpy as np
 
+
 def get_tile_urls(
     month_url,
     easting,
@@ -29,7 +30,10 @@ def get_tile_urls(
             file_urls = [x for x in temp_ if f'_{easting}_{northing}' in x['name']]
         elif isinstance(easting, np.ndarray) and isinstance(northing, np.ndarray):
             for j in range(len(easting)):
-                urls = [x for x in temp_ if f'_{easting.iloc[j]}_{northing.iloc[j]}' in x['name']]
+                urls = [
+                    x for x in temp_
+                    if f'_{easting.iloc[j]}_{northing.iloc[j]}' in x['name']
+                ]
 
                 #     df1 = df.loc[df['name'].str.contains(str(easting[j]))]
                 #     df2 = df.loc[df['name'].str.contains(str(northing[j]))]
diff --git a/requirements.txt b/requirements.txt
@@ -10,3 +10,7 @@ yapf
 sphinx-press-theme
 laspy
 lazrs
+opencv-python
+numpy
+rasterio
+deepforest
diff --git a/tests/raw_data/vst_data.csv b/tests/raw_data/vst_data.csv
diff --git a/tests/test_extract_training_data.py b/tests/test_extract_training_data.py

Original file line number	Diff line number	Diff line change
`@@ -1,5 +1,4 @@`
`1`	`1`	`"""Downloads the AOP data from NEON API."""`
`2`		`-import math`
`3`	`2`	`import os`
`4`	`3`	`import re`
`5`	`4`	`import numpy as np`