fixed h5 retriever

neonwranglerpy/lib/crop_plot_data.py

@@ -19,7 +19,7 @@ def crop_data_to_plot(plt,
                       dpID='DP3.30006.001',
                       dataset_path="",
                       target_year=2018,
-                      bff=520,
+                      bff=20,
                       tasks=1,
                       parallelized=False,
                       savepath=""):

neonwranglerpy/lib/extract_hsi_to_tif.py

@@ -59,9 +59,14 @@
         hdf5_file[sitename]['Reflectance/Metadata/Ancillary_Imagery/Data_Selection_Index']
         [()]
     ]
+    # turn sol_az and sol_zn into arrays
+    sol_zn = np.array(sol_zn)
+    sol_az = np.array(sol_az)
+
     for pt in range(len(solar_angles)):
-        sol_az[flightpaths == solar_angles[pt][0]] = solar_angles[pt][1]
-        sol_zn[flightpaths == solar_angles[pt][0]] = solar_angles[pt][2]
+        good_pixels = flightpaths == solar_angles[pt][0]
+        sol_az[good_pixels] = solar_angles[pt][1]
+        sol_zn[good_pixels] = solar_angles[pt][2]
 
 
 #
@@ -101,8 +106,8 @@
     file_attrs_string = str(list(hdf5_file.items()))
     file_attrs_string_split = file_attrs_string.split("'")
     sitename = file_attrs_string_split[1]
-    flight_paths = hdf5_file[sitename]
-    ["Reflectance/Metadata/Ancillary_Imagery/Data_Selection_Index"].attrs["Data_Files"]
+    flight_paths = hdf5_file[sitename][
+        "Reflectance/Metadata/Ancillary_Imagery/Data_Selection_Index"].attrs["Data_Files"]
     flight_paths = str(flight_paths).split(",")
     which_paths = np.unique(
         hdf5_file[sitename]["Reflectance/Metadata/Ancillary_Imagery/Data_Selection_Index"]
@@ -187,6 +192,9 @@
     originY = extent['yMax']
     res = reflArray_metadata['res']['pixelWidth']
     transform = Affine.translation(originX, originY) * Affine.scale(res, -res)
+
+    if not os.path.exists(ras_dir):
+        os.makedirs(ras_dir)
     with rasterio.open(
             "{}/{}".format(ras_dir, newRaster),
             'w',
@@ -291,10 +299,17 @@
             os.path.basename(rgb_filename))[0] + "_hyperspectral{}.tif".format(suffix)
 
     # stach solar and sensor data to be used for corrections
-    sol_sens_angle = np.array([sol_az, sol_zn, sns_az, sns_zn])
+    sol_az_reshaped = sol_az.squeeze()  # Remove the extra dimension
+    sol_zn_reshaped = sol_zn.squeeze()  # Remove the extra dimension
+    sns_az_reshaped = sns_az  # No need to reshape sns_az
+    sns_zn_reshaped = sns_zn
+
+    sol_sens_angle = np.array(
+        [sol_az_reshaped, sol_zn_reshaped, sns_az_reshaped, sns_zn_reshaped])
     solar_tilename = os.path.splitext(
         os.path.basename(rgb_filename))[0] + "_solar_sensor_angle{}.tif".format(suffix)
     # Save georeference crop to file
+    save_dir = os.path.abspath(save_dir)
     array2raster(solar_tilename,
                  sol_sens_angle,
                  metadata,

tests/test_h5refl2array.py

@@ -0,0 +1,52 @@
+import unittest
+import numpy as np
+import subprocess
+import os
+import pytest
+
+from neonwranglerpy.lib.extract_hsi_to_tif import h5refl2array
+
+# Main Paths
+file_location = os.path.dirname(os.path.realpath(__file__))
+neonwranglerpy_root_dir = os.path.abspath(os.path.join(file_location, os.pardir))
+
+# Paths of the raw data files used
+raw_dir_files = os.path.normpath(os.path.join(neonwranglerpy_root_dir, 'raw_data'))
+
+test_reflection2array_data = [
+    ('test_reflection2array', "h5_data/DP3.30006.001/2017/FullSite/D16/2017_ABBY_1/L3/Spectrometer/Reflectance"
+                              "/NEON_D16_ABBY_DP3_559000_5070000_reflectance.h5")
+]
+
+
+def setup_module():
+    """Automatically sets up the environment before the module runs."""
+    os.chdir(neonwranglerpy_root_dir)
+    subprocess.call(['cp', '-r', 'tests/raw_data', neonwranglerpy_root_dir])
+
+
+def teardown_module():
+    """Automatically clean up after the module."""
+    os.chdir(neonwranglerpy_root_dir)
+    subprocess.call(['rm', '-r', 'raw_data'])
+
+
+def setup_functions():
+    """Set up functions."""
+    teardown_module()
+    setup_module()
+
+
+@pytest.mark.parametrize("test_name, path", test_reflection2array_data)
+def test_reflection2array(test_name, path):
+    setup_functions()
+    path = os.path.join(raw_dir_files, path)
+    reflArray, metadata, sol_az, sol_zn, sns_az, sns_zn = h5refl2array(path)
+
+    # Test the type of the returned objects
+    assert (isinstance(reflArray, np.ndarray))
+    assert (isinstance(metadata, dict))
+    assert (isinstance(sol_az,  np.ndarray))
+    assert (isinstance(sol_zn,  np.ndarray))
+    assert (isinstance(sns_az, np.ndarray))
+    assert (isinstance(sns_zn, np.ndarray))

tests/test_lib.py

@@ -49,7 +49,15 @@
         'utmZone': ['6N', '6N'],
         'easting': [559120, 559120],
         'northing': [5070120, 5070120]
-    }, "DP3.30015.001", "raster_data", "2018", ["NEON_D16_ABBY_DP3_559000_5070000_CHM.tif", True])
+    }, "DP3.30015.001", "raster_data", "2018", ["NEON_D16_ABBY_DP3_559000_5070000_CHM.tif", True]),
+    ('test_extract_h5', {
+        'plotID': ['TEST_0000', 'TEST_0000'],
+        'subplotID': ['A', "A"],
+        'siteID': ['ABBY', 'ABBY'],
+        'utmZone': ['6N', '6N'],
+        'easting': [559120, 559120],
+        'northing': [5070120, 5070120]
+    }, "DP3.30006.001", "h5_data", "2017", ["NEON_D16_ABBY_DP3_559000_5070000_reflectance_hyperspectral.tif", True])
 ]