diff --git a/neonwranglerpy/fetcher/fetcher.py b/neonwranglerpy/fetcher/fetcher.py index 5aae4a9..40bfbbb 100644 --- a/neonwranglerpy/fetcher/fetcher.py +++ b/neonwranglerpy/fetcher/fetcher.py @@ -35,7 +35,7 @@ async def _request(session, url): return await response.json() -async def _download(session, url, filename, sem, month, size=None): +async def _download(session, url, filename, sem, size=None): """Asynchronous function to download file from url. Parameters @@ -58,6 +58,7 @@ async def _download(session, url, filename, sem, month, size=None): size = response.content_length if not size else size block = size copied = 0 + with open(filename, mode='wb') as f: async for chunk in response.content.iter_chunked(block): f.write(chunk) @@ -66,26 +67,44 @@ async def _download(session, url, filename, sem, month, size=None): # update_progressbar(progress, size) -async def _fetcher(data, rate_limit, headers, files_to_stack_path="filesToStack"): +async def _fetcher(data, + rate_limit, + headers, + files_to_stack_path="filesToStack", + data_type="vst"): """Fetcher for downloading files.""" sem = asyncio.Semaphore(rate_limit) data = data['data'] - dir_name = '.'.join( - ['NEON', data['productCode'], data['siteCode'], data['month'], data['release']]) - zip_dir_path = os.path.join(files_to_stack_path, f'{dir_name}') - if not os.path.isdir(zip_dir_path): - os.mkdir(zip_dir_path) d_urls = [f['url'] for f in data["files"]] sizes = [f['size'] for f in data["files"]] f_names = [f['name'] for f in data["files"]] - f_paths = [pjoin(zip_dir_path, name) for name in f_names] - month = [data['month']] + if data_type == "vst": + dir_name = '.'.join([ + 'NEON', data['productCode'], data['siteCode'], data['month'], data['release'] + ]) + zip_dir_path = os.path.join(files_to_stack_path, f'{dir_name}') + if not os.path.isdir(zip_dir_path): + os.mkdir(zip_dir_path) + f_paths = [pjoin(zip_dir_path, name) for name in f_names] + else: + f_paths = [] + zip_dir_path = os.path.join(files_to_stack_path, data['productCode']) + if not os.path.isdir(zip_dir_path): + os.mkdir(zip_dir_path) + for i in range(len(d_urls)): + split_path = d_urls[i].split('/') + dir_path = '/'.join(split_path[4:len(split_path) - 1]) + save_dir_path = pjoin(zip_dir_path, dir_path) + if not os.path.exists(save_dir_path): + os.makedirs(save_dir_path) + f_paths.append(os.path.join(save_dir_path, f_names[i])) + zip_url = zip(d_urls, f_paths, sizes) async with aiohttp.ClientSession() as session: tasks = [] for url, name, sz in zip_url: - task = asyncio.create_task(_download(session, url, name, sem, month, sz)) + task = asyncio.create_task(_download(session, url, name, sem, sz)) tasks.append(task) await asyncio.gather(*tasks) @@ -94,7 +113,7 @@ async def _fetcher(data, rate_limit, headers, files_to_stack_path="filesToStack" async def vst_fetcher(item, rate_limit, headers, files_to_stack_path="filesToStack"): """Vst fetcher gets the urls for the files of vst data.""" data = requests.get(item).json() - await _fetcher(data, rate_limit, headers, files_to_stack_path) + await _fetcher(data, rate_limit, headers, files_to_stack_path, "vst") def fetcher(batch, data_type, rate_limit, headers, files_to_stack_path): @@ -103,7 +122,8 @@ def fetcher(batch, data_type, rate_limit, headers, files_to_stack_path): if data_type == 'vst': asyncio.run(vst_fetcher(batch, rate_limit, headers, files_to_stack_path)) elif data_type == 'aop': - asyncio.run(_fetcher(batch, rate_limit, headers, files_to_stack_path)) + asyncio.run( + _fetcher(batch, rate_limit, headers, files_to_stack_path, data_type)) except Exception as e: print(f"Error processing URLs: {e}") diff --git a/neonwranglerpy/lib/crop_plot_data.py b/neonwranglerpy/lib/crop_plot_data.py index 185f81f..f70c2e5 100644 --- a/neonwranglerpy/lib/crop_plot_data.py +++ b/neonwranglerpy/lib/crop_plot_data.py @@ -9,8 +9,9 @@ def list_files(path): """List all the files in a path of given format.""" - mask = path + '/**/*.[th][i5]' - files = glob(mask, recursive=True) + mask_tif = path + "/**/*.tif" + mask_h5 = path + "/**/*.h5" + files = glob(mask_tif, recursive=True) + glob(mask_h5, recursive=True) return files @@ -23,12 +24,11 @@ def crop_data_to_plot(plt, parallelized=False, savepath=""): """Create shapefiles out of a vegetation structure data with lat/lon coordinates.""" - dataset_path = os.path.normpath(dataset_path) + dataset_path = os.path.abspath(dataset_path) full_files = list_files(dataset_path) files = [file for file in full_files if dpID in file] files = [file for file in files if str(target_year) in file] - # TODO: add check if files for targeted year and product is not present plots = plt[['plotID', 'subplotID', 'siteID', 'utmZone', 'easting', 'northing']] plots = plots.groupby(['plotID', 'subplotID', 'siteID', diff --git a/neonwranglerpy/lib/retrieve_aop_data.py b/neonwranglerpy/lib/retrieve_aop_data.py index 342aebb..a52dd37 100644 --- a/neonwranglerpy/lib/retrieve_aop_data.py +++ b/neonwranglerpy/lib/retrieve_aop_data.py @@ -23,10 +23,10 @@ def retrieve_aop_data(data, year=2019, dpID=['DP3.30006.001'], savepath=""): """ coords_for_tiles = data[['plotID', 'siteID', 'utmZone', 'easting', 'northing']] # get tiles dimensions - coords_for_tiles['easting'] = (coords_for_tiles[['easting']] / - 1000).astype(int) * 1000 - coords_for_tiles['northing'] = (coords_for_tiles[['northing']] / - 1000).astype(int) * 1000 + coords_for_tiles['easting'] = (coords_for_tiles[['easting']] / 1000).astype( + int, errors='ignore') * 1000 + coords_for_tiles['northing'] = (coords_for_tiles[['northing']] / 1000).astype( + int, errors='ignore') * 1000 # if there are more than 1 row, drop duplicates if coords_for_tiles.easting.shape[0] > 1: # drop duplicates values diff --git a/neonwranglerpy/utilities/byTileAOP.py b/neonwranglerpy/utilities/byTileAOP.py index b86f5d3..b24f8ec 100644 --- a/neonwranglerpy/utilities/byTileAOP.py +++ b/neonwranglerpy/utilities/byTileAOP.py @@ -77,9 +77,9 @@ def by_tile_aop(dpID, site, year, easting, northing, buffer=0, savepath=None): if not len(month_urls): print(f"There is no data for site {site} and year {year}") - if isinstance(easting, (int, float)): - easting = [easting] - northing = [northing] + # if isinstance(easting, (int, float)): + # easting = [easting] + # northing = [northing] # convert the easting and northing for BLAN site if site == "BLAN": if isinstance(easting, (int, list)): @@ -124,7 +124,7 @@ def by_tile_aop(dpID, site, year, easting, northing, buffer=0, savepath=None): tile_northing = np.floor(northing / 1000).astype(int) * 1000 file_urls = get_tile_urls(month_urls, tile_easting, tile_northing) - print(f"Tiles Found for Remote Sensing Data: {len(file_urls)}") + # print(f"Tiles Found for Remote Sensing Data: {len(file_urls)}") if not savepath: savepath = os.path.normpath(os.path.join(os.getcwd(), dpID)) else: @@ -133,14 +133,14 @@ def by_tile_aop(dpID, site, year, easting, northing, buffer=0, savepath=None): if not os.path.isdir(savepath): os.makedirs(savepath) - files_to_stack_path = os.path.join(savepath, "filesToStack") - if not os.path.isdir(files_to_stack_path): - os.mkdir(files_to_stack_path) + # files_to_stack_path = os.path.join(savepath, "filesToStack") + # if not os.path.isdir(files_to_stack_path): + # os.mkdir(files_to_stack_path) - if files_to_stack_path: + if savepath: fetcher.run_threaded_batches(file_urls, 'aop', rate_limit=2, headers=None, - savepath=files_to_stack_path) + savepath=savepath) return savepath diff --git a/requirements.txt b/requirements.txt index 11baf2e..7683839 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,17 +1,18 @@ +deepforest flake8-docstrings geopandas +h5py +laspy +laspy[lazrs,laszip] +lazrs +numpy +opencv-python pandas pytest pytest-cov +rasterio requests +sphinx-press-theme toml tox yapf -sphinx-press-theme -laspy -lazrs -opencv-python -numpy -rasterio -deepforest -laspy[lazrs,laszip] diff --git a/tests/raw_data/raster_data/DP3.30015.001/2018/FullSite/D16/2018_ABBY_2/L3/DiscreteLidar/CanopyHeightModelGtif/NEON_D16_ABBY_DP3_559000_5070000_CHM.tif b/tests/raw_data/raster_data/DP3.30015.001/2018/FullSite/D16/2018_ABBY_2/L3/DiscreteLidar/CanopyHeightModelGtif/NEON_D16_ABBY_DP3_559000_5070000_CHM.tif new file mode 100644 index 0000000..6a538d6 Binary files /dev/null and b/tests/raw_data/raster_data/DP3.30015.001/2018/FullSite/D16/2018_ABBY_2/L3/DiscreteLidar/CanopyHeightModelGtif/NEON_D16_ABBY_DP3_559000_5070000_CHM.tif differ diff --git a/tests/raw_data/raster_data/DP3.30015.001/2018/FullSite/D16/2018_ABBY_2/Metadata/DiscreteLidar/TileBoundary/kmls/NEON_D16_ABBY_DP1_559000_5070000_classified_point_cloud.kml b/tests/raw_data/raster_data/DP3.30015.001/2018/FullSite/D16/2018_ABBY_2/Metadata/DiscreteLidar/TileBoundary/kmls/NEON_D16_ABBY_DP1_559000_5070000_classified_point_cloud.kml new file mode 100644 index 0000000..a429b5a --- /dev/null +++ b/tests/raw_data/raster_data/DP3.30015.001/2018/FullSite/D16/2018_ABBY_2/Metadata/DiscreteLidar/TileBoundary/kmls/NEON_D16_ABBY_DP1_559000_5070000_classified_point_cloud.kml @@ -0,0 +1,179 @@ + + + + + + + #yellow + + + +-122.23238135,45.78998225,472.42 +-122.23251581,45.78998319,502.60 +-122.23293157,45.78998597,495.12 +-122.23323219,45.78998800,528.25 +-122.23336215,45.78998883,533.17 +-122.23352100,45.78998984,529.48 +-122.23391686,45.78999247,536.98 +-122.23418289,45.78999436,550.95 +-122.23446446,45.78999622,557.39 +-122.23488887,45.78999910,579.25 +-122.23518130,45.79000103,596.16 +-122.23558094,45.79000370,596.65 +-122.23583161,45.79000538,607.72 +-122.23637528,45.79000902,614.77 +-122.23667629,45.79001104,630.26 +-122.23691821,45.79001267,627.68 +-122.23746663,45.79001634,638.23 +-122.23787007,45.79001903,647.24 +-122.23829064,45.79002180,649.80 +-122.23889580,45.79002587,642.38 +-122.23941458,45.79002932,647.60 +-122.24003466,45.79003344,620.49 +-122.24060095,45.79003719,602.50 +-122.24072702,45.79003803,616.02 +-122.24081019,45.79003854,616.17 +-122.24090217,45.79003900,630.50 +-122.24090741,45.79003863,605.05 +-122.24090941,45.79003769,619.30 +-122.24090961,45.79003546,624.19 +-122.24091087,45.78994774,632.77 +-122.24091276,45.78981042,627.01 +-122.24091613,45.78956400,629.13 +-122.24091874,45.78937174,622.10 +-122.24092179,45.78914701,649.52 +-122.24092677,45.78878024,663.03 +-122.24092683,45.78877532,662.99 +-122.24092733,45.78873915,662.55 +-122.24093202,45.78839299,670.26 +-122.24093390,45.78825512,658.97 +-122.24093661,45.78805501,677.56 +-122.24093793,45.78795761,671.62 +-122.24094121,45.78771630,697.13 +-122.24094387,45.78751962,696.45 +-122.24094775,45.78723222,691.93 +-122.24095073,45.78701475,716.12 +-122.24095427,45.78675446,739.77 +-122.24095765,45.78650539,741.78 +-122.24095810,45.78647169,740.04 +-122.24095917,45.78639010,728.65 +-122.24096435,45.78600930,755.07 +-122.24096779,45.78575778,774.91 +-122.24096874,45.78568832,774.25 +-122.24096986,45.78560544,753.52 +-122.24097462,45.78525376,772.82 +-122.24097634,45.78512721,795.48 +-122.24098101,45.78478381,801.61 +-122.24098664,45.78436899,819.95 +-122.24098998,45.78411981,819.61 +-122.24099274,45.78391684,831.97 +-122.24099807,45.78352698,840.98 +-122.24099995,45.78338830,840.10 +-122.24100301,45.78316303,845.75 +-122.24100597,45.78294461,851.62 +-122.24101049,45.78261037,830.82 +-122.24101301,45.78242328,831.80 +-122.24101683,45.78214237,827.47 +-122.24102209,45.78175510,845.19 +-122.24102355,45.78164921,840.07 +-122.24102398,45.78161755,838.33 +-122.24102423,45.78159943,845.91 +-122.24102809,45.78131481,832.99 +-122.24102980,45.78118832,825.68 +-122.24103104,45.78109728,840.28 +-122.24103152,45.78105487,831.92 +-122.24103161,45.78104560,830.48 +-122.24103108,45.78104047,834.11 +-122.24103014,45.78103998,837.98 +-122.24101558,45.78103942,835.67 +-122.24100614,45.78103927,832.36 +-122.24084005,45.78103811,844.06 +-122.24046370,45.78103560,842.94 +-122.23990230,45.78103188,820.38 +-122.23967972,45.78103041,832.94 +-122.23910563,45.78102657,823.25 +-122.23871273,45.78102397,823.67 +-122.23841314,45.78102196,806.22 +-122.23814754,45.78102018,811.12 +-122.23750748,45.78101593,804.22 +-122.23735086,45.78101489,806.80 +-122.23683190,45.78101142,782.21 +-122.23654231,45.78100949,775.20 +-122.23599912,45.78100588,769.10 +-122.23558946,45.78100310,761.54 +-122.23505218,45.78099952,753.16 +-122.23487799,45.78099834,760.59 +-122.23446789,45.78099560,748.59 +-122.23445889,45.78099554,761.53 +-122.23389307,45.78099184,760.91 +-122.23373284,45.78099073,745.26 +-122.23346330,45.78098889,744.87 +-122.23342892,45.78098979,743.23 +-122.23342211,45.78099023,731.96 +-122.23339356,45.78099616,732.85 +-122.23337332,45.78101520,731.03 +-122.23337611,45.78113084,732.19 +-122.23337897,45.78121380,733.23 +-122.23338068,45.78159885,735.48 +-122.23337276,45.78171767,734.30 +-122.23336741,45.78183789,733.26 +-122.23335587,45.78205088,731.29 +-122.23334934,45.78225623,729.59 +-122.23334104,45.78236337,728.07 +-122.23326144,45.78280977,709.15 +-122.23314301,45.78324149,682.22 +-122.23308463,45.78343722,666.49 +-122.23308419,45.78346469,666.52 +-122.23312231,45.78388315,676.06 +-122.23312488,45.78390203,676.86 +-122.23317638,45.78427732,688.30 +-122.23320198,45.78454484,693.25 +-122.23321142,45.78467600,695.20 +-122.23322496,45.78479141,697.67 +-122.23324568,45.78503386,702.08 +-122.23324782,45.78505210,702.60 +-122.23325844,45.78545998,703.32 +-122.23325252,45.78553470,701.97 +-122.23321130,45.78598143,690.07 +-122.23319094,45.78613482,684.61 +-122.23318346,45.78620583,682.11 +-122.23315809,45.78642558,675.12 +-122.23313964,45.78656417,669.62 +-122.23312010,45.78675973,663.72 +-122.23306359,45.78712179,648.35 +-122.23298096,45.78754079,625.42 +-122.23290970,45.78781744,606.94 +-122.23288090,45.78791713,598.96 +-122.23284534,45.78803543,589.93 +-122.23278066,45.78826695,573.07 +-122.23277595,45.78828580,571.45 +-122.23272403,45.78848753,558.93 +-122.23265525,45.78876980,542.39 +-122.23260206,45.78897986,530.45 +-122.23258008,45.78908361,525.32 +-122.23250474,45.78943420,506.75 +-122.23245574,45.78963089,494.85 +-122.23242285,45.78975558,487.07 +-122.23238606,45.78992532,475.85 +-122.23238135,45.78998225,472.42 + + + + + + diff --git a/tests/raw_data/raster_data/DP3.30015.001/2018/FullSite/D16/2018_ABBY_2/Metadata/DiscreteLidar/TileBoundary/shps/NEON_D16_ABBY_DP1_559000_5070000_classified_point_cloud.dbf b/tests/raw_data/raster_data/DP3.30015.001/2018/FullSite/D16/2018_ABBY_2/Metadata/DiscreteLidar/TileBoundary/shps/NEON_D16_ABBY_DP1_559000_5070000_classified_point_cloud.dbf new file mode 100644 index 0000000..3c6eafc Binary files /dev/null and b/tests/raw_data/raster_data/DP3.30015.001/2018/FullSite/D16/2018_ABBY_2/Metadata/DiscreteLidar/TileBoundary/shps/NEON_D16_ABBY_DP1_559000_5070000_classified_point_cloud.dbf differ diff --git a/tests/raw_data/raster_data/DP3.30015.001/2018/FullSite/D16/2018_ABBY_2/Metadata/DiscreteLidar/TileBoundary/shps/NEON_D16_ABBY_DP1_559000_5070000_classified_point_cloud.prj b/tests/raw_data/raster_data/DP3.30015.001/2018/FullSite/D16/2018_ABBY_2/Metadata/DiscreteLidar/TileBoundary/shps/NEON_D16_ABBY_DP1_559000_5070000_classified_point_cloud.prj new file mode 100644 index 0000000..69e0b1b --- /dev/null +++ b/tests/raw_data/raster_data/DP3.30015.001/2018/FullSite/D16/2018_ABBY_2/Metadata/DiscreteLidar/TileBoundary/shps/NEON_D16_ABBY_DP1_559000_5070000_classified_point_cloud.prj @@ -0,0 +1 @@ +PROJCS["WGS 84 / UTM 10N",GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]],PROJECTION["Transverse_Mercator"],PARAMETER["latitude_of_origin",0],PARAMETER["central_meridian",-123],PARAMETER["scale_factor",0.9996],PARAMETER["false_easting",500000],PARAMETER["false_northing",0],UNIT["Meter",1]] \ No newline at end of file diff --git a/tests/raw_data/raster_data/DP3.30015.001/2018/FullSite/D16/2018_ABBY_2/Metadata/DiscreteLidar/TileBoundary/shps/NEON_D16_ABBY_DP1_559000_5070000_classified_point_cloud.shp b/tests/raw_data/raster_data/DP3.30015.001/2018/FullSite/D16/2018_ABBY_2/Metadata/DiscreteLidar/TileBoundary/shps/NEON_D16_ABBY_DP1_559000_5070000_classified_point_cloud.shp new file mode 100644 index 0000000..936b0c8 Binary files /dev/null and b/tests/raw_data/raster_data/DP3.30015.001/2018/FullSite/D16/2018_ABBY_2/Metadata/DiscreteLidar/TileBoundary/shps/NEON_D16_ABBY_DP1_559000_5070000_classified_point_cloud.shp differ diff --git a/tests/raw_data/raster_data/DP3.30015.001/2018/FullSite/D16/2018_ABBY_2/Metadata/DiscreteLidar/TileBoundary/shps/NEON_D16_ABBY_DP1_559000_5070000_classified_point_cloud.shx b/tests/raw_data/raster_data/DP3.30015.001/2018/FullSite/D16/2018_ABBY_2/Metadata/DiscreteLidar/TileBoundary/shps/NEON_D16_ABBY_DP1_559000_5070000_classified_point_cloud.shx new file mode 100644 index 0000000..3dbc1f1 Binary files /dev/null and b/tests/raw_data/raster_data/DP3.30015.001/2018/FullSite/D16/2018_ABBY_2/Metadata/DiscreteLidar/TileBoundary/shps/NEON_D16_ABBY_DP1_559000_5070000_classified_point_cloud.shx differ diff --git a/tests/test_lib.py b/tests/test_lib.py index cb931d7..f48afc0 100644 --- a/tests/test_lib.py +++ b/tests/test_lib.py @@ -2,8 +2,9 @@ import os import subprocess import pytest - +import pandas as pd from neonwranglerpy.lib.retrieve_vst_data import retrieve_vst_data +from neonwranglerpy.lib.crop_plot_data import crop_data_to_plot # Main Paths file_location = os.path.dirname(os.path.realpath(__file__)) @@ -15,23 +16,41 @@ test_retrieve_vst = [('VST_DELA', 'DP1.10098.001', 'DELA', '2021-01', '2022-01', [ True, True ], { - 'cols': [ - 'uid_x', 'individualID', 'eventID', 'tagStatus', 'growthForm', 'plantStatus', - 'stemDiameter', 'measurementHeight', 'height', 'baseCrownHeight', 'breakHeight', - 'breakDiameter', 'maxCrownDiameter', 'ninetyCrownDiameter', 'canopyPosition', - 'shape', 'basalStemDiameter', 'basalStemDiameterMsrmntHeight', - 'maxBaseCrownDiameter', 'ninetyBaseCrownDiameter', 'uid_y', 'namedLocation', - 'date', 'tagEventID', 'domainID', 'siteID', 'plotID', 'subplotID', - 'nestedSubplotID', 'pointID', 'stemDistance', 'stemAzimuth', 'recordType', - 'supportingStemIndividualID', 'previouslyTaggedAs', 'samplingProtocolVersion', - 'taxonID', 'scientificName', 'taxonRank', 'identificationReferences', - 'morphospeciesID', 'morphospeciesIDRemarks', 'identificationQualifier', 'remarks', - 'measuredBy', 'recordedBy', 'dataQF', 'plotType', 'subtype', 'latitude', - 'longitude', 'datum', 'utmZone', 'easting', 'northing', 'horzUncert', 'crdSource', - 'elevation', 'vertUncert', 'nlcdClass', 'appMods', 'geometry', 'itcEasting', - 'itcNorthing' - ], - 'data':['09af9259-e01e-4a08-9f77-236c3d205ab4', 'NEON.PLA.D08.DELA.04240', 'vst_DELA_2021', 'ok', 'single bole tree', 'Live', 21.7, 130.0, 17.6, 0, 0, 0, 8.3, 7.6, 'Full sun', 0, 0, 0, 0, 0, 'd83b1872-e247-4439-be37-2727c2c22ad4', 'DELA_053.basePlot.vst', '2015-09-15', 'vst_DELA_2015', 'D08', 'DELA', 'DELA_053', 39.0, 0, '59', 4.8, 185.8, 0, 0, 0, 'NEON.DOC.000987vE', 'FRPE', 'Fraxinus pennsylvanica Marshall', 'species', 0, 0, 0, 'cf. species', 0, 'mrichards@field-ops.org', 'hshirley@field-ops.org', 0, 'tower', 'basePlot', 32.537652, -87.804167, 'WGS84', '16N', 424487.656, 3600317.947, 0.12, 'GeoXH 6000', 27.96, 0.13, 'deciduousForest', 'bbc|cdw|cfc|dhp|hbp|ltr|vst' ]})] + 'cols': [ + 'uid_x', 'individualID', 'eventID', 'tagStatus', 'growthForm', 'plantStatus', + 'stemDiameter', 'measurementHeight', 'height', 'baseCrownHeight', 'breakHeight', + 'breakDiameter', 'maxCrownDiameter', 'ninetyCrownDiameter', 'canopyPosition', + 'shape', 'basalStemDiameter', 'basalStemDiameterMsrmntHeight', + 'maxBaseCrownDiameter', 'ninetyBaseCrownDiameter', 'uid_y', 'namedLocation', + 'date', 'tagEventID', 'domainID', 'siteID', 'plotID', 'subplotID', + 'nestedSubplotID', 'pointID', 'stemDistance', 'stemAzimuth', 'recordType', + 'supportingStemIndividualID', 'previouslyTaggedAs', 'samplingProtocolVersion', + 'taxonID', 'scientificName', 'taxonRank', 'identificationReferences', + 'morphospeciesID', 'morphospeciesIDRemarks', 'identificationQualifier', 'remarks', + 'measuredBy', 'recordedBy', 'dataQF', 'plotType', 'subtype', 'latitude', + 'longitude', 'datum', 'utmZone', 'easting', 'northing', 'horzUncert', 'crdSource', + 'elevation', 'vertUncert', 'nlcdClass', 'appMods', 'geometry', 'itcEasting', + 'itcNorthing' + ], + 'data': ['09af9259-e01e-4a08-9f77-236c3d205ab4', 'NEON.PLA.D08.DELA.04240', 'vst_DELA_2021', + 'ok', 'single bole tree', 'Live', 21.7, 130.0, 17.6, 0, 0, 0, 8.3, 7.6, 'Full sun', + 0, 0, 0, 0, 0, 'd83b1872-e247-4439-be37-2727c2c22ad4', 'DELA_053.basePlot.vst', + '2015-09-15', 'vst_DELA_2015', 'D08', 'DELA', 'DELA_053', 39.0, 0, '59', 4.8, 185.8, + 0, 0, 0, 'NEON.DOC.000987vE', 'FRPE', 'Fraxinus pennsylvanica Marshall', 'species', + 0, 0, 0, 'cf. species', 0, 'mrichards@field-ops.org', 'hshirley@field-ops.org', 0, + 'tower', 'basePlot', 32.537652, -87.804167, 'WGS84', '16N', 424487.656, 3600317.947, + 0.12, 'GeoXH 6000', 27.96, 0.13, 'deciduousForest', 'bbc|cdw|cfc|dhp|hbp|ltr|vst']})] + +test_generate_raster_data = [ + ('test_clip_raster', { + 'plotID': ['TEST_0000', 'TEST_0000'], + 'subplotID': ['A', "A"], + 'siteID': ['ABBY', 'ABBY'], + 'utmZone': ['6N', '6N'], + 'easting': [559120, 559120], + 'northing': [5070120, 5070120] + }, "DP3.30015.001", "raster_data", "2018", ["NEON_D16_ABBY_DP3_559000_5070000_CHM.tif", True]) +] def setup_module(): @@ -69,3 +88,12 @@ def test_retrieve_vst_data(test_name, dpID, site, start_date, end_date, args, ex stacked_df=stacked_df) columns_values = list(data_frame['vst'].dtypes.index) assert columns_values == expected['cols'] + + +@pytest.mark.parametrize("test_name, plt, dpID, path, year, expected", test_generate_raster_data) +def test_crop_plot_data(test_name, plt, dpID, path, year, expected): + setup_functions() + path = os.path.join(raw_dir_files, path) + plt = pd.DataFrame(plt) + out = crop_data_to_plot(plt, dpID, path,year, savepath=raw_dir_files) + assert os.path.exists(os.path.join(raw_dir_files,expected[0])) == expected[1]