diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index fd5ff69..da34cf9 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -21,10 +21,10 @@ jobs: - name: Install Python dependencies run: | python -m pip install --upgrade pip - python -m pip install 'isort==5.12.0' 'ruff==0.1.15' + python -m pip install 'isort==5.12.0' 'black==23.10.0' 'ruff==0.1.15' - name: isort run: python -m isort . --check --diff - # - name: black - # run: python -m black --check --diff . + - name: black + run: python -m black --check --diff . - name: ruff run: ruff check . diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index af61341..9607ecd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -51,7 +51,7 @@ repos: hooks: - id: auto-smart-commit - repo: https://github.com/psf/black - rev: 23.10.0 + rev: "23.10.0" hooks: - id: black language_version: python3.9 diff --git a/pyproject.toml b/pyproject.toml index 4145b12..98b6556 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,10 @@ junit_logging = "log" # Include logging output in JUnit XML report profile = "black" +[tool.black] +line-length = 99 + + [tool.ruff] # line-length = 100 @@ -77,4 +81,3 @@ ignore = [ "E501", # Ignore "line-too-long" issues, let black handle that. ] - diff --git a/src/openeo_gfmap/backend.py b/src/openeo_gfmap/backend.py index f63e054..ba8cdb8 100644 --- a/src/openeo_gfmap/backend.py +++ b/src/openeo_gfmap/backend.py @@ -33,9 +33,7 @@ class BackendContext: backend: Backend -def _create_connection( - url: str, *, env_var_suffix: str, connect_kwargs: Optional[dict] = None -): +def _create_connection(url: str, *, env_var_suffix: str, connect_kwargs: Optional[dict] = None): """ Generic helper to create an openEO connection with support for multiple client credential configurations from environment variables @@ -63,9 +61,7 @@ def _create_connection( # Use a shorter max poll time by default to alleviate the default impression that the test seem to hang # because of the OIDC device code poll loop. - max_poll_time = int( - os.environ.get("OPENEO_OIDC_DEVICE_CODE_MAX_POLL_TIME") or 30 - ) + max_poll_time = int(os.environ.get("OPENEO_OIDC_DEVICE_CODE_MAX_POLL_TIME") or 30) connection.authenticate_oidc(max_poll_time=max_poll_time) return connection diff --git a/src/openeo_gfmap/extractions/s2.py b/src/openeo_gfmap/extractions/s2.py index bce191c..342f963 100644 --- a/src/openeo_gfmap/extractions/s2.py +++ b/src/openeo_gfmap/extractions/s2.py @@ -76,9 +76,7 @@ def s2_l2a_fetch_default( ), "CRS not defined within GeoJSON collection." spatial_extent = dict(spatial_extent) - cube = connection.load_collection( - collection_name, spatial_extent, temporal_extent, bands - ) + cube = connection.load_collection(collection_name, spatial_extent, temporal_extent, bands) # Apply if the collection is a GeoJSON Feature collection if isinstance(spatial_extent, GeoJSON): diff --git a/src/openeo_gfmap/features/feature_extractor.py b/src/openeo_gfmap/features/feature_extractor.py index 28ceb7a..28f6b79 100644 --- a/src/openeo_gfmap/features/feature_extractor.py +++ b/src/openeo_gfmap/features/feature_extractor.py @@ -75,9 +75,7 @@ class FeatureExtractor(ABC): point based extraction or dense Cubes for tile/polygon based extraction. """ - def _common_preparations( - self, inarr: xr.DataArray, parameters: dict - ) -> xr.DataArray: + def _common_preparations(self, inarr: xr.DataArray, parameters: dict) -> xr.DataArray: """Common preparations to be executed before the feature extractor is executed. This method should be called by the `_execute` method of the feature extractor. @@ -102,8 +100,7 @@ def output_labels(self) -> list: def _execute(self, cube: XarrayDataCube, parameters: dict) -> XarrayDataCube: raise NotImplementedError( - "FeatureExtractor is a base abstract class, please implement the " - "_execute method." + "FeatureExtractor is a base abstract class, please implement the " "_execute method." ) @@ -130,8 +127,7 @@ def get_latlons(self, inarr: xr.DataArray) -> xr.DataArray: if self.epsg is None: raise Exception( - "EPSG code was not defined, cannot extract lat/lon array " - "as the CRS is unknown." + "EPSG code was not defined, cannot extract lat/lon array " "as the CRS is unknown." ) # If the coordiantes are not in EPSG:4326, we need to reproject them @@ -257,9 +253,7 @@ def apply_feature_extractor( udf = openeo.UDF(code=udf_code, context=parameters) cube = cube.apply_neighborhood(process=udf, size=size, overlap=overlap) - return cube.rename_labels( - dimension="bands", target=feature_extractor_class().output_labels() - ) + return cube.rename_labels(dimension="bands", target=feature_extractor_class().output_labels()) def apply_feature_extractor_local( diff --git a/src/openeo_gfmap/fetching/commons.py b/src/openeo_gfmap/fetching/commons.py index a589dc8..898c86d 100644 --- a/src/openeo_gfmap/fetching/commons.py +++ b/src/openeo_gfmap/fetching/commons.py @@ -128,8 +128,7 @@ def load_collection( pre_mask = params.get("pre_mask", None) if pre_mask is not None: assert isinstance(pre_mask, openeo.DataCube), ( - f"The 'pre_mask' parameter must be an openeo datacube, " - f"got {pre_mask}." + f"The 'pre_mask' parameter must be an openeo datacube, " f"got {pre_mask}." ) cube = cube.mask(pre_mask.resample_cube_spatial(cube)) diff --git a/src/openeo_gfmap/fetching/s1.py b/src/openeo_gfmap/fetching/s1.py index 4fcee4f..cf061fb 100644 --- a/src/openeo_gfmap/fetching/s1.py +++ b/src/openeo_gfmap/fetching/s1.py @@ -84,9 +84,7 @@ def s1_grd_fetch_default( return s1_grd_fetch_default -def get_s1_grd_default_processor( - collection_name: str, fetch_type: FetchType -) -> Callable: +def get_s1_grd_default_processor(collection_name: str, fetch_type: FetchType) -> Callable: """Builds the preprocessing function from the collection name as it is stored in the target backend. """ @@ -113,9 +111,7 @@ def s1_grd_default_processor(cube: openeo.DataCube, **params): ) cube = resample_reproject( - cube, - params.get("target_resolution", 10.0), - params.get("target_crs", None) + cube, params.get("target_resolution", 10.0), params.get("target_crs", None) ) cube = rename_bands(cube, BASE_SENTINEL1_GRD_MAPPING) @@ -128,15 +124,11 @@ def s1_grd_default_processor(cube: openeo.DataCube, **params): SENTINEL1_GRD_BACKEND_MAP = { Backend.TERRASCOPE: { "default": partial(get_s1_grd_default_fetcher, collection_name="SENTINEL1_GRD"), - "preprocessor": partial( - get_s1_grd_default_processor, collection_name="SENTINEL1_GRD" - ), + "preprocessor": partial(get_s1_grd_default_processor, collection_name="SENTINEL1_GRD"), }, Backend.CDSE: { "default": partial(get_s1_grd_default_fetcher, collection_name="SENTINEL1_GRD"), - "preprocessor": partial( - get_s1_grd_default_processor, collection_name="SENTINEL1_GRD" - ), + "preprocessor": partial(get_s1_grd_default_processor, collection_name="SENTINEL1_GRD"), }, } diff --git a/src/openeo_gfmap/fetching/s2.py b/src/openeo_gfmap/fetching/s2.py index de7e201..0f7b3b3 100644 --- a/src/openeo_gfmap/fetching/s2.py +++ b/src/openeo_gfmap/fetching/s2.py @@ -114,9 +114,7 @@ def s2_l2a_fetch_default( return s2_l2a_fetch_default -def get_s2_l2a_element84_fetcher( - collection_name: str, fetch_type: FetchType -) -> Callable: +def get_s2_l2a_element84_fetcher(collection_name: str, fetch_type: FetchType) -> Callable: """Fetches the collections from the Sentinel-2 Cloud-Optimized GeoTIFFs bucket provided by Amazon and managed by Element84. """ @@ -157,9 +155,7 @@ def s2_l2a_element84_fetcher( return s2_l2a_element84_fetcher -def get_s2_l2a_default_processor( - collection_name: str, fetch_type: FetchType -) -> Callable: +def get_s2_l2a_default_processor(collection_name: str, fetch_type: FetchType) -> Callable: """Builds the preprocessing function from the collection name as it stored in the target backend. """ @@ -188,15 +184,11 @@ def s2_l2a_default_processor(cube: openeo.DataCube, **params): SENTINEL2_L2A_BACKEND_MAP = { Backend.TERRASCOPE: { "fetch": partial(get_s2_l2a_default_fetcher, collection_name="SENTINEL2_L2A"), - "preprocessor": partial( - get_s2_l2a_default_processor, collection_name="SENTINEL2_L2A" - ), + "preprocessor": partial(get_s2_l2a_default_processor, collection_name="SENTINEL2_L2A"), }, Backend.CDSE: { "fetch": partial(get_s2_l2a_default_fetcher, collection_name="SENTINEL2_L2A"), - "preprocessor": partial( - get_s2_l2a_default_processor, collection_name="SENTINEL2_L2A" - ), + "preprocessor": partial(get_s2_l2a_default_processor, collection_name="SENTINEL2_L2A"), }, } diff --git a/src/openeo_gfmap/inference/inference_models.py b/src/openeo_gfmap/inference/inference_models.py index e5f60f4..e83d676 100644 --- a/src/openeo_gfmap/inference/inference_models.py +++ b/src/openeo_gfmap/inference/inference_models.py @@ -29,9 +29,7 @@ class ModelInference(ABC): methods and attributes to be used by other model inference classes. """ - def _common_preparations( - self, inarr: xr.DataArray, parameters: dict - ) -> xr.DataArray: + def _common_preparations(self, inarr: xr.DataArray, parameters: dict) -> xr.DataArray: """Common preparations for all inference models. This method will be executed at the very beginning of the process. """ diff --git a/src/openeo_gfmap/preprocessing/cloudmasking.py b/src/openeo_gfmap/preprocessing/cloudmasking.py index 3cbdeed..e605689 100644 --- a/src/openeo_gfmap/preprocessing/cloudmasking.py +++ b/src/openeo_gfmap/preprocessing/cloudmasking.py @@ -33,9 +33,7 @@ def mask_scl_dilation(cube: openeo.DataCube, **params: dict) -> openeo.DataCube: ) nonoptical_cube = cube.filter_bands( - bands=list( - filter(lambda band: not band.startswith("S2"), cube.metadata.band_names) - ) + bands=list(filter(lambda band: not band.startswith("S2"), cube.metadata.band_names)) ) optical_cube = optical_cube.process( @@ -224,9 +222,7 @@ def bap_masking(cube: openeo.DataCube, period: Union[str, list], **params: dict) ) nonoptical_cube = cube.filter_bands( - bands=list( - filter(lambda band: not band.startswith("S2"), cube.metadata.band_names) - ) + bands=list(filter(lambda band: not band.startswith("S2"), cube.metadata.band_names)) ) rank_mask = get_bap_mask(optical_cube, period, **params) diff --git a/src/openeo_gfmap/preprocessing/compositing.py b/src/openeo_gfmap/preprocessing/compositing.py index 5f7eaf4..a0b12f2 100644 --- a/src/openeo_gfmap/preprocessing/compositing.py +++ b/src/openeo_gfmap/preprocessing/compositing.py @@ -14,6 +14,7 @@ def median_compositing(cube: openeo.DataCube, period: Union[str, list]) -> opene elif isinstance(period, list): return cube.aggregate_temporal(intervals=period, reducer="median", dimension="t") + def mean_compositing(cube: openeo.DataCube, period: str) -> openeo.DataCube: """Perfrom mean compositing on the given datacube.""" if isinstance(period, str): diff --git a/src/openeo_gfmap/preprocessing/interpolation.py b/src/openeo_gfmap/preprocessing/interpolation.py index bffaa49..c404403 100644 --- a/src/openeo_gfmap/preprocessing/interpolation.py +++ b/src/openeo_gfmap/preprocessing/interpolation.py @@ -4,6 +4,8 @@ import openeo -def linear_interpolation(cube: openeo.DataCube,) -> openeo.DataCube: +def linear_interpolation( + cube: openeo.DataCube, +) -> openeo.DataCube: """Perform linear interpolation on the given datacube.""" return cube.apply_dimension(dimension="t", process="array_interpolate_linear") diff --git a/src/openeo_gfmap/preprocessing/udf_rank.py b/src/openeo_gfmap/preprocessing/udf_rank.py index b68dad1..7271b61 100644 --- a/src/openeo_gfmap/preprocessing/udf_rank.py +++ b/src/openeo_gfmap/preprocessing/udf_rank.py @@ -1,4 +1,3 @@ - import numpy as np import xarray as xr from openeo.udf import XarrayDataCube @@ -31,9 +30,7 @@ def select_maximum(score: xr.DataArray): # Convert YYYY-mm-dd to datetime64 objects time_bins = [np.datetime64(interval[0]) for interval in intervals] - rank_mask = bap_score.groupby_bins('t', bins=time_bins).map( - select_maximum - ) + rank_mask = bap_score.groupby_bins("t", bins=time_bins).map(select_maximum) else: raise ValueError("Period is not defined in the UDF. Cannot run it.") diff --git a/src/openeo_gfmap/preprocessing/udf_score.py b/src/openeo_gfmap/preprocessing/udf_score.py index 578a5b9..ab72f65 100644 --- a/src/openeo_gfmap/preprocessing/udf_score.py +++ b/src/openeo_gfmap/preprocessing/udf_score.py @@ -11,9 +11,9 @@ def apply_datacube(cube: XarrayDataCube, context: dict) -> XarrayDataCube: cube_array: xr.DataArray = cube.get_array() cube_array = cube_array.transpose("t", "bands", "y", "x") - clouds = np.logical_or( - np.logical_and(cube_array < 11, cube_array >= 8), cube_array == 3 - ).isel(bands=0) + clouds = np.logical_or(np.logical_and(cube_array < 11, cube_array >= 8), cube_array == 3).isel( + bands=0 + ) weights = [1, 0.8, 0.5] @@ -21,11 +21,7 @@ def apply_datacube(cube: XarrayDataCube, context: dict) -> XarrayDataCube: times = cube_array.t.dt.day.values # returns day of the month for each date sigma = 5 mu = 15 - score_doy = ( - 1 - / (sigma * math.sqrt(2 * math.pi)) - * np.exp(-0.5 * ((times - mu) / sigma) ** 2) - ) + score_doy = 1 / (sigma * math.sqrt(2 * math.pi)) * np.exp(-0.5 * ((times - mu) / sigma) ** 2) score_doy = np.broadcast_to( score_doy[:, np.newaxis, np.newaxis], [cube_array.sizes["t"], cube_array.sizes["y"], cube_array.sizes["x"]], @@ -77,9 +73,9 @@ def erode(image, selem): ) # Final score is weighted average - score = ( - weights[0] * score_clouds + weights[1] * score_doy + weights[2] * score_cov - ) / sum(weights) + score = (weights[0] * score_clouds + weights[1] * score_doy + weights[2] * score_cov) / sum( + weights + ) score = np.where(cube_array.values[:, 0, :, :] == 0, 0, score) score_da = xr.DataArray( diff --git a/src/openeo_gfmap/utils/catalogue.py b/src/openeo_gfmap/utils/catalogue.py index 9e35244..e420a05 100644 --- a/src/openeo_gfmap/utils/catalogue.py +++ b/src/openeo_gfmap/utils/catalogue.py @@ -160,9 +160,7 @@ def s1_area_per_orbitstate( ) ) else: - raise NotImplementedError( - f"This feature is not supported for backend: {backend.backend}." - ) + raise NotImplementedError(f"This feature is not supported for backend: {backend.backend}.") # Builds the shape of the spatial extent and computes the area spatial_extent = spatial_extent.to_geometry() @@ -179,15 +177,13 @@ def s1_area_per_orbitstate( "ASCENDING": { "full_overlap": ascending_covers, "area": sum( - product.intersection(spatial_extent).area - for product in ascending_products + product.intersection(spatial_extent).area for product in ascending_products ), }, "DESCENDING": { "full_overlap": descending_covers, "area": sum( - product.intersection(spatial_extent).area - for product in descending_products + product.intersection(spatial_extent).area for product in descending_products ), }, } diff --git a/src/openeo_gfmap/utils/tile_processing.py b/src/openeo_gfmap/utils/tile_processing.py index b763be4..fa1efab 100644 --- a/src/openeo_gfmap/utils/tile_processing.py +++ b/src/openeo_gfmap/utils/tile_processing.py @@ -20,9 +20,7 @@ def normalize_array(inarr: xr.DataArray, percentile: float = 0.99) -> xr.DataArr def select_optical_bands(inarr: xr.DataArray) -> xr.DataArray: """Filters and keep only the optical bands for a given array.""" return inarr.sel( - bands=[ - band for band in inarr.coords["bands"].to_numpy() if band.startswith("S2-B") - ] + bands=[band for band in inarr.coords["bands"].to_numpy() if band.startswith("S2-B")] ) @@ -30,9 +28,7 @@ def select_sar_bands(inarr: xr.DataArray) -> xr.DataArray: """Filters and keep only the SAR bands for a given array.""" return inarr.sel( bands=[ - band - for band in inarr.coords["bands"].to_numpy() - if band in ["VV", "VH", "HH", "HV"] + band for band in inarr.coords["bands"].to_numpy() if band in ["VV", "VH", "HH", "HV"] ] ) @@ -47,9 +43,7 @@ def array_bounds(inarr: xr.DataArray) -> tuple: ) -def arrays_cosine_similarity( - first_array: xr.DataArray, second_array: xr.DataArray -) -> float: +def arrays_cosine_similarity(first_array: xr.DataArray, second_array: xr.DataArray) -> float: """Returns a similarity score based on normalized cosine distance. The input arrays must have similar ranges to obtain a valid score. 1.0 represents the best score (same tiles), while 0.0 is the worst score. diff --git a/tests/test_openeo_gfmap/test_cloud_masking.py b/tests/test_openeo_gfmap/test_cloud_masking.py index e0c3db4..2328f9d 100644 --- a/tests/test_openeo_gfmap/test_cloud_masking.py +++ b/tests/test_openeo_gfmap/test_cloud_masking.py @@ -53,9 +53,7 @@ def test_bap_score(backend: Backend): bap_score = get_bap_score(cube, **preprocessing_parameters) ndvi = cube.ndvi(nir="S2-B08", red="S2-B04") - cube = bap_score.merge_cubes(ndvi).rename_labels( - "bands", ["S2-BAPSCORE", "S2-NDVI"] - ) + cube = bap_score.merge_cubes(ndvi).rename_labels("bands", ["S2-BAPSCORE", "S2-NDVI"]) job = cube.create_job( title="BAP score unittest", @@ -66,9 +64,7 @@ def test_bap_score(backend: Backend): for asset in job.get_results().get_assets(): if asset.metadata["type"].startswith("application/x-netcdf"): - asset.download( - Path(__file__).parent / f"results/bap_score_{backend.value}.nc" - ) + asset.download(Path(__file__).parent / f"results/bap_score_{backend.value}.nc") @pytest.mark.parametrize("backend", backends) @@ -100,9 +96,7 @@ def test_bap_masking(backend: Backend): cube = cube.linear_scale_range(0, 65535, 0, 65535) # Remove SCL - cube = cube.filter_bands( - [band for band in cube.metadata.band_names if band != "S2-SCL"] - ) + cube = cube.filter_bands([band for band in cube.metadata.band_names if band != "S2-SCL"]) job = cube.create_job( title="BAP compositing unittest", @@ -113,9 +107,7 @@ def test_bap_masking(backend: Backend): for asset in job.get_results().get_assets(): if asset.metadata["type"].startswith("application/x-netcdf"): - asset.download( - Path(__file__).parent / f"results/bap_composited_{backend.value}.nc" - ) + asset.download(Path(__file__).parent / f"results/bap_composited_{backend.value}.nc") @pytest.mark.parametrize("backend", backends) @@ -124,12 +116,8 @@ def test_bap_quintad(backend: Backend): backend_context = BackendContext(backend=backend) # Additional parameters - fetching_parameters = { - "fetching_resolution": 10.0 - } - preprocessing_parameters = { - "apply_scl_dilation": True - } + fetching_parameters = {"fetching_resolution": 10.0} + preprocessing_parameters = {"apply_scl_dilation": True} # Fetch the datacube s2_extractor = build_sentinel2_l2a_extractor( @@ -173,9 +161,7 @@ def test_bap_quintad(backend: Backend): assert compositing_intervals == expected_intervals # Perform masking with BAP, masking optical bands - bap_mask = get_bap_mask( - cube, period=compositing_intervals, **preprocessing_parameters - ) + bap_mask = get_bap_mask(cube, period=compositing_intervals, **preprocessing_parameters) # Create a new extractor for the whole data now fetching_parameters = { @@ -206,6 +192,4 @@ def test_bap_quintad(backend: Backend): for asset in job.get_results().get_assets(): if asset.metadata["type"].startswith("application/x-netcdf"): - asset.download( - Path(__file__).parent / f"results/bap_quintad_{backend.value}.nc" - ) + asset.download(Path(__file__).parent / f"results/bap_quintad_{backend.value}.nc") diff --git a/tests/test_openeo_gfmap/test_feature_extractors.py b/tests/test_openeo_gfmap/test_feature_extractors.py index d984bf6..621c010 100644 --- a/tests/test_openeo_gfmap/test_feature_extractors.py +++ b/tests/test_openeo_gfmap/test_feature_extractors.py @@ -43,9 +43,7 @@ def execute(self, inarr: xr.DataArray): ) # Compute the median on the time band - rgb_bands = rgb_bands.median(dim="t").assign_coords( - {"bands": ["red", "green", "blue"]} - ) + rgb_bands = rgb_bands.median(dim="t").assign_coords({"bands": ["red", "green", "blue"]}) # Returns the rgb bands only in the feature, y, x order return rgb_bands.transpose("bands", "y", "x") @@ -83,9 +81,7 @@ def test_patch_feature_udf(backend: Backend, connection_fn: Callable): bands_to_extract = ["S2-B04", "S2-B03", "S2-B02"] # Setup the RGB cube extraction - extractor = build_sentinel2_l2a_extractor( - backend_context, bands_to_extract, FetchType.TILE - ) + extractor = build_sentinel2_l2a_extractor(backend_context, bands_to_extract, FetchType.TILE) rgb_cube = extractor.get_cube(connection, SPATIAL_CONTEXT, TEMPORAL_EXTENT) @@ -122,15 +118,11 @@ def test_latlon_extractor(backend: Backend, connection_fn: Callable): connection = connection_fn() output_path = Path(__file__).parent / f"results/latlon_features_{backend.value}.nc" - REDUCED_TEMPORAL_CONTEXT = TemporalContext( - start_date="2023-06-01", end_date="2023-06-30" - ) + REDUCED_TEMPORAL_CONTEXT = TemporalContext(start_date="2023-06-01", end_date="2023-06-30") bands_to_extract = ["S2-B04"] - extractor = build_sentinel2_l2a_extractor( - backend_context, bands_to_extract, FetchType.TILE - ) + extractor = build_sentinel2_l2a_extractor(backend_context, bands_to_extract, FetchType.TILE) cube = extractor.get_cube(connection, SPATIAL_CONTEXT, REDUCED_TEMPORAL_CONTEXT) @@ -165,9 +157,9 @@ def test_patch_feature_local(): inds = xr.open_dataset(input_path).to_array(dim="bands") - inds = inds.sel( - bands=[band for band in inds.bands.to_numpy() if band != "crs"] - ).transpose("bands", "t", "y", "x") + inds = inds.sel(bands=[band for band in inds.bands.to_numpy() if band != "crs"]).transpose( + "bands", "t", "y", "x" + ) features = apply_feature_extractor_local(DummyPatchExtractor, inds, parameters={}) diff --git a/tests/test_openeo_gfmap/test_s1_fetchers.py b/tests/test_openeo_gfmap/test_s1_fetchers.py index d0bbcf2..bf1c7d9 100644 --- a/tests/test_openeo_gfmap/test_s1_fetchers.py +++ b/tests/test_openeo_gfmap/test_s1_fetchers.py @@ -76,14 +76,9 @@ def sentinel1_grd( cube = extractor.get_cube(connection, spatial_extent, temporal_extent) - output_file = ( - Path(__file__).parent - / f"results/{country}_{backend.value}_sentinel1_grd.nc" - ) + output_file = Path(__file__).parent / f"results/{country}_{backend.value}_sentinel1_grd.nc" - job = cube.create_job( - title="Sentinel1 GRD tile extraction", out_format="NetCDF" - ) + job = cube.create_job(title="Sentinel1 GRD tile extraction", out_format="NetCDF") job.start_and_wait() @@ -113,8 +108,7 @@ def compare_sentinel1_tiles(): loaded_tiles = [] for backend in backend_types: tile_path = ( - Path(__file__).parent - / f"results/{country}_{backend.value}_sentinel1_grd.nc" + Path(__file__).parent / f"results/{country}_{backend.value}_sentinel1_grd.nc" ) loaded_tiles.append(xr.open_dataset(tile_path, engine="h5netcdf")) @@ -182,9 +176,7 @@ def sentinel1_grd_point_based( cube = cube.aggregate_spatial(spatial_context, reducer="mean") - output_file = ( - Path(__file__).parent / f"results/points_{backend.value}_sentinel1_grd.nc" - ) + output_file = Path(__file__).parent / f"results/points_{backend.value}_sentinel1_grd.nc" cube.download(output_file, format="JSON") @@ -247,23 +239,17 @@ def sentinel1_grd_polygon_based( results.download_files(output_folder) # List all the files available in the folder - extracted_files = list( - filter(lambda file: file.suffix == ".nc", output_folder.iterdir()) - ) + extracted_files = list(filter(lambda file: file.suffix == ".nc", output_folder.iterdir())) # Check if there is one file for each polygon assert len(extracted_files) == len(spatial_context["features"]) -@pytest.mark.parametrize( - "spatial_context, temporal_context, backend", test_configurations -) +@pytest.mark.parametrize("spatial_context, temporal_context, backend", test_configurations) def test_sentinel1_grd( spatial_context: SpatialContext, temporal_context: TemporalContext, backend: Backend ): connection = BACKEND_CONNECTIONS[backend]() - TestS1Extractors.sentinel1_grd( - spatial_context, temporal_context, backend, connection - ) + TestS1Extractors.sentinel1_grd(spatial_context, temporal_context, backend, connection) @pytest.mark.depends(on=["test_sentinel1_grd"]) diff --git a/tests/test_openeo_gfmap/test_s2_fetchers.py b/tests/test_openeo_gfmap/test_s2_fetchers.py index 59e9749..c6bbfe7 100644 --- a/tests/test_openeo_gfmap/test_s2_fetchers.py +++ b/tests/test_openeo_gfmap/test_s2_fetchers.py @@ -50,14 +50,10 @@ TEMPORAL_EXTENT_2 = ["2023-01-01", "2023-02-01"] # Dataset of polygons for POINT based extraction -POINT_EXTRACTION_DF = ( - Path(__file__).parent / "resources/malawi_extraction_polygons.gpkg" -) +POINT_EXTRACTION_DF = Path(__file__).parent / "resources/malawi_extraction_polygons.gpkg" # Datase of polygons for Polygon based extraction -POLYGON_EXTRACTION_DF = ( - Path(__file__).parent / "resources/puglia_extraction_polygons.gpkg" -) +POLYGON_EXTRACTION_DF = Path(__file__).parent / "resources/puglia_extraction_polygons.gpkg" # test_backends = [Backend.TERRASCOPE, Backend.CDSE] test_backends = [Backend.CDSE] @@ -100,9 +96,7 @@ def sentinel2_l2a( "S2-AOT", ] fetching_parameters = ( - {"target_resolution": 10.0, "target_crs": 3035} - if country == "Belgium" - else {} + {"target_resolution": 10.0, "target_crs": 3035} if country == "Belgium" else {} ) extractor: CollectionFetcher = build_sentinel2_l2a_extractor( context=context, @@ -125,10 +119,7 @@ def sentinel2_l2a( cube = extractor.get_cube(connection, spatial_extent, temporal_extent) - output_file = ( - Path(__file__).parent - / f"results/{country}_{backend.value}_sentinel2_l2a.nc" - ) + output_file = Path(__file__).parent / f"results/{country}_{backend.value}_sentinel2_l2a.nc" cube.download(output_file, format="NetCDF") @@ -153,8 +144,7 @@ def compare_sentinel2_tiles(): if backend == Backend.EODC: # TODO fix EDOC backend first continue tile_path = ( - Path(__file__).parent - / f"results/{country}_{backend.value}_sentinel2_l2a.nc" + Path(__file__).parent / f"results/{country}_{backend.value}_sentinel2_l2a.nc" ) loaded_tiles.append(xr.open_dataset(tile_path, engine="h5netcdf")) @@ -216,9 +206,7 @@ def sentinel2_l2a_point_based( cube = cube.aggregate_spatial(spatial_context, reducer="mean") - output_file = ( - Path(__file__).parent / f"results/points_{backend.value}_sentinel2_l2a.json" - ) + output_file = Path(__file__).parent / f"results/points_{backend.value}_sentinel2_l2a.json" cube.download(output_file, format="JSON") @@ -273,23 +261,17 @@ def sentinel2_l2a_polygon_based( results.download_files(output_folder) # List all the files available in the folder - extracted_files = list( - filter(lambda file: file.suffix == ".nc", output_folder.iterdir()) - ) + extracted_files = list(filter(lambda file: file.suffix == ".nc", output_folder.iterdir())) # Check if there is one file for each polygon assert len(extracted_files) == len(spatial_context["features"]) -@pytest.mark.parametrize( - "spatial_context, temporal_context, backend", test_configurations -) +@pytest.mark.parametrize("spatial_context, temporal_context, backend", test_configurations) def test_sentinel2_l2a( spatial_context: SpatialContext, temporal_context: TemporalContext, backend: Backend ): connection = BACKEND_CONNECTIONS[backend]() - TestS2Extractors.sentinel2_l2a( - spatial_context, temporal_context, backend, connection - ) + TestS2Extractors.sentinel2_l2a(spatial_context, temporal_context, backend, connection) @pytest.mark.depends(on=["test_sentinel2_l2a"]) diff --git a/tests/test_openeo_gfmap/test_utils.py b/tests/test_openeo_gfmap/test_utils.py index 598de11..2eb574c 100644 --- a/tests/test_openeo_gfmap/test_utils.py +++ b/tests/test_openeo_gfmap/test_utils.py @@ -2,9 +2,7 @@ from openeo_gfmap.utils.catalogue import s1_area_per_orbitstate, select_S1_orbitstate # Region of Paris, France -SPATIAL_CONTEXT = BoundingBoxExtent( - west=1.979, south=48.705, east=2.926, north=49.151, epsg=4326 -) +SPATIAL_CONTEXT = BoundingBoxExtent(west=1.979, south=48.705, east=2.926, north=49.151, epsg=4326) # Summer 2023 TEMPORAL_CONTEXT = TemporalContext(start_date="2023-06-21", end_date="2023-09-21")