From bfbe2127af42c92bdc7974041b395b09fb124eb4 Mon Sep 17 00:00:00 2001 From: Emile Sonneveld Date: Tue, 28 May 2024 16:28:42 +0200 Subject: [PATCH] Add testMissingS2DateLine https://github.com/Open-EO/openeo-geotrellis-extensions/issues/279 --- .../org/openeo/geotrellis/LayerFixtures.scala | 54 +++++++++++++------ .../layers/FileLayerProviderTest.scala | 23 ++++++++ 2 files changed, 62 insertions(+), 15 deletions(-) diff --git a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/LayerFixtures.scala b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/LayerFixtures.scala index cde6d0f5..53738852 100644 --- a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/LayerFixtures.scala +++ b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/LayerFixtures.scala @@ -19,6 +19,7 @@ import org.openeo.geotrellisaccumulo import org.openeo.geotrelliscommon.{DataCubeParameters, SparseSpaceTimePartitioner} import org.openeo.opensearch.OpenSearchClient import org.openeo.opensearch.OpenSearchResponses.CreoFeatureCollection +import org.openeo.opensearch.backends.CreodiasClient import java.awt.image.DataBufferByte import java.io.File @@ -356,9 +357,12 @@ object LayerFixtures { /** * Creates a Sentinel-2 cube by downloading data locally. */ - def sentinel2Cube(localDate: LocalDate, projected_polygons_native_crs: ProjectedPolygons, jsonPath: String, + def sentinel2Cube(localDate: LocalDate, + projected_polygons_native_crs: ProjectedPolygons, + jsonPath: String, dataCubeParameters: DataCubeParameters = new DataCubeParameters, - bandNames: util.List[String] = util.Arrays.asList("IMG_DATA_Band_B04_10m_Tile1_Data", "S2_Level-2A_Tile1_Metadata##1", "S2_Level-2A_Tile1_Metadata##0")) = { + bandNames: util.List[String] = util.Arrays.asList("IMG_DATA_Band_B04_10m_Tile1_Data", "S2_Level-2A_Tile1_Metadata##1", "S2_Level-2A_Tile1_Metadata##0") + ) = { val jsonPathFull = getClass.getResource(jsonPath) val fileSource = Source.fromURL(jsonPathFull) @@ -368,7 +372,7 @@ object LayerFixtures { // Use artifactory to avoid heavy git repo val basePathArtifactory = "https://artifactory.vgt.vito.be/artifactory/testdata-public" - for (rest <- Seq( + val artifactoryPaths = Set( "/eodata/Sentinel-2/MSI/L2A/2023/01/17/S2B_MSIL2A_20230117T104259_N0509_R008_T31UGS_20230117T120337.SAFE/manifest.safe", "/eodata/Sentinel-2/MSI/L2A/2023/01/17/S2B_MSIL2A_20230117T104259_N0509_R008_T31UGS_20230117T120337.SAFE/MTD_MSIL2A.xml", "/eodata/Sentinel-2/MSI/L2A/2023/01/17/S2B_MSIL2A_20230117T104259_N0509_R008_T31UGS_20230117T120337.SAFE/GRANULE/L2A_T31UGS_A030636_20230117T104258/MTD_TL.xml", @@ -402,18 +406,43 @@ object LayerFixtures { "/eodata/Sentinel-2/MSI/L2A/2024/03/24/S2B_MSIL2A_20240324T230529_N0510_R044_T04WDD_20240324T234241.SAFE/GRANULE/L2A_T04WDD_A036821_20240324T230529/MTD_TL.xml", "/eodata/Sentinel-2/MSI/L2A/2024/03/24/S2B_MSIL2A_20240324T230529_N0510_R044_T04WDD_20240324T234241.SAFE/manifest.safe", "/eodata/Sentinel-2/MSI/L2A/2024/03/24/S2B_MSIL2A_20240324T230529_N0510_R044_T04WDD_20240324T234241.SAFE/MTD_MSIL2A.xml", - )) { - val jp2File = new File(basePath, rest) + ) + + for (path <- artifactoryPaths) { + val jp2File = new File(basePath, path) if (!jp2File.exists()) { println("Copy from artifactory to: " + jp2File) - FileUtils.copyURLToFile(new URL(basePathArtifactory + rest), jp2File) + FileUtils.copyURLToFile(new URL(basePathArtifactory + path), jp2File) + } + } + + val folders = artifactoryPaths.map(x => x.substring(0, x.indexOf(".SAFE") + ".SAFE".length)) + val matches = "/eodata/.*?.SAFE".r.findAllIn(txt).toList + for (m <- matches) { + if (folders.contains(m)) { + txt = txt.replace(m, basePath + m) } } - txt = txt.replace("\"/eodata/", "\"" + basePath + "/eodata/") + + val mockedFeatures = CreoFeatureCollection.parse(txt) val client = new MockOpenSearchFeatures(mockedFeatures.features) // val client = CreodiasClient() // More difficult to capture a nodata piece + val localFromDate = localDate + val localToDate = localDate.plusDays(1) + val ZonedFromDate = ZonedDateTime.of(localFromDate, java.time.LocalTime.MIDNIGHT, UTC) + val zonedToDate = ZonedDateTime.of(localToDate, java.time.LocalTime.MIDNIGHT, UTC) + + sentinel2CubeCDSEGeneric((ZonedFromDate, zonedToDate), projected_polygons_native_crs, client, dataCubeParameters, bandNames) + } + + def sentinel2CubeCDSEGeneric(dateInterval: (ZonedDateTime, ZonedDateTime), + projected_polygons_native_crs: ProjectedPolygons, + client: OpenSearchClient = CreodiasClient(), + dataCubeParameters: DataCubeParameters = new DataCubeParameters, + bandNames: util.List[String] = util.Arrays.asList("IMG_DATA_Band_B04_10m_Tile1_Data", "S2_Level-2A_Tile1_Metadata##1", "S2_Level-2A_Tile1_Metadata##0") + ): MultibandTileLayerRDD[SpaceTimeKey] = { val factory = new PyramidFactory( client, "Sentinel2", bandNames, null, @@ -421,17 +450,12 @@ object LayerFixtures { ) factory.crs = projected_polygons_native_crs.crs - val localFromDate = localDate - val localToDate = localDate.plusDays(1) - val ZonedFromDate = ZonedDateTime.of(localFromDate, java.time.LocalTime.MIDNIGHT, UTC) - val zonedToDate = ZonedDateTime.of(localToDate, java.time.LocalTime.MIDNIGHT, UTC) - val from_date = DateTimeFormatter.ISO_OFFSET_DATE_TIME format ZonedFromDate - val to_date = DateTimeFormatter.ISO_OFFSET_DATE_TIME format zonedToDate - + val from_date = DateTimeFormatter.ISO_OFFSET_DATE_TIME format dateInterval._1 + val to_date = DateTimeFormatter.ISO_OFFSET_DATE_TIME format dateInterval._2 val cube: Seq[(Int, MultibandTileLayerRDD[SpaceTimeKey])] = factory.datacube_seq( projected_polygons_native_crs, - from_date, to_date, Collections.emptyMap(), "",dataCubeParameters = dataCubeParameters + from_date, to_date, Collections.emptyMap(), "", dataCubeParameters = dataCubeParameters ) cube.head._2 } diff --git a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala index c3b2e905..1efb6767 100644 --- a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala +++ b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala @@ -1134,6 +1134,29 @@ class FileLayerProviderTest extends RasterMatchers{ assertEquals(8, band.get(200, 200)) } + @Test + def testMissingS2DateLine(): Unit = { + val outDir = Paths.get("tmp/FileLayerProviderTest/") + new Directory(outDir.toFile).deleteRecursively() + Files.createDirectories(outDir) + + val dateFrom = ZonedDateTime.parse("2024-04-02T00:00:00Z") + val dateTo = ZonedDateTime.parse("2024-04-03T00:00:00Z") + + val extent = Extent(178.7384, 70.769, 178.8548, 70.8254) +// val extent = Extent(178.0, 70.0, 178.99, 70.99) + val latlon = CRS.fromName("EPSG:4326") + val projected_polygons_native_crs = ProjectedPolygons.fromExtent(extent, latlon.toString()) + val utmCrs = CRS.fromName("EPSG:32601") + val reprojected = projected_polygons_native_crs.polygons.head.reproject(projected_polygons_native_crs.crs, utmCrs) + val poly2 = ProjectedPolygons(Array(reprojected), utmCrs) + + val layer = LayerFixtures.sentinel2CubeCDSEGeneric((dateFrom, dateTo), poly2, bandNames = java.util.Arrays.asList("IMG_DATA_Band_SCL_20m_Tile1_Data")) + + val cubeSpatial = layer.toSpatial() + cubeSpatial.writeGeoTiff(outDir + "/testMissingS2DateLine.tiff") + } + private def keysForLargeArea(useBBox:Boolean=false) = { val date = LocalDate.of(2022, 2, 11).atStartOfDay(UTC) val crs = CRS.fromEpsgCode(32630)