From bfbe2127af42c92bdc7974041b395b09fb124eb4 Mon Sep 17 00:00:00 2001 From: Emile Sonneveld Date: Tue, 28 May 2024 16:28:42 +0200 Subject: [PATCH 01/15] Add testMissingS2DateLine https://github.com/Open-EO/openeo-geotrellis-extensions/issues/279 --- .../org/openeo/geotrellis/LayerFixtures.scala | 54 +++++++++++++------ .../layers/FileLayerProviderTest.scala | 23 ++++++++ 2 files changed, 62 insertions(+), 15 deletions(-) diff --git a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/LayerFixtures.scala b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/LayerFixtures.scala index cde6d0f5..53738852 100644 --- a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/LayerFixtures.scala +++ b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/LayerFixtures.scala @@ -19,6 +19,7 @@ import org.openeo.geotrellisaccumulo import org.openeo.geotrelliscommon.{DataCubeParameters, SparseSpaceTimePartitioner} import org.openeo.opensearch.OpenSearchClient import org.openeo.opensearch.OpenSearchResponses.CreoFeatureCollection +import org.openeo.opensearch.backends.CreodiasClient import java.awt.image.DataBufferByte import java.io.File @@ -356,9 +357,12 @@ object LayerFixtures { /** * Creates a Sentinel-2 cube by downloading data locally. */ - def sentinel2Cube(localDate: LocalDate, projected_polygons_native_crs: ProjectedPolygons, jsonPath: String, + def sentinel2Cube(localDate: LocalDate, + projected_polygons_native_crs: ProjectedPolygons, + jsonPath: String, dataCubeParameters: DataCubeParameters = new DataCubeParameters, - bandNames: util.List[String] = util.Arrays.asList("IMG_DATA_Band_B04_10m_Tile1_Data", "S2_Level-2A_Tile1_Metadata##1", "S2_Level-2A_Tile1_Metadata##0")) = { + bandNames: util.List[String] = util.Arrays.asList("IMG_DATA_Band_B04_10m_Tile1_Data", "S2_Level-2A_Tile1_Metadata##1", "S2_Level-2A_Tile1_Metadata##0") + ) = { val jsonPathFull = getClass.getResource(jsonPath) val fileSource = Source.fromURL(jsonPathFull) @@ -368,7 +372,7 @@ object LayerFixtures { // Use artifactory to avoid heavy git repo val basePathArtifactory = "https://artifactory.vgt.vito.be/artifactory/testdata-public" - for (rest <- Seq( + val artifactoryPaths = Set( "/eodata/Sentinel-2/MSI/L2A/2023/01/17/S2B_MSIL2A_20230117T104259_N0509_R008_T31UGS_20230117T120337.SAFE/manifest.safe", "/eodata/Sentinel-2/MSI/L2A/2023/01/17/S2B_MSIL2A_20230117T104259_N0509_R008_T31UGS_20230117T120337.SAFE/MTD_MSIL2A.xml", "/eodata/Sentinel-2/MSI/L2A/2023/01/17/S2B_MSIL2A_20230117T104259_N0509_R008_T31UGS_20230117T120337.SAFE/GRANULE/L2A_T31UGS_A030636_20230117T104258/MTD_TL.xml", @@ -402,18 +406,43 @@ object LayerFixtures { "/eodata/Sentinel-2/MSI/L2A/2024/03/24/S2B_MSIL2A_20240324T230529_N0510_R044_T04WDD_20240324T234241.SAFE/GRANULE/L2A_T04WDD_A036821_20240324T230529/MTD_TL.xml", "/eodata/Sentinel-2/MSI/L2A/2024/03/24/S2B_MSIL2A_20240324T230529_N0510_R044_T04WDD_20240324T234241.SAFE/manifest.safe", "/eodata/Sentinel-2/MSI/L2A/2024/03/24/S2B_MSIL2A_20240324T230529_N0510_R044_T04WDD_20240324T234241.SAFE/MTD_MSIL2A.xml", - )) { - val jp2File = new File(basePath, rest) + ) + + for (path <- artifactoryPaths) { + val jp2File = new File(basePath, path) if (!jp2File.exists()) { println("Copy from artifactory to: " + jp2File) - FileUtils.copyURLToFile(new URL(basePathArtifactory + rest), jp2File) + FileUtils.copyURLToFile(new URL(basePathArtifactory + path), jp2File) + } + } + + val folders = artifactoryPaths.map(x => x.substring(0, x.indexOf(".SAFE") + ".SAFE".length)) + val matches = "/eodata/.*?.SAFE".r.findAllIn(txt).toList + for (m <- matches) { + if (folders.contains(m)) { + txt = txt.replace(m, basePath + m) } } - txt = txt.replace("\"/eodata/", "\"" + basePath + "/eodata/") + + val mockedFeatures = CreoFeatureCollection.parse(txt) val client = new MockOpenSearchFeatures(mockedFeatures.features) // val client = CreodiasClient() // More difficult to capture a nodata piece + val localFromDate = localDate + val localToDate = localDate.plusDays(1) + val ZonedFromDate = ZonedDateTime.of(localFromDate, java.time.LocalTime.MIDNIGHT, UTC) + val zonedToDate = ZonedDateTime.of(localToDate, java.time.LocalTime.MIDNIGHT, UTC) + + sentinel2CubeCDSEGeneric((ZonedFromDate, zonedToDate), projected_polygons_native_crs, client, dataCubeParameters, bandNames) + } + + def sentinel2CubeCDSEGeneric(dateInterval: (ZonedDateTime, ZonedDateTime), + projected_polygons_native_crs: ProjectedPolygons, + client: OpenSearchClient = CreodiasClient(), + dataCubeParameters: DataCubeParameters = new DataCubeParameters, + bandNames: util.List[String] = util.Arrays.asList("IMG_DATA_Band_B04_10m_Tile1_Data", "S2_Level-2A_Tile1_Metadata##1", "S2_Level-2A_Tile1_Metadata##0") + ): MultibandTileLayerRDD[SpaceTimeKey] = { val factory = new PyramidFactory( client, "Sentinel2", bandNames, null, @@ -421,17 +450,12 @@ object LayerFixtures { ) factory.crs = projected_polygons_native_crs.crs - val localFromDate = localDate - val localToDate = localDate.plusDays(1) - val ZonedFromDate = ZonedDateTime.of(localFromDate, java.time.LocalTime.MIDNIGHT, UTC) - val zonedToDate = ZonedDateTime.of(localToDate, java.time.LocalTime.MIDNIGHT, UTC) - val from_date = DateTimeFormatter.ISO_OFFSET_DATE_TIME format ZonedFromDate - val to_date = DateTimeFormatter.ISO_OFFSET_DATE_TIME format zonedToDate - + val from_date = DateTimeFormatter.ISO_OFFSET_DATE_TIME format dateInterval._1 + val to_date = DateTimeFormatter.ISO_OFFSET_DATE_TIME format dateInterval._2 val cube: Seq[(Int, MultibandTileLayerRDD[SpaceTimeKey])] = factory.datacube_seq( projected_polygons_native_crs, - from_date, to_date, Collections.emptyMap(), "",dataCubeParameters = dataCubeParameters + from_date, to_date, Collections.emptyMap(), "", dataCubeParameters = dataCubeParameters ) cube.head._2 } diff --git a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala index c3b2e905..1efb6767 100644 --- a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala +++ b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala @@ -1134,6 +1134,29 @@ class FileLayerProviderTest extends RasterMatchers{ assertEquals(8, band.get(200, 200)) } + @Test + def testMissingS2DateLine(): Unit = { + val outDir = Paths.get("tmp/FileLayerProviderTest/") + new Directory(outDir.toFile).deleteRecursively() + Files.createDirectories(outDir) + + val dateFrom = ZonedDateTime.parse("2024-04-02T00:00:00Z") + val dateTo = ZonedDateTime.parse("2024-04-03T00:00:00Z") + + val extent = Extent(178.7384, 70.769, 178.8548, 70.8254) +// val extent = Extent(178.0, 70.0, 178.99, 70.99) + val latlon = CRS.fromName("EPSG:4326") + val projected_polygons_native_crs = ProjectedPolygons.fromExtent(extent, latlon.toString()) + val utmCrs = CRS.fromName("EPSG:32601") + val reprojected = projected_polygons_native_crs.polygons.head.reproject(projected_polygons_native_crs.crs, utmCrs) + val poly2 = ProjectedPolygons(Array(reprojected), utmCrs) + + val layer = LayerFixtures.sentinel2CubeCDSEGeneric((dateFrom, dateTo), poly2, bandNames = java.util.Arrays.asList("IMG_DATA_Band_SCL_20m_Tile1_Data")) + + val cubeSpatial = layer.toSpatial() + cubeSpatial.writeGeoTiff(outDir + "/testMissingS2DateLine.tiff") + } + private def keysForLargeArea(useBBox:Boolean=false) = { val date = LocalDate.of(2022, 2, 11).atStartOfDay(UTC) val crs = CRS.fromEpsgCode(32630) From 2882a530e357cc44829516e51fe2bce5ab926b17 Mon Sep 17 00:00:00 2001 From: Emile Sonneveld Date: Tue, 4 Jun 2024 11:01:00 +0200 Subject: [PATCH 02/15] Case that better shows error. --- .../org/openeo/geotrellis/layers/FileLayerProviderTest.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala index 1efb6767..d1b89b8f 100644 --- a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala +++ b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala @@ -1147,7 +1147,8 @@ class FileLayerProviderTest extends RasterMatchers{ // val extent = Extent(178.0, 70.0, 178.99, 70.99) val latlon = CRS.fromName("EPSG:4326") val projected_polygons_native_crs = ProjectedPolygons.fromExtent(extent, latlon.toString()) - val utmCrs = CRS.fromName("EPSG:32601") +// val utmCrs = CRS.fromName("EPSG:32601") // gives good result + val utmCrs = CRS.fromName("EPSG:32660") // gives empty result val reprojected = projected_polygons_native_crs.polygons.head.reproject(projected_polygons_native_crs.crs, utmCrs) val poly2 = ProjectedPolygons(Array(reprojected), utmCrs) From 5eec0bfa4746dd6ee634c37ec0e32422ec3c9364 Mon Sep 17 00:00:00 2001 From: Emile Sonneveld Date: Mon, 10 Jun 2024 18:16:22 +0200 Subject: [PATCH 03/15] Avoid north pole impricision when reprojecting between LatLng and UTM. Needs more testing. https://github.com/Open-EO/openeo-geotrellis-extensions/issues/279 --- .../geotrellis/layers/FileLayerProvider.scala | 10 ++++---- .../layers/FileLayerProviderTest.scala | 24 +++++++++++-------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala b/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala index 28005fa1..e9febc48 100644 --- a/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala +++ b/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala @@ -1243,11 +1243,13 @@ class FileLayerProvider private(openSearch: OpenSearchClient, openSearchCollecti * - if feature is in utm, target extent may be invalid in feature crs * this is why we take intersection */ - val targetExtentInLatLon = targetExtent.reproject(feature.crs.get) - val featureExtentInLatLon = feature.rasterExtent.get.reproject(feature.crs.get,LatLng) + val commonCRS = targetExtent.crs + val targetExtentInCommonCRS = targetExtent.reproject(commonCRS) + val featureExtentInCommonCRS = feature.rasterExtent.get.reproject(feature.crs.get, commonCRS) - val intersection = featureExtentInLatLon.intersection(targetExtentInLatLon).map(_.buffer(1.0)).getOrElse(featureExtentInLatLon) - val tmp = expandToCellSize(intersection.reproject(LatLng, targetExtent.crs), theResolution) + val intersection = featureExtentInCommonCRS.intersection(targetExtentInCommonCRS).map(_.buffer(1.0)) + .getOrElse(featureExtentInCommonCRS) + val tmp = expandToCellSize(intersection.reproject(commonCRS, targetExtent.crs), theResolution) val alignedToTargetExtent = re.createAlignedRasterExtent(tmp) Some(alignedToTargetExtent.toGridType[Long]) diff --git a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala index d1b89b8f..648fdeba 100644 --- a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala +++ b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala @@ -1134,28 +1134,32 @@ class FileLayerProviderTest extends RasterMatchers{ assertEquals(8, band.get(200, 200)) } - @Test - def testMissingS2DateLine(): Unit = { - val outDir = Paths.get("tmp/FileLayerProviderTest/") - new Directory(outDir.toFile).deleteRecursively() + @ParameterizedTest + @ValueSource(strings = Array("EPSG:32601", "EPSG:32660")) + def testMissingS2DateLine(crsName: String): Unit = { + val outDir = Paths.get("tmp/FileLayerProviderTest_" + crsName.replace(":", "_") + "/") + new Directory(outDir.toFile).deepFiles.foreach(_.delete()) Files.createDirectories(outDir) val dateFrom = ZonedDateTime.parse("2024-04-02T00:00:00Z") val dateTo = ZonedDateTime.parse("2024-04-03T00:00:00Z") val extent = Extent(178.7384, 70.769, 178.8548, 70.8254) -// val extent = Extent(178.0, 70.0, 178.99, 70.99) - val latlon = CRS.fromName("EPSG:4326") - val projected_polygons_native_crs = ProjectedPolygons.fromExtent(extent, latlon.toString()) -// val utmCrs = CRS.fromName("EPSG:32601") // gives good result - val utmCrs = CRS.fromName("EPSG:32660") // gives empty result + val projected_polygons_native_crs = ProjectedPolygons.fromExtent(extent, LatLng.proj4jCrs.toString) + val utmCrs = CRS.fromName(crsName) val reprojected = projected_polygons_native_crs.polygons.head.reproject(projected_polygons_native_crs.crs, utmCrs) val poly2 = ProjectedPolygons(Array(reprojected), utmCrs) val layer = LayerFixtures.sentinel2CubeCDSEGeneric((dateFrom, dateTo), poly2, bandNames = java.util.Arrays.asList("IMG_DATA_Band_SCL_20m_Tile1_Data")) + val layer_collected = layer.collect() + assert(layer_collected.nonEmpty) + for { + (_, multiBandTile) <- layer_collected + tile <- multiBandTile.bands + } assert(!tile.isNoDataTile) val cubeSpatial = layer.toSpatial() - cubeSpatial.writeGeoTiff(outDir + "/testMissingS2DateLine.tiff") + cubeSpatial.writeGeoTiff(outDir + "/testMissingS2DateLine_" + crsName.replace(":", "_") + ".tiff") } private def keysForLargeArea(useBBox:Boolean=false) = { From 059eb58668e833410345e84bb1864f0bcb2f0261 Mon Sep 17 00:00:00 2001 From: Emile Sonneveld Date: Tue, 11 Jun 2024 15:32:40 +0200 Subject: [PATCH 04/15] Avoid RATE_LIMIT_EXCEEDED. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 90804d68..f8a33caf 100644 --- a/pom.xml +++ b/pom.xml @@ -52,7 +52,7 @@ geotrellis-accumulo-extensions geotrellis-s3-extensions - geotrellis-sentinelhub + geotrellis-extensions geotrellis-seeder openeo-geotrellis From 93c2b754264bddd80fba69c734352b73d0aadfb9 Mon Sep 17 00:00:00 2001 From: Emile Sonneveld Date: Fri, 28 Jun 2024 13:29:55 +0200 Subject: [PATCH 05/15] Fix to make tests run on Jenkins. https://github.com/Open-EO/openeo-geotrellis-extensions/issues/279 --- .../geotrellis/layers/FileLayerProvider.scala | 10 +- .../geotrellis/testMissingS2DateLine.json | 569 ++++++++++++++++++ .../org/openeo/geotrellis/LayerFixtures.scala | 31 + .../layers/FileLayerProviderTest.scala | 18 +- pom.xml | 2 +- 5 files changed, 617 insertions(+), 13 deletions(-) create mode 100644 openeo-geotrellis/src/test/resources/org/openeo/geotrellis/testMissingS2DateLine.json diff --git a/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala b/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala index e9febc48..9ab0332d 100644 --- a/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala +++ b/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala @@ -1243,13 +1243,11 @@ class FileLayerProvider private(openSearch: OpenSearchClient, openSearchCollecti * - if feature is in utm, target extent may be invalid in feature crs * this is why we take intersection */ - val commonCRS = targetExtent.crs - val targetExtentInCommonCRS = targetExtent.reproject(commonCRS) - val featureExtentInCommonCRS = feature.rasterExtent.get.reproject(feature.crs.get, commonCRS) + val featureExtentInTargetCRS = feature.rasterExtent.get.reproject(feature.crs.get, targetExtent.crs) - val intersection = featureExtentInCommonCRS.intersection(targetExtentInCommonCRS).map(_.buffer(1.0)) - .getOrElse(featureExtentInCommonCRS) - val tmp = expandToCellSize(intersection.reproject(commonCRS, targetExtent.crs), theResolution) + val intersection = featureExtentInTargetCRS.intersection(targetExtent.extent).map(_.buffer(1.0)) + .getOrElse(featureExtentInTargetCRS) + val tmp = expandToCellSize(intersection, theResolution) val alignedToTargetExtent = re.createAlignedRasterExtent(tmp) Some(alignedToTargetExtent.toGridType[Long]) diff --git a/openeo-geotrellis/src/test/resources/org/openeo/geotrellis/testMissingS2DateLine.json b/openeo-geotrellis/src/test/resources/org/openeo/geotrellis/testMissingS2DateLine.json new file mode 100644 index 00000000..5e72a6ca --- /dev/null +++ b/openeo-geotrellis/src/test/resources/org/openeo/geotrellis/testMissingS2DateLine.json @@ -0,0 +1,569 @@ +{ + "type": "FeatureCollection", + "properties": { + "id": "7c6f676b-ea06-5f51-bd17-70a38f7ff1d2", + "totalResults": null, + "exactCount": 0, + "startIndex": 1, + "itemsPerPage": 1000, + "query": { + "originalFilters": { + "box": "178.72633867329634,70.76628200377719,178.866724486903,70.82813491803448", + "status": "ONLINE", + "dataset": "ESA-DATASET", + "startDate": "2024-04-02T00:00:00Z", + "completionDate": "2024-04-03T00:00:00Z", + "collection": "SENTINEL-2" + }, + "appliedFilters": { + "box": "178.72633867329634,70.76628200377719,178.866724486903,70.82813491803448", + "status": "ONLINE", + "dataset": "ESA-DATASET", + "startDate": "2024-04-02T00:00:00Z", + "completionDate": "2024-04-03T00:00:00Z", + "collection": "SENTINEL-2" + }, + "processingTime": 0.118192746 + }, + "links": [ + { + "rel": "self", + "type": "application/json", + "title": "self", + "href": "https://catalogue.dataspace.copernicus.eu/resto/api/collections/Sentinel2/search.json?box=178.72633867329634%2C70.76628200377719%2C178.866724486903%2C70.82813491803448&page=1&maxRecords=1000&status=ONLINE&dataset=ESA-DATASET&startDate=2024-04-02T00%3A00%3A00Z&completionDate=2024-04-03T00%3A00%3A00Z" + }, + { + "rel": "search", + "type": "application/opensearchdescription+xml", + "title": "OpenSearch Description Document", + "href": "https://catalogue.dataspace.copernicus.eu/resto/api/collections/Sentinel2/describe.xml" + } + ] + }, + "features": [ + { + "type": "Feature", + "id": "a6cdb338-1ab4-447b-abdd-78cde56927a4", + "geometry": { + "type": "MultiPolygon", + "coordinates": [ + [ + [ + [ + 180, + 70.8418254254962 + ], + [ + 179.9040632080704, + 70.19484288998889 + ], + [ + 176.9994705028295, + 70.21832076563753 + ], + [ + 176.9994518989893, + 70.90607981912203 + ], + [ + 177.0133971094673, + 70.91645944454088 + ], + [ + 177.199839761927, + 71.05302740049166 + ], + [ + 177.2061930151708, + 71.05760818625669 + ], + [ + 177.3903933490624, + 71.19021030698168 + ], + [ + 177.4035927539655, + 71.19955639631998 + ], + [ + 180, + 71.17844481285532 + ], + [ + 180, + 70.8418254254962 + ] + ] + ], + [ + [ + [ + -180, + 71.17844481285532 + ], + [ + -179.95015, + 71.1780394383418 + ], + [ + -180, + 70.8418254254962 + ], + [ + -180, + 71.17844481285532 + ] + ] + ] + ] + }, + "properties": { + "collection": "SENTINEL-2", + "status": "ONLINE", + "license": { + "licenseId": "unlicensed", + "hasToBeSigned": "never", + "grantedCountries": null, + "grantedOrganizationCountries": null, + "grantedFlags": null, + "viewService": "public", + "signatureQuota": -1, + "description": { + "shortName": "No license" + } + }, + "parentIdentifier": null, + "title": "S2B_MSIL1C_20240402T000609_N0510_R016_T60WWD_20240402T001958.SAFE", + "description": "The Copernicus Sentinel-2 mission consists of two polar-orbiting satellites that are positioned in the same sun-synchronous orbit, with a phase difference of 180. It aims to monitor changes in land surface conditions. The satellites have a wide swath width (290 km) and a high revisit time. Sentinel-2 is equipped with an optical instrument payload that samples 13 spectral bands: four bands at 10 m, six bands at 20 m and three bands at 60 m spatial resolution [https://dataspace.copernicus.eu/explore-data/data-collections/sentinel-data/sentinel-2].", + "organisationName": null, + "startDate": "2024-04-02T00:06:09.024Z", + "completionDate": "2024-04-02T00:06:09.024Z", + "productType": "S2MSI1C", + "processingLevel": "S2MSI1C", + "platform": "S2B", + "instrument": "MSI", + "resolution": 0, + "sensorMode": "INS-NOBS", + "orbitNumber": 36936, + "quicklook": null, + "thumbnail": "https://catalogue.dataspace.copernicus.eu/get-object?path=/Sentinel-2/MSI/L1C/2024/04/02/S2B_MSIL1C_20240402T000609_N0510_R016_T60WWD_20240402T001958.SAFE/S2B_MSIL1C_20240402T000609_N0510_R016_T60WWD_20240402T001958-ql.jpg", + "updated": "2024-04-02T01:07:35.742Z", + "published": "2024-04-02T01:05:49.640Z", + "snowCover": 0, + "cloudCover": 79.515270411362, + "gmlgeometry": "180.0,70.8418254254962 179.90406320807,70.1948428899889 176.999470502829,70.2183207656375 176.999451898989,70.906079819122 177.013397109467,70.9164594445409 177.199839761927,71.0530274004917 177.206193015171,71.0576081862567 177.390393349062,71.1902103069817 177.403592753966,71.19955639632 180.0,71.1784448128553 180.0,70.8418254254962-180,71.1784448128553 -179.95015,71.1780394383418 -180,70.8418254254962 -180,71.1784448128553", + "centroid": { + "type": "Point", + "coordinates": [ + 177.464743426394, + 70.6942065367668 + ] + }, + "productIdentifier": "/eodata/Sentinel-2/MSI/L1C/2024/04/02/S2B_MSIL1C_20240402T000609_N0510_R016_T60WWD_20240402T001958.SAFE", + "orbitDirection": null, + "timeliness": null, + "relativeOrbitNumber": 16, + "processingBaseline": 5.1, + "missionTakeId": "GS2B_20240402T000609_036936_N05.10", + "services": { + "download": { + "url": "https://catalogue.dataspace.copernicus.eu/download/a6cdb338-1ab4-447b-abdd-78cde56927a4", + "mimeType": "application/octet-stream", + "size": 748892301 + } + }, + "links": [ + { + "rel": "self", + "type": "application/json", + "title": "GeoJSON link for a6cdb338-1ab4-447b-abdd-78cde56927a4", + "href": "https://catalogue.dataspace.copernicus.eu/resto/collections/SENTINEL-2/a6cdb338-1ab4-447b-abdd-78cde56927a4.json" + } + ] + } + }, + { + "type": "Feature", + "id": "4fe42670-54fb-4e66-8ea5-f13ed74c394d", + "geometry": { + "type": "MultiPolygon", + "coordinates": [ + [ + [ + [ + -180, + 71.1754685740731 + ], + [ + -179.50662, + 71.18609391597852 + ], + [ + -179.38673, + 70.20246735196007 + ], + [ + -180, + 70.1893406780604 + ], + [ + -180, + 71.1754685740731 + ] + ] + ], + [ + [ + [ + 180, + 70.1893406780604 + ], + [ + 177.7191700376, + 70.14052089926992 + ], + [ + 177.4551847198166, + 71.12066368510631 + ], + [ + 180, + 71.1754685740731 + ], + [ + 180, + 70.1893406780604 + ] + ] + ] + ] + }, + "properties": { + "collection": "SENTINEL-2", + "status": "ONLINE", + "license": { + "licenseId": "unlicensed", + "hasToBeSigned": "never", + "grantedCountries": null, + "grantedOrganizationCountries": null, + "grantedFlags": null, + "viewService": "public", + "signatureQuota": -1, + "description": { + "shortName": "No license" + } + }, + "parentIdentifier": null, + "title": "S2B_MSIL1C_20240402T000609_N0510_R016_T01WCU_20240402T001958.SAFE", + "description": "The Copernicus Sentinel-2 mission consists of two polar-orbiting satellites that are positioned in the same sun-synchronous orbit, with a phase difference of 180. It aims to monitor changes in land surface conditions. The satellites have a wide swath width (290 km) and a high revisit time. Sentinel-2 is equipped with an optical instrument payload that samples 13 spectral bands: four bands at 10 m, six bands at 20 m and three bands at 60 m spatial resolution [https://dataspace.copernicus.eu/explore-data/data-collections/sentinel-data/sentinel-2].", + "organisationName": null, + "startDate": "2024-04-02T00:06:09.024Z", + "completionDate": "2024-04-02T00:06:09.024Z", + "productType": "S2MSI1C", + "processingLevel": "S2MSI1C", + "platform": "S2B", + "instrument": "MSI", + "resolution": 0, + "sensorMode": "INS-NOBS", + "orbitNumber": 36936, + "quicklook": null, + "thumbnail": "https://catalogue.dataspace.copernicus.eu/get-object?path=/Sentinel-2/MSI/L1C/2024/04/02/S2B_MSIL1C_20240402T000609_N0510_R016_T01WCU_20240402T001958.SAFE/S2B_MSIL1C_20240402T000609_N0510_R016_T01WCU_20240402T001958-ql.jpg", + "updated": "2024-04-02T01:09:53.787Z", + "published": "2024-04-02T01:08:08.468Z", + "snowCover": 0, + "cloudCover": 84.510346681, + "gmlgeometry": "-180,71.1754685740731 -179.50662,71.1860939159785 -179.38673,70.2024673519601 -180,70.1893406780604 -180,71.1754685740731180.0,70.1893406780604 177.7191700376,70.1405208992699 177.455184719817,71.1206636851063 180.0,71.1754685740731 180.0,70.1893406780604", + "centroid": { + "type": "Point", + "coordinates": [ + 111.902011001716, + 70.6664156555398 + ] + }, + "productIdentifier": "/eodata/Sentinel-2/MSI/L1C/2024/04/02/S2B_MSIL1C_20240402T000609_N0510_R016_T01WCU_20240402T001958.SAFE", + "orbitDirection": null, + "timeliness": null, + "relativeOrbitNumber": 16, + "processingBaseline": 5.1, + "missionTakeId": "GS2B_20240402T000609_036936_N05.10", + "services": { + "download": { + "url": "https://catalogue.dataspace.copernicus.eu/download/4fe42670-54fb-4e66-8ea5-f13ed74c394d", + "mimeType": "application/octet-stream", + "size": 762837400 + } + }, + "links": [ + { + "rel": "self", + "type": "application/json", + "title": "GeoJSON link for 4fe42670-54fb-4e66-8ea5-f13ed74c394d", + "href": "https://catalogue.dataspace.copernicus.eu/resto/collections/SENTINEL-2/4fe42670-54fb-4e66-8ea5-f13ed74c394d.json" + } + ] + } + }, + { + "type": "Feature", + "id": "b25acc88-80ce-49b7-b625-6ac6688275a5", + "geometry": { + "type": "MultiPolygon", + "coordinates": [ + [ + [ + [ + 180, + 70.8418254254962 + ], + [ + 179.9040632080704, + 70.19484288998889 + ], + [ + 176.9994705028295, + 70.21832076563753 + ], + [ + 176.9994518989893, + 70.90607981912203 + ], + [ + 177.0133971094673, + 70.91645944454088 + ], + [ + 177.199839761927, + 71.05302740049166 + ], + [ + 177.2061930151708, + 71.05760818625669 + ], + [ + 177.3903933490624, + 71.19021030698168 + ], + [ + 177.4035927539655, + 71.19955639631998 + ], + [ + 180, + 71.17844481285532 + ], + [ + 180, + 70.8418254254962 + ] + ] + ], + [ + [ + [ + -180, + 71.17844481285532 + ], + [ + -179.95015, + 71.1780394383418 + ], + [ + -180, + 70.8418254254962 + ], + [ + -180, + 71.17844481285532 + ] + ] + ] + ] + }, + "properties": { + "collection": "SENTINEL-2", + "status": "ONLINE", + "license": { + "licenseId": "unlicensed", + "hasToBeSigned": "never", + "grantedCountries": null, + "grantedOrganizationCountries": null, + "grantedFlags": null, + "viewService": "public", + "signatureQuota": -1, + "description": { + "shortName": "No license" + } + }, + "parentIdentifier": null, + "title": "S2B_MSIL2A_20240402T000609_N0510_R016_T60WWD_20240402T003652.SAFE", + "description": "The Copernicus Sentinel-2 mission consists of two polar-orbiting satellites that are positioned in the same sun-synchronous orbit, with a phase difference of 180. It aims to monitor changes in land surface conditions. The satellites have a wide swath width (290 km) and a high revisit time. Sentinel-2 is equipped with an optical instrument payload that samples 13 spectral bands: four bands at 10 m, six bands at 20 m and three bands at 60 m spatial resolution [https://dataspace.copernicus.eu/explore-data/data-collections/sentinel-data/sentinel-2].", + "organisationName": null, + "startDate": "2024-04-02T00:06:09.024Z", + "completionDate": "2024-04-02T00:06:09.024Z", + "productType": "S2MSI2A", + "processingLevel": "S2MSI2A", + "platform": "S2B", + "instrument": "MSI", + "resolution": 0, + "sensorMode": "INS-NOBS", + "orbitNumber": 36936, + "quicklook": null, + "thumbnail": "https://catalogue.dataspace.copernicus.eu/get-object?path=/Sentinel-2/MSI/L2A/2024/04/02/S2B_MSIL2A_20240402T000609_N0510_R016_T60WWD_20240402T003652.SAFE/S2B_MSIL2A_20240402T000609_N0510_R016_T60WWD_20240402T003652-ql.jpg", + "updated": "2024-04-02T01:53:23.144Z", + "published": "2024-04-02T01:52:55.925Z", + "snowCover": 0, + "cloudCover": 82.784283, + "gmlgeometry": "180.0,70.8418254254962 179.90406320807,70.1948428899889 176.999470502829,70.2183207656375 176.999451898989,70.906079819122 177.013397109467,70.9164594445409 177.199839761927,71.0530274004917 177.206193015171,71.0576081862567 177.390393349062,71.1902103069817 177.403592753966,71.19955639632 180.0,71.1784448128553 180.0,70.8418254254962-180,71.1784448128553 -179.95015,71.1780394383418 -180,70.8418254254962 -180,71.1784448128553", + "centroid": { + "type": "Point", + "coordinates": [ + 177.464743426394, + 70.6942065367668 + ] + }, + "productIdentifier": "/eodata/Sentinel-2/MSI/L2A/2024/04/02/S2B_MSIL2A_20240402T000609_N0510_R016_T60WWD_20240402T003652.SAFE", + "orbitDirection": null, + "timeliness": null, + "relativeOrbitNumber": 16, + "processingBaseline": 5.1, + "missionTakeId": "GS2B_20240402T000609_036936_N05.10", + "services": { + "download": { + "url": "https://catalogue.dataspace.copernicus.eu/download/b25acc88-80ce-49b7-b625-6ac6688275a5", + "mimeType": "application/octet-stream", + "size": 940045162 + } + }, + "links": [ + { + "rel": "self", + "type": "application/json", + "title": "GeoJSON link for b25acc88-80ce-49b7-b625-6ac6688275a5", + "href": "https://catalogue.dataspace.copernicus.eu/resto/collections/SENTINEL-2/b25acc88-80ce-49b7-b625-6ac6688275a5.json" + } + ] + } + }, + { + "type": "Feature", + "id": "c164d927-d2a6-4a4f-a432-8f5b451b5f06", + "geometry": { + "type": "MultiPolygon", + "coordinates": [ + [ + [ + [ + -180, + 71.1754685740731 + ], + [ + -179.50662, + 71.18609391597852 + ], + [ + -179.38673, + 70.20246735196007 + ], + [ + -180, + 70.1893406780604 + ], + [ + -180, + 71.1754685740731 + ] + ] + ], + [ + [ + [ + 180, + 70.1893406780604 + ], + [ + 177.7191700376, + 70.14052089926992 + ], + [ + 177.4551847198166, + 71.12066368510631 + ], + [ + 180, + 71.1754685740731 + ], + [ + 180, + 70.1893406780604 + ] + ] + ] + ] + }, + "properties": { + "collection": "SENTINEL-2", + "status": "ONLINE", + "license": { + "licenseId": "unlicensed", + "hasToBeSigned": "never", + "grantedCountries": null, + "grantedOrganizationCountries": null, + "grantedFlags": null, + "viewService": "public", + "signatureQuota": -1, + "description": { + "shortName": "No license" + } + }, + "parentIdentifier": null, + "title": "S2B_MSIL2A_20240402T000609_N0510_R016_T01WCU_20240402T003652.SAFE", + "description": "The Copernicus Sentinel-2 mission consists of two polar-orbiting satellites that are positioned in the same sun-synchronous orbit, with a phase difference of 180. It aims to monitor changes in land surface conditions. The satellites have a wide swath width (290 km) and a high revisit time. Sentinel-2 is equipped with an optical instrument payload that samples 13 spectral bands: four bands at 10 m, six bands at 20 m and three bands at 60 m spatial resolution [https://dataspace.copernicus.eu/explore-data/data-collections/sentinel-data/sentinel-2].", + "organisationName": null, + "startDate": "2024-04-02T00:06:09.024Z", + "completionDate": "2024-04-02T00:06:09.024Z", + "productType": "S2MSI2A", + "processingLevel": "S2MSI2A", + "platform": "S2B", + "instrument": "MSI", + "resolution": 0, + "sensorMode": "INS-NOBS", + "orbitNumber": 36936, + "quicklook": null, + "thumbnail": "https://catalogue.dataspace.copernicus.eu/get-object?path=/Sentinel-2/MSI/L2A/2024/04/02/S2B_MSIL2A_20240402T000609_N0510_R016_T01WCU_20240402T003652.SAFE/S2B_MSIL2A_20240402T000609_N0510_R016_T01WCU_20240402T003652-ql.jpg", + "updated": "2024-04-02T01:53:17.222Z", + "published": "2024-04-02T01:52:50.903Z", + "snowCover": 0, + "cloudCover": 88.21907, + "gmlgeometry": "-180,71.1754685740731 -179.50662,71.1860939159785 -179.38673,70.2024673519601 -180,70.1893406780604 -180,71.1754685740731180.0,70.1893406780604 177.7191700376,70.1405208992699 177.455184719817,71.1206636851063 180.0,71.1754685740731 180.0,70.1893406780604", + "centroid": { + "type": "Point", + "coordinates": [ + 111.902011001716, + 70.6664156555398 + ] + }, + "productIdentifier": "/eodata/Sentinel-2/MSI/L2A/2024/04/02/S2B_MSIL2A_20240402T000609_N0510_R016_T01WCU_20240402T003652.SAFE", + "orbitDirection": null, + "timeliness": null, + "relativeOrbitNumber": 16, + "processingBaseline": 5.1, + "missionTakeId": "GS2B_20240402T000609_036936_N05.10", + "services": { + "download": { + "url": "https://catalogue.dataspace.copernicus.eu/download/c164d927-d2a6-4a4f-a432-8f5b451b5f06", + "mimeType": "application/octet-stream", + "size": 938587592 + } + }, + "links": [ + { + "rel": "self", + "type": "application/json", + "title": "GeoJSON link for c164d927-d2a6-4a4f-a432-8f5b451b5f06", + "href": "https://catalogue.dataspace.copernicus.eu/resto/collections/SENTINEL-2/c164d927-d2a6-4a4f-a432-8f5b451b5f06.json" + } + ] + } + } + ] +} \ No newline at end of file diff --git a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/LayerFixtures.scala b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/LayerFixtures.scala index 53738852..a38546f2 100644 --- a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/LayerFixtures.scala +++ b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/LayerFixtures.scala @@ -372,6 +372,26 @@ object LayerFixtures { // Use artifactory to avoid heavy git repo val basePathArtifactory = "https://artifactory.vgt.vito.be/artifactory/testdata-public" + /* + To upload new files; + - mount /eodata + - change openeo-opensearch-client to use fs instead of s3 + - run this Python script, and copy the printed paths here: +root = Path("/tmp/EODATA/") +l = set(filter(lambda p: p.is_file(), root.rglob("*.*"))) +l = {f for f in l if os.stat(f).st_size > 0} +l = set(map(lambda p: os.path.relpath(p, root), l)) +for p in l: + cmd = f'curl -uUSERNAME:PASS -T {root / p} "https://artifactory.vgt.vito.be/artifactory/testdata-public/eodata/{p}"' + print(cmd) + code = os.system(cmd) + if code != 0: + raise Exception("Failed: " + cmd) +print("\n") +for p in l: + print(f'"/eodata/{p}",') + */ + val artifactoryPaths = Set( "/eodata/Sentinel-2/MSI/L2A/2023/01/17/S2B_MSIL2A_20230117T104259_N0509_R008_T31UGS_20230117T120337.SAFE/manifest.safe", "/eodata/Sentinel-2/MSI/L2A/2023/01/17/S2B_MSIL2A_20230117T104259_N0509_R008_T31UGS_20230117T120337.SAFE/MTD_MSIL2A.xml", @@ -406,6 +426,17 @@ object LayerFixtures { "/eodata/Sentinel-2/MSI/L2A/2024/03/24/S2B_MSIL2A_20240324T230529_N0510_R044_T04WDD_20240324T234241.SAFE/GRANULE/L2A_T04WDD_A036821_20240324T230529/MTD_TL.xml", "/eodata/Sentinel-2/MSI/L2A/2024/03/24/S2B_MSIL2A_20240324T230529_N0510_R044_T04WDD_20240324T234241.SAFE/manifest.safe", "/eodata/Sentinel-2/MSI/L2A/2024/03/24/S2B_MSIL2A_20240324T230529_N0510_R044_T04WDD_20240324T234241.SAFE/MTD_MSIL2A.xml", + // testMissingS2DateLine + "/eodata/Sentinel-2/MSI/L2A/2024/04/02/S2B_MSIL2A_20240402T000609_N0510_R016_T01WCU_20240402T003652.SAFE/manifest.safe", + "/eodata/Sentinel-2/MSI/L2A/2024/04/02/S2B_MSIL2A_20240402T000609_N0510_R016_T01WCU_20240402T003652.SAFE/MTD_MSIL2A.xml", + "/eodata/Sentinel-2/MSI/L2A/2024/04/02/S2B_MSIL2A_20240402T000609_N0510_R016_T60WWD_20240402T003652.SAFE/MTD_MSIL2A.xml", + "/eodata/Sentinel-2/MSI/L1C/2024/04/02/S2B_MSIL1C_20240402T000609_N0510_R016_T01WCU_20240402T001958.SAFE/manifest.safe", + "/eodata/Sentinel-2/MSI/L1C/2024/04/02/S2B_MSIL1C_20240402T000609_N0510_R016_T01WCU_20240402T001958.SAFE/MTD_MSIL1C.xml", + "/eodata/Sentinel-2/MSI/L2A/2024/04/02/S2B_MSIL2A_20240402T000609_N0510_R016_T60WWD_20240402T003652.SAFE/GRANULE/L2A_T60WWD_A036936_20240402T000609/IMG_DATA/R20m/T60WWD_20240402T000609_SCL_20m.jp2", + "/eodata/Sentinel-2/MSI/L1C/2024/04/02/S2B_MSIL1C_20240402T000609_N0510_R016_T60WWD_20240402T001958.SAFE/MTD_MSIL1C.xml", + "/eodata/Sentinel-2/MSI/L1C/2024/04/02/S2B_MSIL1C_20240402T000609_N0510_R016_T60WWD_20240402T001958.SAFE/manifest.safe", + "/eodata/Sentinel-2/MSI/L2A/2024/04/02/S2B_MSIL2A_20240402T000609_N0510_R016_T01WCU_20240402T003652.SAFE/GRANULE/L2A_T01WCU_A036936_20240402T000609/IMG_DATA/R20m/T01WCU_20240402T000609_SCL_20m.jp2", + "/eodata/Sentinel-2/MSI/L2A/2024/04/02/S2B_MSIL2A_20240402T000609_N0510_R016_T60WWD_20240402T003652.SAFE/manifest.safe", ) for (path <- artifactoryPaths) { diff --git a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala index 648fdeba..942186ac 100644 --- a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala +++ b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala @@ -1076,7 +1076,7 @@ class FileLayerProviderTest extends RasterMatchers{ cubeSpatial.writeGeoTiff("tmp/testPixelValueOffsetNeededCorner.tiff") val arr = cubeSpatial.collect().array assertTrue(isNoData(arr(1)._2.toArrayTile().band(0).get(162, 250))) - assertEquals(172, arr(0)._2.toArrayTile().band(0).get(5, 5), 1) + assertEquals(187, arr(0)._2.toArrayTile().band(0).get(160, 5), 1) } @Test @@ -1091,7 +1091,7 @@ class FileLayerProviderTest extends RasterMatchers{ cubeSpatial.writeGeoTiff("tmp/testPixelValueOffsetNeededDark.tiff") val band = cubeSpatial.collect().array(0)._2.toArrayTile().band(0) - assertEquals(888, band.get(0, 0), 1) + assertEquals(682, band.get(20, 140), 1) assertEquals(-582, band.get(133, 151), 1) } @@ -1141,16 +1141,22 @@ class FileLayerProviderTest extends RasterMatchers{ new Directory(outDir.toFile).deepFiles.foreach(_.delete()) Files.createDirectories(outDir) - val dateFrom = ZonedDateTime.parse("2024-04-02T00:00:00Z") - val dateTo = ZonedDateTime.parse("2024-04-03T00:00:00Z") - val extent = Extent(178.7384, 70.769, 178.8548, 70.8254) val projected_polygons_native_crs = ProjectedPolygons.fromExtent(extent, LatLng.proj4jCrs.toString) val utmCrs = CRS.fromName(crsName) val reprojected = projected_polygons_native_crs.polygons.head.reproject(projected_polygons_native_crs.crs, utmCrs) val poly2 = ProjectedPolygons(Array(reprojected), utmCrs) - val layer = LayerFixtures.sentinel2CubeCDSEGeneric((dateFrom, dateTo), poly2, bandNames = java.util.Arrays.asList("IMG_DATA_Band_SCL_20m_Tile1_Data")) + val jsonPath = "/org/openeo/geotrellis/testMissingS2DateLine.json" + + + val bandNames = java.util.Arrays.asList("IMG_DATA_Band_SCL_20m_Tile1_Data") + + val layer = LayerFixtures.sentinel2Cube( + LocalDate.of(2024, 4, 2), poly2, jsonPath, + new DataCubeParameters, + bandNames, + ) val layer_collected = layer.collect() assert(layer_collected.nonEmpty) diff --git a/pom.xml b/pom.xml index f8a33caf..90804d68 100644 --- a/pom.xml +++ b/pom.xml @@ -52,7 +52,7 @@ geotrellis-accumulo-extensions geotrellis-s3-extensions - + geotrellis-sentinelhub geotrellis-extensions geotrellis-seeder openeo-geotrellis From 14cfc609d5da9178902c03c48b93dff796d4fc59 Mon Sep 17 00:00:00 2001 From: Emile Sonneveld Date: Fri, 28 Jun 2024 14:13:16 +0200 Subject: [PATCH 06/15] formatting --- .../geotrellis/layers/FileLayerProviderTest.scala | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala index 942186ac..3f133ff4 100644 --- a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala +++ b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala @@ -1147,15 +1147,12 @@ class FileLayerProviderTest extends RasterMatchers{ val reprojected = projected_polygons_native_crs.polygons.head.reproject(projected_polygons_native_crs.crs, utmCrs) val poly2 = ProjectedPolygons(Array(reprojected), utmCrs) - val jsonPath = "/org/openeo/geotrellis/testMissingS2DateLine.json" - - - val bandNames = java.util.Arrays.asList("IMG_DATA_Band_SCL_20m_Tile1_Data") - val layer = LayerFixtures.sentinel2Cube( - LocalDate.of(2024, 4, 2), poly2, jsonPath, + LocalDate.of(2024, 4, 2), + poly2, + "/org/openeo/geotrellis/testMissingS2DateLine.json", new DataCubeParameters, - bandNames, + java.util.Arrays.asList("IMG_DATA_Band_SCL_20m_Tile1_Data"), ) val layer_collected = layer.collect() From 223b5797b9f62584c4a144c4d6229a7fd14f44ef Mon Sep 17 00:00:00 2001 From: Emile Sonneveld Date: Tue, 23 Jul 2024 16:24:58 +0200 Subject: [PATCH 07/15] Assert sensible extends in UTM zones. https://github.com/Open-EO/openeo-geotrellis-extensions/issues/279 --- .../org/openeo/geotrellis/layers/FileLayerProvider.scala | 6 ++++++ .../openeo/geotrellis/layers/FileLayerProviderTest.scala | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala b/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala index 9ab0332d..259e65c3 100644 --- a/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala +++ b/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala @@ -886,6 +886,12 @@ class FileLayerProvider private(openSearch: OpenSearchClient, openSearchCollecti val reprojectedBoundingBox: ProjectedExtent = DatacubeSupport.targetBoundingBox(fullBBox, layoutScheme) val alignedExtent = worldLayout.createAlignedRasterExtent(reprojectedBoundingBox.extent) + val polygonIsUTM = polygons_crs.proj4jCrs.getProjection.getName == "utm" + if (polygonIsUTM) { + // This is an extend that has the highest sensible values for northern and/or southern hemisphere UTM zones + val utmProjectedBounds = Extent(166021.44, 0000000.00, 833978.56, 10000000) + assert(alignedExtent.extent.intersects(utmProjectedBounds), "Extend should be in valid values of UTM zone to avoid distortion when reprojecting.") + } logger.info(s"Loading ${openSearchCollectionId} with params ${datacubeParams.getOrElse(new DataCubeParameters)} and bands ${openSearchLinkTitles.toList.mkString(";")} initial layout: ${worldLayout}") diff --git a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala index 3f133ff4..952714c2 100644 --- a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala +++ b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala @@ -1165,6 +1165,11 @@ class FileLayerProviderTest extends RasterMatchers{ cubeSpatial.writeGeoTiff(outDir + "/testMissingS2DateLine_" + crsName.replace(":", "_") + ".tiff") } + @Test + def testMissingS2DateLineOutside(): Unit = { + assertThrows[AssertionError](testMissingS2DateLine("EPSG:32631")) + } + private def keysForLargeArea(useBBox:Boolean=false) = { val date = LocalDate.of(2022, 2, 11).atStartOfDay(UTC) val crs = CRS.fromEpsgCode(32630) From 7b580ec828c463878055b2b3b8ac1a15b1a64ef9 Mon Sep 17 00:00:00 2001 From: Emile Sonneveld Date: Wed, 24 Jul 2024 19:34:21 +0200 Subject: [PATCH 08/15] Better choose common CRS. Add test for UTM->LatLng and LatLng->UTM. Add health check for ProjectedExtends. https://github.com/Open-EO/openeo-geotrellis-extensions/issues/279 --- .../geotrellis/layers/FileLayerProvider.scala | 50 +++++++++++++---- .../readDataCubeWithOpensearchClientUTM.tif | Bin 0 -> 586 bytes .../org/openeo/geotrellis/LayerFixtures.scala | 3 +- .../layers/FileLayerProviderTest.scala | 1 + .../GlobalNetCdfFileLayerProviderTest.scala | 52 ++++++++++++++++-- 5 files changed, 87 insertions(+), 19 deletions(-) create mode 100644 openeo-geotrellis/src/test/resources/org/openeo/geotrellis/GlobalNetCdfFileLayerProviderTest/readDataCubeWithOpensearchClientUTM.tif diff --git a/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala b/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala index 259e65c3..72ab6856 100644 --- a/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala +++ b/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala @@ -804,6 +804,25 @@ object FileLayerProvider { Some(bbox, dates) } }) + + private def healthCheckExtent(projectedExtent: ProjectedExtent): Unit = { + val horizontal_tolerance = 1.5 + val polygonIsUTM = projectedExtent.crs.proj4jCrs.getProjection.getName == "utm" + if (polygonIsUTM) { + // This is an extend that has the highest sensible values for northern and/or southern hemisphere UTM zones + val utmProjectedBoundsOriginal = Extent(166021.44, 0000000.00, 833978.56, 10000000) + val utmProjectedBounds = utmProjectedBoundsOriginal.buffer( + utmProjectedBoundsOriginal.width * horizontal_tolerance, 0) + assert(projectedExtent.extent.intersects(utmProjectedBounds), + "Extend should be in valid values of UTM zone to avoid distortion when reprojecting: " + projectedExtent.extent) + } else if (projectedExtent.crs == LatLng) { + assert(projectedExtent.extent.xmin >= -180 * horizontal_tolerance) + assert(projectedExtent.extent.xmax <= +180 * horizontal_tolerance) + assert(projectedExtent.extent.ymin >= -90) + assert(projectedExtent.extent.ymax <= +90) + } + } + } class FileLayerProvider private(openSearch: OpenSearchClient, openSearchCollectionId: String, openSearchLinkTitles: NonEmptyList[String], rootPath: String, @@ -886,12 +905,6 @@ class FileLayerProvider private(openSearch: OpenSearchClient, openSearchCollecti val reprojectedBoundingBox: ProjectedExtent = DatacubeSupport.targetBoundingBox(fullBBox, layoutScheme) val alignedExtent = worldLayout.createAlignedRasterExtent(reprojectedBoundingBox.extent) - val polygonIsUTM = polygons_crs.proj4jCrs.getProjection.getName == "utm" - if (polygonIsUTM) { - // This is an extend that has the highest sensible values for northern and/or southern hemisphere UTM zones - val utmProjectedBounds = Extent(166021.44, 0000000.00, 833978.56, 10000000) - assert(alignedExtent.extent.intersects(utmProjectedBounds), "Extend should be in valid values of UTM zone to avoid distortion when reprojecting.") - } logger.info(s"Loading ${openSearchCollectionId} with params ${datacubeParams.getOrElse(new DataCubeParameters)} and bands ${openSearchLinkTitles.toList.mkString(";")} initial layout: ${worldLayout}") @@ -1247,13 +1260,26 @@ class FileLayerProvider private(openSearch: OpenSearchClient, openSearchCollecti * Several edge cases to cover: * - if feature extent is whole world, it may be invalid in target crs * - if feature is in utm, target extent may be invalid in feature crs - * this is why we take intersection + * this is why we take intersection. + * We convert both extents to a common CRS before converting ti to the target CRS. + * If one of the CRSes can cover the whole world (Non-UTM), we choose this CRS. */ - val featureExtentInTargetCRS = feature.rasterExtent.get.reproject(feature.crs.get, targetExtent.crs) - - val intersection = featureExtentInTargetCRS.intersection(targetExtent.extent).map(_.buffer(1.0)) - .getOrElse(featureExtentInTargetCRS) - val tmp = expandToCellSize(intersection, theResolution) + val featureIsUTM = feature.crs.get.proj4jCrs.getProjection.getName == "utm" + val targetIsUTM = targetExtent.crs.proj4jCrs.getProjection.getName == "utm" + val commonCrs = if (!targetIsUTM) targetExtent.crs + else if (!featureIsUTM) feature.crs.get + else targetExtent.crs // Avoid conversion imprecision by intersecting directly in the target CRS + + val featureExtentInCommonCRS = feature.rasterExtent.get.reproject(feature.crs.get, commonCrs) + val targetExtentInCommonCRS = targetExtent.extent.reproject(targetExtent.crs, commonCrs) + + val intersection = featureExtentInCommonCRS.intersection(targetExtentInCommonCRS).map(_.buffer(1.0)) + val intersectionTargetCrs = intersection match { + case None => targetExtent.extent.reproject(targetExtent.crs, targetExtent.crs) + case Some(value) => value.reproject(commonCrs, targetExtent.crs) + } + val tmp = expandToCellSize(intersectionTargetCrs, theResolution) + healthCheckExtent(ProjectedExtent(tmp, targetExtent.crs)) val alignedToTargetExtent = re.createAlignedRasterExtent(tmp) Some(alignedToTargetExtent.toGridType[Long]) diff --git a/openeo-geotrellis/src/test/resources/org/openeo/geotrellis/GlobalNetCdfFileLayerProviderTest/readDataCubeWithOpensearchClientUTM.tif b/openeo-geotrellis/src/test/resources/org/openeo/geotrellis/GlobalNetCdfFileLayerProviderTest/readDataCubeWithOpensearchClientUTM.tif new file mode 100644 index 0000000000000000000000000000000000000000..c86736b95130f0c4fbb59a00e8f47a9261c0ecee GIT binary patch literal 586 zcmebEWzb?^VBla7Vq{=o0kVJ;0}~@75}S#E87$5Llw?L?vq9NF9gLz-HWQF7gT%H) zQsV?OlaYbZ8Hw!zW&?FHJY#f)v)?fWBlI@&G4KH8LGJk0(gbF+0d3-IUk?_O0J80t zG%$e7kO#8!mNbIdOa?#!2w?05ioXT26M#6@pf^i&xKf~3fNWM!zyVo|EFd0`<^Z~mfl;{+ Date: Wed, 24 Jul 2024 22:53:28 +0200 Subject: [PATCH 09/15] Clean up healthCheckExtent(). --- .../geotrellis/layers/FileLayerProvider.scala | 25 ++---------- .../scala/org/openeo/geotrellis/package.scala | 26 +++++++++++++ .../org/openeo/geotrellis/PackageTest.scala | 38 ++++++++++++++++++- 3 files changed, 65 insertions(+), 24 deletions(-) diff --git a/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala b/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala index 72ab6856..98f528f2 100644 --- a/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala +++ b/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala @@ -27,11 +27,11 @@ import org.locationtech.jts.geom.Geometry import org.openeo.geotrellis.OpenEOProcessScriptBuilder.AnyProcess import org.openeo.geotrellis.file.{AbstractPyramidFactory, FixedFeaturesOpenSearchClient} import org.openeo.geotrellis.tile_grid.TileGrid -import org.openeo.geotrellis.{OpenEOProcessScriptBuilder, sortableSourceName} +import org.openeo.geotrellis.{OpenEOProcessScriptBuilder, healthCheckExtent, sortableSourceName} import org.openeo.geotrelliscommon.{BatchJobMetadataTracker, ByKeyPartitioner, CloudFilterStrategy, ConfigurableSpatialPartitioner, DataCubeParameters, DatacubeSupport, L1CCloudFilterStrategy, MaskTileLoader, NoCloudFilterStrategy, ResampledTile, SCLConvolutionFilterStrategy, SpaceTimeByMonthPartitioner, SparseSpaceTimePartitioner, autoUtmEpsg, retryForever} import org.openeo.opensearch.OpenSearchClient import org.openeo.opensearch.OpenSearchResponses.{Feature, Link} -import org.slf4j.LoggerFactory +import org.slf4j.{Logger, LoggerFactory} import java.io.{IOException, Serializable} import java.net.URI @@ -224,7 +224,7 @@ class MultibandCompositeRasterSource(val sourcesListWithBandIds: NonEmptyList[(R object FileLayerProvider { - private val logger = LoggerFactory.getLogger(classOf[FileLayerProvider]) + private implicit val logger: Logger = LoggerFactory.getLogger(classOf[FileLayerProvider]) private val maxRetries = sys.env.getOrElse("GDALREAD_MAXRETRIES", "10").toInt { @@ -804,25 +804,6 @@ object FileLayerProvider { Some(bbox, dates) } }) - - private def healthCheckExtent(projectedExtent: ProjectedExtent): Unit = { - val horizontal_tolerance = 1.5 - val polygonIsUTM = projectedExtent.crs.proj4jCrs.getProjection.getName == "utm" - if (polygonIsUTM) { - // This is an extend that has the highest sensible values for northern and/or southern hemisphere UTM zones - val utmProjectedBoundsOriginal = Extent(166021.44, 0000000.00, 833978.56, 10000000) - val utmProjectedBounds = utmProjectedBoundsOriginal.buffer( - utmProjectedBoundsOriginal.width * horizontal_tolerance, 0) - assert(projectedExtent.extent.intersects(utmProjectedBounds), - "Extend should be in valid values of UTM zone to avoid distortion when reprojecting: " + projectedExtent.extent) - } else if (projectedExtent.crs == LatLng) { - assert(projectedExtent.extent.xmin >= -180 * horizontal_tolerance) - assert(projectedExtent.extent.xmax <= +180 * horizontal_tolerance) - assert(projectedExtent.extent.ymin >= -90) - assert(projectedExtent.extent.ymax <= +90) - } - } - } class FileLayerProvider private(openSearch: OpenSearchClient, openSearchCollectionId: String, openSearchLinkTitles: NonEmptyList[String], rootPath: String, diff --git a/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/package.scala b/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/package.scala index 468dd919..2fbed400 100644 --- a/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/package.scala +++ b/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/package.scala @@ -1,6 +1,8 @@ package org.openeo +import _root_.geotrellis.proj4.LatLng import _root_.geotrellis.raster._ +import _root_.geotrellis.vector._ import org.slf4j.Logger import software.amazon.awssdk.awscore.retry.conditions.RetryOnErrorCodeCondition import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration @@ -152,4 +154,28 @@ package object geotrellis { object TemporalResolution extends Enumeration { val seconds, days, undefined = Value } + + def healthCheckExtent(projectedExtent: ProjectedExtent)(implicit logger: Logger): Boolean = { + val horizontal_tolerance = 1.1 + val polygonIsUTM = projectedExtent.crs.proj4jCrs.getProjection.getName == "utm" + if (polygonIsUTM) { + // This is an extend that has the highest sensible values for northern and/or southern hemisphere UTM zones + val utmProjectedBoundsOriginal = Extent(166021.44, 0000000.00, 833978.56, 10000000) + val utmProjectedBounds = utmProjectedBoundsOriginal.buffer( + utmProjectedBoundsOriginal.width * horizontal_tolerance, 0) + if (!projectedExtent.extent.intersects(utmProjectedBounds)) { + logger.warn("healthCheckExtent dangerous extent: " + projectedExtent) + return false + } + } else if (projectedExtent.crs == LatLng) { + if ((projectedExtent.extent.xmin < -180 * horizontal_tolerance) + || (projectedExtent.extent.xmax > +180 * horizontal_tolerance) + || (projectedExtent.extent.ymin < -90) + || (projectedExtent.extent.ymax > +90)) { + logger.warn("healthCheckExtent dangerous extent: " + projectedExtent) + return false + } + } + true + } } diff --git a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/PackageTest.scala b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/PackageTest.scala index 1eae19c9..5be9e588 100644 --- a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/PackageTest.scala +++ b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/PackageTest.scala @@ -1,10 +1,33 @@ package org.openeo.geotrellis +import geotrellis.proj4.{CRS, LatLng} import geotrellis.raster.{ByteCellType, ByteUserDefinedNoDataCellType, FloatUserDefinedNoDataCellType, UByteCellType, UByteUserDefinedNoDataCellType} -import org.junit.Assert._ -import org.junit.Test +import geotrellis.vector.{Extent, ProjectedExtent} +import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse} +import org.junit.jupiter.api.Test +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.Arguments.arguments +import org.junit.jupiter.params.provider.{Arguments, MethodSource} +import org.openeo.geotrellis.layers.FileLayerProvider +import org.slf4j.{Logger, LoggerFactory} + +object PackageTest { + private implicit val logger: Logger = LoggerFactory.getLogger(classOf[FileLayerProvider]) + + def testHealthCheckExtentParamsOk: java.util.stream.Stream[Arguments] = java.util.Arrays.stream(Array( + arguments(ProjectedExtent(Extent(40, 40, 50, 50), LatLng)), + arguments(ProjectedExtent(Extent(11000, 40, 22000, 50), CRS.fromName("EPSG:32631"))), + )) + def testHealthCheckExtentParamsNok: java.util.stream.Stream[Arguments] = java.util.Arrays.stream(Array( + arguments(ProjectedExtent(Extent(-400, 40, -300, 50), LatLng)), + arguments(ProjectedExtent(Extent(5000111, 40, 5000222, 50), CRS.fromName("EPSG:32631"))), + )) +} class PackageTest { + + import PackageTest._ + @Test def testToSigned(): Unit = { assertEquals(ByteCellType, toSigned(UByteCellType)) @@ -12,4 +35,15 @@ class PackageTest { assertEquals(FloatUserDefinedNoDataCellType(42), toSigned(FloatUserDefinedNoDataCellType(42))) assertEquals(ByteUserDefinedNoDataCellType(42), toSigned(ByteUserDefinedNoDataCellType(42))) } + + @ParameterizedTest + @MethodSource(Array("testHealthCheckExtentParamsOk")) + def testHealthCheckExtentOk(projectedExtent: ProjectedExtent): Unit = { + assert(healthCheckExtent(projectedExtent)) + } + @ParameterizedTest + @MethodSource(Array("testHealthCheckExtentParamsNok")) + def testHealthCheckExtentNok(projectedExtent: ProjectedExtent): Unit = { + assertFalse(healthCheckExtent(projectedExtent)) + } } From 7b6760d87a3f2026b62070d46e85dc1822e1235c Mon Sep 17 00:00:00 2001 From: Emile Sonneveld Date: Thu, 25 Jul 2024 08:56:36 +0200 Subject: [PATCH 10/15] fix --- .../org/openeo/geotrellis/layers/FileLayerProviderTest.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala index 9e453d25..50f60a99 100644 --- a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala +++ b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala @@ -1168,7 +1168,7 @@ class FileLayerProviderTest extends RasterMatchers{ @Test def testMissingS2DateLineOutside(): Unit = { - assertThrows[AssertionError](testMissingS2DateLine("EPSG:32631")) + assertThrows[Exception](testMissingS2DateLine("EPSG:32631")) } private def keysForLargeArea(useBBox:Boolean=false) = { From 500fccf48f84c505251afb3739cf3e25427edf9e Mon Sep 17 00:00:00 2001 From: Emile Sonneveld Date: Thu, 25 Jul 2024 13:27:08 +0200 Subject: [PATCH 11/15] comment --- .../org/openeo/geotrellis/layers/FileLayerProvider.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala b/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala index a764d520..03851950 100644 --- a/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala +++ b/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala @@ -1250,8 +1250,9 @@ class FileLayerProvider private(openSearch: OpenSearchClient, openSearchCollecti * - if feature extent is whole world, it may be invalid in target crs * - if feature is in utm, target extent may be invalid in feature crs * this is why we take intersection. - * We convert both extents to a common CRS before converting ti to the target CRS. - * If one of the CRSes can cover the whole world (Non-UTM), we choose this CRS. + * We convert both extents to a common CRS before taking the intersection. + * If one of the CRSes can cover the whole world (non-UTM), this will be used as common CRS. + * We give priority to use the target CRS as common one, because the intersection will be converted to it anyway */ val featureIsUTM = feature.crs.get.proj4jCrs.getProjection.getName == "utm" val targetIsUTM = targetExtent.crs.proj4jCrs.getProjection.getName == "utm" From d8691bc04c5002bdba6067ef7f5010825d8ae26b Mon Sep 17 00:00:00 2001 From: Emile Sonneveld Date: Fri, 9 Aug 2024 08:31:55 +0200 Subject: [PATCH 12/15] Add log to see if different UTM test case is already covered. --- .../org/openeo/geotrellis/layers/FileLayerProvider.scala | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala b/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala index 03851950..ed1924b7 100644 --- a/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala +++ b/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala @@ -1256,6 +1256,12 @@ class FileLayerProvider private(openSearch: OpenSearchClient, openSearchCollecti */ val featureIsUTM = feature.crs.get.proj4jCrs.getProjection.getName == "utm" val targetIsUTM = targetExtent.crs.proj4jCrs.getProjection.getName == "utm" + if (featureIsUTM && targetIsUTM) { + if (feature.crs.get != targetExtent.crs) { + // Check logs if this test-case is already covered. TODO: Remove this log before merge + logger.info(s"Special case: Feature and target extent are both in UTM, but have different CRSes: ${feature.crs.get} and ${targetExtent.crs}.") + } + } val commonCrs = if (!targetIsUTM) targetExtent.crs else if (!featureIsUTM) feature.crs.get else targetExtent.crs // Avoid conversion imprecision by intersecting directly in the target CRS From 4896876453664e3c3293be26b101c6c6a2a8c642 Mon Sep 17 00:00:00 2001 From: Emile Sonneveld Date: Fri, 9 Aug 2024 10:48:22 +0200 Subject: [PATCH 13/15] Revert "Add log to see if different UTM test case is already covered." This reverts commit d8691bc04c5002bdba6067ef7f5010825d8ae26b. --- .../org/openeo/geotrellis/layers/FileLayerProvider.scala | 6 ------ 1 file changed, 6 deletions(-) diff --git a/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala b/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala index b41e650f..fb88b05b 100644 --- a/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala +++ b/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala @@ -1268,12 +1268,6 @@ class FileLayerProvider private(openSearch: OpenSearchClient, openSearchCollecti */ val featureIsUTM = feature.crs.get.proj4jCrs.getProjection.getName == "utm" val targetIsUTM = targetExtent.crs.proj4jCrs.getProjection.getName == "utm" - if (featureIsUTM && targetIsUTM) { - if (feature.crs.get != targetExtent.crs) { - // Check logs if this test-case is already covered. TODO: Remove this log before merge - logger.info(s"Special case: Feature and target extent are both in UTM, but have different CRSes: ${feature.crs.get} and ${targetExtent.crs}.") - } - } val commonCrs = if (!targetIsUTM) targetExtent.crs else if (!featureIsUTM) feature.crs.get else targetExtent.crs // Avoid conversion imprecision by intersecting directly in the target CRS From 0309ef2bc8e8e7f67492f0265841ed0deee98608 Mon Sep 17 00:00:00 2001 From: Emile Sonneveld Date: Tue, 17 Sep 2024 10:19:04 +0200 Subject: [PATCH 14/15] Test EPSG:3035 and EPSG:4326 too. https://github.com/Open-EO/openeo-geotrellis-extensions/issues/279 --- .../geotrellis/layers/FileLayerProvider.scala | 3 --- .../layers/FileLayerProviderTest.scala | 26 ++++++++++++++++--- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala b/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala index 7f17c5f8..b2a1aa79 100644 --- a/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala +++ b/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala @@ -28,10 +28,7 @@ import org.apache.spark.util.LongAccumulator import org.locationtech.jts.geom.Geometry import org.openeo.geotrellis.OpenEOProcessScriptBuilder.AnyProcess import org.openeo.geotrellis.file.{AbstractPyramidFactory, FixedFeaturesOpenSearchClient} -import org.openeo.geotrellis.tile_grid.TileGrid import org.openeo.geotrellis.{OpenEOProcessScriptBuilder, healthCheckExtent, sortableSourceName} -import org.openeo.geotrelliscommon.{BatchJobMetadataTracker, ByKeyPartitioner, CloudFilterStrategy, ConfigurableSpatialPartitioner, DataCubeParameters, DatacubeSupport, L1CCloudFilterStrategy, MaskTileLoader, NoCloudFilterStrategy, ResampledTile, SCLConvolutionFilterStrategy, SpaceTimeByMonthPartitioner, SparseSpaceTimePartitioner, autoUtmEpsg, retryForever} -import org.openeo.geotrellis.{OpenEOProcessScriptBuilder, sortableSourceName} import org.openeo.geotrelliscommon.DatacubeSupport.prepareMask import org.openeo.geotrelliscommon.{BatchJobMetadataTracker, ByKeyPartitioner, CloudFilterStrategy, ConfigurableSpatialPartitioner, DataCubeParameters, DatacubeSupport, L1CCloudFilterStrategy, MaskTileLoader, NoCloudFilterStrategy, ResampledTile, SCLConvolutionFilterStrategy, SpaceTimeByMonthPartitioner, SparseSpaceTimePartitioner, autoUtmEpsg} import org.openeo.opensearch.OpenSearchClient diff --git a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala index c0b53ff4..f6f87dd9 100644 --- a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala +++ b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/FileLayerProviderTest.scala @@ -1140,19 +1140,29 @@ class FileLayerProviderTest extends RasterMatchers{ } @ParameterizedTest - @ValueSource(strings = Array("EPSG:32601", "EPSG:32660")) + @ValueSource(strings = Array("EPSG:32601", "EPSG:32660", "EPSG:3035", "EPSG:4326")) def testMissingS2DateLine(crsName: String): Unit = { - // crsName "EPSG:4326" only works with the environment variable PROJ_LIB=/usr/share/proj + if ((crsName == "EPSG:3035" || crsName == "EPSG:4326") && System.getenv("PROJ_LIB") == null) { + println("PROJ_LIB is not set in the environment variables. Skipping this test") + // A typical value would be: PROJ_LIB=/usr/share/proj + return + } val outDir = Paths.get("tmp/FileLayerProviderTest_" + crsName.replace(":", "_") + "/") new Directory(outDir.toFile).deepFiles.foreach(_.delete()) Files.createDirectories(outDir) - val extent = Extent(178.7384, 70.769, 178.8548, 70.8254) + val extent = Extent(178.1, 70.3, 178.9, 70.9) val projected_polygons_native_crs = ProjectedPolygons.fromExtent(extent, LatLng.proj4jCrs.toString) val utmCrs = CRS.fromName(crsName) val reprojected = projected_polygons_native_crs.polygons.head.reproject(projected_polygons_native_crs.crs, utmCrs) val poly2 = ProjectedPolygons(Array(reprojected), utmCrs) + if (crsName == "EPSG:4326") { + val poly2GeoJson = poly2.polygons.head.toGeoJson + // geojson only officially suports latLon + Files.writeString(Paths.get(outDir + "/polygons.geojson"), poly2GeoJson) + } + val layer = LayerFixtures.sentinel2Cube( LocalDate.of(2024, 4, 2), poly2, @@ -1163,10 +1173,18 @@ class FileLayerProviderTest extends RasterMatchers{ val layer_collected = layer.collect() assert(layer_collected.nonEmpty) + var found10 = false // SCL value for 'snow or ice' for { (_, multiBandTile) <- layer_collected tile <- multiBandTile.bands - } assert(!tile.isNoDataTile) + } { + assert(!tile.isNoDataTile) + val values = tile.toArrayDouble() + if (values.contains(10.0)) { + found10 = true + } + } + assert(found10) val cubeSpatial = layer.toSpatial() cubeSpatial.writeGeoTiff(outDir + "/testMissingS2DateLine_" + crsName.replace(":", "_") + ".tiff") } From ad69dd766fb8d3138ee9c75b77cf6be489bb252b Mon Sep 17 00:00:00 2001 From: Emile Sonneveld Date: Mon, 28 Oct 2024 22:49:35 +0100 Subject: [PATCH 15/15] fix after merge --- .../src/test/scala/org/openeo/geotrellis/PackageTest.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/PackageTest.scala b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/PackageTest.scala index 082779e2..1aba772b 100644 --- a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/PackageTest.scala +++ b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/PackageTest.scala @@ -4,7 +4,6 @@ import geotrellis.raster.io.geotiff.GeoTiff import geotrellis.proj4.{CRS, LatLng} import geotrellis.raster.{ByteCellType, ByteUserDefinedNoDataCellType, FloatUserDefinedNoDataCellType, UByteCellType, UByteUserDefinedNoDataCellType} import org.junit.Assert._ -import org.junit.Test import org.openeo.geotrellis.geotiff._ import java.nio.file.{Files, Path}