Skip to content

Commit 5ef4a96

Browse files
EmileSonneveldEmileSonneveld
committed
Merge branch 'develop' into dateline
2 parents 4896876 + 72856ef commit 5ef4a96

File tree

30 files changed

+876
-208
lines changed

30 files changed

+876
-208
lines changed

‎Jenkinsfile

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,9 @@ pipeline {
3737
PACKAGE_NAME = "${package_name}"
3838
WORKSPACE = "${env.WORKSPACE}"
3939
}
40-
40+
parameters {
41+
booleanParam(name: 'skip_tests', defaultValue: false, description: 'Check this if you want to skip running tests.')
42+
}
4143
stages {
4244
stage('Checkout') {
4345
steps {
@@ -55,7 +57,7 @@ pipeline {
5557
steps {
5658
script {
5759
rel_version = getMavenVersion()
58-
build()
60+
build( !params.skip_tests)
5961
utils.setWorkspacePermissions()
6062
}
6163
}
@@ -196,12 +198,17 @@ void build(tests = true){
196198
sh "dnf install -y maven git java-11-openjdk-devel gdal-3.8.4"
197199
def server = Artifactory.server('vitoartifactory')
198200
def rtMaven = Artifactory.newMavenBuild()
199-
rtMaven.deployer server: server, releaseRepo: 'libs-release-public', snapshotRepo: 'libs-snapshot-public'
201+
def snapshotRepo = 'libs-snapshot-public'
202+
if (!publishable_branches.contains(env.BRANCH_NAME)) {
203+
snapshotRepo = 'openeo-branch-builds'
204+
rtMaven.opts += " -Drevision=${env.BRANCH_NAME}"
205+
}
206+
rtMaven.deployer server: server, releaseRepo: 'libs-release-public', snapshotRepo: snapshotRepo
200207
rtMaven.tool = maven
201208
if (!tests) {
202209
rtMaven.opts += ' -DskipTests=true'
203210
}
204-
rtMaven.deployer.deployArtifacts = publishable_branches.contains(env.BRANCH_NAME) || publishable_branches.contains(env.CHANGE_BRANCH)
211+
rtMaven.deployer.deployArtifacts = true
205212
//use '--projects StatisticsMapReduce' in 'goals' to build specific module
206213
try {
207214
withCredentials([

‎geopyspark-geotrellis/pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
<parent>
66
<artifactId>openeo-geotrellis-extensions</artifactId>
77
<groupId>org.openeo</groupId>
8-
<version>2.4.0_2.12-SNAPSHOT</version>
8+
<version>${revision}</version>
99
</parent>
1010
<modelVersion>4.0.0</modelVersion>
1111

‎geotrellis-accumulo-extensions/pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
<parent>
44
<artifactId>openeo-geotrellis-extensions</artifactId>
55
<groupId>org.openeo</groupId>
6-
<version>2.4.0_2.12-SNAPSHOT</version>
6+
<version>${revision}</version>
77
</parent>
88
<modelVersion>4.0.0</modelVersion>
99

‎geotrellis-benchmarks/pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
<parent>
66
<artifactId>openeo-geotrellis-extensions</artifactId>
77
<groupId>org.openeo</groupId>
8-
<version>2.4.0_2.12-SNAPSHOT</version>
8+
<version>${revision}</version>
99
</parent>
1010
<modelVersion>4.0.0</modelVersion>
1111

‎geotrellis-common/pom.xml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
<parent>
66
<artifactId>openeo-geotrellis-extensions</artifactId>
77
<groupId>org.openeo</groupId>
8-
<version>2.4.0_2.12-SNAPSHOT</version>
8+
<version>${revision}</version>
99
</parent>
1010
<modelVersion>4.0.0</modelVersion>
1111

@@ -55,13 +55,13 @@
5555
<dependency>
5656
<groupId>org.junit.jupiter</groupId>
5757
<artifactId>junit-jupiter-api</artifactId>
58-
<version>5.3.2</version>
58+
<version>5.10.3</version>
5959
<scope>test</scope>
6060
</dependency>
6161
<dependency>
6262
<groupId>org.junit.vintage</groupId>
6363
<artifactId>junit-vintage-engine</artifactId>
64-
<version>5.3.2</version>
64+
<version>5.10.3</version>
6565
<scope>test</scope>
6666
</dependency>
6767
<dependency>

‎geotrellis-common/src/main/scala/org/openeo/geotrelliscommon/DatacubeSupport.scala

Lines changed: 44 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@ import geotrellis.spark.partition.{PartitionerIndex, SpacePartitioner}
88
import geotrellis.spark.{MultibandTileLayerRDD, _}
99
import geotrellis.util.GetComponent
1010
import geotrellis.vector.{Extent, MultiPolygon, ProjectedExtent}
11-
import org.apache.spark.rdd.RDD
11+
import org.apache.spark.Partitioner
12+
import org.apache.spark.rdd.{CoGroupedRDD, RDD}
1213
import org.slf4j.LoggerFactory
1314

1415
import java.time.ZonedDateTime
@@ -194,7 +195,25 @@ object DatacubeSupport {
194195
ignoreKeysWithoutMask: Boolean = false,
195196
): RDD[(K, MultibandTile)] with Metadata[M] = {
196197
val joined = if (ignoreKeysWithoutMask) {
197-
val tmpRdd = SpatialJoin.join(datacube, mask).mapValues(v => (v._1, Option(v._2)))
198+
//inner join, try to preserve partitioner
199+
val tmpRdd: RDD[(K, (MultibandTile, Option[MultibandTile]))] =
200+
if(datacube.partitioner.isDefined && datacube.partitioner.get.isInstanceOf[SpacePartitioner[K]]){
201+
val part = datacube.partitioner.get.asInstanceOf[SpacePartitioner[K]]
202+
new CoGroupedRDD[K](List(datacube, part(mask)), part)
203+
.flatMapValues { case Array(l, r) =>
204+
if (l.isEmpty) {
205+
Seq.empty[(MultibandTile, Option[MultibandTile])]
206+
}
207+
else if (r.isEmpty)
208+
Seq.empty[(MultibandTile, Option[MultibandTile])]
209+
else
210+
for (v <- l.iterator; w <- r.iterator) yield (v, Some(w))
211+
}.asInstanceOf[RDD[(K, (MultibandTile, Option[MultibandTile]))]]
212+
}else{
213+
SpatialJoin.join(datacube, mask).mapValues(v => (v._1, Option(v._2)))
214+
}
215+
216+
198217
ContextRDD(tmpRdd, datacube.metadata)
199218
} else {
200219
SpatialJoin.leftOuterJoin(datacube, mask)
@@ -210,7 +229,8 @@ object DatacubeSupport {
210229
if (dataTile.bandCount == maskTile.bandCount) {
211230
maskIndex = index
212231
}
213-
tile.dualCombine(maskTile.band(maskIndex))((v1, v2) => if (v2 != 0 && isData(v1)) replacementInt else v1)((v1, v2) => if (v2 != 0.0 && isData(v1)) replacementDouble else v1)
232+
//tile has to be 'mutable', for instant ConstantTile implements dualCombine, but not correctly converting celltype!!
233+
tile.mutable.dualCombine(maskTile.band(maskIndex))((v1, v2) => if (v2 != 0 && isData(v1)) replacementInt else v1)((v1, v2) => if (v2 != 0.0 && isData(v1)) replacementDouble else v1)
214234
})
215235

216236
} else {
@@ -236,16 +256,8 @@ object DatacubeSupport {
236256
logger.debug(s"Spacetime mask is used to reduce input.")
237257
}
238258

239-
val alignedMask: MultibandTileLayerRDD[SpaceTimeKey] =
240-
if(spacetimeMask.metadata.crs.equals(metadata.crs) && spacetimeMask.metadata.layout.equals(metadata.layout)) {
241-
spacetimeMask
242-
}else{
243-
logger.debug(s"mask: automatically resampling mask to match datacube: ${spacetimeMask.metadata}")
244-
spacetimeMask.reproject(metadata.crs,metadata.layout,16,rdd.partitioner)._2
245-
}
246-
247-
// retain only tiles where there is at least one valid pixel (mask value == 0), others will be fully removed
248-
val filtered = alignedMask.withContext{_.filter(_._2.band(0).toArray().exists(pixel => pixel == 0))}
259+
val partitioner = rdd.partitioner
260+
val filtered = prepareMask(spacetimeMask, metadata, partitioner)
249261

250262
if (pixelwiseMasking) {
251263
val spacetimeDataContextRDD = ContextRDD(rdd, metadata)
@@ -263,4 +275,23 @@ object DatacubeSupport {
263275
rdd
264276
}
265277
}
278+
279+
def prepareMask(spacetimeMask: MultibandTileLayerRDD[SpaceTimeKey], metadata: TileLayerMetadata[SpaceTimeKey], partitioner: Option[Partitioner]): ContextRDD[SpaceTimeKey, MultibandTile, TileLayerMetadata[SpaceTimeKey]] = {
280+
val alignedMask: MultibandTileLayerRDD[SpaceTimeKey] =
281+
if (spacetimeMask.metadata.crs.equals(metadata.crs) && spacetimeMask.metadata.layout.equals(metadata.layout)) {
282+
spacetimeMask
283+
} else {
284+
logger.debug(s"mask: automatically resampling mask to match datacube: ${spacetimeMask.metadata}")
285+
spacetimeMask.reproject(metadata.crs, metadata.layout, 16, partitioner)._2
286+
}
287+
288+
val keyBounds = metadata.bounds.get
289+
// retain only tiles where there is at least one valid pixel (mask value == 0), others will be fully removed
290+
val filtered = alignedMask.withContext {
291+
_.filter(t => {
292+
keyBounds.includes(t._1) && t._2.band(0).toArray().exists(pixel => pixel == 0)
293+
})
294+
}
295+
filtered
296+
}
266297
}

‎geotrellis-common/src/main/scala/org/openeo/geotrelliscommon/package.scala

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -231,16 +231,16 @@ package object geotrelliscommon {
231231
import java.util.concurrent.TimeUnit
232232

233233

234-
def retryForever[R](delay: Duration, retries: Int = 20, onAttemptFailed: Exception => Unit = _ => ())(f: => R): R = {
234+
def retryForever[R](delay: Duration, attempts: Int = 20, onAttemptFailed: Exception => Unit = _ => ())(f: => R): R = {
235235
var lastException: Exception = null
236-
var countDown = retries
236+
var countDown = attempts
237237
while (countDown>0) {
238238
try return f
239239
catch {
240240
case e: Exception =>
241241
onAttemptFailed(e)
242242
lastException = e
243-
TimeUnit.SECONDS.sleep(delay.getSeconds)
243+
if (countDown > 1) TimeUnit.SECONDS.sleep(delay.getSeconds)
244244
}
245245
countDown = countDown - 1
246246
}

‎geotrellis-common/src/test/scala/org/openeo/geotrelliscommon/PackageTest.scala

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
package org.openeo.geotrelliscommon
2+
3+
import org.junit.jupiter.api.Assertions.{assertEquals, assertThrowsExactly, fail}
4+
import org.junit.jupiter.api.{Test, Timeout}
5+
6+
import java.time.Duration
7+
8+
class PackageTest {
9+
class FailedAttempt extends Exception
10+
11+
@Test
12+
def retryForeverNumberOfAttempts(): Unit = {
13+
var attempts = 0
14+
15+
try {
16+
retryForever(delay = Duration.ZERO, attempts = 3, onAttemptFailed = _ => attempts += 1) {
17+
println("attempting...")
18+
throw new FailedAttempt
19+
}
20+
21+
fail("should have thrown a FailedAttempt")
22+
} catch {
23+
case _: FailedAttempt =>
24+
}
25+
26+
// count the number of failures to get the number of attempts
27+
assertEquals(3, attempts)
28+
}
29+
30+
@Test
31+
@Timeout(5) // less than RetryForever's delay below
32+
def retryForeverNoDelayAfterFinalFailure(): Unit =
33+
assertThrowsExactly(classOf[FailedAttempt], () =>
34+
retryForever(delay = Duration.ofSeconds(60), attempts = 1) {
35+
println("attempting...")
36+
throw new FailedAttempt
37+
})
38+
}

‎geotrellis-extensions/pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
<parent>
44
<artifactId>openeo-geotrellis-extensions</artifactId>
55
<groupId>org.openeo</groupId>
6-
<version>2.4.0_2.12-SNAPSHOT</version>
6+
<version>${revision}</version>
77
</parent>
88
<modelVersion>4.0.0</modelVersion>
99

‎geotrellis-integrationtests/pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
<parent>
66
<artifactId>openeo-geotrellis-extensions</artifactId>
77
<groupId>org.openeo</groupId>
8-
<version>2.4.0_2.12-SNAPSHOT</version>
8+
<version>${revision}</version>
99
</parent>
1010
<modelVersion>4.0.0</modelVersion>
1111

0 commit comments

Comments
 (0)