From 6a9bc4bc4f893cfe5aedfc5738f0c8b6baf95067 Mon Sep 17 00:00:00 2001 From: Emile Sonneveld Date: Wed, 6 Nov 2024 22:35:01 +0100 Subject: [PATCH] Allow override target files again to be sure. --- .../scala/org/openeo/geotrellis/creo/CreoS3Utils.scala | 10 +++++----- .../scala/org/openeo/geotrellis/geotiff/package.scala | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/creo/CreoS3Utils.scala b/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/creo/CreoS3Utils.scala index ee0d1c9c..ca8f16b4 100644 --- a/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/creo/CreoS3Utils.scala +++ b/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/creo/CreoS3Utils.scala @@ -212,20 +212,20 @@ object CreoS3Utils { } } - def moveOverwriteWithRetries(oldPath: Path, newPath: Path): Unit = { + def moveOverwriteWithRetries(oldPath: String, newPath: String): Unit = { var try_count = 1 breakable { while (true) { try { - if (assetExists(newPath.toString)) { + if (assetExists(newPath)) { // It might be a partial result of a previous failing task. logger.info(f"Will replace $newPath. (try $try_count)") + assetDelete(newPath) } - Files.deleteIfExists(newPath) - Files.move(oldPath, newPath) + moveAsset(oldPath, newPath) break } catch { - case e: FileAlreadyExistsException => + case e: Exception => // Here if another executor wrote the file between the delete and the move statement. try_count += 1 if (try_count > 5) { diff --git a/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/geotiff/package.scala b/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/geotiff/package.scala index acd872a4..5d8b2a8f 100644 --- a/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/geotiff/package.scala +++ b/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/geotiff/package.scala @@ -130,7 +130,7 @@ package object geotiff { if (!CreoS3Utils.isS3(parentDirectory.toString)) { Files.createDirectories(destinationPath.getParent) } - CreoS3Utils.moveAsset(absoluteFilePath, destinationPath.toString) + CreoS3Utils.moveOverwriteWithRetries(absoluteFilePath, destinationPath.toString) destinationPath } @@ -923,7 +923,7 @@ package object geotiff { CreoS3Utils.uploadToS3(tempFile, correctS3Path) } else { // Retry should not be needed at this point, but it is almost free to keep it. - CreoS3Utils.moveOverwriteWithRetries(tempFile, Path.of(path)) + CreoS3Utils.moveOverwriteWithRetries(tempFile.toString, path) path } }