From 7bca8720d111ed024ff3cc2819983856a9099065 Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 7 Oct 2025 11:58:33 +0200 Subject: [PATCH 1/2] Merge RemoteSourceDescriptorService into DataVaultService --- test/backend/DataVaultTestSuite.scala | 24 +-- .../datastore/DataStoreModule.scala | 3 +- .../controllers/DataSourceController.scala | 6 +- .../DatasetArrayBucketProvider.scala | 16 +- .../datavault/GoogleCloudDataVault.scala | 8 +- .../datastore/datavault/HttpsDataVault.scala | 8 +- .../datastore/datavault/S3DataVault.scala | 9 +- .../explore/ExploreLocalLayerService.scala | 9 +- .../explore/ExploreRemoteLayerService.scala | 5 +- .../explore/NeuroglancerUriExplorer.scala | 5 +- .../models/datasource/DataLayer.scala | 8 +- .../services/BinaryDataService.scala | 6 +- .../services/BinaryDataServiceHolder.scala | 6 +- .../services/DSUsedStorageService.scala | 7 +- .../services/DataSourceService.scala | 23 +-- .../ZarrConnectomeFileService.scala | 9 +- .../mapping/ZarrAgglomerateService.scala | 6 +- ...uroglancerPrecomputedMeshFileService.scala | 10 +- .../services/mesh/ZarrMeshFileService.scala | 9 +- .../ZarrSegmentIndexFileService.scala | 9 +- .../services/uploading/UploadService.scala | 10 +- .../datastore/storage/DataVaultService.scala | 178 ++++++++++++++++-- .../RemoteSourceDescriptorService.scala | 160 ---------------- .../EditableMappingLayer.scala | 6 +- .../tracings/volume/VolumeTracingLayer.scala | 4 +- 25 files changed, 256 insertions(+), 288 deletions(-) delete mode 100644 webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/RemoteSourceDescriptorService.scala diff --git a/test/backend/DataVaultTestSuite.scala b/test/backend/DataVaultTestSuite.scala index 670db08c94d..6f2762b0813 100644 --- a/test/backend/DataVaultTestSuite.scala +++ b/test/backend/DataVaultTestSuite.scala @@ -14,7 +14,7 @@ import com.scalableminds.webknossos.datastore.datavault.{ S3DataVault, VaultPath } -import com.scalableminds.webknossos.datastore.storage.{GoogleServiceAccountCredential, RemoteSourceDescriptor} +import com.scalableminds.webknossos.datastore.storage.{GoogleServiceAccountCredential, CredentializedUPath} import com.scalableminds.util.tools.{Box, Empty, EmptyBox, Failure, Full} import com.scalableminds.webknossos.datastore.helpers.UPath import play.api.libs.json.JsString @@ -41,7 +41,7 @@ class DataVaultTestSuite extends PlaySpec { WsTestClient.withClient { ws => val upath = UPath.fromStringUnsafe("http://storage.googleapis.com/") val vaultPath = - new VaultPath(upath, HttpsDataVault.create(RemoteSourceDescriptor(upath, None), ws, dummyDataStoreHost)) + new VaultPath(upath, HttpsDataVault.create(CredentializedUPath(upath, None), ws, dummyDataStoreHost)) val bytes = (vaultPath / s"neuroglancer-fafb-data/fafb_v14/fafb_v14_orig/$dataKey") .readBytes(Some(range))(globalExecutionContext, emptyTokenContext) @@ -55,7 +55,7 @@ class DataVaultTestSuite extends PlaySpec { "using Google Cloud Storage Vault" should { val upath = UPath.fromStringUnsafe("gs://neuroglancer-fafb-data/fafb_v14/fafb_v14_orig") - val vaultPath = new VaultPath(upath, GoogleCloudDataVault.create(RemoteSourceDescriptor(upath, None))) + val vaultPath = new VaultPath(upath, GoogleCloudDataVault.create(CredentializedUPath(upath, None))) "return correct response" in { val bytes = (vaultPath / dataKey) @@ -87,7 +87,7 @@ class DataVaultTestSuite extends PlaySpec { new VaultPath( upath, GoogleCloudDataVault.create( - RemoteSourceDescriptor( + CredentializedUPath( upath, Some(GoogleServiceAccountCredential("name", JsString("secret"), Some("user"), Some("org"))))) ) @@ -104,7 +104,7 @@ class DataVaultTestSuite extends PlaySpec { val upath = UPath.fromStringUnsafe("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5/em/fibsem-uint16/") WsTestClient.withClient { ws => val vaultPath = - new VaultPath(upath, S3DataVault.create(RemoteSourceDescriptor(upath, None), ws)(globalExecutionContext)) + new VaultPath(upath, S3DataVault.create(CredentializedUPath(upath, None), ws)(globalExecutionContext)) val bytes = (vaultPath / "s0/5/5/5") .readBytes(Some(range))(globalExecutionContext, emptyTokenContext) @@ -125,7 +125,7 @@ class DataVaultTestSuite extends PlaySpec { WsTestClient.withClient { ws => val upath = UPath.fromStringUnsafe("http://storage.googleapis.com/") val vaultPath = - new VaultPath(upath, HttpsDataVault.create(RemoteSourceDescriptor(upath, None), ws, dummyDataStoreHost)) + new VaultPath(upath, HttpsDataVault.create(CredentializedUPath(upath, None), ws, dummyDataStoreHost)) val bytes = (vaultPath / s"neuroglancer-fafb-data/fafb_v14/fafb_v14_orig/$dataKey") .readBytes()(globalExecutionContext, emptyTokenContext) .get(handleFoxJustification) @@ -139,7 +139,7 @@ class DataVaultTestSuite extends PlaySpec { "using Google Cloud Storage Vault" should { "return correct response" in { val upath = UPath.fromStringUnsafe("gs://neuroglancer-fafb-data/fafb_v14/fafb_v14_orig") - val vaultPath = new VaultPath(upath, GoogleCloudDataVault.create(RemoteSourceDescriptor(upath, None))) + val vaultPath = new VaultPath(upath, GoogleCloudDataVault.create(CredentializedUPath(upath, None))) val bytes = (vaultPath / dataKey).readBytes()(globalExecutionContext, emptyTokenContext).get(handleFoxJustification) @@ -153,7 +153,7 @@ class DataVaultTestSuite extends PlaySpec { val upath = UPath.fromStringUnsafe("s3://open-neurodata/bock11/image/4_4_40") WsTestClient.withClient { ws => val vaultPath = - new VaultPath(upath, S3DataVault.create(RemoteSourceDescriptor(upath, None), ws)(globalExecutionContext)) + new VaultPath(upath, S3DataVault.create(CredentializedUPath(upath, None), ws)(globalExecutionContext)) val bytes = (vaultPath / "33792-34304_29696-30208_3216-3232") .readBytes()(globalExecutionContext, emptyTokenContext) @@ -166,7 +166,7 @@ class DataVaultTestSuite extends PlaySpec { "requesting a non-existent bucket" in { val upath = UPath.fromStringUnsafe(s"s3://non-existent-bucket${UUID.randomUUID}/non-existent-object") WsTestClient.withClient { ws => - val s3DataVault = S3DataVault.create(RemoteSourceDescriptor(upath, None), ws)(globalExecutionContext) + val s3DataVault = S3DataVault.create(CredentializedUPath(upath, None), ws)(globalExecutionContext) val vaultPath = new VaultPath(upath, s3DataVault) val result = vaultPath.readBytes()(globalExecutionContext, emptyTokenContext).await(handleFoxJustification) @@ -179,7 +179,7 @@ class DataVaultTestSuite extends PlaySpec { "requesting a non-existent object in existent bucket" in { val upath = UPath.fromStringUnsafe(s"s3://open-neurodata/non-existent-object${UUID.randomUUID}") WsTestClient.withClient { ws => - val s3DataVault = S3DataVault.create(RemoteSourceDescriptor(upath, None), ws)(globalExecutionContext) + val s3DataVault = S3DataVault.create(CredentializedUPath(upath, None), ws)(globalExecutionContext) val vaultPath = new VaultPath(upath, s3DataVault) val result = vaultPath.readBytes()(globalExecutionContext, emptyTokenContext).await(handleFoxJustification) @@ -194,7 +194,7 @@ class DataVaultTestSuite extends PlaySpec { val upath = UPath.fromStringUnsafe("s3://janelia-cosem-datasets/jrc_hela-3/jrc_hela-3.n5/em/fibsem-uint16/") WsTestClient.withClient { ws => val vaultPath = - new VaultPath(upath, S3DataVault.create(RemoteSourceDescriptor(upath, None), ws)(globalExecutionContext)) + new VaultPath(upath, S3DataVault.create(CredentializedUPath(upath, None), ws)(globalExecutionContext)) "using s3 data vault" should { "list available directories" in { @@ -208,7 +208,7 @@ class DataVaultTestSuite extends PlaySpec { "return failure" when { "requesting directory listing on non-existent bucket" in { val upath = UPath.fromStringUnsafe(f"s3://non-existent-bucket${UUID.randomUUID}/non-existent-object/") - val s3DataVault = S3DataVault.create(RemoteSourceDescriptor(upath, None), ws)(globalExecutionContext) + val s3DataVault = S3DataVault.create(CredentializedUPath(upath, None), ws)(globalExecutionContext) val vaultPath = new VaultPath(upath, s3DataVault) val result = vaultPath.listDirectory(maxItems = 5)(globalExecutionContext).await(handleFoxJustification) assertBoxFailure(result) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/DataStoreModule.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/DataStoreModule.scala index f5630730bb2..920eeff0cf3 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/DataStoreModule.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/DataStoreModule.scala @@ -28,7 +28,7 @@ import com.scalableminds.webknossos.datastore.services.segmentindex.{ ZarrSegmentIndexFileService } import com.scalableminds.webknossos.datastore.services.uploading.UploadService -import com.scalableminds.webknossos.datastore.storage.{DataVaultService, RemoteSourceDescriptorService} +import com.scalableminds.webknossos.datastore.storage.DataVaultService class DataStoreModule extends AbstractModule { @@ -60,7 +60,6 @@ class DataStoreModule extends AbstractModule { bind(classOf[ZarrConnectomeFileService]).asEagerSingleton() bind(classOf[Hdf5ConnectomeFileService]).asEagerSingleton() bind(classOf[NeuroglancerPrecomputedMeshFileService]).asEagerSingleton() - bind(classOf[RemoteSourceDescriptorService]).asEagerSingleton() bind(classOf[ChunkCacheService]).asEagerSingleton() bind(classOf[DatasetCache]).asEagerSingleton() } diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala index 4ace35de6a0..dead44f434c 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala @@ -26,13 +26,13 @@ import com.scalableminds.webknossos.datastore.services.connectome.ConnectomeFile import com.scalableminds.webknossos.datastore.services.mesh.{MeshFileService, MeshMappingHelper} import com.scalableminds.webknossos.datastore.services.segmentindex.SegmentIndexFileService import com.scalableminds.webknossos.datastore.services.uploading._ -import com.scalableminds.webknossos.datastore.storage.RemoteSourceDescriptorService import com.scalableminds.webknossos.datastore.services.connectome.{ ByAgglomerateIdsRequest, BySynapseIdsRequest, SynapticPartnerDirection } import com.scalableminds.webknossos.datastore.services.mapping.AgglomerateService +import com.scalableminds.webknossos.datastore.storage.DataVaultService import play.api.data.Form import play.api.data.Forms.{longNumber, nonEmptyText, number, tuple} import play.api.libs.Files @@ -67,7 +67,7 @@ class DataSourceController @Inject()( exploreRemoteLayerService: ExploreRemoteLayerService, uploadService: UploadService, meshFileService: MeshFileService, - remoteSourceDescriptorService: RemoteSourceDescriptorService, + dataVaultService: DataVaultService, val dsRemoteWebknossosClient: DSRemoteWebknossosClient, val dsRemoteTracingstoreClient: DSRemoteTracingstoreClient, )(implicit bodyParsers: PlayBodyParsers, ec: ExecutionContext) @@ -647,7 +647,7 @@ class DataSourceController @Inject()( accessTokenService.validateAccessFromTokenContext(UserAccessRequest.webknossos) { for { _ <- Fox.successful(()) - pathsAllowed = request.body.map(remoteSourceDescriptorService.pathIsAllowedToAddDirectly) + pathsAllowed = request.body.map(dataVaultService.pathIsAllowedToAddDirectly) result = request.body.zip(pathsAllowed).map { case (path, isAllowed) => PathValidationResult(path, isAllowed) } diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/DatasetArrayBucketProvider.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/DatasetArrayBucketProvider.scala index 05b136d7877..4c3426d5254 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/DatasetArrayBucketProvider.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/DatasetArrayBucketProvider.scala @@ -14,7 +14,7 @@ import com.scalableminds.webknossos.datastore.datareaders.zarr3.Zarr3Array import com.scalableminds.webknossos.datastore.datavault.VaultPath import com.scalableminds.webknossos.datastore.models.datasource.{DataFormat, DataSourceId, ElementClass, StaticLayer} import com.scalableminds.webknossos.datastore.models.requests.DataReadInstruction -import com.scalableminds.webknossos.datastore.storage.RemoteSourceDescriptorService +import com.scalableminds.webknossos.datastore.storage.DataVaultService import com.typesafe.scalalogging.LazyLogging import scala.concurrent.duration._ @@ -24,7 +24,7 @@ import scala.concurrent.ExecutionContext class DatasetArrayBucketProvider(dataLayer: StaticLayer, dataSourceId: DataSourceId, - remoteSourceDescriptorServiceOpt: Option[RemoteSourceDescriptorService], + dataVaultServiceOpt: Option[DataVaultService], sharedChunkContentsCacheOpt: Option[AlfuCache[String, MultiArray]]) extends BucketProvider with FoxImplicits @@ -69,13 +69,13 @@ class DatasetArrayBucketProvider(dataLayer: StaticLayer, magLocatorOpt match { case None => Fox.empty case Some(magLocator) => - remoteSourceDescriptorServiceOpt match { - case Some(remoteSourceDescriptorService: RemoteSourceDescriptorService) => + dataVaultServiceOpt match { + case Some(dataVaultServiceOpt: DataVaultService) => for { - magPath: VaultPath <- remoteSourceDescriptorService.vaultPathFor(readInstruction.baseDir, - readInstruction.dataSourceId, - readInstruction.dataLayer.name, - magLocator) + magPath: VaultPath <- dataVaultServiceOpt.vaultPathFor(readInstruction.baseDir, + readInstruction.dataSourceId, + readInstruction.dataLayer.name, + magLocator) chunkContentsCache <- sharedChunkContentsCacheOpt.toFox datasetArray <- dataLayer.dataFormat match { case DataFormat.zarr => diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datavault/GoogleCloudDataVault.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datavault/GoogleCloudDataVault.scala index 9f994dc1475..ff2e062360a 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datavault/GoogleCloudDataVault.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datavault/GoogleCloudDataVault.scala @@ -4,7 +4,7 @@ import com.google.auth.oauth2.ServiceAccountCredentials import com.google.cloud.storage.{BlobId, BlobInfo, Storage, StorageException, StorageOptions} import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.tools.{Fox, FoxImplicits} -import com.scalableminds.webknossos.datastore.storage.{GoogleServiceAccountCredential, RemoteSourceDescriptor} +import com.scalableminds.webknossos.datastore.storage.{GoogleServiceAccountCredential, CredentializedUPath} import com.scalableminds.util.tools.Box.tryo import com.scalableminds.webknossos.datastore.helpers.UPath import org.apache.commons.lang3.builder.HashCodeBuilder @@ -114,8 +114,8 @@ class GoogleCloudDataVault(uri: URI, credential: Option[GoogleServiceAccountCred } object GoogleCloudDataVault { - def create(remoteSourceDescriptor: RemoteSourceDescriptor): GoogleCloudDataVault = { - val credential = remoteSourceDescriptor.credential.map(f => f.asInstanceOf[GoogleServiceAccountCredential]) - new GoogleCloudDataVault(remoteSourceDescriptor.toUriUnsafe, credential) + def create(credentializedUpath: CredentializedUPath): GoogleCloudDataVault = { + val credential = credentializedUpath.credential.map(f => f.asInstanceOf[GoogleServiceAccountCredential]) + new GoogleCloudDataVault(credentializedUpath.upath.toRemoteUriUnsafe, credential) } } diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datavault/HttpsDataVault.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datavault/HttpsDataVault.scala index a5bce9463b3..d4708a93dee 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datavault/HttpsDataVault.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datavault/HttpsDataVault.scala @@ -7,7 +7,7 @@ import com.scalableminds.webknossos.datastore.storage.{ DataVaultCredential, HttpBasicAuthCredential, LegacyDataVaultCredential, - RemoteSourceDescriptor + CredentializedUPath } import com.typesafe.scalalogging.LazyLogging import org.apache.commons.lang3.builder.HashCodeBuilder @@ -158,11 +158,11 @@ object HttpsDataVault { /** * Factory method to create a new HttpsDataVault instance. - * @param remoteSourceDescriptor + * @param credentializedUpath * @param ws * @param dataStoreHost The host of the local data store that this vault is accessing. This is used to determine if a user token should be applied in requests. * @return */ - def create(remoteSourceDescriptor: RemoteSourceDescriptor, ws: WSClient, dataStoreHost: String): HttpsDataVault = - new HttpsDataVault(remoteSourceDescriptor.credential, ws, dataStoreHost) + def create(credentializedUpath: CredentializedUPath, ws: WSClient, dataStoreHost: String): HttpsDataVault = + new HttpsDataVault(credentializedUpath.credential, ws, dataStoreHost) } diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datavault/S3DataVault.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datavault/S3DataVault.scala index 02141d47325..9dd35053832 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datavault/S3DataVault.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/datavault/S3DataVault.scala @@ -4,7 +4,7 @@ import com.scalableminds.util.accesscontext.TokenContext import com.scalableminds.util.tools.{Fox, FoxImplicits} import com.scalableminds.webknossos.datastore.storage.{ LegacyDataVaultCredential, - RemoteSourceDescriptor, + CredentializedUPath, S3AccessKeyCredential } import com.scalableminds.util.tools.Box.tryo @@ -182,14 +182,13 @@ class S3DataVault(s3AccessKeyCredential: Option[S3AccessKeyCredential], } object S3DataVault { - def create(remoteSourceDescriptor: RemoteSourceDescriptor, ws: WSClient)( - implicit ec: ExecutionContext): S3DataVault = { - val credential = remoteSourceDescriptor.credential.flatMap { + def create(credentializedUpath: CredentializedUPath, ws: WSClient)(implicit ec: ExecutionContext): S3DataVault = { + val credential = credentializedUpath.credential.flatMap { case f: S3AccessKeyCredential => Some(f) case f: LegacyDataVaultCredential => Some(f.toS3AccessKey) case _ => None } - new S3DataVault(credential, remoteSourceDescriptor.toUriUnsafe, ws, ec) + new S3DataVault(credential, credentializedUpath.upath.toRemoteUriUnsafe, ws, ec) } private def hostBucketFromUri(uri: URI): Option[String] = { diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/ExploreLocalLayerService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/ExploreLocalLayerService.scala index 937af86d2ca..44e5da48e15 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/ExploreLocalLayerService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/ExploreLocalLayerService.scala @@ -6,7 +6,7 @@ import com.scalableminds.util.io.PathUtils import com.scalableminds.util.tools.{Fox, FoxImplicits, JsonHelper} import com.scalableminds.webknossos.datastore.datareaders.n5.N5Header import com.scalableminds.webknossos.datastore.models.datasource.{DataSourceId, StaticLayer, UsableDataSource} -import com.scalableminds.webknossos.datastore.storage.{DataVaultService, RemoteSourceDescriptor} +import com.scalableminds.webknossos.datastore.storage.DataVaultService import com.scalableminds.util.tools.Box.tryo import com.scalableminds.webknossos.datastore.helpers.UPath import play.api.libs.json.Json @@ -44,11 +44,10 @@ class ExploreLocalLayerService @Inject()(dataVaultService: DataVaultService) magDirectories <- tryo(Files.list(path.resolve(layerDirectory)).iterator().asScala.toList).toFox ?~> s"Could not resolve color directory as child of $path" layersWithVoxelSizes <- Fox.combined(magDirectories.map(dir => for { - remoteSourceDescriptor <- Fox.successful(RemoteSourceDescriptor(UPath.fromLocalPath(dir), None)) mag <- Vec3Int .fromMagLiteral(dir.getFileName.toString, allowScalar = true) .toFox ?~> s"invalid mag: ${dir.getFileName}" - vaultPath <- dataVaultService.getVaultPath(remoteSourceDescriptor) ?~> "dataVault.setup.failed" + vaultPath <- dataVaultService.vaultPathFor(dir) ?~> "dataVault.setup.failed" layersWithVoxelSizes <- new ZarrArrayExplorer(mag).explore(vaultPath, None)(TokenContext(None)) } yield layersWithVoxelSizes)) (layers, voxelSize) <- adaptLayersAndVoxelSize(layersWithVoxelSizes.flatten, None) @@ -134,9 +133,7 @@ class ExploreLocalLayerService @Inject()(dataVaultService: DataVaultService) val subdirs = Files.list(path).iterator().asScala.toList if (subdirs.size == 1) subdirs.head.getFileName.toString else layerDirectory } else layerDirectory - fullPath = path.resolve(layer) - remoteSourceDescriptor <- Fox.successful(RemoteSourceDescriptor(UPath.fromLocalPath(fullPath), None)) - vaultPath <- dataVaultService.getVaultPath(remoteSourceDescriptor) ?~> "dataVault.setup.failed" + vaultPath <- dataVaultService.vaultPathFor(path.resolve(layer)) ?~> "dataVault.setup.failed" layersWithVoxelSizes <- explorer.explore(vaultPath, None)(TokenContext(None)) (layers, voxelSize) <- adaptLayersAndVoxelSize(layersWithVoxelSizes, None) relativeLayers = makeLayersRelative(layers) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/ExploreRemoteLayerService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/ExploreRemoteLayerService.scala index 7fd5ecf5a57..a42f90d6a92 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/ExploreRemoteLayerService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/ExploreRemoteLayerService.scala @@ -9,7 +9,7 @@ import com.scalableminds.webknossos.datastore.datavault.VaultPath import com.scalableminds.webknossos.datastore.models.VoxelSize import com.scalableminds.webknossos.datastore.models.datasource.{DataSourceId, StaticLayer, UsableDataSource} import com.scalableminds.webknossos.datastore.services.DSRemoteWebknossosClient -import com.scalableminds.webknossos.datastore.storage.{DataVaultCredential, DataVaultService, RemoteSourceDescriptor} +import com.scalableminds.webknossos.datastore.storage.{DataVaultCredential, DataVaultService, CredentializedUPath} import com.typesafe.scalalogging.LazyLogging import com.scalableminds.util.tools.{Box, Empty, Failure, Full} import com.scalableminds.webknossos.datastore.helpers.UPath @@ -85,8 +85,7 @@ class ExploreRemoteLayerService @Inject()(dataVaultService: DataVaultService, .toFox ?~> s"Received invalid URI: $layerUri" _ <- assertLocalPathInWhitelist(upath) credentialOpt: Option[DataVaultCredential] <- Fox.runOptional(credentialId)(remoteWebknossosClient.getCredential) - remoteSource = RemoteSourceDescriptor(upath, credentialOpt) - remotePath <- dataVaultService.getVaultPath(remoteSource) ?~> "dataVault.setup.failed" + remotePath <- dataVaultService.vaultPathFor(CredentializedUPath(upath, credentialOpt)) ?~> "dataVault.setup.failed" layersWithVoxelSizes <- recursivelyExploreRemoteLayerAtPaths( List((remotePath, 0)), credentialId, diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NeuroglancerUriExplorer.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NeuroglancerUriExplorer.scala index 6ff3b16e1bf..ccc18722acd 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NeuroglancerUriExplorer.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/explore/NeuroglancerUriExplorer.scala @@ -7,7 +7,7 @@ import com.scalableminds.webknossos.datastore.datavault.VaultPath import com.scalableminds.webknossos.datastore.models.VoxelSize import com.scalableminds.webknossos.datastore.models.datasource.LayerViewConfiguration.LayerViewConfiguration import com.scalableminds.webknossos.datastore.models.datasource.{LayerViewConfiguration, StaticLayer} -import com.scalableminds.webknossos.datastore.storage.{DataVaultService, RemoteSourceDescriptor} +import com.scalableminds.webknossos.datastore.storage.DataVaultService import com.scalableminds.util.tools.Box.tryo import com.scalableminds.webknossos.datastore.helpers.UPath import play.api.libs.json._ @@ -44,8 +44,7 @@ class NeuroglancerUriExplorer(dataVaultService: DataVaultService)(implicit val e layerType = new URI(source.value).getScheme name <- JsonHelper.as[JsString](obj \ "name").toFox upath <- UPath.fromString(source.value.substring(f"$layerType://".length)).toFox - remoteSourceDescriptor = RemoteSourceDescriptor(upath, None) - remotePath <- dataVaultService.getVaultPath(remoteSourceDescriptor) ?~> "dataVault.setup.failed" + remotePath <- dataVaultService.vaultPathFor(upath) ?~> "dataVault.setup.failed" viewConfiguration = getViewConfig(obj) layer <- exploreLayer(layerType, remotePath, name.value) layerWithViewConfiguration <- assignViewConfiguration(layer, viewConfiguration) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/datasource/DataLayer.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/datasource/DataLayer.scala index acc24ee476f..cf91266662f 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/datasource/DataLayer.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/models/datasource/DataLayer.scala @@ -12,7 +12,7 @@ import com.scalableminds.util.geometry.{BoundingBox, Vec3Int} import com.scalableminds.webknossos.datastore.helpers.UPath import ucar.ma2.{Array => MultiArray} import com.scalableminds.webknossos.datastore.models.datasource.LayerViewConfiguration.LayerViewConfiguration -import com.scalableminds.webknossos.datastore.storage.RemoteSourceDescriptorService +import com.scalableminds.webknossos.datastore.storage.DataVaultService import play.api.libs.json._ trait DataLayer { @@ -22,7 +22,7 @@ trait DataLayer { def resolutions: List[Vec3Int] def elementClass: ElementClass.Value - def bucketProvider(remoteSourceDescriptorServiceOpt: Option[RemoteSourceDescriptorService], + def bucketProvider(dataVaultServiceOpt: Option[DataVaultService], dataSourceId: DataSourceId, sharedChunkContentsCache: Option[AlfuCache[String, MultiArray]]): BucketProvider @@ -70,10 +70,10 @@ trait StaticLayer extends DataLayer { def dataFormat: DataFormat.Value - def bucketProvider(remoteSourceDescriptorServiceOpt: Option[RemoteSourceDescriptorService], + def bucketProvider(dataVaultServiceOpt: Option[DataVaultService], dataSourceId: DataSourceId, sharedChunkContentsCache: Option[AlfuCache[String, MultiArray]]): BucketProvider = - new DatasetArrayBucketProvider(this, dataSourceId, remoteSourceDescriptorServiceOpt, sharedChunkContentsCache) + new DatasetArrayBucketProvider(this, dataSourceId, dataVaultServiceOpt, sharedChunkContentsCache) def bucketProviderCacheKey: String = this.name diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala index 4bc469a4e4f..d1b62dd2527 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataService.scala @@ -21,7 +21,7 @@ import scala.concurrent.ExecutionContext class BinaryDataService(val dataBaseDir: Path, val agglomerateServiceOpt: Option[AgglomerateService], - remoteSourceDescriptorServiceOpt: Option[RemoteSourceDescriptorService], + dataVaultServiceOpt: Option[DataVaultService], sharedChunkContentsCache: Option[AlfuCache[String, MultiArray]], datasetErrorLoggingService: DatasetErrorLoggingService)(implicit ec: ExecutionContext) extends FoxImplicits @@ -82,7 +82,7 @@ class BinaryDataService(val dataBaseDir: Path, r.cuboid.topLeft.toBucket.copy(additionalCoordinates = r.settings.additionalCoordinates), r.settings.version)) bucketProvider = bucketProviderCache.getOrLoadAndPut((dataSourceId, dataLayer.bucketProviderCacheKey))(_ => - dataLayer.bucketProvider(remoteSourceDescriptorServiceOpt, dataSourceId, sharedChunkContentsCache)) + dataLayer.bucketProvider(dataVaultServiceOpt, dataSourceId, sharedChunkContentsCache)) bucketBoxes <- datasetErrorLoggingService.withErrorLoggingMultiple( dataSourceId, s"Loading ${requests.length} buckets for $dataSourceId layer ${dataLayer.name}, first request: ${firstRequest.cuboid.topLeft.toBucket}", @@ -195,7 +195,7 @@ class BinaryDataService(val dataBaseDir: Path, val dataSourceId = request.dataSourceIdOrVolumeDummy val bucketProvider = bucketProviderCache.getOrLoadAndPut((dataSourceId, request.dataLayer.bucketProviderCacheKey))(_ => - request.dataLayer.bucketProvider(remoteSourceDescriptorServiceOpt, dataSourceId, sharedChunkContentsCache)) + request.dataLayer.bucketProvider(dataVaultServiceOpt, dataSourceId, sharedChunkContentsCache)) datasetErrorLoggingService.withErrorLogging( dataSourceId, s"loading bucket for layer ${request.dataLayer.name} at ${readInstruction.bucket}, cuboid: ${request.cuboid}", diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataServiceHolder.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataServiceHolder.scala index 6c7ab8de5fa..5af4af5068a 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataServiceHolder.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/BinaryDataServiceHolder.scala @@ -2,7 +2,7 @@ package com.scalableminds.webknossos.datastore.services import com.scalableminds.webknossos.datastore.DataStoreConfig import com.scalableminds.webknossos.datastore.services.mapping.AgglomerateService -import com.scalableminds.webknossos.datastore.storage.RemoteSourceDescriptorService +import com.scalableminds.webknossos.datastore.storage.DataVaultService import javax.inject.Inject import scala.concurrent.ExecutionContext @@ -16,7 +16,7 @@ import scala.concurrent.ExecutionContext */ class BinaryDataServiceHolder @Inject()(config: DataStoreConfig, - remoteSourceDescriptorService: RemoteSourceDescriptorService, + dataVaultService: DataVaultService, datasetErrorLoggingService: DSDatasetErrorLoggingService, chunkCacheService: ChunkCacheService, agglomerateService: AgglomerateService)(implicit ec: ExecutionContext) { @@ -24,7 +24,7 @@ class BinaryDataServiceHolder @Inject()(config: DataStoreConfig, val binaryDataService: BinaryDataService = new BinaryDataService( config.Datastore.baseDirectory, Some(agglomerateService), - Some(remoteSourceDescriptorService), + Some(dataVaultService), Some(chunkCacheService.sharedChunkContentsCache), datasetErrorLoggingService ) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSUsedStorageService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSUsedStorageService.scala index 21512ba8226..63f0583e4f5 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSUsedStorageService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DSUsedStorageService.scala @@ -5,7 +5,7 @@ import com.scalableminds.util.tools.{Fox, FoxImplicits, Full} import com.scalableminds.webknossos.datastore.DataStoreConfig import com.scalableminds.webknossos.datastore.helpers.UPath import com.typesafe.scalalogging.LazyLogging -import com.scalableminds.webknossos.datastore.storage.RemoteSourceDescriptorService +import com.scalableminds.webknossos.datastore.storage.DataVaultService import play.api.libs.json.{Json, OFormat} import javax.inject.Inject @@ -31,8 +31,7 @@ object PathStorageUsageResponse { case class PathPair(original: String, upath: UPath) -class DSUsedStorageService @Inject()(config: DataStoreConfig, - remoteSourceDescriptorService: RemoteSourceDescriptorService) +class DSUsedStorageService @Inject()(config: DataStoreConfig, dataVaultService: DataVaultService) extends FoxImplicits with LazyLogging { @@ -65,7 +64,7 @@ class DSUsedStorageService @Inject()(config: DataStoreConfig, .map(registeredPath => path.upath.startsWith(registeredPath)) .getOrElse(false))) vaultPathsForPathPairsToMeasure <- Fox.serialCombined(pathPairsToMeasure)(pathPair => - remoteSourceDescriptorService.vaultPathFor(pathPair.upath)) + dataVaultService.vaultPathFor(pathPair.upath)) usedBytes <- Fox.fromFuture( Fox.serialSequence(vaultPathsForPathPairsToMeasure)(vaultPath => vaultPath.getUsedStorageBytes)) pathPairsWithStorageUsedBox = pathPairsToMeasure.zip(usedBytes) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DataSourceService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DataSourceService.scala index b636e448256..e8de4cf244e 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DataSourceService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DataSourceService.scala @@ -10,7 +10,7 @@ import com.scalableminds.webknossos.datastore.DataStoreConfig import com.scalableminds.webknossos.datastore.dataformats.{MagLocator, MappingProvider} import com.scalableminds.webknossos.datastore.helpers.{DatasetDeleter, IntervalScheduler, UPath} import com.scalableminds.webknossos.datastore.models.datasource._ -import com.scalableminds.webknossos.datastore.storage.RemoteSourceDescriptorService +import com.scalableminds.webknossos.datastore.storage.DataVaultService import com.typesafe.scalalogging.LazyLogging import com.scalableminds.util.tools.Box.tryo import com.scalableminds.util.tools._ @@ -24,7 +24,7 @@ import scala.concurrent.duration._ class DataSourceService @Inject()( config: DataStoreConfig, - remoteSourceDescriptorService: RemoteSourceDescriptorService, + dataVaultService: DataVaultService, val remoteWebknossosClient: DSRemoteWebknossosClient, val lifecycle: ApplicationLifecycle, @Named("webknossos-datastore") val actorSystem: ActorSystem @@ -130,7 +130,12 @@ class DataSourceService @Inject()( absoluteRawLayerPath: Path, dataLayer: DataLayer, mag: MagLocator) = { - val resolvedMagPath = resolveMagPath(absoluteDatasetPath, absoluteRealLayerPath, mag) + val resolvedMagPath = dataVaultService.resolveMagPath( + absoluteDatasetPath, + absoluteRealLayerPath, + absoluteRealLayerPath.getFileName.toString, + mag + ) if (resolvedMagPath.isRemote) { MagPathInfo(dataLayer.name, mag.mag, resolvedMagPath, resolvedMagPath, hasLocalData = false) } else { @@ -147,14 +152,6 @@ class DataSourceService @Inject()( } } - private def resolveMagPath(datasetPath: Path, layerPath: Path, mag: MagLocator): UPath = - remoteSourceDescriptorService.resolveMagPath( - datasetPath, - layerPath, - layerPath.getFileName.toString, - mag - ) - private def resolveRelativePath(basePath: Path, relativePath: Path): Path = if (relativePath.isAbsolute) { relativePath @@ -276,9 +273,9 @@ class DataSourceService @Inject()( dataLayerOpt <- dataLayers dataLayer <- dataLayerOpt _ = dataLayer.mags.foreach(mag => - remoteSourceDescriptorService.removeVaultFromCache(dataBaseDir, dataSource.id, dataLayer.name, mag)) + dataVaultService.removeVaultFromCache(dataBaseDir, dataSource.id, dataLayer.name, mag)) _ = dataLayer.attachments.foreach(_.allAttachments.foreach(attachment => - remoteSourceDescriptorService.removeVaultFromCache(attachment))) + dataVaultService.removeVaultFromCache(attachment))) } yield dataLayer.mags.length } yield removedEntriesList.sum diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/connectome/ZarrConnectomeFileService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/connectome/ZarrConnectomeFileService.scala index 48cd0e2ebd7..dd69da8515a 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/connectome/ZarrConnectomeFileService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/connectome/ZarrConnectomeFileService.scala @@ -10,7 +10,7 @@ import com.scalableminds.webknossos.datastore.datareaders.zarr3.Zarr3Array import com.scalableminds.webknossos.datastore.models.datasource.DataSourceId import com.scalableminds.webknossos.datastore.services.{ChunkCacheService, VoxelyticsZarrArtifactUtils} import com.scalableminds.webknossos.datastore.services.connectome.SynapticPartnerDirection.SynapticPartnerDirection -import com.scalableminds.webknossos.datastore.storage.RemoteSourceDescriptorService +import com.scalableminds.webknossos.datastore.storage.DataVaultService import jakarta.inject.Inject import play.api.libs.json.{JsResult, JsValue, Reads} @@ -41,8 +41,7 @@ object ConnectomeFileAttributes extends VoxelyticsZarrArtifactUtils with Connect } } -class ZarrConnectomeFileService @Inject()(remoteSourceDescriptorService: RemoteSourceDescriptorService, - chunkCacheService: ChunkCacheService) +class ZarrConnectomeFileService @Inject()(dataVaultService: DataVaultService, chunkCacheService: ChunkCacheService) extends FoxImplicits with ConnectomeFileUtils { private lazy val openArraysCache = AlfuCache[(ConnectomeFileKey, String), DatasetArray]() @@ -55,7 +54,7 @@ class ZarrConnectomeFileService @Inject()(remoteSourceDescriptorService: RemoteS connectomeFileKey, _ => for { - groupVaultPath <- remoteSourceDescriptorService.vaultPathFor(connectomeFileKey.attachment) + groupVaultPath <- dataVaultService.vaultPathFor(connectomeFileKey.attachment) groupHeaderBytes <- (groupVaultPath / ConnectomeFileAttributes.FILENAME_ZARR_JSON) .readBytes() ?~> "Could not read connectome file zarr group file." connectomeFileAttributes <- JsonHelper @@ -180,7 +179,7 @@ class ZarrConnectomeFileService @Inject()(remoteSourceDescriptorService: RemoteS (connectomeFileKey, zarrArrayName), _ => for { - groupVaultPath <- remoteSourceDescriptorService.vaultPathFor(connectomeFileKey.attachment) + groupVaultPath <- dataVaultService.vaultPathFor(connectomeFileKey.attachment) zarrArray <- Zarr3Array.open(groupVaultPath / zarrArrayName, DataSourceId("dummy", "unused"), "layer", diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mapping/ZarrAgglomerateService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mapping/ZarrAgglomerateService.scala index 75950cb4fdb..6f7f0fb397a 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mapping/ZarrAgglomerateService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mapping/ZarrAgglomerateService.scala @@ -14,7 +14,7 @@ import com.scalableminds.webknossos.datastore.geometry.Vec3IntProto import com.scalableminds.webknossos.datastore.helpers.{NativeBucketScanner, NodeDefaults, SkeletonTracingDefaults} import com.scalableminds.webknossos.datastore.models.datasource.{DataSourceId, ElementClass} import com.scalableminds.webknossos.datastore.services.{ChunkCacheService, DataConverter} -import com.scalableminds.webknossos.datastore.storage.{AgglomerateFileKey, RemoteSourceDescriptorService} +import com.scalableminds.webknossos.datastore.storage.{AgglomerateFileKey, DataVaultService} import com.typesafe.scalalogging.LazyLogging import ucar.ma2.{Array => MultiArray} @@ -24,7 +24,7 @@ import scala.collection.compat.immutable.ArraySeq import scala.concurrent.ExecutionContext class ZarrAgglomerateService @Inject()(config: DataStoreConfig, - remoteSourceDescriptorService: RemoteSourceDescriptorService, + dataVaultService: DataVaultService, chunkCacheService: ChunkCacheService) extends DataConverter with AgglomerateFileUtils @@ -51,7 +51,7 @@ class ZarrAgglomerateService @Inject()(config: DataStoreConfig, private def openZarrArray(agglomerateFileKey: AgglomerateFileKey, zarrArrayName: String)(implicit ec: ExecutionContext, tc: TokenContext): Fox[DatasetArray] = for { - groupVaultPath <- remoteSourceDescriptorService.vaultPathFor(agglomerateFileKey.attachment) + groupVaultPath <- dataVaultService.vaultPathFor(agglomerateFileKey.attachment) segmentToAgglomeratePath = groupVaultPath / zarrArrayName zarrArray <- Zarr3Array.open(segmentToAgglomeratePath, DataSourceId("dummy", "unused"), diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/NeuroglancerPrecomputedMeshFileService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/NeuroglancerPrecomputedMeshFileService.scala index 26f93e93c1b..3f54814d238 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/NeuroglancerPrecomputedMeshFileService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/NeuroglancerPrecomputedMeshFileService.scala @@ -7,7 +7,7 @@ import com.scalableminds.util.tools.{Fox, FoxImplicits} import com.scalableminds.webknossos.datastore.datareaders.precomputed.ShardingSpecification import com.scalableminds.webknossos.datastore.datavault.VaultPath import com.scalableminds.webknossos.datastore.models.datasource.DataSourceId -import com.scalableminds.webknossos.datastore.storage.RemoteSourceDescriptorService +import com.scalableminds.webknossos.datastore.storage.DataVaultService import play.api.libs.json.{Json, OFormat} import javax.inject.Inject @@ -23,7 +23,7 @@ object NeuroglancerPrecomputedMeshInfo { implicit val jsonFormat: OFormat[NeuroglancerPrecomputedMeshInfo] = Json.format[NeuroglancerPrecomputedMeshInfo] } -class NeuroglancerPrecomputedMeshFileService @Inject()(remoteSourceDescriptorService: RemoteSourceDescriptorService)( +class NeuroglancerPrecomputedMeshFileService @Inject()(dataVaultService: DataVaultService)( implicit ec: ExecutionContext) extends FoxImplicits with NeuroglancerMeshHelper { @@ -32,7 +32,7 @@ class NeuroglancerPrecomputedMeshFileService @Inject()(remoteSourceDescriptorSer private def loadRemoteMeshInfo(meshFileKey: MeshFileKey)(implicit tc: TokenContext): Fox[NeuroglancerMesh] = for { - vaultPath <- remoteSourceDescriptorService.vaultPathFor(meshFileKey.attachment) + vaultPath <- dataVaultService.vaultPathFor(meshFileKey.attachment) meshInfoPath = vaultPath / NeuroglancerMesh.FILENAME_INFO meshInfo <- meshInfoPath.parseAsJson[NeuroglancerPrecomputedMeshInfo] ?~> "Failed to read mesh info" _ <- Fox.fromBool(meshInfo.transform.length == 12) ?~> "Invalid mesh info: transform has to be of length 12" @@ -75,7 +75,7 @@ class NeuroglancerPrecomputedMeshFileService @Inject()(remoteSourceDescriptorSer def listMeshChunksForMultipleSegments(meshFileKey: MeshFileKey, segmentId: Seq[Long])( implicit tc: TokenContext): Fox[WebknossosSegmentInfo] = for { - vaultPath <- remoteSourceDescriptorService.vaultPathFor(meshFileKey.attachment) + vaultPath <- dataVaultService.vaultPathFor(meshFileKey.attachment) mesh <- meshInfoCache.getOrLoad(meshFileKey, loadRemoteMeshInfo) chunkScale = Array.fill(3)(1 / math.pow(2, mesh.meshInfo.vertex_quantization_bits)) meshSegmentInfos <- Fox.serialCombined(segmentId)(id => listMeshChunks(vaultPath, mesh, id)) @@ -104,7 +104,7 @@ class NeuroglancerPrecomputedMeshFileService @Inject()(remoteSourceDescriptorSer def readMeshChunk(meshFileKey: MeshFileKey, meshChunkDataRequests: Seq[MeshChunkDataRequest])( implicit tc: TokenContext): Fox[(Array[Byte], String)] = for { - vaultPath <- remoteSourceDescriptorService.vaultPathFor(meshFileKey.attachment) + vaultPath <- dataVaultService.vaultPathFor(meshFileKey.attachment) segmentId <- meshChunkDataRequests.head.segmentId.toFox ?~> "Segment id parameter is required" _ <- Fox.fromBool(meshChunkDataRequests.flatMap(_.segmentId).distinct.length == 1) ?~> "All requests must have the same segment id" mesh <- meshInfoCache.getOrLoad(meshFileKey, loadRemoteMeshInfo) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/ZarrMeshFileService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/ZarrMeshFileService.scala index e69feda8b32..7150d8ef3fa 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/ZarrMeshFileService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/ZarrMeshFileService.scala @@ -13,7 +13,7 @@ import com.scalableminds.webknossos.datastore.services.{ ChunkCacheService, VoxelyticsZarrArtifactUtils } -import com.scalableminds.webknossos.datastore.storage.RemoteSourceDescriptorService +import com.scalableminds.webknossos.datastore.storage.DataVaultService import play.api.i18n.{Messages, MessagesProvider} import play.api.libs.json.{JsResult, JsValue, Reads} import ucar.ma2.{Array => MultiArray} @@ -59,8 +59,7 @@ object MeshFileAttributes extends MeshFileUtils with VoxelyticsZarrArtifactUtils } } -class ZarrMeshFileService @Inject()(chunkCacheService: ChunkCacheService, - remoteSourceDescriptorService: RemoteSourceDescriptorService) +class ZarrMeshFileService @Inject()(chunkCacheService: ChunkCacheService, dataVaultService: DataVaultService) extends FoxImplicits with MeshFileUtils with NeuroglancerMeshHelper { @@ -71,7 +70,7 @@ class ZarrMeshFileService @Inject()(chunkCacheService: ChunkCacheService, private def readMeshFileAttributesImpl(meshFileKey: MeshFileKey)(implicit ec: ExecutionContext, tc: TokenContext): Fox[MeshFileAttributes] = for { - groupVaultPath <- remoteSourceDescriptorService.vaultPathFor(meshFileKey.attachment) + groupVaultPath <- dataVaultService.vaultPathFor(meshFileKey.attachment) groupHeaderBytes <- (groupVaultPath / MeshFileAttributes.FILENAME_ZARR_JSON) .readBytes() ?~> "Could not read mesh file zarr group file" meshFileAttributes <- JsonHelper @@ -150,7 +149,7 @@ class ZarrMeshFileService @Inject()(chunkCacheService: ChunkCacheService, private def openZarrArrayImpl(meshFileKey: MeshFileKey, zarrArrayName: String)(implicit ec: ExecutionContext, tc: TokenContext): Fox[DatasetArray] = for { - groupVaultPath <- remoteSourceDescriptorService.vaultPathFor(meshFileKey.attachment) + groupVaultPath <- dataVaultService.vaultPathFor(meshFileKey.attachment) zarrArray <- Zarr3Array.open(groupVaultPath / zarrArrayName, DataSourceId("dummy", "unused"), "layer", diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/segmentindex/ZarrSegmentIndexFileService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/segmentindex/ZarrSegmentIndexFileService.scala index 8852fbc19b3..ed39ccb83b6 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/segmentindex/ZarrSegmentIndexFileService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/segmentindex/ZarrSegmentIndexFileService.scala @@ -14,7 +14,7 @@ import com.scalableminds.webknossos.datastore.services.{ VoxelyticsZarrArtifactUtils } import ucar.ma2.{Array => MultiArray} -import com.scalableminds.webknossos.datastore.storage.RemoteSourceDescriptorService +import com.scalableminds.webknossos.datastore.storage.DataVaultService import play.api.libs.json.{JsResult, JsValue, Reads} import javax.inject.Inject @@ -49,8 +49,7 @@ object SegmentIndexFileAttributes extends SegmentIndexFileUtils with VoxelyticsZ } } -class ZarrSegmentIndexFileService @Inject()(remoteSourceDescriptorService: RemoteSourceDescriptorService, - chunkCacheService: ChunkCacheService) +class ZarrSegmentIndexFileService @Inject()(dataVaultService: DataVaultService, chunkCacheService: ChunkCacheService) extends FoxImplicits with SegmentIndexFileUtils { @@ -66,7 +65,7 @@ class ZarrSegmentIndexFileService @Inject()(remoteSourceDescriptorService: Remot implicit ec: ExecutionContext, tc: TokenContext): Fox[SegmentIndexFileAttributes] = for { - groupVaultPath <- remoteSourceDescriptorService.vaultPathFor(segmentIndexFileKey.attachment) + groupVaultPath <- dataVaultService.vaultPathFor(segmentIndexFileKey.attachment) groupHeaderBytes <- (groupVaultPath / SegmentIndexFileAttributes.FILENAME_ZARR_JSON) .readBytes() ?~> "Could not read segment index file zarr group file" segmentIndexFileAttributes <- JsonHelper @@ -127,7 +126,7 @@ class ZarrSegmentIndexFileService @Inject()(remoteSourceDescriptorService: Remot implicit ec: ExecutionContext, tc: TokenContext): Fox[DatasetArray] = for { - groupVaultPath <- remoteSourceDescriptorService.vaultPathFor(segmentIndexFileKey.attachment) + groupVaultPath <- dataVaultService.vaultPathFor(segmentIndexFileKey.attachment) zarrArray <- Zarr3Array.open(groupVaultPath / zarrArrayName, DataSourceId("dummy", "unused"), "layer", diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/uploading/UploadService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/uploading/UploadService.scala index 32eb409e877..a0dcf6e283d 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/uploading/UploadService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/uploading/UploadService.scala @@ -17,7 +17,7 @@ import com.scalableminds.webknossos.datastore.models.UnfinishedUpload import com.scalableminds.webknossos.datastore.models.datasource.UsableDataSource.FILENAME_DATASOURCE_PROPERTIES_JSON import com.scalableminds.webknossos.datastore.models.datasource._ import com.scalableminds.webknossos.datastore.services.{DSRemoteWebknossosClient, DataSourceService} -import com.scalableminds.webknossos.datastore.storage.{DataStoreRedisStore, RemoteSourceDescriptorService} +import com.scalableminds.webknossos.datastore.storage.{DataStoreRedisStore, DataVaultService} import com.typesafe.scalalogging.LazyLogging import org.apache.commons.io.FileUtils import play.api.libs.json.{Json, OFormat, Reads} @@ -92,7 +92,7 @@ object CancelUploadInformation { class UploadService @Inject()(dataSourceService: DataSourceService, runningUploadMetadataStore: DataStoreRedisStore, - remoteSourceDescriptorService: RemoteSourceDescriptorService, + dataVaultService: DataVaultService, exploreLocalLayerService: ExploreLocalLayerService, datasetSymlinkService: DatasetSymlinkService, val remoteWebknossosClient: DSRemoteWebknossosClient)(implicit ec: ExecutionContext) @@ -382,10 +382,8 @@ class UploadService @Inject()(dataSourceService: DataSourceService, private def checkPathsInUploadedDatasourcePropertiesJson(unpackToDir: Path, organizationId: String): Fox[Unit] = { val dataSource = dataSourceService.dataSourceFromDir(unpackToDir, organizationId) for { - _ <- Fox.runOptional(dataSource.toUsable)( - usableDataSource => - Fox.fromBool( - usableDataSource.allExplicitPaths.forall(remoteSourceDescriptorService.pathIsAllowedToAddDirectly))) + _ <- Fox.runOptional(dataSource.toUsable)(usableDataSource => + Fox.fromBool(usableDataSource.allExplicitPaths.forall(dataVaultService.pathIsAllowedToAddDirectly))) } yield () } diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/DataVaultService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/DataVaultService.scala index 652fd3c1de5..6255c6aeb41 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/DataVaultService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/DataVaultService.scala @@ -1,7 +1,8 @@ package com.scalableminds.webknossos.datastore.storage import com.scalableminds.util.cache.AlfuCache -import com.scalableminds.util.tools.Fox +import com.scalableminds.util.tools.Box.tryo +import com.scalableminds.util.tools.{Box, Fox, FoxImplicits, Full} import com.scalableminds.webknossos.datastore.DataStoreConfig import com.scalableminds.webknossos.datastore.datavault.{ DataVault, @@ -12,42 +13,185 @@ import com.scalableminds.webknossos.datastore.datavault.{ VaultPath } import com.typesafe.scalalogging.LazyLogging -import com.scalableminds.util.tools.Full -import com.scalableminds.webknossos.datastore.helpers.PathSchemes +import com.scalableminds.webknossos.datastore.dataformats.MagLocator +import com.scalableminds.webknossos.datastore.helpers.{PathSchemes, UPath} +import com.scalableminds.webknossos.datastore.models.datasource.{DataSourceId, LayerAttachment} +import com.scalableminds.webknossos.datastore.services.DSRemoteWebknossosClient import play.api.libs.ws.WSClient +import java.nio.file.Path import javax.inject.Inject import scala.concurrent.ExecutionContext -class DataVaultService @Inject()(ws: WSClient, config: DataStoreConfig) extends LazyLogging { +case class CredentializedUPath(upath: UPath, credential: Option[DataVaultCredential]) - private val vaultCache: AlfuCache[RemoteSourceDescriptor, DataVault] = +class DataVaultService @Inject()(ws: WSClient, + config: DataStoreConfig, + remoteWebknossosClient: DSRemoteWebknossosClient) + extends LazyLogging + with FoxImplicits { + + // TODO baseDir should be available here, not passed in + + private val vaultCache: AlfuCache[CredentializedUPath, DataVault] = AlfuCache(maxCapacity = 100) - def getVaultPath(remoteSourceDescriptor: RemoteSourceDescriptor)(implicit ec: ExecutionContext): Fox[VaultPath] = + def vaultPathFor(upath: UPath)(implicit ec: ExecutionContext): Fox[VaultPath] = { + val credentialOpt = findGlobalCredentialFor(Some(upath)) + vaultPathFor(CredentializedUPath(upath, credentialOpt)) + } + + def vaultPathFor(localPath: Path)(implicit ec: ExecutionContext): Fox[VaultPath] = + vaultPathFor(UPath.fromLocalPath(localPath)) + + // TODO move magLocator to first argument + def vaultPathFor(baseDir: Path, datasetId: DataSourceId, layerName: String, magLocator: MagLocator)( + implicit ec: ExecutionContext): Fox[VaultPath] = + for { + credentializedUpath <- credentializedUPathForMag(baseDir, datasetId, layerName, magLocator) + vaultPath <- vaultPathFor(credentializedUpath) + } yield vaultPath + + // Note that attachment paths are already resolved with baseDir in local case so we don’t need to do it here. + def vaultPathFor(attachment: LayerAttachment)(implicit ec: ExecutionContext): Fox[VaultPath] = + for { + credentialBox <- credentialFor(attachment).shiftBox + vaultPath <- vaultPathFor(CredentializedUPath(attachment.path, credentialBox.toOption)) + } yield vaultPath + + def removeVaultFromCache(baseDir: Path, datasetId: DataSourceId, layerName: String, magLocator: MagLocator)( + implicit ec: ExecutionContext): Fox[Unit] = + for { + credentializedUpath <- credentializedUPathForMag(baseDir, datasetId, layerName, magLocator) + _ = removeVaultFromCache(credentializedUpath) + } yield () + + def removeVaultFromCache(attachment: LayerAttachment)(implicit ec: ExecutionContext): Fox[Unit] = + for { + credentialBox <- credentialFor(attachment).shiftBox + _ = removeVaultFromCache(CredentializedUPath(attachment.path, credentialBox.toOption)) + } yield () + + private def credentializedUPathForMag( + baseDir: Path, + datasetId: DataSourceId, + layerName: String, + magLocator: MagLocator)(implicit ec: ExecutionContext): Fox[CredentializedUPath] = + for { + credentialBox <- credentialFor(magLocator: MagLocator).shiftBox + resolvedMagPath <- resolveMagPath(baseDir, datasetId, layerName, magLocator).toFox + } yield CredentializedUPath(resolvedMagPath, credentialBox.toOption) + + // TODO move magLocator to first argument + def resolveMagPath(localDatasetDir: Path, layerDir: Path, layerName: String, magLocator: MagLocator): UPath = + magLocator.path match { + case Some(magLocatorPath) => + if (magLocatorPath.isAbsolute) { + magLocatorPath + } else { + // relative local path, resolve in dataset dir + val pathRelativeToDataset = localDatasetDir.resolve(magLocatorPath.toLocalPathUnsafe).normalize + val pathRelativeToLayer = + localDatasetDir.resolve(layerName).resolve(magLocatorPath.toLocalPathUnsafe).normalize + if (pathRelativeToDataset.toFile.exists) { + UPath.fromLocalPath(pathRelativeToDataset) + } else { + UPath.fromLocalPath(pathRelativeToLayer) + } + } + case _ => + val localDirWithScalarMag = layerDir.resolve(magLocator.mag.toMagLiteral(allowScalar = true)) + val localDirWithVec3Mag = layerDir.resolve(magLocator.mag.toMagLiteral()) + if (localDirWithScalarMag.toFile.exists) { + UPath.fromLocalPath(localDirWithScalarMag) + } else { + UPath.fromLocalPath(localDirWithVec3Mag) + } + } + + private def resolveMagPath(dataBaseDir: Path, + dataSourceId: DataSourceId, + layerName: String, + magLocator: MagLocator): Box[UPath] = tryo { + val localDatasetDir = dataBaseDir.resolve(dataSourceId.organizationId).resolve(dataSourceId.directoryName) + val localLayerDir = localDatasetDir.resolve(layerName) + resolveMagPath(localDatasetDir, localLayerDir, layerName, magLocator) + } + + private lazy val globalCredentials = { + val res = config.Datastore.DataVaults.credentials.flatMap { credentialConfig => + new CredentialConfigReader(credentialConfig).getCredential + } + logger.info(s"Parsed ${res.length} global data vault credentials from datastore config.") + res + } + + private def findGlobalCredentialFor(pathOpt: Option[UPath]): Option[DataVaultCredential] = + pathOpt.flatMap(path => globalCredentials.find(c => path.toString.startsWith(c.name))) + + private def credentialFor(magLocator: MagLocator)(implicit ec: ExecutionContext): Fox[DataVaultCredential] = + magLocator.credentialId match { + case Some(credentialId) => + remoteWebknossosClient.getCredential(credentialId) + case None => + magLocator.credentials match { + case Some(credential) => Fox.successful(credential) + case None => findGlobalCredentialFor(magLocator.path).toFox + } + } + + private def credentialFor(attachment: LayerAttachment)(implicit ec: ExecutionContext): Fox[DataVaultCredential] = + attachment.credentialId match { + case Some(credentialId) => + remoteWebknossosClient.getCredential(credentialId) + case None => + findGlobalCredentialFor(Some(attachment.path)).toFox + } + + def pathIsAllowedToAddDirectly(path: UPath): Boolean = + if (path.isLocal) + pathIsDataSourceLocal(path) || pathIsInLocalDirectoryWhitelist(path) + else + !pathMatchesGlobalCredentials(path) + + private def pathIsDataSourceLocal(path: UPath): Boolean = + path.isLocal && { + val workingDir = Path.of(".").toAbsolutePath.normalize + val inWorkingDir = workingDir.resolve(path.toLocalPathUnsafe).toAbsolutePath.normalize + !path.isAbsolute && inWorkingDir.startsWith(workingDir) + } + + private def pathMatchesGlobalCredentials(path: UPath): Boolean = + findGlobalCredentialFor(Some(path)).isDefined + + private def pathIsInLocalDirectoryWhitelist(path: UPath): Boolean = + path.isLocal && + config.Datastore.localDirectoryWhitelist.exists(whitelistEntry => path.toString.startsWith(whitelistEntry)) + + def vaultPathFor(credentializedUpath: CredentializedUPath)(implicit ec: ExecutionContext): Fox[VaultPath] = for { - vault <- vaultCache.getOrLoad(remoteSourceDescriptor, createVault) ?~> "dataVault.setup.failed" - } yield new VaultPath(remoteSourceDescriptor.upath, vault) + vault <- vaultCache.getOrLoad(credentializedUpath, createVault) ?~> "dataVault.setup.failed" + } yield new VaultPath(credentializedUpath.upath, vault) - def removeVaultFromCache(remoteSourceDescriptor: RemoteSourceDescriptor)(implicit ec: ExecutionContext): Fox[Unit] = - Fox.successful(vaultCache.remove(remoteSourceDescriptor)) + private def removeVaultFromCache(credentializedUpath: CredentializedUPath)(implicit ec: ExecutionContext): Fox[Unit] = + Fox.successful(vaultCache.remove(credentializedUpath)) - private def createVault(remoteSource: RemoteSourceDescriptor)(implicit ec: ExecutionContext): Fox[DataVault] = { - val scheme = remoteSource.upath.getScheme + private def createVault(credentializedUpath: CredentializedUPath)(implicit ec: ExecutionContext): Fox[DataVault] = { + val scheme = credentializedUpath.upath.getScheme try { val fs: DataVault = scheme match { - case Some(PathSchemes.schemeGS) => GoogleCloudDataVault.create(remoteSource) - case Some(PathSchemes.schemeS3) => S3DataVault.create(remoteSource, ws) + case Some(PathSchemes.schemeGS) => GoogleCloudDataVault.create(credentializedUpath) + case Some(PathSchemes.schemeS3) => S3DataVault.create(credentializedUpath, ws) case Some(PathSchemes.schemeHttps) | Some(PathSchemes.schemeHttp) => - HttpsDataVault.create(remoteSource, ws, config.Http.uri) + HttpsDataVault.create(credentializedUpath, ws, config.Http.uri) case None => FileSystemDataVault.create case _ => throw new Exception(s"Unknown file system scheme $scheme") } - logger.info(s"Created data vault for ${remoteSource.upath.toString}") + logger.info(s"Created data vault for ${credentializedUpath.upath.toString}") Fox.successful(fs) } catch { case e: Exception => - val msg = s"Creating data vault errored for ${remoteSource.upath.toString}:" + val msg = s"Creating data vault errored for ${credentializedUpath.upath.toString}:" logger.error(msg, e) Fox.failure(msg, Full(e)) } diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/RemoteSourceDescriptorService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/RemoteSourceDescriptorService.scala deleted file mode 100644 index e35afe7d4e3..00000000000 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/RemoteSourceDescriptorService.scala +++ /dev/null @@ -1,160 +0,0 @@ -package com.scalableminds.webknossos.datastore.storage - -import com.scalableminds.util.tools.{Fox, FoxImplicits} -import com.scalableminds.webknossos.datastore.DataStoreConfig -import com.scalableminds.webknossos.datastore.dataformats.MagLocator -import com.scalableminds.webknossos.datastore.datavault.VaultPath -import com.scalableminds.webknossos.datastore.models.datasource.{DataSourceId, LayerAttachment} -import com.scalableminds.webknossos.datastore.services.DSRemoteWebknossosClient -import com.typesafe.scalalogging.LazyLogging -import com.scalableminds.util.tools.Box -import com.scalableminds.util.tools.Box.tryo -import com.scalableminds.webknossos.datastore.helpers.UPath - -import java.net.URI -import java.nio.file.Path -import javax.inject.Inject -import scala.concurrent.ExecutionContext - -case class RemoteSourceDescriptor(upath: UPath, credential: Option[DataVaultCredential]) { - def toUriUnsafe: URI = upath.toRemoteUriUnsafe -} - -class RemoteSourceDescriptorService @Inject()(dSRemoteWebknossosClient: DSRemoteWebknossosClient, - dataStoreConfig: DataStoreConfig, - dataVaultService: DataVaultService) - extends LazyLogging - with FoxImplicits { - - def vaultPathFor(baseDir: Path, datasetId: DataSourceId, layerName: String, magLocator: MagLocator)( - implicit ec: ExecutionContext): Fox[VaultPath] = - for { - remoteSourceDescriptor <- remoteSourceDescriptorFor(baseDir, datasetId, layerName, magLocator) - vaultPath <- dataVaultService.getVaultPath(remoteSourceDescriptor) - } yield vaultPath - - def removeVaultFromCache(baseDir: Path, datasetId: DataSourceId, layerName: String, magLocator: MagLocator)( - implicit ec: ExecutionContext): Fox[Unit] = - for { - remoteSourceDescriptor <- remoteSourceDescriptorFor(baseDir, datasetId, layerName, magLocator) - _ = dataVaultService.removeVaultFromCache(remoteSourceDescriptor) - } yield () - - // Note that attachment paths are already resolved with baseDir in local case so we don’t need to do it here. - def vaultPathFor(attachment: LayerAttachment)(implicit ec: ExecutionContext): Fox[VaultPath] = - for { - credentialBox <- credentialFor(attachment).shiftBox - remoteSourceDescriptor = RemoteSourceDescriptor(attachment.path, credentialBox.toOption) - vaultPath <- dataVaultService.getVaultPath(remoteSourceDescriptor) - } yield vaultPath - - def vaultPathFor(upath: UPath)(implicit ec: ExecutionContext): Fox[VaultPath] = { - val credentialBox = findGlobalCredentialFor(Some(upath)) - val remoteSourceDescriptor = RemoteSourceDescriptor(upath, credentialBox) - dataVaultService.getVaultPath(remoteSourceDescriptor) - } - - def removeVaultFromCache(attachment: LayerAttachment)(implicit ec: ExecutionContext): Fox[Unit] = - for { - credentialBox <- credentialFor(attachment).shiftBox - remoteSourceDescriptor = RemoteSourceDescriptor(attachment.path, credentialBox.toOption) - _ = dataVaultService.removeVaultFromCache(remoteSourceDescriptor) - } yield () - - private def remoteSourceDescriptorFor( - baseDir: Path, - datasetId: DataSourceId, - layerName: String, - magLocator: MagLocator)(implicit ec: ExecutionContext): Fox[RemoteSourceDescriptor] = - for { - credentialBox <- credentialFor(magLocator: MagLocator).shiftBox - uri <- resolveMagPath(baseDir, datasetId, layerName, magLocator).toFox - remoteSource = RemoteSourceDescriptor(UPath.fromStringUnsafe(uri.toString), credentialBox.toOption) - } yield remoteSource - - def resolveMagPath(localDatasetDir: Path, layerDir: Path, layerName: String, magLocator: MagLocator): UPath = - magLocator.path match { - case Some(magLocatorPath) => - if (magLocatorPath.isAbsolute) { - magLocatorPath - } else { - // relative local path, resolve in dataset dir - val pathRelativeToDataset = localDatasetDir.resolve(magLocatorPath.toLocalPathUnsafe).normalize - val pathRelativeToLayer = - localDatasetDir.resolve(layerName).resolve(magLocatorPath.toLocalPathUnsafe).normalize - if (pathRelativeToDataset.toFile.exists) { - UPath.fromLocalPath(pathRelativeToDataset) - } else { - UPath.fromLocalPath(pathRelativeToLayer) - } - } - case _ => - val localDirWithScalarMag = layerDir.resolve(magLocator.mag.toMagLiteral(allowScalar = true)) - val localDirWithVec3Mag = layerDir.resolve(magLocator.mag.toMagLiteral()) - if (localDirWithScalarMag.toFile.exists) { - UPath.fromLocalPath(localDirWithScalarMag) - } else { - UPath.fromLocalPath(localDirWithVec3Mag) - } - } - - private def resolveMagPath(baseDir: Path, - dataSourceId: DataSourceId, - layerName: String, - magLocator: MagLocator): Box[UPath] = tryo { - val localDatasetDir = baseDir.resolve(dataSourceId.organizationId).resolve(dataSourceId.directoryName) - val localLayerDir = localDatasetDir.resolve(layerName) - resolveMagPath(localDatasetDir, localLayerDir, layerName, magLocator) - } - - private lazy val globalCredentials = { - val res = dataStoreConfig.Datastore.DataVaults.credentials.flatMap { credentialConfig => - new CredentialConfigReader(credentialConfig).getCredential - } - logger.info(s"Parsed ${res.length} global data vault credentials from datastore config.") - res - } - - private def findGlobalCredentialFor(pathOpt: Option[UPath]): Option[DataVaultCredential] = - pathOpt.flatMap(path => globalCredentials.find(c => path.toString.startsWith(c.name))) - - private def credentialFor(magLocator: MagLocator)(implicit ec: ExecutionContext): Fox[DataVaultCredential] = - magLocator.credentialId match { - case Some(credentialId) => - dSRemoteWebknossosClient.getCredential(credentialId) - case None => - magLocator.credentials match { - case Some(credential) => Fox.successful(credential) - case None => findGlobalCredentialFor(magLocator.path).toFox - } - } - - private def credentialFor(attachment: LayerAttachment)(implicit ec: ExecutionContext): Fox[DataVaultCredential] = - attachment.credentialId match { - case Some(credentialId) => - dSRemoteWebknossosClient.getCredential(credentialId) - case None => - findGlobalCredentialFor(Some(attachment.path)).toFox - } - - def pathIsAllowedToAddDirectly(path: UPath): Boolean = - if (path.isLocal) - pathIsDataSourceLocal(path) || pathIsInLocalDirectoryWhitelist(path) - else - !pathMatchesGlobalCredentials(path) - - private def pathIsDataSourceLocal(path: UPath): Boolean = - path.isLocal && { - val workingDir = Path.of(".").toAbsolutePath.normalize - val inWorkingDir = workingDir.resolve(path.toLocalPathUnsafe).toAbsolutePath.normalize - !path.isAbsolute && inWorkingDir.startsWith(workingDir) - } - - private def pathMatchesGlobalCredentials(path: UPath): Boolean = - findGlobalCredentialFor(Some(path)).isDefined - - private def pathIsInLocalDirectoryWhitelist(path: UPath): Boolean = - path.isLocal && - dataStoreConfig.Datastore.localDirectoryWhitelist.exists(whitelistEntry => - path.toString.startsWith(whitelistEntry)) -} diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala index 5a1a8b3de28..77f652198ff 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/editablemapping/EditableMappingLayer.scala @@ -14,14 +14,14 @@ import com.scalableminds.webknossos.datastore.models.datasource.{ AdditionalAxis, CoordinateTransformation, DataLayer, - DataSourceId, DataLayerAttachments, + DataSourceId, ElementClass, SegmentationLayer } import ucar.ma2.{Array => MultiArray} import com.scalableminds.webknossos.datastore.models.requests.DataReadInstruction -import com.scalableminds.webknossos.datastore.storage.RemoteSourceDescriptorService +import com.scalableminds.webknossos.datastore.storage.DataVaultService import com.scalableminds.webknossos.tracingstore.annotation.TSAnnotationService import com.typesafe.scalalogging.LazyLogging @@ -84,7 +84,7 @@ case class EditableMappingLayer(name: String, // set to tracing id override def coordinateTransformations: Option[List[CoordinateTransformation]] = None - override def bucketProvider(remoteSourceDescriptorServiceOpt: Option[RemoteSourceDescriptorService], + override def bucketProvider(dataVaultServiceOpt: Option[DataVaultService], dataSourceId: DataSourceId, sharedChunkContentsCache: Option[AlfuCache[String, MultiArray]]): BucketProvider = new EditableMappingBucketProvider(layer = this) diff --git a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingLayer.scala b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingLayer.scala index 4d40964873c..34bb1fdb868 100644 --- a/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingLayer.scala +++ b/webknossos-tracingstore/app/com/scalableminds/webknossos/tracingstore/tracings/volume/VolumeTracingLayer.scala @@ -12,7 +12,7 @@ import com.scalableminds.webknossos.datastore.models.BucketPosition import com.scalableminds.webknossos.datastore.models.datasource.LayerViewConfiguration.LayerViewConfiguration import com.scalableminds.webknossos.datastore.models.datasource._ import com.scalableminds.webknossos.datastore.models.requests.DataReadInstruction -import com.scalableminds.webknossos.datastore.storage.RemoteSourceDescriptorService +import com.scalableminds.webknossos.datastore.storage.DataVaultService import com.scalableminds.webknossos.tracingstore.tracings.{FossilDBClient, TemporaryTracingService} import com.scalableminds.util.tools.Box import ucar.ma2.{Array => MultiArray} @@ -114,7 +114,7 @@ case class VolumeTracingLayer( else new VolumeTracingBucketProvider(this) - override def bucketProvider(remoteSourceDescriptorServiceOpt: Option[RemoteSourceDescriptorService], + override def bucketProvider(dataVaultServiceOpt: Option[DataVaultService], dataSourceId: DataSourceId, sharedChunkContentsCache: Option[AlfuCache[String, MultiArray]]): BucketProvider = volumeBucketProvider From 188d3163430a855b270d327ebaf2d5e78260a62f Mon Sep 17 00:00:00 2001 From: Florian M Date: Tue, 7 Oct 2025 13:48:33 +0200 Subject: [PATCH 2/2] =?UTF-8?q?don=E2=80=99t=20pass=20redundant=20baseDir?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../DatasetArrayBucketProvider.scala | 5 ++- .../services/DataSourceService.scala | 5 ++- .../datastore/storage/DataVaultService.scala | 33 +++++++------------ 3 files changed, 16 insertions(+), 27 deletions(-) diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/DatasetArrayBucketProvider.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/DatasetArrayBucketProvider.scala index 4c3426d5254..6fdea002b47 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/DatasetArrayBucketProvider.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/DatasetArrayBucketProvider.scala @@ -72,10 +72,9 @@ class DatasetArrayBucketProvider(dataLayer: StaticLayer, dataVaultServiceOpt match { case Some(dataVaultServiceOpt: DataVaultService) => for { - magPath: VaultPath <- dataVaultServiceOpt.vaultPathFor(readInstruction.baseDir, + magPath: VaultPath <- dataVaultServiceOpt.vaultPathFor(magLocator, readInstruction.dataSourceId, - readInstruction.dataLayer.name, - magLocator) + readInstruction.dataLayer.name) chunkContentsCache <- sharedChunkContentsCacheOpt.toFox datasetArray <- dataLayer.dataFormat match { case DataFormat.zarr => diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DataSourceService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DataSourceService.scala index e8de4cf244e..128cfc888ad 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DataSourceService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/DataSourceService.scala @@ -131,10 +131,10 @@ class DataSourceService @Inject()( dataLayer: DataLayer, mag: MagLocator) = { val resolvedMagPath = dataVaultService.resolveMagPath( + mag, absoluteDatasetPath, absoluteRealLayerPath, absoluteRealLayerPath.getFileName.toString, - mag ) if (resolvedMagPath.isRemote) { MagPathInfo(dataLayer.name, mag.mag, resolvedMagPath, resolvedMagPath, hasLocalData = false) @@ -272,8 +272,7 @@ class DataSourceService @Inject()( removedEntriesList = for { dataLayerOpt <- dataLayers dataLayer <- dataLayerOpt - _ = dataLayer.mags.foreach(mag => - dataVaultService.removeVaultFromCache(dataBaseDir, dataSource.id, dataLayer.name, mag)) + _ = dataLayer.mags.foreach(mag => dataVaultService.removeVaultFromCache(mag, dataSource.id, dataLayer.name)) _ = dataLayer.attachments.foreach(_.allAttachments.foreach(attachment => dataVaultService.removeVaultFromCache(attachment))) } yield dataLayer.mags.length diff --git a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/DataVaultService.scala b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/DataVaultService.scala index 6255c6aeb41..cf1e3234e31 100644 --- a/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/DataVaultService.scala +++ b/webknossos-datastore/app/com/scalableminds/webknossos/datastore/storage/DataVaultService.scala @@ -31,8 +31,6 @@ class DataVaultService @Inject()(ws: WSClient, extends LazyLogging with FoxImplicits { - // TODO baseDir should be available here, not passed in - private val vaultCache: AlfuCache[CredentializedUPath, DataVault] = AlfuCache(maxCapacity = 100) @@ -44,11 +42,10 @@ class DataVaultService @Inject()(ws: WSClient, def vaultPathFor(localPath: Path)(implicit ec: ExecutionContext): Fox[VaultPath] = vaultPathFor(UPath.fromLocalPath(localPath)) - // TODO move magLocator to first argument - def vaultPathFor(baseDir: Path, datasetId: DataSourceId, layerName: String, magLocator: MagLocator)( + def vaultPathFor(magLocator: MagLocator, dataSourceId: DataSourceId, layerName: String)( implicit ec: ExecutionContext): Fox[VaultPath] = for { - credentializedUpath <- credentializedUPathForMag(baseDir, datasetId, layerName, magLocator) + credentializedUpath <- credentializedUPathForMag(dataSourceId, layerName, magLocator) vaultPath <- vaultPathFor(credentializedUpath) } yield vaultPath @@ -59,10 +56,10 @@ class DataVaultService @Inject()(ws: WSClient, vaultPath <- vaultPathFor(CredentializedUPath(attachment.path, credentialBox.toOption)) } yield vaultPath - def removeVaultFromCache(baseDir: Path, datasetId: DataSourceId, layerName: String, magLocator: MagLocator)( + def removeVaultFromCache(magLocator: MagLocator, datasetId: DataSourceId, layerName: String)( implicit ec: ExecutionContext): Fox[Unit] = for { - credentializedUpath <- credentializedUPathForMag(baseDir, datasetId, layerName, magLocator) + credentializedUpath <- credentializedUPathForMag(datasetId, layerName, magLocator) _ = removeVaultFromCache(credentializedUpath) } yield () @@ -72,18 +69,14 @@ class DataVaultService @Inject()(ws: WSClient, _ = removeVaultFromCache(CredentializedUPath(attachment.path, credentialBox.toOption)) } yield () - private def credentializedUPathForMag( - baseDir: Path, - datasetId: DataSourceId, - layerName: String, - magLocator: MagLocator)(implicit ec: ExecutionContext): Fox[CredentializedUPath] = + private def credentializedUPathForMag(datasetId: DataSourceId, layerName: String, magLocator: MagLocator)( + implicit ec: ExecutionContext): Fox[CredentializedUPath] = for { credentialBox <- credentialFor(magLocator: MagLocator).shiftBox - resolvedMagPath <- resolveMagPath(baseDir, datasetId, layerName, magLocator).toFox + resolvedMagPath <- resolveMagPath(datasetId, layerName, magLocator).toFox } yield CredentializedUPath(resolvedMagPath, credentialBox.toOption) - // TODO move magLocator to first argument - def resolveMagPath(localDatasetDir: Path, layerDir: Path, layerName: String, magLocator: MagLocator): UPath = + def resolveMagPath(magLocator: MagLocator, localDatasetDir: Path, layerDir: Path, layerName: String): UPath = magLocator.path match { case Some(magLocatorPath) => if (magLocatorPath.isAbsolute) { @@ -109,13 +102,11 @@ class DataVaultService @Inject()(ws: WSClient, } } - private def resolveMagPath(dataBaseDir: Path, - dataSourceId: DataSourceId, - layerName: String, - magLocator: MagLocator): Box[UPath] = tryo { - val localDatasetDir = dataBaseDir.resolve(dataSourceId.organizationId).resolve(dataSourceId.directoryName) + private def resolveMagPath(dataSourceId: DataSourceId, layerName: String, magLocator: MagLocator): Box[UPath] = tryo { + val localDatasetDir = + config.Datastore.baseDirectory.resolve(dataSourceId.organizationId).resolve(dataSourceId.directoryName) val localLayerDir = localDatasetDir.resolve(layerName) - resolveMagPath(localDatasetDir, localLayerDir, layerName, magLocator) + resolveMagPath(magLocator, localDatasetDir, localLayerDir, layerName) } private lazy val globalCredentials = {