Skip to content

Commit 192cf21

Browse files
fm3fm3
authored
Don’t re-scan attachments in their list routes (#8829)
With #8708 the datastore uses the datasource properties as stored in postgres. Since wk already scans for attachments not registered in the on-disk datasource-properties.json and writes those into postgres, we don’t need to re-scan when answering the attachment list routes. There we can now rely on the attachment being already registered. ### Steps to test: - Open a local dataset with hdf5 attachments that are not registered in the local datasource-properties.json - They should still be usable. ### Issues: - contributes to #8567 ------ - [x] Considered [common edge cases](../blob/master/.github/common_edge_cases.md) - [x] Needs datastore update after deployment
1 parent 0fd771b commit 192cf21

File tree

5 files changed

+45
-216
lines changed

5 files changed

+45
-216
lines changed

‎webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/DataSourceController.scala‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,14 @@ import com.scalableminds.webknossos.datastore.helpers.{
2121
import com.scalableminds.webknossos.datastore.models.datasource.inbox.InboxDataSource
2222
import com.scalableminds.webknossos.datastore.models.datasource.{DataLayer, DataSource, DataSourceId, GenericDataSource}
2323
import com.scalableminds.webknossos.datastore.services._
24+
import com.scalableminds.webknossos.datastore.services.connectome.ConnectomeFileService
2425
import com.scalableminds.webknossos.datastore.services.mesh.{MeshFileService, MeshMappingHelper}
2526
import com.scalableminds.webknossos.datastore.services.segmentindex.SegmentIndexFileService
2627
import com.scalableminds.webknossos.datastore.services.uploading._
2728
import com.scalableminds.webknossos.datastore.storage.DataVaultService
2829
import com.scalableminds.webknossos.datastore.services.connectome.{
2930
ByAgglomerateIdsRequest,
3031
BySynapseIdsRequest,
31-
ConnectomeFileService,
3232
SynapticPartnerDirection
3333
}
3434
import com.scalableminds.webknossos.datastore.services.mapping.AgglomerateService
@@ -266,8 +266,8 @@ class DataSourceController @Inject()(
266266
): Action[AnyContent] = Action.async { implicit request =>
267267
accessTokenService.validateAccessFromTokenContext(UserAccessRequest.readDataset(datasetId)) {
268268
for {
269-
(dataSource, dataLayer) <- datasetCache.getWithLayer(datasetId, dataLayerName) ~> NOT_FOUND
270-
agglomerateList = agglomerateService.listAgglomeratesFiles(dataSource.id, dataLayer)
269+
(_, dataLayer) <- datasetCache.getWithLayer(datasetId, dataLayerName) ~> NOT_FOUND
270+
agglomerateList = agglomerateService.listAgglomeratesFiles(dataLayer)
271271
} yield Ok(Json.toJson(agglomerateList))
272272
}
273273
}

‎webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/connectome/ConnectomeFileService.scala‎

Lines changed: 10 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -2,24 +2,18 @@ package com.scalableminds.webknossos.datastore.services.connectome
22

33
import com.scalableminds.util.accesscontext.TokenContext
44
import com.scalableminds.util.cache.AlfuCache
5-
import com.scalableminds.util.io.PathUtils
6-
import com.scalableminds.util.tools.Box.tryo
7-
import com.scalableminds.util.tools.{Box, Fox, FoxImplicits}
8-
import com.scalableminds.webknossos.datastore.DataStoreConfig
5+
import com.scalableminds.util.tools.{Fox, FoxImplicits}
96
import com.scalableminds.webknossos.datastore.models.datasource.{
107
DataLayer,
118
DataSourceId,
129
LayerAttachment,
1310
LayerAttachmentDataformat
1411
}
1512
import com.scalableminds.webknossos.datastore.services.connectome.SynapticPartnerDirection.SynapticPartnerDirection
16-
import com.scalableminds.webknossos.datastore.storage.RemoteSourceDescriptorService
1713
import com.typesafe.scalalogging.LazyLogging
18-
import org.apache.commons.io.FilenameUtils
1914
import play.api.i18n.{Messages, MessagesProvider}
2015
import play.api.libs.json.{Json, OFormat}
2116

22-
import java.nio.file.Path
2317
import javax.inject.Inject
2418
import scala.collection.mutable.ListBuffer
2519
import scala.concurrent.ExecutionContext
@@ -82,17 +76,11 @@ object ConnectomeFileNameWithMappingName {
8276

8377
case class ConnectomeFileKey(dataSourceId: DataSourceId, layerName: String, attachment: LayerAttachment)
8478

85-
class ConnectomeFileService @Inject()(config: DataStoreConfig,
86-
remoteSourceDescriptorService: RemoteSourceDescriptorService,
87-
hdf5ConnectomeFileService: Hdf5ConnectomeFileService,
79+
class ConnectomeFileService @Inject()(hdf5ConnectomeFileService: Hdf5ConnectomeFileService,
8880
zarrConnectomeFileService: ZarrConnectomeFileService)
8981
extends FoxImplicits
9082
with LazyLogging {
9183

92-
private val dataBaseDir = Path.of(config.Datastore.baseDirectory)
93-
private val localConnectomesDir = "connectomes"
94-
private val hdf5ConnectomeFileExtension = "hdf5"
95-
9684
private val connectomeFileKeyCache
9785
: AlfuCache[(DataSourceId, String, String), ConnectomeFileKey] = AlfuCache() // dataSourceId, layerName, connectomeFileName → ConnectomeFileKey
9886

@@ -104,60 +92,28 @@ class ConnectomeFileService @Inject()(config: DataStoreConfig,
10492

10593
private def lookUpConnectomeFileKeyImpl(dataSourceId: DataSourceId,
10694
dataLayer: DataLayer,
107-
connectomeFileName: String): Box[ConnectomeFileKey] = {
108-
val registeredAttachment: Option[LayerAttachment] = dataLayer.attachments match {
109-
case Some(attachments) => attachments.connectomes.find(_.name == connectomeFileName)
110-
case None => None
111-
}
112-
val localDatasetDir = dataBaseDir.resolve(dataSourceId.organizationId).resolve(dataSourceId.directoryName)
95+
connectomeFileName: String): Option[ConnectomeFileKey] =
11396
for {
114-
registeredAttachmentNormalized <- tryo(registeredAttachment.map { attachment =>
115-
attachment.copy(
116-
path =
117-
remoteSourceDescriptorService.uriFromPathLiteral(attachment.path.toString, localDatasetDir, dataLayer.name))
118-
})
119-
localFallbackAttachment = LayerAttachment(
120-
connectomeFileName,
121-
localDatasetDir
122-
.resolve(dataLayer.name)
123-
.resolve(localConnectomesDir)
124-
.resolve(connectomeFileName + "." + hdf5ConnectomeFileExtension)
125-
.toUri,
126-
LayerAttachmentDataformat.hdf5
127-
)
128-
selectedAttachment = registeredAttachmentNormalized.getOrElse(localFallbackAttachment)
97+
attachment <- dataLayer.attachments match {
98+
case Some(attachments) => attachments.connectomes.find(_.name == connectomeFileName)
99+
case None => None
100+
}
129101
} yield
130102
ConnectomeFileKey(
131103
dataSourceId,
132104
dataLayer.name,
133-
selectedAttachment
105+
attachment
134106
)
135-
}
136107

137108
def listConnectomeFiles(dataSourceId: DataSourceId, dataLayer: DataLayer)(
138109
implicit ec: ExecutionContext,
139110
tc: TokenContext,
140111
m: MessagesProvider): Fox[List[ConnectomeFileNameWithMappingName]] = {
141-
val attachedConnectomeFileNames = dataLayer.attachments.map(_.connectomes).getOrElse(Seq.empty).map(_.name).toSet
142-
143-
val layerDir =
144-
dataBaseDir.resolve(dataSourceId.organizationId).resolve(dataSourceId.directoryName).resolve(dataLayer.name)
145-
val scannedConnectomeFileNames = PathUtils
146-
.listFiles(layerDir.resolve(localConnectomesDir),
147-
silent = true,
148-
PathUtils.fileExtensionFilter(hdf5ConnectomeFileExtension))
149-
.map { paths =>
150-
paths.map(path => FilenameUtils.removeExtension(path.getFileName.toString))
151-
}
152-
.toOption
153-
.getOrElse(Nil)
154-
.toSet
155-
156-
val allConnectomeFileNames = attachedConnectomeFileNames ++ scannedConnectomeFileNames
112+
val connectomeFileNames = dataLayer.attachments.map(_.connectomes).getOrElse(Seq.empty).map(_.name)
157113

158114
Fox.fromFuture(
159115
Fox
160-
.serialSequence(allConnectomeFileNames.toSeq) { connectomeFileName =>
116+
.serialSequence(connectomeFileNames) { connectomeFileName =>
161117
for {
162118
connectomeFileKey <- lookUpConnectomeFileKey(dataSourceId, dataLayer, connectomeFileName) ?~> Messages(
163119
"connectome.file.lookup.failed",

‎webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mapping/AgglomerateService.scala‎

Lines changed: 13 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -3,60 +3,29 @@ package com.scalableminds.webknossos.datastore.services.mapping
33
import com.scalableminds.util.accesscontext.TokenContext
44
import com.scalableminds.util.cache.AlfuCache
55
import com.scalableminds.util.geometry.Vec3Int
6-
import com.scalableminds.util.io.PathUtils
76
import com.scalableminds.util.time.Instant
8-
import com.scalableminds.util.tools.Box.tryo
9-
import com.scalableminds.util.tools.{Box, Fox, FoxImplicits}
7+
import com.scalableminds.util.tools.{Fox, FoxImplicits}
108
import com.scalableminds.webknossos.datastore.AgglomerateGraph.AgglomerateGraph
11-
import com.scalableminds.webknossos.datastore.DataStoreConfig
129
import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing
13-
import com.scalableminds.webknossos.datastore.models.datasource.{
14-
DataLayer,
15-
DataSourceId,
16-
LayerAttachment,
17-
LayerAttachmentDataformat
18-
}
10+
import com.scalableminds.webknossos.datastore.models.datasource.{DataLayer, DataSourceId, LayerAttachmentDataformat}
1911
import com.scalableminds.webknossos.datastore.models.requests.DataServiceDataRequest
20-
import com.scalableminds.webknossos.datastore.storage.{AgglomerateFileKey, RemoteSourceDescriptorService}
12+
import com.scalableminds.webknossos.datastore.storage.AgglomerateFileKey
2113
import com.typesafe.scalalogging.LazyLogging
22-
import org.apache.commons.io.FilenameUtils
2314

24-
import java.nio.file.Path
2515
import javax.inject.Inject
2616
import scala.concurrent.ExecutionContext
2717
import scala.concurrent.duration.DurationInt
2818

29-
class AgglomerateService @Inject()(config: DataStoreConfig,
30-
zarrAgglomerateService: ZarrAgglomerateService,
31-
hdf5AgglomerateService: Hdf5AgglomerateService,
32-
remoteSourceDescriptorService: RemoteSourceDescriptorService)
19+
class AgglomerateService @Inject()(zarrAgglomerateService: ZarrAgglomerateService,
20+
hdf5AgglomerateService: Hdf5AgglomerateService)
3321
extends LazyLogging
3422
with FoxImplicits {
35-
private val localAgglomeratesDir = "agglomerates"
36-
private val hdf5AgglomerateFileExtension = "hdf5"
37-
private val dataBaseDir = Path.of(config.Datastore.baseDirectory)
3823

3924
private val agglomerateFileKeyCache
4025
: AlfuCache[(DataSourceId, String, String), AgglomerateFileKey] = AlfuCache() // dataSourceId, layerName, mappingName → AgglomerateFileKey
4126

42-
def listAgglomeratesFiles(dataSourceId: DataSourceId, dataLayer: DataLayer): Set[String] = {
43-
val attachedAgglomerateFileNames = dataLayer.attachments.map(_.agglomerates).getOrElse(Seq.empty).map(_.name).toSet
44-
45-
val layerDir =
46-
dataBaseDir.resolve(dataSourceId.organizationId).resolve(dataSourceId.directoryName).resolve(dataLayer.name)
47-
val scannedAgglomerateFileNames = PathUtils
48-
.listFiles(layerDir.resolve(localAgglomeratesDir),
49-
silent = true,
50-
PathUtils.fileExtensionFilter(hdf5AgglomerateFileExtension))
51-
.map { paths =>
52-
paths.map(path => FilenameUtils.removeExtension(path.getFileName.toString))
53-
}
54-
.toOption
55-
.getOrElse(Nil)
56-
.toSet
57-
58-
attachedAgglomerateFileNames ++ scannedAgglomerateFileNames
59-
}
27+
def listAgglomeratesFiles(dataLayer: DataLayer): Seq[String] =
28+
dataLayer.attachments.map(_.agglomerates).getOrElse(Seq.empty).map(_.name)
6029

6130
def clearCaches(dataSourceId: DataSourceId, layerNameOpt: Option[String]): Int = {
6231
agglomerateFileKeyCache.clear {
@@ -83,35 +52,18 @@ class AgglomerateService @Inject()(config: DataStoreConfig,
8352

8453
private def lookUpAgglomerateFileImpl(dataSourceId: DataSourceId,
8554
dataLayer: DataLayer,
86-
mappingName: String): Box[AgglomerateFileKey] = {
87-
val registeredAttachment: Option[LayerAttachment] = dataLayer.attachments match {
88-
case Some(attachments) => attachments.agglomerates.find(_.name == mappingName)
89-
case None => None
90-
}
91-
val localDatasetDir = dataBaseDir.resolve(dataSourceId.organizationId).resolve(dataSourceId.directoryName)
55+
mappingName: String): Option[AgglomerateFileKey] =
9256
for {
93-
registeredAttachmentNormalized <- tryo(registeredAttachment.map { attachment =>
94-
attachment.copy(
95-
path =
96-
remoteSourceDescriptorService.uriFromPathLiteral(attachment.path.toString, localDatasetDir, dataLayer.name))
97-
})
98-
localFallbackAttachment = LayerAttachment(
99-
mappingName,
100-
localDatasetDir
101-
.resolve(dataLayer.name)
102-
.resolve(localAgglomeratesDir)
103-
.resolve(mappingName + "." + hdf5AgglomerateFileExtension)
104-
.toUri,
105-
LayerAttachmentDataformat.hdf5
106-
)
107-
selectedAttachment = registeredAttachmentNormalized.getOrElse(localFallbackAttachment)
57+
attachment <- dataLayer.attachments match {
58+
case Some(attachments) => attachments.agglomerates.find(_.name == mappingName)
59+
case None => None
60+
}
10861
} yield
10962
AgglomerateFileKey(
11063
dataSourceId,
11164
dataLayer.name,
112-
selectedAttachment
65+
attachment
11366
)
114-
}
11567

11668
def applyAgglomerate(request: DataServiceDataRequest)(data: Array[Byte])(implicit ec: ExecutionContext,
11769
tc: TokenContext): Fox[Array[Byte]] =

‎webknossos-datastore/app/com/scalableminds/webknossos/datastore/services/mesh/MeshFileService.scala‎

Lines changed: 11 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -2,24 +2,16 @@ package com.scalableminds.webknossos.datastore.services.mesh
22

33
import com.scalableminds.util.accesscontext.TokenContext
44
import com.scalableminds.util.cache.AlfuCache
5-
import com.scalableminds.util.io.PathUtils
65
import com.scalableminds.util.tools.{Fox, FoxImplicits}
7-
import com.scalableminds.webknossos.datastore.DataStoreConfig
86
import com.scalableminds.webknossos.datastore.models.datasource.{
97
DataLayer,
108
DataSourceId,
119
LayerAttachment,
1210
LayerAttachmentDataformat
1311
}
14-
import com.scalableminds.webknossos.datastore.services.ArrayArtifactHashing
15-
import com.scalableminds.webknossos.datastore.storage.RemoteSourceDescriptorService
16-
import com.scalableminds.util.tools.Box.tryo
17-
import com.scalableminds.util.tools.Box
18-
import org.apache.commons.io.FilenameUtils
1912
import play.api.i18n.{Messages, MessagesProvider}
2013
import play.api.libs.json.{Json, OFormat}
2114

22-
import java.nio.file.Path
2315
import javax.inject.Inject
2416
import scala.concurrent.ExecutionContext
2517

@@ -64,17 +56,10 @@ object MeshFileInfo {
6456
implicit val jsonFormat: OFormat[MeshFileInfo] = Json.format[MeshFileInfo]
6557
}
6658

67-
class MeshFileService @Inject()(config: DataStoreConfig,
68-
hdf5MeshFileService: Hdf5MeshFileService,
59+
class MeshFileService @Inject()(hdf5MeshFileService: Hdf5MeshFileService,
6960
zarrMeshFileService: ZarrMeshFileService,
70-
neuroglancerPrecomputedMeshService: NeuroglancerPrecomputedMeshFileService,
71-
remoteSourceDescriptorService: RemoteSourceDescriptorService)
72-
extends FoxImplicits
73-
with ArrayArtifactHashing {
74-
75-
private val dataBaseDir = Path.of(config.Datastore.baseDirectory)
76-
private val localMeshesDir = "meshes"
77-
private val hdf5MeshFileExtension = "hdf5"
61+
neuroglancerPrecomputedMeshService: NeuroglancerPrecomputedMeshFileService)
62+
extends FoxImplicits {
7863

7964
private val meshFileKeyCache
8065
: AlfuCache[(DataSourceId, String, String), MeshFileKey] = AlfuCache() // dataSourceId, layerName, meshFileName → MeshFileKey
@@ -86,57 +71,27 @@ class MeshFileService @Inject()(config: DataStoreConfig,
8671

8772
private def lookUpMeshFileKeyImpl(dataSourceId: DataSourceId,
8873
dataLayer: DataLayer,
89-
meshFileName: String): Box[MeshFileKey] = {
90-
val registeredAttachment: Option[LayerAttachment] = dataLayer.attachments match {
91-
case Some(attachments) => attachments.meshes.find(_.name == meshFileName)
92-
case None => None
93-
}
94-
val localDatasetDir = dataBaseDir.resolve(dataSourceId.organizationId).resolve(dataSourceId.directoryName)
74+
meshFileName: String): Option[MeshFileKey] =
9575
for {
96-
registeredAttachmentNormalized <- tryo(registeredAttachment.map { attachment =>
97-
attachment.copy(
98-
path =
99-
remoteSourceDescriptorService.uriFromPathLiteral(attachment.path.toString, localDatasetDir, dataLayer.name))
100-
})
101-
localFallbackAttachment = LayerAttachment(
102-
meshFileName,
103-
localDatasetDir
104-
.resolve(dataLayer.name)
105-
.resolve(localMeshesDir)
106-
.resolve(meshFileName + "." + hdf5MeshFileExtension)
107-
.toUri,
108-
LayerAttachmentDataformat.hdf5
109-
)
110-
selectedAttachment = registeredAttachmentNormalized.getOrElse(localFallbackAttachment)
76+
registeredAttachment <- dataLayer.attachments match {
77+
case Some(attachments) => attachments.meshes.find(_.name == meshFileName)
78+
case None => None
79+
}
11180
} yield
11281
MeshFileKey(
11382
dataSourceId,
11483
dataLayer.name,
115-
selectedAttachment
84+
registeredAttachment
11685
)
117-
}
11886

11987
def listMeshFiles(dataSourceId: DataSourceId, dataLayer: DataLayer)(implicit ec: ExecutionContext,
12088
tc: TokenContext,
12189
m: MessagesProvider): Fox[Seq[MeshFileInfo]] = {
122-
val attachedMeshFileNames = dataLayer.attachments.map(_.meshes).getOrElse(Seq.empty).map(_.name).toSet
123-
124-
val layerDir =
125-
dataBaseDir.resolve(dataSourceId.organizationId).resolve(dataSourceId.directoryName).resolve(dataLayer.name)
126-
val scannedMeshFileNames = PathUtils
127-
.listFiles(layerDir.resolve(localMeshesDir), silent = true, PathUtils.fileExtensionFilter(hdf5MeshFileExtension))
128-
.map { paths =>
129-
paths.map(path => FilenameUtils.removeExtension(path.getFileName.toString))
130-
}
131-
.toOption
132-
.getOrElse(Nil)
133-
.toSet
134-
135-
val allMeshFileNames = attachedMeshFileNames ++ scannedMeshFileNames
90+
val meshFileNames = dataLayer.attachments.map(_.meshes).getOrElse(Seq.empty).map(_.name)
13691

13792
Fox.fromFuture(
13893
Fox
139-
.serialSequence(allMeshFileNames.toSeq) { meshFileName =>
94+
.serialSequence(meshFileNames) { meshFileName =>
14095
for {
14196
meshFileKey <- lookUpMeshFileKey(dataSourceId, dataLayer, meshFileName) ?~> Messages(
14297
"mesh.file.lookup.failed",

0 commit comments

Comments
 (0)