Virtual Datasets (#8708)

Co-authored-by: Michael Büßemeyer <[email protected]>

Author: frcroth

app/controllers/AnnotationIOController.scala

@@ -137,7 +137,8 @@ class AnnotationIOController @Inject()(
                                                          volumeLayersGrouped,
                                                          tracingStoreClient,
                                                          parsedFiles.otherFiles,
-                                                         usableDataSource)
+                                                         usableDataSource,
+                                                         dataset._id)
           mergedSkeletonLayers <- mergeAndSaveSkeletonLayers(skeletonTracings, tracingStoreClient)
           annotation <- annotationService.createFrom(request.identity,
                                                      dataset,
@@ -167,7 +168,8 @@ class AnnotationIOController @Inject()(
                                        volumeLayersGrouped: Seq[List[UploadedVolumeLayer]],
                                        client: WKRemoteTracingStoreClient,
                                        otherFiles: Map[String, File],
-                                       dataSource: DataSourceLike): Fox[List[AnnotationLayer]] =
+                                       dataSource: DataSourceLike,
+                                       datasetId: ObjectId): Fox[List[AnnotationLayer]] =
     if (volumeLayersGrouped.isEmpty)
       Fox.successful(List())
     else if (volumeLayersGrouped.length > 1 && volumeLayersGrouped.exists(_.length > 1))
@@ -182,7 +184,8 @@ class AnnotationIOController @Inject()(
                                         newTracingId,
                                         uploadedVolumeLayer.tracing,
                                         uploadedVolumeLayer.getDataZipFrom(otherFiles),
-                                        dataSource = dataSource)
+                                        dataSource = dataSource,
+                                        datasetId = datasetId)
         } yield
           AnnotationLayer(
             newTracingId,
@@ -200,6 +203,7 @@ class AnnotationIOController @Inject()(
           newTracingId,
           VolumeTracings(uploadedVolumeLayersFlat.map(v => VolumeTracingOpt(Some(v.tracing)))),
           dataSource,
+          datasetId,
           uploadedVolumeLayersFlat.map(v => v.getDataZipFrom(otherFiles))
         )
       } yield
@@ -337,8 +341,7 @@ class AnnotationIOController @Inject()(
       else volumeTracing.boundingBox
 
     for {
-      tracingCanHaveSegmentIndex <- canHaveSegmentIndex(organizationId,
-                                                        dataset.name,
+      tracingCanHaveSegmentIndex <- canHaveSegmentIndex(dataset._id,
                                                         fallbackLayerOpt.map(_.name),
                                                         remoteDataStoreClient)
       elementClassProto <- fallbackLayerOpt
@@ -358,13 +361,12 @@ class AnnotationIOController @Inject()(
   }
 
   private def canHaveSegmentIndex(
-      organizationId: String,
-      datasetName: String,
+      datasetId: ObjectId,
       fallbackLayerName: Option[String],
       remoteDataStoreClient: WKRemoteDataStoreClient)(implicit ec: ExecutionContext): Fox[Boolean] =
     fallbackLayerName match {
       case Some(layerName) =>
-        remoteDataStoreClient.hasSegmentIndexFile(organizationId, datasetName, layerName)
+        remoteDataStoreClient.hasSegmentIndexFile(datasetId, layerName)
       case None =>
         Fox.successful(true)
     }

app/controllers/DatasetController.scala

@@ -89,6 +89,7 @@ class DatasetController @Inject()(userService: UserService,
                                   analyticsService: AnalyticsService,
                                   mailchimpClient: MailchimpClient,
                                   wkExploreRemoteLayerService: WKExploreRemoteLayerService,
+                                  composeService: ComposeService,
                                   sil: Silhouette[WkEnv])(implicit ec: ExecutionContext, bodyParsers: PlayBodyParsers)
     extends Controller
     with MetadataAssertions {
@@ -145,10 +146,10 @@ class DatasetController @Inject()(userService: UserService,
         _ <- Fox.fromBool(dataSource.dataLayers.nonEmpty) ?~> "dataset.explore.zeroLayers"
         folderIdOpt <- Fox.runOptional(request.body.folderPath)(folderPath =>
           folderService.getOrCreateFromPathLiteral(folderPath, request.identity._organization)) ?~> "dataset.explore.autoAdd.getFolder.failed"
-        _ <- wkExploreRemoteLayerService.addRemoteDatasource(dataSource,
-                                                             request.body.datasetName,
-                                                             request.identity,
-                                                             folderIdOpt) ?~> "dataset.explore.autoAdd.failed"
+        _ <- wkExploreRemoteLayerService.addRemoteDatasourceToDatabase(dataSource,
+                                                                       request.body.datasetName,
+                                                                       request.identity,
+                                                                       folderIdOpt) ?~> "dataset.explore.autoAdd.failed"
       } yield Ok
     }
 
@@ -490,4 +491,11 @@ class DatasetController @Inject()(userService: UserService,
       }
     }
 
+  def compose(): Action[ComposeRequest] =
+    sil.SecuredAction.async(validateJson[ComposeRequest]) { implicit request =>
+      for {
+        (dataSource, newDatasetId) <- composeService.composeDataset(request.body, request.identity) ?~> "dataset.compose.failed"
+      } yield Ok(Json.obj("newDatasetId" -> newDatasetId))
+    }
+
 }

app/controllers/UserTokenController.scala

@@ -182,10 +182,19 @@ class UserTokenController @Inject()(datasetDAO: DatasetDAO,
         isAllowed <- datasetService.isEditableBy(dataset, Some(user))
       } yield UserAccessAnswer(isAllowed)
 
+    def tryDelete: Fox[UserAccessAnswer] =
+      for {
+        _ <- Fox.fromBool(conf.Features.allowDeleteDatasets) ?~> "dataset.delete.disabled"
+        datasetId <- ObjectId.fromString(id)
+        dataset <- datasetDAO.findOne(datasetId)(GlobalAccessContext) ?~> "dataset.notFound"
+        user <- userBox.toFox ?~> "auth.token.noUser"
+      } yield UserAccessAnswer(user._organization == dataset._organization && user.isAdmin)
+
     mode match {
-      case AccessMode.read  => tryRead
-      case AccessMode.write => tryWrite
-      case _                => Fox.successful(UserAccessAnswer(granted = false, Some("invalid access token")))
+      case AccessMode.read   => tryRead
+      case AccessMode.write  => tryWrite
+      case AccessMode.delete => tryDelete
+      case _                 => Fox.successful(UserAccessAnswer(granted = false, Some("invalid access token")))
     }
   }
 

app/controllers/WKRemoteDataStoreController.scala

@@ -7,9 +7,9 @@ import com.scalableminds.util.tools.Fox
 import com.scalableminds.webknossos.datastore.controllers.JobExportProperties
 import com.scalableminds.webknossos.datastore.helpers.{LayerMagLinkInfo, MagLinkInfo}
 import com.scalableminds.webknossos.datastore.models.UnfinishedUpload
-import com.scalableminds.webknossos.datastore.models.datasource.DataSourceId
+import com.scalableminds.webknossos.datastore.models.datasource.{AbstractDataLayer, DataSource, DataSourceId}
 import com.scalableminds.webknossos.datastore.models.datasource.inbox.{InboxDataSourceLike => InboxDataSource}
-import com.scalableminds.webknossos.datastore.services.{DataSourcePathInfo, DataStoreStatus}
+import com.scalableminds.webknossos.datastore.services.{DataSourcePathInfo, DataSourceRegistrationInfo, DataStoreStatus}
 import com.scalableminds.webknossos.datastore.services.uploading.{
   LinkedLayerIdentifier,
   ReserveAdditionalInformation,
@@ -242,13 +242,38 @@ class WKRemoteDataStoreController @Inject()(
       }
   }
 
-  def getPaths(name: String, key: String, organizationId: String, directoryName: String): Action[AnyContent] =
+  def deleteVirtualDataset(name: String, key: String): Action[ObjectId] =
+    Action.async(validateJson[ObjectId]) { implicit request =>
+      dataStoreService.validateAccess(name, key) { _ =>
+        for {
+          dataset <- datasetDAO.findOne(request.body)(GlobalAccessContext) ~> NOT_FOUND
+          _ <- Fox.fromBool(dataset.isVirtual) ?~> "dataset.delete.notVirtual" ~> FORBIDDEN
+          _ <- datasetDAO.deleteDataset(dataset._id, onlyMarkAsDeleted = true)
+        } yield Ok
+      }
+    }
+
+  def findDatasetId(name: String,
+                    key: String,
+                    datasetDirectoryName: String,
+                    organizationId: String): Action[AnyContent] =
     Action.async { implicit request =>
       dataStoreService.validateAccess(name, key) { _ =>
         for {
-          organization <- organizationDAO.findOne(organizationId)(GlobalAccessContext)
-          dataset <- datasetDAO.findOneByDirectoryNameAndOrganization(directoryName, organization._id)(
+          organization <- organizationDAO.findOne(organizationId)(GlobalAccessContext) ?~> Messages(
+            "organization.notFound",
+            organizationId) ~> NOT_FOUND
+          dataset <- datasetDAO.findOneByNameAndOrganization(datasetDirectoryName, organization._id)(
             GlobalAccessContext)
+        } yield Ok(Json.toJson(dataset._id))
+      }
+    }
+
+  def getPaths(name: String, key: String, datasetId: ObjectId): Action[AnyContent] =
+    Action.async { implicit request =>
+      dataStoreService.validateAccess(name, key) { _ =>
+        for {
+          dataset <- datasetDAO.findOne(datasetId)(GlobalAccessContext) ~> NOT_FOUND
           layers <- datasetLayerDAO.findAllForDataset(dataset._id)
           magsAndLinkedMags <- Fox.serialCombined(layers)(l => datasetService.getPathsForDataLayer(dataset._id, l.name))
           magLinkInfos = magsAndLinkedMags.map(_.map { case (mag, linkedMags) => MagLinkInfo(mag, linkedMags) })
@@ -270,6 +295,53 @@ class WKRemoteDataStoreController @Inject()(
 
     }
 
+  // Register a datasource from the datastore as a dataset in the database.
+  // This is called when adding remote virtual datasets (that should only exist in the database)
+  // by the data store after exploration.
+  def registerDataSource(name: String,
+                         key: String,
+                         organizationId: String,
+                         directoryName: String,
+                         token: String): Action[DataSourceRegistrationInfo] =
+    Action.async(validateJson[DataSourceRegistrationInfo]) { implicit request =>
+      dataStoreService.validateAccess(name, key) { dataStore =>
+        for {
+          user <- bearerTokenService.userForToken(token)
+          organization <- organizationDAO.findOne(organizationId)(GlobalAccessContext) ?~> Messages(
+            "organization.notFound",
+            organizationId) ~> NOT_FOUND
+          _ <- Fox.fromBool(organization._id == user._organization) ?~> "notAllowed" ~> FORBIDDEN
+          dataset <- datasetService.createVirtualDataset(
+            directoryName,
+            organizationId,
+            dataStore,
+            request.body.dataSource,
+            request.body.folderId,
+            user
+          )
+        } yield Ok(dataset._id.toString)
+      }
+    }
+
+  def updateDataSource(name: String, key: String, datasetId: ObjectId, allowNewPaths: Boolean): Action[DataSource] =
+    Action.async(validateJson[DataSource]) { implicit request =>
+      dataStoreService.validateAccess(name, key) { _ =>
+        for {
+          dataset <- datasetDAO.findOne(datasetId)(GlobalAccessContext) ~> NOT_FOUND
+          abstractDataSource = request.body.copy(dataLayers = request.body.dataLayers.map(AbstractDataLayer.from))
+          oldDataSource <- datasetService.fullDataSourceFor(dataset)
+          oldPaths = oldDataSource.toUsable.map(_.allExplicitPaths).getOrElse(List.empty)
+          newPaths = request.body.allExplicitPaths
+          _ <- Fox.fromBool(allowNewPaths || newPaths.forall(oldPaths.contains)) ?~> "New mag paths must be a subset of old mag paths" ~> BAD_REQUEST
+          _ <- datasetDAO.updateDataSourceByDatasetId(datasetId,
+                                                      name,
+                                                      abstractDataSource.hashCode(),
+                                                      abstractDataSource,
+                                                      isUsable = true)(GlobalAccessContext)
+        } yield Ok
+      }
+    }
+
   def jobExportProperties(name: String, key: String, jobId: ObjectId): Action[AnyContent] = Action.async {
     implicit request =>
       dataStoreService.validateAccess(name, key) { _ =>

app/controllers/WKRemoteTracingStoreController.scala

@@ -8,18 +8,15 @@ import com.scalableminds.webknossos.datastore.Annotation.AnnotationProto
 import com.scalableminds.webknossos.datastore.SkeletonTracing.SkeletonTracing
 import com.scalableminds.webknossos.datastore.VolumeTracing.VolumeTracing
 import com.scalableminds.webknossos.datastore.models.annotation.AnnotationLayer
-import com.scalableminds.webknossos.datastore.models.datasource.DataSourceId
 import com.scalableminds.webknossos.tracingstore.AnnotationUpdatesReport
 import com.scalableminds.webknossos.tracingstore.annotation.AnnotationLayerParameters
 import com.scalableminds.webknossos.tracingstore.tracings.TracingId
 import models.analytics.{AnalyticsService, UpdateAnnotationEvent, UpdateAnnotationViewOnlyEvent}
 import models.annotation.AnnotationState._
 import models.annotation._
 import models.dataset.{DatasetDAO, DatasetService}
-import models.organization.OrganizationDAO
 import models.user.UserDAO
 import models.user.time.TimeSpanService
-import play.api.i18n.Messages
 import play.api.libs.json.Json
 import play.api.mvc.{Action, AnyContent, PlayBodyParsers}
 import scalapb.GeneratedMessage
@@ -33,7 +30,6 @@ class WKRemoteTracingStoreController @Inject()(tracingStoreService: TracingStore
                                                wkSilhouetteEnvironment: WkSilhouetteEnvironment,
                                                timeSpanService: TimeSpanService,
                                                datasetService: DatasetService,
-                                               organizationDAO: OrganizationDAO,
                                                userDAO: UserDAO,
                                                annotationInformationProvider: AnnotationInformationProvider,
                                                analyticsService: AnalyticsService,
@@ -114,7 +110,7 @@ class WKRemoteTracingStoreController @Inject()(tracingStoreService: TracingStore
       tracingStoreService.validateAccess(name, key) { _ =>
         implicit val ctx: DBAccessContext = GlobalAccessContext
         annotationDataSourceTemporaryStore.find(annotationId) match {
-          case Some(dataSource) => Fox.successful(Ok(Json.toJson(dataSource)))
+          case Some(dataSourceAndDatasetId) => Fox.successful(Ok(Json.toJson(dataSourceAndDatasetId._1)))
           case None =>
             for {
               annotation <- annotationDAO.findOne(annotationId) ?~> "annotation.notFound"
@@ -125,15 +121,18 @@ class WKRemoteTracingStoreController @Inject()(tracingStoreService: TracingStore
       }
     }
 
-  def dataSourceIdForAnnotation(name: String, key: String, annotationId: ObjectId): Action[AnyContent] =
+  def datasetIdForAnnotation(name: String, key: String, annotationId: ObjectId): Action[AnyContent] =
     Action.async { implicit request =>
       tracingStoreService.validateAccess(name, key) { _ =>
         implicit val ctx: DBAccessContext = GlobalAccessContext
-        for {
-          annotation <- annotationDAO.findOne(annotationId) ?~> "annotation.notFound"
-          dataset <- datasetDAO.findOne(annotation._dataset)
-          organization <- organizationDAO.findOne(dataset._organization)
-        } yield Ok(Json.toJson(DataSourceId(dataset.directoryName, organization._id)))
+        annotationDataSourceTemporaryStore.find(annotationId) match {
+          case Some(dataSourceAndDatasetId) => Fox.successful(Ok(Json.toJson(dataSourceAndDatasetId._2)))
+          case None =>
+            for {
+              annotation <- annotationDAO.findOne(annotationId) ?~> "annotation.notFound"
+              dataset <- datasetDAO.findOne(annotation._dataset) ?~> "dataset.notFound"
+            } yield Ok(Json.toJson(dataset._id))
+        }
       }
     }
 
@@ -151,20 +150,12 @@ class WKRemoteTracingStoreController @Inject()(tracingStoreService: TracingStore
       }
     }
 
-  def dataStoreUriForDataset(name: String,
-                             key: String,
-                             organizationId: Option[String],
-                             datasetDirectory: String): Action[AnyContent] =
+  def dataStoreUriForDataset(name: String, key: String, datasetId: ObjectId): Action[AnyContent] =
     Action.async { implicit request =>
       tracingStoreService.validateAccess(name, key) { _ =>
         implicit val ctx: DBAccessContext = GlobalAccessContext
         for {
-          organizationIdWithFallback <- Fox.fillOption(organizationId) {
-            datasetDAO.getOrganizationIdForDataset(datasetDirectory)(GlobalAccessContext)
-          } ?~> Messages("dataset.noAccess", datasetDirectory) ~> FORBIDDEN
-          dataset <- datasetDAO.findOneByDirectoryNameAndOrganization(datasetDirectory, organizationIdWithFallback) ?~> Messages(
-            "dataset.noAccess",
-            datasetDirectory) ~> FORBIDDEN
+          dataset <- datasetDAO.findOne(datasetId) ?~> "dataset.notFound" ~> NOT_FOUND
           dataStore <- datasetService.dataStoreFor(dataset)
         } yield Ok(Json.toJson(dataStore.url))
       }

app/models/annotation/AnnotationDataSourceTemporaryStore.scala

@@ -9,18 +9,21 @@ import scala.concurrent.ExecutionContext
 import scala.concurrent.duration.DurationInt
 
 /**
-  * Used to store a mapping from annotation id to datasource. This makes it possible for WK to answer a
-  * /tracingstores/:name/dataSource request before an annotation is created. This happens when uploading an annotation.
-  * It also provides a mapping from temporary/compound annotation id (e.g. taskTypeId, projectId) to datasource
+  * Used to store a mapping from annotation id to datasource and datasetId. This makes it possible for WK to answer a
+  * /tracingstores/:name/dataSource or /tracingstores/:name/datasetId request before an annotation is created.
+  * This happens when uploading an annotation.
+  * It also provides a mapping from temporary/compound annotation id (e.g. taskTypeId, projectId) to datasource.
   */
-class AnnotationDataSourceTemporaryStore @Inject()(temporaryStore: TemporaryStore[ObjectId, DataSourceLike]) {
+class AnnotationDataSourceTemporaryStore @Inject()(
+    temporaryStore: TemporaryStore[ObjectId, (DataSourceLike, ObjectId)]) {
 
   private val timeOut = 7 * 24 hours
 
-  def store(annotationId: ObjectId, dataSource: DataSourceLike)(implicit ec: ExecutionContext): Unit =
-    temporaryStore.insert(annotationId, dataSource, Some(timeOut))
+  def store(annotationId: ObjectId, dataSource: DataSourceLike, datasetId: ObjectId)(
+      implicit ec: ExecutionContext): Unit =
+    temporaryStore.insert(annotationId, (dataSource, datasetId), Some(timeOut))
 
-  def find(annotationId: ObjectId): Option[DataSourceLike] =
+  def find(annotationId: ObjectId): Option[(DataSourceLike, ObjectId)] =
     temporaryStore.get(annotationId)
 
 }