Include a remote dataset in the dev setup initial data (#8891)

Based on and thus blocked by #8844 

### Steps to test:
- refresh local database schema
- start up, log in, should see l4_sample_remote in the dashboard dataset
list
- should be usable.

### Issues

- contributes to #8813 (we can easily add a few more once we have them
publicly statically hosted somewhere. Ideal would be something with
plenty of attachments and something with a different data type, like n5
or neuroglancerPrecomputed)

------
- [x] Added changelog entry (create a `$PR_NUMBER.md` file in
`unreleased_changes` or use `./tools/create-changelog-entry.py`)
- [x] Considered [common edge
cases](../blob/master/.github/common_edge_cases.md)

---------

Co-authored-by: valentin-pinkau <[email protected]>

Author: fm3

app/controllers/InitialDataController.scala

@@ -2,6 +2,7 @@ package controllers
 
 import play.silhouette.api.{LoginInfo, Silhouette}
 import com.scalableminds.util.accesscontext.GlobalAccessContext
+import com.scalableminds.util.geometry.{BoundingBox, Vec3Double, Vec3Int}
 import com.scalableminds.util.objectid.ObjectId
 import com.scalableminds.util.time.Instant
 import com.scalableminds.util.tools.{Fox, FoxImplicits}
@@ -15,6 +16,17 @@ import models.task.{TaskType, TaskTypeDAO}
 import models.team._
 import models.user._
 import com.scalableminds.util.tools.Full
+import com.scalableminds.webknossos.datastore.dataformats.MagLocator
+import com.scalableminds.webknossos.datastore.helpers.UPath
+import com.scalableminds.webknossos.datastore.models.{LengthUnit, VoxelSize}
+import com.scalableminds.webknossos.datastore.models.datasource.{
+  DataFormat,
+  DataSourceId,
+  ElementClass,
+  StaticColorLayer,
+  StaticSegmentationLayer,
+  UsableDataSource
+}
 import play.api.libs.json.{JsArray, Json}
 import utils.{StoreModules, WkConf}
 
@@ -39,6 +51,8 @@ class InitialDataController @Inject()(initialDataService: InitialDataService, si
 
 class InitialDataService @Inject()(userService: UserService,
                                    userDAO: UserDAO,
+                                   datasetDAO: DatasetDAO,
+                                   datasetLayerDAO: DatasetLayerDAO,
                                    multiUserDAO: MultiUserDAO,
                                    userExperiencesDAO: UserExperiencesDAO,
                                    taskTypeDAO: TaskTypeDAO,
@@ -156,6 +170,95 @@ Samplecountry
     Some("Works if model files are manually placed at binaryData/sample_organization/66544a56d20000af0e42ba0f/"),
     Some(AiModelCategory.em_neurons)
   )
+  private val defaultDataSource = UsableDataSource(
+    id = DataSourceId("l4_sample_remote", defaultOrganization._id),
+    dataLayers = List(
+      StaticColorLayer(
+        name = "color",
+        dataFormat = DataFormat.zarr3,
+        boundingBox = BoundingBox(Vec3Int(3072, 3072, 512), 1024, 1024, 1024),
+        elementClass = ElementClass.uint8,
+        mags = List(
+          MagLocator(
+            mag = Vec3Int(1, 1, 1),
+            path = Some(UPath.fromStringUnsafe("https://static.webknossos.org/data/zarr_v3/l4_sample/color/1"))
+          ),
+          MagLocator(
+            mag = Vec3Int(2, 2, 1),
+            path = Some(UPath.fromStringUnsafe("https://static.webknossos.org/data/zarr_v3/l4_sample/color/2-2-1"))
+          ),
+          MagLocator(
+            mag = Vec3Int(4, 4, 1),
+            path = Some(UPath.fromStringUnsafe("https://static.webknossos.org/data/zarr_v3/l4_sample/color/4-4-1"))
+          ),
+          MagLocator(
+            mag = Vec3Int(8, 8, 2),
+            path = Some(UPath.fromStringUnsafe("https://static.webknossos.org/data/zarr_v3/l4_sample/color/8-8-2"))
+          ),
+          MagLocator(
+            mag = Vec3Int(16, 16, 4),
+            path = Some(UPath.fromStringUnsafe("https://static.webknossos.org/data/zarr_v3/l4_sample/color/16-16-4"))
+          )
+        )
+      ),
+      StaticSegmentationLayer(
+        name = "segmentation",
+        dataFormat = DataFormat.zarr3,
+        boundingBox = BoundingBox(Vec3Int(3072, 3072, 512), 1024, 1024, 1024),
+        elementClass = ElementClass.uint32,
+        mags = List(
+          MagLocator(
+            mag = Vec3Int(1, 1, 1),
+            path = Some(UPath.fromStringUnsafe("https://static.webknossos.org/data/zarr_v3/l4_sample/segmentation/1"))
+          ),
+          MagLocator(
+            mag = Vec3Int(2, 2, 1),
+            path =
+              Some(UPath.fromStringUnsafe("https://static.webknossos.org/data/zarr_v3/l4_sample/segmentation/2-2-1"))
+          ),
+          MagLocator(
+            mag = Vec3Int(4, 4, 1),
+            path =
+              Some(UPath.fromStringUnsafe("https://static.webknossos.org/data/zarr_v3/l4_sample/segmentation/4-4-1"))
+          ),
+          MagLocator(
+            mag = Vec3Int(8, 8, 2),
+            path =
+              Some(UPath.fromStringUnsafe("https://static.webknossos.org/data/zarr_v3/l4_sample/segmentation/8-8-2"))
+          ),
+          MagLocator(
+            mag = Vec3Int(16, 16, 4),
+            path =
+              Some(UPath.fromStringUnsafe("https://static.webknossos.org/data/zarr_v3/l4_sample/segmentation/16-16-4"))
+          )
+        ),
+        largestSegmentId = Some(2504697),
+      )
+    ),
+    scale = VoxelSize(Vec3Double(11.239999771118164, 11.239999771118164, 28), LengthUnit.nanometer)
+  )
+
+  private val defaultDataset = Dataset(
+    _id = ObjectId("68b80290d4000090f8f4aa62"),
+    _dataStore = defaultDataStore.name,
+    _organization = defaultOrganization._id,
+    _publication = Some(defaultPublication._id),
+    _uploader = Some(defaultUser._id),
+    _folder = defaultOrganization._rootFolder,
+    inboxSourceHash = Some(defaultDataSource.hashCode()),
+    defaultViewConfiguration = None,
+    adminViewConfiguration = None,
+    description = None,
+    directoryName = defaultDataSource.id.directoryName,
+    isPublic = false,
+    isUsable = true,
+    isVirtual = true,
+    name = "l4_sample_remote",
+    voxelSize = Some(defaultDataSource.scale),
+    sharingToken = None,
+    status = "",
+    logoUrl = None,
+  )
 
   def insert: Fox[Unit] =
     for {
@@ -175,6 +278,7 @@ Samplecountry
       _ <- insertTaskType()
       _ <- insertProject()
       _ <- insertPublication()
+      _ <- insertDataset()
       _ <- insertAiModel()
     } yield ()
 
@@ -276,6 +380,15 @@ Samplecountry
     } else Fox.successful(())
   }
 
+  private def insertDataset(): Fox[Unit] = datasetDAO.findOne(defaultDataset._id).shiftBox.flatMap { maybeDataset =>
+    if (maybeDataset.isEmpty) {
+      for {
+        _ <- datasetDAO.insertOne(defaultDataset)
+        _ <- datasetLayerDAO.updateLayers(defaultDataset._id, defaultDataSource)
+      } yield ()
+    } else Fox.successful(())
+  }
+
   private def insertAiModel(): Fox[Unit] = aiModelDAO.findAll.flatMap { aiModels =>
     if (aiModels.isEmpty) {
       aiModelDAO.insertOne(defaultAiModel)

app/models/dataset/DatasetService.scala

@@ -27,7 +27,6 @@ import play.api.http.Status.NOT_FOUND
 import play.api.i18n.{Messages, MessagesProvider}
 import play.api.libs.json.{JsObject, Json}
 import security.RandomIDGenerator
-import utils.WkConf
 
 import javax.inject.Inject
 import scala.concurrent.duration._
@@ -45,8 +44,7 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
                                teamService: TeamService,
                                thumbnailCachingService: ThumbnailCachingService,
                                userService: UserService,
-                               rpc: RPC,
-                               conf: WkConf)(implicit ec: ExecutionContext)
+                               rpc: RPC)(implicit ec: ExecutionContext)
     extends FoxImplicits
     with LazyLogging {
 
@@ -216,7 +214,7 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
         case Some(foundDataset) => // This only returns None for Datasets that are present on a normal Datastore but also got reported from a scratch Datastore
           updateDataSourceDifferentDataStore(foundDataset, dataSource, dataStore)
         case _ =>
-          insertNewDataset(dataSource, dataSource.id.directoryName, dataStore).map(Some(_))
+          createDataset(dataStore, ObjectId.generate, dataSource.id.directoryName, dataSource).map(ds => Some(ds._id))
       }
     }
   }
@@ -262,11 +260,6 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
       }
     }).flatten
 
-  private def insertNewDataset(dataSource: DataSource, datasetName: String, dataStore: DataStore) =
-    publicationForFirstDataset.flatMap { publicationId: Option[ObjectId] =>
-      createDataset(dataStore, ObjectId.generate, datasetName, dataSource, publicationId).map(_._id)
-    }
-
   def updateDataSourceFromUserChanges(dataset: Dataset, dataSourceUpdates: UsableDataSource)(
       implicit ctx: DBAccessContext,
       mp: MessagesProvider): Fox[Unit] =
@@ -368,16 +361,6 @@ class DatasetService @Inject()(organizationDAO: OrganizationDAO,
         }
     }
 
-  private def publicationForFirstDataset: Fox[Option[ObjectId]] =
-    if (conf.WebKnossos.SampleOrganization.enabled) {
-      datasetDAO.isEmpty.map { isEmpty =>
-        if (isEmpty)
-          Some(ObjectId("5c766bec6c01006c018c7459"))
-        else
-          None
-      }
-    } else Fox.successful(None)
-
   def deactivateUnreportedDataSources(reportedDatasetIds: List[ObjectId],
                                       dataStore: DataStore,
                                       organizationId: Option[String]): Fox[Unit] =