Experiments on numeric arrays

.circleci/config.yml

@@ -87,10 +87,6 @@ jobs:
           name: Build webknossos-tracingstore (sbt)
           command: docker-compose run base sbt -no-colors "project webknossosTracingstore" copyMessages compile stage
 
-      - run:
-          name: Assemble webknossos-datastore fat-jar (sbt)
-          command: docker-compose run base sbt -no-colors "project webknossosDatastore" assembly
-
       - save_cache:
           name: Save target cache
           key: target-cache-{{ checksum ".circleci/cache_version" }}-{{ .Branch }}-{{ .Revision }}

frontend/javascripts/admin/admin_rest_api.ts

@@ -1579,21 +1579,6 @@ export async function getAgglomeratesForDatasetLayer(
   );
 }
 
-export async function getMeanAndStdDevFromDataset(
-  datastoreUrl: string,
-  datasetId: APIDatasetId,
-  layerName: string,
-): Promise<{
-  mean: number;
-  stdDev: number;
-}> {
-  return doWithToken((token) =>
-    Request.receiveJSON(
-      `${datastoreUrl}/data/datasets/${datasetId.owningOrganization}/${datasetId.name}/layers/${layerName}/colorStatistics?token=${token}`,
-    ),
-  );
-}
-
 // #### Datastores
 export async function getDatastores(): Promise<Array<APIDataStore>> {
   const datastores = await Request.receiveJSON("/api/datastores");

project/Dependencies.scala

@@ -48,6 +48,7 @@ object Dependencies {
   private val s3fs = "org.lasersonlab" % "s3fs" % "2.2.3"
   private val jblosc = "org.lasersonlab" % "jblosc" % "1.0.1"
   private val scalajHttp = "org.scalaj" %% "scalaj-http" % "2.4.2"
+  private val tensorflow = "org.platanios" %% "tensorflow" % "0.6.4" classifier "linux"
 
   private val sql = Seq(
     "com.typesafe.slick" %% "slick" % "3.2.3",
@@ -83,6 +84,7 @@ object Dependencies {
     webknossosWrap,
     playIterateesStreams,
     filters,
+    tensorflow,
     ws,
     guice,
     swagger,

webknossos-datastore/app/com/scalableminds/webknossos/datastore/arrays/NumericArray.scala

@@ -0,0 +1,151 @@
+package com.scalableminds.webknossos.datastore.arrays
+
+import java.io.{ByteArrayInputStream, ByteArrayOutputStream}
+import java.nio.ByteOrder
+
+import com.scalableminds.webknossos.datastore.models.{UInt16, UInt32, UInt64, UInt8, UnsignedInteger}
+import com.scalableminds.webknossos.datastore.models.datasource.ElementClass
+import javax.imageio.stream.{MemoryCacheImageInputStream, MemoryCacheImageOutputStream}
+
+import scala.util.Using
+
+trait NumericArray extends IndexedSeq[UnsignedInteger] {
+  def update(i: Int, x: UnsignedInteger)
+
+  val elementClass: ElementClass.Value
+
+  lazy val bytesPerElement: Int = ElementClass.bytesPerElement(elementClass)
+
+  def toBytes: Array[Byte] = toBytes(byteOrder = ByteOrder.LITTLE_ENDIAN)
+
+  def toBytes(byteOrder: ByteOrder): Array[Byte] =
+    Using.Manager { use =>
+      val baos = use(new ByteArrayOutputStream())
+      val ios = use(new MemoryCacheImageOutputStream(baos))
+      ios.setByteOrder(byteOrder)
+      writeToImageOutputStream(ios)
+      ios.flush()
+      baos.toByteArray
+    }.get
+
+  def filterNonZero: NumericArray
+
+  protected def writeToImageOutputStream(ios: MemoryCacheImageOutputStream): Unit
+}
+
+protected class NumericArrayByte(val elementClass: ElementClass.Value, val underlyingTyped: Array[Byte])
+    extends NumericArray {
+  override lazy val length: Int =
+    if (elementClass == ElementClass.uint24) underlyingTyped.length / 3 else underlyingTyped.length
+
+  override def apply(idx: Int): UnsignedInteger =
+    UnsignedInteger.fromTypedUnderlying(underlyingTyped(idx), elementClass)
+
+  override def toBytes(byteOrder: ByteOrder = ByteOrder.LITTLE_ENDIAN): Array[Byte] = underlyingTyped
+
+  override protected def writeToImageOutputStream(ios: MemoryCacheImageOutputStream): Unit =
+    throw new Exception("Byte array should output bytes directly, not be called via writeToImageOutputStream")
+
+  protected val zero: Byte = 0
+
+  override def filterNonZero: NumericArray = new NumericArrayByte(elementClass, underlyingTyped.filterNot(_ == zero))
+
+  override def update(i: Int, x: UnsignedInteger): Unit = underlyingTyped(i) = x.asInstanceOf[UInt8].signed
+}
+
+protected class NumericArrayShort(val elementClass: ElementClass.Value, val underlyingTyped: Array[Short])
+    extends NumericArray {
+  override lazy val length: Int = underlyingTyped.length
+
+  override def apply(idx: Int): UnsignedInteger =
+    UnsignedInteger.fromTypedUnderlying(underlyingTyped(idx), elementClass)
+
+  override protected def writeToImageOutputStream(ios: MemoryCacheImageOutputStream): Unit =
+    ios.writeShorts(underlyingTyped, 0, underlyingTyped.length)
+
+  protected val zero: Short = 0
+
+  override def filterNonZero: NumericArray = new NumericArrayShort(elementClass, underlyingTyped.filterNot(_ == zero))
+
+  override def update(i: Int, x: UnsignedInteger): Unit = underlyingTyped(i) = x.asInstanceOf[UInt16].signed
+}
+
+protected class NumericArrayInt(val elementClass: ElementClass.Value, val underlyingTyped: Array[Int])
+    extends NumericArray {
+  override lazy val length: Int = underlyingTyped.length
+
+  override def apply(idx: Int): UnsignedInteger =
+    UnsignedInteger.fromTypedUnderlying(underlyingTyped(idx), elementClass)
+
+  override protected def writeToImageOutputStream(ios: MemoryCacheImageOutputStream): Unit =
+    ios.writeInts(underlyingTyped, 0, underlyingTyped.length)
+
+  protected val zero: Int = 0
+
+  override def filterNonZero: NumericArray = new NumericArrayInt(elementClass, underlyingTyped.filterNot(_ == zero))
+
+  override def update(i: Int, x: UnsignedInteger): Unit = underlyingTyped(i) = x.asInstanceOf[UInt32].signed
+}
+
+protected class NumericArrayLong(val elementClass: ElementClass.Value, val underlyingTyped: Array[Long])
+    extends NumericArray {
+  override lazy val length: Int = underlyingTyped.length
+
+  override def apply(idx: Int): UnsignedInteger =
+    UnsignedInteger.fromTypedUnderlying(underlyingTyped(idx), elementClass)
+
+  override protected def writeToImageOutputStream(ios: MemoryCacheImageOutputStream): Unit =
+    ios.writeLongs(underlyingTyped, 0, underlyingTyped.length)
+
+  protected val zero: Long = 0
+
+  override def filterNonZero: NumericArray = new NumericArrayLong(elementClass, underlyingTyped.filterNot(_ == zero))
+
+  override def update(i: Int, x: UnsignedInteger): Unit = underlyingTyped(i) = x.asInstanceOf[UInt64].signed
+}
+
+object NumericArray {
+
+  def fromBytes(bytes: Array[Byte],
+                elementClass: ElementClass.Value,
+                byteOrder: ByteOrder = ByteOrder.LITTLE_ENDIAN): NumericArray = {
+
+    elementClass match {
+      case ElementClass.uint8 | ElementClass.int8 | ElementClass.uint24 =>
+        return new NumericArrayByte(elementClass, bytes)
+      case _ => ()
+    }
+
+    val numElements = bytes.length / ElementClass.bytesPerElement(elementClass)
+    val bais = new ByteArrayInputStream(bytes)
+    val iis = new MemoryCacheImageInputStream(bais)
+    iis.setByteOrder(byteOrder)
+
+    val result = elementClass match {
+      case ElementClass.uint16 | ElementClass.int16 =>
+        val underlyingTyped = new Array[Short](numElements)
+        iis.readFully(underlyingTyped, 0, underlyingTyped.length)
+        new NumericArrayShort(elementClass, underlyingTyped)
+      case ElementClass.uint32 | ElementClass.int32 =>
+        val underlyingTyped = new Array[Int](numElements)
+        iis.readFully(underlyingTyped, 0, underlyingTyped.length)
+        new NumericArrayInt(elementClass, underlyingTyped)
+      case ElementClass.uint64 | ElementClass.int64 =>
+        val underlyingTyped = new Array[Long](numElements)
+        iis.readFully(underlyingTyped, 0, underlyingTyped.length)
+        new NumericArrayLong(elementClass, underlyingTyped)
+      case _ => ???
+    }
+    iis.close()
+    bais.close()
+    result
+  }
+
+  def fromUnsignedIntegers(uints: IndexedSeq[UnsignedInteger], elementClass: ElementClass.Value): NumericArray =
+    elementClass match {
+      case ElementClass.uint8  => new NumericArrayByte(elementClass, uints.map(_.asInstanceOf[UInt8].signed).toArray)
+      case ElementClass.uint16 => new NumericArrayShort(elementClass, uints.map(_.asInstanceOf[UInt16].signed).toArray)
+      case ElementClass.uint32 => new NumericArrayInt(elementClass, uints.map(_.asInstanceOf[UInt32].signed).toArray)
+      case ElementClass.uint64 => new NumericArrayLong(elementClass, uints.map(_.asInstanceOf[UInt64].signed).toArray)
+    }
+}

webknossos-datastore/app/com/scalableminds/webknossos/datastore/controllers/Application.scala

@@ -1,13 +1,27 @@
 package com.scalableminds.webknossos.datastore.controllers
 
+import java.io.{ByteArrayInputStream, ByteArrayOutputStream}
+import java.lang.reflect.Method
+import java.nio.{ByteBuffer, ByteOrder}
+
 import com.scalableminds.util.tools.Fox
+import com.scalableminds.webknossos.datastore.arrays.NumericArray
+import com.scalableminds.webknossos.datastore.models.datasource.ElementClass
+import com.scalableminds.webknossos.datastore.models.{UInt32, UnsignedIntegerArray}
+import com.scalableminds.webknossos.datastore.services.DataConverter
 import com.scalableminds.webknossos.datastore.storage.DataStoreRedisStore
+import javax.imageio.stream.{MemoryCacheImageInputStream, MemoryCacheImageOutputStream}
 import javax.inject.Inject
+import org.platanios.tensorflow.api.{Shape, Tensor, UInt}
 import play.api.mvc.{Action, AnyContent}
 
 import scala.concurrent.ExecutionContext
+import scala.util.Using
+import org.platanios.tensorflow.jni.{Tensor => NativeTensor}
 
-class Application @Inject()(redisClient: DataStoreRedisStore)(implicit ec: ExecutionContext) extends Controller {
+class Application @Inject()(redisClient: DataStoreRedisStore)(implicit ec: ExecutionContext)
+    extends Controller
+    with DataConverter {
 
   override def allowRemoteOrigin: Boolean = true
 
@@ -22,4 +36,155 @@ class Application @Inject()(redisClient: DataStoreRedisStore)(implicit ec: Execu
     }
   }
 
+  val bytesIn: Array[Byte] = Array.fill(Math.pow(256, 3).intValue())((scala.util.Random.nextInt(256) - 128).toByte)
+
+  def testByteConversions: Action[AnyContent] = Action { implicit request =>
+    val elementClass = ElementClass.uint32
+
+    val iterations = 20
+
+    //testOneMethod("UnsignedInteger", elementClass, iterations, convertAndBackUnsignedInteger)
+    //testOneMethod("spire", elementClass, iterations, convertAndBackSpire)
+    testOneMethod("imageio", elementClass, iterations, convertAndBackImageIO)
+    //testOneMethod("hybrid", elementClass, iterations, convertAndBackHybrid)
+    //testOneMethod("NumericArray", elementClass, iterations, convertAndBackNumericArray)
+    testOneMethod("TensorflowScala", elementClass, iterations, convertAndBackTensorflowScala)
+
+
+    Ok
+  }
+
+  private def testOneMethod(label: String,
+                            elementClass: ElementClass.Value,
+                            iterations: Int,
+                            convertAndBack: ElementClass.Value => (Long, Array[Byte])): Unit = {
+
+    var fromDurations = List[Long]()
+    var toDurations = List[Long]()
+    for (_ <- 1 to iterations) {
+
+      val before = System.currentTimeMillis()
+
+      val (afterTyped, bytesOut) = convertAndBack(elementClass)
+
+      val afterBack = System.currentTimeMillis()
+
+      val isSameLabel = if (bytesIn.sameElements(bytesOut)) " Content is the same!" else ""
+
+      val fromDuration = afterTyped - before
+      val toDuration = afterBack - afterTyped
+
+      fromDurations = fromDuration :: fromDurations
+      toDurations = toDuration :: toDurations
+
+      println(f"$label Done!$isSameLabel fromBytes: ${afterTyped - before} ms, toBytes: ${afterBack - afterTyped}")
+
+    }
+    println(f"$label avg: fromBytes: ${mean(fromDurations)} ms, toBytes: ${mean(toDurations)}")
+  }
+
+  def convertAndBackUnsignedInteger(elementClass: ElementClass.Value): (Long, Array[Byte]) = {
+    val typed = UnsignedIntegerArray.fromByteArray(bytesIn, elementClass)
+
+    val afterTyped = System.currentTimeMillis()
+
+    val bytesOut = UnsignedIntegerArray.toByteArray(typed, elementClass)
+
+    (afterTyped, bytesOut)
+  }
+
+  def convertAndBackSpire(elementClass: ElementClass.Value): (Long, Array[Byte]) = {
+    val typed = convertData(bytesIn, elementClass)
+
+    val afterTyped = System.currentTimeMillis()
+
+    val bytesOut = toBytes(typed, elementClass)
+
+    (afterTyped, bytesOut)
+  }
+
+  def convertAndBackImageIO(elementClass: ElementClass.Value): (Long, Array[Byte]) = {
+    val numElements = bytesIn.length / ElementClass.bytesPerElement(elementClass)
+
+    val typed = new Array[Int](numElements)
+    Using.Manager { use =>
+      val bais = use(new ByteArrayInputStream(bytesIn))
+      val iis = use(new MemoryCacheImageInputStream(bais))
+      iis.setByteOrder(ByteOrder.LITTLE_ENDIAN)
+      iis.readFully(typed, 0, typed.length)
+    }
+
+    val afterTyped = System.currentTimeMillis()
+
+    val bytesOut: Array[Byte] = Using.Manager { use =>
+      val baos = use(new ByteArrayOutputStream())
+      val ios = use(new MemoryCacheImageOutputStream(baos))
+      ios.setByteOrder(ByteOrder.LITTLE_ENDIAN)
+      ios.writeInts(typed, 0, typed.length)
+      ios.flush()
+      baos.toByteArray
+    }.get
+
+    (afterTyped, bytesOut)
+  }
+
+  def convertAndBackNumericArray(elementClass: ElementClass.Value): (Long, Array[Byte]) = {
+    val typed = NumericArray.fromBytes(bytesIn, elementClass)
+
+    val afterTyped = System.currentTimeMillis()
+
+    val bytesOut = typed.toBytes
+
+    (afterTyped, bytesOut)
+  }
+
+  def convertAndBackTensorflowScala(elementClass: ElementClass.Value): (Long, Array[Byte]) = {
+    val typed: Tensor[UInt] = Tensor.fromBuffer[UInt](Shape(bytesIn.length / ElementClass.bytesPerElement(elementClass)), bytesIn.length, ByteBuffer.wrap(bytesIn))
+
+    val afterTyped = System.currentTimeMillis()
+
+    val method: Method = classOf[Tensor[UInt]].getDeclaredMethod("resolve")
+    method.setAccessible(true)
+    val resolved: Long = method.invoke(typed).asInstanceOf[Long]
+    val buffer = NativeTensor.buffer(resolved).order(ByteOrder.nativeOrder)
+    val bytesOut = new Array[Byte](bytesIn.length)
+    buffer.get(bytesOut)
+
+    (afterTyped, bytesOut)
+  }
+
+  def convertAndBackHybrid(elementClass: ElementClass.Value): (Long, Array[Byte]) = {
+    val numElements = bytesIn.length / ElementClass.bytesPerElement(elementClass)
+
+    val typedUnderlying = new Array[Int](numElements)
+    Using.Manager { use =>
+      val bais = use(new ByteArrayInputStream(bytesIn))
+      val iis = use(new MemoryCacheImageInputStream(bais))
+      iis.setByteOrder(ByteOrder.LITTLE_ENDIAN)
+      iis.readFully(typedUnderlying, 0, typedUnderlying.length)
+    }
+
+    val typed: Array[UInt32] = typedUnderlying.map(int => UInt32(int))
+
+    val afterTyped = System.currentTimeMillis()
+
+    val typedUnderlyingOut: Array[Int] = typed.map(t => t.signed)
+
+    val bytesOut: Array[Byte] = Using.Manager { use =>
+      val baos = use(new ByteArrayOutputStream())
+      val ios = use(new MemoryCacheImageOutputStream(baos))
+      ios.setByteOrder(ByteOrder.LITTLE_ENDIAN)
+      ios.writeInts(typedUnderlyingOut, 0, typedUnderlyingOut.length)
+      ios.flush()
+      baos.toByteArray
+    }.get
+
+    (afterTyped, bytesOut)
+  }
+
+  def mean(list: List[Long]): String = {
+    val float = list.sum.toFloat / list.size.toFloat
+    f"$float%1.1f"
+  }
+
 }