From caa3e383e4054f9d26f3a7628d0c412c6a6066d5 Mon Sep 17 00:00:00 2001 From: carlosuc3m <100329787@alumnos.uc3m.es> Date: Mon, 16 Oct 2023 21:55:04 +0200 Subject: [PATCH] bring back Bioengine --- .../bioengine/BioEngineAvailableModels.java | 220 ++++++ .../bioengine/BioengineInterface.java | 474 +++++++++++++ .../bioengine/tensor/BioEngineOutput.java | 401 +++++++++++ .../tensor/BioEngineOutputArray.java | 453 +++++++++++++ .../bioengine/tensor/BioengineTensor.java | 624 ++++++++++++++++++ 5 files changed, 2172 insertions(+) create mode 100644 src/main/java/io/bioimage/modelrunner/bioimageio/bioengine/BioEngineAvailableModels.java create mode 100644 src/main/java/io/bioimage/modelrunner/bioimageio/bioengine/BioengineInterface.java create mode 100644 src/main/java/io/bioimage/modelrunner/bioimageio/bioengine/tensor/BioEngineOutput.java create mode 100644 src/main/java/io/bioimage/modelrunner/bioimageio/bioengine/tensor/BioEngineOutputArray.java create mode 100644 src/main/java/io/bioimage/modelrunner/bioimageio/bioengine/tensor/BioengineTensor.java diff --git a/src/main/java/io/bioimage/modelrunner/bioimageio/bioengine/BioEngineAvailableModels.java b/src/main/java/io/bioimage/modelrunner/bioimageio/bioengine/BioEngineAvailableModels.java new file mode 100644 index 00000000..e358a0bc --- /dev/null +++ b/src/main/java/io/bioimage/modelrunner/bioimageio/bioengine/BioEngineAvailableModels.java @@ -0,0 +1,220 @@ +/*- + * #%L + * Use deep learning frameworks from Java in an agnostic and isolated way. + * %% + * Copyright (C) 2022 - 2023 Institut Pasteur and BioImage.IO developers. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ +package io.bioimage.modelrunner.bioimageio.bioengine; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import javax.net.ssl.HttpsURLConnection; + +import io.bioimage.modelrunner.utils.Constants; + +import com.google.gson.Gson; +import com.google.gson.internal.LinkedTreeMap; + +/** + * Class to handle the JSON file that contains the information about the + * models supported by the Bioengine. The model is located at: + * https://raw.githubusercontent.com/bioimage-io/bioengine-model-runner/gh-pages/manifest.bioengine.json + * @author Carlos Garcia Lopez de Haro + * + */ +public class BioEngineAvailableModels { + + /** + * Collection of models available at the bioengine + */ + private ArrayList> collection; + /** + * Logs for the CI of the Bioengine + */ + private ArrayList conversion_logs; + /** + * Description of the file + */ + private String description; + /** + * Format version + */ + private String format_version; + /** + * Name of the file + */ + private String name; + /** + * Tags fo the file + */ + private ArrayList tags; + /*** + * Type of the file + */ + private String type; + /** + * Version of the file + */ + private String version; + + /** + * URL of the Json file containing the bioengine compatible models json + */ + final private static String BIOENGINE_COMPATIBLE_JSON = + "https://raw.githubusercontent.com/bioimage-io/bioengine-model-runner/gh-pages/manifest.bioengine.json"; + /** + * Key for the ID of the models supported by the bioengine in the collections object + */ + final private static String ID_KEY = "id"; + + private static BioEngineAvailableModels BAM; + /** + * Address of the first public server that hosts a bioengine instance + */ + public static final String PUBLIC_BIOENGINE_SERVER = "https://ai.imjoy.io"; + + /** + * Method that parses the json file that contains all the supported models by the BioEngine + * @return obejct containing the spported models by the bioengine + * @throws IOException if the bioengine website cannot + * be accessed or if there is any error parsing the json + */ + public static BioEngineAvailableModels load() throws IOException + { + HttpsURLConnection con; + try { + URL url = new URL(BIOENGINE_COMPATIBLE_JSON); + con = (HttpsURLConnection) url.openConnection(); + } catch (IOException ex) { + throw new IOException("Unable to access the following link: " + BIOENGINE_COMPATIBLE_JSON + + System.lineSeparator() + "Please, check your Internet connection. If the site does " + + "not exist, please report it at : " + Constants.ISSUES_LINK); + } + try (InputStream inputStream = con.getInputStream(); + InputStreamReader inpStreamReader = new InputStreamReader(inputStream); + BufferedReader br = new BufferedReader(inpStreamReader);){ + Gson g = new Gson(); + return g.fromJson(br, BioEngineAvailableModels.class); + } catch (IOException ex) { + throw new IOException("Unable to parse JSON that contains BioEngine compatible models at: " + + System.lineSeparator() + BIOENGINE_COMPATIBLE_JSON + System.lineSeparator() + + ex.getCause().toString()); + } + } + + /** + * Creates empty BioEngineAvailableModels object that works with all its methods. + * Created to avoid errors caused by {@link NullPointerException} + * @return + */ + private static BioEngineAvailableModels createEmptyObject() { + BioEngineAvailableModels availableModels = new BioEngineAvailableModels(); + availableModels.collection = new ArrayList>(); + availableModels.conversion_logs = new ArrayList(); + return availableModels; + } + + /** + * Method that returns a list of the IDs corresponding to the models that are supported by + * the bioengine + * @return a list of the IDs supported by the bioengine + */ + public List getListOfSupportedIDs() { + return collection.stream().map(x -> x.get(ID_KEY)).collect(Collectors.toList()); + } + + /** + * Method that finds whether a model is supported by the Bioengine by looking if its + * model ID is spedified in the JSON file that contains all valid models + * @param modelID + * ID of the model of interest + * @return true if supported, false otherwise + */ + public boolean isModelSupportedByBioengine(String modelID) { + LinkedTreeMap isFound = collection.stream() + .filter(x -> modelID.startsWith(x.get(ID_KEY)+ "/")).findFirst().orElse(null); + if (isFound == null) + return false; + return true; + } + + /** + * Static method that finds whether a model is supported by the Bioengine by looking if its + * model ID is spedified in the JSON file that contains all valid models. + * This method only recovers the file that contains the information of the models + * supported by the Bioengine the first time. In order to retrieve the file each + * time, do the following: + * BioEngineAvailableModels bam = BioEngineAvailableModels.load(); + * String id = "model/ID": + * boolean supported = bam.isModelSupportedByBioengine(id); + * @param modelID + * ID of the model of interest + * @return true if supported, false otherwise + */ + public static boolean isModelSupportedInBioengine(String modelID) { + if (BAM == null) { + try { + BAM = BioEngineAvailableModels.load(); + } catch (IOException ex) { + BAM = createEmptyObject(); + } + } + return BAM.isModelSupportedByBioengine(modelID); + } + + public ArrayList> getCollection() { + return this.collection; + } + + public String getType() { + return this.type; + } + + public ArrayList getConversionLogs() { + return this.conversion_logs; + } + + public ArrayList getTags() { + return this.tags; + } + + public String getName() { + return this.name; + } + + public String getFormatVersion() { + return this.format_version; + } + + public String getVersion() { + return this.version; + } + + public String getDescription() { + return this.description; + } + + public static String getBioengineJson() { + return BIOENGINE_COMPATIBLE_JSON; + } +} diff --git a/src/main/java/io/bioimage/modelrunner/bioimageio/bioengine/BioengineInterface.java b/src/main/java/io/bioimage/modelrunner/bioimageio/bioengine/BioengineInterface.java new file mode 100644 index 00000000..b7ec5728 --- /dev/null +++ b/src/main/java/io/bioimage/modelrunner/bioimageio/bioengine/BioengineInterface.java @@ -0,0 +1,474 @@ +/*- + * #%L + * Use deep learning frameworks from Java in an agnostic and isolated way. + * %% + * Copyright (C) 2022 - 2023 Institut Pasteur and BioImage.IO developers. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ +package io.bioimage.modelrunner.bioimageio.bioengine; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.ProtocolException; +import java.net.URL; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.zip.GZIPOutputStream; + +import org.msgpack.jackson.dataformat.MessagePackFactory; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import io.bioimage.modelrunner.bioimageio.bioengine.tensor.BioEngineOutput; +import io.bioimage.modelrunner.bioimageio.bioengine.tensor.BioEngineOutputArray; +import io.bioimage.modelrunner.bioimageio.bioengine.tensor.BioengineTensor; +import io.bioimage.modelrunner.bioimageio.description.ModelDescriptor; +import io.bioimage.modelrunner.engine.DeepLearningEngineInterface; +import io.bioimage.modelrunner.exceptions.LoadModelException; +import io.bioimage.modelrunner.exceptions.RunModelException; +import io.bioimage.modelrunner.tensor.Tensor; +import io.bioimage.modelrunner.utils.Constants; +import net.imglib2.img.Img; +import net.imglib2.img.ImgFactory; +import net.imglib2.img.array.ArrayImgFactory; +import net.imglib2.type.numeric.real.FloatType; + +/** + * This class sends the corresponding inputs to the wanted model available + * on the Bioengine, retrieves the corresponding results and converts them into + * JDLL tensors + * + * + * Class to create inputs that can be sent to the BioEngine server. + * The input tensors should be defined as an array of inputs, where + * each of the inputs can be either a String, int or any other type, + * but in case the input is an array, it should be encoded as a + * {@link HashMap}. It needs to have the key "_rtype" with the + * corresponding value "ndarray", the key "_rvalue" with an array of + * bytes corresponding to the data wanted to be encoded, a key "_rshape" + * which should contain the shape of the array and finally the "_rdtype" + * corresponding to the array datatype. + * The array of inputs should be then included in another hashmap under + * the key "inputs", together with the key "model_name" and the name of + * the model and "decode_json" true. + * There is an example defined in Python at: + * https://gist.github.com/oeway/b6a6b810f94c91bb902e80a2f788b9e2#file-access_triton_service_hyhpa-py-L22 + * + * + * @author Carlos Garcia Lopez de Haro + * + */ +public class BioengineInterface implements DeepLearningEngineInterface { + /** + * Server where the Bioengine is hosted + */ + private String server; + /** + * Object that contains all the info described in the rdf.yaml file of the model + */ + private ModelDescriptor rdf; + /** + * Input for the Bioengine, contains all the required info that needs to be + * The input needs to have: + * -An entry called "inputs", whose value is another Map that contains + * the info about the input tensors + * -An entry called model_name with the name of the model + * -A fixed entry called decoe_json that equals to true + */ + Map kwargs = new HashMap(); + /** + * Map containing the information needed to provide an input to the + * server for models that are defined in the bioimage.io repo. + * Model that are not in the bioimage.io repo do not need this map + * This map is sent inside the {@link #kwargs} map + */ + private HashMap bioimageioKwargs = new HashMap(); + /** + * Bioimage.io model ID of the model of interest + */ + private String modelID; + /** + * In the input Bioengine map, key for the model name + */ + private static final String MODEL_NAME_KEY = "model_name"; + /** + * In the input Bioengine map, key for the input tensors. + * THe inputs value should always be a list of objects, which + * corresponds to encoded tensors and/or parameters for the + * Bioengine to use.. + * The entries of this list can be either: + * -A @see java.util.LinkedHashMap generated containing the name, shape, + * dtype and data of a tensor + * -A @see java.util.LinkedHashMap for non dimensional parameters + */ + private static final String INPUTS_KEY = "inputs"; + /** + * In the input Bioengine map, key for whether to + * return the rdf.yaml of the model or not + */ + private static final String RDF_KEY = "return_rdf"; + /** + * In the input Bioengine map, key for the Bioimage.io model id + */ + private static final String ID_KEY = "model_id"; + /** + * Name of the default model used to run a model coming from the BioImage.io repo + */ + public static final String DEFAULT_BMZ_MODEL_NAME = "bioengine-model-runner"; + /** + * String key corresponding to the decode Json parameter in the + * {@link #kwargs} map + */ + private static final String DECODE_JSON_KEY = "decode_json"; + /** + * Value corresponding to the decode Json parameter in the + * {@link #kwargs} map. It is fixed. + */ + private static final boolean DECODE_JSON_VAL = true; + /** + * Key for the input of the BioEngine corresponding to the type of serialization + */ + private static final String SERIALIZATION_KEY = "serialization"; + /** + * Value for the BioEngine serialization + */ + private static final String SERIALIZATION_VAL = "imjoy"; + + public static void main(String[] args) throws LoadModelException, RunModelException { + BioengineInterface bi = new BioengineInterface(); + String path = "C:\\Users\\angel\\OneDrive\\Documentos\\" + + "pasteur\\git\\deep-icy\\models\\2D UNet Arabidopsis Ap" + + "ical Stem Cells_27062023_125425"; + bi.loadModel(path, path); + bi.addServer(BioEngineAvailableModels.PUBLIC_BIOENGINE_SERVER); + final ImgFactory< FloatType > imgFactory = new ArrayImgFactory<>( new FloatType() ); + final Img< FloatType > img1 = imgFactory.create( 1, 1, 512, 512 ); + Tensor inpTensor = Tensor.build("input0", "bcyx", img1); + List> inputs = new ArrayList>(); + inputs.add(inpTensor); + final Img< FloatType > img2 = imgFactory.create( 1, 2, 512, 512 ); + Tensor outTensor = Tensor.build("output0", "bcyx", img2); + List> outputs = new ArrayList>(); + outputs.add(outTensor); + bi.run(inputs, outputs); + System.out.print(DECODE_JSON_VAL); + } + + @Override + /** + * {@inheritDoc} + */ + public void run(List> inputTensors, List> outputTensors) throws RunModelException { + + List inputs = new ArrayList(); + for (Tensor tt : inputTensors) { + inputs.add(BioengineTensor.build(tt).getAsMap()); + } + if (rdf.getName().equals("cellpose-python")) { + Map pp = new HashMap(); + pp.put("diameter", 30); + inputs.add(pp); + } + if (modelID != null) { + bioimageioKwargs.put(INPUTS_KEY, inputs); + ArrayList auxList = new ArrayList(); + auxList.add(bioimageioKwargs); + kwargs.put(INPUTS_KEY, auxList); + } else { + kwargs.put(INPUTS_KEY, inputs); + } + byte[] byteResult; + try { + byteResult = executeModelOnBioEngine(compress(serialize(kwargs))); + bioimageioKwargs.put(INPUTS_KEY, null); + } catch (IOException e) { + throw new RunModelException(e.toString()); + } + BioEngineOutput bioengineOutputs; + try { + bioengineOutputs = BioEngineOutput.build(byteResult); + } catch (Exception e) { + throw new RunModelException("Error retrieving the Bioengine results." + System.lineSeparator() + + e.toString()); + } + fillOutputTensors(outputTensors, bioengineOutputs.getArrayOutputs()); + } + + /** + * Fill the expected output tensors with the data from teh Bioengine outputs + * @param outputTensors + * tensors expeted as defined by the rdf.yaml file + * @param arrayOutputs + * results from the Bioengine + * @throws RunModelException if the number of tensors expected is not the same as + * the number of tensors returned by the Bioengine + */ + private void fillOutputTensors(List> outputTensors, List arrayOutputs) throws RunModelException { + if (outputTensors.size() != arrayOutputs.size()) { + throw new RunModelException("The rdf.yaml file specifies '" + outputTensors.size() + + "' tensors, but the Bioengine has " + + "produced '" + arrayOutputs.size() + "' output tensors."); + } + int c = 0; + for (Tensor tt : outputTensors) { + tt.setData(arrayOutputs.get(c ++).getImg()); + } + } + + @Override + /** + * {@inheritDoc} + */ + public void loadModel(String modelFolder, String modelSource) throws LoadModelException { + try { + rdf = ModelDescriptor.readFromLocalFile(modelFolder + File.separator + Constants.RDF_FNAME, false); + } catch (Exception e) { + throw new LoadModelException("The rdf.yaml file for " + + "model at '" + modelFolder + "' cannot be read."); + } + + if (rdf.getName().equals("cellpose-python")) { + kwargs.put(MODEL_NAME_KEY, "cellpose-python"); + kwargs.put(DECODE_JSON_KEY, DECODE_JSON_VAL); + } else { + kwargs.put(MODEL_NAME_KEY, DEFAULT_BMZ_MODEL_NAME); + kwargs.put(SERIALIZATION_KEY, SERIALIZATION_VAL); + bioimageioKwargs.put(RDF_KEY, false); + workaroundModelID(); + findBioEngineWeightsIfPossible(); + } + } + + @Override + /** + * {@inheritDoc} + */ + public void closeModel() { + this.bioimageioKwargs = null; + this.kwargs = null; + this.modelID = null; + this.rdf = null; + this.server = null; + } + + public void addServer(String server) { + this.server = server; + } + + /** TODO + * Workaround for BioImage.io model runner. It does not work with the full version, it + * only works with: major_version_/second_version + * @throws LoadModelException if not model ID is not found. Without a model ID, the model + * cannot run on the Bioengine + */ + private void workaroundModelID() throws LoadModelException { + modelID = rdf.getModelID(); + if (modelID == null) + throw new LoadModelException("The selected model does not have a model ID, " + + "thus it is not sppported to run on the Bioengine."); + int nSubversions = modelID.length() - modelID.replace("/", "").length(); + if (nSubversions == 2) { + modelID = modelID.substring(0, modelID.lastIndexOf("/")); + } + bioimageioKwargs.put(ID_KEY, modelID); + } + + /** + * Identifies if the model has weights that are compatible with the Bioengine. + * @throws LoadModelException if the model does not have a pair of weights + * compatible with the bioengine. The Bioengine only supports + * keras, torchscript and onnx + */ + private void findBioEngineWeightsIfPossible() throws LoadModelException { + if (BioEngineAvailableModels.isModelSupportedInBioengine(modelID)) + return; + throw new LoadModelException("For some reason, the selected model ('" + modelID + + "') is not supported by the Bioengine. The possible reasons range from" + + " needing inputs or/and outputs in a certain format not supported to " + + "requiring a certain DL framework not supported (too old or too specific)." + + " See the list of the models currently available: " + + BioEngineAvailableModels.getBioengineJson()); + /** + * TODO talk with Wei to see whihc are the weights currently supported by the + * Bioengine + for (String entry : rdf.getWeights().getSupportedDLFrameworks()) { + if (entry.equals(ModelWeight.getKerasID())) { + bioimageioKwargs.put(MODEL_WEIGHTS_KEY, ModelWeight.getKerasID()); + return; + } else if (entry.equals(ModelWeight.getOnnxID())) { + bioimageioKwargs.put(MODEL_WEIGHTS_KEY, ModelWeight.getOnnxID()); + return; + } else if (entry.equals(ModelWeight.getTorchscriptID())) { + bioimageioKwargs.put(MODEL_WEIGHTS_KEY, ModelWeight.getTorchscriptID()); + return; + } + } + if (!rdf.getWeights().getSupportedDLFrameworks().contains(ModelWeight.getTensorflowID())) + throw new LoadModelException(""); + throw new LoadModelException("The Bioengine does not support the DL framework " + + "compatible with the model selected. The bioengine supports the following" + + " frameworks: " + SUPPORTED_BIOENGINE_WEIGHTS + " whereas the model is " + + "only compatible with: " + rdf.getWeights().getSupportedDLFrameworks()); + */ + } + + /** + * Serilize the input map to bytes + * @param kwargs + * map contianing the info we want to serialize + * @return an array of bytes that contians the info that we want to send to + * the Bioengine + * @throws IOException if there is any error in the serialization + */ + private static byte[] serialize(Map kwargs) throws IOException { + ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); + + byte[] bytes = objectMapper.writeValueAsBytes(kwargs); + return bytes; + } + + /** + * Compress an array of bytes + * @param arr + * the array of bytes we want to compresss + * @return the compressed array of bytes + * @throws IOException if there is any error during the compression + */ + private static byte[] compress(byte[] arr) throws IOException { + byte[] result = new byte[]{}; + try (ByteArrayOutputStream bos = new ByteArrayOutputStream(arr.length); + GZIPOutputStream gzipOS = new GZIPOutputStream(bos)) { + gzipOS.write(arr); + // You need to close it before using bos + gzipOS.close(); + result = bos.toByteArray(); + } + return result; + } + + /** + * Sends a byte array to a model in the BioEngine server, where inference + * is performed and it fetches the output array of bytes produced by the server + * @param data + * the data corresponding to the input to the model + * @return the output of the server + * @throws ProtocolException if the connection with the server cannot be opened + * or the server is not found + * @throws MalformedURLException if the url is not correct + * @throws IOException if the connection with the server cannot be opened + * or the server is not found + */ + public byte[] executeModelOnBioEngine(byte[] data) throws ProtocolException, + MalformedURLException, + IOException { + byte[] result = sendDataToServerAndReceiveResponse(data); + // Set received data bytes to null to save memory + data = null; + return result; + } + + /** + * Creates a connection, sends information and receives a response + * @param data + * byte array we want to send to the server + * @return a byte array response from the server + * @throws ProtocolExceptionif the connection with the server cannot be opened + * or the server is not found + * @throws MalformedURLException if the url is not correct + * @throws IOException if the connection with the server cannot be opened + * or the server is not found + */ + private byte[] sendDataToServerAndReceiveResponse(byte[] data) throws ProtocolException, + MalformedURLException, + IOException { + HttpURLConnection conn = createConnection(data); + + byte[] respon; + try (InputStream inputStream = conn.getInputStream();) { + respon = readInputStream(inputStream); + } catch (IOException ex) { + InputStream errorStream = conn.getErrorStream(); + respon = readInputStream(errorStream); + errorStream.close(); + } + conn.disconnect(); + + return respon; + } + + /** + * Create a post connection with the BioEngine server + * @param data + * byte array we want to send to the server + * @return the connection + * @throws ProtocolExceptionif the connection with the server cannot be opened + * or the server is not found + * @throws MalformedURLException if the url is not correct + * @throws IOException if the connection with the server cannot be opened + * or the server is not found + */ + private HttpURLConnection createConnection(byte[] data) throws ProtocolException, + MalformedURLException, + IOException{ + URL url = new URL(getExecutionURL()); + HttpURLConnection conn= (HttpURLConnection) url.openConnection(); + conn.setDoOutput( true ); + conn.setDoInput(true); + conn.setRequestMethod( "POST" ); + conn.setRequestProperty( "Content-Type", "application/msgpack"); + conn.setRequestProperty( "Content-Encoding", "gzip"); + conn.setRequestProperty( "Content-Length", Integer.toString(data.length)); + try( DataOutputStream wr = new DataOutputStream( conn.getOutputStream())) { + wr.write(data); + wr.flush(); + } + return conn; + } + + /** + * Convert an input stream into a byte array + * @param inputStream + * input stream to be converted + * @return byte array that contains the same info as the {@link InputStream} + * @throws IOException if there is any error accessing the {@link InputStream} + */ + private static byte[] readInputStream(InputStream inputStream) throws IOException { + byte[] bytes; + try (ByteArrayOutputStream byteOut = new ByteArrayOutputStream()) { + byte[] buffer = new byte[1024]; + int bytesRead; + while ((bytesRead = inputStream.read(buffer)) != -1) { + byteOut.write(buffer, 0, bytesRead); + } + bytes = byteOut.toByteArray(); + } + return bytes; + } + + /** + * Get the URL of to send the data to be run in the BioEngine + * @return the post BioEngine URL + */ + private String getExecutionURL() { + return server + "/public/services/triton-client/execute"; + } +} diff --git a/src/main/java/io/bioimage/modelrunner/bioimageio/bioengine/tensor/BioEngineOutput.java b/src/main/java/io/bioimage/modelrunner/bioimageio/bioengine/tensor/BioEngineOutput.java new file mode 100644 index 00000000..0b48099b --- /dev/null +++ b/src/main/java/io/bioimage/modelrunner/bioimageio/bioengine/tensor/BioEngineOutput.java @@ -0,0 +1,401 @@ +/*- + * #%L + * Use deep learning frameworks from Java in an agnostic and isolated way. + * %% + * Copyright (C) 2022 - 2023 Institut Pasteur and BioImage.IO developers. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ +package io.bioimage.modelrunner.bioimageio.bioengine.tensor; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import org.msgpack.jackson.dataformat.MessagePackFactory; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + +import io.bioimage.modelrunner.bioimageio.bioengine.BioengineInterface; + + +/** + * Class manage outputs produced by a BioEngine server. + * The output of the BioEngine is an array of bytes that encode a Map with the actual + * outputs of the model and a field called "__info__" that contains the information + * about the output produced. + * There is an example defined in Python at: + * https://gist.github.com/oeway/b6a6b810f94c91bb902e80a2f788b9e2#file-access_triton_service_hyhpa-py-L22 + * + * @author Carlos Javier Garcia Lopez de Haro + */ +public class BioEngineOutput { + + /** + * The deserialized output of the BioEngine server. The BioEngine encodes the + * model output into bytes so it can post the result to local consumers + * via Https Request. The deserialized output is basically the output of the + * model before the encoding by the BioEngine. + * The output is a Map that consists on: + * - An information field "__info__" containing information about the name, + * shape and data type of the outputs + * - A map for each of the individual outputs, which can be ndarrays or + * parameters + */ + public LinkedHashMap deserializedOutput = new LinkedHashMap(); + /** + * Key for the entry in the outputs HashMap that contains information + * about the other outputs of the BioEngine. This information covers + * the output names, the shape, the datatype and the size + */ + private static String outputInfoKey = "__info__"; + /** + * Key to the information about the outputs inside the __info__ + * dictionary + */ + private static String outputInfoListKey = "outputs"; + /** + * List of dictionaries, containing each of them information about + * each of the outputs. This information corresponds to the name of the + * output, the datatype, the shape and the size + */ + private List> outputsInfo; + /** + * Key in the __info__ HashMap that contains the name of each + * of teh outputs of the model + */ + private static String outputInfoNameKey = "name"; + /** + * Key in the __info__ HashMap that contains the name of the BioENgine model run. + * It can be either the name of the model or "bioimage.model-runner" if the model + * being run comes from the bioimage.io repo + */ + private static String modelNameKey = "model_name"; + /** + * Key for the field in the output dictionary that specifies whether the model + * execution has been a success or not + */ + private static String successKey = "success"; + /** + * For each output,key that defined the output object that it is. + * Currently it is only available for the type "ndarray" which corresponds + * to an image. If it does not exist, te output is assumed to be a + * parameter + */ + private static String outputRTypeKey = "_rtype"; + /** + * Key for the field of the BioEngine output dictionary that contains the results + * of the model execution for BioImage.io models ONLY + */ + private static String bioimageioResultKey = "outputs"; + /** + * Value of the {@link #outputRTypeKey} that corresponds to an + * image array + */ + private String imageArrayValue = "ndarray"; + /** + * Whether the model whose output this object represents comes from the + * BioImage.io repo + */ + private boolean bioimageio = false; + /** + * List containing the output arrays of the BioEngine model + * The entries are {@link BioEngineOuputArray} + */ + private List list = new ArrayList(); + /** + * Whether the outputs have been closed or not + */ + private boolean closed = false; + /** + * Error message displayed when there is an error of execution inside the BioEngine + */ + private static String errMsg = "Python error at the BioEngine.\n"; + + /** + * Create the Bioengine input + * @param modelName + * name of the model that is going to be called + * @throws IOException if there is any error deserializing the raw output bytes + * @throws Exception if the BioEngine sends an error message after execution + */ + private BioEngineOutput(byte[] rawOutput) throws IOException, Exception { + this.deserializedOutput = deserialize(rawOutput); + // Remove the bytes from memory + rawOutput = null; + processOutputs(); + } + + /** + * Build the object that contains the inputs to + * the BioEngine in the corresponding format + * @param rawOutput + * the raw byte array output of the bioengine + * @return an object that can be used to provide + * inputs to the BioEngine + * @throws IOException if there is any error deserializing the raw output bytes + * @throws Exception if the BioEngine sends an error message after execution + */ + public static BioEngineOutput build(byte[] rawOutput) throws IOException, + Exception { + return new BioEngineOutput(rawOutput); + } + + /** + * Creates readable outputs by the Java software from the BioEngine + * output. The BioEngine output consists in bytes for every image and + * parameter. This method will create buffers of data that can easily be + * transformed into an Icy sequence or Fiji/ImageJ image + * @throws Exception if the BioEngine sends an error message after execution + */ + private void processOutputs() throws Exception { + lookForErrors(); + setOutputsInfo(); + try { + createOutputsFromInfo(); + } catch (NullPointerException ex) { + String msg = "JDLL does not recognize the format of the BioEngine output. Either the " + + "output is incorrect or the format of the BioEngine has changed and JDLL has" + + " not adapted yet."; + throw new NullPointerException(msg + System.lineSeparator() + ex.toString()); + } catch (Exception ex) { + throw new Exception("Empty/Incorrect BioEngine output" + System.lineSeparator() + ex.toString()); + } + } + + @SuppressWarnings("unchecked") + /** + * Find if the BioEngine sends any error message after execution + * @throws Exception if the BioEngine sends an error message after execution + */ + private void lookForErrors() throws Exception { + LinkedHashMap results = + (LinkedHashMap) this.deserializedOutput.get("result"); + // Special models such as cell pose do not have the same keys as the Bioimage.io models + if (results == null ||results.get("success") == null) + return; + boolean success = (boolean) results.get("success"); + if (!success) { + String msg = (String) results.get("error"); + msg = errMsg + msg; + throw new Exception(msg); + } + } + + /** + * Create readable objects from the deserialized output of the BioEngine using + * the information provided by the BioEngine + * @throws Exception if no output can be retrieved from the data returned by the BioEngine + */ + private void createOutputsFromInfo() throws Exception { + if (this.outputsInfo != null && this.outputsInfo.size() != 0) + for (LinkedHashMap output : this.outputsInfo) { + createOutput((String) output.get(outputInfoNameKey)); + } else if (this.deserializedOutput != null && this.deserializedOutput.size() != 0) { + for (String outputName : this.deserializedOutput.keySet()) + createOutput(outputName); + } + if (list == null || list.size() == 0) { + String msg = "The deserialized BioEngine output did not contain (or\n" + + "the program could not find) any information."; + throw new Exception(msg); + } + } + + /** + * Create an array output readable by Java from a single output map produced + * by the BioEngine + * @param outputName + * the name of the output of interest + * @param output + * the map containing the info of interest + * @throws Exception + */ + private void createOutputBioImageIo(String outputName, LinkedHashMap output) throws Exception { + Objects.requireNonNull(output); + if (output.get(successKey) == null || !((boolean) output.get(successKey))) + return; + List> outputList = (List>) output.get(bioimageioResultKey); + Objects.requireNonNull(outputList); + createOutputsFromList(outputName, outputList); + } + + /** + * Create an array output readable by Java from a single output map produced + * by the BioEngine + * @param outputName + * the name of the output of interest + * @param output + * the map containing the info of interest + * @throws Exception + */ + private void createOutputFromMap(String outputName, LinkedHashMap output) throws Exception { + if (output.get(outputRTypeKey) != null && output.get(outputRTypeKey).equals(imageArrayValue)) { + addOutputToList(outputName, output); + } + } + + /** + * Adds output to list of outputs obtained from the BioEngine + * @param outputName + * name of the output + * @param output + * output Map + * @throws Exception + */ + private void addOutputToList(String outputName, LinkedHashMap output) throws Exception { + try { + this.list.add(BioEngineOutputArray.buildOutput(outputName, output)); + } catch (IllegalArgumentException ex) { + throw new IllegalArgumentException("Invalid output" + System.lineSeparator() + ex.toString()); + } catch (Exception ex) { + throw new Exception("Error retrieving output '" + outputName + "'." + + System.lineSeparator() + ex.toString()); + } + } + + /** + * Create an array output readable by Java from a single output List produced + * by the BioEngine + * @param outputList + * the list containing the info of interest + * @throws Exception + */ + private void createOutputsFromList(String name, List> outputList) throws Exception { + for (int i = 0; i < outputList.size(); i ++) { + LinkedHashMap output = outputList.get(i); + String outputName = name + "_" + i; + createOutputFromMap(outputName, output); + } + } + + /** + * Create an array output readable by Java from a single entry of the output map produced + * by the BioEngine + * @param outputName + * the name of the output of interest + * @throws Exception + */ + private void createOutput(String outputName) throws Exception { + Object outObject = deserializedOutput.get(outputName); + if (!bioimageio && outObject instanceof LinkedHashMap) { + createOutputFromMap(outputName, (LinkedHashMap) outObject); + } else if (bioimageio && outObject instanceof LinkedHashMap) { + createOutputBioImageIo(outputName, (LinkedHashMap) outObject); + } else if (this.deserializedOutput.get(outputName) instanceof List) { + // TODO what to do with other types of output + } + } + + /** + * Set the information about the outputs. This information is obtained + * from the same output of the BioEngine + */ + public void setOutputsInfo() { + throwExceptionIfClosed(); + if (this.outputsInfo == null) { + LinkedHashMap __info__ = (LinkedHashMap) this.deserializedOutput.get(outputInfoKey); + this.outputsInfo = (List>) __info__.get(outputInfoListKey ); + bioimageio = isBioImageIoKey((String) __info__.get(modelNameKey)); + } + } + + /** + * REturns the output of the BioEngine server after deserialization. + * @return the output of the BioEngine + */ + public LinkedHashMap getDeserializedOutput(){ + throwExceptionIfClosed(); + return this.deserializedOutput; + } + + /** + * Get the array outputs produced by the model called in the BioEngine + * @return the array outputs produced + */ + public List getArrayOutputs(){ + throwExceptionIfClosed(); + return this.list; + } + + /** + * Close the outputs of the BioEngine to free the memory + */ + public void close() { + list = null; + deserializedOutput = null; + outputsInfo = null; + imageArrayValue = null; + closed = true; + } + + /** + * Whether the tensor is closed or not + * @return true if closed, false otherwise + */ + public boolean isClosed() { + return closed; + } + + /** + * Throw {@link IllegalStateException} if the tensor has been closed + */ + private void throwExceptionIfClosed() { + if (!closed) + return; + throw new IllegalStateException("The tensor that is trying to be modified has already been " + + "closed."); + } + + /** + * Return the error message that is displayed when the BioEngine fails + * @return standard BioEngine error message + */ + public static String getBioEngineErrorMsg() { + return errMsg; + } + + /** + * Method that deserializes a byte array into a Map + * @param arr + * array of bytes + * @return map of deserailized bytes + * @throws IOException if something goes wrong in the deserialization + * @throws ClassNotFoundException if the deserialized object is not a Map + */ + private static LinkedHashMap deserialize(byte[] arr) throws IOException, ClassNotFoundException{ + ObjectMapper objectMapper = new ObjectMapper(new MessagePackFactory()); + LinkedHashMap map = objectMapper.readValue(arr, new TypeReference>() {}); + return map; + } + + /** + * Whether the name of the model corresponds to the key used to run BioImage.io models + * @param name + * name of a model + * @return true if the model name correpsonds to the Bioimage.io runner and false otherwise + */ + public static boolean isBioImageIoKey(String name) { + if (name != null && name.equals(BioengineInterface.DEFAULT_BMZ_MODEL_NAME)) + return true; + else + return false; + } +} diff --git a/src/main/java/io/bioimage/modelrunner/bioimageio/bioengine/tensor/BioEngineOutputArray.java b/src/main/java/io/bioimage/modelrunner/bioimageio/bioengine/tensor/BioEngineOutputArray.java new file mode 100644 index 00000000..22cbbefe --- /dev/null +++ b/src/main/java/io/bioimage/modelrunner/bioimageio/bioengine/tensor/BioEngineOutputArray.java @@ -0,0 +1,453 @@ +/*- + * #%L + * Use deep learning frameworks from Java in an agnostic and isolated way. + * %% + * Copyright (C) 2022 - 2023 Institut Pasteur and BioImage.IO developers. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ +package io.bioimage.modelrunner.bioimageio.bioengine.tensor; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.DoubleBuffer; +import java.nio.FloatBuffer; +import java.nio.IntBuffer; +import java.nio.ShortBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Objects; + +import io.bioimage.modelrunner.numpy.ByteArrayUtils; +import io.bioimage.modelrunner.numpy.DecodeNumpy; +import net.imglib2.img.Img; +import net.imglib2.type.NativeType; +import net.imglib2.type.numeric.RealType; + +/** + * Class that converts each of the particular output arrays produced by the BioEngine + * server into readable buffers with their corresponding shape and data type. + * The produced object can be used to create an image easily. + * + * @author Carlos Javier García López de Haro + */ +public class BioEngineOutputArray { + /** + * Name of the output image array + */ + private String name; + /** + * Array containing the shape of the output array + */ + private long[] shape; + /** + * String containing the data type of the array. + * Look at TODO to see data types supported + */ + private String dtype; + /** + * byte array containing the data of the array + */ + private byte[] arr; + + /** TODO add possibility of having parameters + * TODO we need the shape of the array too + * Manage ouputs from the BioEngine to be able to be handled by a common + * Java consumer + * @param name + * Name of the output + * @param outputMap + * Map containing the data produced by the BioEngine for a particular output + * @throws IllegalArgumentException if the data type of the array is not supported + */ + private BioEngineOutputArray(String name, HashMap outputMap) throws IllegalArgumentException { + setName(name); + setShape(outputMap.get(BioengineTensor.SHAPE_KEY)); + setDType((String) outputMap.get(BioengineTensor.DTYPE_KEY)); + setData(outputMap.get(BioengineTensor.VALUE_KEY)); + } + + /** + * Creates a BioEngine output array in a readable way for Java consumers + * @param name + * name of the output + * @param dataType + * data type of the tensor + * @param shape + * shape of the tensor + * @param buffer + * data of the tensor as a byte array + * @throws IllegalArgumentException if the data type of the array is not supported + */ + public BioEngineOutputArray(String name, String dataType, Object shape, byte[] buffer) { + setName(name); + setShape(shape); + setDType(dataType); + setData(buffer); + } + + /** + * REtrieve the byte array that contains the data of the tensor + * @param byteArray + * the byte array that contains the array data + * @throws IllegalArgumentException if the data type of the array is not supported + */ + private void setData(Object byteArrayObject) throws IllegalArgumentException { + try { + arr = getByteArray(byteArrayObject); + } catch (Exception ex) { + throw new IllegalArgumentException("Error retrieving information from the BioEngine output '" + this.name + "'.\n" + + "The array data is not correctly defined and cannot be read, it should be either a byte array or List of bytes."); + } + } + + + /** + * Create an array from the bytes received by the BioEngine and using the corresponding shape + * and data types + * @param + * possible ImgLib2 data types of the image that will be returned + * @return an ImgLib2 {@link Img} containing the data of one of the outputs of the bioengine + * @throws IllegalArgumentException if the data type of the array is not supported + */ + @SuppressWarnings("unchecked") + public < T extends RealType< T > & NativeType< T > > Img getImg() + throws IllegalArgumentException { + return getImg(ByteOrder.LITTLE_ENDIAN); + } + + + /** + * Create an array from the bytes received by the BioEngine and using the corresponding shape + * and data types + * @param + * possible ImgLib2 data types of the image that will be returned + * @param byteOrder + * order of the bytes + * @return an ImgLib2 {@link Img} containing the data of one of the outputs of the bioengine + * @throws IllegalArgumentException if the data type of the array is not supported + */ + @SuppressWarnings("unchecked") + public < T extends RealType< T > & NativeType< T > > Img getImg(ByteOrder byteOrder) + throws IllegalArgumentException { + Objects.requireNonNull(arr); + ByteBuffer buf = ByteBuffer.wrap(arr).order(byteOrder); + return DecodeNumpy.build(buf, byteOrder, dtype, shape); + } + + /** + * Retrieve the byte array containing the data of the tensor + * @return the byte array with the data of the tensor + */ + public byte[] getArray() { + return this.arr; + } + + /** + * Converts byte array into a signed integer 16 bit array stored in + * a buffer. + * @param arr + * the byte array + * @return a integer 16 buffer containing the wanted data + */ + public static ShortBuffer convertIntoSignedInt16(byte[] arr) { + return ShortBuffer.wrap(ByteArrayUtils.toInt16(arr)); + } + + /** + * Converts byte array into a signed integer 32 bit array stored in + * a buffer. + * @param arr + * the byte array + * @return a int buffer containing the wanted data + */ + public static IntBuffer convertIntoSignedInt32(byte[] arr) { + return IntBuffer.wrap(ByteArrayUtils.toInt32(arr)); + } + + /** + * Converts byte array into a boolean array stored in + * a buffer. + * @param arr + * the byte array + * @return a int buffer containing the wanted boolean data + */ + public static IntBuffer convertIntoBoolean(byte[] arr) { + int[] boolArr = new int[arr.length]; + for (int i = 0; i < arr.length; i ++) { + boolArr[i] = (int) arr[i]; + } + return IntBuffer.wrap(boolArr); + } + + /** + * Converts byte array into a unsigned integer 16 bit array stored in + * a buffer. + * However, as this data type does not exist in Java, the values are stored + * in an int32 array containing the values that would correspond to + * an uin16 array + * @param arr + * the byte array + * @return an int buffer containing the wanted data + */ + public static IntBuffer convertIntoUnsignedInt16(byte[] arr) { + return IntBuffer.wrap(ByteArrayUtils.toUInt16(arr)); + } + + /** + * Converts byte array into a signed integer 8 bit array stored in + * a buffer. + * However, as this data type does not exist in Java, the values are stored + * in an int32 array containing the values that would correspond to + * an int8 array + * @param arr + * the byte array + * @return an int buffer containing the wanted data + */ + public static ByteBuffer convertIntoSignedInt8(byte[] arr) { + return ByteBuffer.wrap(arr); + } + + /** + * Converts byte array into an unsigned integer 8 bit array stored in + * a buffer. However, as this data type does not exist in Java, the values are stored + * in an int32 array containing the values that would correspond to + * an uint8 array + * @param arr + * the byte array + * @return an int buffer containing the wanted data + */ + public static IntBuffer convertIntoUnignedInt8(byte[] arr) { + int[] uint8 = new int[arr.length]; + for (int i = 0; i < arr.length; i ++) { + uint8[i] = arr[i] & 0xff; + } + return IntBuffer.wrap(uint8); + } + + /** + * Converts byte array into a signed float 32 bit array stored in + * a buffer. + * @param arr + * the byte array + * @return a float buffer containing the wanted data + */ + public static FloatBuffer convertIntoSignedFloat32(byte[] arr) { + return FloatBuffer.wrap(ByteArrayUtils.toFloat32(arr)); + } + + /** + * Converts byte array into a signed float 64 bit array stored in + * a buffer. + * @param arr + * the byte array + * @return a double buffer containing the wanted data + */ + public static DoubleBuffer convertIntoSignedFloat64(byte[] arr) { + return DoubleBuffer.wrap(ByteArrayUtils.toFloat64(arr)); + } + + /** + * Creates a BioEngine output array in a readable way for Java consumers + * @param name + * name of the output + * @param outputMap + * Map containing the information about the output + * @return an object with ordered information about the output that can be managed + * by the consumer software + * @throws IllegalArgumentException if the data type of the array is not supported + */ + public static BioEngineOutputArray buildOutput(String name, HashMap outputMap) throws IllegalArgumentException { + return new BioEngineOutputArray(name, outputMap); + } + + /** + * Creates a BioEngine output array in a readable way for Java consumers + * @param name + * name of the output + * @param dataType + * data type of the tensor + * @param shape + * shape of the tensor + * @param buffer + * data of the tensor as a byte array + * @return an understandable tensor + * @throws IllegalArgumentException if the data type of the array is not supported + */ + public static BioEngineOutputArray buildOutput(String name, String dataType, Object shape, byte[] buffer) throws IllegalArgumentException { + return new BioEngineOutputArray(name, dataType, shape, buffer); + } + + /** + * Sets the data type of the array + * @param dtype + * the data type + */ + private void setDType(String dtype) { + this.dtype = dtype; + } + + /** + * Gets the data type of the array + * @return the data type of the array + */ + public String getDType() { + return this.dtype; + } + + /** + * Sets the shape of the array + * @param shape + * the shape of the array + * @throws IllegalArgumentException if the shape object does not correspond to an array + */ + private void setShape(Object shape) throws IllegalArgumentException{ + try { + this.shape = getLongArray(shape); + } catch (Exception ex) { + throw new IllegalArgumentException("Error retrieving information from the BioEngine output '" + this.name + "'.\n" + + "The shape is not correctly defined, it should be either an int array or ArrayList."); + } + } + + /** + * Gets the shape of the array + * @return the shape of the array + */ + public long[] getShape() { + return this.shape; + } + + /** + * Sets the name of the array + * @param name + * the name of the array + */ + private void setName(String name) { + this.name = name; + } + + /** + * Gets the name of the array + * @return the name of the array + */ + public String getName() { + return this.name; + } + + /** + * Casts an long array in an object that contains an long array + * @param shape + * the object containing the long array + * @return the long array + * @throws Exception if it was impossible to cast the array from the object + */ + public long[] getLongArray(Object shape) throws Exception { + long[] shapeArr = null; + if (shape != null && shape instanceof ArrayList) { + ArrayList ll = (ArrayList) shape; + shapeArr = new long [ll.size()]; + for (int c = 0; c < ll.size(); c ++) + shapeArr[c] = castUnknownToLong(ll.get(c)); + } else if (shape != null&& shape instanceof int[]){ + shapeArr = (long[]) shape; + } else if (shape != null&& shape instanceof double[]){ + double[] shapeArrDouble = (double[]) shape; + shapeArr = new long[shapeArrDouble.length]; + for (int i = 0; i < shapeArrDouble.length; i ++) { + shapeArr[i] = (int) shapeArrDouble[i]; + } + } else { + throw new Exception("Datatype of shape array cannot be casted to int or double."); + } + return shapeArr; + } + + /** + * Cast unknown number to int + * @param unknownNumber + * the unknown number + * @return the int + */ + private int castUnknownToInt(Object unknownNumber) { + if (unknownNumber instanceof Integer){ + return (int) unknownNumber; + } else if (unknownNumber instanceof Double){ + return ((Double) unknownNumber).intValue(); + } else if (unknownNumber instanceof Float){ + return ((Float) unknownNumber).intValue(); + } else if (unknownNumber instanceof Short){ + return ((Short) unknownNumber).intValue(); + } else if (unknownNumber instanceof Byte){ + return ((Byte) unknownNumber).intValue(); + } else if (unknownNumber instanceof Long){ + return ((Long) unknownNumber).intValue(); + } else if (unknownNumber instanceof Number){ + return ((Number) unknownNumber).intValue(); + } else { + throw new IllegalArgumentException("Shape of the output '" + this.name + + "' is not defined with allowed Java types."); + } + } + + /** + * Cast unknown number to long + * @param unknownNumber + * the unknown number + * @return the long + */ + private long castUnknownToLong(Object unknownNumber) { + if (unknownNumber instanceof Integer){ + return (int) unknownNumber; + } else if (unknownNumber instanceof Double){ + return ((Double) unknownNumber).longValue(); + } else if (unknownNumber instanceof Float){ + return ((Float) unknownNumber).longValue(); + } else if (unknownNumber instanceof Short){ + return ((Short) unknownNumber).longValue(); + } else if (unknownNumber instanceof Byte){ + return ((Byte) unknownNumber).longValue(); + } else if (unknownNumber instanceof Long){ + return ((Long) unknownNumber).longValue(); + } else if (unknownNumber instanceof Number){ + return ((Number) unknownNumber).longValue(); + } else { + throw new IllegalArgumentException("Shape of the output '" + this.name + + "' is not defined with allowed Java types."); + } + } + + /** + * Casts a byte array in an object that contains a byte array + * @param listArr + * the object containing the byte array + * @return the byte array + * @throws Exception if it was impossible to cast the array from the object + */ + public static byte[] getByteArray(Object listArr) throws Exception { + byte[] arr = null; + if (listArr != null && listArr instanceof ArrayList) { + @SuppressWarnings("unchecked") + ArrayList ll = (ArrayList) listArr; + arr = new byte[ll.size()]; + for (byte c = 0; c < ll.size(); c ++) + arr[c] = (byte) ll.get(c); + } else if (listArr != null && listArr instanceof byte[]){ + arr = (byte[]) listArr; + } else { + throw new Exception(); + } + return arr; + } +} diff --git a/src/main/java/io/bioimage/modelrunner/bioimageio/bioengine/tensor/BioengineTensor.java b/src/main/java/io/bioimage/modelrunner/bioimageio/bioengine/tensor/BioengineTensor.java new file mode 100644 index 00000000..9e09f7fa --- /dev/null +++ b/src/main/java/io/bioimage/modelrunner/bioimageio/bioengine/tensor/BioengineTensor.java @@ -0,0 +1,624 @@ +/*- + * #%L + * Use deep learning frameworks from Java in an agnostic and isolated way. + * %% + * Copyright (C) 2022 - 2023 Institut Pasteur and BioImage.IO developers. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ +package io.bioimage.modelrunner.bioimageio.bioengine.tensor; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.HashMap; +import java.util.Map; + +import io.bioimage.modelrunner.tensor.Tensor; +import io.bioimage.modelrunner.utils.IndexingUtils; +import net.imglib2.Cursor; +import net.imglib2.RandomAccessibleInterval; +import net.imglib2.img.Img; +import net.imglib2.type.NativeType; +import net.imglib2.type.numeric.RealType; +import net.imglib2.type.numeric.integer.ByteType; +import net.imglib2.type.numeric.integer.IntType; +import net.imglib2.type.numeric.integer.LongType; +import net.imglib2.type.numeric.integer.ShortType; +import net.imglib2.type.numeric.integer.UnsignedByteType; +import net.imglib2.type.numeric.integer.UnsignedIntType; +import net.imglib2.type.numeric.integer.UnsignedShortType; +import net.imglib2.type.numeric.real.DoubleType; +import net.imglib2.type.numeric.real.FloatType; +import net.imglib2.util.Cast; +import net.imglib2.util.Util; +import net.imglib2.view.IntervalView; + +/** + * Class that converts {@link RandomAccessibleInterval}-based JDLL tensors into + * byte based maps that can be sent to the Bioengine for processing + * @author Carlos Garcia Lopez de Haro + * + */ +public class BioengineTensor { + + /** + * Map containing the instances needed to provide an input to the + * server. + * The input needs to have: + * -An entry called "inputs", whose value is another Map that contains + * the info about the input tensors + * -An entry called model_name with the name of the model + * -A fixed entry called decoe_json that equals to true + */ + private Map inputs = new HashMap(); + /** + * String key corresponding to the type of object being specified + */ + protected final static String OBJECT_KEY = "_rtype"; + /** + * Value corresponding to the type of the array in the + * {@link #inputs} map + */ + protected static final String NDARRAY_VALUE = "ndarray"; + /** + * Value corresponding to a parameter + */ + private static final String PARAMETER_VALUE = "parameter"; + /** + * String key corresponding to the value of the array in the + * {@link #inputs} map + */ + protected static final String VALUE_KEY = "_rvalue"; + /** + * String key corresponding to the shape of the array in the + * {@link #inputs} map + */ + protected static final String SHAPE_KEY = "_rshape"; + /** + * String key corresponding to the dtype of the array in the + * {@link #inputs} map + */ + protected static final String DTYPE_KEY = "_rdtype"; + /** + * String used as tag for the float32 np dtype + */ + protected static final String FLOAT32_STR = "float32"; + /** + * String used as tag for the float64 np dtype + */ + protected static final String FLOAT64_STR = "float64"; + /** + * String used as tag for the byte or int8 np dtype + */ + protected static final String BYTE_STR = "int8"; + /** + * String used as tag for the byte or int16 np dtype + */ + protected static final String INT16_STR = "int16"; + /** + * String used as tag for the int32 np dtype + */ + protected static final String INT32_STR = "int32"; + /** + * String used as tag for the int64 np dtype + */ + protected static final String INT64_STR = "int64"; + /** + * String used as tag for the ubyte or uint8 np dtype + */ + protected static final String UBYTE_STR = "uint8"; + /** + * String used as tag for the uint16 np dtype + */ + protected static final String UINT16_STR = "uint16"; + /** + * String used as tag for the uint32 np dtype + */ + protected static final String UINT32_STR = "uint32"; + /** + * String used as tag for the boolean np dtype + */ + protected static final String BOOL_STR = "bool"; + + /** + * Utility class. + */ + private BioengineTensor() {} + + /** + * Create an object that can be sent and understood by the bioengine. + * This method creates the needed object for a parameter in the bioengine + * + * @param params + * the parameters that the model needs + * @return an object readable by the bioengine for parameters + */ + public static BioengineTensor build(Map params) { + BioengineTensor bt = new BioengineTensor(); + bt.inputs.put(OBJECT_KEY, PARAMETER_VALUE); + bt.inputs.put(SHAPE_KEY, new int[] {1}); + bt.inputs.put(DTYPE_KEY, "BYTES"); + bt.inputs.put(VALUE_KEY, params); + return bt; + } + + @SuppressWarnings("unchecked") + /** + * From a JDLL tensors, this method creates an object that contains + * a map that can be sent to the bioengine. + * This method creates the needed object for an Image or ndarray + * + * @param + * ImgLib2 datatype that the tensor can have + * @param tensor + * the tensor containing all the needed data + * @return an object contianing the map that can be sent to the bioengine + */ + public static < T extends RealType< T > & NativeType< T > > + BioengineTensor build(Tensor tensor) { + return build(tensor, ByteOrder.LITTLE_ENDIAN); + } + + @SuppressWarnings("unchecked") + /** + * From a JDLL tensors, this method creates an object that contains + * a map that can be sent to the bioengine. + * This method creates the needed object for an Image or ndarray + * + * @param + * ImgLib2 datatype that the tensor can have + * @param tensor + * the tensor containing all the needed data + * @param order + * order of the bytes + * @return an object contianing the map that can be sent to the bioengine + */ + public static < T extends RealType< T > & NativeType< T > > + BioengineTensor build(Tensor tensor, ByteOrder order) { + BioengineTensor bt = new BioengineTensor(); + bt.inputs.put(OBJECT_KEY, NDARRAY_VALUE); + bt.inputs.put(SHAPE_KEY, tensor.getShape()); + bt.inputs.put(VALUE_KEY, tensor.getShape()); + RandomAccessibleInterval rai = tensor.getData(); + if (Util.getTypeFromInterval(rai) instanceof ByteType) { + bt.inputs.put( VALUE_KEY, buildByte( Cast.unchecked( rai ) ) ); + bt.inputs.put(DTYPE_KEY, BYTE_STR); + } else if (Util.getTypeFromInterval(rai) instanceof UnsignedByteType) { + bt.inputs.put( VALUE_KEY, buildUByte( ( Cast.unchecked( rai ) ) ) ); + bt.inputs.put(DTYPE_KEY, UBYTE_STR); + } else if (Util.getTypeFromInterval(rai) instanceof ShortType) { + bt.inputs.put( VALUE_KEY, buildShort( Cast.unchecked( rai ), order ) ); + bt.inputs.put(DTYPE_KEY, INT16_STR); + } else if (Util.getTypeFromInterval(rai) instanceof UnsignedShortType) { + bt.inputs.put( VALUE_KEY, buildUShort( Cast.unchecked( rai ), order ) ); + bt.inputs.put(DTYPE_KEY, UINT16_STR); + } else if (Util.getTypeFromInterval(rai) instanceof IntType) { + bt.inputs.put( VALUE_KEY, buildInt( Cast.unchecked( rai ), order ) ); + bt.inputs.put(DTYPE_KEY, INT32_STR); + } else if (Util.getTypeFromInterval(rai) instanceof UnsignedIntType) { + bt.inputs.put( VALUE_KEY, buildUInt( Cast.unchecked( rai ), order ) ); + bt.inputs.put(DTYPE_KEY, UINT32_STR); + } else if (Util.getTypeFromInterval(rai) instanceof FloatType) { + bt.inputs.put( VALUE_KEY, buildFloat( Cast.unchecked( rai ), order ) ); + bt.inputs.put(DTYPE_KEY, FLOAT32_STR); + } else if (Util.getTypeFromInterval(rai) instanceof DoubleType) { + bt.inputs.put( VALUE_KEY, buildDouble( Cast.unchecked( rai ), order ) ); + bt.inputs.put(DTYPE_KEY, FLOAT64_STR); + } else if (Util.getTypeFromInterval(rai) instanceof LongType) { + bt.inputs.put( VALUE_KEY, buildLong( Cast.unchecked( rai ), order ) ); + bt.inputs.put(DTYPE_KEY, INT64_STR); + } else { + throw new IllegalArgumentException("The image has an unsupported type: " + Util.getTypeFromInterval(rai).getClass().toString()); + } + return bt; + } + + /** + * + * @return the map that is actually serialized and sent to the bioengine + */ + public Map getAsMap() { + return this.inputs; + } + + /** + * Create byte array from the backend {@link RandomAccessibleInterval} of a JDLL tensor + * @param + * ImgLib2 datatype that the tensor can have + * @param tensor + * the tensor containing all the needed data + * @return the backend {@link RandomAccessibleInterval} of a JDLL tensor as a byte array, + * does not contain information about dimensions or data type + */ + public static < T extends RealType< T > & NativeType< T > > + byte[] createByteArray(Tensor tensor) { + return imglib2ToByteArray(tensor.getData()); + } + + /** + * Create byte array from the backend {@link RandomAccessibleInterval} of a JDLL tensor + * @param + * ImgLib2 datatype that the tensor can have + * @param tensor + * the tensor containing all the needed data + * @param order + * order of the bytes + * @return the backend {@link RandomAccessibleInterval} of a JDLL tensor as a byte array, + * does not contain information about dimensions or data type + */ + public static < T extends RealType< T > & NativeType< T > > + byte[] createByteArray(Tensor tensor, ByteOrder order) { + return imglib2ToByteArray(tensor.getData(), order); + } + + + @SuppressWarnings("unchecked") + /** + * + * Create byte array from a {@link RandomAccessibleInterval} + * @param + * ImgLib2 datatype that the image can have + * @param rai + * the image containing all the needed data + * @return the {@link RandomAccessibleInterval} as a byte array, does not contain information + * about dimensions or data type + */ + public static < T extends RealType< T > & NativeType< T > > + byte[] imglib2ToByteArray(RandomAccessibleInterval rai) { + return imglib2ToByteArray(rai, ByteOrder.LITTLE_ENDIAN); + } + + + @SuppressWarnings("unchecked") + /** + * + * Create byte array from a {@link RandomAccessibleInterval} + * @param + * ImgLib2 datatype that the image can have + * @param rai + * the image containing all the needed data + * @param order + * order of the bytes + * @return the {@link RandomAccessibleInterval} as a byte array, does not contain information + * about dimensions or data type + */ + public static < T extends RealType< T > & NativeType< T > > + byte[] imglib2ToByteArray(RandomAccessibleInterval rai, ByteOrder order) { + if (Util.getTypeFromInterval(rai) instanceof ByteType) { + return buildByte( Cast.unchecked( rai ) ); + } else if (Util.getTypeFromInterval(rai) instanceof UnsignedByteType) { + return buildUByte( Cast.unchecked( rai ) ); + } else if (Util.getTypeFromInterval(rai) instanceof ShortType) { + return buildShort( Cast.unchecked( rai ), order ); + } else if (Util.getTypeFromInterval(rai) instanceof UnsignedShortType) { + return buildUShort( Cast.unchecked( rai ), order ); + } else if (Util.getTypeFromInterval(rai) instanceof IntType) { + return buildInt( Cast.unchecked( rai ), order ); + } else if (Util.getTypeFromInterval(rai) instanceof UnsignedIntType) { + return buildUInt( Cast.unchecked( rai ), order ); + } else if (Util.getTypeFromInterval(rai) instanceof FloatType) { + return buildFloat( Cast.unchecked( rai ), order ); + } else if (Util.getTypeFromInterval(rai) instanceof DoubleType) { + return buildDouble( Cast.unchecked( rai ), order ); + } else if (Util.getTypeFromInterval(rai) instanceof LongType) { + return buildLong( Cast.unchecked( rai ), order ); + } else { + throw new IllegalArgumentException("The image has an unsupported type: " + Util.getTypeFromInterval(rai).getClass().toString()); + } + } + + /** + * Creates a byte array from a {@link ByteType} {@link RandomAccessibleInterval}. + * + * @param imgTensor + * {@link RandomAccessibleInterval} to be mapped into byte buffer + * @param order + * order of the bytes + * @return byte array containing the data of the image as a flat byte array + */ + private static byte[] buildByte(RandomAccessibleInterval imgTensor) + { + Cursor tensorCursor; + if (imgTensor instanceof IntervalView) + tensorCursor = ((IntervalView) imgTensor).cursor(); + else if (imgTensor instanceof Img) + tensorCursor = ((Img) imgTensor).cursor(); + else + throw new IllegalArgumentException("The data of the " + Tensor.class + " has " + + "to be an instance of " + Img.class + " or " + IntervalView.class); + long flatSize = 1; + for (long ss : imgTensor.dimensionsAsLongArray()) {flatSize *= ss;} + byte[] byteArr = new byte[(int) flatSize]; + int cc = 0; + while (tensorCursor.hasNext()) { + tensorCursor.fwd(); + byteArr[cc ++] = tensorCursor.get().getByte(); + } + return byteArr; + } + + /** + * Creates a byte array from a {@link UnsignedByteType} {@link RandomAccessibleInterval}. + * + * @param imgTensor + * {@link RandomAccessibleInterval} to be mapped into byte buffer + * @param order + * order of the bytes + * @return byte array containing the data of the image as a flat byte array + */ + private static byte[] buildUByte(RandomAccessibleInterval imgTensor) + { + Cursor tensorCursor; + if (imgTensor instanceof IntervalView) + tensorCursor = ((IntervalView) imgTensor).cursor(); + else if (imgTensor instanceof Img) + tensorCursor = ((Img) imgTensor).cursor(); + else + throw new IllegalArgumentException("The data of the " + Tensor.class + " has " + + "to be an instance of " + Img.class + " or " + IntervalView.class); + long flatSize = 1; + for (long ss : imgTensor.dimensionsAsLongArray()) {flatSize *= ss;} + byte[] byteArr = new byte[(int) flatSize]; + int cc = 0; + while (tensorCursor.hasNext()) { + tensorCursor.fwd(); + int val = tensorCursor.get().get(); + if (val > 127) + val = val - 256; + byteArr[cc ++] = (byte) val; + } + return byteArr; + } + + /** + * Creates a byte array from a {@link ShortType} {@link RandomAccessibleInterval}. + * + * @param imgTensor + * {@link RandomAccessibleInterval} to be mapped into byte buffer + * @param order + * order of the bytes + * @return byte array containing the data of the image as a flat byte array + */ + private static byte[] buildShort(RandomAccessibleInterval imgTensor, ByteOrder order) + { + Cursor tensorCursor; + if (imgTensor instanceof IntervalView) + tensorCursor = ((IntervalView) imgTensor).cursor(); + else if (imgTensor instanceof Img) + tensorCursor = ((Img) imgTensor).cursor(); + else + throw new IllegalArgumentException("The data of the " + Tensor.class + " has " + + "to be an instance of " + Img.class + " or " + IntervalView.class); + long flatSize = 2; + for (long ss : imgTensor.dimensionsAsLongArray()) {flatSize *= ss;} + byte[] byteArr = new byte[(int) flatSize]; + int cc = 0; + while (tensorCursor.hasNext()) { + tensorCursor.fwd(); + short val = tensorCursor.get().get(); + byte[] arr = ByteBuffer.allocate(2).order(order).putShort(val).array(); + System.arraycopy(arr, 0, byteArr, cc * 2, 2); + cc ++; + } + return byteArr; + } + + /** + * Creates a byte array from a {@link UnsignedShortType} {@link RandomAccessibleInterval}. + * + * @param imgTensor + * {@link RandomAccessibleInterval} to be mapped into byte buffer + * @param order + * order of the bytes + * @return byte array containing the data of the image as a flat byte array + */ + private static byte[] buildUShort(RandomAccessibleInterval imgTensor, ByteOrder order) + { + Cursor tensorCursor; + if (imgTensor instanceof IntervalView) + tensorCursor = ((IntervalView) imgTensor).cursor(); + else if (imgTensor instanceof Img) + tensorCursor = ((Img) imgTensor).cursor(); + else + throw new IllegalArgumentException("The data of the " + Tensor.class + " has " + + "to be an instance of " + Img.class + " or " + IntervalView.class); + long flatSize = 2; + for (long ss : imgTensor.dimensionsAsLongArray()) {flatSize *= ss;} + byte[] byteArr = new byte[(int) flatSize]; + int cc = 0; + while (tensorCursor.hasNext()) { + tensorCursor.fwd(); + int val = tensorCursor.get().get(); + short shortval; + if (val >= Math.pow(2, 15)) + shortval = (short) (val - Math.pow(2, 16)); + else + shortval = (short) val; + byte[] arr = ByteBuffer.allocate(2).order(order).putShort(shortval).array(); + System.arraycopy(arr, 0, byteArr, cc * 2, 2); + cc ++; + } + return byteArr; + } + + /** + * Creates a byte array from a {@link IntType} {@link RandomAccessibleInterval}. + * + * @param imgTensor + * {@link RandomAccessibleInterval} to be mapped into byte buffer + * @param order + * order of the bytes + * @return byte array containing the data of the image as a flat byte array + */ + private static byte[] buildInt(RandomAccessibleInterval imgTensor, ByteOrder order) + { + Cursor tensorCursor; + if (imgTensor instanceof IntervalView) + tensorCursor = ((IntervalView) imgTensor).cursor(); + else if (imgTensor instanceof Img) + tensorCursor = ((Img) imgTensor).cursor(); + else + throw new IllegalArgumentException("The data of the " + Tensor.class + " has " + + "to be an instance of " + Img.class + " or " + IntervalView.class); + long flatSize = 4; + for (long ss : imgTensor.dimensionsAsLongArray()) {flatSize *= ss;} + byte[] byteArr = new byte[(int) flatSize]; + int cc = 0; + while (tensorCursor.hasNext()) { + tensorCursor.fwd(); + int val = tensorCursor.get().getInt(); + byte[] arr = ByteBuffer.allocate(4).order(order).putInt(val).array(); + System.arraycopy(arr, 0, byteArr, cc * 4, 4); + cc ++; + } + return byteArr; + } + + /** + * Creates a byte array from a {@link UnsignedIntType} {@link RandomAccessibleInterval}. + * + * @param imgTensor + * {@link RandomAccessibleInterval} to be mapped into byte buffer + * @param order + * order of the bytes + * @return byte array containing the data of the image as a flat byte array + */ + private static byte[] buildUInt(RandomAccessibleInterval imgTensor, ByteOrder order) + { + Cursor tensorCursor; + if (imgTensor instanceof IntervalView) + tensorCursor = ((IntervalView) imgTensor).cursor(); + else if (imgTensor instanceof Img) + tensorCursor = ((Img) imgTensor).cursor(); + else + throw new IllegalArgumentException("The data of the " + Tensor.class + " has " + + "to be an instance of " + Img.class + " or " + IntervalView.class); + long flatSize = 4; + for (long ss : imgTensor.dimensionsAsLongArray()) {flatSize *= ss;} + byte[] byteArr = new byte[(int) flatSize]; + int cc = 0; + while (tensorCursor.hasNext()) { + tensorCursor.fwd(); + long val = tensorCursor.get().get(); + int intval; + if (val >= Math.pow(2, 31)) + intval = (int) (val - Math.pow(2, 32)); + else + intval = (int) val; + byte[] arr = ByteBuffer.allocate(4).order(order).putInt(intval).array(); + System.arraycopy(arr, 0, byteArr, cc * 4, 4); + cc ++; + } + return byteArr; + } + + /** + * Creates a byte array from a {@link LongType} {@link RandomAccessibleInterval}. + * + * @param imgTensor + * {@link RandomAccessibleInterval} to be mapped into byte buffer + * @param order + * order of the bytes + * @return byte array containing the data of the image as a flat byte array + */ + private static byte[] buildLong(RandomAccessibleInterval imgTensor, ByteOrder order) + { + Cursor tensorCursor; + if (imgTensor instanceof IntervalView) + tensorCursor = ((IntervalView) imgTensor).cursor(); + else if (imgTensor instanceof Img) + tensorCursor = ((Img) imgTensor).cursor(); + else + throw new IllegalArgumentException("The data of the " + Tensor.class + " has " + + "to be an instance of " + Img.class + " or " + IntervalView.class); + long flatSize = 8; + for (long ss : imgTensor.dimensionsAsLongArray()) {flatSize *= ss;} + byte[] byteArr = new byte[(int) flatSize]; + int cc = 0; + while (tensorCursor.hasNext()) { + tensorCursor.fwd(); + long val = tensorCursor.get().get(); + byte[] arr = ByteBuffer.allocate(4).order(order).putLong(val).array();; + System.arraycopy(arr, 0, byteArr, cc * 8, 8); + cc ++; + } + return byteArr; + } + + /** + * Creates a byte array from a {@link FloatType} {@link RandomAccessibleInterval}. + * + * @param imgTensor + * {@link RandomAccessibleInterval} to be mapped into byte buffer + * @param order + * order of the bytes + * @return byte array containing the data of the image as a flat byte array + */ + private static byte[] buildFloat(RandomAccessibleInterval imgTensor, ByteOrder order) + { + Cursor tensorCursor; + if (imgTensor instanceof IntervalView) + tensorCursor = ((IntervalView) imgTensor).cursor(); + else if (imgTensor instanceof Img) + tensorCursor = ((Img) imgTensor).cursor(); + else + throw new IllegalArgumentException("The data of the " + Tensor.class + " has " + + "to be an instance of " + Img.class + " or " + IntervalView.class); + long flatSize = 4; + for (long ss : imgTensor.dimensionsAsLongArray()) {flatSize *= ss;} + byte[] byteArr = new byte[(int) flatSize]; + long[] shape = imgTensor.dimensionsAsLongArray(); + while (tensorCursor.hasNext()) { + tensorCursor.fwd(); + int flatPos = IndexingUtils.multidimensionalIntoFlatIndex( tensorCursor.positionAsLongArray(), shape); + float val = tensorCursor.get().get(); + byte[] arr = ByteBuffer.allocate(4).order(order).putFloat(val).array();; + System.arraycopy(arr, 0, byteArr, flatPos * 4, 4); + } + return byteArr; + } + + /** + * Creates a byte array from a {@link DoubleType} {@link RandomAccessibleInterval}. + * + * @param imgTensor + * {@link RandomAccessibleInterval} to be mapped into byte buffer + * @param order + * order of the bytes + * @return byte array containing the data of the image as a flat byte array + */ + private static byte[] buildDouble(RandomAccessibleInterval imgTensor, ByteOrder order) + { + Cursor tensorCursor; + if (imgTensor instanceof IntervalView) + tensorCursor = ((IntervalView) imgTensor).cursor(); + else if (imgTensor instanceof Img) + tensorCursor = ((Img) imgTensor).cursor(); + else + throw new IllegalArgumentException("The data of the " + Tensor.class + " has " + + "to be an instance of " + Img.class + " or " + IntervalView.class); + long flatSize = 8; + for (long ss : imgTensor.dimensionsAsLongArray()) {flatSize *= ss;} + byte[] byteArr = new byte[(int) flatSize]; + int cc = 0; + while (tensorCursor.hasNext()) { + tensorCursor.fwd(); + double val = tensorCursor.get().getRealDouble(); + byte[] arr = ByteBuffer.allocate(4).order(order).putDouble(val).array();; + System.arraycopy(arr, 0, byteArr, cc * 8, 8); + cc ++; + } + return byteArr; + } +}