Skip to content

Commit

Permalink
keep correcting chain of errors
Browse files Browse the repository at this point in the history
  • Loading branch information
@carlosuc3m
carlosuc3m committed Oct 18, 2024
1 parent 510d93c commit 7a4f2cd
Show file tree
Hide file tree
Showing 5 changed files with 56 additions and 50 deletions.
12 changes: 8 additions & 4 deletions src/main/java/io/bioimage/modelrunner/numpy/DecodeNumpy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -565,13 +565,15 @@ private static Img<ByteType> buildBoolean(ByteBuffer buf, ByteOrder byteOrder, l
* possible ImgLib2 datatypes of the {@link RandomAccessibleInterval}
* @param rai
* the n-dimensional array of interest
* @param
* whether the array is in fortran order or not
* @return a long value specifying the number of bytes it would take to store the nd-array in Numpy format
*/
public static < T extends RealType< T > & NativeType< T > >
long calculateNpyStyleByteArrayLength(RandomAccessibleInterval<T> rai) {
long calculateNpyStyleByteArrayLength(RandomAccessibleInterval<T> rai, boolean fortranOrder) {
String strHeader = "{'descr': '<";
strHeader += getDataType(rai.getAt(rai.minAsLongArray()));
strHeader += "', 'fortran_order': False, 'shape': (";
strHeader += "', 'fortran_order': " + (fortranOrder ? "True" : "False") + ", 'shape': (";
for (long ll : rai.dimensionsAsLongArray()) strHeader += ll + ", ";
strHeader = strHeader.substring(0, strHeader.length() - 2);
strHeader += "), }" + System.lineSeparator();
Expand Down Expand Up @@ -614,13 +616,15 @@ long calculateNpyStyleByteArrayLength(RandomAccessibleInterval<T> rai) {
* the dimensions of the nd array
* @param datatype
* the data type of the nd array
* @param fortranOrder
* whether the array is going to be saved on fortan order or not
* @return a long value specifying the number of bytes it would take to store the nd-array in Numpy npy format
*/
public static < T extends RealType< T > & NativeType< T > >
long calculateNpyStyleByteArrayLength(long[] shape, T datatype) {
long calculateNpyStyleByteArrayLength(long[] shape, T datatype, boolean fortranOrder) {
String strHeader = "{'descr': '<";
strHeader += getDataType(datatype);
strHeader += "', 'fortran_order': False, 'shape': (";
strHeader += "', 'fortran_order': " + (fortranOrder ? "True" : "False") + ", 'shape': (";
for (long ll : shape) strHeader += ll + ", ";
strHeader = strHeader.substring(0, strHeader.length() - 2);
strHeader += "), }" + System.lineSeparator();
Expand Down
12 changes: 7 additions & 5 deletions src/main/java/io/bioimage/modelrunner/tensor/shm/SharedMemoryArray.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,7 @@ SharedMemoryArray readOrCreate(String name, long[] shape, T datatype, boolean is
String strDType = CommonUtils.getDataType(datatype);
int size = DecodeNumpy.DATA_TYPES_MAP.get(strDType);
for (long i : shape) {size *= i;}
if (isNpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(shape, datatype);
if (isNpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(shape, datatype, isFortran);
if (PlatformDetection.isWindows())
return SharedMemoryArrayWin.readOrCreate(name, size, shape, strDType, isNpy, isFortran);
else if (PlatformDetection.isLinux())
Expand Down Expand Up @@ -300,7 +300,7 @@ SharedMemoryArray create(long[] shape, T datatype, boolean isFortran, boolean is
String strDType = CommonUtils.getDataType(datatype);
int size = DecodeNumpy.DATA_TYPES_MAP.get(strDType);
for (long i : shape) {size *= i;}
if (isNpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(shape, datatype);
if (isNpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(shape, datatype, isFortran);
if (PlatformDetection.isWindows())
return SharedMemoryArrayWin.create(size, shape, strDType, isNpy, isFortran);
else if (PlatformDetection.isLinux())
Expand Down Expand Up @@ -524,7 +524,7 @@ static String createShmName() {
* @return the number of bytes needed to store the nd array
*/
public static <T extends RealType<T> & NativeType<T>> int getArrayByteSize(long[] shape, T type) {
return getArrayByteSize(shape, type, false);
return getArrayByteSize(shape, type, false, false);
}

/**
Expand All @@ -537,12 +537,14 @@ public static <T extends RealType<T> & NativeType<T>> int getArrayByteSize(long[
* ImgLib2 data type of the array
* @param isNpy
* whether the array is stored with a Numpy npy header at the beginning
* @param isFortran
* whether the array is stored as fortran or not. Only relevant for this method if the aisNpy argument is true
* @return the number of bytes needed to store the nd array
*/
public static <T extends RealType<T> & NativeType<T>> int getArrayByteSize(long[] shape, T type, boolean isNpy) {
public static <T extends RealType<T> & NativeType<T>> int getArrayByteSize(long[] shape, T type, boolean isNpy, boolean isFortran) {
int noByteSize = 1;
int headerSize = 0;
if (isNpy) headerSize = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(shape, type);
if (isNpy) headerSize = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(shape, type, isFortran);
for (long l : shape) {noByteSize *= l;}
if (type instanceof ByteType || type instanceof UnsignedByteType) {
return noByteSize * 1 + headerSize;
Expand Down
40 changes: 20 additions & 20 deletions src/main/java/io/bioimage/modelrunner/tensor/shm/SharedMemoryArrayLinux.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -353,55 +353,55 @@ SharedMemoryArrayLinux createSHMAFromRAI(String name, RandomAccessibleInterval<T
if (Util.getTypeFromInterval(rai) instanceof ByteType) {
int size = 1;
for (long i : rai.dimensionsAsLongArray()) {size *= i;}
if (isNumpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(rai);
if (isNumpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(rai, isFortranOrder);
shma = new SharedMemoryArrayLinux(name, size, CommonUtils.getDataTypeFromRAI(rai), rai.dimensionsAsLongArray(), isNumpy, isFortranOrder);
shma.buildInt8(Cast.unchecked(rai), isFortranOrder, isNumpy);
} else if (Util.getTypeFromInterval(rai) instanceof UnsignedByteType) {
int size = 1;
for (long i : rai.dimensionsAsLongArray()) {size *= i;}
if (isNumpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(rai);
if (isNumpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(rai, isFortranOrder);
shma = new SharedMemoryArrayLinux(name, size, CommonUtils.getDataTypeFromRAI(rai), rai.dimensionsAsLongArray(), isNumpy, isFortranOrder);
shma.buildUint8(Cast.unchecked(rai), isFortranOrder, isNumpy);
} else if (Util.getTypeFromInterval(rai) instanceof ShortType) {
int size = 2;
for (long i : rai.dimensionsAsLongArray()) {size *= i;}
if (isNumpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(rai);
if (isNumpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(rai, isFortranOrder);
shma = new SharedMemoryArrayLinux(name, size, CommonUtils.getDataTypeFromRAI(rai), rai.dimensionsAsLongArray(), isNumpy, isFortranOrder);
shma.buildInt16(Cast.unchecked(rai), isFortranOrder, isNumpy);
} else if (Util.getTypeFromInterval(rai) instanceof UnsignedShortType) {
int size = 2;
for (long i : rai.dimensionsAsLongArray()) {size *= i;}
if (isNumpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(rai);
if (isNumpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(rai, isFortranOrder);
shma = new SharedMemoryArrayLinux(name, size, CommonUtils.getDataTypeFromRAI(rai), rai.dimensionsAsLongArray(), isNumpy, isFortranOrder);
shma.buildUint16(Cast.unchecked(rai), isFortranOrder, isNumpy);
} else if (Util.getTypeFromInterval(rai) instanceof IntType) {
int size = 4;
for (long i : rai.dimensionsAsLongArray()) {size *= i;}
if (isNumpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(rai);
if (isNumpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(rai, isFortranOrder);
shma = new SharedMemoryArrayLinux(name, size, CommonUtils.getDataTypeFromRAI(rai), rai.dimensionsAsLongArray(), isNumpy, isFortranOrder);
shma.buildInt32(Cast.unchecked(rai), isFortranOrder, isNumpy);
} else if (Util.getTypeFromInterval(rai) instanceof UnsignedIntType) {
int size = 4;
for (long i : rai.dimensionsAsLongArray()) {size *= i;}
if (isNumpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(rai);
if (isNumpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(rai, isFortranOrder);
shma = new SharedMemoryArrayLinux(name, size, CommonUtils.getDataTypeFromRAI(rai), rai.dimensionsAsLongArray(), isNumpy, isFortranOrder);
shma.buildUint32(Cast.unchecked(rai), isFortranOrder, isNumpy);
} else if (Util.getTypeFromInterval(rai) instanceof LongType) {
int size = 8;
for (long i : rai.dimensionsAsLongArray()) {size *= i;}
if (isNumpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(rai);
if (isNumpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(rai, isFortranOrder);
shma = new SharedMemoryArrayLinux(name, size, CommonUtils.getDataTypeFromRAI(rai), rai.dimensionsAsLongArray(), isNumpy, isFortranOrder);
shma.buildInt64(Cast.unchecked(rai), isFortranOrder, isNumpy);
} else if (Util.getTypeFromInterval(rai) instanceof FloatType) {
int size = 4;
for (long i : rai.dimensionsAsLongArray()) {size *= i;}
if (isNumpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(rai);
if (isNumpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(rai, isFortranOrder);
shma = new SharedMemoryArrayLinux(name, size, CommonUtils.getDataTypeFromRAI(rai), rai.dimensionsAsLongArray(), isNumpy, isFortranOrder);
shma.buildFloat32(Cast.unchecked(rai), isFortranOrder, isNumpy);
} else if (Util.getTypeFromInterval(rai) instanceof DoubleType) {
int size = 8;
for (long i : rai.dimensionsAsLongArray()) {size *= i;}
if (isNumpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(rai);
if (isNumpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(rai, isFortranOrder);
shma = new SharedMemoryArrayLinux(name, size, CommonUtils.getDataTypeFromRAI(rai), rai.dimensionsAsLongArray(), isNumpy, isFortranOrder);
shma.buildFloat64(Cast.unchecked(rai), isFortranOrder, isNumpy);
} else {
Expand Down Expand Up @@ -476,7 +476,7 @@ private void addByteArray(byte[] arr) {
private void buildInt8(RandomAccessibleInterval<ByteType> tensor, boolean isFortranOrder, boolean isNumpy)
{
byte[] header = new byte[0];
if (isNumpy) header = getNpyHeader(tensor);
if (isNumpy) header = getNpyHeader(tensor, this.isFortran);
long offset = 0;
for (byte b : header) {
this.pSharedMemory.setByte(offset, b);
Expand All @@ -494,7 +494,7 @@ private void buildInt8(RandomAccessibleInterval<ByteType> tensor, boolean isFort
private void buildUint8(RandomAccessibleInterval<UnsignedByteType> tensor, boolean isFortranOrder, boolean isNumpy)
{
byte[] header = new byte[0];
if (isNumpy) header = getNpyHeader(tensor);
if (isNumpy) header = getNpyHeader(tensor, isFortranOrder);
long offset = 0;
for (byte b : header) {
this.pSharedMemory.setByte(offset, b);
Expand All @@ -512,7 +512,7 @@ private void buildUint8(RandomAccessibleInterval<UnsignedByteType> tensor, boole
private void buildInt16(RandomAccessibleInterval<ShortType> tensor, boolean isFortranOrder, boolean isNumpy)
{
byte[] header = new byte[0];
if (isNumpy) header = getNpyHeader(tensor);
if (isNumpy) header = getNpyHeader(tensor, this.isFortran);
long offset = 0;
for (byte b : header) {
this.pSharedMemory.setByte(offset, b);
Expand All @@ -531,7 +531,7 @@ private void buildInt16(RandomAccessibleInterval<ShortType> tensor, boolean isFo
private void buildUint16(RandomAccessibleInterval<UnsignedShortType> tensor, boolean isFortranOrder, boolean isNumpy)
{
byte[] header = new byte[0];
if (isNumpy) header = getNpyHeader(tensor);
if (isNumpy) header = getNpyHeader(tensor, this.isFortran);
long offset = 0;
for (byte b : header) {
this.pSharedMemory.setByte(offset, b);
Expand All @@ -550,7 +550,7 @@ private void buildUint16(RandomAccessibleInterval<UnsignedShortType> tensor, boo
private void buildInt32(RandomAccessibleInterval<IntType> tensor, boolean isFortranOrder, boolean isNumpy)
{
byte[] header = new byte[0];
if (isNumpy) header = getNpyHeader(tensor);
if (isNumpy) header = getNpyHeader(tensor, this.isFortran);
long offset = 0;
for (byte b : header) {
this.pSharedMemory.setByte(offset, b);
Expand All @@ -569,7 +569,7 @@ private void buildInt32(RandomAccessibleInterval<IntType> tensor, boolean isFort
private void buildUint32(RandomAccessibleInterval<UnsignedIntType> tensor, boolean isFortranOrder, boolean isNumpy)
{
byte[] header = new byte[0];
if (isNumpy) header = getNpyHeader(tensor);
if (isNumpy) header = getNpyHeader(tensor, this.isFortran);
long offset = 0;
for (byte b : header) {
this.pSharedMemory.setByte(offset, b);
Expand All @@ -588,7 +588,7 @@ private void buildUint32(RandomAccessibleInterval<UnsignedIntType> tensor, boole
private void buildInt64(RandomAccessibleInterval<LongType> tensor, boolean isFortranOrder, boolean isNumpy)
{
byte[] header = new byte[0];
if (isNumpy) header = getNpyHeader(tensor);
if (isNumpy) header = getNpyHeader(tensor, this.isFortran);
long offset = 0;
for (byte b : header) {
this.pSharedMemory.setByte(offset, b);
Expand All @@ -607,7 +607,7 @@ private void buildInt64(RandomAccessibleInterval<LongType> tensor, boolean isFor
private void buildFloat32(RandomAccessibleInterval<FloatType> tensor, boolean isFortranOrder, boolean isNumpy)
{
byte[] header = new byte[0];
if (isNumpy) header = getNpyHeader(tensor);
if (isNumpy) header = getNpyHeader(tensor, this.isFortran);
long offset = 0;
for (byte b : header) {
this.pSharedMemory.setByte(offset, b);
Expand All @@ -626,7 +626,7 @@ private void buildFloat32(RandomAccessibleInterval<FloatType> tensor, boolean is
private void buildFloat64(RandomAccessibleInterval<DoubleType> tensor, boolean isFortranOrder, boolean isNumpy)
{
byte[] header = new byte[0];
if (isNumpy) header = getNpyHeader(tensor);
if (isNumpy) header = getNpyHeader(tensor, this.isFortran);
long offset = 0;
for (byte b : header) {
this.pSharedMemory.setByte(offset, b);
Expand All @@ -643,10 +643,10 @@ private void buildFloat64(RandomAccessibleInterval<DoubleType> tensor, boolean i
}

private static <T extends RealType<T> & NativeType<T>>
byte[] getNpyHeader(RandomAccessibleInterval<T> tensor) {
byte[] getNpyHeader(RandomAccessibleInterval<T> tensor, boolean fortranOrder) {
String strHeader = "{'descr': '<";
strHeader += DecodeNumpy.getDataType(tensor.getAt(tensor.minAsLongArray()));
strHeader += "', 'fortran_order': False, 'shape': (";
strHeader += "', 'fortran_order': " + (fortranOrder ? "True" : "False") + ", 'shape': (";
for (long ll : tensor.dimensionsAsLongArray()) strHeader += ll + ", ";
strHeader = strHeader.substring(0, strHeader.length() - 2);
strHeader += "), }" + System.lineSeparator();
Expand Down
18 changes: 9 additions & 9 deletions src/main/java/io/bioimage/modelrunner/tensor/shm/SharedMemoryArrayMacOS.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -326,55 +326,55 @@ SharedMemoryArrayMacOS createSHMAFromRAI(String name, RandomAccessibleInterval<T
if (Util.getTypeFromInterval(rai) instanceof ByteType) {
int size = 1;
for (long i : rai.dimensionsAsLongArray()) {size *= i;}
if (isNumpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(rai);
if (isNumpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(rai, isFortranOrder);
shma = new SharedMemoryArrayMacOS(name, size, CommonUtils.getDataTypeFromRAI(rai), rai.dimensionsAsLongArray(), isNumpy, isFortranOrder);
shma.buildInt8(Cast.unchecked(rai), isFortranOrder, isNumpy);
} else if (Util.getTypeFromInterval(rai) instanceof UnsignedByteType) {
int size = 1;
for (long i : rai.dimensionsAsLongArray()) {size *= i;}
if (isNumpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(rai);
if (isNumpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(rai, isFortranOrder);
shma = new SharedMemoryArrayMacOS(name, size, CommonUtils.getDataTypeFromRAI(rai), rai.dimensionsAsLongArray(), isNumpy, isFortranOrder);
shma.buildUint8(Cast.unchecked(rai), isFortranOrder, isNumpy);
} else if (Util.getTypeFromInterval(rai) instanceof ShortType) {
int size = 2;
for (long i : rai.dimensionsAsLongArray()) {size *= i;}
if (isNumpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(rai);
if (isNumpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(rai, isFortranOrder);
shma = new SharedMemoryArrayMacOS(name, size, CommonUtils.getDataTypeFromRAI(rai), rai.dimensionsAsLongArray(), isNumpy, isFortranOrder);
shma.buildInt16(Cast.unchecked(rai), isFortranOrder, isNumpy);
} else if (Util.getTypeFromInterval(rai) instanceof UnsignedShortType) {
int size = 2;
for (long i : rai.dimensionsAsLongArray()) {size *= i;}
if (isNumpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(rai);
if (isNumpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(rai, isFortranOrder);
shma = new SharedMemoryArrayMacOS(name, size, CommonUtils.getDataTypeFromRAI(rai), rai.dimensionsAsLongArray(), isNumpy, isFortranOrder);
shma.buildUint16(Cast.unchecked(rai), isFortranOrder, isNumpy);
} else if (Util.getTypeFromInterval(rai) instanceof IntType) {
int size = 4;
for (long i : rai.dimensionsAsLongArray()) {size *= i;}
if (isNumpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(rai);
if (isNumpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(rai, isFortranOrder);
shma = new SharedMemoryArrayMacOS(name, size, CommonUtils.getDataTypeFromRAI(rai), rai.dimensionsAsLongArray(), isNumpy, isFortranOrder);
shma.buildInt32(Cast.unchecked(rai), isFortranOrder, isNumpy);
} else if (Util.getTypeFromInterval(rai) instanceof UnsignedIntType) {
int size = 4;
for (long i : rai.dimensionsAsLongArray()) {size *= i;}
if (isNumpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(rai);
if (isNumpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(rai, isFortranOrder);
shma = new SharedMemoryArrayMacOS(name, size, CommonUtils.getDataTypeFromRAI(rai), rai.dimensionsAsLongArray(), isNumpy, isFortranOrder);
shma.buildUint32(Cast.unchecked(rai), isFortranOrder, isNumpy);
} else if (Util.getTypeFromInterval(rai) instanceof LongType) {
int size = 8;
for (long i : rai.dimensionsAsLongArray()) {size *= i;}
if (isNumpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(rai);
if (isNumpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(rai, isFortranOrder);
shma = new SharedMemoryArrayMacOS(name, size, CommonUtils.getDataTypeFromRAI(rai), rai.dimensionsAsLongArray(), isNumpy, isFortranOrder);
shma.buildInt64(Cast.unchecked(rai), isFortranOrder, isNumpy);
} else if (Util.getTypeFromInterval(rai) instanceof FloatType) {
int size = 4;
for (long i : rai.dimensionsAsLongArray()) {size *= i;}
if (isNumpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(rai);
if (isNumpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(rai, isFortranOrder);
shma = new SharedMemoryArrayMacOS(name, size, CommonUtils.getDataTypeFromRAI(rai), rai.dimensionsAsLongArray(), isNumpy, isFortranOrder);
shma.buildFloat32(Cast.unchecked(rai), isFortranOrder, isNumpy);
} else if (Util.getTypeFromInterval(rai) instanceof DoubleType) {
int size = 8;
for (long i : rai.dimensionsAsLongArray()) {size *= i;}
if (isNumpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(rai);
if (isNumpy) size = (int) DecodeNumpy.calculateNpyStyleByteArrayLength(rai, isFortranOrder);
shma = new SharedMemoryArrayMacOS(name, size, CommonUtils.getDataTypeFromRAI(rai), rai.dimensionsAsLongArray(), isNumpy, isFortranOrder);
shma.buildFloat64(Cast.unchecked(rai), isFortranOrder, isNumpy);
} else {
Expand Down
Loading

0 comments on commit 7a4f2cd

Please sign in to comment.