From c1c59a02ec95b8ebcca08f4fafe6874116f2b472 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Thu, 4 Apr 2024 16:15:07 -0500 Subject: [PATCH 1/8] Update JPEGTurboService to return compressed tiles, tile dimensions --- .../formats/services/JPEGTurboService.java | 71 ++++++++++++++++++- .../services/JPEGTurboServiceImpl.java | 64 ++++++++++++----- 2 files changed, 115 insertions(+), 20 deletions(-) diff --git a/components/formats-bsd/src/loci/formats/services/JPEGTurboService.java b/components/formats-bsd/src/loci/formats/services/JPEGTurboService.java index 85570cbaf2c..9ecc8b9c9e9 100644 --- a/components/formats-bsd/src/loci/formats/services/JPEGTurboService.java +++ b/components/formats-bsd/src/loci/formats/services/JPEGTurboService.java @@ -44,21 +44,90 @@ */ public interface JPEGTurboService extends Service { + /** + * @return the restart markers associated with the current JPEG stream + */ long[] getRestartMarkers(); + /** + * @param markers precalculated restart markers associated with + * the current JPEG stream + */ void setRestartMarkers(long[] markers); + /** + * Initialize the given stream, which represents an image of the given + * width and height. This service is primarily intended for very large + * JPEG images whose width and/or height exceed 65535 (the maximum + * that can be recorded in a JPEG stream). + * + * @param jpeg open stream containing JPEG data + * @param width total image width + * @param height total image height + */ void initialize(RandomAccessInputStream jpeg, int width, int height) throws ServiceException, IOException; + /** + * @return the width (in pixels) of a tile + */ + int getTileWidth(); + + /** + * @return the height (in pixels) of a tile + */ + int getTileHeight(); + + /** + * @return the number of rows of tiles + */ + int getTileRows(); + + /** + * @return the number of columns of tiles + */ + int getTileColumns(); + + /** + * Get the uncompressed bytes representing the given bounding box. + * + * @param buf array in which to store uncompressed bytes + * @param xCoordinate upper-left X coordinate of bounding box + * @param yCoordinate upper-left Y coordinate of bounding box + * @param width width of bounding box + * @param height height of bounding box + * @return uncompressed bytes + */ byte[] getTile(byte[] buf, int xCoordinate, int yCoordinate, int width, int height) throws IOException; + /** + * Get the uncompressed bytes representing the given tile index. + * + * @param xTile column index of the tile + * @param yTile row index of the tile + * @return uncompressed bytes + */ byte[] getTile(int xTile, int yTile) throws IOException; + /** + * Similar to getTile(int, int), but returns the JPEG-compressed bytes. + * + * @param xTile column index of the tile + * @param yTile row index of the tile + * @return JPEG-compressed bytes + */ + byte[] getCompressedTile(int xTile, int yTile) throws IOException; + + /** + * Free resources associated with the initialized stream. + */ void close() throws IOException; - + + /** + * @return true if the underlying native library was successfully loaded + */ boolean isLibraryLoaded(); } diff --git a/components/formats-bsd/src/loci/formats/services/JPEGTurboServiceImpl.java b/components/formats-bsd/src/loci/formats/services/JPEGTurboServiceImpl.java index cfe316b1513..2e79544413b 100644 --- a/components/formats-bsd/src/loci/formats/services/JPEGTurboServiceImpl.java +++ b/components/formats-bsd/src/loci/formats/services/JPEGTurboServiceImpl.java @@ -229,6 +229,26 @@ else if (marker == SOS) { } } + @Override + public int getTileWidth() { + return tileWidth; + } + + @Override + public int getTileHeight() { + return tileHeight; + } + + @Override + public int getTileRows() { + return yTiles; + } + + @Override + public int getTileColumns() { + return xTiles; + } + @Override public byte[] getTile(byte[] buf, int xCoordinate, int yCoordinate, int width, int height) @@ -287,6 +307,30 @@ public byte[] getTile(byte[] buf, int xCoordinate, int yCoordinate, @Override public byte[] getTile(int tileX, int tileY) throws IOException { + byte[] compressedData = getCompressedTile(tileX, tileY); + + // and here we actually decompress it... + + try { + int pixelType = TJ.PF_RGB; + int pixelSize = TJ.getPixelSize(pixelType); + + TJDecompressor decoder = new TJDecompressor(compressedData); + byte[] decompressed = decoder.decompress(tileWidth, tileWidth * pixelSize, + tileHeight, pixelType, pixelType); + compressedData = null; + decoder.close(); + return decompressed; + } + catch (Exception e) { + IOException ioe = new IOException(e.getMessage()); + ioe.initCause(e); + throw ioe; + } + } + + @Override + public byte[] getCompressedTile(int tileX, int tileY) throws IOException { if (header == null) { header = getFixedHeader(); } @@ -343,25 +387,7 @@ public byte[] getTile(int tileX, int tileY) throws IOException { } DataTools.unpackBytes(EOI, data, offset, 2, false); - - // and here we actually decompress it... - - try { - int pixelType = TJ.PF_RGB; - int pixelSize = TJ.getPixelSize(pixelType); - - TJDecompressor decoder = new TJDecompressor(data); - byte[] decompressed = decoder.decompress(tileWidth, tileWidth * pixelSize, - tileHeight, pixelType, pixelType); - data = null; - decoder.close(); - return decompressed; - } - catch (Exception e) { - IOException ioe = new IOException(e.getMessage()); - ioe.initCause(e); - throw ioe; - } + return data; } @Override From 32c9fecc09e233fb2c70cb7542b38fb6cca4444f Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Mon, 8 Apr 2024 19:36:50 -0500 Subject: [PATCH 2/8] NDPI: first version of precompressed tile reading --- .../formats/services/JPEGTurboService.java | 10 + .../services/JPEGTurboServiceImpl.java | 11 +- .../src/loci/formats/in/NDPIReader.java | 264 ++++++++++++++---- 3 files changed, 232 insertions(+), 53 deletions(-) diff --git a/components/formats-bsd/src/loci/formats/services/JPEGTurboService.java b/components/formats-bsd/src/loci/formats/services/JPEGTurboService.java index 9ecc8b9c9e9..bddf1be24bb 100644 --- a/components/formats-bsd/src/loci/formats/services/JPEGTurboService.java +++ b/components/formats-bsd/src/loci/formats/services/JPEGTurboService.java @@ -120,6 +120,16 @@ byte[] getTile(byte[] buf, int xCoordinate, int yCoordinate, int width, */ byte[] getCompressedTile(int xTile, int yTile) throws IOException; + /** + * Similar to getTile(int, int), but returns the JPEG-compressed bytes. + * + * @param data preallocated array for storing tile + * @param xTile column index of the tile + * @param yTile row index of the tile + * @return JPEG-compressed bytes + */ + byte[] getCompressedTile(byte[] data, int xTile, int yTile) throws IOException; + /** * Free resources associated with the initialized stream. */ diff --git a/components/formats-bsd/src/loci/formats/services/JPEGTurboServiceImpl.java b/components/formats-bsd/src/loci/formats/services/JPEGTurboServiceImpl.java index 2e79544413b..c2e194801a5 100644 --- a/components/formats-bsd/src/loci/formats/services/JPEGTurboServiceImpl.java +++ b/components/formats-bsd/src/loci/formats/services/JPEGTurboServiceImpl.java @@ -356,12 +356,21 @@ public byte[] getCompressedTile(int tileX, int tileY) throws IOException { } byte[] data = new byte[(int) dataLength]; + return getCompressedTile(data, tileX, tileY); + } + + @Override + public byte[] getCompressedTile(byte[] data, int tileX, int tileY) throws IOException { + if (header == null) { + header = getFixedHeader(); + } int offset = 0; System.arraycopy(header, 0, data, offset, header.length); offset += header.length; - start = tileX + (tileY * xTiles * mult); + int mult = tileHeight / mcuHeight; // was restartInterval + int start = tileX + (tileY * xTiles * mult); for (int row=0; row Date: Tue, 9 Apr 2024 14:28:33 -0500 Subject: [PATCH 3/8] Fix interpretation of x and y coordinates --- .../src/loci/formats/in/NDPIReader.java | 24 ++----------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/components/formats-gpl/src/loci/formats/in/NDPIReader.java b/components/formats-gpl/src/loci/formats/in/NDPIReader.java index 3771cf3d829..85e6f5a0285 100644 --- a/components/formats-gpl/src/loci/formats/in/NDPIReader.java +++ b/components/formats-gpl/src/loci/formats/in/NDPIReader.java @@ -158,17 +158,7 @@ public byte[] openCompressedBytes(int no, int x, int y) throws FormatException, initializedSeries = getCoreIndex(); initializedPlane = no; } - if ((x % service.getTileWidth()) != 0) { - throw new FormatException("Invalid x: " + x + " must be multiple of " + - service.getTileWidth()); - } - if ((y % service.getTileHeight()) != 0) { - throw new FormatException("Invalid y: " + y + " must be multiple of " + - service.getTileHeight()); - } - int tileRow = y / service.getTileHeight(); - int tileColumn = x / service.getTileWidth(); - return service.getCompressedTile(tileColumn, tileRow); + return service.getCompressedTile(x, y); } @Override @@ -188,17 +178,7 @@ public byte[] openCompressedBytes(int no, byte[] buf, int x, int y) throws Forma initializedSeries = getCoreIndex(); initializedPlane = no; } - if ((x % service.getTileWidth()) != 0) { - throw new FormatException("Invalid x: " + x + " must be multiple of " + - service.getTileWidth()); - } - if ((y % service.getTileHeight()) != 0) { - throw new FormatException("Invalid y: " + y + " must be multiple of " + - service.getTileHeight()); - } - int tileRow = y / service.getTileHeight(); - int tileColumn = x / service.getTileWidth(); - service.getCompressedTile(buf, tileColumn, tileRow); + service.getCompressedTile(buf, x, y); return buf; } From e21ed517903f80a5b093ad6ae729a9f3e8645eb7 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Tue, 9 Apr 2024 15:27:24 -0500 Subject: [PATCH 4/8] Refactor generic TIFF tile logic --- .../loci/formats/in/MinimalTiffReader.java | 69 ++++++++++++++++++ .../src/loci/formats/in/NDPIReader.java | 7 +- .../src/loci/formats/in/SVSReader.java | 72 +------------------ 3 files changed, 74 insertions(+), 74 deletions(-) diff --git a/components/formats-bsd/src/loci/formats/in/MinimalTiffReader.java b/components/formats-bsd/src/loci/formats/in/MinimalTiffReader.java index 75eb93e728f..7e3907efe0f 100644 --- a/components/formats-bsd/src/loci/formats/in/MinimalTiffReader.java +++ b/components/formats-bsd/src/loci/formats/in/MinimalTiffReader.java @@ -705,4 +705,73 @@ protected void initTiffParser() { tiffParser.setUse64BitOffsets(use64Bit); } + /** + * Get the index of the tile corresponding to given IFD (plane) + * and tile XY indexes. + * + * @param ifd IFD for the requested tile's plane + * @param x tile X index + * @param y tile Y index + * @return corresponding tile index + */ + protected int getTileIndex(IFD ifd, int x, int y) throws FormatException { + int rows = (int) ifd.getTilesPerColumn(); + int cols = (int) ifd.getTilesPerRow(); + + if (x < 0 || x >= cols) { + throw new IllegalArgumentException("X index " + x + " not in range [0, " + cols + ")"); + } + if (y < 0 || y >= rows) { + throw new IllegalArgumentException("Y index " + y + " not in range [0, " + rows + ")"); + } + + return (cols * y) + x; + } + + protected long getCompressedByteCount(IFD ifd, int x, int y) throws FormatException, IOException { + long[] byteCounts = ifd.getStripByteCounts(); + int tileIndex = getTileIndex(ifd, x, y); + byte[] jpegTable = (byte[]) ifd.getIFDValue(IFD.JPEG_TABLES); + int jpegTableBytes = jpegTable == null ? 0 : jpegTable.length - 2; + long expectedBytes = byteCounts[tileIndex]; + if (expectedBytes > 0) { + expectedBytes += jpegTableBytes; + } + if (expectedBytes < 0 || expectedBytes > Integer.MAX_VALUE) { + throw new IOException("Invalid compressed tile size: " + expectedBytes); + } + return expectedBytes; + } + + protected byte[] copyTile(IFD ifd, byte[] buf, int x, int y) throws FormatException, IOException { + long[] offsets = ifd.getStripOffsets(); + long[] byteCounts = ifd.getStripByteCounts(); + + int tileIndex = getTileIndex(ifd, x, y); + + byte[] jpegTable = (byte[]) ifd.getIFDValue(IFD.JPEG_TABLES); + int jpegTableBytes = jpegTable == null ? 0 : jpegTable.length - 2; + long expectedBytes = getCompressedByteCount(ifd, x, y); + + if (buf.length < expectedBytes) { + throw new IllegalArgumentException("Tile buffer too small: expected >=" + + expectedBytes + ", got " + buf.length); + } + else if (expectedBytes < 0 || expectedBytes > Integer.MAX_VALUE) { + throw new IOException("Invalid compressed tile size: " + expectedBytes); + } + + if (jpegTable != null && expectedBytes > 0) { + System.arraycopy(jpegTable, 0, buf, 0, jpegTable.length - 2); + // skip over the duplicate SOI marker + tiffParser.getStream().seek(offsets[tileIndex] + 2); + tiffParser.getStream().readFully(buf, jpegTable.length - 2, (int) byteCounts[tileIndex]); + } + else if (byteCounts[tileIndex] > 0) { + tiffParser.getStream().seek(offsets[tileIndex]); + tiffParser.getStream().readFully(buf, 0, (int) byteCounts[tileIndex]); + } + return buf; + } + } diff --git a/components/formats-gpl/src/loci/formats/in/NDPIReader.java b/components/formats-gpl/src/loci/formats/in/NDPIReader.java index 85e6f5a0285..4b3611b2412 100644 --- a/components/formats-gpl/src/loci/formats/in/NDPIReader.java +++ b/components/formats-gpl/src/loci/formats/in/NDPIReader.java @@ -149,8 +149,8 @@ public byte[] openCompressedBytes(int no, int x, int y) throws FormatException, IFD ifd = ifds.get(ifdIndex); if (useTiffParser(ifd)) { - // TODO - return null; + byte[] buf = new byte[(int) getCompressedByteCount(ifd, x, y)]; + return openCompressedBytes(no, buf, x, y); } if (initializedSeries != getCoreIndex() || initializedPlane != no) { @@ -169,8 +169,7 @@ public byte[] openCompressedBytes(int no, byte[] buf, int x, int y) throws Forma IFD ifd = ifds.get(ifdIndex); if (useTiffParser(ifd)) { - // TODO - return buf; + return copyTile(ifd, buf, x, y); } if (initializedSeries != getCoreIndex() || initializedPlane != no) { diff --git a/components/formats-gpl/src/loci/formats/in/SVSReader.java b/components/formats-gpl/src/loci/formats/in/SVSReader.java index b88b5478517..ef4ec80d074 100644 --- a/components/formats-gpl/src/loci/formats/in/SVSReader.java +++ b/components/formats-gpl/src/loci/formats/in/SVSReader.java @@ -306,21 +306,7 @@ public int getTileColumns(int no) { public byte[] openCompressedBytes(int no, int x, int y) throws FormatException, IOException { FormatTools.assertId(currentId, true, 1); IFD ifd = getIFD(no); - long[] byteCounts = ifd.getStripByteCounts(); - int tileIndex = getTileIndex(ifd, x, y); - - byte[] jpegTable = (byte[]) ifd.getIFDValue(IFD.JPEG_TABLES); - int jpegTableBytes = jpegTable == null ? 0 : jpegTable.length - 2; - long expectedBytes = byteCounts[tileIndex]; - if (expectedBytes > 0) { - expectedBytes += jpegTableBytes; - } - - if (expectedBytes < 0 || expectedBytes > Integer.MAX_VALUE) { - throw new IOException("Invalid compressed tile size: " + expectedBytes); - } - - byte[] buf = new byte[(int) expectedBytes]; + byte[] buf = new byte[(int) getCompressedByteCount(ifd, x, y)]; return openCompressedBytes(no, buf, x, y); } @@ -328,38 +314,7 @@ public byte[] openCompressedBytes(int no, int x, int y) throws FormatException, public byte[] openCompressedBytes(int no, byte[] buf, int x, int y) throws FormatException, IOException { FormatTools.assertId(currentId, true, 1); IFD ifd = getIFD(no); - long[] offsets = ifd.getStripOffsets(); - long[] byteCounts = ifd.getStripByteCounts(); - - int tileIndex = getTileIndex(ifd, x, y); - - byte[] jpegTable = (byte[]) ifd.getIFDValue(IFD.JPEG_TABLES); - int jpegTableBytes = jpegTable == null ? 0 : jpegTable.length - 2; - long expectedBytes = byteCounts[tileIndex]; - if (expectedBytes > 0) { - expectedBytes += jpegTableBytes; - } - - if (buf.length < expectedBytes) { - throw new IllegalArgumentException("Tile buffer too small: expected >=" + - expectedBytes + ", got " + buf.length); - } - else if (expectedBytes < 0 || expectedBytes > Integer.MAX_VALUE) { - throw new IOException("Invalid compressed tile size: " + expectedBytes); - } - - if (jpegTable != null && expectedBytes > 0) { - System.arraycopy(jpegTable, 0, buf, 0, jpegTable.length - 2); - // skip over the duplicate SOI marker - tiffParser.getStream().seek(offsets[tileIndex] + 2); - tiffParser.getStream().readFully(buf, jpegTable.length - 2, (int) byteCounts[tileIndex]); - } - else if (byteCounts[tileIndex] > 0) { - tiffParser.getStream().seek(offsets[tileIndex]); - tiffParser.getStream().readFully(buf, 0, (int) byteCounts[tileIndex]); - } - - return buf; + return copyTile(ifd, buf, x, y); } @Override @@ -873,27 +828,4 @@ protected IFD getIFD(int no) { return ifds.get(ifd); } - /** - * Get the index of the tile corresponding to given IFD (plane) - * and tile XY indexes. - * - * @param ifd IFD for the requested tile's plane - * @param x tile X index - * @param y tile Y index - * @return corresponding tile index - */ - protected int getTileIndex(IFD ifd, int x, int y) throws FormatException { - int rows = (int) ifd.getTilesPerColumn(); - int cols = (int) ifd.getTilesPerRow(); - - if (x < 0 || x >= cols) { - throw new IllegalArgumentException("X index " + x + " not in range [0, " + cols + ")"); - } - if (y < 0 || y >= rows) { - throw new IllegalArgumentException("Y index " + y + " not in range [0, " + rows + ")"); - } - - return (cols * y) + x; - } - } From b78a93fd0d35a828345d04761cd2b4b0f9fda194 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Fri, 12 Apr 2024 10:58:50 -0500 Subject: [PATCH 5/8] DICOM writer: allow different tile sizes for each resolution This provides better precompressed tile support for input formats that don't have a constant tile size across all resolutions. --- .../src/loci/formats/out/DicomWriter.java | 98 ++++++++++++++++--- 1 file changed, 86 insertions(+), 12 deletions(-) diff --git a/components/formats-bsd/src/loci/formats/out/DicomWriter.java b/components/formats-bsd/src/loci/formats/out/DicomWriter.java index b3f25f64dd4..04c0c867444 100644 --- a/components/formats-bsd/src/loci/formats/out/DicomWriter.java +++ b/components/formats-bsd/src/loci/formats/out/DicomWriter.java @@ -113,6 +113,9 @@ public class DicomWriter extends FormatWriter implements IExtraMetadataWriter { private int baseTileHeight = 256; private int[] tileWidth; private int[] tileHeight; + private long[] tileWidthPointer; + private long[] tileHeightPointer; + private long[] tileCountPointer; private PlaneOffset[][] planeOffsets; private Integer currentPlane = null; private UIDCreator uids; @@ -279,6 +282,13 @@ public void saveCompressedBytes(int no, byte[] buf, int x, int y, int w, int h) boolean first = x == 0 && y == 0; boolean last = x + w == getSizeX() && y + h == getSizeY(); + int width = getSizeX(); + int height = getSizeY(); + int sizeZ = r.getPixelsSizeZ(series).getValue().intValue(); + + int tileCountX = (int) Math.ceil((double) width / tileWidth[resolutionIndex]); + int tileCountY = (int) Math.ceil((double) height / tileHeight[resolutionIndex]); + // the compression type isn't supplied to the writer until // after setId is called, so metadata that indicates or // depends on the compression type needs to be set in @@ -296,6 +306,15 @@ public void saveCompressedBytes(int no, byte[] buf, int x, int y, int w, int h) if (getTIFFCompression() == TiffCompression.JPEG) { ifds[resolutionIndex][no].put(IFD.PHOTOMETRIC_INTERPRETATION, PhotoInterp.Y_CB_CR.getCode()); } + + out.seek(tileWidthPointer[resolutionIndex]); + out.writeShort((short) getTileSizeX()); + out.seek(tileHeightPointer[resolutionIndex]); + out.writeShort((short) getTileSizeY()); + out.seek(tileCountPointer[resolutionIndex]); + + out.writeBytes(padString(String.valueOf( + tileCountX * tileCountY * sizeZ * r.getChannelCount(series)))); } out.seek(out.length()); @@ -334,6 +353,17 @@ public void saveCompressedBytes(int no, byte[] buf, int x, int y, int w, int h) if (ifds[resolutionIndex][no] != null) { tileByteCounts = (long[]) ifds[resolutionIndex][no].getIFDValue(IFD.TILE_BYTE_COUNTS); tileOffsets = (long[]) ifds[resolutionIndex][no].getIFDValue(IFD.TILE_OFFSETS); + + if (tileByteCounts.length < tileCountX * tileCountY) { + long[] newTileByteCounts = new long[tileCountX * tileCountY]; + long[] newTileOffsets = new long[tileCountX * tileCountY]; + System.arraycopy(tileByteCounts, 0, newTileByteCounts, 0, tileByteCounts.length); + System.arraycopy(tileOffsets, 0, newTileOffsets, 0, tileOffsets.length); + tileByteCounts = newTileByteCounts; + tileOffsets = newTileOffsets; + ifds[resolutionIndex][no].put(IFD.TILE_BYTE_COUNTS, tileByteCounts); + ifds[resolutionIndex][no].put(IFD.TILE_OFFSETS, tileOffsets); + } } if (tileByteCounts != null) { @@ -640,6 +670,9 @@ public void setId(String id) throws FormatException, IOException { planeOffsets = new PlaneOffset[totalFiles][]; tileWidth = new int[totalFiles]; tileHeight = new int[totalFiles]; + tileWidthPointer = new long[totalFiles]; + tileHeightPointer = new long[totalFiles]; + tileCountPointer = new long[totalFiles]; // create UIDs that must be consistent across all files in the dataset String specimenUIDValue = uids.getUID(); @@ -739,8 +772,9 @@ public void setId(String id) throws FormatException, IOException { int tileCountX = (int) Math.ceil((double) width / tileWidth[resolutionIndex]); int tileCountY = (int) Math.ceil((double) height / tileHeight[resolutionIndex]); DicomTag numberOfFrames = new DicomTag(NUMBER_OF_FRAMES, IS); + // save space for up to 10 digits numberOfFrames.value = padString(String.valueOf( - tileCountX * tileCountY * sizeZ * r.getChannelCount(pyramid))); + tileCountX * tileCountY * sizeZ * r.getChannelCount(pyramid)), " ", 10); tags.add(numberOfFrames); DicomTag matrixFrames = new DicomTag(TOTAL_PIXEL_MATRIX_FOCAL_PLANES, UL); @@ -1374,6 +1408,9 @@ public void close() throws IOException { ifds = null; tiffSaver = null; validPixelCount = null; + tileWidthPointer = null; + tileHeightPointer = null; + tileCountPointer = null; tagProviders.clear(); @@ -1382,33 +1419,46 @@ public void close() throws IOException { @Override public int setTileSizeX(int tileSize) throws FormatException { - // TODO: this currently enforces the same tile size across all resolutions - // since the tile size is written during setId - // the tile size should probably be configurable per resolution, - // for better pre-compressed tile support if (currentId == null) { baseTileWidth = tileSize; + return baseTileWidth; } - return baseTileWidth; + + int resolutionIndex = getIndex(series, resolution); + tileWidth[resolutionIndex] = tileSize; + return tileWidth[resolutionIndex]; } @Override public int getTileSizeX() { - return baseTileWidth; + if (currentId == null) { + return baseTileWidth; + } + + int resolutionIndex = getIndex(series, resolution); + return tileWidth[resolutionIndex]; } @Override public int setTileSizeY(int tileSize) throws FormatException { - // TODO: see note in setTileSizeX above if (currentId == null) { baseTileHeight = tileSize; + return baseTileHeight; } - return baseTileHeight; + + int resolutionIndex = getIndex(series, resolution); + tileHeight[resolutionIndex] = tileSize; + return tileHeight[resolutionIndex]; } @Override public int getTileSizeY() { - return baseTileHeight; + if (currentId == null) { + return baseTileHeight; + } + + int resolutionIndex = getIndex(series, resolution); + return tileHeight[resolutionIndex]; } // -- DicomWriter-specific methods -- @@ -1468,15 +1518,25 @@ private void writeTag(DicomTag tag) throws IOException { out.writeShort((short) getStoredLength(tag)); } + int resolutionIndex = getIndex(series, resolution); if (tag.attribute == TRANSFER_SYNTAX_UID) { - transferSyntaxPointer[getIndex(series, resolution)] = out.getFilePointer(); + transferSyntaxPointer[resolutionIndex] = out.getFilePointer(); } else if (tag.attribute == LOSSY_IMAGE_COMPRESSION_METHOD) { - compressionMethodPointer[getIndex(series, resolution)] = out.getFilePointer(); + compressionMethodPointer[resolutionIndex] = out.getFilePointer(); } else if (tag.attribute == FILE_META_INFO_GROUP_LENGTH) { fileMetaLengthPointer = out.getFilePointer(); } + else if (tag.attribute == ROWS) { + tileHeightPointer[resolutionIndex] = out.getFilePointer(); + } + else if (tag.attribute == COLUMNS) { + tileWidthPointer[resolutionIndex] = out.getFilePointer(); + } + else if (tag.attribute == NUMBER_OF_FRAMES) { + tileCountPointer[resolutionIndex] = out.getFilePointer(); + } // sequences with no items still need to write a SequenceDelimitationItem below if (tag.children.size() == 0 && tag.value == null && tag.vr != SQ) { @@ -1665,6 +1725,17 @@ private String padString(String value, String append) { return value + append; } + private String padString(String value, String append, int length) { + String rtn = ""; + if (value != null) { + rtn += value; + } + while (rtn.length() < length) { + rtn += append; + } + return rtn; + } + /** * @return transfer syntax UID corresponding to the current compression type */ @@ -1918,6 +1989,9 @@ private void writeIFDs(int resIndex) throws IOException { out.seek(ifdStart); for (int no=0; no Date: Fri, 12 Apr 2024 10:59:41 -0500 Subject: [PATCH 6/8] NDPI: fix tile size and precompressed tile reading for extra images --- .../src/loci/formats/in/NDPIReader.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/components/formats-gpl/src/loci/formats/in/NDPIReader.java b/components/formats-gpl/src/loci/formats/in/NDPIReader.java index 4b3611b2412..505df562974 100644 --- a/components/formats-gpl/src/loci/formats/in/NDPIReader.java +++ b/components/formats-gpl/src/loci/formats/in/NDPIReader.java @@ -169,7 +169,14 @@ public byte[] openCompressedBytes(int no, byte[] buf, int x, int y) throws Forma IFD ifd = ifds.get(ifdIndex); if (useTiffParser(ifd)) { - return copyTile(ifd, buf, x, y); + try (RandomAccessInputStream s = new RandomAccessInputStream(currentId)) { + tiffParser = new TiffParser(s); + tiffParser.setUse64BitOffsets(true); + return copyTile(ifd, buf, x, y); + } + finally { + tiffParser.getStream().close(); + } } if (initializedSeries != getCoreIndex() || initializedPlane != no) { @@ -352,8 +359,7 @@ public int getOptimalTileWidth() { IFD ifd = ifds.get(ifdIndex); try { if (useTiffParser(ifd)) { - // TODO: - return 1024; + return (int) ifd.getTileWidth(); } if (initializedSeries != getCoreIndex() || initializedPlane != no) { @@ -379,8 +385,7 @@ public int getOptimalTileHeight() { IFD ifd = ifds.get(ifdIndex); try { if (useTiffParser(ifd)) { - // TODO: - return 1024; + return (int) ifd.getTileLength(); } if (initializedSeries != getCoreIndex() || initializedPlane != no) { From 23780deb20faba5b66f7f07b647a069ac3987164 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Thu, 20 Jun 2024 18:35:56 -0500 Subject: [PATCH 7/8] Fix tile size/count writing when calling saveBytes --- .../src/loci/formats/out/DicomWriter.java | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/components/formats-bsd/src/loci/formats/out/DicomWriter.java b/components/formats-bsd/src/loci/formats/out/DicomWriter.java index 04c0c867444..275986a606a 100644 --- a/components/formats-bsd/src/loci/formats/out/DicomWriter.java +++ b/components/formats-bsd/src/loci/formats/out/DicomWriter.java @@ -415,6 +415,10 @@ else if (x % thisTileWidth != 0 || y % thisTileHeight != 0 || boolean first = x == 0 && y == 0; boolean last = x + w == getSizeX() && y + h == getSizeY(); + int xTiles = (int) Math.ceil((double) getSizeX() / thisTileWidth); + int yTiles = (int) Math.ceil((double) getSizeY() / thisTileHeight); + int sizeZ = r.getPixelsSizeZ(series).getValue().intValue(); + // the compression type isn't supplied to the writer until // after setId is called, so metadata that indicates or // depends on the compression type needs to be set in @@ -436,6 +440,15 @@ else if (x % thisTileWidth != 0 || y % thisTileHeight != 0 || ifds[resolutionIndex][no].put(IFD.PHOTOMETRIC_INTERPRETATION, PhotoInterp.Y_CB_CR.getCode()); } } + + out.seek(tileWidthPointer[resolutionIndex]); + out.writeShort((short) getTileSizeX()); + out.seek(tileHeightPointer[resolutionIndex]); + out.writeShort((short) getTileSizeY()); + out.seek(tileCountPointer[resolutionIndex]); + + out.writeBytes(padString(String.valueOf( + xTiles * yTiles * sizeZ * r.getChannelCount(series)))); } // TILED_SPARSE, so the tile coordinates must be written @@ -528,7 +541,6 @@ else if (x % thisTileWidth != 0 || y % thisTileHeight != 0 || // in the IFD // this tries to calculate the index without assuming sequential tile // writing, but maybe there is a better way to calculate this? - int xTiles = (int) Math.ceil((double) getSizeX() / tileWidth[resolutionIndex]); int xTile = x / tileWidth[resolutionIndex]; int yTile = y / tileHeight[resolutionIndex]; int tileIndex = (yTile * xTiles) + xTile; @@ -538,6 +550,17 @@ else if (x % thisTileWidth != 0 || y % thisTileHeight != 0 || if (ifds[resolutionIndex][no] != null) { tileByteCounts = (long[]) ifds[resolutionIndex][no].getIFDValue(IFD.TILE_BYTE_COUNTS); tileOffsets = (long[]) ifds[resolutionIndex][no].getIFDValue(IFD.TILE_OFFSETS); + + if (tileByteCounts.length < xTiles * yTiles) { + long[] newTileByteCounts = new long[xTiles * yTiles]; + long[] newTileOffsets = new long[xTiles * yTiles]; + System.arraycopy(tileByteCounts, 0, newTileByteCounts, 0, tileByteCounts.length); + System.arraycopy(tileOffsets, 0, newTileOffsets, 0, tileOffsets.length); + tileByteCounts = newTileByteCounts; + tileOffsets = newTileOffsets; + ifds[resolutionIndex][no].put(IFD.TILE_BYTE_COUNTS, tileByteCounts); + ifds[resolutionIndex][no].put(IFD.TILE_OFFSETS, tileOffsets); + } } if (compression == null || compression.equals(CompressionType.UNCOMPRESSED.getCompression())) { From 4f2a96e3264c5342716fc56d8f99dfc67bb5e119 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Wed, 26 Jun 2024 17:47:13 -0500 Subject: [PATCH 8/8] bfconvert: convert DICOM data tile-wise any time `-precompressed` is used This makes sure that the tiles actually saved match the tile sizes supplied to the writer. --- .../src/loci/formats/tools/ImageConverter.java | 8 ++++++++ .../src/loci/formats/out/DicomWriter.java | 15 +-------------- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/components/bio-formats-tools/src/loci/formats/tools/ImageConverter.java b/components/bio-formats-tools/src/loci/formats/tools/ImageConverter.java index 8f1c3816929..29cb160d8fd 100644 --- a/components/bio-formats-tools/src/loci/formats/tools/ImageConverter.java +++ b/components/bio-formats-tools/src/loci/formats/tools/ImageConverter.java @@ -1228,6 +1228,14 @@ private boolean doTileConversion(IFormatWriter writer, String outputFile) if (writer instanceof DicomWriter || (writer instanceof ImageWriter && ((ImageWriter) writer).getWriter(outputFile) instanceof DicomWriter)) { + // if we asked to try a precompressed conversion, + // then the writer's tile sizes will have been set automatically + // according to the input data + // the conversion must then be performed tile-wise to match the tile sizes, + // even if precompression doesn't end up being possible + if (precompressed) { + return true; + } MetadataStore r = reader.getMetadataStore(); return !(r instanceof IPyramidStore) || ((IPyramidStore) r).getResolutionCount(reader.getSeries()) > 1; } diff --git a/components/formats-bsd/src/loci/formats/out/DicomWriter.java b/components/formats-bsd/src/loci/formats/out/DicomWriter.java index 275986a606a..96b4d6fcfee 100644 --- a/components/formats-bsd/src/loci/formats/out/DicomWriter.java +++ b/components/formats-bsd/src/loci/formats/out/DicomWriter.java @@ -235,14 +235,7 @@ public void saveCompressedBytes(int no, byte[] buf, int x, int y, int w, int h) LOGGER.debug("savePrecompressedBytes(series={}, resolution={}, no={}, x={}, y={})", series, resolution, no, x, y); - // TODO: may want better handling of non-tiled "extra" images (e.g. label, macro) MetadataRetrieve r = getMetadataRetrieve(); - if ((!(r instanceof IPyramidStore) || - ((IPyramidStore) r).getResolutionCount(series) == 1) && - !isFullPlane(x, y, w, h)) - { - throw new FormatException("DicomWriter does not allow tiles for non-pyramid images"); - } int bytesPerPixel = FormatTools.getBytesPerPixel( FormatTools.pixelTypeFromString( @@ -397,13 +390,7 @@ public void saveBytes(int no, byte[] buf, int x, int y, int w, int h) int thisTileHeight = tileHeight[resolutionIndex]; MetadataRetrieve r = getMetadataRetrieve(); - if ((!(r instanceof IPyramidStore) || - ((IPyramidStore) r).getResolutionCount(series) == 1) && - !isFullPlane(x, y, w, h)) - { - throw new FormatException("DicomWriter does not allow tiles for non-pyramid images"); - } - else if (x % thisTileWidth != 0 || y % thisTileHeight != 0 || + if (x % thisTileWidth != 0 || y % thisTileHeight != 0 || (w != thisTileWidth && x + w != getSizeX()) || (h != thisTileHeight && y + h != getSizeY())) {