From 894444bc548a2109c45319f4c7b33989ea7327e5 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Tue, 16 Apr 2024 16:24:08 -0500 Subject: [PATCH 01/12] DICOM: initial precompressed tile reading --- .../src/loci/formats/in/DicomReader.java | 200 ++++++++++++++---- 1 file changed, 164 insertions(+), 36 deletions(-) diff --git a/components/formats-bsd/src/loci/formats/in/DicomReader.java b/components/formats-bsd/src/loci/formats/in/DicomReader.java index 4b6c9afeb24..22702106b48 100644 --- a/components/formats-bsd/src/loci/formats/in/DicomReader.java +++ b/components/formats-bsd/src/loci/formats/in/DicomReader.java @@ -63,6 +63,7 @@ import loci.formats.codec.JPEG2000Codec; import loci.formats.codec.JPEGCodec; import loci.formats.codec.PackbitsCodec; +import loci.formats.codec.PassthroughCodec; import loci.formats.meta.MetadataStore; import ome.xml.model.primitives.Timestamp; import ome.units.quantity.Length; @@ -145,6 +146,91 @@ public DicomReader() { hasCompanionFiles = true; } + // -- ICompressedTileReader API methods -- + + @Override + public int getTileRows(int no) { + FormatTools.assertId(currentId, true, 1); + + return (int) Math.ceil((double) getSizeY() / originalY); + } + + @Override + public int getTileColumns(int no) { + FormatTools.assertId(currentId, true, 1); + + return (int) Math.ceil((double) getSizeX() / originalY); + } + + @Override + public byte[] openCompressedBytes(int no, int x, int y) throws FormatException, IOException { + FormatTools.assertId(currentId, true, 1); + + Region boundingBox = new Region(x * originalX, y * originalY, originalX, originalY); + + List tiles = getTileList(no, boundingBox, true); + if (tiles == null || tiles.size() == 0) { + throw new FormatException("Could not find valid tile; no=" + no + ", x=" + x + ", y=" + y); + } + DicomTile tile = tiles.get(0); + byte[] buf = new byte[(int) (tile.endOffset - tile.fileOffset)]; + try (RandomAccessInputStream stream = new RandomAccessInputStream(tile.file)) { + if (tile.fileOffset >= stream.length()) { + LOGGER.error("attempted to read beyond end of file ({}, {})", tile.fileOffset, tile.file); + return buf; + } + LOGGER.debug("reading from offset = {}, file = {}", tile.fileOffset, tile.file); + stream.seek(tile.fileOffset); + stream.read(buf, 0, (int) (tile.endOffset - tile.fileOffset)); + } + return buf; + } + + @Override + public byte[] openCompressedBytes(int no, byte[] buf, int x, int y) throws FormatException, IOException { + FormatTools.assertId(currentId, true, 1); + + Region boundingBox = new Region(x * originalX, y * originalY, originalX, originalY); + + List tiles = getTileList(no, boundingBox, true); + if (tiles == null || tiles.size() == 0) { + throw new FormatException("Could not find valid tile; no=" + no + ", x=" + x + ", y=" + y); + } + DicomTile tile = tiles.get(0); + try (RandomAccessInputStream stream = new RandomAccessInputStream(tile.file)) { + if (tile.fileOffset >= stream.length()) { + LOGGER.error("attempted to read beyond end of file ({}, {})", tile.fileOffset, tile.file); + return buf; + } + LOGGER.debug("reading from offset = {}, file = {}", tile.fileOffset, tile.file); + stream.seek(tile.fileOffset); + stream.read(buf, 0, (int) (tile.endOffset - tile.fileOffset)); + } + return buf; + } + + @Override + public Codec getTileCodec(int no) throws FormatException, IOException { + FormatTools.assertId(currentId, true, 1); + + List tiles = getTileList(no, null, true); + if (tiles == null || tiles.size() == 0) { + throw new FormatException("Could not find valid tile; no=" + no); + } + return getTileCodec(tiles.get(0)); + } + + @Override + public CodecOptions getTileCodecOptions(int no, int x, int y) throws FormatException, IOException { + FormatTools.assertId(currentId, true, 1); + + List tiles = getTileList(no, null, true); + if (tiles == null || tiles.size() == 0) { + throw new FormatException("Could not find valid tile; no=" + no + ", x=" + x + ", y=" + y); + } + return getTileCodecOptions(tiles.get(0)); + } + // -- IFormatReader API methods -- /* @see loci.formats.IFormatReader#isThisType(String, boolean) */ @@ -282,33 +368,18 @@ public byte[] openBytes(int no, byte[] buf, int x, int y, int w, int h) int bpp = FormatTools.getBytesPerPixel(getPixelType()); int pixel = bpp * getRGBChannelCount(); Region currentRegion = new Region(x, y, w, h); - int z = getZCTCoords(no)[0]; - int c = getZCTCoords(no)[1]; - - if (!tilePositions.containsKey(getCoreIndex())) { - LOGGER.warn("No tiles for core index = {}", getCoreIndex()); - return buf; - } - // look for any tiles that match the requested tile and plane - List zs = zOffsets.get(getCoreIndex()); - List tiles = tilePositions.get(getCoreIndex()); - for (int t=0; t tiles = getTileList(no, currentRegion, false); + for (DicomTile tile : tiles) { + byte[] tileBuf = new byte[tile.region.width * tile.region.height * pixel]; + Region intersection = tile.region.intersection(currentRegion); + getTile(tile, tileBuf, intersection.x - tile.region.x, intersection.y - tile.region.y, + intersection.width, intersection.height); + + for (int row=0; row getTileList(int no, Region boundingBox, boolean firstTileOnly) { + int z = getZCTCoords(no)[0]; + int c = getZCTCoords(no)[1]; + + List tileList = new ArrayList(); + if (!tilePositions.containsKey(getCoreIndex())) { + LOGGER.warn("No tiles for core index = {}", getCoreIndex()); + return tileList; + } + + // look for any tiles that match the requested tile and plane + List zs = zOffsets.get(getCoreIndex()); + List tiles = tilePositions.get(getCoreIndex()); + for (int t=0; t 1) { @@ -1611,13 +1746,6 @@ else if (pt < b.length - 2) { System.arraycopy(tmp, 0, b, 0, b.length); } - Codec codec = null; - CodecOptions options = new CodecOptions(); - options.littleEndian = isLittleEndian(); - options.interleaved = isInterleaved(); - if (tile.isJPEG) codec = new JPEGCodec(); - else codec = new JPEG2000Codec(); - try { b = codec.decompress(b, options); } From 52ab11d08ba8201d5dc2eaae8f460ca012cc8a69 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Thu, 18 Apr 2024 14:58:23 -0500 Subject: [PATCH 02/12] Add basic precompressed TIFF tile writing Only "plain" TIFF, OME-TIFF not supported yet. --- .../src/loci/formats/out/TiffWriter.java | 84 +++++++++++++++++-- .../src/loci/formats/tiff/TiffSaver.java | 4 +- 2 files changed, 80 insertions(+), 8 deletions(-) diff --git a/components/formats-bsd/src/loci/formats/out/TiffWriter.java b/components/formats-bsd/src/loci/formats/out/TiffWriter.java index 9f0bc132cdc..668c46e07b2 100644 --- a/components/formats-bsd/src/loci/formats/out/TiffWriter.java +++ b/components/formats-bsd/src/loci/formats/out/TiffWriter.java @@ -39,6 +39,7 @@ import loci.formats.FormatTools; import loci.formats.FormatWriter; import loci.formats.ImageTools; +import loci.formats.codec.Codec; import loci.formats.codec.CompressionType; import loci.formats.gui.AWTImageTools; import loci.formats.meta.MetadataRetrieve; @@ -100,13 +101,10 @@ public class TiffWriter extends FormatWriter { protected int tileSizeY; /** - * Sets the compression code for the specified IFD. - * - * @param ifd The IFD table to handle. + * Get the TIFF compression enum value that corresponds to + * the current compression type. */ - private void formatCompression(IFD ifd) - throws FormatException - { + private TiffCompression getTIFFCompression() { if (compression == null) compression = ""; TiffCompression compressType = TiffCompression.UNCOMPRESSED; if (compression.equals(COMPRESSION_LZW)) { @@ -124,6 +122,18 @@ else if (compression.equals(COMPRESSION_JPEG)) { else if (compression.equals(COMPRESSION_ZLIB)) { compressType = TiffCompression.DEFLATE; } + return compressType; + } + + /** + * Sets the compression code for the specified IFD. + * + * @param ifd The IFD table to handle. + */ + private void formatCompression(IFD ifd) + throws FormatException + { + TiffCompression compressType = getTIFFCompression(); Object v = ifd.get(new Integer(IFD.COMPRESSION)); if (v == null) ifd.put(new Integer(IFD.COMPRESSION), compressType.getCode()); @@ -149,6 +159,68 @@ public TiffWriter(String format, String[] exts) { isBigTiff = false; } + // -- ICompressedTileWriter API methods -- + + @Override + public Codec getCodec() { + return getTIFFCompression().getCodec(); + } + + @Override + public void saveCompressedBytes(int no, byte[] buf, int x, int y, int w, int h) + throws FormatException, IOException + { + if (!sequential) { + throw new UnsupportedOperationException( + "Sequential tile writing must be enabled to write precompressed tiles"); + } + + LOGGER.warn("saveCompressedBytes(series={}, resolution={}, no={}, x={}, y={})", + series, resolution, no, x, y); + + IFD ifd = new IFD(); + MetadataRetrieve retrieve = getMetadataRetrieve(); + int type = FormatTools.pixelTypeFromString( + retrieve.getPixelsType(series).toString()); + int index = no; + int currentTileSizeX = getTileSizeX(); + int currentTileSizeY = getTileSizeY(); + + if (x % currentTileSizeX != 0 || y % currentTileSizeY != 0 || + (currentTileSizeX != w && x + w != getSizeX()) || + (currentTileSizeY != h && y + h != getSizeY())) + { + throw new IllegalArgumentException("Compressed tile dimensions must match tile size"); + } + + boolean usingTiling = currentTileSizeX > 0 && currentTileSizeY > 0; + if (usingTiling) { + ifd.put(new Integer(IFD.TILE_WIDTH), new Long(currentTileSizeX)); + ifd.put(new Integer(IFD.TILE_LENGTH), new Long(currentTileSizeY)); + } + + // This operation is synchronized + synchronized (this) { + // This operation is synchronized against the TIFF saver. + synchronized (tiffSaver) { + index = prepareToWriteImage(no, buf, ifd, x, y, w, h); + if (index == -1) { + return; + } + } + } + + boolean lastPlane = no == getPlaneCount() - 1; + boolean lastSeries = getSeries() == retrieve.getImageCount() - 1; + boolean lastResolution = getResolution() == getResolutionCount() - 1; + + int nChannels = getSamplesPerPixel(); + + tiffSaver.makeValidIFD(ifd, type, nChannels); + tiffSaver.writeImageIFD(ifd, index, new byte[][] {buf}, + nChannels, lastPlane && lastSeries && lastResolution, x, y); + } + // -- FormatWriter API methods -- /* @see loci.formats.FormatWriter#setId(String) */ diff --git a/components/formats-bsd/src/loci/formats/tiff/TiffSaver.java b/components/formats-bsd/src/loci/formats/tiff/TiffSaver.java index 8a683cc1d44..f803895bca2 100644 --- a/components/formats-bsd/src/loci/formats/tiff/TiffSaver.java +++ b/components/formats-bsd/src/loci/formats/tiff/TiffSaver.java @@ -450,7 +450,7 @@ public void writeImage(byte[] buf, IFD ifd, int no, int pixelType, int x, * @throws FormatException * @throws IOException */ - private void writeImageIFD(IFD ifd, int no, byte[][] strips, + public void writeImageIFD(IFD ifd, int no, byte[][] strips, int nChannels, boolean last, int x, int y) throws FormatException, IOException { LOGGER.debug("Attempting to write image IFD."); @@ -1003,7 +1003,7 @@ private void writeIntValue(RandomAccessOutputStream out, long offset) * @param pixelType The pixel type. * @param nChannels The number of channels. */ - private void makeValidIFD(IFD ifd, int pixelType, int nChannels) { + public void makeValidIFD(IFD ifd, int pixelType, int nChannels) { int bytesPerPixel = FormatTools.getBytesPerPixel(pixelType); int bps = 8 * bytesPerPixel; int[] bpsArray = new int[nChannels]; From d1c0e13036c4e87f3d04bb9299552da65841da2e Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Wed, 1 May 2024 18:05:10 -0500 Subject: [PATCH 03/12] bfconvert: always do tiled conversion when a pyramid is present Instead of requiring DICOM output and a pyramid. This makes sure that the smaller resolutions of a pyramid are written tile-wise when writing OME-TIFF. --- .../src/loci/formats/tools/ImageConverter.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/components/bio-formats-tools/src/loci/formats/tools/ImageConverter.java b/components/bio-formats-tools/src/loci/formats/tools/ImageConverter.java index ad50dcf9b85..b249f3e88cf 100644 --- a/components/bio-formats-tools/src/loci/formats/tools/ImageConverter.java +++ b/components/bio-formats-tools/src/loci/formats/tools/ImageConverter.java @@ -1260,11 +1260,9 @@ private boolean isTiledWriter(IFormatWriter writer, String outputFile) private boolean doTileConversion(IFormatWriter writer, String outputFile) throws FormatException { - if (writer instanceof DicomWriter || - (writer instanceof ImageWriter && ((ImageWriter) writer).getWriter(outputFile) instanceof DicomWriter)) - { - MetadataStore r = reader.getMetadataStore(); - return !(r instanceof IPyramidStore) || ((IPyramidStore) r).getResolutionCount(reader.getSeries()) > 1; + MetadataStore r = reader.getMetadataStore(); + if ((r instanceof IPyramidStore) && ((IPyramidStore) r).getResolutionCount(reader.getSeries()) > 1) { + return true; } return DataTools.safeMultiply64(width, height) >= DataTools.safeMultiply64(4096, 4096) || saveTileWidth > 0 || saveTileHeight > 0; From 3146c90ae6ef143ae2798331022fbde8bf0d33f0 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Wed, 1 May 2024 18:07:02 -0500 Subject: [PATCH 04/12] DICOM reader: fix optimal tile size calculation Different resolutions may have different tile sizes. --- .../src/loci/formats/in/DicomReader.java | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/components/formats-bsd/src/loci/formats/in/DicomReader.java b/components/formats-bsd/src/loci/formats/in/DicomReader.java index 22702106b48..f4c0a4cf841 100644 --- a/components/formats-bsd/src/loci/formats/in/DicomReader.java +++ b/components/formats-bsd/src/loci/formats/in/DicomReader.java @@ -166,11 +166,14 @@ public int getTileColumns(int no) { public byte[] openCompressedBytes(int no, int x, int y) throws FormatException, IOException { FormatTools.assertId(currentId, true, 1); - Region boundingBox = new Region(x * originalX, y * originalY, originalX, originalY); + // TODO: this will result in a lot of redundant lookups, and should be optimized + int tileWidth = getOptimalTileWidth(); + int tileHeight = getOptimalTileHeight(); + Region boundingBox = new Region(x * tileWidth, y * tileHeight, tileWidth, tileHeight); List tiles = getTileList(no, boundingBox, true); if (tiles == null || tiles.size() == 0) { - throw new FormatException("Could not find valid tile; no=" + no + ", x=" + x + ", y=" + y); + throw new FormatException("Could not find valid tile; no=" + no + ", boundingBox=" + boundingBox); } DicomTile tile = tiles.get(0); byte[] buf = new byte[(int) (tile.endOffset - tile.fileOffset)]; @@ -336,7 +339,10 @@ public int fileGroupOption(String id) throws FormatException, IOException { public int getOptimalTileWidth() { FormatTools.assertId(currentId, true, 1); if (tilePositions.containsKey(getCoreIndex())) { - return tilePositions.get(getCoreIndex()).get(0).region.width; + List tile = getTileList(0, null, true); + if (tile != null && tile.size() >= 1) { + return tile.get(0).region.width; + } } if (originalX < getSizeX() && originalX > 0) { return originalX; @@ -348,7 +354,10 @@ public int getOptimalTileWidth() { public int getOptimalTileHeight() { FormatTools.assertId(currentId, true, 1); if (tilePositions.containsKey(getCoreIndex())) { - return tilePositions.get(getCoreIndex()).get(0).region.height; + List tile = getTileList(0, null, true); + if (tile != null && tile.size() >= 1) { + return tile.get(0).region.height; + } } if (originalY < getSizeY() && originalY > 0) { return originalY; From 92ec5815b9393b8e1d273c2aed6409f187575230 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Wed, 1 May 2024 18:07:50 -0500 Subject: [PATCH 05/12] Extend precompressed tile writing to OME-TIFF Mostly this reuses logic in the basic TIFF writer. --- .../src/loci/formats/out/OMETiffWriter.java | 18 ++++++++++++++++++ .../loci/formats/out/PyramidOMETiffWriter.java | 13 +++++++++++++ .../src/loci/formats/out/TiffWriter.java | 18 +++++++++++------- 3 files changed, 42 insertions(+), 7 deletions(-) diff --git a/components/formats-bsd/src/loci/formats/out/OMETiffWriter.java b/components/formats-bsd/src/loci/formats/out/OMETiffWriter.java index eb4afa8fe55..e0785aeb443 100644 --- a/components/formats-bsd/src/loci/formats/out/OMETiffWriter.java +++ b/components/formats-bsd/src/loci/formats/out/OMETiffWriter.java @@ -199,6 +199,24 @@ public void close() throws IOException { // -- IFormatWriter API methods -- + @Override + public void saveCompressedBytes(int no, byte[] buf, int x, int y, int w, int h) + throws FormatException, IOException + { + super.saveCompressedBytes(no, buf, x, y, w, h); + + int index = no; + while (imageLocations[series][index] != null) { + if (index < imageLocations[series].length - 1) { + index++; + } + else { + break; + } + } + imageLocations[series][index] = currentId; + } + /** * @see loci.formats.IFormatWriter#saveBytes(int, byte[], int, int, int, int) */ diff --git a/components/formats-bsd/src/loci/formats/out/PyramidOMETiffWriter.java b/components/formats-bsd/src/loci/formats/out/PyramidOMETiffWriter.java index a1da832176f..d287493b8ac 100644 --- a/components/formats-bsd/src/loci/formats/out/PyramidOMETiffWriter.java +++ b/components/formats-bsd/src/loci/formats/out/PyramidOMETiffWriter.java @@ -70,6 +70,19 @@ public boolean isThisType(String name) { // -- IFormatWriter API methods -- + protected IFD makeIFD() throws FormatException, IOException { + IFD ifd = super.makeIFD(); + if (getResolution() > 0) { + ifd.put(IFD.NEW_SUBFILE_TYPE, 1); + } + else { + if (!ifd.containsKey(IFD.SUB_IFD)) { + ifd.put(IFD.SUB_IFD, (long) 0); + } + } + return ifd; + } + @Override public void saveBytes(int no, byte[] buf, IFD ifd, int x, int y, int w, int h) throws FormatException, IOException diff --git a/components/formats-bsd/src/loci/formats/out/TiffWriter.java b/components/formats-bsd/src/loci/formats/out/TiffWriter.java index 668c46e07b2..a58842114b9 100644 --- a/components/formats-bsd/src/loci/formats/out/TiffWriter.java +++ b/components/formats-bsd/src/loci/formats/out/TiffWriter.java @@ -178,7 +178,7 @@ public void saveCompressedBytes(int no, byte[] buf, int x, int y, int w, int h) LOGGER.warn("saveCompressedBytes(series={}, resolution={}, no={}, x={}, y={})", series, resolution, no, x, y); - IFD ifd = new IFD(); + IFD ifd = makeIFD(); MetadataRetrieve retrieve = getMetadataRetrieve(); int type = FormatTools.pixelTypeFromString( retrieve.getPixelsType(series).toString()); @@ -193,12 +193,6 @@ public void saveCompressedBytes(int no, byte[] buf, int x, int y, int w, int h) throw new IllegalArgumentException("Compressed tile dimensions must match tile size"); } - boolean usingTiling = currentTileSizeX > 0 && currentTileSizeY > 0; - if (usingTiling) { - ifd.put(new Integer(IFD.TILE_WIDTH), new Long(currentTileSizeX)); - ifd.put(new Integer(IFD.TILE_LENGTH), new Long(currentTileSizeY)); - } - // This operation is synchronized synchronized (this) { // This operation is synchronized against the TIFF saver. @@ -221,6 +215,16 @@ public void saveCompressedBytes(int no, byte[] buf, int x, int y, int w, int h) nChannels, lastPlane && lastSeries && lastResolution, x, y); } + protected IFD makeIFD() throws FormatException, IOException { + IFD ifd = new IFD(); + boolean usingTiling = getTileSizeX() > 0 && getTileSizeY() > 0; + if (usingTiling) { + ifd.put(new Integer(IFD.TILE_WIDTH), new Long(getTileSizeX())); + ifd.put(new Integer(IFD.TILE_LENGTH), new Long(getTileSizeY())); + } + return ifd; + } + // -- FormatWriter API methods -- /* @see loci.formats.FormatWriter#setId(String) */ From 6136fb3806b683f1ae0d359ba945cc995ff53370 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Sun, 16 Jun 2024 13:00:14 -0500 Subject: [PATCH 06/12] Fix typos --- .../formats-api/src/loci/formats/ICompressedTileReader.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/formats-api/src/loci/formats/ICompressedTileReader.java b/components/formats-api/src/loci/formats/ICompressedTileReader.java index b90fc44a4d0..ad7baee3b7c 100644 --- a/components/formats-api/src/loci/formats/ICompressedTileReader.java +++ b/components/formats-api/src/loci/formats/ICompressedTileReader.java @@ -66,7 +66,7 @@ default int getTileColumns(int no) { * * @param no plane index * @param x tile X index (indexed from 0, @see getTileColumns(int)) - * @param y tile Y index (indexed frmo 0, @see getTileRows(int)) + * @param y tile Y index (indexed from 0, @see getTileRows(int)) * @return compressed tile bytes */ default byte[] openCompressedBytes(int no, int x, int y) throws FormatException, IOException { @@ -79,7 +79,7 @@ default byte[] openCompressedBytes(int no, int x, int y) throws FormatException, * @param no plane index * @param buf pre-allocated buffer in which to store compressed bytes * @param x tile X index (indexed from 0, @see getTileColumns(int)) - * @param y tile Y index (indexed frmo 0, @see getTileRows(int)) + * @param y tile Y index (indexed from 0, @see getTileRows(int)) * @return compressed tile bytes */ default byte[] openCompressedBytes(int no, byte[] buf, int x, int y) throws FormatException, IOException { @@ -102,7 +102,7 @@ default Codec getTileCodec(int no) throws FormatException, IOException { * * @param no plane index * @param x tile X index (indexed from 0, @see getTileColumns(int)) - * @param y tile Y index (indexed frmo 0, @see getTileRows(int)) + * @param y tile Y index (indexed from 0, @see getTileRows(int)) * @return codec options * @see getTileCodec(int) */ From 93e38d8ec6b073168852ebac09edf502b28ee19b Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Sun, 16 Jun 2024 13:31:35 -0500 Subject: [PATCH 07/12] If `-precompressed` specified without `-compression`, use reader compression type automatically This doesn't yet completely support mixed compression types within the same dataset, but something like: $ bfconvert -noflat -precompressed CMU-1.svs CMU-1.dcm should run without error and convert the pyramid without recompressing. --- .../loci/formats/tools/ImageConverter.java | 20 ++++++++++++++++ .../loci/formats/codec/CompressionType.java | 24 ++++++++++++++++++- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/components/bio-formats-tools/src/loci/formats/tools/ImageConverter.java b/components/bio-formats-tools/src/loci/formats/tools/ImageConverter.java index b249f3e88cf..2b88165fbb6 100644 --- a/components/bio-formats-tools/src/loci/formats/tools/ImageConverter.java +++ b/components/bio-formats-tools/src/loci/formats/tools/ImageConverter.java @@ -62,6 +62,7 @@ import loci.formats.FileStitcher; import loci.formats.FormatException; import loci.formats.FormatTools; +import loci.formats.ICompressedTileReader; import loci.formats.IFormatReader; import loci.formats.IFormatWriter; import loci.formats.ImageReader; @@ -71,7 +72,9 @@ import loci.formats.MetadataTools; import loci.formats.MinMaxCalculator; import loci.formats.MissingLibraryException; +import loci.formats.codec.Codec; import loci.formats.codec.CodecOptions; +import loci.formats.codec.CompressionType; import loci.formats.codec.JPEG2000CodecOptions; import loci.formats.gui.Index16ColorModel; import loci.formats.in.DynamicMetadataOptions; @@ -558,6 +561,11 @@ public boolean testConvert(IFormatWriter writer, String[] args) reader.setId(in); + if (compression == null && precompressed) { + compression = getReaderCodecName(); + LOGGER.info("Implicitly using compression = {}", compression); + } + if (swapOrder != null) { dimSwapper.swapDimensions(swapOrder); } @@ -1308,6 +1316,18 @@ private void setCodecOptions(IFormatWriter writer) { } } + private String getReaderCodecName() throws FormatException, IOException { + if (reader instanceof ICompressedTileReader) { + ICompressedTileReader r = (ICompressedTileReader) reader; + Codec c = r.getTileCodec(0); + CompressionType type = CompressionType.get(c); + if (type != null) { + return type.getCompression(); + } + } + return null; + } + // -- Main method -- public static void main(String[] args) throws FormatException, IOException { diff --git a/components/formats-bsd/src/loci/formats/codec/CompressionType.java b/components/formats-bsd/src/loci/formats/codec/CompressionType.java index 01b595a7727..de9e8fa0111 100644 --- a/components/formats-bsd/src/loci/formats/codec/CompressionType.java +++ b/components/formats-bsd/src/loci/formats/codec/CompressionType.java @@ -115,5 +115,27 @@ public int getCode() { public String getCompression() { return compression; } - + + /** + * Look up the compression type by Codec instance. + */ + public static CompressionType get(Codec c) { + if (c instanceof ZlibCodec) { + return ZLIB; + } + if (c instanceof LZWCodec) { + return LZW; + } + if (c instanceof JPEGCodec) { + return JPEG; + } + if (c instanceof JPEG2000Codec) { + return J2K; + } + if (c instanceof PassthroughCodec) { + return UNCOMPRESSED; + } + return null; + } + } From d341f4c9ea4fe5fff1c3fd6013f7e21d74cca8c1 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Thu, 8 Aug 2024 13:28:04 -0500 Subject: [PATCH 08/12] Fix strip padding when working with planar strips --- .../src/loci/formats/tiff/TiffSaver.java | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/components/formats-bsd/src/loci/formats/tiff/TiffSaver.java b/components/formats-bsd/src/loci/formats/tiff/TiffSaver.java index f803895bca2..06c01da240b 100644 --- a/components/formats-bsd/src/loci/formats/tiff/TiffSaver.java +++ b/components/formats-bsd/src/loci/formats/tiff/TiffSaver.java @@ -350,15 +350,22 @@ public void writeImage(byte[] buf, IFD ifd, int no, int pixelType, int x, stripOut[strip].write(buf, strip * stripSize, stripSize); } } else { - for (int strip = 0; strip < nStrips - 1; strip++) { - stripOut[strip].write(buf, strip * stripSize, stripSize); - } - // Sigh. Need to pad the last strip. - int pos = (nStrips - 1) * stripSize; - int len = buf.length - pos; - stripOut[nStrips - 1].write(buf, pos, len); - for (int n = len; n < stripSize; n++) { - stripOut[nStrips - 1].writeByte(0); + int effectiveStrips = !interleaved ? nStrips / nChannels : nStrips; + int planarChannels = !interleaved ? nChannels : 1; + int totalBytesPerChannel = buf.length / planarChannels; + for (int p=0; p Date: Thu, 8 Aug 2024 13:58:44 -0500 Subject: [PATCH 09/12] One more fix to tile-wise conversion check Always perform tile-wise conversion if precompressed conversion was requested, independent of the format and presence of image pyramid. --- .../src/loci/formats/tools/ImageConverter.java | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/components/bio-formats-tools/src/loci/formats/tools/ImageConverter.java b/components/bio-formats-tools/src/loci/formats/tools/ImageConverter.java index 746b6080a70..934691e072a 100644 --- a/components/bio-formats-tools/src/loci/formats/tools/ImageConverter.java +++ b/components/bio-formats-tools/src/loci/formats/tools/ImageConverter.java @@ -1268,16 +1268,13 @@ private boolean isTiledWriter(IFormatWriter writer, String outputFile) private boolean doTileConversion(IFormatWriter writer, String outputFile) throws FormatException { - MetadataStore r = reader.getMetadataStore(); - if ((r instanceof IPyramidStore) && ((IPyramidStore) r).getResolutionCount(reader.getSeries()) > 1) { - // if we asked to try a precompressed conversion, - // then the writer's tile sizes will have been set automatically - // according to the input data - // the conversion must then be performed tile-wise to match the tile sizes, - // even if precompression doesn't end up being possible - if (precompressed) { - return true; - } + // if we asked to try a precompressed conversion, + // then the writer's tile sizes will have been set automatically + // according to the input data + // the conversion must then be performed tile-wise to match the tile sizes, + // even if precompression doesn't end up being possible + if (precompressed) { + return true; } return DataTools.safeMultiply64(width, height) >= DataTools.safeMultiply64(4096, 4096) || saveTileWidth > 0 || saveTileHeight > 0; From 117078ead882510f88996eb4376249dd0daeaf8a Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Fri, 9 Aug 2024 15:55:44 -0500 Subject: [PATCH 10/12] Override writer's interleaved setting if precompressed JPEG tiles are supplied --- .../src/loci/formats/tools/ImageConverter.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/components/bio-formats-tools/src/loci/formats/tools/ImageConverter.java b/components/bio-formats-tools/src/loci/formats/tools/ImageConverter.java index 934691e072a..0ff352a2b7b 100644 --- a/components/bio-formats-tools/src/loci/formats/tools/ImageConverter.java +++ b/components/bio-formats-tools/src/loci/formats/tools/ImageConverter.java @@ -767,7 +767,9 @@ else if (writer instanceof ImageWriter) { int writerSeries = series == -1 ? q : 0; writer.setSeries(writerSeries); writer.setResolution(res); + writer.setInterleaved(reader.isInterleaved() && !autoscale); + writer.setValidBitsPerPixel(reader.getBitsPerPixel()); int numImages = writer.canDoStacks() ? reader.getImageCount() : 1; @@ -840,6 +842,12 @@ else if (saveTileWidth > 0 && saveTileHeight > 0) { } } + if (precompressed && FormatTools.canUsePrecompressedTiles(reader, writer, writer.getSeries(), writer.getResolution())) { + if (getReaderCodecName().startsWith("JPEG")) { + writer.setInterleaved(true); + } + } + int outputIndex = 0; if (nextOutputIndex.containsKey(outputName)) { outputIndex = nextOutputIndex.get(outputName); From f6a0c75fd88e54efa3ba98c601194ff589ee34ab Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Fri, 9 Aug 2024 15:56:14 -0500 Subject: [PATCH 11/12] Turn down log level for saveCompressedBytes logging --- components/formats-bsd/src/loci/formats/out/TiffWriter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/formats-bsd/src/loci/formats/out/TiffWriter.java b/components/formats-bsd/src/loci/formats/out/TiffWriter.java index bd2bf785c0f..8bc78476238 100644 --- a/components/formats-bsd/src/loci/formats/out/TiffWriter.java +++ b/components/formats-bsd/src/loci/formats/out/TiffWriter.java @@ -176,7 +176,7 @@ public void saveCompressedBytes(int no, byte[] buf, int x, int y, int w, int h) "Sequential tile writing must be enabled to write precompressed tiles"); } - LOGGER.warn("saveCompressedBytes(series={}, resolution={}, no={}, x={}, y={})", + LOGGER.debug("saveCompressedBytes(series={}, resolution={}, no={}, x={}, y={})", series, resolution, no, x, y); IFD ifd = makeIFD(); From 29e8c05f22b9b647efa6899d95969edf204c9762 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Tue, 13 Aug 2024 10:49:11 -0500 Subject: [PATCH 12/12] One more check for performing tile-wise conversion --- .../src/loci/formats/tools/ImageConverter.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/components/bio-formats-tools/src/loci/formats/tools/ImageConverter.java b/components/bio-formats-tools/src/loci/formats/tools/ImageConverter.java index 0ff352a2b7b..77d0c2b316c 100644 --- a/components/bio-formats-tools/src/loci/formats/tools/ImageConverter.java +++ b/components/bio-formats-tools/src/loci/formats/tools/ImageConverter.java @@ -1284,6 +1284,14 @@ private boolean doTileConversion(IFormatWriter writer, String outputFile) if (precompressed) { return true; } + // tile size has already been set in the writer, + // so tile-wise conversion should be performed + // independent of image size + if ((writer.getTileSizeX() > 0 && writer.getTileSizeX() < width) || + (writer.getTileSizeY() > 0 && writer.getTileSizeY() < height)) + { + return true; + } return DataTools.safeMultiply64(width, height) >= DataTools.safeMultiply64(4096, 4096) || saveTileWidth > 0 || saveTileHeight > 0; }