Merge pull request #3992 from melissalinkert/precompressed-tiles

Allow reading and writing compressed tiles

components/bio-formats-tools/src/loci/formats/tools/ImageConverter.java

@@ -129,6 +129,8 @@ public final class ImageConverter {
   private boolean noSequential = false;
   private String swapOrder = null;
   private Byte fillColor = null;
+  private boolean precompressed = false;
+  private boolean tryPrecompressed = false;
 
   private IFormatReader reader;
   private MinMaxCalculator minMax;
@@ -179,6 +181,7 @@ private boolean parseArgs(String[] args) {
         else if (args[i].equals("-padded")) zeroPadding = true;
         else if (args[i].equals("-noflat")) flat = false;
         else if (args[i].equals("-no-sas")) originalMetadata = false;
+        else if (args[i].equals("-precompressed")) precompressed = true;
         else if (args[i].equals("-cache")) useMemoizer = true;
         else if (args[i].equals("-cache-dir")) {
           cacheDir = args[++i];
@@ -340,6 +343,7 @@ private void printUsage() {
       "    [-option key value] [-novalid] [-validate] [-tilex tileSizeX]", 
       "    [-tiley tileSizeY] [-pyramid-scale scale]", 
       "    [-swap dimensionsOrderString] [-fill color]",
+      "    [-precompressed]",
       "    [-pyramid-resolutions numResolutionLevels] in_file out_file",
       "",
       "            -version: print the library version and exit",
@@ -384,6 +388,10 @@ private void printUsage() {
       "      -no-sequential: do not assume that planes are written in sequential order",
       "               -swap: override the default input dimension order; argument is f.i. XYCTZ",
       "               -fill: byte value to use for undefined pixels (0-255)",
+      "      -precompressed: transfer compressed bytes from input dataset directly to output.",
+      "                      Most input and output formats do not support this option.",
+      "                      Do not use -crop, -fill, or -autoscale, or pyramid generation options",
+      "                      with this option.",
       "",
       "The extension of the output file specifies the file format to use",
       "for the conversion. The list of available formats and extensions is:",
@@ -455,6 +463,18 @@ public boolean testConvert(IFormatWriter writer, String[] args)
       return false;
     }
 
+    // TODO: there may be other options not compatible with -precompressed
+    if (precompressed &&
+      (width_crop > 0 || height_crop > 0 ||
+      pyramidResolutions > 1 ||
+      fillColor != null ||
+      autoscale
+      ))
+    {
+      throw new UnsupportedOperationException("-precompressed not supported with " +
+        "-autoscale, -crop, -fill, -pyramid-scale, -pyramid-resolutions");
+    }
+
     CommandLineTools.runUpgradeCheck(args);
 
     if (new Location(out).exists()) {
@@ -712,8 +732,14 @@ else if (w instanceof DicomWriter) {
 
         total += numImages;
 
-        writer.setTileSizeX(saveTileWidth);
-        writer.setTileSizeY(saveTileHeight);
+        if (precompressed) {
+          writer.setTileSizeX(reader.getOptimalTileWidth());
+          writer.setTileSizeY(reader.getOptimalTileHeight());
+        }
+        else if (saveTileWidth > 0 && saveTileHeight > 0) {
+          writer.setTileSizeX(saveTileWidth);
+          writer.setTileSizeY(saveTileHeight);
+        }
 
         int count = 0;
         for (int i=startPlane; i<endPlane; i++) {
@@ -834,13 +860,28 @@ private long convertPlane(IFormatWriter writer, int index, int outputIndex,
       }
     }
 
+    tryPrecompressed = precompressed && FormatTools.canUsePrecompressedTiles(reader, writer, writer.getSeries(), writer.getResolution());
+
     byte[] buf = getTile(reader, writer.getResolution(), index,
       xCoordinate, yCoordinate, width, height);
 
+    // if we asked for precompressed tiles, but that wasn't possible,
+    // then log that decompression/recompression happened
+    // TODO: decide if an exception is better here?
+    if (precompressed && !tryPrecompressed) {
+      LOGGER.warn("Decompressed tile: series={}, resolution={}, x={}, y={}",
+        writer.getSeries(), writer.getResolution(), xCoordinate, yCoordinate);
+    }
+
     autoscalePlane(buf, index);
     applyLUT(writer);
     long m = System.currentTimeMillis();
-    writer.saveBytes(outputIndex, buf);
+    if (tryPrecompressed) {
+      writer.saveCompressedBytes(outputIndex, buf, 0, 0, reader.getSizeX(), reader.getSizeY());
+    }
+    else {
+      writer.saveBytes(outputIndex, buf);
+    }
     return m;
   }
 
@@ -886,16 +927,37 @@ private long convertTilePlane(IFormatWriter writer, int index, int outputIndex,
       nYTiles++;
     }
 
+    // only warn once if the whole resolution will need to
+    // be decompressed and recompressed
+    boolean canPrecompressResolution = precompressed && FormatTools.canUsePrecompressedTiles(reader, writer, writer.getSeries(), writer.getResolution());
+    if (precompressed && !canPrecompressResolution) {
+      LOGGER.warn("Decompressing resolution: series={}, resolution={}",
+        writer.getSeries(), writer.getResolution());
+      tryPrecompressed = false;
+    }
+
     Long m = null;
     for (int y=0; y<nYTiles; y++) {
       for (int x=0; x<nXTiles; x++) {
         int tileX = xCoordinate + x * w;
         int tileY = yCoordinate + y * h;
         int tileWidth = x < nXTiles - 1 ? w : width - (w * x);
         int tileHeight = y < nYTiles - 1 ? h : height - (h * y);
+
+        tryPrecompressed = precompressed && canPrecompressResolution &&
+          FormatTools.canUsePrecompressedTiles(reader, writer, writer.getSeries(), writer.getResolution());
         byte[] buf = getTile(reader, writer.getResolution(),
           index, tileX, tileY, tileWidth, tileHeight);
 
+        // if we asked for precompressed tiles, but that wasn't possible,
+        // then log that decompression/recompression happened
+        // this is mainly expected for edge tiles, which might be smaller than expected
+        // TODO: decide if an exception is better here?
+        if (precompressed && canPrecompressResolution && !tryPrecompressed) {
+          LOGGER.warn("Decompressed tile: series={}, resolution={}, x={}, y={}",
+            writer.getSeries(), writer.getResolution(), x, y);
+        }
+
         String tileName =
           FormatTools.getTileFilename(x, y, y * nXTiles + x, currentFile);
         if (!currentFile.equals(tileName)) {
@@ -957,18 +1019,8 @@ private long convertTilePlane(IFormatWriter writer, int index, int outputIndex,
           outputY = 0;
         }
 
-        if (writer instanceof TiffWriter) {
-          ((TiffWriter) writer).saveBytes(outputIndex, buf,
-            outputX, outputY, tileWidth, tileHeight);
-        }
-        else if (writer instanceof ImageWriter) {
-          if (baseWriter instanceof TiffWriter) {
-            ((TiffWriter) baseWriter).saveBytes(outputIndex, buf,
-              outputX, outputY, tileWidth, tileHeight);
-          }
-          else {
-            writer.saveBytes(outputIndex, buf, outputX, outputY, tileWidth, tileHeight);
-          }
+        if (tryPrecompressed) {
+          writer.saveCompressedBytes(outputIndex, buf, outputX, outputY, tileWidth, tileHeight);
         }
         else {
           writer.saveBytes(outputIndex, buf, outputX, outputY, tileWidth, tileHeight);
@@ -1033,7 +1085,7 @@ public int getTileColumns(String outputName) {
   private void autoscalePlane(byte[] buf, int index)
     throws FormatException, IOException
   {
-    if (autoscale) {
+    if (autoscale && !tryPrecompressed) {
       Double min = null;
       Double max = null;
 
@@ -1123,8 +1175,17 @@ private byte[] getTile(IFormatReader reader, int resolution,
   {
     if (resolution < reader.getResolutionCount()) {
       reader.setResolution(resolution);
+      int optimalWidth = reader.getOptimalTileWidth();
+      int optimalHeight = reader.getOptimalTileHeight();
+      if (tryPrecompressed) {
+        return reader.openCompressedBytes(no, x / optimalWidth, y / optimalHeight);
+      }
+      tryPrecompressed = false;
       return reader.openBytes(no, x, y, w, h);
     }
+    if (tryPrecompressed) {
+      throw new UnsupportedOperationException("Cannot generate resolutions with precompressed tiles");
+    }
     reader.setResolution(0);
     IImageScaler scaler = new SimpleImageScaler();
     int scale = (int) Math.pow(pyramidScale, resolution);

components/formats-api/src/loci/formats/FormatReader.java

@@ -1310,6 +1310,20 @@ public int getOptimalTileHeight() {
      return (int) Math.min(maxHeight, getSizeY());
   }
 
+  // -- ICompressedTileReader API methods --
+
+  @Override
+  public int getTileRows(int no) {
+    double rows = (double) getSizeY() / getOptimalTileHeight();
+    return (int) Math.ceil(rows);
+  }
+
+  @Override
+  public int getTileColumns(int no) {
+    double cols = (double) getSizeX() / getOptimalTileWidth();
+    return (int) Math.ceil(cols);
+  }
+
   // -- Sub-resolution API methods --
 
   @Override

components/formats-api/src/loci/formats/FormatTools.java

@@ -54,6 +54,7 @@
 import loci.common.services.DependencyException;
 import loci.common.services.ServiceException;
 import loci.common.services.ServiceFactory;
+import loci.formats.codec.Codec;
 import loci.formats.meta.DummyMetadata;
 import loci.formats.meta.MetadataRetrieve;
 import loci.formats.meta.MetadataStore;
@@ -2127,4 +2128,57 @@ public static int getRequiredDirectories(String[] files)
 
     return (int) Math.max(maxDirCount - dirCount, 0);
   }
+
+  /**
+   * Compare the given reader and writer at the specified series and resolution to
+   * see if pre-compressed tiles can be transferred directly from the reader to the writer.
+   */
+  public static boolean canUsePrecompressedTiles(IFormatReader reader, IFormatWriter writer,
+    int series, int resolution)
+    throws FormatException, IOException
+  {
+    if (!(reader instanceof ICompressedTileReader)) {
+      return false;
+    }
+    if (!(writer instanceof ICompressedTileWriter)) {
+      return false;
+    }
+
+    int readerSeries = reader.getSeries();
+    int readerRes = reader.getResolution();
+    reader.setSeries(series);
+    reader.setResolution(resolution);
+
+    int writerSeries = writer.getSeries();
+    int writerRes = writer.getResolution();
+    writer.setSeries(series);
+    writer.setResolution(resolution);
+
+    boolean sameTileWidth = reader.getOptimalTileWidth() == writer.getTileSizeX();
+    boolean sameTileHeight = reader.getOptimalTileHeight() == writer.getTileSizeY();
+
+    // reader and writer must use equivalent codecs
+    // the Codec objects are not expected to be strictly equal,
+    // but both should either be null, or non-null and instances of the same class
+    boolean sameCodec = true;
+    Codec writerCodec = ((ICompressedTileWriter) writer).getCodec();
+    for (int no=0; no<reader.getImageCount(); no++) {
+      Codec readerCodec = ((ICompressedTileReader) reader).getTileCodec(no);
+      if ((writerCodec == null && readerCodec != null) ||
+        (writerCodec != null && readerCodec == null) ||
+        !writerCodec.getClass().equals(readerCodec.getClass()))
+      {
+        sameCodec = false;
+        break;
+      }
+    }
+
+    reader.setSeries(readerSeries);
+    reader.setResolution(resolution);
+    writer.setSeries(series);
+    writer.setResolution(resolution);
+
+    return sameTileWidth && sameTileHeight && sameCodec;
+  }
+
 }

components/formats-api/src/loci/formats/FormatWriter.java

@@ -266,6 +266,11 @@ public void setCodecOptions(CodecOptions options) {
     this.options = options;
   }
 
+  @Override
+  public CodecOptions getCodecOptions() {
+    return options;
+  }
+
   /* @see IFormatWriter#getCompression() */
   @Override
   public String getCompression() {