Merge branch 'ome:develop' into 4110_czi_zstd_pyramid_levels

components/bio-formats-plugins/src/loci/plugins/config/ND2Widgets.java

@@ -83,7 +83,7 @@ public Component[] getWidgets() {
   @Override
   public void itemStateChanged(ItemEvent e) {
     JCheckBox box = (JCheckBox) e.getSource();
-    if (box.equals(getWidgets()[1])) {
+    if (box.equals(getWidgets()[0])) {
       Prefs.set(LociPrefs.PREF_ND2_CHUNKMAP, box.isSelected());
     }
   }

components/formats-api/src/loci/formats/MetadataList.java

@@ -131,6 +131,27 @@ public void set(int i1, int i2, T value) {
     data.get(i1).set(i2, value);
   }
 
+  /**
+   * Remove the array element at the specified indexes.
+   *
+   * @param i1 The primary array index
+   * @param i2 The secondary array index
+   * @return the removed element
+   */
+  public T remove(int i1, int i2) {
+    return data.get(i1).remove(i2);
+  }
+
+  /**
+   * Remove the entire primary array element at the specified index.
+   *
+   * @param i1 The primary array index
+   * @return the removed element
+   */
+  public List<T> remove(int i1) {
+    return data.remove(i1);
+  }
+
   /**
    * Add a empty primary array element.
    */

components/formats-bsd/src/loci/formats/in/TiffReader.java

@@ -320,6 +320,7 @@ else if (eq > 0) {
       t = getImageCount();
     }
     m.dimensionOrder = "XYCZT";
+    m.orderCertain = true;
 
     if (z * t * (isRGB() ? 1 : c) == ifds.size()) {
       m.sizeZ = z;

components/formats-bsd/src/loci/formats/services/JPEGTurboServiceImpl.java

@@ -88,6 +88,8 @@ public class JPEGTurboServiceImpl implements JPEGTurboService {
   private long sos;
   private long imageDimensions;
 
+  private int mcuWidth;
+  private int mcuHeight;
   private int tileWidth;
   private int tileHeight;
   private int xTiles;
@@ -155,6 +157,10 @@ public void initialize(RandomAccessInputStream jpeg, int width, int height)
       }
       else if (marker == SOF0) {
         imageDimensions = in.getFilePointer() + 1;
+        parseSOF();
+      }
+      else if (marker > 0xFFC0 && marker < 0xFFD0 && marker % 4 != 0) {
+        throw new IOException("Unsupported JPEG SOF marker: " + marker);
       }
       else if (marker == SOS) {
         sos = end;
@@ -205,7 +211,7 @@ else if (marker == SOS) {
       }
     }
 
-    tileWidth = restartInterval * 8;
+    tileWidth = restartInterval * mcuWidth;
     tileHeight = (int) Math.min(tileWidth, 512);
 
     xTiles = imageWidth / tileWidth;
@@ -219,8 +225,7 @@ else if (marker == SOS) {
     }
 
     if (restartInterval == 1 && restartMarkers.size() <= 1) {
-      // interval and markers are not present or invalid
-      throw new IOException("Restart interval and markers invalid");
+      throw new IOException("The tiled-JPEG reader only supports images encoded with restart markers");
     }
   }
 
@@ -288,9 +293,9 @@ public byte[] getTile(int tileX, int tileY) throws IOException {
 
     long dataLength = header.length + 2;
 
-    int mult = tileHeight / 8; // was restartInterval
+    int mult = tileHeight / mcuHeight; // was restartInterval
     int start = tileX + (tileY * xTiles * mult);
-    for (int row=0; row<tileHeight/8; row++) {
+    for (int row=0; row<tileHeight/mcuHeight; row++) {
       int end = start + 1;
 
       long startOffset = restartMarkers.get(start);
@@ -313,7 +318,7 @@ public byte[] getTile(int tileX, int tileY) throws IOException {
     offset += header.length;
 
     start = tileX + (tileY * xTiles * mult);
-    for (int row=0; row<tileHeight/8; row++) {
+    for (int row=0; row<tileHeight/mcuHeight; row++) {
       int end = start + 1;
 
       long endOffset = in.length();
@@ -373,6 +378,8 @@ public void close() throws IOException {
     restartInterval = 1;
     sos = 0;
     imageDimensions = 0;
+    mcuWidth = 0;
+    mcuHeight = 0;
     tileWidth = 0;
     tileHeight = 0;
     xTiles = 0;
@@ -400,4 +407,48 @@ private byte[] getFixedHeader() throws IOException {
     return header;
   }
 
+  private void parseSOF() throws IOException {
+    // https://mykb.cipindanci.com/archive/SuperKB/1294/JPEG%20File%20Layout%20and%20Format.htm
+    // example: FFC00011 08001100 11030122 00021101 031101
+    int bpc = in.readByte() & 0xff;
+    if (bpc != 8) {
+      throw new IOException("Only 8-bit channels supported by this reader");
+    }
+    in.skipBytes(4);
+    int channels = in.readByte() & 0xff;
+    if (channels != 3) {
+      // https://stackoverflow.com/questions/51008883/is-there-a-grayscale-jpg-format
+      throw new IOException("Only images with 3 channels are supported by this reader");
+    }
+
+    // Sampling factors: https://stackoverflow.com/q/43225439
+    // https://stackoverflow.com/q/27918757 https://stackoverflow.com/q/43225439
+    // mcu_tool.sh shows MCU size for some images
+    // JpegSnoop or identify -verbose show subsampling rates
+    // convert -sampling rate 2x2 or ffmpeg -i .. -vf format=yuv420p for making images
+
+    // MCU size in X direction divided by 8 is taken as the largest X.
+    // Likewise for Y. For each coomponent, first 4 bits is X, the next 4 bits is Y.
+    // some examples: 2x1,2x1,2x1 (0x21,0x21,0x21) is 16x8.
+    // 2x1,1x1,1x1 is 16x8. 3x2,1x1,1x1 is 24x16
+
+    int maxX = 0x00;
+    int maxY = 0x00;
+
+    for (int i = 0; i < channels; i++) {
+      int componentId = in.readByte() & 0xff;
+
+      int rates = in.readByte();
+      int X = rates & 0xf0; // Shift right later
+      int Y = rates & 0x0f;
+      maxX = Math.max(maxX, X);
+      maxY = Math.max(maxY, Y);
+
+      int quantTableNumber = in.readByte() & 0xff;
+    }
+
+    mcuWidth = (maxX >> 4) * 8;
+    mcuHeight = maxY * 8;
+  }
+
 }

components/formats-gpl/src/loci/formats/in/OIRReader.java

@@ -687,7 +687,9 @@ private void readXMLBlock(String file, RandomAccessInputStream s) throws FormatE
     }
     // total block length could include multiple strings of XML
     int totalBlockLength = s.readInt();
+    LOGGER.debug("total block length = {}", totalBlockLength);
     long end = s.getFilePointer() + totalBlockLength - 4;
+    LOGGER.debug("end = {}", end);
     s.skipBytes(4);
 
     while (s.getFilePointer() < end) {
@@ -735,6 +737,9 @@ else if (xmlLength < 0 || xmlLength >= (s.length() - s.getFilePointer())) {
       long fp = s.getFilePointer();
       String xml = s.readString(xmlLength).trim();
       LOGGER.trace("xml = {}", xml);
+      if (!xml.startsWith("<?xml")) {
+        break;
+      }
       if (isCurrentFile(file) || xml.indexOf("lut:LUT") > 0) {
         parseXML(s, xml, fp);
       }
@@ -1249,6 +1254,15 @@ private void parseImageProperties(Element root) throws FormatException {
           parseChannel(channel, appendChannels);
         }
       }
+
+      // calls to parseChannel may have reset the channels list
+      // so there may be null elements that need to be removed
+      for (int i=0; i<channels.size(); i++) {
+        if (channels.get(i) == null) {
+          channels.remove(i);
+          i--;
+        }
+      }
     }
   }
 
@@ -1277,9 +1291,14 @@ private void parseChannel(Element channel, boolean appendChannels) {
       channelName = name.getTextContent();
     }
 
+    int index = Integer.parseInt(channel.getAttribute("order")) - 1;
+
     if (id != null) {
       boolean foundChannel = false;
       for (Channel ch : channels) {
+        if (ch == null) {
+          continue;
+        }
         if (ch.id.equals(id)) {
           foundChannel = true;
           if (laserId != null) {
@@ -1306,7 +1325,16 @@ private void parseChannel(Element channel, boolean appendChannels) {
             }
           }
         }
-        channels.add(c);
+
+        while (index > channels.size()) {
+          channels.add(null);
+        }
+        if (index == channels.size()) {
+          channels.add(c);
+        }
+        else {
+          channels.set(index, c);
+        }
       }
     }
   }

components/formats-gpl/src/loci/formats/in/SVSReader.java

@@ -71,6 +71,9 @@ public class SVSReader extends BaseTiffReader {
 
   // -- Constants --
 
+  public static final String REMOVE_THUMBNAIL_KEY = "svs.remove_thumbnail";
+  public static final boolean REMOVE_THUMBNAIL_DEFAULT = true;
+
   /** Logger for this class. */
   private static final Logger LOGGER =
     LoggerFactory.getLogger(SVSReader.class);
@@ -118,8 +121,19 @@ public SVSReader() {
 
   public SVSReader(String name, String[] suffixes) {
     super(name, suffixes);
-  }  
-
+  }
+
+  // -- SVSReader API methods --
+
+  public boolean removeThumbnail() {
+    MetadataOptions options = getMetadataOptions();
+    if (options instanceof DynamicMetadataOptions) {
+      return ((DynamicMetadataOptions) options).getBoolean(
+        REMOVE_THUMBNAIL_KEY, REMOVE_THUMBNAIL_DEFAULT);
+    }
+    return REMOVE_THUMBNAIL_DEFAULT;
+  }
+
   // -- IFormatReader API methods --
 
   /* @see loci.formats.IFormatReader#fileGroupOption(String) */
@@ -389,9 +403,15 @@ protected void initStandardMetadata() throws FormatException, IOException {
     for (int i=0; i<seriesCount; i++) {
       setSeries(i);
       int index = i;
-      tiffParser.fillInIFD(ifds.get(index));
 
-      String comment = ifds.get(index).getComment();
+      IFD currentIFD = ifds.get(index);
+      tiffParser.fillInIFD(currentIFD);
+
+      String comment = currentIFD.getComment();
+      int subfileType = currentIFD.getIFDIntValue(IFD.NEW_SUBFILE_TYPE);
+
+      // if there is no identifying comment, assign this IFD
+      // to the label or macro (if a label or macro was not already found)
       if (comment == null) {
         if (labelIndex == -1) {
           labelIndex = i;
@@ -401,10 +421,15 @@ else if (macroIndex == -1) {
         }
         continue;
       }
+
+      // when the comment exists, check it for any information that
+      // identifies the image type
       comments[i] = comment;
       String[] lines = comment.split("\n");
       String[] tokens;
       String key, value;
+      boolean foundLabel = false;
+      boolean foundMacro = false;
       for (String line : lines) {
         tokens = line.split("[|]");
         for (String t : tokens) {
@@ -420,15 +445,28 @@ else if (key.equals("OffsetZ")) {
           }
           else if (t.toLowerCase().indexOf("label") >= 0) {
             labelIndex = i;
+            foundLabel = true;
           }
           else if (t.toLowerCase().indexOf("macro") >= 0) {
             macroIndex = i;
+            foundMacro = true;
           }
         }
       }
       if (zPosition[index] != null) {
         uniqueZ.add(zPosition[index]);
       }
+
+      // if the comment existed but didn't identify a label or macro
+      // check the subfile type to see if we suspect a label or macro anyway
+      if (!foundLabel && !foundMacro && subfileType != 0) {
+        if (labelIndex == -1) {
+          labelIndex = i;
+        }
+        else if (macroIndex == -1) {
+          macroIndex = i;
+        }
+      }
     }
     setSeries(0);
 
@@ -603,6 +641,26 @@ else if (t.toLowerCase().indexOf("macro") >= 0) {
     setSeries(0);
 
     core.reorder();
+
+    if (removeThumbnail()) {
+      // if the smallest resolution uses strips instead of tiles
+      // then it's a "thumbnail" image instead of a real resolution
+      // remove it by default, see https://github.com/ome/bioformats/issues/3757
+      IFD lastResolution = ifds.get(getIFDIndex(core.size(0) - 1, 0));
+      if (lastResolution.get(IFD.STRIP_BYTE_COUNTS) != null) {
+        int index = core.flattenedIndex(0, core.size(0) - 1);
+        core.remove(0, core.size(0) - 1);
+
+        // update the label and macro indexes
+        // otherwise image names won't be set correctly
+        if (index < labelIndex) {
+          labelIndex--;
+        }
+        if (index < macroIndex) {
+          macroIndex--;
+        }
+      }
+    }
   }
 
   /* @see loci.formats.BaseTiffReader#initMetadataStore() */