From 6b827cbe553bd6b45eb3314dd464208cb365cc9c Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Mon, 12 Feb 2024 16:28:47 -0600 Subject: [PATCH 1/4] Add stopwatches to DICOM reader and writer --- .../src/loci/formats/in/DicomReader.java | 69 +++++++++++++++++++ .../src/loci/formats/out/DicomWriter.java | 53 +++++++++++++- 2 files changed, 121 insertions(+), 1 deletion(-) diff --git a/components/formats-bsd/src/loci/formats/in/DicomReader.java b/components/formats-bsd/src/loci/formats/in/DicomReader.java index aba35119fd4..c5916a6c3f4 100644 --- a/components/formats-bsd/src/loci/formats/in/DicomReader.java +++ b/components/formats-bsd/src/loci/formats/in/DicomReader.java @@ -65,6 +65,8 @@ import ome.xml.model.primitives.Timestamp; import ome.units.quantity.Length; import ome.units.UNITS; +import org.perf4j.StopWatch; +import org.perf4j.slf4j.Slf4JStopWatch; import loci.formats.dicom.DicomAttribute; import loci.formats.dicom.DicomFileInfo; @@ -275,6 +277,8 @@ public byte[] openBytes(int no, byte[] buf, int x, int y, int w, int h) { FormatTools.checkPlaneParameters(this, no, buf.length, x, y, w, h); + StopWatch watch = stopWatch(); + int bpp = FormatTools.getBytesPerPixel(getPixelType()); int pixel = bpp * getRGBChannelCount(); Region currentRegion = new Region(x, y, w, h); @@ -289,12 +293,16 @@ public byte[] openBytes(int no, byte[] buf, int x, int y, int w, int h) // look for any tiles that match the requested tile and plane List zs = zOffsets.get(getCoreIndex()); List tiles = tilePositions.get(getCoreIndex()); + watch.stop("openBytes setup, w=" + w + ", h=" + h); + watch.start(); + // TODO: stop tile scan as soon as whole requested area is read? for (int t=0; t 0; invert the values so that // white -> 255 (or 65535) if (bpp == 1) { @@ -327,6 +338,7 @@ else if (bpp == 2) { DataTools.unpackBytes(maxPixelValue - s, buf, i, 2, little); } } + watch.stop("inverted pixel values"); } // NB: do *not* apply the rescale function @@ -377,6 +389,9 @@ public void close(boolean fileOnly) throws IOException { @Override protected void initFile(String id) throws FormatException, IOException { super.initFile(id); + + StopWatch watch = stopWatch(); + if (in != null) { in.close(); } @@ -384,8 +399,14 @@ protected void initFile(String id) throws FormatException, IOException { in.order(true); CoreMetadata m = core.get(0, 0); + watch.stop("open selected file"); + // look for companion files + watch.start(); attachCompanionFiles(); + watch.stop("companion file scan"); + + watch.start(); m.littleEndian = true; long location = 0; @@ -432,7 +453,9 @@ protected void initFile(String id) throws FormatException, IOException { int opticalChannels = 0; List opticalPathIDs = new ArrayList(); + watch.stop("header check and variable init"); + watch.start(); while (decodingTags) { if (in.getFilePointer() + 4 >= in.length()) { break; @@ -682,9 +705,11 @@ else if (child.attribute == OPTICAL_PATH_DESCRIPTION) { decodingTags = false; } } + watch.stop("tag decoding"); if (imagesPerFile == 0) imagesPerFile = 1; if (new Location(currentId).getName().equals("DICOMDIR")) { + watch.start(); String parent = new Location(currentId).getAbsoluteFile().getParent(); Integer[] fileKeys = fileList.keySet().toArray(new Integer[0]); Arrays.sort(fileKeys); @@ -705,8 +730,10 @@ else if (child.attribute == OPTICAL_PATH_DESCRIPTION) { tilePositions = new HashMap>(); zOffsets = new HashMap>(); core.clear(); + watch.stop("DICOMDIR parsing"); } else { + watch.start(); if (m.sizeZ == 0) { m.sizeZ = 1; } @@ -787,7 +814,10 @@ else if (y + originalY < getSizeY()) { LOGGER.info("Calculating image offsets"); calculatePixelsOffsets(baseOffset); + + watch.stop("tile and dimension calculation"); } + watch.start(); makeFileList(); LOGGER.info("Populating metadata"); @@ -796,6 +826,7 @@ else if (y + originalY < getSizeY()) { Integer[] keys = fileList.keySet().toArray(new Integer[0]); Arrays.sort(keys); + watch.stop("assembled file list"); // at this point, we have a list of all files to be grouped together // and have parsed tags from the current file @@ -804,6 +835,7 @@ else if (y + originalY < getSizeY()) { if (seriesCount > 1) { for (int i=0; i currentFileList = fileList.get(keys[i]); DicomFileInfo fileInfo = createFileInfo(currentFileList.get(0)); zOffsets.put(i, fileInfo.zOffsets); @@ -827,17 +859,22 @@ else if (y + originalY < getSizeY()) { fileInfo.positionZ = z; metadataInfo.add(fileInfo); tilePositions.put(i, positions); + seriesWatch.stop("populated series #" + i); } } else { List allFiles = fileList.get(keys[0]); List infos = new ArrayList(); + StopWatch singleSeriesWatch = stopWatch(); + // parse tags for each file for (String file : allFiles) { DicomFileInfo info = createFileInfo(file); infos.add(info); } + singleSeriesWatch.stop("created " + infos.size() + " file infos"); + singleSeriesWatch.start(); if (infos.size() > 1) { infos.sort(null); @@ -931,8 +968,11 @@ else if (info.concatenationIndex == 0) { metadataInfo.add(infos.get(0)); zOffsets.put(0, infos.get(0).zOffsets); } + singleSeriesWatch.stop("updated metadata from file infos"); } + watch.start(); + // The metadata store we're working with. MetadataStore store = makeFilterMetadata(); MetadataTools.populatePixels(store, this, true); @@ -1001,6 +1041,7 @@ else if (info.concatenationIndex == 0) { } } setSeries(0); + watch.stop("populated MetadataStore"); } // -- Helper methods -- @@ -1008,6 +1049,7 @@ else if (info.concatenationIndex == 0) { // TODO: target for refactoring, this can possibly be combined with the // tag parsing loop that calls this private void addInfo(DicomTag info) throws IOException { + StopWatch infoWatch = stopWatch(); CoreMetadata m = core.get(0, 0); m.littleEndian = in.isLittleEndian(); @@ -1152,6 +1194,7 @@ else if (infoString.startsWith("MONOCHROME")) { addOriginalMetadata(key, info); } } + infoWatch.stop("addInfo attribute = " + info.attribute); } /** @@ -1195,6 +1238,7 @@ private void addOriginalMetadata(String key, DicomTag info) { * Build a list of files that belong with the current file. */ private void makeFileList() throws FormatException, IOException { + StopWatch fileScanWatch = stopWatch(); LOGGER.info("Building file list"); if (fileList == null && originalInstance != null && originalDate != null && @@ -1227,11 +1271,13 @@ private void makeFileList() throws FormatException, IOException { } } } + fileScanWatch.stop("finished file scanning"); } else if (fileList == null || !isGroupFiles()) { fileList = new HashMap>(); fileList.put(0, new ArrayList()); fileList.get(0).add(new Location(currentId).getAbsolutePath()); + fileScanWatch.stop("single file, no scanning needed"); } } @@ -1245,6 +1291,7 @@ private void scanDirectory(Location dir, boolean checkSeries) { String[] files = dir.list(true); if (files == null) return; + StopWatch directoryWatch = stopWatch(); Arrays.sort(files); for (String f : files) { String file = new Location(dir, f).getAbsolutePath(); @@ -1253,6 +1300,7 @@ private void scanDirectory(Location dir, boolean checkSeries) addFileToList(file, checkSeries); } } + directoryWatch.stop("scanned directory " + dir); } /** @@ -1261,6 +1309,8 @@ private void scanDirectory(Location dir, boolean checkSeries) private void addFileToList(String file, boolean checkSeries) throws FormatException, IOException { + StopWatch addFileWatch = stopWatch(); + int currentX = 0, currentY = 0; int fileSeries = -1; String thisSpecimen = null; @@ -1341,6 +1391,9 @@ private void addFileToList(String file, boolean checkSeries) } } } + finally { + addFileWatch.stop("checked tags from " + file); + } LOGGER.debug("file = {}", file); LOGGER.debug(" date = {}, originalDate = {}", date, originalDate); @@ -1508,6 +1561,7 @@ private void getTile(DicomTile tile, byte[] buf, int x, int y, int w, int h) CodecOptions options = new CodecOptions(); options.maxBytes = tile.region.width * tile.region.height; for (int c=0; c getTags() { return tags; } + protected Slf4JStopWatch stopWatch() { + return new Slf4JStopWatch(LOGGER, Slf4JStopWatch.DEBUG_LEVEL); + } + } diff --git a/components/formats-bsd/src/loci/formats/out/DicomWriter.java b/components/formats-bsd/src/loci/formats/out/DicomWriter.java index b3f25f64dd4..bdc4f9c1eb5 100644 --- a/components/formats-bsd/src/loci/formats/out/DicomWriter.java +++ b/components/formats-bsd/src/loci/formats/out/DicomWriter.java @@ -78,6 +78,9 @@ import ome.units.UNITS; import ome.units.quantity.Length; +import org.perf4j.StopWatch; +import org.perf4j.slf4j.Slf4JStopWatch; + import static loci.formats.dicom.DicomAttribute.*; import static loci.formats.dicom.DicomVR.*; @@ -143,6 +146,8 @@ public DicomWriter() { public void setExtraMetadata(String tagSource) { FormatTools.assertId(currentId, false, 1); + StopWatch metadataWatch = stopWatch(); + // get the provider (parser) from the source name // uses the file extension, this might need improvement @@ -163,6 +168,7 @@ public void setExtraMetadata(String tagSource) { LOGGER.error("Could not parse extra metadata: " + tagSource, e); } } + metadataWatch.stop("parsed extra metadata from " + tagSource); } /** @@ -193,23 +199,31 @@ public boolean writeDualPersonality() { @Override public void setSeries(int s) throws FormatException { super.setSeries(s); + StopWatch seriesWatch = stopWatch(); try { openFile(series, resolution); } catch (IOException e) { LOGGER.error("Could not open file for series #" + s, e); } + finally { + seriesWatch.stop("setSeries(" + s + ")"); + } } @Override public void setResolution(int r) { super.setResolution(r); + StopWatch resolutionWatch = stopWatch(); try { openFile(series, resolution); } catch (IOException e) { LOGGER.error("Could not open file for series #" + series + ", resolution #" + r, e); } + finally { + resolutionWatch.stop("setResolution(" + r + ")"); + } } @Override @@ -241,6 +255,8 @@ public void saveCompressedBytes(int no, byte[] buf, int x, int y, int w, int h) throw new FormatException("DicomWriter does not allow tiles for non-pyramid images"); } + StopWatch precompressedWatch = stopWatch(); + int bytesPerPixel = FormatTools.getBytesPerPixel( FormatTools.pixelTypeFromString( r.getPixelsType(series).toString())); @@ -297,6 +313,9 @@ public void saveCompressedBytes(int no, byte[] buf, int x, int y, int w, int h) ifds[resolutionIndex][no].put(IFD.PHOTOMETRIC_INTERPRETATION, PhotoInterp.Y_CB_CR.getCode()); } } + precompressedWatch.stop("precompressed tile setup"); + + precompressedWatch.start(); out.seek(out.length()); long start = out.getFilePointer(); @@ -322,6 +341,10 @@ public void saveCompressedBytes(int no, byte[] buf, int x, int y, int w, int h) out.writeByte(0); } + precompressedWatch.stop("wrote precompressed tile"); + + precompressedWatch.start(); + // update the IFD to include this tile int xTiles = (int) Math.ceil((double) getSizeX() / tileWidth[resolutionIndex]); int xTile = x / tileWidth[resolutionIndex]; @@ -351,6 +374,7 @@ public void saveCompressedBytes(int no, byte[] buf, int x, int y, int w, int h) end.elementLength = 0; writeTag(end); } + precompressedWatch.stop("updated IFD"); } /** @@ -380,6 +404,7 @@ else if (x % thisTileWidth != 0 || y % thisTileHeight != 0 || throw new FormatException("Tile too small, expected " + thisTileWidth + "x" + thisTileHeight + ". Setting the tile size to " + getSizeX() + "x" + getSizeY() + " or smaller may work."); } + StopWatch tileWatch = stopWatch(); checkPixelCount(false); boolean first = x == 0 && y == 0; @@ -437,6 +462,9 @@ else if (x % thisTileWidth != 0 || y % thisTileHeight != 0 || } } } + tileWatch.stop("setup tile writing"); + + tileWatch.start(); int bytesPerPixel = FormatTools.getBytesPerPixel( FormatTools.pixelTypeFromString( @@ -491,9 +519,12 @@ else if (x % thisTileWidth != 0 || y % thisTileHeight != 0 || paddedBuf = interleavedBuf; } + tileWatch.stop("repacked tile for compression"); // now we actually compress and write the pixel data + tileWatch.start(); + // we need to know the tile index to write save the tile offset // in the IFD // this tries to calculate the index without assuming sequential tile @@ -577,7 +608,7 @@ else if (x % thisTileWidth != 0 || y % thisTileHeight != 0 || writeTag(end); } } - + tileWatch.stop("compressed and wrote tile"); } /* @see loci.formats.IFormatWriter#canDoStacks() */ @@ -609,6 +640,8 @@ public void setId(String id) throws FormatException, IOException { out.close(); } + StopWatch initWatch = stopWatch(); + checkPixelCount(true); uids = new UIDCreator(); @@ -647,6 +680,9 @@ public void setId(String id) throws FormatException, IOException { String seriesInstanceUID = uids.getUID(); String studyInstanceUID = uids.getUID(); + initWatch.stop("setup data structures"); + initWatch.start(); + for (int pyramid=0; pyramid> 16)); @@ -1627,6 +1669,7 @@ else if (tag.value != null) { throw new IllegalArgumentException(String.valueOf(tag.vr.getCode())); } } + tagWatch.stop("wrote single tag: " + tag); } /** @@ -1710,6 +1753,7 @@ private void openFile(int pyramid, int res) throws IOException { // filename for this series/resolution return; } + StopWatch openWatch = stopWatch(); if (out != null) { out.close(); } @@ -1731,6 +1775,7 @@ else if (r.getPixelsBinDataCount(pyramid) == 0) { if (out.length() == 0) { writeHeader(); } + openWatch.stop("opened " + filename); } /** @@ -1738,6 +1783,7 @@ else if (r.getPixelsBinDataCount(pyramid) == 0) { * See http://dicom.nema.org/medical/dicom/current/output/html/part10.html#sect_7.1 */ private void writeHeader() throws IOException { + StopWatch headerWatch = stopWatch(); boolean littleEndian = out.isLittleEndian(); if (writeDualPersonality()) { // write a TIFF header in the preamble @@ -1811,6 +1857,7 @@ private void writeHeader() throws IOException { out.skipBytes(fileMetaBytes); out.order(littleEndian); + headerWatch.stop("wrote header for series = " + series + ", resolution = " + resolution); } private String getFilename(int pyramid, int res) { @@ -2043,6 +2090,10 @@ private void checkPixelCount(boolean warn) throws FormatException { } } + protected Slf4JStopWatch stopWatch() { + return new Slf4JStopWatch(LOGGER, Slf4JStopWatch.DEBUG_LEVEL); + } + class PlaneOffset { public long xOffset; public long yOffset; From 4e81795abba41f6fdf5e9ca12370d55ef2422f44 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Wed, 13 Mar 2024 18:05:51 -0500 Subject: [PATCH 2/4] Write header to an in-memory buffer before saving to disk This reduces the number of writes during initialization, which reduces overall conversion time. --- .../src/loci/formats/out/DicomWriter.java | 125 ++++++++++-------- 1 file changed, 69 insertions(+), 56 deletions(-) diff --git a/components/formats-bsd/src/loci/formats/out/DicomWriter.java b/components/formats-bsd/src/loci/formats/out/DicomWriter.java index bdc4f9c1eb5..4bab31a2d7b 100644 --- a/components/formats-bsd/src/loci/formats/out/DicomWriter.java +++ b/components/formats-bsd/src/loci/formats/out/DicomWriter.java @@ -45,6 +45,7 @@ import java.util.Comparator; import java.util.List; +import loci.common.ByteArrayHandle; import loci.common.Constants; import loci.common.DataTools; import loci.common.DateTools; @@ -1485,45 +1486,49 @@ private int getStoredLength(DicomTag tag) { } private void writeTag(DicomTag tag) throws IOException { + writeTag(tag, out); + } + + private void writeTag(DicomTag tag, RandomAccessOutputStream output) throws IOException { StopWatch tagWatch = stopWatch(); int tagCode = tag.attribute == null ? tag.tag : tag.attribute.getTag(); - out.writeShort((short) ((tagCode & 0xffff0000) >> 16)); - out.writeShort((short) (tagCode & 0xffff)); + output.writeShort((short) ((tagCode & 0xffff0000) >> 16)); + output.writeShort((short) (tagCode & 0xffff)); if (tag.vr == IMPLICIT) { - out.writeInt(getStoredLength(tag)); + output.writeInt(getStoredLength(tag)); } else { - boolean order = out.isLittleEndian(); - out.order(false); - out.writeShort(tag.vr.getCode()); - out.order(order); + boolean order = output.isLittleEndian(); + output.order(false); + output.writeShort(tag.vr.getCode()); + output.order(order); if (tag.vr == OB || tag.vr == OW || tag.vr == SQ || tag.vr == UN || tag.vr == UT || tag.vr == UC) { - out.writeShort((short) 0); - out.writeInt(getStoredLength(tag)); + output.writeShort((short) 0); + output.writeInt(getStoredLength(tag)); } else { - out.writeShort((short) getStoredLength(tag)); + output.writeShort((short) getStoredLength(tag)); } if (tag.attribute == TRANSFER_SYNTAX_UID) { - transferSyntaxPointer[getIndex(series, resolution)] = out.getFilePointer(); + transferSyntaxPointer[getIndex(series, resolution)] = output.getFilePointer(); } else if (tag.attribute == LOSSY_IMAGE_COMPRESSION_METHOD) { - compressionMethodPointer[getIndex(series, resolution)] = out.getFilePointer(); + compressionMethodPointer[getIndex(series, resolution)] = output.getFilePointer(); } else if (tag.attribute == FILE_META_INFO_GROUP_LENGTH) { - fileMetaLengthPointer = out.getFilePointer(); + fileMetaLengthPointer = output.getFilePointer(); } // sequences with no items still need to write a SequenceDelimitationItem below if (tag.children.size() == 0 && tag.value == null && tag.vr != SQ) { if (tag.attribute != PIXEL_DATA) { - out.skipBytes(tag.elementLength); + output.skipBytes(tag.elementLength); } return; } @@ -1574,27 +1579,27 @@ else if (tag.value != null) { } switch (tag.attribute) { case OPTICAL_PATH_ID: - planeOffsets[resolutionIndex][currentPlane].cOffset = out.getFilePointer(); + planeOffsets[resolutionIndex][currentPlane].cOffset = output.getFilePointer(); break; case ROW_POSITION_IN_MATRIX: - planeOffsets[resolutionIndex][currentPlane].yOffset = out.getFilePointer(); + planeOffsets[resolutionIndex][currentPlane].yOffset = output.getFilePointer(); break; case COLUMN_POSITION_IN_MATRIX: - planeOffsets[resolutionIndex][currentPlane].xOffset = out.getFilePointer(); + planeOffsets[resolutionIndex][currentPlane].xOffset = output.getFilePointer(); break; case DIMENSION_INDEX_VALUES: - planeOffsets[resolutionIndex][currentPlane].dimensionIndex = out.getFilePointer(); + planeOffsets[resolutionIndex][currentPlane].dimensionIndex = output.getFilePointer(); break; case X_OFFSET_IN_SLIDE: - planeOffsets[resolutionIndex][currentPlane].xOffsetReal = out.getFilePointer(); + planeOffsets[resolutionIndex][currentPlane].xOffsetReal = output.getFilePointer(); planeOffsets[resolutionIndex][currentPlane].xOffsetSize = tag.elementLength; break; case Y_OFFSET_IN_SLIDE: - planeOffsets[resolutionIndex][currentPlane].yOffsetReal = out.getFilePointer(); + planeOffsets[resolutionIndex][currentPlane].yOffsetReal = output.getFilePointer(); planeOffsets[resolutionIndex][currentPlane].yOffsetSize = tag.elementLength; break; case Z_OFFSET_IN_SLIDE: - planeOffsets[resolutionIndex][currentPlane].zOffset = out.getFilePointer(); + planeOffsets[resolutionIndex][currentPlane].zOffset = output.getFilePointer(); planeOffsets[resolutionIndex][currentPlane].zOffsetSize = tag.elementLength; break; } @@ -1617,53 +1622,53 @@ else if (tag.value != null) { case UI: case UR: case UT: - out.writeBytes(tag.value.toString()); + output.writeBytes(tag.value.toString()); break; case AT: for (short s : (short[]) tag.value) { - out.writeShort(s); + output.writeShort(s); } break; case FL: for (float f : (float[]) tag.value) { - out.writeFloat(f); + output.writeFloat(f); } break; case FD: for (double d : (double[]) tag.value) { - out.writeDouble(d); + output.writeDouble(d); } break; case OB: - out.write((byte[]) tag.value); + output.write((byte[]) tag.value); break; case SL: for (int v : (int[]) tag.value) { - out.writeInt(v); + output.writeInt(v); } break; case SS: for (short s : (short[]) tag.value) { - out.writeShort(s); + output.writeShort(s); } break; case SV: for (long v : (long[]) tag.value) { - out.writeLong(v); + output.writeLong(v); } break; case UL: for (long v : (long[]) tag.value) { - out.writeInt((int) (v & 0xffffffff)); + output.writeInt((int) (v & 0xffffffff)); } break; case US: for (short s : (short[]) tag.value) { - out.writeShort(s); + output.writeShort(s); } break; case IMPLICIT: - out.write((byte[]) tag.value); + output.write((byte[]) tag.value); break; default: throw new IllegalArgumentException(String.valueOf(tag.vr.getCode())); @@ -1784,78 +1789,86 @@ else if (r.getPixelsBinDataCount(pyramid) == 0) { */ private void writeHeader() throws IOException { StopWatch headerWatch = stopWatch(); + ByteArrayHandle buffer = new ByteArrayHandle(); + RandomAccessOutputStream headerBuffer = new RandomAccessOutputStream(buffer); boolean littleEndian = out.isLittleEndian(); + headerBuffer.order(littleEndian); if (writeDualPersonality()) { // write a TIFF header in the preamble if (littleEndian) { - out.writeByte(TiffConstants.LITTLE); - out.writeByte(TiffConstants.LITTLE); + headerBuffer.writeByte(TiffConstants.LITTLE); + headerBuffer.writeByte(TiffConstants.LITTLE); } else { - out.writeByte(TiffConstants.BIG); - out.writeByte(TiffConstants.BIG); + headerBuffer.writeByte(TiffConstants.BIG); + headerBuffer.writeByte(TiffConstants.BIG); } if (bigTiff) { - out.writeShort(TiffConstants.BIG_TIFF_MAGIC_NUMBER); - out.writeShort(8); // number of bytes in an offset - out.writeShort(0); // reserved + headerBuffer.writeShort(TiffConstants.BIG_TIFF_MAGIC_NUMBER); + headerBuffer.writeShort(8); // number of bytes in an offset + headerBuffer.writeShort(0); // reserved - nextIFDPointer[getIndex(series, resolution)] = out.getFilePointer(); - out.writeLong(-1); // placeholder to first IFD + nextIFDPointer[getIndex(series, resolution)] = headerBuffer.getFilePointer(); + headerBuffer.writeLong(-1); // placeholder to first IFD } else { - out.writeShort(TiffConstants.MAGIC_NUMBER); - nextIFDPointer[getIndex(series, resolution)] = out.getFilePointer(); - out.writeInt(-1); // placeholder to first IFD + headerBuffer.writeShort(TiffConstants.MAGIC_NUMBER); + nextIFDPointer[getIndex(series, resolution)] = headerBuffer.getFilePointer(); + headerBuffer.writeInt(-1); // placeholder to first IFD } } else { byte[] preamble = new byte[128]; - out.write(preamble); + headerBuffer.write(preamble); } // seek to end of preamble, then write DICOM header - out.seek(128); - out.order(true); - out.writeBytes("DICM"); + headerBuffer.seek(128); + headerBuffer.order(true); + headerBuffer.writeBytes("DICM"); DicomTag fileMetaLength = new DicomTag(FILE_META_INFO_GROUP_LENGTH, UL); // placeholder value, overwritten at the end of this method fileMetaLength.value = new long[] {0}; - writeTag(fileMetaLength); + writeTag(fileMetaLength, headerBuffer); DicomTag fileMetaVersion = new DicomTag(FILE_META_INFO_VERSION, OB); fileMetaVersion.value = new byte[] {0, 1}; - writeTag(fileMetaVersion); + writeTag(fileMetaVersion, headerBuffer); DicomTag mediaStorageClassUID = new DicomTag(MEDIA_SOP_CLASS_UID, UI); mediaStorageClassUID.value = padUID(SOP_CLASS_UID_VALUE); - writeTag(mediaStorageClassUID); + writeTag(mediaStorageClassUID, headerBuffer); DicomTag mediaStorageInstanceUID = new DicomTag(MEDIA_SOP_INSTANCE_UID, UI); mediaStorageInstanceUID.value = padUID(instanceUIDValue); - writeTag(mediaStorageInstanceUID); + writeTag(mediaStorageInstanceUID, headerBuffer); // placeholder, will be overwritten on the first call to saveBytes DicomTag transferSyntaxUID = new DicomTag(TRANSFER_SYNTAX_UID, UI); transferSyntaxUID.elementLength = 22; - writeTag(transferSyntaxUID); + writeTag(transferSyntaxUID, headerBuffer); DicomTag implementationClassUID = new DicomTag(IMPLEMENTATION_UID, UI); implementationClassUID.value = padUID(implementationUID); - writeTag(implementationClassUID); + writeTag(implementationClassUID, headerBuffer); DicomTag implementationVersionName = new DicomTag(IMPLEMENTATION_VERSION, SH); implementationVersionName.value = padString(FormatTools.VERSION); - writeTag(implementationVersionName); + writeTag(implementationVersionName, headerBuffer); + + int bufferBytes = (int) headerBuffer.getFilePointer(); + out.order(headerBuffer.isLittleEndian()); + headerBuffer.close(); + out.write(buffer.getBytes(), 0, bufferBytes); // count all bytes after the file meta length value int fileMetaBytes = (int) (out.getFilePointer() - fileMetaLengthPointer - 4); out.seek(fileMetaLengthPointer); out.writeInt(fileMetaBytes); fileMetaLengthPointer = 0; - out.skipBytes(fileMetaBytes); + out.skipBytes(fileMetaBytes); out.order(littleEndian); headerWatch.stop("wrote header for series = " + series + ", resolution = " + resolution); } From 4ea1fce3d7549125f1568f36d43a7341fd8160a8 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Sun, 24 Mar 2024 14:56:30 -0500 Subject: [PATCH 3/4] A few improvements to initialization and tile read time --- .../src/loci/formats/in/DicomReader.java | 333 ++++++++++-------- 1 file changed, 183 insertions(+), 150 deletions(-) diff --git a/components/formats-bsd/src/loci/formats/in/DicomReader.java b/components/formats-bsd/src/loci/formats/in/DicomReader.java index c5916a6c3f4..5ea3bf935c9 100644 --- a/components/formats-bsd/src/loci/formats/in/DicomReader.java +++ b/components/formats-bsd/src/loci/formats/in/DicomReader.java @@ -129,6 +129,9 @@ public class DicomReader extends SubResolutionFormatReader { private List tags; + private transient String currentTileFile = null; + private transient RandomAccessInputStream currentTileStream = null; + // -- Constructor -- /** Constructs a new DICOM reader. */ @@ -295,7 +298,7 @@ public byte[] openBytes(int no, byte[] buf, int x, int y, int w, int h) List tiles = tilePositions.get(getCoreIndex()); watch.stop("openBytes setup, w=" + w + ", h=" + h); watch.start(); - // TODO: stop tile scan as soon as whole requested area is read? + for (int t=0; t= stream.length()) { - LOGGER.error("attempted to read beyond end of file ({}, {})", tile.fileOffset, tile.file); - return; - } - LOGGER.debug("reading from offset = {}, file = {}", tile.fileOffset, tile.file); - stream.seek(tile.fileOffset); - if (tile.isRLE) { - // plane is compressed using run-length encoding - CodecOptions options = new CodecOptions(); - options.maxBytes = tile.region.width * tile.region.height; - for (int c=0; c 1) { - int plane = bytes / (bpp * ec); - byte[][] tmp = new byte[bpp][]; - long start = stream.getFilePointer(); - for (int i=0; i= stream.length()) { + LOGGER.error("attempted to read beyond end of file ({}, {})", tile.fileOffset, tile.file); + return; + } + LOGGER.debug("reading from offset = {}, file = {}", tile.fileOffset, tile.file); + stream.seek(tile.fileOffset); + + if (tile.isRLE) { + // plane is compressed using run-length encoding + CodecOptions options = new CodecOptions(); + options.maxBytes = tile.region.width * tile.region.height; + for (int c=0; c 1) { + int plane = bytes / (bpp * ec); + byte[][] tmp = new byte[bpp][]; + long start = stream.getFilePointer(); + for (int i=0; i 0 && tmp[i].length > options.maxBytes) { + stream.seek(start); tmp[i] = codec.decompress(stream, options); - if (i > 0 && tmp[i].length > options.maxBytes) { - stream.seek(start); - tmp[i] = codec.decompress(stream, options); - } - if (!tile.last || i < bpp - 1) { - start = stream.getFilePointer(); - while (stream.read() == 0); - long end = stream.getFilePointer(); - stream.seek(end - 1); - } } - t = new byte[bytes / ec]; - for (int i=0; i= 0 && (b[pt] != (byte) 0xff || b[pt + 1] != (byte) 0xd9)) { - pt--; - } - if (pt < 0) { - byte[] tmp = b; - b = new byte[tmp.length + 2]; - System.arraycopy(tmp, 0, b, 0, tmp.length); - b[b.length - 2] = (byte) 0xff; - b[b.length - 1] = (byte) 0xd9; - } - else if (pt < b.length - 2) { - byte[] tmp = b; - b = new byte[pt + 2]; - System.arraycopy(tmp, 0, b, 0, b.length); - } + int pt = b.length - 2; + while (pt >= 0 && (b[pt] != (byte) 0xff || b[pt + 1] != (byte) 0xd9)) { + pt--; + } + if (pt < 0) { + byte[] tmp = b; + b = new byte[tmp.length + 2]; + System.arraycopy(tmp, 0, b, 0, tmp.length); + b[b.length - 2] = (byte) 0xff; + b[b.length - 1] = (byte) 0xd9; + } + else if (pt < b.length - 2) { + byte[] tmp = b; + b = new byte[pt + 2]; + System.arraycopy(tmp, 0, b, 0, b.length); + } - Codec codec = null; - CodecOptions options = new CodecOptions(); - options.littleEndian = isLittleEndian(); - options.interleaved = isInterleaved(); - if (tile.isJPEG) codec = new JPEGCodec(); - else codec = new JPEG2000Codec(); + Codec codec = null; + CodecOptions options = new CodecOptions(); + options.littleEndian = isLittleEndian(); + options.interleaved = isInterleaved(); + if (tile.isJPEG) codec = new JPEGCodec(); + else codec = new JPEG2000Codec(); - try { - b = codec.decompress(b, options); - } - catch (NullPointerException e) { - LOGGER.debug("Could not read empty or invalid tile", e); - return; - } - finally { - jpegWatch.stop("decompressed (jpeg = " + tile.isJPEG + ")"); - } - jpegWatch.start(); + try { + b = codec.decompress(b, options); + } + catch (NullPointerException e) { + LOGGER.debug("Could not read empty or invalid tile", e); + return; + } + finally { + jpegWatch.stop("decompressed (jpeg = " + tile.isJPEG + ")"); + } + jpegWatch.start(); - int rowLen = w * bpp; - int srcRowLen = tile.region.width * bpp; + int rowLen = w * bpp; + int srcRowLen = tile.region.width * bpp; - if (isInterleaved()) { - rowLen *= ec; - srcRowLen *= ec; - for (int row=0; row currentTilePositions = tilePositions.get(getCoreIndex()); + long offset = baseOffset; + int bufferSize = 4096; for (int i=0; i 0) { - tilePositions.get(getCoreIndex()).get(i - 1).endOffset = tilePositions.get(getCoreIndex()).get(i).fileOffset; + currentTilePositions.get(i - 1).endOffset = currentTile.fileOffset; } if (i == imagesPerFile - 1) { - tilePositions.get(getCoreIndex()).get(i).endOffset = in.length(); + currentTile.endOffset = in.length(); } if (!zOffsets.containsKey(getCoreIndex())) { zOffsets.put(getCoreIndex(), new ArrayList()); } - Double z = tilePositions.get(getCoreIndex()).get(i).zOffset; + Double z = currentTile.zOffset; if (!zOffsets.get(getCoreIndex()).contains(z)) { zOffsets.get(getCoreIndex()).add(z); } - offsetWatch.stop("calculated offset #" + i); } } @@ -1888,6 +1917,10 @@ private DicomFileInfo createFileInfo(String file) throws FormatException, IOExce return new DicomFileInfo(new Location(file).getAbsolutePath()); } + private boolean encloses(Region a, Region b) { + return a.intersection(b).equals(b); + } + private void updateCoreMetadata(CoreMetadata ms) { if (ms.sizeC == 0) ms.sizeC = 1; ms.sizeT = 1; From 99462118cc80f1f8e9c6407f4f8825c82b301c24 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Thu, 29 Aug 2024 17:19:45 -0500 Subject: [PATCH 4/4] Reduce number of calls to getSamplesPerPixel() Especially when repacking tiles, many duplicate calls to this method can get expensive. --- .../src/loci/formats/out/DicomWriter.java | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/components/formats-bsd/src/loci/formats/out/DicomWriter.java b/components/formats-bsd/src/loci/formats/out/DicomWriter.java index de3850248ea..d578d10e06c 100644 --- a/components/formats-bsd/src/loci/formats/out/DicomWriter.java +++ b/components/formats-bsd/src/loci/formats/out/DicomWriter.java @@ -500,6 +500,7 @@ public void saveBytes(int no, byte[] buf, int x, int y, int w, int h) int bytesPerPixel = FormatTools.getBytesPerPixel( FormatTools.pixelTypeFromString( r.getPixelsType(series).toString())); + int samplesPerPixel = getSamplesPerPixel(); out.seek(out.length()); long start = out.getFilePointer(); @@ -512,9 +513,9 @@ public void saveBytes(int no, byte[] buf, int x, int y, int w, int h) if ((x + w == getSizeX() && w < thisTileWidth) || (y + h == getSizeY() && h < thisTileHeight)) { - if (interleaved || getSamplesPerPixel() == 1) { - int srcRowLen = w * bytesPerPixel * getSamplesPerPixel(); - int destRowLen = thisTileWidth * bytesPerPixel * getSamplesPerPixel(); + if (interleaved || samplesPerPixel == 1) { + int srcRowLen = w * bytesPerPixel * samplesPerPixel; + int destRowLen = thisTileWidth * bytesPerPixel * samplesPerPixel; paddedBuf = new byte[thisTileHeight * destRowLen]; for (int row=0; row