Skip to content

Commit

Permalink
Add 3D shard test
Browse files Browse the repository at this point in the history
  • Loading branch information
melissalinkert committed Jul 30, 2024
1 parent 629da6c commit 3116a33
Show file tree
Hide file tree
Showing 2 changed files with 68 additions and 3 deletions.
10 changes: 9 additions & 1 deletion src/main/java/com/glencoesoftware/zarr/Convert.java
Original file line number Diff line number Diff line change
Expand Up @@ -305,9 +305,17 @@ public void convertToV3() throws Exception {
// no changes needed
break;
case SUPERCHUNK:
// each shard covers 2x2 chunks
// each shard covers 2x2 chunks in XY
chunkSizes[4] *= 2;
chunkSizes[3] *= 2;

// shard across other dimensions too, but only
// if the dimension is greater than the chunk size
for (int i=0; i<=2; i++) {
if (shape[i] > chunkSizes[i]) {
chunkSizes[i] *= 2;
}
}
break;
case CUSTOM:
// TODO
Expand Down
61 changes: 59 additions & 2 deletions src/test/java/com/glencoesoftware/zarr/test/ConversionTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -375,12 +375,69 @@ public void testSharding() throws Exception {
}
}

@Test
public void test3DSharding() throws Exception {
input = fake("sizeX", "4096", "sizeY", "4096", "sizeZ", "10");
// start with a Z chunk size of 2 (== 5 Z chunks)
assertBioFormats2Raw("-z", "2");

String[] shardOptions = new String[] {
"SINGLE", "CHUNK", "SUPERCHUNK"
};
int[][] shardSizes = new int[][] {
{1, 1, 10, 4096, 4096},
{1, 1, 2, 1024, 1024},
{1, 1, 4, 2048, 2048}
};

for (int opt=0; opt<shardOptions.length; opt++) {
// first convert v2 produced by bioformats2raw to v3
Path v3Output = tmp.newFolder().toPath().resolve("v3-test");
Convert v3Converter = new Convert();
v3Converter.setInput(output.toString());
v3Converter.setOutput(v3Output.toString());

v3Converter.setSharding(shardOptions[opt]);
v3Converter.convertToV3();

// check list of codecs in the v3 arrays

Store store = new FilesystemStore(v3Output);
Array resolution = Array.open(store.resolve("0", "0"));

int[] shardSize = shardSizes[opt];
Assert.assertArrayEquals(resolution.metadata.chunkShape(), shardSize);

// now convert v3 back to v2
Path roundtripOutput = tmp.newFolder().toPath().resolve("v2-roundtrip-test");
Convert v2Converter = new Convert();
v2Converter.setInput(v3Output.toString());
v2Converter.setOutput(roundtripOutput.toString());
v2Converter.setWriteV2(true);
v2Converter.convertToV2();

Path originalOMEXML = output.resolve("OME").resolve("METADATA.ome.xml");
Path roundtripOMEXML = roundtripOutput.resolve("OME").resolve("METADATA.ome.xml");

// make sure the OME-XML is present and not changed
Assert.assertEquals(Files.readAllLines(originalOMEXML), Files.readAllLines(roundtripOMEXML));

// since the image is small, make sure all pixels are identical in both resolutions
for (int r=0; r<4; r++) {
ZarrArray original = ZarrGroup.open(output.resolve("0")).openArray(String.valueOf(r));
ZarrArray roundtrip = ZarrGroup.open(roundtripOutput.resolve("0")).openArray(String.valueOf(r));

compareZarrArrays(original, roundtrip);
}
}
}

private void compareZarrArrays(ZarrArray original, ZarrArray roundtrip) throws Exception {
Assert.assertArrayEquals(original.getShape(), roundtrip.getShape());

int[] shape = original.getShape();
byte[] originalImage = new byte[shape[3] * shape[4]];
byte[] roundtripImage = new byte[shape[3] * shape[4]];
byte[] originalImage = new byte[shape[2] * shape[3] * shape[4]];
byte[] roundtripImage = new byte[shape[2] * shape[3] * shape[4]];
original.read(originalImage, shape);
roundtrip.read(roundtripImage, shape);

Expand Down

0 comments on commit 3116a33

Please sign in to comment.