WIP: N5S3

pom.xml

@@ -209,6 +209,11 @@
 			<artifactId>n5</artifactId>
 			<version>2.1.6</version>
 		</dependency>
+		<dependency>
+			<groupId>org.janelia.saalfeldlab</groupId>
+			<artifactId>n5-aws-s3</artifactId>
+			<version>2.1.1-SNAPSHOT</version>
+		</dependency>
 		<dependency>
 			<groupId>com.miglayout</groupId>
 			<artifactId>miglayout-swing</artifactId>

src/main/java/bdv/export/n5/WriteSequenceToN5.java

@@ -8,7 +8,7 @@
 import bdv.export.ExportScalePyramid.AfterEachPlane;
 import bdv.export.ExportScalePyramid.LoopbackHeuristic;
 import bdv.img.cache.SimpleCacheArrayLoader;
-import bdv.img.n5.N5ImageLoader;
+import bdv.img.n5.N5FSImageLoader;
 import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -325,7 +325,7 @@ public RandomAccessibleInterval< T > getImage( final int level ) throws IOExcept
 			final long[] dimensions = attributes.getDimensions();
 			final int[] cellDimensions = attributes.getBlockSize();
 			final CellGrid grid = new CellGrid( dimensions, cellDimensions );
-			final SimpleCacheArrayLoader< ? > cacheArrayLoader = N5ImageLoader.createCacheArrayLoader( n5, pathName );
+			final SimpleCacheArrayLoader< ? > cacheArrayLoader = N5FSImageLoader.createCacheArrayLoader( n5, pathName );
 			return new ReadOnlyCachedCellImgFactory().createWithCacheLoader(
 					dimensions, type,
 					key -> {

src/main/java/bdv/img/n5/N5FSImageLoader.java

@@ -0,0 +1,52 @@
+/*
+ * #%L
+ * BigDataViewer core classes with minimal dependencies
+ * %%
+ * Copyright (C) 2012 - 2016 Tobias Pietzsch, Stephan Saalfeld, Stephan Preibisch,
+ * Jean-Yves Tinevez, HongKee Moon, Johannes Schindelin, Curtis Rueden, John Bogovic
+ * %%
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * #L%
+ */
+package bdv.img.n5;
+
+import mpicbg.spim.data.generic.sequence.AbstractSequenceDescription;
+import org.janelia.saalfeldlab.n5.N5FSReader;
+
+import java.io.File;
+import java.io.IOException;
+
+public class N5FSImageLoader extends N5ImageLoader
+{
+	private final File n5File;
+
+	public N5FSImageLoader( final File n5File, final AbstractSequenceDescription< ?, ?, ? > sequenceDescription ) throws IOException
+	{
+		super( new N5FSReader( n5File.getAbsolutePath() ), sequenceDescription );
+		this.n5File = n5File;
+	}
+
+	public File getN5File()
+	{
+		return n5File;
+	}
+}

src/main/java/bdv/img/n5/N5ImageLoader.java

@@ -36,98 +36,63 @@
 import bdv.img.cache.VolatileGlobalCellCache;
 import bdv.util.ConstantRandomAccessible;
 import bdv.util.MipmapTransforms;
-import java.io.File;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.Callable;
-import java.util.function.Function;
+import com.amazonaws.SdkClientException;
 import mpicbg.spim.data.generic.sequence.AbstractSequenceDescription;
 import mpicbg.spim.data.generic.sequence.BasicViewSetup;
 import mpicbg.spim.data.generic.sequence.ImgLoaderHint;
 import mpicbg.spim.data.sequence.MultiResolutionImgLoader;
 import mpicbg.spim.data.sequence.MultiResolutionSetupImgLoader;
 import mpicbg.spim.data.sequence.VoxelDimensions;
-import net.imglib2.Dimensions;
-import net.imglib2.FinalDimensions;
-import net.imglib2.FinalInterval;
-import net.imglib2.RandomAccessibleInterval;
-import net.imglib2.Volatile;
+import net.imglib2.*;
 import net.imglib2.cache.queue.BlockingFetchQueues;
 import net.imglib2.cache.queue.FetcherThreads;
 import net.imglib2.cache.volatiles.CacheHints;
 import net.imglib2.cache.volatiles.LoadingStrategy;
-import net.imglib2.img.basictypeaccess.volatiles.array.VolatileByteArray;
-import net.imglib2.img.basictypeaccess.volatiles.array.VolatileDoubleArray;
-import net.imglib2.img.basictypeaccess.volatiles.array.VolatileFloatArray;
-import net.imglib2.img.basictypeaccess.volatiles.array.VolatileIntArray;
-import net.imglib2.img.basictypeaccess.volatiles.array.VolatileLongArray;
-import net.imglib2.img.basictypeaccess.volatiles.array.VolatileShortArray;
+import net.imglib2.img.basictypeaccess.volatiles.array.*;
 import net.imglib2.img.cell.CellGrid;
 import net.imglib2.img.cell.CellImg;
 import net.imglib2.realtransform.AffineTransform3D;
 import net.imglib2.type.NativeType;
-import net.imglib2.type.numeric.integer.ByteType;
-import net.imglib2.type.numeric.integer.IntType;
-import net.imglib2.type.numeric.integer.LongType;
-import net.imglib2.type.numeric.integer.ShortType;
-import net.imglib2.type.numeric.integer.UnsignedByteType;
-import net.imglib2.type.numeric.integer.UnsignedIntType;
-import net.imglib2.type.numeric.integer.UnsignedLongType;
-import net.imglib2.type.numeric.integer.UnsignedShortType;
+import net.imglib2.type.numeric.integer.*;
 import net.imglib2.type.numeric.real.DoubleType;
 import net.imglib2.type.numeric.real.FloatType;
-import net.imglib2.type.volatiles.VolatileByteType;
-import net.imglib2.type.volatiles.VolatileDoubleType;
-import net.imglib2.type.volatiles.VolatileFloatType;
-import net.imglib2.type.volatiles.VolatileIntType;
-import net.imglib2.type.volatiles.VolatileLongType;
-import net.imglib2.type.volatiles.VolatileShortType;
-import net.imglib2.type.volatiles.VolatileUnsignedByteType;
-import net.imglib2.type.volatiles.VolatileUnsignedIntType;
-import net.imglib2.type.volatiles.VolatileUnsignedLongType;
-import net.imglib2.type.volatiles.VolatileUnsignedShortType;
+import net.imglib2.type.volatiles.*;
 import net.imglib2.util.Cast;
 import net.imglib2.view.Views;
-import org.janelia.saalfeldlab.n5.DataBlock;
-import org.janelia.saalfeldlab.n5.DataType;
-import org.janelia.saalfeldlab.n5.DatasetAttributes;
-import org.janelia.saalfeldlab.n5.N5FSReader;
-import org.janelia.saalfeldlab.n5.N5Reader;
+import org.janelia.saalfeldlab.n5.*;
 
-import static bdv.img.n5.BdvN5Format.DATA_TYPE_KEY;
-import static bdv.img.n5.BdvN5Format.DOWNSAMPLING_FACTORS_KEY;
-import static bdv.img.n5.BdvN5Format.getPathName;
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.function.Function;
+
+import static bdv.img.n5.BdvN5Format.*;
 
 public class N5ImageLoader implements ViewerImgLoader, MultiResolutionImgLoader
 {
-	private final File n5File;
+	protected final N5Reader n5;
 
 	// TODO: it would be good if this would not be needed
 	//       find available setups from the n5
-	private final AbstractSequenceDescription< ?, ?, ? > seq;
+	protected final AbstractSequenceDescription< ?, ?, ? > seq;
 
 	/**
 	 * Maps setup id to {@link SetupImgLoader}.
 	 */
 	private final Map< Integer, SetupImgLoader > setupImgLoaders = new HashMap<>();
 
-	public N5ImageLoader( final File n5File, final AbstractSequenceDescription< ?, ?, ? > sequenceDescription )
-	{
-		this.n5File = n5File;
-		this.seq = sequenceDescription;
-	}
-
-	public File getN5File()
-	{
-		return n5File;
-	}
-
 	private volatile boolean isOpen = false;
 	private FetcherThreads fetchers;
 	private VolatileGlobalCellCache cache;
-	private N5Reader n5;
+
+	public N5ImageLoader( N5Reader n5Reader, AbstractSequenceDescription< ?, ?, ? > sequenceDescription )
+	{
+		this.n5 = n5Reader;
+		this.seq = sequenceDescription;
+	}
 
 	private void open()
 	{
@@ -140,8 +105,6 @@ private void open()
 
 				try
 				{
-					this.n5 = new N5FSReader( n5File.getAbsolutePath() );
-
 					int maxNumLevels = 0;
 					final List< ? extends BasicViewSetup > setups = seq.getViewSetupsOrdered();
 					for ( final BasicViewSetup setup : setups )
@@ -353,7 +316,46 @@ private static class N5CacheArrayLoader< A > implements SimpleCacheArrayLoader<
 		@Override
 		public A loadArray( final long[] gridPosition ) throws IOException
 		{
-			return createArray.apply( n5.readBlock( pathName, attributes, gridPosition ) );
+			DataBlock< ? > block = null;
+
+			try {
+				block = n5.readBlock( pathName, attributes, gridPosition );
+			}
+			catch ( SdkClientException e )
+			{
+				System.err.println( e ); // this happens sometimes, not sure yet why...
+			}
+
+			if ( block == null )
+			{
+				final int[] blockSize = attributes.getBlockSize();
+				final int n = blockSize[ 0 ] * blockSize[ 1 ] * blockSize[ 2 ];
+				switch ( attributes.getDataType() )
+				{
+					case UINT8:
+					case INT8:
+						return createArray.apply( new ByteArrayDataBlock( blockSize, gridPosition, new byte[ n ] ) );
+					case UINT16:
+					case INT16:
+						return createArray.apply( new ShortArrayDataBlock( blockSize, gridPosition, new short[ n ] ) );
+					case UINT32:
+					case INT32:
+						return createArray.apply( new IntArrayDataBlock( blockSize, gridPosition, new int[ n ] ) );
+					case UINT64:
+					case INT64:
+						return createArray.apply( new LongArrayDataBlock( blockSize, gridPosition, new long[ n ] ) );
+					case FLOAT32:
+						return createArray.apply( new FloatArrayDataBlock( blockSize, gridPosition, new float[ n ] ) );
+					case FLOAT64:
+						return createArray.apply( new DoubleArrayDataBlock( blockSize, gridPosition, new double[ n ] ) );
+					default:
+						throw new IllegalArgumentException();
+				}
+			}
+			else
+			{
+				return createArray.apply( block );
+			}
 		}
 	}