Merge pull request #26 from joshmoore/submission-run

Submission run

.env-2D

@@ -0,0 +1,24 @@
+AWS_ACCESS_KEY_ID=minioadmin
+AWS_SECRET_ACCESS_KEY=minioadmin
+BUCKET=ngff-latency-benchmark
+HOST=nginx
+HOST=10.0.1.229
+
+XY=65536
+Z=1
+C=8
+T=1
+XC=256
+ZC=1
+ROOT=/var/lib/docker
+ROOT=/uod/idr-scratch/ngff-latency-benchmark
+ROOT=/uod/idr/scratch/idr1-slot2/ngff-latency-benchmark
+ROOT=/home/ubuntu/ngff-latency-benchmark
+
+NAME=${XY}-Z-${Z}-T-${T}-C-${C}-XYC-${XC}-ZC-${ZC}
+DIR=${ROOT}/${NAME}
+BASE=IMS_XY-${NAME}
+ROUNDS=100
+TEST_REPEATS=1
+
+#S3ARGS={"anon": true, "client_kwargs": {"endpoint_url": "http://${HOST}:9000"}}

.env-3D

@@ -0,0 +1,24 @@
+AWS_ACCESS_KEY_ID=minioadmin
+AWS_SECRET_ACCESS_KEY=minioadmin
+BUCKET=ngff-latency-benchmark
+HOST=nginx
+HOST=10.0.1.22
+
+XY=1024
+Z=1024
+C=1
+T=100
+XC=32
+ZC=32
+ROOT=/var/lib/docker
+ROOT=/uod/idr-scratch/ngff-latency-benchmark
+ROOT=/uod/idr/scratch/idr1-slot2/ngff-latency-benchmark
+ROOT=/home/ubuntu/ngff-latency-benchmark
+
+NAME=${XY}-Z-${Z}-T-${T}-C-${C}-XYC-${XC}-ZC-${ZC}
+DIR=${ROOT}/${NAME}
+BASE=IMS_XY-${NAME}
+ROUNDS=100
+TEST_REPEATS=1
+
+#S3ARGS={"anon": true, "client_kwargs": {"endpoint_url": "http://${HOST}:9000"}}

convert/Dockerfile

@@ -5,9 +5,9 @@ RUN conda install -c ome bioformats2raw raw2ometiff
 # Temporary
 #
 RUN conda install -y -c anaconda -c conda-forge maven git
-RUN git clone -b nested-re-re --depth=1 git://github.com/joshmoore/bioformats2raw /tmp/bioformats2raw
-RUN git clone -b nested --depth=1 git://github.com/joshmoore/raw2ometiff /tmp/raw2ometiff
-RUN git clone -b nested --depth=1 git://github.com/joshmoore/jzarr /tmp/jzarr
+RUN git clone -b nested-re-re --depth=1 https://github.com/joshmoore/bioformats2raw /tmp/bioformats2raw
+RUN git clone -b nested --depth=1 https://github.com/joshmoore/raw2ometiff /tmp/raw2ometiff
+RUN git clone -b nested --depth=1 https://github.com/joshmoore/jzarr /tmp/jzarr
 RUN cd /tmp/jzarr \
  && echo "bloscJnaLibraryPath=/opt/conda/lib" > maven.properties \
  && mvn install -DskipTests

docker-compose.yml

@@ -36,6 +36,8 @@ services:
       - HOST
       - MINIO_ACCESS_KEY=${AWS_ACCESS_KEY_ID}
       - MINIO_SECRET_KEY=${AWS_SECRET_ACCESS_KEY}
+      - AWS_ACCESS_KEY_ID
+      - AWS_SECRET_ACCESS_KEY
 
   benchmark:
     build: benchmark

upload.sh

@@ -11,13 +11,29 @@ set -e
 set -u
 set -x
 
-# Avoiding nginx for upload
-mc config host add benchmark http://minio1:9000 ${MINIO_ACCESS_KEY} ${MINIO_SECRET_KEY}
-mc mb -p benchmark/${BUCKET}
-mc policy set public benchmark/${BUCKET}
+if [ $# -eq 0 ]
+then
 
-time mc mirror --overwrite  \
+    # Avoiding nginx for upload
+    mc config host add benchmark http://minio1:9000 ${MINIO_ACCESS_KEY} ${MINIO_SECRET_KEY}
+    mc mb -p benchmark/${BUCKET}
+    mc policy set public benchmark/${BUCKET}
+
+    time mc mirror --overwrite  \
 	--exclude "out/*" \
 	--exclude "*.csv" \
 	--exclude ".*bfmemo"  \
 	${DIR} benchmark/${BUCKET}/${NAME}/
+elif [ "$1" = "tiff" ];
+then
+	time aws s3 cp ${DIR}/*.tiff s3://${BUCKET}/${NAME}/
+elif [ "$1" = "ims" ];
+then
+	time aws s3 cp ${DIR}/*.ims s3://${BUCKET}/${NAME}/
+elif [ "$1" = "zarr" ];
+then
+	time aws s3 cp --recursive ${DIR}/*.zarr/ s3://${BUCKET}/${NAME}/IMS_XY-${NAME}.ome.zarr/
+else
+    echo unknown upload: $1
+    exit 1
+fi