diff --git a/.env-2D b/.env-2D new file mode 100644 index 0000000..7ead99f --- /dev/null +++ b/.env-2D @@ -0,0 +1,24 @@ +AWS_ACCESS_KEY_ID=minioadmin +AWS_SECRET_ACCESS_KEY=minioadmin +BUCKET=ngff-latency-benchmark +HOST=nginx +HOST=10.0.1.229 + +XY=65536 +Z=1 +C=8 +T=1 +XC=256 +ZC=1 +ROOT=/var/lib/docker +ROOT=/uod/idr-scratch/ngff-latency-benchmark +ROOT=/uod/idr/scratch/idr1-slot2/ngff-latency-benchmark +ROOT=/home/ubuntu/ngff-latency-benchmark + +NAME=${XY}-Z-${Z}-T-${T}-C-${C}-XYC-${XC}-ZC-${ZC} +DIR=${ROOT}/${NAME} +BASE=IMS_XY-${NAME} +ROUNDS=100 +TEST_REPEATS=1 + +#S3ARGS={"anon": true, "client_kwargs": {"endpoint_url": "http://${HOST}:9000"}} diff --git a/.env-3D b/.env-3D new file mode 100644 index 0000000..161124e --- /dev/null +++ b/.env-3D @@ -0,0 +1,24 @@ +AWS_ACCESS_KEY_ID=minioadmin +AWS_SECRET_ACCESS_KEY=minioadmin +BUCKET=ngff-latency-benchmark +HOST=nginx +HOST=10.0.1.22 + +XY=1024 +Z=1024 +C=1 +T=100 +XC=32 +ZC=32 +ROOT=/var/lib/docker +ROOT=/uod/idr-scratch/ngff-latency-benchmark +ROOT=/uod/idr/scratch/idr1-slot2/ngff-latency-benchmark +ROOT=/home/ubuntu/ngff-latency-benchmark + +NAME=${XY}-Z-${Z}-T-${T}-C-${C}-XYC-${XC}-ZC-${ZC} +DIR=${ROOT}/${NAME} +BASE=IMS_XY-${NAME} +ROUNDS=100 +TEST_REPEATS=1 + +#S3ARGS={"anon": true, "client_kwargs": {"endpoint_url": "http://${HOST}:9000"}} diff --git a/convert/Dockerfile b/convert/Dockerfile index 5a54b35..6a4703d 100644 --- a/convert/Dockerfile +++ b/convert/Dockerfile @@ -5,9 +5,9 @@ RUN conda install -c ome bioformats2raw raw2ometiff # Temporary # RUN conda install -y -c anaconda -c conda-forge maven git -RUN git clone -b nested-re-re --depth=1 git://github.com/joshmoore/bioformats2raw /tmp/bioformats2raw -RUN git clone -b nested --depth=1 git://github.com/joshmoore/raw2ometiff /tmp/raw2ometiff -RUN git clone -b nested --depth=1 git://github.com/joshmoore/jzarr /tmp/jzarr +RUN git clone -b nested-re-re --depth=1 https://github.com/joshmoore/bioformats2raw /tmp/bioformats2raw +RUN git clone -b nested --depth=1 https://github.com/joshmoore/raw2ometiff /tmp/raw2ometiff +RUN git clone -b nested --depth=1 https://github.com/joshmoore/jzarr /tmp/jzarr RUN cd /tmp/jzarr \ && echo "bloscJnaLibraryPath=/opt/conda/lib" > maven.properties \ && mvn install -DskipTests diff --git a/docker-compose.yml b/docker-compose.yml index fb3039d..00eaf93 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,6 +36,8 @@ services: - HOST - MINIO_ACCESS_KEY=${AWS_ACCESS_KEY_ID} - MINIO_SECRET_KEY=${AWS_SECRET_ACCESS_KEY} + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY benchmark: build: benchmark diff --git a/upload.sh b/upload.sh index e21dce2..10cfdd3 100755 --- a/upload.sh +++ b/upload.sh @@ -11,13 +11,29 @@ set -e set -u set -x -# Avoiding nginx for upload -mc config host add benchmark http://minio1:9000 ${MINIO_ACCESS_KEY} ${MINIO_SECRET_KEY} -mc mb -p benchmark/${BUCKET} -mc policy set public benchmark/${BUCKET} +if [ $# -eq 0 ] +then -time mc mirror --overwrite \ + # Avoiding nginx for upload + mc config host add benchmark http://minio1:9000 ${MINIO_ACCESS_KEY} ${MINIO_SECRET_KEY} + mc mb -p benchmark/${BUCKET} + mc policy set public benchmark/${BUCKET} + + time mc mirror --overwrite \ --exclude "out/*" \ --exclude "*.csv" \ --exclude ".*bfmemo" \ ${DIR} benchmark/${BUCKET}/${NAME}/ +elif [ "$1" = "tiff" ]; +then + time aws s3 cp ${DIR}/*.tiff s3://${BUCKET}/${NAME}/ +elif [ "$1" = "ims" ]; +then + time aws s3 cp ${DIR}/*.ims s3://${BUCKET}/${NAME}/ +elif [ "$1" = "zarr" ]; +then + time aws s3 cp --recursive ${DIR}/*.zarr/ s3://${BUCKET}/${NAME}/IMS_XY-${NAME}.ome.zarr/ +else + echo unknown upload: $1 + exit 1 +fi