Minor changes on way to 1.0 actual release

umccr · Sep 18, 2024 · 7bb94fb · 7bb94fb
1 parent 66fc86f
commit 7bb94fb
Show file tree

Hide file tree

Showing 6 changed files with 40 additions and 24 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -19,13 +19,15 @@ jobs:
 
       - run: pnpm install
 
-      - name: Setup Rclone
-        uses: animmouse/setup-rclone@v1
+      - run: pnpm run build
 
-      - run: ./test-go-direct.sh
-        working-directory: packages/aws-copy-out-sharer/docker/rclone-batch-docker-image
+#      - name: Setup Rclone
+#        uses: animmouse/setup-rclone@v1
 
-      - run: ./test-docker-direct.sh
-        working-directory: packages/aws-copy-out-sharer/docker/rclone-batch-docker-image
+#      - run: ./test-go-direct.sh
+#        working-directory: packages/aws-copy-out-sharer/docker/rclone-batch-docker-image
+
+#      - run: ./test-docker-direct.sh
+#        working-directory: packages/aws-copy-out-sharer/docker/rclone-batch-docker-image
 #      - run: pnpm run build
 #      - run: pnpm run package
diff --git a/.github/workflows/release-package.yml b/.github/workflows/release-package.yml
@@ -23,7 +23,7 @@ jobs:
       # first set the versions of our package to the release version
 
       - run: pnpm version ${{github.ref_name}}
-        working-directory: packages/aws-copy-out-sharer
+        working-directory: packages/steps-s3-copy
 
       # build and package
 
@@ -34,7 +34,7 @@ jobs:
       # if all the builds have completed - we can publish the package
 
       - run: pnpm exec publib-npm
-        working-directory: packages/aws-copy-out-sharer
+        working-directory: packages/steps-s3-copy
         env:
           NPM_TOKEN: ${{secrets.NPM_PUBLISH_TOKEN}}
           NPM_ACCESS_LEVEL: public
diff --git a/dev-testsuite/test-2.mts b/dev-testsuite/test-2.mts
@@ -54,35 +54,36 @@ export async function test2(
     uniqueTestId,
   );
 
-  const LARGE_SIZE = 256 * 1024;
+  const REAL_LARGE_SIZE = 256 * 1024 * 1024;
+  const FAKE_LARGE_SIZE = 256 * 1024;
   const SMALL_SIZE = 1024;
 
   // a complex folder structure that is realistic
   const sourceObjects: Record<string, TestObjectParams> = {
     ["production/analysis_data/SBJ12345/wgs_tumor_normal/2024082738bf62ad/L2401304_L2401303_dragen_somatic/MDX654321_normal.bam"]:
-      { sizeInBytes: LARGE_SIZE },
+      { sizeInBytes: FAKE_LARGE_SIZE },
     ["production/analysis_data/SBJ12345/wgs_tumor_normal/2024082738bf62ad/L2401304_L2401303_dragen_somatic/MDX654326_tumor.bam"]:
-      { sizeInBytes: LARGE_SIZE },
+      { sizeInBytes: FAKE_LARGE_SIZE },
     ["production/analysis_data/SBJ12345/wgs_tumor_normal/2024082738bf62ad/L2401304_L2401303_dragen_somatic/MDX654321_normal.bam.bai"]:
       { sizeInBytes: SMALL_SIZE },
     ["production/analysis_data/SBJ12345/wgs_tumor_normal/2024082738bf62ad/L2401304_L2401303_dragen_somatic/MDX654326_tumor.bam.bai"]:
       { sizeInBytes: SMALL_SIZE },
     ["production/primary_data/240823_A98765_4321_ABCDEFGHIJ/20240827933c0a00/WGS_TsqNano/MDX654321_L2401303_S4_L002_R1_001.fastq.gz"]:
-      { sizeInBytes: LARGE_SIZE },
+      { sizeInBytes: REAL_LARGE_SIZE },
     ["production/primary_data/240823_A98765_4321_ABCDEFGHIJ/20240827933c0a00/WGS_TsqNano/MDX654321_L2401303_S4_L002_R2_001.fastq.gz"]:
-      { sizeInBytes: LARGE_SIZE },
+      { sizeInBytes: REAL_LARGE_SIZE },
     ["production/primary_data/240823_A98765_4321_ABCDEFGHIJ/20240827933c0a00/WGS_TsqNano/MDX654321_L2401303_S4_L003_R1_001.fastq.gz"]:
-      { sizeInBytes: LARGE_SIZE },
+      { sizeInBytes: FAKE_LARGE_SIZE },
     ["production/primary_data/240823_A98765_4321_ABCDEFGHIJ/20240827933c0a00/WGS_TsqNano/MDX654321_L2401303_S4_L003_R2_001.fastq.gz"]:
-      { sizeInBytes: LARGE_SIZE },
+      { sizeInBytes: FAKE_LARGE_SIZE },
     ["production/primary_data/240823_A98765_4321_ABCDEFGHIJ/20240827933c0a00/WGS_TsqNano/MDX654326_L2401304_S5_L002_R1_001.fastq.gz"]:
-      { sizeInBytes: LARGE_SIZE },
+      { sizeInBytes: FAKE_LARGE_SIZE },
     ["production/primary_data/240823_A98765_4321_ABCDEFGHIJ/20240827933c0a00/WGS_TsqNano/MDX654326_L2401304_S5_L002_R2_001.fastq.gz"]:
-      { sizeInBytes: LARGE_SIZE },
+      { sizeInBytes: FAKE_LARGE_SIZE },
     ["production/primary_data/240823_A98765_4321_ABCDEFGHIJ/20240827933c0a00/WGS_TsqNano/MDX654326_L2401304_S5_L003_R1_001.fastq.gz"]:
-      { sizeInBytes: LARGE_SIZE },
+      { sizeInBytes: FAKE_LARGE_SIZE },
     ["production/primary_data/240823_A98765_4321_ABCDEFGHIJ/20240827933c0a00/WGS_TsqNano/MDX654326_L2401304_S5_L003_R2_001.fastq.gz"]:
-      { sizeInBytes: LARGE_SIZE },
+      { sizeInBytes: FAKE_LARGE_SIZE },
     ["production/analysis_data/SBJ12345/umccrise/2024082790265f5d/L2401304__L2401303/SBJ12345__MDX654326/SBJ12345__MDX654326-normal.cpsr.html"]:
       { sizeInBytes: SMALL_SIZE },
     ["production/analysis_data/SBJ12345/umccrise/2024082790265f5d/L2401304__L2401303/SBJ12345__MDX654326/SBJ12345__MDX654326_cancer_report.html"]:

diff --git a/dev/example-copy-manifest-hard.csv b/dev/example-copy-manifest-hard.csv
@@ -0,0 +1,14 @@
+umccr-10f-data-dev,"ASHKENAZIM/HG002-HG003-HG004.joint.filter.bcf"
+umccr-10f-data-dev,"ASHKENAZIM/HG002-HG003-HG004.joint.filter.bcf.csi"
+umccr-10f-data-dev,"ASHKENAZIM/HG002-HG003-HG004.joint.filter.vcf"
+umccr-10f-data-dev,"ASHKENAZIM/HG002-HG003-HG004.joint.filter.vcf.gz"
+umccr-10f-data-dev,"ASHKENAZIM/HG002-HG003-HG004.joint.filter.vcf.gz.csi"
+umccr-10f-data-dev,"ASHKENAZIM/HG002-HG003-HG004.joint.filter.vcf.gz.tbi"
+umccr-10f-data-dev,AFILETHATDOESNOTEXIST.txt
+umccr-10f-data-dev,ASHKENAZIM/HG002.bam
+umccr-10f-data-dev,ASHKENAZIM/HG002.bam.bai
+umccr-10f-data-dev,ASHKENAZIM/HG003.bam
+umccr-10f-data-dev,ASHKENAZIM/HG003.bam.bai
+umccr-10f-data-dev,ASHKENAZIM/HG004.bam
+umccr-10f-data-dev,ASHKENAZIM/HG004.bam.bai
+umccr-10f-data-dev,STRUCTURED
diff --git a/dev/example-copy-manifest.csv b/dev/example-copy-manifest.csv
@@ -4,10 +4,9 @@ umccr-10f-data-dev,"ASHKENAZIM/HG002-HG003-HG004.joint.filter.vcf"
 umccr-10f-data-dev,"ASHKENAZIM/HG002-HG003-HG004.joint.filter.vcf.gz"
 umccr-10f-data-dev,"ASHKENAZIM/HG002-HG003-HG004.joint.filter.vcf.gz.csi"
 umccr-10f-data-dev,"ASHKENAZIM/HG002-HG003-HG004.joint.filter.vcf.gz.tbi"
-umccr-10f-data-dev,AFILETHATDOESNOTEXIST.txt
 umccr-10f-data-dev,ASHKENAZIM/HG002.bam
 umccr-10f-data-dev,ASHKENAZIM/HG002.bam.bai
 umccr-10f-data-dev,ASHKENAZIM/HG003.bam
 umccr-10f-data-dev,ASHKENAZIM/HG003.bam.bai
 umccr-10f-data-dev,ASHKENAZIM/HG004.bam
-umccr-10f-data-dev,ASHKENAZIM/HG004.bam.bai
+umccr-10f-data-dev,ASHKENAZIM/HG004.bam.bai
diff --git a/packages/steps-s3-copy/package.json b/packages/steps-s3-copy/package.json
@@ -1,12 +1,12 @@
 {
   "name": "@umccr/steps-s3-copy",
   "version": "0.0.0",
-  "description": "CDK construct for an AWS Steps machine that copies objects",
+  "description": "CDK construct for an AWS Steps machine that copies S3 objects",
   "homepage": "https://github.com/umccr/steps-s3-copy",
   "license": "MIT",
   "author": {
-    "name": "Andrew Patterson",
-    "email": "[email protected]"
+    "name": "University of Melbourne Centre for Cancer Research",
+    "organization": true
   },
   "repository": {
     "type": "git",