Merge pull request #7 from umccr/feature/bump-rclone-version

Feature/bump rclone version
umccr · Nov 15, 2024 · b836792 · b836792
2 parents 129c8f5 + c831dfb
commit b836792
Show file tree

Hide file tree

Showing 11 changed files with 62 additions and 98 deletions.
diff --git a/dev-testsuite/README.md b/dev-testsuite/README.md
@@ -0,0 +1,11 @@
+# Test suite
+
+WIP!!!
+
+Trying to create a test suite that produces a variety of objects in the test buckets
+and then exhibits copies between them.
+
+Is stuck half way between a useful test suite and some random experiments
+on S3 multipart uploads - so apologies for that.
+
+Not really ready for running out of the box.
diff --git a/dev-testsuite/test-1.mts b/dev-testsuite/test-1.mts
@@ -24,7 +24,6 @@ type TestObjectParams = {
 
 const MiB = 1024 * 1024;
 
-
 /**
  * Execute a test on the given state machine, copying newly created objects from
  * sourceBucket to destinationBucket, and using workingBucket for working artifacts.
@@ -60,7 +59,7 @@ export async function test1(
   );
 
   // 3 files in standard storage (no thawing)
-/*  const sourceObjects: Record<string, TestObjectParams> = {
+  const sourceObjects: Record<string, TestObjectParams> = {
     [`1.bin`]: {
       sizeInBytes: 256 * 124,
     },
@@ -81,21 +80,21 @@ export async function test1(
       partSizeInBytes: 5 * 1024 * 1024,
       checksumAlgorithm: "SHA1",
     },
-  }; */
+  };
 
-  const sourceObjects: Record<string, TestObjectParams> = {
+  /*const sourceObjects: Record<string, TestObjectParams> = {
     //[`many-parts.bin`]: {
     //  // 1010 parts (above the 1000 part boundary) of 5 MiB and then some "extra" to test that edge case as well
     //  sizeInBytes: (5 * MiB * 1010) + 4111,
     //  partSizeInBytes: 5 * MiB,
     //},
-    [`many-parts-with-crc32.bin`]: {
-      // 1010 parts (above the 1000 part boundary) of 5 MiB and then some "extra" to test that edge case as well
-      sizeInBytes: (5 * MiB * 1010) + 4111,
-      partSizeInBytes: 5 * MiB,
-      checksumAlgorithm: "CRC32"
-    },
-    /*[`uneven-parts.bin`]: {
+    // [`many-parts-with-crc32.bin`]: {
+    //  // 1010 parts (above the 1000 part boundary) of 5 MiB and then some "extra" to test that edge case as well
+    //  sizeInBytes: (5 * MiB * 1010) + 4111,
+    //  partSizeInBytes: 5 * MiB,
+    //  checksumAlgorithm: "CRC32"
+    //},
+    [`uneven-parts.bin`]: {
       sizeInBytes: 45 * MiB,
       partSizeInBytes: 6 * MiB,
     },
@@ -112,7 +111,7 @@ export async function test1(
     [`skip-part-numbers.bin`]: {
       sizeInBytes: 45 * MiB,
       partSizeInBytes: 6 * MiB,
-    }, */
+    },
     // You cannot skip part numbers if using checksums
     // [`skip-part-numbers-with-crc32.bin`]: {
     //  sizeInBytes: 45 * MiB,
@@ -123,7 +122,7 @@ export async function test1(
     //  sizeInBytes: 5000 * MiB,
     //  partSizeInBytes: undefined
     //},
-  };
+  }; */
 
   const testObjects: Record<string, TestObject> = {};
 
@@ -140,8 +139,7 @@ export async function test1(
     );
   }
 
-  if (uniqueTestId === "")
-    return;
+  if (uniqueTestId === "") return;
 
   await makeObjectDictionaryCsv(workingBucket, testFolderObjectsTsvAbsolute, {
     [sourceBucket]: Object.keys(sourceObjects).map(

diff --git a/dev-testsuite/test.mts b/dev-testsuite/test.mts
@@ -3,8 +3,8 @@ import {
   DescribeStacksCommand,
 } from "@aws-sdk/client-cloudformation";
 import { test1 } from "./test-1.mjs";
-import {test2} from "./test-2.mjs";
-import {randomBytes} from "node:crypto";
+import { test2 } from "./test-2.mjs";
+import { randomBytes } from "node:crypto";
 
 const cloudFormationClient = new CloudFormationClient({});
 
@@ -140,31 +140,40 @@ async function doTest3(stateMachineArn: string) {
         (o) => o.OutputKey === "DestinationBucket",
       );
 
-      if (smOutput && smOutput.OutputValue && sourceOutput && sourceOutput.OutputValue && workingOutput && workingOutput.OutputValue && destinationOutput && destinationOutput.OutputValue) {
+      if (
+        smOutput &&
+        smOutput.OutputValue &&
+        sourceOutput &&
+        sourceOutput.OutputValue &&
+        workingOutput &&
+        workingOutput.OutputValue &&
+        destinationOutput &&
+        destinationOutput.OutputValue
+      ) {
         console.log(`Steps Arn = ${smOutput.OutputValue}`);
 
         const allTestPromises = [
-          /*test1(
+          test1(
             randomBytes(8).toString("hex"),
             smOutput.OutputValue!,
             sourceOutput.OutputValue!,
             workingOutput.OutputValue!,
             destinationOutput.OutputValue!,
-          ),*/
+          ),
           /*test1(
               "",
               smOutput.OutputValue,
               "umccr-10f-data-dev",
               workingOutput.OutputValue,
               destinationOutput.OutputValue,
           ),*/
-          test2(
+          /*test2(
               randomBytes(8).toString("hex"),
               smOutput.OutputValue,
               sourceOutput.OutputValue,
               workingOutput.OutputValue,
               "org-umccr-demo-data-copy-demo" //destinationOutput.OutputValue,
-          ),
+          ),*/
         ];
 
         const allTestResults = await Promise.allSettled(allTestPromises);

diff --git a/dev/EXAMPLE-COPY-README.md b/dev/EXAMPLE-COPY-README.md
diff --git a/dev/README.md b/dev/README.md
@@ -1,3 +1,6 @@
 # Deploy
 
-CDKs for deploying manually to UMCCR and AG.
+CDK for deploying manually to UMCCR dev.
+
+See accompanying dev test suite for some code
+that exercises this as deployed to dev.
diff --git a/dev/dev.ts b/dev/dev.ts
@@ -45,7 +45,10 @@ class StepsS3CopyStack extends Stack {
         },
       ],
       removalPolicy: RemovalPolicy.DESTROY,
-      autoDeleteObjects: true
+      // whilst we don't want to keep any objects and can feel free to delete
+      // these buckets, given the lifecycle rules they will generally be empty anyway
+      // (setting autoDeleteObjects adds extra lambda/roles that clog everything up)
+      autoDeleteObjects: false,
     });
 
     const workingBucket = new Bucket(this, "Working", {
@@ -58,7 +61,10 @@ class StepsS3CopyStack extends Stack {
         },
       ],
       removalPolicy: RemovalPolicy.DESTROY,
-      autoDeleteObjects: true
+      // whilst we don't want to keep any objects and can feel free to delete
+      // these buckets, given the lifecycle rules they will generally be empty anyway
+      // (setting autoDeleteObjects adds extra lambda/roles that clog everything up)
+      autoDeleteObjects: false,
     });
 
     const destinationBucket = new Bucket(this, "Destination", {
@@ -71,7 +77,10 @@ class StepsS3CopyStack extends Stack {
         },
       ],
       removalPolicy: RemovalPolicy.DESTROY,
-      autoDeleteObjects: true
+      // whilst we don't want to keep any objects and can feel free to delete
+      // these buckets, given the lifecycle rules they will generally be empty anyway
+      // (setting autoDeleteObjects adds extra lambda/roles that clog everything up)
+      autoDeleteObjects: false,
     });
 
     const stepsS3Copy = new StepsS3CopyConstruct(this, "StepsS3Copy", {
@@ -80,7 +89,7 @@ class StepsS3CopyStack extends Stack {
       workingBucket: workingBucket.bucketName,
       workingBucketPrefixKey: WORKING_BUCKET_PREFIX,
       aggressiveTimes: true,
-      writerRoleName: "umccr-wehi-data-sharing-role",
+      writerRoleName: "steps-s3-copy-role",
       allowWriteToInstalledAccount: true,
     });
 

diff --git a/dev/example-copy-manifest-ag.csv b/dev/example-copy-manifest-ag.csv
diff --git a/dev/example-copy-manifest-hard.csv b/dev/example-copy-manifest-hard.csv
diff --git a/dev/example-copy-manifest.csv b/dev/example-copy-manifest.csv
diff --git a/dev/package.json b/dev/package.json
@@ -2,11 +2,10 @@
   "name": "manual",
   "private": true,
   "version": "0.0.0",
-  "description": "Manual CDK deployment for development",
+  "description": "Manual CDK deployment for development at UMCCR",
   "scripts": {
     "deploy": "pnpm -w run build && cdk deploy",
-    "destroy": "pnpm -w run build && cdk destroy",
-    "test": "ts-node --prefer-ts-exts tests/test.ts StepsS3Copy"
+    "destroy": "pnpm -w run build && cdk destroy"
   },
   "dependencies": {
     "@aws-sdk/client-cloudformation": "3.629.0",

diff --git a/packages/steps-s3-copy/docker/rclone-batch-docker-image/Dockerfile b/packages/steps-s3-copy/docker/rclone-batch-docker-image/Dockerfile
@@ -1,4 +1,4 @@
-FROM public.ecr.aws/docker/library/golang:1.22.6-alpine as builder
+FROM public.ecr.aws/docker/library/golang:1.22.6-alpine AS builder
 
 # git is needed for go mod download to work
 RUN apk add --no-cache git
@@ -16,7 +16,7 @@ ADD *.go ./
 RUN env GOOS=linux CGO_ENABLED=0 GOARCH=amd64 go build rclone-batch.go
 
 
-FROM rclone/rclone:1.67.0
+FROM rclone/rclone:1.68.1
 
 # if we want signals to propogate through to the app we need a lightweight initd
 RUN apk add --no-cache tini
@@ -26,6 +26,6 @@ WORKDIR /data
 
 COPY --from=builder /build/rclone-batch ./
 
-ENV RB_RCLONE_BINARY "/usr/local/bin/rclone"
+ENV RB_RCLONE_BINARY="/usr/local/bin/rclone"
 
 ENTRYPOINT ["/sbin/tini", "-vv", "--", "/data/rclone-batch"]