From 032750b99e2b0f6c826610dbf0c64b247865c5b6 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 28 May 2025 16:15:24 +0100 Subject: [PATCH 01/11] Updated archive logic Removed git-archive.sh Use tar instead of git archive as done elsewhere. Updated evergreen copy pasta tgz file name Ensure all task groups have their setup and tear down config in the same place JAVA-5886 --- .evergreen/.evg.yml | 38 +++++++++++++++++++++++--------------- .evergreen/git-archive.sh | 20 -------------------- 2 files changed, 23 insertions(+), 35 deletions(-) delete mode 100755 .evergreen/git-archive.sh diff --git a/.evergreen/.evg.yml b/.evergreen/.evg.yml index c20533906e..ed41dd12c8 100644 --- a/.evergreen/.evg.yml +++ b/.evergreen/.evg.yml @@ -830,13 +830,15 @@ functions: type: test params: shell: bash + working-directory: "src" include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] script: |- set -o errexit ${PREPARE_SHELL} export K8S_VARIANT=${VARIANT} - export K8S_DRIVERS_TAR_FILE=$(./.evergreen/git-archive.sh) + export K8S_DRIVERS_TAR_FILE="/tmp/mongo-java-driver.tgz" export K8S_TEST_CMD="OIDC_ENV=k8s VARIANT=${VARIANT} ./.evergreen/run-mongodb-oidc-test.sh" + tar czf $K8S_DRIVERS_TAR_FILE . bash $DRIVERS_TOOLS/.evergreen/auth_oidc/k8s/setup-pod.sh bash $DRIVERS_TOOLS/.evergreen/auth_oidc/k8s/run-self-test.sh source $DRIVERS_TOOLS/.evergreen/auth_oidc/k8s/secrets-export.sh @@ -929,29 +931,33 @@ tasks: commands: - command: shell.exec params: - shell: bash + shell: "bash" + working_dir: "src" env: JAVA_HOME: ${JAVA_HOME} script: |- set -o errexit ${PREPARE_SHELL} - export AZUREOIDC_DRIVERS_TAR_FILE=$(./.evergreen/git-archive.sh) + export AZUREOIDC_DRIVERS_TAR_FILE=/tmp/mongo-java-driver.tgz export AZUREOIDC_TEST_CMD="OIDC_ENV=azure ./.evergreen/run-mongodb-oidc-test.sh" + tar czf $AZUREOIDC_DRIVERS_TAR_FILE . bash $DRIVERS_TOOLS/.evergreen/auth_oidc/azure/run-driver-test.sh - name: "oidc-auth-test-gcp" commands: - command: shell.exec params: - shell: bash + shell: "bash" + working_dir: "src" script: |- set -o errexit ${PREPARE_SHELL} - export GCPOIDC_DRIVERS_TAR_FILE=$(./.evergreen/git-archive.sh) + export GCPOIDC_DRIVERS_TAR_FILE=/tmp/mongo-java-driver.tgz # Define the command to run on the VM. # Ensure that we source the environment file created for us, set up any other variables we need, # and then run our test suite on the vm. export GCPOIDC_TEST_CMD="OIDC_ENV=gcp ./.evergreen/run-mongodb-oidc-test.sh" + tar czf $GCPOIDC_DRIVERS_TAR_FILE . bash $DRIVERS_TOOLS/.evergreen/auth_oidc/gcp/run-driver-test.sh - name: "oidc-auth-test-k8s" @@ -971,8 +977,6 @@ tasks: - func: "oidc-auth-test-k8s-func" vars: VARIANT: gke - params: - include_expansions_in_env: [ "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN" ] - name: serverless-test commands: @@ -1706,11 +1710,11 @@ tasks: export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey - tar czf /tmp/mongo-csharp-driver.tgz . - AZUREKMS_SRC=/tmp/mongo-csharp-driver.tgz AZUREKMS_DST="~/" $DRIVERS_TOOLS/.evergreen/csfle/azurekms/copy-file.sh + tar czf /tmp/mongo-java-driver.tgz . + AZUREKMS_SRC=/tmp/mongo-java-driver.tgz AZUREKMS_DST="~/" $DRIVERS_TOOLS/.evergreen/csfle/azurekms/copy-file.sh echo "Copying files ... end" echo "Untarring file ... begin" - AZUREKMS_CMD="tar xf mongo-csharp-driver.tgz" $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh + AZUREKMS_CMD="tar xf mongo-java-driver.tgz" $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh echo "Untarring file ... end" - command: shell.exec type: test @@ -2071,6 +2075,10 @@ task_groups: - oidc-auth-test - name: test-oidc-azure-task-group + setup_group_can_fail_task: true + setup_group_timeout_secs: 1800 + teardown_task_can_fail_task: true + teardown_group_timeout_secs: 1800 setup_group: - func: fetch source - func: prepare resources @@ -2088,12 +2096,14 @@ task_groups: binary: bash args: - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/azure/delete-vm.sh - setup_group_can_fail_task: true - setup_group_timeout_secs: 1800 tasks: - oidc-auth-test-azure - name: test-oidc-gcp-task-group + setup_group_can_fail_task: true + setup_group_timeout_secs: 1800 + teardown_task_can_fail_task: true + teardown_group_timeout_secs: 1800 setup_group: - func: fetch source - func: prepare resources @@ -2112,8 +2122,6 @@ task_groups: binary: bash args: - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/gcp/teardown.sh - setup_group_can_fail_task: true - setup_group_timeout_secs: 1800 tasks: - oidc-auth-test-gcp @@ -2121,7 +2129,7 @@ task_groups: setup_group_can_fail_task: true setup_group_timeout_secs: 1800 teardown_task_can_fail_task: true - teardown_group_timeout_secs: 180 + teardown_group_timeout_secs: 180 # 3 minutes (max allowed time) setup_group: - func: fetch source - func: prepare resources diff --git a/.evergreen/git-archive.sh b/.evergreen/git-archive.sh deleted file mode 100755 index 5c22c9170a..0000000000 --- a/.evergreen/git-archive.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -# Exit the script with error if any of the commands fail -set -o errexit - -# Returns the path to the root archive file which includes all git submodules. - -echo "Creating root archive" -export GIT_ARCHIVE_FILE="/tmp/mongo-java-driver.tar" - -# create root archive -git archive --output $GIT_ARCHIVE_FILE HEAD - -echo "Appending submodule archives" -git submodule status --recursive | awk '{ print $2 }' | xargs tar -rf $GIT_ARCHIVE_FILE - -echo "Appending .git directory to the root archive" -tar -rf $GIT_ARCHIVE_FILE .git - -echo "$GIT_ARCHIVE_FILE" From 6defbce93332980080f288eb7293fc5470aff2d0 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 28 May 2025 17:49:32 +0100 Subject: [PATCH 02/11] Reduce the size of the tar files and allow gradle to handle a missing git repo --- .evergreen/.evg.yml | 11 ++++++----- .../main/kotlin/conventions/git-version.gradle.kts | 6 +++++- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.evergreen/.evg.yml b/.evergreen/.evg.yml index ed41dd12c8..fa665f5924 100644 --- a/.evergreen/.evg.yml +++ b/.evergreen/.evg.yml @@ -838,7 +838,8 @@ functions: export K8S_VARIANT=${VARIANT} export K8S_DRIVERS_TAR_FILE="/tmp/mongo-java-driver.tgz" export K8S_TEST_CMD="OIDC_ENV=k8s VARIANT=${VARIANT} ./.evergreen/run-mongodb-oidc-test.sh" - tar czf $K8S_DRIVERS_TAR_FILE . + tar --exclude="**build/ --exclude-vcs" -czf $K8S_DRIVERS_TAR_FILE . + tar -tvf $K8S_DRIVERS_TAR_FILE bash $DRIVERS_TOOLS/.evergreen/auth_oidc/k8s/setup-pod.sh bash $DRIVERS_TOOLS/.evergreen/auth_oidc/k8s/run-self-test.sh source $DRIVERS_TOOLS/.evergreen/auth_oidc/k8s/secrets-export.sh @@ -940,7 +941,7 @@ tasks: ${PREPARE_SHELL} export AZUREOIDC_DRIVERS_TAR_FILE=/tmp/mongo-java-driver.tgz export AZUREOIDC_TEST_CMD="OIDC_ENV=azure ./.evergreen/run-mongodb-oidc-test.sh" - tar czf $AZUREOIDC_DRIVERS_TAR_FILE . + tar --exclude="**build/ --exclude-vcs" -czf $AZUREOIDC_DRIVERS_TAR_FILE . bash $DRIVERS_TOOLS/.evergreen/auth_oidc/azure/run-driver-test.sh - name: "oidc-auth-test-gcp" @@ -957,7 +958,7 @@ tasks: # Ensure that we source the environment file created for us, set up any other variables we need, # and then run our test suite on the vm. export GCPOIDC_TEST_CMD="OIDC_ENV=gcp ./.evergreen/run-mongodb-oidc-test.sh" - tar czf $GCPOIDC_DRIVERS_TAR_FILE . + tar --exclude="**build/ --exclude-vcs" -czf $GCPOIDC_DRIVERS_TAR_FILE . bash $DRIVERS_TOOLS/.evergreen/auth_oidc/gcp/run-driver-test.sh - name: "oidc-auth-test-k8s" @@ -1678,7 +1679,7 @@ tasks: export GCPKMS_PROJECT=${GCPKMS_PROJECT} export GCPKMS_ZONE=${GCPKMS_ZONE} export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} - tar czf /tmp/mongo-java-driver.tgz . + tar --exclude="**build/ --exclude-vcs -czf /tmp/mongo-java-driver.tgz . GCPKMS_SRC=/tmp/mongo-java-driver.tgz GCPKMS_DST=$GCPKMS_INSTANCENAME: $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/copy-file.sh echo "Copying files ... end" echo "Untarring file ... begin" @@ -1710,7 +1711,7 @@ tasks: export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey - tar czf /tmp/mongo-java-driver.tgz . + tar --exclude="**build/ --exclude-vcs -czf /tmp/mongo-java-driver.tgz . AZUREKMS_SRC=/tmp/mongo-java-driver.tgz AZUREKMS_DST="~/" $DRIVERS_TOOLS/.evergreen/csfle/azurekms/copy-file.sh echo "Copying files ... end" echo "Untarring file ... begin" diff --git a/buildSrc/src/main/kotlin/conventions/git-version.gradle.kts b/buildSrc/src/main/kotlin/conventions/git-version.gradle.kts index 5370f59cea..17d8165116 100644 --- a/buildSrc/src/main/kotlin/conventions/git-version.gradle.kts +++ b/buildSrc/src/main/kotlin/conventions/git-version.gradle.kts @@ -19,9 +19,13 @@ package conventions val gitVersion: Provider = providers - .exec { commandLine("git", "describe", "--tags", "--always", "--dirty") } + .exec { + commandLine("git", "describe", "--tags", "--always", "--dirty") + isIgnoreExitValue = true + } .standardOutput .asText + .orElse("UNKNOWN") .map { it.trim().removePrefix("r") } // Allows access to gitVersion extension to other conventions From a7819a8ab65e059d7ec6b7f4dfa2b489446cbc49 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 28 May 2025 19:02:53 +0100 Subject: [PATCH 03/11] working_dir fix --- .evergreen/.evg.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.evergreen/.evg.yml b/.evergreen/.evg.yml index fa665f5924..a6c4cbdace 100644 --- a/.evergreen/.evg.yml +++ b/.evergreen/.evg.yml @@ -830,7 +830,7 @@ functions: type: test params: shell: bash - working-directory: "src" + working_dir: "src" include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] script: |- set -o errexit @@ -839,7 +839,6 @@ functions: export K8S_DRIVERS_TAR_FILE="/tmp/mongo-java-driver.tgz" export K8S_TEST_CMD="OIDC_ENV=k8s VARIANT=${VARIANT} ./.evergreen/run-mongodb-oidc-test.sh" tar --exclude="**build/ --exclude-vcs" -czf $K8S_DRIVERS_TAR_FILE . - tar -tvf $K8S_DRIVERS_TAR_FILE bash $DRIVERS_TOOLS/.evergreen/auth_oidc/k8s/setup-pod.sh bash $DRIVERS_TOOLS/.evergreen/auth_oidc/k8s/run-self-test.sh source $DRIVERS_TOOLS/.evergreen/auth_oidc/k8s/secrets-export.sh From da121aa56504762921bf562d90b40bd026575b2f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 28 May 2025 19:16:06 +0100 Subject: [PATCH 04/11] fix tar exclude syntax --- .evergreen/.evg.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.evergreen/.evg.yml b/.evergreen/.evg.yml index a6c4cbdace..e59d16932f 100644 --- a/.evergreen/.evg.yml +++ b/.evergreen/.evg.yml @@ -838,7 +838,7 @@ functions: export K8S_VARIANT=${VARIANT} export K8S_DRIVERS_TAR_FILE="/tmp/mongo-java-driver.tgz" export K8S_TEST_CMD="OIDC_ENV=k8s VARIANT=${VARIANT} ./.evergreen/run-mongodb-oidc-test.sh" - tar --exclude="**build/ --exclude-vcs" -czf $K8S_DRIVERS_TAR_FILE . + tar --exclude="**build/" --exclude-vcs -czf $K8S_DRIVERS_TAR_FILE . bash $DRIVERS_TOOLS/.evergreen/auth_oidc/k8s/setup-pod.sh bash $DRIVERS_TOOLS/.evergreen/auth_oidc/k8s/run-self-test.sh source $DRIVERS_TOOLS/.evergreen/auth_oidc/k8s/secrets-export.sh @@ -940,7 +940,7 @@ tasks: ${PREPARE_SHELL} export AZUREOIDC_DRIVERS_TAR_FILE=/tmp/mongo-java-driver.tgz export AZUREOIDC_TEST_CMD="OIDC_ENV=azure ./.evergreen/run-mongodb-oidc-test.sh" - tar --exclude="**build/ --exclude-vcs" -czf $AZUREOIDC_DRIVERS_TAR_FILE . + tar --exclude="**build/" --exclude-vcs -czf $AZUREOIDC_DRIVERS_TAR_FILE . bash $DRIVERS_TOOLS/.evergreen/auth_oidc/azure/run-driver-test.sh - name: "oidc-auth-test-gcp" @@ -957,7 +957,7 @@ tasks: # Ensure that we source the environment file created for us, set up any other variables we need, # and then run our test suite on the vm. export GCPOIDC_TEST_CMD="OIDC_ENV=gcp ./.evergreen/run-mongodb-oidc-test.sh" - tar --exclude="**build/ --exclude-vcs" -czf $GCPOIDC_DRIVERS_TAR_FILE . + tar --exclude="**build/" --exclude-vcs -czf $GCPOIDC_DRIVERS_TAR_FILE . bash $DRIVERS_TOOLS/.evergreen/auth_oidc/gcp/run-driver-test.sh - name: "oidc-auth-test-k8s" @@ -1678,7 +1678,7 @@ tasks: export GCPKMS_PROJECT=${GCPKMS_PROJECT} export GCPKMS_ZONE=${GCPKMS_ZONE} export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} - tar --exclude="**build/ --exclude-vcs -czf /tmp/mongo-java-driver.tgz . + tar --exclude="**build/" --exclude-vcs -czf /tmp/mongo-java-driver.tgz . GCPKMS_SRC=/tmp/mongo-java-driver.tgz GCPKMS_DST=$GCPKMS_INSTANCENAME: $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/copy-file.sh echo "Copying files ... end" echo "Untarring file ... begin" @@ -1710,7 +1710,7 @@ tasks: export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey - tar --exclude="**build/ --exclude-vcs -czf /tmp/mongo-java-driver.tgz . + tar --exclude="**build/" --exclude-vcs -czf /tmp/mongo-java-driver.tgz . AZUREKMS_SRC=/tmp/mongo-java-driver.tgz AZUREKMS_DST="~/" $DRIVERS_TOOLS/.evergreen/csfle/azurekms/copy-file.sh echo "Copying files ... end" echo "Untarring file ... begin" From 8d4243c906a050023c37aaff510084519abf4224 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Wed, 28 May 2025 19:35:59 +0100 Subject: [PATCH 05/11] git-version.gradle.kts - ocd fix --- buildSrc/src/main/kotlin/conventions/git-version.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildSrc/src/main/kotlin/conventions/git-version.gradle.kts b/buildSrc/src/main/kotlin/conventions/git-version.gradle.kts index 17d8165116..9ddfd25cab 100644 --- a/buildSrc/src/main/kotlin/conventions/git-version.gradle.kts +++ b/buildSrc/src/main/kotlin/conventions/git-version.gradle.kts @@ -25,8 +25,8 @@ val gitVersion: Provider = } .standardOutput .asText - .orElse("UNKNOWN") .map { it.trim().removePrefix("r") } + .orElse("UNKNOWN") // Allows access to gitVersion extension to other conventions extensions.add("gitVersion", gitVersion) From e521d911170edec6fb5fe5a06c3b7bc0494d990f Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 29 May 2025 10:03:27 +0100 Subject: [PATCH 06/11] Fix evergreen validation warnings --- .evergreen/.evg.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.evergreen/.evg.yml b/.evergreen/.evg.yml index e59d16932f..0c92b42733 100644 --- a/.evergreen/.evg.yml +++ b/.evergreen/.evg.yml @@ -2078,7 +2078,7 @@ task_groups: setup_group_can_fail_task: true setup_group_timeout_secs: 1800 teardown_task_can_fail_task: true - teardown_group_timeout_secs: 1800 + teardown_group_timeout_secs: 180 # 3 minutes (max allowed time) setup_group: - func: fetch source - func: prepare resources @@ -2103,7 +2103,7 @@ task_groups: setup_group_can_fail_task: true setup_group_timeout_secs: 1800 teardown_task_can_fail_task: true - teardown_group_timeout_secs: 1800 + teardown_group_timeout_secs: 180 # 3 minutes (max allowed time) setup_group: - func: fetch source - func: prepare resources From 9ecdf974661d747e15f46eef1f48bbea682d9da4 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 29 May 2025 12:02:17 +0100 Subject: [PATCH 07/11] DRY up creating archive file --- .evergreen/.evg.yml | 52 ++++++++++++++++++++++++++++----------------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/.evergreen/.evg.yml b/.evergreen/.evg.yml index 0c92b42733..6906bea827 100644 --- a/.evergreen/.evg.yml +++ b/.evergreen/.evg.yml @@ -61,6 +61,8 @@ functions: export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" export UPLOAD_BUCKET="${project}" export PROJECT_DIRECTORY="$(pwd)" + export ARCHIVE_FILE_NAME="mongo-java-driver.tgz" + export ARCHIVE_FILE_PATH="/tmp/${ARCHIVE_FILE_NAME}" cat < expansion.yml CURRENT_VERSION: "$CURRENT_VERSION" @@ -77,11 +79,12 @@ functions: export MONGODB_BINARIES="$MONGODB_BINARIES" export UPLOAD_BUCKET="$UPLOAD_BUCKET" export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" - export TMPDIR="$MONGO_ORCHESTRATION_HOME/db" export PATH="$MONGODB_BINARIES:$PATH" export PROJECT="${project}" - EOT + export ARCHIVE_FILE_NAME="$ARCHIVE_FILE_NAME" + export ARCHIVE_FILE_PATH="$ARCHIVE_FILE_PATH" + EOT # See what we've done cat expansion.yml @@ -266,7 +269,7 @@ functions: . ./activate-authawsvenv.sh python ./lib/aws_assign_instance_profile.py fi - + "run tests": - command: shell.exec type: test @@ -766,6 +769,12 @@ functions: bash ${DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh || true rm -rf $DRIVERS_TOOLS || true + "create archive tar file": + - command: shell.exec + params: + script: | + tar --exclude="**build/" --exclude-vcs -czf "${ARCHIVE_FILE_PATH}" . + "fix absolute paths": - command: shell.exec params: @@ -836,9 +845,8 @@ functions: set -o errexit ${PREPARE_SHELL} export K8S_VARIANT=${VARIANT} - export K8S_DRIVERS_TAR_FILE="/tmp/mongo-java-driver.tgz" + export K8S_DRIVERS_TAR_FILE=$ARCHIVE_FILE_PATH export K8S_TEST_CMD="OIDC_ENV=k8s VARIANT=${VARIANT} ./.evergreen/run-mongodb-oidc-test.sh" - tar --exclude="**build/" --exclude-vcs -czf $K8S_DRIVERS_TAR_FILE . bash $DRIVERS_TOOLS/.evergreen/auth_oidc/k8s/setup-pod.sh bash $DRIVERS_TOOLS/.evergreen/auth_oidc/k8s/run-self-test.sh source $DRIVERS_TOOLS/.evergreen/auth_oidc/k8s/secrets-export.sh @@ -938,7 +946,7 @@ tasks: script: |- set -o errexit ${PREPARE_SHELL} - export AZUREOIDC_DRIVERS_TAR_FILE=/tmp/mongo-java-driver.tgz + export AZUREOIDC_DRIVERS_TAR_FILE=$ARCHIVE_FILE_PATH export AZUREOIDC_TEST_CMD="OIDC_ENV=azure ./.evergreen/run-mongodb-oidc-test.sh" tar --exclude="**build/" --exclude-vcs -czf $AZUREOIDC_DRIVERS_TAR_FILE . bash $DRIVERS_TOOLS/.evergreen/auth_oidc/azure/run-driver-test.sh @@ -952,7 +960,7 @@ tasks: script: |- set -o errexit ${PREPARE_SHELL} - export GCPOIDC_DRIVERS_TAR_FILE=/tmp/mongo-java-driver.tgz + export GCPOIDC_DRIVERS_TAR_FILE=$ARCHIVE_FILE_PATH # Define the command to run on the VM. # Ensure that we source the environment file created for us, set up any other variables we need, # and then run our test suite on the vm. @@ -1678,11 +1686,10 @@ tasks: export GCPKMS_PROJECT=${GCPKMS_PROJECT} export GCPKMS_ZONE=${GCPKMS_ZONE} export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} - tar --exclude="**build/" --exclude-vcs -czf /tmp/mongo-java-driver.tgz . - GCPKMS_SRC=/tmp/mongo-java-driver.tgz GCPKMS_DST=$GCPKMS_INSTANCENAME: $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/copy-file.sh + GCPKMS_SRC=$ARCHIVE_FILE_PATH GCPKMS_DST=$GCPKMS_INSTANCENAME: $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/copy-file.sh echo "Copying files ... end" echo "Untarring file ... begin" - GCPKMS_CMD="tar xf mongo-java-driver.tgz" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh + GCPKMS_CMD="tar xf $ARCHIVE_FILE_NAME" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh echo "Untarring file ... end" - command: shell.exec type: test @@ -1710,11 +1717,11 @@ tasks: export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup} export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey - tar --exclude="**build/" --exclude-vcs -czf /tmp/mongo-java-driver.tgz . - AZUREKMS_SRC=/tmp/mongo-java-driver.tgz AZUREKMS_DST="~/" $DRIVERS_TOOLS/.evergreen/csfle/azurekms/copy-file.sh + tar --exclude="**build/" --exclude-vcs -czf $ARCHIVE_FILE_PATH . + AZUREKMS_SRC=$ARCHIVE_FILE_PATH AZUREKMS_DST="~/" $DRIVERS_TOOLS/.evergreen/csfle/azurekms/copy-file.sh echo "Copying files ... end" echo "Untarring file ... begin" - AZUREKMS_CMD="tar xf mongo-java-driver.tgz" $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh + AZUREKMS_CMD="tar xf $ARCHIVE_FILE_NAME" $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh echo "Untarring file ... end" - command: shell.exec type: test @@ -1975,6 +1982,7 @@ task_groups: - func: fetch source - func: prepare resources - func: fix absolute paths + - func: create archive tar file - command: shell.exec params: shell: "bash" @@ -2011,6 +2019,7 @@ task_groups: - func: fetch source - func: prepare resources - func: fix absolute paths + - func: create archive tar file - command: shell.exec params: shell: "bash" @@ -2050,6 +2059,10 @@ task_groups: - testazurekms-task - name: test-oidc-task-group + setup_group_can_fail_task: true + setup_group_timeout_secs: 1800 + teardown_task_can_fail_task: true + teardown_task_timeout_secs: 1800 setup_group: - func: fetch source - func: prepare resources @@ -2069,8 +2082,6 @@ task_groups: binary: bash args: - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/teardown.sh - setup_group_can_fail_task: true - setup_group_timeout_secs: 1800 tasks: - oidc-auth-test @@ -2078,11 +2089,12 @@ task_groups: setup_group_can_fail_task: true setup_group_timeout_secs: 1800 teardown_task_can_fail_task: true - teardown_group_timeout_secs: 180 # 3 minutes (max allowed time) + teardown_task_timeout_secs: 1800 setup_group: - func: fetch source - func: prepare resources - func: fix absolute paths + - func: create archive tar file - command: subprocess.exec params: binary: bash @@ -2103,11 +2115,12 @@ task_groups: setup_group_can_fail_task: true setup_group_timeout_secs: 1800 teardown_task_can_fail_task: true - teardown_group_timeout_secs: 180 # 3 minutes (max allowed time) + teardown_task_timeout_secs: 1800 setup_group: - func: fetch source - func: prepare resources - func: fix absolute paths + - func: create archive tar file - command: subprocess.exec params: binary: bash @@ -2129,11 +2142,12 @@ task_groups: setup_group_can_fail_task: true setup_group_timeout_secs: 1800 teardown_task_can_fail_task: true - teardown_group_timeout_secs: 180 # 3 minutes (max allowed time) + teardown_task_timeout_secs: 1800 setup_group: - func: fetch source - func: prepare resources - func: fix absolute paths + - func: create archive tar file - command: ec2.assume_role params: role_arn: ${aws_test_secrets_role} @@ -2143,7 +2157,7 @@ task_groups: include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] args: - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/k8s/setup.sh - teardown_group: + teardown_task: - command: subprocess.exec params: binary: bash From d69322f1c2798fcb768abde4e7e593c86ad1118b Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 29 May 2025 12:23:49 +0100 Subject: [PATCH 08/11] Fix fetch source --- .evergreen/.evg.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.evergreen/.evg.yml b/.evergreen/.evg.yml index 6906bea827..eec6e70ef2 100644 --- a/.evergreen/.evg.yml +++ b/.evergreen/.evg.yml @@ -71,6 +71,8 @@ functions: MONGODB_BINARIES: "$MONGODB_BINARIES" UPLOAD_BUCKET: "$UPLOAD_BUCKET" PROJECT_DIRECTORY: "$PROJECT_DIRECTORY" + ARCHIVE_FILE_NAME: "$ARCHIVE_FILE_NAME" + ARCHIVE_FILE_PATH: "$ARCHIVE_FILE_PATH" PREPARE_SHELL: | set -o errexit set -o xtrace @@ -84,7 +86,7 @@ functions: export PROJECT="${project}" export ARCHIVE_FILE_NAME="$ARCHIVE_FILE_NAME" export ARCHIVE_FILE_PATH="$ARCHIVE_FILE_PATH" - EOT + EOT # See what we've done cat expansion.yml From 58bbf715f2b81ce0cb4d6841333331ba811c076c Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 29 May 2025 12:37:05 +0100 Subject: [PATCH 09/11] Fix fetch source variable --- .evergreen/.evg.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/.evg.yml b/.evergreen/.evg.yml index eec6e70ef2..79936db39b 100644 --- a/.evergreen/.evg.yml +++ b/.evergreen/.evg.yml @@ -62,7 +62,7 @@ functions: export UPLOAD_BUCKET="${project}" export PROJECT_DIRECTORY="$(pwd)" export ARCHIVE_FILE_NAME="mongo-java-driver.tgz" - export ARCHIVE_FILE_PATH="/tmp/${ARCHIVE_FILE_NAME}" + export ARCHIVE_FILE_PATH="/tmp/$ARCHIVE_FILE_NAME" cat < expansion.yml CURRENT_VERSION: "$CURRENT_VERSION" From a5faa88bfb858e1689e3a8141d2ccb9db3991d05 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 29 May 2025 14:09:54 +0100 Subject: [PATCH 10/11] Add assume_role and ensure params are available --- .evergreen/.evg.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.evergreen/.evg.yml b/.evergreen/.evg.yml index 79936db39b..918e4e1707 100644 --- a/.evergreen/.evg.yml +++ b/.evergreen/.evg.yml @@ -775,7 +775,9 @@ functions: - command: shell.exec params: script: | + echo "Creating archive tar file at ${ARCHIVE_FILE_PATH}" tar --exclude="**build/" --exclude-vcs -czf "${ARCHIVE_FILE_PATH}" . + echo "Created archive tar file at ${ARCHIVE_FILE_PATH}" "fix absolute paths": - command: shell.exec @@ -842,7 +844,7 @@ functions: params: shell: bash working_dir: "src" - include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "ARCHIVE_FILE_PATH"] script: |- set -o errexit ${PREPARE_SHELL} @@ -943,6 +945,7 @@ tasks: params: shell: "bash" working_dir: "src" + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "ARCHIVE_FILE_PATH"] env: JAVA_HOME: ${JAVA_HOME} script: |- @@ -959,6 +962,7 @@ tasks: params: shell: "bash" working_dir: "src" + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "ARCHIVE_FILE_PATH"] script: |- set -o errexit ${PREPARE_SHELL} @@ -2097,9 +2101,13 @@ task_groups: - func: prepare resources - func: fix absolute paths - func: create archive tar file + - command: ec2.assume_role + params: + role_arn: ${aws_test_secrets_role} - command: subprocess.exec params: binary: bash + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] env: AZUREOIDC_VMNAME_PREFIX: "JAVA_DRIVER" args: @@ -2123,9 +2131,13 @@ task_groups: - func: prepare resources - func: fix absolute paths - func: create archive tar file + - command: ec2.assume_role + params: + role_arn: ${aws_test_secrets_role} - command: subprocess.exec params: binary: bash + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] env: GCPOIDC_VMNAME_PREFIX: "JAVA_DRIVER" GCPKMS_MACHINETYPE: "e2-medium" # comparable elapsed time to Azure; default was starved, caused timeouts From ea38c70af05d945ff71023fd690caec3d1ad4de3 Mon Sep 17 00:00:00 2001 From: Ross Lawley Date: Thu, 29 May 2025 14:25:16 +0100 Subject: [PATCH 11/11] Fix tar location --- .evergreen/.evg.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.evergreen/.evg.yml b/.evergreen/.evg.yml index 918e4e1707..66e5b63c2e 100644 --- a/.evergreen/.evg.yml +++ b/.evergreen/.evg.yml @@ -774,6 +774,7 @@ functions: "create archive tar file": - command: shell.exec params: + working_dir: "src" script: | echo "Creating archive tar file at ${ARCHIVE_FILE_PATH}" tar --exclude="**build/" --exclude-vcs -czf "${ARCHIVE_FILE_PATH}" .