diff --git a/.github/jsonnet/GIT_VERSION b/.github/jsonnet/GIT_VERSION index 6ee27ed..f0a00a6 100644 --- a/.github/jsonnet/GIT_VERSION +++ b/.github/jsonnet/GIT_VERSION @@ -1 +1 @@ -545ab086d65446b210d470713a69ba3beecf28e9 +5985c99fa9e401a6a47e10cd49e141524cf63d4d diff --git a/.github/jsonnet/cache.jsonnet b/.github/jsonnet/cache.jsonnet index 9bd425f..20860b3 100644 --- a/.github/jsonnet/cache.jsonnet +++ b/.github/jsonnet/cache.jsonnet @@ -1,12 +1,14 @@ local base = import 'base.jsonnet'; +local images = import 'images.jsonnet'; +local misc = import 'misc.jsonnet'; { /** * Fetch a cache from the cache server. - * + * * This is a generic function that can be used to fetch any cache. It is advised to wrap this function * in a more specific function that fetches a specific cache, setting the cacheName and folders parameters. - * + * * To be paired with the uploadCache function. * * @param {string} cacheName - The name of the cache to fetch. The name of the repository is usually a good option. @@ -82,10 +84,10 @@ local base = import 'base.jsonnet'; /** * Uploads a cache to the cache server. - * + * * This is a generic function that can be used to upload any cache. It is advised to wrap this function * in a more specific function that uploads a specific cache, setting the cacheName and folders parameters. - * + * * To be paired with the fetchCache function. * * @param {string} cacheName - The name of the cache to upload. The name of the repository is usually a good option. @@ -141,4 +143,85 @@ local base = import 'base.jsonnet'; 'echo "Cache removed"\n', ifClause=ifClause, ), + +/** + * Daily (weekday) backup of the full repository (working tree + .git) to GCS as a zstd tar archive, + * then refreshes a 7-day signed HTTPS URL into the GIT_HTTPS_ARCHIVE_MIRROR repo secret. + * + * @param {string} [cron='0 1 * * 1-5'] - Schedule (UTC). Default 01:00 on weekdays. + * @param {string} [bucketPath='gs://gynzy-internal-files/git-mirror'] - Destination prefix. + * @returns {workflows} - GitHub Actions pipeline that mirrors the repository to GCS. + */ + updateGitCacheCron( + cron='0 1 * * 1-5', + ):: + local secret = self.secret; + local destUrl = 'gs://gynzy-internal-files/git-mirror/${GITHUB_REPOSITORY}.tar.zst'; + base.pipeline( + 'git-mirror-backup', + [ + base.ghJob( + 'git-mirror-backup', + image=images.cloud_sdk_image, + useCredentials=true, + timeoutMinutes=60, + steps=[ + // blobless=false: a backup must contain ALL git objects, not lazily-fetched blobs. + misc.checkout(fullClone=true, blobless=false, cloneTimeout=30, skipSeed=true), + base.step( + 'authenticate gcloud', + ||| + set -euo pipefail + # Write the key OUTSIDE the workspace so it never lands in the archive. + printf '%s' "$SERVICE_JSON" > "$RUNNER_TEMP/gce.json" + gcloud auth activate-service-account --key-file="$RUNNER_TEMP/gce.json" + |||, + shell='bash', + env={ SERVICE_JSON: misc.secret('SERVICE_JSON') }, + ), + base.step( + 'create and upload archive', + ||| + set -euo pipefail + # checkStat=minimal makes git compare only mtime+size (which tar preserves), + # not ctime/inode. Consumers that seed from this archive then see a clean tree + # and only write the real diff during checkout instead of rewriting every file. + git -C "$GITHUB_WORKSPACE" config --local core.checkStat minimal + + DEST="%s" + + # Upload to a temp object first, then atomically move into place so a partial + # upload can never sit at the final destination. + tar -C "$GITHUB_WORKSPACE" -c . | zstdmt -10 | gcloud storage cp - "${DEST}.tmp" + gcloud storage mv "${DEST}.tmp" "$DEST" + ||| % destUrl, + shell='bash', + ), + base.step( + 'refresh signed-url secret', + ||| + set -euo pipefail + DEST="%s" + URL="$(gcloud storage sign-url "$DEST" \ + --private-key-file="$RUNNER_TEMP/gce.json" --http-verb=GET --duration=7d \ + --format='value(signed_url)')" + echo "::add-mask::$URL" + gh secret set "GIT_HTTPS_ARCHIVE_MIRROR" --repo "$GITHUB_REPOSITORY" --body "$URL" + ||| % destUrl, + shell='bash', + env={ + GH_TOKEN: misc.secret('VIRKO_GITHUB_TOKEN'), + }, + ), + base.step( + 'remove credentials', + 'rm -f "$RUNNER_TEMP/gce.json"', + ifClause='${{ always() }}', + ), + ], + ), + ], + event={ schedule: [{ cron: cron }], workflow_dispatch: {} }, + permissions={ contents: 'read' }, + ), } diff --git a/.github/jsonnet/deployment.jsonnet b/.github/jsonnet/deployment.jsonnet index 61e066d..35cee6b 100644 --- a/.github/jsonnet/deployment.jsonnet +++ b/.github/jsonnet/deployment.jsonnet @@ -160,10 +160,11 @@ local notifications = import 'notifications.jsonnet'; * Generate a GitHub ifClause for the provided deployment targets. * * @param {array} targets - Array of deployment target environment names + * @param {boolean} [virkoOnly=true] - If true, also require the deployment event to be created by 'gynzy-virko', preventing manually created deployment events from triggering the job * @returns {string} - GitHub Actions conditional expression that matches any of the provided targets */ - deploymentTargets(targets):: - '${{ ' + std.join(' || ', std.map(function(target) "github.event.deployment.environment == '" + target + "'", targets)) + ' }}', + deploymentTargets(targets, virkoOnly=true):: + '${{ github.event_name == \'deployment\' && (' + std.join(' || ', std.map(function(target) "github.event.deployment.environment == '" + target + "'", targets)) + ')' + (if virkoOnly then " && github.event.deployment.creator.login == 'gynzy-virko'" else '') + ' }}', /** * Creates a step to update deployment status (success/failure) based on the result from the current job diff --git a/.github/jsonnet/helm.jsonnet b/.github/jsonnet/helm.jsonnet index 40befec..e387cb4 100644 --- a/.github/jsonnet/helm.jsonnet +++ b/.github/jsonnet/helm.jsonnet @@ -1,6 +1,7 @@ local base = import 'base.jsonnet'; local clusters = import 'clusters.jsonnet'; local databases = import 'databases.jsonnet'; +local deployment = import 'deployment.jsonnet'; local images = import 'images.jsonnet'; local misc = import 'misc.jsonnet'; local services = import 'services.jsonnet'; @@ -151,7 +152,7 @@ local services = import 'services.jsonnet'; base.ghJob( 'deploy-prod', runsOn=runsOn, - ifClause="${{ github.event.deployment.environment == '" + environment + "' }}", + ifClause=deployment.deploymentTargets([environment]), image=image, useCredentials=useCredentials, steps=[ @@ -234,7 +235,7 @@ local services = import 'services.jsonnet'; base.ghJob( 'deploy-test', runsOn=runsOn, - ifClause="${{ github.event.deployment.environment == 'test' }}", + ifClause=deployment.deploymentTargets(['test']), image=image, useCredentials=useCredentials, steps=[ @@ -515,7 +516,7 @@ local services = import 'services.jsonnet'; runsOn=runsOn, image=image, useCredentials=useCredentials, - ifClause="${{ github.event.deployment.environment == 'canary' }}", + ifClause=deployment.deploymentTargets(['canary']), steps=[ misc.checkout(), self.helmDeployCanary( @@ -597,7 +598,7 @@ local services = import 'services.jsonnet'; base.ghJob( 'kill-canary', runsOn=runsOn, - ifClause="${{ github.event.deployment.environment == 'kill-canary' || github.event.deployment.environment == 'production' }}", + ifClause=deployment.deploymentTargets(['kill-canary', 'production']), image=images.default_job_image, useCredentials=false, steps=[ diff --git a/.github/jsonnet/images.jsonnet b/.github/jsonnet/images.jsonnet index c3ef14e..def35f7 100644 --- a/.github/jsonnet/images.jsonnet +++ b/.github/jsonnet/images.jsonnet @@ -20,6 +20,7 @@ default_mongodb_image: 'europe-docker.pkg.dev/unicorn-985/private-images/docker-images_mongo8-replicated:v1', mongo_job_image: 'europe-docker.pkg.dev/unicorn-985/public-images/docker-images_mongo-cloner-job:v1', default_python_image: 'mirror.gcr.io/python:3.12.1', + cloud_sdk_image: 'europe-docker.pkg.dev/unicorn-985/private-images/docker-images_gcloud:v2', default_pulumi_node_image: 'mirror.gcr.io/node:22', job_poster_image: 'europe-docker.pkg.dev/unicorn-985/public-images/docker-images_job-poster:v2', } diff --git a/.github/jsonnet/misc.jsonnet b/.github/jsonnet/misc.jsonnet index cbdb462..5603f26 100644 --- a/.github/jsonnet/misc.jsonnet +++ b/.github/jsonnet/misc.jsonnet @@ -17,9 +17,11 @@ local images = import 'images.jsonnet'; * @param {boolean} [blobless=null] - Whether to perform a blobless clone (--filter=blob:none); null uses default (false) * @param {number} [retryAttempts=null] - Number of additional checkout attempts on failure; null uses default (0) * @param {number} [cloneTimeout=null] - Timeout for git clone operation in minutes; null uses default (10) + * @param {boolean} [skipSeed=false] - Boolean where the archive based git initialization can be skipped * @returns {steps} - GitHub Actions steps for repository checkout */ - checkout(ifClause=null, fullClone=false, ref=null, preferSshClone=true, includeSubmodules=true, blobless=null, retryAttempts=null, cloneTimeout=null):: + checkout(ifClause=null, fullClone=false, ref=null, preferSshClone=true, includeSubmodules=true, blobless=null, retryAttempts=null, cloneTimeout=null, skipSeed=false):: + local secret = self.secret; local actualBlobless = if blobless == null then false else blobless; local actualRetryAttempts = if retryAttempts == null then 0 else retryAttempts; local actualCloneTimeout = if cloneTimeout == null then 10 else cloneTimeout; @@ -73,6 +75,34 @@ local images = import 'images.jsonnet'; id='check-binaries', ) else []); + // Cache seed: unpack the mirror archive (full working tree + .git) so the checkout + // below only fetches the delta. Skipped at runtime when the mirror secret is empty. + // Any failure leaves a clean workspace and continues normally. + local seedSteps = (if skipSeed then [] else base.step( + 'fetch git-mirror archive', + ||| + seed() { + if ! command -v zstd >/dev/null 2>&1; then + if command -v apk >/dev/null 2>&1; then apk add --no-cache zstd tar wget || return 1; + elif command -v apt >/dev/null 2>&1; then apt update && apt install -y zstd tar wget || return 1; + else return 1; fi + fi + wget -q -O - "$GIT_HTTPS_ARCHIVE_MIRROR" | tar -C "$GITHUB_WORKSPACE" --extract --zstd -f - || return 1 + } + if ! seed; then + echo "git-mirror seed failed; continuing with a clean checkout" + find "$GITHUB_WORKSPACE" -mindepth 1 -delete 2>/dev/null || true + exit 1 + fi + echo "git-mirror seed succeeded" + exit 0 + |||, + env={ GIT_HTTPS_ARCHIVE_MIRROR: secret('GIT_HTTPS_ARCHIVE_MIRROR') }, + ifClause="${{ env.GIT_HTTPS_ARCHIVE_MIRROR != '' }}", + shell='bash', + continueOnError=true, + )); + // strip the ${{ }} from the IfClause so we can inject and add our own if clause local localIfClause = (if ifClause == null then null else std.strReplace(std.strReplace(ifClause, '${{ ', ''), ' }}', '')); local userIfPart = if ifClause == null then '' else '( ' + localIfClause + ' ) && '; @@ -97,6 +127,7 @@ local images = import 'images.jsonnet'; ); if (preferSshClone) then + seedSteps + sshSteps + retrySteps( 'Check out repository code via ssh', @@ -112,6 +143,7 @@ local images = import 'images.jsonnet'; ) + base.step('git safe directory', "command -v git && git config --global --add safe.directory '*' || true") else + seedSteps + self.checkoutWithoutSshMagic(ifClause, fullClone, ref), /** diff --git a/.github/jsonnet/newrelic.jsonnet b/.github/jsonnet/newrelic.jsonnet index cb8d934..7bfc83d 100644 --- a/.github/jsonnet/newrelic.jsonnet +++ b/.github/jsonnet/newrelic.jsonnet @@ -1,4 +1,5 @@ local base = import 'base.jsonnet'; +local deployment = import 'deployment.jsonnet'; local images = import 'images.jsonnet'; local misc = import 'misc.jsonnet'; local yarn = import 'yarn.jsonnet'; @@ -31,7 +32,7 @@ local pnpm = import 'pnpm.jsonnet'; runsOn=runsOn, image=image, useCredentials=useCredentials, - ifClause="${{ github.event.deployment.environment == 'production' }}", + ifClause=deployment.deploymentTargets(['production']), steps= ( if packageManager == 'yarn' then diff --git a/.github/jsonnet/pnpm.jsonnet b/.github/jsonnet/pnpm.jsonnet index bd59adc..fc38a6c 100644 --- a/.github/jsonnet/pnpm.jsonnet +++ b/.github/jsonnet/pnpm.jsonnet @@ -1,6 +1,7 @@ local actions = import 'actions.jsonnet'; local base = import 'base.jsonnet'; local cache = import 'cache.jsonnet'; +local deployment = import 'deployment.jsonnet'; local misc = import 'misc.jsonnet'; local yarn = import 'yarn.jsonnet'; @@ -144,7 +145,7 @@ local yarn = import 'yarn.jsonnet'; runsOn=runsOn, image=image, useCredentials=useCredentials, - ifClause="${{ github.event.deployment.environment == 'production' || github.event.deployment.environment == 'prod' }}", + ifClause=deployment.deploymentTargets(['production']), steps=[ self.checkoutAndPnpm( cacheName=null, // to populate cache we want a clean install diff --git a/.github/jsonnet/pulumi.jsonnet b/.github/jsonnet/pulumi.jsonnet index 419a35e..ecf96c4 100644 --- a/.github/jsonnet/pulumi.jsonnet +++ b/.github/jsonnet/pulumi.jsonnet @@ -1,14 +1,17 @@ local actions = import 'actions.jsonnet'; local base = import 'base.jsonnet'; +local deployment = import 'deployment.jsonnet'; local images = import 'images.jsonnet'; local misc = import 'misc.jsonnet'; local notifications = import 'notifications.jsonnet'; local pnpm = import 'pnpm.jsonnet'; local yarn = import 'yarn.jsonnet'; +local defaultPulumiVersion = '3.248.0'; + // Standard setup steps required for all Pulumi operations // Includes authentication, cloud setup, and tool installation -local pulumiSetupSteps = +local pulumiSetupSteps(pulumiVersion) = base.action( 'auth', uses=actions.gcp_auth_action, @@ -18,7 +21,7 @@ local pulumiSetupSteps = } ) + base.action('setup-gcloud', uses=actions.gcp_setup_gcloud_action) + - base.action('pulumi-cli-setup', actions.pulumi_action) + + base.action('pulumi-cli-setup', actions.pulumi_action, with={'pulumi-version': pulumiVersion}) + base.action('jsonnet-setup', 'kobtea/setup-jsonnet-action@78f57bb20bd6cf4914c27dd44610a7d923455ecf') + // v2 misc.install1Password() + misc.getLockStep(lockName='lock-pulumi', lockTimeout='1200'); @@ -156,6 +159,7 @@ local pulumiDefaultEnvironment(stack) = { * @param {boolean} [blobless=null] - Whether to perform a blobless clone (--filter=blob:none); null uses checkout default * @param {number} [retryAttempts=null] - Number of additional checkout attempts on failure; null uses checkout default * @param {number} [cloneTimeout=null] - Timeout for git clone operation in minutes; null uses checkout default + * @param {string} [pulumiVersion=defaultPulumiVersion] - Pulumi CLI version to install * @returns {jobs} - Complete GitHub Actions job for Pulumi preview */ pulumiPreviewJob( @@ -170,6 +174,7 @@ local pulumiDefaultEnvironment(stack) = { additionalSetupSteps=[], ignoreEngines=false, runsOn=null, + pulumiVersion=defaultPulumiVersion, ):: base.ghJob( 'pulumi-preview-' + stack, @@ -178,7 +183,7 @@ local pulumiDefaultEnvironment(stack) = { useCredentials=false, steps=[ yarn.checkoutAndYarn(ref=gitCloneRef, cacheName=cacheName, fullClone=false, workingDirectory=yarnDir, source=yarnNpmSource, ignoreEngines=ignoreEngines), - pulumiSetupSteps, + pulumiSetupSteps(pulumiVersion), additionalSetupSteps, self.pulumiPreview(stack, pulumiDir=pulumiDir, environmentVariables=environmentVariables), ], @@ -196,6 +201,7 @@ local pulumiDefaultEnvironment(stack) = { * @param {string} [image=images.default_pulumi_node_image] - Container image * @param {object} [environmentVariables={}] - Additional environment variables * @param {array} [additionalSetupSteps=[]] - Extra setup steps + * @param {string} [pulumiVersion=defaultPulumiVersion] - Pulumi CLI version to install * @returns {jobs} - GitHub Actions job for test environment Pulumi preview */ pulumiPreviewTestJob( @@ -208,6 +214,7 @@ local pulumiDefaultEnvironment(stack) = { image=images.default_pulumi_node_image, environmentVariables={}, additionalSetupSteps=[], + pulumiVersion=defaultPulumiVersion, ):: self.pulumiPreviewJob( stack, @@ -219,6 +226,7 @@ local pulumiDefaultEnvironment(stack) = { image=image, environmentVariables=environmentVariables, additionalSetupSteps=additionalSetupSteps, + pulumiVersion=pulumiVersion, ), /** @@ -233,6 +241,7 @@ local pulumiDefaultEnvironment(stack) = { * @param {string} [image=images.default_pulumi_node_image] - Container image * @param {object} [environmentVariables={}] - Additional environment variables * @param {array} [additionalSetupSteps=[]] - Extra setup steps + * @param {string} [pulumiVersion=defaultPulumiVersion] - Pulumi CLI version to install * @returns {jobs} - GitHub Actions job for production Pulumi preview */ pulumiPreviewProdJob( @@ -245,6 +254,7 @@ local pulumiDefaultEnvironment(stack) = { image=images.default_pulumi_node_image, environmentVariables={}, additionalSetupSteps=[], + pulumiVersion=defaultPulumiVersion, ):: self.pulumiPreviewJob( stack, @@ -256,6 +266,7 @@ local pulumiDefaultEnvironment(stack) = { image=image, environmentVariables=environmentVariables, additionalSetupSteps=additionalSetupSteps, + pulumiVersion=pulumiVersion, ), /** @@ -275,6 +286,7 @@ local pulumiDefaultEnvironment(stack) = { * @param {string} [packageManager='yarn'] - Package manager to use ('yarn' or 'pnpm') * @param {array} [pnpmInstallArgs=[]] - Additional arguments for pnpm install * @param {string} [runsOn=null] - GitHub Actions runner to use for the job + * @param {string} [pulumiVersion=defaultPulumiVersion] - Pulumi CLI version to install * @returns {jobs} - GitHub Actions job that previews both test and production stacks */ pulumiPreviewTestAndProdJob( @@ -295,6 +307,7 @@ local pulumiDefaultEnvironment(stack) = { blobless=null, retryAttempts=null, cloneTimeout=null, + pulumiVersion=defaultPulumiVersion, ):: base.ghJob( 'pulumi-preview', @@ -306,7 +319,7 @@ local pulumiDefaultEnvironment(stack) = { if packageManager == 'yarn' then yarn.checkoutAndYarn(ref=gitCloneRef, cacheName=cacheName, fullClone=false, workingDirectory=yarnDir, source=yarnNpmSource, ignoreEngines=ignoreEngines, blobless=blobless, retryAttempts=retryAttempts, cloneTimeout=cloneTimeout) else if packageManager == 'pnpm' then pnpm.checkoutAndPnpm(ref=gitCloneRef, cacheName=cacheName, fullClone=false, workingDirectory=yarnDir, source=yarnNpmSource, pnpmInstallArgs=pnpmInstallArgs, blobless=blobless, retryAttempts=retryAttempts, cloneTimeout=cloneTimeout) ), - pulumiSetupSteps, + pulumiSetupSteps(pulumiVersion), additionalSetupSteps, self.pulumiPreview(testStack, pulumiDir=pulumiDir, environmentVariables=environmentVariables), self.pulumiPreview(productionStack, pulumiDir=pulumiDir, environmentVariables=environmentVariables), @@ -335,6 +348,7 @@ local pulumiDefaultEnvironment(stack) = { * @param {boolean} [blobless=null] - Whether to perform a blobless clone (--filter=blob:none); null uses checkout default * @param {number} [retryAttempts=null] - Number of additional checkout attempts on failure; null uses checkout default * @param {number} [cloneTimeout=null] - Timeout for git clone operation in minutes; null uses checkout default + * @param {string} [pulumiVersion=defaultPulumiVersion] - Pulumi CLI version to install * @returns {jobs} - GitHub Actions job for Pulumi deployment with failure notifications */ pulumiDeployJob( @@ -357,6 +371,7 @@ local pulumiDefaultEnvironment(stack) = { blobless=null, retryAttempts=null, cloneTimeout=null, + pulumiVersion=defaultPulumiVersion, ):: base.ghJob( name=jobName, @@ -369,7 +384,7 @@ local pulumiDefaultEnvironment(stack) = { if packageManager == 'yarn' then yarn.checkoutAndYarn(ref=gitCloneRef, cacheName=cacheName, fullClone=false, workingDirectory=yarnDir, source=yarnNpmSource, ignoreEngines=ignoreEngines, blobless=blobless, retryAttempts=retryAttempts, cloneTimeout=cloneTimeout) else if packageManager == 'pnpm' then pnpm.checkoutAndPnpm(ref=gitCloneRef, cacheName=cacheName, fullClone=false, workingDirectory=yarnDir, source=yarnNpmSource, pnpmInstallArgs=pnpmInstallArgs, blobless=blobless, retryAttempts=retryAttempts, cloneTimeout=cloneTimeout) ), - pulumiSetupSteps, + pulumiSetupSteps(pulumiVersion), additionalSetupSteps, self.pulumiDeploy(stack, pulumiDir=pulumiDir, stepName=jobName, environmentVariables=environmentVariables), if notifyOnFailure then notifications.notifiyDeployFailure(environment=stack) else [], @@ -386,7 +401,7 @@ local pulumiDefaultEnvironment(stack) = { * @param {string} [gitCloneRef='${{ github.sha }}'] - Git reference to checkout * @param {string} [cacheName=null] - Cache key for dependency caching * @param {string} [image=images.default_pulumi_node_image] - Container image - * @param {string} [ifClause="${{ github.event.deployment.environment == 'test' }}"] - Conditional for test deployments + * @param {string} [ifClause=deployment.deploymentTargets(['test'])] - Conditional for test deployments * @param {object} [environmentVariables={}] - Additional environment variables * @param {array} [additionalSetupSteps=[]] - Extra setup steps * @param {boolean} [ignoreEngines=false] - Whether to ignore Node.js engine requirements @@ -395,6 +410,7 @@ local pulumiDefaultEnvironment(stack) = { * @param {boolean} [blobless=null] - Whether to perform a blobless clone (--filter=blob:none); null uses checkout default * @param {number} [retryAttempts=null] - Number of additional checkout attempts on failure; null uses checkout default * @param {number} [cloneTimeout=null] - Timeout for git clone operation in minutes; null uses checkout default + * @param {string} [pulumiVersion=defaultPulumiVersion] - Pulumi CLI version to install * @returns {jobs} - GitHub Actions job for test environment deployment */ pulumiDeployTestJob( @@ -405,7 +421,7 @@ local pulumiDefaultEnvironment(stack) = { gitCloneRef='${{ github.sha }}', cacheName=null, image=images.default_pulumi_node_image, - ifClause="${{ github.event.deployment.environment == 'test' }}", + ifClause=deployment.deploymentTargets(['test']), environmentVariables={}, additionalSetupSteps=[], ignoreEngines=false, @@ -414,6 +430,7 @@ local pulumiDefaultEnvironment(stack) = { blobless=null, retryAttempts=null, cloneTimeout=null, + pulumiVersion=defaultPulumiVersion, ):: self.pulumiDeployJob( stack, @@ -432,6 +449,7 @@ local pulumiDefaultEnvironment(stack) = { blobless=blobless, retryAttempts=retryAttempts, cloneTimeout=cloneTimeout, + pulumiVersion=pulumiVersion, ), /** @@ -444,7 +462,7 @@ local pulumiDefaultEnvironment(stack) = { * @param {string} [gitCloneRef='${{ github.sha }}'] - Git reference to checkout * @param {string} [cacheName=null] - Cache key for dependency caching * @param {string} [image=images.default_pulumi_node_image] - Container image - * @param {string} [ifClause="${{ github.event.deployment.environment == 'prod' || github.event.deployment.environment == 'production' }}"] - Conditional for production deployments + * @param {string} [ifClause=deployment.deploymentTargets(['production'])] - Conditional for production deployments * @param {object} [environmentVariables={}] - Additional environment variables * @param {array} [additionalSetupSteps=[]] - Extra setup steps * @param {boolean} [ignoreEngines=false] - Whether to ignore Node.js engine requirements @@ -453,6 +471,7 @@ local pulumiDefaultEnvironment(stack) = { * @param {boolean} [blobless=null] - Whether to perform a blobless clone (--filter=blob:none); null uses checkout default * @param {number} [retryAttempts=null] - Number of additional checkout attempts on failure; null uses checkout default * @param {number} [cloneTimeout=null] - Timeout for git clone operation in minutes; null uses checkout default + * @param {string} [pulumiVersion=defaultPulumiVersion] - Pulumi CLI version to install * @returns {jobs} - GitHub Actions job for production deployment */ pulumiDeployProdJob( @@ -463,7 +482,7 @@ local pulumiDefaultEnvironment(stack) = { gitCloneRef='${{ github.sha }}', cacheName=null, image=images.default_pulumi_node_image, - ifClause="${{ github.event.deployment.environment == 'prod' || github.event.deployment.environment == 'production' }}", + ifClause=deployment.deploymentTargets(['production']), environmentVariables={}, additionalSetupSteps=[], ignoreEngines=false, @@ -472,6 +491,7 @@ local pulumiDefaultEnvironment(stack) = { blobless=null, retryAttempts=null, cloneTimeout=null, + pulumiVersion=defaultPulumiVersion, ):: self.pulumiDeployJob( stack, @@ -490,6 +510,7 @@ local pulumiDefaultEnvironment(stack) = { blobless=blobless, retryAttempts=retryAttempts, cloneTimeout=cloneTimeout, + pulumiVersion=pulumiVersion, ), /** @@ -513,6 +534,7 @@ local pulumiDefaultEnvironment(stack) = { * @param {string} [packageManager='yarn'] - Package manager to use ('yarn' or 'pnpm') * @param {array} [pnpmInstallArgs=[]] - Additional arguments for pnpm install * @param {string} [runsOn=null] - GitHub Actions runner to use for the job + * @param {string} [pulumiVersion=defaultPulumiVersion] - Pulumi CLI version to install * @returns {jobs} - GitHub Actions job for Pulumi infrastructure destruction */ pulumiDestroyJob( @@ -532,6 +554,7 @@ local pulumiDefaultEnvironment(stack) = { packageManager='yarn', pnpmInstallArgs=[], runsOn=null, + pulumiVersion=defaultPulumiVersion, ):: base.ghJob( name=jobName, @@ -544,7 +567,7 @@ local pulumiDefaultEnvironment(stack) = { if packageManager == 'yarn' then yarn.checkoutAndYarn(ref=gitCloneRef, cacheName=cacheName, fullClone=false, workingDirectory=yarnDir, source=yarnNpmSource, ignoreEngines=ignoreEngines) else if packageManager == 'pnpm' then pnpm.checkoutAndPnpm(ref=gitCloneRef, cacheName=cacheName, fullClone=false, workingDirectory=yarnDir, source=yarnNpmSource, pnpmInstallArgs=pnpmInstallArgs) ), - pulumiSetupSteps, + pulumiSetupSteps(pulumiVersion), additionalSetupSteps, self.pulumiDestroy(stack, pulumiDir=pulumiDir, stepName=jobName, environmentVariables=environmentVariables), if notifyOnFailure then notifications.notifiyDeployFailure(environment=stack) else [], @@ -573,6 +596,7 @@ local pulumiDefaultEnvironment(stack) = { * @param {boolean} [blobless=null] - Whether to perform a blobless clone (--filter=blob:none); null uses checkout default * @param {number} [retryAttempts=null] - Number of additional checkout attempts on failure; null uses checkout default * @param {number} [cloneTimeout=null] - Timeout for git clone operation in minutes; null uses checkout default + * @param {string} [pulumiVersion=defaultPulumiVersion] - Pulumi CLI version to install * @returns {workflows} - Complete set of Pulumi preview and deployment pipelines */ pulumiDefaultPipeline( @@ -592,6 +616,7 @@ local pulumiDefaultEnvironment(stack) = { blobless=null, retryAttempts=null, cloneTimeout=null, + pulumiVersion=defaultPulumiVersion, ):: base.pipeline( 'pulumi-preview', @@ -612,6 +637,7 @@ local pulumiDefaultEnvironment(stack) = { blobless=blobless, retryAttempts=retryAttempts, cloneTimeout=cloneTimeout, + pulumiVersion=pulumiVersion, ), ], ) + @@ -627,12 +653,13 @@ local pulumiDefaultEnvironment(stack) = { image=image, environmentVariables=environmentVariables, additionalSetupSteps=additionalSetupSteps, - ifClause=if deployTestWithProd then "${{ github.event.deployment.environment == 'test' || github.event.deployment.environment == 'prod' || github.event.deployment.environment == 'production' }}" else "${{ github.event.deployment.environment == 'test' }}", + ifClause=if deployTestWithProd then deployment.deploymentTargets(['test', 'production']) else deployment.deploymentTargets(['test']), ignoreEngines=ignoreEngines, runsOn=runsOn, blobless=blobless, retryAttempts=retryAttempts, cloneTimeout=cloneTimeout, + pulumiVersion=pulumiVersion, ), self.pulumiDeployProdJob( pulumiDir=pulumiDir, @@ -649,6 +676,7 @@ local pulumiDefaultEnvironment(stack) = { blobless=blobless, retryAttempts=retryAttempts, cloneTimeout=cloneTimeout, + pulumiVersion=pulumiVersion, ), ], event='deployment', diff --git a/.github/jsonnet/ruby.jsonnet b/.github/jsonnet/ruby.jsonnet index d19f2bf..5ef7b32 100644 --- a/.github/jsonnet/ruby.jsonnet +++ b/.github/jsonnet/ruby.jsonnet @@ -1,6 +1,7 @@ local actions = import 'actions.jsonnet'; local base = import 'base.jsonnet'; local database = import 'databases.jsonnet'; +local deployment = import 'deployment.jsonnet'; local docker = import 'docker.jsonnet'; local helm = import 'helm.jsonnet'; local misc = import 'misc.jsonnet'; @@ -150,7 +151,7 @@ local servicesImport = import 'services.jsonnet'; 'apidocs', runsOn=runsOn, image=rubyImageName, - ifClause="${{ github.event.deployment.environment == 'production' }}", + ifClause=deployment.deploymentTargets(['production']), steps=[ misc.checkout(), base.step( @@ -276,7 +277,7 @@ local servicesImport = import 'services.jsonnet'; base.ghJob( 'deploy-test', runsOn=runsOn, - ifClause="${{ github.event.deployment.environment == 'test' }}", + ifClause=deployment.deploymentTargets(['test']), image=image, useCredentials=useCredentials, steps= @@ -330,7 +331,7 @@ local servicesImport = import 'services.jsonnet'; base.ghJob( 'deploy-prod', runsOn=runsOn, - ifClause="${{ github.event.deployment.environment == 'production' }}", + ifClause=deployment.deploymentTargets(['production']), image=image, useCredentials=useCredentials, steps=[misc.checkout()] + diff --git a/.github/jsonnet/yarn.jsonnet b/.github/jsonnet/yarn.jsonnet index a4efa72..43bec7d 100644 --- a/.github/jsonnet/yarn.jsonnet +++ b/.github/jsonnet/yarn.jsonnet @@ -1,6 +1,7 @@ local actions = import 'actions.jsonnet'; local base = import 'base.jsonnet'; local cache = import 'cache.jsonnet'; +local deployment = import 'deployment.jsonnet'; local misc = import 'misc.jsonnet'; { @@ -143,7 +144,7 @@ local misc = import 'misc.jsonnet'; runsOn=runsOn, image=image, useCredentials=useCredentials, - ifClause="${{ github.event.deployment.environment == 'production' || github.event.deployment.environment == 'prod' }}", + ifClause=deployment.deploymentTargets(['production']), steps=[ misc.checkout() + self.setGynzyNpmToken() + diff --git a/.github/workflows/misc.yml b/.github/workflows/misc.yml index 2a00885..d6707a0 100644 --- a/.github/workflows/misc.yml +++ b/.github/workflows/misc.yml @@ -8,6 +8,28 @@ "image": "europe-docker.pkg.dev/unicorn-985/private-images/docker-images_jsonnet:v2" "runs-on": "ubuntu-latest" "steps": + - "continue-on-error": true + "env": + "GIT_HTTPS_ARCHIVE_MIRROR": "${{ secrets.GIT_HTTPS_ARCHIVE_MIRROR }}" + "if": "${{ env.GIT_HTTPS_ARCHIVE_MIRROR != '' }}" + "name": "fetch git-mirror archive" + "run": | + seed() { + if ! command -v zstd >/dev/null 2>&1; then + if command -v apk >/dev/null 2>&1; then apk add --no-cache zstd tar wget || return 1; + elif command -v apt >/dev/null 2>&1; then apt update && apt install -y zstd tar wget || return 1; + else return 1; fi + fi + wget -q -O - "$GIT_HTTPS_ARCHIVE_MIRROR" | tar -C "$GITHUB_WORKSPACE" --extract --zstd -f - || return 1 + } + if ! seed; then + echo "git-mirror seed failed; continuing with a clean checkout" + find "$GITHUB_WORKSPACE" -mindepth 1 -delete 2>/dev/null || true + exit 1 + fi + echo "git-mirror seed succeeded" + exit 0 + "shell": "bash" - "id": "check-binaries" "name": "check for ssh/git binaries" "run": | diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 960b789..77725f6 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -9,6 +9,28 @@ "pull-requests": "read" "runs-on": "ubuntu-latest" "steps": + - "continue-on-error": true + "env": + "GIT_HTTPS_ARCHIVE_MIRROR": "${{ secrets.GIT_HTTPS_ARCHIVE_MIRROR }}" + "if": "${{ env.GIT_HTTPS_ARCHIVE_MIRROR != '' }}" + "name": "fetch git-mirror archive" + "run": | + seed() { + if ! command -v zstd >/dev/null 2>&1; then + if command -v apk >/dev/null 2>&1; then apk add --no-cache zstd tar wget || return 1; + elif command -v apt >/dev/null 2>&1; then apt update && apt install -y zstd tar wget || return 1; + else return 1; fi + fi + wget -q -O - "$GIT_HTTPS_ARCHIVE_MIRROR" | tar -C "$GITHUB_WORKSPACE" --extract --zstd -f - || return 1 + } + if ! seed; then + echo "git-mirror seed failed; continuing with a clean checkout" + find "$GITHUB_WORKSPACE" -mindepth 1 -delete 2>/dev/null || true + exit 1 + fi + echo "git-mirror seed succeeded" + exit 0 + "shell": "bash" - "id": "check-binaries" "name": "check for ssh/git binaries" "run": | @@ -134,6 +156,28 @@ "image": "mirror.gcr.io/node:24" "runs-on": "ubuntu-latest" "steps": + - "continue-on-error": true + "env": + "GIT_HTTPS_ARCHIVE_MIRROR": "${{ secrets.GIT_HTTPS_ARCHIVE_MIRROR }}" + "if": "${{ env.GIT_HTTPS_ARCHIVE_MIRROR != '' }}" + "name": "fetch git-mirror archive" + "run": | + seed() { + if ! command -v zstd >/dev/null 2>&1; then + if command -v apk >/dev/null 2>&1; then apk add --no-cache zstd tar wget || return 1; + elif command -v apt >/dev/null 2>&1; then apt update && apt install -y zstd tar wget || return 1; + else return 1; fi + fi + wget -q -O - "$GIT_HTTPS_ARCHIVE_MIRROR" | tar -C "$GITHUB_WORKSPACE" --extract --zstd -f - || return 1 + } + if ! seed; then + echo "git-mirror seed failed; continuing with a clean checkout" + find "$GITHUB_WORKSPACE" -mindepth 1 -delete 2>/dev/null || true + exit 1 + fi + echo "git-mirror seed succeeded" + exit 0 + "shell": "bash" - "id": "check-binaries" "name": "check for ssh/git binaries" "run": | diff --git a/.github/workflows/publish-prod.yml b/.github/workflows/publish-prod.yml index c286e4d..ec73626 100644 --- a/.github/workflows/publish-prod.yml +++ b/.github/workflows/publish-prod.yml @@ -9,6 +9,28 @@ "pull-requests": "read" "runs-on": "ubuntu-latest" "steps": + - "continue-on-error": true + "env": + "GIT_HTTPS_ARCHIVE_MIRROR": "${{ secrets.GIT_HTTPS_ARCHIVE_MIRROR }}" + "if": "${{ env.GIT_HTTPS_ARCHIVE_MIRROR != '' }}" + "name": "fetch git-mirror archive" + "run": | + seed() { + if ! command -v zstd >/dev/null 2>&1; then + if command -v apk >/dev/null 2>&1; then apk add --no-cache zstd tar wget || return 1; + elif command -v apt >/dev/null 2>&1; then apt update && apt install -y zstd tar wget || return 1; + else return 1; fi + fi + wget -q -O - "$GIT_HTTPS_ARCHIVE_MIRROR" | tar -C "$GITHUB_WORKSPACE" --extract --zstd -f - || return 1 + } + if ! seed; then + echo "git-mirror seed failed; continuing with a clean checkout" + find "$GITHUB_WORKSPACE" -mindepth 1 -delete 2>/dev/null || true + exit 1 + fi + echo "git-mirror seed succeeded" + exit 0 + "shell": "bash" - "id": "check-binaries" "name": "check for ssh/git binaries" "run": |