Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/jsonnet/GIT_VERSION
Original file line number Diff line number Diff line change
@@ -1 +1 @@
545ab086d65446b210d470713a69ba3beecf28e9
5985c99fa9e401a6a47e10cd49e141524cf63d4d
91 changes: 87 additions & 4 deletions .github/jsonnet/cache.jsonnet
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
local base = import 'base.jsonnet';
local images = import 'images.jsonnet';
local misc = import 'misc.jsonnet';

{
/**
* Fetch a cache from the cache server.
*
*
* This is a generic function that can be used to fetch any cache. It is advised to wrap this function
* in a more specific function that fetches a specific cache, setting the cacheName and folders parameters.
*
*
* To be paired with the uploadCache function.
*
* @param {string} cacheName - The name of the cache to fetch. The name of the repository is usually a good option.
Expand Down Expand Up @@ -82,10 +84,10 @@ local base = import 'base.jsonnet';

/**
* Uploads a cache to the cache server.
*
*
* This is a generic function that can be used to upload any cache. It is advised to wrap this function
* in a more specific function that uploads a specific cache, setting the cacheName and folders parameters.
*
*
* To be paired with the fetchCache function.
*
* @param {string} cacheName - The name of the cache to upload. The name of the repository is usually a good option.
Expand Down Expand Up @@ -141,4 +143,85 @@ local base = import 'base.jsonnet';
'echo "Cache removed"\n',
ifClause=ifClause,
),

/**
* Daily (weekday) backup of the full repository (working tree + .git) to GCS as a zstd tar archive,
* then refreshes a 7-day signed HTTPS URL into the GIT_HTTPS_ARCHIVE_MIRROR repo secret.
*
* @param {string} [cron='0 1 * * 1-5'] - Schedule (UTC). Default 01:00 on weekdays.
* @param {string} [bucketPath='gs://gynzy-internal-files/git-mirror'] - Destination prefix.
* @returns {workflows} - GitHub Actions pipeline that mirrors the repository to GCS.
*/
updateGitCacheCron(
cron='0 1 * * 1-5',
)::
local secret = self.secret;
local destUrl = 'gs://gynzy-internal-files/git-mirror/${GITHUB_REPOSITORY}.tar.zst';
base.pipeline(
'git-mirror-backup',
[
base.ghJob(
'git-mirror-backup',
image=images.cloud_sdk_image,
useCredentials=true,
timeoutMinutes=60,
steps=[
// blobless=false: a backup must contain ALL git objects, not lazily-fetched blobs.
misc.checkout(fullClone=true, blobless=false, cloneTimeout=30, skipSeed=true),
base.step(
'authenticate gcloud',
|||
set -euo pipefail
# Write the key OUTSIDE the workspace so it never lands in the archive.
printf '%s' "$SERVICE_JSON" > "$RUNNER_TEMP/gce.json"
gcloud auth activate-service-account --key-file="$RUNNER_TEMP/gce.json"
|||,
shell='bash',
env={ SERVICE_JSON: misc.secret('SERVICE_JSON') },
),
base.step(
'create and upload archive',
|||
set -euo pipefail
# checkStat=minimal makes git compare only mtime+size (which tar preserves),
# not ctime/inode. Consumers that seed from this archive then see a clean tree
# and only write the real diff during checkout instead of rewriting every file.
git -C "$GITHUB_WORKSPACE" config --local core.checkStat minimal

DEST="%s"

# Upload to a temp object first, then atomically move into place so a partial
# upload can never sit at the final destination.
tar -C "$GITHUB_WORKSPACE" -c . | zstdmt -10 | gcloud storage cp - "${DEST}.tmp"
gcloud storage mv "${DEST}.tmp" "$DEST"
||| % destUrl,
shell='bash',
),
base.step(
'refresh signed-url secret',
|||
set -euo pipefail
DEST="%s"
URL="$(gcloud storage sign-url "$DEST" \
--private-key-file="$RUNNER_TEMP/gce.json" --http-verb=GET --duration=7d \
--format='value(signed_url)')"
echo "::add-mask::$URL"
gh secret set "GIT_HTTPS_ARCHIVE_MIRROR" --repo "$GITHUB_REPOSITORY" --body "$URL"
||| % destUrl,
shell='bash',
env={
GH_TOKEN: misc.secret('VIRKO_GITHUB_TOKEN'),
},
),
base.step(
'remove credentials',
'rm -f "$RUNNER_TEMP/gce.json"',
ifClause='${{ always() }}',
),
],
),
],
event={ schedule: [{ cron: cron }], workflow_dispatch: {} },
permissions={ contents: 'read' },
),
}
5 changes: 3 additions & 2 deletions .github/jsonnet/deployment.jsonnet
Original file line number Diff line number Diff line change
Expand Up @@ -160,10 +160,11 @@ local notifications = import 'notifications.jsonnet';
* Generate a GitHub ifClause for the provided deployment targets.
*
* @param {array} targets - Array of deployment target environment names
* @param {boolean} [virkoOnly=true] - If true, also require the deployment event to be created by 'gynzy-virko', preventing manually created deployment events from triggering the job
* @returns {string} - GitHub Actions conditional expression that matches any of the provided targets
*/
deploymentTargets(targets)::
'${{ ' + std.join(' || ', std.map(function(target) "github.event.deployment.environment == '" + target + "'", targets)) + ' }}',
deploymentTargets(targets, virkoOnly=true)::
'${{ github.event_name == \'deployment\' && (' + std.join(' || ', std.map(function(target) "github.event.deployment.environment == '" + target + "'", targets)) + ')' + (if virkoOnly then " && github.event.deployment.creator.login == 'gynzy-virko'" else '') + ' }}',

/**
* Creates a step to update deployment status (success/failure) based on the result from the current job
Expand Down
9 changes: 5 additions & 4 deletions .github/jsonnet/helm.jsonnet
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
local base = import 'base.jsonnet';
local clusters = import 'clusters.jsonnet';
local databases = import 'databases.jsonnet';
local deployment = import 'deployment.jsonnet';
local images = import 'images.jsonnet';
local misc = import 'misc.jsonnet';
local services = import 'services.jsonnet';
Expand Down Expand Up @@ -151,7 +152,7 @@ local services = import 'services.jsonnet';
base.ghJob(
'deploy-prod',
runsOn=runsOn,
ifClause="${{ github.event.deployment.environment == '" + environment + "' }}",
ifClause=deployment.deploymentTargets([environment]),
image=image,
useCredentials=useCredentials,
steps=[
Expand Down Expand Up @@ -234,7 +235,7 @@ local services = import 'services.jsonnet';
base.ghJob(
'deploy-test',
runsOn=runsOn,
ifClause="${{ github.event.deployment.environment == 'test' }}",
ifClause=deployment.deploymentTargets(['test']),
image=image,
useCredentials=useCredentials,
steps=[
Expand Down Expand Up @@ -515,7 +516,7 @@ local services = import 'services.jsonnet';
runsOn=runsOn,
image=image,
useCredentials=useCredentials,
ifClause="${{ github.event.deployment.environment == 'canary' }}",
ifClause=deployment.deploymentTargets(['canary']),
steps=[
misc.checkout(),
self.helmDeployCanary(
Expand Down Expand Up @@ -597,7 +598,7 @@ local services = import 'services.jsonnet';
base.ghJob(
'kill-canary',
runsOn=runsOn,
ifClause="${{ github.event.deployment.environment == 'kill-canary' || github.event.deployment.environment == 'production' }}",
ifClause=deployment.deploymentTargets(['kill-canary', 'production']),
image=images.default_job_image,
useCredentials=false,
steps=[
Expand Down
1 change: 1 addition & 0 deletions .github/jsonnet/images.jsonnet
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
default_mongodb_image: 'europe-docker.pkg.dev/unicorn-985/private-images/docker-images_mongo8-replicated:v1',
mongo_job_image: 'europe-docker.pkg.dev/unicorn-985/public-images/docker-images_mongo-cloner-job:v1',
default_python_image: 'mirror.gcr.io/python:3.12.1',
cloud_sdk_image: 'europe-docker.pkg.dev/unicorn-985/private-images/docker-images_gcloud:v2',
default_pulumi_node_image: 'mirror.gcr.io/node:22',
job_poster_image: 'europe-docker.pkg.dev/unicorn-985/public-images/docker-images_job-poster:v2',
}
34 changes: 33 additions & 1 deletion .github/jsonnet/misc.jsonnet
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,11 @@ local images = import 'images.jsonnet';
* @param {boolean} [blobless=null] - Whether to perform a blobless clone (--filter=blob:none); null uses default (false)
* @param {number} [retryAttempts=null] - Number of additional checkout attempts on failure; null uses default (0)
* @param {number} [cloneTimeout=null] - Timeout for git clone operation in minutes; null uses default (10)
* @param {boolean} [skipSeed=false] - Boolean where the archive based git initialization can be skipped
* @returns {steps} - GitHub Actions steps for repository checkout
*/
checkout(ifClause=null, fullClone=false, ref=null, preferSshClone=true, includeSubmodules=true, blobless=null, retryAttempts=null, cloneTimeout=null)::
checkout(ifClause=null, fullClone=false, ref=null, preferSshClone=true, includeSubmodules=true, blobless=null, retryAttempts=null, cloneTimeout=null, skipSeed=false)::
local secret = self.secret;
local actualBlobless = if blobless == null then false else blobless;
local actualRetryAttempts = if retryAttempts == null then 0 else retryAttempts;
local actualCloneTimeout = if cloneTimeout == null then 10 else cloneTimeout;
Expand Down Expand Up @@ -73,6 +75,34 @@ local images = import 'images.jsonnet';
id='check-binaries',
) else []);

// Cache seed: unpack the mirror archive (full working tree + .git) so the checkout
// below only fetches the delta. Skipped at runtime when the mirror secret is empty.
// Any failure leaves a clean workspace and continues normally.
local seedSteps = (if skipSeed then [] else base.step(
'fetch git-mirror archive',
|||
seed() {
if ! command -v zstd >/dev/null 2>&1; then
if command -v apk >/dev/null 2>&1; then apk add --no-cache zstd tar wget || return 1;
elif command -v apt >/dev/null 2>&1; then apt update && apt install -y zstd tar wget || return 1;
else return 1; fi
fi
wget -q -O - "$GIT_HTTPS_ARCHIVE_MIRROR" | tar -C "$GITHUB_WORKSPACE" --extract --zstd -f - || return 1
}
if ! seed; then
echo "git-mirror seed failed; continuing with a clean checkout"
find "$GITHUB_WORKSPACE" -mindepth 1 -delete 2>/dev/null || true
exit 1
fi
echo "git-mirror seed succeeded"
exit 0
|||,
env={ GIT_HTTPS_ARCHIVE_MIRROR: secret('GIT_HTTPS_ARCHIVE_MIRROR') },
ifClause="${{ env.GIT_HTTPS_ARCHIVE_MIRROR != '' }}",
shell='bash',
continueOnError=true,
));

// strip the ${{ }} from the IfClause so we can inject and add our own if clause
local localIfClause = (if ifClause == null then null else std.strReplace(std.strReplace(ifClause, '${{ ', ''), ' }}', ''));
local userIfPart = if ifClause == null then '' else '( ' + localIfClause + ' ) && ';
Expand All @@ -97,6 +127,7 @@ local images = import 'images.jsonnet';
);

if (preferSshClone) then
seedSteps +
sshSteps +
retrySteps(
'Check out repository code via ssh',
Expand All @@ -112,6 +143,7 @@ local images = import 'images.jsonnet';
) +
base.step('git safe directory', "command -v git && git config --global --add safe.directory '*' || true")
else
seedSteps +
self.checkoutWithoutSshMagic(ifClause, fullClone, ref),

/**
Expand Down
3 changes: 2 additions & 1 deletion .github/jsonnet/newrelic.jsonnet
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
local base = import 'base.jsonnet';
local deployment = import 'deployment.jsonnet';
local images = import 'images.jsonnet';
local misc = import 'misc.jsonnet';
local yarn = import 'yarn.jsonnet';
Expand Down Expand Up @@ -31,7 +32,7 @@ local pnpm = import 'pnpm.jsonnet';
runsOn=runsOn,
image=image,
useCredentials=useCredentials,
ifClause="${{ github.event.deployment.environment == 'production' }}",
ifClause=deployment.deploymentTargets(['production']),
steps=
(
if packageManager == 'yarn' then
Expand Down
3 changes: 2 additions & 1 deletion .github/jsonnet/pnpm.jsonnet
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
local actions = import 'actions.jsonnet';
local base = import 'base.jsonnet';
local cache = import 'cache.jsonnet';
local deployment = import 'deployment.jsonnet';
local misc = import 'misc.jsonnet';
local yarn = import 'yarn.jsonnet';

Expand Down Expand Up @@ -144,7 +145,7 @@ local yarn = import 'yarn.jsonnet';
runsOn=runsOn,
image=image,
useCredentials=useCredentials,
ifClause="${{ github.event.deployment.environment == 'production' || github.event.deployment.environment == 'prod' }}",
ifClause=deployment.deploymentTargets(['production']),
steps=[
self.checkoutAndPnpm(
cacheName=null, // to populate cache we want a clean install
Expand Down
Loading
Loading