From f191f0187737c697c778eb0f30cb49b62b1cfc24 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sun, 31 Dec 2023 19:05:44 -0500 Subject: [PATCH 01/26] feat(ci): streamline Kubernetes preview and deploy workflows --- ...elease-diff.yml => preview-kubernetes.yml} | 30 ++++++--- .../workflows/service-release-candidate.yml | 67 ------------------- .github/workflows/service-release-channel.yml | 44 ------------ 3 files changed, 22 insertions(+), 119 deletions(-) rename .github/workflows/{service-release-diff.yml => preview-kubernetes.yml} (66%) delete mode 100644 .github/workflows/service-release-candidate.yml delete mode 100644 .github/workflows/service-release-channel.yml diff --git a/.github/workflows/service-release-diff.yml b/.github/workflows/preview-kubernetes.yml similarity index 66% rename from .github/workflows/service-release-diff.yml rename to .github/workflows/preview-kubernetes.yml index 63ef31bfb7..501cabd513 100644 --- a/.github/workflows/service-release-diff.yml +++ b/.github/workflows/preview-kubernetes.yml @@ -3,11 +3,15 @@ name: Show diff for release channel on: pull_request: branches: - - 'releases/*' - types: - - opened - - synchronize - - reopened + - 'main' + paths: + - '.holo/config.toml' + - '.holo/sources/jarvus-cluster-template.toml' + - '.holo/branches/releases/**' + - '.github/workflows/service-*' + - 'ci/**' + - 'kubernetes/apps/**' + - 'kubernetes/system/**' env: GITHUB_TOKEN: ${{ secrets.GH_ACTIONS_TOKEN }} @@ -22,9 +26,7 @@ jobs: USE_GKE_GCLOUD_AUTH_PLUGIN: True steps: # Setup - - uses: actions/checkout@v2 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - uses: 'google-github-actions/auth@v2' with: @@ -36,6 +38,18 @@ jobs: cluster_name: ${{ env.GKE_NAME }} location: ${{ env.GKE_REGION }} - run: curl -sSL https://install.python-poetry.org | python - + - name: Set up hologit + env: + BIO_RELEASE: 1.6.821 + run: | + curl -LO "https://github.com/biome-sh/biome/releases/download/$BIO_RELEASE/bio-$BIO_RELEASE-x86_64-linux.tar.gz" + tar xzvf "bio-$BIO_RELEASE-x86_64-linux.tar.gz" + sudo mv bio /usr/local/bin/bio + sudo bio pkg install --binlink jarvus/hologit + + # Render Kubernetes content with parent underlay and checkout + - run: git holo project release-candidate --commit-to=kubernetes + - run: git checkout kubernetes # Diff and write back to PR - id: diff diff --git a/.github/workflows/service-release-candidate.yml b/.github/workflows/service-release-candidate.yml deleted file mode 100644 index af8b885acb..0000000000 --- a/.github/workflows/service-release-candidate.yml +++ /dev/null @@ -1,67 +0,0 @@ -name: Build release candidate - -on: - push: - branches: - - '*' - paths: - - '.holo/config.toml' - - '.holo/sources/jarvus-cluster-template.toml' - - '.holo/branches/releases/**' - - '.github/workflows/service-*' - - 'ci/**' - - 'kubernetes/apps/**' - - 'kubernetes/system/**' - - -env: - GITHUB_TOKEN: ${{ secrets.GH_ACTIONS_TOKEN }} - BIO_RELEASE: 1.6.372 - - -jobs: - build-release-candidate: - runs-on: ubuntu-latest - steps: - - - name: Check out repo - uses: actions/checkout@v2 - with: - fetch-depth: 0 - token: ${{ secrets.GH_ACTIONS_TOKEN }} - - - name: 'Build release candidate branch' - run: | - - # dependency setup - curl -LO "https://github.com/biome-sh/biome/releases/download/v1.6.372/bio-$BIO_RELEASE-x86_64-linux.tar.gz" - tar xzvf "bio-$BIO_RELEASE-x86_64-linux.tar.gz" - sudo mv bio /usr/local/bin/bio - sudo bio pkg install --binlink jarvus/hologit - - # environment setup - [[ ! $(git status --porcelain=v1) ]] || git stash -u - git config user.name "Github Action $GITHUB_JOB" - git config user.email "$(whoami)@$(uname -n)" - remote_name=origin - source_branch=${GITHUB_REF#refs/heads/} - candidate_ref=refs/heads/candidates/$source_branch - prod_ref=refs/heads/releases/prod - prod_remote_ref=refs/remotes/$remote_name/${prod_ref#refs/heads/} - - # branch setup - git update-ref -d "$prod_remote_ref" - git update-ref -d "$candidate_ref" - if [[ $(git ls-remote "$remote_name" "$prod_ref") ]]; then - git fetch "$remote_name" "$prod_ref" - fi - - # build & push - commit_msg="Release candidate: $source_branch - - Source-holobranch: release-candidate - Source-commit: $(git rev-list HEAD -1)" - - git holo project release-candidate --commit-to "$prod_remote_ref" --commit-message "$commit_msg" - git update-ref "$candidate_ref" "$prod_remote_ref" - git push -f "$remote_name" "$candidate_ref" diff --git a/.github/workflows/service-release-channel.yml b/.github/workflows/service-release-channel.yml deleted file mode 100644 index 95c0bd1559..0000000000 --- a/.github/workflows/service-release-channel.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: Deploy into release channel - -on: - push: - branches: - - 'releases/*' - -jobs: - release: - runs-on: ubuntu-latest - env: - CLOUDSDK_CORE_PROJECT: cal-itp-data-infra - GKE_NAME: data-infra-apps - GKE_REGION: us-west1 - USE_GKE_GCLOUD_AUTH_PLUGIN: True - steps: - # Setup - - name: Check out repo - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - uses: 'google-github-actions/auth@v2' - with: - credentials_json: '${{ secrets.GCP_SA_KEY }}' - - uses: google-github-actions/setup-gcloud@v2 - - name: install auth plugin - run: gcloud components install gke-gcloud-auth-plugin - - uses: google-github-actions/get-gke-credentials@v1 - with: - cluster_name: ${{ env.GKE_NAME }} - location: ${{ env.GKE_REGION }} - - run: curl -sSL https://install.python-poetry.org | python - - - # Release to channel - - id: poetry-invoke - name: Run poetry invoke - shell: bash - working-directory: ci - run: | - export RELEASE_CHANNEL=${GITHUB_REF#refs/heads/releases/} - printf 'WORKFLOW: service-release-channel; RELEASE_CHANNEL=%s\n' "$RELEASE_CHANNEL" - poetry install - poetry run invoke secrets -f "./channels/$RELEASE_CHANNEL.yaml" - poetry run invoke release -f "./channels/$RELEASE_CHANNEL.yaml" From f482c854b801ccdaf0e8f40c61b344aefb1734f3 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Sun, 31 Dec 2023 19:21:14 -0500 Subject: [PATCH 02/26] fix(ci): apply diff formatting to markdown content --- .github/workflows/preview-kubernetes.yml | 2 +- ci/tasks.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/preview-kubernetes.yml b/.github/workflows/preview-kubernetes.yml index 501cabd513..72ac0ae222 100644 --- a/.github/workflows/preview-kubernetes.yml +++ b/.github/workflows/preview-kubernetes.yml @@ -60,7 +60,7 @@ jobs: export RELEASE_CHANNEL=${GITHUB_BASE_REF#releases/} printf 'WORKFLOW: service-release-diff; RELEASE_CHANNEL=%s\n' "$RELEASE_CHANNEL" poetry install - poetry run invoke diff -f "./channels/$RELEASE_CHANNEL.yaml" --outfile=diff.txt + poetry run invoke diff -f "./channels/$RELEASE_CHANNEL.yaml" --outfile=diff.md - uses: peter-evans/find-comment@v2 id: fc diff --git a/ci/tasks.py b/ci/tasks.py index 05dd1bb357..1ee63fdcce 100644 --- a/ci/tasks.py +++ b/ci/tasks.py @@ -208,7 +208,7 @@ def diff( full_diff += result.stdout msg = ( - f"```{full_diff}```" + f"```diff\n{full_diff}\n```" if full_diff else f"No {driver if driver else 'manifest'} changes found for {c.calitp_config.channel}.\n" ) From 02802ba14efed6caf1cd3d6ea50970c711778364 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Tue, 2 Jan 2024 19:18:56 -0500 Subject: [PATCH 03/26] feat(ci): add support for including secret helm values when diffing --- ci/channels/prod.yaml | 4 +++ ci/tasks.py | 63 +++++++++++++++++++++++++++++-------------- 2 files changed, 47 insertions(+), 20 deletions(-) diff --git a/ci/channels/prod.yaml b/ci/channels/prod.yaml index 30c0f70522..06e4a05535 100644 --- a/ci/channels/prod.yaml +++ b/ci/channels/prod.yaml @@ -26,6 +26,8 @@ calitp: namespace: jupyterhub helm_name: jupyterhub helm_chart: kubernetes/apps/charts/jupyterhub + secret_helm_values: + - jupyterhub_jupyterhub-sensitive-helm-values secrets: - jupyterhub_jupyterhub-gcloud-service-key - jupyterhub_jupyterhub-github-config @@ -97,6 +99,8 @@ calitp: namespace: sentry helm_name: sentry helm_chart: kubernetes/apps/charts/sentry + secret_helm_values: + - sentry_sentry-sensitive-helm-values secrets: - sentry_sentry-secret - sentry_sentry-sentry-postgresql diff --git a/ci/tasks.py b/ci/tasks.py index 1ee63fdcce..fe6fa9d2ee 100644 --- a/ci/tasks.py +++ b/ci/tasks.py @@ -37,6 +37,7 @@ class Release(BaseModel): helm_name: Optional[str] helm_chart: Optional[Path] helm_values: List[Path] = [] + secret_helm_values: List[str] = [] timeout: Optional[str] # for kustomize @@ -168,6 +169,8 @@ def diff( full_diff = "" result: Result + secrets_client = secretmanager.SecretManagerServiceClient() + for release in get_releases(c, driver=actual_driver, app=app): if release.driver == ReleaseDriver.kustomize: assert release.kustomize_dir is not None @@ -178,28 +181,48 @@ def diff( chart_path = c.calitp_config.git_root / Path(release.helm_chart) c.run(f"helm dependency update {chart_path}") - values_str = " ".join( - [ - f"--values {c.calitp_config.git_root / Path(values_file)}" - for values_file in release.helm_values - ] - ) - assert release.helm_name is not None - result = c.run( - " ".join( + with tempfile.TemporaryDirectory() as tmpdir: + secret_helm_value_paths = [] + for secret_helm_values in release.secret_helm_values: + secret_path = Path(tmpdir) / Path(f"{secret_helm_values}.yaml") + name = f"projects/1005246706141/secrets/{secret_helm_values}/versions/latest" + secret_contents = secrets_client.access_secret_version( + request={"name": name} + ).payload.data.decode("UTF-8") + + with open(secret_path, "w") as f: + f.write(secret_contents) + + secret_helm_value_paths.append(secret_path) + print(f"Downloaded secret helm values: {secret_path}", flush=True) + + values_str = " ".join( [ - "helm", - "diff", - "upgrade", - release.helm_name, - str(chart_path), - f"--namespace={release.namespace}", - values_str, - "-C 5", # only include 5 lines of context + f"--values={c.calitp_config.git_root / Path(values_file)}" + for values_file in release.helm_values ] - ), - warn=True, - ) + ).join( + [ + f"--values={secret_path}" + for secret_path in secret_helm_value_paths + ] + ) + assert release.helm_name is not None + result = c.run( + " ".join( + [ + "helm", + "diff", + "upgrade", + release.helm_name, + str(chart_path), + f"--namespace={release.namespace}", + values_str, + "-C 5", # only include 5 lines of context + ] + ), + warn=True, + ) else: print(f"Encountered unknown driver: {release.driver}", flush=True) raise RuntimeError From 72847d9b103f52d432f54f6a7cf83cb5ea51626d Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Tue, 2 Jan 2024 22:13:22 -0500 Subject: [PATCH 04/26] fix(ci): capture activeDeadlineSeconds already applied to cluster --- kubernetes/apps/charts/sentry/values.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kubernetes/apps/charts/sentry/values.yaml b/kubernetes/apps/charts/sentry/values.yaml index f7016ea343..28ca2ff940 100644 --- a/kubernetes/apps/charts/sentry/values.yaml +++ b/kubernetes/apps/charts/sentry/values.yaml @@ -108,3 +108,6 @@ sentry: persistentVolumeClaim: dataPersistentVolume: storage: "50Gi" + + hooks: + activeDeadlineSeconds: 1200 From 5edb2b1bf7edee39aa5378437bf4bbfa5f6dc6bd Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Tue, 2 Jan 2024 22:13:40 -0500 Subject: [PATCH 05/26] fix(ci): configure sentry to use existing secret --- kubernetes/apps/charts/sentry/values.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/kubernetes/apps/charts/sentry/values.yaml b/kubernetes/apps/charts/sentry/values.yaml index 28ca2ff940..c9fd05bbf8 100644 --- a/kubernetes/apps/charts/sentry/values.yaml +++ b/kubernetes/apps/charts/sentry/values.yaml @@ -81,6 +81,7 @@ sentry: days: 30 postProcessForwardTransactions: replicas: 2 + existingSecret: sentry-sentry-secret snuba: subscriptionConsumerTransactions: From 5d38e2705e0c8f781fcea538bcf6e1edba038dcd Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Tue, 2 Jan 2024 22:13:57 -0500 Subject: [PATCH 06/26] fix(ci): move diff newline --- ci/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/tasks.py b/ci/tasks.py index fe6fa9d2ee..eb718b1be5 100644 --- a/ci/tasks.py +++ b/ci/tasks.py @@ -231,7 +231,7 @@ def diff( full_diff += result.stdout msg = ( - f"```diff\n{full_diff}\n```" + f"```diff\n{full_diff}```\n" if full_diff else f"No {driver if driver else 'manifest'} changes found for {c.calitp_config.channel}.\n" ) From 7b1eb8c16e72aa2db612193f9d679b2245207e0d Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Tue, 2 Jan 2024 22:14:18 -0500 Subject: [PATCH 07/26] fix(ci): remove sentry secrets that aren't needed --- ci/channels/prod.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci/channels/prod.yaml b/ci/channels/prod.yaml index 06e4a05535..faf422769f 100644 --- a/ci/channels/prod.yaml +++ b/ci/channels/prod.yaml @@ -99,8 +99,6 @@ calitp: namespace: sentry helm_name: sentry helm_chart: kubernetes/apps/charts/sentry - secret_helm_values: - - sentry_sentry-sensitive-helm-values secrets: - sentry_sentry-secret - sentry_sentry-sentry-postgresql From 1c626213fcbc684dc79b0d7b5886697460583579 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Tue, 2 Jan 2024 22:20:50 -0500 Subject: [PATCH 08/26] fix(ci): update workflow title --- .github/workflows/preview-kubernetes.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/preview-kubernetes.yml b/.github/workflows/preview-kubernetes.yml index 72ac0ae222..d01c1ed182 100644 --- a/.github/workflows/preview-kubernetes.yml +++ b/.github/workflows/preview-kubernetes.yml @@ -1,4 +1,4 @@ -name: Show diff for release channel +name: Generate diff of Kubernetes changes on: pull_request: From 1ae2bc48a176584c198368c31793f25a4d3c653a Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Tue, 2 Jan 2024 22:25:23 -0500 Subject: [PATCH 09/26] fix(ci): bio download path --- .github/workflows/preview-kubernetes.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/preview-kubernetes.yml b/.github/workflows/preview-kubernetes.yml index d01c1ed182..e45119885f 100644 --- a/.github/workflows/preview-kubernetes.yml +++ b/.github/workflows/preview-kubernetes.yml @@ -42,8 +42,8 @@ jobs: env: BIO_RELEASE: 1.6.821 run: | - curl -LO "https://github.com/biome-sh/biome/releases/download/$BIO_RELEASE/bio-$BIO_RELEASE-x86_64-linux.tar.gz" - tar xzvf "bio-$BIO_RELEASE-x86_64-linux.tar.gz" + curl -LO "https://github.com/biome-sh/biome/releases/download/v${BIO_RELEASE}/bio-${BIO_RELEASE}-x86_64-linux.tar.gz" + tar xzvf "bio-${BIO_RELEASE}-x86_64-linux.tar.gz" sudo mv bio /usr/local/bin/bio sudo bio pkg install --binlink jarvus/hologit From 09a8de0c7328df65332b52db1e262f21a3d6e430 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Tue, 2 Jan 2024 22:33:09 -0500 Subject: [PATCH 10/26] fix(ci): configure git user --- .github/workflows/preview-kubernetes.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/preview-kubernetes.yml b/.github/workflows/preview-kubernetes.yml index e45119885f..81fced479e 100644 --- a/.github/workflows/preview-kubernetes.yml +++ b/.github/workflows/preview-kubernetes.yml @@ -47,6 +47,9 @@ jobs: sudo mv bio /usr/local/bin/bio sudo bio pkg install --binlink jarvus/hologit + git config user.name "Github Action $GITHUB_JOB" + git config user.email "$(whoami)@$(uname -n)" + # Render Kubernetes content with parent underlay and checkout - run: git holo project release-candidate --commit-to=kubernetes - run: git checkout kubernetes From 34dad839d4730d7db48835442e3d81e3c9990462 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Tue, 2 Jan 2024 22:41:23 -0500 Subject: [PATCH 11/26] fix(ci): correct diff filename --- .github/workflows/preview-kubernetes.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/preview-kubernetes.yml b/.github/workflows/preview-kubernetes.yml index 81fced479e..a461463978 100644 --- a/.github/workflows/preview-kubernetes.yml +++ b/.github/workflows/preview-kubernetes.yml @@ -75,5 +75,5 @@ jobs: with: comment-id: ${{ steps.fc.outputs.comment-id }} issue-number: ${{ github.event.number }} - body-file: "ci/diff.txt" + body-file: "ci/diff.md" edit-mode: replace From 90a0d341b65124c5ecc2f6279cd9b85211b44a49 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Tue, 2 Jan 2024 22:46:42 -0500 Subject: [PATCH 12/26] fix(ci): remove deprecated RELEASE_CHANNEL variable --- .github/workflows/preview-kubernetes.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/preview-kubernetes.yml b/.github/workflows/preview-kubernetes.yml index a461463978..d02621e9a3 100644 --- a/.github/workflows/preview-kubernetes.yml +++ b/.github/workflows/preview-kubernetes.yml @@ -60,10 +60,8 @@ jobs: shell: bash working-directory: ci run: | - export RELEASE_CHANNEL=${GITHUB_BASE_REF#releases/} - printf 'WORKFLOW: service-release-diff; RELEASE_CHANNEL=%s\n' "$RELEASE_CHANNEL" poetry install - poetry run invoke diff -f "./channels/$RELEASE_CHANNEL.yaml" --outfile=diff.md + poetry run invoke diff -f "./channels/prod.yaml" --outfile=diff.md - uses: peter-evans/find-comment@v2 id: fc From 0c0e4627a6a565c8de0db0c14ff35a4dc17125bc Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Tue, 2 Jan 2024 22:59:10 -0500 Subject: [PATCH 13/26] docs(ci): move k8s GitOps to kubernetes/README.md --- .github/workflows/README.md | 17 ----------------- kubernetes/README.md | 10 ++++++++++ 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index c1c5001a35..c9ba897c27 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -26,20 +26,3 @@ Workflows prefixed with `service-` deal with Kubernetes deployments. - `service-release-channel.yml` deploys to a given channel (i.e. environment) on updates to a release branch Some of these workflows use hologit or invoke. See the READMEs in [.holo](../../.holo) and [ci](../../ci) for documentation regarding hologit and invoke, respectively. - -## GitOps - -The workflows described above also define their triggers. In general, developer workflows should follow these steps. - -1. Check out a feature branch -2. Put up a PR for that feature branch, targeting `main` - - `service-release-candidate` will run and create a remote branch named `candidate/ Date: Tue, 2 Jan 2024 22:59:21 -0500 Subject: [PATCH 14/26] docs(ci): fix header --- kubernetes/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/README.md b/kubernetes/README.md index 447018b72c..5d6a88a765 100644 --- a/kubernetes/README.md +++ b/kubernetes/README.md @@ -138,7 +138,7 @@ At the time of this writing, a JupyterHub deployment is available at [https://no - `ingress.hosts` - `ingress.tls.hosts` -# Backups +## Backups For most of our backups we utilize [Restic](https://restic.readthedocs.io/en/latest/010_introduction.html); this section uses the Metabase database backup as an example. From aee380594854e42a5aa933170d01cbbb6da80414 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Tue, 2 Jan 2024 23:00:09 -0500 Subject: [PATCH 15/26] fix(ci): update holobranch name --- .github/workflows/preview-kubernetes.yml | 2 +- .../_data-infra.toml | 0 .../kubernetes/apps/charts/grafana.toml | 0 .../kubernetes/apps/charts/loki.toml | 0 .../kubernetes/apps/charts/prometheus.toml | 0 .../kubernetes/apps/charts/promtail.toml | 0 6 files changed, 1 insertion(+), 1 deletion(-) rename .holo/branches/{release-candidate => kubernetes-workspace}/_data-infra.toml (100%) rename .holo/branches/{release-candidate => kubernetes-workspace}/kubernetes/apps/charts/grafana.toml (100%) rename .holo/branches/{release-candidate => kubernetes-workspace}/kubernetes/apps/charts/loki.toml (100%) rename .holo/branches/{release-candidate => kubernetes-workspace}/kubernetes/apps/charts/prometheus.toml (100%) rename .holo/branches/{release-candidate => kubernetes-workspace}/kubernetes/apps/charts/promtail.toml (100%) diff --git a/.github/workflows/preview-kubernetes.yml b/.github/workflows/preview-kubernetes.yml index d02621e9a3..0bfda799d8 100644 --- a/.github/workflows/preview-kubernetes.yml +++ b/.github/workflows/preview-kubernetes.yml @@ -51,7 +51,7 @@ jobs: git config user.email "$(whoami)@$(uname -n)" # Render Kubernetes content with parent underlay and checkout - - run: git holo project release-candidate --commit-to=kubernetes + - run: git holo project kubernetes-workspace --commit-to=kubernetes - run: git checkout kubernetes # Diff and write back to PR diff --git a/.holo/branches/release-candidate/_data-infra.toml b/.holo/branches/kubernetes-workspace/_data-infra.toml similarity index 100% rename from .holo/branches/release-candidate/_data-infra.toml rename to .holo/branches/kubernetes-workspace/_data-infra.toml diff --git a/.holo/branches/release-candidate/kubernetes/apps/charts/grafana.toml b/.holo/branches/kubernetes-workspace/kubernetes/apps/charts/grafana.toml similarity index 100% rename from .holo/branches/release-candidate/kubernetes/apps/charts/grafana.toml rename to .holo/branches/kubernetes-workspace/kubernetes/apps/charts/grafana.toml diff --git a/.holo/branches/release-candidate/kubernetes/apps/charts/loki.toml b/.holo/branches/kubernetes-workspace/kubernetes/apps/charts/loki.toml similarity index 100% rename from .holo/branches/release-candidate/kubernetes/apps/charts/loki.toml rename to .holo/branches/kubernetes-workspace/kubernetes/apps/charts/loki.toml diff --git a/.holo/branches/release-candidate/kubernetes/apps/charts/prometheus.toml b/.holo/branches/kubernetes-workspace/kubernetes/apps/charts/prometheus.toml similarity index 100% rename from .holo/branches/release-candidate/kubernetes/apps/charts/prometheus.toml rename to .holo/branches/kubernetes-workspace/kubernetes/apps/charts/prometheus.toml diff --git a/.holo/branches/release-candidate/kubernetes/apps/charts/promtail.toml b/.holo/branches/kubernetes-workspace/kubernetes/apps/charts/promtail.toml similarity index 100% rename from .holo/branches/release-candidate/kubernetes/apps/charts/promtail.toml rename to .holo/branches/kubernetes-workspace/kubernetes/apps/charts/promtail.toml From 761a79d7979e2c43fa907b841ec6684d80c1093b Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Tue, 2 Jan 2024 23:00:31 -0500 Subject: [PATCH 16/26] fix(ci): delete workflow for cleaning up deprecated candidate branches --- .../workflows/cleanup-release-candidates.yml | 33 ------------------- 1 file changed, 33 deletions(-) delete mode 100644 .github/workflows/cleanup-release-candidates.yml diff --git a/.github/workflows/cleanup-release-candidates.yml b/.github/workflows/cleanup-release-candidates.yml deleted file mode 100644 index 75dc99103c..0000000000 --- a/.github/workflows/cleanup-release-candidates.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Cleanup release candidate branches - -on: - delete: - branches: - - '!candidates/*' - -env: - GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }} - REPO_URL: ${{ github.server_url }}/${{ github.repository }} - - -jobs: - delete-release-candidate: - runs-on: ubuntu-latest - steps: - - # This step is only for credential setup - - name: Check out repo - uses: actions/checkout@v2 - - - name: 'Delete orphaned release candidates' - run: | - while read candidate_sha1 candidate_ref; do - test "$candidate_ref" || continue - source_branch=${candidate_ref#refs/heads/candidates/} - source_ref=refs/heads/$source_branch - if ! [[ $(git ls-remote "$REPO_URL" "$source_ref") ]]; then - printf 'Pruning orphaned release candidate: %s\n' "$candidate_ref" - git push "$REPO_URL" --delete "$candidate_ref" - fi - done <<< "$(git ls-remote "$REPO_URL" 'refs/heads/candidates/*')" - printf 'All orphaned release candidates successfully pruned\n' From fd33f78e9eda9116b708ac782745964c2ec05220 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Tue, 2 Jan 2024 23:02:00 -0500 Subject: [PATCH 17/26] docs(ci): update github workflows docs --- .github/workflows/README.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index c9ba897c27..f09bc45b9e 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -13,16 +13,18 @@ While we're using GCP Composer, "deployment" of Airflow consists of two parts: This workflow builds a static website from the Svelte app and deploys it to Netlify. +## deploy-kubernetes.yml + +This workflow deploys changes to the production Kubernetes cluster when they get merged into the `main` branch. + ## build-\*.yml workflows Workflows prefixed with `build-` generally lint, test, and (usually) publish either a Python package or a Docker image. -## service-\*.yml workflows +## preview-\*.yml workflows -Workflows prefixed with `service-` deal with Kubernetes deployments. +Workflows prefixed with `preview-` deal with generating previews for pull request changes -- `service-release-candidate.yml` creates candidate branches, using [hologit](https://github.com/JarvusInnovations/hologit) to bring in external Helm charts and remove irrelevant (i.e. non-infra) code -- `service-release-diff.yml` renders kubectl diffs on PRs targeting release branches -- `service-release-channel.yml` deploys to a given channel (i.e. environment) on updates to a release branch +- `preview-kubernetes.yml` renders kubectl diffs on PRs changing cluster content Some of these workflows use hologit or invoke. See the READMEs in [.holo](../../.holo) and [ci](../../ci) for documentation regarding hologit and invoke, respectively. From 6c75ed2a9876536711b599fc003530734338805f Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Tue, 2 Jan 2024 23:02:10 -0500 Subject: [PATCH 18/26] docs(ci): update hologit docs --- .holo/README.md | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/.holo/README.md b/.holo/README.md index 979e360b18..b360bb817a 100644 --- a/.holo/README.md +++ b/.holo/README.md @@ -10,13 +10,9 @@ hologit allows: 3. Applying transformations to files as part of #1 - These transformations are called "lenses" -In this repository, we declare one holobranch named [release-candidate](../branches/release-candidate). -By projecting this holobranch in GitHub Actions, individual "candidate" branches end up containing -only the code relevant to infra/Kubernetes as well as Kubernetes code from the upstream [cluster-template](https://github.com/JarvusInnovations/cluster-template) -repository. Then, a PR from a `candidate/` to `releases/` (such as `releases/test`) will only show changes/content -relevant to infra in addition to `releases/*` branches only ever containing infra code. For example: +In this repository, we declare one holobranch named [kubernetes-workspace](../branches/kubernetes-workspace). +By projecting this holobranch in GitHub Actions, a tree containing only the code relevant to infra/Kubernetes +as well as Kubernetes code from the upstream [cluster-template](https://github.com/JarvusInnovations/cluster-template) +repository is generated. -1. Create a [PR making an infra-related change](https://github.com/cal-itp/data-infra/pull/2828) -2. Create and merge a [PR to deploy a candidate branch to test](https://github.com/cal-itp/data-infra/pull/2829) -3. Merge the PR from #1 -4. After merge, [PR to deploy the main candidate branch to prod](https://github.com/cal-itp/data-infra/pull/2832) +See [`ci/README.md`](../ci/README.md) for details on the pull request workflow for previewing and deploying Kubernetes changes. From 1279cc3bcedf6176aed42d64eacb90a7abcb3dc9 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Wed, 3 Jan 2024 11:05:47 -0500 Subject: [PATCH 19/26] fix(ci): update workflow step name --- .github/workflows/preview-kubernetes.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/preview-kubernetes.yml b/.github/workflows/preview-kubernetes.yml index 0bfda799d8..c885683e5c 100644 --- a/.github/workflows/preview-kubernetes.yml +++ b/.github/workflows/preview-kubernetes.yml @@ -17,7 +17,7 @@ env: GITHUB_TOKEN: ${{ secrets.GH_ACTIONS_TOKEN }} jobs: - release: + preview-kubernetes: runs-on: ubuntu-latest env: CLOUDSDK_CORE_PROJECT: cal-itp-data-infra From d8aa877fa7808c93ffd5f9f80f48901b64176430 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Wed, 3 Jan 2024 11:25:16 -0500 Subject: [PATCH 20/26] fix(ci): exclude hooks from helm diff --- ci/tasks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/tasks.py b/ci/tasks.py index eb718b1be5..f5f5b827ed 100644 --- a/ci/tasks.py +++ b/ci/tasks.py @@ -218,7 +218,8 @@ def diff( str(chart_path), f"--namespace={release.namespace}", values_str, - "-C 5", # only include 5 lines of context + "-C 5", # only include 5 lines of context + "--no-hooks", # exclude hooks that get recreated every upgrade from diff ] ), warn=True, From 865b0c3bdd9df48aeffb7ced5d3e22d937315cbc Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Wed, 3 Jan 2024 11:29:09 -0500 Subject: [PATCH 21/26] fix(docs): update links to gitops --- images/dask/README.md | 2 +- images/jupyter-singleuser/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/images/dask/README.md b/images/dask/README.md index bb2e8cb01b..dc06699e0b 100644 --- a/images/dask/README.md +++ b/images/dask/README.md @@ -22,4 +22,4 @@ docker build -t ghcr.io/cal-itp/data-infra/dask:[NEW VERSION TAG] . docker push ghcr.io/cal-itp/data-infra/dask:[NEW VERSION TAG] ``` -After deploying, you will likely need to change references to the version of the image in use by Kubernetes-managed services, such as [here](../../kubernetes/apps/charts/dask/values.yaml). See [our GitHub workflows documentation](../../.github/workflows#service-yml-workflows) for how to manage deployment of updated Kubernetes services and their associated workloads. +After deploying, you will likely need to change references to the version of the image in use by Kubernetes-managed services, such as [here](../../kubernetes/apps/charts/dask/values.yaml). See [our GitHub workflows documentation](../../kubernetes/README.md#gitops) for how to manage deployment of updated Kubernetes services and their associated workloads. diff --git a/images/jupyter-singleuser/README.md b/images/jupyter-singleuser/README.md index ac0f0dfe8d..bca477a588 100644 --- a/images/jupyter-singleuser/README.md +++ b/images/jupyter-singleuser/README.md @@ -20,4 +20,4 @@ docker build -t ghcr.io/cal-itp/data-infra/jupyter-singleuser:[NEW VERSION TAG] docker push ghcr.io/cal-itp/data-infra/jupyter-singleuser:[NEW VERSION TAG] ``` -After deploying, you will likely need to change references to the version of the image in use by Kubernetes-managed services, such as [here](../../kubernetes/apps/charts/jupyterhub/values.yaml). See [our GitHub workflows documentation](../../.github/workflows#service-yml-workflows) for how to manage deployment of updated Kubernetes services and their associated workloads. +After deploying, you will likely need to change references to the version of the image in use by Kubernetes-managed services, such as [here](../../kubernetes/apps/charts/jupyterhub/values.yaml). See [our GitHub workflows documentation](../../kubernetes/README.md#gitops) for how to manage deployment of updated Kubernetes services and their associated workloads. From c5383b8fa3c6aeb84d02b0777405bf141a8d9fda Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Wed, 3 Jan 2024 11:29:20 -0500 Subject: [PATCH 22/26] fix(ci): update path filters for new workflows --- .github/workflows/preview-kubernetes.yml | 2 +- .holo/branches/kubernetes-workspace/_data-infra.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/preview-kubernetes.yml b/.github/workflows/preview-kubernetes.yml index c885683e5c..21dbea5142 100644 --- a/.github/workflows/preview-kubernetes.yml +++ b/.github/workflows/preview-kubernetes.yml @@ -8,7 +8,7 @@ on: - '.holo/config.toml' - '.holo/sources/jarvus-cluster-template.toml' - '.holo/branches/releases/**' - - '.github/workflows/service-*' + - '.github/workflows/*-kubernetes.yml' - 'ci/**' - 'kubernetes/apps/**' - 'kubernetes/system/**' diff --git a/.holo/branches/kubernetes-workspace/_data-infra.toml b/.holo/branches/kubernetes-workspace/_data-infra.toml index a888f90082..7a29e7ad4e 100644 --- a/.holo/branches/kubernetes-workspace/_data-infra.toml +++ b/.holo/branches/kubernetes-workspace/_data-infra.toml @@ -1,3 +1,3 @@ [holomapping] -files = [ "ci/**", "kubernetes/apps/**", "kubernetes/system/**", ".github/workflows/service-*" ] +files = [ "ci/**", "kubernetes/apps/**", "kubernetes/system/**", ".github/workflows/*-kubernetes.yml" ] before = "*" From 20e7d160d17170a7d5b852bf332cf9cfc554e526 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Wed, 3 Jan 2024 11:40:31 -0500 Subject: [PATCH 23/26] docs(ci): add warnings about diff limitations --- ci/tasks.py | 15 +++++++++++---- kubernetes/README.md | 3 +++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/ci/tasks.py b/ci/tasks.py index f5f5b827ed..ebebaed880 100644 --- a/ci/tasks.py +++ b/ci/tasks.py @@ -231,11 +231,18 @@ def diff( if result.stdout: full_diff += result.stdout - msg = ( - f"```diff\n{full_diff}```\n" - if full_diff - else f"No {driver if driver else 'manifest'} changes found for {c.calitp_config.channel}.\n" + msg = "\n\n".join( + [ + "The following changes will be applied to the production Kubernetes cluster upon merge.", + "**BE AWARE** this may not reveal changes that have been manually applied to the cluster getting undone—applying manual changes to the cluster should be avoided.", + ( + f"```diff\n{full_diff}```\n" + if full_diff + else f"No {driver if driver else 'manifest'} changes found for {c.calitp_config.channel}.\n" + ) + ] ) + if outfile: print(f"writing {len(msg)=} to {outfile}", flush=True) with open(outfile, "w") as f: diff --git a/kubernetes/README.md b/kubernetes/README.md index 5d6a88a765..e4c856be89 100644 --- a/kubernetes/README.md +++ b/kubernetes/README.md @@ -15,6 +15,9 @@ The workflows described above also define their triggers. In general, developer 1. Check out a feature branch 2. Put up a PR for that feature branch, targeting `main` - `preview-kubernetes` will run and add a comment showing the diff of changes that will affect the production Kubernetes cluster + + **BE AWARE**: This diff may *NOT* reveal any changes that have been manually applied to the cluster being undone. The `helm diff` plugin used under the hood compares the new manifests against the saved snapshot of the last ones Helm deployed rather than the current state of the cluster. It has to work that way because that most accurately reflects how helm will apply the changes. This is why it is important to avoid making manual changes to the cluster. + 3. Merge the PR - `deploy-kubernetes` will run and deploy to `prod` this time From 800766003fbfc9bc69977755a7e51aa3a8bf390a Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Wed, 3 Jan 2024 22:56:00 -0500 Subject: [PATCH 24/26] feat(ci): add deploy-kubernetes workflow --- .github/workflows/deploy-kubernetes.yml | 65 +++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 .github/workflows/deploy-kubernetes.yml diff --git a/.github/workflows/deploy-kubernetes.yml b/.github/workflows/deploy-kubernetes.yml new file mode 100644 index 0000000000..57b5e6e91b --- /dev/null +++ b/.github/workflows/deploy-kubernetes.yml @@ -0,0 +1,65 @@ +name: Generate diff of Kubernetes changes + +on: + push: + branches: + - 'main' + paths: + - '.holo/config.toml' + - '.holo/sources/jarvus-cluster-template.toml' + - '.holo/branches/releases/**' + - '.github/workflows/*-kubernetes.yml' + - 'ci/**' + - 'kubernetes/apps/**' + - 'kubernetes/system/**' + +env: + GITHUB_TOKEN: ${{ secrets.GH_ACTIONS_TOKEN }} + +jobs: + preview-kubernetes: + runs-on: ubuntu-latest + env: + CLOUDSDK_CORE_PROJECT: cal-itp-data-infra + GKE_NAME: data-infra-apps + GKE_REGION: us-west1 + USE_GKE_GCLOUD_AUTH_PLUGIN: True + steps: + # Setup + - uses: actions/checkout@v4 + + - uses: 'google-github-actions/auth@v2' + with: + credentials_json: '${{ secrets.GCP_SA_KEY }}' + - uses: google-github-actions/setup-gcloud@v2 + - run: gcloud components install gke-gcloud-auth-plugin + - uses: google-github-actions/get-gke-credentials@v1 + with: + cluster_name: ${{ env.GKE_NAME }} + location: ${{ env.GKE_REGION }} + - run: curl -sSL https://install.python-poetry.org | python - + - name: Set up hologit + env: + BIO_RELEASE: 1.6.821 + run: | + curl -LO "https://github.com/biome-sh/biome/releases/download/v${BIO_RELEASE}/bio-${BIO_RELEASE}-x86_64-linux.tar.gz" + tar xzvf "bio-${BIO_RELEASE}-x86_64-linux.tar.gz" + sudo mv bio /usr/local/bin/bio + sudo bio pkg install --binlink jarvus/hologit + + git config user.name "Github Action $GITHUB_JOB" + git config user.email "$(whoami)@$(uname -n)" + + # Render Kubernetes content with parent underlay and checkout + - run: git holo project kubernetes-workspace --commit-to=kubernetes + - run: git checkout kubernetes + + # Diff and write back to PR + - id: diff + name: Run poetry invoke + shell: bash + working-directory: ci + run: | + poetry install + poetry run invoke secrets -f "./channels/prod.yaml" + poetry run invoke release -f "./channels/prod.yaml" From a508bf2164b27bd83af8fd0d7b9071fe572ba48f Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Wed, 3 Jan 2024 23:20:32 -0500 Subject: [PATCH 25/26] chore(ci): delint inline comments --- ci/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/tasks.py b/ci/tasks.py index ebebaed880..d9e0e9dd8e 100644 --- a/ci/tasks.py +++ b/ci/tasks.py @@ -218,8 +218,8 @@ def diff( str(chart_path), f"--namespace={release.namespace}", values_str, - "-C 5", # only include 5 lines of context - "--no-hooks", # exclude hooks that get recreated every upgrade from diff + "-C 5", # only include 5 lines of context + "--no-hooks", # exclude hooks that get recreated every upgrade from diff ] ), warn=True, From 0d557ccb318f95f24123ff5018cb334e1db5f351 Mon Sep 17 00:00:00 2001 From: Chris Alfano Date: Wed, 3 Jan 2024 23:24:29 -0500 Subject: [PATCH 26/26] chore(ci): delint trailing comma --- ci/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/tasks.py b/ci/tasks.py index d9e0e9dd8e..4a4badb343 100644 --- a/ci/tasks.py +++ b/ci/tasks.py @@ -239,7 +239,7 @@ def diff( f"```diff\n{full_diff}```\n" if full_diff else f"No {driver if driver else 'manifest'} changes found for {c.calitp_config.channel}.\n" - ) + ), ] )