Refactor: implement tag-based deployment (#2213)

.devcontainer/Dockerfile

@@ -12,10 +12,3 @@ RUN pip install -e .[dev,test]
 # docs requirements are in a separate file for the GitHub Action
 COPY docs/requirements.txt docs/requirements.txt
 RUN pip install -r docs/requirements.txt
-
-# install pre-commit environments in throwaway Git repository
-# https://stackoverflow.com/a/68758943
-COPY .pre-commit-config.yaml .
-RUN git init . && \
-    pre-commit install-hooks && \
-    rm -rf .git

.devcontainer/postAttach.sh

@@ -4,4 +4,9 @@ set -eu
 # initialize pre-commit
 
 git config --global --add safe.directory /calitp/app
-pre-commit install --overwrite
+
+# initialize hook environments
+pre-commit install --install-hooks --overwrite
+
+# manage commit-msg hooks
+pre-commit install --hook-type commit-msg

.dockerignore

@@ -1,4 +1,3 @@
-.git/
 .github/
 .vscode/
 **/__pycache__/

.github/workflows/check-migrations-and-messages.yml

@@ -1,5 +1,5 @@
 name: Check for up-to-date Django migrations and messages
-on: [push, pull_request]
+on: [push, pull_request, workflow_call]
 
 jobs:
   check-migrations-and-messages:

.github/workflows/deploy.yml

@@ -5,18 +5,44 @@ on:
   push:
     branches:
       - main
-      - test
-      - prod
+    tags:
+      # pre-release tag
+      - "202[3-9].[0-9][0-9].[0-9]+-rc[0-9]+"
+      # release tags
+      - "202[3-9].[0-9][0-9].[0-9]+"
 
 defaults:
   run:
     shell: bash
 
+concurrency:
+  # this ternary operator like expression gives us the name of the deployment environment (see https://docs.github.com/en/actions/learn-github-actions/expressions#example)
+  group: ${{ github.ref_type != 'tag' && github.ref_name || contains(github.ref, '-rc') && 'test' || 'prod' }}
+  cancel-in-progress: true
+
 jobs:
+  tests-ui:
+    uses: ./.github/workflows/tests-ui.yml
+    if: github.ref_type == 'tag'
+
+  tests-pytest:
+    uses: ./.github/workflows/tests-pytest.yml
+    if: github.ref_type == 'tag'
+
+  tests-cypress:
+    uses: ./.github/workflows/tests-cypress.yml
+    if: github.ref_type == 'tag'
+
+  check-migrations-and-messages:
+    uses: ./.github/workflows/check-migrations-and-messages.yml
+    if: github.ref_type == 'tag'
+
   deploy:
     runs-on: ubuntu-latest
-    environment: ${{ github.ref_name }}
-    concurrency: ${{ github.ref_name }}
+    needs:
+      [tests-ui, tests-pytest, tests-cypress, check-migrations-and-messages]
+    if: (!cancelled())
+    environment: ${{ github.ref_type != 'tag' && github.ref_name || contains(github.ref, '-rc') && 'test' || 'prod' }}
 
     steps:
       - name: Checkout
@@ -69,3 +95,23 @@ jobs:
           app-name: ${{ vars.AZURE_WEBAPP_NAME }}
           images: ghcr.io/${{ github.repository }}:${{ github.sha }}
           publish-profile: ${{ secrets.AZURE_WEBAPP_PUBLISH_PROFILE }}
+
+  release:
+    needs: deploy
+    if: ${{ github.ref_type == 'tag' && !contains(github.ref, '-rc') }}
+    runs-on: ubuntu-latest
+    permissions:
+      # https://github.com/softprops/action-gh-release#permissions
+      contents: write
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Release
+        uses: softprops/action-gh-release@v2
+        with:
+          prerelease: false
+          generate_release_notes: true

.github/workflows/tests-cypress.yml

@@ -1,6 +1,6 @@
 name: Cypress tests
 
-on: push
+on: [push, workflow_call]
 
 defaults:
   run:

.github/workflows/tests-pytest.yml

@@ -1,6 +1,6 @@
 name: Pytest
 
-on: [push, pull_request]
+on: [push, pull_request, workflow_call]
 
 jobs:
   pytest:

.github/workflows/tests-ui.yml

@@ -1,6 +1,7 @@
 name: UI & a11y tests
 
 on:
+  workflow_call:
   workflow_dispatch:
   pull_request:
     branches: [main, test, prod]

appcontainer/Dockerfile

@@ -7,6 +7,9 @@ RUN python -m pip install --upgrade pip
 COPY appcontainer/nginx.conf /etc/nginx/nginx.conf
 COPY appcontainer/proxy.conf /calitp/run/proxy.conf
 
+# copy files needed for version metadata
+COPY .git .git
+
 # copy source files
 COPY manage.py manage.py
 COPY bin bin

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "benefits"
-version = "2024.07.1"
+dynamic = ["version"]
 description = "Cal-ITP Benefits is an application that enables automated eligibility verification and enrollment for transit benefits onto customers’ existing contactless bank (credit/debit) cards."
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -27,6 +27,7 @@ dev = [
     "djlint",
     "flake8",
     "pre-commit",
+    "setuptools_scm>=8"
 ]
 test = [
     "coverage",
@@ -42,7 +43,7 @@ Documentation = "https://docs.calitp.org/benefits"
 Issues = "https://github.com/cal-itp/benefits/issues"
 
 [build-system]
-requires = ["setuptools>=65", "wheel"]
+requires = ["setuptools>=65", "wheel", "setuptools-scm>=8"]
 build-backend = "setuptools.build_meta"
 
 [tool.black]
@@ -78,3 +79,6 @@ markers = [
 
 [tool.setuptools]
 packages = ["benefits"]
+
+[tool.setuptools_scm]
+# intentionally left blank, but we need the section header to activate the tool

terraform/azure-pipelines.yml

@@ -4,8 +4,8 @@ trigger:
   branches:
     include:
       - main
-      - test
-      - prod
+      - refs/tags/20??.??.?*-rc?*
+      - refs/tags/20??.??.?*
   # only run for changes to Terraform files
   paths:
     include:
@@ -26,8 +26,14 @@ stages:
             value: $[variables['Build.SourceBranchName']]
           - name: TARGET
             value: $[variables['System.PullRequest.TargetBranch']]
+          - name: IS_TAG
+            value: $[startsWith(variables['Build.SourceBranch'], 'refs/tags/')]
         steps:
-          - bash: python terraform/pipeline/workspace.py
+          - bash: |
+              python terraform/pipeline/workspace.py
+
+              TAG_TYPE=$(python terraform/pipeline/tag.py)
+              echo "##vso[task.setvariable variable=tag_type;isOutput=true]$TAG_TYPE"
             displayName: Set environment-related variables
             # save the values
             # https://learn.microsoft.com/en-us/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch#use-outputs-in-a-different-job
@@ -39,18 +45,22 @@ stages:
         condition: eq(dependencies.environment.outputs['env_select.service_connection'], 'Development')
         variables:
           workspace: $[ dependencies.environment.outputs['env_select.workspace'] ]
+          tag_type: $[ dependencies.environment.outputs['env_select.tag_type'] ]
         steps:
           - template: pipeline/deploy.yml
             parameters:
               service_connection: Development
               workspace: $(workspace)
+              tag_type: $(tag_type)
       - job: prod
         dependsOn: environment
         condition: eq(dependencies.environment.outputs['env_select.service_connection'], 'Production')
         variables:
           workspace: $[ dependencies.environment.outputs['env_select.workspace'] ]
+          tag_type: $[ dependencies.environment.outputs['env_select.tag_type'] ]
         steps:
           - template: pipeline/deploy.yml
             parameters:
               service_connection: Production
               workspace: $(workspace)
+              tag_type: $(tag_type)

terraform/pipeline/deploy.yml

@@ -3,6 +3,8 @@ parameters:
     type: string
   - name: workspace
     type: string
+  - name: tag_type
+    type: string
 
 steps:
   # https://github.com/microsoft/azure-pipelines-terraform/tree/main/Tasks/TerraformInstaller#readme
@@ -48,7 +50,13 @@ steps:
       # service connection
       environmentServiceNameAzureRM: "${{ parameters.service_connection }}"
     # the plan is done as part of the apply (below), so don't bother doing it twice
-    condition: notIn(variables['Build.SourceBranchName'], 'main', 'test', 'prod')
+    condition: |
+      ${{ and(
+            ne(variables['Build.SourceBranchName'], 'main'),
+            ne(parameters.tag_type, 'test'),
+            ne(parameters.tag_type, 'prod')
+          )
+      }}
   - task: TerraformTaskV3@3
     displayName: Terraform apply
     inputs:
@@ -59,5 +67,11 @@ steps:
       workingDirectory: "$(System.DefaultWorkingDirectory)/terraform"
       # service connection
       environmentServiceNameAzureRM: "${{ parameters.service_connection }}"
-    # only run on certain branches
-    condition: in(variables['Build.SourceBranchName'], 'main', 'test', 'prod')
+    # only run on main branch OR if it's a tag for test or prod
+    condition: |
+      ${{ or(
+            eq(variables['Build.SourceBranchName'], 'main'),
+            eq(parameters.tag_type, 'test'),
+            eq(parameters.tag_type, 'prod')
+          )
+      }}

terraform/pipeline/tag.py

@@ -0,0 +1,19 @@
+import os
+import re
+
+REASON = os.environ["REASON"]
+# use variable corresponding to tag triggers
+SOURCE = os.environ["INDIVIDUAL_SOURCE"]
+IS_TAG = os.environ["IS_TAG"].lower() == "true"
+
+if REASON == "IndividualCI" and IS_TAG:
+    if re.fullmatch(r"20\d\d.\d\d.\d+-rc\d+", SOURCE):
+        tag_type = "test"
+    elif re.fullmatch(r"20\d\d.\d\d.\d+", SOURCE):
+        tag_type = "prod"
+    else:
+        tag_type = None
+else:
+    tag_type = None
+
+print(tag_type)

terraform/pipeline/workspace.py

@@ -2,6 +2,7 @@
 time) so that all the necessary pipeline variables are available."""
 
 import os
+import re
 import sys
 
 REASON = os.environ["REASON"]
@@ -11,22 +12,27 @@
 SOURCE = os.environ.get("OTHER_SOURCE") or os.environ["INDIVIDUAL_SOURCE"]
 
 TARGET = os.environ["TARGET"]
+IS_TAG = os.environ["IS_TAG"].lower() == "true"
 
-# branch to workspace mapping
-WORKSPACES = {"main": "dev", "test": "test", "prod": "default"}
 # workspace to service connection mapping
 SERVICE_CONNECTIONS = {"dev": "Development", "test": "Development", "default": "Production"}
 
-if REASON == "PullRequest" and TARGET in WORKSPACES:
-    # it's a pull request against one of the environment branches, so use the
-    # target branch
-    workspace = WORKSPACES[TARGET]
-elif REASON in ["IndividualCI", "Manual"] and SOURCE in WORKSPACES:
-    # it's being run on one of the environment branches, so use that
-    workspace = WORKSPACES[SOURCE]
+if REASON == "PullRequest" and TARGET == "main":
+    # it's a pull request against main, this is for the dev environment
+    environment = "dev"
+elif REASON in ["IndividualCI", "Manual"] and SOURCE == "main":
+    # it's being run on the main branch, this is for the dev environment
+    environment = "dev"
+elif REASON in ["IndividualCI"] and IS_TAG and re.fullmatch(r"20\d\d.\d\d.\d+-rc\d+", SOURCE):
+    environment = "test"
+elif REASON in ["IndividualCI"] and IS_TAG and re.fullmatch(r"20\d\d.\d\d.\d+", SOURCE):
+    environment = "prod"
 else:
     # default to running against dev
-    workspace = "dev"
+    environment = "dev"
+
+# matching logic in ../init.sh
+workspace = "default" if environment == "prod" else environment
 
 service_connection = SERVICE_CONNECTIONS[workspace]
 

tests/pytest/test_version.py

@@ -6,4 +6,4 @@ def test_version():
 
     assert __version__ is not None
     assert __version__ == VERSION
-    assert re.match(r"\d{4}\.\d{1,2}\.\d+", __version__)
+    assert re.match(r"\d+\.\d+\..+", __version__)