Update dbt workflow template

github/serverless/dbt/deploy.yml

@@ -1,109 +1,128 @@
-name: Serverless Prod Deployment
+name: Serverless Deployment
 on:
   push:
     branches:
       - "main"
       - "master"
+  pull_request:
+    types: [opened, synchronize, reopened, closed]
 
 concurrency:
   # Cancel in-progress deploys to same branch
   group: ${{ github.ref }}/deploy
   cancel-in-progress: true
 env:
   DAGSTER_CLOUD_URL: ${{ secrets.DAGSTER_CLOUD_URL }}
-  DAGSTER_CLOUD_ORGANIZATION: ${{ secrets.DAGSTER_CLOUD_ORGANIZATION }}
   DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }}
   ENABLE_FAST_DEPLOYS: 'true'
   PYTHON_VERSION: '3.8'
+  DAGSTER_PROJECT_DIR: '.'
   DAGSTER_CLOUD_FILE: 'dagster_cloud.yaml'
   DAGSTER_PROJECT_NAME: 'dagster_dbt_scaffold'
+  DAGSTER_CLOUD_ORGANIZATION: ${{ secrets.DAGSTER_CLOUD_ORGANIZATION }}
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
 jobs:
   dagster_cloud_default_deploy:
     name: Dagster Serverless Deploy
     runs-on: ubuntu-20.04
-    outputs:
-      build_info: ${{ steps.parse-workspace.outputs.build_info }}
 
     steps:
       - name: Prerun Checks
         id: prerun
         uses: dagster-io/dagster-cloud-action/actions/utils/[email protected]
 
-      - name: Launch Docker Deploy
-        if: steps.prerun.outputs.result == 'docker-deploy'
-        id: parse-workspace
-        uses: dagster-io/dagster-cloud-action/actions/utils/[email protected]
-        with:
-          dagster_cloud_file: $DAGSTER_CLOUD_FILE
-
       - name: Checkout
-        if: steps.prerun.outputs.result == 'pex-deploy'
         uses: actions/checkout@v3
         with:
           ref: ${{ github.head_ref }}
-          path: project-repo
 
+      # Validate dagster_cloud.yaml and the connection to dagster.cloud
       - name: Validate configuration
         id: ci-validate
-        if: steps.prerun.outputs.result == 'pex-deploy'
+        if: steps.prerun.outputs.result != 'skip'
         uses: dagster-io/dagster-cloud-action/actions/utils/[email protected]
         with:
-          command: "ci check --project-dir project-repo --dagster-cloud-yaml-path ${{ env.DAGSTER_CLOUD_FILE }}"
+          command: "ci check --project-dir ${{ env.DAGSTER_PROJECT_DIR }} --dagster-cloud-yaml-path ${{ env.DAGSTER_CLOUD_FILE }}"
 
+      # Parse dagster_cloud.yaml, detect if this is branch deployment and initialize the build session
       - name: Initialize build session
         id: ci-init
-        if: steps.prerun.outputs.result == 'pex-deploy'
         uses: dagster-io/dagster-cloud-action/actions/utils/[email protected]
         with:
-          project_dir: project-repo
+          project_dir: ${{ env.DAGSTER_PROJECT_DIR }}
           dagster_cloud_yaml_path: ${{ env.DAGSTER_CLOUD_FILE }}
+          # The full deployment name. If this run is for a PR, this value is ignored and a branch
+          # deployment is used.
           deployment: 'prod'
 
+
+      # If using fast build, build the PEX
+      # First ensure the correct Python version is installed
+      - name: Set up Python ${{ env.PYTHON_VERSION }} for target
+        id: custom-python-version
+        if: steps.pre-run.outputs.result == 'pex-deploy' && ${{ env.PYTHON_VERSION != '3.8' }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+      - name: Install setuptools
+        if: steps.pre-run.outputs.result == 'pex-deploy' && ${{ env.PYTHON_VERSION != '3.8' }}
+        run: ${{ steps.custom-python-version.outputs.python-path }} -m pip install setuptools
+        shell: bash
+
       - name: Prepare DBT project for deployment
         if: steps.prerun.outputs.result == 'pex-deploy'
         # --upgrade-strategy eager picks up newer packages that are required for things to work
         run: |
           pip install pip --upgrade
-          cd project-repo/${{ env.DAGSTER_PROJECT_NAME }}
+          cd ${{ env.DAGSTER_PROJECT_DIR }}/${{ env.DAGSTER_PROJECT_NAME }}
           pip install . --upgrade --upgrade-strategy eager
-          dagster-dbt project prepare-for-deployment --file ${{ env.DAGSTER_PROJECT_NAME }}/project.py
+          dagster-dbt project prepare-and-package --file ${{ env.DAGSTER_PROJECT_NAME }}/project.py
           # The cli command below can be used to manage syncing the prod manifest to branches if state_path is set on the DbtProject
           # dagster-cloud ci dagster-dbt project manage-state --file ${{ env.DAGSTER_PROJECT_NAME }}/project.py
         shell: bash
 
-      - name: Python Executable Deploy
+      - name: Run PEX build
+        id: run-pex-build
         if: steps.prerun.outputs.result == 'pex-deploy'
-        uses: dagster-io/dagster-cloud-action/actions/build_deploy_python_executable@v0.1
+        uses: dagster-io/dagster-cloud-action/actions/utils/dagster-cloud-cli@v0.1
         with:
-          dagster_cloud_file: "$GITHUB_WORKSPACE/project-repo/$DAGSTER_CLOUD_FILE"
-          build_output_dir: "$GITHUB_WORKSPACE/build"
-          python_version: "${{ env.PYTHON_VERSION }}"
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-  dagster_cloud_docker_deploy:
-    name: Docker Deploy
-    runs-on: ubuntu-20.04
-    if: needs.dagster_cloud_default_deploy.outputs.build_info
-    needs: dagster_cloud_default_deploy
-    strategy:
-      fail-fast: false
-      matrix:
-        location: ${{ fromJSON(needs.dagster_cloud_default_deploy.outputs.build_info) }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
+          command: "ci build --build-strategy=python-executable --python-version ${{ env.PYTHON_VERSION }}  --pex-deps-cache-from='${{ github.repository }}' --pex-deps-cache-to='${{ github.repository }}'"
+
+
+      # Otherwise, enable buildx for caching and build the Docker image
+      - name: Set up Docker Buildx
+        if: steps.prerun.outputs.result == 'docker-deploy'
+        uses: docker/setup-buildx-action@v2
+
+      - name: Run Docker build
+        id: run-docker-build
+        if: steps.prerun.outputs.result == 'docker-deploy'
+        uses: dagster-io/dagster-cloud-action/actions/utils/[email protected]
         with:
-          ref: ${{ github.head_ref }}
-      - name: Build and deploy to Dagster Cloud serverless
-        uses: dagster-io/dagster-cloud-action/actions/[email protected]
+          command: "ci build --build-strategy=docker --python-version ${{ env.PYTHON_VERSION }}"
+
+
+      # Deploy all code locations in this build session to Dagster Cloud
+      - name: Deploy to Dagster Cloud
+        id: ci-deploy
+        if: steps.prerun.outputs.result != 'skip'
+        uses: dagster-io/dagster-cloud-action/actions/utils/[email protected]
+        with:
+          command: "ci deploy"
+
+      # Update a PR comment - this runs always() so the comment is updated on success and failure
+      - name: Update PR comment for branch deployments
+        id: ci-notify
+        if: steps.prerun.outputs.result != 'skip'
+        uses: dagster-io/dagster-cloud-action/actions/utils/[email protected]
+        with:
+          command: "ci notify --project-dir=${{ env.DAGSTER_PROJECT_DIR }}"
+
+      # Generate a summary that shows up on the Workflow Summary page
+      - name: Generate a summary
+        id: ci-summary
+        if: steps.prerun.outputs.result != 'skip'
+        uses: dagster-io/dagster-cloud-action/actions/utils/[email protected]
         with:
-          dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }}
-          location: ${{ toJson(matrix.location) }}
-          base_image: "python:${{ env.PYTHON_VERSION }}-slim"
-          # Uncomment to pass through Github Action secrets as a JSON string of key-value pairs
-          # env_vars: ${{ toJson(secrets) }}
-          organization_id: ${{ secrets.ORGANIZATION_ID }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          command: "ci status --output-format=markdown >> $GITHUB_STEP_SUMMARY"