feat: add the supabase admin agent to the ami build

Author: Crispy1975

.github/actions/download-supabase-artifacts/action.yml

@@ -0,0 +1,72 @@
+name: "Download Supabase project artifacts"
+description: "Authenticate with AWS shared account and download artifacts from S3"
+
+inputs:
+  region:
+    description: "AWS region"
+    required: true
+  auth-role:
+    description: "Initial role to assume using GitHub OIDC"
+    required: true
+  download-role:
+    description: "Role to assume for S3 access"
+    required: true
+  bucket:
+    description: "S3 bucket name"
+    required: true
+  artifacts:
+    description: "Newline-separated list of artefact filenames to download"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: GitHub OIDC Auth
+      uses: aws-actions/[email protected]
+      with:
+        aws-region: ${{ inputs.region }}
+        role-to-assume: ${{ inputs.auth-role }}
+        role-session-name: github-oidc-session
+
+    - name: Assume Destination Role
+      uses: aws-actions/[email protected]
+      with:
+        aws-region: ${{ inputs.region }}
+        role-to-assume: ${{ inputs.download-role }}
+        role-session-name: s3-access
+        role-skip-session-tagging: true
+        role-chaining: true
+
+    - name: Download artifacts from S3
+      shell: bash
+      run: |
+        set -euo pipefail
+        mkdir -p /tmp/supabase-dist
+
+        bucket="${{ inputs.bucket }}"
+        mapfile -t entries <<< "${{ inputs.artifacts }}"
+
+        for entry in "${entries[@]}"; do
+          # Trim whitespace
+          entry="${entry#"${entry%%[![:space:]]*}"}"
+          entry="${entry%"${entry##*[![:space:]]}"}"
+
+          # Skip empty lines
+          if [[ -z "$entry" ]]; then
+            continue
+          fi
+
+          IFS=',' read -r tool version platform <<< "$entry"
+
+          # Validate expected parts
+          if [[ -z "$tool" || -z "$version" || -z "$platform" ]]; then
+            echo "Warning: skipping malformed artifact entry: '$entry'" >&2
+            continue
+          fi
+
+          filename="${tool}-${version}-${platform}.tar.xz"
+          s3_path="s3://${bucket}/${tool}/v${version}/${filename}"
+
+          echo "Downloading $s3_path"
+          aws s3 cp --no-progress "$s3_path" "/tmp/supabase-dist/$filename"
+        done

.github/workflows/nix-build.yml

@@ -79,19 +79,19 @@ jobs:
             trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
       - name: Build psql bundle
         run: >
-            nix run "github:Mic92/nix-fast-build?rev=b1dae483ab7d4139a6297e02b6de9e5d30e43d48" 
-            -- --skip-cached --no-nom 
-            --flake ".#checks.$(nix eval --raw --impure --expr 'builtins.currentSystem')"
+          nix run "github:Mic92/nix-fast-build?rev=b1dae483ab7d4139a6297e02b6de9e5d30e43d48" 
+          -- --skip-cached --no-nom 
+          --flake ".#checks.$(nix eval --raw --impure --expr 'builtins.currentSystem')"
         env:
           AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
           AWS_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }}
-  
+
   run-testinfra:
     needs: build-run-image
     if: ${{ success() }}
     uses: ./.github/workflows/testinfra-ami-build.yml
-    
+
   run-tests:
     needs: build-run-image
     if: ${{ success() }}

.github/workflows/testinfra-ami-build.yml

@@ -37,7 +37,7 @@ jobs:
             ubuntu_release: focal
             ubuntu_version: 20.04
             mcpu: neoverse-n1
-    runs-on: ${{ matrix.runner }}    
+    runs-on: ${{ matrix.runner }}
     timeout-minutes: 150
     permissions:
       contents: write
@@ -53,6 +53,18 @@ jobs:
         with:
           cmd: yq 'to_entries | map(select(.value|type == "!!str")) |  map(.key + "=" + .value) | join("\n")' 'ansible/vars.yml'
 
+      - name: Download Supabase Artifacts
+        id: download-artifacts
+        uses: ./.github/actions/download-supabase-artifacts
+        with:
+          region: ap-southeast-1
+          auth-role: ${{ secrets.SUPABASE_GITHUB_OIDC_ROLE }}
+          download-role: ${{ secrets.SHARED_AWS_ARTIFACTS_S3_ROLE }}
+          bucket: ${{ secrets.SHARED_AWS_ARTIFACTS_BUCKET }}
+          artifacts: |
+            supabase-admin-agent,1.4.35,linux-arm64
+            supabase-admin-agent,1.4.35,linux-amd64
+
       - run: docker context create builders
 
       - uses: docker/setup-buildx-action@v3
@@ -73,7 +85,7 @@ jobs:
           echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl
           # Ensure there's a newline at the end of the file
           echo "" >> common-nix.vars.pkr.hcl
-  
+
       - name: Build AMI stage 1
         run: |
           packer init amazon-arm64-nix.pkr.hcl
@@ -84,7 +96,7 @@ jobs:
         run: |
           packer init stage2-nix-psql.pkr.hcl
           GIT_SHA=${{github.sha}}
-          packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl"  -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=ap-southeast-1" -var 'ami_regions=["ap-southeast-1"]' -var "force-deregister=true" -var "git_sha=${GITHUB_SHA}"  stage2-nix-psql.pkr.hcl 
+          packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl"  -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=ap-southeast-1" -var 'ami_regions=["ap-southeast-1"]' -var "force-deregister=true" -var "git_sha=${GITHUB_SHA}"  stage2-nix-psql.pkr.hcl
 
       - name: Run tests
         timeout-minutes: 10
@@ -93,8 +105,8 @@ jobs:
         run: |
           # TODO: use poetry for pkg mgmt
           pip3 install boto3 boto3-stubs[essential] docker ec2instanceconnectcli pytest pytest-testinfra[paramiko,docker] requests
-          pytest -vv -s testinfra/test_ami_nix.py 
-      
+          pytest -vv -s testinfra/test_ami_nix.py
+
       - name: Cleanup resources on build cancellation
         if: ${{ cancelled() }}
         run: |
@@ -111,7 +123,7 @@ jobs:
           # Define AMI name patterns
           STAGE1_AMI_NAME="supabase-postgres-ci-ami-test-stage-1"
           STAGE2_AMI_NAME="${{ steps.random.outputs.random_string }}"
-          
+
           # Function to deregister AMIs by name pattern
           deregister_ami_by_name() {
             local ami_name_pattern=$1
@@ -121,7 +133,7 @@ jobs:
               aws ec2 deregister-image --region ap-southeast-1 --image-id $ami_id
             done
           }
-          
+
           # Deregister AMIs
           deregister_ami_by_name "$STAGE1_AMI_NAME"
           deregister_ami_by_name "$STAGE2_AMI_NAME"

ansible/files/permission_check.py

@@ -94,11 +94,17 @@
     "systemd-coredump": [
         {"groupname": "systemd-coredump", "username": "systemd-coredump"}
     ],
+    "supabase-admin-agent": [
+        {"groupname": "supabase-admin-agent", "username": "supabase-admin-agent"},
+        {"groupname": "admin", "username": "supabase-admin-agent"},
+        {"groupname": "salt", "username": "supabase-admin-agent"},
+    ],
 }
 
 # postgresql.service is expected to mount /etc as read-only
 expected_mount = "/etc ro"
 
+
 # This program depends on osquery being installed on the system
 # Function to run osquery
 def run_osquery(query):
@@ -154,6 +160,7 @@ def check_nixbld_users():
 
     print("All nixbld users are in the 'nixbld' group.")
 
+
 def check_postgresql_mount():
     # processes table has the nix .postgres-wrapped path as the
     # binary path, rather than /usr/lib/postgresql/bin/postgres which
@@ -182,6 +189,7 @@ def check_postgresql_mount():
 
     print("postgresql.service mounts /etc as read-only.")
 
+
 def main():
     parser = argparse.ArgumentParser(
         prog="Supabase Postgres Artifact Permissions Checker",
@@ -234,6 +242,7 @@ def main():
         "postgrest",
         "tcpdump",
         "systemd-coredump",
+        "supabase-admin-agent",
     ]
     if not qemu_artifact:
         usernames.append("ec2-instance-connect")
@@ -251,5 +260,6 @@ def main():
     # Check if postgresql.service is using a read-only mount for /etc
     check_postgresql_mount()
 
+
 if __name__ == "__main__":
     main()

ansible/files/supabase_admin_agent_config/supabase-admin-agent.sudoers.conf

@@ -0,0 +1,2 @@
+%supabase-admin-agent ALL= NOPASSWD: /usr/bin/salt-call
+%supabase-admin-agent ALL= NOPASSWD: /usr/bin/gpg --homedir {{ gpgdir }} --import, /usr/bin/gpg --homedir {{ gpgdir }} --list-secret-keys *

ansible/files/supabase_admin_agent_config/supabase-admin-agent_salt.service

@@ -0,0 +1,19 @@
+[Unit]
+Description=Configuration management via supabase-admin-agent salt
+After=network.target
+
+[Service]
+Type=oneshot
+ExecStart=/opt/supabase-admin-agent/supabase-admin-agent --config /opt/supabase-admin-agent/config.yaml salt --apply --store-result
+User=supabase-admin-agent
+Group=supabase-admin-agent
+StandardOutput=journal
+StandardError=journal
+StateDirectory=supabase-admin-agent
+CacheDirectory=supabase-admin-agent
+
+# Security hardening
+PrivateTmp=true
+
+[Install]
+WantedBy=multi-user.target

ansible/files/supabase_admin_agent_config/supabase-admin-agent_salt.timer

@@ -0,0 +1,13 @@
+[Unit]
+Description=Run Supabase supabase-admin-agent salt on a schedule
+Requires=supabase-admin-agent_salt.service
+
+[Timer]
+OnCalendar=*:0/10
+# Random delay up to {{ splay }} seconds splay
+RandomizedDelaySec={{ splay }}
+AccuracySec=1s
+Persistent=true
+
+[Install]
+WantedBy=timers.target

ansible/tasks/internal/supabase-admin-agent.yml

@@ -0,0 +1,87 @@
+- name: supabase-admin-agent - system group
+  group:
+    name: supabase-admin-agent
+    system: yes
+
+- name: supabase-admin-agent - system user
+  user:
+    name: supabase-admin-agent
+    group: supabase-admin-agent
+    groups: admin,salt
+    append: yes
+    system: yes
+    shell: /bin/sh
+
+- name: supabase-admin-agent - config dir
+  file:
+    path: /opt/supabase-admin-agent
+    owner: supabase-admin-agent
+    state: directory
+
+- name: supabase-admin-agent - gpg dir
+  file:
+    path: /etc/salt/gpgkeys
+    owner: root
+    group: salt
+    state: directory
+
+- name: give supabase-admin-agent user permissions
+  copy:
+    src: files/supabase_admin_agent_config/supabase-admin-agent.sudoers.conf
+    dest: /etc/sudoers.d/supabase-admin-agent
+    mode: "0644"
+
+- name: Setting arch (x86)
+  set_fact:
+    arch: "x86"
+  when: platform == "amd64"
+
+- name: Setting arch (arm)
+  set_fact:
+    arch: "arm64"
+  when: platform == "arm64"
+
+- name: Copy supabase-admin-agent archive
+  copy:
+    src: "/tmp/supabase-dist/supabase-admin-agent-{{ supabase_admin_agent_release }}-linux-{{ arch }}.tar.xz"
+    dest: "/tmp/supabase-admin-agent.tar.xz"
+    mode: "0755"
+
+- name: supabase-admin-agent - unpack archive in /opt
+  unarchive:
+    remote_src: yes
+    src: /tmp/supabase-admin-agent.tar.xz
+    dest: /opt/supabase-admin-agent/
+    owner: supabase-admin-agent
+    extra_opts:
+      - --strip-components=1
+
+- name: supabase-admin-agent - create symlink
+  ansible.builtin.file:
+    path: /opt/supabase-admin-agent/supabase-admin-agent
+    src: "/opt/supabase-admin-agent/supabase-admin-agent-linux-{{ arch }}"
+    state: link
+    owner: supabase-admin-agent
+    mode: "0755"
+    force: yes
+
+- name: supabase-admin-agent - create salt systemd timer file
+  copy:
+    src: files/supabase_admin_agent_config/supabase-admin-agent_salt.timer
+    dest: /etc/systemd/system/supabase-admin-agent_salt.timer
+
+- name: supabase-admin-agent - create salt service file
+  copy:
+    src: files/supabase_admin_agent_config/supabase-admin-agent_salt.service
+    dest: /etc/systemd/system/supabase-admin-agent_salt.service
+
+- name: supabase-admin-agent - reload systemd
+  systemd:
+    daemon_reload: yes
+
+# Initially ensure supabase-admin-agent is installed but not started
+- name: supabase-admin-agent - DISABLE service
+  systemd:
+    name: supabase-admin-agent_salt
+    enabled: no
+    state: stopped