feat: snowpipe streaming #18036
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: builds | |
| on: | |
| release: | |
| types: [ created ] | |
| push: | |
| branches: | |
| - master | |
| - main | |
| pull_request: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.sha }} | |
| cancel-in-progress: true | |
| env: | |
| arch_amd64: amd64 | |
| arch_arm64: arm64 | |
| docker_oss_images: | | |
| name=rudderlabs/develop-rudder-server | |
| name=rudderlabs/rudder-server,enable=${{ github.ref == format('refs/heads/{0}', 'master') || github.event_name == 'release' }} | |
| docker_oss_tags: | | |
| type=ref,event=branch | |
| type=raw,value=1-alpine,enable=${{ github.event_name == 'release' && !github.event.release.prerelease }} | |
| type=raw,value=latest,enable=${{ github.event_name == 'release' && !github.event.release.prerelease }} | |
| type=raw,value=${{ github.head_ref }},enable=${{ github.event_name == 'pull_request' }} | |
| type=semver,pattern={{version}} | |
| type=semver,pattern={{major}}.{{minor}} | |
| type=semver,pattern={{major}} | |
| docker_ent_images: | | |
| name=rudderstack/develop-rudder-server-enterprise | |
| name=rudderstack/rudder-server-enterprise,enable=${{ github.ref == format('refs/heads/{0}', 'master') || github.event_name == 'release' }} | |
| docker_ent_tags: | | |
| type=ref,event=branch | |
| type=raw,value=latest,enable=${{ github.event_name == 'release' }} | |
| type=raw,value=${{ github.head_ref }},enable=${{ github.event_name == 'pull_request' }} | |
| type=semver,pattern={{version}} | |
| type=semver,pattern={{major}}.{{minor}} | |
| type=semver,pattern={{major}} | |
| docker_sbsvc_images: | | |
| name=rudderstack/develop-suppression-backup-service | |
| name=rudderstack/suppression-backup-service,enable=${{ github.ref == format('refs/heads/{0}', 'master') || github.event_name == 'release' }} | |
| docker_sbsvc_tags: | | |
| type=ref,event=branch | |
| type=raw,value=latest,enable=${{ github.event_name == 'release' }} | |
| type=raw,value=${{ github.head_ref }},enable=${{ github.event_name == 'pull_request' }} | |
| type=semver,pattern={{version}} | |
| type=semver,pattern={{major}}.{{minor}} | |
| type=semver,pattern={{major}} | |
| jobs: | |
| docker-oss-meta: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| labels: ${{ steps.meta.outputs.labels }} | |
| build-date: ${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.created'] }} | |
| version: ${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.version'] }} | |
| revision: ${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.revision'] }} | |
| tags: ${{ steps.meta.outputs.tags }} | |
| arm64_tags: ${{ steps.arm64_meta.outputs.tags }} | |
| arm64_labels: ${{ steps.arm64_meta.outputs.labels }} | |
| amd64_tags: ${{ steps.amd64_meta.outputs.tags }} | |
| amd64_labels: ${{ steps.amd64_meta.outputs.labels }} | |
| steps: | |
| - name: Docker meta | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{env.docker_oss_images}} | |
| tags: ${{env.docker_oss_tags}} | |
| - name: Docker arm64 meta | |
| id: arm64_meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{env.docker_oss_images}} | |
| tags: ${{env.docker_oss_tags}} | |
| flavor: | | |
| suffix=-${{env.arch_arm64}},onlatest=true | |
| - name: Docker amd64 meta | |
| id: amd64_meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{env.docker_oss_images}} | |
| tags: ${{env.docker_oss_tags}} | |
| flavor: | | |
| suffix=-${{env.arch_amd64}},onlatest=true | |
| docker-oss: | |
| needs: | |
| - docker-oss-meta | |
| strategy: | |
| matrix: | |
| build-config: | |
| - os: [ self-hosted, Linux, ARM64 ] | |
| tags: ${{needs.docker-oss-meta.outputs.arm64_tags}} | |
| labels: ${{needs.docker-oss-meta.outputs.arm64_labels}} | |
| platform: linux/arm64 | |
| - os: ubuntu-latest | |
| tags: ${{needs.docker-oss-meta.outputs.amd64_tags}} | |
| labels: ${{needs.docker-oss-meta.outputs.amd64_labels}} | |
| platform: linux/amd64 | |
| runs-on: ${{matrix.build-config.os}} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Login to DockerHub | |
| uses: docker/[email protected] | |
| with: | |
| username: rudderlabs | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Build image (no-push) | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| platforms: ${{ matrix.build-config.platform }} | |
| push: false | |
| tags: ${{ matrix.build-config.tags }} | |
| labels: ${{ matrix.build-config.labels }} | |
| build-args: | | |
| BUILD_DATE=${{ needs.docker-oss-meta.outputs.build-date }} | |
| VERSION=${{ needs.docker-oss-meta.outputs.version }} | |
| COMMIT_HASH=${{ github.sha }} | |
| REVISION=${{ needs.docker-oss-meta.outputs.revision }} | |
| outputs: type=docker,dest=${{ runner.temp }}/local-rudder-server.tar | |
| cache-to: type=local,dest=${{ runner.temp }}/.buildx-cache,mode=max,tag=temp | |
| - name: Run TruffleHog | |
| run: | | |
| docker pull trufflesecurity/trufflehog:latest | |
| docker run --rm -v ${{ runner.temp }}:/tmp \ | |
| trufflesecurity/trufflehog:latest \ | |
| docker --image file:///tmp/local-rudder-server.tar \ | |
| --github-actions \ | |
| --no-verification \ | |
| --fail | |
| - name: Build and push | |
| if: success() | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| platforms: ${{ matrix.build-config.platform }} | |
| push: true | |
| tags: ${{ matrix.build-config.tags }} | |
| labels: ${{ matrix.build-config.labels }} | |
| build-args: | | |
| BUILD_DATE=${{ needs.docker-oss-meta.outputs.build-date }} | |
| VERSION=${{ needs.docker-oss-meta.outputs.version }} | |
| COMMIT_HASH=${{ github.sha }} | |
| REVISION=${{ needs.docker-oss-meta.outputs.revision }} | |
| cache-from: type=local,src=${{ runner.temp }}/.buildx-cache,tag=temp | |
| docker-ent-meta: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| labels: ${{ steps.meta.outputs.labels }} | |
| build-date: ${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.created'] }} | |
| version: ${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.version'] }} | |
| revision: ${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.revision'] }} | |
| tags: ${{ steps.meta.outputs.tags }} | |
| arm64_tags: ${{ steps.arm64_meta.outputs.tags }} | |
| arm64_labels: ${{ steps.arm64_meta.outputs.labels }} | |
| amd64_tags: ${{ steps.amd64_meta.outputs.tags }} | |
| amd64_labels: ${{ steps.amd64_meta.outputs.labels }} | |
| steps: | |
| - name: Docker meta | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{env.docker_ent_images}} | |
| tags: ${{env.docker_ent_tags}} | |
| - name: Docker arm64 meta | |
| id: arm64_meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{env.docker_ent_images}} | |
| tags: ${{env.docker_ent_tags}} | |
| flavor: | | |
| suffix=-${{env.arch_arm64}},onlatest=true | |
| - name: Docker amd64 meta | |
| id: amd64_meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{env.docker_ent_images}} | |
| tags: ${{env.docker_ent_tags}} | |
| flavor: | | |
| suffix=-${{env.arch_amd64}},onlatest=true | |
| docker-ent: | |
| needs: | |
| - docker-ent-meta | |
| strategy: | |
| matrix: | |
| build-config: | |
| - os: [ self-hosted, Linux, ARM64 ] | |
| tags: ${{needs.docker-ent-meta.outputs.arm64_tags}} | |
| labels: ${{needs.docker-ent-meta.outputs.arm64_labels}} | |
| platform: linux/arm64 | |
| - os: ubuntu-latest | |
| tags: ${{needs.docker-ent-meta.outputs.amd64_tags}} | |
| labels: ${{needs.docker-ent-meta.outputs.amd64_labels}} | |
| platforms: linux/amd64 | |
| runs-on: ${{matrix.build-config.os}} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to DockerHub | |
| uses: docker/[email protected] | |
| with: | |
| username: rudderlabs | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Build image (no-push) | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| platforms: ${{ matrix.build-config.platform }} | |
| push: false | |
| tags: ${{ matrix.build-config.tags }} | |
| labels: ${{ matrix.build-config.labels }} | |
| build-args: | | |
| BUILD_DATE=${{ needs.docker-ent-meta.outputs.build-date }} | |
| VERSION=${{ needs.docker-ent-meta.outputs.version }} | |
| COMMIT_HASH=${{ github.sha }} | |
| REVISION=${{ needs.docker-ent-meta.outputs.revision }} | |
| ENTERPRISE_TOKEN=${{ secrets.ENTERPRISE_TOKEN }} | |
| outputs: type=docker,dest=${{ runner.temp }}/local-rudder-server.tar | |
| cache-to: type=local,dest=${{ runner.temp }}/.buildx-cache,mode=max,tag=temp | |
| - name: Run TruffleHog | |
| run: | | |
| docker pull trufflesecurity/trufflehog:latest | |
| docker run --rm -v ${{ runner.temp }}:/tmp \ | |
| trufflesecurity/trufflehog:latest \ | |
| docker --image file:///tmp/local-rudder-server.tar \ | |
| --github-actions \ | |
| --no-verification \ | |
| --fail | |
| - name: Build and push | |
| if: success() | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| platforms: ${{ matrix.build-config.platform }} | |
| push: true | |
| tags: ${{ matrix.build-config.tags }} | |
| labels: ${{ matrix.build-config.labels }} | |
| build-args: | | |
| BUILD_DATE=${{ needs.docker-ent-meta.outputs.build-date }} | |
| VERSION=${{ needs.docker-ent-meta.outputs.version }} | |
| COMMIT_HASH=${{ github.sha }} | |
| REVISION=${{ needs.docker-ent-meta.outputs.revision }} | |
| ENTERPRISE_TOKEN=${{ secrets.ENTERPRISE_TOKEN }} | |
| cache-from: type=local,src=${{ runner.temp }}/.buildx-cache,tag=temp | |
| docker-sbsvc-meta: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| labels: ${{ steps.meta.outputs.labels }} | |
| build-date: ${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.created'] }} | |
| version: ${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.version'] }} | |
| revision: ${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.revision'] }} | |
| tags: ${{ steps.meta.outputs.tags }} | |
| arm64_tags: ${{ steps.arm64_meta.outputs.tags }} | |
| arm64_labels: ${{ steps.arm64_meta.outputs.labels }} | |
| amd64_tags: ${{ steps.amd64_meta.outputs.tags }} | |
| amd64_labels: ${{ steps.amd64_meta.outputs.labels }} | |
| steps: | |
| - name: Docker meta | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{env.docker_sbsvc_images}} | |
| tags: ${{env.docker_sbsvc_tags}} | |
| - name: Docker arm64 meta | |
| id: arm64_meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{env.docker_sbsvc_images}} | |
| tags: ${{env.docker_sbsvc_tags}} | |
| flavor: | | |
| suffix=-${{env.arch_arm64}},onlatest=true | |
| - name: Docker amd64 meta | |
| id: amd64_meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{env.docker_sbsvc_images}} | |
| tags: ${{env.docker_sbsvc_tags}} | |
| flavor: | | |
| suffix=-${{env.arch_amd64}},onlatest=true | |
| docker-suppression-backup-service: | |
| needs: | |
| - docker-sbsvc-meta | |
| strategy: | |
| matrix: | |
| build-config: | |
| - os: [ self-hosted, Linux, ARM64 ] | |
| tags: ${{needs.docker-sbsvc-meta.outputs.arm64_tags}} | |
| labels: ${{needs.docker-sbsvc-meta.outputs.arm64_labels}} | |
| platform: linux/arm64 | |
| - os: ubuntu-latest | |
| tags: ${{needs.docker-sbsvc-meta.outputs.amd64_tags}} | |
| labels: ${{needs.docker-sbsvc-meta.outputs.amd64_labels}} | |
| platform: linux/amd64 | |
| runs-on: ${{matrix.build-config.os}} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to DockerHub | |
| uses: docker/[email protected] | |
| with: | |
| username: rudderlabs | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Build image (no-push) | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| platforms: ${{ matrix.build-config.platform }} | |
| file: ./suppression-backup-service/Dockerfile | |
| push: false | |
| tags: ${{ matrix.build-config.tags }} | |
| labels: ${{ matrix.build-config.labels }} | |
| build-args: | | |
| BUILD_DATE=${{ needs.docker-sbsvc-meta.outputs.build-date }} | |
| VERSION=${{ needs.docker-sbsvc-meta.outputs.version }} | |
| COMMIT_HASH=${{ github.sha }} | |
| REVISION=${{ needs.docker-sbsvc-meta.outputs.revision }} | |
| outputs: type=docker,dest=${{ runner.temp }}/suppression-backup-service.tar | |
| cache-to: type=local,dest=${{ runner.temp }}/.buildx-cache,mode=max,tag=temp | |
| - name: Run TruffleHog | |
| run: | | |
| docker pull trufflesecurity/trufflehog:latest | |
| docker run --rm -v ${{ runner.temp }}:/tmp \ | |
| trufflesecurity/trufflehog:latest \ | |
| docker --image file:///tmp/suppression-backup-service.tar \ | |
| --github-actions \ | |
| --no-verification \ | |
| --fail | |
| - name: Build and push | |
| if: success() | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| platforms: ${{ matrix.build-config.platform }} | |
| file: ./suppression-backup-service/Dockerfile | |
| push: true | |
| tags: ${{ matrix.build-config.tags }} | |
| labels: ${{ matrix.build-config.labels }} | |
| build-args: | | |
| BUILD_DATE=${{ needs.docker-sbsvc-meta.outputs.build-date }} | |
| VERSION=${{ needs.docker-sbsvc-meta.outputs.version }} | |
| COMMIT_HASH=${{ github.sha }} | |
| REVISION=${{ needs.docker-sbsvc-meta.outputs.revision }} | |
| cache-from: type=local,src=${{ runner.temp }}/.buildx-cache,tag=temp | |
| create-manifest-docker-oss: | |
| runs-on: ubuntu-latest | |
| needs: [ docker-oss, docker-oss-meta ] | |
| steps: | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to DockerHub | |
| uses: docker/[email protected] | |
| with: | |
| username: rudderlabs | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Create multi-arch manifest for Docker OSS | |
| run: | | |
| while read -r tag; do | |
| echo "$tag" | |
| arm_tag=$(echo "${{ needs.docker-oss-meta.outputs.arm64_tags }}" | grep "$tag") | |
| amd_tag=$(echo "${{ needs.docker-oss-meta.outputs.amd64_tags }}" | grep "$tag") | |
| docker buildx imagetools create -t $tag $arm_tag $amd_tag | |
| done <<< "${{ needs.docker-oss-meta.outputs.tags }}" | |
| create-manifest-docker-ent: | |
| runs-on: ubuntu-latest | |
| needs: [ docker-ent, docker-ent-meta ] | |
| steps: | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to DockerHub | |
| uses: docker/[email protected] | |
| with: | |
| username: rudderlabs | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Create multi-arch manifest for Docker ENT | |
| run: | | |
| while read -r tag; do | |
| echo "$tag" | |
| arm_tag=$(echo "${{ needs.docker-ent-meta.outputs.arm64_tags }}" | grep "$tag") | |
| amd_tag=$(echo "${{ needs.docker-ent-meta.outputs.amd64_tags }}" | grep "$tag") | |
| docker buildx imagetools create -t $tag $arm_tag $amd_tag | |
| done <<< "${{ needs.docker-ent-meta.outputs.tags }}" | |
| create-manifest-docker-sbsvc: | |
| runs-on: ubuntu-latest | |
| needs: [ docker-suppression-backup-service, docker-sbsvc-meta ] | |
| steps: | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to DockerHub | |
| uses: docker/[email protected] | |
| with: | |
| username: rudderlabs | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Create multi-arch manifest for Docker Suppression Backup Service | |
| run: | | |
| while read -r tag; do | |
| echo "$tag" | |
| arm_tag=$(echo "${{ needs.docker-sbsvc-meta.outputs.arm64_tags }}" | grep "$tag") | |
| amd_tag=$(echo "${{ needs.docker-sbsvc-meta.outputs.amd64_tags }}" | grep "$tag") | |
| docker buildx imagetools create -t $tag $arm_tag $amd_tag | |
| done <<< "${{ needs.docker-sbsvc-meta.outputs.tags }}" |