[PI-618] bulk etl local (part 1) #2282

Workflow file for this run

.github/workflows/pull-requests.yml at 910fc3e

	name: "Workflow: Pull Request"

	on: pull_request

	permissions:
	id-token: write
	contents: read
	actions: write

	env:
	BASE_CACHE_SUFFIX: base
	BASE_BRANCH_NAME: ${{ github.event.pull_request.base.ref }}
	BRANCH_NAME: ${{ github.event.pull_request.head.ref }}
	CI_ROLE_NAME: ${{ secrets.CI_ROLE_NAME }}
	BRANCH_GITHUB_SHA_SHORT: $(echo ${{ github.event.pull_request.head.sha }} \| cut -c 1-7)
	TF_CLI_ARGS: -no-color

	jobs:
	workflow--check--branch-name:
	runs-on: [self-hosted, ci]
	steps:
	- uses: actions/checkout@v4
	with:
	ref: ${{ env.BRANCH_NAME }}
	- uses: ./.github/actions/make/
	with:
	command: workflow--check--branch-name

	workflow--check--rebased-on-main:
	runs-on: [self-hosted, ci]
	steps:
	- uses: actions/checkout@v4
	with:
	fetch-depth: 0
	- uses: ./.github/actions/make/
	with:
	command: workflow--check--rebased-on-main

	workflow--check--release-branch-name:
	runs-on: [self-hosted, ci]
	steps:
	- uses: actions/checkout@v4
	with:
	ref: ${{ github.event.pull_request.head.ref }}
	- uses: ./.github/actions/make/
	with:
	command: workflow--check--release-branch-name

	parse-secrets:
	runs-on: [self-hosted, ci]
	needs: [workflow--check--branch-name]
	steps:
	- id: parse-secrets
	run: \|
	echo "::add-mask::${{ secrets.CI_ROLE_NAME }}"

	create-workspace-name:
	runs-on: [self-hosted, ci]
	needs: [workflow--check--branch-name]
	outputs:
	workspace: ${{ steps.create_workspace_name.outputs.workspace }}
	steps:
	- id: create_workspace_name
	# will create a workspace in the format ci-PI-70-b25lkjd or for release branches rel-2023-10-30-lskjdld
	run: \|
	if [[ ${{ env.BRANCH_NAME }} == "release/" ]]; then
	echo "workspace=rel-$(echo ${{ env.BRANCH_NAME }} \| sed 's/release\///')-${{ env.BRANCH_GITHUB_SHA_SHORT }}" >> $GITHUB_OUTPUT
	else
	echo "workspace=ci-$(echo ${{ env.BRANCH_NAME }} \| sed -n 's/.\/$[^-]$-$[^-]$./\1-\2/p' \| tr '[:upper:]' '[:lower:]')-${{ env.BRANCH_GITHUB_SHA_SHORT }}" >> $GITHUB_OUTPUT
	fi

	build-head:
	runs-on: [self-hosted, ci]
	needs: [parse-secrets]
	steps:
	- uses: actions/checkout@v4
	with:
	ref: ${{ env.BRANCH_NAME }}
	- uses: ./.github/actions/make/
	with:
	command: build
	save-to-cache: "true"
	restore-from-cache: "false"

	build-base:
	runs-on: [self-hosted, ci]
	needs: [workflow--check--branch-name]
	steps:
	- uses: actions/checkout@v4
	with:
	ref: ${{ env.BASE_BRANCH_NAME }}
	- uses: ./.github/actions/make/
	with:
	command: build
	save-to-cache: "true"
	restore-from-cache: "false"
	cache-suffix: ${{ env.BASE_CACHE_SUFFIX }}

	workflow--codebase-checks:
	runs-on: [self-hosted, ci]
	needs: [build-head]
	steps:
	- uses: actions/checkout@v4
	with:
	ref: ${{ env.BRANCH_NAME }}
	- uses: ./.github/actions/make/
	with:
	command: workflow--codebase-checks

	test--unit:
	needs: [build-head]
	runs-on: [self-hosted, ci]
	strategy:
	matrix:
	test-type: [unit, slow]
	steps:
	- uses: actions/checkout@v4
	with:
	ref: ${{ env.BRANCH_NAME }}
	- uses: ./.github/actions/pytest-with-rerun/
	with:
	test-type: ${{ matrix.test-type }}

	test--feature--local:
	needs: [build-head]
	runs-on: [self-hosted, ci]
	steps:
	- uses: actions/checkout@v4
	with:
	ref: ${{ env.BRANCH_NAME }}
	- uses: ./.github/actions/make/
	with:
	command: test--feature--local

	terraform-base-build:
	needs: [create-workspace-name, build-base, parse-secrets]
	runs-on: [self-hosted, ci]
	steps:
	- uses: actions/checkout@v4
	with:
	ref: ${{ env.BASE_BRANCH_NAME }}
	- uses: ./.github/actions/terraform/
	with:
	command: plan
	workspace: ${{ needs.create-workspace-name.outputs.workspace }}
	cache-suffix: ${{ env.BASE_CACHE_SUFFIX }}
	- uses: ./.github/actions/terraform/
	with:
	command: apply
	workspace: ${{ needs.create-workspace-name.outputs.workspace }}
	save-to-cache: "true"
	restore-from-cache: "false"
	cache-suffix: ${{ env.BASE_CACHE_SUFFIX }}

	terraform-head-build:
	needs:
	[
	create-workspace-name,
	build-head,
	workflow--codebase-checks,
	test--unit,
	test--feature--local,
	parse-secrets,
	terraform-base-build,
	]
	runs-on: [self-hosted, ci]
	steps:
	- uses: actions/checkout@v4
	with:
	ref: ${{ env.BRANCH_NAME }}
	- uses: ./.github/actions/terraform/
	with:
	command: plan
	workspace: ${{ needs.create-workspace-name.outputs.workspace }}
	- uses: ./.github/actions/terraform/
	with:
	command: apply
	save-to-cache: "true"
	restore-from-cache: "false"
	workspace: ${{ needs.create-workspace-name.outputs.workspace }}

	apigee--deploy:
	needs: [build-head, terraform-head-build]
	runs-on: [self-hosted, ci]
	steps:
	- uses: actions/checkout@v4
	with:
	ref: ${{ env.BRANCH_NAME }}
	- uses: ./.github/actions/make/
	with:
	command: apigee--deploy DEV_BUILD="true"
	save-to-cache: "false"
	restore-from-cache: "true"
	requires-aws: true

	apigee--attach-product:
	needs: [build-head, apigee--deploy]
	runs-on: [self-hosted, ci]
	steps:
	- uses: actions/checkout@v4
	with:
	ref: ${{ env.BRANCH_NAME }}
	- run: \|
	poetry add pytest-nhsd-apim
	- uses: ./.github/actions/make/
	with:
	command: apigee--attach-product DEV_BUILD="true"
	save-to-cache: "false"
	restore-from-cache: "true"
	requires-aws: true

	test--integration:
	needs:
	[
	build-head,
	workflow--codebase-checks,
	test--unit,
	test--feature--local,
	terraform-head-build,
	apigee--attach-product,
	]
	runs-on: [self-hosted, ci]
	strategy:
	matrix:
	test-type: [integration, s3]
	steps:
	- uses: actions/checkout@v4
	with:
	ref: ${{ env.BRANCH_NAME }}
	- uses: ./.github/actions/pytest-with-rerun/
	with:
	test-type: ${{ matrix.test-type }}
	requires-aws: true

	test--feature--integration:
	needs:
	[
	build-head,
	workflow--codebase-checks,
	test--unit,
	test--feature--local,
	terraform-head-build,
	test--integration,
	]
	runs-on: [self-hosted, ci]
	steps:
	- uses: actions/checkout@v4
	with:
	ref: ${{ env.BRANCH_NAME }}
	- uses: ./.github/actions/make/
	with:
	command: test--feature--integration
	requires-aws: true

	test--smoke:
	needs:
	[
	build-head,
	workflow--codebase-checks,
	test--unit,
	test--feature--local,
	terraform-head-build,
	test--feature--integration,
	test--integration,
	apigee--deploy,
	apigee--attach-product,
	]
	runs-on: [self-hosted, ci]
	steps:
	- uses: actions/checkout@v4
	with:
	ref: ${{ env.BRANCH_NAME }}
	- uses: ./.github/actions/make/
	with:
	command: test--smoke
	requires-aws: true

	apigee--detach-product:
	needs:
	[
	build-head,
	test--smoke,
	apigee--attach-product,
	test--feature--integration,
	]
	runs-on: [self-hosted, ci]
	if: ${{ needs.apigee--attach-product.result == 'success' }}
	steps:
	- uses: actions/checkout@v4
	with:
	ref: ${{ env.BRANCH_NAME }}
	- run: \|
	poetry add pytest-nhsd-apim
	- uses: ./.github/actions/make/
	with:
	command: apigee--detach-product DEV_BUILD="true"
	save-to-cache: "false"
	restore-from-cache: "true"
	requires-aws: true

	delete--proxy:
	needs: [test--smoke, apigee--deploy, apigee--detach-product]
	runs-on: [self-hosted, ci]
	if: ${{ needs.apigee--deploy.result == 'success' }}
	steps:
	- uses: actions/checkout@v4
	with:
	ref: ${{ env.BRANCH_NAME }}
	- uses: ./.github/actions/make/
	with:
	command: apigee--delete DEV_BUILD="true"
	save-to-cache: "false"
	restore-from-cache: "true"
	requires-aws: true

	destroy-redundant-workspaces:
	runs-on: [self-hosted, ci]
	needs: [build-head]
	steps:
	- uses: actions/checkout@v4
	with:
	ref: ${{ env.BRANCH_NAME }}
	fetch-depth: 0
	- name: Remove redundant workspaces
	uses: ./.github/actions/make/
	with:
	command: destroy--redundant-workspaces BRANCH_NAME=origin/${{ env.BRANCH_NAME }} DESTROY_ALL_COMMITS_ON_BRANCH=false KILL_ALL=false CURRENT_COMMIT="${{ env.BRANCH_GITHUB_SHA_SHORT }}" TF_CLI_ARGS=${{ env.TF_CLI_ARGS }}
	requires-aws: true

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

[PI-618] bulk etl local (part 1) #2282

Workflow file

[PI-618] bulk etl local (part 1) #2282

Jobs

Run details

Workflow file for this run