From ab52a92acdce86b02e2e5f06f9278e2132d9fe2f Mon Sep 17 00:00:00 2001 From: Ziga Cernigoj Date: Mon, 23 Oct 2023 14:33:55 +0200 Subject: [PATCH] add lint, test and deploy github actions --- .github/workflows/deploy_production.yam | 97 ++++++++++++++++++ .github/workflows/deploy_testing.yaml | 98 +++++++++++++++++++ .github/workflows/lint_and_test.yaml | 7 ++ .github/workflows/reusable_lint_and_test.yaml | 56 +++++++++++ 4 files changed, 258 insertions(+) create mode 100644 .github/workflows/deploy_production.yam create mode 100644 .github/workflows/deploy_testing.yaml create mode 100644 .github/workflows/lint_and_test.yaml create mode 100644 .github/workflows/reusable_lint_and_test.yaml diff --git a/.github/workflows/deploy_production.yam b/.github/workflows/deploy_production.yam new file mode 100644 index 00000000..f9426d93 --- /dev/null +++ b/.github/workflows/deploy_production.yam @@ -0,0 +1,97 @@ +name: Deploy production +on: + push: + tags: + - 'v[0-9]+.[0-9]+.[0-9]+' + +jobs: + + run-reusable-lint-and-test: + uses: ./.github/workflows/reusable_lint_and_test.yaml + secrets: inherit + + deploy-production: + needs: run-reusable-lint-and-test + + if: startsWith(github.ref, 'refs/tags/v') + + runs-on: ubuntu-latest + + env: + DEPLOYMENT_NAME: "production-docker" + PROD_ECR_REGISTRY: ${{ secrets.PROD_ECR_REGISTRY }} + PROD_ECR_REGISTRY_IMAGE: ${{ secrets.PROD_ECR_REGISTRY_IMAGE }} + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - name: configure aws access credentials + run: | + mkdir -p ~/.aws + echo -e "[default]\nregion=eu-central-1" > ~/.aws/config + echo -e "[default]\naws_access_key_id=${{ secrets.PRODZAPPA_AWS_ACCESS_KEY_ID }}\naws_secret_access_key=${{ secrets.PRODZAPPA_AWS_SECRET_ACCESS_KEY }}" > ~/.aws/credentials + + - name: install pipenv and aws + run: | + pip install pipenv==2021.5.29 + pip install awscli --no-build-isolation + + - name: download process definitions + run: | + chmod +x download-process-definitions.sh + ./download-process-definitions.sh + + - name: install dependencies needed for deployment + working-directory: ./rest + run: pipenv install --dev + + - name: create zappa_settings.json on-the-fly + working-directory: ./rest + run: | + cp zappa_settings.json.template zappa_settings.json + sed -i "s/@@AWS_ACCESS_KEY_ID@@/${{ secrets.PRODDATA_AWS_ACCESS_KEY_ID }}/g" zappa_settings.json + sed -i "s#@@AWS_SECRET_ACCESS_KEY@@#${{ secrets.PRODDATA_AWS_SECRET_ACCESS_KEY }}#g" zappa_settings.json + sed -i "s/@@PRODUCTION_SH_CLIENT_ID@@/${{ secrets.PRODUCTION_SH_CLIENT_ID }}/g" zappa_settings.json + sed -i "s/@@PRODUCTION_SH_CLIENT_SECRET@@/${{ secrets.PRODUCTION_SH_CLIENT_SECRET }}/g" zappa_settings.json + sed -i "s/@@BACKEND_VERSION@@/$GITHUB_REF_NAME/g" zappa_settings.json + sed -i "s/@@RESULTS_S3_BUCKET_NAME_MAIN@@/${{ secrets.RESULTS_S3_BUCKET_NAME_MAIN_PRODUCTION }}/g" zappa_settings.json + sed -i "s/@@RESULTS_S3_BUCKET_NAME_CREODIAS@@/${{ secrets.RESULTS_S3_BUCKET_NAME_CREODIAS_PRODUCTION }}/g" zappa_settings.json + sed -i "s/@@RESULTS_S3_BUCKET_NAME_USWEST@@/${{ secrets.RESULTS_S3_BUCKET_NAME_USWEST_PRODUCTION }}/g" zappa_settings.json + sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN@@/${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN_PRODUCTION }}/g" zappa_settings.json + sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS@@/${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS_PRODUCTION }}/g" zappa_settings.json + sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST@@/${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST_PRODUCTION }}/g" zappa_settings.json + sed -i "s#@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN@@#${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN_PRODUCTION }}#g" zappa_settings.json + sed -i "s/@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS@@/${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS_PRODUCTION }}/g" zappa_settings.json + sed -i "s#@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST@@#${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST_PRODUCTION }}#g" zappa_settings.json + sed -i "s#@@USAGE_REPORTING_BASE_URL_PRODUCTION@@#${{ secrets.USAGE_REPORTING_BASE_URL_PRODUCTION }}#g" zappa_settings.json + sed -i "s#@@USAGE_REPORTING_AUTH_URL_PRODUCTION@@#${{ secrets.USAGE_REPORTING_AUTH_URL_PRODUCTION }}#g" zappa_settings.json + sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_ID_PRODUCTION@@#${{ secrets.USAGE_REPORTING_AUTH_CLIENT_ID_PRODUCTION }}#g" zappa_settings.json + sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_SECRET_PRODUCTION@@#${{ secrets.USAGE_REPORTING_AUTH_CLIENT_SECRET_PRODUCTION }}#g" zappa_settings.json + sed -i "s#@@LOGGING_LEVEL@@#${{ secrets.LOGGING_LEVEL_PRODUCTION }}#g" zappa_settings.json + + - name: generate zappa_settings.py for docker image from zappa_settings.json + working-directory: ./rest + run: pipenv run zappa save-python-settings-file "$DEPLOYMENT_NAME" + + - name: build docker image with correct tags + working-directory: ./rest + run: docker build -t "$PROD_ECR_REGISTRY/$PROD_ECR_REGISTRY_IMAGE:$GITHUB_REF_NAME" -t "$PROD_ECR_REGISTRY/$PROD_ECR_REGISTRY_IMAGE:latest" --build-arg VERSION=$GITHUB_REF_NAME --build-arg VCS_REF=$GITHUB_SHA --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') . + + - name: login for AWS ECR docker + working-directory: ./rest + run: aws ecr get-login-password --region eu-central-1 | docker login --username AWS --password-stdin "$PROD_ECR_REGISTRY" + + - name: push docker images (versioned with CI tag and "latest") to AWS ECR with docker + working-directory: ./rest + run: | + docker push "$PROD_ECR_REGISTRY/$PROD_ECR_REGISTRY_IMAGE:$GITHUB_REF_NAME" + docker push "$PROD_ECR_REGISTRY/$PROD_ECR_REGISTRY_IMAGE:latest" + + - name: deploy lambda with new docker image + working-directory: ./rest + run: pipenv run zappa deploy "$DEPLOYMENT_NAME" -d "$PROD_ECR_REGISTRY/$PROD_ECR_REGISTRY_IMAGE:latest" || pipenv run zappa update "$DEPLOYMENT_NAME" -d "$PROD_ECR_REGISTRY/$PROD_ECR_REGISTRY_IMAGE:latest" \ No newline at end of file diff --git a/.github/workflows/deploy_testing.yaml b/.github/workflows/deploy_testing.yaml new file mode 100644 index 00000000..f8cc0bfb --- /dev/null +++ b/.github/workflows/deploy_testing.yaml @@ -0,0 +1,98 @@ +name: Deploy testing +on: + push: + tags: + - 'v[0-9]+.[0-9]+.[0-9]+' + - 'v[0-9]+.[0-9]+.[0-9]+-rc.[0-9]+' + +jobs: + + run-reusable-lint-and-test: + uses: ./.github/workflows/reusable_lint_and_test.yaml + secrets: inherit + + deploy-testing: + needs: run-reusable-lint-and-test + + if: startsWith(github.ref, 'refs/tags/v') + + runs-on: ubuntu-latest + + env: + DEPLOYMENT_NAME: "testing-docker" + TESTING_ECR_REGISTRY: ${{ secrets.TESTING_ECR_REGISTRY }} + TESTING_ECR_REGISTRY_IMAGE: ${{ secrets.TESTING_ECR_REGISTRY_IMAGE }} + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - name: configure aws access credentials + run: | + mkdir -p ~/.aws + echo -e "[default]\nregion=eu-central-1" > ~/.aws/config + echo -e "[default]\naws_access_key_id=${{ secrets.TESTINGZAPPA_AWS_ACCESS_KEY_ID }}\naws_secret_access_key=${{ secrets.TESTINGZAPPA_AWS_SECRET_ACCESS_KEY }}" > ~/.aws/credentials + + - name: install pipenv and aws + run: | + pip install pipenv==2021.5.29 + pip install awscli --no-build-isolation + + - name: download process definitions + run: | + chmod +x download-process-definitions.sh + ./download-process-definitions.sh + + - name: install dependencies needed for deployment + working-directory: ./rest + run: pipenv install --dev + + - name: create zappa_settings.json on-the-fly + working-directory: ./rest + run: | + cp zappa_settings.json.template zappa_settings.json + sed -i "s/@@AWS_ACCESS_KEY_ID@@/${{ secrets.TESTINGDATA_AWS_ACCESS_KEY_ID }}/g" zappa_settings.json + sed -i "s#@@AWS_SECRET_ACCESS_KEY@@#${{ secrets.TESTINGDATA_AWS_SECRET_ACCESS_KEY }}#g" zappa_settings.json + sed -i "s/@@TESTING_SH_CLIENT_ID@@/${{ secrets.TESTING_SH_CLIENT_ID }}/g" zappa_settings.json + sed -i "s/@@TESTING_SH_CLIENT_SECRET@@/${{ secrets.TESTING_SH_CLIENT_SECRET }}/g" zappa_settings.json + sed -i "s/@@BACKEND_VERSION@@/$GITHUB_REF_NAME/g" zappa_settings.json + sed -i "s/@@RESULTS_S3_BUCKET_NAME_MAIN@@/${{ secrets.RESULTS_S3_BUCKET_NAME_MAIN }}/g" zappa_settings.json + sed -i "s/@@RESULTS_S3_BUCKET_NAME_CREODIAS@@/${{ secrets.RESULTS_S3_BUCKET_NAME_CREODIAS }}/g" zappa_settings.json + sed -i "s/@@RESULTS_S3_BUCKET_NAME_USWEST@@/${{ secrets.RESULTS_S3_BUCKET_NAME_USWEST }}/g" zappa_settings.json + sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN@@/${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN }}/g" zappa_settings.json + sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS@@/${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS }}/g" zappa_settings.json + sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST@@/${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST }}/g" zappa_settings.json + sed -i "s#@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN@@#${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN }}#g" zappa_settings.json + sed -i "s/@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS@@/${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS }}/g" zappa_settings.json + sed -i "s#@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST@@#${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST }}#g" zappa_settings.json + sed -i "s#@@USAGE_REPORTING_BASE_URL_TESTING@@#${{ secrets.USAGE_REPORTING_BASE_URL_TESTING }}#g" zappa_settings.json + sed -i "s#@@USAGE_REPORTING_AUTH_URL_TESTING@@#${{ secrets.USAGE_REPORTING_AUTH_URL_TESTING }}#g" zappa_settings.json + sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_ID_TESTING@@#${{ secrets.USAGE_REPORTING_AUTH_CLIENT_ID_TESTING }}#g" zappa_settings.json + sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING@@#${{ secrets.USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING }}#g" zappa_settings.json + sed -i "s#@@LOGGING_LEVEL@@#${{ secrets.LOGGING_LEVEL_TESTING }}#g" zappa_settings.json + + - name: generate zappa_settings.py for docker image from zappa_settings.json + working-directory: ./rest + run: pipenv run zappa save-python-settings-file "$DEPLOYMENT_NAME" + + - name: build docker image with correct tags + working-directory: ./rest + run: docker build -t "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:$GITHUB_REF_NAME" -t "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:latest" --build-arg VERSION=$GITHUB_REF_NAME --build-arg VCS_REF=$GITHUB_SHA --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') . + + - name: login for AWS ECR docker + working-directory: ./rest + run: aws ecr get-login-password --region eu-central-1 | docker login --username AWS --password-stdin "$TESTING_ECR_REGISTRY" + + - name: push docker images (versioned with CI tag and "latest") to AWS ECR with docker + working-directory: ./rest + run: | + docker push "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:$GITHUB_REF_NAME" + docker push "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:latest" + + - name: deploy lambda with new docker image + working-directory: ./rest + run: pipenv run zappa deploy "$DEPLOYMENT_NAME" -d "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:latest" || pipenv run zappa update "$DEPLOYMENT_NAME" -d "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:latest" \ No newline at end of file diff --git a/.github/workflows/lint_and_test.yaml b/.github/workflows/lint_and_test.yaml new file mode 100644 index 00000000..fd4c4c6f --- /dev/null +++ b/.github/workflows/lint_and_test.yaml @@ -0,0 +1,7 @@ +name: Lint and test +on: [push, pull_request] + +jobs: + run-reusable-lint-and-test: + uses: ./.github/workflows/reusable_lint_and_test.yaml + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/reusable_lint_and_test.yaml b/.github/workflows/reusable_lint_and_test.yaml new file mode 100644 index 00000000..77c3f213 --- /dev/null +++ b/.github/workflows/reusable_lint_and_test.yaml @@ -0,0 +1,56 @@ +name: Reusable lint and test +on: + workflow_call: + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - name: Install black + run: python -m pip install --upgrade black==22.3.0 + + - name: make sure that the *same* version is used in Pipfiles to avoid incompatibilities + run: grep 'black = "==22.3.0"' rest/Pipfile + + - name: check the files for correct formatting + run: black -l 120 --check . || exit 1 + + integration-tests: + runs-on: ubuntu-latest + + env: + TESTS_SH_CLIENT_ID: ${{ secrets.TESTS_SH_CLIENT_ID }} + TESTS_SH_CLIENT_SECRET: ${{ secrets.TESTS_SH_CLIENT_SECRET }} + RESULTS_S3_BUCKET_NAME_MAIN: ${{ secrets.RESULTS_S3_BUCKET_NAME_MAIN }} + RESULTS_S3_BUCKET_NAME_CREODIAS: ${{ secrets.RESULTS_S3_BUCKET_NAME_CREODIAS }} + RESULTS_S3_BUCKET_NAME_USWEST: ${{ secrets.RESULTS_S3_BUCKET_NAME_USWEST }} + RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN: ${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN }} + RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS: ${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS }} + RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST: ${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST }} + RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN: ${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN }} + RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS: ${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS }} + RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST: ${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST }} + USAGE_REPORTING_BASE_URL_TESTING: ${{ secrets.USAGE_REPORTING_BASE_URL_TESTING }} + USAGE_REPORTING_AUTH_URL_TESTING: ${{ secrets.USAGE_REPORTING_AUTH_URL_TESTING }} + USAGE_REPORTING_AUTH_CLIENT_ID_TESTING: ${{ secrets.USAGE_REPORTING_AUTH_CLIENT_ID_TESTING }} + USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING: ${{ secrets.USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING }} + + steps: + - uses: actions/checkout@v4 + + - name: Docker-compose build + run: docker-compose -f docker-compose.yml -f docker-compose.pytest.yml build + + - name: Docker-compose up and run pytest + run: docker-compose -f docker-compose.yml -f docker-compose.pytest.yml up --exit-code-from pytest + + - name: Docker-compose cleanup + if: always() + run: docker-compose -f docker-compose.yml -f docker-compose.pytest.yml down -v \ No newline at end of file