Merge pull request #44 from Open-EO/sync/master

sync master branches from gitlab and github

.github/workflows/deploy_production.yaml

@@ -0,0 +1,97 @@
+name: Deploy production
+on:
+  push:
+    tags:
+    - 'v[0-9]+.[0-9]+.[0-9]+'
+
+jobs:
+
+  run-reusable-lint-and-test:
+    uses: ./.github/workflows/reusable_lint_and_test.yaml
+    secrets: inherit
+
+  deploy-production:
+    needs: run-reusable-lint-and-test
+
+    if: startsWith(github.ref, 'refs/tags/v')
+
+    runs-on: ubuntu-latest
+
+    env:
+      DEPLOYMENT_NAME: "production-docker"
+      PROD_ECR_REGISTRY: ${{ secrets.PROD_ECR_REGISTRY }}
+      PROD_ECR_REGISTRY_IMAGE: ${{ secrets.PROD_ECR_REGISTRY_IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+
+      - name: configure aws access credentials
+        run: |
+          mkdir -p ~/.aws
+          echo -e "[default]\nregion=eu-central-1" > ~/.aws/config
+          echo -e "[default]\naws_access_key_id=${{ secrets.PRODZAPPA_AWS_ACCESS_KEY_ID }}\naws_secret_access_key=${{ secrets.PRODZAPPA_AWS_SECRET_ACCESS_KEY }}" > ~/.aws/credentials
+
+      - name: install pipenv and aws
+        run: |
+          pip install pipenv==2021.5.29
+          pip install awscli --no-build-isolation
+
+      - name: download process definitions
+        run: |
+          chmod +x download-process-definitions.sh
+          ./download-process-definitions.sh
+      
+      - name: install dependencies needed for deployment
+        working-directory: ./rest
+        run: pipenv install --dev
+
+      - name: create zappa_settings.json on-the-fly
+        working-directory: ./rest
+        run: |
+          cp zappa_settings.json.template zappa_settings.json
+          sed -i "s/@@AWS_ACCESS_KEY_ID@@/${{ secrets.PRODDATA_AWS_ACCESS_KEY_ID }}/g" zappa_settings.json
+          sed -i "s#@@AWS_SECRET_ACCESS_KEY@@#${{ secrets.PRODDATA_AWS_SECRET_ACCESS_KEY }}#g" zappa_settings.json
+          sed -i "s/@@PRODUCTION_SH_CLIENT_ID@@/${{ secrets.PRODUCTION_SH_CLIENT_ID }}/g" zappa_settings.json
+          sed -i "s/@@PRODUCTION_SH_CLIENT_SECRET@@/${{ secrets.PRODUCTION_SH_CLIENT_SECRET }}/g" zappa_settings.json
+          sed -i "s/@@BACKEND_VERSION@@/$GITHUB_REF_NAME/g" zappa_settings.json
+          sed -i "s/@@RESULTS_S3_BUCKET_NAME_MAIN@@/${{ secrets.RESULTS_S3_BUCKET_NAME_MAIN_PRODUCTION }}/g" zappa_settings.json
+          sed -i "s/@@RESULTS_S3_BUCKET_NAME_CREODIAS@@/${{ secrets.RESULTS_S3_BUCKET_NAME_CREODIAS_PRODUCTION }}/g" zappa_settings.json
+          sed -i "s/@@RESULTS_S3_BUCKET_NAME_USWEST@@/${{ secrets.RESULTS_S3_BUCKET_NAME_USWEST_PRODUCTION }}/g" zappa_settings.json
+          sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN@@/${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN_PRODUCTION }}/g" zappa_settings.json
+          sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS@@/${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS_PRODUCTION }}/g" zappa_settings.json
+          sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST@@/${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST_PRODUCTION }}/g" zappa_settings.json
+          sed -i "s#@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN@@#${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN_PRODUCTION }}#g" zappa_settings.json
+          sed -i "s/@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS@@/${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS_PRODUCTION }}/g" zappa_settings.json
+          sed -i "s#@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST@@#${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST_PRODUCTION }}#g" zappa_settings.json
+          sed -i "s#@@USAGE_REPORTING_BASE_URL_PRODUCTION@@#${{ secrets.USAGE_REPORTING_BASE_URL_PRODUCTION }}#g" zappa_settings.json
+          sed -i "s#@@USAGE_REPORTING_AUTH_URL_PRODUCTION@@#${{ secrets.USAGE_REPORTING_AUTH_URL_PRODUCTION }}#g" zappa_settings.json
+          sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_ID_PRODUCTION@@#${{ secrets.USAGE_REPORTING_AUTH_CLIENT_ID_PRODUCTION }}#g" zappa_settings.json
+          sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_SECRET_PRODUCTION@@#${{ secrets.USAGE_REPORTING_AUTH_CLIENT_SECRET_PRODUCTION }}#g" zappa_settings.json
+          sed -i "s#@@LOGGING_LEVEL@@#${{ secrets.LOGGING_LEVEL_PRODUCTION }}#g" zappa_settings.json
+
+      - name: generate zappa_settings.py for docker image from zappa_settings.json
+        working-directory: ./rest
+        run: pipenv run zappa save-python-settings-file "$DEPLOYMENT_NAME"
+
+      - name: build docker image with correct tags
+        working-directory: ./rest
+        run: docker build -t "$PROD_ECR_REGISTRY/$PROD_ECR_REGISTRY_IMAGE:$GITHUB_REF_NAME" -t "$PROD_ECR_REGISTRY/$PROD_ECR_REGISTRY_IMAGE:latest" --build-arg VERSION=$GITHUB_REF_NAME --build-arg VCS_REF=$GITHUB_SHA --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') .
+
+      - name: login for AWS ECR docker
+        working-directory: ./rest
+        run: aws ecr get-login-password --region eu-central-1 | docker login --username AWS --password-stdin "$PROD_ECR_REGISTRY"
+
+      - name: push docker images (versioned with CI tag and "latest") to AWS ECR with docker
+        working-directory: ./rest
+        run: |
+          docker push "$PROD_ECR_REGISTRY/$PROD_ECR_REGISTRY_IMAGE:$GITHUB_REF_NAME"
+          docker push "$PROD_ECR_REGISTRY/$PROD_ECR_REGISTRY_IMAGE:latest"
+      
+      - name: deploy lambda with new docker image
+        working-directory: ./rest
+        run: pipenv run zappa deploy "$DEPLOYMENT_NAME" -d "$PROD_ECR_REGISTRY/$PROD_ECR_REGISTRY_IMAGE:latest" || pipenv run zappa update "$DEPLOYMENT_NAME" -d "$PROD_ECR_REGISTRY/$PROD_ECR_REGISTRY_IMAGE:latest"

.github/workflows/deploy_testing.yaml

@@ -0,0 +1,98 @@
+name: Deploy testing
+on:
+  push:
+    tags:
+    - 'v[0-9]+.[0-9]+.[0-9]+'
+    - 'v[0-9]+.[0-9]+.[0-9]+-rc.[0-9]+'
+
+jobs:
+
+  run-reusable-lint-and-test:
+    uses: ./.github/workflows/reusable_lint_and_test.yaml
+    secrets: inherit
+
+  deploy-testing:
+    needs: run-reusable-lint-and-test
+
+    if: startsWith(github.ref, 'refs/tags/v')
+
+    runs-on: ubuntu-latest
+
+    env:
+      DEPLOYMENT_NAME: "testing-docker"
+      TESTING_ECR_REGISTRY: ${{ secrets.TESTING_ECR_REGISTRY }}
+      TESTING_ECR_REGISTRY_IMAGE: ${{ secrets.TESTING_ECR_REGISTRY_IMAGE }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+
+      - name: configure aws access credentials
+        run: |
+          mkdir -p ~/.aws
+          echo -e "[default]\nregion=eu-central-1" > ~/.aws/config
+          echo -e "[default]\naws_access_key_id=${{ secrets.TESTINGZAPPA_AWS_ACCESS_KEY_ID }}\naws_secret_access_key=${{ secrets.TESTINGZAPPA_AWS_SECRET_ACCESS_KEY }}" > ~/.aws/credentials
+
+      - name: install pipenv and aws
+        run: |
+          pip install pipenv==2021.5.29
+          pip install awscli --no-build-isolation
+
+      - name: download process definitions
+        run: |
+          chmod +x download-process-definitions.sh
+          ./download-process-definitions.sh
+      
+      - name: install dependencies needed for deployment
+        working-directory: ./rest
+        run: pipenv install --dev
+
+      - name: create zappa_settings.json on-the-fly
+        working-directory: ./rest
+        run: |
+          cp zappa_settings.json.template zappa_settings.json
+          sed -i "s/@@AWS_ACCESS_KEY_ID@@/${{ secrets.TESTINGDATA_AWS_ACCESS_KEY_ID }}/g" zappa_settings.json
+          sed -i "s#@@AWS_SECRET_ACCESS_KEY@@#${{ secrets.TESTINGDATA_AWS_SECRET_ACCESS_KEY }}#g" zappa_settings.json
+          sed -i "s/@@TESTING_SH_CLIENT_ID@@/${{ secrets.TESTING_SH_CLIENT_ID }}/g" zappa_settings.json
+          sed -i "s/@@TESTING_SH_CLIENT_SECRET@@/${{ secrets.TESTING_SH_CLIENT_SECRET }}/g" zappa_settings.json
+          sed -i "s/@@BACKEND_VERSION@@/$GITHUB_REF_NAME/g" zappa_settings.json
+          sed -i "s/@@RESULTS_S3_BUCKET_NAME_MAIN@@/${{ secrets.RESULTS_S3_BUCKET_NAME_MAIN }}/g" zappa_settings.json
+          sed -i "s/@@RESULTS_S3_BUCKET_NAME_CREODIAS@@/${{ secrets.RESULTS_S3_BUCKET_NAME_CREODIAS }}/g" zappa_settings.json
+          sed -i "s/@@RESULTS_S3_BUCKET_NAME_USWEST@@/${{ secrets.RESULTS_S3_BUCKET_NAME_USWEST }}/g" zappa_settings.json
+          sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN@@/${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN }}/g" zappa_settings.json
+          sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS@@/${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS }}/g" zappa_settings.json
+          sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST@@/${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST }}/g" zappa_settings.json
+          sed -i "s#@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN@@#${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN }}#g" zappa_settings.json
+          sed -i "s/@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS@@/${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS }}/g" zappa_settings.json
+          sed -i "s#@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST@@#${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST }}#g" zappa_settings.json
+          sed -i "s#@@USAGE_REPORTING_BASE_URL_TESTING@@#${{ secrets.USAGE_REPORTING_BASE_URL_TESTING }}#g" zappa_settings.json
+          sed -i "s#@@USAGE_REPORTING_AUTH_URL_TESTING@@#${{ secrets.USAGE_REPORTING_AUTH_URL_TESTING }}#g" zappa_settings.json
+          sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_ID_TESTING@@#${{ secrets.USAGE_REPORTING_AUTH_CLIENT_ID_TESTING }}#g" zappa_settings.json
+          sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING@@#${{ secrets.USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING }}#g" zappa_settings.json
+          sed -i "s#@@LOGGING_LEVEL@@#${{ secrets.LOGGING_LEVEL_TESTING }}#g" zappa_settings.json
+
+      - name: generate zappa_settings.py for docker image from zappa_settings.json
+        working-directory: ./rest
+        run: pipenv run zappa save-python-settings-file "$DEPLOYMENT_NAME"
+
+      - name: build docker image with correct tags
+        working-directory: ./rest
+        run: docker build -t "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:$GITHUB_REF_NAME" -t "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:latest" --build-arg VERSION=$GITHUB_REF_NAME --build-arg VCS_REF=$GITHUB_SHA --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') .
+
+      - name: login for AWS ECR docker
+        working-directory: ./rest
+        run: aws ecr get-login-password --region eu-central-1 | docker login --username AWS --password-stdin "$TESTING_ECR_REGISTRY"
+
+      - name: push docker images (versioned with CI tag and "latest") to AWS ECR with docker
+        working-directory: ./rest
+        run: |
+          docker push "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:$GITHUB_REF_NAME"
+          docker push "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:latest"
+      
+      - name: deploy lambda with new docker image
+        working-directory: ./rest
+        run: pipenv run zappa deploy "$DEPLOYMENT_NAME" -d "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:latest" || pipenv run zappa update "$DEPLOYMENT_NAME" -d "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:latest"

.github/workflows/lint_and_test.yaml

@@ -0,0 +1,7 @@
+name: Lint and test
+on: [push, pull_request]
+
+jobs:
+  run-reusable-lint-and-test:
+    uses: ./.github/workflows/reusable_lint_and_test.yaml
+    secrets: inherit

.github/workflows/reusable_lint_and_test.yaml

@@ -0,0 +1,56 @@
+name: Reusable lint and test
+on:
+  workflow_call:
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+
+      - name: Install black
+        run: python -m pip install --upgrade black==22.3.0
+
+      - name: make sure that the *same* version is used in Pipfiles to avoid incompatibilities
+        run: grep 'black = "==22.3.0"' rest/Pipfile
+
+      - name: check the files for correct formatting
+        run: black -l 120 --check . || exit 1
+
+  integration-tests:
+    runs-on: ubuntu-latest
+
+    env:
+      TESTS_SH_CLIENT_ID: ${{ secrets.TESTS_SH_CLIENT_ID }}
+      TESTS_SH_CLIENT_SECRET: ${{ secrets.TESTS_SH_CLIENT_SECRET }}
+      RESULTS_S3_BUCKET_NAME_MAIN: ${{ secrets.RESULTS_S3_BUCKET_NAME_MAIN }}
+      RESULTS_S3_BUCKET_NAME_CREODIAS: ${{ secrets.RESULTS_S3_BUCKET_NAME_CREODIAS }}
+      RESULTS_S3_BUCKET_NAME_USWEST: ${{ secrets.RESULTS_S3_BUCKET_NAME_USWEST }}
+      RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN: ${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN }}
+      RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS: ${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS }}
+      RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST: ${{ secrets.RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST }}
+      RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN: ${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN }}
+      RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS: ${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS }}
+      RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST: ${{ secrets.RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST }}
+      USAGE_REPORTING_BASE_URL_TESTING: ${{ secrets.USAGE_REPORTING_BASE_URL_TESTING }}
+      USAGE_REPORTING_AUTH_URL_TESTING: ${{ secrets.USAGE_REPORTING_AUTH_URL_TESTING }}
+      USAGE_REPORTING_AUTH_CLIENT_ID_TESTING: ${{ secrets.USAGE_REPORTING_AUTH_CLIENT_ID_TESTING }}
+      USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING: ${{ secrets.USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Docker-compose build
+        run: docker-compose -f docker-compose.yml -f docker-compose.pytest.yml build
+
+      - name: Docker-compose up and run pytest
+        run: docker-compose -f docker-compose.yml -f docker-compose.pytest.yml up --exit-code-from pytest
+
+      - name: Docker-compose cleanup
+        if: always()
+        run: docker-compose -f docker-compose.yml -f docker-compose.pytest.yml down -v