Merge branch 'feature/lambda-docker' into 'master'

Deploy openEO-sh-driver as docker image on AWS lambda

See merge request team-6/openeo-sentinelhub-python-driver!317

.gitlab-ci.yml

@@ -8,6 +8,9 @@ services:
 stages:
   - test
   - deploy
+  - deploy-generate-zappa-settings
+  - deploy-create-docker-image
+  - deploy-update-lambda
 
 linting:
   stage: test
@@ -28,7 +31,6 @@ integration tests:
   after_script:
     - docker-compose -f docker-compose.yml -f docker-compose.pytest.yml down -v
 
-
 ## GITHUB:
 publish_github:
   stage: deploy
@@ -57,6 +59,8 @@ publish_github:
     - if git remote | grep github > /dev/null; then git remote rm github; fi
 
 
+# BEFORE 2023-10-16
+# DEPLOY TESTING: uploading .zip file on lambda
 deploy-testing-rest-lambda:
   stage: deploy
   when: manual
@@ -102,13 +106,131 @@ deploy-testing-rest-lambda:
     - export AWS_SECRET_ACCESS_KEY="$TESTING_AWS_SECRET_ACCESS_KEY"
     - pipenv run bash -c "DEPLOYMENT_TYPE=testing python dynamodb/dynamodb.py"
 
+# AFTER 2023-10-16
+# DEPLOY TESTING: create docker image, upload it to AWS ECR and use it in AWS Lambda
+# - step 1: generate zappa settings for the app inside the docker image
+# - step 2: create docker image and push it to AWS ECR
+# - step 3: update lambda function to use the new docker image
+
+# step 1: generate zappa settings for the app inside the docker image
+deploy-testing-generate-zappa-settings:
+  stage: deploy-generate-zappa-settings
+  when: manual
+  only:
+    variables:
+      - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+[-]rc.[0-9]+$/
+  image: python:3.9.13
+  variables:
+    DEPLOYMENT_NAME: "testing-docker"
+  before_script:
+    # configure aws access credentials:
+    - mkdir -p ~/.aws
+    - echo -e "[default]\nregion=eu-central-1" > ~/.aws/config
+    - echo -e "[default]\naws_access_key_id=$TESTINGZAPPA_AWS_ACCESS_KEY_ID\naws_secret_access_key=$TESTINGZAPPA_AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials
+    - pip install pipenv==2021.5.29
+  script:
+    - chmod +x download-process-definitions.sh
+    - ./download-process-definitions.sh
+    - cd rest/
+    - pipenv install --dev
+    # create zappa_settings.json on-the-fly:
+    - cp zappa_settings.json.template zappa_settings.json
+    - sed -i "s/@@AWS_ACCESS_KEY_ID@@/$TESTINGDATA_AWS_ACCESS_KEY_ID/g" zappa_settings.json
+    - sed -i "s#@@AWS_SECRET_ACCESS_KEY@@#$TESTINGDATA_AWS_SECRET_ACCESS_KEY#g" zappa_settings.json
+    - sed -i "s/@@TESTING_SH_CLIENT_ID@@/$TESTING_SH_CLIENT_ID/g" zappa_settings.json
+    - sed -i "s/@@TESTING_SH_CLIENT_SECRET@@/$TESTING_SH_CLIENT_SECRET/g" zappa_settings.json
+    - sed -i "s/@@BACKEND_VERSION@@/$CI_COMMIT_TAG/g" zappa_settings.json
+    - sed -i "s/@@RESULTS_S3_BUCKET_NAME_MAIN@@/$RESULTS_S3_BUCKET_NAME_MAIN/g" zappa_settings.json
+    - sed -i "s/@@RESULTS_S3_BUCKET_NAME_CREODIAS@@/$RESULTS_S3_BUCKET_NAME_CREODIAS/g" zappa_settings.json
+    - sed -i "s/@@RESULTS_S3_BUCKET_NAME_USWEST@@/$RESULTS_S3_BUCKET_NAME_USWEST/g" zappa_settings.json
+    - sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN@@/$RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN/g" zappa_settings.json
+    - sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS@@/$RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS/g" zappa_settings.json
+    - sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST@@/$RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST/g" zappa_settings.json
+    - sed -i "s#@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN@@#$RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN#g" zappa_settings.json
+    - sed -i "s/@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS@@/$RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS/g" zappa_settings.json
+    - sed -i "s#@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST@@#$RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST#g" zappa_settings.json
+    - sed -i "s#@@USAGE_REPORTING_BASE_URL_TESTING@@#$USAGE_REPORTING_BASE_URL_TESTING#g" zappa_settings.json
+    - sed -i "s#@@USAGE_REPORTING_AUTH_URL_TESTING@@#$USAGE_REPORTING_AUTH_URL_TESTING#g" zappa_settings.json
+    - sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_ID_TESTING@@#$USAGE_REPORTING_AUTH_CLIENT_ID_TESTING#g" zappa_settings.json
+    - sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING@@#$USAGE_REPORTING_AUTH_CLIENT_SECRET_TESTING#g" zappa_settings.json
+    - sed -i "s#@@LOGGING_LEVEL@@#$LOGGING_LEVEL_TESTING#g" zappa_settings.json
+    # generate zappa_settings.py for docker image from zappa_settings.json
+    - pipenv run zappa save-python-settings-file "$DEPLOYMENT_NAME"
+  # keep downloaded process definitions and zappa configuration for the following CI jobs
+  artifacts:
+    untracked: true
+    expire_in: 1 day
+
+# step 2: create docker image and push it to AWS ECR
+deploy-testing-create-docker-image:
+  stage: deploy-create-docker-image
+  only:
+    variables:
+      - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+[-]rc.[0-9]+$/
+  dependencies:
+    - deploy-testing-generate-zappa-settings
+  needs: ["deploy-testing-generate-zappa-settings"]
+  before_script:
+    - apk add --update python python-dev py-pip
+    - pip install awscli --no-build-isolation
+    # configure aws access credentials:
+    - mkdir -p ~/.aws
+    - echo -e "[default]\nregion=eu-central-1" > ~/.aws/config
+    - echo -e "[default]\naws_access_key_id=$TESTINGZAPPA_AWS_ACCESS_KEY_ID\naws_secret_access_key=$TESTINGZAPPA_AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials
+  script:
+    - cd rest/
+    # build docker image with correct tags
+    - docker build -t "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:$CI_COMMIT_TAG" -t "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:latest" --build-arg VERSION=$CI_COMMIT_TAG --build-arg VCS_REF=$CI_COMMIT_SHA --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') .
+    # login for AWS ECR docker
+    - aws ecr get-login-password --region eu-central-1 | docker login --username AWS --password-stdin "$TESTING_ECR_REGISTRY"
+    # create a repository - needs to be executed only once!
+    # - aws ecr create-repository --repository-name openeo-sh-zappa --image-scanning-configuration scanOnPush=true --region eu-central-1
+    # push docker images (versioned with CI tag and "latest") to AWS ECR with docker
+    - docker push "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:$CI_COMMIT_TAG"
+    - docker push "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:latest"
+  # keep downloaded process definitions and zappa configuration for the following CI jobs
+  artifacts:
+    untracked: true
+    expire_in: 1 day
 
+# step 3: update lambda function to use the new docker image
+deploy-testing-rest-lambda-docker:
+  stage: deploy-update-lambda
+  only:
+    variables:
+      - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+[-]rc.[0-9]+$/
+  dependencies:
+    - deploy-testing-create-docker-image
+  needs: ["deploy-testing-create-docker-image"]
+  image: python:3.9.13
+  variables:
+    DEPLOYMENT_NAME: "testing-docker"
+  before_script:
+    # configure aws access credentials:
+    - mkdir -p ~/.aws
+    - echo -e "[default]\nregion=eu-central-1" > ~/.aws/config
+    - echo -e "[default]\naws_access_key_id=$TESTINGZAPPA_AWS_ACCESS_KEY_ID\naws_secret_access_key=$TESTINGZAPPA_AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials
+    - pip install pipenv==2021.5.29
+  script:
+    - chmod +x download-process-definitions.sh
+    - ./download-process-definitions.sh
+    - cd rest/
+    - pipenv install --dev
+    # deploy lambda with new docker image
+    - pipenv run zappa deploy "$DEPLOYMENT_NAME" -d "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:latest" || pipenv run zappa update "$DEPLOYMENT_NAME" -d "$TESTING_ECR_REGISTRY/$TESTING_ECR_REGISTRY_IMAGE:latest"
+    # ensure tables are created:
+    # - export AWS_ACCESS_KEY_ID="$TESTING_AWS_ACCESS_KEY_ID"
+    # - export AWS_SECRET_ACCESS_KEY="$TESTING_AWS_SECRET_ACCESS_KEY"
+    # - pipenv run bash -c "DEPLOYMENT_TYPE=testing python dynamodb/dynamodb.py"
+
+# BEFORE 2023-10-16
+# DEPLOY PRODUCTION: uploading .zip file on lambda
 deploy-production-rest-lambda:
   stage: deploy
   when: manual
   only:
     variables:
-      - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]$/
+      - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+$/
   image: python:3.9.13
   before_script:
     # configure aws access credentials:
@@ -147,3 +269,120 @@ deploy-production-rest-lambda:
     - export AWS_ACCESS_KEY_ID="$PRODDATA_AWS_ACCESS_KEY_ID"
     - export AWS_SECRET_ACCESS_KEY="$PRODDATA_AWS_SECRET_ACCESS_KEY"
     - pipenv run bash -c "DEPLOYMENT_TYPE=production python dynamodb/dynamodb.py"
+
+# AFTER 2023-10-16
+# DEPLOY PRODUCTION: create docker image, upload it to AWS ECR and use it in AWS Lambda
+# - step 1: generate zappa settings for the app inside the docker image
+# - step 2: create docker image and push it to AWS ECR
+# - step 3: update lambda function to use the new docker image
+
+# step 1: generate zappa settings for the app inside the docker image
+deploy-production-generate-zappa-settings:
+  stage: deploy-generate-zappa-settings
+  when: manual
+  only:
+    variables:
+      - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+$/
+  image: python:3.9.13
+  variables:
+    DEPLOYMENT_NAME: "production-docker"
+  before_script:
+    # configure aws access credentials:
+    - mkdir -p ~/.aws
+    - echo -e "[default]\nregion=eu-central-1" > ~/.aws/config
+    - echo -e "[default]\naws_access_key_id=$PRODZAPPA_AWS_ACCESS_KEY_ID\naws_secret_access_key=$PRODZAPPA_AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials
+    - pip install pipenv==2021.5.29
+  script:
+    - chmod +x download-process-definitions.sh
+    - ./download-process-definitions.sh
+    - cd rest/
+    - pipenv install --dev
+    # create zappa_settings.json on-the-fly:
+    - cp zappa_settings.json.template zappa_settings.json
+    - sed -i "s/@@AWS_ACCESS_KEY_ID@@/$PRODDATA_AWS_ACCESS_KEY_ID/g" zappa_settings.json
+    - sed -i "s#@@AWS_SECRET_ACCESS_KEY@@#$PRODDATA_AWS_SECRET_ACCESS_KEY#g" zappa_settings.json
+    - sed -i "s/@@PRODUCTION_SH_CLIENT_ID@@/$PRODUCTION_SH_CLIENT_ID/g" zappa_settings.json
+    - sed -i "s/@@PRODUCTION_SH_CLIENT_SECRET@@/$PRODUCTION_SH_CLIENT_SECRET/g" zappa_settings.json
+    - sed -i "s/@@BACKEND_VERSION@@/$CI_COMMIT_TAG/g" zappa_settings.json
+    - sed -i "s/@@RESULTS_S3_BUCKET_NAME_MAIN@@/$RESULTS_S3_BUCKET_NAME_MAIN_PRODUCTION/g" zappa_settings.json
+    - sed -i "s/@@RESULTS_S3_BUCKET_NAME_CREODIAS@@/$RESULTS_S3_BUCKET_NAME_CREODIAS_PRODUCTION/g" zappa_settings.json
+    - sed -i "s/@@RESULTS_S3_BUCKET_NAME_USWEST@@/$RESULTS_S3_BUCKET_NAME_USWEST_PRODUCTION/g" zappa_settings.json
+    - sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN@@/$RESULTS_S3_BUCKET_ACCESS_KEY_ID_MAIN_PRODUCTION/g" zappa_settings.json
+    - sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS@@/$RESULTS_S3_BUCKET_ACCESS_KEY_ID_CREODIAS_PRODUCTION/g" zappa_settings.json
+    - sed -i "s/@@RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST@@/$RESULTS_S3_BUCKET_ACCESS_KEY_ID_USWEST_PRODUCTION/g" zappa_settings.json
+    - sed -i "s#@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN@@#$RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_MAIN_PRODUCTION#g" zappa_settings.json
+    - sed -i "s/@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS@@/$RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_CREODIAS_PRODUCTION/g" zappa_settings.json
+    - sed -i "s#@@RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST@@#$RESULTS_S3_BUCKET_SECRET_ACCESS_KEY_USWEST_PRODUCTION#g" zappa_settings.json
+    - sed -i "s#@@USAGE_REPORTING_BASE_URL_PRODUCTION@@#$USAGE_REPORTING_BASE_URL_PRODUCTION#g" zappa_settings.json
+    - sed -i "s#@@USAGE_REPORTING_AUTH_URL_PRODUCTION@@#$USAGE_REPORTING_AUTH_URL_PRODUCTION#g" zappa_settings.json
+    - sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_ID_PRODUCTION@@#$USAGE_REPORTING_AUTH_CLIENT_ID_PRODUCTION#g" zappa_settings.json
+    - sed -i "s#@@USAGE_REPORTING_AUTH_CLIENT_SECRET_PRODUCTION@@#$USAGE_REPORTING_AUTH_CLIENT_SECRET_PRODUCTION#g" zappa_settings.json
+    - sed -i "s#@@LOGGING_LEVEL@@#$LOGGING_LEVEL_PRODUCTION#g" zappa_settings.json
+    # generate zappa_settings.py for docker image from zappa_settings.json
+    - pipenv run zappa save-python-settings-file "$DEPLOYMENT_NAME"
+  # keep downloaded process definitions and zappa configuration for the following CI jobs
+  artifacts:
+    untracked: true
+    expire_in: 1 day
+
+# step 2: create docker image and push it to AWS ECR
+deploy-production-create-docker-image:
+  stage: deploy-create-docker-image
+  only:
+    variables:
+      - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+$/
+  dependencies:
+    - deploy-production-generate-zappa-settings
+  needs: ["deploy-production-generate-zappa-settings"]
+  before_script:
+    - apk add --update python python-dev py-pip
+    - pip install awscli --no-build-isolation
+    # configure aws access credentials:
+    - mkdir -p ~/.aws
+    - echo -e "[default]\nregion=eu-central-1" > ~/.aws/config
+    - echo -e "[default]\naws_access_key_id=$PRODZAPPA_AWS_ACCESS_KEY_ID\naws_secret_access_key=$PRODZAPPA_AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials
+  script:
+    - cd rest/
+    # build docker image with correct tags
+    - docker build -t "$PROD_ECR_REGISTRY/$PROD_ECR_REGISTRY_IMAGE:$CI_COMMIT_TAG" -t "$PROD_ECR_REGISTRY/$PROD_ECR_REGISTRY_IMAGE:latest" --build-arg VERSION=$CI_COMMIT_TAG --build-arg VCS_REF=$CI_COMMIT_SHA --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') .
+    # login for AWS ECR docker
+    - aws ecr get-login-password --region eu-central-1 | docker login --username AWS --password-stdin "$PROD_ECR_REGISTRY"
+    # create a repository - needs to be executed only once!
+    # - aws ecr create-repository --repository-name openeo-sh-zappa --image-scanning-configuration scanOnPush=true --region eu-central-1
+    # push docker images (versioned with CI tag and "latest") to AWS ECR with docker
+    - docker push "$PROD_ECR_REGISTRY/$PROD_ECR_REGISTRY_IMAGE:$CI_COMMIT_TAG"
+    - docker push "$PROD_ECR_REGISTRY/$PROD_ECR_REGISTRY_IMAGE:latest"
+  # keep downloaded process definitions and zappa configuration for the following CI jobs
+  artifacts:
+    untracked: true
+    expire_in: 1 day
+
+# step 3: update lambda function to use the new docker image
+deploy-production-rest-lambda-docker:
+  stage: deploy-update-lambda
+  only:
+    variables:
+      - $CI_COMMIT_TAG =~ /^v[0-9]+[.][0-9]+[.][0-9]+$/
+  dependencies:
+    - deploy-production-create-docker-image
+  needs: ["deploy-production-create-docker-image"]
+  image: python:3.9.13
+  variables:
+    DEPLOYMENT_NAME: "production-docker"
+  before_script:
+    # configure aws access credentials:
+    - mkdir -p ~/.aws
+    - echo -e "[default]\nregion=eu-central-1" > ~/.aws/config
+    - echo -e "[default]\naws_access_key_id=$PRODZAPPA_AWS_ACCESS_KEY_ID\naws_secret_access_key=$PRODZAPPA_AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials
+    - pip install pipenv==2021.5.29
+  script:
+    - chmod +x download-process-definitions.sh
+    - ./download-process-definitions.sh
+    - cd rest/
+    - pipenv install --dev
+    # deploy lambda with new docker image
+    - pipenv run zappa deploy "$DEPLOYMENT_NAME" -d "$PROD_ECR_REGISTRY/$PROD_ECR_REGISTRY_IMAGE:latest" || pipenv run zappa update "$DEPLOYMENT_NAME" -d "$PROD_ECR_REGISTRY/$PROD_ECR_REGISTRY_IMAGE:latest"
+    # ensure tables are created:
+    # - export AWS_ACCESS_KEY_ID="$PRODDATA_AWS_ACCESS_KEY_ID"
+    # - export AWS_SECRET_ACCESS_KEY="$PRODDATA_AWS_SECRET_ACCESS_KEY"
+    # - pipenv run bash -c "DEPLOYMENT_TYPE=production python dynamodb/dynamodb.py"

rest/Dockerfile

@@ -0,0 +1,38 @@
+# resources
+# https://www.cloudtechsimplified.com/run-docker-containers-images-from-ecr-in-aws-lambda-along-with-cicd-python/
+# https://docs.aws.amazon.com/lambda/latest/dg/python-image.html#python-image-instructions
+# https://gallery.ecr.aws/lambda/python/
+# https://zebradil.me/post/2018-05-25-pipenv-for-aws-lambda/
+# https://stackoverflow.com/questions/48381918/how-do-i-install-git-using-aws-lambda
+# https://stackoverflow.com/a/68787552
+# https://github.com/zappa/Zappa#docker-workflows
+# https://ianwhitestone.work/zappa-serverless-docker/
+
+# use base aws lambda docker image for python (all the sources listed bellow)
+FROM public.ecr.aws/lambda/python:3.9
+
+# install pipenv and git that are needed to install dependencies
+RUN yum -y install git
+RUN pip install pipenv
+
+# copy the /rest folder
+COPY ./ ${LAMBDA_TASK_ROOT}
+
+# create requirements.txt file and
+# install dependencies with pip
+RUN pipenv requirements > requirements.txt
+RUN pip install -r requirements.txt
+
+# Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile)
+# app.app is the same as in zappa_settings.json.template
+# CMD [ "app.app" ]
+
+# https://ianwhitestone.work/zappa-serverless-docker/
+# Grab the zappa handler.py and put it in the working directory
+RUN ZAPPA_HANDLER_PATH=$( \
+  python -c "from zappa import handler; print (handler.__file__)" \
+  ) \
+  && echo $ZAPPA_HANDLER_PATH \
+  && cp $ZAPPA_HANDLER_PATH ${LAMBDA_TASK_ROOT}
+
+CMD [ "handler.lambda_handler" ]

rest/Pipfile

@@ -9,11 +9,13 @@ pylint = "==2.12.2"
 responses = "==0.21.0"
 cryptography = "==36.0.1"
 black = "==22.3.0"
+awscli = "*"
 
 [requires]
 python_version = "3.9"
 
 [packages]
+zappa = "==0.54.1"
 flask = "==2.0.3"
 pytest = "==7.0.1"
 boto3 = "==1.21.18"
@@ -31,4 +33,4 @@ placebo = "==0.9.0"
 rioxarray = "*"
 zarr = "*"
 netcdf4 = "*"
-pip-tools = "==6.1.0"
+werkzeug = "==2.3.7"