From f0f5da993e3c4cec1144110e39f813fd1932a396 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Mon, 17 May 2021 09:54:27 +0000 Subject: [PATCH 01/20] Update default docker-libs image --- settings.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.conf b/settings.conf index 6aa349a..df717b9 100644 --- a/settings.conf +++ b/settings.conf @@ -22,7 +22,7 @@ BACKEND_PORT=8000 FLOWER_PORT=5555 # Base iamge for building the Back-end image -DOCKER_LIBS_IMAGE=dhealth/pylibs:develop_build77 +DOCKER_LIBS_IMAGE=dhealth/pylibs-toolkit:0.10.0-gpu # Docker image name DOCKER_BACKEND_IMAGE=dhealth/backend From 0cf3c6be542ef0d3047113ec801c36973e1afd79 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Mon, 17 May 2021 15:25:48 +0000 Subject: [PATCH 02/20] Update image: make data paths relative to DATA_DIR --- config.template | 7 +++---- docker-compose.template.yml | 15 +++------------ docker/Dockerfile | 17 ++++------------- settings.conf | 4 +--- setup.sh | 5 +---- 5 files changed, 12 insertions(+), 36 deletions(-) diff --git a/config.template b/config.template index 2d47b68..f55ee23 100644 --- a/config.template +++ b/config.template @@ -17,10 +17,9 @@ CORS_ORIGIN_WHITELIST="${CORS_ORIGIN_WHITELIST}" # Relative path for static resources STATIC_URL="${STATIC_URL}" -# Data directories -TRAINING_DIR="${TRAINING_DIR}" -INFERENCE_DIR="${INFERENCE_DIR}" -DATASETS_DIR="${DATASETS_DIR}" + +# Data directory +DATA_DIR="${DATA_DIR}" # RabbitMQ Credentials # see https://hub.docker.com/_/rabbitmq diff --git a/docker-compose.template.yml b/docker-compose.template.yml index 0882777..ed725ad 100644 --- a/docker-compose.template.yml +++ b/docker-compose.template.yml @@ -23,12 +23,9 @@ services: ports: - "${BACKEND_PORT}:8000" volumes: - ${BACKEND_VOLUME} - ./.config:/app/config - ./settings.conf:/app/settings.conf - - datasets:${DATASETS_DIR} - - training:${TRAINING_DIR} - - inference:${INFERENCE_DIR} + - data:${DATA_DIR} depends_on: - broker - db @@ -48,12 +45,9 @@ services: - DATABASE_URL=psql://${POSTGRESQL_USERNAME}:${POSTGRESQL_PASSWORD}@db:5432/${POSTGRESQL_DATABASE} # PostgreSQL DB command: ["start.sh","celery"] volumes: - ${BACKEND_VOLUME} - ./.config:/app/config - ./settings.conf:/app/settings.conf - - datasets:${DATASETS_DIR} - - training:${TRAINING_DIR} - - inference:${INFERENCE_DIR} + - data:${DATA_DIR} depends_on: - broker - db @@ -65,7 +59,6 @@ services: environment: - PYTHONPATH="$PYTHONPATH:.%" volumes: - ${BACKEND_VOLUME} - ./.config:/app/config - ./settings.conf:/app/settings.conf - postgresql:/bitnami/postgresql @@ -89,6 +82,4 @@ services: volumes: postgresql: - datasets: - inference: - training: \ No newline at end of file + data: diff --git a/docker/Dockerfile b/docker/Dockerfile index b26f691..d263f3c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -15,12 +15,8 @@ ENV BACKEND_PORT ${BACKEND_PORT} ENV APP_PATH "/app" # Set data paths -ARG DATASETS_DIR="/data/datasets" -ENV DATASETS_DIR=${DATASETS_DIR} -ARG TRAINING_DIR="/data/training" -ARG TRAINING_DIR=${TRAINING_DIR} -ARG INFERENCE_DIR="/data/inference" -ARG INFERENCE_DIR=${INFERENCE_DIR} +ARG DATA_DIR="/data" +ENV DATA_DIR=${DATA_DIR} # Copy source path COPY src ${APP_PATH} @@ -52,13 +48,8 @@ RUN \ && groupadd ${USERNAME} -g ${GID} || true \ && useradd -m -u ${UID} -g ${GID} ${USERNAME} \ && chown -R ${USERNAME} ${APP_PATH} \ - && mkdir -p ${DATASETS_DIR} ${INFERENCE_DIR} ${TRAINING_DIR} \ - && chown -R ${USERNAME} ${DATASETS_DIR} ${TRAINING_DIR} ${INFERENCE_DIR} - -# Declare data volumes -#VOLUME ["${DATASETS_DIR}"] -#VOLUME ["${INFERENCE_DIR}"] -#VOLUME ["${TRAINING_DIR}"] + && mkdir -p ${DATA_DIR} \ + && chown -R ${USERNAME} ${DATA_DIR} # Set the default user USER ${USERNAME} diff --git a/settings.conf b/settings.conf index df717b9..6af94a7 100644 --- a/settings.conf +++ b/settings.conf @@ -46,9 +46,7 @@ ALLOWED_HOSTS=localhost,127.0.0.1 CORS_ORIGIN_WHITELIST=http://localhost:4200 # Data directories -TRAINING_DIR=/data/training -INFERENCE_DIR=/data/inference -DATASETS_DIR=/data/datasets +DATA_DIR=/data # Celery settings CELERY_ACCEPT_CONTENT=json diff --git a/setup.sh b/setup.sh index 6c6b64e..b738ec5 100755 --- a/setup.sh +++ b/setup.sh @@ -45,10 +45,7 @@ fi # build images docker-compose build \ --build-arg DOCKER_LIBS_IMAGE="${DOCKER_LIBS_IMAGE:-}" \ - --build-arg DATASETS_DIR="${DATASETS_DIR:-}" \ - --build-arg TRAINING_DIR="${TRAINING_DIR:-}" \ - --build-arg INFERENCE_DIR="${INFERENCE_DIR:-}" - + --build-arg DATA_DIR="${DATA_DIR:-}" # clean up source #rm -rf docker/src From ecb49fde406a6c727fe0045e807b2dfed4f9b7be Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Mon, 17 May 2021 16:34:56 +0000 Subject: [PATCH 03/20] Fix configuration of back-end port --- docker-compose.template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.template.yml b/docker-compose.template.yml index ed725ad..fb2a119 100644 --- a/docker-compose.template.yml +++ b/docker-compose.template.yml @@ -21,7 +21,7 @@ services: - DATABASE_URL=psql://${POSTGRESQL_USERNAME}:${POSTGRESQL_PASSWORD}@db:5432/${POSTGRESQL_DATABASE} # PostgreSQL DB - POSTGRESQL_HOST=db ports: - - "${BACKEND_PORT}:8000" + - "${BACKEND_PORT}:${BACKEND_PORT}" volumes: - ./.config:/app/config - ./settings.conf:/app/settings.conf From aa3f654abba95c1d25ddb731a062ee23175c5454 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Mon, 17 May 2021 16:35:28 +0000 Subject: [PATCH 04/20] Set pylibs-toolkit:latest as default base image --- settings.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.conf b/settings.conf index 6af94a7..b776c26 100644 --- a/settings.conf +++ b/settings.conf @@ -22,7 +22,7 @@ BACKEND_PORT=8000 FLOWER_PORT=5555 # Base iamge for building the Back-end image -DOCKER_LIBS_IMAGE=dhealth/pylibs-toolkit:0.10.0-gpu +DOCKER_LIBS_IMAGE=dhealth/pylibs-toolkit:latest # Docker image name DOCKER_BACKEND_IMAGE=dhealth/backend From 8e22b3e3bb2c8960cb86693c49d4dd3917223571 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Mon, 17 May 2021 16:37:01 +0000 Subject: [PATCH 05/20] Config GPU support via EDDL_WITH_CUDA; hide Docker runtime --- config.template | 3 +++ settings.conf | 7 ++----- setup.sh | 10 ++++++++++ 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/config.template b/config.template index f55ee23..a3bedde 100644 --- a/config.template +++ b/config.template @@ -1,6 +1,9 @@ # Allow DEBUG mode. Don't run with debug turned on in production! DEBUG=${DEBUG} +# Enable eddl CS_GPU computing service +EDDL_WITH_CUDA=${EDDL_WITH_CUDA} + # Set SECRET_KEY. Keep the secret key used in production secret! SECRET_KEY="${SECRET_KEY}" diff --git a/settings.conf b/settings.conf index b776c26..9e85426 100644 --- a/settings.conf +++ b/settings.conf @@ -27,11 +27,8 @@ DOCKER_LIBS_IMAGE=dhealth/pylibs-toolkit:latest # Docker image name DOCKER_BACKEND_IMAGE=dhealth/backend -# Set Docker Runtime: runc | nvidia -# The current implementation only support 'nvidia' runtime -# Notice that to use GPUs you need to use the 'nvidia' -# DOCKER_RUNTIME *and* use a GPU-enabled pylibs container image. -DOCKER_RUNTIME=runc #"nvidia" +# Set to 'True' to enable eddl CS_GPU computing service (default: False) +EDDL_WITH_CUDA=False # Set path to your local copy of the Back-end #BACKEND_LOCAL_PATH="" diff --git a/setup.sh b/setup.sh index b738ec5..06dcf7d 100755 --- a/setup.sh +++ b/setup.sh @@ -21,6 +21,16 @@ export SECRET_KEY=$(\ docker run -it --rm "${DOCKER_BACKEND_IMAGE}" \ python3 -c 'from django.core.management.utils import get_random_secret_key; print(get_random_secret_key())' | tr -d '[:space:]') +# Set Docker Runtime: runc | nvidia +# The current implementation only support 'nvidia' runtime +# Notice that to use GPUs you need to use the 'nvidia' +# DOCKER_RUNTIME *and* use a GPU-enabled pylibs container image. +DOCKER_RUNTIME=runc +if [[ "${EDDL_WITH_CUDA}" == "True" || "${EDDL_WITH_CUDA}" == "true" ]]; then + DOCKER_RUNTIME=nvidia +fi +export DOCKER_RUNTIME + # generate config file from settings envsubst < config.template > .config From 11fc725880e62868c7aeb8f6f97d161647460bb1 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Mon, 17 May 2021 16:38:52 +0000 Subject: [PATCH 06/20] Fix missing STATIC and MEDIA urls on config template --- config.template | 4 +++- settings.conf | 3 --- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/config.template b/config.template index a3bedde..7dc1647 100644 --- a/config.template +++ b/config.template @@ -18,8 +18,10 @@ CORS_ORIGIN_WHITELIST="${CORS_ORIGIN_WHITELIST}" # DATABASE_URL=psql://urser:un-githubbedpassword@127.0.0.1:8458/database # PostgreSQL DB # Relative path for static resources -STATIC_URL="${STATIC_URL}" +STATIC_URL='/backend/static/' +# Base url to serve media files +MEDIA_URL='/backend/media/' # Data directory DATA_DIR="${DATA_DIR}" diff --git a/settings.conf b/settings.conf index 9e85426..dc183a5 100644 --- a/settings.conf +++ b/settings.conf @@ -33,9 +33,6 @@ EDDL_WITH_CUDA=False # Set path to your local copy of the Back-end #BACKEND_LOCAL_PATH="" -# Django static -STATIC_URL=/backend/static/ - # Comma separated allowed hosts ALLOWED_HOSTS=localhost,127.0.0.1 From aee49d3eb84bb03a8ad0aaf88a09d2fa4b39d461 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Mon, 17 May 2021 16:53:17 +0000 Subject: [PATCH 07/20] Allow to configure email account and OAuth credentials for external identity providers --- config.template | 7 +++++++ k8s/deephealth-backend/templates/secrets.yaml | 12 ++++++++++++ k8s/deephealth-backend/values.yaml | 13 ++++++++++++- k8s/values.yaml | 12 +++++++++++- settings.conf | 7 +++++++ 5 files changed, 49 insertions(+), 2 deletions(-) diff --git a/config.template b/config.template index 7dc1647..2f2b9a6 100644 --- a/config.template +++ b/config.template @@ -35,3 +35,10 @@ CELERY_ACCEPT_CONTENT="${CELERY_ACCEPT_CONTENT}" # list of comma separated value CELERY_RESULT_BACKEND="${CELERY_RESULT_BACKEND}" CELERY_TASK_SERIALIZER="${CELERY_TASK_SERIALIZER}" +# OAuth settings +DRFSO2_PROPRIETARY_BACKEND_NAME='GithubOAuth2' +DRFSO2_URL_NAMESPACE='social' + +# GitHub configuration +SOCIAL_AUTH_GITHUB_KEY='' +SOCIAL_AUTH_GITHUB_SECRET='' diff --git a/k8s/deephealth-backend/templates/secrets.yaml b/k8s/deephealth-backend/templates/secrets.yaml index 4668937..8e06fc4 100644 --- a/k8s/deephealth-backend/templates/secrets.yaml +++ b/k8s/deephealth-backend/templates/secrets.yaml @@ -58,6 +58,18 @@ stringData: CELERY_ACCEPT_CONTENT={{ .Values.celery.acceptContent | quote }} # CELERY_RESULT_BACKEND=db+sqlite:///results.sqlite CELERY_TASK_SERIALIZER={{ .Values.celery.taskSerializer | quote }} + + # OAuth settings + DRFSO2_PROPRIETARY_BACKEND_NAME='GithubOAuth2' + DRFSO2_URL_NAMESPACE='social' + + # GitHub configuration + SOCIAL_AUTH_GITHUB_KEY={{ .Values.backend.social_auth.github.client_id | quote }} + SOCIAL_AUTH_GITHUB_SECRET={{ .Values.backend.social_auth.github.client_secret | quote }} + + # Email backend settings + EMAIL_HOST_USER={{ .Values.backend.email.user | quote }} + EMAIL_HOST_PASSWORD={{ .Values.backend.email.password | quote }} --- {{- end }} {{- if or .Release.IsInstall (and .Release.IsUpgrade (or .Values.postgresql.postgresqlPassword .Values.postgresql.postgresqlPostgresPassword)) }} diff --git a/k8s/deephealth-backend/values.yaml b/k8s/deephealth-backend/values.yaml index 72b5f05..37324bb 100644 --- a/k8s/deephealth-backend/values.yaml +++ b/k8s/deephealth-backend/values.yaml @@ -82,8 +82,19 @@ backend: staticFiles: storageClass: *globalStorageClass size: 10Mi - + # Secrets to access social authentication providers + social_auth: + github: + client_id: + client_secret: + + # Email settings + email: + user: + password: + + # Configure pod resources resources: {} # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little diff --git a/k8s/values.yaml b/k8s/values.yaml index c62adc6..7fd1bc4 100644 --- a/k8s/values.yaml +++ b/k8s/values.yaml @@ -74,7 +74,17 @@ backend: allowedHosts: "*" # Cross-Origin Resource Sharing (CORS) whitelist - corsOriginWhiteList: "" + + # Secrets to access social authentication providers + social_auth: + github: + client_id: + client_secret: + + # Email settings + email: + user: + password: resources: {} # We usually recommend not to specify default resources and to leave this as a conscious diff --git a/settings.conf b/settings.conf index dc183a5..4640dcc 100644 --- a/settings.conf +++ b/settings.conf @@ -47,3 +47,10 @@ CELERY_ACCEPT_CONTENT=json CELERY_RESULT_BACKEND=db+sqlite:///results.sqlite CELERY_TASK_SERIALIZER=json +# Social login settings +SOCIAL_AUTH_GITHUB_KEY= +SOCIAL_AUTH_GITHUB_SECRET= + +# Email backend settings +EMAIL_HOST_USER= +EMAIL_HOST_PASSWORD= From af20950dbf322c5bd8864f7801291f1273327f9a Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Mon, 17 May 2021 17:05:34 +0000 Subject: [PATCH 08/20] Fix trailing whitespaces and EOF --- docker/Dockerfile | 4 +- .../templates/init-job.yaml | 6 +- k8s/deephealth-backend/templates/secrets.yaml | 8 +-- k8s/deephealth-backend/values.yaml | 38 +++++------ k8s/values.yaml | 65 +++++++++---------- 5 files changed, 58 insertions(+), 63 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index d263f3c..c883d71 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -9,7 +9,7 @@ ARG USERNAME=backend # Set defaults ARG BACKEND_PORT=8000 -ENV BACKEND_PORT ${BACKEND_PORT} +ENV BACKEND_PORT ${BACKEND_PORT} # Set the path where the app will be installed ENV APP_PATH "/app" @@ -55,4 +55,4 @@ RUN \ USER ${USERNAME} # Default command -CMD start.sh backend \ No newline at end of file +CMD start.sh backend diff --git a/k8s/deephealth-backend/templates/init-job.yaml b/k8s/deephealth-backend/templates/init-job.yaml index ffa3621..5ba58ff 100644 --- a/k8s/deephealth-backend/templates/init-job.yaml +++ b/k8s/deephealth-backend/templates/init-job.yaml @@ -17,10 +17,10 @@ spec: - name: wait-for-postgresql image: "{{ .Values.postgresql.image.repository }}:{{ .Values.postgresql.image.tag }}" imagePullPolicy: {{ .Values.backend.image.pullPolicy }} - command: + command: - /bin/sh - -c - - until PGPASSWORD=${POSTGRES_PASSWORD} psql -h "{{ include "deephealth-backend.fullname" . }}-postgresql" -U "{{ .Values.postgresql.postgresqlUsername }}" {{ .Values.postgresql.postgresqlDatabase }} -c '\q'; do + - until PGPASSWORD=${POSTGRES_PASSWORD} psql -h "{{ include "deephealth-backend.fullname" . }}-postgresql" -U "{{ .Values.postgresql.postgresqlUsername }}" {{ .Values.postgresql.postgresqlDatabase }} -c '\q'; do >&2 echo "PostgreSQL is unavailable -- sleep 2 seconds and retry" ; sleep 2 ; done ; @@ -83,4 +83,4 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} - backoffLimit: 4 \ No newline at end of file + backoffLimit: 4 diff --git a/k8s/deephealth-backend/templates/secrets.yaml b/k8s/deephealth-backend/templates/secrets.yaml index 8e06fc4..89f1e93 100644 --- a/k8s/deephealth-backend/templates/secrets.yaml +++ b/k8s/deephealth-backend/templates/secrets.yaml @@ -36,7 +36,7 @@ stringData: # Cross-Origin Resource Sharing (CORS) whitelist CORS_ORIGIN_WHITELIST="{{- range .Values.ingress.hosts }},http://{{ .host }},https://{{ .host }}{{- end }}{{- if .Values.backend.corsOriginWhiteList }},{{ .Values.backend.corsOriginWhiteList }}{{- end }}" - + # Database connection settings # DATABASE_URL=sqlite:///my-local-sqlite.db # SQLite DB # DATABASE_URL=psql://{{ .Values.postgresql.postgresqlUsername }}:{{ .Values.postgresql.postgresqlPassword }}@{{ include "deephealth-backend.fullname" . }}-postgresql:{{ .Values.postgresql.service.port }}/{{ .Values.postgresql.postgresqlDatabase }} @@ -49,7 +49,7 @@ stringData: # Data directories DATA_DIR={{ .Values.backend.persistence.data.path }} - + # RabbitMQ Credentials # see https://hub.docker.com/_/rabbitmq # RABBITMQ_BROKER_URL=amqp://{{ .Values.broker.rabbitmq.username }}:{{ .Values.broker.rabbitmq.password }}@{{ include "deephealth-backend.fullname" . }}-rabbitmq:{{ .Values.broker.service.port }} @@ -87,7 +87,7 @@ metadata: helm.sh/resource-policy: keep type: Opaque data: - postgresql-postgres-password: {{ include "deephealth-backend.postgresql.postgres-password" . | b64enc | quote }} + postgresql-postgres-password: {{ include "deephealth-backend.postgresql.postgres-password" . | b64enc | quote }} postgresql-password: {{ include "deephealth-backend.postgresql.password" . | b64enc | quote }} {{- if .Values.postgresql.replication.enabled }} postgresql-replication-password: {{ include "deephealth-backend.postgresql.replication.password" . | b64enc | quote }} @@ -122,4 +122,4 @@ data: {{ else }} rabbitmq-erlang-cookie: {{ randAlphaNum 32 | b64enc | quote }} {{ end }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/k8s/deephealth-backend/values.yaml b/k8s/deephealth-backend/values.yaml index 37324bb..7af3aee 100644 --- a/k8s/deephealth-backend/values.yaml +++ b/k8s/deephealth-backend/values.yaml @@ -56,7 +56,7 @@ backend: # number of replicas replicaCount: 1 - + # number of Gunicorn workers workers: 3 @@ -106,9 +106,9 @@ backend: # requests: # cpu: 100m # memory: 128Mi - # To use GPU you need to install - # https://github.com/NVIDIA/k8s-device-plugin) - # on your k8s Cluster + # To use GPU you need to install + # https://github.com/NVIDIA/k8s-device-plugin) + # on your k8s Cluster # limits: # nvidia.com/gpu: 2 # requesting 2 GPUs @@ -124,11 +124,11 @@ nginx: nameOverride: "nginx" image: - # repository: + # repository: # tag: latest pullPolicy: *imagePullPolicy debug: *debug - + service: type: ClusterIP port: 80 @@ -148,7 +148,7 @@ nginx: # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: "true" # hosts: - # - host: + # - host: # name: pippo.backend.172.30.10.101.nip.io # path: / @@ -175,7 +175,7 @@ celery: acceptContent: 'json' # list of comma separated values taskSerializer: 'json' - + service: # type: ClusterIP # port: 80 @@ -205,9 +205,9 @@ celery: # requests: # cpu: 100m # memory: 128Mi - # To use GPU you need to install - # https://github.com/NVIDIA/k8s-device-plugin) - # on your k8s Cluster + # To use GPU you need to install + # https://github.com/NVIDIA/k8s-device-plugin) + # on your k8s Cluster # limits: # nvidia.com/gpu: 2 # requesting 2 GPUs @@ -218,7 +218,7 @@ celery: affinity: {} -# RabbitMQ settings. You can find all the available settings +# RabbitMQ settings. You can find all the available settings # at https://github.com/bitnami/charts/tree/master/bitnami/rabbitmq broker: &broker nameOverride: "rabbitmq" @@ -226,11 +226,11 @@ broker: &broker image: debug: *debug pullPolicy: *imagePullPolicy - + rabbitmq: username: user # password: password - + service: port: 5672 @@ -242,7 +242,7 @@ broker: &broker rabbitmq: existingPasswordSecret: deephealth-backend-rabbitmq-secrets existingErlangSecret: deephealth-backend-rabbitmq-secrets - + resources: {} # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little @@ -319,7 +319,7 @@ console: debug: *debug replicaCount: 1 - + resources: {} # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little @@ -331,9 +331,9 @@ console: # requests: # cpu: 100m # memory: 128Mi - # To use GPU you need to install - # https://github.com/NVIDIA/k8s-device-plugin) - # on your k8s Cluster + # To use GPU you need to install + # https://github.com/NVIDIA/k8s-device-plugin) + # on your k8s Cluster # limits: # nvidia.com/gpu: 2 # requesting 2 GPUs diff --git a/k8s/values.yaml b/k8s/values.yaml index 7fd1bc4..51ed483 100644 --- a/k8s/values.yaml +++ b/k8s/values.yaml @@ -14,13 +14,11 @@ global: # preserve PVC when a release is deleted retainPVCs: True - # ServiceType of the Back-end EndPoint # endpoint: # service: # type: NodePort - # Set external access to the services # ingress: # enabled: true @@ -28,14 +26,14 @@ global: # kubernetes.io/ingress.class: nginx # # kubernetes.io/tls-acme: "true" # hosts: -# - host: backend.172.30.10.101.nip.io +# - host: backend.172.30.10.100.nip.io # serviceName: endpoint # service name without ReleasePrefix # servicePort: 80 # tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local +# - secretName: chart-example-tls +# hosts: +# - chart-example.local # Backend settings backend: @@ -86,7 +84,9 @@ backend: user: password: - resources: {} + # Configure pod resources + resources: + {} # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little # resources, such as Minikube. If you do want to specify resources, uncomment the following @@ -97,9 +97,9 @@ backend: # requests: # cpu: 100m # memory: 128Mi - # To use GPU you need to install - # https://github.com/NVIDIA/k8s-device-plugin) - # on your k8s Cluster + # To use GPU you need to install + # https://github.com/NVIDIA/k8s-device-plugin) + # on your k8s Cluster # limits: # nvidia.com/gpu: 2 # requesting 2 GPUs @@ -109,14 +109,13 @@ backend: affinity: {} - # nginx settings nginx: nameOverride: "nginx" image: debug: *debug - + # service: # type: NodePort # port: 80 @@ -125,16 +124,15 @@ nginx: #serverBlockConfigMap: proxy-config # serverDataVolumeClaim: static-files # serverDataVolumePath: /app/static - ingress: # enabled: false # hostname: nginx.backend.172.30.10.101.nip.io # annotations: # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" + # kubernetes.io/tls-acme: "true" # hosts: - # - host: + # - host: # name: pippo.backend.172.30.10.101.nip.io # path: / @@ -144,7 +142,8 @@ nginx: # hosts: # - chart-example.local - resources: {} + resources: + {} # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little # resources, such as Minikube. If you do want to specify resources, uncomment the following @@ -159,9 +158,6 @@ nginx: # Celery settings celery: - acceptContent: 'json' # list of comma separated values - taskSerializer: 'json' - service: # type: ClusterIP # port: 80 @@ -180,7 +176,8 @@ celery: # # hosts: # # - chart-example.local - resources: {} + resources: + {} # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little # resources, such as Minikube. If you do want to specify resources, uncomment the following @@ -191,9 +188,9 @@ celery: # requests: # cpu: 100m # memory: 128Mi - # To use GPU you need to install - # https://github.com/NVIDIA/k8s-device-plugin) - # on your k8s Cluster + # To use GPU you need to install + # https://github.com/NVIDIA/k8s-device-plugin) + # on your k8s Cluster # limits: # nvidia.com/gpu: 2 # requesting 2 GPUs @@ -203,19 +200,18 @@ celery: affinity: {} - -# RabbitMQ settings. You can find all the available settings +# RabbitMQ settings. You can find all the available settings # at https://github.com/bitnami/charts/tree/master/bitnami/rabbitmq broker: &broker nameOverride: "rabbitmq" image: debug: *debug - + rabbitmq: username: user # password: password - + service: port: 5672 @@ -223,7 +219,7 @@ broker: &broker storageClass: *globalStorageClass size: 1Gi # existingClaim: "" - + resources: {} # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little @@ -245,7 +241,6 @@ broker: &broker # Copy broker properties as values of the subchart RabbitMQ rabbitmq: *broker - # PostgreSQL settings. # You can find all the available settings at https://github.com/bitnami/charts/tree/master/bitnami/postgresql postgresql: @@ -285,7 +280,6 @@ postgresql: affinity: {} - # Debug console settings console: enabled: *debug @@ -296,8 +290,9 @@ console: # debug: *debug replicaCount: 1 - - resources: {} + + resources: + {} # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little # resources, such as Minikube. If you do want to specify resources, uncomment the following @@ -308,9 +303,9 @@ console: # requests: # cpu: 100m # memory: 128Mi - # To use GPU you need to install - # https://github.com/NVIDIA/k8s-device-plugin) - # on your k8s Cluster + # To use GPU you need to install + # https://github.com/NVIDIA/k8s-device-plugin) + # on your k8s Cluster # limits: # nvidia.com/gpu: 2 # requesting 2 GPUs From 0375e199dec0da6ecd9f49e8cbea5de0ebad54d7 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Mon, 17 May 2021 17:11:29 +0000 Subject: [PATCH 09/20] Set "latest" as default image tag on values --- k8s/deephealth-backend/values.yaml | 2 +- k8s/values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/k8s/deephealth-backend/values.yaml b/k8s/deephealth-backend/values.yaml index 7af3aee..42431aa 100644 --- a/k8s/deephealth-backend/values.yaml +++ b/k8s/deephealth-backend/values.yaml @@ -62,7 +62,7 @@ backend: image: &backend_image repository: dhealth/backend - tag: 2fd828d8 + tag: latest pullPolicy: *imagePullPolicy service: diff --git a/k8s/values.yaml b/k8s/values.yaml index 51ed483..5b6e617 100644 --- a/k8s/values.yaml +++ b/k8s/values.yaml @@ -46,7 +46,7 @@ backend: image: &backend_image # repository: dhealth/backend - tag: 2fd828d8 + tag: latest # service: # type: NodePort From dda1fcb8d6cdc583ba38606fc606a9c0ea96f7b1 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Mon, 17 May 2021 17:12:20 +0000 Subject: [PATCH 10/20] Use double quotes --- k8s/values.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/k8s/values.yaml b/k8s/values.yaml index 5b6e617..f5c2b0e 100644 --- a/k8s/values.yaml +++ b/k8s/values.yaml @@ -59,7 +59,7 @@ backend: persistence: data: storageClass: *globalStorageClass - path: '/data' + path: "/data" size: 10Gi # existingClaim: datasets staticFiles: @@ -157,6 +157,8 @@ nginx: # Celery settings celery: + acceptContent: "json" # list of comma separated values + taskSerializer: "json" service: # type: ClusterIP From 26d5611d74da1191055403ae98b7dc8b67d1375d Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Mon, 17 May 2021 17:14:19 +0000 Subject: [PATCH 11/20] Update init script --- docker/init.sh | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/docker/init.sh b/docker/init.sh index 76f7b7e..f9f08cd 100644 --- a/docker/init.sh +++ b/docker/init.sh @@ -3,12 +3,26 @@ ## Notice that we assume /app as current working directory # Apply all available migrations -python3 manage.py migrate +python manage.py migrate -# Create the user if doesn't exist +# Creating an admin user (non-interactive) +# (~equivalent to the interactive cmd `python manage.py createsuperuser`) cat < Date: Tue, 18 May 2021 16:38:11 +0000 Subject: [PATCH 12/20] Update README: add new configuration properties --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ad14ed8..374a439 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,11 @@ The following tables lists the main configurable parameters of the `deephealth-b | `backend.image.tag` | Back-end App Docker Image Tag | `2fd828d8` | | `backend.admin.username` | Username of the administrator of the backend app | `admin` | | `backend.admin.password` | Password of the administrator of the backend app (autogenerated if not defined) | `nil` | -| `backend.admin.email` | Email of the administrator of the backend app | `nil` | +| `backend.admin.password` | Password of the administrator of the backend app (autogenerated if not defined) | `nil` | +| `backend.social_auth.github.client_id` | ClientID of the back-end registered as OAuth2 app on Github | `nil` | +| `backend.social_auth.github.client_secret` | Client Secret of the back-end registered as OAuth2 app on Github | `nil` | +| `backend.email.user` | Username for the the back-end email service | `nil` | +| `backend.email.password` | Password for the the back-end email service | `nil` | | `backend.replicaCount` | Number of replicase of the the backend (Gunicorn) server replicas | `1` | | `backend.workers` | Number of workers of every backend (Gunicorn) server replicas | `3` | | `backend.persistence.data.storageClass` | Storage class used for backend data (requires support for `ReadWriteMany` access mode) | `*globalStorageClass` | From 5f114ee0fd7d22908e4c6b9761b65c73a8d34acc Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 18 May 2021 16:38:58 +0000 Subject: [PATCH 13/20] Fix trailing whitespaces --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 374a439..001f157 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ helm repo update curl https://raw.githubusercontent.com/deephealthproject/docker-backend/develop/k8s/deephealth-backend/values.yaml -o values.yaml ``` -3. Edit the `values.yaml` to configure your deployment ([here](#helm-chart-parameters) the available parameters) +3. Edit the `values.yaml` to configure your deployment ([here](#helm-chart-parameters) the available parameters) 4. To install the chart with the release name `deephealth-backend` : ```bash @@ -62,7 +62,7 @@ helm repo add deephealth https://deephealthproject.github.io/helm-charts/ helm dependency build k8s/deephealth-backend ``` -3. Edit the ´k8s/values.yaml´ template to configure your deployment ([here](#helm-chart-parameters) the available parameters) +3. Edit the ´k8s/values.yaml´ template to configure your deployment ([here](#helm-chart-parameters) the available parameters) 4. Deploy on Kubernetes through `helm`: @@ -94,7 +94,7 @@ ingress: ### Parameters -The following tables lists the main configurable parameters of the `deephealth-backend` chart and their default values. +The following tables lists the main configurable parameters of the `deephealth-backend` chart and their default values. | Parameter | Description | Default | |-----------------------------------------------|-----------------------------------------------------------------------------------------------|-----------------------------------------------| @@ -108,7 +108,7 @@ The following tables lists the main configurable parameters of the `deephealth-b | `ingress.annations` | Annotations for the ingress realated with the API Endpoint service | `kubernetes.io/ingress.class: nginx` | | `ingress.hosts` | Hosts paths for the ingress realated with the API Endpoint service (see example) | `nil` | | `backend.image.repository` | Back-end App Docker Image | `dhealth/backend` | -| `backend.image.tag` | Back-end App Docker Image Tag | `2fd828d8` | +| `backend.image.tag` | Back-end App Docker Image Tag | `2fd828d8` | | `backend.admin.username` | Username of the administrator of the backend app | `admin` | | `backend.admin.password` | Password of the administrator of the backend app (autogenerated if not defined) | `nil` | | `backend.admin.password` | Password of the administrator of the backend app (autogenerated if not defined) | `nil` | From e60db2b0aae076f94bff47e7b6a087fbbb2b9d95 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 18 May 2021 16:54:53 +0000 Subject: [PATCH 14/20] Ignore pyc files on Docker image --- docker/.dockerignore | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/.dockerignore b/docker/.dockerignore index 8184442..469da21 100644 --- a/docker/.dockerignore +++ b/docker/.dockerignore @@ -4,4 +4,5 @@ **/.vscode **/.DS_Store **/.env +**/*.pyc **/venv From f7a2395954197ee009d74768c6d6d9b89a89e918 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 18 May 2021 17:20:32 +0000 Subject: [PATCH 15/20] Fix missing wait-for-postgres on app container --- docker-compose.template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.template.yml b/docker-compose.template.yml index fb2a119..cd8ddd7 100644 --- a/docker-compose.template.yml +++ b/docker-compose.template.yml @@ -12,7 +12,7 @@ services: restart: "no" user: "${UID}:${GID}" runtime: ${DOCKER_RUNTIME} - command: ["/bin/bash","-c","init.sh && start.sh backend"] + command: ["/bin/bash","-c","wait-for-postgres.sh && init.sh && start.sh backend"] env_file: &envfile - settings.conf environment: From bd0f71a4e0cdb44fa1f29fa1bc221eea049881e3 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 18 May 2021 20:18:10 +0000 Subject: [PATCH 16/20] Ensure locale settings required by celery --- docker/Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index c883d71..b0e4f6b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -18,6 +18,10 @@ ENV APP_PATH "/app" ARG DATA_DIR="/data" ENV DATA_DIR=${DATA_DIR} +# Locale settings +ENV LC_ALL C.UTF-8 +ENV LANG C.UTF-8 + # Copy source path COPY src ${APP_PATH} From 43d3e54f7a0ce07ecf69dd98d540e3da988ae499 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 18 May 2021 20:19:04 +0000 Subject: [PATCH 17/20] Force proper version of click library --- docker/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index b0e4f6b..d6c10c7 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -48,6 +48,7 @@ RUN \ /usr/local/bin/wait-for-postgres.sh \ # install app requirements && pip --no-cache-dir install -r requirements.txt \ + && pip --no-cache-dir install click==7.1.2 \ # create the user && groupadd ${USERNAME} -g ${GID} || true \ && useradd -m -u ${UID} -g ${GID} ${USERNAME} \ From f8e54e77d49f2d9e03c688e11c0610556b18195e Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 18 May 2021 20:29:10 +0000 Subject: [PATCH 18/20] Set k8s default image tag to the current app version --- k8s/deephealth-backend/values.yaml | 2 +- k8s/values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/k8s/deephealth-backend/values.yaml b/k8s/deephealth-backend/values.yaml index 42431aa..5d82900 100644 --- a/k8s/deephealth-backend/values.yaml +++ b/k8s/deephealth-backend/values.yaml @@ -62,7 +62,7 @@ backend: image: &backend_image repository: dhealth/backend - tag: latest + tag: 11f2b98 pullPolicy: *imagePullPolicy service: diff --git a/k8s/values.yaml b/k8s/values.yaml index f5c2b0e..a1f8f4c 100644 --- a/k8s/values.yaml +++ b/k8s/values.yaml @@ -46,7 +46,7 @@ backend: image: &backend_image # repository: dhealth/backend - tag: latest + # tag: latest # service: # type: NodePort From f8161cfbd33c31ee64cc162be12d8b163b2e49be Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 18 May 2021 20:30:36 +0000 Subject: [PATCH 19/20] Update chart version metadata --- k8s/deephealth-backend/Chart.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/k8s/deephealth-backend/Chart.yaml b/k8s/deephealth-backend/Chart.yaml index 5e6cd75..bf9b78e 100644 --- a/k8s/deephealth-backend/Chart.yaml +++ b/k8s/deephealth-backend/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 -appVersion: "2fd828d8" +appVersion: "11f2b98" description: A Helm chart for Kubernetes name: deephealth-backend -version: 0.1.2 +version: 0.2.0 From 5c053d42c201979c7ef62032fb6c40f7cced1870 Mon Sep 17 00:00:00 2001 From: Marco Enrico Piras Date: Tue, 18 May 2021 20:55:22 +0000 Subject: [PATCH 20/20] Update sample values --- k8s/values.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/k8s/values.yaml b/k8s/values.yaml index a1f8f4c..f435714 100644 --- a/k8s/values.yaml +++ b/k8s/values.yaml @@ -45,8 +45,9 @@ backend: replicaCount: 1 image: &backend_image - # repository: dhealth/backend - # tag: latest + repository: dhealth/backend + tag: 11f2b98 + pullPolicy: *imagePullPolicy # service: # type: NodePort