Merge pull request #13 from deephealthproject/update-deployments

Update deployments

README.md

@@ -34,7 +34,7 @@ helm repo update
 curl https://raw.githubusercontent.com/deephealthproject/docker-backend/develop/k8s/deephealth-backend/values.yaml -o values.yaml
 ```
 
-3. Edit the `values.yaml` to configure your deployment ([here](#helm-chart-parameters) the available parameters) 
+3. Edit the `values.yaml` to configure your deployment ([here](#helm-chart-parameters) the available parameters)
 4. To install the chart with the release name `deephealth-backend` :
 
 ```bash
@@ -62,7 +62,7 @@ helm repo add deephealth https://deephealthproject.github.io/helm-charts/
 helm dependency build k8s/deephealth-backend
 ```
 
-3. Edit the ´k8s/values.yaml´ template to configure your deployment ([here](#helm-chart-parameters) the available parameters) 
+3. Edit the ´k8s/values.yaml´ template to configure your deployment ([here](#helm-chart-parameters) the available parameters)
 
 4. Deploy on Kubernetes through `helm`:
 
@@ -94,7 +94,7 @@ ingress:
 
 ### Parameters
 
-The following tables lists the main configurable parameters of the `deephealth-backend` chart and their default values. 
+The following tables lists the main configurable parameters of the `deephealth-backend` chart and their default values.
 
 |           Parameter                           |                    Description                                                                |            Default                            |
 |-----------------------------------------------|-----------------------------------------------------------------------------------------------|-----------------------------------------------|
@@ -108,10 +108,14 @@ The following tables lists the main configurable parameters of the `deephealth-b
 | `ingress.annations`                           | Annotations for the ingress realated with the API Endpoint service                            |  `kubernetes.io/ingress.class: nginx`         |
 | `ingress.hosts`                               | Hosts paths for the ingress realated with the API Endpoint service (see example)              |  `nil`                                        |
 | `backend.image.repository`                    | Back-end App Docker Image                                                                     |  `dhealth/backend`                            |
-| `backend.image.tag`                           | Back-end App Docker Image Tag                                                                 |  `2fd828d8`                                    |
+| `backend.image.tag`                           | Back-end App Docker Image Tag                                                                 |  `2fd828d8`                                   |
 | `backend.admin.username`                      | Username of the administrator of the backend app                                              |  `admin`                                      |
 | `backend.admin.password`                      | Password of the administrator of the backend app (autogenerated if not defined)               |  `nil`                                        |
-| `backend.admin.email`                         | Email of the administrator of the backend app                                                 |  `nil`                                        |
+| `backend.admin.password`                      | Password of the administrator of the backend app (autogenerated if not defined)               |  `nil`                                        |
+| `backend.social_auth.github.client_id`        | ClientID of the back-end registered as OAuth2 app on Github                                   |  `nil`                                        |
+| `backend.social_auth.github.client_secret`    | Client Secret of the back-end registered as OAuth2 app on Github                              |  `nil`                                        |
+| `backend.email.user`                          | Username for the the back-end email service                                                   |  `nil`                                        |
+| `backend.email.password`                      | Password for the the back-end email service                                                   |  `nil`                                        |
 | `backend.replicaCount`                        | Number of replicase of the the backend (Gunicorn) server replicas                             |  `1`                                          |
 | `backend.workers`                             | Number of workers of every backend (Gunicorn) server replicas                                 |  `3`                                          |
 | `backend.persistence.data.storageClass`       | Storage class used for backend data (requires support for `ReadWriteMany` access mode)        |  `*globalStorageClass`                        |

config.template

@@ -1,6 +1,9 @@
 # Allow DEBUG mode. Don't run with debug turned on in production!
 DEBUG=${DEBUG}
 
+# Enable eddl CS_GPU computing service
+EDDL_WITH_CUDA=${EDDL_WITH_CUDA}
+
 # Set SECRET_KEY. Keep the secret key used in production secret!
 SECRET_KEY="${SECRET_KEY}"
 
@@ -15,12 +18,13 @@ CORS_ORIGIN_WHITELIST="${CORS_ORIGIN_WHITELIST}"
 # DATABASE_URL=psql://urser:[email protected]:8458/database # PostgreSQL DB
 
 # Relative path for static resources
-STATIC_URL="${STATIC_URL}"
+STATIC_URL='/backend/static/'
+
+# Base url to serve media files
+MEDIA_URL='/backend/media/'
 
-# Data directories
-TRAINING_DIR="${TRAINING_DIR}"
-INFERENCE_DIR="${INFERENCE_DIR}"
-DATASETS_DIR="${DATASETS_DIR}"
+# Data directory
+DATA_DIR="${DATA_DIR}"
 
 # RabbitMQ Credentials
 # see https://hub.docker.com/_/rabbitmq
@@ -31,3 +35,10 @@ CELERY_ACCEPT_CONTENT="${CELERY_ACCEPT_CONTENT}" # list of comma separated value
 CELERY_RESULT_BACKEND="${CELERY_RESULT_BACKEND}"
 CELERY_TASK_SERIALIZER="${CELERY_TASK_SERIALIZER}"
 
+# OAuth settings
+DRFSO2_PROPRIETARY_BACKEND_NAME='GithubOAuth2'
+DRFSO2_URL_NAMESPACE='social'
+
+# GitHub configuration
+SOCIAL_AUTH_GITHUB_KEY=''
+SOCIAL_AUTH_GITHUB_SECRET=''

docker-compose.template.yml

@@ -12,7 +12,7 @@ services:
     restart: "no"
     user: "${UID}:${GID}"
     runtime: ${DOCKER_RUNTIME}
-    command: ["/bin/bash","-c","init.sh && start.sh backend"]
+    command: ["/bin/bash","-c","wait-for-postgres.sh && init.sh && start.sh backend"]
     env_file: &envfile
       - settings.conf
     environment:
@@ -21,14 +21,11 @@ services:
       - DATABASE_URL=psql://${POSTGRESQL_USERNAME}:${POSTGRESQL_PASSWORD}@db:5432/${POSTGRESQL_DATABASE} # PostgreSQL DB
       - POSTGRESQL_HOST=db
     ports:
-      - "${BACKEND_PORT}:8000"
+      - "${BACKEND_PORT}:${BACKEND_PORT}"
     volumes:
-      ${BACKEND_VOLUME}
       - ./.config:/app/config
       - ./settings.conf:/app/settings.conf
-      - datasets:${DATASETS_DIR}
-      - training:${TRAINING_DIR}
-      - inference:${INFERENCE_DIR}
+      - data:${DATA_DIR}
     depends_on:
       - broker
       - db
@@ -48,12 +45,9 @@ services:
       - DATABASE_URL=psql://${POSTGRESQL_USERNAME}:${POSTGRESQL_PASSWORD}@db:5432/${POSTGRESQL_DATABASE} # PostgreSQL DB
     command: ["start.sh","celery"]
     volumes:
-      ${BACKEND_VOLUME}
       - ./.config:/app/config
       - ./settings.conf:/app/settings.conf
-      - datasets:${DATASETS_DIR}
-      - training:${TRAINING_DIR}
-      - inference:${INFERENCE_DIR}
+      - data:${DATA_DIR}
     depends_on:
       - broker
       - db
@@ -65,7 +59,6 @@ services:
     environment:
       - PYTHONPATH="$PYTHONPATH:.%"
     volumes:
-      ${BACKEND_VOLUME}
       - ./.config:/app/config
       - ./settings.conf:/app/settings.conf
       - postgresql:/bitnami/postgresql
@@ -89,6 +82,4 @@ services:
 
 volumes:
   postgresql:
-  datasets:
-  inference:
-  training:
+  data:

docker/.dockerignore

@@ -4,4 +4,5 @@
 **/.vscode
 **/.DS_Store
 **/.env
+**/*.pyc
 **/venv

docker/Dockerfile

@@ -9,18 +9,18 @@ ARG USERNAME=backend
 
 # Set defaults
 ARG BACKEND_PORT=8000
-ENV BACKEND_PORT ${BACKEND_PORT} 
+ENV BACKEND_PORT ${BACKEND_PORT}
 
 # Set the path where the app will be installed
 ENV APP_PATH "/app"
 
 # Set data paths
-ARG DATASETS_DIR="/data/datasets"
-ENV DATASETS_DIR=${DATASETS_DIR}
-ARG TRAINING_DIR="/data/training"
-ARG TRAINING_DIR=${TRAINING_DIR}
-ARG INFERENCE_DIR="/data/inference"
-ARG INFERENCE_DIR=${INFERENCE_DIR}
+ARG DATA_DIR="/data"
+ENV DATA_DIR=${DATA_DIR}
+
+# Locale settings
+ENV LC_ALL C.UTF-8
+ENV LANG C.UTF-8
 
 # Copy source path
 COPY src ${APP_PATH}
@@ -48,20 +48,16 @@ RUN \
                 /usr/local/bin/wait-for-postgres.sh \
     # install app requirements
     && pip --no-cache-dir install -r requirements.txt \
+    && pip --no-cache-dir install click==7.1.2 \
     # create the user
     && groupadd ${USERNAME} -g ${GID} || true \
     && useradd -m -u ${UID} -g ${GID} ${USERNAME} \
     && chown -R ${USERNAME} ${APP_PATH} \
-    && mkdir -p ${DATASETS_DIR} ${INFERENCE_DIR} ${TRAINING_DIR} \
-    && chown -R ${USERNAME} ${DATASETS_DIR} ${TRAINING_DIR} ${INFERENCE_DIR} 
-
-# Declare data volumes
-#VOLUME ["${DATASETS_DIR}"]
-#VOLUME ["${INFERENCE_DIR}"]
-#VOLUME ["${TRAINING_DIR}"]
+    && mkdir -p ${DATA_DIR} \
+    && chown -R ${USERNAME} ${DATA_DIR}
 
 # Set the default user
 USER ${USERNAME}
 
 # Default command
-CMD start.sh backend
+CMD start.sh backend

docker/init.sh

@@ -3,12 +3,26 @@
 ## Notice that we assume /app as current working directory
 
 # Apply all available migrations
-python3 manage.py migrate
+python manage.py migrate
 
-# Create the user if doesn't exist
+# Creating an admin user (non-interactive)
+# (~equivalent to the interactive cmd `python manage.py createsuperuser`)
 cat <<EOF | python manage.py shell
 from django.contrib.auth import get_user_model
 User = get_user_model()  # get the currently active user model,
 User.objects.filter(username='${ADMIN_USER}').exists() or \
     User.objects.create_superuser('${ADMIN_USER}', '${ADMIN_EMAIL}', '${ADMIN_PASSWORD}')
 EOF
+
+# Initialize Django fixtures for the DB init
+# This will also download ONNX weights of the models
+python scripts/init_fixtures.py
+
+# Load fixtures into the DB (default entries)
+python manage.py loaddata \
+                    tasks.json \
+                    property.json \
+                    allowedproperty.json \
+                    dataset.json \
+                    model.json \
+                    modelweights.json # auth.json

k8s/deephealth-backend/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v1
-appVersion: "2fd828d8"
+appVersion: "11f2b98"
 description: A Helm chart for Kubernetes
 name: deephealth-backend
-version: 0.1.2
+version: 0.2.0

k8s/deephealth-backend/templates/init-job.yaml

@@ -17,10 +17,10 @@ spec:
         - name: wait-for-postgresql
           image: "{{ .Values.postgresql.image.repository }}:{{ .Values.postgresql.image.tag }}"
           imagePullPolicy: {{ .Values.backend.image.pullPolicy }}
-          command: 
+          command:
             - /bin/sh
             - -c
-            - until PGPASSWORD=${POSTGRES_PASSWORD} psql -h "{{ include "deephealth-backend.fullname" . }}-postgresql" -U "{{ .Values.postgresql.postgresqlUsername }}" {{ .Values.postgresql.postgresqlDatabase }} -c '\q'; do 
+            - until PGPASSWORD=${POSTGRES_PASSWORD} psql -h "{{ include "deephealth-backend.fullname" . }}-postgresql" -U "{{ .Values.postgresql.postgresqlUsername }}" {{ .Values.postgresql.postgresqlDatabase }} -c '\q'; do
                 >&2 echo "PostgreSQL is unavailable -- sleep 2 seconds and retry" ;
                 sleep 2 ;
               done ;
@@ -83,4 +83,4 @@ spec:
       tolerations:
         {{- toYaml . | nindent 8 }}
     {{- end }}
-  backoffLimit: 4
+  backoffLimit: 4

k8s/deephealth-backend/templates/secrets.yaml

@@ -36,7 +36,7 @@ stringData:
 
     # Cross-Origin Resource Sharing (CORS) whitelist
     CORS_ORIGIN_WHITELIST="{{- range .Values.ingress.hosts }},http://{{ .host }},https://{{ .host }}{{- end }}{{- if .Values.backend.corsOriginWhiteList }},{{ .Values.backend.corsOriginWhiteList }}{{- end }}"
-    
+
     # Database connection settings
     # DATABASE_URL=sqlite:///my-local-sqlite.db # SQLite DB
     # DATABASE_URL=psql://{{ .Values.postgresql.postgresqlUsername }}:{{ .Values.postgresql.postgresqlPassword }}@{{ include "deephealth-backend.fullname" . }}-postgresql:{{ .Values.postgresql.service.port }}/{{ .Values.postgresql.postgresqlDatabase }}
@@ -49,7 +49,7 @@ stringData:
 
     # Data directories
     DATA_DIR={{ .Values.backend.persistence.data.path }}
-    
+
     # RabbitMQ Credentials
     # see https://hub.docker.com/_/rabbitmq
     # RABBITMQ_BROKER_URL=amqp://{{ .Values.broker.rabbitmq.username }}:{{ .Values.broker.rabbitmq.password }}@{{ include "deephealth-backend.fullname" . }}-rabbitmq:{{ .Values.broker.service.port }}
@@ -58,6 +58,18 @@ stringData:
     CELERY_ACCEPT_CONTENT={{ .Values.celery.acceptContent | quote }}
     # CELERY_RESULT_BACKEND=db+sqlite:///results.sqlite
     CELERY_TASK_SERIALIZER={{ .Values.celery.taskSerializer | quote }}
+
+    # OAuth settings
+    DRFSO2_PROPRIETARY_BACKEND_NAME='GithubOAuth2'
+    DRFSO2_URL_NAMESPACE='social'
+
+    # GitHub configuration
+    SOCIAL_AUTH_GITHUB_KEY={{ .Values.backend.social_auth.github.client_id | quote }}
+    SOCIAL_AUTH_GITHUB_SECRET={{ .Values.backend.social_auth.github.client_secret | quote }}
+
+    # Email backend settings
+    EMAIL_HOST_USER={{ .Values.backend.email.user | quote }}
+    EMAIL_HOST_PASSWORD={{ .Values.backend.email.password | quote }}
 ---
 {{- end }}
 {{- if or .Release.IsInstall (and .Release.IsUpgrade (or .Values.postgresql.postgresqlPassword .Values.postgresql.postgresqlPostgresPassword)) }}
@@ -75,7 +87,7 @@ metadata:
     helm.sh/resource-policy: keep
 type: Opaque
 data:
-  postgresql-postgres-password: {{ include "deephealth-backend.postgresql.postgres-password" . | b64enc | quote }}  
+  postgresql-postgres-password: {{ include "deephealth-backend.postgresql.postgres-password" . | b64enc | quote }}
   postgresql-password: {{ include "deephealth-backend.postgresql.password" . | b64enc | quote }}
   {{- if .Values.postgresql.replication.enabled }}
   postgresql-replication-password: {{ include "deephealth-backend.postgresql.replication.password" . | b64enc | quote }}
@@ -110,4 +122,4 @@ data:
   {{ else }}
   rabbitmq-erlang-cookie: {{ randAlphaNum 32 | b64enc | quote }}
   {{ end }}
-{{- end }}
+{{- end }}