diff --git a/docker/worker_entrypoint.sh b/docker/worker_entrypoint.sh index 056a20b17..9e78003c1 100755 --- a/docker/worker_entrypoint.sh +++ b/docker/worker_entrypoint.sh @@ -41,11 +41,17 @@ if [[ -n "${WORKER_PROCESSES:-}" ]]; then log "Worker starting ${WORKER_PROCESSES} processes" fi +if [[ -n "${WORKER_THREADS:-}" ]]; then + processes="--threads ${WORKER_THREADS}" + log "Worker starting with ${WORKER_THREADS} threads per process" +fi + while : ; do /usr/local/bin/dramatiq \ ${verbose:-} \ ${watch:-} \ ${processes:-} \ + ${threads:-} \ lifemonitor.tasks.worker:broker lifemonitor.tasks.tasks exit_code=$? if [[ $exit_code == 3 ]]; then diff --git a/k8s/Chart.yaml b/k8s/Chart.yaml index 208c09375..1d3baf92c 100644 --- a/k8s/Chart.yaml +++ b/k8s/Chart.yaml @@ -7,12 +7,12 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.3.0 +version: 0.4.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. -appVersion: 0.1.0 +appVersion: 0.4.0 # Chart dependencies dependencies: diff --git a/k8s/templates/_helpers.tpl b/k8s/templates/_helpers.tpl index ebeb4ae3e..40f949c38 100644 --- a/k8s/templates/_helpers.tpl +++ b/k8s/templates/_helpers.tpl @@ -35,9 +35,9 @@ Common labels */}} {{- define "chart.labels" -}} app.kubernetes.io/name: {{ include "chart.name" . }} -helm.sh/chart: {{ include "chart.chart" . }} app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/managed-by: {{ .Release.Service }} +helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" {{- end }} {{/* @@ -48,6 +48,19 @@ app.kubernetes.io/name: {{ include "chart.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} +{{/* + +Define lifemonitor image +*/}} +{{- define "chart.lifemonitor.image" -}} +{{- if .Values.lifemonitor.image }} +{{- printf "%s" .Values.lifemonitor.image }} +{{- else }} +{{- printf "crs4/lifemonitor:%s" .Chart.AppVersion }} +{{- end }} +{{- end }} + + {{/* Create the name of the service account to use */}} @@ -77,6 +90,10 @@ Define environment variables shared by some pods. value: "{{ .Values.postgresql.postgresqlPassword }}" - name: POSTGRESQL_DATABASE value: "{{ .Values.postgresql.postgresqlDatabase }}" +- name: REDIS_HOST + value: "{{ .Release.Name }}-redis-master" +- name: WORKER_PROCESSES + value: "{{ .Values.worker.processes }}" - name: LIFEMONITOR_TLS_KEY value: "/lm/certs/tls.key" - name: LIFEMONITOR_TLS_CERT diff --git a/k8s/templates/backend-deployment.yaml b/k8s/templates/backend-deployment.yaml index 7ee59cd8f..3c9ec93df 100644 --- a/k8s/templates/backend-deployment.yaml +++ b/k8s/templates/backend-deployment.yaml @@ -4,6 +4,7 @@ metadata: name: {{ include "chart.fullname" . }}-backend labels: {{- include "chart.labels" . | nindent 4 }} + app.kubernetes.io/component: backend spec: {{- if not .Values.lifemonitor.autoscaling.enabled }} replicas: {{ .Values.lifemonitor.replicaCount }} @@ -11,14 +12,17 @@ spec: selector: matchLabels: {{- include "chart.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: backend template: metadata: - {{- with .Values.lifemonitor.podAnnotations }} annotations: + checksum/settings: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + {{- with .Values.lifemonitor.podAnnotations }} {{- toYaml . | nindent 8 }} - {{- end }} + {{- end }} labels: {{- include "chart.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: backend spec: {{- with .Values.lifemonitor.imagePullSecrets }} imagePullSecrets: @@ -36,12 +40,12 @@ spec: - name: app securityContext: {{- toYaml .Values.lifemonitor.securityContext | nindent 12 }} - image: {{ .Values.lifemonitor.image }} + image: {{ include "chart.lifemonitor.image" . }} imagePullPolicy: {{ .Values.lifemonitor.imagePullPolicy }} env: -{{ include "lifemonitor.common-env" . | indent 12 }} + {{- include "lifemonitor.common-env" . | nindent 12 }} volumeMounts: -{{ include "lifemonitor.common-volume-mounts" . | indent 12 }} + {{- include "lifemonitor.common-volume-mounts" . | nindent 12 }} ports: - name: http containerPort: 8000 @@ -61,7 +65,7 @@ spec: resources: {{- toYaml .Values.lifemonitor.resources | nindent 12 }} volumes: -{{ include "lifemonitor.common-volume" . | indent 12 }} + {{- include "lifemonitor.common-volume" . | nindent 8 }} {{- with .Values.lifemonitor.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/k8s/templates/job-init.yaml b/k8s/templates/job-init.yaml new file mode 100644 index 000000000..a20595777 --- /dev/null +++ b/k8s/templates/job-init.yaml @@ -0,0 +1,35 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "chart.fullname" . }}-init + labels: + {{- include "chart.labels" . | nindent 4 }} +spec: + template: + spec: + containers: + - name: lifemonitor-init + image: {{ include "chart.lifemonitor.image" . }} + imagePullPolicy: {{ .Values.lifemonitor.imagePullPolicy }} + command: ["/bin/sh","-c"] + args: ["wait-for-postgres.sh && flask init db && flask task-queue reset"] + env: + {{- include "lifemonitor.common-env" . | nindent 10 }} + volumeMounts: + {{- include "lifemonitor.common-volume-mounts" . | nindent 10 }} + restartPolicy: OnFailure + volumes: + {{- include "lifemonitor.common-volume" . | nindent 8 }} + {{- with .Values.lifemonitor.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.lifemonitor.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.lifemonitor.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + backoffLimit: 4 diff --git a/k8s/templates/init-job.yaml b/k8s/templates/job-upgrade.yaml similarity index 62% rename from k8s/templates/init-job.yaml rename to k8s/templates/job-upgrade.yaml index 243a1c8a6..34008f66f 100644 --- a/k8s/templates/init-job.yaml +++ b/k8s/templates/job-upgrade.yaml @@ -1,18 +1,24 @@ apiVersion: batch/v1 kind: Job metadata: - name: {{ include "chart.fullname" . }}-init + name: {{ include "chart.fullname" . }}-upgrade labels: {{- include "chart.labels" . | nindent 4 }} + annotations: + # This is what defines this resource as a hook. Without this line, the + # job is considered part of the release. + "helm.sh/hook": post-upgrade + "helm.sh/hook-weight": "-5" + "helm.sh/hook-delete-policy": hook-succeeded spec: - template: + template: spec: containers: - - name: lifemonitor-init - image: "{{ .Values.lifemonitor.image }}" + - name: lifemonitor-upgrade + image: {{ include "chart.lifemonitor.image" . }} imagePullPolicy: {{ .Values.lifemonitor.imagePullPolicy }} command: ["/bin/sh","-c"] - args: ["wait-for-postgres.sh && flask init db"] + args: ["wait-for-postgres.sh && flask init db && flask task-queue reset"] env: {{ include "lifemonitor.common-env" . | indent 10 }} volumeMounts: @@ -32,4 +38,4 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} - backoffLimit: 4 \ No newline at end of file + backoffLimit: 4 diff --git a/k8s/templates/secret.yaml b/k8s/templates/secret.yaml index 936cfc7e0..73af94f14 100644 --- a/k8s/templates/secret.yaml +++ b/k8s/templates/secret.yaml @@ -37,7 +37,10 @@ stringData: POSTGRESQL_PASSWORD={{ .Values.postgresql.postgresqlPassword }} # Dramatiq worker settings - WORKER_PROCESSES={{ .Values.lifemonitor.worker_processes }} + WORKER_PROCESSES={{ .Values.worker.processes }} + {{- if .Values.worker.threads }} + WORKER_THREADS={{ .Values.worker.threads }} + {{- end }} # Redis settings REDIS_HOST={{ .Release.Name }}-redis-master @@ -45,11 +48,14 @@ stringData: REDIS_PASSWORD={{ .Values.redis.auth.password }} # Redis Cache - CACHE_REDIS_DB=0 - CACHE_REDIS_URL=redis://:{{ .Values.redis.auth.password }}@{{ .Release.Name }}-redis-master:{{ .Values.redis.master.service.port }}/0 - CACHE_DEFAULT_TIMEOUT=300 - CACHE_SESSION_TIMEOUT=3600 - CACHE_BUILDS_TIMEOUT=84600 + CACHE_REDIS_HOST={{ .Release.Name }}-redis-master + CACHE_REDIS_DB={{ .Values.cache.db }} + CACHE_REDIS_URL=redis://:{{ .Values.redis.auth.password }}@{{ .Release.Name }}-redis-master:{{ .Values.redis.master.service.port }}/{{ .Values.cache.db }} + CACHE_DEFAULT_TIMEOUT={{ .Values.cache.timeout.default }} + CACHE_REQUEST_TIMEOUT={{ .Values.cache.timeout.request }} + CACHE_SESSION_TIMEOUT={{ .Values.cache.timeout.session }} + CACHE_WORKFLOW_TIMEOUT={{ .Values.cache.timeout.workflow }} + CACHE_BUILD_TIMEOUT={{ .Values.cache.timeout.build }} # Set admin credentials LIFEMONITOR_ADMIN_PASSWORD={{ .Values.lifemonitor.administrator.password }} diff --git a/k8s/templates/service.yaml b/k8s/templates/service.yaml index 90cb29f12..ac8c8ccf6 100644 --- a/k8s/templates/service.yaml +++ b/k8s/templates/service.yaml @@ -13,3 +13,4 @@ spec: name: http selector: {{- include "chart.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: backend diff --git a/k8s/templates/worker-deployment.yaml b/k8s/templates/worker-deployment.yaml index cede78b95..ce99c4c61 100644 --- a/k8s/templates/worker-deployment.yaml +++ b/k8s/templates/worker-deployment.yaml @@ -5,70 +5,71 @@ metadata: labels: {{- include "chart.labels" . | nindent 4 }} spec: - {{- if not .Values.lifemonitor.autoscaling.enabled }} - replicas: {{ .Values.lifemonitor.replicaCount }} + {{- if not .Values.worker.autoscaling.enabled }} + replicas: {{ .Values.worker.replicaCount }} {{- end }} selector: matchLabels: {{- include "chart.selectorLabels" . | nindent 6 }} template: metadata: - {{- with .Values.lifemonitor.podAnnotations }} annotations: + checksum/settings: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + {{- with .Values.worker.podAnnotations }} {{- toYaml . | nindent 8 }} {{- end }} labels: {{- include "chart.selectorLabels" . | nindent 8 }} spec: - {{- with .Values.lifemonitor.imagePullSecrets }} + {{- with .Values.worker.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} serviceAccountName: {{ include "chart.serviceAccountName" . }} securityContext: - {{- toYaml .Values.lifemonitor.podSecurityContext | nindent 8 }} + {{- toYaml .Values.worker.podSecurityContext | nindent 8 }} initContainers: - name: init image: "crs4/k8s-wait-for:latest" imagePullPolicy: IfNotPresent args: ["job", "{{ include "chart.fullname" . }}-init"] containers: - - name: app + - name: worker securityContext: - {{- toYaml .Values.lifemonitor.securityContext | nindent 12 }} - image: {{ .Values.lifemonitor.image }} - imagePullPolicy: {{ .Values.lifemonitor.imagePullPolicy }} + {{- toYaml .Values.worker.securityContext | nindent 12 }} + image: {{ include "chart.lifemonitor.image" . }} + imagePullPolicy: {{ .Values.worker.imagePullPolicy }} command: ["/bin/sh","-c"] args: ["/usr/local/bin/worker_entrypoint.sh"] env: -{{ include "lifemonitor.common-env" . | indent 12 }} + {{- include "lifemonitor.common-env" . | nindent 12 }} volumeMounts: -{{ include "lifemonitor.common-volume-mounts" . | indent 12 }} - livenessProbe: - httpGet: - scheme: HTTPS - path: /health - port: 8000 - readinessProbe: - httpGet: - scheme: HTTPS - path: /health - port: 8000 - initialDelaySeconds: 5 - periodSeconds: 3 + {{- include "lifemonitor.common-volume-mounts" . | nindent 12 }} + # livenessProbe: + # httpGet: + # scheme: HTTPS + # path: /health + # port: 8000 + # readinessProbe: + # httpGet: + # scheme: HTTPS + # path: /health + # port: 8000 + # initialDelaySeconds: 5 + # periodSeconds: 3 resources: - {{- toYaml .Values.lifemonitor.resources | nindent 12 }} + {{- toYaml .Values.worker.resources | nindent 12 }} volumes: -{{ include "lifemonitor.common-volume" . | indent 12 }} - {{- with .Values.lifemonitor.nodeSelector }} + {{- include "lifemonitor.common-volume" . | nindent 8 }} + {{- with .Values.worker.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.lifemonitor.affinity }} + {{- with .Values.worker.affinity }} affinity: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.lifemonitor.tolerations }} + {{- with .Values.worker.tolerations }} tolerations: {{- toYaml . | nindent 8 }} {{- end }} diff --git a/k8s/values.yaml b/k8s/values.yaml index e0c9fef4c..07683f742 100644 --- a/k8s/values.yaml +++ b/k8s/values.yaml @@ -59,10 +59,19 @@ testing_services: # token: # type: travis +cache: + db: 0 + timeout: + default: 30 + request: 15 + session: 3600 + workflow: 1800 + build: 84600 + lifemonitor: replicaCount: 1 - image: &lifemonitorImage crs4/lifemonitor:master + image: &lifemonitorImage crs4/lifemonitor:0.4.0 imagePullPolicy: &lifemonitorImagePullPolicy Always imagePullSecrets: [] @@ -96,16 +105,65 @@ lifemonitor: type: ClusterIP port: 8000 - # Dramatiq worker settings - worker_processes: 1 - persistence: storageClass: *storageClass # Enable/Disable the pod to test connection to the LifeMonitor back-end enableTestConnection: false - resources: {} + resources: + {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + + nodeSelector: {} + + tolerations: [] + + affinity: {} + +worker: + image: *lifemonitorImage + imagePullPolicy: *lifemonitorImagePullPolicy + imagePullSecrets: [] + + processes: 1 + #threads: 1 + + podAnnotations: {} + + podSecurityContext: + {} + # fsGroup: 2000 + + securityContext: + {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + + replicaCount: 1 + + resources: + {} # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little # resources, such as Minikube. If you do want to specify resources, uncomment the following diff --git a/settings.conf b/settings.conf index 78c557d28..33119552e 100644 --- a/settings.conf +++ b/settings.conf @@ -35,7 +35,7 @@ FLASK_ENV=development LIFEMONITOR_ADMIN_PASSWORD=admin # PostgreSQL DBMS settings -#POSTGRESQL_HOST=0.0.0.0 +POSTGRESQL_HOST=db POSTGRESQL_PORT=5432 POSTGRESQL_DATABASE=lm POSTGRESQL_USERNAME=lm @@ -47,6 +47,7 @@ GUNICORN_THREADS=2 # Dramatiq worker settings WORKER_PROCESSES=1 +WORKER_THREADS=3 # Redis settings REDIS_HOST=redis