From 6e0dd9e84a4238cad13389b4bf659e5b6280f8f1 Mon Sep 17 00:00:00 2001 From: cgonzalez509 Date: Wed, 3 Jul 2024 10:17:12 +0200 Subject: [PATCH] Orion.initData process for slow deployment scenarios (#275) * orion.post-hook-initdata.entrypoint.sh changed entrypoint.sh retries several times curl post of new entities before failing. In some deployments orion server needs some seconds to be ready * orion.values.initData.hookDeletePolicy variable added Addition done to allow setting custom hook delete policies. eg. hook-succeded to delete init job on success. Tested on minikube * Added initData.numberDeploymentTries (20) It customizes the number of tries before deployment of initial data fails. This solves the issue in low resourced k8s environments that some pods can take some time to be fully deployed * Added initData.numberDeploymentTries (20) It customizes the number of tries before deployment of initial data fails. This solves the issue in low resourced k8s environments that some pods can take some time to be fully deployed * Restoring tabular changes in values.yaml * Restoring not required changes in orion.Chart.yaml --------- Co-authored-by: cgonzalez --- charts/orion/Chart.yaml | 2 +- charts/orion/templates/initdata-cm.yaml | 40 +++++++++++++++++-- .../orion/templates/post-hook-initdata.yaml | 2 +- charts/orion/values.yaml | 7 ++++ 4 files changed, 45 insertions(+), 6 deletions(-) diff --git a/charts/orion/Chart.yaml b/charts/orion/Chart.yaml index 273c05ae..0b450caf 100644 --- a/charts/orion/Chart.yaml +++ b/charts/orion/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: orion -version: 1.3.0 +version: 1.3.1 appVersion: 1.0.1 kubeVersion: '>= 1.19-0' home: https://github.com/FIWARE/context.Orion-LD diff --git a/charts/orion/templates/initdata-cm.yaml b/charts/orion/templates/initdata-cm.yaml index 610279f5..7512b706 100644 --- a/charts/orion/templates/initdata-cm.yaml +++ b/charts/orion/templates/initdata-cm.yaml @@ -5,12 +5,44 @@ metadata: name: {{ include "orion.fullname" . }}-init-data-cm data: entrypoint.sh: |- - {{- range $d := .Values.initData.entities }} - curl -X POST ${ORION_HOST}:${ORION_PORT}/ngsi-ld/v1/entities --header "Content-Type: application/ld+json" -d @/data/{{ $d.name }} - {{- end }} + #!/bin/sh + RC=-1 + TRY=1 + NFILES_DEPLOYED=0 + NFILES_WITH_MSG=0 + NFILES={{ len .Values.initData.entities | default 0 }} + NTRIES={{ .Values.initData.numberDeploymentTries | default 20 }} + while [ $TRY -lt $NTRIES ]; do + IDX=0 +{{- range $d := .Values.initData.entities }} + IDX=$(( $IDX + 1)); + echo -e "---\n(Try $TRY.$IDX) Deploying FILE /data/{{ $d.name }}..." + RESPONSE=$(curl --silent -X POST ${ORION_HOST}:${ORION_PORT}/ngsi-ld/v1/entities --header "Content-Type: application/ld+json" -d @/data/{{ $d.name }} ) + RC=$?; + if [ "$RC" -eq 0 ]; then + if [ ${#RESPONSE} -ne 0 ]; then + NFILES_WITH_MSG=$(( $NFILES_WITH_MSG + 1)) + fi + NFILES_DEPLOYED=$(( $NFILES_DEPLOYED + 1)); + echo "($NFILES_DEPLOYED/$NFILES) FILE /data/{{ $d.name }} posted with response [$RESPONSE]" + if [ "$NFILES" -eq "$NFILES_DEPLOYED" ]; then + break; + fi + else + echo "Error in Curl RC=$RC" + sleep 1 + fi +{{- end }} + TRY=$(( $TRY + 1)); + done + echo -e "\n---\n$NFILES_DEPLOYED/$NFILES files posted" + if [ $NFILES_WITH_MSG -gt 0 ]; then + echo "$NFILES_WITH_MSG/$NFILES with message (review them for potential errors)" + fi + return $RC {{ range $d := .Values.initData.entities }} {{ $d.name }}: | {{- $d.data | nindent 4 }} {{- end }} -{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/orion/templates/post-hook-initdata.yaml b/charts/orion/templates/post-hook-initdata.yaml index 73ef7d78..110e870c 100644 --- a/charts/orion/templates/post-hook-initdata.yaml +++ b/charts/orion/templates/post-hook-initdata.yaml @@ -6,7 +6,7 @@ metadata: name: {{ include "orion.fullname" . }}-init-data annotations: "helm.sh/hook": {{ .Values.initData.hook }} - "helm.sh/hook-delete-policy": before-hook-creation + "helm.sh/hook-delete-policy": {{ .Values.initData.hookDeletePolicy | default "before-hook-creation" }} "helm.sh/hook-weight": "1" spec: backoffLimit: {{ .Values.initData.backoffLimit }} diff --git a/charts/orion/values.yaml b/charts/orion/values.yaml index 62b054b4..d6a99f23 100644 --- a/charts/orion/values.yaml +++ b/charts/orion/values.yaml @@ -314,8 +314,15 @@ initData: initEnabled: false # -- Annotation value for the Hook hook: post-install,post-upgrade + # -- Annotation value for the Hook hook-delete-policy. + # hook-succeeded removes the job on success, leaving it (and its pods) in case of error for perusal + # hookDeletePolicy: before-hook-creation, hook-succeeded + hookDeletePolicy: before-hook-creation # -- Number of retries before considering a Job as failed backoffLimit: 1 + # initData.numberDeploymentTries customizes the number of tries before deployment of initial data fails. + # This solves the issue in low resourced k8s environments that some pods can take some time to be fully deployed + numberDeploymentTries: 20 # -- Array of entities to be created entities: [] # - name: entity001.json