Orion.initData process for slow deployment scenarios (#275)

* orion.post-hook-initdata.entrypoint.sh changed

entrypoint.sh retries several times curl post of new entities before failing. In some deployments orion server needs some seconds to be ready

* orion.values.initData.hookDeletePolicy variable added

Addition done  to allow setting custom hook delete policies. eg.  hook-succeded to delete init job on success. Tested on minikube

* Added initData.numberDeploymentTries (20)

It customizes the number of tries before deployment of initial data fails. This solves the issue in low resourced k8s environments that some pods can take some time to be fully deployed

* Added initData.numberDeploymentTries (20)

It customizes the number of tries before deployment of initial data fails. This solves the issue in low resourced k8s environments that some pods can take some time to be fully deployed

* Restoring tabular changes in values.yaml

* Restoring not required changes in orion.Chart.yaml

---------

Co-authored-by: cgonzalez <cgonzalez@V22088>

charts/orion/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v2
 name: orion
-version: 1.3.0
+version: 1.3.1
 appVersion: 1.0.1
 kubeVersion: '>= 1.19-0'
 home: https://github.com/FIWARE/context.Orion-LD

charts/orion/templates/initdata-cm.yaml

@@ -5,12 +5,44 @@ metadata:
   name: {{ include "orion.fullname" . }}-init-data-cm
 data:
   entrypoint.sh: |-
-    {{- range $d := .Values.initData.entities }}
-    curl -X POST ${ORION_HOST}:${ORION_PORT}/ngsi-ld/v1/entities --header "Content-Type: application/ld+json" -d @/data/{{ $d.name }}
-    {{- end }}
+    #!/bin/sh    
+    RC=-1
+    TRY=1
+    NFILES_DEPLOYED=0
+    NFILES_WITH_MSG=0
+    NFILES={{ len .Values.initData.entities | default 0 }}
+    NTRIES={{ .Values.initData.numberDeploymentTries | default 20 }}
+    while [ $TRY -lt $NTRIES ]; do
+      IDX=0
+{{- range $d := .Values.initData.entities }}
+      IDX=$(( $IDX + 1));
+      echo -e "---\n(Try $TRY.$IDX) Deploying FILE /data/{{ $d.name }}..."
+      RESPONSE=$(curl --silent -X POST ${ORION_HOST}:${ORION_PORT}/ngsi-ld/v1/entities --header "Content-Type: application/ld+json" -d @/data/{{ $d.name }} )
+      RC=$?;
+      if [ "$RC" -eq 0 ]; then 
+        if [ ${#RESPONSE} -ne 0 ]; then
+          NFILES_WITH_MSG=$(( $NFILES_WITH_MSG + 1))
+        fi
+        NFILES_DEPLOYED=$(( $NFILES_DEPLOYED + 1));
+        echo "($NFILES_DEPLOYED/$NFILES) FILE /data/{{ $d.name }} posted with response [$RESPONSE]"
+        if [ "$NFILES" -eq "$NFILES_DEPLOYED" ]; then
+          break;
+        fi
+      else
+        echo "Error in Curl RC=$RC"
+        sleep 1
+      fi
+{{- end }}
+      TRY=$(( $TRY + 1));
+    done
+    echo -e "\n---\n$NFILES_DEPLOYED/$NFILES files posted"
+    if [ $NFILES_WITH_MSG -gt 0 ]; then
+      echo "$NFILES_WITH_MSG/$NFILES with message (review them for potential errors)"
+    fi
+    return $RC
   {{ range $d := .Values.initData.entities }}
   {{ $d.name }}: |
     {{- $d.data | nindent 4 }}
   {{- end }}
 
-{{- end }}
+{{- end }}

charts/orion/templates/post-hook-initdata.yaml

@@ -6,7 +6,7 @@ metadata:
   name: {{ include "orion.fullname" . }}-init-data
   annotations:
     "helm.sh/hook": {{ .Values.initData.hook }}
-    "helm.sh/hook-delete-policy": before-hook-creation
+    "helm.sh/hook-delete-policy": {{ .Values.initData.hookDeletePolicy | default "before-hook-creation" }}
     "helm.sh/hook-weight": "1"
 spec:
   backoffLimit: {{ .Values.initData.backoffLimit }}

charts/orion/values.yaml

@@ -314,8 +314,15 @@ initData:
   initEnabled: false
   # -- Annotation value for the Hook
   hook: post-install,post-upgrade
+  # -- Annotation value for the Hook hook-delete-policy.
+  # hook-succeeded removes the job on success, leaving it (and its pods) in case of error for perusal
+  # hookDeletePolicy: before-hook-creation, hook-succeeded
+  hookDeletePolicy: before-hook-creation
   # -- Number of retries before considering a Job as failed
   backoffLimit: 1
+  # initData.numberDeploymentTries customizes the number of tries before deployment of initial data fails.
+  # This solves the issue in low resourced k8s environments that some pods can take some time to be fully deployed
+  numberDeploymentTries: 20
   # -- Array of entities to be created
   entities: []
   #  - name: entity001.json