add distributed option to helm-chart

parseablehq · Apr 8, 2024 · 9cfd164 · 9cfd164
1 parent be469a3
commit 9cfd164
Show file tree

Hide file tree

Showing 9 changed files with 805 additions and 10 deletions.
diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl
@@ -35,7 +35,19 @@ Common labels
 */}}
 {{- define "parseable.labels" -}}
 helm.sh/chart: {{ include "parseable.chart" . }}
-{{ include "parseable.selectorLabels" . }}
+{{ include "parseable.labelsSelector" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Ingestor Labels
+*/}}
+{{- define "parseable.ingestorLabels" -}}
+helm.sh/chart: {{ include "parseable.chart" . }}
+{{ include "parseable.ingestorLabelsSelector" . }}
 {{- if .Chart.AppVersion }}
 app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
 {{- end }}
@@ -45,9 +57,18 @@ app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{/*
 Selector labels
 */}}
-{{- define "parseable.selectorLabels" -}}
+{{- define "parseable.labelsSelector" -}}
+app.kubernetes.io/name: {{ include "parseable.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Ingestor Labels Selector
+*/}}
+{{- define "parseable.ingestorLabelsSelector" -}}
 app.kubernetes.io/name: {{ include "parseable.name" . }}
 app.kubernetes.io/instance: {{ .Release.Name }}
+app.parseable.com/type: ingestor
 {{- end }}
 
 {{/*

diff --git a/helm/templates/ingestor-service.yaml b/helm/templates/ingestor-service.yaml
@@ -0,0 +1,15 @@
+{{- if eq .Values.parseable.highAvailability.enabled true }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "parseable.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  type: {{ $.Values.parseable.service.type }}
+  ports:
+    - port: {{ $.Values.parseable.service.port }}
+      targetPort: 8000
+      protocol: TCP
+  selector:
+    {{- include "parseable.ingestorLabelsSelector" . | nindent 4 }}
+{{- end }}
diff --git a/helm/templates/ingestor-statefulset.yaml b/helm/templates/ingestor-statefulset.yaml
@@ -0,0 +1,75 @@
+{{- if eq .Values.parseable.highAvailability.enabled true }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "parseable.fullname" . }}-headless
+  namespace: {{ .Release.Namespace }}
+spec:
+  ports:
+    - port: 8000
+      name: "parseable-port"
+  clusterIP: None
+  selector:
+    {{- include "parseable.ingestorLabelsSelector" . | nindent 4 }}
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ include "parseable.fullname" . }}-ingestor
+  namespace: {{ .Release.Namespace }}
+spec:
+  selector:
+    matchLabels:
+      {{- include "parseable.ingestorLabelsSelector" . | nindent 6 }}
+  serviceName: {{ include "parseable.fullname" . }}-headless
+  replicas: {{ .Values.parseable.highAvailability.ingestor.count }}
+  minReadySeconds: 10
+  template:
+    metadata:
+      labels:
+        {{- include "parseable.ingestorLabelsSelector" . | nindent 6 }}
+    spec:
+      terminationGracePeriodSeconds: 10 
+      containers:
+      - name: {{ .Chart.Name }}
+        securityContext:
+          {{- toYaml .Values.parseable.securityContext | nindent 8 }}
+        image: {{ .Values.parseable.image.repository }}:{{ .Values.parseable.image.tag | default .Chart.AppVersion }}
+        imagePullPolicy: {{ .Values.parseable.image.pullPolicy }}
+        # Uncomment to debug
+        # command: [ "/bin/sh", "-c", "sleep 1000000" ]
+        args: ["parseable", "s3-store"]
+        env:
+          {{- range $key, $value :=  .Values.parseable.highAvailability.ingestor.env }}
+          - name: {{ $key }}
+            value: {{ tpl $value $ | quote }}
+          {{- end }}
+          {{- range $secret := .Values.parseable.s3ModeSecret }}
+          {{- range $key := $secret.keys }}
+          {{- $envPrefix := $secret.prefix | default "" | upper }}
+          {{- $envKey := $key | upper | replace "." "_" | replace "-" "_" }}
+          - name: {{ $envPrefix }}{{ $envKey }}
+            valueFrom:
+              secretKeyRef:
+                name: {{ $secret.name }}
+                key: {{ $key }}
+          {{- end }}
+          {{- end }}
+        ports:
+          - containerPort: 8000
+        resources:
+          {{- toYaml .Values.parseable.highAvailability.ingestor.resources | nindent 12 }}
+        volumeMounts:
+        ## No data volume for ingestor
+        - mountPath: "/parseable/staging"
+          name: stage-volume
+  volumeClaimTemplates:
+  - metadata:
+      name: stage-volume
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+      storageClassName: "my-storage-class"
+      resources:
+        requests:
+          storage: 10Gi
+{{- end }}
diff --git a/helm/templates/logstream-job.yaml b/helm/templates/logstream-job.yaml
@@ -15,7 +15,7 @@ spec:
         {{- toYaml . | nindent 8 }}
       {{- end }}
       labels:
-        {{- include "parseable.selectorLabels" . | nindent 8 }}
+        {{- include "parseable.labelsSelector" . | nindent 8 }}
     spec:
       restartPolicy: OnFailure
       securityContext:

diff --git a/helm/templates/service-monitor.yaml b/helm/templates/service-monitor.yaml
@@ -4,7 +4,7 @@ kind: ServiceMonitor
 metadata:
   name: {{ include "parseable.fullname" . }}
   namespace: {{ default .Release.Namespace .Values.parseable.metrics.serviceMonitor.namespace | quote }}
-  labels: {{- include "parseable.selectorLabels" . | nindent 4 }}
+  labels: {{- include "parseable.labelsSelector" . | nindent 4 }}
     {{- if .Values.parseable.metrics.serviceMonitor.additionalLabels }}
     {{- include (dict "value" .Values.parseable.metrics.serviceMonitor.additionalLabels "context" $) | nindent 4 }}
     {{- end }}
@@ -33,5 +33,5 @@ spec:
     matchNames:
       - {{ .Release.Namespace }}
   selector:
-    matchLabels: {{- include "parseable.selectorLabels" . | nindent 6 }}
+    matchLabels: {{- include "parseable.labelsSelector" . | nindent 6 }}
 {{- end }}
diff --git a/helm/templates/deployment.yaml → helm/templates/standalone-deployment.yaml b/helm/templates/deployment.yaml → helm/templates/standalone-deployment.yaml
@@ -1,3 +1,4 @@
+{{- if eq .Values.parseable.highAvailability.enabled false }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -6,18 +7,18 @@ metadata:
   labels:
     {{- include "parseable.labels" . | nindent 4 }}
 spec:
-  replicas: {{ .Values.parseable.replicaCount }}
+  replicas: 1
   selector:
     matchLabels:
-      {{- include "parseable.selectorLabels" . | nindent 6 }}
+      {{- include "parseable.labelsSelector" . | nindent 6 }}
   template:
     metadata:
       {{- with .Values.parseable.podAnnotations }}
       annotations:
         {{- toYaml . | nindent 8 }}
       {{- end }}
       labels:
-        {{- include "parseable.selectorLabels" . | nindent 8 }}
+        {{- include "parseable.labelsSelector" . | nindent 8 }}
     spec:
       {{- with .Values.parseable.imagePullSecrets }}
       imagePullSecrets:
@@ -107,3 +108,4 @@ spec:
       tolerations:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+{{- end }}
diff --git a/helm/templates/service.yaml → helm/templates/standalone-service.yaml b/helm/templates/service.yaml → helm/templates/standalone-service.yaml
@@ -1,3 +1,4 @@
+{{- if eq .Values.parseable.highAvailability.enabled false }}
 apiVersion: v1
 kind: Service
 metadata:
@@ -10,4 +11,5 @@ spec:
       targetPort: 8000
       protocol: TCP
   selector:
-    {{- include "parseable.selectorLabels" . | nindent 4 }}
+    {{- include "parseable.labelsSelector" . | nindent 4 }}
+{{- end }}
diff --git a/helm/values.yaml b/helm/values.yaml
@@ -3,10 +3,27 @@ parseable:
     repository: parseable/parseable
     tag: v0.9.0
     pullPolicy: Always
-  replicaCount: 1
   ## Set to true if you want to deploy Parseable in local mode (store logs
   ## on local mount point instead of S3 bucket)
   local: false
+  ## Set to true if you want to deploy Parseable in a HA mode (multiple ingestors)
+  ## Please note that highAvailability is not supported in local mode
+  highAvailability:
+    enabled: true
+    ingestor: 
+      count: 3
+      env: 
+        RUST_LOG: warn
+      service:
+        type: ClusterIP
+        port: 80
+      resources:
+        limits:
+          cpu: 500m
+          memory: 4Gi
+        requests:
+          cpu: 250m
+          memory: 1Gi
   ## Add environment variables to the Parseable Deployment
   env:
     RUST_LOG: warn