feat: add distributed option to helm-chart and update console (#749)

parseablehq · Apr 20, 2024 · 09b9e40 · 09b9e40
1 parent c79b255
commit 09b9e40
Show file tree

Hide file tree

Showing 16 changed files with 313 additions and 52 deletions.
diff --git a/helm-releases/operator-0.0.3.tgz b/helm-releases/operator-0.0.3.tgz
diff --git a/helm-releases/parseable-1.0.0.tgz b/helm-releases/parseable-1.0.0.tgz
diff --git a/helm/Chart.yaml b/helm/Chart.yaml
@@ -2,8 +2,8 @@ apiVersion: v2
 name: parseable
 description: Helm chart for Parseable Server
 type: application
-version: 0.9.0
-appVersion: "v0.9.0"
+version: 1.0.0
+appVersion: "v1.0.0"
 maintainers:
   - name: Parseable Team
     email: [email protected]

diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl
@@ -35,7 +35,19 @@ Common labels
 */}}
 {{- define "parseable.labels" -}}
 helm.sh/chart: {{ include "parseable.chart" . }}
-{{ include "parseable.selectorLabels" . }}
+{{ include "parseable.labelsSelector" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Ingestor Labels
+*/}}
+{{- define "parseable.ingestorLabels" -}}
+helm.sh/chart: {{ include "parseable.chart" . }}
+{{ include "parseable.ingestorLabelsSelector" . }}
 {{- if .Chart.AppVersion }}
 app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
 {{- end }}
@@ -45,9 +57,18 @@ app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{/*
 Selector labels
 */}}
-{{- define "parseable.selectorLabels" -}}
+{{- define "parseable.labelsSelector" -}}
+app.kubernetes.io/name: {{ include "parseable.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Ingestor Labels Selector for ingestor statefulset
+*/}}
+{{- define "parseable.ingestorLabelsSelector" -}}
 app.kubernetes.io/name: {{ include "parseable.name" . }}
 app.kubernetes.io/instance: {{ .Release.Name }}
+app.parseable.com/type: ingestor
 {{- end }}
 
 {{/*

diff --git a/helm/templates/ingestor-service.yaml b/helm/templates/ingestor-service.yaml
@@ -0,0 +1,15 @@
+{{- if eq .Values.parseable.highAvailability.enabled true }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "parseable.fullname" . }}-ingestor-service
+  namespace: {{ .Release.Namespace }}
+spec:
+  type: {{ $.Values.parseable.highAvailability.ingestor.service.type }}
+  ports:
+    - port: {{ $.Values.parseable.highAvailability.ingestor.service.port }}
+      targetPort: {{ .Values.parseable.highAvailability.ingestor.port }}
+      protocol: TCP
+  selector:
+    {{- include "parseable.ingestorLabelsSelector" . | nindent 4 }}
+{{- end }}
diff --git a/helm/templates/ingestor-statefulset.yaml b/helm/templates/ingestor-statefulset.yaml
@@ -0,0 +1,78 @@
+{{- if eq .Values.parseable.highAvailability.enabled true }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "parseable.fullname" . }}-ingestor-headless
+  namespace: {{ .Release.Namespace }}
+spec:
+  ports:
+    - port: 8000
+      name: "parseable-port"
+  clusterIP: None
+  selector:
+    {{- include "parseable.ingestorLabelsSelector" . | nindent 4 }}
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ include "parseable.fullname" . }}-ingestor
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "parseable.ingestorLabels" . | nindent 4 }}
+spec:
+  selector:
+    matchLabels:
+      {{- include "parseable.ingestorLabelsSelector" . | nindent 6 }}
+  serviceName: {{ include "parseable.fullname" . }}-headless
+  replicas: {{ .Values.parseable.highAvailability.ingestor.count }}
+  minReadySeconds: 2
+  template:
+    metadata:
+      labels:
+        {{- include "parseable.ingestorLabelsSelector" . | nindent 8 }}
+    spec:
+      terminationGracePeriodSeconds: 10 
+      containers:
+      - name: {{ .Chart.Name }}
+        securityContext:
+          {{- toYaml .Values.parseable.securityContext | nindent 8 }}
+        image: {{ .Values.parseable.image.repository }}:{{ .Values.parseable.image.tag | default .Chart.AppVersion }}
+        imagePullPolicy: {{ .Values.parseable.image.pullPolicy }}
+        command: ["/bin/bash", "-c"]
+        args: ["parseable s3-store --ingestor-endpoint=${HOSTNAME}.{{ include "parseable.fullname" . }}-ingestor-headless.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.parseable.highAvailability.ingestor.port }}"]
+        env:
+          {{- range $key, $value :=  .Values.parseable.highAvailability.ingestor.env }}
+          - name: {{ $key }}
+            value: {{ tpl $value $ | quote }}
+          {{- end }}
+          {{- range $secret := .Values.parseable.s3ModeSecret }}
+          {{- range $key := $secret.keys }}
+          {{- $envPrefix := $secret.prefix | default "" | upper }}
+          {{- $envKey := $key | upper | replace "." "_" | replace "-" "_" }}
+          - name: {{ $envPrefix }}{{ $envKey }}
+            valueFrom:
+              secretKeyRef:
+                name: {{ $secret.name }}
+                key: {{ $key }}
+          {{- end }}
+          {{- end }}
+          - name: P_MODE
+            value: "ingest"
+        ports:
+          - containerPort: {{ .Values.parseable.highAvailability.ingestor.port }}
+        resources:
+          {{- toYaml .Values.parseable.highAvailability.ingestor.resources | nindent 12 }}
+        volumeMounts:
+          ## No data volume for ingestor
+        - mountPath: "/parseable/staging"
+          name: stage-volume
+  volumeClaimTemplates:
+  - metadata:
+      name: stage-volume
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+      storageClassName: "do-block-storage"
+      resources:
+        requests:
+          storage: 10Gi
+{{- end }}
diff --git a/helm/templates/configmap.yaml → helm/templates/logstream-configmap.yaml b/helm/templates/configmap.yaml → helm/templates/logstream-configmap.yaml
diff --git a/helm/templates/logstream-job.yaml b/helm/templates/logstream-job.yaml
@@ -15,7 +15,7 @@ spec:
         {{- toYaml . | nindent 8 }}
       {{- end }}
       labels:
-        {{- include "parseable.selectorLabels" . | nindent 8 }}
+        {{- include "parseable.labelsSelector" . | nindent 8 }}
     spec:
       restartPolicy: OnFailure
       securityContext:

diff --git a/helm/templates/querier-deployment.yaml b/helm/templates/querier-deployment.yaml
@@ -0,0 +1,85 @@
+{{- if eq .Values.parseable.highAvailability.enabled true }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "parseable.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "parseable.labels" . | nindent 4 }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      {{- include "parseable.labelsSelector" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.parseable.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "parseable.labelsSelector" . | nindent 8 }}
+    spec:
+      {{- with .Values.parseable.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "parseable.serviceAccountName" . }}
+      securityContext:
+        {{- toYaml .Values.parseable.podSecurityContext | nindent 8 }}
+      containers:
+        - name: {{ .Chart.Name }}
+          securityContext:
+            {{- toYaml .Values.parseable.securityContext | nindent 12 }}
+          image: "{{ .Values.parseable.image.repository }}:{{ .Values.parseable.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.parseable.image.pullPolicy }}
+          # Uncomment to debug
+          # command: [ "/bin/sh", "-c", "sleep 1000000" ]
+          args: ["parseable", "s3-store"]
+          env:
+            - name: P_MODE
+              value: "query"
+            {{- range $key, $value :=  .Values.parseable.env }}
+            - name: {{ $key }}
+              value: {{ tpl $value $ | quote }}
+            {{- end }}
+            {{- range $secret := .Values.parseable.s3ModeSecret }}
+            {{- range $key := $secret.keys }}
+            {{- $envPrefix := $secret.prefix | default "" | upper }}
+            {{- $envKey := $key | upper | replace "." "_" | replace "-" "_" }}
+            - name: {{ $envPrefix }}{{ $envKey }}
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $secret.name }}
+                  key: {{ $key }}
+            {{- end }}
+            {{- end }}
+          ports:
+            - containerPort: 8000
+          resources:
+            {{- toYaml .Values.parseable.resources | nindent 12 }}
+          volumeMounts:
+          - mountPath: "/parseable/staging"
+            name: stage-volume
+      volumes:
+      {{- if .Values.parseable.persistence.staging.enabled }}
+      - name: stage-volume
+        persistentVolumeClaim:
+          claimName: {{ include "parseable.fullname" . }}-staging-pvc
+      {{- else }}
+      - name: stage-volume
+        emptyDir: {}
+      {{- end }}
+      {{- with .Values.parseable.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.parseable.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.parseable.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+{{- end }}
diff --git a/helm/templates/querier-service.yaml b/helm/templates/querier-service.yaml
@@ -0,0 +1,15 @@
+{{- if eq .Values.parseable.highAvailability.enabled true }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "parseable.fullname" . }}-ingestor-service
+  namespace: {{ .Release.Namespace }}
+spec:
+  type: {{ $.Values.parseable.service.type }}
+  ports:
+    - port: {{ $.Values.parseable.service.port }}
+      targetPort: 8000
+      protocol: TCP
+  selector:
+    {{- include "parseable.labelsSelector" . | nindent 4 }}
+{{- end }}
diff --git a/helm/templates/service-monitor.yaml b/helm/templates/service-monitor.yaml
@@ -4,7 +4,7 @@ kind: ServiceMonitor
 metadata:
   name: {{ include "parseable.fullname" . }}
   namespace: {{ default .Release.Namespace .Values.parseable.metrics.serviceMonitor.namespace | quote }}
-  labels: {{- include "parseable.selectorLabels" . | nindent 4 }}
+  labels: {{- include "parseable.labelsSelector" . | nindent 4 }}
     {{- if .Values.parseable.metrics.serviceMonitor.additionalLabels }}
     {{- include (dict "value" .Values.parseable.metrics.serviceMonitor.additionalLabels "context" $) | nindent 4 }}
     {{- end }}
@@ -33,5 +33,5 @@ spec:
     matchNames:
       - {{ .Release.Namespace }}
   selector:
-    matchLabels: {{- include "parseable.selectorLabels" . | nindent 6 }}
+    matchLabels: {{- include "parseable.labelsSelector" . | nindent 6 }}
 {{- end }}
diff --git a/helm/templates/deployment.yaml → helm/templates/standalone-deployment.yaml b/helm/templates/deployment.yaml → helm/templates/standalone-deployment.yaml
@@ -1,3 +1,4 @@
+{{- if eq .Values.parseable.highAvailability.enabled false }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -6,18 +7,18 @@ metadata:
   labels:
     {{- include "parseable.labels" . | nindent 4 }}
 spec:
-  replicas: {{ .Values.parseable.replicaCount }}
+  replicas: 1
   selector:
     matchLabels:
-      {{- include "parseable.selectorLabels" . | nindent 6 }}
+      {{- include "parseable.labelsSelector" . | nindent 6 }}
   template:
     metadata:
       {{- with .Values.parseable.podAnnotations }}
       annotations:
         {{- toYaml . | nindent 8 }}
       {{- end }}
       labels:
-        {{- include "parseable.selectorLabels" . | nindent 8 }}
+        {{- include "parseable.labelsSelector" . | nindent 8 }}
     spec:
       {{- with .Values.parseable.imagePullSecrets }}
       imagePullSecrets:
@@ -107,3 +108,4 @@ spec:
       tolerations:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+{{- end }}
diff --git a/helm/templates/service.yaml → helm/templates/standalone-service.yaml b/helm/templates/service.yaml → helm/templates/standalone-service.yaml
@@ -1,3 +1,4 @@
+{{- if eq .Values.parseable.highAvailability.enabled false }}
 apiVersion: v1
 kind: Service
 metadata:
@@ -10,4 +11,5 @@ spec:
       targetPort: 8000
       protocol: TCP
   selector:
-    {{- include "parseable.selectorLabels" . | nindent 4 }}
+    {{- include "parseable.labelsSelector" . | nindent 4 }}
+{{- end }}
diff --git a/helm/values.yaml b/helm/values.yaml
@@ -1,12 +1,32 @@
 parseable:
   image:
     repository: parseable/parseable
-    tag: v0.9.0
+    tag: v1.0.0
     pullPolicy: Always
-  replicaCount: 1
   ## Set to true if you want to deploy Parseable in local mode (store logs
   ## on local mount point instead of S3 bucket)
-  local: false
+  local: true
+  ## Set to true if you want to deploy Parseable in a HA mode (multiple ingestors)
+  ## Please note that highAvailability is not supported in local mode
+  highAvailability:
+    enabled: false
+    ingestor: 
+      port: 8000
+      count: 3
+      env: 
+        RUST_LOG: warn
+      ## Use this endpoint to send events to ingestors
+      ## Console (UI) is available on the other service (that points to the query pod)
+      service:
+        type: ClusterIP
+        port: 80 
+      resources:
+        limits:
+          cpu: 500m
+          memory: 4Gi
+        requests:
+          cpu: 250m
+          memory: 1Gi
   ## Add environment variables to the Parseable Deployment
   env:
     RUST_LOG: warn