diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e7ce48f..824da6ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,10 @@ # Changelog ## In Development + +## v0.10.0 * Bump versions of all dependencies (#50) +* Allow st2sensorcontainer to be partitioned (#51) * Replace single-node `etcd` coordination backend with 3-node etcd HA cluster, deployed as a Helm dependency (#52) * Fixed improper job load order for enterprise edition failing due to missing RBAC roles & assignments (#53) diff --git a/README.md b/README.md index 72be7535..356c22de 100644 --- a/README.md +++ b/README.md @@ -112,12 +112,27 @@ In an HA deployment there must be a minimum of `2` replicas of st2notifier runni which in our case is `etcd`. ### [st2sensorcontainer](https://docs.stackstorm.com/reference/ha.html#st2sensorcontainer) -st2sensorcontainer manages StackStorm sensors: starts, stops and restarts them as a subprocesses. -At the moment K8s configuration consists of Deployment with hardcoded `1` replica. -Future plans are to re-work this setup and benefit from Docker-friendly [single-sensor-per-container mode #4179](https://github.com/StackStorm/st2/pull/4179) -(since st2 `v2.9`) as a way of [Sensor Partitioning](https://docs.stackstorm.com/latest/reference/sensor_partitioning.html), distributing the computing load -between many pods and relying on K8s failover/reschedule mechanisms, instead of running everything on `1` single instance of st2sensorcontainer. +st2sensorcontainer manages StackStorm sensors: It starts, stops and restarts them as subprocesses. +By default, deployment is configured with `1` replica containing all the sensors. +st2sensorcontainer also supports a more Docker-friendly single-sensor-per-container mode as a way of +[Sensor Partitioning](https://docs.stackstorm.com/latest/reference/sensor_partitioning.html). This +distributes the computing load between many pods and relies on K8s failover/reschedule mechanisms, +instead of running everything on a single instance of st2sensorcontainer. The sensor(s) must be +deployed as part of the custom packs image. + +As an example, override the default Helm values as follows: + +``` +st2: + packs: + sensors: + - name: github + ref: githubwebhook.GitHubWebhookSensor + - name: circleci + ref: circle_ci.CircleCIWebhookSensor +``` + ### [st2actionrunner](https://docs.stackstorm.com/reference/ha.html#st2actionrunner) Stackstorm workers that actually execute actions. `5` replicas for K8s Deployment are configured by default to increase StackStorm ability to execute actions without excessive queuing. diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl index bbc11087..a2c9c14c 100644 --- a/templates/_helpers.tpl +++ b/templates/_helpers.tpl @@ -27,3 +27,8 @@ stackstorm {{- define "enterpriseSuffix" -}} {{ if required "Missing context '.Values.enterprise.enabled'!" .Values.enterprise.enabled }}-enterprise{{ end }} {{- end -}} + +# Generate '-' prefix only when the variable is defined +{{- define "hyphenPrefix" -}} +{{ if . }}-{{ . }}{{end}} +{{- end -}} diff --git a/templates/deployments.yaml b/templates/deployments.yaml index c79b42dd..9762d5ee 100644 --- a/templates/deployments.yaml +++ b/templates/deployments.yaml @@ -780,56 +780,56 @@ spec: {{ toYaml . | indent 8 }} {{- end }} +{{- range .Values.st2.packs.sensors }} --- apiVersion: apps/v1 kind: Deployment metadata: - name: {{ .Release.Name }}-st2sensorcontainer{{ template "enterpriseSuffix" . }} + name: {{ $.Release.Name }}-st2sensorcontainer{{ template "hyphenPrefix" .name }}{{ template "enterpriseSuffix" $ }} labels: - app: st2sensorcontainer + app: st2sensorcontainer{{ template "hyphenPrefix" .name }} tier: backend vendor: stackstorm - support: {{ template "supportMethod" . }} - chart: {{ .Chart.Name }}-{{ .Chart.Version }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} + support: {{ template "supportMethod" $ }} + chart: {{ $.Chart.Name }}-{{ $.Chart.Version }} + release: {{ $.Release.Name }} + heritage: {{ $.Release.Service }} spec: selector: matchLabels: - app: st2sensorcontainer - support: {{ template "supportMethod" . }} - release: {{ .Release.Name }} + app: st2sensorcontainer{{ template "hyphenPrefix" .name }} + support: {{ template "supportMethod" $ }} + release: {{ $.Release.Name }} # https://docs.stackstorm.com/reference/ha.html#st2sensorcontainer # It is possible to run st2sensorcontainer in HA mode by running one process on each compute instance. Each sensor node needs to be # provided with proper partition information to share work with other sensor nodes so that the same sensor does not run on different nodes. # See Partitioning Sensors for information on how to partition sensors. - # TODO: Re-work to use single-sensor-per-container mode instead of running 1 node. Proper implementation is possible with Helm templating (#4) replicas: 1 template: metadata: labels: - app: st2sensorcontainer + app: st2sensorcontainer{{ template "hyphenPrefix" .name }} tier: backend vendor: stackstorm - support: {{ template "supportMethod" . }} - chart: {{ .Chart.Name }}-{{ .Chart.Version }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} + support: {{ template "supportMethod" $ }} + chart: {{ $.Chart.Name }}-{{ $.Chart.Version }} + release: {{ $.Release.Name }} + heritage: {{ $.Release.Service }} annotations: - checksum/config: {{ include (print $.Template.BasePath "/configmaps_st2-conf.yaml") . | sha256sum }} - checksum/packs: {{ include (print $.Template.BasePath "/configmaps_packs.yaml") . | sha256sum }} + checksum/config: {{ include (print $.Template.BasePath "/configmaps_st2-conf.yaml") $ | sha256sum }} + checksum/packs: {{ include (print $.Template.BasePath "/configmaps_packs.yaml") $ | sha256sum }} spec: - {{- if .Values.enterprise.enabled }} + {{- if $.Values.enterprise.enabled }} imagePullSecrets: - - name: {{ .Release.Name }}-st2-license + - name: {{ $.Release.Name }}-st2-license {{- end }} - {{- if .Values.st2.packs.image.repository }} + {{- if $.Values.st2.packs.image.repository }} initContainers: # Merge packs and virtualenvs from st2sensorcontainer with those from the st2.packs image # Custom packs - name: st2-custom-packs - image: "{{ .Values.st2.packs.image.repository }}/{{ .Values.st2.packs.image.name }}:{{ .Values.st2.packs.image.tag }}" - imagePullPolicy: {{ .Values.st2.packs.image.pullPolicy | quote }} + image: "{{ $.Values.st2.packs.image.repository }}/{{ $.Values.st2.packs.image.name }}:{{ $.Values.st2.packs.image.tag }}" + imagePullPolicy: {{ $.Values.st2.packs.image.pullPolicy | quote }} volumeMounts: - name: st2-packs-vol mountPath: /opt/stackstorm/packs-shared @@ -843,8 +843,8 @@ spec: /bin/cp -aR /opt/stackstorm/virtualenvs/. /opt/stackstorm/virtualenvs-shared # System packs - name: st2-system-packs - image: "{{ template "imageRepository" . }}/st2actionrunner{{ template "enterpriseSuffix" . }}:{{ .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} + image: "{{ template "imageRepository" $ }}/st2actionrunner{{ template "enterpriseSuffix" $ }}:{{ $.Chart.AppVersion }}" + imagePullPolicy: {{ $.Values.image.pullPolicy }} volumeMounts: - name: st2-packs-vol mountPath: /opt/stackstorm/packs-shared @@ -856,15 +856,30 @@ spec: /bin/cp -aR /opt/stackstorm/virtualenvs/. /opt/stackstorm/virtualenvs-shared {{- end }} containers: - - name: st2sensorcontainer{{ template "enterpriseSuffix" . }} - image: "{{ template "imageRepository" . }}/st2sensorcontainer{{ template "enterpriseSuffix" . }}:{{ .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - # TODO: Add liveness/readiness probes (#3) - #livenessProbe: - #readinessProbe: + - name: st2sensorcontainer{{ template "hyphenPrefix" .name }}{{ template "enterpriseSuffix" $ }} + image: "{{ template "imageRepository" $ }}/st2sensorcontainer{{ template "enterpriseSuffix" $ }}:{{ $.Chart.AppVersion }}" + imagePullPolicy: {{ $.Values.image.pullPolicy }} + {{- with .readinessProbe }} + # Probe to check if app is running. Failure will lead to a pod restart. + readinessProbe: +{{ toYaml . | indent 10 }} + {{- end }} + {{- with .livenessProbe }} + livenessProbe: +{{ toYaml . | indent 10 }} + {{- end }} + {{- if .ref }} + command: + - /opt/stackstorm/st2/bin/st2sensorcontainer + - --config-file=/etc/st2/st2.conf + - --config-file=/etc/st2/st2.docker.conf + - --config-file=/etc/st2/st2.user.conf + - --single-sensor-mode + - --sensor-ref={{ .ref }} + {{- end }} envFrom: - configMapRef: - name: {{ .Release.Name }}-st2-urls + name: {{ $.Release.Name }}-st2-urls volumeMounts: - name: st2-config-vol mountPath: /etc/st2/st2.docker.conf @@ -872,7 +887,7 @@ spec: - name: st2-config-vol mountPath: /etc/st2/st2.user.conf subPath: st2.user.conf - {{- if .Values.st2.packs.image.repository }} + {{- if $.Values.st2.packs.image.repository }} - name: st2-packs-vol mountPath: /opt/stackstorm/packs readOnly: true @@ -881,29 +896,30 @@ spec: readOnly: true {{- end }} resources: -{{ toYaml .Values.st2sensorcontainer.resources | indent 10 }} +{{ toYaml .resources | indent 10 }} volumes: - name: st2-config-vol configMap: - name: {{ .Release.Name }}-st2-config - {{- if .Values.st2.packs.image.repository }} + name: {{ $.Release.Name }}-st2-config + {{- if $.Values.st2.packs.image.repository }} - name: st2-packs-vol emptyDir: {} - name: st2-virtualenvs-vol emptyDir: {} {{- end }} - {{- with .Values.st2sensorcontainer.nodeSelector }} + {{- with .nodeSelector }} nodeSelector: {{ toYaml . | indent 8 }} {{- end }} - {{- with .Values.st2sensorcontainer.affinity }} + {{- with .affinity }} affinity: {{ toYaml . | indent 8 }} {{- end }} - {{- with .Values.st2sensorcontainer.tolerations }} + {{- with .tolerations }} tolerations: {{ toYaml . | indent 8 }} {{- end }} +{{- end }} --- apiVersion: apps/v1 diff --git a/values.yaml b/values.yaml index 3a794b71..0580cbab 100644 --- a/values.yaml +++ b/values.yaml @@ -85,6 +85,24 @@ st2: name: st2packs tag: latest pullPolicy: Always + + # https://docs.stackstorm.com/reference/ha.html#st2sensorcontainer + # It is possible to run st2sensorcontainer in HA mode by running one process on each compute instance. + # Each sensor node needs to be provided with proper partition information to share work with other sensor + # nodes so that the same sensor does not run on different nodes. + sensors: + # Specify default container that executes all sensors. + # To partition sensors with one sensor per node, override st2.packs.sensors. + # NOTE: Do not modify this file. + - name: + livenessProbe: {} + readinessProbe: {} + # TODO: Find out recommended/default resources for this specific service (#5) + resources: {} + # Additional advanced settings to control pod/deployment placement + affinity: {} + nodeSelector: {} + tolerations: [] # Import data into StackStorm's Key/Value datastore (https://docs.stackstorm.com/datastore.html) keyvalue: #- name: st2_version @@ -314,19 +332,6 @@ st2notifier: nodeSelector: {} tolerations: [] affinity: {} -# https://docs.stackstorm.com/reference/ha.html#st2sensorcontainer -# It is possible to run st2sensorcontainer in HA mode by running one process on each compute instance. Each sensor node needs to be -# provided with proper partition information to share work with other sensor nodes so that the same sensor does not run on different nodes. -st2sensorcontainer: - # TODO: Re-work to use single-sensor-per-container mode partitioning instead of running 1 single node of st2sensorcontainer. Proper implementation is now possible with Helm templating (#4) - # NB! Number of replicas are hardcoded to 1, see above T0D0 about using single-sensor-per-container mode in future as way of Sensor Partitioning. - # replicas: 1 - # TODO: Find out recommended/default resources for this specific service (#5) - resources: {} - # Additional advanced settings to control pod/deployment placement - nodeSelector: {} - tolerations: [] - affinity: {} # https://docs.stackstorm.com/reference/ha.html#st2actionrunner # Multiple st2actionrunner processes can run in active-active with only connections to MongoDB and RabbitMQ. Work gets naturally # distributed across runners via RabbitMQ. Adding more st2actionrunner processes increases the ability of StackStorm to execute actions.