Added tolerations to the installer chart (#819)

* Added tolerations definition to installer Jobs & all _supplemental files as they need to be always aligned * Updated documentation * Created Unit Tests for the tolerations, improved the old test in such a way to assure the backward compatibility of the new implementation ---------
pegasystems · Nov 11, 2024 · 055c775 · 055c775
1 parent e4bdbba
commit 055c775
Show file tree

Hide file tree

Showing 7 changed files with 153 additions and 71 deletions.
diff --git a/charts/pega/README.md b/charts/pega/README.md
@@ -1100,6 +1100,7 @@ Parameter   | Description   | Default value
 `imagePullPolicy` | Specify when to pull an image. | `IfNotPresent`
 `adminPassword` | Specify a temporary, initial password to log into the Pega application. This will need to be changed at first login. The adminPassword value cannot start with "@". | `"ADMIN_PASSWORD"`
 `affinity` | Configures policy to assign the pods to the nodes. See the official [Kubernetes Documentation](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/). | `""`
+`tolerations` | Configures the tolerations in order to schedule the pods into the appropiate nodes. | `""`
 `upgrade.upgradeType:` |Specify the type of process, applying a patch or upgrading. | See the next table for details.
 `upgrade.upgradeSteps:` |Specify the steps of a `custom` upgrade process that you want to complete. For `zero-downtime`, `out-of-place-rules`, `out-of-place-data`, or `in-place` upgrades, leave this parameter empty. | <ul>`enable_cluster_upgrade` `rules_migration` `rules_upgrade` `data_upgrade` `disable_cluster_upgrade`</ul>
 `upgrade.targetRulesSchema:` |Specify the name of the schema you created the process creates for the new rules schema. | `""`

diff --git a/charts/pega/charts/hazelcast/templates/_supplemental.tpl b/charts/pega/charts/hazelcast/templates/_supplemental.tpl
@@ -12,6 +12,7 @@ pega-hz-secret-name
 deployDBSecret
 deployNonExtDBSecret
 podAffinity
+tolerations
 secretResolver are copied from pega/templates/_helpers.tpl because helm lint requires
 charts to render standalone. See: https://github.com/helm/helm/issues/11260 for more details.
 */}}
@@ -123,4 +124,11 @@ false
 affinity:
 {{- toYaml .affinity | nindent 2 }}
 {{- end }}
+{{ end }}
+
+{{- define "tolerations" }}
+{{- if .tolerations }}
+tolerations:
+{{- toYaml .tolerations | nindent 2 }}
+{{- end }}
 {{ end }}
diff --git a/charts/pega/charts/installer/templates/_pega-installer-job.tpl b/charts/pega/charts/installer/templates/_pega-installer-job.tpl
@@ -189,5 +189,6 @@ spec:
       imagePullSecrets:
 {{- include "imagePullSecrets" .root | indent 6 }}
 {{- include "podAffinity" .root.Values | indent 6 }}
+{{- include "tolerations" .root.Values | indent 6 }}
 ---
 {{- end -}}
diff --git a/charts/pega/charts/installer/templates/_supplemental.tpl b/charts/pega/charts/installer/templates/_supplemental.tpl
@@ -12,6 +12,7 @@ pega-hz-secret-name
 deployDBSecret
 deployNonExtDBSecret
 podAffinity
+tolerations
 secretResolver are copied from pega/templates/_helpers.tpl because helm lint requires
 charts to render standalone. See: https://github.com/helm/helm/issues/11260 for more details.
 */}}
@@ -123,4 +124,11 @@ false
 affinity:
 {{- toYaml .affinity | nindent 2 }}
 {{- end }}
+{{ end }}
+
+{{- define "tolerations" }}
+{{- if .tolerations }}
+tolerations:
+{{- toYaml .tolerations | nindent 2 }}
+{{- end }}
 {{ end }}
diff --git a/charts/pega/charts/pegasearch/templates/_supplemental.tpl b/charts/pega/charts/pegasearch/templates/_supplemental.tpl
@@ -12,6 +12,7 @@ pega-hz-secret-name
 deployDBSecret
 deployNonExtDBSecret
 podAffinity
+tolerations
 secretResolver are copied from pega/templates/_helpers.tpl because helm lint requires
 charts to render standalone. See: https://github.com/helm/helm/issues/11260 for more details.
 */}}
@@ -123,4 +124,11 @@ false
 affinity:
 {{- toYaml .affinity | nindent 2 }}
 {{- end }}
+{{ end }}
+
+{{- define "tolerations" }}
+{{- if .tolerations }}
+tolerations:
+{{- toYaml .tolerations | nindent 2 }}
+{{- end }}
 {{ end }}
diff --git a/charts/pega/templates/_supplemental.tpl b/charts/pega/templates/_supplemental.tpl
@@ -12,6 +12,7 @@ pega-hz-secret-name
 deployDBSecret
 deployNonExtDBSecret
 podAffinity
+tolerations
 secretResolver are copied from pega/templates/_helpers.tpl because helm lint requires
 charts to render standalone. See: https://github.com/helm/helm/issues/11260 for more details.
 */}}
@@ -123,4 +124,11 @@ false
 affinity:
 {{- toYaml .affinity | nindent 2 }}
 {{- end }}
+{{ end }}
+
+{{- define "tolerations" }}
+{{- if .tolerations }}
+tolerations:
+{{- toYaml .tolerations | nindent 2 }}
+{{- end }}
 {{ end }}
diff --git a/terratest/src/test/pega/pega-installer-job_test.go b/terratest/src/test/pega/pega-installer-job_test.go
@@ -81,64 +81,6 @@ func TestPegaInstallerJob(t *testing.T) {
 	}
 }
 
-func TestPegaInstallerJobWithNodeSelector(t *testing.T) {
-	var options = &helm.Options{
-		SetValues: map[string]string{
-			"global.deployment.name":         "install-ns",
-			"global.provider":                "k8s",
-			"global.actions.execute":         "install",
-			"installer.imagePullPolicy":      "Always",
-			"installer.upgrade.upgradeType":  "zero-downtime",
-			"installer.nodeSelector.mylabel": "myvalue",
-		},
-	}
-
-	helmChartPath, err := filepath.Abs(PegaHelmChartPath)
-	require.NoError(t, err)
-
-	yamlContent := RenderTemplate(t, options, helmChartPath, []string{"charts/installer/templates/pega-installer-job.yaml"})
-	yamlSplit := strings.Split(yamlContent, "---")
-
-	var jobObj k8sbatch.Job
-	UnmarshalK8SYaml(t, yamlSplit[1], &jobObj)
-
-	require.Equal(t, "myvalue", jobObj.Spec.Template.Spec.NodeSelector["mylabel"])
-
-}
-
-func TestPegaInstallerJobWithAffinity(t *testing.T) {
-
-	var affintiyBasePath = "installer.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0]."
-
-	var options = &helm.Options{
-		SetValues: map[string]string{
-			"global.deployment.name":        "install-ns",
-			"global.provider":               "k8s",
-			"global.actions.execute":        "install",
-			"installer.imagePullPolicy":     "Always",
-			"installer.upgrade.upgradeType": "zero-downtime",
-			affintiyBasePath + "key":        "kubernetes.io/os",
-			affintiyBasePath + "operator":   "In",
-			affintiyBasePath + "values[0]":  "linux",
-		},
-	}
-
-	helmChartPath, err := filepath.Abs(PegaHelmChartPath)
-	require.NoError(t, err)
-
-	yamlContent := RenderTemplate(t, options, helmChartPath, []string{"charts/installer/templates/pega-installer-job.yaml"})
-	yamlSplit := strings.Split(yamlContent, "---")
-
-	var jobObj k8sbatch.Job
-	UnmarshalK8SYaml(t, yamlSplit[1], &jobObj)
-
-	jobAffinity := jobObj.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution
-
-	require.Equal(t, "kubernetes.io/os", jobAffinity.NodeSelectorTerms[0].MatchExpressions[0].Key)
-	require.Equal(t, "In", string(jobAffinity.NodeSelectorTerms[0].MatchExpressions[0].Operator))
-	require.Equal(t, "linux", jobAffinity.NodeSelectorTerms[0].MatchExpressions[0].Values[0])
-}
-
 func assertJob(t *testing.T, jobYaml string, expectedJob pegaDbJob, options *helm.Options, pullPolicy string) {
 	var jobObj k8sbatch.Job
 	UnmarshalK8SYaml(t, jobYaml, &jobObj)
@@ -148,6 +90,8 @@ func assertJob(t *testing.T, jobYaml string, expectedJob pegaDbJob, options *hel
 
 	var containerPort int32 = 8080
 
+	require.Empty(t, jobSpec.Affinity)
+	require.Empty(t, jobSpec.Tolerations)
 	require.Equal(t, jobSpec.Volumes[0].Name, "pega-installer-credentials-volume")
 	require.Equal(t, jobSpec.Volumes[0].VolumeSource.Projected.Sources[0].Secret.Name, getObjName(options, "-db-secret"))
 	require.Equal(t, jobSpec.Volumes[0].VolumeSource.Projected.Sources[1].Secret.Name, customArtifactorySecret)
@@ -189,15 +133,15 @@ func assertJob(t *testing.T, jobYaml string, expectedJob pegaDbJob, options *hel
 	VerifyInitContainerData(t, actualInitContainers, options)
 }
 
-func TestPegaInstallerJobResourcesWithEphemeralStorage(t *testing.T) {
+func TestPegaInstallerJobWithNodeSelector(t *testing.T) {
 	var options = &helm.Options{
 		SetValues: map[string]string{
-			"global.deployment.name":         "pega",
+			"global.deployment.name":         "install-ns",
 			"global.provider":                "k8s",
 			"global.actions.execute":         "install",
-			"installer.resources.requests.ephemeralStorage": "10Gi",
-			"installer.resources.limits.ephemeralStorage": "11Gi",
 			"installer.imagePullPolicy":      "Always",
+			"installer.upgrade.upgradeType":  "zero-downtime",
+			"installer.nodeSelector.mylabel": "myvalue",
 		},
 	}
 
@@ -210,19 +154,123 @@ func TestPegaInstallerJobResourcesWithEphemeralStorage(t *testing.T) {
 	var jobObj k8sbatch.Job
 	UnmarshalK8SYaml(t, yamlSplit[1], &jobObj)
 
-    require.Equal(t, "11Gi", jobObj.Spec.Template.Spec.Containers[0].Resources.Limits.StorageEphemeral().String())
-    require.Equal(t, "10Gi", jobObj.Spec.Template.Spec.Containers[0].Resources.Requests.StorageEphemeral().String())
+	require.Equal(t, "myvalue", jobObj.Spec.Template.Spec.NodeSelector["mylabel"])
+
+}
+
+func TestPegaInstallerJobWithAffinity(t *testing.T) {
+
+	var affintiyBasePath = "installer.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0]."
+	var supportedVendors = []string{"k8s", "openshift", "eks", "gke", "aks", "pks"}
+	var supportedOperations = []string{"install", "install-deploy", "upgrade-deploy"}
+
+	for _, vendor := range supportedVendors {
+		for _, operation := range supportedOperations {
+
+			var options = &helm.Options{
+				SetValues: map[string]string{
+					"global.deployment.name":        "install-ns",
+					"global.provider":               vendor,
+					"global.actions.execute":        operation,
+					"installer.imagePullPolicy":     "Always",
+					"installer.upgrade.upgradeType": "zero-downtime",
+					affintiyBasePath + "key":        "kubernetes.io/os",
+					affintiyBasePath + "operator":   "In",
+					affintiyBasePath + "values[0]":  "linux",
+				},
+			}
+
+			helmChartPath, err := filepath.Abs(PegaHelmChartPath)
+			require.NoError(t, err)
+
+			yamlContent := RenderTemplate(t, options, helmChartPath, []string{"charts/installer/templates/pega-installer-job.yaml"})
+			yamlSplit := strings.Split(yamlContent, "---")
+
+			var jobObj k8sbatch.Job
+			UnmarshalK8SYaml(t, yamlSplit[1], &jobObj)
+
+			jobAffinity := jobObj.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution
+
+			require.Equal(t, "kubernetes.io/os", jobAffinity.NodeSelectorTerms[0].MatchExpressions[0].Key)
+			require.Equal(t, "In", string(jobAffinity.NodeSelectorTerms[0].MatchExpressions[0].Operator))
+			require.Equal(t, "linux", jobAffinity.NodeSelectorTerms[0].MatchExpressions[0].Values[0])
+		}
+	}
 }
 
+func TestPegaInstallerJobWithTolerations(t *testing.T) {
+
+	var supportedVendors = []string{"k8s", "openshift", "eks", "gke", "aks", "pks"}
+	var supportedOperations = []string{"install", "install-deploy", "upgrade-deploy"}
+
+	for _, vendor := range supportedVendors {
+		for _, operation := range supportedOperations {
+
+			var options = &helm.Options{
+				SetValues: map[string]string{
+					"global.deployment.name":            "install-ns",
+					"global.provider":                   vendor,
+					"global.actions.execute":            operation,
+					"installer.imagePullPolicy":         "Always",
+					"installer.upgrade.upgradeType":     "zero-downtime",
+					"installer.tolerations[0].key":      "key1",
+					"installer.tolerations[0].operator": "Equal",
+					"installer.tolerations[0].value":    "value1",
+					"installer.tolerations[0].effect":   "NoSchedule",
+				},
+			}
+
+			helmChartPath, err := filepath.Abs(PegaHelmChartPath)
+			require.NoError(t, err)
+
+			yamlContent := RenderTemplate(t, options, helmChartPath, []string{"charts/installer/templates/pega-installer-job.yaml"})
+			yamlSplit := strings.Split(yamlContent, "---")
 
+			var jobObj k8sbatch.Job
+			UnmarshalK8SYaml(t, yamlSplit[1], &jobObj)
+
+			jobTolerations := jobObj.Spec.Template.Spec.Tolerations
+
+			require.Equal(t, "key1", jobTolerations[0].Key)
+			require.Equal(t, "Equal", string(jobTolerations[0].Operator))
+			require.Equal(t, "value1", jobTolerations[0].Value)
+			require.Equal(t, "NoSchedule", string(jobTolerations[0].Effect))
+		}
+	}
+}
+
+func TestPegaInstallerJobResourcesWithEphemeralStorage(t *testing.T) {
+	var options = &helm.Options{
+		SetValues: map[string]string{
+			"global.deployment.name":                        "pega",
+			"global.provider":                               "k8s",
+			"global.actions.execute":                        "install",
+			"installer.resources.requests.ephemeralStorage": "10Gi",
+			"installer.resources.limits.ephemeralStorage":   "11Gi",
+			"installer.imagePullPolicy":                     "Always",
+		},
+	}
+
+	helmChartPath, err := filepath.Abs(PegaHelmChartPath)
+	require.NoError(t, err)
+
+	yamlContent := RenderTemplate(t, options, helmChartPath, []string{"charts/installer/templates/pega-installer-job.yaml"})
+	yamlSplit := strings.Split(yamlContent, "---")
+
+	var jobObj k8sbatch.Job
+	UnmarshalK8SYaml(t, yamlSplit[1], &jobObj)
+
+	require.Equal(t, "11Gi", jobObj.Spec.Template.Spec.Containers[0].Resources.Limits.StorageEphemeral().String())
+	require.Equal(t, "10Gi", jobObj.Spec.Template.Spec.Containers[0].Resources.Requests.StorageEphemeral().String())
+}
 
 func TestPegaInstallerJobResourcesWithNoEphemeralStorage(t *testing.T) {
 	var options = &helm.Options{
 		SetValues: map[string]string{
-			"global.deployment.name":         "pega",
-			"global.provider":                "k8s",
-			"global.actions.execute":         "install",
-			"installer.imagePullPolicy":      "Always",
+			"global.deployment.name":    "pega",
+			"global.provider":           "k8s",
+			"global.actions.execute":    "install",
+			"installer.imagePullPolicy": "Always",
 		},
 	}
 
@@ -235,7 +283,7 @@ func TestPegaInstallerJobResourcesWithNoEphemeralStorage(t *testing.T) {
 	var jobObj k8sbatch.Job
 	UnmarshalK8SYaml(t, yamlSplit[1], &jobObj)
 
-    require.Equal(t, "0", jobObj.Spec.Template.Spec.Containers[0].Resources.Limits.StorageEphemeral().String())
-    require.Equal(t, "0", jobObj.Spec.Template.Spec.Containers[0].Resources.Requests.StorageEphemeral().String())
+	require.Equal(t, "0", jobObj.Spec.Template.Spec.Containers[0].Resources.Limits.StorageEphemeral().String())
+	require.Equal(t, "0", jobObj.Spec.Template.Spec.Containers[0].Resources.Requests.StorageEphemeral().String())
 
-}
+}