[PLAT-520] Added kube-state-metrics support

stable/yugaware/templates/_helpers.tpl

@@ -112,3 +112,16 @@ Make list of allowed CORS origins
 {{- end -}}
 ]
 {{- end -}}
+
+{{/*
+kube-state-metrics
+*/}}
+{{- define "yugaware.kubeStateMetrics.endpoint" -}}
+{{- if .Values.kubeStateMetrics.customEndpoint -}}
+{{- .Values.kubeStateMetrics.customEndpoint -}}
+{{- else if .Values.kubeStateMetrics.install -}}
+{{- printf "%s-ksm.%s.svc.%s:8080" .Release.Name .Release.Namespace .Values.domainName -}}
+{{- else -}}
+{{- printf "kube-state-metrics.kube-system.svc.%s:8080" .Values.domainName -}}
+{{- end -}}
+{{- end -}}

stable/yugaware/templates/configs.yaml

@@ -332,7 +332,7 @@ data:
 
       - job_name: 'kube-state-metrics'
         static_configs:
-        - targets: ['kube-state-metrics.kube-system.svc.{{.Values.domainName}}:8080']
+        - targets: ['{{ template "yugaware.kubeStateMetrics.endpoint" . }}']
         metric_relabel_configs:
           # Save the name of the metric so we can group_by since we cannot by __name__ directly...
           - source_labels: ["__name__"]
@@ -347,6 +347,16 @@ data:
             regex: "(.*)"
             target_label: "container_name"
             replacement: "$1"
+          # rename old name of the CPU metric to the new name and label
+          # ref: https://github.com/kubernetes/kube-state-metrics/blob/master/CHANGELOG.md#v200-alpha--2020-09-16
+          - source_labels: ["__name__"]
+            regex: "kube_pod_container_resource_requests_cpu_cores"
+            target_label: "unit"
+            replacement: "core"
+          - source_labels: ["__name__"]
+            regex: "kube_pod_container_resource_requests_cpu_cores"
+            target_label: "__name__"
+            replacement: "kube_pod_container_resource_requests"
 
       - job_name: 'kubernetes-cadvisor'
 

stable/yugaware/templates/kube-state-metrics/kube-state-metrics-clusterrole.yaml

@@ -0,0 +1,182 @@
+{{- /*
+  Adapted from https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-state-metrics (Apache-2.0 license).
+  */}}
+{{- if and .Values.kubeStateMetrics.install (not .Values.kubeStateMetrics.customEndpoint) -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app: {{ .Release.Name }}-yugaware-ksm
+    chart: {{ template "yugaware.chart" . }}
+    release: {{ .Release.Name }}
+  name: {{ .Release.Name }}-ksm
+rules:
+{{ if has "certificatesigningrequests" $.Values.kubeStateMetrics.collectors }}
+- apiGroups: ["certificates.k8s.io"]
+  resources:
+  - certificatesigningrequests
+  verbs: ["list", "watch"]
+{{ end -}}
+{{ if has "configmaps" $.Values.kubeStateMetrics.collectors }}
+- apiGroups: [""]
+  resources:
+  - configmaps
+  verbs: ["list", "watch"]
+{{ end -}}
+{{ if has "cronjobs" $.Values.kubeStateMetrics.collectors }}
+- apiGroups: ["batch"]
+  resources:
+  - cronjobs
+  verbs: ["list", "watch"]
+{{ end -}}
+{{ if has "daemonsets" $.Values.kubeStateMetrics.collectors }}
+- apiGroups: ["extensions", "apps"]
+  resources:
+  - daemonsets
+  verbs: ["list", "watch"]
+{{ end -}}
+{{ if has "deployments" $.Values.kubeStateMetrics.collectors }}
+- apiGroups: ["extensions", "apps"]
+  resources:
+  - deployments
+  verbs: ["list", "watch"]
+{{ end -}}
+{{ if has "endpoints" $.Values.kubeStateMetrics.collectors }}
+- apiGroups: [""]
+  resources:
+  - endpoints
+  verbs: ["list", "watch"]
+{{ end -}}
+{{ if has "horizontalpodautoscalers" $.Values.kubeStateMetrics.collectors }}
+- apiGroups: ["autoscaling"]
+  resources:
+  - horizontalpodautoscalers
+  verbs: ["list", "watch"]
+{{ end -}}
+{{ if has "ingresses" $.Values.kubeStateMetrics.collectors }}
+- apiGroups: ["extensions", "networking.k8s.io"]
+  resources:
+  - ingresses
+  verbs: ["list", "watch"]
+{{ end -}}
+{{ if has "jobs" $.Values.kubeStateMetrics.collectors }}
+- apiGroups: ["batch"]
+  resources:
+  - jobs
+  verbs: ["list", "watch"]
+{{ end -}}
+{{ if has "limitranges" $.Values.kubeStateMetrics.collectors }}
+- apiGroups: [""]
+  resources:
+  - limitranges
+  verbs: ["list", "watch"]
+{{ end -}}
+{{ if has "mutatingwebhookconfigurations" $.Values.kubeStateMetrics.collectors }}
+- apiGroups: ["admissionregistration.k8s.io"]
+  resources:
+    - mutatingwebhookconfigurations
+  verbs: ["list", "watch"]
+{{ end -}}
+{{ if has "namespaces" $.Values.kubeStateMetrics.collectors }}
+- apiGroups: [""]
+  resources:
+  - namespaces
+  verbs: ["list", "watch"]
+{{ end -}}
+{{ if has "networkpolicies" $.Values.kubeStateMetrics.collectors }}
+- apiGroups: ["networking.k8s.io"]
+  resources:
+  - networkpolicies
+  verbs: ["list", "watch"]
+{{ end -}}
+{{ if has "nodes" $.Values.kubeStateMetrics.collectors }}
+- apiGroups: [""]
+  resources:
+  - nodes
+  verbs: ["list", "watch"]
+{{ end -}}
+{{ if has "persistentvolumeclaims" $.Values.kubeStateMetrics.collectors }}
+- apiGroups: [""]
+  resources:
+  - persistentvolumeclaims
+  verbs: ["list", "watch"]
+{{ end -}}
+{{ if has "persistentvolumes" $.Values.kubeStateMetrics.collectors }}
+- apiGroups: [""]
+  resources:
+  - persistentvolumes
+  verbs: ["list", "watch"]
+{{ end -}}
+{{ if has "poddisruptionbudgets" $.Values.kubeStateMetrics.collectors }}
+- apiGroups: ["policy"]
+  resources:
+    - poddisruptionbudgets
+  verbs: ["list", "watch"]
+{{ end -}}
+{{ if has "pods" $.Values.kubeStateMetrics.collectors }}
+- apiGroups: [""]
+  resources:
+  - pods
+  verbs: ["list", "watch"]
+{{ end -}}
+{{ if has "replicasets" $.Values.kubeStateMetrics.collectors }}
+- apiGroups: ["extensions", "apps"]
+  resources:
+  - replicasets
+  verbs: ["list", "watch"]
+{{ end -}}
+{{ if has "replicationcontrollers" $.Values.kubeStateMetrics.collectors }}
+- apiGroups: [""]
+  resources:
+  - replicationcontrollers
+  verbs: ["list", "watch"]
+{{ end -}}
+{{ if has "resourcequotas" $.Values.kubeStateMetrics.collectors }}
+- apiGroups: [""]
+  resources:
+  - resourcequotas
+  verbs: ["list", "watch"]
+{{ end -}}
+{{ if has "secrets" $.Values.kubeStateMetrics.collectors }}
+- apiGroups: [""]
+  resources:
+  - secrets
+  verbs: ["list", "watch"]
+{{ end -}}
+{{ if has "services" $.Values.kubeStateMetrics.collectors }}
+- apiGroups: [""]
+  resources:
+  - services
+  verbs: ["list", "watch"]
+{{ end -}}
+{{ if has "statefulsets" $.Values.kubeStateMetrics.collectors }}
+- apiGroups: ["apps"]
+  resources:
+  - statefulsets
+  verbs: ["list", "watch"]
+{{ end -}}
+{{ if has "storageclasses" $.Values.kubeStateMetrics.collectors }}
+- apiGroups: ["storage.k8s.io"]
+  resources:
+    - storageclasses
+  verbs: ["list", "watch"]
+{{ end -}}
+{{ if has "validatingwebhookconfigurations" $.Values.kubeStateMetrics.collectors }}
+- apiGroups: ["admissionregistration.k8s.io"]
+  resources:
+    - validatingwebhookconfigurations
+  verbs: ["list", "watch"]
+{{ end -}}
+{{ if has "volumeattachments" $.Values.kubeStateMetrics.collectors }}
+- apiGroups: ["storage.k8s.io"]
+  resources:
+    - volumeattachments
+  verbs: ["list", "watch"]
+{{ end -}}
+{{ if has "verticalpodautoscalers" $.Values.kubeStateMetrics.collectors }}
+- apiGroups: ["autoscaling.k8s.io"]
+  resources:
+    - verticalpodautoscalers
+  verbs: ["list", "watch"]
+{{ end -}}
+{{- end -}}

stable/yugaware/templates/kube-state-metrics/kube-state-metrics-clusterrolebinding.yaml

@@ -0,0 +1,18 @@
+{{- if and .Values.kubeStateMetrics.install (not .Values.kubeStateMetrics.customEndpoint) -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app: {{ .Release.Name }}-yugaware-ksm
+    chart: {{ template "yugaware.chart" . }}
+    release: {{ .Release.Name }}
+  name: {{ .Release.Name }}-ksm
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ .Release.Name }}-ksm
+subjects:
+- kind: ServiceAccount
+  name: {{ .Release.Name }}-ksm
+  namespace: {{ .Release.Namespace }}
+{{- end -}}

stable/yugaware/templates/kube-state-metrics/kube-state-metrics-deployment.yaml

@@ -0,0 +1,61 @@
+{{- /*
+  Adapted from https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-state-metrics (Apache-2.0 license)
+  */}}
+{{- if and .Values.kubeStateMetrics.install (not .Values.kubeStateMetrics.customEndpoint) -}}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Release.Name }}-ksm
+  labels:
+    app: {{ .Release.Name }}-yugaware-ksm
+    chart: {{ template "yugaware.chart" . }}
+    release: {{ .Release.Name }}
+spec:
+  selector:
+    matchLabels:
+      app: {{ .Release.Name }}-yugaware-ksm
+  replicas: {{ .Values.kubeStateMetrics.replicas }}
+  template:
+    metadata:
+      labels:
+        app: {{ .Release.Name }}-yugaware-ksm
+    spec:
+      serviceAccountName: {{ .Release.Name }}
+      {{- if .Values.kubeStateMetrics.securityContext.enabled }}
+      securityContext:
+        fsGroup: {{ .Values.kubeStateMetrics.securityContext.fsGroup }}
+        runAsGroup: {{ .Values.kubeStateMetrics.securityContext.runAsGroup }}
+        runAsUser: {{ .Values.kubeStateMetrics.securityContext.runAsUser }}
+      {{- end }}
+      containers:
+      - name: kube-state-metrics-yugaware
+        args:
+        {{-  if .Values.kubeStateMetrics.extraArgs  }}
+        {{- range .Values.kubeStateMetrics.extraArgs  }}
+        - {{ . }}
+        {{- end  }}
+        {{-  end  }}
+        - --port=8080
+        - --resources={{ .Values.kubeStateMetrics.collectors | join "," }}
+        imagePullPolicy: {{ .Values.kubeStateMetrics.image.pullPolicy }}
+        image: "{{ .Values.kubeStateMetrics.image.repository }}:{{ .Values.kubeStateMetrics.image.tag }}"
+        ports:
+        - containerPort: 8080
+          name: "http"
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8080
+          initialDelaySeconds: 5
+          timeoutSeconds: 5
+        readinessProbe:
+          httpGet:
+            path: /
+            port: 8080
+          initialDelaySeconds: 5
+          timeoutSeconds: 5
+        {{- if .Values.kubeStateMetrics.resources }}
+        resources:
+{{ toYaml .Values.kubeStateMetrics.resources | indent 10 }}
+{{- end }}
+{{- end -}}

stable/yugaware/templates/kube-state-metrics/kube-state-metrics-service-account.yaml

@@ -0,0 +1,10 @@
+{{- if and .Values.kubeStateMetrics.install (not .Values.kubeStateMetrics.customEndpoint) -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app: {{ .Release.Name }}-yugaware-ksm
+    chart: {{ template "yugaware.chart" . }}
+    release: {{ .Release.Name }}
+  name: {{ .Release.Name }}-ksm
+{{- end -}}

stable/yugaware/templates/kube-state-metrics/kube-state-metrics-service.yaml

@@ -0,0 +1,19 @@
+{{- if and .Values.kubeStateMetrics.install (not .Values.kubeStateMetrics.customEndpoint) -}}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Release.Name }}-ksm
+  labels:
+    app: {{ .Release.Name }}-yugaware-ksm
+    chart: {{ template "yugaware.chart" . }}
+    release: {{ .Release.Name }}
+spec:
+  type: "ClusterIP"
+  ports:
+  - name: "http"
+    protocol: TCP
+    port: 8080
+    targetPort: 8080
+  selector:
+    app: {{ .Release.Name }}-yugaware-ksm
+{{- end -}}

stable/yugaware/values.yaml

@@ -176,3 +176,36 @@ additionalAppConf:
 ## to modify this unless you are using helm template command i.e. GKE
 ## app's deployer image against a Kubernetes cluster >= 1.21.
 # pdbPolicyVersionOverride: "v1beta1"
+
+## kube-state-metrics support
+## Custom endpoint has higher priority than installation. The endpoint value defaults to kube-state-metrics.kube-system.svc.<domainName>:8080.
+kubeStateMetrics:
+  ## To provide custom kube-state-metric endpoint
+  # customEndpoint: "kube-state-metrics.svc.cluster.local:8080"
+  ## To enable the kube-state-metric installation along with platform
+  install: false
+  image:
+    repository: k8s.gcr.io/kube-state-metrics/kube-state-metrics
+    tag: v2.4.1
+    pullPolicy: IfNotPresent
+  replicas: 1
+  securityContext:
+    enabled: true
+    runAsGroup: 65534
+    runAsUser: 65534
+    fsGroup: 65534
+  resources:
+    limits:
+      cpu: 100m
+      memory: 64Mi
+    requests:
+      cpu: 10m
+      memory: 32Mi
+  ## List of additional cli arguments to configure kube-state-metrics
+  ## for example: --enable-gzip-encoding, --log-file, etc.
+  ## all the possible args can be found here: https://github.com/kubernetes/kube-state-metrics/blob/master/docs/cli-arguments.md
+  extraArgs: []
+  ## As we know platform need only pods metrics from kube-state-metrics
+  ## So we just enable the collector for pods.
+  collectors:
+    - pods