Skip to content

Commit

Permalink
Added Mimir / Loki Rules Sync Support
Browse files Browse the repository at this point in the history
Resolves #564
  • Loading branch information
bentonam committed Jun 10, 2024
1 parent eaee988 commit 4d3a91c
Show file tree
Hide file tree
Showing 44 changed files with 55,873 additions and 4 deletions.
7 changes: 5 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ METRICS_CONFIG_FILES = $(subst values.yaml,metrics.alloy,$(INPUT_FILES))
EVENTS_CONFIG_FILES = $(subst values.yaml,events.alloy,$(INPUT_FILES))
LOGS_CONFIG_FILES = $(subst values.yaml,logs.alloy,$(INPUT_FILES))
PROFILES_CONFIG_FILES = $(subst values.yaml,profiles.alloy,$(INPUT_FILES))
RULES_CONFIG_FILES = $(subst values.yaml,rules.alloy,$(INPUT_FILES))

CT_CONFIGFILE ?= .github/configs/ct.yaml
LINT_CONFIGFILE ?= .github/configs/lintconf.yaml
Expand Down Expand Up @@ -39,7 +40,7 @@ lint-chart:
ct lint --debug --config "$(CT_CONFIGFILE)" --lint-conf "$(LINT_CONFIGFILE)" --check-version-increment=false

lint-config lint-configs lint-alloy:
@./scripts/lint-alloy.sh $(METRICS_CONFIG_FILES) $(EVENTS_CONFIG_FILES) $(LOGS_CONFIG_FILES) --public-preview $(PROFILES_CONFIG_FILES) || true
@./scripts/lint-alloy.sh $(METRICS_CONFIG_FILES) $(EVENTS_CONFIG_FILES) $(LOGS_CONFIG_FILES) $(RULES_CONFIG_FILES) --public-preview $(PROFILES_CONFIG_FILES) || true

# Shell Linting
lint-sh lint-shell:
Expand Down Expand Up @@ -98,7 +99,9 @@ test: scripts/test-runner.sh lint-chart lint-config
%/profiles.alloy: %/output.yaml
yq -r "select(.metadata.name==\"k8smon-alloy-profiles\") | .data[\"config.alloy\"] | select( . != null )" $< > $@

%/rules.alloy: %/output.yaml
yq -r "select(.metadata.name==\"k8smon-alloy-rules\") | .data[\"config.alloy\"] | select( . != null )" $< > $@

generate-example-outputs: $(OUTPUT_FILES) $(METRICS_CONFIG_FILES) $(EVENTS_CONFIG_FILES) $(LOGS_CONFIG_FILES) $(PROFILES_CONFIG_FILES)
generate-example-outputs: $(OUTPUT_FILES) $(METRICS_CONFIG_FILES) $(EVENTS_CONFIG_FILES) $(LOGS_CONFIG_FILES) $(PROFILES_CONFIG_FILES) $(RULES_CONFIG_FILES)

regenerate-example-outputs: clean generate-example-outputs
7 changes: 5 additions & 2 deletions charts/k8s-monitoring/Chart.lock
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@ dependencies:
- name: alloy
repository: https://grafana.github.io/helm-charts
version: 0.3.2
- name: alloy
repository: https://grafana.github.io/helm-charts
version: 0.3.2
- name: kube-state-metrics
repository: https://prometheus-community.github.io/helm-charts
version: 5.20.0
Expand All @@ -26,5 +29,5 @@ dependencies:
- name: opencost
repository: https://opencost.github.io/opencost-helm-chart
version: 1.35.0
digest: sha256:9b840af4845bec9c212d0cbf3dc7666f15eba4527a603f695c85f8a4ee2c0650
generated: "2024-06-07T21:51:33.506459-05:00"
digest: sha256:5cdbcb273ce1c65625de0b58465503498c18980df88de412bd4895451b3937bf
generated: "2024-06-10T11:02:51.469871-04:00"
5 changes: 5 additions & 0 deletions charts/k8s-monitoring/Chart.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,11 @@ dependencies:
version: 0.3.2
repository: https://grafana.github.io/helm-charts
condition: profiles.enabled
- alias: alloy-rules
name: alloy
version: 0.3.2
repository: https://grafana.github.io/helm-charts
condition: rules.enabled
- name: kube-state-metrics
version: 5.20.0
repository: https://prometheus-community.github.io/helm-charts
Expand Down
37 changes: 37 additions & 0 deletions charts/k8s-monitoring/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@ The Prometheus and Loki services may be hosted on the same cluster, or remotely
| https://grafana.github.io/helm-charts | alloy-events(alloy) | 0.3.2 |
| https://grafana.github.io/helm-charts | alloy-logs(alloy) | 0.3.2 |
| https://grafana.github.io/helm-charts | alloy-profiles(alloy) | 0.3.2 |
| https://grafana.github.io/helm-charts | alloy-rules(alloy) | 0.3.2 |
| https://opencost.github.io/opencost-helm-chart | opencost | 1.35.0 |
| https://prometheus-community.github.io/helm-charts | kube-state-metrics | 5.20.0 |
| https://prometheus-community.github.io/helm-charts | prometheus-node-exporter | 4.34.0 |
Expand Down Expand Up @@ -702,6 +703,24 @@ The Prometheus and Loki services may be hosted on the same cluster, or remotely
| receivers.zipkin.enabled | bool | `false` | Receive Zipkin traces |
| receivers.zipkin.port | int | `9411` | Which port to use for the Zipkin receiver. This port needs to be opened in the alloy section below. |

### Rules

| Key | Type | Default | Description |
|-----|------|---------|-------------|
| rules.enabled | bool | `false` | Whether or not to enable the rules synchronization |

### Rules (Loki)

| Key | Type | Default | Description |
|-----|------|---------|-------------|
| rules.loki.enabled | bool | `true` | Whether or not to enable the Mimir rules synchronization |

### Rules (Mimir)

| Key | Type | Default | Description |
|-----|------|---------|-------------|
| rules.mimir.enabled | bool | `true` | Whether or not to enable the Mimir rules synchronization |

### Test Job

| Key | Type | Default | Description |
Expand All @@ -727,6 +746,24 @@ The Prometheus and Loki services may be hosted on the same cluster, or remotely
| traces.receiver.filters | object | `{"span":[],"spanevent":[]}` | Apply a filter to traces received via the OTLP or OTLP HTTP receivers. ([docs](https://grafana.com/docs/alloy/latest/reference/components/otelcol.processor.filter/)) |
| traces.receiver.transforms | object | `{"resource":[],"span":[],"spanevent":[]}` | Apply a transformation to traces received via the OTLP or OTLP HTTP receivers. ([docs](https://grafana.com/docs/alloy/latest/reference/components/otelcol.processor.transform/)) |

### Other Values

| Key | Type | Default | Description |
|-----|------|---------|-------------|
| rules.loki.namespace.label_expressions | list | `[]` | Label expressions for Namespace resources. |
| rules.loki.namespace.label_selectors | object | `{}` | Label selector for Namespace resources. |
| rules.loki.prefix | string | alloy | Prefix to be added to the rule namespace, used to differentiate multiple Alloy deployments added. |
| rules.loki.rule.label_expressions | list | `[]` | Label expressions for PrometheusRule resources. Example: ```alloy - key: team operator: In values: ["ops"] ``` |
| rules.loki.rule.label_selectors | object | `{"rule_type":"loki"}` | Label selectors for PrometheusRule resources as key/pair values. Example: ```alloy label_selectors: rule_type: loki sync: "true" loki: "true" ``` |
| rules.loki.sync_interval | string | 5m | Amount of time between reconciliations with Mimir. |
| rules.mimir.namespace.label_expressions | list | `[]` | Label expressions for Namespace resources. |
| rules.mimir.namespace.label_selectors | object | `{}` | Label selector for Namespace resources. |
| rules.mimir.prefix | string | alloy | Prefix to be added to the rule namespace, used to differentiate multiple Alloy deployments added. |
| rules.mimir.prometheus_http_prefix | string | /api/prom | Path prefix for Mimir’s Prometheus endpoint (gem-path-prefix). |
| rules.mimir.rule.label_expressions | list | `[]` | Label expressions for PrometheusRule resources. Example: ```alloy - key: team operator: In values: ["ops"] ``` |
| rules.mimir.rule.label_selectors | object | `{"rule_type":"mimir"}` | Label selectors for PrometheusRule resources as key/pair values. Example: ```alloy label_selectors: rule_type: mimir sync: "true" mimir: "true" ``` |
| rules.mimir.sync_interval | string | 5m | Amount of time between reconciliations with Mimir. |

## Customizing the configuration

There are several options for customizing the configuration generated by this chart. This can be used to add extra
Expand Down
8 changes: 8 additions & 0 deletions charts/k8s-monitoring/templates/_configs.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -144,3 +144,11 @@
{{- include "alloy.config.profilesService" . }}
{{- include "alloy.config.logging" (index .Values "alloy-profiles").logging }}
{{- end -}}

{{/* Grafana Alloy for Rules config */}}
{{- define "alloyRulesConfig" -}}
{{- include "alloy.config.rulesMimir" . }}
{{- include "alloy.config.rulesLoki" . }}

{{- include "alloy.config.logging" (index .Values "alloy-rules").logging }}
{{- end -}}
10 changes: 10 additions & 0 deletions charts/k8s-monitoring/templates/alloy-rules-config.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{{- if .Values.rules.enabled }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "alloy.fullname" (index .Subcharts "alloy-rules") }}
namespace: {{ .Release.Namespace }}
data:
config.alloy: |-
{{- include "alloyRulesConfig" . | trim | nindent 4 }}
{{- end }}
74 changes: 74 additions & 0 deletions charts/k8s-monitoring/templates/alloy_config/_rules_loki.alloy.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
{{ define "alloy.config.rulesLoki" }}
{{- if .Values.rules.loki.enabled }}
// Logs Service
remote.kubernetes.secret "logs_service" {
name = {{ include "kubernetes_monitoring.logs_service.secret.name" . | quote}}
namespace = {{ .Values.externalServices.loki.secret.namespace | default .Release.Namespace | quote }}
}
// Rules
loki.rules.kubernetes "rules_service" {
address = nonsensitive(remote.kubernetes.secret.logs_service.data[{{ .Values.externalServices.loki.hostKey | quote }}])
{{- if or (and (eq .Values.externalServices.loki.secret.create true) (.Values.externalServices.loki.tenantId) (eq .Values.externalServices.loki.secret.create false) ) }}
headers = nonsensitive(coalesce(remote.kubernetes.secret.logs_service.data[{{ .Values.externalServices.loki.tenantIdKey | quote }}], ""))
{{- end }}
sync_interval = {{ .Values.rules.loki.sync_interval | quote }}
loki_namespace_prefix = {{ .Values.rules.loki.prefix | quote }}
{{- if .Values.externalServices.loki.proxyURL }}
proxy_url = {{ .Values.externalServices.loki.proxyURL | quote }}
{{- end }}
{{ if eq .Values.externalServices.loki.authMode "basic" }}
basic_auth {
username = nonsensitive(remote.kubernetes.secret.logs_service.data[{{ .Values.externalServices.loki.basicAuth.usernameKey | quote }}])
password = remote.kubernetes.secret.logs_service.data[{{ .Values.externalServices.loki.basicAuth.passwordKey | quote }}]
}
{{- end }}
rule_namespace_selector {
{{- if .Values.rules.loki.namespace.label_selectors }}
match_labels = {
{{- range $key, $value := .Values.rules.loki.namespace.label_selectors }}
{{ $key }} = "{{ $value }}",
{{- end }}
}
{{- end }}

{{- if .Values.rules.loki.namespace.label_expressions }}
{{- range $expr := .Values.rules.loki.namespace.label_expressions }}
match_expression {
key = "{{ $expr.key }}"
operator = "{{ $expr.operator }}"
values = [
{{- range $index, $value := $expr.values }}
{{- if $index }}, {{ end }}"{{ $value }}"
{{- end }}
]
}
{{- end }}
{{- end }}
}

rule_selector {
{{- if .Values.rules.loki.rule.label_selectors }}
match_labels = {
{{- range $key, $value := .Values.rules.loki.rule.label_selectors }}
{{ $key }} = "{{ $value }}",
{{- end }}
}
{{- end }}

{{- if .Values.rules.loki.rule.label_expressions }}
{{- range $expr := .Values.rules.loki.rule.label_expressions }}
match_expression {
key = "{{ $expr.key }}"
operator = "{{ $expr.operator }}"
values = [
{{- range $index, $value := $expr.values }}
{{- if $index }}, {{ end }}"{{ $value }}"
{{- end }}
]
}
{{- end }}
{{- end }}
}
}
{{- end }}
{{- end }}
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
{{ define "alloy.config.rulesMimir" }}
{{- if .Values.rules.mimir.enabled }}
// Metrics Service
remote.kubernetes.secret "metrics_service" {
name = {{ include "kubernetes_monitoring.metrics_service.secret.name" . | quote }}
namespace = {{ .Values.externalServices.prometheus.secret.namespace | default .Release.Namespace | quote }}
}
// Rules
mimir.rules.kubernetes "rules_service" {
address = nonsensitive(remote.kubernetes.secret.metrics_service.data[{{ .Values.externalServices.prometheus.hostKey | quote }}])
{{- if or (and (eq .Values.externalServices.prometheus.secret.create true) (.Values.externalServices.prometheus.tenantId) (eq .Values.externalServices.prometheus.secret.create false) ) }}
headers = nonsensitive(coalesce(remote.kubernetes.secret.metrics_service.data[{{ .Values.externalServices.prometheus.tenantIdKey | quote }}], ""))
{{- end }}
sync_interval = {{ .Values.rules.mimir.sync_interval | quote }}
prometheus_http_prefix = {{ .Values.rules.mimir.prometheus_http_prefix | quote }}
mimir_namespace_prefix = {{ .Values.rules.mimir.prefix | quote }}
{{- if .Values.externalServices.prometheus.proxyURL }}
proxy_url = {{ .Values.externalServices.prometheus.proxyURL | quote }}
{{- end }}
{{ if eq .Values.externalServices.prometheus.authMode "basic" }}
basic_auth {
username = nonsensitive(remote.kubernetes.secret.metrics_service.data[{{ .Values.externalServices.prometheus.basicAuth.usernameKey | quote }}])
password = remote.kubernetes.secret.metrics_service.data[{{ .Values.externalServices.prometheus.basicAuth.passwordKey | quote }}]
}
{{- end }}
rule_namespace_selector {
{{- if .Values.rules.mimir.namespace.label_selectors }}
match_labels = {
{{- range $key, $value := .Values.rules.mimir.namespace.label_selectors }}
{{ $key }} = "{{ $value }}",
{{- end }}
}
{{- end }}

{{- if .Values.rules.mimir.namespace.label_expressions }}
{{- range $expr := .Values.rules.mimir.namespace.label_expressions }}
match_expression {
key = "{{ $expr.key }}"
operator = "{{ $expr.operator }}"
values = [
{{- range $index, $value := $expr.values }}
{{- if $index }}, {{ end }}"{{ $value }}"
{{- end }}
]
}
{{- end }}
{{- end }}
}

rule_selector {
{{- if .Values.rules.mimir.rule.label_selectors }}
match_labels = {
{{- range $key, $value := .Values.rules.mimir.rule.label_selectors }}
{{ $key }} = "{{ $value }}",
{{- end }}
}
{{- end }}

{{- if .Values.rules.mimir.rule.label_expressions }}
{{- range $expr := .Values.rules.mimir.rule.label_expressions }}
match_expression {
key = "{{ $expr.key }}"
operator = "{{ $expr.operator }}"
values = [
{{- range $index, $value := $expr.values }}
{{- if $index }}, {{ end }}"{{ $value }}"
{{- end }}
]
}
{{- end }}
{{- end }}
}
}
{{- end }}
{{- end }}
Loading

0 comments on commit 4d3a91c

Please sign in to comment.