From e1695629b27b2d3509031de47fe78cf98cf30db8 Mon Sep 17 00:00:00 2001 From: Aaron Benton Date: Mon, 10 Jun 2024 13:34:08 -0400 Subject: [PATCH] Added Mimir / Loki Rules Sync Support Resolves #564 --- Makefile | 7 +- charts/k8s-monitoring/Chart.lock | 7 +- charts/k8s-monitoring/Chart.yaml | 5 + charts/k8s-monitoring/README.md | 32 + charts/k8s-monitoring/charts/alloy-0.3.2.tgz | Bin 0 -> 20268 bytes charts/k8s-monitoring/templates/_configs.tpl | 8 + .../templates/alloy-rules-config.yaml | 10 + .../alloy_config/_rules_loki.alloy.txt | 74 + .../alloy_config/_rules_mimir.alloy.txt | 75 + charts/k8s-monitoring/values.schema.json | 168 + charts/k8s-monitoring/values.yaml | 122 + .../alloy-autoscaling-and-storage/rules.alloy | 0 examples/control-plane-metrics/rules.alloy | 0 examples/custom-config/rules.alloy | 0 examples/custom-metrics-tuning/rules.alloy | 0 examples/custom-pricing/rules.alloy | 0 .../rules.alloy | 0 examples/default-values/rules.alloy | 0 examples/eks-fargate/rules.alloy | 0 examples/extra-rules/rules.alloy | 0 examples/gke-autopilot/rules.alloy | 0 examples/ibm-cloud/rules.alloy | 0 examples/logs-journal/rules.alloy | 0 examples/logs-only/rules.alloy | 0 .../rules.alloy | 0 examples/metric-module-imports/rules.alloy | 0 examples/metrics-only/rules.alloy | 0 examples/openshift-compatible/rules.alloy | 0 examples/otel-metrics-service/rules.alloy | 0 examples/pod-labels/rules.alloy | 0 examples/private-image-registry/rules.alloy | 0 examples/profiles-enabled/rules.alloy | 0 examples/proxies/rules.alloy | 0 examples/rules-sync/events.alloy | 48 + examples/rules-sync/logs.alloy | 155 + examples/rules-sync/metrics.alloy | 824 + examples/rules-sync/output.yaml | 54223 ++++++++++++++++ examples/rules-sync/profiles.alloy | 0 examples/rules-sync/rules.alloy | 53 + examples/rules-sync/values.yaml | 61 + examples/scrape-intervals/rules.alloy | 0 examples/service-integrations/rules.alloy | 0 examples/specific-namespace/rules.alloy | 0 examples/traces-enabled/rules.alloy | 0 examples/windows-exporter/rules.alloy | 0 45 files changed, 55868 insertions(+), 4 deletions(-) create mode 100644 charts/k8s-monitoring/charts/alloy-0.3.2.tgz create mode 100644 charts/k8s-monitoring/templates/alloy-rules-config.yaml create mode 100644 charts/k8s-monitoring/templates/alloy_config/_rules_loki.alloy.txt create mode 100644 charts/k8s-monitoring/templates/alloy_config/_rules_mimir.alloy.txt create mode 100644 examples/alloy-autoscaling-and-storage/rules.alloy create mode 100644 examples/control-plane-metrics/rules.alloy create mode 100644 examples/custom-config/rules.alloy create mode 100644 examples/custom-metrics-tuning/rules.alloy create mode 100644 examples/custom-pricing/rules.alloy create mode 100644 examples/custom-prometheus-operator-rules/rules.alloy create mode 100644 examples/default-values/rules.alloy create mode 100644 examples/eks-fargate/rules.alloy create mode 100644 examples/extra-rules/rules.alloy create mode 100644 examples/gke-autopilot/rules.alloy create mode 100644 examples/ibm-cloud/rules.alloy create mode 100644 examples/logs-journal/rules.alloy create mode 100644 examples/logs-only/rules.alloy create mode 100644 examples/metric-module-imports-extra-config/rules.alloy create mode 100644 examples/metric-module-imports/rules.alloy create mode 100644 examples/metrics-only/rules.alloy create mode 100644 examples/openshift-compatible/rules.alloy create mode 100644 examples/otel-metrics-service/rules.alloy create mode 100644 examples/pod-labels/rules.alloy create mode 100644 examples/private-image-registry/rules.alloy create mode 100644 examples/profiles-enabled/rules.alloy create mode 100644 examples/proxies/rules.alloy create mode 100644 examples/rules-sync/events.alloy create mode 100644 examples/rules-sync/logs.alloy create mode 100644 examples/rules-sync/metrics.alloy create mode 100644 examples/rules-sync/output.yaml create mode 100644 examples/rules-sync/profiles.alloy create mode 100644 examples/rules-sync/rules.alloy create mode 100644 examples/rules-sync/values.yaml create mode 100644 examples/scrape-intervals/rules.alloy create mode 100644 examples/service-integrations/rules.alloy create mode 100644 examples/specific-namespace/rules.alloy create mode 100644 examples/traces-enabled/rules.alloy create mode 100644 examples/windows-exporter/rules.alloy diff --git a/Makefile b/Makefile index b22c137f8..5a547d381 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,7 @@ METRICS_CONFIG_FILES = $(subst values.yaml,metrics.alloy,$(INPUT_FILES)) EVENTS_CONFIG_FILES = $(subst values.yaml,events.alloy,$(INPUT_FILES)) LOGS_CONFIG_FILES = $(subst values.yaml,logs.alloy,$(INPUT_FILES)) PROFILES_CONFIG_FILES = $(subst values.yaml,profiles.alloy,$(INPUT_FILES)) +RULES_CONFIG_FILES = $(subst values.yaml,rules.alloy,$(INPUT_FILES)) CT_CONFIGFILE ?= .github/configs/ct.yaml LINT_CONFIGFILE ?= .github/configs/lintconf.yaml @@ -39,7 +40,7 @@ lint-chart: ct lint --debug --config "$(CT_CONFIGFILE)" --lint-conf "$(LINT_CONFIGFILE)" --check-version-increment=false lint-config lint-configs lint-alloy: - @./scripts/lint-alloy.sh $(METRICS_CONFIG_FILES) $(EVENTS_CONFIG_FILES) $(LOGS_CONFIG_FILES) --public-preview $(PROFILES_CONFIG_FILES) + @./scripts/lint-alloy.sh $(METRICS_CONFIG_FILES) $(EVENTS_CONFIG_FILES) $(LOGS_CONFIG_FILES) $(RULES_CONFIG_FILES) --public-preview $(PROFILES_CONFIG_FILES) # Shell Linting lint-sh lint-shell: @@ -98,7 +99,9 @@ test: scripts/test-runner.sh lint-chart lint-config %/profiles.alloy: %/output.yaml yq -r "select(.metadata.name==\"k8smon-alloy-profiles\") | .data[\"config.alloy\"] | select( . != null )" $< > $@ +%/rules.alloy: %/output.yaml + yq -r "select(.metadata.name==\"k8smon-alloy-rules\") | .data[\"config.alloy\"] | select( . != null )" $< > $@ -generate-example-outputs: $(OUTPUT_FILES) $(METRICS_CONFIG_FILES) $(EVENTS_CONFIG_FILES) $(LOGS_CONFIG_FILES) $(PROFILES_CONFIG_FILES) +generate-example-outputs: $(OUTPUT_FILES) $(METRICS_CONFIG_FILES) $(EVENTS_CONFIG_FILES) $(LOGS_CONFIG_FILES) $(PROFILES_CONFIG_FILES) $(RULES_CONFIG_FILES) regenerate-example-outputs: clean generate-example-outputs diff --git a/charts/k8s-monitoring/Chart.lock b/charts/k8s-monitoring/Chart.lock index f0a5da827..3e419964e 100644 --- a/charts/k8s-monitoring/Chart.lock +++ b/charts/k8s-monitoring/Chart.lock @@ -11,6 +11,9 @@ dependencies: - name: alloy repository: https://grafana.github.io/helm-charts version: 0.6.0 +- name: alloy + repository: https://grafana.github.io/helm-charts + version: 0.3.2 - name: kube-state-metrics repository: https://prometheus-community.github.io/helm-charts version: 5.25.1 @@ -29,5 +32,5 @@ dependencies: - name: kepler repository: https://sustainable-computing-io.github.io/kepler-helm-chart version: 0.5.9 -digest: sha256:78cc014e2a726be60e168fa7d09facff16ff7ed399948403ff2e692ae8d24d91 -generated: "2024-08-14T17:25:14.684591-05:00" +digest: sha256:02e225df81ff2034d306bd6950a033fdcba0285c8ac4f7be31bbd13a03389e6b +generated: "2024-08-16T11:13:53.900883-05:00" diff --git a/charts/k8s-monitoring/Chart.yaml b/charts/k8s-monitoring/Chart.yaml index c6850e816..fdeecb25c 100644 --- a/charts/k8s-monitoring/Chart.yaml +++ b/charts/k8s-monitoring/Chart.yaml @@ -32,6 +32,11 @@ dependencies: version: 0.6.0 repository: https://grafana.github.io/helm-charts condition: profiles.enabled + - alias: alloy-rules + name: alloy + version: 0.3.2 + repository: https://grafana.github.io/helm-charts + condition: rules.enabled - name: kube-state-metrics version: 5.25.1 repository: https://prometheus-community.github.io/helm-charts diff --git a/charts/k8s-monitoring/README.md b/charts/k8s-monitoring/README.md index 82c4eef98..43e50b8b9 100644 --- a/charts/k8s-monitoring/README.md +++ b/charts/k8s-monitoring/README.md @@ -140,6 +140,7 @@ The Prometheus and Loki services may be hosted on the same cluster, or remotely | https://grafana.github.io/helm-charts | alloy-events(alloy) | 0.6.0 | | https://grafana.github.io/helm-charts | alloy-logs(alloy) | 0.6.0 | | https://grafana.github.io/helm-charts | alloy-profiles(alloy) | 0.6.0 | +| https://grafana.github.io/helm-charts | alloy-rules(alloy) | 0.3.2 | | https://opencost.github.io/opencost-helm-chart | opencost | 1.41.0 | | https://prometheus-community.github.io/helm-charts | kube-state-metrics | 5.25.1 | | https://prometheus-community.github.io/helm-charts | prometheus-node-exporter | 4.38.0 | @@ -841,6 +842,24 @@ The Prometheus and Loki services may be hosted on the same cluster, or remotely | receivers.zipkin.port | int | `9411` | Which port to use for the Zipkin receiver. This port needs to be opened in the alloy section below. | | receivers.zipkin.tls | object | `{}` | [TLS settings](https://grafana.com/docs/alloy/latest/reference/components/otelcol.receiver.zipkin/#tls-block) to configure for the Zipkin receiver. | +### Rules + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| rules.enabled | bool | `false` | Whether or not to enable the rules synchronization | + +### Rules (Loki) + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| rules.loki.enabled | bool | `true` | Whether or not to enable the Mimir rules synchronization | + +### Rules (Mimir) + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| rules.mimir.enabled | bool | `true` | Whether or not to enable the Mimir rules synchronization | + ### Test Job | Key | Type | Default | Description | @@ -872,6 +891,19 @@ The Prometheus and Loki services may be hosted on the same cluster, or remotely | Key | Type | Default | Description | |-----|------|---------|-------------| | extraObjects | list | `[]` | Deploy additional manifest objects | +| rules.loki.namespace.label_expressions | list | `[]` | Label expressions for Namespace resources. | +| rules.loki.namespace.label_selectors | object | `{}` | Label selector for Namespace resources. | +| rules.loki.prefix | string | alloy | Prefix to be added to the rule namespace, used to differentiate multiple Alloy deployments added. | +| rules.loki.rule.label_expressions | list | `[]` | Label expressions for PrometheusRule resources. Example: ```alloy - key: team operator: In values: ["ops"] ``` | +| rules.loki.rule.label_selectors | object | `{"rule_type":"loki"}` | Label selectors for PrometheusRule resources as key/pair values. Example: ```alloy label_selectors: rule_type: loki sync: "true" loki: "true" ``` | +| rules.loki.sync_interval | string | 5m | Amount of time between reconciliations with Mimir. | +| rules.mimir.namespace.label_expressions | list | `[]` | Label expressions for Namespace resources. | +| rules.mimir.namespace.label_selectors | object | `{}` | Label selector for Namespace resources. | +| rules.mimir.prefix | string | alloy | Prefix to be added to the rule namespace, used to differentiate multiple Alloy deployments added. | +| rules.mimir.prometheus_http_prefix | string | /api/prom | Path prefix for Mimir’s Prometheus endpoint (gem-path-prefix). | +| rules.mimir.rule.label_expressions | list | `[]` | Label expressions for PrometheusRule resources. Example: ```alloy - key: team operator: In values: ["ops"] ``` | +| rules.mimir.rule.label_selectors | object | `{"rule_type":"mimir"}` | Label selectors for PrometheusRule resources as key/pair values. Example: ```alloy label_selectors: rule_type: mimir sync: "true" mimir: "true" ``` | +| rules.mimir.sync_interval | string | 5m | Amount of time between reconciliations with Mimir. | ## Customizing the configuration diff --git a/charts/k8s-monitoring/charts/alloy-0.3.2.tgz b/charts/k8s-monitoring/charts/alloy-0.3.2.tgz new file mode 100644 index 0000000000000000000000000000000000000000..cddb68d1970a6c1699cb83871d63e106517d45f3 GIT binary patch literal 20268 zcmV)pK%2iGiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0POvHciT3$Fb>bZho1sR>9ZX7F{z7hsm|%V$783h*Cei2?6hYl zt4|D(kc65fSOAoxY5IQl-@*n!f;UM{9H;GwU8}K3U}Iwg*f(r!;5cT>{^=ZZ87y%U z|K$@rgTY{MbZ`Lw4hDnzzk`E=!@mp3(C?lT8uoLW#!LdFbsxXL^6^_Bn>GMW6wt+OCu_wM93o%gq&bW zJcLr55dF?Wh?9&7Dj8pnQLpDAw6I_Y!5|oT5fLG$nTGQ%$5WhQbfPAsM?>{)E@dXh z{XWNc!HmjzKFI~)AxkAmWe~EYKhw4PdI~3)ru~S8qAys^Ln8WE2qK04{8j;d%Q#MF zBoK?4r;&wko?K???<|XHh!sYjgY14V@;>(2R ze~hQTy6?A$>5K?DMq-XfhezY#SI-6q`$N1RJsUh5jiwWv1Y1AlPn?+>nrhvUQj@!@`O zbU6I#=&R#r2mkE-nb+iFK6?ExaGVqIsT6?K>wk1~w0}@v|A$Az?fQR==LzzC^!)-yxd94^Fhah2kOr!D``+QJTlG=dEZY zWJ#7w5}|1x$DJ*zk0|3rwacd7lSZqycBADd=-Zf0u&T{6!X+gFO&Lct!83xgJdSBP zL(Eiz#`I!95t-7IL?}-q!qE*>2soKhA^GwK;WR=wAxo!p_KL&|M}!BpGJ$ITe2SJV zM|U`ts#8TvERC0lrNm%jEA63NXf#AdLOP`(no<%+0!?y>Xa_JkCy7#KbnTlNbh(a> zr(cgfq}x<~AOvG$XjDZ-wV=S!DWVdkghWEUnvkO16i6vp;$W+4gjf)Po|0gOViw}q zf}}+3p)ePcCAuxKhk_v31w?GKAZkj~z6Tu<$4C%~*c8oNRMq$)K|)TX2%wI7bERL5 z(VL$XAOLek!kkmNM4&;w1M(@24|=J{UC}{N#v%nTW09?Rs_Tu>ufKVol8<8x`o}qu zbHWuMnhJ^I7@fX4)6*GohfiyRURYBa{_bo2h1tUg=tg}DQP+r0s4kfyg^eM6Frm_YL!s51pwnG1f8XJ04q=fgsxS`s6d&r0~EfJ zp&OaSH=y<3slG$gB*|hd;U{N_S^WkB-PRcG>^L*>0w+YxMDmUbs8{wyuNcF^?&c&l zoltm8aV$*Fw2}oJs(|`;vQ#v!z?uXxvV0QL(9bwoP;%GXLyV)I);{BS z>Ei{av4Yl9Bq`8%p|BXY7c7v!#AfI423L8zp$mG~)zEG*vywIGYd=DCr904xB6&YPK&~&dsV3 zdmuF=L%r{=uP?89MUxu*7^&Aqo9J&?5R|C_)T3v5Plcfe$b_6K`d)%EGDiKP_H&qj z0tu2v3LQK2FYrW7R+cUk#p-jR1ecIl!Z^%HoFSTIoGpsk0uW)wsP^={enr$`QS01D z58LWH$uT17f^wFs362(+Q;msa!2B$O)N`;HJ%FuhKTj7GQTY;QLa_;`bW4^3F~vhl z&cry-$>JN%k_y(E5SY&@{l0+(pg7S@=~6O+6P!qZkv4*2EQyc~dSOiaJWu&5YqF@$5!nVO?fx5Yd)R#$p-*#njJB_%%l7(+egqIT1?j@34~E z&7+2K#3wjxvtx|b`s!~dr{(;Cby|p}sx2c3U(k@8grQn~D?nmoiXyDb78+%h3K8hA zN~$y(E0q~)IvwK)iHkLF`3uyE>g_3TxTyL9YQoB}VAGUI*af%`z_OAH>w2yo6lz2i zgu-H|)4*8T%3^@JRdNO`MucNRcu&mmi-MNGrGO1^#Z9@RscUnBgUbTU(y9+*Tvums zE?I&lRp#Tetf#G;ikwq$wa-SBdP`IcTHWVx>S99p=FF$UerEH~GM|EPTINzSp=vFa z7M076ex^x6qorVJUsp?T=Ifee6W4fFE;2KA?ZidMFrLLA7>lg#&%9zu2Ar7k)JUfM!^dWMrK2#l&%nX}`r+*<7X89QPhDp!- zPDrA6E)HMP-o2%Dl~Nms?Q*phYv_8J5pA^=3(*;0&Ei~Bp|IG26xycNR-iFztwFmJ zVUn;^5V^C5c1m8k1D5QLvM!VnuD*llgAGonQlVsA?_lBOub$^a*EfWA-TrDks(m;$f{ zNi2X@ERI<&YgU!!O}R4GFAZI+TNaeql*l{AZ_9-Svxn!#k~bE?9LvIO1{fx06pgb~ z>m>@{g@)&-wi}^wd48rH;292yg>`v;wgQf-dvds1Qbms35lzoA^n0>#%^q(q#M${w6)#B%jctq(o; zBmydc4GyKROySC$Nm)d$h_WEGTc}vv4F@kka*J2sT(el&rp4r44MvuwE5^Y!%UH~2 zOLUcSf+IB&s*5z0n?cpJb!De+rHzbJ2HYsdLV%NQJ>*6m>*K80wUMjXm?do7;$2I& ztXfgZOf5&>qAo8>t%3xv^kjMIN90TExa=)@E@hONu{6(Uu| zw&89qb&NSdF}ovNi^^i9rUl2gELVFQX{dJvaM^r_Q-?&lxrC;#99CUvF3+-uX)e!9 zWB6U!*kRe&$%1gD?3`Y{M!BRh{TB=!!Xalu`<^(7$pWVmX6zoqC?h;1sZ`aBHf`iT ziL}qYd~Ng%hqs9UUFppgbfaA=oV>D=X z^(81HpR2dJ^^H4hkZtMn3P5A@Y@nSoFbU4sU1~Kmpu`KR^;{4Mlo(}05}wf1?0Bkr zJ}hVJ*bfW@TgfPv1z4lnD7oOgMw&)rbWnMj5l&%4kEM|qqoaX~-kC*3nAvRzoN>0G zkuv)xvi_^7v-9RX~n@^Txui}qHHTF z5+vp?!I)f+P^cVTgsI?0uBF*)1o zhR{2rm<4qojWLmRO*I~c@o1Fx^UK0=WD$Y=J!R||Gfd)JA`sCobQs z^VS<0`@`YrS z1T@No)>f7#I1?K19lLWuv5umIC@)i>u>7HoD?&Qt;v{CX{%9~7^alt1!KiOy6;$_y zFPSgr#J6qYjQTY7oy~lOY5$4APibGOc&M88+0@6WtqMrgF&m?FM$>oj?FSu{X(WOP z+9FYn2?GZ%7;&{D26_Okm!O*kWh{Ig!lt|uix4#8GSN(j*LVm`F=y_cJYcFmUmH~h zf;`nw-#)Vg_qBm=trcGgBk_#u&UC=nk$k?14hkTKQY$dVy@KUjhdFQv-v{T(g7Bqw z3s$tgpa_IBfmtAzjq6Sa6P6T26D-Odu2Su|R1G?d&ye6tGx?PsOVVg;ej*D%526sU z&q+i%31z*6rd}?HUm~G<`aLfRr|9-od(#b4uEwZ*VYkp)u`?kU&Tm`1hN@<*1!WXp zg8fz9^U5&#G8PtKG1vxt`-neA{I?E$vFl0nFTTHieig`f@&Vgecm8LvKdQz59}EVE z+xY*-cz#rZR=bQVM%{mI57_)$JoWYeRtt|nW)F7~XwCW`jt=VQe+~`@$J_P)7|*Z2 z_P_LADPxB$NEx=T!&ZvOJvWF2Bo#82h7+znhfPlIIj*xW`@j9>{ranK&eEVB%(g(c z;wLx@6yP56O|dV`(VOwt=xLG*`I7J%9O=2^R1y?uB{0x2olhAr-^@rVQA9(zyIVk` zQwuOC`UMTv+pB*zUr8*=qBITTJSw_i&l~knP!vz2+MgZ}w;lxKoO4iGnR;cdY(vyr zvMk71dLm6+fLESz*nqybWcjG%Rj7AUSADG$*YF!1{?|^&&%H+e%uj>=&q$WT`(5VC<48gJw_h2S^k0j62 z5FPEqpESA3r&Ib4_55Bd`z%h4*%lP*U7}xd9Ba0SGcBzG??<9xLwTuMP(2W6LPDGi zq7N!Mo)j(P^yx~>iQf6Pv29eKr;}w7FO%xia*7-82HyG9o_W(ijPG6;;Rx?&E%*lu zI4~E-2HOs@mXgI`w3d?fW3HBBo!>X1w&KK=Wdr8puQwmg632oD8c6WU916T}L~8Zr zJ(0^eP308z{*UngkLcB4=|(r7T&pCunR3ops?}j~AE zh`h5DMZGVe#9K*(6uoADdzEo`Z=H;&_kVk+_qO-3gmlMP|7}C)Kdh+^vYx=nLmjkG zt{rfPSI-sOgT=5T>z9Q}XF9|HhAf%D32h*rSjDHHe(Q=VzIH9C&H^q)D|1mJyKauL zBw?v>EOw!;ziPbaBu)Y`??Y$D=+|HCDyf-dMjx$z6(+{*g#j6$%_k$yvY_fBE%Jc$ z7XKUkBlu^prA&fTJR^}mSpv2NiR|`bgN*$zk;o@Pa+p^FMI@%Mv0+dCE4;K#qWc8V z%jXedxy*8XfQJOL0PT!-^i~L->0slX|IsgWe6u;+`%U>`58V+Ia;B{qefC60m>v7j z=u5K13HI6rga z)D6~Mlq%+S3wiYU0&~W}>KNwd`0JI$sNR%*I>*AU1asJ-XF=@NL4I9`Hd9oIw=b%2 z9a`AJ|I=11)|oG=C`vr5SB5q^OSOsJu_JU(%HwJeGgij-w+ASCEyGqTvT^`h&x@=b z&kbt*&W5+^T+GVRURSAe@Oz6v@GKat)tFupnR9&_1ZNp%8K+>|6gm4$Sc~4+>|CJc zds(eA)Ji};{USKUnND*+i3mCo?0Y=u%|b z)5)9es30b;RoGBv9Znaqd=eAiWL_arnPVYPThBOi=kPK%a#W>bpUQMJi*oYG~|sj`rpUuR9x zddeL=s*qV4eW4nAi8Hm7EQ~VgkA>m(z*H#A^*YY>d$tl?TdI8DhVQSxqLAU3h>$pd z&1c;kU2VG!`)~6!mF4|2)p~NO2*hW3*NH7cng#ApSq4 z*f(boEJEGgGf7DSrW?^RM(=&7_EcsuT2T`=25QK;UWg8`!8bG}BKVd_>(}es7-A@B zHxoL5Z+ZN``_sbz?IFjvu)H5Y4y@t-2S-PB{(p3IbgT9$8 z>7-z=uV<>X<}OF?%ZQZ9R!dRfx8M2>5c{#n1oqo+O)*k#%4&`Xzb~CcdkuhP7Cv1A z^I36Zhv%vJt50NTQi`_h($``$gVHGQeUra8$grG$?U*4>Uf)V_!WV=Gb15@psrWdG z_}2{s<|K+Z`t7&zO3S7WF{i%5!`GYdE>JfEuC49t=@f=fHF2%mj!87(bNjJ?LDHxS zafL6z0cewDnNMFAIm6wLKQpQ+M1&yX>wEYW*5Fpn5M3Qn^Cf^3*4066tK`UH`_q_~ zt~q(aXO1x+|@m4c9Zw@AzY@pG@pw0vXU=`7SDBVmxTh?t{>on3mKF5DjZNV0$6B(5lJN3B$< z4Teb$j4TOCbRDSm!mHGyiUY=DzECxxdSxrP)iSe??urbsbMVfRaCck_k=Gs`IPQx7NoY8g1mwwv<(&VT%U)0!spTHIpTbUCrT;?$>`$-%50Cci_djm+|Hpco z`ag`i|4FTY)j>w?R#BA+6){U6WjU<8w5euDbd>F|D>@^qf{Lcn0lZ9~TemJ;9JN#vJw{6t41<_YUp)*k`;@i2YqQwVizf|3J_GSy$GFS_| zh2-n1bd5+MVq3v&4#$;4btgy5^r>bFG&HH^C0siN+hE*tl~jI&Ep0a)p;p1{1YWDs z&`j^W)M_~t@7d8c1JTUGt(`~BRNGYrb4~pXr-743UKg$8m2YpcQRg0QU7nwn?Lf}? z?ly|D75Gj{`1Yzrfo{Fb7PaY2jgD)Rn?8i0aNO!SK&t}`fiMdV`#ajn((8a7}vXXvJ)%r&^^6wL^FN>jPwAk0geGQNvO zBMl1f-@s9APpM{Gxhh4Xl-?H(ZSHM^nym!8#ycroHT>^M=jx3-x_s@Ho)313eeN~# zXMR@l|MIGuj}idu`2Rt}|95epMc%|~9 zymq6vxMr!-Am7TI6*5O`rOmybRs7$%p5UVdz#9I)KRi0B^8e9bxZVGMoM!|6?@WUt zfq>&=SbdA0tuVzjMm>EO%g*X5JNB&LgrHPkc}7#XEN92L#0D{a=W$R^YS1m8oCdc7 z>Jtj6Hs5yjx>e&DXVFVYUZqq9>k7U!B_CDzIky#TFb)1R!t$dGwAIbEUi{^_f@`;V z)k5Y1H$Luuk-+r{1DP<|GFXiEH%wFWv~A6<l{ z!2&MaF^LxUkYMPJpgXKnl;QXeFNMAd)m(pSZi@rB#kHpj4kcWlp@zJpoWx7jBq#oq z$Z!tV04g-#0+5uDNT1h+#3XRJo`5hO`9nECuZVg}?bNdh~8=bF4lo46HrSC_0Ppu4;1 z9_oyv13lF@2d=5R_L|$>g0+9~);+UP#2W_Yrd|QrbAQPxClTsZh}c$`in26t=v*j{ z?c%1Ru{LWwNc0`P_rbx|S&xdjCyJ#gY20hvg66u2&5{bO_^z+^LB!h_yrs*BA@`tE zEhAAraAtR1TKz|v)?t?4Z%(TU_aVJ1|L2~~s)rvql^=$fLq~wVq{GAj+^y= zXK5nv0@K)><}A!_!1NjxU2#0J%{KvIuh7|cdmCh+MPuW>t_+rPhN`rYU8~=DVM@3+ z`9q2Fg)U)l{H_NZ7MH=>E;Me3w_bzdR_-E9x41RD2#Rq`#Ab0++$XLu1iCrjies+o zqrOAG^0jumxGGh7<(966-PErKS9RSts1kTwt+?VsZvfNzz10G4Q2upRp?i6w`pnT9B4EHeO& zI-qQJVXjJ;QZA%Pw}xKo`;w;$KRjcaI5lg>B1ARNz#^KsmE>bhfmA5;$m3O|@^R~96(`idBtvHreZp~n|Ywjf}bzr@{MW^ek5Obf;r`GG8 ztgoo;WQpx^{GrFUR=~s?Z3vnS1Gi+kXRKxz$FeE-hJO1EC@C|W`2>C8UMjOn^wf3} zaT={(zV*J;)5z;an%BL%371w@{na@nyMl(&4vKWAhoZt$J3zZYiB-E?Jdd?5+`%g0 zE^ky;G>UR-sY2g?6V!G{0`x_xn!-8j${_&ZY8ORc-KAU6s5_?LaL_DP(TO!;`qY7% zH!kz36e>?Xbn(_&@$!!L8r-(Rt$0-~bYFpqE1qfqDp$Z)pDs33ee8|!joNuKAzasY zIeY`eueC#h4a_b!z@u_+{$@>f?vA?1`0FN~za4^)AA;&@y^dF`vtls@8?KT^TbM2b z-tO!EfuFkl$0s;^=nddC_TRzbem(yCV0bjx+JBGn)RY<+{`O?e@CoN~&bW2w7gxpD zG|oC<;wu(^$UP39Qj=H6dxa+V1iey!qPsZ_=a5R`4nyIMAMg>* z=(n8ZSs@yI)a#XhIzWIl&Zou=JNjEKO^qr_(kNpzm7@73rXeX`8;0$?fvBSDuYSha zyJhX2n~D&$J{FZfj5yOT7KBe6+-5`;f5%jmKj3VB2Wew=O9S0am$=H5a6e@%irEZ^ zt|+yR;4PG^bVL1clF9j{Rrz5(YsV#FDU}S;lFm4u;uHrVOIBg1`uJd^O=v=S028}5 zZ`(+S-njUNj1wjR-Jd$~sK$Fx|NC83fFde%D97!yU@S< z_)fCO)MzY%#`}!3$pbD8vLLCP<1~s1|A>hXU_KNp9SR5(<|JW~tf6ikZ~3gAsAPz6idnJB$qHMHB@nyi3YmpP0z+KE3|%c@Ir5TGa`F? zsHc!o{{kLRrUtsa3-DnY@O9~0n%O2RwOK{dnMOi`Te;yyUm`u{{N8ozS(Du{y!QX*YkfK91pkq zKacW!lK9WsmDHc|^0!BG|2Le>wsV^~>hE=Orwpg4wN*w}bWpmuf=-umZlr60Rx7;X zKaFmza%`x@ioWODGkdwjXEN&1BJj@jN}pD0 zUbPa&eq0ej@~Ll?`}NOtr`+}!$nW&&kpEa~Ko6q^tdakNC~vKR9pe28}$NTCs}bcC3Hn_jr`` zKiVG*>g#`hxb^=(%Ja$2e|6aawf8U0MC&pUT3vblIs-CLerNVW{lcHNqsp5|@()9ge+Xi$rlO36zK_*bC|<0T zJMBmEKS{2HtqJj9kCy+zoWu#8rHqqL(8fCb|LAzXCjUo=qpkk`D9;mgiKQewg-t^} z1@y7LNlxR4rZbe`@D|UA2t0eGzX>ZsVou^1&0;puD;qBHSNurQ1%c=y=RHm%?+HrD zOy{e3s_&H~k>*AJxf`JGQ^;5WH5E!IBOJvvC4m>5UA?`Mj1%t(I%P@1QuM>=6^bYq zUNEDw5C7Nbd%@&iybu4kFXpqp`k(zN7HPi(G{NC*o}np?iSWJ*#9iin8BFl4_hldx z^*7^m=6(79_nx31FsCdR==|)t@Pdr9pGhdafJOxOb#cyq_JV~7Sw#At_XM4>bVphe z7Aamt;Jtc&a`y6hkVL_Z$s~&#g$0}#l5iq47iI_S^gQtT@UH(btWU51)9+3$zJ31U z`)`9JdJwRcXPy3kFsh&b7>x#7{r^!Ob6SnWZ031?MQ=}8s<%@a7d8(UB&=aMM{h#+ z>nx`c`RUD1FfnN=B+o-_ud)=)*`4=S#Zq*RA(+ddZ7F!`dB8~`7Uqc(>n5$aKp4%b zPzryDW7rW!Y-%bBG$HbikW{f4f&~$pWq9_poMVY-lHpJ)^}x>s;l;UBggF6#B>Jxc zpkxIKnkzBxS!XmlxjZils>&&oDC0zsR0f{+IyG2Co?kqkHwgBF5qdfrj1K(4p}#-a zbv~EHo;-Pi{+7=a0~DU;qt}K?41KGkNYDvbTZ>_!{zFgyhfg_{lI`jyhGi3@(PmA2 zL*JqD5?wV$YRqsN5`lQ0rs}0)#WY3dLQ;kjBIana)R78soRJxNz-WSZ*_>;1;3~1E99{9uIJvwp{MJS;Hj^S%s(U7Hzf~RxLwG+2Kg@z^B4*bDa-K~8? zu*|t`=scC2MR`aNt_{w2Bu*4cTn6ZRPCCCLs;o~XaqlpX1X9*AmUKd6qV)hZn~XWC z&_+|v5_oxI!mE66I@|zE57{yhqFbLK*oaZLF662M5EWdi?M3=y14||BvxJ>0GLw_n&X{;Q*XLFq2w2B`2iG zED&=_;s{vb1V=N{SLJ=DxKGn5;|bQDt^c2p%b0vUjj{B_FFEF97yaj(!qo?&b4>%L zun%Q7fqs%@&bEbOvTIr>EBLzn+EzTzVL6J|=*(7%8^5U#SLsTUp;FWYo>wd3oZc^^ zNwu8OD;2BgWDk{;wc54GVvMbV{Zc8|4hHrHEXdWP#;Vdjw0&y6sD24S zXQ1VQHcsx)X#oIfM-3Gy!$Lp;Wy#Qu%;K9;orf-HhXnjonwC+$+Og7`!6p4A&w>1> z`U9>aQIJSO0RKum;9HnDWmi)gE3j%l(bNGb8JZDUz~d}whnLpJ&H%hH77KYt1$4Or zIu|5BuZ7KOWqL~mf@$Wqq-WwQqQ+jgP^ZXqC?Sv_5S3dnq!)N15KdXTOjxdX2c8im zB$i-`&Pkjp`<1grO@pqszW7EB)q8CozehK3eo}3|#F;>r@7|InnBmyzBjZS4J&CYD zBwbL>QZT+3m{Y(vKr8GVPJn}iq(37mQA%^<6!URL`-@>PI_N(k=|T;T4u|#^LqBAk zxIKV`*ZQvNY9F4%Rb!azQy^;Ys()|v&Lhv$n@#0oiEJ3tKF`yBz5VCuqNDj%(zy62 z0&8AV+{PBv!Rz3x_6D@`RB`-w856|j1zrJC@64S<5wG%uK>+^&YfxW_Mo<^m3s8sD zA;s)WY16FsFTOrk6__?Di@W8TZA1Lmd&Ngo!^6@3!J%ptUU#=xhf?e5(H2Jz*jv7hprp?S|keGQoiEy zz0xk7Ga)s5wGtgl7=bfGp^~*>UD_B_?A3a0vqa~~*4LrKs4A_kXsD{wno+{hsz_i}q?{sIQ8ANGsAL{(C_dB(6?v%_!q|>6;9JF@c%>lBP7t6I9eiF;r$# zRD=)c-WT3W#VtUn1n8ww)FuS9UN%KN@-8Esf)6w9?V(wf~oI8Gowg0eOd$EGBzE%O5$F#6Ou69HAF% zwp-Pyp%N7+X3#n7=bbPnE8Hlj3WH17JE|cH`29`U46yU0F*H1m3d8jx4Xl>xGR%c! zNvlZuenB`_YMX)tlF6u4^_kxaxT&<(RRo+%ri{x_4URI97zULAl#q&8P)d*S`n9&D zZX=fbo6CyEzLt7P-eB@5>bU8^rWcw)XXaD z-@jF$+3uxOe!H_I|rRQ?B=Z_1btq{#=FJ(r)9P z=w@Rl_LBydn|l@D;3tEUfZ`BD5}wdhvv>gFtF{W^yJ4f1+@qEC{1U(GX+MQ&$sPhW z-Wp}`QKj)3pt^Y%M?LH5v~>GmPOQ_)5o&8N>`rT1jNATTU`H;pYXCM@-Ejd26j8#6 zio|Xc$Bf;=)(+B}PYJ$N6FH=CN*3V>%cbq%hJ*P!`}m>EvwIBW>+B(iG0z@$5EXVC z4B?{;;2NH=Ch=+&*Atx<6`fwbcGHC;9C9YKQ-hP3EO07;7w;jAvJ#jfzmZTRGJey z`cL48uU~8Qs(7!t)~-d;|2QY0Ef=O*soU%On_%a8s;s`)T;&wypwnST!j)G#?HNa8 zEfqW!Xvet`Wv4;$#(i=vx{mYJcorn9?02-mJat_rRc3rrgs(eZB)HZpPZ5q;Iuq80 zs@c+@Pb$UhwB~(sTAwo^FZ5Wps#xXJxnw#+w$hRdFNK)~rIiBeBhQUtYTR*(MZ(0k zdo_u%v%49zf_`~^_8~Z_YQ=e{2H+|^8q}^APgz8+Owe@8XjeYFDsk29tclkGIELm(APjkXuJKyfv2gx!Pv)S@0 z;{->iEESStnzlH{uT4p$iz2t|eH+!Akj{V<9oI&+`VQVUcLUa%*5&?veIwZoc567| z9OCYwozjBa0T0~j30}LMu{keIRGhuu81RHjpiDzsBm!2`fZci(W1aY|g^XzUur;e8 zZsOy1Wl+=W3Su8%^GI}WLON{7MiHF8I@5`xoI_*)$sPeKe0wdMh4z~$;}~{yyjrgG zDhtAF8CLi8)kmb`+qI2r)GlE78iiTRCO9?@pe&DLy*1b3f6RO-17mQWcTO zuhas!-IE3@;9D0=QO+y^Iz?1guA8t9J?#$>EFopYgsKr@K?HhAf*Fcgh+_+q60wK0 zcU89~_D~Q6yDqL7T=wy>uKHrqbUH=NFxOVND!n6WzS0M9gO+|z7cDh>ZOZZa^n%Gt z9ctX`z*3LB)?Mo`H=RpelW5iKDg3FOh;8JDUF~?S1pTaN$kukzu6t9v2O3E{Gh+rf zsA+ZZ)GT#e1cH@eKF~@_g7)TotrT719<Ya?7riz8rv}!#&Y~0P4x>6>YGOzx2a$07JD@R*HYfCBALEL5zpO#=D69jdBxo3mW zjPY~9QnIHfJ?a4m!zntyR6-bLq1!_D7FOxnHpj5tkRN+byKpqLmHB>J{R7YseQUfv zWn#`|*dPPT|8`9I9Z>X|hP^Q#v|_!_hNPr|l=T){Lzp@huUE=q+^6e{S5A zLgS*+O{JBl-g#=rNAJNII_Yv{#m>wUEv9>EX;&{dDk~bYAtq!e_huOtrTHk^;V4*bf<-RTW_jSVe5|aZ!(Csrxq8RfTOK} z72>3=cyXk@WluQdSn2Yn1O(TlOllQIp>tWJTrC1ZMYT>C=uxHRbRN`NO zX&sWBBy6F9U^LBT&Wm9Bz+z(;=%NspH}wcUz{y7ML2NXgrDb@&f#4w<&j-=XG&S*o zl}PgdT{Faq#6~-Ua3fao35dhl>?RXmu_z zVO2#iRT*EUsBt&P(sSagj7nsM+0%$}63TdqU?e!pXLCdoMFJ9`ghhEw#BP8t7#wHC z(i3LiM9!5wfbjxnzTWXJ8&B{O;aC*g0)-Z1_9b(e6WFWO+e0D`=Mb#xM0nZDAXz49 zVm4?*;CVmbm@0`3DCq;MH*UR~J#nF$~2U6HadOb7Or5 zk=Q6+;5eeyLid~)4TusjySW7lPJQQ~u)bFKO%V|M)0>|H&-*L-^2;AjUR|7DeEa2> zWAqxZXrda~(8xJnfVQD;sX4hkS2zIK61?>OiYA=k+ad}L3__G=Gmayo(;F4Yp_#I? zaTJvSVfKuF;CT+7A0D{e7zqe1t_}sfs*HsVVgdcedTLl^*0h>1nq|NkL`4y7H>vcd zh*=_2mZuRiTgdKF*-I7`L$msnY}v0`%=jJSw=u(!Usy%`j79!p*na{z>;|3(+PRJy z5#@GFu|YFH|Bc?E|3*iLhx>9~34Zsh z)9cH?9!COR1W8E)qmDEJ?2Cm}zN#!NPLvWIN*$vKnf#5>tc??0O(2$fs<_#EbfV%B zl>&WteSHZ%s-G>B=G@MW3D&^WvC*ZKV|$5bBNnigH+BKRnJ~L zY-9aByQ5(<0Mzz2T-sF5B~9t9pitw$(hZukI1=rl(@spCIc?*4=T;SJgGGf&xuBzJ z_7FyItd3KQ(W5c>$$sWRwY2w|l~(Wn)BILPFA_4dXnFj1m? z>o$XiDq2m|&t0lD)%Q0@FKJMstDf;{@UbDXSC3EkJaCKa{DCetkzAEQKIo~M<8HF5}sKl{oy$Vbf@Kpi7EPy7&oDcL8dyTd^?dzlo zFq)g36WHv?w2DUI2(KDm#}DXvH_opcQ&p6X6C+`r`wd{hHZOAkhU2X`haie`Yp(vE z5f0fPn2vSOyvtR5V|q)_o3;y&%n4=SSQ%GlWW0o_-d6SlRGI>Zej7xi>a<5wd6o0E zEy1_ll0<_9GZD;Uvn_2&Zt?IAS@>^J4Wn$9O!? z!SloQ*(U`MU-i`UfIXXY*iLvNPzEFSFtoE>uT~7G6lV0jt_^I*!mGw1Ie_;Ng?W%M zA*kksUaU-j98QMbIAPW|I&iBvo$=k}3Cxa4SnK%|Em^+9mA2+N>hi+uC*Az@>tQh3 z_Xk>^(#O|~vwn_Y0(&=Ip=UQ}s#CI{8J0!=_Y@QzEKQ_$vpRb1#s)xva#1;e0)cA8 ziBStYZ_U)>YY@O^vu|zfn!wbZ^=}D$Y{yUvNpdISJ|r zdJbx3P5@^rTccSxKnwfY%m{sUQSokTu?U@IZz&{6^x;A1qj1s=3#Qsj{9haC9ohd$aNY zt@h}_zD`iB(9mlo8+!=d+}td25_`ze7>lgJJWo%=YeD!J4RqQ10&LOqJny=pvqHJj zi3ZAFp|iqSRz-XCRQsXr30HBmi*zPwFqq88jI&TP>zlqsRjp(JAYEXJ_NS zZLG`xJ2wg zNC%RhQ9dr-ASCF&$P1cKIYx(r!At6S1*_3!OKT(W=@QHN81w+^yWS^`jT!L{2%UF!T%pj0bH;DHST{uJlfv>_$bfA^8dAb{&RPC`_!KL z`gdIV52gUFTmJ`xX|A$fl z*2@2*VO{Ek2aQK8=-O}s_ z!s&ErN`7wg{o$WA>;FO2fVJ}fxE}v~e6YV=|Bv!~~_W!}*!SQzeKgRO}UBQvqrODc+^L|YkM+=a%It!&`4sL!k$Pyw-4ZutwEwKI4 zpYV!I$F(SKIE9O2<80dUg3tJr>)&L<{RAGsweo+zVgHXt+x7n_PpAJ6wu8nx&9PT5 z-P$yPFSlelM#<8=GFh)acgMERMvt@pCC5HV7Z0xitXu#4jri}w!FK=uah?a1|Em7; zbYT?$%`d)v_5Atu`Ng+hB}B3JBjyR3aYA&qTFH8qTHk$t@%^jwZ_ZC^)vo6Ry<|e7 z?^wz>Tw-3ScX9Iad96yBvu1m4z2&K`e;EFU<^XH;|HGsD`ajqoZP))}JP*A7ArWzv z1N@B1jBsDhIi1QPGMdXV8Xk;}o|!jgj@{~eo8EPdu1_zw3;4mGV*O7sXTG^#=Ybi( zdi#G=U;l^u+w)(K@~mCHJvA|U)pa}CKR9k%x6b==-9ox$&ofhN>k0gQJ;nOhm+|>< zF`A#{acpjcdxZQC`$vuQ--E6F|2WTv{{OC9Cu`Awfu{?qRSsn}FLCZ9tTGof=|WB- z&r|HBeiNjL-7T}o4F~3Q6TJJDvph2|y`sTdbG9+K#7?*MxvBp9dD_>%y~6Qv_W$;e z>i+-Z{jL0eoaaNIX(UU^~Lqu^Rt?# z(CNTill$~7je4DcUYwskzqoq-_V3UCu>w*|Ly`*e_LeNWK|Mb^zdrx|;_dm_RSnd) zguS4WjQo?U3sn3#rE=-~QmFD3;LrZ{_U!ys4RT54-bR_fHW{vsa(rzL?sG4We}kvs z|5`i#6z1Os=f4{I-_haWmj6G-v!VXC-tgO6Uw_D_b^SlA`M1ITPd)#`@o<~};c=c# z*MAz(1$4M>StZw0nkcZKD*q4j3Ve{OR9>7HW!rz|2m_Yvd5$7$qC788AA)JJLo zYyCg_jr|GY zd;Uv~<6h17D8Ye!FrAW6j#2M|U4?TJM-VD|2r5p?teNQZ0)~CdDiA-xAWo(=EZ3_FSKw- z@Vl#9ayLdpt^AqNTJ1AGoaA0gUi^!t`b6Q~oTRT)fh84FiYGA{BkQ?2Ntbtc38mor z(W+Alp#;lt?%q$S9-L*hrZlI~2NJtW(NsvBhDFKGJtmL&6zl(v%DErW84(YD0BGI% zKWyaxI^3TBf25~nZ>DEL8?9)!_kFpUQL`_IT=7#>VUgQF0WugHFg501!W z+S_e6+*5ApP?Xn$}pjD{n;PmcDd!{Om@bU1zX z)#T_X8cpz5&xTX-Y($O@_6H*z?jH}r!-KDm28V}(!>__Wtf$$me;`tyyu(Qr6McgF z;o4Zc{_FQY4iEPCxBI`3@{~C}|D#OhNz*CodH>PO;Q5>9wWsW~xjO%&lA^P-IBYgn z%BwdkDwuRa)jHYFc3;C__4*%;j_dnBqtW4L8~^_(kMH}Q%L8ym$va6> z^-Bb|&%i)l482>LMq_jenYmw?n>EhJl%`ZtmU;=1IKmQ~e7L?N>D?<{+}OuziwxA|s({c&2xqJ4$I_7DFt_ zjB#2znN3l@^d0z)A3;;&l0`4rOkd;{Qz8HEzI{Q3&UTQ+ImdB4CH8u%eB2H8*C{G=D#yNssGER_EmB(lgq3xnx}QBc){b1)cerhjS`PD05EwIHNwGp2E5blITfsf6Yz>S9_y|n$XS% zn2{?+K+R1=1;VDPIY?I!+Gh87_5A9(xH<<0qfULGds$JGL!byo(<$M45T~3a0MjbW zIN2ck7pPgbB~mTfT*b6j6OfFMqQ7sB}6l4U1QRoSNbZ!fWW==5_-mfas~PxfXP894p$tpb zF->PJQ=~HXuXIU$p#|6>N^mXJAE3ld$W&>v5sB#nf-jrh>RiE9Ex*zd*sFH}MavW? zG(?=-aVnkIPAJ<-K|$mmnh;<`AY&uY|6q5hz?Y^oB2>cxI+ul(dh_22k(`Ea4Sn9~ zy(RJ28()yH#hlX*o1#n35+dg$7bY2TuFsjYD*9PbSn2u^3x)HluWh`31x)SNlXf(B zTCiKE-C*cex1I-yO2(GSZ=?37oRg0oq+%PH5vu`M_qDeVX(Zbn23v5xRe$z5y3gL~$k}h z`cffCF03cistbxrmX<4BBxZ3G6S4-RWp+(hK-3cw#%>S9oz|ok_qWoztB-5Uy$Wq% z7%p%^v{j;$tT~#PcE6;m04%T-wqc@dd8N)9e|m z^>3a=go}{zZk~yheJcr1$LRmP!T!Ha{C~dv$^0?!zk2)AmtS-falKW$&Z5-`Zh=f2 zO;#wW#7N0tIT3j*OQ$;(NNM7dq=>d_!w8%|X86r;gO&|EIE~a?T!QnrTE;S}ZvUWW zVC{_R!xr=g)>a#J#RvPJ9appJu68$U-N{>bawj>q?qqnj?&QvA>rO^nck+Gl+q#pt z?&Jq@C$~>?zmQGd+BO?61>p9eZm(U~KHF#esOSF+009606(6Pk0N?=t D4!zG2 literal 0 HcmV?d00001 diff --git a/charts/k8s-monitoring/templates/_configs.tpl b/charts/k8s-monitoring/templates/_configs.tpl index 2b9d1eec2..5621b497b 100644 --- a/charts/k8s-monitoring/templates/_configs.tpl +++ b/charts/k8s-monitoring/templates/_configs.tpl @@ -153,3 +153,11 @@ {{- include "alloy.config.logging" (index .Values "alloy-profiles").logging }} {{- include "alloy.config.liveDebugging" (index .Values "alloy-profiles").liveDebugging}} {{- end -}} + +{{/* Grafana Alloy for Rules config */}} +{{- define "alloyRulesConfig" -}} + {{- include "alloy.config.rulesMimir" . }} + {{- include "alloy.config.rulesLoki" . }} + + {{- include "alloy.config.logging" (index .Values "alloy-rules").logging }} +{{- end -}} diff --git a/charts/k8s-monitoring/templates/alloy-rules-config.yaml b/charts/k8s-monitoring/templates/alloy-rules-config.yaml new file mode 100644 index 000000000..0f6651103 --- /dev/null +++ b/charts/k8s-monitoring/templates/alloy-rules-config.yaml @@ -0,0 +1,10 @@ +{{- if .Values.rules.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "alloy.fullname" (index .Subcharts "alloy-rules") }} + namespace: {{ .Release.Namespace }} +data: + config.alloy: |- + {{- include "alloyRulesConfig" . | trim | nindent 4 }} +{{- end }} diff --git a/charts/k8s-monitoring/templates/alloy_config/_rules_loki.alloy.txt b/charts/k8s-monitoring/templates/alloy_config/_rules_loki.alloy.txt new file mode 100644 index 000000000..198c7e70d --- /dev/null +++ b/charts/k8s-monitoring/templates/alloy_config/_rules_loki.alloy.txt @@ -0,0 +1,74 @@ +{{ define "alloy.config.rulesLoki" }} +{{- if .Values.rules.loki.enabled }} +// Logs Service +remote.kubernetes.secret "logs_service" { + name = {{ include "kubernetes_monitoring.logs_service.secret.name" . | quote}} + namespace = {{ .Values.externalServices.loki.secret.namespace | default .Release.Namespace | quote }} +} +// Rules +loki.rules.kubernetes "rules_service" { + address = nonsensitive(remote.kubernetes.secret.logs_service.data[{{ .Values.externalServices.loki.hostKey | quote }}]) + {{- if or (and (eq .Values.externalServices.loki.secret.create true) (.Values.externalServices.loki.tenantId) (eq .Values.externalServices.loki.secret.create false) ) }} + headers = nonsensitive(coalesce(remote.kubernetes.secret.logs_service.data[{{ .Values.externalServices.loki.tenantIdKey | quote }}], "")) + {{- end }} + sync_interval = {{ .Values.rules.loki.sync_interval | quote }} + loki_namespace_prefix = {{ .Values.rules.loki.prefix | quote }} +{{- if .Values.externalServices.loki.proxyURL }} + proxy_url = {{ .Values.externalServices.loki.proxyURL | quote }} +{{- end }} +{{ if eq .Values.externalServices.loki.authMode "basic" }} + basic_auth { + username = nonsensitive(remote.kubernetes.secret.logs_service.data[{{ .Values.externalServices.loki.basicAuth.usernameKey | quote }}]) + password = remote.kubernetes.secret.logs_service.data[{{ .Values.externalServices.loki.basicAuth.passwordKey | quote }}] + } +{{- end }} + rule_namespace_selector { + {{- if .Values.rules.loki.namespace.label_selectors }} + match_labels = { + {{- range $key, $value := .Values.rules.loki.namespace.label_selectors }} + {{ $key }} = "{{ $value }}", + {{- end }} + } + {{- end }} + + {{- if .Values.rules.loki.namespace.label_expressions }} + {{- range $expr := .Values.rules.loki.namespace.label_expressions }} + match_expression { + key = "{{ $expr.key }}" + operator = "{{ $expr.operator }}" + values = [ + {{- range $index, $value := $expr.values }} + {{- if $index }}, {{ end }}"{{ $value }}" + {{- end }} + ] + } + {{- end }} + {{- end }} + } + + rule_selector { + {{- if .Values.rules.loki.rule.label_selectors }} + match_labels = { + {{- range $key, $value := .Values.rules.loki.rule.label_selectors }} + {{ $key }} = "{{ $value }}", + {{- end }} + } + {{- end }} + + {{- if .Values.rules.loki.rule.label_expressions }} + {{- range $expr := .Values.rules.loki.rule.label_expressions }} + match_expression { + key = "{{ $expr.key }}" + operator = "{{ $expr.operator }}" + values = [ + {{- range $index, $value := $expr.values }} + {{- if $index }}, {{ end }}"{{ $value }}" + {{- end }} + ] + } + {{- end }} + {{- end }} + } +} +{{- end }} +{{- end }} diff --git a/charts/k8s-monitoring/templates/alloy_config/_rules_mimir.alloy.txt b/charts/k8s-monitoring/templates/alloy_config/_rules_mimir.alloy.txt new file mode 100644 index 000000000..d5d5982c4 --- /dev/null +++ b/charts/k8s-monitoring/templates/alloy_config/_rules_mimir.alloy.txt @@ -0,0 +1,75 @@ +{{ define "alloy.config.rulesMimir" }} +{{- if .Values.rules.mimir.enabled }} +// Metrics Service +remote.kubernetes.secret "metrics_service" { + name = {{ include "kubernetes_monitoring.metrics_service.secret.name" . | quote }} + namespace = {{ .Values.externalServices.prometheus.secret.namespace | default .Release.Namespace | quote }} +} +// Rules +mimir.rules.kubernetes "rules_service" { + address = nonsensitive(remote.kubernetes.secret.metrics_service.data[{{ .Values.externalServices.prometheus.hostKey | quote }}]) + {{- if or (and (eq .Values.externalServices.prometheus.secret.create true) (.Values.externalServices.prometheus.tenantId) (eq .Values.externalServices.prometheus.secret.create false) ) }} + headers = nonsensitive(coalesce(remote.kubernetes.secret.metrics_service.data[{{ .Values.externalServices.prometheus.tenantIdKey | quote }}], "")) + {{- end }} + sync_interval = {{ .Values.rules.mimir.sync_interval | quote }} + prometheus_http_prefix = {{ .Values.rules.mimir.prometheus_http_prefix | quote }} + mimir_namespace_prefix = {{ .Values.rules.mimir.prefix | quote }} +{{- if .Values.externalServices.prometheus.proxyURL }} + proxy_url = {{ .Values.externalServices.prometheus.proxyURL | quote }} +{{- end }} +{{ if eq .Values.externalServices.prometheus.authMode "basic" }} + basic_auth { + username = nonsensitive(remote.kubernetes.secret.metrics_service.data[{{ .Values.externalServices.prometheus.basicAuth.usernameKey | quote }}]) + password = remote.kubernetes.secret.metrics_service.data[{{ .Values.externalServices.prometheus.basicAuth.passwordKey | quote }}] + } +{{- end }} + rule_namespace_selector { + {{- if .Values.rules.mimir.namespace.label_selectors }} + match_labels = { + {{- range $key, $value := .Values.rules.mimir.namespace.label_selectors }} + {{ $key }} = "{{ $value }}", + {{- end }} + } + {{- end }} + + {{- if .Values.rules.mimir.namespace.label_expressions }} + {{- range $expr := .Values.rules.mimir.namespace.label_expressions }} + match_expression { + key = "{{ $expr.key }}" + operator = "{{ $expr.operator }}" + values = [ + {{- range $index, $value := $expr.values }} + {{- if $index }}, {{ end }}"{{ $value }}" + {{- end }} + ] + } + {{- end }} + {{- end }} + } + + rule_selector { + {{- if .Values.rules.mimir.rule.label_selectors }} + match_labels = { + {{- range $key, $value := .Values.rules.mimir.rule.label_selectors }} + {{ $key }} = "{{ $value }}", + {{- end }} + } + {{- end }} + + {{- if .Values.rules.mimir.rule.label_expressions }} + {{- range $expr := .Values.rules.mimir.rule.label_expressions }} + match_expression { + key = "{{ $expr.key }}" + operator = "{{ $expr.operator }}" + values = [ + {{- range $index, $value := $expr.values }} + {{- if $index }}, {{ end }}"{{ $value }}" + {{- end }} + ] + } + {{- end }} + {{- end }} + } +} +{{- end }} +{{- end }} diff --git a/charts/k8s-monitoring/values.schema.json b/charts/k8s-monitoring/values.schema.json index 5bf71489e..84a23e782 100644 --- a/charts/k8s-monitoring/values.schema.json +++ b/charts/k8s-monitoring/values.schema.json @@ -138,6 +138,81 @@ } } }, + "alloy-rules": { + "type": "object", + "properties": { + "alloy": { + "type": "object", + "properties": { + "clustering": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + } + } + }, + "configMap": { + "type": "object", + "properties": { + "create": { + "type": "boolean" + } + } + } + } + }, + "controller": { + "type": "object", + "properties": { + "nodeSelector": { + "type": "object", + "properties": { + "kubernetes.io/os": { + "type": "string" + } + } + }, + "tolerations": { + "type": "array", + "items": { + "type": "object", + "properties": { + "effect": { + "type": "string" + }, + "operator": { + "type": "string" + } + } + } + }, + "type": { + "type": "string" + } + } + }, + "crds": { + "type": "object", + "properties": { + "create": { + "type": "boolean" + } + } + }, + "logging": { + "type": "object", + "properties": { + "format": { + "type": "string" + }, + "level": { + "type": "string" + } + } + } + } + }, "cluster": { "type": "object", "properties": { @@ -2283,6 +2358,99 @@ } } }, + "rules": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "loki": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "namespace": { + "type": "object", + "properties": { + "label_expressions": { + "type": "array" + }, + "label_selectors": { + "type": "object" + } + } + }, + "prefix": { + "type": "string" + }, + "rule": { + "type": "object", + "properties": { + "label_expressions": { + "type": "array" + }, + "label_selectors": { + "type": "object", + "properties": { + "rule_type": { + "type": "string" + } + } + } + } + }, + "sync_interval": { + "type": "string" + } + } + }, + "mimir": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "namespace": { + "type": "object", + "properties": { + "label_expressions": { + "type": "array" + }, + "label_selectors": { + "type": "object" + } + } + }, + "prefix": { + "type": "string" + }, + "prometheus_http_prefix": { + "type": "string" + }, + "rule": { + "type": "object", + "properties": { + "label_expressions": { + "type": "array" + }, + "label_selectors": { + "type": "object", + "properties": { + "rule_type": { + "type": "string" + } + } + } + } + }, + "sync_interval": { + "type": "string" + } + } + } + } + }, "test": { "type": "object", "properties": { diff --git a/charts/k8s-monitoring/values.yaml b/charts/k8s-monitoring/values.yaml index 8bf6cc696..ee1adfa38 100644 --- a/charts/k8s-monitoring/values.yaml +++ b/charts/k8s-monitoring/values.yaml @@ -1709,6 +1709,91 @@ profiles: - mutex - fgprof +# Settings related to the Rule syncing +rules: + # -- Whether or not to enable the rules synchronization + # @section -- Rules + enabled: false + + # Settings for Mimir rules, see: https://grafana.com/docs/alloy/latest/reference/components/mimir.rules.kubernetes/ + mimir: + # -- Whether or not to enable the Mimir rules synchronization + # @section -- Rules (Mimir) + enabled: true + # -- Amount of time between reconciliations with Mimir. + # @default -- 5m + sync_interval: 5m + # -- Path prefix for Mimir’s Prometheus endpoint (gem-path-prefix). + # @default -- /api/prom + prometheus_http_prefix: /api/prom + # -- Prefix to be added to the rule namespace, used to differentiate multiple Alloy deployments added. + # @default -- alloy + prefix: alloy + # Namespace selectors for Mimir rules. + namespace: + # -- Label selector for Namespace resources. + label_selectors: {} + # -- Label expressions for Namespace resources. + label_expressions: [] + # Selector for PrometheusRule resources. + rule: + # -- Label selectors for PrometheusRule resources as key/pair values. + # Example: + # ```alloy + # label_selectors: + # rule_type: mimir + # sync: "true" + # mimir: "true" + # ``` + label_selectors: + rule_type: mimir + # -- Label expressions for PrometheusRule resources. + # Example: + # ```alloy + # - key: team + # operator: In + # values: ["ops"] + # ``` + label_expressions: [] + + # Settings for Loki rules, see: https://grafana.com/docs/alloy/latest/reference/components/loki.rules.kubernetes/ + loki: + # -- Whether or not to enable the Mimir rules synchronization + # @section -- Rules (Loki) + enabled: true + # -- Amount of time between reconciliations with Mimir. + # @default -- 5m + sync_interval: 5m + # -- Prefix to be added to the rule namespace, used to differentiate multiple Alloy deployments added. + # @default -- alloy + prefix: alloy + # Namespace selectors for Loki rules. + namespace: + # -- Label selector for Namespace resources. + label_selectors: {} + # -- Label expressions for Namespace resources. + label_expressions: [] + # Selector for PrometheusRule resources. + rule: + # -- Label selectors for PrometheusRule resources as key/pair values. + # Example: + # ```alloy + # label_selectors: + # rule_type: loki + # sync: "true" + # loki: "true" + # ``` + label_selectors: + rule_type: loki + # -- Label expressions for PrometheusRule resources. + # Example: + # ```alloy + # - key: team + # operator: In + # values: ["ops"] + # ``` + label_expressions: [] + # Telemetry data receiver settings receivers: grpc: @@ -2404,6 +2489,43 @@ alloy-profiles: # @ignored crds: {create: false} + +# Settings for the Grafana Alloy instance that syncs PrometheusRules to Mimir and Loki +# See https://github.com/grafana/alloy/tree/main/operations/helm/charts/alloy for available values. +# @ignored -- This skips including these values in README.md +alloy-rules: + logging: + # -- Level at which Alloy log lines should be written. + # @section -- Chart + level: info + # -- Format to use for writing Alloy log lines. + # @section -- Chart + format: logfmt + + alloy: + # This chart is creating the configuration, so the alloy chart does not need to. + # @ignored + configMap: {create: false} + + # Disabling clustering by default, because the default log gathering format does not require clusters. + # @ignored + clustering: {enabled: false} + + controller: + # @ignored + type: statefulset + # @ignored + nodeSelector: + kubernetes.io/os: linux + + tolerations: + - effect: NoSchedule + operator: Exists + + # Skip installation of the Grafana Alloy CRDs, since we don't use them in this chart + # @ignored + crds: {create: false} + # -- Deploy additional manifest objects extraObjects: [] # - apiVersion: external-secrets.io/v1beta1 diff --git a/examples/alloy-autoscaling-and-storage/rules.alloy b/examples/alloy-autoscaling-and-storage/rules.alloy new file mode 100644 index 000000000..e69de29bb diff --git a/examples/control-plane-metrics/rules.alloy b/examples/control-plane-metrics/rules.alloy new file mode 100644 index 000000000..e69de29bb diff --git a/examples/custom-config/rules.alloy b/examples/custom-config/rules.alloy new file mode 100644 index 000000000..e69de29bb diff --git a/examples/custom-metrics-tuning/rules.alloy b/examples/custom-metrics-tuning/rules.alloy new file mode 100644 index 000000000..e69de29bb diff --git a/examples/custom-pricing/rules.alloy b/examples/custom-pricing/rules.alloy new file mode 100644 index 000000000..e69de29bb diff --git a/examples/custom-prometheus-operator-rules/rules.alloy b/examples/custom-prometheus-operator-rules/rules.alloy new file mode 100644 index 000000000..e69de29bb diff --git a/examples/default-values/rules.alloy b/examples/default-values/rules.alloy new file mode 100644 index 000000000..e69de29bb diff --git a/examples/eks-fargate/rules.alloy b/examples/eks-fargate/rules.alloy new file mode 100644 index 000000000..e69de29bb diff --git a/examples/extra-rules/rules.alloy b/examples/extra-rules/rules.alloy new file mode 100644 index 000000000..e69de29bb diff --git a/examples/gke-autopilot/rules.alloy b/examples/gke-autopilot/rules.alloy new file mode 100644 index 000000000..e69de29bb diff --git a/examples/ibm-cloud/rules.alloy b/examples/ibm-cloud/rules.alloy new file mode 100644 index 000000000..e69de29bb diff --git a/examples/logs-journal/rules.alloy b/examples/logs-journal/rules.alloy new file mode 100644 index 000000000..e69de29bb diff --git a/examples/logs-only/rules.alloy b/examples/logs-only/rules.alloy new file mode 100644 index 000000000..e69de29bb diff --git a/examples/metric-module-imports-extra-config/rules.alloy b/examples/metric-module-imports-extra-config/rules.alloy new file mode 100644 index 000000000..e69de29bb diff --git a/examples/metric-module-imports/rules.alloy b/examples/metric-module-imports/rules.alloy new file mode 100644 index 000000000..e69de29bb diff --git a/examples/metrics-only/rules.alloy b/examples/metrics-only/rules.alloy new file mode 100644 index 000000000..e69de29bb diff --git a/examples/openshift-compatible/rules.alloy b/examples/openshift-compatible/rules.alloy new file mode 100644 index 000000000..e69de29bb diff --git a/examples/otel-metrics-service/rules.alloy b/examples/otel-metrics-service/rules.alloy new file mode 100644 index 000000000..e69de29bb diff --git a/examples/pod-labels/rules.alloy b/examples/pod-labels/rules.alloy new file mode 100644 index 000000000..e69de29bb diff --git a/examples/private-image-registry/rules.alloy b/examples/private-image-registry/rules.alloy new file mode 100644 index 000000000..e69de29bb diff --git a/examples/profiles-enabled/rules.alloy b/examples/profiles-enabled/rules.alloy new file mode 100644 index 000000000..e69de29bb diff --git a/examples/proxies/rules.alloy b/examples/proxies/rules.alloy new file mode 100644 index 000000000..e69de29bb diff --git a/examples/rules-sync/events.alloy b/examples/rules-sync/events.alloy new file mode 100644 index 000000000..266de8610 --- /dev/null +++ b/examples/rules-sync/events.alloy @@ -0,0 +1,48 @@ +// Cluster Events +loki.source.kubernetes_events "cluster_events" { + job_name = "integrations/kubernetes/eventhandler" + log_format = "logfmt" + forward_to = [ + loki.process.cluster_events.receiver, + ] +} + +loki.process "cluster_events" { + forward_to = [ + loki.process.logs_service.receiver, + ] +} + +// Logs Service +remote.kubernetes.secret "logs_service" { + name = "loki-k8s-monitoring" + namespace = "default" +} + +loki.process "logs_service" { + stage.static_labels { + values = { + cluster = "rules-sync", + } + } + forward_to = [loki.write.logs_service.receiver] +} + +// Loki +loki.write "logs_service" { + endpoint { + url = nonsensitive(remote.kubernetes.secret.logs_service.data["host"]) + "/loki/api/v1/push" + tenant_id = nonsensitive(remote.kubernetes.secret.logs_service.data["tenantId"]) + + basic_auth { + username = nonsensitive(remote.kubernetes.secret.logs_service.data["username"]) + password = remote.kubernetes.secret.logs_service.data["password"] + } + } +} + + +logging { + level = "info" + format = "logfmt" +} diff --git a/examples/rules-sync/logs.alloy b/examples/rules-sync/logs.alloy new file mode 100644 index 000000000..a7ea0e979 --- /dev/null +++ b/examples/rules-sync/logs.alloy @@ -0,0 +1,155 @@ +// Pod Logs +discovery.kubernetes "pods" { + role = "pod" + selectors { + role = "pod" + field = "spec.nodeName=" + env("HOSTNAME") + } +} + +discovery.relabel "pod_logs" { + targets = discovery.kubernetes.pods.targets + rule { + source_labels = ["__meta_kubernetes_namespace"] + action = "replace" + target_label = "namespace" + } + + rule { + source_labels = ["__meta_kubernetes_pod_name"] + action = "replace" + target_label = "pod" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + action = "replace" + target_label = "container" + } + rule { + source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"] + separator = "/" + action = "replace" + replacement = "$1" + target_label = "job" + } + rule { + source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"] + separator = "/" + action = "replace" + replacement = "/var/log/pods/*$1/*.log" + target_label = "__path__" + } + + // set the container runtime as a label + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_container_id"] + regex = "^(\\S+):\\/\\/.+$" + replacement = "$1" + target_label = "tmp_container_runtime" + } +} + +discovery.relabel "filtered_pod_logs" { + targets = discovery.relabel.pod_logs.output + rule { // Drop anything with a "falsy" annotation value + source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_logs_autogather"] + regex = "(false|no|skip)" + action = "drop" + } +} + +local.file_match "pod_logs" { + path_targets = discovery.relabel.filtered_pod_logs.output +} + +loki.source.file "pod_logs" { + targets = local.file_match.pod_logs.targets + forward_to = [loki.process.pod_logs.receiver] +} + +loki.process "pod_logs" { + stage.match { + selector = "{tmp_container_runtime=\"containerd\"}" + // the cri processing stage extracts the following k/v pairs: log, stream, time, flags + stage.cri {} + + // Set the extract flags and stream values as labels + stage.labels { + values = { + flags = "", + stream = "", + } + } + } + + stage.match { + selector = "{tmp_container_runtime=\"cri-o\"}" + // the cri processing stage extracts the following k/v pairs: log, stream, time, flags + stage.cri {} + + // Set the extract flags and stream values as labels + stage.labels { + values = { + flags = "", + stream = "", + } + } + } + + // if the label tmp_container_runtime from above is docker parse using docker + stage.match { + selector = "{tmp_container_runtime=\"docker\"}" + // the docker processing stage extracts the following k/v pairs: log, stream, time + stage.docker {} + + // Set the extract stream value as a label + stage.labels { + values = { + stream = "", + } + } + } + + // Drop the filename label, since it's not really useful in the context of Kubernetes, where we already have + // cluster, namespace, pod, and container labels. + // Also drop the temporary container runtime label as it is no longer needed. + stage.label_drop { + values = ["filename", "tmp_container_runtime"] + } + forward_to = [loki.process.logs_service.receiver] +} + +// Logs Service +remote.kubernetes.secret "logs_service" { + name = "loki-k8s-monitoring" + namespace = "default" +} + +loki.process "logs_service" { + stage.static_labels { + values = { + cluster = "rules-sync", + } + } + forward_to = [loki.write.logs_service.receiver] +} + +// Loki +loki.write "logs_service" { + endpoint { + url = nonsensitive(remote.kubernetes.secret.logs_service.data["host"]) + "/loki/api/v1/push" + tenant_id = nonsensitive(remote.kubernetes.secret.logs_service.data["tenantId"]) + + basic_auth { + username = nonsensitive(remote.kubernetes.secret.logs_service.data["username"]) + password = remote.kubernetes.secret.logs_service.data["password"] + } + } +} + + +logging { + level = "info" + format = "logfmt" +} diff --git a/examples/rules-sync/metrics.alloy b/examples/rules-sync/metrics.alloy new file mode 100644 index 000000000..b9b41e0f3 --- /dev/null +++ b/examples/rules-sync/metrics.alloy @@ -0,0 +1,824 @@ +discovery.kubernetes "nodes" { + role = "node" +} + +discovery.kubernetes "services" { + role = "service" +} + +discovery.kubernetes "endpoints" { + role = "endpoints" +} + +discovery.kubernetes "pods" { + role = "pod" +} + +// OTLP Receivers +otelcol.receiver.otlp "receiver" { + debug_metrics { + disable_high_cardinality_metrics = true + } + + grpc { + endpoint = "0.0.0.0:4317" + } + + http { + endpoint = "0.0.0.0:4318" + } + output { + metrics = [otelcol.processor.resourcedetection.default.input] + logs = [otelcol.processor.resourcedetection.default.input] + } +} + + + + +// Processors +otelcol.processor.transform "add_metric_datapoint_attributes" { + // Grafana Cloud Kubernetes monitoring expects Loki labels `cluster`, `pod`, and `namespace` + error_mode = "ignore" + metric_statements { + context = "datapoint" + statements = [ + "set(attributes[\"deployment.environment\"], resource.attributes[\"deployment.environment\"])", + "set(attributes[\"service.version\"], resource.attributes[\"service.version\"])", + ] + } + output { + metrics = [otelcol.processor.k8sattributes.default.input] + } +} + +otelcol.processor.resourcedetection "default" { + detectors = ["env", "system"] + + system { + hostname_sources = ["os"] + } + + output { + metrics = [otelcol.processor.transform.add_metric_datapoint_attributes.input] + logs = [otelcol.processor.k8sattributes.default.input] + } +} + +otelcol.processor.k8sattributes "default" { + extract { + metadata = ["k8s.namespace.name","k8s.pod.name","k8s.deployment.name","k8s.statefulset.name","k8s.daemonset.name","k8s.cronjob.name","k8s.job.name","k8s.node.name","k8s.pod.uid","k8s.pod.start_time"] + } + pod_association { + source { + from = "connection" + } + } + + output { + metrics = [otelcol.processor.transform.default.input] + logs = [otelcol.processor.transform.default.input] + } +} + +otelcol.processor.transform "default" { + // Grafana Cloud Kubernetes monitoring expects Loki labels `cluster`, `pod`, and `namespace` + error_mode = "ignore" + metric_statements { + context = "resource" + statements = [ + "set(attributes[\"k8s.cluster.name\"], \"rules-sync\") where attributes[\"k8s.cluster.name\"] == nil", + ] + } + log_statements { + context = "resource" + statements = [ + "set(attributes[\"pod\"], attributes[\"k8s.pod.name\"])", + "set(attributes[\"namespace\"], attributes[\"k8s.namespace.name\"])", + "set(attributes[\"loki.resource.labels\"], \"cluster, namespace, job, pod\")", + "set(attributes[\"k8s.cluster.name\"], \"rules-sync\") where attributes[\"k8s.cluster.name\"] == nil", + ] + } + output { + metrics = [otelcol.processor.filter.default.input] + logs = [otelcol.processor.filter.default.input] + } +} + +otelcol.processor.filter "default" { + error_mode = "ignore" + + output { + metrics = [otelcol.processor.batch.batch_processor.input] + logs = [otelcol.processor.batch.batch_processor.input] + } +} + +otelcol.processor.batch "batch_processor" { + send_batch_size = 16384 + send_batch_max_size = 0 + timeout = "2s" + output { + metrics = [otelcol.exporter.prometheus.metrics_converter.input] + logs = [otelcol.exporter.loki.logs_converter.input] + } +} +otelcol.exporter.prometheus "metrics_converter" { + forward_to = [prometheus.relabel.metrics_service.receiver] +} +otelcol.exporter.loki "logs_converter" { + forward_to = [loki.process.pod_logs.receiver] +} +// Annotation Autodiscovery +discovery.relabel "annotation_autodiscovery_pods" { + targets = discovery.kubernetes.pods.targets + rule { + source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_scrape"] + regex = "true" + action = "keep" + } + rule { + source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_job"] + action = "replace" + target_label = "job" + } + rule { + source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_instance"] + action = "replace" + target_label = "instance" + } + rule { + source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_metrics_path"] + action = "replace" + target_label = "__metrics_path__" + } + + // Choose the pod port + // The discovery generates a target for each declared container port of the pod. + // If the metricsPortName annotation has value, keep only the target where the port name matches the one of the annotation. + rule { + source_labels = ["__meta_kubernetes_pod_container_port_name"] + target_label = "__tmp_port" + } + rule { + source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_metrics_portName"] + regex = "(.+)" + target_label = "__tmp_port" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_port_name"] + action = "keepequal" + target_label = "__tmp_port" + } + + // If the metrics port number annotation has a value, override the target address to use it, regardless whether it is + // one of the declared ports on that Pod. + rule { + source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_metrics_portNumber", "__meta_kubernetes_pod_ip"] + regex = "(\\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})" + replacement = "[$2]:$1" // IPv6 + target_label = "__address__" + } + rule { + source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_metrics_portNumber", "__meta_kubernetes_pod_ip"] + regex = "(\\d+);((([0-9]+?)(\\.|$)){4})" // IPv4, takes priority over IPv6 when both exists + replacement = "$2:$1" + target_label = "__address__" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_k8s_grafana_com_metrics_scheme"] + action = "replace" + target_label = "__scheme__" + } +} + +discovery.relabel "annotation_autodiscovery_services" { + targets = discovery.kubernetes.services.targets + rule { + source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_scrape"] + regex = "true" + action = "keep" + } + rule { + source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_job"] + action = "replace" + target_label = "job" + } + rule { + source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_instance"] + action = "replace" + target_label = "instance" + } + rule { + source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_metrics_path"] + action = "replace" + target_label = "__metrics_path__" + } + + // Choose the service port + rule { + source_labels = ["__meta_kubernetes_service_port_name"] + target_label = "__tmp_port" + } + rule { + source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_metrics_portName"] + regex = "(.+)" + target_label = "__tmp_port" + } + rule { + source_labels = ["__meta_kubernetes_service_port_name"] + action = "keepequal" + target_label = "__tmp_port" + } + + rule { + source_labels = ["__meta_kubernetes_service_port_number"] + target_label = "__tmp_port" + } + rule { + source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_metrics_portNumber"] + regex = "(.+)" + target_label = "__tmp_port" + } + rule { + source_labels = ["__meta_kubernetes_service_port_number"] + action = "keepequal" + target_label = "__tmp_port" + } + + rule { + source_labels = ["__meta_kubernetes_service_annotation_k8s_grafana_com_metrics_scheme"] + action = "replace" + target_label = "__scheme__" + } +} + +discovery.relabel "annotation_autodiscovery_http" { + targets = concat(discovery.relabel.annotation_autodiscovery_pods.output, discovery.relabel.annotation_autodiscovery_services.output) + rule { + source_labels = ["__scheme__"] + regex = "https" + action = "drop" + } +} + +discovery.relabel "annotation_autodiscovery_https" { + targets = concat(discovery.relabel.annotation_autodiscovery_pods.output, discovery.relabel.annotation_autodiscovery_services.output) + rule { + source_labels = ["__scheme__"] + regex = "https" + action = "keep" + } +} + +prometheus.scrape "annotation_autodiscovery_http" { + targets = discovery.relabel.annotation_autodiscovery_http.output + honor_labels = true + clustering { + enabled = true + } + forward_to = [prometheus.relabel.annotation_autodiscovery.receiver] +} + +prometheus.scrape "annotation_autodiscovery_https" { + targets = discovery.relabel.annotation_autodiscovery_https.output + honor_labels = true + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + tls_config { + insecure_skip_verify = true + } + clustering { + enabled = true + } + forward_to = [prometheus.relabel.annotation_autodiscovery.receiver] +} + +prometheus.relabel "annotation_autodiscovery" { + max_cache_size = 100000 + forward_to = [prometheus.relabel.metrics_service.receiver] +} + +// Grafana Alloy +discovery.relabel "alloy" { + targets = discovery.kubernetes.pods.targets + rule { + source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_instance"] + regex = "k8smon" + action = "keep" + } + rule { + source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"] + regex = "alloy.*" + action = "keep" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_port_name"] + regex = "http-metrics" + action = "keep" + } + rule { + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + rule { + source_labels = ["__meta_kubernetes_pod_name"] + target_label = "pod" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + target_label = "container" + } +} + +prometheus.scrape "alloy" { + job_name = "integrations/alloy" + targets = discovery.relabel.alloy.output + scrape_interval = "60s" + forward_to = [prometheus.relabel.alloy.receiver] + clustering { + enabled = true + } +} + +prometheus.relabel "alloy" { + max_cache_size = 100000 + rule { + source_labels = ["__name__"] + regex = "up|alloy_build_info" + action = "keep" + } + forward_to = [prometheus.relabel.metrics_service.receiver] +} + +// Kubernetes Monitoring Telemetry +prometheus.exporter.unix "kubernetes_monitoring_telemetry" { + set_collectors = ["textfile"] + textfile { + directory = "/etc/kubernetes-monitoring-telemetry" + } +} + +prometheus.scrape "kubernetes_monitoring_telemetry" { + job_name = "integrations/kubernetes/kubernetes_monitoring_telemetry" + targets = prometheus.exporter.unix.kubernetes_monitoring_telemetry.targets + scrape_interval = "60s" + clustering { + enabled = true + } + forward_to = [prometheus.relabel.kubernetes_monitoring_telemetry.receiver] +} + +prometheus.relabel "kubernetes_monitoring_telemetry" { + max_cache_size = 100000 + rule { + target_label = "job" + action = "replace" + replacement = "integrations/kubernetes/kubernetes_monitoring_telemetry" + } + rule { + target_label = "instance" + action = "replace" + replacement = "k8smon" + } + rule { + source_labels = ["__name__"] + regex = "up|grafana_kubernetes_monitoring_.*" + action = "keep" + } + forward_to = [prometheus.relabel.metrics_service.receiver] +} + +// Kubelet +discovery.relabel "kubelet" { + targets = discovery.kubernetes.nodes.targets +} + +prometheus.scrape "kubelet" { + job_name = "integrations/kubernetes/kubelet" + targets = discovery.relabel.kubelet.output + scheme = "https" + scrape_interval = "60s" + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + tls_config { + insecure_skip_verify = true + } + clustering { + enabled = true + } + forward_to = [prometheus.relabel.kubelet.receiver] +} + +prometheus.relabel "kubelet" { + max_cache_size = 100000 + rule { + source_labels = ["__name__"] + regex = "up|container_cpu_usage_seconds_total|kubelet_certificate_manager_client_expiration_renew_errors|kubelet_certificate_manager_client_ttl_seconds|kubelet_certificate_manager_server_ttl_seconds|kubelet_cgroup_manager_duration_seconds_bucket|kubelet_cgroup_manager_duration_seconds_count|kubelet_node_config_error|kubelet_node_name|kubelet_pleg_relist_duration_seconds_bucket|kubelet_pleg_relist_duration_seconds_count|kubelet_pleg_relist_interval_seconds_bucket|kubelet_pod_start_duration_seconds_bucket|kubelet_pod_start_duration_seconds_count|kubelet_pod_worker_duration_seconds_bucket|kubelet_pod_worker_duration_seconds_count|kubelet_running_container_count|kubelet_running_containers|kubelet_running_pod_count|kubelet_running_pods|kubelet_runtime_operations_errors_total|kubelet_runtime_operations_total|kubelet_server_expiration_renew_errors|kubelet_volume_stats_available_bytes|kubelet_volume_stats_capacity_bytes|kubelet_volume_stats_inodes|kubelet_volume_stats_inodes_used|kubernetes_build_info|namespace_workload_pod|rest_client_requests_total|storage_operation_duration_seconds_count|storage_operation_errors_total|volume_manager_total_volumes" + action = "keep" + } + forward_to = [prometheus.relabel.metrics_service.receiver] +} + +// cAdvisor +discovery.relabel "cadvisor" { + targets = discovery.kubernetes.nodes.targets + rule { + replacement = "/metrics/cadvisor" + target_label = "__metrics_path__" + } +} + +prometheus.scrape "cadvisor" { + job_name = "integrations/kubernetes/cadvisor" + targets = discovery.relabel.cadvisor.output + scheme = "https" + scrape_interval = "60s" + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + tls_config { + insecure_skip_verify = true + } + clustering { + enabled = true + } + forward_to = [prometheus.relabel.cadvisor.receiver] +} + +prometheus.relabel "cadvisor" { + max_cache_size = 100000 + rule { + source_labels = ["__name__"] + regex = "up|container_cpu_cfs_periods_total|container_cpu_cfs_throttled_periods_total|container_cpu_usage_seconds_total|container_fs_reads_bytes_total|container_fs_reads_total|container_fs_writes_bytes_total|container_fs_writes_total|container_memory_cache|container_memory_rss|container_memory_swap|container_memory_working_set_bytes|container_network_receive_bytes_total|container_network_receive_packets_dropped_total|container_network_receive_packets_total|container_network_transmit_bytes_total|container_network_transmit_packets_dropped_total|container_network_transmit_packets_total|machine_memory_bytes" + action = "keep" + } + // Drop empty container labels, addressing https://github.com/google/cadvisor/issues/2688 + rule { + source_labels = ["__name__","container"] + separator = "@" + regex = "(container_cpu_.*|container_fs_.*|container_memory_.*)@" + action = "drop" + } + // Drop empty image labels, addressing https://github.com/google/cadvisor/issues/2688 + rule { + source_labels = ["__name__","image"] + separator = "@" + regex = "(container_cpu_.*|container_fs_.*|container_memory_.*|container_network_.*)@" + action = "drop" + } + // Normalizing unimportant labels (not deleting to continue satisfying