Add MaxSurge and MaxUnavailable strategy to all Loki k8 workloads. (

#5227) * Add `MaxSurge` and `MaxUnavailable` strategy to all loki k8 workloads. This fixes couple of issues. 1. By default these configs are 25% in k8, meaning during rollout 25% of pods are allowed to shutdown immediately. 2. Due to (1), during graceful shutdown process, 25% of all the pods access consul to `unregister()` from shared key value. (2) makes CAS rate of underlying KV store high (leads to lots of retry and failing) sometimes failing to unregister leaving the ring "unhealthy" Also this PR make these configs consistent across all k8 workloads. More details: grafana/dskit#117 * Remove it from statefulset workloads Signed-off-by: Kaviraj <[email protected]>
grafana · Jan 25, 2022 · db283da · db283da
1 parent 3f496e7
commit db283da
Show file tree

Hide file tree

Showing 6 changed files with 18 additions and 6 deletions.
diff --git a/production/ksonnet/loki/distributor.libsonnet b/production/ksonnet/loki/distributor.libsonnet
@@ -29,7 +29,9 @@ local k = import 'ksonnet-util/kausal.libsonnet';
       $._config.overrides_configmap_mount_name,
       $._config.overrides_configmap_mount_path,
     ) +
-    k.util.antiAffinity,
+    k.util.antiAffinity +
+    deployment.mixin.spec.strategy.rollingUpdate.withMaxSurge(5) +
+    deployment.mixin.spec.strategy.rollingUpdate.withMaxUnavailable(1),
 
   distributor_service:
     k.util.serviceFor($.distributor_deployment),

diff --git a/production/ksonnet/loki/gateway.libsonnet b/production/ksonnet/loki/gateway.libsonnet
@@ -98,7 +98,9 @@ local k = import 'ksonnet-util/kausal.libsonnet';
     }) +
     k.util.configVolumeMount('gateway-config', '/etc/nginx') +
     k.util.secretVolumeMount('gateway-secret', '/etc/nginx/secrets', defaultMode=420) +
-    k.util.antiAffinity,
+    k.util.antiAffinity +
+    deployment.mixin.spec.strategy.rollingUpdate.withMaxSurge(5) +
+    deployment.mixin.spec.strategy.rollingUpdate.withMaxUnavailable(1),
 
   gateway_service:
     k.util.serviceFor($.gateway_deployment),

diff --git a/production/ksonnet/loki/querier.libsonnet b/production/ksonnet/loki/querier.libsonnet
@@ -34,7 +34,9 @@ local k = import 'ksonnet-util/kausal.libsonnet';
       $._config.overrides_configmap_mount_name,
       $._config.overrides_configmap_mount_path,
     ) +
-    k.util.antiAffinity
+    k.util.antiAffinity +
+    deployment.mixin.spec.strategy.rollingUpdate.withMaxSurge(5) +
+    deployment.mixin.spec.strategy.rollingUpdate.withMaxUnavailable(1)
   else {},
 
   // PVC for queriers when running as statefulsets

diff --git a/production/ksonnet/loki/query-frontend.libsonnet b/production/ksonnet/loki/query-frontend.libsonnet
@@ -35,7 +35,9 @@ local k = import 'ksonnet-util/kausal.libsonnet';
       $._config.overrides_configmap_mount_name,
       $._config.overrides_configmap_mount_path,
     ) +
-    k.util.antiAffinity,
+    k.util.antiAffinity +
+    deployment.mixin.spec.strategy.rollingUpdate.withMaxSurge(5) +
+    deployment.mixin.spec.strategy.rollingUpdate.withMaxUnavailable(1),
 
   local service = k.core.v1.service,
 

diff --git a/production/ksonnet/loki/query-scheduler.libsonnet b/production/ksonnet/loki/query-scheduler.libsonnet
@@ -51,7 +51,9 @@ local k = import 'ksonnet-util/kausal.libsonnet';
       $._config.overrides_configmap_mount_name,
       $._config.overrides_configmap_mount_path,
     ) +
-    k.util.antiAffinity
+    k.util.antiAffinity +
+    deployment.mixin.spec.strategy.rollingUpdate.withMaxSurge(5) +
+    deployment.mixin.spec.strategy.rollingUpdate.withMaxUnavailable(1)
   else {},
 
   local service = k.core.v1.service,

diff --git a/production/ksonnet/loki/ruler.libsonnet b/production/ksonnet/loki/ruler.libsonnet
@@ -44,7 +44,9 @@ local k = import 'ksonnet-util/kausal.libsonnet';
         $._config.overrides_configmap_mount_name,
         $._config.overrides_configmap_mount_path,
       ) +
-      k.util.antiAffinity
+      k.util.antiAffinity +
+      deployment.mixin.spec.strategy.rollingUpdate.withMaxSurge(5) +
+      deployment.mixin.spec.strategy.rollingUpdate.withMaxUnavailable(1)
     else {},
 
   ruler_service: if !$._config.ruler_enabled