Add memory requests for nodes and controlplane workloads

This commit adds memory requests for all nodes and controlplane
workloads. The reason behind it is to better show user available
resources on both worker and controller nodes e.g. when doing 'kubectl
describe node'. This is important while one scales the controlplane
deployments and may prevent node eviction.

The measurment was done on freshly created cluster, with
prometheus-operator and metrics-server deployed,
on controller node and on worker node, so the numbers might be lower
than the numbers on long-running cluster, but they give at least some
initial visibility.

The values were measured using 'systemd-cgtop -m -1 / --depth=1', not
using 'free', as 'systemd-cgtop' also includes page cache usage, the
same way 'kubelet' is measuring the memory usage.

Before the measurment, following command has been executed:
'sync; echo 1 | sudo tee /proc/sys/vm/drop_caches; sleep 10'
To make sure only active memory has been captured.

system.slice uses ~250Mi, init.scope uses ~200Mi, which sums up to
roughly 500Mi, which is needed for system.

Kubelet in /docker slice was using ~100Mi. etcd in /docker slice was
using ~200Mi, so workers has 100Mi reserved for 'kube' and controllers
has 300Mi.

Memory usage for self-hosted components has been measured using the
following command: 'kubectl top pods --sort-by=memory | sort -h -k3 -r'.

Then, the read values were rounded up a bit.

Signed-off-by: Mateusz Gozdek <[email protected]>

assets/lokomotive-kubernetes/aws/flatcar-linux/kubernetes/cl/controller.yaml.tmpl

@@ -95,7 +95,9 @@ systemd:
           --pod-manifest-path=/etc/kubernetes/manifests \
           --read-only-port=0 \
           --register-with-taints=$${NODE_TAINTS} \
-          --volume-plugin-dir=/var/lib/kubelet/volumeplugins
+          --volume-plugin-dir=/var/lib/kubelet/volumeplugins \
+          --system-reserved=memory=500Mi \
+          --kube-reserved=$${KUBE_RESERVED}
         ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid
         Restart=always
         RestartSec=10
@@ -126,6 +128,7 @@ storage:
           KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet"
           NODE_LABELS="node.kubernetes.io/master,node.kubernetes.io/controller=true"
           NODE_TAINTS="node-role.kubernetes.io/master=:NoSchedule"
+          KUBE_RESERVED="memory=300Mi"
     - path: /etc/sysctl.d/max-user-watches.conf
       filesystem: root
       contents:

assets/lokomotive-kubernetes/aws/flatcar-linux/kubernetes/workers/cl/worker.yaml.tmpl

@@ -68,7 +68,9 @@ systemd:
           --node-labels=$${NODE_LABELS} \
           --pod-manifest-path=/etc/kubernetes/manifests \
           --read-only-port=0 \
-          --volume-plugin-dir=/var/lib/kubelet/volumeplugins
+          --volume-plugin-dir=/var/lib/kubelet/volumeplugins \
+          --system-reserved=memory=500Mi \
+          --kube-reserved=$${KUBE_RESERVED}
         ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid
         Restart=always
         RestartSec=5
@@ -103,6 +105,7 @@ storage:
           KUBELET_IMAGE_TAG=v1.18.0
           KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet"
           NODE_LABELS="node.kubernetes.io/node"
+          KUBE_RESERVED="memory=100Mi"
     - path: /etc/sysctl.d/max-user-watches.conf
       filesystem: root
       contents:

assets/lokomotive-kubernetes/azure/flatcar-linux/kubernetes/cl/controller.yaml.tmpl

@@ -91,7 +91,9 @@ systemd:
           --pod-manifest-path=/etc/kubernetes/manifests \
           --read-only-port=0 \
           --register-with-taints=$${NODE_TAINTS} \
-          --volume-plugin-dir=/var/lib/kubelet/volumeplugins
+          --volume-plugin-dir=/var/lib/kubelet/volumeplugins \
+          --system-reserved=memory=500Mi \
+          --kube-reserved=$${KUBE_RESERVED}
         ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid
         Restart=always
         RestartSec=10
@@ -128,6 +130,7 @@ storage:
           KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet"
           NODE_LABELS="node.kubernetes.io/master,node.kubernetes.io/controller=true"
           NODE_TAINTS="node-role.kubernetes.io/master=:NoSchedule"
+          KUBE_RESERVED="memory=300Mi"
     - path: /etc/sysctl.d/max-user-watches.conf
       filesystem: root
       contents:

assets/lokomotive-kubernetes/azure/flatcar-linux/kubernetes/workers/cl/worker.yaml.tmpl

@@ -66,7 +66,9 @@ systemd:
           --node-labels=$${NODE_LABELS} \
           --pod-manifest-path=/etc/kubernetes/manifests \
           --read-only-port=0 \
-          --volume-plugin-dir=/var/lib/kubelet/volumeplugins
+          --volume-plugin-dir=/var/lib/kubelet/volumeplugins \
+          --system-reserved=memory=500Mi \
+          --kube-reserved=$${KUBE_RESERVED}
         ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid
         Restart=always
         RestartSec=5
@@ -101,6 +103,7 @@ storage:
           KUBELET_IMAGE_TAG=v1.18.0
           KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet"
           NODE_LABELS="node.kubernetes.io/node"
+          KUBE_RESERVED="memory=100Mi"
     - path: /etc/sysctl.d/max-user-watches.conf
       filesystem: root
       contents:

assets/lokomotive-kubernetes/bare-metal/flatcar-linux/kubernetes/cl/controller.yaml.tmpl

@@ -104,7 +104,9 @@ systemd:
           --pod-manifest-path=/etc/kubernetes/manifests \
           --read-only-port=0 \
           --register-with-taints=$${NODE_TAINTS} \
-          --volume-plugin-dir=/var/lib/kubelet/volumeplugins
+          --volume-plugin-dir=/var/lib/kubelet/volumeplugins \
+          --system-reserved=memory=500Mi \
+          --kube-reserved=$${KUBE_RESERVED}
         ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid
         Restart=always
         RestartSec=10
@@ -133,6 +135,7 @@ storage:
           KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet"
           NODE_LABELS="node.kubernetes.io/master,node.kubernetes.io/controller=true"
           NODE_TAINTS="node-role.kubernetes.io/master=:NoSchedule"
+          KUBE_RESERVED="memory=300Mi"
     - path: /etc/hostname
       filesystem: root
       mode: 0644

assets/lokomotive-kubernetes/bare-metal/flatcar-linux/kubernetes/cl/worker.yaml.tmpl

@@ -74,7 +74,9 @@ systemd:
           --node-labels=$${NODE_LABELS} \
           --pod-manifest-path=/etc/kubernetes/manifests \
           --read-only-port=0 \
-          --volume-plugin-dir=/var/lib/kubelet/volumeplugins
+          --volume-plugin-dir=/var/lib/kubelet/volumeplugins \
+          --system-reserved=memory=500Mi \
+          --kube-reserved=$${KUBE_RESERVED}
         ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid
         Restart=always
         RestartSec=5
@@ -92,6 +94,7 @@ storage:
           KUBELET_IMAGE_TAG=v1.18.0
           KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet"
           NODE_LABELS="node.kubernetes.io/node"
+          KUBE_RESERVED="memory=100Mi"
     - path: /etc/hostname
       filesystem: root
       mode: 0644

assets/lokomotive-kubernetes/bootkube/resources/charts/calico/templates/daemonset.yaml

@@ -147,6 +147,7 @@ spec:
           resources:
             requests:
               cpu: 100m
+              memory: "60Mi"
           livenessProbe:
             exec:
               command:

assets/lokomotive-kubernetes/bootkube/resources/charts/kube-apiserver/templates/kube-apiserver.yaml

@@ -148,6 +148,9 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: status.podIP
+        resources:
+          requests:
+            memory: "500Mi"
         volumeMounts:
         - name: secrets
           mountPath: /etc/kubernetes/secrets

assets/lokomotive-kubernetes/bootkube/resources/charts/kubelet/templates/kubelet-ds.yaml

@@ -46,7 +46,9 @@ spec:
           --volume-plugin-dir=/var/lib/kubelet/volumeplugins \
           --node-labels=$(grep NODE_LABELS /etc/kubernetes/kubelet.env | cut -d'"' -f2) \
           --register-with-taints=$(grep NODE_TAINTS /etc/kubernetes/kubelet.env | cut -d'"' -f2) \
-          --address=$(HOST_IP)
+          --address=$(HOST_IP) \
+          --system-reserved=memory=500Mi \
+          --kube-reserved=$(grep KUBE_RESERVED /etc/kubernetes/kubelet.env | cut -d'"' -f2)
           " | tee /tmp/start-kubelet.sh &&
           chmod +x /tmp/start-kubelet.sh &&
           /tmp/start-kubelet.sh
@@ -103,6 +105,9 @@ spec:
           mountPath: /usr/sbin/iscsiadm
         - name: modules
           mountPath: /lib/modules
+        resources:
+          requests:
+            memory: "100Mi"
       hostNetwork: true
       hostPID: true
       # Tolerate all the taints. This ensures that the pod runs on all the nodes.

assets/lokomotive-kubernetes/bootkube/resources/charts/kubernetes/templates/kube-controller-manager.yaml

@@ -82,6 +82,9 @@ spec:
         - name: ssl-host
           mountPath: /etc/ssl/certs
           readOnly: true
+        resources:
+          requests:
+            memory: "50Mi"
       volumes:
       - name: secrets
         secret:

assets/lokomotive-kubernetes/bootkube/resources/charts/kubernetes/templates/kube-proxy.yaml

@@ -69,6 +69,9 @@ spec:
         - name: ssl-certs-host
           mountPath: /etc/ssl/certs
           readOnly: true
+        resources:
+          requests:
+            memory: "20Mi"
       volumes:
       - name: kubeconfig
         configMap:

assets/lokomotive-kubernetes/bootkube/resources/charts/kubernetes/templates/kube-scheduler.yaml

@@ -60,3 +60,6 @@ spec:
             port: 10259
           initialDelaySeconds: 15
           timeoutSeconds: 15
+        resources:
+          requests:
+            memory: "30Mi"

assets/lokomotive-kubernetes/bootkube/resources/charts/pod-checkpointer/templates/daemonset.yaml

@@ -60,6 +60,9 @@ spec:
           mountPath: /etc/kubernetes
         - name: var-run
           mountPath: /var/run
+        resources:
+          requests:
+            memory: "5Mi"
       volumes:
       - name: kubeconfig
         configMap:

assets/lokomotive-kubernetes/google-cloud/flatcar-linux/kubernetes/cl/controller.yaml.tmpl

@@ -92,7 +92,9 @@ systemd:
           --pod-manifest-path=/etc/kubernetes/manifests \
           --register-with-taints=$${NODE_TAINTS} \
           --read-only-port=0 \
-          --volume-plugin-dir=/var/lib/kubelet/volumeplugins
+          --volume-plugin-dir=/var/lib/kubelet/volumeplugins \
+          --system-reserved=memory=500Mi \
+          --kube-reserved=$${KUBE_RESERVED}
         ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid
         Restart=always
         RestartSec=10
@@ -129,6 +131,7 @@ storage:
           KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet"
           NODE_LABELS="node.kubernetes.io/master,node.kubernetes.io/controller=true"
           NODE_TAINTS="node-role.kubernetes.io/master=:NoSchedule"
+          KUBE_RESERVED="memory=300Mi"
     - path: /etc/sysctl.d/max-user-watches.conf
       filesystem: root
       contents:

assets/lokomotive-kubernetes/google-cloud/flatcar-linux/kubernetes/workers/cl/worker.yaml.tmpl

@@ -62,7 +62,9 @@ systemd:
           --node-labels=$${NODE_LABELS} \
           --pod-manifest-path=/etc/kubernetes/manifests \
           --read-only-port=0 \
-          --volume-plugin-dir=/var/lib/kubelet/volumeplugins
+          --volume-plugin-dir=/var/lib/kubelet/volumeplugins \
+          --system-reserved=memory=500Mi \
+          --kube-reserved=$${KUBE_RESERVED}
         ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid
         Restart=always
         RestartSec=5
@@ -97,6 +99,7 @@ storage:
           KUBELET_IMAGE_TAG=v1.18.0
           KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet"
           NODE_LABELS="node.kubernetes.io/node"
+          KUBE_RESERVED="memory=100Mi"
     - path: /etc/sysctl.d/max-user-watches.conf
       filesystem: root
       contents:

assets/lokomotive-kubernetes/kvm-libvirt/flatcar-linux/kubernetes/cl/controller.yaml.tmpl

@@ -94,7 +94,9 @@ systemd:
           --pod-manifest-path=/etc/kubernetes/manifests \
           --read-only-port=0 \
           --register-with-taints=$${NODE_TAINTS} \
-          --volume-plugin-dir=/var/lib/kubelet/volumeplugins
+          --volume-plugin-dir=/var/lib/kubelet/volumeplugins \
+          --system-reserved=memory=500Mi \
+          --kube-reserved=$${KUBE_RESERVED}
         ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid
         Restart=always
         RestartSec=10
@@ -131,6 +133,7 @@ storage:
           KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet"
           NODE_LABELS="node.kubernetes.io/master,node.kubernetes.io/controller=true"
           NODE_TAINTS="node-role.kubernetes.io/master=:NoSchedule"
+          KUBE_RESERVED="memory=300Mi"
     - path: /etc/sysctl.d/max-user-watches.conf
       filesystem: root
       contents:

assets/lokomotive-kubernetes/kvm-libvirt/flatcar-linux/kubernetes/workers/cl/worker.yaml.tmpl

@@ -69,7 +69,9 @@ systemd:
           --node-labels=$${NODE_LABELS} \
           --pod-manifest-path=/etc/kubernetes/manifests \
           --read-only-port=0 \
-          --volume-plugin-dir=/var/lib/kubelet/volumeplugins
+          --volume-plugin-dir=/var/lib/kubelet/volumeplugins \
+          --system-reserved=memory=500Mi \
+          --kube-reserved=$${KUBE_RESERVED}
         ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid
         Restart=always
         RestartSec=5
@@ -104,6 +106,7 @@ storage:
           KUBELET_IMAGE_TAG=v1.18.0
           KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet"
           NODE_LABELS="node.kubernetes.io/node"
+          KUBE_RESERVED="memory=100Mi"
     - path: /etc/sysctl.d/max-user-watches.conf
       filesystem: root
       contents:

assets/lokomotive-kubernetes/packet/flatcar-linux/kubernetes/cl/controller.yaml.tmpl

@@ -130,7 +130,9 @@ systemd:
           --read-only-port=0 \
           --register-with-taints=$${NODE_TAINTS} \
           --address=$${COREOS_PACKET_IPV4_PRIVATE_0} \
-          --volume-plugin-dir=/var/lib/kubelet/volumeplugins
+          --volume-plugin-dir=/var/lib/kubelet/volumeplugins \
+          --system-reserved=memory=500Mi \
+          --kube-reserved=$${KUBE_RESERVED}
         ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid
         Restart=always
         RestartSec=10
@@ -173,6 +175,7 @@ storage:
           KUBELET_IMAGE_ARGS="--exec=/usr/local/bin/kubelet"
           NODE_LABELS="node.kubernetes.io/master,node.kubernetes.io/controller=true"
           NODE_TAINTS="node-role.kubernetes.io/master=:NoSchedule"
+          KUBE_RESERVED="memory=300Mi"
     - path: /etc/sysctl.d/max-user-watches.conf
       filesystem: root
       contents:

assets/lokomotive-kubernetes/packet/flatcar-linux/kubernetes/workers/cl/worker.yaml.tmpl

@@ -108,7 +108,9 @@ systemd:
             --read-only-port=0 \
             --register-with-taints=$${NODE_TAINTS} \
             --address=$${COREOS_PACKET_IPV4_PRIVATE_0} \
-            --volume-plugin-dir=/var/lib/kubelet/volumeplugins"
+            --volume-plugin-dir=/var/lib/kubelet/volumeplugins \
+            --system-reserved=memory=500Mi \
+            --kube-reserved=$${KUBE_RESERVED}
         ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid
         Restart=always
         RestartSec=5
@@ -281,6 +283,7 @@ storage:
           NODE_LABELS="node.kubernetes.io/node,${node_labels}"
           BGP_NODE_LABELS="${bgp_node_labels}"
           NODE_TAINTS="${taints}"
+          KUBE_RESERVED="memory=100Mi"
     - path: /etc/sysctl.d/max-user-watches.conf
       filesystem: root
       contents: