Merge pull request #2043 from cybozu-go/enable-cilium-l4lb

Enable Cilium L4LB
cybozu-go · Aug 1, 2022 · 3273ef6 · 3273ef6
2 parents 693d8ae + cd0b7f0
commit 3273ef6
Show file tree

Hide file tree

Showing 10 changed files with 432 additions and 12 deletions.
diff --git a/cilium/pre/bgp_config.yaml b/cilium/pre/bgp_config.yaml
@@ -0,0 +1,26 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: bgp-config
+  namespace: kube-system
+data:
+  config.yaml: |
+    peers:
+    - peer-address: 127.0.0.1
+      peer-asn: 64699
+      my-asn: 64698
+    address-pools:
+    - name: default
+      protocol: bgp
+      addresses:
+      - {{ .lbAddressDefault }}
+    - name: bastion
+      protocol: bgp
+      addresses:
+      - {{ .lbAddressBastion }}
+      auto-assign: false
+    - name: internet
+      protocol: bgp
+      addresses:
+      - {{ .lbAddressInternet }}
+      auto-assign: false
diff --git a/cilium/pre/kustomization.yaml b/cilium/pre/kustomization.yaml
@@ -2,4 +2,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - ../base
+  - bgp_config.yaml
   - upstream.yaml
diff --git a/cilium/pre/upstream.yaml b/cilium/pre/upstream.yaml
@@ -104,6 +104,7 @@ data:
   # Specifies the ratio (0.0-1.0) of total system memory to use for dynamic
   # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps.
   bpf-map-dynamic-size-ratio: "0.0025"
+  enable-host-legacy-routing: "true"
   # bpf-policy-map-max specifies the maximum number of entries in endpoint
   # policy map (per endpoint)
   bpf-policy-map-max: "16384"
@@ -177,10 +178,21 @@ data:
   # host-firewall and BPF masquerading)
   devices: "eth+ eno1+ eno2+"
 
-  kube-proxy-replacement:  "disabled"
+  kube-proxy-replacement:  "partial"
+  kube-proxy-replacement-healthz-bind-address: ""
+  enable-host-reachable-services: "true"
+  bpf-lb-sock-hostns-only: "true"
+  enable-host-port: "false"
+  enable-external-ips: "true"
+  enable-node-port: "true"
+  direct-routing-device: "e+"
   enable-health-check-nodeport: "true"
   node-port-bind-protection: "true"
   enable-auto-protect-node-port-range: "true"
+  bpf-lb-mode: "dsr"
+  bpf-lb-algorithm: "maglev"
+  bpf-lb-acceleration: "disabled"
+  bpf-lb-maglev-hash-seed: "3HCx6JennjWtot2U"
   enable-session-affinity: "true"
   enable-l2-neigh-discovery: "true"
   arping-refresh-period: "30s"
@@ -218,6 +230,7 @@ data:
   cluster-pool-ipv4-cidr: "10.0.0.0/8"
   cluster-pool-ipv4-mask-size: "24"
   disable-cnp-status-updates: "true"
+  bgp-announce-lb-ip: "true"
   cgroup-root: "/run/cilium/cgroupv2"
   enable-k8s-terminating-endpoint: "true"
   annotate-k8s-node: "true"
@@ -614,7 +627,7 @@ spec:
         prometheus.io/port: "9090"
         prometheus.io/scrape: "true"
         # ensure pods roll when configmap updates
-        cilium.io/cilium-configmap-checksum: "31b347a67277e80a7c19349a86db7ebc2edffb39a58175dad27d049feabb3d4f"
+        cilium.io/cilium-configmap-checksum: "27e2fa4fbcb9b18d162c6531b4bc8160031da83e4e99c6eab77f4cfd89d443de"
       labels:
         k8s-app: cilium
     spec:
@@ -712,6 +725,10 @@ spec:
               name: cilium-config
               key: custom-cni-conf
               optional: true
+        - name: KUBERNETES_SERVICE_HOST
+          value: "127.0.0.1"
+        - name: KUBERNETES_SERVICE_PORT
+          value: "16443"
         lifecycle:
           postStart:
             exec:
@@ -764,6 +781,9 @@ spec:
           readOnly: true
         - name: xtables-lock
           mountPath: /run/xtables.lock
+        - name: bgp-config-path
+          mountPath: /var/lib/cilium/bgp
+          readOnly: true
         - name: hubble-tls
           mountPath: /var/lib/cilium/tls/hubble
           readOnly: true
@@ -816,6 +836,10 @@ spec:
               name: cilium-config
               key: clean-cilium-bpf-state
               optional: true
+        - name: KUBERNETES_SERVICE_HOST
+          value: "127.0.0.1"
+        - name: KUBERNETES_SERVICE_PORT
+          value: "16443"
         securityContext:
           privileged: true
         volumeMounts:
@@ -889,6 +913,9 @@ spec:
       - name: cilium-config-path
         configMap:
           name: cilium-config
+      - name: bgp-config-path
+        configMap:
+          name: bgp-config
       - name: hubble-tls
         projected:
           # note: the leading zero means this number is in octal representation: do not remove it
@@ -931,7 +958,7 @@ spec:
     metadata:
       annotations:
         # ensure pods roll when configmap updates
-        cilium.io/cilium-configmap-checksum: "31b347a67277e80a7c19349a86db7ebc2edffb39a58175dad27d049feabb3d4f"
+        cilium.io/cilium-configmap-checksum: "27e2fa4fbcb9b18d162c6531b4bc8160031da83e4e99c6eab77f4cfd89d443de"
         prometheus.io/port: "6942"
         prometheus.io/scrape: "true"
       labels:
@@ -976,6 +1003,10 @@ spec:
               key: debug
               name: cilium-config
               optional: true
+        - name: KUBERNETES_SERVICE_HOST
+          value: "127.0.0.1"
+        - name: KUBERNETES_SERVICE_PORT
+          value: "16443"
         ports:
         - name: prometheus
           containerPort: 6942
@@ -994,6 +1025,9 @@ spec:
         - name: cilium-config-path
           mountPath: /tmp/cilium/config-map
           readOnly: true
+        - name: bgp-config-path
+          mountPath: /var/lib/cilium/bgp
+          readOnly: true
         resources:
           requests:
             cpu: 100m
@@ -1010,6 +1044,9 @@ spec:
       - name: cilium-config-path
         configMap:
           name: cilium-config
+      - name: bgp-config-path
+        configMap:
+          name: bgp-config
 ---
 # Source: cilium/templates/hubble-relay/deployment.yaml
 apiVersion: apps/v1

diff --git a/cilium/pre/values.yaml b/cilium/pre/values.yaml
@@ -1,10 +1,23 @@
+bgp:
+  announce:
+    loadbalancerIP: true
+  enabled: true
+bpf:
+  hostRouting: true
 cni:
   chainingMode: "generic-veth"
   customConf: true
 datapathMode: "veth"
 devices: "eth+ eno1+ eno2+"
 enableIPv4Masquerade: false
 enableIdentityMark: false
+externalIPs:
+  enabled: true
+hostPort:
+  enabled: false
+hostServices:
+  enabled: true
+  hostNamespaceOnly: true
 hubble:
   relay:
     enabled: true
@@ -27,7 +40,19 @@ hubble:
       - "flow:destinationContext=pod|dns|ip;sourceContext=pod|dns|ip"
       - "icmp"
       - "http"
-kubeProxyReplacement: "disabled"
+k8sServiceHost: 127.0.0.1
+k8sServicePort: 16443
+kubeProxyReplacement: "partial"
+loadBalancer:
+  # We can't enable XDP Acceleration because of the issue https://github.com/cilium/cilium/issues/19453
+  acceleration: disabled
+  algorithm: maglev
+  mode: dsr
+maglev:
+  hashSeed: 3HCx6JennjWtot2U
+nodePort:
+  directRoutingDevice: "e+"
+  enabled: true
 operator:
   rollOutPods: true
   prometheus: