fix(cilium): turn on l2-announcements (ARP) again to troubleshoot connection problems

I'm experiencing issues reaching services directly exposed to the internet

At first only some carriers didn't load the pages, but it appears to be getting worse somehow

Signed-off-by: Vegard Hagen <[email protected]>

Author: vehagn

k8s/infra/network/cilium/kustomization.yaml

@@ -9,6 +9,8 @@ resources:
   - bgp-cluster-config.yaml
   - dashboards/cilium.yaml
   - dashboards/cilium-operator.yaml
+  - l2-announce.yaml
+  - l2-ip-pool.yaml
 
 helmCharts:
   - name: cilium

k8s/infra/network/cilium/l2-announce.yaml

@@ -0,0 +1,8 @@
+apiVersion: cilium.io/v2alpha1
+kind: CiliumL2AnnouncementPolicy
+metadata:
+  name: default-l2-announcement-policy
+  namespace: kube-system
+spec:
+  externalIPs: true
+  loadBalancerIPs: true

k8s/infra/network/cilium/l2-ip-pool.yaml

@@ -0,0 +1,11 @@
+apiVersion: cilium.io/v2
+kind: CiliumLoadBalancerIPPool
+metadata:
+  name: first-pool
+spec:
+  blocks:
+    - start: 192.168.1.220
+      stop: 192.168.1.255
+  serviceSelector:
+    matchLabels:
+      l2.cilium.io/ip-pool: default

k8s/infra/network/cilium/values.yaml

@@ -59,6 +59,13 @@ resources:
 #debug:
 #  enabled: true
 
+k8sClientRateLimit:
+  qps: 20
+  burst: 100
+
+l2announcements:
+  enabled: true
+
 # https://docs.cilium.io/en/latest/network/bgp-control-plane/bgp-control-plane-v2/
 bgpControlPlane:
   enabled: true

k8s/infra/network/dns/unbound/svc.yaml

@@ -10,6 +10,7 @@ metadata:
     lb-ipam.cilium.io/ip-pool: default-bgp
 spec:
   type: LoadBalancer
+  externalTrafficPolicy: Local
   # https://kubernetes.io/docs/concepts/services-networking/cluster-ip-allocation/
   clusterIP: 10.96.0.11
   ports:

k8s/infra/network/gateway/gw-external.yaml

@@ -9,9 +9,12 @@ spec:
     labels:
       bgp.cilium.io/advertise-service: default
       lb-ipam.cilium.io/ip-pool: default-bgp
+      l2.cilium.io/ip-pool: default
   addresses:
     - type: IPAddress
       value: 172.20.10.110
+    - type: IPAddress
+      value: 192.168.1.222
   listeners:
     - protocol: HTTPS
       port: 443