Add ability to use Cilium CNI, Hubble, GatewayAPI

.github/workflows/ci.yml

@@ -47,7 +47,8 @@ jobs:
             playbook: converge.yml
           - distro: debian11
             playbook: converge.yml
-
+          - distro: debian11
+            playbook: cilium.yml
           - distro: debian11
             playbook: calico.yml
 

README.md

@@ -129,7 +129,7 @@ kubernetes_pod_network:
   # cidr: '192.168.0.0/16'
 ```
 
-This role currently supports `flannel` (default), `calico` or `weave` for cluster pod networking. Choose only one for your cluster; converting between them is not done automatically and could result in broken networking; if you need to switch from one to another, it should be done outside of this role.
+This role currently supports `flannel` (default), `cilium`, `calico` or `weave` for cluster pod networking. Choose only one for your cluster; converting between them is not done automatically and could result in broken networking; if you need to switch from one to another, it should be done outside of this role.
 
 ```yaml
 kubernetes_apiserver_advertise_address: ''`
@@ -168,6 +168,30 @@ kubernetes_calico_manifest_file: https://projectcalico.docs.tigera.io/manifests/
 
 Calico manifest file to apply to the Kubernetes cluster (if using Calico instead of Flannel).
 
+#### Cilium
+
+```yaml
+kubernetes_cilium_hubble_client: true
+kubernetes_cilium_values:
+  kubeProxyReplacement: true
+  gatewayAPI:
+    enabled: true
+```
+
+Cilium Helm chart values can be specified under `kubernetes_cilium_values`.
+
+##### Kube Proxy Replacement
+
+[Kube Proxy Replacement](https://docs.cilium.io/en/latest/network/kubernetes/kubeproxy-free/) is supported through this method. The is a prerequisite for Gateway API Support.
+
+##### Gateway API Support
+
+[Gateway API Support](https://docs.cilium.io/en/latest/network/servicemesh/gateway-api/gateway-api/#gs-gateway-api). This will install additional CRDs to support GatewayAPI.
+
+##### Hubble Observability
+
+In addition to setting up [Setting up Hubble Observability](https://docs.cilium.io/en/stable/gettingstarted/hubble_setup/#hubble-setup), you may choose to install the hubble client with `kubernetes_cilium_hubble_client: true`
+
 ## Dependencies
 
 None.

defaults/main.yml

@@ -11,6 +11,10 @@
 
 kubernetes_version: '1.25'
 kubernetes_version_rhel_package: '1.25.1'
+kubernetes_cilium_version: '1.14.5'
+kubernetes_cilium_datapath: 'native'
+kubernetes_cilium_hubble_client: false
+kubernetes_cilium_values: ""
 
 kubernetes_role: control_plane
 
@@ -53,7 +57,7 @@
 
 kubernetes_apt_release_channel: "stable"
 kubernetes_apt_keyring_file: "/etc/apt/keyrings/kubernetes-apt-keyring.asc"
 kubernetes_apt_repository: "deb [signed-by={{ kubernetes_apt_keyring_file }}] https://pkgs.k8s.io/core:/{{ kubernetes_apt_release_channel }}:/v{{ kubernetes_version }}/deb/ /"
 
 kubernetes_yum_base_url: "https://pkgs.k8s.io/core:/stable:/v{{ kubernetes_version }}/rpm/"
 kubernetes_yum_gpg_key: "https://pkgs.k8s.io/core:/stable:/v{{ kubernetes_version }}/rpm/repodata/repomd.xml.key"

molecule/default/cilium.yml

@@ -0,0 +1,74 @@
+---
+- name: Converge
+  hosts: all
+  become: true
+
+  vars:
+    kubernetes_cilium_hubble_client: true
+    kubernetes_cilium_values:
+      envoy:
+        enabled: true
+      kubeProxyReplacement: true
+      l7Proxy: true
+      loadBalancer:
+        l7:
+          backend: envoy
+      ingressController:
+        enabled: true
+        loadbalancerMode: dedicated
+        default: true
+      hubble:
+        relay:
+          enabled: true
+        ui:
+          enabled: true
+      gatewayAPI:
+        enabled: true
+    kubernetes_pod_network:
+      cni: 'cilium'
+      cidr: 10.244.0.0/16
+
+    # Allow swap in test environments (hard to control in some envs).
+    kubernetes_config_kubelet_configuration:
+      cgroupDriver: "systemd"
+      failSwapOn: false
+      cgroupsPerQOS: true
+      enforceNodeAllocatable: ['pods']
+    containerd_config_cgroup_driver_systemd: true
+
+  pre_tasks:
+    - name: Update apt cache.
+      apt: update_cache=true cache_valid_time=600
+      when: ansible_os_family == 'Debian'
+
+    - name: Ensure test dependencies are installed (RedHat).
+      package: name=iproute state=present
+      when: ansible_os_family == 'RedHat'
+
+    - name: Ensure test dependencies are installed (Debian).
+      package: name=iproute2 state=present
+      when: ansible_os_family == 'Debian'
+
+    - name: Gather facts.
+      action: setup
+
+  roles:
+    - role: geerlingguy.containerd
+    - role: geerlingguy.kubernetes
+
+  post_tasks:
+    - name: Get cluster info.
+      command: kubectl cluster-info
+      changed_when: false
+      register: kubernetes_info
+
+    - name: Print cluster info.
+      debug: var=kubernetes_info.stdout
+
+    - name: Get all running pods.
+      command: kubectl get pods --all-namespaces
+      changed_when: false
+      register: kubernetes_pods
+
+    - name: Print list of running pods.
+      debug: var=kubernetes_pods.stdout

tasks/cilium-client-setup.yml

@@ -0,0 +1,52 @@
+---
+- name: Check if Cilium CLI has already been Installed.
+  stat:
+    path: /usr/local/bin/cilium
+  register: cilium_init_stat
+  when:
+    - kubernetes_pod_network.cni == 'cilium'
+
+- name: Install Cilium CLI
+  when:
+    - kubernetes_pod_network.cni == 'cilium'
+    - not cilium_init_stat.stat.exists
+  block:
+    - name: Get Cilium CLI version
+      shell: curl -s https://raw.githubusercontent.com/cilium/cilium-cli/main/stable.txt
+      register: cilium_cli_version
+      changed_when: false
+
+    - name: Set CLI architecture
+      set_fact:
+        cli_arch: "{{ 'arm64' if ansible_architecture == 'aarch64' else 'amd64' }}"
+
+    - name: Download Cilium CLI
+      get_url:
+        url: "https://github.com/cilium/cilium-cli/releases/download/{{ cilium_cli_version.stdout }}/cilium-linux-{{ cli_arch }}.tar.gz"
+        dest: "/tmp/cilium-linux-{{ cli_arch }}.tar.gz"
+        mode: '0644'
+
+    - name: Download Cilium CLI checksum
+      get_url:
+        url: "https://github.com/cilium/cilium-cli/releases/download/{{ cilium_cli_version.stdout }}/cilium-linux-{{ cli_arch }}.tar.gz.sha256sum"
+        dest: "/tmp/cilium-linux-{{ cli_arch }}.tar.gz.sha256sum"
+        mode: '0644'
+
+    - name: Verify Cilium CLI checksum
+      shell: sha256sum --check /tmp/cilium-linux-{{ cli_arch }}.tar.gz.sha256sum
+      args:
+        chdir: /tmp
+
+    - name: Extract Cilium CLI
+      unarchive:
+        src: "/tmp/cilium-linux-{{ cli_arch }}.tar.gz"
+        dest: /usr/local/bin
+        remote_src: true
+
+    - name: Remove downloaded files
+      file:
+        path: "/tmp/cilium-linux-{{ cli_arch }}.tar.gz{{ item }}"
+        state: absent
+      loop:
+        - ''
+        - '.sha256sum'

tasks/control-plane-setup.yml

@@ -9,13 +9,27 @@
     src: "kubeadm-kubelet-config.j2"
     dest: "{{ kubernetes_kubeadm_kubelet_config_file_path }}"
 
+- name: Determine if we are replacing kube-proxy with cilium
+  set_fact: replace_kube_proxy="{{ (kubernetes_pod_network.cni == 'cilium') and
+    (kubernetes_cilium_values is defined) and
+    (kubernetes_cilium_values.kubeProxyReplacement is defined) and
+    (kubernetes_cilium_values.kubeProxyReplacement) | bool }}"
+
+- name: Determine if we installing Gateway API
+  set_fact: install_gateway_api="{{ (replace_kube_proxy) and
+    (kubernetes_cilium_values.gatewayAPI.enabled is defined) and
+    (kubernetes_cilium_values.gatewayAPI.enabled)| bool }}"
+
 - name: Initialize Kubernetes control plane with kubeadm init
   command: >
     kubeadm init
     --config {{ kubernetes_kubeadm_kubelet_config_file_path }}
     {{ kubernetes_kubeadm_init_extra_opts }}
   register: kubeadmin_init
-  when: (not kubernetes_init_stat.stat.exists) and (kubernetes_ignore_preflight_errors is not defined)
+  when:
+    - not kubernetes_init_stat.stat.exists
+    - kubernetes_ignore_preflight_errors is not defined
+    - not replace_kube_proxy
 
 - name: Initialize Kubernetes control plane with kubeadm init and ignore_preflight_errors
   command: >
@@ -24,7 +38,35 @@
     --ignore-preflight-errors={{ kubernetes_ignore_preflight_errors }}
     {{ kubernetes_kubeadm_init_extra_opts }}
   register: kubeadmin_init
-  when: (not kubernetes_init_stat.stat.exists) and (kubernetes_ignore_preflight_errors is defined)
+  when:
+    - not kubernetes_init_stat.stat.exists
+    - kubernetes_ignore_preflight_errors is defined
+    - not replace_kube_proxy
+
+- name: Initialize Kubernetes control plane with kubeadm init without kube-proxy
+  command: >
+    kubeadm init
+    --config {{ kubernetes_kubeadm_kubelet_config_file_path }}
+    --skip-phases=addon/kube-proxy
+    {{ kubernetes_kubeadm_init_extra_opts }}
+  register: kubeadmin_init
+  when:
+    - not kubernetes_init_stat.stat.exists
+    - kubernetes_ignore_preflight_errors is not defined
+    - replace_kube_proxy
+
+- name: Initialize Kubernetes control plane with kubeadm init without kube-proxy and ignore_preflight_errors
+  command: >
+    kubeadm init
+    --config {{ kubernetes_kubeadm_kubelet_config_file_path }}
+    --ignore-preflight-errors={{ kubernetes_ignore_preflight_errors }}
+    --skip-phases=addon/kube-proxy
+    {{ kubernetes_kubeadm_init_extra_opts }}
+  register: kubeadmin_init
+  when:
+    - not kubernetes_init_stat.stat.exists
+    - kubernetes_ignore_preflight_errors is defined
+    - replace_kube_proxy
 
 - name: Print the init output to screen.
   debug:
@@ -45,6 +87,75 @@
     state: link
     mode: 0644
 
+- name: Write Cilium Helm values to file
+  copy:
+    dest: "/tmp/cilium_helm.yaml"
+    content: "{{ kubernetes_cilium_values | to_yaml }}"
+  when:
+    - kubernetes_pod_network.cni == 'cilium'
+    - not kubernetes_init_stat.stat.exists
+
+- name: Taint nodes with cilium agent-not-ready
+  command: kubectl taint nodes --all node.cilium.io/agent-not-ready=true:NoExecute
+  when:
+    - kubernetes_pod_network.cni == 'cilium'
+    - not kubernetes_init_stat.stat.exists
+  register: cilium_taint
+  changed_when: "'tainted' in cilium_taint.stdout"
+  until: cilium_taint is not failed
+  retries: 12
+  delay: 5
+
+- name: Install Prerequisite CRDs for Cilium Gateway API support.
+  when: install_gateway_api
+  register: gateway_crds
+  changed_when: "'created' in gateway_crds.stdout"
+  command: "kubectl apply -f {{ item }}"
+  loop:
+    - https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.0.0/config/crd/standard/gateway.networking.k8s.io_gatewayclasses.yaml
+    - https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.0.0/config/crd/standard/gateway.networking.k8s.io_gateways.yaml
+    - https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.0.0/config/crd/standard/gateway.networking.k8s.io_httproutes.yaml
+    - https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.0.0/config/crd/standard/gateway.networking.k8s.io_referencegrants.yaml
+    - https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.0.0/config/crd/experimental/gateway.networking.k8s.io_grpcroutes.yaml
+    - https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.0.0/config/crd/experimental/gateway.networking.k8s.io_tlsroutes.yaml
+    - https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.0.0/config/crd/experimental/gateway.networking.k8s.io_tcproutes.yaml
+    - https://raw.githubusercontent.com/kubernetes-sigs/gateway-api/v1.0.0/config/crd/experimental/gateway.networking.k8s.io_udproutes.yaml
+
+- name: Configure Cilium networking
+  command: >
+    /usr/local/bin/cilium install
+    --version {{ kubernetes_cilium_version }}
+    --datapath-mode {{ kubernetes_cilium_datapath }}
+    --values /tmp/cilium_helm.yaml
+  when:
+    - kubernetes_pod_network.cni == 'cilium'
+    - not kubernetes_init_stat.stat.exists
+    - not replace_kube_proxy
+  register: cilium_result
+  changed_when: "'Error' not in cilium_result.stdout"
+  until: cilium_result is not failed
+  retries: 20
+  delay: 5
+
+- name: Configure Cilium networking and replace kube-proxy
+  command: >
+    /usr/local/bin/cilium install
+    --version {{ kubernetes_cilium_version }}
+    --datapath-mode {{ kubernetes_cilium_datapath }}
+    --values /tmp/cilium_helm.yaml
+    --set k8sServiceHost="{{ kubernetes_apiserver_advertise_address | default(ansible_default_ipv4.address, true) }}"
+    --set k8sServicePort=6443
+  when:
+    - kubernetes_pod_network.cni == 'cilium'
+    - not kubernetes_init_stat.stat.exists
+    - replace_kube_proxy
+
+  register: cilium_result
+  changed_when: "'Error' not in cilium_result.stdout"
+  until: cilium_result is not failed
+  retries: 20
+  delay: 5
+
 - name: Configure Flannel networking.
   command: "kubectl apply -f {{ kubernetes_flannel_manifest_file }}"
   register: flannel_result

tasks/hubble-client-setup.yml

@@ -0,0 +1,53 @@
+---
+- name: Check if Hubble CLI has already been Installed.
+  stat:
+    path: /usr/local/bin/hubble
+  register: hubble_init_stat
+  when:
+    - kubernetes_pod_network.cni == 'cilium'
+
+- name: Install Hubble CLI
+  when:
+    - kubernetes_pod_network.cni == 'cilium'
+    - not hubble_init_stat.stat.exists
+  block:
+    - name: Get Hubble CLI version
+      shell: curl -s https://raw.githubusercontent.com/cilium/hubble/master/stable.txt
+      register: hubble_cli_version
+      changed_when: false
+
+    - name: Set CLI architecture
+      set_fact:
+        cli_arch: "{{ 'arm64' if ansible_architecture == 'aarch64' else 'amd64' }}"
+
+
+    - name: Download Hubble CLI
+      get_url:
+        url: "https://github.com/cilium/hubble/releases/download/{{ hubble_cli_version.stdout }}/hubble-linux-{{ cli_arch }}.tar.gz"
+        dest: "/tmp/hubble-linux-{{ cli_arch }}.tar.gz"
+        mode: '0644'
+
+    - name: Download Hubble CLI checksum
+      get_url:
+        url: "https://github.com/cilium/hubble/releases/download/{{ hubble_cli_version.stdout }}/hubble-linux-{{ cli_arch }}.tar.gz.sha256sum"
+        dest: "/tmp/hubble-linux-{{ cli_arch }}.tar.gz.sha256sum"
+        mode: '0644'
+
+    - name: Verify Hubble CLI checksum
+      shell: sha256sum --check /tmp/hubble-linux-{{ cli_arch }}.tar.gz.sha256sum
+      args:
+        chdir: /tmp
+
+    - name: Extract Hubble CLI
+      unarchive:
+        src: "/tmp/hubble-linux-{{ cli_arch }}.tar.gz"
+        dest: /usr/local/bin
+        remote_src: true
+
+    - name: Remove downloaded files
+      file:
+        path: "/tmp/hubble-linux-{{ cli_arch }}.tar.gz{{ item }}"
+        state: absent
+      loop:
+        - ''
+        - '.sha256sum'