diff --git a/README.md b/README.md index f6318766c..697930404 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,7 @@ For deploying telco-cloud with cloud-native functions, loxilb can be used as a S #### loxilb as in-cluster pod - [K3s : loxilb in-cluster mode](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/k3s_quick_start_incluster.md) - [K0s : loxilb in-cluster mode](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/k0s_quick_start_incluster.md) +- [MicroK8s : loxilb in-cluster mode](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/microk8s_quick_start_incluster.md) ## Knowledge-Base - [What is eBPF](ebpf.md) diff --git a/cicd/microk8s-incluster/Vagrantfile b/cicd/microk8s-incluster/Vagrantfile new file mode 100644 index 000000000..9d6f61d0d --- /dev/null +++ b/cicd/microk8s-incluster/Vagrantfile @@ -0,0 +1,28 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +k3snodes = (ENV['K8SNodes'] || "1").to_i +box_name = (ENV['VAGRANT_BOX'] || "sysnet4admin/Ubuntu-k8s") +box_version = "0.7.1" +Vagrant.configure("2") do |config| + config.vm.box = "#{box_name}" + config.vm.box_version = "#{box_version}" + + if Vagrant.has_plugin?("vagrant-vbguest") + config.vbguest.auto_update = false + end + + (1..k3snodes).each do |node_number| + config.vm.define "mk8s-node#{node_number}" do |node| + node.vm.hostname = "mk8s-node#{node_number}" + ip = node_number + 1 + node.vm.network :private_network, ip: "192.168.82.#{ip}", :netmask => "255.255.255.0" + node.vm.provision :shell, :path => "mk8s-install.sh" + node.vm.provider :virtualbox do |vbox| + vbox.customize ["modifyvm", :id, "--memory", 16000] + vbox.customize ["modifyvm", :id, "--cpus", 8] + vbox.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"] + end + end + end +end diff --git a/cicd/microk8s-incluster/common.sh b/cicd/microk8s-incluster/common.sh new file mode 100755 index 000000000..b82f6e0e4 --- /dev/null +++ b/cicd/microk8s-incluster/common.sh @@ -0,0 +1,555 @@ +#!/bin/bash + +if [[ "$1" == "init" ]]; then + pull_dockers +fi + +hn="netns" +pid="" +vrn="/var/run/" +hexec="sudo ip netns exec " +dexec="sudo docker exec -i " +hns="sudo ip netns " +hexist="$vrn$hn" +lxdocker="ghcr.io/loxilb-io/loxilb:latest" +var=$(lsb_release -r | cut -f2) +if [[ $var == *"22.04"* ]];then + lxdocker="ghcr.io/loxilb-io/loxilb:latestU22" +fi + +loxilbs=() + +## Given a docker name(arg1), return its pid +get_docker_pid() { + id=`docker ps -f name=$1| grep -w $1 | cut -d " " -f 1 | grep -iv "CONTAINER"` + pid=`docker inspect -f '{{.State.Pid}}' $id` +} + +## Pull all necessary dockers for testbed +pull_dockers() { + ## loxilb docker + docker pull $lxdocker + ## Host docker + docker pull eyes852/ubuntu-iperf-test:0.5 + ## BGP host docker + docker pull ewindisch/quagga + ## Keepalive docker + docker pull osixia/keepalived:2.0.20 +} + +## Creates a docker host +## arg1 - "loxilb"|"host" +## arg2 - instance-name +spawn_docker_host() { + POSITIONAL_ARGS=() + local bpath + local kpath + local ka + local bgp + while [[ $# -gt 0 ]]; do + case "$1" in + -t | --dock-type ) + dtype="$2" + shift 2 + ;; + -d | --dock-name ) + dname="$2" + shift 2 + ;; + -b | --with-bgp ) + if [[ "$2" == "yes" ]]; then + bgp=$2 + fi + shift 2 + ;; + -c | --bgp-config ) + bpath="$2" + bgp="yes" + shift 2 + ;; + -k | --with-ka ) + ka="in" + if [[ "$2" == "out" ]]; then + ka=$2 + fi + shift 2 + ;; + -d | --ka-config ) + kpath="$2" + if [[ -z ${ka+x} ]]; then + ka="in" + fi + shift 2 + ;; + -*|--*) + echo "Unknown option $1" + exit + ;; + esac + done + set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters + echo "Spawning $dname($dtype)" >&2 + if [[ "$dtype" == "loxilb" ]]; then + loxilbs+=("$dname") + if [[ "$pick_config" == "yes" ]]; then + echo "$dname will pick config from $(pwd)/${dname}_config" + loxilb_config="-v $(pwd)/${dname}_config:/etc/loxilb/" + fi + if [[ "$bgp" == "yes" ]]; then + bgp_opts="-b" + if [[ ! -z "$bpath" ]]; then + bgp_conf="-v $bpath:/etc/gobgp/" + fi + fi + if [[ "$dname" == "llb1" ]]; then + cluster_opts=" --cluster=172.17.0.3 --self=0" + elif [[ "$dname" == "llb2" ]]; then + cluster_opts=" --cluster=172.17.0.2 --self=1" + fi + + if [[ ! -z ${ka+x} ]]; then + sudo mkdir -p /etc/shared/$dname/ + if [[ "$ka" == "in" ]];then + ka_opts="-k in" + if [[ ! -z "$kpath" ]]; then + ka_conf="-v $kpath:/etc/keepalived/" + fi + fi + docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dt --entrypoint /bin/bash $bgp_conf -v /dev/log:/dev/log -v /etc/shared/$dname:/etc/shared $loxilb_config $ka_conf --name $dname $lxdocker + docker exec -dt $dname /root/loxilb-io/loxilb/loxilb $bgp_opts $cluster_opts $ka_opts + + if [[ "$ka" == "out" ]];then + ka_opts="-k out" + if [[ ! -z "$kpath" ]]; then + ka_conf="-v $kpath:/container/service/keepalived/assets/" + fi + + docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dit --network=container:$dname $ka_conf -v /etc/shared/$dname:/etc/shared --name ka_$dname osixia/keepalived:2.0.20 + fi + else + docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dt --entrypoint /bin/bash $bgp_conf -v /dev/log:/dev/log $loxilb_config --name $dname $lxdocker $bgp_opts + docker exec -dt $dname /root/loxilb-io/loxilb/loxilb $bgp_opts $cluster_opts + fi + elif [[ "$dtype" == "host" ]]; then + if [[ ! -z "$bpath" ]]; then + bgp_conf="--volume $bpath:/etc/quagga" + fi + if [[ "$bgp" == "yes" || ! -z "$bpath" ]]; then + docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dit $bgp_conf --name $dname ewindisch/quagga + else + docker run -u root --cap-add SYS_ADMIN -dit --name $dname eyes852/ubuntu-iperf-test:0.5 + fi + fi + + pid="" + + sleep 2 + get_docker_pid $dname + echo $pid + if [ ! -f "$hexist/$dname" -a "$pid" != "" ]; then + sudo mkdir -p /var/run/netns + sudo touch /var/run/netns/$dname + #echo "sudo mount -o bind /proc/$pid/ns/net /var/run/netns/$2" + sudo mount -o bind /proc/$pid/ns/net /var/run/netns/$dname + fi + + $hexec $dname ifconfig lo up + $hexec $dname sysctl net.ipv6.conf.all.disable_ipv6=1 2>&1 >> /dev/null + #$hexec $dname sysctl net.ipv4.conf.all.arp_accept=1 2>&1 >> /dev/null + $hexec $dname sysctl net.ipv4.conf.eth0.arp_ignore=2 2>&1 >> /dev/null +} + +## Deletes a docker host +## arg1 - hostname +delete_docker_host() { + id=`docker ps -f name=$1| grep -w $1 | cut -d " " -f 1 | grep -iv "CONTAINER"` + if [ "$id" != "" ]; then + docker stop $1 2>&1 >> /dev/null + hd="true" + ka=`docker ps -f name=ka_$1| grep -w ka_$1 | cut -d " " -f 1 | grep -iv "CONTAINER"` + loxilbs=( "${loxilbs[@]/$1}" ) + if [ "$ka" != "" ]; then + docker stop ka_$1 2>&1 >> /dev/null + docker rm ka_$1 2>&1 >> /dev/null + fi + fi + if [ -f "$hexist/$1" ]; then + $hns del $1 + sudo rm -fr "$hexist/$1" 2>&1 >> /dev/null + fi + if [ "$id" != "" ]; then + docker rm $1 2>&1 >> /dev/null + fi +} + +## Connects two docker hosts +## arg1 - hostname1 +## arg2 - hostname2 +## arg3 - mtu +connect_docker_hosts() { + link1=e$1$2 + link2=e$2$1 + + mtu="9000" + if [[ $# -gt 2 ]]; then + mtu=$3 + fi + + #echo $link1 $link2 + sudo ip -n $1 link add $link1 type veth peer name $link2 netns $2 + sudo ip -n $1 link set $link1 mtu $mtu up + sudo ip -n $2 link set $link2 mtu $mtu up +} + +## arg1 - hostname1 +## arg2 - hostname2 +disconnect_docker_hosts() { + link1=e$1$2 + link2=e$2$1 + # echo $link1 $link2 + if [ -f "$hexist/$1" ]; then + ifexist1=`sudo ip -n $1 link show $link1 | grep -w $link1` + if [ "chk$ifexist1" != "chk" ]; then + sudo ip -n $1 link set $link1 down 2>&1 >> /dev/null + sudo ip -n $1 link del $link1 2>&1 >> /dev/null + fi + fi + + if [ -f "$hexist/$2" ]; then + ifexist2=`sudo ip -n $2 link show | grep -w $link2` + if [ "chk$ifexist2" != "chk" ]; then + sudo ip -n $2 link set $link2 down 2>&1 >> /dev/null + sudo ip -n $2 link del $link2 2>&1 >> /dev/null + fi + fi +} + +## arg1 - hostname1 +## arg2 - hostname2 +## arg3 - ip_addr +## arg4 - gw +config_docker_host() { + POSITIONAL_ARGS=() + while [[ $# -gt 0 ]]; do + case $1 in + --host1) + local h1="$2" + shift + shift + ;; + --host2) + local h2="$2" + shift + shift + ;; + --ptype) + local ptype="$2" + shift + shift + ;; + --id) + local xid="$2" + shift + shift + ;; + --addr) + local addr="$2" + shift + shift + ;; + --gw) + local gw="$2" + shift + shift + ;; + -*|--*) + echo "Unknown option $1" + exit 1 + ;; + *) + POSITIONAL_ARGS+=("$1") # save positional arg + shift # past argument + ;; + esac + done + set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters + + link1=e$h1$h2 + link2=e$h2$h1 + #echo "$h1:$link1->$h2:$link2" + + #if [[ -n "${loxilbs[$h1]}" && "$pick_config" == "yes" ]]; then + if [[ ${loxilbs[*]} =~ (^|[[:space:]])$h1($|[[:space:]]) && "$pick_config" == "yes" ]]; then + return + fi + + if [[ "$ptype" == "phy" ]]; then + sudo ip -n $h1 addr add $addr dev $link1 + elif [[ "$ptype" == "vlan" ]]; then + sudo ip -n $h1 addr add $addr dev vlan$xid + elif [[ "$ptype" == "vxlan" ]]; then + sudo ip -n $h1 addr add $addr dev vxlan$xid + elif [[ "$ptype" == "trunk" ]]; then + trunk="bond$xid" + sudo ip -n $h1 link set $link1 down + sudo ip -n $h1 link add $trunk type bond + sudo ip -n $h1 link set $link1 master $trunk + sudo ip -n $h1 link set $link1 up + sudo ip -n $h1 link set $trunk up + + sudo ip -n $h2 link set $link2 down + sudo ip -n $h2 link add $trunk type bond + sudo ip -n $h2 link set $link2 master $trunk + sudo ip -n $h2 link set $link2 up + sudo ip -n $h2 link set $trunk up + + sudo ip -n $h1 addr add $addr dev bond$xid + if [[ "$gw" != "" ]]; then + sudo ip -n $h2 addr add $gw/24 dev bond$xid + sudo ip -n $h1 route add default via $gw proto static + fi + else + echo "Check port-type" + fi + + if [[ "$gw" != "" ]]; then + sudo ip -n $h1 route del default 2>&1 >> /dev/null + sudo ip -n $h1 route add default via $gw + fi +} + +## arg1 - hostname1 +## arg2 - hostname2 +## arg3 - vlan +## arg4 - tagged/untagged +create_docker_host_vlan() { + local addr="" + POSITIONAL_ARGS=() + while [[ $# -gt 0 ]]; do + case $1 in + --host1) + local h1="$2" + shift + shift + ;; + --host2) + local h2="$2" + shift + shift + ;; + --ptype) + local ptype="$2" + shift + shift + ;; + --id) + local vid="$2" + shift + shift + ;; + --addr) + addr="$2" + shift + shift + ;; + -*|--*) + echo "Unknown option $1" + exit 1 + ;; + *) + POSITIONAL_ARGS+=("$1") # save positional arg + shift # past argument + ;; + esac + done + + if [[ ${loxilbs[*]} =~ (^|[[:space:]])$h1($|[[:space:]]) && "$pick_config" == "yes" ]]; then + return + fi + + set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters + link1=e$h1$h2 + link2=e$h2$h1 + + #echo "$h1:$link1->$h2:$link2" + + if [[ "$ptype" == "tagged" ]]; then + brport="$link1.$vid" + sudo ip -n $h1 link add link $link1 name $brport type vlan id $vid + sudo ip -n $h1 link set $brport up + else + brport=$link1 + fi + + sudo ip -n $h1 link add vlan$vid type bridge 2>&1 | true + sudo ip -n $h1 link set $brport master vlan$vid + sudo ip -n $h1 link set vlan$vid up + if [[ "$addr" != "" ]]; then + sudo ip -n $h1 addr add $addr dev vlan$vid + fi +} + +## arg1 - hostname1 +## arg2 - hostname2 +## arg3 - vxlan-id +## arg4 - phy/vlan +## arg5 - local ip if arg4 is phy/vlan-id if arg4 is vlan +## arg6 - local ip if arg4 is vlan +create_docker_host_vxlan() { + POSITIONAL_ARGS=() + while [[ $# -gt 0 ]]; do + case $1 in + --host1) + local h1="$2" + shift + shift + ;; + --host2) + local h2="$2" + shift + shift + ;; + --uif) + local uifType="$2" + shift + shift + ;; + --vid) + local vid="$2" + shift + shift + ;; + --pvid) + local pvid="$2" + shift + shift + ;; + --id) + local vxid="$2" + shift + shift + ;; + --ep) + local ep="$2" + shift + shift + ;; + --lip) + local lip="$2" + shift + shift + ;; + -*|--*) + echo "Unknown option $1" + exit 1 + ;; + *) + POSITIONAL_ARGS+=("$1") # save positional arg + shift # past argument + ;; + esac + done + + if [[ ${loxilbs[*]} =~ (^|[[:space:]])$h1($|[[:space:]]) && "$pick_config" == "yes" ]]; then + return + fi + + set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters + link1=e$h1$h2 + link2=e$h2$h1 + + #echo "$h1:$link1->$h2:$link2" + + if [[ "$uifType" == "phy" ]]; then + sudo ip -n $h1 link add vxlan$vxid type vxlan id $vxid local $lip dev $link1 dstport 4789 + sudo ip -n $h1 link set vxlan$vxid up + elif [[ "$uifType" == "vlan" ]]; then + sudo ip -n $h1 link add vxlan$vxid type vxlan id $vxid local $lip dev vlan$vid dstport 4789 + sudo ip -n $h1 link set vxlan$vxid up + fi + + if [[ "$pvid" != "" ]]; then + sudo ip -n $h1 link add vlan$pvid type bridge 2>&1 | true + sudo ip -n $h1 link set vxlan$vxid master vlan$pvid + sudo ip -n $h1 link set vlan$pvid up + fi + + if [[ "$ep" != "" ]]; then + sudo bridge -n $h1 fdb append 00:00:00:00:00:00 dst $ep dev vxlan$vxid + fi + +} + +## arg1 - hostname1 +## arg2 - hostname2 +create_docker_host_cnbridge() { + POSITIONAL_ARGS=() + while [[ $# -gt 0 ]]; do + case $1 in + --host1) + local h1="$2" + shift + shift + ;; + --host2) + local h2="$2" + shift + shift + ;; + -*|--*) + echo "Unknown option $1" + exit 1 + ;; + *) + POSITIONAL_ARGS+=("$1") # save positional arg + shift # past argument + ;; + esac + done + + if [[ ${loxilbs[*]} =~ (^|[[:space:]])$h1($|[[:space:]]) && "$pick_config" == "yes" ]]; then + return + fi + + set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters + link1=e$h1$h2 + link2=e$h2$h1 + + #echo "$h1:$link1->$h2:$link2" + + brport=$link1 + + sudo ip -n $h1 link add br$h1 type bridge 2>&1 | true + sudo ip -n $h1 link set $brport master br$h1 + sudo ip -n $h1 link set br$h1 up +} + +#Arg1: host name +#Arg2: --:: +#Arg3: --endpoints::,.. +function create_lb_rule() { + if [[ ${loxilbs[*]} =~ (^|[[:space:]])$1($|[[:space:]]) && "$pick_config" == "yes" ]]; then + return + fi + args=( "$@" ) + args=( "${args[@]/$1}" ) + echo "$1: loxicmd create lb ${args[*]}" + $dexec $1 loxicmd create lb ${args[*]} + + hook=$($dexec llb1 ntc filter show dev eth0 ingress | grep tc_packet_hook) + if [[ $hook != *"tc_packet_hook"* ]]; then + echo "ERROR : No hook point found"; + exit 1 + fi +} + +#Arg1: host name +#Arg2: +#Arg3: +function add_route() { + if [[ ${loxilbs[*]} =~ (^|[[:space:]])$1($|[[:space:]]) && "$pick_config" == "yes" ]]; then + return + fi + echo "$1: ip route add $2 via $3 proto static" + $hexec $1 ip route add $2 via $3 proto static +} diff --git a/cicd/microk8s-incluster/config.sh b/cicd/microk8s-incluster/config.sh new file mode 100755 index 000000000..6b8ee48ef --- /dev/null +++ b/cicd/microk8s-incluster/config.sh @@ -0,0 +1,3 @@ +#!/bin/bash +vagrant global-status | grep -i virtualbox | cut -f 1 -d ' ' | xargs -L 1 vagrant destroy -f +vagrant up diff --git a/cicd/microk8s-incluster/kube-loxilb.yml b/cicd/microk8s-incluster/kube-loxilb.yml new file mode 100644 index 000000000..6a7e7aa41 --- /dev/null +++ b/cicd/microk8s-incluster/kube-loxilb.yml @@ -0,0 +1,130 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-loxilb + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kube-loxilb +rules: + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - watch + - list + - patch + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list + - patch + - apiGroups: + - "" + resources: + - endpoints + - services + - services/status + verbs: + - get + - watch + - list + - patch + - update + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - watch + - list + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kube-loxilb +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-loxilb +subjects: + - kind: ServiceAccount + name: kube-loxilb + namespace: kube-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kube-loxilb + namespace: kube-system + labels: + app: loxilb +spec: + replicas: 1 + selector: + matchLabels: + app: loxilb + template: + metadata: + labels: + app: loxilb + spec: + hostNetwork: true + tolerations: + - effect: NoSchedule + operator: Exists + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + priorityClassName: system-node-critical + serviceAccountName: kube-loxilb + terminationGracePeriodSeconds: 0 + containers: + - name: kube-loxilb + image: ghcr.io/loxilb-io/kube-loxilb:latest + imagePullPolicy: Always + command: + - /bin/kube-loxilb + args: + #- --loxiURL=http://172.17.0.2:11111 + - --externalCIDR=192.168.82.100/32 + - --setRoles=0.0.0.0 + #- --monitor + #- --setBGP + #- --setLBMode=1 + #- --config=/opt/loxilb/agent/kube-loxilb.conf + resources: + requests: + cpu: "100m" + memory: "50Mi" + limits: + cpu: "100m" + memory: "50Mi" + securityContext: + privileged: true + capabilities: + add: ["NET_ADMIN", "NET_RAW"] diff --git a/cicd/microk8s-incluster/loxilb.yml b/cicd/microk8s-incluster/loxilb.yml new file mode 100644 index 000000000..9a673e565 --- /dev/null +++ b/cicd/microk8s-incluster/loxilb.yml @@ -0,0 +1,63 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: loxilb-lb + namespace: kube-system +spec: + selector: + matchLabels: + app: loxilb-app + template: + metadata: + name: loxilb-lb + labels: + app: loxilb-app + spec: + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + tolerations: + - effect: NoSchedule + operator: Exists + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + + containers: + - name: loxilb-app + image: "ghcr.io/loxilb-io/loxilb:latest" + imagePullPolicy: Always + command: [ "/root/loxilb-io/loxilb/loxilb", "--egr-hooks", "--blacklist=cni[0-9a-z]|veth.|flannel.|cali.|tunl.|vxlan[.]calico|kube[-]bridge" ] + ports: + - containerPort: 11111 + - containerPort: 179 + - containerPort: 50051 + securityContext: + privileged: true + capabilities: + add: + - SYS_ADMIN +--- +apiVersion: v1 +kind: Service +metadata: + name: loxilb-lb-service + namespace: kube-system +spec: + clusterIP: None + selector: + app: loxilb-app + ports: + - name: loxilb-app + port: 11111 + targetPort: 11111 + protocol: TCP + - name: loxilb-app-bgp + port: 179 + targetPort: 179 + protocol: TCP + - name: loxilb-app-gobgp + port: 50051 + targetPort: 50051 + protocol: TCP diff --git a/cicd/microk8s-incluster/mk8s-install.sh b/cicd/microk8s-incluster/mk8s-install.sh new file mode 100755 index 000000000..337badf84 --- /dev/null +++ b/cicd/microk8s-incluster/mk8s-install.sh @@ -0,0 +1,70 @@ +source /vagrant/common.sh + +function wait_mk8s_cluster_ready { + Res=$(microk8s kubectl get pods -A | + while IFS= read -r line; do + if [[ "$line" != *"Running"* && "$line" != *"READY"* ]]; then + echo "not ready" + return + fi + done) + if [[ $Res == *"not ready"* ]]; then + return 1 + fi + return 0 +} + +function wait_mk8s_cluster_ready_full { + i=1 + nr=0 + for ((;;)) do + wait_mk8s_cluster_ready + nr=$? + if [[ $nr == 0 ]]; then + echo "Cluster is ready" + break + fi + i=$(( $i + 1 )) + if [[ $i -ge 40 ]]; then + echo "Cluster is not ready.Giving up" + microk8s kubectl get svc + microk8s kubectl get pods -A + exit 1 + fi + echo "Cluster is not ready...." + sleep 10 + done +} + +export LOXILB_IP=$(ip a |grep global | grep -v '10.0.2.15' | grep -v '192.168.80' | awk '{print $2}' | cut -f1 -d '/') + +apt-get update +apt-get install -y software-properties-common ethtool +apt install -y snapd + +echo "Start micro-k8s installation" + +snap install microk8s --classic --channel=1.28/stable + +sleep 30 +microk8s status --wait-ready + +# Check kubectl works +microk8s kubectl get pods -A + +echo "End micro-k8s installation" +sleep 60 +wait_mk8s_cluster_ready_full + +microk8s kubectl apply -f /vagrant/loxilb.yml +sleep 60 +wait_mk8s_cluster_ready_full + +microk8s kubectl apply -f /vagrant/kube-loxilb.yml +sleep 30 +wait_mk8s_cluster_ready_full + +microk8s kubectl apply -f /vagrant/tcp-svc-lb.yml + +# Wait for cluster to be ready +wait_mk8s_cluster_ready_full diff --git a/cicd/microk8s-incluster/rmconfig.sh b/cicd/microk8s-incluster/rmconfig.sh new file mode 100755 index 000000000..d2eeabe5a --- /dev/null +++ b/cicd/microk8s-incluster/rmconfig.sh @@ -0,0 +1,2 @@ +#!/bin/bash +vagrant destroy -f mk8s-node1 diff --git a/cicd/microk8s-incluster/tcp-svc-lb.yml b/cicd/microk8s-incluster/tcp-svc-lb.yml new file mode 100644 index 000000000..b3d345483 --- /dev/null +++ b/cicd/microk8s-incluster/tcp-svc-lb.yml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: Service +metadata: + name: tcp-lb-onearm + annotations: + loxilb.io/liveness: "yes" + loxilb.io/lbmode: "onearm" +spec: + externalTrafficPolicy: Local + loadBalancerClass: loxilb.io/loxilb + selector: + what: tcp-onearm-test + ports: + - port: 56002 + targetPort: 80 + type: LoadBalancer +--- +apiVersion: v1 +kind: Pod +metadata: + name: tcp-onearm-test + labels: + what: tcp-onearm-test +spec: + containers: + - name: tcp-onearm-test + image: ghcr.io/loxilb-io/nginx:stable + ports: + - containerPort: 80 diff --git a/cicd/microk8s-incluster/validation.sh b/cicd/microk8s-incluster/validation.sh new file mode 100755 index 000000000..0de67fbeb --- /dev/null +++ b/cicd/microk8s-incluster/validation.sh @@ -0,0 +1,61 @@ +#!/bin/bash +source ../common.sh +echo microk8s-incluster + +if [ "$1" ]; then + KUBECONFIG="$1" +fi + +# Set space as the delimiter +IFS=' ' + +for((i=0; i<120; i++)) +do + extLB=$(vagrant ssh mk8s-node1 -c 'sudo microk8s kubectl get svc' 2> /dev/null | grep tcp-lb"") + read -a strarr <<< "$extLB" + len=${#strarr[*]} + if [[ $((len)) -lt 6 ]]; then + echo "Can't find tcp-lb service" + sleep 1 + continue + fi + if [[ ${strarr[3]} != *"none"* ]]; then + extIP="$(cut -d'-' -f2 <<<${strarr[3]})" + break + fi + echo "No external LB allocated" + sleep 1 +done + +## Any routing updates ?? +#sleep 30 +echo Service IP : $extIP +echo $extIP + +echo -e "\nEnd Points List" +echo "******************************************************************************" +vagrant ssh mk8s-node1 -c 'sudo microk8s kubectl get endpoints -A' 2> /dev/null +echo "******************************************************************************" +echo -e "\nSVC List" +echo "******************************************************************************" +vagrant ssh mk8s-node1 -c 'sudo microk8s kubectl get svc' 2> /dev/null +echo "******************************************************************************" +echo -e "\nPod List" +echo "******************************************************************************" +vagrant ssh mk8s-node1 -c 'sudo microk8s kubectl get pods -A' 2> /dev/null +echo "******************************************************************************" + +echo -e "\nTEST RESULTS" +echo "******************************************************************************" + +echo -e "Command: curl --connect-time 10 http://${extIP}:56002'\n" +res=`curl -s --connect-time 10 http://${extIP}:56002` +echo "Result" +echo $res +if [[ "$res" == *"Welcome to nginx"* ]]; then + echo -e "\n\nmicrok8s-incluster TCP service (loxilb) [OK]" +else + echo -e "\n\nmicrok8s-incluster TCP service (loxilb) [NOK]" + exit 1 +fi +