Merge pull request #370 from TrekkieCoder/main

PR - cicd: updated various in-cluster tests
loxilb-io · Jul 31, 2023 · 8c838ff · 8c838ff
2 parents df31068 + 1904979
commit 8c838ff
Show file tree

Hide file tree

Showing 11 changed files with 58 additions and 112 deletions.
diff --git a/cicd/k3s-calico-incluster/Vagrantfile b/cicd/k3s-calico-incluster/Vagrantfile
@@ -31,8 +31,8 @@ Vagrant.configure("2") do |config|
     master.vm.network :private_network, ip: "192.168.90.10", :netmask => "255.255.255.0"
     master.vm.provision :shell, :path => "master1.sh"
     master.vm.provider :virtualbox do |vbox|
-        vbox.customize ["modifyvm", :id, "--memory", 2048]
-        vbox.customize ["modifyvm", :id, "--cpus", 2]
+        vbox.customize ["modifyvm", :id, "--memory", 4096]
+        vbox.customize ["modifyvm", :id, "--cpus", 3]
     end
   end
 
@@ -42,8 +42,8 @@ Vagrant.configure("2") do |config|
     master.vm.network :private_network, ip: "192.168.90.11", :netmask => "255.255.255.0"
     master.vm.provision :shell, :path => "master2.sh"
     master.vm.provider :virtualbox do |vbox|
-        vbox.customize ["modifyvm", :id, "--memory", 2048]
-        vbox.customize ["modifyvm", :id, "--cpus", 2]
+        vbox.customize ["modifyvm", :id, "--memory", 4096]
+        vbox.customize ["modifyvm", :id, "--cpus", 3]
     end
   end
 

diff --git a/cicd/k3s-calico-incluster/loxilb.yml b/cicd/k3s-calico-incluster/loxilb.yml
@@ -32,7 +32,7 @@ spec:
       containers:
       - name: loxilb-app
         image: "ghcr.io/loxilb-io/loxilb:latest"
-        command: [ "/root/loxilb-io/loxilb/loxilb", "--bgp", "--egr-hooks", "--blacklist=cali.|tunl.|vxlan[.]calico|veth." ]
+        command: [ "/root/loxilb-io/loxilb/loxilb", "--bgp", "--egr-hooks", "--blacklist=cali.|tunl.|vxlan[.]calico|veth.|cni[0-9a-z]" ]
         ports:
         - containerPort: 11111
         - containerPort: 1791

diff --git a/cicd/k3s-calico-incluster/master1.sh b/cicd/k3s-calico-incluster/master1.sh
@@ -6,8 +6,8 @@ echo $MASTER_IP > /vagrant/master-ip
 cp /var/lib/rancher/k3s/server/node-token /vagrant/node-token
 sed -i -e "s/127.0.0.1/${MASTER_IP}/g" /etc/rancher/k3s/k3s.yaml
 cp /etc/rancher/k3s/k3s.yaml /vagrant/k3s.yaml
-#sudo kubectl apply -f /vagrant/loxilb.yml
-#sudo kubectl apply -f /vagrant/kube-loxilb.yml
+sudo kubectl apply -f /vagrant/loxilb.yml
+sudo kubectl apply -f /vagrant/kube-loxilb.yml
 sudo kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.26.0/manifests/tigera-operator.yaml
 sudo kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.26.0/manifests/custom-resources.yaml
 /vagrant/wait_ready.sh
diff --git a/cicd/k3s-calico-incluster/master2.sh b/cicd/k3s-calico-incluster/master2.sh
@@ -3,7 +3,6 @@ export WORKER_ADDR=$(ip a |grep global | grep -v '10.0.2.15' | grep '192.168.80'
 export MASTER_ADDR=$(cat /vagrant/master-ip)
 export NODE_TOKEN=$(cat /vagrant/node-token)
 
-#curl -fL https://get.k3s.io | K3S_TOKEN=${NODE_TOKEN} sh -s - server --server https://192.168.80.10:6443 --disable traefik --disable servicelb --node-ip=192.168.80.11 external-hostname=192.168.80.11 --node-external-ip=192.168.80.11 --disable-cloud-controller -t ${NODE_TOKEN}
 curl -fL https://get.k3s.io | K3S_TOKEN=${NODE_TOKEN} sh -s - server --server https://192.168.80.10:6443 --disable traefik --disable servicelb --node-ip=192.168.80.11 external-hostname=192.168.80.11 --node-external-ip=192.168.80.11 -t ${NODE_TOKEN}
 
 /vagrant/wait_ready.sh
diff --git a/cicd/k3s-calico-incluster/validation.sh b/cicd/k3s-calico-incluster/validation.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 source ../common.sh
-echo k3s-flannel-cluster
+echo k3s-calico-incluster
 
 if [ "$1" ]; then
   KUBECONFIG="$1"
@@ -9,48 +9,34 @@ fi
 # Set space as the delimiter
 IFS=' '
 
-sleep 45
+sleep 5
 extIP="123.123.123.1"
 echo $extIP
 
 echo "Service Info"
-vagrant ssh master -c 'sudo kubectl get svc'
-echo "LB Info"
-vagrant ssh loxilb -c 'sudo docker exec -i loxilb loxicmd get lb -o wide'
-echo "EP Info"
-vagrant ssh loxilb -c 'sudo docker exec -i loxilb loxicmd get ep -o wide'
+vagrant ssh master1 -c 'sudo kubectl get svc'
 
 print_debug_info() {
-  echo "llb1 route-info"
-  vagrant ssh loxilb -c 'ip route'
-  vagrant ssh master -c 'sudo kubectl get pods -A'
-  vagrant ssh master -c 'sudo kubectl get svc'
-  vagrant ssh master -c 'sudo kubectl get nodes'
+  echo "cluster-info"
+  vagrant ssh master1 -c 'sudo kubectl get pods -A'
+  vagrant ssh master1 -c 'sudo kubectl get svc'
+  vagrant ssh master1 -c 'sudo kubectl get nodes'
 }
 
 out=$(curl -s --connect-timeout 10 http://$extIP:55002) 
 if [[ ${out} == *"Welcome to nginx"* ]]; then
-  echo "k3s-flannel-cluster (kube-loxilb) tcp [OK]"
+  echo "k3s-calico-incluster (kube-loxilb) tcp [OK]"
 else
-  echo "k3s-flannel-cluster (kube-loxilb) tcp [FAILED]"
+  echo "k3s-calico-incluster (kube-loxilb) tcp [FAILED]"
   print_debug_info
   exit 1
 fi
 
 out=$(timeout 10 ../common/udp_client $extIP 55003)
 if [[ ${out} == *"Client"* ]]; then
-  echo "k3s-flannel-cluster (kube-loxilb) udp [OK]"
+  echo "k3s-calico-incluster (kube-loxilb) udp [OK]"
 else
-  echo "k3s-flannel-cluster (kube-loxilb) udp [FAILED]"
-  print_debug_info
-  exit 1
-fi
-
-out=$(timeout 10 ../common/sctp_client 192.168.90.1 41291 $extIP 55004)
-if [[ ${out} == *"server1"* ]]; then
-  echo "k3s-flannel-cluster (kube-loxilb) sctp [OK]"
-else
-  echo "k3s-flannel-cluster (kube-loxilb) sctp [FAILED]"
+  echo "k3s-calico-incluster (kube-loxilb) udp [FAILED]"
   print_debug_info
   exit 1
 fi

diff --git a/cicd/k3s-calico-incluster/worker.sh b/cicd/k3s-calico-incluster/worker.sh
@@ -5,8 +5,7 @@ export NODE_TOKEN=$(cat /vagrant/node-token)
 mkdir -p /etc/rancher/k3s
 cp -f /vagrant/k3s.yaml /etc/rancher/k3s/k3s.yaml
 curl -sfL https://get.k3s.io | K3S_TOKEN=${NODE_TOKEN} sh -s - agent --server https://192.168.80.10:6443 --node-ip=${WORKER_ADDR} --node-external-ip=${WORKER_ADDR} -t ${NODE_TOKEN}
-#sudo kubectl apply -f /vagrant/loxilb-peer.yml
+sudo kubectl apply -f /vagrant/loxilb-peer.yml
 sudo kubectl apply -f /vagrant/nginx.yml
-#sudo kubectl apply -f /vagrant/udp.yml
-#sudo kubectl apply -f /vagrant/sctp.yml
+sudo kubectl apply -f /vagrant/udp.yml
 /vagrant/wait_ready.sh
diff --git a/cicd/k3s-flannel-incluster/validation.sh b/cicd/k3s-flannel-incluster/validation.sh
@@ -9,23 +9,18 @@ fi
 # Set space as the delimiter
 IFS=' '
 
-sleep 45
+sleep 5
 extIP="123.123.123.1"
 echo $extIP
 
 echo "Service Info"
-vagrant ssh master -c 'sudo kubectl get svc'
-echo "LB Info"
-vagrant ssh loxilb -c 'sudo docker exec -i loxilb loxicmd get lb -o wide'
-echo "EP Info"
-vagrant ssh loxilb -c 'sudo docker exec -i loxilb loxicmd get ep -o wide'
+vagrant ssh master1 -c 'sudo kubectl get svc'
 
 print_debug_info() {
-  echo "llb1 route-info"
-  vagrant ssh loxilb -c 'ip route'
-  vagrant ssh master -c 'sudo kubectl get pods -A'
-  vagrant ssh master -c 'sudo kubectl get svc'
-  vagrant ssh master -c 'sudo kubectl get nodes'
+  echo "cluster-info"
+  vagrant ssh master1 -c 'sudo kubectl get pods -A'
+  vagrant ssh master1 -c 'sudo kubectl get svc'
+  vagrant ssh master1 -c 'sudo kubectl get nodes'
 }
 
 out=$(curl -s --connect-timeout 10 http://$extIP:55002) 

diff --git a/cicd/k8s-calico-incluster/Vagrantfile b/cicd/k8s-calico-incluster/Vagrantfile
@@ -6,21 +6,18 @@ settings = YAML.load_file "yaml/settings.yaml"
 
 workers = settings["nodes"]["workers"]["count"]
 
-bname = ("sysnet4admin/Ubuntu-k8s")
-bversion = "0.7.1"
-
 Vagrant.configure("2") do |config|
 
   if Vagrant.has_plugin?("vagrant-vbguest")
     config.vbguest.auto_update = false
   end
 
-  config.vm.box = "#{bname}"
-  config.vm.box_version = "#{bversion}"
+  config.vm.box = settings["software"]["cluster"]["box"]["name"]
+  config.vm.box_version = settings["software"]["cluster"]["box"]["version"]
 
   config.vm.define "host" do |host|
     host.vm.hostname = 'host1'
-    #loxilb.vm.network "forwarded_port", guest: 55002, host: 5502, protocol: "tcp"
+    host.vm.network :private_network, ip: settings["network"]["client_ip"], :netmask => "255.255.255.0"
     host.vm.network :private_network, ip: "192.168.80.9", :netmask => "255.255.255.0"
     host.vm.network :private_network, ip: "192.168.90.9", :netmask => "255.255.255.0"
     host.vm.provision :shell, :path => "node_scripts/host.sh"
@@ -30,10 +27,10 @@ Vagrant.configure("2") do |config|
     end
   end
 
-  #config.vm.box = settings["software"]["cluster"]["box"]
   config.vm.define "master" do |master|
-    master.vm.hostname = 'master'
+    master.vm.hostname = 'master1'
     master.vm.network :private_network, ip: settings["network"]["control_ip"], :netmask => "255.255.255.0"
+    master.vm.network :private_network, ip: settings["network"]["control_ip2"], :netmask => "255.255.255.0"
     master.vm.provision "shell",
       env: {
         "DNS_SERVERS" => settings["network"]["dns_servers"].join(" "),
@@ -52,8 +49,8 @@ Vagrant.configure("2") do |config|
       path: "node_scripts/master.sh"
 
     master.vm.provider :virtualbox do |vbox|
-        vbox.customize ["modifyvm", :id, "--memory", 2048]
-        vbox.customize ["modifyvm", :id, "--cpus", 2]
+        vbox.customize ["modifyvm", :id, "--memory", 4096]
+        vbox.customize ["modifyvm", :id, "--cpus", 3]
     end
   end
 

diff --git a/cicd/k8s-calico-incluster/config.sh b/cicd/k8s-calico-incluster/config.sh
@@ -30,7 +30,7 @@ do
     sleep 10
 done
 
-#Create fullnat Service
+# Create fullnat Services
 vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/tcp_fullnat.yml' 2> /dev/null
 vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/udp_fullnat.yml' 2> /dev/null
 vagrant ssh master -c 'kubectl apply -f /vagrant/yaml/sctp_fullnat.yml' 2> /dev/null
diff --git a/cicd/k8s-calico-incluster/validation.sh b/cicd/k8s-calico-incluster/validation.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 source ../common.sh
-echo k8s-calico
+echo k8s-calico-incluster
 
 if [ "$1" ]; then
   KUBECONFIG="$1"
@@ -11,11 +11,11 @@ IFS=' '
 
 for((i=0; i<120; i++))
 do
-  extLB=$(vagrant ssh master -c 'kubectl get svc' 2> /dev/null | grep "tcp-lb-default")
+  extLB=$(vagrant ssh master -c 'kubectl get svc' 2> /dev/null | grep "tcp-lb-fullnat")
   read -a strarr <<< "$extLB"
   len=${#strarr[*]}
   if [[ $((len)) -lt 6 ]]; then
-    echo "Can't find tcp-lb service"
+    echo "Can't find tcp-lb-fullnat"
     sleep 1
     continue
   fi 
@@ -42,61 +42,31 @@ echo "**************************************************************************
 echo -e "\nPod List"
 echo "******************************************************************************"
 vagrant ssh master -c 'kubectl get pods -A' 2> /dev/null
-echo "******************************************************************************"
-echo -e "\nLB List"
-echo "******************************************************************************"
-vagrant ssh loxilb -c 'sudo docker exec -it loxilb loxicmd get lb -o wide' 2> /dev/null
-echo "******************************************************************************"
-echo -e "\nEP List"
-echo "******************************************************************************"
-vagrant ssh loxilb -c 'sudo docker exec -it loxilb loxicmd get ep -o wide' 2> /dev/null
-echo "******************************************************************************"
 
 echo -e "\nTEST RESULTS"
 echo "******************************************************************************"
-mode=( "default" "onearm" "fullnat" )
-tcp_port=( 55002 56002 57002 )
-udp_port=( 55003 56003 57003 )
-sctp_port=( 55004 56004 57004 )
+mode=( "fullnat" )
+tcp_port=( 57002 )
+udp_port=( 57003 )
+sctp_port=( 57004 )
 code=0
-for ((i=0;i<=2;i++)); do
-out=$(curl -s --connect-timeout 10 http://$extIP:${tcp_port[i]})
-if [[ ${out} == *"Welcome to nginx"* ]]; then
-  echo -e "K8s-calico TCP\t(${mode[i]})\t[OK]"
+for ((i=0;i<1;i++)); do
+out=$(vagrant ssh host -c "curl -s --connect-timeout 10 http://$extIP:${tcp_port[i]}")
+echo $out
+if [[ ${out} == *"nginx"* ]]; then
+  echo -e "K8s-calico-incluster TCP\t(${mode[i]})\t[OK]"
 else
-  echo -e "K8s-calico TCP\t(${mode[i]})\t[FAILED]"
-  ## Dump some debug info
-  echo "llb1 lb-info"
-  vagrant ssh loxilb -c 'sudo docker exec -it llb1 loxicmd get lb -o wide' 2> /dev/null
-  echo "llb1 route-info"
-  vagrant ssh loxilb -c 'sudo docker exec -it llb1 ip route' 2> /dev/null
+  echo -e "K8s-calico-incluster TCP\t(${mode[i]})\t[FAILED]"
   code=1
 fi
 
-out=$(timeout 5 ../common/udp_client $extIP ${udp_port[i]})
+out=$(vagrant ssh host -c "timeout 5 /vagrant/tools/udp_client $extIP ${udp_port[i]}")
 if [[ ${out} == *"Client"* ]]; then
-  echo -e "K8s-calico UDP\t(${mode[i]})\t[OK]"
+  echo -e "K8s-calico-incluster UDP\t(${mode[i]})\t[OK]"
 else
-  echo -e "K8s-calico UDP\t(${mode[i]})\t[FAILED]"
-  ## Dump some debug info
-  echo "llb1 lb-info"
-  vagrant ssh loxilb -c 'sudo docker exec -it llb1 loxicmd get lb -o wide' 2> /dev/null
-  echo "llb1 route-info"
-  vagrant ssh loxilb -c 'sudo docker exec -it llb1 ip route' 2> /dev/null
-  code=1
-fi
-
-out=$(timeout 5 ../common/sctp_client 192.168.90.1 34951 $extIP ${sctp_port[i]})
-if [[ ${out} == *"server1"* ]]; then
-  echo -e "K8s-calico SCTP\t(${mode[i]})\t[OK]"
-else
-  echo -e "K8s-calico SCTP\t(${mode[i]})\t[FAILED]"
-  ## Dump some debug info
-  echo "llb1 lb-info"
-  vagrant ssh loxilb -c 'sudo docker exec -it llb1 loxicmd get lb -o wide' 2> /dev/null
-  echo "llb1 route-info"
-  vagrant ssh loxilb -c 'sudo docker exec -it llb1 ip route' 2> /dev/null
+  echo -e "K8s-calico-incluster UDP\t(${mode[i]})\t[FAILED]"
   code=1
 fi
 done
+
 exit $code
diff --git a/cicd/k8s-calico-incluster/yaml/settings.yaml b/cicd/k8s-calico-incluster/yaml/settings.yaml
@@ -9,35 +9,35 @@ cluster_name: Kubernetes Cluster
 #   NO_PROXY=127.0.0.1,localhost,master-node,node01,node02,node03
 # All IPs/CIDRs should be private and allowed in /etc/vbox/networks.conf.
 network:
-  iloxilb_ip: 192.168.80.9
-  oloxilb_ip: 192.168.90.9
   # Worker IPs are simply incremented from the control IP.
+  client_ip: 192.168.90.9
   control_ip: 192.168.80.10
+  control_ip2: 192.168.90.10
+  control2_ip: 192.168.80.11
+  control2_ip2: 192.168.90.11
   dns_servers:
     - 8.8.8.8
     - 1.1.1.1
   pod_cidr: 172.16.1.0/16
   service_cidr: 172.17.1.0/18
 nodes:
   control:
-    cpu: 2
+    cpu: 4
     memory: 4096
   workers:
     count: 2
-    cpu: 1
+    cpu: 2
     memory: 2048
 # Mount additional shared folders from the host into each virtual machine.
 # Note that the project directory is automatically mounted at /vagrant.
 # shared_folders:
 #   - host_path: ../images
 #     vm_path: /vagrant/images
 software:
-  loxilb:
+  cluster:
     box: 
      name: sysnet4admin/Ubuntu-k8s
      version: 0.7.1
-  cluster:
-    box: bento/ubuntu-22.04
   calico: 3.26.0
   # To skip the dashboard installation, set its version to an empty value or comment it out:
   kubernetes: 1.27.1-00