Merge pull request #779 from nik-netlox/main

gh-87 longrun test CICD with tcpkali tool added

.github/workflows/tcpkali-longrun.yml

@@ -0,0 +1,36 @@
+name: TCPKALI-Longrun-CI
+on:
+ schedule:
+   # Runs "At 13:00 UTC every day-of-week"
+   - cron: '0 13 * * *'
+ workflow_dispatch:
+    inputs:
+      testName:
+        description: 'Test Run-Name'     
+        required: true
+        default: 'tcpkali-longrun'
+jobs:
+  test-runner:
+    name: tcpkali-longrun
+    runs-on: [self-hosted, sb]
+    if: github.repository == 'loxilb-io/loxilb' 
+          &&  github.event.inputs.tagName == ''
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+          submodules: recursive
+
+    - name: Run the test
+      run: |
+         cd cicd/tcpkali
+         ./config.sh
+         ./validation.sh
+         cd -
+         
+    - name: Clean test-bed
+      if: success() || failure()
+      run: |
+        cd cicd/tcpkali || true
+        ./rmconfig.sh
+        cd -

cicd/tcpkali/config.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+source ../common.sh
+
+echo "#########################################"
+echo "Spawning all hosts"
+echo "#########################################"
+
+spawn_docker_host --dock-type loxilb --dock-name llb1
+spawn_docker_host --dock-type host --dock-name l3h1
+spawn_docker_host --dock-type host --dock-name l3h2
+spawn_docker_host --dock-type host --dock-name l3ep1
+spawn_docker_host --dock-type host --dock-name l3ep2
+spawn_docker_host --dock-type host --dock-name l3ep3
+
+echo "#########################################"
+echo "Connecting and configuring  hosts"
+echo "#########################################"
+
+
+connect_docker_hosts l3h1 llb1
+connect_docker_hosts l3h2 llb1
+connect_docker_hosts l3ep1 llb1
+connect_docker_hosts l3ep2 llb1
+connect_docker_hosts l3ep3 llb1
+
+sleep 5
+
+#L3 config
+config_docker_host --host1 l3h1 --host2 llb1 --ptype phy --addr 10.10.10.1/24 --gw 10.10.10.254
+config_docker_host --host1 l3h2 --host2 llb1 --ptype phy --addr 10.10.11.1/24 --gw 10.10.11.254
+config_docker_host --host1 l3ep1 --host2 llb1 --ptype phy --addr 31.31.31.1/24 --gw 31.31.31.254
+config_docker_host --host1 l3ep2 --host2 llb1 --ptype phy --addr 32.32.32.1/24 --gw 32.32.32.254
+config_docker_host --host1 l3ep3 --host2 llb1 --ptype phy --addr 33.33.33.1/24 --gw 33.33.33.254
+config_docker_host --host1 llb1 --host2 l3h1 --ptype phy --addr 10.10.10.254/24
+config_docker_host --host1 llb1 --host2 l3h2 --ptype phy --addr 10.10.11.254/24
+config_docker_host --host1 llb1 --host2 l3ep1 --ptype phy --addr 31.31.31.254/24
+config_docker_host --host1 llb1 --host2 l3ep2 --ptype phy --addr 32.32.32.254/24
+config_docker_host --host1 llb1 --host2 l3ep3 --ptype phy --addr 33.33.33.254/24
+
+$dexec llb1 ip addr add 10.10.10.3/32 dev lo
+
+sleep 5
+create_lb_rule llb1 20.20.20.1 --tcp=2020:8080 --endpoints=31.31.31.1:1,32.32.32.1:1,33.33.33.1:1
+create_lb_rule llb1 10.10.10.254 --tcp=2020:8080 --endpoints=31.31.31.1:1,32.32.32.1:1,33.33.33.1:1
+create_lb_rule llb1 10.10.10.3 --tcp=2020:8080 --endpoints=31.31.31.1:1,32.32.32.1:1,33.33.33.1:1

cicd/tcpkali/pps.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+i="1"     # one second
+time=$1
+int1=$2   # network interface
+int2=$3   # network interface
+
+end=$((SECONDS + $time))
+
+while [ $SECONDS -lt $end ]; do
+		txpkts_old1="`cat /sys/class/net/$int1/statistics/tx_packets`" # sent packets
+		rxpkts_old1="`cat /sys/class/net/$int1/statistics/rx_packets`" # recv packets
+		txpkts_old2="`cat /sys/class/net/$int2/statistics/tx_packets`" # sent packets
+		rxpkts_old2="`cat /sys/class/net/$int2/statistics/rx_packets`" # recv packets
+        txpkts_old="`expr $txpkts_old1 + $txpkts_old2`"
+        rxpkts_old="`expr $rxpkts_old1 + $rxpkts_old2`"
+   	    sleep $i
+		txpkts_new1="`cat /sys/class/net/$int1/statistics/tx_packets`" # sent packets
+        rxpkts_new1="`cat /sys/class/net/$int1/statistics/rx_packets`" # recv packets
+		txpkts_new2="`cat /sys/class/net/$int2/statistics/tx_packets`" # sent packets
+		rxpkts_new2="`cat /sys/class/net/$int2/statistics/rx_packets`" # recv packets
+
+        txpkts_new="`expr $txpkts_new1 + $txpkts_new2`"
+        rxpkts_new="`expr $rxpkts_new1 + $rxpkts_new2`"
+
+		txpkts="`expr $txpkts_new - $txpkts_old`"		     # evaluate expressions for sent packets
+		rxpkts="`expr $rxpkts_new - $rxpkts_old`"		     # evaluate expressions for recv packets
+		echo "tx $txpkts pkts/s - rx $rxpkts pkts/ on interface $int1 and $int2"
+done
+

cicd/tcpkali/rmconfig.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+source ../common.sh
+
+disconnect_docker_hosts l3h1 llb1
+disconnect_docker_hosts l3h2 llb1
+disconnect_docker_hosts l3ep1 llb1
+disconnect_docker_hosts l3ep2 llb1
+disconnect_docker_hosts l3ep3 llb1
+
+delete_docker_host llb1
+delete_docker_host l3h1
+delete_docker_host l3h2
+delete_docker_host l3ep1
+delete_docker_host l3ep2
+delete_docker_host l3ep3
+
+echo "#########################################"
+echo "Deleted testbed"
+echo "#########################################"

cicd/tcpkali/validation.sh

@@ -0,0 +1,85 @@
+#!/bin/bash
+source ../common.sh
+echo SCENARIO-tcpkali
+$hexec l3ep1 node ../common/tcp_server.js server1 &
+$hexec l3ep2 node ../common/tcp_server.js server2 &
+$hexec l3ep3 node ../common/tcp_server.js server3 &
+
+conn=400
+
+time=14400 #4hrs
+
+stime=$(( $time + 10 ))
+sleep 2
+code=0
+servIP="20.20.20.1"
+servArr=( "server1" "server2" "server3" )
+ep=( "31.31.31.1" "32.32.32.1" "33.33.33.1" )
+host=( "l3h1" "l3h2" )
+i=0
+waitCount=0
+while [ $i -le 1 ]
+do
+j=0
+echo "Check connectivity from ${host[i]}"
+while [ $j -le 2 ]
+do
+    res=$($hexec ${host[i]} curl --max-time 10 -s ${ep[j]}:8080)
+    #echo $res
+    if [[ $res == "${servArr[j]}" ]]
+    then
+        echo "$res UP"
+        j=$(( $j + 1 ))
+    else
+        echo "Waiting for ${servArr[j]}(${ep[j]})"
+        waitCount=$(( $waitCount + 1 ))
+        if [[ $waitCount == 10 ]];
+        then
+            echo "All Servers are not UP"
+            echo SCENARIO-tcpkali [FAILED]
+            sudo killall -9 node 2>&1 > /dev/null
+            exit 1
+        fi
+    fi
+    sleep 1
+done
+i=$(( $i + 1 ))
+done
+
+sudo killall -9 node 2>&1 > /dev/null
+
+$hexec l3ep1 tcpkali -l 8080 -T $stime 2>&1> /dev/null &
+$hexec l3ep2 tcpkali -l 8080 -T $stime 2>&1> /dev/null &
+$hexec l3ep3 tcpkali -l 8080 -T $stime 2>&1> /dev/null &
+
+sleep 2
+
+for k in {0..1}
+do
+    echo "${host[k]}: Testing Service IP: $servIP - connections: $conn duration: $time secs"
+    lcode=0
+    $hexec ${host[k]} stdbuf -oL tcpkali -c $conn -T $time -w 8 -m "message" -r 2000 $servIP:2020 2> ${host[k]}.log &
+done
+
+#sleep $stime
+$hexec llb1 ./pps.sh $time ellb1l3h1 ellb1l3h2
+
+conn1=$( tail -n 2 l3h1.log | xargs | cut -d ' ' -f 11 | cut -d ')' -f 1 )
+conn2=$( tail -n 2 l3h2.log | xargs | cut -d ' ' -f 11 | cut -d ')' -f 1 )
+
+if [[ $conn1 == $conn && $conn2 == $conn ]]
+then
+    echo SCENARIO-tcpkali with ${servIP[k]} [OK]
+else
+    echo SCENARIO-tcpkali with ${servIP[k]} [FAILED]
+    echo "l3h1 tcpkali"
+    tail -n 2 l3h1.log
+    echo "l3h2 tcpkali"
+    tail -n 2 l3h2.log
+    code=1
+fi
+
+rm l3h1.log l3h2.log
+
+#sudo killall -9 tcpkali 2>&1 > /dev/null
+exit $code