PR - Merge latest to aws-support

.github/workflows/__k3s-base-sanity-module.yml

@@ -13,7 +13,7 @@ jobs:
              go-version: '>=1.18.0'
       - run: |
          sudo apt-get update
-         sudo apt-get -y install bridge-utils net-tools iperf curl wget clang-10 llvm libelf-dev gcc-multilib libpcap-dev linux-tools-$(uname -r) elfutils dwarves git libbsd-dev bridge-utils unzip build-essential bison flex iperf iproute2 nodejs socat ethtool
+         sudo apt-get -y install bridge-utils net-tools iperf curl lksctp-tools wget clang-10 llvm libelf-dev gcc-multilib libpcap-dev linux-tools-$(uname -r) elfutils dwarves git libbsd-dev bridge-utils unzip build-essential bison flex iperf iproute2 nodejs socat ethtool
          git clone --recursive https://github.com/loxilb-io/loxilb
       - working-directory: ./loxilb
         run: |

.github/workflows/basic-sanity-ubuntu-22.yml

@@ -35,7 +35,7 @@ jobs:
         with:
              go-version: '>=1.18.0'
       - run: sudo apt-get update
-      - run: sudo apt-get -y install clang-13 llvm libelf-dev gcc-multilib libpcap-dev linux-tools-$(uname -r) elfutils dwarves git libbsd-dev bridge-utils unzip build-essential bison flex iproute2
+      - run: sudo apt-get -y install clang llvm libelf-dev gcc-multilib libpcap-dev linux-tools-$(uname -r) elfutils dwarves git libbsd-dev bridge-utils unzip build-essential bison flex iproute2
       - run: | 
             sudo ip netns add test
             sudo ip netns del test

.github/workflows/k3s-calico.yml

@@ -23,10 +23,10 @@ jobs:
       - run: curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.22.9+k3s1 INSTALL_K3S_EXEC="server --disable traefik --disable servicelb --disable-cloud-controller --kubelet-arg cloud-provider=external --flannel-backend=none --cluster-cidr=10.42.0.0/16" K3S_KUBECONFIG_MODE="644" sh -
       - run: |
          sleep 10
-         kubectl "${{ env.KUBECONFIG }}" create -f https://raw.githubusercontent.com/projectcalico/calico/v3.26.0/manifests/tigera-operator.yaml
+         kubectl "${{ env.KUBECONFIG }}" create -f https://raw.githubusercontent.com/projectcalico/calico/v3.26.1/manifests/tigera-operator.yaml
       - run: |
          sleep 10
-         kubectl "${{ env.KUBECONFIG }}" create -f https://raw.githubusercontent.com/projectcalico/calico/v3.26.0/manifests/custom-resources.yaml
+         kubectl "${{ env.KUBECONFIG }}" create -f https://raw.githubusercontent.com/projectcalico/calico/v3.26.1/manifests/custom-resources.yaml
       - run: |
          sleep 10
          kubectl "${{ env.KUBECONFIG }}" taint nodes --all node.cloudprovider.kubernetes.io/uninitialized=false:NoSchedule-

.github/workflows/sctpmh-sanity.yml

@@ -0,0 +1,55 @@
+name: SCTP-MH-LB-Sanity-CI
+
+on:
+  #push:
+  #  branches:
+  #    - main
+  #pull_request:
+  #  branches: [ "main" ]
+  workflow_dispatch:
+    inputs:
+      logLevel:
+        description: 'Log level'     
+        required: true
+        default: 'warning'
+      tags:
+        description: 'SCTP MH LB Sanity'
+
+jobs:
+  build:
+    name: sctpmh-lb-sanity
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: recursive
+      - uses: actions/setup-python@v2
+      - uses: actions/setup-go@v3
+        with:
+             go-version: '>=1.18.0'
+      - run: sudo apt-get update
+      - run: sudo apt-get -y install clang-10 llvm libelf-dev gcc-multilib libpcap-dev linux-tools-$(uname -r) elfutils dwarves git libbsd-dev bridge-utils unzip build-essential bison flex iperf iproute2 nodejs socat ethtool curl lksctp-tools
+      - run: |
+            git clone --recurse-submodules https://github.com/loxilb-io/iproute2 iproute2-main
+            cd iproute2-main/libbpf/src/
+            sudo make install
+            mkdir build
+            DESTDIR=build OBJDIR=build make install
+            cd -
+            cd iproute2-main/
+            export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:`pwd`/libbpf/src/ && LIBBPF_FORCE=on LIBBPF_DIR=`pwd`/libbpf/src/build ./configure && make && sudo cp -f tc/tc /usr/local/sbin/ntc && cd -
+      - run: loxilb-ebpf/utils/mkllb_bpffs.sh
+      - run: sudo -E env "PATH=$PATH" make
+      - run: sudo -E env "PATH=$PATH" make test
+      - run: docker pull ghcr.io/loxilb-io/loxilb:latest
+      - run: docker run -u root --cap-add SYS_ADMIN   --restart unless-stopped --privileged -dit -v /dev/log:/dev/log --name loxilb ghcr.io/loxilb-io/loxilb:latest
+      - run: pwd && ls && sudo -E env "PATH=$PATH" make docker-cp
+      - run: docker exec -dit loxilb mkllb_bpffs
+      - run: id=`docker ps -f name=loxilb | cut  -d " "  -f 1 | grep -iv  "CONTAINER"` && docker commit $id ghcr.io/loxilb-io/loxilb:latest
+      - run: docker stop loxilb && docker rm loxilb
+      - run: |
+             cd cicd/sctpmh/
+             ./config.sh
+             ./validation.sh
+             ./rmconfig.sh
+             cd -

Dockerfile

@@ -10,8 +10,8 @@ ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/lib64/"
 
 # Install loxilb related packages
 RUN mkdir -p /opt/loxilb && \
-    mkdir -p /opt/loxilb/cert/ && \
     mkdir -p /root/loxilb-io/loxilb/ && \
+    mkdir -p /opt/loxilb/cert/ && \
     mkdir -p /etc/bash_completion.d/ && \
     # Update Ubuntu Software repository
     apt-get update && apt-get install -y wget && \
@@ -44,7 +44,6 @@ RUN mkdir -p /opt/loxilb && \
     cd /root/loxilb-io/loxilb/ && go get . && if [ "$arch" = "arm64" ] ; then DOCKER_BUILDX_ARM64=true make; \
     else make ;fi && cp loxilb-ebpf/utils/mkllb_bpffs.sh /usr/local/sbin/mkllb_bpffs && \
     cp loxilb-ebpf/utils/mkllb_cgroup.sh /usr/local/sbin/mkllb_cgroup && \
-    cp api/certification/* /opt/loxilb/cert/ && cd - && \
     cp /root/loxilb-io/loxilb/loxilb-ebpf/kernel/loxilb_dp_debug  /usr/local/sbin/loxilb_dp_debug && \
     cp /root/loxilb-io/loxilb/loxilb /usr/local/sbin/loxilb && \
     rm -fr /root/loxilb-io/loxilb/* && rm -fr /root/loxilb-io/loxilb/.git && \

Makefile

@@ -31,6 +31,9 @@ docker-cp: build
 	docker cp /opt/loxilb/llb_ebpf_emain.o $(loxilbid):/opt/loxilb/llb_ebpf_emain.o
 	docker cp /opt/loxilb/llb_xdp_main.o $(loxilbid):/opt/loxilb/llb_xdp_main.o
 	docker cp /opt/loxilb/llb_kern_sock.o $(loxilbid):/opt/loxilb/llb_kern_sock.o
+	docker cp /opt/loxilb/llb_kern_sockmap.o $(loxilbid):/opt/loxilb/llb_kern_sockmap.o
+	docker cp /opt/loxilb/llb_kern_sockstream.o $(loxilbid):/opt/loxilb/llb_kern_sockstream.o
+	docker cp /opt/loxilb/llb_kern_sockdirect.o $(loxilbid):/opt/loxilb/llb_kern_sockdirect.o
 	docker cp loxilb-ebpf/kernel/loxilb_dp_debug  $(loxilbid):/usr/local/sbin/
 	docker cp loxilb-ebpf/libbpf/src/libbpf.so.0.8.1 $(loxilbid):/usr/lib64/
 	docker cp loxilb-ebpf/utils/loxilb_dp_tool $(loxilbid):/usr/local/sbin/

README.md

@@ -18,7 +18,7 @@ All these services are provided by load-balancers/proxies operating at Layer4/La
 
 Service type load-balancer is usually provided by public cloud-provider(s) as a managed entity. But for on-prem and self-managed clusters, there are only a few good options available. Even for provider-managed K8s like EKS, there are many who would want to bring their own LB to clusters running anywhere. <b>loxilb provides service type load-balancer as its main use-case</b>. loxilb can be run in-cluster or ext-to-cluster as per user need.  
 
-Additionally, loxilb can also support cluster-ip and node-port services, thereby providing end-to-end connectivity for Kubernetes.
+Additionally, loxilb can also support cluster-ip and node-port services, thereby providing full cluster-mesh implementation for Kubernetes (replacment of kube-proxy).
 
 ## Why choose loxilb?
 
@@ -30,6 +30,7 @@ Additionally, loxilb can also support cluster-ip and node-port services, thereby
 - Utitlizes ebpf which makes it ```flexible``` as well as ```customizable```
 - Advanced ```quality of service``` for workloads (per LB, per end-point or per client)
 - Works with ```any``` Kubernetes distribution/CNI - k8s/k3s/k0s/kind/OpenShift + Calico/Flannel/Cilium/Weave/Multus etc
+- Kube-proxy replacement with loxilb allows ```simple plug-in``` with any existing/deployed pod-networking software
 - Extensive support for ```SCTP workloads``` (with multi-homing) on K8s
 - Dual stack with ```NAT66, NAT64``` support for K8s
 - K8s ```multi-cluster``` support (planned 🚧)
@@ -53,16 +54,17 @@ Additionally, loxilb can also support cluster-ip and node-port services, thereby
 - A kubernetes agent [kube-loxilb](https://github.com/loxilb-io/kube-loxilb) written in Go
 
 ## Layer4 Vs Layer7
-loxilb works as a L4 load-balancer/service-mesh by default. Although it provides great performance, at times, L7 load-balancing might become necessary in K8s. There are many good L7 proxies already available for K8s. Still, we are working on providing a great L7 solution natively in eBPF. It is a tough endeavor one which should reap great benefits once completed. Please keep an eye for updates on this.
+loxilb works as a L4 load-balancer/service-proxy by default. Although it provides great performance, at times, L7 load-balancing might become necessary in K8s. There are many good L7 proxies already available for K8s. Still, we are working on providing a great L7 solution natively in eBPF. It is a tough endeavor one which should reap great benefits once completed. Please keep an eye for updates on this.
 
 ## Telco-Cloud with loxilb
-For deploying telco-cloud with cloud-native functions, loxilb can be used as a SCP(service communication proxy). SCP is nothing but a glorified term for Kubernetes load-balancing/proxy. But telco-cloud requires load-balancing across various interfaces/standards like N2, N4, E2(ORAN), S6x, 5GLAN, GTP etc. Each of these interfaces present its own unique challenges(and DPI) for load-balancing which loxilb aims to solve e.g.
+For deploying telco-cloud with cloud-native functions, loxilb can be used as a SCP(service communication proxy). SCP is nothing but a communication hub for telco micro-services running in Kubernetes. But telco-cloud requires load-balancing and communication across various interfaces/standards like N2, N4, E2(ORAN), S6x, 5GLAN, GTP etc. Each of these present its own unique challenges which loxilb aims to solve e.g.
 - N4 requires PFCP level session-intelligence
 - N2 requires NGAP parsing capability
 - S6x requires Diameter/SCTP multi-homing LB support
 - MEC use-cases might require UL-CL understanding
 - Hitless failover support might be essential for mission-critical applications
 - E2 might require SCTP-LB with OpenVPN bundled together
+- SIP support is needed to enable cloud-native VOIP
 
 ## How-To Guides
 - [How-To : Deploy loxilb in K8s with kube-loxilb](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/kube-loxilb.md)
@@ -71,7 +73,9 @@ For deploying telco-cloud with cloud-native functions, loxilb can be used as a S
 - [How-To : Run loxilb in standalone mode](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/standalone.md)
 - [How-To : Manual build/run](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/run.md)
 - [How-To : Standalone configuration](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/cmd.md)
-- [How-To : debug](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/debugging.md)
+- [How-To : Debug loxilb](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/debugging.md)
+- [How-To : Access end-points outside K8s](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/ext-ep.md)
+- [How-To : Deploy multi-server K3s HA with loxilb](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/k3s-multi-master.md)
 
 ## Getting started with different K8s distributions/tools   
 
@@ -86,6 +90,10 @@ For deploying telco-cloud with cloud-native functions, loxilb can be used as a S
 - [K0s : loxilb in-cluster mode](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/k0s_quick_start_incluster.md)
 - [MicroK8s : loxilb in-cluster mode](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/microk8s_quick_start_incluster.md)
 
+#### loxilb as service-proxy (kube-proxy replacement)
+- [K3s : loxilb service-proxy with flannel](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/service-proxy-flannel.md)
+- [K3s : loxilb service-proxy with calico](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/service-proxy-calico.md)
+
 ## Knowledge-Base   
 - [What is eBPF](ebpf.md)
 - [What is k8s service - load-balancer](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/lb.md)
@@ -122,7 +130,7 @@ Feel free to post your queries in github [discussion](https://github.com/loxilb-
 | [![udp-lb-sanity-CI](https://github.com/loxilb-io/loxilb/actions/workflows/udp-sanity.yml/badge.svg)](https://github.com/loxilb-io/loxilb/actions/workflows/udp-sanity.yml) | [![udp-lb-sanity-CI](https://github.com/loxilb-io/loxilb/actions/workflows/udp-sanity-ubuntu-22.yml/badge.svg)](https://github.com/loxilb-io/loxilb/actions/workflows/udp-sanity-ubuntu-22.yml) | [![UDP-LB-Sanity-CI-RH9](https://github.com/loxilb-io/loxilb/actions/workflows/udp-sanity-rh9.yml/badge.svg)](https://github.com/loxilb-io/loxilb/actions/workflows/udp-sanity-rh9.yml) |
 | [![sctp-lb-sanity-CI](https://github.com/loxilb-io/loxilb/actions/workflows/sctp-sanity.yml/badge.svg)](https://github.com/loxilb-io/loxilb/actions/workflows/sctp-sanity.yml)  | ![ipsec-sanity-CI](https://github.com/loxilb-io/loxilb/actions/workflows/ipsec-sanity-ubuntu-22.yml/badge.svg)  | [![IPsec-Sanity-CI-RH9](https://github.com/loxilb-io/loxilb/actions/workflows/ipsec-sanity-rh9.yml/badge.svg)](https://github.com/loxilb-io/loxilb/actions/workflows/ipsec-sanity-rh9.yml) |
 | ![extlb workflow](https://github.com/loxilb-io/loxilb/actions/workflows/advanced-lb-sanity.yml/badge.svg) | ![nat66-sanity-CI](https://github.com/loxilb-io/loxilb/actions/workflows/nat66-sanity-ubuntu-22.yml/badge.svg)  | [![NAT66-LB-Sanity-CI-RH9](https://github.com/loxilb-io/loxilb/actions/workflows/nat66-sanity-rh9.yml/badge.svg)](https://github.com/loxilb-io/loxilb/actions/workflows/nat66-sanity-rh9.yml) | 
-| ![ipsec-sanity-CI](https://github.com/loxilb-io/loxilb/actions/workflows/ipsec-sanity.yml/badge.svg)   | [![Scale-Sanity-CI-Ubuntu-22](https://github.com/loxilb-io/loxilb/actions/workflows/scale-sanity-ubuntu-22.yml/badge.svg)](https://github.com/loxilb-io/loxilb/actions/workflows/scale-sanity-ubuntu-22.yml) | |
+| ![ipsec-sanity-CI](https://github.com/loxilb-io/loxilb/actions/workflows/ipsec-sanity.yml/badge.svg)   | [![Scale-Sanity-CI-Ubuntu-22](https://github.com/loxilb-io/loxilb/actions/workflows/scale-sanity-ubuntu-22.yml/badge.svg)](https://github.com/loxilb-io/loxilb/actions/workflows/scale-sanity-ubuntu-22.yml) | [![Adv-LB-Sanity-CI-RH9](https://github.com/loxilb-io/loxilb/actions/workflows/advanced-lb-sanity-rh9.yml/badge.svg)](https://github.com/loxilb-io/loxilb/actions/workflows/advanced-lb-sanity-rh9.yml) |
 | ![scale-sanity-CI](https://github.com/loxilb-io/loxilb/actions/workflows/scale-sanity.yml/badge.svg)  | [![perf-CI](https://github.com/loxilb-io/loxilb/actions/workflows/perf.yml/badge.svg)](https://github.com/loxilb-io/loxilb/actions/workflows/perf.yml) | | 
 | [![liveness-sanity-CI](https://github.com/loxilb-io/loxilb/actions/workflows/liveness-sanity.yml/badge.svg)](https://github.com/loxilb-io/loxilb/actions/workflows/liveness-sanity.yml)  | | |
 | ![nat66-sanity-CI](https://github.com/loxilb-io/loxilb/actions/workflows/nat66-sanity.yml/badge.svg)   | | |

api/api.go

@@ -17,6 +17,11 @@
 package api
 
 import (
+	"log"
+	"os"
+	"runtime/debug"
+	"time"
+
 	"github.com/go-openapi/loads"
 	flags "github.com/jessevdk/go-flags"
 	"github.com/loxilb-io/loxilb/api/restapi"
@@ -25,10 +30,6 @@ import (
 	cmn "github.com/loxilb-io/loxilb/common"
 	"github.com/loxilb-io/loxilb/options"
 	tk "github.com/loxilb-io/loxilib"
-	"log"
-	"os"
-	"runtime/debug"
-	"time"
 )
 
 var (
@@ -117,11 +118,17 @@ func RunAPIServer() {
 	server.ConfigureAPI()
 	// API server host list
 	server.Host = options.Opts.Host
-	server.TLSHost = options.Opts.TLSHost
-	server.TLSCertificateKey = options.Opts.TLSCertificateKey
-	server.TLSCertificate = options.Opts.TLSCertificate
 	server.Port = options.Opts.Port
-	server.TLSPort = options.Opts.TLSPort
+
+	// HTTPs List
+	if options.Opts.TLS {
+		server.TLSHost = options.Opts.TLSHost
+		server.TLSPort = options.Opts.TLSPort
+
+		server.TLSCertificateKey = options.Opts.TLSCertificateKey
+		server.TLSCertificate = options.Opts.TLSCertificate
+	}
+
 	api.ServerShutdown = func() {
 		waitApiServerShutOk()
 		os.Exit(0)

api/build_api.sh

@@ -0,0 +1,3 @@
+sudo docker run --rm -it  --user $(id -u):$(id -g) -e GOPATH=$(go env GOPATH):/go -v $HOME:$HOME -w $(pwd) quay.io/goswagger/swagger:0.30.3 generate server
+sed -i 's/s\.hasScheme(schemeHTTPS)/s\.hasScheme(schemeHTTPS) \&\& options\.Opts\.TLS/gi' restapi/server.go
+sed -i'' -r -e '/import/a\\t\"github.com/loxilb-io/loxilb/options\"' restapi/server.go