PR - Merge from main

.github/workflows/docker-multiarch.yml

@@ -18,6 +18,11 @@ jobs:
     - uses: actions/checkout@v2
       with:
           submodules: recursive
+    #- name : Checkout Branch
+    #  if:  github.event.inputs.tagName != ''
+    #  run: |
+    #      git fetch --all --tags 
+    #      git checkout ${{ github.event.inputs.tagName }}
 
     - name: Login to GitHub Container Registry
       uses: docker/login-action@v1
@@ -64,5 +69,6 @@ jobs:
         context: .
         platforms: linux/amd64, linux/arm64
         push: true
+        build-args: TAG=${{ github.event.inputs.tagName }}
         tags: ghcr.io/loxilb-io/loxilb:${{ github.event.inputs.tagName }}
 

Dockerfile

@@ -4,14 +4,18 @@ FROM ubuntu:20.04 as build
 # Disable Prompt During Packages Installation
 ARG DEBIAN_FRONTEND=noninteractive
 
+ARG TAG=main
+
 # Env variables
 ENV PATH="${PATH}:/usr/local/go/bin"
 ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/lib64/"
 
 # Install loxilb related packages
 RUN mkdir -p /opt/loxilb && \
     mkdir -p /root/loxilb-io/loxilb/ && \
+    mkdir -p /usr/lib64/ && \
     mkdir -p /opt/loxilb/cert/ && \
+    mkdir -p /etc/loxilb/certs/ && \
     mkdir -p /etc/bash_completion.d/ && \
     # Update Ubuntu Software repository
     apt-get update && apt-get install -y wget && \
@@ -22,6 +26,14 @@ RUN mkdir -p /opt/loxilb && \
     apt-get install -y clang llvm libelf-dev libpcap-dev vim net-tools \
     elfutils dwarves git libbsd-dev bridge-utils wget unzip build-essential \
     bison flex sudo iproute2 pkg-config tcpdump iputils-ping curl bash-completion && \
+    # Install openssl-3.0.0
+    wget https://www.openssl.org/source/openssl-3.0.0.tar.gz && tar -xvzf openssl-3.0.0.tar.gz && \
+    cd openssl-3.0.0 && ./Configure enable-ktls '-Wl,-rpath,$(LIBRPATH)' --prefix=/usr/local/build && \
+    make -j$(nproc) && make install_dev install_modules && cd - && \
+    cp -a /usr/local/build/include/openssl /usr/include/ && \
+    if [ -d /usr/local/build/lib64  ] ; then mv /usr/local/build/lib64  /usr/local/build/lib; fi && \
+    cp -fr /usr/local/build/lib/* /usr/lib/ && ldconfig && \
+    rm -fr openssl-3.0.0*  && \
     # Install loxilb's custom ntc tool
     wget https://github.com/loxilb-io/iproute2/archive/refs/heads/main.zip && \
     unzip main.zip && cd iproute2-main/ && rm -fr libbpf && wget https://github.com/loxilb-io/libbpf/archive/refs/heads/main.zip && \
@@ -31,17 +43,20 @@ RUN mkdir -p /opt/loxilb && \
     LIBBPF_FORCE=on LIBBPF_DIR=`pwd`/libbpf/src/build ./configure && make && \
     cp -f tc/tc /usr/local/sbin/ntc && cd .. && rm -fr main.zip iproute2-main && \
     # Install bpftool
-    git clone --recurse-submodules https://github.com/libbpf/bpftool.git && cd bpftool/src/ && \
-    git switch --detach v7.2.0 && \
+    wget https://github.com/libbpf/bpftool/releases/download/v7.2.0/bpftool-libbpf-v7.2.0-sources.tar.gz && \
+    tar -xvzf bpftool-libbpf-v7.2.0-sources.tar.gz && cd bpftool/src/ && \
     make clean && 	make -j $(nproc) && cp -f ./bpftool /usr/local/sbin/bpftool && \
-    cd - && rm -fr bpftool && \
+    cd - && rm -fr bpftool* && \
     # Install loxicmd
-    git clone https://github.com/loxilb-io/loxicmd.git && cd loxicmd && go get . && \
+    git clone https://github.com/loxilb-io/loxicmd.git && cd loxicmd && git fetch --all --tags && \
+    git checkout $TAG && go get . && \
     make && cp ./loxicmd /usr/local/sbin/loxicmd && cd - && rm -fr loxicmd && \
     /usr/local/sbin/loxicmd completion bash > /etc/bash_completion.d/loxi_completion && \
     # Install loxilb
     git clone --recurse-submodules https://github.com/loxilb-io/loxilb  /root/loxilb-io/loxilb/ && \
-    cd /root/loxilb-io/loxilb/ && go get . && if [ "$arch" = "arm64" ] ; then DOCKER_BUILDX_ARM64=true make; \
+    cd /root/loxilb-io/loxilb/ && git fetch --all --tags && git checkout $TAG && \
+    cd loxilb-ebpf && git fetch --all --tags && git checkout $TAG && cd .. \
+    go get . && if [ "$arch" = "arm64" ] ; then DOCKER_BUILDX_ARM64=true make; \
     else make ;fi && cp loxilb-ebpf/utils/mkllb_bpffs.sh /usr/local/sbin/mkllb_bpffs && \
     cp loxilb-ebpf/utils/mkllb_cgroup.sh /usr/local/sbin/mkllb_cgroup && \
     cp /root/loxilb-io/loxilb/loxilb-ebpf/kernel/loxilb_dp_debug  /usr/local/sbin/loxilb_dp_debug && \
@@ -85,6 +100,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends sudo \
     rm -rf /var/lib/apt/lists/* && apt clean
 
 COPY --from=build /usr/lib64/libbpf* /usr/lib64/
+COPY --from=build /usr/local/build/lib/* /usr/lib64
 COPY --from=build /usr/local/go/bin /usr/local/go/bin
 COPY --from=build /usr/local/sbin/mkllb_bpffs /usr/local/sbin/mkllb_bpffs
 COPY --from=build /usr/local/sbin/mkllb_cgroup /usr/local/sbin/mkllb_cgroup

README.md

@@ -76,6 +76,7 @@ For deploying telco-cloud with cloud-native functions, loxilb can be used as a S
 - [How-To : Debug loxilb](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/debugging.md)
 - [How-To : Access end-points outside K8s](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/ext-ep.md)
 - [How-To : Deploy multi-server K3s HA with loxilb](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/k3s-multi-master.md)
+- [How-To : Deploy loxilb with multi-AZ HA support in AWS](https://github.com/loxilb-io/loxilbdocs/blob/main/docs/aws-multi-az.md)
 
 ## Getting started with different K8s distributions/tools   
 

api/loxinlp/nlp.go

@@ -30,6 +30,7 @@ import (
 	"time"
 
 	cmn "github.com/loxilb-io/loxilb/common"
+	opt "github.com/loxilb-io/loxilb/options"
 	tk "github.com/loxilb-io/loxilib"
 	nlp "github.com/vishvananda/netlink"
 	"golang.org/x/sys/unix"
@@ -105,7 +106,8 @@ func iSBlackListedIntf(name string, masterIdx int) bool {
 }
 
 func applyAllConfig(name string) bool {
-	command := "loxicmd apply --per-intf " + name + " -c /etc/loxilb/ipconfig/"
+
+	command := "loxicmd apply --per-intf " + name + " -c " + opt.Opts.ConfigPath + "/ipconfig/"
 	cmd := exec.Command("bash", "-c", command)
 	output, err := cmd.Output()
 	if err != nil {
@@ -120,7 +122,8 @@ func applyLoadBalancerConfig() bool {
 	var resp struct {
 		Attr []cmn.LbRuleMod `json:"lbAttr"`
 	}
-	byteBuf, err := os.ReadFile("/etc/loxilb/lbconfig.txt")
+	dpath := opt.Opts.ConfigPath + "/lbconfig.txt"
+	byteBuf, err := os.ReadFile(dpath)
 	if err != nil {
 		fmt.Println(err.Error())
 		return false
@@ -141,7 +144,8 @@ func applySessionConfig() bool {
 	var resp struct {
 		Attr []cmn.SessionMod `json:"sessionAttr"`
 	}
-	byteBuf, err := os.ReadFile("/etc/loxilb/sessionconfig.txt")
+	dpath := opt.Opts.ConfigPath + "/sessionconfig.txt"
+	byteBuf, err := os.ReadFile(dpath)
 	if err != nil {
 		fmt.Println(err.Error())
 		return false
@@ -162,7 +166,8 @@ func applyUlClConfig() bool {
 	var resp struct {
 		Attr []cmn.SessionUlClMod `json:"ulclAttr"`
 	}
-	byteBuf, err := os.ReadFile("/etc/loxilb/sessionulclconfig.txt")
+	dpath := opt.Opts.ConfigPath + "/sessionulclconfig.txt"
+	byteBuf, err := os.ReadFile(dpath)
 	if err != nil {
 		fmt.Println(err.Error())
 		return false
@@ -183,7 +188,8 @@ func applyFWConfig() bool {
 	var resp struct {
 		Attr []cmn.FwRuleMod `json:"fwAttr"`
 	}
-	byteBuf, err := os.ReadFile("/etc/loxilb/FWconfig.txt")
+	dpath := opt.Opts.ConfigPath + "/FWconfig.txt"
+	byteBuf, err := os.ReadFile(dpath)
 	if err != nil {
 		fmt.Println(err.Error())
 		return false
@@ -204,7 +210,8 @@ func applyEPConfig() bool {
 	var resp struct {
 		Attr []cmn.EndPointMod `json:"Attr"`
 	}
-	byteBuf, err := os.ReadFile("/etc/loxilb/EPconfig.txt")
+	dpath := opt.Opts.ConfigPath + "/EPconfig.txt"
+	byteBuf, err := os.ReadFile(dpath)
 	if err != nil {
 		fmt.Println(err.Error())
 		return false
@@ -225,7 +232,8 @@ func ApplyBFDConfig() bool {
 	var resp struct {
 		Attr []cmn.BFDMod `json:"Attr"`
 	}
-	byteBuf, err := os.ReadFile("/etc/loxilb/BFDconfig.txt")
+	dpath := opt.Opts.ConfigPath + "/BFDconfig.txt"
+	byteBuf, err := os.ReadFile(dpath)
 	if err != nil {
 		fmt.Println(err.Error())
 		return false
@@ -244,7 +252,7 @@ func ApplyBFDConfig() bool {
 
 func applyRoutes(name string) {
 	tk.LogIt(tk.LogDebug, "[NLP] Applying Route Config for %s \n", name)
-	command := "loxicmd apply --per-intf " + name + " -r -c /etc/loxilb/ipconfig/"
+	command := "loxicmd apply --per-intf " + name + " -r -c " + opt.Opts.ConfigPath + "/ipconfig/"
 	cmd := exec.Command("bash", "-c", command)
 	output, err := cmd.Output()
 	if err != nil {
@@ -257,7 +265,9 @@ func applyRoutes(name string) {
 func applyConfigMap(name string, state bool, add bool) {
 	var configApplied bool
 	var needRouteApply bool
-	if _, err := os.Stat("/etc/loxilb/ipconfig/"); errors.Is(err, os.ErrNotExist) {
+	dpath := opt.Opts.ConfigPath + "/ipconfig/"
+
+	if _, err := os.Stat(dpath); errors.Is(err, os.ErrNotExist) {
 		return
 	}
 	if add {
@@ -1200,7 +1210,7 @@ func AddRoute(route nlp.Route) int {
 	return ret
 }
 
-func AddRouteNoHook(DestinationIPNet, gateway string) int {
+func AddRouteNoHook(DestinationIPNet, gateway, proto string) int {
 	var ret int
 	var route nlp.Route
 	_, Ipnet, err := net.ParseCIDR(DestinationIPNet)
@@ -1210,6 +1220,10 @@ func AddRouteNoHook(DestinationIPNet, gateway string) int {
 	Gw := net.ParseIP(gateway)
 	route.Dst = Ipnet
 	route.Gw = Gw
+
+	if proto == "static" {
+		route.Protocol = 4 // 4 means Proto Static.
+	}
 	err = nlp.RouteAdd(&route)
 	if err != nil {
 		return -1
@@ -1577,7 +1591,7 @@ func LbSessionGet(done bool) int {
 
 	if done {
 
-		if _, err := os.Stat("/etc/loxilb/EPconfig.txt"); errors.Is(err, os.ErrNotExist) {
+		if _, err := os.Stat(opt.Opts.ConfigPath + "/EPconfig.txt"); errors.Is(err, os.ErrNotExist) {
 			if err != nil {
 				tk.LogIt(tk.LogInfo, "[NLP] No EndPoint config file : %s \n", err.Error())
 			}
@@ -1586,7 +1600,7 @@ func LbSessionGet(done bool) int {
 		}
 		tk.LogIt(tk.LogInfo, "[NLP] EndPoint done\n")
 
-		if _, err := os.Stat("/etc/loxilb/lbconfig.txt"); errors.Is(err, os.ErrNotExist) {
+		if _, err := os.Stat(opt.Opts.ConfigPath + "/lbconfig.txt"); errors.Is(err, os.ErrNotExist) {
 			if err != nil {
 				tk.LogIt(tk.LogInfo, "[NLP] No load balancer config file : %s \n", err.Error())
 			}
@@ -1595,7 +1609,7 @@ func LbSessionGet(done bool) int {
 		}
 
 		tk.LogIt(tk.LogInfo, "[NLP] LoadBalancer done\n")
-		if _, err := os.Stat("/etc/loxilb/sessionconfig.txt"); errors.Is(err, os.ErrNotExist) {
+		if _, err := os.Stat(opt.Opts.ConfigPath + "/sessionconfig.txt"); errors.Is(err, os.ErrNotExist) {
 			if err != nil {
 				tk.LogIt(tk.LogInfo, "[NLP] No Session config file : %s \n", err.Error())
 			}
@@ -1604,7 +1618,7 @@ func LbSessionGet(done bool) int {
 		}
 
 		tk.LogIt(tk.LogInfo, "[NLP] Session done\n")
-		if _, err := os.Stat("/etc/loxilb/sessionulclconfig.txt"); errors.Is(err, os.ErrNotExist) {
+		if _, err := os.Stat(opt.Opts.ConfigPath + "/sessionulclconfig.txt"); errors.Is(err, os.ErrNotExist) {
 			if err != nil {
 				tk.LogIt(tk.LogInfo, "[NLP] No UlCl config file : %s \n", err.Error())
 			}
@@ -1613,7 +1627,7 @@ func LbSessionGet(done bool) int {
 		}
 
 		tk.LogIt(tk.LogInfo, "[NLP] Session UlCl done\n")
-		if _, err := os.Stat("/etc/loxilb/FWconfig.txt"); errors.Is(err, os.ErrNotExist) {
+		if _, err := os.Stat(opt.Opts.ConfigPath + "/FWconfig.txt"); errors.Is(err, os.ErrNotExist) {
 			if err != nil {
 				tk.LogIt(tk.LogInfo, "[NLP] No Firewall config file : %s \n", err.Error())
 			}

api/restapi/configure_loxilb_rest_api.go

@@ -176,8 +176,9 @@ func configureAPI(api *operations.LoxilbRestAPIAPI) http.Handler {
 
 	// BGP Policy Apply
 	api.PostConfigBgpPolicyApplyHandler = operations.PostConfigBgpPolicyApplyHandlerFunc(handler.ConfigPostBGPPolicyApply)
-	api.PreServerShutdown = func() {}
+	api.DeleteConfigBgpPolicyApplyHandler = operations.DeleteConfigBgpPolicyApplyHandlerFunc(handler.ConfigDeleteBGPPolicyApply)
 
+	api.PreServerShutdown = func() {}
 	api.ServerShutdown = func() {}
 
 	return setupGlobalMiddleware(api.Serve(setupMiddlewares))