gh-730 Initial support for ecmp load-balancing

api/loxinlp/nlp.go

@@ -1189,8 +1189,18 @@ func AddRoute(route nlp.Route) int {
 		ipNet = *route.Dst
 	}
 
-	ret, err := hooks.NetRouteAdd(&cmn.RouteMod{Protocol: int(route.Protocol), Flags: route.Flags,
-		Gw: route.Gw, LinkIndex: route.LinkIndex, Dst: ipNet})
+	var gws []cmn.GWInfo
+
+	if len(route.MultiPath) <= 0 {
+		gw := cmn.GWInfo{Gw: route.Gw, LinkIndex: route.LinkIndex}
+		gws = append(gws, gw)
+	} else {
+		for i := range route.MultiPath {
+			gws = append(gws, cmn.GWInfo{Gw: route.MultiPath[i].Gw, LinkIndex: route.MultiPath[i].LinkIndex})
+		}
+	}
+
+	ret, err := hooks.NetRouteAdd(&cmn.RouteMod{Protocol: int(route.Protocol), Flags: route.Flags, Dst: ipNet, GWs: gws})
 	if err != nil {
 		if route.Gw != nil {
 			tk.LogIt(tk.LogError, "[NLP] RT  %s via %s proto %d add failed-%s\n", ipNet.String(),
@@ -1350,14 +1360,28 @@ func NUWorkSingle(m nlp.NeighUpdate) int {
 func RUWorkSingle(m nlp.RouteUpdate) int {
 	var ret int
 
-	link, err := nlp.LinkByIndex(m.LinkIndex)
-	if err != nil {
-		fmt.Println(err)
-		return -1
-	}
+	if len(m.MultiPath) <= 0 {
+		link, err := nlp.LinkByIndex(m.LinkIndex)
+		if err != nil {
+			tk.LogIt(tk.LogError, "RUWorkSingle: link find error %s", err)
+			return -1
+		}
 
-	if iSBlackListedIntf(link.Attrs().Name, link.Attrs().MasterIndex) {
-		return -1
+		if iSBlackListedIntf(link.Attrs().Name, link.Attrs().MasterIndex) {
+			return -1
+		}
+	} else {
+		for _, path := range m.MultiPath {
+			link, err := nlp.LinkByIndex(path.LinkIndex)
+			if err != nil {
+				tk.LogIt(tk.LogError, "RUWorkSingle: link find error %s", err)
+				return -1
+			}
+
+			if iSBlackListedIntf(link.Attrs().Name, link.Attrs().MasterIndex) {
+				return -1
+			}
+		}
 	}
 
 	if skipIfRoute {
@@ -1559,25 +1583,28 @@ func NlpGet(ch chan bool) int {
 				AddNeigh(neigh, link)
 			}
 		}
+	}
 
-		/* Get Routes */
-		routes, err := nlp.RouteList(link, nlp.FAMILY_ALL)
-		if err != nil {
-			tk.LogIt(tk.LogError, "[NLP] Error getting route list %v\n", err)
-		}
+	/* Get Routes */
+	routes, err := nlp.RouteList(nil, nlp.FAMILY_ALL)
+	if err != nil {
+		tk.LogIt(tk.LogError, "[NLP] Error getting route list %v\n", err)
+	}
 
-		if len(routes) == 0 {
-			tk.LogIt(tk.LogDebug, "[NLP] No STATIC routes found for intf %s\n", link.Attrs().Name)
-		} else {
-			for _, route := range routes {
-				if skipIfRoute {
-					if route.Scope.String() == "link" && tk.IsNetIPv4(route.Dst.IP.String()) {
-						continue
-					}
+	if len(routes) == 0 {
+		tk.LogIt(tk.LogDebug, "[NLP] No STATIC routes found\n")
+	} else {
+		for _, route := range routes {
+			var m nlp.RouteUpdate
+			if skipIfRoute {
+				if route.Scope.String() == "link" && tk.IsNetIPv4(route.Dst.IP.String()) {
+					continue
 				}
-
-				AddRoute(route)
 			}
+			m.Type = syscall.RTM_NEWROUTE
+			m.Route = route
+
+			RUWorkSingle(m)
 		}
 	}
 	tk.LogIt(tk.LogInfo, "[NLP] nlp get done\n")

common/common.go

@@ -369,16 +369,22 @@ type RouteGet struct {
 	Sync DpStatusT
 }
 
+// GWInfo - Info about gateway
+type GWInfo struct {
+	// Gw - gateway information if any
+	Gw net.IP
+	// LinkIndex - OS allocated index
+	LinkIndex int
+}
+
 // RouteMod - Info about a route
 type RouteMod struct {
 	// Protocol - Protocol type
 	Protocol int
 	// Flags - flag type
 	Flags int
-	// Gw - gateway information if any
-	Gw net.IP
-	// LinkIndex - OS allocated index
-	LinkIndex int
+	// GWs - gateway information if any
+	GWs []GWInfo
 	// Dst - ip addr
 	Dst net.IPNet
 }

pkg/loxinet/apiclient.go

@@ -279,25 +279,32 @@ func (na *NetAPIStruct) NetRouteAdd(rm *cmn.RouteMod) (int, error) {
 	var ret int
 	var err error
 
+	if len(rm.GWs) <= 0 {
+		return RtNhErr, errors.New("invalid gws")
+	}
 	if na.BgpPeerMode {
 		return RtNhErr, errors.New("running in bgp only mode")
 	}
 	intfRt := false
 	mlen, _ := rm.Dst.Mask.Size()
-	if rm.Gw == nil {
+	if rm.GWs[0].Gw == nil {
 		// This is an interface route
 		if (tk.IsNetIPv4(rm.Dst.IP.String()) && mlen == 32) || (tk.IsNetIPv6(rm.Dst.IP.String()) && mlen == 128) {
 			intfRt = true
-			rm.Gw = rm.Dst.IP
+			rm.GWs[0].Gw = rm.Dst.IP
 		}
 	}
 	mh.mtx.Lock()
 	defer mh.mtx.Unlock()
 
-	ra := RtAttr{Protocol: rm.Protocol, OSFlags: rm.Flags, HostRoute: false, Ifi: rm.LinkIndex, IfRoute: intfRt}
-	if rm.Gw != nil {
-		na := []RtNhAttr{{rm.Gw, rm.LinkIndex}}
+	ra := RtAttr{Protocol: rm.Protocol, OSFlags: rm.Flags, HostRoute: false, Ifi: rm.GWs[0].LinkIndex, IfRoute: intfRt}
+	if rm.GWs[0].Gw != nil {
+		var na []RtNhAttr
+		for _, gw := range rm.GWs {
+			na = append(na, RtNhAttr{gw.Gw, gw.LinkIndex})
+		}
 		ret, err = mh.zr.Rt.RtAdd(rm.Dst, RootZone, ra, na)
+
 	} else {
 		ret, err = mh.zr.Rt.RtAdd(rm.Dst, RootZone, ra, nil)
 	}

pkg/loxinet/dpbroker.go

@@ -168,7 +168,8 @@ type RouteDpWorkQ struct {
 	Dst     net.IPNet
 	RtType  int
 	RtMark  int
-	NMark   int
+	NMax    int
+	NMark   [8]int
 }
 
 // StatDpWorkQ - work queue entry for stat operation

pkg/loxinet/dpebpf_linux.go

@@ -698,7 +698,7 @@ func DpRouterMacMod(w *RouterMacDpWorkQ) int {
 				rtNhAct := (*rtNhAct)(getPtrOffset(unsafe.Pointer(dat),
 					C.sizeof_struct_dp_cmn_act))
 				C.memset(unsafe.Pointer(rtNhAct), 0, C.sizeof_struct_dp_rt_nh_act)
-				rtNhAct.nh_num = 0
+				rtNhAct.nh_num[0] = 0
 				rtNhAct.tid = 0
 				rtNhAct.bd = C.ushort(w.BD)
 			} else {
@@ -710,7 +710,7 @@ func DpRouterMacMod(w *RouterMacDpWorkQ) int {
 					C.sizeof_struct_dp_cmn_act))
 				C.memset(unsafe.Pointer(rtNhAct), 0, C.sizeof_struct_dp_rt_nh_act)
 
-				rtNhAct.nh_num = C.ushort(w.NhNum)
+				rtNhAct.nh_num[0] = C.ushort(w.NhNum)
 				tid := ((w.TunID << 8) & 0xffffff00)
 				rtNhAct.tid = C.uint(tk.Htonl(tid))
 			}
@@ -882,11 +882,16 @@ func DpRouteMod(w *RouteDpWorkQ) int {
 		dat := new(rtDat)
 		C.memset(unsafe.Pointer(dat), 0, C.sizeof_struct_dp_rt_tact)
 
-		if w.NMark >= 0 {
+		if w.NMax > 0 {
 			dat.ca.act_type = C.DP_SET_RT_NHNUM
 			act = (*rtL3NhAct)(getPtrOffset(unsafe.Pointer(dat),
 				C.sizeof_struct_dp_cmn_act))
-			act.nh_num = C.ushort(w.NMark)
+			act.naps = C.ushort(w.NMax)
+			for i := range w.NMark {
+				if i < C.DP_MAX_ACTIVE_PATHS {
+					act.nh_num[i] = C.ushort(w.NMark[i])
+				}
+			}
 		} else {
 			dat.ca.act_type = C.DP_SET_TOCP
 		}

pkg/loxinet/route.go

@@ -138,7 +138,7 @@ func (r *RtH) RtFind(Dst net.IPNet, Zone string) *Rt {
 	key := RtKey{Dst.String(), Zone}
 	rt, found := r.RtMap[key]
 
-	if found == true {
+	if found {
 		return rt
 	}
 	return nil
@@ -151,9 +151,12 @@ func (r *RtH) RouteGet() ([]cmn.RouteGet, error) {
 		var tmpRt cmn.RouteGet
 		tmpRt.Dst = rk.RtCidr
 		tmpRt.Flags = GetFlagToString(r2.TFlags)
-		if len(r2.NhAttr) != 0 {
-			// TODO : Current multiple gw not showing. So I added as a static code
-			tmpRt.Gw = r2.NhAttr[0].NhAddr.String()
+		tmpRt.Gw = ""
+		for i, gw := range r2.NextHops {
+			if i != 0 {
+				tmpRt.Gw += ","
+			}
+			tmpRt.Gw += gw.Addr.String()
 		}
 		tmpRt.HardwareMark = int(r2.Mark)
 		tmpRt.Protocol = r2.Attr.Protocol
@@ -199,20 +202,15 @@ func (r *RtH) RtAdd(Dst net.IPNet, Zone string, Ra RtAttr, Na []RtNhAttr) (int,
 		}
 	}
 
-	if nhLen > 1 {
-		tk.LogIt(tk.LogError, "rt add - %s:%s ecmp not supported\n", Dst.String(), Zone)
-		return RtNhErr, errors.New("ecmp-rt error not supported")
-	}
-
 	rt, found := r.RtMap[key]
-	if found == true {
+	if found {
 		rtMod := false
 		if len(rt.NhAttr) != nhLen {
 			rtMod = true
 		} else {
 			for i := 0; i < nhLen; i++ {
 				// FIXME - Need to sort before comparing
-				if Na[i].NhAddr.Equal(rt.NhAttr[i].NhAddr) == false {
+				if !Na[i].NhAddr.Equal(rt.NhAttr[i].NhAddr) {
 					rtMod = false
 					break
 				}
@@ -275,7 +273,7 @@ func (r *RtH) RtAdd(Dst net.IPNet, Zone string, Ra RtAttr, Na []RtNhAttr) (int,
 
 		hwmac, _ := net.ParseMAC("00:00:00:00:00:00")
 
-		for i := 0; i < len(Na); i++ {
+		for i := range Na {
 			nh, _ := r.Zone.Nh.NeighFind(Na[i].NhAddr, Zone)
 			if nh == nil {
 
@@ -307,7 +305,7 @@ func (r *RtH) RtAdd(Dst net.IPNet, Zone string, Ra RtAttr, Na []RtNhAttr) (int,
 
 	// Pair this route with appropriate neighbor
 	//if rt.TFlags & RT_TYPE_HOST != RT_TYPE_HOST {
-	for i := 0; i < len(rt.NextHops); i++ {
+	for i := range rt.NextHops {
 		r.Zone.Nh.NeighPairRt(rt.NextHops[i], rt)
 	}
 	//}
@@ -334,14 +332,13 @@ func (r *RtH) RtAdd(Dst net.IPNet, Zone string, Ra RtAttr, Na []RtNhAttr) (int,
 		}
 		delete(r.RtMap, rt.Key)
 		r.Mark.PutCounter(rt.Mark)
-		fmt.Printf("rt add - %s:%s lpm add fail\n", Dst.String(), Zone)
 		tk.LogIt(tk.LogError, "rt add - %s:%s lpm add fail\n", Dst.String(), Zone)
 		return RtTrieAddErr, errors.New("RT Trie Err")
 	}
 
 	rt.DP(DpCreate)
 
-	tk.LogIt(tk.LogDebug, "rt added - %s:%s mark:%v\n", Dst.String(), Zone, rt.RtGetNhMark())
+	tk.LogIt(tk.LogDebug, "rt added - %s:%s mark:%s\n", Dst.String(), Zone, rt.RtNhMarkString())
 
 	return 0, nil
 }
@@ -534,10 +531,22 @@ func (r *RtH) RoutesTicker() {
 	r.RoutesSync()
 }
 
+// RtNhMarkString - get the rt-entry's neighbor in string format
+func (rt *Rt) RtNhMarkString() string {
+	str := ""
+	for i, nh := range rt.NextHops {
+		if i != 0 {
+			str += ","
+		}
+		str += fmt.Sprintf("%v", nh.Mark)
+	}
+	return str
+}
+
 // RtGetNhMark - get the rt-entry's neighbor identifier
-func (rt *Rt) RtGetNhMark() uint64 {
-	if len(rt.NextHops) > 0 {
-		return rt.NextHops[0].Mark
+func (rt *Rt) RtGetNhMark(n int) uint64 {
+	if len(rt.NextHops) > 0 && n < len(rt.NextHops) {
+		return rt.NextHops[n].Mark
 	}
 	return ^uint64(0)
 }
@@ -570,7 +579,10 @@ func (rt *Rt) DP(work DpWorkT) int {
 	rtWq.Dst = *rtNet
 	rtWq.RtType = rt.TFlags
 	rtWq.RtMark = int(rt.Mark)
-	rtWq.NMark = int(rt.RtGetNhMark())
+	rtWq.NMax = len(rt.NextHops)
+	for i := range rt.NextHops {
+		rtWq.NMark[i] = int(rt.RtGetNhMark(i))
+	}
 
 	mh.dp.ToDpCh <- rtWq