fix: classic network metadata server revisit

Author: Qiu Jian

.circleci/config.yml

@@ -5,7 +5,7 @@ version: 2
 jobs:
   build:
     docker:
-      - image: yunion/centos-build:go-1.18.3-0
+      - image: yunion/centos-build:go-1.21.10-0
 
     working_directory: /root/go/src/yunion.io/x/sdnagent
     steps:

.github/workflows/ci-pr-build-test.yml

@@ -13,7 +13,7 @@ jobs:
     - name: Setup Go
       uses: actions/setup-go@v4
       with:
-        go-version: '1.18'
+        go-version: '1.21'
 
     - name: Run go test
       shell: bash

go.mod

@@ -1,26 +1,26 @@
 module yunion.io/x/sdnagent
 
-go 1.18
+go 1.21
 
 require (
 	github.com/coreos/go-iptables v0.6.0
-	github.com/digitalocean/go-openvswitch v0.0.0-20240130171624-c0f7d42efe24
+	github.com/digitalocean/go-openvswitch v0.0.20240920
 	github.com/fsnotify/fsnotify v1.5.4
-	github.com/golang/protobuf v1.5.2
+	github.com/golang/protobuf v1.5.3
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/cobra v0.0.5
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.3.2
 	github.com/vishvananda/netlink v1.1.0
 	github.com/vishvananda/netns v0.0.5-0.20240412164733-9469873f4601
-	golang.org/x/net v0.9.0
-	google.golang.org/grpc v1.38.0
-	google.golang.org/protobuf v1.27.1
+	golang.org/x/net v0.21.0
+	google.golang.org/grpc v1.62.0
+	google.golang.org/protobuf v1.32.0
 	yunion.io/x/jsonutils v1.0.1-0.20240203102553-4096f103b401
 	yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91
-	yunion.io/x/onecloud v0.0.0-20240824114415-077e7975f242
-	yunion.io/x/pkg v1.10.1-0.20240812013427-0163ba9c86b1
+	yunion.io/x/onecloud v0.0.0-20240920070610-6e42b181d337
+	yunion.io/x/pkg v1.10.1-0.20240905110705-77c46e716318
 )
 
 require (
@@ -40,7 +40,7 @@ require (
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
 	github.com/gin-gonic/gin v1.7.7 // indirect
-	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-playground/locales v0.13.0 // indirect
 	github.com/go-playground/universal-translator v0.17.0 // indirect
@@ -94,15 +94,16 @@ require (
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
 	go.uber.org/zap v1.17.0 // indirect
-	golang.org/x/crypto v0.8.0 // indirect
-	golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c // indirect
-	golang.org/x/sync v0.1.0 // indirect
-	golang.org/x/sys v0.9.0 // indirect
-	golang.org/x/term v0.7.0 // indirect
-	golang.org/x/text v0.9.0 // indirect
-	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
-	google.golang.org/appengine v1.6.6 // indirect
-	google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect
+	golang.org/x/crypto v0.19.0 // indirect
+	golang.org/x/oauth2 v0.17.0 // indirect
+	golang.org/x/sync v0.6.0 // indirect
+	golang.org/x/sys v0.17.0 // indirect
+	golang.org/x/term v0.17.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
+	google.golang.org/appengine v1.6.8 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240304161311-37d4d3c04a78 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240228224816-df926f6c8641 // indirect
 	gopkg.in/fatih/set.v0 v0.2.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.62.0 // indirect
@@ -113,20 +114,19 @@ require (
 	k8s.io/client-go v0.19.3 // indirect
 	k8s.io/cluster-bootstrap v0.19.3 // indirect
 	k8s.io/cri-api v0.22.17 // indirect
-	k8s.io/klog/v2 v2.2.0 // indirect
+	k8s.io/klog/v2 v2.20.0 // indirect
 	k8s.io/utils v0.0.0-20201110183641-67b214c5f920 // indirect
 	moul.io/http2curl/v2 v2.3.0 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.0.1 // indirect
 	sigs.k8s.io/yaml v1.2.0 // indirect
-	yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20240823015832-84392f4b7c49 // indirect
+	yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20240918093754-91627585360f // indirect
 	yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32 // indirect
 	yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e // indirect
-	yunion.io/x/sqlchemy v1.1.3-0.20240823033059-be6fe90dab22 // indirect
+	yunion.io/x/sqlchemy v1.1.3-0.20240916001724-cb6bf30b6f63 // indirect
 	yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c // indirect
 )
 
 replace (
-	github.com/digitalocean/go-openvswitch => yunion.io/x/go-openvswitch v0.0.20240615
-	github.com/go-logr/logr => github.com/go-logr/logr v0.4.0
+	github.com/digitalocean/go-openvswitch => yunion.io/x/go-openvswitch v0.0.20240920
 	github.com/jaypipes/ghw => github.com/zexi/ghw v0.9.1
 )

go.sum

@@ -27,6 +27,7 @@ import (
 
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
+	pkgutils "yunion.io/x/pkg/utils"
 
 	"yunion.io/x/onecloud/pkg/util/fileutils2"
 
@@ -60,7 +61,7 @@ type FlowMan struct {
 	waitCount int32
 }
 
-func (fm *FlowMan) doDumpFlows() (*utils.FlowSet, error) {
+func (fm *FlowMan) doDumpFlows(excludeOvsTables []int) (*utils.FlowSet, error) {
 	// check existence of ovs-db's sock file
 	const ovsDbSock = "/var/run/openvswitch/db.sock"
 	if !fileutils2.Exists(ovsDbSock) {
@@ -75,18 +76,37 @@ func (fm *FlowMan) doDumpFlows() (*utils.FlowSet, error) {
 	// for _, of := range flows {
 	// 	utils.OVSFlowOrderMatch(of)
 	// }
+	{
+		// filter out dynamic flow tables
+		nflows := make([]*ovs.Flow, 0, len(flows))
+		for i := range flows {
+			if len(excludeOvsTables) > 0 && pkgutils.IsInArray(flows[i].Table, excludeOvsTables) {
+				continue
+			}
+			nflows = append(nflows, flows[i])
+		}
+		flows = nflows
+	}
+
 	fs := utils.NewFlowSetFromList(flows)
 	return fs, nil
 }
 
+var (
+	excludeOvsTables = []int{
+		10,
+		12,
+	}
+)
+
 func (fm *FlowMan) doCheck() {
 	if atomic.LoadInt32(&fm.waitCount) != 0 {
 		return
 	}
 	defer log.Infof("flowman %s: check done", fm.bridge)
 	var err error
 	// fs0: current flows
-	fs0, err := fm.doDumpFlows()
+	fs0, err := fm.doDumpFlows(excludeOvsTables)
 	if err != nil {
 		log.Errorf("FlowMan doCheck doDumpFlows fail %s", err)
 		return
@@ -136,7 +156,7 @@ func (fm *FlowMan) bufWriteFlows(buf *bytes.Buffer, prefix string, flows []*ovs.
 	}
 }
 
-func (fm *FlowMan) doCommitChange(flowsAdd, flowsDel []*ovs.Flow) {
+func (fm *FlowMan) doCommitChange(flowsAdd, flowsDel []*ovs.Flow) error {
 	ofCli := ovs.New(ovs.Strict(), ovs.Debug(false)).OpenFlow
 	err := ofCli.AddFlowBundle(fm.bridge, func(tx *ovs.FlowTransaction) error {
 		mfs := make([]*ovs.MatchFlow, len(flowsDel))
@@ -150,8 +170,9 @@ func (fm *FlowMan) doCommitChange(flowsAdd, flowsDel []*ovs.Flow) {
 	})
 	if err != nil {
 		log.Errorf("flowman %s: add flow bundle failed: %s", fm.bridge, err)
-		return
+		return errors.Wrapf(err, "AddFlowBundle %s", fm.bridge)
 	}
+	return nil
 }
 
 func (fm *FlowMan) doCmd(cmd *flowManCmd) {

pkg/agent/server/flowman.go

@@ -18,16 +18,20 @@ import (
 	"context"
 
 	"yunion.io/x/log"
+
 	"yunion.io/x/sdnagent/pkg/agent/utils"
 )
 
 type HostLocal struct {
 	watcher *serversWatcher
+
+	bridgeMap map[string]*utils.HostLocal
 }
 
 func NewHostLocal(watcher *serversWatcher) *HostLocal {
 	return &HostLocal{
-		watcher: watcher,
+		watcher:   watcher,
+		bridgeMap: make(map[string]*utils.HostLocal),
 	}
 }
 
@@ -44,16 +48,22 @@ func (hl *HostLocal) updateFlows(ctx context.Context) {
 			Ifname:     hcn.Ifname,
 			IP:         ip,
 			MAC:        mac,
-
-			HostLocalNets: hcn.HostLocalNets,
 		}
+		hostLocal = utils.FetchHostLocal(hostLocal, hl.watcher)
+
 		flows, err := hostLocal.FlowsMap()
 		if err != nil {
 			log.Errorf("prepare %s hostlocal flows failed: %s", hcn.Bridge, err)
 			continue
 		}
 		flowman := hl.watcher.agent.GetFlowMan(hcn.Bridge)
 		flowman.updateFlows(ctx, hostLocal.Who(), flows[hcn.Bridge])
+
+		err = hostLocal.EnsureFakeLocalMetadataRoute()
+		if err != nil {
+			log.Errorf("EnsureFakeLocalMetadataRoute %s hostlocal flows failed: %s", hcn.Bridge, err)
+			continue
+		}
 	}
 }
 

pkg/agent/server/hostlocal.go

@@ -219,8 +219,8 @@ func (man *ovnMan) ensureGeneveFastpath(ctx context.Context) {
 
 func (man *ovnMan) ensureBasicFlows(ctx context.Context) {
 	var (
-		p       = apis.VpcMappedCidr()
-		hexMac  = "0x" + strings.TrimLeft(strings.ReplaceAll(man.mac, ":", ""), "0")
+		p = apis.VpcMappedCidr()
+		/*hexMac  = "0x" + strings.TrimLeft(strings.ReplaceAll(man.mac, ":", ""), "0")
 		actions = []string{
 			"move:NXM_OF_ETH_SRC[]->NXM_OF_ETH_DST[]",
 			fmt.Sprintf("load:%s->NXM_OF_ETH_SRC[]", hexMac),
@@ -230,12 +230,12 @@ func (man *ovnMan) ensureBasicFlows(ctx context.Context) {
 			"move:NXM_NX_ARP_SHA[]->NXM_NX_ARP_THA[]",
 			"move:NXM_OF_ARP_SPA[]->NXM_OF_ARP_TPA[]",
 			"in_port",
-		}
+		}*/
 	)
 	flows := []*ovs.Flow{
 		utils.F(0, 3050,
 			fmt.Sprintf("in_port=LOCAL,arp,arp_op=1,arp_tpa=%s", p.String()),
-			strings.Join(actions, ",")),
+			utils.FakeArpRespActions(man.mac) /*strings.Join(actions, ",")*/),
 		utils.F(0, 32000,
 			fmt.Sprintf("ip,nw_dst=%s", p.String()),
 			"drop"),

pkg/agent/server/ovn.go

@@ -45,6 +45,7 @@ type wCmd int
 
 const (
 	wCmdFindGuestDescByIdIP wCmd = iota
+	wCmdFindGuestDescByHostLocalIP
 )
 
 type wCmdFindGuestDescByIdIPData struct {
@@ -53,6 +54,12 @@ type wCmdFindGuestDescByIdIPData struct {
 	RespCh chan<- *desc.SGuestDesc
 }
 
+type wCmdFindGuestDescByHostLocalIPData struct {
+	HostLocal *utils.HostLocal
+	IP        string
+	RespCh    chan<- *desc.SGuestDesc
+}
+
 type wCmdReq struct {
 	cmd  wCmd
 	data interface{}
@@ -316,6 +323,25 @@ func (w *serversWatcher) Start(ctx context.Context, agent *AgentServer) {
 							log.Errorf("guest %s: GetJSONObjectDesc: %v", guestId, err)
 						}
 						robj = obj
+						break
+					}
+				}
+				data.RespCh <- robj
+			case wCmdFindGuestDescByHostLocalIP:
+				var (
+					data      = cmd.data.(wCmdFindGuestDescByHostLocalIPData)
+					hostLocal = data.HostLocal
+					ip        = data.IP
+					robj      *desc.SGuestDesc
+				)
+				for guestId, guest := range w.guests {
+					if nic := guest.FindNicByHostLocalIP(hostLocal, ip); nic != nil {
+						obj, err := guest.GetJSONObjectDesc()
+						if err != nil {
+							log.Errorf("guest %s: GetJSONObjectDesc: %v", guestId, err)
+						}
+						robj = obj
+						break
 					}
 				}
 				data.RespCh <- robj
@@ -344,6 +370,22 @@ func (w *serversWatcher) FindGuestDescByNetIdIP(netId, ip string) *desc.SGuestDe
 	return obj
 }
 
+func (w *serversWatcher) FindGuestDescByHostLocalIp(hostLocal *utils.HostLocal, ip string) *desc.SGuestDesc {
+	respCh := make(chan *desc.SGuestDesc)
+	reqData := wCmdFindGuestDescByHostLocalIPData{
+		HostLocal: hostLocal,
+		IP:        ip,
+		RespCh:    respCh,
+	}
+	req := wCmdReq{
+		cmd:  wCmdFindGuestDescByHostLocalIP,
+		data: reqData,
+	}
+	w.cmdCh <- req
+	obj := <-respCh
+	return obj
+}
+
 func (w *serversWatcher) watchEvent(ev *fsnotify.Event) (wev *watchEvent) {
 	dir, file := filepath.Split(ev.Name)
 	dir = path.Clean(dir)
@@ -385,3 +427,12 @@ func (w *serversWatcher) watchEvent(ev *fsnotify.Event) (wev *watchEvent) {
 	}
 	return nil
 }
+
+func (w *serversWatcher) GetHostLocalByIp(ip string) *utils.HostLocal {
+	for _, hl := range w.hostLocal.bridgeMap {
+		if hl.IP.String() == ip {
+			return hl
+		}
+	}
+	return nil
+}