Enable cleanup in E2E

.codespellrc

@@ -1,3 +1,3 @@
 [codespell]
 skip = .git,go.sum,go.mod,*.png,*.svg
-ignore-words-list = kmesh,outter,nd,Donot,donot,doesnot
+ignore-words-list = kmesh,outter,nd,Donot,donot,doesnot,Failer

bpf/kmesh/probes/metrics.h

@@ -9,6 +9,8 @@
 struct metric_key {
     struct ip_addr src_ip;
     struct ip_addr dst_ip;
+    __u32 direction;
+    __u32 dst_port;
 };
 
 struct metric_data {
@@ -34,28 +36,42 @@ struct {
     __uint(max_entries, RINGBUF_SIZE);
 } map_of_metric_notify SEC(".maps");
 
-static inline void construct_metric_key(struct bpf_sock *sk, struct metric_key *key)
+static inline void construct_metric_key(struct bpf_sock *sk, __u8 direction, struct metric_key *key)
 {
     bpf_memset(key, 0, sizeof(struct metric_key));
-    if (sk->family == AF_INET) {
-        key->src_ip.ip4 = sk->src_ip4;
-        key->dst_ip.ip4 = sk->dst_ip4;
+
+    key->direction = direction;
+    if (direction == OUTBOUND) {
+        if (sk->family == AF_INET) {
+            key->src_ip.ip4 = sk->src_ip4;
+            key->dst_ip.ip4 = sk->dst_ip4;
+        } else {
+            bpf_memcpy(key->src_ip.ip6, sk->src_ip6, IPV6_ADDR_LEN);
+            bpf_memcpy(key->dst_ip.ip6, sk->dst_ip6, IPV6_ADDR_LEN);
+        }
+        key->dst_port = bpf_ntohl(sk->dst_port);
     } else {
-        bpf_memcpy(key->src_ip.ip6, sk->src_ip6, IPV6_ADDR_LEN);
-        bpf_memcpy(key->dst_ip.ip6, sk->dst_ip6, IPV6_ADDR_LEN);
+        if (sk->family == AF_INET) {
+            key->src_ip.ip4 = sk->dst_ip4;
+            key->dst_ip.ip4 = sk->src_ip4;
+        } else {
+            bpf_memcpy(key->src_ip.ip6, sk->dst_ip6, IPV6_ADDR_LEN);
+            bpf_memcpy(key->dst_ip.ip6, sk->src_ip6, IPV6_ADDR_LEN);
+        }
+        key->dst_port = sk->src_port;
     }
     return;
 }
 
-static inline void report_metrics(struct bpf_sock *sk)
+static inline void report_metrics(struct metric_key *mk)
 {
     struct metric_key *key = bpf_ringbuf_reserve(&map_of_metric_notify, sizeof(struct metric_key), 0);
     if (!key) {
         BPF_LOG(ERR, PROBE, "report_metrics bpf_ringbuf_reserve failed\n");
         return;
     }
 
-    construct_metric_key(sk, key);
+    bpf_memcpy(key, mk, sizeof(struct metric_key));
     bpf_ringbuf_submit(key, 0);
     return;
 }
@@ -67,7 +83,7 @@ metric_on_connect(struct bpf_sock *sk, struct bpf_tcp_sock *tcp_sock, struct soc
     struct metric_data data = {0};
     struct metric_data *metric = NULL;
 
-    construct_metric_key(sk, &key);
+    construct_metric_key(sk, storage->direction, &key);
     metric = (struct metric_data *)bpf_map_lookup_elem(&map_of_metrics, &key);
     if (!metric) {
         data.conn_open++;
@@ -83,7 +99,7 @@ metric_on_connect(struct bpf_sock *sk, struct bpf_tcp_sock *tcp_sock, struct soc
     metric->conn_open++;
     metric->direction = storage->direction;
 notify:
-    report_metrics(sk);
+    report_metrics(&key);
     return;
 }
 
@@ -94,7 +110,7 @@ metric_on_close(struct bpf_sock *sk, struct bpf_tcp_sock *tcp_sock, struct sock_
     struct metric_data data = {0};
     struct metric_data *metric = NULL;
 
-    construct_metric_key(sk, &key);
+    construct_metric_key(sk, storage->direction, &key);
     metric = (struct metric_data *)bpf_map_lookup_elem(&map_of_metrics, &key);
     if (!metric) {
         // connect failed
@@ -113,7 +129,7 @@ metric_on_close(struct bpf_sock *sk, struct bpf_tcp_sock *tcp_sock, struct sock_
     metric->sent_bytes += tcp_sock->delivered;
     metric->received_bytes += tcp_sock->bytes_received;
 notify:
-    report_metrics(sk);
+    report_metrics(&key);
     return;
 }
 

docs/kmesh_compile-zh.md

@@ -19,7 +19,7 @@ Kmesh需要在拥有Kmesh内核增强特性的Linux环境中编译构建。当
   注意：kmesh-build镜像需要和源码版本相匹配
 
   ```bash
-  docker pull ghcr.io/kmesh-net/kmesh-build-x86:latest
+  docker pull ghcr.io/kmesh-net/kmesh-build:latest
   ```
 
 ### 源码编译

docs/kmesh_compile.md

@@ -19,7 +19,7 @@ The Kmesh needs to be compiled and built in the Linux environment with the Kmesh
   Note: The `kmesh-build` image needs to match the version of the source code.
 
   ```
-  docker pull ghcr.io/kmesh-net/kmesh-build-x86:latest
+  docker pull ghcr.io/kmesh-net/kmesh-build:latest
   ```
 
 ### build from source

docs/kmesh_deploy_and_develop_in_kind.md

@@ -79,7 +79,7 @@ You can follow the steps below to develop in kind:
 + Build your docker image locally:
 
     ```shell
-    docker build --build-arg arch=amd64 -f build/docker/kmesh.dockerfile -t $image_name .
+    docker build -f build/docker/kmesh.dockerfile -t $image_name .
     ```
 
     You should specify the `image_name`.

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/agiledragon/gomonkey/v2 v2.12.0
 	github.com/cespare/xxhash/v2 v2.3.0
 	github.com/cilium/ebpf v0.15.0
-	github.com/containernetworking/cni v1.2.2
+	github.com/containernetworking/cni v1.2.3
 	github.com/containernetworking/plugins v1.5.1
 	github.com/envoyproxy/go-control-plane v0.12.1-0.20240614044803-82e2a76dbddd
 	github.com/golang/protobuf v1.5.4

go.sum

@@ -101,8 +101,8 @@ github.com/cncf/xds/go v0.0.0-20240423153145-555b57ec207b/go.mod h1:W+zGtBO5Y1Ig
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/containerd/stargz-snapshotter/estargz v0.15.1 h1:eXJjw9RbkLFgioVaTG+G/ZW/0kEe2oEKCdS/ZxIyoCU=
 github.com/containerd/stargz-snapshotter/estargz v0.15.1/go.mod h1:gr2RNwukQ/S9Nv33Lt6UC7xEx58C+LHRdoqbEKjz1Kk=
-github.com/containernetworking/cni v1.2.2 h1:9IbP6KJQQxVKo4hhnm8r50YcVKrJbJu3Dqw+Rbt1vYk=
-github.com/containernetworking/cni v1.2.2/go.mod h1:DuLgF+aPd3DzcTQTtp/Nvl1Kim23oFKdm2okJzBQA5M=
+github.com/containernetworking/cni v1.2.3 h1:hhOcjNVUQTnzdRJ6alC5XF+wd9mfGIUaj8FuJbEslXM=
+github.com/containernetworking/cni v1.2.3/go.mod h1:DuLgF+aPd3DzcTQTtp/Nvl1Kim23oFKdm2okJzBQA5M=
 github.com/containernetworking/plugins v1.5.1 h1:T5ji+LPYjjgW0QM+KyrigZbLsZ8jaX+E5J/EcKOE4gQ=
 github.com/containernetworking/plugins v1.5.1/go.mod h1:MIQfgMayGuHYs0XdNudf31cLLAC+i242hNm6KuDGqCM=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=

pkg/cni/plugin/plugin.go

@@ -20,8 +20,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"strconv"
-	"strings"
 
 	"github.com/cilium/ebpf"
 	"github.com/containernetworking/cni/pkg/skel"
@@ -30,14 +28,10 @@ import (
 	"github.com/containernetworking/cni/pkg/version"
 	netns "github.com/containernetworking/plugins/pkg/ns"
 	"github.com/vishvananda/netlink"
-	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	k8stypes "k8s.io/apimachinery/pkg/types"
-	"k8s.io/client-go/kubernetes"
 
 	"kmesh.net/kmesh/pkg/constants"
 	"kmesh.net/kmesh/pkg/logger"
-	"kmesh.net/kmesh/pkg/nets"
 	"kmesh.net/kmesh/pkg/utils"
 )
 
@@ -86,100 +80,6 @@ func parseSkelArgs(args *skel.CmdArgs) (*cniConf, *k8sArgs, *cniv1.Result, error
 	return &cniConf, &k8sCommonArgs, result, nil
 }
 
-// checkKmesh checks whether we should enable kmesh for the given pod
-func checkKmesh(client kubernetes.Interface, pod *v1.Pod) (bool, error) {
-	namespace, err := client.CoreV1().Namespaces().Get(context.TODO(), pod.Namespace, metav1.GetOptions{})
-	if err != nil {
-		return false, err
-	}
-	var enableSidecar bool
-	injectLabel := namespace.Labels["istio-injection"]
-	if injectLabel == "enabled" {
-		enableSidecar = true
-	}
-	// According to istio, it support per pod config.
-	injValue := pod.Annotations["sidecar.istio.io/inject"]
-	if v, ok := pod.Labels["sidecar.istio.io/inject"]; ok {
-		injValue = v
-	}
-	if inject, err := strconv.ParseBool(injValue); err == nil {
-		enableSidecar = inject
-	}
-
-	// If sidecar inject enabled, kmesh do not take charge of it.
-	if enableSidecar {
-		return false, nil
-	}
-
-	// Exclude istio managed gateway
-	if gateway, ok := pod.Labels["gateway.istio.io/managed"]; ok {
-		if strings.EqualFold(gateway, "istio.io-mesh-controller") {
-			return false, nil
-		}
-	}
-
-	mode := namespace.Labels[constants.DataPlaneModeLabel]
-	if strings.EqualFold(mode, constants.DataPlaneModeKmesh) {
-		return true, nil
-	}
-
-	return false, nil
-}
-
-func disableKmeshControl(ns string) error {
-	if ns == "" {
-		return nil
-	}
-
-	execFunc := func(netns.NetNS) error {
-		/*
-		 * Attempt to connect to a special IP address. The
-		 * connection triggers the cgroup/connect4/6 ebpf
-		 * program and records the netns cookie information
-		 * of the current connection. The cookie can be used
-		 * to determine whether the netns is managed by Kmesh.
-		 * ControlCommandIp4/6:930(0x3a2) is "cipher key" for cgroup/connect4/6
-		 * ebpf program disable kmesh control
-		 */
-		return nets.TriggerControlCommand(constants.OperDisableControl)
-	}
-
-	if err := netns.WithNetNSPath(ns, execFunc); err != nil {
-		err = fmt.Errorf("enter ns path :%v, run execFunc failed: %v", ns, err)
-		return err
-	}
-	return nil
-}
-
-func enableKmeshControl(ns string) error {
-	execFunc := func(netns.NetNS) error {
-		/*
-		 * Attempt to connect to a special IP address. The
-		 * connection triggers the cgroup/connect4/6 ebpf
-		 * program and records the netns cookie information
-		 * of the current connection. The cookie can be used
-		 * to determine whether the netns is managed by Kmesh.
-		 * ControlCommandIp4/6:929(0x3a1) is "cipher key" for cgroup/connect4/6
-		 * ebpf program.
-		 */
-		return nets.TriggerControlCommand(constants.OperEnableControl)
-	}
-
-	if err := netns.WithNetNSPath(ns, execFunc); err != nil {
-		err = fmt.Errorf("enter ns path :%v, run execFunc failed: %v", ns, err)
-		return err
-	}
-	return nil
-}
-
-const KmeshRedirection = "kmesh.net/redirection"
-
-var annotationPatch = []byte(fmt.Sprintf(
-	`{"metadata":{"annotations":{"%s":"%s"}}}`,
-	KmeshRedirection,
-	"enabled",
-))
-
 func getPrevCniResult(conf *cniConf) (*cniv1.Result, error) {
 	var err error
 	if conf.RawPrevResult == nil {
@@ -228,17 +128,6 @@ func enableXdpAuth(ifname string) error {
 	return nil
 }
 
-func patchKmeshAnnotation(client kubernetes.Interface, pod *v1.Pod) error {
-	_, err := client.CoreV1().Pods(pod.Namespace).Patch(
-		context.Background(),
-		pod.Name,
-		k8stypes.MergePatchType,
-		annotationPatch,
-		metav1.PatchOptions{},
-	)
-	return err
-}
-
 // if cmdadd failed, then we cannot return failed, do nothing and print pre result
 func CmdAdd(args *skel.CmdArgs) error {
 	var err error
@@ -271,22 +160,23 @@ func CmdAdd(args *skel.CmdArgs) error {
 		return err
 	}
 
-	enableKmesh, err := checkKmesh(client, pod)
+	namespace, err := client.CoreV1().Namespaces().Get(context.TODO(), pod.Namespace, metav1.GetOptions{})
 	if err != nil {
-		log.Errorf("failed to check enable kmesh information: %v", err)
-		return err
+		return fmt.Errorf("failed to get namespace %s: %v", pod.Namespace, err)
 	}
 
+	enableKmesh := utils.ShouldEnroll(pod, namespace)
+
 	if !enableKmesh {
 		return types.PrintResult(preResult, cniConf.CNIVersion)
 	}
 
-	if err := enableKmeshControl(args.Netns); err != nil {
+	if err := utils.HandleKmeshManage(args.Netns, true); err != nil {
 		log.Errorf("failed to enable kmesh control, err is %v", err)
 		return err
 	}
 
-	if err := patchKmeshAnnotation(client, pod); err != nil {
+	if err := utils.PatchKmeshRedirectAnnotation(client, pod); err != nil {
 		log.Errorf("failed to annotate kmesh redirection, err is %v", err)
 	}
 
@@ -314,7 +204,7 @@ func CmdCheck(args *skel.CmdArgs) (err error) {
 
 func CmdDelete(args *skel.CmdArgs) error {
 	// clean
-	if err := disableKmeshControl(args.Netns); err != nil {
+	if err := utils.HandleKmeshManage(args.Netns, false); err != nil {
 		log.Errorf("failed to disable Kmesh control, err: %v", err)
 	}