Merge pull request #661 from YaoZengzeng/e2e-restart

e2e test for Kmesh daemon restart

test/e2e/main_test.go

@@ -74,6 +74,9 @@ type EchoDeployments struct {
 	// The echo service which is enrolled to Kmesh without waypoint.
 	EnrolledToKmesh echo.Instances
 
+	// The echo service which is enrolled to Kmesh and with service waypoint.
+	ServiceWithWaypointAtServiceGranularity echo.Instances
+
 	// WaypointProxies by
 	WaypointProxies map[string]ambient.WaypointProxy
 }
@@ -84,6 +87,7 @@ const (
 	WaypointImageAnnotation                 = "sidecar.istio.io/proxyImage"
 	Timeout                                 = 2 * time.Minute
 	KmeshReleaseName                        = "kmesh"
+	KmeshDaemonsetName                      = "kmesh"
 	KmeshNamespace                          = "kmesh-system"
 )
 
@@ -176,6 +180,7 @@ func SetupApps(t resource.Context, i istio.Instance, apps *EchoDeployments) erro
 	}
 	apps.All = echos
 	apps.EnrolledToKmesh = match.ServiceName(echo.NamespacedName{Name: EnrolledToKmesh, Namespace: apps.Namespace}).GetMatches(echos)
+	apps.ServiceWithWaypointAtServiceGranularity = match.ServiceName(echo.NamespacedName{Name: ServiceWithWaypointAtServiceGranularity, Namespace: apps.Namespace}).GetMatches(echos)
 
 	if apps.WaypointProxies == nil {
 		apps.WaypointProxies = make(map[string]ambient.WaypointProxy)

test/e2e/restart_test.go

@@ -0,0 +1,107 @@
+//go:build integ
+// +build integ
+
+/*
+ * Copyright 2024 The Kmesh Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// NOTE: THE CODE IN THIS FILE IS MAINLY REFERENCED FROM ISTIO INTEGRATION
+// FRAMEWORK(https://github.com/istio/istio/tree/master/tests/integration)
+// AND ADAPTED FOR KMESH.
+
+package kmesh
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"istio.io/istio/pkg/test/framework"
+	"istio.io/istio/pkg/test/framework/components/echo"
+	"istio.io/istio/pkg/test/framework/components/echo/util/traffic"
+	kubetest "istio.io/istio/pkg/test/kube"
+	"istio.io/istio/pkg/test/util/retry"
+	appsv1 "k8s.io/api/apps/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+func TestKmeshRestart(t *testing.T) {
+	framework.NewTest(t).Run(func(t framework.TestContext) {
+		src := apps.EnrolledToKmesh[0]
+		dst := apps.ServiceWithWaypointAtServiceGranularity
+		options := echo.CallOptions{
+			To:    dst,
+			Count: 1,
+			// Determine whether it is managed by Kmesh by passing through Waypoint.
+			Check: httpValidator,
+			Port: echo.Port{
+				Name: "http",
+			},
+			Retry: echo.Retry{NoRetry: true},
+		}
+
+		g := traffic.NewGenerator(t, traffic.Config{
+			Source:   src,
+			Options:  options,
+			Interval: 50 * time.Millisecond,
+		}).Start()
+
+		restartKmesh(t)
+
+		g.Stop().CheckSuccessRate(t, 1)
+	})
+}
+
+func restartKmesh(t framework.TestContext) {
+	patchOpts := metav1.PatchOptions{}
+	patchData := fmt.Sprintf(`{
+			"spec": {
+				"template": {
+					"metadata": {
+						"annotations": {
+							"kubectl.kubernetes.io/restartedAt": %q
+						}
+					}
+				}
+			}
+		}`, time.Now().Format(time.RFC3339))
+	ds := t.Clusters().Default().Kube().AppsV1().DaemonSets(KmeshNamespace)
+	_, err := ds.Patch(context.Background(), KmeshDaemonsetName, types.StrategicMergePatchType, []byte(patchData), patchOpts)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := retry.UntilSuccess(func() error {
+		d, err := ds.Get(context.Background(), KmeshDaemonsetName, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+		if !daemonsetsetComplete(d) {
+			return fmt.Errorf("rollout is not yet done")
+		}
+		return nil
+	}, retry.Timeout(60*time.Second), retry.Delay(2*time.Second)); err != nil {
+		t.Fatal("failed to wait for Kmesh rollout status for: %v", err)
+	}
+	if _, err := kubetest.CheckPodsAreReady(kubetest.NewPodFetch(t.AllClusters()[0], KmeshNamespace, "app=kmesh")); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func daemonsetsetComplete(ds *appsv1.DaemonSet) bool {
+	return ds.Status.UpdatedNumberScheduled == ds.Status.DesiredNumberScheduled && ds.Status.NumberReady == ds.Status.DesiredNumberScheduled && ds.Status.ObservedGeneration >= ds.Generation
+}

test/e2e/run_test.sh

@@ -79,6 +79,11 @@ EOF
 EOF
     done
 
+    # For KinD environment we need to mount bpf for each node, ref: https://github.com/kmesh-net/kmesh/issues/662
+    for node in $(kind get nodes --name="${NAME}"); do
+        docker exec "${node}" sh -c "mount -t bpf none /sys/fs/bpf"
+    done
+
     # Document the local registry
     cat <<EOF | kubectl apply -f -
 apiVersion: v1