Skip to content

Commit 8db4de4

Browse files
xiaozhiche320xiaozhiche320
authored
chore: Add e2e tests to validate hubble-relay and hubble-ui deployment (#896)
# Description Add e2e tests to validate Hubble-relay and Hubble-UI deployment, in order not to touch the legacy test, the install-helm-chart func was re-written again. The test Hubble job is able to validate Hubble resource and basic metrics but not migrating all the advanced metrics validation since there are many heavy liftings work remain and could be addressed in the future migration. ## Related Issue #422 ## Checklist - [ ] I have read the [contributing documentation](https://retina.sh/docs/contributing). - [ ] I signed and signed-off the commits (`git commit -S -s ...`). See [this documentation](https://docs.github.com/en/authentication/managing-commit-signature-verification/about-commit-signature-verification) on signing commits. - [ ] I have correctly attributed the author(s) of the code. - [ ] I have tested the changes locally. - [ ] I have followed the project's style guidelines. - [ ] I have updated the documentation, if necessary. - [ ] I have added tests, if applicable. ## Screenshots (if applicable) or Testing Completed <img width="925" alt="image" src="https://github.com/user-attachments/assets/fa0207b7-c004-4a34-a9d4-8cef09fff327"> ## Additional Notes Add any additional notes or context about the pull request here. --- Please refer to the [CONTRIBUTING.md](../CONTRIBUTING.md) file for more information on how to contribute to this project. --------- Signed-off-by: Yilin <[email protected]>
1 parent 36ee056 commit 8db4de4

File tree

6 files changed

+424
-0
lines changed

6 files changed

+424
-0
lines changed

‎test/e2e/framework/kubernetes/install-hubble-helm.go

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
package kubernetes
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"log"
7+
"os"
8+
"strings"
9+
"time"
10+
11+
"github.com/microsoft/retina/test/e2e/common"
12+
generic "github.com/microsoft/retina/test/e2e/framework/generic"
13+
"helm.sh/helm/v3/pkg/action"
14+
"helm.sh/helm/v3/pkg/chart/loader"
15+
"helm.sh/helm/v3/pkg/cli"
16+
"k8s.io/client-go/kubernetes"
17+
"k8s.io/client-go/tools/clientcmd"
18+
)
19+
20+
const (
21+
HubbleNamespace = "kube-system"
22+
HubbleUIApp = "hubble-ui"
23+
HubbleRelayApp = "hubble-relay"
24+
)
25+
26+
type ValidateHubbleStep struct {
27+
Namespace string
28+
ReleaseName string
29+
KubeConfigFilePath string
30+
ChartPath string
31+
TagEnv string
32+
}
33+
34+
func (v *ValidateHubbleStep) Run() error {
35+
ctx, cancel := context.WithTimeout(context.Background(), defaultTimeoutSeconds*time.Second)
36+
defer cancel()
37+
38+
settings := cli.New()
39+
settings.KubeConfig = v.KubeConfigFilePath
40+
actionConfig := new(action.Configuration)
41+
42+
err := actionConfig.Init(settings.RESTClientGetter(), v.Namespace, os.Getenv("HELM_DRIVER"), log.Printf)
43+
if err != nil {
44+
return fmt.Errorf("failed to initialize helm action config: %w", err)
45+
}
46+
47+
// Creating extra namespace to deploy test pods
48+
err = CreateNamespaceFn(v.KubeConfigFilePath, common.TestPodNamespace)
49+
if err != nil {
50+
return fmt.Errorf("failed to create namespace %s: %w", v.Namespace, err)
51+
}
52+
53+
tag := os.Getenv(generic.DefaultTagEnv)
54+
if tag == "" {
55+
return fmt.Errorf("tag is not set: %w", errEmpty)
56+
}
57+
imageRegistry := os.Getenv(generic.DefaultImageRegistry)
58+
if imageRegistry == "" {
59+
return fmt.Errorf("image registry is not set: %w", errEmpty)
60+
}
61+
62+
imageNamespace := os.Getenv(generic.DefaultImageNamespace)
63+
if imageNamespace == "" {
64+
return fmt.Errorf("image namespace is not set: %w", errEmpty)
65+
}
66+
67+
// load chart from the path
68+
chart, err := loader.Load(v.ChartPath)
69+
if err != nil {
70+
return fmt.Errorf("failed to load chart from path %s: %w", v.ChartPath, err)
71+
}
72+
73+
chart.Values["imagePullSecrets"] = []map[string]interface{}{
74+
{
75+
"name": "acr-credentials",
76+
},
77+
}
78+
chart.Values["operator"].(map[string]interface{})["enabled"] = true
79+
chart.Values["operator"].(map[string]interface{})["repository"] = imageRegistry + "/" + imageNamespace + "/retina-operator"
80+
chart.Values["operator"].(map[string]interface{})["tag"] = tag
81+
chart.Values["agent"].(map[string]interface{})["enabled"] = true
82+
chart.Values["agent"].(map[string]interface{})["repository"] = imageRegistry + "/" + imageNamespace + "/retina-agent"
83+
chart.Values["agent"].(map[string]interface{})["tag"] = tag
84+
chart.Values["agent"].(map[string]interface{})["init"].(map[string]interface{})["enabled"] = true
85+
chart.Values["agent"].(map[string]interface{})["init"].(map[string]interface{})["repository"] = imageRegistry + "/" + imageNamespace + "/retina-init"
86+
chart.Values["agent"].(map[string]interface{})["init"].(map[string]interface{})["tag"] = tag
87+
chart.Values["hubble"].(map[string]interface{})["tls"].(map[string]interface{})["enabled"] = false
88+
chart.Values["hubble"].(map[string]interface{})["relay"].(map[string]interface{})["tls"].(map[string]interface{})["server"].(map[string]interface{})["enabled"] = false
89+
chart.Values["hubble"].(map[string]interface{})["tls"].(map[string]interface{})["auto"].(map[string]interface{})["enabled"] = false
90+
91+
getclient := action.NewGet(actionConfig)
92+
release, err := getclient.Run(v.ReleaseName)
93+
if err == nil && release != nil {
94+
log.Printf("found existing release by same name, removing before installing %s", release.Name)
95+
delclient := action.NewUninstall(actionConfig)
96+
delclient.Wait = true
97+
delclient.Timeout = deleteTimeout
98+
_, err = delclient.Run(v.ReleaseName)
99+
if err != nil {
100+
return fmt.Errorf("failed to delete existing release %s: %w", v.ReleaseName, err)
101+
}
102+
} else if err != nil && !strings.Contains(err.Error(), "not found") {
103+
return fmt.Errorf("failed to get release %s: %w", v.ReleaseName, err)
104+
}
105+
106+
client := action.NewInstall(actionConfig)
107+
client.Namespace = v.Namespace
108+
client.ReleaseName = v.ReleaseName
109+
client.Timeout = createTimeout
110+
client.Wait = true
111+
client.WaitForJobs = true
112+
113+
// install the chart here
114+
rel, err := client.RunWithContext(ctx, chart, chart.Values)
115+
if err != nil {
116+
return fmt.Errorf("failed to install chart: %w", err)
117+
}
118+
119+
log.Printf("installed chart from path: %s in namespace: %s\n", rel.Name, rel.Namespace)
120+
// this will confirm the values set during installation
121+
log.Printf("chart values: %v\n", rel.Config)
122+
123+
// ensure all pods are running, since helm doesn't care about windows
124+
config, err := clientcmd.BuildConfigFromFlags("", v.KubeConfigFilePath)
125+
if err != nil {
126+
return fmt.Errorf("error building kubeconfig: %w", err)
127+
}
128+
129+
clientset, err := kubernetes.NewForConfig(config)
130+
if err != nil {
131+
return fmt.Errorf("error creating Kubernetes client: %w", err)
132+
}
133+
134+
// Validate Hubble Relay Pod
135+
if err := WaitForPodReady(ctx, clientset, HubbleNamespace, "k8s-app="+HubbleRelayApp); err != nil {
136+
return fmt.Errorf("error waiting for Hubble Relay pods to be ready: %w", err)
137+
}
138+
log.Printf("Hubble Relay Pod is ready")
139+
140+
// Validate Hubble UI Pod
141+
if err := WaitForPodReady(ctx, clientset, HubbleNamespace, "k8s-app="+HubbleUIApp); err != nil {
142+
return fmt.Errorf("error waiting for Hubble UI pods to be ready: %w", err)
143+
}
144+
log.Printf("Hubble UI Pod is ready")
145+
146+
return nil
147+
}
148+
149+
func (v *ValidateHubbleStep) Prevalidate() error {
150+
return nil
151+
}
152+
153+
func (v *ValidateHubbleStep) Stop() error {
154+
return nil
155+
}

‎test/e2e/framework/kubernetes/validate-service.go

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
package kubernetes
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"time"
7+
8+
corev1 "k8s.io/api/core/v1"
9+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
10+
"k8s.io/client-go/kubernetes"
11+
"k8s.io/client-go/tools/clientcmd"
12+
)
13+
14+
type ResourceTypes string
15+
16+
const (
17+
ResourceTypePod = "pod"
18+
ResourceTypeService = "service"
19+
)
20+
21+
type ValidateResource struct {
22+
ResourceName string
23+
ResourceNamespace string
24+
ResourceType string
25+
Labels string
26+
KubeConfigFilePath string
27+
}
28+
29+
func (v *ValidateResource) Run() error {
30+
config, err := clientcmd.BuildConfigFromFlags("", v.KubeConfigFilePath)
31+
if err != nil {
32+
return fmt.Errorf("error building kubeconfig: %w", err)
33+
}
34+
35+
clientset, err := kubernetes.NewForConfig(config)
36+
if err != nil {
37+
return fmt.Errorf("error creating Kubernetes client: %w", err)
38+
}
39+
40+
ctx, cancel := context.WithTimeout(context.Background(), defaultTimeoutSeconds*time.Second)
41+
defer cancel()
42+
43+
switch v.ResourceType {
44+
case ResourceTypePod:
45+
err = WaitForPodReady(ctx, clientset, v.ResourceNamespace, v.Labels)
46+
if err != nil {
47+
return fmt.Errorf("pod not found: %w", err)
48+
}
49+
case ResourceTypeService:
50+
exists, err := serviceExists(ctx, clientset, v.ResourceNamespace, v.ResourceName, v.Labels)
51+
if err != nil || !exists {
52+
return fmt.Errorf("service not found: %w", err)
53+
}
54+
55+
default:
56+
return fmt.Errorf("resource type %s not supported", v.ResourceType)
57+
}
58+
59+
if err != nil {
60+
return fmt.Errorf("error waiting for pod to be ready: %w", err)
61+
}
62+
return nil
63+
}
64+
65+
func serviceExists(ctx context.Context, clientset *kubernetes.Clientset, namespace, serviceName, labels string) (bool, error) {
66+
var serviceList *corev1.ServiceList
67+
serviceList, err := clientset.CoreV1().Services(namespace).List(ctx, metav1.ListOptions{LabelSelector: labels})
68+
if err != nil {
69+
return false, fmt.Errorf("error listing Services: %w", err)
70+
}
71+
if len(serviceList.Items) == 0 {
72+
return false, nil
73+
}
74+
return true, nil
75+
}
76+
77+
func (v *ValidateResource) Prevalidate() error {
78+
return nil
79+
}
80+
81+
func (v *ValidateResource) Stop() error {
82+
return nil
83+
}

‎test/e2e/framework/kubernetes/validateHttp.go

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
package kubernetes
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"log"
7+
"net/http"
8+
"time"
9+
)
10+
11+
const (
12+
RequestTimeout = 30 * time.Second
13+
)
14+
15+
type ValidateHTTPResponse struct {
16+
URL string
17+
ExpectedStatus int
18+
}
19+
20+
func (v *ValidateHTTPResponse) Run() error {
21+
ctx, cancel := context.WithTimeout(context.Background(), RequestTimeout)
22+
defer cancel()
23+
24+
req, err := http.NewRequestWithContext(ctx, http.MethodGet, v.URL, nil)
25+
if err != nil {
26+
return fmt.Errorf("error creating HTTP request: %w", err)
27+
}
28+
29+
client := &http.Client{}
30+
resp, err := client.Do(req)
31+
if err != nil {
32+
return fmt.Errorf("error making HTTP request: %w", err)
33+
}
34+
defer resp.Body.Close()
35+
36+
if resp.StatusCode != v.ExpectedStatus {
37+
return fmt.Errorf("unexpected status code: got %d, want %d", resp.StatusCode, v.ExpectedStatus)
38+
}
39+
log.Printf("HTTP validation succeeded for URL: %s with status code %d\n", v.URL, resp.StatusCode)
40+
41+
return nil
42+
}
43+
44+
func (v *ValidateHTTPResponse) Prevalidate() error {
45+
return nil
46+
}
47+
48+
func (v *ValidateHTTPResponse) Stop() error {
49+
return nil
50+
}

‎test/e2e/hubble/scenario.go

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
package hubble
2+
3+
import (
4+
"net/http"
5+
6+
k8s "github.com/microsoft/retina/test/e2e/framework/kubernetes"
7+
"github.com/microsoft/retina/test/e2e/framework/types"
8+
)
9+
10+
func ValidateHubbleRelayService() *types.Scenario {
11+
name := "Validate Hubble Relay Service"
12+
steps := []*types.StepWrapper{
13+
{
14+
Step: &k8s.ValidateResource{
15+
ResourceName: "hubble-relay-service",
16+
ResourceNamespace: k8s.HubbleNamespace,
17+
ResourceType: k8s.ResourceTypeService,
18+
Labels: "k8s-app=" + k8s.HubbleRelayApp,
19+
},
20+
},
21+
}
22+
23+
return types.NewScenario(name, steps...)
24+
}
25+
26+
func ValidateHubbleUIService(kubeConfigFilePath string) *types.Scenario {
27+
name := "Validate Hubble UI Services"
28+
steps := []*types.StepWrapper{
29+
{
30+
Step: &k8s.ValidateResource{
31+
ResourceName: k8s.HubbleUIApp,
32+
ResourceNamespace: k8s.HubbleNamespace,
33+
ResourceType: k8s.ResourceTypeService,
34+
Labels: "k8s-app=" + k8s.HubbleUIApp,
35+
},
36+
},
37+
{
38+
Step: &k8s.PortForward{
39+
LabelSelector: "k8s-app=hubble-ui",
40+
LocalPort: "8080",
41+
RemotePort: "8081",
42+
OptionalLabelAffinity: "k8s-app=hubble-ui",
43+
Endpoint: "?namespace=default", // set as default namespace query since endpoint can't be nil
44+
KubeConfigFilePath: kubeConfigFilePath,
45+
},
46+
},
47+
{
48+
Step: &k8s.ValidateHTTPResponse{
49+
URL: "http://localhost:8080",
50+
ExpectedStatus: http.StatusOK,
51+
},
52+
},
53+
}
54+
55+
return types.NewScenario(name, steps...)
56+
}

0 commit comments

Comments
 (0)