From 73ee309ee83e219f075662bf60dbce41b1c327f7 Mon Sep 17 00:00:00 2001 From: Kashif Khan Date: Thu, 2 Jan 2025 13:19:19 +0200 Subject: [PATCH] Fix metrics service validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since the cmd block is outside the eventually block, when the same cmd object is used multiple times, we see ´exec: Stdout already set´ error. Moving it inside the eventually block should solve it. Also BMO optional test is not deploying BMO via e2e_suite_test.go but that is done separately in upgrade test. Hence the check is failing since we are not deploying BMO in e2e suite test but we are checking for metrics service to be up. This PR makes sure that metrics service is only checked when BMO is deployed. Signed-off-by: Kashif Khan --- test/e2e/e2e_suite_test.go | 116 +++++++++++++++++++------------------ 1 file changed, 61 insertions(+), 55 deletions(-) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index aa7bf3a29e..6d5145e32c 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -226,63 +226,69 @@ var _ = SynchronizedBeforeSuite(func() []byte { }) }) Expect(err).NotTo(HaveOccurred()) - } - - // Metrics test start - By("creating a ClusterRoleBinding for the service account to allow access to metrics") - cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, - "--clusterrole=baremetal-operator-metrics-reader", - fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), - ) - _, err := cmd.CombinedOutput() - Expect(err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") - - By("validating that the metrics service is available") - cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) - _, err = cmd.CombinedOutput() - Expect(err).NotTo(HaveOccurred(), "Metrics service should exist") - - By("getting the service account token") - token, err := serviceAccountToken() - Expect(err).NotTo(HaveOccurred()) - Expect(token).NotTo(BeEmpty()) - By("waiting for the metrics endpoint to be ready") - verifyMetricsEndpointReady := func(g Gomega) { - cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) - output, err := cmd.CombinedOutput() - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(output).To(ContainSubstring("8443"), "Metrics endpoint is not ready") - } - Eventually(verifyMetricsEndpointReady).Should(Succeed()) - - By("creating the curl-metrics pod to access the metrics endpoint") - cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", - "--namespace", namespace, - "--image=curlimages/curl:7.87.0", - "--command", - "--", "curl", "-v", "--tlsv1.3", "-k", "-H", fmt.Sprintf("Authorization:Bearer %s", token), - fmt.Sprintf("https://%s.%s.svc.cluster.local:8443/metrics", metricsServiceName, namespace)) - _, err = cmd.CombinedOutput() - Expect(err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") - - By("waiting for the curl-metrics pod to complete.") - verifyCurlUp := func(g Gomega) { - cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", - "-o", "jsonpath={.status.phase}", - "-n", namespace) - output, err := cmd.CombinedOutput() - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(string(output)).To(Equal("Succeeded"), "curl pod in wrong status") + // Metrics test start + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, + "--clusterrole=baremetal-operator-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), + ) + _, err = cmd.CombinedOutput() + Expect(err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + Eventually(func() error { + cmd := exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + output, err := cmd.CombinedOutput() + if err != nil { + fmt.Printf("Service check output: %s\n", string(output)) + return err + } + return nil + }, "30s", "5s").Should(Succeed(), "Metrics service is not available") + + By("getting the service account token") + token, err := serviceAccountToken() + Expect(err).NotTo(HaveOccurred()) + Expect(token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + output, err := cmd.CombinedOutput() + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring("8443"), "Metrics endpoint is not ready") + } + Eventually(verifyMetricsEndpointReady).Should(Succeed()) + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:7.87.0", + "--command", + "--", "curl", "-v", "--tlsv1.3", "-k", "-H", fmt.Sprintf("Authorization:Bearer %s", token), + fmt.Sprintf("https://%s.%s.svc.cluster.local:8443/metrics", metricsServiceName, namespace)) + _, err = cmd.CombinedOutput() + Expect(err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + output, err := cmd.CombinedOutput() + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(string(output)).To(Equal("Succeeded"), "curl pod in wrong status") + } + Eventually(verifyCurlUp, 5*time.Minute).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + Expect(metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) + // Metrics test end } - Eventually(verifyCurlUp, 5*time.Minute).Should(Succeed()) - - By("getting the metrics by checking curl-metrics logs") - metricsOutput := getMetricsOutput() - Expect(metricsOutput).To(ContainSubstring( - "controller_runtime_reconcile_total", - )) - // Metrics test end return []byte(strings.Join([]string{clusterProxy.GetKubeconfigPath()}, ",")) }, func(data []byte) {