diff --git a/test/extended/router/gatewayapicontroller.go b/test/extended/router/gatewayapicontroller.go index a0543de8195f..abca26d74f9f 100644 --- a/test/extended/router/gatewayapicontroller.go +++ b/test/extended/router/gatewayapicontroller.go @@ -3,9 +3,12 @@ package router import ( "context" "crypto/tls" + "errors" "fmt" "net" "net/http" + "sort" + "strconv" "strings" "time" @@ -41,7 +44,8 @@ const ( // namespace. ingressNamespace = "openshift-ingress" // istioName is the name of the Istio CR. - istioName = "openshift-gateway" + istioName = "openshift-gateway" + istiodDeployment = "istiod-openshift-gateway" // The name of the default gatewayclass, which is used to install OSSM. gatewayClassName = "openshift-default" @@ -96,7 +100,8 @@ var _ = g.Describe("[sig-network-edge][OCPFeatureGate:GatewayAPIController][Feat // gatewayClassControllerName is the name that must be used to create a supported gatewayClass. gatewayClassControllerName = "openshift.io/gateway-controller/v1" //OSSM Deployment Pod Name - deploymentOSSMName = "servicemesh-operator3" + deploymentOSSMName = "servicemesh-operator3" + openshiftOperatorsNamespace = "openshift-operators" ) g.BeforeEach(func() { isokd, err := isOKD(oc) @@ -392,6 +397,84 @@ var _ = g.Describe("[sig-network-edge][OCPFeatureGate:GatewayAPIController][Feat }) o.Expect(waitIstioErr).NotTo(o.HaveOccurred(), "Timed out waiting for Istio to remove GIE env variable") }) + + g.It("Ensure OSSM subscription, istiod deployment and the istio could be deleted and then get recreated [Serial]", func() { + // delete the OSSM subscription and then check if it is restored + g.By(fmt.Sprintf("Try to delete the subscription %s", expectedSubscriptionName)) + subscriptionOriginalCreatedTimestamp, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("-n", openshiftOperatorsNamespace, "subscription", expectedSubscriptionName, `-o=jsonpath={.metadata.creationTimestamp}`).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + _, err = oc.AsAdmin().WithoutNamespace().Run("delete").Args("-n", openshiftOperatorsNamespace, "subscription/"+expectedSubscriptionName).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By(fmt.Sprintf("Wait until the OSSM subscription %s is automatically created successfully", expectedSubscriptionName)) + pollWaitSubscriptionCreated(oc, openshiftOperatorsNamespace, expectedSubscriptionName, subscriptionOriginalCreatedTimestamp) + + // delete the istiod deployment and then checked if it is restored + g.By(fmt.Sprintf("Try to delete the istiod deployment in %s namespace", ingressNamespace)) + pollWaitDeploymentReady(oc, ingressNamespace, istiodDeployment) + deployment, err := oc.AdminKubeClient().AppsV1().Deployments(ingressNamespace).Get(context.Background(), istiodDeployment, metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + err = oc.AdminKubeClient().AppsV1().Deployments(ingressNamespace).Delete(context.Background(), istiodDeployment, metav1.DeleteOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By(fmt.Sprintf("Wait until the istiod deployment in %s namespace is automatically created successfully", ingressNamespace)) + pollWaitDeploymentCreated(oc, ingressNamespace, istiodDeployment, deployment.CreationTimestamp) + + // delete the istio and check if it is restored + g.By(fmt.Sprintf("Try to delete the istio %s", istioName)) + istioOriginalCreatedTimestamp, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("-n", ingressNamespace, "istio/"+istioName, `-o=jsonpath={.metadata.creationTimestamp}`).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + _, err = oc.AsAdmin().WithoutNamespace().Run("delete").Args("-n", ingressNamespace, "istio/"+istioName).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By(fmt.Sprintf("Wait until the the istiod %s is automatically created successfully", istioName)) + pollWaitIstioCreated(oc, ingressNamespace, istioName, istioOriginalCreatedTimestamp) + }) + + g.It("Ensure gateway loadbalancer service and dnsrecords could be deleted and then get recreated [Serial]", func() { + g.By("Getting the default domain for creating a custom Gateway") + defaultIngressDomain, err := getDefaultIngressClusterDomainName(oc, time.Minute) + o.Expect(err).NotTo(o.HaveOccurred(), "Failed to find default domain name") + customDomain := strings.Replace(defaultIngressDomain, "apps.", "gw-custom.", 1) + + g.By("Create a custom Gateway") + gw := names.SimpleNameGenerator.GenerateName("gateway-") + gateways = append(gateways, gw) + _, gwerr := createAndCheckGateway(oc, gw, gatewayClassName, customDomain) + o.Expect(gwerr).NotTo(o.HaveOccurred(), "Failed to create Gateway") + + // verify the gateway's LoadBalancer service + assertGatewayLoadbalancerReady(oc, gw, gw+"-openshift-default") + gatewayLbService := gw + "-openshift-default" + + // make sure the DNSRecord is ready to use. + assertDNSRecordStatus(oc, gw) + + g.By(fmt.Sprintf("Try to delete the gateway lb service %s", gatewayLbService)) + lbService, err := oc.AdminKubeClient().CoreV1().Services(ingressNamespace).Get(context.Background(), gatewayLbService, metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + err = oc.AdminKubeClient().CoreV1().Services(ingressNamespace).Delete(context.Background(), gatewayLbService, metav1.DeleteOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By(fmt.Sprintf("Wait until the gateway lb service %s is automatically recreated successfully", gatewayLbService)) + pollWaitGWLBServiceRecreated(oc, ingressNamespace, gatewayLbService, lbService.ObjectMeta.CreationTimestamp) + + // make sure the DNSRecord is ready to use. + assertDNSRecordStatus(oc, gw) + + // delete the gateway dnsrecords then checked if it is restored + g.By(fmt.Sprintf("Get some info of the gateway dnsrecords in %s namespace, then try to delete it", ingressNamespace)) + dnsrecordList, err := oc.AdminIngressClient().IngressV1().DNSRecords(ingressNamespace).List(context.Background(), metav1.ListOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + dnsrecord, err := getGWDNSRecords(dnsrecordList, gw) + o.Expect(err).NotTo(o.HaveOccurred()) + err = oc.AdminIngressClient().IngressV1().DNSRecords(ingressNamespace).Delete(context.Background(), dnsrecord.ObjectMeta.Name, metav1.DeleteOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By(fmt.Sprintf("Wait unitl the gateway dnsrecords in %s namespace is automatically created successfully", ingressNamespace)) + pollWaitGWDNSRecordsRecreated(oc, gw, ingressNamespace, getSortedString(dnsrecord.Spec.Targets), dnsrecord.ObjectMeta.CreationTimestamp) + }) }) func skipGatewayIfNonCloudPlatform(oc *exutil.CLI) { @@ -889,3 +972,213 @@ func extractObjectReference(v map[string]any) corev1.ObjectReference { FieldPath: getNestedString(v, "fieldPath"), } } + +// used to wait for a subscription is created successfully by checking its CatalogSourcesUnhealthy +func pollWaitSubscriptionCreated(oc *exutil.CLI, openshiftOperatorsNamespace, expectedSubscriptionName, originalCreatedTimestamp string) { + err := wait.Poll(3*time.Second, 300*time.Second, func() (bool, error) { + unhealthy, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("-n", openshiftOperatorsNamespace, "subscription", expectedSubscriptionName, `-o=jsonpath={.status.conditions[?(@.type=="CatalogSourcesUnhealthy")].status}`).Output() + if err != nil { + e2e.Logf("Failed to get %q subscription, error: %v, retrying...", expectedSubscriptionName, err) + return false, nil + } + + if unhealthy != "False" { + e2e.Logf("Wait CatalogSourcesUnhealthy status to be False, and got %q", unhealthy) + return false, nil + } + + currentCreatedTimestamp, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("-n", openshiftOperatorsNamespace, "subscription", expectedSubscriptionName, `-o=jsonpath={.metadata.creationTimestamp}`).Output() + if err != nil { + e2e.Logf("Failed to get %q subscription, error: %v, retrying...", expectedSubscriptionName, err) + return false, nil + } + + if currentCreatedTimestamp == originalCreatedTimestamp { + e2e.Logf("Original subscription %q in namespace %q is not deleted yet, retrying...", expectedSubscriptionName, openshiftOperatorsNamespace) + return false, nil + } + + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred()) +} + +// used to wait for a deployment is ready +func pollWaitDeploymentReady(oc *exutil.CLI, ns, deploymentName string) { + err := wait.Poll(3*time.Second, 300*time.Second, func() (bool, error) { + deployment, err := oc.AdminKubeClient().AppsV1().Deployments(ns).Get(context.Background(), deploymentName, metav1.GetOptions{}) + if err != nil { + e2e.Logf("Failed to get %q deployment: %v, retrying...", deploymentName, err) + return false, nil + } + + if readyReplicas := deployment.Status.ReadyReplicas; readyReplicas < 1 { + e2e.Logf(`The deployment %s in %s namespace is not ready(ReadyReplicas: %v), retrying...`, deploymentName, ns, readyReplicas) + return false, nil + } + + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred()) +} + +// used to wait for a deployment is automatically recreated +func pollWaitDeploymentCreated(oc *exutil.CLI, ns, deploymentName string, originalCreatedTime metav1.Time) { + err := wait.Poll(3*time.Second, 300*time.Second, func() (bool, error) { + deployment, err := oc.AdminKubeClient().AppsV1().Deployments(ns).Get(context.Background(), deploymentName, metav1.GetOptions{}) + if err != nil { + e2e.Logf("Failed to get %q deployment: %v, retrying...", deploymentName, err) + return false, nil + } + + if deployment.CreationTimestamp == originalCreatedTime { + e2e.Logf("Orignal deployment %q in namespace %q is not deleted yet, retrying...", deploymentName, ns) + return false, nil + } + + if readyReplicas := deployment.Status.ReadyReplicas; readyReplicas < 1 { + e2e.Logf(`The deployment %s in %s namespace is not ready(ReadyReplicas: %v), retrying...`, deploymentName, ns, readyReplicas) + return false, nil + } + + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred()) +} + +// used to wait for the istio is created successfully by checking its readyReplicas +func pollWaitIstioCreated(oc *exutil.CLI, ingressNamespace, istioName, originalCreatedTimestamp string) { + err := wait.Poll(3*time.Second, 300*time.Second, func() (bool, error) { + readyReplicasStr, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("-n", ingressNamespace, "istio/"+istioName, `-o=jsonpath={.status.revisions.ready}`).Output() + if err != nil { + e2e.Logf("Failed to check istio %q, error: %v, retrying...", istioName, err) + return false, nil + } + + readyReplicas, err := strconv.Atoi(readyReplicasStr) + if err != nil { + e2e.Logf("Failed to convert readyReplicasStr %q to int, error: %v, retrying...", readyReplicasStr, err) + return false, nil + } + + if readyReplicas < 1 { + e2e.Logf("No ready replicas found for istio %q", istioName) + return false, nil + } + + currentCreatedTimestamp, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("-n", ingressNamespace, "istio/"+istioName, `-o=jsonpath={.metadata.creationTimestamp}`).Output() + if err != nil { + e2e.Logf("Failed to check istio %q, error: %v, retrying...", istioName, err) + return false, nil + } + + if currentCreatedTimestamp == originalCreatedTimestamp { + e2e.Logf("Original istio %q in namespace %q is not deleted yet, retrying...", istioName, ingressNamespace) + return false, nil + } + + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred()) +} + +// used to wait for the gateway lb service is automatically recreated successfully +func pollWaitGWLBServiceRecreated(oc *exutil.CLI, ingressNamespace, gatewayLbService string, originalCreatedTime metav1.Time) { + var lbAddress string + err := wait.Poll(3*time.Second, 300*time.Second, func() (bool, error) { + lbService, err := oc.AdminKubeClient().CoreV1().Services(ingressNamespace).Get(context.Background(), gatewayLbService, metav1.GetOptions{}) + if err != nil { + e2e.Logf("Failed to get the gateway lb service %q: %v, retrying...", gatewayLbService, err) + return false, nil + } + + if lbService.ObjectMeta.CreationTimestamp == originalCreatedTime { + e2e.Logf("Original gateway lb service %q is not deleted yet, retrying...", gatewayLbService) + return false, nil + } + + if len(lbService.Status.LoadBalancer.Ingress) == 0 { + e2e.Logf("New gateway lb service %q is created, but without lb hostname or ip, retrying...", gatewayLbService) + return false, nil + } + + if lbService.Status.LoadBalancer.Ingress[0].Hostname != "" { + lbAddress = lbService.Status.LoadBalancer.Ingress[0].Hostname + } else { + lbAddress = lbService.Status.LoadBalancer.Ingress[0].IP + } + if lbAddress == "" { + e2e.Logf("No load balancer address for service %q, retrying...", gatewayLbService) + return false, nil + } + + e2e.Logf("Got load balancer address for service %q: %v", gatewayLbService, lbAddress) + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred()) +} + +// used to get a gateway dnsrecord from a given dnsrecordList +func getGWDNSRecords(dnsrecordList *operatoringressv1.DNSRecordList, gwName string) (operatoringressv1.DNSRecord, error) { + for _, dnsrecord := range dnsrecordList.Items { + if strings.Contains(dnsrecord.ObjectMeta.Name, gwName+"-") { + return dnsrecord, nil + } + } + + return operatoringressv1.DNSRecord{}, errors.New("Could not get the name of the gw dnsrecord") +} + +// used to wait for the gateway dnsrecord is automatically recreated successfully +func pollWaitGWDNSRecordsRecreated(oc *exutil.CLI, gwName, ingressNamespace, expectedtargets string, originalCreatedTime metav1.Time) { + err := wait.Poll(3*time.Second, 300*time.Second, func() (bool, error) { + dnsrecordList, err := oc.AdminIngressClient().IngressV1().DNSRecords(ingressNamespace).List(context.Background(), metav1.ListOptions{}) + if err != nil { + e2e.Logf("Failed to List DNSRecords in namespace %q: %v, retrying...", ingressNamespace, err) + return false, nil + } + + dnsrecord, err := getGWDNSRecords(dnsrecordList, gwName) + if err != nil { + e2e.Logf("Failed to get the DNSRecord name for gateway %q in namespace %q: %v, retrying...", gwName, ingressNamespace, err) + return false, nil + } + + if dnsrecord.ObjectMeta.CreationTimestamp == originalCreatedTime { + e2e.Logf("Original DNSRecord of GW %q is not deleted yet, retrying...", gwName) + return false, nil + } + + currentTargets := getSortedString(dnsrecord.Spec.Targets) + if currentTargets != expectedtargets { + e2e.Logf("Current DNSRecord targets %q for gateway %q differ from expected %q, retrying...", currentTargets, gwName, expectedtargets) + return false, nil + } + + for _, zone := range dnsrecord.Status.Zones { + for _, condition := range zone.Conditions { + if condition.Type == "Published" && condition.Status != "True" { + e2e.Logf(`DNSRecord %q is not published in zone %q, retrying...`, dnsrecord.Name, zone) + return false, nil + } + } + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred()) +} + +// used to sort string type of slice or string which can be split to the slice by the space character +func getSortedString(obj interface{}) string { + objList := []string{} + str, ok := obj.(string) + if ok { + objList = strings.Split(str, " ") + } + strList, ok := obj.([]string) + if ok { + objList = strList + } + sort.Strings(objList) + return strings.Join(objList, " ") +}