Merge pull request #405 from dcurran90/workloadResources

Adding support for daemonSet and statefulSet resources in charts
redhat-certification · Sep 1, 2023 · ab27c64 · ab27c64
2 parents 2b786ba + 562919e
commit ab27c64
Show file tree

Hide file tree

Showing 3 changed files with 307 additions and 46 deletions.
diff --git a/internal/chartverifier/checks/charttesting.go b/internal/chartverifier/checks/charttesting.go
@@ -235,7 +235,7 @@ func testRelease(
 	release, namespace, releaseSelector string,
 	cleanupHelmTests bool,
 ) error {
-	if err := kubectl.WaitForDeployments(ctx, namespace, releaseSelector); err != nil {
+	if err := kubectl.WaitForWorkloadResources(ctx, namespace, releaseSelector); err != nil {
 		return err
 	}
 	if err := helm.Test(ctx, namespace, release); err != nil {

diff --git a/internal/tool/kubectl.go b/internal/tool/kubectl.go
@@ -29,6 +29,8 @@ var content embed.FS
 var (
 	kubeOpenShiftVersionMap map[string]string
 	listDeployments         = getDeploymentsList
+	listDaemonSets          = getDaemonSetsList
+	listStatefulSets        = getStatefulSetsList
 	latestKubeVersion       *semver.Version
 )
 
@@ -41,9 +43,10 @@ type versionMapping struct {
 	OcpVersion  string `yaml:"ocp-version"`
 }
 
-type deploymentNotReady struct {
-	Name        string
-	Unavailable int32
+type workloadNotReady struct {
+	ResourceType string
+	Name         string
+	Unavailable  int32
 }
 
 func init() {
@@ -94,49 +97,81 @@ func NewKubectl(kubeConfig clientcmd.ClientConfig) (*Kubectl, error) {
 	return kubectl, nil
 }
 
-func (k Kubectl) WaitForDeployments(context context.Context, namespace string, selector string) error {
+// WaitForWorkloadResources returns nil when all pods for requested workload resources are confirmed ready
+// or an error if resources cannot be confirmed ready before the timeout is exceeded.
+// Currently checks deployments, daemonSets, and statefulSets.
+func (k Kubectl) WaitForWorkloadResources(context context.Context, namespace string, selector string) error {
 	deadline, _ := context.Deadline()
-	unavailableDeployments := []deploymentNotReady{{Name: "none", Unavailable: 1}}
-	getDeploymentsError := ""
-
-	utils.LogInfo(fmt.Sprintf("Start wait for deployments. --timeout time left: %s ", time.Until(deadline).String()))
-
-	for deadline.After(time.Now()) && len(unavailableDeployments) > 0 {
-		unavailableDeployments = []deploymentNotReady{}
-		deployments, err := listDeployments(k, context, namespace, selector)
-		if err != nil {
-			unavailableDeployments = []deploymentNotReady{{Name: "none", Unavailable: 1}}
-			getDeploymentsError = fmt.Sprintf("error getting deployments from namespace %s : %v", namespace, err)
-			utils.LogWarning(getDeploymentsError)
-			time.Sleep(time.Second)
-		} else {
-			getDeploymentsError = ""
+	unavailableWorkloadResources := []workloadNotReady{{Name: "none", Unavailable: 1}}
+	getWorkloadResourceError := ""
+
+	// Loop until timeout reached or all requested pods are available
+	utils.LogInfo(fmt.Sprintf("Start wait for workloads resources. --timeout time left: %s ", time.Until(deadline).String()))
+	for deadline.After(time.Now()) && len(unavailableWorkloadResources) > 0 {
+		unavailableWorkloadResources = []workloadNotReady{}
+
+		deployments, errDeployments := listDeployments(k, context, namespace, selector)
+		daemonSets, errDaemonSets := listDaemonSets(k, context, namespace, selector)
+		statefulSets, errStatefulSets := listStatefulSets(k, context, namespace, selector)
+
+		// Inspect the resources that are successfully returned or handle API request errors
+		if errDeployments == nil && errDaemonSets == nil && errStatefulSets == nil {
+			getWorkloadResourceError = ""
+			// Check the number of unavailable replicas for each workload type
 			for _, deployment := range deployments {
-				// Just after rollout, pods from the previous deployment revision may still be in a
-				// terminating state.
 				if deployment.Status.UnavailableReplicas > 0 {
-					unavailableDeployments = append(unavailableDeployments, deploymentNotReady{Name: deployment.Name, Unavailable: deployment.Status.UnavailableReplicas})
+					unavailableWorkloadResources = append(unavailableWorkloadResources, workloadNotReady{Name: deployment.Name, ResourceType: "Deployment", Unavailable: deployment.Status.UnavailableReplicas})
+				}
+			}
+			for _, daemonSet := range daemonSets {
+				if daemonSet.Status.NumberUnavailable > 0 {
+					unavailableWorkloadResources = append(unavailableWorkloadResources, workloadNotReady{Name: daemonSet.Name, ResourceType: "DaemonSet", Unavailable: daemonSet.Status.NumberUnavailable})
 				}
 			}
-			if len(unavailableDeployments) > 0 {
-				utils.LogInfo(fmt.Sprintf("Wait for %d deployments:", len(unavailableDeployments)))
-				for _, unavailableDeployment := range unavailableDeployments {
-					utils.LogInfo(fmt.Sprintf("    - %s with %d unavailable replicas", unavailableDeployment.Name, unavailableDeployment.Unavailable))
+			for _, statefulSet := range statefulSets {
+				// StatefulSet doesn't report unavailable replicas so it is calculated here
+				unavailableReplicas := statefulSet.Status.Replicas - statefulSet.Status.AvailableReplicas
+				if unavailableReplicas > 0 {
+					unavailableWorkloadResources = append(unavailableWorkloadResources, workloadNotReady{Name: statefulSet.Name, ResourceType: "StatefulSet", Unavailable: unavailableReplicas})
+				}
+			}
+
+			// If any pods are unavailable report it and sleep until the next loop
+			// Else everything is available and the loop will exit
+			if len(unavailableWorkloadResources) > 0 {
+				utils.LogInfo(fmt.Sprintf("Wait for %d workload resources:", len(unavailableWorkloadResources)))
+				for _, unavailableWorkloadResource := range unavailableWorkloadResources {
+					utils.LogInfo(fmt.Sprintf("    - %s %s with %d unavailable pods", unavailableWorkloadResource.ResourceType, unavailableWorkloadResource.Name, unavailableWorkloadResource.Unavailable))
 				}
 				time.Sleep(time.Second)
 			} else {
-				utils.LogInfo(fmt.Sprintf("Finish wait for deployments, --timeout time left %s", time.Until(deadline).String()))
+				utils.LogInfo(fmt.Sprintf("Finish wait for workload resources, --timeout time left %s", time.Until(deadline).String()))
+			}
+		} else {
+			resourceType := "Deployment"
+			errMsg := errDeployments
+			if errDaemonSets != nil {
+				resourceType = "DaemonSet"
+				errMsg = errDaemonSets
+			} else if errStatefulSets != nil {
+				resourceType = "StatefulSet"
+				errMsg = errStatefulSets
 			}
+			unavailableWorkloadResources = []workloadNotReady{{Name: "none", ResourceType: resourceType, Unavailable: 1}}
+			getWorkloadResourceError = fmt.Sprintf("error getting %s from namespace %s : %v", resourceType, namespace, errMsg)
+			utils.LogWarning(getWorkloadResourceError)
+			time.Sleep(time.Second)
 		}
 	}
 
-	if len(getDeploymentsError) > 0 {
-		errorMsg := fmt.Sprintf("Time out retrying after %s", getDeploymentsError)
+	// Any errors or resources that are still unavailable returns an error at this point
+	if getWorkloadResourceError != "" {
+		errorMsg := fmt.Sprintf("Time out retrying after %s", getWorkloadResourceError)
 		utils.LogError(errorMsg)
 		return errors.New(errorMsg)
 	}
-	if len(unavailableDeployments) > 0 {
-		errorMsg := "error unavailable deployments, timeout has expired, please consider increasing the timeout using the chart-verifier --timeout flag"
+	if len(unavailableWorkloadResources) > 0 {
+		errorMsg := "error unavailable workload resources, timeout has expired, please consider increasing the timeout using the chart-verifier --timeout flag"
 		utils.LogError(errorMsg)
 		return errors.New(errorMsg)
 	}
@@ -182,6 +217,22 @@ func getDeploymentsList(k Kubectl, context context.Context, namespace string, se
 	return list.Items, err
 }
 
+func getStatefulSetsList(k Kubectl, context context.Context, namespace string, selector string) ([]v1.StatefulSet, error) {
+	list, err := k.clientset.AppsV1().StatefulSets(namespace).List(context, metav1.ListOptions{LabelSelector: selector})
+	if err != nil {
+		return nil, err
+	}
+	return list.Items, err
+}
+
+func getDaemonSetsList(k Kubectl, context context.Context, namespace string, selector string) ([]v1.DaemonSet, error) {
+	list, err := k.clientset.AppsV1().DaemonSets(namespace).List(context, metav1.ListOptions{LabelSelector: selector})
+	if err != nil {
+		return nil, err
+	}
+	return list.Items, err
+}
+
 func GetLatestKubeVersion() string {
 	return latestKubeVersion.String()
 }