diff --git a/internal/chartverifier/checks/charttesting.go b/internal/chartverifier/checks/charttesting.go index adab0479..49877727 100644 --- a/internal/chartverifier/checks/charttesting.go +++ b/internal/chartverifier/checks/charttesting.go @@ -235,7 +235,7 @@ func testRelease( release, namespace, releaseSelector string, cleanupHelmTests bool, ) error { - if err := kubectl.WaitForDeployments(ctx, namespace, releaseSelector); err != nil { + if err := kubectl.WaitForWorkloadResources(ctx, namespace, releaseSelector); err != nil { return err } if err := helm.Test(ctx, namespace, release); err != nil { diff --git a/internal/tool/kubectl.go b/internal/tool/kubectl.go index 1422f36a..b4eab169 100644 --- a/internal/tool/kubectl.go +++ b/internal/tool/kubectl.go @@ -29,6 +29,8 @@ var content embed.FS var ( kubeOpenShiftVersionMap map[string]string listDeployments = getDeploymentsList + listDaemonSets = getDaemonSetsList + listStatefulSets = getStatefulSetsList latestKubeVersion *semver.Version ) @@ -41,9 +43,10 @@ type versionMapping struct { OcpVersion string `yaml:"ocp-version"` } -type deploymentNotReady struct { - Name string - Unavailable int32 +type workloadNotReady struct { + ResourceType string + Name string + Unavailable int32 } func init() { @@ -94,49 +97,81 @@ func NewKubectl(kubeConfig clientcmd.ClientConfig) (*Kubectl, error) { return kubectl, nil } -func (k Kubectl) WaitForDeployments(context context.Context, namespace string, selector string) error { +// WaitForWorkloadResources returns nil when all pods for requested workload resources are confirmed ready +// or an error if resources cannot be confirmed ready before the timeout is exceeded. +// Currently checks deployments, daemonSets, and statefulSets. +func (k Kubectl) WaitForWorkloadResources(context context.Context, namespace string, selector string) error { deadline, _ := context.Deadline() - unavailableDeployments := []deploymentNotReady{{Name: "none", Unavailable: 1}} - getDeploymentsError := "" - - utils.LogInfo(fmt.Sprintf("Start wait for deployments. --timeout time left: %s ", time.Until(deadline).String())) - - for deadline.After(time.Now()) && len(unavailableDeployments) > 0 { - unavailableDeployments = []deploymentNotReady{} - deployments, err := listDeployments(k, context, namespace, selector) - if err != nil { - unavailableDeployments = []deploymentNotReady{{Name: "none", Unavailable: 1}} - getDeploymentsError = fmt.Sprintf("error getting deployments from namespace %s : %v", namespace, err) - utils.LogWarning(getDeploymentsError) - time.Sleep(time.Second) - } else { - getDeploymentsError = "" + unavailableWorkloadResources := []workloadNotReady{{Name: "none", Unavailable: 1}} + getWorkloadResourceError := "" + + // Loop until timeout reached or all requested pods are available + utils.LogInfo(fmt.Sprintf("Start wait for workloads resources. --timeout time left: %s ", time.Until(deadline).String())) + for deadline.After(time.Now()) && len(unavailableWorkloadResources) > 0 { + unavailableWorkloadResources = []workloadNotReady{} + + deployments, errDeployments := listDeployments(k, context, namespace, selector) + daemonSets, errDaemonSets := listDaemonSets(k, context, namespace, selector) + statefulSets, errStatefulSets := listStatefulSets(k, context, namespace, selector) + + // Inspect the resources that are successfully returned or handle API request errors + if errDeployments == nil && errDaemonSets == nil && errStatefulSets == nil { + getWorkloadResourceError = "" + // Check the number of unavailable replicas for each workload type for _, deployment := range deployments { - // Just after rollout, pods from the previous deployment revision may still be in a - // terminating state. if deployment.Status.UnavailableReplicas > 0 { - unavailableDeployments = append(unavailableDeployments, deploymentNotReady{Name: deployment.Name, Unavailable: deployment.Status.UnavailableReplicas}) + unavailableWorkloadResources = append(unavailableWorkloadResources, workloadNotReady{Name: deployment.Name, ResourceType: "Deployment", Unavailable: deployment.Status.UnavailableReplicas}) + } + } + for _, daemonSet := range daemonSets { + if daemonSet.Status.NumberUnavailable > 0 { + unavailableWorkloadResources = append(unavailableWorkloadResources, workloadNotReady{Name: daemonSet.Name, ResourceType: "DaemonSet", Unavailable: daemonSet.Status.NumberUnavailable}) } } - if len(unavailableDeployments) > 0 { - utils.LogInfo(fmt.Sprintf("Wait for %d deployments:", len(unavailableDeployments))) - for _, unavailableDeployment := range unavailableDeployments { - utils.LogInfo(fmt.Sprintf(" - %s with %d unavailable replicas", unavailableDeployment.Name, unavailableDeployment.Unavailable)) + for _, statefulSet := range statefulSets { + // StatefulSet doesn't report unavailable replicas so it is calculated here + unavailableReplicas := statefulSet.Status.Replicas - statefulSet.Status.AvailableReplicas + if unavailableReplicas > 0 { + unavailableWorkloadResources = append(unavailableWorkloadResources, workloadNotReady{Name: statefulSet.Name, ResourceType: "StatefulSet", Unavailable: unavailableReplicas}) + } + } + + // If any pods are unavailable report it and sleep until the next loop + // Else everything is available and the loop will exit + if len(unavailableWorkloadResources) > 0 { + utils.LogInfo(fmt.Sprintf("Wait for %d workload resources:", len(unavailableWorkloadResources))) + for _, unavailableWorkloadResource := range unavailableWorkloadResources { + utils.LogInfo(fmt.Sprintf(" - %s %s with %d unavailable pods", unavailableWorkloadResource.ResourceType, unavailableWorkloadResource.Name, unavailableWorkloadResource.Unavailable)) } time.Sleep(time.Second) } else { - utils.LogInfo(fmt.Sprintf("Finish wait for deployments, --timeout time left %s", time.Until(deadline).String())) + utils.LogInfo(fmt.Sprintf("Finish wait for workload resources, --timeout time left %s", time.Until(deadline).String())) + } + } else { + resourceType := "Deployment" + errMsg := errDeployments + if errDaemonSets != nil { + resourceType = "DaemonSet" + errMsg = errDaemonSets + } else if errStatefulSets != nil { + resourceType = "StatefulSet" + errMsg = errStatefulSets } + unavailableWorkloadResources = []workloadNotReady{{Name: "none", ResourceType: resourceType, Unavailable: 1}} + getWorkloadResourceError = fmt.Sprintf("error getting %s from namespace %s : %v", resourceType, namespace, errMsg) + utils.LogWarning(getWorkloadResourceError) + time.Sleep(time.Second) } } - if len(getDeploymentsError) > 0 { - errorMsg := fmt.Sprintf("Time out retrying after %s", getDeploymentsError) + // Any errors or resources that are still unavailable returns an error at this point + if getWorkloadResourceError != "" { + errorMsg := fmt.Sprintf("Time out retrying after %s", getWorkloadResourceError) utils.LogError(errorMsg) return errors.New(errorMsg) } - if len(unavailableDeployments) > 0 { - errorMsg := "error unavailable deployments, timeout has expired, please consider increasing the timeout using the chart-verifier --timeout flag" + if len(unavailableWorkloadResources) > 0 { + errorMsg := "error unavailable workload resources, timeout has expired, please consider increasing the timeout using the chart-verifier --timeout flag" utils.LogError(errorMsg) return errors.New(errorMsg) } @@ -182,6 +217,22 @@ func getDeploymentsList(k Kubectl, context context.Context, namespace string, se return list.Items, err } +func getStatefulSetsList(k Kubectl, context context.Context, namespace string, selector string) ([]v1.StatefulSet, error) { + list, err := k.clientset.AppsV1().StatefulSets(namespace).List(context, metav1.ListOptions{LabelSelector: selector}) + if err != nil { + return nil, err + } + return list.Items, err +} + +func getDaemonSetsList(k Kubectl, context context.Context, namespace string, selector string) ([]v1.DaemonSet, error) { + list, err := k.clientset.AppsV1().DaemonSets(namespace).List(context, metav1.ListOptions{LabelSelector: selector}) + if err != nil { + return nil, err + } + return list.Items, err +} + func GetLatestKubeVersion() string { return latestKubeVersion.String() } diff --git a/internal/tool/kubectl_test.go b/internal/tool/kubectl_test.go index df13185a..7990f2fb 100644 --- a/internal/tool/kubectl_test.go +++ b/internal/tool/kubectl_test.go @@ -124,18 +124,108 @@ var testDeployments = []v1.Deployment{ }, } -var DeploymentList1 []v1.Deployment +var testDaemonSets = []v1.DaemonSet{ + { + ObjectMeta: metav1.ObjectMeta{Name: "test0"}, + Status: v1.DaemonSetStatus{NumberUnavailable: 1}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "test1"}, + Status: v1.DaemonSetStatus{NumberUnavailable: 2}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "test2"}, + Status: v1.DaemonSetStatus{NumberUnavailable: 3}, + }, +} + +var testStatefulSets = []v1.StatefulSet{ + { + ObjectMeta: metav1.ObjectMeta{Name: "test0"}, + Status: v1.StatefulSetStatus{Replicas: 1, AvailableReplicas: 0}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "test1"}, + Status: v1.StatefulSetStatus{Replicas: 2, AvailableReplicas: 0}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "test0"}, + Status: v1.StatefulSetStatus{Replicas: 3, AvailableReplicas: 0}, + }, +} + +var ( + DeploymentList1 []v1.Deployment + DaemonSetList1 []v1.DaemonSet + StatefulSetList1 []v1.StatefulSet +) + +func TestGetDeploymentList(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + clientset := fake.NewSimpleClientset() + _, err := clientset.AppsV1().Deployments("").Create(ctx, &testDeployments[0], metav1.CreateOptions{}) + require.NoError(t, err) + + kubectl := Kubectl{clientset: clientset} + deps, err := getDeploymentsList(kubectl, ctx, "", "") + + require.NoError(t, err) + require.Equal(t, 1, len(deps)) + require.Equal(t, testDeployments[0], deps[0]) +} + +func TestGetDaemonSetList(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + clientset := fake.NewSimpleClientset() + _, err := clientset.AppsV1().DaemonSets("").Create(ctx, &testDaemonSets[0], metav1.CreateOptions{}) + require.NoError(t, err) + + kubectl := Kubectl{clientset: clientset} + daemons, err := getDaemonSetsList(kubectl, ctx, "", "") + + require.NoError(t, err) + require.Equal(t, 1, len(daemons)) + require.Equal(t, testDaemonSets[0], daemons[0]) +} + +func TestGetStatefulSetList(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + clientset := fake.NewSimpleClientset() + _, err := clientset.AppsV1().StatefulSets("").Create(ctx, &testStatefulSets[0], metav1.CreateOptions{}) + require.NoError(t, err) + + kubectl := Kubectl{clientset: clientset} + sts, err := getStatefulSetsList(kubectl, ctx, "", "") + + require.NoError(t, err) + require.Equal(t, 1, len(sts)) + require.Equal(t, testStatefulSets[0], sts[0]) +} -func TestWaitForDeployments(t *testing.T) { +func TestWaitForWorkloadResources(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() listDeployments = deploymentTestListGood + listDaemonSets = daemonSetTestListGood + listStatefulSets = statefulSetTestListGood + DeploymentList1 = make([]v1.Deployment, len(testDeployments)) + DaemonSetList1 = make([]v1.DaemonSet, len(testDaemonSets)) + StatefulSetList1 = make([]v1.StatefulSet, len(testStatefulSets)) + copy(DeploymentList1, testDeployments) + copy(DaemonSetList1, testDaemonSets) + copy(StatefulSetList1, testStatefulSets) k := new(Kubectl) - err := k.WaitForDeployments(ctx, "testNameSpace", "selector") + err := k.WaitForWorkloadResources(ctx, "testNameSpace", "selector") require.NoError(t, err) } @@ -147,12 +237,62 @@ func TestBadToGoodWaitForDeployments(t *testing.T) { DeploymentList1 = make([]v1.Deployment, len(testDeployments)) copy(DeploymentList1, testDeployments) + listDaemonSets = daemonSetTestListGood + DaemonSetList1 = make([]v1.DaemonSet, len(testDaemonSets)) + copy(DaemonSetList1, testDaemonSets) + + listStatefulSets = statefulSetTestListGood + StatefulSetList1 = make([]v1.StatefulSet, len(testStatefulSets)) + copy(StatefulSetList1, testStatefulSets) + + k := new(Kubectl) + err := k.WaitForWorkloadResources(ctx, "testNameSpace", "selector") + require.NoError(t, err) +} + +func TestBadToGoodWaitForDaemonSets(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + listDeployments = deploymentTestListGood + DeploymentList1 = make([]v1.Deployment, len(testDeployments)) + copy(DeploymentList1, testDeployments) + + listDaemonSets = daemonSetTestListBadToGood + DaemonSetList1 = make([]v1.DaemonSet, len(testDaemonSets)) + copy(DaemonSetList1, testDaemonSets) + + listStatefulSets = statefulSetTestListGood + StatefulSetList1 = make([]v1.StatefulSet, len(testStatefulSets)) + copy(StatefulSetList1, testStatefulSets) + + k := new(Kubectl) + err := k.WaitForWorkloadResources(ctx, "testNameSpace", "selector") + require.NoError(t, err) +} + +func TestBadToGoodWaitForStatefulSets(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + listDeployments = deploymentTestListGood + DeploymentList1 = make([]v1.Deployment, len(testDeployments)) + copy(DeploymentList1, testDeployments) + + listDaemonSets = daemonSetTestListGood + DaemonSetList1 = make([]v1.DaemonSet, len(testDaemonSets)) + copy(DaemonSetList1, testDaemonSets) + + listStatefulSets = statefulSetTestListBadToGood + StatefulSetList1 = make([]v1.StatefulSet, len(testStatefulSets)) + copy(StatefulSetList1, testStatefulSets) + k := new(Kubectl) - err := k.WaitForDeployments(ctx, "testNameSpace", "selector") + err := k.WaitForWorkloadResources(ctx, "testNameSpace", "selector") require.NoError(t, err) } -func TestTimeExpirationWaitingForDeployments(t *testing.T) { +func TestTimeExpirationWaitingForWorkloadResources(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() @@ -160,10 +300,13 @@ func TestTimeExpirationWaitingForDeployments(t *testing.T) { DeploymentList1 = make([]v1.Deployment, len(testDeployments)) copy(DeploymentList1, testDeployments) + listDaemonSets = daemonSetTestListGood + listStatefulSets = statefulSetTestListGood + k := new(Kubectl) - err := k.WaitForDeployments(ctx, "testNameSpace", "selector") + err := k.WaitForWorkloadResources(ctx, "testNameSpace", "selector") require.Error(t, err) - require.Contains(t, err.Error(), "error unavailable deployments, timeout has expired,") + require.Contains(t, err.Error(), "error unavailable workload resources, timeout has expired,") } func TestTimeExpirationGetDeploymentsFailure(t *testing.T) { @@ -175,14 +318,17 @@ func TestTimeExpirationGetDeploymentsFailure(t *testing.T) { copy(DeploymentList1, testDeployments) k := new(Kubectl) - err := k.WaitForDeployments(ctx, "testNameSpace", "selector") + err := k.WaitForWorkloadResources(ctx, "testNameSpace", "selector") require.Error(t, err) require.Contains(t, err.Error(), "Time out retrying after") - require.Contains(t, err.Error(), "error getting deployments from namespace") + require.Contains(t, err.Error(), "error getting Deployment from namespace") require.Contains(t, err.Error(), "pretend error getting deployment list") } func deploymentTestListGood(k Kubectl, context context.Context, namespace string, selector string) ([]v1.Deployment, error) { + // Mock listing Deployments + // Return contents of testDeployments and decrease + // the nuber of unavailable pods on each call fmt.Println("deploymentTestListGood called") for index := 0; index < len(DeploymentList1); index++ { if DeploymentList1[index].Status.UnavailableReplicas > 0 { @@ -193,19 +339,83 @@ func deploymentTestListGood(k Kubectl, context context.Context, namespace string return DeploymentList1, nil } +func daemonSetTestListGood(k Kubectl, context context.Context, namespace string, selector string) ([]v1.DaemonSet, error) { + // Mock listing DaemonSets + // Return contents of testDaemonSets and decrease + // the nuber of unavailable pods on each call + for index := 0; index < len(testDaemonSets); index++ { + if DaemonSetList1[index].Status.NumberUnavailable > 0 { + DaemonSetList1[index].Status.NumberUnavailable-- + fmt.Printf("NumberUnavailable set to %d for daemonset %s\n", DaemonSetList1[index].Status.NumberUnavailable, DaemonSetList1[index].Name) + } + } + return DaemonSetList1, nil +} + +func statefulSetTestListGood(k Kubectl, context context.Context, namespace string, selector string) ([]v1.StatefulSet, error) { + // Mock listing StatefulSets + // Return contents of testStatefulSets and increase + // the nuber of available pods on each call + fmt.Println("statefulSetSetTestListGood called") + for index := 0; index < len(testDaemonSets); index++ { + unavailableReplicas := StatefulSetList1[index].Status.Replicas - StatefulSetList1[index].Status.AvailableReplicas + if unavailableReplicas > 0 { + StatefulSetList1[index].Status.AvailableReplicas++ + unavailableReplicas = StatefulSetList1[index].Status.Replicas - StatefulSetList1[index].Status.AvailableReplicas + + fmt.Printf("Unavailable Replicas set to %d for statefulset %s\n", unavailableReplicas, StatefulSetList1[index].Name) + } + } + return StatefulSetList1, nil +} + func deploymentTestListBad(k Kubectl, context context.Context, namespace string, selector string) ([]v1.Deployment, error) { + // Mock listing Deployments with error fmt.Println("deploymentTestListBad called") return nil, errors.New("pretend error getting deployment list") } -var errorSent = false +var ( + deploymentErrorSent = false + daemonSetErrorSent = false + statefulSetErrorSent = false +) func deploymentTestListBadToGood(k Kubectl, context context.Context, namespace string, selector string) ([]v1.Deployment, error) { - if !errorSent { - fmt.Println("deploymentTestListBadToGoodToBad bad path") - errorSent = true + // Mock listing Deployments + // Return error on first call and pass to deploymentTestListGood + // on subsequent calls + if !deploymentErrorSent { + fmt.Println("deploymentTestListBadToGood bad path") + deploymentErrorSent = true return nil, errors.New("pretend error getting deployment list") } - fmt.Println("deploymentTestListBadToGoodToBad good path") + fmt.Println("deploymentTestListBadToGood good path") return deploymentTestListGood(k, context, namespace, selector) } + +func daemonSetTestListBadToGood(k Kubectl, context context.Context, namespace string, selector string) ([]v1.DaemonSet, error) { + // Mock listing DaemonSets + // Return error on first call and pass to daemonSetTestListGood + // on subsequent calls + if !daemonSetErrorSent { + fmt.Println("daemonSetTestListBadToGood bad path") + daemonSetErrorSent = true + return nil, errors.New("pretend error getting daemonSet list") + } + fmt.Println("deploymentTestListBadToGood good path") + return daemonSetTestListGood(k, context, namespace, selector) +} + +func statefulSetTestListBadToGood(k Kubectl, context context.Context, namespace string, selector string) ([]v1.StatefulSet, error) { + // Mock listing StatefulSets + // Return error on first call and pass to statefulSetTestListGood + // on subsequent calls + if !statefulSetErrorSent { + fmt.Println("statefulSetTestListBadToGood bad path") + statefulSetErrorSent = true + return nil, errors.New("pretend error getting statefulSet list") + } + fmt.Println("statefulSetTestListBadToGood good path") + return statefulSetTestListGood(k, context, namespace, selector) +}