MCO-2006: Migrate pinned images sets private test cases

Author: sergiordlr

cmd/machine-config-tests-ext/main.go

@@ -72,6 +72,18 @@ func main() {
 		},
 		ClusterStability: e.ClusterStabilityDisruptive,
 		TestTimeout:      &defaultTimeout,
+		Description:      "Suite that daily sends signals to component readiness"
+	})
+
+	// This suite will include all tests not included in the previous suite inheriting from openshift/disruptive
+	ext.AddGlobalSuite(e.Suite{
+		Name: "openshift/machine-config-operator/longduration",
+		Qualifiers: []string{
+			`name.contains("[Suite:openshift/machine-config-operator/longduration]")`,
+		},
+		ClusterStability: e.ClusterStabilityDisruptive,
+		TestTimeout:      &defaultTimeout,
+		Description:      "A long-running, resource-intensive test suite executed on a scheduled basis to provide deep validation beyond the standard executions",
 	})
 
 	// If using Ginkgo, build test specs automatically

test/extended-priv/const.go

@@ -75,4 +75,9 @@ const (
 	MachineSetResource = "machinesets"
 	// ControlPlaneMachineSetResource is the resource name for controlplanemachinesets
 	ControlPlaneMachineSetResource = "controlplanemachinesets"
+
+	// BusyBoxImage the multiplatform busybox image stored in openshifttest
+	BusyBoxImage = "quay.io/openshifttest/busybox@sha256:c5439d7db88ab5423999530349d327b04279ad3161d7596d2126dfb5b02bfd1f"
+	// AlpineImage the multiplatform alpine image stored in openshifttest
+	AlpineImage = "quay.io/openshifttest/alpine@sha256:dc1536cbff0ba235d4219462aeccd4caceab9def96ae8064257d049166890083"
 )

test/extended-priv/controller.go

@@ -130,3 +130,19 @@ func (mcc *Controller) HasAcquiredLease() (bool, error) {
 
 	return strings.Contains(podAllLogs, "successfully acquired lease"), nil
 }
+
+// GetNode return the node where the machine controller is running
+func (mcc *Controller) GetNode() (*Node, error) {
+	controllerPodName, err := mcc.GetCachedPodName()
+	if err != nil {
+		return nil, err
+	}
+
+	controllerPod := NewNamespacedResource(mcc.oc, "pod", MachineConfigNamespace, controllerPodName)
+	nodeName, err := controllerPod.Get(`{.spec.nodeName}`)
+	if err != nil {
+		return nil, err
+	}
+
+	return NewNode(mcc.oc, nodeName), nil
+}

test/extended-priv/gomega_matchers.go

@@ -152,3 +152,8 @@ func (matcher *AvailableMatcher) NegatedFailureMessage(actual interface{}) (mess
 
 	return message
 }
+
+// BeAvailable returns the gomega matcher to check if a resource is available or not.
+func BeAvailable() types.GomegaMatcher {
+	return &DegradedMatcher{&conditionMatcher{conditionType: "Available", field: "status", expected: "True"}}
+}

test/extended-priv/kubeletconfig.go

@@ -0,0 +1,73 @@
+package extended
+
+import (
+	"fmt"
+
+	o "github.com/onsi/gomega"
+
+	exutil "github.com/openshift/machine-config-operator/test/extended-priv/util"
+	logger "github.com/openshift/machine-config-operator/test/extended-priv/util/logext"
+)
+
+// KubeletConfig struct is used to handle KubeletConfig resources in OCP
+type KubeletConfig struct {
+	Resource
+	template string
+}
+
+// KubeletConfigList handles list of nodes
+type KubeletConfigList struct {
+	ResourceList
+}
+
+// NewKubeletConfig create a NewKubeletConfig struct
+func NewKubeletConfig(oc *exutil.CLI, name, template string) *KubeletConfig {
+	return &KubeletConfig{Resource: *NewResource(oc, "KubeletConfig", name), template: template}
+}
+
+// NewKubeletConfigList create a NewKubeletConfigList struct
+func NewKubeletConfigList(oc *exutil.CLI) *KubeletConfigList {
+	return &KubeletConfigList{*NewResourceList(oc, "KubeletConfig")}
+}
+
+func (kc *KubeletConfig) create(parameters ...string) {
+	allParams := []string{"--ignore-unknown-parameters=true", "-f", kc.template,
+		"-p", "NAME=" + kc.name}
+	allParams = append(allParams, parameters...)
+	exutil.CreateClusterResourceFromTemplate(kc.oc, allParams...)
+}
+
+func (kc KubeletConfig) waitUntilSuccess(timeout string) {
+	logger.Infof("wait for %s to report success", kc.name)
+	o.EventuallyWithOffset(1, &kc, timeout, "2s").Should(o.SatisfyAll(
+		HaveConditionField("Success", "status", "True"),
+		HaveConditionField("Success", "message", "Success"),
+	), "KubeletConfig '%s' should report Success in status.conditions, but the current status is not success", kc.GetName())
+}
+
+func (kc KubeletConfig) waitUntilFailure(expectedMsg, timeout string) {
+	logger.Infof("wait for %s to report failure", kc.name)
+	o.EventuallyWithOffset(1, &kc, timeout, "2s").Should(o.SatisfyAll(
+		HaveConditionField("Failure", "status", "False"),
+		HaveConditionField("Failure", "message", o.ContainSubstring(expectedMsg)),
+	), "KubeletConfig '%s' should report Failure in status.conditions and report failure message %s. But it doesn't.", kc.GetName(), expectedMsg)
+}
+
+// GetGeneratedMCName returns the name of the MC that was generated by this KubeletConfig resource
+func (kc KubeletConfig) GetGeneratedMCName() (string, error) {
+	mcName, err := kc.Get(`{.metadata.finalizers[0]}`)
+	if err != nil {
+		return "", err
+	}
+	if mcName == "" {
+		return "", fmt.Errorf("It was not possible to get the finalizer from %s %s: %s", kc.GetKind(), kc.GetName(), kc.PrettyString())
+	}
+	return mcName, nil
+}
+
+// GetGeneratedMCNameOrFail returns the name of the MC that was generated by this KubeletConfig resource and fails the test case if it cannot be done
+func (kc KubeletConfig) GetGeneratedMCNameOrFail() string {
+	mcName, err := kc.GetGeneratedMCName()
+	o.Expect(err).NotTo(o.HaveOccurred(), "Error getting the generated MC for %s %s", kc.GetKind(), kc.GetName())
+	return mcName
+}

test/extended-priv/machineconfignode.go

@@ -13,3 +13,18 @@ type MachineConfigNode struct {
 func NewMachineConfigNode(oc *exutil.CLI, node string) *MachineConfigNode {
 	return &MachineConfigNode{Resource: *NewResource(oc, "machineconfignode", node)}
 }
+
+// IsPinnedImageSetsDegraded returns true if the PinnedImageSetsDegraded condition is true
+func (mcn *MachineConfigNode) IsPinnedImageSetsDegraded() bool {
+	return mcn.IsConditionStatusTrue("PinnedImageSetsDegraded")
+}
+
+// IsPinnedImageSetsProgressing returns true if the PinnedImageSetsProgressing condition is true
+func (mcn *MachineConfigNode) IsPinnedImageSetsProgressing() bool {
+	return mcn.IsConditionStatusTrue("PinnedImageSetsProgressing")
+}
+
+// GetPinnedImageSetLastFailedError returns the last failed generation error for pinned image sets
+func (mcn *MachineConfigNode) GetPinnedImageSetLastFailedError() string {
+	return mcn.GetOrFail(`{.status.pinnedImageSets[*].lastFailedGenerationError}`)
+}

test/extended-priv/machineconfigpool.go

@@ -12,6 +12,7 @@ import (
 	o "github.com/onsi/gomega"
 	exutil "github.com/openshift/machine-config-operator/test/extended-priv/util"
 	logger "github.com/openshift/machine-config-operator/test/extended-priv/util/logext"
+	"github.com/tidwall/gjson"
 	"k8s.io/apimachinery/pkg/util/wait"
 	e2e "k8s.io/kubernetes/test/e2e/framework"
 )
@@ -848,3 +849,197 @@ func DebugDegradedStatus(mcp *MachineConfigPool) {
 	logger.Infof("Last %d lines of MCC:\n%s", maxMCCLines, GetLastNLines(mccLogs, maxMCCLines))
 	logger.Infof("END DEBUG")
 }
+
+// DeleteCustomMCP deletes a custom MCP and removes its labels from all nodes
+func DeleteCustomMCP(oc *exutil.CLI, name string) error {
+	mcp := NewMachineConfigPool(oc, name)
+	if !mcp.Exists() {
+		logger.Infof("MCP %s does not exist. No need to remove it", mcp.GetName())
+		return nil
+	}
+
+	exutil.By(fmt.Sprintf("Removing custom MCP %s", name))
+
+	nodes, err := mcp.GetNodes()
+	if err != nil {
+		logger.Errorf("Could not get the nodes that belong to MCP %s: %s", mcp.GetName(), err)
+		return err
+	}
+
+	label := fmt.Sprintf("node-role.kubernetes.io/%s", mcp.GetName())
+	for _, node := range nodes {
+		logger.Infof("Removing pool label from node %s", node.GetName())
+		err := node.RemoveLabel(label)
+		if err != nil {
+			logger.Errorf("Could not remove the role label from node %s: %s", node.GetName(), err)
+			return err
+		}
+	}
+
+	for _, node := range nodes {
+		err := node.WaitForLabelRemoved(label)
+		if err != nil {
+			logger.Errorf("The label %s was not removed from node %s", label, node.GetName())
+		}
+	}
+
+	err = mcp.WaitForMachineCount(0, 5*time.Minute)
+	if err != nil {
+		logger.Errorf("The %s MCP already contains nodes, it cannot be deleted: %s", mcp.GetName(), err)
+		return err
+	}
+
+	// Wait for worker MCP to be updated before removing the custom pool
+	// in order to make sure that no node has any annotation pointing to resources that depend on the custom pool that we want to delete
+	wMcp := NewMachineConfigPool(oc, MachineConfigPoolWorker)
+	wMcp.waitForComplete()
+
+	err = mcp.Delete()
+	if err != nil {
+		logger.Errorf("Could not delete %s MCP", mcp.GetName())
+		return err
+	}
+
+	return nil
+}
+
+// GetPoolSynchronizersStatusByType returns the pool synchronizer status for a given type
+func (mcp *MachineConfigPool) GetPoolSynchronizersStatusByType(pType string) (string, error) {
+	return mcp.Get(`{.status.poolSynchronizersStatus[?(@.poolSynchronizerType=="` + pType + `")]}`)
+}
+
+// IsPinnedImagesComplete returns if the MCP is reporting that there is no pinnedimages operation in progress
+func (mcp *MachineConfigPool) IsPinnedImagesComplete() (bool, error) {
+
+	pinnedStatus, err := mcp.GetPoolSynchronizersStatusByType("PinnedImageSets")
+	if err != nil {
+		return false, err
+	}
+
+	logger.Infof("Pinned status: %s", pinnedStatus)
+
+	mcpMachineCount, err := mcp.Get(`{.status.machineCount}`)
+	if err != nil {
+		return false, err
+	}
+
+	if mcpMachineCount == "" {
+		return false, fmt.Errorf("status.machineCount is empty in mcp %s", mcp.GetName())
+	}
+
+	pinnedMachineCount := gjson.Get(pinnedStatus, "machineCount").String()
+	if pinnedMachineCount == "" {
+		return false, fmt.Errorf("pinned status machineCount is empty in mcp %s", mcp.GetName())
+	}
+
+	pinnedUnavailableMachineCount := gjson.Get(pinnedStatus, "unavailableMachineCount").String()
+	if pinnedUnavailableMachineCount == "" {
+		return false, fmt.Errorf("pinned status unavailableMachineCount is empty in mcp %s", mcp.GetName())
+	}
+
+	updatedMachineCount := gjson.Get(pinnedStatus, "updatedMachineCount").String()
+	if updatedMachineCount == "" {
+		return false, fmt.Errorf("pinned status updatedMachineCount is empty in mcp %s", mcp.GetName())
+	}
+
+	return mcpMachineCount == pinnedMachineCount && updatedMachineCount == pinnedMachineCount && pinnedUnavailableMachineCount == "0", nil
+}
+
+// allNodesReportingPinnedSuccess returns true if all nodes in the pool are reporting pinned images success
+func (mcp *MachineConfigPool) allNodesReportingPinnedSuccess() (bool, error) {
+	allNodes, err := mcp.GetNodes()
+	if err != nil {
+		return false, err
+	}
+
+	if len(allNodes) == 0 {
+		logger.Infof("Warning, pool %s has no nodes!! We consider all nodes as correctly pinned", mcp.GetName())
+	}
+
+	for _, node := range allNodes {
+		nodeMCN := node.GetMachineConfigNode()
+		if nodeMCN.IsPinnedImageSetsDegraded() {
+			logger.Infof("Node %s is pinned degraded. Condition:\n%s", node.GetName(), nodeMCN.GetConditionByType("PinnedImageSetsDegraded"))
+			return false, nil
+		}
+
+		if nodeMCN.IsPinnedImageSetsProgressing() {
+			return false, nil
+		}
+	}
+
+	return true, nil
+}
+
+// waitForPinComplete waits for the MCP to complete pinning images
+func (mcp *MachineConfigPool) waitForPinComplete(timeToWait time.Duration) error {
+	logger.Infof("Waiting %s for MCP %s to complete pinned images.", timeToWait, mcp.name)
+
+	immediate := false
+	err := wait.PollUntilContextTimeout(context.TODO(), 1*time.Minute, timeToWait, immediate, func(_ context.Context) (bool, error) {
+		pinnedComplete, err := mcp.IsPinnedImagesComplete()
+		if err != nil {
+
+			logger.Infof("Error getting pinned complete: %s", err)
+			return false, err
+		}
+
+		if !pinnedComplete {
+			logger.Infof("Waiting for PinnedImageSets poolSynchronizersStatus status to repot success")
+			return false, nil
+		}
+
+		allNodesComplete, err := mcp.allNodesReportingPinnedSuccess()
+		if err != nil {
+			logger.Infof("Error getting if all nodes finished")
+			return false, err
+		}
+
+		if !allNodesComplete {
+			logger.Infof("Waiting for all nodes to report pinned images success")
+			return false, nil
+		}
+
+		logger.Infof("Pool %s successfully pinned the images! Complete!", mcp.GetName())
+		return true, nil
+	})
+
+	if err != nil {
+		logger.Infof("Pinned images operation is not completed on mcp %s", mcp.name)
+	}
+	return err
+}
+
+// GetPinnedImageSets returns a list with the nodes that match the .spec.nodeSelector.matchLabels criteria plus the provided extraLabels
+func (mcp *MachineConfigPool) GetPinnedImageSets() ([]*PinnedImageSet, error) {
+	mcp.oc.NotShowInfo()
+	defer mcp.oc.SetShowInfo()
+
+	labelsString, err := mcp.Get(`{.spec.machineConfigSelector.matchLabels}`)
+	if err != nil {
+		return nil, err
+	}
+
+	if labelsString == "" {
+		return nil, fmt.Errorf("No machineConfigSelector found in %s", mcp)
+	}
+
+	labels := gjson.Parse(labelsString)
+
+	requiredLabel := ""
+	labels.ForEach(func(key, value gjson.Result) bool {
+		requiredLabel += fmt.Sprintf("%s=%s,", key.String(), value.String())
+		return true // keep iterating
+	})
+
+	if requiredLabel == "" {
+		return nil, fmt.Errorf("No labels matcher could be built for %s", mcp)
+	}
+	// remove the last comma
+	requiredLabel = strings.TrimSuffix(requiredLabel, ",")
+
+	pisList := NewPinnedImageSetList(mcp.oc)
+	pisList.ByLabel(requiredLabel)
+
+	return pisList.GetAll()
+}