Merge pull request #1527 from Nordix/mquhuy/e2e-parallel-test

 🌱 Enable parallel testing in E2E

Makefile

@@ -48,11 +48,12 @@ export GOFLAGS=
 #
 GINKGO_FOCUS ?=
 GINKGO_SKIP ?=
-GINKGO_NODES ?= 1
+GINKGO_NODES ?= 2
 GINKGO_TIMEOUT ?= 2h
 GINKGO_POLL_PROGRESS_AFTER ?= 60m
 GINKGO_POLL_PROGRESS_INTERVAL ?= 5m
 E2E_CONF_FILE ?= $(ROOT_DIR)/test/e2e/config/fixture.yaml
+E2E_BMCS_CONF_FILE ?= $(ROOT_DIR)/test/e2e/config/bmcs-fixture.yaml
 USE_EXISTING_CLUSTER ?= false
 SKIP_RESOURCE_CLEANUP ?= false
 GINKGO_NOCOLOR ?= false
@@ -118,7 +119,8 @@ test-e2e: $(GINKGO) ## Run the end-to-end tests
 		-poll-progress-interval=$(GINKGO_POLL_PROGRESS_INTERVAL) --tags=e2e --focus="$(GINKGO_FOCUS)" \
 		$(_SKIP_ARGS) --nodes=$(GINKGO_NODES) --timeout=$(GINKGO_TIMEOUT) --no-color=$(GINKGO_NOCOLOR) \
 		--output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) test/e2e -- \
-		-e2e.config="$(E2E_CONF_FILE)" -e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER) \
+		-e2e.config="$(E2E_CONF_FILE)" -e2e.bmcsConfig="$(E2E_BMCS_CONF_FILE)" \
+		-e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER) \
 		-e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) -e2e.artifacts-folder="$(ARTIFACTS)"
 
 ## --------------------------------------

hack/ci-e2e.sh

@@ -38,6 +38,7 @@ export PATH="${PATH}:/usr/local/go/bin"
 "${REPO_ROOT}/hack/e2e/ensure_htpasswd.sh"
 # CAPI test framework uses kubectl in the background
 "${REPO_ROOT}/hack/e2e/ensure_kubectl.sh"
+"${REPO_ROOT}/hack/e2e/ensure_yq.sh"
 
 # Build the container image with e2e tag (used in tests)
 IMG=quay.io/metal3-io/baremetal-operator:e2e make docker
@@ -59,39 +60,20 @@ minikube start
 # Load the BMO e2e image into it
 minikube image load quay.io/metal3-io/baremetal-operator:e2e
 
-# Create libvirt domain
-VM_NAME="bmo-e2e-0"
-export BOOT_MAC_ADDRESS="00:60:2f:31:81:01"
-
-"${REPO_ROOT}/tools/bmh_test/create_vm.sh" "${VM_NAME}" "${BOOT_MAC_ADDRESS}"
-
 # This IP is defined by the network we created above.
 IP_ADDRESS="192.168.222.1"
 
-# These variables are used by the tests. They override variables in the config file.
-# These are the VBMC defaults (used since we did not specify anything else for `vbmc add`).
-export BMC_USER=admin
-export BMC_PASSWORD=password
-
 if [[ "${BMO_E2E_EMULATOR}" == "vbmc" ]]; then
-	# VBMC variables
-  VBMC_PORT="16230"
-  export BMC_ADDRESS="ipmi://${IP_ADDRESS}:${VBMC_PORT}"
-
   # Start VBMC
   docker run --name vbmc --network host -d \
     -v /var/run/libvirt/libvirt-sock:/var/run/libvirt/libvirt-sock \
     -v /var/run/libvirt/libvirt-sock-ro:/var/run/libvirt/libvirt-sock-ro \
     quay.io/metal3-io/vbmc
 
-  # Add BMH VM to VBMC
-  "${REPO_ROOT}/tools/bmh_test/vm2vbmc.sh" "${VM_NAME}" "${VBMC_PORT}"
 
 elif [[ "${BMO_E2E_EMULATOR}" == "sushy-tools" ]]; then
   # Sushy-tools variables
   SUSHY_EMULATOR_FILE="${REPO_ROOT}"/test/e2e/sushy-tools/sushy-emulator.conf
-  SUSHY_PORT="8000"
-  export BMC_ADDRESS="redfish+http://${IP_ADDRESS}:${SUSHY_PORT}/redfish/v1/Systems/${VM_NAME}"
 
   # Start sushy-tools
   docker run --name sushy-tools -d --network host \
@@ -105,6 +87,13 @@ else
   exit 1
 fi
 
+export E2E_BMCS_CONF_FILE="${REPO_ROOT}/test/e2e/config/bmcs-${BMO_E2E_EMULATOR}.yaml"
+"${REPO_ROOT}/hack/create_bmcs.sh" "${E2E_BMCS_CONF_FILE}" baremetal-e2e
+
+# Set the number of ginkgo processes to the number of BMCs
+n_vms=$(yq '. | length' "${E2E_BMCS_CONF_FILE}")
+export GINKGO_NODES="${n_vms}"
+
 # Image server variables
 CIRROS_VERSION="0.6.2"
 IMAGE_FILE="cirros-${CIRROS_VERSION}-x86_64-disk.img"
@@ -153,6 +142,7 @@ echo "IRONIC_HTPASSWD=$(htpasswd -n -b -B "${IRONIC_USERNAME}" "${IRONIC_PASSWOR
 echo "INSPECTOR_HTPASSWD=$(htpasswd -n -b -B "${IRONIC_INSPECTOR_USERNAME}" \
   "${IRONIC_INSPECTOR_PASSWORD}")" > "${IRONIC_OVERLAY}/ironic-inspector-htpasswd"
 
+
 # We need to gather artifacts/logs before exiting also if there are errors
 set +e
 

hack/clean-e2e.sh

@@ -7,11 +7,19 @@ minikube delete
 docker rm -f vbmc
 docker rm -f image-server-e2e
 docker rm -f sushy-tools
-virsh -c qemu:///system destroy --domain bmo-e2e-0
-virsh -c qemu:///system undefine --domain bmo-e2e-0 --nvram --remove-all-storage
+
+virsh_vms=$(virsh list --name --all)
+
+for vm in ${virsh_vms}; do
+  if [[ "${vm}" =~ "bmo-e2e-" ]]; then
+    virsh -c qemu:///system destroy --domain "${vm}"
+    virsh -c qemu:///system undefine --domain "${vm}" --nvram --remove-all-storage
+  fi
+done
+
 virsh -c qemu:///system net-destroy baremetal-e2e
 virsh -c qemu:///system net-undefine baremetal-e2e
 
 rm -rf "${REPO_ROOT}/test/e2e/_artifacts"
-rm -rf "${REPO_ROOT}/artifacts.tar.gz"
+rm -rf "${REPO_ROOT}"/artifacts-*
 rm -rf "${REPO_ROOT}/test/e2e/images"

hack/create_bmcs.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+#
+# This script reads BMC information in a config file and prepare VMs 
+# whose info match those config
+#
+set -eux
+
+REPO_ROOT=$(realpath "$(dirname "${BASH_SOURCE[0]}")/..")
+cd "${REPO_ROOT}"
+
+CONFIG_FILE=$1
+NETWORK=${2:-"baremetal-e2e"}
+
+readarray -t BMCS < <(yq e -o=j -I=0 '.[]' "${CONFIG_FILE}")
+
+for bmc in "${BMCS[@]}"; do
+  address=$(echo "${bmc}" | jq -r '.address')
+  bootMacAddress=$(echo "${bmc}" | jq -r '.bootMacAddress')
+  hostName=$(echo "${bmc}" | jq -r '.hostName')
+  ipAddress=$(echo "${bmc}" | jq -r '.ipAddress')
+
+  # Add the the VM to the network host list
+  virsh -c qemu:///system net-update "${NETWORK}" add-last ip-dhcp-host \
+    "<host mac='${bootMacAddress}' name='${hostName}' ip='${ipAddress}' />" \
+    --live --config --parent-index 0
+
+  # Create VM
+  "${REPO_ROOT}/tools/bmh_test/create_vm.sh" "${hostName}" "${bootMacAddress}"
+
+  # Add BMH VM to VBMC
+  if [[ "${address}" =~ "ipmi://" ]]; then
+    vbmc_port="${address##*:}"
+    "${REPO_ROOT}/tools/bmh_test/vm2vbmc.sh" "${hostName}" "${vbmc_port}"
+  fi
+done

hack/e2e/ensure_yq.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -eux
+
+USR_LOCAL_BIN="/usr/local/bin"
+YQ_VERSION="v4.40.5"
+
+# Check if yq tool is installed and install it if not
+verify_yq()
+{
+    if ! [[ -x "$(command -v yq)" ]]; then
+        if [[ "${OSTYPE}" == "linux-gnu" ]]; then
+            echo "yq not found, installing"
+            curl -LO "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64.tar.gz"
+            tar xvf yq_linux_amd64.tar.gz
+            sudo install yq_linux_amd64 "${USR_LOCAL_BIN}/yq"
+        else
+            echo "Missing required binary in path: yq"
+            return 2
+        fi
+    fi
+}
+
+verify_yq

hack/e2e/net.xml

@@ -10,7 +10,6 @@
     <dhcp>
       <range start='192.168.222.2' end='192.168.222.199'/>
       <host mac='52:54:00:6c:3c:01' name='minikube' ip='192.168.222.199'/>
-      <host mac='00:60:2f:31:81:01' name='bmo-e2e-0' ip='192.168.222.122'/>
       <bootp file='http://192.168.222.199:6180/boot.ipxe'/>
     </dhcp>
   </ip>

test/e2e/README.md

@@ -46,6 +46,14 @@ test suite with it:
 make test-e2e
 ```
 
+## BMCs config
+
+In case you want to run the tests with your own hardware, the information
+regarding BMCs should be stored in a yaml file, whose path is exported to
+`E2E_BMCS_CONF_FILE` variable (please take a look at
+[bmcs-sushy-tools.yaml](../../test/e2e/config/bmcs-sushy-tools.yaml)
+to understand the file structure).
+
 ## Tests
 
 Here is a list of the currently implemented tests. As mentioned above, we divide

test/e2e/basic_ops_test.go

@@ -19,25 +19,16 @@ import (
 
 var _ = Describe("basic", func() {
 	var (
-		specName       = "basic-ops"
-		secretName     = "bmc-credentials"
-		namespace      *corev1.Namespace
-		cancelWatches  context.CancelFunc
-		bmcUser        string
-		bmcPassword    string
-		bmcAddress     string
-		bootMacAddress string
+		specName      = "basic-ops"
+		secretName    = "bmc-credentials"
+		namespace     *corev1.Namespace
+		cancelWatches context.CancelFunc
 	)
 	const (
 		rebootAnnotation   = "reboot.metal3.io"
 		poweroffAnnotation = "reboot.metal3.io/poweroff"
 	)
 	BeforeEach(func() {
-		bmcUser = e2eConfig.GetVariable("BMC_USER")
-		bmcPassword = e2eConfig.GetVariable("BMC_PASSWORD")
-		bmcAddress = e2eConfig.GetVariable("BMC_ADDRESS")
-		bootMacAddress = e2eConfig.GetVariable("BOOT_MAC_ADDRESS")
-
 		namespace, cancelWatches = framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{
 			Creator:   clusterProxy.GetClient(),
 			ClientSet: clusterProxy.GetClientSet(),
@@ -49,8 +40,8 @@ var _ = Describe("basic", func() {
 	It("should control power cycle of BMH though annotations", func() {
 		By("creating a secret with BMH credentials")
 		bmcCredentialsData := map[string]string{
-			"username": bmcUser,
-			"password": bmcPassword,
+			"username": bmc.User,
+			"password": bmc.Password,
 		}
 		CreateSecret(ctx, clusterProxy.GetClient(), namespace.Name, secretName, bmcCredentialsData)
 
@@ -66,11 +57,11 @@ var _ = Describe("basic", func() {
 			Spec: metal3api.BareMetalHostSpec{
 				Online: true,
 				BMC: metal3api.BMCDetails{
-					Address:         bmcAddress,
+					Address:         bmc.Address,
 					CredentialsName: "bmc-credentials",
 				},
 				BootMode:       metal3api.Legacy,
-				BootMACAddress: bootMacAddress,
+				BootMACAddress: bmc.BootMacAddress,
 			},
 		}
 		err := clusterProxy.GetClient().Create(ctx, &bmh)

test/e2e/bmc.go

@@ -0,0 +1,37 @@
+package e2e
+
+import (
+	"os"
+
+	. "github.com/onsi/gomega"
+	"gopkg.in/yaml.v2"
+)
+
+// BMC defines a BMH to use in the tests
+type BMC struct {
+	// BMC initial username
+	User string `yaml:"user,omitempty"`
+	// BMC initial password
+	Password string `yaml:"password,omitempty"`
+	// BMC initial address
+	Address string `yaml:"address,omitempty"`
+	// BMC Mac address
+	BootMacAddress string `yaml:"bootMacAddress,omitempty"`
+	// The Hostname of the node, which will be read into BMH object
+	HostName string `yaml:"hostName,omitempty"`
+	// The IP address of the node
+	// Optional. Only needed if e2eConfig variable
+	// SSH_CHECK_PROVISIONED is true
+	IpAddress string `yaml:"ipAddress,omitempty"`
+	// Optional. Only needed if e2eConfig variable
+	// SSH_CHECK_PROVISIONED is true
+	SSHPort string `yaml:"sshPort,omitempty"`
+}
+
+func LoadBMCConfig(configPath string) *[]BMC {
+	configData, err := os.ReadFile(configPath) //#nosec
+	Expect(err).ToNot(HaveOccurred(), "Failed to read the bmcs config file")
+	var bmcs []BMC
+	Expect(yaml.Unmarshal(configData, &bmcs)).To(Succeed())
+	return &bmcs
+}

test/e2e/common.go

@@ -391,13 +391,10 @@ echo "%s" >> /root/.ssh/authorized_keys`, sshPubKeyData, sshPubKeyData)
 // PerformSSHBootCheck performs an SSH check to verify the node's boot source.
 // The `expectedBootMode` parameter should be "disk" or "memory".
 // The `auth` parameter is an ssh.AuthMethod for authentication.
-func PerformSSHBootCheck(e2eConfig *Config, expectedBootMode string, auth ssh.AuthMethod) {
-	ip := e2eConfig.GetVariable("IP_BMO_E2E_0")
-	sshPort := e2eConfig.GetVariable("SSH_PORT")
-	address := fmt.Sprintf("%s:%s", ip, sshPort)
+func PerformSSHBootCheck(e2eConfig *Config, expectedBootMode string, auth ssh.AuthMethod, sshAddress string) {
 	user := e2eConfig.GetVariable("CIRROS_USERNAME")
 
-	client := EstablishSSHConnection(e2eConfig, auth, user, address)
+	client := EstablishSSHConnection(e2eConfig, auth, user, sshAddress)
 	defer func() {
 		if client != nil {
 			client.Close()

test/e2e/config/bmcs-fixture.yaml

@@ -0,0 +1,16 @@
+# For fixture config, only the `hostName` field matters. The number of BMCs,
+# however, needs to be equal or larger than `GINKGO_NODES`.
+- user: admin
+  password: password
+  address: "redfish+http://192.168.222.1:8000/redfish/v1/Systems/bmo-e2e-0"
+  bootMacAddress: "00:60:2f:31:81:01"
+  hostName: ""
+  ipAddress: "192.168.222.122"
+  sshPort: "22"
+- user: admin
+  password: password
+  address: "redfish+http://192.168.222.1:8000/redfish/v1/Systems/bmo-e2e-1"
+  bootMacAddress: "00:60:2f:31:81:02"
+  hostName: ""
+  ipAddress: "192.168.222.123"
+  sshPort: "22"

test/e2e/config/bmcs-sushy-tools.yaml

@@ -0,0 +1,14 @@
+- user: admin
+  password: password
+  address: "redfish+http://192.168.222.1:8000/redfish/v1/Systems/bmo-e2e-0"
+  bootMacAddress: "00:60:2f:31:81:01"
+  hostName: "bmo-e2e-0"
+  ipAddress: "192.168.222.122"
+  sshPort: "22"
+- user: admin
+  password: password
+  address: "redfish+http://192.168.222.1:8000/redfish/v1/Systems/bmo-e2e-1"
+  bootMacAddress: "00:60:2f:31:81:02"
+  hostName: "bmo-e2e-1"
+  ipAddress: "192.168.222.123"
+  sshPort: "22"

test/e2e/config/bmcs-vbmc.yaml

@@ -0,0 +1,14 @@
+- user: admin
+  password: password
+  address: "ipmi://192.168.222.1:16230"
+  bootMacAddress: "00:60:2f:31:81:01"
+  hostName: "bmo-e2e-0"
+  ipAddress: "192.168.222.122"
+  sshPort: "22"
+- user: admin
+  password: password
+  address: "ipmi://192.168.222.1:16231"
+  bootMacAddress: "00:60:2f:31:81:02"
+  hostName: "bmo-e2e-1"
+  ipAddress: "192.168.222.123"
+  sshPort: "22"

test/e2e/config/fixture.yaml

@@ -25,13 +25,6 @@ variables:
   UPGRADE_DEPLOY_CERT_MANAGER: "true"
   UPGRADE_BMO_KUSTOMIZATION_FROM: "../../config/overlays/fixture-release-0.4"
 
-  BMC_ADDRESS: ipmi://192.168.222.1:16230
-  BOOT_MAC_ADDRESS: "00:60:2f:31:81:01"
-  EXPECTED_HOST_NAME: ""
-  # Test credentials. The tests will create a BMH with these.
-  # For the fixture provider it does not matter much what we put here.
-  BMC_USER: admin
-  BMC_PASSWORD: password
   IMAGE_URL: "http://192.168.222.1/cirros-0.6.2-x86_64-disk.img"
   IMAGE_CHECKSUM: "c8fc807773e5354afe61636071771906"
   CERT_MANAGER_VERSION: "v1.13.1"
@@ -45,13 +38,13 @@ intervals:
   inspection/wait-available: ["5s", "1ms"]
   external-inspection/wait-available: ["5s", "1ms"]
   default/wait-deployment: ["5m", "1s"]
-  default/wait-namespace-deleted: ["20s", "1s"]
+  default/wait-namespace-deleted: ["1m", "1s"]
   ironic/wait-deployment: ["10m", "2s"]
   default/wait-registration: ["5s", "5ms"]
-  default/wait-provisioning: ["30s", "10ms"]
-  default/wait-provisioned: ["1m", "1s"]
+  default/wait-provisioning: ["1m", "10ms"]
+  default/wait-provisioned: ["5m", "1s"]
   default/wait-deprovisioning: ["5s", "10ms"]
-  default/wait-available: ["2m", "1s"]
+  default/wait-available: ["5m", "1s"]
   default/wait-deleting: ["5s", "10ms"]
   default/wait-deleted: ["5s", "10ms"]
   default/wait-secret-deletion: ["5s", "10ms"]