Merge pull request #1520 from Nordix/lentzi90/bmo-e2e-virtualmedia

🌱 Test virtualmedia booting

hack/ci-e2e.sh

@@ -4,6 +4,10 @@
 # Description: This script sets up the environment and runs E2E tests for the
 #              BMO project. It uses either vbmc or sushy-tools based on
 #              the BMO_E2E_EMULATOR environment variable.
+#              With sushy-tools, it is also possible to choose between
+#              redfish-virtualmedia and redfish protocols using the
+#              SUSHY_TOOLS_PROTOCOL environment variable.
+#              By default, sushy-tools and redfish-virtualmedia will be used.
 # Usage:       export BMO_E2E_EMULATOR="vbmc"  # Or "sushy-tools"
 #              ./ci-e2e.sh
 # -----------------------------------------------------------------------------
@@ -16,6 +20,8 @@ cd "${REPO_ROOT}" || exit 1
 
 # BMO_E2E_EMULATOR can be set to either "vbmc" or "sushy-tools"
 BMO_E2E_EMULATOR=${BMO_E2E_EMULATOR:-"sushy-tools"}
+# We can choose to use redfish-virtualmedia or redfish as the protocol when using sushy-tools
+SUSHY_TOOLS_PROTOCOL=${SUSHY_TOOLS_PROTOCOL:-"redfish-virtualmedia"}
 
 export E2E_CONF_FILE="${REPO_ROOT}/test/e2e/config/ironic.yaml"
 
@@ -68,6 +74,8 @@ rm /tmp/bmo-e2e.tar
 IP_ADDRESS="192.168.222.1"
 
 if [[ "${BMO_E2E_EMULATOR}" == "vbmc" ]]; then
+  BMC_PROTOCOL="ipmi"
+  export E2E_BMCS_CONF_FILE="${REPO_ROOT}/test/e2e/config/bmcs-ipmi.yaml"
   # Start VBMC
   docker run --name vbmc --network host -d \
     -v /var/run/libvirt/libvirt-sock:/var/run/libvirt/libvirt-sock \
@@ -76,6 +84,8 @@ if [[ "${BMO_E2E_EMULATOR}" == "vbmc" ]]; then
 
 
 elif [[ "${BMO_E2E_EMULATOR}" == "sushy-tools" ]]; then
+  BMC_PROTOCOL=${SUSHY_TOOLS_PROTOCOL}
+  export E2E_BMCS_CONF_FILE="${REPO_ROOT}/test/e2e/config/bmcs-${SUSHY_TOOLS_PROTOCOL}.yaml"
   # Sushy-tools variables
   SUSHY_EMULATOR_FILE="${REPO_ROOT}"/test/e2e/sushy-tools/sushy-emulator.conf
 
@@ -91,7 +101,6 @@ else
   exit 1
 fi
 
-export E2E_BMCS_CONF_FILE="${REPO_ROOT}/test/e2e/config/bmcs-${BMO_E2E_EMULATOR}.yaml"
 "${REPO_ROOT}/hack/create_bmcs.sh" "${E2E_BMCS_CONF_FILE}" baremetal-e2e
 
 # Set the number of ginkgo processes to the number of BMCs
@@ -106,16 +115,42 @@ export IMAGE_URL="http://${IP_ADDRESS}/${IMAGE_FILE}"
 IMAGE_DIR="${REPO_ROOT}/test/e2e/images"
 mkdir -p "${IMAGE_DIR}"
 
-## Download and run image server
-wget --quiet -P "${IMAGE_DIR}"/ https://artifactory.nordix.org/artifactory/metal3/images/iso/"${IMAGE_FILE}"
+## Download disk images
+wget --quiet -P "${IMAGE_DIR}/" https://artifactory.nordix.org/artifactory/metal3/images/iso/"${IMAGE_FILE}"
+wget --quiet -P "${IMAGE_DIR}/" https://fastly-cdn.system-rescue.org/releases/11.00/systemrescue-11.00-amd64.iso
 
+## Start the image server
 docker run --name image-server-e2e -d \
   -p 80:8080 \
   -v "${IMAGE_DIR}:/usr/share/nginx/html" nginxinc/nginx-unprivileged
 
-# Generate the key pair
+# Generate ssh key pair for verifying provisioned BMHs
 ssh-keygen -t ed25519 -f "${IMAGE_DIR}/ssh_testkey" -q -N ""
 
+# Build an ISO image with baked ssh key
+# See https://www.system-rescue.org/scripts/sysrescue-customize/
+# We use the systemrescue ISO and their script for customizing it.
+pushd "${IMAGE_DIR}"
+wget -O sysrescue-customize https://gitlab.com/systemrescue/systemrescue-sources/-/raw/main/airootfs/usr/share/sysrescue/bin/sysrescue-customize?inline=false
+chmod +x sysrescue-customize
+
+pub_ssh_key=$(cut -d " " -f "1,2" "ssh_testkey.pub")
+
+mkdir -p recipe/iso_add/sysrescue.d
+# Reference: https://www.system-rescue.org/manual/Configuring_SystemRescue/
+cat << EOF > recipe/iso_add/sysrescue.d/90-config.yaml
+---
+global:
+    nofirewall: true
+sysconfig:
+    authorized_keys:
+        "[email protected]": "${pub_ssh_key}"
+EOF
+
+./sysrescue-customize --auto --recipe-dir recipe --source systemrescue-11.00-amd64.iso --dest=sysrescue-out.iso
+export ISO_IMAGE_URL="http://${IP_ADDRESS}/sysrescue-out.iso"
+popd
+
 # Generate credentials
 BMO_OVERLAYS=("${REPO_ROOT}/config/overlays/e2e" "${REPO_ROOT}/config/overlays/e2e-release-0.4" "${REPO_ROOT}/config/overlays/e2e-release-0.5")
 IRONIC_OVERLAY="${REPO_ROOT}/ironic-deployment/overlays/e2e"
@@ -157,6 +192,6 @@ sudo sh -c "cp -r /var/log/libvirt/qemu/* ${LOGS_DIR}/qemu/"
 sudo chown -R "${USER}:${USER}" "${LOGS_DIR}/qemu"
 
 # Collect all artifacts
-tar --directory test/e2e/ -czf "artifacts-e2e-${BMO_E2E_EMULATOR}.tar.gz" _artifacts
+tar --directory test/e2e/ -czf "artifacts-e2e-${BMO_E2E_EMULATOR}-${BMC_PROTOCOL}.tar.gz" _artifacts
 
 exit "${test_status}"

test/e2e/common.go

@@ -202,8 +202,9 @@ func HasRootOnDisk(output string) bool {
 			continue // Skip malformed lines
 		}
 
-		if fields[5] == "/" && !strings.Contains(fields[0], "tmpfs") {
-			return true // Found a non-tmpfs root filesystem
+		// When booting from memory or live-ISO we can have root on tmpfs or airootfs
+		if fields[5] == "/" && !(strings.Contains(fields[0], "tmpfs") || strings.Contains(fields[0], "airootfs")) {
+			return true
 		}
 	}
 
@@ -257,12 +258,8 @@ func createCirrosInstanceAndHostnameUserdata(ctx context.Context, client client.
 
 	userDataContent := fmt.Sprintf(`#!/bin/sh
 mkdir /root/.ssh
-mkdir /home/cirros/.ssh
 chmod 700 /root/.ssh
-chmod 700 /home/cirros/.ssh
-chown cirros /home/cirros/.ssh
-echo "%s" >> /home/cirros/.ssh/authorized_keys
-echo "%s" >> /root/.ssh/authorized_keys`, sshPubKeyData, sshPubKeyData)
+echo "%s" >> /root/.ssh/authorized_keys`, sshPubKeyData)
 
 	CreateSecret(ctx, client, namespace, secretName, map[string]string{"userData": userDataContent})
 }
@@ -271,7 +268,7 @@ echo "%s" >> /root/.ssh/authorized_keys`, sshPubKeyData, sshPubKeyData)
 // The `expectedBootMode` parameter should be "disk" or "memory".
 // The `auth` parameter is an ssh.AuthMethod for authentication.
 func PerformSSHBootCheck(e2eConfig *Config, expectedBootMode string, auth ssh.AuthMethod, sshAddress string) {
-	user := e2eConfig.GetVariable("CIRROS_USERNAME")
+	user := e2eConfig.GetVariable("SSH_USERNAME")
 
 	client := EstablishSSHConnection(e2eConfig, auth, user, sshAddress)
 	defer func() {

test/e2e/config/bmcs-redfish-virtualmedia.yaml

@@ -0,0 +1,14 @@
+- user: admin
+  password: password
+  address: "redfish-virtualmedia+http://192.168.222.1:8000/redfish/v1/Systems/bmo-e2e-0"
+  bootMacAddress: "00:60:2f:31:81:01"
+  hostName: "bmo-e2e-0"
+  ipAddress: "192.168.222.122"
+  sshPort: "22"
+- user: admin
+  password: password
+  address: "redfish-virtualmedia+http://192.168.222.1:8000/redfish/v1/Systems/bmo-e2e-1"
+  bootMacAddress: "00:60:2f:31:81:02"
+  hostName: "bmo-e2e-1"
+  ipAddress: "192.168.222.123"
+  sshPort: "22"

test/e2e/config/fixture.yaml

@@ -26,6 +26,7 @@ variables:
   UPGRADE_BMO_KUSTOMIZATION_FROM: "../../config/overlays/fixture-release-0.5"
 
   IMAGE_URL: "http://192.168.222.1/cirros-0.6.2-x86_64-disk.img"
+  ISO_IMAGE_URL: "http://192.168.222.1/cirros.iso"
   IMAGE_CHECKSUM: "c8fc807773e5354afe61636071771906"
   CERT_MANAGER_VERSION: "v1.13.1"
   SSH_CHECK_PROVISIONED: "false"

test/e2e/config/ironic.yaml

@@ -30,11 +30,11 @@ variables:
   UPGRADE_BMO_KUSTOMIZATION_FROM: "../../config/overlays/e2e-release-0.5"
 
   IMAGE_URL: "http://192.168.222.1/cirros-0.6.2-x86_64-disk.img"
+  ISO_IMAGE_URL: "http://192.168.222.1/minimal_linux_live-v2.iso"
   IMAGE_CHECKSUM: "c8fc807773e5354afe61636071771906"
   CERT_MANAGER_VERSION: "v1.13.1"
   SSH_CHECK_PROVISIONED: "true"
-  CIRROS_USERNAME: "cirros"
-  CIRROS_PASSWORD: "gocubsgo"
+  SSH_USERNAME: "root"
   SSH_PRIV_KEY: "./images/ssh_testkey"
   SSH_PUB_KEY: "./images/ssh_testkey.pub"
 

test/e2e/live_iso_test.go

@@ -3,6 +3,7 @@ package e2e
 import (
 	"context"
 	"fmt"
+	"os"
 
 	"golang.org/x/crypto/ssh"
 
@@ -16,6 +17,7 @@ import (
 	"sigs.k8s.io/cluster-api/util/patch"
 
 	metal3api "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1"
+	metal3bmc "github.com/metal3-io/baremetal-operator/pkg/hardwareutils/bmc"
 
 	capm3_e2e "github.com/metal3-io/cluster-api-provider-metal3/test/e2e"
 )
@@ -30,7 +32,17 @@ var _ = Describe("Live-ISO", Label("required", "live-iso"), func() {
 	)
 
 	BeforeEach(func() {
-		imageURL = e2eConfig.GetVariable("IMAGE_URL")
+		// Check what kind of BMC we are dealing with
+		// It may be *possible* to boot a live-ISO over (i)PXE, but there are severe limitations.
+		// Therefore we skip the test if it doesn't support ISO preprovisioning images.
+		// See https://docs.openstack.org/ironic/latest/admin/ramdisk-boot.html
+		accessDetails, err := metal3bmc.NewAccessDetails(bmc.Address, false)
+		Expect(err).NotTo(HaveOccurred())
+		if !accessDetails.SupportsISOPreprovisioningImage() {
+			Skip(fmt.Sprintf("BMC does not support ISO images. It does not make sense to test live-ISO here. BMC address: %s", bmc.Address))
+		}
+
+		imageURL = e2eConfig.GetVariable("ISO_IMAGE_URL")
 
 		namespace, cancelWatches = framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{
 			Creator:   clusterProxy.GetClient(),
@@ -68,8 +80,9 @@ var _ = Describe("Live-ISO", Label("required", "live-iso"), func() {
 					URL:        imageURL,
 					DiskFormat: pointer.String("live-iso"),
 				},
-				BootMode:       metal3api.Legacy,
-				BootMACAddress: bmc.BootMacAddress,
+				BootMode:              metal3api.Legacy,
+				BootMACAddress:        bmc.BootMacAddress,
+				AutomatedCleaningMode: metal3api.CleaningModeDisabled,
 			},
 		}
 		err := clusterProxy.GetClient().Create(ctx, &bmh)
@@ -92,8 +105,12 @@ var _ = Describe("Live-ISO", Label("required", "live-iso"), func() {
 		// The ssh check is not possible in all situations (e.g. fixture) so it can be skipped
 		if e2eConfig.GetVariable("SSH_CHECK_PROVISIONED") == "true" {
 			By("Verifying the node booted from live ISO image")
-			password := e2eConfig.GetVariable("CIRROS_PASSWORD")
-			auth := ssh.Password(password)
+			keyPath := e2eConfig.GetVariable("SSH_PRIV_KEY")
+			key, err := os.ReadFile(keyPath)
+			Expect(err).NotTo(HaveOccurred(), "unable to read private key")
+			signer, err := ssh.ParsePrivateKey(key)
+			Expect(err).NotTo(HaveOccurred(), "unable to parse private key")
+			auth := ssh.PublicKeys(signer)
 			PerformSSHBootCheck(e2eConfig, "memory", auth, fmt.Sprintf("%s:%s", bmc.IPAddress, bmc.SSHPort))
 		} else {
 			capm3_e2e.Logf("WARNING: Skipping SSH check since SSH_CHECK_PROVISIONED != true")

test/go.mod

@@ -5,6 +5,7 @@ go 1.21
 require (
 	github.com/cert-manager/cert-manager v1.10.0
 	github.com/metal3-io/baremetal-operator/apis v0.5.1
+	github.com/metal3-io/baremetal-operator/pkg/hardwareutils v0.5.1
 	github.com/metal3-io/cluster-api-provider-metal3/test v1.6.1
 	github.com/onsi/ginkgo/v2 v2.13.2
 	github.com/onsi/gomega v1.30.0
@@ -79,7 +80,6 @@ require (
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
-	github.com/metal3-io/baremetal-operator/pkg/hardwareutils v0.5.1 // indirect
 	github.com/metal3-io/cluster-api-provider-metal3/api v1.5.1 // indirect
 	github.com/metal3-io/ip-address-manager/api v1.6.2 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect

tools/bmh_test/clean_local_bmh_test_setup.sh

@@ -10,15 +10,15 @@ if [[ -n "${VM_LIST}" ]]; then
     # Loop through the list and delete each virtual machine
     for vm_name in ${VM_LIST}; do
         virsh -c qemu:///system destroy --domain "${vm_name}"
-        virsh -c qemu:///system undefine --domain "${vm_name}" --remove-all-storage
+        virsh -c qemu:///system undefine --domain "${vm_name}" --remove-all-storage --nvram
         kubectl delete baremetalhost "${vm_name}"
     done
 else
     echo "No virtual machines found. Skipping..."
 fi
 
 # Clear vbmc
-docker rm -f vbmc 
+docker rm -f vbmc
 
 # Clear network
 virsh -c qemu:///system net-destroy baremetal-e2e