diff --git a/.github/workflows/e2e-fixture-test.yml b/.github/workflows/e2e-fixture-test.yml index 22e7551cde..76e4670c86 100644 --- a/.github/workflows/e2e-fixture-test.yml +++ b/.github/workflows/e2e-fixture-test.yml @@ -22,6 +22,11 @@ jobs: with: go-version: ${{ steps.vars.outputs.go_version }} + - name: Install libvirt + run: | + sudo apt-get update + sudo apt-get install -y libvirt-dev + - name: Build BMO e2e Docker Image env: IMG: quay.io/metal3-io/baremetal-operator:e2e diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index fd001e93a8..5c32b88896 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -31,7 +31,7 @@ jobs: - name: Install libvirt run: | sudo apt-get update - sudo apt-get install -y libvirt-daemon-system qemu-kvm virt-manager + sudo apt-get install -y libvirt-daemon-system qemu-kvm virt-manager libvirt-dev - name: Run BMO e2e Tests env: diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 08a022bfe3..57da717b87 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -20,6 +20,12 @@ jobs: - pkg/hardwareutils steps: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + - name: Install libvirt + run: | + sudo apt-get update + sudo apt-get install -y libvirt-dev + - name: Calculate go version id: vars run: echo "go_version=$(make go-version)" >> $GITHUB_OUTPUT diff --git a/clean_bmcs.sh b/clean_bmcs.sh new file mode 100755 index 0000000000..21a81068ab --- /dev/null +++ b/clean_bmcs.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# +# This script reads BMC information in a config file and prepare VMs +# whose info match those config +# +set -x + +REPO_ROOT=$(realpath "$(dirname "${BASH_SOURCE[0]}")") + +virsh pool-destroy default || true +virsh pool-delete default || true +virsh pool-undefine default || true + +CONFIG_FILE=$1 +NETWORK=${2:-"baremetal-e2e"} + +readarray -t BMCS < <(yq e -o=j -I=0 '.[]' "${CONFIG_FILE}") + +for bmc in "${BMCS[@]}"; do + bootMacAddress=$(echo "${bmc}" | jq -r '.bootMacAddress') + ipAddress=$(echo "${bmc}" | jq -r '.ipAddress') + virsh -c qemu:///system net-update "${NETWORK}" delete ip-dhcp-host "" --live --config +done +"${REPO_ROOT}/tools/bmh_test/clean_local_bmh_test_setup.sh" "^bmo-e2e-" +rm -rf /tmp/bmo-e2e-* +rm -rf /tmp/pool_oo diff --git a/hack/ci-e2e.sh b/hack/ci-e2e.sh index 55da95ff50..795a6d9988 100755 --- a/hack/ci-e2e.sh +++ b/hack/ci-e2e.sh @@ -82,6 +82,10 @@ rm /tmp/bmo-e2e.tar # This IP is defined by the network we created above. IP_ADDRESS="192.168.222.1" +pushd "${REPO_ROOT}/test/createVM" || exit 1 +go run main.go --yaml-source-file "${E2E_BMCS_CONF_FILE}" +popd + if [[ "${BMO_E2E_EMULATOR}" == "vbmc" ]]; then # Start VBMC docker run --name vbmc --network host -d \ @@ -89,6 +93,13 @@ if [[ "${BMO_E2E_EMULATOR}" == "vbmc" ]]; then -v /var/run/libvirt/libvirt-sock-ro:/var/run/libvirt/libvirt-sock-ro \ quay.io/metal3-io/vbmc + readarray -t BMCS < <(yq e -o=j -I=0 '.[]' "${E2E_BMCS_CONF_FILE}") + for bmc in "${BMCS[@]}"; do + address=$(echo "${bmc}" | jq -r '.address') + hostName=$(echo "${bmc}" | jq -r '.hostName') + vbmc_port="${address##*:}" + "${REPO_ROOT}/tools/bmh_test/vm2vbmc.sh" "${hostName}" "${vbmc_port}" + done elif [[ "${BMO_E2E_EMULATOR}" == "sushy-tools" ]]; then # Sushy-tools variables @@ -105,8 +116,6 @@ else exit 1 fi -"${REPO_ROOT}/hack/create_bmcs.sh" "${E2E_BMCS_CONF_FILE}" baremetal-e2e - # Image server variables CIRROS_VERSION="0.6.2" IMAGE_FILE="cirros-${CIRROS_VERSION}-x86_64-disk.img" @@ -118,6 +127,7 @@ mkdir -p "${IMAGE_DIR}" ## Download disk images wget --quiet -P "${IMAGE_DIR}/" https://artifactory.nordix.org/artifactory/metal3/images/iso/"${IMAGE_FILE}" wget --quiet -P "${IMAGE_DIR}/" https://fastly-cdn.system-rescue.org/releases/11.00/systemrescue-11.00-amd64.iso +wget --quiet -P "${IMAGE_DIR}/" https://artifactory.nordix.org/artifactory/metal3/images/iso/minimal_linux_live-v2.iso ## Start the image server docker run --name image-server-e2e -d \ diff --git a/hack/clean-e2e.sh b/hack/clean-e2e.sh index 1a36c5f55c..17c8f7f010 100755 --- a/hack/clean-e2e.sh +++ b/hack/clean-e2e.sh @@ -13,3 +13,15 @@ docker rm -f sushy-tools rm -rf "${REPO_ROOT}/test/e2e/_artifacts" rm -rf "${REPO_ROOT}"/artifacts-* rm -rf "${REPO_ROOT}/test/e2e/images" + +# Clear network +virsh -c qemu:///system net-destroy baremetal-e2e +virsh -c qemu:///system net-undefine baremetal-e2e + +# Clean volume pool directory +rm -rf /tmp/pool_oo/* + +# Clean volume pool +virsh pool-destroy default || true +virsh pool-delete default || true +virsh pool-undefine default || true diff --git a/hack/create_bmcs.sh b/hack/create_bmcs.sh deleted file mode 100755 index d8a3aee3f4..0000000000 --- a/hack/create_bmcs.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash -# -# This script reads BMC information in a config file and prepare VMs -# whose info match those config -# -set -eux - -REPO_ROOT=$(realpath "$(dirname "${BASH_SOURCE[0]}")/..") -cd "${REPO_ROOT}" - -CONFIG_FILE=$1 -NETWORK=${2:-"baremetal-e2e"} - -readarray -t BMCS < <(yq e -o=j -I=0 '.[]' "${CONFIG_FILE}") - -for bmc in "${BMCS[@]}"; do - address=$(echo "${bmc}" | jq -r '.address') - bootMacAddress=$(echo "${bmc}" | jq -r '.bootMacAddress') - hostName=$(echo "${bmc}" | jq -r '.hostName') - ipAddress=$(echo "${bmc}" | jq -r '.ipAddress') - - # Add the the VM to the network host list - virsh -c qemu:///system net-update "${NETWORK}" add-last ip-dhcp-host \ - "" \ - --live --config --parent-index 0 - - # Create VM - "${REPO_ROOT}/tools/bmh_test/create_vm.sh" "${hostName}" "${bootMacAddress}" - - # Add BMH VM to VBMC - if [[ "${address}" =~ "ipmi://" ]]; then - vbmc_port="${address##*:}" - "${REPO_ROOT}/tools/bmh_test/vm2vbmc.sh" "${hostName}" "${vbmc_port}" - fi -done diff --git a/test/createVM/main.go b/test/createVM/main.go new file mode 100644 index 0000000000..afffb8e5a6 --- /dev/null +++ b/test/createVM/main.go @@ -0,0 +1,295 @@ +package main + +import ( + "bytes" + "embed" + "flag" + "fmt" + "os" + "text/template" + + "github.com/dypflying/go-qcow2lib/qcow2" + bmoe2e "github.com/metal3-io/baremetal-operator/test/e2e" + "libvirt.org/go/libvirt" +) + +var ( + //go:embed templates/*.tpl + templateFiles embed.FS +) + +func RenderTemplate(inputFile string, data interface{}) (string, error) { + tmpl, err := template.ParseFS(templateFiles, inputFile) + if err != nil { + return "", err + } + + var buf bytes.Buffer + + if err = tmpl.Execute(&buf, data); err != nil { + return "", err + } + + return buf.String(), nil +} + +// CreateVolumePool creates a volume pool with specified name if a pool with +// that name does not exist yet. +func CreateVolumePool(poolName, poolPath string) (*libvirt.StoragePool, error) { + // Connect to libvirt daemon + conn, err := libvirt.NewConnect("qemu:///system") + if err != nil { + fmt.Println("Failed to connect to qemu:///system") + return nil, err + } + defer conn.Close() + + pool, err := conn.LookupStoragePoolByName(poolName) + + if err == nil { + fmt.Println("Pool already exists") + return pool, nil + } + + if err = os.Mkdir(poolPath, 0777); err != nil && !os.IsExist(err) { + fmt.Println("Cannot determine the state of the poolPath") + return nil, err + } + + data := struct { + PoolName string + PoolPath string + }{ + PoolName: poolName, + PoolPath: poolPath, + } + + poolCfg, err := RenderTemplate("templates/pool.xml.tpl", data) + + if err != nil { + fmt.Println("Failed to read pool XML file") + fmt.Printf("Error occurred: %v\n", err) + return nil, err + } + + // Create the volume pool + pool, err = conn.StoragePoolDefineXML(poolCfg, 0) + + if err != nil { + fmt.Println("Failed to create volume pool") + fmt.Printf("Error occurred: %v\n", err) + return nil, err + } + + if err = pool.SetAutostart(true); err != nil { + fmt.Println("Failed to Set the pool autostart") + fmt.Printf("Error occurred: %v\n", err) + return nil, err + } + + if err = pool.Create(0); err != nil { + fmt.Println("Failed to Start the pool") + fmt.Printf("Error occurred: %v\n", err) + return nil, err + } + + fmt.Println("Volume pool created successfully") + return pool, nil +} + +func CreateVolume(volumeName, poolName, poolPath string, capacityInGB int) error { + // Connect to libvirt daemon + conn, err := libvirt.NewConnect("qemu:///system") + if err != nil { + fmt.Println("Failed to connect to qemu:///system") + return err + } + defer conn.Close() + + pool, err := CreateVolumePool(poolName, poolPath) + + if err != nil { + fmt.Println("Failed to create storage pool") + fmt.Printf("Error occurred: %v\n", err) + return err + } + + data := struct { + VolumeName string + VolumeCapacityInGB int + }{ + VolumeName: volumeName, + VolumeCapacityInGB: capacityInGB, + } + + volumeCfg, err := RenderTemplate("templates/volume.xml.tpl", data) + + if err != nil { + fmt.Println("Failed to read volume XML file") + fmt.Printf("Error occurred: %v\n", err) + return err + } + + // Create the volume + _, err = pool.StorageVolCreateXML(volumeCfg, 0) + + if err != nil { + fmt.Println("Failed to create volume") + fmt.Printf("Error occurred: %v\n", err) + return err + } + + fmt.Println("Volume created successfully") + return nil +} + +func CreateLibvirtVM(hostName, networkName, macAddress string) error { + poolName := "default" + poolPath := "/tmp/pool_oo" + opts := make(map[string]any) + opts[qcow2.OPT_SIZE] = 3 * (1 << 30) // qcow2 file's size is 3g + opts[qcow2.OPT_FMT] = "qcow2" // qcow2 format + opts[qcow2.OPT_SUBCLUSTER] = true // enable sub-cluster + + err := qcow2.Blk_Create("/tmp/"+hostName+".qcow2", opts) + + if err != nil { + fmt.Println("Failed to create qcow2 file") + fmt.Printf("Error occurred: %v\n", err) + return err + } + + if err = CreateVolume(hostName, poolName, poolPath, 20); err != nil { + return err + } + + conn, err := libvirt.NewConnect("qemu:///system") + if err != nil { + fmt.Println("Failed to connect to qemu:///system") + return err + } + defer conn.Close() + + data := struct { + HostName string + Network string + MacAddress string + PoolPath string + }{ + HostName: hostName, + Network: networkName, + MacAddress: macAddress, + PoolPath: poolPath, + } + + vmCfg, err := RenderTemplate("templates/VM.xml.tpl", data) + + if err != nil { + return err + } + + dom, err := conn.DomainDefineXML(vmCfg) + + if err != nil { + fmt.Println("Failed to define domain") + fmt.Printf("Error occurred: %v\n", err) + return err + } + + if err := dom.Create(); err != nil { + fmt.Println("Failed to create domain") + fmt.Printf("Error occurred: %v\n", err) + return err + } + + fmt.Println("Domain created successfully") + return nil +} + +func CreateLibvirtBMC(macAddress, hostName, ipAddress, networkName string) error { + var err error + conn, err := libvirt.NewConnect("qemu:///system") + if err != nil { + return err + } + defer conn.Close() + + network, err := conn.LookupNetworkByName(networkName) + if err != nil { + return err + } + + xmlTpl, err := template.New("xml").Parse("") + + if err != nil { + return err + } + + data := struct { + MacAddress string + HostName string + IPAddress string + }{ + MacAddress: macAddress, + HostName: hostName, + IPAddress: ipAddress, + } + + var buf bytes.Buffer + + err = xmlTpl.Execute(&buf, data) + + if err != nil { + fmt.Println("Failed to create BMC") + fmt.Printf("Error occurred: %v\n", err) + return err + } + + if err = network.Update( + libvirt.NETWORK_UPDATE_COMMAND_ADD_LAST, + libvirt.NETWORK_SECTION_IP_DHCP_HOST, + -1, + buf.String(), + libvirt.NETWORK_UPDATE_AFFECT_LIVE|libvirt.NETWORK_UPDATE_AFFECT_CONFIG, + ); err != nil { + fmt.Printf("Error occurred: %v\n", err) + return err + } + if err = CreateLibvirtVM(hostName, networkName, macAddress); err != nil { + fmt.Printf("Error occurred: %v\n", err) + return err + } + return nil +} + +func main() { + var vmName = flag.String( + "vm-name", "VM-1", "The name of the VM to create") + var networkName = flag.String( + "network-name", "baremetal-e2e", "The name of the network that the new VM should be attached to") + var macAddress = flag.String( + "mac-address", "00:60:2f:31:81:01", "Mac address of the VM on the network") + var ipAddress = flag.String( + "ip-address", "192.168.222.122", "IP address of the VM on the network") + var configFile = flag.String( + "yaml-source-file", "", "yaml file where BMCS are defined. If this is set, ignore all other options") + flag.Parse() + var err error + if *configFile == "" { + if err = CreateLibvirtBMC(*macAddress, *vmName, *ipAddress, *networkName); err != nil { + fmt.Printf("Error occurred: %v\n", err) + os.Exit(1) + } + } else { + bmcs, err := bmoe2e.LoadBMCConfig(*configFile) + if err != nil { + os.Exit(1) + } + for _, bmc := range *bmcs { + if err = CreateLibvirtBMC(bmc.BootMacAddress, bmc.HostName, bmc.IPAddress, "baremetal-e2e"); err != nil { + fmt.Printf("Error occurred: %v\n", err) + os.Exit(1) + } + } + } +} diff --git a/test/createVM/templates/VM.xml.tpl b/test/createVM/templates/VM.xml.tpl new file mode 100644 index 0000000000..bd3de83fb7 --- /dev/null +++ b/test/createVM/templates/VM.xml.tpl @@ -0,0 +1,105 @@ + + {{ .HostName }} + 4194304 + 4194304 + 2 + + hvm + + + + + + + + + + + + destroy + restart + restart + + /usr/bin/qemu-system-x86_64 + + + + +
+ + + + + + + +
+ + +
+ + +
+ + + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + + + + + + + + + + + + + + + + + + +