Support VM disk resize without reboot (from Incus)

doc/api-extensions.md

@@ -2516,3 +2516,7 @@ Adds support for using a bridge network with a specified VLAN ID as an OVN uplin
 
 Adds `logical_cpus` field to `GET /1.0/cluster/members/{name}/state` which
 contains the total available logical CPUs available when LXD started.
+
+## `vm_storage_disk_live_resize`
+
+This enables resizing of virtual machine `zfs`, `lvm` and `ceph` disks without requiring a restart for the changes to take effect.

lxd/api_internal.go

@@ -52,6 +52,7 @@ var apiInternal = []APIEndpoint{
 	internalContainerOnStartCmd,
 	internalContainerOnStopCmd,
 	internalContainerOnStopNSCmd,
+	internalVirtualMachineOnResizeCmd,
 	internalGarbageCollectorCmd,
 	internalImageOptimizeCmd,
 	internalImageRefreshCmd,
@@ -93,6 +94,12 @@ var internalContainerOnStopCmd = APIEndpoint{
 	Get: APIEndpointAction{Handler: internalContainerOnStop, AccessHandler: allowPermission(entity.TypeServer, auth.EntitlementCanEdit)},
 }
 
+var internalVirtualMachineOnResizeCmd = APIEndpoint{
+	Path: "virtual-machines/{instanceRef}/onresize",
+
+	Get: APIEndpointAction{Handler: internalVirtualMachineOnResize, AccessHandler: allowPermission(entity.TypeServer, auth.EntitlementCanEdit)},
+}
+
 var internalSQLCmd = APIEndpoint{
 	Path: "sql",
 
@@ -387,6 +394,56 @@ func internalContainerOnStop(d *Daemon, r *http.Request) response.Response {
 	return response.EmptySyncResponse
 }
 
+func internalVirtualMachineOnResize(d *Daemon, r *http.Request) response.Response {
+	s := d.State()
+
+	// Get the instance ID.
+	instanceID, err := strconv.Atoi(mux.Vars(r)["instanceRef"])
+	if err != nil {
+		return response.BadRequest(err)
+	}
+
+	// Get the devices list.
+	devices := request.QueryParam(r, "devices")
+	if devices == "" {
+		return response.BadRequest(fmt.Errorf("Resize hook requires a list of devices"))
+	}
+
+	// Load by ID.
+	inst, err := instance.LoadByID(s, instanceID)
+	if err != nil {
+		return response.SmartError(err)
+	}
+
+	// Update the local instance.
+	for _, dev := range strings.Split(devices, ",") {
+		fields := strings.SplitN(dev, ":", 2)
+		if len(fields) != 2 {
+			return response.BadRequest(fmt.Errorf("Invalid device/size tuple: %s", dev))
+		}
+
+		size, err := strconv.ParseInt(fields[1], 16, 64)
+		if err != nil {
+			return response.BadRequest(err)
+		}
+
+		runConf := deviceConfig.RunConfig{}
+		runConf.Mounts = []deviceConfig.MountEntryItem{
+			{
+				DevName: fields[0],
+				Size:    size,
+			},
+		}
+
+		err = inst.DeviceEventHandler(&runConf)
+		if err != nil {
+			return response.InternalError(err)
+		}
+	}
+
+	return response.EmptySyncResponse
+}
+
 type internalSQLDump struct {
 	Text string `json:"text" yaml:"text"`
 }

lxd/cluster/connect.go

@@ -16,11 +16,17 @@ import (
 	"github.com/canonical/lxd/lxd/instance/instancetype"
 	"github.com/canonical/lxd/lxd/request"
 	"github.com/canonical/lxd/lxd/state"
+	storagePools "github.com/canonical/lxd/lxd/storage"
 	"github.com/canonical/lxd/shared"
 	"github.com/canonical/lxd/shared/api"
 	"github.com/canonical/lxd/shared/version"
 )
 
+// Set references.
+func init() {
+	storagePools.ConnectIfInstanceIsRemote = ConnectIfInstanceIsRemote
+}
+
 // Connect is a convenience around lxd.ConnectLXD that configures the client
 // with the correct parameters for node-to-node communication.
 //

lxd/device/config/device_runconfig.go

@@ -30,6 +30,7 @@ type MountEntryItem struct {
 	PassNo     int         // Used by fsck(8) to determine the order in which filesystem checks are done at boot time. Defaults to zero (don't fsck) if not present.
 	OwnerShift string      // Ownership shifting mode, use constants MountOwnerShiftNone, MountOwnerShiftStatic or MountOwnerShiftDynamic.
 	Limits     *DiskLimits // Disk limits.
+	Size       int64       // Expected disk size in bytes.
 }
 
 // RootFSEntryItem represents the root filesystem options for an Instance.

lxd/device/disk.go

@@ -1334,6 +1334,26 @@ func (d *disk) Update(oldDevices deviceConfig.Devices, isRunning bool) error {
 				d.logger.Warn("Could not apply quota because disk is in use, deferring until next start")
 			} else if err != nil {
 				return err
+			} else if d.inst.Type() == instancetype.VM && d.inst.IsRunning() {
+				// Get the disk size in bytes.
+				size, err := units.ParseByteSizeString(newRootDiskDeviceSize)
+				if err != nil {
+					return err
+				}
+
+				// Notify to reload disk size.
+				runConf := deviceConfig.RunConfig{}
+				runConf.Mounts = []deviceConfig.MountEntryItem{
+					{
+						DevName: d.name,
+						Size:    size,
+					},
+				}
+
+				err = d.inst.DeviceEventHandler(&runConf)
+				if err != nil {
+					return err
+				}
 			}
 		}
 	}

lxd/instance/drivers/driver_qemu.go

@@ -8082,7 +8082,7 @@ func (d *qemu) DeviceEventHandler(runConf *deviceConfig.RunConfig) error {
 
 	// Handle disk reconfiguration.
 	for _, mount := range runConf.Mounts {
-		if mount.Limits == nil {
+		if mount.Limits == nil && mount.Size == 0 {
 			continue
 		}
 
@@ -8095,10 +8095,20 @@ func (d *qemu) DeviceEventHandler(runConf *deviceConfig.RunConfig) error {
 		// Figure out the QEMU device ID.
 		devID := fmt.Sprintf("%s%s", qemuDeviceIDPrefix, filesystem.PathNameEncode(mount.DevName))
 
-		// Apply the limits.
-		err = m.SetBlockThrottle(devID, int(mount.Limits.ReadBytes), int(mount.Limits.WriteBytes), int(mount.Limits.ReadIOps), int(mount.Limits.WriteIOps))
-		if err != nil {
-			return fmt.Errorf("Failed applying limits for disk device %q: %w", mount.DevName, err)
+		if mount.Limits != nil {
+			// Apply the limits.
+			err = m.SetBlockThrottle(devID, int(mount.Limits.ReadBytes), int(mount.Limits.WriteBytes), int(mount.Limits.ReadIOps), int(mount.Limits.WriteIOps))
+			if err != nil {
+				return fmt.Errorf("Failed applying limits for disk device %q: %w", mount.DevName, err)
+			}
+		}
+
+		if mount.Size > 0 {
+			// Update the size.
+			err = m.UpdateBlockSize(strings.SplitN(devID, "-", 2)[1])
+			if err != nil {
+				return fmt.Errorf("Failed updating disk size %q: %w", mount.DevName, err)
+			}
 		}
 	}
 

lxd/instance/drivers/qmp/commands.go

@@ -1015,6 +1015,24 @@ func (m *Monitor) Eject(id string) error {
 	return nil
 }
 
+// UpdateBlockSize updates the size of a disk.
+func (m *Monitor) UpdateBlockSize(id string) error {
+	var args struct {
+		NodeName string `json:"node-name"`
+		Size     int64  `json:"size"`
+	}
+
+	args.NodeName = id
+	args.Size = 1
+
+	err := m.run("block_resize", args, nil)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // SetBlockThrottle applies an I/O limit on a disk.
 func (m *Monitor) SetBlockThrottle(id string, bytesRead int, bytesWrite int, iopsRead int, iopsWrite int) error {
 	var args struct {

lxd/storage/backend_lxd.go

@@ -26,12 +26,14 @@ import (
 	"golang.org/x/sys/unix"
 	"gopkg.in/yaml.v2"
 
+	"github.com/canonical/lxd/client"
 	"github.com/canonical/lxd/lxd/apparmor"
 	"github.com/canonical/lxd/lxd/backup"
 	backupConfig "github.com/canonical/lxd/lxd/backup/config"
 	"github.com/canonical/lxd/lxd/cluster/request"
 	"github.com/canonical/lxd/lxd/db"
 	"github.com/canonical/lxd/lxd/db/cluster"
+	deviceConfig "github.com/canonical/lxd/lxd/device/config"
 	"github.com/canonical/lxd/lxd/instance"
 	"github.com/canonical/lxd/lxd/instance/instancetype"
 	"github.com/canonical/lxd/lxd/instancewriter"
@@ -64,6 +66,11 @@ import (
 var unavailablePools = make(map[string]struct{})
 var unavailablePoolsMu = sync.Mutex{}
 
+// ConnectIfInstanceIsRemote is a reference to cluster.ConnectIfInstanceIsRemote.
+//
+//nolint:typecheck
+var ConnectIfInstanceIsRemote func(s *state.State, projectName string, instName string, r *http.Request, instanceType instancetype.Type) (lxd.InstanceServer, error)
+
 // instanceDiskVolumeEffectiveFields fields from the instance disks that are applied to the volume's effective
 // config (but not stored in the disk's volume database record).
 var instanceDiskVolumeEffectiveFields = []string{
@@ -6068,6 +6075,80 @@ func (b *lxdBackend) UpdateCustomVolume(projectName string, volName string, newD
 		delete(newConfig, "volatile.idmap.next")
 	}
 
+	// Notify instances of disk size changes as needed.
+	newSize, ok := changedConfig["size"]
+	if ok && newSize != "" && contentType == drivers.ContentTypeBlock {
+		// Get the disk size in bytes.
+		size, err := units.ParseByteSizeString(changedConfig["size"])
+		if err != nil {
+			return err
+		}
+
+		type instDevice struct {
+			args    db.InstanceArgs
+			devices []string
+		}
+
+		instDevices := []instDevice{}
+		err = VolumeUsedByInstanceDevices(b.state, b.name, projectName, &curVol.StorageVolume, true, func(dbInst db.InstanceArgs, project api.Project, usedByDevices []string) error {
+			if dbInst.Type != instancetype.VM {
+				return nil
+			}
+
+			instDevices = append(instDevices, instDevice{args: dbInst, devices: usedByDevices})
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+
+		for _, entry := range instDevices {
+			c, err := ConnectIfInstanceIsRemote(b.state, entry.args.Project, entry.args.Name, nil, entry.args.Type)
+			if err != nil {
+				return err
+			}
+
+			if c != nil {
+				// Send a remote notification.
+				devs := []string{}
+				for _, devName := range entry.devices {
+					devs = append(devs, fmt.Sprintf("%s:%d", devName, size))
+				}
+
+				uri := fmt.Sprintf("/internal/virtual-machines/%d/onresize?devices=%s", entry.args.ID, strings.Join(devs, ","))
+				_, _, err := c.RawQuery("GET", uri, nil, "")
+				if err != nil {
+					return err
+				}
+			} else {
+				// Update the local instance.
+				inst, err := instance.LoadByProjectAndName(b.state, entry.args.Project, entry.args.Name)
+				if err != nil {
+					return err
+				}
+
+				if !inst.IsRunning() {
+					continue
+				}
+
+				for _, devName := range entry.devices {
+					runConf := deviceConfig.RunConfig{}
+					runConf.Mounts = []deviceConfig.MountEntryItem{
+						{
+							DevName: devName,
+							Size:    size,
+						},
+					}
+
+					err = inst.DeviceEventHandler(&runConf)
+					if err != nil {
+						return err
+					}
+				}
+			}
+		}
+	}
+
 	// Update the database if something changed.
 	if len(changedConfig) != 0 || newDesc != curVol.Description {
 		err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {