diff --git a/virtcontainers/fc.go b/virtcontainers/fc.go index 96ac05829..52f3fa2f6 100644 --- a/virtcontainers/fc.go +++ b/virtcontainers/fc.go @@ -683,8 +683,8 @@ func (fc *firecracker) hypervisorConfig() HypervisorConfig { return fc.config } -func (fc *firecracker) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32) (uint32, error) { - return 0, nil +func (fc *firecracker) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, memoryDevice, error) { + return 0, memoryDevice{}, nil } func (fc *firecracker) resizeVCPUs(reqVCPUs uint32) (currentVCPUs uint32, newVCPUs uint32, err error) { diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go index 1157d4214..5dd008c7b 100644 --- a/virtcontainers/hypervisor.go +++ b/virtcontainers/hypervisor.go @@ -91,6 +91,8 @@ const ( type memoryDevice struct { slot int sizeMB int + addr uint64 + probe bool } // Set sets an hypervisor type based on the input string. @@ -592,7 +594,7 @@ type hypervisor interface { addDevice(devInfo interface{}, devType deviceType) error hotplugAddDevice(devInfo interface{}, devType deviceType) (interface{}, error) hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error) - resizeMemory(memMB uint32, memoryBlockSizeMB uint32) (uint32, error) + resizeMemory(memMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, memoryDevice, error) resizeVCPUs(vcpus uint32) (uint32, uint32, error) getSandboxConsole(sandboxID string) (string, error) disconnect() diff --git a/virtcontainers/mock_hypervisor.go b/virtcontainers/mock_hypervisor.go index 780ae566f..86e3739f7 100644 --- a/virtcontainers/mock_hypervisor.go +++ b/virtcontainers/mock_hypervisor.go @@ -84,8 +84,8 @@ func (m *mockHypervisor) getSandboxConsole(sandboxID string) (string, error) { return "", nil } -func (m *mockHypervisor) resizeMemory(memMB uint32, memorySectionSizeMB uint32) (uint32, error) { - return 0, nil +func (m *mockHypervisor) resizeMemory(memMB uint32, memorySectionSizeMB uint32, probe bool) (uint32, memoryDevice, error) { + return 0, memoryDevice{}, nil } func (m *mockHypervisor) resizeVCPUs(cpus uint32) (uint32, uint32, error) { return 0, 0, nil diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go index e461b7edf..23a0271f1 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/qemu.go @@ -1240,7 +1240,19 @@ func (q *qemu) hotplugAddMemory(memDev *memoryDevice) (int, error) { q.Logger().WithError(err).Error("hotplug memory") return 0, err } - + // if guest kernel only supports memory hotplug via probe interface, we need to get address of hot-add memory device + if memDev.probe { + memoryDevices, err := q.qmpMonitorCh.qmp.ExecQueryMemoryDevices(q.qmpMonitorCh.ctx) + if err != nil { + return 0, fmt.Errorf("failed to query memory devices: %v", err) + } + if len(memoryDevices) != 0 { + q.Logger().WithField("addr", fmt.Sprintf("0x%x", memoryDevices[len(memoryDevices)-1].Data.Addr)).Debug("recently hot-add memory device") + memDev.addr = memoryDevices[len(memoryDevices)-1].Data.Addr + } else { + return 0, fmt.Errorf("failed to probe address of recently hot-add memory device, no device exists") + } + } q.state.HotpluggedMemory += memDev.sizeMB return memDev.sizeMB, q.store.Store(store.Hypervisor, q.state) } @@ -1370,32 +1382,33 @@ func (q *qemu) disconnect() { // the memory to remove has to be at least the size of one slot. // To return memory back we are resizing the VM memory balloon. // A longer term solution is evaluate solutions like virtio-mem -func (q *qemu) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32) (uint32, error) { +func (q *qemu) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, memoryDevice, error) { currentMemory := q.config.MemorySize + uint32(q.state.HotpluggedMemory) err := q.qmpSetup() if err != nil { - return 0, err + return 0, memoryDevice{}, err } + var addMemDevice memoryDevice switch { case currentMemory < reqMemMB: //hotplug addMemMB := reqMemMB - currentMemory memHotplugMB, err := calcHotplugMemMiBSize(addMemMB, memoryBlockSizeMB) if err != nil { - return currentMemory, err + return currentMemory, memoryDevice{}, err } - addMemDevice := &memoryDevice{ - sizeMB: int(memHotplugMB), - } - data, err := q.hotplugAddDevice(addMemDevice, memoryDev) + addMemDevice.sizeMB = int(memHotplugMB) + addMemDevice.probe = probe + + data, err := q.hotplugAddDevice(&addMemDevice, memoryDev) if err != nil { - return currentMemory, err + return currentMemory, addMemDevice, err } memoryAdded, ok := data.(int) if !ok { - return currentMemory, fmt.Errorf("Could not get the memory added, got %+v", data) + return currentMemory, addMemDevice, fmt.Errorf("Could not get the memory added, got %+v", data) } currentMemory += uint32(memoryAdded) case currentMemory > reqMemMB: @@ -1403,31 +1416,31 @@ func (q *qemu) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32) (uint32, addMemMB := currentMemory - reqMemMB memHotunplugMB, err := calcHotplugMemMiBSize(addMemMB, memoryBlockSizeMB) if err != nil { - return currentMemory, err + return currentMemory, memoryDevice{}, err } - addMemDevice := &memoryDevice{ - sizeMB: int(memHotunplugMB), - } - data, err := q.hotplugRemoveDevice(addMemDevice, memoryDev) + addMemDevice.sizeMB = int(memHotunplugMB) + addMemDevice.probe = probe + + data, err := q.hotplugRemoveDevice(&addMemDevice, memoryDev) if err != nil { - return currentMemory, err + return currentMemory, addMemDevice, err } memoryRemoved, ok := data.(int) if !ok { - return currentMemory, fmt.Errorf("Could not get the memory removed, got %+v", data) + return currentMemory, addMemDevice, fmt.Errorf("Could not get the memory removed, got %+v", data) } //FIXME: This is to check memory hotplugRemoveDevice reported 0, as this is not supported. // In the future if this is implemented this validation should be removed. if memoryRemoved != 0 { - return currentMemory, fmt.Errorf("memory hot unplug is not supported, something went wrong") + return currentMemory, addMemDevice, fmt.Errorf("memory hot unplug is not supported, something went wrong") } currentMemory -= uint32(memoryRemoved) } // currentMemory is the current memory (updated) of the VM, return to caller to allow verify // the current VM memory state. - return currentMemory, nil + return currentMemory, addMemDevice, nil } // genericAppendBridges appends to devices the given bridges diff --git a/virtcontainers/qemu_test.go b/virtcontainers/qemu_test.go index 28b4f3cd6..cf073134f 100644 --- a/virtcontainers/qemu_test.go +++ b/virtcontainers/qemu_test.go @@ -393,9 +393,9 @@ func TestHotplugUnsupportedDeviceType(t *testing.T) { } q.store = vcStore - _, err = q.hotplugAddDevice(&memoryDevice{0, 128}, fsDev) + _, err = q.hotplugAddDevice(&memoryDevice{0, 128, uint64(0), false}, fsDev) assert.Error(err) - _, err = q.hotplugRemoveDevice(&memoryDevice{0, 128}, fsDev) + _, err = q.hotplugRemoveDevice(&memoryDevice{0, 128, uint64(0), false}, fsDev) assert.Error(err) } diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index 6a82d2d6e..79c443491 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -1673,7 +1673,7 @@ func (s *Sandbox) updateResources() error { // Update Memory s.Logger().WithField("memory-sandbox-size-byte", sandboxMemoryByte).Debugf("Request to hypervisor to update memory") - newMemory, err := s.hypervisor.resizeMemory(uint32(sandboxMemoryByte>>utils.MibToBytesShift), s.state.GuestMemoryBlockSizeMB) + newMemory, _, err := s.hypervisor.resizeMemory(uint32(sandboxMemoryByte>>utils.MibToBytesShift), s.state.GuestMemoryBlockSizeMB, s.state.GuestMemoryHotplugProbe) if err != nil { return err } diff --git a/virtcontainers/vm.go b/virtcontainers/vm.go index dc9d6f097..b621830bf 100644 --- a/virtcontainers/vm.go +++ b/virtcontainers/vm.go @@ -373,7 +373,7 @@ func (v *VM) AddCPUs(num uint32) error { func (v *VM) AddMemory(numMB uint32) error { if numMB > 0 { v.logger().Infof("hot adding %d MB memory", numMB) - dev := &memoryDevice{1, int(numMB)} + dev := &memoryDevice{1, int(numMB), 0, false} if _, err := v.hypervisor.hotplugAddDevice(dev, memoryDev); err != nil { return err }