From 47670fcf73dc8fc2ad3768514ae056282cceb82d Mon Sep 17 00:00:00 2001 From: Penny Zheng Date: Fri, 11 Jan 2019 14:09:03 +0800 Subject: [PATCH] memoryDevice: reconstruct memoryDevice If kata-runtime supports memory hotplug via probe interface, we need to reconstruct memoryDevice to store relevant status, which are addr and probe. addr specifies the physical address of the memory device, and probe determines it is hotplugged via acpi-driven or probe interface. Fixes: #1149 Signed-off-by: Penny Zheng --- virtcontainers/fc.go | 4 +-- virtcontainers/hypervisor.go | 4 ++- virtcontainers/mock_hypervisor.go | 4 +-- virtcontainers/qemu.go | 51 +++++++++++++++++++------------ virtcontainers/qemu_test.go | 4 +-- virtcontainers/sandbox.go | 2 +- virtcontainers/vm.go | 2 +- 7 files changed, 43 insertions(+), 28 deletions(-) diff --git a/virtcontainers/fc.go b/virtcontainers/fc.go index 96ac05829..52f3fa2f6 100644 --- a/virtcontainers/fc.go +++ b/virtcontainers/fc.go @@ -683,8 +683,8 @@ func (fc *firecracker) hypervisorConfig() HypervisorConfig { return fc.config } -func (fc *firecracker) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32) (uint32, error) { - return 0, nil +func (fc *firecracker) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, memoryDevice, error) { + return 0, memoryDevice{}, nil } func (fc *firecracker) resizeVCPUs(reqVCPUs uint32) (currentVCPUs uint32, newVCPUs uint32, err error) { diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go index 1157d4214..5dd008c7b 100644 --- a/virtcontainers/hypervisor.go +++ b/virtcontainers/hypervisor.go @@ -91,6 +91,8 @@ const ( type memoryDevice struct { slot int sizeMB int + addr uint64 + probe bool } // Set sets an hypervisor type based on the input string. @@ -592,7 +594,7 @@ type hypervisor interface { addDevice(devInfo interface{}, devType deviceType) error hotplugAddDevice(devInfo interface{}, devType deviceType) (interface{}, error) hotplugRemoveDevice(devInfo interface{}, devType deviceType) (interface{}, error) - resizeMemory(memMB uint32, memoryBlockSizeMB uint32) (uint32, error) + resizeMemory(memMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, memoryDevice, error) resizeVCPUs(vcpus uint32) (uint32, uint32, error) getSandboxConsole(sandboxID string) (string, error) disconnect() diff --git a/virtcontainers/mock_hypervisor.go b/virtcontainers/mock_hypervisor.go index 780ae566f..86e3739f7 100644 --- a/virtcontainers/mock_hypervisor.go +++ b/virtcontainers/mock_hypervisor.go @@ -84,8 +84,8 @@ func (m *mockHypervisor) getSandboxConsole(sandboxID string) (string, error) { return "", nil } -func (m *mockHypervisor) resizeMemory(memMB uint32, memorySectionSizeMB uint32) (uint32, error) { - return 0, nil +func (m *mockHypervisor) resizeMemory(memMB uint32, memorySectionSizeMB uint32, probe bool) (uint32, memoryDevice, error) { + return 0, memoryDevice{}, nil } func (m *mockHypervisor) resizeVCPUs(cpus uint32) (uint32, uint32, error) { return 0, 0, nil diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go index e461b7edf..23a0271f1 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/qemu.go @@ -1240,7 +1240,19 @@ func (q *qemu) hotplugAddMemory(memDev *memoryDevice) (int, error) { q.Logger().WithError(err).Error("hotplug memory") return 0, err } - + // if guest kernel only supports memory hotplug via probe interface, we need to get address of hot-add memory device + if memDev.probe { + memoryDevices, err := q.qmpMonitorCh.qmp.ExecQueryMemoryDevices(q.qmpMonitorCh.ctx) + if err != nil { + return 0, fmt.Errorf("failed to query memory devices: %v", err) + } + if len(memoryDevices) != 0 { + q.Logger().WithField("addr", fmt.Sprintf("0x%x", memoryDevices[len(memoryDevices)-1].Data.Addr)).Debug("recently hot-add memory device") + memDev.addr = memoryDevices[len(memoryDevices)-1].Data.Addr + } else { + return 0, fmt.Errorf("failed to probe address of recently hot-add memory device, no device exists") + } + } q.state.HotpluggedMemory += memDev.sizeMB return memDev.sizeMB, q.store.Store(store.Hypervisor, q.state) } @@ -1370,32 +1382,33 @@ func (q *qemu) disconnect() { // the memory to remove has to be at least the size of one slot. // To return memory back we are resizing the VM memory balloon. // A longer term solution is evaluate solutions like virtio-mem -func (q *qemu) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32) (uint32, error) { +func (q *qemu) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32, probe bool) (uint32, memoryDevice, error) { currentMemory := q.config.MemorySize + uint32(q.state.HotpluggedMemory) err := q.qmpSetup() if err != nil { - return 0, err + return 0, memoryDevice{}, err } + var addMemDevice memoryDevice switch { case currentMemory < reqMemMB: //hotplug addMemMB := reqMemMB - currentMemory memHotplugMB, err := calcHotplugMemMiBSize(addMemMB, memoryBlockSizeMB) if err != nil { - return currentMemory, err + return currentMemory, memoryDevice{}, err } - addMemDevice := &memoryDevice{ - sizeMB: int(memHotplugMB), - } - data, err := q.hotplugAddDevice(addMemDevice, memoryDev) + addMemDevice.sizeMB = int(memHotplugMB) + addMemDevice.probe = probe + + data, err := q.hotplugAddDevice(&addMemDevice, memoryDev) if err != nil { - return currentMemory, err + return currentMemory, addMemDevice, err } memoryAdded, ok := data.(int) if !ok { - return currentMemory, fmt.Errorf("Could not get the memory added, got %+v", data) + return currentMemory, addMemDevice, fmt.Errorf("Could not get the memory added, got %+v", data) } currentMemory += uint32(memoryAdded) case currentMemory > reqMemMB: @@ -1403,31 +1416,31 @@ func (q *qemu) resizeMemory(reqMemMB uint32, memoryBlockSizeMB uint32) (uint32, addMemMB := currentMemory - reqMemMB memHotunplugMB, err := calcHotplugMemMiBSize(addMemMB, memoryBlockSizeMB) if err != nil { - return currentMemory, err + return currentMemory, memoryDevice{}, err } - addMemDevice := &memoryDevice{ - sizeMB: int(memHotunplugMB), - } - data, err := q.hotplugRemoveDevice(addMemDevice, memoryDev) + addMemDevice.sizeMB = int(memHotunplugMB) + addMemDevice.probe = probe + + data, err := q.hotplugRemoveDevice(&addMemDevice, memoryDev) if err != nil { - return currentMemory, err + return currentMemory, addMemDevice, err } memoryRemoved, ok := data.(int) if !ok { - return currentMemory, fmt.Errorf("Could not get the memory removed, got %+v", data) + return currentMemory, addMemDevice, fmt.Errorf("Could not get the memory removed, got %+v", data) } //FIXME: This is to check memory hotplugRemoveDevice reported 0, as this is not supported. // In the future if this is implemented this validation should be removed. if memoryRemoved != 0 { - return currentMemory, fmt.Errorf("memory hot unplug is not supported, something went wrong") + return currentMemory, addMemDevice, fmt.Errorf("memory hot unplug is not supported, something went wrong") } currentMemory -= uint32(memoryRemoved) } // currentMemory is the current memory (updated) of the VM, return to caller to allow verify // the current VM memory state. - return currentMemory, nil + return currentMemory, addMemDevice, nil } // genericAppendBridges appends to devices the given bridges diff --git a/virtcontainers/qemu_test.go b/virtcontainers/qemu_test.go index 28b4f3cd6..cf073134f 100644 --- a/virtcontainers/qemu_test.go +++ b/virtcontainers/qemu_test.go @@ -393,9 +393,9 @@ func TestHotplugUnsupportedDeviceType(t *testing.T) { } q.store = vcStore - _, err = q.hotplugAddDevice(&memoryDevice{0, 128}, fsDev) + _, err = q.hotplugAddDevice(&memoryDevice{0, 128, uint64(0), false}, fsDev) assert.Error(err) - _, err = q.hotplugRemoveDevice(&memoryDevice{0, 128}, fsDev) + _, err = q.hotplugRemoveDevice(&memoryDevice{0, 128, uint64(0), false}, fsDev) assert.Error(err) } diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index 6a82d2d6e..79c443491 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -1673,7 +1673,7 @@ func (s *Sandbox) updateResources() error { // Update Memory s.Logger().WithField("memory-sandbox-size-byte", sandboxMemoryByte).Debugf("Request to hypervisor to update memory") - newMemory, err := s.hypervisor.resizeMemory(uint32(sandboxMemoryByte>>utils.MibToBytesShift), s.state.GuestMemoryBlockSizeMB) + newMemory, _, err := s.hypervisor.resizeMemory(uint32(sandboxMemoryByte>>utils.MibToBytesShift), s.state.GuestMemoryBlockSizeMB, s.state.GuestMemoryHotplugProbe) if err != nil { return err } diff --git a/virtcontainers/vm.go b/virtcontainers/vm.go index dc9d6f097..b621830bf 100644 --- a/virtcontainers/vm.go +++ b/virtcontainers/vm.go @@ -373,7 +373,7 @@ func (v *VM) AddCPUs(num uint32) error { func (v *VM) AddMemory(numMB uint32) error { if numMB > 0 { v.logger().Infof("hot adding %d MB memory", numMB) - dev := &memoryDevice{1, int(numMB)} + dev := &memoryDevice{1, int(numMB), 0, false} if _, err := v.hypervisor.hotplugAddDevice(dev, memoryDev); err != nil { return err }