diff --git a/Makefile b/Makefile index 96b26dc6e..b074e485e 100644 --- a/Makefile +++ b/Makefile @@ -125,6 +125,13 @@ DEFVCPUS := 1 DEFMAXVCPUS := 0 # Default memory size in MiB DEFMEMSZ := 2048 +# Default memory slots +# Cases to consider : +# - nvdimm rootfs image +# - preallocated memory +# - vm template memory +# - hugepage memory +DEFMEMSLOTS := 10 #Default number of bridges DEFBRIDGES := 1 #Default network model @@ -202,6 +209,7 @@ USER_VARS += SYSCONFDIR USER_VARS += DEFVCPUS USER_VARS += DEFMAXVCPUS USER_VARS += DEFMEMSZ +USER_VARS += DEFMEMSLOTS USER_VARS += DEFBRIDGES USER_VARS += DEFNETWORKMODEL USER_VARS += DEFDISABLEBLOCK @@ -302,6 +310,7 @@ const defaultRootDirectory = "$(PKGRUNDIR)" const defaultVCPUCount uint32 = $(DEFVCPUS) const defaultMaxVCPUCount uint32 = $(DEFMAXVCPUS) const defaultMemSize uint32 = $(DEFMEMSZ) // MiB +const defaultMemSlots uint32 = $(DEFMEMSLOTS) const defaultBridgesCount uint32 = $(DEFBRIDGES) const defaultInterNetworkingModel = "$(DEFNETWORKMODEL)" const defaultDisableBlockDeviceUse bool = $(DEFDISABLEBLOCK) @@ -391,6 +400,7 @@ $(GENERATED_FILES): %: %.in Makefile VERSION -e "s|@DEFVCPUS@|$(DEFVCPUS)|g" \ -e "s|@DEFMAXVCPUS@|$(DEFMAXVCPUS)|g" \ -e "s|@DEFMEMSZ@|$(DEFMEMSZ)|g" \ + -e "s|@DEFMEMSLOTS@|$(DEFMEMSLOTS)|g" \ -e "s|@DEFBRIDGES@|$(DEFBRIDGES)|g" \ -e "s|@DEFNETWORKMODEL@|$(DEFNETWORKMODEL)|g" \ -e "s|@DEFDISABLEBLOCK@|$(DEFDISABLEBLOCK)|g" \ diff --git a/cli/config.go b/cli/config.go index 7c69fb0ff..a7011d4fc 100644 --- a/cli/config.go +++ b/cli/config.go @@ -84,6 +84,7 @@ type hypervisor struct { NumVCPUs int32 `toml:"default_vcpus"` DefaultMaxVCPUs uint32 `toml:"default_maxvcpus"` MemorySize uint32 `toml:"default_memory"` + MemSlots uint32 `toml:"memory_slots"` DefaultBridges uint32 `toml:"default_bridges"` Msize9p uint32 `toml:"msize_9p"` DisableBlockDeviceUse bool `toml:"disable_block_device_use"` @@ -247,6 +248,15 @@ func (h hypervisor) defaultMemSz() uint32 { return h.MemorySize } +func (h hypervisor) defaultMemSlots() uint32 { + slots := h.MemSlots + if slots == 0 { + slots = defaultMemSlots + } + + return slots +} + func (h hypervisor) defaultBridges() uint32 { if h.DefaultBridges == 0 { return defaultBridgesCount @@ -392,6 +402,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) { NumVCPUs: h.defaultVCPUs(), DefaultMaxVCPUs: h.defaultMaxVCPUs(), MemorySize: h.defaultMemSz(), + MemSlots: h.defaultMemSlots(), DefaultBridges: h.defaultBridges(), DisableBlockDeviceUse: h.DisableBlockDeviceUse, MemPrealloc: h.MemPrealloc, diff --git a/cli/config/configuration.toml.in b/cli/config/configuration.toml.in index 305fc275f..e76871538 100644 --- a/cli/config/configuration.toml.in +++ b/cli/config/configuration.toml.in @@ -76,6 +76,11 @@ default_bridges = @DEFBRIDGES@ # Default memory size in MiB for SB/VM. # If unspecified then it will be set @DEFMEMSZ@ MiB. #default_memory = @DEFMEMSZ@ +# +# Default memory slots per SB/VM. +# If unspecified then it will be set @DEFMEMSLOTS@. +# This is will determine the times that memory will be hotadded to sandbox/VM. +#memory_slots = @DEFMEMSLOTS@ # Disable block device from being used for a container's rootfs. # In case of a storage driver like devicemapper where a container's diff --git a/cli/config_test.go b/cli/config_test.go index c83563966..8a155e1ef 100644 --- a/cli/config_test.go +++ b/cli/config_test.go @@ -159,6 +159,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf EnableIOThreads: enableIOThreads, HotplugVFIOOnRootBus: hotplugVFIOOnRootBus, Msize9p: defaultMsize9p, + MemSlots: defaultMemSlots, } agentConfig := vc.KataAgentConfig{} diff --git a/cli/kata-env.go b/cli/kata-env.go index e2a24c746..2327be56e 100644 --- a/cli/kata-env.go +++ b/cli/kata-env.go @@ -25,7 +25,7 @@ import ( // // XXX: Increment for every change to the output format // (meaning any change to the EnvInfo type). -const formatVersion = "1.0.16" +const formatVersion = "1.0.17" // MetaInfo stores information on the format of the output itself type MetaInfo struct { @@ -82,6 +82,7 @@ type HypervisorInfo struct { Path string BlockDeviceDriver string Msize9p uint32 + MemorySlots uint32 Debug bool UseVSock bool } @@ -317,6 +318,7 @@ func getHypervisorInfo(config oci.RuntimeConfig) HypervisorInfo { BlockDeviceDriver: config.HypervisorConfig.BlockDeviceDriver, Msize9p: config.HypervisorConfig.Msize9p, UseVSock: config.HypervisorConfig.UseVSock, + MemorySlots: config.HypervisorConfig.MemSlots, } } diff --git a/cli/kata-env_test.go b/cli/kata-env_test.go index c3a888e8b..0a71f0320 100644 --- a/cli/kata-env_test.go +++ b/cli/kata-env_test.go @@ -262,6 +262,7 @@ func getExpectedHypervisor(config oci.RuntimeConfig) HypervisorInfo { MachineType: config.HypervisorConfig.HypervisorMachineType, BlockDeviceDriver: config.HypervisorConfig.BlockDeviceDriver, Msize9p: config.HypervisorConfig.Msize9p, + MemorySlots: config.HypervisorConfig.MemSlots, Debug: config.HypervisorConfig.Debug, } } diff --git a/virtcontainers/hypervisor.go b/virtcontainers/hypervisor.go index 8d2a8a1fb..951257f89 100644 --- a/virtcontainers/hypervisor.go +++ b/virtcontainers/hypervisor.go @@ -136,6 +136,25 @@ type Param struct { // HypervisorConfig is the hypervisor configuration. type HypervisorConfig struct { + // NumVCPUs specifies default number of vCPUs for the VM. + NumVCPUs uint32 + + //DefaultMaxVCPUs specifies the maximum number of vCPUs for the VM. + DefaultMaxVCPUs uint32 + + // DefaultMem specifies default memory size in MiB for the VM. + MemorySize uint32 + + // DefaultBridges specifies default number of bridges for the VM. + // Bridges can be used to hot plug devices + DefaultBridges uint32 + + // Msize9p is used as the msize for 9p shares + Msize9p uint32 + + // MemSlots specifies default memory slots the VM. + MemSlots uint32 + // KernelParams are additional guest kernel parameters. KernelParams []Param @@ -169,25 +188,20 @@ type HypervisorConfig struct { // emulated. HypervisorMachineType string + // MemoryPath is the memory file path of VM memory. Used when either BootToBeTemplate or + // BootFromTemplate is true. + MemoryPath string + + // DevicesStatePath is the VM device state file path. Used when either BootToBeTemplate or + // BootFromTemplate is true. + DevicesStatePath string + // customAssets is a map of assets. // Each value in that map takes precedence over the configured assets. // For example, if there is a value for the "kernel" key in this map, // it will be used for the sandbox's kernel path instead of KernelPath. customAssets map[assetType]*asset - // NumVCPUs specifies default number of vCPUs for the VM. - NumVCPUs uint32 - - //DefaultMaxVCPUs specifies the maximum number of vCPUs for the VM. - DefaultMaxVCPUs uint32 - - // DefaultMem specifies default memory size in MiB for the VM. - MemorySize uint32 - - // DefaultBridges specifies default number of bridges for the VM. - // Bridges can be used to hot plug devices - DefaultBridges uint32 - // DisableBlockDeviceUse disallows a block device from being used. DisableBlockDeviceUse bool @@ -217,9 +231,6 @@ type HypervisorConfig struct { // when running on top of another VMM. DisableNestingChecks bool - // Msize9p is used as the msize for 9p shares - Msize9p uint32 - // UseVSock use a vsock for agent communication UseVSock bool @@ -233,14 +244,6 @@ type HypervisorConfig struct { // BootFromTemplate used to indicate if the VM should be created from a template VM BootFromTemplate bool - // MemoryPath is the memory file path of VM memory. Used when either BootToBeTemplate or - // BootFromTemplate is true. - MemoryPath string - - // DevicesStatePath is the VM device state file path. Used when either BootToBeTemplate or - // BootFromTemplate is true. - DevicesStatePath string - // DisableVhostNet is used to indicate if host supports vhost_net DisableVhostNet bool } diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go index e2c236ccc..4b026286f 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/qemu.go @@ -283,7 +283,7 @@ func (q *qemu) memoryTopology() (govmmQemu.Memory, error) { memMb := uint64(q.config.MemorySize) - return q.arch.memoryTopology(memMb, hostMemMb), nil + return q.arch.memoryTopology(memMb, hostMemMb, uint8(q.config.MemSlots)), nil } func (q *qemu) qmpSocketPath(id string) (string, error) { @@ -1248,7 +1248,7 @@ func genericBridges(number uint32, machineType string) []Bridge { return bridges } -func genericMemoryTopology(memoryMb, hostMemoryMb uint64) govmmQemu.Memory { +func genericMemoryTopology(memoryMb, hostMemoryMb uint64, slots uint8) govmmQemu.Memory { // NVDIMM device needs memory space 1024MB // See https://github.com/clearcontainers/runtime/issues/380 memoryOffset := 1024 @@ -1260,7 +1260,7 @@ func genericMemoryTopology(memoryMb, hostMemoryMb uint64) govmmQemu.Memory { memory := govmmQemu.Memory{ Size: mem, - Slots: defaultMemSlots, + Slots: slots, MaxMem: memMax, } diff --git a/virtcontainers/qemu_amd64.go b/virtcontainers/qemu_amd64.go index 2fae39cb5..5f3fd898d 100644 --- a/virtcontainers/qemu_amd64.go +++ b/virtcontainers/qemu_amd64.go @@ -122,8 +122,8 @@ func (q *qemuAmd64) cpuModel() string { return cpuModel } -func (q *qemuAmd64) memoryTopology(memoryMb, hostMemoryMb uint64) govmmQemu.Memory { - return genericMemoryTopology(memoryMb, hostMemoryMb) +func (q *qemuAmd64) memoryTopology(memoryMb, hostMemoryMb uint64, slots uint8) govmmQemu.Memory { + return genericMemoryTopology(memoryMb, hostMemoryMb, slots) } func (q *qemuAmd64) appendImage(devices []govmmQemu.Device, path string) ([]govmmQemu.Device, error) { diff --git a/virtcontainers/qemu_amd64_test.go b/virtcontainers/qemu_amd64_test.go index b8b824c76..c71c8022e 100644 --- a/virtcontainers/qemu_amd64_test.go +++ b/virtcontainers/qemu_amd64_test.go @@ -86,13 +86,14 @@ func TestQemuAmd64MemoryTopology(t *testing.T) { hostMem := uint64(100) mem := uint64(120) + slots := uint8(10) expectedMemory := govmmQemu.Memory{ Size: fmt.Sprintf("%dM", mem), - Slots: defaultMemSlots, + Slots: slots, MaxMem: fmt.Sprintf("%dM", hostMem+uint64(memoryOffset)), } - m := amd64.memoryTopology(mem, hostMem) + m := amd64.memoryTopology(mem, hostMem, slots) assert.Equal(expectedMemory, m) } diff --git a/virtcontainers/qemu_arch_base.go b/virtcontainers/qemu_arch_base.go index 8dd607fa5..16e7e2bd1 100644 --- a/virtcontainers/qemu_arch_base.go +++ b/virtcontainers/qemu_arch_base.go @@ -53,7 +53,7 @@ type qemuArch interface { cpuModel() string // memoryTopology returns the memory topology using the given amount of memoryMb and hostMemoryMb - memoryTopology(memoryMb, hostMemoryMb uint64) govmmQemu.Memory + memoryTopology(memoryMb, hostMemoryMb uint64, slots uint8) govmmQemu.Memory // appendConsole appends a console to devices appendConsole(devices []govmmQemu.Device, path string) []govmmQemu.Device @@ -110,7 +110,6 @@ type qemuArchBase struct { const ( defaultCores uint32 = 1 defaultThreads uint32 = 1 - defaultMemSlots uint8 = 2 defaultCPUModel = "host" defaultBridgeBus = "pcie.0" maxDevIDSize = 31 @@ -258,12 +257,12 @@ func (q *qemuArchBase) cpuModel() string { return defaultCPUModel } -func (q *qemuArchBase) memoryTopology(memoryMb, hostMemoryMb uint64) govmmQemu.Memory { +func (q *qemuArchBase) memoryTopology(memoryMb, hostMemoryMb uint64, slots uint8) govmmQemu.Memory { memMax := fmt.Sprintf("%dM", hostMemoryMb) mem := fmt.Sprintf("%dM", memoryMb) memory := govmmQemu.Memory{ Size: mem, - Slots: defaultMemSlots, + Slots: slots, MaxMem: memMax, } diff --git a/virtcontainers/qemu_arch_base_test.go b/virtcontainers/qemu_arch_base_test.go index 601341fbb..7da056647 100644 --- a/virtcontainers/qemu_arch_base_test.go +++ b/virtcontainers/qemu_arch_base_test.go @@ -185,13 +185,14 @@ func TestQemuArchBaseMemoryTopology(t *testing.T) { hostMem := uint64(100) mem := uint64(120) + slots := uint8(12) expectedMemory := govmmQemu.Memory{ Size: fmt.Sprintf("%dM", mem), - Slots: defaultMemSlots, + Slots: slots, MaxMem: fmt.Sprintf("%dM", hostMem), } - m := qemuArchBase.memoryTopology(mem, hostMem) + m := qemuArchBase.memoryTopology(mem, hostMem, slots) assert.Equal(expectedMemory, m) } diff --git a/virtcontainers/qemu_ppc64le.go b/virtcontainers/qemu_ppc64le.go index 3c22ec116..6b5d8b654 100644 --- a/virtcontainers/qemu_ppc64le.go +++ b/virtcontainers/qemu_ppc64le.go @@ -109,7 +109,7 @@ func (q *qemuPPC64le) cpuModel() string { return cpuModel } -func (q *qemuPPC64le) memoryTopology(memoryMb, hostMemoryMb uint64) govmmQemu.Memory { +func (q *qemuPPC64le) memoryTopology(memoryMb, hostMemoryMb uint64, slots uint8) govmmQemu.Memory { if qemuMajorVersion >= 2 && qemuMinorVersion >= 10 { q.Logger().Debug("Aligning maxmem to multiples of 256MB. Assumption: Kernel Version >= 4.11") @@ -119,7 +119,7 @@ func (q *qemuPPC64le) memoryTopology(memoryMb, hostMemoryMb uint64) govmmQemu.Me hostMemoryMb = defaultMemMaxPPC64le } - return genericMemoryTopology(memoryMb, hostMemoryMb) + return genericMemoryTopology(memoryMb, hostMemoryMb, slots) } func (q *qemuPPC64le) appendImage(devices []govmmQemu.Device, path string) ([]govmmQemu.Device, error) { diff --git a/virtcontainers/qemu_ppc64le_test.go b/virtcontainers/qemu_ppc64le_test.go index 22752b2c7..ddb25fc00 100644 --- a/virtcontainers/qemu_ppc64le_test.go +++ b/virtcontainers/qemu_ppc64le_test.go @@ -41,12 +41,13 @@ func TestQemuPPC64leMemoryTopology(t *testing.T) { hostMem := uint64(1024) mem := uint64(120) + slots := uint8(10) expectedMemory := govmmQemu.Memory{ Size: fmt.Sprintf("%dM", mem), - Slots: defaultMemSlots, + Slots: slots, MaxMem: fmt.Sprintf("%dM", hostMem+uint64(memoryOffset)), } - m := ppc64le.memoryTopology(mem, hostMem) + m := ppc64le.memoryTopology(mem, hostMem, slots) assert.Equal(expectedMemory, m) } diff --git a/virtcontainers/qemu_test.go b/virtcontainers/qemu_test.go index b46e80645..f4b96ee72 100644 --- a/virtcontainers/qemu_test.go +++ b/virtcontainers/qemu_test.go @@ -151,11 +151,13 @@ func TestQemuCPUTopology(t *testing.T) { func TestQemuMemoryTopology(t *testing.T) { mem := uint32(1000) + slots := uint32(8) q := &qemu{ arch: &qemuArchBase{}, config: HypervisorConfig{ MemorySize: mem, + MemSlots: slots, }, } @@ -167,7 +169,7 @@ func TestQemuMemoryTopology(t *testing.T) { expectedOut := govmmQemu.Memory{ Size: fmt.Sprintf("%dM", mem), - Slots: defaultMemSlots, + Slots: uint8(slots), MaxMem: memMax, } diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index 4247e8df4..f46d778df 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -293,7 +293,8 @@ type Cmd struct { // Resources describes VM resources configuration. type Resources struct { // Memory is the amount of available memory in MiB. - Memory uint + Memory uint + MemorySlots uint8 } // SandboxStatus describes a sandbox status.