diff --git a/tools/packaging/kernel/kata_config_version b/tools/packaging/kernel/kata_config_version index f96ac0672..fe4afb0df 100644 --- a/tools/packaging/kernel/kata_config_version +++ b/tools/packaging/kernel/kata_config_version @@ -1 +1 @@ -105 +106 diff --git a/tools/packaging/kernel/patches/5.10.x/dragonball-experimental/0005-upcall-dragonball-devmgr-suppots-cpu-hotplug-on-arm6.patch b/tools/packaging/kernel/patches/5.10.x/dragonball-experimental/0005-upcall-dragonball-devmgr-suppots-cpu-hotplug-on-arm6.patch new file mode 100644 index 000000000..74dcc732e --- /dev/null +++ b/tools/packaging/kernel/patches/5.10.x/dragonball-experimental/0005-upcall-dragonball-devmgr-suppots-cpu-hotplug-on-arm6.patch @@ -0,0 +1,163 @@ +From 16e3b3da9fb8b79b006d8c9d1f68b2dec9980d72 Mon Sep 17 00:00:00 2001 +Message-Id: <16e3b3da9fb8b79b006d8c9d1f68b2dec9980d72.1685428663.git.jiyunxue@linux.alibaba.com> +From: xuejun-xj +Date: Wed, 10 May 2023 13:55:43 +0800 +Subject: [PATCH 1/3] upcall: dragonball-devmgr suppots cpu hotplug on arm64 + +Enable vcpuhotplug feature on aarch64 in guest kernel. It communicates +with dragonball by using upcall. This commit does these changes: + +1. Wraps x86 related fields with CONFIG_X86_64. +2. Add "cpu_event_notification" for arm64. +3. Add "add_cpu_dev" and "del_cpu_dev" for arm64. + +Signed-off-by: xuejun-xj +Reviewed-by : Chao Wu +Reviewed-by: Zizheng Bian +Reviewed-by: Baolin Wang +--- + .../upcall_srv/dragonball_device_manager.c | 84 ++++++++++++++++++- + 1 file changed, 81 insertions(+), 3 deletions(-) + +diff --git a/drivers/misc/dragonball/upcall_srv/dragonball_device_manager.c b/drivers/misc/dragonball/upcall_srv/dragonball_device_manager.c +index 5a95b2ba63e8..088d38623b8d 100644 +--- a/drivers/misc/dragonball/upcall_srv/dragonball_device_manager.c ++++ b/drivers/misc/dragonball/upcall_srv/dragonball_device_manager.c +@@ -85,15 +85,21 @@ struct devmgr_req { + #if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) + struct { + uint8_t count; ++#ifdef CONFIG_X86_64 + uint8_t apic_ver; + uint8_t apic_ids[256]; ++#endif + } cpu_dev_info; + #endif + } msg_load; + }; + + struct cpu_dev_reply_info { ++#if defined(CONFIG_X86_64) + uint32_t apic_index; ++#elif defined(CONFIG_ARM64) ++ uint32_t cpu_id; ++#endif + }; + + struct devmgr_reply { +@@ -190,7 +196,8 @@ static void _fill_msg_header(struct devmgr_msg_header *msg, uint32_t msg_size, + msg->msg_flags = msg_flags; + } + +-#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) && defined(CONFIG_X86_64) ++#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) ++#if defined(CONFIG_X86_64) + static int get_cpu_id(int apic_id) + { + int i; +@@ -219,6 +226,24 @@ static void cpu_event_notification( + _fill_msg_header(&rep->msg_header, + sizeof(struct cpu_dev_reply_info), action_type, 0); + } ++#elif defined(CONFIG_ARM64) ++/** ++ * Return the first failed hotplug index of the cpu_id to dragonball. ++ * If hotplug/hotunplug succeeds, it will equals to the expected cpu count. ++ */ ++static void cpu_event_notification( ++ uint8_t cpu_id, ++ int ret, ++ uint32_t action_type, ++ struct devmgr_reply *rep) ++{ ++ pr_info("cpu event notification: cpu_id %d\n", cpu_id); ++ rep->msg_load.cpu_dev_info.cpu_id = cpu_id; ++ rep->ret = ret; ++ _fill_msg_header(&rep->msg_header, ++ sizeof(struct cpu_dev_reply_info), action_type, 0); ++} ++#endif + #endif + + #if defined(CONFIG_DRAGONBALL_HOTPLUG_VIRTIO_MMIO) +@@ -262,7 +287,8 @@ static int del_mmio_dev(struct devmgr_req *req, + #endif + + +-#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) && defined(CONFIG_X86_64) ++#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) ++#if defined(CONFIG_X86_64) + static int add_cpu_upcall(int apic_id, uint8_t apic_ver) + { + int cpu_id, node_id; +@@ -430,6 +456,58 @@ static int del_cpu_dev(struct devmgr_req *req, + cpu_event_notification(i, ret, DEL_CPU, rep); + return ret; + } ++#elif defined(CONFIG_ARM64) ++static int add_cpu_dev(struct devmgr_req *req, struct devmgr_reply *rep) ++{ ++ int i, ret = 0; ++ unsigned int cpu_id, nr_online_cpus; ++ uint8_t count = req->msg_load.cpu_dev_info.count; ++ ++ nr_online_cpus = num_online_cpus(); ++ ++ pr_info("Current vcpu number: %d, Add vcpu number: %d\n", ++ nr_online_cpus, count); ++ ++ for (i = 0; i < count; ++i) { ++ cpu_id = nr_online_cpus + i; ++ ret = add_cpu(cpu_id); ++ if (ret != 0) ++ break; ++ } ++ ++ cpu_event_notification(nr_online_cpus + i, ret, ADD_CPU, rep); ++ return ret; ++} ++ ++static int del_cpu_dev(struct devmgr_req *req, struct devmgr_reply *rep) ++{ ++ int i, ret = 0; ++ unsigned int cpu_id, nr_online_cpus; ++ uint8_t count = req->msg_load.cpu_dev_info.count; ++ ++ nr_online_cpus = num_online_cpus(); ++ ++ pr_info("Current vcpu number: %d, Delete vcpu number: %d\n", ++ nr_online_cpus, count); ++ ++ if (count >= nr_online_cpus) { ++ pr_err("cpu del parameter check error: cannot remove all vcpus\n"); ++ ret = -EINVAL; ++ cpu_event_notification(0, ret, DEL_CPU, rep); ++ return ret; ++ } ++ ++ for (i = 0; i < count; ++i) { ++ cpu_id = nr_online_cpus - i - 1; ++ ret = remove_cpu(cpu_id); ++ if (ret != 0) ++ break; ++ } ++ ++ cpu_event_notification(nr_online_cpus - i, ret, DEL_CPU, rep); ++ return ret; ++} ++#endif + #endif + + static struct { +@@ -440,7 +518,7 @@ static struct { + {ADD_MMIO, add_mmio_dev}, + {DEL_MMIO, del_mmio_dev}, + #endif +-#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) && defined(CONFIG_X86_64) ++#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) + {ADD_CPU, add_cpu_dev}, + {DEL_CPU, del_cpu_dev}, + #endif +-- +2.28.0 + diff --git a/tools/packaging/kernel/patches/5.10.x/dragonball-experimental/0006-msi-control-msi-irq-number-activated.patch b/tools/packaging/kernel/patches/5.10.x/dragonball-experimental/0006-msi-control-msi-irq-number-activated.patch new file mode 100644 index 000000000..b40cf6666 --- /dev/null +++ b/tools/packaging/kernel/patches/5.10.x/dragonball-experimental/0006-msi-control-msi-irq-number-activated.patch @@ -0,0 +1,67 @@ +From 6e07ca77fe7b5c15e0e98d9e86294c7dd2553a5a Mon Sep 17 00:00:00 2001 +Message-Id: <6e07ca77fe7b5c15e0e98d9e86294c7dd2553a5a.1685428663.git.jiyunxue@linux.alibaba.com> +In-Reply-To: <16e3b3da9fb8b79b006d8c9d1f68b2dec9980d72.1685428663.git.jiyunxue@linux.alibaba.com> +References: <16e3b3da9fb8b79b006d8c9d1f68b2dec9980d72.1685428663.git.jiyunxue@linux.alibaba.com> +From: xuejun-xj +Date: Wed, 10 May 2023 14:51:40 +0800 +Subject: [PATCH 2/3] msi: control msi irq number activated + +When passthroughing pci device, kernel will initialize and activate +(max_cpu_count+1) msi irq. However, in vcpu hotplugging situation, +because of vgic, max_cpu_count may be greater than online_cpu_count. +Those offline cpus will also be activated by kernel, which cause failure +of passthroughing pci device. + +To solve this problem, this patch add a function +"check_affinity_mask_online" to check if msi_desc->affinity contains +online cpus. If current cpu is offline, it will continue the for loop to +skip activating related irq. + +Signed-off-by: xuejun-xj +Reviewed-by: Shuo Tan +Reviewed-by: Baolin Wang +--- + kernel/irq/msi.c | 20 ++++++++++++++++++++ + 1 file changed, 20 insertions(+) + +diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c +index d924676c8781..d60a3fc654e6 100644 +--- a/kernel/irq/msi.c ++++ b/kernel/irq/msi.c +@@ -395,6 +395,23 @@ static bool msi_check_reservation_mode(struct irq_domain *domain, + return desc->msi_attrib.is_msix || desc->msi_attrib.maskbit; + } + ++/* This function is used for check whether the cpu affinity belongs to the ++ * online cpus. When we passthrough the nvme devices, the kernel will allocate ++ * maxcpus+1 MSI irqs and then activate them. In vcpu hotplug situations, it ++ * may happen that kernel activates the offline cpus when bootcpus < maxcpus. ++ * To avoid this conflict, this function check the affinities. ++ */ ++static inline bool check_affinity_mask_online(struct irq_affinity_desc *affinity) ++{ ++ int cpu; ++ ++ for_each_cpu(cpu, &affinity->mask) ++ if (cpu_online(cpu)) ++ return true; ++ ++ return false; ++} ++ + int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, + int nvec) + { +@@ -445,6 +462,9 @@ int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, + goto skip_activate; + + for_each_msi_vector(desc, i, dev) { ++ if (desc->affinity ++ && !check_affinity_mask_online(desc->affinity)) ++ continue; + if (desc->irq == i) { + virq = desc->irq; + dev_dbg(dev, "irq [%d-%d] for MSI\n", +-- +2.28.0 + diff --git a/tools/packaging/kernel/patches/5.10.x/dragonball-experimental/0007-smp-update-bringup_nonboot_cpus-parameters.patch b/tools/packaging/kernel/patches/5.10.x/dragonball-experimental/0007-smp-update-bringup_nonboot_cpus-parameters.patch new file mode 100644 index 000000000..71d4fb977 --- /dev/null +++ b/tools/packaging/kernel/patches/5.10.x/dragonball-experimental/0007-smp-update-bringup_nonboot_cpus-parameters.patch @@ -0,0 +1,139 @@ +From a05086142be13d43c7fc92500bcb870a2f37e485 Mon Sep 17 00:00:00 2001 +Message-Id: +In-Reply-To: <16e3b3da9fb8b79b006d8c9d1f68b2dec9980d72.1685428663.git.jiyunxue@linux.alibaba.com> +References: <16e3b3da9fb8b79b006d8c9d1f68b2dec9980d72.1685428663.git.jiyunxue@linux.alibaba.com> +From: xuejun-xj +Date: Tue, 23 May 2023 09:43:02 +0800 +Subject: [PATCH 3/3] smp: update bringup_nonboot_cpus parameters + +On aarch64, kvm doesn't allow vmm to call KVM_CREATE_VCPU ioctls after +vm has already started, which is caused by vgic_initialized check in +kvm_arch_vcpu_precreate() function. Therefore, to support vcpu hotplug +feature on aarch64, all the vcpus should be created and configured ready +for start at booting procedure. + +To solve the problem, dragonball will add a property in each cpu node, +called "boot-onlined". This property indicates whether this cpu should +be onlined at first boot. It has two values: 0 and 1. 0 means offline, +while 1 means online. + +This commit also add a helper function called "of_get_cpu_boot_onlined", +which parse the cpu node and get the value of boot-onlined property. +Then update the global variable "boot_onlined_cpu". + +When kernel calling smp_init(), bringup_nonboot_cpus will start all the +other cpus except cpu0. The activated cpu number equals setup_max_cpus. +In vcpu hotplug scenario, vmm will create all the vcpufd before vm is +initialized, while activating only a few vcpus at first boot. The +setup_max_cpus variable will be initialized as all vcpu count. This +cause that the other cpus cannot find enough cpu threads, and they will +wait for 5 seconds each cpu. + +Therefore, we use boot_onlined_cpu instead of setup_max_cpus to give +"bringup_nonboot_cpus" correct cpu number it needs. + +Signed-off-by: xuejun-xj +--- + .../devicetree/bindings/arm/cpus.yaml | 11 +++++++++ + arch/arm64/kernel/smp.c | 24 +++++++++++++++++++ + kernel/smp.c | 10 +++++++- + 3 files changed, 44 insertions(+), 1 deletion(-) + +diff --git a/Documentation/devicetree/bindings/arm/cpus.yaml b/Documentation/devicetree/bindings/arm/cpus.yaml +index 14cd727d3c4b..691bb352d842 100644 +--- a/Documentation/devicetree/bindings/arm/cpus.yaml ++++ b/Documentation/devicetree/bindings/arm/cpus.yaml +@@ -316,6 +316,17 @@ properties: + formed by encoding the target CPU id into the low bits of the + physical start address it should jump to. + ++ boot-onlined: ++ $ref: '/schemas/types.yaml#/definitions/uint32' ++ description: | ++ The boot-onlined property is an optional u32 value that indicates ++ whether the cpu device should be activated at first boot. This is ++ useful in vcpu hotplug scenario to pass correct value of activated ++ cpu number. ++ ++ This property has two values: 0 and 1. 1 means the cpu should be ++ activated while 0 means it shouldn't. ++ + if: + # If the enable-method property contains one of those values + properties: +diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c +index 18e9727d3f64..5db8041929a6 100644 +--- a/arch/arm64/kernel/smp.c ++++ b/arch/arm64/kernel/smp.c +@@ -464,6 +464,27 @@ void __init smp_prepare_boot_cpu(void) + init_gic_priority_masking(); + } + ++#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) && defined(CONFIG_ARM64) ++extern unsigned int boot_onlined_cpu; ++static void __init of_get_cpu_boot_onlined(struct device_node *dn) ++{ ++ unsigned int boot_onlined; ++ int r; ++ ++ r = of_property_read_u32(dn, "boot-onlined", &boot_onlined); ++ if (r) { ++ pr_err("%pOF: missing boot-onlined property\n", dn); ++ return; ++ } ++ /* ++ * Property boot-onlined has two values: 0 and 1. ++ * 0 means offline, and 1 means online. ++ * Here just count the number of boot_onlined_cpu. ++ */ ++ boot_onlined_cpu += boot_onlined; ++} ++#endif ++ + static u64 __init of_get_cpu_mpidr(struct device_node *dn) + { + const __be32 *cell; +@@ -654,6 +675,9 @@ static void __init of_parse_and_init_cpus(void) + struct device_node *dn; + + for_each_of_cpu_node(dn) { ++#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) && defined(CONFIG_ARM64) ++ of_get_cpu_boot_onlined(dn); ++#endif + u64 hwid = of_get_cpu_mpidr(dn); + + if (hwid == INVALID_HWID) +diff --git a/kernel/smp.c b/kernel/smp.c +index 25240fb2df94..567615b9a008 100644 +--- a/kernel/smp.c ++++ b/kernel/smp.c +@@ -801,17 +801,25 @@ void __init setup_nr_cpu_ids(void) + nr_cpu_ids = find_last_bit(cpumask_bits(cpu_possible_mask),NR_CPUS) + 1; + } + ++/* Setup number of CPUs to activate */ ++unsigned int boot_onlined_cpu = 0; ++ + /* Called by boot processor to activate the rest. */ + void __init smp_init(void) + { + int num_nodes, num_cpus; ++ int num_onlined_cpu = setup_max_cpus; + + idle_threads_init(); + cpuhp_threads_init(); + + pr_info("Bringing up secondary CPUs ...\n"); + +- bringup_nonboot_cpus(setup_max_cpus); ++#if defined(CONFIG_DRAGONBALL_HOTPLUG_CPU) && defined(CONFIG_ARM64) ++ if (boot_onlined_cpu != 0) ++ num_onlined_cpu = boot_onlined_cpu; ++#endif ++ bringup_nonboot_cpus(num_onlined_cpu); + + num_nodes = num_online_nodes(); + num_cpus = num_online_cpus(); +-- +2.28.0 +