[RFC PATCH v2] powerpc: Make crashing cpu to be discovered first in kdump kernel.
Mahesh Salgaonkar
mahesh at linux.ibm.com
Sat Sep 9 22:42:07 AEST 2023
The kernel boot parameter 'nr_cpus=' allows one to specify number of
possible cpus in the system. In the normal scenario the first cpu (cpu0)
that shows up is the boot cpu and hence it gets covered under nr_cpus
limit.
But this assumption is broken in kdump scenario where kdump kernel after a
crash can boot up on an non-zero boot cpu. The paca structure allocation
depends on value of nr_cpus and is indexed using logical cpu ids. The cpu
discovery code brings up the cpus as they appear sequentially on device
tree and assigns logical cpu ids starting from 0. This definitely becomes
an issue if boot cpu id > nr_cpus. When this occurs it results into
In past there were proposals to fix this by making changes to cpu discovery
code to identify non-zero boot cpu and map it to logical cpu 0. However,
the changes were very invasive, making discovery code more complicated and
risky.
Considering that the non-zero boot cpu scenario is more specific to kdump
kernel, limiting the changes in panic/crash kexec path would probably be a
best approach to have.
Hence proposed change is, in crash kexec path, move the crashing cpu's
device node to the first position under '/cpus' node, which will make the
crashing cpu to be discovered as part of the first core in kdump kernel.
In order to accommodate boot cpu for the case where boot_cpuid >
nr_cpu_ids, align up the nr_cpu_ids to SMT threads in
early_init_dt_scan_cpus(). This will allow kdump kernel to work with
nr_cpus=X where X will be aligned up in multiple of SMT threads per core.
Signed-off-by: Mahesh Salgaonkar <mahesh at linux.ibm.com>
---
Change in RFCv2:
- Fix checkpatch.pl reported errors and build issues against few powerpc
configs.
- Move add_node_props() from file_load_64.c to core_64.c so that it can be
reused.
---
arch/powerpc/include/asm/kexec.h | 1
arch/powerpc/kernel/prom.c | 13 +++
arch/powerpc/kexec/core_64.c | 153 +++++++++++++++++++++++++++++++++++++
arch/powerpc/kexec/file_load_64.c | 27 -------
4 files changed, 167 insertions(+), 27 deletions(-)
diff --git a/arch/powerpc/include/asm/kexec.h b/arch/powerpc/include/asm/kexec.h
index a1ddba01e7d13..1f95be94e5b80 100644
--- a/arch/powerpc/include/asm/kexec.h
+++ b/arch/powerpc/include/asm/kexec.h
@@ -98,6 +98,7 @@ void relocate_new_kernel(unsigned long indirection_page, unsigned long reboot_co
unsigned long start_address) __noreturn;
void kexec_copy_flush(struct kimage *image);
+int add_node_props(void *fdt, int node_offset, const struct device_node *dn);
#if defined(CONFIG_CRASH_DUMP) && defined(CONFIG_PPC_RTAS)
void crash_free_reserved_phys_range(unsigned long begin, unsigned long end);
diff --git a/arch/powerpc/kernel/prom.c b/arch/powerpc/kernel/prom.c
index 0b5878c3125b1..02af26b01c2b5 100644
--- a/arch/powerpc/kernel/prom.c
+++ b/arch/powerpc/kernel/prom.c
@@ -322,6 +322,9 @@ static void __init check_cpu_feature_properties(unsigned long node)
}
}
+/* align nr_cpu on nthreads per core. adjust nr_cpus up */
+#define NR_CPU_ALIGN_UP(n, align_to) (((n)+((align_to)-1))&(~((typeof(n))(align_to)-1)))
+
static int __init early_init_dt_scan_cpus(unsigned long node,
const char *uname, int depth,
void *data)
@@ -348,6 +351,16 @@ static int __init early_init_dt_scan_cpus(unsigned long node,
nthreads = len / sizeof(int);
+ /*
+ * Align nr_cpu_ids to correct SMT value. This will help us to allocate
+ * pacas correctly to accommodate boot_cpu != 0 scenario e.g. in kdump
+ * kernel the boot cpu can be any cpu between 0 through nthreads.
+ */
+ if (nr_cpu_ids % nthreads) {
+ set_nr_cpu_ids(NR_CPU_ALIGN_UP(nr_cpu_ids, nthreads));
+ pr_info("Aligned nr_cpus to SMT=%d, nr_cpu_ids = %d\n", nthreads, nr_cpu_ids);
+ }
+
/*
* Now see if any of these threads match our boot cpu.
* NOTE: This must match the parsing done in smp_setup_cpu_maps.
diff --git a/arch/powerpc/kexec/core_64.c b/arch/powerpc/kexec/core_64.c
index a79e28c91e2be..0b5e2c6b51d64 100644
--- a/arch/powerpc/kexec/core_64.c
+++ b/arch/powerpc/kexec/core_64.c
@@ -17,6 +17,7 @@
#include <linux/cpu.h>
#include <linux/hardirq.h>
#include <linux/of.h>
+#include <linux/libfdt.h>
#include <asm/page.h>
#include <asm/current.h>
@@ -298,6 +299,144 @@ extern void kexec_sequence(void *newstack, unsigned long start,
void (*clear_all)(void),
bool copy_with_mmu_off) __noreturn;
+/**
+ * add_node_props - Reads node properties from device node structure and add
+ * them to fdt.
+ * @fdt: Flattened device tree of the kernel
+ * @node_offset: offset of the node to add a property at
+ * @dn: device node pointer
+ *
+ * Returns 0 on success, negative errno on error.
+ */
+int add_node_props(void *fdt, int node_offset, const struct device_node *dn)
+{
+ int ret = 0;
+ struct property *pp;
+
+ if (!dn)
+ return -EINVAL;
+
+ for_each_property_of_node(dn, pp) {
+ ret = fdt_setprop(fdt, node_offset, pp->name, pp->value, pp->length);
+ if (ret < 0) {
+ pr_err("Unable to add %s property: %s\n", pp->name, fdt_strerror(ret));
+ return ret;
+ }
+ }
+ return ret;
+}
+
+/*
+ * Move the crashing cpus FDT node as the first node under '/cpus' node.
+ *
+ * - Get the FDT segment from the crash image segments.
+ * - Locate the crashing CPUs fdt subnode 'A' under '/cpus' node.
+ * - Now locate the crashing cpu device node 'B' from of_root device tree.
+ * - Delete the crashing cpu FDT node 'A' from kexec FDT segment.
+ * - Insert the crashing cpu device node 'B' into kexec FDT segment as first
+ * subnode under '/cpus' node.
+ */
+static void move_crashing_cpu(struct kimage *image)
+{
+ void *fdt, *ptr;
+ const char *pathp = NULL;
+ unsigned long mem;
+ const __be32 *intserv;
+ struct device_node *dn;
+ bool first_node = true;
+ int cpus_offset, offset, fdt_index = -1;
+ int initial_depth, depth, len, i, ret, nthreads;
+
+ /* Find the FDT segment index in kexec segment array. */
+ for (i = 0; i < image->nr_segments; i++) {
+ mem = image->segment[i].mem;
+ ptr = __va(mem);
+ if (ptr && fdt_magic(ptr) == FDT_MAGIC) {
+ fdt_index = i;
+ break;
+ }
+ }
+ if (fdt_index < 0) {
+ pr_err("Unable to locate FDT segment.\n");
+ return;
+ }
+
+ fdt = __va((void *)image->segment[fdt_index].mem);
+
+ offset = cpus_offset = fdt_path_offset(fdt, "/cpus");
+ if (cpus_offset < 0) {
+ if (cpus_offset != -FDT_ERR_NOTFOUND)
+ pr_err("Malformed device tree: error reading /cpus node: %s\n",
+ fdt_strerror(cpus_offset));
+ return;
+ }
+
+ /* Locate crashing cpus fdt node */
+ initial_depth = depth = 0;
+ for (offset = fdt_next_node(fdt, offset, &depth);
+ offset >= 0 && depth > initial_depth;
+ offset = fdt_next_node(fdt, offset, &depth)) {
+
+
+ intserv = fdt_getprop(fdt, offset, "ibm,ppc-interrupt-server#s", &len);
+ if (!intserv) {
+ pr_err("Unable to fetch ibm,ppc-interrupt-server#s property\n");
+ return;
+ }
+
+ /* Find the match for crashing cpus phys id. */
+ nthreads = len / sizeof(int);
+ for (i = 0; i < nthreads; i++) {
+ if (be32_to_cpu(intserv[i]) == get_paca()->hw_cpu_id)
+ break;
+ }
+ if (i < nthreads) {
+ /* Found the match */
+ pathp = fdt_get_name(fdt, offset, NULL);
+ break;
+ }
+
+ first_node = false;
+ }
+
+ /*
+ * Nothing to be done if crashing cpu's fdt node is already at first
+ * position OR crashing cpu's fdt node isn't present in kexec FDT
+ * segment, which is not possible unless kexec FDT segment hasn't been
+ * refreshed after DLPAR.
+ */
+ if (first_node || offset < 0)
+ return;
+
+ /* Locate the device node of crashing cpu from of_root */
+ for_each_node_by_type(dn, "cpu") {
+ if (!strcmp(dn->full_name, pathp))
+ break;
+ }
+ if (!dn) {
+ pr_err("Could not locate device node of crashing cpu: %s\n", pathp);
+ return;
+ }
+
+ /* Delete the crashing cpu FDT node from kexec FDT segment */
+ ret = fdt_del_node(fdt, offset);
+ if (ret < 0) {
+ pr_err("Error deleting node /cpus/%s: %s\n", pathp, fdt_strerror(ret));
+ return;
+ }
+
+ /* Add it as first subnode under /cpus node. */
+ offset = fdt_add_subnode(fdt, cpus_offset, dn->full_name);
+ if (offset < 0) {
+ pr_err("Unable to add %s subnode: %s\n", dn->full_name,
+ fdt_strerror(offset));
+ return;
+ }
+
+ /* Copy rest of the properties of crashing cpus */
+ add_node_props(fdt, offset, dn);
+}
+
/* too late to fail here */
void default_machine_kexec(struct kimage *image)
{
@@ -341,6 +480,20 @@ void default_machine_kexec(struct kimage *image)
printk("kexec: Unshared all shared pages.\n");
}
+ /*
+ * Move the crashing cpus FDT node as the first node under /cpus node.
+ * This will make the core (where crashing cpu belongs) to
+ * automatically become first core to show up in kdump kernel and
+ * crashing cpu as boot cpu within first n threads of that core.
+ *
+ * Currently this will work with kexec_file_load only.
+ *
+ * XXX: For kexec_load, change is required in kexec tool to exclude FDT
+ * segment from purgatory checksum check.
+ */
+ if (image->type == KEXEC_TYPE_CRASH && image->file_mode)
+ move_crashing_cpu(image);
+
paca_ptrs[kexec_paca.paca_index] = &kexec_paca;
setup_paca(&kexec_paca);
diff --git a/arch/powerpc/kexec/file_load_64.c b/arch/powerpc/kexec/file_load_64.c
index 110d28bede2a7..a1ab070a3a38c 100644
--- a/arch/powerpc/kexec/file_load_64.c
+++ b/arch/powerpc/kexec/file_load_64.c
@@ -1018,33 +1018,6 @@ unsigned int kexec_extra_fdt_size_ppc64(struct kimage *image)
return extra_size;
}
-/**
- * add_node_props - Reads node properties from device node structure and add
- * them to fdt.
- * @fdt: Flattened device tree of the kernel
- * @node_offset: offset of the node to add a property at
- * @dn: device node pointer
- *
- * Returns 0 on success, negative errno on error.
- */
-static int add_node_props(void *fdt, int node_offset, const struct device_node *dn)
-{
- int ret = 0;
- struct property *pp;
-
- if (!dn)
- return -EINVAL;
-
- for_each_property_of_node(dn, pp) {
- ret = fdt_setprop(fdt, node_offset, pp->name, pp->value, pp->length);
- if (ret < 0) {
- pr_err("Unable to add %s property: %s\n", pp->name, fdt_strerror(ret));
- return ret;
- }
- }
- return ret;
-}
-
/**
* update_cpus_node - Update cpus node of flattened device tree using of_root
* device node.
More information about the Linuxppc-dev
mailing list