[PATCH 1/2] powerpc/rtas: use device model APIs and serialization during LPM

[Nathan Lynch]

During LPAR migration, cpu hotplug and migration operations can
interleave like so:

cd /sys/devices/system/cpu/cpu7/ | drmgr -m -c pmig -p pre \
echo 0 > online                  | -s 0xd7a884f83d830e6d -t 19 \
echo 1 > online                  | -n -d 1 5
---------------------------------+-------------------------------------------
online_store() {                 |
  device_offline() {             |
    cpu_subsys_offline() {       |
      cpu_down(7);               |
    }                            |
    dev->offline = true;         |
  }                              | migration_store() {
}                                |   rtas_ibm_suspend_me() {
                                 |     rtas_online_cpus_mask() {
                                 |       cpu_up(7);
				 |     }
                                 |     cpu_hotplug_disable();
                                 |     on_each_cpu(rtas_percpu_suspend_me());
                                 |     cpu_hotplug_enable();
online_store() {                 |
  device_online() {              |
    cpu_subsys_online() {        |
      cpu_up(7);                 |
    }                            |
    dev->offline = false;        |
  }                              |     rtas_offline_cpus_mask() {
}                                |       rtas_cpu_state_change_mask() {
                                 |         cpu_down(7);
				 |       }
				 |     }
				 |   }
				 | }

This leaves cpu7 in a state where the driver core considers the cpu
device online, but in all other respects it is offline and
unused. Attempts to online the cpu via sysfs appear to succeed but the
driver core actually does not pass the request to the lower-level
cpuhp support code. This makes the cpu unusable until the system is
rebooted.

Instead of directly calling cpu_up/cpu_down, the migration code should
use the higher-level device core APIs to maintain consistent state and
serialize operations.

Fixes: 120496ac2d2d ("powerpc: Bring all threads online prior to migration/hibernation")
Signed-off-by: Nathan Lynch <nathanl at linux.ibm.com>
---
 arch/powerpc/kernel/rtas.c | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/arch/powerpc/kernel/rtas.c b/arch/powerpc/kernel/rtas.c
index 9b4d2a2ffb4f..fbefd9ff6dab 100644
--- a/arch/powerpc/kernel/rtas.c
+++ b/arch/powerpc/kernel/rtas.c
@@ -875,15 +875,17 @@ static int rtas_cpu_state_change_mask(enum rtas_cpu_state state,
 		return 0;
 
 	for_each_cpu(cpu, cpus) {
+		struct device *dev = get_cpu_device(cpu);
+
 		switch (state) {
 		case DOWN:
-			cpuret = cpu_down(cpu);
+			cpuret = device_offline(dev);
 			break;
 		case UP:
-			cpuret = cpu_up(cpu);
+			cpuret = device_online(dev);
 			break;
 		}
-		if (cpuret) {
+		if (cpuret < 0) {
 			pr_debug("%s: cpu_%s for cpu#%d returned %d.\n",
 					__func__,
 					((state == UP) ? "up" : "down"),
@@ -972,6 +974,8 @@ int rtas_ibm_suspend_me(u64 handle)
 	data.token = rtas_token("ibm,suspend-me");
 	data.complete = &done;
 
+	lock_device_hotplug();
+
 	/* All present CPUs must be online */
 	cpumask_andnot(offline_mask, cpu_present_mask, cpu_online_mask);
 	cpuret = rtas_online_cpus_mask(offline_mask);
@@ -1011,6 +1015,7 @@ int rtas_ibm_suspend_me(u64 handle)
 				__func__);
 
 out:
+	unlock_device_hotplug();
 	free_cpumask_var(offline_mask);
 	return atomic_read(&data.error);
 }
-- 
2.20.1