[PATCH 02/47] x86/cpu/cacheinfo: Move CPUHP_AP_CACHECTRL_STARTING to post stop machine

[Frederic Weisbecker]

Between CPUHP_TEARDOWN_CPU and CPUHP_AP_OFFLINE, stop_machine executes
all the teardown callbacks while stopping all CPUs in the system. This
doesn't mix-up well with running RT or isolated tasks.

In order to transition smoothly from global stop_machine to finer
grained synchronization between teardown callbacks and the other CPUs,
a new stop machine post function has been introduced to be executed
after the main one with IRQs still disabled while other CPUs can exit
stop_machine().

The cpu_cacheinfo_mask is only accessed by hotplug operations and
therefore the current CPU can be cleared from it while other CPUs
have exited hotplug stop machine.

Move this teardown callback to post stop_machine() accordingly.

Signed-off-by: Frederic Weisbecker <frederic@xxxxxxxxxx>
---
 kernel/cpu.c        | 21 +++++++++++++++++++--
 kernel/sched/core.c |  1 -
 2 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/kernel/cpu.c b/kernel/cpu.c
index 1ae03f1e8f84..c48cdff149f6 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -1275,7 +1275,7 @@ void clear_tasks_mm_cpumask(int cpu)
 static int take_cpu_down(void *_param)
 {
 	struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
-	enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE);
+	enum cpuhp_state target = max((int)st->target, CPUHP_AP_CACHECTRL_STARTING);
 	int err, cpu = smp_processor_id();
 
 	/* Ensure this CPU doesn't handle any more interrupts. */
@@ -1294,9 +1294,26 @@ static int take_cpu_down(void *_param)
 	 */
 	cpuhp_invoke_callback_range_nofail(false, cpu, st, target);
 
+	return 0;
+}
+
+static int take_cpu_down_post(void *_param)
+{
+	struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
+	enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE);
+	int cpu = smp_processor_id();
+
+	WARN_ON(st->state != CPUHP_AP_CACHECTRL_STARTING);
+
+	/*
+	 * Invoke the former CPU_DYING callbacks. DYING must not fail!
+	 */
+	cpuhp_invoke_callback_range_nofail(false, cpu, st, target);
+
 	/* Park the stopper thread */
 	stop_machine_park(cpu);
 	return 0;
+
 }
 
 static int takedown_cpu(unsigned int cpu)
@@ -1313,7 +1330,7 @@ static int takedown_cpu(unsigned int cpu)
 	 */
 	irq_lock_sparse();
 
-	err = stop_machine_cpuslocked(take_cpu_down, NULL, NULL,
+	err = stop_machine_cpuslocked(take_cpu_down, take_cpu_down_post, NULL,
 				      cpumask_of(cpu));
 	if (err) {
 		/* CPU refused to die */
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 04b62c16fa8c..eb6bbc610a06 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -6003,7 +6003,6 @@ static void sched_tick_stop(int cpu)
 	WARN_ON_ONCE(!tick_work_cpu);
 
 	twork = per_cpu_ptr(tick_work_cpu, cpu);
-	/* There cannot be competing actions, but don't rely on stop-machine. */
 	os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_OFFLINING);
 	WARN_ON_ONCE(os != TICK_SCHED_REMOTE_RUNNING);
 	/* Don't cancel, as this would mess up the state machine. */
-- 
2.53.0