Re: [PATCH v4 2/9] stop_machine: Add NMI-based execution path
From: Peter Zijlstra
Date: Thu Nov 19 2026 - 08:03:11 EST
On Wed, Nov 04, 2026 at 06:23:15PM +0000, Chang S. Bae wrote:
> The x86 microcode loader is currently the primary user for this. But
> other architectures are not expected to use it. Add a build option to
> make it opt-in.
FWIW, that's disgusting! The micro-code that loads the micro-code should
damn well disable all exceptions itself, it can manage the NMI state
much saner than we can, but also SMI and god knows what else.
Anyway, how about something a little like this completely untested
below?
I didn't bother with the noinstr, but it shouldn't be too hard to add
that back in I suppose. But I see very little reason for it, since (1)
see above, micro-code should do this, and (2) see the CONFIG_DEBUG_ENTRY
bit in entry_64.S right above repeat_nmi:.
---
arch/x86/include/asm/stop_machine.h | 20 +++++++
arch/x86/kernel/irq_work.c | 17 +++++-
arch/x86/kernel/nmi.c | 4 ++
kernel/stop_machine.c | 105 ++++++++++++++++++++++++++++++++++--
4 files changed, 140 insertions(+), 6 deletions(-)
diff --git a/arch/x86/include/asm/stop_machine.h b/arch/x86/include/asm/stop_machine.h
new file mode 100644
index 000000000000..c178aa1d0697
--- /dev/null
+++ b/arch/x86/include/asm/stop_machine.h
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_STOP_MACHINE_H
+#define _ASM_STOP_MACHINE_H
+
+#include <linux/static_call_types.h>
+#include <asm/irq_work.h>
+
+extern void stop_machine_ipi_proto(void);
+
+DECLARE_STATIC_CALL(stop_machine_ipi, stop_machine_ipi_proto);
+
+static inline bool arch_stop_machine_has_ipi(void)
+{
+ return arch_irq_work_has_interrupt();
+}
+#define arch_stop_machine_has_ipi arch_stop_machine_has_ipi
+
+extern void arch_stop_machine_ipi_raise(bool nmi);
+
+#endif /* _ASM_STOP_MACHINE_H */
diff --git a/arch/x86/kernel/irq_work.c b/arch/x86/kernel/irq_work.c
index 308c62411ff4..7ef8ceff4236 100644
--- a/arch/x86/kernel/irq_work.c
+++ b/arch/x86/kernel/irq_work.c
@@ -12,6 +12,8 @@
#include <asm/idtentry.h>
#include <asm/trace/irq_vectors.h>
#include <linux/interrupt.h>
+#include <asm/stop_machine.h>
+#include <linux/static_call.h>
#ifdef CONFIG_X86_LOCAL_APIC
DEFINE_IDTENTRY_SYSVEC(sysvec_irq_work)
@@ -19,16 +21,27 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_irq_work)
apic_eoi();
trace_irq_work_entry(IRQ_WORK_VECTOR);
inc_irq_stat(IRQ_WORK);
+ static_call_cond(stop_machine_ipi)();
irq_work_run();
trace_irq_work_exit(IRQ_WORK_VECTOR);
}
+static inline void __arch_irq_work_raise(int vector)
+{
+ __apic_send_IPI_self(vector);
+ apic_wait_icr_idle();
+}
+
void arch_irq_work_raise(void)
{
if (!arch_irq_work_has_interrupt())
return;
- __apic_send_IPI_self(IRQ_WORK_VECTOR);
- apic_wait_icr_idle();
+ __arch_irq_work_raise(IRQ_WORK_VECTOR);
+}
+
+void arch_stop_machine_ipi_raise(bool nmi)
+{
+ __arch_irq_work_raise(nmi ? NMI_VECTOR : IRQ_WORK_VECTOR);
}
#endif
diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c
index cb46da482c85..6389437862f2 100644
--- a/arch/x86/kernel/nmi.c
+++ b/arch/x86/kernel/nmi.c
@@ -25,6 +25,7 @@
#include <linux/atomic.h>
#include <linux/sched/clock.h>
#include <linux/kvm_types.h>
+#include <linux/static_call.h>
#include <asm/cpu_entry_area.h>
#include <asm/traps.h>
@@ -37,6 +38,7 @@
#include <asm/microcode.h>
#include <asm/sev.h>
#include <asm/fred.h>
+#include <asm/stop_machine.h>
#define CREATE_TRACE_POINTS
#include <trace/events/nmi.h>
@@ -390,6 +392,8 @@ static noinstr void default_do_nmi(struct pt_regs *regs)
instrumentation_begin();
+ static_call_cond(stop_machine_ipi)();
+
if (microcode_nmi_handler_enabled() && microcode_nmi_handler())
goto out;
diff --git a/kernel/stop_machine.c b/kernel/stop_machine.c
index d085ba1f4b44..a11ce3181c61 100644
--- a/kernel/stop_machine.c
+++ b/kernel/stop_machine.c
@@ -23,6 +23,8 @@
#include <linux/atomic.h>
#include <linux/nmi.h>
#include <linux/sched/wake_q.h>
+#include <linux/static_call.h>
+#include <asm/stop_machine.h>
/*
* Structure to determine completion condition and record errors. May
@@ -45,6 +47,9 @@ struct cpu_stopper {
struct cpu_stop_work stop_work; /* for stop_cpus */
unsigned long caller;
cpu_stop_fn_t fn;
+
+ struct multi_stop_data *msdata;
+ int ret;
};
static DEFINE_PER_CPU(struct cpu_stopper, cpu_stopper);
@@ -162,6 +167,8 @@ enum multi_stop_state {
MULTI_STOP_DISABLE_IRQ,
/* Run the function */
MULTI_STOP_RUN,
+ /* Cleanup */
+ MULTI_STOP_CLEANUP,
/* Exit */
MULTI_STOP_EXIT,
};
@@ -175,6 +182,7 @@ struct multi_stop_data {
enum multi_stop_state state;
atomic_t thread_ack;
+ bool nmi;
};
static void set_state(struct multi_stop_data *msdata,
@@ -182,8 +190,7 @@ static void set_state(struct multi_stop_data *msdata,
{
/* Reset ack counter. */
atomic_set(&msdata->thread_ack, msdata->num_threads);
- smp_wmb();
- WRITE_ONCE(msdata->state, newstate);
+ smp_store_release(&msdata->state, newstate);
}
/* Last one to ack a state moves to the next state. */
@@ -198,6 +205,50 @@ notrace void __weak stop_machine_yield(const struct cpumask *cpumask)
cpu_relax();
}
+static void stop_machine_ipi_func(void);
+
+DEFINE_STATIC_CALL_NULL(stop_machine_ipi, stop_machine_ipi_func);
+
+static void stop_machine_ipi_prepare(bool is_first, bool is_active)
+{
+ struct cpu_stopper *stopper = this_cpu_ptr(&cpu_stopper);
+
+ stopper->ret = !!is_active;
+ if (is_first)
+ static_call_update(stop_machine_ipi, stop_machine_ipi_func);
+}
+
+#ifndef arch_stop_machine_has_ipi
+static inline bool stop_machine_has_ipi(void)
+{
+ return false;
+}
+#endif
+
+static void stop_machine_ipi_raise(struct multi_stop_data *msdata)
+{
+ struct cpu_stopper *stopper = this_cpu_ptr(&cpu_stopper);
+
+ scoped_guard(irq) {
+ stopper->msdata = msdata;
+ arch_stop_machine_ipi_raise(msdata->nmi);
+ }
+
+ /* Wait for IPI to hit. */
+ while (READ_ONCE(stopper->msdata))
+ cpu_relax();
+}
+
+static int stop_machine_ipi_cleanup(bool is_first)
+{
+ struct cpu_stopper *stopper = this_cpu_ptr(&cpu_stopper);
+
+ if (is_first)
+ static_call_update(stop_machine_ipi, NULL);
+
+ return stopper->ret;
+}
+
/* This is the cpu_stop function which stops the CPU. */
static int multi_cpu_stop(void *data)
{
@@ -205,8 +256,8 @@ static int multi_cpu_stop(void *data)
enum multi_stop_state newstate, curstate = MULTI_STOP_NONE;
int cpu = smp_processor_id(), err = 0;
const struct cpumask *cpumask;
+ bool is_active, is_first, ipi;
unsigned long flags;
- bool is_active;
/*
* When called from stop_machine_from_inactive_cpu(), irq might
@@ -216,21 +267,34 @@ static int multi_cpu_stop(void *data)
if (!msdata->active_cpus) {
cpumask = cpu_online_mask;
- is_active = cpu == cpumask_first(cpumask);
+ is_first = cpu == cpumask_first(cpumask);
+ is_active = is_first;
} else {
cpumask = msdata->active_cpus;
+ is_first = cpu == cpumask_first(cpumask);
is_active = cpumask_test_cpu(cpu, cpumask);
}
+ ipi = arch_stop_machine_has_ipi();
+
/* Simple state machine */
do {
/* Chill out and ensure we re-read multi_stop_state. */
+again:
stop_machine_yield(cpumask);
newstate = READ_ONCE(msdata->state);
if (newstate != curstate) {
curstate = newstate;
switch (curstate) {
+ case MULTI_STOP_PREPARE:
+ if (ipi)
+ stop_machine_ipi_prepare(is_first, is_active);
+ break;
case MULTI_STOP_DISABLE_IRQ:
+ if (ipi) {
+ stop_machine_ipi_raise(msdata);
+ goto again;
+ }
local_irq_disable();
hard_irq_disable();
break;
@@ -238,6 +302,10 @@ static int multi_cpu_stop(void *data)
if (is_active)
err = msdata->fn(msdata->data);
break;
+ case MULTI_STOP_CLEANUP:
+ if (ipi)
+ err = stop_machine_ipi_cleanup(is_first);
+ break;
default:
break;
}
@@ -258,6 +326,35 @@ static int multi_cpu_stop(void *data)
return err;
}
+static void stop_machine_ipi_func(void)
+{
+ struct cpu_stopper *stopper = this_cpu_ptr(&cpu_stopper);
+ struct multi_stop_data *msdata = stopper->msdata;
+ enum multi_stop_state newstate, curstate = MULTI_STOP_NONE;
+
+ if (!msdata)
+ return;
+
+ do {
+ cpu_relax();
+ newstate = READ_ONCE(msdata->state);
+ if (newstate != curstate) {
+ curstate = newstate;
+ switch (curstate) {
+ case MULTI_STOP_RUN:
+ if (stopper->ret)
+ stopper->ret = msdata->fn(msdata->data);
+ break;
+ default:
+ break;
+ }
+ }
+ ack_state(msdata);
+ } while (curstate <= MULTI_STOP_RUN);
+
+ stopper->msdata = NULL;
+}
+
static int cpu_stop_queue_two_works(int cpu1, struct cpu_stop_work *work1,
int cpu2, struct cpu_stop_work *work2)
{