[PATCH 2/2] printk: Defer legacy console flushes while a CPU runs a stopper callback

[Aditya Chillara]

The cpu stopper thread runs above every other scheduling class, so while a
stopper callback executes, nothing else on that CPU is scheduled. If such a
callback emits a normal-priority printk(), the legacy console path can
synchronously flush the pending console backlog. On systems with a slow
UART and a large backlog, this holds the CPU long enough to starve RT
kthreads such as the watchdog pet, and for multi_cpu_stop() prevents the
CPU from advancing the state machine while the other CPUs wait. The
resulting delay can prevent watchdog servicing long enough to trigger a
watchdog bark or bite.

The same can happen from an interrupt taken during the callback:
multi_cpu_stop() keeps interrupts enabled during MULTI_STOP_PREPARE, and a
printk() may be emitted from a softirq run on irq exit.

Defer normal-priority legacy console output while a CPU is executing a
stopper callback (in_cpu_stop()), and reschedule any pending output after
the callback exits. Keep emergency and panic console behavior unchanged.

Assisted-by: LLM
Signed-off-by: Aditya Chillara <aditya.chillara@xxxxxxxxxxxxxxxx>
---
 include/linux/printk.h   |  5 +++++
 kernel/printk/internal.h |  9 ++++++++-
 kernel/printk/printk.c   | 23 +++++++++++++++++++++++
 kernel/stop_machine.c    |  7 +++++++
 4 files changed, 43 insertions(+), 1 deletion(-)

diff --git a/include/linux/printk.h b/include/linux/printk.h
index f594c1266bfd..4f3fb4ef5b92 100644
--- a/include/linux/printk.h
+++ b/include/linux/printk.h
@@ -166,6 +166,7 @@ __printf(1, 2) __cold int _printk_deferred(const char *fmt, ...);
 
 extern void __printk_deferred_enter(void);
 extern void __printk_deferred_exit(void);
+void printk_defer_console_output(void);
 
 extern void printk_force_console_enter(void);
 extern void printk_force_console_exit(void);
@@ -239,6 +240,10 @@ static inline void printk_deferred_exit(void)
 {
 }
 
+static inline void printk_defer_console_output(void)
+{
+}
+
 static inline void printk_force_console_enter(void)
 {
 }
diff --git a/kernel/printk/internal.h b/kernel/printk/internal.h
index 85fbf1801cbe..8c02f93e798c 100644
--- a/kernel/printk/internal.h
+++ b/kernel/printk/internal.h
@@ -3,6 +3,7 @@
  * internal.h - printk internal definitions
  */
 #include <linux/console.h>
+#include <linux/stop_machine.h>
 #include <linux/types.h>
 #include <linux/sysctl.h>
 
@@ -206,7 +207,13 @@ static inline void printk_get_console_flush_type(struct console_flush_type *ft)
 
 		/* Legacy consoles are flushed directly when possible. */
 		if (have_legacy_console || have_boot_console) {
-			if (!is_printk_legacy_deferred())
+			/*
+			 * Offload while this CPU runs a stopper callback so that a
+			 * slow flush cannot starve RT kthreads (e.g. the watchdog
+			 * pet) or stall the multi_cpu_stop() state machine.
+			 * Flushed once the stopper exits.
+			 */
+			if (!is_printk_legacy_deferred() && !in_cpu_stop())
 				ft->legacy_direct = true;
 			else if (!console_irqwork_blocked)
 				ft->legacy_offload = true;
diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c
index 3fcdf4b4e2e5..379fcbe6914c 100644
--- a/kernel/printk/printk.c
+++ b/kernel/printk/printk.c
@@ -49,6 +49,7 @@
 #include <linux/sched/debug.h>
 #include <linux/sched/task_stack.h>
 #include <linux/panic.h>
+#include <linux/stop_machine.h>
 
 #include <linux/uaccess.h>
 #include <asm/sections.h>
@@ -4602,6 +4603,16 @@ static void wake_up_klogd_work_func(struct irq_work *irq_work)
 {
 	int pending = this_cpu_xchg(printk_pending, 0);
 
+	/*
+	 * Don't flush the legacy console from irq_work while this CPU runs a
+	 * stopper callback; keep the bit pending and re-run once it exits (see
+	 * cpu_stopper_thread()).
+	 */
+	if ((pending & PRINTK_PENDING_OUTPUT) && in_cpu_stop()) {
+		this_cpu_or(printk_pending, PRINTK_PENDING_OUTPUT);
+		pending &= ~PRINTK_PENDING_OUTPUT;
+	}
+
 	if (pending & PRINTK_PENDING_OUTPUT) {
 		if (force_legacy_kthread()) {
 			if (printk_legacy_kthread)
@@ -4687,6 +4698,18 @@ void defer_console_output(void)
 	__wake_up_klogd(PRINTK_PENDING_WAKEUP | PRINTK_PENDING_OUTPUT);
 }
 
+void printk_defer_console_output(void)
+{
+	bool pending_output;
+
+	preempt_disable();
+	pending_output = this_cpu_read(printk_pending) & PRINTK_PENDING_OUTPUT;
+	preempt_enable();
+
+	if (pending_output)
+		defer_console_output();
+}
+
 /**
  * printk_trigger_flush - Attempt to flush printk buffer to consoles.
  *
diff --git a/kernel/stop_machine.c b/kernel/stop_machine.c
index d38fabb0b9f1..18ae7ff2d8b8 100644
--- a/kernel/stop_machine.c
+++ b/kernel/stop_machine.c
@@ -22,6 +22,7 @@
 #include <linux/smpboot.h>
 #include <linux/atomic.h>
 #include <linux/nmi.h>
+#include <linux/printk.h>
 #include <linux/sched/wake_q.h>
 
 /*
@@ -530,6 +531,12 @@ static void cpu_stopper_thread(unsigned int cpu)
 		ret = fn(arg);
 		this_cpu_write(cpu_stop_active, false);
 
+		/*
+		 * Flush console output that was deferred while the callback ran,
+		 * now that it is safe to do so from this CPU.
+		 */
+		printk_defer_console_output();
+
 		if (done) {
 			if (ret)
 				done->ret = ret;

-- 
2.34.1