[PATCH] x86/nmi: Enable panic() when NMI handler runs too long

[Paul E. McKenney]

Firmware, device-driver, and configuration issues can result in
long-running NMI handlers, for example, nmi_cpu_backtrace_handler()
running for more than two seconds, and with no console issues.  In this
case, this is not the fault of nmi_cpu_backtrace_handler(), but it is
the first symptom of the underlying problem.  These NMI handlers result
in various other problems, including CSD-lock warnings, clocksource
watchdog false positives, scheduler frequency invariance wobbliness,
systemd excessive-CPU complaints, and even the occasional RCU CPU
stall warning.  Eventually, the automation will take the system out of
service, but with quite a bit of noise obscuring the underlying issue,
and thus confusing both automation and human beings.

Therefore, provide a toolong_nmi_panic __setup() parameter that makes
nmi_check_duration() invoke nmi_panic() if the just-completed NMI handler
ran for more than the specified number of milliseconds.  This parameter
defaults to zero, which disables the nmi_panic().  As in, by default
you get the same behavior that you get today.

[ paulmck: Apply kernel test robot feedback. ]

Signed-off-by: Paul E. McKenney <paulmck@xxxxxxxxxx>
Cc: Thomas Gleixner <tglx@xxxxxxxxxx>
Cc: Ingo Molnar <mingo@xxxxxxxxxx>
Cc: Borislav Petkov <bp@xxxxxxxxx>
Cc: Dave Hansen <dave.hansen@xxxxxxxxxxxxxxx>
Cc: "H. Peter Anvin" <hpa@xxxxxxxxx>
Cc: <x86@xxxxxxxxxx>

diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c
index 3c9f60d6ca5a..aae725477468 100644
--- a/arch/x86/kernel/nmi.c
+++ b/arch/x86/kernel/nmi.c
@@ -85,6 +85,7 @@ static DEFINE_PER_CPU(struct nmi_stats, nmi_stats);
 static int ignore_nmis __read_mostly;
 
 int unknown_nmi_panic;
+static unsigned long toolong_nmi_panic;
 int panic_on_unrecovered_nmi;
 int panic_on_io_nmi;
 
@@ -101,6 +102,20 @@ static int __init setup_unknown_nmi_panic(char *str)
 }
 __setup("unknown_nmi_panic", setup_unknown_nmi_panic);
 
+static int __init setup_toolong_nmi_panic(char *str)
+{
+	int ret;
+
+	ret = kstrtoul(str, 0, &toolong_nmi_panic);
+	if (ret) {
+		pr_alert("%s: %s on '%s', disabling to zero.\n",
+			 __func__, ret == -ERANGE ? "Overflow" : "Parse error", str);
+		toolong_nmi_panic = 0;
+	}
+	return 1;
+}
+__setup("toolong_nmi_panic=", setup_toolong_nmi_panic);
+
 static u64 nmi_longest_ns = 1 * NSEC_PER_MSEC;
 
 static int __init nmi_warning_debugfs(void)
@@ -111,7 +126,7 @@ static int __init nmi_warning_debugfs(void)
 }
 fs_initcall(nmi_warning_debugfs);
 
-static void nmi_check_duration(struct nmiaction *action, u64 duration)
+static void nmi_check_duration(struct nmiaction *action, u64 duration, struct pt_regs *regs)
 {
 	int remainder_ns, decimal_msecs;
 
@@ -126,6 +141,8 @@ static void nmi_check_duration(struct nmiaction *action, u64 duration)
 
 	pr_info_ratelimited("INFO: NMI handler (%ps) took too long to run: %lld.%03d msecs\n",
 			    action->handler, duration, decimal_msecs);
+	if (toolong_nmi_panic && duration >= toolong_nmi_panic)
+		nmi_panic(regs, "Panic per toolong_nmi_panic");
 }
 
 static int nmi_handle(unsigned int type, struct pt_regs *regs)
@@ -165,7 +182,7 @@ static int nmi_handle(unsigned int type, struct pt_regs *regs)
 		delta = sched_clock() - delta;
 		trace_nmi_handler(a->handler, (int)delta, thishandled);
 
-		nmi_check_duration(a, delta);
+		nmi_check_duration(a, delta, regs);
 	}
 
 	rcu_read_unlock();