Re: Unknown NMI after S4 resume

From: Tony W Wang-oc
Date: Mon Jul 29 2024 - 05:15:08 EST



On 2024/7/25 16:27, Pavel Machek wrote:

Yes, please.

The watchdog driver should get suspend/resume hooks, and probably do
same init on boot and on resume.


Two patch methods were tried and the results are as follows:

Patch 1:
diff -aNur linux-6.10/kernel/watchdog.c linux-6.10_test/kernel/watchdog.c
--- linux-6.10/kernel/watchdog.c 2024-07-15 06:43:32.000000000 +0800
+++ linux-6.10_test/kernel/watchdog.c 2024-07-26 18:38:16.442600567 +0800
@@ -25,6 +25,7 @@
#include <linux/stop_machine.h>
#include <linux/sysctl.h>
#include <linux/tick.h>
+#include <linux/syscore_ops.h>

#include <linux/sched/clock.h>
#include <linux/sched/debug.h>
@@ -1264,3 +1265,35 @@

lockup_detector_setup();
}
+
+static int watchdog_suspend(void)
+{
+ if (cpumask_test_cpu(0, &watchdog_allowed_mask))
+ watchdog_hardlockup_disable(0);
+
+ lockup_detector_cleanup();
+
+ return 0;
+}
+
+static void watchdog_resume(void)
+{
+ if (cpumask_test_cpu(0, &watchdog_allowed_mask)
+ && (watchdog_enabled & WATCHDOG_HARDLOCKUP_ENABLED))
+ watchdog_hardlockup_enable(0);
+}
+
+static struct syscore_ops watchdog_syscore_ops = {
+ .resume = watchdog_resume,
+ .suspend = watchdog_suspend,
+};
+
+static int __init init_watchdog_sysfs(void)
+{
+ register_syscore_ops(&watchdog_syscore_ops);
+
+ return 0;
+}
+
+late_initcall(init_watchdog_sysfs);
+

Patch 1 result is: There is no unknown NMI after S4 resume, but this patch introduces the following warning:

[ 8.994619] ------------[ cut here ]------------
[ 8.999205] Interrupts enabled after watchdog_suspend+0x0/0x50
[ 9.005011] WARNING: CPU: 0 PID: 219 at drivers/base/syscore.c:68 syscore_suspend+0x17c/0x290
[ 9.013463] Modules linked in: uas usb_storage i2c_hid_acpi i2c_hid r8169 via_rng crc32_pclmul ahci drm xhci_pci psmouse libahci xhci_pci_renesas realtek hid video wmi
[ 9.028289] CPU: 0 PID: 219 Comm: sh Not tainted 6.10.0 #2
[ 9.033726] Hardware name: Shanghai Zhaoxin Semiconductor Co., Ltd. ZXE CRB/HX005EZ, BIOS 47.00 06/19/2024
[ 9.043289] RIP: 0010:syscore_suspend+0x17c/0x290
[ 9.047953] Code: 80 fc 01 0f 87 0f 06 5f 00 41 83 e4 01 0f 85 1a ff ff ff 48 8b 73 10 48 c7 c7 0e c8 81 b0 c6 05 06 d4 48 01 01 e8 a4 de 5a ff <0f> 0b e9 fc fe ff ff 65 8b 05 f2 5e 6e 50 48 0f a3 05 7e 04 4e 01
[ 9.066546] RSP: 0018:ffffbabf8073fbe8 EFLAGS: 00010246
[ 9.071723] RAX: 0000000000000000 RBX: ffffffffb0bf29e0 RCX: 0000000000000000
[ 9.078792] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000000
[ 9.085864] RBP: ffffbabf8073fbf8 R08: 0000000000000000 R09: 0000000000000000
[ 9.092933] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000000
[ 9.100000] R13: fffffffffffffff2 R14: ffff8edd5106e000 R15: 0000000000000004
[ 9.107070] FS: 000070a4a2ac3740(0000) GS:ffff8ee06fa00000(0000) knlGS:0000000000000000
[ 9.115086] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 9.120779] CR2: 00005d4095519bd8 CR3: 00000001030e8002 CR4: 0000000000770ef0
[ 9.127849] PKRU: 55555554
[ 9.130534] Call Trace:
[ 9.132961] <TASK>
[ 9.135042] ? show_regs+0x71/0x90
[ 9.138416] ? __warn+0x88/0x140
[ 9.141617] ? syscore_suspend+0x17c/0x290
[ 9.145678] ? report_bug+0x1ab/0x1c0
[ 9.149313] ? console_unlock+0x77/0x120
[ 9.153204] ? handle_bug+0x46/0x90
[ 9.156665] ? exc_invalid_op+0x19/0x80
[ 9.160464] ? asm_exc_invalid_op+0x1b/0x20
[ 9.164614] ? syscore_suspend+0x17c/0x290
[ 9.168674] ? syscore_suspend+0x17c/0x290
......

Patch 2:
diff -aNur linux-6.10/include/linux/nmi.h linux-6.10_test/include/linux/nmi.h
--- linux-6.10/include/linux/nmi.h 2024-07-15 06:43:32.000000000 +0800
+++ linux-6.10_test/include/linux/nmi.h 2024-07-25 18:57:38.886105105 +0800
@@ -19,6 +19,9 @@
void lockup_detector_soft_poweroff(void);
void lockup_detector_cleanup(void);

+extern int nmi_watchdog_bsp_suspend(void);
+extern void nmi_watchdog_bsp_resume(void);
+
extern int watchdog_user_enabled;
extern int watchdog_thresh;
extern unsigned long watchdog_enabled;
diff -aNur linux-6.10/kernel/power/hibernate.c linux-6.10_test/kernel/power/hibernate.c
--- linux-6.10/kernel/power/hibernate.c 2024-07-15 06:43:32.000000000 +0800
+++ linux-6.10_test/kernel/power/hibernate.c 2024-07-25 19:07:16.470095318 +0800
@@ -313,6 +313,8 @@
if (error || hibernation_test(TEST_PLATFORM))
goto Platform_finish;

+ nmi_watchdog_bsp_suspend();
+
error = pm_sleep_disable_secondary_cpus();
if (error || hibernation_test(TEST_CPUS))
goto Enable_cpus;
@@ -356,7 +358,9 @@

Enable_cpus:
pm_sleep_enable_secondary_cpus();
-
+
+ nmi_watchdog_bsp_resume();
+
/* Allow architectures to do nosmt-specific post-resume dances */
if (!in_suspend)
error = arch_resume_nosmt();
@@ -481,6 +485,8 @@

cpuidle_pause();

+ nmi_watchdog_bsp_suspend();
+
error = hibernate_resume_nonboot_cpu_disable();
if (error)
goto Enable_cpus;
@@ -526,6 +532,8 @@
Enable_cpus:
pm_sleep_enable_secondary_cpus();

+ nmi_watchdog_bsp_resume();
+
Cleanup:
platform_restore_cleanup(platform_mode);

diff -aNur linux-6.10/kernel/watchdog.c linux-6.10_test/kernel/watchdog.c
--- linux-6.10/kernel/watchdog.c 2024-07-15 06:43:32.000000000 +0800
+++ linux-6.10_test/kernel/watchdog.c 2024-07-26 09:53:04.579979589 +0800
@@ -1249,6 +1249,36 @@
}
late_initcall_sync(lockup_detector_check);

+int bsp_watchdog_enable(void *data)
+{
+ if (cpumask_test_cpu(smp_processor_id(), &watchdog_allowed_mask))
+ watchdog_hardlockup_enable(smp_processor_id());
+ return 0;
+}
+int bsp_watchdog_disable(void *data)
+{
+ if (cpumask_test_cpu(smp_processor_id(), &watchdog_allowed_mask))
+ watchdog_hardlockup_disable(smp_processor_id());
+ return 0;
+}
+
+int nmi_watchdog_bsp_suspend(void)
+{
+ smp_call_on_cpu(0,bsp_watchdog_disable,NULL,false);
+ return 0;
+}
+EXPORT_SYMBOL(nmi_watchdog_bsp_suspend);
+
+void nmi_watchdog_bsp_resume(void)
+{
+ /* Enable the perf event */
+ if (watchdog_enabled & WATCHDOG_HARDLOCKUP_ENABLED){
+ smp_call_on_cpu(0,bsp_watchdog_enable,NULL,false);
+ }
+ return;
+}
+EXPORT_SYMBOL(nmi_watchdog_bsp_resume);
+
void __init lockup_detector_init(void)
{
if (tick_nohz_full_enabled())

Patch 2 result is: There is no unknown NMI after S4 resume, and the results of multiple tests are ok.

Can patch 2 be used to fix this problem?

BRs!
TonyWWang-oc