[PATCH v6 2/2] watchdog: add watchdog_cpumask sysctl to assist nohz

[cmetcalf]

From: Chris Metcalf <cmetcalf@xxxxxxxxxx>

Change the default behavior of watchdog so it only runs on the
housekeeping cores when nohz_full is enabled at build and boot time.

Allow modifying the set of cores the watchdog is currently running
on with a new kernel.watchdog_cpumask sysctl.

Acked-by: Don Zickus <dzickus@xxxxxxxxxx>
Signed-off-by: Chris Metcalf <cmetcalf@xxxxxxxxxx>
---
v6: use alloc_cpumask_var() [Sasha Levin]
    switch from watchdog_exclude to watchdog_cpumask [Frederic]
    simplify the smp_hotplug_thread API to watchdog [Frederic]
    add Don's Acked-by

 Documentation/lockup-watchdogs.txt |  6 ++++
 Documentation/sysctl/kernel.txt    | 11 ++++++++
 include/linux/nmi.h                |  3 ++
 kernel/sysctl.c                    |  7 +++++
 kernel/watchdog.c                  | 56 ++++++++++++++++++++++++++++++++++++++
 5 files changed, 83 insertions(+)

diff --git a/Documentation/lockup-watchdogs.txt b/Documentation/lockup-watchdogs.txt
index ab0baa692c13..31c312853d4c 100644
--- a/Documentation/lockup-watchdogs.txt
+++ b/Documentation/lockup-watchdogs.txt
@@ -61,3 +61,9 @@ As explained above, a kernel knob is provided that allows
 administrators to configure the period of the hrtimer and the perf
 event. The right value for a particular environment is a trade-off
 between fast response to lockups and detection overhead.
+
+By default, the watchdog runs on all online cores.  However, on a
+kernel configured with NO_HZ_FULL, by default the watchdog runs only
+on the housekeeping cores, not the cores specified in the "nohz_full"
+boot argument.  In either case, the set of cores excluded from running
+the watchdog may be adjusted via the kernel.watchdog_cpumask sysctl.
diff --git a/Documentation/sysctl/kernel.txt b/Documentation/sysctl/kernel.txt
index c831001c45f1..f6a9dca8c100 100644
--- a/Documentation/sysctl/kernel.txt
+++ b/Documentation/sysctl/kernel.txt
@@ -923,6 +923,17 @@ and nmi_watchdog.
 
 ==============================================================
 
+watchdog_cpumask:
+
+This value can be used to control on which cpus the watchdog may run.
+The default cpumask is all possible cores, but if NO_HZ_FULL is
+enabled in the kernel config, and cores are specified with the
+nohz_full= boot argument, those cores are excluded by default.
+Offline cores can be included in this mask, and if the core is later
+brought online, the watchdog will be started based on the mask value.
+
+==============================================================
+
 watchdog_thresh:
 
 This value can be used to control the frequency of hrtimer and NMI
diff --git a/include/linux/nmi.h b/include/linux/nmi.h
index 3d46fb4708e0..f94da0e65dea 100644
--- a/include/linux/nmi.h
+++ b/include/linux/nmi.h
@@ -67,6 +67,7 @@ extern int nmi_watchdog_enabled;
 extern int soft_watchdog_enabled;
 extern int watchdog_user_enabled;
 extern int watchdog_thresh;
+extern unsigned long *watchdog_cpumask_bits;
 extern int sysctl_softlockup_all_cpu_backtrace;
 struct ctl_table;
 extern int proc_watchdog(struct ctl_table *, int ,
@@ -77,6 +78,8 @@ extern int proc_soft_watchdog(struct ctl_table *, int ,
 			      void __user *, size_t *, loff_t *);
 extern int proc_watchdog_thresh(struct ctl_table *, int ,
 				void __user *, size_t *, loff_t *);
+extern int proc_watchdog_cpumask(struct ctl_table *, int,
+				 void __user *, size_t *, loff_t *);
 #endif
 
 #ifdef CONFIG_HAVE_ACPI_APEI_NMI
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index 2082b1a88fb9..699571a74e3b 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -881,6 +881,13 @@ static struct ctl_table kern_table[] = {
 		.extra2		= &one,
 	},
 	{
+		.procname	= "watchdog_cpumask",
+		.data		= &watchdog_cpumask_bits,
+		.maxlen		= NR_CPUS,
+		.mode		= 0644,
+		.proc_handler	= proc_watchdog_cpumask,
+	},
+	{
 		.procname	= "softlockup_panic",
 		.data		= &softlockup_panic,
 		.maxlen		= sizeof(int),
diff --git a/kernel/watchdog.c b/kernel/watchdog.c
index 2316f50b07a4..fc0a90684639 100644
--- a/kernel/watchdog.c
+++ b/kernel/watchdog.c
@@ -19,6 +19,7 @@
 #include <linux/sysctl.h>
 #include <linux/smpboot.h>
 #include <linux/sched/rt.h>
+#include <linux/tick.h>
 
 #include <asm/irq_regs.h>
 #include <linux/kvm_para.h>
@@ -56,6 +57,8 @@ int __read_mostly sysctl_softlockup_all_cpu_backtrace;
 #else
 #define sysctl_softlockup_all_cpu_backtrace 0
 #endif
+static cpumask_var_t watchdog_cpumask;
+unsigned long *watchdog_cpumask_bits;
 
 static int __read_mostly watchdog_running;
 static u64 __read_mostly sample_period;
@@ -477,6 +480,16 @@ static int watchdog_should_run(unsigned int cpu)
 }
 
 /*
+ * This test is not serialized with updates to watchdog_cpumask,
+ * but when the update is complete we will disable and re-enable all
+ * the watchdog threads anyway.
+ */
+static int watchdog_valid_cpu(unsigned int cpu)
+{
+	return cpumask_test_cpu(cpu, watchdog_cpumask);
+}
+
+/*
  * The watchdog thread function - touches the timestamp.
  *
  * It only runs once every sample_period seconds (4 seconds by
@@ -645,6 +658,7 @@ static struct smp_hotplug_thread watchdog_threads = {
 	.cleanup		= watchdog_cleanup,
 	.park			= watchdog_disable,
 	.unpark			= watchdog_enable,
+	.valid_cpu		= watchdog_valid_cpu,
 };
 
 static void restart_watchdog_hrtimer(void *info)
@@ -869,12 +883,54 @@ out:
 	mutex_unlock(&watchdog_proc_mutex);
 	return err;
 }
+
+/*
+ * The cpumask is the mask of possible cpus that the watchdog can run
+ * on, not the mask of cpus it is actually running on.  This allows the
+ * user to specify a mask that will include cpus that have not yet
+ * been brought online, if desired.
+ */
+int proc_watchdog_cpumask(struct ctl_table *table, int write,
+			  void __user *buffer, size_t *lenp, loff_t *ppos)
+{
+	int err;
+
+	mutex_lock(&watchdog_proc_mutex);
+	err = proc_do_large_bitmap(table, write, buffer, lenp, ppos);
+	if (!err && write) {
+		/* Remove impossible cpus to keep sysctl output cleaner. */
+		cpumask_and(watchdog_cpumask, watchdog_cpumask,
+			    cpu_possible_mask);
+
+		if (watchdog_enabled && watchdog_thresh) {
+			watchdog_disable_all_cpus();
+			watchdog_enable_all_cpus();
+		}
+	}
+	mutex_unlock(&watchdog_proc_mutex);
+	return err;
+}
+
 #endif /* CONFIG_SYSCTL */
 
 void __init lockup_detector_init(void)
 {
 	set_sample_period();
 
+	alloc_cpumask_var(&watchdog_cpumask, GFP_KERNEL);
+
+#ifdef CONFIG_NO_HZ_FULL
+	if (!cpumask_empty(tick_nohz_full_mask))
+		pr_info("Disabling watchdog on nohz_full cores by default\n");
+	cpumask_andnot(watchdog_cpumask, cpu_possible_mask,
+		       tick_nohz_full_mask);
+#else
+	cpumask_copy(watchdog_cpumask, cpu_possible_mask);
+#endif
+
+	/* The sysctl API requires a variable holding a pointer to the mask. */
+	watchdog_cpumask_bits = cpumask_bits(watchdog_cpumask);
+
 	if (watchdog_enabled)
 		watchdog_enable_all_cpus();
 }
-- 
2.1.2

--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Please read the FAQ at  http://www.tux.org/lkml/