[PATCH -rcu/dev 4/4] hazptrtorture: Nest a second hazard pointer in readers

[Andrea Parri]

hazptrtorture rarely exercises the race fixed by "hazptr: Record per-CPU
slot owner after claiming the slot": an interrupt handler acquiring a
hazard pointer after the interrupted task chooses a slot in
__hazptr_acquire() and before it claims it.  Readers get there only when
their pointer is NULL or changes under them.

This commit therefore adds the nest_acquire module parameter, which
makes a reader acquire and release a second hazard pointer while holding
the first, once per the specified number of acquisitions on average
(default two, zero disables).  While the outer hazard pointer is in
slot 0 of the reader's CPU, the nested acquisition finds that slot in
use and goes to __hazptr_acquire(), where hazptr_torture_acquire()
handlers from other readers can interrupt it.

Without that fix, and with the checks added by "hazptr: Check per-CPU
slot owner with CONFIG_HAZPTR_DEBUG", the PREEMPT and NOPREEMPT
scenarios at default parameters report the race 6.3 and 6.0 seconds
after boot, and NOPREEMPT oopses 1.3 seconds later.  With
nest_acquire=0, both run 10 minutes without a report.

With the fix, 30-minute runs of both scenarios pass, as does a 10-minute
PREEMPT run with irq_acquire=100 and nest_acquire=1.

The nesting costs about a third of the outer reader iterations, as seen
in the IPI acquisitions, which readers send once per irq_acquire
iterations.  PREEMPT counts acqirq: 270645 in 10 minutes without nesting
and acqirq: 544976 in 30 minutes with it.

Signed-off-by: Andrea Parri <parri.andrea@xxxxxxxxx>
---
 .../admin-guide/kernel-parameters.txt         |  6 +++++
 kernel/rcu/hazptrtorture.c                    | 27 +++++++++++++++++++
 2 files changed, 33 insertions(+)

diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index e3c5a6b68fc34..63bb7719457dd 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -1992,6 +1992,12 @@ Kernel parameters
 			is equal to zero, in which case it instead
 			defaults to zero (disabled).
 
+	hazptrtorture.nest_acquire= [KNL]
+			Acquire a second, nested hazard pointer on
+			average one out of the specified number of
+			reader acquisitions.  Zero disables.  Defaults
+			to two.
+
 	hazptrtorture.nreaders= [KNL]
 			Number of hazard-pointer reader kthreads, each
 			of which repeatedly acquires and releases hazard
diff --git a/kernel/rcu/hazptrtorture.c b/kernel/rcu/hazptrtorture.c
index c46083a8af093..4373bc5fe11e5 100644
--- a/kernel/rcu/hazptrtorture.c
+++ b/kernel/rcu/hazptrtorture.c
@@ -37,6 +37,8 @@ torture_param(int, irq_release, -1,
 	      "Release hazard pointers from irq handlers once per specified #, zero to disable");
 torture_param(int, kthread_do_pending_ms, -1,
 	      "Delay between cleanups for deferred hazard pointers (ms), zero to disable");
+torture_param(int, nest_acquire, 2,
+	      "Nest a second hazard pointer once per specified # of reader acquires, zero to disable");
 torture_param(int, nreaders, -1, "Number of hazard-pointer reader threads");
 torture_param(bool, nwriters, 1, "Number of hazard-pointer writer threads, 0 or 1");
 torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)");
@@ -449,6 +451,22 @@ static void hazptr_torture_defer(struct hazptr_pending *hppp, struct torture_ran
 	atomic_long_inc(per_cpu_ptr(&hazptr_torture_releases_defer, raw_smp_processor_id()));
 }
 
+/*
+ * Acquire and release a second hazard pointer while the caller holds one.
+ * If the caller's hazard pointer is in this CPU's slot 0, this acquisition
+ * takes the slow path, which first tries another per-CPU slot and can race
+ * there with hazptr_torture_acquire() handlers.
+ */
+static void hazptr_torture_nested_reader(struct torture_random_state *trsp)
+{
+	struct hazptr_pending hpp;
+
+	hpp.hpp_htp = cur_ops->readlock(&hpp.hpp_hc);
+	atomic_long_inc(per_cpu_ptr(&hazptr_torture_acquires, raw_smp_processor_id()));
+	if (hpp.hpp_htp)
+		hazptr_torture_reader_tail(&hpp, trsp);
+}
+
 /*
  * Hazard-pointer torture reader kthread.  Repeatedly dereferences
  * hazptr_torture_current, incrementing the corresponding element of the
@@ -497,6 +515,8 @@ static int hazptr_torture_reader(void *arg)
 			schedule_timeout_interruptible(HZ / 10);
 			continue;
 		}
+		if (nest_acquire && !(torture_random(&rand) % nest_acquire))
+			hazptr_torture_nested_reader(&rand);
 		if (time_after(jiffies, lastsleep) && !torture_must_stop()) {
 			torture_hrtimeout_us(500, 1000, &rand);
 			lastsleep = jiffies + 10;
@@ -721,6 +741,7 @@ hazptr_torture_print_module_parms(struct hazptr_torture_ops *cur_ops, const char
 	pr_alert("%s" TORTURE_FLAG
 		 "--- %s: nreaders=%d nwriters=%d "
 		 "defer_modulus=%d irq_acquire=%d irq_release=%d kthread_do_pending_ms=%d "
+		 "nest_acquire=%d "
 		 "onoff_interval=%d onoff_holdoff=%d "
 		 "preempt_duration=%d preempt_interval=%d "
 		 "reader_sleep_us=%d "
@@ -728,6 +749,7 @@ hazptr_torture_print_module_parms(struct hazptr_torture_ops *cur_ops, const char
 		 "verbose=%d\n",
 		 torture_type, tag, nrealreaders, nwriters,
 		 defer_modulus, irq_acquire, irq_release, kthread_do_pending_ms,
+		 nest_acquire,
 		 onoff_interval, onoff_holdoff,
 		 preempt_duration, preempt_interval,
 		 reader_sleep_us,
@@ -891,6 +913,11 @@ static int __init hazptr_torture_init(void)
 		WARN_ON(IS_BUILTIN(CONFIG_HAZPTR_TORTURE_TEST));
 		irq_release = 0;
 	}
+	if (nest_acquire < 0) {
+		pr_alert("Cannot have negative nest_acquire (%d), disabling.\n", nest_acquire);
+		WARN_ON(IS_BUILTIN(CONFIG_HAZPTR_TORTURE_TEST));
+		nest_acquire = 0;
+	}
 	reader_tasks = kzalloc_objs(reader_tasks[0], nrealreaders);
 	for (i = 0; i < nrealreaders; i++) {
 		firsterr = torture_create_kthread(hazptr_torture_reader, (void *)i,
-- 
2.53.0