[PATCH v2 06/10] rcu/tasks: Check RCU watching state for holdout idle tasks

[neeraj . upadhyay]

From: Neeraj Upadhyay <neeraj.upadhyay@xxxxxxxxxx>

Use RCU watching state of a CPU to check whether RCU-tasks GP
need to wait for idle task on that CPU. Idle tasks which are
in deep-idle states where RCU is not watching or which have
transitioned to/from deep-idle state do not block RCU-tasks
grace period.

Signed-off-by: Neeraj Upadhyay <neeraj.upadhyay@xxxxxxxxxx>
---
 kernel/rcu/tasks.h | 36 +++++++++++++++++++++++++++++++++---
 1 file changed, 33 insertions(+), 3 deletions(-)

diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h
index 9523aff6cdae..d8506d2e6f54 100644
--- a/kernel/rcu/tasks.h
+++ b/kernel/rcu/tasks.h
@@ -36,6 +36,8 @@ typedef void (*postgp_func_t)(struct rcu_tasks *rtp);
  * @cpu: CPU number corresponding to this entry.
  * @index: Index of this CPU in rtpcp_array of the rcu_tasks structure.
  * @rtpp: Pointer to the rcu_tasks structure.
+ * @rcu_watching_snap: Per-GP RCU-watching snapshot for idle tasks.
+ * @rcu_watching_snap_rec: RCU-watching snapshot recorded for idle task.
  */
 struct rcu_tasks_percpu {
 	struct rcu_segcblist cblist;
@@ -52,6 +54,8 @@ struct rcu_tasks_percpu {
 	int cpu;
 	int index;
 	struct rcu_tasks *rtpp;
+	int rcu_watching_snap;
+	bool rcu_watching_snap_rec;
 };
 
 /**
@@ -957,9 +961,14 @@ static void rcu_tasks_wait_gp(struct rcu_tasks *rtp)
 // rcu_tasks_pregp_step() and by the scheduler's locks and interrupt
 // disabling.
 
+void call_rcu_tasks(struct rcu_head *rhp, rcu_callback_t func);
+DEFINE_RCU_TASKS(rcu_tasks, rcu_tasks_wait_gp, call_rcu_tasks, "RCU Tasks");
+
 /* Pre-grace-period preparation. */
 static void rcu_tasks_pregp_step(struct list_head *hop)
 {
+	int cpu;
+
 	/*
 	 * Wait for all pre-existing t->on_rq and t->nvcsw transitions
 	 * to complete.  Invoking synchronize_rcu() suffices because all
@@ -974,11 +983,20 @@ static void rcu_tasks_pregp_step(struct list_head *hop)
 	 * grace period.
 	 */
 	synchronize_rcu();
+
+	/* Initialize watching snapshots for this GP */
+	for_each_possible_cpu(cpu) {
+		struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rcu_tasks.rtpcpu, cpu);
+
+		rtpcp->rcu_watching_snap_rec = false;
+	}
 }
 
 #ifdef CONFIG_SMP
 static bool rcu_idle_task_is_holdout(struct task_struct *t, int cpu)
 {
+	struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rcu_tasks.rtpcpu, cpu);
+
 	/* Idle tasks on offline CPUs are RCU-tasks quiescent states. */
 	if (!rcu_cpu_online(cpu))
 		return false;
@@ -992,6 +1010,21 @@ static bool rcu_idle_task_is_holdout(struct task_struct *t, int cpu)
 	if (!t->on_cpu)
 		return false;
 
+	if (!rtpcp->rcu_watching_snap_rec) {
+		/*
+		 * Do plain access. Ordering between remote CPU's pre idle accesses
+		 * and post rcu-tasks grace period is provided by synchronize_rcu()
+		 * in rcu_tasks_postgp().
+		 */
+		rtpcp->rcu_watching_snap = ct_rcu_watching_cpu(cpu);
+		rtpcp->rcu_watching_snap_rec = true;
+		/* RCU-idle contexts are RCU-tasks quiescent state for idle tasks. */
+		if (rcu_watching_snap_in_eqs(rtpcp->rcu_watching_snap))
+			return false;
+	} else if (rcu_watching_snap_stopped_since(cpu, rtpcp->rcu_watching_snap)) {
+		return false;
+	}
+
 	return true;
 }
 #else /* #ifdef CONFIG_SMP */
@@ -1042,9 +1075,6 @@ static void rcu_tasks_pertask(struct task_struct *t, struct list_head *hop)
 	}
 }
 
-void call_rcu_tasks(struct rcu_head *rhp, rcu_callback_t func);
-DEFINE_RCU_TASKS(rcu_tasks, rcu_tasks_wait_gp, call_rcu_tasks, "RCU Tasks");
-
 /* Processing between scanning taskslist and draining the holdout list. */
 static void rcu_tasks_postscan(struct list_head *hop)
 {
-- 
2.40.1