[PATCH RFC 08/16] prcu: Implement PRCU callback processing

[lianglihao]

From: Lihao Liang <lianglihao@xxxxxxxxxx>

Currently, PRCU core processing only consists of callback processing
in prcu_process_callbacks(), which is triggered by the scheduling-clock
interrupt.

Reviewed-by: Heng Zhang <heng.z@xxxxxxxxxx>
Signed-off-by: Lihao Liang <lianglihao@xxxxxxxxxx>
---
 include/linux/interrupt.h |  3 ++
 include/linux/prcu.h      |  8 +++++
 kernel/rcu/prcu.c         | 86 +++++++++++++++++++++++++++++++++++++++++++++++
 kernel/rcu/tree.c         |  1 +
 kernel/time/timer.c       |  2 ++
 5 files changed, 100 insertions(+)

diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
index 0991f973..f05ef62a 100644
--- a/include/linux/interrupt.h
+++ b/include/linux/interrupt.h
@@ -456,6 +456,9 @@ enum
 	SCHED_SOFTIRQ,
 	HRTIMER_SOFTIRQ, /* Unused, but kept as tools rely on the
 			    numbering. Sigh! */
+#ifdef CONFIG_PRCU
+	PRCU_SOFTIRQ,
+#endif
 	RCU_SOFTIRQ,    /* Preferable RCU should always be the last softirq */
 
 	NR_SOFTIRQS
diff --git a/include/linux/prcu.h b/include/linux/prcu.h
index e5e09c9b..4e7d5d65 100644
--- a/include/linux/prcu.h
+++ b/include/linux/prcu.h
@@ -31,11 +31,13 @@ struct prcu_local_struct {
 	unsigned int locked;
 	unsigned int online;
 	unsigned long long version;
+	unsigned long long cb_version;
 	struct prcu_cblist cblist;
 };
 
 struct prcu_struct {
 	atomic64_t global_version;
+	atomic64_t cb_version;
 	atomic_t active_ctr;
 	struct mutex mtx;
 	wait_queue_head_t wait_q;
@@ -48,6 +50,9 @@ void synchronize_prcu(void);
 void call_prcu(struct rcu_head *head, rcu_callback_t func);
 void prcu_init(void);
 void prcu_note_context_switch(void);
+int prcu_pending(void);
+void invoke_prcu_core(void);
+void prcu_check_callbacks(void);
 
 #else /* #ifdef CONFIG_PRCU */
 
@@ -57,6 +62,9 @@ void prcu_note_context_switch(void);
 #define call_prcu() do {} while (0)
 #define prcu_init() do {} while (0)
 #define prcu_note_context_switch() do {} while (0)
+#define prcu_pending() 0
+#define invoke_prcu_core() do {} while (0)
+#define prcu_check_callbacks() do {} while (0)
 
 #endif /* #ifdef CONFIG_PRCU */
 #endif /* __LINUX_PRCU_H */
diff --git a/kernel/rcu/prcu.c b/kernel/rcu/prcu.c
index f198285c..373039c5 100644
--- a/kernel/rcu/prcu.c
+++ b/kernel/rcu/prcu.c
@@ -1,6 +1,7 @@
 #include <linux/smp.h>
 #include <linux/percpu.h>
 #include <linux/prcu.h>
+#include <linux/interrupt.h>
 #include <linux/sched.h>
 #include <linux/slab.h>
 #include <asm/barrier.h>
@@ -11,6 +12,7 @@ DEFINE_PER_CPU_SHARED_ALIGNED(struct prcu_local_struct, prcu_local);
 
 struct prcu_struct global_prcu = {
 	.global_version = ATOMIC64_INIT(0),
+	.cb_version = ATOMIC64_INIT(0),
 	.active_ctr = ATOMIC_INIT(0),
 	.mtx = __MUTEX_INITIALIZER(global_prcu.mtx),
 	.wait_q = __WAIT_QUEUE_HEAD_INITIALIZER(global_prcu.wait_q)
@@ -27,6 +29,35 @@ static void prcu_cblist_init(struct prcu_cblist *rclp)
 	rclp->len = 0;
 }
 
+/*
+ * Dequeue the oldest rcu_head structure from the specified callback list;
+ * store the callback grace period version number into the version pointer.
+ */
+static struct rcu_head *prcu_cblist_dequeue(struct prcu_cblist *rclp)
+{
+	struct rcu_head *rhp;
+	struct prcu_version_head *vhp;
+
+	rhp = rclp->head;
+	if (!rhp) {
+		WARN_ON(vhp);
+		WARN_ON(rclp->len);
+		return NULL;
+	}
+
+	vhp = rclp->version_head;
+	rclp->version_head = vhp->next;
+	rclp->head = rhp->next;
+	rclp->len--;
+
+	if (!rclp->head) {
+		rclp->tail = &rclp->head;
+		rclp->version_tail = &rclp->version_head;
+	}
+
+	return rhp;
+}
+
 static inline void prcu_report(struct prcu_local_struct *local)
 {
 	unsigned long long global_version;
@@ -117,6 +148,7 @@ void synchronize_prcu(void)
 	if (atomic_read(&prcu->active_ctr))
 		wait_event(prcu->wait_q, !atomic_read(&prcu->active_ctr));
 
+	atomic64_set(&prcu->cb_version, version);
 	mutex_unlock(&prcu->mtx);
 }
 EXPORT_SYMBOL(synchronize_prcu);
@@ -166,6 +198,58 @@ void call_prcu(struct rcu_head *head, rcu_callback_t func)
 }
 EXPORT_SYMBOL(call_prcu);
 
+int prcu_pending(void)
+{
+	struct prcu_local_struct *local = get_cpu_ptr(&prcu_local);
+	unsigned long long cb_version = local->cb_version;
+	struct prcu_cblist *rclp = &local->cblist;
+
+	put_cpu_ptr(&prcu_local);
+	return cb_version < atomic64_read(&prcu->cb_version) && rclp->head;
+}
+
+void invoke_prcu_core(void)
+{
+	if (cpu_online(smp_processor_id()))
+		raise_softirq(PRCU_SOFTIRQ);
+}
+
+void prcu_check_callbacks(void)
+{
+	if (prcu_pending())
+		invoke_prcu_core();
+}
+
+static __latent_entropy void prcu_process_callbacks(struct softirq_action *unused)
+{
+	unsigned long flags;
+	unsigned long long cb_version;
+	struct prcu_local_struct *local;
+	struct prcu_cblist *rclp;
+	struct rcu_head *rhp;
+	struct prcu_version_head *vhp;
+
+	if (cpu_is_offline(smp_processor_id()))
+		return;
+
+	cb_version = atomic64_read(&prcu->cb_version);
+
+	/* Disable interrupts to prevent races with call_prcu() */
+	local_irq_save(flags);
+	local = this_cpu_ptr(&prcu_local);
+	rclp = &local->cblist;
+	rhp = rclp->head;
+	vhp = rclp->version_head;
+	for (; rhp && vhp && vhp->version < cb_version;
+	     rhp = rclp->head, vhp = rclp->version_head) {
+		rhp = prcu_cblist_dequeue(rclp);
+		debug_rcu_head_unqueue(rhp);
+		rhp->func(rhp);
+	}
+	local->cb_version = cb_version;
+	local_irq_restore(flags);
+}
+
 void prcu_init_local_struct(int cpu)
 {
 	struct prcu_local_struct *local;
@@ -174,6 +258,7 @@ void prcu_init_local_struct(int cpu)
 	local->locked = 0;
 	local->online = 0;
 	local->version = 0;
+	local->cb_version = 0;
 	prcu_cblist_init(&local->cblist);
 }
 
@@ -181,6 +266,7 @@ void __init prcu_init(void)
 {
 	int cpu;
 
+	open_softirq(PRCU_SOFTIRQ, prcu_process_callbacks);
 	for_each_possible_cpu(cpu)
 		prcu_init_local_struct(cpu);
 }
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index e354e475..46910114 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -2852,6 +2852,7 @@ void rcu_check_callbacks(int user)
 {
 	trace_rcu_utilization(TPS("Start scheduler-tick"));
 	increment_cpu_stall_ticks();
+
 	if (user || rcu_is_cpu_rrupt_from_idle()) {
 
 		/*
diff --git a/kernel/time/timer.c b/kernel/time/timer.c
index d3f33020..ed863e63 100644
--- a/kernel/time/timer.c
+++ b/kernel/time/timer.c
@@ -44,6 +44,7 @@
 #include <linux/sched/debug.h>
 #include <linux/slab.h>
 #include <linux/compat.h>
+#include <linux/prcu.h>
 
 #include <linux/uaccess.h>
 #include <asm/unistd.h>
@@ -1568,6 +1569,7 @@ void update_process_times(int user_tick)
 	/* Note: this timer irq context must be accounted for as well. */
 	account_process_tick(p, user_tick);
 	run_local_timers();
+	prcu_check_callbacks();
 	rcu_check_callbacks(user_tick);
 #ifdef CONFIG_IRQ_WORK
 	if (in_irq())
-- 
2.14.1.729.g59c0ea183