[PATCH] srcutiny: Enable atomic SRCU flavor tracking

[Kunwu Chan]

From: Kunwu Chan <kunwu.chan@xxxxxxxxx>

Set srcu_reader_flavor to SRCU_READ_FLAVOR_ATOMIC in the Tiny SRCU
atomic initialization paths, so that the entry-point checks added by
the previous commit can identify atomic SRCU domains.

For static initialization, add a flavor parameter to
__SRCU_STRUCT_INIT() and pass SRCU_READ_FLAVOR_ATOMIC through
DEFINE_SRCU_ATOMIC() and DEFINE_STATIC_SRCU_ATOMIC().

For dynamic initialization, initialize srcu_reader_flavor to zero in
init_srcu_struct_fields() for the generic initialization path, and
set it to SRCU_READ_FLAVOR_ATOMIC in init_srcu_struct_atomic().
Handle both CONFIG_DEBUG_LOCK_ALLOC and non-debug initialization paths.

Enable srcu_check_read_flavor() for Tiny SRCU, matching the Tree SRCU
behavior, so that readers can verify that the requested flavor matches
the SRCU domain.

Signed-off-by: Kunwu Chan <kunwu.chan@xxxxxxxxx>
---
 include/linux/srcutiny.h | 39 +++++++++++++++++++++++++++++----------
 kernel/rcu/srcutiny.c    |  1 +
 2 files changed, 30 insertions(+), 10 deletions(-)

diff --git a/include/linux/srcutiny.h b/include/linux/srcutiny.h
index 2b293336525a..9dce4b5aa084 100644
--- a/include/linux/srcutiny.h
+++ b/include/linux/srcutiny.h
@@ -40,7 +40,7 @@ void srcu_drive_gp(struct work_struct *wp);
 void srcu_tiny_irq_work(struct irq_work *irq_work);
 void srcu_defer_drain(struct irq_work *irq_work);
 
-#define __SRCU_STRUCT_INIT(name, __ignored, ___ignored, ____ignored)	\
+#define __SRCU_STRUCT_INIT(name, __ignored, ___ignored, flavor)		\
 {									\
 	.srcu_wq = __SWAIT_QUEUE_HEAD_INITIALIZER(name.srcu_wq),	\
 	.srcu_cb_tail = &name.srcu_cb_head,				\
@@ -49,6 +49,7 @@ void srcu_defer_drain(struct irq_work *irq_work);
 	.defer_cbs = LLIST_HEAD_INIT(name.defer_cbs),			\
 	.defer_iw = { .node = { .u_flags = IRQ_WORK_HARD_IRQ },		\
 		      .func = srcu_defer_drain },			\
+	.srcu_reader_flavor = flavor,					\
 	__SRCU_DEP_MAP_INIT(name)					\
 }
 
@@ -57,29 +58,43 @@ void srcu_defer_drain(struct irq_work *irq_work);
  * Tree SRCU, which needs some per-CPU data.
  */
 #define DEFINE_SRCU(name) \
-	struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name, name)
+	struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name, 0)
 #define DEFINE_STATIC_SRCU(name) \
-	static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name, name)
+	static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name, 0)
 #define DEFINE_SRCU_FAST(name) DEFINE_SRCU(name)
 #define DEFINE_STATIC_SRCU_FAST(name) \
-	static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name, name)
+	static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name, 0)
 #define DEFINE_SRCU_FAST_UPDOWN(name) DEFINE_SRCU(name)
 #define DEFINE_STATIC_SRCU_FAST_UPDOWN(name) \
-	static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name, name)
-#define DEFINE_SRCU_ATOMIC(name) DEFINE_SRCU(name)
+	static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name, 0)
+#define DEFINE_SRCU_ATOMIC(name) \
+	struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name, SRCU_READ_FLAVOR_ATOMIC)
 #define DEFINE_STATIC_SRCU_ATOMIC(name) \
-	static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name, name)
+	static struct srcu_struct name = \
+		__SRCU_STRUCT_INIT(name, name, name, SRCU_READ_FLAVOR_ATOMIC)
 
 // Dummy structure for srcu_notifier_head.
 struct srcu_usage { };
 #define __SRCU_USAGE_INIT(name) { }
 #define __init_srcu_struct_fast __init_srcu_struct
 #define __init_srcu_struct_fast_updown __init_srcu_struct
-#define __init_srcu_struct_atomic __init_srcu_struct
+#define __init_srcu_struct_atomic(ssp, name, key) \
+({ \
+	int __ret = __init_srcu_struct(ssp, name, key); \
+	if (!__ret) \
+		(ssp)->srcu_reader_flavor = SRCU_READ_FLAVOR_ATOMIC; \
+	__ret; \
+})
 #ifndef CONFIG_DEBUG_LOCK_ALLOC
 #define init_srcu_struct_fast init_srcu_struct
 #define init_srcu_struct_fast_updown init_srcu_struct
-#define init_srcu_struct_atomic init_srcu_struct
+#define init_srcu_struct_atomic(ssp)					\
+	({								\
+		int __ret = init_srcu_struct(ssp);			\
+		if (!__ret)						\
+			(ssp)->srcu_reader_flavor = SRCU_READ_FLAVOR_ATOMIC;\
+		__ret;							\
+	})
 #endif // #ifndef CONFIG_DEBUG_LOCK_ALLOC
 
 void synchronize_srcu(struct srcu_struct *ssp);
@@ -148,7 +163,11 @@ static inline void synchronize_srcu_expedited(struct srcu_struct *ssp)
 void srcu_barrier(struct srcu_struct *ssp);
 
 static inline void srcu_expedite_current(struct srcu_struct *ssp) { }
-#define srcu_check_read_flavor(ssp, read_flavor) do { } while (0)
+#define srcu_check_read_flavor(ssp, read_flavor) \
+	({ \
+		u8 __f = (ssp)->srcu_reader_flavor; \
+		WARN_ON_ONCE(__f && !(__f & (read_flavor))); \
+	})
 
 /* Defined here to avoid size increase for non-torture kernels. */
 static inline void srcu_torture_stats_print(struct srcu_struct *ssp,
diff --git a/kernel/rcu/srcutiny.c b/kernel/rcu/srcutiny.c
index 22f7716cbb0e..873b30ccf563 100644
--- a/kernel/rcu/srcutiny.c
+++ b/kernel/rcu/srcutiny.c
@@ -42,6 +42,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp)
 	ssp->srcu_gp_running = false;
 	ssp->srcu_gp_waiting = false;
 	ssp->srcu_atomic_gp_flag = 0;
+	ssp->srcu_reader_flavor = 0;
 	ssp->srcu_idx = 0;
 	ssp->srcu_idx_max = 0;
 	INIT_WORK(&ssp->srcu_work, srcu_drive_gp);
-- 
2.43.0