[PATCH 08/19] context_tracking: Take NMI eqs entrypoints over RCU
From: Frederic Weisbecker
Date: Wed Mar 02 2022 - 10:50:15 EST
The RCU dynticks counter is going to be merged into the context tracking
subsystem. Prepare with moving the NMI extended quiescent states
entrypoints to context tracking. For now those are dumb redirection to
existing RCU calls.
Signed-off-by: Frederic Weisbecker <frederic@xxxxxxxxxx>
Cc: Paul E. McKenney <paulmck@xxxxxxxxxx>
Cc: Peter Zijlstra <peterz@xxxxxxxxxxxxx>
Cc: Thomas Gleixner <tglx@xxxxxxxxxxxxx>
Cc: Neeraj Upadhyay <quic_neeraju@xxxxxxxxxxx>
Cc: Uladzislau Rezki <uladzislau.rezki@xxxxxxxx>
Cc: Joel Fernandes <joel@xxxxxxxxxxxxxxxxx>
Cc: Boqun Feng <boqun.feng@xxxxxxxxx>
Cc: Nicolas Saenz Julienne <nsaenz@xxxxxxxxxx>
Cc: Marcelo Tosatti <mtosatti@xxxxxxxxxx>
Cc: Xiongfeng Wang <wangxiongfeng2@xxxxxxxxxx>
Cc: Yu Liao<liaoyu15@xxxxxxxxxx>
Cc: Phil Auld <pauld@xxxxxxxxxx>
Cc: Paul Gortmaker<paul.gortmaker@xxxxxxxxxxxxx>
Cc: Alex Belits <abelits@xxxxxxxxxxx>
---
Documentation/RCU/Design/Requirements/Requirements.rst | 2 +-
arch/Kconfig | 2 +-
arch/arm64/kernel/entry-common.c | 8 ++++----
include/linux/context_tracking_irq.h | 4 ++++
include/linux/hardirq.h | 4 ++--
kernel/context_tracking.c | 10 ++++++++++
kernel/entry/common.c | 4 ++--
kernel/extable.c | 4 ++--
kernel/trace/trace.c | 2 +-
9 files changed, 27 insertions(+), 13 deletions(-)
diff --git a/Documentation/RCU/Design/Requirements/Requirements.rst b/Documentation/RCU/Design/Requirements/Requirements.rst
index e3dd5d71c798..256cf260e864 100644
--- a/Documentation/RCU/Design/Requirements/Requirements.rst
+++ b/Documentation/RCU/Design/Requirements/Requirements.rst
@@ -1847,7 +1847,7 @@ normal interrupts. One way that this can happen is for code that
directly invokes ct_irq_enter() and ct_irq_exit() to be called
from an NMI handler. This astonishing fact of life prompted the current
code structure, which has ct_irq_enter() invoking
-rcu_nmi_enter() and ct_irq_exit() invoking rcu_nmi_exit().
+ct_nmi_enter() and ct_irq_exit() invoking ct_nmi_exit().
And yes, I also learned of this requirement the hard way.
Loadable Modules
diff --git a/arch/Kconfig b/arch/Kconfig
index 66b2b6d4717b..c22b8ca0eb01 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -785,7 +785,7 @@ config HAVE_CONTEXT_TRACKING_USER_OFFSTACK
- Critical entry code isn't preemptible (or better yet:
not interruptible).
- - No use of RCU read side critical sections, unless rcu_nmi_enter()
+ - No use of RCU read side critical sections, unless ct_nmi_enter()
got called.
- No use of instrumentation, unless instrumentation_begin() got
called.
diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c
index 43ca8cf4e1dd..6a1ea28731c8 100644
--- a/arch/arm64/kernel/entry-common.c
+++ b/arch/arm64/kernel/entry-common.c
@@ -158,7 +158,7 @@ static void noinstr arm64_enter_nmi(struct pt_regs *regs)
__nmi_enter();
lockdep_hardirqs_off(CALLER_ADDR0);
lockdep_hardirq_enter();
- rcu_nmi_enter();
+ ct_nmi_enter();
trace_hardirqs_off_finish();
ftrace_nmi_enter();
@@ -179,7 +179,7 @@ static void noinstr arm64_exit_nmi(struct pt_regs *regs)
lockdep_hardirqs_on_prepare(CALLER_ADDR0);
}
- rcu_nmi_exit();
+ ct_nmi_exit();
lockdep_hardirq_exit();
if (restore)
lockdep_hardirqs_on(CALLER_ADDR0);
@@ -196,7 +196,7 @@ static void noinstr arm64_enter_el1_dbg(struct pt_regs *regs)
regs->lockdep_hardirqs = lockdep_hardirqs_enabled();
lockdep_hardirqs_off(CALLER_ADDR0);
- rcu_nmi_enter();
+ ct_nmi_enter();
trace_hardirqs_off_finish();
}
@@ -215,7 +215,7 @@ static void noinstr arm64_exit_el1_dbg(struct pt_regs *regs)
lockdep_hardirqs_on_prepare(CALLER_ADDR0);
}
- rcu_nmi_exit();
+ ct_nmi_exit();
if (restore)
lockdep_hardirqs_on(CALLER_ADDR0);
}
diff --git a/include/linux/context_tracking_irq.h b/include/linux/context_tracking_irq.h
index 60e3ed15a04e..11043bf724b7 100644
--- a/include/linux/context_tracking_irq.h
+++ b/include/linux/context_tracking_irq.h
@@ -7,11 +7,15 @@ void ct_irq_enter(void);
void ct_irq_exit(void);
void ct_irq_enter_irqson(void);
void ct_irq_exit_irqson(void);
+void ct_nmi_enter(void);
+void ct_nmi_exit(void);
#else
static inline void ct_irq_enter(void) { }
static inline void ct_irq_exit(void) { }
static inline void ct_irq_enter_irqson(void) { }
static inline void ct_irq_exit_irqson(void) { }
+static inline void ct_nmi_enter(void) { }
+static inline void ct_nmi_exit(void) { }
#endif
#endif
diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h
index 76878b357ffa..345cdbe9c1b7 100644
--- a/include/linux/hardirq.h
+++ b/include/linux/hardirq.h
@@ -124,7 +124,7 @@ extern void rcu_nmi_exit(void);
do { \
__nmi_enter(); \
lockdep_hardirq_enter(); \
- rcu_nmi_enter(); \
+ ct_nmi_enter(); \
instrumentation_begin(); \
ftrace_nmi_enter(); \
instrumentation_end(); \
@@ -143,7 +143,7 @@ extern void rcu_nmi_exit(void);
instrumentation_begin(); \
ftrace_nmi_exit(); \
instrumentation_end(); \
- rcu_nmi_exit(); \
+ ct_nmi_exit(); \
lockdep_hardirq_exit(); \
__nmi_exit(); \
} while (0)
diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c
index b63ff851472e..1686cd528966 100644
--- a/kernel/context_tracking.c
+++ b/kernel/context_tracking.c
@@ -267,3 +267,13 @@ void ct_irq_exit_irqson(void)
{
rcu_irq_exit_irqson();
}
+
+noinstr void ct_nmi_enter(void)
+{
+ rcu_nmi_enter();
+}
+
+noinstr void ct_nmi_exit(void)
+{
+ rcu_nmi_exit();
+}
diff --git a/kernel/entry/common.c b/kernel/entry/common.c
index cebc98b8adc6..08230507793f 100644
--- a/kernel/entry/common.c
+++ b/kernel/entry/common.c
@@ -449,7 +449,7 @@ irqentry_state_t noinstr irqentry_nmi_enter(struct pt_regs *regs)
__nmi_enter();
lockdep_hardirqs_off(CALLER_ADDR0);
lockdep_hardirq_enter();
- rcu_nmi_enter();
+ ct_nmi_enter();
instrumentation_begin();
trace_hardirqs_off_finish();
@@ -469,7 +469,7 @@ void noinstr irqentry_nmi_exit(struct pt_regs *regs, irqentry_state_t irq_state)
}
instrumentation_end();
- rcu_nmi_exit();
+ ct_nmi_exit();
lockdep_hardirq_exit();
if (irq_state.lockdep)
lockdep_hardirqs_on(CALLER_ADDR0);
diff --git a/kernel/extable.c b/kernel/extable.c
index b6f330f0fe74..88d4d739c5a1 100644
--- a/kernel/extable.c
+++ b/kernel/extable.c
@@ -113,7 +113,7 @@ int kernel_text_address(unsigned long addr)
/* Treat this like an NMI as it can happen anywhere */
if (no_rcu)
- rcu_nmi_enter();
+ ct_nmi_enter();
if (is_module_text_address(addr))
goto out;
@@ -126,7 +126,7 @@ int kernel_text_address(unsigned long addr)
ret = 0;
out:
if (no_rcu)
- rcu_nmi_exit();
+ ct_nmi_exit();
return ret;
}
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 7c500c708180..9434da82af8a 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -3086,7 +3086,7 @@ void __trace_stack(struct trace_array *tr, unsigned int trace_ctx,
}
/*
- * When an NMI triggers, RCU is enabled via rcu_nmi_enter(),
+ * When an NMI triggers, RCU is enabled via ct_nmi_enter(),
* but if the above rcu_is_watching() failed, then the NMI
* triggered someplace critical, and ct_irq_enter() should
* not be called from NMI.
--
2.25.1