[patch V2 10/13] x86: Select CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK

From: Thomas Gleixner
Date: Tue Feb 09 2021 - 21:09:48 EST


From: Thomas Gleixner <tglx@xxxxxxxxxxxxx>

Now that all invocations of irq_exit_rcu() happen on the irq stack, turn on
CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK which causes the core code to invoke
__do_softirq() directly without going through do_softirq_own_stack().

That means do_softirq_own_stack() is only invoked from task context which
means it can't be on the irq stack. Remove the conditional from
run_softirq_on_irqstack_cond() and rename the function accordingly.

Signed-off-by: Thomas Gleixner <tglx@xxxxxxxxxxxxx>
Reviewed-by: Kees Cook <keescook@xxxxxxxxxxxx>

---
arch/x86/Kconfig | 1 +
arch/x86/include/asm/irq_stack.h | 19 ++++++++-----------
arch/x86/kernel/irq_64.c | 2 +-
3 files changed, 10 insertions(+), 12 deletions(-)

--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -187,6 +187,7 @@ config X86
select HAVE_HW_BREAKPOINT
select HAVE_IDE
select HAVE_IOREMAP_PROT
+ select HAVE_IRQ_EXIT_ON_IRQ_STACK if X86_64
select HAVE_IRQ_TIME_ACCOUNTING
select HAVE_KERNEL_BZIP2
select HAVE_KERNEL_GZIP
--- a/arch/x86/include/asm/irq_stack.h
+++ b/arch/x86/include/asm/irq_stack.h
@@ -189,19 +189,16 @@
"call %P[__func] \n"

/*
- * Macro to invoke __do_softirq on the irq stack. Contrary to the above
- * the only check which is necessary is whether the interrupt stack is
- * in use already.
+ * Macro to invoke __do_softirq on the irq stack. This is only called from
+ * task context when bottom halfs are about to be reenabled and soft
+ * interrupts are pending to be processed. The interrupt stack cannot be in
+ * use here.
*/
-#define run_softirq_on_irqstack_cond() \
+#define run_softirq_on_irqstack() \
{ \
- if (__this_cpu_read(hardirq_stack_inuse)) { \
- __do_softirq(); \
- } else { \
- __this_cpu_write(hardirq_stack_inuse, true); \
- call_on_irqstack(__do_softirq, ASM_CALL_SOFTIRQ); \
- __this_cpu_write(hardirq_stack_inuse, false); \
- } \
+ __this_cpu_write(hardirq_stack_inuse, true); \
+ call_on_irqstack(__do_softirq, ASM_CALL_SOFTIRQ); \
+ __this_cpu_write(hardirq_stack_inuse, false); \
}

#else /* CONFIG_X86_64 */
--- a/arch/x86/kernel/irq_64.c
+++ b/arch/x86/kernel/irq_64.c
@@ -76,5 +76,5 @@ int irq_init_percpu_irqstack(unsigned in

void do_softirq_own_stack(void)
{
- run_softirq_on_irqstack_cond();
+ run_softirq_on_irqstack();
}