[PATCH v3 8/8] x86/irq: merge and functionalize common code in DECLARE/DEFINE_IDTENTRY_*
From: H. Peter Anvin
Date: Wed May 19 2021 - 17:22:42 EST
From: "H. Peter Anvin (Intel)" <hpa@xxxxxxxxx>
Move as much of the common code in the _IDTENTRY_ and
run_*_on_irq_stack() into inline functions instead of macros. A lot of
them differ only in types and/or the presence or absence of an
additional argument; the differential amount of code is neglible and
after bending the types a little bit, they unify nicely.
Signed-off-by: H. Peter Anvin (Intel) <hpa@xxxxxxxxx>
---
arch/x86/entry/common.c | 5 +-
arch/x86/include/asm/idtentry.h | 170 +++++++++++++++----------------
arch/x86/include/asm/irq_stack.h | 73 +++++--------
arch/x86/kernel/apic/apic.c | 2 +-
arch/x86/kernel/irq.c | 1 +
arch/x86/kernel/sev-es.c | 6 +-
arch/x86/kernel/traps.c | 2 +-
7 files changed, 117 insertions(+), 142 deletions(-)
diff --git a/arch/x86/entry/common.c b/arch/x86/entry/common.c
index 7b2542b13ebd..1e46a1a35b20 100644
--- a/arch/x86/entry/common.c
+++ b/arch/x86/entry/common.c
@@ -253,7 +253,8 @@ static __always_inline bool get_and_clear_inhcall(void) { return false; }
static __always_inline void restore_inhcall(bool inhcall) { }
#endif
-static void __xen_pv_evtchn_do_upcall(struct pt_regs *regs)
+static void __xen_pv_evtchn_do_upcall(struct pt_regs *regs,
+ u32 __dummy __always_unused)
{
struct pt_regs *old_regs = set_irq_regs(regs);
@@ -269,7 +270,7 @@ __visible noinstr void xen_pv_evtchn_do_upcall(struct pt_regs *regs)
irqentry_state_t state = irqentry_enter(regs);
bool inhcall;
- run_sysvec_on_irqstack_cond(__xen_pv_evtchn_do_upcall, regs);
+ run_on_irqstack_cond(__xen_pv_evtchn_do_upcall, regs, 0);
inhcall = get_and_clear_inhcall();
if (inhcall && !WARN_ON_ONCE(state.exit_rcu)) {
diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentry.h
index c03a18cac78e..3932c770551e 100644
--- a/arch/x86/include/asm/idtentry.h
+++ b/arch/x86/include/asm/idtentry.h
@@ -11,6 +11,65 @@
#include <asm/irq_stack.h>
+/**
+ * idtentry_call - Common code for non-trivial IDT entry points
+ * @func: What should ultimately be called
+ * @regs: Pointer to entry struct pt_regs
+ * @arg: Extra argument to func (vector, error_code)
+ * @flush: If kvm_set_cpu_l1tf_flush_l1d() should be called
+ * @wrapper: Wrapper to call func; NULL for just call
+ */
+static __always_inline void
+idtentry_call(irq_func_t func, struct pt_regs *regs, bool flush,
+ void (*wrapper)(irq_func_t, struct pt_regs *, u32),
+ u32 arg)
+{
+ irqentry_state_t state = irqentry_enter(regs);
+
+ instrumentation_begin();
+ if (flush)
+ kvm_set_cpu_l1tf_flush_l1d();
+ if (wrapper)
+ wrapper(func, regs, arg);
+ else
+ func(regs, arg);
+ instrumentation_end();
+ irqentry_exit(regs, state);
+}
+
+/**
+ * _DEFINE_IDTENTRY - Common macro for defining idtentries with no argument
+ * @func: Function name of the entry point
+ * @flush: If wrapper should call kvm_set_cpu_l1tf_flush_l1d()
+ * @inline_opt: __always_inline or noinline as appropriate for __func
+ * @wrapper: Wrapper function for calling __func
+ *
+ */
+#define _DEFINE_IDTENTRY(func, flush, inline_opt, wrapper) \
+static inline_opt void __##func(struct pt_regs *regs, u32); \
+__visible noinstr void func(struct pt_regs *regs) \
+{ \
+ idtentry_call(__##func, regs, flush, wrapper, 0); \
+} \
+static inline_opt void \
+__##func(struct pt_regs *regs, u32 __dummy __always_unused)
+
+/**
+ * _DEFINE_IDTENTRY_ERRORCODE - Common macro for defining idtentries with argument
+ * @func: Function name of the entry point
+ * @flush: If wrapper should call kvm_set_cpu_l1tf_flush_l1d()
+ * @inline_opt: __always_inline or noinline as appropriate for __func
+ * @wrapper: Wrapper function for calling __func
+ *
+ */
+#define _DEFINE_IDTENTRY_ERRORCODE(func, flush, inline_opt, wrapper) \
+static inline_opt void __##func(struct pt_regs *regs, u32 error_code); \
+__visible noinstr void func(struct pt_regs *regs, u32 error_code) \
+{ \
+ idtentry_call(__##func, regs, flush, wrapper, error_code); \
+} \
+static inline_opt void __##func(struct pt_regs *regs, u32 error_code)
+
/**
* DECLARE_IDTENTRY - Declare functions for simple IDT entry points
* No error code pushed by hardware
@@ -45,19 +104,7 @@
* which has to run before returning to the low level assembly code.
*/
#define DEFINE_IDTENTRY(func) \
-static __always_inline void __##func(struct pt_regs *regs); \
- \
-__visible noinstr void func(struct pt_regs *regs) \
-{ \
- irqentry_state_t state = irqentry_enter(regs); \
- \
- instrumentation_begin(); \
- __##func (regs); \
- instrumentation_end(); \
- irqentry_exit(regs, state); \
-} \
- \
-static __always_inline void __##func(struct pt_regs *regs)
+ _DEFINE_IDTENTRY(func, false, __always_inline, NULL)
/* Special case for 32bit IRET 'trap' */
#define DECLARE_IDTENTRY_SW DECLARE_IDTENTRY
@@ -80,7 +127,7 @@ static __always_inline void __##func(struct pt_regs *regs)
#define DECLARE_IDTENTRY_ERRORCODE(vector, func) \
asmlinkage void asm_##func(void); \
asmlinkage void xen_asm_##func(void); \
- __visible void func(struct pt_regs *regs, unsigned long error_code)
+ __visible void func(struct pt_regs *regs, u32 error_code)
/**
* DEFINE_IDTENTRY_ERRORCODE - Emit code for simple IDT entry points
@@ -90,22 +137,7 @@ static __always_inline void __##func(struct pt_regs *regs)
* Same as DEFINE_IDTENTRY, but has an extra error_code argument
*/
#define DEFINE_IDTENTRY_ERRORCODE(func) \
-static __always_inline void __##func(struct pt_regs *regs, \
- unsigned long error_code); \
- \
-__visible noinstr void func(struct pt_regs *regs, \
- unsigned long error_code) \
-{ \
- irqentry_state_t state = irqentry_enter(regs); \
- \
- instrumentation_begin(); \
- __##func (regs, error_code); \
- instrumentation_end(); \
- irqentry_exit(regs, state); \
-} \
- \
-static __always_inline void __##func(struct pt_regs *regs, \
- unsigned long error_code)
+ _DEFINE_IDTENTRY_ERRORCODE(func, false, __always_inline, NULL)
/**
* DECLARE_IDTENTRY_RAW - Declare functions for raw IDT entry points
@@ -161,7 +193,7 @@ __visible noinstr void func(struct pt_regs *regs)
* is required before the enter/exit() helpers are invoked.
*/
#define DEFINE_IDTENTRY_RAW_ERRORCODE(func) \
-__visible noinstr void func(struct pt_regs *regs, unsigned long error_code)
+__visible noinstr void func(struct pt_regs *regs, u32 error_code)
/**
* DECLARE_IDTENTRY_IRQ - Declare functions for device interrupt IDT entry
@@ -187,25 +219,11 @@ __visible noinstr void func(struct pt_regs *regs, unsigned long error_code)
* has to be done in the function body if necessary.
*/
#define DEFINE_IDTENTRY_IRQ(func) \
-static void __##func(struct pt_regs *regs, u32 vector); \
- \
-__visible noinstr void func(struct pt_regs *regs, \
- unsigned long error_code) \
-{ \
- irqentry_state_t state = irqentry_enter(regs); \
- u32 vector = (u32)(u8)error_code; \
- \
- instrumentation_begin(); \
- kvm_set_cpu_l1tf_flush_l1d(); \
- run_irq_on_irqstack_cond(__##func, regs, vector); \
- instrumentation_end(); \
- irqentry_exit(regs, state); \
-} \
- \
-static noinline void __##func(struct pt_regs *regs, u32 vector)
+ _DEFINE_IDTENTRY_ERRORCODE(func, true, noinline, run_on_irqstack_cond)
/**
- * DECLARE_IDTENTRY_SYSVEC - Declare functions for system vector entry points
+ * DECLARE_IDTENTRY - Declare functions for simple IDT entry points
+ * No error code pushed by hardware
* @vector: Vector number (ignored for C)
* @func: Function name of the entry point
*
@@ -214,9 +232,11 @@ static noinline void __##func(struct pt_regs *regs, u32 vector)
* - The XEN PV trap entry point: xen_##func (maybe unused)
* - The C handler called from the ASM entry point
*
- * Maps to DECLARE_IDTENTRY().
+ * Note: This is the C variant of DECLARE_IDTENTRY(). As the name says it
+ * declares the entry points for usage in C code. There is an ASM variant
+ * as well which is used to emit the entry stubs in entry_32/64.S.
*/
-#define DECLARE_IDTENTRY_SYSVEC(vector, func) \
+#define DECLARE_IDTENTRY_SYSVEC(vector, func) \
DECLARE_IDTENTRY(vector, func)
/**
@@ -229,20 +249,7 @@ static noinline void __##func(struct pt_regs *regs, u32 vector)
* Runs the function on the interrupt stack if the entry hit kernel mode
*/
#define DEFINE_IDTENTRY_SYSVEC(func) \
-static void __##func(struct pt_regs *regs); \
- \
-__visible noinstr void func(struct pt_regs *regs) \
-{ \
- irqentry_state_t state = irqentry_enter(regs); \
- \
- instrumentation_begin(); \
- kvm_set_cpu_l1tf_flush_l1d(); \
- run_sysvec_on_irqstack_cond(__##func, regs); \
- instrumentation_end(); \
- irqentry_exit(regs, state); \
-} \
- \
-static noinline void __##func(struct pt_regs *regs)
+ _DEFINE_IDTENTRY(func, true, noinline, run_on_irqstack_cond)
/**
* DEFINE_IDTENTRY_SYSVEC_SIMPLE - Emit code for simple system vector IDT
@@ -255,23 +262,16 @@ static noinline void __##func(struct pt_regs *regs)
* Only use for 'empty' vectors like reschedule IPI and KVM posted
* interrupt vectors.
*/
+static __always_inline void
+call_sysvec_simple(irq_func_t func, struct pt_regs *regs, u32 arg)
+{
+ __irq_enter_raw();
+ func(regs, arg);
+ __irq_exit_raw();
+}
+
#define DEFINE_IDTENTRY_SYSVEC_SIMPLE(func) \
-static __always_inline void __##func(struct pt_regs *regs); \
- \
-__visible noinstr void func(struct pt_regs *regs) \
-{ \
- irqentry_state_t state = irqentry_enter(regs); \
- \
- instrumentation_begin(); \
- __irq_enter_raw(); \
- kvm_set_cpu_l1tf_flush_l1d(); \
- __##func (regs); \
- __irq_exit_raw(); \
- instrumentation_end(); \
- irqentry_exit(regs, state); \
-} \
- \
-static __always_inline void __##func(struct pt_regs *regs)
+ _DEFINE_IDTENTRY(func, true, __always_inline, call_sysvec_simple)
/**
* DECLARE_IDTENTRY_XENCB - Declare functions for XEN HV callback entry point
@@ -312,8 +312,8 @@ static __always_inline void __##func(struct pt_regs *regs)
*/
#define DECLARE_IDTENTRY_VC(vector, func) \
DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func); \
- __visible noinstr void ist_##func(struct pt_regs *regs, unsigned long error_code); \
- __visible noinstr void safe_stack_##func(struct pt_regs *regs, unsigned long error_code)
+ __visible noinstr void ist_##func(struct pt_regs *regs, u32 error_code); \
+ __visible noinstr void safe_stack_##func(struct pt_regs *regs, u32 error_code)
/**
* DEFINE_IDTENTRY_IST - Emit code for IST entry points
@@ -396,8 +396,7 @@ static __always_inline void __##func(struct pt_regs *regs)
*/
#define DECLARE_IDTENTRY_DF(vector, func) \
asmlinkage void asm_##func(void); \
- __visible void func(struct pt_regs *regs, \
- unsigned long error_code, \
+ __visible void func(struct pt_regs *regs, u32 error_code, \
unsigned long address)
/**
@@ -408,8 +407,7 @@ static __always_inline void __##func(struct pt_regs *regs)
* cr2 in the address argument.
*/
#define DEFINE_IDTENTRY_DF(func) \
-__visible noinstr void func(struct pt_regs *regs, \
- unsigned long error_code, \
+__visible noinstr void func(struct pt_regs *regs, u32 error_code, \
unsigned long address)
#endif /* !CONFIG_X86_64 */
diff --git a/arch/x86/include/asm/irq_stack.h b/arch/x86/include/asm/irq_stack.h
index 562854c60808..23d518c06b1d 100644
--- a/arch/x86/include/asm/irq_stack.h
+++ b/arch/x86/include/asm/irq_stack.h
@@ -6,6 +6,8 @@
#include <asm/processor.h>
+typedef void (*irq_func_t)(struct pt_regs *regs, u32 arg);
+
#ifdef CONFIG_X86_64
/*
@@ -132,7 +134,7 @@
}
/*
- * Function call sequence for __call_on_irqstack() for system vectors.
+ * Function call sequence for __call_on_irqstack() for irqs and system vectors.
*
* Note that irq_enter_rcu() and irq_exit_rcu() do not use the input
* mechanism because these functions are global and cannot be optimized out
@@ -145,27 +147,6 @@
* call to idtentry_exit() anyway, it's likely that it does not cause extra
* effort for this asm magic.
*/
-#define ASM_CALL_SYSVEC \
- "call irq_enter_rcu \n" \
- "movq %[arg1], %%rdi \n" \
- "call %P[__func] \n" \
- "call irq_exit_rcu \n"
-
-#define SYSVEC_CONSTRAINTS , [arg1] "r" (regs)
-
-#define run_sysvec_on_irqstack_cond(func, regs) \
-{ \
- assert_function_type(func, void (*)(struct pt_regs *)); \
- assert_arg_type(regs, struct pt_regs *); \
- \
- call_on_irqstack_cond(func, regs, ASM_CALL_SYSVEC, \
- SYSVEC_CONSTRAINTS, regs); \
-}
-
-/*
- * As in ASM_CALL_SYSVEC above the clobbers force the compiler to store
- * @regs and @vector in callee saved registers.
- */
#define ASM_CALL_IRQ \
"call irq_enter_rcu \n" \
"movq %[arg1], %%rdi \n" \
@@ -173,16 +154,13 @@
"call %P[__func] \n" \
"call irq_exit_rcu \n"
-#define IRQ_CONSTRAINTS , [arg1] "r" (regs), [arg2] "r" (vector)
+#define IRQ_CONSTRAINTS , [arg1] "r" (regs), [arg2] "r" (arg)
-#define run_irq_on_irqstack_cond(func, regs, vector) \
-{ \
- assert_function_type(func, void (*)(struct pt_regs *, u32)); \
- assert_arg_type(regs, struct pt_regs *); \
- assert_arg_type(vector, u32); \
- \
- call_on_irqstack_cond(func, regs, ASM_CALL_IRQ, \
- IRQ_CONSTRAINTS, regs, vector); \
+static __always_inline void
+run_on_irqstack_cond(irq_func_t func, struct pt_regs *regs, u32 arg)
+{
+ call_on_irqstack_cond(func, regs, ASM_CALL_IRQ,
+ IRQ_CONSTRAINTS, regs, arg);
}
#define ASM_CALL_SOFTIRQ \
@@ -194,28 +172,25 @@
* interrupts are pending to be processed. The interrupt stack cannot be in
* use here.
*/
-#define do_softirq_own_stack() \
-{ \
- __this_cpu_write(hardirq_stack_inuse, true); \
- call_on_irqstack(__do_softirq, ASM_CALL_SOFTIRQ); \
- __this_cpu_write(hardirq_stack_inuse, false); \
+static __always_inline void do_softirq_own_stack(void)
+{
+ __this_cpu_write(hardirq_stack_inuse, true);
+ call_on_irqstack(__do_softirq, ASM_CALL_SOFTIRQ);
+ __this_cpu_write(hardirq_stack_inuse, false);
}
#else /* CONFIG_X86_64 */
-/* System vector handlers always run on the stack they interrupted. */
-#define run_sysvec_on_irqstack_cond(func, regs) \
-{ \
- irq_enter_rcu(); \
- func(regs); \
- irq_exit_rcu(); \
-}
-/* Switches to the irq stack within func() */
-#define run_irq_on_irqstack_cond(func, regs, vector) \
-{ \
- irq_enter_rcu(); \
- func(regs, vector); \
- irq_exit_rcu(); \
+/*
+ * System vector handlers always run on the stack they interrupted;
+ * irqs switch to the irq stack within func().
+ */
+static __always_inline void
+run_on_irqstack_cond(irq_func_t func, struct pt_regs *regs, u32 arg)
+{
+ irq_enter_rcu();
+ func(regs, arg);
+ irq_exit_rcu();
}
#endif /* !CONFIG_X86_64 */
diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c
index 4a39fb429f15..64832869e1a1 100644
--- a/arch/x86/kernel/apic/apic.c
+++ b/arch/x86/kernel/apic/apic.c
@@ -2188,7 +2188,7 @@ static noinline void handle_spurious_interrupt(u8 vector)
*/
DEFINE_IDTENTRY_IRQ(spurious_interrupt)
{
- handle_spurious_interrupt(vector);
+ handle_spurious_interrupt((u8)error_code);
}
DEFINE_IDTENTRY_SYSVEC(sysvec_spurious_apic_interrupt)
diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c
index e28f6a5d14f1..27994818d9b1 100644
--- a/arch/x86/kernel/irq.c
+++ b/arch/x86/kernel/irq.c
@@ -239,6 +239,7 @@ static __always_inline void handle_irq(struct irq_desc *desc,
*/
DEFINE_IDTENTRY_IRQ(common_interrupt)
{
+ const u8 vector = error_code;
struct pt_regs *old_regs = set_irq_regs(regs);
struct irq_desc *desc;
diff --git a/arch/x86/kernel/sev-es.c b/arch/x86/kernel/sev-es.c
index 73873b007838..bdd75b71d0f3 100644
--- a/arch/x86/kernel/sev-es.c
+++ b/arch/x86/kernel/sev-es.c
@@ -1335,15 +1335,15 @@ DEFINE_IDTENTRY_VC_SAFE_STACK(exc_vmm_communication)
vc_finish_insn(&ctxt);
break;
case ES_UNSUPPORTED:
- pr_err_ratelimited("Unsupported exit-code 0x%02lx in early #VC exception (IP: 0x%lx)\n",
+ pr_err_ratelimited("Unsupported exit-code 0x%02x in early #VC exception (IP: 0x%lx)\n",
error_code, regs->ip);
goto fail;
case ES_VMM_ERROR:
- pr_err_ratelimited("Failure in communication with VMM (exit-code 0x%02lx IP: 0x%lx)\n",
+ pr_err_ratelimited("Failure in communication with VMM (exit-code 0x%02x IP: 0x%lx)\n",
error_code, regs->ip);
goto fail;
case ES_DECODE_FAILED:
- pr_err_ratelimited("Failed to decode instruction (exit-code 0x%02lx IP: 0x%lx)\n",
+ pr_err_ratelimited("Failed to decode instruction (exit-code 0x%02x IP: 0x%lx)\n",
error_code, regs->ip);
goto fail;
case ES_EXCEPTION:
diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index 853ea7a80806..f5aecbb44bc7 100644
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -461,7 +461,7 @@ DEFINE_IDTENTRY_DF(exc_double_fault)
}
#endif
- pr_emerg("PANIC: double fault, error_code: 0x%lx\n", error_code);
+ pr_emerg("PANIC: double fault, error_code: 0x%x\n", error_code);
die("double fault", regs, error_code);
panic("Machine halted.");
instrumentation_end();
--
2.31.1