Re: [PATCH v2 2/6] riscv: Deduplicate IRQ stack switching

[Clément Léger]

On 15/08/2023 22:34, Sami Tolvanen wrote:
> With CONFIG_IRQ_STACKS, we switch to a separate per-CPU IRQ stack
> before calling handle_riscv_irq or __do_softirq. We currently
> have duplicate inline assembly snippets for stack switching in
> both code paths. Now that we can access per-CPU variables in
> assembly, implement call_on_irq_stack in assembly, and use that
> instead of redudant inline assembly.
> 
> Signed-off-by: Sami Tolvanen <samitolvanen@xxxxxxxxxx>
> ---
>  arch/riscv/include/asm/asm.h       |  5 +++++
>  arch/riscv/include/asm/irq_stack.h |  3 +++
>  arch/riscv/kernel/entry.S          | 32 ++++++++++++++++++++++++++++++
>  arch/riscv/kernel/irq.c            | 32 ++++++++----------------------
>  arch/riscv/kernel/traps.c          | 29 ++++-----------------------
>  5 files changed, 52 insertions(+), 49 deletions(-)
> 
> diff --git a/arch/riscv/include/asm/asm.h b/arch/riscv/include/asm/asm.h
> index bfb4c26f113c..8e446be2d57c 100644
> --- a/arch/riscv/include/asm/asm.h
> +++ b/arch/riscv/include/asm/asm.h
> @@ -104,6 +104,11 @@
>  .endm
>  #endif /* CONFIG_SMP */
>  
> +.macro load_per_cpu dst ptr tmp
> +	asm_per_cpu \dst \ptr \tmp
> +	REG_L \dst, 0(\dst)
> +.endm
> +
>  	/* save all GPs except x1 ~ x5 */
>  	.macro save_from_x6_to_x31
>  	REG_S x6,  PT_T1(sp)
> diff --git a/arch/riscv/include/asm/irq_stack.h b/arch/riscv/include/asm/irq_stack.h
> index e4042d297580..6441ded3b0cf 100644
> --- a/arch/riscv/include/asm/irq_stack.h
> +++ b/arch/riscv/include/asm/irq_stack.h
> @@ -12,6 +12,9 @@
>  
>  DECLARE_PER_CPU(ulong *, irq_stack_ptr);
>  
> +asmlinkage void call_on_irq_stack(struct pt_regs *regs,
> +				  void (*func)(struct pt_regs *));
> +
>  #ifdef CONFIG_VMAP_STACK
>  /*
>   * To ensure that VMAP'd stack overflow detection works correctly, all VMAP'd
> diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S
> index 3d11aa3af105..39875f5e08a6 100644
> --- a/arch/riscv/kernel/entry.S
> +++ b/arch/riscv/kernel/entry.S
> @@ -218,6 +218,38 @@ SYM_CODE_START(ret_from_fork)
>  	tail syscall_exit_to_user_mode
>  SYM_CODE_END(ret_from_fork)
>  
> +#ifdef CONFIG_IRQ_STACKS
> +/*
> + * void call_on_irq_stack(struct pt_regs *regs,
> + * 		          void (*func)(struct pt_regs *));
> + *
> + * Calls func(regs) using the per-CPU IRQ stack.
> + */
> +SYM_FUNC_START(call_on_irq_stack)
> +	/* Create a frame record to save ra and s0 (fp) */
> +	addi	sp, sp, -RISCV_SZPTR
> +	REG_S	ra, (sp)
> +	addi	sp, sp, -RISCV_SZPTR
> +	REG_S	s0, (sp)
> +	addi	s0, sp, 2*RISCV_SZPTR

Hi Sami,

Some defines for stack frame offsets could be added in asm-offsets for
the offsets:

	DEFINE(STACKFRAME_SIZE_ON_STACK, ALIGN(sizeof(struct stackframe),
STACK_ALIGN));
	OFFSET(STACKFRAME_FP, stackframe, fp);
	OFFSET(STACKFRAME_RA, stackframe, ra);

And you can probably increment the stack at once (saving two addi in
prologue/epilogue) for both RA and FP and reuse the asm-offsets defines:

	addi	sp, sp, -STACKFRAME_SIZE_ON_STACK
	REG_S	ra, STACKFRAME_RA(sp)
	REG_S	s0, STACKFRAME_FP(sp)
	addi	s0, sp, STACKFRAME_SIZE_ON_STACK

Clément

> +
> +	/* Switch to the per-CPU IRQ stack and call the handler */
> +	load_per_cpu t0, irq_stack_ptr, t1
> +	li	t1, IRQ_STACK_SIZE
> +	add	sp, t0, t1
> +	jalr	a1
> +
> +	/* Switch back to the thread stack and restore ra and s0 */
> +	addi	sp, s0, -2*RISCV_SZPTR
> +	REG_L	s0, (sp)
> +	addi	sp, sp, RISCV_SZPTR
> +	REG_L	ra, (sp)
> +	addi	sp, sp, RISCV_SZPTR
> +
> +	ret
> +SYM_FUNC_END(call_on_irq_stack)
> +#endif /* CONFIG_IRQ_STACKS */
> +
>  /*
>   * Integer register context switch
>   * The callee-saved registers must be saved and restored.
> diff --git a/arch/riscv/kernel/irq.c b/arch/riscv/kernel/irq.c
> index d0577cc6a081..95dafdcbd135 100644
> --- a/arch/riscv/kernel/irq.c
> +++ b/arch/riscv/kernel/irq.c
> @@ -61,32 +61,16 @@ static void init_irq_stacks(void)
>  #endif /* CONFIG_VMAP_STACK */
>  
>  #ifdef CONFIG_HAVE_SOFTIRQ_ON_OWN_STACK
> +static void ___do_softirq(struct pt_regs *regs)
> +{
> +	__do_softirq();
> +}
> +
>  void do_softirq_own_stack(void)
>  {
> -#ifdef CONFIG_IRQ_STACKS
> -	if (on_thread_stack()) {
> -		ulong *sp = per_cpu(irq_stack_ptr, smp_processor_id())
> -					+ IRQ_STACK_SIZE/sizeof(ulong);
> -		__asm__ __volatile(
> -		"addi	sp, sp, -"RISCV_SZPTR  "\n"
> -		REG_S"  ra, (sp)		\n"
> -		"addi	sp, sp, -"RISCV_SZPTR  "\n"
> -		REG_S"  s0, (sp)		\n"
> -		"addi	s0, sp, 2*"RISCV_SZPTR "\n"
> -		"move	sp, %[sp]		\n"
> -		"call	__do_softirq		\n"
> -		"addi	sp, s0, -2*"RISCV_SZPTR"\n"
> -		REG_L"  s0, (sp)		\n"
> -		"addi	sp, sp, "RISCV_SZPTR   "\n"
> -		REG_L"  ra, (sp)		\n"
> -		"addi	sp, sp, "RISCV_SZPTR   "\n"
> -		:
> -		: [sp] "r" (sp)
> -		: "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7",
> -		  "t0", "t1", "t2", "t3", "t4", "t5", "t6",
> -		  "memory");
> -	} else
> -#endif
> +	if (on_thread_stack())
> +		call_on_irq_stack(NULL, ___do_softirq);
> +	else
>  		__do_softirq();
>  }
>  #endif /* CONFIG_HAVE_SOFTIRQ_ON_OWN_STACK */
> diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c
> index deb2144d9143..83319b6816da 100644
> --- a/arch/riscv/kernel/traps.c
> +++ b/arch/riscv/kernel/traps.c
> @@ -350,31 +350,10 @@ static void noinstr handle_riscv_irq(struct pt_regs *regs)
>  asmlinkage void noinstr do_irq(struct pt_regs *regs)
>  {
>  	irqentry_state_t state = irqentry_enter(regs);
> -#ifdef CONFIG_IRQ_STACKS
> -	if (on_thread_stack()) {
> -		ulong *sp = per_cpu(irq_stack_ptr, smp_processor_id())
> -					+ IRQ_STACK_SIZE/sizeof(ulong);
> -		__asm__ __volatile(
> -		"addi	sp, sp, -"RISCV_SZPTR  "\n"
> -		REG_S"  ra, (sp)		\n"
> -		"addi	sp, sp, -"RISCV_SZPTR  "\n"
> -		REG_S"  s0, (sp)		\n"
> -		"addi	s0, sp, 2*"RISCV_SZPTR "\n"
> -		"move	sp, %[sp]		\n"
> -		"move	a0, %[regs]		\n"
> -		"call	handle_riscv_irq	\n"
> -		"addi	sp, s0, -2*"RISCV_SZPTR"\n"
> -		REG_L"  s0, (sp)		\n"
> -		"addi	sp, sp, "RISCV_SZPTR   "\n"
> -		REG_L"  ra, (sp)		\n"
> -		"addi	sp, sp, "RISCV_SZPTR   "\n"
> -		:
> -		: [sp] "r" (sp), [regs] "r" (regs)
> -		: "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7",
> -		  "t0", "t1", "t2", "t3", "t4", "t5", "t6",
> -		  "memory");
> -	} else
> -#endif
> +
> +	if (IS_ENABLED(CONFIG_IRQ_STACKS) && on_thread_stack())
> +		call_on_irq_stack(regs, handle_riscv_irq);
> +	else
>  		handle_riscv_irq(regs);
>  
>  	irqentry_exit(regs, state);