Re: [PATCH v2 06/21] arm64: Store 'current' in TPIDRRO_EL0 instead of SP_EL0

[Jinjie Ruan]

在 2026/9/19 0:13, Will Deacon 写道:
> To make SP_EL0 available to point at the per-cpu kernel overflow stack,
> we must relocate 'current' elsewhere. With TPIDRRO_EL0 now restored late
> in the return-to-user path, we can otherwise use it to hold 'current'
> and update the various accessors accordingly.
> 
> Move 'current' from SP_EL0 to TPIDRRO_EL0.
> 
> Cc: Mark Rutland <mark.rutland@xxxxxxx>
> Signed-off-by: Will Deacon <will@xxxxxxxxxx>
> ---
>  arch/arm64/include/asm/assembler.h |  2 +-
>  arch/arm64/include/asm/current.h   |  6 +++---
>  arch/arm64/kernel/entry.S          | 20 ++++++++++----------
>  arch/arm64/kernel/head.S           |  2 +-
>  arch/arm64/kernel/process.c        |  5 +++--
>  arch/arm64/mm/proc.S               |  6 +++++-
>  arch/arm64/net/bpf_jit_comp.c      |  4 ++--
>  7 files changed, 25 insertions(+), 20 deletions(-)
> 
> diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
> index 0b58b550e8dc..fd1ae5337935 100644
> --- a/arch/arm64/include/asm/assembler.h
> +++ b/arch/arm64/include/asm/assembler.h
> @@ -587,7 +587,7 @@ alternative_else_nop_endif
>   * Return the current task_struct.
>   */
>  	.macro	get_current_task, rd
> -	mrs	\rd, sp_el0
> +	mrs	\rd, tpidrro_el0
>  	.endm
>  
>  /*
> diff --git a/arch/arm64/include/asm/current.h b/arch/arm64/include/asm/current.h
> index c92912eaf186..8c7efac05394 100644
> --- a/arch/arm64/include/asm/current.h
> +++ b/arch/arm64/include/asm/current.h
> @@ -14,11 +14,11 @@ struct task_struct;
>   */
>  static __always_inline struct task_struct *get_current(void)
>  {
> -	unsigned long sp_el0;
> +	unsigned long tpidrro_el0;
>  
> -	asm ("mrs %0, sp_el0" : "=r" (sp_el0));
> +	asm ("mrs %0, tpidrro_el0" : "=r" (tpidrro_el0));
>  
> -	return (struct task_struct *)sp_el0;
> +	return (struct task_struct *)tpidrro_el0;
>  }
>  
>  #define current get_current()
> diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
> index b5d8277f608a..a45be0a837c8 100644
> --- a/arch/arm64/kernel/entry.S
> +++ b/arch/arm64/kernel/entry.S
> @@ -81,9 +81,9 @@
>  	/* Stash the original SP (minus PT_REGS_SIZE) in tpidr_el0. */
>  	msr	tpidr_el0, x0
>  
> -	/* Recover the original x0 value and stash it in tpidrro_el0 */
> +	/* Recover the original x0 value and stash it in sp_el0 */
>  	sub	x0, sp, x0
> -	msr	tpidrro_el0, x0
> +	msr	sp_el0, x0
>  
>  	/* Switch to the overflow stack */
>  	adr_this_cpu sp, overflow_stack + OVERFLOW_STACK_SIZE, x0
> @@ -99,7 +99,7 @@
>  
>  	/* We were already on the overflow stack. Restore sp/x0 and carry on. */
>  	sub	sp, sp, x0
> -	mrs	x0, tpidrro_el0
> +	mrs	x0, sp_el0
>  	b	el\el\ht\()_\regsize\()_\label
>  	.endif
>  .org .Lventry_start\@ + 128	// Did we overflow the ventry slot?
> @@ -226,7 +226,7 @@ alternative_cb_end
>  	clear_gp_regs
>  	mrs	x21, sp_el0
>  	ldr_this_cpu	tsk, __entry_task, x20
> -	msr	sp_el0, tsk
> +	msr	tpidrro_el0, tsk
>  
>  	/*
>  	 * Ensure MDSCR_EL1.SS is clear, since we can unmask debug exceptions
> @@ -548,7 +548,7 @@ SYM_CODE_START_LOCAL(__bad_stack)
>  	 */
>  
>  	/* Restore the original x0 value */
> -	mrs	x0, tpidrro_el0
> +	mrs	x0, sp_el0
>  
>  	/*
>  	 * Store the original GPRs to the new stack. The orginal SP (minus
> @@ -843,7 +843,7 @@ SYM_FUNC_START(cpu_switch_to)
>  	ldp	x29, x9, [x8], #16
>  	ldr	lr, [x8]
>  	mov	sp, x9
> -	msr	sp_el0, x1
> +	msr	tpidrro_el0, x1
>  	ptrauth_keys_install_kernel x1, x8, x9, x10
>  	scs_save x0
>  	scs_load_current
> @@ -1031,11 +1031,11 @@ SYM_CODE_START(__sdei_asm_handler)
>  
>  	/*
>  	 * We may have interrupted userspace, or a guest, or exit-from or
> -	 * return-to either of these. We can't trust sp_el0, restore it.
> +	 * return-to either of these. We can't trust tpidrro_el0, restore it.
>  	 */
> -	mrs	x28, sp_el0
> +	mrs	x28, tpidrro_el0
>  	ldr_this_cpu	dst=x0, sym=__entry_task, tmp=x1
> -	msr	sp_el0, x0
> +	msr	tpidrro_el0, x0
>  
>  	/* If we interrupted the kernel point to the previous stack/frame. */
>  	and     x0, x3, #0xc
> @@ -1051,7 +1051,7 @@ SYM_CODE_START(__sdei_asm_handler)
>  	mov	x1, x19
>  	bl	__sdei_handler
>  
> -	msr	sp_el0, x28
> +	msr	tpidrro_el0, x28
>  	/* restore regs >x17 that we clobbered */
>  	mov	x4, x19         // keep x4 for __sdei_asm_exit_trampoline
>  	ldp	x28, x29, [x4, #SDEI_EVENT_INTREGS + 16 * 14]
> diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
> index 87a822e5c4ca..c33551ee6572 100644
> --- a/arch/arm64/kernel/head.S
> +++ b/arch/arm64/kernel/head.S
> @@ -193,7 +193,7 @@ SYM_CODE_END(preserve_boot_args)
>  	 * for consistency with user tasks and kthreads.
>  	 */
>  	.macro	init_cpu_task tsk, tmp1, tmp2
> -	msr	sp_el0, \tsk
> +	msr	tpidrro_el0, \tsk
>  
>  	ldr	\tmp1, [\tsk, #TSK_STACK]
>  	add	sp, \tmp1, #THREAD_SIZE
> diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c
> index bfdc12166895..79a19be88ced 100644
> --- a/arch/arm64/kernel/process.c
> +++ b/arch/arm64/kernel/process.c
> @@ -551,8 +551,9 @@ static void ssbs_thread_switch(struct task_struct *next)
>  }
>  
>  /*
> - * We store our current task in sp_el0, which is clobbered by userspace. Keep a
> - * shadow copy so that we can restore this upon entry from userspace.
> + * We store our current task in tpidrro_el0, which is clobbered when returning
> + * to userspace. Keep a shadow copy so that we can restore this upon entry from
> + * userspace.
>   *
>   * This is *only* for exception entry from EL0, and is not valid until we
>   * __switch_to() a user task.
> diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S
> index 12aacc74f764..ad4965d34978 100644
> --- a/arch/arm64/mm/proc.S
> +++ b/arch/arm64/mm/proc.S
> @@ -129,6 +129,10 @@ SYM_FUNC_START(cpu_do_resume)
>  	ldp	x9, x10, [x0, #48]
>  	ldp	x11, x12, [x0, #64]
>  	ldp	x13, x14, [x0, #80]
> +
> +	/* Move 'current' somewhere safe */
> +	mov	x15, x3
> +
>  	/*
>  	 * Restore x18, as it may be used as a platform register, and clear
>  	 * the buffer to minimize the risk of exposure when used for shadow
> @@ -176,7 +180,7 @@ alternative_if ARM64_HAS_GIC_PRIO_MASKING
>  alternative_else_nop_endif
>  #endif
>  
> -	ptrauth_keys_install_kernel_nosync x14, x1, x2, x3
> +	ptrauth_keys_install_kernel_nosync x15, x1, x2, x3
>  	isb
>  	ret
>  SYM_FUNC_END(cpu_do_resume)
> diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
> index c18e005a41db..51d853b92443 100644
> --- a/arch/arm64/net/bpf_jit_comp.c
> +++ b/arch/arm64/net/bpf_jit_comp.c
> @@ -1694,7 +1694,7 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn
>  		if (insn->src_reg == 0 && insn->imm == BPF_FUNC_get_smp_processor_id) {
>  			cpu_offset = offsetof(struct thread_info, cpu);
>  
> -			emit(A64_MRS_SP_EL0(tmp), ctx);
> +			emit(A64_MRS_TPIDRRO_EL0(tmp), ctx);
>  			if (is_lsi_offset(cpu_offset, 2)) {
>  				emit(A64_LDR32I(r0, tmp, cpu_offset), ctx);
>  			} else {
> @@ -1707,7 +1707,7 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn
>  		/* Implement helper call to bpf_get_current_task/_btf() inline */
>  		if (insn->src_reg == 0 && (insn->imm == BPF_FUNC_get_current_task ||
>  					   insn->imm == BPF_FUNC_get_current_task_btf)) {
> -			emit(A64_MRS_SP_EL0(r0), ctx);
> +			emit(A64_MRS_TPIDRRO_EL0(r0), ctx);

Since A64_MRS_SP_EL0 is no longer used after this, can it be removed?

>  			break;
>  		}
>  

-- 
Best regards,
Jinjie