[PATCH v4 2/2] arm64: copy_from_user: use privileged LDP/STP for large copies
From: Qi Xi
Date: Mon Nov 16 2026 - 02:31:59 EST
From: Qi Xi <xiqi2@xxxxxxxxxx>
For copies of 4096 bytes or more, __arch_copy_from_user_opt() toggles
PSTATE.PAN, allowing it to use privileged LDP instead of LDTR,
together with the offset-addressed loop from copy_template.S. Smaller
copies keep using the original code.
copy_to_user is left unchanged. Using privileged stores there would
need the same PAN toggle, but a PAN update in the middle of the
user-side store stream breaks hardware write streaming, which costs
more than the paired stores save.
Signed-off-by: Qi Xi <xiqi2@xxxxxxxxxx>
---
arch/arm64/lib/Makefile | 2 +-
arch/arm64/lib/copy_from_user.S | 8 ++
arch/arm64/lib/copy_from_user_opt.S | 114 ++++++++++++++++++++++++++++
3 files changed, 123 insertions(+), 1 deletion(-)
create mode 100644 arch/arm64/lib/copy_from_user_opt.S
diff --git a/arch/arm64/lib/Makefile b/arch/arm64/lib/Makefile
index 38ffd359123c..b7c1c579aec0 100644
--- a/arch/arm64/lib/Makefile
+++ b/arch/arm64/lib/Makefile
@@ -4,7 +4,7 @@
KCSAN_SANITIZE_delay.o := n
lib-y := clear_user.o delay.o copy_from_user.o \
- copy_to_user.o copy_page.o \
+ copy_from_user_opt.o copy_to_user.o copy_page.o \
csum.o insn.o memchr.o memcpy.o \
memset.o memcmp.o strcmp.o strncmp.o strlen.o \
strnlen.o strchr.o strrchr.o tishift.o
diff --git a/arch/arm64/lib/copy_from_user.S b/arch/arm64/lib/copy_from_user.S
index 400057d607ec..ec23066c2a37 100644
--- a/arch/arm64/lib/copy_from_user.S
+++ b/arch/arm64/lib/copy_from_user.S
@@ -64,6 +64,12 @@ srcin .req x15
SYM_FUNC_START(__arch_copy_from_user)
add end, x0, x2
mov srcin, x1
+ /*
+ * Use the privileged LDP/STP path for large copies. Below 4096
+ * bytes the PAN toggle costs more than it saves.
+ */
+ cmp x2, #4096
+ b.hs 9002f
#include "copy_template.S"
mov x0, #0 // Nothing to copy
ret
@@ -79,5 +85,7 @@ USER(9998f, ldtrb tmp1w, [srcin])
strb tmp1w, [dst], #1
9998: sub x0, end, dst // bytes not copied
ret
+
+9002: b __arch_copy_from_user_opt
SYM_FUNC_END(__arch_copy_from_user)
EXPORT_SYMBOL(__arch_copy_from_user)
diff --git a/arch/arm64/lib/copy_from_user_opt.S b/arch/arm64/lib/copy_from_user_opt.S
new file mode 100644
index 000000000000..2c0e3cb4798a
--- /dev/null
+++ b/arch/arm64/lib/copy_from_user_opt.S
@@ -0,0 +1,114 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2026 Huawei Ltd.
+ */
+
+#include <linux/linkage.h>
+
+#include <asm/asm-uaccess.h>
+#include <asm/assembler.h>
+#include <asm/cache.h>
+
+#define COPY_TEMPLATE_PAIRS
+
+ .macro ldrb1 reg, ptr, val
+ user_ldst 9998f, ldtrb, \reg, \ptr, \val
+ .endm
+
+ .macro strb1 reg, ptr, val
+ strb \reg, [\ptr], \val
+ .endm
+
+ .macro ldrh1 reg, ptr, val
+ user_ldst 9997f, ldtrh, \reg, \ptr, \val
+ .endm
+
+ .macro strh1 reg, ptr, val
+ strh \reg, [\ptr], \val
+ .endm
+
+ .macro ldr1 reg, ptr, val
+ user_ldst 9997f, ldtr, \reg, \ptr, \val
+ .endm
+
+ .macro str1 reg, ptr, val
+ str \reg, [\ptr], \val
+ .endm
+
+ .macro ldp1 reg1, reg2, ptr, val
+ user_ldp 9997f, \reg1, \reg2, \ptr, \val
+ .endm
+
+ .macro stp1 reg1, reg2, ptr, val
+ stp \reg1, \reg2, [\ptr], \val
+ .endm
+
+ .macro ldp2 reg1, reg2, ptr, val, fixup
+8888: ldp \reg1, \reg2, [\ptr, \val]
+ _asm_extable_uaccess 8888b, \fixup
+ .endm
+
+ .macro stp2 reg1, reg2, ptr, val
+ stp \reg1, \reg2, [\ptr, \val]
+ .endm
+
+ .macro cpy1 dst, src, count
+ .arch_extension mops
+ USER_CPY(9997f, 0, cpyfprt [\dst]!, [\src]!, \count!)
+ USER_CPY(9996f, 0, cpyfmrt [\dst]!, [\src]!, \count!)
+ USER_CPY(9996f, 0, cpyfert [\dst]!, [\src]!, \count!)
+ .endm
+
+end .req x5
+srcin .req x15
+SYM_FUNC_START(__arch_copy_from_user_opt)
+ add end, x0, x2
+ mov srcin, x1
+
+#ifdef CONFIG_KASAN_HW_TAGS
+ /*
+ * KASAN enables tag checking for EL1 accesses (SCTLR_EL1.TCF)
+ * while the user's TCF0 mode may have checking disabled.
+ * Suppress it for the copy, as uaccess_enable_privileged() does.
+ */
+ ALTERNATIVE("nop", __stringify(SET_PSTATE_TCO(1)), ARM64_MTE)
+#endif
+ /*
+ * PSTATE.PAN blocks EL1 access to user memory, so clear it for
+ * the privileged LDP/STP below and restore it on exit.
+ */
+ ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_HAS_PAN)
+
+#include "copy_template.S"
+
+ mov x0, #0 // Nothing to copy
+
+.L__arch_copy_from_user_opt_exit:
+#ifdef CONFIG_KASAN_HW_TAGS
+ ALTERNATIVE("nop", __stringify(SET_PSTATE_TCO(0)), ARM64_MTE)
+#endif
+ ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_HAS_PAN)
+ ret
+
+ // Exception fixups
+.Lcopy_opt_fixup64:
+ add dst, dst, #16
+.Lcopy_opt_fixup48:
+ add dst, dst, #16
+.Lcopy_opt_fixup32:
+ add dst, dst, #16
+.Lcopy_opt_fixup16:
+ add dst, dst, #16
+ b 9998f
+
+9996: b.cs 9997f
+ // Registers are in Option A format
+ add dst, dst, count
+9997: cmp dst, dstin
+ b.ne 9998f
+ // Before being absolutely sure we couldn't copy anything, try harder
+USER(9998f, ldtrb tmp1w, [srcin])
+ strb tmp1w, [dst], #1
+9998: sub x0, end, dst // bytes not copied
+ b .L__arch_copy_from_user_opt_exit
+SYM_FUNC_END(__arch_copy_from_user_opt)
--
2.53.0