[PATCH v4 1/2] arm64: copy_template: add an offset-addressed 64-byte loop

From: Qi Xi

Date: Mon Nov 16 2026 - 02:31:59 EST


From: Qi Xi <xiqi2@xxxxxxxxxx>

copy_template.S is shared by copy_from_user.S and copy_to_user.S. In
its 64-byte loop every load and store is post-indexed, which on the
user side means two LDTR/STTR accesses followed by an address update.

Add a COPY_TEMPLATE_PAIRS variant of the loop that uses offset
addressing and advances src and dst once per iteration. Without
per-access writeback, the loads and stores of an iteration can overlap
rather than serialise on the base register.

Defining COPY_TEMPLATE_PAIRS before including copy_template.S selects
the variant.

Co-developed-by: Ben Niu <benniu@xxxxxxxx>
Signed-off-by: Ben Niu <benniu@xxxxxxxx>
Signed-off-by: Jinjiang Tu <tujinjiang@xxxxxxxxxx>
Signed-off-by: Qi Xi <xiqi2@xxxxxxxxxx>
---
arch/arm64/lib/copy_template.S | 35 ++++++++++++++++++++++++++++++++++
1 file changed, 35 insertions(+)

diff --git a/arch/arm64/lib/copy_template.S b/arch/arm64/lib/copy_template.S
index 7f2f5a0e2fb9..12ab3bc8afa3 100644
--- a/arch/arm64/lib/copy_template.S
+++ b/arch/arm64/lib/copy_template.S
@@ -161,6 +161,40 @@ alternative_else_nop_endif
*/
.p2align L1_CACHE_SHIFT
.Lcpy_body_large:
+#ifdef COPY_TEMPLATE_PAIRS
+ /* pre-get 64 bytes data. */
+ ldp2 A_l, A_h, src, #0, 9997f
+ ldp2 B_l, B_h, src, #16, 9997f
+ ldp2 C_l, C_h, src, #32, 9997f
+ ldp2 D_l, D_h, src, #48, 9997f
+ add src, src, #64
+1:
+ /*
+ * interlace the load of next 64 bytes data block with store of the last
+ * loaded 64 bytes data.
+ *
+ * The loads can fault on user memory. Since dst is not advanced
+ * per access, each fault is fixed up to account for the 16-byte
+ * blocks already stored.
+ */
+ stp2 A_l, A_h, dst, #0
+ ldp2 A_l, A_h, src, #0, .Lcopy_opt_fixup16
+ stp2 B_l, B_h, dst, #16
+ ldp2 B_l, B_h, src, #16, .Lcopy_opt_fixup32
+ stp2 C_l, C_h, dst, #32
+ ldp2 C_l, C_h, src, #32, .Lcopy_opt_fixup48
+ stp2 D_l, D_h, dst, #48
+ ldp2 D_l, D_h, src, #48, .Lcopy_opt_fixup64
+ add dst, dst, #64
+ add src, src, #64
+ subs count, count, #64
+ b.ge 1b
+ stp2 A_l, A_h, dst, #0
+ stp2 B_l, B_h, dst, #16
+ stp2 C_l, C_h, dst, #32
+ stp2 D_l, D_h, dst, #48
+ add dst, dst, #64
+#else
/* pre-get 64 bytes data. */
ldp1 A_l, A_h, src, #16
ldp1 B_l, B_h, src, #16
@@ -185,6 +219,7 @@ alternative_else_nop_endif
stp1 B_l, B_h, dst, #16
stp1 C_l, C_h, dst, #16
stp1 D_l, D_h, dst, #16
+#endif

tst count, #0x3f
b.ne .Ltail63
--
2.53.0