[PATCH RFC v4 07/13] bpf, x86: Take a Tasks Trace reader in the trampoline around its call-outs
From: Josef Bacik
Date: Fri Sep 18 2026 - 11:26:14 EST
On HAVE_RCU_TRAMPOLINE_READERS kernels Tasks RCU keeps a BPF trampoline
image allocated only while a task using it is a Tasks Trace RCU reader
or is executing text that rcu_tasks_trampoline_text() recognises. The
image itself is such text, but the C glue and the programs it calls are
not, and only sleepable programs take rcu_read_lock_trace() today.
Have the x86-64 JIT open-code rcu_read_lock_trace() and
rcu_read_unlock_trace() in the trampoline, as ftrace_64.S does for
ftrace_caller: one reader from just after the frame is set up to just
before the original function is called, covering __bpf_tramp_enter()
and the fentry and fmod_ret programs, and a second one from just after
the original function returns to just before the final register
restore, covering the fexit programs and __bpf_tramp_exit(). The
original function itself runs outside both, since it may run for a long
time and the image is pinned by im->pcref across it. Trampolines that
do not call the original function get a single reader around all their
programs. The second reader is entered before ip_after_call, so the
ip_after_call -> ip_epilogue jump that bpf_tramp_image_put() patches in
is inside it, and the fmod_ret early-exit branch lands after that point
still holding the first reader, so exactly one is held on every path.
The sequence uses r10 and r11, which are scratch at each emission point,
and references current_task and rcu_tasks_trace_srcu_struct by absolute
sign-extended address, the form the JIT already relies on for
this_cpu_off. Sleepable programs' own rcu_read_lock_trace() simply
nests. Nothing is emitted on other configurations.
Suggested-by: Alexei Starovoitov <ast@xxxxxxxxxx>
Assisted-by: LLM
Signed-off-by: Josef Bacik <josef@xxxxxxxxxxxxxx>
---
arch/x86/net/bpf_jit_comp.c | 113 ++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 113 insertions(+)
diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 2853e87797a7..c991f7ceacdf 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -14,6 +14,7 @@
#include <linux/memory.h>
#include <linux/sort.h>
#include <linux/execmem.h>
+#include <linux/rcupdate_trace.h>
#include <asm/extable.h>
#include <asm/ftrace.h>
#include <asm/set_memory.h>
@@ -722,6 +723,97 @@ static void emit_indirect_jump(u8 **pprog, int bpf_reg, u8 *ip)
*pprog = prog;
}
+/*
+ * Open-coded rcu_read_lock_trace() / rcu_read_unlock_trace() for the
+ * trampoline, see CONFIG_HAVE_RCU_TRAMPOLINE_READERS and the equivalent
+ * macros in arch/x86/kernel/ftrace_64.S. The image is not relocated, so
+ * current_task and rcu_tasks_trace_srcu_struct are referenced by absolute
+ * (sign-extended 32-bit) address, the form the JIT already relies on for
+ * this_cpu_off. Uses r10 and r11, which are scratch at every emission
+ * point, and clobbers flags.
+ *
+ * lock: unlock:
+ * mov r11, gs:[current_task] mov r11, gs:[current_task]
+ * mov r10d, [r11+nesting] mov r10d, [r11+nesting]
+ * inc dword ptr [r11+nesting] sub r10d, 1
+ * test r10d, r10d jnz 2f
+ * jnz 1f mov r10, [r11+scp]
+ * mov r10, [&srcu.srcu_ctrp] mov dword ptr [r11+nesting], 0
+ * inc qword ptr gs:[r10+locks] (smp_mb)
+ * mov [r11+scp], r10 inc qword ptr gs:[r10+unlocks]
+ * (smp_mb) jmp 3f
+ * 1: 2: mov [r11+nesting], r10d
+ * 3:
+ */
+static void emit_trace_rcu_reader(u8 **pprog, bool lock)
+{
+#ifdef CONFIG_TASKS_RCU_TRAMPOLINE_READERS
+ const u32 nesting = offsetof(struct task_struct, trc_reader_nesting);
+ const u32 scp = offsetof(struct task_struct, trc_reader_scp);
+ const bool mb = !IS_ENABLED(CONFIG_TASKS_TRACE_RCU_NO_MB);
+ u8 *prog = *pprog;
+
+ BUILD_BUG_ON(IS_ENABLED(CONFIG_NEED_SRCU_NMI_SAFE));
+ BUILD_BUG_ON(offsetof(struct srcu_ctr, srcu_locks) != 0);
+ BUILD_BUG_ON(offsetof(struct srcu_ctr, srcu_unlocks) != 8);
+
+ /* mov r11, gs:[abs32 current_task] */
+ EMIT2(0x65, 0x4C);
+ EMIT3(0x8B, 0x1C, 0x25);
+ EMIT((u32)(unsigned long)¤t_task, 4);
+ /* mov r10d, dword ptr [r11 + nesting] */
+ EMIT3(0x45, 0x8B, 0x93);
+ EMIT(nesting, 4);
+
+ if (lock) {
+ /* inc dword ptr [r11 + nesting] */
+ EMIT3(0x41, 0xFF, 0x83);
+ EMIT(nesting, 4);
+ /* test r10d, r10d */
+ EMIT3(0x45, 0x85, 0xD2);
+ /* jnz 1f */
+ EMIT2(X86_JNE, 8 + 4 + 7 + (mb ? 6 : 0));
+ /* mov r10, qword ptr [abs32 &rcu_tasks_trace_srcu_struct.srcu_ctrp] */
+ EMIT4(0x4C, 0x8B, 0x14, 0x25);
+ EMIT((u32)(unsigned long)&rcu_tasks_trace_srcu_struct.srcu_ctrp, 4);
+ /* inc qword ptr gs:[r10] */
+ EMIT4(0x65, 0x49, 0xFF, 0x02);
+ /* mov qword ptr [r11 + scp], r10 */
+ EMIT3(0x4D, 0x89, 0x93);
+ EMIT(scp, 4);
+ /* smp_mb(): lock add dword ptr [rsp - 4], 0 */
+ if (mb)
+ EMIT2_off32(0xF0, 0x83, 0x00FC2444);
+ /* 1: */
+ } else {
+ /* sub r10d, 1 */
+ EMIT4(0x41, 0x83, 0xEA, 0x01);
+ /* jnz 2f */
+ EMIT2(X86_JNE, 7 + 11 + (mb ? 6 : 0) + 5 + 2);
+ /* mov r10, qword ptr [r11 + scp] */
+ EMIT3(0x4D, 0x8B, 0x93);
+ EMIT(scp, 4);
+ /* mov dword ptr [r11 + nesting], 0 */
+ EMIT3(0x41, 0xC7, 0x83);
+ EMIT(nesting, 4);
+ EMIT(0, 4);
+ if (mb)
+ EMIT2_off32(0xF0, 0x83, 0x00FC2444);
+ /* inc qword ptr gs:[r10 + 8] */
+ EMIT4(0x65, 0x49, 0xFF, 0x42);
+ EMIT1(0x08);
+ /* jmp 3f */
+ EMIT2(0xEB, 7);
+ /* 2: mov dword ptr [r11 + nesting], r10d */
+ EMIT3(0x45, 0x89, 0x93);
+ EMIT(nesting, 4);
+ /* 3: */
+ }
+
+ *pprog = prog;
+#endif
+}
+
static void emit_return(u8 **pprog, u8 *ip)
{
u8 *prog = *pprog;
@@ -3610,6 +3702,16 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
/* mov QWORD PTR [rbp - rbx_off], rbx */
emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_6, -rbx_off);
+ /*
+ * Tasks RCU keeps this image alive only while we are a Tasks Trace
+ * reader; the instructions before this point (and after the final
+ * unlock) are covered by the irq-exit IP check. One reader spans
+ * __bpf_tramp_enter() and the fentry/fmod_ret progs, a second one
+ * the fexit progs and __bpf_tramp_exit(); the original function runs
+ * outside both, with the image pinned by im->pcref instead.
+ */
+ emit_trace_rcu_reader(&prog, true);
+
func_meta = nr_regs;
/* Store number of argument registers of the traced function */
emit_store_stack_imm64(&prog, BPF_REG_0, -func_meta_off, func_meta);
@@ -3660,6 +3762,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
}
if (flags & BPF_TRAMP_F_CALL_ORIG) {
+ emit_trace_rcu_reader(&prog, false);
restore_regs(m, &prog, regs_off);
save_args(m, &prog, arg_stack_off, true, flags, 0);
@@ -3682,6 +3785,13 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
}
/* remember return value in a stack for bpf prog to access */
emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8);
+ /*
+ * Second reader. Taken before ip_after_call so that the
+ * ip_after_call -> ip_epilogue jump patched in at teardown is
+ * inside it too; the fmod_ret early exit jumps past this still
+ * holding the first reader, so either way exactly one is held.
+ */
+ emit_trace_rcu_reader(&prog, true);
im->ip_after_call = image + (prog - (u8 *)rw_image);
emit_nops(&prog, X86_PATCH_SIZE);
}
@@ -3737,6 +3847,9 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
LOAD_TRAMP_TAIL_CALL_CNT_PTR(stack_size);
}
+ /* Remaining instructions are covered by the irq-exit IP check. */
+ emit_trace_rcu_reader(&prog, false);
+
/* restore return value of orig_call or fentry prog back into RAX */
if (save_ret)
emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, -8);
--
2.55.0