[PATCH bpf-next v2 14/22] bpf: Move bpf_convert_ctx_accesses() to the patch list
From: Qiliang Yuan
Date: Fri Oct 30 2026 - 08:00:10 EST
bpf_convert_ctx_accesses() patches the program for every context
access that it rewrites and every barrier that it adds, one
bpf_patch_insn_data() at a time.
Each of these patches costs O(prog->len), so a program with many
context accesses pays it once for every one of them.
Queue the prologue, the store of the context pointer for the epilogue
and the rewrites, and commit them at the end of the pass. Queue a
barrier in front of an instruction before rewriting the instruction,
whose rewrite then replaces the last instruction of the barrier. As
ops->gen_prologue() ends the prologue with the first instruction of the
program, insert the prologue without it with bpf_patch_list_insert().
As the epilogue is now found by its index in the unpatched program,
which can be 0, mark that there is none yet with -1.
Signed-off-by: Qiliang Yuan <odys.yuan@xxxxxxxxx>
---
kernel/bpf/fixups.c | 132 +++++++++++++++++++---------------------------------
1 file changed, 49 insertions(+), 83 deletions(-)
diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index 6d99de1be2581..08c89f2baffb6 100644
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -1090,16 +1090,15 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env)
{
struct bpf_subprog_info *subprogs = env->subprog_info;
const struct bpf_verifier_ops *ops = env->ops;
- int i, cnt, size, ctx_field_size, ret, delta = 0, epilogue_cnt = 0;
+ int i, cnt, size, ctx_field_size, ret, epilogue_cnt = 0;
const int insn_cnt = env->prog->len;
struct bpf_insn *epilogue_buf = env->epilogue_buf;
struct bpf_insn *insn_buf = env->insn_buf;
struct bpf_insn *insn;
u32 target_size, size_default, off;
- struct bpf_prog *new_prog;
enum bpf_access_type type;
bool is_narrower_load;
- int epilogue_idx = 0;
+ int epilogue_idx = -1;
if (ops->gen_epilogue) {
epilogue_cnt = ops->gen_epilogue(epilogue_buf, env->prog,
@@ -1109,17 +1108,12 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env)
return -EFAULT;
} else if (epilogue_cnt) {
/* Save the ARG_PTR_TO_CTX for the epilogue to use */
- cnt = 0;
subprogs[0].stack_depth += 8;
- insn_buf[cnt++] = BPF_STX_MEM(BPF_DW, BPF_REG_FP, BPF_REG_1,
- -subprogs[0].stack_depth);
- insn_buf[cnt++] = env->prog->insnsi[0];
- new_prog = __bpf_patch_insn_data(env, 0, insn_buf, cnt,
- BPF_PATCH_MOVE_TARGET);
- if (!new_prog)
- return -ENOMEM;
- env->prog = new_prog;
- delta += cnt - 1;
+ insn_buf[0] = BPF_STX_MEM(BPF_DW, BPF_REG_FP, BPF_REG_1,
+ -subprogs[0].stack_depth);
+ ret = bpf_patch_list_insert(env, 0, insn_buf, 1);
+ if (ret)
+ return ret;
ret = add_kfunc_in_insns(env, epilogue_buf, epilogue_cnt - 1);
if (ret < 0)
@@ -1150,13 +1144,14 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env)
env->prog->aux->attach_btf_id))
return -EINVAL;
env->prog->aux->has_prologue = true;
- new_prog = __bpf_patch_insn_data(env, 0, insn_buf, cnt,
- BPF_PATCH_MOVE_TARGET);
- if (!new_prog)
- return -ENOMEM;
-
- env->prog = new_prog;
- delta += cnt - 1;
+ /* the prologue ends with the first insn of the program */
+ if (verifier_bug_if(memcmp(&insn_buf[cnt - 1], env->prog->insnsi,
+ sizeof(*insn_buf)), env,
+ "prologue doesn't end with the first insn"))
+ return -EFAULT;
+ ret = bpf_patch_list_insert(env, 0, insn_buf, cnt - 1);
+ if (ret)
+ return ret;
ret = add_kfunc_in_insns(env, insn_buf, cnt - 1);
if (ret < 0)
@@ -1164,21 +1159,18 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env)
}
}
- if (delta)
- WARN_ON(adjust_jmp_off(env->prog, 0, delta));
-
if (bpf_prog_is_offloaded(env->prog->aux))
- return 0;
-
- insn = env->prog->insnsi + delta;
+ return bpf_patch_list_commit(env);
- for (i = 0; i < insn_cnt; i++, insn++) {
+ for (i = 0; i < insn_cnt; i++) {
bpf_convert_ctx_access_t convert_ctx_access;
enum bpf_reg_type ptr_type;
u8 mode;
- if (env->insn_aux_data[i + delta].nospec) {
- WARN_ON_ONCE(env->insn_aux_data[i + delta].alu_state);
+ insn = &env->prog->insnsi[i];
+
+ if (env->insn_aux_data[i].nospec) {
+ WARN_ON_ONCE(env->insn_aux_data[i].alu_state);
struct bpf_insn *patch = insn_buf;
*patch++ = BPF_ST_NOSPEC();
@@ -1186,17 +1178,13 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env)
ret = bpf_adj_moved_insn(patch++, 1);
if (ret) {
verbose(env, "insn %d cannot be patched due to 16-bit range\n",
- env->insn_aux_data[i + delta].orig_idx);
+ env->insn_aux_data[i].orig_idx);
return ret;
}
cnt = patch - insn_buf;
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
/* This can not be easily merged with the
* nospec_result-case, because an insn may require a
* nospec before and after itself. Therefore also do not
@@ -1226,42 +1214,34 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env)
insn->code == (BPF_STX | BPF_ATOMIC | BPF_H) ||
insn->code == (BPF_STX | BPF_ATOMIC | BPF_W) ||
insn->code == (BPF_STX | BPF_ATOMIC | BPF_DW)) &&
- env->insn_aux_data[i + delta].ptr_type == PTR_TO_ARENA) {
+ env->insn_aux_data[i].ptr_type == PTR_TO_ARENA) {
insn->code = BPF_STX | BPF_PROBE_ATOMIC | BPF_SIZE(insn->code);
env->prog->aux->num_exentries++;
continue;
} else if (insn->code == (BPF_STX | BPF_ATOMIC | BPF_DW) &&
insn->imm == BPF_LOAD_ACQ &&
- env->insn_aux_data[i + delta].arena_member) {
+ env->insn_aux_data[i].arena_member) {
/* an acquire load of an __arena member is rewritten like a plain load */
- cnt = arena_member_fixup(env, i + delta, insn, insn_buf);
+ cnt = arena_member_fixup(env, i, insn, insn_buf);
if (cnt < 0)
return cnt;
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
continue;
} else if (insn->code == (BPF_JMP | BPF_EXIT) &&
epilogue_cnt &&
- i + delta < subprogs[1].start) {
+ i < subprogs[1].start) {
/* Generate epilogue for the main prog */
- if (epilogue_idx) {
+ if (epilogue_idx >= 0) {
/* jump back to the earlier generated epilogue */
- insn_buf[0] = BPF_JMP32_A(epilogue_idx - i - delta - 1);
+ insn_buf[0] = BPF_JMP32_A(epilogue_idx - i - 1);
cnt = 1;
} else {
memcpy(insn_buf, epilogue_buf,
epilogue_cnt * sizeof(*epilogue_buf));
cnt = epilogue_cnt;
- /* epilogue_idx cannot be 0. It must have at
- * least one ctx ptr saving insn before the
- * epilogue.
- */
- epilogue_idx = i + delta;
+ epilogue_idx = i;
}
goto patch_insn_buf;
} else {
@@ -1269,7 +1249,7 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env)
}
if (type == BPF_WRITE &&
- env->insn_aux_data[i + delta].nospec_result) {
+ env->insn_aux_data[i].nospec_result) {
/* nospec_result is only used to mitigate Spectre v4 and
* to limit verification-time for Spectre v1.
*/
@@ -1278,17 +1258,13 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env)
*patch++ = *insn;
*patch++ = BPF_ST_NOSPEC();
cnt = patch - insn_buf;
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
continue;
}
- ptr_type = env->insn_aux_data[i + delta].ptr_type;
+ ptr_type = env->insn_aux_data[i].ptr_type;
switch ((int)ptr_type) {
case PTR_TO_CTX:
if (!ops->convert_ctx_access)
@@ -1341,22 +1317,18 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env)
return -EFAULT;
}
- cnt = arena_member_fixup(env, i + delta, insn, insn_buf);
+ cnt = arena_member_fixup(env, i, insn, insn_buf);
if (cnt < 0)
return cnt;
if (!cnt)
continue;
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
- env->prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
continue;
}
- ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size;
+ ctx_field_size = env->insn_aux_data[i].ctx_field_size;
size = BPF_LDST_BYTES(insn);
mode = BPF_MODE(insn->code);
@@ -1424,18 +1396,12 @@ int bpf_convert_ctx_accesses(struct bpf_verifier_env *env)
size * 8, 0);
patch_insn_buf:
- new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
- if (!new_prog)
- return -ENOMEM;
-
- delta += cnt - 1;
-
- /* keep walking new program and skip insns we just inserted */
- env->prog = new_prog;
- insn = new_prog->insnsi + i + delta;
+ insn = bpf_patch_list_add(env, i, insn_buf, cnt);
+ if (IS_ERR(insn))
+ return PTR_ERR(insn);
}
- return 0;
+ return bpf_patch_list_commit(env);
}
static u32 *bpf_dup_subprog_starts(struct bpf_verifier_env *env)
--
2.43.0