[PATCH bpf-next v5 2/5] bpf: Pass bpf_verifier_env to jit
From: Xu Kuohai
Date: Mon Mar 02 2026 - 05:07:57 EST
From: Xu Kuohai <xukuohai@xxxxxxxxxx>
Pass bpf_verifier_env to bpf_int_jit_compile() and bpf_jit_blind_constants().
The follow-up patch will use env->insn_aux_data in the JIT stage to detect
indirect jump targets.
Signed-off-by: Xu Kuohai <xukuohai@xxxxxxxxxx>
---
arch/arc/net/bpf_jit_core.c | 19 ++++++++++---------
arch/arm/net/bpf_jit_32.c | 4 ++--
arch/arm64/net/bpf_jit_comp.c | 4 ++--
arch/loongarch/net/bpf_jit.c | 4 ++--
arch/mips/net/bpf_jit_comp.c | 4 ++--
arch/parisc/net/bpf_jit_core.c | 4 ++--
arch/powerpc/net/bpf_jit_comp.c | 4 ++--
arch/riscv/net/bpf_jit_core.c | 4 ++--
arch/s390/net/bpf_jit_comp.c | 4 ++--
arch/sparc/net/bpf_jit_comp_64.c | 4 ++--
arch/x86/net/bpf_jit_comp.c | 4 ++--
arch/x86/net/bpf_jit_comp32.c | 4 ++--
include/linux/filter.h | 6 +++---
kernel/bpf/core.c | 10 +++++-----
kernel/bpf/verifier.c | 6 +++---
15 files changed, 43 insertions(+), 42 deletions(-)
diff --git a/arch/arc/net/bpf_jit_core.c b/arch/arc/net/bpf_jit_core.c
index 1421eeced0f5..076aaf52cb80 100644
--- a/arch/arc/net/bpf_jit_core.c
+++ b/arch/arc/net/bpf_jit_core.c
@@ -157,14 +157,15 @@ static void jit_dump(const struct jit_context *ctx)
}
/* Initialise the context so there's no garbage. */
-static int jit_ctx_init(struct jit_context *ctx, struct bpf_prog *prog)
+static int jit_ctx_init(struct jit_context *ctx, struct bpf_verifier_env *env,
+ struct bpf_prog *prog)
{
memset(ctx, 0, sizeof(*ctx));
ctx->orig_prog = prog;
/* If constant blinding was requested but failed, scram. */
- ctx->prog = bpf_jit_blind_constants(prog);
+ ctx->prog = bpf_jit_blind_constants(env, prog);
if (IS_ERR(ctx->prog))
return PTR_ERR(ctx->prog);
ctx->blinded = (ctx->prog != ctx->orig_prog);
@@ -1335,7 +1336,7 @@ static int jit_patch_relocations(struct jit_context *ctx)
* to get the necessary data for the real compilation phase,
* jit_compile().
*/
-static struct bpf_prog *do_normal_pass(struct bpf_prog *prog)
+static struct bpf_prog *do_normal_pass(struct bpf_verifier_env *env, struct bpf_prog *prog)
{
struct jit_context ctx;
@@ -1343,7 +1344,7 @@ static struct bpf_prog *do_normal_pass(struct bpf_prog *prog)
if (!prog->jit_requested)
return prog;
- if (jit_ctx_init(&ctx, prog)) {
+ if (jit_ctx_init(&ctx, env, prog)) {
jit_ctx_cleanup(&ctx);
return prog;
}
@@ -1374,7 +1375,7 @@ static struct bpf_prog *do_normal_pass(struct bpf_prog *prog)
* again to get the newly translated addresses in order to resolve
* the "call"s.
*/
-static struct bpf_prog *do_extra_pass(struct bpf_prog *prog)
+static struct bpf_prog *do_extra_pass(struct bpf_verifier_env *env, struct bpf_prog *prog)
{
struct jit_context ctx;
@@ -1382,7 +1383,7 @@ static struct bpf_prog *do_extra_pass(struct bpf_prog *prog)
if (check_jit_context(prog))
return prog;
- if (jit_ctx_init(&ctx, prog)) {
+ if (jit_ctx_init(&ctx, env, prog)) {
jit_ctx_cleanup(&ctx);
return prog;
}
@@ -1411,15 +1412,15 @@ static struct bpf_prog *do_extra_pass(struct bpf_prog *prog)
* (re)locations involved that their addresses are not known
* during the first run.
*/
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
{
vm_dump(prog);
/* Was this program already translated? */
if (!prog->jited)
- return do_normal_pass(prog);
+ return do_normal_pass(env, prog);
else
- return do_extra_pass(prog);
+ return do_extra_pass(env, prog);
return prog;
}
diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c
index deeb8f292454..9c07cbf1dbfc 100644
--- a/arch/arm/net/bpf_jit_32.c
+++ b/arch/arm/net/bpf_jit_32.c
@@ -2142,7 +2142,7 @@ bool bpf_jit_needs_zext(void)
return true;
}
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
{
struct bpf_prog *tmp, *orig_prog = prog;
struct bpf_binary_header *header;
@@ -2162,7 +2162,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
* then we must fall back to the interpreter. Otherwise, we save
* the new JITed code.
*/
- tmp = bpf_jit_blind_constants(prog);
+ tmp = bpf_jit_blind_constants(env, prog);
if (IS_ERR(tmp))
return orig_prog;
diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
index adf84962d579..823246c7ff5d 100644
--- a/arch/arm64/net/bpf_jit_comp.c
+++ b/arch/arm64/net/bpf_jit_comp.c
@@ -2006,7 +2006,7 @@ struct arm64_jit_data {
struct jit_ctx ctx;
};
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
{
int image_size, prog_size, extable_size, extable_align, extable_offset;
struct bpf_prog *tmp, *orig_prog = prog;
@@ -2027,7 +2027,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
if (!prog->jit_requested)
return orig_prog;
- tmp = bpf_jit_blind_constants(prog);
+ tmp = bpf_jit_blind_constants(env, prog);
/* If blinding was requested and we failed during blinding,
* we must fall back to the interpreter.
*/
diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c
index 3bd89f55960d..b578b176ef01 100644
--- a/arch/loongarch/net/bpf_jit.c
+++ b/arch/loongarch/net/bpf_jit.c
@@ -1909,7 +1909,7 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
return ret < 0 ? ret : ret * LOONGARCH_INSN_SIZE;
}
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
{
bool tmp_blinded = false, extra_pass = false;
u8 *image_ptr, *ro_image_ptr;
@@ -1927,7 +1927,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
if (!prog->jit_requested)
return orig_prog;
- tmp = bpf_jit_blind_constants(prog);
+ tmp = bpf_jit_blind_constants(env, prog);
/*
* If blinding was requested and we failed during blinding,
* we must fall back to the interpreter. Otherwise, we save
diff --git a/arch/mips/net/bpf_jit_comp.c b/arch/mips/net/bpf_jit_comp.c
index e355dfca4400..faf0ba098a86 100644
--- a/arch/mips/net/bpf_jit_comp.c
+++ b/arch/mips/net/bpf_jit_comp.c
@@ -909,7 +909,7 @@ bool bpf_jit_needs_zext(void)
return true;
}
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
{
struct bpf_prog *tmp, *orig_prog = prog;
struct bpf_binary_header *header = NULL;
@@ -931,7 +931,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
* then we must fall back to the interpreter. Otherwise, we save
* the new JITed code.
*/
- tmp = bpf_jit_blind_constants(prog);
+ tmp = bpf_jit_blind_constants(env, prog);
if (IS_ERR(tmp))
return orig_prog;
if (tmp != prog) {
diff --git a/arch/parisc/net/bpf_jit_core.c b/arch/parisc/net/bpf_jit_core.c
index a5eb6b51e27a..e85b6e336b19 100644
--- a/arch/parisc/net/bpf_jit_core.c
+++ b/arch/parisc/net/bpf_jit_core.c
@@ -41,7 +41,7 @@ bool bpf_jit_needs_zext(void)
return true;
}
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
{
unsigned int prog_size = 0, extable_size = 0;
bool tmp_blinded = false, extra_pass = false;
@@ -53,7 +53,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
if (!prog->jit_requested)
return orig_prog;
- tmp = bpf_jit_blind_constants(prog);
+ tmp = bpf_jit_blind_constants(env, prog);
if (IS_ERR(tmp))
return orig_prog;
if (tmp != prog) {
diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c
index 52162e4a7f84..fb77e8beb161 100644
--- a/arch/powerpc/net/bpf_jit_comp.c
+++ b/arch/powerpc/net/bpf_jit_comp.c
@@ -129,7 +129,7 @@ bool bpf_jit_needs_zext(void)
return true;
}
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *fp)
{
u32 proglen;
u32 alloclen;
@@ -154,7 +154,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
if (!fp->jit_requested)
return org_fp;
- tmp_fp = bpf_jit_blind_constants(org_fp);
+ tmp_fp = bpf_jit_blind_constants(env, org_fp);
if (IS_ERR(tmp_fp))
return org_fp;
diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c
index b3581e926436..ce157319459f 100644
--- a/arch/riscv/net/bpf_jit_core.c
+++ b/arch/riscv/net/bpf_jit_core.c
@@ -41,7 +41,7 @@ bool bpf_jit_needs_zext(void)
return true;
}
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
{
unsigned int prog_size = 0, extable_size = 0;
bool tmp_blinded = false, extra_pass = false;
@@ -53,7 +53,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
if (!prog->jit_requested)
return orig_prog;
- tmp = bpf_jit_blind_constants(prog);
+ tmp = bpf_jit_blind_constants(env, prog);
if (IS_ERR(tmp))
return orig_prog;
if (tmp != prog) {
diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c
index 211226748662..84aabfc8a9d6 100644
--- a/arch/s390/net/bpf_jit_comp.c
+++ b/arch/s390/net/bpf_jit_comp.c
@@ -2303,7 +2303,7 @@ static struct bpf_binary_header *bpf_jit_alloc(struct bpf_jit *jit,
/*
* Compile eBPF program "fp"
*/
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *fp)
{
struct bpf_prog *tmp, *orig_fp = fp;
struct bpf_binary_header *header;
@@ -2316,7 +2316,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
if (!fp->jit_requested)
return orig_fp;
- tmp = bpf_jit_blind_constants(fp);
+ tmp = bpf_jit_blind_constants(env, fp);
/*
* If blinding was requested and we failed during blinding,
* we must fall back to the interpreter.
diff --git a/arch/sparc/net/bpf_jit_comp_64.c b/arch/sparc/net/bpf_jit_comp_64.c
index b23d1c645ae5..55da61ca2967 100644
--- a/arch/sparc/net/bpf_jit_comp_64.c
+++ b/arch/sparc/net/bpf_jit_comp_64.c
@@ -1477,7 +1477,7 @@ struct sparc64_jit_data {
struct jit_ctx ctx;
};
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
{
struct bpf_prog *tmp, *orig_prog = prog;
struct sparc64_jit_data *jit_data;
@@ -1492,7 +1492,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
if (!prog->jit_requested)
return orig_prog;
- tmp = bpf_jit_blind_constants(prog);
+ tmp = bpf_jit_blind_constants(env, prog);
/* If blinding was requested and we failed during blinding,
* we must fall back to the interpreter.
*/
diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 8f10080e6fe3..43beacaed56d 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -3722,7 +3722,7 @@ struct x64_jit_data {
#define MAX_PASSES 20
#define PADDING_PASSES (MAX_PASSES - 5)
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
{
struct bpf_binary_header *rw_header = NULL;
struct bpf_binary_header *header = NULL;
@@ -3744,7 +3744,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
if (!prog->jit_requested)
return orig_prog;
- tmp = bpf_jit_blind_constants(prog);
+ tmp = bpf_jit_blind_constants(env, prog);
/*
* If blinding was requested and we failed during blinding,
* we must fall back to the interpreter.
diff --git a/arch/x86/net/bpf_jit_comp32.c b/arch/x86/net/bpf_jit_comp32.c
index dda423025c3d..957f7aa951ba 100644
--- a/arch/x86/net/bpf_jit_comp32.c
+++ b/arch/x86/net/bpf_jit_comp32.c
@@ -2518,7 +2518,7 @@ bool bpf_jit_needs_zext(void)
return true;
}
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
{
struct bpf_binary_header *header = NULL;
struct bpf_prog *tmp, *orig_prog = prog;
@@ -2533,7 +2533,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
if (!prog->jit_requested)
return orig_prog;
- tmp = bpf_jit_blind_constants(prog);
+ tmp = bpf_jit_blind_constants(env, prog);
/*
* If blinding was requested and we failed during blinding,
* we must fall back to the interpreter.
diff --git a/include/linux/filter.h b/include/linux/filter.h
index 632c03e126d9..8b5e9ac9eee4 100644
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -1108,7 +1108,7 @@ static inline int sk_filter_reason(struct sock *sk, struct sk_buff *skb,
return sk_filter_trim_cap(sk, skb, 1, reason);
}
-struct bpf_prog *bpf_prog_select_jit(struct bpf_prog *fp, int *err);
+struct bpf_prog *bpf_prog_select_jit(struct bpf_verifier_env *env, struct bpf_prog *fp, int *err);
struct bpf_prog *__bpf_prog_select_runtime(struct bpf_prog *fp, bool jit_attempted, int *err);
struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err);
void bpf_prog_free(struct bpf_prog *fp);
@@ -1155,7 +1155,7 @@ u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
((u64 (*)(u64, u64, u64, u64, u64, const struct bpf_insn *)) \
(void *)__bpf_call_base)
-struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog);
+struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog);
void bpf_jit_compile(struct bpf_prog *prog);
bool bpf_jit_needs_zext(void);
bool bpf_jit_inlines_helper_call(s32 imm);
@@ -1312,7 +1312,7 @@ int bpf_jit_get_func_addr(const struct bpf_prog *prog,
const char *bpf_jit_get_prog_name(struct bpf_prog *prog);
-struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *fp);
+struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bpf_prog *prog);
void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other);
static inline void bpf_jit_dump(unsigned int flen, unsigned int proglen,
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 00be578a438d..7702c232c62e 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -1486,7 +1486,7 @@ static void adjust_insn_arrays(struct bpf_prog *prog, u32 off, u32 len)
#endif
}
-struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *prog)
+struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bpf_prog *prog)
{
struct bpf_insn insn_buff[16], aux[2];
struct bpf_prog *clone, *tmp;
@@ -2505,13 +2505,13 @@ static bool bpf_prog_select_interpreter(struct bpf_prog *fp)
return select_interpreter;
}
-struct bpf_prog *bpf_prog_select_jit(struct bpf_prog *fp, int *err)
+struct bpf_prog *bpf_prog_select_jit(struct bpf_verifier_env *env, struct bpf_prog *fp, int *err)
{
*err = bpf_prog_alloc_jited_linfo(fp);
if (*err)
return fp;
- fp = bpf_int_jit_compile(fp);
+ fp = bpf_int_jit_compile(env, fp);
bpf_prog_jit_attempt_done(fp);
return fp;
}
@@ -2541,7 +2541,7 @@ struct bpf_prog *__bpf_prog_select_runtime(struct bpf_prog *fp, bool jit_attempt
*/
if (!bpf_prog_is_offloaded(fp->aux)) {
if (!jit_attempted) {
- fp = bpf_prog_select_jit(fp, err);
+ fp = bpf_prog_select_jit(NULL, fp, err);
if (*err)
return fp;
}
@@ -3072,7 +3072,7 @@ const struct bpf_func_proto bpf_tail_call_proto = {
* It is encouraged to implement bpf_int_jit_compile() instead, so that
* eBPF and implicitly also cBPF can get JITed!
*/
-struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_prog *prog)
+struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
{
return prog;
}
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index ab2bc0850770..1d2d42078ddf 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -22844,7 +22844,7 @@ static int jit_subprogs(struct bpf_verifier_env *env)
* all instruction adjustments should be accumulated
*/
old_len = func[i]->len;
- func[i] = bpf_int_jit_compile(func[i]);
+ func[i] = bpf_int_jit_compile(env, func[i]);
subprog_start_adjustment += func[i]->len - old_len;
if (!func[i]->jited) {
@@ -22890,7 +22890,7 @@ static int jit_subprogs(struct bpf_verifier_env *env)
}
for (i = 0; i < env->subprog_cnt; i++) {
old_bpf_func = func[i]->bpf_func;
- tmp = bpf_int_jit_compile(func[i]);
+ tmp = bpf_int_jit_compile(env, func[i]);
if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) {
verbose(env, "JIT doesn't support bpf-to-bpf calls\n");
err = -ENOTSUPP;
@@ -26089,7 +26089,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3
/* constants blinding in the JIT may increase prog->len */
len = env->prog->len;
if (env->subprog_cnt == 1)
- env->prog = bpf_prog_select_jit(env->prog, &ret);
+ env->prog = bpf_prog_select_jit(env, env->prog, &ret);
adjust_btf_func(env);
--
2.47.3