Re: [PATCH v3 3/3] LoongArch: BPF: Optimize 32-bit operations using native instructions

From: Hengqi Chen

Date: Wed Oct 28 2026 - 02:55:38 EST

On Wed, Oct 28, 2026 at 12:19 PM Huacai Chen <chenhuacai@xxxxxxxxxx> wrote:
>
> Hi, Hengqi,
>
> On Fri, Oct 16, 2026 at 4:10 PM Tiezhu Yang <yangtiezhu@xxxxxxxxxxx> wrote:
> >
> > Currently, the LoongArch BPF JIT emits numerous redundant zero and sign
> > extensions across 32-bit operations, bloating the final JIT image size
> > and also leading to runtime overhead, particularly in the division and
> > modulo operations. Thus, it is necessary to optimize 32-bit operations
> > by emitting LoongArch native 32-bit word instructions.
> >
> > For general ALU32 operations, utilize native 32-bit word instructions
> > for core calculations. Given that the BPF spec requires the upper 32
> > bits of the destination register to be zero-extended upon completion,
> > use a unified 'bstrpickd' instruction to clear the upper 32 bits at
> > the common exit of build_insn(), while immediate loads are optimized
> > via (u32) casting to bypass the trailing extension.
> >
> > According to the LoongArch Reference Manual, native 32-bit division
> > and modulo instructions (DIV.W[U] and MOD.W[U]) require the upper
> > bits [63:31] of both input operands to be all 0s or all 1s (0x0 or
> > 0x1ffffffff), that is to say, they need to be properly sign-extended
> > by the software on CPU cores without the DIV32 feature. Otherwise,
> > the 32-bit division and modulo instructions could trigger undefined
> > hardware behaviors and return unexpected values.
> >
> > To ensure both performance and hardware compatibility, leverage the
> > newly introduced cpu_has_div32 helper:
> > (1) on newer CPUs with DIV32, emit native 32-bit div/mod instructions
> > directly for optimal performance;
> > (2) on older CPUs without DIV32, sign-extend the destination register
> > in place and the source register into a temporary register via 'addw'
> > before executing the native 32-bit div/mod instructions. Note that the
> > JIT compiler must never alter the state of the src register because it
> > remains read-only for the duration of the instruction.
> >
> > For JMP32 conditional branches, the source operands may originate from
> > 64-bit contexts where the upper 32 bits contain arbitrary data, so use
> > 6 independent case structures to prevent 64-bit comparison inversions.
> > Furthermore, use 'bstrpickd' to mask out upper bits for unsigned JMP32,
> > and 'addw' with GPR_ZERO to re-align sign bits for signed JMP32.
> >
> > With this patch, the emit_{zext, sext}_32() and is_signed_bpf_cond()
> > helpers are completely wiped out from the header and source files,
> > delivering a native execution stream with a reduced JIT image size,
> > especially for division and mod operations.
> >
> > All of the testcases of test_bpf passed on Loongson-3A6000 (with DIV32)
> > and Loongson-3A5000 (without DIV32), here are the test results:
> >
> > $ sudo modprobe test_bpf
> > $ dmesg -t | grep Summary
> > test_bpf: Summary: 1061 PASSED, 0 FAILED, [1049/1049 JIT'ed]
> > test_bpf: test_tail_calls: Summary: 10 PASSED, 0 FAILED, [10/10 JIT'ed]
> > test_bpf: test_skb_segment: Summary: 2 PASSED, 0 FAILED
> >
> > For the BPF selftests test_verifier and test_progs, there are no any
> > regressions.
> >
> > Signed-off-by: Tiezhu Yang <yangtiezhu@xxxxxxxxxxx>
> > ---
> > arch/loongarch/include/asm/inst.h | 6 +
> > arch/loongarch/net/bpf_jit.c | 313 +++++++++++++++++-------------
> > arch/loongarch/net/bpf_jit.h | 18 --
> > 3 files changed, 183 insertions(+), 154 deletions(-)
> >
> > diff --git a/arch/loongarch/include/asm/inst.h b/arch/loongarch/include/asm/inst.h
> > index 585667e361c2..b55aae325300 100644
> > --- a/arch/loongarch/include/asm/inst.h
> > +++ b/arch/loongarch/include/asm/inst.h
> > @@ -752,11 +752,17 @@ static inline void emit_##NAME(union loongarch_instruction *insn, \
> >
> > DEF_EMIT_REG3_FORMAT(addw, addw_op)
> > DEF_EMIT_REG3_FORMAT(addd, addd_op)
> > +DEF_EMIT_REG3_FORMAT(subw, subw_op)
> > DEF_EMIT_REG3_FORMAT(subd, subd_op)
> > +DEF_EMIT_REG3_FORMAT(mulw, mulw_op)
> > DEF_EMIT_REG3_FORMAT(muld, muld_op)
> > +DEF_EMIT_REG3_FORMAT(divw, divw_op)
> > DEF_EMIT_REG3_FORMAT(divd, divd_op)
> > +DEF_EMIT_REG3_FORMAT(modw, modw_op)
> > DEF_EMIT_REG3_FORMAT(modd, modd_op)
> > +DEF_EMIT_REG3_FORMAT(divwu, divwu_op)
> > DEF_EMIT_REG3_FORMAT(divdu, divdu_op)
> > +DEF_EMIT_REG3_FORMAT(modwu, modwu_op)
> > DEF_EMIT_REG3_FORMAT(moddu, moddu_op)
> > DEF_EMIT_REG3_FORMAT(maskeqz, maskeqz_op)
> > DEF_EMIT_REG3_FORMAT(masknez, masknez_op)
> > diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c
> > index 77cd96a6f84c..1a7c8effc37b 100644
> > --- a/arch/loongarch/net/bpf_jit.c
> > +++ b/arch/loongarch/net/bpf_jit.c
> > @@ -288,7 +288,7 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn)
> > * if (index >= array->map.max_entries)
> > * goto out;
> > */
> > - emit_zext_32(ctx, a2, true);
> > + emit_insn(ctx, bstrpickd, a2, a2, 31, 0);
> >
> > off = offsetof(struct bpf_array, map.max_entries);
> > emit_insn(ctx, ldwu, t1, a1, off);
> > @@ -412,7 +412,6 @@ static int emit_atomic_rmw(const struct bpf_insn *insn, struct jit_ctx *ctx)
> > return -EINVAL;
> > }
> > emit_insn(ctx, amadddbb, src, t1, t3);
> > - emit_zext_32(ctx, src, true);
> > break;
> > case BPF_H:
> > if (!cpu_has_lam_bh) {
> > @@ -420,23 +419,23 @@ static int emit_atomic_rmw(const struct bpf_insn *insn, struct jit_ctx *ctx)
> > return -EINVAL;
> > }
> > emit_insn(ctx, amadddbh, src, t1, t3);
> > - emit_zext_32(ctx, src, true);
> > break;
> > case BPF_W:
> > emit_insn(ctx, amadddbw, src, t1, t3);
> > - emit_zext_32(ctx, src, true);
> > break;
> > case BPF_DW:
> > emit_insn(ctx, amadddbd, src, t1, t3);
> > break;
> > }
> > + if (!isdw)
> > + emit_insn(ctx, bstrpickd, src, src, 31, 0);
> > break;
> > case BPF_AND | BPF_FETCH:
> > if (isdw) {
> > emit_insn(ctx, amanddbd, src, t1, t3);
> > } else {
> > emit_insn(ctx, amanddbw, src, t1, t3);
> > - emit_zext_32(ctx, src, true);
> > + emit_insn(ctx, bstrpickd, src, src, 31, 0);
> > }
> > break;
> > case BPF_OR | BPF_FETCH:
> > @@ -444,7 +443,7 @@ static int emit_atomic_rmw(const struct bpf_insn *insn, struct jit_ctx *ctx)
> > emit_insn(ctx, amordbd, src, t1, t3);
> > } else {
> > emit_insn(ctx, amordbw, src, t1, t3);
> > - emit_zext_32(ctx, src, true);
> > + emit_insn(ctx, bstrpickd, src, src, 31, 0);
> > }
> > break;
> > case BPF_XOR | BPF_FETCH:
> > @@ -452,7 +451,7 @@ static int emit_atomic_rmw(const struct bpf_insn *insn, struct jit_ctx *ctx)
> > emit_insn(ctx, amxordbd, src, t1, t3);
> > } else {
> > emit_insn(ctx, amxordbw, src, t1, t3);
> > - emit_zext_32(ctx, src, true);
> > + emit_insn(ctx, bstrpickd, src, src, 31, 0);
> > }
> > break;
> > /* src = atomic_xchg(dst + off, src); */
> > @@ -464,7 +463,6 @@ static int emit_atomic_rmw(const struct bpf_insn *insn, struct jit_ctx *ctx)
> > return -EINVAL;
> > }
> > emit_insn(ctx, amswapdbb, src, t1, t3);
> > - emit_zext_32(ctx, src, true);
> > break;
> > case BPF_H:
> > if (!cpu_has_lam_bh) {
> > @@ -472,16 +470,16 @@ static int emit_atomic_rmw(const struct bpf_insn *insn, struct jit_ctx *ctx)
> > return -EINVAL;
> > }
> > emit_insn(ctx, amswapdbh, src, t1, t3);
> > - emit_zext_32(ctx, src, true);
> > break;
> > case BPF_W:
> > emit_insn(ctx, amswapdbw, src, t1, t3);
> > - emit_zext_32(ctx, src, true);
> > break;
> > case BPF_DW:
> > emit_insn(ctx, amswapdbd, src, t1, t3);
> > break;
> > }
> > + if (!isdw)
> > + emit_insn(ctx, bstrpickd, src, src, 31, 0);
> > break;
> > /* r0 = atomic_cmpxchg(dst + off, r0, src); */
> > case BPF_CMPXCHG:
> > @@ -494,13 +492,13 @@ static int emit_atomic_rmw(const struct bpf_insn *insn, struct jit_ctx *ctx)
> > emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -4);
> > } else {
> > emit_insn(ctx, llw, r0, t1, 0);
> > - emit_zext_32(ctx, t2, true);
> > - emit_zext_32(ctx, r0, true);
> > + emit_insn(ctx, bstrpickd, t2, t2, 31, 0);
> > + emit_insn(ctx, bstrpickd, r0, r0, 31, 0);
> > emit_insn(ctx, bne, t2, r0, 4);
> > move_reg(ctx, t3, src);
> > emit_insn(ctx, scw, t3, t1, 0);
> > emit_insn(ctx, beq, t3, LOONGARCH_GPR_ZERO, -6);
> > - emit_zext_32(ctx, r0, true);
> > + emit_insn(ctx, bstrpickd, r0, r0, 31, 0);
> > }
> > emit_insn(ctx, dbar, DBAR_LLSC_MB);
> > break;
> > @@ -605,12 +603,6 @@ static int emit_atomic_ld_st(const struct bpf_insn *insn, struct jit_ctx *ctx)
> > return 0;
> > }
> >
> > -static bool is_signed_bpf_cond(u8 cond)
> > -{
> > - return cond == BPF_JSGT || cond == BPF_JSLT ||
> > - cond == BPF_JSGE || cond == BPF_JSLE;
> > -}
> > -
> > #define BPF_FIXUP_REG_MASK GENMASK(31, 27)
> > #define BPF_FIXUP_OFFSET_MASK GENMASK(26, 0)
> > #define REG_DONT_CLEAR_MARKER 0
> > @@ -707,7 +699,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext
> > u8 dst = regmap[insn->dst_reg];
> > const s16 off = insn->off;
> > const s32 imm = insn->imm;
> > - const bool is32 = BPF_CLASS(insn->code) == BPF_ALU || BPF_CLASS(insn->code) == BPF_JMP32;
> > + const bool is32 = BPF_CLASS(insn->code) == BPF_ALU && BPF_OP(insn->code) != BPF_END;
> >
> > switch (code) {
> > /* dst = src */
> > @@ -715,7 +707,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext
> > case BPF_ALU64 | BPF_MOV | BPF_X:
> > if (insn_is_cast_user(insn)) {
> > move_reg(ctx, t1, src);
> > - emit_zext_32(ctx, t1, true);
> > + emit_insn(ctx, bstrpickd, t1, t1, 31, 0);
> > move_imm(ctx, dst, (ctx->user_vm_start >> 32) << 32);
> > emit_insn(ctx, beq, t1, LOONGARCH_GPR_ZERO, 2);
> > emit_insn(ctx, or, t1, dst, t1);
> > @@ -734,15 +726,12 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext
> > switch (off) {
> > case 0:
> > move_reg(ctx, dst, src);
> > - emit_zext_32(ctx, dst, is32);
> > break;
> > case 8:
> > emit_insn(ctx, extwb, dst, src);
> > - emit_zext_32(ctx, dst, is32);
> > break;
> > case 16:
> > emit_insn(ctx, extwh, dst, src);
> > - emit_zext_32(ctx, dst, is32);
> > break;
> > case 32:
> > emit_insn(ctx, addw, dst, src, LOONGARCH_GPR_ZERO);
> > @@ -752,151 +741,188 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext
> >
> > /* dst = imm */
> > case BPF_ALU | BPF_MOV | BPF_K:
> > + move_imm(ctx, dst, (u32)imm);
> > + return 0;
> > case BPF_ALU64 | BPF_MOV | BPF_K:
> > move_imm(ctx, dst, imm);
> > - emit_zext_32(ctx, dst, is32);
> > break;
> dst = imm is the simplest case, I think the 64bit version can also
> "return 0" here. If you agree, I will modify it when apply.
>

I am fine with the change. Cheers.

> Huacai
>
> >
> > /* dst = dst + src */
> > case BPF_ALU | BPF_ADD | BPF_X:
> > + emit_insn(ctx, addw, dst, dst, src);
> > + break;
> > case BPF_ALU64 | BPF_ADD | BPF_X:
> > emit_insn(ctx, addd, dst, dst, src);
> > - emit_zext_32(ctx, dst, is32);
> > break;
> >
> > /* dst = dst + imm */
> > case BPF_ALU | BPF_ADD | BPF_K:
> > + if (is_signed_imm12(imm)) {
> > + emit_insn(ctx, addiw, dst, dst, imm);
> > + } else {
> > + move_imm(ctx, t1, (u32)imm);
> > + emit_insn(ctx, addw, dst, dst, t1);
> > + }
> > + break;
> > case BPF_ALU64 | BPF_ADD | BPF_K:
> > if (is_signed_imm12(imm)) {
> > emit_insn(ctx, addid, dst, dst, imm);
> > } else {
> > move_imm(ctx, t1, imm);
> > - emit_zext_32(ctx, t1, is32);
> > emit_insn(ctx, addd, dst, dst, t1);
> > }
> > - emit_zext_32(ctx, dst, is32);
> > break;
> >
> > /* dst = dst - src */
> > case BPF_ALU | BPF_SUB | BPF_X:
> > + emit_insn(ctx, subw, dst, dst, src);
> > + break;
> > case BPF_ALU64 | BPF_SUB | BPF_X:
> > emit_insn(ctx, subd, dst, dst, src);
> > - emit_zext_32(ctx, dst, is32);
> > break;
> >
> > /* dst = dst - imm */
> > case BPF_ALU | BPF_SUB | BPF_K:
> > + if (is_signed_imm12(-imm)) {
> > + emit_insn(ctx, addiw, dst, dst, -imm);
> > + } else {
> > + move_imm(ctx, t1, (u32)imm);
> > + emit_insn(ctx, subw, dst, dst, t1);
> > + }
> > + break;
> > case BPF_ALU64 | BPF_SUB | BPF_K:
> > if (is_signed_imm12(-imm)) {
> > emit_insn(ctx, addid, dst, dst, -imm);
> > } else {
> > move_imm(ctx, t1, imm);
> > - emit_zext_32(ctx, t1, is32);
> > emit_insn(ctx, subd, dst, dst, t1);
> > }
> > - emit_zext_32(ctx, dst, is32);
> > break;
> >
> > /* dst = dst * src */
> > case BPF_ALU | BPF_MUL | BPF_X:
> > + emit_insn(ctx, mulw, dst, dst, src);
> > + break;
> > case BPF_ALU64 | BPF_MUL | BPF_X:
> > emit_insn(ctx, muld, dst, dst, src);
> > - emit_zext_32(ctx, dst, is32);
> > break;
> >
> > /* dst = dst * imm */
> > case BPF_ALU | BPF_MUL | BPF_K:
> > + move_imm(ctx, t1, (u32)imm);
> > + emit_insn(ctx, mulw, dst, dst, t1);
> > + break;
> > case BPF_ALU64 | BPF_MUL | BPF_K:
> > move_imm(ctx, t1, imm);
> > - emit_zext_32(ctx, t1, is32);
> > emit_insn(ctx, muld, dst, dst, t1);
> > - emit_zext_32(ctx, dst, is32);
> > break;
> >
> > /* dst = dst / src */
> > case BPF_ALU | BPF_DIV | BPF_X:
> > - case BPF_ALU64 | BPF_DIV | BPF_X:
> > - if (!off) {
> > - emit_zext_32(ctx, dst, is32);
> > - move_reg(ctx, t1, src);
> > - emit_zext_32(ctx, t1, is32);
> > - emit_insn(ctx, divdu, dst, dst, t1);
> > - emit_zext_32(ctx, dst, is32);
> > - } else {
> > - emit_sext_32(ctx, dst, is32);
> > - move_reg(ctx, t1, src);
> > - emit_sext_32(ctx, t1, is32);
> > - emit_insn(ctx, divd, dst, dst, t1);
> > - emit_zext_32(ctx, dst, is32);
> > + if (cpu_has_div32) {
> > + if (!off)
> > + emit_insn(ctx, divwu, dst, dst, src);
> > + else
> > + emit_insn(ctx, divw, dst, dst, src);
> > + break;
> > }
> > + /*
> > + * Legacy fallback for !div32: src must be extended into a
> > + * temporary register to keep the original src register intact.
> > + */
> > + emit_insn(ctx, addw, dst, dst, LOONGARCH_GPR_ZERO);
> > + emit_insn(ctx, addw, t1, src, LOONGARCH_GPR_ZERO);
> > + if (!off)
> > + emit_insn(ctx, divwu, dst, dst, t1);
> > + else
> > + emit_insn(ctx, divw, dst, dst, t1);
> > + break;
> > + case BPF_ALU64 | BPF_DIV | BPF_X:
> > + if (!off)
> > + emit_insn(ctx, divdu, dst, dst, src);
> > + else
> > + emit_insn(ctx, divd, dst, dst, src);
> > break;
> >
> > /* dst = dst / imm */
> > case BPF_ALU | BPF_DIV | BPF_K:
> > + move_imm(ctx, t1, (u32)imm);
> > + if (!cpu_has_div32) {
> > + emit_insn(ctx, addw, dst, dst, LOONGARCH_GPR_ZERO);
> > + emit_insn(ctx, addw, t1, t1, LOONGARCH_GPR_ZERO);
> > + }
> > + if (!off)
> > + emit_insn(ctx, divwu, dst, dst, t1);
> > + else
> > + emit_insn(ctx, divw, dst, dst, t1);
> > + break;
> > case BPF_ALU64 | BPF_DIV | BPF_K:
> > - if (!off) {
> > - move_imm(ctx, t1, imm);
> > - emit_zext_32(ctx, t1, is32);
> > - emit_zext_32(ctx, dst, is32);
> > + move_imm(ctx, t1, imm);
> > + if (!off)
> > emit_insn(ctx, divdu, dst, dst, t1);
> > - emit_zext_32(ctx, dst, is32);
> > - } else {
> > - move_imm(ctx, t1, imm);
> > - emit_sext_32(ctx, t1, is32);
> > - emit_sext_32(ctx, dst, is32);
> > + else
> > emit_insn(ctx, divd, dst, dst, t1);
> > - emit_zext_32(ctx, dst, is32);
> > - }
> > break;
> >
> > /* dst = dst % src */
> > case BPF_ALU | BPF_MOD | BPF_X:
> > - case BPF_ALU64 | BPF_MOD | BPF_X:
> > - if (!off) {
> > - emit_zext_32(ctx, dst, is32);
> > - move_reg(ctx, t1, src);
> > - emit_zext_32(ctx, t1, is32);
> > - emit_insn(ctx, moddu, dst, dst, t1);
> > - emit_zext_32(ctx, dst, is32);
> > - } else {
> > - emit_sext_32(ctx, dst, is32);
> > - move_reg(ctx, t1, src);
> > - emit_sext_32(ctx, t1, is32);
> > - emit_insn(ctx, modd, dst, dst, t1);
> > - emit_zext_32(ctx, dst, is32);
> > + if (cpu_has_div32) {
> > + if (!off)
> > + emit_insn(ctx, modwu, dst, dst, src);
> > + else
> > + emit_insn(ctx, modw, dst, dst, src);
> > + break;
> > }
> > + /*
> > + * Legacy fallback for !div32: src must be extended into a
> > + * temporary register to keep the original src register intact.
> > + */
> > + emit_insn(ctx, addw, dst, dst, LOONGARCH_GPR_ZERO);
> > + emit_insn(ctx, addw, t1, src, LOONGARCH_GPR_ZERO);
> > + if (!off)
> > + emit_insn(ctx, modwu, dst, dst, t1);
> > + else
> > + emit_insn(ctx, modw, dst, dst, t1);
> > + break;
> > + case BPF_ALU64 | BPF_MOD | BPF_X:
> > + if (!off)
> > + emit_insn(ctx, moddu, dst, dst, src);
> > + else
> > + emit_insn(ctx, modd, dst, dst, src);
> > break;
> >
> > /* dst = dst % imm */
> > case BPF_ALU | BPF_MOD | BPF_K:
> > + move_imm(ctx, t1, (u32)imm);
> > + if (!cpu_has_div32) {
> > + emit_insn(ctx, addw, dst, dst, LOONGARCH_GPR_ZERO);
> > + emit_insn(ctx, addw, t1, t1, LOONGARCH_GPR_ZERO);
> > + }
> > + if (!off)
> > + emit_insn(ctx, modwu, dst, dst, t1);
> > + else
> > + emit_insn(ctx, modw, dst, dst, t1);
> > + break;
> > case BPF_ALU64 | BPF_MOD | BPF_K:
> > - if (!off) {
> > - move_imm(ctx, t1, imm);
> > - emit_zext_32(ctx, t1, is32);
> > - emit_zext_32(ctx, dst, is32);
> > + move_imm(ctx, t1, imm);
> > + if (!off)
> > emit_insn(ctx, moddu, dst, dst, t1);
> > - emit_zext_32(ctx, dst, is32);
> > - } else {
> > - move_imm(ctx, t1, imm);
> > - emit_sext_32(ctx, t1, is32);
> > - emit_sext_32(ctx, dst, is32);
> > + else
> > emit_insn(ctx, modd, dst, dst, t1);
> > - emit_zext_32(ctx, dst, is32);
> > - }
> > break;
> >
> > /* dst = -dst */
> > case BPF_ALU | BPF_NEG:
> > + emit_insn(ctx, subw, dst, LOONGARCH_GPR_ZERO, dst);
> > + break;
> > case BPF_ALU64 | BPF_NEG:
> > emit_insn(ctx, subd, dst, LOONGARCH_GPR_ZERO, dst);
> > - emit_zext_32(ctx, dst, is32);
> > break;
> >
> > /* dst = dst & src */
> > case BPF_ALU | BPF_AND | BPF_X:
> > case BPF_ALU64 | BPF_AND | BPF_X:
> > emit_insn(ctx, and, dst, dst, src);
> > - emit_zext_32(ctx, dst, is32);
> > break;
> >
> > /* dst = dst & imm */
> > @@ -905,18 +931,18 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext
> > if (is_unsigned_imm12(imm)) {
> > emit_insn(ctx, andi, dst, dst, imm);
> > } else {
> > - move_imm(ctx, t1, imm);
> > - emit_zext_32(ctx, t1, is32);
> > + if (is32)
> > + move_imm(ctx, t1, (u32)imm);
> > + else
> > + move_imm(ctx, t1, imm);
> > emit_insn(ctx, and, dst, dst, t1);
> > }
> > - emit_zext_32(ctx, dst, is32);
> > break;
> >
> > /* dst = dst | src */
> > case BPF_ALU | BPF_OR | BPF_X:
> > case BPF_ALU64 | BPF_OR | BPF_X:
> > emit_insn(ctx, or, dst, dst, src);
> > - emit_zext_32(ctx, dst, is32);
> > break;
> >
> > /* dst = dst | imm */
> > @@ -925,18 +951,18 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext
> > if (is_unsigned_imm12(imm)) {
> > emit_insn(ctx, ori, dst, dst, imm);
> > } else {
> > - move_imm(ctx, t1, imm);
> > - emit_zext_32(ctx, t1, is32);
> > + if (is32)
> > + move_imm(ctx, t1, (u32)imm);
> > + else
> > + move_imm(ctx, t1, imm);
> > emit_insn(ctx, or, dst, dst, t1);
> > }
> > - emit_zext_32(ctx, dst, is32);
> > break;
> >
> > /* dst = dst ^ src */
> > case BPF_ALU | BPF_XOR | BPF_X:
> > case BPF_ALU64 | BPF_XOR | BPF_X:
> > emit_insn(ctx, xor, dst, dst, src);
> > - emit_zext_32(ctx, dst, is32);
> > break;
> >
> > /* dst = dst ^ imm */
> > @@ -945,19 +971,18 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext
> > if (is_unsigned_imm12(imm)) {
> > emit_insn(ctx, xori, dst, dst, imm);
> > } else {
> > - move_imm(ctx, t1, imm);
> > - emit_zext_32(ctx, t1, is32);
> > + if (is32)
> > + move_imm(ctx, t1, (u32)imm);
> > + else
> > + move_imm(ctx, t1, imm);
> > emit_insn(ctx, xor, dst, dst, t1);
> > }
> > - emit_zext_32(ctx, dst, is32);
> > break;
> >
> > /* dst = dst << src (logical) */
> > case BPF_ALU | BPF_LSH | BPF_X:
> > emit_insn(ctx, sllw, dst, dst, src);
> > - emit_zext_32(ctx, dst, is32);
> > break;
> > -
> > case BPF_ALU64 | BPF_LSH | BPF_X:
> > emit_insn(ctx, slld, dst, dst, src);
> > break;
> > @@ -965,9 +990,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext
> > /* dst = dst << imm (logical) */
> > case BPF_ALU | BPF_LSH | BPF_K:
> > emit_insn(ctx, slliw, dst, dst, imm);
> > - emit_zext_32(ctx, dst, is32);
> > break;
> > -
> > case BPF_ALU64 | BPF_LSH | BPF_K:
> > emit_insn(ctx, sllid, dst, dst, imm);
> > break;
> > @@ -975,9 +998,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext
> > /* dst = dst >> src (logical) */
> > case BPF_ALU | BPF_RSH | BPF_X:
> > emit_insn(ctx, srlw, dst, dst, src);
> > - emit_zext_32(ctx, dst, is32);
> > break;
> > -
> > case BPF_ALU64 | BPF_RSH | BPF_X:
> > emit_insn(ctx, srld, dst, dst, src);
> > break;
> > @@ -985,9 +1006,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext
> > /* dst = dst >> imm (logical) */
> > case BPF_ALU | BPF_RSH | BPF_K:
> > emit_insn(ctx, srliw, dst, dst, imm);
> > - emit_zext_32(ctx, dst, is32);
> > break;
> > -
> > case BPF_ALU64 | BPF_RSH | BPF_K:
> > emit_insn(ctx, srlid, dst, dst, imm);
> > break;
> > @@ -995,9 +1014,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext
> > /* dst = dst >> src (arithmetic) */
> > case BPF_ALU | BPF_ARSH | BPF_X:
> > emit_insn(ctx, sraw, dst, dst, src);
> > - emit_zext_32(ctx, dst, is32);
> > break;
> > -
> > case BPF_ALU64 | BPF_ARSH | BPF_X:
> > emit_insn(ctx, srad, dst, dst, src);
> > break;
> > @@ -1005,9 +1022,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext
> > /* dst = dst >> imm (arithmetic) */
> > case BPF_ALU | BPF_ARSH | BPF_K:
> > emit_insn(ctx, sraiw, dst, dst, imm);
> > - emit_zext_32(ctx, dst, is32);
> > break;
> > -
> > case BPF_ALU64 | BPF_ARSH | BPF_K:
> > emit_insn(ctx, sraid, dst, dst, imm);
> > break;
> > @@ -1016,12 +1031,10 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext
> > case BPF_ALU | BPF_END | BPF_FROM_LE:
> > switch (imm) {
> > case 16:
> > - /* zero-extend 16 bits into 64 bits */
> > emit_insn(ctx, bstrpickd, dst, dst, 15, 0);
> > break;
> > case 32:
> > - /* zero-extend 32 bits into 64 bits */
> > - emit_zext_32(ctx, dst, is32);
> > + emit_insn(ctx, bstrpickd, dst, dst, 31, 0);
> > break;
> > case 64:
> > /* do nothing */
> > @@ -1034,13 +1047,11 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext
> > switch (imm) {
> > case 16:
> > emit_insn(ctx, revb2h, dst, dst);
> > - /* zero-extend 16 bits into 64 bits */
> > emit_insn(ctx, bstrpickd, dst, dst, 15, 0);
> > break;
> > case 32:
> > emit_insn(ctx, revb2w, dst, dst);
> > - /* clear the upper 32 bits */
> > - emit_zext_32(ctx, dst, true);
> > + emit_insn(ctx, bstrpickd, dst, dst, 31, 0);
> > break;
> > case 64:
> > emit_insn(ctx, revbd, dst, dst);
> > @@ -1048,7 +1059,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext
> > }
> > break;
> >
> > - /* PC += off if dst cond src */
> > + /* PC += off if dst cond src (64-bit) */
> > case BPF_JMP | BPF_JEQ | BPF_X:
> > case BPF_JMP | BPF_JNE | BPF_X:
> > case BPF_JMP | BPF_JGT | BPF_X:
> > @@ -1059,31 +1070,38 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext
> > case BPF_JMP | BPF_JSGE | BPF_X:
> > case BPF_JMP | BPF_JSLT | BPF_X:
> > case BPF_JMP | BPF_JSLE | BPF_X:
> > + jmp_offset = bpf2la_offset(i, off, ctx);
> > + if (emit_cond_jmp(ctx, cond, dst, src, jmp_offset) < 0)
> > + goto toofar;
> > + break;
> > +
> > + /* PC += off if dst cond src (32-bit unsigned) */
> > case BPF_JMP32 | BPF_JEQ | BPF_X:
> > case BPF_JMP32 | BPF_JNE | BPF_X:
> > case BPF_JMP32 | BPF_JGT | BPF_X:
> > case BPF_JMP32 | BPF_JGE | BPF_X:
> > case BPF_JMP32 | BPF_JLT | BPF_X:
> > case BPF_JMP32 | BPF_JLE | BPF_X:
> > + jmp_offset = bpf2la_offset(i, off, ctx);
> > + emit_insn(ctx, bstrpickd, t1, dst, 31, 0);
> > + emit_insn(ctx, bstrpickd, t2, src, 31, 0);
> > + if (emit_cond_jmp(ctx, cond, t1, t2, jmp_offset) < 0)
> > + goto toofar;
> > + break;
> > +
> > + /* PC += off if dst cond src (32-bit signed) */
> > case BPF_JMP32 | BPF_JSGT | BPF_X:
> > case BPF_JMP32 | BPF_JSGE | BPF_X:
> > case BPF_JMP32 | BPF_JSLT | BPF_X:
> > case BPF_JMP32 | BPF_JSLE | BPF_X:
> > jmp_offset = bpf2la_offset(i, off, ctx);
> > - move_reg(ctx, t1, dst);
> > - move_reg(ctx, t2, src);
> > - if (is_signed_bpf_cond(BPF_OP(code))) {
> > - emit_sext_32(ctx, t1, is32);
> > - emit_sext_32(ctx, t2, is32);
> > - } else {
> > - emit_zext_32(ctx, t1, is32);
> > - emit_zext_32(ctx, t2, is32);
> > - }
> > + emit_insn(ctx, addw, t1, dst, LOONGARCH_GPR_ZERO);
> > + emit_insn(ctx, addw, t2, src, LOONGARCH_GPR_ZERO);
> > if (emit_cond_jmp(ctx, cond, t1, t2, jmp_offset) < 0)
> > goto toofar;
> > break;
> >
> > - /* PC += off if dst cond imm */
> > + /* PC += off if dst cond imm (64-bit) */
> > case BPF_JMP | BPF_JEQ | BPF_K:
> > case BPF_JMP | BPF_JNE | BPF_K:
> > case BPF_JMP | BPF_JGT | BPF_K:
> > @@ -1094,12 +1112,38 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext
> > case BPF_JMP | BPF_JSGE | BPF_K:
> > case BPF_JMP | BPF_JSLT | BPF_K:
> > case BPF_JMP | BPF_JSLE | BPF_K:
> > + jmp_offset = bpf2la_offset(i, off, ctx);
> > + if (imm) {
> > + move_imm(ctx, t1, imm);
> > + tm = t1;
> > + } else {
> > + tm = LOONGARCH_GPR_ZERO;
> > + }
> > + if (emit_cond_jmp(ctx, cond, dst, tm, jmp_offset) < 0)
> > + goto toofar;
> > + break;
> > +
> > + /* PC += off if dst cond imm (32-bit unsigned) */
> > case BPF_JMP32 | BPF_JEQ | BPF_K:
> > case BPF_JMP32 | BPF_JNE | BPF_K:
> > case BPF_JMP32 | BPF_JGT | BPF_K:
> > case BPF_JMP32 | BPF_JGE | BPF_K:
> > case BPF_JMP32 | BPF_JLT | BPF_K:
> > case BPF_JMP32 | BPF_JLE | BPF_K:
> > + jmp_offset = bpf2la_offset(i, off, ctx);
> > + if (imm) {
> > + move_imm(ctx, t1, imm);
> > + emit_insn(ctx, bstrpickd, t1, t1, 31, 0);
> > + tm = t1;
> > + } else {
> > + tm = LOONGARCH_GPR_ZERO;
> > + }
> > + emit_insn(ctx, bstrpickd, t2, dst, 31, 0);
> > + if (emit_cond_jmp(ctx, cond, t2, tm, jmp_offset) < 0)
> > + goto toofar;
> > + break;
> > +
> > + /* PC += off if dst cond imm (32-bit signed) */
> > case BPF_JMP32 | BPF_JSGT | BPF_K:
> > case BPF_JMP32 | BPF_JSGE | BPF_K:
> > case BPF_JMP32 | BPF_JSLT | BPF_K:
> > @@ -1107,19 +1151,12 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext
> > jmp_offset = bpf2la_offset(i, off, ctx);
> > if (imm) {
> > move_imm(ctx, t1, imm);
> > + emit_insn(ctx, addw, t1, t1, LOONGARCH_GPR_ZERO);
> > tm = t1;
> > } else {
> > - /* If imm is 0, simply use zero register. */
> > tm = LOONGARCH_GPR_ZERO;
> > }
> > - move_reg(ctx, t2, dst);
> > - if (is_signed_bpf_cond(BPF_OP(code))) {
> > - emit_sext_32(ctx, tm, is32);
> > - emit_sext_32(ctx, t2, is32);
> > - } else {
> > - emit_zext_32(ctx, tm, is32);
> > - emit_zext_32(ctx, t2, is32);
> > - }
> > + emit_insn(ctx, addw, t2, dst, LOONGARCH_GPR_ZERO);
> > if (emit_cond_jmp(ctx, cond, t2, tm, jmp_offset) < 0)
> > goto toofar;
> > break;
> > @@ -1129,7 +1166,8 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext
> > case BPF_JMP32 | BPF_JSET | BPF_X:
> > jmp_offset = bpf2la_offset(i, off, ctx);
> > emit_insn(ctx, and, t1, dst, src);
> > - emit_zext_32(ctx, t1, is32);
> > + if (BPF_CLASS(code) == BPF_JMP32)
> > + emit_insn(ctx, bstrpickd, t1, t1, 31, 0);
> > if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
> > goto toofar;
> > break;
> > @@ -1139,9 +1177,9 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext
> > case BPF_JMP32 | BPF_JSET | BPF_K:
> > jmp_offset = bpf2la_offset(i, off, ctx);
> > move_imm(ctx, t1, imm);
> > - emit_zext_32(ctx, t1, is32);
> > emit_insn(ctx, and, t1, dst, t1);
> > - emit_zext_32(ctx, t1, is32);
> > + if (BPF_CLASS(code) == BPF_JMP32)
> > + emit_insn(ctx, bstrpickd, t1, t1, 31, 0);
> > if (emit_cond_jmp(ctx, cond, t1, LOONGARCH_GPR_ZERO, jmp_offset) < 0)
> > goto toofar;
> > break;
> > @@ -1468,6 +1506,9 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext
> > return -EINVAL;
> > }
> >
> > + if (is32)
> > + emit_insn(ctx, bstrpickd, dst, dst, 31, 0);
> > +
> > return 0;
> >
> > toofar:
> > diff --git a/arch/loongarch/net/bpf_jit.h b/arch/loongarch/net/bpf_jit.h
> > index 57a98f8a055b..2bd0dc5b911c 100644
> > --- a/arch/loongarch/net/bpf_jit.h
> > +++ b/arch/loongarch/net/bpf_jit.h
> > @@ -72,24 +72,6 @@ static inline int epilogue_offset(const struct jit_ctx *ctx)
> > return (to - from);
> > }
> >
> > -/* Zero-extend 32 bits into 64 bits */
> > -static inline void emit_zext_32(struct jit_ctx *ctx, enum loongarch_gpr reg, bool is32)
> > -{
> > - if (!is32)
> > - return;
> > -
> > - emit_insn(ctx, lu32id, reg, 0);
> > -}
> > -
> > -/* Signed-extend 32 bits into 64 bits */
> > -static inline void emit_sext_32(struct jit_ctx *ctx, enum loongarch_gpr reg, bool is32)
> > -{
> > - if (!is32)
> > - return;
> > -
> > - emit_insn(ctx, addiw, reg, reg, 0);
> > -}
> > -
> > /* Emit proper extension according to ABI requirements.
> > * Note that it requires a value of size `size` already resides in register `reg`.
> > */
> > --
> > 2.42.0
> >