Re: [PATCH bpf-next v5] m68k, bpf: Add initial BPF JIT compiler support
From: Greg Ungerer
Date: Fri Aug 28 2026 - 06:52:17 EST
Hi Kuan-Wei,
On 24/8/26 02:38, Kuan-Wei Chiu wrote:
> Add a BPF JIT compiler for the m68k architecture.
>
> The JIT generates m68k machine code targeting m68020+ processors and
> coldfire parts with an mmu (ISA_B/ISA_C). It excludes plain 68000/68010
> and non-mmu coldfire processors, as it relies on 32-bit branch
> displacements (b<cc>.l) to handle large bpf programs.
>
> Tested with the test_bpf.ko:
> test_bpf: Summary: 1061 PASSED, 0 FAILED, [1049/1049 JIT'ed]
> test_bpf: test_tail_calls: Summary: 10 PASSED, 0 FAILED, [10/10 JIT'ed]
>
> Tested-by: Daniel Palmer <daniel@xxxxxxxxx>
> Signed-off-by: Kuan-Wei Chiu <visitorckw@xxxxxxxxx>
> ---
> Changes in v5:
> - Folded coldfire support provided by Greg
> - Updated Kconfig to select HAVE_EBPF_JIT if MMU
> - Optimized alu32 immediate & shift operations to use the 2 byte moveq
>
> P.S.
> Sorry for the delay in sending out v5.
> I'll try my best to keep the intervals between future versions shorter.
>
> To Greg: I'm not sure if I should include tags like Suggested-by or
> Co-developed-by for you, but I assume I'd need your explicit consent to
> do so anyway. Please feel free to add the appropriate tags.
Suggested-by doesn't quite feel right here, Co-developed-by is probably
right. Feel free to add in the appropriate place:
Co-developed-by: Greg Ungerer <gerg@xxxxxxxxxxxxxx>
Signed-off-by: Greg Ungerer <gerg@xxxxxxxxxxxxxx>
>From here it is really up to Geert now if and when he will pick this up.
I am happy with this patch in its current form.
Regards
Greg
>
> MAINTAINERS | 6 +
> arch/m68k/Kbuild | 2 +-
> arch/m68k/Kconfig | 1 +
> arch/m68k/net/Makefile | 2 +
> arch/m68k/net/bpf_jit_comp.c | 1811 ++++++++++++++++++++++++++++++++++
> 5 files changed, 1821 insertions(+), 1 deletion(-)
> create mode 100644 arch/m68k/net/Makefile
> create mode 100644 arch/m68k/net/bpf_jit_comp.c
>
> diff --git a/MAINTAINERS b/MAINTAINERS
> index c31a32a2c748..b3b310c972a4 100644
> --- a/MAINTAINERS
> +++ b/MAINTAINERS
> @@ -4807,6 +4807,12 @@ L: bpf@xxxxxxxxxxxxxxx
> S: Maintained
> F: arch/loongarch/net/
>
> +BPF JIT for M68K
> +M: Kuan-Wei Chiu <visitorckw@xxxxxxxxx>
> +L: bpf@xxxxxxxxxxxxxxx
> +S: Maintained
> +F: arch/m68k/net/
> +
> BPF JIT for MIPS (32-BIT AND 64-BIT)
> M: Johan Almbladh <johan.almbladh@xxxxxxxxxxxxxxxxx>
> M: Paul Burton <paulburton@xxxxxxxxxx>
> diff --git a/arch/m68k/Kbuild b/arch/m68k/Kbuild
> index 7762af9f6def..058ad41a728c 100644
> --- a/arch/m68k/Kbuild
> +++ b/arch/m68k/Kbuild
> @@ -1,5 +1,5 @@
> # SPDX-License-Identifier: GPL-2.0-only
> -obj-y += kernel/ mm/
> +obj-y += kernel/ mm/ net/
> obj-$(CONFIG_Q40) += q40/
> obj-$(CONFIG_AMIGA) += amiga/
> obj-$(CONFIG_ATARI) += atari/
> diff --git a/arch/m68k/Kconfig b/arch/m68k/Kconfig
> index 11835eb59d94..d9c6028f4928 100644
> --- a/arch/m68k/Kconfig
> +++ b/arch/m68k/Kconfig
> @@ -8,6 +8,7 @@ config M68K
> select ARCH_HAS_CPU_FINALIZE_INIT if MMU
> select ARCH_HAS_CURRENT_STACK_POINTER
> select ARCH_HAS_DMA_PREP_COHERENT if M68K_NONCOHERENT_DMA && !COLDFIRE
> + select HAVE_EBPF_JIT if MMU
> select ARCH_HAS_SYNC_DMA_FOR_DEVICE if M68K_NONCOHERENT_DMA
> select ARCH_HAVE_NMI_SAFE_CMPXCHG if RMW_INSNS
> select ARCH_MIGHT_HAVE_PC_PARPORT if ISA
> diff --git a/arch/m68k/net/Makefile b/arch/m68k/net/Makefile
> new file mode 100644
> index 000000000000..ec5b14763316
> --- /dev/null
> +++ b/arch/m68k/net/Makefile
> @@ -0,0 +1,2 @@
> +# SPDX-License-Identifier: GPL-2.0-only
> +obj-$(CONFIG_BPF_JIT) += bpf_jit_comp.o
> diff --git a/arch/m68k/net/bpf_jit_comp.c b/arch/m68k/net/bpf_jit_comp.c
> new file mode 100644
> index 000000000000..e3a5dca70a3f
> --- /dev/null
> +++ b/arch/m68k/net/bpf_jit_comp.c
> @@ -0,0 +1,1811 @@
> +// SPDX-License-Identifier: GPL-2.0-only
> +/*
> + * BPF JIT compiler for m68k
> + *
> + * Copyright (C) 2026 Kuan-Wei Chiu <visitorckw@xxxxxxxxx>
> + */
> +
> +#include <linux/bpf.h>
> +#include <linux/filter.h>
> +#include <linux/irqflags.h>
> +#include <linux/slab.h>
> +#include <asm/cacheflush.h>
> +#include <linux/math64.h>
> +#include <linux/limits.h>
> +
> +#define STACK_OFFSET(k) (-4 - (4 * (k)))
> +
> +enum {
> + BPF_R1_HI, BPF_R1_LO,
> + BPF_R2_HI, BPF_R2_LO,
> + BPF_R3_HI, BPF_R3_LO,
> + BPF_R4_HI, BPF_R4_LO,
> + BPF_R5_HI, BPF_R5_LO,
> + BPF_R6_HI, BPF_R6_LO,
> + BPF_R7_HI, BPF_R7_LO,
> + BPF_R8_HI, BPF_R8_LO,
> + BPF_R9_HI, BPF_R9_LO,
> + BPF_R10_HI, BPF_R10_LO,
> + BPF_AX_HI, BPF_AX_LO,
> + BPF_TC_HI, BPF_TC_LO,
> + BPF_JIT_SCRATCH_REGS,
> +};
> +
> +#define SCRATCH_SIZE (BPF_JIT_SCRATCH_REGS * 4)
> +
> +#define TMP_REG_1 14
> +#define TMP_REG_2 15
> +
> +#define M68K_D0 0
> +#define M68K_D1 1
> +#define M68K_D2 2
> +#define M68K_D3 3
> +#define M68K_D4 4
> +#define M68K_D5 5
> +
> +static const s8 bpf2m68k[16][2] = {
> + [BPF_REG_0] = {M68K_D1, M68K_D0},
> + [BPF_REG_1] = {STACK_OFFSET(BPF_R1_HI), STACK_OFFSET(BPF_R1_LO)},
> + [BPF_REG_2] = {STACK_OFFSET(BPF_R2_HI), STACK_OFFSET(BPF_R2_LO)},
> + [BPF_REG_3] = {STACK_OFFSET(BPF_R3_HI), STACK_OFFSET(BPF_R3_LO)},
> + [BPF_REG_4] = {STACK_OFFSET(BPF_R4_HI), STACK_OFFSET(BPF_R4_LO)},
> + [BPF_REG_5] = {STACK_OFFSET(BPF_R5_HI), STACK_OFFSET(BPF_R5_LO)},
> + [BPF_REG_6] = {STACK_OFFSET(BPF_R6_HI), STACK_OFFSET(BPF_R6_LO)},
> + [BPF_REG_7] = {STACK_OFFSET(BPF_R7_HI), STACK_OFFSET(BPF_R7_LO)},
> + [BPF_REG_8] = {STACK_OFFSET(BPF_R8_HI), STACK_OFFSET(BPF_R8_LO)},
> + [BPF_REG_9] = {STACK_OFFSET(BPF_R9_HI), STACK_OFFSET(BPF_R9_LO)},
> + [BPF_REG_10] = {STACK_OFFSET(BPF_R10_HI), STACK_OFFSET(BPF_R10_LO)},
> + [BPF_REG_AX] = {STACK_OFFSET(BPF_AX_HI), STACK_OFFSET(BPF_AX_LO)},
> + [TMP_REG_1] = {M68K_D3, M68K_D2},
> + [TMP_REG_2] = {M68K_D5, M68K_D4},
> +};
> +
> +struct jit_ctx {
> + const struct bpf_prog *prog;
> + u16 *target;
> + unsigned int idx;
> + int *offsets;
> +};
> +
> +static noinline u32 jit_mul32(u32 a, u32 b)
> +{
> + return a * b;
> +}
> +
> +static noinline u32 jit_div32(u32 a, u32 b)
> +{
> + return b ? a / b : 0;
> +}
> +
> +static noinline u32 jit_mod32(u32 a, u32 b)
> +{
> + return b ? a % b : a;
> +}
> +
> +static noinline s32 jit_sdiv32(s32 a, s32 b)
> +{
> + if (b == 0)
> + return 0;
> + if (a == S32_MIN && b == -1)
> + return S32_MIN;
> + return a / b;
> +}
> +
> +static noinline s32 jit_smod32(s32 a, s32 b)
> +{
> + if (b == 0)
> + return a;
> + if (a == S32_MIN && b == -1)
> + return 0;
> + return a % b;
> +}
> +
> +static noinline u64 jit_mul64(u64 a, u64 b)
> +{
> + return a * b;
> +}
> +
> +static noinline u64 jit_div64(u64 a, u64 b)
> +{
> + return b ? div64_u64(a, b) : 0;
> +}
> +
> +static noinline s64 jit_sdiv64(s64 a, s64 b)
> +{
> + if (b == 0)
> + return 0;
> + if (a == S64_MIN && b == -1)
> + return S64_MIN;
> + return div64_s64(a, b);
> +}
> +
> +static noinline u64 jit_mod64(u64 a, u64 b)
> +{
> + u64 rem;
> +
> + if (b == 0)
> + return a;
> + div64_u64_rem(a, b, &rem);
> + return rem;
> +}
> +
> +static noinline s64 jit_smod64(s64 a, s64 b)
> +{
> + if (b == 0)
> + return a;
> + if (a == S64_MIN && b == -1)
> + return 0;
> + return a - div64_s64(a, b) * b;
> +}
> +
> +static u32 jit_atomic32(u32 *ptr, u32 val, u32 op, u32 cmp)
> +{
> + unsigned long flags;
> + u32 old;
> +
> + local_irq_save(flags);
> + old = *ptr;
> + switch (op) {
> + case BPF_ADD:
> + case BPF_ADD | BPF_FETCH:
> + *ptr += val;
> + break;
> + case BPF_AND:
> + case BPF_AND | BPF_FETCH:
> + *ptr &= val;
> + break;
> + case BPF_OR:
> + case BPF_OR | BPF_FETCH:
> + *ptr |= val;
> + break;
> + case BPF_XOR:
> + case BPF_XOR | BPF_FETCH:
> + *ptr ^= val;
> + break;
> + case BPF_XCHG:
> + *ptr = val;
> + break;
> + case BPF_CMPXCHG:
> + if (old == cmp)
> + *ptr = val;
> + break;
> + }
> + local_irq_restore(flags);
> +
> + return old;
> +}
> +
> +static u64 jit_atomic64(u32 *ptr, u32 val_hi, u32 val_lo, u32 op, u32 cmp_hi, u32 cmp_lo)
> +{
> + unsigned long flags;
> + u64 old, val, cmp;
> + u64 *ptr64 = (u64 *)ptr;
> +
> + val = ((u64)val_hi << 32) | val_lo;
> + cmp = ((u64)cmp_hi << 32) | cmp_lo;
> +
> + local_irq_save(flags);
> + old = *ptr64;
> + switch (op) {
> + case BPF_ADD:
> + case BPF_ADD | BPF_FETCH:
> + *ptr64 += val;
> + break;
> + case BPF_AND:
> + case BPF_AND | BPF_FETCH:
> + *ptr64 &= val;
> + break;
> + case BPF_OR:
> + case BPF_OR | BPF_FETCH:
> + *ptr64 |= val;
> + break;
> + case BPF_XOR:
> + case BPF_XOR | BPF_FETCH:
> + *ptr64 ^= val;
> + break;
> + case BPF_XCHG:
> + *ptr64 = val;
> + break;
> + case BPF_CMPXCHG:
> + if (old == cmp)
> + *ptr64 = val;
> + break;
> + }
> + local_irq_restore(flags);
> +
> + return old;
> +}
> +
> +static inline void emit_16(struct jit_ctx *ctx, u16 inst)
> +{
> + if (ctx->target)
> + ctx->target[ctx->idx] = inst;
> + ctx->idx++;
> +}
> +
> +static inline void emit_32(struct jit_ctx *ctx, u32 val)
> +{
> + emit_16(ctx, val >> 16);
> + emit_16(ctx, val & 0xffff);
> +}
> +
> +static inline bool is_stacked(s8 reg)
> +{
> + return reg < 0;
> +}
> +
> +static s8 bpf_get_reg32(s8 bpf_reg, s8 tmp_reg, struct jit_ctx *ctx)
> +{
> + if (is_stacked(bpf_reg)) {
> + emit_16(ctx, 0x202e | (tmp_reg << 9)); /* move.l d16(%fp), reg */
> + emit_16(ctx, (u16)bpf_reg);
> + return tmp_reg;
> + }
> + return bpf_reg;
> +}
> +
> +static void bpf_put_reg32(s8 bpf_reg, s8 src_reg, struct jit_ctx *ctx)
> +{
> + if (is_stacked(bpf_reg)) {
> + emit_16(ctx, 0x2d40 | src_reg); /* move.l reg, d16(%fp) */
> + emit_16(ctx, (u16)bpf_reg);
> + } else if (bpf_reg != src_reg) {
> + emit_16(ctx, 0x2000 | (bpf_reg << 9) | src_reg); /* move.l src, dst */
> + }
> +}
> +
> +static void emit_alu32_neg(const struct bpf_insn *insn, struct jit_ctx *ctx)
> +{
> + const s8 *dst = bpf2m68k[insn->dst_reg];
> + const s8 *tmp1 = bpf2m68k[TMP_REG_1];
> + s8 d_reg = bpf_get_reg32(dst[1], tmp1[1], ctx);
> +
> + emit_16(ctx, 0x4480 | d_reg); /* neg.l d_reg */
> +
> + bpf_put_reg32(dst[1], d_reg, ctx);
> + emit_16(ctx, 0x7000 | (tmp1[0] << 9)); /* moveq #0, tmp1 */
> + bpf_put_reg32(dst[0], tmp1[0], ctx);
> +}
> +
> +static void emit_alu64_neg(const struct bpf_insn *insn, struct jit_ctx *ctx)
> +{
> + const s8 *dst = bpf2m68k[insn->dst_reg];
> + const s8 *tmp1 = bpf2m68k[TMP_REG_1];
> + s8 d_lo = bpf_get_reg32(dst[1], tmp1[1], ctx);
> + s8 d_hi = bpf_get_reg32(dst[0], tmp1[0], ctx);
> +
> + emit_16(ctx, 0x4480 | d_lo); /* neg.l d_lo */
> + emit_16(ctx, 0x4080 | d_hi); /* negx.l d_hi */
> +
> + bpf_put_reg32(dst[1], d_lo, ctx);
> + bpf_put_reg32(dst[0], d_hi, ctx);
> +}
> +
> +static void emit_alu32_x(const struct bpf_insn *insn, struct jit_ctx *ctx)
> +{
> + const s8 *dst = bpf2m68k[insn->dst_reg];
> + const s8 *src = bpf2m68k[insn->src_reg];
> + const s8 *tmp1 = bpf2m68k[TMP_REG_1];
> + const s8 *tmp2 = bpf2m68k[TMP_REG_2];
> + s8 d_reg, s_reg;
> +
> + d_reg = bpf_get_reg32(dst[1], tmp1[1], ctx);
> + s_reg = bpf_get_reg32(src[1], tmp2[1], ctx);
> +
> + switch (BPF_OP(insn->code)) {
> + case BPF_MOV:
> + if (insn->off == 8 || insn->off == 16) {
> + if (d_reg != s_reg)
> + emit_16(ctx, 0x2000 | (d_reg << 9) | s_reg); /* move.l src, dst */
> + if (insn->off == 8)
> + emit_16(ctx, 0x49c0 | d_reg); /* extb.l d_reg */
> + else
> + emit_16(ctx, 0x48c0 | d_reg); /* ext.l d_reg */
> + } else {
> + if (d_reg != s_reg)
> + emit_16(ctx, 0x2000 | (d_reg << 9) | s_reg); /* move.l src, dst */
> + }
> + break;
> + case BPF_ADD:
> + emit_16(ctx, 0xd080 | (d_reg << 9) | s_reg); /* add.l src, dst */
> + break;
> + case BPF_SUB:
> + emit_16(ctx, 0x9080 | (d_reg << 9) | s_reg); /* sub.l src, dst */
> + break;
> + case BPF_AND:
> + emit_16(ctx, 0xc080 | (d_reg << 9) | s_reg); /* and.l src, dst */
> + break;
> + case BPF_OR:
> + emit_16(ctx, 0x8080 | (d_reg << 9) | s_reg); /* or.l src, dst */
> + break;
> + case BPF_XOR:
> + emit_16(ctx, 0xb180 | (s_reg << 9) | d_reg); /* eor.l src, dst */
> + break;
> + case BPF_LSH:
> + case BPF_RSH:
> + case BPF_ARSH: {
> + s8 count_reg = tmp2[1];
> +
> + if (s_reg != count_reg)
> + emit_16(ctx, 0x2000 | (count_reg << 9) | s_reg); /* move.l src, count */
> +
> + emit_16(ctx, 0x0280 | count_reg); /* andi.l #imm, count */
> + emit_32(ctx, 0x1f);
> +
> + if (BPF_OP(insn->code) == BPF_LSH)
> + emit_16(ctx, 0xe1a8 | (count_reg << 9) | d_reg); /* lsl.l count, dst */
> + else if (BPF_OP(insn->code) == BPF_RSH)
> + emit_16(ctx, 0xe0a8 | (count_reg << 9) | d_reg); /* lsr.l count, dst */
> + else
> + emit_16(ctx, 0xe0a0 | (count_reg << 9) | d_reg); /* asr.l count, dst */
> + break;
> + }
> + }
> +
> + bpf_put_reg32(dst[1], d_reg, ctx);
> + emit_16(ctx, 0x7000 | (tmp1[0] << 9)); /* moveq #0, tmp1 */
> + bpf_put_reg32(dst[0], tmp1[0], ctx);
> +}
> +
> +static void emit_alu32_k(const struct bpf_insn *insn, struct jit_ctx *ctx)
> +{
> + const s8 *dst = bpf2m68k[insn->dst_reg];
> + const s8 *tmp1 = bpf2m68k[TMP_REG_1];
> + const s8 *tmp2 = bpf2m68k[TMP_REG_2];
> + s8 d_reg;
> + s32 imm = insn->imm;
> + bool is_moveq = (imm >= -128 && imm <= 127);
> +
> + d_reg = bpf_get_reg32(dst[1], tmp1[1], ctx);
> +
> + switch (BPF_OP(insn->code)) {
> + case BPF_MOV:
> + if (is_moveq) {
> + emit_16(ctx, 0x7000 | (d_reg << 9) | (imm & 0xff)); /* moveq #imm, dst */
> + } else {
> + emit_16(ctx, 0x203c | (d_reg << 9)); /* move.l #imm, dst */
> + emit_32(ctx, imm);
> + }
> + break;
> + case BPF_ADD:
> + if (is_moveq) {
> + emit_16(ctx, 0x7000 | (tmp2[1] << 9) | (imm & 0xff)); /* moveq #imm, tmp2 */
> + emit_16(ctx, 0xd080 | (d_reg << 9) | tmp2[1]); /* add.l tmp2, dst */
> + } else {
> + emit_16(ctx, 0x0680 | d_reg); /* addi.l #imm, dst */
> + emit_32(ctx, imm);
> + }
> + break;
> + case BPF_SUB:
> + if (is_moveq) {
> + emit_16(ctx, 0x7000 | (tmp2[1] << 9) | (imm & 0xff)); /* moveq #imm, tmp2 */
> + emit_16(ctx, 0x9080 | (d_reg << 9) | tmp2[1]); /* sub.l tmp2, dst */
> + } else {
> + emit_16(ctx, 0x0480 | d_reg); /* subi.l #imm, dst */
> + emit_32(ctx, imm);
> + }
> + break;
> + case BPF_AND:
> + if (is_moveq) {
> + emit_16(ctx, 0x7000 | (tmp2[1] << 9) | (imm & 0xff)); /* moveq #imm, tmp2 */
> + emit_16(ctx, 0xc080 | (d_reg << 9) | tmp2[1]); /* and.l tmp2, dst */
> + } else {
> + emit_16(ctx, 0x0280 | d_reg); /* andi.l #imm, dst */
> + emit_32(ctx, imm);
> + }
> + break;
> + case BPF_OR:
> + if (is_moveq) {
> + emit_16(ctx, 0x7000 | (tmp2[1] << 9) | (imm & 0xff)); /* moveq #imm, tmp2 */
> + emit_16(ctx, 0x8080 | (d_reg << 9) | tmp2[1]); /* or.l tmp2, dst */
> + } else {
> + emit_16(ctx, 0x0080 | d_reg); /* ori.l #imm, dst */
> + emit_32(ctx, imm);
> + }
> + break;
> + case BPF_XOR:
> + if (is_moveq) {
> + emit_16(ctx, 0x7000 | (tmp2[1] << 9) | (imm & 0xff)); /* moveq #imm, tmp2 */
> + emit_16(ctx, 0xb180 | (tmp2[1] << 9) | d_reg); /* eor.l tmp2, dst */
> + } else {
> + emit_16(ctx, 0x0a80 | d_reg); /* eori.l #imm, dst */
> + emit_32(ctx, imm);
> + }
> + break;
> + case BPF_LSH:
> + case BPF_RSH:
> + case BPF_ARSH:
> + emit_16(ctx, 0x7000 | (tmp2[1] << 9) | (imm & 0x1f)); /* moveq #imm, count */
> +
> + if (BPF_OP(insn->code) == BPF_LSH)
> + emit_16(ctx, 0xe1a8 | (tmp2[1] << 9) | d_reg); /* lsl.l count, dst */
> + else if (BPF_OP(insn->code) == BPF_RSH)
> + emit_16(ctx, 0xe0a8 | (tmp2[1] << 9) | d_reg); /* lsr.l count, dst */
> + else
> + emit_16(ctx, 0xe0a0 | (tmp2[1] << 9) | d_reg); /* asr.l count, dst */
> + break;
> + }
> +
> + bpf_put_reg32(dst[1], d_reg, ctx);
> + emit_16(ctx, 0x7000 | (tmp1[0] << 9)); /* moveq #0, tmp1 */
> + bpf_put_reg32(dst[0], tmp1[0], ctx);
> +}
> +
> +static void emit_alu64_x(const struct bpf_insn *insn, struct jit_ctx *ctx)
> +{
> + const s8 *dst = bpf2m68k[insn->dst_reg];
> + const s8 *src = bpf2m68k[insn->src_reg];
> + const s8 *tmp1 = bpf2m68k[TMP_REG_1];
> + const s8 *tmp2 = bpf2m68k[TMP_REG_2];
> + s8 d_lo, d_hi, s_lo, s_hi;
> +
> + d_lo = bpf_get_reg32(dst[1], tmp1[1], ctx);
> + d_hi = bpf_get_reg32(dst[0], tmp1[0], ctx);
> + s_lo = bpf_get_reg32(src[1], tmp2[1], ctx);
> + s_hi = bpf_get_reg32(src[0], tmp2[0], ctx);
> +
> + switch (BPF_OP(insn->code)) {
> + case BPF_MOV:
> + if (insn->off == 8 || insn->off == 16 || insn->off == 32) {
> + if (d_lo != s_lo)
> + emit_16(ctx, 0x2000 | (d_lo << 9) | s_lo); /* move.l src, dst */
> +
> + if (insn->off == 8)
> + emit_16(ctx, 0x49c0 | d_lo); /* extb.l d_lo */
> + else if (insn->off == 16)
> + emit_16(ctx, 0x48c0 | d_lo); /* ext.l d_lo */
> +
> + emit_16(ctx, 0x4a80 | d_lo); /* tst.l d_lo */
> + emit_16(ctx, 0x5bc0 | d_hi); /* smi d_hi */
> + emit_16(ctx, 0x49c0 | d_hi); /* extb.l d_hi */
> + } else {
> + if (d_lo != s_lo)
> + emit_16(ctx, 0x2000 | (d_lo << 9) | s_lo); /* move.l s_lo, d_lo */
> + if (d_hi != s_hi)
> + emit_16(ctx, 0x2000 | (d_hi << 9) | s_hi); /* move.l s_hi, d_hi */
> + }
> + break;
> + case BPF_ADD:
> + emit_16(ctx, 0xd080 | (d_lo << 9) | s_lo); /* add.l s_lo, d_lo */
> + emit_16(ctx, 0xd180 | (d_hi << 9) | s_hi); /* addx.l s_hi, d_hi */
> + break;
> + case BPF_SUB:
> + emit_16(ctx, 0x9080 | (d_lo << 9) | s_lo); /* sub.l s_lo, d_lo */
> + emit_16(ctx, 0x9180 | (d_hi << 9) | s_hi); /* subx.l s_hi, d_hi */
> + break;
> + case BPF_AND:
> + emit_16(ctx, 0xc080 | (d_lo << 9) | s_lo); /* and.l s_lo, d_lo */
> + emit_16(ctx, 0xc080 | (d_hi << 9) | s_hi); /* and.l s_hi, d_hi */
> + break;
> + case BPF_OR:
> + emit_16(ctx, 0x8080 | (d_lo << 9) | s_lo); /* or.l s_lo, d_lo */
> + emit_16(ctx, 0x8080 | (d_hi << 9) | s_hi); /* or.l s_hi, d_hi */
> + break;
> + case BPF_XOR:
> + emit_16(ctx, 0xb180 | (s_lo << 9) | d_lo); /* eor.l s_lo, d_lo */
> + emit_16(ctx, 0xb180 | (s_hi << 9) | d_hi); /* eor.l s_hi, d_hi */
> + break;
> + }
> +
> + bpf_put_reg32(dst[1], d_lo, ctx);
> + bpf_put_reg32(dst[0], d_hi, ctx);
> +}
> +
> +static void emit_alu64_k(const struct bpf_insn *insn, struct jit_ctx *ctx)
> +{
> + const s8 *dst = bpf2m68k[insn->dst_reg];
> + const s8 *tmp1 = bpf2m68k[TMP_REG_1];
> + const s8 *tmp2 = bpf2m68k[TMP_REG_2];
> + s8 d_lo, d_hi, s_lo, s_hi;
> + u32 imm_lo = insn->imm;
> + u32 imm_hi = insn->imm < 0 ? 0xffffffff : 0;
> +
> + d_lo = bpf_get_reg32(dst[1], tmp1[1], ctx);
> + d_hi = bpf_get_reg32(dst[0], tmp1[0], ctx);
> +
> + if (BPF_OP(insn->code) == BPF_MOV) {
> + emit_16(ctx, 0x203c | (d_lo << 9)); /* move.l #imm, d_lo */
> + emit_32(ctx, imm_lo);
> + emit_16(ctx, 0x203c | (d_hi << 9)); /* move.l #imm, d_hi */
> + emit_32(ctx, imm_hi);
> + bpf_put_reg32(dst[1], d_lo, ctx);
> + bpf_put_reg32(dst[0], d_hi, ctx);
> + return;
> + }
> +
> + s_lo = tmp2[1];
> + s_hi = tmp2[0];
> +
> + emit_16(ctx, 0x203c | (s_lo << 9)); /* move.l #imm, s_lo */
> + emit_32(ctx, imm_lo);
> + emit_16(ctx, 0x203c | (s_hi << 9)); /* move.l #imm, s_hi */
> + emit_32(ctx, imm_hi);
> +
> + switch (BPF_OP(insn->code)) {
> + case BPF_ADD:
> + emit_16(ctx, 0xd080 | (d_lo << 9) | s_lo); /* add.l s_lo, d_lo */
> + emit_16(ctx, 0xd180 | (d_hi << 9) | s_hi); /* addx.l s_hi, d_hi */
> + break;
> + case BPF_SUB:
> + emit_16(ctx, 0x9080 | (d_lo << 9) | s_lo); /* sub.l s_lo, d_lo */
> + emit_16(ctx, 0x9180 | (d_hi << 9) | s_hi); /* subx.l s_hi, d_hi */
> + break;
> + case BPF_AND:
> + emit_16(ctx, 0xc080 | (d_lo << 9) | s_lo); /* and.l s_lo, d_lo */
> + emit_16(ctx, 0xc080 | (d_hi << 9) | s_hi); /* and.l s_hi, d_hi */
> + break;
> + case BPF_OR:
> + emit_16(ctx, 0x8080 | (d_lo << 9) | s_lo); /* or.l s_lo, d_lo */
> + emit_16(ctx, 0x8080 | (d_hi << 9) | s_hi); /* or.l s_hi, d_hi */
> + break;
> + case BPF_XOR:
> + emit_16(ctx, 0xb180 | (s_lo << 9) | d_lo); /* eor.l s_lo, d_lo */
> + emit_16(ctx, 0xb180 | (s_hi << 9) | d_hi); /* eor.l s_hi, d_hi */
> + break;
> + }
> +
> + bpf_put_reg32(dst[1], d_lo, ctx);
> + bpf_put_reg32(dst[0], d_hi, ctx);
> +}
> +
> +static inline void emit_lsh64(struct jit_ctx *ctx, s8 d_lo, s8 d_hi)
> +{
> + if (CPU_IS_COLDFIRE) {
> + emit_16(ctx, 0xd080 | (d_lo << 9) | d_lo); /* add.l d_lo, d_lo */
> + emit_16(ctx, 0xd180 | (d_hi << 9) | d_hi); /* addx.l d_hi, d_hi */
> + } else {
> + emit_16(ctx, 0xe388 | d_lo); /* lsll #1, d_lo */
> + emit_16(ctx, 0xe390 | d_hi); /* roxl.l #1, d_hi */
> + }
> +}
> +
> +static inline void emit_rsh64(struct jit_ctx *ctx, s8 d_lo, s8 d_hi)
> +{
> + if (CPU_IS_COLDFIRE) {
> + emit_16(ctx, 0xe288 | d_lo); /* lsr.l #1, d_lo */
> + emit_16(ctx, 0xe288 | d_hi); /* lsr.l #1, d_hi */
> + emit_16(ctx, 0x6400); /* bcc 1f */
> + emit_16(ctx, 0x0006);
> + emit_16(ctx, 0x08c0 | d_lo); /* bset #31, d_lo */
> + emit_16(ctx, 0x001f);
> + } else {
> + emit_16(ctx, 0xe288 | d_hi); /* lsrl #1, d_hi */
> + emit_16(ctx, 0xe290 | d_lo); /* roxr.l #1, d_lo */
> + }
> +}
> +
> +static inline void emit_arsh64(struct jit_ctx *ctx, s8 d_lo, s8 d_hi)
> +{
> + if (CPU_IS_COLDFIRE) {
> + emit_16(ctx, 0xe288 | d_lo); /* lsr.l #1, d_lo */
> + emit_16(ctx, 0xe280 | d_hi); /* asr.l #1, d_hi */
> + emit_16(ctx, 0x6400); /* bcc 1f */
> + emit_16(ctx, 0x0006);
> + emit_16(ctx, 0x08c0 | d_lo); /* bset #31, d_lo */
> + emit_16(ctx, 0x001f);
> + } else {
> + emit_16(ctx, 0xe280 | d_hi); /* asrl #1, d_hi */
> + emit_16(ctx, 0xe290 | d_lo); /* roxr.l #1, d_lo */
> + }
> +}
> +
> +static void emit_alu64_shift(const struct bpf_insn *insn, struct jit_ctx *ctx, bool is_imm)
> +{
> + const s8 *dst = bpf2m68k[insn->dst_reg];
> + const s8 *tmp1 = bpf2m68k[TMP_REG_1];
> + const s8 *tmp2 = bpf2m68k[TMP_REG_2];
> + s8 d_lo, d_hi, count_reg;
> + int loop_start, done_idx;
> +
> + d_lo = bpf_get_reg32(dst[1], tmp1[1], ctx);
> + d_hi = bpf_get_reg32(dst[0], tmp1[0], ctx);
> + count_reg = tmp2[1];
> +
> + if (is_imm) {
> + emit_16(ctx, 0x203c | (count_reg << 9)); /* move.l #imm, count_reg */
> + emit_32(ctx, insn->imm);
> + } else {
> + const s8 *src = bpf2m68k[insn->src_reg];
> + s8 s_lo = bpf_get_reg32(src[1], tmp2[1], ctx);
> +
> + if (count_reg != s_lo)
> + emit_16(ctx, 0x2000 | (count_reg << 9) | s_lo); /* move.l s_lo, count_reg */
> + }
> +
> + emit_16(ctx, 0x0280 | count_reg); /* andi.l #0x3f, count_reg */
> + emit_32(ctx, 0x3f);
> +
> + emit_16(ctx, 0x4a80 | count_reg); /* tst.l count_reg */
> + emit_16(ctx, 0x6700); /* beq.w done_idx */
> + done_idx = ctx->idx;
> + emit_16(ctx, 0);
> +
> + loop_start = ctx->idx;
> +
> + if (BPF_OP(insn->code) == BPF_LSH)
> + emit_lsh64(ctx, d_lo, d_hi);
> + else if (BPF_OP(insn->code) == BPF_RSH)
> + emit_rsh64(ctx, d_lo, d_hi);
> + else if (BPF_OP(insn->code) == BPF_ARSH)
> + emit_arsh64(ctx, d_lo, d_hi);
> +
> + emit_16(ctx, 0x5380 | count_reg); /* subq.l #1, count_reg */
> + emit_16(ctx, 0x6600); /* bne.w loop_start */
> + emit_16(ctx, (loop_start - (int)ctx->idx) * 2);
> +
> + if (ctx->target)
> + ctx->target[done_idx] = (ctx->idx - done_idx) * 2;
> +
> + bpf_put_reg32(dst[1], d_lo, ctx);
> + bpf_put_reg32(dst[0], d_hi, ctx);
> +}
> +
> +static inline void emit_to_le16(struct jit_ctx *ctx, s8 d_lo, s8 d_hi)
> +{
> + if (CPU_IS_COLDFIRE) {
> + emit_16(ctx, 0x71c0 | (d_lo << 9) | d_lo); /* mvz.w d_lo, d_lo */
> + emit_16(ctx, 0x7180 | (d_hi << 9) | d_lo); /* mvz.b d_lo, d_hi */
> + emit_16(ctx, 0xe088 | d_lo); /* lsr.l #8, d_lo */
> + emit_16(ctx, 0xe188 | d_hi); /* lsl.l #8, d_hi */
> + emit_16(ctx, 0x8080 | (d_lo << 9) | d_hi); /* or.l d_hi, d_lo */
> + } else {
> + emit_16(ctx, 0x0280 | d_lo); /* andi.l #0xffff, d_lo */
> + emit_32(ctx, 0xffff);
> + emit_16(ctx, 0xe058 | d_lo); /* ror.w #8, d_lo */
> + }
> +
> + emit_16(ctx, 0x7000 | (d_hi << 9)); /* moveq #0, d_hi */
> +}
> +
> +static inline void emit_to_le32(struct jit_ctx *ctx, s8 d_lo, s8 d_hi)
> +{
> + if (CPU_IS_COLDFIRE) {
> + emit_16(ctx, 0x2f00 | d_lo); /* move.l d_lo, -(%sp) */
> + emit_16(ctx, 0x4840 | d_lo); /* swap d_lo */
> + emit_16(ctx, 0x71c0 | (d_lo << 9) | d_lo); /* mvz.w d_lo, d_lo */
> + emit_16(ctx, 0x7180 | (d_hi << 9) | d_lo); /* mvz.b d_lo, d_hi */
> + emit_16(ctx, 0xe088 | d_lo); /* lsr.l #8, d_lo */
> + emit_16(ctx, 0xe188 | d_hi); /* lsl.l #8, d_hi */
> + emit_16(ctx, 0x8080 | (d_hi << 9) | d_lo); /* or.l d_lo, d_hi */
> + emit_16(ctx, 0x2017 | (d_lo << 9)); /* move.l (%sp), d_lo */
> + emit_16(ctx, 0x2e80 | d_hi); /* move.l d_hi, (%sp) */
> + emit_16(ctx, 0x71c0 | (d_lo << 9) | d_lo); /* mvz.w d_lo, d_lo */
> + emit_16(ctx, 0x7180 | (d_hi << 9) | d_lo); /* mvz.b d_lo, d_hi */
> + emit_16(ctx, 0xe088 | d_lo); /* lsr.l #8, d_lo */
> + emit_16(ctx, 0xe188 | d_hi); /* lsl.l #8, d_hi */
> + emit_16(ctx, 0x8080 | (d_lo << 9) | d_hi); /* or.l d_hi, d_lo */
> + emit_16(ctx, 0x4840 | d_lo); /* swap d_lo */
> + emit_16(ctx, 0x809f | (d_lo << 9)); /* or.l (%sp)+, d_lo */
> + } else {
> + emit_16(ctx, 0xe058 | d_lo); /* ror.w #8, d_lo */
> + emit_16(ctx, 0x4840 | d_lo); /* swap d_lo */
> + emit_16(ctx, 0xe058 | d_lo); /* ror.w #8, d_lo */
> + }
> +
> + emit_16(ctx, 0x7000 | (d_hi << 9)); /* moveq #0, d_hi */
> +}
> +
> +static inline void emit_to_le64(struct jit_ctx *ctx, s8 d_lo, s8 d_hi)
> +{
> + if (CPU_IS_COLDFIRE) {
> + emit_16(ctx, 0x2f00 | d_lo); /* move.l d_lo, -(%sp) */
> + emit_16(ctx, 0x2f00 | d_hi); /* move.l d_hi, -(%sp) */
> +
> + emit_16(ctx, 0x4840 | d_lo); /* swap d_lo */
> + emit_16(ctx, 0x71c0 | (d_lo << 9) | d_lo); /* mvz.w d_lo, d_lo */
> + emit_16(ctx, 0x7180 | (d_hi << 9) | d_lo); /* mvz.b d_lo, d_hi */
> + emit_16(ctx, 0xe088 | d_lo); /* lsr.l #8, d_lo */
> + emit_16(ctx, 0xe188 | d_hi); /* lsl.l #8, d_hi */
> + emit_16(ctx, 0x8080 | (d_hi << 9) | d_lo); /* or.l d_lo, d_hi */
> + emit_16(ctx, 0x202f | (d_lo << 9)); /* move.l 4(%sp), d_lo */
> + emit_16(ctx, 0x0004);
> + emit_16(ctx, 0x2f40 | d_hi); /* move.l d_hi, 4(%sp) */
> + emit_16(ctx, 0x0004);
> + emit_16(ctx, 0x71c0 | (d_lo << 9) | d_lo); /* mvz.w d_lo, d_lo */
> + emit_16(ctx, 0x7180 | (d_hi << 9) | d_lo); /* mvz.b d_lo, d_hi */
> + emit_16(ctx, 0xe088 | d_lo); /* lsr.l #8, d_lo */
> + emit_16(ctx, 0xe188 | d_hi); /* lsl.l #8, d_hi */
> + emit_16(ctx, 0x8080 | (d_lo << 9) | d_hi); /* or.l d_hi, d_lo */
> + emit_16(ctx, 0x4840 | d_lo); /* swap d_lo */
> + emit_16(ctx, 0x81af | (d_lo << 9)); /* or.l d_lo, 4(%sp) */
> + emit_16(ctx, 0x0004);
> +
> + emit_16(ctx, 0x2017 | (d_lo << 9)); /* move.l (%sp), d_lo */
> + emit_16(ctx, 0x4840 | d_lo); /* swap d_lo */
> + emit_16(ctx, 0x71c0 | (d_lo << 9) | d_lo); /* mvz.w d_lo, d_lo */
> + emit_16(ctx, 0x7180 | (d_hi << 9) | d_lo); /* mvz.b d_lo, d_hi */
> + emit_16(ctx, 0xe088 | d_lo); /* lsr.l #8, d_lo */
> + emit_16(ctx, 0xe188 | d_hi); /* lsl.l #8, d_hi */
> + emit_16(ctx, 0x8080 | (d_hi << 9) | d_lo); /* or.l d_lo, d_hi */
> + emit_16(ctx, 0x2017 | (d_lo << 9)); /* move.l (%sp), d_lo */
> + emit_16(ctx, 0x2e80 | d_hi); /* move.l d_hi, (%sp) */
> + emit_16(ctx, 0x71c0 | (d_lo << 9) | d_lo); /* mvz.w d_lo, d_lo */
> + emit_16(ctx, 0x7180 | (d_hi << 9) | d_lo); /* mvz.b d_lo, d_hi */
> + emit_16(ctx, 0xe088 | d_lo); /* lsr.l #8, d_lo */
> + emit_16(ctx, 0xe188 | d_hi); /* lsl.l #8, d_hi */
> + emit_16(ctx, 0x8080 | (d_lo << 9) | d_hi); /* or.l d_hi, d_lo */
> + emit_16(ctx, 0x4840 | d_lo); /* swap d_lo */
> + emit_16(ctx, 0x809f | (d_lo << 9)); /* or.l (%sp)+, d_lo */
> +
> + emit_16(ctx, 0x201f | (d_hi << 9)); /* move.l (%sp)+, d_hi */
> + } else {
> + emit_16(ctx, 0xe058 | d_lo); /* ror.w #8, d_lo */
> + emit_16(ctx, 0x4840 | d_lo); /* swap d_lo */
> + emit_16(ctx, 0xe058 | d_lo); /* ror.w #8, d_lo */
> +
> + emit_16(ctx, 0xe058 | d_hi); /* ror.w #8, d_hi */
> + emit_16(ctx, 0x4840 | d_hi); /* swap d_hi */
> + emit_16(ctx, 0xe058 | d_hi); /* ror.w #8, d_hi */
> +
> + emit_16(ctx, 0xc140 | (d_hi << 9) | d_lo); /* exg d_lo, d_hi */
> + }
> +}
> +
> +static void emit_bpf_end(const struct bpf_insn *insn, struct jit_ctx *ctx)
> +{
> + const s8 *dst = bpf2m68k[insn->dst_reg];
> + const s8 *tmp1 = bpf2m68k[TMP_REG_1];
> + s8 d_lo, d_hi;
> + bool to_le = BPF_SRC(insn->code) == BPF_TO_LE;
> + int imm = insn->imm;
> +
> + d_lo = bpf_get_reg32(dst[1], tmp1[1], ctx);
> + d_hi = bpf_get_reg32(dst[0], tmp1[0], ctx);
> +
> + if (to_le) {
> + switch (imm) {
> + case 16:
> + emit_to_le16(ctx, d_lo, d_hi);
> + break;
> + case 32:
> + emit_to_le32(ctx, d_lo, d_hi);
> + break;
> + case 64:
> + emit_to_le64(ctx, d_lo, d_hi);
> + break;
> + }
> + } else {
> + switch (imm) {
> + case 16:
> + emit_16(ctx, 0x0280 | d_lo); /* andi.l #0xffff, d_lo */
> + emit_32(ctx, 0xffff);
> + emit_16(ctx, 0x7000 | (d_hi << 9)); /* moveq #0, d_hi */
> + break;
> + case 32:
> + emit_16(ctx, 0x7000 | (d_hi << 9)); /* moveq #0, d_hi */
> + break;
> + case 64:
> + break;
> + }
> + }
> +
> + bpf_put_reg32(dst[1], d_lo, ctx);
> + bpf_put_reg32(dst[0], d_hi, ctx);
> +}
> +
> +static void emit_math_call(const struct bpf_insn *insn, struct jit_ctx *ctx, bool is_64)
> +{
> + const s8 *dst = bpf2m68k[insn->dst_reg];
> + const s8 *src = bpf2m68k[insn->src_reg];
> + const s8 *tmp1 = bpf2m68k[TMP_REG_1];
> + const s8 *tmp2 = bpf2m68k[TMP_REG_2];
> + s8 d_lo, d_hi, s_lo, s_hi = 0;
> + void *func = NULL;
> + u8 op = BPF_OP(insn->code);
> +
> + d_lo = bpf_get_reg32(dst[1], tmp1[1], ctx);
> + d_hi = bpf_get_reg32(dst[0], tmp1[0], ctx);
> +
> + if (BPF_SRC(insn->code) == BPF_X) {
> + s_lo = bpf_get_reg32(src[1], tmp2[1], ctx);
> + if (is_64)
> + s_hi = bpf_get_reg32(src[0], tmp2[0], ctx);
> + } else {
> + s_lo = tmp2[1];
> + emit_16(ctx, 0x203c | (s_lo << 9)); /* move.l #imm, s_lo */
> + emit_32(ctx, insn->imm);
> + if (is_64) {
> + s_hi = tmp2[0];
> + emit_16(ctx, 0x203c | (s_hi << 9)); /* move.l #imm, s_hi */
> + emit_32(ctx, insn->imm < 0 ? 0xffffffff : 0);
> + }
> + }
> +
> + if (is_64) {
> + switch (op) {
> + case BPF_MUL:
> + func = (void *)jit_mul64;
> + break;
> + case BPF_DIV:
> + func = (void *)jit_div64;
> + break;
> + case BPF_MOD:
> + func = (void *)jit_mod64;
> + break;
> + }
> + if (BPF_CLASS(insn->code) == BPF_ALU64 && op == BPF_DIV && insn->off == 1)
> + func = (void *)jit_sdiv64;
> + if (BPF_CLASS(insn->code) == BPF_ALU64 && op == BPF_MOD && insn->off == 1)
> + func = (void *)jit_smod64;
> + } else {
> + switch (op) {
> + case BPF_MUL:
> + func = (void *)jit_mul32;
> + break;
> + case BPF_DIV:
> + func = (void *)jit_div32;
> + break;
> + case BPF_MOD:
> + func = (void *)jit_mod32;
> + break;
> + }
> + if (BPF_CLASS(insn->code) == BPF_ALU && op == BPF_DIV && insn->off == 1)
> + func = (void *)jit_sdiv32;
> + if (BPF_CLASS(insn->code) == BPF_ALU && op == BPF_MOD && insn->off == 1)
> + func = (void *)jit_smod32;
> + }
> +
> + if (is_64) {
> + emit_16(ctx, 0x2f01); /* move.l d1, -(%sp) */
> + emit_16(ctx, 0x2f00); /* move.l d0, -(%sp) */
> +
> + emit_16(ctx, 0x2f00 | s_lo); /* move.l s_lo, -(%sp) */
> + emit_16(ctx, 0x2f00 | s_hi); /* move.l s_hi, -(%sp) */
> + emit_16(ctx, 0x2f00 | d_lo); /* move.l d_lo, -(%sp) */
> + emit_16(ctx, 0x2f00 | d_hi); /* move.l d_hi, -(%sp) */
> +
> + emit_16(ctx, 0x207c); /* movea.l #func, %a0 */
> + emit_32(ctx, (u32)func);
> + emit_16(ctx, 0x4e90); /* jsr (%a0) */
> +
> + emit_16(ctx, 0x4fef); /* lea 16(%sp), %sp */
> + emit_16(ctx, 16);
> +
> + emit_16(ctx, 0x2601); /* move.l %d1, %d3 */
> + emit_16(ctx, 0x2400); /* move.l %d0, %d2 */
> +
> + emit_16(ctx, 0x201f); /* move.l (%sp)+, d0 */
> + emit_16(ctx, 0x221f); /* move.l (%sp)+, d1 */
> + } else {
> + emit_16(ctx, 0x2f01); /* move.l d1, -(%sp) */
> + emit_16(ctx, 0x2f00); /* move.l d0, -(%sp) */
> +
> + emit_16(ctx, 0x2f00 | s_lo); /* move.l s_lo, -(%sp) */
> + emit_16(ctx, 0x2f00 | d_lo); /* move.l d_lo, -(%sp) */
> +
> + emit_16(ctx, 0x207c); /* movea.l #func, %a0 */
> + emit_32(ctx, (u32)func);
> + emit_16(ctx, 0x4e90); /* jsr (%a0) */
> +
> + emit_16(ctx, 0x4fef); /* lea 8(%sp), %sp */
> + emit_16(ctx, 8);
> +
> + emit_16(ctx, 0x2600); /* move.l %d0, %d3 */
> + emit_16(ctx, 0x7400); /* moveq #0, %d2 */
> +
> + emit_16(ctx, 0x201f); /* move.l (%sp)+, d0 */
> + emit_16(ctx, 0x221f); /* move.l (%sp)+, d1 */
> + }
> +
> + bpf_put_reg32(dst[1], M68K_D3, ctx);
> + bpf_put_reg32(dst[0], M68K_D2, ctx);
> +}
> +
> +static void emit_ldx(const struct bpf_insn *insn, struct jit_ctx *ctx)
> +{
> + const s8 *dst = bpf2m68k[insn->dst_reg];
> + const s8 *src = bpf2m68k[insn->src_reg];
> + const s8 *tmp1 = bpf2m68k[TMP_REG_1];
> + s8 d_lo, d_hi;
> + bool is_mem_sx = BPF_MODE(insn->code) == BPF_MEMSX;
> +
> + if (is_stacked(src[1])) {
> + emit_16(ctx, 0x206e); /* movea.l d16(%fp), %a0 */
> + emit_16(ctx, (u16)src[1]);
> + } else {
> + emit_16(ctx, 0x2040 | src[1]); /* movea.l src, %a0 */
> + }
> +
> + d_lo = is_stacked(dst[1]) ? tmp1[1] : dst[1];
> + d_hi = is_stacked(dst[0]) ? tmp1[0] : dst[0];
> +
> + switch (BPF_SIZE(insn->code)) {
> + case BPF_B:
> + if (!is_mem_sx)
> + emit_16(ctx, 0x7000 | (d_lo << 9)); /* moveq #0, d_lo */
> + emit_16(ctx, 0x1028 | (d_lo << 9)); /* move.b d16(%a0), d_lo */
> + emit_16(ctx, insn->off);
> + if (is_mem_sx)
> + emit_16(ctx, 0x49c0 | d_lo); /* extb.l d_lo */
> + break;
> + case BPF_H:
> + if (!is_mem_sx)
> + emit_16(ctx, 0x7000 | (d_lo << 9)); /* moveq #0, d_lo */
> + emit_16(ctx, 0x3028 | (d_lo << 9)); /* move.w d16(%a0), d_lo */
> + emit_16(ctx, insn->off);
> + if (is_mem_sx)
> + emit_16(ctx, 0x48c0 | d_lo); /* ext.l d_lo */
> + break;
> + case BPF_W:
> + emit_16(ctx, 0x2028 | (d_lo << 9)); /* move.l d16(%a0), d_lo */
> + emit_16(ctx, insn->off);
> + break;
> + case BPF_DW:
> + emit_16(ctx, 0x2028 | (d_hi << 9)); /* move.l d16(%a0), d_hi */
> + emit_16(ctx, insn->off);
> + emit_16(ctx, 0x41e8); /* lea 4(%a0), %a0 */
> + emit_16(ctx, 4);
> + emit_16(ctx, 0x2028 | (d_lo << 9)); /* move.l d16(%a0), d_lo */
> + emit_16(ctx, insn->off);
> + break;
> + }
> +
> + if (BPF_SIZE(insn->code) != BPF_DW) {
> + if (is_mem_sx) {
> + emit_16(ctx, 0x4a80 | d_lo); /* tst.l d_lo */
> + emit_16(ctx, 0x5bc0 | d_hi); /* smi d_hi */
> + emit_16(ctx, 0x49c0 | d_hi); /* extb.l d_hi */
> + } else {
> + emit_16(ctx, 0x7000 | (d_hi << 9)); /* moveq #0, d_hi */
> + }
> + }
> +
> + bpf_put_reg32(dst[1], d_lo, ctx);
> + bpf_put_reg32(dst[0], d_hi, ctx);
> +}
> +
> +static void emit_stx(const struct bpf_insn *insn, struct jit_ctx *ctx)
> +{
> + const s8 *dst = bpf2m68k[insn->dst_reg];
> + const s8 *src = bpf2m68k[insn->src_reg];
> + const s8 *tmp2 = bpf2m68k[TMP_REG_2];
> + s8 s_lo, s_hi;
> +
> + if (is_stacked(dst[1])) {
> + emit_16(ctx, 0x206e); /* movea.l d16(%fp), %a0 */
> + emit_16(ctx, (u16)dst[1]);
> + } else {
> + emit_16(ctx, 0x2040 | dst[1]); /* movea.l dst, %a0 */
> + }
> +
> + s_lo = bpf_get_reg32(src[1], tmp2[1], ctx);
> +
> + switch (BPF_SIZE(insn->code)) {
> + case BPF_B:
> + emit_16(ctx, 0x1140 | s_lo); /* move.b s_lo, d16(%a0) */
> + emit_16(ctx, insn->off);
> + break;
> + case BPF_H:
> + emit_16(ctx, 0x3140 | s_lo); /* move.w s_lo, d16(%a0) */
> + emit_16(ctx, insn->off);
> + break;
> + case BPF_W:
> + emit_16(ctx, 0x2140 | s_lo); /* move.l s_lo, d16(%a0) */
> + emit_16(ctx, insn->off);
> + break;
> + case BPF_DW:
> + s_hi = bpf_get_reg32(src[0], tmp2[0], ctx);
> + emit_16(ctx, 0x2140 | s_hi); /* move.l s_hi, d16(%a0) */
> + emit_16(ctx, insn->off);
> + emit_16(ctx, 0x41e8); /* lea 4(%a0), %a0 */
> + emit_16(ctx, 4);
> + emit_16(ctx, 0x2140 | s_lo); /* move.l s_lo, d16(%a0) */
> + emit_16(ctx, insn->off);
> + break;
> + }
> +}
> +
> +static void emit_st(const struct bpf_insn *insn, struct jit_ctx *ctx)
> +{
> + const s8 *dst = bpf2m68k[insn->dst_reg];
> + const s8 *tmp = bpf2m68k[TMP_REG_1];
> +
> + if (is_stacked(dst[1])) {
> + emit_16(ctx, 0x206e); /* movea.l d16(%fp), %a0 */
> + emit_16(ctx, (u16)dst[1]);
> + } else {
> + emit_16(ctx, 0x2040 | dst[1]); /* movea.l dst, %a0 */
> + }
> +
> + emit_16(ctx, 0x203c | (tmp[1] << 9)); /* move.l #imm, tmp */
> + emit_32(ctx, insn->imm);
> +
> + switch (BPF_SIZE(insn->code)) {
> + case BPF_B:
> + emit_16(ctx, 0x1140 | tmp[1]); /* move.b tmp, d16(%a0) */
> + emit_16(ctx, insn->off);
> + break;
> + case BPF_H:
> + emit_16(ctx, 0x3140 | tmp[1]); /* move.w tmp, d16(%a0) */
> + emit_16(ctx, insn->off);
> + break;
> + case BPF_W:
> + emit_16(ctx, 0x2140 | tmp[1]); /* move.l tmp, d16(%a0) */
> + emit_16(ctx, insn->off);
> + break;
> + case BPF_DW:
> + emit_16(ctx, 0x203c | (tmp[0] << 9)); /* move.l #imm, tmp */
> + emit_32(ctx, insn->imm < 0 ? 0xffffffff : 0);
> +
> + emit_16(ctx, 0x2140 | tmp[0]); /* move.l tmp, d16(%a0) */
> + emit_16(ctx, insn->off);
> + emit_16(ctx, 0x41e8); /* lea 4(%a0), %a0 */
> + emit_16(ctx, 4);
> + emit_16(ctx, 0x2140 | tmp[1]); /* move.l tmp, d16(%a0) */
> + emit_16(ctx, insn->off);
> + break;
> + }
> +}
> +
> +static void emit_atomic(const struct bpf_insn *insn, struct jit_ctx *ctx)
> +{
> + const s8 *dst = bpf2m68k[insn->dst_reg];
> + const s8 *src = bpf2m68k[insn->src_reg];
> + const s8 *tmp2 = bpf2m68k[TMP_REG_2];
> +
> + if (is_stacked(dst[1])) {
> + emit_16(ctx, 0x206e); /* movea.l d16(%fp), %a0 */
> + emit_16(ctx, (u16)dst[1]);
> + } else {
> + emit_16(ctx, 0x2040 | dst[1]); /* movea.l dst, %a0 */
> + }
> +
> + if (insn->off) {
> + emit_16(ctx, 0x41e8); /* lea d16(%a0), %a0 */
> + emit_16(ctx, insn->off);
> + }
> +
> + if (BPF_SIZE(insn->code) == BPF_W) {
> + s8 s_lo = bpf_get_reg32(src[1], tmp2[1], ctx);
> +
> + if (insn->imm != BPF_CMPXCHG) {
> + emit_16(ctx, 0x2f01); /* move.l d1, -(%sp) */
> + emit_16(ctx, 0x2f00); /* move.l d0, -(%sp) */
> + }
> +
> + emit_16(ctx, 0x2f00 | M68K_D0); /* move.l %d0, -(%sp) */
> + emit_16(ctx, 0x2f3c); /* move.l #imm, -(%sp) */
> + emit_32(ctx, insn->imm);
> + emit_16(ctx, 0x2f00 | s_lo); /* move.l s_lo, -(%sp) */
> + emit_16(ctx, 0x2f08); /* move.l %a0, -(%sp) */
> +
> + emit_16(ctx, 0x207c); /* movea.l #func, %a0 */
> + emit_32(ctx, (u32)jit_atomic32);
> + emit_16(ctx, 0x4e90); /* jsr (%a0) */
> +
> + emit_16(ctx, 0x4fef); /* lea 16(%sp), %sp */
> + emit_16(ctx, 16);
> +
> + if (insn->imm == BPF_CMPXCHG) {
> + emit_16(ctx, 0x7200); /* moveq #0, %d1 */
> + } else {
> + bool is_fetch = (insn->imm & BPF_FETCH) || insn->imm == BPF_XCHG;
> +
> + if (is_fetch)
> + emit_16(ctx, 0x2600); /* move.l %d0, %d3 */
> +
> + emit_16(ctx, 0x201f); /* move.l (%sp)+, d0 */
> + emit_16(ctx, 0x221f); /* move.l (%sp)+, d1 */
> +
> + if (is_fetch) {
> + bpf_put_reg32(src[1], M68K_D3, ctx);
> + emit_16(ctx, 0x7400); /* moveq #0, %d2 */
> + bpf_put_reg32(src[0], M68K_D2, ctx);
> + }
> + }
> +
> + } else if (BPF_SIZE(insn->code) == BPF_DW) {
> + s8 s_lo = bpf_get_reg32(src[1], tmp2[1], ctx);
> + s8 s_hi = bpf_get_reg32(src[0], tmp2[0], ctx);
> +
> + if (insn->imm != BPF_CMPXCHG) {
> + emit_16(ctx, 0x2f01); /* move.l d1, -(%sp) */
> + emit_16(ctx, 0x2f00); /* move.l d0, -(%sp) */
> + }
> +
> + emit_16(ctx, 0x2f00 | M68K_D0); /* move.l %d0, -(%sp) */
> + emit_16(ctx, 0x2f00 | M68K_D1); /* move.l %d1, -(%sp) */
> + emit_16(ctx, 0x2f3c); /* move.l #imm, -(%sp) */
> + emit_32(ctx, insn->imm);
> + emit_16(ctx, 0x2f00 | s_lo); /* move.l s_lo, -(%sp) */
> + emit_16(ctx, 0x2f00 | s_hi); /* move.l s_hi, -(%sp) */
> + emit_16(ctx, 0x2f08); /* move.l %a0, -(%sp) */
> +
> + emit_16(ctx, 0x207c); /* movea.l #func, %a0 */
> + emit_32(ctx, (u32)jit_atomic64);
> + emit_16(ctx, 0x4e90); /* jsr (%a0) */
> +
> + emit_16(ctx, 0x4fef); /* lea 24(%sp), %sp */
> + emit_16(ctx, 24);
> +
> + if (insn->imm == BPF_CMPXCHG) {
> + if (CPU_IS_COLDFIRE) {
> + emit_16(ctx, 0x2f00); /* move.l %d0, -(%sp) */
> + emit_16(ctx, 0x2001); /* move.l %d1, %d0 */
> + emit_16(ctx, 0x221f); /* move.l (%sp)+, %d1 */
> + } else {
> + emit_16(ctx, 0xc141); /* exg %d0, %d1 */
> + }
> + } else {
> + bool is_fetch = (insn->imm & BPF_FETCH) || insn->imm == BPF_XCHG;
> +
> + if (is_fetch) {
> + emit_16(ctx, 0x2600); /* move.l %d0, %d3 */
> + emit_16(ctx, 0x2401); /* move.l %d1, %d2 */
> + }
> +
> + emit_16(ctx, 0x201f); /* move.l (%sp)+, d0 */
> + emit_16(ctx, 0x221f); /* move.l (%sp)+, d1 */
> +
> + if (is_fetch) {
> + bpf_put_reg32(src[1], M68K_D2, ctx);
> + bpf_put_reg32(src[0], M68K_D3, ctx);
> + }
> + }
> + }
> +}
> +
> +static void emit_tail_call(const struct bpf_insn *insn, struct jit_ctx *ctx)
> +{
> + const s8 *r2 = bpf2m68k[BPF_REG_2];
> + const s8 *r3 = bpf2m68k[BPF_REG_3];
> + int jmp_out_1, jmp_out_2, jmp_out_3;
> +
> + if (is_stacked(r2[1])) {
> + emit_16(ctx, 0x206e); /* movea.l d16(%fp), %a0 */
> + emit_16(ctx, (u16)r2[1]);
> + } else {
> + emit_16(ctx, 0x2040 | r2[1]); /* movea.l r2, %a0 */
> + }
> +
> + if (is_stacked(r3[1])) {
> + emit_16(ctx, 0x242e); /* move.l d16(%fp), %d2 */
> + emit_16(ctx, (u16)r3[1]);
> + } else {
> + emit_16(ctx, 0x2400 | r3[1]); /* move.l r3, %d2 */
> + }
> +
> + emit_16(ctx, 0xb4a8); /* cmp.l d16(%a0), %d2 */
> + emit_16(ctx, offsetof(struct bpf_array, map.max_entries));
> + emit_16(ctx, 0x6400); /* bcc.w jmp_out_1 */
> + jmp_out_1 = ctx->idx;
> + emit_16(ctx, 0);
> +
> + emit_16(ctx, 0x262e); /* move.l d16(%fp), %d3 */
> + emit_16(ctx, (u16)STACK_OFFSET(BPF_TC_LO));
> + emit_16(ctx, 0x4a83); /* tst.l %d3 */
> + emit_16(ctx, 0x6700); /* beq.w jmp_out_2 */
> + jmp_out_2 = ctx->idx;
> + emit_16(ctx, 0);
> +
> + emit_16(ctx, 0x5383); /* subq.l #1, %d3 */
> + emit_16(ctx, 0x2d43); /* move.l %d3, d16(%fp) */
> + emit_16(ctx, (u16)STACK_OFFSET(BPF_TC_LO));
> +
> + emit_16(ctx, 0xd482); /* add.l %d2, %d2 */
> + emit_16(ctx, 0xd482); /* add.l %d2, %d2 */
> + emit_16(ctx, 0x43e8); /* lea d16(%a0), %a1 */
> + emit_16(ctx, offsetof(struct bpf_array, ptrs));
> + emit_16(ctx, 0xd3c2); /* adda.l %d2, %a1 */
> +
> + emit_16(ctx, 0x2051); /* movea.l (%a1), %a0 */
> +
> + emit_16(ctx, 0x2408); /* move.l %a0, %d2 */
> + emit_16(ctx, 0x4a82); /* tst.l %d2 */
> + emit_16(ctx, 0x6700); /* beq.w jmp_out_3 */
> + jmp_out_3 = ctx->idx;
> + emit_16(ctx, 0);
> +
> + emit_16(ctx, 0x2068); /* movea.l d16(%a0), %a0 */
> + emit_16(ctx, offsetof(struct bpf_prog, bpf_func));
> +
> + emit_16(ctx, 0x4ee8); /* jmp d16(%a0) */
> + emit_16(ctx, 44);
> +
> + if (ctx->target) {
> + ctx->target[jmp_out_1] = (ctx->idx - jmp_out_1) * 2;
> + ctx->target[jmp_out_2] = (ctx->idx - jmp_out_2) * 2;
> + ctx->target[jmp_out_3] = (ctx->idx - jmp_out_3) * 2;
> + }
> +}
> +
> +static int emit_jmp(const struct bpf_insn *insn, struct jit_ctx *ctx, int curr_i)
> +{
> + const s8 *dst = bpf2m68k[insn->dst_reg];
> + const s8 *src = bpf2m68k[insn->src_reg];
> + const s8 *tmp1 = bpf2m68k[TMP_REG_1];
> + const s8 *tmp2 = bpf2m68k[TMP_REG_2];
> + s8 d_lo, d_hi = 0, s_lo, s_hi = 0;
> + bool is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32;
> + u8 op = BPF_OP(insn->code);
> + int target_insn;
> +
> + if (is_jmp32 && op == BPF_JA)
> + target_insn = curr_i + 1 + insn->imm;
> + else
> + target_insn = curr_i + 1 + insn->off;
> +
> + if (target_insn < 0 || target_insn > ctx->prog->len)
> + return -EOPNOTSUPP;
> +
> + #define EMIT_JMP32(op_base) \
> + do { \
> + int disp = (ctx->offsets[target_insn] - (ctx->idx + 1)) * 2; \
> + emit_16(ctx, (op_base) | 0x00ff); \
> + emit_32(ctx, disp); \
> + } while (0)
> +
> + if (op == BPF_JA) {
> + EMIT_JMP32(0x6000); /* bra.l */
> + return 0;
> + }
> +
> + d_lo = bpf_get_reg32(dst[1], tmp1[1], ctx);
> + if (!is_jmp32)
> + d_hi = bpf_get_reg32(dst[0], tmp1[0], ctx);
> +
> + if (BPF_SRC(insn->code) == BPF_X) {
> + s_lo = bpf_get_reg32(src[1], tmp2[1], ctx);
> + if (!is_jmp32)
> + s_hi = bpf_get_reg32(src[0], tmp2[0], ctx);
> + } else {
> + s_lo = tmp2[1];
> + emit_16(ctx, 0x203c | (s_lo << 9)); /* move.l #imm, s_lo */
> + emit_32(ctx, insn->imm);
> + if (!is_jmp32) {
> + s_hi = tmp2[0];
> + emit_16(ctx, 0x203c | (s_hi << 9)); /* move.l #imm, s_hi */
> + emit_32(ctx, insn->imm < 0 ? 0xffffffff : 0);
> + }
> + }
> +
> + if (op == BPF_JSET) {
> + emit_16(ctx, 0x2000 | (tmp1[1] << 9) | d_lo); /* move.l d_lo, tmp1_lo */
> + emit_16(ctx, 0xc080 | (tmp1[1] << 9) | s_lo); /* and.l s_lo, tmp1_lo */
> +
> + if (!is_jmp32) {
> + EMIT_JMP32(0x6600); /* bne.l */
> + emit_16(ctx, 0x2000 | (tmp1[0] << 9) | d_hi); /* move.l d_hi, tmp1_hi */
> + emit_16(ctx, 0xc080 | (tmp1[0] << 9) | s_hi); /* and.l s_hi, tmp1_hi */
> + }
> + EMIT_JMP32(0x6600); /* bne.l */
> + return 0;
> + }
> +
> + if (is_jmp32) {
> + emit_16(ctx, 0xb080 | (d_lo << 9) | s_lo); /* cmp.l s_lo, d_lo */
> + switch (op) {
> + case BPF_JEQ:
> + EMIT_JMP32(0x6700); /* beq.l */
> + break;
> + case BPF_JNE:
> + EMIT_JMP32(0x6600); /* bne.l */
> + break;
> + case BPF_JGT:
> + EMIT_JMP32(0x6200); /* bhi.l */
> + break;
> + case BPF_JGE:
> + EMIT_JMP32(0x6400); /* bcc.l */
> + break;
> + case BPF_JLT:
> + EMIT_JMP32(0x6500); /* bcs.l */
> + break;
> + case BPF_JLE:
> + EMIT_JMP32(0x6300); /* bls.l */
> + break;
> + case BPF_JSGT:
> + EMIT_JMP32(0x6e00); /* bgt.l */
> + break;
> + case BPF_JSGE:
> + EMIT_JMP32(0x6c00); /* bge.l */
> + break;
> + case BPF_JSLT:
> + EMIT_JMP32(0x6d00); /* blt.l */
> + break;
> + case BPF_JSLE:
> + EMIT_JMP32(0x6f00); /* ble.l */
> + break;
> + }
> + } else {
> + emit_16(ctx, 0xb080 | (d_hi << 9) | s_hi); /* cmp.l s_hi, d_hi */
> +
> + switch (op) {
> + case BPF_JEQ:
> + emit_16(ctx, 0x6608); /* bne.s .+8 */
> + emit_16(ctx, 0xb080 | (d_lo << 9) | s_lo); /* cmp.l s_lo, d_lo */
> + EMIT_JMP32(0x6700); /* beq.l */
> + break;
> + case BPF_JNE:
> + EMIT_JMP32(0x6600); /* bne.l */
> + emit_16(ctx, 0xb080 | (d_lo << 9) | s_lo); /* cmp.l s_lo, d_lo */
> + EMIT_JMP32(0x6600); /* bne.l */
> + break;
> + case BPF_JGT:
> + EMIT_JMP32(0x6200); /* bhi.l */
> + emit_16(ctx, 0x6508); /* bcs.s .+8 */
> + emit_16(ctx, 0xb080 | (d_lo << 9) | s_lo); /* cmp.l s_lo, d_lo */
> + EMIT_JMP32(0x6200); /* bhi.l */
> + break;
> + case BPF_JGE:
> + EMIT_JMP32(0x6200); /* bhi.l */
> + emit_16(ctx, 0x6508); /* bcs.s .+8 */
> + emit_16(ctx, 0xb080 | (d_lo << 9) | s_lo); /* cmp.l s_lo, d_lo */
> + EMIT_JMP32(0x6400); /* bcc.l */
> + break;
> + case BPF_JLT:
> + EMIT_JMP32(0x6500); /* bcs.l */
> + emit_16(ctx, 0x6208); /* bhi.s .+8 */
> + emit_16(ctx, 0xb080 | (d_lo << 9) | s_lo); /* cmp.l s_lo, d_lo */
> + EMIT_JMP32(0x6500); /* bcs.l */
> + break;
> + case BPF_JLE:
> + EMIT_JMP32(0x6500); /* bcs.l */
> + emit_16(ctx, 0x6208); /* bhi.s .+8 */
> + emit_16(ctx, 0xb080 | (d_lo << 9) | s_lo); /* cmp.l s_lo, d_lo */
> + EMIT_JMP32(0x6300); /* bls.l */
> + break;
> + case BPF_JSGT:
> + EMIT_JMP32(0x6e00); /* bgt.l */
> + emit_16(ctx, 0x6d08); /* blt.s .+8 */
> + emit_16(ctx, 0xb080 | (d_lo << 9) | s_lo); /* cmp.l s_lo, d_lo */
> + EMIT_JMP32(0x6200); /* bhi.l */
> + break;
> + case BPF_JSGE:
> + EMIT_JMP32(0x6e00); /* bgt.l */
> + emit_16(ctx, 0x6d08); /* blt.s .+8 */
> + emit_16(ctx, 0xb080 | (d_lo << 9) | s_lo); /* cmp.l s_lo, d_lo */
> + EMIT_JMP32(0x6400); /* bcc.l */
> + break;
> + case BPF_JSLT:
> + EMIT_JMP32(0x6d00); /* blt.l */
> + emit_16(ctx, 0x6e08); /* bgt.s .+8 */
> + emit_16(ctx, 0xb080 | (d_lo << 9) | s_lo); /* cmp.l s_lo, d_lo */
> + EMIT_JMP32(0x6500); /* bcs.l */
> + break;
> + case BPF_JSLE:
> + EMIT_JMP32(0x6d00); /* blt.l */
> + emit_16(ctx, 0x6e08); /* bgt.s .+8 */
> + emit_16(ctx, 0xb080 | (d_lo << 9) | s_lo); /* cmp.l s_lo, d_lo */
> + EMIT_JMP32(0x6300); /* bls.l */
> + break;
> + }
> + }
> + #undef EMIT_JMP32
> + return 0;
> +}
> +
> +static int emit_call(const struct bpf_insn *insn, struct jit_ctx *ctx)
> +{
> + const s8 arg_regs[] = { BPF_REG_5, BPF_REG_4, BPF_REG_3, BPF_REG_2, BPF_REG_1 };
> + const s8 *tmp1 = bpf2m68k[TMP_REG_1];
> + bool extra_pass = ctx->prog->jited;
> + u64 func_addr;
> + bool fixed;
> + int i, err;
> +
> + err = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, &func_addr, &fixed);
> + if (err)
> + return err;
> +
> + for (i = 0; i < 5; i++) {
> + const s8 *reg = bpf2m68k[arg_regs[i]];
> + s8 d_lo = bpf_get_reg32(reg[1], tmp1[1], ctx);
> + s8 d_hi = bpf_get_reg32(reg[0], tmp1[0], ctx);
> +
> + emit_16(ctx, 0x2f00 | d_lo); /* move.l d_lo, -(%sp) */
> + emit_16(ctx, 0x2f00 | d_hi); /* move.l d_hi, -(%sp) */
> + }
> +
> + emit_16(ctx, 0x207c); /* movea.l #func_addr, %a0 */
> + emit_32(ctx, (u32)func_addr);
> +
> + emit_16(ctx, 0x4e90); /* jsr (%a0) */
> +
> + emit_16(ctx, 0x4fef); /* lea 40(%sp), %sp */
> + emit_16(ctx, 40);
> +
> + if (insn->src_reg != BPF_PSEUDO_CALL) {
> + if (CPU_IS_COLDFIRE) {
> + emit_16(ctx, 0x2f00); /* move.l %d0, -(%sp) */
> + emit_16(ctx, 0x2001); /* move.l %d1, %d0 */
> + emit_16(ctx, 0x221f); /* move.l (%sp)+, %d1 */
> + } else {
> + emit_16(ctx, 0xc340); /* exg %d0, %d1 */
> + }
> + }
> +
> + return 0;
> +}
> +
> +static void build_prologue(struct jit_ctx *ctx)
> +{
> + int bpf_stack = MAX_BPF_STACK;
> + int total_stack = SCRATCH_SIZE + bpf_stack;
> + int i;
> +
> + emit_16(ctx, 0x7021); /* moveq #33, %d0 */
> +
> + emit_16(ctx, 0x4e56); /* link %a6, #-total_stack */
> + emit_16(ctx, -total_stack);
> +
> + if (CPU_IS_COLDFIRE) {
> + emit_16(ctx, 0x4fef); /* lea -16(%sp), %sp */
> + emit_16(ctx, 0xfff0);
> + emit_16(ctx, 0x48d7); /* movem.l d2-d5, (%sp) */
> + emit_16(ctx, 0x003c);
> + } else {
> + emit_16(ctx, 0x48e7); /* movem.l d2-d5, -(%sp) */
> + emit_16(ctx, 0x3c00);
> + }
> +
> + emit_16(ctx, 0x2d40); /* move.l %d0, off(%fp) */
> + emit_16(ctx, (u16)STACK_OFFSET(BPF_TC_LO));
> +
> + if (ctx->prog->is_func) {
> + const s8 arg_regs[] = { BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5 };
> +
> + for (i = 0; i < 5; i++) {
> + int offset = 8 + i * 8;
> + const s8 *reg = bpf2m68k[arg_regs[i]];
> +
> + emit_16(ctx, 0x202e); /* move.l offset(%fp), %d0 */
> + emit_16(ctx, offset);
> + bpf_put_reg32(reg[0], M68K_D0, ctx);
> +
> + emit_16(ctx, 0x202e); /* move.l offset+4(%fp), %d0 */
> + emit_16(ctx, offset + 4);
> + bpf_put_reg32(reg[1], M68K_D0, ctx);
> + }
> + } else {
> + emit_16(ctx, 0x202e); /* move.l 8(%fp), %d0 */
> + emit_16(ctx, 8);
> + bpf_put_reg32(bpf2m68k[BPF_REG_1][1], M68K_D0, ctx);
> +
> + emit_16(ctx, 0x7000 | (M68K_D0 << 9)); /* moveq #0, %d0 */
> + bpf_put_reg32(bpf2m68k[BPF_REG_1][0], M68K_D0, ctx);
> + }
> +
> + emit_16(ctx, 0x41ee); /* lea -SCRATCH_SIZE(%fp), %a0 */
> + emit_16(ctx, -SCRATCH_SIZE);
> + emit_16(ctx, 0x2008); /* move.l %a0, %d0 */
> +
> + bpf_put_reg32(bpf2m68k[BPF_REG_10][1], M68K_D0, ctx);
> + emit_16(ctx, 0x7000 | (M68K_D0 << 9)); /* moveq #0, %d0 */
> + bpf_put_reg32(bpf2m68k[BPF_REG_10][0], M68K_D0, ctx);
> +}
> +
> +static void build_epilogue(struct jit_ctx *ctx)
> +{
> + if (CPU_IS_COLDFIRE) {
> + emit_16(ctx, 0x4cd7); /* movem.l (%sp), d2-d5 */
> + emit_16(ctx, 0x003c);
> + emit_16(ctx, 0x4fef); /* lea 16(%sp), %sp */
> + emit_16(ctx, 0x0010);
> + } else {
> + emit_16(ctx, 0x4cdf); /* movem.l (%sp)+, d2-d5 */
> + emit_16(ctx, 0x003c);
> + }
> +
> + emit_16(ctx, 0x4e5e); /* unlk %fp */
> + emit_16(ctx, 0x4e75); /* rts */
> +}
> +
> +static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, int i)
> +{
> + u8 code = insn->code;
> +
> + switch (code) {
> + case BPF_ALU | BPF_NEG:
> + emit_alu32_neg(insn, ctx);
> + break;
> + case BPF_ALU64 | BPF_NEG:
> + emit_alu64_neg(insn, ctx);
> + break;
> +
> + case BPF_ALU | BPF_MOV | BPF_X:
> + case BPF_ALU | BPF_ADD | BPF_X:
> + case BPF_ALU | BPF_SUB | BPF_X:
> + case BPF_ALU | BPF_AND | BPF_X:
> + case BPF_ALU | BPF_OR | BPF_X:
> + case BPF_ALU | BPF_XOR | BPF_X:
> + case BPF_ALU | BPF_LSH | BPF_X:
> + case BPF_ALU | BPF_RSH | BPF_X:
> + case BPF_ALU | BPF_ARSH | BPF_X:
> + emit_alu32_x(insn, ctx);
> + break;
> +
> + case BPF_ALU | BPF_MOV | BPF_K:
> + case BPF_ALU | BPF_ADD | BPF_K:
> + case BPF_ALU | BPF_SUB | BPF_K:
> + case BPF_ALU | BPF_AND | BPF_K:
> + case BPF_ALU | BPF_OR | BPF_K:
> + case BPF_ALU | BPF_XOR | BPF_K:
> + case BPF_ALU | BPF_LSH | BPF_K:
> + case BPF_ALU | BPF_RSH | BPF_K:
> + case BPF_ALU | BPF_ARSH | BPF_K:
> + emit_alu32_k(insn, ctx);
> + break;
> +
> + case BPF_ALU64 | BPF_MOV | BPF_X:
> + case BPF_ALU64 | BPF_ADD | BPF_X:
> + case BPF_ALU64 | BPF_SUB | BPF_X:
> + case BPF_ALU64 | BPF_AND | BPF_X:
> + case BPF_ALU64 | BPF_OR | BPF_X:
> + case BPF_ALU64 | BPF_XOR | BPF_X:
> + emit_alu64_x(insn, ctx);
> + break;
> +
> + case BPF_ALU64 | BPF_MOV | BPF_K:
> + case BPF_ALU64 | BPF_ADD | BPF_K:
> + case BPF_ALU64 | BPF_SUB | BPF_K:
> + case BPF_ALU64 | BPF_AND | BPF_K:
> + case BPF_ALU64 | BPF_OR | BPF_K:
> + case BPF_ALU64 | BPF_XOR | BPF_K:
> + emit_alu64_k(insn, ctx);
> + break;
> +
> + case BPF_ALU64 | BPF_LSH | BPF_X:
> + case BPF_ALU64 | BPF_RSH | BPF_X:
> + case BPF_ALU64 | BPF_ARSH | BPF_X:
> + emit_alu64_shift(insn, ctx, false);
> + break;
> +
> + case BPF_ALU64 | BPF_LSH | BPF_K:
> + case BPF_ALU64 | BPF_RSH | BPF_K:
> + case BPF_ALU64 | BPF_ARSH | BPF_K:
> + emit_alu64_shift(insn, ctx, true);
> + break;
> +
> + case BPF_ALU | BPF_MUL | BPF_X:
> + case BPF_ALU | BPF_DIV | BPF_X:
> + case BPF_ALU | BPF_MOD | BPF_X:
> + case BPF_ALU | BPF_MUL | BPF_K:
> + case BPF_ALU | BPF_DIV | BPF_K:
> + case BPF_ALU | BPF_MOD | BPF_K:
> + emit_math_call(insn, ctx, false);
> + break;
> +
> + case BPF_ALU64 | BPF_MUL | BPF_X:
> + case BPF_ALU64 | BPF_DIV | BPF_X:
> + case BPF_ALU64 | BPF_MOD | BPF_X:
> + case BPF_ALU64 | BPF_MUL | BPF_K:
> + case BPF_ALU64 | BPF_DIV | BPF_K:
> + case BPF_ALU64 | BPF_MOD | BPF_K:
> + emit_math_call(insn, ctx, true);
> + break;
> +
> + case BPF_ALU | BPF_END | BPF_TO_BE:
> + case BPF_ALU | BPF_END | BPF_TO_LE:
> + case BPF_ALU64 | BPF_END | BPF_TO_BE:
> + case BPF_ALU64 | BPF_END | BPF_TO_LE:
> + emit_bpf_end(insn, ctx);
> + break;
> +
> + case BPF_LD | BPF_IMM | BPF_DW: {
> + const struct bpf_insn *insn1 = insn + 1;
> + const s8 *dst = bpf2m68k[insn->dst_reg];
> + const s8 *tmp1 = bpf2m68k[TMP_REG_1];
> + s8 d_lo = bpf_get_reg32(dst[1], tmp1[1], ctx);
> + s8 d_hi = bpf_get_reg32(dst[0], tmp1[0], ctx);
> +
> + emit_16(ctx, 0x203c | (d_lo << 9)); /* move.l #imm, d_lo */
> + emit_32(ctx, insn->imm);
> + emit_16(ctx, 0x203c | (d_hi << 9)); /* move.l #imm, d_hi */
> + emit_32(ctx, insn1->imm);
> +
> + bpf_put_reg32(dst[1], d_lo, ctx);
> + bpf_put_reg32(dst[0], d_hi, ctx);
> + break;
> + }
> +
> + case BPF_LDX | BPF_MEM | BPF_B:
> + case BPF_LDX | BPF_MEM | BPF_H:
> + case BPF_LDX | BPF_MEM | BPF_W:
> + case BPF_LDX | BPF_MEM | BPF_DW:
> + case BPF_LDX | BPF_MEMSX | BPF_B:
> + case BPF_LDX | BPF_MEMSX | BPF_H:
> + case BPF_LDX | BPF_MEMSX | BPF_W:
> + emit_ldx(insn, ctx);
> + break;
> +
> + case BPF_STX | BPF_MEM | BPF_B:
> + case BPF_STX | BPF_MEM | BPF_H:
> + case BPF_STX | BPF_MEM | BPF_W:
> + case BPF_STX | BPF_MEM | BPF_DW:
> + emit_stx(insn, ctx);
> + break;
> +
> + case BPF_ST | BPF_MEM | BPF_B:
> + case BPF_ST | BPF_MEM | BPF_H:
> + case BPF_ST | BPF_MEM | BPF_W:
> + case BPF_ST | BPF_MEM | BPF_DW:
> + emit_st(insn, ctx);
> + break;
> +
> + case BPF_STX | BPF_ATOMIC | BPF_W:
> + case BPF_STX | BPF_ATOMIC | BPF_DW:
> + emit_atomic(insn, ctx);
> + break;
> +
> + case BPF_JMP | BPF_TAIL_CALL:
> + emit_tail_call(insn, ctx);
> + break;
> +
> + case BPF_JMP | BPF_JA:
> + case BPF_JMP32 | BPF_JA:
> + case BPF_JMP | BPF_JEQ | BPF_X:
> + case BPF_JMP | BPF_JEQ | BPF_K:
> + case BPF_JMP | BPF_JNE | BPF_X:
> + case BPF_JMP | BPF_JNE | BPF_K:
> + case BPF_JMP | BPF_JGT | BPF_X:
> + case BPF_JMP | BPF_JGT | BPF_K:
> + case BPF_JMP | BPF_JGE | BPF_X:
> + case BPF_JMP | BPF_JGE | BPF_K:
> + case BPF_JMP | BPF_JLT | BPF_X:
> + case BPF_JMP | BPF_JLT | BPF_K:
> + case BPF_JMP | BPF_JLE | BPF_X:
> + case BPF_JMP | BPF_JLE | BPF_K:
> + case BPF_JMP | BPF_JSGT | BPF_X:
> + case BPF_JMP | BPF_JSGT | BPF_K:
> + case BPF_JMP | BPF_JSGE | BPF_X:
> + case BPF_JMP | BPF_JSGE | BPF_K:
> + case BPF_JMP | BPF_JSLT | BPF_X:
> + case BPF_JMP | BPF_JSLT | BPF_K:
> + case BPF_JMP | BPF_JSLE | BPF_X:
> + case BPF_JMP | BPF_JSLE | BPF_K:
> + case BPF_JMP | BPF_JSET | BPF_X:
> + case BPF_JMP | BPF_JSET | BPF_K:
> + case BPF_JMP32 | BPF_JEQ | BPF_X:
> + case BPF_JMP32 | BPF_JEQ | BPF_K:
> + case BPF_JMP32 | BPF_JNE | BPF_X:
> + case BPF_JMP32 | BPF_JNE | BPF_K:
> + case BPF_JMP32 | BPF_JGT | BPF_X:
> + case BPF_JMP32 | BPF_JGT | BPF_K:
> + case BPF_JMP32 | BPF_JGE | BPF_X:
> + case BPF_JMP32 | BPF_JGE | BPF_K:
> + case BPF_JMP32 | BPF_JLT | BPF_X:
> + case BPF_JMP32 | BPF_JLT | BPF_K:
> + case BPF_JMP32 | BPF_JLE | BPF_X:
> + case BPF_JMP32 | BPF_JLE | BPF_K:
> + case BPF_JMP32 | BPF_JSGT | BPF_X:
> + case BPF_JMP32 | BPF_JSGT | BPF_K:
> + case BPF_JMP32 | BPF_JSGE | BPF_X:
> + case BPF_JMP32 | BPF_JSGE | BPF_K:
> + case BPF_JMP32 | BPF_JSLT | BPF_X:
> + case BPF_JMP32 | BPF_JSLT | BPF_K:
> + case BPF_JMP32 | BPF_JSLE | BPF_X:
> + case BPF_JMP32 | BPF_JSLE | BPF_K:
> + case BPF_JMP32 | BPF_JSET | BPF_X:
> + case BPF_JMP32 | BPF_JSET | BPF_K:
> + return emit_jmp(insn, ctx, i);
> +
> + case BPF_JMP | BPF_CALL:
> + return emit_call(insn, ctx);
> +
> + case BPF_JMP | BPF_EXIT:
> + build_epilogue(ctx);
> + break;
> +
> + default:
> + return -EOPNOTSUPP;
> + }
> +
> + return 0;
> +}
> +
> +static int build_body(struct jit_ctx *ctx)
> +{
> + const struct bpf_prog *prog = ctx->prog;
> + int i;
> +
> + for (i = 0; i < prog->len; i++) {
> + const struct bpf_insn *insn = &prog->insnsi[i];
> + int ret;
> +
> + if (!ctx->target)
> + ctx->offsets[i] = ctx->idx;
> +
> + ret = build_insn(insn, ctx, i);
> + if (ret < 0)
> + return ret;
> +
> + if (insn->code == (BPF_LD | BPF_IMM | BPF_DW)) {
> + i++;
> + if (!ctx->target)
> + ctx->offsets[i] = ctx->idx;
> + }
> + }
> +
> + if (!ctx->target)
> + ctx->offsets[prog->len] = ctx->idx;
> +
> + return 0;
> +}
> +
> +bool bpf_jit_needs_zext(void)
> +{
> + return true;
> +}
> +
> +static void jit_fill_hole(void *area, unsigned int size)
> +{
> + u16 *ptr;
> +
> + for (ptr = area; size >= 2; size -= 2)
> + *ptr++ = 0x4afc;
> +}
> +
> +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
> +{
> + struct bpf_binary_header *header;
> + struct jit_ctx ctx;
> + unsigned int image_size;
> + u8 *image_ptr;
> + bool extra_pass = prog->jited;
> +
> + if (!prog->jit_requested)
> + return prog;
> +
> + if (prog->jited && !prog->aux->func)
> + return prog;
> +
> + memset(&ctx, 0, sizeof(ctx));
> + ctx.prog = prog;
> +
> + ctx.offsets = kvcalloc(prog->len + 1, sizeof(int), GFP_KERNEL);
> + if (!ctx.offsets)
> + return prog;
> +
> + build_prologue(&ctx);
> + if (build_body(&ctx) < 0)
> + goto out_off;
> + build_epilogue(&ctx);
> +
> + if (extra_pass) {
> + header = prog->aux->jit_data;
> + image_ptr = (u8 *)prog->bpf_func;
> + image_size = prog->jited_len;
> + } else {
> + image_size = ctx.idx * 2;
> + header = bpf_jit_binary_alloc(image_size, &image_ptr, 2, jit_fill_hole);
> + if (!header)
> + goto out_off;
> + prog->aux->jit_data = header;
> + }
> +
> + ctx.target = (u16 *)image_ptr;
> + ctx.idx = 0;
> +
> + build_prologue(&ctx);
> + if (build_body(&ctx) < 0) {
> + bpf_jit_binary_free(header);
> + prog->aux->jit_data = NULL;
> + prog->bpf_func = NULL;
> + prog->jited = 0;
> + prog->jited_len = 0;
> + goto out_off;
> + }
> + build_epilogue(&ctx);
> +
> + flush_icache_range((u32)header, (u32)(ctx.target + ctx.idx));
> +
> + if (bpf_jit_enable > 1)
> + bpf_jit_dump(prog->len, image_size, 2, ctx.target);
> +
> + if ((!prog->aux->func || extra_pass) && bpf_jit_binary_lock_ro(header)) {
> + bpf_jit_binary_free(header);
> + prog->bpf_func = NULL;
> + prog->jited = 0;
> + prog->jited_len = 0;
> + prog->aux->jit_data = NULL;
> + goto out_off;
> + }
> +
> + prog->bpf_func = (void *)ctx.target;
> + prog->jited = 1;
> + prog->jited_len = image_size;
> +
> +out_off:
> + kvfree(ctx.offsets);
> + return prog;
> +}