diff --git a/arch/riscv/net/bpf_jit_comp32.c b/arch/riscv/net/bpf_jit_comp32.c index 592dd86fbf81..a9e0bd5cc81d 100644 --- a/arch/riscv/net/bpf_jit_comp32.c +++ b/arch/riscv/net/bpf_jit_comp32.c @@ -509,12 +509,15 @@ static void emit_alu_r64(const s8 *dst, const s8 *src, } static void emit_alu_r32(const s8 *dst, const s8 *src, - struct rv_jit_context *ctx, const u8 op) + struct rv_jit_context *ctx, + const struct bpf_insn *insn) { const s8 *tmp1 = bpf2rv32[TMP_REG_1]; const s8 *tmp2 = bpf2rv32[TMP_REG_2]; const s8 *rd = bpf_get_reg32(dst, tmp1, ctx); const s8 *rs = bpf_get_reg32(src, tmp2, ctx); + u8 op = BPF_OP(insn->code); + bool is_signed = insn->off == 1; switch (op) { case BPF_MOV: @@ -539,10 +542,12 @@ static void emit_alu_r32(const s8 *dst, const s8 *src, emit(rv_mul(lo(rd), lo(rd), lo(rs)), ctx); break; case BPF_DIV: - emit(rv_divu(lo(rd), lo(rd), lo(rs)), ctx); + emit(is_signed ? rv_div(lo(rd), lo(rd), lo(rs)) : + rv_divu(lo(rd), lo(rd), lo(rs)), ctx); break; case BPF_MOD: - emit(rv_remu(lo(rd), lo(rd), lo(rs)), ctx); + emit(is_signed ? rv_rem(lo(rd), lo(rd), lo(rs)) : + rv_remu(lo(rd), lo(rd), lo(rs)), ctx); break; case BPF_LSH: emit(rv_sll(lo(rd), lo(rd), lo(rs)), ctx); @@ -869,14 +874,58 @@ static int emit_load_r64(const s8 *dst, const s8 *src, s16 off, return 0; } -static int emit_store_r64(const s8 *dst, const s8 *src, s16 off, - struct rv_jit_context *ctx, const u8 size, - const u8 mode) +static int emit_bpf_atomic(s8 dst, const s8 *src, const s8 *rs, + struct rv_jit_context *ctx, + const struct bpf_insn *insn) +{ + s32 imm = insn->imm; + bool is_fetch = (imm & BPF_FETCH) || (imm == BPF_XCHG); + s8 fetch_reg = is_fetch ? lo(rs) : RV_REG_ZERO; + int aq = is_fetch ? 1 : 0; + int rl = is_fetch ? 1 : 0; + + switch (imm) { + case BPF_ADD: + case BPF_ADD | BPF_FETCH: + emit(rv_amoadd_w(fetch_reg, lo(rs), dst, aq, rl), ctx); + break; + case BPF_AND: + case BPF_AND | BPF_FETCH: + emit(rv_amoand_w(fetch_reg, lo(rs), dst, aq, rl), ctx); + break; + case BPF_OR: + case BPF_OR | BPF_FETCH: + emit(rv_amoor_w(fetch_reg, lo(rs), dst, aq, rl), ctx); + break; + case BPF_XOR: + case BPF_XOR | BPF_FETCH: + emit(rv_amoxor_w(fetch_reg, lo(rs), dst, aq, rl), ctx); + break; + case BPF_XCHG: + emit(rv_amoswap_w(fetch_reg, lo(rs), dst, aq, rl), ctx); + break; + default: + return -1; + } + + if (is_fetch) { + emit(rv_addi(hi(rs), RV_REG_ZERO, 0), ctx); + bpf_put_reg64(src, rs, ctx); + } + return 0; +} + +static int emit_store_r64(const s8 *dst, const s8 *src, + struct rv_jit_context *ctx, + const struct bpf_insn *insn) { const s8 *tmp1 = bpf2rv32[TMP_REG_1]; const s8 *tmp2 = bpf2rv32[TMP_REG_2]; const s8 *rd = bpf_get_reg64(dst, tmp1, ctx); const s8 *rs = bpf_get_reg64(src, tmp2, ctx); + u8 size = BPF_SIZE(insn->code); + u8 mode = BPF_MODE(insn->code); + s16 off = insn->off; if (mode == BPF_ATOMIC && size != BPF_W) return -1; @@ -896,9 +945,9 @@ static int emit_store_r64(const s8 *dst, const s8 *src, s16 off, case BPF_MEM: emit(rv_sw(RV_REG_T0, 0, lo(rs)), ctx); break; - case BPF_ATOMIC: /* Only BPF_ADD supported */ - emit(rv_amoadd_w(RV_REG_ZERO, lo(rs), RV_REG_T0, 0, 0), - ctx); + case BPF_ATOMIC: + if (emit_bpf_atomic(RV_REG_T0, src, rs, ctx, insn)) + return -1; break; } break; @@ -967,6 +1016,24 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, switch (code) { case BPF_ALU64 | BPF_MOV | BPF_X: + if (insn->off != 0) { + const s8 *rd = bpf_get_reg64(dst, tmp1, ctx); + const s8 *rs = bpf_get_reg64(src, tmp2, ctx); + + if (insn->off == 8) { + emit(rv_slli(lo(rd), lo(rs), 24), ctx); + emit(rv_srai(lo(rd), lo(rd), 24), ctx); + } else if (insn->off == 16) { + emit(rv_slli(lo(rd), lo(rs), 16), ctx); + emit(rv_srai(lo(rd), lo(rd), 16), ctx); + } else { + emit(rv_addi(lo(rd), lo(rs), 0), ctx); + } + emit(rv_srai(hi(rd), lo(rd), 31), ctx); + bpf_put_reg64(dst, rd, ctx); + break; + } + fallthrough; case BPF_ALU64 | BPF_ADD | BPF_X: case BPF_ALU64 | BPF_ADD | BPF_K: @@ -1017,6 +1084,20 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, emit_zext64(dst, ctx); break; } + if (insn->off != 0) { + const s8 *rd = bpf_get_reg32(dst, tmp1, ctx); + const s8 *rs = bpf_get_reg32(src, tmp2, ctx); + + if (insn->off == 8) { + emit(rv_slli(lo(rd), lo(rs), 24), ctx); + emit(rv_srai(lo(rd), lo(rd), 24), ctx); + } else if (insn->off == 16) { + emit(rv_slli(lo(rd), lo(rs), 16), ctx); + emit(rv_srai(lo(rd), lo(rd), 16), ctx); + } + bpf_put_reg32(dst, rd, ctx); + break; + } fallthrough; case BPF_ALU | BPF_ADD | BPF_X: @@ -1041,7 +1122,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, emit_imm32(tmp2, imm, ctx); src = tmp2; } - emit_alu_r32(dst, src, ctx, BPF_OP(code)); + emit_alu_r32(dst, src, ctx, insn); break; case BPF_ALU | BPF_MOV | BPF_K: @@ -1065,7 +1146,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, * src is ignored---choose tmp2 as a dummy register since it * is not on the stack. */ - emit_alu_r32(dst, tmp2, ctx, BPF_OP(code)); + emit_alu_r32(dst, tmp2, ctx, insn); break; case BPF_ALU | BPF_END | BPF_FROM_LE: @@ -1266,21 +1347,19 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, src = tmp2; } - if (emit_store_r64(dst, src, off, ctx, BPF_SIZE(code), - BPF_MODE(code))) + if (emit_store_r64(dst, src, ctx, insn)) return -1; break; case BPF_STX | BPF_ATOMIC | BPF_W: - if (insn->imm != BPF_ADD) { + if (insn->imm == BPF_CMPXCHG) { pr_info_once( "bpf-jit: not supported: atomic operation %02x ***\n", insn->imm); return -EFAULT; } - if (emit_store_r64(dst, src, off, ctx, BPF_SIZE(code), - BPF_MODE(code))) + if (emit_store_r64(dst, src, ctx, insn)) return -1; break;