aboutsummaryrefslogtreecommitdiff
path: root/arch/powerpc/net/bpf_jit_comp.c
diff options
context:
space:
mode:
Diffstat (limited to 'arch/powerpc/net/bpf_jit_comp.c')
-rw-r--r--arch/powerpc/net/bpf_jit_comp.c275
1 files changed, 141 insertions, 134 deletions
diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c
index 73619d3aeb6..82e82cadcde 100644
--- a/arch/powerpc/net/bpf_jit_comp.c
+++ b/arch/powerpc/net/bpf_jit_comp.c
@@ -13,16 +13,12 @@
#include <asm/cacheflush.h>
#include <linux/netdevice.h>
#include <linux/filter.h>
-#include "bpf_jit.h"
+#include <linux/if_vlan.h>
-#ifndef __BIG_ENDIAN
-/* There are endianness assumptions herein. */
-#error "Little-endian PPC not supported in BPF compiler"
-#endif
+#include "bpf_jit.h"
int bpf_jit_enable __read_mostly;
-
static inline void bpf_flush_icache(void *start, void *end)
{
smp_wmb();
@@ -39,7 +35,7 @@ static void bpf_jit_build_prologue(struct sk_filter *fp, u32 *image,
/* Make stackframe */
if (ctx->seen & SEEN_DATAREF) {
/* If we call any helpers (for loads), save LR */
- EMIT(PPC_INST_MFLR | __PPC_RT(0));
+ EMIT(PPC_INST_MFLR | __PPC_RT(R0));
PPC_STD(0, 1, 16);
/* Back up non-volatile regs. */
@@ -56,7 +52,7 @@ static void bpf_jit_build_prologue(struct sk_filter *fp, u32 *image,
PPC_STD(i, 1, -(8*(32-i)));
}
}
- EMIT(PPC_INST_STDU | __PPC_RS(1) | __PPC_RA(1) |
+ EMIT(PPC_INST_STDU | __PPC_RS(R1) | __PPC_RA(R1) |
(-BPF_PPC_STACKFRAME & 0xfffc));
}
@@ -83,17 +79,11 @@ static void bpf_jit_build_prologue(struct sk_filter *fp, u32 *image,
}
switch (filter[0].code) {
- case BPF_S_RET_K:
- case BPF_S_LD_W_LEN:
- case BPF_S_ANC_PROTOCOL:
- case BPF_S_ANC_IFINDEX:
- case BPF_S_ANC_MARK:
- case BPF_S_ANC_RXHASH:
- case BPF_S_ANC_CPU:
- case BPF_S_ANC_QUEUE:
- case BPF_S_LD_W_ABS:
- case BPF_S_LD_H_ABS:
- case BPF_S_LD_B_ABS:
+ case BPF_RET | BPF_K:
+ case BPF_LD | BPF_W | BPF_LEN:
+ case BPF_LD | BPF_W | BPF_ABS:
+ case BPF_LD | BPF_H | BPF_ABS:
+ case BPF_LD | BPF_B | BPF_ABS:
/* first instruction sets A register (or is RET 'constant') */
break;
default:
@@ -127,6 +117,9 @@ static void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx)
PPC_BLR();
}
+#define CHOOSE_LOAD_FUNC(K, func) \
+ ((int)K < 0 ? ((int)K >= SKF_LL_OFF ? func##_negative_offset : func) : func##_positive_offset)
+
/* Assemble the body code between the prologue & epilogue. */
static int bpf_jit_build_body(struct sk_filter *fp, u32 *image,
struct codegen_context *ctx,
@@ -143,6 +136,7 @@ static int bpf_jit_build_body(struct sk_filter *fp, u32 *image,
for (i = 0; i < flen; i++) {
unsigned int K = filter[i].k;
+ u16 code = bpf_anc_helper(&filter[i]);
/*
* addrs[] maps a BPF bytecode address into a real offset from
@@ -150,35 +144,35 @@ static int bpf_jit_build_body(struct sk_filter *fp, u32 *image,
*/
addrs[i] = ctx->idx * 4;
- switch (filter[i].code) {
+ switch (code) {
/*** ALU ops ***/
- case BPF_S_ALU_ADD_X: /* A += X; */
+ case BPF_ALU | BPF_ADD | BPF_X: /* A += X; */
ctx->seen |= SEEN_XREG;
PPC_ADD(r_A, r_A, r_X);
break;
- case BPF_S_ALU_ADD_K: /* A += K; */
+ case BPF_ALU | BPF_ADD | BPF_K: /* A += K; */
if (!K)
break;
PPC_ADDI(r_A, r_A, IMM_L(K));
if (K >= 32768)
PPC_ADDIS(r_A, r_A, IMM_HA(K));
break;
- case BPF_S_ALU_SUB_X: /* A -= X; */
+ case BPF_ALU | BPF_SUB | BPF_X: /* A -= X; */
ctx->seen |= SEEN_XREG;
PPC_SUB(r_A, r_A, r_X);
break;
- case BPF_S_ALU_SUB_K: /* A -= K */
+ case BPF_ALU | BPF_SUB | BPF_K: /* A -= K */
if (!K)
break;
PPC_ADDI(r_A, r_A, IMM_L(-K));
if (K >= 32768)
PPC_ADDIS(r_A, r_A, IMM_HA(-K));
break;
- case BPF_S_ALU_MUL_X: /* A *= X; */
+ case BPF_ALU | BPF_MUL | BPF_X: /* A *= X; */
ctx->seen |= SEEN_XREG;
PPC_MUL(r_A, r_A, r_X);
break;
- case BPF_S_ALU_MUL_K: /* A *= K */
+ case BPF_ALU | BPF_MUL | BPF_K: /* A *= K */
if (K < 32768)
PPC_MULI(r_A, r_A, K);
else {
@@ -186,7 +180,27 @@ static int bpf_jit_build_body(struct sk_filter *fp, u32 *image,
PPC_MUL(r_A, r_A, r_scratch1);
}
break;
- case BPF_S_ALU_DIV_X: /* A /= X; */
+ case BPF_ALU | BPF_MOD | BPF_X: /* A %= X; */
+ ctx->seen |= SEEN_XREG;
+ PPC_CMPWI(r_X, 0);
+ if (ctx->pc_ret0 != -1) {
+ PPC_BCC(COND_EQ, addrs[ctx->pc_ret0]);
+ } else {
+ PPC_BCC_SHORT(COND_NE, (ctx->idx*4)+12);
+ PPC_LI(r_ret, 0);
+ PPC_JMP(exit_addr);
+ }
+ PPC_DIVWU(r_scratch1, r_A, r_X);
+ PPC_MUL(r_scratch1, r_X, r_scratch1);
+ PPC_SUB(r_A, r_A, r_scratch1);
+ break;
+ case BPF_ALU | BPF_MOD | BPF_K: /* A %= K; */
+ PPC_LI32(r_scratch2, K);
+ PPC_DIVWU(r_scratch1, r_A, r_scratch2);
+ PPC_MUL(r_scratch1, r_scratch2, r_scratch1);
+ PPC_SUB(r_A, r_A, r_scratch1);
+ break;
+ case BPF_ALU | BPF_DIV | BPF_X: /* A /= X; */
ctx->seen |= SEEN_XREG;
PPC_CMPWI(r_X, 0);
if (ctx->pc_ret0 != -1) {
@@ -202,16 +216,17 @@ static int bpf_jit_build_body(struct sk_filter *fp, u32 *image,
}
PPC_DIVWU(r_A, r_A, r_X);
break;
- case BPF_S_ALU_DIV_K: /* A = reciprocal_divide(A, K); */
+ case BPF_ALU | BPF_DIV | BPF_K: /* A /= K */
+ if (K == 1)
+ break;
PPC_LI32(r_scratch1, K);
- /* Top 32 bits of 64bit result -> A */
- PPC_MULHWU(r_A, r_A, r_scratch1);
+ PPC_DIVWU(r_A, r_A, r_scratch1);
break;
- case BPF_S_ALU_AND_X:
+ case BPF_ALU | BPF_AND | BPF_X:
ctx->seen |= SEEN_XREG;
PPC_AND(r_A, r_A, r_X);
break;
- case BPF_S_ALU_AND_K:
+ case BPF_ALU | BPF_AND | BPF_K:
if (!IMM_H(K))
PPC_ANDI(r_A, r_A, K);
else {
@@ -219,40 +234,51 @@ static int bpf_jit_build_body(struct sk_filter *fp, u32 *image,
PPC_AND(r_A, r_A, r_scratch1);
}
break;
- case BPF_S_ALU_OR_X:
+ case BPF_ALU | BPF_OR | BPF_X:
ctx->seen |= SEEN_XREG;
PPC_OR(r_A, r_A, r_X);
break;
- case BPF_S_ALU_OR_K:
+ case BPF_ALU | BPF_OR | BPF_K:
if (IMM_L(K))
PPC_ORI(r_A, r_A, IMM_L(K));
if (K >= 65536)
PPC_ORIS(r_A, r_A, IMM_H(K));
break;
- case BPF_S_ALU_LSH_X: /* A <<= X; */
+ case BPF_ANC | SKF_AD_ALU_XOR_X:
+ case BPF_ALU | BPF_XOR | BPF_X: /* A ^= X */
+ ctx->seen |= SEEN_XREG;
+ PPC_XOR(r_A, r_A, r_X);
+ break;
+ case BPF_ALU | BPF_XOR | BPF_K: /* A ^= K */
+ if (IMM_L(K))
+ PPC_XORI(r_A, r_A, IMM_L(K));
+ if (K >= 65536)
+ PPC_XORIS(r_A, r_A, IMM_H(K));
+ break;
+ case BPF_ALU | BPF_LSH | BPF_X: /* A <<= X; */
ctx->seen |= SEEN_XREG;
PPC_SLW(r_A, r_A, r_X);
break;
- case BPF_S_ALU_LSH_K:
+ case BPF_ALU | BPF_LSH | BPF_K:
if (K == 0)
break;
else
PPC_SLWI(r_A, r_A, K);
break;
- case BPF_S_ALU_RSH_X: /* A >>= X; */
+ case BPF_ALU | BPF_RSH | BPF_X: /* A >>= X; */
ctx->seen |= SEEN_XREG;
PPC_SRW(r_A, r_A, r_X);
break;
- case BPF_S_ALU_RSH_K: /* A >>= K; */
+ case BPF_ALU | BPF_RSH | BPF_K: /* A >>= K; */
if (K == 0)
break;
else
PPC_SRWI(r_A, r_A, K);
break;
- case BPF_S_ALU_NEG:
+ case BPF_ALU | BPF_NEG:
PPC_NEG(r_A, r_A);
break;
- case BPF_S_RET_K:
+ case BPF_RET | BPF_K:
PPC_LI32(r_ret, K);
if (!K) {
if (ctx->pc_ret0 == -1)
@@ -279,7 +305,7 @@ static int bpf_jit_build_body(struct sk_filter *fp, u32 *image,
PPC_BLR();
}
break;
- case BPF_S_RET_A:
+ case BPF_RET | BPF_A:
PPC_MR(r_ret, r_A);
if (i != flen - 1) {
if (ctx->seen)
@@ -288,60 +314,53 @@ static int bpf_jit_build_body(struct sk_filter *fp, u32 *image,
PPC_BLR();
}
break;
- case BPF_S_MISC_TAX: /* X = A */
+ case BPF_MISC | BPF_TAX: /* X = A */
PPC_MR(r_X, r_A);
break;
- case BPF_S_MISC_TXA: /* A = X */
+ case BPF_MISC | BPF_TXA: /* A = X */
ctx->seen |= SEEN_XREG;
PPC_MR(r_A, r_X);
break;
/*** Constant loads/M[] access ***/
- case BPF_S_LD_IMM: /* A = K */
+ case BPF_LD | BPF_IMM: /* A = K */
PPC_LI32(r_A, K);
break;
- case BPF_S_LDX_IMM: /* X = K */
+ case BPF_LDX | BPF_IMM: /* X = K */
PPC_LI32(r_X, K);
break;
- case BPF_S_LD_MEM: /* A = mem[K] */
+ case BPF_LD | BPF_MEM: /* A = mem[K] */
PPC_MR(r_A, r_M + (K & 0xf));
ctx->seen |= SEEN_MEM | (1<<(K & 0xf));
break;
- case BPF_S_LDX_MEM: /* X = mem[K] */
+ case BPF_LDX | BPF_MEM: /* X = mem[K] */
PPC_MR(r_X, r_M + (K & 0xf));
ctx->seen |= SEEN_MEM | (1<<(K & 0xf));
break;
- case BPF_S_ST: /* mem[K] = A */
+ case BPF_ST: /* mem[K] = A */
PPC_MR(r_M + (K & 0xf), r_A);
ctx->seen |= SEEN_MEM | (1<<(K & 0xf));
break;
- case BPF_S_STX: /* mem[K] = X */
+ case BPF_STX: /* mem[K] = X */
PPC_MR(r_M + (K & 0xf), r_X);
ctx->seen |= SEEN_XREG | SEEN_MEM | (1<<(K & 0xf));
break;
- case BPF_S_LD_W_LEN: /* A = skb->len; */
+ case BPF_LD | BPF_W | BPF_LEN: /* A = skb->len; */
BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, len) != 4);
PPC_LWZ_OFFS(r_A, r_skb, offsetof(struct sk_buff, len));
break;
- case BPF_S_LDX_W_LEN: /* X = skb->len; */
+ case BPF_LDX | BPF_W | BPF_LEN: /* X = skb->len; */
PPC_LWZ_OFFS(r_X, r_skb, offsetof(struct sk_buff, len));
break;
/*** Ancillary info loads ***/
-
- /* None of the BPF_S_ANC* codes appear to be passed by
- * sk_chk_filter(). The interpreter and the x86 BPF
- * compiler implement them so we do too -- they may be
- * planted in future.
- */
- case BPF_S_ANC_PROTOCOL: /* A = ntohs(skb->protocol); */
+ case BPF_ANC | SKF_AD_PROTOCOL: /* A = ntohs(skb->protocol); */
BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff,
protocol) != 2);
- PPC_LHZ_OFFS(r_A, r_skb, offsetof(struct sk_buff,
- protocol));
- /* ntohs is a NOP with BE loads. */
+ PPC_NTOHS_OFFS(r_A, r_skb, offsetof(struct sk_buff,
+ protocol));
break;
- case BPF_S_ANC_IFINDEX:
+ case BPF_ANC | SKF_AD_IFINDEX:
PPC_LD_OFFS(r_scratch1, r_skb, offsetof(struct sk_buff,
dev));
PPC_CMPDI(r_scratch1, 0);
@@ -358,23 +377,37 @@ static int bpf_jit_build_body(struct sk_filter *fp, u32 *image,
PPC_LWZ_OFFS(r_A, r_scratch1,
offsetof(struct net_device, ifindex));
break;
- case BPF_S_ANC_MARK:
+ case BPF_ANC | SKF_AD_MARK:
BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4);
PPC_LWZ_OFFS(r_A, r_skb, offsetof(struct sk_buff,
mark));
break;
- case BPF_S_ANC_RXHASH:
- BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, rxhash) != 4);
+ case BPF_ANC | SKF_AD_RXHASH:
+ BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, hash) != 4);
PPC_LWZ_OFFS(r_A, r_skb, offsetof(struct sk_buff,
- rxhash));
+ hash));
+ break;
+ case BPF_ANC | SKF_AD_VLAN_TAG:
+ case BPF_ANC | SKF_AD_VLAN_TAG_PRESENT:
+ BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, vlan_tci) != 2);
+ BUILD_BUG_ON(VLAN_TAG_PRESENT != 0x1000);
+
+ PPC_LHZ_OFFS(r_A, r_skb, offsetof(struct sk_buff,
+ vlan_tci));
+ if (code == (BPF_ANC | SKF_AD_VLAN_TAG)) {
+ PPC_ANDI(r_A, r_A, ~VLAN_TAG_PRESENT);
+ } else {
+ PPC_ANDI(r_A, r_A, VLAN_TAG_PRESENT);
+ PPC_SRWI(r_A, r_A, 12);
+ }
break;
- case BPF_S_ANC_QUEUE:
+ case BPF_ANC | SKF_AD_QUEUE:
BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff,
queue_mapping) != 2);
PPC_LHZ_OFFS(r_A, r_skb, offsetof(struct sk_buff,
queue_mapping));
break;
- case BPF_S_ANC_CPU:
+ case BPF_ANC | SKF_AD_CPU:
#ifdef CONFIG_SMP
/*
* PACA ptr is r13:
@@ -390,22 +423,17 @@ static int bpf_jit_build_body(struct sk_filter *fp, u32 *image,
break;
/*** Absolute loads from packet header/data ***/
- case BPF_S_LD_W_ABS:
- func = sk_load_word;
+ case BPF_LD | BPF_W | BPF_ABS:
+ func = CHOOSE_LOAD_FUNC(K, sk_load_word);
goto common_load;
- case BPF_S_LD_H_ABS:
- func = sk_load_half;
+ case BPF_LD | BPF_H | BPF_ABS:
+ func = CHOOSE_LOAD_FUNC(K, sk_load_half);
goto common_load;
- case BPF_S_LD_B_ABS:
- func = sk_load_byte;
+ case BPF_LD | BPF_B | BPF_ABS:
+ func = CHOOSE_LOAD_FUNC(K, sk_load_byte);
common_load:
- /*
- * Load from [K]. Reference with the (negative)
- * SKF_NET_OFF/SKF_LL_OFF offsets is unsupported.
- */
+ /* Load from [K]. */
ctx->seen |= SEEN_DATAREF;
- if ((int)K < 0)
- return -ENOTSUPP;
PPC_LI64(r_scratch1, func);
PPC_MTLR(r_scratch1);
PPC_LI32(r_addr, K);
@@ -418,18 +446,18 @@ static int bpf_jit_build_body(struct sk_filter *fp, u32 *image,
break;
/*** Indirect loads from packet header/data ***/
- case BPF_S_LD_W_IND:
+ case BPF_LD | BPF_W | BPF_IND:
func = sk_load_word;
goto common_load_ind;
- case BPF_S_LD_H_IND:
+ case BPF_LD | BPF_H | BPF_IND:
func = sk_load_half;
goto common_load_ind;
- case BPF_S_LD_B_IND:
+ case BPF_LD | BPF_B | BPF_IND:
func = sk_load_byte;
common_load_ind:
/*
* Load from [X + K]. Negative offsets are tested for
- * in the helper functions, and result in a 'ret 0'.
+ * in the helper functions.
*/
ctx->seen |= SEEN_DATAREF | SEEN_XREG;
PPC_LI64(r_scratch1, func);
@@ -442,37 +470,31 @@ static int bpf_jit_build_body(struct sk_filter *fp, u32 *image,
PPC_BCC(COND_LT, exit_addr);
break;
- case BPF_S_LDX_B_MSH:
- /*
- * x86 version drops packet (RET 0) when K<0, whereas
- * interpreter does allow K<0 (__load_pointer, special
- * ancillary data). common_load returns ENOTSUPP if K<0,
- * so we fall back to interpreter & filter works.
- */
- func = sk_load_byte_msh;
+ case BPF_LDX | BPF_B | BPF_MSH:
+ func = CHOOSE_LOAD_FUNC(K, sk_load_byte_msh);
goto common_load;
break;
/*** Jump and branches ***/
- case BPF_S_JMP_JA:
+ case BPF_JMP | BPF_JA:
if (K != 0)
PPC_JMP(addrs[i + 1 + K]);
break;
- case BPF_S_JMP_JGT_K:
- case BPF_S_JMP_JGT_X:
+ case BPF_JMP | BPF_JGT | BPF_K:
+ case BPF_JMP | BPF_JGT | BPF_X:
true_cond = COND_GT;
goto cond_branch;
- case BPF_S_JMP_JGE_K:
- case BPF_S_JMP_JGE_X:
+ case BPF_JMP | BPF_JGE | BPF_K:
+ case BPF_JMP | BPF_JGE | BPF_X:
true_cond = COND_GE;
goto cond_branch;
- case BPF_S_JMP_JEQ_K:
- case BPF_S_JMP_JEQ_X:
+ case BPF_JMP | BPF_JEQ | BPF_K:
+ case BPF_JMP | BPF_JEQ | BPF_X:
true_cond = COND_EQ;
goto cond_branch;
- case BPF_S_JMP_JSET_K:
- case BPF_S_JMP_JSET_X:
+ case BPF_JMP | BPF_JSET | BPF_K:
+ case BPF_JMP | BPF_JSET | BPF_X:
true_cond = COND_NE;
/* Fall through */
cond_branch:
@@ -483,20 +505,20 @@ static int bpf_jit_build_body(struct sk_filter *fp, u32 *image,
break;
}
- switch (filter[i].code) {
- case BPF_S_JMP_JGT_X:
- case BPF_S_JMP_JGE_X:
- case BPF_S_JMP_JEQ_X:
+ switch (code) {
+ case BPF_JMP | BPF_JGT | BPF_X:
+ case BPF_JMP | BPF_JGE | BPF_X:
+ case BPF_JMP | BPF_JEQ | BPF_X:
ctx->seen |= SEEN_XREG;
PPC_CMPLW(r_A, r_X);
break;
- case BPF_S_JMP_JSET_X:
+ case BPF_JMP | BPF_JSET | BPF_X:
ctx->seen |= SEEN_XREG;
PPC_AND_DOT(r_scratch1, r_A, r_X);
break;
- case BPF_S_JMP_JEQ_K:
- case BPF_S_JMP_JGT_K:
- case BPF_S_JMP_JGE_K:
+ case BPF_JMP | BPF_JEQ | BPF_K:
+ case BPF_JMP | BPF_JGT | BPF_K:
+ case BPF_JMP | BPF_JGE | BPF_K:
if (K < 32768)
PPC_CMPLWI(r_A, K);
else {
@@ -504,7 +526,7 @@ static int bpf_jit_build_body(struct sk_filter *fp, u32 *image,
PPC_CMPLW(r_A, r_scratch1);
}
break;
- case BPF_S_JMP_JSET_K:
+ case BPF_JMP | BPF_JSET | BPF_K:
if (K < 32768)
/* PPC_ANDI is /only/ dot-form */
PPC_ANDI(r_scratch1, r_A, K);
@@ -633,8 +655,7 @@ void bpf_jit_compile(struct sk_filter *fp)
proglen = cgctx.idx * 4;
alloclen = proglen + FUNCTION_DESCR_SIZE;
- image = module_alloc(max_t(unsigned int, alloclen,
- sizeof(struct work_struct)));
+ image = module_alloc(alloclen);
if (!image)
goto out;
@@ -654,41 +675,27 @@ void bpf_jit_compile(struct sk_filter *fp)
}
if (bpf_jit_enable > 1)
- pr_info("flen=%d proglen=%u pass=%d image=%p\n",
- flen, proglen, pass, image);
+ /* Note that we output the base address of the code_base
+ * rather than image, since opcodes are in code_base.
+ */
+ bpf_jit_dump(flen, proglen, pass, code_base);
if (image) {
- if (bpf_jit_enable > 1)
- print_hex_dump(KERN_ERR, "JIT code: ",
- DUMP_PREFIX_ADDRESS,
- 16, 1, code_base,
- proglen, false);
-
bpf_flush_icache(code_base, code_base + (proglen/4));
/* Function descriptor nastiness: Address + TOC */
((u64 *)image)[0] = (u64)code_base;
((u64 *)image)[1] = local_paca->kernel_toc;
fp->bpf_func = (void *)image;
+ fp->jited = 1;
}
out:
kfree(addrs);
return;
}
-static void jit_free_defer(struct work_struct *arg)
-{
- module_free(NULL, arg);
-}
-
-/* run from softirq, we must use a work_struct to call
- * module_free() from process context
- */
void bpf_jit_free(struct sk_filter *fp)
{
- if (fp->bpf_func != sk_run_filter) {
- struct work_struct *work = (struct work_struct *)fp->bpf_func;
-
- INIT_WORK(work, jit_free_defer);
- schedule_work(work);
- }
+ if (fp->jited)
+ module_free(NULL, fp->bpf_func);
+ kfree(fp);
}
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-11-09 12:08:52 +0000