powerpc: bpf: eliminate zero extension code-gen [Linux 5.3]

This Linux kernel change "powerpc: bpf: eliminate zero extension code-gen" is included in the Linux 5.3 release. This change is authored by Jiong Wang <jiong.wang [at] netronome.com> on Fri May 24 23:25:23 2019 +0100. The commit for this change in Linux stable tree is a4c9277 (patch).

powerpc: bpf: eliminate zero extension code-gen

Cc: Naveen N. Rao <naveen.n.rao@linux.ibm.com>
Cc: Sandipan Das <sandipan@linux.ibm.com>
Signed-off-by: Jiong Wang <jiong.wang@netronome.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

There are 36 lines of Linux source code added/deleted in this change. Code changes to Linux kernel are as follows.

 arch/powerpc/net/bpf_jit_comp64.c | 36 +++++++++++++++++++++++++++++++++---
 1 file changed, 33 insertions(+), 3 deletions(-)

diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c
index 21a1dcd..0ebd946 100644
--- a/arch/powerpc/net/bpf_jit_comp64.c
+++ b/arch/powerpc/net/bpf_jit_comp64.c
@@ -504,6 +504,9 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
        case BPF_ALU | BPF_LSH | BPF_X: /* (u32) dst <<= (u32) src */
            /* slw clears top 32 bits */
            PPC_SLW(dst_reg, dst_reg, src_reg);
+           /* skip zero extension move, but set address map. */
+           if (insn_is_zext(&insn[i + 1]))
+               addrs[++i] = ctx->idx * 4;
            break;
        case BPF_ALU64 | BPF_LSH | BPF_X: /* dst <<= src; */
            PPC_SLD(dst_reg, dst_reg, src_reg);
@@ -511,6 +514,8 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
        case BPF_ALU | BPF_LSH | BPF_K: /* (u32) dst <<== (u32) imm */
            /* with imm 0, we still need to clear top 32 bits */
            PPC_SLWI(dst_reg, dst_reg, imm);
+           if (insn_is_zext(&insn[i + 1]))
+               addrs[++i] = ctx->idx * 4;
            break;
        case BPF_ALU64 | BPF_LSH | BPF_K: /* dst <<== imm */
            if (imm != 0)
@@ -518,12 +523,16 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
            break;
        case BPF_ALU | BPF_RSH | BPF_X: /* (u32) dst >>= (u32) src */
            PPC_SRW(dst_reg, dst_reg, src_reg);
+           if (insn_is_zext(&insn[i + 1]))
+               addrs[++i] = ctx->idx * 4;
            break;
        case BPF_ALU64 | BPF_RSH | BPF_X: /* dst >>= src */
            PPC_SRD(dst_reg, dst_reg, src_reg);
            break;
        case BPF_ALU | BPF_RSH | BPF_K: /* (u32) dst >>= (u32) imm */
            PPC_SRWI(dst_reg, dst_reg, imm);
+           if (insn_is_zext(&insn[i + 1]))
+               addrs[++i] = ctx->idx * 4;
            break;
        case BPF_ALU64 | BPF_RSH | BPF_K: /* dst >>= imm */
            if (imm != 0)
@@ -548,6 +557,11 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
         */
        case BPF_ALU | BPF_MOV | BPF_X: /* (u32) dst = src */
        case BPF_ALU64 | BPF_MOV | BPF_X: /* dst = src */
+           if (imm == 1) {
+               /* special mov32 for zext */
+               PPC_RLWINM(dst_reg, dst_reg, 0, 0, 31);
+               break;
+           }
            PPC_MR(dst_reg, src_reg);
            goto bpf_alu32_trunc;
        case BPF_ALU | BPF_MOV | BPF_K: /* (u32) dst = imm */
@@ -555,11 +569,13 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
            PPC_LI32(dst_reg, imm);
            if (imm < 0)
                goto bpf_alu32_trunc;
+           else if (insn_is_zext(&insn[i + 1]))
+               addrs[++i] = ctx->idx * 4;
            break;

 bpf_alu32_trunc:
        /* Truncate to 32-bits */
-       if (BPF_CLASS(code) == BPF_ALU)
+       if (BPF_CLASS(code) == BPF_ALU && !fp->aux->verifier_zext)
            PPC_RLWINM(dst_reg, dst_reg, 0, 0, 31);
        break;

@@ -618,10 +634,13 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
            case 16:
                /* zero-extend 16 bits into 64 bits */
                PPC_RLDICL(dst_reg, dst_reg, 0, 48);
+               if (insn_is_zext(&insn[i + 1]))
+                   addrs[++i] = ctx->idx * 4;
                break;
            case 32:
-               /* zero-extend 32 bits into 64 bits */
-               PPC_RLDICL(dst_reg, dst_reg, 0, 32);
+               if (!fp->aux->verifier_zext)
+                   /* zero-extend 32 bits into 64 bits */
+                   PPC_RLDICL(dst_reg, dst_reg, 0, 32);
                break;
            case 64:
                /* nop */
@@ -698,14 +717,20 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
        /* dst = *(u8 *)(ul) (src + off) */
        case BPF_LDX | BPF_MEM | BPF_B:
            PPC_LBZ(dst_reg, src_reg, off);
+           if (insn_is_zext(&insn[i + 1]))
+               addrs[++i] = ctx->idx * 4;
            break;
        /* dst = *(u16 *)(ul) (src + off) */
        case BPF_LDX | BPF_MEM | BPF_H:
            PPC_LHZ(dst_reg, src_reg, off);
+           if (insn_is_zext(&insn[i + 1]))
+               addrs[++i] = ctx->idx * 4;
            break;
        /* dst = *(u32 *)(ul) (src + off) */
        case BPF_LDX | BPF_MEM | BPF_W:
            PPC_LWZ(dst_reg, src_reg, off);
+           if (insn_is_zext(&insn[i + 1]))
+               addrs[++i] = ctx->idx * 4;
            break;
        /* dst = *(u64 *)(ul) (src + off) */
        case BPF_LDX | BPF_MEM | BPF_DW:
@@ -1046,6 +1071,11 @@ struct powerpc64_jit_data {
    struct codegen_context ctx;
 };

+bool bpf_jit_needs_zext(void)
+{
+   return true;
+}
+
 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
 {
    u32 proglen;

Leave a Reply

Your email address will not be published. Required fields are marked *