bpfjit.c revision 1.1.2.4
1 1.1.2.4 yamt /* $NetBSD: bpfjit.c,v 1.1.2.4 2014/05/22 11:41:08 yamt Exp $ */ 2 1.1.2.4 yamt 3 1.1.2.2 yamt /*- 4 1.1.2.2 yamt * Copyright (c) 2011-2012 Alexander Nasonov. 5 1.1.2.2 yamt * All rights reserved. 6 1.1.2.2 yamt * 7 1.1.2.2 yamt * Redistribution and use in source and binary forms, with or without 8 1.1.2.2 yamt * modification, are permitted provided that the following conditions 9 1.1.2.2 yamt * are met: 10 1.1.2.2 yamt * 11 1.1.2.2 yamt * 1. Redistributions of source code must retain the above copyright 12 1.1.2.2 yamt * notice, this list of conditions and the following disclaimer. 13 1.1.2.2 yamt * 2. Redistributions in binary form must reproduce the above copyright 14 1.1.2.2 yamt * notice, this list of conditions and the following disclaimer in 15 1.1.2.2 yamt * the documentation and/or other materials provided with the 16 1.1.2.2 yamt * distribution. 17 1.1.2.2 yamt * 18 1.1.2.2 yamt * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 1.1.2.2 yamt * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 1.1.2.2 yamt * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 21 1.1.2.2 yamt * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 22 1.1.2.2 yamt * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 23 1.1.2.2 yamt * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 24 1.1.2.2 yamt * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 25 1.1.2.2 yamt * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 26 1.1.2.2 yamt * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 27 1.1.2.2 yamt * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 28 1.1.2.2 yamt * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 1.1.2.2 yamt * SUCH DAMAGE. 30 1.1.2.2 yamt */ 31 1.1.2.2 yamt 32 1.1.2.3 yamt #include <sys/cdefs.h> 33 1.1.2.3 yamt #ifdef _KERNEL 34 1.1.2.4 yamt __KERNEL_RCSID(0, "$NetBSD: bpfjit.c,v 1.1.2.4 2014/05/22 11:41:08 yamt Exp $"); 35 1.1.2.3 yamt #else 36 1.1.2.4 yamt __RCSID("$NetBSD: bpfjit.c,v 1.1.2.4 2014/05/22 11:41:08 yamt Exp $"); 37 1.1.2.3 yamt #endif 38 1.1.2.3 yamt 39 1.1.2.4 yamt #include <sys/types.h> 40 1.1.2.4 yamt #include <sys/queue.h> 41 1.1.2.2 yamt 42 1.1.2.2 yamt #ifndef _KERNEL 43 1.1.2.4 yamt #include <stdlib.h> 44 1.1.2.2 yamt #include <assert.h> 45 1.1.2.4 yamt #define BPFJIT_ALLOC(sz) malloc(sz) 46 1.1.2.4 yamt #define BPFJIT_FREE(p, sz) free(p) 47 1.1.2.2 yamt #define BPFJIT_ASSERT(c) assert(c) 48 1.1.2.2 yamt #else 49 1.1.2.4 yamt #include <sys/kmem.h> 50 1.1.2.4 yamt #define BPFJIT_ALLOC(sz) kmem_alloc(sz, KM_SLEEP) 51 1.1.2.4 yamt #define BPFJIT_FREE(p, sz) kmem_free(p, sz) 52 1.1.2.2 yamt #define BPFJIT_ASSERT(c) KASSERT(c) 53 1.1.2.2 yamt #endif 54 1.1.2.2 yamt 55 1.1.2.2 yamt #ifndef _KERNEL 56 1.1.2.2 yamt #include <limits.h> 57 1.1.2.4 yamt #include <stdio.h> 58 1.1.2.2 yamt #include <stdbool.h> 59 1.1.2.2 yamt #include <stddef.h> 60 1.1.2.2 yamt #include <stdint.h> 61 1.1.2.2 yamt #else 62 1.1.2.2 yamt #include <sys/atomic.h> 63 1.1.2.2 yamt #include <sys/module.h> 64 1.1.2.2 yamt #endif 65 1.1.2.2 yamt 66 1.1.2.4 yamt #define __BPF_PRIVATE 67 1.1.2.4 yamt #include <net/bpf.h> 68 1.1.2.4 yamt #include <net/bpfjit.h> 69 1.1.2.2 yamt #include <sljitLir.h> 70 1.1.2.2 yamt 71 1.1.2.2 yamt #define BPFJIT_A SLJIT_TEMPORARY_REG1 72 1.1.2.2 yamt #define BPFJIT_X SLJIT_TEMPORARY_EREG1 73 1.1.2.2 yamt #define BPFJIT_TMP1 SLJIT_TEMPORARY_REG2 74 1.1.2.2 yamt #define BPFJIT_TMP2 SLJIT_TEMPORARY_REG3 75 1.1.2.2 yamt #define BPFJIT_BUF SLJIT_SAVED_REG1 76 1.1.2.2 yamt #define BPFJIT_WIRELEN SLJIT_SAVED_REG2 77 1.1.2.2 yamt #define BPFJIT_BUFLEN SLJIT_SAVED_REG3 78 1.1.2.2 yamt #define BPFJIT_KERN_TMP SLJIT_TEMPORARY_EREG2 79 1.1.2.2 yamt 80 1.1.2.2 yamt /* 81 1.1.2.2 yamt * Flags for bpfjit_optimization_hints(). 82 1.1.2.2 yamt */ 83 1.1.2.2 yamt #define BPFJIT_INIT_X 0x10000 84 1.1.2.2 yamt #define BPFJIT_INIT_A 0x20000 85 1.1.2.2 yamt 86 1.1.2.2 yamt /* 87 1.1.2.2 yamt * Node of bj_jumps list. 88 1.1.2.2 yamt */ 89 1.1.2.4 yamt struct bpfjit_jump { 90 1.1.2.2 yamt struct sljit_jump *bj_jump; 91 1.1.2.2 yamt SLIST_ENTRY(bpfjit_jump) bj_entries; 92 1.1.2.2 yamt uint32_t bj_safe_length; 93 1.1.2.2 yamt }; 94 1.1.2.2 yamt 95 1.1.2.2 yamt /* 96 1.1.2.2 yamt * Data for BPF_JMP instruction. 97 1.1.2.2 yamt */ 98 1.1.2.4 yamt struct bpfjit_jump_data { 99 1.1.2.2 yamt /* 100 1.1.2.2 yamt * These entries make up bj_jumps list: 101 1.1.2.2 yamt * bj_jtf[0] - when coming from jt path, 102 1.1.2.2 yamt * bj_jtf[1] - when coming from jf path. 103 1.1.2.2 yamt */ 104 1.1.2.2 yamt struct bpfjit_jump bj_jtf[2]; 105 1.1.2.2 yamt }; 106 1.1.2.2 yamt 107 1.1.2.2 yamt /* 108 1.1.2.2 yamt * Data for "read from packet" instructions. 109 1.1.2.2 yamt * See also read_pkt_insn() function below. 110 1.1.2.2 yamt */ 111 1.1.2.4 yamt struct bpfjit_read_pkt_data { 112 1.1.2.2 yamt /* 113 1.1.2.2 yamt * If positive, emit "if (buflen < bj_check_length) return 0". 114 1.1.2.2 yamt * We assume that buflen is never equal to UINT32_MAX (otherwise, 115 1.1.2.2 yamt * we need a special bool variable to emit unconditional "return 0"). 116 1.1.2.2 yamt */ 117 1.1.2.2 yamt uint32_t bj_check_length; 118 1.1.2.2 yamt }; 119 1.1.2.2 yamt 120 1.1.2.2 yamt /* 121 1.1.2.2 yamt * Additional (optimization-related) data for bpf_insn. 122 1.1.2.2 yamt */ 123 1.1.2.4 yamt struct bpfjit_insn_data { 124 1.1.2.2 yamt /* List of jumps to this insn. */ 125 1.1.2.2 yamt SLIST_HEAD(, bpfjit_jump) bj_jumps; 126 1.1.2.2 yamt 127 1.1.2.2 yamt union { 128 1.1.2.2 yamt struct bpfjit_jump_data bj_jdata; 129 1.1.2.2 yamt struct bpfjit_read_pkt_data bj_rdata; 130 1.1.2.2 yamt } bj_aux; 131 1.1.2.2 yamt 132 1.1.2.2 yamt bool bj_unreachable; 133 1.1.2.2 yamt }; 134 1.1.2.2 yamt 135 1.1.2.2 yamt #ifdef _KERNEL 136 1.1.2.2 yamt 137 1.1.2.2 yamt uint32_t m_xword(const struct mbuf *, uint32_t, int *); 138 1.1.2.2 yamt uint32_t m_xhalf(const struct mbuf *, uint32_t, int *); 139 1.1.2.2 yamt uint32_t m_xbyte(const struct mbuf *, uint32_t, int *); 140 1.1.2.2 yamt 141 1.1.2.2 yamt MODULE(MODULE_CLASS_MISC, bpfjit, "sljit") 142 1.1.2.2 yamt 143 1.1.2.2 yamt static int 144 1.1.2.2 yamt bpfjit_modcmd(modcmd_t cmd, void *arg) 145 1.1.2.2 yamt { 146 1.1.2.2 yamt 147 1.1.2.2 yamt switch (cmd) { 148 1.1.2.2 yamt case MODULE_CMD_INIT: 149 1.1.2.2 yamt bpfjit_module_ops.bj_free_code = &bpfjit_free_code; 150 1.1.2.2 yamt membar_producer(); 151 1.1.2.2 yamt bpfjit_module_ops.bj_generate_code = &bpfjit_generate_code; 152 1.1.2.2 yamt membar_producer(); 153 1.1.2.2 yamt return 0; 154 1.1.2.2 yamt 155 1.1.2.2 yamt case MODULE_CMD_FINI: 156 1.1.2.2 yamt return EOPNOTSUPP; 157 1.1.2.2 yamt 158 1.1.2.2 yamt default: 159 1.1.2.2 yamt return ENOTTY; 160 1.1.2.2 yamt } 161 1.1.2.2 yamt } 162 1.1.2.2 yamt #endif 163 1.1.2.2 yamt 164 1.1.2.2 yamt static uint32_t 165 1.1.2.2 yamt read_width(struct bpf_insn *pc) 166 1.1.2.2 yamt { 167 1.1.2.2 yamt 168 1.1.2.2 yamt switch (BPF_SIZE(pc->code)) { 169 1.1.2.2 yamt case BPF_W: 170 1.1.2.2 yamt return 4; 171 1.1.2.2 yamt case BPF_H: 172 1.1.2.2 yamt return 2; 173 1.1.2.2 yamt case BPF_B: 174 1.1.2.2 yamt return 1; 175 1.1.2.2 yamt default: 176 1.1.2.2 yamt BPFJIT_ASSERT(false); 177 1.1.2.2 yamt return 0; 178 1.1.2.2 yamt } 179 1.1.2.2 yamt } 180 1.1.2.2 yamt 181 1.1.2.2 yamt /* 182 1.1.2.2 yamt * Get offset of M[k] on the stack. 183 1.1.2.2 yamt */ 184 1.1.2.3 yamt static size_t 185 1.1.2.3 yamt mem_local_offset(uint32_t k, unsigned int minm) 186 1.1.2.2 yamt { 187 1.1.2.3 yamt size_t moff = (k - minm) * sizeof(uint32_t); 188 1.1.2.2 yamt 189 1.1.2.2 yamt #ifdef _KERNEL 190 1.1.2.2 yamt /* 191 1.1.2.2 yamt * 4 bytes for the third argument of m_xword/m_xhalf/m_xbyte. 192 1.1.2.2 yamt */ 193 1.1.2.2 yamt return sizeof(uint32_t) + moff; 194 1.1.2.2 yamt #else 195 1.1.2.2 yamt return moff; 196 1.1.2.2 yamt #endif 197 1.1.2.2 yamt } 198 1.1.2.2 yamt 199 1.1.2.2 yamt /* 200 1.1.2.2 yamt * Generate code for BPF_LD+BPF_B+BPF_ABS A <- P[k:1]. 201 1.1.2.2 yamt */ 202 1.1.2.2 yamt static int 203 1.1.2.2 yamt emit_read8(struct sljit_compiler* compiler, uint32_t k) 204 1.1.2.2 yamt { 205 1.1.2.2 yamt 206 1.1.2.2 yamt return sljit_emit_op1(compiler, 207 1.1.2.2 yamt SLJIT_MOV_UB, 208 1.1.2.2 yamt BPFJIT_A, 0, 209 1.1.2.2 yamt SLJIT_MEM1(BPFJIT_BUF), k); 210 1.1.2.2 yamt } 211 1.1.2.2 yamt 212 1.1.2.2 yamt /* 213 1.1.2.2 yamt * Generate code for BPF_LD+BPF_H+BPF_ABS A <- P[k:2]. 214 1.1.2.2 yamt */ 215 1.1.2.2 yamt static int 216 1.1.2.2 yamt emit_read16(struct sljit_compiler* compiler, uint32_t k) 217 1.1.2.2 yamt { 218 1.1.2.2 yamt int status; 219 1.1.2.2 yamt 220 1.1.2.2 yamt /* tmp1 = buf[k]; */ 221 1.1.2.2 yamt status = sljit_emit_op1(compiler, 222 1.1.2.2 yamt SLJIT_MOV_UB, 223 1.1.2.2 yamt BPFJIT_TMP1, 0, 224 1.1.2.2 yamt SLJIT_MEM1(BPFJIT_BUF), k); 225 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 226 1.1.2.2 yamt return status; 227 1.1.2.2 yamt 228 1.1.2.2 yamt /* A = buf[k+1]; */ 229 1.1.2.2 yamt status = sljit_emit_op1(compiler, 230 1.1.2.2 yamt SLJIT_MOV_UB, 231 1.1.2.2 yamt BPFJIT_A, 0, 232 1.1.2.2 yamt SLJIT_MEM1(BPFJIT_BUF), k+1); 233 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 234 1.1.2.2 yamt return status; 235 1.1.2.2 yamt 236 1.1.2.2 yamt /* tmp1 = tmp1 << 8; */ 237 1.1.2.2 yamt status = sljit_emit_op2(compiler, 238 1.1.2.2 yamt SLJIT_SHL, 239 1.1.2.2 yamt BPFJIT_TMP1, 0, 240 1.1.2.2 yamt BPFJIT_TMP1, 0, 241 1.1.2.2 yamt SLJIT_IMM, 8); 242 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 243 1.1.2.2 yamt return status; 244 1.1.2.2 yamt 245 1.1.2.2 yamt /* A = A + tmp1; */ 246 1.1.2.2 yamt status = sljit_emit_op2(compiler, 247 1.1.2.2 yamt SLJIT_ADD, 248 1.1.2.2 yamt BPFJIT_A, 0, 249 1.1.2.2 yamt BPFJIT_A, 0, 250 1.1.2.2 yamt BPFJIT_TMP1, 0); 251 1.1.2.2 yamt return status; 252 1.1.2.2 yamt } 253 1.1.2.2 yamt 254 1.1.2.2 yamt /* 255 1.1.2.2 yamt * Generate code for BPF_LD+BPF_W+BPF_ABS A <- P[k:4]. 256 1.1.2.2 yamt */ 257 1.1.2.2 yamt static int 258 1.1.2.2 yamt emit_read32(struct sljit_compiler* compiler, uint32_t k) 259 1.1.2.2 yamt { 260 1.1.2.2 yamt int status; 261 1.1.2.2 yamt 262 1.1.2.2 yamt /* tmp1 = buf[k]; */ 263 1.1.2.2 yamt status = sljit_emit_op1(compiler, 264 1.1.2.2 yamt SLJIT_MOV_UB, 265 1.1.2.2 yamt BPFJIT_TMP1, 0, 266 1.1.2.2 yamt SLJIT_MEM1(BPFJIT_BUF), k); 267 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 268 1.1.2.2 yamt return status; 269 1.1.2.2 yamt 270 1.1.2.2 yamt /* tmp2 = buf[k+1]; */ 271 1.1.2.2 yamt status = sljit_emit_op1(compiler, 272 1.1.2.2 yamt SLJIT_MOV_UB, 273 1.1.2.2 yamt BPFJIT_TMP2, 0, 274 1.1.2.2 yamt SLJIT_MEM1(BPFJIT_BUF), k+1); 275 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 276 1.1.2.2 yamt return status; 277 1.1.2.2 yamt 278 1.1.2.2 yamt /* A = buf[k+3]; */ 279 1.1.2.2 yamt status = sljit_emit_op1(compiler, 280 1.1.2.2 yamt SLJIT_MOV_UB, 281 1.1.2.2 yamt BPFJIT_A, 0, 282 1.1.2.2 yamt SLJIT_MEM1(BPFJIT_BUF), k+3); 283 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 284 1.1.2.2 yamt return status; 285 1.1.2.2 yamt 286 1.1.2.2 yamt /* tmp1 = tmp1 << 24; */ 287 1.1.2.2 yamt status = sljit_emit_op2(compiler, 288 1.1.2.2 yamt SLJIT_SHL, 289 1.1.2.2 yamt BPFJIT_TMP1, 0, 290 1.1.2.2 yamt BPFJIT_TMP1, 0, 291 1.1.2.2 yamt SLJIT_IMM, 24); 292 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 293 1.1.2.2 yamt return status; 294 1.1.2.2 yamt 295 1.1.2.2 yamt /* A = A + tmp1; */ 296 1.1.2.2 yamt status = sljit_emit_op2(compiler, 297 1.1.2.2 yamt SLJIT_ADD, 298 1.1.2.2 yamt BPFJIT_A, 0, 299 1.1.2.2 yamt BPFJIT_A, 0, 300 1.1.2.2 yamt BPFJIT_TMP1, 0); 301 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 302 1.1.2.2 yamt return status; 303 1.1.2.2 yamt 304 1.1.2.2 yamt /* tmp1 = buf[k+2]; */ 305 1.1.2.2 yamt status = sljit_emit_op1(compiler, 306 1.1.2.2 yamt SLJIT_MOV_UB, 307 1.1.2.2 yamt BPFJIT_TMP1, 0, 308 1.1.2.2 yamt SLJIT_MEM1(BPFJIT_BUF), k+2); 309 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 310 1.1.2.2 yamt return status; 311 1.1.2.2 yamt 312 1.1.2.2 yamt /* tmp2 = tmp2 << 16; */ 313 1.1.2.2 yamt status = sljit_emit_op2(compiler, 314 1.1.2.2 yamt SLJIT_SHL, 315 1.1.2.2 yamt BPFJIT_TMP2, 0, 316 1.1.2.2 yamt BPFJIT_TMP2, 0, 317 1.1.2.2 yamt SLJIT_IMM, 16); 318 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 319 1.1.2.2 yamt return status; 320 1.1.2.2 yamt 321 1.1.2.2 yamt /* A = A + tmp2; */ 322 1.1.2.2 yamt status = sljit_emit_op2(compiler, 323 1.1.2.2 yamt SLJIT_ADD, 324 1.1.2.2 yamt BPFJIT_A, 0, 325 1.1.2.2 yamt BPFJIT_A, 0, 326 1.1.2.2 yamt BPFJIT_TMP2, 0); 327 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 328 1.1.2.2 yamt return status; 329 1.1.2.2 yamt 330 1.1.2.2 yamt /* tmp1 = tmp1 << 8; */ 331 1.1.2.2 yamt status = sljit_emit_op2(compiler, 332 1.1.2.2 yamt SLJIT_SHL, 333 1.1.2.2 yamt BPFJIT_TMP1, 0, 334 1.1.2.2 yamt BPFJIT_TMP1, 0, 335 1.1.2.2 yamt SLJIT_IMM, 8); 336 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 337 1.1.2.2 yamt return status; 338 1.1.2.2 yamt 339 1.1.2.2 yamt /* A = A + tmp1; */ 340 1.1.2.2 yamt status = sljit_emit_op2(compiler, 341 1.1.2.2 yamt SLJIT_ADD, 342 1.1.2.2 yamt BPFJIT_A, 0, 343 1.1.2.2 yamt BPFJIT_A, 0, 344 1.1.2.2 yamt BPFJIT_TMP1, 0); 345 1.1.2.2 yamt return status; 346 1.1.2.2 yamt } 347 1.1.2.2 yamt 348 1.1.2.2 yamt #ifdef _KERNEL 349 1.1.2.2 yamt /* 350 1.1.2.2 yamt * Generate m_xword/m_xhalf/m_xbyte call. 351 1.1.2.2 yamt * 352 1.1.2.2 yamt * pc is one of: 353 1.1.2.2 yamt * BPF_LD+BPF_W+BPF_ABS A <- P[k:4] 354 1.1.2.2 yamt * BPF_LD+BPF_H+BPF_ABS A <- P[k:2] 355 1.1.2.2 yamt * BPF_LD+BPF_B+BPF_ABS A <- P[k:1] 356 1.1.2.2 yamt * BPF_LD+BPF_W+BPF_IND A <- P[X+k:4] 357 1.1.2.2 yamt * BPF_LD+BPF_H+BPF_IND A <- P[X+k:2] 358 1.1.2.2 yamt * BPF_LD+BPF_B+BPF_IND A <- P[X+k:1] 359 1.1.2.2 yamt * BPF_LDX+BPF_B+BPF_MSH X <- 4*(P[k:1]&0xf) 360 1.1.2.2 yamt * 361 1.1.2.2 yamt * dst must be BPFJIT_A for BPF_LD instructions and BPFJIT_X 362 1.1.2.2 yamt * or any of BPFJIT_TMP* registrers for BPF_MSH instruction. 363 1.1.2.2 yamt */ 364 1.1.2.2 yamt static int 365 1.1.2.2 yamt emit_xcall(struct sljit_compiler* compiler, struct bpf_insn *pc, 366 1.1.2.2 yamt int dst, sljit_w dstw, struct sljit_jump **ret0_jump, 367 1.1.2.2 yamt uint32_t (*fn)(const struct mbuf *, uint32_t, int *)) 368 1.1.2.2 yamt { 369 1.1.2.2 yamt #if BPFJIT_X != SLJIT_TEMPORARY_EREG1 || \ 370 1.1.2.2 yamt BPFJIT_X == SLJIT_RETURN_REG 371 1.1.2.2 yamt #error "Not supported assignment of registers." 372 1.1.2.2 yamt #endif 373 1.1.2.2 yamt int status; 374 1.1.2.2 yamt 375 1.1.2.2 yamt /* 376 1.1.2.2 yamt * The third argument of fn is an address on stack. 377 1.1.2.2 yamt */ 378 1.1.2.2 yamt const int arg3_offset = 0; 379 1.1.2.2 yamt 380 1.1.2.2 yamt if (BPF_CLASS(pc->code) == BPF_LDX) { 381 1.1.2.2 yamt /* save A */ 382 1.1.2.2 yamt status = sljit_emit_op1(compiler, 383 1.1.2.2 yamt SLJIT_MOV, 384 1.1.2.2 yamt BPFJIT_KERN_TMP, 0, 385 1.1.2.2 yamt BPFJIT_A, 0); 386 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 387 1.1.2.2 yamt return status; 388 1.1.2.2 yamt } 389 1.1.2.2 yamt 390 1.1.2.2 yamt /* 391 1.1.2.2 yamt * Prepare registers for fn(buf, k, &err) call. 392 1.1.2.2 yamt */ 393 1.1.2.2 yamt status = sljit_emit_op1(compiler, 394 1.1.2.2 yamt SLJIT_MOV, 395 1.1.2.2 yamt SLJIT_TEMPORARY_REG1, 0, 396 1.1.2.2 yamt BPFJIT_BUF, 0); 397 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 398 1.1.2.2 yamt return status; 399 1.1.2.2 yamt 400 1.1.2.2 yamt if (BPF_CLASS(pc->code) == BPF_LD && BPF_MODE(pc->code) == BPF_IND) { 401 1.1.2.2 yamt status = sljit_emit_op2(compiler, 402 1.1.2.2 yamt SLJIT_ADD, 403 1.1.2.2 yamt SLJIT_TEMPORARY_REG2, 0, 404 1.1.2.2 yamt BPFJIT_X, 0, 405 1.1.2.2 yamt SLJIT_IMM, (uint32_t)pc->k); 406 1.1.2.2 yamt } else { 407 1.1.2.2 yamt status = sljit_emit_op1(compiler, 408 1.1.2.2 yamt SLJIT_MOV, 409 1.1.2.2 yamt SLJIT_TEMPORARY_REG2, 0, 410 1.1.2.2 yamt SLJIT_IMM, (uint32_t)pc->k); 411 1.1.2.2 yamt } 412 1.1.2.2 yamt 413 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 414 1.1.2.2 yamt return status; 415 1.1.2.2 yamt 416 1.1.2.2 yamt status = sljit_get_local_base(compiler, 417 1.1.2.2 yamt SLJIT_TEMPORARY_REG3, 0, arg3_offset); 418 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 419 1.1.2.2 yamt return status; 420 1.1.2.2 yamt 421 1.1.2.2 yamt /* fn(buf, k, &err); */ 422 1.1.2.2 yamt status = sljit_emit_ijump(compiler, 423 1.1.2.2 yamt SLJIT_CALL3, 424 1.1.2.2 yamt SLJIT_IMM, SLJIT_FUNC_OFFSET(fn)); 425 1.1.2.2 yamt 426 1.1.2.2 yamt if (BPF_CLASS(pc->code) == BPF_LDX) { 427 1.1.2.2 yamt 428 1.1.2.2 yamt /* move return value to dst */ 429 1.1.2.2 yamt BPFJIT_ASSERT(dst != SLJIT_RETURN_REG); 430 1.1.2.2 yamt status = sljit_emit_op1(compiler, 431 1.1.2.2 yamt SLJIT_MOV, 432 1.1.2.2 yamt dst, dstw, 433 1.1.2.2 yamt SLJIT_RETURN_REG, 0); 434 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 435 1.1.2.2 yamt return status; 436 1.1.2.2 yamt 437 1.1.2.2 yamt /* restore A */ 438 1.1.2.2 yamt status = sljit_emit_op1(compiler, 439 1.1.2.2 yamt SLJIT_MOV, 440 1.1.2.2 yamt BPFJIT_A, 0, 441 1.1.2.2 yamt BPFJIT_KERN_TMP, 0); 442 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 443 1.1.2.2 yamt return status; 444 1.1.2.2 yamt 445 1.1.2.2 yamt } else if (dst != SLJIT_RETURN_REG) { 446 1.1.2.2 yamt status = sljit_emit_op1(compiler, 447 1.1.2.2 yamt SLJIT_MOV, 448 1.1.2.2 yamt dst, dstw, 449 1.1.2.2 yamt SLJIT_RETURN_REG, 0); 450 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 451 1.1.2.2 yamt return status; 452 1.1.2.2 yamt } 453 1.1.2.2 yamt 454 1.1.2.2 yamt /* tmp3 = *err; */ 455 1.1.2.2 yamt status = sljit_emit_op1(compiler, 456 1.1.2.2 yamt SLJIT_MOV_UI, 457 1.1.2.2 yamt SLJIT_TEMPORARY_REG3, 0, 458 1.1.2.2 yamt SLJIT_MEM1(SLJIT_LOCALS_REG), arg3_offset); 459 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 460 1.1.2.2 yamt return status; 461 1.1.2.2 yamt 462 1.1.2.2 yamt /* if (tmp3 != 0) return 0; */ 463 1.1.2.2 yamt *ret0_jump = sljit_emit_cmp(compiler, 464 1.1.2.2 yamt SLJIT_C_NOT_EQUAL, 465 1.1.2.2 yamt SLJIT_TEMPORARY_REG3, 0, 466 1.1.2.2 yamt SLJIT_IMM, 0); 467 1.1.2.2 yamt if (*ret0_jump == NULL) 468 1.1.2.2 yamt return SLJIT_ERR_ALLOC_FAILED; 469 1.1.2.2 yamt 470 1.1.2.2 yamt return status; 471 1.1.2.2 yamt } 472 1.1.2.2 yamt #endif 473 1.1.2.2 yamt 474 1.1.2.2 yamt /* 475 1.1.2.2 yamt * Generate code for 476 1.1.2.2 yamt * BPF_LD+BPF_W+BPF_ABS A <- P[k:4] 477 1.1.2.2 yamt * BPF_LD+BPF_H+BPF_ABS A <- P[k:2] 478 1.1.2.2 yamt * BPF_LD+BPF_B+BPF_ABS A <- P[k:1] 479 1.1.2.2 yamt * BPF_LD+BPF_W+BPF_IND A <- P[X+k:4] 480 1.1.2.2 yamt * BPF_LD+BPF_H+BPF_IND A <- P[X+k:2] 481 1.1.2.2 yamt * BPF_LD+BPF_B+BPF_IND A <- P[X+k:1] 482 1.1.2.2 yamt */ 483 1.1.2.2 yamt static int 484 1.1.2.2 yamt emit_pkt_read(struct sljit_compiler* compiler, 485 1.1.2.2 yamt struct bpf_insn *pc, struct sljit_jump *to_mchain_jump, 486 1.1.2.2 yamt struct sljit_jump **ret0, size_t *ret0_size) 487 1.1.2.2 yamt { 488 1.1.2.4 yamt int status = 0; /* XXX gcc 4.1 */ 489 1.1.2.2 yamt uint32_t width; 490 1.1.2.2 yamt struct sljit_jump *jump; 491 1.1.2.2 yamt #ifdef _KERNEL 492 1.1.2.2 yamt struct sljit_label *label; 493 1.1.2.2 yamt struct sljit_jump *over_mchain_jump; 494 1.1.2.2 yamt const bool check_zero_buflen = (to_mchain_jump != NULL); 495 1.1.2.2 yamt #endif 496 1.1.2.2 yamt const uint32_t k = pc->k; 497 1.1.2.2 yamt 498 1.1.2.2 yamt #ifdef _KERNEL 499 1.1.2.2 yamt if (to_mchain_jump == NULL) { 500 1.1.2.2 yamt to_mchain_jump = sljit_emit_cmp(compiler, 501 1.1.2.2 yamt SLJIT_C_EQUAL, 502 1.1.2.2 yamt BPFJIT_BUFLEN, 0, 503 1.1.2.2 yamt SLJIT_IMM, 0); 504 1.1.2.2 yamt if (to_mchain_jump == NULL) 505 1.1.2.2 yamt return SLJIT_ERR_ALLOC_FAILED; 506 1.1.2.2 yamt } 507 1.1.2.2 yamt #endif 508 1.1.2.2 yamt 509 1.1.2.2 yamt width = read_width(pc); 510 1.1.2.2 yamt 511 1.1.2.2 yamt if (BPF_MODE(pc->code) == BPF_IND) { 512 1.1.2.2 yamt /* tmp1 = buflen - (pc->k + width); */ 513 1.1.2.2 yamt status = sljit_emit_op2(compiler, 514 1.1.2.2 yamt SLJIT_SUB, 515 1.1.2.2 yamt BPFJIT_TMP1, 0, 516 1.1.2.2 yamt BPFJIT_BUFLEN, 0, 517 1.1.2.2 yamt SLJIT_IMM, k + width); 518 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 519 1.1.2.2 yamt return status; 520 1.1.2.2 yamt 521 1.1.2.2 yamt /* buf += X; */ 522 1.1.2.2 yamt status = sljit_emit_op2(compiler, 523 1.1.2.2 yamt SLJIT_ADD, 524 1.1.2.2 yamt BPFJIT_BUF, 0, 525 1.1.2.2 yamt BPFJIT_BUF, 0, 526 1.1.2.2 yamt BPFJIT_X, 0); 527 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 528 1.1.2.2 yamt return status; 529 1.1.2.2 yamt 530 1.1.2.2 yamt /* if (tmp1 < X) return 0; */ 531 1.1.2.2 yamt jump = sljit_emit_cmp(compiler, 532 1.1.2.2 yamt SLJIT_C_LESS, 533 1.1.2.2 yamt BPFJIT_TMP1, 0, 534 1.1.2.2 yamt BPFJIT_X, 0); 535 1.1.2.2 yamt if (jump == NULL) 536 1.1.2.2 yamt return SLJIT_ERR_ALLOC_FAILED; 537 1.1.2.2 yamt ret0[(*ret0_size)++] = jump; 538 1.1.2.2 yamt } 539 1.1.2.2 yamt 540 1.1.2.2 yamt switch (width) { 541 1.1.2.2 yamt case 4: 542 1.1.2.2 yamt status = emit_read32(compiler, k); 543 1.1.2.2 yamt break; 544 1.1.2.2 yamt case 2: 545 1.1.2.2 yamt status = emit_read16(compiler, k); 546 1.1.2.2 yamt break; 547 1.1.2.2 yamt case 1: 548 1.1.2.2 yamt status = emit_read8(compiler, k); 549 1.1.2.2 yamt break; 550 1.1.2.2 yamt } 551 1.1.2.2 yamt 552 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 553 1.1.2.2 yamt return status; 554 1.1.2.2 yamt 555 1.1.2.2 yamt if (BPF_MODE(pc->code) == BPF_IND) { 556 1.1.2.2 yamt /* buf -= X; */ 557 1.1.2.2 yamt status = sljit_emit_op2(compiler, 558 1.1.2.2 yamt SLJIT_SUB, 559 1.1.2.2 yamt BPFJIT_BUF, 0, 560 1.1.2.2 yamt BPFJIT_BUF, 0, 561 1.1.2.2 yamt BPFJIT_X, 0); 562 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 563 1.1.2.2 yamt return status; 564 1.1.2.2 yamt } 565 1.1.2.2 yamt 566 1.1.2.2 yamt #ifdef _KERNEL 567 1.1.2.2 yamt over_mchain_jump = sljit_emit_jump(compiler, SLJIT_JUMP); 568 1.1.2.2 yamt if (over_mchain_jump == NULL) 569 1.1.2.2 yamt return SLJIT_ERR_ALLOC_FAILED; 570 1.1.2.2 yamt 571 1.1.2.2 yamt /* entry point to mchain handler */ 572 1.1.2.2 yamt label = sljit_emit_label(compiler); 573 1.1.2.2 yamt if (label == NULL) 574 1.1.2.2 yamt return SLJIT_ERR_ALLOC_FAILED; 575 1.1.2.2 yamt sljit_set_label(to_mchain_jump, label); 576 1.1.2.2 yamt 577 1.1.2.2 yamt if (check_zero_buflen) { 578 1.1.2.2 yamt /* if (buflen != 0) return 0; */ 579 1.1.2.2 yamt jump = sljit_emit_cmp(compiler, 580 1.1.2.2 yamt SLJIT_C_NOT_EQUAL, 581 1.1.2.2 yamt BPFJIT_BUFLEN, 0, 582 1.1.2.2 yamt SLJIT_IMM, 0); 583 1.1.2.2 yamt if (jump == NULL) 584 1.1.2.2 yamt return SLJIT_ERR_ALLOC_FAILED; 585 1.1.2.2 yamt ret0[(*ret0_size)++] = jump; 586 1.1.2.2 yamt } 587 1.1.2.2 yamt 588 1.1.2.2 yamt switch (width) { 589 1.1.2.2 yamt case 4: 590 1.1.2.2 yamt status = emit_xcall(compiler, pc, BPFJIT_A, 0, &jump, &m_xword); 591 1.1.2.2 yamt break; 592 1.1.2.2 yamt case 2: 593 1.1.2.2 yamt status = emit_xcall(compiler, pc, BPFJIT_A, 0, &jump, &m_xhalf); 594 1.1.2.2 yamt break; 595 1.1.2.2 yamt case 1: 596 1.1.2.2 yamt status = emit_xcall(compiler, pc, BPFJIT_A, 0, &jump, &m_xbyte); 597 1.1.2.2 yamt break; 598 1.1.2.2 yamt } 599 1.1.2.2 yamt 600 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 601 1.1.2.2 yamt return status; 602 1.1.2.2 yamt 603 1.1.2.2 yamt ret0[(*ret0_size)++] = jump; 604 1.1.2.2 yamt 605 1.1.2.2 yamt label = sljit_emit_label(compiler); 606 1.1.2.2 yamt if (label == NULL) 607 1.1.2.2 yamt return SLJIT_ERR_ALLOC_FAILED; 608 1.1.2.2 yamt sljit_set_label(over_mchain_jump, label); 609 1.1.2.2 yamt #endif 610 1.1.2.2 yamt 611 1.1.2.2 yamt return status; 612 1.1.2.2 yamt } 613 1.1.2.2 yamt 614 1.1.2.2 yamt /* 615 1.1.2.2 yamt * Generate code for BPF_LDX+BPF_B+BPF_MSH X <- 4*(P[k:1]&0xf). 616 1.1.2.2 yamt */ 617 1.1.2.2 yamt static int 618 1.1.2.2 yamt emit_msh(struct sljit_compiler* compiler, 619 1.1.2.2 yamt struct bpf_insn *pc, struct sljit_jump *to_mchain_jump, 620 1.1.2.2 yamt struct sljit_jump **ret0, size_t *ret0_size) 621 1.1.2.2 yamt { 622 1.1.2.2 yamt int status; 623 1.1.2.2 yamt #ifdef _KERNEL 624 1.1.2.2 yamt struct sljit_label *label; 625 1.1.2.2 yamt struct sljit_jump *jump, *over_mchain_jump; 626 1.1.2.2 yamt const bool check_zero_buflen = (to_mchain_jump != NULL); 627 1.1.2.2 yamt #endif 628 1.1.2.2 yamt const uint32_t k = pc->k; 629 1.1.2.2 yamt 630 1.1.2.2 yamt #ifdef _KERNEL 631 1.1.2.2 yamt if (to_mchain_jump == NULL) { 632 1.1.2.2 yamt to_mchain_jump = sljit_emit_cmp(compiler, 633 1.1.2.2 yamt SLJIT_C_EQUAL, 634 1.1.2.2 yamt BPFJIT_BUFLEN, 0, 635 1.1.2.2 yamt SLJIT_IMM, 0); 636 1.1.2.2 yamt if (to_mchain_jump == NULL) 637 1.1.2.2 yamt return SLJIT_ERR_ALLOC_FAILED; 638 1.1.2.2 yamt } 639 1.1.2.2 yamt #endif 640 1.1.2.2 yamt 641 1.1.2.2 yamt /* tmp1 = buf[k] */ 642 1.1.2.2 yamt status = sljit_emit_op1(compiler, 643 1.1.2.2 yamt SLJIT_MOV_UB, 644 1.1.2.2 yamt BPFJIT_TMP1, 0, 645 1.1.2.2 yamt SLJIT_MEM1(BPFJIT_BUF), k); 646 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 647 1.1.2.2 yamt return status; 648 1.1.2.2 yamt 649 1.1.2.2 yamt /* tmp1 &= 0xf */ 650 1.1.2.2 yamt status = sljit_emit_op2(compiler, 651 1.1.2.2 yamt SLJIT_AND, 652 1.1.2.2 yamt BPFJIT_TMP1, 0, 653 1.1.2.2 yamt BPFJIT_TMP1, 0, 654 1.1.2.2 yamt SLJIT_IMM, 0xf); 655 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 656 1.1.2.2 yamt return status; 657 1.1.2.2 yamt 658 1.1.2.2 yamt /* tmp1 = tmp1 << 2 */ 659 1.1.2.2 yamt status = sljit_emit_op2(compiler, 660 1.1.2.2 yamt SLJIT_SHL, 661 1.1.2.2 yamt BPFJIT_X, 0, 662 1.1.2.2 yamt BPFJIT_TMP1, 0, 663 1.1.2.2 yamt SLJIT_IMM, 2); 664 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 665 1.1.2.2 yamt return status; 666 1.1.2.2 yamt 667 1.1.2.2 yamt #ifdef _KERNEL 668 1.1.2.2 yamt over_mchain_jump = sljit_emit_jump(compiler, SLJIT_JUMP); 669 1.1.2.2 yamt if (over_mchain_jump == NULL) 670 1.1.2.2 yamt return SLJIT_ERR_ALLOC_FAILED; 671 1.1.2.2 yamt 672 1.1.2.2 yamt /* entry point to mchain handler */ 673 1.1.2.2 yamt label = sljit_emit_label(compiler); 674 1.1.2.2 yamt if (label == NULL) 675 1.1.2.2 yamt return SLJIT_ERR_ALLOC_FAILED; 676 1.1.2.2 yamt sljit_set_label(to_mchain_jump, label); 677 1.1.2.2 yamt 678 1.1.2.2 yamt if (check_zero_buflen) { 679 1.1.2.2 yamt /* if (buflen != 0) return 0; */ 680 1.1.2.2 yamt jump = sljit_emit_cmp(compiler, 681 1.1.2.2 yamt SLJIT_C_NOT_EQUAL, 682 1.1.2.2 yamt BPFJIT_BUFLEN, 0, 683 1.1.2.2 yamt SLJIT_IMM, 0); 684 1.1.2.2 yamt if (jump == NULL) 685 1.1.2.2 yamt return SLJIT_ERR_ALLOC_FAILED; 686 1.1.2.2 yamt ret0[(*ret0_size)++] = jump; 687 1.1.2.2 yamt } 688 1.1.2.2 yamt 689 1.1.2.2 yamt status = emit_xcall(compiler, pc, BPFJIT_TMP1, 0, &jump, &m_xbyte); 690 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 691 1.1.2.2 yamt return status; 692 1.1.2.2 yamt ret0[(*ret0_size)++] = jump; 693 1.1.2.2 yamt 694 1.1.2.2 yamt /* tmp1 &= 0xf */ 695 1.1.2.2 yamt status = sljit_emit_op2(compiler, 696 1.1.2.2 yamt SLJIT_AND, 697 1.1.2.2 yamt BPFJIT_TMP1, 0, 698 1.1.2.2 yamt BPFJIT_TMP1, 0, 699 1.1.2.2 yamt SLJIT_IMM, 0xf); 700 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 701 1.1.2.2 yamt return status; 702 1.1.2.2 yamt 703 1.1.2.2 yamt /* tmp1 = tmp1 << 2 */ 704 1.1.2.2 yamt status = sljit_emit_op2(compiler, 705 1.1.2.2 yamt SLJIT_SHL, 706 1.1.2.2 yamt BPFJIT_X, 0, 707 1.1.2.2 yamt BPFJIT_TMP1, 0, 708 1.1.2.2 yamt SLJIT_IMM, 2); 709 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 710 1.1.2.2 yamt return status; 711 1.1.2.2 yamt 712 1.1.2.2 yamt 713 1.1.2.2 yamt label = sljit_emit_label(compiler); 714 1.1.2.2 yamt if (label == NULL) 715 1.1.2.2 yamt return SLJIT_ERR_ALLOC_FAILED; 716 1.1.2.2 yamt sljit_set_label(over_mchain_jump, label); 717 1.1.2.2 yamt #endif 718 1.1.2.2 yamt 719 1.1.2.2 yamt return status; 720 1.1.2.2 yamt } 721 1.1.2.2 yamt 722 1.1.2.2 yamt static int 723 1.1.2.2 yamt emit_pow2_division(struct sljit_compiler* compiler, uint32_t k) 724 1.1.2.2 yamt { 725 1.1.2.2 yamt int shift = 0; 726 1.1.2.2 yamt int status = SLJIT_SUCCESS; 727 1.1.2.2 yamt 728 1.1.2.2 yamt while (k > 1) { 729 1.1.2.2 yamt k >>= 1; 730 1.1.2.2 yamt shift++; 731 1.1.2.2 yamt } 732 1.1.2.2 yamt 733 1.1.2.2 yamt BPFJIT_ASSERT(k == 1 && shift < 32); 734 1.1.2.2 yamt 735 1.1.2.2 yamt if (shift != 0) { 736 1.1.2.2 yamt status = sljit_emit_op2(compiler, 737 1.1.2.2 yamt SLJIT_LSHR|SLJIT_INT_OP, 738 1.1.2.2 yamt BPFJIT_A, 0, 739 1.1.2.2 yamt BPFJIT_A, 0, 740 1.1.2.2 yamt SLJIT_IMM, shift); 741 1.1.2.2 yamt } 742 1.1.2.2 yamt 743 1.1.2.2 yamt return status; 744 1.1.2.2 yamt } 745 1.1.2.2 yamt 746 1.1.2.2 yamt #if !defined(BPFJIT_USE_UDIV) 747 1.1.2.2 yamt static sljit_uw 748 1.1.2.2 yamt divide(sljit_uw x, sljit_uw y) 749 1.1.2.2 yamt { 750 1.1.2.2 yamt 751 1.1.2.2 yamt return (uint32_t)x / (uint32_t)y; 752 1.1.2.2 yamt } 753 1.1.2.2 yamt #endif 754 1.1.2.2 yamt 755 1.1.2.2 yamt /* 756 1.1.2.2 yamt * Generate A = A / div. 757 1.1.2.2 yamt * divt,divw are either SLJIT_IMM,pc->k or BPFJIT_X,0. 758 1.1.2.2 yamt */ 759 1.1.2.2 yamt static int 760 1.1.2.2 yamt emit_division(struct sljit_compiler* compiler, int divt, sljit_w divw) 761 1.1.2.2 yamt { 762 1.1.2.2 yamt int status; 763 1.1.2.2 yamt 764 1.1.2.2 yamt #if BPFJIT_X == SLJIT_TEMPORARY_REG1 || \ 765 1.1.2.2 yamt BPFJIT_X == SLJIT_RETURN_REG || \ 766 1.1.2.2 yamt BPFJIT_X == SLJIT_TEMPORARY_REG2 || \ 767 1.1.2.2 yamt BPFJIT_A == SLJIT_TEMPORARY_REG2 768 1.1.2.2 yamt #error "Not supported assignment of registers." 769 1.1.2.2 yamt #endif 770 1.1.2.2 yamt 771 1.1.2.2 yamt #if BPFJIT_A != SLJIT_TEMPORARY_REG1 772 1.1.2.2 yamt status = sljit_emit_op1(compiler, 773 1.1.2.2 yamt SLJIT_MOV, 774 1.1.2.2 yamt SLJIT_TEMPORARY_REG1, 0, 775 1.1.2.2 yamt BPFJIT_A, 0); 776 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 777 1.1.2.2 yamt return status; 778 1.1.2.2 yamt #endif 779 1.1.2.2 yamt 780 1.1.2.2 yamt status = sljit_emit_op1(compiler, 781 1.1.2.2 yamt SLJIT_MOV, 782 1.1.2.2 yamt SLJIT_TEMPORARY_REG2, 0, 783 1.1.2.2 yamt divt, divw); 784 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 785 1.1.2.2 yamt return status; 786 1.1.2.2 yamt 787 1.1.2.2 yamt #if defined(BPFJIT_USE_UDIV) 788 1.1.2.2 yamt status = sljit_emit_op0(compiler, SLJIT_UDIV|SLJIT_INT_OP); 789 1.1.2.2 yamt 790 1.1.2.2 yamt #if BPFJIT_A != SLJIT_TEMPORARY_REG1 791 1.1.2.2 yamt status = sljit_emit_op1(compiler, 792 1.1.2.2 yamt SLJIT_MOV, 793 1.1.2.2 yamt BPFJIT_A, 0, 794 1.1.2.2 yamt SLJIT_TEMPORARY_REG1, 0); 795 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 796 1.1.2.2 yamt return status; 797 1.1.2.2 yamt #endif 798 1.1.2.2 yamt #else 799 1.1.2.2 yamt status = sljit_emit_ijump(compiler, 800 1.1.2.2 yamt SLJIT_CALL2, 801 1.1.2.2 yamt SLJIT_IMM, SLJIT_FUNC_OFFSET(divide)); 802 1.1.2.2 yamt 803 1.1.2.2 yamt #if BPFJIT_A != SLJIT_RETURN_REG 804 1.1.2.2 yamt status = sljit_emit_op1(compiler, 805 1.1.2.2 yamt SLJIT_MOV, 806 1.1.2.2 yamt BPFJIT_A, 0, 807 1.1.2.2 yamt SLJIT_RETURN_REG, 0); 808 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 809 1.1.2.2 yamt return status; 810 1.1.2.2 yamt #endif 811 1.1.2.2 yamt #endif 812 1.1.2.2 yamt 813 1.1.2.2 yamt return status; 814 1.1.2.2 yamt } 815 1.1.2.2 yamt 816 1.1.2.2 yamt /* 817 1.1.2.2 yamt * Count BPF_RET instructions. 818 1.1.2.2 yamt */ 819 1.1.2.2 yamt static size_t 820 1.1.2.2 yamt count_returns(struct bpf_insn *insns, size_t insn_count) 821 1.1.2.2 yamt { 822 1.1.2.2 yamt size_t i; 823 1.1.2.2 yamt size_t rv; 824 1.1.2.2 yamt 825 1.1.2.2 yamt rv = 0; 826 1.1.2.2 yamt for (i = 0; i < insn_count; i++) { 827 1.1.2.2 yamt if (BPF_CLASS(insns[i].code) == BPF_RET) 828 1.1.2.2 yamt rv++; 829 1.1.2.2 yamt } 830 1.1.2.2 yamt 831 1.1.2.2 yamt return rv; 832 1.1.2.2 yamt } 833 1.1.2.2 yamt 834 1.1.2.2 yamt /* 835 1.1.2.2 yamt * Return true if pc is a "read from packet" instruction. 836 1.1.2.2 yamt * If length is not NULL and return value is true, *length will 837 1.1.2.2 yamt * be set to a safe length required to read a packet. 838 1.1.2.2 yamt */ 839 1.1.2.2 yamt static bool 840 1.1.2.2 yamt read_pkt_insn(struct bpf_insn *pc, uint32_t *length) 841 1.1.2.2 yamt { 842 1.1.2.2 yamt bool rv; 843 1.1.2.2 yamt uint32_t width; 844 1.1.2.2 yamt 845 1.1.2.2 yamt switch (BPF_CLASS(pc->code)) { 846 1.1.2.2 yamt default: 847 1.1.2.2 yamt rv = false; 848 1.1.2.2 yamt break; 849 1.1.2.2 yamt 850 1.1.2.2 yamt case BPF_LD: 851 1.1.2.2 yamt rv = BPF_MODE(pc->code) == BPF_ABS || 852 1.1.2.2 yamt BPF_MODE(pc->code) == BPF_IND; 853 1.1.2.2 yamt if (rv) 854 1.1.2.2 yamt width = read_width(pc); 855 1.1.2.2 yamt break; 856 1.1.2.2 yamt 857 1.1.2.2 yamt case BPF_LDX: 858 1.1.2.2 yamt rv = pc->code == (BPF_LDX|BPF_B|BPF_MSH); 859 1.1.2.2 yamt width = 1; 860 1.1.2.2 yamt break; 861 1.1.2.2 yamt } 862 1.1.2.2 yamt 863 1.1.2.2 yamt if (rv && length != NULL) { 864 1.1.2.2 yamt *length = (pc->k > UINT32_MAX - width) ? 865 1.1.2.2 yamt UINT32_MAX : pc->k + width; 866 1.1.2.2 yamt } 867 1.1.2.2 yamt 868 1.1.2.2 yamt return rv; 869 1.1.2.2 yamt } 870 1.1.2.2 yamt 871 1.1.2.2 yamt /* 872 1.1.2.2 yamt * Set bj_check_length for all "read from packet" instructions 873 1.1.2.2 yamt * in a linear block of instructions [from, to). 874 1.1.2.2 yamt */ 875 1.1.2.2 yamt static void 876 1.1.2.2 yamt set_check_length(struct bpf_insn *insns, struct bpfjit_insn_data *insn_dat, 877 1.1.2.2 yamt size_t from, size_t to, uint32_t length) 878 1.1.2.2 yamt { 879 1.1.2.2 yamt 880 1.1.2.2 yamt for (; from < to; from++) { 881 1.1.2.2 yamt if (read_pkt_insn(&insns[from], NULL)) { 882 1.1.2.2 yamt insn_dat[from].bj_aux.bj_rdata.bj_check_length = length; 883 1.1.2.2 yamt length = 0; 884 1.1.2.2 yamt } 885 1.1.2.2 yamt } 886 1.1.2.2 yamt } 887 1.1.2.2 yamt 888 1.1.2.2 yamt /* 889 1.1.2.2 yamt * The function divides instructions into blocks. Destination of a jump 890 1.1.2.2 yamt * instruction starts a new block. BPF_RET and BPF_JMP instructions 891 1.1.2.2 yamt * terminate a block. Blocks are linear, that is, there are no jumps out 892 1.1.2.2 yamt * from the middle of a block and there are no jumps in to the middle of 893 1.1.2.2 yamt * a block. 894 1.1.2.2 yamt * If a block has one or more "read from packet" instructions, 895 1.1.2.2 yamt * bj_check_length will be set to one value for the whole block and that 896 1.1.2.2 yamt * value will be equal to the greatest value of safe lengths of "read from 897 1.1.2.2 yamt * packet" instructions inside the block. 898 1.1.2.2 yamt */ 899 1.1.2.2 yamt static int 900 1.1.2.2 yamt optimize(struct bpf_insn *insns, 901 1.1.2.2 yamt struct bpfjit_insn_data *insn_dat, size_t insn_count) 902 1.1.2.2 yamt { 903 1.1.2.2 yamt size_t i; 904 1.1.2.2 yamt size_t first_read; 905 1.1.2.2 yamt bool unreachable; 906 1.1.2.2 yamt uint32_t jt, jf; 907 1.1.2.2 yamt uint32_t length, safe_length; 908 1.1.2.2 yamt struct bpfjit_jump *jmp, *jtf; 909 1.1.2.2 yamt 910 1.1.2.2 yamt for (i = 0; i < insn_count; i++) 911 1.1.2.2 yamt SLIST_INIT(&insn_dat[i].bj_jumps); 912 1.1.2.2 yamt 913 1.1.2.2 yamt safe_length = 0; 914 1.1.2.2 yamt unreachable = false; 915 1.1.2.2 yamt first_read = SIZE_MAX; 916 1.1.2.2 yamt 917 1.1.2.2 yamt for (i = 0; i < insn_count; i++) { 918 1.1.2.2 yamt 919 1.1.2.2 yamt if (!SLIST_EMPTY(&insn_dat[i].bj_jumps)) { 920 1.1.2.2 yamt unreachable = false; 921 1.1.2.2 yamt 922 1.1.2.2 yamt set_check_length(insns, insn_dat, 923 1.1.2.2 yamt first_read, i, safe_length); 924 1.1.2.2 yamt first_read = SIZE_MAX; 925 1.1.2.2 yamt 926 1.1.2.2 yamt safe_length = UINT32_MAX; 927 1.1.2.2 yamt SLIST_FOREACH(jmp, &insn_dat[i].bj_jumps, bj_entries) { 928 1.1.2.2 yamt if (jmp->bj_safe_length < safe_length) 929 1.1.2.2 yamt safe_length = jmp->bj_safe_length; 930 1.1.2.2 yamt } 931 1.1.2.2 yamt } 932 1.1.2.2 yamt 933 1.1.2.2 yamt insn_dat[i].bj_unreachable = unreachable; 934 1.1.2.2 yamt if (unreachable) 935 1.1.2.2 yamt continue; 936 1.1.2.2 yamt 937 1.1.2.2 yamt if (read_pkt_insn(&insns[i], &length)) { 938 1.1.2.2 yamt if (first_read == SIZE_MAX) 939 1.1.2.2 yamt first_read = i; 940 1.1.2.2 yamt if (length > safe_length) 941 1.1.2.2 yamt safe_length = length; 942 1.1.2.2 yamt } 943 1.1.2.2 yamt 944 1.1.2.2 yamt switch (BPF_CLASS(insns[i].code)) { 945 1.1.2.2 yamt case BPF_RET: 946 1.1.2.2 yamt unreachable = true; 947 1.1.2.2 yamt continue; 948 1.1.2.2 yamt 949 1.1.2.2 yamt case BPF_JMP: 950 1.1.2.2 yamt if (insns[i].code == (BPF_JMP|BPF_JA)) { 951 1.1.2.2 yamt jt = jf = insns[i].k; 952 1.1.2.2 yamt } else { 953 1.1.2.2 yamt jt = insns[i].jt; 954 1.1.2.2 yamt jf = insns[i].jf; 955 1.1.2.2 yamt } 956 1.1.2.2 yamt 957 1.1.2.2 yamt if (jt >= insn_count - (i + 1) || 958 1.1.2.2 yamt jf >= insn_count - (i + 1)) { 959 1.1.2.2 yamt return -1; 960 1.1.2.2 yamt } 961 1.1.2.2 yamt 962 1.1.2.2 yamt if (jt > 0 && jf > 0) 963 1.1.2.2 yamt unreachable = true; 964 1.1.2.2 yamt 965 1.1.2.2 yamt jtf = insn_dat[i].bj_aux.bj_jdata.bj_jtf; 966 1.1.2.2 yamt 967 1.1.2.2 yamt jtf[0].bj_jump = NULL; 968 1.1.2.2 yamt jtf[0].bj_safe_length = safe_length; 969 1.1.2.2 yamt SLIST_INSERT_HEAD(&insn_dat[i + 1 + jt].bj_jumps, 970 1.1.2.2 yamt &jtf[0], bj_entries); 971 1.1.2.2 yamt 972 1.1.2.2 yamt if (jf != jt) { 973 1.1.2.2 yamt jtf[1].bj_jump = NULL; 974 1.1.2.2 yamt jtf[1].bj_safe_length = safe_length; 975 1.1.2.2 yamt SLIST_INSERT_HEAD(&insn_dat[i + 1 + jf].bj_jumps, 976 1.1.2.2 yamt &jtf[1], bj_entries); 977 1.1.2.2 yamt } 978 1.1.2.2 yamt 979 1.1.2.2 yamt continue; 980 1.1.2.2 yamt } 981 1.1.2.2 yamt } 982 1.1.2.2 yamt 983 1.1.2.2 yamt set_check_length(insns, insn_dat, first_read, insn_count, safe_length); 984 1.1.2.2 yamt 985 1.1.2.2 yamt return 0; 986 1.1.2.2 yamt } 987 1.1.2.2 yamt 988 1.1.2.2 yamt /* 989 1.1.2.2 yamt * Count out-of-bounds and division by zero jumps. 990 1.1.2.2 yamt * 991 1.1.2.2 yamt * insn_dat should be initialized by optimize(). 992 1.1.2.2 yamt */ 993 1.1.2.2 yamt static size_t 994 1.1.2.2 yamt get_ret0_size(struct bpf_insn *insns, struct bpfjit_insn_data *insn_dat, 995 1.1.2.2 yamt size_t insn_count) 996 1.1.2.2 yamt { 997 1.1.2.2 yamt size_t rv = 0; 998 1.1.2.2 yamt size_t i; 999 1.1.2.2 yamt 1000 1.1.2.2 yamt for (i = 0; i < insn_count; i++) { 1001 1.1.2.2 yamt 1002 1.1.2.2 yamt if (read_pkt_insn(&insns[i], NULL)) { 1003 1.1.2.2 yamt if (insn_dat[i].bj_aux.bj_rdata.bj_check_length > 0) 1004 1.1.2.2 yamt rv++; 1005 1.1.2.2 yamt #ifdef _KERNEL 1006 1.1.2.2 yamt rv++; 1007 1.1.2.2 yamt #endif 1008 1.1.2.2 yamt } 1009 1.1.2.2 yamt 1010 1.1.2.2 yamt if (insns[i].code == (BPF_LD|BPF_IND|BPF_B) || 1011 1.1.2.2 yamt insns[i].code == (BPF_LD|BPF_IND|BPF_H) || 1012 1.1.2.2 yamt insns[i].code == (BPF_LD|BPF_IND|BPF_W)) { 1013 1.1.2.2 yamt rv++; 1014 1.1.2.2 yamt } 1015 1.1.2.2 yamt 1016 1.1.2.2 yamt if (insns[i].code == (BPF_ALU|BPF_DIV|BPF_X)) 1017 1.1.2.2 yamt rv++; 1018 1.1.2.2 yamt 1019 1.1.2.2 yamt if (insns[i].code == (BPF_ALU|BPF_DIV|BPF_K) && 1020 1.1.2.2 yamt insns[i].k == 0) { 1021 1.1.2.2 yamt rv++; 1022 1.1.2.2 yamt } 1023 1.1.2.2 yamt } 1024 1.1.2.2 yamt 1025 1.1.2.2 yamt return rv; 1026 1.1.2.2 yamt } 1027 1.1.2.2 yamt 1028 1.1.2.2 yamt /* 1029 1.1.2.2 yamt * Convert BPF_ALU operations except BPF_NEG and BPF_DIV to sljit operation. 1030 1.1.2.2 yamt */ 1031 1.1.2.2 yamt static int 1032 1.1.2.2 yamt bpf_alu_to_sljit_op(struct bpf_insn *pc) 1033 1.1.2.2 yamt { 1034 1.1.2.2 yamt 1035 1.1.2.2 yamt /* 1036 1.1.2.2 yamt * Note: all supported 64bit arches have 32bit multiply 1037 1.1.2.2 yamt * instruction so SLJIT_INT_OP doesn't have any overhead. 1038 1.1.2.2 yamt */ 1039 1.1.2.2 yamt switch (BPF_OP(pc->code)) { 1040 1.1.2.2 yamt case BPF_ADD: return SLJIT_ADD; 1041 1.1.2.2 yamt case BPF_SUB: return SLJIT_SUB; 1042 1.1.2.2 yamt case BPF_MUL: return SLJIT_MUL|SLJIT_INT_OP; 1043 1.1.2.2 yamt case BPF_OR: return SLJIT_OR; 1044 1.1.2.2 yamt case BPF_AND: return SLJIT_AND; 1045 1.1.2.2 yamt case BPF_LSH: return SLJIT_SHL; 1046 1.1.2.2 yamt case BPF_RSH: return SLJIT_LSHR|SLJIT_INT_OP; 1047 1.1.2.2 yamt default: 1048 1.1.2.2 yamt BPFJIT_ASSERT(false); 1049 1.1.2.2 yamt return 0; 1050 1.1.2.2 yamt } 1051 1.1.2.2 yamt } 1052 1.1.2.2 yamt 1053 1.1.2.2 yamt /* 1054 1.1.2.2 yamt * Convert BPF_JMP operations except BPF_JA to sljit condition. 1055 1.1.2.2 yamt */ 1056 1.1.2.2 yamt static int 1057 1.1.2.2 yamt bpf_jmp_to_sljit_cond(struct bpf_insn *pc, bool negate) 1058 1.1.2.2 yamt { 1059 1.1.2.2 yamt /* 1060 1.1.2.2 yamt * Note: all supported 64bit arches have 32bit comparison 1061 1.1.2.2 yamt * instructions so SLJIT_INT_OP doesn't have any overhead. 1062 1.1.2.2 yamt */ 1063 1.1.2.2 yamt int rv = SLJIT_INT_OP; 1064 1.1.2.2 yamt 1065 1.1.2.2 yamt switch (BPF_OP(pc->code)) { 1066 1.1.2.2 yamt case BPF_JGT: 1067 1.1.2.2 yamt rv |= negate ? SLJIT_C_LESS_EQUAL : SLJIT_C_GREATER; 1068 1.1.2.2 yamt break; 1069 1.1.2.2 yamt case BPF_JGE: 1070 1.1.2.2 yamt rv |= negate ? SLJIT_C_LESS : SLJIT_C_GREATER_EQUAL; 1071 1.1.2.2 yamt break; 1072 1.1.2.2 yamt case BPF_JEQ: 1073 1.1.2.2 yamt rv |= negate ? SLJIT_C_NOT_EQUAL : SLJIT_C_EQUAL; 1074 1.1.2.2 yamt break; 1075 1.1.2.2 yamt case BPF_JSET: 1076 1.1.2.2 yamt rv |= negate ? SLJIT_C_EQUAL : SLJIT_C_NOT_EQUAL; 1077 1.1.2.2 yamt break; 1078 1.1.2.2 yamt default: 1079 1.1.2.2 yamt BPFJIT_ASSERT(false); 1080 1.1.2.2 yamt } 1081 1.1.2.2 yamt 1082 1.1.2.2 yamt return rv; 1083 1.1.2.2 yamt } 1084 1.1.2.2 yamt 1085 1.1.2.2 yamt static unsigned int 1086 1.1.2.2 yamt bpfjit_optimization_hints(struct bpf_insn *insns, size_t insn_count) 1087 1.1.2.2 yamt { 1088 1.1.2.2 yamt unsigned int rv = BPFJIT_INIT_A; 1089 1.1.2.2 yamt struct bpf_insn *pc; 1090 1.1.2.3 yamt unsigned int minm, maxm; 1091 1.1.2.2 yamt 1092 1.1.2.2 yamt BPFJIT_ASSERT(BPF_MEMWORDS - 1 <= 0xff); 1093 1.1.2.2 yamt 1094 1.1.2.2 yamt maxm = 0; 1095 1.1.2.2 yamt minm = BPF_MEMWORDS - 1; 1096 1.1.2.2 yamt 1097 1.1.2.2 yamt for (pc = insns; pc != insns + insn_count; pc++) { 1098 1.1.2.2 yamt switch (BPF_CLASS(pc->code)) { 1099 1.1.2.2 yamt case BPF_LD: 1100 1.1.2.2 yamt if (BPF_MODE(pc->code) == BPF_IND) 1101 1.1.2.2 yamt rv |= BPFJIT_INIT_X; 1102 1.1.2.2 yamt if (BPF_MODE(pc->code) == BPF_MEM && 1103 1.1.2.2 yamt (uint32_t)pc->k < BPF_MEMWORDS) { 1104 1.1.2.2 yamt if (pc->k > maxm) 1105 1.1.2.2 yamt maxm = pc->k; 1106 1.1.2.2 yamt if (pc->k < minm) 1107 1.1.2.2 yamt minm = pc->k; 1108 1.1.2.2 yamt } 1109 1.1.2.2 yamt continue; 1110 1.1.2.2 yamt case BPF_LDX: 1111 1.1.2.2 yamt rv |= BPFJIT_INIT_X; 1112 1.1.2.2 yamt if (BPF_MODE(pc->code) == BPF_MEM && 1113 1.1.2.2 yamt (uint32_t)pc->k < BPF_MEMWORDS) { 1114 1.1.2.2 yamt if (pc->k > maxm) 1115 1.1.2.2 yamt maxm = pc->k; 1116 1.1.2.2 yamt if (pc->k < minm) 1117 1.1.2.2 yamt minm = pc->k; 1118 1.1.2.2 yamt } 1119 1.1.2.2 yamt continue; 1120 1.1.2.2 yamt case BPF_ST: 1121 1.1.2.2 yamt if ((uint32_t)pc->k < BPF_MEMWORDS) { 1122 1.1.2.2 yamt if (pc->k > maxm) 1123 1.1.2.2 yamt maxm = pc->k; 1124 1.1.2.2 yamt if (pc->k < minm) 1125 1.1.2.2 yamt minm = pc->k; 1126 1.1.2.2 yamt } 1127 1.1.2.2 yamt continue; 1128 1.1.2.2 yamt case BPF_STX: 1129 1.1.2.2 yamt rv |= BPFJIT_INIT_X; 1130 1.1.2.2 yamt if ((uint32_t)pc->k < BPF_MEMWORDS) { 1131 1.1.2.2 yamt if (pc->k > maxm) 1132 1.1.2.2 yamt maxm = pc->k; 1133 1.1.2.2 yamt if (pc->k < minm) 1134 1.1.2.2 yamt minm = pc->k; 1135 1.1.2.2 yamt } 1136 1.1.2.2 yamt continue; 1137 1.1.2.2 yamt case BPF_ALU: 1138 1.1.2.2 yamt if (pc->code == (BPF_ALU|BPF_NEG)) 1139 1.1.2.2 yamt continue; 1140 1.1.2.2 yamt if (BPF_SRC(pc->code) == BPF_X) 1141 1.1.2.2 yamt rv |= BPFJIT_INIT_X; 1142 1.1.2.2 yamt continue; 1143 1.1.2.2 yamt case BPF_JMP: 1144 1.1.2.2 yamt if (pc->code == (BPF_JMP|BPF_JA)) 1145 1.1.2.2 yamt continue; 1146 1.1.2.2 yamt if (BPF_SRC(pc->code) == BPF_X) 1147 1.1.2.2 yamt rv |= BPFJIT_INIT_X; 1148 1.1.2.2 yamt continue; 1149 1.1.2.2 yamt case BPF_RET: 1150 1.1.2.2 yamt continue; 1151 1.1.2.2 yamt case BPF_MISC: 1152 1.1.2.2 yamt rv |= BPFJIT_INIT_X; 1153 1.1.2.2 yamt continue; 1154 1.1.2.2 yamt default: 1155 1.1.2.2 yamt BPFJIT_ASSERT(false); 1156 1.1.2.2 yamt } 1157 1.1.2.2 yamt } 1158 1.1.2.2 yamt 1159 1.1.2.2 yamt return rv | (maxm << 8) | minm; 1160 1.1.2.2 yamt } 1161 1.1.2.2 yamt 1162 1.1.2.2 yamt /* 1163 1.1.2.2 yamt * Convert BPF_K and BPF_X to sljit register. 1164 1.1.2.2 yamt */ 1165 1.1.2.2 yamt static int 1166 1.1.2.2 yamt kx_to_reg(struct bpf_insn *pc) 1167 1.1.2.2 yamt { 1168 1.1.2.2 yamt 1169 1.1.2.2 yamt switch (BPF_SRC(pc->code)) { 1170 1.1.2.2 yamt case BPF_K: return SLJIT_IMM; 1171 1.1.2.2 yamt case BPF_X: return BPFJIT_X; 1172 1.1.2.2 yamt default: 1173 1.1.2.2 yamt BPFJIT_ASSERT(false); 1174 1.1.2.2 yamt return 0; 1175 1.1.2.2 yamt } 1176 1.1.2.2 yamt } 1177 1.1.2.2 yamt 1178 1.1.2.2 yamt static sljit_w 1179 1.1.2.2 yamt kx_to_reg_arg(struct bpf_insn *pc) 1180 1.1.2.2 yamt { 1181 1.1.2.2 yamt 1182 1.1.2.2 yamt switch (BPF_SRC(pc->code)) { 1183 1.1.2.2 yamt case BPF_K: return (uint32_t)pc->k; /* SLJIT_IMM, pc->k, */ 1184 1.1.2.2 yamt case BPF_X: return 0; /* BPFJIT_X, 0, */ 1185 1.1.2.2 yamt default: 1186 1.1.2.2 yamt BPFJIT_ASSERT(false); 1187 1.1.2.2 yamt return 0; 1188 1.1.2.2 yamt } 1189 1.1.2.2 yamt } 1190 1.1.2.2 yamt 1191 1.1.2.4 yamt bpfjit_func_t 1192 1.1.2.4 yamt bpfjit_generate_code(bpf_ctx_t *bc, struct bpf_insn *insns, size_t insn_count) 1193 1.1.2.2 yamt { 1194 1.1.2.2 yamt void *rv; 1195 1.1.2.2 yamt size_t i; 1196 1.1.2.2 yamt int status; 1197 1.1.2.2 yamt int branching, negate; 1198 1.1.2.2 yamt unsigned int rval, mode, src; 1199 1.1.2.2 yamt int ntmp; 1200 1.1.2.3 yamt unsigned int locals_size; 1201 1.1.2.3 yamt unsigned int minm, maxm; /* min/max k for M[k] */ 1202 1.1.2.3 yamt size_t mem_locals_start; /* start of M[] array */ 1203 1.1.2.2 yamt unsigned int opts; 1204 1.1.2.2 yamt struct bpf_insn *pc; 1205 1.1.2.2 yamt struct sljit_compiler* compiler; 1206 1.1.2.2 yamt 1207 1.1.2.2 yamt /* a list of jumps to a normal return from a generated function */ 1208 1.1.2.2 yamt struct sljit_jump **returns; 1209 1.1.2.2 yamt size_t returns_size, returns_maxsize; 1210 1.1.2.2 yamt 1211 1.1.2.2 yamt /* a list of jumps to out-of-bound return from a generated function */ 1212 1.1.2.2 yamt struct sljit_jump **ret0; 1213 1.1.2.4 yamt size_t ret0_size = 0, ret0_maxsize = 0; 1214 1.1.2.2 yamt 1215 1.1.2.2 yamt struct bpfjit_insn_data *insn_dat; 1216 1.1.2.2 yamt 1217 1.1.2.2 yamt /* for local use */ 1218 1.1.2.2 yamt struct sljit_label *label; 1219 1.1.2.2 yamt struct sljit_jump *jump; 1220 1.1.2.2 yamt struct bpfjit_jump *bjump, *jtf; 1221 1.1.2.2 yamt 1222 1.1.2.2 yamt struct sljit_jump *to_mchain_jump; 1223 1.1.2.2 yamt 1224 1.1.2.2 yamt uint32_t jt, jf; 1225 1.1.2.2 yamt 1226 1.1.2.2 yamt rv = NULL; 1227 1.1.2.2 yamt compiler = NULL; 1228 1.1.2.2 yamt insn_dat = NULL; 1229 1.1.2.2 yamt returns = NULL; 1230 1.1.2.2 yamt ret0 = NULL; 1231 1.1.2.2 yamt 1232 1.1.2.2 yamt opts = bpfjit_optimization_hints(insns, insn_count); 1233 1.1.2.2 yamt minm = opts & 0xff; 1234 1.1.2.2 yamt maxm = (opts >> 8) & 0xff; 1235 1.1.2.2 yamt mem_locals_start = mem_local_offset(0, 0); 1236 1.1.2.2 yamt locals_size = (minm <= maxm) ? 1237 1.1.2.2 yamt mem_local_offset(maxm + 1, minm) : mem_locals_start; 1238 1.1.2.2 yamt 1239 1.1.2.2 yamt ntmp = 4; 1240 1.1.2.2 yamt #ifdef _KERNEL 1241 1.1.2.2 yamt ntmp += 1; /* for BPFJIT_KERN_TMP */ 1242 1.1.2.2 yamt #endif 1243 1.1.2.2 yamt 1244 1.1.2.2 yamt returns_maxsize = count_returns(insns, insn_count); 1245 1.1.2.2 yamt if (returns_maxsize == 0) 1246 1.1.2.2 yamt goto fail; 1247 1.1.2.2 yamt 1248 1.1.2.4 yamt insn_dat = BPFJIT_ALLOC(insn_count * sizeof(insn_dat[0])); 1249 1.1.2.2 yamt if (insn_dat == NULL) 1250 1.1.2.2 yamt goto fail; 1251 1.1.2.2 yamt 1252 1.1.2.2 yamt if (optimize(insns, insn_dat, insn_count) < 0) 1253 1.1.2.2 yamt goto fail; 1254 1.1.2.2 yamt 1255 1.1.2.2 yamt ret0_size = 0; 1256 1.1.2.2 yamt ret0_maxsize = get_ret0_size(insns, insn_dat, insn_count); 1257 1.1.2.2 yamt if (ret0_maxsize > 0) { 1258 1.1.2.4 yamt ret0 = BPFJIT_ALLOC(ret0_maxsize * sizeof(ret0[0])); 1259 1.1.2.2 yamt if (ret0 == NULL) 1260 1.1.2.2 yamt goto fail; 1261 1.1.2.2 yamt } 1262 1.1.2.2 yamt 1263 1.1.2.2 yamt returns_size = 0; 1264 1.1.2.4 yamt returns = BPFJIT_ALLOC(returns_maxsize * sizeof(returns[0])); 1265 1.1.2.2 yamt if (returns == NULL) 1266 1.1.2.2 yamt goto fail; 1267 1.1.2.2 yamt 1268 1.1.2.2 yamt compiler = sljit_create_compiler(); 1269 1.1.2.2 yamt if (compiler == NULL) 1270 1.1.2.2 yamt goto fail; 1271 1.1.2.2 yamt 1272 1.1.2.2 yamt #if !defined(_KERNEL) && defined(SLJIT_VERBOSE) && SLJIT_VERBOSE 1273 1.1.2.2 yamt sljit_compiler_verbose(compiler, stderr); 1274 1.1.2.2 yamt #endif 1275 1.1.2.2 yamt 1276 1.1.2.2 yamt status = sljit_emit_enter(compiler, 3, ntmp, 3, locals_size); 1277 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 1278 1.1.2.2 yamt goto fail; 1279 1.1.2.2 yamt 1280 1.1.2.2 yamt for (i = mem_locals_start; i < locals_size; i+= sizeof(uint32_t)) { 1281 1.1.2.2 yamt status = sljit_emit_op1(compiler, 1282 1.1.2.2 yamt SLJIT_MOV_UI, 1283 1.1.2.2 yamt SLJIT_MEM1(SLJIT_LOCALS_REG), i, 1284 1.1.2.2 yamt SLJIT_IMM, 0); 1285 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 1286 1.1.2.2 yamt goto fail; 1287 1.1.2.2 yamt } 1288 1.1.2.2 yamt 1289 1.1.2.2 yamt if (opts & BPFJIT_INIT_A) { 1290 1.1.2.2 yamt /* A = 0; */ 1291 1.1.2.2 yamt status = sljit_emit_op1(compiler, 1292 1.1.2.2 yamt SLJIT_MOV, 1293 1.1.2.2 yamt BPFJIT_A, 0, 1294 1.1.2.2 yamt SLJIT_IMM, 0); 1295 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 1296 1.1.2.2 yamt goto fail; 1297 1.1.2.2 yamt } 1298 1.1.2.2 yamt 1299 1.1.2.2 yamt if (opts & BPFJIT_INIT_X) { 1300 1.1.2.2 yamt /* X = 0; */ 1301 1.1.2.2 yamt status = sljit_emit_op1(compiler, 1302 1.1.2.2 yamt SLJIT_MOV, 1303 1.1.2.2 yamt BPFJIT_X, 0, 1304 1.1.2.2 yamt SLJIT_IMM, 0); 1305 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 1306 1.1.2.2 yamt goto fail; 1307 1.1.2.2 yamt } 1308 1.1.2.2 yamt 1309 1.1.2.2 yamt for (i = 0; i < insn_count; i++) { 1310 1.1.2.2 yamt if (insn_dat[i].bj_unreachable) 1311 1.1.2.2 yamt continue; 1312 1.1.2.2 yamt 1313 1.1.2.2 yamt to_mchain_jump = NULL; 1314 1.1.2.2 yamt 1315 1.1.2.2 yamt /* 1316 1.1.2.2 yamt * Resolve jumps to the current insn. 1317 1.1.2.2 yamt */ 1318 1.1.2.2 yamt label = NULL; 1319 1.1.2.2 yamt SLIST_FOREACH(bjump, &insn_dat[i].bj_jumps, bj_entries) { 1320 1.1.2.2 yamt if (bjump->bj_jump != NULL) { 1321 1.1.2.2 yamt if (label == NULL) 1322 1.1.2.2 yamt label = sljit_emit_label(compiler); 1323 1.1.2.2 yamt if (label == NULL) 1324 1.1.2.2 yamt goto fail; 1325 1.1.2.2 yamt sljit_set_label(bjump->bj_jump, label); 1326 1.1.2.2 yamt } 1327 1.1.2.2 yamt } 1328 1.1.2.2 yamt 1329 1.1.2.2 yamt if (read_pkt_insn(&insns[i], NULL) && 1330 1.1.2.2 yamt insn_dat[i].bj_aux.bj_rdata.bj_check_length > 0) { 1331 1.1.2.2 yamt /* if (buflen < bj_check_length) return 0; */ 1332 1.1.2.2 yamt jump = sljit_emit_cmp(compiler, 1333 1.1.2.2 yamt SLJIT_C_LESS, 1334 1.1.2.2 yamt BPFJIT_BUFLEN, 0, 1335 1.1.2.2 yamt SLJIT_IMM, 1336 1.1.2.2 yamt insn_dat[i].bj_aux.bj_rdata.bj_check_length); 1337 1.1.2.2 yamt if (jump == NULL) 1338 1.1.2.2 yamt goto fail; 1339 1.1.2.2 yamt #ifdef _KERNEL 1340 1.1.2.2 yamt to_mchain_jump = jump; 1341 1.1.2.2 yamt #else 1342 1.1.2.2 yamt ret0[ret0_size++] = jump; 1343 1.1.2.2 yamt #endif 1344 1.1.2.2 yamt } 1345 1.1.2.2 yamt 1346 1.1.2.2 yamt pc = &insns[i]; 1347 1.1.2.2 yamt switch (BPF_CLASS(pc->code)) { 1348 1.1.2.2 yamt 1349 1.1.2.2 yamt default: 1350 1.1.2.2 yamt goto fail; 1351 1.1.2.2 yamt 1352 1.1.2.2 yamt case BPF_LD: 1353 1.1.2.2 yamt /* BPF_LD+BPF_IMM A <- k */ 1354 1.1.2.2 yamt if (pc->code == (BPF_LD|BPF_IMM)) { 1355 1.1.2.2 yamt status = sljit_emit_op1(compiler, 1356 1.1.2.2 yamt SLJIT_MOV, 1357 1.1.2.2 yamt BPFJIT_A, 0, 1358 1.1.2.2 yamt SLJIT_IMM, (uint32_t)pc->k); 1359 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 1360 1.1.2.2 yamt goto fail; 1361 1.1.2.2 yamt 1362 1.1.2.2 yamt continue; 1363 1.1.2.2 yamt } 1364 1.1.2.2 yamt 1365 1.1.2.2 yamt /* BPF_LD+BPF_MEM A <- M[k] */ 1366 1.1.2.2 yamt if (pc->code == (BPF_LD|BPF_MEM)) { 1367 1.1.2.2 yamt if (pc->k < minm || pc->k > maxm) 1368 1.1.2.2 yamt goto fail; 1369 1.1.2.2 yamt status = sljit_emit_op1(compiler, 1370 1.1.2.2 yamt SLJIT_MOV_UI, 1371 1.1.2.2 yamt BPFJIT_A, 0, 1372 1.1.2.2 yamt SLJIT_MEM1(SLJIT_LOCALS_REG), 1373 1.1.2.2 yamt mem_local_offset(pc->k, minm)); 1374 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 1375 1.1.2.2 yamt goto fail; 1376 1.1.2.2 yamt 1377 1.1.2.2 yamt continue; 1378 1.1.2.2 yamt } 1379 1.1.2.2 yamt 1380 1.1.2.2 yamt /* BPF_LD+BPF_W+BPF_LEN A <- len */ 1381 1.1.2.2 yamt if (pc->code == (BPF_LD|BPF_W|BPF_LEN)) { 1382 1.1.2.2 yamt status = sljit_emit_op1(compiler, 1383 1.1.2.2 yamt SLJIT_MOV, 1384 1.1.2.2 yamt BPFJIT_A, 0, 1385 1.1.2.2 yamt BPFJIT_WIRELEN, 0); 1386 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 1387 1.1.2.2 yamt goto fail; 1388 1.1.2.2 yamt 1389 1.1.2.2 yamt continue; 1390 1.1.2.2 yamt } 1391 1.1.2.2 yamt 1392 1.1.2.2 yamt mode = BPF_MODE(pc->code); 1393 1.1.2.2 yamt if (mode != BPF_ABS && mode != BPF_IND) 1394 1.1.2.2 yamt goto fail; 1395 1.1.2.2 yamt 1396 1.1.2.2 yamt status = emit_pkt_read(compiler, pc, 1397 1.1.2.2 yamt to_mchain_jump, ret0, &ret0_size); 1398 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 1399 1.1.2.2 yamt goto fail; 1400 1.1.2.2 yamt 1401 1.1.2.2 yamt continue; 1402 1.1.2.2 yamt 1403 1.1.2.2 yamt case BPF_LDX: 1404 1.1.2.2 yamt mode = BPF_MODE(pc->code); 1405 1.1.2.2 yamt 1406 1.1.2.2 yamt /* BPF_LDX+BPF_W+BPF_IMM X <- k */ 1407 1.1.2.2 yamt if (mode == BPF_IMM) { 1408 1.1.2.2 yamt if (BPF_SIZE(pc->code) != BPF_W) 1409 1.1.2.2 yamt goto fail; 1410 1.1.2.2 yamt status = sljit_emit_op1(compiler, 1411 1.1.2.2 yamt SLJIT_MOV, 1412 1.1.2.2 yamt BPFJIT_X, 0, 1413 1.1.2.2 yamt SLJIT_IMM, (uint32_t)pc->k); 1414 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 1415 1.1.2.2 yamt goto fail; 1416 1.1.2.2 yamt 1417 1.1.2.2 yamt continue; 1418 1.1.2.2 yamt } 1419 1.1.2.2 yamt 1420 1.1.2.2 yamt /* BPF_LDX+BPF_W+BPF_LEN X <- len */ 1421 1.1.2.2 yamt if (mode == BPF_LEN) { 1422 1.1.2.2 yamt if (BPF_SIZE(pc->code) != BPF_W) 1423 1.1.2.2 yamt goto fail; 1424 1.1.2.2 yamt status = sljit_emit_op1(compiler, 1425 1.1.2.2 yamt SLJIT_MOV, 1426 1.1.2.2 yamt BPFJIT_X, 0, 1427 1.1.2.2 yamt BPFJIT_WIRELEN, 0); 1428 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 1429 1.1.2.2 yamt goto fail; 1430 1.1.2.2 yamt 1431 1.1.2.2 yamt continue; 1432 1.1.2.2 yamt } 1433 1.1.2.2 yamt 1434 1.1.2.2 yamt /* BPF_LDX+BPF_W+BPF_MEM X <- M[k] */ 1435 1.1.2.2 yamt if (mode == BPF_MEM) { 1436 1.1.2.2 yamt if (BPF_SIZE(pc->code) != BPF_W) 1437 1.1.2.2 yamt goto fail; 1438 1.1.2.2 yamt if (pc->k < minm || pc->k > maxm) 1439 1.1.2.2 yamt goto fail; 1440 1.1.2.2 yamt status = sljit_emit_op1(compiler, 1441 1.1.2.2 yamt SLJIT_MOV_UI, 1442 1.1.2.2 yamt BPFJIT_X, 0, 1443 1.1.2.2 yamt SLJIT_MEM1(SLJIT_LOCALS_REG), 1444 1.1.2.2 yamt mem_local_offset(pc->k, minm)); 1445 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 1446 1.1.2.2 yamt goto fail; 1447 1.1.2.2 yamt 1448 1.1.2.2 yamt continue; 1449 1.1.2.2 yamt } 1450 1.1.2.2 yamt 1451 1.1.2.2 yamt /* BPF_LDX+BPF_B+BPF_MSH X <- 4*(P[k:1]&0xf) */ 1452 1.1.2.2 yamt if (mode != BPF_MSH || BPF_SIZE(pc->code) != BPF_B) 1453 1.1.2.2 yamt goto fail; 1454 1.1.2.2 yamt 1455 1.1.2.2 yamt status = emit_msh(compiler, pc, 1456 1.1.2.2 yamt to_mchain_jump, ret0, &ret0_size); 1457 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 1458 1.1.2.2 yamt goto fail; 1459 1.1.2.2 yamt 1460 1.1.2.2 yamt continue; 1461 1.1.2.2 yamt 1462 1.1.2.2 yamt case BPF_ST: 1463 1.1.2.2 yamt if (pc->code != BPF_ST || pc->k < minm || pc->k > maxm) 1464 1.1.2.2 yamt goto fail; 1465 1.1.2.2 yamt 1466 1.1.2.2 yamt status = sljit_emit_op1(compiler, 1467 1.1.2.2 yamt SLJIT_MOV_UI, 1468 1.1.2.2 yamt SLJIT_MEM1(SLJIT_LOCALS_REG), 1469 1.1.2.2 yamt mem_local_offset(pc->k, minm), 1470 1.1.2.2 yamt BPFJIT_A, 0); 1471 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 1472 1.1.2.2 yamt goto fail; 1473 1.1.2.2 yamt 1474 1.1.2.2 yamt continue; 1475 1.1.2.2 yamt 1476 1.1.2.2 yamt case BPF_STX: 1477 1.1.2.2 yamt if (pc->code != BPF_STX || pc->k < minm || pc->k > maxm) 1478 1.1.2.2 yamt goto fail; 1479 1.1.2.2 yamt 1480 1.1.2.2 yamt status = sljit_emit_op1(compiler, 1481 1.1.2.2 yamt SLJIT_MOV_UI, 1482 1.1.2.2 yamt SLJIT_MEM1(SLJIT_LOCALS_REG), 1483 1.1.2.2 yamt mem_local_offset(pc->k, minm), 1484 1.1.2.2 yamt BPFJIT_X, 0); 1485 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 1486 1.1.2.2 yamt goto fail; 1487 1.1.2.2 yamt 1488 1.1.2.2 yamt continue; 1489 1.1.2.2 yamt 1490 1.1.2.2 yamt case BPF_ALU: 1491 1.1.2.2 yamt 1492 1.1.2.2 yamt if (pc->code == (BPF_ALU|BPF_NEG)) { 1493 1.1.2.2 yamt status = sljit_emit_op1(compiler, 1494 1.1.2.2 yamt SLJIT_NEG, 1495 1.1.2.2 yamt BPFJIT_A, 0, 1496 1.1.2.2 yamt BPFJIT_A, 0); 1497 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 1498 1.1.2.2 yamt goto fail; 1499 1.1.2.2 yamt 1500 1.1.2.2 yamt continue; 1501 1.1.2.2 yamt } 1502 1.1.2.2 yamt 1503 1.1.2.2 yamt if (BPF_OP(pc->code) != BPF_DIV) { 1504 1.1.2.2 yamt status = sljit_emit_op2(compiler, 1505 1.1.2.2 yamt bpf_alu_to_sljit_op(pc), 1506 1.1.2.2 yamt BPFJIT_A, 0, 1507 1.1.2.2 yamt BPFJIT_A, 0, 1508 1.1.2.2 yamt kx_to_reg(pc), kx_to_reg_arg(pc)); 1509 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 1510 1.1.2.2 yamt goto fail; 1511 1.1.2.2 yamt 1512 1.1.2.2 yamt continue; 1513 1.1.2.2 yamt } 1514 1.1.2.2 yamt 1515 1.1.2.2 yamt /* BPF_DIV */ 1516 1.1.2.2 yamt 1517 1.1.2.2 yamt src = BPF_SRC(pc->code); 1518 1.1.2.2 yamt if (src != BPF_X && src != BPF_K) 1519 1.1.2.2 yamt goto fail; 1520 1.1.2.2 yamt 1521 1.1.2.2 yamt /* division by zero? */ 1522 1.1.2.2 yamt if (src == BPF_X) { 1523 1.1.2.2 yamt jump = sljit_emit_cmp(compiler, 1524 1.1.2.2 yamt SLJIT_C_EQUAL|SLJIT_INT_OP, 1525 1.1.2.2 yamt BPFJIT_X, 0, 1526 1.1.2.2 yamt SLJIT_IMM, 0); 1527 1.1.2.2 yamt if (jump == NULL) 1528 1.1.2.2 yamt goto fail; 1529 1.1.2.2 yamt ret0[ret0_size++] = jump; 1530 1.1.2.2 yamt } else if (pc->k == 0) { 1531 1.1.2.2 yamt jump = sljit_emit_jump(compiler, SLJIT_JUMP); 1532 1.1.2.2 yamt if (jump == NULL) 1533 1.1.2.2 yamt goto fail; 1534 1.1.2.2 yamt ret0[ret0_size++] = jump; 1535 1.1.2.2 yamt } 1536 1.1.2.2 yamt 1537 1.1.2.2 yamt if (src == BPF_X) { 1538 1.1.2.2 yamt status = emit_division(compiler, BPFJIT_X, 0); 1539 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 1540 1.1.2.2 yamt goto fail; 1541 1.1.2.2 yamt } else if (pc->k != 0) { 1542 1.1.2.2 yamt if (pc->k & (pc->k - 1)) { 1543 1.1.2.2 yamt status = emit_division(compiler, 1544 1.1.2.2 yamt SLJIT_IMM, (uint32_t)pc->k); 1545 1.1.2.2 yamt } else { 1546 1.1.2.2 yamt status = emit_pow2_division(compiler, 1547 1.1.2.2 yamt (uint32_t)pc->k); 1548 1.1.2.2 yamt } 1549 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 1550 1.1.2.2 yamt goto fail; 1551 1.1.2.2 yamt } 1552 1.1.2.2 yamt 1553 1.1.2.2 yamt continue; 1554 1.1.2.2 yamt 1555 1.1.2.2 yamt case BPF_JMP: 1556 1.1.2.2 yamt 1557 1.1.2.2 yamt if (pc->code == (BPF_JMP|BPF_JA)) { 1558 1.1.2.2 yamt jt = jf = pc->k; 1559 1.1.2.2 yamt } else { 1560 1.1.2.2 yamt jt = pc->jt; 1561 1.1.2.2 yamt jf = pc->jf; 1562 1.1.2.2 yamt } 1563 1.1.2.2 yamt 1564 1.1.2.2 yamt negate = (jt == 0) ? 1 : 0; 1565 1.1.2.2 yamt branching = (jt == jf) ? 0 : 1; 1566 1.1.2.2 yamt jtf = insn_dat[i].bj_aux.bj_jdata.bj_jtf; 1567 1.1.2.2 yamt 1568 1.1.2.2 yamt if (branching) { 1569 1.1.2.2 yamt if (BPF_OP(pc->code) != BPF_JSET) { 1570 1.1.2.2 yamt jump = sljit_emit_cmp(compiler, 1571 1.1.2.2 yamt bpf_jmp_to_sljit_cond(pc, negate), 1572 1.1.2.2 yamt BPFJIT_A, 0, 1573 1.1.2.2 yamt kx_to_reg(pc), kx_to_reg_arg(pc)); 1574 1.1.2.2 yamt } else { 1575 1.1.2.2 yamt status = sljit_emit_op2(compiler, 1576 1.1.2.2 yamt SLJIT_AND, 1577 1.1.2.2 yamt BPFJIT_TMP1, 0, 1578 1.1.2.2 yamt BPFJIT_A, 0, 1579 1.1.2.2 yamt kx_to_reg(pc), kx_to_reg_arg(pc)); 1580 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 1581 1.1.2.2 yamt goto fail; 1582 1.1.2.2 yamt 1583 1.1.2.2 yamt jump = sljit_emit_cmp(compiler, 1584 1.1.2.2 yamt bpf_jmp_to_sljit_cond(pc, negate), 1585 1.1.2.2 yamt BPFJIT_TMP1, 0, 1586 1.1.2.2 yamt SLJIT_IMM, 0); 1587 1.1.2.2 yamt } 1588 1.1.2.2 yamt 1589 1.1.2.2 yamt if (jump == NULL) 1590 1.1.2.2 yamt goto fail; 1591 1.1.2.2 yamt 1592 1.1.2.2 yamt BPFJIT_ASSERT(jtf[negate].bj_jump == NULL); 1593 1.1.2.2 yamt jtf[negate].bj_jump = jump; 1594 1.1.2.2 yamt } 1595 1.1.2.2 yamt 1596 1.1.2.2 yamt if (!branching || (jt != 0 && jf != 0)) { 1597 1.1.2.2 yamt jump = sljit_emit_jump(compiler, SLJIT_JUMP); 1598 1.1.2.2 yamt if (jump == NULL) 1599 1.1.2.2 yamt goto fail; 1600 1.1.2.2 yamt 1601 1.1.2.2 yamt BPFJIT_ASSERT(jtf[branching].bj_jump == NULL); 1602 1.1.2.2 yamt jtf[branching].bj_jump = jump; 1603 1.1.2.2 yamt } 1604 1.1.2.2 yamt 1605 1.1.2.2 yamt continue; 1606 1.1.2.2 yamt 1607 1.1.2.2 yamt case BPF_RET: 1608 1.1.2.2 yamt 1609 1.1.2.2 yamt rval = BPF_RVAL(pc->code); 1610 1.1.2.2 yamt if (rval == BPF_X) 1611 1.1.2.2 yamt goto fail; 1612 1.1.2.2 yamt 1613 1.1.2.2 yamt /* BPF_RET+BPF_K accept k bytes */ 1614 1.1.2.2 yamt if (rval == BPF_K) { 1615 1.1.2.2 yamt status = sljit_emit_op1(compiler, 1616 1.1.2.2 yamt SLJIT_MOV, 1617 1.1.2.2 yamt BPFJIT_A, 0, 1618 1.1.2.2 yamt SLJIT_IMM, (uint32_t)pc->k); 1619 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 1620 1.1.2.2 yamt goto fail; 1621 1.1.2.2 yamt } 1622 1.1.2.2 yamt 1623 1.1.2.2 yamt /* BPF_RET+BPF_A accept A bytes */ 1624 1.1.2.2 yamt if (rval == BPF_A) { 1625 1.1.2.2 yamt #if BPFJIT_A != SLJIT_RETURN_REG 1626 1.1.2.2 yamt status = sljit_emit_op1(compiler, 1627 1.1.2.2 yamt SLJIT_MOV, 1628 1.1.2.2 yamt SLJIT_RETURN_REG, 0, 1629 1.1.2.2 yamt BPFJIT_A, 0); 1630 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 1631 1.1.2.2 yamt goto fail; 1632 1.1.2.2 yamt #endif 1633 1.1.2.2 yamt } 1634 1.1.2.2 yamt 1635 1.1.2.2 yamt /* 1636 1.1.2.2 yamt * Save a jump to a normal return. If the program 1637 1.1.2.2 yamt * ends with BPF_RET, no jump is needed because 1638 1.1.2.2 yamt * the normal return is generated right after the 1639 1.1.2.2 yamt * last instruction. 1640 1.1.2.2 yamt */ 1641 1.1.2.2 yamt if (i != insn_count - 1) { 1642 1.1.2.2 yamt jump = sljit_emit_jump(compiler, SLJIT_JUMP); 1643 1.1.2.2 yamt if (jump == NULL) 1644 1.1.2.2 yamt goto fail; 1645 1.1.2.2 yamt returns[returns_size++] = jump; 1646 1.1.2.2 yamt } 1647 1.1.2.2 yamt 1648 1.1.2.2 yamt continue; 1649 1.1.2.2 yamt 1650 1.1.2.2 yamt case BPF_MISC: 1651 1.1.2.2 yamt 1652 1.1.2.2 yamt if (pc->code == (BPF_MISC|BPF_TAX)) { 1653 1.1.2.2 yamt status = sljit_emit_op1(compiler, 1654 1.1.2.2 yamt SLJIT_MOV_UI, 1655 1.1.2.2 yamt BPFJIT_X, 0, 1656 1.1.2.2 yamt BPFJIT_A, 0); 1657 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 1658 1.1.2.2 yamt goto fail; 1659 1.1.2.2 yamt 1660 1.1.2.2 yamt continue; 1661 1.1.2.2 yamt } 1662 1.1.2.2 yamt 1663 1.1.2.2 yamt if (pc->code == (BPF_MISC|BPF_TXA)) { 1664 1.1.2.2 yamt status = sljit_emit_op1(compiler, 1665 1.1.2.2 yamt SLJIT_MOV, 1666 1.1.2.2 yamt BPFJIT_A, 0, 1667 1.1.2.2 yamt BPFJIT_X, 0); 1668 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 1669 1.1.2.2 yamt goto fail; 1670 1.1.2.2 yamt 1671 1.1.2.2 yamt continue; 1672 1.1.2.2 yamt } 1673 1.1.2.2 yamt 1674 1.1.2.2 yamt goto fail; 1675 1.1.2.2 yamt } /* switch */ 1676 1.1.2.2 yamt } /* main loop */ 1677 1.1.2.2 yamt 1678 1.1.2.2 yamt BPFJIT_ASSERT(ret0_size == ret0_maxsize); 1679 1.1.2.2 yamt BPFJIT_ASSERT(returns_size <= returns_maxsize); 1680 1.1.2.2 yamt 1681 1.1.2.2 yamt if (returns_size > 0) { 1682 1.1.2.2 yamt label = sljit_emit_label(compiler); 1683 1.1.2.2 yamt if (label == NULL) 1684 1.1.2.2 yamt goto fail; 1685 1.1.2.2 yamt for (i = 0; i < returns_size; i++) 1686 1.1.2.2 yamt sljit_set_label(returns[i], label); 1687 1.1.2.2 yamt } 1688 1.1.2.2 yamt 1689 1.1.2.2 yamt status = sljit_emit_return(compiler, 1690 1.1.2.2 yamt SLJIT_MOV_UI, 1691 1.1.2.2 yamt BPFJIT_A, 0); 1692 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 1693 1.1.2.2 yamt goto fail; 1694 1.1.2.2 yamt 1695 1.1.2.2 yamt if (ret0_size > 0) { 1696 1.1.2.2 yamt label = sljit_emit_label(compiler); 1697 1.1.2.2 yamt if (label == NULL) 1698 1.1.2.2 yamt goto fail; 1699 1.1.2.2 yamt 1700 1.1.2.2 yamt for (i = 0; i < ret0_size; i++) 1701 1.1.2.2 yamt sljit_set_label(ret0[i], label); 1702 1.1.2.2 yamt 1703 1.1.2.2 yamt status = sljit_emit_op1(compiler, 1704 1.1.2.2 yamt SLJIT_MOV, 1705 1.1.2.2 yamt SLJIT_RETURN_REG, 0, 1706 1.1.2.2 yamt SLJIT_IMM, 0); 1707 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 1708 1.1.2.2 yamt goto fail; 1709 1.1.2.2 yamt 1710 1.1.2.2 yamt status = sljit_emit_return(compiler, 1711 1.1.2.2 yamt SLJIT_MOV_UI, 1712 1.1.2.2 yamt SLJIT_RETURN_REG, 0); 1713 1.1.2.2 yamt if (status != SLJIT_SUCCESS) 1714 1.1.2.2 yamt goto fail; 1715 1.1.2.2 yamt } 1716 1.1.2.2 yamt 1717 1.1.2.2 yamt rv = sljit_generate_code(compiler); 1718 1.1.2.2 yamt 1719 1.1.2.2 yamt fail: 1720 1.1.2.2 yamt if (compiler != NULL) 1721 1.1.2.2 yamt sljit_free_compiler(compiler); 1722 1.1.2.2 yamt 1723 1.1.2.2 yamt if (insn_dat != NULL) 1724 1.1.2.4 yamt BPFJIT_FREE(insn_dat, insn_count * sizeof(insn_dat[0])); 1725 1.1.2.2 yamt 1726 1.1.2.2 yamt if (returns != NULL) 1727 1.1.2.4 yamt BPFJIT_FREE(returns, returns_maxsize * sizeof(returns[0])); 1728 1.1.2.2 yamt 1729 1.1.2.2 yamt if (ret0 != NULL) 1730 1.1.2.4 yamt BPFJIT_FREE(ret0, ret0_maxsize * sizeof(ret0[0])); 1731 1.1.2.2 yamt 1732 1.1.2.4 yamt return (bpfjit_func_t)rv; 1733 1.1.2.2 yamt } 1734 1.1.2.2 yamt 1735 1.1.2.2 yamt void 1736 1.1.2.4 yamt bpfjit_free_code(bpfjit_func_t code) 1737 1.1.2.2 yamt { 1738 1.1.2.2 yamt sljit_free_code((void *)code); 1739 1.1.2.2 yamt } 1740
Indexes created Thu Oct 02 10:09:58 GMT 2025