bpfjit.c revision 1.2.2.2
1 1.2.2.2 tls /*- 2 1.2.2.2 tls * Copyright (c) 2011-2012 Alexander Nasonov. 3 1.2.2.2 tls * All rights reserved. 4 1.2.2.2 tls * 5 1.2.2.2 tls * Redistribution and use in source and binary forms, with or without 6 1.2.2.2 tls * modification, are permitted provided that the following conditions 7 1.2.2.2 tls * are met: 8 1.2.2.2 tls * 9 1.2.2.2 tls * 1. Redistributions of source code must retain the above copyright 10 1.2.2.2 tls * notice, this list of conditions and the following disclaimer. 11 1.2.2.2 tls * 2. Redistributions in binary form must reproduce the above copyright 12 1.2.2.2 tls * notice, this list of conditions and the following disclaimer in 13 1.2.2.2 tls * the documentation and/or other materials provided with the 14 1.2.2.2 tls * distribution. 15 1.2.2.2 tls * 16 1.2.2.2 tls * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 1.2.2.2 tls * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 1.2.2.2 tls * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 19 1.2.2.2 tls * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 20 1.2.2.2 tls * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 21 1.2.2.2 tls * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 22 1.2.2.2 tls * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 23 1.2.2.2 tls * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 24 1.2.2.2 tls * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 25 1.2.2.2 tls * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 26 1.2.2.2 tls * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 1.2.2.2 tls * SUCH DAMAGE. 28 1.2.2.2 tls */ 29 1.2.2.2 tls 30 1.2.2.2 tls #include <sys/cdefs.h> 31 1.2.2.2 tls #ifdef _KERNEL 32 1.2.2.2 tls __KERNEL_RCSID(0, "$NetBSD: bpfjit.c,v 1.2.2.2 2012/11/20 03:02:46 tls Exp $"); 33 1.2.2.2 tls #else 34 1.2.2.2 tls __RCSID("$NetBSD: bpfjit.c,v 1.2.2.2 2012/11/20 03:02:46 tls Exp $"); 35 1.2.2.2 tls #endif 36 1.2.2.2 tls 37 1.2.2.2 tls #include <net/bpfjit.h> 38 1.2.2.2 tls 39 1.2.2.2 tls #ifndef _KERNEL 40 1.2.2.2 tls #include <assert.h> 41 1.2.2.2 tls #define BPFJIT_ASSERT(c) assert(c) 42 1.2.2.2 tls #else 43 1.2.2.2 tls #define BPFJIT_ASSERT(c) KASSERT(c) 44 1.2.2.2 tls #endif 45 1.2.2.2 tls 46 1.2.2.2 tls #ifndef _KERNEL 47 1.2.2.2 tls #include <stdlib.h> 48 1.2.2.2 tls #define BPFJIT_MALLOC(sz) malloc(sz) 49 1.2.2.2 tls #define BPFJIT_FREE(p) free(p) 50 1.2.2.2 tls #else 51 1.2.2.2 tls #include <sys/malloc.h> 52 1.2.2.2 tls #define BPFJIT_MALLOC(sz) kern_malloc(sz, M_WAITOK) 53 1.2.2.2 tls #define BPFJIT_FREE(p) kern_free(p) 54 1.2.2.2 tls #endif 55 1.2.2.2 tls 56 1.2.2.2 tls #ifndef _KERNEL 57 1.2.2.2 tls #include <limits.h> 58 1.2.2.2 tls #include <stdbool.h> 59 1.2.2.2 tls #include <stddef.h> 60 1.2.2.2 tls #include <stdint.h> 61 1.2.2.2 tls #else 62 1.2.2.2 tls #include <machine/limits.h> 63 1.2.2.2 tls #include <sys/null.h> 64 1.2.2.2 tls #include <sys/types.h> 65 1.2.2.2 tls #include <sys/atomic.h> 66 1.2.2.2 tls #include <sys/module.h> 67 1.2.2.2 tls #endif 68 1.2.2.2 tls 69 1.2.2.2 tls #include <sys/queue.h> 70 1.2.2.2 tls #include <sys/types.h> 71 1.2.2.2 tls 72 1.2.2.2 tls #include <sljitLir.h> 73 1.2.2.2 tls 74 1.2.2.2 tls #if !defined(_KERNEL) && defined(SLJIT_VERBOSE) && SLJIT_VERBOSE 75 1.2.2.2 tls #include <stdio.h> /* for stderr */ 76 1.2.2.2 tls #endif 77 1.2.2.2 tls 78 1.2.2.2 tls 79 1.2.2.2 tls #define BPFJIT_A SLJIT_TEMPORARY_REG1 80 1.2.2.2 tls #define BPFJIT_X SLJIT_TEMPORARY_EREG1 81 1.2.2.2 tls #define BPFJIT_TMP1 SLJIT_TEMPORARY_REG2 82 1.2.2.2 tls #define BPFJIT_TMP2 SLJIT_TEMPORARY_REG3 83 1.2.2.2 tls #define BPFJIT_BUF SLJIT_SAVED_REG1 84 1.2.2.2 tls #define BPFJIT_WIRELEN SLJIT_SAVED_REG2 85 1.2.2.2 tls #define BPFJIT_BUFLEN SLJIT_SAVED_REG3 86 1.2.2.2 tls #define BPFJIT_KERN_TMP SLJIT_TEMPORARY_EREG2 87 1.2.2.2 tls 88 1.2.2.2 tls /* 89 1.2.2.2 tls * Flags for bpfjit_optimization_hints(). 90 1.2.2.2 tls */ 91 1.2.2.2 tls #define BPFJIT_INIT_X 0x10000 92 1.2.2.2 tls #define BPFJIT_INIT_A 0x20000 93 1.2.2.2 tls 94 1.2.2.2 tls 95 1.2.2.2 tls /* 96 1.2.2.2 tls * Node of bj_jumps list. 97 1.2.2.2 tls */ 98 1.2.2.2 tls struct bpfjit_jump 99 1.2.2.2 tls { 100 1.2.2.2 tls struct sljit_jump *bj_jump; 101 1.2.2.2 tls SLIST_ENTRY(bpfjit_jump) bj_entries; 102 1.2.2.2 tls uint32_t bj_safe_length; 103 1.2.2.2 tls }; 104 1.2.2.2 tls 105 1.2.2.2 tls /* 106 1.2.2.2 tls * Data for BPF_JMP instruction. 107 1.2.2.2 tls */ 108 1.2.2.2 tls struct bpfjit_jump_data 109 1.2.2.2 tls { 110 1.2.2.2 tls /* 111 1.2.2.2 tls * These entries make up bj_jumps list: 112 1.2.2.2 tls * bj_jtf[0] - when coming from jt path, 113 1.2.2.2 tls * bj_jtf[1] - when coming from jf path. 114 1.2.2.2 tls */ 115 1.2.2.2 tls struct bpfjit_jump bj_jtf[2]; 116 1.2.2.2 tls }; 117 1.2.2.2 tls 118 1.2.2.2 tls /* 119 1.2.2.2 tls * Data for "read from packet" instructions. 120 1.2.2.2 tls * See also read_pkt_insn() function below. 121 1.2.2.2 tls */ 122 1.2.2.2 tls struct bpfjit_read_pkt_data 123 1.2.2.2 tls { 124 1.2.2.2 tls /* 125 1.2.2.2 tls * If positive, emit "if (buflen < bj_check_length) return 0". 126 1.2.2.2 tls * We assume that buflen is never equal to UINT32_MAX (otherwise, 127 1.2.2.2 tls * we need a special bool variable to emit unconditional "return 0"). 128 1.2.2.2 tls */ 129 1.2.2.2 tls uint32_t bj_check_length; 130 1.2.2.2 tls }; 131 1.2.2.2 tls 132 1.2.2.2 tls /* 133 1.2.2.2 tls * Additional (optimization-related) data for bpf_insn. 134 1.2.2.2 tls */ 135 1.2.2.2 tls struct bpfjit_insn_data 136 1.2.2.2 tls { 137 1.2.2.2 tls /* List of jumps to this insn. */ 138 1.2.2.2 tls SLIST_HEAD(, bpfjit_jump) bj_jumps; 139 1.2.2.2 tls 140 1.2.2.2 tls union { 141 1.2.2.2 tls struct bpfjit_jump_data bj_jdata; 142 1.2.2.2 tls struct bpfjit_read_pkt_data bj_rdata; 143 1.2.2.2 tls } bj_aux; 144 1.2.2.2 tls 145 1.2.2.2 tls bool bj_unreachable; 146 1.2.2.2 tls }; 147 1.2.2.2 tls 148 1.2.2.2 tls #ifdef _KERNEL 149 1.2.2.2 tls 150 1.2.2.2 tls uint32_t m_xword(const struct mbuf *, uint32_t, int *); 151 1.2.2.2 tls uint32_t m_xhalf(const struct mbuf *, uint32_t, int *); 152 1.2.2.2 tls uint32_t m_xbyte(const struct mbuf *, uint32_t, int *); 153 1.2.2.2 tls 154 1.2.2.2 tls MODULE(MODULE_CLASS_MISC, bpfjit, "sljit") 155 1.2.2.2 tls 156 1.2.2.2 tls static int 157 1.2.2.2 tls bpfjit_modcmd(modcmd_t cmd, void *arg) 158 1.2.2.2 tls { 159 1.2.2.2 tls 160 1.2.2.2 tls switch (cmd) { 161 1.2.2.2 tls case MODULE_CMD_INIT: 162 1.2.2.2 tls bpfjit_module_ops.bj_free_code = &bpfjit_free_code; 163 1.2.2.2 tls membar_producer(); 164 1.2.2.2 tls bpfjit_module_ops.bj_generate_code = &bpfjit_generate_code; 165 1.2.2.2 tls membar_producer(); 166 1.2.2.2 tls return 0; 167 1.2.2.2 tls 168 1.2.2.2 tls case MODULE_CMD_FINI: 169 1.2.2.2 tls return EOPNOTSUPP; 170 1.2.2.2 tls 171 1.2.2.2 tls default: 172 1.2.2.2 tls return ENOTTY; 173 1.2.2.2 tls } 174 1.2.2.2 tls } 175 1.2.2.2 tls #endif 176 1.2.2.2 tls 177 1.2.2.2 tls static uint32_t 178 1.2.2.2 tls read_width(struct bpf_insn *pc) 179 1.2.2.2 tls { 180 1.2.2.2 tls 181 1.2.2.2 tls switch (BPF_SIZE(pc->code)) { 182 1.2.2.2 tls case BPF_W: 183 1.2.2.2 tls return 4; 184 1.2.2.2 tls case BPF_H: 185 1.2.2.2 tls return 2; 186 1.2.2.2 tls case BPF_B: 187 1.2.2.2 tls return 1; 188 1.2.2.2 tls default: 189 1.2.2.2 tls BPFJIT_ASSERT(false); 190 1.2.2.2 tls return 0; 191 1.2.2.2 tls } 192 1.2.2.2 tls } 193 1.2.2.2 tls 194 1.2.2.2 tls /* 195 1.2.2.2 tls * Get offset of M[k] on the stack. 196 1.2.2.2 tls */ 197 1.2.2.2 tls static size_t 198 1.2.2.2 tls mem_local_offset(uint32_t k, unsigned int minm) 199 1.2.2.2 tls { 200 1.2.2.2 tls size_t moff = (k - minm) * sizeof(uint32_t); 201 1.2.2.2 tls 202 1.2.2.2 tls #ifdef _KERNEL 203 1.2.2.2 tls /* 204 1.2.2.2 tls * 4 bytes for the third argument of m_xword/m_xhalf/m_xbyte. 205 1.2.2.2 tls */ 206 1.2.2.2 tls return sizeof(uint32_t) + moff; 207 1.2.2.2 tls #else 208 1.2.2.2 tls return moff; 209 1.2.2.2 tls #endif 210 1.2.2.2 tls } 211 1.2.2.2 tls 212 1.2.2.2 tls /* 213 1.2.2.2 tls * Generate code for BPF_LD+BPF_B+BPF_ABS A <- P[k:1]. 214 1.2.2.2 tls */ 215 1.2.2.2 tls static int 216 1.2.2.2 tls emit_read8(struct sljit_compiler* compiler, uint32_t k) 217 1.2.2.2 tls { 218 1.2.2.2 tls 219 1.2.2.2 tls return sljit_emit_op1(compiler, 220 1.2.2.2 tls SLJIT_MOV_UB, 221 1.2.2.2 tls BPFJIT_A, 0, 222 1.2.2.2 tls SLJIT_MEM1(BPFJIT_BUF), k); 223 1.2.2.2 tls } 224 1.2.2.2 tls 225 1.2.2.2 tls /* 226 1.2.2.2 tls * Generate code for BPF_LD+BPF_H+BPF_ABS A <- P[k:2]. 227 1.2.2.2 tls */ 228 1.2.2.2 tls static int 229 1.2.2.2 tls emit_read16(struct sljit_compiler* compiler, uint32_t k) 230 1.2.2.2 tls { 231 1.2.2.2 tls int status; 232 1.2.2.2 tls 233 1.2.2.2 tls /* tmp1 = buf[k]; */ 234 1.2.2.2 tls status = sljit_emit_op1(compiler, 235 1.2.2.2 tls SLJIT_MOV_UB, 236 1.2.2.2 tls BPFJIT_TMP1, 0, 237 1.2.2.2 tls SLJIT_MEM1(BPFJIT_BUF), k); 238 1.2.2.2 tls if (status != SLJIT_SUCCESS) 239 1.2.2.2 tls return status; 240 1.2.2.2 tls 241 1.2.2.2 tls /* A = buf[k+1]; */ 242 1.2.2.2 tls status = sljit_emit_op1(compiler, 243 1.2.2.2 tls SLJIT_MOV_UB, 244 1.2.2.2 tls BPFJIT_A, 0, 245 1.2.2.2 tls SLJIT_MEM1(BPFJIT_BUF), k+1); 246 1.2.2.2 tls if (status != SLJIT_SUCCESS) 247 1.2.2.2 tls return status; 248 1.2.2.2 tls 249 1.2.2.2 tls /* tmp1 = tmp1 << 8; */ 250 1.2.2.2 tls status = sljit_emit_op2(compiler, 251 1.2.2.2 tls SLJIT_SHL, 252 1.2.2.2 tls BPFJIT_TMP1, 0, 253 1.2.2.2 tls BPFJIT_TMP1, 0, 254 1.2.2.2 tls SLJIT_IMM, 8); 255 1.2.2.2 tls if (status != SLJIT_SUCCESS) 256 1.2.2.2 tls return status; 257 1.2.2.2 tls 258 1.2.2.2 tls /* A = A + tmp1; */ 259 1.2.2.2 tls status = sljit_emit_op2(compiler, 260 1.2.2.2 tls SLJIT_ADD, 261 1.2.2.2 tls BPFJIT_A, 0, 262 1.2.2.2 tls BPFJIT_A, 0, 263 1.2.2.2 tls BPFJIT_TMP1, 0); 264 1.2.2.2 tls return status; 265 1.2.2.2 tls } 266 1.2.2.2 tls 267 1.2.2.2 tls /* 268 1.2.2.2 tls * Generate code for BPF_LD+BPF_W+BPF_ABS A <- P[k:4]. 269 1.2.2.2 tls */ 270 1.2.2.2 tls static int 271 1.2.2.2 tls emit_read32(struct sljit_compiler* compiler, uint32_t k) 272 1.2.2.2 tls { 273 1.2.2.2 tls int status; 274 1.2.2.2 tls 275 1.2.2.2 tls /* tmp1 = buf[k]; */ 276 1.2.2.2 tls status = sljit_emit_op1(compiler, 277 1.2.2.2 tls SLJIT_MOV_UB, 278 1.2.2.2 tls BPFJIT_TMP1, 0, 279 1.2.2.2 tls SLJIT_MEM1(BPFJIT_BUF), k); 280 1.2.2.2 tls if (status != SLJIT_SUCCESS) 281 1.2.2.2 tls return status; 282 1.2.2.2 tls 283 1.2.2.2 tls /* tmp2 = buf[k+1]; */ 284 1.2.2.2 tls status = sljit_emit_op1(compiler, 285 1.2.2.2 tls SLJIT_MOV_UB, 286 1.2.2.2 tls BPFJIT_TMP2, 0, 287 1.2.2.2 tls SLJIT_MEM1(BPFJIT_BUF), k+1); 288 1.2.2.2 tls if (status != SLJIT_SUCCESS) 289 1.2.2.2 tls return status; 290 1.2.2.2 tls 291 1.2.2.2 tls /* A = buf[k+3]; */ 292 1.2.2.2 tls status = sljit_emit_op1(compiler, 293 1.2.2.2 tls SLJIT_MOV_UB, 294 1.2.2.2 tls BPFJIT_A, 0, 295 1.2.2.2 tls SLJIT_MEM1(BPFJIT_BUF), k+3); 296 1.2.2.2 tls if (status != SLJIT_SUCCESS) 297 1.2.2.2 tls return status; 298 1.2.2.2 tls 299 1.2.2.2 tls /* tmp1 = tmp1 << 24; */ 300 1.2.2.2 tls status = sljit_emit_op2(compiler, 301 1.2.2.2 tls SLJIT_SHL, 302 1.2.2.2 tls BPFJIT_TMP1, 0, 303 1.2.2.2 tls BPFJIT_TMP1, 0, 304 1.2.2.2 tls SLJIT_IMM, 24); 305 1.2.2.2 tls if (status != SLJIT_SUCCESS) 306 1.2.2.2 tls return status; 307 1.2.2.2 tls 308 1.2.2.2 tls /* A = A + tmp1; */ 309 1.2.2.2 tls status = sljit_emit_op2(compiler, 310 1.2.2.2 tls SLJIT_ADD, 311 1.2.2.2 tls BPFJIT_A, 0, 312 1.2.2.2 tls BPFJIT_A, 0, 313 1.2.2.2 tls BPFJIT_TMP1, 0); 314 1.2.2.2 tls if (status != SLJIT_SUCCESS) 315 1.2.2.2 tls return status; 316 1.2.2.2 tls 317 1.2.2.2 tls /* tmp1 = buf[k+2]; */ 318 1.2.2.2 tls status = sljit_emit_op1(compiler, 319 1.2.2.2 tls SLJIT_MOV_UB, 320 1.2.2.2 tls BPFJIT_TMP1, 0, 321 1.2.2.2 tls SLJIT_MEM1(BPFJIT_BUF), k+2); 322 1.2.2.2 tls if (status != SLJIT_SUCCESS) 323 1.2.2.2 tls return status; 324 1.2.2.2 tls 325 1.2.2.2 tls /* tmp2 = tmp2 << 16; */ 326 1.2.2.2 tls status = sljit_emit_op2(compiler, 327 1.2.2.2 tls SLJIT_SHL, 328 1.2.2.2 tls BPFJIT_TMP2, 0, 329 1.2.2.2 tls BPFJIT_TMP2, 0, 330 1.2.2.2 tls SLJIT_IMM, 16); 331 1.2.2.2 tls if (status != SLJIT_SUCCESS) 332 1.2.2.2 tls return status; 333 1.2.2.2 tls 334 1.2.2.2 tls /* A = A + tmp2; */ 335 1.2.2.2 tls status = sljit_emit_op2(compiler, 336 1.2.2.2 tls SLJIT_ADD, 337 1.2.2.2 tls BPFJIT_A, 0, 338 1.2.2.2 tls BPFJIT_A, 0, 339 1.2.2.2 tls BPFJIT_TMP2, 0); 340 1.2.2.2 tls if (status != SLJIT_SUCCESS) 341 1.2.2.2 tls return status; 342 1.2.2.2 tls 343 1.2.2.2 tls /* tmp1 = tmp1 << 8; */ 344 1.2.2.2 tls status = sljit_emit_op2(compiler, 345 1.2.2.2 tls SLJIT_SHL, 346 1.2.2.2 tls BPFJIT_TMP1, 0, 347 1.2.2.2 tls BPFJIT_TMP1, 0, 348 1.2.2.2 tls SLJIT_IMM, 8); 349 1.2.2.2 tls if (status != SLJIT_SUCCESS) 350 1.2.2.2 tls return status; 351 1.2.2.2 tls 352 1.2.2.2 tls /* A = A + tmp1; */ 353 1.2.2.2 tls status = sljit_emit_op2(compiler, 354 1.2.2.2 tls SLJIT_ADD, 355 1.2.2.2 tls BPFJIT_A, 0, 356 1.2.2.2 tls BPFJIT_A, 0, 357 1.2.2.2 tls BPFJIT_TMP1, 0); 358 1.2.2.2 tls return status; 359 1.2.2.2 tls } 360 1.2.2.2 tls 361 1.2.2.2 tls #ifdef _KERNEL 362 1.2.2.2 tls /* 363 1.2.2.2 tls * Generate m_xword/m_xhalf/m_xbyte call. 364 1.2.2.2 tls * 365 1.2.2.2 tls * pc is one of: 366 1.2.2.2 tls * BPF_LD+BPF_W+BPF_ABS A <- P[k:4] 367 1.2.2.2 tls * BPF_LD+BPF_H+BPF_ABS A <- P[k:2] 368 1.2.2.2 tls * BPF_LD+BPF_B+BPF_ABS A <- P[k:1] 369 1.2.2.2 tls * BPF_LD+BPF_W+BPF_IND A <- P[X+k:4] 370 1.2.2.2 tls * BPF_LD+BPF_H+BPF_IND A <- P[X+k:2] 371 1.2.2.2 tls * BPF_LD+BPF_B+BPF_IND A <- P[X+k:1] 372 1.2.2.2 tls * BPF_LDX+BPF_B+BPF_MSH X <- 4*(P[k:1]&0xf) 373 1.2.2.2 tls * 374 1.2.2.2 tls * dst must be BPFJIT_A for BPF_LD instructions and BPFJIT_X 375 1.2.2.2 tls * or any of BPFJIT_TMP* registrers for BPF_MSH instruction. 376 1.2.2.2 tls */ 377 1.2.2.2 tls static int 378 1.2.2.2 tls emit_xcall(struct sljit_compiler* compiler, struct bpf_insn *pc, 379 1.2.2.2 tls int dst, sljit_w dstw, struct sljit_jump **ret0_jump, 380 1.2.2.2 tls uint32_t (*fn)(const struct mbuf *, uint32_t, int *)) 381 1.2.2.2 tls { 382 1.2.2.2 tls #if BPFJIT_X != SLJIT_TEMPORARY_EREG1 || \ 383 1.2.2.2 tls BPFJIT_X == SLJIT_RETURN_REG 384 1.2.2.2 tls #error "Not supported assignment of registers." 385 1.2.2.2 tls #endif 386 1.2.2.2 tls int status; 387 1.2.2.2 tls 388 1.2.2.2 tls /* 389 1.2.2.2 tls * The third argument of fn is an address on stack. 390 1.2.2.2 tls */ 391 1.2.2.2 tls const int arg3_offset = 0; 392 1.2.2.2 tls 393 1.2.2.2 tls if (BPF_CLASS(pc->code) == BPF_LDX) { 394 1.2.2.2 tls /* save A */ 395 1.2.2.2 tls status = sljit_emit_op1(compiler, 396 1.2.2.2 tls SLJIT_MOV, 397 1.2.2.2 tls BPFJIT_KERN_TMP, 0, 398 1.2.2.2 tls BPFJIT_A, 0); 399 1.2.2.2 tls if (status != SLJIT_SUCCESS) 400 1.2.2.2 tls return status; 401 1.2.2.2 tls } 402 1.2.2.2 tls 403 1.2.2.2 tls /* 404 1.2.2.2 tls * Prepare registers for fn(buf, k, &err) call. 405 1.2.2.2 tls */ 406 1.2.2.2 tls status = sljit_emit_op1(compiler, 407 1.2.2.2 tls SLJIT_MOV, 408 1.2.2.2 tls SLJIT_TEMPORARY_REG1, 0, 409 1.2.2.2 tls BPFJIT_BUF, 0); 410 1.2.2.2 tls if (status != SLJIT_SUCCESS) 411 1.2.2.2 tls return status; 412 1.2.2.2 tls 413 1.2.2.2 tls if (BPF_CLASS(pc->code) == BPF_LD && BPF_MODE(pc->code) == BPF_IND) { 414 1.2.2.2 tls status = sljit_emit_op2(compiler, 415 1.2.2.2 tls SLJIT_ADD, 416 1.2.2.2 tls SLJIT_TEMPORARY_REG2, 0, 417 1.2.2.2 tls BPFJIT_X, 0, 418 1.2.2.2 tls SLJIT_IMM, (uint32_t)pc->k); 419 1.2.2.2 tls } else { 420 1.2.2.2 tls status = sljit_emit_op1(compiler, 421 1.2.2.2 tls SLJIT_MOV, 422 1.2.2.2 tls SLJIT_TEMPORARY_REG2, 0, 423 1.2.2.2 tls SLJIT_IMM, (uint32_t)pc->k); 424 1.2.2.2 tls } 425 1.2.2.2 tls 426 1.2.2.2 tls if (status != SLJIT_SUCCESS) 427 1.2.2.2 tls return status; 428 1.2.2.2 tls 429 1.2.2.2 tls status = sljit_get_local_base(compiler, 430 1.2.2.2 tls SLJIT_TEMPORARY_REG3, 0, arg3_offset); 431 1.2.2.2 tls if (status != SLJIT_SUCCESS) 432 1.2.2.2 tls return status; 433 1.2.2.2 tls 434 1.2.2.2 tls /* fn(buf, k, &err); */ 435 1.2.2.2 tls status = sljit_emit_ijump(compiler, 436 1.2.2.2 tls SLJIT_CALL3, 437 1.2.2.2 tls SLJIT_IMM, SLJIT_FUNC_OFFSET(fn)); 438 1.2.2.2 tls 439 1.2.2.2 tls if (BPF_CLASS(pc->code) == BPF_LDX) { 440 1.2.2.2 tls 441 1.2.2.2 tls /* move return value to dst */ 442 1.2.2.2 tls BPFJIT_ASSERT(dst != SLJIT_RETURN_REG); 443 1.2.2.2 tls status = sljit_emit_op1(compiler, 444 1.2.2.2 tls SLJIT_MOV, 445 1.2.2.2 tls dst, dstw, 446 1.2.2.2 tls SLJIT_RETURN_REG, 0); 447 1.2.2.2 tls if (status != SLJIT_SUCCESS) 448 1.2.2.2 tls return status; 449 1.2.2.2 tls 450 1.2.2.2 tls /* restore A */ 451 1.2.2.2 tls status = sljit_emit_op1(compiler, 452 1.2.2.2 tls SLJIT_MOV, 453 1.2.2.2 tls BPFJIT_A, 0, 454 1.2.2.2 tls BPFJIT_KERN_TMP, 0); 455 1.2.2.2 tls if (status != SLJIT_SUCCESS) 456 1.2.2.2 tls return status; 457 1.2.2.2 tls 458 1.2.2.2 tls } else if (dst != SLJIT_RETURN_REG) { 459 1.2.2.2 tls status = sljit_emit_op1(compiler, 460 1.2.2.2 tls SLJIT_MOV, 461 1.2.2.2 tls dst, dstw, 462 1.2.2.2 tls SLJIT_RETURN_REG, 0); 463 1.2.2.2 tls if (status != SLJIT_SUCCESS) 464 1.2.2.2 tls return status; 465 1.2.2.2 tls } 466 1.2.2.2 tls 467 1.2.2.2 tls /* tmp3 = *err; */ 468 1.2.2.2 tls status = sljit_emit_op1(compiler, 469 1.2.2.2 tls SLJIT_MOV_UI, 470 1.2.2.2 tls SLJIT_TEMPORARY_REG3, 0, 471 1.2.2.2 tls SLJIT_MEM1(SLJIT_LOCALS_REG), arg3_offset); 472 1.2.2.2 tls if (status != SLJIT_SUCCESS) 473 1.2.2.2 tls return status; 474 1.2.2.2 tls 475 1.2.2.2 tls /* if (tmp3 != 0) return 0; */ 476 1.2.2.2 tls *ret0_jump = sljit_emit_cmp(compiler, 477 1.2.2.2 tls SLJIT_C_NOT_EQUAL, 478 1.2.2.2 tls SLJIT_TEMPORARY_REG3, 0, 479 1.2.2.2 tls SLJIT_IMM, 0); 480 1.2.2.2 tls if (*ret0_jump == NULL) 481 1.2.2.2 tls return SLJIT_ERR_ALLOC_FAILED; 482 1.2.2.2 tls 483 1.2.2.2 tls return status; 484 1.2.2.2 tls } 485 1.2.2.2 tls #endif 486 1.2.2.2 tls 487 1.2.2.2 tls /* 488 1.2.2.2 tls * Generate code for 489 1.2.2.2 tls * BPF_LD+BPF_W+BPF_ABS A <- P[k:4] 490 1.2.2.2 tls * BPF_LD+BPF_H+BPF_ABS A <- P[k:2] 491 1.2.2.2 tls * BPF_LD+BPF_B+BPF_ABS A <- P[k:1] 492 1.2.2.2 tls * BPF_LD+BPF_W+BPF_IND A <- P[X+k:4] 493 1.2.2.2 tls * BPF_LD+BPF_H+BPF_IND A <- P[X+k:2] 494 1.2.2.2 tls * BPF_LD+BPF_B+BPF_IND A <- P[X+k:1] 495 1.2.2.2 tls */ 496 1.2.2.2 tls static int 497 1.2.2.2 tls emit_pkt_read(struct sljit_compiler* compiler, 498 1.2.2.2 tls struct bpf_insn *pc, struct sljit_jump *to_mchain_jump, 499 1.2.2.2 tls struct sljit_jump **ret0, size_t *ret0_size) 500 1.2.2.2 tls { 501 1.2.2.2 tls int status; 502 1.2.2.2 tls uint32_t width; 503 1.2.2.2 tls struct sljit_jump *jump; 504 1.2.2.2 tls #ifdef _KERNEL 505 1.2.2.2 tls struct sljit_label *label; 506 1.2.2.2 tls struct sljit_jump *over_mchain_jump; 507 1.2.2.2 tls const bool check_zero_buflen = (to_mchain_jump != NULL); 508 1.2.2.2 tls #endif 509 1.2.2.2 tls const uint32_t k = pc->k; 510 1.2.2.2 tls 511 1.2.2.2 tls #ifdef _KERNEL 512 1.2.2.2 tls if (to_mchain_jump == NULL) { 513 1.2.2.2 tls to_mchain_jump = sljit_emit_cmp(compiler, 514 1.2.2.2 tls SLJIT_C_EQUAL, 515 1.2.2.2 tls BPFJIT_BUFLEN, 0, 516 1.2.2.2 tls SLJIT_IMM, 0); 517 1.2.2.2 tls if (to_mchain_jump == NULL) 518 1.2.2.2 tls return SLJIT_ERR_ALLOC_FAILED; 519 1.2.2.2 tls } 520 1.2.2.2 tls #endif 521 1.2.2.2 tls 522 1.2.2.2 tls width = read_width(pc); 523 1.2.2.2 tls 524 1.2.2.2 tls if (BPF_MODE(pc->code) == BPF_IND) { 525 1.2.2.2 tls /* tmp1 = buflen - (pc->k + width); */ 526 1.2.2.2 tls status = sljit_emit_op2(compiler, 527 1.2.2.2 tls SLJIT_SUB, 528 1.2.2.2 tls BPFJIT_TMP1, 0, 529 1.2.2.2 tls BPFJIT_BUFLEN, 0, 530 1.2.2.2 tls SLJIT_IMM, k + width); 531 1.2.2.2 tls if (status != SLJIT_SUCCESS) 532 1.2.2.2 tls return status; 533 1.2.2.2 tls 534 1.2.2.2 tls /* buf += X; */ 535 1.2.2.2 tls status = sljit_emit_op2(compiler, 536 1.2.2.2 tls SLJIT_ADD, 537 1.2.2.2 tls BPFJIT_BUF, 0, 538 1.2.2.2 tls BPFJIT_BUF, 0, 539 1.2.2.2 tls BPFJIT_X, 0); 540 1.2.2.2 tls if (status != SLJIT_SUCCESS) 541 1.2.2.2 tls return status; 542 1.2.2.2 tls 543 1.2.2.2 tls /* if (tmp1 < X) return 0; */ 544 1.2.2.2 tls jump = sljit_emit_cmp(compiler, 545 1.2.2.2 tls SLJIT_C_LESS, 546 1.2.2.2 tls BPFJIT_TMP1, 0, 547 1.2.2.2 tls BPFJIT_X, 0); 548 1.2.2.2 tls if (jump == NULL) 549 1.2.2.2 tls return SLJIT_ERR_ALLOC_FAILED; 550 1.2.2.2 tls ret0[(*ret0_size)++] = jump; 551 1.2.2.2 tls } 552 1.2.2.2 tls 553 1.2.2.2 tls switch (width) { 554 1.2.2.2 tls case 4: 555 1.2.2.2 tls status = emit_read32(compiler, k); 556 1.2.2.2 tls break; 557 1.2.2.2 tls case 2: 558 1.2.2.2 tls status = emit_read16(compiler, k); 559 1.2.2.2 tls break; 560 1.2.2.2 tls case 1: 561 1.2.2.2 tls status = emit_read8(compiler, k); 562 1.2.2.2 tls break; 563 1.2.2.2 tls } 564 1.2.2.2 tls 565 1.2.2.2 tls if (status != SLJIT_SUCCESS) 566 1.2.2.2 tls return status; 567 1.2.2.2 tls 568 1.2.2.2 tls if (BPF_MODE(pc->code) == BPF_IND) { 569 1.2.2.2 tls /* buf -= X; */ 570 1.2.2.2 tls status = sljit_emit_op2(compiler, 571 1.2.2.2 tls SLJIT_SUB, 572 1.2.2.2 tls BPFJIT_BUF, 0, 573 1.2.2.2 tls BPFJIT_BUF, 0, 574 1.2.2.2 tls BPFJIT_X, 0); 575 1.2.2.2 tls if (status != SLJIT_SUCCESS) 576 1.2.2.2 tls return status; 577 1.2.2.2 tls } 578 1.2.2.2 tls 579 1.2.2.2 tls #ifdef _KERNEL 580 1.2.2.2 tls over_mchain_jump = sljit_emit_jump(compiler, SLJIT_JUMP); 581 1.2.2.2 tls if (over_mchain_jump == NULL) 582 1.2.2.2 tls return SLJIT_ERR_ALLOC_FAILED; 583 1.2.2.2 tls 584 1.2.2.2 tls /* entry point to mchain handler */ 585 1.2.2.2 tls label = sljit_emit_label(compiler); 586 1.2.2.2 tls if (label == NULL) 587 1.2.2.2 tls return SLJIT_ERR_ALLOC_FAILED; 588 1.2.2.2 tls sljit_set_label(to_mchain_jump, label); 589 1.2.2.2 tls 590 1.2.2.2 tls if (check_zero_buflen) { 591 1.2.2.2 tls /* if (buflen != 0) return 0; */ 592 1.2.2.2 tls jump = sljit_emit_cmp(compiler, 593 1.2.2.2 tls SLJIT_C_NOT_EQUAL, 594 1.2.2.2 tls BPFJIT_BUFLEN, 0, 595 1.2.2.2 tls SLJIT_IMM, 0); 596 1.2.2.2 tls if (jump == NULL) 597 1.2.2.2 tls return SLJIT_ERR_ALLOC_FAILED; 598 1.2.2.2 tls ret0[(*ret0_size)++] = jump; 599 1.2.2.2 tls } 600 1.2.2.2 tls 601 1.2.2.2 tls switch (width) { 602 1.2.2.2 tls case 4: 603 1.2.2.2 tls status = emit_xcall(compiler, pc, BPFJIT_A, 0, &jump, &m_xword); 604 1.2.2.2 tls break; 605 1.2.2.2 tls case 2: 606 1.2.2.2 tls status = emit_xcall(compiler, pc, BPFJIT_A, 0, &jump, &m_xhalf); 607 1.2.2.2 tls break; 608 1.2.2.2 tls case 1: 609 1.2.2.2 tls status = emit_xcall(compiler, pc, BPFJIT_A, 0, &jump, &m_xbyte); 610 1.2.2.2 tls break; 611 1.2.2.2 tls } 612 1.2.2.2 tls 613 1.2.2.2 tls if (status != SLJIT_SUCCESS) 614 1.2.2.2 tls return status; 615 1.2.2.2 tls 616 1.2.2.2 tls ret0[(*ret0_size)++] = jump; 617 1.2.2.2 tls 618 1.2.2.2 tls label = sljit_emit_label(compiler); 619 1.2.2.2 tls if (label == NULL) 620 1.2.2.2 tls return SLJIT_ERR_ALLOC_FAILED; 621 1.2.2.2 tls sljit_set_label(over_mchain_jump, label); 622 1.2.2.2 tls #endif 623 1.2.2.2 tls 624 1.2.2.2 tls return status; 625 1.2.2.2 tls } 626 1.2.2.2 tls 627 1.2.2.2 tls /* 628 1.2.2.2 tls * Generate code for BPF_LDX+BPF_B+BPF_MSH X <- 4*(P[k:1]&0xf). 629 1.2.2.2 tls */ 630 1.2.2.2 tls static int 631 1.2.2.2 tls emit_msh(struct sljit_compiler* compiler, 632 1.2.2.2 tls struct bpf_insn *pc, struct sljit_jump *to_mchain_jump, 633 1.2.2.2 tls struct sljit_jump **ret0, size_t *ret0_size) 634 1.2.2.2 tls { 635 1.2.2.2 tls int status; 636 1.2.2.2 tls #ifdef _KERNEL 637 1.2.2.2 tls struct sljit_label *label; 638 1.2.2.2 tls struct sljit_jump *jump, *over_mchain_jump; 639 1.2.2.2 tls const bool check_zero_buflen = (to_mchain_jump != NULL); 640 1.2.2.2 tls #endif 641 1.2.2.2 tls const uint32_t k = pc->k; 642 1.2.2.2 tls 643 1.2.2.2 tls #ifdef _KERNEL 644 1.2.2.2 tls if (to_mchain_jump == NULL) { 645 1.2.2.2 tls to_mchain_jump = sljit_emit_cmp(compiler, 646 1.2.2.2 tls SLJIT_C_EQUAL, 647 1.2.2.2 tls BPFJIT_BUFLEN, 0, 648 1.2.2.2 tls SLJIT_IMM, 0); 649 1.2.2.2 tls if (to_mchain_jump == NULL) 650 1.2.2.2 tls return SLJIT_ERR_ALLOC_FAILED; 651 1.2.2.2 tls } 652 1.2.2.2 tls #endif 653 1.2.2.2 tls 654 1.2.2.2 tls /* tmp1 = buf[k] */ 655 1.2.2.2 tls status = sljit_emit_op1(compiler, 656 1.2.2.2 tls SLJIT_MOV_UB, 657 1.2.2.2 tls BPFJIT_TMP1, 0, 658 1.2.2.2 tls SLJIT_MEM1(BPFJIT_BUF), k); 659 1.2.2.2 tls if (status != SLJIT_SUCCESS) 660 1.2.2.2 tls return status; 661 1.2.2.2 tls 662 1.2.2.2 tls /* tmp1 &= 0xf */ 663 1.2.2.2 tls status = sljit_emit_op2(compiler, 664 1.2.2.2 tls SLJIT_AND, 665 1.2.2.2 tls BPFJIT_TMP1, 0, 666 1.2.2.2 tls BPFJIT_TMP1, 0, 667 1.2.2.2 tls SLJIT_IMM, 0xf); 668 1.2.2.2 tls if (status != SLJIT_SUCCESS) 669 1.2.2.2 tls return status; 670 1.2.2.2 tls 671 1.2.2.2 tls /* tmp1 = tmp1 << 2 */ 672 1.2.2.2 tls status = sljit_emit_op2(compiler, 673 1.2.2.2 tls SLJIT_SHL, 674 1.2.2.2 tls BPFJIT_X, 0, 675 1.2.2.2 tls BPFJIT_TMP1, 0, 676 1.2.2.2 tls SLJIT_IMM, 2); 677 1.2.2.2 tls if (status != SLJIT_SUCCESS) 678 1.2.2.2 tls return status; 679 1.2.2.2 tls 680 1.2.2.2 tls #ifdef _KERNEL 681 1.2.2.2 tls over_mchain_jump = sljit_emit_jump(compiler, SLJIT_JUMP); 682 1.2.2.2 tls if (over_mchain_jump == NULL) 683 1.2.2.2 tls return SLJIT_ERR_ALLOC_FAILED; 684 1.2.2.2 tls 685 1.2.2.2 tls /* entry point to mchain handler */ 686 1.2.2.2 tls label = sljit_emit_label(compiler); 687 1.2.2.2 tls if (label == NULL) 688 1.2.2.2 tls return SLJIT_ERR_ALLOC_FAILED; 689 1.2.2.2 tls sljit_set_label(to_mchain_jump, label); 690 1.2.2.2 tls 691 1.2.2.2 tls if (check_zero_buflen) { 692 1.2.2.2 tls /* if (buflen != 0) return 0; */ 693 1.2.2.2 tls jump = sljit_emit_cmp(compiler, 694 1.2.2.2 tls SLJIT_C_NOT_EQUAL, 695 1.2.2.2 tls BPFJIT_BUFLEN, 0, 696 1.2.2.2 tls SLJIT_IMM, 0); 697 1.2.2.2 tls if (jump == NULL) 698 1.2.2.2 tls return SLJIT_ERR_ALLOC_FAILED; 699 1.2.2.2 tls ret0[(*ret0_size)++] = jump; 700 1.2.2.2 tls } 701 1.2.2.2 tls 702 1.2.2.2 tls status = emit_xcall(compiler, pc, BPFJIT_TMP1, 0, &jump, &m_xbyte); 703 1.2.2.2 tls if (status != SLJIT_SUCCESS) 704 1.2.2.2 tls return status; 705 1.2.2.2 tls ret0[(*ret0_size)++] = jump; 706 1.2.2.2 tls 707 1.2.2.2 tls /* tmp1 &= 0xf */ 708 1.2.2.2 tls status = sljit_emit_op2(compiler, 709 1.2.2.2 tls SLJIT_AND, 710 1.2.2.2 tls BPFJIT_TMP1, 0, 711 1.2.2.2 tls BPFJIT_TMP1, 0, 712 1.2.2.2 tls SLJIT_IMM, 0xf); 713 1.2.2.2 tls if (status != SLJIT_SUCCESS) 714 1.2.2.2 tls return status; 715 1.2.2.2 tls 716 1.2.2.2 tls /* tmp1 = tmp1 << 2 */ 717 1.2.2.2 tls status = sljit_emit_op2(compiler, 718 1.2.2.2 tls SLJIT_SHL, 719 1.2.2.2 tls BPFJIT_X, 0, 720 1.2.2.2 tls BPFJIT_TMP1, 0, 721 1.2.2.2 tls SLJIT_IMM, 2); 722 1.2.2.2 tls if (status != SLJIT_SUCCESS) 723 1.2.2.2 tls return status; 724 1.2.2.2 tls 725 1.2.2.2 tls 726 1.2.2.2 tls label = sljit_emit_label(compiler); 727 1.2.2.2 tls if (label == NULL) 728 1.2.2.2 tls return SLJIT_ERR_ALLOC_FAILED; 729 1.2.2.2 tls sljit_set_label(over_mchain_jump, label); 730 1.2.2.2 tls #endif 731 1.2.2.2 tls 732 1.2.2.2 tls return status; 733 1.2.2.2 tls } 734 1.2.2.2 tls 735 1.2.2.2 tls static int 736 1.2.2.2 tls emit_pow2_division(struct sljit_compiler* compiler, uint32_t k) 737 1.2.2.2 tls { 738 1.2.2.2 tls int shift = 0; 739 1.2.2.2 tls int status = SLJIT_SUCCESS; 740 1.2.2.2 tls 741 1.2.2.2 tls while (k > 1) { 742 1.2.2.2 tls k >>= 1; 743 1.2.2.2 tls shift++; 744 1.2.2.2 tls } 745 1.2.2.2 tls 746 1.2.2.2 tls BPFJIT_ASSERT(k == 1 && shift < 32); 747 1.2.2.2 tls 748 1.2.2.2 tls if (shift != 0) { 749 1.2.2.2 tls status = sljit_emit_op2(compiler, 750 1.2.2.2 tls SLJIT_LSHR|SLJIT_INT_OP, 751 1.2.2.2 tls BPFJIT_A, 0, 752 1.2.2.2 tls BPFJIT_A, 0, 753 1.2.2.2 tls SLJIT_IMM, shift); 754 1.2.2.2 tls } 755 1.2.2.2 tls 756 1.2.2.2 tls return status; 757 1.2.2.2 tls } 758 1.2.2.2 tls 759 1.2.2.2 tls #if !defined(BPFJIT_USE_UDIV) 760 1.2.2.2 tls static sljit_uw 761 1.2.2.2 tls divide(sljit_uw x, sljit_uw y) 762 1.2.2.2 tls { 763 1.2.2.2 tls 764 1.2.2.2 tls return (uint32_t)x / (uint32_t)y; 765 1.2.2.2 tls } 766 1.2.2.2 tls #endif 767 1.2.2.2 tls 768 1.2.2.2 tls /* 769 1.2.2.2 tls * Generate A = A / div. 770 1.2.2.2 tls * divt,divw are either SLJIT_IMM,pc->k or BPFJIT_X,0. 771 1.2.2.2 tls */ 772 1.2.2.2 tls static int 773 1.2.2.2 tls emit_division(struct sljit_compiler* compiler, int divt, sljit_w divw) 774 1.2.2.2 tls { 775 1.2.2.2 tls int status; 776 1.2.2.2 tls 777 1.2.2.2 tls #if BPFJIT_X == SLJIT_TEMPORARY_REG1 || \ 778 1.2.2.2 tls BPFJIT_X == SLJIT_RETURN_REG || \ 779 1.2.2.2 tls BPFJIT_X == SLJIT_TEMPORARY_REG2 || \ 780 1.2.2.2 tls BPFJIT_A == SLJIT_TEMPORARY_REG2 781 1.2.2.2 tls #error "Not supported assignment of registers." 782 1.2.2.2 tls #endif 783 1.2.2.2 tls 784 1.2.2.2 tls #if BPFJIT_A != SLJIT_TEMPORARY_REG1 785 1.2.2.2 tls status = sljit_emit_op1(compiler, 786 1.2.2.2 tls SLJIT_MOV, 787 1.2.2.2 tls SLJIT_TEMPORARY_REG1, 0, 788 1.2.2.2 tls BPFJIT_A, 0); 789 1.2.2.2 tls if (status != SLJIT_SUCCESS) 790 1.2.2.2 tls return status; 791 1.2.2.2 tls #endif 792 1.2.2.2 tls 793 1.2.2.2 tls status = sljit_emit_op1(compiler, 794 1.2.2.2 tls SLJIT_MOV, 795 1.2.2.2 tls SLJIT_TEMPORARY_REG2, 0, 796 1.2.2.2 tls divt, divw); 797 1.2.2.2 tls if (status != SLJIT_SUCCESS) 798 1.2.2.2 tls return status; 799 1.2.2.2 tls 800 1.2.2.2 tls #if defined(BPFJIT_USE_UDIV) 801 1.2.2.2 tls status = sljit_emit_op0(compiler, SLJIT_UDIV|SLJIT_INT_OP); 802 1.2.2.2 tls 803 1.2.2.2 tls #if BPFJIT_A != SLJIT_TEMPORARY_REG1 804 1.2.2.2 tls status = sljit_emit_op1(compiler, 805 1.2.2.2 tls SLJIT_MOV, 806 1.2.2.2 tls BPFJIT_A, 0, 807 1.2.2.2 tls SLJIT_TEMPORARY_REG1, 0); 808 1.2.2.2 tls if (status != SLJIT_SUCCESS) 809 1.2.2.2 tls return status; 810 1.2.2.2 tls #endif 811 1.2.2.2 tls #else 812 1.2.2.2 tls status = sljit_emit_ijump(compiler, 813 1.2.2.2 tls SLJIT_CALL2, 814 1.2.2.2 tls SLJIT_IMM, SLJIT_FUNC_OFFSET(divide)); 815 1.2.2.2 tls 816 1.2.2.2 tls #if BPFJIT_A != SLJIT_RETURN_REG 817 1.2.2.2 tls status = sljit_emit_op1(compiler, 818 1.2.2.2 tls SLJIT_MOV, 819 1.2.2.2 tls BPFJIT_A, 0, 820 1.2.2.2 tls SLJIT_RETURN_REG, 0); 821 1.2.2.2 tls if (status != SLJIT_SUCCESS) 822 1.2.2.2 tls return status; 823 1.2.2.2 tls #endif 824 1.2.2.2 tls #endif 825 1.2.2.2 tls 826 1.2.2.2 tls return status; 827 1.2.2.2 tls } 828 1.2.2.2 tls 829 1.2.2.2 tls /* 830 1.2.2.2 tls * Count BPF_RET instructions. 831 1.2.2.2 tls */ 832 1.2.2.2 tls static size_t 833 1.2.2.2 tls count_returns(struct bpf_insn *insns, size_t insn_count) 834 1.2.2.2 tls { 835 1.2.2.2 tls size_t i; 836 1.2.2.2 tls size_t rv; 837 1.2.2.2 tls 838 1.2.2.2 tls rv = 0; 839 1.2.2.2 tls for (i = 0; i < insn_count; i++) { 840 1.2.2.2 tls if (BPF_CLASS(insns[i].code) == BPF_RET) 841 1.2.2.2 tls rv++; 842 1.2.2.2 tls } 843 1.2.2.2 tls 844 1.2.2.2 tls return rv; 845 1.2.2.2 tls } 846 1.2.2.2 tls 847 1.2.2.2 tls /* 848 1.2.2.2 tls * Return true if pc is a "read from packet" instruction. 849 1.2.2.2 tls * If length is not NULL and return value is true, *length will 850 1.2.2.2 tls * be set to a safe length required to read a packet. 851 1.2.2.2 tls */ 852 1.2.2.2 tls static bool 853 1.2.2.2 tls read_pkt_insn(struct bpf_insn *pc, uint32_t *length) 854 1.2.2.2 tls { 855 1.2.2.2 tls bool rv; 856 1.2.2.2 tls uint32_t width; 857 1.2.2.2 tls 858 1.2.2.2 tls switch (BPF_CLASS(pc->code)) { 859 1.2.2.2 tls default: 860 1.2.2.2 tls rv = false; 861 1.2.2.2 tls break; 862 1.2.2.2 tls 863 1.2.2.2 tls case BPF_LD: 864 1.2.2.2 tls rv = BPF_MODE(pc->code) == BPF_ABS || 865 1.2.2.2 tls BPF_MODE(pc->code) == BPF_IND; 866 1.2.2.2 tls if (rv) 867 1.2.2.2 tls width = read_width(pc); 868 1.2.2.2 tls break; 869 1.2.2.2 tls 870 1.2.2.2 tls case BPF_LDX: 871 1.2.2.2 tls rv = pc->code == (BPF_LDX|BPF_B|BPF_MSH); 872 1.2.2.2 tls width = 1; 873 1.2.2.2 tls break; 874 1.2.2.2 tls } 875 1.2.2.2 tls 876 1.2.2.2 tls if (rv && length != NULL) { 877 1.2.2.2 tls *length = (pc->k > UINT32_MAX - width) ? 878 1.2.2.2 tls UINT32_MAX : pc->k + width; 879 1.2.2.2 tls } 880 1.2.2.2 tls 881 1.2.2.2 tls return rv; 882 1.2.2.2 tls } 883 1.2.2.2 tls 884 1.2.2.2 tls /* 885 1.2.2.2 tls * Set bj_check_length for all "read from packet" instructions 886 1.2.2.2 tls * in a linear block of instructions [from, to). 887 1.2.2.2 tls */ 888 1.2.2.2 tls static void 889 1.2.2.2 tls set_check_length(struct bpf_insn *insns, struct bpfjit_insn_data *insn_dat, 890 1.2.2.2 tls size_t from, size_t to, uint32_t length) 891 1.2.2.2 tls { 892 1.2.2.2 tls 893 1.2.2.2 tls for (; from < to; from++) { 894 1.2.2.2 tls if (read_pkt_insn(&insns[from], NULL)) { 895 1.2.2.2 tls insn_dat[from].bj_aux.bj_rdata.bj_check_length = length; 896 1.2.2.2 tls length = 0; 897 1.2.2.2 tls } 898 1.2.2.2 tls } 899 1.2.2.2 tls } 900 1.2.2.2 tls 901 1.2.2.2 tls /* 902 1.2.2.2 tls * The function divides instructions into blocks. Destination of a jump 903 1.2.2.2 tls * instruction starts a new block. BPF_RET and BPF_JMP instructions 904 1.2.2.2 tls * terminate a block. Blocks are linear, that is, there are no jumps out 905 1.2.2.2 tls * from the middle of a block and there are no jumps in to the middle of 906 1.2.2.2 tls * a block. 907 1.2.2.2 tls * If a block has one or more "read from packet" instructions, 908 1.2.2.2 tls * bj_check_length will be set to one value for the whole block and that 909 1.2.2.2 tls * value will be equal to the greatest value of safe lengths of "read from 910 1.2.2.2 tls * packet" instructions inside the block. 911 1.2.2.2 tls */ 912 1.2.2.2 tls static int 913 1.2.2.2 tls optimize(struct bpf_insn *insns, 914 1.2.2.2 tls struct bpfjit_insn_data *insn_dat, size_t insn_count) 915 1.2.2.2 tls { 916 1.2.2.2 tls size_t i; 917 1.2.2.2 tls size_t first_read; 918 1.2.2.2 tls bool unreachable; 919 1.2.2.2 tls uint32_t jt, jf; 920 1.2.2.2 tls uint32_t length, safe_length; 921 1.2.2.2 tls struct bpfjit_jump *jmp, *jtf; 922 1.2.2.2 tls 923 1.2.2.2 tls for (i = 0; i < insn_count; i++) 924 1.2.2.2 tls SLIST_INIT(&insn_dat[i].bj_jumps); 925 1.2.2.2 tls 926 1.2.2.2 tls safe_length = 0; 927 1.2.2.2 tls unreachable = false; 928 1.2.2.2 tls first_read = SIZE_MAX; 929 1.2.2.2 tls 930 1.2.2.2 tls for (i = 0; i < insn_count; i++) { 931 1.2.2.2 tls 932 1.2.2.2 tls if (!SLIST_EMPTY(&insn_dat[i].bj_jumps)) { 933 1.2.2.2 tls unreachable = false; 934 1.2.2.2 tls 935 1.2.2.2 tls set_check_length(insns, insn_dat, 936 1.2.2.2 tls first_read, i, safe_length); 937 1.2.2.2 tls first_read = SIZE_MAX; 938 1.2.2.2 tls 939 1.2.2.2 tls safe_length = UINT32_MAX; 940 1.2.2.2 tls SLIST_FOREACH(jmp, &insn_dat[i].bj_jumps, bj_entries) { 941 1.2.2.2 tls if (jmp->bj_safe_length < safe_length) 942 1.2.2.2 tls safe_length = jmp->bj_safe_length; 943 1.2.2.2 tls } 944 1.2.2.2 tls } 945 1.2.2.2 tls 946 1.2.2.2 tls insn_dat[i].bj_unreachable = unreachable; 947 1.2.2.2 tls if (unreachable) 948 1.2.2.2 tls continue; 949 1.2.2.2 tls 950 1.2.2.2 tls if (read_pkt_insn(&insns[i], &length)) { 951 1.2.2.2 tls if (first_read == SIZE_MAX) 952 1.2.2.2 tls first_read = i; 953 1.2.2.2 tls if (length > safe_length) 954 1.2.2.2 tls safe_length = length; 955 1.2.2.2 tls } 956 1.2.2.2 tls 957 1.2.2.2 tls switch (BPF_CLASS(insns[i].code)) { 958 1.2.2.2 tls case BPF_RET: 959 1.2.2.2 tls unreachable = true; 960 1.2.2.2 tls continue; 961 1.2.2.2 tls 962 1.2.2.2 tls case BPF_JMP: 963 1.2.2.2 tls if (insns[i].code == (BPF_JMP|BPF_JA)) { 964 1.2.2.2 tls jt = jf = insns[i].k; 965 1.2.2.2 tls } else { 966 1.2.2.2 tls jt = insns[i].jt; 967 1.2.2.2 tls jf = insns[i].jf; 968 1.2.2.2 tls } 969 1.2.2.2 tls 970 1.2.2.2 tls if (jt >= insn_count - (i + 1) || 971 1.2.2.2 tls jf >= insn_count - (i + 1)) { 972 1.2.2.2 tls return -1; 973 1.2.2.2 tls } 974 1.2.2.2 tls 975 1.2.2.2 tls if (jt > 0 && jf > 0) 976 1.2.2.2 tls unreachable = true; 977 1.2.2.2 tls 978 1.2.2.2 tls jtf = insn_dat[i].bj_aux.bj_jdata.bj_jtf; 979 1.2.2.2 tls 980 1.2.2.2 tls jtf[0].bj_jump = NULL; 981 1.2.2.2 tls jtf[0].bj_safe_length = safe_length; 982 1.2.2.2 tls SLIST_INSERT_HEAD(&insn_dat[i + 1 + jt].bj_jumps, 983 1.2.2.2 tls &jtf[0], bj_entries); 984 1.2.2.2 tls 985 1.2.2.2 tls if (jf != jt) { 986 1.2.2.2 tls jtf[1].bj_jump = NULL; 987 1.2.2.2 tls jtf[1].bj_safe_length = safe_length; 988 1.2.2.2 tls SLIST_INSERT_HEAD(&insn_dat[i + 1 + jf].bj_jumps, 989 1.2.2.2 tls &jtf[1], bj_entries); 990 1.2.2.2 tls } 991 1.2.2.2 tls 992 1.2.2.2 tls continue; 993 1.2.2.2 tls } 994 1.2.2.2 tls } 995 1.2.2.2 tls 996 1.2.2.2 tls set_check_length(insns, insn_dat, first_read, insn_count, safe_length); 997 1.2.2.2 tls 998 1.2.2.2 tls return 0; 999 1.2.2.2 tls } 1000 1.2.2.2 tls 1001 1.2.2.2 tls /* 1002 1.2.2.2 tls * Count out-of-bounds and division by zero jumps. 1003 1.2.2.2 tls * 1004 1.2.2.2 tls * insn_dat should be initialized by optimize(). 1005 1.2.2.2 tls */ 1006 1.2.2.2 tls static size_t 1007 1.2.2.2 tls get_ret0_size(struct bpf_insn *insns, struct bpfjit_insn_data *insn_dat, 1008 1.2.2.2 tls size_t insn_count) 1009 1.2.2.2 tls { 1010 1.2.2.2 tls size_t rv = 0; 1011 1.2.2.2 tls size_t i; 1012 1.2.2.2 tls 1013 1.2.2.2 tls for (i = 0; i < insn_count; i++) { 1014 1.2.2.2 tls 1015 1.2.2.2 tls if (read_pkt_insn(&insns[i], NULL)) { 1016 1.2.2.2 tls if (insn_dat[i].bj_aux.bj_rdata.bj_check_length > 0) 1017 1.2.2.2 tls rv++; 1018 1.2.2.2 tls #ifdef _KERNEL 1019 1.2.2.2 tls rv++; 1020 1.2.2.2 tls #endif 1021 1.2.2.2 tls } 1022 1.2.2.2 tls 1023 1.2.2.2 tls if (insns[i].code == (BPF_LD|BPF_IND|BPF_B) || 1024 1.2.2.2 tls insns[i].code == (BPF_LD|BPF_IND|BPF_H) || 1025 1.2.2.2 tls insns[i].code == (BPF_LD|BPF_IND|BPF_W)) { 1026 1.2.2.2 tls rv++; 1027 1.2.2.2 tls } 1028 1.2.2.2 tls 1029 1.2.2.2 tls if (insns[i].code == (BPF_ALU|BPF_DIV|BPF_X)) 1030 1.2.2.2 tls rv++; 1031 1.2.2.2 tls 1032 1.2.2.2 tls if (insns[i].code == (BPF_ALU|BPF_DIV|BPF_K) && 1033 1.2.2.2 tls insns[i].k == 0) { 1034 1.2.2.2 tls rv++; 1035 1.2.2.2 tls } 1036 1.2.2.2 tls } 1037 1.2.2.2 tls 1038 1.2.2.2 tls return rv; 1039 1.2.2.2 tls } 1040 1.2.2.2 tls 1041 1.2.2.2 tls /* 1042 1.2.2.2 tls * Convert BPF_ALU operations except BPF_NEG and BPF_DIV to sljit operation. 1043 1.2.2.2 tls */ 1044 1.2.2.2 tls static int 1045 1.2.2.2 tls bpf_alu_to_sljit_op(struct bpf_insn *pc) 1046 1.2.2.2 tls { 1047 1.2.2.2 tls 1048 1.2.2.2 tls /* 1049 1.2.2.2 tls * Note: all supported 64bit arches have 32bit multiply 1050 1.2.2.2 tls * instruction so SLJIT_INT_OP doesn't have any overhead. 1051 1.2.2.2 tls */ 1052 1.2.2.2 tls switch (BPF_OP(pc->code)) { 1053 1.2.2.2 tls case BPF_ADD: return SLJIT_ADD; 1054 1.2.2.2 tls case BPF_SUB: return SLJIT_SUB; 1055 1.2.2.2 tls case BPF_MUL: return SLJIT_MUL|SLJIT_INT_OP; 1056 1.2.2.2 tls case BPF_OR: return SLJIT_OR; 1057 1.2.2.2 tls case BPF_AND: return SLJIT_AND; 1058 1.2.2.2 tls case BPF_LSH: return SLJIT_SHL; 1059 1.2.2.2 tls case BPF_RSH: return SLJIT_LSHR|SLJIT_INT_OP; 1060 1.2.2.2 tls default: 1061 1.2.2.2 tls BPFJIT_ASSERT(false); 1062 1.2.2.2 tls return 0; 1063 1.2.2.2 tls } 1064 1.2.2.2 tls } 1065 1.2.2.2 tls 1066 1.2.2.2 tls /* 1067 1.2.2.2 tls * Convert BPF_JMP operations except BPF_JA to sljit condition. 1068 1.2.2.2 tls */ 1069 1.2.2.2 tls static int 1070 1.2.2.2 tls bpf_jmp_to_sljit_cond(struct bpf_insn *pc, bool negate) 1071 1.2.2.2 tls { 1072 1.2.2.2 tls /* 1073 1.2.2.2 tls * Note: all supported 64bit arches have 32bit comparison 1074 1.2.2.2 tls * instructions so SLJIT_INT_OP doesn't have any overhead. 1075 1.2.2.2 tls */ 1076 1.2.2.2 tls int rv = SLJIT_INT_OP; 1077 1.2.2.2 tls 1078 1.2.2.2 tls switch (BPF_OP(pc->code)) { 1079 1.2.2.2 tls case BPF_JGT: 1080 1.2.2.2 tls rv |= negate ? SLJIT_C_LESS_EQUAL : SLJIT_C_GREATER; 1081 1.2.2.2 tls break; 1082 1.2.2.2 tls case BPF_JGE: 1083 1.2.2.2 tls rv |= negate ? SLJIT_C_LESS : SLJIT_C_GREATER_EQUAL; 1084 1.2.2.2 tls break; 1085 1.2.2.2 tls case BPF_JEQ: 1086 1.2.2.2 tls rv |= negate ? SLJIT_C_NOT_EQUAL : SLJIT_C_EQUAL; 1087 1.2.2.2 tls break; 1088 1.2.2.2 tls case BPF_JSET: 1089 1.2.2.2 tls rv |= negate ? SLJIT_C_EQUAL : SLJIT_C_NOT_EQUAL; 1090 1.2.2.2 tls break; 1091 1.2.2.2 tls default: 1092 1.2.2.2 tls BPFJIT_ASSERT(false); 1093 1.2.2.2 tls } 1094 1.2.2.2 tls 1095 1.2.2.2 tls return rv; 1096 1.2.2.2 tls } 1097 1.2.2.2 tls 1098 1.2.2.2 tls static unsigned int 1099 1.2.2.2 tls bpfjit_optimization_hints(struct bpf_insn *insns, size_t insn_count) 1100 1.2.2.2 tls { 1101 1.2.2.2 tls unsigned int rv = BPFJIT_INIT_A; 1102 1.2.2.2 tls struct bpf_insn *pc; 1103 1.2.2.2 tls unsigned int minm, maxm; 1104 1.2.2.2 tls 1105 1.2.2.2 tls BPFJIT_ASSERT(BPF_MEMWORDS - 1 <= 0xff); 1106 1.2.2.2 tls 1107 1.2.2.2 tls maxm = 0; 1108 1.2.2.2 tls minm = BPF_MEMWORDS - 1; 1109 1.2.2.2 tls 1110 1.2.2.2 tls for (pc = insns; pc != insns + insn_count; pc++) { 1111 1.2.2.2 tls switch (BPF_CLASS(pc->code)) { 1112 1.2.2.2 tls case BPF_LD: 1113 1.2.2.2 tls if (BPF_MODE(pc->code) == BPF_IND) 1114 1.2.2.2 tls rv |= BPFJIT_INIT_X; 1115 1.2.2.2 tls if (BPF_MODE(pc->code) == BPF_MEM && 1116 1.2.2.2 tls (uint32_t)pc->k < BPF_MEMWORDS) { 1117 1.2.2.2 tls if (pc->k > maxm) 1118 1.2.2.2 tls maxm = pc->k; 1119 1.2.2.2 tls if (pc->k < minm) 1120 1.2.2.2 tls minm = pc->k; 1121 1.2.2.2 tls } 1122 1.2.2.2 tls continue; 1123 1.2.2.2 tls case BPF_LDX: 1124 1.2.2.2 tls rv |= BPFJIT_INIT_X; 1125 1.2.2.2 tls if (BPF_MODE(pc->code) == BPF_MEM && 1126 1.2.2.2 tls (uint32_t)pc->k < BPF_MEMWORDS) { 1127 1.2.2.2 tls if (pc->k > maxm) 1128 1.2.2.2 tls maxm = pc->k; 1129 1.2.2.2 tls if (pc->k < minm) 1130 1.2.2.2 tls minm = pc->k; 1131 1.2.2.2 tls } 1132 1.2.2.2 tls continue; 1133 1.2.2.2 tls case BPF_ST: 1134 1.2.2.2 tls if ((uint32_t)pc->k < BPF_MEMWORDS) { 1135 1.2.2.2 tls if (pc->k > maxm) 1136 1.2.2.2 tls maxm = pc->k; 1137 1.2.2.2 tls if (pc->k < minm) 1138 1.2.2.2 tls minm = pc->k; 1139 1.2.2.2 tls } 1140 1.2.2.2 tls continue; 1141 1.2.2.2 tls case BPF_STX: 1142 1.2.2.2 tls rv |= BPFJIT_INIT_X; 1143 1.2.2.2 tls if ((uint32_t)pc->k < BPF_MEMWORDS) { 1144 1.2.2.2 tls if (pc->k > maxm) 1145 1.2.2.2 tls maxm = pc->k; 1146 1.2.2.2 tls if (pc->k < minm) 1147 1.2.2.2 tls minm = pc->k; 1148 1.2.2.2 tls } 1149 1.2.2.2 tls continue; 1150 1.2.2.2 tls case BPF_ALU: 1151 1.2.2.2 tls if (pc->code == (BPF_ALU|BPF_NEG)) 1152 1.2.2.2 tls continue; 1153 1.2.2.2 tls if (BPF_SRC(pc->code) == BPF_X) 1154 1.2.2.2 tls rv |= BPFJIT_INIT_X; 1155 1.2.2.2 tls continue; 1156 1.2.2.2 tls case BPF_JMP: 1157 1.2.2.2 tls if (pc->code == (BPF_JMP|BPF_JA)) 1158 1.2.2.2 tls continue; 1159 1.2.2.2 tls if (BPF_SRC(pc->code) == BPF_X) 1160 1.2.2.2 tls rv |= BPFJIT_INIT_X; 1161 1.2.2.2 tls continue; 1162 1.2.2.2 tls case BPF_RET: 1163 1.2.2.2 tls continue; 1164 1.2.2.2 tls case BPF_MISC: 1165 1.2.2.2 tls rv |= BPFJIT_INIT_X; 1166 1.2.2.2 tls continue; 1167 1.2.2.2 tls default: 1168 1.2.2.2 tls BPFJIT_ASSERT(false); 1169 1.2.2.2 tls } 1170 1.2.2.2 tls } 1171 1.2.2.2 tls 1172 1.2.2.2 tls return rv | (maxm << 8) | minm; 1173 1.2.2.2 tls } 1174 1.2.2.2 tls 1175 1.2.2.2 tls /* 1176 1.2.2.2 tls * Convert BPF_K and BPF_X to sljit register. 1177 1.2.2.2 tls */ 1178 1.2.2.2 tls static int 1179 1.2.2.2 tls kx_to_reg(struct bpf_insn *pc) 1180 1.2.2.2 tls { 1181 1.2.2.2 tls 1182 1.2.2.2 tls switch (BPF_SRC(pc->code)) { 1183 1.2.2.2 tls case BPF_K: return SLJIT_IMM; 1184 1.2.2.2 tls case BPF_X: return BPFJIT_X; 1185 1.2.2.2 tls default: 1186 1.2.2.2 tls BPFJIT_ASSERT(false); 1187 1.2.2.2 tls return 0; 1188 1.2.2.2 tls } 1189 1.2.2.2 tls } 1190 1.2.2.2 tls 1191 1.2.2.2 tls static sljit_w 1192 1.2.2.2 tls kx_to_reg_arg(struct bpf_insn *pc) 1193 1.2.2.2 tls { 1194 1.2.2.2 tls 1195 1.2.2.2 tls switch (BPF_SRC(pc->code)) { 1196 1.2.2.2 tls case BPF_K: return (uint32_t)pc->k; /* SLJIT_IMM, pc->k, */ 1197 1.2.2.2 tls case BPF_X: return 0; /* BPFJIT_X, 0, */ 1198 1.2.2.2 tls default: 1199 1.2.2.2 tls BPFJIT_ASSERT(false); 1200 1.2.2.2 tls return 0; 1201 1.2.2.2 tls } 1202 1.2.2.2 tls } 1203 1.2.2.2 tls 1204 1.2.2.2 tls bpfjit_function_t 1205 1.2.2.2 tls bpfjit_generate_code(struct bpf_insn *insns, size_t insn_count) 1206 1.2.2.2 tls { 1207 1.2.2.2 tls void *rv; 1208 1.2.2.2 tls size_t i; 1209 1.2.2.2 tls int status; 1210 1.2.2.2 tls int branching, negate; 1211 1.2.2.2 tls unsigned int rval, mode, src; 1212 1.2.2.2 tls int ntmp; 1213 1.2.2.2 tls unsigned int locals_size; 1214 1.2.2.2 tls unsigned int minm, maxm; /* min/max k for M[k] */ 1215 1.2.2.2 tls size_t mem_locals_start; /* start of M[] array */ 1216 1.2.2.2 tls unsigned int opts; 1217 1.2.2.2 tls struct bpf_insn *pc; 1218 1.2.2.2 tls struct sljit_compiler* compiler; 1219 1.2.2.2 tls 1220 1.2.2.2 tls /* a list of jumps to a normal return from a generated function */ 1221 1.2.2.2 tls struct sljit_jump **returns; 1222 1.2.2.2 tls size_t returns_size, returns_maxsize; 1223 1.2.2.2 tls 1224 1.2.2.2 tls /* a list of jumps to out-of-bound return from a generated function */ 1225 1.2.2.2 tls struct sljit_jump **ret0; 1226 1.2.2.2 tls size_t ret0_size, ret0_maxsize; 1227 1.2.2.2 tls 1228 1.2.2.2 tls struct bpfjit_insn_data *insn_dat; 1229 1.2.2.2 tls 1230 1.2.2.2 tls /* for local use */ 1231 1.2.2.2 tls struct sljit_label *label; 1232 1.2.2.2 tls struct sljit_jump *jump; 1233 1.2.2.2 tls struct bpfjit_jump *bjump, *jtf; 1234 1.2.2.2 tls 1235 1.2.2.2 tls struct sljit_jump *to_mchain_jump; 1236 1.2.2.2 tls 1237 1.2.2.2 tls uint32_t jt, jf; 1238 1.2.2.2 tls 1239 1.2.2.2 tls rv = NULL; 1240 1.2.2.2 tls compiler = NULL; 1241 1.2.2.2 tls insn_dat = NULL; 1242 1.2.2.2 tls returns = NULL; 1243 1.2.2.2 tls ret0 = NULL; 1244 1.2.2.2 tls 1245 1.2.2.2 tls opts = bpfjit_optimization_hints(insns, insn_count); 1246 1.2.2.2 tls minm = opts & 0xff; 1247 1.2.2.2 tls maxm = (opts >> 8) & 0xff; 1248 1.2.2.2 tls mem_locals_start = mem_local_offset(0, 0); 1249 1.2.2.2 tls locals_size = (minm <= maxm) ? 1250 1.2.2.2 tls mem_local_offset(maxm + 1, minm) : mem_locals_start; 1251 1.2.2.2 tls 1252 1.2.2.2 tls ntmp = 4; 1253 1.2.2.2 tls #ifdef _KERNEL 1254 1.2.2.2 tls ntmp += 1; /* for BPFJIT_KERN_TMP */ 1255 1.2.2.2 tls #endif 1256 1.2.2.2 tls 1257 1.2.2.2 tls returns_maxsize = count_returns(insns, insn_count); 1258 1.2.2.2 tls if (returns_maxsize == 0) 1259 1.2.2.2 tls goto fail; 1260 1.2.2.2 tls 1261 1.2.2.2 tls insn_dat = BPFJIT_MALLOC(insn_count * sizeof(insn_dat[0])); 1262 1.2.2.2 tls if (insn_dat == NULL) 1263 1.2.2.2 tls goto fail; 1264 1.2.2.2 tls 1265 1.2.2.2 tls if (optimize(insns, insn_dat, insn_count) < 0) 1266 1.2.2.2 tls goto fail; 1267 1.2.2.2 tls 1268 1.2.2.2 tls ret0_size = 0; 1269 1.2.2.2 tls ret0_maxsize = get_ret0_size(insns, insn_dat, insn_count); 1270 1.2.2.2 tls if (ret0_maxsize > 0) { 1271 1.2.2.2 tls ret0 = BPFJIT_MALLOC(ret0_maxsize * sizeof(ret0[0])); 1272 1.2.2.2 tls if (ret0 == NULL) 1273 1.2.2.2 tls goto fail; 1274 1.2.2.2 tls } 1275 1.2.2.2 tls 1276 1.2.2.2 tls returns_size = 0; 1277 1.2.2.2 tls returns = BPFJIT_MALLOC(returns_maxsize * sizeof(returns[0])); 1278 1.2.2.2 tls if (returns == NULL) 1279 1.2.2.2 tls goto fail; 1280 1.2.2.2 tls 1281 1.2.2.2 tls compiler = sljit_create_compiler(); 1282 1.2.2.2 tls if (compiler == NULL) 1283 1.2.2.2 tls goto fail; 1284 1.2.2.2 tls 1285 1.2.2.2 tls #if !defined(_KERNEL) && defined(SLJIT_VERBOSE) && SLJIT_VERBOSE 1286 1.2.2.2 tls sljit_compiler_verbose(compiler, stderr); 1287 1.2.2.2 tls #endif 1288 1.2.2.2 tls 1289 1.2.2.2 tls status = sljit_emit_enter(compiler, 3, ntmp, 3, locals_size); 1290 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1291 1.2.2.2 tls goto fail; 1292 1.2.2.2 tls 1293 1.2.2.2 tls for (i = mem_locals_start; i < locals_size; i+= sizeof(uint32_t)) { 1294 1.2.2.2 tls status = sljit_emit_op1(compiler, 1295 1.2.2.2 tls SLJIT_MOV_UI, 1296 1.2.2.2 tls SLJIT_MEM1(SLJIT_LOCALS_REG), i, 1297 1.2.2.2 tls SLJIT_IMM, 0); 1298 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1299 1.2.2.2 tls goto fail; 1300 1.2.2.2 tls } 1301 1.2.2.2 tls 1302 1.2.2.2 tls if (opts & BPFJIT_INIT_A) { 1303 1.2.2.2 tls /* A = 0; */ 1304 1.2.2.2 tls status = sljit_emit_op1(compiler, 1305 1.2.2.2 tls SLJIT_MOV, 1306 1.2.2.2 tls BPFJIT_A, 0, 1307 1.2.2.2 tls SLJIT_IMM, 0); 1308 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1309 1.2.2.2 tls goto fail; 1310 1.2.2.2 tls } 1311 1.2.2.2 tls 1312 1.2.2.2 tls if (opts & BPFJIT_INIT_X) { 1313 1.2.2.2 tls /* X = 0; */ 1314 1.2.2.2 tls status = sljit_emit_op1(compiler, 1315 1.2.2.2 tls SLJIT_MOV, 1316 1.2.2.2 tls BPFJIT_X, 0, 1317 1.2.2.2 tls SLJIT_IMM, 0); 1318 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1319 1.2.2.2 tls goto fail; 1320 1.2.2.2 tls } 1321 1.2.2.2 tls 1322 1.2.2.2 tls for (i = 0; i < insn_count; i++) { 1323 1.2.2.2 tls if (insn_dat[i].bj_unreachable) 1324 1.2.2.2 tls continue; 1325 1.2.2.2 tls 1326 1.2.2.2 tls to_mchain_jump = NULL; 1327 1.2.2.2 tls 1328 1.2.2.2 tls /* 1329 1.2.2.2 tls * Resolve jumps to the current insn. 1330 1.2.2.2 tls */ 1331 1.2.2.2 tls label = NULL; 1332 1.2.2.2 tls SLIST_FOREACH(bjump, &insn_dat[i].bj_jumps, bj_entries) { 1333 1.2.2.2 tls if (bjump->bj_jump != NULL) { 1334 1.2.2.2 tls if (label == NULL) 1335 1.2.2.2 tls label = sljit_emit_label(compiler); 1336 1.2.2.2 tls if (label == NULL) 1337 1.2.2.2 tls goto fail; 1338 1.2.2.2 tls sljit_set_label(bjump->bj_jump, label); 1339 1.2.2.2 tls } 1340 1.2.2.2 tls } 1341 1.2.2.2 tls 1342 1.2.2.2 tls if (read_pkt_insn(&insns[i], NULL) && 1343 1.2.2.2 tls insn_dat[i].bj_aux.bj_rdata.bj_check_length > 0) { 1344 1.2.2.2 tls /* if (buflen < bj_check_length) return 0; */ 1345 1.2.2.2 tls jump = sljit_emit_cmp(compiler, 1346 1.2.2.2 tls SLJIT_C_LESS, 1347 1.2.2.2 tls BPFJIT_BUFLEN, 0, 1348 1.2.2.2 tls SLJIT_IMM, 1349 1.2.2.2 tls insn_dat[i].bj_aux.bj_rdata.bj_check_length); 1350 1.2.2.2 tls if (jump == NULL) 1351 1.2.2.2 tls goto fail; 1352 1.2.2.2 tls #ifdef _KERNEL 1353 1.2.2.2 tls to_mchain_jump = jump; 1354 1.2.2.2 tls #else 1355 1.2.2.2 tls ret0[ret0_size++] = jump; 1356 1.2.2.2 tls #endif 1357 1.2.2.2 tls } 1358 1.2.2.2 tls 1359 1.2.2.2 tls pc = &insns[i]; 1360 1.2.2.2 tls switch (BPF_CLASS(pc->code)) { 1361 1.2.2.2 tls 1362 1.2.2.2 tls default: 1363 1.2.2.2 tls goto fail; 1364 1.2.2.2 tls 1365 1.2.2.2 tls case BPF_LD: 1366 1.2.2.2 tls /* BPF_LD+BPF_IMM A <- k */ 1367 1.2.2.2 tls if (pc->code == (BPF_LD|BPF_IMM)) { 1368 1.2.2.2 tls status = sljit_emit_op1(compiler, 1369 1.2.2.2 tls SLJIT_MOV, 1370 1.2.2.2 tls BPFJIT_A, 0, 1371 1.2.2.2 tls SLJIT_IMM, (uint32_t)pc->k); 1372 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1373 1.2.2.2 tls goto fail; 1374 1.2.2.2 tls 1375 1.2.2.2 tls continue; 1376 1.2.2.2 tls } 1377 1.2.2.2 tls 1378 1.2.2.2 tls /* BPF_LD+BPF_MEM A <- M[k] */ 1379 1.2.2.2 tls if (pc->code == (BPF_LD|BPF_MEM)) { 1380 1.2.2.2 tls if (pc->k < minm || pc->k > maxm) 1381 1.2.2.2 tls goto fail; 1382 1.2.2.2 tls status = sljit_emit_op1(compiler, 1383 1.2.2.2 tls SLJIT_MOV_UI, 1384 1.2.2.2 tls BPFJIT_A, 0, 1385 1.2.2.2 tls SLJIT_MEM1(SLJIT_LOCALS_REG), 1386 1.2.2.2 tls mem_local_offset(pc->k, minm)); 1387 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1388 1.2.2.2 tls goto fail; 1389 1.2.2.2 tls 1390 1.2.2.2 tls continue; 1391 1.2.2.2 tls } 1392 1.2.2.2 tls 1393 1.2.2.2 tls /* BPF_LD+BPF_W+BPF_LEN A <- len */ 1394 1.2.2.2 tls if (pc->code == (BPF_LD|BPF_W|BPF_LEN)) { 1395 1.2.2.2 tls status = sljit_emit_op1(compiler, 1396 1.2.2.2 tls SLJIT_MOV, 1397 1.2.2.2 tls BPFJIT_A, 0, 1398 1.2.2.2 tls BPFJIT_WIRELEN, 0); 1399 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1400 1.2.2.2 tls goto fail; 1401 1.2.2.2 tls 1402 1.2.2.2 tls continue; 1403 1.2.2.2 tls } 1404 1.2.2.2 tls 1405 1.2.2.2 tls mode = BPF_MODE(pc->code); 1406 1.2.2.2 tls if (mode != BPF_ABS && mode != BPF_IND) 1407 1.2.2.2 tls goto fail; 1408 1.2.2.2 tls 1409 1.2.2.2 tls status = emit_pkt_read(compiler, pc, 1410 1.2.2.2 tls to_mchain_jump, ret0, &ret0_size); 1411 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1412 1.2.2.2 tls goto fail; 1413 1.2.2.2 tls 1414 1.2.2.2 tls continue; 1415 1.2.2.2 tls 1416 1.2.2.2 tls case BPF_LDX: 1417 1.2.2.2 tls mode = BPF_MODE(pc->code); 1418 1.2.2.2 tls 1419 1.2.2.2 tls /* BPF_LDX+BPF_W+BPF_IMM X <- k */ 1420 1.2.2.2 tls if (mode == BPF_IMM) { 1421 1.2.2.2 tls if (BPF_SIZE(pc->code) != BPF_W) 1422 1.2.2.2 tls goto fail; 1423 1.2.2.2 tls status = sljit_emit_op1(compiler, 1424 1.2.2.2 tls SLJIT_MOV, 1425 1.2.2.2 tls BPFJIT_X, 0, 1426 1.2.2.2 tls SLJIT_IMM, (uint32_t)pc->k); 1427 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1428 1.2.2.2 tls goto fail; 1429 1.2.2.2 tls 1430 1.2.2.2 tls continue; 1431 1.2.2.2 tls } 1432 1.2.2.2 tls 1433 1.2.2.2 tls /* BPF_LDX+BPF_W+BPF_LEN X <- len */ 1434 1.2.2.2 tls if (mode == BPF_LEN) { 1435 1.2.2.2 tls if (BPF_SIZE(pc->code) != BPF_W) 1436 1.2.2.2 tls goto fail; 1437 1.2.2.2 tls status = sljit_emit_op1(compiler, 1438 1.2.2.2 tls SLJIT_MOV, 1439 1.2.2.2 tls BPFJIT_X, 0, 1440 1.2.2.2 tls BPFJIT_WIRELEN, 0); 1441 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1442 1.2.2.2 tls goto fail; 1443 1.2.2.2 tls 1444 1.2.2.2 tls continue; 1445 1.2.2.2 tls } 1446 1.2.2.2 tls 1447 1.2.2.2 tls /* BPF_LDX+BPF_W+BPF_MEM X <- M[k] */ 1448 1.2.2.2 tls if (mode == BPF_MEM) { 1449 1.2.2.2 tls if (BPF_SIZE(pc->code) != BPF_W) 1450 1.2.2.2 tls goto fail; 1451 1.2.2.2 tls if (pc->k < minm || pc->k > maxm) 1452 1.2.2.2 tls goto fail; 1453 1.2.2.2 tls status = sljit_emit_op1(compiler, 1454 1.2.2.2 tls SLJIT_MOV_UI, 1455 1.2.2.2 tls BPFJIT_X, 0, 1456 1.2.2.2 tls SLJIT_MEM1(SLJIT_LOCALS_REG), 1457 1.2.2.2 tls mem_local_offset(pc->k, minm)); 1458 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1459 1.2.2.2 tls goto fail; 1460 1.2.2.2 tls 1461 1.2.2.2 tls continue; 1462 1.2.2.2 tls } 1463 1.2.2.2 tls 1464 1.2.2.2 tls /* BPF_LDX+BPF_B+BPF_MSH X <- 4*(P[k:1]&0xf) */ 1465 1.2.2.2 tls if (mode != BPF_MSH || BPF_SIZE(pc->code) != BPF_B) 1466 1.2.2.2 tls goto fail; 1467 1.2.2.2 tls 1468 1.2.2.2 tls status = emit_msh(compiler, pc, 1469 1.2.2.2 tls to_mchain_jump, ret0, &ret0_size); 1470 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1471 1.2.2.2 tls goto fail; 1472 1.2.2.2 tls 1473 1.2.2.2 tls continue; 1474 1.2.2.2 tls 1475 1.2.2.2 tls case BPF_ST: 1476 1.2.2.2 tls if (pc->code != BPF_ST || pc->k < minm || pc->k > maxm) 1477 1.2.2.2 tls goto fail; 1478 1.2.2.2 tls 1479 1.2.2.2 tls status = sljit_emit_op1(compiler, 1480 1.2.2.2 tls SLJIT_MOV_UI, 1481 1.2.2.2 tls SLJIT_MEM1(SLJIT_LOCALS_REG), 1482 1.2.2.2 tls mem_local_offset(pc->k, minm), 1483 1.2.2.2 tls BPFJIT_A, 0); 1484 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1485 1.2.2.2 tls goto fail; 1486 1.2.2.2 tls 1487 1.2.2.2 tls continue; 1488 1.2.2.2 tls 1489 1.2.2.2 tls case BPF_STX: 1490 1.2.2.2 tls if (pc->code != BPF_STX || pc->k < minm || pc->k > maxm) 1491 1.2.2.2 tls goto fail; 1492 1.2.2.2 tls 1493 1.2.2.2 tls status = sljit_emit_op1(compiler, 1494 1.2.2.2 tls SLJIT_MOV_UI, 1495 1.2.2.2 tls SLJIT_MEM1(SLJIT_LOCALS_REG), 1496 1.2.2.2 tls mem_local_offset(pc->k, minm), 1497 1.2.2.2 tls BPFJIT_X, 0); 1498 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1499 1.2.2.2 tls goto fail; 1500 1.2.2.2 tls 1501 1.2.2.2 tls continue; 1502 1.2.2.2 tls 1503 1.2.2.2 tls case BPF_ALU: 1504 1.2.2.2 tls 1505 1.2.2.2 tls if (pc->code == (BPF_ALU|BPF_NEG)) { 1506 1.2.2.2 tls status = sljit_emit_op1(compiler, 1507 1.2.2.2 tls SLJIT_NEG, 1508 1.2.2.2 tls BPFJIT_A, 0, 1509 1.2.2.2 tls BPFJIT_A, 0); 1510 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1511 1.2.2.2 tls goto fail; 1512 1.2.2.2 tls 1513 1.2.2.2 tls continue; 1514 1.2.2.2 tls } 1515 1.2.2.2 tls 1516 1.2.2.2 tls if (BPF_OP(pc->code) != BPF_DIV) { 1517 1.2.2.2 tls status = sljit_emit_op2(compiler, 1518 1.2.2.2 tls bpf_alu_to_sljit_op(pc), 1519 1.2.2.2 tls BPFJIT_A, 0, 1520 1.2.2.2 tls BPFJIT_A, 0, 1521 1.2.2.2 tls kx_to_reg(pc), kx_to_reg_arg(pc)); 1522 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1523 1.2.2.2 tls goto fail; 1524 1.2.2.2 tls 1525 1.2.2.2 tls continue; 1526 1.2.2.2 tls } 1527 1.2.2.2 tls 1528 1.2.2.2 tls /* BPF_DIV */ 1529 1.2.2.2 tls 1530 1.2.2.2 tls src = BPF_SRC(pc->code); 1531 1.2.2.2 tls if (src != BPF_X && src != BPF_K) 1532 1.2.2.2 tls goto fail; 1533 1.2.2.2 tls 1534 1.2.2.2 tls /* division by zero? */ 1535 1.2.2.2 tls if (src == BPF_X) { 1536 1.2.2.2 tls jump = sljit_emit_cmp(compiler, 1537 1.2.2.2 tls SLJIT_C_EQUAL|SLJIT_INT_OP, 1538 1.2.2.2 tls BPFJIT_X, 0, 1539 1.2.2.2 tls SLJIT_IMM, 0); 1540 1.2.2.2 tls if (jump == NULL) 1541 1.2.2.2 tls goto fail; 1542 1.2.2.2 tls ret0[ret0_size++] = jump; 1543 1.2.2.2 tls } else if (pc->k == 0) { 1544 1.2.2.2 tls jump = sljit_emit_jump(compiler, SLJIT_JUMP); 1545 1.2.2.2 tls if (jump == NULL) 1546 1.2.2.2 tls goto fail; 1547 1.2.2.2 tls ret0[ret0_size++] = jump; 1548 1.2.2.2 tls } 1549 1.2.2.2 tls 1550 1.2.2.2 tls if (src == BPF_X) { 1551 1.2.2.2 tls status = emit_division(compiler, BPFJIT_X, 0); 1552 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1553 1.2.2.2 tls goto fail; 1554 1.2.2.2 tls } else if (pc->k != 0) { 1555 1.2.2.2 tls if (pc->k & (pc->k - 1)) { 1556 1.2.2.2 tls status = emit_division(compiler, 1557 1.2.2.2 tls SLJIT_IMM, (uint32_t)pc->k); 1558 1.2.2.2 tls } else { 1559 1.2.2.2 tls status = emit_pow2_division(compiler, 1560 1.2.2.2 tls (uint32_t)pc->k); 1561 1.2.2.2 tls } 1562 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1563 1.2.2.2 tls goto fail; 1564 1.2.2.2 tls } 1565 1.2.2.2 tls 1566 1.2.2.2 tls continue; 1567 1.2.2.2 tls 1568 1.2.2.2 tls case BPF_JMP: 1569 1.2.2.2 tls 1570 1.2.2.2 tls if (pc->code == (BPF_JMP|BPF_JA)) { 1571 1.2.2.2 tls jt = jf = pc->k; 1572 1.2.2.2 tls } else { 1573 1.2.2.2 tls jt = pc->jt; 1574 1.2.2.2 tls jf = pc->jf; 1575 1.2.2.2 tls } 1576 1.2.2.2 tls 1577 1.2.2.2 tls negate = (jt == 0) ? 1 : 0; 1578 1.2.2.2 tls branching = (jt == jf) ? 0 : 1; 1579 1.2.2.2 tls jtf = insn_dat[i].bj_aux.bj_jdata.bj_jtf; 1580 1.2.2.2 tls 1581 1.2.2.2 tls if (branching) { 1582 1.2.2.2 tls if (BPF_OP(pc->code) != BPF_JSET) { 1583 1.2.2.2 tls jump = sljit_emit_cmp(compiler, 1584 1.2.2.2 tls bpf_jmp_to_sljit_cond(pc, negate), 1585 1.2.2.2 tls BPFJIT_A, 0, 1586 1.2.2.2 tls kx_to_reg(pc), kx_to_reg_arg(pc)); 1587 1.2.2.2 tls } else { 1588 1.2.2.2 tls status = sljit_emit_op2(compiler, 1589 1.2.2.2 tls SLJIT_AND, 1590 1.2.2.2 tls BPFJIT_TMP1, 0, 1591 1.2.2.2 tls BPFJIT_A, 0, 1592 1.2.2.2 tls kx_to_reg(pc), kx_to_reg_arg(pc)); 1593 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1594 1.2.2.2 tls goto fail; 1595 1.2.2.2 tls 1596 1.2.2.2 tls jump = sljit_emit_cmp(compiler, 1597 1.2.2.2 tls bpf_jmp_to_sljit_cond(pc, negate), 1598 1.2.2.2 tls BPFJIT_TMP1, 0, 1599 1.2.2.2 tls SLJIT_IMM, 0); 1600 1.2.2.2 tls } 1601 1.2.2.2 tls 1602 1.2.2.2 tls if (jump == NULL) 1603 1.2.2.2 tls goto fail; 1604 1.2.2.2 tls 1605 1.2.2.2 tls BPFJIT_ASSERT(jtf[negate].bj_jump == NULL); 1606 1.2.2.2 tls jtf[negate].bj_jump = jump; 1607 1.2.2.2 tls } 1608 1.2.2.2 tls 1609 1.2.2.2 tls if (!branching || (jt != 0 && jf != 0)) { 1610 1.2.2.2 tls jump = sljit_emit_jump(compiler, SLJIT_JUMP); 1611 1.2.2.2 tls if (jump == NULL) 1612 1.2.2.2 tls goto fail; 1613 1.2.2.2 tls 1614 1.2.2.2 tls BPFJIT_ASSERT(jtf[branching].bj_jump == NULL); 1615 1.2.2.2 tls jtf[branching].bj_jump = jump; 1616 1.2.2.2 tls } 1617 1.2.2.2 tls 1618 1.2.2.2 tls continue; 1619 1.2.2.2 tls 1620 1.2.2.2 tls case BPF_RET: 1621 1.2.2.2 tls 1622 1.2.2.2 tls rval = BPF_RVAL(pc->code); 1623 1.2.2.2 tls if (rval == BPF_X) 1624 1.2.2.2 tls goto fail; 1625 1.2.2.2 tls 1626 1.2.2.2 tls /* BPF_RET+BPF_K accept k bytes */ 1627 1.2.2.2 tls if (rval == BPF_K) { 1628 1.2.2.2 tls status = sljit_emit_op1(compiler, 1629 1.2.2.2 tls SLJIT_MOV, 1630 1.2.2.2 tls BPFJIT_A, 0, 1631 1.2.2.2 tls SLJIT_IMM, (uint32_t)pc->k); 1632 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1633 1.2.2.2 tls goto fail; 1634 1.2.2.2 tls } 1635 1.2.2.2 tls 1636 1.2.2.2 tls /* BPF_RET+BPF_A accept A bytes */ 1637 1.2.2.2 tls if (rval == BPF_A) { 1638 1.2.2.2 tls #if BPFJIT_A != SLJIT_RETURN_REG 1639 1.2.2.2 tls status = sljit_emit_op1(compiler, 1640 1.2.2.2 tls SLJIT_MOV, 1641 1.2.2.2 tls SLJIT_RETURN_REG, 0, 1642 1.2.2.2 tls BPFJIT_A, 0); 1643 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1644 1.2.2.2 tls goto fail; 1645 1.2.2.2 tls #endif 1646 1.2.2.2 tls } 1647 1.2.2.2 tls 1648 1.2.2.2 tls /* 1649 1.2.2.2 tls * Save a jump to a normal return. If the program 1650 1.2.2.2 tls * ends with BPF_RET, no jump is needed because 1651 1.2.2.2 tls * the normal return is generated right after the 1652 1.2.2.2 tls * last instruction. 1653 1.2.2.2 tls */ 1654 1.2.2.2 tls if (i != insn_count - 1) { 1655 1.2.2.2 tls jump = sljit_emit_jump(compiler, SLJIT_JUMP); 1656 1.2.2.2 tls if (jump == NULL) 1657 1.2.2.2 tls goto fail; 1658 1.2.2.2 tls returns[returns_size++] = jump; 1659 1.2.2.2 tls } 1660 1.2.2.2 tls 1661 1.2.2.2 tls continue; 1662 1.2.2.2 tls 1663 1.2.2.2 tls case BPF_MISC: 1664 1.2.2.2 tls 1665 1.2.2.2 tls if (pc->code == (BPF_MISC|BPF_TAX)) { 1666 1.2.2.2 tls status = sljit_emit_op1(compiler, 1667 1.2.2.2 tls SLJIT_MOV_UI, 1668 1.2.2.2 tls BPFJIT_X, 0, 1669 1.2.2.2 tls BPFJIT_A, 0); 1670 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1671 1.2.2.2 tls goto fail; 1672 1.2.2.2 tls 1673 1.2.2.2 tls continue; 1674 1.2.2.2 tls } 1675 1.2.2.2 tls 1676 1.2.2.2 tls if (pc->code == (BPF_MISC|BPF_TXA)) { 1677 1.2.2.2 tls status = sljit_emit_op1(compiler, 1678 1.2.2.2 tls SLJIT_MOV, 1679 1.2.2.2 tls BPFJIT_A, 0, 1680 1.2.2.2 tls BPFJIT_X, 0); 1681 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1682 1.2.2.2 tls goto fail; 1683 1.2.2.2 tls 1684 1.2.2.2 tls continue; 1685 1.2.2.2 tls } 1686 1.2.2.2 tls 1687 1.2.2.2 tls goto fail; 1688 1.2.2.2 tls } /* switch */ 1689 1.2.2.2 tls } /* main loop */ 1690 1.2.2.2 tls 1691 1.2.2.2 tls BPFJIT_ASSERT(ret0_size == ret0_maxsize); 1692 1.2.2.2 tls BPFJIT_ASSERT(returns_size <= returns_maxsize); 1693 1.2.2.2 tls 1694 1.2.2.2 tls if (returns_size > 0) { 1695 1.2.2.2 tls label = sljit_emit_label(compiler); 1696 1.2.2.2 tls if (label == NULL) 1697 1.2.2.2 tls goto fail; 1698 1.2.2.2 tls for (i = 0; i < returns_size; i++) 1699 1.2.2.2 tls sljit_set_label(returns[i], label); 1700 1.2.2.2 tls } 1701 1.2.2.2 tls 1702 1.2.2.2 tls status = sljit_emit_return(compiler, 1703 1.2.2.2 tls SLJIT_MOV_UI, 1704 1.2.2.2 tls BPFJIT_A, 0); 1705 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1706 1.2.2.2 tls goto fail; 1707 1.2.2.2 tls 1708 1.2.2.2 tls if (ret0_size > 0) { 1709 1.2.2.2 tls label = sljit_emit_label(compiler); 1710 1.2.2.2 tls if (label == NULL) 1711 1.2.2.2 tls goto fail; 1712 1.2.2.2 tls 1713 1.2.2.2 tls for (i = 0; i < ret0_size; i++) 1714 1.2.2.2 tls sljit_set_label(ret0[i], label); 1715 1.2.2.2 tls 1716 1.2.2.2 tls status = sljit_emit_op1(compiler, 1717 1.2.2.2 tls SLJIT_MOV, 1718 1.2.2.2 tls SLJIT_RETURN_REG, 0, 1719 1.2.2.2 tls SLJIT_IMM, 0); 1720 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1721 1.2.2.2 tls goto fail; 1722 1.2.2.2 tls 1723 1.2.2.2 tls status = sljit_emit_return(compiler, 1724 1.2.2.2 tls SLJIT_MOV_UI, 1725 1.2.2.2 tls SLJIT_RETURN_REG, 0); 1726 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1727 1.2.2.2 tls goto fail; 1728 1.2.2.2 tls } 1729 1.2.2.2 tls 1730 1.2.2.2 tls rv = sljit_generate_code(compiler); 1731 1.2.2.2 tls 1732 1.2.2.2 tls fail: 1733 1.2.2.2 tls if (compiler != NULL) 1734 1.2.2.2 tls sljit_free_compiler(compiler); 1735 1.2.2.2 tls 1736 1.2.2.2 tls if (insn_dat != NULL) 1737 1.2.2.2 tls BPFJIT_FREE(insn_dat); 1738 1.2.2.2 tls 1739 1.2.2.2 tls if (returns != NULL) 1740 1.2.2.2 tls BPFJIT_FREE(returns); 1741 1.2.2.2 tls 1742 1.2.2.2 tls if (ret0 != NULL) 1743 1.2.2.2 tls BPFJIT_FREE(ret0); 1744 1.2.2.2 tls 1745 1.2.2.2 tls return (bpfjit_function_t)rv; 1746 1.2.2.2 tls } 1747 1.2.2.2 tls 1748 1.2.2.2 tls void 1749 1.2.2.2 tls bpfjit_free_code(bpfjit_function_t code) 1750 1.2.2.2 tls { 1751 1.2.2.2 tls 1752 1.2.2.2 tls sljit_free_code((void *)code); 1753 1.2.2.2 tls } 1754
Indexes created Thu Oct 02 10:09:58 GMT 2025