bpfjit.c revision 1.2.2.4
1 1.2.2.3 tls /* $NetBSD: bpfjit.c,v 1.2.2.4 2017/12/03 11:39:02 jdolecek Exp $ */ 2 1.2.2.3 tls 3 1.2.2.2 tls /*- 4 1.2.2.4 jdolecek * Copyright (c) 2011-2015 Alexander Nasonov. 5 1.2.2.2 tls * All rights reserved. 6 1.2.2.2 tls * 7 1.2.2.2 tls * Redistribution and use in source and binary forms, with or without 8 1.2.2.2 tls * modification, are permitted provided that the following conditions 9 1.2.2.2 tls * are met: 10 1.2.2.2 tls * 11 1.2.2.2 tls * 1. Redistributions of source code must retain the above copyright 12 1.2.2.2 tls * notice, this list of conditions and the following disclaimer. 13 1.2.2.2 tls * 2. Redistributions in binary form must reproduce the above copyright 14 1.2.2.2 tls * notice, this list of conditions and the following disclaimer in 15 1.2.2.2 tls * the documentation and/or other materials provided with the 16 1.2.2.2 tls * distribution. 17 1.2.2.2 tls * 18 1.2.2.2 tls * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 1.2.2.2 tls * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 1.2.2.2 tls * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 21 1.2.2.2 tls * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 22 1.2.2.2 tls * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 23 1.2.2.2 tls * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 24 1.2.2.2 tls * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 25 1.2.2.2 tls * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 26 1.2.2.2 tls * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 27 1.2.2.2 tls * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 28 1.2.2.2 tls * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 1.2.2.2 tls * SUCH DAMAGE. 30 1.2.2.2 tls */ 31 1.2.2.2 tls 32 1.2.2.2 tls #include <sys/cdefs.h> 33 1.2.2.2 tls #ifdef _KERNEL 34 1.2.2.3 tls __KERNEL_RCSID(0, "$NetBSD: bpfjit.c,v 1.2.2.4 2017/12/03 11:39:02 jdolecek Exp $"); 35 1.2.2.2 tls #else 36 1.2.2.3 tls __RCSID("$NetBSD: bpfjit.c,v 1.2.2.4 2017/12/03 11:39:02 jdolecek Exp $"); 37 1.2.2.2 tls #endif 38 1.2.2.2 tls 39 1.2.2.3 tls #include <sys/types.h> 40 1.2.2.3 tls #include <sys/queue.h> 41 1.2.2.2 tls 42 1.2.2.2 tls #ifndef _KERNEL 43 1.2.2.2 tls #include <assert.h> 44 1.2.2.3 tls #define BJ_ASSERT(c) assert(c) 45 1.2.2.2 tls #else 46 1.2.2.3 tls #define BJ_ASSERT(c) KASSERT(c) 47 1.2.2.2 tls #endif 48 1.2.2.2 tls 49 1.2.2.2 tls #ifndef _KERNEL 50 1.2.2.2 tls #include <stdlib.h> 51 1.2.2.3 tls #define BJ_ALLOC(sz) malloc(sz) 52 1.2.2.3 tls #define BJ_FREE(p, sz) free(p) 53 1.2.2.2 tls #else 54 1.2.2.3 tls #include <sys/kmem.h> 55 1.2.2.3 tls #define BJ_ALLOC(sz) kmem_alloc(sz, KM_SLEEP) 56 1.2.2.3 tls #define BJ_FREE(p, sz) kmem_free(p, sz) 57 1.2.2.2 tls #endif 58 1.2.2.2 tls 59 1.2.2.2 tls #ifndef _KERNEL 60 1.2.2.2 tls #include <limits.h> 61 1.2.2.2 tls #include <stdbool.h> 62 1.2.2.2 tls #include <stddef.h> 63 1.2.2.2 tls #include <stdint.h> 64 1.2.2.2 tls #else 65 1.2.2.2 tls #include <sys/atomic.h> 66 1.2.2.2 tls #include <sys/module.h> 67 1.2.2.2 tls #endif 68 1.2.2.2 tls 69 1.2.2.3 tls #define __BPF_PRIVATE 70 1.2.2.3 tls #include <net/bpf.h> 71 1.2.2.3 tls #include <net/bpfjit.h> 72 1.2.2.2 tls #include <sljitLir.h> 73 1.2.2.2 tls 74 1.2.2.2 tls #if !defined(_KERNEL) && defined(SLJIT_VERBOSE) && SLJIT_VERBOSE 75 1.2.2.2 tls #include <stdio.h> /* for stderr */ 76 1.2.2.2 tls #endif 77 1.2.2.2 tls 78 1.2.2.3 tls /* 79 1.2.2.4 jdolecek * Number of saved registers to pass to sljit_emit_enter() function. 80 1.2.2.4 jdolecek */ 81 1.2.2.4 jdolecek #define NSAVEDS 3 82 1.2.2.4 jdolecek 83 1.2.2.4 jdolecek /* 84 1.2.2.3 tls * Arguments of generated bpfjit_func_t. 85 1.2.2.3 tls * The first argument is reassigned upon entry 86 1.2.2.3 tls * to a more frequently used buf argument. 87 1.2.2.3 tls */ 88 1.2.2.4 jdolecek #define BJ_CTX_ARG SLJIT_S0 89 1.2.2.4 jdolecek #define BJ_ARGS SLJIT_S1 90 1.2.2.3 tls 91 1.2.2.3 tls /* 92 1.2.2.3 tls * Permanent register assignments. 93 1.2.2.3 tls */ 94 1.2.2.4 jdolecek #define BJ_BUF SLJIT_S0 95 1.2.2.4 jdolecek //#define BJ_ARGS SLJIT_S1 96 1.2.2.4 jdolecek #define BJ_BUFLEN SLJIT_S2 97 1.2.2.4 jdolecek #define BJ_AREG SLJIT_R0 98 1.2.2.4 jdolecek #define BJ_TMP1REG SLJIT_R1 99 1.2.2.4 jdolecek #define BJ_TMP2REG SLJIT_R2 100 1.2.2.4 jdolecek #define BJ_XREG SLJIT_R3 101 1.2.2.4 jdolecek #define BJ_TMP3REG SLJIT_R4 102 1.2.2.3 tls 103 1.2.2.3 tls #ifdef _KERNEL 104 1.2.2.3 tls #define MAX_MEMWORDS BPF_MAX_MEMWORDS 105 1.2.2.3 tls #else 106 1.2.2.3 tls #define MAX_MEMWORDS BPF_MEMWORDS 107 1.2.2.3 tls #endif 108 1.2.2.2 tls 109 1.2.2.3 tls #define BJ_INIT_NOBITS ((bpf_memword_init_t)0) 110 1.2.2.3 tls #define BJ_INIT_MBIT(k) BPF_MEMWORD_INIT(k) 111 1.2.2.3 tls #define BJ_INIT_ABIT BJ_INIT_MBIT(MAX_MEMWORDS) 112 1.2.2.3 tls #define BJ_INIT_XBIT BJ_INIT_MBIT(MAX_MEMWORDS + 1) 113 1.2.2.2 tls 114 1.2.2.3 tls /* 115 1.2.2.3 tls * Get a number of memwords and external memwords from a bpf_ctx object. 116 1.2.2.2 tls */ 117 1.2.2.3 tls #define GET_EXTWORDS(bc) ((bc) ? (bc)->extwords : 0) 118 1.2.2.3 tls #define GET_MEMWORDS(bc) (GET_EXTWORDS(bc) ? GET_EXTWORDS(bc) : BPF_MEMWORDS) 119 1.2.2.2 tls 120 1.2.2.3 tls /* 121 1.2.2.3 tls * Optimization hints. 122 1.2.2.3 tls */ 123 1.2.2.3 tls typedef unsigned int bpfjit_hint_t; 124 1.2.2.3 tls #define BJ_HINT_ABS 0x01 /* packet read at absolute offset */ 125 1.2.2.3 tls #define BJ_HINT_IND 0x02 /* packet read at variable offset */ 126 1.2.2.3 tls #define BJ_HINT_MSH 0x04 /* BPF_MSH instruction */ 127 1.2.2.3 tls #define BJ_HINT_COP 0x08 /* BPF_COP or BPF_COPX instruction */ 128 1.2.2.3 tls #define BJ_HINT_COPX 0x10 /* BPF_COPX instruction */ 129 1.2.2.3 tls #define BJ_HINT_XREG 0x20 /* BJ_XREG is needed */ 130 1.2.2.3 tls #define BJ_HINT_LDX 0x40 /* BPF_LDX instruction */ 131 1.2.2.3 tls #define BJ_HINT_PKT (BJ_HINT_ABS|BJ_HINT_IND|BJ_HINT_MSH) 132 1.2.2.2 tls 133 1.2.2.2 tls /* 134 1.2.2.3 tls * Datatype for Array Bounds Check Elimination (ABC) pass. 135 1.2.2.2 tls */ 136 1.2.2.3 tls typedef uint64_t bpfjit_abc_length_t; 137 1.2.2.3 tls #define MAX_ABC_LENGTH (UINT32_MAX + UINT64_C(4)) /* max. width is 4 */ 138 1.2.2.3 tls 139 1.2.2.3 tls struct bpfjit_stack 140 1.2.2.2 tls { 141 1.2.2.3 tls bpf_ctx_t *ctx; 142 1.2.2.3 tls uint32_t *extmem; /* pointer to external memory store */ 143 1.2.2.3 tls uint32_t reg; /* saved A or X register */ 144 1.2.2.3 tls #ifdef _KERNEL 145 1.2.2.3 tls int err; /* 3rd argument for m_xword/m_xhalf/m_xbyte function call */ 146 1.2.2.3 tls #endif 147 1.2.2.3 tls uint32_t mem[BPF_MEMWORDS]; /* internal memory store */ 148 1.2.2.2 tls }; 149 1.2.2.2 tls 150 1.2.2.2 tls /* 151 1.2.2.2 tls * Data for BPF_JMP instruction. 152 1.2.2.3 tls * Forward declaration for struct bpfjit_jump. 153 1.2.2.2 tls */ 154 1.2.2.3 tls struct bpfjit_jump_data; 155 1.2.2.3 tls 156 1.2.2.3 tls /* 157 1.2.2.3 tls * Node of bjumps list. 158 1.2.2.3 tls */ 159 1.2.2.3 tls struct bpfjit_jump { 160 1.2.2.3 tls struct sljit_jump *sjump; 161 1.2.2.3 tls SLIST_ENTRY(bpfjit_jump) entries; 162 1.2.2.3 tls struct bpfjit_jump_data *jdata; 163 1.2.2.3 tls }; 164 1.2.2.3 tls 165 1.2.2.3 tls /* 166 1.2.2.3 tls * Data for BPF_JMP instruction. 167 1.2.2.3 tls */ 168 1.2.2.3 tls struct bpfjit_jump_data { 169 1.2.2.3 tls /* 170 1.2.2.3 tls * These entries make up bjumps list: 171 1.2.2.3 tls * jtf[0] - when coming from jt path, 172 1.2.2.3 tls * jtf[1] - when coming from jf path. 173 1.2.2.3 tls */ 174 1.2.2.3 tls struct bpfjit_jump jtf[2]; 175 1.2.2.3 tls /* 176 1.2.2.3 tls * Length calculated by Array Bounds Check Elimination (ABC) pass. 177 1.2.2.3 tls */ 178 1.2.2.3 tls bpfjit_abc_length_t abc_length; 179 1.2.2.2 tls /* 180 1.2.2.3 tls * Length checked by the last out-of-bounds check. 181 1.2.2.2 tls */ 182 1.2.2.3 tls bpfjit_abc_length_t checked_length; 183 1.2.2.2 tls }; 184 1.2.2.2 tls 185 1.2.2.2 tls /* 186 1.2.2.2 tls * Data for "read from packet" instructions. 187 1.2.2.2 tls * See also read_pkt_insn() function below. 188 1.2.2.2 tls */ 189 1.2.2.3 tls struct bpfjit_read_pkt_data { 190 1.2.2.3 tls /* 191 1.2.2.3 tls * Length calculated by Array Bounds Check Elimination (ABC) pass. 192 1.2.2.3 tls */ 193 1.2.2.3 tls bpfjit_abc_length_t abc_length; 194 1.2.2.2 tls /* 195 1.2.2.3 tls * If positive, emit "if (buflen < check_length) return 0" 196 1.2.2.3 tls * out-of-bounds check. 197 1.2.2.3 tls * Values greater than UINT32_MAX generate unconditional "return 0". 198 1.2.2.2 tls */ 199 1.2.2.3 tls bpfjit_abc_length_t check_length; 200 1.2.2.2 tls }; 201 1.2.2.2 tls 202 1.2.2.2 tls /* 203 1.2.2.2 tls * Additional (optimization-related) data for bpf_insn. 204 1.2.2.2 tls */ 205 1.2.2.3 tls struct bpfjit_insn_data { 206 1.2.2.2 tls /* List of jumps to this insn. */ 207 1.2.2.3 tls SLIST_HEAD(, bpfjit_jump) bjumps; 208 1.2.2.2 tls 209 1.2.2.2 tls union { 210 1.2.2.3 tls struct bpfjit_jump_data jdata; 211 1.2.2.3 tls struct bpfjit_read_pkt_data rdata; 212 1.2.2.3 tls } u; 213 1.2.2.2 tls 214 1.2.2.3 tls bpf_memword_init_t invalid; 215 1.2.2.3 tls bool unreachable; 216 1.2.2.2 tls }; 217 1.2.2.2 tls 218 1.2.2.2 tls #ifdef _KERNEL 219 1.2.2.2 tls 220 1.2.2.2 tls uint32_t m_xword(const struct mbuf *, uint32_t, int *); 221 1.2.2.2 tls uint32_t m_xhalf(const struct mbuf *, uint32_t, int *); 222 1.2.2.2 tls uint32_t m_xbyte(const struct mbuf *, uint32_t, int *); 223 1.2.2.2 tls 224 1.2.2.2 tls MODULE(MODULE_CLASS_MISC, bpfjit, "sljit") 225 1.2.2.2 tls 226 1.2.2.2 tls static int 227 1.2.2.2 tls bpfjit_modcmd(modcmd_t cmd, void *arg) 228 1.2.2.2 tls { 229 1.2.2.2 tls 230 1.2.2.2 tls switch (cmd) { 231 1.2.2.2 tls case MODULE_CMD_INIT: 232 1.2.2.2 tls bpfjit_module_ops.bj_free_code = &bpfjit_free_code; 233 1.2.2.2 tls membar_producer(); 234 1.2.2.2 tls bpfjit_module_ops.bj_generate_code = &bpfjit_generate_code; 235 1.2.2.2 tls membar_producer(); 236 1.2.2.2 tls return 0; 237 1.2.2.2 tls 238 1.2.2.2 tls case MODULE_CMD_FINI: 239 1.2.2.2 tls return EOPNOTSUPP; 240 1.2.2.2 tls 241 1.2.2.2 tls default: 242 1.2.2.2 tls return ENOTTY; 243 1.2.2.2 tls } 244 1.2.2.2 tls } 245 1.2.2.2 tls #endif 246 1.2.2.2 tls 247 1.2.2.3 tls /* 248 1.2.2.3 tls * Return a number of scratch registers to pass 249 1.2.2.3 tls * to sljit_emit_enter() function. 250 1.2.2.3 tls */ 251 1.2.2.4 jdolecek static sljit_s32 252 1.2.2.3 tls nscratches(bpfjit_hint_t hints) 253 1.2.2.3 tls { 254 1.2.2.4 jdolecek sljit_s32 rv = 2; 255 1.2.2.3 tls 256 1.2.2.3 tls #ifdef _KERNEL 257 1.2.2.3 tls if (hints & BJ_HINT_PKT) 258 1.2.2.3 tls rv = 3; /* xcall with three arguments */ 259 1.2.2.3 tls #endif 260 1.2.2.3 tls 261 1.2.2.3 tls if (hints & BJ_HINT_IND) 262 1.2.2.3 tls rv = 3; /* uses BJ_TMP2REG */ 263 1.2.2.3 tls 264 1.2.2.3 tls if (hints & BJ_HINT_COP) 265 1.2.2.3 tls rv = 3; /* calls copfunc with three arguments */ 266 1.2.2.3 tls 267 1.2.2.3 tls if (hints & BJ_HINT_XREG) 268 1.2.2.3 tls rv = 4; /* uses BJ_XREG */ 269 1.2.2.3 tls 270 1.2.2.3 tls #ifdef _KERNEL 271 1.2.2.3 tls if (hints & BJ_HINT_LDX) 272 1.2.2.3 tls rv = 5; /* uses BJ_TMP3REG */ 273 1.2.2.3 tls #endif 274 1.2.2.3 tls 275 1.2.2.3 tls if (hints & BJ_HINT_COPX) 276 1.2.2.3 tls rv = 5; /* uses BJ_TMP3REG */ 277 1.2.2.3 tls 278 1.2.2.3 tls return rv; 279 1.2.2.3 tls } 280 1.2.2.3 tls 281 1.2.2.2 tls static uint32_t 282 1.2.2.3 tls read_width(const struct bpf_insn *pc) 283 1.2.2.2 tls { 284 1.2.2.2 tls 285 1.2.2.2 tls switch (BPF_SIZE(pc->code)) { 286 1.2.2.4 jdolecek case BPF_W: return 4; 287 1.2.2.4 jdolecek case BPF_H: return 2; 288 1.2.2.4 jdolecek case BPF_B: return 1; 289 1.2.2.4 jdolecek default: return 0; 290 1.2.2.2 tls } 291 1.2.2.2 tls } 292 1.2.2.2 tls 293 1.2.2.2 tls /* 294 1.2.2.3 tls * Copy buf and buflen members of bpf_args from BJ_ARGS 295 1.2.2.3 tls * pointer to BJ_BUF and BJ_BUFLEN registers. 296 1.2.2.2 tls */ 297 1.2.2.3 tls static int 298 1.2.2.3 tls load_buf_buflen(struct sljit_compiler *compiler) 299 1.2.2.2 tls { 300 1.2.2.3 tls int status; 301 1.2.2.2 tls 302 1.2.2.3 tls status = sljit_emit_op1(compiler, 303 1.2.2.3 tls SLJIT_MOV_P, 304 1.2.2.3 tls BJ_BUF, 0, 305 1.2.2.3 tls SLJIT_MEM1(BJ_ARGS), 306 1.2.2.3 tls offsetof(struct bpf_args, pkt)); 307 1.2.2.3 tls if (status != SLJIT_SUCCESS) 308 1.2.2.3 tls return status; 309 1.2.2.3 tls 310 1.2.2.3 tls status = sljit_emit_op1(compiler, 311 1.2.2.3 tls SLJIT_MOV, /* size_t source */ 312 1.2.2.3 tls BJ_BUFLEN, 0, 313 1.2.2.3 tls SLJIT_MEM1(BJ_ARGS), 314 1.2.2.3 tls offsetof(struct bpf_args, buflen)); 315 1.2.2.3 tls 316 1.2.2.3 tls return status; 317 1.2.2.3 tls } 318 1.2.2.3 tls 319 1.2.2.3 tls static bool 320 1.2.2.3 tls grow_jumps(struct sljit_jump ***jumps, size_t *size) 321 1.2.2.3 tls { 322 1.2.2.3 tls struct sljit_jump **newptr; 323 1.2.2.3 tls const size_t elemsz = sizeof(struct sljit_jump *); 324 1.2.2.3 tls size_t old_size = *size; 325 1.2.2.3 tls size_t new_size = 2 * old_size; 326 1.2.2.3 tls 327 1.2.2.3 tls if (new_size < old_size || new_size > SIZE_MAX / elemsz) 328 1.2.2.3 tls return false; 329 1.2.2.3 tls 330 1.2.2.3 tls newptr = BJ_ALLOC(new_size * elemsz); 331 1.2.2.3 tls if (newptr == NULL) 332 1.2.2.3 tls return false; 333 1.2.2.3 tls 334 1.2.2.3 tls memcpy(newptr, *jumps, old_size * elemsz); 335 1.2.2.3 tls BJ_FREE(*jumps, old_size * elemsz); 336 1.2.2.3 tls 337 1.2.2.3 tls *jumps = newptr; 338 1.2.2.3 tls *size = new_size; 339 1.2.2.3 tls return true; 340 1.2.2.3 tls } 341 1.2.2.3 tls 342 1.2.2.3 tls static bool 343 1.2.2.3 tls append_jump(struct sljit_jump *jump, struct sljit_jump ***jumps, 344 1.2.2.3 tls size_t *size, size_t *max_size) 345 1.2.2.3 tls { 346 1.2.2.3 tls if (*size == *max_size && !grow_jumps(jumps, max_size)) 347 1.2.2.3 tls return false; 348 1.2.2.3 tls 349 1.2.2.3 tls (*jumps)[(*size)++] = jump; 350 1.2.2.3 tls return true; 351 1.2.2.2 tls } 352 1.2.2.2 tls 353 1.2.2.2 tls /* 354 1.2.2.3 tls * Emit code for BPF_LD+BPF_B+BPF_ABS A <- P[k:1]. 355 1.2.2.2 tls */ 356 1.2.2.2 tls static int 357 1.2.2.4 jdolecek emit_read8(struct sljit_compiler *compiler, sljit_s32 src, uint32_t k) 358 1.2.2.2 tls { 359 1.2.2.2 tls 360 1.2.2.2 tls return sljit_emit_op1(compiler, 361 1.2.2.4 jdolecek SLJIT_MOV_U8, 362 1.2.2.3 tls BJ_AREG, 0, 363 1.2.2.3 tls SLJIT_MEM1(src), k); 364 1.2.2.2 tls } 365 1.2.2.2 tls 366 1.2.2.2 tls /* 367 1.2.2.3 tls * Emit code for BPF_LD+BPF_H+BPF_ABS A <- P[k:2]. 368 1.2.2.2 tls */ 369 1.2.2.2 tls static int 370 1.2.2.4 jdolecek emit_read16(struct sljit_compiler *compiler, sljit_s32 src, uint32_t k) 371 1.2.2.2 tls { 372 1.2.2.2 tls int status; 373 1.2.2.2 tls 374 1.2.2.3 tls BJ_ASSERT(k <= UINT32_MAX - 1); 375 1.2.2.3 tls 376 1.2.2.3 tls /* A = buf[k]; */ 377 1.2.2.2 tls status = sljit_emit_op1(compiler, 378 1.2.2.4 jdolecek SLJIT_MOV_U8, 379 1.2.2.3 tls BJ_AREG, 0, 380 1.2.2.3 tls SLJIT_MEM1(src), k); 381 1.2.2.2 tls if (status != SLJIT_SUCCESS) 382 1.2.2.2 tls return status; 383 1.2.2.2 tls 384 1.2.2.3 tls /* tmp1 = buf[k+1]; */ 385 1.2.2.2 tls status = sljit_emit_op1(compiler, 386 1.2.2.4 jdolecek SLJIT_MOV_U8, 387 1.2.2.3 tls BJ_TMP1REG, 0, 388 1.2.2.3 tls SLJIT_MEM1(src), k+1); 389 1.2.2.2 tls if (status != SLJIT_SUCCESS) 390 1.2.2.2 tls return status; 391 1.2.2.2 tls 392 1.2.2.3 tls /* A = A << 8; */ 393 1.2.2.2 tls status = sljit_emit_op2(compiler, 394 1.2.2.2 tls SLJIT_SHL, 395 1.2.2.3 tls BJ_AREG, 0, 396 1.2.2.3 tls BJ_AREG, 0, 397 1.2.2.2 tls SLJIT_IMM, 8); 398 1.2.2.2 tls if (status != SLJIT_SUCCESS) 399 1.2.2.2 tls return status; 400 1.2.2.2 tls 401 1.2.2.2 tls /* A = A + tmp1; */ 402 1.2.2.2 tls status = sljit_emit_op2(compiler, 403 1.2.2.2 tls SLJIT_ADD, 404 1.2.2.3 tls BJ_AREG, 0, 405 1.2.2.3 tls BJ_AREG, 0, 406 1.2.2.3 tls BJ_TMP1REG, 0); 407 1.2.2.2 tls return status; 408 1.2.2.2 tls } 409 1.2.2.2 tls 410 1.2.2.2 tls /* 411 1.2.2.3 tls * Emit code for BPF_LD+BPF_W+BPF_ABS A <- P[k:4]. 412 1.2.2.2 tls */ 413 1.2.2.2 tls static int 414 1.2.2.4 jdolecek emit_read32(struct sljit_compiler *compiler, sljit_s32 src, uint32_t k) 415 1.2.2.2 tls { 416 1.2.2.2 tls int status; 417 1.2.2.2 tls 418 1.2.2.3 tls BJ_ASSERT(k <= UINT32_MAX - 3); 419 1.2.2.2 tls 420 1.2.2.3 tls /* A = buf[k]; */ 421 1.2.2.2 tls status = sljit_emit_op1(compiler, 422 1.2.2.4 jdolecek SLJIT_MOV_U8, 423 1.2.2.3 tls BJ_AREG, 0, 424 1.2.2.3 tls SLJIT_MEM1(src), k); 425 1.2.2.2 tls if (status != SLJIT_SUCCESS) 426 1.2.2.2 tls return status; 427 1.2.2.2 tls 428 1.2.2.3 tls /* tmp1 = buf[k+1]; */ 429 1.2.2.2 tls status = sljit_emit_op1(compiler, 430 1.2.2.4 jdolecek SLJIT_MOV_U8, 431 1.2.2.3 tls BJ_TMP1REG, 0, 432 1.2.2.3 tls SLJIT_MEM1(src), k+1); 433 1.2.2.2 tls if (status != SLJIT_SUCCESS) 434 1.2.2.2 tls return status; 435 1.2.2.2 tls 436 1.2.2.3 tls /* A = A << 8; */ 437 1.2.2.2 tls status = sljit_emit_op2(compiler, 438 1.2.2.2 tls SLJIT_SHL, 439 1.2.2.3 tls BJ_AREG, 0, 440 1.2.2.3 tls BJ_AREG, 0, 441 1.2.2.3 tls SLJIT_IMM, 8); 442 1.2.2.2 tls if (status != SLJIT_SUCCESS) 443 1.2.2.2 tls return status; 444 1.2.2.2 tls 445 1.2.2.2 tls /* A = A + tmp1; */ 446 1.2.2.2 tls status = sljit_emit_op2(compiler, 447 1.2.2.2 tls SLJIT_ADD, 448 1.2.2.3 tls BJ_AREG, 0, 449 1.2.2.3 tls BJ_AREG, 0, 450 1.2.2.3 tls BJ_TMP1REG, 0); 451 1.2.2.2 tls if (status != SLJIT_SUCCESS) 452 1.2.2.2 tls return status; 453 1.2.2.2 tls 454 1.2.2.2 tls /* tmp1 = buf[k+2]; */ 455 1.2.2.2 tls status = sljit_emit_op1(compiler, 456 1.2.2.4 jdolecek SLJIT_MOV_U8, 457 1.2.2.3 tls BJ_TMP1REG, 0, 458 1.2.2.3 tls SLJIT_MEM1(src), k+2); 459 1.2.2.2 tls if (status != SLJIT_SUCCESS) 460 1.2.2.2 tls return status; 461 1.2.2.2 tls 462 1.2.2.3 tls /* A = A << 8; */ 463 1.2.2.2 tls status = sljit_emit_op2(compiler, 464 1.2.2.2 tls SLJIT_SHL, 465 1.2.2.3 tls BJ_AREG, 0, 466 1.2.2.3 tls BJ_AREG, 0, 467 1.2.2.3 tls SLJIT_IMM, 8); 468 1.2.2.2 tls if (status != SLJIT_SUCCESS) 469 1.2.2.2 tls return status; 470 1.2.2.2 tls 471 1.2.2.3 tls /* A = A + tmp1; */ 472 1.2.2.2 tls status = sljit_emit_op2(compiler, 473 1.2.2.2 tls SLJIT_ADD, 474 1.2.2.3 tls BJ_AREG, 0, 475 1.2.2.3 tls BJ_AREG, 0, 476 1.2.2.3 tls BJ_TMP1REG, 0); 477 1.2.2.3 tls if (status != SLJIT_SUCCESS) 478 1.2.2.3 tls return status; 479 1.2.2.3 tls 480 1.2.2.3 tls /* tmp1 = buf[k+3]; */ 481 1.2.2.3 tls status = sljit_emit_op1(compiler, 482 1.2.2.4 jdolecek SLJIT_MOV_U8, 483 1.2.2.3 tls BJ_TMP1REG, 0, 484 1.2.2.3 tls SLJIT_MEM1(src), k+3); 485 1.2.2.2 tls if (status != SLJIT_SUCCESS) 486 1.2.2.2 tls return status; 487 1.2.2.2 tls 488 1.2.2.3 tls /* A = A << 8; */ 489 1.2.2.2 tls status = sljit_emit_op2(compiler, 490 1.2.2.2 tls SLJIT_SHL, 491 1.2.2.3 tls BJ_AREG, 0, 492 1.2.2.3 tls BJ_AREG, 0, 493 1.2.2.2 tls SLJIT_IMM, 8); 494 1.2.2.2 tls if (status != SLJIT_SUCCESS) 495 1.2.2.2 tls return status; 496 1.2.2.2 tls 497 1.2.2.2 tls /* A = A + tmp1; */ 498 1.2.2.2 tls status = sljit_emit_op2(compiler, 499 1.2.2.2 tls SLJIT_ADD, 500 1.2.2.3 tls BJ_AREG, 0, 501 1.2.2.3 tls BJ_AREG, 0, 502 1.2.2.3 tls BJ_TMP1REG, 0); 503 1.2.2.2 tls return status; 504 1.2.2.2 tls } 505 1.2.2.2 tls 506 1.2.2.2 tls #ifdef _KERNEL 507 1.2.2.2 tls /* 508 1.2.2.3 tls * Emit code for m_xword/m_xhalf/m_xbyte call. 509 1.2.2.2 tls * 510 1.2.2.3 tls * @pc BPF_LD+BPF_W+BPF_ABS A <- P[k:4] 511 1.2.2.3 tls * BPF_LD+BPF_H+BPF_ABS A <- P[k:2] 512 1.2.2.3 tls * BPF_LD+BPF_B+BPF_ABS A <- P[k:1] 513 1.2.2.3 tls * BPF_LD+BPF_W+BPF_IND A <- P[X+k:4] 514 1.2.2.3 tls * BPF_LD+BPF_H+BPF_IND A <- P[X+k:2] 515 1.2.2.3 tls * BPF_LD+BPF_B+BPF_IND A <- P[X+k:1] 516 1.2.2.3 tls * BPF_LDX+BPF_B+BPF_MSH X <- 4*(P[k:1]&0xf) 517 1.2.2.2 tls */ 518 1.2.2.2 tls static int 519 1.2.2.3 tls emit_xcall(struct sljit_compiler *compiler, bpfjit_hint_t hints, 520 1.2.2.3 tls const struct bpf_insn *pc, int dst, struct sljit_jump ***ret0, 521 1.2.2.3 tls size_t *ret0_size, size_t *ret0_maxsize, 522 1.2.2.2 tls uint32_t (*fn)(const struct mbuf *, uint32_t, int *)) 523 1.2.2.2 tls { 524 1.2.2.3 tls #if BJ_XREG == SLJIT_RETURN_REG || \ 525 1.2.2.4 jdolecek BJ_XREG == SLJIT_R0 || \ 526 1.2.2.4 jdolecek BJ_XREG == SLJIT_R1 || \ 527 1.2.2.4 jdolecek BJ_XREG == SLJIT_R2 528 1.2.2.2 tls #error "Not supported assignment of registers." 529 1.2.2.2 tls #endif 530 1.2.2.3 tls struct sljit_jump *jump; 531 1.2.2.4 jdolecek sljit_s32 save_reg; 532 1.2.2.2 tls int status; 533 1.2.2.2 tls 534 1.2.2.3 tls save_reg = (BPF_CLASS(pc->code) == BPF_LDX) ? BJ_AREG : BJ_XREG; 535 1.2.2.2 tls 536 1.2.2.3 tls if (save_reg == BJ_AREG || (hints & BJ_HINT_XREG)) { 537 1.2.2.3 tls /* save A or X */ 538 1.2.2.2 tls status = sljit_emit_op1(compiler, 539 1.2.2.4 jdolecek SLJIT_MOV_U32, 540 1.2.2.4 jdolecek SLJIT_MEM1(SLJIT_SP), 541 1.2.2.3 tls offsetof(struct bpfjit_stack, reg), 542 1.2.2.3 tls save_reg, 0); 543 1.2.2.2 tls if (status != SLJIT_SUCCESS) 544 1.2.2.2 tls return status; 545 1.2.2.2 tls } 546 1.2.2.2 tls 547 1.2.2.2 tls /* 548 1.2.2.3 tls * Prepare registers for fn(mbuf, k, &err) call. 549 1.2.2.2 tls */ 550 1.2.2.2 tls status = sljit_emit_op1(compiler, 551 1.2.2.2 tls SLJIT_MOV, 552 1.2.2.4 jdolecek SLJIT_R0, 0, 553 1.2.2.3 tls BJ_BUF, 0); 554 1.2.2.2 tls if (status != SLJIT_SUCCESS) 555 1.2.2.2 tls return status; 556 1.2.2.2 tls 557 1.2.2.2 tls if (BPF_CLASS(pc->code) == BPF_LD && BPF_MODE(pc->code) == BPF_IND) { 558 1.2.2.3 tls if (pc->k == 0) { 559 1.2.2.3 tls /* k = X; */ 560 1.2.2.3 tls status = sljit_emit_op1(compiler, 561 1.2.2.3 tls SLJIT_MOV, 562 1.2.2.4 jdolecek SLJIT_R1, 0, 563 1.2.2.3 tls BJ_XREG, 0); 564 1.2.2.3 tls if (status != SLJIT_SUCCESS) 565 1.2.2.3 tls return status; 566 1.2.2.3 tls } else { 567 1.2.2.3 tls /* if (X > UINT32_MAX - pc->k) return 0; */ 568 1.2.2.3 tls jump = sljit_emit_cmp(compiler, 569 1.2.2.4 jdolecek SLJIT_GREATER, 570 1.2.2.3 tls BJ_XREG, 0, 571 1.2.2.3 tls SLJIT_IMM, UINT32_MAX - pc->k); 572 1.2.2.3 tls if (jump == NULL) 573 1.2.2.3 tls return SLJIT_ERR_ALLOC_FAILED; 574 1.2.2.3 tls if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 575 1.2.2.3 tls return SLJIT_ERR_ALLOC_FAILED; 576 1.2.2.3 tls 577 1.2.2.3 tls /* k = X + pc->k; */ 578 1.2.2.3 tls status = sljit_emit_op2(compiler, 579 1.2.2.3 tls SLJIT_ADD, 580 1.2.2.4 jdolecek SLJIT_R1, 0, 581 1.2.2.3 tls BJ_XREG, 0, 582 1.2.2.3 tls SLJIT_IMM, (uint32_t)pc->k); 583 1.2.2.3 tls if (status != SLJIT_SUCCESS) 584 1.2.2.3 tls return status; 585 1.2.2.3 tls } 586 1.2.2.2 tls } else { 587 1.2.2.3 tls /* k = pc->k */ 588 1.2.2.2 tls status = sljit_emit_op1(compiler, 589 1.2.2.2 tls SLJIT_MOV, 590 1.2.2.4 jdolecek SLJIT_R1, 0, 591 1.2.2.2 tls SLJIT_IMM, (uint32_t)pc->k); 592 1.2.2.3 tls if (status != SLJIT_SUCCESS) 593 1.2.2.3 tls return status; 594 1.2.2.2 tls } 595 1.2.2.2 tls 596 1.2.2.3 tls /* 597 1.2.2.3 tls * The third argument of fn is an address on stack. 598 1.2.2.3 tls */ 599 1.2.2.2 tls status = sljit_get_local_base(compiler, 600 1.2.2.4 jdolecek SLJIT_R2, 0, 601 1.2.2.3 tls offsetof(struct bpfjit_stack, err)); 602 1.2.2.2 tls if (status != SLJIT_SUCCESS) 603 1.2.2.2 tls return status; 604 1.2.2.2 tls 605 1.2.2.2 tls /* fn(buf, k, &err); */ 606 1.2.2.2 tls status = sljit_emit_ijump(compiler, 607 1.2.2.2 tls SLJIT_CALL3, 608 1.2.2.2 tls SLJIT_IMM, SLJIT_FUNC_OFFSET(fn)); 609 1.2.2.3 tls if (status != SLJIT_SUCCESS) 610 1.2.2.3 tls return status; 611 1.2.2.2 tls 612 1.2.2.3 tls if (dst != SLJIT_RETURN_REG) { 613 1.2.2.2 tls /* move return value to dst */ 614 1.2.2.2 tls status = sljit_emit_op1(compiler, 615 1.2.2.2 tls SLJIT_MOV, 616 1.2.2.3 tls dst, 0, 617 1.2.2.2 tls SLJIT_RETURN_REG, 0); 618 1.2.2.2 tls if (status != SLJIT_SUCCESS) 619 1.2.2.2 tls return status; 620 1.2.2.3 tls } 621 1.2.2.3 tls 622 1.2.2.3 tls /* if (*err != 0) return 0; */ 623 1.2.2.3 tls jump = sljit_emit_cmp(compiler, 624 1.2.2.4 jdolecek SLJIT_NOT_EQUAL|SLJIT_I32_OP, 625 1.2.2.4 jdolecek SLJIT_MEM1(SLJIT_SP), 626 1.2.2.3 tls offsetof(struct bpfjit_stack, err), 627 1.2.2.3 tls SLJIT_IMM, 0); 628 1.2.2.3 tls if (jump == NULL) 629 1.2.2.3 tls return SLJIT_ERR_ALLOC_FAILED; 630 1.2.2.3 tls 631 1.2.2.3 tls if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 632 1.2.2.3 tls return SLJIT_ERR_ALLOC_FAILED; 633 1.2.2.2 tls 634 1.2.2.3 tls if (save_reg == BJ_AREG || (hints & BJ_HINT_XREG)) { 635 1.2.2.3 tls /* restore A or X */ 636 1.2.2.2 tls status = sljit_emit_op1(compiler, 637 1.2.2.4 jdolecek SLJIT_MOV_U32, 638 1.2.2.3 tls save_reg, 0, 639 1.2.2.4 jdolecek SLJIT_MEM1(SLJIT_SP), 640 1.2.2.3 tls offsetof(struct bpfjit_stack, reg)); 641 1.2.2.3 tls if (status != SLJIT_SUCCESS) 642 1.2.2.3 tls return status; 643 1.2.2.3 tls } 644 1.2.2.3 tls 645 1.2.2.3 tls return SLJIT_SUCCESS; 646 1.2.2.3 tls } 647 1.2.2.3 tls #endif 648 1.2.2.3 tls 649 1.2.2.3 tls /* 650 1.2.2.3 tls * Emit code for BPF_COP and BPF_COPX instructions. 651 1.2.2.3 tls */ 652 1.2.2.3 tls static int 653 1.2.2.3 tls emit_cop(struct sljit_compiler *compiler, bpfjit_hint_t hints, 654 1.2.2.3 tls const bpf_ctx_t *bc, const struct bpf_insn *pc, 655 1.2.2.3 tls struct sljit_jump ***ret0, size_t *ret0_size, size_t *ret0_maxsize) 656 1.2.2.3 tls { 657 1.2.2.3 tls #if BJ_XREG == SLJIT_RETURN_REG || \ 658 1.2.2.4 jdolecek BJ_XREG == SLJIT_R0 || \ 659 1.2.2.4 jdolecek BJ_XREG == SLJIT_R1 || \ 660 1.2.2.4 jdolecek BJ_XREG == SLJIT_R2 || \ 661 1.2.2.4 jdolecek BJ_TMP3REG == SLJIT_R0 || \ 662 1.2.2.4 jdolecek BJ_TMP3REG == SLJIT_R1 || \ 663 1.2.2.4 jdolecek BJ_TMP3REG == SLJIT_R2 664 1.2.2.3 tls #error "Not supported assignment of registers." 665 1.2.2.3 tls #endif 666 1.2.2.3 tls 667 1.2.2.3 tls struct sljit_jump *jump; 668 1.2.2.4 jdolecek sljit_s32 call_reg; 669 1.2.2.3 tls sljit_sw call_off; 670 1.2.2.3 tls int status; 671 1.2.2.3 tls 672 1.2.2.3 tls BJ_ASSERT(bc != NULL && bc->copfuncs != NULL); 673 1.2.2.3 tls 674 1.2.2.3 tls if (hints & BJ_HINT_LDX) { 675 1.2.2.3 tls /* save X */ 676 1.2.2.3 tls status = sljit_emit_op1(compiler, 677 1.2.2.4 jdolecek SLJIT_MOV_U32, 678 1.2.2.4 jdolecek SLJIT_MEM1(SLJIT_SP), 679 1.2.2.3 tls offsetof(struct bpfjit_stack, reg), 680 1.2.2.3 tls BJ_XREG, 0); 681 1.2.2.3 tls if (status != SLJIT_SUCCESS) 682 1.2.2.3 tls return status; 683 1.2.2.3 tls } 684 1.2.2.3 tls 685 1.2.2.3 tls if (BPF_MISCOP(pc->code) == BPF_COP) { 686 1.2.2.3 tls call_reg = SLJIT_IMM; 687 1.2.2.3 tls call_off = SLJIT_FUNC_OFFSET(bc->copfuncs[pc->k]); 688 1.2.2.3 tls } else { 689 1.2.2.3 tls /* if (X >= bc->nfuncs) return 0; */ 690 1.2.2.3 tls jump = sljit_emit_cmp(compiler, 691 1.2.2.4 jdolecek SLJIT_GREATER_EQUAL, 692 1.2.2.3 tls BJ_XREG, 0, 693 1.2.2.3 tls SLJIT_IMM, bc->nfuncs); 694 1.2.2.3 tls if (jump == NULL) 695 1.2.2.3 tls return SLJIT_ERR_ALLOC_FAILED; 696 1.2.2.3 tls if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 697 1.2.2.3 tls return SLJIT_ERR_ALLOC_FAILED; 698 1.2.2.3 tls 699 1.2.2.3 tls /* tmp1 = ctx; */ 700 1.2.2.3 tls status = sljit_emit_op1(compiler, 701 1.2.2.3 tls SLJIT_MOV_P, 702 1.2.2.3 tls BJ_TMP1REG, 0, 703 1.2.2.4 jdolecek SLJIT_MEM1(SLJIT_SP), 704 1.2.2.3 tls offsetof(struct bpfjit_stack, ctx)); 705 1.2.2.3 tls if (status != SLJIT_SUCCESS) 706 1.2.2.3 tls return status; 707 1.2.2.3 tls 708 1.2.2.3 tls /* tmp1 = ctx->copfuncs; */ 709 1.2.2.3 tls status = sljit_emit_op1(compiler, 710 1.2.2.3 tls SLJIT_MOV_P, 711 1.2.2.3 tls BJ_TMP1REG, 0, 712 1.2.2.3 tls SLJIT_MEM1(BJ_TMP1REG), 713 1.2.2.3 tls offsetof(struct bpf_ctx, copfuncs)); 714 1.2.2.2 tls if (status != SLJIT_SUCCESS) 715 1.2.2.2 tls return status; 716 1.2.2.2 tls 717 1.2.2.3 tls /* tmp2 = X; */ 718 1.2.2.2 tls status = sljit_emit_op1(compiler, 719 1.2.2.2 tls SLJIT_MOV, 720 1.2.2.3 tls BJ_TMP2REG, 0, 721 1.2.2.3 tls BJ_XREG, 0); 722 1.2.2.3 tls if (status != SLJIT_SUCCESS) 723 1.2.2.3 tls return status; 724 1.2.2.3 tls 725 1.2.2.3 tls /* tmp3 = ctx->copfuncs[tmp2]; */ 726 1.2.2.3 tls call_reg = BJ_TMP3REG; 727 1.2.2.3 tls call_off = 0; 728 1.2.2.3 tls status = sljit_emit_op1(compiler, 729 1.2.2.3 tls SLJIT_MOV_P, 730 1.2.2.3 tls call_reg, call_off, 731 1.2.2.3 tls SLJIT_MEM2(BJ_TMP1REG, BJ_TMP2REG), 732 1.2.2.3 tls SLJIT_WORD_SHIFT); 733 1.2.2.2 tls if (status != SLJIT_SUCCESS) 734 1.2.2.2 tls return status; 735 1.2.2.2 tls } 736 1.2.2.2 tls 737 1.2.2.3 tls /* 738 1.2.2.3 tls * Copy bpf_copfunc_t arguments to registers. 739 1.2.2.3 tls */ 740 1.2.2.4 jdolecek #if BJ_AREG != SLJIT_R2 741 1.2.2.2 tls status = sljit_emit_op1(compiler, 742 1.2.2.4 jdolecek SLJIT_MOV_U32, 743 1.2.2.4 jdolecek SLJIT_R2, 0, 744 1.2.2.3 tls BJ_AREG, 0); 745 1.2.2.2 tls if (status != SLJIT_SUCCESS) 746 1.2.2.2 tls return status; 747 1.2.2.3 tls #endif 748 1.2.2.2 tls 749 1.2.2.3 tls status = sljit_emit_op1(compiler, 750 1.2.2.3 tls SLJIT_MOV_P, 751 1.2.2.4 jdolecek SLJIT_R0, 0, 752 1.2.2.4 jdolecek SLJIT_MEM1(SLJIT_SP), 753 1.2.2.3 tls offsetof(struct bpfjit_stack, ctx)); 754 1.2.2.3 tls if (status != SLJIT_SUCCESS) 755 1.2.2.3 tls return status; 756 1.2.2.2 tls 757 1.2.2.3 tls status = sljit_emit_op1(compiler, 758 1.2.2.3 tls SLJIT_MOV_P, 759 1.2.2.4 jdolecek SLJIT_R1, 0, 760 1.2.2.3 tls BJ_ARGS, 0); 761 1.2.2.3 tls if (status != SLJIT_SUCCESS) 762 1.2.2.3 tls return status; 763 1.2.2.3 tls 764 1.2.2.3 tls status = sljit_emit_ijump(compiler, 765 1.2.2.3 tls SLJIT_CALL3, call_reg, call_off); 766 1.2.2.3 tls if (status != SLJIT_SUCCESS) 767 1.2.2.3 tls return status; 768 1.2.2.3 tls 769 1.2.2.3 tls #if BJ_AREG != SLJIT_RETURN_REG 770 1.2.2.3 tls status = sljit_emit_op1(compiler, 771 1.2.2.3 tls SLJIT_MOV, 772 1.2.2.3 tls BJ_AREG, 0, 773 1.2.2.3 tls SLJIT_RETURN_REG, 0); 774 1.2.2.3 tls if (status != SLJIT_SUCCESS) 775 1.2.2.3 tls return status; 776 1.2.2.2 tls #endif 777 1.2.2.2 tls 778 1.2.2.3 tls if (hints & BJ_HINT_LDX) { 779 1.2.2.3 tls /* restore X */ 780 1.2.2.3 tls status = sljit_emit_op1(compiler, 781 1.2.2.4 jdolecek SLJIT_MOV_U32, 782 1.2.2.3 tls BJ_XREG, 0, 783 1.2.2.4 jdolecek SLJIT_MEM1(SLJIT_SP), 784 1.2.2.3 tls offsetof(struct bpfjit_stack, reg)); 785 1.2.2.3 tls if (status != SLJIT_SUCCESS) 786 1.2.2.3 tls return status; 787 1.2.2.3 tls } 788 1.2.2.3 tls 789 1.2.2.3 tls return SLJIT_SUCCESS; 790 1.2.2.3 tls } 791 1.2.2.3 tls 792 1.2.2.2 tls /* 793 1.2.2.2 tls * Generate code for 794 1.2.2.2 tls * BPF_LD+BPF_W+BPF_ABS A <- P[k:4] 795 1.2.2.2 tls * BPF_LD+BPF_H+BPF_ABS A <- P[k:2] 796 1.2.2.2 tls * BPF_LD+BPF_B+BPF_ABS A <- P[k:1] 797 1.2.2.2 tls * BPF_LD+BPF_W+BPF_IND A <- P[X+k:4] 798 1.2.2.2 tls * BPF_LD+BPF_H+BPF_IND A <- P[X+k:2] 799 1.2.2.2 tls * BPF_LD+BPF_B+BPF_IND A <- P[X+k:1] 800 1.2.2.2 tls */ 801 1.2.2.2 tls static int 802 1.2.2.3 tls emit_pkt_read(struct sljit_compiler *compiler, bpfjit_hint_t hints, 803 1.2.2.3 tls const struct bpf_insn *pc, struct sljit_jump *to_mchain_jump, 804 1.2.2.3 tls struct sljit_jump ***ret0, size_t *ret0_size, size_t *ret0_maxsize) 805 1.2.2.2 tls { 806 1.2.2.3 tls int status = SLJIT_ERR_ALLOC_FAILED; 807 1.2.2.2 tls uint32_t width; 808 1.2.2.4 jdolecek sljit_s32 ld_reg; 809 1.2.2.2 tls struct sljit_jump *jump; 810 1.2.2.2 tls #ifdef _KERNEL 811 1.2.2.2 tls struct sljit_label *label; 812 1.2.2.2 tls struct sljit_jump *over_mchain_jump; 813 1.2.2.2 tls const bool check_zero_buflen = (to_mchain_jump != NULL); 814 1.2.2.2 tls #endif 815 1.2.2.2 tls const uint32_t k = pc->k; 816 1.2.2.2 tls 817 1.2.2.2 tls #ifdef _KERNEL 818 1.2.2.2 tls if (to_mchain_jump == NULL) { 819 1.2.2.2 tls to_mchain_jump = sljit_emit_cmp(compiler, 820 1.2.2.4 jdolecek SLJIT_EQUAL, 821 1.2.2.3 tls BJ_BUFLEN, 0, 822 1.2.2.2 tls SLJIT_IMM, 0); 823 1.2.2.2 tls if (to_mchain_jump == NULL) 824 1.2.2.3 tls return SLJIT_ERR_ALLOC_FAILED; 825 1.2.2.2 tls } 826 1.2.2.2 tls #endif 827 1.2.2.2 tls 828 1.2.2.3 tls ld_reg = BJ_BUF; 829 1.2.2.2 tls width = read_width(pc); 830 1.2.2.4 jdolecek if (width == 0) 831 1.2.2.4 jdolecek return SLJIT_ERR_ALLOC_FAILED; 832 1.2.2.2 tls 833 1.2.2.2 tls if (BPF_MODE(pc->code) == BPF_IND) { 834 1.2.2.2 tls /* tmp1 = buflen - (pc->k + width); */ 835 1.2.2.2 tls status = sljit_emit_op2(compiler, 836 1.2.2.2 tls SLJIT_SUB, 837 1.2.2.3 tls BJ_TMP1REG, 0, 838 1.2.2.3 tls BJ_BUFLEN, 0, 839 1.2.2.2 tls SLJIT_IMM, k + width); 840 1.2.2.2 tls if (status != SLJIT_SUCCESS) 841 1.2.2.2 tls return status; 842 1.2.2.2 tls 843 1.2.2.3 tls /* ld_reg = buf + X; */ 844 1.2.2.3 tls ld_reg = BJ_TMP2REG; 845 1.2.2.2 tls status = sljit_emit_op2(compiler, 846 1.2.2.2 tls SLJIT_ADD, 847 1.2.2.3 tls ld_reg, 0, 848 1.2.2.3 tls BJ_BUF, 0, 849 1.2.2.3 tls BJ_XREG, 0); 850 1.2.2.2 tls if (status != SLJIT_SUCCESS) 851 1.2.2.2 tls return status; 852 1.2.2.2 tls 853 1.2.2.2 tls /* if (tmp1 < X) return 0; */ 854 1.2.2.2 tls jump = sljit_emit_cmp(compiler, 855 1.2.2.4 jdolecek SLJIT_LESS, 856 1.2.2.3 tls BJ_TMP1REG, 0, 857 1.2.2.3 tls BJ_XREG, 0); 858 1.2.2.2 tls if (jump == NULL) 859 1.2.2.3 tls return SLJIT_ERR_ALLOC_FAILED; 860 1.2.2.3 tls if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 861 1.2.2.3 tls return SLJIT_ERR_ALLOC_FAILED; 862 1.2.2.2 tls } 863 1.2.2.2 tls 864 1.2.2.4 jdolecek /* 865 1.2.2.4 jdolecek * Don't emit wrapped-around reads. They're dead code but 866 1.2.2.4 jdolecek * dead code elimination logic isn't smart enough to figure 867 1.2.2.4 jdolecek * it out. 868 1.2.2.4 jdolecek */ 869 1.2.2.4 jdolecek if (k <= UINT32_MAX - width + 1) { 870 1.2.2.4 jdolecek switch (width) { 871 1.2.2.4 jdolecek case 4: 872 1.2.2.4 jdolecek status = emit_read32(compiler, ld_reg, k); 873 1.2.2.4 jdolecek break; 874 1.2.2.4 jdolecek case 2: 875 1.2.2.4 jdolecek status = emit_read16(compiler, ld_reg, k); 876 1.2.2.4 jdolecek break; 877 1.2.2.4 jdolecek case 1: 878 1.2.2.4 jdolecek status = emit_read8(compiler, ld_reg, k); 879 1.2.2.4 jdolecek break; 880 1.2.2.4 jdolecek } 881 1.2.2.2 tls 882 1.2.2.4 jdolecek if (status != SLJIT_SUCCESS) 883 1.2.2.4 jdolecek return status; 884 1.2.2.4 jdolecek } 885 1.2.2.2 tls 886 1.2.2.2 tls #ifdef _KERNEL 887 1.2.2.2 tls over_mchain_jump = sljit_emit_jump(compiler, SLJIT_JUMP); 888 1.2.2.2 tls if (over_mchain_jump == NULL) 889 1.2.2.3 tls return SLJIT_ERR_ALLOC_FAILED; 890 1.2.2.2 tls 891 1.2.2.2 tls /* entry point to mchain handler */ 892 1.2.2.2 tls label = sljit_emit_label(compiler); 893 1.2.2.2 tls if (label == NULL) 894 1.2.2.3 tls return SLJIT_ERR_ALLOC_FAILED; 895 1.2.2.2 tls sljit_set_label(to_mchain_jump, label); 896 1.2.2.2 tls 897 1.2.2.2 tls if (check_zero_buflen) { 898 1.2.2.2 tls /* if (buflen != 0) return 0; */ 899 1.2.2.2 tls jump = sljit_emit_cmp(compiler, 900 1.2.2.4 jdolecek SLJIT_NOT_EQUAL, 901 1.2.2.3 tls BJ_BUFLEN, 0, 902 1.2.2.2 tls SLJIT_IMM, 0); 903 1.2.2.2 tls if (jump == NULL) 904 1.2.2.2 tls return SLJIT_ERR_ALLOC_FAILED; 905 1.2.2.3 tls if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 906 1.2.2.3 tls return SLJIT_ERR_ALLOC_FAILED; 907 1.2.2.2 tls } 908 1.2.2.2 tls 909 1.2.2.2 tls switch (width) { 910 1.2.2.2 tls case 4: 911 1.2.2.3 tls status = emit_xcall(compiler, hints, pc, BJ_AREG, 912 1.2.2.3 tls ret0, ret0_size, ret0_maxsize, &m_xword); 913 1.2.2.2 tls break; 914 1.2.2.2 tls case 2: 915 1.2.2.3 tls status = emit_xcall(compiler, hints, pc, BJ_AREG, 916 1.2.2.3 tls ret0, ret0_size, ret0_maxsize, &m_xhalf); 917 1.2.2.2 tls break; 918 1.2.2.2 tls case 1: 919 1.2.2.3 tls status = emit_xcall(compiler, hints, pc, BJ_AREG, 920 1.2.2.3 tls ret0, ret0_size, ret0_maxsize, &m_xbyte); 921 1.2.2.2 tls break; 922 1.2.2.2 tls } 923 1.2.2.2 tls 924 1.2.2.2 tls if (status != SLJIT_SUCCESS) 925 1.2.2.2 tls return status; 926 1.2.2.2 tls 927 1.2.2.2 tls label = sljit_emit_label(compiler); 928 1.2.2.2 tls if (label == NULL) 929 1.2.2.2 tls return SLJIT_ERR_ALLOC_FAILED; 930 1.2.2.2 tls sljit_set_label(over_mchain_jump, label); 931 1.2.2.2 tls #endif 932 1.2.2.2 tls 933 1.2.2.3 tls return SLJIT_SUCCESS; 934 1.2.2.3 tls } 935 1.2.2.3 tls 936 1.2.2.3 tls static int 937 1.2.2.3 tls emit_memload(struct sljit_compiler *compiler, 938 1.2.2.4 jdolecek sljit_s32 dst, uint32_t k, size_t extwords) 939 1.2.2.3 tls { 940 1.2.2.3 tls int status; 941 1.2.2.4 jdolecek sljit_s32 src; 942 1.2.2.3 tls sljit_sw srcw; 943 1.2.2.3 tls 944 1.2.2.3 tls srcw = k * sizeof(uint32_t); 945 1.2.2.3 tls 946 1.2.2.3 tls if (extwords == 0) { 947 1.2.2.4 jdolecek src = SLJIT_MEM1(SLJIT_SP); 948 1.2.2.3 tls srcw += offsetof(struct bpfjit_stack, mem); 949 1.2.2.3 tls } else { 950 1.2.2.3 tls /* copy extmem pointer to the tmp1 register */ 951 1.2.2.3 tls status = sljit_emit_op1(compiler, 952 1.2.2.3 tls SLJIT_MOV_P, 953 1.2.2.3 tls BJ_TMP1REG, 0, 954 1.2.2.4 jdolecek SLJIT_MEM1(SLJIT_SP), 955 1.2.2.3 tls offsetof(struct bpfjit_stack, extmem)); 956 1.2.2.3 tls if (status != SLJIT_SUCCESS) 957 1.2.2.3 tls return status; 958 1.2.2.3 tls src = SLJIT_MEM1(BJ_TMP1REG); 959 1.2.2.3 tls } 960 1.2.2.3 tls 961 1.2.2.4 jdolecek return sljit_emit_op1(compiler, SLJIT_MOV_U32, dst, 0, src, srcw); 962 1.2.2.3 tls } 963 1.2.2.3 tls 964 1.2.2.3 tls static int 965 1.2.2.3 tls emit_memstore(struct sljit_compiler *compiler, 966 1.2.2.4 jdolecek sljit_s32 src, uint32_t k, size_t extwords) 967 1.2.2.3 tls { 968 1.2.2.3 tls int status; 969 1.2.2.4 jdolecek sljit_s32 dst; 970 1.2.2.3 tls sljit_sw dstw; 971 1.2.2.3 tls 972 1.2.2.3 tls dstw = k * sizeof(uint32_t); 973 1.2.2.3 tls 974 1.2.2.3 tls if (extwords == 0) { 975 1.2.2.4 jdolecek dst = SLJIT_MEM1(SLJIT_SP); 976 1.2.2.3 tls dstw += offsetof(struct bpfjit_stack, mem); 977 1.2.2.3 tls } else { 978 1.2.2.3 tls /* copy extmem pointer to the tmp1 register */ 979 1.2.2.3 tls status = sljit_emit_op1(compiler, 980 1.2.2.3 tls SLJIT_MOV_P, 981 1.2.2.3 tls BJ_TMP1REG, 0, 982 1.2.2.4 jdolecek SLJIT_MEM1(SLJIT_SP), 983 1.2.2.3 tls offsetof(struct bpfjit_stack, extmem)); 984 1.2.2.3 tls if (status != SLJIT_SUCCESS) 985 1.2.2.3 tls return status; 986 1.2.2.3 tls dst = SLJIT_MEM1(BJ_TMP1REG); 987 1.2.2.3 tls } 988 1.2.2.3 tls 989 1.2.2.4 jdolecek return sljit_emit_op1(compiler, SLJIT_MOV_U32, dst, dstw, src, 0); 990 1.2.2.2 tls } 991 1.2.2.2 tls 992 1.2.2.2 tls /* 993 1.2.2.3 tls * Emit code for BPF_LDX+BPF_B+BPF_MSH X <- 4*(P[k:1]&0xf). 994 1.2.2.2 tls */ 995 1.2.2.2 tls static int 996 1.2.2.3 tls emit_msh(struct sljit_compiler *compiler, bpfjit_hint_t hints, 997 1.2.2.3 tls const struct bpf_insn *pc, struct sljit_jump *to_mchain_jump, 998 1.2.2.3 tls struct sljit_jump ***ret0, size_t *ret0_size, size_t *ret0_maxsize) 999 1.2.2.2 tls { 1000 1.2.2.2 tls int status; 1001 1.2.2.2 tls #ifdef _KERNEL 1002 1.2.2.2 tls struct sljit_label *label; 1003 1.2.2.2 tls struct sljit_jump *jump, *over_mchain_jump; 1004 1.2.2.2 tls const bool check_zero_buflen = (to_mchain_jump != NULL); 1005 1.2.2.2 tls #endif 1006 1.2.2.2 tls const uint32_t k = pc->k; 1007 1.2.2.2 tls 1008 1.2.2.2 tls #ifdef _KERNEL 1009 1.2.2.2 tls if (to_mchain_jump == NULL) { 1010 1.2.2.2 tls to_mchain_jump = sljit_emit_cmp(compiler, 1011 1.2.2.4 jdolecek SLJIT_EQUAL, 1012 1.2.2.3 tls BJ_BUFLEN, 0, 1013 1.2.2.2 tls SLJIT_IMM, 0); 1014 1.2.2.2 tls if (to_mchain_jump == NULL) 1015 1.2.2.3 tls return SLJIT_ERR_ALLOC_FAILED; 1016 1.2.2.2 tls } 1017 1.2.2.2 tls #endif 1018 1.2.2.2 tls 1019 1.2.2.2 tls /* tmp1 = buf[k] */ 1020 1.2.2.2 tls status = sljit_emit_op1(compiler, 1021 1.2.2.4 jdolecek SLJIT_MOV_U8, 1022 1.2.2.3 tls BJ_TMP1REG, 0, 1023 1.2.2.3 tls SLJIT_MEM1(BJ_BUF), k); 1024 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1025 1.2.2.2 tls return status; 1026 1.2.2.2 tls 1027 1.2.2.2 tls #ifdef _KERNEL 1028 1.2.2.2 tls over_mchain_jump = sljit_emit_jump(compiler, SLJIT_JUMP); 1029 1.2.2.2 tls if (over_mchain_jump == NULL) 1030 1.2.2.2 tls return SLJIT_ERR_ALLOC_FAILED; 1031 1.2.2.2 tls 1032 1.2.2.2 tls /* entry point to mchain handler */ 1033 1.2.2.2 tls label = sljit_emit_label(compiler); 1034 1.2.2.2 tls if (label == NULL) 1035 1.2.2.2 tls return SLJIT_ERR_ALLOC_FAILED; 1036 1.2.2.2 tls sljit_set_label(to_mchain_jump, label); 1037 1.2.2.2 tls 1038 1.2.2.2 tls if (check_zero_buflen) { 1039 1.2.2.2 tls /* if (buflen != 0) return 0; */ 1040 1.2.2.2 tls jump = sljit_emit_cmp(compiler, 1041 1.2.2.4 jdolecek SLJIT_NOT_EQUAL, 1042 1.2.2.3 tls BJ_BUFLEN, 0, 1043 1.2.2.2 tls SLJIT_IMM, 0); 1044 1.2.2.2 tls if (jump == NULL) 1045 1.2.2.3 tls return SLJIT_ERR_ALLOC_FAILED; 1046 1.2.2.3 tls if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 1047 1.2.2.3 tls return SLJIT_ERR_ALLOC_FAILED; 1048 1.2.2.2 tls } 1049 1.2.2.2 tls 1050 1.2.2.3 tls status = emit_xcall(compiler, hints, pc, BJ_TMP1REG, 1051 1.2.2.3 tls ret0, ret0_size, ret0_maxsize, &m_xbyte); 1052 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1053 1.2.2.2 tls return status; 1054 1.2.2.3 tls 1055 1.2.2.3 tls label = sljit_emit_label(compiler); 1056 1.2.2.3 tls if (label == NULL) 1057 1.2.2.3 tls return SLJIT_ERR_ALLOC_FAILED; 1058 1.2.2.3 tls sljit_set_label(over_mchain_jump, label); 1059 1.2.2.3 tls #endif 1060 1.2.2.2 tls 1061 1.2.2.2 tls /* tmp1 &= 0xf */ 1062 1.2.2.2 tls status = sljit_emit_op2(compiler, 1063 1.2.2.2 tls SLJIT_AND, 1064 1.2.2.3 tls BJ_TMP1REG, 0, 1065 1.2.2.3 tls BJ_TMP1REG, 0, 1066 1.2.2.2 tls SLJIT_IMM, 0xf); 1067 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1068 1.2.2.2 tls return status; 1069 1.2.2.2 tls 1070 1.2.2.3 tls /* X = tmp1 << 2 */ 1071 1.2.2.2 tls status = sljit_emit_op2(compiler, 1072 1.2.2.2 tls SLJIT_SHL, 1073 1.2.2.3 tls BJ_XREG, 0, 1074 1.2.2.3 tls BJ_TMP1REG, 0, 1075 1.2.2.2 tls SLJIT_IMM, 2); 1076 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1077 1.2.2.2 tls return status; 1078 1.2.2.2 tls 1079 1.2.2.3 tls return SLJIT_SUCCESS; 1080 1.2.2.2 tls } 1081 1.2.2.2 tls 1082 1.2.2.4 jdolecek /* 1083 1.2.2.4 jdolecek * Emit code for A = A / k or A = A % k when k is a power of 2. 1084 1.2.2.4 jdolecek * @pc BPF_DIV or BPF_MOD instruction. 1085 1.2.2.4 jdolecek */ 1086 1.2.2.2 tls static int 1087 1.2.2.4 jdolecek emit_pow2_moddiv(struct sljit_compiler *compiler, const struct bpf_insn *pc) 1088 1.2.2.2 tls { 1089 1.2.2.4 jdolecek uint32_t k = pc->k; 1090 1.2.2.2 tls int status = SLJIT_SUCCESS; 1091 1.2.2.2 tls 1092 1.2.2.4 jdolecek BJ_ASSERT(k != 0 && (k & (k - 1)) == 0); 1093 1.2.2.2 tls 1094 1.2.2.4 jdolecek if (BPF_OP(pc->code) == BPF_MOD) { 1095 1.2.2.2 tls status = sljit_emit_op2(compiler, 1096 1.2.2.4 jdolecek SLJIT_AND, 1097 1.2.2.3 tls BJ_AREG, 0, 1098 1.2.2.3 tls BJ_AREG, 0, 1099 1.2.2.4 jdolecek SLJIT_IMM, k - 1); 1100 1.2.2.4 jdolecek } else { 1101 1.2.2.4 jdolecek int shift = 0; 1102 1.2.2.4 jdolecek 1103 1.2.2.4 jdolecek /* 1104 1.2.2.4 jdolecek * Do shift = __builtin_ctz(k). 1105 1.2.2.4 jdolecek * The loop is slower, but that's ok. 1106 1.2.2.4 jdolecek */ 1107 1.2.2.4 jdolecek while (k > 1) { 1108 1.2.2.4 jdolecek k >>= 1; 1109 1.2.2.4 jdolecek shift++; 1110 1.2.2.4 jdolecek } 1111 1.2.2.4 jdolecek 1112 1.2.2.4 jdolecek if (shift != 0) { 1113 1.2.2.4 jdolecek status = sljit_emit_op2(compiler, 1114 1.2.2.4 jdolecek SLJIT_LSHR|SLJIT_I32_OP, 1115 1.2.2.4 jdolecek BJ_AREG, 0, 1116 1.2.2.4 jdolecek BJ_AREG, 0, 1117 1.2.2.4 jdolecek SLJIT_IMM, shift); 1118 1.2.2.4 jdolecek } 1119 1.2.2.2 tls } 1120 1.2.2.2 tls 1121 1.2.2.2 tls return status; 1122 1.2.2.2 tls } 1123 1.2.2.2 tls 1124 1.2.2.2 tls #if !defined(BPFJIT_USE_UDIV) 1125 1.2.2.2 tls static sljit_uw 1126 1.2.2.2 tls divide(sljit_uw x, sljit_uw y) 1127 1.2.2.2 tls { 1128 1.2.2.2 tls 1129 1.2.2.2 tls return (uint32_t)x / (uint32_t)y; 1130 1.2.2.2 tls } 1131 1.2.2.4 jdolecek 1132 1.2.2.4 jdolecek static sljit_uw 1133 1.2.2.4 jdolecek modulus(sljit_uw x, sljit_uw y) 1134 1.2.2.4 jdolecek { 1135 1.2.2.4 jdolecek 1136 1.2.2.4 jdolecek return (uint32_t)x % (uint32_t)y; 1137 1.2.2.4 jdolecek } 1138 1.2.2.2 tls #endif 1139 1.2.2.2 tls 1140 1.2.2.2 tls /* 1141 1.2.2.4 jdolecek * Emit code for A = A / div or A = A % div. 1142 1.2.2.4 jdolecek * @pc BPF_DIV or BPF_MOD instruction. 1143 1.2.2.2 tls */ 1144 1.2.2.2 tls static int 1145 1.2.2.4 jdolecek emit_moddiv(struct sljit_compiler *compiler, const struct bpf_insn *pc) 1146 1.2.2.2 tls { 1147 1.2.2.2 tls int status; 1148 1.2.2.4 jdolecek const bool xdiv = BPF_OP(pc->code) == BPF_DIV; 1149 1.2.2.4 jdolecek const bool xreg = BPF_SRC(pc->code) == BPF_X; 1150 1.2.2.2 tls 1151 1.2.2.3 tls #if BJ_XREG == SLJIT_RETURN_REG || \ 1152 1.2.2.4 jdolecek BJ_XREG == SLJIT_R0 || \ 1153 1.2.2.4 jdolecek BJ_XREG == SLJIT_R1 || \ 1154 1.2.2.4 jdolecek BJ_AREG == SLJIT_R1 1155 1.2.2.2 tls #error "Not supported assignment of registers." 1156 1.2.2.2 tls #endif 1157 1.2.2.2 tls 1158 1.2.2.4 jdolecek #if BJ_AREG != SLJIT_R0 1159 1.2.2.2 tls status = sljit_emit_op1(compiler, 1160 1.2.2.2 tls SLJIT_MOV, 1161 1.2.2.4 jdolecek SLJIT_R0, 0, 1162 1.2.2.3 tls BJ_AREG, 0); 1163 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1164 1.2.2.2 tls return status; 1165 1.2.2.2 tls #endif 1166 1.2.2.2 tls 1167 1.2.2.2 tls status = sljit_emit_op1(compiler, 1168 1.2.2.2 tls SLJIT_MOV, 1169 1.2.2.4 jdolecek SLJIT_R1, 0, 1170 1.2.2.4 jdolecek xreg ? BJ_XREG : SLJIT_IMM, 1171 1.2.2.4 jdolecek xreg ? 0 : (uint32_t)pc->k); 1172 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1173 1.2.2.2 tls return status; 1174 1.2.2.2 tls 1175 1.2.2.2 tls #if defined(BPFJIT_USE_UDIV) 1176 1.2.2.4 jdolecek status = sljit_emit_op0(compiler, SLJIT_UDIV|SLJIT_I32_OP); 1177 1.2.2.4 jdolecek 1178 1.2.2.4 jdolecek if (BPF_OP(pc->code) == BPF_DIV) { 1179 1.2.2.4 jdolecek #if BJ_AREG != SLJIT_R0 1180 1.2.2.4 jdolecek status = sljit_emit_op1(compiler, 1181 1.2.2.4 jdolecek SLJIT_MOV, 1182 1.2.2.4 jdolecek BJ_AREG, 0, 1183 1.2.2.4 jdolecek SLJIT_R0, 0); 1184 1.2.2.4 jdolecek #endif 1185 1.2.2.4 jdolecek } else { 1186 1.2.2.4 jdolecek #if BJ_AREG != SLJIT_R1 1187 1.2.2.4 jdolecek /* Remainder is in SLJIT_R1. */ 1188 1.2.2.4 jdolecek status = sljit_emit_op1(compiler, 1189 1.2.2.4 jdolecek SLJIT_MOV, 1190 1.2.2.4 jdolecek BJ_AREG, 0, 1191 1.2.2.4 jdolecek SLJIT_R1, 0); 1192 1.2.2.4 jdolecek #endif 1193 1.2.2.4 jdolecek } 1194 1.2.2.2 tls 1195 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1196 1.2.2.2 tls return status; 1197 1.2.2.2 tls #else 1198 1.2.2.2 tls status = sljit_emit_ijump(compiler, 1199 1.2.2.2 tls SLJIT_CALL2, 1200 1.2.2.4 jdolecek SLJIT_IMM, xdiv ? SLJIT_FUNC_OFFSET(divide) : 1201 1.2.2.4 jdolecek SLJIT_FUNC_OFFSET(modulus)); 1202 1.2.2.2 tls 1203 1.2.2.3 tls #if BJ_AREG != SLJIT_RETURN_REG 1204 1.2.2.2 tls status = sljit_emit_op1(compiler, 1205 1.2.2.2 tls SLJIT_MOV, 1206 1.2.2.3 tls BJ_AREG, 0, 1207 1.2.2.2 tls SLJIT_RETURN_REG, 0); 1208 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1209 1.2.2.2 tls return status; 1210 1.2.2.2 tls #endif 1211 1.2.2.2 tls #endif 1212 1.2.2.2 tls 1213 1.2.2.2 tls return status; 1214 1.2.2.2 tls } 1215 1.2.2.2 tls 1216 1.2.2.2 tls /* 1217 1.2.2.2 tls * Return true if pc is a "read from packet" instruction. 1218 1.2.2.2 tls * If length is not NULL and return value is true, *length will 1219 1.2.2.2 tls * be set to a safe length required to read a packet. 1220 1.2.2.2 tls */ 1221 1.2.2.2 tls static bool 1222 1.2.2.3 tls read_pkt_insn(const struct bpf_insn *pc, bpfjit_abc_length_t *length) 1223 1.2.2.2 tls { 1224 1.2.2.2 tls bool rv; 1225 1.2.2.4 jdolecek bpfjit_abc_length_t width = 0; /* XXXuninit */ 1226 1.2.2.2 tls 1227 1.2.2.2 tls switch (BPF_CLASS(pc->code)) { 1228 1.2.2.2 tls default: 1229 1.2.2.2 tls rv = false; 1230 1.2.2.2 tls break; 1231 1.2.2.2 tls 1232 1.2.2.2 tls case BPF_LD: 1233 1.2.2.2 tls rv = BPF_MODE(pc->code) == BPF_ABS || 1234 1.2.2.2 tls BPF_MODE(pc->code) == BPF_IND; 1235 1.2.2.4 jdolecek if (rv) { 1236 1.2.2.2 tls width = read_width(pc); 1237 1.2.2.4 jdolecek rv = (width != 0); 1238 1.2.2.4 jdolecek } 1239 1.2.2.2 tls break; 1240 1.2.2.2 tls 1241 1.2.2.2 tls case BPF_LDX: 1242 1.2.2.4 jdolecek rv = BPF_MODE(pc->code) == BPF_MSH && 1243 1.2.2.4 jdolecek BPF_SIZE(pc->code) == BPF_B; 1244 1.2.2.2 tls width = 1; 1245 1.2.2.2 tls break; 1246 1.2.2.2 tls } 1247 1.2.2.2 tls 1248 1.2.2.2 tls if (rv && length != NULL) { 1249 1.2.2.3 tls /* 1250 1.2.2.3 tls * Values greater than UINT32_MAX will generate 1251 1.2.2.3 tls * unconditional "return 0". 1252 1.2.2.3 tls */ 1253 1.2.2.3 tls *length = (uint32_t)pc->k + width; 1254 1.2.2.2 tls } 1255 1.2.2.2 tls 1256 1.2.2.2 tls return rv; 1257 1.2.2.2 tls } 1258 1.2.2.2 tls 1259 1.2.2.2 tls static void 1260 1.2.2.3 tls optimize_init(struct bpfjit_insn_data *insn_dat, size_t insn_count) 1261 1.2.2.2 tls { 1262 1.2.2.3 tls size_t i; 1263 1.2.2.2 tls 1264 1.2.2.3 tls for (i = 0; i < insn_count; i++) { 1265 1.2.2.3 tls SLIST_INIT(&insn_dat[i].bjumps); 1266 1.2.2.3 tls insn_dat[i].invalid = BJ_INIT_NOBITS; 1267 1.2.2.2 tls } 1268 1.2.2.2 tls } 1269 1.2.2.2 tls 1270 1.2.2.2 tls /* 1271 1.2.2.2 tls * The function divides instructions into blocks. Destination of a jump 1272 1.2.2.2 tls * instruction starts a new block. BPF_RET and BPF_JMP instructions 1273 1.2.2.2 tls * terminate a block. Blocks are linear, that is, there are no jumps out 1274 1.2.2.2 tls * from the middle of a block and there are no jumps in to the middle of 1275 1.2.2.2 tls * a block. 1276 1.2.2.3 tls * 1277 1.2.2.3 tls * The function also sets bits in *initmask for memwords that 1278 1.2.2.3 tls * need to be initialized to zero. Note that this set should be empty 1279 1.2.2.3 tls * for any valid kernel filter program. 1280 1.2.2.2 tls */ 1281 1.2.2.3 tls static bool 1282 1.2.2.3 tls optimize_pass1(const bpf_ctx_t *bc, const struct bpf_insn *insns, 1283 1.2.2.3 tls struct bpfjit_insn_data *insn_dat, size_t insn_count, 1284 1.2.2.3 tls bpf_memword_init_t *initmask, bpfjit_hint_t *hints) 1285 1.2.2.2 tls { 1286 1.2.2.3 tls struct bpfjit_jump *jtf; 1287 1.2.2.2 tls size_t i; 1288 1.2.2.2 tls uint32_t jt, jf; 1289 1.2.2.3 tls bpfjit_abc_length_t length; 1290 1.2.2.3 tls bpf_memword_init_t invalid; /* borrowed from bpf_filter() */ 1291 1.2.2.3 tls bool unreachable; 1292 1.2.2.3 tls 1293 1.2.2.3 tls const size_t memwords = GET_MEMWORDS(bc); 1294 1.2.2.2 tls 1295 1.2.2.3 tls *hints = 0; 1296 1.2.2.3 tls *initmask = BJ_INIT_NOBITS; 1297 1.2.2.2 tls 1298 1.2.2.2 tls unreachable = false; 1299 1.2.2.3 tls invalid = ~BJ_INIT_NOBITS; 1300 1.2.2.2 tls 1301 1.2.2.2 tls for (i = 0; i < insn_count; i++) { 1302 1.2.2.3 tls if (!SLIST_EMPTY(&insn_dat[i].bjumps)) 1303 1.2.2.2 tls unreachable = false; 1304 1.2.2.3 tls insn_dat[i].unreachable = unreachable; 1305 1.2.2.2 tls 1306 1.2.2.2 tls if (unreachable) 1307 1.2.2.2 tls continue; 1308 1.2.2.2 tls 1309 1.2.2.3 tls invalid |= insn_dat[i].invalid; 1310 1.2.2.3 tls 1311 1.2.2.3 tls if (read_pkt_insn(&insns[i], &length) && length > UINT32_MAX) 1312 1.2.2.3 tls unreachable = true; 1313 1.2.2.2 tls 1314 1.2.2.2 tls switch (BPF_CLASS(insns[i].code)) { 1315 1.2.2.2 tls case BPF_RET: 1316 1.2.2.3 tls if (BPF_RVAL(insns[i].code) == BPF_A) 1317 1.2.2.3 tls *initmask |= invalid & BJ_INIT_ABIT; 1318 1.2.2.3 tls 1319 1.2.2.2 tls unreachable = true; 1320 1.2.2.2 tls continue; 1321 1.2.2.2 tls 1322 1.2.2.3 tls case BPF_LD: 1323 1.2.2.3 tls if (BPF_MODE(insns[i].code) == BPF_ABS) 1324 1.2.2.3 tls *hints |= BJ_HINT_ABS; 1325 1.2.2.3 tls 1326 1.2.2.3 tls if (BPF_MODE(insns[i].code) == BPF_IND) { 1327 1.2.2.3 tls *hints |= BJ_HINT_IND | BJ_HINT_XREG; 1328 1.2.2.3 tls *initmask |= invalid & BJ_INIT_XBIT; 1329 1.2.2.3 tls } 1330 1.2.2.3 tls 1331 1.2.2.3 tls if (BPF_MODE(insns[i].code) == BPF_MEM && 1332 1.2.2.3 tls (uint32_t)insns[i].k < memwords) { 1333 1.2.2.3 tls *initmask |= invalid & BJ_INIT_MBIT(insns[i].k); 1334 1.2.2.3 tls } 1335 1.2.2.3 tls 1336 1.2.2.3 tls invalid &= ~BJ_INIT_ABIT; 1337 1.2.2.3 tls continue; 1338 1.2.2.3 tls 1339 1.2.2.3 tls case BPF_LDX: 1340 1.2.2.3 tls *hints |= BJ_HINT_XREG | BJ_HINT_LDX; 1341 1.2.2.3 tls 1342 1.2.2.3 tls if (BPF_MODE(insns[i].code) == BPF_MEM && 1343 1.2.2.3 tls (uint32_t)insns[i].k < memwords) { 1344 1.2.2.3 tls *initmask |= invalid & BJ_INIT_MBIT(insns[i].k); 1345 1.2.2.3 tls } 1346 1.2.2.3 tls 1347 1.2.2.3 tls if (BPF_MODE(insns[i].code) == BPF_MSH && 1348 1.2.2.3 tls BPF_SIZE(insns[i].code) == BPF_B) { 1349 1.2.2.3 tls *hints |= BJ_HINT_MSH; 1350 1.2.2.3 tls } 1351 1.2.2.3 tls 1352 1.2.2.3 tls invalid &= ~BJ_INIT_XBIT; 1353 1.2.2.3 tls continue; 1354 1.2.2.3 tls 1355 1.2.2.3 tls case BPF_ST: 1356 1.2.2.3 tls *initmask |= invalid & BJ_INIT_ABIT; 1357 1.2.2.3 tls 1358 1.2.2.3 tls if ((uint32_t)insns[i].k < memwords) 1359 1.2.2.3 tls invalid &= ~BJ_INIT_MBIT(insns[i].k); 1360 1.2.2.3 tls 1361 1.2.2.3 tls continue; 1362 1.2.2.3 tls 1363 1.2.2.3 tls case BPF_STX: 1364 1.2.2.3 tls *hints |= BJ_HINT_XREG; 1365 1.2.2.3 tls *initmask |= invalid & BJ_INIT_XBIT; 1366 1.2.2.3 tls 1367 1.2.2.3 tls if ((uint32_t)insns[i].k < memwords) 1368 1.2.2.3 tls invalid &= ~BJ_INIT_MBIT(insns[i].k); 1369 1.2.2.3 tls 1370 1.2.2.3 tls continue; 1371 1.2.2.3 tls 1372 1.2.2.3 tls case BPF_ALU: 1373 1.2.2.3 tls *initmask |= invalid & BJ_INIT_ABIT; 1374 1.2.2.3 tls 1375 1.2.2.3 tls if (insns[i].code != (BPF_ALU|BPF_NEG) && 1376 1.2.2.3 tls BPF_SRC(insns[i].code) == BPF_X) { 1377 1.2.2.3 tls *hints |= BJ_HINT_XREG; 1378 1.2.2.3 tls *initmask |= invalid & BJ_INIT_XBIT; 1379 1.2.2.3 tls } 1380 1.2.2.3 tls 1381 1.2.2.3 tls invalid &= ~BJ_INIT_ABIT; 1382 1.2.2.3 tls continue; 1383 1.2.2.3 tls 1384 1.2.2.3 tls case BPF_MISC: 1385 1.2.2.3 tls switch (BPF_MISCOP(insns[i].code)) { 1386 1.2.2.3 tls case BPF_TAX: // X <- A 1387 1.2.2.3 tls *hints |= BJ_HINT_XREG; 1388 1.2.2.3 tls *initmask |= invalid & BJ_INIT_ABIT; 1389 1.2.2.3 tls invalid &= ~BJ_INIT_XBIT; 1390 1.2.2.3 tls continue; 1391 1.2.2.3 tls 1392 1.2.2.3 tls case BPF_TXA: // A <- X 1393 1.2.2.3 tls *hints |= BJ_HINT_XREG; 1394 1.2.2.3 tls *initmask |= invalid & BJ_INIT_XBIT; 1395 1.2.2.3 tls invalid &= ~BJ_INIT_ABIT; 1396 1.2.2.3 tls continue; 1397 1.2.2.3 tls 1398 1.2.2.3 tls case BPF_COPX: 1399 1.2.2.3 tls *hints |= BJ_HINT_XREG | BJ_HINT_COPX; 1400 1.2.2.3 tls /* FALLTHROUGH */ 1401 1.2.2.3 tls 1402 1.2.2.3 tls case BPF_COP: 1403 1.2.2.3 tls *hints |= BJ_HINT_COP; 1404 1.2.2.3 tls *initmask |= invalid & BJ_INIT_ABIT; 1405 1.2.2.3 tls invalid &= ~BJ_INIT_ABIT; 1406 1.2.2.3 tls continue; 1407 1.2.2.3 tls } 1408 1.2.2.3 tls 1409 1.2.2.3 tls continue; 1410 1.2.2.3 tls 1411 1.2.2.2 tls case BPF_JMP: 1412 1.2.2.3 tls /* Initialize abc_length for ABC pass. */ 1413 1.2.2.3 tls insn_dat[i].u.jdata.abc_length = MAX_ABC_LENGTH; 1414 1.2.2.3 tls 1415 1.2.2.4 jdolecek *initmask |= invalid & BJ_INIT_ABIT; 1416 1.2.2.4 jdolecek 1417 1.2.2.4 jdolecek if (BPF_SRC(insns[i].code) == BPF_X) { 1418 1.2.2.4 jdolecek *hints |= BJ_HINT_XREG; 1419 1.2.2.4 jdolecek *initmask |= invalid & BJ_INIT_XBIT; 1420 1.2.2.4 jdolecek } 1421 1.2.2.4 jdolecek 1422 1.2.2.3 tls if (BPF_OP(insns[i].code) == BPF_JA) { 1423 1.2.2.2 tls jt = jf = insns[i].k; 1424 1.2.2.2 tls } else { 1425 1.2.2.2 tls jt = insns[i].jt; 1426 1.2.2.2 tls jf = insns[i].jf; 1427 1.2.2.2 tls } 1428 1.2.2.2 tls 1429 1.2.2.2 tls if (jt >= insn_count - (i + 1) || 1430 1.2.2.2 tls jf >= insn_count - (i + 1)) { 1431 1.2.2.3 tls return false; 1432 1.2.2.2 tls } 1433 1.2.2.2 tls 1434 1.2.2.2 tls if (jt > 0 && jf > 0) 1435 1.2.2.2 tls unreachable = true; 1436 1.2.2.2 tls 1437 1.2.2.3 tls jt += i + 1; 1438 1.2.2.3 tls jf += i + 1; 1439 1.2.2.3 tls 1440 1.2.2.3 tls jtf = insn_dat[i].u.jdata.jtf; 1441 1.2.2.2 tls 1442 1.2.2.3 tls jtf[0].jdata = &insn_dat[i].u.jdata; 1443 1.2.2.3 tls SLIST_INSERT_HEAD(&insn_dat[jt].bjumps, 1444 1.2.2.3 tls &jtf[0], entries); 1445 1.2.2.2 tls 1446 1.2.2.2 tls if (jf != jt) { 1447 1.2.2.3 tls jtf[1].jdata = &insn_dat[i].u.jdata; 1448 1.2.2.3 tls SLIST_INSERT_HEAD(&insn_dat[jf].bjumps, 1449 1.2.2.3 tls &jtf[1], entries); 1450 1.2.2.2 tls } 1451 1.2.2.2 tls 1452 1.2.2.3 tls insn_dat[jf].invalid |= invalid; 1453 1.2.2.3 tls insn_dat[jt].invalid |= invalid; 1454 1.2.2.3 tls invalid = 0; 1455 1.2.2.3 tls 1456 1.2.2.2 tls continue; 1457 1.2.2.2 tls } 1458 1.2.2.2 tls } 1459 1.2.2.2 tls 1460 1.2.2.3 tls return true; 1461 1.2.2.2 tls } 1462 1.2.2.2 tls 1463 1.2.2.2 tls /* 1464 1.2.2.3 tls * Array Bounds Check Elimination (ABC) pass. 1465 1.2.2.2 tls */ 1466 1.2.2.3 tls static void 1467 1.2.2.3 tls optimize_pass2(const bpf_ctx_t *bc, const struct bpf_insn *insns, 1468 1.2.2.3 tls struct bpfjit_insn_data *insn_dat, size_t insn_count) 1469 1.2.2.2 tls { 1470 1.2.2.3 tls struct bpfjit_jump *jmp; 1471 1.2.2.3 tls const struct bpf_insn *pc; 1472 1.2.2.3 tls struct bpfjit_insn_data *pd; 1473 1.2.2.2 tls size_t i; 1474 1.2.2.3 tls bpfjit_abc_length_t length, abc_length = 0; 1475 1.2.2.2 tls 1476 1.2.2.3 tls const size_t extwords = GET_EXTWORDS(bc); 1477 1.2.2.2 tls 1478 1.2.2.3 tls for (i = insn_count; i != 0; i--) { 1479 1.2.2.3 tls pc = &insns[i-1]; 1480 1.2.2.3 tls pd = &insn_dat[i-1]; 1481 1.2.2.3 tls 1482 1.2.2.3 tls if (pd->unreachable) 1483 1.2.2.3 tls continue; 1484 1.2.2.3 tls 1485 1.2.2.3 tls switch (BPF_CLASS(pc->code)) { 1486 1.2.2.3 tls case BPF_RET: 1487 1.2.2.3 tls /* 1488 1.2.2.3 tls * It's quite common for bpf programs to 1489 1.2.2.3 tls * check packet bytes in increasing order 1490 1.2.2.3 tls * and return zero if bytes don't match 1491 1.2.2.3 tls * specified critetion. Such programs disable 1492 1.2.2.3 tls * ABC optimization completely because for 1493 1.2.2.3 tls * every jump there is a branch with no read 1494 1.2.2.3 tls * instruction. 1495 1.2.2.3 tls * With no side effects, BPF_STMT(BPF_RET+BPF_K, 0) 1496 1.2.2.3 tls * is indistinguishable from out-of-bound load. 1497 1.2.2.3 tls * Therefore, abc_length can be set to 1498 1.2.2.3 tls * MAX_ABC_LENGTH and enable ABC for many 1499 1.2.2.3 tls * bpf programs. 1500 1.2.2.3 tls * If this optimization encounters any 1501 1.2.2.3 tls * instruction with a side effect, it will 1502 1.2.2.3 tls * reset abc_length. 1503 1.2.2.3 tls */ 1504 1.2.2.3 tls if (BPF_RVAL(pc->code) == BPF_K && pc->k == 0) 1505 1.2.2.3 tls abc_length = MAX_ABC_LENGTH; 1506 1.2.2.3 tls else 1507 1.2.2.3 tls abc_length = 0; 1508 1.2.2.3 tls break; 1509 1.2.2.3 tls 1510 1.2.2.3 tls case BPF_MISC: 1511 1.2.2.3 tls if (BPF_MISCOP(pc->code) == BPF_COP || 1512 1.2.2.3 tls BPF_MISCOP(pc->code) == BPF_COPX) { 1513 1.2.2.3 tls /* COP instructions can have side effects. */ 1514 1.2.2.3 tls abc_length = 0; 1515 1.2.2.3 tls } 1516 1.2.2.3 tls break; 1517 1.2.2.3 tls 1518 1.2.2.3 tls case BPF_ST: 1519 1.2.2.3 tls case BPF_STX: 1520 1.2.2.3 tls if (extwords != 0) { 1521 1.2.2.3 tls /* Write to memory is visible after a call. */ 1522 1.2.2.3 tls abc_length = 0; 1523 1.2.2.3 tls } 1524 1.2.2.3 tls break; 1525 1.2.2.3 tls 1526 1.2.2.3 tls case BPF_JMP: 1527 1.2.2.3 tls abc_length = pd->u.jdata.abc_length; 1528 1.2.2.3 tls break; 1529 1.2.2.3 tls 1530 1.2.2.3 tls default: 1531 1.2.2.3 tls if (read_pkt_insn(pc, &length)) { 1532 1.2.2.3 tls if (abc_length < length) 1533 1.2.2.3 tls abc_length = length; 1534 1.2.2.3 tls pd->u.rdata.abc_length = abc_length; 1535 1.2.2.3 tls } 1536 1.2.2.3 tls break; 1537 1.2.2.2 tls } 1538 1.2.2.2 tls 1539 1.2.2.3 tls SLIST_FOREACH(jmp, &pd->bjumps, entries) { 1540 1.2.2.3 tls if (jmp->jdata->abc_length > abc_length) 1541 1.2.2.3 tls jmp->jdata->abc_length = abc_length; 1542 1.2.2.2 tls } 1543 1.2.2.3 tls } 1544 1.2.2.3 tls } 1545 1.2.2.2 tls 1546 1.2.2.3 tls static void 1547 1.2.2.3 tls optimize_pass3(const struct bpf_insn *insns, 1548 1.2.2.3 tls struct bpfjit_insn_data *insn_dat, size_t insn_count) 1549 1.2.2.3 tls { 1550 1.2.2.3 tls struct bpfjit_jump *jmp; 1551 1.2.2.3 tls size_t i; 1552 1.2.2.3 tls bpfjit_abc_length_t checked_length = 0; 1553 1.2.2.3 tls 1554 1.2.2.3 tls for (i = 0; i < insn_count; i++) { 1555 1.2.2.3 tls if (insn_dat[i].unreachable) 1556 1.2.2.3 tls continue; 1557 1.2.2.2 tls 1558 1.2.2.3 tls SLIST_FOREACH(jmp, &insn_dat[i].bjumps, entries) { 1559 1.2.2.3 tls if (jmp->jdata->checked_length < checked_length) 1560 1.2.2.3 tls checked_length = jmp->jdata->checked_length; 1561 1.2.2.3 tls } 1562 1.2.2.3 tls 1563 1.2.2.3 tls if (BPF_CLASS(insns[i].code) == BPF_JMP) { 1564 1.2.2.3 tls insn_dat[i].u.jdata.checked_length = checked_length; 1565 1.2.2.3 tls } else if (read_pkt_insn(&insns[i], NULL)) { 1566 1.2.2.3 tls struct bpfjit_read_pkt_data *rdata = 1567 1.2.2.3 tls &insn_dat[i].u.rdata; 1568 1.2.2.3 tls rdata->check_length = 0; 1569 1.2.2.3 tls if (checked_length < rdata->abc_length) { 1570 1.2.2.3 tls checked_length = rdata->abc_length; 1571 1.2.2.3 tls rdata->check_length = checked_length; 1572 1.2.2.3 tls } 1573 1.2.2.2 tls } 1574 1.2.2.2 tls } 1575 1.2.2.3 tls } 1576 1.2.2.2 tls 1577 1.2.2.3 tls static bool 1578 1.2.2.3 tls optimize(const bpf_ctx_t *bc, const struct bpf_insn *insns, 1579 1.2.2.3 tls struct bpfjit_insn_data *insn_dat, size_t insn_count, 1580 1.2.2.3 tls bpf_memword_init_t *initmask, bpfjit_hint_t *hints) 1581 1.2.2.3 tls { 1582 1.2.2.3 tls 1583 1.2.2.3 tls optimize_init(insn_dat, insn_count); 1584 1.2.2.3 tls 1585 1.2.2.3 tls if (!optimize_pass1(bc, insns, insn_dat, insn_count, initmask, hints)) 1586 1.2.2.3 tls return false; 1587 1.2.2.3 tls 1588 1.2.2.3 tls optimize_pass2(bc, insns, insn_dat, insn_count); 1589 1.2.2.3 tls optimize_pass3(insns, insn_dat, insn_count); 1590 1.2.2.3 tls 1591 1.2.2.3 tls return true; 1592 1.2.2.2 tls } 1593 1.2.2.2 tls 1594 1.2.2.2 tls /* 1595 1.2.2.2 tls * Convert BPF_ALU operations except BPF_NEG and BPF_DIV to sljit operation. 1596 1.2.2.2 tls */ 1597 1.2.2.4 jdolecek static bool 1598 1.2.2.4 jdolecek alu_to_op(const struct bpf_insn *pc, int *res) 1599 1.2.2.2 tls { 1600 1.2.2.4 jdolecek const uint32_t k = pc->k; 1601 1.2.2.2 tls 1602 1.2.2.2 tls /* 1603 1.2.2.2 tls * Note: all supported 64bit arches have 32bit multiply 1604 1.2.2.4 jdolecek * instruction so SLJIT_I32_OP doesn't have any overhead. 1605 1.2.2.2 tls */ 1606 1.2.2.2 tls switch (BPF_OP(pc->code)) { 1607 1.2.2.4 jdolecek case BPF_ADD: 1608 1.2.2.4 jdolecek *res = SLJIT_ADD; 1609 1.2.2.4 jdolecek return true; 1610 1.2.2.4 jdolecek case BPF_SUB: 1611 1.2.2.4 jdolecek *res = SLJIT_SUB; 1612 1.2.2.4 jdolecek return true; 1613 1.2.2.4 jdolecek case BPF_MUL: 1614 1.2.2.4 jdolecek *res = SLJIT_MUL|SLJIT_I32_OP; 1615 1.2.2.4 jdolecek return true; 1616 1.2.2.4 jdolecek case BPF_OR: 1617 1.2.2.4 jdolecek *res = SLJIT_OR; 1618 1.2.2.4 jdolecek return true; 1619 1.2.2.4 jdolecek case BPF_XOR: 1620 1.2.2.4 jdolecek *res = SLJIT_XOR; 1621 1.2.2.4 jdolecek return true; 1622 1.2.2.4 jdolecek case BPF_AND: 1623 1.2.2.4 jdolecek *res = SLJIT_AND; 1624 1.2.2.4 jdolecek return true; 1625 1.2.2.4 jdolecek case BPF_LSH: 1626 1.2.2.4 jdolecek *res = SLJIT_SHL; 1627 1.2.2.4 jdolecek return k < 32; 1628 1.2.2.4 jdolecek case BPF_RSH: 1629 1.2.2.4 jdolecek *res = SLJIT_LSHR|SLJIT_I32_OP; 1630 1.2.2.4 jdolecek return k < 32; 1631 1.2.2.2 tls default: 1632 1.2.2.4 jdolecek return false; 1633 1.2.2.2 tls } 1634 1.2.2.2 tls } 1635 1.2.2.2 tls 1636 1.2.2.2 tls /* 1637 1.2.2.2 tls * Convert BPF_JMP operations except BPF_JA to sljit condition. 1638 1.2.2.2 tls */ 1639 1.2.2.4 jdolecek static bool 1640 1.2.2.4 jdolecek jmp_to_cond(const struct bpf_insn *pc, bool negate, int *res) 1641 1.2.2.2 tls { 1642 1.2.2.4 jdolecek 1643 1.2.2.2 tls /* 1644 1.2.2.2 tls * Note: all supported 64bit arches have 32bit comparison 1645 1.2.2.4 jdolecek * instructions so SLJIT_I32_OP doesn't have any overhead. 1646 1.2.2.2 tls */ 1647 1.2.2.4 jdolecek *res = SLJIT_I32_OP; 1648 1.2.2.2 tls 1649 1.2.2.2 tls switch (BPF_OP(pc->code)) { 1650 1.2.2.2 tls case BPF_JGT: 1651 1.2.2.4 jdolecek *res |= negate ? SLJIT_LESS_EQUAL : SLJIT_GREATER; 1652 1.2.2.4 jdolecek return true; 1653 1.2.2.2 tls case BPF_JGE: 1654 1.2.2.4 jdolecek *res |= negate ? SLJIT_LESS : SLJIT_GREATER_EQUAL; 1655 1.2.2.4 jdolecek return true; 1656 1.2.2.2 tls case BPF_JEQ: 1657 1.2.2.4 jdolecek *res |= negate ? SLJIT_NOT_EQUAL : SLJIT_EQUAL; 1658 1.2.2.4 jdolecek return true; 1659 1.2.2.2 tls case BPF_JSET: 1660 1.2.2.4 jdolecek *res |= negate ? SLJIT_EQUAL : SLJIT_NOT_EQUAL; 1661 1.2.2.4 jdolecek return true; 1662 1.2.2.2 tls default: 1663 1.2.2.4 jdolecek return false; 1664 1.2.2.2 tls } 1665 1.2.2.2 tls } 1666 1.2.2.2 tls 1667 1.2.2.2 tls /* 1668 1.2.2.2 tls * Convert BPF_K and BPF_X to sljit register. 1669 1.2.2.2 tls */ 1670 1.2.2.2 tls static int 1671 1.2.2.3 tls kx_to_reg(const struct bpf_insn *pc) 1672 1.2.2.2 tls { 1673 1.2.2.2 tls 1674 1.2.2.2 tls switch (BPF_SRC(pc->code)) { 1675 1.2.2.2 tls case BPF_K: return SLJIT_IMM; 1676 1.2.2.3 tls case BPF_X: return BJ_XREG; 1677 1.2.2.2 tls default: 1678 1.2.2.3 tls BJ_ASSERT(false); 1679 1.2.2.2 tls return 0; 1680 1.2.2.2 tls } 1681 1.2.2.2 tls } 1682 1.2.2.2 tls 1683 1.2.2.3 tls static sljit_sw 1684 1.2.2.3 tls kx_to_reg_arg(const struct bpf_insn *pc) 1685 1.2.2.2 tls { 1686 1.2.2.2 tls 1687 1.2.2.2 tls switch (BPF_SRC(pc->code)) { 1688 1.2.2.2 tls case BPF_K: return (uint32_t)pc->k; /* SLJIT_IMM, pc->k, */ 1689 1.2.2.3 tls case BPF_X: return 0; /* BJ_XREG, 0, */ 1690 1.2.2.2 tls default: 1691 1.2.2.3 tls BJ_ASSERT(false); 1692 1.2.2.2 tls return 0; 1693 1.2.2.2 tls } 1694 1.2.2.2 tls } 1695 1.2.2.2 tls 1696 1.2.2.3 tls static bool 1697 1.2.2.3 tls generate_insn_code(struct sljit_compiler *compiler, bpfjit_hint_t hints, 1698 1.2.2.3 tls const bpf_ctx_t *bc, const struct bpf_insn *insns, 1699 1.2.2.3 tls struct bpfjit_insn_data *insn_dat, size_t insn_count) 1700 1.2.2.2 tls { 1701 1.2.2.2 tls /* a list of jumps to out-of-bound return from a generated function */ 1702 1.2.2.2 tls struct sljit_jump **ret0; 1703 1.2.2.2 tls size_t ret0_size, ret0_maxsize; 1704 1.2.2.2 tls 1705 1.2.2.2 tls struct sljit_jump *jump; 1706 1.2.2.3 tls struct sljit_label *label; 1707 1.2.2.3 tls const struct bpf_insn *pc; 1708 1.2.2.2 tls struct bpfjit_jump *bjump, *jtf; 1709 1.2.2.2 tls struct sljit_jump *to_mchain_jump; 1710 1.2.2.2 tls 1711 1.2.2.3 tls size_t i; 1712 1.2.2.4 jdolecek unsigned int rval, mode, src, op; 1713 1.2.2.3 tls int branching, negate; 1714 1.2.2.4 jdolecek int status, cond, op2; 1715 1.2.2.2 tls uint32_t jt, jf; 1716 1.2.2.2 tls 1717 1.2.2.3 tls bool unconditional_ret; 1718 1.2.2.3 tls bool rv; 1719 1.2.2.2 tls 1720 1.2.2.3 tls const size_t extwords = GET_EXTWORDS(bc); 1721 1.2.2.3 tls const size_t memwords = GET_MEMWORDS(bc); 1722 1.2.2.2 tls 1723 1.2.2.3 tls ret0 = NULL; 1724 1.2.2.3 tls rv = false; 1725 1.2.2.2 tls 1726 1.2.2.2 tls ret0_size = 0; 1727 1.2.2.3 tls ret0_maxsize = 64; 1728 1.2.2.3 tls ret0 = BJ_ALLOC(ret0_maxsize * sizeof(ret0[0])); 1729 1.2.2.3 tls if (ret0 == NULL) 1730 1.2.2.2 tls goto fail; 1731 1.2.2.2 tls 1732 1.2.2.3 tls /* reset sjump members of jdata */ 1733 1.2.2.3 tls for (i = 0; i < insn_count; i++) { 1734 1.2.2.3 tls if (insn_dat[i].unreachable || 1735 1.2.2.3 tls BPF_CLASS(insns[i].code) != BPF_JMP) { 1736 1.2.2.3 tls continue; 1737 1.2.2.3 tls } 1738 1.2.2.2 tls 1739 1.2.2.3 tls jtf = insn_dat[i].u.jdata.jtf; 1740 1.2.2.3 tls jtf[0].sjump = jtf[1].sjump = NULL; 1741 1.2.2.2 tls } 1742 1.2.2.2 tls 1743 1.2.2.3 tls /* main loop */ 1744 1.2.2.2 tls for (i = 0; i < insn_count; i++) { 1745 1.2.2.3 tls if (insn_dat[i].unreachable) 1746 1.2.2.2 tls continue; 1747 1.2.2.2 tls 1748 1.2.2.2 tls /* 1749 1.2.2.2 tls * Resolve jumps to the current insn. 1750 1.2.2.2 tls */ 1751 1.2.2.2 tls label = NULL; 1752 1.2.2.3 tls SLIST_FOREACH(bjump, &insn_dat[i].bjumps, entries) { 1753 1.2.2.3 tls if (bjump->sjump != NULL) { 1754 1.2.2.2 tls if (label == NULL) 1755 1.2.2.2 tls label = sljit_emit_label(compiler); 1756 1.2.2.2 tls if (label == NULL) 1757 1.2.2.2 tls goto fail; 1758 1.2.2.3 tls sljit_set_label(bjump->sjump, label); 1759 1.2.2.2 tls } 1760 1.2.2.2 tls } 1761 1.2.2.2 tls 1762 1.2.2.3 tls to_mchain_jump = NULL; 1763 1.2.2.3 tls unconditional_ret = false; 1764 1.2.2.3 tls 1765 1.2.2.3 tls if (read_pkt_insn(&insns[i], NULL)) { 1766 1.2.2.3 tls if (insn_dat[i].u.rdata.check_length > UINT32_MAX) { 1767 1.2.2.3 tls /* Jump to "return 0" unconditionally. */ 1768 1.2.2.3 tls unconditional_ret = true; 1769 1.2.2.3 tls jump = sljit_emit_jump(compiler, SLJIT_JUMP); 1770 1.2.2.3 tls if (jump == NULL) 1771 1.2.2.3 tls goto fail; 1772 1.2.2.3 tls if (!append_jump(jump, &ret0, 1773 1.2.2.3 tls &ret0_size, &ret0_maxsize)) 1774 1.2.2.3 tls goto fail; 1775 1.2.2.3 tls } else if (insn_dat[i].u.rdata.check_length > 0) { 1776 1.2.2.3 tls /* if (buflen < check_length) return 0; */ 1777 1.2.2.3 tls jump = sljit_emit_cmp(compiler, 1778 1.2.2.4 jdolecek SLJIT_LESS, 1779 1.2.2.3 tls BJ_BUFLEN, 0, 1780 1.2.2.3 tls SLJIT_IMM, 1781 1.2.2.3 tls insn_dat[i].u.rdata.check_length); 1782 1.2.2.3 tls if (jump == NULL) 1783 1.2.2.3 tls goto fail; 1784 1.2.2.2 tls #ifdef _KERNEL 1785 1.2.2.3 tls to_mchain_jump = jump; 1786 1.2.2.2 tls #else 1787 1.2.2.3 tls if (!append_jump(jump, &ret0, 1788 1.2.2.3 tls &ret0_size, &ret0_maxsize)) 1789 1.2.2.3 tls goto fail; 1790 1.2.2.2 tls #endif 1791 1.2.2.3 tls } 1792 1.2.2.2 tls } 1793 1.2.2.2 tls 1794 1.2.2.2 tls pc = &insns[i]; 1795 1.2.2.2 tls switch (BPF_CLASS(pc->code)) { 1796 1.2.2.2 tls 1797 1.2.2.2 tls default: 1798 1.2.2.2 tls goto fail; 1799 1.2.2.2 tls 1800 1.2.2.2 tls case BPF_LD: 1801 1.2.2.2 tls /* BPF_LD+BPF_IMM A <- k */ 1802 1.2.2.2 tls if (pc->code == (BPF_LD|BPF_IMM)) { 1803 1.2.2.2 tls status = sljit_emit_op1(compiler, 1804 1.2.2.2 tls SLJIT_MOV, 1805 1.2.2.3 tls BJ_AREG, 0, 1806 1.2.2.2 tls SLJIT_IMM, (uint32_t)pc->k); 1807 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1808 1.2.2.2 tls goto fail; 1809 1.2.2.2 tls 1810 1.2.2.2 tls continue; 1811 1.2.2.2 tls } 1812 1.2.2.2 tls 1813 1.2.2.2 tls /* BPF_LD+BPF_MEM A <- M[k] */ 1814 1.2.2.2 tls if (pc->code == (BPF_LD|BPF_MEM)) { 1815 1.2.2.3 tls if ((uint32_t)pc->k >= memwords) 1816 1.2.2.2 tls goto fail; 1817 1.2.2.3 tls status = emit_memload(compiler, 1818 1.2.2.3 tls BJ_AREG, pc->k, extwords); 1819 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1820 1.2.2.2 tls goto fail; 1821 1.2.2.2 tls 1822 1.2.2.2 tls continue; 1823 1.2.2.2 tls } 1824 1.2.2.2 tls 1825 1.2.2.2 tls /* BPF_LD+BPF_W+BPF_LEN A <- len */ 1826 1.2.2.2 tls if (pc->code == (BPF_LD|BPF_W|BPF_LEN)) { 1827 1.2.2.2 tls status = sljit_emit_op1(compiler, 1828 1.2.2.3 tls SLJIT_MOV, /* size_t source */ 1829 1.2.2.3 tls BJ_AREG, 0, 1830 1.2.2.3 tls SLJIT_MEM1(BJ_ARGS), 1831 1.2.2.3 tls offsetof(struct bpf_args, wirelen)); 1832 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1833 1.2.2.2 tls goto fail; 1834 1.2.2.2 tls 1835 1.2.2.2 tls continue; 1836 1.2.2.2 tls } 1837 1.2.2.2 tls 1838 1.2.2.2 tls mode = BPF_MODE(pc->code); 1839 1.2.2.2 tls if (mode != BPF_ABS && mode != BPF_IND) 1840 1.2.2.2 tls goto fail; 1841 1.2.2.2 tls 1842 1.2.2.3 tls if (unconditional_ret) 1843 1.2.2.3 tls continue; 1844 1.2.2.3 tls 1845 1.2.2.3 tls status = emit_pkt_read(compiler, hints, pc, 1846 1.2.2.3 tls to_mchain_jump, &ret0, &ret0_size, &ret0_maxsize); 1847 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1848 1.2.2.2 tls goto fail; 1849 1.2.2.2 tls 1850 1.2.2.2 tls continue; 1851 1.2.2.2 tls 1852 1.2.2.2 tls case BPF_LDX: 1853 1.2.2.2 tls mode = BPF_MODE(pc->code); 1854 1.2.2.2 tls 1855 1.2.2.2 tls /* BPF_LDX+BPF_W+BPF_IMM X <- k */ 1856 1.2.2.2 tls if (mode == BPF_IMM) { 1857 1.2.2.2 tls if (BPF_SIZE(pc->code) != BPF_W) 1858 1.2.2.2 tls goto fail; 1859 1.2.2.2 tls status = sljit_emit_op1(compiler, 1860 1.2.2.2 tls SLJIT_MOV, 1861 1.2.2.3 tls BJ_XREG, 0, 1862 1.2.2.2 tls SLJIT_IMM, (uint32_t)pc->k); 1863 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1864 1.2.2.2 tls goto fail; 1865 1.2.2.2 tls 1866 1.2.2.2 tls continue; 1867 1.2.2.2 tls } 1868 1.2.2.2 tls 1869 1.2.2.2 tls /* BPF_LDX+BPF_W+BPF_LEN X <- len */ 1870 1.2.2.2 tls if (mode == BPF_LEN) { 1871 1.2.2.2 tls if (BPF_SIZE(pc->code) != BPF_W) 1872 1.2.2.2 tls goto fail; 1873 1.2.2.2 tls status = sljit_emit_op1(compiler, 1874 1.2.2.3 tls SLJIT_MOV, /* size_t source */ 1875 1.2.2.3 tls BJ_XREG, 0, 1876 1.2.2.3 tls SLJIT_MEM1(BJ_ARGS), 1877 1.2.2.3 tls offsetof(struct bpf_args, wirelen)); 1878 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1879 1.2.2.2 tls goto fail; 1880 1.2.2.2 tls 1881 1.2.2.2 tls continue; 1882 1.2.2.2 tls } 1883 1.2.2.2 tls 1884 1.2.2.2 tls /* BPF_LDX+BPF_W+BPF_MEM X <- M[k] */ 1885 1.2.2.2 tls if (mode == BPF_MEM) { 1886 1.2.2.2 tls if (BPF_SIZE(pc->code) != BPF_W) 1887 1.2.2.2 tls goto fail; 1888 1.2.2.3 tls if ((uint32_t)pc->k >= memwords) 1889 1.2.2.2 tls goto fail; 1890 1.2.2.3 tls status = emit_memload(compiler, 1891 1.2.2.3 tls BJ_XREG, pc->k, extwords); 1892 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1893 1.2.2.2 tls goto fail; 1894 1.2.2.2 tls 1895 1.2.2.2 tls continue; 1896 1.2.2.2 tls } 1897 1.2.2.2 tls 1898 1.2.2.2 tls /* BPF_LDX+BPF_B+BPF_MSH X <- 4*(P[k:1]&0xf) */ 1899 1.2.2.2 tls if (mode != BPF_MSH || BPF_SIZE(pc->code) != BPF_B) 1900 1.2.2.2 tls goto fail; 1901 1.2.2.2 tls 1902 1.2.2.3 tls if (unconditional_ret) 1903 1.2.2.3 tls continue; 1904 1.2.2.3 tls 1905 1.2.2.3 tls status = emit_msh(compiler, hints, pc, 1906 1.2.2.3 tls to_mchain_jump, &ret0, &ret0_size, &ret0_maxsize); 1907 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1908 1.2.2.2 tls goto fail; 1909 1.2.2.2 tls 1910 1.2.2.2 tls continue; 1911 1.2.2.2 tls 1912 1.2.2.2 tls case BPF_ST: 1913 1.2.2.3 tls if (pc->code != BPF_ST || 1914 1.2.2.3 tls (uint32_t)pc->k >= memwords) { 1915 1.2.2.2 tls goto fail; 1916 1.2.2.3 tls } 1917 1.2.2.2 tls 1918 1.2.2.3 tls status = emit_memstore(compiler, 1919 1.2.2.3 tls BJ_AREG, pc->k, extwords); 1920 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1921 1.2.2.2 tls goto fail; 1922 1.2.2.2 tls 1923 1.2.2.2 tls continue; 1924 1.2.2.2 tls 1925 1.2.2.2 tls case BPF_STX: 1926 1.2.2.3 tls if (pc->code != BPF_STX || 1927 1.2.2.3 tls (uint32_t)pc->k >= memwords) { 1928 1.2.2.2 tls goto fail; 1929 1.2.2.3 tls } 1930 1.2.2.2 tls 1931 1.2.2.3 tls status = emit_memstore(compiler, 1932 1.2.2.3 tls BJ_XREG, pc->k, extwords); 1933 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1934 1.2.2.2 tls goto fail; 1935 1.2.2.2 tls 1936 1.2.2.2 tls continue; 1937 1.2.2.2 tls 1938 1.2.2.2 tls case BPF_ALU: 1939 1.2.2.2 tls if (pc->code == (BPF_ALU|BPF_NEG)) { 1940 1.2.2.2 tls status = sljit_emit_op1(compiler, 1941 1.2.2.2 tls SLJIT_NEG, 1942 1.2.2.3 tls BJ_AREG, 0, 1943 1.2.2.3 tls BJ_AREG, 0); 1944 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1945 1.2.2.2 tls goto fail; 1946 1.2.2.2 tls 1947 1.2.2.2 tls continue; 1948 1.2.2.2 tls } 1949 1.2.2.2 tls 1950 1.2.2.4 jdolecek op = BPF_OP(pc->code); 1951 1.2.2.4 jdolecek if (op != BPF_DIV && op != BPF_MOD) { 1952 1.2.2.4 jdolecek if (!alu_to_op(pc, &op2)) 1953 1.2.2.4 jdolecek goto fail; 1954 1.2.2.4 jdolecek 1955 1.2.2.2 tls status = sljit_emit_op2(compiler, 1956 1.2.2.4 jdolecek op2, BJ_AREG, 0, BJ_AREG, 0, 1957 1.2.2.2 tls kx_to_reg(pc), kx_to_reg_arg(pc)); 1958 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1959 1.2.2.2 tls goto fail; 1960 1.2.2.2 tls 1961 1.2.2.2 tls continue; 1962 1.2.2.2 tls } 1963 1.2.2.2 tls 1964 1.2.2.4 jdolecek /* BPF_DIV/BPF_MOD */ 1965 1.2.2.2 tls 1966 1.2.2.2 tls src = BPF_SRC(pc->code); 1967 1.2.2.2 tls if (src != BPF_X && src != BPF_K) 1968 1.2.2.2 tls goto fail; 1969 1.2.2.2 tls 1970 1.2.2.2 tls /* division by zero? */ 1971 1.2.2.2 tls if (src == BPF_X) { 1972 1.2.2.2 tls jump = sljit_emit_cmp(compiler, 1973 1.2.2.4 jdolecek SLJIT_EQUAL|SLJIT_I32_OP, 1974 1.2.2.3 tls BJ_XREG, 0, 1975 1.2.2.2 tls SLJIT_IMM, 0); 1976 1.2.2.2 tls if (jump == NULL) 1977 1.2.2.2 tls goto fail; 1978 1.2.2.3 tls if (!append_jump(jump, &ret0, 1979 1.2.2.3 tls &ret0_size, &ret0_maxsize)) 1980 1.2.2.3 tls goto fail; 1981 1.2.2.2 tls } else if (pc->k == 0) { 1982 1.2.2.2 tls jump = sljit_emit_jump(compiler, SLJIT_JUMP); 1983 1.2.2.2 tls if (jump == NULL) 1984 1.2.2.2 tls goto fail; 1985 1.2.2.3 tls if (!append_jump(jump, &ret0, 1986 1.2.2.3 tls &ret0_size, &ret0_maxsize)) 1987 1.2.2.3 tls goto fail; 1988 1.2.2.2 tls } 1989 1.2.2.2 tls 1990 1.2.2.2 tls if (src == BPF_X) { 1991 1.2.2.4 jdolecek status = emit_moddiv(compiler, pc); 1992 1.2.2.2 tls if (status != SLJIT_SUCCESS) 1993 1.2.2.2 tls goto fail; 1994 1.2.2.2 tls } else if (pc->k != 0) { 1995 1.2.2.2 tls if (pc->k & (pc->k - 1)) { 1996 1.2.2.4 jdolecek status = emit_moddiv(compiler, pc); 1997 1.2.2.2 tls } else { 1998 1.2.2.4 jdolecek status = emit_pow2_moddiv(compiler, pc); 1999 1.2.2.2 tls } 2000 1.2.2.2 tls if (status != SLJIT_SUCCESS) 2001 1.2.2.2 tls goto fail; 2002 1.2.2.2 tls } 2003 1.2.2.2 tls 2004 1.2.2.2 tls continue; 2005 1.2.2.2 tls 2006 1.2.2.2 tls case BPF_JMP: 2007 1.2.2.4 jdolecek op = BPF_OP(pc->code); 2008 1.2.2.4 jdolecek if (op == BPF_JA) { 2009 1.2.2.2 tls jt = jf = pc->k; 2010 1.2.2.2 tls } else { 2011 1.2.2.2 tls jt = pc->jt; 2012 1.2.2.2 tls jf = pc->jf; 2013 1.2.2.2 tls } 2014 1.2.2.2 tls 2015 1.2.2.2 tls negate = (jt == 0) ? 1 : 0; 2016 1.2.2.2 tls branching = (jt == jf) ? 0 : 1; 2017 1.2.2.3 tls jtf = insn_dat[i].u.jdata.jtf; 2018 1.2.2.2 tls 2019 1.2.2.2 tls if (branching) { 2020 1.2.2.4 jdolecek if (op != BPF_JSET) { 2021 1.2.2.4 jdolecek if (!jmp_to_cond(pc, negate, &cond)) 2022 1.2.2.4 jdolecek goto fail; 2023 1.2.2.2 tls jump = sljit_emit_cmp(compiler, 2024 1.2.2.4 jdolecek cond, BJ_AREG, 0, 2025 1.2.2.2 tls kx_to_reg(pc), kx_to_reg_arg(pc)); 2026 1.2.2.2 tls } else { 2027 1.2.2.2 tls status = sljit_emit_op2(compiler, 2028 1.2.2.2 tls SLJIT_AND, 2029 1.2.2.3 tls BJ_TMP1REG, 0, 2030 1.2.2.3 tls BJ_AREG, 0, 2031 1.2.2.2 tls kx_to_reg(pc), kx_to_reg_arg(pc)); 2032 1.2.2.2 tls if (status != SLJIT_SUCCESS) 2033 1.2.2.2 tls goto fail; 2034 1.2.2.2 tls 2035 1.2.2.4 jdolecek if (!jmp_to_cond(pc, negate, &cond)) 2036 1.2.2.4 jdolecek goto fail; 2037 1.2.2.2 tls jump = sljit_emit_cmp(compiler, 2038 1.2.2.4 jdolecek cond, BJ_TMP1REG, 0, SLJIT_IMM, 0); 2039 1.2.2.2 tls } 2040 1.2.2.2 tls 2041 1.2.2.2 tls if (jump == NULL) 2042 1.2.2.2 tls goto fail; 2043 1.2.2.2 tls 2044 1.2.2.3 tls BJ_ASSERT(jtf[negate].sjump == NULL); 2045 1.2.2.3 tls jtf[negate].sjump = jump; 2046 1.2.2.2 tls } 2047 1.2.2.2 tls 2048 1.2.2.2 tls if (!branching || (jt != 0 && jf != 0)) { 2049 1.2.2.2 tls jump = sljit_emit_jump(compiler, SLJIT_JUMP); 2050 1.2.2.2 tls if (jump == NULL) 2051 1.2.2.2 tls goto fail; 2052 1.2.2.2 tls 2053 1.2.2.3 tls BJ_ASSERT(jtf[branching].sjump == NULL); 2054 1.2.2.3 tls jtf[branching].sjump = jump; 2055 1.2.2.2 tls } 2056 1.2.2.2 tls 2057 1.2.2.2 tls continue; 2058 1.2.2.2 tls 2059 1.2.2.2 tls case BPF_RET: 2060 1.2.2.2 tls rval = BPF_RVAL(pc->code); 2061 1.2.2.2 tls if (rval == BPF_X) 2062 1.2.2.2 tls goto fail; 2063 1.2.2.2 tls 2064 1.2.2.2 tls /* BPF_RET+BPF_K accept k bytes */ 2065 1.2.2.2 tls if (rval == BPF_K) { 2066 1.2.2.3 tls status = sljit_emit_return(compiler, 2067 1.2.2.4 jdolecek SLJIT_MOV_U32, 2068 1.2.2.2 tls SLJIT_IMM, (uint32_t)pc->k); 2069 1.2.2.2 tls if (status != SLJIT_SUCCESS) 2070 1.2.2.2 tls goto fail; 2071 1.2.2.2 tls } 2072 1.2.2.2 tls 2073 1.2.2.2 tls /* BPF_RET+BPF_A accept A bytes */ 2074 1.2.2.2 tls if (rval == BPF_A) { 2075 1.2.2.3 tls status = sljit_emit_return(compiler, 2076 1.2.2.4 jdolecek SLJIT_MOV_U32, 2077 1.2.2.3 tls BJ_AREG, 0); 2078 1.2.2.2 tls if (status != SLJIT_SUCCESS) 2079 1.2.2.2 tls goto fail; 2080 1.2.2.2 tls } 2081 1.2.2.2 tls 2082 1.2.2.2 tls continue; 2083 1.2.2.2 tls 2084 1.2.2.2 tls case BPF_MISC: 2085 1.2.2.3 tls switch (BPF_MISCOP(pc->code)) { 2086 1.2.2.3 tls case BPF_TAX: 2087 1.2.2.2 tls status = sljit_emit_op1(compiler, 2088 1.2.2.4 jdolecek SLJIT_MOV_U32, 2089 1.2.2.3 tls BJ_XREG, 0, 2090 1.2.2.3 tls BJ_AREG, 0); 2091 1.2.2.2 tls if (status != SLJIT_SUCCESS) 2092 1.2.2.2 tls goto fail; 2093 1.2.2.2 tls 2094 1.2.2.2 tls continue; 2095 1.2.2.2 tls 2096 1.2.2.3 tls case BPF_TXA: 2097 1.2.2.2 tls status = sljit_emit_op1(compiler, 2098 1.2.2.2 tls SLJIT_MOV, 2099 1.2.2.3 tls BJ_AREG, 0, 2100 1.2.2.3 tls BJ_XREG, 0); 2101 1.2.2.3 tls if (status != SLJIT_SUCCESS) 2102 1.2.2.3 tls goto fail; 2103 1.2.2.3 tls 2104 1.2.2.3 tls continue; 2105 1.2.2.3 tls 2106 1.2.2.3 tls case BPF_COP: 2107 1.2.2.3 tls case BPF_COPX: 2108 1.2.2.3 tls if (bc == NULL || bc->copfuncs == NULL) 2109 1.2.2.3 tls goto fail; 2110 1.2.2.3 tls if (BPF_MISCOP(pc->code) == BPF_COP && 2111 1.2.2.3 tls (uint32_t)pc->k >= bc->nfuncs) { 2112 1.2.2.3 tls goto fail; 2113 1.2.2.3 tls } 2114 1.2.2.3 tls 2115 1.2.2.3 tls status = emit_cop(compiler, hints, bc, pc, 2116 1.2.2.3 tls &ret0, &ret0_size, &ret0_maxsize); 2117 1.2.2.2 tls if (status != SLJIT_SUCCESS) 2118 1.2.2.2 tls goto fail; 2119 1.2.2.2 tls 2120 1.2.2.2 tls continue; 2121 1.2.2.2 tls } 2122 1.2.2.2 tls 2123 1.2.2.2 tls goto fail; 2124 1.2.2.2 tls } /* switch */ 2125 1.2.2.2 tls } /* main loop */ 2126 1.2.2.2 tls 2127 1.2.2.3 tls BJ_ASSERT(ret0_size <= ret0_maxsize); 2128 1.2.2.2 tls 2129 1.2.2.3 tls if (ret0_size > 0) { 2130 1.2.2.2 tls label = sljit_emit_label(compiler); 2131 1.2.2.2 tls if (label == NULL) 2132 1.2.2.2 tls goto fail; 2133 1.2.2.3 tls for (i = 0; i < ret0_size; i++) 2134 1.2.2.3 tls sljit_set_label(ret0[i], label); 2135 1.2.2.2 tls } 2136 1.2.2.2 tls 2137 1.2.2.2 tls status = sljit_emit_return(compiler, 2138 1.2.2.4 jdolecek SLJIT_MOV_U32, 2139 1.2.2.3 tls SLJIT_IMM, 0); 2140 1.2.2.2 tls if (status != SLJIT_SUCCESS) 2141 1.2.2.2 tls goto fail; 2142 1.2.2.2 tls 2143 1.2.2.3 tls rv = true; 2144 1.2.2.3 tls 2145 1.2.2.3 tls fail: 2146 1.2.2.3 tls if (ret0 != NULL) 2147 1.2.2.3 tls BJ_FREE(ret0, ret0_maxsize * sizeof(ret0[0])); 2148 1.2.2.3 tls 2149 1.2.2.3 tls return rv; 2150 1.2.2.3 tls } 2151 1.2.2.3 tls 2152 1.2.2.3 tls bpfjit_func_t 2153 1.2.2.3 tls bpfjit_generate_code(const bpf_ctx_t *bc, 2154 1.2.2.3 tls const struct bpf_insn *insns, size_t insn_count) 2155 1.2.2.3 tls { 2156 1.2.2.3 tls void *rv; 2157 1.2.2.3 tls struct sljit_compiler *compiler; 2158 1.2.2.3 tls 2159 1.2.2.3 tls size_t i; 2160 1.2.2.3 tls int status; 2161 1.2.2.3 tls 2162 1.2.2.3 tls /* optimization related */ 2163 1.2.2.3 tls bpf_memword_init_t initmask; 2164 1.2.2.3 tls bpfjit_hint_t hints; 2165 1.2.2.3 tls 2166 1.2.2.3 tls /* memory store location for initial zero initialization */ 2167 1.2.2.4 jdolecek sljit_s32 mem_reg; 2168 1.2.2.3 tls sljit_sw mem_off; 2169 1.2.2.3 tls 2170 1.2.2.3 tls struct bpfjit_insn_data *insn_dat; 2171 1.2.2.3 tls 2172 1.2.2.3 tls const size_t extwords = GET_EXTWORDS(bc); 2173 1.2.2.3 tls const size_t memwords = GET_MEMWORDS(bc); 2174 1.2.2.3 tls const bpf_memword_init_t preinited = extwords ? bc->preinited : 0; 2175 1.2.2.3 tls 2176 1.2.2.3 tls rv = NULL; 2177 1.2.2.3 tls compiler = NULL; 2178 1.2.2.3 tls insn_dat = NULL; 2179 1.2.2.3 tls 2180 1.2.2.3 tls if (memwords > MAX_MEMWORDS) 2181 1.2.2.3 tls goto fail; 2182 1.2.2.3 tls 2183 1.2.2.3 tls if (insn_count == 0 || insn_count > SIZE_MAX / sizeof(insn_dat[0])) 2184 1.2.2.3 tls goto fail; 2185 1.2.2.3 tls 2186 1.2.2.3 tls insn_dat = BJ_ALLOC(insn_count * sizeof(insn_dat[0])); 2187 1.2.2.3 tls if (insn_dat == NULL) 2188 1.2.2.3 tls goto fail; 2189 1.2.2.3 tls 2190 1.2.2.3 tls if (!optimize(bc, insns, insn_dat, insn_count, &initmask, &hints)) 2191 1.2.2.3 tls goto fail; 2192 1.2.2.3 tls 2193 1.2.2.4 jdolecek compiler = sljit_create_compiler(NULL); 2194 1.2.2.3 tls if (compiler == NULL) 2195 1.2.2.3 tls goto fail; 2196 1.2.2.3 tls 2197 1.2.2.3 tls #if !defined(_KERNEL) && defined(SLJIT_VERBOSE) && SLJIT_VERBOSE 2198 1.2.2.3 tls sljit_compiler_verbose(compiler, stderr); 2199 1.2.2.3 tls #endif 2200 1.2.2.3 tls 2201 1.2.2.4 jdolecek status = sljit_emit_enter(compiler, 0, 2, nscratches(hints), 2202 1.2.2.4 jdolecek NSAVEDS, 0, 0, sizeof(struct bpfjit_stack)); 2203 1.2.2.3 tls if (status != SLJIT_SUCCESS) 2204 1.2.2.3 tls goto fail; 2205 1.2.2.3 tls 2206 1.2.2.3 tls if (hints & BJ_HINT_COP) { 2207 1.2.2.3 tls /* save ctx argument */ 2208 1.2.2.3 tls status = sljit_emit_op1(compiler, 2209 1.2.2.3 tls SLJIT_MOV_P, 2210 1.2.2.4 jdolecek SLJIT_MEM1(SLJIT_SP), 2211 1.2.2.3 tls offsetof(struct bpfjit_stack, ctx), 2212 1.2.2.3 tls BJ_CTX_ARG, 0); 2213 1.2.2.3 tls if (status != SLJIT_SUCCESS) 2214 1.2.2.2 tls goto fail; 2215 1.2.2.3 tls } 2216 1.2.2.2 tls 2217 1.2.2.3 tls if (extwords == 0) { 2218 1.2.2.4 jdolecek mem_reg = SLJIT_MEM1(SLJIT_SP); 2219 1.2.2.3 tls mem_off = offsetof(struct bpfjit_stack, mem); 2220 1.2.2.3 tls } else { 2221 1.2.2.3 tls /* copy "mem" argument from bpf_args to bpfjit_stack */ 2222 1.2.2.3 tls status = sljit_emit_op1(compiler, 2223 1.2.2.3 tls SLJIT_MOV_P, 2224 1.2.2.3 tls BJ_TMP1REG, 0, 2225 1.2.2.3 tls SLJIT_MEM1(BJ_ARGS), offsetof(struct bpf_args, mem)); 2226 1.2.2.3 tls if (status != SLJIT_SUCCESS) 2227 1.2.2.3 tls goto fail; 2228 1.2.2.3 tls 2229 1.2.2.3 tls status = sljit_emit_op1(compiler, 2230 1.2.2.3 tls SLJIT_MOV_P, 2231 1.2.2.4 jdolecek SLJIT_MEM1(SLJIT_SP), 2232 1.2.2.3 tls offsetof(struct bpfjit_stack, extmem), 2233 1.2.2.3 tls BJ_TMP1REG, 0); 2234 1.2.2.3 tls if (status != SLJIT_SUCCESS) 2235 1.2.2.3 tls goto fail; 2236 1.2.2.3 tls 2237 1.2.2.3 tls mem_reg = SLJIT_MEM1(BJ_TMP1REG); 2238 1.2.2.3 tls mem_off = 0; 2239 1.2.2.3 tls } 2240 1.2.2.3 tls 2241 1.2.2.3 tls /* 2242 1.2.2.3 tls * Exclude pre-initialised external memory words but keep 2243 1.2.2.3 tls * initialization statuses of A and X registers in case 2244 1.2.2.3 tls * bc->preinited wrongly sets those two bits. 2245 1.2.2.3 tls */ 2246 1.2.2.3 tls initmask &= ~preinited | BJ_INIT_ABIT | BJ_INIT_XBIT; 2247 1.2.2.2 tls 2248 1.2.2.3 tls #if defined(_KERNEL) 2249 1.2.2.3 tls /* bpf_filter() checks initialization of memwords. */ 2250 1.2.2.3 tls BJ_ASSERT((initmask & (BJ_INIT_MBIT(memwords) - 1)) == 0); 2251 1.2.2.3 tls #endif 2252 1.2.2.3 tls for (i = 0; i < memwords; i++) { 2253 1.2.2.3 tls if (initmask & BJ_INIT_MBIT(i)) { 2254 1.2.2.3 tls /* M[i] = 0; */ 2255 1.2.2.3 tls status = sljit_emit_op1(compiler, 2256 1.2.2.4 jdolecek SLJIT_MOV_U32, 2257 1.2.2.3 tls mem_reg, mem_off + i * sizeof(uint32_t), 2258 1.2.2.3 tls SLJIT_IMM, 0); 2259 1.2.2.3 tls if (status != SLJIT_SUCCESS) 2260 1.2.2.3 tls goto fail; 2261 1.2.2.3 tls } 2262 1.2.2.3 tls } 2263 1.2.2.3 tls 2264 1.2.2.3 tls if (initmask & BJ_INIT_ABIT) { 2265 1.2.2.3 tls /* A = 0; */ 2266 1.2.2.2 tls status = sljit_emit_op1(compiler, 2267 1.2.2.2 tls SLJIT_MOV, 2268 1.2.2.3 tls BJ_AREG, 0, 2269 1.2.2.2 tls SLJIT_IMM, 0); 2270 1.2.2.2 tls if (status != SLJIT_SUCCESS) 2271 1.2.2.2 tls goto fail; 2272 1.2.2.3 tls } 2273 1.2.2.2 tls 2274 1.2.2.3 tls if (initmask & BJ_INIT_XBIT) { 2275 1.2.2.3 tls /* X = 0; */ 2276 1.2.2.3 tls status = sljit_emit_op1(compiler, 2277 1.2.2.3 tls SLJIT_MOV, 2278 1.2.2.3 tls BJ_XREG, 0, 2279 1.2.2.3 tls SLJIT_IMM, 0); 2280 1.2.2.2 tls if (status != SLJIT_SUCCESS) 2281 1.2.2.2 tls goto fail; 2282 1.2.2.2 tls } 2283 1.2.2.2 tls 2284 1.2.2.3 tls status = load_buf_buflen(compiler); 2285 1.2.2.3 tls if (status != SLJIT_SUCCESS) 2286 1.2.2.3 tls goto fail; 2287 1.2.2.3 tls 2288 1.2.2.3 tls if (!generate_insn_code(compiler, hints, 2289 1.2.2.3 tls bc, insns, insn_dat, insn_count)) { 2290 1.2.2.3 tls goto fail; 2291 1.2.2.3 tls } 2292 1.2.2.3 tls 2293 1.2.2.2 tls rv = sljit_generate_code(compiler); 2294 1.2.2.2 tls 2295 1.2.2.2 tls fail: 2296 1.2.2.2 tls if (compiler != NULL) 2297 1.2.2.2 tls sljit_free_compiler(compiler); 2298 1.2.2.2 tls 2299 1.2.2.2 tls if (insn_dat != NULL) 2300 1.2.2.3 tls BJ_FREE(insn_dat, insn_count * sizeof(insn_dat[0])); 2301 1.2.2.2 tls 2302 1.2.2.3 tls return (bpfjit_func_t)rv; 2303 1.2.2.2 tls } 2304 1.2.2.2 tls 2305 1.2.2.2 tls void 2306 1.2.2.3 tls bpfjit_free_code(bpfjit_func_t code) 2307 1.2.2.2 tls { 2308 1.2.2.2 tls 2309 1.2.2.2 tls sljit_free_code((void *)code); 2310 1.2.2.2 tls } 2311
Indexes created Thu Oct 02 10:09:58 GMT 2025