bpfjit.c revision 1.30
1 /* $NetBSD: bpfjit.c,v 1.30 2014/07/22 08:29:51 alnsn Exp $ */ 2 3 /*- 4 * Copyright (c) 2011-2014 Alexander Nasonov. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 21 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 22 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 23 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 24 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 25 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 26 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 27 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 28 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 #ifdef _KERNEL 34 __KERNEL_RCSID(0, "$NetBSD: bpfjit.c,v 1.30 2014/07/22 08:29:51 alnsn Exp $"); 35 #else 36 __RCSID("$NetBSD: bpfjit.c,v 1.30 2014/07/22 08:29:51 alnsn Exp $"); 37 #endif 38 39 #include <sys/types.h> 40 #include <sys/queue.h> 41 42 #ifndef _KERNEL 43 #include <assert.h> 44 #define BJ_ASSERT(c) assert(c) 45 #else 46 #define BJ_ASSERT(c) KASSERT(c) 47 #endif 48 49 #ifndef _KERNEL 50 #include <stdlib.h> 51 #define BJ_ALLOC(sz) malloc(sz) 52 #define BJ_FREE(p, sz) free(p) 53 #else 54 #include <sys/kmem.h> 55 #define BJ_ALLOC(sz) kmem_alloc(sz, KM_SLEEP) 56 #define BJ_FREE(p, sz) kmem_free(p, sz) 57 #endif 58 59 #ifndef _KERNEL 60 #include <limits.h> 61 #include <stdbool.h> 62 #include <stddef.h> 63 #include <stdint.h> 64 #else 65 #include <sys/atomic.h> 66 #include <sys/module.h> 67 #endif 68 69 #define __BPF_PRIVATE 70 #include <net/bpf.h> 71 #include <net/bpfjit.h> 72 #include <sljitLir.h> 73 74 #if !defined(_KERNEL) && defined(SLJIT_VERBOSE) && SLJIT_VERBOSE 75 #include <stdio.h> /* for stderr */ 76 #endif 77 78 /* 79 * Arguments of generated bpfjit_func_t. 80 * The first argument is reassigned upon entry 81 * to a more frequently used buf argument. 82 */ 83 #define BJ_CTX_ARG SLJIT_SAVED_REG1 84 #define BJ_ARGS SLJIT_SAVED_REG2 85 86 /* 87 * Permanent register assignments. 88 */ 89 #define BJ_BUF SLJIT_SAVED_REG1 90 //#define BJ_ARGS SLJIT_SAVED_REG2 91 #define BJ_BUFLEN SLJIT_SAVED_REG3 92 #define BJ_XREG SLJIT_SAVED_EREG1 93 #define BJ_ASAVE SLJIT_SAVED_EREG2 94 #define BJ_AREG SLJIT_SCRATCH_REG1 95 #define BJ_TMP1REG SLJIT_SCRATCH_REG2 96 #define BJ_TMP2REG SLJIT_SCRATCH_REG3 97 #define BJ_TMP3REG SLJIT_TEMPORARY_EREG1 98 99 #ifdef _KERNEL 100 #define MAX_MEMWORDS BPF_MAX_MEMWORDS 101 #else 102 #define MAX_MEMWORDS BPF_MEMWORDS 103 #endif 104 105 #define BJ_INIT_NOBITS ((bpf_memword_init_t)0) 106 #define BJ_INIT_MBIT(k) BPF_MEMWORD_INIT(k) 107 #define BJ_INIT_ABIT BJ_INIT_MBIT(MAX_MEMWORDS) 108 #define BJ_INIT_XBIT BJ_INIT_MBIT(MAX_MEMWORDS + 1) 109 110 /* 111 * Get a number of memwords and external memwords from a bpf_ctx object. 112 */ 113 #define GET_EXTWORDS(bc) ((bc) ? (bc)->extwords : 0) 114 #define GET_MEMWORDS(bc) (GET_EXTWORDS(bc) ? GET_EXTWORDS(bc) : BPF_MEMWORDS) 115 116 /* 117 * Optimization hints. 118 */ 119 typedef unsigned int bpfjit_hint_t; 120 #define BJ_HINT_ABS 0x01 /* packet read at absolute offset */ 121 #define BJ_HINT_IND 0x02 /* packet read at variable offset */ 122 #define BJ_HINT_MSH 0x04 /* BPF_MSH instruction */ 123 #define BJ_HINT_COP 0x08 /* BPF_COP or BPF_COPX instruction */ 124 #define BJ_HINT_COPX 0x10 /* BPF_COPX instruction */ 125 #define BJ_HINT_XREG 0x20 /* BJ_XREG is needed */ 126 #define BJ_HINT_LDX 0x40 /* BPF_LDX instruction */ 127 #define BJ_HINT_PKT (BJ_HINT_ABS|BJ_HINT_IND|BJ_HINT_MSH) 128 129 /* 130 * Datatype for Array Bounds Check Elimination (ABC) pass. 131 */ 132 typedef uint64_t bpfjit_abc_length_t; 133 #define MAX_ABC_LENGTH (UINT32_MAX + UINT64_C(4)) /* max. width is 4 */ 134 135 struct bpfjit_stack 136 { 137 bpf_ctx_t *ctx; 138 uint32_t *extmem; /* pointer to external memory store */ 139 #ifdef _KERNEL 140 int err; /* 3rd argument for m_xword/m_xhalf/m_xbyte function call */ 141 #endif 142 uint32_t mem[BPF_MEMWORDS]; /* internal memory store */ 143 }; 144 145 /* 146 * Data for BPF_JMP instruction. 147 * Forward declaration for struct bpfjit_jump. 148 */ 149 struct bpfjit_jump_data; 150 151 /* 152 * Node of bjumps list. 153 */ 154 struct bpfjit_jump { 155 struct sljit_jump *sjump; 156 SLIST_ENTRY(bpfjit_jump) entries; 157 struct bpfjit_jump_data *jdata; 158 }; 159 160 /* 161 * Data for BPF_JMP instruction. 162 */ 163 struct bpfjit_jump_data { 164 /* 165 * These entries make up bjumps list: 166 * jtf[0] - when coming from jt path, 167 * jtf[1] - when coming from jf path. 168 */ 169 struct bpfjit_jump jtf[2]; 170 /* 171 * Length calculated by Array Bounds Check Elimination (ABC) pass. 172 */ 173 bpfjit_abc_length_t abc_length; 174 /* 175 * Length checked by the last out-of-bounds check. 176 */ 177 bpfjit_abc_length_t checked_length; 178 }; 179 180 /* 181 * Data for "read from packet" instructions. 182 * See also read_pkt_insn() function below. 183 */ 184 struct bpfjit_read_pkt_data { 185 /* 186 * Length calculated by Array Bounds Check Elimination (ABC) pass. 187 */ 188 bpfjit_abc_length_t abc_length; 189 /* 190 * If positive, emit "if (buflen < check_length) return 0" 191 * out-of-bounds check. 192 * Values greater than UINT32_MAX generate unconditional "return 0". 193 */ 194 bpfjit_abc_length_t check_length; 195 }; 196 197 /* 198 * Additional (optimization-related) data for bpf_insn. 199 */ 200 struct bpfjit_insn_data { 201 /* List of jumps to this insn. */ 202 SLIST_HEAD(, bpfjit_jump) bjumps; 203 204 union { 205 struct bpfjit_jump_data jdata; 206 struct bpfjit_read_pkt_data rdata; 207 } u; 208 209 bpf_memword_init_t invalid; 210 bool unreachable; 211 }; 212 213 #ifdef _KERNEL 214 215 uint32_t m_xword(const struct mbuf *, uint32_t, int *); 216 uint32_t m_xhalf(const struct mbuf *, uint32_t, int *); 217 uint32_t m_xbyte(const struct mbuf *, uint32_t, int *); 218 219 MODULE(MODULE_CLASS_MISC, bpfjit, "sljit") 220 221 static int 222 bpfjit_modcmd(modcmd_t cmd, void *arg) 223 { 224 225 switch (cmd) { 226 case MODULE_CMD_INIT: 227 bpfjit_module_ops.bj_free_code = &bpfjit_free_code; 228 membar_producer(); 229 bpfjit_module_ops.bj_generate_code = &bpfjit_generate_code; 230 membar_producer(); 231 return 0; 232 233 case MODULE_CMD_FINI: 234 return EOPNOTSUPP; 235 236 default: 237 return ENOTTY; 238 } 239 } 240 #endif 241 242 /* 243 * Return a number of scratch registers to pass 244 * to sljit_emit_enter() function. 245 */ 246 static sljit_si 247 nscratches(bpfjit_hint_t hints) 248 { 249 sljit_si rv = 2; 250 251 #ifdef _KERNEL 252 if (hints & BJ_HINT_PKT) 253 rv = 3; /* xcall with three arguments */ 254 #endif 255 256 if (hints & BJ_HINT_IND) 257 rv = 3; /* uses BJ_TMP2REG */ 258 259 if (hints & BJ_HINT_COP) 260 rv = 3; /* calls copfunc with three arguments */ 261 262 if (hints & BJ_HINT_COPX) 263 rv = 4; /* uses BJ_TMP3REG */ 264 265 return rv; 266 } 267 268 /* 269 * Return a number of saved registers to pass 270 * to sljit_emit_enter() function. 271 */ 272 static sljit_si 273 nsaveds(bpfjit_hint_t hints) 274 { 275 sljit_si rv = 3; 276 277 if (hints & BJ_HINT_XREG) 278 rv = 4; /* uses BJ_XREG */ 279 280 #ifdef _KERNEL 281 if (hints & BJ_HINT_LDX) 282 rv = 5; /* uses BJ_ASAVE */ 283 #endif 284 285 return rv; 286 } 287 288 static uint32_t 289 read_width(const struct bpf_insn *pc) 290 { 291 292 switch (BPF_SIZE(pc->code)) { 293 case BPF_W: 294 return 4; 295 case BPF_H: 296 return 2; 297 case BPF_B: 298 return 1; 299 default: 300 BJ_ASSERT(false); 301 return 0; 302 } 303 } 304 305 /* 306 * Copy buf and buflen members of bpf_args from BJ_ARGS 307 * pointer to BJ_BUF and BJ_BUFLEN registers. 308 */ 309 static int 310 load_buf_buflen(struct sljit_compiler *compiler) 311 { 312 int status; 313 314 status = sljit_emit_op1(compiler, 315 SLJIT_MOV_P, 316 BJ_BUF, 0, 317 SLJIT_MEM1(BJ_ARGS), 318 offsetof(struct bpf_args, pkt)); 319 if (status != SLJIT_SUCCESS) 320 return status; 321 322 status = sljit_emit_op1(compiler, 323 SLJIT_MOV, /* size_t source */ 324 BJ_BUFLEN, 0, 325 SLJIT_MEM1(BJ_ARGS), 326 offsetof(struct bpf_args, buflen)); 327 328 return status; 329 } 330 331 static bool 332 grow_jumps(struct sljit_jump ***jumps, size_t *size) 333 { 334 struct sljit_jump **newptr; 335 const size_t elemsz = sizeof(struct sljit_jump *); 336 size_t old_size = *size; 337 size_t new_size = 2 * old_size; 338 339 if (new_size < old_size || new_size > SIZE_MAX / elemsz) 340 return false; 341 342 newptr = BJ_ALLOC(new_size * elemsz); 343 if (newptr == NULL) 344 return false; 345 346 memcpy(newptr, *jumps, old_size * elemsz); 347 BJ_FREE(*jumps, old_size * elemsz); 348 349 *jumps = newptr; 350 *size = new_size; 351 return true; 352 } 353 354 static bool 355 append_jump(struct sljit_jump *jump, struct sljit_jump ***jumps, 356 size_t *size, size_t *max_size) 357 { 358 if (*size == *max_size && !grow_jumps(jumps, max_size)) 359 return false; 360 361 (*jumps)[(*size)++] = jump; 362 return true; 363 } 364 365 /* 366 * Emit code for BPF_LD+BPF_B+BPF_ABS A <- P[k:1]. 367 */ 368 static int 369 emit_read8(struct sljit_compiler *compiler, sljit_si src, uint32_t k) 370 { 371 372 return sljit_emit_op1(compiler, 373 SLJIT_MOV_UB, 374 BJ_AREG, 0, 375 SLJIT_MEM1(src), k); 376 } 377 378 /* 379 * Emit code for BPF_LD+BPF_H+BPF_ABS A <- P[k:2]. 380 */ 381 static int 382 emit_read16(struct sljit_compiler *compiler, sljit_si src, uint32_t k) 383 { 384 int status; 385 386 BJ_ASSERT(k <= UINT32_MAX - 1); 387 388 /* A = buf[k]; */ 389 status = sljit_emit_op1(compiler, 390 SLJIT_MOV_UB, 391 BJ_AREG, 0, 392 SLJIT_MEM1(src), k); 393 if (status != SLJIT_SUCCESS) 394 return status; 395 396 /* tmp1 = buf[k+1]; */ 397 status = sljit_emit_op1(compiler, 398 SLJIT_MOV_UB, 399 BJ_TMP1REG, 0, 400 SLJIT_MEM1(src), k+1); 401 if (status != SLJIT_SUCCESS) 402 return status; 403 404 /* A = A << 8; */ 405 status = sljit_emit_op2(compiler, 406 SLJIT_SHL, 407 BJ_AREG, 0, 408 BJ_AREG, 0, 409 SLJIT_IMM, 8); 410 if (status != SLJIT_SUCCESS) 411 return status; 412 413 /* A = A + tmp1; */ 414 status = sljit_emit_op2(compiler, 415 SLJIT_ADD, 416 BJ_AREG, 0, 417 BJ_AREG, 0, 418 BJ_TMP1REG, 0); 419 return status; 420 } 421 422 /* 423 * Emit code for BPF_LD+BPF_W+BPF_ABS A <- P[k:4]. 424 */ 425 static int 426 emit_read32(struct sljit_compiler *compiler, sljit_si src, uint32_t k) 427 { 428 int status; 429 430 BJ_ASSERT(k <= UINT32_MAX - 3); 431 432 /* A = buf[k]; */ 433 status = sljit_emit_op1(compiler, 434 SLJIT_MOV_UB, 435 BJ_AREG, 0, 436 SLJIT_MEM1(src), k); 437 if (status != SLJIT_SUCCESS) 438 return status; 439 440 /* tmp1 = buf[k+1]; */ 441 status = sljit_emit_op1(compiler, 442 SLJIT_MOV_UB, 443 BJ_TMP1REG, 0, 444 SLJIT_MEM1(src), k+1); 445 if (status != SLJIT_SUCCESS) 446 return status; 447 448 /* A = A << 8; */ 449 status = sljit_emit_op2(compiler, 450 SLJIT_SHL, 451 BJ_AREG, 0, 452 BJ_AREG, 0, 453 SLJIT_IMM, 8); 454 if (status != SLJIT_SUCCESS) 455 return status; 456 457 /* A = A + tmp1; */ 458 status = sljit_emit_op2(compiler, 459 SLJIT_ADD, 460 BJ_AREG, 0, 461 BJ_AREG, 0, 462 BJ_TMP1REG, 0); 463 if (status != SLJIT_SUCCESS) 464 return status; 465 466 /* tmp1 = buf[k+2]; */ 467 status = sljit_emit_op1(compiler, 468 SLJIT_MOV_UB, 469 BJ_TMP1REG, 0, 470 SLJIT_MEM1(src), k+2); 471 if (status != SLJIT_SUCCESS) 472 return status; 473 474 /* A = A << 8; */ 475 status = sljit_emit_op2(compiler, 476 SLJIT_SHL, 477 BJ_AREG, 0, 478 BJ_AREG, 0, 479 SLJIT_IMM, 8); 480 if (status != SLJIT_SUCCESS) 481 return status; 482 483 /* A = A + tmp1; */ 484 status = sljit_emit_op2(compiler, 485 SLJIT_ADD, 486 BJ_AREG, 0, 487 BJ_AREG, 0, 488 BJ_TMP1REG, 0); 489 if (status != SLJIT_SUCCESS) 490 return status; 491 492 /* tmp1 = buf[k+3]; */ 493 status = sljit_emit_op1(compiler, 494 SLJIT_MOV_UB, 495 BJ_TMP1REG, 0, 496 SLJIT_MEM1(src), k+3); 497 if (status != SLJIT_SUCCESS) 498 return status; 499 500 /* A = A << 8; */ 501 status = sljit_emit_op2(compiler, 502 SLJIT_SHL, 503 BJ_AREG, 0, 504 BJ_AREG, 0, 505 SLJIT_IMM, 8); 506 if (status != SLJIT_SUCCESS) 507 return status; 508 509 /* A = A + tmp1; */ 510 status = sljit_emit_op2(compiler, 511 SLJIT_ADD, 512 BJ_AREG, 0, 513 BJ_AREG, 0, 514 BJ_TMP1REG, 0); 515 return status; 516 } 517 518 #ifdef _KERNEL 519 /* 520 * Emit code for m_xword/m_xhalf/m_xbyte call. 521 * 522 * @pc BPF_LD+BPF_W+BPF_ABS A <- P[k:4] 523 * BPF_LD+BPF_H+BPF_ABS A <- P[k:2] 524 * BPF_LD+BPF_B+BPF_ABS A <- P[k:1] 525 * BPF_LD+BPF_W+BPF_IND A <- P[X+k:4] 526 * BPF_LD+BPF_H+BPF_IND A <- P[X+k:2] 527 * BPF_LD+BPF_B+BPF_IND A <- P[X+k:1] 528 * BPF_LDX+BPF_B+BPF_MSH X <- 4*(P[k:1]&0xf) 529 */ 530 static int 531 emit_xcall(struct sljit_compiler *compiler, const struct bpf_insn *pc, 532 int dst, struct sljit_jump ***ret0, size_t *ret0_size, size_t *ret0_maxsize, 533 uint32_t (*fn)(const struct mbuf *, uint32_t, int *)) 534 { 535 struct sljit_jump *jump; 536 int status; 537 538 BJ_ASSERT(dst != BJ_ASAVE); 539 540 if (BPF_CLASS(pc->code) == BPF_LDX) { 541 /* save A */ 542 status = sljit_emit_op1(compiler, 543 SLJIT_MOV, 544 BJ_ASAVE, 0, 545 BJ_AREG, 0); 546 if (status != SLJIT_SUCCESS) 547 return status; 548 } 549 550 /* 551 * Prepare registers for fn(mbuf, k, &err) call. 552 */ 553 status = sljit_emit_op1(compiler, 554 SLJIT_MOV, 555 SLJIT_SCRATCH_REG1, 0, 556 BJ_BUF, 0); 557 if (status != SLJIT_SUCCESS) 558 return status; 559 560 if (BPF_CLASS(pc->code) == BPF_LD && BPF_MODE(pc->code) == BPF_IND) { 561 /* if (X > UINT32_MAX - pc->k) return 0; */ 562 jump = sljit_emit_cmp(compiler, 563 SLJIT_C_GREATER, 564 BJ_XREG, 0, 565 SLJIT_IMM, UINT32_MAX - pc->k); 566 if (jump == NULL) 567 return SLJIT_ERR_ALLOC_FAILED; 568 if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 569 return SLJIT_ERR_ALLOC_FAILED; 570 571 /* k = X + pc->k; */ 572 status = sljit_emit_op2(compiler, 573 SLJIT_ADD, 574 SLJIT_SCRATCH_REG2, 0, 575 BJ_XREG, 0, 576 SLJIT_IMM, (uint32_t)pc->k); 577 if (status != SLJIT_SUCCESS) 578 return status; 579 } else { 580 /* k = pc->k */ 581 status = sljit_emit_op1(compiler, 582 SLJIT_MOV, 583 SLJIT_SCRATCH_REG2, 0, 584 SLJIT_IMM, (uint32_t)pc->k); 585 if (status != SLJIT_SUCCESS) 586 return status; 587 } 588 589 /* 590 * The third argument of fn is an address on stack. 591 */ 592 status = sljit_get_local_base(compiler, 593 SLJIT_SCRATCH_REG3, 0, 594 offsetof(struct bpfjit_stack, err)); 595 if (status != SLJIT_SUCCESS) 596 return status; 597 598 /* fn(buf, k, &err); */ 599 status = sljit_emit_ijump(compiler, 600 SLJIT_CALL3, 601 SLJIT_IMM, SLJIT_FUNC_OFFSET(fn)); 602 if (status != SLJIT_SUCCESS) 603 return status; 604 605 if (dst != SLJIT_RETURN_REG) { 606 /* move return value to dst */ 607 status = sljit_emit_op1(compiler, 608 SLJIT_MOV, 609 dst, 0, 610 SLJIT_RETURN_REG, 0); 611 if (status != SLJIT_SUCCESS) 612 return status; 613 } 614 615 /* if (*err != 0) return 0; */ 616 jump = sljit_emit_cmp(compiler, 617 SLJIT_C_NOT_EQUAL|SLJIT_INT_OP, 618 SLJIT_MEM1(SLJIT_LOCALS_REG), 619 offsetof(struct bpfjit_stack, err), 620 SLJIT_IMM, 0); 621 if (jump == NULL) 622 return SLJIT_ERR_ALLOC_FAILED; 623 624 if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 625 return SLJIT_ERR_ALLOC_FAILED; 626 627 if (BPF_CLASS(pc->code) == BPF_LDX) { 628 /* restore A */ 629 status = sljit_emit_op1(compiler, 630 SLJIT_MOV, 631 BJ_AREG, 0, 632 BJ_ASAVE, 0); 633 if (status != SLJIT_SUCCESS) 634 return status; 635 } 636 637 return SLJIT_SUCCESS; 638 } 639 #endif 640 641 /* 642 * Emit code for BPF_COP and BPF_COPX instructions. 643 */ 644 static int 645 emit_cop(struct sljit_compiler *compiler, 646 const bpf_ctx_t *bc, const struct bpf_insn *pc, 647 struct sljit_jump ***ret0, size_t *ret0_size, size_t *ret0_maxsize) 648 { 649 #if BJ_TMP3REG == SLJIT_SCRATCH_REG1 || \ 650 BJ_TMP3REG == SLJIT_SCRATCH_REG2 || \ 651 BJ_TMP3REG == SLJIT_SCRATCH_REG3 652 #error "Not supported assignment of registers." 653 #endif 654 655 struct sljit_jump *jump; 656 sljit_si call_reg; 657 sljit_sw call_off; 658 int status; 659 660 BJ_ASSERT(bc != NULL && bc->copfuncs != NULL); 661 662 if (BPF_MISCOP(pc->code) == BPF_COP) { 663 call_reg = SLJIT_IMM; 664 call_off = SLJIT_FUNC_OFFSET(bc->copfuncs[pc->k]); 665 } else { 666 /* if (X >= bc->nfuncs) return 0; */ 667 jump = sljit_emit_cmp(compiler, 668 SLJIT_C_GREATER_EQUAL, 669 BJ_XREG, 0, 670 SLJIT_IMM, bc->nfuncs); 671 if (jump == NULL) 672 return SLJIT_ERR_ALLOC_FAILED; 673 if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 674 return SLJIT_ERR_ALLOC_FAILED; 675 676 /* tmp1 = ctx; */ 677 status = sljit_emit_op1(compiler, 678 SLJIT_MOV_P, 679 BJ_TMP1REG, 0, 680 SLJIT_MEM1(SLJIT_LOCALS_REG), 681 offsetof(struct bpfjit_stack, ctx)); 682 if (status != SLJIT_SUCCESS) 683 return status; 684 685 /* tmp1 = ctx->copfuncs; */ 686 status = sljit_emit_op1(compiler, 687 SLJIT_MOV_P, 688 BJ_TMP1REG, 0, 689 SLJIT_MEM1(BJ_TMP1REG), 690 offsetof(struct bpf_ctx, copfuncs)); 691 if (status != SLJIT_SUCCESS) 692 return status; 693 694 /* tmp2 = X; */ 695 status = sljit_emit_op1(compiler, 696 SLJIT_MOV, 697 BJ_TMP2REG, 0, 698 BJ_XREG, 0); 699 if (status != SLJIT_SUCCESS) 700 return status; 701 702 /* tmp3 = ctx->copfuncs[tmp2]; */ 703 call_reg = BJ_TMP3REG; 704 call_off = 0; 705 status = sljit_emit_op1(compiler, 706 SLJIT_MOV_P, 707 call_reg, call_off, 708 SLJIT_MEM2(BJ_TMP1REG, BJ_TMP2REG), 709 SLJIT_WORD_SHIFT); 710 if (status != SLJIT_SUCCESS) 711 return status; 712 } 713 714 /* 715 * Copy bpf_copfunc_t arguments to registers. 716 */ 717 #if BJ_AREG != SLJIT_SCRATCH_REG3 718 status = sljit_emit_op1(compiler, 719 SLJIT_MOV_UI, 720 SLJIT_SCRATCH_REG3, 0, 721 BJ_AREG, 0); 722 if (status != SLJIT_SUCCESS) 723 return status; 724 #endif 725 726 status = sljit_emit_op1(compiler, 727 SLJIT_MOV_P, 728 SLJIT_SCRATCH_REG1, 0, 729 SLJIT_MEM1(SLJIT_LOCALS_REG), 730 offsetof(struct bpfjit_stack, ctx)); 731 if (status != SLJIT_SUCCESS) 732 return status; 733 734 status = sljit_emit_op1(compiler, 735 SLJIT_MOV_P, 736 SLJIT_SCRATCH_REG2, 0, 737 BJ_ARGS, 0); 738 if (status != SLJIT_SUCCESS) 739 return status; 740 741 status = sljit_emit_ijump(compiler, 742 SLJIT_CALL3, call_reg, call_off); 743 if (status != SLJIT_SUCCESS) 744 return status; 745 746 #if BJ_AREG != SLJIT_RETURN_REG 747 status = sljit_emit_op1(compiler, 748 SLJIT_MOV, 749 BJ_AREG, 0, 750 SLJIT_RETURN_REG, 0); 751 if (status != SLJIT_SUCCESS) 752 return status; 753 #endif 754 755 return SLJIT_SUCCESS; 756 } 757 758 /* 759 * Generate code for 760 * BPF_LD+BPF_W+BPF_ABS A <- P[k:4] 761 * BPF_LD+BPF_H+BPF_ABS A <- P[k:2] 762 * BPF_LD+BPF_B+BPF_ABS A <- P[k:1] 763 * BPF_LD+BPF_W+BPF_IND A <- P[X+k:4] 764 * BPF_LD+BPF_H+BPF_IND A <- P[X+k:2] 765 * BPF_LD+BPF_B+BPF_IND A <- P[X+k:1] 766 */ 767 static int 768 emit_pkt_read(struct sljit_compiler *compiler, 769 const struct bpf_insn *pc, struct sljit_jump *to_mchain_jump, 770 struct sljit_jump ***ret0, size_t *ret0_size, size_t *ret0_maxsize) 771 { 772 int status = SLJIT_ERR_ALLOC_FAILED; 773 uint32_t width; 774 sljit_si ld_reg; 775 struct sljit_jump *jump; 776 #ifdef _KERNEL 777 struct sljit_label *label; 778 struct sljit_jump *over_mchain_jump; 779 const bool check_zero_buflen = (to_mchain_jump != NULL); 780 #endif 781 const uint32_t k = pc->k; 782 783 #ifdef _KERNEL 784 if (to_mchain_jump == NULL) { 785 to_mchain_jump = sljit_emit_cmp(compiler, 786 SLJIT_C_EQUAL, 787 BJ_BUFLEN, 0, 788 SLJIT_IMM, 0); 789 if (to_mchain_jump == NULL) 790 return SLJIT_ERR_ALLOC_FAILED; 791 } 792 #endif 793 794 ld_reg = BJ_BUF; 795 width = read_width(pc); 796 797 if (BPF_MODE(pc->code) == BPF_IND) { 798 /* tmp1 = buflen - (pc->k + width); */ 799 status = sljit_emit_op2(compiler, 800 SLJIT_SUB, 801 BJ_TMP1REG, 0, 802 BJ_BUFLEN, 0, 803 SLJIT_IMM, k + width); 804 if (status != SLJIT_SUCCESS) 805 return status; 806 807 /* ld_reg = buf + X; */ 808 ld_reg = BJ_TMP2REG; 809 status = sljit_emit_op2(compiler, 810 SLJIT_ADD, 811 ld_reg, 0, 812 BJ_BUF, 0, 813 BJ_XREG, 0); 814 if (status != SLJIT_SUCCESS) 815 return status; 816 817 /* if (tmp1 < X) return 0; */ 818 jump = sljit_emit_cmp(compiler, 819 SLJIT_C_LESS, 820 BJ_TMP1REG, 0, 821 BJ_XREG, 0); 822 if (jump == NULL) 823 return SLJIT_ERR_ALLOC_FAILED; 824 if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 825 return SLJIT_ERR_ALLOC_FAILED; 826 } 827 828 switch (width) { 829 case 4: 830 status = emit_read32(compiler, ld_reg, k); 831 break; 832 case 2: 833 status = emit_read16(compiler, ld_reg, k); 834 break; 835 case 1: 836 status = emit_read8(compiler, ld_reg, k); 837 break; 838 } 839 840 if (status != SLJIT_SUCCESS) 841 return status; 842 843 #ifdef _KERNEL 844 over_mchain_jump = sljit_emit_jump(compiler, SLJIT_JUMP); 845 if (over_mchain_jump == NULL) 846 return SLJIT_ERR_ALLOC_FAILED; 847 848 /* entry point to mchain handler */ 849 label = sljit_emit_label(compiler); 850 if (label == NULL) 851 return SLJIT_ERR_ALLOC_FAILED; 852 sljit_set_label(to_mchain_jump, label); 853 854 if (check_zero_buflen) { 855 /* if (buflen != 0) return 0; */ 856 jump = sljit_emit_cmp(compiler, 857 SLJIT_C_NOT_EQUAL, 858 BJ_BUFLEN, 0, 859 SLJIT_IMM, 0); 860 if (jump == NULL) 861 return SLJIT_ERR_ALLOC_FAILED; 862 if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 863 return SLJIT_ERR_ALLOC_FAILED; 864 } 865 866 switch (width) { 867 case 4: 868 status = emit_xcall(compiler, pc, BJ_AREG, 869 ret0, ret0_size, ret0_maxsize, &m_xword); 870 break; 871 case 2: 872 status = emit_xcall(compiler, pc, BJ_AREG, 873 ret0, ret0_size, ret0_maxsize, &m_xhalf); 874 break; 875 case 1: 876 status = emit_xcall(compiler, pc, BJ_AREG, 877 ret0, ret0_size, ret0_maxsize, &m_xbyte); 878 break; 879 } 880 881 if (status != SLJIT_SUCCESS) 882 return status; 883 884 label = sljit_emit_label(compiler); 885 if (label == NULL) 886 return SLJIT_ERR_ALLOC_FAILED; 887 sljit_set_label(over_mchain_jump, label); 888 #endif 889 890 return SLJIT_SUCCESS; 891 } 892 893 static int 894 emit_memload(struct sljit_compiler *compiler, 895 sljit_si dst, uint32_t k, size_t extwords) 896 { 897 int status; 898 sljit_si src; 899 sljit_sw srcw; 900 901 srcw = k * sizeof(uint32_t); 902 903 if (extwords == 0) { 904 src = SLJIT_MEM1(SLJIT_LOCALS_REG); 905 srcw += offsetof(struct bpfjit_stack, mem); 906 } else { 907 /* copy extmem pointer to the tmp1 register */ 908 status = sljit_emit_op1(compiler, 909 SLJIT_MOV_P, 910 BJ_TMP1REG, 0, 911 SLJIT_MEM1(SLJIT_LOCALS_REG), 912 offsetof(struct bpfjit_stack, extmem)); 913 if (status != SLJIT_SUCCESS) 914 return status; 915 src = SLJIT_MEM1(BJ_TMP1REG); 916 } 917 918 return sljit_emit_op1(compiler, SLJIT_MOV_UI, dst, 0, src, srcw); 919 } 920 921 static int 922 emit_memstore(struct sljit_compiler *compiler, 923 sljit_si src, uint32_t k, size_t extwords) 924 { 925 int status; 926 sljit_si dst; 927 sljit_sw dstw; 928 929 dstw = k * sizeof(uint32_t); 930 931 if (extwords == 0) { 932 dst = SLJIT_MEM1(SLJIT_LOCALS_REG); 933 dstw += offsetof(struct bpfjit_stack, mem); 934 } else { 935 /* copy extmem pointer to the tmp1 register */ 936 status = sljit_emit_op1(compiler, 937 SLJIT_MOV_P, 938 BJ_TMP1REG, 0, 939 SLJIT_MEM1(SLJIT_LOCALS_REG), 940 offsetof(struct bpfjit_stack, extmem)); 941 if (status != SLJIT_SUCCESS) 942 return status; 943 dst = SLJIT_MEM1(BJ_TMP1REG); 944 } 945 946 return sljit_emit_op1(compiler, SLJIT_MOV_UI, dst, dstw, src, 0); 947 } 948 949 /* 950 * Emit code for BPF_LDX+BPF_B+BPF_MSH X <- 4*(P[k:1]&0xf). 951 */ 952 static int 953 emit_msh(struct sljit_compiler *compiler, 954 const struct bpf_insn *pc, struct sljit_jump *to_mchain_jump, 955 struct sljit_jump ***ret0, size_t *ret0_size, size_t *ret0_maxsize) 956 { 957 int status; 958 #ifdef _KERNEL 959 struct sljit_label *label; 960 struct sljit_jump *jump, *over_mchain_jump; 961 const bool check_zero_buflen = (to_mchain_jump != NULL); 962 #endif 963 const uint32_t k = pc->k; 964 965 #ifdef _KERNEL 966 if (to_mchain_jump == NULL) { 967 to_mchain_jump = sljit_emit_cmp(compiler, 968 SLJIT_C_EQUAL, 969 BJ_BUFLEN, 0, 970 SLJIT_IMM, 0); 971 if (to_mchain_jump == NULL) 972 return SLJIT_ERR_ALLOC_FAILED; 973 } 974 #endif 975 976 /* tmp1 = buf[k] */ 977 status = sljit_emit_op1(compiler, 978 SLJIT_MOV_UB, 979 BJ_TMP1REG, 0, 980 SLJIT_MEM1(BJ_BUF), k); 981 if (status != SLJIT_SUCCESS) 982 return status; 983 984 #ifdef _KERNEL 985 over_mchain_jump = sljit_emit_jump(compiler, SLJIT_JUMP); 986 if (over_mchain_jump == NULL) 987 return SLJIT_ERR_ALLOC_FAILED; 988 989 /* entry point to mchain handler */ 990 label = sljit_emit_label(compiler); 991 if (label == NULL) 992 return SLJIT_ERR_ALLOC_FAILED; 993 sljit_set_label(to_mchain_jump, label); 994 995 if (check_zero_buflen) { 996 /* if (buflen != 0) return 0; */ 997 jump = sljit_emit_cmp(compiler, 998 SLJIT_C_NOT_EQUAL, 999 BJ_BUFLEN, 0, 1000 SLJIT_IMM, 0); 1001 if (jump == NULL) 1002 return SLJIT_ERR_ALLOC_FAILED; 1003 if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 1004 return SLJIT_ERR_ALLOC_FAILED; 1005 } 1006 1007 status = emit_xcall(compiler, pc, BJ_TMP1REG, 1008 ret0, ret0_size, ret0_maxsize, &m_xbyte); 1009 if (status != SLJIT_SUCCESS) 1010 return status; 1011 1012 label = sljit_emit_label(compiler); 1013 if (label == NULL) 1014 return SLJIT_ERR_ALLOC_FAILED; 1015 sljit_set_label(over_mchain_jump, label); 1016 #endif 1017 1018 /* tmp1 &= 0xf */ 1019 status = sljit_emit_op2(compiler, 1020 SLJIT_AND, 1021 BJ_TMP1REG, 0, 1022 BJ_TMP1REG, 0, 1023 SLJIT_IMM, 0xf); 1024 if (status != SLJIT_SUCCESS) 1025 return status; 1026 1027 /* X = tmp1 << 2 */ 1028 status = sljit_emit_op2(compiler, 1029 SLJIT_SHL, 1030 BJ_XREG, 0, 1031 BJ_TMP1REG, 0, 1032 SLJIT_IMM, 2); 1033 if (status != SLJIT_SUCCESS) 1034 return status; 1035 1036 return SLJIT_SUCCESS; 1037 } 1038 1039 static int 1040 emit_pow2_division(struct sljit_compiler *compiler, uint32_t k) 1041 { 1042 int shift = 0; 1043 int status = SLJIT_SUCCESS; 1044 1045 while (k > 1) { 1046 k >>= 1; 1047 shift++; 1048 } 1049 1050 BJ_ASSERT(k == 1 && shift < 32); 1051 1052 if (shift != 0) { 1053 status = sljit_emit_op2(compiler, 1054 SLJIT_LSHR|SLJIT_INT_OP, 1055 BJ_AREG, 0, 1056 BJ_AREG, 0, 1057 SLJIT_IMM, shift); 1058 } 1059 1060 return status; 1061 } 1062 1063 #if !defined(BPFJIT_USE_UDIV) 1064 static sljit_uw 1065 divide(sljit_uw x, sljit_uw y) 1066 { 1067 1068 return (uint32_t)x / (uint32_t)y; 1069 } 1070 #endif 1071 1072 /* 1073 * Emit code for A = A / div. 1074 * divt,divw are either SLJIT_IMM,pc->k or BJ_XREG,0. 1075 */ 1076 static int 1077 emit_division(struct sljit_compiler *compiler, int divt, sljit_sw divw) 1078 { 1079 int status; 1080 1081 #if BJ_AREG != SLJIT_SCRATCH_REG1 1082 status = sljit_emit_op1(compiler, 1083 SLJIT_MOV, 1084 SLJIT_SCRATCH_REG1, 0, 1085 BJ_AREG, 0); 1086 if (status != SLJIT_SUCCESS) 1087 return status; 1088 #endif 1089 1090 status = sljit_emit_op1(compiler, 1091 SLJIT_MOV, 1092 SLJIT_SCRATCH_REG2, 0, 1093 divt, divw); 1094 if (status != SLJIT_SUCCESS) 1095 return status; 1096 1097 #if defined(BPFJIT_USE_UDIV) 1098 status = sljit_emit_op0(compiler, SLJIT_UDIV|SLJIT_INT_OP); 1099 1100 #if BJ_AREG != SLJIT_SCRATCH_REG1 1101 status = sljit_emit_op1(compiler, 1102 SLJIT_MOV, 1103 BJ_AREG, 0, 1104 SLJIT_SCRATCH_REG1, 0); 1105 if (status != SLJIT_SUCCESS) 1106 return status; 1107 #endif 1108 #else 1109 status = sljit_emit_ijump(compiler, 1110 SLJIT_CALL2, 1111 SLJIT_IMM, SLJIT_FUNC_OFFSET(divide)); 1112 1113 #if BJ_AREG != SLJIT_RETURN_REG 1114 status = sljit_emit_op1(compiler, 1115 SLJIT_MOV, 1116 BJ_AREG, 0, 1117 SLJIT_RETURN_REG, 0); 1118 if (status != SLJIT_SUCCESS) 1119 return status; 1120 #endif 1121 #endif 1122 1123 return status; 1124 } 1125 1126 /* 1127 * Return true if pc is a "read from packet" instruction. 1128 * If length is not NULL and return value is true, *length will 1129 * be set to a safe length required to read a packet. 1130 */ 1131 static bool 1132 read_pkt_insn(const struct bpf_insn *pc, bpfjit_abc_length_t *length) 1133 { 1134 bool rv; 1135 bpfjit_abc_length_t width; 1136 1137 switch (BPF_CLASS(pc->code)) { 1138 default: 1139 rv = false; 1140 break; 1141 1142 case BPF_LD: 1143 rv = BPF_MODE(pc->code) == BPF_ABS || 1144 BPF_MODE(pc->code) == BPF_IND; 1145 if (rv) 1146 width = read_width(pc); 1147 break; 1148 1149 case BPF_LDX: 1150 rv = pc->code == (BPF_LDX|BPF_B|BPF_MSH); 1151 width = 1; 1152 break; 1153 } 1154 1155 if (rv && length != NULL) { 1156 /* 1157 * Values greater than UINT32_MAX will generate 1158 * unconditional "return 0". 1159 */ 1160 *length = (uint32_t)pc->k + width; 1161 } 1162 1163 return rv; 1164 } 1165 1166 static void 1167 optimize_init(struct bpfjit_insn_data *insn_dat, size_t insn_count) 1168 { 1169 size_t i; 1170 1171 for (i = 0; i < insn_count; i++) { 1172 SLIST_INIT(&insn_dat[i].bjumps); 1173 insn_dat[i].invalid = BJ_INIT_NOBITS; 1174 } 1175 } 1176 1177 /* 1178 * The function divides instructions into blocks. Destination of a jump 1179 * instruction starts a new block. BPF_RET and BPF_JMP instructions 1180 * terminate a block. Blocks are linear, that is, there are no jumps out 1181 * from the middle of a block and there are no jumps in to the middle of 1182 * a block. 1183 * 1184 * The function also sets bits in *initmask for memwords that 1185 * need to be initialized to zero. Note that this set should be empty 1186 * for any valid kernel filter program. 1187 */ 1188 static bool 1189 optimize_pass1(const bpf_ctx_t *bc, const struct bpf_insn *insns, 1190 struct bpfjit_insn_data *insn_dat, size_t insn_count, 1191 bpf_memword_init_t *initmask, bpfjit_hint_t *hints) 1192 { 1193 struct bpfjit_jump *jtf; 1194 size_t i; 1195 uint32_t jt, jf; 1196 bpfjit_abc_length_t length; 1197 bpf_memword_init_t invalid; /* borrowed from bpf_filter() */ 1198 bool unreachable; 1199 1200 const size_t memwords = GET_MEMWORDS(bc); 1201 1202 *hints = 0; 1203 *initmask = BJ_INIT_NOBITS; 1204 1205 unreachable = false; 1206 invalid = ~BJ_INIT_NOBITS; 1207 1208 for (i = 0; i < insn_count; i++) { 1209 if (!SLIST_EMPTY(&insn_dat[i].bjumps)) 1210 unreachable = false; 1211 insn_dat[i].unreachable = unreachable; 1212 1213 if (unreachable) 1214 continue; 1215 1216 invalid |= insn_dat[i].invalid; 1217 1218 if (read_pkt_insn(&insns[i], &length) && length > UINT32_MAX) 1219 unreachable = true; 1220 1221 switch (BPF_CLASS(insns[i].code)) { 1222 case BPF_RET: 1223 if (BPF_RVAL(insns[i].code) == BPF_A) 1224 *initmask |= invalid & BJ_INIT_ABIT; 1225 1226 unreachable = true; 1227 continue; 1228 1229 case BPF_LD: 1230 if (BPF_MODE(insns[i].code) == BPF_ABS) 1231 *hints |= BJ_HINT_ABS; 1232 1233 if (BPF_MODE(insns[i].code) == BPF_IND) { 1234 *hints |= BJ_HINT_IND | BJ_HINT_XREG; 1235 *initmask |= invalid & BJ_INIT_XBIT; 1236 } 1237 1238 if (BPF_MODE(insns[i].code) == BPF_MEM && 1239 (uint32_t)insns[i].k < memwords) { 1240 *initmask |= invalid & BJ_INIT_MBIT(insns[i].k); 1241 } 1242 1243 invalid &= ~BJ_INIT_ABIT; 1244 continue; 1245 1246 case BPF_LDX: 1247 *hints |= BJ_HINT_XREG | BJ_HINT_LDX; 1248 1249 if (BPF_MODE(insns[i].code) == BPF_MEM && 1250 (uint32_t)insns[i].k < memwords) { 1251 *initmask |= invalid & BJ_INIT_MBIT(insns[i].k); 1252 } 1253 1254 if (BPF_MODE(insns[i].code) == BPF_MSH && 1255 BPF_SIZE(insns[i].code) == BPF_B) { 1256 *hints |= BJ_HINT_MSH; 1257 } 1258 1259 invalid &= ~BJ_INIT_XBIT; 1260 continue; 1261 1262 case BPF_ST: 1263 *initmask |= invalid & BJ_INIT_ABIT; 1264 1265 if ((uint32_t)insns[i].k < memwords) 1266 invalid &= ~BJ_INIT_MBIT(insns[i].k); 1267 1268 continue; 1269 1270 case BPF_STX: 1271 *hints |= BJ_HINT_XREG; 1272 *initmask |= invalid & BJ_INIT_XBIT; 1273 1274 if ((uint32_t)insns[i].k < memwords) 1275 invalid &= ~BJ_INIT_MBIT(insns[i].k); 1276 1277 continue; 1278 1279 case BPF_ALU: 1280 *initmask |= invalid & BJ_INIT_ABIT; 1281 1282 if (insns[i].code != (BPF_ALU|BPF_NEG) && 1283 BPF_SRC(insns[i].code) == BPF_X) { 1284 *hints |= BJ_HINT_XREG; 1285 *initmask |= invalid & BJ_INIT_XBIT; 1286 } 1287 1288 invalid &= ~BJ_INIT_ABIT; 1289 continue; 1290 1291 case BPF_MISC: 1292 switch (BPF_MISCOP(insns[i].code)) { 1293 case BPF_TAX: // X <- A 1294 *hints |= BJ_HINT_XREG; 1295 *initmask |= invalid & BJ_INIT_ABIT; 1296 invalid &= ~BJ_INIT_XBIT; 1297 continue; 1298 1299 case BPF_TXA: // A <- X 1300 *hints |= BJ_HINT_XREG; 1301 *initmask |= invalid & BJ_INIT_XBIT; 1302 invalid &= ~BJ_INIT_ABIT; 1303 continue; 1304 1305 case BPF_COPX: 1306 *hints |= BJ_HINT_XREG | BJ_HINT_COPX; 1307 /* FALLTHROUGH */ 1308 1309 case BPF_COP: 1310 *hints |= BJ_HINT_COP; 1311 *initmask |= invalid & BJ_INIT_ABIT; 1312 invalid &= ~BJ_INIT_ABIT; 1313 continue; 1314 } 1315 1316 continue; 1317 1318 case BPF_JMP: 1319 /* Initialize abc_length for ABC pass. */ 1320 insn_dat[i].u.jdata.abc_length = MAX_ABC_LENGTH; 1321 1322 if (BPF_OP(insns[i].code) == BPF_JA) { 1323 jt = jf = insns[i].k; 1324 } else { 1325 jt = insns[i].jt; 1326 jf = insns[i].jf; 1327 } 1328 1329 if (jt >= insn_count - (i + 1) || 1330 jf >= insn_count - (i + 1)) { 1331 return false; 1332 } 1333 1334 if (jt > 0 && jf > 0) 1335 unreachable = true; 1336 1337 jt += i + 1; 1338 jf += i + 1; 1339 1340 jtf = insn_dat[i].u.jdata.jtf; 1341 1342 jtf[0].jdata = &insn_dat[i].u.jdata; 1343 SLIST_INSERT_HEAD(&insn_dat[jt].bjumps, 1344 &jtf[0], entries); 1345 1346 if (jf != jt) { 1347 jtf[1].jdata = &insn_dat[i].u.jdata; 1348 SLIST_INSERT_HEAD(&insn_dat[jf].bjumps, 1349 &jtf[1], entries); 1350 } 1351 1352 insn_dat[jf].invalid |= invalid; 1353 insn_dat[jt].invalid |= invalid; 1354 invalid = 0; 1355 1356 continue; 1357 } 1358 } 1359 1360 return true; 1361 } 1362 1363 /* 1364 * Array Bounds Check Elimination (ABC) pass. 1365 */ 1366 static void 1367 optimize_pass2(const bpf_ctx_t *bc, const struct bpf_insn *insns, 1368 struct bpfjit_insn_data *insn_dat, size_t insn_count) 1369 { 1370 struct bpfjit_jump *jmp; 1371 const struct bpf_insn *pc; 1372 struct bpfjit_insn_data *pd; 1373 size_t i; 1374 bpfjit_abc_length_t length, abc_length = 0; 1375 1376 const size_t extwords = GET_EXTWORDS(bc); 1377 1378 for (i = insn_count; i != 0; i--) { 1379 pc = &insns[i-1]; 1380 pd = &insn_dat[i-1]; 1381 1382 if (pd->unreachable) 1383 continue; 1384 1385 switch (BPF_CLASS(pc->code)) { 1386 case BPF_RET: 1387 /* 1388 * It's quite common for bpf programs to 1389 * check packet bytes in increasing order 1390 * and return zero if bytes don't match 1391 * specified critetion. Such programs disable 1392 * ABC optimization completely because for 1393 * every jump there is a branch with no read 1394 * instruction. 1395 * With no side effects, BPF_STMT(BPF_RET+BPF_K, 0) 1396 * is indistinguishable from out-of-bound load. 1397 * Therefore, abc_length can be set to 1398 * MAX_ABC_LENGTH and enable ABC for many 1399 * bpf programs. 1400 * If this optimization encounters any 1401 * instruction with a side effect, it will 1402 * reset abc_length. 1403 */ 1404 if (BPF_RVAL(pc->code) == BPF_K && pc->k == 0) 1405 abc_length = MAX_ABC_LENGTH; 1406 else 1407 abc_length = 0; 1408 break; 1409 1410 case BPF_MISC: 1411 if (BPF_MISCOP(pc->code) == BPF_COP || 1412 BPF_MISCOP(pc->code) == BPF_COPX) { 1413 /* COP instructions can have side effects. */ 1414 abc_length = 0; 1415 } 1416 break; 1417 1418 case BPF_ST: 1419 case BPF_STX: 1420 if (extwords != 0) { 1421 /* Write to memory is visible after a call. */ 1422 abc_length = 0; 1423 } 1424 break; 1425 1426 case BPF_JMP: 1427 abc_length = pd->u.jdata.abc_length; 1428 break; 1429 1430 default: 1431 if (read_pkt_insn(pc, &length)) { 1432 if (abc_length < length) 1433 abc_length = length; 1434 pd->u.rdata.abc_length = abc_length; 1435 } 1436 break; 1437 } 1438 1439 SLIST_FOREACH(jmp, &pd->bjumps, entries) { 1440 if (jmp->jdata->abc_length > abc_length) 1441 jmp->jdata->abc_length = abc_length; 1442 } 1443 } 1444 } 1445 1446 static void 1447 optimize_pass3(const struct bpf_insn *insns, 1448 struct bpfjit_insn_data *insn_dat, size_t insn_count) 1449 { 1450 struct bpfjit_jump *jmp; 1451 size_t i; 1452 bpfjit_abc_length_t checked_length = 0; 1453 1454 for (i = 0; i < insn_count; i++) { 1455 if (insn_dat[i].unreachable) 1456 continue; 1457 1458 SLIST_FOREACH(jmp, &insn_dat[i].bjumps, entries) { 1459 if (jmp->jdata->checked_length < checked_length) 1460 checked_length = jmp->jdata->checked_length; 1461 } 1462 1463 if (BPF_CLASS(insns[i].code) == BPF_JMP) { 1464 insn_dat[i].u.jdata.checked_length = checked_length; 1465 } else if (read_pkt_insn(&insns[i], NULL)) { 1466 struct bpfjit_read_pkt_data *rdata = 1467 &insn_dat[i].u.rdata; 1468 rdata->check_length = 0; 1469 if (checked_length < rdata->abc_length) { 1470 checked_length = rdata->abc_length; 1471 rdata->check_length = checked_length; 1472 } 1473 } 1474 } 1475 } 1476 1477 static bool 1478 optimize(const bpf_ctx_t *bc, const struct bpf_insn *insns, 1479 struct bpfjit_insn_data *insn_dat, size_t insn_count, 1480 bpf_memword_init_t *initmask, bpfjit_hint_t *hints) 1481 { 1482 1483 optimize_init(insn_dat, insn_count); 1484 1485 if (!optimize_pass1(bc, insns, insn_dat, insn_count, initmask, hints)) 1486 return false; 1487 1488 optimize_pass2(bc, insns, insn_dat, insn_count); 1489 optimize_pass3(insns, insn_dat, insn_count); 1490 1491 return true; 1492 } 1493 1494 /* 1495 * Convert BPF_ALU operations except BPF_NEG and BPF_DIV to sljit operation. 1496 */ 1497 static int 1498 bpf_alu_to_sljit_op(const struct bpf_insn *pc) 1499 { 1500 1501 /* 1502 * Note: all supported 64bit arches have 32bit multiply 1503 * instruction so SLJIT_INT_OP doesn't have any overhead. 1504 */ 1505 switch (BPF_OP(pc->code)) { 1506 case BPF_ADD: return SLJIT_ADD; 1507 case BPF_SUB: return SLJIT_SUB; 1508 case BPF_MUL: return SLJIT_MUL|SLJIT_INT_OP; 1509 case BPF_OR: return SLJIT_OR; 1510 case BPF_AND: return SLJIT_AND; 1511 case BPF_LSH: return SLJIT_SHL; 1512 case BPF_RSH: return SLJIT_LSHR|SLJIT_INT_OP; 1513 default: 1514 BJ_ASSERT(false); 1515 return 0; 1516 } 1517 } 1518 1519 /* 1520 * Convert BPF_JMP operations except BPF_JA to sljit condition. 1521 */ 1522 static int 1523 bpf_jmp_to_sljit_cond(const struct bpf_insn *pc, bool negate) 1524 { 1525 /* 1526 * Note: all supported 64bit arches have 32bit comparison 1527 * instructions so SLJIT_INT_OP doesn't have any overhead. 1528 */ 1529 int rv = SLJIT_INT_OP; 1530 1531 switch (BPF_OP(pc->code)) { 1532 case BPF_JGT: 1533 rv |= negate ? SLJIT_C_LESS_EQUAL : SLJIT_C_GREATER; 1534 break; 1535 case BPF_JGE: 1536 rv |= negate ? SLJIT_C_LESS : SLJIT_C_GREATER_EQUAL; 1537 break; 1538 case BPF_JEQ: 1539 rv |= negate ? SLJIT_C_NOT_EQUAL : SLJIT_C_EQUAL; 1540 break; 1541 case BPF_JSET: 1542 rv |= negate ? SLJIT_C_EQUAL : SLJIT_C_NOT_EQUAL; 1543 break; 1544 default: 1545 BJ_ASSERT(false); 1546 } 1547 1548 return rv; 1549 } 1550 1551 /* 1552 * Convert BPF_K and BPF_X to sljit register. 1553 */ 1554 static int 1555 kx_to_reg(const struct bpf_insn *pc) 1556 { 1557 1558 switch (BPF_SRC(pc->code)) { 1559 case BPF_K: return SLJIT_IMM; 1560 case BPF_X: return BJ_XREG; 1561 default: 1562 BJ_ASSERT(false); 1563 return 0; 1564 } 1565 } 1566 1567 static sljit_sw 1568 kx_to_reg_arg(const struct bpf_insn *pc) 1569 { 1570 1571 switch (BPF_SRC(pc->code)) { 1572 case BPF_K: return (uint32_t)pc->k; /* SLJIT_IMM, pc->k, */ 1573 case BPF_X: return 0; /* BJ_XREG, 0, */ 1574 default: 1575 BJ_ASSERT(false); 1576 return 0; 1577 } 1578 } 1579 1580 static bool 1581 generate_insn_code(struct sljit_compiler *compiler, const bpf_ctx_t *bc, 1582 const struct bpf_insn *insns, struct bpfjit_insn_data *insn_dat, 1583 size_t insn_count) 1584 { 1585 /* a list of jumps to out-of-bound return from a generated function */ 1586 struct sljit_jump **ret0; 1587 size_t ret0_size, ret0_maxsize; 1588 1589 struct sljit_jump *jump; 1590 struct sljit_label *label; 1591 const struct bpf_insn *pc; 1592 struct bpfjit_jump *bjump, *jtf; 1593 struct sljit_jump *to_mchain_jump; 1594 1595 size_t i; 1596 int status; 1597 int branching, negate; 1598 unsigned int rval, mode, src; 1599 uint32_t jt, jf; 1600 1601 bool unconditional_ret; 1602 bool rv; 1603 1604 const size_t extwords = GET_EXTWORDS(bc); 1605 const size_t memwords = GET_MEMWORDS(bc); 1606 1607 ret0 = NULL; 1608 rv = false; 1609 1610 ret0_size = 0; 1611 ret0_maxsize = 64; 1612 ret0 = BJ_ALLOC(ret0_maxsize * sizeof(ret0[0])); 1613 if (ret0 == NULL) 1614 goto fail; 1615 1616 /* reset sjump members of jdata */ 1617 for (i = 0; i < insn_count; i++) { 1618 if (insn_dat[i].unreachable || 1619 BPF_CLASS(insns[i].code) != BPF_JMP) { 1620 continue; 1621 } 1622 1623 jtf = insn_dat[i].u.jdata.jtf; 1624 jtf[0].sjump = jtf[1].sjump = NULL; 1625 } 1626 1627 /* main loop */ 1628 for (i = 0; i < insn_count; i++) { 1629 if (insn_dat[i].unreachable) 1630 continue; 1631 1632 /* 1633 * Resolve jumps to the current insn. 1634 */ 1635 label = NULL; 1636 SLIST_FOREACH(bjump, &insn_dat[i].bjumps, entries) { 1637 if (bjump->sjump != NULL) { 1638 if (label == NULL) 1639 label = sljit_emit_label(compiler); 1640 if (label == NULL) 1641 goto fail; 1642 sljit_set_label(bjump->sjump, label); 1643 } 1644 } 1645 1646 to_mchain_jump = NULL; 1647 unconditional_ret = false; 1648 1649 if (read_pkt_insn(&insns[i], NULL)) { 1650 if (insn_dat[i].u.rdata.check_length > UINT32_MAX) { 1651 /* Jump to "return 0" unconditionally. */ 1652 unconditional_ret = true; 1653 jump = sljit_emit_jump(compiler, SLJIT_JUMP); 1654 if (jump == NULL) 1655 goto fail; 1656 if (!append_jump(jump, &ret0, 1657 &ret0_size, &ret0_maxsize)) 1658 goto fail; 1659 } else if (insn_dat[i].u.rdata.check_length > 0) { 1660 /* if (buflen < check_length) return 0; */ 1661 jump = sljit_emit_cmp(compiler, 1662 SLJIT_C_LESS, 1663 BJ_BUFLEN, 0, 1664 SLJIT_IMM, 1665 insn_dat[i].u.rdata.check_length); 1666 if (jump == NULL) 1667 goto fail; 1668 #ifdef _KERNEL 1669 to_mchain_jump = jump; 1670 #else 1671 if (!append_jump(jump, &ret0, 1672 &ret0_size, &ret0_maxsize)) 1673 goto fail; 1674 #endif 1675 } 1676 } 1677 1678 pc = &insns[i]; 1679 switch (BPF_CLASS(pc->code)) { 1680 1681 default: 1682 goto fail; 1683 1684 case BPF_LD: 1685 /* BPF_LD+BPF_IMM A <- k */ 1686 if (pc->code == (BPF_LD|BPF_IMM)) { 1687 status = sljit_emit_op1(compiler, 1688 SLJIT_MOV, 1689 BJ_AREG, 0, 1690 SLJIT_IMM, (uint32_t)pc->k); 1691 if (status != SLJIT_SUCCESS) 1692 goto fail; 1693 1694 continue; 1695 } 1696 1697 /* BPF_LD+BPF_MEM A <- M[k] */ 1698 if (pc->code == (BPF_LD|BPF_MEM)) { 1699 if ((uint32_t)pc->k >= memwords) 1700 goto fail; 1701 status = emit_memload(compiler, 1702 BJ_AREG, pc->k, extwords); 1703 if (status != SLJIT_SUCCESS) 1704 goto fail; 1705 1706 continue; 1707 } 1708 1709 /* BPF_LD+BPF_W+BPF_LEN A <- len */ 1710 if (pc->code == (BPF_LD|BPF_W|BPF_LEN)) { 1711 status = sljit_emit_op1(compiler, 1712 SLJIT_MOV, /* size_t source */ 1713 BJ_AREG, 0, 1714 SLJIT_MEM1(BJ_ARGS), 1715 offsetof(struct bpf_args, wirelen)); 1716 if (status != SLJIT_SUCCESS) 1717 goto fail; 1718 1719 continue; 1720 } 1721 1722 mode = BPF_MODE(pc->code); 1723 if (mode != BPF_ABS && mode != BPF_IND) 1724 goto fail; 1725 1726 if (unconditional_ret) 1727 continue; 1728 1729 status = emit_pkt_read(compiler, pc, 1730 to_mchain_jump, &ret0, &ret0_size, &ret0_maxsize); 1731 if (status != SLJIT_SUCCESS) 1732 goto fail; 1733 1734 continue; 1735 1736 case BPF_LDX: 1737 mode = BPF_MODE(pc->code); 1738 1739 /* BPF_LDX+BPF_W+BPF_IMM X <- k */ 1740 if (mode == BPF_IMM) { 1741 if (BPF_SIZE(pc->code) != BPF_W) 1742 goto fail; 1743 status = sljit_emit_op1(compiler, 1744 SLJIT_MOV, 1745 BJ_XREG, 0, 1746 SLJIT_IMM, (uint32_t)pc->k); 1747 if (status != SLJIT_SUCCESS) 1748 goto fail; 1749 1750 continue; 1751 } 1752 1753 /* BPF_LDX+BPF_W+BPF_LEN X <- len */ 1754 if (mode == BPF_LEN) { 1755 if (BPF_SIZE(pc->code) != BPF_W) 1756 goto fail; 1757 status = sljit_emit_op1(compiler, 1758 SLJIT_MOV, /* size_t source */ 1759 BJ_XREG, 0, 1760 SLJIT_MEM1(BJ_ARGS), 1761 offsetof(struct bpf_args, wirelen)); 1762 if (status != SLJIT_SUCCESS) 1763 goto fail; 1764 1765 continue; 1766 } 1767 1768 /* BPF_LDX+BPF_W+BPF_MEM X <- M[k] */ 1769 if (mode == BPF_MEM) { 1770 if (BPF_SIZE(pc->code) != BPF_W) 1771 goto fail; 1772 if ((uint32_t)pc->k >= memwords) 1773 goto fail; 1774 status = emit_memload(compiler, 1775 BJ_XREG, pc->k, extwords); 1776 if (status != SLJIT_SUCCESS) 1777 goto fail; 1778 1779 continue; 1780 } 1781 1782 /* BPF_LDX+BPF_B+BPF_MSH X <- 4*(P[k:1]&0xf) */ 1783 if (mode != BPF_MSH || BPF_SIZE(pc->code) != BPF_B) 1784 goto fail; 1785 1786 if (unconditional_ret) 1787 continue; 1788 1789 status = emit_msh(compiler, pc, 1790 to_mchain_jump, &ret0, &ret0_size, &ret0_maxsize); 1791 if (status != SLJIT_SUCCESS) 1792 goto fail; 1793 1794 continue; 1795 1796 case BPF_ST: 1797 if (pc->code != BPF_ST || 1798 (uint32_t)pc->k >= memwords) { 1799 goto fail; 1800 } 1801 1802 status = emit_memstore(compiler, 1803 BJ_AREG, pc->k, extwords); 1804 if (status != SLJIT_SUCCESS) 1805 goto fail; 1806 1807 continue; 1808 1809 case BPF_STX: 1810 if (pc->code != BPF_STX || 1811 (uint32_t)pc->k >= memwords) { 1812 goto fail; 1813 } 1814 1815 status = emit_memstore(compiler, 1816 BJ_XREG, pc->k, extwords); 1817 if (status != SLJIT_SUCCESS) 1818 goto fail; 1819 1820 continue; 1821 1822 case BPF_ALU: 1823 if (pc->code == (BPF_ALU|BPF_NEG)) { 1824 status = sljit_emit_op1(compiler, 1825 SLJIT_NEG, 1826 BJ_AREG, 0, 1827 BJ_AREG, 0); 1828 if (status != SLJIT_SUCCESS) 1829 goto fail; 1830 1831 continue; 1832 } 1833 1834 if (BPF_OP(pc->code) != BPF_DIV) { 1835 status = sljit_emit_op2(compiler, 1836 bpf_alu_to_sljit_op(pc), 1837 BJ_AREG, 0, 1838 BJ_AREG, 0, 1839 kx_to_reg(pc), kx_to_reg_arg(pc)); 1840 if (status != SLJIT_SUCCESS) 1841 goto fail; 1842 1843 continue; 1844 } 1845 1846 /* BPF_DIV */ 1847 1848 src = BPF_SRC(pc->code); 1849 if (src != BPF_X && src != BPF_K) 1850 goto fail; 1851 1852 /* division by zero? */ 1853 if (src == BPF_X) { 1854 jump = sljit_emit_cmp(compiler, 1855 SLJIT_C_EQUAL|SLJIT_INT_OP, 1856 BJ_XREG, 0, 1857 SLJIT_IMM, 0); 1858 if (jump == NULL) 1859 goto fail; 1860 if (!append_jump(jump, &ret0, 1861 &ret0_size, &ret0_maxsize)) 1862 goto fail; 1863 } else if (pc->k == 0) { 1864 jump = sljit_emit_jump(compiler, SLJIT_JUMP); 1865 if (jump == NULL) 1866 goto fail; 1867 if (!append_jump(jump, &ret0, 1868 &ret0_size, &ret0_maxsize)) 1869 goto fail; 1870 } 1871 1872 if (src == BPF_X) { 1873 status = emit_division(compiler, BJ_XREG, 0); 1874 if (status != SLJIT_SUCCESS) 1875 goto fail; 1876 } else if (pc->k != 0) { 1877 if (pc->k & (pc->k - 1)) { 1878 status = emit_division(compiler, 1879 SLJIT_IMM, (uint32_t)pc->k); 1880 } else { 1881 status = emit_pow2_division(compiler, 1882 (uint32_t)pc->k); 1883 } 1884 if (status != SLJIT_SUCCESS) 1885 goto fail; 1886 } 1887 1888 continue; 1889 1890 case BPF_JMP: 1891 if (BPF_OP(pc->code) == BPF_JA) { 1892 jt = jf = pc->k; 1893 } else { 1894 jt = pc->jt; 1895 jf = pc->jf; 1896 } 1897 1898 negate = (jt == 0) ? 1 : 0; 1899 branching = (jt == jf) ? 0 : 1; 1900 jtf = insn_dat[i].u.jdata.jtf; 1901 1902 if (branching) { 1903 if (BPF_OP(pc->code) != BPF_JSET) { 1904 jump = sljit_emit_cmp(compiler, 1905 bpf_jmp_to_sljit_cond(pc, negate), 1906 BJ_AREG, 0, 1907 kx_to_reg(pc), kx_to_reg_arg(pc)); 1908 } else { 1909 status = sljit_emit_op2(compiler, 1910 SLJIT_AND, 1911 BJ_TMP1REG, 0, 1912 BJ_AREG, 0, 1913 kx_to_reg(pc), kx_to_reg_arg(pc)); 1914 if (status != SLJIT_SUCCESS) 1915 goto fail; 1916 1917 jump = sljit_emit_cmp(compiler, 1918 bpf_jmp_to_sljit_cond(pc, negate), 1919 BJ_TMP1REG, 0, 1920 SLJIT_IMM, 0); 1921 } 1922 1923 if (jump == NULL) 1924 goto fail; 1925 1926 BJ_ASSERT(jtf[negate].sjump == NULL); 1927 jtf[negate].sjump = jump; 1928 } 1929 1930 if (!branching || (jt != 0 && jf != 0)) { 1931 jump = sljit_emit_jump(compiler, SLJIT_JUMP); 1932 if (jump == NULL) 1933 goto fail; 1934 1935 BJ_ASSERT(jtf[branching].sjump == NULL); 1936 jtf[branching].sjump = jump; 1937 } 1938 1939 continue; 1940 1941 case BPF_RET: 1942 rval = BPF_RVAL(pc->code); 1943 if (rval == BPF_X) 1944 goto fail; 1945 1946 /* BPF_RET+BPF_K accept k bytes */ 1947 if (rval == BPF_K) { 1948 status = sljit_emit_return(compiler, 1949 SLJIT_MOV_UI, 1950 SLJIT_IMM, (uint32_t)pc->k); 1951 if (status != SLJIT_SUCCESS) 1952 goto fail; 1953 } 1954 1955 /* BPF_RET+BPF_A accept A bytes */ 1956 if (rval == BPF_A) { 1957 status = sljit_emit_return(compiler, 1958 SLJIT_MOV_UI, 1959 BJ_AREG, 0); 1960 if (status != SLJIT_SUCCESS) 1961 goto fail; 1962 } 1963 1964 continue; 1965 1966 case BPF_MISC: 1967 switch (BPF_MISCOP(pc->code)) { 1968 case BPF_TAX: 1969 status = sljit_emit_op1(compiler, 1970 SLJIT_MOV_UI, 1971 BJ_XREG, 0, 1972 BJ_AREG, 0); 1973 if (status != SLJIT_SUCCESS) 1974 goto fail; 1975 1976 continue; 1977 1978 case BPF_TXA: 1979 status = sljit_emit_op1(compiler, 1980 SLJIT_MOV, 1981 BJ_AREG, 0, 1982 BJ_XREG, 0); 1983 if (status != SLJIT_SUCCESS) 1984 goto fail; 1985 1986 continue; 1987 1988 case BPF_COP: 1989 case BPF_COPX: 1990 if (bc == NULL || bc->copfuncs == NULL) 1991 goto fail; 1992 if (BPF_MISCOP(pc->code) == BPF_COP && 1993 (uint32_t)pc->k >= bc->nfuncs) { 1994 goto fail; 1995 } 1996 1997 status = emit_cop(compiler, bc, pc, 1998 &ret0, &ret0_size, &ret0_maxsize); 1999 if (status != SLJIT_SUCCESS) 2000 goto fail; 2001 2002 continue; 2003 } 2004 2005 goto fail; 2006 } /* switch */ 2007 } /* main loop */ 2008 2009 BJ_ASSERT(ret0_size <= ret0_maxsize); 2010 2011 if (ret0_size > 0) { 2012 label = sljit_emit_label(compiler); 2013 if (label == NULL) 2014 goto fail; 2015 for (i = 0; i < ret0_size; i++) 2016 sljit_set_label(ret0[i], label); 2017 } 2018 2019 status = sljit_emit_return(compiler, 2020 SLJIT_MOV_UI, 2021 SLJIT_IMM, 0); 2022 if (status != SLJIT_SUCCESS) 2023 goto fail; 2024 2025 rv = true; 2026 2027 fail: 2028 if (ret0 != NULL) 2029 BJ_FREE(ret0, ret0_maxsize * sizeof(ret0[0])); 2030 2031 return rv; 2032 } 2033 2034 bpfjit_func_t 2035 bpfjit_generate_code(const bpf_ctx_t *bc, 2036 const struct bpf_insn *insns, size_t insn_count) 2037 { 2038 void *rv; 2039 struct sljit_compiler *compiler; 2040 2041 size_t i; 2042 int status; 2043 2044 /* optimization related */ 2045 bpf_memword_init_t initmask; 2046 bpfjit_hint_t hints; 2047 2048 /* memory store location for initial zero initialization */ 2049 sljit_si mem_reg; 2050 sljit_sw mem_off; 2051 2052 struct bpfjit_insn_data *insn_dat; 2053 2054 const size_t extwords = GET_EXTWORDS(bc); 2055 const size_t memwords = GET_MEMWORDS(bc); 2056 const bpf_memword_init_t preinited = extwords ? bc->preinited : 0; 2057 2058 rv = NULL; 2059 compiler = NULL; 2060 insn_dat = NULL; 2061 2062 if (memwords > MAX_MEMWORDS) 2063 goto fail; 2064 2065 if (insn_count == 0 || insn_count > SIZE_MAX / sizeof(insn_dat[0])) 2066 goto fail; 2067 2068 insn_dat = BJ_ALLOC(insn_count * sizeof(insn_dat[0])); 2069 if (insn_dat == NULL) 2070 goto fail; 2071 2072 if (!optimize(bc, insns, insn_dat, insn_count, &initmask, &hints)) 2073 goto fail; 2074 2075 compiler = sljit_create_compiler(); 2076 if (compiler == NULL) 2077 goto fail; 2078 2079 #if !defined(_KERNEL) && defined(SLJIT_VERBOSE) && SLJIT_VERBOSE 2080 sljit_compiler_verbose(compiler, stderr); 2081 #endif 2082 2083 status = sljit_emit_enter(compiler, 2084 2, nscratches(hints), nsaveds(hints), sizeof(struct bpfjit_stack)); 2085 if (status != SLJIT_SUCCESS) 2086 goto fail; 2087 2088 if (hints & BJ_HINT_COP) { 2089 /* save ctx argument */ 2090 status = sljit_emit_op1(compiler, 2091 SLJIT_MOV_P, 2092 SLJIT_MEM1(SLJIT_LOCALS_REG), 2093 offsetof(struct bpfjit_stack, ctx), 2094 BJ_CTX_ARG, 0); 2095 if (status != SLJIT_SUCCESS) 2096 goto fail; 2097 } 2098 2099 if (extwords == 0) { 2100 mem_reg = SLJIT_MEM1(SLJIT_LOCALS_REG); 2101 mem_off = offsetof(struct bpfjit_stack, mem); 2102 } else { 2103 /* copy "mem" argument from bpf_args to bpfjit_stack */ 2104 status = sljit_emit_op1(compiler, 2105 SLJIT_MOV_P, 2106 BJ_TMP1REG, 0, 2107 SLJIT_MEM1(BJ_ARGS), offsetof(struct bpf_args, mem)); 2108 if (status != SLJIT_SUCCESS) 2109 goto fail; 2110 2111 status = sljit_emit_op1(compiler, 2112 SLJIT_MOV_P, 2113 SLJIT_MEM1(SLJIT_LOCALS_REG), 2114 offsetof(struct bpfjit_stack, extmem), 2115 BJ_TMP1REG, 0); 2116 if (status != SLJIT_SUCCESS) 2117 goto fail; 2118 2119 mem_reg = SLJIT_MEM1(BJ_TMP1REG); 2120 mem_off = 0; 2121 } 2122 2123 /* 2124 * Exclude pre-initialised external memory words but keep 2125 * initialization statuses of A and X registers in case 2126 * bc->preinited wrongly sets those two bits. 2127 */ 2128 initmask &= ~preinited | BJ_INIT_ABIT | BJ_INIT_XBIT; 2129 2130 #if defined(_KERNEL) 2131 /* bpf_filter() checks initialization of memwords. */ 2132 BJ_ASSERT((initmask & (BJ_INIT_MBIT(memwords) - 1)) == 0); 2133 #endif 2134 for (i = 0; i < memwords; i++) { 2135 if (initmask & BJ_INIT_MBIT(i)) { 2136 /* M[i] = 0; */ 2137 status = sljit_emit_op1(compiler, 2138 SLJIT_MOV_UI, 2139 mem_reg, mem_off + i * sizeof(uint32_t), 2140 SLJIT_IMM, 0); 2141 if (status != SLJIT_SUCCESS) 2142 goto fail; 2143 } 2144 } 2145 2146 if (initmask & BJ_INIT_ABIT) { 2147 /* A = 0; */ 2148 status = sljit_emit_op1(compiler, 2149 SLJIT_MOV, 2150 BJ_AREG, 0, 2151 SLJIT_IMM, 0); 2152 if (status != SLJIT_SUCCESS) 2153 goto fail; 2154 } 2155 2156 if (initmask & BJ_INIT_XBIT) { 2157 /* X = 0; */ 2158 status = sljit_emit_op1(compiler, 2159 SLJIT_MOV, 2160 BJ_XREG, 0, 2161 SLJIT_IMM, 0); 2162 if (status != SLJIT_SUCCESS) 2163 goto fail; 2164 } 2165 2166 status = load_buf_buflen(compiler); 2167 if (status != SLJIT_SUCCESS) 2168 goto fail; 2169 2170 if (!generate_insn_code(compiler, bc, insns, insn_dat, insn_count)) 2171 goto fail; 2172 2173 rv = sljit_generate_code(compiler); 2174 2175 fail: 2176 if (compiler != NULL) 2177 sljit_free_compiler(compiler); 2178 2179 if (insn_dat != NULL) 2180 BJ_FREE(insn_dat, insn_count * sizeof(insn_dat[0])); 2181 2182 return (bpfjit_func_t)rv; 2183 } 2184 2185 void 2186 bpfjit_free_code(bpfjit_func_t code) 2187 { 2188 2189 sljit_free_code((void *)code); 2190 } 2191
Indexes created Fri Sep 26 20:09:58 GMT 2025