bpfjit.c revision 1.29
1 /* $NetBSD: bpfjit.c,v 1.29 2014/07/22 08:20:08 alnsn Exp $ */ 2 3 /*- 4 * Copyright (c) 2011-2014 Alexander Nasonov. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 21 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 22 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 23 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 24 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 25 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 26 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 27 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 28 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 #ifdef _KERNEL 34 __KERNEL_RCSID(0, "$NetBSD: bpfjit.c,v 1.29 2014/07/22 08:20:08 alnsn Exp $"); 35 #else 36 __RCSID("$NetBSD: bpfjit.c,v 1.29 2014/07/22 08:20:08 alnsn Exp $"); 37 #endif 38 39 #include <sys/types.h> 40 #include <sys/queue.h> 41 42 #ifndef _KERNEL 43 #include <assert.h> 44 #define BJ_ASSERT(c) assert(c) 45 #else 46 #define BJ_ASSERT(c) KASSERT(c) 47 #endif 48 49 #ifndef _KERNEL 50 #include <stdlib.h> 51 #define BJ_ALLOC(sz) malloc(sz) 52 #define BJ_FREE(p, sz) free(p) 53 #else 54 #include <sys/kmem.h> 55 #define BJ_ALLOC(sz) kmem_alloc(sz, KM_SLEEP) 56 #define BJ_FREE(p, sz) kmem_free(p, sz) 57 #endif 58 59 #ifndef _KERNEL 60 #include <limits.h> 61 #include <stdbool.h> 62 #include <stddef.h> 63 #include <stdint.h> 64 #else 65 #include <sys/atomic.h> 66 #include <sys/module.h> 67 #endif 68 69 #define __BPF_PRIVATE 70 #include <net/bpf.h> 71 #include <net/bpfjit.h> 72 #include <sljitLir.h> 73 74 #if !defined(_KERNEL) && defined(SLJIT_VERBOSE) && SLJIT_VERBOSE 75 #include <stdio.h> /* for stderr */ 76 #endif 77 78 /* 79 * Arguments of generated bpfjit_func_t. 80 * The first argument is reassigned upon entry 81 * to a more frequently used buf argument. 82 */ 83 #define BJ_CTX_ARG SLJIT_SAVED_REG1 84 #define BJ_ARGS SLJIT_SAVED_REG2 85 86 /* 87 * Permanent register assignments. 88 */ 89 #define BJ_BUF SLJIT_SAVED_REG1 90 //#define BJ_ARGS SLJIT_SAVED_REG2 91 #define BJ_BUFLEN SLJIT_SAVED_REG3 92 #define BJ_XREG SLJIT_SAVED_EREG1 93 #define BJ_ASAVE SLJIT_SAVED_EREG2 94 #define BJ_AREG SLJIT_SCRATCH_REG1 95 #define BJ_TMP1REG SLJIT_SCRATCH_REG2 96 #define BJ_TMP2REG SLJIT_SCRATCH_REG3 97 #define BJ_TMP3REG SLJIT_TEMPORARY_EREG1 98 99 #ifdef _KERNEL 100 #define MAX_MEMWORDS BPF_MAX_MEMWORDS 101 #else 102 #define MAX_MEMWORDS BPF_MEMWORDS 103 #endif 104 105 #define BJ_INIT_NOBITS ((bpf_memword_init_t)0) 106 #define BJ_INIT_MBIT(k) BPF_MEMWORD_INIT(k) 107 #define BJ_INIT_ABIT BJ_INIT_MBIT(MAX_MEMWORDS) 108 #define BJ_INIT_XBIT BJ_INIT_MBIT(MAX_MEMWORDS + 1) 109 110 /* 111 * Get a number of memwords and external memwords from a bpf_ctx object. 112 */ 113 #define GET_EXTWORDS(bc) ((bc) ? (bc)->extwords : 0) 114 #define GET_MEMWORDS(bc) (GET_EXTWORDS(bc) ? GET_EXTWORDS(bc) : BPF_MEMWORDS) 115 116 /* 117 * Optimization hints. 118 */ 119 typedef unsigned int bpfjit_hint_t; 120 #define BJ_HINT_ABS 0x01 /* packet read at absolute offset */ 121 #define BJ_HINT_IND 0x02 /* packet read at variable offset */ 122 #define BJ_HINT_MSH 0x04 /* BPF_MSH instruction */ 123 #define BJ_HINT_COP 0x08 /* BPF_COP or BPF_COPX instruction */ 124 #define BJ_HINT_COPX 0x10 /* BPF_COPX instruction */ 125 #define BJ_HINT_XREG 0x20 /* BJ_XREG is needed */ 126 #define BJ_HINT_LDX 0x40 /* BPF_LDX instruction */ 127 #define BJ_HINT_PKT (BJ_HINT_ABS|BJ_HINT_IND|BJ_HINT_MSH) 128 129 /* 130 * Datatype for Array Bounds Check Elimination (ABC) pass. 131 */ 132 typedef uint64_t bpfjit_abc_length_t; 133 #define MAX_ABC_LENGTH (UINT32_MAX + UINT64_C(4)) /* max. width is 4 */ 134 135 struct bpfjit_stack 136 { 137 bpf_ctx_t *ctx; 138 uint32_t *extmem; /* pointer to external memory store */ 139 #ifdef _KERNEL 140 int err; /* 3rd argument for m_xword/m_xhalf/m_xbyte function call */ 141 #endif 142 uint32_t mem[BPF_MEMWORDS]; /* internal memory store */ 143 }; 144 145 /* 146 * Data for BPF_JMP instruction. 147 * Forward declaration for struct bpfjit_jump. 148 */ 149 struct bpfjit_jump_data; 150 151 /* 152 * Node of bjumps list. 153 */ 154 struct bpfjit_jump { 155 struct sljit_jump *sjump; 156 SLIST_ENTRY(bpfjit_jump) entries; 157 struct bpfjit_jump_data *jdata; 158 }; 159 160 /* 161 * Data for BPF_JMP instruction. 162 */ 163 struct bpfjit_jump_data { 164 /* 165 * These entries make up bjumps list: 166 * jtf[0] - when coming from jt path, 167 * jtf[1] - when coming from jf path. 168 */ 169 struct bpfjit_jump jtf[2]; 170 /* 171 * Length calculated by Array Bounds Check Elimination (ABC) pass. 172 */ 173 bpfjit_abc_length_t abc_length; 174 /* 175 * Length checked by the last out-of-bounds check. 176 */ 177 bpfjit_abc_length_t checked_length; 178 }; 179 180 /* 181 * Data for "read from packet" instructions. 182 * See also read_pkt_insn() function below. 183 */ 184 struct bpfjit_read_pkt_data { 185 /* 186 * Length calculated by Array Bounds Check Elimination (ABC) pass. 187 */ 188 bpfjit_abc_length_t abc_length; 189 /* 190 * If positive, emit "if (buflen < check_length) return 0" 191 * out-of-bounds check. 192 * Values greater than UINT32_MAX generate unconditional "return 0". 193 */ 194 bpfjit_abc_length_t check_length; 195 }; 196 197 /* 198 * Additional (optimization-related) data for bpf_insn. 199 */ 200 struct bpfjit_insn_data { 201 /* List of jumps to this insn. */ 202 SLIST_HEAD(, bpfjit_jump) bjumps; 203 204 union { 205 struct bpfjit_jump_data jdata; 206 struct bpfjit_read_pkt_data rdata; 207 } u; 208 209 bpf_memword_init_t invalid; 210 bool unreachable; 211 }; 212 213 #ifdef _KERNEL 214 215 uint32_t m_xword(const struct mbuf *, uint32_t, int *); 216 uint32_t m_xhalf(const struct mbuf *, uint32_t, int *); 217 uint32_t m_xbyte(const struct mbuf *, uint32_t, int *); 218 219 MODULE(MODULE_CLASS_MISC, bpfjit, "sljit") 220 221 static int 222 bpfjit_modcmd(modcmd_t cmd, void *arg) 223 { 224 225 switch (cmd) { 226 case MODULE_CMD_INIT: 227 bpfjit_module_ops.bj_free_code = &bpfjit_free_code; 228 membar_producer(); 229 bpfjit_module_ops.bj_generate_code = &bpfjit_generate_code; 230 membar_producer(); 231 return 0; 232 233 case MODULE_CMD_FINI: 234 return EOPNOTSUPP; 235 236 default: 237 return ENOTTY; 238 } 239 } 240 #endif 241 242 /* 243 * Return a number of scratch registers to pass 244 * to sljit_emit_enter() function. 245 */ 246 static sljit_si 247 nscratches(bpfjit_hint_t hints) 248 { 249 sljit_si rv = 2; 250 251 #ifdef _KERNEL 252 if (hints & BJ_HINT_PKT) 253 rv = 3; /* xcall with three arguments */ 254 #endif 255 256 if (hints & BJ_HINT_IND) 257 rv = 3; /* uses BJ_TMP2REG */ 258 259 if (hints & BJ_HINT_COP) 260 rv = 3; /* calls copfunc with three arguments */ 261 262 if (hints & BJ_HINT_COPX) 263 rv = 4; /* uses BJ_TMP3REG */ 264 265 return rv; 266 } 267 268 /* 269 * Return a number of saved registers to pass 270 * to sljit_emit_enter() function. 271 */ 272 static sljit_si 273 nsaveds(bpfjit_hint_t hints) 274 { 275 sljit_si rv = 3; 276 277 if (hints & BJ_HINT_XREG) 278 rv = 4; /* uses BJ_XREG */ 279 280 #ifdef _KERNEL 281 if (hints & BJ_HINT_LDX) 282 rv = 5; /* uses BJ_ASAVE */ 283 #endif 284 285 return rv; 286 } 287 288 static uint32_t 289 read_width(const struct bpf_insn *pc) 290 { 291 292 switch (BPF_SIZE(pc->code)) { 293 case BPF_W: 294 return 4; 295 case BPF_H: 296 return 2; 297 case BPF_B: 298 return 1; 299 default: 300 BJ_ASSERT(false); 301 return 0; 302 } 303 } 304 305 /* 306 * Copy buf and buflen members of bpf_args from BJ_ARGS 307 * pointer to BJ_BUF and BJ_BUFLEN registers. 308 */ 309 static int 310 load_buf_buflen(struct sljit_compiler *compiler) 311 { 312 int status; 313 314 status = sljit_emit_op1(compiler, 315 SLJIT_MOV_P, 316 BJ_BUF, 0, 317 SLJIT_MEM1(BJ_ARGS), 318 offsetof(struct bpf_args, pkt)); 319 if (status != SLJIT_SUCCESS) 320 return status; 321 322 status = sljit_emit_op1(compiler, 323 SLJIT_MOV, /* size_t source */ 324 BJ_BUFLEN, 0, 325 SLJIT_MEM1(BJ_ARGS), 326 offsetof(struct bpf_args, buflen)); 327 328 return status; 329 } 330 331 static bool 332 grow_jumps(struct sljit_jump ***jumps, size_t *size) 333 { 334 struct sljit_jump **newptr; 335 const size_t elemsz = sizeof(struct sljit_jump *); 336 size_t old_size = *size; 337 size_t new_size = 2 * old_size; 338 339 if (new_size < old_size || new_size > SIZE_MAX / elemsz) 340 return false; 341 342 newptr = BJ_ALLOC(new_size * elemsz); 343 if (newptr == NULL) 344 return false; 345 346 memcpy(newptr, *jumps, old_size * elemsz); 347 BJ_FREE(*jumps, old_size * elemsz); 348 349 *jumps = newptr; 350 *size = new_size; 351 return true; 352 } 353 354 static bool 355 append_jump(struct sljit_jump *jump, struct sljit_jump ***jumps, 356 size_t *size, size_t *max_size) 357 { 358 if (*size == *max_size && !grow_jumps(jumps, max_size)) 359 return false; 360 361 (*jumps)[(*size)++] = jump; 362 return true; 363 } 364 365 /* 366 * Emit code for BPF_LD+BPF_B+BPF_ABS A <- P[k:1]. 367 */ 368 static int 369 emit_read8(struct sljit_compiler *compiler, sljit_si src, uint32_t k) 370 { 371 372 return sljit_emit_op1(compiler, 373 SLJIT_MOV_UB, 374 BJ_AREG, 0, 375 SLJIT_MEM1(src), k); 376 } 377 378 /* 379 * Emit code for BPF_LD+BPF_H+BPF_ABS A <- P[k:2]. 380 */ 381 static int 382 emit_read16(struct sljit_compiler *compiler, sljit_si src, uint32_t k) 383 { 384 int status; 385 386 BJ_ASSERT(k <= UINT32_MAX - 1); 387 388 /* A = buf[k]; */ 389 status = sljit_emit_op1(compiler, 390 SLJIT_MOV_UB, 391 BJ_AREG, 0, 392 SLJIT_MEM1(src), k); 393 if (status != SLJIT_SUCCESS) 394 return status; 395 396 /* tmp1 = buf[k+1]; */ 397 status = sljit_emit_op1(compiler, 398 SLJIT_MOV_UB, 399 BJ_TMP1REG, 0, 400 SLJIT_MEM1(src), k+1); 401 if (status != SLJIT_SUCCESS) 402 return status; 403 404 /* A = A << 8; */ 405 status = sljit_emit_op2(compiler, 406 SLJIT_SHL, 407 BJ_AREG, 0, 408 BJ_AREG, 0, 409 SLJIT_IMM, 8); 410 if (status != SLJIT_SUCCESS) 411 return status; 412 413 /* A = A + tmp1; */ 414 status = sljit_emit_op2(compiler, 415 SLJIT_ADD, 416 BJ_AREG, 0, 417 BJ_AREG, 0, 418 BJ_TMP1REG, 0); 419 return status; 420 } 421 422 /* 423 * Emit code for BPF_LD+BPF_W+BPF_ABS A <- P[k:4]. 424 */ 425 static int 426 emit_read32(struct sljit_compiler *compiler, sljit_si src, uint32_t k) 427 { 428 int status; 429 430 BJ_ASSERT(k <= UINT32_MAX - 3); 431 432 /* A = buf[k]; */ 433 status = sljit_emit_op1(compiler, 434 SLJIT_MOV_UB, 435 BJ_AREG, 0, 436 SLJIT_MEM1(src), k); 437 if (status != SLJIT_SUCCESS) 438 return status; 439 440 /* tmp1 = buf[k+1]; */ 441 status = sljit_emit_op1(compiler, 442 SLJIT_MOV_UB, 443 BJ_TMP1REG, 0, 444 SLJIT_MEM1(src), k+1); 445 if (status != SLJIT_SUCCESS) 446 return status; 447 448 /* A = A << 8; */ 449 status = sljit_emit_op2(compiler, 450 SLJIT_SHL, 451 BJ_AREG, 0, 452 BJ_AREG, 0, 453 SLJIT_IMM, 8); 454 if (status != SLJIT_SUCCESS) 455 return status; 456 457 /* A = A + tmp1; */ 458 status = sljit_emit_op2(compiler, 459 SLJIT_ADD, 460 BJ_AREG, 0, 461 BJ_AREG, 0, 462 BJ_TMP1REG, 0); 463 if (status != SLJIT_SUCCESS) 464 return status; 465 466 /* tmp1 = buf[k+2]; */ 467 status = sljit_emit_op1(compiler, 468 SLJIT_MOV_UB, 469 BJ_TMP1REG, 0, 470 SLJIT_MEM1(src), k+2); 471 if (status != SLJIT_SUCCESS) 472 return status; 473 474 /* A = A << 8; */ 475 status = sljit_emit_op2(compiler, 476 SLJIT_SHL, 477 BJ_AREG, 0, 478 BJ_AREG, 0, 479 SLJIT_IMM, 8); 480 if (status != SLJIT_SUCCESS) 481 return status; 482 483 /* A = A + tmp1; */ 484 status = sljit_emit_op2(compiler, 485 SLJIT_ADD, 486 BJ_AREG, 0, 487 BJ_AREG, 0, 488 BJ_TMP1REG, 0); 489 if (status != SLJIT_SUCCESS) 490 return status; 491 492 /* tmp1 = buf[k+3]; */ 493 status = sljit_emit_op1(compiler, 494 SLJIT_MOV_UB, 495 BJ_TMP1REG, 0, 496 SLJIT_MEM1(src), k+3); 497 if (status != SLJIT_SUCCESS) 498 return status; 499 500 /* A = A << 8; */ 501 status = sljit_emit_op2(compiler, 502 SLJIT_SHL, 503 BJ_AREG, 0, 504 BJ_AREG, 0, 505 SLJIT_IMM, 8); 506 if (status != SLJIT_SUCCESS) 507 return status; 508 509 /* A = A + tmp1; */ 510 status = sljit_emit_op2(compiler, 511 SLJIT_ADD, 512 BJ_AREG, 0, 513 BJ_AREG, 0, 514 BJ_TMP1REG, 0); 515 return status; 516 } 517 518 #ifdef _KERNEL 519 /* 520 * Emit code for m_xword/m_xhalf/m_xbyte call. 521 * 522 * @pc BPF_LD+BPF_W+BPF_ABS A <- P[k:4] 523 * BPF_LD+BPF_H+BPF_ABS A <- P[k:2] 524 * BPF_LD+BPF_B+BPF_ABS A <- P[k:1] 525 * BPF_LD+BPF_W+BPF_IND A <- P[X+k:4] 526 * BPF_LD+BPF_H+BPF_IND A <- P[X+k:2] 527 * BPF_LD+BPF_B+BPF_IND A <- P[X+k:1] 528 * BPF_LDX+BPF_B+BPF_MSH X <- 4*(P[k:1]&0xf) 529 */ 530 static int 531 emit_xcall(struct sljit_compiler *compiler, const struct bpf_insn *pc, 532 int dst, struct sljit_jump ***ret0, size_t *ret0_size, size_t *ret0_maxsize, 533 uint32_t (*fn)(const struct mbuf *, uint32_t, int *)) 534 { 535 struct sljit_jump *jump; 536 int status; 537 538 BJ_ASSERT(dst != BJ_ASAVE); 539 540 if (BPF_CLASS(pc->code) == BPF_LDX) { 541 /* save A */ 542 status = sljit_emit_op1(compiler, 543 SLJIT_MOV, 544 BJ_ASAVE, 0, 545 BJ_AREG, 0); 546 if (status != SLJIT_SUCCESS) 547 return status; 548 } 549 550 /* 551 * Prepare registers for fn(mbuf, k, &err) call. 552 */ 553 status = sljit_emit_op1(compiler, 554 SLJIT_MOV, 555 SLJIT_SCRATCH_REG1, 0, 556 BJ_BUF, 0); 557 if (status != SLJIT_SUCCESS) 558 return status; 559 560 if (BPF_CLASS(pc->code) == BPF_LD && BPF_MODE(pc->code) == BPF_IND) { 561 /* if (X > UINT32_MAX - pc->k) return 0; */ 562 jump = sljit_emit_cmp(compiler, 563 SLJIT_C_GREATER, 564 BJ_XREG, 0, 565 SLJIT_IMM, UINT32_MAX - pc->k); 566 if (jump == NULL) 567 return SLJIT_ERR_ALLOC_FAILED; 568 if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 569 return SLJIT_ERR_ALLOC_FAILED; 570 571 /* k = X + pc->k; */ 572 status = sljit_emit_op2(compiler, 573 SLJIT_ADD, 574 SLJIT_SCRATCH_REG2, 0, 575 BJ_XREG, 0, 576 SLJIT_IMM, (uint32_t)pc->k); 577 if (status != SLJIT_SUCCESS) 578 return status; 579 } else { 580 /* k = pc->k */ 581 status = sljit_emit_op1(compiler, 582 SLJIT_MOV, 583 SLJIT_SCRATCH_REG2, 0, 584 SLJIT_IMM, (uint32_t)pc->k); 585 if (status != SLJIT_SUCCESS) 586 return status; 587 } 588 589 /* 590 * The third argument of fn is an address on stack. 591 */ 592 status = sljit_get_local_base(compiler, 593 SLJIT_SCRATCH_REG3, 0, 594 offsetof(struct bpfjit_stack, err)); 595 if (status != SLJIT_SUCCESS) 596 return status; 597 598 /* fn(buf, k, &err); */ 599 status = sljit_emit_ijump(compiler, 600 SLJIT_CALL3, 601 SLJIT_IMM, SLJIT_FUNC_OFFSET(fn)); 602 if (status != SLJIT_SUCCESS) 603 return status; 604 605 if (dst != SLJIT_RETURN_REG) { 606 /* move return value to dst */ 607 status = sljit_emit_op1(compiler, 608 SLJIT_MOV, 609 dst, 0, 610 SLJIT_RETURN_REG, 0); 611 if (status != SLJIT_SUCCESS) 612 return status; 613 } 614 615 /* tmp2 = *err; */ 616 status = sljit_emit_op1(compiler, 617 SLJIT_MOV_UI, 618 BJ_TMP2REG, 0, 619 SLJIT_MEM1(SLJIT_LOCALS_REG), 620 offsetof(struct bpfjit_stack, err)); 621 if (status != SLJIT_SUCCESS) 622 return status; 623 624 /* if (tmp2 != 0) return 0; */ 625 jump = sljit_emit_cmp(compiler, 626 SLJIT_C_NOT_EQUAL, 627 BJ_TMP2REG, 0, 628 SLJIT_IMM, 0); 629 if (jump == NULL) 630 return SLJIT_ERR_ALLOC_FAILED; 631 632 if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 633 return SLJIT_ERR_ALLOC_FAILED; 634 635 if (BPF_CLASS(pc->code) == BPF_LDX) { 636 /* restore A */ 637 status = sljit_emit_op1(compiler, 638 SLJIT_MOV, 639 BJ_AREG, 0, 640 BJ_ASAVE, 0); 641 if (status != SLJIT_SUCCESS) 642 return status; 643 } 644 645 return SLJIT_SUCCESS; 646 } 647 #endif 648 649 /* 650 * Emit code for BPF_COP and BPF_COPX instructions. 651 */ 652 static int 653 emit_cop(struct sljit_compiler *compiler, 654 const bpf_ctx_t *bc, const struct bpf_insn *pc, 655 struct sljit_jump ***ret0, size_t *ret0_size, size_t *ret0_maxsize) 656 { 657 #if BJ_TMP3REG == SLJIT_SCRATCH_REG1 || \ 658 BJ_TMP3REG == SLJIT_SCRATCH_REG2 || \ 659 BJ_TMP3REG == SLJIT_SCRATCH_REG3 660 #error "Not supported assignment of registers." 661 #endif 662 663 struct sljit_jump *jump; 664 sljit_si call_reg; 665 sljit_sw call_off; 666 int status; 667 668 BJ_ASSERT(bc != NULL && bc->copfuncs != NULL); 669 670 if (BPF_MISCOP(pc->code) == BPF_COP) { 671 call_reg = SLJIT_IMM; 672 call_off = SLJIT_FUNC_OFFSET(bc->copfuncs[pc->k]); 673 } else { 674 /* if (X >= bc->nfuncs) return 0; */ 675 jump = sljit_emit_cmp(compiler, 676 SLJIT_C_GREATER_EQUAL, 677 BJ_XREG, 0, 678 SLJIT_IMM, bc->nfuncs); 679 if (jump == NULL) 680 return SLJIT_ERR_ALLOC_FAILED; 681 if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 682 return SLJIT_ERR_ALLOC_FAILED; 683 684 /* tmp1 = ctx; */ 685 status = sljit_emit_op1(compiler, 686 SLJIT_MOV_P, 687 BJ_TMP1REG, 0, 688 SLJIT_MEM1(SLJIT_LOCALS_REG), 689 offsetof(struct bpfjit_stack, ctx)); 690 if (status != SLJIT_SUCCESS) 691 return status; 692 693 /* tmp1 = ctx->copfuncs; */ 694 status = sljit_emit_op1(compiler, 695 SLJIT_MOV_P, 696 BJ_TMP1REG, 0, 697 SLJIT_MEM1(BJ_TMP1REG), 698 offsetof(struct bpf_ctx, copfuncs)); 699 if (status != SLJIT_SUCCESS) 700 return status; 701 702 /* tmp2 = X; */ 703 status = sljit_emit_op1(compiler, 704 SLJIT_MOV, 705 BJ_TMP2REG, 0, 706 BJ_XREG, 0); 707 if (status != SLJIT_SUCCESS) 708 return status; 709 710 /* tmp3 = ctx->copfuncs[tmp2]; */ 711 call_reg = BJ_TMP3REG; 712 call_off = 0; 713 status = sljit_emit_op1(compiler, 714 SLJIT_MOV_P, 715 call_reg, call_off, 716 SLJIT_MEM2(BJ_TMP1REG, BJ_TMP2REG), 717 SLJIT_WORD_SHIFT); 718 if (status != SLJIT_SUCCESS) 719 return status; 720 } 721 722 /* 723 * Copy bpf_copfunc_t arguments to registers. 724 */ 725 #if BJ_AREG != SLJIT_SCRATCH_REG3 726 status = sljit_emit_op1(compiler, 727 SLJIT_MOV_UI, 728 SLJIT_SCRATCH_REG3, 0, 729 BJ_AREG, 0); 730 if (status != SLJIT_SUCCESS) 731 return status; 732 #endif 733 734 status = sljit_emit_op1(compiler, 735 SLJIT_MOV_P, 736 SLJIT_SCRATCH_REG1, 0, 737 SLJIT_MEM1(SLJIT_LOCALS_REG), 738 offsetof(struct bpfjit_stack, ctx)); 739 if (status != SLJIT_SUCCESS) 740 return status; 741 742 status = sljit_emit_op1(compiler, 743 SLJIT_MOV_P, 744 SLJIT_SCRATCH_REG2, 0, 745 BJ_ARGS, 0); 746 if (status != SLJIT_SUCCESS) 747 return status; 748 749 status = sljit_emit_ijump(compiler, 750 SLJIT_CALL3, call_reg, call_off); 751 if (status != SLJIT_SUCCESS) 752 return status; 753 754 #if BJ_AREG != SLJIT_RETURN_REG 755 status = sljit_emit_op1(compiler, 756 SLJIT_MOV, 757 BJ_AREG, 0, 758 SLJIT_RETURN_REG, 0); 759 if (status != SLJIT_SUCCESS) 760 return status; 761 #endif 762 763 return SLJIT_SUCCESS; 764 } 765 766 /* 767 * Generate code for 768 * BPF_LD+BPF_W+BPF_ABS A <- P[k:4] 769 * BPF_LD+BPF_H+BPF_ABS A <- P[k:2] 770 * BPF_LD+BPF_B+BPF_ABS A <- P[k:1] 771 * BPF_LD+BPF_W+BPF_IND A <- P[X+k:4] 772 * BPF_LD+BPF_H+BPF_IND A <- P[X+k:2] 773 * BPF_LD+BPF_B+BPF_IND A <- P[X+k:1] 774 */ 775 static int 776 emit_pkt_read(struct sljit_compiler *compiler, 777 const struct bpf_insn *pc, struct sljit_jump *to_mchain_jump, 778 struct sljit_jump ***ret0, size_t *ret0_size, size_t *ret0_maxsize) 779 { 780 int status = SLJIT_ERR_ALLOC_FAILED; 781 uint32_t width; 782 sljit_si ld_reg; 783 struct sljit_jump *jump; 784 #ifdef _KERNEL 785 struct sljit_label *label; 786 struct sljit_jump *over_mchain_jump; 787 const bool check_zero_buflen = (to_mchain_jump != NULL); 788 #endif 789 const uint32_t k = pc->k; 790 791 #ifdef _KERNEL 792 if (to_mchain_jump == NULL) { 793 to_mchain_jump = sljit_emit_cmp(compiler, 794 SLJIT_C_EQUAL, 795 BJ_BUFLEN, 0, 796 SLJIT_IMM, 0); 797 if (to_mchain_jump == NULL) 798 return SLJIT_ERR_ALLOC_FAILED; 799 } 800 #endif 801 802 ld_reg = BJ_BUF; 803 width = read_width(pc); 804 805 if (BPF_MODE(pc->code) == BPF_IND) { 806 /* tmp1 = buflen - (pc->k + width); */ 807 status = sljit_emit_op2(compiler, 808 SLJIT_SUB, 809 BJ_TMP1REG, 0, 810 BJ_BUFLEN, 0, 811 SLJIT_IMM, k + width); 812 if (status != SLJIT_SUCCESS) 813 return status; 814 815 /* ld_reg = buf + X; */ 816 ld_reg = BJ_TMP2REG; 817 status = sljit_emit_op2(compiler, 818 SLJIT_ADD, 819 ld_reg, 0, 820 BJ_BUF, 0, 821 BJ_XREG, 0); 822 if (status != SLJIT_SUCCESS) 823 return status; 824 825 /* if (tmp1 < X) return 0; */ 826 jump = sljit_emit_cmp(compiler, 827 SLJIT_C_LESS, 828 BJ_TMP1REG, 0, 829 BJ_XREG, 0); 830 if (jump == NULL) 831 return SLJIT_ERR_ALLOC_FAILED; 832 if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 833 return SLJIT_ERR_ALLOC_FAILED; 834 } 835 836 switch (width) { 837 case 4: 838 status = emit_read32(compiler, ld_reg, k); 839 break; 840 case 2: 841 status = emit_read16(compiler, ld_reg, k); 842 break; 843 case 1: 844 status = emit_read8(compiler, ld_reg, k); 845 break; 846 } 847 848 if (status != SLJIT_SUCCESS) 849 return status; 850 851 #ifdef _KERNEL 852 over_mchain_jump = sljit_emit_jump(compiler, SLJIT_JUMP); 853 if (over_mchain_jump == NULL) 854 return SLJIT_ERR_ALLOC_FAILED; 855 856 /* entry point to mchain handler */ 857 label = sljit_emit_label(compiler); 858 if (label == NULL) 859 return SLJIT_ERR_ALLOC_FAILED; 860 sljit_set_label(to_mchain_jump, label); 861 862 if (check_zero_buflen) { 863 /* if (buflen != 0) return 0; */ 864 jump = sljit_emit_cmp(compiler, 865 SLJIT_C_NOT_EQUAL, 866 BJ_BUFLEN, 0, 867 SLJIT_IMM, 0); 868 if (jump == NULL) 869 return SLJIT_ERR_ALLOC_FAILED; 870 if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 871 return SLJIT_ERR_ALLOC_FAILED; 872 } 873 874 switch (width) { 875 case 4: 876 status = emit_xcall(compiler, pc, BJ_AREG, 877 ret0, ret0_size, ret0_maxsize, &m_xword); 878 break; 879 case 2: 880 status = emit_xcall(compiler, pc, BJ_AREG, 881 ret0, ret0_size, ret0_maxsize, &m_xhalf); 882 break; 883 case 1: 884 status = emit_xcall(compiler, pc, BJ_AREG, 885 ret0, ret0_size, ret0_maxsize, &m_xbyte); 886 break; 887 } 888 889 if (status != SLJIT_SUCCESS) 890 return status; 891 892 label = sljit_emit_label(compiler); 893 if (label == NULL) 894 return SLJIT_ERR_ALLOC_FAILED; 895 sljit_set_label(over_mchain_jump, label); 896 #endif 897 898 return SLJIT_SUCCESS; 899 } 900 901 static int 902 emit_memload(struct sljit_compiler *compiler, 903 sljit_si dst, uint32_t k, size_t extwords) 904 { 905 int status; 906 sljit_si src; 907 sljit_sw srcw; 908 909 srcw = k * sizeof(uint32_t); 910 911 if (extwords == 0) { 912 src = SLJIT_MEM1(SLJIT_LOCALS_REG); 913 srcw += offsetof(struct bpfjit_stack, mem); 914 } else { 915 /* copy extmem pointer to the tmp1 register */ 916 status = sljit_emit_op1(compiler, 917 SLJIT_MOV_P, 918 BJ_TMP1REG, 0, 919 SLJIT_MEM1(SLJIT_LOCALS_REG), 920 offsetof(struct bpfjit_stack, extmem)); 921 if (status != SLJIT_SUCCESS) 922 return status; 923 src = SLJIT_MEM1(BJ_TMP1REG); 924 } 925 926 return sljit_emit_op1(compiler, SLJIT_MOV_UI, dst, 0, src, srcw); 927 } 928 929 static int 930 emit_memstore(struct sljit_compiler *compiler, 931 sljit_si src, uint32_t k, size_t extwords) 932 { 933 int status; 934 sljit_si dst; 935 sljit_sw dstw; 936 937 dstw = k * sizeof(uint32_t); 938 939 if (extwords == 0) { 940 dst = SLJIT_MEM1(SLJIT_LOCALS_REG); 941 dstw += offsetof(struct bpfjit_stack, mem); 942 } else { 943 /* copy extmem pointer to the tmp1 register */ 944 status = sljit_emit_op1(compiler, 945 SLJIT_MOV_P, 946 BJ_TMP1REG, 0, 947 SLJIT_MEM1(SLJIT_LOCALS_REG), 948 offsetof(struct bpfjit_stack, extmem)); 949 if (status != SLJIT_SUCCESS) 950 return status; 951 dst = SLJIT_MEM1(BJ_TMP1REG); 952 } 953 954 return sljit_emit_op1(compiler, SLJIT_MOV_UI, dst, dstw, src, 0); 955 } 956 957 /* 958 * Emit code for BPF_LDX+BPF_B+BPF_MSH X <- 4*(P[k:1]&0xf). 959 */ 960 static int 961 emit_msh(struct sljit_compiler *compiler, 962 const struct bpf_insn *pc, struct sljit_jump *to_mchain_jump, 963 struct sljit_jump ***ret0, size_t *ret0_size, size_t *ret0_maxsize) 964 { 965 int status; 966 #ifdef _KERNEL 967 struct sljit_label *label; 968 struct sljit_jump *jump, *over_mchain_jump; 969 const bool check_zero_buflen = (to_mchain_jump != NULL); 970 #endif 971 const uint32_t k = pc->k; 972 973 #ifdef _KERNEL 974 if (to_mchain_jump == NULL) { 975 to_mchain_jump = sljit_emit_cmp(compiler, 976 SLJIT_C_EQUAL, 977 BJ_BUFLEN, 0, 978 SLJIT_IMM, 0); 979 if (to_mchain_jump == NULL) 980 return SLJIT_ERR_ALLOC_FAILED; 981 } 982 #endif 983 984 /* tmp1 = buf[k] */ 985 status = sljit_emit_op1(compiler, 986 SLJIT_MOV_UB, 987 BJ_TMP1REG, 0, 988 SLJIT_MEM1(BJ_BUF), k); 989 if (status != SLJIT_SUCCESS) 990 return status; 991 992 /* tmp1 &= 0xf */ 993 status = sljit_emit_op2(compiler, 994 SLJIT_AND, 995 BJ_TMP1REG, 0, 996 BJ_TMP1REG, 0, 997 SLJIT_IMM, 0xf); 998 if (status != SLJIT_SUCCESS) 999 return status; 1000 1001 /* tmp1 = tmp1 << 2 */ 1002 status = sljit_emit_op2(compiler, 1003 SLJIT_SHL, 1004 BJ_XREG, 0, 1005 BJ_TMP1REG, 0, 1006 SLJIT_IMM, 2); 1007 if (status != SLJIT_SUCCESS) 1008 return status; 1009 1010 #ifdef _KERNEL 1011 over_mchain_jump = sljit_emit_jump(compiler, SLJIT_JUMP); 1012 if (over_mchain_jump == NULL) 1013 return SLJIT_ERR_ALLOC_FAILED; 1014 1015 /* entry point to mchain handler */ 1016 label = sljit_emit_label(compiler); 1017 if (label == NULL) 1018 return SLJIT_ERR_ALLOC_FAILED; 1019 sljit_set_label(to_mchain_jump, label); 1020 1021 if (check_zero_buflen) { 1022 /* if (buflen != 0) return 0; */ 1023 jump = sljit_emit_cmp(compiler, 1024 SLJIT_C_NOT_EQUAL, 1025 BJ_BUFLEN, 0, 1026 SLJIT_IMM, 0); 1027 if (jump == NULL) 1028 return SLJIT_ERR_ALLOC_FAILED; 1029 if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 1030 return SLJIT_ERR_ALLOC_FAILED; 1031 } 1032 1033 status = emit_xcall(compiler, pc, BJ_TMP1REG, 1034 ret0, ret0_size, ret0_maxsize, &m_xbyte); 1035 if (status != SLJIT_SUCCESS) 1036 return status; 1037 1038 /* tmp1 &= 0xf */ 1039 status = sljit_emit_op2(compiler, 1040 SLJIT_AND, 1041 BJ_TMP1REG, 0, 1042 BJ_TMP1REG, 0, 1043 SLJIT_IMM, 0xf); 1044 if (status != SLJIT_SUCCESS) 1045 return status; 1046 1047 /* tmp1 = tmp1 << 2 */ 1048 status = sljit_emit_op2(compiler, 1049 SLJIT_SHL, 1050 BJ_XREG, 0, 1051 BJ_TMP1REG, 0, 1052 SLJIT_IMM, 2); 1053 if (status != SLJIT_SUCCESS) 1054 return status; 1055 1056 1057 label = sljit_emit_label(compiler); 1058 if (label == NULL) 1059 return SLJIT_ERR_ALLOC_FAILED; 1060 sljit_set_label(over_mchain_jump, label); 1061 #endif 1062 1063 return SLJIT_SUCCESS; 1064 } 1065 1066 static int 1067 emit_pow2_division(struct sljit_compiler *compiler, uint32_t k) 1068 { 1069 int shift = 0; 1070 int status = SLJIT_SUCCESS; 1071 1072 while (k > 1) { 1073 k >>= 1; 1074 shift++; 1075 } 1076 1077 BJ_ASSERT(k == 1 && shift < 32); 1078 1079 if (shift != 0) { 1080 status = sljit_emit_op2(compiler, 1081 SLJIT_LSHR|SLJIT_INT_OP, 1082 BJ_AREG, 0, 1083 BJ_AREG, 0, 1084 SLJIT_IMM, shift); 1085 } 1086 1087 return status; 1088 } 1089 1090 #if !defined(BPFJIT_USE_UDIV) 1091 static sljit_uw 1092 divide(sljit_uw x, sljit_uw y) 1093 { 1094 1095 return (uint32_t)x / (uint32_t)y; 1096 } 1097 #endif 1098 1099 /* 1100 * Emit code for A = A / div. 1101 * divt,divw are either SLJIT_IMM,pc->k or BJ_XREG,0. 1102 */ 1103 static int 1104 emit_division(struct sljit_compiler *compiler, int divt, sljit_sw divw) 1105 { 1106 int status; 1107 1108 #if BJ_AREG != SLJIT_SCRATCH_REG1 1109 status = sljit_emit_op1(compiler, 1110 SLJIT_MOV, 1111 SLJIT_SCRATCH_REG1, 0, 1112 BJ_AREG, 0); 1113 if (status != SLJIT_SUCCESS) 1114 return status; 1115 #endif 1116 1117 status = sljit_emit_op1(compiler, 1118 SLJIT_MOV, 1119 SLJIT_SCRATCH_REG2, 0, 1120 divt, divw); 1121 if (status != SLJIT_SUCCESS) 1122 return status; 1123 1124 #if defined(BPFJIT_USE_UDIV) 1125 status = sljit_emit_op0(compiler, SLJIT_UDIV|SLJIT_INT_OP); 1126 1127 #if BJ_AREG != SLJIT_SCRATCH_REG1 1128 status = sljit_emit_op1(compiler, 1129 SLJIT_MOV, 1130 BJ_AREG, 0, 1131 SLJIT_SCRATCH_REG1, 0); 1132 if (status != SLJIT_SUCCESS) 1133 return status; 1134 #endif 1135 #else 1136 status = sljit_emit_ijump(compiler, 1137 SLJIT_CALL2, 1138 SLJIT_IMM, SLJIT_FUNC_OFFSET(divide)); 1139 1140 #if BJ_AREG != SLJIT_RETURN_REG 1141 status = sljit_emit_op1(compiler, 1142 SLJIT_MOV, 1143 BJ_AREG, 0, 1144 SLJIT_RETURN_REG, 0); 1145 if (status != SLJIT_SUCCESS) 1146 return status; 1147 #endif 1148 #endif 1149 1150 return status; 1151 } 1152 1153 /* 1154 * Return true if pc is a "read from packet" instruction. 1155 * If length is not NULL and return value is true, *length will 1156 * be set to a safe length required to read a packet. 1157 */ 1158 static bool 1159 read_pkt_insn(const struct bpf_insn *pc, bpfjit_abc_length_t *length) 1160 { 1161 bool rv; 1162 bpfjit_abc_length_t width; 1163 1164 switch (BPF_CLASS(pc->code)) { 1165 default: 1166 rv = false; 1167 break; 1168 1169 case BPF_LD: 1170 rv = BPF_MODE(pc->code) == BPF_ABS || 1171 BPF_MODE(pc->code) == BPF_IND; 1172 if (rv) 1173 width = read_width(pc); 1174 break; 1175 1176 case BPF_LDX: 1177 rv = pc->code == (BPF_LDX|BPF_B|BPF_MSH); 1178 width = 1; 1179 break; 1180 } 1181 1182 if (rv && length != NULL) { 1183 /* 1184 * Values greater than UINT32_MAX will generate 1185 * unconditional "return 0". 1186 */ 1187 *length = (uint32_t)pc->k + width; 1188 } 1189 1190 return rv; 1191 } 1192 1193 static void 1194 optimize_init(struct bpfjit_insn_data *insn_dat, size_t insn_count) 1195 { 1196 size_t i; 1197 1198 for (i = 0; i < insn_count; i++) { 1199 SLIST_INIT(&insn_dat[i].bjumps); 1200 insn_dat[i].invalid = BJ_INIT_NOBITS; 1201 } 1202 } 1203 1204 /* 1205 * The function divides instructions into blocks. Destination of a jump 1206 * instruction starts a new block. BPF_RET and BPF_JMP instructions 1207 * terminate a block. Blocks are linear, that is, there are no jumps out 1208 * from the middle of a block and there are no jumps in to the middle of 1209 * a block. 1210 * 1211 * The function also sets bits in *initmask for memwords that 1212 * need to be initialized to zero. Note that this set should be empty 1213 * for any valid kernel filter program. 1214 */ 1215 static bool 1216 optimize_pass1(const bpf_ctx_t *bc, const struct bpf_insn *insns, 1217 struct bpfjit_insn_data *insn_dat, size_t insn_count, 1218 bpf_memword_init_t *initmask, bpfjit_hint_t *hints) 1219 { 1220 struct bpfjit_jump *jtf; 1221 size_t i; 1222 uint32_t jt, jf; 1223 bpfjit_abc_length_t length; 1224 bpf_memword_init_t invalid; /* borrowed from bpf_filter() */ 1225 bool unreachable; 1226 1227 const size_t memwords = GET_MEMWORDS(bc); 1228 1229 *hints = 0; 1230 *initmask = BJ_INIT_NOBITS; 1231 1232 unreachable = false; 1233 invalid = ~BJ_INIT_NOBITS; 1234 1235 for (i = 0; i < insn_count; i++) { 1236 if (!SLIST_EMPTY(&insn_dat[i].bjumps)) 1237 unreachable = false; 1238 insn_dat[i].unreachable = unreachable; 1239 1240 if (unreachable) 1241 continue; 1242 1243 invalid |= insn_dat[i].invalid; 1244 1245 if (read_pkt_insn(&insns[i], &length) && length > UINT32_MAX) 1246 unreachable = true; 1247 1248 switch (BPF_CLASS(insns[i].code)) { 1249 case BPF_RET: 1250 if (BPF_RVAL(insns[i].code) == BPF_A) 1251 *initmask |= invalid & BJ_INIT_ABIT; 1252 1253 unreachable = true; 1254 continue; 1255 1256 case BPF_LD: 1257 if (BPF_MODE(insns[i].code) == BPF_ABS) 1258 *hints |= BJ_HINT_ABS; 1259 1260 if (BPF_MODE(insns[i].code) == BPF_IND) { 1261 *hints |= BJ_HINT_IND | BJ_HINT_XREG; 1262 *initmask |= invalid & BJ_INIT_XBIT; 1263 } 1264 1265 if (BPF_MODE(insns[i].code) == BPF_MEM && 1266 (uint32_t)insns[i].k < memwords) { 1267 *initmask |= invalid & BJ_INIT_MBIT(insns[i].k); 1268 } 1269 1270 invalid &= ~BJ_INIT_ABIT; 1271 continue; 1272 1273 case BPF_LDX: 1274 *hints |= BJ_HINT_XREG | BJ_HINT_LDX; 1275 1276 if (BPF_MODE(insns[i].code) == BPF_MEM && 1277 (uint32_t)insns[i].k < memwords) { 1278 *initmask |= invalid & BJ_INIT_MBIT(insns[i].k); 1279 } 1280 1281 if (BPF_MODE(insns[i].code) == BPF_MSH && 1282 BPF_SIZE(insns[i].code) == BPF_B) { 1283 *hints |= BJ_HINT_MSH; 1284 } 1285 1286 invalid &= ~BJ_INIT_XBIT; 1287 continue; 1288 1289 case BPF_ST: 1290 *initmask |= invalid & BJ_INIT_ABIT; 1291 1292 if ((uint32_t)insns[i].k < memwords) 1293 invalid &= ~BJ_INIT_MBIT(insns[i].k); 1294 1295 continue; 1296 1297 case BPF_STX: 1298 *hints |= BJ_HINT_XREG; 1299 *initmask |= invalid & BJ_INIT_XBIT; 1300 1301 if ((uint32_t)insns[i].k < memwords) 1302 invalid &= ~BJ_INIT_MBIT(insns[i].k); 1303 1304 continue; 1305 1306 case BPF_ALU: 1307 *initmask |= invalid & BJ_INIT_ABIT; 1308 1309 if (insns[i].code != (BPF_ALU|BPF_NEG) && 1310 BPF_SRC(insns[i].code) == BPF_X) { 1311 *hints |= BJ_HINT_XREG; 1312 *initmask |= invalid & BJ_INIT_XBIT; 1313 } 1314 1315 invalid &= ~BJ_INIT_ABIT; 1316 continue; 1317 1318 case BPF_MISC: 1319 switch (BPF_MISCOP(insns[i].code)) { 1320 case BPF_TAX: // X <- A 1321 *hints |= BJ_HINT_XREG; 1322 *initmask |= invalid & BJ_INIT_ABIT; 1323 invalid &= ~BJ_INIT_XBIT; 1324 continue; 1325 1326 case BPF_TXA: // A <- X 1327 *hints |= BJ_HINT_XREG; 1328 *initmask |= invalid & BJ_INIT_XBIT; 1329 invalid &= ~BJ_INIT_ABIT; 1330 continue; 1331 1332 case BPF_COPX: 1333 *hints |= BJ_HINT_XREG | BJ_HINT_COPX; 1334 /* FALLTHROUGH */ 1335 1336 case BPF_COP: 1337 *hints |= BJ_HINT_COP; 1338 *initmask |= invalid & BJ_INIT_ABIT; 1339 invalid &= ~BJ_INIT_ABIT; 1340 continue; 1341 } 1342 1343 continue; 1344 1345 case BPF_JMP: 1346 /* Initialize abc_length for ABC pass. */ 1347 insn_dat[i].u.jdata.abc_length = MAX_ABC_LENGTH; 1348 1349 if (BPF_OP(insns[i].code) == BPF_JA) { 1350 jt = jf = insns[i].k; 1351 } else { 1352 jt = insns[i].jt; 1353 jf = insns[i].jf; 1354 } 1355 1356 if (jt >= insn_count - (i + 1) || 1357 jf >= insn_count - (i + 1)) { 1358 return false; 1359 } 1360 1361 if (jt > 0 && jf > 0) 1362 unreachable = true; 1363 1364 jt += i + 1; 1365 jf += i + 1; 1366 1367 jtf = insn_dat[i].u.jdata.jtf; 1368 1369 jtf[0].jdata = &insn_dat[i].u.jdata; 1370 SLIST_INSERT_HEAD(&insn_dat[jt].bjumps, 1371 &jtf[0], entries); 1372 1373 if (jf != jt) { 1374 jtf[1].jdata = &insn_dat[i].u.jdata; 1375 SLIST_INSERT_HEAD(&insn_dat[jf].bjumps, 1376 &jtf[1], entries); 1377 } 1378 1379 insn_dat[jf].invalid |= invalid; 1380 insn_dat[jt].invalid |= invalid; 1381 invalid = 0; 1382 1383 continue; 1384 } 1385 } 1386 1387 return true; 1388 } 1389 1390 /* 1391 * Array Bounds Check Elimination (ABC) pass. 1392 */ 1393 static void 1394 optimize_pass2(const bpf_ctx_t *bc, const struct bpf_insn *insns, 1395 struct bpfjit_insn_data *insn_dat, size_t insn_count) 1396 { 1397 struct bpfjit_jump *jmp; 1398 const struct bpf_insn *pc; 1399 struct bpfjit_insn_data *pd; 1400 size_t i; 1401 bpfjit_abc_length_t length, abc_length = 0; 1402 1403 const size_t extwords = GET_EXTWORDS(bc); 1404 1405 for (i = insn_count; i != 0; i--) { 1406 pc = &insns[i-1]; 1407 pd = &insn_dat[i-1]; 1408 1409 if (pd->unreachable) 1410 continue; 1411 1412 switch (BPF_CLASS(pc->code)) { 1413 case BPF_RET: 1414 /* 1415 * It's quite common for bpf programs to 1416 * check packet bytes in increasing order 1417 * and return zero if bytes don't match 1418 * specified critetion. Such programs disable 1419 * ABC optimization completely because for 1420 * every jump there is a branch with no read 1421 * instruction. 1422 * With no side effects, BPF_STMT(BPF_RET+BPF_K, 0) 1423 * is indistinguishable from out-of-bound load. 1424 * Therefore, abc_length can be set to 1425 * MAX_ABC_LENGTH and enable ABC for many 1426 * bpf programs. 1427 * If this optimization encounters any 1428 * instruction with a side effect, it will 1429 * reset abc_length. 1430 */ 1431 if (BPF_RVAL(pc->code) == BPF_K && pc->k == 0) 1432 abc_length = MAX_ABC_LENGTH; 1433 else 1434 abc_length = 0; 1435 break; 1436 1437 case BPF_MISC: 1438 if (BPF_MISCOP(pc->code) == BPF_COP || 1439 BPF_MISCOP(pc->code) == BPF_COPX) { 1440 /* COP instructions can have side effects. */ 1441 abc_length = 0; 1442 } 1443 break; 1444 1445 case BPF_ST: 1446 case BPF_STX: 1447 if (extwords != 0) { 1448 /* Write to memory is visible after a call. */ 1449 abc_length = 0; 1450 } 1451 break; 1452 1453 case BPF_JMP: 1454 abc_length = pd->u.jdata.abc_length; 1455 break; 1456 1457 default: 1458 if (read_pkt_insn(pc, &length)) { 1459 if (abc_length < length) 1460 abc_length = length; 1461 pd->u.rdata.abc_length = abc_length; 1462 } 1463 break; 1464 } 1465 1466 SLIST_FOREACH(jmp, &pd->bjumps, entries) { 1467 if (jmp->jdata->abc_length > abc_length) 1468 jmp->jdata->abc_length = abc_length; 1469 } 1470 } 1471 } 1472 1473 static void 1474 optimize_pass3(const struct bpf_insn *insns, 1475 struct bpfjit_insn_data *insn_dat, size_t insn_count) 1476 { 1477 struct bpfjit_jump *jmp; 1478 size_t i; 1479 bpfjit_abc_length_t checked_length = 0; 1480 1481 for (i = 0; i < insn_count; i++) { 1482 if (insn_dat[i].unreachable) 1483 continue; 1484 1485 SLIST_FOREACH(jmp, &insn_dat[i].bjumps, entries) { 1486 if (jmp->jdata->checked_length < checked_length) 1487 checked_length = jmp->jdata->checked_length; 1488 } 1489 1490 if (BPF_CLASS(insns[i].code) == BPF_JMP) { 1491 insn_dat[i].u.jdata.checked_length = checked_length; 1492 } else if (read_pkt_insn(&insns[i], NULL)) { 1493 struct bpfjit_read_pkt_data *rdata = 1494 &insn_dat[i].u.rdata; 1495 rdata->check_length = 0; 1496 if (checked_length < rdata->abc_length) { 1497 checked_length = rdata->abc_length; 1498 rdata->check_length = checked_length; 1499 } 1500 } 1501 } 1502 } 1503 1504 static bool 1505 optimize(const bpf_ctx_t *bc, const struct bpf_insn *insns, 1506 struct bpfjit_insn_data *insn_dat, size_t insn_count, 1507 bpf_memword_init_t *initmask, bpfjit_hint_t *hints) 1508 { 1509 1510 optimize_init(insn_dat, insn_count); 1511 1512 if (!optimize_pass1(bc, insns, insn_dat, insn_count, initmask, hints)) 1513 return false; 1514 1515 optimize_pass2(bc, insns, insn_dat, insn_count); 1516 optimize_pass3(insns, insn_dat, insn_count); 1517 1518 return true; 1519 } 1520 1521 /* 1522 * Convert BPF_ALU operations except BPF_NEG and BPF_DIV to sljit operation. 1523 */ 1524 static int 1525 bpf_alu_to_sljit_op(const struct bpf_insn *pc) 1526 { 1527 1528 /* 1529 * Note: all supported 64bit arches have 32bit multiply 1530 * instruction so SLJIT_INT_OP doesn't have any overhead. 1531 */ 1532 switch (BPF_OP(pc->code)) { 1533 case BPF_ADD: return SLJIT_ADD; 1534 case BPF_SUB: return SLJIT_SUB; 1535 case BPF_MUL: return SLJIT_MUL|SLJIT_INT_OP; 1536 case BPF_OR: return SLJIT_OR; 1537 case BPF_AND: return SLJIT_AND; 1538 case BPF_LSH: return SLJIT_SHL; 1539 case BPF_RSH: return SLJIT_LSHR|SLJIT_INT_OP; 1540 default: 1541 BJ_ASSERT(false); 1542 return 0; 1543 } 1544 } 1545 1546 /* 1547 * Convert BPF_JMP operations except BPF_JA to sljit condition. 1548 */ 1549 static int 1550 bpf_jmp_to_sljit_cond(const struct bpf_insn *pc, bool negate) 1551 { 1552 /* 1553 * Note: all supported 64bit arches have 32bit comparison 1554 * instructions so SLJIT_INT_OP doesn't have any overhead. 1555 */ 1556 int rv = SLJIT_INT_OP; 1557 1558 switch (BPF_OP(pc->code)) { 1559 case BPF_JGT: 1560 rv |= negate ? SLJIT_C_LESS_EQUAL : SLJIT_C_GREATER; 1561 break; 1562 case BPF_JGE: 1563 rv |= negate ? SLJIT_C_LESS : SLJIT_C_GREATER_EQUAL; 1564 break; 1565 case BPF_JEQ: 1566 rv |= negate ? SLJIT_C_NOT_EQUAL : SLJIT_C_EQUAL; 1567 break; 1568 case BPF_JSET: 1569 rv |= negate ? SLJIT_C_EQUAL : SLJIT_C_NOT_EQUAL; 1570 break; 1571 default: 1572 BJ_ASSERT(false); 1573 } 1574 1575 return rv; 1576 } 1577 1578 /* 1579 * Convert BPF_K and BPF_X to sljit register. 1580 */ 1581 static int 1582 kx_to_reg(const struct bpf_insn *pc) 1583 { 1584 1585 switch (BPF_SRC(pc->code)) { 1586 case BPF_K: return SLJIT_IMM; 1587 case BPF_X: return BJ_XREG; 1588 default: 1589 BJ_ASSERT(false); 1590 return 0; 1591 } 1592 } 1593 1594 static sljit_sw 1595 kx_to_reg_arg(const struct bpf_insn *pc) 1596 { 1597 1598 switch (BPF_SRC(pc->code)) { 1599 case BPF_K: return (uint32_t)pc->k; /* SLJIT_IMM, pc->k, */ 1600 case BPF_X: return 0; /* BJ_XREG, 0, */ 1601 default: 1602 BJ_ASSERT(false); 1603 return 0; 1604 } 1605 } 1606 1607 static bool 1608 generate_insn_code(struct sljit_compiler *compiler, const bpf_ctx_t *bc, 1609 const struct bpf_insn *insns, struct bpfjit_insn_data *insn_dat, 1610 size_t insn_count) 1611 { 1612 /* a list of jumps to out-of-bound return from a generated function */ 1613 struct sljit_jump **ret0; 1614 size_t ret0_size, ret0_maxsize; 1615 1616 struct sljit_jump *jump; 1617 struct sljit_label *label; 1618 const struct bpf_insn *pc; 1619 struct bpfjit_jump *bjump, *jtf; 1620 struct sljit_jump *to_mchain_jump; 1621 1622 size_t i; 1623 int status; 1624 int branching, negate; 1625 unsigned int rval, mode, src; 1626 uint32_t jt, jf; 1627 1628 bool unconditional_ret; 1629 bool rv; 1630 1631 const size_t extwords = GET_EXTWORDS(bc); 1632 const size_t memwords = GET_MEMWORDS(bc); 1633 1634 ret0 = NULL; 1635 rv = false; 1636 1637 ret0_size = 0; 1638 ret0_maxsize = 64; 1639 ret0 = BJ_ALLOC(ret0_maxsize * sizeof(ret0[0])); 1640 if (ret0 == NULL) 1641 goto fail; 1642 1643 /* reset sjump members of jdata */ 1644 for (i = 0; i < insn_count; i++) { 1645 if (insn_dat[i].unreachable || 1646 BPF_CLASS(insns[i].code) != BPF_JMP) { 1647 continue; 1648 } 1649 1650 jtf = insn_dat[i].u.jdata.jtf; 1651 jtf[0].sjump = jtf[1].sjump = NULL; 1652 } 1653 1654 /* main loop */ 1655 for (i = 0; i < insn_count; i++) { 1656 if (insn_dat[i].unreachable) 1657 continue; 1658 1659 /* 1660 * Resolve jumps to the current insn. 1661 */ 1662 label = NULL; 1663 SLIST_FOREACH(bjump, &insn_dat[i].bjumps, entries) { 1664 if (bjump->sjump != NULL) { 1665 if (label == NULL) 1666 label = sljit_emit_label(compiler); 1667 if (label == NULL) 1668 goto fail; 1669 sljit_set_label(bjump->sjump, label); 1670 } 1671 } 1672 1673 to_mchain_jump = NULL; 1674 unconditional_ret = false; 1675 1676 if (read_pkt_insn(&insns[i], NULL)) { 1677 if (insn_dat[i].u.rdata.check_length > UINT32_MAX) { 1678 /* Jump to "return 0" unconditionally. */ 1679 unconditional_ret = true; 1680 jump = sljit_emit_jump(compiler, SLJIT_JUMP); 1681 if (jump == NULL) 1682 goto fail; 1683 if (!append_jump(jump, &ret0, 1684 &ret0_size, &ret0_maxsize)) 1685 goto fail; 1686 } else if (insn_dat[i].u.rdata.check_length > 0) { 1687 /* if (buflen < check_length) return 0; */ 1688 jump = sljit_emit_cmp(compiler, 1689 SLJIT_C_LESS, 1690 BJ_BUFLEN, 0, 1691 SLJIT_IMM, 1692 insn_dat[i].u.rdata.check_length); 1693 if (jump == NULL) 1694 goto fail; 1695 #ifdef _KERNEL 1696 to_mchain_jump = jump; 1697 #else 1698 if (!append_jump(jump, &ret0, 1699 &ret0_size, &ret0_maxsize)) 1700 goto fail; 1701 #endif 1702 } 1703 } 1704 1705 pc = &insns[i]; 1706 switch (BPF_CLASS(pc->code)) { 1707 1708 default: 1709 goto fail; 1710 1711 case BPF_LD: 1712 /* BPF_LD+BPF_IMM A <- k */ 1713 if (pc->code == (BPF_LD|BPF_IMM)) { 1714 status = sljit_emit_op1(compiler, 1715 SLJIT_MOV, 1716 BJ_AREG, 0, 1717 SLJIT_IMM, (uint32_t)pc->k); 1718 if (status != SLJIT_SUCCESS) 1719 goto fail; 1720 1721 continue; 1722 } 1723 1724 /* BPF_LD+BPF_MEM A <- M[k] */ 1725 if (pc->code == (BPF_LD|BPF_MEM)) { 1726 if ((uint32_t)pc->k >= memwords) 1727 goto fail; 1728 status = emit_memload(compiler, 1729 BJ_AREG, pc->k, extwords); 1730 if (status != SLJIT_SUCCESS) 1731 goto fail; 1732 1733 continue; 1734 } 1735 1736 /* BPF_LD+BPF_W+BPF_LEN A <- len */ 1737 if (pc->code == (BPF_LD|BPF_W|BPF_LEN)) { 1738 status = sljit_emit_op1(compiler, 1739 SLJIT_MOV, /* size_t source */ 1740 BJ_AREG, 0, 1741 SLJIT_MEM1(BJ_ARGS), 1742 offsetof(struct bpf_args, wirelen)); 1743 if (status != SLJIT_SUCCESS) 1744 goto fail; 1745 1746 continue; 1747 } 1748 1749 mode = BPF_MODE(pc->code); 1750 if (mode != BPF_ABS && mode != BPF_IND) 1751 goto fail; 1752 1753 if (unconditional_ret) 1754 continue; 1755 1756 status = emit_pkt_read(compiler, pc, 1757 to_mchain_jump, &ret0, &ret0_size, &ret0_maxsize); 1758 if (status != SLJIT_SUCCESS) 1759 goto fail; 1760 1761 continue; 1762 1763 case BPF_LDX: 1764 mode = BPF_MODE(pc->code); 1765 1766 /* BPF_LDX+BPF_W+BPF_IMM X <- k */ 1767 if (mode == BPF_IMM) { 1768 if (BPF_SIZE(pc->code) != BPF_W) 1769 goto fail; 1770 status = sljit_emit_op1(compiler, 1771 SLJIT_MOV, 1772 BJ_XREG, 0, 1773 SLJIT_IMM, (uint32_t)pc->k); 1774 if (status != SLJIT_SUCCESS) 1775 goto fail; 1776 1777 continue; 1778 } 1779 1780 /* BPF_LDX+BPF_W+BPF_LEN X <- len */ 1781 if (mode == BPF_LEN) { 1782 if (BPF_SIZE(pc->code) != BPF_W) 1783 goto fail; 1784 status = sljit_emit_op1(compiler, 1785 SLJIT_MOV, /* size_t source */ 1786 BJ_XREG, 0, 1787 SLJIT_MEM1(BJ_ARGS), 1788 offsetof(struct bpf_args, wirelen)); 1789 if (status != SLJIT_SUCCESS) 1790 goto fail; 1791 1792 continue; 1793 } 1794 1795 /* BPF_LDX+BPF_W+BPF_MEM X <- M[k] */ 1796 if (mode == BPF_MEM) { 1797 if (BPF_SIZE(pc->code) != BPF_W) 1798 goto fail; 1799 if ((uint32_t)pc->k >= memwords) 1800 goto fail; 1801 status = emit_memload(compiler, 1802 BJ_XREG, pc->k, extwords); 1803 if (status != SLJIT_SUCCESS) 1804 goto fail; 1805 1806 continue; 1807 } 1808 1809 /* BPF_LDX+BPF_B+BPF_MSH X <- 4*(P[k:1]&0xf) */ 1810 if (mode != BPF_MSH || BPF_SIZE(pc->code) != BPF_B) 1811 goto fail; 1812 1813 if (unconditional_ret) 1814 continue; 1815 1816 status = emit_msh(compiler, pc, 1817 to_mchain_jump, &ret0, &ret0_size, &ret0_maxsize); 1818 if (status != SLJIT_SUCCESS) 1819 goto fail; 1820 1821 continue; 1822 1823 case BPF_ST: 1824 if (pc->code != BPF_ST || 1825 (uint32_t)pc->k >= memwords) { 1826 goto fail; 1827 } 1828 1829 status = emit_memstore(compiler, 1830 BJ_AREG, pc->k, extwords); 1831 if (status != SLJIT_SUCCESS) 1832 goto fail; 1833 1834 continue; 1835 1836 case BPF_STX: 1837 if (pc->code != BPF_STX || 1838 (uint32_t)pc->k >= memwords) { 1839 goto fail; 1840 } 1841 1842 status = emit_memstore(compiler, 1843 BJ_XREG, pc->k, extwords); 1844 if (status != SLJIT_SUCCESS) 1845 goto fail; 1846 1847 continue; 1848 1849 case BPF_ALU: 1850 if (pc->code == (BPF_ALU|BPF_NEG)) { 1851 status = sljit_emit_op1(compiler, 1852 SLJIT_NEG, 1853 BJ_AREG, 0, 1854 BJ_AREG, 0); 1855 if (status != SLJIT_SUCCESS) 1856 goto fail; 1857 1858 continue; 1859 } 1860 1861 if (BPF_OP(pc->code) != BPF_DIV) { 1862 status = sljit_emit_op2(compiler, 1863 bpf_alu_to_sljit_op(pc), 1864 BJ_AREG, 0, 1865 BJ_AREG, 0, 1866 kx_to_reg(pc), kx_to_reg_arg(pc)); 1867 if (status != SLJIT_SUCCESS) 1868 goto fail; 1869 1870 continue; 1871 } 1872 1873 /* BPF_DIV */ 1874 1875 src = BPF_SRC(pc->code); 1876 if (src != BPF_X && src != BPF_K) 1877 goto fail; 1878 1879 /* division by zero? */ 1880 if (src == BPF_X) { 1881 jump = sljit_emit_cmp(compiler, 1882 SLJIT_C_EQUAL|SLJIT_INT_OP, 1883 BJ_XREG, 0, 1884 SLJIT_IMM, 0); 1885 if (jump == NULL) 1886 goto fail; 1887 if (!append_jump(jump, &ret0, 1888 &ret0_size, &ret0_maxsize)) 1889 goto fail; 1890 } else if (pc->k == 0) { 1891 jump = sljit_emit_jump(compiler, SLJIT_JUMP); 1892 if (jump == NULL) 1893 goto fail; 1894 if (!append_jump(jump, &ret0, 1895 &ret0_size, &ret0_maxsize)) 1896 goto fail; 1897 } 1898 1899 if (src == BPF_X) { 1900 status = emit_division(compiler, BJ_XREG, 0); 1901 if (status != SLJIT_SUCCESS) 1902 goto fail; 1903 } else if (pc->k != 0) { 1904 if (pc->k & (pc->k - 1)) { 1905 status = emit_division(compiler, 1906 SLJIT_IMM, (uint32_t)pc->k); 1907 } else { 1908 status = emit_pow2_division(compiler, 1909 (uint32_t)pc->k); 1910 } 1911 if (status != SLJIT_SUCCESS) 1912 goto fail; 1913 } 1914 1915 continue; 1916 1917 case BPF_JMP: 1918 if (BPF_OP(pc->code) == BPF_JA) { 1919 jt = jf = pc->k; 1920 } else { 1921 jt = pc->jt; 1922 jf = pc->jf; 1923 } 1924 1925 negate = (jt == 0) ? 1 : 0; 1926 branching = (jt == jf) ? 0 : 1; 1927 jtf = insn_dat[i].u.jdata.jtf; 1928 1929 if (branching) { 1930 if (BPF_OP(pc->code) != BPF_JSET) { 1931 jump = sljit_emit_cmp(compiler, 1932 bpf_jmp_to_sljit_cond(pc, negate), 1933 BJ_AREG, 0, 1934 kx_to_reg(pc), kx_to_reg_arg(pc)); 1935 } else { 1936 status = sljit_emit_op2(compiler, 1937 SLJIT_AND, 1938 BJ_TMP1REG, 0, 1939 BJ_AREG, 0, 1940 kx_to_reg(pc), kx_to_reg_arg(pc)); 1941 if (status != SLJIT_SUCCESS) 1942 goto fail; 1943 1944 jump = sljit_emit_cmp(compiler, 1945 bpf_jmp_to_sljit_cond(pc, negate), 1946 BJ_TMP1REG, 0, 1947 SLJIT_IMM, 0); 1948 } 1949 1950 if (jump == NULL) 1951 goto fail; 1952 1953 BJ_ASSERT(jtf[negate].sjump == NULL); 1954 jtf[negate].sjump = jump; 1955 } 1956 1957 if (!branching || (jt != 0 && jf != 0)) { 1958 jump = sljit_emit_jump(compiler, SLJIT_JUMP); 1959 if (jump == NULL) 1960 goto fail; 1961 1962 BJ_ASSERT(jtf[branching].sjump == NULL); 1963 jtf[branching].sjump = jump; 1964 } 1965 1966 continue; 1967 1968 case BPF_RET: 1969 rval = BPF_RVAL(pc->code); 1970 if (rval == BPF_X) 1971 goto fail; 1972 1973 /* BPF_RET+BPF_K accept k bytes */ 1974 if (rval == BPF_K) { 1975 status = sljit_emit_return(compiler, 1976 SLJIT_MOV_UI, 1977 SLJIT_IMM, (uint32_t)pc->k); 1978 if (status != SLJIT_SUCCESS) 1979 goto fail; 1980 } 1981 1982 /* BPF_RET+BPF_A accept A bytes */ 1983 if (rval == BPF_A) { 1984 status = sljit_emit_return(compiler, 1985 SLJIT_MOV_UI, 1986 BJ_AREG, 0); 1987 if (status != SLJIT_SUCCESS) 1988 goto fail; 1989 } 1990 1991 continue; 1992 1993 case BPF_MISC: 1994 switch (BPF_MISCOP(pc->code)) { 1995 case BPF_TAX: 1996 status = sljit_emit_op1(compiler, 1997 SLJIT_MOV_UI, 1998 BJ_XREG, 0, 1999 BJ_AREG, 0); 2000 if (status != SLJIT_SUCCESS) 2001 goto fail; 2002 2003 continue; 2004 2005 case BPF_TXA: 2006 status = sljit_emit_op1(compiler, 2007 SLJIT_MOV, 2008 BJ_AREG, 0, 2009 BJ_XREG, 0); 2010 if (status != SLJIT_SUCCESS) 2011 goto fail; 2012 2013 continue; 2014 2015 case BPF_COP: 2016 case BPF_COPX: 2017 if (bc == NULL || bc->copfuncs == NULL) 2018 goto fail; 2019 if (BPF_MISCOP(pc->code) == BPF_COP && 2020 (uint32_t)pc->k >= bc->nfuncs) { 2021 goto fail; 2022 } 2023 2024 status = emit_cop(compiler, bc, pc, 2025 &ret0, &ret0_size, &ret0_maxsize); 2026 if (status != SLJIT_SUCCESS) 2027 goto fail; 2028 2029 continue; 2030 } 2031 2032 goto fail; 2033 } /* switch */ 2034 } /* main loop */ 2035 2036 BJ_ASSERT(ret0_size <= ret0_maxsize); 2037 2038 if (ret0_size > 0) { 2039 label = sljit_emit_label(compiler); 2040 if (label == NULL) 2041 goto fail; 2042 for (i = 0; i < ret0_size; i++) 2043 sljit_set_label(ret0[i], label); 2044 } 2045 2046 status = sljit_emit_return(compiler, 2047 SLJIT_MOV_UI, 2048 SLJIT_IMM, 0); 2049 if (status != SLJIT_SUCCESS) 2050 goto fail; 2051 2052 rv = true; 2053 2054 fail: 2055 if (ret0 != NULL) 2056 BJ_FREE(ret0, ret0_maxsize * sizeof(ret0[0])); 2057 2058 return rv; 2059 } 2060 2061 bpfjit_func_t 2062 bpfjit_generate_code(const bpf_ctx_t *bc, 2063 const struct bpf_insn *insns, size_t insn_count) 2064 { 2065 void *rv; 2066 struct sljit_compiler *compiler; 2067 2068 size_t i; 2069 int status; 2070 2071 /* optimization related */ 2072 bpf_memword_init_t initmask; 2073 bpfjit_hint_t hints; 2074 2075 /* memory store location for initial zero initialization */ 2076 sljit_si mem_reg; 2077 sljit_sw mem_off; 2078 2079 struct bpfjit_insn_data *insn_dat; 2080 2081 const size_t extwords = GET_EXTWORDS(bc); 2082 const size_t memwords = GET_MEMWORDS(bc); 2083 const bpf_memword_init_t preinited = extwords ? bc->preinited : 0; 2084 2085 rv = NULL; 2086 compiler = NULL; 2087 insn_dat = NULL; 2088 2089 if (memwords > MAX_MEMWORDS) 2090 goto fail; 2091 2092 if (insn_count == 0 || insn_count > SIZE_MAX / sizeof(insn_dat[0])) 2093 goto fail; 2094 2095 insn_dat = BJ_ALLOC(insn_count * sizeof(insn_dat[0])); 2096 if (insn_dat == NULL) 2097 goto fail; 2098 2099 if (!optimize(bc, insns, insn_dat, insn_count, &initmask, &hints)) 2100 goto fail; 2101 2102 compiler = sljit_create_compiler(); 2103 if (compiler == NULL) 2104 goto fail; 2105 2106 #if !defined(_KERNEL) && defined(SLJIT_VERBOSE) && SLJIT_VERBOSE 2107 sljit_compiler_verbose(compiler, stderr); 2108 #endif 2109 2110 status = sljit_emit_enter(compiler, 2, nscratches(hints), 2111 nsaveds(hints), sizeof(struct bpfjit_stack)); 2112 if (status != SLJIT_SUCCESS) 2113 goto fail; 2114 2115 if (hints & BJ_HINT_COP) { 2116 /* save ctx argument */ 2117 status = sljit_emit_op1(compiler, 2118 SLJIT_MOV_P, 2119 SLJIT_MEM1(SLJIT_LOCALS_REG), 2120 offsetof(struct bpfjit_stack, ctx), 2121 BJ_CTX_ARG, 0); 2122 if (status != SLJIT_SUCCESS) 2123 goto fail; 2124 } 2125 2126 if (extwords == 0) { 2127 mem_reg = SLJIT_MEM1(SLJIT_LOCALS_REG); 2128 mem_off = offsetof(struct bpfjit_stack, mem); 2129 } else { 2130 /* copy "mem" argument from bpf_args to bpfjit_stack */ 2131 status = sljit_emit_op1(compiler, 2132 SLJIT_MOV_P, 2133 BJ_TMP1REG, 0, 2134 SLJIT_MEM1(BJ_ARGS), offsetof(struct bpf_args, mem)); 2135 if (status != SLJIT_SUCCESS) 2136 goto fail; 2137 2138 status = sljit_emit_op1(compiler, 2139 SLJIT_MOV_P, 2140 SLJIT_MEM1(SLJIT_LOCALS_REG), 2141 offsetof(struct bpfjit_stack, extmem), 2142 BJ_TMP1REG, 0); 2143 if (status != SLJIT_SUCCESS) 2144 goto fail; 2145 2146 mem_reg = SLJIT_MEM1(BJ_TMP1REG); 2147 mem_off = 0; 2148 } 2149 2150 /* 2151 * Exclude pre-initialised external memory words but keep 2152 * initialization statuses of A and X registers in case 2153 * bc->preinited wrongly sets those two bits. 2154 */ 2155 initmask &= ~preinited | BJ_INIT_ABIT | BJ_INIT_XBIT; 2156 2157 #if defined(_KERNEL) 2158 /* bpf_filter() checks initialization of memwords. */ 2159 BJ_ASSERT((initmask & (BJ_INIT_MBIT(memwords) - 1)) == 0); 2160 #endif 2161 for (i = 0; i < memwords; i++) { 2162 if (initmask & BJ_INIT_MBIT(i)) { 2163 /* M[i] = 0; */ 2164 status = sljit_emit_op1(compiler, 2165 SLJIT_MOV_UI, 2166 mem_reg, mem_off + i * sizeof(uint32_t), 2167 SLJIT_IMM, 0); 2168 if (status != SLJIT_SUCCESS) 2169 goto fail; 2170 } 2171 } 2172 2173 if (initmask & BJ_INIT_ABIT) { 2174 /* A = 0; */ 2175 status = sljit_emit_op1(compiler, 2176 SLJIT_MOV, 2177 BJ_AREG, 0, 2178 SLJIT_IMM, 0); 2179 if (status != SLJIT_SUCCESS) 2180 goto fail; 2181 } 2182 2183 if (initmask & BJ_INIT_XBIT) { 2184 /* X = 0; */ 2185 status = sljit_emit_op1(compiler, 2186 SLJIT_MOV, 2187 BJ_XREG, 0, 2188 SLJIT_IMM, 0); 2189 if (status != SLJIT_SUCCESS) 2190 goto fail; 2191 } 2192 2193 status = load_buf_buflen(compiler); 2194 if (status != SLJIT_SUCCESS) 2195 goto fail; 2196 2197 if (!generate_insn_code(compiler, bc, insns, insn_dat, insn_count)) 2198 goto fail; 2199 2200 rv = sljit_generate_code(compiler); 2201 2202 fail: 2203 if (compiler != NULL) 2204 sljit_free_compiler(compiler); 2205 2206 if (insn_dat != NULL) 2207 BJ_FREE(insn_dat, insn_count * sizeof(insn_dat[0])); 2208 2209 return (bpfjit_func_t)rv; 2210 } 2211 2212 void 2213 bpfjit_free_code(bpfjit_func_t code) 2214 { 2215 2216 sljit_free_code((void *)code); 2217 } 2218
Indexes created Sat Sep 27 14:09:57 GMT 2025