bpfjit.c revision 1.22
1 /* $NetBSD: bpfjit.c,v 1.22 2014/07/08 11:30:31 alnsn Exp $ */ 2 3 /*- 4 * Copyright (c) 2011-2014 Alexander Nasonov. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 21 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 22 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 23 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 24 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 25 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 26 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 27 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 28 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 #ifdef _KERNEL 34 __KERNEL_RCSID(0, "$NetBSD: bpfjit.c,v 1.22 2014/07/08 11:30:31 alnsn Exp $"); 35 #else 36 __RCSID("$NetBSD: bpfjit.c,v 1.22 2014/07/08 11:30:31 alnsn Exp $"); 37 #endif 38 39 #include <sys/types.h> 40 #include <sys/queue.h> 41 42 #ifndef _KERNEL 43 #include <assert.h> 44 #define BJ_ASSERT(c) assert(c) 45 #else 46 #define BJ_ASSERT(c) KASSERT(c) 47 #endif 48 49 #ifndef _KERNEL 50 #include <stdlib.h> 51 #define BJ_ALLOC(sz) malloc(sz) 52 #define BJ_FREE(p, sz) free(p) 53 #else 54 #include <sys/kmem.h> 55 #define BJ_ALLOC(sz) kmem_alloc(sz, KM_SLEEP) 56 #define BJ_FREE(p, sz) kmem_free(p, sz) 57 #endif 58 59 #ifndef _KERNEL 60 #include <limits.h> 61 #include <stdbool.h> 62 #include <stddef.h> 63 #include <stdint.h> 64 #else 65 #include <sys/atomic.h> 66 #include <sys/module.h> 67 #endif 68 69 #define __BPF_PRIVATE 70 #include <net/bpf.h> 71 #include <net/bpfjit.h> 72 #include <sljitLir.h> 73 74 #if !defined(_KERNEL) && defined(SLJIT_VERBOSE) && SLJIT_VERBOSE 75 #include <stdio.h> /* for stderr */ 76 #endif 77 78 /* 79 * Arguments of generated bpfjit_func_t. 80 * The first argument is reassigned upon entry 81 * to a more frequently used buf argument. 82 */ 83 #define BJ_CTX_ARG SLJIT_SAVED_REG1 84 #define BJ_ARGS SLJIT_SAVED_REG2 85 86 /* 87 * Permanent register assignments. 88 */ 89 #define BJ_BUF SLJIT_SAVED_REG1 90 //#define BJ_ARGS SLJIT_SAVED_REG2 91 #define BJ_BUFLEN SLJIT_SAVED_REG3 92 #define BJ_AREG SLJIT_SCRATCH_REG1 93 #define BJ_TMP1REG SLJIT_SCRATCH_REG2 94 #define BJ_TMP2REG SLJIT_SCRATCH_REG3 95 #define BJ_XREG SLJIT_TEMPORARY_EREG1 96 #define BJ_TMP3REG SLJIT_TEMPORARY_EREG2 97 98 /* 99 * EREG registers can't be used for indirect calls, reuse BJ_BUF and 100 * BJ_BUFLEN registers. They can be easily restored from BJ_ARGS. 101 */ 102 #define BJ_COPF_PTR SLJIT_SAVED_REG1 103 #define BJ_COPF_IDX SLJIT_SAVED_REG3 104 105 #ifdef _KERNEL 106 #define MAX_MEMWORDS BPF_MAX_MEMWORDS 107 #else 108 #define MAX_MEMWORDS BPF_MEMWORDS 109 #endif 110 111 #define BJ_INIT_NOBITS ((bpf_memword_init_t)0) 112 #define BJ_INIT_MBIT(k) BPF_MEMWORD_INIT(k) 113 #define BJ_INIT_ABIT BJ_INIT_MBIT(MAX_MEMWORDS) 114 #define BJ_INIT_XBIT BJ_INIT_MBIT(MAX_MEMWORDS + 1) 115 116 /* 117 * Get a number of memwords and external memwords from a bpf_ctx object. 118 */ 119 #define GET_EXTWORDS(bc) ((bc) ? (bc)->extwords : 0) 120 #define GET_MEMWORDS(bc) (GET_EXTWORDS(bc) ? GET_EXTWORDS(bc) : BPF_MEMWORDS) 121 122 /* 123 * Optimization hints. 124 */ 125 typedef unsigned int bpfjit_hint_t; 126 #define BJ_HINT_LDW 0x01 /* 32-bit packet read */ 127 #define BJ_HINT_IND 0x02 /* packet read at a variable offset */ 128 #define BJ_HINT_COP 0x04 /* BPF_COP or BPF_COPX instruction */ 129 #define BJ_HINT_XREG 0x08 /* BJ_XREG is needed */ 130 #define BJ_HINT_LDX 0x10 /* BPF_LDX instruction */ 131 132 /* 133 * Datatype for Array Bounds Check Elimination (ABC) pass. 134 */ 135 typedef uint64_t bpfjit_abc_length_t; 136 #define MAX_ABC_LENGTH (UINT32_MAX + UINT64_C(4)) /* max. width is 4 */ 137 138 struct bpfjit_stack 139 { 140 bpf_ctx_t *ctx; 141 uint32_t *extmem; /* pointer to external memory store */ 142 #ifdef _KERNEL 143 int err; /* 3rd argument for m_xword/m_xhalf/m_xbyte function call */ 144 #endif 145 uint32_t mem[BPF_MEMWORDS]; /* internal memory store */ 146 }; 147 148 /* 149 * Data for BPF_JMP instruction. 150 * Forward declaration for struct bpfjit_jump. 151 */ 152 struct bpfjit_jump_data; 153 154 /* 155 * Node of bjumps list. 156 */ 157 struct bpfjit_jump { 158 struct sljit_jump *sjump; 159 SLIST_ENTRY(bpfjit_jump) entries; 160 struct bpfjit_jump_data *jdata; 161 }; 162 163 /* 164 * Data for BPF_JMP instruction. 165 */ 166 struct bpfjit_jump_data { 167 /* 168 * These entries make up bjumps list: 169 * jtf[0] - when coming from jt path, 170 * jtf[1] - when coming from jf path. 171 */ 172 struct bpfjit_jump jtf[2]; 173 /* 174 * Length calculated by Array Bounds Check Elimination (ABC) pass. 175 */ 176 bpfjit_abc_length_t abc_length; 177 /* 178 * Length checked by the last out-of-bounds check. 179 */ 180 bpfjit_abc_length_t checked_length; 181 }; 182 183 /* 184 * Data for "read from packet" instructions. 185 * See also read_pkt_insn() function below. 186 */ 187 struct bpfjit_read_pkt_data { 188 /* 189 * Length calculated by Array Bounds Check Elimination (ABC) pass. 190 */ 191 bpfjit_abc_length_t abc_length; 192 /* 193 * If positive, emit "if (buflen < check_length) return 0" 194 * out-of-bounds check. 195 * Values greater than UINT32_MAX generate unconditional "return 0". 196 */ 197 bpfjit_abc_length_t check_length; 198 }; 199 200 /* 201 * Additional (optimization-related) data for bpf_insn. 202 */ 203 struct bpfjit_insn_data { 204 /* List of jumps to this insn. */ 205 SLIST_HEAD(, bpfjit_jump) bjumps; 206 207 union { 208 struct bpfjit_jump_data jdata; 209 struct bpfjit_read_pkt_data rdata; 210 } u; 211 212 bpf_memword_init_t invalid; 213 bool unreachable; 214 }; 215 216 #ifdef _KERNEL 217 218 uint32_t m_xword(const struct mbuf *, uint32_t, int *); 219 uint32_t m_xhalf(const struct mbuf *, uint32_t, int *); 220 uint32_t m_xbyte(const struct mbuf *, uint32_t, int *); 221 222 MODULE(MODULE_CLASS_MISC, bpfjit, "sljit") 223 224 static int 225 bpfjit_modcmd(modcmd_t cmd, void *arg) 226 { 227 228 switch (cmd) { 229 case MODULE_CMD_INIT: 230 bpfjit_module_ops.bj_free_code = &bpfjit_free_code; 231 membar_producer(); 232 bpfjit_module_ops.bj_generate_code = &bpfjit_generate_code; 233 membar_producer(); 234 return 0; 235 236 case MODULE_CMD_FINI: 237 return EOPNOTSUPP; 238 239 default: 240 return ENOTTY; 241 } 242 } 243 #endif 244 245 /* 246 * Return a number of scratch registers to pass 247 * to sljit_emit_enter() function. 248 */ 249 static sljit_si 250 nscratches(bpfjit_hint_t hints) 251 { 252 sljit_si rv = 2; 253 254 #ifdef _KERNEL 255 /* 256 * Most kernel programs load packet bytes and they generate 257 * m_xword/m_xhalf/m_xbyte() calls with three arguments. 258 */ 259 rv = 3; 260 #endif 261 262 if (hints & BJ_HINT_LDW) 263 rv = 3; /* uses BJ_TMP2REG */ 264 265 if (hints & BJ_HINT_COP) 266 rv = 3; /* calls copfunc with three arguments */ 267 268 if (hints & BJ_HINT_XREG) 269 rv = 4; /* uses BJ_XREG */ 270 271 #ifdef _KERNEL 272 if (hints & BJ_HINT_LDX) 273 rv = 5; /* uses BJ_TMP3REG */ 274 #endif 275 276 return rv; 277 } 278 279 static uint32_t 280 read_width(const struct bpf_insn *pc) 281 { 282 283 switch (BPF_SIZE(pc->code)) { 284 case BPF_W: 285 return 4; 286 case BPF_H: 287 return 2; 288 case BPF_B: 289 return 1; 290 default: 291 BJ_ASSERT(false); 292 return 0; 293 } 294 } 295 296 /* 297 * Copy buf and buflen members of bpf_args from BJ_ARGS 298 * pointer to BJ_BUF and BJ_BUFLEN registers. 299 */ 300 static int 301 load_buf_buflen(struct sljit_compiler *compiler) 302 { 303 int status; 304 305 status = sljit_emit_op1(compiler, 306 SLJIT_MOV_P, 307 BJ_BUF, 0, 308 SLJIT_MEM1(BJ_ARGS), 309 offsetof(struct bpf_args, pkt)); 310 if (status != SLJIT_SUCCESS) 311 return status; 312 313 status = sljit_emit_op1(compiler, 314 SLJIT_MOV, /* size_t source */ 315 BJ_BUFLEN, 0, 316 SLJIT_MEM1(BJ_ARGS), 317 offsetof(struct bpf_args, buflen)); 318 319 return status; 320 } 321 322 static bool 323 grow_jumps(struct sljit_jump ***jumps, size_t *size) 324 { 325 struct sljit_jump **newptr; 326 const size_t elemsz = sizeof(struct sljit_jump *); 327 size_t old_size = *size; 328 size_t new_size = 2 * old_size; 329 330 if (new_size < old_size || new_size > SIZE_MAX / elemsz) 331 return false; 332 333 newptr = BJ_ALLOC(new_size * elemsz); 334 if (newptr == NULL) 335 return false; 336 337 memcpy(newptr, *jumps, old_size * elemsz); 338 BJ_FREE(*jumps, old_size * elemsz); 339 340 *jumps = newptr; 341 *size = new_size; 342 return true; 343 } 344 345 static bool 346 append_jump(struct sljit_jump *jump, struct sljit_jump ***jumps, 347 size_t *size, size_t *max_size) 348 { 349 if (*size == *max_size && !grow_jumps(jumps, max_size)) 350 return false; 351 352 (*jumps)[(*size)++] = jump; 353 return true; 354 } 355 356 /* 357 * Generate code for BPF_LD+BPF_B+BPF_ABS A <- P[k:1]. 358 */ 359 static int 360 emit_read8(struct sljit_compiler *compiler, uint32_t k) 361 { 362 363 return sljit_emit_op1(compiler, 364 SLJIT_MOV_UB, 365 BJ_AREG, 0, 366 SLJIT_MEM1(BJ_BUF), k); 367 } 368 369 /* 370 * Generate code for BPF_LD+BPF_H+BPF_ABS A <- P[k:2]. 371 */ 372 static int 373 emit_read16(struct sljit_compiler *compiler, uint32_t k) 374 { 375 int status; 376 377 /* tmp1 = buf[k]; */ 378 status = sljit_emit_op1(compiler, 379 SLJIT_MOV_UB, 380 BJ_TMP1REG, 0, 381 SLJIT_MEM1(BJ_BUF), k); 382 if (status != SLJIT_SUCCESS) 383 return status; 384 385 /* A = buf[k+1]; */ 386 status = sljit_emit_op1(compiler, 387 SLJIT_MOV_UB, 388 BJ_AREG, 0, 389 SLJIT_MEM1(BJ_BUF), k+1); 390 if (status != SLJIT_SUCCESS) 391 return status; 392 393 /* tmp1 = tmp1 << 8; */ 394 status = sljit_emit_op2(compiler, 395 SLJIT_SHL, 396 BJ_TMP1REG, 0, 397 BJ_TMP1REG, 0, 398 SLJIT_IMM, 8); 399 if (status != SLJIT_SUCCESS) 400 return status; 401 402 /* A = A + tmp1; */ 403 status = sljit_emit_op2(compiler, 404 SLJIT_ADD, 405 BJ_AREG, 0, 406 BJ_AREG, 0, 407 BJ_TMP1REG, 0); 408 return status; 409 } 410 411 /* 412 * Generate code for BPF_LD+BPF_W+BPF_ABS A <- P[k:4]. 413 */ 414 static int 415 emit_read32(struct sljit_compiler *compiler, uint32_t k) 416 { 417 int status; 418 419 /* tmp1 = buf[k]; */ 420 status = sljit_emit_op1(compiler, 421 SLJIT_MOV_UB, 422 BJ_TMP1REG, 0, 423 SLJIT_MEM1(BJ_BUF), k); 424 if (status != SLJIT_SUCCESS) 425 return status; 426 427 /* tmp2 = buf[k+1]; */ 428 status = sljit_emit_op1(compiler, 429 SLJIT_MOV_UB, 430 BJ_TMP2REG, 0, 431 SLJIT_MEM1(BJ_BUF), k+1); 432 if (status != SLJIT_SUCCESS) 433 return status; 434 435 /* A = buf[k+3]; */ 436 status = sljit_emit_op1(compiler, 437 SLJIT_MOV_UB, 438 BJ_AREG, 0, 439 SLJIT_MEM1(BJ_BUF), k+3); 440 if (status != SLJIT_SUCCESS) 441 return status; 442 443 /* tmp1 = tmp1 << 24; */ 444 status = sljit_emit_op2(compiler, 445 SLJIT_SHL, 446 BJ_TMP1REG, 0, 447 BJ_TMP1REG, 0, 448 SLJIT_IMM, 24); 449 if (status != SLJIT_SUCCESS) 450 return status; 451 452 /* A = A + tmp1; */ 453 status = sljit_emit_op2(compiler, 454 SLJIT_ADD, 455 BJ_AREG, 0, 456 BJ_AREG, 0, 457 BJ_TMP1REG, 0); 458 if (status != SLJIT_SUCCESS) 459 return status; 460 461 /* tmp1 = buf[k+2]; */ 462 status = sljit_emit_op1(compiler, 463 SLJIT_MOV_UB, 464 BJ_TMP1REG, 0, 465 SLJIT_MEM1(BJ_BUF), k+2); 466 if (status != SLJIT_SUCCESS) 467 return status; 468 469 /* tmp2 = tmp2 << 16; */ 470 status = sljit_emit_op2(compiler, 471 SLJIT_SHL, 472 BJ_TMP2REG, 0, 473 BJ_TMP2REG, 0, 474 SLJIT_IMM, 16); 475 if (status != SLJIT_SUCCESS) 476 return status; 477 478 /* A = A + tmp2; */ 479 status = sljit_emit_op2(compiler, 480 SLJIT_ADD, 481 BJ_AREG, 0, 482 BJ_AREG, 0, 483 BJ_TMP2REG, 0); 484 if (status != SLJIT_SUCCESS) 485 return status; 486 487 /* tmp1 = tmp1 << 8; */ 488 status = sljit_emit_op2(compiler, 489 SLJIT_SHL, 490 BJ_TMP1REG, 0, 491 BJ_TMP1REG, 0, 492 SLJIT_IMM, 8); 493 if (status != SLJIT_SUCCESS) 494 return status; 495 496 /* A = A + tmp1; */ 497 status = sljit_emit_op2(compiler, 498 SLJIT_ADD, 499 BJ_AREG, 0, 500 BJ_AREG, 0, 501 BJ_TMP1REG, 0); 502 return status; 503 } 504 505 #ifdef _KERNEL 506 /* 507 * Generate m_xword/m_xhalf/m_xbyte call. 508 * 509 * pc is one of: 510 * BPF_LD+BPF_W+BPF_ABS A <- P[k:4] 511 * BPF_LD+BPF_H+BPF_ABS A <- P[k:2] 512 * BPF_LD+BPF_B+BPF_ABS A <- P[k:1] 513 * BPF_LD+BPF_W+BPF_IND A <- P[X+k:4] 514 * BPF_LD+BPF_H+BPF_IND A <- P[X+k:2] 515 * BPF_LD+BPF_B+BPF_IND A <- P[X+k:1] 516 * BPF_LDX+BPF_B+BPF_MSH X <- 4*(P[k:1]&0xf) 517 * 518 * The dst variable should be 519 * - BJ_AREG when emitting code for BPF_LD instructions, 520 * - BJ_XREG or any of BJ_TMP[1-3]REG registers when emitting 521 * code for BPF_MSH instruction. 522 */ 523 static int 524 emit_xcall(struct sljit_compiler *compiler, const struct bpf_insn *pc, 525 int dst, sljit_sw dstw, struct sljit_jump **ret0_jump, 526 uint32_t (*fn)(const struct mbuf *, uint32_t, int *)) 527 { 528 #if BJ_XREG == SLJIT_RETURN_REG || \ 529 BJ_XREG == SLJIT_SCRATCH_REG1 || \ 530 BJ_XREG == SLJIT_SCRATCH_REG2 || \ 531 BJ_XREG == SLJIT_SCRATCH_REG3 532 #error "Not supported assignment of registers." 533 #endif 534 int status; 535 536 if (BPF_CLASS(pc->code) == BPF_LDX) { 537 /* save A */ 538 status = sljit_emit_op1(compiler, 539 SLJIT_MOV, 540 BJ_TMP3REG, 0, 541 BJ_AREG, 0); 542 if (status != SLJIT_SUCCESS) 543 return status; 544 } 545 546 /* 547 * Prepare registers for fn(buf, k, &err) call. 548 */ 549 status = sljit_emit_op1(compiler, 550 SLJIT_MOV, 551 SLJIT_SCRATCH_REG1, 0, 552 BJ_BUF, 0); 553 if (status != SLJIT_SUCCESS) 554 return status; 555 556 if (BPF_CLASS(pc->code) == BPF_LD && BPF_MODE(pc->code) == BPF_IND) { 557 status = sljit_emit_op2(compiler, 558 SLJIT_ADD, 559 SLJIT_SCRATCH_REG2, 0, 560 BJ_XREG, 0, 561 SLJIT_IMM, (uint32_t)pc->k); 562 } else { 563 status = sljit_emit_op1(compiler, 564 SLJIT_MOV, 565 SLJIT_SCRATCH_REG2, 0, 566 SLJIT_IMM, (uint32_t)pc->k); 567 } 568 569 if (status != SLJIT_SUCCESS) 570 return status; 571 572 /* 573 * The third argument of fn is an address on stack. 574 */ 575 status = sljit_get_local_base(compiler, 576 SLJIT_SCRATCH_REG3, 0, 577 offsetof(struct bpfjit_stack, err)); 578 if (status != SLJIT_SUCCESS) 579 return status; 580 581 /* fn(buf, k, &err); */ 582 status = sljit_emit_ijump(compiler, 583 SLJIT_CALL3, 584 SLJIT_IMM, SLJIT_FUNC_OFFSET(fn)); 585 586 if (dst != SLJIT_RETURN_REG) { 587 /* move return value to dst */ 588 status = sljit_emit_op1(compiler, 589 SLJIT_MOV, 590 dst, dstw, 591 SLJIT_RETURN_REG, 0); 592 if (status != SLJIT_SUCCESS) 593 return status; 594 } 595 596 if (BPF_CLASS(pc->code) == BPF_LDX) { 597 /* restore A */ 598 status = sljit_emit_op1(compiler, 599 SLJIT_MOV, 600 BJ_AREG, 0, 601 BJ_TMP3REG, 0); 602 if (status != SLJIT_SUCCESS) 603 return status; 604 } 605 606 /* tmp3 = *err; */ 607 status = sljit_emit_op1(compiler, 608 SLJIT_MOV_UI, 609 SLJIT_SCRATCH_REG3, 0, 610 SLJIT_MEM1(SLJIT_LOCALS_REG), 611 offsetof(struct bpfjit_stack, err)); 612 if (status != SLJIT_SUCCESS) 613 return status; 614 615 /* if (tmp3 != 0) return 0; */ 616 *ret0_jump = sljit_emit_cmp(compiler, 617 SLJIT_C_NOT_EQUAL, 618 SLJIT_SCRATCH_REG3, 0, 619 SLJIT_IMM, 0); 620 if (*ret0_jump == NULL) 621 return SLJIT_ERR_ALLOC_FAILED; 622 623 return status; 624 } 625 #endif 626 627 /* 628 * Emit code for BPF_COP and BPF_COPX instructions. 629 */ 630 static int 631 emit_cop(struct sljit_compiler *compiler, const bpf_ctx_t *bc, 632 const struct bpf_insn *pc, struct sljit_jump **ret0_jump) 633 { 634 #if BJ_XREG == SLJIT_RETURN_REG || \ 635 BJ_XREG == SLJIT_SCRATCH_REG1 || \ 636 BJ_XREG == SLJIT_SCRATCH_REG2 || \ 637 BJ_XREG == SLJIT_SCRATCH_REG3 || \ 638 BJ_COPF_PTR == BJ_ARGS || \ 639 BJ_COPF_IDX == BJ_ARGS 640 #error "Not supported assignment of registers." 641 #endif 642 643 struct sljit_jump *jump; 644 int status; 645 646 jump = NULL; 647 648 BJ_ASSERT(bc != NULL && bc->copfuncs != NULL); 649 650 if (BPF_MISCOP(pc->code) == BPF_COPX) { 651 /* if (X >= bc->nfuncs) return 0; */ 652 jump = sljit_emit_cmp(compiler, 653 SLJIT_C_GREATER_EQUAL, 654 BJ_XREG, 0, 655 SLJIT_IMM, bc->nfuncs); 656 if (jump == NULL) 657 return SLJIT_ERR_ALLOC_FAILED; 658 } 659 660 if (jump != NULL) 661 *ret0_jump = jump; 662 663 /* 664 * Copy bpf_copfunc_t arguments to registers. 665 */ 666 #if BJ_AREG != SLJIT_SCRATCH_REG3 667 status = sljit_emit_op1(compiler, 668 SLJIT_MOV_UI, 669 SLJIT_SCRATCH_REG3, 0, 670 BJ_AREG, 0); 671 if (status != SLJIT_SUCCESS) 672 return status; 673 #endif 674 675 status = sljit_emit_op1(compiler, 676 SLJIT_MOV_P, 677 SLJIT_SCRATCH_REG1, 0, 678 SLJIT_MEM1(SLJIT_LOCALS_REG), 679 offsetof(struct bpfjit_stack, ctx)); 680 if (status != SLJIT_SUCCESS) 681 return status; 682 683 status = sljit_emit_op1(compiler, 684 SLJIT_MOV_P, 685 SLJIT_SCRATCH_REG2, 0, 686 BJ_ARGS, 0); 687 if (status != SLJIT_SUCCESS) 688 return status; 689 690 if (BPF_MISCOP(pc->code) == BPF_COP) { 691 status = sljit_emit_ijump(compiler, 692 SLJIT_CALL3, 693 SLJIT_IMM, SLJIT_FUNC_OFFSET(bc->copfuncs[pc->k])); 694 if (status != SLJIT_SUCCESS) 695 return status; 696 } else if (BPF_MISCOP(pc->code) == BPF_COPX) { 697 /* load ctx->copfuncs */ 698 status = sljit_emit_op1(compiler, 699 SLJIT_MOV_P, 700 BJ_COPF_PTR, 0, 701 SLJIT_MEM1(SLJIT_SCRATCH_REG1), 702 offsetof(struct bpf_ctx, copfuncs)); 703 if (status != SLJIT_SUCCESS) 704 return status; 705 706 /* 707 * Load X to a register that can be used for 708 * memory addressing. 709 */ 710 status = sljit_emit_op1(compiler, 711 SLJIT_MOV, 712 BJ_COPF_IDX, 0, 713 BJ_XREG, 0); 714 if (status != SLJIT_SUCCESS) 715 return status; 716 717 status = sljit_emit_ijump(compiler, 718 SLJIT_CALL3, 719 SLJIT_MEM2(BJ_COPF_PTR, BJ_COPF_IDX), 720 SLJIT_WORD_SHIFT); 721 if (status != SLJIT_SUCCESS) 722 return status; 723 724 status = load_buf_buflen(compiler); 725 if (status != SLJIT_SUCCESS) 726 return status; 727 } 728 729 #if BJ_AREG != SLJIT_RETURN_REG 730 status = sljit_emit_op1(compiler, 731 SLJIT_MOV, 732 BJ_AREG, 0, 733 SLJIT_RETURN_REG, 0); 734 if (status != SLJIT_SUCCESS) 735 return status; 736 #endif 737 738 return status; 739 } 740 741 /* 742 * Generate code for 743 * BPF_LD+BPF_W+BPF_ABS A <- P[k:4] 744 * BPF_LD+BPF_H+BPF_ABS A <- P[k:2] 745 * BPF_LD+BPF_B+BPF_ABS A <- P[k:1] 746 * BPF_LD+BPF_W+BPF_IND A <- P[X+k:4] 747 * BPF_LD+BPF_H+BPF_IND A <- P[X+k:2] 748 * BPF_LD+BPF_B+BPF_IND A <- P[X+k:1] 749 */ 750 static int 751 emit_pkt_read(struct sljit_compiler *compiler, 752 const struct bpf_insn *pc, struct sljit_jump *to_mchain_jump, 753 struct sljit_jump ***ret0, size_t *ret0_size, size_t *ret0_maxsize) 754 { 755 int status = 0; /* XXX gcc 4.1 */ 756 uint32_t width; 757 struct sljit_jump *jump; 758 #ifdef _KERNEL 759 struct sljit_label *label; 760 struct sljit_jump *over_mchain_jump; 761 const bool check_zero_buflen = (to_mchain_jump != NULL); 762 #endif 763 const uint32_t k = pc->k; 764 765 #ifdef _KERNEL 766 if (to_mchain_jump == NULL) { 767 to_mchain_jump = sljit_emit_cmp(compiler, 768 SLJIT_C_EQUAL, 769 BJ_BUFLEN, 0, 770 SLJIT_IMM, 0); 771 if (to_mchain_jump == NULL) 772 return SLJIT_ERR_ALLOC_FAILED; 773 } 774 #endif 775 776 width = read_width(pc); 777 778 if (BPF_MODE(pc->code) == BPF_IND) { 779 /* tmp1 = buflen - (pc->k + width); */ 780 status = sljit_emit_op2(compiler, 781 SLJIT_SUB, 782 BJ_TMP1REG, 0, 783 BJ_BUFLEN, 0, 784 SLJIT_IMM, k + width); 785 if (status != SLJIT_SUCCESS) 786 return status; 787 788 /* buf += X; */ 789 status = sljit_emit_op2(compiler, 790 SLJIT_ADD, 791 BJ_BUF, 0, 792 BJ_BUF, 0, 793 BJ_XREG, 0); 794 if (status != SLJIT_SUCCESS) 795 return status; 796 797 /* if (tmp1 < X) return 0; */ 798 jump = sljit_emit_cmp(compiler, 799 SLJIT_C_LESS, 800 BJ_TMP1REG, 0, 801 BJ_XREG, 0); 802 if (jump == NULL) 803 return SLJIT_ERR_ALLOC_FAILED; 804 if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 805 return SLJIT_ERR_ALLOC_FAILED; 806 } 807 808 switch (width) { 809 case 4: 810 status = emit_read32(compiler, k); 811 break; 812 case 2: 813 status = emit_read16(compiler, k); 814 break; 815 case 1: 816 status = emit_read8(compiler, k); 817 break; 818 } 819 820 if (status != SLJIT_SUCCESS) 821 return status; 822 823 if (BPF_MODE(pc->code) == BPF_IND) { 824 /* buf -= X; */ 825 status = sljit_emit_op2(compiler, 826 SLJIT_SUB, 827 BJ_BUF, 0, 828 BJ_BUF, 0, 829 BJ_XREG, 0); 830 if (status != SLJIT_SUCCESS) 831 return status; 832 } 833 834 #ifdef _KERNEL 835 over_mchain_jump = sljit_emit_jump(compiler, SLJIT_JUMP); 836 if (over_mchain_jump == NULL) 837 return SLJIT_ERR_ALLOC_FAILED; 838 839 /* entry point to mchain handler */ 840 label = sljit_emit_label(compiler); 841 if (label == NULL) 842 return SLJIT_ERR_ALLOC_FAILED; 843 sljit_set_label(to_mchain_jump, label); 844 845 if (check_zero_buflen) { 846 /* if (buflen != 0) return 0; */ 847 jump = sljit_emit_cmp(compiler, 848 SLJIT_C_NOT_EQUAL, 849 BJ_BUFLEN, 0, 850 SLJIT_IMM, 0); 851 if (jump == NULL) 852 return SLJIT_ERR_ALLOC_FAILED; 853 if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 854 return SLJIT_ERR_ALLOC_FAILED; 855 } 856 857 switch (width) { 858 case 4: 859 status = emit_xcall(compiler, pc, BJ_AREG, 0, &jump, &m_xword); 860 break; 861 case 2: 862 status = emit_xcall(compiler, pc, BJ_AREG, 0, &jump, &m_xhalf); 863 break; 864 case 1: 865 status = emit_xcall(compiler, pc, BJ_AREG, 0, &jump, &m_xbyte); 866 break; 867 } 868 869 if (status != SLJIT_SUCCESS) 870 return status; 871 872 if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 873 return SLJIT_ERR_ALLOC_FAILED; 874 875 label = sljit_emit_label(compiler); 876 if (label == NULL) 877 return SLJIT_ERR_ALLOC_FAILED; 878 sljit_set_label(over_mchain_jump, label); 879 #endif 880 881 return status; 882 } 883 884 static int 885 emit_memload(struct sljit_compiler *compiler, 886 sljit_si dst, uint32_t k, size_t extwords) 887 { 888 int status; 889 sljit_si src; 890 sljit_sw srcw; 891 892 srcw = k * sizeof(uint32_t); 893 894 if (extwords == 0) { 895 src = SLJIT_MEM1(SLJIT_LOCALS_REG); 896 srcw += offsetof(struct bpfjit_stack, mem); 897 } else { 898 /* copy extmem pointer to the tmp1 register */ 899 status = sljit_emit_op1(compiler, 900 SLJIT_MOV_P, 901 BJ_TMP1REG, 0, 902 SLJIT_MEM1(SLJIT_LOCALS_REG), 903 offsetof(struct bpfjit_stack, extmem)); 904 if (status != SLJIT_SUCCESS) 905 return status; 906 src = SLJIT_MEM1(BJ_TMP1REG); 907 } 908 909 return sljit_emit_op1(compiler, SLJIT_MOV_UI, dst, 0, src, srcw); 910 } 911 912 static int 913 emit_memstore(struct sljit_compiler *compiler, 914 sljit_si src, uint32_t k, size_t extwords) 915 { 916 int status; 917 sljit_si dst; 918 sljit_sw dstw; 919 920 dstw = k * sizeof(uint32_t); 921 922 if (extwords == 0) { 923 dst = SLJIT_MEM1(SLJIT_LOCALS_REG); 924 dstw += offsetof(struct bpfjit_stack, mem); 925 } else { 926 /* copy extmem pointer to the tmp1 register */ 927 status = sljit_emit_op1(compiler, 928 SLJIT_MOV_P, 929 BJ_TMP1REG, 0, 930 SLJIT_MEM1(SLJIT_LOCALS_REG), 931 offsetof(struct bpfjit_stack, extmem)); 932 if (status != SLJIT_SUCCESS) 933 return status; 934 dst = SLJIT_MEM1(BJ_TMP1REG); 935 } 936 937 return sljit_emit_op1(compiler, SLJIT_MOV_UI, dst, dstw, src, 0); 938 } 939 940 /* 941 * Generate code for BPF_LDX+BPF_B+BPF_MSH X <- 4*(P[k:1]&0xf). 942 */ 943 static int 944 emit_msh(struct sljit_compiler *compiler, 945 const struct bpf_insn *pc, struct sljit_jump *to_mchain_jump, 946 struct sljit_jump ***ret0, size_t *ret0_size, size_t *ret0_maxsize) 947 { 948 int status; 949 #ifdef _KERNEL 950 struct sljit_label *label; 951 struct sljit_jump *jump, *over_mchain_jump; 952 const bool check_zero_buflen = (to_mchain_jump != NULL); 953 #endif 954 const uint32_t k = pc->k; 955 956 #ifdef _KERNEL 957 if (to_mchain_jump == NULL) { 958 to_mchain_jump = sljit_emit_cmp(compiler, 959 SLJIT_C_EQUAL, 960 BJ_BUFLEN, 0, 961 SLJIT_IMM, 0); 962 if (to_mchain_jump == NULL) 963 return SLJIT_ERR_ALLOC_FAILED; 964 } 965 #endif 966 967 /* tmp1 = buf[k] */ 968 status = sljit_emit_op1(compiler, 969 SLJIT_MOV_UB, 970 BJ_TMP1REG, 0, 971 SLJIT_MEM1(BJ_BUF), k); 972 if (status != SLJIT_SUCCESS) 973 return status; 974 975 /* tmp1 &= 0xf */ 976 status = sljit_emit_op2(compiler, 977 SLJIT_AND, 978 BJ_TMP1REG, 0, 979 BJ_TMP1REG, 0, 980 SLJIT_IMM, 0xf); 981 if (status != SLJIT_SUCCESS) 982 return status; 983 984 /* tmp1 = tmp1 << 2 */ 985 status = sljit_emit_op2(compiler, 986 SLJIT_SHL, 987 BJ_XREG, 0, 988 BJ_TMP1REG, 0, 989 SLJIT_IMM, 2); 990 if (status != SLJIT_SUCCESS) 991 return status; 992 993 #ifdef _KERNEL 994 over_mchain_jump = sljit_emit_jump(compiler, SLJIT_JUMP); 995 if (over_mchain_jump == NULL) 996 return SLJIT_ERR_ALLOC_FAILED; 997 998 /* entry point to mchain handler */ 999 label = sljit_emit_label(compiler); 1000 if (label == NULL) 1001 return SLJIT_ERR_ALLOC_FAILED; 1002 sljit_set_label(to_mchain_jump, label); 1003 1004 if (check_zero_buflen) { 1005 /* if (buflen != 0) return 0; */ 1006 jump = sljit_emit_cmp(compiler, 1007 SLJIT_C_NOT_EQUAL, 1008 BJ_BUFLEN, 0, 1009 SLJIT_IMM, 0); 1010 if (jump == NULL) 1011 return SLJIT_ERR_ALLOC_FAILED; 1012 if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 1013 return SLJIT_ERR_ALLOC_FAILED; 1014 } 1015 1016 status = emit_xcall(compiler, pc, BJ_TMP1REG, 0, &jump, &m_xbyte); 1017 if (status != SLJIT_SUCCESS) 1018 return status; 1019 1020 if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 1021 return SLJIT_ERR_ALLOC_FAILED; 1022 1023 /* tmp1 &= 0xf */ 1024 status = sljit_emit_op2(compiler, 1025 SLJIT_AND, 1026 BJ_TMP1REG, 0, 1027 BJ_TMP1REG, 0, 1028 SLJIT_IMM, 0xf); 1029 if (status != SLJIT_SUCCESS) 1030 return status; 1031 1032 /* tmp1 = tmp1 << 2 */ 1033 status = sljit_emit_op2(compiler, 1034 SLJIT_SHL, 1035 BJ_XREG, 0, 1036 BJ_TMP1REG, 0, 1037 SLJIT_IMM, 2); 1038 if (status != SLJIT_SUCCESS) 1039 return status; 1040 1041 1042 label = sljit_emit_label(compiler); 1043 if (label == NULL) 1044 return SLJIT_ERR_ALLOC_FAILED; 1045 sljit_set_label(over_mchain_jump, label); 1046 #endif 1047 1048 return status; 1049 } 1050 1051 static int 1052 emit_pow2_division(struct sljit_compiler *compiler, uint32_t k) 1053 { 1054 int shift = 0; 1055 int status = SLJIT_SUCCESS; 1056 1057 while (k > 1) { 1058 k >>= 1; 1059 shift++; 1060 } 1061 1062 BJ_ASSERT(k == 1 && shift < 32); 1063 1064 if (shift != 0) { 1065 status = sljit_emit_op2(compiler, 1066 SLJIT_LSHR|SLJIT_INT_OP, 1067 BJ_AREG, 0, 1068 BJ_AREG, 0, 1069 SLJIT_IMM, shift); 1070 } 1071 1072 return status; 1073 } 1074 1075 #if !defined(BPFJIT_USE_UDIV) 1076 static sljit_uw 1077 divide(sljit_uw x, sljit_uw y) 1078 { 1079 1080 return (uint32_t)x / (uint32_t)y; 1081 } 1082 #endif 1083 1084 /* 1085 * Generate A = A / div. 1086 * divt,divw are either SLJIT_IMM,pc->k or BJ_XREG,0. 1087 */ 1088 static int 1089 emit_division(struct sljit_compiler *compiler, int divt, sljit_sw divw) 1090 { 1091 int status; 1092 1093 #if BJ_XREG == SLJIT_RETURN_REG || \ 1094 BJ_XREG == SLJIT_SCRATCH_REG1 || \ 1095 BJ_XREG == SLJIT_SCRATCH_REG2 || \ 1096 BJ_AREG == SLJIT_SCRATCH_REG2 1097 #error "Not supported assignment of registers." 1098 #endif 1099 1100 #if BJ_AREG != SLJIT_SCRATCH_REG1 1101 status = sljit_emit_op1(compiler, 1102 SLJIT_MOV, 1103 SLJIT_SCRATCH_REG1, 0, 1104 BJ_AREG, 0); 1105 if (status != SLJIT_SUCCESS) 1106 return status; 1107 #endif 1108 1109 status = sljit_emit_op1(compiler, 1110 SLJIT_MOV, 1111 SLJIT_SCRATCH_REG2, 0, 1112 divt, divw); 1113 if (status != SLJIT_SUCCESS) 1114 return status; 1115 1116 #if defined(BPFJIT_USE_UDIV) 1117 status = sljit_emit_op0(compiler, SLJIT_UDIV|SLJIT_INT_OP); 1118 1119 #if BJ_AREG != SLJIT_SCRATCH_REG1 1120 status = sljit_emit_op1(compiler, 1121 SLJIT_MOV, 1122 BJ_AREG, 0, 1123 SLJIT_SCRATCH_REG1, 0); 1124 if (status != SLJIT_SUCCESS) 1125 return status; 1126 #endif 1127 #else 1128 status = sljit_emit_ijump(compiler, 1129 SLJIT_CALL2, 1130 SLJIT_IMM, SLJIT_FUNC_OFFSET(divide)); 1131 1132 #if BJ_AREG != SLJIT_RETURN_REG 1133 status = sljit_emit_op1(compiler, 1134 SLJIT_MOV, 1135 BJ_AREG, 0, 1136 SLJIT_RETURN_REG, 0); 1137 if (status != SLJIT_SUCCESS) 1138 return status; 1139 #endif 1140 #endif 1141 1142 return status; 1143 } 1144 1145 /* 1146 * Return true if pc is a "read from packet" instruction. 1147 * If length is not NULL and return value is true, *length will 1148 * be set to a safe length required to read a packet. 1149 */ 1150 static bool 1151 read_pkt_insn(const struct bpf_insn *pc, bpfjit_abc_length_t *length) 1152 { 1153 bool rv; 1154 bpfjit_abc_length_t width; 1155 1156 switch (BPF_CLASS(pc->code)) { 1157 default: 1158 rv = false; 1159 break; 1160 1161 case BPF_LD: 1162 rv = BPF_MODE(pc->code) == BPF_ABS || 1163 BPF_MODE(pc->code) == BPF_IND; 1164 if (rv) 1165 width = read_width(pc); 1166 break; 1167 1168 case BPF_LDX: 1169 rv = pc->code == (BPF_LDX|BPF_B|BPF_MSH); 1170 width = 1; 1171 break; 1172 } 1173 1174 if (rv && length != NULL) { 1175 /* 1176 * Values greater than UINT32_MAX will generate 1177 * unconditional "return 0". 1178 */ 1179 *length = (uint32_t)pc->k + width; 1180 } 1181 1182 return rv; 1183 } 1184 1185 static void 1186 optimize_init(struct bpfjit_insn_data *insn_dat, size_t insn_count) 1187 { 1188 size_t i; 1189 1190 for (i = 0; i < insn_count; i++) { 1191 SLIST_INIT(&insn_dat[i].bjumps); 1192 insn_dat[i].invalid = BJ_INIT_NOBITS; 1193 } 1194 } 1195 1196 /* 1197 * The function divides instructions into blocks. Destination of a jump 1198 * instruction starts a new block. BPF_RET and BPF_JMP instructions 1199 * terminate a block. Blocks are linear, that is, there are no jumps out 1200 * from the middle of a block and there are no jumps in to the middle of 1201 * a block. 1202 * 1203 * The function also sets bits in *initmask for memwords that 1204 * need to be initialized to zero. Note that this set should be empty 1205 * for any valid kernel filter program. 1206 */ 1207 static bool 1208 optimize_pass1(const bpf_ctx_t *bc, const struct bpf_insn *insns, 1209 struct bpfjit_insn_data *insn_dat, size_t insn_count, 1210 bpf_memword_init_t *initmask, bpfjit_hint_t *hints) 1211 { 1212 struct bpfjit_jump *jtf; 1213 size_t i; 1214 uint32_t jt, jf; 1215 bpfjit_abc_length_t length; 1216 bpf_memword_init_t invalid; /* borrowed from bpf_filter() */ 1217 bool unreachable; 1218 1219 const size_t memwords = GET_MEMWORDS(bc); 1220 1221 *hints = 0; 1222 *initmask = BJ_INIT_NOBITS; 1223 1224 unreachable = false; 1225 invalid = ~BJ_INIT_NOBITS; 1226 1227 for (i = 0; i < insn_count; i++) { 1228 if (!SLIST_EMPTY(&insn_dat[i].bjumps)) 1229 unreachable = false; 1230 insn_dat[i].unreachable = unreachable; 1231 1232 if (unreachable) 1233 continue; 1234 1235 invalid |= insn_dat[i].invalid; 1236 1237 if (read_pkt_insn(&insns[i], &length) && length > UINT32_MAX) 1238 unreachable = true; 1239 1240 switch (BPF_CLASS(insns[i].code)) { 1241 case BPF_RET: 1242 if (BPF_RVAL(insns[i].code) == BPF_A) 1243 *initmask |= invalid & BJ_INIT_ABIT; 1244 1245 unreachable = true; 1246 continue; 1247 1248 case BPF_LD: 1249 if ((BPF_MODE(insns[i].code) == BPF_IND || 1250 BPF_MODE(insns[i].code) == BPF_ABS) && 1251 read_width(&insns[i]) == 4) { 1252 *hints |= BJ_HINT_LDW; 1253 } 1254 1255 if (BPF_MODE(insns[i].code) == BPF_IND) { 1256 *hints |= BJ_HINT_XREG | BJ_HINT_IND; 1257 *initmask |= invalid & BJ_INIT_XBIT; 1258 } 1259 1260 if (BPF_MODE(insns[i].code) == BPF_MEM && 1261 (uint32_t)insns[i].k < memwords) { 1262 *initmask |= invalid & BJ_INIT_MBIT(insns[i].k); 1263 } 1264 1265 invalid &= ~BJ_INIT_ABIT; 1266 continue; 1267 1268 case BPF_LDX: 1269 *hints |= BJ_HINT_XREG | BJ_HINT_LDX; 1270 1271 if (BPF_MODE(insns[i].code) == BPF_MEM && 1272 (uint32_t)insns[i].k < memwords) { 1273 *initmask |= invalid & BJ_INIT_MBIT(insns[i].k); 1274 } 1275 1276 invalid &= ~BJ_INIT_XBIT; 1277 continue; 1278 1279 case BPF_ST: 1280 *initmask |= invalid & BJ_INIT_ABIT; 1281 1282 if ((uint32_t)insns[i].k < memwords) 1283 invalid &= ~BJ_INIT_MBIT(insns[i].k); 1284 1285 continue; 1286 1287 case BPF_STX: 1288 *hints |= BJ_HINT_XREG; 1289 *initmask |= invalid & BJ_INIT_XBIT; 1290 1291 if ((uint32_t)insns[i].k < memwords) 1292 invalid &= ~BJ_INIT_MBIT(insns[i].k); 1293 1294 continue; 1295 1296 case BPF_ALU: 1297 *initmask |= invalid & BJ_INIT_ABIT; 1298 1299 if (insns[i].code != (BPF_ALU|BPF_NEG) && 1300 BPF_SRC(insns[i].code) == BPF_X) { 1301 *hints |= BJ_HINT_XREG; 1302 *initmask |= invalid & BJ_INIT_XBIT; 1303 } 1304 1305 invalid &= ~BJ_INIT_ABIT; 1306 continue; 1307 1308 case BPF_MISC: 1309 switch (BPF_MISCOP(insns[i].code)) { 1310 case BPF_TAX: // X <- A 1311 *hints |= BJ_HINT_XREG; 1312 *initmask |= invalid & BJ_INIT_ABIT; 1313 invalid &= ~BJ_INIT_XBIT; 1314 continue; 1315 1316 case BPF_TXA: // A <- X 1317 *hints |= BJ_HINT_XREG; 1318 *initmask |= invalid & BJ_INIT_XBIT; 1319 invalid &= ~BJ_INIT_ABIT; 1320 continue; 1321 1322 case BPF_COPX: 1323 *hints |= BJ_HINT_XREG; 1324 /* FALLTHROUGH */ 1325 1326 case BPF_COP: 1327 *hints |= BJ_HINT_COP; 1328 *initmask |= invalid & BJ_INIT_ABIT; 1329 invalid &= ~BJ_INIT_ABIT; 1330 continue; 1331 } 1332 1333 continue; 1334 1335 case BPF_JMP: 1336 /* Initialize abc_length for ABC pass. */ 1337 insn_dat[i].u.jdata.abc_length = MAX_ABC_LENGTH; 1338 1339 if (BPF_OP(insns[i].code) == BPF_JA) { 1340 jt = jf = insns[i].k; 1341 } else { 1342 jt = insns[i].jt; 1343 jf = insns[i].jf; 1344 } 1345 1346 if (jt >= insn_count - (i + 1) || 1347 jf >= insn_count - (i + 1)) { 1348 return false; 1349 } 1350 1351 if (jt > 0 && jf > 0) 1352 unreachable = true; 1353 1354 jt += i + 1; 1355 jf += i + 1; 1356 1357 jtf = insn_dat[i].u.jdata.jtf; 1358 1359 jtf[0].sjump = NULL; 1360 jtf[0].jdata = &insn_dat[i].u.jdata; 1361 SLIST_INSERT_HEAD(&insn_dat[jt].bjumps, 1362 &jtf[0], entries); 1363 1364 if (jf != jt) { 1365 jtf[1].sjump = NULL; 1366 jtf[1].jdata = &insn_dat[i].u.jdata; 1367 SLIST_INSERT_HEAD(&insn_dat[jf].bjumps, 1368 &jtf[1], entries); 1369 } 1370 1371 insn_dat[jf].invalid |= invalid; 1372 insn_dat[jt].invalid |= invalid; 1373 invalid = 0; 1374 1375 continue; 1376 } 1377 } 1378 1379 return true; 1380 } 1381 1382 /* 1383 * Array Bounds Check Elimination (ABC) pass. 1384 */ 1385 static void 1386 optimize_pass2(const bpf_ctx_t *bc, const struct bpf_insn *insns, 1387 struct bpfjit_insn_data *insn_dat, size_t insn_count) 1388 { 1389 struct bpfjit_jump *jmp; 1390 const struct bpf_insn *pc; 1391 struct bpfjit_insn_data *pd; 1392 size_t i; 1393 bpfjit_abc_length_t length, abc_length = 0; 1394 1395 const size_t extwords = GET_EXTWORDS(bc); 1396 1397 for (i = insn_count; i != 0; i--) { 1398 pc = &insns[i-1]; 1399 pd = &insn_dat[i-1]; 1400 1401 if (pd->unreachable) 1402 continue; 1403 1404 switch (BPF_CLASS(pc->code)) { 1405 case BPF_RET: 1406 /* 1407 * It's quite common for bpf programs to 1408 * check packet bytes in increasing order 1409 * and return zero if bytes don't match 1410 * specified critetion. Such programs disable 1411 * ABC optimization completely because for 1412 * every jump there is a branch with no read 1413 * instruction. 1414 * With no side effects, BPF_STMT(BPF_RET+BPF_K, 0) 1415 * is indistinguishable from out-of-bound load. 1416 * Therefore, abc_length can be set to 1417 * MAX_ABC_LENGTH and enable ABC for many 1418 * bpf programs. 1419 * If this optimization encounters any 1420 * instruction with a side effect, it will 1421 * reset abc_length. 1422 */ 1423 if (BPF_RVAL(pc->code) == BPF_K && pc->k == 0) 1424 abc_length = MAX_ABC_LENGTH; 1425 else 1426 abc_length = 0; 1427 break; 1428 1429 case BPF_MISC: 1430 if (BPF_MISCOP(pc->code) == BPF_COP || 1431 BPF_MISCOP(pc->code) == BPF_COPX) { 1432 /* COP instructions can have side effects. */ 1433 abc_length = 0; 1434 } 1435 break; 1436 1437 case BPF_ST: 1438 case BPF_STX: 1439 if (extwords != 0) { 1440 /* Write to memory is visible after a call. */ 1441 abc_length = 0; 1442 } 1443 break; 1444 1445 case BPF_JMP: 1446 abc_length = pd->u.jdata.abc_length; 1447 break; 1448 1449 default: 1450 if (read_pkt_insn(pc, &length)) { 1451 if (abc_length < length) 1452 abc_length = length; 1453 pd->u.rdata.abc_length = abc_length; 1454 } 1455 break; 1456 } 1457 1458 SLIST_FOREACH(jmp, &pd->bjumps, entries) { 1459 if (jmp->jdata->abc_length > abc_length) 1460 jmp->jdata->abc_length = abc_length; 1461 } 1462 } 1463 } 1464 1465 static void 1466 optimize_pass3(const struct bpf_insn *insns, 1467 struct bpfjit_insn_data *insn_dat, size_t insn_count) 1468 { 1469 struct bpfjit_jump *jmp; 1470 size_t i; 1471 bpfjit_abc_length_t checked_length = 0; 1472 1473 for (i = 0; i < insn_count; i++) { 1474 if (insn_dat[i].unreachable) 1475 continue; 1476 1477 SLIST_FOREACH(jmp, &insn_dat[i].bjumps, entries) { 1478 if (jmp->jdata->checked_length < checked_length) 1479 checked_length = jmp->jdata->checked_length; 1480 } 1481 1482 if (BPF_CLASS(insns[i].code) == BPF_JMP) { 1483 insn_dat[i].u.jdata.checked_length = checked_length; 1484 } else if (read_pkt_insn(&insns[i], NULL)) { 1485 struct bpfjit_read_pkt_data *rdata = 1486 &insn_dat[i].u.rdata; 1487 rdata->check_length = 0; 1488 if (checked_length < rdata->abc_length) { 1489 checked_length = rdata->abc_length; 1490 rdata->check_length = checked_length; 1491 } 1492 } 1493 } 1494 } 1495 1496 static bool 1497 optimize(const bpf_ctx_t *bc, const struct bpf_insn *insns, 1498 struct bpfjit_insn_data *insn_dat, size_t insn_count, 1499 bpf_memword_init_t *initmask, bpfjit_hint_t *hints) 1500 { 1501 1502 optimize_init(insn_dat, insn_count); 1503 1504 if (!optimize_pass1(bc, insns, insn_dat, insn_count, initmask, hints)) 1505 return false; 1506 1507 optimize_pass2(bc, insns, insn_dat, insn_count); 1508 optimize_pass3(insns, insn_dat, insn_count); 1509 1510 return true; 1511 } 1512 1513 /* 1514 * Convert BPF_ALU operations except BPF_NEG and BPF_DIV to sljit operation. 1515 */ 1516 static int 1517 bpf_alu_to_sljit_op(const struct bpf_insn *pc) 1518 { 1519 1520 /* 1521 * Note: all supported 64bit arches have 32bit multiply 1522 * instruction so SLJIT_INT_OP doesn't have any overhead. 1523 */ 1524 switch (BPF_OP(pc->code)) { 1525 case BPF_ADD: return SLJIT_ADD; 1526 case BPF_SUB: return SLJIT_SUB; 1527 case BPF_MUL: return SLJIT_MUL|SLJIT_INT_OP; 1528 case BPF_OR: return SLJIT_OR; 1529 case BPF_AND: return SLJIT_AND; 1530 case BPF_LSH: return SLJIT_SHL; 1531 case BPF_RSH: return SLJIT_LSHR|SLJIT_INT_OP; 1532 default: 1533 BJ_ASSERT(false); 1534 return 0; 1535 } 1536 } 1537 1538 /* 1539 * Convert BPF_JMP operations except BPF_JA to sljit condition. 1540 */ 1541 static int 1542 bpf_jmp_to_sljit_cond(const struct bpf_insn *pc, bool negate) 1543 { 1544 /* 1545 * Note: all supported 64bit arches have 32bit comparison 1546 * instructions so SLJIT_INT_OP doesn't have any overhead. 1547 */ 1548 int rv = SLJIT_INT_OP; 1549 1550 switch (BPF_OP(pc->code)) { 1551 case BPF_JGT: 1552 rv |= negate ? SLJIT_C_LESS_EQUAL : SLJIT_C_GREATER; 1553 break; 1554 case BPF_JGE: 1555 rv |= negate ? SLJIT_C_LESS : SLJIT_C_GREATER_EQUAL; 1556 break; 1557 case BPF_JEQ: 1558 rv |= negate ? SLJIT_C_NOT_EQUAL : SLJIT_C_EQUAL; 1559 break; 1560 case BPF_JSET: 1561 rv |= negate ? SLJIT_C_EQUAL : SLJIT_C_NOT_EQUAL; 1562 break; 1563 default: 1564 BJ_ASSERT(false); 1565 } 1566 1567 return rv; 1568 } 1569 1570 /* 1571 * Convert BPF_K and BPF_X to sljit register. 1572 */ 1573 static int 1574 kx_to_reg(const struct bpf_insn *pc) 1575 { 1576 1577 switch (BPF_SRC(pc->code)) { 1578 case BPF_K: return SLJIT_IMM; 1579 case BPF_X: return BJ_XREG; 1580 default: 1581 BJ_ASSERT(false); 1582 return 0; 1583 } 1584 } 1585 1586 static sljit_sw 1587 kx_to_reg_arg(const struct bpf_insn *pc) 1588 { 1589 1590 switch (BPF_SRC(pc->code)) { 1591 case BPF_K: return (uint32_t)pc->k; /* SLJIT_IMM, pc->k, */ 1592 case BPF_X: return 0; /* BJ_XREG, 0, */ 1593 default: 1594 BJ_ASSERT(false); 1595 return 0; 1596 } 1597 } 1598 1599 static bool 1600 generate_insn_code(struct sljit_compiler *compiler, const bpf_ctx_t *bc, 1601 const struct bpf_insn *insns, struct bpfjit_insn_data *insn_dat, 1602 size_t insn_count) 1603 { 1604 /* a list of jumps to out-of-bound return from a generated function */ 1605 struct sljit_jump **ret0; 1606 size_t ret0_size, ret0_maxsize; 1607 1608 struct sljit_jump *jump; 1609 struct sljit_label *label; 1610 const struct bpf_insn *pc; 1611 struct bpfjit_jump *bjump, *jtf; 1612 struct sljit_jump *to_mchain_jump; 1613 1614 size_t i; 1615 int status; 1616 int branching, negate; 1617 unsigned int rval, mode, src; 1618 uint32_t jt, jf; 1619 1620 bool unconditional_ret; 1621 bool rv; 1622 1623 const size_t extwords = GET_EXTWORDS(bc); 1624 const size_t memwords = GET_MEMWORDS(bc); 1625 1626 ret0 = NULL; 1627 rv = false; 1628 1629 ret0_size = 0; 1630 ret0_maxsize = 64; 1631 ret0 = BJ_ALLOC(ret0_maxsize * sizeof(ret0[0])); 1632 if (ret0 == NULL) 1633 goto fail; 1634 1635 for (i = 0; i < insn_count; i++) { 1636 if (insn_dat[i].unreachable) 1637 continue; 1638 1639 /* 1640 * Resolve jumps to the current insn. 1641 */ 1642 label = NULL; 1643 SLIST_FOREACH(bjump, &insn_dat[i].bjumps, entries) { 1644 if (bjump->sjump != NULL) { 1645 if (label == NULL) 1646 label = sljit_emit_label(compiler); 1647 if (label == NULL) 1648 goto fail; 1649 sljit_set_label(bjump->sjump, label); 1650 } 1651 } 1652 1653 to_mchain_jump = NULL; 1654 unconditional_ret = false; 1655 1656 if (read_pkt_insn(&insns[i], NULL)) { 1657 if (insn_dat[i].u.rdata.check_length > UINT32_MAX) { 1658 /* Jump to "return 0" unconditionally. */ 1659 unconditional_ret = true; 1660 jump = sljit_emit_jump(compiler, SLJIT_JUMP); 1661 if (jump == NULL) 1662 goto fail; 1663 if (!append_jump(jump, &ret0, 1664 &ret0_size, &ret0_maxsize)) 1665 goto fail; 1666 } else if (insn_dat[i].u.rdata.check_length > 0) { 1667 /* if (buflen < check_length) return 0; */ 1668 jump = sljit_emit_cmp(compiler, 1669 SLJIT_C_LESS, 1670 BJ_BUFLEN, 0, 1671 SLJIT_IMM, 1672 insn_dat[i].u.rdata.check_length); 1673 if (jump == NULL) 1674 goto fail; 1675 #ifdef _KERNEL 1676 to_mchain_jump = jump; 1677 #else 1678 if (!append_jump(jump, &ret0, 1679 &ret0_size, &ret0_maxsize)) 1680 goto fail; 1681 #endif 1682 } 1683 } 1684 1685 pc = &insns[i]; 1686 switch (BPF_CLASS(pc->code)) { 1687 1688 default: 1689 goto fail; 1690 1691 case BPF_LD: 1692 /* BPF_LD+BPF_IMM A <- k */ 1693 if (pc->code == (BPF_LD|BPF_IMM)) { 1694 status = sljit_emit_op1(compiler, 1695 SLJIT_MOV, 1696 BJ_AREG, 0, 1697 SLJIT_IMM, (uint32_t)pc->k); 1698 if (status != SLJIT_SUCCESS) 1699 goto fail; 1700 1701 continue; 1702 } 1703 1704 /* BPF_LD+BPF_MEM A <- M[k] */ 1705 if (pc->code == (BPF_LD|BPF_MEM)) { 1706 if ((uint32_t)pc->k >= memwords) 1707 goto fail; 1708 status = emit_memload(compiler, 1709 BJ_AREG, pc->k, extwords); 1710 if (status != SLJIT_SUCCESS) 1711 goto fail; 1712 1713 continue; 1714 } 1715 1716 /* BPF_LD+BPF_W+BPF_LEN A <- len */ 1717 if (pc->code == (BPF_LD|BPF_W|BPF_LEN)) { 1718 status = sljit_emit_op1(compiler, 1719 SLJIT_MOV, /* size_t source */ 1720 BJ_AREG, 0, 1721 SLJIT_MEM1(BJ_ARGS), 1722 offsetof(struct bpf_args, wirelen)); 1723 if (status != SLJIT_SUCCESS) 1724 goto fail; 1725 1726 continue; 1727 } 1728 1729 mode = BPF_MODE(pc->code); 1730 if (mode != BPF_ABS && mode != BPF_IND) 1731 goto fail; 1732 1733 if (unconditional_ret) 1734 continue; 1735 1736 status = emit_pkt_read(compiler, pc, 1737 to_mchain_jump, &ret0, &ret0_size, &ret0_maxsize); 1738 if (status != SLJIT_SUCCESS) 1739 goto fail; 1740 1741 continue; 1742 1743 case BPF_LDX: 1744 mode = BPF_MODE(pc->code); 1745 1746 /* BPF_LDX+BPF_W+BPF_IMM X <- k */ 1747 if (mode == BPF_IMM) { 1748 if (BPF_SIZE(pc->code) != BPF_W) 1749 goto fail; 1750 status = sljit_emit_op1(compiler, 1751 SLJIT_MOV, 1752 BJ_XREG, 0, 1753 SLJIT_IMM, (uint32_t)pc->k); 1754 if (status != SLJIT_SUCCESS) 1755 goto fail; 1756 1757 continue; 1758 } 1759 1760 /* BPF_LDX+BPF_W+BPF_LEN X <- len */ 1761 if (mode == BPF_LEN) { 1762 if (BPF_SIZE(pc->code) != BPF_W) 1763 goto fail; 1764 status = sljit_emit_op1(compiler, 1765 SLJIT_MOV, /* size_t source */ 1766 BJ_XREG, 0, 1767 SLJIT_MEM1(BJ_ARGS), 1768 offsetof(struct bpf_args, wirelen)); 1769 if (status != SLJIT_SUCCESS) 1770 goto fail; 1771 1772 continue; 1773 } 1774 1775 /* BPF_LDX+BPF_W+BPF_MEM X <- M[k] */ 1776 if (mode == BPF_MEM) { 1777 if (BPF_SIZE(pc->code) != BPF_W) 1778 goto fail; 1779 if ((uint32_t)pc->k >= memwords) 1780 goto fail; 1781 status = emit_memload(compiler, 1782 BJ_XREG, pc->k, extwords); 1783 if (status != SLJIT_SUCCESS) 1784 goto fail; 1785 1786 continue; 1787 } 1788 1789 /* BPF_LDX+BPF_B+BPF_MSH X <- 4*(P[k:1]&0xf) */ 1790 if (mode != BPF_MSH || BPF_SIZE(pc->code) != BPF_B) 1791 goto fail; 1792 1793 if (unconditional_ret) 1794 continue; 1795 1796 status = emit_msh(compiler, pc, 1797 to_mchain_jump, &ret0, &ret0_size, &ret0_maxsize); 1798 if (status != SLJIT_SUCCESS) 1799 goto fail; 1800 1801 continue; 1802 1803 case BPF_ST: 1804 if (pc->code != BPF_ST || 1805 (uint32_t)pc->k >= memwords) { 1806 goto fail; 1807 } 1808 1809 status = emit_memstore(compiler, 1810 BJ_AREG, pc->k, extwords); 1811 if (status != SLJIT_SUCCESS) 1812 goto fail; 1813 1814 continue; 1815 1816 case BPF_STX: 1817 if (pc->code != BPF_STX || 1818 (uint32_t)pc->k >= memwords) { 1819 goto fail; 1820 } 1821 1822 status = emit_memstore(compiler, 1823 BJ_XREG, pc->k, extwords); 1824 if (status != SLJIT_SUCCESS) 1825 goto fail; 1826 1827 continue; 1828 1829 case BPF_ALU: 1830 if (pc->code == (BPF_ALU|BPF_NEG)) { 1831 status = sljit_emit_op1(compiler, 1832 SLJIT_NEG, 1833 BJ_AREG, 0, 1834 BJ_AREG, 0); 1835 if (status != SLJIT_SUCCESS) 1836 goto fail; 1837 1838 continue; 1839 } 1840 1841 if (BPF_OP(pc->code) != BPF_DIV) { 1842 status = sljit_emit_op2(compiler, 1843 bpf_alu_to_sljit_op(pc), 1844 BJ_AREG, 0, 1845 BJ_AREG, 0, 1846 kx_to_reg(pc), kx_to_reg_arg(pc)); 1847 if (status != SLJIT_SUCCESS) 1848 goto fail; 1849 1850 continue; 1851 } 1852 1853 /* BPF_DIV */ 1854 1855 src = BPF_SRC(pc->code); 1856 if (src != BPF_X && src != BPF_K) 1857 goto fail; 1858 1859 /* division by zero? */ 1860 if (src == BPF_X) { 1861 jump = sljit_emit_cmp(compiler, 1862 SLJIT_C_EQUAL|SLJIT_INT_OP, 1863 BJ_XREG, 0, 1864 SLJIT_IMM, 0); 1865 if (jump == NULL) 1866 goto fail; 1867 if (!append_jump(jump, &ret0, 1868 &ret0_size, &ret0_maxsize)) 1869 goto fail; 1870 } else if (pc->k == 0) { 1871 jump = sljit_emit_jump(compiler, SLJIT_JUMP); 1872 if (jump == NULL) 1873 goto fail; 1874 if (!append_jump(jump, &ret0, 1875 &ret0_size, &ret0_maxsize)) 1876 goto fail; 1877 } 1878 1879 if (src == BPF_X) { 1880 status = emit_division(compiler, BJ_XREG, 0); 1881 if (status != SLJIT_SUCCESS) 1882 goto fail; 1883 } else if (pc->k != 0) { 1884 if (pc->k & (pc->k - 1)) { 1885 status = emit_division(compiler, 1886 SLJIT_IMM, (uint32_t)pc->k); 1887 } else { 1888 status = emit_pow2_division(compiler, 1889 (uint32_t)pc->k); 1890 } 1891 if (status != SLJIT_SUCCESS) 1892 goto fail; 1893 } 1894 1895 continue; 1896 1897 case BPF_JMP: 1898 if (BPF_OP(pc->code) == BPF_JA) { 1899 jt = jf = pc->k; 1900 } else { 1901 jt = pc->jt; 1902 jf = pc->jf; 1903 } 1904 1905 negate = (jt == 0) ? 1 : 0; 1906 branching = (jt == jf) ? 0 : 1; 1907 jtf = insn_dat[i].u.jdata.jtf; 1908 1909 if (branching) { 1910 if (BPF_OP(pc->code) != BPF_JSET) { 1911 jump = sljit_emit_cmp(compiler, 1912 bpf_jmp_to_sljit_cond(pc, negate), 1913 BJ_AREG, 0, 1914 kx_to_reg(pc), kx_to_reg_arg(pc)); 1915 } else { 1916 status = sljit_emit_op2(compiler, 1917 SLJIT_AND, 1918 BJ_TMP1REG, 0, 1919 BJ_AREG, 0, 1920 kx_to_reg(pc), kx_to_reg_arg(pc)); 1921 if (status != SLJIT_SUCCESS) 1922 goto fail; 1923 1924 jump = sljit_emit_cmp(compiler, 1925 bpf_jmp_to_sljit_cond(pc, negate), 1926 BJ_TMP1REG, 0, 1927 SLJIT_IMM, 0); 1928 } 1929 1930 if (jump == NULL) 1931 goto fail; 1932 1933 BJ_ASSERT(jtf[negate].sjump == NULL); 1934 jtf[negate].sjump = jump; 1935 } 1936 1937 if (!branching || (jt != 0 && jf != 0)) { 1938 jump = sljit_emit_jump(compiler, SLJIT_JUMP); 1939 if (jump == NULL) 1940 goto fail; 1941 1942 BJ_ASSERT(jtf[branching].sjump == NULL); 1943 jtf[branching].sjump = jump; 1944 } 1945 1946 continue; 1947 1948 case BPF_RET: 1949 rval = BPF_RVAL(pc->code); 1950 if (rval == BPF_X) 1951 goto fail; 1952 1953 /* BPF_RET+BPF_K accept k bytes */ 1954 if (rval == BPF_K) { 1955 status = sljit_emit_return(compiler, 1956 SLJIT_MOV_UI, 1957 SLJIT_IMM, (uint32_t)pc->k); 1958 if (status != SLJIT_SUCCESS) 1959 goto fail; 1960 } 1961 1962 /* BPF_RET+BPF_A accept A bytes */ 1963 if (rval == BPF_A) { 1964 status = sljit_emit_return(compiler, 1965 SLJIT_MOV_UI, 1966 BJ_AREG, 0); 1967 if (status != SLJIT_SUCCESS) 1968 goto fail; 1969 } 1970 1971 continue; 1972 1973 case BPF_MISC: 1974 switch (BPF_MISCOP(pc->code)) { 1975 case BPF_TAX: 1976 status = sljit_emit_op1(compiler, 1977 SLJIT_MOV_UI, 1978 BJ_XREG, 0, 1979 BJ_AREG, 0); 1980 if (status != SLJIT_SUCCESS) 1981 goto fail; 1982 1983 continue; 1984 1985 case BPF_TXA: 1986 status = sljit_emit_op1(compiler, 1987 SLJIT_MOV, 1988 BJ_AREG, 0, 1989 BJ_XREG, 0); 1990 if (status != SLJIT_SUCCESS) 1991 goto fail; 1992 1993 continue; 1994 1995 case BPF_COP: 1996 case BPF_COPX: 1997 if (bc == NULL || bc->copfuncs == NULL) 1998 goto fail; 1999 if (BPF_MISCOP(pc->code) == BPF_COP && 2000 (uint32_t)pc->k >= bc->nfuncs) { 2001 goto fail; 2002 } 2003 2004 jump = NULL; 2005 status = emit_cop(compiler, bc, pc, &jump); 2006 if (status != SLJIT_SUCCESS) 2007 goto fail; 2008 2009 if (jump != NULL && !append_jump(jump, 2010 &ret0, &ret0_size, &ret0_maxsize)) 2011 goto fail; 2012 2013 continue; 2014 } 2015 2016 goto fail; 2017 } /* switch */ 2018 } /* main loop */ 2019 2020 BJ_ASSERT(ret0_size <= ret0_maxsize); 2021 2022 if (ret0_size > 0) { 2023 label = sljit_emit_label(compiler); 2024 if (label == NULL) 2025 goto fail; 2026 for (i = 0; i < ret0_size; i++) 2027 sljit_set_label(ret0[i], label); 2028 } 2029 2030 rv = true; 2031 2032 fail: 2033 if (ret0 != NULL) 2034 BJ_FREE(ret0, ret0_maxsize * sizeof(ret0[0])); 2035 2036 return rv; 2037 } 2038 2039 bpfjit_func_t 2040 bpfjit_generate_code(const bpf_ctx_t *bc, 2041 const struct bpf_insn *insns, size_t insn_count) 2042 { 2043 void *rv; 2044 struct sljit_compiler *compiler; 2045 2046 size_t i; 2047 int status; 2048 2049 /* optimization related */ 2050 bpf_memword_init_t initmask; 2051 bpfjit_hint_t hints; 2052 2053 /* memory store location for initial zero initialization */ 2054 sljit_si mem_reg; 2055 sljit_sw mem_off; 2056 2057 struct bpfjit_insn_data *insn_dat; 2058 2059 const size_t extwords = GET_EXTWORDS(bc); 2060 const size_t memwords = GET_MEMWORDS(bc); 2061 const bpf_memword_init_t preinited = extwords ? bc->preinited : 0; 2062 2063 rv = NULL; 2064 compiler = NULL; 2065 insn_dat = NULL; 2066 2067 if (memwords > MAX_MEMWORDS) 2068 goto fail; 2069 2070 if (insn_count == 0 || insn_count > SIZE_MAX / sizeof(insn_dat[0])) 2071 goto fail; 2072 2073 insn_dat = BJ_ALLOC(insn_count * sizeof(insn_dat[0])); 2074 if (insn_dat == NULL) 2075 goto fail; 2076 2077 if (!optimize(bc, insns, insn_dat, insn_count, &initmask, &hints)) 2078 goto fail; 2079 2080 compiler = sljit_create_compiler(); 2081 if (compiler == NULL) 2082 goto fail; 2083 2084 #if !defined(_KERNEL) && defined(SLJIT_VERBOSE) && SLJIT_VERBOSE 2085 sljit_compiler_verbose(compiler, stderr); 2086 #endif 2087 2088 status = sljit_emit_enter(compiler, 2089 2, nscratches(hints), 3, sizeof(struct bpfjit_stack)); 2090 if (status != SLJIT_SUCCESS) 2091 goto fail; 2092 2093 if (hints & BJ_HINT_COP) { 2094 /* save ctx argument */ 2095 status = sljit_emit_op1(compiler, 2096 SLJIT_MOV_P, 2097 SLJIT_MEM1(SLJIT_LOCALS_REG), 2098 offsetof(struct bpfjit_stack, ctx), 2099 BJ_CTX_ARG, 0); 2100 if (status != SLJIT_SUCCESS) 2101 goto fail; 2102 } 2103 2104 if (extwords == 0) { 2105 mem_reg = SLJIT_MEM1(SLJIT_LOCALS_REG); 2106 mem_off = offsetof(struct bpfjit_stack, mem); 2107 } else { 2108 /* copy "mem" argument from bpf_args to bpfjit_stack */ 2109 status = sljit_emit_op1(compiler, 2110 SLJIT_MOV_P, 2111 BJ_TMP1REG, 0, 2112 SLJIT_MEM1(BJ_ARGS), offsetof(struct bpf_args, mem)); 2113 if (status != SLJIT_SUCCESS) 2114 goto fail; 2115 2116 status = sljit_emit_op1(compiler, 2117 SLJIT_MOV_P, 2118 SLJIT_MEM1(SLJIT_LOCALS_REG), 2119 offsetof(struct bpfjit_stack, extmem), 2120 BJ_TMP1REG, 0); 2121 if (status != SLJIT_SUCCESS) 2122 goto fail; 2123 2124 mem_reg = SLJIT_MEM1(BJ_TMP1REG); 2125 mem_off = 0; 2126 } 2127 2128 /* 2129 * Exclude pre-initialised external memory words but keep 2130 * initialization statuses of A and X registers in case 2131 * bc->preinited wrongly sets those two bits. 2132 */ 2133 initmask &= ~preinited | BJ_INIT_ABIT | BJ_INIT_XBIT; 2134 2135 #if defined(_KERNEL) 2136 /* bpf_filter() checks initialization of memwords. */ 2137 BJ_ASSERT((initmask & (BJ_INIT_MBIT(memwords) - 1)) == 0); 2138 #endif 2139 for (i = 0; i < memwords; i++) { 2140 if (initmask & BJ_INIT_MBIT(i)) { 2141 /* M[i] = 0; */ 2142 status = sljit_emit_op1(compiler, 2143 SLJIT_MOV_UI, 2144 mem_reg, mem_off + i * sizeof(uint32_t), 2145 SLJIT_IMM, 0); 2146 if (status != SLJIT_SUCCESS) 2147 goto fail; 2148 } 2149 } 2150 2151 if (initmask & BJ_INIT_ABIT) { 2152 /* A = 0; */ 2153 status = sljit_emit_op1(compiler, 2154 SLJIT_MOV, 2155 BJ_AREG, 0, 2156 SLJIT_IMM, 0); 2157 if (status != SLJIT_SUCCESS) 2158 goto fail; 2159 } 2160 2161 if (initmask & BJ_INIT_XBIT) { 2162 /* X = 0; */ 2163 status = sljit_emit_op1(compiler, 2164 SLJIT_MOV, 2165 BJ_XREG, 0, 2166 SLJIT_IMM, 0); 2167 if (status != SLJIT_SUCCESS) 2168 goto fail; 2169 } 2170 2171 status = load_buf_buflen(compiler); 2172 if (status != SLJIT_SUCCESS) 2173 goto fail; 2174 2175 if (!generate_insn_code(compiler, bc, insns, insn_dat, insn_count)) 2176 goto fail; 2177 2178 status = sljit_emit_return(compiler, 2179 SLJIT_MOV_UI, 2180 SLJIT_IMM, 0); 2181 if (status != SLJIT_SUCCESS) 2182 goto fail; 2183 2184 rv = sljit_generate_code(compiler); 2185 2186 fail: 2187 if (compiler != NULL) 2188 sljit_free_compiler(compiler); 2189 2190 if (insn_dat != NULL) 2191 BJ_FREE(insn_dat, insn_count * sizeof(insn_dat[0])); 2192 2193 return (bpfjit_func_t)rv; 2194 } 2195 2196 void 2197 bpfjit_free_code(bpfjit_func_t code) 2198 { 2199 2200 sljit_free_code((void *)code); 2201 } 2202
Indexes created Thu Sep 25 16:09:42 GMT 2025