bpfjit.c revision 1.28
1 /* $NetBSD: bpfjit.c,v 1.28 2014/07/13 21:54:46 alnsn Exp $ */ 2 3 /*- 4 * Copyright (c) 2011-2014 Alexander Nasonov. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 21 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 22 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 23 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 24 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 25 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 26 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 27 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 28 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 #ifdef _KERNEL 34 __KERNEL_RCSID(0, "$NetBSD: bpfjit.c,v 1.28 2014/07/13 21:54:46 alnsn Exp $"); 35 #else 36 __RCSID("$NetBSD: bpfjit.c,v 1.28 2014/07/13 21:54:46 alnsn Exp $"); 37 #endif 38 39 #include <sys/types.h> 40 #include <sys/queue.h> 41 42 #ifndef _KERNEL 43 #include <assert.h> 44 #define BJ_ASSERT(c) assert(c) 45 #else 46 #define BJ_ASSERT(c) KASSERT(c) 47 #endif 48 49 #ifndef _KERNEL 50 #include <stdlib.h> 51 #define BJ_ALLOC(sz) malloc(sz) 52 #define BJ_FREE(p, sz) free(p) 53 #else 54 #include <sys/kmem.h> 55 #define BJ_ALLOC(sz) kmem_alloc(sz, KM_SLEEP) 56 #define BJ_FREE(p, sz) kmem_free(p, sz) 57 #endif 58 59 #ifndef _KERNEL 60 #include <limits.h> 61 #include <stdbool.h> 62 #include <stddef.h> 63 #include <stdint.h> 64 #else 65 #include <sys/atomic.h> 66 #include <sys/module.h> 67 #endif 68 69 #define __BPF_PRIVATE 70 #include <net/bpf.h> 71 #include <net/bpfjit.h> 72 #include <sljitLir.h> 73 74 #if !defined(_KERNEL) && defined(SLJIT_VERBOSE) && SLJIT_VERBOSE 75 #include <stdio.h> /* for stderr */ 76 #endif 77 78 /* 79 * Arguments of generated bpfjit_func_t. 80 * The first argument is reassigned upon entry 81 * to a more frequently used buf argument. 82 */ 83 #define BJ_CTX_ARG SLJIT_SAVED_REG1 84 #define BJ_ARGS SLJIT_SAVED_REG2 85 86 /* 87 * Permanent register assignments. 88 */ 89 #define BJ_BUF SLJIT_SAVED_REG1 90 //#define BJ_ARGS SLJIT_SAVED_REG2 91 #define BJ_BUFLEN SLJIT_SAVED_REG3 92 #define BJ_AREG SLJIT_SCRATCH_REG1 93 #define BJ_TMP1REG SLJIT_SCRATCH_REG2 94 #define BJ_TMP2REG SLJIT_SCRATCH_REG3 95 #define BJ_XREG SLJIT_TEMPORARY_EREG1 96 #define BJ_TMP3REG SLJIT_TEMPORARY_EREG2 97 98 #ifdef _KERNEL 99 #define MAX_MEMWORDS BPF_MAX_MEMWORDS 100 #else 101 #define MAX_MEMWORDS BPF_MEMWORDS 102 #endif 103 104 #define BJ_INIT_NOBITS ((bpf_memword_init_t)0) 105 #define BJ_INIT_MBIT(k) BPF_MEMWORD_INIT(k) 106 #define BJ_INIT_ABIT BJ_INIT_MBIT(MAX_MEMWORDS) 107 #define BJ_INIT_XBIT BJ_INIT_MBIT(MAX_MEMWORDS + 1) 108 109 /* 110 * Get a number of memwords and external memwords from a bpf_ctx object. 111 */ 112 #define GET_EXTWORDS(bc) ((bc) ? (bc)->extwords : 0) 113 #define GET_MEMWORDS(bc) (GET_EXTWORDS(bc) ? GET_EXTWORDS(bc) : BPF_MEMWORDS) 114 115 /* 116 * Optimization hints. 117 */ 118 typedef unsigned int bpfjit_hint_t; 119 #define BJ_HINT_ABS 0x01 /* packet read at absolute offset */ 120 #define BJ_HINT_IND 0x02 /* packet read at variable offset */ 121 #define BJ_HINT_COP 0x04 /* BPF_COP or BPF_COPX instruction */ 122 #define BJ_HINT_COPX 0x08 /* BPF_COPX instruction */ 123 #define BJ_HINT_XREG 0x10 /* BJ_XREG is needed */ 124 #define BJ_HINT_LDX 0x20 /* BPF_LDX instruction */ 125 #define BJ_HINT_PKT (BJ_HINT_ABS|BJ_HINT_IND) /* packet read */ 126 127 /* 128 * Datatype for Array Bounds Check Elimination (ABC) pass. 129 */ 130 typedef uint64_t bpfjit_abc_length_t; 131 #define MAX_ABC_LENGTH (UINT32_MAX + UINT64_C(4)) /* max. width is 4 */ 132 133 struct bpfjit_stack 134 { 135 bpf_ctx_t *ctx; 136 uint32_t *extmem; /* pointer to external memory store */ 137 #ifdef _KERNEL 138 int err; /* 3rd argument for m_xword/m_xhalf/m_xbyte function call */ 139 #endif 140 uint32_t mem[BPF_MEMWORDS]; /* internal memory store */ 141 }; 142 143 /* 144 * Data for BPF_JMP instruction. 145 * Forward declaration for struct bpfjit_jump. 146 */ 147 struct bpfjit_jump_data; 148 149 /* 150 * Node of bjumps list. 151 */ 152 struct bpfjit_jump { 153 struct sljit_jump *sjump; 154 SLIST_ENTRY(bpfjit_jump) entries; 155 struct bpfjit_jump_data *jdata; 156 }; 157 158 /* 159 * Data for BPF_JMP instruction. 160 */ 161 struct bpfjit_jump_data { 162 /* 163 * These entries make up bjumps list: 164 * jtf[0] - when coming from jt path, 165 * jtf[1] - when coming from jf path. 166 */ 167 struct bpfjit_jump jtf[2]; 168 /* 169 * Length calculated by Array Bounds Check Elimination (ABC) pass. 170 */ 171 bpfjit_abc_length_t abc_length; 172 /* 173 * Length checked by the last out-of-bounds check. 174 */ 175 bpfjit_abc_length_t checked_length; 176 }; 177 178 /* 179 * Data for "read from packet" instructions. 180 * See also read_pkt_insn() function below. 181 */ 182 struct bpfjit_read_pkt_data { 183 /* 184 * Length calculated by Array Bounds Check Elimination (ABC) pass. 185 */ 186 bpfjit_abc_length_t abc_length; 187 /* 188 * If positive, emit "if (buflen < check_length) return 0" 189 * out-of-bounds check. 190 * Values greater than UINT32_MAX generate unconditional "return 0". 191 */ 192 bpfjit_abc_length_t check_length; 193 }; 194 195 /* 196 * Additional (optimization-related) data for bpf_insn. 197 */ 198 struct bpfjit_insn_data { 199 /* List of jumps to this insn. */ 200 SLIST_HEAD(, bpfjit_jump) bjumps; 201 202 union { 203 struct bpfjit_jump_data jdata; 204 struct bpfjit_read_pkt_data rdata; 205 } u; 206 207 bpf_memword_init_t invalid; 208 bool unreachable; 209 }; 210 211 #ifdef _KERNEL 212 213 uint32_t m_xword(const struct mbuf *, uint32_t, int *); 214 uint32_t m_xhalf(const struct mbuf *, uint32_t, int *); 215 uint32_t m_xbyte(const struct mbuf *, uint32_t, int *); 216 217 MODULE(MODULE_CLASS_MISC, bpfjit, "sljit") 218 219 static int 220 bpfjit_modcmd(modcmd_t cmd, void *arg) 221 { 222 223 switch (cmd) { 224 case MODULE_CMD_INIT: 225 bpfjit_module_ops.bj_free_code = &bpfjit_free_code; 226 membar_producer(); 227 bpfjit_module_ops.bj_generate_code = &bpfjit_generate_code; 228 membar_producer(); 229 return 0; 230 231 case MODULE_CMD_FINI: 232 return EOPNOTSUPP; 233 234 default: 235 return ENOTTY; 236 } 237 } 238 #endif 239 240 /* 241 * Return a number of scratch registers to pass 242 * to sljit_emit_enter() function. 243 */ 244 static sljit_si 245 nscratches(bpfjit_hint_t hints) 246 { 247 sljit_si rv = 2; 248 249 #ifdef _KERNEL 250 if (hints & BJ_HINT_PKT) 251 rv = 3; /* xcall with three arguments */ 252 #endif 253 254 if (hints & BJ_HINT_IND) 255 rv = 3; /* uses BJ_TMP2REG */ 256 257 if (hints & BJ_HINT_COP) 258 rv = 3; /* calls copfunc with three arguments */ 259 260 if (hints & BJ_HINT_XREG) 261 rv = 4; /* uses BJ_XREG */ 262 263 #ifdef _KERNEL 264 if (hints & BJ_HINT_LDX) 265 rv = 5; /* uses BJ_TMP3REG */ 266 #endif 267 268 if (hints & BJ_HINT_COPX) 269 rv = 5; /* uses BJ_TMP3REG */ 270 271 return rv; 272 } 273 274 static uint32_t 275 read_width(const struct bpf_insn *pc) 276 { 277 278 switch (BPF_SIZE(pc->code)) { 279 case BPF_W: 280 return 4; 281 case BPF_H: 282 return 2; 283 case BPF_B: 284 return 1; 285 default: 286 BJ_ASSERT(false); 287 return 0; 288 } 289 } 290 291 /* 292 * Copy buf and buflen members of bpf_args from BJ_ARGS 293 * pointer to BJ_BUF and BJ_BUFLEN registers. 294 */ 295 static int 296 load_buf_buflen(struct sljit_compiler *compiler) 297 { 298 int status; 299 300 status = sljit_emit_op1(compiler, 301 SLJIT_MOV_P, 302 BJ_BUF, 0, 303 SLJIT_MEM1(BJ_ARGS), 304 offsetof(struct bpf_args, pkt)); 305 if (status != SLJIT_SUCCESS) 306 return status; 307 308 status = sljit_emit_op1(compiler, 309 SLJIT_MOV, /* size_t source */ 310 BJ_BUFLEN, 0, 311 SLJIT_MEM1(BJ_ARGS), 312 offsetof(struct bpf_args, buflen)); 313 314 return status; 315 } 316 317 static bool 318 grow_jumps(struct sljit_jump ***jumps, size_t *size) 319 { 320 struct sljit_jump **newptr; 321 const size_t elemsz = sizeof(struct sljit_jump *); 322 size_t old_size = *size; 323 size_t new_size = 2 * old_size; 324 325 if (new_size < old_size || new_size > SIZE_MAX / elemsz) 326 return false; 327 328 newptr = BJ_ALLOC(new_size * elemsz); 329 if (newptr == NULL) 330 return false; 331 332 memcpy(newptr, *jumps, old_size * elemsz); 333 BJ_FREE(*jumps, old_size * elemsz); 334 335 *jumps = newptr; 336 *size = new_size; 337 return true; 338 } 339 340 static bool 341 append_jump(struct sljit_jump *jump, struct sljit_jump ***jumps, 342 size_t *size, size_t *max_size) 343 { 344 if (*size == *max_size && !grow_jumps(jumps, max_size)) 345 return false; 346 347 (*jumps)[(*size)++] = jump; 348 return true; 349 } 350 351 /* 352 * Emit code for BPF_LD+BPF_B+BPF_ABS A <- P[k:1]. 353 */ 354 static int 355 emit_read8(struct sljit_compiler *compiler, sljit_si src, uint32_t k) 356 { 357 358 return sljit_emit_op1(compiler, 359 SLJIT_MOV_UB, 360 BJ_AREG, 0, 361 SLJIT_MEM1(src), k); 362 } 363 364 /* 365 * Emit code for BPF_LD+BPF_H+BPF_ABS A <- P[k:2]. 366 */ 367 static int 368 emit_read16(struct sljit_compiler *compiler, sljit_si src, uint32_t k) 369 { 370 int status; 371 372 BJ_ASSERT(k <= UINT32_MAX - 1); 373 374 /* A = buf[k]; */ 375 status = sljit_emit_op1(compiler, 376 SLJIT_MOV_UB, 377 BJ_AREG, 0, 378 SLJIT_MEM1(src), k); 379 if (status != SLJIT_SUCCESS) 380 return status; 381 382 /* tmp1 = buf[k+1]; */ 383 status = sljit_emit_op1(compiler, 384 SLJIT_MOV_UB, 385 BJ_TMP1REG, 0, 386 SLJIT_MEM1(src), k+1); 387 if (status != SLJIT_SUCCESS) 388 return status; 389 390 /* A = A << 8; */ 391 status = sljit_emit_op2(compiler, 392 SLJIT_SHL, 393 BJ_AREG, 0, 394 BJ_AREG, 0, 395 SLJIT_IMM, 8); 396 if (status != SLJIT_SUCCESS) 397 return status; 398 399 /* A = A + tmp1; */ 400 status = sljit_emit_op2(compiler, 401 SLJIT_ADD, 402 BJ_AREG, 0, 403 BJ_AREG, 0, 404 BJ_TMP1REG, 0); 405 return status; 406 } 407 408 /* 409 * Emit code for BPF_LD+BPF_W+BPF_ABS A <- P[k:4]. 410 */ 411 static int 412 emit_read32(struct sljit_compiler *compiler, sljit_si src, uint32_t k) 413 { 414 int status; 415 416 BJ_ASSERT(k <= UINT32_MAX - 3); 417 418 /* A = buf[k]; */ 419 status = sljit_emit_op1(compiler, 420 SLJIT_MOV_UB, 421 BJ_AREG, 0, 422 SLJIT_MEM1(src), k); 423 if (status != SLJIT_SUCCESS) 424 return status; 425 426 /* tmp1 = buf[k+1]; */ 427 status = sljit_emit_op1(compiler, 428 SLJIT_MOV_UB, 429 BJ_TMP1REG, 0, 430 SLJIT_MEM1(src), k+1); 431 if (status != SLJIT_SUCCESS) 432 return status; 433 434 /* A = A << 8; */ 435 status = sljit_emit_op2(compiler, 436 SLJIT_SHL, 437 BJ_AREG, 0, 438 BJ_AREG, 0, 439 SLJIT_IMM, 8); 440 if (status != SLJIT_SUCCESS) 441 return status; 442 443 /* A = A + tmp1; */ 444 status = sljit_emit_op2(compiler, 445 SLJIT_ADD, 446 BJ_AREG, 0, 447 BJ_AREG, 0, 448 BJ_TMP1REG, 0); 449 if (status != SLJIT_SUCCESS) 450 return status; 451 452 /* tmp1 = buf[k+2]; */ 453 status = sljit_emit_op1(compiler, 454 SLJIT_MOV_UB, 455 BJ_TMP1REG, 0, 456 SLJIT_MEM1(src), k+2); 457 if (status != SLJIT_SUCCESS) 458 return status; 459 460 /* A = A << 8; */ 461 status = sljit_emit_op2(compiler, 462 SLJIT_SHL, 463 BJ_AREG, 0, 464 BJ_AREG, 0, 465 SLJIT_IMM, 8); 466 if (status != SLJIT_SUCCESS) 467 return status; 468 469 /* A = A + tmp1; */ 470 status = sljit_emit_op2(compiler, 471 SLJIT_ADD, 472 BJ_AREG, 0, 473 BJ_AREG, 0, 474 BJ_TMP1REG, 0); 475 if (status != SLJIT_SUCCESS) 476 return status; 477 478 /* tmp1 = buf[k+3]; */ 479 status = sljit_emit_op1(compiler, 480 SLJIT_MOV_UB, 481 BJ_TMP1REG, 0, 482 SLJIT_MEM1(src), k+3); 483 if (status != SLJIT_SUCCESS) 484 return status; 485 486 /* A = A << 8; */ 487 status = sljit_emit_op2(compiler, 488 SLJIT_SHL, 489 BJ_AREG, 0, 490 BJ_AREG, 0, 491 SLJIT_IMM, 8); 492 if (status != SLJIT_SUCCESS) 493 return status; 494 495 /* A = A + tmp1; */ 496 status = sljit_emit_op2(compiler, 497 SLJIT_ADD, 498 BJ_AREG, 0, 499 BJ_AREG, 0, 500 BJ_TMP1REG, 0); 501 return status; 502 } 503 504 #ifdef _KERNEL 505 /* 506 * Emit code for m_xword/m_xhalf/m_xbyte call. 507 * 508 * @pc BPF_LD+BPF_W+BPF_ABS A <- P[k:4] 509 * BPF_LD+BPF_H+BPF_ABS A <- P[k:2] 510 * BPF_LD+BPF_B+BPF_ABS A <- P[k:1] 511 * BPF_LD+BPF_W+BPF_IND A <- P[X+k:4] 512 * BPF_LD+BPF_H+BPF_IND A <- P[X+k:2] 513 * BPF_LD+BPF_B+BPF_IND A <- P[X+k:1] 514 * BPF_LDX+BPF_B+BPF_MSH X <- 4*(P[k:1]&0xf) 515 */ 516 static int 517 emit_xcall(struct sljit_compiler *compiler, const struct bpf_insn *pc, 518 int dst, struct sljit_jump ***ret0, size_t *ret0_size, size_t *ret0_maxsize, 519 uint32_t (*fn)(const struct mbuf *, uint32_t, int *)) 520 { 521 #if BJ_XREG == SLJIT_RETURN_REG || \ 522 BJ_XREG == SLJIT_SCRATCH_REG1 || \ 523 BJ_XREG == SLJIT_SCRATCH_REG2 || \ 524 BJ_XREG == SLJIT_SCRATCH_REG3 || \ 525 BJ_TMP3REG == SLJIT_RETURN_REG || \ 526 BJ_TMP3REG == SLJIT_SCRATCH_REG1 || \ 527 BJ_TMP3REG == SLJIT_SCRATCH_REG2 || \ 528 BJ_TMP3REG == SLJIT_SCRATCH_REG3 529 #error "Not supported assignment of registers." 530 #endif 531 struct sljit_jump *jump; 532 int status; 533 534 BJ_ASSERT(dst != BJ_TMP2REG && dst != BJ_TMP3REG); 535 536 if (BPF_CLASS(pc->code) == BPF_LDX) { 537 /* save A */ 538 status = sljit_emit_op1(compiler, 539 SLJIT_MOV, 540 BJ_TMP3REG, 0, 541 BJ_AREG, 0); 542 if (status != SLJIT_SUCCESS) 543 return status; 544 } 545 546 /* 547 * Prepare registers for fn(mbuf, k, &err) call. 548 */ 549 status = sljit_emit_op1(compiler, 550 SLJIT_MOV, 551 SLJIT_SCRATCH_REG1, 0, 552 BJ_BUF, 0); 553 if (status != SLJIT_SUCCESS) 554 return status; 555 556 if (BPF_CLASS(pc->code) == BPF_LD && BPF_MODE(pc->code) == BPF_IND) { 557 /* if (X > UINT32_MAX - pc->k) return 0; */ 558 jump = sljit_emit_cmp(compiler, 559 SLJIT_C_GREATER, 560 BJ_XREG, 0, 561 SLJIT_IMM, UINT32_MAX - pc->k); 562 if (jump == NULL) 563 return SLJIT_ERR_ALLOC_FAILED; 564 if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 565 return SLJIT_ERR_ALLOC_FAILED; 566 567 /* k = X + pc->k; */ 568 status = sljit_emit_op2(compiler, 569 SLJIT_ADD, 570 SLJIT_SCRATCH_REG2, 0, 571 BJ_XREG, 0, 572 SLJIT_IMM, (uint32_t)pc->k); 573 if (status != SLJIT_SUCCESS) 574 return status; 575 } else { 576 /* k = pc->k */ 577 status = sljit_emit_op1(compiler, 578 SLJIT_MOV, 579 SLJIT_SCRATCH_REG2, 0, 580 SLJIT_IMM, (uint32_t)pc->k); 581 if (status != SLJIT_SUCCESS) 582 return status; 583 } 584 585 /* 586 * The third argument of fn is an address on stack. 587 */ 588 status = sljit_get_local_base(compiler, 589 SLJIT_SCRATCH_REG3, 0, 590 offsetof(struct bpfjit_stack, err)); 591 if (status != SLJIT_SUCCESS) 592 return status; 593 594 /* fn(buf, k, &err); */ 595 status = sljit_emit_ijump(compiler, 596 SLJIT_CALL3, 597 SLJIT_IMM, SLJIT_FUNC_OFFSET(fn)); 598 if (status != SLJIT_SUCCESS) 599 return status; 600 601 if (dst != SLJIT_RETURN_REG) { 602 /* move return value to dst */ 603 status = sljit_emit_op1(compiler, 604 SLJIT_MOV, 605 dst, 0, 606 SLJIT_RETURN_REG, 0); 607 if (status != SLJIT_SUCCESS) 608 return status; 609 } 610 611 /* tmp2 = *err; */ 612 status = sljit_emit_op1(compiler, 613 SLJIT_MOV_UI, 614 BJ_TMP2REG, 0, 615 SLJIT_MEM1(SLJIT_LOCALS_REG), 616 offsetof(struct bpfjit_stack, err)); 617 if (status != SLJIT_SUCCESS) 618 return status; 619 620 /* if (tmp2 != 0) return 0; */ 621 jump = sljit_emit_cmp(compiler, 622 SLJIT_C_NOT_EQUAL, 623 BJ_TMP2REG, 0, 624 SLJIT_IMM, 0); 625 if (jump == NULL) 626 return SLJIT_ERR_ALLOC_FAILED; 627 628 if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 629 return SLJIT_ERR_ALLOC_FAILED; 630 631 if (BPF_CLASS(pc->code) == BPF_LDX) { 632 /* restore A */ 633 status = sljit_emit_op1(compiler, 634 SLJIT_MOV, 635 BJ_AREG, 0, 636 BJ_TMP3REG, 0); 637 if (status != SLJIT_SUCCESS) 638 return status; 639 } 640 641 return SLJIT_SUCCESS; 642 } 643 #endif 644 645 /* 646 * Emit code for BPF_COP and BPF_COPX instructions. 647 */ 648 static int 649 emit_cop(struct sljit_compiler *compiler, 650 const bpf_ctx_t *bc, const struct bpf_insn *pc, 651 struct sljit_jump ***ret0, size_t *ret0_size, size_t *ret0_maxsize) 652 { 653 #if BJ_XREG == SLJIT_RETURN_REG || \ 654 BJ_XREG == SLJIT_SCRATCH_REG1 || \ 655 BJ_XREG == SLJIT_SCRATCH_REG2 || \ 656 BJ_XREG == SLJIT_SCRATCH_REG3 || \ 657 BJ_TMP3REG == SLJIT_SCRATCH_REG1 || \ 658 BJ_TMP3REG == SLJIT_SCRATCH_REG2 || \ 659 BJ_TMP3REG == SLJIT_SCRATCH_REG3 660 #error "Not supported assignment of registers." 661 #endif 662 663 struct sljit_jump *jump; 664 sljit_si call_reg; 665 sljit_sw call_off; 666 int status; 667 668 BJ_ASSERT(bc != NULL && bc->copfuncs != NULL); 669 670 if (BPF_MISCOP(pc->code) == BPF_COP) { 671 call_reg = SLJIT_IMM; 672 call_off = SLJIT_FUNC_OFFSET(bc->copfuncs[pc->k]); 673 } else { 674 /* if (X >= bc->nfuncs) return 0; */ 675 jump = sljit_emit_cmp(compiler, 676 SLJIT_C_GREATER_EQUAL, 677 BJ_XREG, 0, 678 SLJIT_IMM, bc->nfuncs); 679 if (jump == NULL) 680 return SLJIT_ERR_ALLOC_FAILED; 681 if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 682 return SLJIT_ERR_ALLOC_FAILED; 683 684 /* tmp1 = ctx; */ 685 status = sljit_emit_op1(compiler, 686 SLJIT_MOV_P, 687 BJ_TMP1REG, 0, 688 SLJIT_MEM1(SLJIT_LOCALS_REG), 689 offsetof(struct bpfjit_stack, ctx)); 690 if (status != SLJIT_SUCCESS) 691 return status; 692 693 /* tmp1 = ctx->copfuncs; */ 694 status = sljit_emit_op1(compiler, 695 SLJIT_MOV_P, 696 BJ_TMP1REG, 0, 697 SLJIT_MEM1(BJ_TMP1REG), 698 offsetof(struct bpf_ctx, copfuncs)); 699 if (status != SLJIT_SUCCESS) 700 return status; 701 702 /* tmp2 = X; */ 703 status = sljit_emit_op1(compiler, 704 SLJIT_MOV, 705 BJ_TMP2REG, 0, 706 BJ_XREG, 0); 707 if (status != SLJIT_SUCCESS) 708 return status; 709 710 /* tmp3 = ctx->copfuncs[tmp2]; */ 711 call_reg = BJ_TMP3REG; 712 call_off = 0; 713 status = sljit_emit_op1(compiler, 714 SLJIT_MOV_P, 715 call_reg, call_off, 716 SLJIT_MEM2(BJ_TMP1REG, BJ_TMP2REG), 717 SLJIT_WORD_SHIFT); 718 if (status != SLJIT_SUCCESS) 719 return status; 720 } 721 722 /* 723 * Copy bpf_copfunc_t arguments to registers. 724 */ 725 #if BJ_AREG != SLJIT_SCRATCH_REG3 726 status = sljit_emit_op1(compiler, 727 SLJIT_MOV_UI, 728 SLJIT_SCRATCH_REG3, 0, 729 BJ_AREG, 0); 730 if (status != SLJIT_SUCCESS) 731 return status; 732 #endif 733 734 status = sljit_emit_op1(compiler, 735 SLJIT_MOV_P, 736 SLJIT_SCRATCH_REG1, 0, 737 SLJIT_MEM1(SLJIT_LOCALS_REG), 738 offsetof(struct bpfjit_stack, ctx)); 739 if (status != SLJIT_SUCCESS) 740 return status; 741 742 status = sljit_emit_op1(compiler, 743 SLJIT_MOV_P, 744 SLJIT_SCRATCH_REG2, 0, 745 BJ_ARGS, 0); 746 if (status != SLJIT_SUCCESS) 747 return status; 748 749 status = sljit_emit_ijump(compiler, 750 SLJIT_CALL3, call_reg, call_off); 751 if (status != SLJIT_SUCCESS) 752 return status; 753 754 #if BJ_AREG != SLJIT_RETURN_REG 755 status = sljit_emit_op1(compiler, 756 SLJIT_MOV, 757 BJ_AREG, 0, 758 SLJIT_RETURN_REG, 0); 759 if (status != SLJIT_SUCCESS) 760 return status; 761 #endif 762 763 return SLJIT_SUCCESS; 764 } 765 766 /* 767 * Generate code for 768 * BPF_LD+BPF_W+BPF_ABS A <- P[k:4] 769 * BPF_LD+BPF_H+BPF_ABS A <- P[k:2] 770 * BPF_LD+BPF_B+BPF_ABS A <- P[k:1] 771 * BPF_LD+BPF_W+BPF_IND A <- P[X+k:4] 772 * BPF_LD+BPF_H+BPF_IND A <- P[X+k:2] 773 * BPF_LD+BPF_B+BPF_IND A <- P[X+k:1] 774 */ 775 static int 776 emit_pkt_read(struct sljit_compiler *compiler, 777 const struct bpf_insn *pc, struct sljit_jump *to_mchain_jump, 778 struct sljit_jump ***ret0, size_t *ret0_size, size_t *ret0_maxsize) 779 { 780 int status = SLJIT_ERR_ALLOC_FAILED; 781 uint32_t width; 782 sljit_si ld_reg; 783 struct sljit_jump *jump; 784 #ifdef _KERNEL 785 struct sljit_label *label; 786 struct sljit_jump *over_mchain_jump; 787 const bool check_zero_buflen = (to_mchain_jump != NULL); 788 #endif 789 const uint32_t k = pc->k; 790 791 #ifdef _KERNEL 792 if (to_mchain_jump == NULL) { 793 to_mchain_jump = sljit_emit_cmp(compiler, 794 SLJIT_C_EQUAL, 795 BJ_BUFLEN, 0, 796 SLJIT_IMM, 0); 797 if (to_mchain_jump == NULL) 798 return SLJIT_ERR_ALLOC_FAILED; 799 } 800 #endif 801 802 ld_reg = BJ_BUF; 803 width = read_width(pc); 804 805 if (BPF_MODE(pc->code) == BPF_IND) { 806 /* tmp1 = buflen - (pc->k + width); */ 807 status = sljit_emit_op2(compiler, 808 SLJIT_SUB, 809 BJ_TMP1REG, 0, 810 BJ_BUFLEN, 0, 811 SLJIT_IMM, k + width); 812 if (status != SLJIT_SUCCESS) 813 return status; 814 815 /* ld_reg = buf + X; */ 816 ld_reg = BJ_TMP2REG; 817 status = sljit_emit_op2(compiler, 818 SLJIT_ADD, 819 ld_reg, 0, 820 BJ_BUF, 0, 821 BJ_XREG, 0); 822 if (status != SLJIT_SUCCESS) 823 return status; 824 825 /* if (tmp1 < X) return 0; */ 826 jump = sljit_emit_cmp(compiler, 827 SLJIT_C_LESS, 828 BJ_TMP1REG, 0, 829 BJ_XREG, 0); 830 if (jump == NULL) 831 return SLJIT_ERR_ALLOC_FAILED; 832 if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 833 return SLJIT_ERR_ALLOC_FAILED; 834 } 835 836 switch (width) { 837 case 4: 838 status = emit_read32(compiler, ld_reg, k); 839 break; 840 case 2: 841 status = emit_read16(compiler, ld_reg, k); 842 break; 843 case 1: 844 status = emit_read8(compiler, ld_reg, k); 845 break; 846 } 847 848 if (status != SLJIT_SUCCESS) 849 return status; 850 851 #ifdef _KERNEL 852 over_mchain_jump = sljit_emit_jump(compiler, SLJIT_JUMP); 853 if (over_mchain_jump == NULL) 854 return SLJIT_ERR_ALLOC_FAILED; 855 856 /* entry point to mchain handler */ 857 label = sljit_emit_label(compiler); 858 if (label == NULL) 859 return SLJIT_ERR_ALLOC_FAILED; 860 sljit_set_label(to_mchain_jump, label); 861 862 if (check_zero_buflen) { 863 /* if (buflen != 0) return 0; */ 864 jump = sljit_emit_cmp(compiler, 865 SLJIT_C_NOT_EQUAL, 866 BJ_BUFLEN, 0, 867 SLJIT_IMM, 0); 868 if (jump == NULL) 869 return SLJIT_ERR_ALLOC_FAILED; 870 if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 871 return SLJIT_ERR_ALLOC_FAILED; 872 } 873 874 switch (width) { 875 case 4: 876 status = emit_xcall(compiler, pc, BJ_AREG, 877 ret0, ret0_size, ret0_maxsize, &m_xword); 878 break; 879 case 2: 880 status = emit_xcall(compiler, pc, BJ_AREG, 881 ret0, ret0_size, ret0_maxsize, &m_xhalf); 882 break; 883 case 1: 884 status = emit_xcall(compiler, pc, BJ_AREG, 885 ret0, ret0_size, ret0_maxsize, &m_xbyte); 886 break; 887 } 888 889 if (status != SLJIT_SUCCESS) 890 return status; 891 892 label = sljit_emit_label(compiler); 893 if (label == NULL) 894 return SLJIT_ERR_ALLOC_FAILED; 895 sljit_set_label(over_mchain_jump, label); 896 #endif 897 898 return SLJIT_SUCCESS; 899 } 900 901 static int 902 emit_memload(struct sljit_compiler *compiler, 903 sljit_si dst, uint32_t k, size_t extwords) 904 { 905 int status; 906 sljit_si src; 907 sljit_sw srcw; 908 909 srcw = k * sizeof(uint32_t); 910 911 if (extwords == 0) { 912 src = SLJIT_MEM1(SLJIT_LOCALS_REG); 913 srcw += offsetof(struct bpfjit_stack, mem); 914 } else { 915 /* copy extmem pointer to the tmp1 register */ 916 status = sljit_emit_op1(compiler, 917 SLJIT_MOV_P, 918 BJ_TMP1REG, 0, 919 SLJIT_MEM1(SLJIT_LOCALS_REG), 920 offsetof(struct bpfjit_stack, extmem)); 921 if (status != SLJIT_SUCCESS) 922 return status; 923 src = SLJIT_MEM1(BJ_TMP1REG); 924 } 925 926 return sljit_emit_op1(compiler, SLJIT_MOV_UI, dst, 0, src, srcw); 927 } 928 929 static int 930 emit_memstore(struct sljit_compiler *compiler, 931 sljit_si src, uint32_t k, size_t extwords) 932 { 933 int status; 934 sljit_si dst; 935 sljit_sw dstw; 936 937 dstw = k * sizeof(uint32_t); 938 939 if (extwords == 0) { 940 dst = SLJIT_MEM1(SLJIT_LOCALS_REG); 941 dstw += offsetof(struct bpfjit_stack, mem); 942 } else { 943 /* copy extmem pointer to the tmp1 register */ 944 status = sljit_emit_op1(compiler, 945 SLJIT_MOV_P, 946 BJ_TMP1REG, 0, 947 SLJIT_MEM1(SLJIT_LOCALS_REG), 948 offsetof(struct bpfjit_stack, extmem)); 949 if (status != SLJIT_SUCCESS) 950 return status; 951 dst = SLJIT_MEM1(BJ_TMP1REG); 952 } 953 954 return sljit_emit_op1(compiler, SLJIT_MOV_UI, dst, dstw, src, 0); 955 } 956 957 /* 958 * Emit code for BPF_LDX+BPF_B+BPF_MSH X <- 4*(P[k:1]&0xf). 959 */ 960 static int 961 emit_msh(struct sljit_compiler *compiler, 962 const struct bpf_insn *pc, struct sljit_jump *to_mchain_jump, 963 struct sljit_jump ***ret0, size_t *ret0_size, size_t *ret0_maxsize) 964 { 965 int status; 966 #ifdef _KERNEL 967 struct sljit_label *label; 968 struct sljit_jump *jump, *over_mchain_jump; 969 const bool check_zero_buflen = (to_mchain_jump != NULL); 970 #endif 971 const uint32_t k = pc->k; 972 973 #ifdef _KERNEL 974 if (to_mchain_jump == NULL) { 975 to_mchain_jump = sljit_emit_cmp(compiler, 976 SLJIT_C_EQUAL, 977 BJ_BUFLEN, 0, 978 SLJIT_IMM, 0); 979 if (to_mchain_jump == NULL) 980 return SLJIT_ERR_ALLOC_FAILED; 981 } 982 #endif 983 984 /* tmp1 = buf[k] */ 985 status = sljit_emit_op1(compiler, 986 SLJIT_MOV_UB, 987 BJ_TMP1REG, 0, 988 SLJIT_MEM1(BJ_BUF), k); 989 if (status != SLJIT_SUCCESS) 990 return status; 991 992 /* tmp1 &= 0xf */ 993 status = sljit_emit_op2(compiler, 994 SLJIT_AND, 995 BJ_TMP1REG, 0, 996 BJ_TMP1REG, 0, 997 SLJIT_IMM, 0xf); 998 if (status != SLJIT_SUCCESS) 999 return status; 1000 1001 /* tmp1 = tmp1 << 2 */ 1002 status = sljit_emit_op2(compiler, 1003 SLJIT_SHL, 1004 BJ_XREG, 0, 1005 BJ_TMP1REG, 0, 1006 SLJIT_IMM, 2); 1007 if (status != SLJIT_SUCCESS) 1008 return status; 1009 1010 #ifdef _KERNEL 1011 over_mchain_jump = sljit_emit_jump(compiler, SLJIT_JUMP); 1012 if (over_mchain_jump == NULL) 1013 return SLJIT_ERR_ALLOC_FAILED; 1014 1015 /* entry point to mchain handler */ 1016 label = sljit_emit_label(compiler); 1017 if (label == NULL) 1018 return SLJIT_ERR_ALLOC_FAILED; 1019 sljit_set_label(to_mchain_jump, label); 1020 1021 if (check_zero_buflen) { 1022 /* if (buflen != 0) return 0; */ 1023 jump = sljit_emit_cmp(compiler, 1024 SLJIT_C_NOT_EQUAL, 1025 BJ_BUFLEN, 0, 1026 SLJIT_IMM, 0); 1027 if (jump == NULL) 1028 return SLJIT_ERR_ALLOC_FAILED; 1029 if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 1030 return SLJIT_ERR_ALLOC_FAILED; 1031 } 1032 1033 status = emit_xcall(compiler, pc, BJ_TMP1REG, 1034 ret0, ret0_size, ret0_maxsize, &m_xbyte); 1035 if (status != SLJIT_SUCCESS) 1036 return status; 1037 1038 /* tmp1 &= 0xf */ 1039 status = sljit_emit_op2(compiler, 1040 SLJIT_AND, 1041 BJ_TMP1REG, 0, 1042 BJ_TMP1REG, 0, 1043 SLJIT_IMM, 0xf); 1044 if (status != SLJIT_SUCCESS) 1045 return status; 1046 1047 /* tmp1 = tmp1 << 2 */ 1048 status = sljit_emit_op2(compiler, 1049 SLJIT_SHL, 1050 BJ_XREG, 0, 1051 BJ_TMP1REG, 0, 1052 SLJIT_IMM, 2); 1053 if (status != SLJIT_SUCCESS) 1054 return status; 1055 1056 1057 label = sljit_emit_label(compiler); 1058 if (label == NULL) 1059 return SLJIT_ERR_ALLOC_FAILED; 1060 sljit_set_label(over_mchain_jump, label); 1061 #endif 1062 1063 return SLJIT_SUCCESS; 1064 } 1065 1066 static int 1067 emit_pow2_division(struct sljit_compiler *compiler, uint32_t k) 1068 { 1069 int shift = 0; 1070 int status = SLJIT_SUCCESS; 1071 1072 while (k > 1) { 1073 k >>= 1; 1074 shift++; 1075 } 1076 1077 BJ_ASSERT(k == 1 && shift < 32); 1078 1079 if (shift != 0) { 1080 status = sljit_emit_op2(compiler, 1081 SLJIT_LSHR|SLJIT_INT_OP, 1082 BJ_AREG, 0, 1083 BJ_AREG, 0, 1084 SLJIT_IMM, shift); 1085 } 1086 1087 return status; 1088 } 1089 1090 #if !defined(BPFJIT_USE_UDIV) 1091 static sljit_uw 1092 divide(sljit_uw x, sljit_uw y) 1093 { 1094 1095 return (uint32_t)x / (uint32_t)y; 1096 } 1097 #endif 1098 1099 /* 1100 * Emit code for A = A / div. 1101 * divt,divw are either SLJIT_IMM,pc->k or BJ_XREG,0. 1102 */ 1103 static int 1104 emit_division(struct sljit_compiler *compiler, int divt, sljit_sw divw) 1105 { 1106 int status; 1107 1108 #if BJ_XREG == SLJIT_RETURN_REG || \ 1109 BJ_XREG == SLJIT_SCRATCH_REG1 || \ 1110 BJ_XREG == SLJIT_SCRATCH_REG2 || \ 1111 BJ_AREG == SLJIT_SCRATCH_REG2 1112 #error "Not supported assignment of registers." 1113 #endif 1114 1115 #if BJ_AREG != SLJIT_SCRATCH_REG1 1116 status = sljit_emit_op1(compiler, 1117 SLJIT_MOV, 1118 SLJIT_SCRATCH_REG1, 0, 1119 BJ_AREG, 0); 1120 if (status != SLJIT_SUCCESS) 1121 return status; 1122 #endif 1123 1124 status = sljit_emit_op1(compiler, 1125 SLJIT_MOV, 1126 SLJIT_SCRATCH_REG2, 0, 1127 divt, divw); 1128 if (status != SLJIT_SUCCESS) 1129 return status; 1130 1131 #if defined(BPFJIT_USE_UDIV) 1132 status = sljit_emit_op0(compiler, SLJIT_UDIV|SLJIT_INT_OP); 1133 1134 #if BJ_AREG != SLJIT_SCRATCH_REG1 1135 status = sljit_emit_op1(compiler, 1136 SLJIT_MOV, 1137 BJ_AREG, 0, 1138 SLJIT_SCRATCH_REG1, 0); 1139 if (status != SLJIT_SUCCESS) 1140 return status; 1141 #endif 1142 #else 1143 status = sljit_emit_ijump(compiler, 1144 SLJIT_CALL2, 1145 SLJIT_IMM, SLJIT_FUNC_OFFSET(divide)); 1146 1147 #if BJ_AREG != SLJIT_RETURN_REG 1148 status = sljit_emit_op1(compiler, 1149 SLJIT_MOV, 1150 BJ_AREG, 0, 1151 SLJIT_RETURN_REG, 0); 1152 if (status != SLJIT_SUCCESS) 1153 return status; 1154 #endif 1155 #endif 1156 1157 return status; 1158 } 1159 1160 /* 1161 * Return true if pc is a "read from packet" instruction. 1162 * If length is not NULL and return value is true, *length will 1163 * be set to a safe length required to read a packet. 1164 */ 1165 static bool 1166 read_pkt_insn(const struct bpf_insn *pc, bpfjit_abc_length_t *length) 1167 { 1168 bool rv; 1169 bpfjit_abc_length_t width; 1170 1171 switch (BPF_CLASS(pc->code)) { 1172 default: 1173 rv = false; 1174 break; 1175 1176 case BPF_LD: 1177 rv = BPF_MODE(pc->code) == BPF_ABS || 1178 BPF_MODE(pc->code) == BPF_IND; 1179 if (rv) 1180 width = read_width(pc); 1181 break; 1182 1183 case BPF_LDX: 1184 rv = pc->code == (BPF_LDX|BPF_B|BPF_MSH); 1185 width = 1; 1186 break; 1187 } 1188 1189 if (rv && length != NULL) { 1190 /* 1191 * Values greater than UINT32_MAX will generate 1192 * unconditional "return 0". 1193 */ 1194 *length = (uint32_t)pc->k + width; 1195 } 1196 1197 return rv; 1198 } 1199 1200 static void 1201 optimize_init(struct bpfjit_insn_data *insn_dat, size_t insn_count) 1202 { 1203 size_t i; 1204 1205 for (i = 0; i < insn_count; i++) { 1206 SLIST_INIT(&insn_dat[i].bjumps); 1207 insn_dat[i].invalid = BJ_INIT_NOBITS; 1208 } 1209 } 1210 1211 /* 1212 * The function divides instructions into blocks. Destination of a jump 1213 * instruction starts a new block. BPF_RET and BPF_JMP instructions 1214 * terminate a block. Blocks are linear, that is, there are no jumps out 1215 * from the middle of a block and there are no jumps in to the middle of 1216 * a block. 1217 * 1218 * The function also sets bits in *initmask for memwords that 1219 * need to be initialized to zero. Note that this set should be empty 1220 * for any valid kernel filter program. 1221 */ 1222 static bool 1223 optimize_pass1(const bpf_ctx_t *bc, const struct bpf_insn *insns, 1224 struct bpfjit_insn_data *insn_dat, size_t insn_count, 1225 bpf_memword_init_t *initmask, bpfjit_hint_t *hints) 1226 { 1227 struct bpfjit_jump *jtf; 1228 size_t i; 1229 uint32_t jt, jf; 1230 bpfjit_abc_length_t length; 1231 bpf_memword_init_t invalid; /* borrowed from bpf_filter() */ 1232 bool unreachable; 1233 1234 const size_t memwords = GET_MEMWORDS(bc); 1235 1236 *hints = 0; 1237 *initmask = BJ_INIT_NOBITS; 1238 1239 unreachable = false; 1240 invalid = ~BJ_INIT_NOBITS; 1241 1242 for (i = 0; i < insn_count; i++) { 1243 if (!SLIST_EMPTY(&insn_dat[i].bjumps)) 1244 unreachable = false; 1245 insn_dat[i].unreachable = unreachable; 1246 1247 if (unreachable) 1248 continue; 1249 1250 invalid |= insn_dat[i].invalid; 1251 1252 if (read_pkt_insn(&insns[i], &length) && length > UINT32_MAX) 1253 unreachable = true; 1254 1255 switch (BPF_CLASS(insns[i].code)) { 1256 case BPF_RET: 1257 if (BPF_RVAL(insns[i].code) == BPF_A) 1258 *initmask |= invalid & BJ_INIT_ABIT; 1259 1260 unreachable = true; 1261 continue; 1262 1263 case BPF_LD: 1264 if (BPF_MODE(insns[i].code) == BPF_ABS) 1265 *hints |= BJ_HINT_ABS; 1266 1267 if (BPF_MODE(insns[i].code) == BPF_IND) { 1268 *hints |= BJ_HINT_IND | BJ_HINT_XREG; 1269 *initmask |= invalid & BJ_INIT_XBIT; 1270 } 1271 1272 if (BPF_MODE(insns[i].code) == BPF_MEM && 1273 (uint32_t)insns[i].k < memwords) { 1274 *initmask |= invalid & BJ_INIT_MBIT(insns[i].k); 1275 } 1276 1277 invalid &= ~BJ_INIT_ABIT; 1278 continue; 1279 1280 case BPF_LDX: 1281 *hints |= BJ_HINT_XREG | BJ_HINT_LDX; 1282 1283 if (BPF_MODE(insns[i].code) == BPF_MEM && 1284 (uint32_t)insns[i].k < memwords) { 1285 *initmask |= invalid & BJ_INIT_MBIT(insns[i].k); 1286 } 1287 1288 invalid &= ~BJ_INIT_XBIT; 1289 continue; 1290 1291 case BPF_ST: 1292 *initmask |= invalid & BJ_INIT_ABIT; 1293 1294 if ((uint32_t)insns[i].k < memwords) 1295 invalid &= ~BJ_INIT_MBIT(insns[i].k); 1296 1297 continue; 1298 1299 case BPF_STX: 1300 *hints |= BJ_HINT_XREG; 1301 *initmask |= invalid & BJ_INIT_XBIT; 1302 1303 if ((uint32_t)insns[i].k < memwords) 1304 invalid &= ~BJ_INIT_MBIT(insns[i].k); 1305 1306 continue; 1307 1308 case BPF_ALU: 1309 *initmask |= invalid & BJ_INIT_ABIT; 1310 1311 if (insns[i].code != (BPF_ALU|BPF_NEG) && 1312 BPF_SRC(insns[i].code) == BPF_X) { 1313 *hints |= BJ_HINT_XREG; 1314 *initmask |= invalid & BJ_INIT_XBIT; 1315 } 1316 1317 invalid &= ~BJ_INIT_ABIT; 1318 continue; 1319 1320 case BPF_MISC: 1321 switch (BPF_MISCOP(insns[i].code)) { 1322 case BPF_TAX: // X <- A 1323 *hints |= BJ_HINT_XREG; 1324 *initmask |= invalid & BJ_INIT_ABIT; 1325 invalid &= ~BJ_INIT_XBIT; 1326 continue; 1327 1328 case BPF_TXA: // A <- X 1329 *hints |= BJ_HINT_XREG; 1330 *initmask |= invalid & BJ_INIT_XBIT; 1331 invalid &= ~BJ_INIT_ABIT; 1332 continue; 1333 1334 case BPF_COPX: 1335 *hints |= BJ_HINT_XREG | BJ_HINT_COPX; 1336 /* FALLTHROUGH */ 1337 1338 case BPF_COP: 1339 *hints |= BJ_HINT_COP; 1340 *initmask |= invalid & BJ_INIT_ABIT; 1341 invalid &= ~BJ_INIT_ABIT; 1342 continue; 1343 } 1344 1345 continue; 1346 1347 case BPF_JMP: 1348 /* Initialize abc_length for ABC pass. */ 1349 insn_dat[i].u.jdata.abc_length = MAX_ABC_LENGTH; 1350 1351 if (BPF_OP(insns[i].code) == BPF_JA) { 1352 jt = jf = insns[i].k; 1353 } else { 1354 jt = insns[i].jt; 1355 jf = insns[i].jf; 1356 } 1357 1358 if (jt >= insn_count - (i + 1) || 1359 jf >= insn_count - (i + 1)) { 1360 return false; 1361 } 1362 1363 if (jt > 0 && jf > 0) 1364 unreachable = true; 1365 1366 jt += i + 1; 1367 jf += i + 1; 1368 1369 jtf = insn_dat[i].u.jdata.jtf; 1370 1371 jtf[0].jdata = &insn_dat[i].u.jdata; 1372 SLIST_INSERT_HEAD(&insn_dat[jt].bjumps, 1373 &jtf[0], entries); 1374 1375 if (jf != jt) { 1376 jtf[1].jdata = &insn_dat[i].u.jdata; 1377 SLIST_INSERT_HEAD(&insn_dat[jf].bjumps, 1378 &jtf[1], entries); 1379 } 1380 1381 insn_dat[jf].invalid |= invalid; 1382 insn_dat[jt].invalid |= invalid; 1383 invalid = 0; 1384 1385 continue; 1386 } 1387 } 1388 1389 return true; 1390 } 1391 1392 /* 1393 * Array Bounds Check Elimination (ABC) pass. 1394 */ 1395 static void 1396 optimize_pass2(const bpf_ctx_t *bc, const struct bpf_insn *insns, 1397 struct bpfjit_insn_data *insn_dat, size_t insn_count) 1398 { 1399 struct bpfjit_jump *jmp; 1400 const struct bpf_insn *pc; 1401 struct bpfjit_insn_data *pd; 1402 size_t i; 1403 bpfjit_abc_length_t length, abc_length = 0; 1404 1405 const size_t extwords = GET_EXTWORDS(bc); 1406 1407 for (i = insn_count; i != 0; i--) { 1408 pc = &insns[i-1]; 1409 pd = &insn_dat[i-1]; 1410 1411 if (pd->unreachable) 1412 continue; 1413 1414 switch (BPF_CLASS(pc->code)) { 1415 case BPF_RET: 1416 /* 1417 * It's quite common for bpf programs to 1418 * check packet bytes in increasing order 1419 * and return zero if bytes don't match 1420 * specified critetion. Such programs disable 1421 * ABC optimization completely because for 1422 * every jump there is a branch with no read 1423 * instruction. 1424 * With no side effects, BPF_STMT(BPF_RET+BPF_K, 0) 1425 * is indistinguishable from out-of-bound load. 1426 * Therefore, abc_length can be set to 1427 * MAX_ABC_LENGTH and enable ABC for many 1428 * bpf programs. 1429 * If this optimization encounters any 1430 * instruction with a side effect, it will 1431 * reset abc_length. 1432 */ 1433 if (BPF_RVAL(pc->code) == BPF_K && pc->k == 0) 1434 abc_length = MAX_ABC_LENGTH; 1435 else 1436 abc_length = 0; 1437 break; 1438 1439 case BPF_MISC: 1440 if (BPF_MISCOP(pc->code) == BPF_COP || 1441 BPF_MISCOP(pc->code) == BPF_COPX) { 1442 /* COP instructions can have side effects. */ 1443 abc_length = 0; 1444 } 1445 break; 1446 1447 case BPF_ST: 1448 case BPF_STX: 1449 if (extwords != 0) { 1450 /* Write to memory is visible after a call. */ 1451 abc_length = 0; 1452 } 1453 break; 1454 1455 case BPF_JMP: 1456 abc_length = pd->u.jdata.abc_length; 1457 break; 1458 1459 default: 1460 if (read_pkt_insn(pc, &length)) { 1461 if (abc_length < length) 1462 abc_length = length; 1463 pd->u.rdata.abc_length = abc_length; 1464 } 1465 break; 1466 } 1467 1468 SLIST_FOREACH(jmp, &pd->bjumps, entries) { 1469 if (jmp->jdata->abc_length > abc_length) 1470 jmp->jdata->abc_length = abc_length; 1471 } 1472 } 1473 } 1474 1475 static void 1476 optimize_pass3(const struct bpf_insn *insns, 1477 struct bpfjit_insn_data *insn_dat, size_t insn_count) 1478 { 1479 struct bpfjit_jump *jmp; 1480 size_t i; 1481 bpfjit_abc_length_t checked_length = 0; 1482 1483 for (i = 0; i < insn_count; i++) { 1484 if (insn_dat[i].unreachable) 1485 continue; 1486 1487 SLIST_FOREACH(jmp, &insn_dat[i].bjumps, entries) { 1488 if (jmp->jdata->checked_length < checked_length) 1489 checked_length = jmp->jdata->checked_length; 1490 } 1491 1492 if (BPF_CLASS(insns[i].code) == BPF_JMP) { 1493 insn_dat[i].u.jdata.checked_length = checked_length; 1494 } else if (read_pkt_insn(&insns[i], NULL)) { 1495 struct bpfjit_read_pkt_data *rdata = 1496 &insn_dat[i].u.rdata; 1497 rdata->check_length = 0; 1498 if (checked_length < rdata->abc_length) { 1499 checked_length = rdata->abc_length; 1500 rdata->check_length = checked_length; 1501 } 1502 } 1503 } 1504 } 1505 1506 static bool 1507 optimize(const bpf_ctx_t *bc, const struct bpf_insn *insns, 1508 struct bpfjit_insn_data *insn_dat, size_t insn_count, 1509 bpf_memword_init_t *initmask, bpfjit_hint_t *hints) 1510 { 1511 1512 optimize_init(insn_dat, insn_count); 1513 1514 if (!optimize_pass1(bc, insns, insn_dat, insn_count, initmask, hints)) 1515 return false; 1516 1517 optimize_pass2(bc, insns, insn_dat, insn_count); 1518 optimize_pass3(insns, insn_dat, insn_count); 1519 1520 return true; 1521 } 1522 1523 /* 1524 * Convert BPF_ALU operations except BPF_NEG and BPF_DIV to sljit operation. 1525 */ 1526 static int 1527 bpf_alu_to_sljit_op(const struct bpf_insn *pc) 1528 { 1529 1530 /* 1531 * Note: all supported 64bit arches have 32bit multiply 1532 * instruction so SLJIT_INT_OP doesn't have any overhead. 1533 */ 1534 switch (BPF_OP(pc->code)) { 1535 case BPF_ADD: return SLJIT_ADD; 1536 case BPF_SUB: return SLJIT_SUB; 1537 case BPF_MUL: return SLJIT_MUL|SLJIT_INT_OP; 1538 case BPF_OR: return SLJIT_OR; 1539 case BPF_AND: return SLJIT_AND; 1540 case BPF_LSH: return SLJIT_SHL; 1541 case BPF_RSH: return SLJIT_LSHR|SLJIT_INT_OP; 1542 default: 1543 BJ_ASSERT(false); 1544 return 0; 1545 } 1546 } 1547 1548 /* 1549 * Convert BPF_JMP operations except BPF_JA to sljit condition. 1550 */ 1551 static int 1552 bpf_jmp_to_sljit_cond(const struct bpf_insn *pc, bool negate) 1553 { 1554 /* 1555 * Note: all supported 64bit arches have 32bit comparison 1556 * instructions so SLJIT_INT_OP doesn't have any overhead. 1557 */ 1558 int rv = SLJIT_INT_OP; 1559 1560 switch (BPF_OP(pc->code)) { 1561 case BPF_JGT: 1562 rv |= negate ? SLJIT_C_LESS_EQUAL : SLJIT_C_GREATER; 1563 break; 1564 case BPF_JGE: 1565 rv |= negate ? SLJIT_C_LESS : SLJIT_C_GREATER_EQUAL; 1566 break; 1567 case BPF_JEQ: 1568 rv |= negate ? SLJIT_C_NOT_EQUAL : SLJIT_C_EQUAL; 1569 break; 1570 case BPF_JSET: 1571 rv |= negate ? SLJIT_C_EQUAL : SLJIT_C_NOT_EQUAL; 1572 break; 1573 default: 1574 BJ_ASSERT(false); 1575 } 1576 1577 return rv; 1578 } 1579 1580 /* 1581 * Convert BPF_K and BPF_X to sljit register. 1582 */ 1583 static int 1584 kx_to_reg(const struct bpf_insn *pc) 1585 { 1586 1587 switch (BPF_SRC(pc->code)) { 1588 case BPF_K: return SLJIT_IMM; 1589 case BPF_X: return BJ_XREG; 1590 default: 1591 BJ_ASSERT(false); 1592 return 0; 1593 } 1594 } 1595 1596 static sljit_sw 1597 kx_to_reg_arg(const struct bpf_insn *pc) 1598 { 1599 1600 switch (BPF_SRC(pc->code)) { 1601 case BPF_K: return (uint32_t)pc->k; /* SLJIT_IMM, pc->k, */ 1602 case BPF_X: return 0; /* BJ_XREG, 0, */ 1603 default: 1604 BJ_ASSERT(false); 1605 return 0; 1606 } 1607 } 1608 1609 static bool 1610 generate_insn_code(struct sljit_compiler *compiler, const bpf_ctx_t *bc, 1611 const struct bpf_insn *insns, struct bpfjit_insn_data *insn_dat, 1612 size_t insn_count) 1613 { 1614 /* a list of jumps to out-of-bound return from a generated function */ 1615 struct sljit_jump **ret0; 1616 size_t ret0_size, ret0_maxsize; 1617 1618 struct sljit_jump *jump; 1619 struct sljit_label *label; 1620 const struct bpf_insn *pc; 1621 struct bpfjit_jump *bjump, *jtf; 1622 struct sljit_jump *to_mchain_jump; 1623 1624 size_t i; 1625 int status; 1626 int branching, negate; 1627 unsigned int rval, mode, src; 1628 uint32_t jt, jf; 1629 1630 bool unconditional_ret; 1631 bool rv; 1632 1633 const size_t extwords = GET_EXTWORDS(bc); 1634 const size_t memwords = GET_MEMWORDS(bc); 1635 1636 ret0 = NULL; 1637 rv = false; 1638 1639 ret0_size = 0; 1640 ret0_maxsize = 64; 1641 ret0 = BJ_ALLOC(ret0_maxsize * sizeof(ret0[0])); 1642 if (ret0 == NULL) 1643 goto fail; 1644 1645 /* reset sjump members of jdata */ 1646 for (i = 0; i < insn_count; i++) { 1647 if (insn_dat[i].unreachable || 1648 BPF_CLASS(insns[i].code) != BPF_JMP) { 1649 continue; 1650 } 1651 1652 jtf = insn_dat[i].u.jdata.jtf; 1653 jtf[0].sjump = jtf[1].sjump = NULL; 1654 } 1655 1656 /* main loop */ 1657 for (i = 0; i < insn_count; i++) { 1658 if (insn_dat[i].unreachable) 1659 continue; 1660 1661 /* 1662 * Resolve jumps to the current insn. 1663 */ 1664 label = NULL; 1665 SLIST_FOREACH(bjump, &insn_dat[i].bjumps, entries) { 1666 if (bjump->sjump != NULL) { 1667 if (label == NULL) 1668 label = sljit_emit_label(compiler); 1669 if (label == NULL) 1670 goto fail; 1671 sljit_set_label(bjump->sjump, label); 1672 } 1673 } 1674 1675 to_mchain_jump = NULL; 1676 unconditional_ret = false; 1677 1678 if (read_pkt_insn(&insns[i], NULL)) { 1679 if (insn_dat[i].u.rdata.check_length > UINT32_MAX) { 1680 /* Jump to "return 0" unconditionally. */ 1681 unconditional_ret = true; 1682 jump = sljit_emit_jump(compiler, SLJIT_JUMP); 1683 if (jump == NULL) 1684 goto fail; 1685 if (!append_jump(jump, &ret0, 1686 &ret0_size, &ret0_maxsize)) 1687 goto fail; 1688 } else if (insn_dat[i].u.rdata.check_length > 0) { 1689 /* if (buflen < check_length) return 0; */ 1690 jump = sljit_emit_cmp(compiler, 1691 SLJIT_C_LESS, 1692 BJ_BUFLEN, 0, 1693 SLJIT_IMM, 1694 insn_dat[i].u.rdata.check_length); 1695 if (jump == NULL) 1696 goto fail; 1697 #ifdef _KERNEL 1698 to_mchain_jump = jump; 1699 #else 1700 if (!append_jump(jump, &ret0, 1701 &ret0_size, &ret0_maxsize)) 1702 goto fail; 1703 #endif 1704 } 1705 } 1706 1707 pc = &insns[i]; 1708 switch (BPF_CLASS(pc->code)) { 1709 1710 default: 1711 goto fail; 1712 1713 case BPF_LD: 1714 /* BPF_LD+BPF_IMM A <- k */ 1715 if (pc->code == (BPF_LD|BPF_IMM)) { 1716 status = sljit_emit_op1(compiler, 1717 SLJIT_MOV, 1718 BJ_AREG, 0, 1719 SLJIT_IMM, (uint32_t)pc->k); 1720 if (status != SLJIT_SUCCESS) 1721 goto fail; 1722 1723 continue; 1724 } 1725 1726 /* BPF_LD+BPF_MEM A <- M[k] */ 1727 if (pc->code == (BPF_LD|BPF_MEM)) { 1728 if ((uint32_t)pc->k >= memwords) 1729 goto fail; 1730 status = emit_memload(compiler, 1731 BJ_AREG, pc->k, extwords); 1732 if (status != SLJIT_SUCCESS) 1733 goto fail; 1734 1735 continue; 1736 } 1737 1738 /* BPF_LD+BPF_W+BPF_LEN A <- len */ 1739 if (pc->code == (BPF_LD|BPF_W|BPF_LEN)) { 1740 status = sljit_emit_op1(compiler, 1741 SLJIT_MOV, /* size_t source */ 1742 BJ_AREG, 0, 1743 SLJIT_MEM1(BJ_ARGS), 1744 offsetof(struct bpf_args, wirelen)); 1745 if (status != SLJIT_SUCCESS) 1746 goto fail; 1747 1748 continue; 1749 } 1750 1751 mode = BPF_MODE(pc->code); 1752 if (mode != BPF_ABS && mode != BPF_IND) 1753 goto fail; 1754 1755 if (unconditional_ret) 1756 continue; 1757 1758 status = emit_pkt_read(compiler, pc, 1759 to_mchain_jump, &ret0, &ret0_size, &ret0_maxsize); 1760 if (status != SLJIT_SUCCESS) 1761 goto fail; 1762 1763 continue; 1764 1765 case BPF_LDX: 1766 mode = BPF_MODE(pc->code); 1767 1768 /* BPF_LDX+BPF_W+BPF_IMM X <- k */ 1769 if (mode == BPF_IMM) { 1770 if (BPF_SIZE(pc->code) != BPF_W) 1771 goto fail; 1772 status = sljit_emit_op1(compiler, 1773 SLJIT_MOV, 1774 BJ_XREG, 0, 1775 SLJIT_IMM, (uint32_t)pc->k); 1776 if (status != SLJIT_SUCCESS) 1777 goto fail; 1778 1779 continue; 1780 } 1781 1782 /* BPF_LDX+BPF_W+BPF_LEN X <- len */ 1783 if (mode == BPF_LEN) { 1784 if (BPF_SIZE(pc->code) != BPF_W) 1785 goto fail; 1786 status = sljit_emit_op1(compiler, 1787 SLJIT_MOV, /* size_t source */ 1788 BJ_XREG, 0, 1789 SLJIT_MEM1(BJ_ARGS), 1790 offsetof(struct bpf_args, wirelen)); 1791 if (status != SLJIT_SUCCESS) 1792 goto fail; 1793 1794 continue; 1795 } 1796 1797 /* BPF_LDX+BPF_W+BPF_MEM X <- M[k] */ 1798 if (mode == BPF_MEM) { 1799 if (BPF_SIZE(pc->code) != BPF_W) 1800 goto fail; 1801 if ((uint32_t)pc->k >= memwords) 1802 goto fail; 1803 status = emit_memload(compiler, 1804 BJ_XREG, pc->k, extwords); 1805 if (status != SLJIT_SUCCESS) 1806 goto fail; 1807 1808 continue; 1809 } 1810 1811 /* BPF_LDX+BPF_B+BPF_MSH X <- 4*(P[k:1]&0xf) */ 1812 if (mode != BPF_MSH || BPF_SIZE(pc->code) != BPF_B) 1813 goto fail; 1814 1815 if (unconditional_ret) 1816 continue; 1817 1818 status = emit_msh(compiler, pc, 1819 to_mchain_jump, &ret0, &ret0_size, &ret0_maxsize); 1820 if (status != SLJIT_SUCCESS) 1821 goto fail; 1822 1823 continue; 1824 1825 case BPF_ST: 1826 if (pc->code != BPF_ST || 1827 (uint32_t)pc->k >= memwords) { 1828 goto fail; 1829 } 1830 1831 status = emit_memstore(compiler, 1832 BJ_AREG, pc->k, extwords); 1833 if (status != SLJIT_SUCCESS) 1834 goto fail; 1835 1836 continue; 1837 1838 case BPF_STX: 1839 if (pc->code != BPF_STX || 1840 (uint32_t)pc->k >= memwords) { 1841 goto fail; 1842 } 1843 1844 status = emit_memstore(compiler, 1845 BJ_XREG, pc->k, extwords); 1846 if (status != SLJIT_SUCCESS) 1847 goto fail; 1848 1849 continue; 1850 1851 case BPF_ALU: 1852 if (pc->code == (BPF_ALU|BPF_NEG)) { 1853 status = sljit_emit_op1(compiler, 1854 SLJIT_NEG, 1855 BJ_AREG, 0, 1856 BJ_AREG, 0); 1857 if (status != SLJIT_SUCCESS) 1858 goto fail; 1859 1860 continue; 1861 } 1862 1863 if (BPF_OP(pc->code) != BPF_DIV) { 1864 status = sljit_emit_op2(compiler, 1865 bpf_alu_to_sljit_op(pc), 1866 BJ_AREG, 0, 1867 BJ_AREG, 0, 1868 kx_to_reg(pc), kx_to_reg_arg(pc)); 1869 if (status != SLJIT_SUCCESS) 1870 goto fail; 1871 1872 continue; 1873 } 1874 1875 /* BPF_DIV */ 1876 1877 src = BPF_SRC(pc->code); 1878 if (src != BPF_X && src != BPF_K) 1879 goto fail; 1880 1881 /* division by zero? */ 1882 if (src == BPF_X) { 1883 jump = sljit_emit_cmp(compiler, 1884 SLJIT_C_EQUAL|SLJIT_INT_OP, 1885 BJ_XREG, 0, 1886 SLJIT_IMM, 0); 1887 if (jump == NULL) 1888 goto fail; 1889 if (!append_jump(jump, &ret0, 1890 &ret0_size, &ret0_maxsize)) 1891 goto fail; 1892 } else if (pc->k == 0) { 1893 jump = sljit_emit_jump(compiler, SLJIT_JUMP); 1894 if (jump == NULL) 1895 goto fail; 1896 if (!append_jump(jump, &ret0, 1897 &ret0_size, &ret0_maxsize)) 1898 goto fail; 1899 } 1900 1901 if (src == BPF_X) { 1902 status = emit_division(compiler, BJ_XREG, 0); 1903 if (status != SLJIT_SUCCESS) 1904 goto fail; 1905 } else if (pc->k != 0) { 1906 if (pc->k & (pc->k - 1)) { 1907 status = emit_division(compiler, 1908 SLJIT_IMM, (uint32_t)pc->k); 1909 } else { 1910 status = emit_pow2_division(compiler, 1911 (uint32_t)pc->k); 1912 } 1913 if (status != SLJIT_SUCCESS) 1914 goto fail; 1915 } 1916 1917 continue; 1918 1919 case BPF_JMP: 1920 if (BPF_OP(pc->code) == BPF_JA) { 1921 jt = jf = pc->k; 1922 } else { 1923 jt = pc->jt; 1924 jf = pc->jf; 1925 } 1926 1927 negate = (jt == 0) ? 1 : 0; 1928 branching = (jt == jf) ? 0 : 1; 1929 jtf = insn_dat[i].u.jdata.jtf; 1930 1931 if (branching) { 1932 if (BPF_OP(pc->code) != BPF_JSET) { 1933 jump = sljit_emit_cmp(compiler, 1934 bpf_jmp_to_sljit_cond(pc, negate), 1935 BJ_AREG, 0, 1936 kx_to_reg(pc), kx_to_reg_arg(pc)); 1937 } else { 1938 status = sljit_emit_op2(compiler, 1939 SLJIT_AND, 1940 BJ_TMP1REG, 0, 1941 BJ_AREG, 0, 1942 kx_to_reg(pc), kx_to_reg_arg(pc)); 1943 if (status != SLJIT_SUCCESS) 1944 goto fail; 1945 1946 jump = sljit_emit_cmp(compiler, 1947 bpf_jmp_to_sljit_cond(pc, negate), 1948 BJ_TMP1REG, 0, 1949 SLJIT_IMM, 0); 1950 } 1951 1952 if (jump == NULL) 1953 goto fail; 1954 1955 BJ_ASSERT(jtf[negate].sjump == NULL); 1956 jtf[negate].sjump = jump; 1957 } 1958 1959 if (!branching || (jt != 0 && jf != 0)) { 1960 jump = sljit_emit_jump(compiler, SLJIT_JUMP); 1961 if (jump == NULL) 1962 goto fail; 1963 1964 BJ_ASSERT(jtf[branching].sjump == NULL); 1965 jtf[branching].sjump = jump; 1966 } 1967 1968 continue; 1969 1970 case BPF_RET: 1971 rval = BPF_RVAL(pc->code); 1972 if (rval == BPF_X) 1973 goto fail; 1974 1975 /* BPF_RET+BPF_K accept k bytes */ 1976 if (rval == BPF_K) { 1977 status = sljit_emit_return(compiler, 1978 SLJIT_MOV_UI, 1979 SLJIT_IMM, (uint32_t)pc->k); 1980 if (status != SLJIT_SUCCESS) 1981 goto fail; 1982 } 1983 1984 /* BPF_RET+BPF_A accept A bytes */ 1985 if (rval == BPF_A) { 1986 status = sljit_emit_return(compiler, 1987 SLJIT_MOV_UI, 1988 BJ_AREG, 0); 1989 if (status != SLJIT_SUCCESS) 1990 goto fail; 1991 } 1992 1993 continue; 1994 1995 case BPF_MISC: 1996 switch (BPF_MISCOP(pc->code)) { 1997 case BPF_TAX: 1998 status = sljit_emit_op1(compiler, 1999 SLJIT_MOV_UI, 2000 BJ_XREG, 0, 2001 BJ_AREG, 0); 2002 if (status != SLJIT_SUCCESS) 2003 goto fail; 2004 2005 continue; 2006 2007 case BPF_TXA: 2008 status = sljit_emit_op1(compiler, 2009 SLJIT_MOV, 2010 BJ_AREG, 0, 2011 BJ_XREG, 0); 2012 if (status != SLJIT_SUCCESS) 2013 goto fail; 2014 2015 continue; 2016 2017 case BPF_COP: 2018 case BPF_COPX: 2019 if (bc == NULL || bc->copfuncs == NULL) 2020 goto fail; 2021 if (BPF_MISCOP(pc->code) == BPF_COP && 2022 (uint32_t)pc->k >= bc->nfuncs) { 2023 goto fail; 2024 } 2025 2026 status = emit_cop(compiler, bc, pc, 2027 &ret0, &ret0_size, &ret0_maxsize); 2028 if (status != SLJIT_SUCCESS) 2029 goto fail; 2030 2031 continue; 2032 } 2033 2034 goto fail; 2035 } /* switch */ 2036 } /* main loop */ 2037 2038 BJ_ASSERT(ret0_size <= ret0_maxsize); 2039 2040 if (ret0_size > 0) { 2041 label = sljit_emit_label(compiler); 2042 if (label == NULL) 2043 goto fail; 2044 for (i = 0; i < ret0_size; i++) 2045 sljit_set_label(ret0[i], label); 2046 } 2047 2048 status = sljit_emit_return(compiler, 2049 SLJIT_MOV_UI, 2050 SLJIT_IMM, 0); 2051 if (status != SLJIT_SUCCESS) 2052 goto fail; 2053 2054 rv = true; 2055 2056 fail: 2057 if (ret0 != NULL) 2058 BJ_FREE(ret0, ret0_maxsize * sizeof(ret0[0])); 2059 2060 return rv; 2061 } 2062 2063 bpfjit_func_t 2064 bpfjit_generate_code(const bpf_ctx_t *bc, 2065 const struct bpf_insn *insns, size_t insn_count) 2066 { 2067 void *rv; 2068 struct sljit_compiler *compiler; 2069 2070 size_t i; 2071 int status; 2072 2073 /* optimization related */ 2074 bpf_memword_init_t initmask; 2075 bpfjit_hint_t hints; 2076 2077 /* memory store location for initial zero initialization */ 2078 sljit_si mem_reg; 2079 sljit_sw mem_off; 2080 2081 struct bpfjit_insn_data *insn_dat; 2082 2083 const size_t extwords = GET_EXTWORDS(bc); 2084 const size_t memwords = GET_MEMWORDS(bc); 2085 const bpf_memword_init_t preinited = extwords ? bc->preinited : 0; 2086 2087 rv = NULL; 2088 compiler = NULL; 2089 insn_dat = NULL; 2090 2091 if (memwords > MAX_MEMWORDS) 2092 goto fail; 2093 2094 if (insn_count == 0 || insn_count > SIZE_MAX / sizeof(insn_dat[0])) 2095 goto fail; 2096 2097 insn_dat = BJ_ALLOC(insn_count * sizeof(insn_dat[0])); 2098 if (insn_dat == NULL) 2099 goto fail; 2100 2101 if (!optimize(bc, insns, insn_dat, insn_count, &initmask, &hints)) 2102 goto fail; 2103 2104 compiler = sljit_create_compiler(); 2105 if (compiler == NULL) 2106 goto fail; 2107 2108 #if !defined(_KERNEL) && defined(SLJIT_VERBOSE) && SLJIT_VERBOSE 2109 sljit_compiler_verbose(compiler, stderr); 2110 #endif 2111 2112 status = sljit_emit_enter(compiler, 2113 2, nscratches(hints), 3, sizeof(struct bpfjit_stack)); 2114 if (status != SLJIT_SUCCESS) 2115 goto fail; 2116 2117 if (hints & BJ_HINT_COP) { 2118 /* save ctx argument */ 2119 status = sljit_emit_op1(compiler, 2120 SLJIT_MOV_P, 2121 SLJIT_MEM1(SLJIT_LOCALS_REG), 2122 offsetof(struct bpfjit_stack, ctx), 2123 BJ_CTX_ARG, 0); 2124 if (status != SLJIT_SUCCESS) 2125 goto fail; 2126 } 2127 2128 if (extwords == 0) { 2129 mem_reg = SLJIT_MEM1(SLJIT_LOCALS_REG); 2130 mem_off = offsetof(struct bpfjit_stack, mem); 2131 } else { 2132 /* copy "mem" argument from bpf_args to bpfjit_stack */ 2133 status = sljit_emit_op1(compiler, 2134 SLJIT_MOV_P, 2135 BJ_TMP1REG, 0, 2136 SLJIT_MEM1(BJ_ARGS), offsetof(struct bpf_args, mem)); 2137 if (status != SLJIT_SUCCESS) 2138 goto fail; 2139 2140 status = sljit_emit_op1(compiler, 2141 SLJIT_MOV_P, 2142 SLJIT_MEM1(SLJIT_LOCALS_REG), 2143 offsetof(struct bpfjit_stack, extmem), 2144 BJ_TMP1REG, 0); 2145 if (status != SLJIT_SUCCESS) 2146 goto fail; 2147 2148 mem_reg = SLJIT_MEM1(BJ_TMP1REG); 2149 mem_off = 0; 2150 } 2151 2152 /* 2153 * Exclude pre-initialised external memory words but keep 2154 * initialization statuses of A and X registers in case 2155 * bc->preinited wrongly sets those two bits. 2156 */ 2157 initmask &= ~preinited | BJ_INIT_ABIT | BJ_INIT_XBIT; 2158 2159 #if defined(_KERNEL) 2160 /* bpf_filter() checks initialization of memwords. */ 2161 BJ_ASSERT((initmask & (BJ_INIT_MBIT(memwords) - 1)) == 0); 2162 #endif 2163 for (i = 0; i < memwords; i++) { 2164 if (initmask & BJ_INIT_MBIT(i)) { 2165 /* M[i] = 0; */ 2166 status = sljit_emit_op1(compiler, 2167 SLJIT_MOV_UI, 2168 mem_reg, mem_off + i * sizeof(uint32_t), 2169 SLJIT_IMM, 0); 2170 if (status != SLJIT_SUCCESS) 2171 goto fail; 2172 } 2173 } 2174 2175 if (initmask & BJ_INIT_ABIT) { 2176 /* A = 0; */ 2177 status = sljit_emit_op1(compiler, 2178 SLJIT_MOV, 2179 BJ_AREG, 0, 2180 SLJIT_IMM, 0); 2181 if (status != SLJIT_SUCCESS) 2182 goto fail; 2183 } 2184 2185 if (initmask & BJ_INIT_XBIT) { 2186 /* X = 0; */ 2187 status = sljit_emit_op1(compiler, 2188 SLJIT_MOV, 2189 BJ_XREG, 0, 2190 SLJIT_IMM, 0); 2191 if (status != SLJIT_SUCCESS) 2192 goto fail; 2193 } 2194 2195 status = load_buf_buflen(compiler); 2196 if (status != SLJIT_SUCCESS) 2197 goto fail; 2198 2199 if (!generate_insn_code(compiler, bc, insns, insn_dat, insn_count)) 2200 goto fail; 2201 2202 rv = sljit_generate_code(compiler); 2203 2204 fail: 2205 if (compiler != NULL) 2206 sljit_free_compiler(compiler); 2207 2208 if (insn_dat != NULL) 2209 BJ_FREE(insn_dat, insn_count * sizeof(insn_dat[0])); 2210 2211 return (bpfjit_func_t)rv; 2212 } 2213 2214 void 2215 bpfjit_free_code(bpfjit_func_t code) 2216 { 2217 2218 sljit_free_code((void *)code); 2219 } 2220
Indexes created Fri Sep 26 20:09:58 GMT 2025