bpfjit.c revision 1.25
1 /* $NetBSD: bpfjit.c,v 1.25 2014/07/12 16:52:57 alnsn Exp $ */ 2 3 /*- 4 * Copyright (c) 2011-2014 Alexander Nasonov. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 21 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 22 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 23 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 24 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 25 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 26 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 27 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 28 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 #ifdef _KERNEL 34 __KERNEL_RCSID(0, "$NetBSD: bpfjit.c,v 1.25 2014/07/12 16:52:57 alnsn Exp $"); 35 #else 36 __RCSID("$NetBSD: bpfjit.c,v 1.25 2014/07/12 16:52:57 alnsn Exp $"); 37 #endif 38 39 #include <sys/types.h> 40 #include <sys/queue.h> 41 42 #ifndef _KERNEL 43 #include <assert.h> 44 #define BJ_ASSERT(c) assert(c) 45 #else 46 #define BJ_ASSERT(c) KASSERT(c) 47 #endif 48 49 #ifndef _KERNEL 50 #include <stdlib.h> 51 #define BJ_ALLOC(sz) malloc(sz) 52 #define BJ_FREE(p, sz) free(p) 53 #else 54 #include <sys/kmem.h> 55 #define BJ_ALLOC(sz) kmem_alloc(sz, KM_SLEEP) 56 #define BJ_FREE(p, sz) kmem_free(p, sz) 57 #endif 58 59 #ifndef _KERNEL 60 #include <limits.h> 61 #include <stdbool.h> 62 #include <stddef.h> 63 #include <stdint.h> 64 #else 65 #include <sys/atomic.h> 66 #include <sys/module.h> 67 #endif 68 69 #define __BPF_PRIVATE 70 #include <net/bpf.h> 71 #include <net/bpfjit.h> 72 #include <sljitLir.h> 73 74 #if !defined(_KERNEL) && defined(SLJIT_VERBOSE) && SLJIT_VERBOSE 75 #include <stdio.h> /* for stderr */ 76 #endif 77 78 /* 79 * Arguments of generated bpfjit_func_t. 80 * The first argument is reassigned upon entry 81 * to a more frequently used buf argument. 82 */ 83 #define BJ_CTX_ARG SLJIT_SAVED_REG1 84 #define BJ_ARGS SLJIT_SAVED_REG2 85 86 /* 87 * Permanent register assignments. 88 */ 89 #define BJ_BUF SLJIT_SAVED_REG1 90 //#define BJ_ARGS SLJIT_SAVED_REG2 91 #define BJ_BUFLEN SLJIT_SAVED_REG3 92 #define BJ_AREG SLJIT_SCRATCH_REG1 93 #define BJ_TMP1REG SLJIT_SCRATCH_REG2 94 #define BJ_TMP2REG SLJIT_SCRATCH_REG3 95 #define BJ_XREG SLJIT_TEMPORARY_EREG1 96 #define BJ_TMP3REG SLJIT_TEMPORARY_EREG2 97 98 /* 99 * EREG registers can't be used for indirect calls, reuse BJ_BUF and 100 * BJ_BUFLEN registers. They can be easily restored from BJ_ARGS. 101 */ 102 #define BJ_COPF_PTR SLJIT_SAVED_REG1 103 #define BJ_COPF_IDX SLJIT_SAVED_REG3 104 105 #ifdef _KERNEL 106 #define MAX_MEMWORDS BPF_MAX_MEMWORDS 107 #else 108 #define MAX_MEMWORDS BPF_MEMWORDS 109 #endif 110 111 #define BJ_INIT_NOBITS ((bpf_memword_init_t)0) 112 #define BJ_INIT_MBIT(k) BPF_MEMWORD_INIT(k) 113 #define BJ_INIT_ABIT BJ_INIT_MBIT(MAX_MEMWORDS) 114 #define BJ_INIT_XBIT BJ_INIT_MBIT(MAX_MEMWORDS + 1) 115 116 /* 117 * Get a number of memwords and external memwords from a bpf_ctx object. 118 */ 119 #define GET_EXTWORDS(bc) ((bc) ? (bc)->extwords : 0) 120 #define GET_MEMWORDS(bc) (GET_EXTWORDS(bc) ? GET_EXTWORDS(bc) : BPF_MEMWORDS) 121 122 /* 123 * Optimization hints. 124 */ 125 typedef unsigned int bpfjit_hint_t; 126 #define BJ_HINT_PKT 0x01 /* packet read */ 127 #define BJ_HINT_LDW 0x02 /* 32-bit load */ 128 #define BJ_HINT_COP 0x04 /* BPF_COP or BPF_COPX instruction */ 129 #define BJ_HINT_XREG 0x08 /* BJ_XREG is needed */ 130 #define BJ_HINT_LDX 0x10 /* BPF_LDX instruction */ 131 132 /* 133 * Datatype for Array Bounds Check Elimination (ABC) pass. 134 */ 135 typedef uint64_t bpfjit_abc_length_t; 136 #define MAX_ABC_LENGTH (UINT32_MAX + UINT64_C(4)) /* max. width is 4 */ 137 138 struct bpfjit_stack 139 { 140 bpf_ctx_t *ctx; 141 uint32_t *extmem; /* pointer to external memory store */ 142 #ifdef _KERNEL 143 int err; /* 3rd argument for m_xword/m_xhalf/m_xbyte function call */ 144 #endif 145 uint32_t mem[BPF_MEMWORDS]; /* internal memory store */ 146 }; 147 148 /* 149 * Data for BPF_JMP instruction. 150 * Forward declaration for struct bpfjit_jump. 151 */ 152 struct bpfjit_jump_data; 153 154 /* 155 * Node of bjumps list. 156 */ 157 struct bpfjit_jump { 158 struct sljit_jump *sjump; 159 SLIST_ENTRY(bpfjit_jump) entries; 160 struct bpfjit_jump_data *jdata; 161 }; 162 163 /* 164 * Data for BPF_JMP instruction. 165 */ 166 struct bpfjit_jump_data { 167 /* 168 * These entries make up bjumps list: 169 * jtf[0] - when coming from jt path, 170 * jtf[1] - when coming from jf path. 171 */ 172 struct bpfjit_jump jtf[2]; 173 /* 174 * Length calculated by Array Bounds Check Elimination (ABC) pass. 175 */ 176 bpfjit_abc_length_t abc_length; 177 /* 178 * Length checked by the last out-of-bounds check. 179 */ 180 bpfjit_abc_length_t checked_length; 181 }; 182 183 /* 184 * Data for "read from packet" instructions. 185 * See also read_pkt_insn() function below. 186 */ 187 struct bpfjit_read_pkt_data { 188 /* 189 * Length calculated by Array Bounds Check Elimination (ABC) pass. 190 */ 191 bpfjit_abc_length_t abc_length; 192 /* 193 * If positive, emit "if (buflen < check_length) return 0" 194 * out-of-bounds check. 195 * Values greater than UINT32_MAX generate unconditional "return 0". 196 */ 197 bpfjit_abc_length_t check_length; 198 }; 199 200 /* 201 * Additional (optimization-related) data for bpf_insn. 202 */ 203 struct bpfjit_insn_data { 204 /* List of jumps to this insn. */ 205 SLIST_HEAD(, bpfjit_jump) bjumps; 206 207 union { 208 struct bpfjit_jump_data jdata; 209 struct bpfjit_read_pkt_data rdata; 210 } u; 211 212 bpf_memword_init_t invalid; 213 bool unreachable; 214 }; 215 216 #ifdef _KERNEL 217 218 uint32_t m_xword(const struct mbuf *, uint32_t, int *); 219 uint32_t m_xhalf(const struct mbuf *, uint32_t, int *); 220 uint32_t m_xbyte(const struct mbuf *, uint32_t, int *); 221 222 MODULE(MODULE_CLASS_MISC, bpfjit, "sljit") 223 224 static int 225 bpfjit_modcmd(modcmd_t cmd, void *arg) 226 { 227 228 switch (cmd) { 229 case MODULE_CMD_INIT: 230 bpfjit_module_ops.bj_free_code = &bpfjit_free_code; 231 membar_producer(); 232 bpfjit_module_ops.bj_generate_code = &bpfjit_generate_code; 233 membar_producer(); 234 return 0; 235 236 case MODULE_CMD_FINI: 237 return EOPNOTSUPP; 238 239 default: 240 return ENOTTY; 241 } 242 } 243 #endif 244 245 /* 246 * Return a number of scratch registers to pass 247 * to sljit_emit_enter() function. 248 */ 249 static sljit_si 250 nscratches(bpfjit_hint_t hints) 251 { 252 sljit_si rv = 2; 253 254 #ifdef _KERNEL 255 if (hints & BJ_HINT_PKT) 256 rv = 3; /* xcall with three arguments */ 257 #endif 258 259 if (hints & (BJ_HINT_LDW|BJ_HINT_PKT)) 260 rv = 3; /* uses BJ_TMP2REG */ 261 262 if (hints & BJ_HINT_COP) 263 rv = 3; /* calls copfunc with three arguments */ 264 265 if (hints & BJ_HINT_XREG) 266 rv = 4; /* uses BJ_XREG */ 267 268 #ifdef _KERNEL 269 if (hints & BJ_HINT_LDX) 270 rv = 5; /* uses BJ_TMP3REG */ 271 #endif 272 273 return rv; 274 } 275 276 static uint32_t 277 read_width(const struct bpf_insn *pc) 278 { 279 280 switch (BPF_SIZE(pc->code)) { 281 case BPF_W: 282 return 4; 283 case BPF_H: 284 return 2; 285 case BPF_B: 286 return 1; 287 default: 288 BJ_ASSERT(false); 289 return 0; 290 } 291 } 292 293 /* 294 * Copy buf and buflen members of bpf_args from BJ_ARGS 295 * pointer to BJ_BUF and BJ_BUFLEN registers. 296 */ 297 static int 298 load_buf_buflen(struct sljit_compiler *compiler) 299 { 300 int status; 301 302 status = sljit_emit_op1(compiler, 303 SLJIT_MOV_P, 304 BJ_BUF, 0, 305 SLJIT_MEM1(BJ_ARGS), 306 offsetof(struct bpf_args, pkt)); 307 if (status != SLJIT_SUCCESS) 308 return status; 309 310 status = sljit_emit_op1(compiler, 311 SLJIT_MOV, /* size_t source */ 312 BJ_BUFLEN, 0, 313 SLJIT_MEM1(BJ_ARGS), 314 offsetof(struct bpf_args, buflen)); 315 316 return status; 317 } 318 319 static bool 320 grow_jumps(struct sljit_jump ***jumps, size_t *size) 321 { 322 struct sljit_jump **newptr; 323 const size_t elemsz = sizeof(struct sljit_jump *); 324 size_t old_size = *size; 325 size_t new_size = 2 * old_size; 326 327 if (new_size < old_size || new_size > SIZE_MAX / elemsz) 328 return false; 329 330 newptr = BJ_ALLOC(new_size * elemsz); 331 if (newptr == NULL) 332 return false; 333 334 memcpy(newptr, *jumps, old_size * elemsz); 335 BJ_FREE(*jumps, old_size * elemsz); 336 337 *jumps = newptr; 338 *size = new_size; 339 return true; 340 } 341 342 static bool 343 append_jump(struct sljit_jump *jump, struct sljit_jump ***jumps, 344 size_t *size, size_t *max_size) 345 { 346 if (*size == *max_size && !grow_jumps(jumps, max_size)) 347 return false; 348 349 (*jumps)[(*size)++] = jump; 350 return true; 351 } 352 353 /* 354 * Emit code for BPF_LD+BPF_B+BPF_ABS A <- P[k:1]. 355 */ 356 static int 357 emit_read8(struct sljit_compiler *compiler, uint32_t k) 358 { 359 360 return sljit_emit_op1(compiler, 361 SLJIT_MOV_UB, 362 BJ_AREG, 0, 363 SLJIT_MEM1(BJ_BUF), k); 364 } 365 366 /* 367 * Emit code for BPF_LD+BPF_H+BPF_ABS A <- P[k:2]. 368 */ 369 static int 370 emit_read16(struct sljit_compiler *compiler, uint32_t k) 371 { 372 int status; 373 374 /* tmp1 = buf[k]; */ 375 status = sljit_emit_op1(compiler, 376 SLJIT_MOV_UB, 377 BJ_TMP1REG, 0, 378 SLJIT_MEM1(BJ_BUF), k); 379 if (status != SLJIT_SUCCESS) 380 return status; 381 382 /* A = buf[k+1]; */ 383 status = sljit_emit_op1(compiler, 384 SLJIT_MOV_UB, 385 BJ_AREG, 0, 386 SLJIT_MEM1(BJ_BUF), k+1); 387 if (status != SLJIT_SUCCESS) 388 return status; 389 390 /* tmp1 = tmp1 << 8; */ 391 status = sljit_emit_op2(compiler, 392 SLJIT_SHL, 393 BJ_TMP1REG, 0, 394 BJ_TMP1REG, 0, 395 SLJIT_IMM, 8); 396 if (status != SLJIT_SUCCESS) 397 return status; 398 399 /* A = A + tmp1; */ 400 status = sljit_emit_op2(compiler, 401 SLJIT_ADD, 402 BJ_AREG, 0, 403 BJ_AREG, 0, 404 BJ_TMP1REG, 0); 405 return status; 406 } 407 408 /* 409 * Emit code for BPF_LD+BPF_W+BPF_ABS A <- P[k:4]. 410 */ 411 static int 412 emit_read32(struct sljit_compiler *compiler, uint32_t k) 413 { 414 int status; 415 416 /* tmp1 = buf[k]; */ 417 status = sljit_emit_op1(compiler, 418 SLJIT_MOV_UB, 419 BJ_TMP1REG, 0, 420 SLJIT_MEM1(BJ_BUF), k); 421 if (status != SLJIT_SUCCESS) 422 return status; 423 424 /* tmp2 = buf[k+1]; */ 425 status = sljit_emit_op1(compiler, 426 SLJIT_MOV_UB, 427 BJ_TMP2REG, 0, 428 SLJIT_MEM1(BJ_BUF), k+1); 429 if (status != SLJIT_SUCCESS) 430 return status; 431 432 /* A = buf[k+3]; */ 433 status = sljit_emit_op1(compiler, 434 SLJIT_MOV_UB, 435 BJ_AREG, 0, 436 SLJIT_MEM1(BJ_BUF), k+3); 437 if (status != SLJIT_SUCCESS) 438 return status; 439 440 /* tmp1 = tmp1 << 24; */ 441 status = sljit_emit_op2(compiler, 442 SLJIT_SHL, 443 BJ_TMP1REG, 0, 444 BJ_TMP1REG, 0, 445 SLJIT_IMM, 24); 446 if (status != SLJIT_SUCCESS) 447 return status; 448 449 /* A = A + tmp1; */ 450 status = sljit_emit_op2(compiler, 451 SLJIT_ADD, 452 BJ_AREG, 0, 453 BJ_AREG, 0, 454 BJ_TMP1REG, 0); 455 if (status != SLJIT_SUCCESS) 456 return status; 457 458 /* tmp1 = buf[k+2]; */ 459 status = sljit_emit_op1(compiler, 460 SLJIT_MOV_UB, 461 BJ_TMP1REG, 0, 462 SLJIT_MEM1(BJ_BUF), k+2); 463 if (status != SLJIT_SUCCESS) 464 return status; 465 466 /* tmp2 = tmp2 << 16; */ 467 status = sljit_emit_op2(compiler, 468 SLJIT_SHL, 469 BJ_TMP2REG, 0, 470 BJ_TMP2REG, 0, 471 SLJIT_IMM, 16); 472 if (status != SLJIT_SUCCESS) 473 return status; 474 475 /* A = A + tmp2; */ 476 status = sljit_emit_op2(compiler, 477 SLJIT_ADD, 478 BJ_AREG, 0, 479 BJ_AREG, 0, 480 BJ_TMP2REG, 0); 481 if (status != SLJIT_SUCCESS) 482 return status; 483 484 /* tmp1 = tmp1 << 8; */ 485 status = sljit_emit_op2(compiler, 486 SLJIT_SHL, 487 BJ_TMP1REG, 0, 488 BJ_TMP1REG, 0, 489 SLJIT_IMM, 8); 490 if (status != SLJIT_SUCCESS) 491 return status; 492 493 /* A = A + tmp1; */ 494 status = sljit_emit_op2(compiler, 495 SLJIT_ADD, 496 BJ_AREG, 0, 497 BJ_AREG, 0, 498 BJ_TMP1REG, 0); 499 return status; 500 } 501 502 #ifdef _KERNEL 503 /* 504 * Emit code for m_xword/m_xhalf/m_xbyte call. 505 * 506 * @pc BPF_LD+BPF_W+BPF_ABS A <- P[k:4] 507 * BPF_LD+BPF_H+BPF_ABS A <- P[k:2] 508 * BPF_LD+BPF_B+BPF_ABS A <- P[k:1] 509 * BPF_LD+BPF_W+BPF_IND A <- P[X+k:4] 510 * BPF_LD+BPF_H+BPF_IND A <- P[X+k:2] 511 * BPF_LD+BPF_B+BPF_IND A <- P[X+k:1] 512 * BPF_LDX+BPF_B+BPF_MSH X <- 4*(P[k:1]&0xf) 513 */ 514 static int 515 emit_xcall(struct sljit_compiler *compiler, const struct bpf_insn *pc, 516 int dst, struct sljit_jump ***ret0, size_t *ret0_size, size_t *ret0_maxsize, 517 uint32_t (*fn)(const struct mbuf *, uint32_t, int *)) 518 { 519 #if BJ_XREG == SLJIT_RETURN_REG || \ 520 BJ_XREG == SLJIT_SCRATCH_REG1 || \ 521 BJ_XREG == SLJIT_SCRATCH_REG2 || \ 522 BJ_XREG == SLJIT_SCRATCH_REG3 || \ 523 BJ_TMP3REG == SLJIT_RETURN_REG || \ 524 BJ_TMP3REG == SLJIT_SCRATCH_REG1 || \ 525 BJ_TMP3REG == SLJIT_SCRATCH_REG2 || \ 526 BJ_TMP3REG == SLJIT_SCRATCH_REG3 527 #error "Not supported assignment of registers." 528 #endif 529 struct sljit_jump *jump; 530 int status; 531 532 BJ_ASSERT(dst != BJ_TMP2REG && dst != BJ_TMP3REG); 533 534 if (BPF_CLASS(pc->code) == BPF_LDX) { 535 /* save A */ 536 status = sljit_emit_op1(compiler, 537 SLJIT_MOV, 538 BJ_TMP3REG, 0, 539 BJ_AREG, 0); 540 if (status != SLJIT_SUCCESS) 541 return status; 542 } 543 544 /* 545 * Prepare registers for fn(mbuf, k, &err) call. 546 */ 547 status = sljit_emit_op1(compiler, 548 SLJIT_MOV, 549 SLJIT_SCRATCH_REG1, 0, 550 BJ_BUF, 0); 551 if (status != SLJIT_SUCCESS) 552 return status; 553 554 if (BPF_CLASS(pc->code) == BPF_LD && BPF_MODE(pc->code) == BPF_IND) { 555 /* k = X + pc->k; */ 556 status = sljit_emit_op2(compiler, 557 SLJIT_ADD | SLJIT_INT_OP, 558 SLJIT_SCRATCH_REG2, 0, 559 BJ_XREG, 0, 560 SLJIT_IMM, (uint32_t)pc->k); 561 if (status != SLJIT_SUCCESS) 562 return status; 563 564 /* if (k < X) return 0; */ 565 jump = sljit_emit_cmp(compiler, 566 SLJIT_C_LESS, 567 SLJIT_SCRATCH_REG2, 0, 568 BJ_XREG, 0); 569 if (jump == NULL) 570 return SLJIT_ERR_ALLOC_FAILED; 571 572 if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 573 return SLJIT_ERR_ALLOC_FAILED; 574 } else { 575 /* k = pc->k */ 576 status = sljit_emit_op1(compiler, 577 SLJIT_MOV, 578 SLJIT_SCRATCH_REG2, 0, 579 SLJIT_IMM, (uint32_t)pc->k); 580 if (status != SLJIT_SUCCESS) 581 return status; 582 } 583 584 /* 585 * The third argument of fn is an address on stack. 586 */ 587 status = sljit_get_local_base(compiler, 588 SLJIT_SCRATCH_REG3, 0, 589 offsetof(struct bpfjit_stack, err)); 590 if (status != SLJIT_SUCCESS) 591 return status; 592 593 /* fn(buf, k, &err); */ 594 status = sljit_emit_ijump(compiler, 595 SLJIT_CALL3, 596 SLJIT_IMM, SLJIT_FUNC_OFFSET(fn)); 597 if (status != SLJIT_SUCCESS) 598 return status; 599 600 if (dst != SLJIT_RETURN_REG) { 601 /* move return value to dst */ 602 status = sljit_emit_op1(compiler, 603 SLJIT_MOV, 604 dst, 0, 605 SLJIT_RETURN_REG, 0); 606 if (status != SLJIT_SUCCESS) 607 return status; 608 } 609 610 if (BPF_CLASS(pc->code) == BPF_LDX) { 611 /* restore A */ 612 status = sljit_emit_op1(compiler, 613 SLJIT_MOV, 614 BJ_AREG, 0, 615 BJ_TMP3REG, 0); 616 if (status != SLJIT_SUCCESS) 617 return status; 618 } 619 620 /* tmp2 = *err; */ 621 status = sljit_emit_op1(compiler, 622 SLJIT_MOV_UI, 623 BJ_TMP2REG, 0, 624 SLJIT_MEM1(SLJIT_LOCALS_REG), 625 offsetof(struct bpfjit_stack, err)); 626 if (status != SLJIT_SUCCESS) 627 return status; 628 629 /* if (tmp2 != 0) return 0; */ 630 jump = sljit_emit_cmp(compiler, 631 SLJIT_C_NOT_EQUAL, 632 BJ_TMP2REG, 0, 633 SLJIT_IMM, 0); 634 if (jump == NULL) 635 return SLJIT_ERR_ALLOC_FAILED; 636 637 if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 638 return SLJIT_ERR_ALLOC_FAILED; 639 640 return SLJIT_SUCCESS; 641 } 642 #endif 643 644 /* 645 * Emit code for BPF_COP and BPF_COPX instructions. 646 */ 647 static int 648 emit_cop(struct sljit_compiler *compiler, const bpf_ctx_t *bc, 649 const struct bpf_insn *pc, struct sljit_jump **ret0_jump) 650 { 651 #if BJ_XREG == SLJIT_RETURN_REG || \ 652 BJ_XREG == SLJIT_SCRATCH_REG1 || \ 653 BJ_XREG == SLJIT_SCRATCH_REG2 || \ 654 BJ_XREG == SLJIT_SCRATCH_REG3 || \ 655 BJ_COPF_PTR == BJ_ARGS || \ 656 BJ_COPF_IDX == BJ_ARGS 657 #error "Not supported assignment of registers." 658 #endif 659 660 struct sljit_jump *jump; 661 int status; 662 663 jump = NULL; 664 665 BJ_ASSERT(bc != NULL && bc->copfuncs != NULL); 666 667 if (BPF_MISCOP(pc->code) == BPF_COPX) { 668 /* if (X >= bc->nfuncs) return 0; */ 669 jump = sljit_emit_cmp(compiler, 670 SLJIT_C_GREATER_EQUAL, 671 BJ_XREG, 0, 672 SLJIT_IMM, bc->nfuncs); 673 if (jump == NULL) 674 return SLJIT_ERR_ALLOC_FAILED; 675 } 676 677 if (jump != NULL) 678 *ret0_jump = jump; 679 680 /* 681 * Copy bpf_copfunc_t arguments to registers. 682 */ 683 #if BJ_AREG != SLJIT_SCRATCH_REG3 684 status = sljit_emit_op1(compiler, 685 SLJIT_MOV_UI, 686 SLJIT_SCRATCH_REG3, 0, 687 BJ_AREG, 0); 688 if (status != SLJIT_SUCCESS) 689 return status; 690 #endif 691 692 status = sljit_emit_op1(compiler, 693 SLJIT_MOV_P, 694 SLJIT_SCRATCH_REG1, 0, 695 SLJIT_MEM1(SLJIT_LOCALS_REG), 696 offsetof(struct bpfjit_stack, ctx)); 697 if (status != SLJIT_SUCCESS) 698 return status; 699 700 status = sljit_emit_op1(compiler, 701 SLJIT_MOV_P, 702 SLJIT_SCRATCH_REG2, 0, 703 BJ_ARGS, 0); 704 if (status != SLJIT_SUCCESS) 705 return status; 706 707 if (BPF_MISCOP(pc->code) == BPF_COP) { 708 status = sljit_emit_ijump(compiler, 709 SLJIT_CALL3, 710 SLJIT_IMM, SLJIT_FUNC_OFFSET(bc->copfuncs[pc->k])); 711 if (status != SLJIT_SUCCESS) 712 return status; 713 } else if (BPF_MISCOP(pc->code) == BPF_COPX) { 714 /* load ctx->copfuncs */ 715 status = sljit_emit_op1(compiler, 716 SLJIT_MOV_P, 717 BJ_COPF_PTR, 0, 718 SLJIT_MEM1(SLJIT_SCRATCH_REG1), 719 offsetof(struct bpf_ctx, copfuncs)); 720 if (status != SLJIT_SUCCESS) 721 return status; 722 723 /* 724 * Load X to a register that can be used for 725 * memory addressing. 726 */ 727 status = sljit_emit_op1(compiler, 728 SLJIT_MOV, 729 BJ_COPF_IDX, 0, 730 BJ_XREG, 0); 731 if (status != SLJIT_SUCCESS) 732 return status; 733 734 status = sljit_emit_ijump(compiler, 735 SLJIT_CALL3, 736 SLJIT_MEM2(BJ_COPF_PTR, BJ_COPF_IDX), 737 SLJIT_WORD_SHIFT); 738 if (status != SLJIT_SUCCESS) 739 return status; 740 741 status = load_buf_buflen(compiler); 742 if (status != SLJIT_SUCCESS) 743 return status; 744 } 745 746 #if BJ_AREG != SLJIT_RETURN_REG 747 status = sljit_emit_op1(compiler, 748 SLJIT_MOV, 749 BJ_AREG, 0, 750 SLJIT_RETURN_REG, 0); 751 if (status != SLJIT_SUCCESS) 752 return status; 753 #endif 754 755 return SLJIT_SUCCESS; 756 } 757 758 /* 759 * Generate code for 760 * BPF_LD+BPF_W+BPF_ABS A <- P[k:4] 761 * BPF_LD+BPF_H+BPF_ABS A <- P[k:2] 762 * BPF_LD+BPF_B+BPF_ABS A <- P[k:1] 763 * BPF_LD+BPF_W+BPF_IND A <- P[X+k:4] 764 * BPF_LD+BPF_H+BPF_IND A <- P[X+k:2] 765 * BPF_LD+BPF_B+BPF_IND A <- P[X+k:1] 766 */ 767 static int 768 emit_pkt_read(struct sljit_compiler *compiler, 769 const struct bpf_insn *pc, struct sljit_jump *to_mchain_jump, 770 struct sljit_jump ***ret0, size_t *ret0_size, size_t *ret0_maxsize) 771 { 772 int status = SLJIT_ERR_ALLOC_FAILED; 773 uint32_t width; 774 struct sljit_jump *jump; 775 #ifdef _KERNEL 776 struct sljit_label *label; 777 struct sljit_jump *over_mchain_jump; 778 const bool check_zero_buflen = (to_mchain_jump != NULL); 779 #endif 780 const uint32_t k = pc->k; 781 782 #ifdef _KERNEL 783 if (to_mchain_jump == NULL) { 784 to_mchain_jump = sljit_emit_cmp(compiler, 785 SLJIT_C_EQUAL, 786 BJ_BUFLEN, 0, 787 SLJIT_IMM, 0); 788 if (to_mchain_jump == NULL) 789 return SLJIT_ERR_ALLOC_FAILED; 790 } 791 #endif 792 793 width = read_width(pc); 794 795 if (BPF_MODE(pc->code) == BPF_IND) { 796 /* tmp1 = buflen - (pc->k + width); */ 797 status = sljit_emit_op2(compiler, 798 SLJIT_SUB, 799 BJ_TMP1REG, 0, 800 BJ_BUFLEN, 0, 801 SLJIT_IMM, k + width); 802 if (status != SLJIT_SUCCESS) 803 return status; 804 805 /* buf += X; */ 806 status = sljit_emit_op2(compiler, 807 SLJIT_ADD, 808 BJ_BUF, 0, 809 BJ_BUF, 0, 810 BJ_XREG, 0); 811 if (status != SLJIT_SUCCESS) 812 return status; 813 814 /* if (tmp1 < X) return 0; */ 815 jump = sljit_emit_cmp(compiler, 816 SLJIT_C_LESS, 817 BJ_TMP1REG, 0, 818 BJ_XREG, 0); 819 if (jump == NULL) 820 return SLJIT_ERR_ALLOC_FAILED; 821 if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 822 return SLJIT_ERR_ALLOC_FAILED; 823 } 824 825 switch (width) { 826 case 4: 827 status = emit_read32(compiler, k); 828 break; 829 case 2: 830 status = emit_read16(compiler, k); 831 break; 832 case 1: 833 status = emit_read8(compiler, k); 834 break; 835 } 836 837 if (status != SLJIT_SUCCESS) 838 return status; 839 840 if (BPF_MODE(pc->code) == BPF_IND) { 841 /* buf -= X; */ 842 status = sljit_emit_op2(compiler, 843 SLJIT_SUB, 844 BJ_BUF, 0, 845 BJ_BUF, 0, 846 BJ_XREG, 0); 847 if (status != SLJIT_SUCCESS) 848 return status; 849 } 850 851 #ifdef _KERNEL 852 over_mchain_jump = sljit_emit_jump(compiler, SLJIT_JUMP); 853 if (over_mchain_jump == NULL) 854 return SLJIT_ERR_ALLOC_FAILED; 855 856 /* entry point to mchain handler */ 857 label = sljit_emit_label(compiler); 858 if (label == NULL) 859 return SLJIT_ERR_ALLOC_FAILED; 860 sljit_set_label(to_mchain_jump, label); 861 862 if (check_zero_buflen) { 863 /* if (buflen != 0) return 0; */ 864 jump = sljit_emit_cmp(compiler, 865 SLJIT_C_NOT_EQUAL, 866 BJ_BUFLEN, 0, 867 SLJIT_IMM, 0); 868 if (jump == NULL) 869 return SLJIT_ERR_ALLOC_FAILED; 870 if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 871 return SLJIT_ERR_ALLOC_FAILED; 872 } 873 874 switch (width) { 875 case 4: 876 status = emit_xcall(compiler, pc, BJ_AREG, 877 ret0, ret0_size, ret0_maxsize, &m_xword); 878 break; 879 case 2: 880 status = emit_xcall(compiler, pc, BJ_AREG, 881 ret0, ret0_size, ret0_maxsize, &m_xhalf); 882 break; 883 case 1: 884 status = emit_xcall(compiler, pc, BJ_AREG, 885 ret0, ret0_size, ret0_maxsize, &m_xbyte); 886 break; 887 } 888 889 if (status != SLJIT_SUCCESS) 890 return status; 891 892 label = sljit_emit_label(compiler); 893 if (label == NULL) 894 return SLJIT_ERR_ALLOC_FAILED; 895 sljit_set_label(over_mchain_jump, label); 896 #endif 897 898 return SLJIT_SUCCESS; 899 } 900 901 static int 902 emit_memload(struct sljit_compiler *compiler, 903 sljit_si dst, uint32_t k, size_t extwords) 904 { 905 int status; 906 sljit_si src; 907 sljit_sw srcw; 908 909 srcw = k * sizeof(uint32_t); 910 911 if (extwords == 0) { 912 src = SLJIT_MEM1(SLJIT_LOCALS_REG); 913 srcw += offsetof(struct bpfjit_stack, mem); 914 } else { 915 /* copy extmem pointer to the tmp1 register */ 916 status = sljit_emit_op1(compiler, 917 SLJIT_MOV_P, 918 BJ_TMP1REG, 0, 919 SLJIT_MEM1(SLJIT_LOCALS_REG), 920 offsetof(struct bpfjit_stack, extmem)); 921 if (status != SLJIT_SUCCESS) 922 return status; 923 src = SLJIT_MEM1(BJ_TMP1REG); 924 } 925 926 return sljit_emit_op1(compiler, SLJIT_MOV_UI, dst, 0, src, srcw); 927 } 928 929 static int 930 emit_memstore(struct sljit_compiler *compiler, 931 sljit_si src, uint32_t k, size_t extwords) 932 { 933 int status; 934 sljit_si dst; 935 sljit_sw dstw; 936 937 dstw = k * sizeof(uint32_t); 938 939 if (extwords == 0) { 940 dst = SLJIT_MEM1(SLJIT_LOCALS_REG); 941 dstw += offsetof(struct bpfjit_stack, mem); 942 } else { 943 /* copy extmem pointer to the tmp1 register */ 944 status = sljit_emit_op1(compiler, 945 SLJIT_MOV_P, 946 BJ_TMP1REG, 0, 947 SLJIT_MEM1(SLJIT_LOCALS_REG), 948 offsetof(struct bpfjit_stack, extmem)); 949 if (status != SLJIT_SUCCESS) 950 return status; 951 dst = SLJIT_MEM1(BJ_TMP1REG); 952 } 953 954 return sljit_emit_op1(compiler, SLJIT_MOV_UI, dst, dstw, src, 0); 955 } 956 957 /* 958 * Emit code for BPF_LDX+BPF_B+BPF_MSH X <- 4*(P[k:1]&0xf). 959 */ 960 static int 961 emit_msh(struct sljit_compiler *compiler, 962 const struct bpf_insn *pc, struct sljit_jump *to_mchain_jump, 963 struct sljit_jump ***ret0, size_t *ret0_size, size_t *ret0_maxsize) 964 { 965 int status; 966 #ifdef _KERNEL 967 struct sljit_label *label; 968 struct sljit_jump *jump, *over_mchain_jump; 969 const bool check_zero_buflen = (to_mchain_jump != NULL); 970 #endif 971 const uint32_t k = pc->k; 972 973 #ifdef _KERNEL 974 if (to_mchain_jump == NULL) { 975 to_mchain_jump = sljit_emit_cmp(compiler, 976 SLJIT_C_EQUAL, 977 BJ_BUFLEN, 0, 978 SLJIT_IMM, 0); 979 if (to_mchain_jump == NULL) 980 return SLJIT_ERR_ALLOC_FAILED; 981 } 982 #endif 983 984 /* tmp1 = buf[k] */ 985 status = sljit_emit_op1(compiler, 986 SLJIT_MOV_UB, 987 BJ_TMP1REG, 0, 988 SLJIT_MEM1(BJ_BUF), k); 989 if (status != SLJIT_SUCCESS) 990 return status; 991 992 /* tmp1 &= 0xf */ 993 status = sljit_emit_op2(compiler, 994 SLJIT_AND, 995 BJ_TMP1REG, 0, 996 BJ_TMP1REG, 0, 997 SLJIT_IMM, 0xf); 998 if (status != SLJIT_SUCCESS) 999 return status; 1000 1001 /* tmp1 = tmp1 << 2 */ 1002 status = sljit_emit_op2(compiler, 1003 SLJIT_SHL, 1004 BJ_XREG, 0, 1005 BJ_TMP1REG, 0, 1006 SLJIT_IMM, 2); 1007 if (status != SLJIT_SUCCESS) 1008 return status; 1009 1010 #ifdef _KERNEL 1011 over_mchain_jump = sljit_emit_jump(compiler, SLJIT_JUMP); 1012 if (over_mchain_jump == NULL) 1013 return SLJIT_ERR_ALLOC_FAILED; 1014 1015 /* entry point to mchain handler */ 1016 label = sljit_emit_label(compiler); 1017 if (label == NULL) 1018 return SLJIT_ERR_ALLOC_FAILED; 1019 sljit_set_label(to_mchain_jump, label); 1020 1021 if (check_zero_buflen) { 1022 /* if (buflen != 0) return 0; */ 1023 jump = sljit_emit_cmp(compiler, 1024 SLJIT_C_NOT_EQUAL, 1025 BJ_BUFLEN, 0, 1026 SLJIT_IMM, 0); 1027 if (jump == NULL) 1028 return SLJIT_ERR_ALLOC_FAILED; 1029 if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 1030 return SLJIT_ERR_ALLOC_FAILED; 1031 } 1032 1033 status = emit_xcall(compiler, pc, BJ_TMP1REG, 1034 ret0, ret0_size, ret0_maxsize, &m_xbyte); 1035 if (status != SLJIT_SUCCESS) 1036 return status; 1037 1038 /* tmp1 &= 0xf */ 1039 status = sljit_emit_op2(compiler, 1040 SLJIT_AND, 1041 BJ_TMP1REG, 0, 1042 BJ_TMP1REG, 0, 1043 SLJIT_IMM, 0xf); 1044 if (status != SLJIT_SUCCESS) 1045 return status; 1046 1047 /* tmp1 = tmp1 << 2 */ 1048 status = sljit_emit_op2(compiler, 1049 SLJIT_SHL, 1050 BJ_XREG, 0, 1051 BJ_TMP1REG, 0, 1052 SLJIT_IMM, 2); 1053 if (status != SLJIT_SUCCESS) 1054 return status; 1055 1056 1057 label = sljit_emit_label(compiler); 1058 if (label == NULL) 1059 return SLJIT_ERR_ALLOC_FAILED; 1060 sljit_set_label(over_mchain_jump, label); 1061 #endif 1062 1063 return SLJIT_SUCCESS; 1064 } 1065 1066 static int 1067 emit_pow2_division(struct sljit_compiler *compiler, uint32_t k) 1068 { 1069 int shift = 0; 1070 int status = SLJIT_SUCCESS; 1071 1072 while (k > 1) { 1073 k >>= 1; 1074 shift++; 1075 } 1076 1077 BJ_ASSERT(k == 1 && shift < 32); 1078 1079 if (shift != 0) { 1080 status = sljit_emit_op2(compiler, 1081 SLJIT_LSHR|SLJIT_INT_OP, 1082 BJ_AREG, 0, 1083 BJ_AREG, 0, 1084 SLJIT_IMM, shift); 1085 } 1086 1087 return status; 1088 } 1089 1090 #if !defined(BPFJIT_USE_UDIV) 1091 static sljit_uw 1092 divide(sljit_uw x, sljit_uw y) 1093 { 1094 1095 return (uint32_t)x / (uint32_t)y; 1096 } 1097 #endif 1098 1099 /* 1100 * Emit code for A = A / div. 1101 * divt,divw are either SLJIT_IMM,pc->k or BJ_XREG,0. 1102 */ 1103 static int 1104 emit_division(struct sljit_compiler *compiler, int divt, sljit_sw divw) 1105 { 1106 int status; 1107 1108 #if BJ_XREG == SLJIT_RETURN_REG || \ 1109 BJ_XREG == SLJIT_SCRATCH_REG1 || \ 1110 BJ_XREG == SLJIT_SCRATCH_REG2 || \ 1111 BJ_AREG == SLJIT_SCRATCH_REG2 1112 #error "Not supported assignment of registers." 1113 #endif 1114 1115 #if BJ_AREG != SLJIT_SCRATCH_REG1 1116 status = sljit_emit_op1(compiler, 1117 SLJIT_MOV, 1118 SLJIT_SCRATCH_REG1, 0, 1119 BJ_AREG, 0); 1120 if (status != SLJIT_SUCCESS) 1121 return status; 1122 #endif 1123 1124 status = sljit_emit_op1(compiler, 1125 SLJIT_MOV, 1126 SLJIT_SCRATCH_REG2, 0, 1127 divt, divw); 1128 if (status != SLJIT_SUCCESS) 1129 return status; 1130 1131 #if defined(BPFJIT_USE_UDIV) 1132 status = sljit_emit_op0(compiler, SLJIT_UDIV|SLJIT_INT_OP); 1133 1134 #if BJ_AREG != SLJIT_SCRATCH_REG1 1135 status = sljit_emit_op1(compiler, 1136 SLJIT_MOV, 1137 BJ_AREG, 0, 1138 SLJIT_SCRATCH_REG1, 0); 1139 if (status != SLJIT_SUCCESS) 1140 return status; 1141 #endif 1142 #else 1143 status = sljit_emit_ijump(compiler, 1144 SLJIT_CALL2, 1145 SLJIT_IMM, SLJIT_FUNC_OFFSET(divide)); 1146 1147 #if BJ_AREG != SLJIT_RETURN_REG 1148 status = sljit_emit_op1(compiler, 1149 SLJIT_MOV, 1150 BJ_AREG, 0, 1151 SLJIT_RETURN_REG, 0); 1152 if (status != SLJIT_SUCCESS) 1153 return status; 1154 #endif 1155 #endif 1156 1157 return status; 1158 } 1159 1160 /* 1161 * Return true if pc is a "read from packet" instruction. 1162 * If length is not NULL and return value is true, *length will 1163 * be set to a safe length required to read a packet. 1164 */ 1165 static bool 1166 read_pkt_insn(const struct bpf_insn *pc, bpfjit_abc_length_t *length) 1167 { 1168 bool rv; 1169 bpfjit_abc_length_t width; 1170 1171 switch (BPF_CLASS(pc->code)) { 1172 default: 1173 rv = false; 1174 break; 1175 1176 case BPF_LD: 1177 rv = BPF_MODE(pc->code) == BPF_ABS || 1178 BPF_MODE(pc->code) == BPF_IND; 1179 if (rv) 1180 width = read_width(pc); 1181 break; 1182 1183 case BPF_LDX: 1184 rv = pc->code == (BPF_LDX|BPF_B|BPF_MSH); 1185 width = 1; 1186 break; 1187 } 1188 1189 if (rv && length != NULL) { 1190 /* 1191 * Values greater than UINT32_MAX will generate 1192 * unconditional "return 0". 1193 */ 1194 *length = (uint32_t)pc->k + width; 1195 } 1196 1197 return rv; 1198 } 1199 1200 static void 1201 optimize_init(struct bpfjit_insn_data *insn_dat, size_t insn_count) 1202 { 1203 size_t i; 1204 1205 for (i = 0; i < insn_count; i++) { 1206 SLIST_INIT(&insn_dat[i].bjumps); 1207 insn_dat[i].invalid = BJ_INIT_NOBITS; 1208 } 1209 } 1210 1211 /* 1212 * The function divides instructions into blocks. Destination of a jump 1213 * instruction starts a new block. BPF_RET and BPF_JMP instructions 1214 * terminate a block. Blocks are linear, that is, there are no jumps out 1215 * from the middle of a block and there are no jumps in to the middle of 1216 * a block. 1217 * 1218 * The function also sets bits in *initmask for memwords that 1219 * need to be initialized to zero. Note that this set should be empty 1220 * for any valid kernel filter program. 1221 */ 1222 static bool 1223 optimize_pass1(const bpf_ctx_t *bc, const struct bpf_insn *insns, 1224 struct bpfjit_insn_data *insn_dat, size_t insn_count, 1225 bpf_memword_init_t *initmask, bpfjit_hint_t *hints) 1226 { 1227 struct bpfjit_jump *jtf; 1228 size_t i; 1229 uint32_t jt, jf; 1230 bpfjit_abc_length_t length; 1231 bpf_memword_init_t invalid; /* borrowed from bpf_filter() */ 1232 bool unreachable; 1233 1234 const size_t memwords = GET_MEMWORDS(bc); 1235 1236 *hints = 0; 1237 *initmask = BJ_INIT_NOBITS; 1238 1239 unreachable = false; 1240 invalid = ~BJ_INIT_NOBITS; 1241 1242 for (i = 0; i < insn_count; i++) { 1243 if (!SLIST_EMPTY(&insn_dat[i].bjumps)) 1244 unreachable = false; 1245 insn_dat[i].unreachable = unreachable; 1246 1247 if (unreachable) 1248 continue; 1249 1250 invalid |= insn_dat[i].invalid; 1251 1252 if (read_pkt_insn(&insns[i], &length) && length > UINT32_MAX) 1253 unreachable = true; 1254 1255 switch (BPF_CLASS(insns[i].code)) { 1256 case BPF_RET: 1257 if (BPF_RVAL(insns[i].code) == BPF_A) 1258 *initmask |= invalid & BJ_INIT_ABIT; 1259 1260 unreachable = true; 1261 continue; 1262 1263 case BPF_LD: 1264 if (BPF_MODE(insns[i].code) == BPF_IND || 1265 BPF_MODE(insns[i].code) == BPF_ABS) { 1266 *hints |= BJ_HINT_PKT; 1267 } 1268 1269 if (BPF_SIZE(insns[i].code) == BPF_W) 1270 *hints |= BJ_HINT_LDW; 1271 1272 if (BPF_MODE(insns[i].code) == BPF_IND) { 1273 *hints |= BJ_HINT_XREG; 1274 *initmask |= invalid & BJ_INIT_XBIT; 1275 } 1276 1277 if (BPF_MODE(insns[i].code) == BPF_MEM && 1278 (uint32_t)insns[i].k < memwords) { 1279 *initmask |= invalid & BJ_INIT_MBIT(insns[i].k); 1280 } 1281 1282 invalid &= ~BJ_INIT_ABIT; 1283 continue; 1284 1285 case BPF_LDX: 1286 *hints |= BJ_HINT_XREG | BJ_HINT_LDX; 1287 1288 if (BPF_MODE(insns[i].code) == BPF_MEM && 1289 (uint32_t)insns[i].k < memwords) { 1290 *initmask |= invalid & BJ_INIT_MBIT(insns[i].k); 1291 } 1292 1293 invalid &= ~BJ_INIT_XBIT; 1294 continue; 1295 1296 case BPF_ST: 1297 *initmask |= invalid & BJ_INIT_ABIT; 1298 1299 if ((uint32_t)insns[i].k < memwords) 1300 invalid &= ~BJ_INIT_MBIT(insns[i].k); 1301 1302 continue; 1303 1304 case BPF_STX: 1305 *hints |= BJ_HINT_XREG; 1306 *initmask |= invalid & BJ_INIT_XBIT; 1307 1308 if ((uint32_t)insns[i].k < memwords) 1309 invalid &= ~BJ_INIT_MBIT(insns[i].k); 1310 1311 continue; 1312 1313 case BPF_ALU: 1314 *initmask |= invalid & BJ_INIT_ABIT; 1315 1316 if (insns[i].code != (BPF_ALU|BPF_NEG) && 1317 BPF_SRC(insns[i].code) == BPF_X) { 1318 *hints |= BJ_HINT_XREG; 1319 *initmask |= invalid & BJ_INIT_XBIT; 1320 } 1321 1322 invalid &= ~BJ_INIT_ABIT; 1323 continue; 1324 1325 case BPF_MISC: 1326 switch (BPF_MISCOP(insns[i].code)) { 1327 case BPF_TAX: // X <- A 1328 *hints |= BJ_HINT_XREG; 1329 *initmask |= invalid & BJ_INIT_ABIT; 1330 invalid &= ~BJ_INIT_XBIT; 1331 continue; 1332 1333 case BPF_TXA: // A <- X 1334 *hints |= BJ_HINT_XREG; 1335 *initmask |= invalid & BJ_INIT_XBIT; 1336 invalid &= ~BJ_INIT_ABIT; 1337 continue; 1338 1339 case BPF_COPX: 1340 *hints |= BJ_HINT_XREG; 1341 /* FALLTHROUGH */ 1342 1343 case BPF_COP: 1344 *hints |= BJ_HINT_COP; 1345 *initmask |= invalid & BJ_INIT_ABIT; 1346 invalid &= ~BJ_INIT_ABIT; 1347 continue; 1348 } 1349 1350 continue; 1351 1352 case BPF_JMP: 1353 /* Initialize abc_length for ABC pass. */ 1354 insn_dat[i].u.jdata.abc_length = MAX_ABC_LENGTH; 1355 1356 if (BPF_OP(insns[i].code) == BPF_JA) { 1357 jt = jf = insns[i].k; 1358 } else { 1359 jt = insns[i].jt; 1360 jf = insns[i].jf; 1361 } 1362 1363 if (jt >= insn_count - (i + 1) || 1364 jf >= insn_count - (i + 1)) { 1365 return false; 1366 } 1367 1368 if (jt > 0 && jf > 0) 1369 unreachable = true; 1370 1371 jt += i + 1; 1372 jf += i + 1; 1373 1374 jtf = insn_dat[i].u.jdata.jtf; 1375 1376 jtf[0].jdata = &insn_dat[i].u.jdata; 1377 SLIST_INSERT_HEAD(&insn_dat[jt].bjumps, 1378 &jtf[0], entries); 1379 1380 if (jf != jt) { 1381 jtf[1].jdata = &insn_dat[i].u.jdata; 1382 SLIST_INSERT_HEAD(&insn_dat[jf].bjumps, 1383 &jtf[1], entries); 1384 } 1385 1386 insn_dat[jf].invalid |= invalid; 1387 insn_dat[jt].invalid |= invalid; 1388 invalid = 0; 1389 1390 continue; 1391 } 1392 } 1393 1394 return true; 1395 } 1396 1397 /* 1398 * Array Bounds Check Elimination (ABC) pass. 1399 */ 1400 static void 1401 optimize_pass2(const bpf_ctx_t *bc, const struct bpf_insn *insns, 1402 struct bpfjit_insn_data *insn_dat, size_t insn_count) 1403 { 1404 struct bpfjit_jump *jmp; 1405 const struct bpf_insn *pc; 1406 struct bpfjit_insn_data *pd; 1407 size_t i; 1408 bpfjit_abc_length_t length, abc_length = 0; 1409 1410 const size_t extwords = GET_EXTWORDS(bc); 1411 1412 for (i = insn_count; i != 0; i--) { 1413 pc = &insns[i-1]; 1414 pd = &insn_dat[i-1]; 1415 1416 if (pd->unreachable) 1417 continue; 1418 1419 switch (BPF_CLASS(pc->code)) { 1420 case BPF_RET: 1421 /* 1422 * It's quite common for bpf programs to 1423 * check packet bytes in increasing order 1424 * and return zero if bytes don't match 1425 * specified critetion. Such programs disable 1426 * ABC optimization completely because for 1427 * every jump there is a branch with no read 1428 * instruction. 1429 * With no side effects, BPF_STMT(BPF_RET+BPF_K, 0) 1430 * is indistinguishable from out-of-bound load. 1431 * Therefore, abc_length can be set to 1432 * MAX_ABC_LENGTH and enable ABC for many 1433 * bpf programs. 1434 * If this optimization encounters any 1435 * instruction with a side effect, it will 1436 * reset abc_length. 1437 */ 1438 if (BPF_RVAL(pc->code) == BPF_K && pc->k == 0) 1439 abc_length = MAX_ABC_LENGTH; 1440 else 1441 abc_length = 0; 1442 break; 1443 1444 case BPF_MISC: 1445 if (BPF_MISCOP(pc->code) == BPF_COP || 1446 BPF_MISCOP(pc->code) == BPF_COPX) { 1447 /* COP instructions can have side effects. */ 1448 abc_length = 0; 1449 } 1450 break; 1451 1452 case BPF_ST: 1453 case BPF_STX: 1454 if (extwords != 0) { 1455 /* Write to memory is visible after a call. */ 1456 abc_length = 0; 1457 } 1458 break; 1459 1460 case BPF_JMP: 1461 abc_length = pd->u.jdata.abc_length; 1462 break; 1463 1464 default: 1465 if (read_pkt_insn(pc, &length)) { 1466 if (abc_length < length) 1467 abc_length = length; 1468 pd->u.rdata.abc_length = abc_length; 1469 } 1470 break; 1471 } 1472 1473 SLIST_FOREACH(jmp, &pd->bjumps, entries) { 1474 if (jmp->jdata->abc_length > abc_length) 1475 jmp->jdata->abc_length = abc_length; 1476 } 1477 } 1478 } 1479 1480 static void 1481 optimize_pass3(const struct bpf_insn *insns, 1482 struct bpfjit_insn_data *insn_dat, size_t insn_count) 1483 { 1484 struct bpfjit_jump *jmp; 1485 size_t i; 1486 bpfjit_abc_length_t checked_length = 0; 1487 1488 for (i = 0; i < insn_count; i++) { 1489 if (insn_dat[i].unreachable) 1490 continue; 1491 1492 SLIST_FOREACH(jmp, &insn_dat[i].bjumps, entries) { 1493 if (jmp->jdata->checked_length < checked_length) 1494 checked_length = jmp->jdata->checked_length; 1495 } 1496 1497 if (BPF_CLASS(insns[i].code) == BPF_JMP) { 1498 insn_dat[i].u.jdata.checked_length = checked_length; 1499 } else if (read_pkt_insn(&insns[i], NULL)) { 1500 struct bpfjit_read_pkt_data *rdata = 1501 &insn_dat[i].u.rdata; 1502 rdata->check_length = 0; 1503 if (checked_length < rdata->abc_length) { 1504 checked_length = rdata->abc_length; 1505 rdata->check_length = checked_length; 1506 } 1507 } 1508 } 1509 } 1510 1511 static bool 1512 optimize(const bpf_ctx_t *bc, const struct bpf_insn *insns, 1513 struct bpfjit_insn_data *insn_dat, size_t insn_count, 1514 bpf_memword_init_t *initmask, bpfjit_hint_t *hints) 1515 { 1516 1517 optimize_init(insn_dat, insn_count); 1518 1519 if (!optimize_pass1(bc, insns, insn_dat, insn_count, initmask, hints)) 1520 return false; 1521 1522 optimize_pass2(bc, insns, insn_dat, insn_count); 1523 optimize_pass3(insns, insn_dat, insn_count); 1524 1525 return true; 1526 } 1527 1528 /* 1529 * Convert BPF_ALU operations except BPF_NEG and BPF_DIV to sljit operation. 1530 */ 1531 static int 1532 bpf_alu_to_sljit_op(const struct bpf_insn *pc) 1533 { 1534 1535 /* 1536 * Note: all supported 64bit arches have 32bit multiply 1537 * instruction so SLJIT_INT_OP doesn't have any overhead. 1538 */ 1539 switch (BPF_OP(pc->code)) { 1540 case BPF_ADD: return SLJIT_ADD; 1541 case BPF_SUB: return SLJIT_SUB; 1542 case BPF_MUL: return SLJIT_MUL|SLJIT_INT_OP; 1543 case BPF_OR: return SLJIT_OR; 1544 case BPF_AND: return SLJIT_AND; 1545 case BPF_LSH: return SLJIT_SHL; 1546 case BPF_RSH: return SLJIT_LSHR|SLJIT_INT_OP; 1547 default: 1548 BJ_ASSERT(false); 1549 return 0; 1550 } 1551 } 1552 1553 /* 1554 * Convert BPF_JMP operations except BPF_JA to sljit condition. 1555 */ 1556 static int 1557 bpf_jmp_to_sljit_cond(const struct bpf_insn *pc, bool negate) 1558 { 1559 /* 1560 * Note: all supported 64bit arches have 32bit comparison 1561 * instructions so SLJIT_INT_OP doesn't have any overhead. 1562 */ 1563 int rv = SLJIT_INT_OP; 1564 1565 switch (BPF_OP(pc->code)) { 1566 case BPF_JGT: 1567 rv |= negate ? SLJIT_C_LESS_EQUAL : SLJIT_C_GREATER; 1568 break; 1569 case BPF_JGE: 1570 rv |= negate ? SLJIT_C_LESS : SLJIT_C_GREATER_EQUAL; 1571 break; 1572 case BPF_JEQ: 1573 rv |= negate ? SLJIT_C_NOT_EQUAL : SLJIT_C_EQUAL; 1574 break; 1575 case BPF_JSET: 1576 rv |= negate ? SLJIT_C_EQUAL : SLJIT_C_NOT_EQUAL; 1577 break; 1578 default: 1579 BJ_ASSERT(false); 1580 } 1581 1582 return rv; 1583 } 1584 1585 /* 1586 * Convert BPF_K and BPF_X to sljit register. 1587 */ 1588 static int 1589 kx_to_reg(const struct bpf_insn *pc) 1590 { 1591 1592 switch (BPF_SRC(pc->code)) { 1593 case BPF_K: return SLJIT_IMM; 1594 case BPF_X: return BJ_XREG; 1595 default: 1596 BJ_ASSERT(false); 1597 return 0; 1598 } 1599 } 1600 1601 static sljit_sw 1602 kx_to_reg_arg(const struct bpf_insn *pc) 1603 { 1604 1605 switch (BPF_SRC(pc->code)) { 1606 case BPF_K: return (uint32_t)pc->k; /* SLJIT_IMM, pc->k, */ 1607 case BPF_X: return 0; /* BJ_XREG, 0, */ 1608 default: 1609 BJ_ASSERT(false); 1610 return 0; 1611 } 1612 } 1613 1614 static bool 1615 generate_insn_code(struct sljit_compiler *compiler, const bpf_ctx_t *bc, 1616 const struct bpf_insn *insns, struct bpfjit_insn_data *insn_dat, 1617 size_t insn_count) 1618 { 1619 /* a list of jumps to out-of-bound return from a generated function */ 1620 struct sljit_jump **ret0; 1621 size_t ret0_size, ret0_maxsize; 1622 1623 struct sljit_jump *jump; 1624 struct sljit_label *label; 1625 const struct bpf_insn *pc; 1626 struct bpfjit_jump *bjump, *jtf; 1627 struct sljit_jump *to_mchain_jump; 1628 1629 size_t i; 1630 int status; 1631 int branching, negate; 1632 unsigned int rval, mode, src; 1633 uint32_t jt, jf; 1634 1635 bool unconditional_ret; 1636 bool rv; 1637 1638 const size_t extwords = GET_EXTWORDS(bc); 1639 const size_t memwords = GET_MEMWORDS(bc); 1640 1641 ret0 = NULL; 1642 rv = false; 1643 1644 ret0_size = 0; 1645 ret0_maxsize = 64; 1646 ret0 = BJ_ALLOC(ret0_maxsize * sizeof(ret0[0])); 1647 if (ret0 == NULL) 1648 goto fail; 1649 1650 /* reset sjump members of jdata */ 1651 for (i = 0; i < insn_count; i++) { 1652 if (insn_dat[i].unreachable || 1653 BPF_CLASS(insns[i].code) != BPF_JMP) { 1654 continue; 1655 } 1656 1657 jtf = insn_dat[i].u.jdata.jtf; 1658 jtf[0].sjump = jtf[1].sjump = NULL; 1659 } 1660 1661 /* main loop */ 1662 for (i = 0; i < insn_count; i++) { 1663 if (insn_dat[i].unreachable) 1664 continue; 1665 1666 /* 1667 * Resolve jumps to the current insn. 1668 */ 1669 label = NULL; 1670 SLIST_FOREACH(bjump, &insn_dat[i].bjumps, entries) { 1671 if (bjump->sjump != NULL) { 1672 if (label == NULL) 1673 label = sljit_emit_label(compiler); 1674 if (label == NULL) 1675 goto fail; 1676 sljit_set_label(bjump->sjump, label); 1677 } 1678 } 1679 1680 to_mchain_jump = NULL; 1681 unconditional_ret = false; 1682 1683 if (read_pkt_insn(&insns[i], NULL)) { 1684 if (insn_dat[i].u.rdata.check_length > UINT32_MAX) { 1685 /* Jump to "return 0" unconditionally. */ 1686 unconditional_ret = true; 1687 jump = sljit_emit_jump(compiler, SLJIT_JUMP); 1688 if (jump == NULL) 1689 goto fail; 1690 if (!append_jump(jump, &ret0, 1691 &ret0_size, &ret0_maxsize)) 1692 goto fail; 1693 } else if (insn_dat[i].u.rdata.check_length > 0) { 1694 /* if (buflen < check_length) return 0; */ 1695 jump = sljit_emit_cmp(compiler, 1696 SLJIT_C_LESS, 1697 BJ_BUFLEN, 0, 1698 SLJIT_IMM, 1699 insn_dat[i].u.rdata.check_length); 1700 if (jump == NULL) 1701 goto fail; 1702 #ifdef _KERNEL 1703 to_mchain_jump = jump; 1704 #else 1705 if (!append_jump(jump, &ret0, 1706 &ret0_size, &ret0_maxsize)) 1707 goto fail; 1708 #endif 1709 } 1710 } 1711 1712 pc = &insns[i]; 1713 switch (BPF_CLASS(pc->code)) { 1714 1715 default: 1716 goto fail; 1717 1718 case BPF_LD: 1719 /* BPF_LD+BPF_IMM A <- k */ 1720 if (pc->code == (BPF_LD|BPF_IMM)) { 1721 status = sljit_emit_op1(compiler, 1722 SLJIT_MOV, 1723 BJ_AREG, 0, 1724 SLJIT_IMM, (uint32_t)pc->k); 1725 if (status != SLJIT_SUCCESS) 1726 goto fail; 1727 1728 continue; 1729 } 1730 1731 /* BPF_LD+BPF_MEM A <- M[k] */ 1732 if (pc->code == (BPF_LD|BPF_MEM)) { 1733 if ((uint32_t)pc->k >= memwords) 1734 goto fail; 1735 status = emit_memload(compiler, 1736 BJ_AREG, pc->k, extwords); 1737 if (status != SLJIT_SUCCESS) 1738 goto fail; 1739 1740 continue; 1741 } 1742 1743 /* BPF_LD+BPF_W+BPF_LEN A <- len */ 1744 if (pc->code == (BPF_LD|BPF_W|BPF_LEN)) { 1745 status = sljit_emit_op1(compiler, 1746 SLJIT_MOV, /* size_t source */ 1747 BJ_AREG, 0, 1748 SLJIT_MEM1(BJ_ARGS), 1749 offsetof(struct bpf_args, wirelen)); 1750 if (status != SLJIT_SUCCESS) 1751 goto fail; 1752 1753 continue; 1754 } 1755 1756 mode = BPF_MODE(pc->code); 1757 if (mode != BPF_ABS && mode != BPF_IND) 1758 goto fail; 1759 1760 if (unconditional_ret) 1761 continue; 1762 1763 status = emit_pkt_read(compiler, pc, 1764 to_mchain_jump, &ret0, &ret0_size, &ret0_maxsize); 1765 if (status != SLJIT_SUCCESS) 1766 goto fail; 1767 1768 continue; 1769 1770 case BPF_LDX: 1771 mode = BPF_MODE(pc->code); 1772 1773 /* BPF_LDX+BPF_W+BPF_IMM X <- k */ 1774 if (mode == BPF_IMM) { 1775 if (BPF_SIZE(pc->code) != BPF_W) 1776 goto fail; 1777 status = sljit_emit_op1(compiler, 1778 SLJIT_MOV, 1779 BJ_XREG, 0, 1780 SLJIT_IMM, (uint32_t)pc->k); 1781 if (status != SLJIT_SUCCESS) 1782 goto fail; 1783 1784 continue; 1785 } 1786 1787 /* BPF_LDX+BPF_W+BPF_LEN X <- len */ 1788 if (mode == BPF_LEN) { 1789 if (BPF_SIZE(pc->code) != BPF_W) 1790 goto fail; 1791 status = sljit_emit_op1(compiler, 1792 SLJIT_MOV, /* size_t source */ 1793 BJ_XREG, 0, 1794 SLJIT_MEM1(BJ_ARGS), 1795 offsetof(struct bpf_args, wirelen)); 1796 if (status != SLJIT_SUCCESS) 1797 goto fail; 1798 1799 continue; 1800 } 1801 1802 /* BPF_LDX+BPF_W+BPF_MEM X <- M[k] */ 1803 if (mode == BPF_MEM) { 1804 if (BPF_SIZE(pc->code) != BPF_W) 1805 goto fail; 1806 if ((uint32_t)pc->k >= memwords) 1807 goto fail; 1808 status = emit_memload(compiler, 1809 BJ_XREG, pc->k, extwords); 1810 if (status != SLJIT_SUCCESS) 1811 goto fail; 1812 1813 continue; 1814 } 1815 1816 /* BPF_LDX+BPF_B+BPF_MSH X <- 4*(P[k:1]&0xf) */ 1817 if (mode != BPF_MSH || BPF_SIZE(pc->code) != BPF_B) 1818 goto fail; 1819 1820 if (unconditional_ret) 1821 continue; 1822 1823 status = emit_msh(compiler, pc, 1824 to_mchain_jump, &ret0, &ret0_size, &ret0_maxsize); 1825 if (status != SLJIT_SUCCESS) 1826 goto fail; 1827 1828 continue; 1829 1830 case BPF_ST: 1831 if (pc->code != BPF_ST || 1832 (uint32_t)pc->k >= memwords) { 1833 goto fail; 1834 } 1835 1836 status = emit_memstore(compiler, 1837 BJ_AREG, pc->k, extwords); 1838 if (status != SLJIT_SUCCESS) 1839 goto fail; 1840 1841 continue; 1842 1843 case BPF_STX: 1844 if (pc->code != BPF_STX || 1845 (uint32_t)pc->k >= memwords) { 1846 goto fail; 1847 } 1848 1849 status = emit_memstore(compiler, 1850 BJ_XREG, pc->k, extwords); 1851 if (status != SLJIT_SUCCESS) 1852 goto fail; 1853 1854 continue; 1855 1856 case BPF_ALU: 1857 if (pc->code == (BPF_ALU|BPF_NEG)) { 1858 status = sljit_emit_op1(compiler, 1859 SLJIT_NEG, 1860 BJ_AREG, 0, 1861 BJ_AREG, 0); 1862 if (status != SLJIT_SUCCESS) 1863 goto fail; 1864 1865 continue; 1866 } 1867 1868 if (BPF_OP(pc->code) != BPF_DIV) { 1869 status = sljit_emit_op2(compiler, 1870 bpf_alu_to_sljit_op(pc), 1871 BJ_AREG, 0, 1872 BJ_AREG, 0, 1873 kx_to_reg(pc), kx_to_reg_arg(pc)); 1874 if (status != SLJIT_SUCCESS) 1875 goto fail; 1876 1877 continue; 1878 } 1879 1880 /* BPF_DIV */ 1881 1882 src = BPF_SRC(pc->code); 1883 if (src != BPF_X && src != BPF_K) 1884 goto fail; 1885 1886 /* division by zero? */ 1887 if (src == BPF_X) { 1888 jump = sljit_emit_cmp(compiler, 1889 SLJIT_C_EQUAL|SLJIT_INT_OP, 1890 BJ_XREG, 0, 1891 SLJIT_IMM, 0); 1892 if (jump == NULL) 1893 goto fail; 1894 if (!append_jump(jump, &ret0, 1895 &ret0_size, &ret0_maxsize)) 1896 goto fail; 1897 } else if (pc->k == 0) { 1898 jump = sljit_emit_jump(compiler, SLJIT_JUMP); 1899 if (jump == NULL) 1900 goto fail; 1901 if (!append_jump(jump, &ret0, 1902 &ret0_size, &ret0_maxsize)) 1903 goto fail; 1904 } 1905 1906 if (src == BPF_X) { 1907 status = emit_division(compiler, BJ_XREG, 0); 1908 if (status != SLJIT_SUCCESS) 1909 goto fail; 1910 } else if (pc->k != 0) { 1911 if (pc->k & (pc->k - 1)) { 1912 status = emit_division(compiler, 1913 SLJIT_IMM, (uint32_t)pc->k); 1914 } else { 1915 status = emit_pow2_division(compiler, 1916 (uint32_t)pc->k); 1917 } 1918 if (status != SLJIT_SUCCESS) 1919 goto fail; 1920 } 1921 1922 continue; 1923 1924 case BPF_JMP: 1925 if (BPF_OP(pc->code) == BPF_JA) { 1926 jt = jf = pc->k; 1927 } else { 1928 jt = pc->jt; 1929 jf = pc->jf; 1930 } 1931 1932 negate = (jt == 0) ? 1 : 0; 1933 branching = (jt == jf) ? 0 : 1; 1934 jtf = insn_dat[i].u.jdata.jtf; 1935 1936 if (branching) { 1937 if (BPF_OP(pc->code) != BPF_JSET) { 1938 jump = sljit_emit_cmp(compiler, 1939 bpf_jmp_to_sljit_cond(pc, negate), 1940 BJ_AREG, 0, 1941 kx_to_reg(pc), kx_to_reg_arg(pc)); 1942 } else { 1943 status = sljit_emit_op2(compiler, 1944 SLJIT_AND, 1945 BJ_TMP1REG, 0, 1946 BJ_AREG, 0, 1947 kx_to_reg(pc), kx_to_reg_arg(pc)); 1948 if (status != SLJIT_SUCCESS) 1949 goto fail; 1950 1951 jump = sljit_emit_cmp(compiler, 1952 bpf_jmp_to_sljit_cond(pc, negate), 1953 BJ_TMP1REG, 0, 1954 SLJIT_IMM, 0); 1955 } 1956 1957 if (jump == NULL) 1958 goto fail; 1959 1960 BJ_ASSERT(jtf[negate].sjump == NULL); 1961 jtf[negate].sjump = jump; 1962 } 1963 1964 if (!branching || (jt != 0 && jf != 0)) { 1965 jump = sljit_emit_jump(compiler, SLJIT_JUMP); 1966 if (jump == NULL) 1967 goto fail; 1968 1969 BJ_ASSERT(jtf[branching].sjump == NULL); 1970 jtf[branching].sjump = jump; 1971 } 1972 1973 continue; 1974 1975 case BPF_RET: 1976 rval = BPF_RVAL(pc->code); 1977 if (rval == BPF_X) 1978 goto fail; 1979 1980 /* BPF_RET+BPF_K accept k bytes */ 1981 if (rval == BPF_K) { 1982 status = sljit_emit_return(compiler, 1983 SLJIT_MOV_UI, 1984 SLJIT_IMM, (uint32_t)pc->k); 1985 if (status != SLJIT_SUCCESS) 1986 goto fail; 1987 } 1988 1989 /* BPF_RET+BPF_A accept A bytes */ 1990 if (rval == BPF_A) { 1991 status = sljit_emit_return(compiler, 1992 SLJIT_MOV_UI, 1993 BJ_AREG, 0); 1994 if (status != SLJIT_SUCCESS) 1995 goto fail; 1996 } 1997 1998 continue; 1999 2000 case BPF_MISC: 2001 switch (BPF_MISCOP(pc->code)) { 2002 case BPF_TAX: 2003 status = sljit_emit_op1(compiler, 2004 SLJIT_MOV_UI, 2005 BJ_XREG, 0, 2006 BJ_AREG, 0); 2007 if (status != SLJIT_SUCCESS) 2008 goto fail; 2009 2010 continue; 2011 2012 case BPF_TXA: 2013 status = sljit_emit_op1(compiler, 2014 SLJIT_MOV, 2015 BJ_AREG, 0, 2016 BJ_XREG, 0); 2017 if (status != SLJIT_SUCCESS) 2018 goto fail; 2019 2020 continue; 2021 2022 case BPF_COP: 2023 case BPF_COPX: 2024 if (bc == NULL || bc->copfuncs == NULL) 2025 goto fail; 2026 if (BPF_MISCOP(pc->code) == BPF_COP && 2027 (uint32_t)pc->k >= bc->nfuncs) { 2028 goto fail; 2029 } 2030 2031 jump = NULL; 2032 status = emit_cop(compiler, bc, pc, &jump); 2033 if (status != SLJIT_SUCCESS) 2034 goto fail; 2035 2036 if (jump != NULL && !append_jump(jump, 2037 &ret0, &ret0_size, &ret0_maxsize)) 2038 goto fail; 2039 2040 continue; 2041 } 2042 2043 goto fail; 2044 } /* switch */ 2045 } /* main loop */ 2046 2047 BJ_ASSERT(ret0_size <= ret0_maxsize); 2048 2049 if (ret0_size > 0) { 2050 label = sljit_emit_label(compiler); 2051 if (label == NULL) 2052 goto fail; 2053 for (i = 0; i < ret0_size; i++) 2054 sljit_set_label(ret0[i], label); 2055 } 2056 2057 status = sljit_emit_return(compiler, 2058 SLJIT_MOV_UI, 2059 SLJIT_IMM, 0); 2060 if (status != SLJIT_SUCCESS) 2061 goto fail; 2062 2063 rv = true; 2064 2065 fail: 2066 if (ret0 != NULL) 2067 BJ_FREE(ret0, ret0_maxsize * sizeof(ret0[0])); 2068 2069 return rv; 2070 } 2071 2072 bpfjit_func_t 2073 bpfjit_generate_code(const bpf_ctx_t *bc, 2074 const struct bpf_insn *insns, size_t insn_count) 2075 { 2076 void *rv; 2077 struct sljit_compiler *compiler; 2078 2079 size_t i; 2080 int status; 2081 2082 /* optimization related */ 2083 bpf_memword_init_t initmask; 2084 bpfjit_hint_t hints; 2085 2086 /* memory store location for initial zero initialization */ 2087 sljit_si mem_reg; 2088 sljit_sw mem_off; 2089 2090 struct bpfjit_insn_data *insn_dat; 2091 2092 const size_t extwords = GET_EXTWORDS(bc); 2093 const size_t memwords = GET_MEMWORDS(bc); 2094 const bpf_memword_init_t preinited = extwords ? bc->preinited : 0; 2095 2096 rv = NULL; 2097 compiler = NULL; 2098 insn_dat = NULL; 2099 2100 if (memwords > MAX_MEMWORDS) 2101 goto fail; 2102 2103 if (insn_count == 0 || insn_count > SIZE_MAX / sizeof(insn_dat[0])) 2104 goto fail; 2105 2106 insn_dat = BJ_ALLOC(insn_count * sizeof(insn_dat[0])); 2107 if (insn_dat == NULL) 2108 goto fail; 2109 2110 if (!optimize(bc, insns, insn_dat, insn_count, &initmask, &hints)) 2111 goto fail; 2112 2113 compiler = sljit_create_compiler(); 2114 if (compiler == NULL) 2115 goto fail; 2116 2117 #if !defined(_KERNEL) && defined(SLJIT_VERBOSE) && SLJIT_VERBOSE 2118 sljit_compiler_verbose(compiler, stderr); 2119 #endif 2120 2121 status = sljit_emit_enter(compiler, 2122 2, nscratches(hints), 3, sizeof(struct bpfjit_stack)); 2123 if (status != SLJIT_SUCCESS) 2124 goto fail; 2125 2126 if (hints & BJ_HINT_COP) { 2127 /* save ctx argument */ 2128 status = sljit_emit_op1(compiler, 2129 SLJIT_MOV_P, 2130 SLJIT_MEM1(SLJIT_LOCALS_REG), 2131 offsetof(struct bpfjit_stack, ctx), 2132 BJ_CTX_ARG, 0); 2133 if (status != SLJIT_SUCCESS) 2134 goto fail; 2135 } 2136 2137 if (extwords == 0) { 2138 mem_reg = SLJIT_MEM1(SLJIT_LOCALS_REG); 2139 mem_off = offsetof(struct bpfjit_stack, mem); 2140 } else { 2141 /* copy "mem" argument from bpf_args to bpfjit_stack */ 2142 status = sljit_emit_op1(compiler, 2143 SLJIT_MOV_P, 2144 BJ_TMP1REG, 0, 2145 SLJIT_MEM1(BJ_ARGS), offsetof(struct bpf_args, mem)); 2146 if (status != SLJIT_SUCCESS) 2147 goto fail; 2148 2149 status = sljit_emit_op1(compiler, 2150 SLJIT_MOV_P, 2151 SLJIT_MEM1(SLJIT_LOCALS_REG), 2152 offsetof(struct bpfjit_stack, extmem), 2153 BJ_TMP1REG, 0); 2154 if (status != SLJIT_SUCCESS) 2155 goto fail; 2156 2157 mem_reg = SLJIT_MEM1(BJ_TMP1REG); 2158 mem_off = 0; 2159 } 2160 2161 /* 2162 * Exclude pre-initialised external memory words but keep 2163 * initialization statuses of A and X registers in case 2164 * bc->preinited wrongly sets those two bits. 2165 */ 2166 initmask &= ~preinited | BJ_INIT_ABIT | BJ_INIT_XBIT; 2167 2168 #if defined(_KERNEL) 2169 /* bpf_filter() checks initialization of memwords. */ 2170 BJ_ASSERT((initmask & (BJ_INIT_MBIT(memwords) - 1)) == 0); 2171 #endif 2172 for (i = 0; i < memwords; i++) { 2173 if (initmask & BJ_INIT_MBIT(i)) { 2174 /* M[i] = 0; */ 2175 status = sljit_emit_op1(compiler, 2176 SLJIT_MOV_UI, 2177 mem_reg, mem_off + i * sizeof(uint32_t), 2178 SLJIT_IMM, 0); 2179 if (status != SLJIT_SUCCESS) 2180 goto fail; 2181 } 2182 } 2183 2184 if (initmask & BJ_INIT_ABIT) { 2185 /* A = 0; */ 2186 status = sljit_emit_op1(compiler, 2187 SLJIT_MOV, 2188 BJ_AREG, 0, 2189 SLJIT_IMM, 0); 2190 if (status != SLJIT_SUCCESS) 2191 goto fail; 2192 } 2193 2194 if (initmask & BJ_INIT_XBIT) { 2195 /* X = 0; */ 2196 status = sljit_emit_op1(compiler, 2197 SLJIT_MOV, 2198 BJ_XREG, 0, 2199 SLJIT_IMM, 0); 2200 if (status != SLJIT_SUCCESS) 2201 goto fail; 2202 } 2203 2204 status = load_buf_buflen(compiler); 2205 if (status != SLJIT_SUCCESS) 2206 goto fail; 2207 2208 if (!generate_insn_code(compiler, bc, insns, insn_dat, insn_count)) 2209 goto fail; 2210 2211 rv = sljit_generate_code(compiler); 2212 2213 fail: 2214 if (compiler != NULL) 2215 sljit_free_compiler(compiler); 2216 2217 if (insn_dat != NULL) 2218 BJ_FREE(insn_dat, insn_count * sizeof(insn_dat[0])); 2219 2220 return (bpfjit_func_t)rv; 2221 } 2222 2223 void 2224 bpfjit_free_code(bpfjit_func_t code) 2225 { 2226 2227 sljit_free_code((void *)code); 2228 } 2229
Indexes created Thu Sep 25 16:09:42 GMT 2025