bpfjit.c revision 1.26
1 /* $NetBSD: bpfjit.c,v 1.26 2014/07/12 20:14:18 alnsn Exp $ */ 2 3 /*- 4 * Copyright (c) 2011-2014 Alexander Nasonov. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 21 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 22 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 23 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 24 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 25 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 26 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 27 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 28 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 #ifdef _KERNEL 34 __KERNEL_RCSID(0, "$NetBSD: bpfjit.c,v 1.26 2014/07/12 20:14:18 alnsn Exp $"); 35 #else 36 __RCSID("$NetBSD: bpfjit.c,v 1.26 2014/07/12 20:14:18 alnsn Exp $"); 37 #endif 38 39 #include <sys/types.h> 40 #include <sys/queue.h> 41 42 #ifndef _KERNEL 43 #include <assert.h> 44 #define BJ_ASSERT(c) assert(c) 45 #else 46 #define BJ_ASSERT(c) KASSERT(c) 47 #endif 48 49 #ifndef _KERNEL 50 #include <stdlib.h> 51 #define BJ_ALLOC(sz) malloc(sz) 52 #define BJ_FREE(p, sz) free(p) 53 #else 54 #include <sys/kmem.h> 55 #define BJ_ALLOC(sz) kmem_alloc(sz, KM_SLEEP) 56 #define BJ_FREE(p, sz) kmem_free(p, sz) 57 #endif 58 59 #ifndef _KERNEL 60 #include <limits.h> 61 #include <stdbool.h> 62 #include <stddef.h> 63 #include <stdint.h> 64 #else 65 #include <sys/atomic.h> 66 #include <sys/module.h> 67 #endif 68 69 #define __BPF_PRIVATE 70 #include <net/bpf.h> 71 #include <net/bpfjit.h> 72 #include <sljitLir.h> 73 74 #if !defined(_KERNEL) && defined(SLJIT_VERBOSE) && SLJIT_VERBOSE 75 #include <stdio.h> /* for stderr */ 76 #endif 77 78 /* 79 * Arguments of generated bpfjit_func_t. 80 * The first argument is reassigned upon entry 81 * to a more frequently used buf argument. 82 */ 83 #define BJ_CTX_ARG SLJIT_SAVED_REG1 84 #define BJ_ARGS SLJIT_SAVED_REG2 85 86 /* 87 * Permanent register assignments. 88 */ 89 #define BJ_BUF SLJIT_SAVED_REG1 90 //#define BJ_ARGS SLJIT_SAVED_REG2 91 #define BJ_BUFLEN SLJIT_SAVED_REG3 92 #define BJ_AREG SLJIT_SCRATCH_REG1 93 #define BJ_TMP1REG SLJIT_SCRATCH_REG2 94 #define BJ_TMP2REG SLJIT_SCRATCH_REG3 95 #define BJ_XREG SLJIT_TEMPORARY_EREG1 96 #define BJ_TMP3REG SLJIT_TEMPORARY_EREG2 97 98 /* 99 * EREG registers can't be used for indirect calls, reuse BJ_BUF and 100 * BJ_BUFLEN registers. They can be easily restored from BJ_ARGS. 101 */ 102 #define BJ_COPF_PTR SLJIT_SAVED_REG1 103 #define BJ_COPF_IDX SLJIT_SAVED_REG3 104 105 #ifdef _KERNEL 106 #define MAX_MEMWORDS BPF_MAX_MEMWORDS 107 #else 108 #define MAX_MEMWORDS BPF_MEMWORDS 109 #endif 110 111 #define BJ_INIT_NOBITS ((bpf_memword_init_t)0) 112 #define BJ_INIT_MBIT(k) BPF_MEMWORD_INIT(k) 113 #define BJ_INIT_ABIT BJ_INIT_MBIT(MAX_MEMWORDS) 114 #define BJ_INIT_XBIT BJ_INIT_MBIT(MAX_MEMWORDS + 1) 115 116 /* 117 * Get a number of memwords and external memwords from a bpf_ctx object. 118 */ 119 #define GET_EXTWORDS(bc) ((bc) ? (bc)->extwords : 0) 120 #define GET_MEMWORDS(bc) (GET_EXTWORDS(bc) ? GET_EXTWORDS(bc) : BPF_MEMWORDS) 121 122 /* 123 * Optimization hints. 124 */ 125 typedef unsigned int bpfjit_hint_t; 126 #define BJ_HINT_PKT 0x01 /* packet read */ 127 #define BJ_HINT_LDW 0x02 /* 32-bit load */ 128 #define BJ_HINT_COP 0x04 /* BPF_COP or BPF_COPX instruction */ 129 #define BJ_HINT_XREG 0x08 /* BJ_XREG is needed */ 130 #define BJ_HINT_LDX 0x10 /* BPF_LDX instruction */ 131 132 /* 133 * Datatype for Array Bounds Check Elimination (ABC) pass. 134 */ 135 typedef uint64_t bpfjit_abc_length_t; 136 #define MAX_ABC_LENGTH (UINT32_MAX + UINT64_C(4)) /* max. width is 4 */ 137 138 struct bpfjit_stack 139 { 140 bpf_ctx_t *ctx; 141 uint32_t *extmem; /* pointer to external memory store */ 142 #ifdef _KERNEL 143 int err; /* 3rd argument for m_xword/m_xhalf/m_xbyte function call */ 144 #endif 145 uint32_t mem[BPF_MEMWORDS]; /* internal memory store */ 146 }; 147 148 /* 149 * Data for BPF_JMP instruction. 150 * Forward declaration for struct bpfjit_jump. 151 */ 152 struct bpfjit_jump_data; 153 154 /* 155 * Node of bjumps list. 156 */ 157 struct bpfjit_jump { 158 struct sljit_jump *sjump; 159 SLIST_ENTRY(bpfjit_jump) entries; 160 struct bpfjit_jump_data *jdata; 161 }; 162 163 /* 164 * Data for BPF_JMP instruction. 165 */ 166 struct bpfjit_jump_data { 167 /* 168 * These entries make up bjumps list: 169 * jtf[0] - when coming from jt path, 170 * jtf[1] - when coming from jf path. 171 */ 172 struct bpfjit_jump jtf[2]; 173 /* 174 * Length calculated by Array Bounds Check Elimination (ABC) pass. 175 */ 176 bpfjit_abc_length_t abc_length; 177 /* 178 * Length checked by the last out-of-bounds check. 179 */ 180 bpfjit_abc_length_t checked_length; 181 }; 182 183 /* 184 * Data for "read from packet" instructions. 185 * See also read_pkt_insn() function below. 186 */ 187 struct bpfjit_read_pkt_data { 188 /* 189 * Length calculated by Array Bounds Check Elimination (ABC) pass. 190 */ 191 bpfjit_abc_length_t abc_length; 192 /* 193 * If positive, emit "if (buflen < check_length) return 0" 194 * out-of-bounds check. 195 * Values greater than UINT32_MAX generate unconditional "return 0". 196 */ 197 bpfjit_abc_length_t check_length; 198 }; 199 200 /* 201 * Additional (optimization-related) data for bpf_insn. 202 */ 203 struct bpfjit_insn_data { 204 /* List of jumps to this insn. */ 205 SLIST_HEAD(, bpfjit_jump) bjumps; 206 207 union { 208 struct bpfjit_jump_data jdata; 209 struct bpfjit_read_pkt_data rdata; 210 } u; 211 212 bpf_memword_init_t invalid; 213 bool unreachable; 214 }; 215 216 #ifdef _KERNEL 217 218 uint32_t m_xword(const struct mbuf *, uint32_t, int *); 219 uint32_t m_xhalf(const struct mbuf *, uint32_t, int *); 220 uint32_t m_xbyte(const struct mbuf *, uint32_t, int *); 221 222 MODULE(MODULE_CLASS_MISC, bpfjit, "sljit") 223 224 static int 225 bpfjit_modcmd(modcmd_t cmd, void *arg) 226 { 227 228 switch (cmd) { 229 case MODULE_CMD_INIT: 230 bpfjit_module_ops.bj_free_code = &bpfjit_free_code; 231 membar_producer(); 232 bpfjit_module_ops.bj_generate_code = &bpfjit_generate_code; 233 membar_producer(); 234 return 0; 235 236 case MODULE_CMD_FINI: 237 return EOPNOTSUPP; 238 239 default: 240 return ENOTTY; 241 } 242 } 243 #endif 244 245 /* 246 * Return a number of scratch registers to pass 247 * to sljit_emit_enter() function. 248 */ 249 static sljit_si 250 nscratches(bpfjit_hint_t hints) 251 { 252 sljit_si rv = 2; 253 254 #ifdef _KERNEL 255 if (hints & BJ_HINT_PKT) 256 rv = 3; /* xcall with three arguments */ 257 #endif 258 259 if (hints & (BJ_HINT_LDW|BJ_HINT_PKT)) 260 rv = 3; /* uses BJ_TMP2REG */ 261 262 if (hints & BJ_HINT_COP) 263 rv = 3; /* calls copfunc with three arguments */ 264 265 if (hints & BJ_HINT_XREG) 266 rv = 4; /* uses BJ_XREG */ 267 268 #ifdef _KERNEL 269 if (hints & BJ_HINT_LDX) 270 rv = 5; /* uses BJ_TMP3REG */ 271 #endif 272 273 return rv; 274 } 275 276 static uint32_t 277 read_width(const struct bpf_insn *pc) 278 { 279 280 switch (BPF_SIZE(pc->code)) { 281 case BPF_W: 282 return 4; 283 case BPF_H: 284 return 2; 285 case BPF_B: 286 return 1; 287 default: 288 BJ_ASSERT(false); 289 return 0; 290 } 291 } 292 293 /* 294 * Copy buf and buflen members of bpf_args from BJ_ARGS 295 * pointer to BJ_BUF and BJ_BUFLEN registers. 296 */ 297 static int 298 load_buf_buflen(struct sljit_compiler *compiler) 299 { 300 int status; 301 302 status = sljit_emit_op1(compiler, 303 SLJIT_MOV_P, 304 BJ_BUF, 0, 305 SLJIT_MEM1(BJ_ARGS), 306 offsetof(struct bpf_args, pkt)); 307 if (status != SLJIT_SUCCESS) 308 return status; 309 310 status = sljit_emit_op1(compiler, 311 SLJIT_MOV, /* size_t source */ 312 BJ_BUFLEN, 0, 313 SLJIT_MEM1(BJ_ARGS), 314 offsetof(struct bpf_args, buflen)); 315 316 return status; 317 } 318 319 static bool 320 grow_jumps(struct sljit_jump ***jumps, size_t *size) 321 { 322 struct sljit_jump **newptr; 323 const size_t elemsz = sizeof(struct sljit_jump *); 324 size_t old_size = *size; 325 size_t new_size = 2 * old_size; 326 327 if (new_size < old_size || new_size > SIZE_MAX / elemsz) 328 return false; 329 330 newptr = BJ_ALLOC(new_size * elemsz); 331 if (newptr == NULL) 332 return false; 333 334 memcpy(newptr, *jumps, old_size * elemsz); 335 BJ_FREE(*jumps, old_size * elemsz); 336 337 *jumps = newptr; 338 *size = new_size; 339 return true; 340 } 341 342 static bool 343 append_jump(struct sljit_jump *jump, struct sljit_jump ***jumps, 344 size_t *size, size_t *max_size) 345 { 346 if (*size == *max_size && !grow_jumps(jumps, max_size)) 347 return false; 348 349 (*jumps)[(*size)++] = jump; 350 return true; 351 } 352 353 /* 354 * Emit code for BPF_LD+BPF_B+BPF_ABS A <- P[k:1]. 355 */ 356 static int 357 emit_read8(struct sljit_compiler *compiler, uint32_t k) 358 { 359 360 return sljit_emit_op1(compiler, 361 SLJIT_MOV_UB, 362 BJ_AREG, 0, 363 SLJIT_MEM1(BJ_BUF), k); 364 } 365 366 /* 367 * Emit code for BPF_LD+BPF_H+BPF_ABS A <- P[k:2]. 368 */ 369 static int 370 emit_read16(struct sljit_compiler *compiler, uint32_t k) 371 { 372 int status; 373 374 /* tmp1 = buf[k]; */ 375 status = sljit_emit_op1(compiler, 376 SLJIT_MOV_UB, 377 BJ_TMP1REG, 0, 378 SLJIT_MEM1(BJ_BUF), k); 379 if (status != SLJIT_SUCCESS) 380 return status; 381 382 /* A = buf[k+1]; */ 383 status = sljit_emit_op1(compiler, 384 SLJIT_MOV_UB, 385 BJ_AREG, 0, 386 SLJIT_MEM1(BJ_BUF), k+1); 387 if (status != SLJIT_SUCCESS) 388 return status; 389 390 /* tmp1 = tmp1 << 8; */ 391 status = sljit_emit_op2(compiler, 392 SLJIT_SHL, 393 BJ_TMP1REG, 0, 394 BJ_TMP1REG, 0, 395 SLJIT_IMM, 8); 396 if (status != SLJIT_SUCCESS) 397 return status; 398 399 /* A = A + tmp1; */ 400 status = sljit_emit_op2(compiler, 401 SLJIT_ADD, 402 BJ_AREG, 0, 403 BJ_AREG, 0, 404 BJ_TMP1REG, 0); 405 return status; 406 } 407 408 /* 409 * Emit code for BPF_LD+BPF_W+BPF_ABS A <- P[k:4]. 410 */ 411 static int 412 emit_read32(struct sljit_compiler *compiler, uint32_t k) 413 { 414 int status; 415 416 /* tmp1 = buf[k]; */ 417 status = sljit_emit_op1(compiler, 418 SLJIT_MOV_UB, 419 BJ_TMP1REG, 0, 420 SLJIT_MEM1(BJ_BUF), k); 421 if (status != SLJIT_SUCCESS) 422 return status; 423 424 /* tmp2 = buf[k+1]; */ 425 status = sljit_emit_op1(compiler, 426 SLJIT_MOV_UB, 427 BJ_TMP2REG, 0, 428 SLJIT_MEM1(BJ_BUF), k+1); 429 if (status != SLJIT_SUCCESS) 430 return status; 431 432 /* A = buf[k+3]; */ 433 status = sljit_emit_op1(compiler, 434 SLJIT_MOV_UB, 435 BJ_AREG, 0, 436 SLJIT_MEM1(BJ_BUF), k+3); 437 if (status != SLJIT_SUCCESS) 438 return status; 439 440 /* tmp1 = tmp1 << 24; */ 441 status = sljit_emit_op2(compiler, 442 SLJIT_SHL, 443 BJ_TMP1REG, 0, 444 BJ_TMP1REG, 0, 445 SLJIT_IMM, 24); 446 if (status != SLJIT_SUCCESS) 447 return status; 448 449 /* A = A + tmp1; */ 450 status = sljit_emit_op2(compiler, 451 SLJIT_ADD, 452 BJ_AREG, 0, 453 BJ_AREG, 0, 454 BJ_TMP1REG, 0); 455 if (status != SLJIT_SUCCESS) 456 return status; 457 458 /* tmp1 = buf[k+2]; */ 459 status = sljit_emit_op1(compiler, 460 SLJIT_MOV_UB, 461 BJ_TMP1REG, 0, 462 SLJIT_MEM1(BJ_BUF), k+2); 463 if (status != SLJIT_SUCCESS) 464 return status; 465 466 /* tmp2 = tmp2 << 16; */ 467 status = sljit_emit_op2(compiler, 468 SLJIT_SHL, 469 BJ_TMP2REG, 0, 470 BJ_TMP2REG, 0, 471 SLJIT_IMM, 16); 472 if (status != SLJIT_SUCCESS) 473 return status; 474 475 /* A = A + tmp2; */ 476 status = sljit_emit_op2(compiler, 477 SLJIT_ADD, 478 BJ_AREG, 0, 479 BJ_AREG, 0, 480 BJ_TMP2REG, 0); 481 if (status != SLJIT_SUCCESS) 482 return status; 483 484 /* tmp1 = tmp1 << 8; */ 485 status = sljit_emit_op2(compiler, 486 SLJIT_SHL, 487 BJ_TMP1REG, 0, 488 BJ_TMP1REG, 0, 489 SLJIT_IMM, 8); 490 if (status != SLJIT_SUCCESS) 491 return status; 492 493 /* A = A + tmp1; */ 494 status = sljit_emit_op2(compiler, 495 SLJIT_ADD, 496 BJ_AREG, 0, 497 BJ_AREG, 0, 498 BJ_TMP1REG, 0); 499 return status; 500 } 501 502 #ifdef _KERNEL 503 /* 504 * Emit code for m_xword/m_xhalf/m_xbyte call. 505 * 506 * @pc BPF_LD+BPF_W+BPF_ABS A <- P[k:4] 507 * BPF_LD+BPF_H+BPF_ABS A <- P[k:2] 508 * BPF_LD+BPF_B+BPF_ABS A <- P[k:1] 509 * BPF_LD+BPF_W+BPF_IND A <- P[X+k:4] 510 * BPF_LD+BPF_H+BPF_IND A <- P[X+k:2] 511 * BPF_LD+BPF_B+BPF_IND A <- P[X+k:1] 512 * BPF_LDX+BPF_B+BPF_MSH X <- 4*(P[k:1]&0xf) 513 */ 514 static int 515 emit_xcall(struct sljit_compiler *compiler, const struct bpf_insn *pc, 516 int dst, struct sljit_jump ***ret0, size_t *ret0_size, size_t *ret0_maxsize, 517 uint32_t (*fn)(const struct mbuf *, uint32_t, int *)) 518 { 519 #if BJ_XREG == SLJIT_RETURN_REG || \ 520 BJ_XREG == SLJIT_SCRATCH_REG1 || \ 521 BJ_XREG == SLJIT_SCRATCH_REG2 || \ 522 BJ_XREG == SLJIT_SCRATCH_REG3 || \ 523 BJ_TMP3REG == SLJIT_RETURN_REG || \ 524 BJ_TMP3REG == SLJIT_SCRATCH_REG1 || \ 525 BJ_TMP3REG == SLJIT_SCRATCH_REG2 || \ 526 BJ_TMP3REG == SLJIT_SCRATCH_REG3 527 #error "Not supported assignment of registers." 528 #endif 529 struct sljit_jump *jump; 530 int status; 531 532 BJ_ASSERT(dst != BJ_TMP2REG && dst != BJ_TMP3REG); 533 534 if (BPF_CLASS(pc->code) == BPF_LDX) { 535 /* save A */ 536 status = sljit_emit_op1(compiler, 537 SLJIT_MOV, 538 BJ_TMP3REG, 0, 539 BJ_AREG, 0); 540 if (status != SLJIT_SUCCESS) 541 return status; 542 } 543 544 /* 545 * Prepare registers for fn(mbuf, k, &err) call. 546 */ 547 status = sljit_emit_op1(compiler, 548 SLJIT_MOV, 549 SLJIT_SCRATCH_REG1, 0, 550 BJ_BUF, 0); 551 if (status != SLJIT_SUCCESS) 552 return status; 553 554 if (BPF_CLASS(pc->code) == BPF_LD && BPF_MODE(pc->code) == BPF_IND) { 555 /* if (X > UINT32_MAX - pc->k) return 0; */ 556 jump = sljit_emit_cmp(compiler, 557 SLJIT_C_GREATER, 558 BJ_XREG, 0, 559 SLJIT_IMM, UINT32_MAX - pc->k); 560 if (jump == NULL) 561 return SLJIT_ERR_ALLOC_FAILED; 562 if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 563 return SLJIT_ERR_ALLOC_FAILED; 564 565 /* k = X + pc->k; */ 566 status = sljit_emit_op2(compiler, 567 SLJIT_ADD, 568 SLJIT_SCRATCH_REG2, 0, 569 BJ_XREG, 0, 570 SLJIT_IMM, (uint32_t)pc->k); 571 if (status != SLJIT_SUCCESS) 572 return status; 573 } else { 574 /* k = pc->k */ 575 status = sljit_emit_op1(compiler, 576 SLJIT_MOV, 577 SLJIT_SCRATCH_REG2, 0, 578 SLJIT_IMM, (uint32_t)pc->k); 579 if (status != SLJIT_SUCCESS) 580 return status; 581 } 582 583 /* 584 * The third argument of fn is an address on stack. 585 */ 586 status = sljit_get_local_base(compiler, 587 SLJIT_SCRATCH_REG3, 0, 588 offsetof(struct bpfjit_stack, err)); 589 if (status != SLJIT_SUCCESS) 590 return status; 591 592 /* fn(buf, k, &err); */ 593 status = sljit_emit_ijump(compiler, 594 SLJIT_CALL3, 595 SLJIT_IMM, SLJIT_FUNC_OFFSET(fn)); 596 if (status != SLJIT_SUCCESS) 597 return status; 598 599 if (dst != SLJIT_RETURN_REG) { 600 /* move return value to dst */ 601 status = sljit_emit_op1(compiler, 602 SLJIT_MOV, 603 dst, 0, 604 SLJIT_RETURN_REG, 0); 605 if (status != SLJIT_SUCCESS) 606 return status; 607 } 608 609 /* tmp2 = *err; */ 610 status = sljit_emit_op1(compiler, 611 SLJIT_MOV_UI, 612 BJ_TMP2REG, 0, 613 SLJIT_MEM1(SLJIT_LOCALS_REG), 614 offsetof(struct bpfjit_stack, err)); 615 if (status != SLJIT_SUCCESS) 616 return status; 617 618 /* if (tmp2 != 0) return 0; */ 619 jump = sljit_emit_cmp(compiler, 620 SLJIT_C_NOT_EQUAL, 621 BJ_TMP2REG, 0, 622 SLJIT_IMM, 0); 623 if (jump == NULL) 624 return SLJIT_ERR_ALLOC_FAILED; 625 626 if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 627 return SLJIT_ERR_ALLOC_FAILED; 628 629 if (BPF_CLASS(pc->code) == BPF_LDX) { 630 /* restore A */ 631 status = sljit_emit_op1(compiler, 632 SLJIT_MOV, 633 BJ_AREG, 0, 634 BJ_TMP3REG, 0); 635 if (status != SLJIT_SUCCESS) 636 return status; 637 } 638 639 return SLJIT_SUCCESS; 640 } 641 #endif 642 643 /* 644 * Emit code for BPF_COP and BPF_COPX instructions. 645 */ 646 static int 647 emit_cop(struct sljit_compiler *compiler, const bpf_ctx_t *bc, 648 const struct bpf_insn *pc, struct sljit_jump **ret0_jump) 649 { 650 #if BJ_XREG == SLJIT_RETURN_REG || \ 651 BJ_XREG == SLJIT_SCRATCH_REG1 || \ 652 BJ_XREG == SLJIT_SCRATCH_REG2 || \ 653 BJ_XREG == SLJIT_SCRATCH_REG3 || \ 654 BJ_COPF_PTR == BJ_ARGS || \ 655 BJ_COPF_IDX == BJ_ARGS 656 #error "Not supported assignment of registers." 657 #endif 658 659 struct sljit_jump *jump; 660 int status; 661 662 jump = NULL; 663 664 BJ_ASSERT(bc != NULL && bc->copfuncs != NULL); 665 666 if (BPF_MISCOP(pc->code) == BPF_COPX) { 667 /* if (X >= bc->nfuncs) return 0; */ 668 jump = sljit_emit_cmp(compiler, 669 SLJIT_C_GREATER_EQUAL, 670 BJ_XREG, 0, 671 SLJIT_IMM, bc->nfuncs); 672 if (jump == NULL) 673 return SLJIT_ERR_ALLOC_FAILED; 674 } 675 676 if (jump != NULL) 677 *ret0_jump = jump; 678 679 /* 680 * Copy bpf_copfunc_t arguments to registers. 681 */ 682 #if BJ_AREG != SLJIT_SCRATCH_REG3 683 status = sljit_emit_op1(compiler, 684 SLJIT_MOV_UI, 685 SLJIT_SCRATCH_REG3, 0, 686 BJ_AREG, 0); 687 if (status != SLJIT_SUCCESS) 688 return status; 689 #endif 690 691 status = sljit_emit_op1(compiler, 692 SLJIT_MOV_P, 693 SLJIT_SCRATCH_REG1, 0, 694 SLJIT_MEM1(SLJIT_LOCALS_REG), 695 offsetof(struct bpfjit_stack, ctx)); 696 if (status != SLJIT_SUCCESS) 697 return status; 698 699 status = sljit_emit_op1(compiler, 700 SLJIT_MOV_P, 701 SLJIT_SCRATCH_REG2, 0, 702 BJ_ARGS, 0); 703 if (status != SLJIT_SUCCESS) 704 return status; 705 706 if (BPF_MISCOP(pc->code) == BPF_COP) { 707 status = sljit_emit_ijump(compiler, 708 SLJIT_CALL3, 709 SLJIT_IMM, SLJIT_FUNC_OFFSET(bc->copfuncs[pc->k])); 710 if (status != SLJIT_SUCCESS) 711 return status; 712 } else if (BPF_MISCOP(pc->code) == BPF_COPX) { 713 /* load ctx->copfuncs */ 714 status = sljit_emit_op1(compiler, 715 SLJIT_MOV_P, 716 BJ_COPF_PTR, 0, 717 SLJIT_MEM1(SLJIT_SCRATCH_REG1), 718 offsetof(struct bpf_ctx, copfuncs)); 719 if (status != SLJIT_SUCCESS) 720 return status; 721 722 /* 723 * Load X to a register that can be used for 724 * memory addressing. 725 */ 726 status = sljit_emit_op1(compiler, 727 SLJIT_MOV, 728 BJ_COPF_IDX, 0, 729 BJ_XREG, 0); 730 if (status != SLJIT_SUCCESS) 731 return status; 732 733 status = sljit_emit_ijump(compiler, 734 SLJIT_CALL3, 735 SLJIT_MEM2(BJ_COPF_PTR, BJ_COPF_IDX), 736 SLJIT_WORD_SHIFT); 737 if (status != SLJIT_SUCCESS) 738 return status; 739 740 status = load_buf_buflen(compiler); 741 if (status != SLJIT_SUCCESS) 742 return status; 743 } 744 745 #if BJ_AREG != SLJIT_RETURN_REG 746 status = sljit_emit_op1(compiler, 747 SLJIT_MOV, 748 BJ_AREG, 0, 749 SLJIT_RETURN_REG, 0); 750 if (status != SLJIT_SUCCESS) 751 return status; 752 #endif 753 754 return SLJIT_SUCCESS; 755 } 756 757 /* 758 * Generate code for 759 * BPF_LD+BPF_W+BPF_ABS A <- P[k:4] 760 * BPF_LD+BPF_H+BPF_ABS A <- P[k:2] 761 * BPF_LD+BPF_B+BPF_ABS A <- P[k:1] 762 * BPF_LD+BPF_W+BPF_IND A <- P[X+k:4] 763 * BPF_LD+BPF_H+BPF_IND A <- P[X+k:2] 764 * BPF_LD+BPF_B+BPF_IND A <- P[X+k:1] 765 */ 766 static int 767 emit_pkt_read(struct sljit_compiler *compiler, 768 const struct bpf_insn *pc, struct sljit_jump *to_mchain_jump, 769 struct sljit_jump ***ret0, size_t *ret0_size, size_t *ret0_maxsize) 770 { 771 int status = SLJIT_ERR_ALLOC_FAILED; 772 uint32_t width; 773 struct sljit_jump *jump; 774 #ifdef _KERNEL 775 struct sljit_label *label; 776 struct sljit_jump *over_mchain_jump; 777 const bool check_zero_buflen = (to_mchain_jump != NULL); 778 #endif 779 const uint32_t k = pc->k; 780 781 #ifdef _KERNEL 782 if (to_mchain_jump == NULL) { 783 to_mchain_jump = sljit_emit_cmp(compiler, 784 SLJIT_C_EQUAL, 785 BJ_BUFLEN, 0, 786 SLJIT_IMM, 0); 787 if (to_mchain_jump == NULL) 788 return SLJIT_ERR_ALLOC_FAILED; 789 } 790 #endif 791 792 width = read_width(pc); 793 794 if (BPF_MODE(pc->code) == BPF_IND) { 795 /* tmp1 = buflen - (pc->k + width); */ 796 status = sljit_emit_op2(compiler, 797 SLJIT_SUB, 798 BJ_TMP1REG, 0, 799 BJ_BUFLEN, 0, 800 SLJIT_IMM, k + width); 801 if (status != SLJIT_SUCCESS) 802 return status; 803 804 /* buf += X; */ 805 status = sljit_emit_op2(compiler, 806 SLJIT_ADD, 807 BJ_BUF, 0, 808 BJ_BUF, 0, 809 BJ_XREG, 0); 810 if (status != SLJIT_SUCCESS) 811 return status; 812 813 /* if (tmp1 < X) return 0; */ 814 jump = sljit_emit_cmp(compiler, 815 SLJIT_C_LESS, 816 BJ_TMP1REG, 0, 817 BJ_XREG, 0); 818 if (jump == NULL) 819 return SLJIT_ERR_ALLOC_FAILED; 820 if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 821 return SLJIT_ERR_ALLOC_FAILED; 822 } 823 824 switch (width) { 825 case 4: 826 status = emit_read32(compiler, k); 827 break; 828 case 2: 829 status = emit_read16(compiler, k); 830 break; 831 case 1: 832 status = emit_read8(compiler, k); 833 break; 834 } 835 836 if (status != SLJIT_SUCCESS) 837 return status; 838 839 if (BPF_MODE(pc->code) == BPF_IND) { 840 /* buf -= X; */ 841 status = sljit_emit_op2(compiler, 842 SLJIT_SUB, 843 BJ_BUF, 0, 844 BJ_BUF, 0, 845 BJ_XREG, 0); 846 if (status != SLJIT_SUCCESS) 847 return status; 848 } 849 850 #ifdef _KERNEL 851 over_mchain_jump = sljit_emit_jump(compiler, SLJIT_JUMP); 852 if (over_mchain_jump == NULL) 853 return SLJIT_ERR_ALLOC_FAILED; 854 855 /* entry point to mchain handler */ 856 label = sljit_emit_label(compiler); 857 if (label == NULL) 858 return SLJIT_ERR_ALLOC_FAILED; 859 sljit_set_label(to_mchain_jump, label); 860 861 if (check_zero_buflen) { 862 /* if (buflen != 0) return 0; */ 863 jump = sljit_emit_cmp(compiler, 864 SLJIT_C_NOT_EQUAL, 865 BJ_BUFLEN, 0, 866 SLJIT_IMM, 0); 867 if (jump == NULL) 868 return SLJIT_ERR_ALLOC_FAILED; 869 if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 870 return SLJIT_ERR_ALLOC_FAILED; 871 } 872 873 switch (width) { 874 case 4: 875 status = emit_xcall(compiler, pc, BJ_AREG, 876 ret0, ret0_size, ret0_maxsize, &m_xword); 877 break; 878 case 2: 879 status = emit_xcall(compiler, pc, BJ_AREG, 880 ret0, ret0_size, ret0_maxsize, &m_xhalf); 881 break; 882 case 1: 883 status = emit_xcall(compiler, pc, BJ_AREG, 884 ret0, ret0_size, ret0_maxsize, &m_xbyte); 885 break; 886 } 887 888 if (status != SLJIT_SUCCESS) 889 return status; 890 891 label = sljit_emit_label(compiler); 892 if (label == NULL) 893 return SLJIT_ERR_ALLOC_FAILED; 894 sljit_set_label(over_mchain_jump, label); 895 #endif 896 897 return SLJIT_SUCCESS; 898 } 899 900 static int 901 emit_memload(struct sljit_compiler *compiler, 902 sljit_si dst, uint32_t k, size_t extwords) 903 { 904 int status; 905 sljit_si src; 906 sljit_sw srcw; 907 908 srcw = k * sizeof(uint32_t); 909 910 if (extwords == 0) { 911 src = SLJIT_MEM1(SLJIT_LOCALS_REG); 912 srcw += offsetof(struct bpfjit_stack, mem); 913 } else { 914 /* copy extmem pointer to the tmp1 register */ 915 status = sljit_emit_op1(compiler, 916 SLJIT_MOV_P, 917 BJ_TMP1REG, 0, 918 SLJIT_MEM1(SLJIT_LOCALS_REG), 919 offsetof(struct bpfjit_stack, extmem)); 920 if (status != SLJIT_SUCCESS) 921 return status; 922 src = SLJIT_MEM1(BJ_TMP1REG); 923 } 924 925 return sljit_emit_op1(compiler, SLJIT_MOV_UI, dst, 0, src, srcw); 926 } 927 928 static int 929 emit_memstore(struct sljit_compiler *compiler, 930 sljit_si src, uint32_t k, size_t extwords) 931 { 932 int status; 933 sljit_si dst; 934 sljit_sw dstw; 935 936 dstw = k * sizeof(uint32_t); 937 938 if (extwords == 0) { 939 dst = SLJIT_MEM1(SLJIT_LOCALS_REG); 940 dstw += offsetof(struct bpfjit_stack, mem); 941 } else { 942 /* copy extmem pointer to the tmp1 register */ 943 status = sljit_emit_op1(compiler, 944 SLJIT_MOV_P, 945 BJ_TMP1REG, 0, 946 SLJIT_MEM1(SLJIT_LOCALS_REG), 947 offsetof(struct bpfjit_stack, extmem)); 948 if (status != SLJIT_SUCCESS) 949 return status; 950 dst = SLJIT_MEM1(BJ_TMP1REG); 951 } 952 953 return sljit_emit_op1(compiler, SLJIT_MOV_UI, dst, dstw, src, 0); 954 } 955 956 /* 957 * Emit code for BPF_LDX+BPF_B+BPF_MSH X <- 4*(P[k:1]&0xf). 958 */ 959 static int 960 emit_msh(struct sljit_compiler *compiler, 961 const struct bpf_insn *pc, struct sljit_jump *to_mchain_jump, 962 struct sljit_jump ***ret0, size_t *ret0_size, size_t *ret0_maxsize) 963 { 964 int status; 965 #ifdef _KERNEL 966 struct sljit_label *label; 967 struct sljit_jump *jump, *over_mchain_jump; 968 const bool check_zero_buflen = (to_mchain_jump != NULL); 969 #endif 970 const uint32_t k = pc->k; 971 972 #ifdef _KERNEL 973 if (to_mchain_jump == NULL) { 974 to_mchain_jump = sljit_emit_cmp(compiler, 975 SLJIT_C_EQUAL, 976 BJ_BUFLEN, 0, 977 SLJIT_IMM, 0); 978 if (to_mchain_jump == NULL) 979 return SLJIT_ERR_ALLOC_FAILED; 980 } 981 #endif 982 983 /* tmp1 = buf[k] */ 984 status = sljit_emit_op1(compiler, 985 SLJIT_MOV_UB, 986 BJ_TMP1REG, 0, 987 SLJIT_MEM1(BJ_BUF), k); 988 if (status != SLJIT_SUCCESS) 989 return status; 990 991 /* tmp1 &= 0xf */ 992 status = sljit_emit_op2(compiler, 993 SLJIT_AND, 994 BJ_TMP1REG, 0, 995 BJ_TMP1REG, 0, 996 SLJIT_IMM, 0xf); 997 if (status != SLJIT_SUCCESS) 998 return status; 999 1000 /* tmp1 = tmp1 << 2 */ 1001 status = sljit_emit_op2(compiler, 1002 SLJIT_SHL, 1003 BJ_XREG, 0, 1004 BJ_TMP1REG, 0, 1005 SLJIT_IMM, 2); 1006 if (status != SLJIT_SUCCESS) 1007 return status; 1008 1009 #ifdef _KERNEL 1010 over_mchain_jump = sljit_emit_jump(compiler, SLJIT_JUMP); 1011 if (over_mchain_jump == NULL) 1012 return SLJIT_ERR_ALLOC_FAILED; 1013 1014 /* entry point to mchain handler */ 1015 label = sljit_emit_label(compiler); 1016 if (label == NULL) 1017 return SLJIT_ERR_ALLOC_FAILED; 1018 sljit_set_label(to_mchain_jump, label); 1019 1020 if (check_zero_buflen) { 1021 /* if (buflen != 0) return 0; */ 1022 jump = sljit_emit_cmp(compiler, 1023 SLJIT_C_NOT_EQUAL, 1024 BJ_BUFLEN, 0, 1025 SLJIT_IMM, 0); 1026 if (jump == NULL) 1027 return SLJIT_ERR_ALLOC_FAILED; 1028 if (!append_jump(jump, ret0, ret0_size, ret0_maxsize)) 1029 return SLJIT_ERR_ALLOC_FAILED; 1030 } 1031 1032 status = emit_xcall(compiler, pc, BJ_TMP1REG, 1033 ret0, ret0_size, ret0_maxsize, &m_xbyte); 1034 if (status != SLJIT_SUCCESS) 1035 return status; 1036 1037 /* tmp1 &= 0xf */ 1038 status = sljit_emit_op2(compiler, 1039 SLJIT_AND, 1040 BJ_TMP1REG, 0, 1041 BJ_TMP1REG, 0, 1042 SLJIT_IMM, 0xf); 1043 if (status != SLJIT_SUCCESS) 1044 return status; 1045 1046 /* tmp1 = tmp1 << 2 */ 1047 status = sljit_emit_op2(compiler, 1048 SLJIT_SHL, 1049 BJ_XREG, 0, 1050 BJ_TMP1REG, 0, 1051 SLJIT_IMM, 2); 1052 if (status != SLJIT_SUCCESS) 1053 return status; 1054 1055 1056 label = sljit_emit_label(compiler); 1057 if (label == NULL) 1058 return SLJIT_ERR_ALLOC_FAILED; 1059 sljit_set_label(over_mchain_jump, label); 1060 #endif 1061 1062 return SLJIT_SUCCESS; 1063 } 1064 1065 static int 1066 emit_pow2_division(struct sljit_compiler *compiler, uint32_t k) 1067 { 1068 int shift = 0; 1069 int status = SLJIT_SUCCESS; 1070 1071 while (k > 1) { 1072 k >>= 1; 1073 shift++; 1074 } 1075 1076 BJ_ASSERT(k == 1 && shift < 32); 1077 1078 if (shift != 0) { 1079 status = sljit_emit_op2(compiler, 1080 SLJIT_LSHR|SLJIT_INT_OP, 1081 BJ_AREG, 0, 1082 BJ_AREG, 0, 1083 SLJIT_IMM, shift); 1084 } 1085 1086 return status; 1087 } 1088 1089 #if !defined(BPFJIT_USE_UDIV) 1090 static sljit_uw 1091 divide(sljit_uw x, sljit_uw y) 1092 { 1093 1094 return (uint32_t)x / (uint32_t)y; 1095 } 1096 #endif 1097 1098 /* 1099 * Emit code for A = A / div. 1100 * divt,divw are either SLJIT_IMM,pc->k or BJ_XREG,0. 1101 */ 1102 static int 1103 emit_division(struct sljit_compiler *compiler, int divt, sljit_sw divw) 1104 { 1105 int status; 1106 1107 #if BJ_XREG == SLJIT_RETURN_REG || \ 1108 BJ_XREG == SLJIT_SCRATCH_REG1 || \ 1109 BJ_XREG == SLJIT_SCRATCH_REG2 || \ 1110 BJ_AREG == SLJIT_SCRATCH_REG2 1111 #error "Not supported assignment of registers." 1112 #endif 1113 1114 #if BJ_AREG != SLJIT_SCRATCH_REG1 1115 status = sljit_emit_op1(compiler, 1116 SLJIT_MOV, 1117 SLJIT_SCRATCH_REG1, 0, 1118 BJ_AREG, 0); 1119 if (status != SLJIT_SUCCESS) 1120 return status; 1121 #endif 1122 1123 status = sljit_emit_op1(compiler, 1124 SLJIT_MOV, 1125 SLJIT_SCRATCH_REG2, 0, 1126 divt, divw); 1127 if (status != SLJIT_SUCCESS) 1128 return status; 1129 1130 #if defined(BPFJIT_USE_UDIV) 1131 status = sljit_emit_op0(compiler, SLJIT_UDIV|SLJIT_INT_OP); 1132 1133 #if BJ_AREG != SLJIT_SCRATCH_REG1 1134 status = sljit_emit_op1(compiler, 1135 SLJIT_MOV, 1136 BJ_AREG, 0, 1137 SLJIT_SCRATCH_REG1, 0); 1138 if (status != SLJIT_SUCCESS) 1139 return status; 1140 #endif 1141 #else 1142 status = sljit_emit_ijump(compiler, 1143 SLJIT_CALL2, 1144 SLJIT_IMM, SLJIT_FUNC_OFFSET(divide)); 1145 1146 #if BJ_AREG != SLJIT_RETURN_REG 1147 status = sljit_emit_op1(compiler, 1148 SLJIT_MOV, 1149 BJ_AREG, 0, 1150 SLJIT_RETURN_REG, 0); 1151 if (status != SLJIT_SUCCESS) 1152 return status; 1153 #endif 1154 #endif 1155 1156 return status; 1157 } 1158 1159 /* 1160 * Return true if pc is a "read from packet" instruction. 1161 * If length is not NULL and return value is true, *length will 1162 * be set to a safe length required to read a packet. 1163 */ 1164 static bool 1165 read_pkt_insn(const struct bpf_insn *pc, bpfjit_abc_length_t *length) 1166 { 1167 bool rv; 1168 bpfjit_abc_length_t width; 1169 1170 switch (BPF_CLASS(pc->code)) { 1171 default: 1172 rv = false; 1173 break; 1174 1175 case BPF_LD: 1176 rv = BPF_MODE(pc->code) == BPF_ABS || 1177 BPF_MODE(pc->code) == BPF_IND; 1178 if (rv) 1179 width = read_width(pc); 1180 break; 1181 1182 case BPF_LDX: 1183 rv = pc->code == (BPF_LDX|BPF_B|BPF_MSH); 1184 width = 1; 1185 break; 1186 } 1187 1188 if (rv && length != NULL) { 1189 /* 1190 * Values greater than UINT32_MAX will generate 1191 * unconditional "return 0". 1192 */ 1193 *length = (uint32_t)pc->k + width; 1194 } 1195 1196 return rv; 1197 } 1198 1199 static void 1200 optimize_init(struct bpfjit_insn_data *insn_dat, size_t insn_count) 1201 { 1202 size_t i; 1203 1204 for (i = 0; i < insn_count; i++) { 1205 SLIST_INIT(&insn_dat[i].bjumps); 1206 insn_dat[i].invalid = BJ_INIT_NOBITS; 1207 } 1208 } 1209 1210 /* 1211 * The function divides instructions into blocks. Destination of a jump 1212 * instruction starts a new block. BPF_RET and BPF_JMP instructions 1213 * terminate a block. Blocks are linear, that is, there are no jumps out 1214 * from the middle of a block and there are no jumps in to the middle of 1215 * a block. 1216 * 1217 * The function also sets bits in *initmask for memwords that 1218 * need to be initialized to zero. Note that this set should be empty 1219 * for any valid kernel filter program. 1220 */ 1221 static bool 1222 optimize_pass1(const bpf_ctx_t *bc, const struct bpf_insn *insns, 1223 struct bpfjit_insn_data *insn_dat, size_t insn_count, 1224 bpf_memword_init_t *initmask, bpfjit_hint_t *hints) 1225 { 1226 struct bpfjit_jump *jtf; 1227 size_t i; 1228 uint32_t jt, jf; 1229 bpfjit_abc_length_t length; 1230 bpf_memword_init_t invalid; /* borrowed from bpf_filter() */ 1231 bool unreachable; 1232 1233 const size_t memwords = GET_MEMWORDS(bc); 1234 1235 *hints = 0; 1236 *initmask = BJ_INIT_NOBITS; 1237 1238 unreachable = false; 1239 invalid = ~BJ_INIT_NOBITS; 1240 1241 for (i = 0; i < insn_count; i++) { 1242 if (!SLIST_EMPTY(&insn_dat[i].bjumps)) 1243 unreachable = false; 1244 insn_dat[i].unreachable = unreachable; 1245 1246 if (unreachable) 1247 continue; 1248 1249 invalid |= insn_dat[i].invalid; 1250 1251 if (read_pkt_insn(&insns[i], &length) && length > UINT32_MAX) 1252 unreachable = true; 1253 1254 switch (BPF_CLASS(insns[i].code)) { 1255 case BPF_RET: 1256 if (BPF_RVAL(insns[i].code) == BPF_A) 1257 *initmask |= invalid & BJ_INIT_ABIT; 1258 1259 unreachable = true; 1260 continue; 1261 1262 case BPF_LD: 1263 if (BPF_MODE(insns[i].code) == BPF_IND || 1264 BPF_MODE(insns[i].code) == BPF_ABS) { 1265 *hints |= BJ_HINT_PKT; 1266 } 1267 1268 if (BPF_SIZE(insns[i].code) == BPF_W) 1269 *hints |= BJ_HINT_LDW; 1270 1271 if (BPF_MODE(insns[i].code) == BPF_IND) { 1272 *hints |= BJ_HINT_XREG; 1273 *initmask |= invalid & BJ_INIT_XBIT; 1274 } 1275 1276 if (BPF_MODE(insns[i].code) == BPF_MEM && 1277 (uint32_t)insns[i].k < memwords) { 1278 *initmask |= invalid & BJ_INIT_MBIT(insns[i].k); 1279 } 1280 1281 invalid &= ~BJ_INIT_ABIT; 1282 continue; 1283 1284 case BPF_LDX: 1285 *hints |= BJ_HINT_XREG | BJ_HINT_LDX; 1286 1287 if (BPF_MODE(insns[i].code) == BPF_MEM && 1288 (uint32_t)insns[i].k < memwords) { 1289 *initmask |= invalid & BJ_INIT_MBIT(insns[i].k); 1290 } 1291 1292 invalid &= ~BJ_INIT_XBIT; 1293 continue; 1294 1295 case BPF_ST: 1296 *initmask |= invalid & BJ_INIT_ABIT; 1297 1298 if ((uint32_t)insns[i].k < memwords) 1299 invalid &= ~BJ_INIT_MBIT(insns[i].k); 1300 1301 continue; 1302 1303 case BPF_STX: 1304 *hints |= BJ_HINT_XREG; 1305 *initmask |= invalid & BJ_INIT_XBIT; 1306 1307 if ((uint32_t)insns[i].k < memwords) 1308 invalid &= ~BJ_INIT_MBIT(insns[i].k); 1309 1310 continue; 1311 1312 case BPF_ALU: 1313 *initmask |= invalid & BJ_INIT_ABIT; 1314 1315 if (insns[i].code != (BPF_ALU|BPF_NEG) && 1316 BPF_SRC(insns[i].code) == BPF_X) { 1317 *hints |= BJ_HINT_XREG; 1318 *initmask |= invalid & BJ_INIT_XBIT; 1319 } 1320 1321 invalid &= ~BJ_INIT_ABIT; 1322 continue; 1323 1324 case BPF_MISC: 1325 switch (BPF_MISCOP(insns[i].code)) { 1326 case BPF_TAX: // X <- A 1327 *hints |= BJ_HINT_XREG; 1328 *initmask |= invalid & BJ_INIT_ABIT; 1329 invalid &= ~BJ_INIT_XBIT; 1330 continue; 1331 1332 case BPF_TXA: // A <- X 1333 *hints |= BJ_HINT_XREG; 1334 *initmask |= invalid & BJ_INIT_XBIT; 1335 invalid &= ~BJ_INIT_ABIT; 1336 continue; 1337 1338 case BPF_COPX: 1339 *hints |= BJ_HINT_XREG; 1340 /* FALLTHROUGH */ 1341 1342 case BPF_COP: 1343 *hints |= BJ_HINT_COP; 1344 *initmask |= invalid & BJ_INIT_ABIT; 1345 invalid &= ~BJ_INIT_ABIT; 1346 continue; 1347 } 1348 1349 continue; 1350 1351 case BPF_JMP: 1352 /* Initialize abc_length for ABC pass. */ 1353 insn_dat[i].u.jdata.abc_length = MAX_ABC_LENGTH; 1354 1355 if (BPF_OP(insns[i].code) == BPF_JA) { 1356 jt = jf = insns[i].k; 1357 } else { 1358 jt = insns[i].jt; 1359 jf = insns[i].jf; 1360 } 1361 1362 if (jt >= insn_count - (i + 1) || 1363 jf >= insn_count - (i + 1)) { 1364 return false; 1365 } 1366 1367 if (jt > 0 && jf > 0) 1368 unreachable = true; 1369 1370 jt += i + 1; 1371 jf += i + 1; 1372 1373 jtf = insn_dat[i].u.jdata.jtf; 1374 1375 jtf[0].jdata = &insn_dat[i].u.jdata; 1376 SLIST_INSERT_HEAD(&insn_dat[jt].bjumps, 1377 &jtf[0], entries); 1378 1379 if (jf != jt) { 1380 jtf[1].jdata = &insn_dat[i].u.jdata; 1381 SLIST_INSERT_HEAD(&insn_dat[jf].bjumps, 1382 &jtf[1], entries); 1383 } 1384 1385 insn_dat[jf].invalid |= invalid; 1386 insn_dat[jt].invalid |= invalid; 1387 invalid = 0; 1388 1389 continue; 1390 } 1391 } 1392 1393 return true; 1394 } 1395 1396 /* 1397 * Array Bounds Check Elimination (ABC) pass. 1398 */ 1399 static void 1400 optimize_pass2(const bpf_ctx_t *bc, const struct bpf_insn *insns, 1401 struct bpfjit_insn_data *insn_dat, size_t insn_count) 1402 { 1403 struct bpfjit_jump *jmp; 1404 const struct bpf_insn *pc; 1405 struct bpfjit_insn_data *pd; 1406 size_t i; 1407 bpfjit_abc_length_t length, abc_length = 0; 1408 1409 const size_t extwords = GET_EXTWORDS(bc); 1410 1411 for (i = insn_count; i != 0; i--) { 1412 pc = &insns[i-1]; 1413 pd = &insn_dat[i-1]; 1414 1415 if (pd->unreachable) 1416 continue; 1417 1418 switch (BPF_CLASS(pc->code)) { 1419 case BPF_RET: 1420 /* 1421 * It's quite common for bpf programs to 1422 * check packet bytes in increasing order 1423 * and return zero if bytes don't match 1424 * specified critetion. Such programs disable 1425 * ABC optimization completely because for 1426 * every jump there is a branch with no read 1427 * instruction. 1428 * With no side effects, BPF_STMT(BPF_RET+BPF_K, 0) 1429 * is indistinguishable from out-of-bound load. 1430 * Therefore, abc_length can be set to 1431 * MAX_ABC_LENGTH and enable ABC for many 1432 * bpf programs. 1433 * If this optimization encounters any 1434 * instruction with a side effect, it will 1435 * reset abc_length. 1436 */ 1437 if (BPF_RVAL(pc->code) == BPF_K && pc->k == 0) 1438 abc_length = MAX_ABC_LENGTH; 1439 else 1440 abc_length = 0; 1441 break; 1442 1443 case BPF_MISC: 1444 if (BPF_MISCOP(pc->code) == BPF_COP || 1445 BPF_MISCOP(pc->code) == BPF_COPX) { 1446 /* COP instructions can have side effects. */ 1447 abc_length = 0; 1448 } 1449 break; 1450 1451 case BPF_ST: 1452 case BPF_STX: 1453 if (extwords != 0) { 1454 /* Write to memory is visible after a call. */ 1455 abc_length = 0; 1456 } 1457 break; 1458 1459 case BPF_JMP: 1460 abc_length = pd->u.jdata.abc_length; 1461 break; 1462 1463 default: 1464 if (read_pkt_insn(pc, &length)) { 1465 if (abc_length < length) 1466 abc_length = length; 1467 pd->u.rdata.abc_length = abc_length; 1468 } 1469 break; 1470 } 1471 1472 SLIST_FOREACH(jmp, &pd->bjumps, entries) { 1473 if (jmp->jdata->abc_length > abc_length) 1474 jmp->jdata->abc_length = abc_length; 1475 } 1476 } 1477 } 1478 1479 static void 1480 optimize_pass3(const struct bpf_insn *insns, 1481 struct bpfjit_insn_data *insn_dat, size_t insn_count) 1482 { 1483 struct bpfjit_jump *jmp; 1484 size_t i; 1485 bpfjit_abc_length_t checked_length = 0; 1486 1487 for (i = 0; i < insn_count; i++) { 1488 if (insn_dat[i].unreachable) 1489 continue; 1490 1491 SLIST_FOREACH(jmp, &insn_dat[i].bjumps, entries) { 1492 if (jmp->jdata->checked_length < checked_length) 1493 checked_length = jmp->jdata->checked_length; 1494 } 1495 1496 if (BPF_CLASS(insns[i].code) == BPF_JMP) { 1497 insn_dat[i].u.jdata.checked_length = checked_length; 1498 } else if (read_pkt_insn(&insns[i], NULL)) { 1499 struct bpfjit_read_pkt_data *rdata = 1500 &insn_dat[i].u.rdata; 1501 rdata->check_length = 0; 1502 if (checked_length < rdata->abc_length) { 1503 checked_length = rdata->abc_length; 1504 rdata->check_length = checked_length; 1505 } 1506 } 1507 } 1508 } 1509 1510 static bool 1511 optimize(const bpf_ctx_t *bc, const struct bpf_insn *insns, 1512 struct bpfjit_insn_data *insn_dat, size_t insn_count, 1513 bpf_memword_init_t *initmask, bpfjit_hint_t *hints) 1514 { 1515 1516 optimize_init(insn_dat, insn_count); 1517 1518 if (!optimize_pass1(bc, insns, insn_dat, insn_count, initmask, hints)) 1519 return false; 1520 1521 optimize_pass2(bc, insns, insn_dat, insn_count); 1522 optimize_pass3(insns, insn_dat, insn_count); 1523 1524 return true; 1525 } 1526 1527 /* 1528 * Convert BPF_ALU operations except BPF_NEG and BPF_DIV to sljit operation. 1529 */ 1530 static int 1531 bpf_alu_to_sljit_op(const struct bpf_insn *pc) 1532 { 1533 1534 /* 1535 * Note: all supported 64bit arches have 32bit multiply 1536 * instruction so SLJIT_INT_OP doesn't have any overhead. 1537 */ 1538 switch (BPF_OP(pc->code)) { 1539 case BPF_ADD: return SLJIT_ADD; 1540 case BPF_SUB: return SLJIT_SUB; 1541 case BPF_MUL: return SLJIT_MUL|SLJIT_INT_OP; 1542 case BPF_OR: return SLJIT_OR; 1543 case BPF_AND: return SLJIT_AND; 1544 case BPF_LSH: return SLJIT_SHL; 1545 case BPF_RSH: return SLJIT_LSHR|SLJIT_INT_OP; 1546 default: 1547 BJ_ASSERT(false); 1548 return 0; 1549 } 1550 } 1551 1552 /* 1553 * Convert BPF_JMP operations except BPF_JA to sljit condition. 1554 */ 1555 static int 1556 bpf_jmp_to_sljit_cond(const struct bpf_insn *pc, bool negate) 1557 { 1558 /* 1559 * Note: all supported 64bit arches have 32bit comparison 1560 * instructions so SLJIT_INT_OP doesn't have any overhead. 1561 */ 1562 int rv = SLJIT_INT_OP; 1563 1564 switch (BPF_OP(pc->code)) { 1565 case BPF_JGT: 1566 rv |= negate ? SLJIT_C_LESS_EQUAL : SLJIT_C_GREATER; 1567 break; 1568 case BPF_JGE: 1569 rv |= negate ? SLJIT_C_LESS : SLJIT_C_GREATER_EQUAL; 1570 break; 1571 case BPF_JEQ: 1572 rv |= negate ? SLJIT_C_NOT_EQUAL : SLJIT_C_EQUAL; 1573 break; 1574 case BPF_JSET: 1575 rv |= negate ? SLJIT_C_EQUAL : SLJIT_C_NOT_EQUAL; 1576 break; 1577 default: 1578 BJ_ASSERT(false); 1579 } 1580 1581 return rv; 1582 } 1583 1584 /* 1585 * Convert BPF_K and BPF_X to sljit register. 1586 */ 1587 static int 1588 kx_to_reg(const struct bpf_insn *pc) 1589 { 1590 1591 switch (BPF_SRC(pc->code)) { 1592 case BPF_K: return SLJIT_IMM; 1593 case BPF_X: return BJ_XREG; 1594 default: 1595 BJ_ASSERT(false); 1596 return 0; 1597 } 1598 } 1599 1600 static sljit_sw 1601 kx_to_reg_arg(const struct bpf_insn *pc) 1602 { 1603 1604 switch (BPF_SRC(pc->code)) { 1605 case BPF_K: return (uint32_t)pc->k; /* SLJIT_IMM, pc->k, */ 1606 case BPF_X: return 0; /* BJ_XREG, 0, */ 1607 default: 1608 BJ_ASSERT(false); 1609 return 0; 1610 } 1611 } 1612 1613 static bool 1614 generate_insn_code(struct sljit_compiler *compiler, const bpf_ctx_t *bc, 1615 const struct bpf_insn *insns, struct bpfjit_insn_data *insn_dat, 1616 size_t insn_count) 1617 { 1618 /* a list of jumps to out-of-bound return from a generated function */ 1619 struct sljit_jump **ret0; 1620 size_t ret0_size, ret0_maxsize; 1621 1622 struct sljit_jump *jump; 1623 struct sljit_label *label; 1624 const struct bpf_insn *pc; 1625 struct bpfjit_jump *bjump, *jtf; 1626 struct sljit_jump *to_mchain_jump; 1627 1628 size_t i; 1629 int status; 1630 int branching, negate; 1631 unsigned int rval, mode, src; 1632 uint32_t jt, jf; 1633 1634 bool unconditional_ret; 1635 bool rv; 1636 1637 const size_t extwords = GET_EXTWORDS(bc); 1638 const size_t memwords = GET_MEMWORDS(bc); 1639 1640 ret0 = NULL; 1641 rv = false; 1642 1643 ret0_size = 0; 1644 ret0_maxsize = 64; 1645 ret0 = BJ_ALLOC(ret0_maxsize * sizeof(ret0[0])); 1646 if (ret0 == NULL) 1647 goto fail; 1648 1649 /* reset sjump members of jdata */ 1650 for (i = 0; i < insn_count; i++) { 1651 if (insn_dat[i].unreachable || 1652 BPF_CLASS(insns[i].code) != BPF_JMP) { 1653 continue; 1654 } 1655 1656 jtf = insn_dat[i].u.jdata.jtf; 1657 jtf[0].sjump = jtf[1].sjump = NULL; 1658 } 1659 1660 /* main loop */ 1661 for (i = 0; i < insn_count; i++) { 1662 if (insn_dat[i].unreachable) 1663 continue; 1664 1665 /* 1666 * Resolve jumps to the current insn. 1667 */ 1668 label = NULL; 1669 SLIST_FOREACH(bjump, &insn_dat[i].bjumps, entries) { 1670 if (bjump->sjump != NULL) { 1671 if (label == NULL) 1672 label = sljit_emit_label(compiler); 1673 if (label == NULL) 1674 goto fail; 1675 sljit_set_label(bjump->sjump, label); 1676 } 1677 } 1678 1679 to_mchain_jump = NULL; 1680 unconditional_ret = false; 1681 1682 if (read_pkt_insn(&insns[i], NULL)) { 1683 if (insn_dat[i].u.rdata.check_length > UINT32_MAX) { 1684 /* Jump to "return 0" unconditionally. */ 1685 unconditional_ret = true; 1686 jump = sljit_emit_jump(compiler, SLJIT_JUMP); 1687 if (jump == NULL) 1688 goto fail; 1689 if (!append_jump(jump, &ret0, 1690 &ret0_size, &ret0_maxsize)) 1691 goto fail; 1692 } else if (insn_dat[i].u.rdata.check_length > 0) { 1693 /* if (buflen < check_length) return 0; */ 1694 jump = sljit_emit_cmp(compiler, 1695 SLJIT_C_LESS, 1696 BJ_BUFLEN, 0, 1697 SLJIT_IMM, 1698 insn_dat[i].u.rdata.check_length); 1699 if (jump == NULL) 1700 goto fail; 1701 #ifdef _KERNEL 1702 to_mchain_jump = jump; 1703 #else 1704 if (!append_jump(jump, &ret0, 1705 &ret0_size, &ret0_maxsize)) 1706 goto fail; 1707 #endif 1708 } 1709 } 1710 1711 pc = &insns[i]; 1712 switch (BPF_CLASS(pc->code)) { 1713 1714 default: 1715 goto fail; 1716 1717 case BPF_LD: 1718 /* BPF_LD+BPF_IMM A <- k */ 1719 if (pc->code == (BPF_LD|BPF_IMM)) { 1720 status = sljit_emit_op1(compiler, 1721 SLJIT_MOV, 1722 BJ_AREG, 0, 1723 SLJIT_IMM, (uint32_t)pc->k); 1724 if (status != SLJIT_SUCCESS) 1725 goto fail; 1726 1727 continue; 1728 } 1729 1730 /* BPF_LD+BPF_MEM A <- M[k] */ 1731 if (pc->code == (BPF_LD|BPF_MEM)) { 1732 if ((uint32_t)pc->k >= memwords) 1733 goto fail; 1734 status = emit_memload(compiler, 1735 BJ_AREG, pc->k, extwords); 1736 if (status != SLJIT_SUCCESS) 1737 goto fail; 1738 1739 continue; 1740 } 1741 1742 /* BPF_LD+BPF_W+BPF_LEN A <- len */ 1743 if (pc->code == (BPF_LD|BPF_W|BPF_LEN)) { 1744 status = sljit_emit_op1(compiler, 1745 SLJIT_MOV, /* size_t source */ 1746 BJ_AREG, 0, 1747 SLJIT_MEM1(BJ_ARGS), 1748 offsetof(struct bpf_args, wirelen)); 1749 if (status != SLJIT_SUCCESS) 1750 goto fail; 1751 1752 continue; 1753 } 1754 1755 mode = BPF_MODE(pc->code); 1756 if (mode != BPF_ABS && mode != BPF_IND) 1757 goto fail; 1758 1759 if (unconditional_ret) 1760 continue; 1761 1762 status = emit_pkt_read(compiler, pc, 1763 to_mchain_jump, &ret0, &ret0_size, &ret0_maxsize); 1764 if (status != SLJIT_SUCCESS) 1765 goto fail; 1766 1767 continue; 1768 1769 case BPF_LDX: 1770 mode = BPF_MODE(pc->code); 1771 1772 /* BPF_LDX+BPF_W+BPF_IMM X <- k */ 1773 if (mode == BPF_IMM) { 1774 if (BPF_SIZE(pc->code) != BPF_W) 1775 goto fail; 1776 status = sljit_emit_op1(compiler, 1777 SLJIT_MOV, 1778 BJ_XREG, 0, 1779 SLJIT_IMM, (uint32_t)pc->k); 1780 if (status != SLJIT_SUCCESS) 1781 goto fail; 1782 1783 continue; 1784 } 1785 1786 /* BPF_LDX+BPF_W+BPF_LEN X <- len */ 1787 if (mode == BPF_LEN) { 1788 if (BPF_SIZE(pc->code) != BPF_W) 1789 goto fail; 1790 status = sljit_emit_op1(compiler, 1791 SLJIT_MOV, /* size_t source */ 1792 BJ_XREG, 0, 1793 SLJIT_MEM1(BJ_ARGS), 1794 offsetof(struct bpf_args, wirelen)); 1795 if (status != SLJIT_SUCCESS) 1796 goto fail; 1797 1798 continue; 1799 } 1800 1801 /* BPF_LDX+BPF_W+BPF_MEM X <- M[k] */ 1802 if (mode == BPF_MEM) { 1803 if (BPF_SIZE(pc->code) != BPF_W) 1804 goto fail; 1805 if ((uint32_t)pc->k >= memwords) 1806 goto fail; 1807 status = emit_memload(compiler, 1808 BJ_XREG, pc->k, extwords); 1809 if (status != SLJIT_SUCCESS) 1810 goto fail; 1811 1812 continue; 1813 } 1814 1815 /* BPF_LDX+BPF_B+BPF_MSH X <- 4*(P[k:1]&0xf) */ 1816 if (mode != BPF_MSH || BPF_SIZE(pc->code) != BPF_B) 1817 goto fail; 1818 1819 if (unconditional_ret) 1820 continue; 1821 1822 status = emit_msh(compiler, pc, 1823 to_mchain_jump, &ret0, &ret0_size, &ret0_maxsize); 1824 if (status != SLJIT_SUCCESS) 1825 goto fail; 1826 1827 continue; 1828 1829 case BPF_ST: 1830 if (pc->code != BPF_ST || 1831 (uint32_t)pc->k >= memwords) { 1832 goto fail; 1833 } 1834 1835 status = emit_memstore(compiler, 1836 BJ_AREG, pc->k, extwords); 1837 if (status != SLJIT_SUCCESS) 1838 goto fail; 1839 1840 continue; 1841 1842 case BPF_STX: 1843 if (pc->code != BPF_STX || 1844 (uint32_t)pc->k >= memwords) { 1845 goto fail; 1846 } 1847 1848 status = emit_memstore(compiler, 1849 BJ_XREG, pc->k, extwords); 1850 if (status != SLJIT_SUCCESS) 1851 goto fail; 1852 1853 continue; 1854 1855 case BPF_ALU: 1856 if (pc->code == (BPF_ALU|BPF_NEG)) { 1857 status = sljit_emit_op1(compiler, 1858 SLJIT_NEG, 1859 BJ_AREG, 0, 1860 BJ_AREG, 0); 1861 if (status != SLJIT_SUCCESS) 1862 goto fail; 1863 1864 continue; 1865 } 1866 1867 if (BPF_OP(pc->code) != BPF_DIV) { 1868 status = sljit_emit_op2(compiler, 1869 bpf_alu_to_sljit_op(pc), 1870 BJ_AREG, 0, 1871 BJ_AREG, 0, 1872 kx_to_reg(pc), kx_to_reg_arg(pc)); 1873 if (status != SLJIT_SUCCESS) 1874 goto fail; 1875 1876 continue; 1877 } 1878 1879 /* BPF_DIV */ 1880 1881 src = BPF_SRC(pc->code); 1882 if (src != BPF_X && src != BPF_K) 1883 goto fail; 1884 1885 /* division by zero? */ 1886 if (src == BPF_X) { 1887 jump = sljit_emit_cmp(compiler, 1888 SLJIT_C_EQUAL|SLJIT_INT_OP, 1889 BJ_XREG, 0, 1890 SLJIT_IMM, 0); 1891 if (jump == NULL) 1892 goto fail; 1893 if (!append_jump(jump, &ret0, 1894 &ret0_size, &ret0_maxsize)) 1895 goto fail; 1896 } else if (pc->k == 0) { 1897 jump = sljit_emit_jump(compiler, SLJIT_JUMP); 1898 if (jump == NULL) 1899 goto fail; 1900 if (!append_jump(jump, &ret0, 1901 &ret0_size, &ret0_maxsize)) 1902 goto fail; 1903 } 1904 1905 if (src == BPF_X) { 1906 status = emit_division(compiler, BJ_XREG, 0); 1907 if (status != SLJIT_SUCCESS) 1908 goto fail; 1909 } else if (pc->k != 0) { 1910 if (pc->k & (pc->k - 1)) { 1911 status = emit_division(compiler, 1912 SLJIT_IMM, (uint32_t)pc->k); 1913 } else { 1914 status = emit_pow2_division(compiler, 1915 (uint32_t)pc->k); 1916 } 1917 if (status != SLJIT_SUCCESS) 1918 goto fail; 1919 } 1920 1921 continue; 1922 1923 case BPF_JMP: 1924 if (BPF_OP(pc->code) == BPF_JA) { 1925 jt = jf = pc->k; 1926 } else { 1927 jt = pc->jt; 1928 jf = pc->jf; 1929 } 1930 1931 negate = (jt == 0) ? 1 : 0; 1932 branching = (jt == jf) ? 0 : 1; 1933 jtf = insn_dat[i].u.jdata.jtf; 1934 1935 if (branching) { 1936 if (BPF_OP(pc->code) != BPF_JSET) { 1937 jump = sljit_emit_cmp(compiler, 1938 bpf_jmp_to_sljit_cond(pc, negate), 1939 BJ_AREG, 0, 1940 kx_to_reg(pc), kx_to_reg_arg(pc)); 1941 } else { 1942 status = sljit_emit_op2(compiler, 1943 SLJIT_AND, 1944 BJ_TMP1REG, 0, 1945 BJ_AREG, 0, 1946 kx_to_reg(pc), kx_to_reg_arg(pc)); 1947 if (status != SLJIT_SUCCESS) 1948 goto fail; 1949 1950 jump = sljit_emit_cmp(compiler, 1951 bpf_jmp_to_sljit_cond(pc, negate), 1952 BJ_TMP1REG, 0, 1953 SLJIT_IMM, 0); 1954 } 1955 1956 if (jump == NULL) 1957 goto fail; 1958 1959 BJ_ASSERT(jtf[negate].sjump == NULL); 1960 jtf[negate].sjump = jump; 1961 } 1962 1963 if (!branching || (jt != 0 && jf != 0)) { 1964 jump = sljit_emit_jump(compiler, SLJIT_JUMP); 1965 if (jump == NULL) 1966 goto fail; 1967 1968 BJ_ASSERT(jtf[branching].sjump == NULL); 1969 jtf[branching].sjump = jump; 1970 } 1971 1972 continue; 1973 1974 case BPF_RET: 1975 rval = BPF_RVAL(pc->code); 1976 if (rval == BPF_X) 1977 goto fail; 1978 1979 /* BPF_RET+BPF_K accept k bytes */ 1980 if (rval == BPF_K) { 1981 status = sljit_emit_return(compiler, 1982 SLJIT_MOV_UI, 1983 SLJIT_IMM, (uint32_t)pc->k); 1984 if (status != SLJIT_SUCCESS) 1985 goto fail; 1986 } 1987 1988 /* BPF_RET+BPF_A accept A bytes */ 1989 if (rval == BPF_A) { 1990 status = sljit_emit_return(compiler, 1991 SLJIT_MOV_UI, 1992 BJ_AREG, 0); 1993 if (status != SLJIT_SUCCESS) 1994 goto fail; 1995 } 1996 1997 continue; 1998 1999 case BPF_MISC: 2000 switch (BPF_MISCOP(pc->code)) { 2001 case BPF_TAX: 2002 status = sljit_emit_op1(compiler, 2003 SLJIT_MOV_UI, 2004 BJ_XREG, 0, 2005 BJ_AREG, 0); 2006 if (status != SLJIT_SUCCESS) 2007 goto fail; 2008 2009 continue; 2010 2011 case BPF_TXA: 2012 status = sljit_emit_op1(compiler, 2013 SLJIT_MOV, 2014 BJ_AREG, 0, 2015 BJ_XREG, 0); 2016 if (status != SLJIT_SUCCESS) 2017 goto fail; 2018 2019 continue; 2020 2021 case BPF_COP: 2022 case BPF_COPX: 2023 if (bc == NULL || bc->copfuncs == NULL) 2024 goto fail; 2025 if (BPF_MISCOP(pc->code) == BPF_COP && 2026 (uint32_t)pc->k >= bc->nfuncs) { 2027 goto fail; 2028 } 2029 2030 jump = NULL; 2031 status = emit_cop(compiler, bc, pc, &jump); 2032 if (status != SLJIT_SUCCESS) 2033 goto fail; 2034 2035 if (jump != NULL && !append_jump(jump, 2036 &ret0, &ret0_size, &ret0_maxsize)) 2037 goto fail; 2038 2039 continue; 2040 } 2041 2042 goto fail; 2043 } /* switch */ 2044 } /* main loop */ 2045 2046 BJ_ASSERT(ret0_size <= ret0_maxsize); 2047 2048 if (ret0_size > 0) { 2049 label = sljit_emit_label(compiler); 2050 if (label == NULL) 2051 goto fail; 2052 for (i = 0; i < ret0_size; i++) 2053 sljit_set_label(ret0[i], label); 2054 } 2055 2056 status = sljit_emit_return(compiler, 2057 SLJIT_MOV_UI, 2058 SLJIT_IMM, 0); 2059 if (status != SLJIT_SUCCESS) 2060 goto fail; 2061 2062 rv = true; 2063 2064 fail: 2065 if (ret0 != NULL) 2066 BJ_FREE(ret0, ret0_maxsize * sizeof(ret0[0])); 2067 2068 return rv; 2069 } 2070 2071 bpfjit_func_t 2072 bpfjit_generate_code(const bpf_ctx_t *bc, 2073 const struct bpf_insn *insns, size_t insn_count) 2074 { 2075 void *rv; 2076 struct sljit_compiler *compiler; 2077 2078 size_t i; 2079 int status; 2080 2081 /* optimization related */ 2082 bpf_memword_init_t initmask; 2083 bpfjit_hint_t hints; 2084 2085 /* memory store location for initial zero initialization */ 2086 sljit_si mem_reg; 2087 sljit_sw mem_off; 2088 2089 struct bpfjit_insn_data *insn_dat; 2090 2091 const size_t extwords = GET_EXTWORDS(bc); 2092 const size_t memwords = GET_MEMWORDS(bc); 2093 const bpf_memword_init_t preinited = extwords ? bc->preinited : 0; 2094 2095 rv = NULL; 2096 compiler = NULL; 2097 insn_dat = NULL; 2098 2099 if (memwords > MAX_MEMWORDS) 2100 goto fail; 2101 2102 if (insn_count == 0 || insn_count > SIZE_MAX / sizeof(insn_dat[0])) 2103 goto fail; 2104 2105 insn_dat = BJ_ALLOC(insn_count * sizeof(insn_dat[0])); 2106 if (insn_dat == NULL) 2107 goto fail; 2108 2109 if (!optimize(bc, insns, insn_dat, insn_count, &initmask, &hints)) 2110 goto fail; 2111 2112 compiler = sljit_create_compiler(); 2113 if (compiler == NULL) 2114 goto fail; 2115 2116 #if !defined(_KERNEL) && defined(SLJIT_VERBOSE) && SLJIT_VERBOSE 2117 sljit_compiler_verbose(compiler, stderr); 2118 #endif 2119 2120 status = sljit_emit_enter(compiler, 2121 2, nscratches(hints), 3, sizeof(struct bpfjit_stack)); 2122 if (status != SLJIT_SUCCESS) 2123 goto fail; 2124 2125 if (hints & BJ_HINT_COP) { 2126 /* save ctx argument */ 2127 status = sljit_emit_op1(compiler, 2128 SLJIT_MOV_P, 2129 SLJIT_MEM1(SLJIT_LOCALS_REG), 2130 offsetof(struct bpfjit_stack, ctx), 2131 BJ_CTX_ARG, 0); 2132 if (status != SLJIT_SUCCESS) 2133 goto fail; 2134 } 2135 2136 if (extwords == 0) { 2137 mem_reg = SLJIT_MEM1(SLJIT_LOCALS_REG); 2138 mem_off = offsetof(struct bpfjit_stack, mem); 2139 } else { 2140 /* copy "mem" argument from bpf_args to bpfjit_stack */ 2141 status = sljit_emit_op1(compiler, 2142 SLJIT_MOV_P, 2143 BJ_TMP1REG, 0, 2144 SLJIT_MEM1(BJ_ARGS), offsetof(struct bpf_args, mem)); 2145 if (status != SLJIT_SUCCESS) 2146 goto fail; 2147 2148 status = sljit_emit_op1(compiler, 2149 SLJIT_MOV_P, 2150 SLJIT_MEM1(SLJIT_LOCALS_REG), 2151 offsetof(struct bpfjit_stack, extmem), 2152 BJ_TMP1REG, 0); 2153 if (status != SLJIT_SUCCESS) 2154 goto fail; 2155 2156 mem_reg = SLJIT_MEM1(BJ_TMP1REG); 2157 mem_off = 0; 2158 } 2159 2160 /* 2161 * Exclude pre-initialised external memory words but keep 2162 * initialization statuses of A and X registers in case 2163 * bc->preinited wrongly sets those two bits. 2164 */ 2165 initmask &= ~preinited | BJ_INIT_ABIT | BJ_INIT_XBIT; 2166 2167 #if defined(_KERNEL) 2168 /* bpf_filter() checks initialization of memwords. */ 2169 BJ_ASSERT((initmask & (BJ_INIT_MBIT(memwords) - 1)) == 0); 2170 #endif 2171 for (i = 0; i < memwords; i++) { 2172 if (initmask & BJ_INIT_MBIT(i)) { 2173 /* M[i] = 0; */ 2174 status = sljit_emit_op1(compiler, 2175 SLJIT_MOV_UI, 2176 mem_reg, mem_off + i * sizeof(uint32_t), 2177 SLJIT_IMM, 0); 2178 if (status != SLJIT_SUCCESS) 2179 goto fail; 2180 } 2181 } 2182 2183 if (initmask & BJ_INIT_ABIT) { 2184 /* A = 0; */ 2185 status = sljit_emit_op1(compiler, 2186 SLJIT_MOV, 2187 BJ_AREG, 0, 2188 SLJIT_IMM, 0); 2189 if (status != SLJIT_SUCCESS) 2190 goto fail; 2191 } 2192 2193 if (initmask & BJ_INIT_XBIT) { 2194 /* X = 0; */ 2195 status = sljit_emit_op1(compiler, 2196 SLJIT_MOV, 2197 BJ_XREG, 0, 2198 SLJIT_IMM, 0); 2199 if (status != SLJIT_SUCCESS) 2200 goto fail; 2201 } 2202 2203 status = load_buf_buflen(compiler); 2204 if (status != SLJIT_SUCCESS) 2205 goto fail; 2206 2207 if (!generate_insn_code(compiler, bc, insns, insn_dat, insn_count)) 2208 goto fail; 2209 2210 rv = sljit_generate_code(compiler); 2211 2212 fail: 2213 if (compiler != NULL) 2214 sljit_free_compiler(compiler); 2215 2216 if (insn_dat != NULL) 2217 BJ_FREE(insn_dat, insn_count * sizeof(insn_dat[0])); 2218 2219 return (bpfjit_func_t)rv; 2220 } 2221 2222 void 2223 bpfjit_free_code(bpfjit_func_t code) 2224 { 2225 2226 sljit_free_code((void *)code); 2227 } 2228
Indexes created Sat Sep 27 14:09:57 GMT 2025