1 /* 2 * Copyright 2019 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 */ 23 24 #ifndef GEN_MI_BUILDER_H 25 #define GEN_MI_BUILDER_H 26 27 #include "util/bitscan.h" 28 #include "util/fast_idiv_by_const.h" 29 #include "util/u_math.h" 30 31 #ifndef GEN_MI_BUILDER_NUM_ALLOC_GPRS 32 /** The number of GPRs the MI builder is allowed to allocate 33 * 34 * This may be set by a user of this API so that it can reserve some GPRs at 35 * the top end for its own use. 36 */ 37 #define GEN_MI_BUILDER_NUM_ALLOC_GPRS 16 38 #endif 39 40 /** These must be defined by the user of the builder 41 * 42 * void *__gen_get_batch_dwords(__gen_user_data *user_data, 43 * unsigned num_dwords); 44 * 45 * __gen_address_type 46 * __gen_address_offset(__gen_address_type addr, uint64_t offset); 47 * 48 */ 49 50 /* 51 * Start of the actual MI builder 52 */ 53 54 #define __genxml_cmd_length(cmd) cmd ## _length 55 #define __genxml_cmd_header(cmd) cmd ## _header 56 #define __genxml_cmd_pack(cmd) cmd ## _pack 57 58 #define gen_mi_builder_pack(b, cmd, dst, name) \ 59 for (struct cmd name = { __genxml_cmd_header(cmd) }, \ 60 *_dst = (struct cmd *)(dst); __builtin_expect(_dst != NULL, 1); \ 61 __genxml_cmd_pack(cmd)((b)->user_data, (void *)_dst, &name), \ 62 _dst = NULL) 63 64 #define gen_mi_builder_emit(b, cmd, name) \ 65 gen_mi_builder_pack((b), cmd, __gen_get_batch_dwords((b)->user_data, __genxml_cmd_length(cmd)), name) 66 67 68 enum gen_mi_value_type { 69 GEN_MI_VALUE_TYPE_IMM, 70 GEN_MI_VALUE_TYPE_MEM32, 71 GEN_MI_VALUE_TYPE_MEM64, 72 GEN_MI_VALUE_TYPE_REG32, 73 GEN_MI_VALUE_TYPE_REG64, 74 }; 75 76 struct gen_mi_value { 77 enum gen_mi_value_type type; 78 79 union { 80 uint64_t imm; 81 __gen_address_type addr; 82 uint32_t reg; 83 }; 84 85 #if GEN_GEN >= 7 || GEN_IS_HASWELL 86 bool invert; 87 #endif 88 }; 89 90 #if GEN_GEN >= 9 91 #define GEN_MI_BUILDER_MAX_MATH_DWORDS 256 92 #else 93 #define GEN_MI_BUILDER_MAX_MATH_DWORDS 64 94 #endif 95 96 struct gen_mi_builder { 97 __gen_user_data *user_data; 98 99 #if GEN_GEN >= 8 || GEN_IS_HASWELL 100 uint32_t gprs; 101 uint8_t gpr_refs[GEN_MI_BUILDER_NUM_ALLOC_GPRS]; 102 103 unsigned num_math_dwords; 104 uint32_t math_dwords[GEN_MI_BUILDER_MAX_MATH_DWORDS]; 105 #endif 106 }; 107 108 static inline void 109 gen_mi_builder_init(struct gen_mi_builder *b, __gen_user_data *user_data) 110 { 111 memset(b, 0, sizeof(*b)); 112 b->user_data = user_data; 113 114 #if GEN_GEN >= 8 || GEN_IS_HASWELL 115 b->gprs = 0; 116 b->num_math_dwords = 0; 117 #endif 118 } 119 120 static inline void 121 gen_mi_builder_flush_math(struct gen_mi_builder *b) 122 { 123 #if GEN_GEN >= 8 || GEN_IS_HASWELL 124 if (b->num_math_dwords == 0) 125 return; 126 127 uint32_t *dw = (uint32_t *)__gen_get_batch_dwords(b->user_data, 128 1 + b->num_math_dwords); 129 gen_mi_builder_pack(b, GENX(MI_MATH), dw, math) { 130 math.DWordLength = 1 + b->num_math_dwords - GENX(MI_MATH_length_bias); 131 } 132 memcpy(dw + 1, b->math_dwords, b->num_math_dwords * sizeof(uint32_t)); 133 b->num_math_dwords = 0; 134 #endif 135 } 136 137 #define _GEN_MI_BUILDER_GPR_BASE 0x2600 138 /* The actual hardware limit on GPRs */ 139 #define _GEN_MI_BUILDER_NUM_HW_GPRS 16 140 141 #if GEN_GEN >= 8 || GEN_IS_HASWELL 142 143 static inline bool 144 gen_mi_value_is_gpr(struct gen_mi_value val) 145 { 146 return (val.type == GEN_MI_VALUE_TYPE_REG32 || 147 val.type == GEN_MI_VALUE_TYPE_REG64) && 148 val.reg >= _GEN_MI_BUILDER_GPR_BASE && 149 val.reg < _GEN_MI_BUILDER_GPR_BASE + 150 _GEN_MI_BUILDER_NUM_HW_GPRS * 8; 151 } 152 153 static inline bool 154 _gen_mi_value_is_allocated_gpr(struct gen_mi_value val) 155 { 156 return (val.type == GEN_MI_VALUE_TYPE_REG32 || 157 val.type == GEN_MI_VALUE_TYPE_REG64) && 158 val.reg >= _GEN_MI_BUILDER_GPR_BASE && 159 val.reg < _GEN_MI_BUILDER_GPR_BASE + 160 GEN_MI_BUILDER_NUM_ALLOC_GPRS * 8; 161 } 162 163 static inline uint32_t 164 _gen_mi_value_as_gpr(struct gen_mi_value val) 165 { 166 assert(gen_mi_value_is_gpr(val)); 167 assert(val.reg % 8 == 0); 168 return (val.reg - _GEN_MI_BUILDER_GPR_BASE) / 8; 169 } 170 171 static inline struct gen_mi_value 172 gen_mi_new_gpr(struct gen_mi_builder *b) 173 { 174 unsigned gpr = ffs(~b->gprs) - 1; 175 assert(gpr < GEN_MI_BUILDER_NUM_ALLOC_GPRS); 176 assert(b->gpr_refs[gpr] == 0); 177 b->gprs |= (1u << gpr); 178 b->gpr_refs[gpr] = 1; 179 180 return (struct gen_mi_value) { 181 .type = GEN_MI_VALUE_TYPE_REG64, 182 .reg = _GEN_MI_BUILDER_GPR_BASE + gpr * 8, 183 }; 184 } 185 #endif /* GEN_GEN >= 8 || GEN_IS_HASWELL */ 186 187 /** Take a reference to a gen_mi_value 188 * 189 * The MI builder uses reference counting to automatically free ALU GPRs for 190 * re-use in calculations. All gen_mi_* math functions consume the reference 191 * they are handed for each source and return a reference to a value which the 192 * caller must consume. In particular, if you pas the same value into a 193 * single gen_mi_* math function twice (say to add a number to itself), you 194 * are responsible for calling gen_mi_value_ref() to get a second reference 195 * because the gen_mi_* math function will consume it twice. 196 */ 197 static inline struct gen_mi_value 198 gen_mi_value_ref(struct gen_mi_builder *b, struct gen_mi_value val) 199 { 200 #if GEN_GEN >= 8 || GEN_IS_HASWELL 201 if (_gen_mi_value_is_allocated_gpr(val)) { 202 unsigned gpr = _gen_mi_value_as_gpr(val); 203 assert(gpr < GEN_MI_BUILDER_NUM_ALLOC_GPRS); 204 assert(b->gprs & (1u << gpr)); 205 assert(b->gpr_refs[gpr] < UINT8_MAX); 206 b->gpr_refs[gpr]++; 207 } 208 #endif /* GEN_GEN >= 8 || GEN_IS_HASWELL */ 209 210 return val; 211 } 212 213 /** Drop a reference to a gen_mi_value 214 * 215 * See also gen_mi_value_ref. 216 */ 217 static inline void 218 gen_mi_value_unref(struct gen_mi_builder *b, struct gen_mi_value val) 219 { 220 #if GEN_GEN >= 8 || GEN_IS_HASWELL 221 if (_gen_mi_value_is_allocated_gpr(val)) { 222 unsigned gpr = _gen_mi_value_as_gpr(val); 223 assert(gpr < GEN_MI_BUILDER_NUM_ALLOC_GPRS); 224 assert(b->gprs & (1u << gpr)); 225 assert(b->gpr_refs[gpr] > 0); 226 if (--b->gpr_refs[gpr] == 0) 227 b->gprs &= ~(1u << gpr); 228 } 229 #endif /* GEN_GEN >= 8 || GEN_IS_HASWELL */ 230 } 231 232 static inline struct gen_mi_value 233 gen_mi_imm(uint64_t imm) 234 { 235 return (struct gen_mi_value) { 236 .type = GEN_MI_VALUE_TYPE_IMM, 237 .imm = imm, 238 }; 239 } 240 241 static inline struct gen_mi_value 242 gen_mi_reg32(uint32_t reg) 243 { 244 struct gen_mi_value val = { 245 .type = GEN_MI_VALUE_TYPE_REG32, 246 .reg = reg, 247 }; 248 #if GEN_GEN >= 8 || GEN_IS_HASWELL 249 assert(!_gen_mi_value_is_allocated_gpr(val)); 250 #endif 251 return val; 252 } 253 254 static inline struct gen_mi_value 255 gen_mi_reg64(uint32_t reg) 256 { 257 struct gen_mi_value val = { 258 .type = GEN_MI_VALUE_TYPE_REG64, 259 .reg = reg, 260 }; 261 #if GEN_GEN >= 8 || GEN_IS_HASWELL 262 assert(!_gen_mi_value_is_allocated_gpr(val)); 263 #endif 264 return val; 265 } 266 267 static inline struct gen_mi_value 268 gen_mi_mem32(__gen_address_type addr) 269 { 270 return (struct gen_mi_value) { 271 .type = GEN_MI_VALUE_TYPE_MEM32, 272 .addr = addr, 273 }; 274 } 275 276 static inline struct gen_mi_value 277 gen_mi_mem64(__gen_address_type addr) 278 { 279 return (struct gen_mi_value) { 280 .type = GEN_MI_VALUE_TYPE_MEM64, 281 .addr = addr, 282 }; 283 } 284 285 static inline struct gen_mi_value 286 gen_mi_value_half(struct gen_mi_value value, bool top_32_bits) 287 { 288 switch (value.type) { 289 case GEN_MI_VALUE_TYPE_IMM: 290 if (top_32_bits) 291 value.imm >>= 32; 292 else 293 value.imm &= 0xffffffffu; 294 return value; 295 296 case GEN_MI_VALUE_TYPE_MEM32: 297 assert(!top_32_bits); 298 return value; 299 300 case GEN_MI_VALUE_TYPE_MEM64: 301 if (top_32_bits) 302 value.addr = __gen_address_offset(value.addr, 4); 303 value.type = GEN_MI_VALUE_TYPE_MEM32; 304 return value; 305 306 case GEN_MI_VALUE_TYPE_REG32: 307 assert(!top_32_bits); 308 return value; 309 310 case GEN_MI_VALUE_TYPE_REG64: 311 if (top_32_bits) 312 value.reg += 4; 313 value.type = GEN_MI_VALUE_TYPE_REG32; 314 return value; 315 } 316 317 unreachable("Invalid gen_mi_value type"); 318 } 319 320 static inline void 321 _gen_mi_copy_no_unref(struct gen_mi_builder *b, 322 struct gen_mi_value dst, struct gen_mi_value src) 323 { 324 #if GEN_GEN >= 7 || GEN_IS_HASWELL 325 /* TODO: We could handle src.invert by emitting a bit of math if we really 326 * wanted to. 327 */ 328 assert(!dst.invert && !src.invert); 329 #endif 330 gen_mi_builder_flush_math(b); 331 332 switch (dst.type) { 333 case GEN_MI_VALUE_TYPE_IMM: 334 unreachable("Cannot copy to an immediate"); 335 336 case GEN_MI_VALUE_TYPE_MEM64: 337 case GEN_MI_VALUE_TYPE_REG64: 338 /* If the destination is 64 bits, we have to copy in two halves */ 339 _gen_mi_copy_no_unref(b, gen_mi_value_half(dst, false), 340 gen_mi_value_half(src, false)); 341 switch (src.type) { 342 case GEN_MI_VALUE_TYPE_IMM: 343 case GEN_MI_VALUE_TYPE_MEM64: 344 case GEN_MI_VALUE_TYPE_REG64: 345 /* TODO: Use MI_STORE_DATA_IMM::StoreQWord when we have it */ 346 _gen_mi_copy_no_unref(b, gen_mi_value_half(dst, true), 347 gen_mi_value_half(src, true)); 348 break; 349 default: 350 _gen_mi_copy_no_unref(b, gen_mi_value_half(dst, true), 351 gen_mi_imm(0)); 352 break; 353 } 354 break; 355 356 case GEN_MI_VALUE_TYPE_MEM32: 357 switch (src.type) { 358 case GEN_MI_VALUE_TYPE_IMM: 359 gen_mi_builder_emit(b, GENX(MI_STORE_DATA_IMM), sdi) { 360 sdi.Address = dst.addr; 361 sdi.ImmediateData = src.imm; 362 } 363 break; 364 365 case GEN_MI_VALUE_TYPE_MEM32: 366 case GEN_MI_VALUE_TYPE_MEM64: 367 #if GEN_GEN >= 8 368 gen_mi_builder_emit(b, GENX(MI_COPY_MEM_MEM), cmm) { 369 cmm.DestinationMemoryAddress = dst.addr; 370 cmm.SourceMemoryAddress = src.addr; 371 } 372 #elif GEN_IS_HASWELL 373 { 374 struct gen_mi_value tmp = gen_mi_new_gpr(b); 375 _gen_mi_copy_no_unref(b, tmp, src); 376 _gen_mi_copy_no_unref(b, dst, tmp); 377 gen_mi_value_unref(b, tmp); 378 } 379 #else 380 unreachable("Cannot do mem <-> mem copy on IVB and earlier"); 381 #endif 382 break; 383 384 case GEN_MI_VALUE_TYPE_REG32: 385 case GEN_MI_VALUE_TYPE_REG64: 386 gen_mi_builder_emit(b, GENX(MI_STORE_REGISTER_MEM), srm) { 387 srm.RegisterAddress = src.reg; 388 srm.MemoryAddress = dst.addr; 389 } 390 break; 391 392 default: 393 unreachable("Invalid gen_mi_value type"); 394 } 395 break; 396 397 case GEN_MI_VALUE_TYPE_REG32: 398 switch (src.type) { 399 case GEN_MI_VALUE_TYPE_IMM: 400 gen_mi_builder_emit(b, GENX(MI_LOAD_REGISTER_IMM), lri) { 401 lri.RegisterOffset = dst.reg; 402 lri.DataDWord = src.imm; 403 } 404 break; 405 406 case GEN_MI_VALUE_TYPE_MEM32: 407 case GEN_MI_VALUE_TYPE_MEM64: 408 gen_mi_builder_emit(b, GENX(MI_LOAD_REGISTER_MEM), lrm) { 409 lrm.RegisterAddress = dst.reg; 410 lrm.MemoryAddress = src.addr; 411 } 412 break; 413 414 case GEN_MI_VALUE_TYPE_REG32: 415 case GEN_MI_VALUE_TYPE_REG64: 416 #if GEN_GEN >= 8 || GEN_IS_HASWELL 417 gen_mi_builder_emit(b, GENX(MI_LOAD_REGISTER_REG), lrr) { 418 lrr.SourceRegisterAddress = src.reg; 419 lrr.DestinationRegisterAddress = dst.reg; 420 } 421 #else 422 unreachable("Cannot do reg <-> reg copy on IVB and earlier"); 423 #endif 424 break; 425 426 default: 427 unreachable("Invalid gen_mi_value type"); 428 } 429 break; 430 431 default: 432 unreachable("Invalid gen_mi_value type"); 433 } 434 } 435 436 /** Store the value in src to the value represented by dst 437 * 438 * If the bit size of src and dst mismatch, this function does an unsigned 439 * integer cast. If src has more bits than dst, it takes the bottom bits. If 440 * src has fewer bits then dst, it fills the top bits with zeros. 441 * 442 * This function consumes one reference for each of src and dst. 443 */ 444 static inline void 445 gen_mi_store(struct gen_mi_builder *b, 446 struct gen_mi_value dst, struct gen_mi_value src) 447 { 448 _gen_mi_copy_no_unref(b, dst, src); 449 gen_mi_value_unref(b, src); 450 gen_mi_value_unref(b, dst); 451 } 452 453 static inline void 454 gen_mi_memset(struct gen_mi_builder *b, __gen_address_type dst, 455 uint32_t value, uint32_t size) 456 { 457 #if GEN_GEN >= 8 || GEN_IS_HASWELL 458 assert(b->num_math_dwords == 0); 459 #endif 460 461 /* This memset operates in units of dwords. */ 462 assert(size % 4 == 0); 463 464 for (uint32_t i = 0; i < size; i += 4) { 465 gen_mi_store(b, gen_mi_mem32(__gen_address_offset(dst, i)), 466 gen_mi_imm(value)); 467 } 468 } 469 470 /* NOTE: On IVB, this function stomps GEN7_3DPRIM_BASE_VERTEX */ 471 static inline void 472 gen_mi_memcpy(struct gen_mi_builder *b, __gen_address_type dst, 473 __gen_address_type src, uint32_t size) 474 { 475 #if GEN_GEN >= 8 || GEN_IS_HASWELL 476 assert(b->num_math_dwords == 0); 477 #endif 478 479 /* This memcpy operates in units of dwords. */ 480 assert(size % 4 == 0); 481 482 for (uint32_t i = 0; i < size; i += 4) { 483 struct gen_mi_value dst_val = gen_mi_mem32(__gen_address_offset(dst, i)); 484 struct gen_mi_value src_val = gen_mi_mem32(__gen_address_offset(src, i)); 485 #if GEN_GEN >= 8 || GEN_IS_HASWELL 486 gen_mi_store(b, dst_val, src_val); 487 #else 488 /* IVB does not have a general purpose register for command streamer 489 * commands. Therefore, we use an alternate temporary register. 490 */ 491 struct gen_mi_value tmp_reg = gen_mi_reg32(0x2440); /* GEN7_3DPRIM_BASE_VERTEX */ 492 gen_mi_store(b, tmp_reg, src_val); 493 gen_mi_store(b, dst_val, tmp_reg); 494 #endif 495 } 496 } 497 498 /* 499 * MI_MATH Section. Only available on Haswell+ 500 */ 501 502 #if GEN_GEN >= 8 || GEN_IS_HASWELL 503 504 static inline void 505 _gen_mi_builder_push_math(struct gen_mi_builder *b, 506 const uint32_t *dwords, 507 unsigned num_dwords) 508 { 509 assert(num_dwords < GEN_MI_BUILDER_MAX_MATH_DWORDS); 510 if (b->num_math_dwords + num_dwords > GEN_MI_BUILDER_MAX_MATH_DWORDS) 511 gen_mi_builder_flush_math(b); 512 513 memcpy(&b->math_dwords[b->num_math_dwords], 514 dwords, num_dwords * sizeof(*dwords)); 515 b->num_math_dwords += num_dwords; 516 } 517 518 static inline uint32_t 519 _gen_mi_pack_alu(uint32_t opcode, uint32_t operand1, uint32_t operand2) 520 { 521 struct GENX(MI_MATH_ALU_INSTRUCTION) instr = { 522 .Operand2 = operand2, 523 .Operand1 = operand1, 524 .ALUOpcode = opcode, 525 }; 526 527 uint32_t dw; 528 GENX(MI_MATH_ALU_INSTRUCTION_pack)(NULL, &dw, &instr); 529 530 return dw; 531 } 532 533 static inline struct gen_mi_value 534 gen_mi_value_to_gpr(struct gen_mi_builder *b, struct gen_mi_value val) 535 { 536 if (gen_mi_value_is_gpr(val)) 537 return val; 538 539 /* Save off the invert flag because it makes copy() grumpy */ 540 bool invert = val.invert; 541 val.invert = false; 542 543 struct gen_mi_value tmp = gen_mi_new_gpr(b); 544 _gen_mi_copy_no_unref(b, tmp, val); 545 tmp.invert = invert; 546 547 return tmp; 548 } 549 550 static inline uint32_t 551 _gen_mi_math_load_src(struct gen_mi_builder *b, 552 unsigned src, struct gen_mi_value *val) 553 { 554 if (val->type == GEN_MI_VALUE_TYPE_IMM && 555 (val->imm == 0 || val->imm == UINT64_MAX)) { 556 uint64_t imm = val->invert ? ~val->imm : val->imm; 557 return _gen_mi_pack_alu(imm ? MI_ALU_LOAD1 : MI_ALU_LOAD0, src, 0); 558 } else { 559 *val = gen_mi_value_to_gpr(b, *val); 560 return _gen_mi_pack_alu(val->invert ? MI_ALU_LOADINV : MI_ALU_LOAD, 561 src, _gen_mi_value_as_gpr(*val)); 562 } 563 } 564 565 static inline struct gen_mi_value 566 gen_mi_math_binop(struct gen_mi_builder *b, uint32_t opcode, 567 struct gen_mi_value src0, struct gen_mi_value src1, 568 uint32_t store_op, uint32_t store_src) 569 { 570 struct gen_mi_value dst = gen_mi_new_gpr(b); 571 572 uint32_t dw[4]; 573 dw[0] = _gen_mi_math_load_src(b, MI_ALU_SRCA, &src0); 574 dw[1] = _gen_mi_math_load_src(b, MI_ALU_SRCB, &src1); 575 dw[2] = _gen_mi_pack_alu(opcode, 0, 0); 576 dw[3] = _gen_mi_pack_alu(store_op, _gen_mi_value_as_gpr(dst), store_src); 577 _gen_mi_builder_push_math(b, dw, 4); 578 579 gen_mi_value_unref(b, src0); 580 gen_mi_value_unref(b, src1); 581 582 return dst; 583 } 584 585 static inline struct gen_mi_value 586 gen_mi_inot(struct gen_mi_builder *b, struct gen_mi_value val) 587 { 588 /* TODO These currently can't be passed into gen_mi_copy */ 589 val.invert = !val.invert; 590 return val; 591 } 592 593 static inline struct gen_mi_value 594 gen_mi_iadd(struct gen_mi_builder *b, 595 struct gen_mi_value src0, struct gen_mi_value src1) 596 { 597 return gen_mi_math_binop(b, MI_ALU_ADD, src0, src1, 598 MI_ALU_STORE, MI_ALU_ACCU); 599 } 600 601 static inline struct gen_mi_value 602 gen_mi_iadd_imm(struct gen_mi_builder *b, 603 struct gen_mi_value src, uint64_t N) 604 { 605 if (N == 0) 606 return src; 607 608 return gen_mi_iadd(b, src, gen_mi_imm(N)); 609 } 610 611 static inline struct gen_mi_value 612 gen_mi_isub(struct gen_mi_builder *b, 613 struct gen_mi_value src0, struct gen_mi_value src1) 614 { 615 return gen_mi_math_binop(b, MI_ALU_SUB, src0, src1, 616 MI_ALU_STORE, MI_ALU_ACCU); 617 } 618 619 static inline struct gen_mi_value 620 gen_mi_ult(struct gen_mi_builder *b, 621 struct gen_mi_value src0, struct gen_mi_value src1) 622 { 623 /* Compute "less than" by subtracting and storing the carry bit */ 624 return gen_mi_math_binop(b, MI_ALU_SUB, src0, src1, 625 MI_ALU_STORE, MI_ALU_CF); 626 } 627 628 static inline struct gen_mi_value 629 gen_mi_uge(struct gen_mi_builder *b, 630 struct gen_mi_value src0, struct gen_mi_value src1) 631 { 632 /* Compute "less than" by subtracting and storing the carry bit */ 633 return gen_mi_math_binop(b, MI_ALU_SUB, src0, src1, 634 MI_ALU_STOREINV, MI_ALU_CF); 635 } 636 637 static inline struct gen_mi_value 638 gen_mi_iand(struct gen_mi_builder *b, 639 struct gen_mi_value src0, struct gen_mi_value src1) 640 { 641 return gen_mi_math_binop(b, MI_ALU_AND, src0, src1, 642 MI_ALU_STORE, MI_ALU_ACCU); 643 } 644 645 static inline struct gen_mi_value 646 gen_mi_imul_imm(struct gen_mi_builder *b, 647 struct gen_mi_value src, uint32_t N) 648 { 649 if (N == 0) { 650 gen_mi_value_unref(b, src); 651 return gen_mi_imm(0); 652 } 653 654 if (N == 1) 655 return src; 656 657 src = gen_mi_value_to_gpr(b, src); 658 659 struct gen_mi_value res = gen_mi_value_ref(b, src); 660 661 unsigned top_bit = 31 - __builtin_clz(N); 662 for (int i = top_bit - 1; i >= 0; i--) { 663 res = gen_mi_iadd(b, res, gen_mi_value_ref(b, res)); 664 if (N & (1 << i)) 665 res = gen_mi_iadd(b, res, gen_mi_value_ref(b, src)); 666 } 667 668 gen_mi_value_unref(b, src); 669 670 return res; 671 } 672 673 static inline struct gen_mi_value 674 gen_mi_ishl_imm(struct gen_mi_builder *b, 675 struct gen_mi_value src, uint32_t shift) 676 { 677 struct gen_mi_value res = gen_mi_value_to_gpr(b, src); 678 679 for (unsigned i = 0; i < shift; i++) 680 res = gen_mi_iadd(b, res, gen_mi_value_ref(b, res)); 681 682 return res; 683 } 684 685 static inline struct gen_mi_value 686 gen_mi_ushr32_imm(struct gen_mi_builder *b, 687 struct gen_mi_value src, uint32_t shift) 688 { 689 /* We right-shift by left-shifting by 32 - shift and taking the top 32 bits 690 * of the result. This assumes the top 32 bits are zero. 691 */ 692 if (shift > 64) 693 return gen_mi_imm(0); 694 695 if (shift > 32) { 696 struct gen_mi_value tmp = gen_mi_new_gpr(b); 697 _gen_mi_copy_no_unref(b, gen_mi_value_half(tmp, false), 698 gen_mi_value_half(src, true)); 699 _gen_mi_copy_no_unref(b, gen_mi_value_half(tmp, true), gen_mi_imm(0)); 700 gen_mi_value_unref(b, src); 701 src = tmp; 702 shift -= 32; 703 } 704 assert(shift <= 32); 705 struct gen_mi_value tmp = gen_mi_ishl_imm(b, src, 32 - shift); 706 struct gen_mi_value dst = gen_mi_new_gpr(b); 707 _gen_mi_copy_no_unref(b, gen_mi_value_half(dst, false), 708 gen_mi_value_half(tmp, true)); 709 _gen_mi_copy_no_unref(b, gen_mi_value_half(dst, true), gen_mi_imm(0)); 710 gen_mi_value_unref(b, tmp); 711 return dst; 712 } 713 714 static inline struct gen_mi_value 715 gen_mi_udiv32_imm(struct gen_mi_builder *b, 716 struct gen_mi_value N, uint32_t D) 717 { 718 /* We implicitly assume that N is only a 32-bit value */ 719 if (D == 0) { 720 /* This is invalid but we should do something */ 721 return gen_mi_imm(0); 722 } else if (util_is_power_of_two_or_zero(D)) { 723 return gen_mi_ushr32_imm(b, N, util_logbase2(D)); 724 } else { 725 struct util_fast_udiv_info m = util_compute_fast_udiv_info(D, 32, 32); 726 assert(m.multiplier <= UINT32_MAX); 727 728 if (m.pre_shift) 729 N = gen_mi_ushr32_imm(b, N, m.pre_shift); 730 731 /* Do the 32x32 multiply into gpr0 */ 732 N = gen_mi_imul_imm(b, N, m.multiplier); 733 734 if (m.increment) 735 N = gen_mi_iadd(b, N, gen_mi_imm(m.multiplier)); 736 737 N = gen_mi_ushr32_imm(b, N, 32); 738 739 if (m.post_shift) 740 N = gen_mi_ushr32_imm(b, N, m.post_shift); 741 742 return N; 743 } 744 } 745 746 #endif /* MI_MATH section */ 747 748 #endif /* GEN_MI_BUILDER_H */ 749
Indexes created Thu Jul 23 00:25:05 UTC 2026