1/* 2 * Copyright © 2019 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 */ 23 24#ifndef GEN_MI_BUILDER_H 25#define GEN_MI_BUILDER_H 26 27#include "util/bitscan.h" 28#include "util/fast_idiv_by_const.h" 29#include "util/u_math.h" 30 31#ifndef GEN_MI_BUILDER_NUM_ALLOC_GPRS 32/** The number of GPRs the MI builder is allowed to allocate 33 * 34 * This may be set by a user of this API so that it can reserve some GPRs at 35 * the top end for its own use. 36 */ 37#define GEN_MI_BUILDER_NUM_ALLOC_GPRS 16 38#endif 39 40/** These must be defined by the user of the builder 41 * 42 * void *__gen_get_batch_dwords(__gen_user_data *user_data, 43 * unsigned num_dwords); 44 * 45 * __gen_address_type 46 * __gen_address_offset(__gen_address_type addr, uint64_t offset); 47 * 48 */ 49 50/* 51 * Start of the actual MI builder 52 */ 53 54#define __genxml_cmd_length(cmd) cmd ## _length 55#define __genxml_cmd_header(cmd) cmd ## _header 56#define __genxml_cmd_pack(cmd) cmd ## _pack 57 58#define gen_mi_builder_pack(b, cmd, dst, name) \ 59 for (struct cmd name = { __genxml_cmd_header(cmd) }, \ 60 *_dst = (struct cmd *)(dst); __builtin_expect(_dst != NULL, 1); \ 61 __genxml_cmd_pack(cmd)((b)->user_data, (void *)_dst, &name), \ 62 _dst = NULL) 63 64#define gen_mi_builder_emit(b, cmd, name) \ 65 gen_mi_builder_pack((b), cmd, __gen_get_batch_dwords((b)->user_data, __genxml_cmd_length(cmd)), name) 66 67 68enum gen_mi_value_type { 69 GEN_MI_VALUE_TYPE_IMM, 70 GEN_MI_VALUE_TYPE_MEM32, 71 GEN_MI_VALUE_TYPE_MEM64, 72 GEN_MI_VALUE_TYPE_REG32, 73 GEN_MI_VALUE_TYPE_REG64, 74}; 75 76struct gen_mi_value { 77 enum gen_mi_value_type type; 78 79 union { 80 uint64_t imm; 81 __gen_address_type addr; 82 uint32_t reg; 83 }; 84 85#if GEN_GEN >= 7 || GEN_IS_HASWELL 86 bool invert; 87#endif 88}; 89 90#if GEN_GEN >= 9 91#define GEN_MI_BUILDER_MAX_MATH_DWORDS 256 92#else 93#define GEN_MI_BUILDER_MAX_MATH_DWORDS 64 94#endif 95 96struct gen_mi_builder { 97 __gen_user_data *user_data; 98 99#if GEN_GEN >= 8 || GEN_IS_HASWELL 100 uint32_t gprs; 101 uint8_t gpr_refs[GEN_MI_BUILDER_NUM_ALLOC_GPRS]; 102 103 unsigned num_math_dwords; 104 uint32_t math_dwords[GEN_MI_BUILDER_MAX_MATH_DWORDS]; 105#endif 106}; 107 108static inline void 109gen_mi_builder_init(struct gen_mi_builder *b, __gen_user_data *user_data) 110{ 111 memset(b, 0, sizeof(*b)); 112 b->user_data = user_data; 113 114#if GEN_GEN >= 8 || GEN_IS_HASWELL 115 b->gprs = 0; 116 b->num_math_dwords = 0; 117#endif 118} 119 120static inline void 121gen_mi_builder_flush_math(struct gen_mi_builder *b) 122{ 123#if GEN_GEN >= 8 || GEN_IS_HASWELL 124 if (b->num_math_dwords == 0) 125 return; 126 127 uint32_t *dw = (uint32_t *)__gen_get_batch_dwords(b->user_data, 128 1 + b->num_math_dwords); 129 gen_mi_builder_pack(b, GENX(MI_MATH), dw, math) { 130 math.DWordLength = 1 + b->num_math_dwords - GENX(MI_MATH_length_bias); 131 } 132 memcpy(dw + 1, b->math_dwords, b->num_math_dwords * sizeof(uint32_t)); 133 b->num_math_dwords = 0; 134#endif 135} 136 137#define _GEN_MI_BUILDER_GPR_BASE 0x2600 138/* The actual hardware limit on GPRs */ 139#define _GEN_MI_BUILDER_NUM_HW_GPRS 16 140 141#if GEN_GEN >= 8 || GEN_IS_HASWELL 142 143static inline bool 144gen_mi_value_is_gpr(struct gen_mi_value val) 145{ 146 return (val.type == GEN_MI_VALUE_TYPE_REG32 || 147 val.type == GEN_MI_VALUE_TYPE_REG64) && 148 val.reg >= _GEN_MI_BUILDER_GPR_BASE && 149 val.reg < _GEN_MI_BUILDER_GPR_BASE + 150 _GEN_MI_BUILDER_NUM_HW_GPRS * 8; 151} 152 153static inline bool 154_gen_mi_value_is_allocated_gpr(struct gen_mi_value val) 155{ 156 return (val.type == GEN_MI_VALUE_TYPE_REG32 || 157 val.type == GEN_MI_VALUE_TYPE_REG64) && 158 val.reg >= _GEN_MI_BUILDER_GPR_BASE && 159 val.reg < _GEN_MI_BUILDER_GPR_BASE + 160 GEN_MI_BUILDER_NUM_ALLOC_GPRS * 8; 161} 162 163static inline uint32_t 164_gen_mi_value_as_gpr(struct gen_mi_value val) 165{ 166 assert(gen_mi_value_is_gpr(val)); 167 assert(val.reg % 8 == 0); 168 return (val.reg - _GEN_MI_BUILDER_GPR_BASE) / 8; 169} 170 171static inline struct gen_mi_value 172gen_mi_new_gpr(struct gen_mi_builder *b) 173{ 174 unsigned gpr = ffs(~b->gprs) - 1; 175 assert(gpr < GEN_MI_BUILDER_NUM_ALLOC_GPRS); 176 assert(b->gpr_refs[gpr] == 0); 177 b->gprs |= (1u << gpr); 178 b->gpr_refs[gpr] = 1; 179 180 return (struct gen_mi_value) { 181 .type = GEN_MI_VALUE_TYPE_REG64, 182 .reg = _GEN_MI_BUILDER_GPR_BASE + gpr * 8, 183 }; 184} 185#endif /* GEN_GEN >= 8 || GEN_IS_HASWELL */ 186 187/** Take a reference to a gen_mi_value 188 * 189 * The MI builder uses reference counting to automatically free ALU GPRs for 190 * re-use in calculations. All gen_mi_* math functions consume the reference 191 * they are handed for each source and return a reference to a value which the 192 * caller must consume. In particular, if you pas the same value into a 193 * single gen_mi_* math function twice (say to add a number to itself), you 194 * are responsible for calling gen_mi_value_ref() to get a second reference 195 * because the gen_mi_* math function will consume it twice. 196 */ 197static inline struct gen_mi_value 198gen_mi_value_ref(struct gen_mi_builder *b, struct gen_mi_value val) 199{ 200#if GEN_GEN >= 8 || GEN_IS_HASWELL 201 if (_gen_mi_value_is_allocated_gpr(val)) { 202 unsigned gpr = _gen_mi_value_as_gpr(val); 203 assert(gpr < GEN_MI_BUILDER_NUM_ALLOC_GPRS); 204 assert(b->gprs & (1u << gpr)); 205 assert(b->gpr_refs[gpr] < UINT8_MAX); 206 b->gpr_refs[gpr]++; 207 } 208#endif /* GEN_GEN >= 8 || GEN_IS_HASWELL */ 209 210 return val; 211} 212 213/** Drop a reference to a gen_mi_value 214 * 215 * See also gen_mi_value_ref. 216 */ 217static inline void 218gen_mi_value_unref(struct gen_mi_builder *b, struct gen_mi_value val) 219{ 220#if GEN_GEN >= 8 || GEN_IS_HASWELL 221 if (_gen_mi_value_is_allocated_gpr(val)) { 222 unsigned gpr = _gen_mi_value_as_gpr(val); 223 assert(gpr < GEN_MI_BUILDER_NUM_ALLOC_GPRS); 224 assert(b->gprs & (1u << gpr)); 225 assert(b->gpr_refs[gpr] > 0); 226 if (--b->gpr_refs[gpr] == 0) 227 b->gprs &= ~(1u << gpr); 228 } 229#endif /* GEN_GEN >= 8 || GEN_IS_HASWELL */ 230} 231 232static inline struct gen_mi_value 233gen_mi_imm(uint64_t imm) 234{ 235 return (struct gen_mi_value) { 236 .type = GEN_MI_VALUE_TYPE_IMM, 237 .imm = imm, 238 }; 239} 240 241static inline struct gen_mi_value 242gen_mi_reg32(uint32_t reg) 243{ 244 struct gen_mi_value val = { 245 .type = GEN_MI_VALUE_TYPE_REG32, 246 .reg = reg, 247 }; 248#if GEN_GEN >= 8 || GEN_IS_HASWELL 249 assert(!_gen_mi_value_is_allocated_gpr(val)); 250#endif 251 return val; 252} 253 254static inline struct gen_mi_value 255gen_mi_reg64(uint32_t reg) 256{ 257 struct gen_mi_value val = { 258 .type = GEN_MI_VALUE_TYPE_REG64, 259 .reg = reg, 260 }; 261#if GEN_GEN >= 8 || GEN_IS_HASWELL 262 assert(!_gen_mi_value_is_allocated_gpr(val)); 263#endif 264 return val; 265} 266 267static inline struct gen_mi_value 268gen_mi_mem32(__gen_address_type addr) 269{ 270 return (struct gen_mi_value) { 271 .type = GEN_MI_VALUE_TYPE_MEM32, 272 .addr = addr, 273 }; 274} 275 276static inline struct gen_mi_value 277gen_mi_mem64(__gen_address_type addr) 278{ 279 return (struct gen_mi_value) { 280 .type = GEN_MI_VALUE_TYPE_MEM64, 281 .addr = addr, 282 }; 283} 284 285static inline struct gen_mi_value 286gen_mi_value_half(struct gen_mi_value value, bool top_32_bits) 287{ 288 switch (value.type) { 289 case GEN_MI_VALUE_TYPE_IMM: 290 if (top_32_bits) 291 value.imm >>= 32; 292 else 293 value.imm &= 0xffffffffu; 294 return value; 295 296 case GEN_MI_VALUE_TYPE_MEM32: 297 assert(!top_32_bits); 298 return value; 299 300 case GEN_MI_VALUE_TYPE_MEM64: 301 if (top_32_bits) 302 value.addr = __gen_address_offset(value.addr, 4); 303 value.type = GEN_MI_VALUE_TYPE_MEM32; 304 return value; 305 306 case GEN_MI_VALUE_TYPE_REG32: 307 assert(!top_32_bits); 308 return value; 309 310 case GEN_MI_VALUE_TYPE_REG64: 311 if (top_32_bits) 312 value.reg += 4; 313 value.type = GEN_MI_VALUE_TYPE_REG32; 314 return value; 315 } 316 317 unreachable("Invalid gen_mi_value type"); 318} 319 320static inline void 321_gen_mi_copy_no_unref(struct gen_mi_builder *b, 322 struct gen_mi_value dst, struct gen_mi_value src) 323{ 324#if GEN_GEN >= 7 || GEN_IS_HASWELL 325 /* TODO: We could handle src.invert by emitting a bit of math if we really 326 * wanted to. 327 */ 328 assert(!dst.invert && !src.invert); 329#endif 330 gen_mi_builder_flush_math(b); 331 332 switch (dst.type) { 333 case GEN_MI_VALUE_TYPE_IMM: 334 unreachable("Cannot copy to an immediate"); 335 336 case GEN_MI_VALUE_TYPE_MEM64: 337 case GEN_MI_VALUE_TYPE_REG64: 338 /* If the destination is 64 bits, we have to copy in two halves */ 339 _gen_mi_copy_no_unref(b, gen_mi_value_half(dst, false), 340 gen_mi_value_half(src, false)); 341 switch (src.type) { 342 case GEN_MI_VALUE_TYPE_IMM: 343 case GEN_MI_VALUE_TYPE_MEM64: 344 case GEN_MI_VALUE_TYPE_REG64: 345 /* TODO: Use MI_STORE_DATA_IMM::StoreQWord when we have it */ 346 _gen_mi_copy_no_unref(b, gen_mi_value_half(dst, true), 347 gen_mi_value_half(src, true)); 348 break; 349 default: 350 _gen_mi_copy_no_unref(b, gen_mi_value_half(dst, true), 351 gen_mi_imm(0)); 352 break; 353 } 354 break; 355 356 case GEN_MI_VALUE_TYPE_MEM32: 357 switch (src.type) { 358 case GEN_MI_VALUE_TYPE_IMM: 359 gen_mi_builder_emit(b, GENX(MI_STORE_DATA_IMM), sdi) { 360 sdi.Address = dst.addr; 361 sdi.ImmediateData = src.imm; 362 } 363 break; 364 365 case GEN_MI_VALUE_TYPE_MEM32: 366 case GEN_MI_VALUE_TYPE_MEM64: 367#if GEN_GEN >= 8 368 gen_mi_builder_emit(b, GENX(MI_COPY_MEM_MEM), cmm) { 369 cmm.DestinationMemoryAddress = dst.addr; 370 cmm.SourceMemoryAddress = src.addr; 371 } 372#elif GEN_IS_HASWELL 373 { 374 struct gen_mi_value tmp = gen_mi_new_gpr(b); 375 _gen_mi_copy_no_unref(b, tmp, src); 376 _gen_mi_copy_no_unref(b, dst, tmp); 377 gen_mi_value_unref(b, tmp); 378 } 379#else 380 unreachable("Cannot do mem <-> mem copy on IVB and earlier"); 381#endif 382 break; 383 384 case GEN_MI_VALUE_TYPE_REG32: 385 case GEN_MI_VALUE_TYPE_REG64: 386 gen_mi_builder_emit(b, GENX(MI_STORE_REGISTER_MEM), srm) { 387 srm.RegisterAddress = src.reg; 388 srm.MemoryAddress = dst.addr; 389 } 390 break; 391 392 default: 393 unreachable("Invalid gen_mi_value type"); 394 } 395 break; 396 397 case GEN_MI_VALUE_TYPE_REG32: 398 switch (src.type) { 399 case GEN_MI_VALUE_TYPE_IMM: 400 gen_mi_builder_emit(b, GENX(MI_LOAD_REGISTER_IMM), lri) { 401 lri.RegisterOffset = dst.reg; 402 lri.DataDWord = src.imm; 403 } 404 break; 405 406 case GEN_MI_VALUE_TYPE_MEM32: 407 case GEN_MI_VALUE_TYPE_MEM64: 408 gen_mi_builder_emit(b, GENX(MI_LOAD_REGISTER_MEM), lrm) { 409 lrm.RegisterAddress = dst.reg; 410 lrm.MemoryAddress = src.addr; 411 } 412 break; 413 414 case GEN_MI_VALUE_TYPE_REG32: 415 case GEN_MI_VALUE_TYPE_REG64: 416#if GEN_GEN >= 8 || GEN_IS_HASWELL 417 gen_mi_builder_emit(b, GENX(MI_LOAD_REGISTER_REG), lrr) { 418 lrr.SourceRegisterAddress = src.reg; 419 lrr.DestinationRegisterAddress = dst.reg; 420 } 421#else 422 unreachable("Cannot do reg <-> reg copy on IVB and earlier"); 423#endif 424 break; 425 426 default: 427 unreachable("Invalid gen_mi_value type"); 428 } 429 break; 430 431 default: 432 unreachable("Invalid gen_mi_value type"); 433 } 434} 435 436/** Store the value in src to the value represented by dst 437 * 438 * If the bit size of src and dst mismatch, this function does an unsigned 439 * integer cast. If src has more bits than dst, it takes the bottom bits. If 440 * src has fewer bits then dst, it fills the top bits with zeros. 441 * 442 * This function consumes one reference for each of src and dst. 443 */ 444static inline void 445gen_mi_store(struct gen_mi_builder *b, 446 struct gen_mi_value dst, struct gen_mi_value src) 447{ 448 _gen_mi_copy_no_unref(b, dst, src); 449 gen_mi_value_unref(b, src); 450 gen_mi_value_unref(b, dst); 451} 452 453static inline void 454gen_mi_memset(struct gen_mi_builder *b, __gen_address_type dst, 455 uint32_t value, uint32_t size) 456{ 457#if GEN_GEN >= 8 || GEN_IS_HASWELL 458 assert(b->num_math_dwords == 0); 459#endif 460 461 /* This memset operates in units of dwords. */ 462 assert(size % 4 == 0); 463 464 for (uint32_t i = 0; i < size; i += 4) { 465 gen_mi_store(b, gen_mi_mem32(__gen_address_offset(dst, i)), 466 gen_mi_imm(value)); 467 } 468} 469 470/* NOTE: On IVB, this function stomps GEN7_3DPRIM_BASE_VERTEX */ 471static inline void 472gen_mi_memcpy(struct gen_mi_builder *b, __gen_address_type dst, 473 __gen_address_type src, uint32_t size) 474{ 475#if GEN_GEN >= 8 || GEN_IS_HASWELL 476 assert(b->num_math_dwords == 0); 477#endif 478 479 /* This memcpy operates in units of dwords. */ 480 assert(size % 4 == 0); 481 482 for (uint32_t i = 0; i < size; i += 4) { 483 struct gen_mi_value dst_val = gen_mi_mem32(__gen_address_offset(dst, i)); 484 struct gen_mi_value src_val = gen_mi_mem32(__gen_address_offset(src, i)); 485#if GEN_GEN >= 8 || GEN_IS_HASWELL 486 gen_mi_store(b, dst_val, src_val); 487#else 488 /* IVB does not have a general purpose register for command streamer 489 * commands. Therefore, we use an alternate temporary register. 490 */ 491 struct gen_mi_value tmp_reg = gen_mi_reg32(0x2440); /* GEN7_3DPRIM_BASE_VERTEX */ 492 gen_mi_store(b, tmp_reg, src_val); 493 gen_mi_store(b, dst_val, tmp_reg); 494#endif 495 } 496} 497 498/* 499 * MI_MATH Section. Only available on Haswell+ 500 */ 501 502#if GEN_GEN >= 8 || GEN_IS_HASWELL 503 504static inline void 505_gen_mi_builder_push_math(struct gen_mi_builder *b, 506 const uint32_t *dwords, 507 unsigned num_dwords) 508{ 509 assert(num_dwords < GEN_MI_BUILDER_MAX_MATH_DWORDS); 510 if (b->num_math_dwords + num_dwords > GEN_MI_BUILDER_MAX_MATH_DWORDS) 511 gen_mi_builder_flush_math(b); 512 513 memcpy(&b->math_dwords[b->num_math_dwords], 514 dwords, num_dwords * sizeof(*dwords)); 515 b->num_math_dwords += num_dwords; 516} 517 518static inline uint32_t 519_gen_mi_pack_alu(uint32_t opcode, uint32_t operand1, uint32_t operand2) 520{ 521 struct GENX(MI_MATH_ALU_INSTRUCTION) instr = { 522 .Operand2 = operand2, 523 .Operand1 = operand1, 524 .ALUOpcode = opcode, 525 }; 526 527 uint32_t dw; 528 GENX(MI_MATH_ALU_INSTRUCTION_pack)(NULL, &dw, &instr); 529 530 return dw; 531} 532 533static inline struct gen_mi_value 534gen_mi_value_to_gpr(struct gen_mi_builder *b, struct gen_mi_value val) 535{ 536 if (gen_mi_value_is_gpr(val)) 537 return val; 538 539 /* Save off the invert flag because it makes copy() grumpy */ 540 bool invert = val.invert; 541 val.invert = false; 542 543 struct gen_mi_value tmp = gen_mi_new_gpr(b); 544 _gen_mi_copy_no_unref(b, tmp, val); 545 tmp.invert = invert; 546 547 return tmp; 548} 549 550static inline uint32_t 551_gen_mi_math_load_src(struct gen_mi_builder *b, 552 unsigned src, struct gen_mi_value *val) 553{ 554 if (val->type == GEN_MI_VALUE_TYPE_IMM && 555 (val->imm == 0 || val->imm == UINT64_MAX)) { 556 uint64_t imm = val->invert ? ~val->imm : val->imm; 557 return _gen_mi_pack_alu(imm ? MI_ALU_LOAD1 : MI_ALU_LOAD0, src, 0); 558 } else { 559 *val = gen_mi_value_to_gpr(b, *val); 560 return _gen_mi_pack_alu(val->invert ? MI_ALU_LOADINV : MI_ALU_LOAD, 561 src, _gen_mi_value_as_gpr(*val)); 562 } 563} 564 565static inline struct gen_mi_value 566gen_mi_math_binop(struct gen_mi_builder *b, uint32_t opcode, 567 struct gen_mi_value src0, struct gen_mi_value src1, 568 uint32_t store_op, uint32_t store_src) 569{ 570 struct gen_mi_value dst = gen_mi_new_gpr(b); 571 572 uint32_t dw[4]; 573 dw[0] = _gen_mi_math_load_src(b, MI_ALU_SRCA, &src0); 574 dw[1] = _gen_mi_math_load_src(b, MI_ALU_SRCB, &src1); 575 dw[2] = _gen_mi_pack_alu(opcode, 0, 0); 576 dw[3] = _gen_mi_pack_alu(store_op, _gen_mi_value_as_gpr(dst), store_src); 577 _gen_mi_builder_push_math(b, dw, 4); 578 579 gen_mi_value_unref(b, src0); 580 gen_mi_value_unref(b, src1); 581 582 return dst; 583} 584 585static inline struct gen_mi_value 586gen_mi_inot(struct gen_mi_builder *b, struct gen_mi_value val) 587{ 588 /* TODO These currently can't be passed into gen_mi_copy */ 589 val.invert = !val.invert; 590 return val; 591} 592 593static inline struct gen_mi_value 594gen_mi_iadd(struct gen_mi_builder *b, 595 struct gen_mi_value src0, struct gen_mi_value src1) 596{ 597 return gen_mi_math_binop(b, MI_ALU_ADD, src0, src1, 598 MI_ALU_STORE, MI_ALU_ACCU); 599} 600 601static inline struct gen_mi_value 602gen_mi_iadd_imm(struct gen_mi_builder *b, 603 struct gen_mi_value src, uint64_t N) 604{ 605 if (N == 0) 606 return src; 607 608 return gen_mi_iadd(b, src, gen_mi_imm(N)); 609} 610 611static inline struct gen_mi_value 612gen_mi_isub(struct gen_mi_builder *b, 613 struct gen_mi_value src0, struct gen_mi_value src1) 614{ 615 return gen_mi_math_binop(b, MI_ALU_SUB, src0, src1, 616 MI_ALU_STORE, MI_ALU_ACCU); 617} 618 619static inline struct gen_mi_value 620gen_mi_ult(struct gen_mi_builder *b, 621 struct gen_mi_value src0, struct gen_mi_value src1) 622{ 623 /* Compute "less than" by subtracting and storing the carry bit */ 624 return gen_mi_math_binop(b, MI_ALU_SUB, src0, src1, 625 MI_ALU_STORE, MI_ALU_CF); 626} 627 628static inline struct gen_mi_value 629gen_mi_uge(struct gen_mi_builder *b, 630 struct gen_mi_value src0, struct gen_mi_value src1) 631{ 632 /* Compute "less than" by subtracting and storing the carry bit */ 633 return gen_mi_math_binop(b, MI_ALU_SUB, src0, src1, 634 MI_ALU_STOREINV, MI_ALU_CF); 635} 636 637static inline struct gen_mi_value 638gen_mi_iand(struct gen_mi_builder *b, 639 struct gen_mi_value src0, struct gen_mi_value src1) 640{ 641 return gen_mi_math_binop(b, MI_ALU_AND, src0, src1, 642 MI_ALU_STORE, MI_ALU_ACCU); 643} 644 645static inline struct gen_mi_value 646gen_mi_imul_imm(struct gen_mi_builder *b, 647 struct gen_mi_value src, uint32_t N) 648{ 649 if (N == 0) { 650 gen_mi_value_unref(b, src); 651 return gen_mi_imm(0); 652 } 653 654 if (N == 1) 655 return src; 656 657 src = gen_mi_value_to_gpr(b, src); 658 659 struct gen_mi_value res = gen_mi_value_ref(b, src); 660 661 unsigned top_bit = 31 - __builtin_clz(N); 662 for (int i = top_bit - 1; i >= 0; i--) { 663 res = gen_mi_iadd(b, res, gen_mi_value_ref(b, res)); 664 if (N & (1 << i)) 665 res = gen_mi_iadd(b, res, gen_mi_value_ref(b, src)); 666 } 667 668 gen_mi_value_unref(b, src); 669 670 return res; 671} 672 673static inline struct gen_mi_value 674gen_mi_ishl_imm(struct gen_mi_builder *b, 675 struct gen_mi_value src, uint32_t shift) 676{ 677 struct gen_mi_value res = gen_mi_value_to_gpr(b, src); 678 679 for (unsigned i = 0; i < shift; i++) 680 res = gen_mi_iadd(b, res, gen_mi_value_ref(b, res)); 681 682 return res; 683} 684 685static inline struct gen_mi_value 686gen_mi_ushr32_imm(struct gen_mi_builder *b, 687 struct gen_mi_value src, uint32_t shift) 688{ 689 /* We right-shift by left-shifting by 32 - shift and taking the top 32 bits 690 * of the result. This assumes the top 32 bits are zero. 691 */ 692 if (shift > 64) 693 return gen_mi_imm(0); 694 695 if (shift > 32) { 696 struct gen_mi_value tmp = gen_mi_new_gpr(b); 697 _gen_mi_copy_no_unref(b, gen_mi_value_half(tmp, false), 698 gen_mi_value_half(src, true)); 699 _gen_mi_copy_no_unref(b, gen_mi_value_half(tmp, true), gen_mi_imm(0)); 700 gen_mi_value_unref(b, src); 701 src = tmp; 702 shift -= 32; 703 } 704 assert(shift <= 32); 705 struct gen_mi_value tmp = gen_mi_ishl_imm(b, src, 32 - shift); 706 struct gen_mi_value dst = gen_mi_new_gpr(b); 707 _gen_mi_copy_no_unref(b, gen_mi_value_half(dst, false), 708 gen_mi_value_half(tmp, true)); 709 _gen_mi_copy_no_unref(b, gen_mi_value_half(dst, true), gen_mi_imm(0)); 710 gen_mi_value_unref(b, tmp); 711 return dst; 712} 713 714static inline struct gen_mi_value 715gen_mi_udiv32_imm(struct gen_mi_builder *b, 716 struct gen_mi_value N, uint32_t D) 717{ 718 /* We implicitly assume that N is only a 32-bit value */ 719 if (D == 0) { 720 /* This is invalid but we should do something */ 721 return gen_mi_imm(0); 722 } else if (util_is_power_of_two_or_zero(D)) { 723 return gen_mi_ushr32_imm(b, N, util_logbase2(D)); 724 } else { 725 struct util_fast_udiv_info m = util_compute_fast_udiv_info(D, 32, 32); 726 assert(m.multiplier <= UINT32_MAX); 727 728 if (m.pre_shift) 729 N = gen_mi_ushr32_imm(b, N, m.pre_shift); 730 731 /* Do the 32x32 multiply into gpr0 */ 732 N = gen_mi_imul_imm(b, N, m.multiplier); 733 734 if (m.increment) 735 N = gen_mi_iadd(b, N, gen_mi_imm(m.multiplier)); 736 737 N = gen_mi_ushr32_imm(b, N, 32); 738 739 if (m.post_shift) 740 N = gen_mi_ushr32_imm(b, N, m.post_shift); 741 742 return N; 743 } 744} 745 746#endif /* MI_MATH section */ 747 748#endif /* GEN_MI_BUILDER_H */ 749