1/* 2 * Copyright (C) 2014 Rob Clark <robclark@freedesktop.org> 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 * SOFTWARE. 22 * 23 * Authors: 24 * Rob Clark <robclark@freedesktop.org> 25 */ 26 27#include "util/ralloc.h" 28#include "util/u_math.h" 29 30#include "ir3.h" 31#include "ir3_compiler.h" 32 33/* 34 * Legalize: 35 * 36 * We currently require that scheduling ensures that we have enough nop's 37 * in all the right places. The legalize step mostly handles fixing up 38 * instruction flags ((ss)/(sy)/(ei)), and collapses sequences of nop's 39 * into fewer nop's w/ rpt flag. 40 */ 41 42struct ir3_legalize_ctx { 43 struct ir3_compiler *compiler; 44 gl_shader_stage type; 45 bool has_ssbo; 46 bool need_pixlod; 47 int max_bary; 48}; 49 50struct ir3_legalize_state { 51 regmask_t needs_ss; 52 regmask_t needs_ss_war; /* write after read */ 53 regmask_t needs_sy; 54}; 55 56struct ir3_legalize_block_data { 57 bool valid; 58 struct ir3_legalize_state state; 59}; 60 61/* We want to evaluate each block from the position of any other 62 * predecessor block, in order that the flags set are the union of 63 * all possible program paths. 64 * 65 * To do this, we need to know the output state (needs_ss/ss_war/sy) 66 * of all predecessor blocks. The tricky thing is loops, which mean 67 * that we can't simply recursively process each predecessor block 68 * before legalizing the current block. 69 * 70 * How we handle that is by looping over all the blocks until the 71 * results converge. If the output state of a given block changes 72 * in a given pass, this means that all successor blocks are not 73 * yet fully legalized. 74 */ 75 76static bool 77legalize_block(struct ir3_legalize_ctx *ctx, struct ir3_block *block) 78{ 79 struct ir3_legalize_block_data *bd = block->data; 80 81 if (bd->valid) 82 return false; 83 84 struct ir3_instruction *last_input = NULL; 85 struct ir3_instruction *last_rel = NULL; 86 struct ir3_instruction *last_n = NULL; 87 struct list_head instr_list; 88 struct ir3_legalize_state prev_state = bd->state; 89 struct ir3_legalize_state *state = &bd->state; 90 91 /* our input state is the OR of all predecessor blocks' state: */ 92 for (unsigned i = 0; i < block->predecessors_count; i++) { 93 struct ir3_legalize_block_data *pbd = block->predecessors[i]->data; 94 struct ir3_legalize_state *pstate = &pbd->state; 95 96 /* Our input (ss)/(sy) state is based on OR'ing the output 97 * state of all our predecessor blocks 98 */ 99 regmask_or(&state->needs_ss, 100 &state->needs_ss, &pstate->needs_ss); 101 regmask_or(&state->needs_ss_war, 102 &state->needs_ss_war, &pstate->needs_ss_war); 103 regmask_or(&state->needs_sy, 104 &state->needs_sy, &pstate->needs_sy); 105 } 106 107 /* remove all the instructions from the list, we'll be adding 108 * them back in as we go 109 */ 110 list_replace(&block->instr_list, &instr_list); 111 list_inithead(&block->instr_list); 112 113 list_for_each_entry_safe (struct ir3_instruction, n, &instr_list, node) { 114 struct ir3_register *reg; 115 unsigned i; 116 117 n->flags &= ~(IR3_INSTR_SS | IR3_INSTR_SY); 118 119 if (is_meta(n)) 120 continue; 121 122 if (is_input(n)) { 123 struct ir3_register *inloc = n->regs[1]; 124 assert(inloc->flags & IR3_REG_IMMED); 125 ctx->max_bary = MAX2(ctx->max_bary, inloc->iim_val); 126 } 127 128 if (last_n && is_barrier(last_n)) 129 n->flags |= IR3_INSTR_SS | IR3_INSTR_SY; 130 131 /* NOTE: consider dst register too.. it could happen that 132 * texture sample instruction (for example) writes some 133 * components which are unused. A subsequent instruction 134 * that writes the same register can race w/ the sam instr 135 * resulting in undefined results: 136 */ 137 for (i = 0; i < n->regs_count; i++) { 138 reg = n->regs[i]; 139 140 if (reg_gpr(reg)) { 141 142 /* TODO: we probably only need (ss) for alu 143 * instr consuming sfu result.. need to make 144 * some tests for both this and (sy).. 145 */ 146 if (regmask_get(&state->needs_ss, reg)) { 147 n->flags |= IR3_INSTR_SS; 148 regmask_init(&state->needs_ss_war); 149 regmask_init(&state->needs_ss); 150 } 151 152 if (regmask_get(&state->needs_sy, reg)) { 153 n->flags |= IR3_INSTR_SY; 154 regmask_init(&state->needs_sy); 155 } 156 } 157 158 /* TODO: is it valid to have address reg loaded from a 159 * relative src (ie. mova a0, c<a0.x+4>)? If so, the 160 * last_rel check below should be moved ahead of this: 161 */ 162 if (reg->flags & IR3_REG_RELATIV) 163 last_rel = n; 164 } 165 166 if (n->regs_count > 0) { 167 reg = n->regs[0]; 168 if (regmask_get(&state->needs_ss_war, reg)) { 169 n->flags |= IR3_INSTR_SS; 170 regmask_init(&state->needs_ss_war); 171 regmask_init(&state->needs_ss); 172 } 173 174 if (last_rel && (reg->num == regid(REG_A0, 0))) { 175 last_rel->flags |= IR3_INSTR_UL; 176 last_rel = NULL; 177 } 178 } 179 180 /* cat5+ does not have an (ss) bit, if needed we need to 181 * insert a nop to carry the sync flag. Would be kinda 182 * clever if we were aware of this during scheduling, but 183 * this should be a pretty rare case: 184 */ 185 if ((n->flags & IR3_INSTR_SS) && (opc_cat(n->opc) >= 5)) { 186 struct ir3_instruction *nop; 187 nop = ir3_NOP(block); 188 nop->flags |= IR3_INSTR_SS; 189 n->flags &= ~IR3_INSTR_SS; 190 } 191 192 /* need to be able to set (ss) on first instruction: */ 193 if (list_empty(&block->instr_list) && (opc_cat(n->opc) >= 5)) 194 ir3_NOP(block); 195 196 if (is_nop(n) && !list_empty(&block->instr_list)) { 197 struct ir3_instruction *last = list_last_entry(&block->instr_list, 198 struct ir3_instruction, node); 199 if (is_nop(last) && (last->repeat < 5)) { 200 last->repeat++; 201 last->flags |= n->flags; 202 continue; 203 } 204 205 /* NOTE: I think the nopN encoding works for a5xx and 206 * probably a4xx, but not a3xx. So far only tested on 207 * a6xx. 208 */ 209 if ((ctx->compiler->gpu_id >= 600) && !n->flags && (last->nop < 3) && 210 ((opc_cat(last->opc) == 2) || (opc_cat(last->opc) == 3))) { 211 last->nop++; 212 continue; 213 } 214 } 215 216 if (ctx->compiler->samgq_workaround && 217 ctx->type == MESA_SHADER_VERTEX && n->opc == OPC_SAMGQ) { 218 struct ir3_instruction *samgp; 219 220 for (i = 0; i < 4; i++) { 221 samgp = ir3_instr_clone(n); 222 samgp->opc = OPC_SAMGP0 + i; 223 if (i > 1) 224 samgp->flags |= IR3_INSTR_SY; 225 } 226 list_delinit(&n->node); 227 } else { 228 list_addtail(&n->node, &block->instr_list); 229 } 230 231 if (is_sfu(n)) 232 regmask_set(&state->needs_ss, n->regs[0]); 233 234 if (is_tex(n)) { 235 regmask_set(&state->needs_sy, n->regs[0]); 236 ctx->need_pixlod = true; 237 } else if (n->opc == OPC_RESINFO) { 238 regmask_set(&state->needs_ss, n->regs[0]); 239 ir3_NOP(block)->flags |= IR3_INSTR_SS; 240 } else if (is_load(n)) { 241 /* seems like ldlv needs (ss) bit instead?? which is odd but 242 * makes a bunch of flat-varying tests start working on a4xx. 243 */ 244 if ((n->opc == OPC_LDLV) || (n->opc == OPC_LDL)) 245 regmask_set(&state->needs_ss, n->regs[0]); 246 else 247 regmask_set(&state->needs_sy, n->regs[0]); 248 } else if (is_atomic(n->opc)) { 249 if (n->flags & IR3_INSTR_G) { 250 if (ctx->compiler->gpu_id >= 600) { 251 /* New encoding, returns result via second src: */ 252 regmask_set(&state->needs_sy, n->regs[3]); 253 } else { 254 regmask_set(&state->needs_sy, n->regs[0]); 255 } 256 } else { 257 regmask_set(&state->needs_ss, n->regs[0]); 258 } 259 } 260 261 if (is_ssbo(n->opc) || (is_atomic(n->opc) && (n->flags & IR3_INSTR_G))) 262 ctx->has_ssbo = true; 263 264 /* both tex/sfu appear to not always immediately consume 265 * their src register(s): 266 */ 267 if (is_tex(n) || is_sfu(n) || is_mem(n)) { 268 foreach_src(reg, n) { 269 if (reg_gpr(reg)) 270 regmask_set(&state->needs_ss_war, reg); 271 } 272 } 273 274 if (is_input(n)) 275 last_input = n; 276 277 last_n = n; 278 } 279 280 if (last_input) { 281 /* special hack.. if using ldlv to bypass interpolation, 282 * we need to insert a dummy bary.f on which we can set 283 * the (ei) flag: 284 */ 285 if (is_mem(last_input) && (last_input->opc == OPC_LDLV)) { 286 struct ir3_instruction *baryf; 287 288 /* (ss)bary.f (ei)r63.x, 0, r0.x */ 289 baryf = ir3_instr_create(block, OPC_BARY_F); 290 baryf->flags |= IR3_INSTR_SS; 291 ir3_reg_create(baryf, regid(63, 0), 0); 292 ir3_reg_create(baryf, 0, IR3_REG_IMMED)->iim_val = 0; 293 ir3_reg_create(baryf, regid(0, 0), 0); 294 295 /* insert the dummy bary.f after last_input: */ 296 list_delinit(&baryf->node); 297 list_add(&baryf->node, &last_input->node); 298 299 last_input = baryf; 300 } 301 last_input->regs[0]->flags |= IR3_REG_EI; 302 } 303 304 if (last_rel) 305 last_rel->flags |= IR3_INSTR_UL; 306 307 bd->valid = true; 308 309 if (memcmp(&prev_state, state, sizeof(*state))) { 310 /* our output state changed, this invalidates all of our 311 * successors: 312 */ 313 for (unsigned i = 0; i < ARRAY_SIZE(block->successors); i++) { 314 if (!block->successors[i]) 315 break; 316 struct ir3_legalize_block_data *pbd = block->successors[i]->data; 317 pbd->valid = false; 318 } 319 } 320 321 return true; 322} 323 324/* NOTE: branch instructions are always the last instruction(s) 325 * in the block. We take advantage of this as we resolve the 326 * branches, since "if (foo) break;" constructs turn into 327 * something like: 328 * 329 * block3 { 330 * ... 331 * 0029:021: mov.s32s32 r62.x, r1.y 332 * 0082:022: br !p0.x, target=block5 333 * 0083:023: br p0.x, target=block4 334 * // succs: if _[0029:021: mov.s32s32] block4; else block5; 335 * } 336 * block4 { 337 * 0084:024: jump, target=block6 338 * // succs: block6; 339 * } 340 * block5 { 341 * 0085:025: jump, target=block7 342 * // succs: block7; 343 * } 344 * 345 * ie. only instruction in block4/block5 is a jump, so when 346 * resolving branches we can easily detect this by checking 347 * that the first instruction in the target block is itself 348 * a jump, and setup the br directly to the jump's target 349 * (and strip back out the now unreached jump) 350 * 351 * TODO sometimes we end up with things like: 352 * 353 * br !p0.x, #2 354 * br p0.x, #12 355 * add.u r0.y, r0.y, 1 356 * 357 * If we swapped the order of the branches, we could drop one. 358 */ 359static struct ir3_block * 360resolve_dest_block(struct ir3_block *block) 361{ 362 /* special case for last block: */ 363 if (!block->successors[0]) 364 return block; 365 366 /* NOTE that we may or may not have inserted the jump 367 * in the target block yet, so conditions to resolve 368 * the dest to the dest block's successor are: 369 * 370 * (1) successor[1] == NULL && 371 * (2) (block-is-empty || only-instr-is-jump) 372 */ 373 if (block->successors[1] == NULL) { 374 if (list_empty(&block->instr_list)) { 375 return block->successors[0]; 376 } else if (list_length(&block->instr_list) == 1) { 377 struct ir3_instruction *instr = list_first_entry( 378 &block->instr_list, struct ir3_instruction, node); 379 if (instr->opc == OPC_JUMP) 380 return block->successors[0]; 381 } 382 } 383 return block; 384} 385 386static bool 387resolve_jump(struct ir3_instruction *instr) 388{ 389 struct ir3_block *tblock = 390 resolve_dest_block(instr->cat0.target); 391 struct ir3_instruction *target; 392 393 if (tblock != instr->cat0.target) { 394 list_delinit(&instr->cat0.target->node); 395 instr->cat0.target = tblock; 396 return true; 397 } 398 399 target = list_first_entry(&tblock->instr_list, 400 struct ir3_instruction, node); 401 402 /* TODO maybe a less fragile way to do this. But we are expecting 403 * a pattern from sched_block() that looks like: 404 * 405 * br !p0.x, #else-block 406 * br p0.x, #if-block 407 * 408 * if the first branch target is +2, or if 2nd branch target is +1 409 * then we can just drop the jump. 410 */ 411 unsigned next_block; 412 if (instr->cat0.inv == true) 413 next_block = 2; 414 else 415 next_block = 1; 416 417 if ((!target) || (target->ip == (instr->ip + next_block))) { 418 list_delinit(&instr->node); 419 return true; 420 } else { 421 instr->cat0.immed = 422 (int)target->ip - (int)instr->ip; 423 } 424 return false; 425} 426 427/* resolve jumps, removing jumps/branches to immediately following 428 * instruction which we end up with from earlier stages. Since 429 * removing an instruction can invalidate earlier instruction's 430 * branch offsets, we need to do this iteratively until no more 431 * branches are removed. 432 */ 433static bool 434resolve_jumps(struct ir3 *ir) 435{ 436 list_for_each_entry (struct ir3_block, block, &ir->block_list, node) 437 list_for_each_entry (struct ir3_instruction, instr, &block->instr_list, node) 438 if (is_flow(instr) && instr->cat0.target) 439 if (resolve_jump(instr)) 440 return true; 441 442 return false; 443} 444 445/* we want to mark points where divergent flow control re-converges 446 * with (jp) flags. For now, since we don't do any optimization for 447 * things that start out as a 'do {} while()', re-convergence points 448 * will always be a branch or jump target. Note that this is overly 449 * conservative, since unconditional jump targets are not convergence 450 * points, we are just assuming that the other path to reach the jump 451 * target was divergent. If we were clever enough to optimize the 452 * jump at end of a loop back to a conditional branch into a single 453 * conditional branch, ie. like: 454 * 455 * add.f r1.w, r0.x, (neg)(r)c2.x <= loop start 456 * mul.f r1.z, r1.z, r0.x 457 * mul.f r1.y, r1.y, r0.x 458 * mul.f r0.z, r1.x, r0.x 459 * mul.f r0.w, r0.y, r0.x 460 * cmps.f.ge r0.x, (r)c2.y, (r)r1.w 461 * add.s r0.x, (r)r0.x, (r)-1 462 * sel.f32 r0.x, (r)c3.y, (r)r0.x, c3.x 463 * cmps.f.eq p0.x, r0.x, c3.y 464 * mov.f32f32 r0.x, r1.w 465 * mov.f32f32 r0.y, r0.w 466 * mov.f32f32 r1.x, r0.z 467 * (rpt2)nop 468 * br !p0.x, #-13 469 * (jp)mul.f r0.x, c263.y, r1.y 470 * 471 * Then we'd have to be more clever, as the convergence point is no 472 * longer a branch or jump target. 473 */ 474static void 475mark_convergence_points(struct ir3 *ir) 476{ 477 list_for_each_entry (struct ir3_block, block, &ir->block_list, node) { 478 list_for_each_entry (struct ir3_instruction, instr, &block->instr_list, node) { 479 if (is_flow(instr) && instr->cat0.target) { 480 struct ir3_instruction *target = 481 list_first_entry(&instr->cat0.target->instr_list, 482 struct ir3_instruction, node); 483 target->flags |= IR3_INSTR_JP; 484 } 485 } 486 } 487} 488 489void 490ir3_legalize(struct ir3 *ir, bool *has_ssbo, bool *need_pixlod, int *max_bary) 491{ 492 struct ir3_legalize_ctx *ctx = rzalloc(ir, struct ir3_legalize_ctx); 493 bool progress; 494 495 ctx->max_bary = -1; 496 ctx->compiler = ir->compiler; 497 ctx->type = ir->type; 498 499 /* allocate per-block data: */ 500 list_for_each_entry (struct ir3_block, block, &ir->block_list, node) { 501 block->data = rzalloc(ctx, struct ir3_legalize_block_data); 502 } 503 504 /* process each block: */ 505 do { 506 progress = false; 507 list_for_each_entry (struct ir3_block, block, &ir->block_list, node) { 508 progress |= legalize_block(ctx, block); 509 } 510 } while (progress); 511 512 *has_ssbo = ctx->has_ssbo; 513 *need_pixlod = ctx->need_pixlod; 514 *max_bary = ctx->max_bary; 515 516 do { 517 ir3_count_instructions(ir); 518 } while(resolve_jumps(ir)); 519 520 mark_convergence_points(ir); 521 522 ralloc_free(ctx); 523} 524