Home | History | Annotate | Line # | Download | only in sfn
      1 #include "sfn_nir.h"
      2 
      3 bool r600_lower_tess_io_filter(const nir_instr *instr, gl_shader_stage stage)
      4 {
      5    if (instr->type != nir_instr_type_intrinsic)
      6       return false;
      7 
      8    nir_intrinsic_instr *op = nir_instr_as_intrinsic(instr);
      9    switch (op->intrinsic) {
     10    case nir_intrinsic_load_input:
     11       return stage == MESA_SHADER_TESS_CTRL || stage == MESA_SHADER_TESS_EVAL;
     12    case nir_intrinsic_load_output:
     13    case nir_intrinsic_load_per_vertex_input:
     14    case nir_intrinsic_load_per_vertex_output:
     15    case nir_intrinsic_store_per_vertex_output:
     16    case nir_intrinsic_load_patch_vertices_in:
     17    case nir_intrinsic_load_tess_level_outer:
     18    case nir_intrinsic_load_tess_level_inner:
     19       return true;
     20    case nir_intrinsic_store_output:
     21       return stage == MESA_SHADER_TESS_CTRL || stage == MESA_SHADER_VERTEX;
     22    default:
     23       ;
     24    }
     25    return false;
     26 }
     27 
     28 static nir_ssa_def *
     29 emit_load_param_base(nir_builder *b, nir_intrinsic_op op)
     30 {
     31    nir_intrinsic_instr *result = nir_intrinsic_instr_create(b->shader, op);
     32 	nir_ssa_dest_init(&result->instr, &result->dest,
     33                      4, 32, NULL);
     34    nir_builder_instr_insert(b, &result->instr);
     35    return &result->dest.ssa;
     36 }
     37 
     38 static int get_tcs_varying_offset(nir_intrinsic_instr *op)
     39 {
     40    unsigned location = nir_intrinsic_io_semantics(op).location;
     41 
     42    switch (location) {
     43    case VARYING_SLOT_POS:
     44       return 0;
     45    case VARYING_SLOT_PSIZ:
     46       return 0x10;
     47    case VARYING_SLOT_CLIP_DIST0:
     48       return 0x20;
     49    case VARYING_SLOT_CLIP_DIST1:
     50       return 0x30;
     51    case VARYING_SLOT_TESS_LEVEL_OUTER:
     52       return 0;
     53    case VARYING_SLOT_TESS_LEVEL_INNER:
     54       return 0x10;
     55    default:
     56       if (location >= VARYING_SLOT_VAR0 &&
     57           location <= VARYING_SLOT_VAR31)
     58          return 0x10 * (location - VARYING_SLOT_VAR0) + 0x40;
     59 
     60       if (location >=  VARYING_SLOT_PATCH0) {
     61          return 0x10 * (location - VARYING_SLOT_PATCH0) + 0x20;
     62       }
     63    }
     64    return 0;
     65 }
     66 
     67 static inline nir_ssa_def *
     68 r600_umad_24(nir_builder *b, nir_ssa_def *op1, nir_ssa_def *op2, nir_ssa_def *op3)
     69 {
     70    return nir_build_alu(b, nir_op_umad24, op1, op2, op3, NULL);
     71 }
     72 
     73 static inline nir_ssa_def *
     74 r600_tcs_base_address(nir_builder *b, nir_ssa_def *param_base, nir_ssa_def *rel_patch_id)
     75 {
     76    return r600_umad_24(b,  nir_channel(b, param_base, 0),
     77                        rel_patch_id,
     78                        nir_channel(b, param_base, 3));
     79 }
     80 
     81 
     82 static nir_ssa_def *
     83 emil_lsd_in_addr(nir_builder *b, nir_ssa_def *base, nir_ssa_def *patch_id, nir_intrinsic_instr *op)
     84 {
     85    nir_ssa_def *addr = nir_build_alu(b, nir_op_umul24,
     86                                       nir_channel(b, base, 0),
     87                                       patch_id, NULL, NULL);
     88 
     89    auto idx1 = nir_src_as_const_value(op->src[0]);
     90    if (!idx1 || idx1->u32 != 0)
     91       addr = r600_umad_24(b, nir_channel(b, base, 1),
     92                           op->src[0].ssa, addr);
     93 
     94    auto offset = nir_imm_int(b, get_tcs_varying_offset(op));
     95 
     96    auto idx2 = nir_src_as_const_value(op->src[1]);
     97    if (!idx2 || idx2->u32 != 0)
     98       offset = nir_iadd(b, offset, nir_ishl(b, op->src[1].ssa, nir_imm_int(b, 4)));
     99 
    100    return nir_iadd(b, addr, offset);
    101 }
    102 
    103 static nir_ssa_def *
    104 emil_lsd_out_addr(nir_builder *b, nir_ssa_def *base, nir_ssa_def *patch_id, nir_intrinsic_instr *op, nir_variable_mode mode, int src_offset)
    105 {
    106 
    107    nir_ssa_def *addr1 = r600_umad_24(b, nir_channel(b, base, 0),
    108                                      patch_id,
    109                                      nir_channel(b, base, 2));
    110    nir_ssa_def *addr2 = r600_umad_24(b, nir_channel(b, base, 1),
    111                                      op->src[src_offset].ssa, addr1);
    112    int offset = get_tcs_varying_offset(op);
    113    return nir_iadd(b, nir_iadd(b, addr2,
    114                                nir_ishl(b, op->src[src_offset + 1].ssa, nir_imm_int(b,4))),
    115                                nir_imm_int(b, offset));
    116 }
    117 
    118 static nir_ssa_def *load_offset_group(nir_builder *b, int ncomponents)
    119 {
    120    switch (ncomponents) {
    121    /* tess outer offsets */
    122    case 1: return nir_imm_int(b, 0);
    123    case 2: return nir_imm_ivec2(b, 0, 4);
    124    case 3: return r600_imm_ivec3(b, 0, 4, 8);
    125    case 4: return nir_imm_ivec4(b, 0, 4, 8, 12);
    126       /* tess inner offsets */
    127    case 5: return nir_imm_int(b, 16);
    128    case 6: return nir_imm_ivec2(b, 16, 20);
    129    default:
    130       debug_printf("Got %d components\n", ncomponents);
    131       unreachable("Unsupported component count");
    132    }
    133 }
    134 
    135 static nir_ssa_def *load_offset_group_from_mask(nir_builder *b, uint32_t mask)
    136 {
    137    auto full_mask = nir_imm_ivec4(b, 0, 4, 8, 12);
    138    return nir_channels(b, full_mask, mask);
    139 }
    140 
    141 struct MaskQuery {
    142    uint32_t mask;
    143    uint32_t ssa_index;
    144    nir_alu_instr *alu;
    145    int index;
    146    uint32_t full_mask;
    147 };
    148 
    149 static bool update_alu_mask(nir_src *src, void *data)
    150 {
    151    auto mq = reinterpret_cast<MaskQuery *>(data);
    152 
    153    if (mq->ssa_index == src->ssa->index) {
    154       mq->mask |= nir_alu_instr_src_read_mask(mq->alu, mq->index);
    155    }
    156    ++mq->index;
    157 
    158    return mq->mask != mq->full_mask;
    159 }
    160 
    161 static uint32_t get_dest_usee_mask(nir_intrinsic_instr *op)
    162 {
    163    assert(op->dest.is_ssa);
    164 
    165    MaskQuery mq = {0};
    166    mq.full_mask = (1 << nir_dest_num_components(op->dest)) - 1;
    167 
    168    nir_foreach_use(use_src,  &op->dest.ssa) {
    169       auto use_instr = use_src->parent_instr;
    170       mq.ssa_index = use_src->ssa->index;
    171 
    172       switch (use_instr->type) {
    173       case nir_instr_type_alu: {
    174          mq.alu = nir_instr_as_alu(use_instr);
    175          mq.index = 0;
    176          if (!nir_foreach_src(use_instr, update_alu_mask, &mq))
    177             return 0xf;
    178          break;
    179       }
    180       case nir_instr_type_intrinsic:  {
    181          auto intr = nir_instr_as_intrinsic(use_instr);
    182          switch (intr->intrinsic) {
    183          case nir_intrinsic_store_output:
    184          case nir_intrinsic_store_per_vertex_output:
    185             mq.mask |= nir_intrinsic_write_mask(intr) << nir_intrinsic_component(intr);
    186             break;
    187          case nir_intrinsic_store_scratch:
    188          case nir_intrinsic_store_local_shared_r600:
    189             mq.mask |= nir_intrinsic_write_mask(intr);
    190             break;
    191          default:
    192             return 0xf;
    193          }
    194          break;
    195       }
    196       default:
    197          return 0xf;
    198       }
    199 
    200    }
    201    return mq.mask;
    202 }
    203 
    204 static void replace_load_instr(nir_builder *b, nir_intrinsic_instr *op, nir_ssa_def *addr)
    205 {
    206    uint32_t mask = get_dest_usee_mask(op);
    207    if (mask) {
    208       nir_ssa_def *addr_outer = nir_iadd(b, addr, load_offset_group_from_mask(b, mask));
    209       if (nir_intrinsic_component(op))
    210          addr_outer = nir_iadd(b, addr_outer, nir_imm_int(b, 4 * nir_intrinsic_component(op)));
    211 
    212       auto new_load = nir_load_local_shared_r600(b, 32, addr_outer);
    213 
    214       auto undef = nir_ssa_undef(b, 1, 32);
    215       int comps = nir_dest_num_components(op->dest);
    216       nir_ssa_def *remix[4] = {undef, undef, undef, undef};
    217 
    218       int chan = 0;
    219       for (int i = 0; i < comps; ++i) {
    220          if (mask & (1 << i)) {
    221             remix[i] = nir_channel(b, new_load, chan++);
    222          }
    223       }
    224       auto new_load_remixed = nir_vec(b, remix, comps);
    225       nir_ssa_def_rewrite_uses(&op->dest.ssa, new_load_remixed);
    226    }
    227    nir_instr_remove(&op->instr);
    228 }
    229 
    230 static nir_ssa_def *
    231 r600_load_rel_patch_id(nir_builder *b)
    232 {
    233    auto patch_id = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_tcs_rel_patch_id_r600);
    234    nir_ssa_dest_init(&patch_id->instr, &patch_id->dest,
    235                      1, 32, NULL);
    236    nir_builder_instr_insert(b, &patch_id->instr);
    237    return &patch_id->dest.ssa;
    238 }
    239 
    240 static void
    241 emit_store_lds(nir_builder *b, nir_intrinsic_instr *op, nir_ssa_def *addr)
    242 {
    243    uint32_t orig_writemask = nir_intrinsic_write_mask(op) << nir_intrinsic_component(op);
    244 
    245    for (int i = 0; i < 2; ++i) {
    246       unsigned test_mask = (0x3 << 2 * i);
    247       if (!(orig_writemask & test_mask))
    248          continue;
    249 
    250       uint32_t writemask =  test_mask >> nir_intrinsic_component(op);
    251 
    252       auto store_tcs_out = nir_intrinsic_instr_create(b->shader, nir_intrinsic_store_local_shared_r600);
    253       nir_intrinsic_set_write_mask(store_tcs_out, writemask);
    254       store_tcs_out->src[0] = nir_src_for_ssa(op->src[0].ssa);
    255       store_tcs_out->num_components = store_tcs_out->src[0].ssa->num_components;
    256       bool start_even = (orig_writemask & (1u << (2 * i)));
    257 
    258       auto addr2 = nir_iadd(b, addr, nir_imm_int(b, 8 * i + (start_even ? 0 : 4)));
    259       store_tcs_out->src[1] = nir_src_for_ssa(addr2);
    260 
    261       nir_builder_instr_insert(b, &store_tcs_out->instr);
    262    }
    263 }
    264 
    265 static nir_ssa_def *
    266 emil_tcs_io_offset(nir_builder *b, nir_ssa_def *addr, nir_intrinsic_instr *op, int src_offset)
    267 {
    268    int offset = get_tcs_varying_offset(op);
    269    return nir_iadd(b, nir_iadd(b, addr,
    270                                nir_ishl(b, op->src[src_offset].ssa, nir_imm_int(b,4))),
    271                                nir_imm_int(b, offset));
    272 }
    273 
    274 
    275 inline unsigned
    276 outer_tf_components(pipe_prim_type prim_type)
    277 {
    278    switch (prim_type) {
    279    case PIPE_PRIM_LINES: return 2;
    280    case PIPE_PRIM_TRIANGLES: return 3;
    281    case PIPE_PRIM_QUADS: return 4;
    282    default:
    283       return 0;
    284    }
    285 }
    286 
    287 
    288 
    289 static bool
    290 r600_lower_tess_io_impl(nir_builder *b, nir_instr *instr, enum pipe_prim_type prim_type)
    291 {
    292    static nir_ssa_def *load_in_param_base = nullptr;
    293    static nir_ssa_def *load_out_param_base = nullptr;
    294 
    295    b->cursor = nir_before_instr(instr);
    296    nir_intrinsic_instr *op = nir_instr_as_intrinsic(instr);
    297 
    298    if (b->shader->info.stage == MESA_SHADER_TESS_CTRL) {
    299       load_in_param_base = emit_load_param_base(b, nir_intrinsic_load_tcs_in_param_base_r600);
    300       load_out_param_base = emit_load_param_base(b, nir_intrinsic_load_tcs_out_param_base_r600);
    301    } else if (b->shader->info.stage == MESA_SHADER_TESS_EVAL) {
    302       load_in_param_base = emit_load_param_base(b, nir_intrinsic_load_tcs_out_param_base_r600);
    303    } else if (b->shader->info.stage == MESA_SHADER_VERTEX) {
    304       load_out_param_base = emit_load_param_base(b, nir_intrinsic_load_tcs_in_param_base_r600);
    305    }
    306 
    307    auto rel_patch_id = r600_load_rel_patch_id(b);
    308 
    309    unsigned tf_inner_address_offset = 0;
    310    unsigned ncomps_correct = 0;
    311 
    312    switch (op->intrinsic) {
    313    case nir_intrinsic_load_patch_vertices_in: {
    314       nir_ssa_def *vertices_in;
    315       if (b->shader->info.stage == MESA_SHADER_TESS_CTRL)
    316          vertices_in = nir_channel(b, load_in_param_base, 2);
    317       else {
    318          auto base = emit_load_param_base(b, nir_intrinsic_load_tcs_in_param_base_r600);
    319          vertices_in = nir_channel(b, base, 2);
    320       }
    321       nir_ssa_def_rewrite_uses(&op->dest.ssa, vertices_in);
    322       nir_instr_remove(&op->instr);
    323       return true;
    324    }
    325    case nir_intrinsic_load_per_vertex_input: {
    326       nir_ssa_def *addr =
    327             b->shader->info.stage == MESA_SHADER_TESS_CTRL ?
    328                emil_lsd_in_addr(b, load_in_param_base, rel_patch_id, op) :
    329                emil_lsd_out_addr(b, load_in_param_base, rel_patch_id, op, nir_var_shader_in, 0);
    330       replace_load_instr(b, op, addr);
    331       return true;
    332    }
    333    case nir_intrinsic_store_per_vertex_output: {
    334       nir_ssa_def *addr = emil_lsd_out_addr(b, load_out_param_base, rel_patch_id, op, nir_var_shader_out, 1);
    335       emit_store_lds(b, op, addr);
    336       nir_instr_remove(instr);
    337       return true;
    338    }
    339    case nir_intrinsic_load_per_vertex_output: {
    340       nir_ssa_def *addr = emil_lsd_out_addr(b, load_out_param_base, rel_patch_id, op, nir_var_shader_out, 0);
    341       replace_load_instr(b, op, addr);
    342       return true;
    343    }
    344    case nir_intrinsic_store_output: {
    345       nir_ssa_def *addr = (b->shader->info.stage == MESA_SHADER_TESS_CTRL) ?
    346                              r600_tcs_base_address(b, load_out_param_base, rel_patch_id):
    347                              nir_build_alu(b, nir_op_umul24,
    348                                            nir_channel(b, load_out_param_base, 1),
    349                                            rel_patch_id, NULL, NULL);
    350       addr = emil_tcs_io_offset(b, addr, op, 1);
    351       emit_store_lds(b, op, addr);
    352       nir_instr_remove(instr);
    353       return true;
    354    }
    355    case nir_intrinsic_load_output: {
    356       nir_ssa_def *addr = r600_tcs_base_address(b, load_out_param_base, rel_patch_id);
    357       addr = emil_tcs_io_offset(b, addr, op, 0);
    358       replace_load_instr(b, op, addr);
    359       return true;
    360    }
    361    case nir_intrinsic_load_input: {
    362       nir_ssa_def *addr = r600_tcs_base_address(b, load_in_param_base, rel_patch_id);
    363       addr = emil_tcs_io_offset(b, addr, op, 0);
    364       replace_load_instr(b, op, addr);
    365       return true;
    366    }
    367    case nir_intrinsic_load_tess_level_inner:
    368       tf_inner_address_offset = 4;
    369       ncomps_correct = 2;
    370       FALLTHROUGH;
    371    case nir_intrinsic_load_tess_level_outer: {
    372       auto ncomps = outer_tf_components(prim_type);
    373       if (!ncomps)
    374          return false;
    375       ncomps -= ncomps_correct;
    376       auto base = emit_load_param_base(b, nir_intrinsic_load_tcs_out_param_base_r600);
    377       auto rel_patch_id = r600_load_rel_patch_id(b);
    378       nir_ssa_def *addr0 = r600_tcs_base_address(b, base, rel_patch_id);
    379       nir_ssa_def *addr_outer = nir_iadd(b, addr0, load_offset_group(b, tf_inner_address_offset + ncomps));
    380 
    381       auto tf = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_local_shared_r600);
    382       tf->num_components = ncomps;
    383       tf->src[0] = nir_src_for_ssa(addr_outer);
    384       nir_ssa_dest_init(&tf->instr, &tf->dest,
    385                         tf->num_components, 32, NULL);
    386       nir_builder_instr_insert(b, &tf->instr);
    387 
    388       nir_ssa_def_rewrite_uses(&op->dest.ssa, &tf->dest.ssa);
    389       nir_instr_remove(instr);
    390       return true;
    391    }
    392    default:
    393       ;
    394    }
    395 
    396    return false;
    397 }
    398 
    399 bool r600_lower_tess_io(nir_shader *shader, enum pipe_prim_type prim_type)
    400 {
    401    bool progress = false;
    402    nir_foreach_function(function, shader) {
    403       if (function->impl) {
    404          nir_builder b;
    405          nir_builder_init(&b, function->impl);
    406 
    407          nir_foreach_block(block, function->impl) {
    408             nir_foreach_instr_safe(instr, block) {
    409                if (instr->type != nir_instr_type_intrinsic)
    410                   continue;
    411 
    412                if (r600_lower_tess_io_filter(instr, shader->info.stage))
    413                   progress |= r600_lower_tess_io_impl(&b, instr, prim_type);
    414             }
    415          }
    416       }
    417    }
    418    return progress;
    419 }
    420 
    421 bool r600_emit_tf(nir_builder *b, nir_ssa_def *val)
    422 {
    423    nir_intrinsic_instr *store_tf = nir_intrinsic_instr_create(b->shader, nir_intrinsic_store_tf_r600);
    424    store_tf->num_components = val->num_components;
    425    store_tf->src[0] = nir_src_for_ssa(val);
    426    nir_builder_instr_insert(b, &store_tf->instr);
    427    return true;
    428 }
    429 
    430 bool r600_append_tcs_TF_emission(nir_shader *shader, enum pipe_prim_type prim_type) {
    431    if (shader->info.stage != MESA_SHADER_TESS_CTRL)
    432       return false;
    433 
    434    nir_foreach_function(function, shader) {
    435       nir_foreach_block(block, function->impl) {
    436          nir_foreach_instr_safe(instr, block) {
    437             if (instr->type != nir_instr_type_intrinsic)
    438                continue;
    439             nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
    440             if (intr->intrinsic == nir_intrinsic_store_tf_r600) {
    441                return false;
    442             }
    443          }
    444       }
    445    }
    446    nir_builder builder;
    447    nir_builder *b = &builder;
    448 
    449    assert(exec_list_length(&shader->functions) == 1);
    450    nir_function *f = (nir_function *)shader->functions.get_head();
    451    nir_builder_init(b, f->impl);
    452 
    453    auto outer_comps = outer_tf_components(prim_type);
    454    if (!outer_comps)
    455       return false;
    456 
    457    unsigned inner_comps = outer_comps - 2;
    458    unsigned stride = (inner_comps + outer_comps) * 4;
    459 
    460    b->cursor = nir_after_cf_list(&f->impl->body);
    461 
    462    auto invocation_id = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_invocation_id);
    463 	nir_ssa_dest_init(&invocation_id->instr, &invocation_id->dest,
    464                      1, 32, NULL);
    465    nir_builder_instr_insert(b, &invocation_id->instr);
    466 
    467    nir_push_if(b, nir_ieq_imm(b, &invocation_id->dest.ssa, 0));
    468    auto base = emit_load_param_base(b, nir_intrinsic_load_tcs_out_param_base_r600);
    469    auto rel_patch_id = r600_load_rel_patch_id(b);
    470 
    471    nir_ssa_def *addr0 = r600_tcs_base_address(b, base, rel_patch_id);
    472 
    473    nir_ssa_def *addr_outer = nir_iadd(b, addr0, load_offset_group(b, outer_comps));
    474    auto tf_outer = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_local_shared_r600);
    475    tf_outer->num_components = outer_comps;
    476    tf_outer->src[0] = nir_src_for_ssa(addr_outer);
    477    nir_ssa_dest_init(&tf_outer->instr, &tf_outer->dest,
    478                      tf_outer->num_components, 32, NULL);
    479    nir_builder_instr_insert(b, &tf_outer->instr);
    480 
    481    std::vector<nir_ssa_def *> tf_out;
    482 
    483 
    484    auto tf_out_base = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_tcs_tess_factor_base_r600);
    485 	nir_ssa_dest_init(&tf_out_base->instr, &tf_out_base->dest,
    486                      1, 32, NULL);
    487    nir_builder_instr_insert(b, &tf_out_base->instr);
    488 
    489    auto out_addr0 = nir_build_alu(b, nir_op_umad24,
    490                                   rel_patch_id,
    491                                   nir_imm_int(b, stride),
    492                                   &tf_out_base->dest.ssa,
    493                                   NULL);
    494    int chanx = 0;
    495    int chany = 1;
    496 
    497    if (prim_type == PIPE_PRIM_LINES)
    498       std::swap(chanx, chany);
    499 
    500 
    501    auto v0 = nir_vec4(b, out_addr0, nir_channel(b, &tf_outer->dest.ssa, chanx),
    502                       nir_iadd(b, out_addr0, nir_imm_int(b, 4)),
    503                       nir_channel(b, &tf_outer->dest.ssa, chany));
    504 
    505    tf_out.push_back(v0);
    506    if (outer_comps > 2) {
    507       auto v1 = (outer_comps > 3) ? nir_vec4(b, nir_iadd(b, out_addr0, nir_imm_int(b, 8)),
    508                                              nir_channel(b, &tf_outer->dest.ssa, 2),
    509                                              nir_iadd(b, out_addr0, nir_imm_int(b, 12)),
    510                                              nir_channel(b, &tf_outer->dest.ssa, 3)) :
    511                                     nir_vec2(b, nir_iadd(b, out_addr0, nir_imm_int(b, 8)),
    512                                              nir_channel(b, &tf_outer->dest.ssa, 2));
    513       tf_out.push_back(v1);
    514    }
    515 
    516    if (inner_comps) {
    517       nir_ssa_def *addr1 = nir_iadd(b, addr0, load_offset_group(b, 4 + inner_comps));
    518       auto tf_inner = nir_intrinsic_instr_create(b->shader, nir_intrinsic_load_local_shared_r600);
    519       tf_inner->num_components = inner_comps;
    520       tf_inner->src[0] = nir_src_for_ssa(addr1);
    521       nir_ssa_dest_init(&tf_inner->instr, &tf_inner->dest,
    522                         tf_inner->num_components, 32, NULL);
    523       nir_builder_instr_insert(b, &tf_inner->instr);
    524 
    525       auto v2 = (inner_comps > 1) ? nir_vec4(b, nir_iadd(b, out_addr0, nir_imm_int(b, 16)),
    526                                              nir_channel(b, &tf_inner->dest.ssa, 0),
    527                                              nir_iadd(b, out_addr0, nir_imm_int(b, 20)),
    528                                              nir_channel(b, &tf_inner->dest.ssa, 1)):
    529                                     nir_vec2(b, nir_iadd(b, out_addr0, nir_imm_int(b, 12)),
    530                                              nir_channel(b, &tf_inner->dest.ssa, 0));
    531       tf_out.push_back(v2);
    532    }
    533 
    534    for (auto tf: tf_out)
    535       r600_emit_tf(b, tf);
    536 
    537    nir_pop_if(b, nullptr);
    538 
    539    nir_metadata_preserve(f->impl, nir_metadata_none);
    540 
    541    return true;
    542 }
    543 
    544 static bool
    545 r600_lower_tess_coord_filter(const nir_instr *instr, UNUSED const void *_options)
    546 {
    547    if (instr->type != nir_instr_type_intrinsic)
    548       return false;
    549    auto intr = nir_instr_as_intrinsic(instr);
    550    return intr->intrinsic == nir_intrinsic_load_tess_coord;
    551 }
    552 
    553 static nir_ssa_def *
    554 r600_lower_tess_coord_impl(nir_builder *b, nir_instr *instr, void *_options)
    555 {
    556    pipe_prim_type prim_type = *(pipe_prim_type *)_options;
    557 
    558    auto tc_xy = nir_load_tess_coord_r600(b);
    559 
    560    auto tc_x = nir_channel(b, tc_xy, 0);
    561    auto tc_y = nir_channel(b, tc_xy, 1);
    562 
    563    if (prim_type == PIPE_PRIM_TRIANGLES)
    564       return nir_vec3(b, tc_x, tc_y, nir_fsub(b, nir_imm_float(b, 1.0),
    565                                               nir_fadd(b, tc_x, tc_y)));
    566    else
    567       return nir_vec3(b, tc_x, tc_y, nir_imm_float(b, 0.0));
    568 }
    569 
    570 
    571 bool r600_lower_tess_coord(nir_shader *sh, enum pipe_prim_type prim_type)
    572 {
    573    return nir_shader_lower_instructions(sh, r600_lower_tess_coord_filter,
    574                                         r600_lower_tess_coord_impl, &prim_type);
    575 }
    576