vtn_variables.c revision 993e1d59
1/* 2 * Copyright © 2015 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Jason Ekstrand (jason@jlekstrand.net) 25 * 26 */ 27 28#include "vtn_private.h" 29#include "spirv_info.h" 30#include "nir_deref.h" 31 32static struct vtn_access_chain * 33vtn_access_chain_create(struct vtn_builder *b, unsigned length) 34{ 35 struct vtn_access_chain *chain; 36 37 /* Subtract 1 from the length since there's already one built in */ 38 size_t size = sizeof(*chain) + 39 (MAX2(length, 1) - 1) * sizeof(chain->link[0]); 40 chain = rzalloc_size(b, size); 41 chain->length = length; 42 43 return chain; 44} 45 46static struct vtn_access_chain * 47vtn_access_chain_extend(struct vtn_builder *b, struct vtn_access_chain *old, 48 unsigned new_ids) 49{ 50 struct vtn_access_chain *chain; 51 52 unsigned old_len = old ? old->length : 0; 53 chain = vtn_access_chain_create(b, old_len + new_ids); 54 55 for (unsigned i = 0; i < old_len; i++) 56 chain->link[i] = old->link[i]; 57 58 return chain; 59} 60 61static bool 62vtn_pointer_uses_ssa_offset(struct vtn_builder *b, 63 struct vtn_pointer *ptr) 64{ 65 return ptr->mode == vtn_variable_mode_ubo || 66 ptr->mode == vtn_variable_mode_ssbo || 67 ptr->mode == vtn_variable_mode_push_constant || 68 (ptr->mode == vtn_variable_mode_workgroup && 69 b->options->lower_workgroup_access_to_offsets); 70} 71 72static bool 73vtn_pointer_is_external_block(struct vtn_builder *b, 74 struct vtn_pointer *ptr) 75{ 76 return ptr->mode == vtn_variable_mode_ssbo || 77 ptr->mode == vtn_variable_mode_ubo || 78 ptr->mode == vtn_variable_mode_push_constant || 79 (ptr->mode == vtn_variable_mode_workgroup && 80 b->options->lower_workgroup_access_to_offsets); 81} 82 83/* Dereference the given base pointer by the access chain */ 84static struct vtn_pointer * 85vtn_access_chain_pointer_dereference(struct vtn_builder *b, 86 struct vtn_pointer *base, 87 struct vtn_access_chain *deref_chain) 88{ 89 struct vtn_access_chain *chain = 90 vtn_access_chain_extend(b, base->chain, deref_chain->length); 91 struct vtn_type *type = base->type; 92 enum gl_access_qualifier access = base->access; 93 94 /* OpPtrAccessChain is only allowed on things which support variable 95 * pointers. For everything else, the client is expected to just pass us 96 * the right access chain. 97 */ 98 vtn_assert(!deref_chain->ptr_as_array); 99 100 unsigned start = base->chain ? base->chain->length : 0; 101 for (unsigned i = 0; i < deref_chain->length; i++) { 102 chain->link[start + i] = deref_chain->link[i]; 103 104 if (glsl_type_is_struct(type->type)) { 105 vtn_assert(deref_chain->link[i].mode == vtn_access_mode_literal); 106 type = type->members[deref_chain->link[i].id]; 107 } else { 108 type = type->array_element; 109 } 110 111 access |= type->access; 112 } 113 114 struct vtn_pointer *ptr = rzalloc(b, struct vtn_pointer); 115 ptr->mode = base->mode; 116 ptr->type = type; 117 ptr->var = base->var; 118 ptr->deref = base->deref; 119 ptr->chain = chain; 120 ptr->access = access; 121 122 return ptr; 123} 124 125static nir_ssa_def * 126vtn_access_link_as_ssa(struct vtn_builder *b, struct vtn_access_link link, 127 unsigned stride) 128{ 129 vtn_assert(stride > 0); 130 if (link.mode == vtn_access_mode_literal) { 131 return nir_imm_int(&b->nb, link.id * stride); 132 } else if (stride == 1) { 133 nir_ssa_def *ssa = vtn_ssa_value(b, link.id)->def; 134 if (ssa->bit_size != 32) 135 ssa = nir_i2i32(&b->nb, ssa); 136 return ssa; 137 } else { 138 nir_ssa_def *src0 = vtn_ssa_value(b, link.id)->def; 139 if (src0->bit_size != 32) 140 src0 = nir_i2i32(&b->nb, src0); 141 return nir_imul(&b->nb, src0, nir_imm_int(&b->nb, stride)); 142 } 143} 144 145static nir_ssa_def * 146vtn_variable_resource_index(struct vtn_builder *b, struct vtn_variable *var, 147 nir_ssa_def *desc_array_index) 148{ 149 if (!desc_array_index) { 150 vtn_assert(glsl_type_is_struct(var->type->type)); 151 desc_array_index = nir_imm_int(&b->nb, 0); 152 } 153 154 nir_intrinsic_instr *instr = 155 nir_intrinsic_instr_create(b->nb.shader, 156 nir_intrinsic_vulkan_resource_index); 157 instr->src[0] = nir_src_for_ssa(desc_array_index); 158 nir_intrinsic_set_desc_set(instr, var->descriptor_set); 159 nir_intrinsic_set_binding(instr, var->binding); 160 161 nir_ssa_dest_init(&instr->instr, &instr->dest, 1, 32, NULL); 162 nir_builder_instr_insert(&b->nb, &instr->instr); 163 164 return &instr->dest.ssa; 165} 166 167static nir_ssa_def * 168vtn_resource_reindex(struct vtn_builder *b, nir_ssa_def *base_index, 169 nir_ssa_def *offset_index) 170{ 171 nir_intrinsic_instr *instr = 172 nir_intrinsic_instr_create(b->nb.shader, 173 nir_intrinsic_vulkan_resource_reindex); 174 instr->src[0] = nir_src_for_ssa(base_index); 175 instr->src[1] = nir_src_for_ssa(offset_index); 176 177 nir_ssa_dest_init(&instr->instr, &instr->dest, 1, 32, NULL); 178 nir_builder_instr_insert(&b->nb, &instr->instr); 179 180 return &instr->dest.ssa; 181} 182 183static struct vtn_pointer * 184vtn_ssa_offset_pointer_dereference(struct vtn_builder *b, 185 struct vtn_pointer *base, 186 struct vtn_access_chain *deref_chain) 187{ 188 nir_ssa_def *block_index = base->block_index; 189 nir_ssa_def *offset = base->offset; 190 struct vtn_type *type = base->type; 191 enum gl_access_qualifier access = base->access; 192 193 unsigned idx = 0; 194 if (base->mode == vtn_variable_mode_ubo || 195 base->mode == vtn_variable_mode_ssbo) { 196 if (!block_index) { 197 vtn_assert(base->var && base->type); 198 nir_ssa_def *desc_arr_idx; 199 if (glsl_type_is_array(type->type)) { 200 if (deref_chain->length >= 1) { 201 desc_arr_idx = 202 vtn_access_link_as_ssa(b, deref_chain->link[0], 1); 203 idx++; 204 /* This consumes a level of type */ 205 type = type->array_element; 206 access |= type->access; 207 } else { 208 /* This is annoying. We've been asked for a pointer to the 209 * array of UBOs/SSBOs and not a specifc buffer. Return a 210 * pointer with a descriptor index of 0 and we'll have to do 211 * a reindex later to adjust it to the right thing. 212 */ 213 desc_arr_idx = nir_imm_int(&b->nb, 0); 214 } 215 } else if (deref_chain->ptr_as_array) { 216 /* You can't have a zero-length OpPtrAccessChain */ 217 vtn_assert(deref_chain->length >= 1); 218 desc_arr_idx = vtn_access_link_as_ssa(b, deref_chain->link[0], 1); 219 } else { 220 /* We have a regular non-array SSBO. */ 221 desc_arr_idx = NULL; 222 } 223 block_index = vtn_variable_resource_index(b, base->var, desc_arr_idx); 224 } else if (deref_chain->ptr_as_array && 225 type->base_type == vtn_base_type_struct && type->block) { 226 /* We are doing an OpPtrAccessChain on a pointer to a struct that is 227 * decorated block. This is an interesting corner in the SPIR-V 228 * spec. One interpretation would be that they client is clearly 229 * trying to treat that block as if it's an implicit array of blocks 230 * repeated in the buffer. However, the SPIR-V spec for the 231 * OpPtrAccessChain says: 232 * 233 * "Base is treated as the address of the first element of an 234 * array, and the Element element’s address is computed to be the 235 * base for the Indexes, as per OpAccessChain." 236 * 237 * Taken literally, that would mean that your struct type is supposed 238 * to be treated as an array of such a struct and, since it's 239 * decorated block, that means an array of blocks which corresponds 240 * to an array descriptor. Therefore, we need to do a reindex 241 * operation to add the index from the first link in the access chain 242 * to the index we recieved. 243 * 244 * The downside to this interpretation (there always is one) is that 245 * this might be somewhat surprising behavior to apps if they expect 246 * the implicit array behavior described above. 247 */ 248 vtn_assert(deref_chain->length >= 1); 249 nir_ssa_def *offset_index = 250 vtn_access_link_as_ssa(b, deref_chain->link[0], 1); 251 idx++; 252 253 block_index = vtn_resource_reindex(b, block_index, offset_index); 254 } 255 } 256 257 if (!offset) { 258 if (base->mode == vtn_variable_mode_workgroup) { 259 /* SLM doesn't need nor have a block index */ 260 vtn_assert(!block_index); 261 262 /* We need the variable for the base offset */ 263 vtn_assert(base->var); 264 265 /* We need ptr_type for size and alignment */ 266 vtn_assert(base->ptr_type); 267 268 /* Assign location on first use so that we don't end up bloating SLM 269 * address space for variables which are never statically used. 270 */ 271 if (base->var->shared_location < 0) { 272 vtn_assert(base->ptr_type->length > 0 && base->ptr_type->align > 0); 273 b->shader->num_shared = vtn_align_u32(b->shader->num_shared, 274 base->ptr_type->align); 275 base->var->shared_location = b->shader->num_shared; 276 b->shader->num_shared += base->ptr_type->length; 277 } 278 279 offset = nir_imm_int(&b->nb, base->var->shared_location); 280 } else if (base->mode == vtn_variable_mode_push_constant) { 281 /* Push constants neither need nor have a block index */ 282 vtn_assert(!block_index); 283 284 /* Start off with at the start of the push constant block. */ 285 offset = nir_imm_int(&b->nb, 0); 286 } else { 287 /* The code above should have ensured a block_index when needed. */ 288 vtn_assert(block_index); 289 290 /* Start off with at the start of the buffer. */ 291 offset = nir_imm_int(&b->nb, 0); 292 } 293 } 294 295 if (deref_chain->ptr_as_array && idx == 0) { 296 /* We need ptr_type for the stride */ 297 vtn_assert(base->ptr_type); 298 299 /* We need at least one element in the chain */ 300 vtn_assert(deref_chain->length >= 1); 301 302 nir_ssa_def *elem_offset = 303 vtn_access_link_as_ssa(b, deref_chain->link[idx], 304 base->ptr_type->stride); 305 offset = nir_iadd(&b->nb, offset, elem_offset); 306 idx++; 307 } 308 309 for (; idx < deref_chain->length; idx++) { 310 switch (glsl_get_base_type(type->type)) { 311 case GLSL_TYPE_UINT: 312 case GLSL_TYPE_INT: 313 case GLSL_TYPE_UINT16: 314 case GLSL_TYPE_INT16: 315 case GLSL_TYPE_UINT8: 316 case GLSL_TYPE_INT8: 317 case GLSL_TYPE_UINT64: 318 case GLSL_TYPE_INT64: 319 case GLSL_TYPE_FLOAT: 320 case GLSL_TYPE_FLOAT16: 321 case GLSL_TYPE_DOUBLE: 322 case GLSL_TYPE_BOOL: 323 case GLSL_TYPE_ARRAY: { 324 nir_ssa_def *elem_offset = 325 vtn_access_link_as_ssa(b, deref_chain->link[idx], type->stride); 326 offset = nir_iadd(&b->nb, offset, elem_offset); 327 type = type->array_element; 328 access |= type->access; 329 break; 330 } 331 332 case GLSL_TYPE_STRUCT: { 333 vtn_assert(deref_chain->link[idx].mode == vtn_access_mode_literal); 334 unsigned member = deref_chain->link[idx].id; 335 nir_ssa_def *mem_offset = nir_imm_int(&b->nb, type->offsets[member]); 336 offset = nir_iadd(&b->nb, offset, mem_offset); 337 type = type->members[member]; 338 access |= type->access; 339 break; 340 } 341 342 default: 343 vtn_fail("Invalid type for deref"); 344 } 345 } 346 347 struct vtn_pointer *ptr = rzalloc(b, struct vtn_pointer); 348 ptr->mode = base->mode; 349 ptr->type = type; 350 ptr->block_index = block_index; 351 ptr->offset = offset; 352 ptr->access = access; 353 354 return ptr; 355} 356 357/* Dereference the given base pointer by the access chain */ 358static struct vtn_pointer * 359vtn_pointer_dereference(struct vtn_builder *b, 360 struct vtn_pointer *base, 361 struct vtn_access_chain *deref_chain) 362{ 363 if (vtn_pointer_uses_ssa_offset(b, base)) { 364 return vtn_ssa_offset_pointer_dereference(b, base, deref_chain); 365 } else { 366 return vtn_access_chain_pointer_dereference(b, base, deref_chain); 367 } 368} 369 370struct vtn_pointer * 371vtn_pointer_for_variable(struct vtn_builder *b, 372 struct vtn_variable *var, struct vtn_type *ptr_type) 373{ 374 struct vtn_pointer *pointer = rzalloc(b, struct vtn_pointer); 375 376 pointer->mode = var->mode; 377 pointer->type = var->type; 378 vtn_assert(ptr_type->base_type == vtn_base_type_pointer); 379 vtn_assert(ptr_type->deref->type == var->type->type); 380 pointer->ptr_type = ptr_type; 381 pointer->var = var; 382 pointer->access = var->access | var->type->access; 383 384 return pointer; 385} 386 387/* Returns an atomic_uint type based on the original uint type. The returned 388 * type will be equivalent to the original one but will have an atomic_uint 389 * type as leaf instead of an uint. 390 * 391 * Manages uint scalars, arrays, and arrays of arrays of any nested depth. 392 */ 393static const struct glsl_type * 394repair_atomic_type(const struct glsl_type *type) 395{ 396 assert(glsl_get_base_type(glsl_without_array(type)) == GLSL_TYPE_UINT); 397 assert(glsl_type_is_scalar(glsl_without_array(type))); 398 399 if (glsl_type_is_array(type)) { 400 const struct glsl_type *atomic = 401 repair_atomic_type(glsl_get_array_element(type)); 402 403 return glsl_array_type(atomic, glsl_get_length(type)); 404 } else { 405 return glsl_atomic_uint_type(); 406 } 407} 408 409nir_deref_instr * 410vtn_pointer_to_deref(struct vtn_builder *b, struct vtn_pointer *ptr) 411{ 412 /* Do on-the-fly copy propagation for samplers. */ 413 if (ptr->var && ptr->var->copy_prop_sampler) 414 return vtn_pointer_to_deref(b, ptr->var->copy_prop_sampler); 415 416 nir_deref_instr *tail; 417 if (ptr->deref) { 418 tail = ptr->deref; 419 } else { 420 assert(ptr->var && ptr->var->var); 421 tail = nir_build_deref_var(&b->nb, ptr->var->var); 422 } 423 424 /* Raw variable access */ 425 if (!ptr->chain) 426 return tail; 427 428 struct vtn_access_chain *chain = ptr->chain; 429 vtn_assert(chain); 430 431 for (unsigned i = 0; i < chain->length; i++) { 432 if (glsl_type_is_struct(tail->type)) { 433 vtn_assert(chain->link[i].mode == vtn_access_mode_literal); 434 unsigned idx = chain->link[i].id; 435 tail = nir_build_deref_struct(&b->nb, tail, idx); 436 } else { 437 nir_ssa_def *index; 438 if (chain->link[i].mode == vtn_access_mode_literal) { 439 index = nir_imm_int(&b->nb, chain->link[i].id); 440 } else { 441 vtn_assert(chain->link[i].mode == vtn_access_mode_id); 442 index = vtn_ssa_value(b, chain->link[i].id)->def; 443 } 444 tail = nir_build_deref_array(&b->nb, tail, index); 445 } 446 } 447 448 return tail; 449} 450 451static void 452_vtn_local_load_store(struct vtn_builder *b, bool load, nir_deref_instr *deref, 453 struct vtn_ssa_value *inout) 454{ 455 if (glsl_type_is_vector_or_scalar(deref->type)) { 456 if (load) { 457 inout->def = nir_load_deref(&b->nb, deref); 458 } else { 459 nir_store_deref(&b->nb, deref, inout->def, ~0); 460 } 461 } else if (glsl_type_is_array(deref->type) || 462 glsl_type_is_matrix(deref->type)) { 463 unsigned elems = glsl_get_length(deref->type); 464 for (unsigned i = 0; i < elems; i++) { 465 nir_deref_instr *child = 466 nir_build_deref_array(&b->nb, deref, nir_imm_int(&b->nb, i)); 467 _vtn_local_load_store(b, load, child, inout->elems[i]); 468 } 469 } else { 470 vtn_assert(glsl_type_is_struct(deref->type)); 471 unsigned elems = glsl_get_length(deref->type); 472 for (unsigned i = 0; i < elems; i++) { 473 nir_deref_instr *child = nir_build_deref_struct(&b->nb, deref, i); 474 _vtn_local_load_store(b, load, child, inout->elems[i]); 475 } 476 } 477} 478 479nir_deref_instr * 480vtn_nir_deref(struct vtn_builder *b, uint32_t id) 481{ 482 struct vtn_pointer *ptr = vtn_value(b, id, vtn_value_type_pointer)->pointer; 483 return vtn_pointer_to_deref(b, ptr); 484} 485 486/* 487 * Gets the NIR-level deref tail, which may have as a child an array deref 488 * selecting which component due to OpAccessChain supporting per-component 489 * indexing in SPIR-V. 490 */ 491static nir_deref_instr * 492get_deref_tail(nir_deref_instr *deref) 493{ 494 if (deref->deref_type != nir_deref_type_array) 495 return deref; 496 497 nir_deref_instr *parent = 498 nir_instr_as_deref(deref->parent.ssa->parent_instr); 499 500 if (glsl_type_is_vector(parent->type)) 501 return parent; 502 else 503 return deref; 504} 505 506struct vtn_ssa_value * 507vtn_local_load(struct vtn_builder *b, nir_deref_instr *src) 508{ 509 nir_deref_instr *src_tail = get_deref_tail(src); 510 struct vtn_ssa_value *val = vtn_create_ssa_value(b, src_tail->type); 511 _vtn_local_load_store(b, true, src_tail, val); 512 513 if (src_tail != src) { 514 val->type = src->type; 515 if (nir_src_is_const(src->arr.index)) 516 val->def = vtn_vector_extract(b, val->def, 517 nir_src_as_uint(src->arr.index)); 518 else 519 val->def = vtn_vector_extract_dynamic(b, val->def, src->arr.index.ssa); 520 } 521 522 return val; 523} 524 525void 526vtn_local_store(struct vtn_builder *b, struct vtn_ssa_value *src, 527 nir_deref_instr *dest) 528{ 529 nir_deref_instr *dest_tail = get_deref_tail(dest); 530 531 if (dest_tail != dest) { 532 struct vtn_ssa_value *val = vtn_create_ssa_value(b, dest_tail->type); 533 _vtn_local_load_store(b, true, dest_tail, val); 534 535 if (nir_src_is_const(dest->arr.index)) 536 val->def = vtn_vector_insert(b, val->def, src->def, 537 nir_src_as_uint(dest->arr.index)); 538 else 539 val->def = vtn_vector_insert_dynamic(b, val->def, src->def, 540 dest->arr.index.ssa); 541 _vtn_local_load_store(b, false, dest_tail, val); 542 } else { 543 _vtn_local_load_store(b, false, dest_tail, src); 544 } 545} 546 547nir_ssa_def * 548vtn_pointer_to_offset(struct vtn_builder *b, struct vtn_pointer *ptr, 549 nir_ssa_def **index_out) 550{ 551 assert(vtn_pointer_uses_ssa_offset(b, ptr)); 552 if (!ptr->offset) { 553 struct vtn_access_chain chain = { 554 .length = 0, 555 }; 556 ptr = vtn_ssa_offset_pointer_dereference(b, ptr, &chain); 557 } 558 *index_out = ptr->block_index; 559 return ptr->offset; 560} 561 562/* Tries to compute the size of an interface block based on the strides and 563 * offsets that are provided to us in the SPIR-V source. 564 */ 565static unsigned 566vtn_type_block_size(struct vtn_builder *b, struct vtn_type *type) 567{ 568 enum glsl_base_type base_type = glsl_get_base_type(type->type); 569 switch (base_type) { 570 case GLSL_TYPE_UINT: 571 case GLSL_TYPE_INT: 572 case GLSL_TYPE_UINT16: 573 case GLSL_TYPE_INT16: 574 case GLSL_TYPE_UINT8: 575 case GLSL_TYPE_INT8: 576 case GLSL_TYPE_UINT64: 577 case GLSL_TYPE_INT64: 578 case GLSL_TYPE_FLOAT: 579 case GLSL_TYPE_FLOAT16: 580 case GLSL_TYPE_BOOL: 581 case GLSL_TYPE_DOUBLE: { 582 unsigned cols = type->row_major ? glsl_get_vector_elements(type->type) : 583 glsl_get_matrix_columns(type->type); 584 if (cols > 1) { 585 vtn_assert(type->stride > 0); 586 return type->stride * cols; 587 } else { 588 unsigned type_size = glsl_get_bit_size(type->type) / 8; 589 return glsl_get_vector_elements(type->type) * type_size; 590 } 591 } 592 593 case GLSL_TYPE_STRUCT: 594 case GLSL_TYPE_INTERFACE: { 595 unsigned size = 0; 596 unsigned num_fields = glsl_get_length(type->type); 597 for (unsigned f = 0; f < num_fields; f++) { 598 unsigned field_end = type->offsets[f] + 599 vtn_type_block_size(b, type->members[f]); 600 size = MAX2(size, field_end); 601 } 602 return size; 603 } 604 605 case GLSL_TYPE_ARRAY: 606 vtn_assert(type->stride > 0); 607 vtn_assert(glsl_get_length(type->type) > 0); 608 return type->stride * glsl_get_length(type->type); 609 610 default: 611 vtn_fail("Invalid block type"); 612 return 0; 613 } 614} 615 616static void 617_vtn_load_store_tail(struct vtn_builder *b, nir_intrinsic_op op, bool load, 618 nir_ssa_def *index, nir_ssa_def *offset, 619 unsigned access_offset, unsigned access_size, 620 struct vtn_ssa_value **inout, const struct glsl_type *type, 621 enum gl_access_qualifier access) 622{ 623 nir_intrinsic_instr *instr = nir_intrinsic_instr_create(b->nb.shader, op); 624 instr->num_components = glsl_get_vector_elements(type); 625 626 int src = 0; 627 if (!load) { 628 nir_intrinsic_set_write_mask(instr, (1 << instr->num_components) - 1); 629 instr->src[src++] = nir_src_for_ssa((*inout)->def); 630 } 631 632 if (op == nir_intrinsic_load_push_constant) { 633 nir_intrinsic_set_base(instr, access_offset); 634 nir_intrinsic_set_range(instr, access_size); 635 } 636 637 if (op == nir_intrinsic_load_ssbo || 638 op == nir_intrinsic_store_ssbo) { 639 nir_intrinsic_set_access(instr, access); 640 } 641 642 if (index) 643 instr->src[src++] = nir_src_for_ssa(index); 644 645 if (op == nir_intrinsic_load_push_constant) { 646 /* We need to subtract the offset from where the intrinsic will load the 647 * data. */ 648 instr->src[src++] = 649 nir_src_for_ssa(nir_isub(&b->nb, offset, 650 nir_imm_int(&b->nb, access_offset))); 651 } else { 652 instr->src[src++] = nir_src_for_ssa(offset); 653 } 654 655 if (load) { 656 nir_ssa_dest_init(&instr->instr, &instr->dest, 657 instr->num_components, 658 glsl_get_bit_size(type), NULL); 659 (*inout)->def = &instr->dest.ssa; 660 } 661 662 nir_builder_instr_insert(&b->nb, &instr->instr); 663 664 if (load && glsl_get_base_type(type) == GLSL_TYPE_BOOL) 665 (*inout)->def = nir_ine(&b->nb, (*inout)->def, nir_imm_int(&b->nb, 0)); 666} 667 668static void 669_vtn_block_load_store(struct vtn_builder *b, nir_intrinsic_op op, bool load, 670 nir_ssa_def *index, nir_ssa_def *offset, 671 unsigned access_offset, unsigned access_size, 672 struct vtn_type *type, enum gl_access_qualifier access, 673 struct vtn_ssa_value **inout) 674{ 675 if (load && *inout == NULL) 676 *inout = vtn_create_ssa_value(b, type->type); 677 678 enum glsl_base_type base_type = glsl_get_base_type(type->type); 679 switch (base_type) { 680 case GLSL_TYPE_UINT: 681 case GLSL_TYPE_INT: 682 case GLSL_TYPE_UINT16: 683 case GLSL_TYPE_INT16: 684 case GLSL_TYPE_UINT8: 685 case GLSL_TYPE_INT8: 686 case GLSL_TYPE_UINT64: 687 case GLSL_TYPE_INT64: 688 case GLSL_TYPE_FLOAT: 689 case GLSL_TYPE_FLOAT16: 690 case GLSL_TYPE_DOUBLE: 691 case GLSL_TYPE_BOOL: 692 /* This is where things get interesting. At this point, we've hit 693 * a vector, a scalar, or a matrix. 694 */ 695 if (glsl_type_is_matrix(type->type)) { 696 /* Loading the whole matrix */ 697 struct vtn_ssa_value *transpose; 698 unsigned num_ops, vec_width, col_stride; 699 if (type->row_major) { 700 num_ops = glsl_get_vector_elements(type->type); 701 vec_width = glsl_get_matrix_columns(type->type); 702 col_stride = type->array_element->stride; 703 if (load) { 704 const struct glsl_type *transpose_type = 705 glsl_matrix_type(base_type, vec_width, num_ops); 706 *inout = vtn_create_ssa_value(b, transpose_type); 707 } else { 708 transpose = vtn_ssa_transpose(b, *inout); 709 inout = &transpose; 710 } 711 } else { 712 num_ops = glsl_get_matrix_columns(type->type); 713 vec_width = glsl_get_vector_elements(type->type); 714 col_stride = type->stride; 715 } 716 717 for (unsigned i = 0; i < num_ops; i++) { 718 nir_ssa_def *elem_offset = 719 nir_iadd(&b->nb, offset, nir_imm_int(&b->nb, i * col_stride)); 720 _vtn_load_store_tail(b, op, load, index, elem_offset, 721 access_offset, access_size, 722 &(*inout)->elems[i], 723 glsl_vector_type(base_type, vec_width), 724 type->access | access); 725 } 726 727 if (load && type->row_major) 728 *inout = vtn_ssa_transpose(b, *inout); 729 } else { 730 unsigned elems = glsl_get_vector_elements(type->type); 731 unsigned type_size = glsl_get_bit_size(type->type) / 8; 732 if (elems == 1 || type->stride == type_size) { 733 /* This is a tightly-packed normal scalar or vector load */ 734 vtn_assert(glsl_type_is_vector_or_scalar(type->type)); 735 _vtn_load_store_tail(b, op, load, index, offset, 736 access_offset, access_size, 737 inout, type->type, 738 type->access | access); 739 } else { 740 /* This is a strided load. We have to load N things separately. 741 * This is the single column of a row-major matrix case. 742 */ 743 vtn_assert(type->stride > type_size); 744 vtn_assert(type->stride % type_size == 0); 745 746 nir_ssa_def *per_comp[4]; 747 for (unsigned i = 0; i < elems; i++) { 748 nir_ssa_def *elem_offset = 749 nir_iadd(&b->nb, offset, 750 nir_imm_int(&b->nb, i * type->stride)); 751 struct vtn_ssa_value *comp, temp_val; 752 if (!load) { 753 temp_val.def = nir_channel(&b->nb, (*inout)->def, i); 754 temp_val.type = glsl_scalar_type(base_type); 755 } 756 comp = &temp_val; 757 _vtn_load_store_tail(b, op, load, index, elem_offset, 758 access_offset, access_size, 759 &comp, glsl_scalar_type(base_type), 760 type->access | access); 761 per_comp[i] = comp->def; 762 } 763 764 if (load) { 765 if (*inout == NULL) 766 *inout = vtn_create_ssa_value(b, type->type); 767 (*inout)->def = nir_vec(&b->nb, per_comp, elems); 768 } 769 } 770 } 771 return; 772 773 case GLSL_TYPE_ARRAY: { 774 unsigned elems = glsl_get_length(type->type); 775 for (unsigned i = 0; i < elems; i++) { 776 nir_ssa_def *elem_off = 777 nir_iadd(&b->nb, offset, nir_imm_int(&b->nb, i * type->stride)); 778 _vtn_block_load_store(b, op, load, index, elem_off, 779 access_offset, access_size, 780 type->array_element, 781 type->array_element->access | access, 782 &(*inout)->elems[i]); 783 } 784 return; 785 } 786 787 case GLSL_TYPE_STRUCT: { 788 unsigned elems = glsl_get_length(type->type); 789 for (unsigned i = 0; i < elems; i++) { 790 nir_ssa_def *elem_off = 791 nir_iadd(&b->nb, offset, nir_imm_int(&b->nb, type->offsets[i])); 792 _vtn_block_load_store(b, op, load, index, elem_off, 793 access_offset, access_size, 794 type->members[i], 795 type->members[i]->access | access, 796 &(*inout)->elems[i]); 797 } 798 return; 799 } 800 801 default: 802 vtn_fail("Invalid block member type"); 803 } 804} 805 806static struct vtn_ssa_value * 807vtn_block_load(struct vtn_builder *b, struct vtn_pointer *src) 808{ 809 nir_intrinsic_op op; 810 unsigned access_offset = 0, access_size = 0; 811 switch (src->mode) { 812 case vtn_variable_mode_ubo: 813 op = nir_intrinsic_load_ubo; 814 break; 815 case vtn_variable_mode_ssbo: 816 op = nir_intrinsic_load_ssbo; 817 break; 818 case vtn_variable_mode_push_constant: 819 op = nir_intrinsic_load_push_constant; 820 access_size = b->shader->num_uniforms; 821 break; 822 case vtn_variable_mode_workgroup: 823 op = nir_intrinsic_load_shared; 824 break; 825 default: 826 vtn_fail("Invalid block variable mode"); 827 } 828 829 nir_ssa_def *offset, *index = NULL; 830 offset = vtn_pointer_to_offset(b, src, &index); 831 832 struct vtn_ssa_value *value = NULL; 833 _vtn_block_load_store(b, op, true, index, offset, 834 access_offset, access_size, 835 src->type, src->access, &value); 836 return value; 837} 838 839static void 840vtn_block_store(struct vtn_builder *b, struct vtn_ssa_value *src, 841 struct vtn_pointer *dst) 842{ 843 nir_intrinsic_op op; 844 switch (dst->mode) { 845 case vtn_variable_mode_ssbo: 846 op = nir_intrinsic_store_ssbo; 847 break; 848 case vtn_variable_mode_workgroup: 849 op = nir_intrinsic_store_shared; 850 break; 851 default: 852 vtn_fail("Invalid block variable mode"); 853 } 854 855 nir_ssa_def *offset, *index = NULL; 856 offset = vtn_pointer_to_offset(b, dst, &index); 857 858 _vtn_block_load_store(b, op, false, index, offset, 859 0, 0, dst->type, dst->access, &src); 860} 861 862static void 863_vtn_variable_load_store(struct vtn_builder *b, bool load, 864 struct vtn_pointer *ptr, 865 struct vtn_ssa_value **inout) 866{ 867 enum glsl_base_type base_type = glsl_get_base_type(ptr->type->type); 868 switch (base_type) { 869 case GLSL_TYPE_UINT: 870 case GLSL_TYPE_INT: 871 case GLSL_TYPE_UINT16: 872 case GLSL_TYPE_INT16: 873 case GLSL_TYPE_UINT8: 874 case GLSL_TYPE_INT8: 875 case GLSL_TYPE_UINT64: 876 case GLSL_TYPE_INT64: 877 case GLSL_TYPE_FLOAT: 878 case GLSL_TYPE_FLOAT16: 879 case GLSL_TYPE_BOOL: 880 case GLSL_TYPE_DOUBLE: 881 /* At this point, we have a scalar, vector, or matrix so we know that 882 * there cannot be any structure splitting still in the way. By 883 * stopping at the matrix level rather than the vector level, we 884 * ensure that matrices get loaded in the optimal way even if they 885 * are storred row-major in a UBO. 886 */ 887 if (load) { 888 *inout = vtn_local_load(b, vtn_pointer_to_deref(b, ptr)); 889 } else { 890 vtn_local_store(b, *inout, vtn_pointer_to_deref(b, ptr)); 891 } 892 return; 893 894 case GLSL_TYPE_ARRAY: 895 case GLSL_TYPE_STRUCT: { 896 unsigned elems = glsl_get_length(ptr->type->type); 897 if (load) { 898 vtn_assert(*inout == NULL); 899 *inout = rzalloc(b, struct vtn_ssa_value); 900 (*inout)->type = ptr->type->type; 901 (*inout)->elems = rzalloc_array(b, struct vtn_ssa_value *, elems); 902 } 903 904 struct vtn_access_chain chain = { 905 .length = 1, 906 .link = { 907 { .mode = vtn_access_mode_literal, }, 908 } 909 }; 910 for (unsigned i = 0; i < elems; i++) { 911 chain.link[0].id = i; 912 struct vtn_pointer *elem = vtn_pointer_dereference(b, ptr, &chain); 913 _vtn_variable_load_store(b, load, elem, &(*inout)->elems[i]); 914 } 915 return; 916 } 917 918 default: 919 vtn_fail("Invalid access chain type"); 920 } 921} 922 923struct vtn_ssa_value * 924vtn_variable_load(struct vtn_builder *b, struct vtn_pointer *src) 925{ 926 if (vtn_pointer_is_external_block(b, src)) { 927 return vtn_block_load(b, src); 928 } else { 929 struct vtn_ssa_value *val = NULL; 930 _vtn_variable_load_store(b, true, src, &val); 931 return val; 932 } 933} 934 935void 936vtn_variable_store(struct vtn_builder *b, struct vtn_ssa_value *src, 937 struct vtn_pointer *dest) 938{ 939 if (vtn_pointer_is_external_block(b, dest)) { 940 vtn_assert(dest->mode == vtn_variable_mode_ssbo || 941 dest->mode == vtn_variable_mode_workgroup); 942 vtn_block_store(b, src, dest); 943 } else { 944 _vtn_variable_load_store(b, false, dest, &src); 945 } 946} 947 948static void 949_vtn_variable_copy(struct vtn_builder *b, struct vtn_pointer *dest, 950 struct vtn_pointer *src) 951{ 952 vtn_assert(src->type->type == dest->type->type); 953 enum glsl_base_type base_type = glsl_get_base_type(src->type->type); 954 switch (base_type) { 955 case GLSL_TYPE_UINT: 956 case GLSL_TYPE_INT: 957 case GLSL_TYPE_UINT16: 958 case GLSL_TYPE_INT16: 959 case GLSL_TYPE_UINT8: 960 case GLSL_TYPE_INT8: 961 case GLSL_TYPE_UINT64: 962 case GLSL_TYPE_INT64: 963 case GLSL_TYPE_FLOAT: 964 case GLSL_TYPE_FLOAT16: 965 case GLSL_TYPE_DOUBLE: 966 case GLSL_TYPE_BOOL: 967 /* At this point, we have a scalar, vector, or matrix so we know that 968 * there cannot be any structure splitting still in the way. By 969 * stopping at the matrix level rather than the vector level, we 970 * ensure that matrices get loaded in the optimal way even if they 971 * are storred row-major in a UBO. 972 */ 973 vtn_variable_store(b, vtn_variable_load(b, src), dest); 974 return; 975 976 case GLSL_TYPE_ARRAY: 977 case GLSL_TYPE_STRUCT: { 978 struct vtn_access_chain chain = { 979 .length = 1, 980 .link = { 981 { .mode = vtn_access_mode_literal, }, 982 } 983 }; 984 unsigned elems = glsl_get_length(src->type->type); 985 for (unsigned i = 0; i < elems; i++) { 986 chain.link[0].id = i; 987 struct vtn_pointer *src_elem = 988 vtn_pointer_dereference(b, src, &chain); 989 struct vtn_pointer *dest_elem = 990 vtn_pointer_dereference(b, dest, &chain); 991 992 _vtn_variable_copy(b, dest_elem, src_elem); 993 } 994 return; 995 } 996 997 default: 998 vtn_fail("Invalid access chain type"); 999 } 1000} 1001 1002static void 1003vtn_variable_copy(struct vtn_builder *b, struct vtn_pointer *dest, 1004 struct vtn_pointer *src) 1005{ 1006 /* TODO: At some point, we should add a special-case for when we can 1007 * just emit a copy_var intrinsic. 1008 */ 1009 _vtn_variable_copy(b, dest, src); 1010} 1011 1012static void 1013set_mode_system_value(struct vtn_builder *b, nir_variable_mode *mode) 1014{ 1015 vtn_assert(*mode == nir_var_system_value || *mode == nir_var_shader_in); 1016 *mode = nir_var_system_value; 1017} 1018 1019static void 1020vtn_get_builtin_location(struct vtn_builder *b, 1021 SpvBuiltIn builtin, int *location, 1022 nir_variable_mode *mode) 1023{ 1024 switch (builtin) { 1025 case SpvBuiltInPosition: 1026 *location = VARYING_SLOT_POS; 1027 break; 1028 case SpvBuiltInPointSize: 1029 *location = VARYING_SLOT_PSIZ; 1030 break; 1031 case SpvBuiltInClipDistance: 1032 *location = VARYING_SLOT_CLIP_DIST0; /* XXX CLIP_DIST1? */ 1033 break; 1034 case SpvBuiltInCullDistance: 1035 *location = VARYING_SLOT_CULL_DIST0; 1036 break; 1037 case SpvBuiltInVertexId: 1038 case SpvBuiltInVertexIndex: 1039 /* The Vulkan spec defines VertexIndex to be non-zero-based and doesn't 1040 * allow VertexId. The ARB_gl_spirv spec defines VertexId to be the 1041 * same as gl_VertexID, which is non-zero-based, and removes 1042 * VertexIndex. Since they're both defined to be non-zero-based, we use 1043 * SYSTEM_VALUE_VERTEX_ID for both. 1044 */ 1045 *location = SYSTEM_VALUE_VERTEX_ID; 1046 set_mode_system_value(b, mode); 1047 break; 1048 case SpvBuiltInInstanceIndex: 1049 *location = SYSTEM_VALUE_INSTANCE_INDEX; 1050 set_mode_system_value(b, mode); 1051 break; 1052 case SpvBuiltInInstanceId: 1053 *location = SYSTEM_VALUE_INSTANCE_ID; 1054 set_mode_system_value(b, mode); 1055 break; 1056 case SpvBuiltInPrimitiveId: 1057 if (b->shader->info.stage == MESA_SHADER_FRAGMENT) { 1058 vtn_assert(*mode == nir_var_shader_in); 1059 *location = VARYING_SLOT_PRIMITIVE_ID; 1060 } else if (*mode == nir_var_shader_out) { 1061 *location = VARYING_SLOT_PRIMITIVE_ID; 1062 } else { 1063 *location = SYSTEM_VALUE_PRIMITIVE_ID; 1064 set_mode_system_value(b, mode); 1065 } 1066 break; 1067 case SpvBuiltInInvocationId: 1068 *location = SYSTEM_VALUE_INVOCATION_ID; 1069 set_mode_system_value(b, mode); 1070 break; 1071 case SpvBuiltInLayer: 1072 *location = VARYING_SLOT_LAYER; 1073 if (b->shader->info.stage == MESA_SHADER_FRAGMENT) 1074 *mode = nir_var_shader_in; 1075 else if (b->shader->info.stage == MESA_SHADER_GEOMETRY) 1076 *mode = nir_var_shader_out; 1077 else if (b->options && b->options->caps.shader_viewport_index_layer && 1078 (b->shader->info.stage == MESA_SHADER_VERTEX || 1079 b->shader->info.stage == MESA_SHADER_TESS_EVAL)) 1080 *mode = nir_var_shader_out; 1081 else 1082 vtn_fail("invalid stage for SpvBuiltInLayer"); 1083 break; 1084 case SpvBuiltInViewportIndex: 1085 *location = VARYING_SLOT_VIEWPORT; 1086 if (b->shader->info.stage == MESA_SHADER_GEOMETRY) 1087 *mode = nir_var_shader_out; 1088 else if (b->options && b->options->caps.shader_viewport_index_layer && 1089 (b->shader->info.stage == MESA_SHADER_VERTEX || 1090 b->shader->info.stage == MESA_SHADER_TESS_EVAL)) 1091 *mode = nir_var_shader_out; 1092 else if (b->shader->info.stage == MESA_SHADER_FRAGMENT) 1093 *mode = nir_var_shader_in; 1094 else 1095 vtn_fail("invalid stage for SpvBuiltInViewportIndex"); 1096 break; 1097 case SpvBuiltInTessLevelOuter: 1098 *location = VARYING_SLOT_TESS_LEVEL_OUTER; 1099 break; 1100 case SpvBuiltInTessLevelInner: 1101 *location = VARYING_SLOT_TESS_LEVEL_INNER; 1102 break; 1103 case SpvBuiltInTessCoord: 1104 *location = SYSTEM_VALUE_TESS_COORD; 1105 set_mode_system_value(b, mode); 1106 break; 1107 case SpvBuiltInPatchVertices: 1108 *location = SYSTEM_VALUE_VERTICES_IN; 1109 set_mode_system_value(b, mode); 1110 break; 1111 case SpvBuiltInFragCoord: 1112 *location = VARYING_SLOT_POS; 1113 vtn_assert(*mode == nir_var_shader_in); 1114 break; 1115 case SpvBuiltInPointCoord: 1116 *location = VARYING_SLOT_PNTC; 1117 vtn_assert(*mode == nir_var_shader_in); 1118 break; 1119 case SpvBuiltInFrontFacing: 1120 *location = SYSTEM_VALUE_FRONT_FACE; 1121 set_mode_system_value(b, mode); 1122 break; 1123 case SpvBuiltInSampleId: 1124 *location = SYSTEM_VALUE_SAMPLE_ID; 1125 set_mode_system_value(b, mode); 1126 break; 1127 case SpvBuiltInSamplePosition: 1128 *location = SYSTEM_VALUE_SAMPLE_POS; 1129 set_mode_system_value(b, mode); 1130 break; 1131 case SpvBuiltInSampleMask: 1132 if (*mode == nir_var_shader_out) { 1133 *location = FRAG_RESULT_SAMPLE_MASK; 1134 } else { 1135 *location = SYSTEM_VALUE_SAMPLE_MASK_IN; 1136 set_mode_system_value(b, mode); 1137 } 1138 break; 1139 case SpvBuiltInFragDepth: 1140 *location = FRAG_RESULT_DEPTH; 1141 vtn_assert(*mode == nir_var_shader_out); 1142 break; 1143 case SpvBuiltInHelperInvocation: 1144 *location = SYSTEM_VALUE_HELPER_INVOCATION; 1145 set_mode_system_value(b, mode); 1146 break; 1147 case SpvBuiltInNumWorkgroups: 1148 *location = SYSTEM_VALUE_NUM_WORK_GROUPS; 1149 set_mode_system_value(b, mode); 1150 break; 1151 case SpvBuiltInWorkgroupSize: 1152 *location = SYSTEM_VALUE_LOCAL_GROUP_SIZE; 1153 set_mode_system_value(b, mode); 1154 break; 1155 case SpvBuiltInWorkgroupId: 1156 *location = SYSTEM_VALUE_WORK_GROUP_ID; 1157 set_mode_system_value(b, mode); 1158 break; 1159 case SpvBuiltInLocalInvocationId: 1160 *location = SYSTEM_VALUE_LOCAL_INVOCATION_ID; 1161 set_mode_system_value(b, mode); 1162 break; 1163 case SpvBuiltInLocalInvocationIndex: 1164 *location = SYSTEM_VALUE_LOCAL_INVOCATION_INDEX; 1165 set_mode_system_value(b, mode); 1166 break; 1167 case SpvBuiltInGlobalInvocationId: 1168 *location = SYSTEM_VALUE_GLOBAL_INVOCATION_ID; 1169 set_mode_system_value(b, mode); 1170 break; 1171 case SpvBuiltInBaseVertex: 1172 /* OpenGL gl_BaseVertex (SYSTEM_VALUE_BASE_VERTEX) is not the same 1173 * semantic as SPIR-V BaseVertex (SYSTEM_VALUE_FIRST_VERTEX). 1174 */ 1175 *location = SYSTEM_VALUE_FIRST_VERTEX; 1176 set_mode_system_value(b, mode); 1177 break; 1178 case SpvBuiltInBaseInstance: 1179 *location = SYSTEM_VALUE_BASE_INSTANCE; 1180 set_mode_system_value(b, mode); 1181 break; 1182 case SpvBuiltInDrawIndex: 1183 *location = SYSTEM_VALUE_DRAW_ID; 1184 set_mode_system_value(b, mode); 1185 break; 1186 case SpvBuiltInSubgroupSize: 1187 *location = SYSTEM_VALUE_SUBGROUP_SIZE; 1188 set_mode_system_value(b, mode); 1189 break; 1190 case SpvBuiltInSubgroupId: 1191 *location = SYSTEM_VALUE_SUBGROUP_ID; 1192 set_mode_system_value(b, mode); 1193 break; 1194 case SpvBuiltInSubgroupLocalInvocationId: 1195 *location = SYSTEM_VALUE_SUBGROUP_INVOCATION; 1196 set_mode_system_value(b, mode); 1197 break; 1198 case SpvBuiltInNumSubgroups: 1199 *location = SYSTEM_VALUE_NUM_SUBGROUPS; 1200 set_mode_system_value(b, mode); 1201 break; 1202 case SpvBuiltInDeviceIndex: 1203 *location = SYSTEM_VALUE_DEVICE_INDEX; 1204 set_mode_system_value(b, mode); 1205 break; 1206 case SpvBuiltInViewIndex: 1207 *location = SYSTEM_VALUE_VIEW_INDEX; 1208 set_mode_system_value(b, mode); 1209 break; 1210 case SpvBuiltInSubgroupEqMask: 1211 *location = SYSTEM_VALUE_SUBGROUP_EQ_MASK, 1212 set_mode_system_value(b, mode); 1213 break; 1214 case SpvBuiltInSubgroupGeMask: 1215 *location = SYSTEM_VALUE_SUBGROUP_GE_MASK, 1216 set_mode_system_value(b, mode); 1217 break; 1218 case SpvBuiltInSubgroupGtMask: 1219 *location = SYSTEM_VALUE_SUBGROUP_GT_MASK, 1220 set_mode_system_value(b, mode); 1221 break; 1222 case SpvBuiltInSubgroupLeMask: 1223 *location = SYSTEM_VALUE_SUBGROUP_LE_MASK, 1224 set_mode_system_value(b, mode); 1225 break; 1226 case SpvBuiltInSubgroupLtMask: 1227 *location = SYSTEM_VALUE_SUBGROUP_LT_MASK, 1228 set_mode_system_value(b, mode); 1229 break; 1230 case SpvBuiltInFragStencilRefEXT: 1231 *location = FRAG_RESULT_STENCIL; 1232 vtn_assert(*mode == nir_var_shader_out); 1233 break; 1234 case SpvBuiltInWorkDim: 1235 *location = SYSTEM_VALUE_WORK_DIM; 1236 set_mode_system_value(b, mode); 1237 break; 1238 case SpvBuiltInGlobalSize: 1239 *location = SYSTEM_VALUE_GLOBAL_GROUP_SIZE; 1240 set_mode_system_value(b, mode); 1241 break; 1242 default: 1243 vtn_fail("unsupported builtin: %u", builtin); 1244 } 1245} 1246 1247static void 1248apply_var_decoration(struct vtn_builder *b, 1249 struct nir_variable_data *var_data, 1250 const struct vtn_decoration *dec) 1251{ 1252 switch (dec->decoration) { 1253 case SpvDecorationRelaxedPrecision: 1254 break; /* FIXME: Do nothing with this for now. */ 1255 case SpvDecorationNoPerspective: 1256 var_data->interpolation = INTERP_MODE_NOPERSPECTIVE; 1257 break; 1258 case SpvDecorationFlat: 1259 var_data->interpolation = INTERP_MODE_FLAT; 1260 break; 1261 case SpvDecorationCentroid: 1262 var_data->centroid = true; 1263 break; 1264 case SpvDecorationSample: 1265 var_data->sample = true; 1266 break; 1267 case SpvDecorationInvariant: 1268 var_data->invariant = true; 1269 break; 1270 case SpvDecorationConstant: 1271 var_data->read_only = true; 1272 break; 1273 case SpvDecorationNonReadable: 1274 var_data->image.access |= ACCESS_NON_READABLE; 1275 break; 1276 case SpvDecorationNonWritable: 1277 var_data->read_only = true; 1278 var_data->image.access |= ACCESS_NON_WRITEABLE; 1279 break; 1280 case SpvDecorationRestrict: 1281 var_data->image.access |= ACCESS_RESTRICT; 1282 break; 1283 case SpvDecorationVolatile: 1284 var_data->image.access |= ACCESS_VOLATILE; 1285 break; 1286 case SpvDecorationCoherent: 1287 var_data->image.access |= ACCESS_COHERENT; 1288 break; 1289 case SpvDecorationComponent: 1290 var_data->location_frac = dec->literals[0]; 1291 break; 1292 case SpvDecorationIndex: 1293 var_data->index = dec->literals[0]; 1294 break; 1295 case SpvDecorationBuiltIn: { 1296 SpvBuiltIn builtin = dec->literals[0]; 1297 1298 nir_variable_mode mode = var_data->mode; 1299 vtn_get_builtin_location(b, builtin, &var_data->location, &mode); 1300 var_data->mode = mode; 1301 1302 switch (builtin) { 1303 case SpvBuiltInTessLevelOuter: 1304 case SpvBuiltInTessLevelInner: 1305 var_data->compact = true; 1306 break; 1307 case SpvBuiltInFragCoord: 1308 var_data->pixel_center_integer = b->pixel_center_integer; 1309 /* fallthrough */ 1310 case SpvBuiltInSamplePosition: 1311 var_data->origin_upper_left = b->origin_upper_left; 1312 break; 1313 default: 1314 break; 1315 } 1316 } 1317 1318 case SpvDecorationSpecId: 1319 case SpvDecorationRowMajor: 1320 case SpvDecorationColMajor: 1321 case SpvDecorationMatrixStride: 1322 case SpvDecorationAliased: 1323 case SpvDecorationUniform: 1324 case SpvDecorationLinkageAttributes: 1325 break; /* Do nothing with these here */ 1326 1327 case SpvDecorationPatch: 1328 var_data->patch = true; 1329 break; 1330 1331 case SpvDecorationLocation: 1332 vtn_fail("Handled above"); 1333 1334 case SpvDecorationBlock: 1335 case SpvDecorationBufferBlock: 1336 case SpvDecorationArrayStride: 1337 case SpvDecorationGLSLShared: 1338 case SpvDecorationGLSLPacked: 1339 break; /* These can apply to a type but we don't care about them */ 1340 1341 case SpvDecorationBinding: 1342 case SpvDecorationDescriptorSet: 1343 case SpvDecorationNoContraction: 1344 case SpvDecorationInputAttachmentIndex: 1345 vtn_warn("Decoration not allowed for variable or structure member: %s", 1346 spirv_decoration_to_string(dec->decoration)); 1347 break; 1348 1349 case SpvDecorationXfbBuffer: 1350 var_data->explicit_xfb_buffer = true; 1351 var_data->xfb_buffer = dec->literals[0]; 1352 var_data->always_active_io = true; 1353 break; 1354 case SpvDecorationXfbStride: 1355 var_data->explicit_xfb_stride = true; 1356 var_data->xfb_stride = dec->literals[0]; 1357 break; 1358 case SpvDecorationOffset: 1359 var_data->explicit_offset = true; 1360 var_data->offset = dec->literals[0]; 1361 break; 1362 1363 case SpvDecorationStream: 1364 var_data->stream = dec->literals[0]; 1365 break; 1366 1367 case SpvDecorationCPacked: 1368 case SpvDecorationSaturatedConversion: 1369 case SpvDecorationFuncParamAttr: 1370 case SpvDecorationFPRoundingMode: 1371 case SpvDecorationFPFastMathMode: 1372 case SpvDecorationAlignment: 1373 vtn_warn("Decoration only allowed for CL-style kernels: %s", 1374 spirv_decoration_to_string(dec->decoration)); 1375 break; 1376 1377 case SpvDecorationHlslSemanticGOOGLE: 1378 /* HLSL semantic decorations can safely be ignored by the driver. */ 1379 break; 1380 1381 default: 1382 vtn_fail("Unhandled decoration"); 1383 } 1384} 1385 1386static void 1387var_is_patch_cb(struct vtn_builder *b, struct vtn_value *val, int member, 1388 const struct vtn_decoration *dec, void *out_is_patch) 1389{ 1390 if (dec->decoration == SpvDecorationPatch) { 1391 *((bool *) out_is_patch) = true; 1392 } 1393} 1394 1395static void 1396var_decoration_cb(struct vtn_builder *b, struct vtn_value *val, int member, 1397 const struct vtn_decoration *dec, void *void_var) 1398{ 1399 struct vtn_variable *vtn_var = void_var; 1400 1401 /* Handle decorations that apply to a vtn_variable as a whole */ 1402 switch (dec->decoration) { 1403 case SpvDecorationBinding: 1404 vtn_var->binding = dec->literals[0]; 1405 vtn_var->explicit_binding = true; 1406 return; 1407 case SpvDecorationDescriptorSet: 1408 vtn_var->descriptor_set = dec->literals[0]; 1409 return; 1410 case SpvDecorationInputAttachmentIndex: 1411 vtn_var->input_attachment_index = dec->literals[0]; 1412 return; 1413 case SpvDecorationPatch: 1414 vtn_var->patch = true; 1415 break; 1416 case SpvDecorationOffset: 1417 vtn_var->offset = dec->literals[0]; 1418 break; 1419 case SpvDecorationNonWritable: 1420 vtn_var->access |= ACCESS_NON_WRITEABLE; 1421 break; 1422 case SpvDecorationNonReadable: 1423 vtn_var->access |= ACCESS_NON_READABLE; 1424 break; 1425 case SpvDecorationVolatile: 1426 vtn_var->access |= ACCESS_VOLATILE; 1427 break; 1428 case SpvDecorationCoherent: 1429 vtn_var->access |= ACCESS_COHERENT; 1430 break; 1431 case SpvDecorationHlslCounterBufferGOOGLE: 1432 /* HLSL semantic decorations can safely be ignored by the driver. */ 1433 break; 1434 default: 1435 break; 1436 } 1437 1438 if (val->value_type == vtn_value_type_pointer) { 1439 assert(val->pointer->var == void_var); 1440 assert(val->pointer->chain == NULL); 1441 assert(member == -1); 1442 } else { 1443 assert(val->value_type == vtn_value_type_type); 1444 } 1445 1446 /* Location is odd. If applied to a split structure, we have to walk the 1447 * whole thing and accumulate the location. It's easier to handle as a 1448 * special case. 1449 */ 1450 if (dec->decoration == SpvDecorationLocation) { 1451 unsigned location = dec->literals[0]; 1452 bool is_vertex_input = false; 1453 if (b->shader->info.stage == MESA_SHADER_FRAGMENT && 1454 vtn_var->mode == vtn_variable_mode_output) { 1455 location += FRAG_RESULT_DATA0; 1456 } else if (b->shader->info.stage == MESA_SHADER_VERTEX && 1457 vtn_var->mode == vtn_variable_mode_input) { 1458 is_vertex_input = true; 1459 location += VERT_ATTRIB_GENERIC0; 1460 } else if (vtn_var->mode == vtn_variable_mode_input || 1461 vtn_var->mode == vtn_variable_mode_output) { 1462 location += vtn_var->patch ? VARYING_SLOT_PATCH0 : VARYING_SLOT_VAR0; 1463 } else if (vtn_var->mode != vtn_variable_mode_uniform) { 1464 vtn_warn("Location must be on input, output, uniform, sampler or " 1465 "image variable"); 1466 return; 1467 } 1468 1469 if (vtn_var->var->num_members == 0) { 1470 /* This handles the member and lone variable cases */ 1471 vtn_var->var->data.location = location; 1472 } else { 1473 /* This handles the structure member case */ 1474 assert(vtn_var->var->members); 1475 for (unsigned i = 0; i < vtn_var->var->num_members; i++) { 1476 vtn_var->var->members[i].location = location; 1477 const struct glsl_type *member_type = 1478 glsl_get_struct_field(vtn_var->var->interface_type, i); 1479 location += glsl_count_attribute_slots(member_type, 1480 is_vertex_input); 1481 } 1482 } 1483 return; 1484 } else { 1485 if (vtn_var->var) { 1486 if (vtn_var->var->num_members == 0) { 1487 assert(member == -1); 1488 apply_var_decoration(b, &vtn_var->var->data, dec); 1489 } else if (member >= 0) { 1490 /* Member decorations must come from a type */ 1491 assert(val->value_type == vtn_value_type_type); 1492 apply_var_decoration(b, &vtn_var->var->members[member], dec); 1493 } else { 1494 unsigned length = 1495 glsl_get_length(glsl_without_array(vtn_var->type->type)); 1496 for (unsigned i = 0; i < length; i++) 1497 apply_var_decoration(b, &vtn_var->var->members[i], dec); 1498 } 1499 } else { 1500 /* A few variables, those with external storage, have no actual 1501 * nir_variables associated with them. Fortunately, all decorations 1502 * we care about for those variables are on the type only. 1503 */ 1504 vtn_assert(vtn_var->mode == vtn_variable_mode_ubo || 1505 vtn_var->mode == vtn_variable_mode_ssbo || 1506 vtn_var->mode == vtn_variable_mode_push_constant || 1507 (vtn_var->mode == vtn_variable_mode_workgroup && 1508 b->options->lower_workgroup_access_to_offsets)); 1509 } 1510 } 1511} 1512 1513static enum vtn_variable_mode 1514vtn_storage_class_to_mode(struct vtn_builder *b, 1515 SpvStorageClass class, 1516 struct vtn_type *interface_type, 1517 nir_variable_mode *nir_mode_out) 1518{ 1519 enum vtn_variable_mode mode; 1520 nir_variable_mode nir_mode; 1521 switch (class) { 1522 case SpvStorageClassUniform: 1523 if (interface_type->block) { 1524 mode = vtn_variable_mode_ubo; 1525 nir_mode = 0; 1526 } else if (interface_type->buffer_block) { 1527 mode = vtn_variable_mode_ssbo; 1528 nir_mode = 0; 1529 } else { 1530 /* Default-block uniforms, coming from gl_spirv */ 1531 mode = vtn_variable_mode_uniform; 1532 nir_mode = nir_var_uniform; 1533 } 1534 break; 1535 case SpvStorageClassStorageBuffer: 1536 mode = vtn_variable_mode_ssbo; 1537 nir_mode = 0; 1538 break; 1539 case SpvStorageClassUniformConstant: 1540 mode = vtn_variable_mode_uniform; 1541 nir_mode = nir_var_uniform; 1542 break; 1543 case SpvStorageClassPushConstant: 1544 mode = vtn_variable_mode_push_constant; 1545 nir_mode = nir_var_uniform; 1546 break; 1547 case SpvStorageClassInput: 1548 mode = vtn_variable_mode_input; 1549 nir_mode = nir_var_shader_in; 1550 break; 1551 case SpvStorageClassOutput: 1552 mode = vtn_variable_mode_output; 1553 nir_mode = nir_var_shader_out; 1554 break; 1555 case SpvStorageClassPrivate: 1556 mode = vtn_variable_mode_global; 1557 nir_mode = nir_var_global; 1558 break; 1559 case SpvStorageClassFunction: 1560 mode = vtn_variable_mode_local; 1561 nir_mode = nir_var_local; 1562 break; 1563 case SpvStorageClassWorkgroup: 1564 mode = vtn_variable_mode_workgroup; 1565 nir_mode = nir_var_shared; 1566 break; 1567 case SpvStorageClassAtomicCounter: 1568 mode = vtn_variable_mode_uniform; 1569 nir_mode = nir_var_uniform; 1570 break; 1571 case SpvStorageClassCrossWorkgroup: 1572 case SpvStorageClassGeneric: 1573 default: 1574 vtn_fail("Unhandled variable storage class"); 1575 } 1576 1577 if (nir_mode_out) 1578 *nir_mode_out = nir_mode; 1579 1580 return mode; 1581} 1582 1583nir_ssa_def * 1584vtn_pointer_to_ssa(struct vtn_builder *b, struct vtn_pointer *ptr) 1585{ 1586 if (vtn_pointer_uses_ssa_offset(b, ptr)) { 1587 /* This pointer needs to have a pointer type with actual storage */ 1588 vtn_assert(ptr->ptr_type); 1589 vtn_assert(ptr->ptr_type->type); 1590 1591 if (!ptr->offset) { 1592 /* If we don't have an offset then we must be a pointer to the variable 1593 * itself. 1594 */ 1595 vtn_assert(!ptr->offset && !ptr->block_index); 1596 1597 struct vtn_access_chain chain = { 1598 .length = 0, 1599 }; 1600 ptr = vtn_ssa_offset_pointer_dereference(b, ptr, &chain); 1601 } 1602 1603 vtn_assert(ptr->offset); 1604 if (ptr->block_index) { 1605 vtn_assert(ptr->mode == vtn_variable_mode_ubo || 1606 ptr->mode == vtn_variable_mode_ssbo); 1607 return nir_vec2(&b->nb, ptr->block_index, ptr->offset); 1608 } else { 1609 vtn_assert(ptr->mode == vtn_variable_mode_workgroup); 1610 return ptr->offset; 1611 } 1612 } else { 1613 return &vtn_pointer_to_deref(b, ptr)->dest.ssa; 1614 } 1615} 1616 1617struct vtn_pointer * 1618vtn_pointer_from_ssa(struct vtn_builder *b, nir_ssa_def *ssa, 1619 struct vtn_type *ptr_type) 1620{ 1621 vtn_assert(ssa->num_components <= 2 && ssa->bit_size == 32); 1622 vtn_assert(ptr_type->base_type == vtn_base_type_pointer); 1623 1624 struct vtn_type *interface_type = ptr_type->deref; 1625 while (interface_type->base_type == vtn_base_type_array) 1626 interface_type = interface_type->array_element; 1627 1628 struct vtn_pointer *ptr = rzalloc(b, struct vtn_pointer); 1629 nir_variable_mode nir_mode; 1630 ptr->mode = vtn_storage_class_to_mode(b, ptr_type->storage_class, 1631 interface_type, &nir_mode); 1632 ptr->type = ptr_type->deref; 1633 ptr->ptr_type = ptr_type; 1634 1635 if (ptr->mode == vtn_variable_mode_ubo || 1636 ptr->mode == vtn_variable_mode_ssbo) { 1637 /* This pointer type needs to have actual storage */ 1638 vtn_assert(ptr_type->type); 1639 vtn_assert(ssa->num_components == 2); 1640 ptr->block_index = nir_channel(&b->nb, ssa, 0); 1641 ptr->offset = nir_channel(&b->nb, ssa, 1); 1642 } else if (ptr->mode == vtn_variable_mode_workgroup || 1643 ptr->mode == vtn_variable_mode_push_constant) { 1644 /* This pointer type needs to have actual storage */ 1645 vtn_assert(ptr_type->type); 1646 vtn_assert(ssa->num_components == 1); 1647 ptr->block_index = NULL; 1648 ptr->offset = ssa; 1649 } else { 1650 ptr->deref = nir_build_deref_cast(&b->nb, ssa, nir_mode, 1651 ptr_type->deref->type); 1652 } 1653 1654 return ptr; 1655} 1656 1657static bool 1658is_per_vertex_inout(const struct vtn_variable *var, gl_shader_stage stage) 1659{ 1660 if (var->patch || !glsl_type_is_array(var->type->type)) 1661 return false; 1662 1663 if (var->mode == vtn_variable_mode_input) { 1664 return stage == MESA_SHADER_TESS_CTRL || 1665 stage == MESA_SHADER_TESS_EVAL || 1666 stage == MESA_SHADER_GEOMETRY; 1667 } 1668 1669 if (var->mode == vtn_variable_mode_output) 1670 return stage == MESA_SHADER_TESS_CTRL; 1671 1672 return false; 1673} 1674 1675static void 1676vtn_create_variable(struct vtn_builder *b, struct vtn_value *val, 1677 struct vtn_type *ptr_type, SpvStorageClass storage_class, 1678 nir_constant *initializer) 1679{ 1680 vtn_assert(ptr_type->base_type == vtn_base_type_pointer); 1681 struct vtn_type *type = ptr_type->deref; 1682 1683 struct vtn_type *without_array = type; 1684 while(glsl_type_is_array(without_array->type)) 1685 without_array = without_array->array_element; 1686 1687 enum vtn_variable_mode mode; 1688 nir_variable_mode nir_mode; 1689 mode = vtn_storage_class_to_mode(b, storage_class, without_array, &nir_mode); 1690 1691 switch (mode) { 1692 case vtn_variable_mode_ubo: 1693 b->shader->info.num_ubos++; 1694 break; 1695 case vtn_variable_mode_ssbo: 1696 b->shader->info.num_ssbos++; 1697 break; 1698 case vtn_variable_mode_uniform: 1699 if (glsl_type_is_image(without_array->type)) 1700 b->shader->info.num_images++; 1701 else if (glsl_type_is_sampler(without_array->type)) 1702 b->shader->info.num_textures++; 1703 break; 1704 case vtn_variable_mode_push_constant: 1705 b->shader->num_uniforms = vtn_type_block_size(b, type); 1706 break; 1707 default: 1708 /* No tallying is needed */ 1709 break; 1710 } 1711 1712 struct vtn_variable *var = rzalloc(b, struct vtn_variable); 1713 var->type = type; 1714 var->mode = mode; 1715 1716 vtn_assert(val->value_type == vtn_value_type_pointer); 1717 val->pointer = vtn_pointer_for_variable(b, var, ptr_type); 1718 1719 switch (var->mode) { 1720 case vtn_variable_mode_local: 1721 case vtn_variable_mode_global: 1722 case vtn_variable_mode_uniform: 1723 /* For these, we create the variable normally */ 1724 var->var = rzalloc(b->shader, nir_variable); 1725 var->var->name = ralloc_strdup(var->var, val->name); 1726 1727 /* Need to tweak the nir type here as at vtn_handle_type we don't have 1728 * the access to storage_class, that is the one that points us that is 1729 * an atomic uint. 1730 */ 1731 if (storage_class == SpvStorageClassAtomicCounter) { 1732 var->var->type = repair_atomic_type(var->type->type); 1733 } else { 1734 var->var->type = var->type->type; 1735 } 1736 var->var->data.mode = nir_mode; 1737 var->var->data.location = -1; 1738 var->var->interface_type = NULL; 1739 break; 1740 1741 case vtn_variable_mode_workgroup: 1742 if (b->options->lower_workgroup_access_to_offsets) { 1743 var->shared_location = -1; 1744 } else { 1745 /* Create the variable normally */ 1746 var->var = rzalloc(b->shader, nir_variable); 1747 var->var->name = ralloc_strdup(var->var, val->name); 1748 var->var->type = var->type->type; 1749 var->var->data.mode = nir_var_shared; 1750 } 1751 break; 1752 1753 case vtn_variable_mode_input: 1754 case vtn_variable_mode_output: { 1755 /* In order to know whether or not we're a per-vertex inout, we need 1756 * the patch qualifier. This means walking the variable decorations 1757 * early before we actually create any variables. Not a big deal. 1758 * 1759 * GLSLang really likes to place decorations in the most interior 1760 * thing it possibly can. In particular, if you have a struct, it 1761 * will place the patch decorations on the struct members. This 1762 * should be handled by the variable splitting below just fine. 1763 * 1764 * If you have an array-of-struct, things get even more weird as it 1765 * will place the patch decorations on the struct even though it's 1766 * inside an array and some of the members being patch and others not 1767 * makes no sense whatsoever. Since the only sensible thing is for 1768 * it to be all or nothing, we'll call it patch if any of the members 1769 * are declared patch. 1770 */ 1771 var->patch = false; 1772 vtn_foreach_decoration(b, val, var_is_patch_cb, &var->patch); 1773 if (glsl_type_is_array(var->type->type) && 1774 glsl_type_is_struct(without_array->type)) { 1775 vtn_foreach_decoration(b, vtn_value(b, without_array->id, 1776 vtn_value_type_type), 1777 var_is_patch_cb, &var->patch); 1778 } 1779 1780 /* For inputs and outputs, we immediately split structures. This 1781 * is for a couple of reasons. For one, builtins may all come in 1782 * a struct and we really want those split out into separate 1783 * variables. For another, interpolation qualifiers can be 1784 * applied to members of the top-level struct ane we need to be 1785 * able to preserve that information. 1786 */ 1787 1788 struct vtn_type *interface_type = var->type; 1789 if (is_per_vertex_inout(var, b->shader->info.stage)) { 1790 /* In Geometry shaders (and some tessellation), inputs come 1791 * in per-vertex arrays. However, some builtins come in 1792 * non-per-vertex, hence the need for the is_array check. In 1793 * any case, there are no non-builtin arrays allowed so this 1794 * check should be sufficient. 1795 */ 1796 interface_type = var->type->array_element; 1797 } 1798 1799 var->var = rzalloc(b->shader, nir_variable); 1800 var->var->name = ralloc_strdup(var->var, val->name); 1801 var->var->type = var->type->type; 1802 var->var->interface_type = interface_type->type; 1803 var->var->data.mode = nir_mode; 1804 var->var->data.patch = var->patch; 1805 1806 if (glsl_type_is_struct(interface_type->type)) { 1807 /* It's a struct. Set it up as per-member. */ 1808 var->var->num_members = glsl_get_length(interface_type->type); 1809 var->var->members = rzalloc_array(var->var, struct nir_variable_data, 1810 var->var->num_members); 1811 1812 for (unsigned i = 0; i < var->var->num_members; i++) { 1813 var->var->members[i].mode = nir_mode; 1814 var->var->members[i].patch = var->patch; 1815 } 1816 } 1817 1818 /* For inputs and outputs, we need to grab locations and builtin 1819 * information from the interface type. 1820 */ 1821 vtn_foreach_decoration(b, vtn_value(b, interface_type->id, 1822 vtn_value_type_type), 1823 var_decoration_cb, var); 1824 break; 1825 } 1826 1827 case vtn_variable_mode_ubo: 1828 case vtn_variable_mode_ssbo: 1829 case vtn_variable_mode_push_constant: 1830 /* These don't need actual variables. */ 1831 break; 1832 } 1833 1834 if (initializer) { 1835 var->var->constant_initializer = 1836 nir_constant_clone(initializer, var->var); 1837 } 1838 1839 vtn_foreach_decoration(b, val, var_decoration_cb, var); 1840 1841 if (var->mode == vtn_variable_mode_uniform) { 1842 /* XXX: We still need the binding information in the nir_variable 1843 * for these. We should fix that. 1844 */ 1845 var->var->data.binding = var->binding; 1846 var->var->data.explicit_binding = var->explicit_binding; 1847 var->var->data.descriptor_set = var->descriptor_set; 1848 var->var->data.index = var->input_attachment_index; 1849 var->var->data.offset = var->offset; 1850 1851 if (glsl_type_is_image(without_array->type)) 1852 var->var->data.image.format = without_array->image_format; 1853 } 1854 1855 if (var->mode == vtn_variable_mode_local) { 1856 vtn_assert(var->var != NULL && var->var->members == NULL); 1857 nir_function_impl_add_variable(b->nb.impl, var->var); 1858 } else if (var->var) { 1859 nir_shader_add_variable(b->shader, var->var); 1860 } else { 1861 vtn_assert(vtn_pointer_is_external_block(b, val->pointer)); 1862 } 1863} 1864 1865static void 1866vtn_assert_types_equal(struct vtn_builder *b, SpvOp opcode, 1867 struct vtn_type *dst_type, 1868 struct vtn_type *src_type) 1869{ 1870 if (dst_type->id == src_type->id) 1871 return; 1872 1873 if (vtn_types_compatible(b, dst_type, src_type)) { 1874 /* Early versions of GLSLang would re-emit types unnecessarily and you 1875 * would end up with OpLoad, OpStore, or OpCopyMemory opcodes which have 1876 * mismatched source and destination types. 1877 * 1878 * https://github.com/KhronosGroup/glslang/issues/304 1879 * https://github.com/KhronosGroup/glslang/issues/307 1880 * https://bugs.freedesktop.org/show_bug.cgi?id=104338 1881 * https://bugs.freedesktop.org/show_bug.cgi?id=104424 1882 */ 1883 vtn_warn("Source and destination types of %s do not have the same " 1884 "ID (but are compatible): %u vs %u", 1885 spirv_op_to_string(opcode), dst_type->id, src_type->id); 1886 return; 1887 } 1888 1889 vtn_fail("Source and destination types of %s do not match: %s vs. %s", 1890 spirv_op_to_string(opcode), 1891 glsl_get_type_name(dst_type->type), 1892 glsl_get_type_name(src_type->type)); 1893} 1894 1895void 1896vtn_handle_variables(struct vtn_builder *b, SpvOp opcode, 1897 const uint32_t *w, unsigned count) 1898{ 1899 switch (opcode) { 1900 case SpvOpUndef: { 1901 struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_undef); 1902 val->type = vtn_value(b, w[1], vtn_value_type_type)->type; 1903 break; 1904 } 1905 1906 case SpvOpVariable: { 1907 struct vtn_type *ptr_type = vtn_value(b, w[1], vtn_value_type_type)->type; 1908 1909 struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_pointer); 1910 1911 SpvStorageClass storage_class = w[3]; 1912 nir_constant *initializer = NULL; 1913 if (count > 4) 1914 initializer = vtn_value(b, w[4], vtn_value_type_constant)->constant; 1915 1916 vtn_create_variable(b, val, ptr_type, storage_class, initializer); 1917 break; 1918 } 1919 1920 case SpvOpAccessChain: 1921 case SpvOpPtrAccessChain: 1922 case SpvOpInBoundsAccessChain: { 1923 struct vtn_access_chain *chain = vtn_access_chain_create(b, count - 4); 1924 chain->ptr_as_array = (opcode == SpvOpPtrAccessChain); 1925 1926 unsigned idx = 0; 1927 for (int i = 4; i < count; i++) { 1928 struct vtn_value *link_val = vtn_untyped_value(b, w[i]); 1929 if (link_val->value_type == vtn_value_type_constant) { 1930 chain->link[idx].mode = vtn_access_mode_literal; 1931 chain->link[idx].id = link_val->constant->values[0].u32[0]; 1932 } else { 1933 chain->link[idx].mode = vtn_access_mode_id; 1934 chain->link[idx].id = w[i]; 1935 1936 } 1937 idx++; 1938 } 1939 1940 struct vtn_type *ptr_type = vtn_value(b, w[1], vtn_value_type_type)->type; 1941 struct vtn_value *base_val = vtn_untyped_value(b, w[3]); 1942 if (base_val->value_type == vtn_value_type_sampled_image) { 1943 /* This is rather insane. SPIR-V allows you to use OpSampledImage 1944 * to combine an array of images with a single sampler to get an 1945 * array of sampled images that all share the same sampler. 1946 * Fortunately, this means that we can more-or-less ignore the 1947 * sampler when crawling the access chain, but it does leave us 1948 * with this rather awkward little special-case. 1949 */ 1950 struct vtn_value *val = 1951 vtn_push_value(b, w[2], vtn_value_type_sampled_image); 1952 val->sampled_image = ralloc(b, struct vtn_sampled_image); 1953 val->sampled_image->type = base_val->sampled_image->type; 1954 val->sampled_image->image = 1955 vtn_pointer_dereference(b, base_val->sampled_image->image, chain); 1956 val->sampled_image->sampler = base_val->sampled_image->sampler; 1957 } else { 1958 vtn_assert(base_val->value_type == vtn_value_type_pointer); 1959 struct vtn_value *val = 1960 vtn_push_value(b, w[2], vtn_value_type_pointer); 1961 val->pointer = vtn_pointer_dereference(b, base_val->pointer, chain); 1962 val->pointer->ptr_type = ptr_type; 1963 } 1964 break; 1965 } 1966 1967 case SpvOpCopyMemory: { 1968 struct vtn_value *dest = vtn_value(b, w[1], vtn_value_type_pointer); 1969 struct vtn_value *src = vtn_value(b, w[2], vtn_value_type_pointer); 1970 1971 vtn_assert_types_equal(b, opcode, dest->type->deref, src->type->deref); 1972 1973 vtn_variable_copy(b, dest->pointer, src->pointer); 1974 break; 1975 } 1976 1977 case SpvOpLoad: { 1978 struct vtn_type *res_type = 1979 vtn_value(b, w[1], vtn_value_type_type)->type; 1980 struct vtn_value *src_val = vtn_value(b, w[3], vtn_value_type_pointer); 1981 struct vtn_pointer *src = src_val->pointer; 1982 1983 vtn_assert_types_equal(b, opcode, res_type, src_val->type->deref); 1984 1985 if (glsl_type_is_image(res_type->type) || 1986 glsl_type_is_sampler(res_type->type)) { 1987 vtn_push_value(b, w[2], vtn_value_type_pointer)->pointer = src; 1988 return; 1989 } 1990 1991 vtn_push_ssa(b, w[2], res_type, vtn_variable_load(b, src)); 1992 break; 1993 } 1994 1995 case SpvOpStore: { 1996 struct vtn_value *dest_val = vtn_value(b, w[1], vtn_value_type_pointer); 1997 struct vtn_pointer *dest = dest_val->pointer; 1998 struct vtn_value *src_val = vtn_untyped_value(b, w[2]); 1999 2000 /* OpStore requires us to actually have a storage type */ 2001 vtn_fail_if(dest->type->type == NULL, 2002 "Invalid destination type for OpStore"); 2003 2004 if (glsl_get_base_type(dest->type->type) == GLSL_TYPE_BOOL && 2005 glsl_get_base_type(src_val->type->type) == GLSL_TYPE_UINT) { 2006 /* Early versions of GLSLang would use uint types for UBOs/SSBOs but 2007 * would then store them to a local variable as bool. Work around 2008 * the issue by doing an implicit conversion. 2009 * 2010 * https://github.com/KhronosGroup/glslang/issues/170 2011 * https://bugs.freedesktop.org/show_bug.cgi?id=104424 2012 */ 2013 vtn_warn("OpStore of value of type OpTypeInt to a pointer to type " 2014 "OpTypeBool. Doing an implicit conversion to work around " 2015 "the problem."); 2016 struct vtn_ssa_value *bool_ssa = 2017 vtn_create_ssa_value(b, dest->type->type); 2018 bool_ssa->def = nir_i2b(&b->nb, vtn_ssa_value(b, w[2])->def); 2019 vtn_variable_store(b, bool_ssa, dest); 2020 break; 2021 } 2022 2023 vtn_assert_types_equal(b, opcode, dest_val->type->deref, src_val->type); 2024 2025 if (glsl_type_is_sampler(dest->type->type)) { 2026 vtn_warn("OpStore of a sampler detected. Doing on-the-fly copy " 2027 "propagation to workaround the problem."); 2028 vtn_assert(dest->var->copy_prop_sampler == NULL); 2029 dest->var->copy_prop_sampler = 2030 vtn_value(b, w[2], vtn_value_type_pointer)->pointer; 2031 break; 2032 } 2033 2034 struct vtn_ssa_value *src = vtn_ssa_value(b, w[2]); 2035 vtn_variable_store(b, src, dest); 2036 break; 2037 } 2038 2039 case SpvOpArrayLength: { 2040 struct vtn_pointer *ptr = 2041 vtn_value(b, w[3], vtn_value_type_pointer)->pointer; 2042 const uint32_t field = w[4]; 2043 2044 vtn_fail_if(ptr->type->base_type != vtn_base_type_struct, 2045 "OpArrayLength must take a pointer to a structure type"); 2046 vtn_fail_if(field != ptr->type->length - 1 || 2047 ptr->type->members[field]->base_type != vtn_base_type_array, 2048 "OpArrayLength must reference the last memeber of the " 2049 "structure and that must be an array"); 2050 2051 const uint32_t offset = ptr->type->offsets[field]; 2052 const uint32_t stride = ptr->type->members[field]->stride; 2053 2054 if (!ptr->block_index) { 2055 struct vtn_access_chain chain = { 2056 .length = 0, 2057 }; 2058 ptr = vtn_ssa_offset_pointer_dereference(b, ptr, &chain); 2059 vtn_assert(ptr->block_index); 2060 } 2061 2062 nir_intrinsic_instr *instr = 2063 nir_intrinsic_instr_create(b->nb.shader, 2064 nir_intrinsic_get_buffer_size); 2065 instr->src[0] = nir_src_for_ssa(ptr->block_index); 2066 nir_ssa_dest_init(&instr->instr, &instr->dest, 1, 32, NULL); 2067 nir_builder_instr_insert(&b->nb, &instr->instr); 2068 nir_ssa_def *buf_size = &instr->dest.ssa; 2069 2070 /* array_length = max(buffer_size - offset, 0) / stride */ 2071 nir_ssa_def *array_length = 2072 nir_idiv(&b->nb, 2073 nir_imax(&b->nb, 2074 nir_isub(&b->nb, 2075 buf_size, 2076 nir_imm_int(&b->nb, offset)), 2077 nir_imm_int(&b->nb, 0u)), 2078 nir_imm_int(&b->nb, stride)); 2079 2080 struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_ssa); 2081 val->ssa = vtn_create_ssa_value(b, glsl_uint_type()); 2082 val->ssa->def = array_length; 2083 break; 2084 } 2085 2086 case SpvOpCopyMemorySized: 2087 default: 2088 vtn_fail("Unhandled opcode"); 2089 } 2090} 2091