1/* 2 * Copyright © 2015 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Jason Ekstrand (jason@jlekstrand.net) 25 * 26 */ 27 28#include "vtn_private.h" 29#include "spirv_info.h" 30#include "nir_deref.h" 31#include <vulkan/vulkan_core.h> 32 33static void ptr_decoration_cb(struct vtn_builder *b, 34 struct vtn_value *val, int member, 35 const struct vtn_decoration *dec, 36 void *void_ptr); 37 38struct vtn_value * 39vtn_push_value_pointer(struct vtn_builder *b, uint32_t value_id, 40 struct vtn_pointer *ptr) 41{ 42 struct vtn_value *val = vtn_push_value(b, value_id, vtn_value_type_pointer); 43 val->pointer = ptr; 44 vtn_foreach_decoration(b, val, ptr_decoration_cb, ptr); 45 return val; 46} 47 48static void 49ssa_decoration_cb(struct vtn_builder *b, struct vtn_value *val, int member, 50 const struct vtn_decoration *dec, void *void_ssa) 51{ 52 struct vtn_ssa_value *ssa = void_ssa; 53 54 switch (dec->decoration) { 55 case SpvDecorationNonUniformEXT: 56 ssa->access |= ACCESS_NON_UNIFORM; 57 break; 58 59 default: 60 break; 61 } 62} 63 64struct vtn_value * 65vtn_push_ssa(struct vtn_builder *b, uint32_t value_id, 66 struct vtn_type *type, struct vtn_ssa_value *ssa) 67{ 68 struct vtn_value *val; 69 if (type->base_type == vtn_base_type_pointer) { 70 val = vtn_push_value_pointer(b, value_id, vtn_pointer_from_ssa(b, ssa->def, type)); 71 } else { 72 val = vtn_push_value(b, value_id, vtn_value_type_ssa); 73 val->ssa = ssa; 74 vtn_foreach_decoration(b, val, ssa_decoration_cb, val->ssa); 75 } 76 return val; 77} 78 79static struct vtn_access_chain * 80vtn_access_chain_create(struct vtn_builder *b, unsigned length) 81{ 82 struct vtn_access_chain *chain; 83 84 /* Subtract 1 from the length since there's already one built in */ 85 size_t size = sizeof(*chain) + 86 (MAX2(length, 1) - 1) * sizeof(chain->link[0]); 87 chain = rzalloc_size(b, size); 88 chain->length = length; 89 90 return chain; 91} 92 93bool 94vtn_pointer_uses_ssa_offset(struct vtn_builder *b, 95 struct vtn_pointer *ptr) 96{ 97 return ((ptr->mode == vtn_variable_mode_ubo || 98 ptr->mode == vtn_variable_mode_ssbo) && 99 b->options->lower_ubo_ssbo_access_to_offsets) || 100 ptr->mode == vtn_variable_mode_push_constant || 101 (ptr->mode == vtn_variable_mode_workgroup && 102 b->options->lower_workgroup_access_to_offsets); 103} 104 105static bool 106vtn_pointer_is_external_block(struct vtn_builder *b, 107 struct vtn_pointer *ptr) 108{ 109 return ptr->mode == vtn_variable_mode_ssbo || 110 ptr->mode == vtn_variable_mode_ubo || 111 ptr->mode == vtn_variable_mode_phys_ssbo || 112 ptr->mode == vtn_variable_mode_push_constant || 113 (ptr->mode == vtn_variable_mode_workgroup && 114 b->options->lower_workgroup_access_to_offsets); 115} 116 117static nir_ssa_def * 118vtn_access_link_as_ssa(struct vtn_builder *b, struct vtn_access_link link, 119 unsigned stride, unsigned bit_size) 120{ 121 vtn_assert(stride > 0); 122 if (link.mode == vtn_access_mode_literal) { 123 return nir_imm_intN_t(&b->nb, link.id * stride, bit_size); 124 } else { 125 nir_ssa_def *ssa = vtn_ssa_value(b, link.id)->def; 126 if (ssa->bit_size != bit_size) 127 ssa = nir_i2i(&b->nb, ssa, bit_size); 128 return nir_imul_imm(&b->nb, ssa, stride); 129 } 130} 131 132static VkDescriptorType 133vk_desc_type_for_mode(struct vtn_builder *b, enum vtn_variable_mode mode) 134{ 135 switch (mode) { 136 case vtn_variable_mode_ubo: 137 return VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER; 138 case vtn_variable_mode_ssbo: 139 return VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; 140 default: 141 vtn_fail("Invalid mode for vulkan_resource_index"); 142 } 143} 144 145static const struct glsl_type * 146vtn_ptr_type_for_mode(struct vtn_builder *b, enum vtn_variable_mode mode) 147{ 148 switch (mode) { 149 case vtn_variable_mode_ubo: 150 return b->options->ubo_ptr_type; 151 case vtn_variable_mode_ssbo: 152 return b->options->ssbo_ptr_type; 153 default: 154 vtn_fail("Invalid mode for vulkan_resource_index"); 155 } 156} 157 158static nir_ssa_def * 159vtn_variable_resource_index(struct vtn_builder *b, struct vtn_variable *var, 160 nir_ssa_def *desc_array_index) 161{ 162 if (!desc_array_index) { 163 vtn_assert(glsl_type_is_struct_or_ifc(var->type->type)); 164 desc_array_index = nir_imm_int(&b->nb, 0); 165 } 166 167 nir_intrinsic_instr *instr = 168 nir_intrinsic_instr_create(b->nb.shader, 169 nir_intrinsic_vulkan_resource_index); 170 instr->src[0] = nir_src_for_ssa(desc_array_index); 171 nir_intrinsic_set_desc_set(instr, var->descriptor_set); 172 nir_intrinsic_set_binding(instr, var->binding); 173 nir_intrinsic_set_desc_type(instr, vk_desc_type_for_mode(b, var->mode)); 174 175 const struct glsl_type *index_type = 176 b->options->lower_ubo_ssbo_access_to_offsets ? 177 glsl_uint_type() : vtn_ptr_type_for_mode(b, var->mode); 178 179 instr->num_components = glsl_get_vector_elements(index_type); 180 nir_ssa_dest_init(&instr->instr, &instr->dest, instr->num_components, 181 glsl_get_bit_size(index_type), NULL); 182 nir_builder_instr_insert(&b->nb, &instr->instr); 183 184 return &instr->dest.ssa; 185} 186 187static nir_ssa_def * 188vtn_resource_reindex(struct vtn_builder *b, enum vtn_variable_mode mode, 189 nir_ssa_def *base_index, nir_ssa_def *offset_index) 190{ 191 nir_intrinsic_instr *instr = 192 nir_intrinsic_instr_create(b->nb.shader, 193 nir_intrinsic_vulkan_resource_reindex); 194 instr->src[0] = nir_src_for_ssa(base_index); 195 instr->src[1] = nir_src_for_ssa(offset_index); 196 nir_intrinsic_set_desc_type(instr, vk_desc_type_for_mode(b, mode)); 197 198 const struct glsl_type *index_type = 199 b->options->lower_ubo_ssbo_access_to_offsets ? 200 glsl_uint_type() : vtn_ptr_type_for_mode(b, mode); 201 202 instr->num_components = glsl_get_vector_elements(index_type); 203 nir_ssa_dest_init(&instr->instr, &instr->dest, instr->num_components, 204 glsl_get_bit_size(index_type), NULL); 205 nir_builder_instr_insert(&b->nb, &instr->instr); 206 207 return &instr->dest.ssa; 208} 209 210static nir_ssa_def * 211vtn_descriptor_load(struct vtn_builder *b, enum vtn_variable_mode mode, 212 nir_ssa_def *desc_index) 213{ 214 nir_intrinsic_instr *desc_load = 215 nir_intrinsic_instr_create(b->nb.shader, 216 nir_intrinsic_load_vulkan_descriptor); 217 desc_load->src[0] = nir_src_for_ssa(desc_index); 218 nir_intrinsic_set_desc_type(desc_load, vk_desc_type_for_mode(b, mode)); 219 220 const struct glsl_type *ptr_type = vtn_ptr_type_for_mode(b, mode); 221 222 desc_load->num_components = glsl_get_vector_elements(ptr_type); 223 nir_ssa_dest_init(&desc_load->instr, &desc_load->dest, 224 desc_load->num_components, 225 glsl_get_bit_size(ptr_type), NULL); 226 nir_builder_instr_insert(&b->nb, &desc_load->instr); 227 228 return &desc_load->dest.ssa; 229} 230 231/* Dereference the given base pointer by the access chain */ 232static struct vtn_pointer * 233vtn_nir_deref_pointer_dereference(struct vtn_builder *b, 234 struct vtn_pointer *base, 235 struct vtn_access_chain *deref_chain) 236{ 237 struct vtn_type *type = base->type; 238 enum gl_access_qualifier access = base->access | deref_chain->access; 239 unsigned idx = 0; 240 241 nir_deref_instr *tail; 242 if (base->deref) { 243 tail = base->deref; 244 } else if (vtn_pointer_is_external_block(b, base)) { 245 nir_ssa_def *block_index = base->block_index; 246 247 /* We dereferencing an external block pointer. Correctness of this 248 * operation relies on one particular line in the SPIR-V spec, section 249 * entitled "Validation Rules for Shader Capabilities": 250 * 251 * "Block and BufferBlock decorations cannot decorate a structure 252 * type that is nested at any level inside another structure type 253 * decorated with Block or BufferBlock." 254 * 255 * This means that we can detect the point where we cross over from 256 * descriptor indexing to buffer indexing by looking for the block 257 * decorated struct type. Anything before the block decorated struct 258 * type is a descriptor indexing operation and anything after the block 259 * decorated struct is a buffer offset operation. 260 */ 261 262 /* Figure out the descriptor array index if any 263 * 264 * Some of the Vulkan CTS tests with hand-rolled SPIR-V have been known 265 * to forget the Block or BufferBlock decoration from time to time. 266 * It's more robust if we check for both !block_index and for the type 267 * to contain a block. This way there's a decent chance that arrays of 268 * UBOs/SSBOs will work correctly even if variable pointers are 269 * completley toast. 270 */ 271 nir_ssa_def *desc_arr_idx = NULL; 272 if (!block_index || vtn_type_contains_block(b, type)) { 273 /* If our type contains a block, then we're still outside the block 274 * and we need to process enough levels of dereferences to get inside 275 * of it. 276 */ 277 if (deref_chain->ptr_as_array) { 278 unsigned aoa_size = glsl_get_aoa_size(type->type); 279 desc_arr_idx = vtn_access_link_as_ssa(b, deref_chain->link[idx], 280 MAX2(aoa_size, 1), 32); 281 idx++; 282 } 283 284 for (; idx < deref_chain->length; idx++) { 285 if (type->base_type != vtn_base_type_array) { 286 vtn_assert(type->base_type == vtn_base_type_struct); 287 break; 288 } 289 290 unsigned aoa_size = glsl_get_aoa_size(type->array_element->type); 291 nir_ssa_def *arr_offset = 292 vtn_access_link_as_ssa(b, deref_chain->link[idx], 293 MAX2(aoa_size, 1), 32); 294 if (desc_arr_idx) 295 desc_arr_idx = nir_iadd(&b->nb, desc_arr_idx, arr_offset); 296 else 297 desc_arr_idx = arr_offset; 298 299 type = type->array_element; 300 access |= type->access; 301 } 302 } 303 304 if (!block_index) { 305 vtn_assert(base->var && base->type); 306 block_index = vtn_variable_resource_index(b, base->var, desc_arr_idx); 307 } else if (desc_arr_idx) { 308 block_index = vtn_resource_reindex(b, base->mode, 309 block_index, desc_arr_idx); 310 } 311 312 if (idx == deref_chain->length) { 313 /* The entire deref was consumed in finding the block index. Return 314 * a pointer which just has a block index and a later access chain 315 * will dereference deeper. 316 */ 317 struct vtn_pointer *ptr = rzalloc(b, struct vtn_pointer); 318 ptr->mode = base->mode; 319 ptr->type = type; 320 ptr->block_index = block_index; 321 ptr->access = access; 322 return ptr; 323 } 324 325 /* If we got here, there's more access chain to handle and we have the 326 * final block index. Insert a descriptor load and cast to a deref to 327 * start the deref chain. 328 */ 329 nir_ssa_def *desc = vtn_descriptor_load(b, base->mode, block_index); 330 331 assert(base->mode == vtn_variable_mode_ssbo || 332 base->mode == vtn_variable_mode_ubo); 333 nir_variable_mode nir_mode = 334 base->mode == vtn_variable_mode_ssbo ? nir_var_mem_ssbo : nir_var_mem_ubo; 335 336 tail = nir_build_deref_cast(&b->nb, desc, nir_mode, type->type, 337 base->ptr_type->stride); 338 } else { 339 assert(base->var && base->var->var); 340 tail = nir_build_deref_var(&b->nb, base->var->var); 341 if (base->ptr_type && base->ptr_type->type) { 342 tail->dest.ssa.num_components = 343 glsl_get_vector_elements(base->ptr_type->type); 344 tail->dest.ssa.bit_size = glsl_get_bit_size(base->ptr_type->type); 345 } 346 } 347 348 if (idx == 0 && deref_chain->ptr_as_array) { 349 /* We start with a deref cast to get the stride. Hopefully, we'll be 350 * able to delete that cast eventually. 351 */ 352 tail = nir_build_deref_cast(&b->nb, &tail->dest.ssa, tail->mode, 353 tail->type, base->ptr_type->stride); 354 355 nir_ssa_def *index = vtn_access_link_as_ssa(b, deref_chain->link[0], 1, 356 tail->dest.ssa.bit_size); 357 tail = nir_build_deref_ptr_as_array(&b->nb, tail, index); 358 idx++; 359 } 360 361 for (; idx < deref_chain->length; idx++) { 362 if (glsl_type_is_struct_or_ifc(type->type)) { 363 vtn_assert(deref_chain->link[idx].mode == vtn_access_mode_literal); 364 unsigned field = deref_chain->link[idx].id; 365 tail = nir_build_deref_struct(&b->nb, tail, field); 366 type = type->members[field]; 367 } else { 368 nir_ssa_def *arr_index = 369 vtn_access_link_as_ssa(b, deref_chain->link[idx], 1, 370 tail->dest.ssa.bit_size); 371 tail = nir_build_deref_array(&b->nb, tail, arr_index); 372 type = type->array_element; 373 } 374 375 access |= type->access; 376 } 377 378 struct vtn_pointer *ptr = rzalloc(b, struct vtn_pointer); 379 ptr->mode = base->mode; 380 ptr->type = type; 381 ptr->var = base->var; 382 ptr->deref = tail; 383 ptr->access = access; 384 385 return ptr; 386} 387 388static struct vtn_pointer * 389vtn_ssa_offset_pointer_dereference(struct vtn_builder *b, 390 struct vtn_pointer *base, 391 struct vtn_access_chain *deref_chain) 392{ 393 nir_ssa_def *block_index = base->block_index; 394 nir_ssa_def *offset = base->offset; 395 struct vtn_type *type = base->type; 396 enum gl_access_qualifier access = base->access; 397 398 unsigned idx = 0; 399 if (base->mode == vtn_variable_mode_ubo || 400 base->mode == vtn_variable_mode_ssbo) { 401 if (!block_index) { 402 vtn_assert(base->var && base->type); 403 nir_ssa_def *desc_arr_idx; 404 if (glsl_type_is_array(type->type)) { 405 if (deref_chain->length >= 1) { 406 desc_arr_idx = 407 vtn_access_link_as_ssa(b, deref_chain->link[0], 1, 32); 408 idx++; 409 /* This consumes a level of type */ 410 type = type->array_element; 411 access |= type->access; 412 } else { 413 /* This is annoying. We've been asked for a pointer to the 414 * array of UBOs/SSBOs and not a specifc buffer. Return a 415 * pointer with a descriptor index of 0 and we'll have to do 416 * a reindex later to adjust it to the right thing. 417 */ 418 desc_arr_idx = nir_imm_int(&b->nb, 0); 419 } 420 } else if (deref_chain->ptr_as_array) { 421 /* You can't have a zero-length OpPtrAccessChain */ 422 vtn_assert(deref_chain->length >= 1); 423 desc_arr_idx = vtn_access_link_as_ssa(b, deref_chain->link[0], 1, 32); 424 } else { 425 /* We have a regular non-array SSBO. */ 426 desc_arr_idx = NULL; 427 } 428 block_index = vtn_variable_resource_index(b, base->var, desc_arr_idx); 429 } else if (deref_chain->ptr_as_array && 430 type->base_type == vtn_base_type_struct && type->block) { 431 /* We are doing an OpPtrAccessChain on a pointer to a struct that is 432 * decorated block. This is an interesting corner in the SPIR-V 433 * spec. One interpretation would be that they client is clearly 434 * trying to treat that block as if it's an implicit array of blocks 435 * repeated in the buffer. However, the SPIR-V spec for the 436 * OpPtrAccessChain says: 437 * 438 * "Base is treated as the address of the first element of an 439 * array, and the Element element’s address is computed to be the 440 * base for the Indexes, as per OpAccessChain." 441 * 442 * Taken literally, that would mean that your struct type is supposed 443 * to be treated as an array of such a struct and, since it's 444 * decorated block, that means an array of blocks which corresponds 445 * to an array descriptor. Therefore, we need to do a reindex 446 * operation to add the index from the first link in the access chain 447 * to the index we recieved. 448 * 449 * The downside to this interpretation (there always is one) is that 450 * this might be somewhat surprising behavior to apps if they expect 451 * the implicit array behavior described above. 452 */ 453 vtn_assert(deref_chain->length >= 1); 454 nir_ssa_def *offset_index = 455 vtn_access_link_as_ssa(b, deref_chain->link[0], 1, 32); 456 idx++; 457 458 block_index = vtn_resource_reindex(b, base->mode, 459 block_index, offset_index); 460 } 461 } 462 463 if (!offset) { 464 if (base->mode == vtn_variable_mode_workgroup) { 465 /* SLM doesn't need nor have a block index */ 466 vtn_assert(!block_index); 467 468 /* We need the variable for the base offset */ 469 vtn_assert(base->var); 470 471 /* We need ptr_type for size and alignment */ 472 vtn_assert(base->ptr_type); 473 474 /* Assign location on first use so that we don't end up bloating SLM 475 * address space for variables which are never statically used. 476 */ 477 if (base->var->shared_location < 0) { 478 vtn_assert(base->ptr_type->length > 0 && base->ptr_type->align > 0); 479 b->shader->num_shared = vtn_align_u32(b->shader->num_shared, 480 base->ptr_type->align); 481 base->var->shared_location = b->shader->num_shared; 482 b->shader->num_shared += base->ptr_type->length; 483 } 484 485 offset = nir_imm_int(&b->nb, base->var->shared_location); 486 } else if (base->mode == vtn_variable_mode_push_constant) { 487 /* Push constants neither need nor have a block index */ 488 vtn_assert(!block_index); 489 490 /* Start off with at the start of the push constant block. */ 491 offset = nir_imm_int(&b->nb, 0); 492 } else { 493 /* The code above should have ensured a block_index when needed. */ 494 vtn_assert(block_index); 495 496 /* Start off with at the start of the buffer. */ 497 offset = nir_imm_int(&b->nb, 0); 498 } 499 } 500 501 if (deref_chain->ptr_as_array && idx == 0) { 502 /* We need ptr_type for the stride */ 503 vtn_assert(base->ptr_type); 504 505 /* We need at least one element in the chain */ 506 vtn_assert(deref_chain->length >= 1); 507 508 nir_ssa_def *elem_offset = 509 vtn_access_link_as_ssa(b, deref_chain->link[idx], 510 base->ptr_type->stride, offset->bit_size); 511 offset = nir_iadd(&b->nb, offset, elem_offset); 512 idx++; 513 } 514 515 for (; idx < deref_chain->length; idx++) { 516 switch (glsl_get_base_type(type->type)) { 517 case GLSL_TYPE_UINT: 518 case GLSL_TYPE_INT: 519 case GLSL_TYPE_UINT16: 520 case GLSL_TYPE_INT16: 521 case GLSL_TYPE_UINT8: 522 case GLSL_TYPE_INT8: 523 case GLSL_TYPE_UINT64: 524 case GLSL_TYPE_INT64: 525 case GLSL_TYPE_FLOAT: 526 case GLSL_TYPE_FLOAT16: 527 case GLSL_TYPE_DOUBLE: 528 case GLSL_TYPE_BOOL: 529 case GLSL_TYPE_ARRAY: { 530 nir_ssa_def *elem_offset = 531 vtn_access_link_as_ssa(b, deref_chain->link[idx], 532 type->stride, offset->bit_size); 533 offset = nir_iadd(&b->nb, offset, elem_offset); 534 type = type->array_element; 535 access |= type->access; 536 break; 537 } 538 539 case GLSL_TYPE_INTERFACE: 540 case GLSL_TYPE_STRUCT: { 541 vtn_assert(deref_chain->link[idx].mode == vtn_access_mode_literal); 542 unsigned member = deref_chain->link[idx].id; 543 offset = nir_iadd_imm(&b->nb, offset, type->offsets[member]); 544 type = type->members[member]; 545 access |= type->access; 546 break; 547 } 548 549 default: 550 vtn_fail("Invalid type for deref"); 551 } 552 } 553 554 struct vtn_pointer *ptr = rzalloc(b, struct vtn_pointer); 555 ptr->mode = base->mode; 556 ptr->type = type; 557 ptr->block_index = block_index; 558 ptr->offset = offset; 559 ptr->access = access; 560 561 return ptr; 562} 563 564/* Dereference the given base pointer by the access chain */ 565static struct vtn_pointer * 566vtn_pointer_dereference(struct vtn_builder *b, 567 struct vtn_pointer *base, 568 struct vtn_access_chain *deref_chain) 569{ 570 if (vtn_pointer_uses_ssa_offset(b, base)) { 571 return vtn_ssa_offset_pointer_dereference(b, base, deref_chain); 572 } else { 573 return vtn_nir_deref_pointer_dereference(b, base, deref_chain); 574 } 575} 576 577struct vtn_pointer * 578vtn_pointer_for_variable(struct vtn_builder *b, 579 struct vtn_variable *var, struct vtn_type *ptr_type) 580{ 581 struct vtn_pointer *pointer = rzalloc(b, struct vtn_pointer); 582 583 pointer->mode = var->mode; 584 pointer->type = var->type; 585 vtn_assert(ptr_type->base_type == vtn_base_type_pointer); 586 vtn_assert(ptr_type->deref->type == var->type->type); 587 pointer->ptr_type = ptr_type; 588 pointer->var = var; 589 pointer->access = var->access | var->type->access; 590 591 return pointer; 592} 593 594/* Returns an atomic_uint type based on the original uint type. The returned 595 * type will be equivalent to the original one but will have an atomic_uint 596 * type as leaf instead of an uint. 597 * 598 * Manages uint scalars, arrays, and arrays of arrays of any nested depth. 599 */ 600static const struct glsl_type * 601repair_atomic_type(const struct glsl_type *type) 602{ 603 assert(glsl_get_base_type(glsl_without_array(type)) == GLSL_TYPE_UINT); 604 assert(glsl_type_is_scalar(glsl_without_array(type))); 605 606 if (glsl_type_is_array(type)) { 607 const struct glsl_type *atomic = 608 repair_atomic_type(glsl_get_array_element(type)); 609 610 return glsl_array_type(atomic, glsl_get_length(type), 611 glsl_get_explicit_stride(type)); 612 } else { 613 return glsl_atomic_uint_type(); 614 } 615} 616 617nir_deref_instr * 618vtn_pointer_to_deref(struct vtn_builder *b, struct vtn_pointer *ptr) 619{ 620 if (b->wa_glslang_179) { 621 /* Do on-the-fly copy propagation for samplers. */ 622 if (ptr->var && ptr->var->copy_prop_sampler) 623 return vtn_pointer_to_deref(b, ptr->var->copy_prop_sampler); 624 } 625 626 vtn_assert(!vtn_pointer_uses_ssa_offset(b, ptr)); 627 if (!ptr->deref) { 628 struct vtn_access_chain chain = { 629 .length = 0, 630 }; 631 ptr = vtn_nir_deref_pointer_dereference(b, ptr, &chain); 632 } 633 634 return ptr->deref; 635} 636 637static void 638_vtn_local_load_store(struct vtn_builder *b, bool load, nir_deref_instr *deref, 639 struct vtn_ssa_value *inout, 640 enum gl_access_qualifier access) 641{ 642 if (glsl_type_is_vector_or_scalar(deref->type)) { 643 if (load) { 644 inout->def = nir_load_deref_with_access(&b->nb, deref, access); 645 } else { 646 nir_store_deref_with_access(&b->nb, deref, inout->def, ~0, access); 647 } 648 } else if (glsl_type_is_array(deref->type) || 649 glsl_type_is_matrix(deref->type)) { 650 unsigned elems = glsl_get_length(deref->type); 651 for (unsigned i = 0; i < elems; i++) { 652 nir_deref_instr *child = 653 nir_build_deref_array_imm(&b->nb, deref, i); 654 _vtn_local_load_store(b, load, child, inout->elems[i], access); 655 } 656 } else { 657 vtn_assert(glsl_type_is_struct_or_ifc(deref->type)); 658 unsigned elems = glsl_get_length(deref->type); 659 for (unsigned i = 0; i < elems; i++) { 660 nir_deref_instr *child = nir_build_deref_struct(&b->nb, deref, i); 661 _vtn_local_load_store(b, load, child, inout->elems[i], access); 662 } 663 } 664} 665 666nir_deref_instr * 667vtn_nir_deref(struct vtn_builder *b, uint32_t id) 668{ 669 struct vtn_pointer *ptr = vtn_value(b, id, vtn_value_type_pointer)->pointer; 670 return vtn_pointer_to_deref(b, ptr); 671} 672 673/* 674 * Gets the NIR-level deref tail, which may have as a child an array deref 675 * selecting which component due to OpAccessChain supporting per-component 676 * indexing in SPIR-V. 677 */ 678static nir_deref_instr * 679get_deref_tail(nir_deref_instr *deref) 680{ 681 if (deref->deref_type != nir_deref_type_array) 682 return deref; 683 684 nir_deref_instr *parent = 685 nir_instr_as_deref(deref->parent.ssa->parent_instr); 686 687 if (glsl_type_is_vector(parent->type)) 688 return parent; 689 else 690 return deref; 691} 692 693struct vtn_ssa_value * 694vtn_local_load(struct vtn_builder *b, nir_deref_instr *src, 695 enum gl_access_qualifier access) 696{ 697 nir_deref_instr *src_tail = get_deref_tail(src); 698 struct vtn_ssa_value *val = vtn_create_ssa_value(b, src_tail->type); 699 _vtn_local_load_store(b, true, src_tail, val, access); 700 701 if (src_tail != src) { 702 val->type = src->type; 703 if (nir_src_is_const(src->arr.index)) 704 val->def = vtn_vector_extract(b, val->def, 705 nir_src_as_uint(src->arr.index)); 706 else 707 val->def = vtn_vector_extract_dynamic(b, val->def, src->arr.index.ssa); 708 } 709 710 return val; 711} 712 713void 714vtn_local_store(struct vtn_builder *b, struct vtn_ssa_value *src, 715 nir_deref_instr *dest, enum gl_access_qualifier access) 716{ 717 nir_deref_instr *dest_tail = get_deref_tail(dest); 718 719 if (dest_tail != dest) { 720 struct vtn_ssa_value *val = vtn_create_ssa_value(b, dest_tail->type); 721 _vtn_local_load_store(b, true, dest_tail, val, access); 722 723 if (nir_src_is_const(dest->arr.index)) 724 val->def = vtn_vector_insert(b, val->def, src->def, 725 nir_src_as_uint(dest->arr.index)); 726 else 727 val->def = vtn_vector_insert_dynamic(b, val->def, src->def, 728 dest->arr.index.ssa); 729 _vtn_local_load_store(b, false, dest_tail, val, access); 730 } else { 731 _vtn_local_load_store(b, false, dest_tail, src, access); 732 } 733} 734 735nir_ssa_def * 736vtn_pointer_to_offset(struct vtn_builder *b, struct vtn_pointer *ptr, 737 nir_ssa_def **index_out) 738{ 739 assert(vtn_pointer_uses_ssa_offset(b, ptr)); 740 if (!ptr->offset) { 741 struct vtn_access_chain chain = { 742 .length = 0, 743 }; 744 ptr = vtn_ssa_offset_pointer_dereference(b, ptr, &chain); 745 } 746 *index_out = ptr->block_index; 747 return ptr->offset; 748} 749 750/* Tries to compute the size of an interface block based on the strides and 751 * offsets that are provided to us in the SPIR-V source. 752 */ 753static unsigned 754vtn_type_block_size(struct vtn_builder *b, struct vtn_type *type) 755{ 756 enum glsl_base_type base_type = glsl_get_base_type(type->type); 757 switch (base_type) { 758 case GLSL_TYPE_UINT: 759 case GLSL_TYPE_INT: 760 case GLSL_TYPE_UINT16: 761 case GLSL_TYPE_INT16: 762 case GLSL_TYPE_UINT8: 763 case GLSL_TYPE_INT8: 764 case GLSL_TYPE_UINT64: 765 case GLSL_TYPE_INT64: 766 case GLSL_TYPE_FLOAT: 767 case GLSL_TYPE_FLOAT16: 768 case GLSL_TYPE_BOOL: 769 case GLSL_TYPE_DOUBLE: { 770 unsigned cols = type->row_major ? glsl_get_vector_elements(type->type) : 771 glsl_get_matrix_columns(type->type); 772 if (cols > 1) { 773 vtn_assert(type->stride > 0); 774 return type->stride * cols; 775 } else { 776 unsigned type_size = glsl_get_bit_size(type->type) / 8; 777 return glsl_get_vector_elements(type->type) * type_size; 778 } 779 } 780 781 case GLSL_TYPE_STRUCT: 782 case GLSL_TYPE_INTERFACE: { 783 unsigned size = 0; 784 unsigned num_fields = glsl_get_length(type->type); 785 for (unsigned f = 0; f < num_fields; f++) { 786 unsigned field_end = type->offsets[f] + 787 vtn_type_block_size(b, type->members[f]); 788 size = MAX2(size, field_end); 789 } 790 return size; 791 } 792 793 case GLSL_TYPE_ARRAY: 794 vtn_assert(type->stride > 0); 795 vtn_assert(glsl_get_length(type->type) > 0); 796 return type->stride * glsl_get_length(type->type); 797 798 default: 799 vtn_fail("Invalid block type"); 800 return 0; 801 } 802} 803 804static void 805_vtn_load_store_tail(struct vtn_builder *b, nir_intrinsic_op op, bool load, 806 nir_ssa_def *index, nir_ssa_def *offset, 807 unsigned access_offset, unsigned access_size, 808 struct vtn_ssa_value **inout, const struct glsl_type *type, 809 enum gl_access_qualifier access) 810{ 811 nir_intrinsic_instr *instr = nir_intrinsic_instr_create(b->nb.shader, op); 812 instr->num_components = glsl_get_vector_elements(type); 813 814 /* Booleans usually shouldn't show up in external memory in SPIR-V. 815 * However, they do for certain older GLSLang versions and can for shared 816 * memory when we lower access chains internally. 817 */ 818 const unsigned data_bit_size = glsl_type_is_boolean(type) ? 32 : 819 glsl_get_bit_size(type); 820 821 int src = 0; 822 if (!load) { 823 nir_intrinsic_set_write_mask(instr, (1 << instr->num_components) - 1); 824 instr->src[src++] = nir_src_for_ssa((*inout)->def); 825 } 826 827 if (op == nir_intrinsic_load_push_constant) { 828 nir_intrinsic_set_base(instr, access_offset); 829 nir_intrinsic_set_range(instr, access_size); 830 } 831 832 if (op == nir_intrinsic_load_ubo || 833 op == nir_intrinsic_load_ssbo || 834 op == nir_intrinsic_store_ssbo) { 835 nir_intrinsic_set_access(instr, access); 836 } 837 838 /* With extensions like relaxed_block_layout, we really can't guarantee 839 * much more than scalar alignment. 840 */ 841 if (op != nir_intrinsic_load_push_constant) 842 nir_intrinsic_set_align(instr, data_bit_size / 8, 0); 843 844 if (index) 845 instr->src[src++] = nir_src_for_ssa(index); 846 847 if (op == nir_intrinsic_load_push_constant) { 848 /* We need to subtract the offset from where the intrinsic will load the 849 * data. */ 850 instr->src[src++] = 851 nir_src_for_ssa(nir_isub(&b->nb, offset, 852 nir_imm_int(&b->nb, access_offset))); 853 } else { 854 instr->src[src++] = nir_src_for_ssa(offset); 855 } 856 857 if (load) { 858 nir_ssa_dest_init(&instr->instr, &instr->dest, 859 instr->num_components, data_bit_size, NULL); 860 (*inout)->def = &instr->dest.ssa; 861 } 862 863 nir_builder_instr_insert(&b->nb, &instr->instr); 864 865 if (load && glsl_get_base_type(type) == GLSL_TYPE_BOOL) 866 (*inout)->def = nir_ine(&b->nb, (*inout)->def, nir_imm_int(&b->nb, 0)); 867} 868 869static void 870_vtn_block_load_store(struct vtn_builder *b, nir_intrinsic_op op, bool load, 871 nir_ssa_def *index, nir_ssa_def *offset, 872 unsigned access_offset, unsigned access_size, 873 struct vtn_type *type, enum gl_access_qualifier access, 874 struct vtn_ssa_value **inout) 875{ 876 if (load && *inout == NULL) 877 *inout = vtn_create_ssa_value(b, type->type); 878 879 enum glsl_base_type base_type = glsl_get_base_type(type->type); 880 switch (base_type) { 881 case GLSL_TYPE_UINT: 882 case GLSL_TYPE_INT: 883 case GLSL_TYPE_UINT16: 884 case GLSL_TYPE_INT16: 885 case GLSL_TYPE_UINT8: 886 case GLSL_TYPE_INT8: 887 case GLSL_TYPE_UINT64: 888 case GLSL_TYPE_INT64: 889 case GLSL_TYPE_FLOAT: 890 case GLSL_TYPE_FLOAT16: 891 case GLSL_TYPE_DOUBLE: 892 case GLSL_TYPE_BOOL: 893 /* This is where things get interesting. At this point, we've hit 894 * a vector, a scalar, or a matrix. 895 */ 896 if (glsl_type_is_matrix(type->type)) { 897 /* Loading the whole matrix */ 898 struct vtn_ssa_value *transpose; 899 unsigned num_ops, vec_width, col_stride; 900 if (type->row_major) { 901 num_ops = glsl_get_vector_elements(type->type); 902 vec_width = glsl_get_matrix_columns(type->type); 903 col_stride = type->array_element->stride; 904 if (load) { 905 const struct glsl_type *transpose_type = 906 glsl_matrix_type(base_type, vec_width, num_ops); 907 *inout = vtn_create_ssa_value(b, transpose_type); 908 } else { 909 transpose = vtn_ssa_transpose(b, *inout); 910 inout = &transpose; 911 } 912 } else { 913 num_ops = glsl_get_matrix_columns(type->type); 914 vec_width = glsl_get_vector_elements(type->type); 915 col_stride = type->stride; 916 } 917 918 for (unsigned i = 0; i < num_ops; i++) { 919 nir_ssa_def *elem_offset = 920 nir_iadd_imm(&b->nb, offset, i * col_stride); 921 _vtn_load_store_tail(b, op, load, index, elem_offset, 922 access_offset, access_size, 923 &(*inout)->elems[i], 924 glsl_vector_type(base_type, vec_width), 925 type->access | access); 926 } 927 928 if (load && type->row_major) 929 *inout = vtn_ssa_transpose(b, *inout); 930 } else { 931 unsigned elems = glsl_get_vector_elements(type->type); 932 unsigned type_size = glsl_get_bit_size(type->type) / 8; 933 if (elems == 1 || type->stride == type_size) { 934 /* This is a tightly-packed normal scalar or vector load */ 935 vtn_assert(glsl_type_is_vector_or_scalar(type->type)); 936 _vtn_load_store_tail(b, op, load, index, offset, 937 access_offset, access_size, 938 inout, type->type, 939 type->access | access); 940 } else { 941 /* This is a strided load. We have to load N things separately. 942 * This is the single column of a row-major matrix case. 943 */ 944 vtn_assert(type->stride > type_size); 945 vtn_assert(type->stride % type_size == 0); 946 947 nir_ssa_def *per_comp[4]; 948 for (unsigned i = 0; i < elems; i++) { 949 nir_ssa_def *elem_offset = 950 nir_iadd_imm(&b->nb, offset, i * type->stride); 951 struct vtn_ssa_value *comp, temp_val; 952 if (!load) { 953 temp_val.def = nir_channel(&b->nb, (*inout)->def, i); 954 temp_val.type = glsl_scalar_type(base_type); 955 } 956 comp = &temp_val; 957 _vtn_load_store_tail(b, op, load, index, elem_offset, 958 access_offset, access_size, 959 &comp, glsl_scalar_type(base_type), 960 type->access | access); 961 per_comp[i] = comp->def; 962 } 963 964 if (load) { 965 if (*inout == NULL) 966 *inout = vtn_create_ssa_value(b, type->type); 967 (*inout)->def = nir_vec(&b->nb, per_comp, elems); 968 } 969 } 970 } 971 return; 972 973 case GLSL_TYPE_ARRAY: { 974 unsigned elems = glsl_get_length(type->type); 975 for (unsigned i = 0; i < elems; i++) { 976 nir_ssa_def *elem_off = 977 nir_iadd_imm(&b->nb, offset, i * type->stride); 978 _vtn_block_load_store(b, op, load, index, elem_off, 979 access_offset, access_size, 980 type->array_element, 981 type->array_element->access | access, 982 &(*inout)->elems[i]); 983 } 984 return; 985 } 986 987 case GLSL_TYPE_INTERFACE: 988 case GLSL_TYPE_STRUCT: { 989 unsigned elems = glsl_get_length(type->type); 990 for (unsigned i = 0; i < elems; i++) { 991 nir_ssa_def *elem_off = 992 nir_iadd_imm(&b->nb, offset, type->offsets[i]); 993 _vtn_block_load_store(b, op, load, index, elem_off, 994 access_offset, access_size, 995 type->members[i], 996 type->members[i]->access | access, 997 &(*inout)->elems[i]); 998 } 999 return; 1000 } 1001 1002 default: 1003 vtn_fail("Invalid block member type"); 1004 } 1005} 1006 1007static struct vtn_ssa_value * 1008vtn_block_load(struct vtn_builder *b, struct vtn_pointer *src) 1009{ 1010 nir_intrinsic_op op; 1011 unsigned access_offset = 0, access_size = 0; 1012 switch (src->mode) { 1013 case vtn_variable_mode_ubo: 1014 op = nir_intrinsic_load_ubo; 1015 break; 1016 case vtn_variable_mode_ssbo: 1017 op = nir_intrinsic_load_ssbo; 1018 break; 1019 case vtn_variable_mode_push_constant: 1020 op = nir_intrinsic_load_push_constant; 1021 access_size = b->shader->num_uniforms; 1022 break; 1023 case vtn_variable_mode_workgroup: 1024 op = nir_intrinsic_load_shared; 1025 break; 1026 default: 1027 vtn_fail("Invalid block variable mode"); 1028 } 1029 1030 nir_ssa_def *offset, *index = NULL; 1031 offset = vtn_pointer_to_offset(b, src, &index); 1032 1033 struct vtn_ssa_value *value = NULL; 1034 _vtn_block_load_store(b, op, true, index, offset, 1035 access_offset, access_size, 1036 src->type, src->access, &value); 1037 return value; 1038} 1039 1040static void 1041vtn_block_store(struct vtn_builder *b, struct vtn_ssa_value *src, 1042 struct vtn_pointer *dst) 1043{ 1044 nir_intrinsic_op op; 1045 switch (dst->mode) { 1046 case vtn_variable_mode_ssbo: 1047 op = nir_intrinsic_store_ssbo; 1048 break; 1049 case vtn_variable_mode_workgroup: 1050 op = nir_intrinsic_store_shared; 1051 break; 1052 default: 1053 vtn_fail("Invalid block variable mode"); 1054 } 1055 1056 nir_ssa_def *offset, *index = NULL; 1057 offset = vtn_pointer_to_offset(b, dst, &index); 1058 1059 _vtn_block_load_store(b, op, false, index, offset, 1060 0, 0, dst->type, dst->access, &src); 1061} 1062 1063static void 1064_vtn_variable_load_store(struct vtn_builder *b, bool load, 1065 struct vtn_pointer *ptr, 1066 enum gl_access_qualifier access, 1067 struct vtn_ssa_value **inout) 1068{ 1069 enum glsl_base_type base_type = glsl_get_base_type(ptr->type->type); 1070 switch (base_type) { 1071 case GLSL_TYPE_UINT: 1072 case GLSL_TYPE_INT: 1073 case GLSL_TYPE_UINT16: 1074 case GLSL_TYPE_INT16: 1075 case GLSL_TYPE_UINT8: 1076 case GLSL_TYPE_INT8: 1077 case GLSL_TYPE_UINT64: 1078 case GLSL_TYPE_INT64: 1079 case GLSL_TYPE_FLOAT: 1080 case GLSL_TYPE_FLOAT16: 1081 case GLSL_TYPE_BOOL: 1082 case GLSL_TYPE_DOUBLE: 1083 if (glsl_type_is_vector_or_scalar(ptr->type->type)) { 1084 /* We hit a vector or scalar; go ahead and emit the load[s] */ 1085 nir_deref_instr *deref = vtn_pointer_to_deref(b, ptr); 1086 if (vtn_pointer_is_external_block(b, ptr)) { 1087 /* If it's external, we call nir_load/store_deref directly. The 1088 * vtn_local_load/store helpers are too clever and do magic to 1089 * avoid array derefs of vectors. That magic is both less 1090 * efficient than the direct load/store and, in the case of 1091 * stores, is broken because it creates a race condition if two 1092 * threads are writing to different components of the same vector 1093 * due to the load+insert+store it uses to emulate the array 1094 * deref. 1095 */ 1096 if (load) { 1097 *inout = vtn_create_ssa_value(b, ptr->type->type); 1098 (*inout)->def = nir_load_deref_with_access(&b->nb, deref, 1099 ptr->type->access | access); 1100 } else { 1101 nir_store_deref_with_access(&b->nb, deref, (*inout)->def, ~0, 1102 ptr->type->access | access); 1103 } 1104 } else { 1105 if (load) { 1106 *inout = vtn_local_load(b, deref, ptr->type->access | access); 1107 } else { 1108 vtn_local_store(b, *inout, deref, ptr->type->access | access); 1109 } 1110 } 1111 return; 1112 } 1113 /* Fall through */ 1114 1115 case GLSL_TYPE_INTERFACE: 1116 case GLSL_TYPE_ARRAY: 1117 case GLSL_TYPE_STRUCT: { 1118 unsigned elems = glsl_get_length(ptr->type->type); 1119 if (load) { 1120 vtn_assert(*inout == NULL); 1121 *inout = rzalloc(b, struct vtn_ssa_value); 1122 (*inout)->type = ptr->type->type; 1123 (*inout)->elems = rzalloc_array(b, struct vtn_ssa_value *, elems); 1124 } 1125 1126 struct vtn_access_chain chain = { 1127 .length = 1, 1128 .link = { 1129 { .mode = vtn_access_mode_literal, }, 1130 } 1131 }; 1132 for (unsigned i = 0; i < elems; i++) { 1133 chain.link[0].id = i; 1134 struct vtn_pointer *elem = vtn_pointer_dereference(b, ptr, &chain); 1135 _vtn_variable_load_store(b, load, elem, ptr->type->access | access, 1136 &(*inout)->elems[i]); 1137 } 1138 return; 1139 } 1140 1141 default: 1142 vtn_fail("Invalid access chain type"); 1143 } 1144} 1145 1146struct vtn_ssa_value * 1147vtn_variable_load(struct vtn_builder *b, struct vtn_pointer *src) 1148{ 1149 if (vtn_pointer_uses_ssa_offset(b, src)) { 1150 return vtn_block_load(b, src); 1151 } else { 1152 struct vtn_ssa_value *val = NULL; 1153 _vtn_variable_load_store(b, true, src, src->access, &val); 1154 return val; 1155 } 1156} 1157 1158void 1159vtn_variable_store(struct vtn_builder *b, struct vtn_ssa_value *src, 1160 struct vtn_pointer *dest) 1161{ 1162 if (vtn_pointer_uses_ssa_offset(b, dest)) { 1163 vtn_assert(dest->mode == vtn_variable_mode_ssbo || 1164 dest->mode == vtn_variable_mode_workgroup); 1165 vtn_block_store(b, src, dest); 1166 } else { 1167 _vtn_variable_load_store(b, false, dest, dest->access, &src); 1168 } 1169} 1170 1171static void 1172_vtn_variable_copy(struct vtn_builder *b, struct vtn_pointer *dest, 1173 struct vtn_pointer *src) 1174{ 1175 vtn_assert(src->type->type == dest->type->type); 1176 enum glsl_base_type base_type = glsl_get_base_type(src->type->type); 1177 switch (base_type) { 1178 case GLSL_TYPE_UINT: 1179 case GLSL_TYPE_INT: 1180 case GLSL_TYPE_UINT16: 1181 case GLSL_TYPE_INT16: 1182 case GLSL_TYPE_UINT8: 1183 case GLSL_TYPE_INT8: 1184 case GLSL_TYPE_UINT64: 1185 case GLSL_TYPE_INT64: 1186 case GLSL_TYPE_FLOAT: 1187 case GLSL_TYPE_FLOAT16: 1188 case GLSL_TYPE_DOUBLE: 1189 case GLSL_TYPE_BOOL: 1190 /* At this point, we have a scalar, vector, or matrix so we know that 1191 * there cannot be any structure splitting still in the way. By 1192 * stopping at the matrix level rather than the vector level, we 1193 * ensure that matrices get loaded in the optimal way even if they 1194 * are storred row-major in a UBO. 1195 */ 1196 vtn_variable_store(b, vtn_variable_load(b, src), dest); 1197 return; 1198 1199 case GLSL_TYPE_INTERFACE: 1200 case GLSL_TYPE_ARRAY: 1201 case GLSL_TYPE_STRUCT: { 1202 struct vtn_access_chain chain = { 1203 .length = 1, 1204 .link = { 1205 { .mode = vtn_access_mode_literal, }, 1206 } 1207 }; 1208 unsigned elems = glsl_get_length(src->type->type); 1209 for (unsigned i = 0; i < elems; i++) { 1210 chain.link[0].id = i; 1211 struct vtn_pointer *src_elem = 1212 vtn_pointer_dereference(b, src, &chain); 1213 struct vtn_pointer *dest_elem = 1214 vtn_pointer_dereference(b, dest, &chain); 1215 1216 _vtn_variable_copy(b, dest_elem, src_elem); 1217 } 1218 return; 1219 } 1220 1221 default: 1222 vtn_fail("Invalid access chain type"); 1223 } 1224} 1225 1226static void 1227vtn_variable_copy(struct vtn_builder *b, struct vtn_pointer *dest, 1228 struct vtn_pointer *src) 1229{ 1230 /* TODO: At some point, we should add a special-case for when we can 1231 * just emit a copy_var intrinsic. 1232 */ 1233 _vtn_variable_copy(b, dest, src); 1234} 1235 1236static void 1237set_mode_system_value(struct vtn_builder *b, nir_variable_mode *mode) 1238{ 1239 vtn_assert(*mode == nir_var_system_value || *mode == nir_var_shader_in); 1240 *mode = nir_var_system_value; 1241} 1242 1243static void 1244vtn_get_builtin_location(struct vtn_builder *b, 1245 SpvBuiltIn builtin, int *location, 1246 nir_variable_mode *mode) 1247{ 1248 switch (builtin) { 1249 case SpvBuiltInPosition: 1250 *location = VARYING_SLOT_POS; 1251 break; 1252 case SpvBuiltInPointSize: 1253 *location = VARYING_SLOT_PSIZ; 1254 break; 1255 case SpvBuiltInClipDistance: 1256 *location = VARYING_SLOT_CLIP_DIST0; /* XXX CLIP_DIST1? */ 1257 break; 1258 case SpvBuiltInCullDistance: 1259 *location = VARYING_SLOT_CULL_DIST0; 1260 break; 1261 case SpvBuiltInVertexId: 1262 case SpvBuiltInVertexIndex: 1263 /* The Vulkan spec defines VertexIndex to be non-zero-based and doesn't 1264 * allow VertexId. The ARB_gl_spirv spec defines VertexId to be the 1265 * same as gl_VertexID, which is non-zero-based, and removes 1266 * VertexIndex. Since they're both defined to be non-zero-based, we use 1267 * SYSTEM_VALUE_VERTEX_ID for both. 1268 */ 1269 *location = SYSTEM_VALUE_VERTEX_ID; 1270 set_mode_system_value(b, mode); 1271 break; 1272 case SpvBuiltInInstanceIndex: 1273 *location = SYSTEM_VALUE_INSTANCE_INDEX; 1274 set_mode_system_value(b, mode); 1275 break; 1276 case SpvBuiltInInstanceId: 1277 *location = SYSTEM_VALUE_INSTANCE_ID; 1278 set_mode_system_value(b, mode); 1279 break; 1280 case SpvBuiltInPrimitiveId: 1281 if (b->shader->info.stage == MESA_SHADER_FRAGMENT) { 1282 vtn_assert(*mode == nir_var_shader_in); 1283 *location = VARYING_SLOT_PRIMITIVE_ID; 1284 } else if (*mode == nir_var_shader_out) { 1285 *location = VARYING_SLOT_PRIMITIVE_ID; 1286 } else { 1287 *location = SYSTEM_VALUE_PRIMITIVE_ID; 1288 set_mode_system_value(b, mode); 1289 } 1290 break; 1291 case SpvBuiltInInvocationId: 1292 *location = SYSTEM_VALUE_INVOCATION_ID; 1293 set_mode_system_value(b, mode); 1294 break; 1295 case SpvBuiltInLayer: 1296 *location = VARYING_SLOT_LAYER; 1297 if (b->shader->info.stage == MESA_SHADER_FRAGMENT) 1298 *mode = nir_var_shader_in; 1299 else if (b->shader->info.stage == MESA_SHADER_GEOMETRY) 1300 *mode = nir_var_shader_out; 1301 else if (b->options && b->options->caps.shader_viewport_index_layer && 1302 (b->shader->info.stage == MESA_SHADER_VERTEX || 1303 b->shader->info.stage == MESA_SHADER_TESS_EVAL)) 1304 *mode = nir_var_shader_out; 1305 else 1306 vtn_fail("invalid stage for SpvBuiltInLayer"); 1307 break; 1308 case SpvBuiltInViewportIndex: 1309 *location = VARYING_SLOT_VIEWPORT; 1310 if (b->shader->info.stage == MESA_SHADER_GEOMETRY) 1311 *mode = nir_var_shader_out; 1312 else if (b->options && b->options->caps.shader_viewport_index_layer && 1313 (b->shader->info.stage == MESA_SHADER_VERTEX || 1314 b->shader->info.stage == MESA_SHADER_TESS_EVAL)) 1315 *mode = nir_var_shader_out; 1316 else if (b->shader->info.stage == MESA_SHADER_FRAGMENT) 1317 *mode = nir_var_shader_in; 1318 else 1319 vtn_fail("invalid stage for SpvBuiltInViewportIndex"); 1320 break; 1321 case SpvBuiltInTessLevelOuter: 1322 *location = VARYING_SLOT_TESS_LEVEL_OUTER; 1323 break; 1324 case SpvBuiltInTessLevelInner: 1325 *location = VARYING_SLOT_TESS_LEVEL_INNER; 1326 break; 1327 case SpvBuiltInTessCoord: 1328 *location = SYSTEM_VALUE_TESS_COORD; 1329 set_mode_system_value(b, mode); 1330 break; 1331 case SpvBuiltInPatchVertices: 1332 *location = SYSTEM_VALUE_VERTICES_IN; 1333 set_mode_system_value(b, mode); 1334 break; 1335 case SpvBuiltInFragCoord: 1336 *location = VARYING_SLOT_POS; 1337 vtn_assert(*mode == nir_var_shader_in); 1338 break; 1339 case SpvBuiltInPointCoord: 1340 *location = VARYING_SLOT_PNTC; 1341 vtn_assert(*mode == nir_var_shader_in); 1342 break; 1343 case SpvBuiltInFrontFacing: 1344 *location = SYSTEM_VALUE_FRONT_FACE; 1345 set_mode_system_value(b, mode); 1346 break; 1347 case SpvBuiltInSampleId: 1348 *location = SYSTEM_VALUE_SAMPLE_ID; 1349 set_mode_system_value(b, mode); 1350 break; 1351 case SpvBuiltInSamplePosition: 1352 *location = SYSTEM_VALUE_SAMPLE_POS; 1353 set_mode_system_value(b, mode); 1354 break; 1355 case SpvBuiltInSampleMask: 1356 if (*mode == nir_var_shader_out) { 1357 *location = FRAG_RESULT_SAMPLE_MASK; 1358 } else { 1359 *location = SYSTEM_VALUE_SAMPLE_MASK_IN; 1360 set_mode_system_value(b, mode); 1361 } 1362 break; 1363 case SpvBuiltInFragDepth: 1364 *location = FRAG_RESULT_DEPTH; 1365 vtn_assert(*mode == nir_var_shader_out); 1366 break; 1367 case SpvBuiltInHelperInvocation: 1368 *location = SYSTEM_VALUE_HELPER_INVOCATION; 1369 set_mode_system_value(b, mode); 1370 break; 1371 case SpvBuiltInNumWorkgroups: 1372 *location = SYSTEM_VALUE_NUM_WORK_GROUPS; 1373 set_mode_system_value(b, mode); 1374 break; 1375 case SpvBuiltInWorkgroupSize: 1376 *location = SYSTEM_VALUE_LOCAL_GROUP_SIZE; 1377 set_mode_system_value(b, mode); 1378 break; 1379 case SpvBuiltInWorkgroupId: 1380 *location = SYSTEM_VALUE_WORK_GROUP_ID; 1381 set_mode_system_value(b, mode); 1382 break; 1383 case SpvBuiltInLocalInvocationId: 1384 *location = SYSTEM_VALUE_LOCAL_INVOCATION_ID; 1385 set_mode_system_value(b, mode); 1386 break; 1387 case SpvBuiltInLocalInvocationIndex: 1388 *location = SYSTEM_VALUE_LOCAL_INVOCATION_INDEX; 1389 set_mode_system_value(b, mode); 1390 break; 1391 case SpvBuiltInGlobalInvocationId: 1392 *location = SYSTEM_VALUE_GLOBAL_INVOCATION_ID; 1393 set_mode_system_value(b, mode); 1394 break; 1395 case SpvBuiltInGlobalLinearId: 1396 *location = SYSTEM_VALUE_GLOBAL_INVOCATION_INDEX; 1397 set_mode_system_value(b, mode); 1398 break; 1399 case SpvBuiltInBaseVertex: 1400 /* OpenGL gl_BaseVertex (SYSTEM_VALUE_BASE_VERTEX) is not the same 1401 * semantic as SPIR-V BaseVertex (SYSTEM_VALUE_FIRST_VERTEX). 1402 */ 1403 *location = SYSTEM_VALUE_FIRST_VERTEX; 1404 set_mode_system_value(b, mode); 1405 break; 1406 case SpvBuiltInBaseInstance: 1407 *location = SYSTEM_VALUE_BASE_INSTANCE; 1408 set_mode_system_value(b, mode); 1409 break; 1410 case SpvBuiltInDrawIndex: 1411 *location = SYSTEM_VALUE_DRAW_ID; 1412 set_mode_system_value(b, mode); 1413 break; 1414 case SpvBuiltInSubgroupSize: 1415 *location = SYSTEM_VALUE_SUBGROUP_SIZE; 1416 set_mode_system_value(b, mode); 1417 break; 1418 case SpvBuiltInSubgroupId: 1419 *location = SYSTEM_VALUE_SUBGROUP_ID; 1420 set_mode_system_value(b, mode); 1421 break; 1422 case SpvBuiltInSubgroupLocalInvocationId: 1423 *location = SYSTEM_VALUE_SUBGROUP_INVOCATION; 1424 set_mode_system_value(b, mode); 1425 break; 1426 case SpvBuiltInNumSubgroups: 1427 *location = SYSTEM_VALUE_NUM_SUBGROUPS; 1428 set_mode_system_value(b, mode); 1429 break; 1430 case SpvBuiltInDeviceIndex: 1431 *location = SYSTEM_VALUE_DEVICE_INDEX; 1432 set_mode_system_value(b, mode); 1433 break; 1434 case SpvBuiltInViewIndex: 1435 *location = SYSTEM_VALUE_VIEW_INDEX; 1436 set_mode_system_value(b, mode); 1437 break; 1438 case SpvBuiltInSubgroupEqMask: 1439 *location = SYSTEM_VALUE_SUBGROUP_EQ_MASK, 1440 set_mode_system_value(b, mode); 1441 break; 1442 case SpvBuiltInSubgroupGeMask: 1443 *location = SYSTEM_VALUE_SUBGROUP_GE_MASK, 1444 set_mode_system_value(b, mode); 1445 break; 1446 case SpvBuiltInSubgroupGtMask: 1447 *location = SYSTEM_VALUE_SUBGROUP_GT_MASK, 1448 set_mode_system_value(b, mode); 1449 break; 1450 case SpvBuiltInSubgroupLeMask: 1451 *location = SYSTEM_VALUE_SUBGROUP_LE_MASK, 1452 set_mode_system_value(b, mode); 1453 break; 1454 case SpvBuiltInSubgroupLtMask: 1455 *location = SYSTEM_VALUE_SUBGROUP_LT_MASK, 1456 set_mode_system_value(b, mode); 1457 break; 1458 case SpvBuiltInFragStencilRefEXT: 1459 *location = FRAG_RESULT_STENCIL; 1460 vtn_assert(*mode == nir_var_shader_out); 1461 break; 1462 case SpvBuiltInWorkDim: 1463 *location = SYSTEM_VALUE_WORK_DIM; 1464 set_mode_system_value(b, mode); 1465 break; 1466 case SpvBuiltInGlobalSize: 1467 *location = SYSTEM_VALUE_GLOBAL_GROUP_SIZE; 1468 set_mode_system_value(b, mode); 1469 break; 1470 default: 1471 vtn_fail("Unsupported builtin: %s (%u)", 1472 spirv_builtin_to_string(builtin), builtin); 1473 } 1474} 1475 1476static void 1477apply_var_decoration(struct vtn_builder *b, 1478 struct nir_variable_data *var_data, 1479 const struct vtn_decoration *dec) 1480{ 1481 switch (dec->decoration) { 1482 case SpvDecorationRelaxedPrecision: 1483 break; /* FIXME: Do nothing with this for now. */ 1484 case SpvDecorationNoPerspective: 1485 var_data->interpolation = INTERP_MODE_NOPERSPECTIVE; 1486 break; 1487 case SpvDecorationFlat: 1488 var_data->interpolation = INTERP_MODE_FLAT; 1489 break; 1490 case SpvDecorationCentroid: 1491 var_data->centroid = true; 1492 break; 1493 case SpvDecorationSample: 1494 var_data->sample = true; 1495 break; 1496 case SpvDecorationInvariant: 1497 var_data->invariant = true; 1498 break; 1499 case SpvDecorationConstant: 1500 var_data->read_only = true; 1501 break; 1502 case SpvDecorationNonReadable: 1503 var_data->image.access |= ACCESS_NON_READABLE; 1504 break; 1505 case SpvDecorationNonWritable: 1506 var_data->read_only = true; 1507 var_data->image.access |= ACCESS_NON_WRITEABLE; 1508 break; 1509 case SpvDecorationRestrict: 1510 var_data->image.access |= ACCESS_RESTRICT; 1511 break; 1512 case SpvDecorationVolatile: 1513 var_data->image.access |= ACCESS_VOLATILE; 1514 break; 1515 case SpvDecorationCoherent: 1516 var_data->image.access |= ACCESS_COHERENT; 1517 break; 1518 case SpvDecorationComponent: 1519 var_data->location_frac = dec->operands[0]; 1520 break; 1521 case SpvDecorationIndex: 1522 var_data->index = dec->operands[0]; 1523 break; 1524 case SpvDecorationBuiltIn: { 1525 SpvBuiltIn builtin = dec->operands[0]; 1526 1527 nir_variable_mode mode = var_data->mode; 1528 vtn_get_builtin_location(b, builtin, &var_data->location, &mode); 1529 var_data->mode = mode; 1530 1531 switch (builtin) { 1532 case SpvBuiltInTessLevelOuter: 1533 case SpvBuiltInTessLevelInner: 1534 case SpvBuiltInClipDistance: 1535 case SpvBuiltInCullDistance: 1536 var_data->compact = true; 1537 break; 1538 default: 1539 break; 1540 } 1541 } 1542 1543 case SpvDecorationSpecId: 1544 case SpvDecorationRowMajor: 1545 case SpvDecorationColMajor: 1546 case SpvDecorationMatrixStride: 1547 case SpvDecorationAliased: 1548 case SpvDecorationUniform: 1549 case SpvDecorationLinkageAttributes: 1550 break; /* Do nothing with these here */ 1551 1552 case SpvDecorationPatch: 1553 var_data->patch = true; 1554 break; 1555 1556 case SpvDecorationLocation: 1557 vtn_fail("Handled above"); 1558 1559 case SpvDecorationBlock: 1560 case SpvDecorationBufferBlock: 1561 case SpvDecorationArrayStride: 1562 case SpvDecorationGLSLShared: 1563 case SpvDecorationGLSLPacked: 1564 break; /* These can apply to a type but we don't care about them */ 1565 1566 case SpvDecorationBinding: 1567 case SpvDecorationDescriptorSet: 1568 case SpvDecorationNoContraction: 1569 case SpvDecorationInputAttachmentIndex: 1570 vtn_warn("Decoration not allowed for variable or structure member: %s", 1571 spirv_decoration_to_string(dec->decoration)); 1572 break; 1573 1574 case SpvDecorationXfbBuffer: 1575 var_data->explicit_xfb_buffer = true; 1576 var_data->xfb_buffer = dec->operands[0]; 1577 var_data->always_active_io = true; 1578 break; 1579 case SpvDecorationXfbStride: 1580 var_data->explicit_xfb_stride = true; 1581 var_data->xfb_stride = dec->operands[0]; 1582 break; 1583 case SpvDecorationOffset: 1584 var_data->explicit_offset = true; 1585 var_data->offset = dec->operands[0]; 1586 break; 1587 1588 case SpvDecorationStream: 1589 var_data->stream = dec->operands[0]; 1590 break; 1591 1592 case SpvDecorationCPacked: 1593 case SpvDecorationSaturatedConversion: 1594 case SpvDecorationFuncParamAttr: 1595 case SpvDecorationFPRoundingMode: 1596 case SpvDecorationFPFastMathMode: 1597 case SpvDecorationAlignment: 1598 if (b->shader->info.stage != MESA_SHADER_KERNEL) { 1599 vtn_warn("Decoration only allowed for CL-style kernels: %s", 1600 spirv_decoration_to_string(dec->decoration)); 1601 } 1602 break; 1603 1604 case SpvDecorationHlslSemanticGOOGLE: 1605 /* HLSL semantic decorations can safely be ignored by the driver. */ 1606 break; 1607 1608 case SpvDecorationRestrictPointerEXT: 1609 case SpvDecorationAliasedPointerEXT: 1610 /* TODO: We should actually plumb alias information through NIR. */ 1611 break; 1612 1613 default: 1614 vtn_fail_with_decoration("Unhandled decoration", dec->decoration); 1615 } 1616} 1617 1618static void 1619var_is_patch_cb(struct vtn_builder *b, struct vtn_value *val, int member, 1620 const struct vtn_decoration *dec, void *out_is_patch) 1621{ 1622 if (dec->decoration == SpvDecorationPatch) { 1623 *((bool *) out_is_patch) = true; 1624 } 1625} 1626 1627static void 1628var_decoration_cb(struct vtn_builder *b, struct vtn_value *val, int member, 1629 const struct vtn_decoration *dec, void *void_var) 1630{ 1631 struct vtn_variable *vtn_var = void_var; 1632 1633 /* Handle decorations that apply to a vtn_variable as a whole */ 1634 switch (dec->decoration) { 1635 case SpvDecorationBinding: 1636 vtn_var->binding = dec->operands[0]; 1637 vtn_var->explicit_binding = true; 1638 return; 1639 case SpvDecorationDescriptorSet: 1640 vtn_var->descriptor_set = dec->operands[0]; 1641 return; 1642 case SpvDecorationInputAttachmentIndex: 1643 vtn_var->input_attachment_index = dec->operands[0]; 1644 return; 1645 case SpvDecorationPatch: 1646 vtn_var->patch = true; 1647 break; 1648 case SpvDecorationOffset: 1649 vtn_var->offset = dec->operands[0]; 1650 break; 1651 case SpvDecorationNonWritable: 1652 vtn_var->access |= ACCESS_NON_WRITEABLE; 1653 break; 1654 case SpvDecorationNonReadable: 1655 vtn_var->access |= ACCESS_NON_READABLE; 1656 break; 1657 case SpvDecorationVolatile: 1658 vtn_var->access |= ACCESS_VOLATILE; 1659 break; 1660 case SpvDecorationCoherent: 1661 vtn_var->access |= ACCESS_COHERENT; 1662 break; 1663 case SpvDecorationHlslCounterBufferGOOGLE: 1664 /* HLSL semantic decorations can safely be ignored by the driver. */ 1665 break; 1666 default: 1667 break; 1668 } 1669 1670 if (val->value_type == vtn_value_type_pointer) { 1671 assert(val->pointer->var == void_var); 1672 assert(member == -1); 1673 } else { 1674 assert(val->value_type == vtn_value_type_type); 1675 } 1676 1677 /* Location is odd. If applied to a split structure, we have to walk the 1678 * whole thing and accumulate the location. It's easier to handle as a 1679 * special case. 1680 */ 1681 if (dec->decoration == SpvDecorationLocation) { 1682 unsigned location = dec->operands[0]; 1683 if (b->shader->info.stage == MESA_SHADER_FRAGMENT && 1684 vtn_var->mode == vtn_variable_mode_output) { 1685 location += FRAG_RESULT_DATA0; 1686 } else if (b->shader->info.stage == MESA_SHADER_VERTEX && 1687 vtn_var->mode == vtn_variable_mode_input) { 1688 location += VERT_ATTRIB_GENERIC0; 1689 } else if (vtn_var->mode == vtn_variable_mode_input || 1690 vtn_var->mode == vtn_variable_mode_output) { 1691 location += vtn_var->patch ? VARYING_SLOT_PATCH0 : VARYING_SLOT_VAR0; 1692 } else if (vtn_var->mode != vtn_variable_mode_uniform) { 1693 vtn_warn("Location must be on input, output, uniform, sampler or " 1694 "image variable"); 1695 return; 1696 } 1697 1698 if (vtn_var->var->num_members == 0) { 1699 /* This handles the member and lone variable cases */ 1700 vtn_var->var->data.location = location; 1701 } else { 1702 /* This handles the structure member case */ 1703 assert(vtn_var->var->members); 1704 1705 if (member == -1) 1706 vtn_var->base_location = location; 1707 else 1708 vtn_var->var->members[member].location = location; 1709 } 1710 1711 return; 1712 } else { 1713 if (vtn_var->var) { 1714 if (vtn_var->var->num_members == 0) { 1715 /* We call this function on types as well as variables and not all 1716 * struct types get split so we can end up having stray member 1717 * decorations; just ignore them. 1718 */ 1719 if (member == -1) 1720 apply_var_decoration(b, &vtn_var->var->data, dec); 1721 } else if (member >= 0) { 1722 /* Member decorations must come from a type */ 1723 assert(val->value_type == vtn_value_type_type); 1724 apply_var_decoration(b, &vtn_var->var->members[member], dec); 1725 } else { 1726 unsigned length = 1727 glsl_get_length(glsl_without_array(vtn_var->type->type)); 1728 for (unsigned i = 0; i < length; i++) 1729 apply_var_decoration(b, &vtn_var->var->members[i], dec); 1730 } 1731 } else { 1732 /* A few variables, those with external storage, have no actual 1733 * nir_variables associated with them. Fortunately, all decorations 1734 * we care about for those variables are on the type only. 1735 */ 1736 vtn_assert(vtn_var->mode == vtn_variable_mode_ubo || 1737 vtn_var->mode == vtn_variable_mode_ssbo || 1738 vtn_var->mode == vtn_variable_mode_push_constant || 1739 (vtn_var->mode == vtn_variable_mode_workgroup && 1740 b->options->lower_workgroup_access_to_offsets)); 1741 } 1742 } 1743} 1744 1745static void 1746ptr_decoration_cb(struct vtn_builder *b, struct vtn_value *val, int member, 1747 const struct vtn_decoration *dec, void *void_ptr) 1748{ 1749 struct vtn_pointer *ptr = void_ptr; 1750 1751 switch (dec->decoration) { 1752 case SpvDecorationNonUniformEXT: 1753 ptr->access |= ACCESS_NON_UNIFORM; 1754 break; 1755 1756 default: 1757 break; 1758 } 1759} 1760 1761static enum vtn_variable_mode 1762vtn_storage_class_to_mode(struct vtn_builder *b, 1763 SpvStorageClass class, 1764 struct vtn_type *interface_type, 1765 nir_variable_mode *nir_mode_out) 1766{ 1767 enum vtn_variable_mode mode; 1768 nir_variable_mode nir_mode; 1769 switch (class) { 1770 case SpvStorageClassUniform: 1771 if (interface_type->block) { 1772 mode = vtn_variable_mode_ubo; 1773 nir_mode = nir_var_mem_ubo; 1774 } else if (interface_type->buffer_block) { 1775 mode = vtn_variable_mode_ssbo; 1776 nir_mode = nir_var_mem_ssbo; 1777 } else { 1778 /* Default-block uniforms, coming from gl_spirv */ 1779 mode = vtn_variable_mode_uniform; 1780 nir_mode = nir_var_uniform; 1781 } 1782 break; 1783 case SpvStorageClassStorageBuffer: 1784 mode = vtn_variable_mode_ssbo; 1785 nir_mode = nir_var_mem_ssbo; 1786 break; 1787 case SpvStorageClassPhysicalStorageBufferEXT: 1788 mode = vtn_variable_mode_phys_ssbo; 1789 nir_mode = nir_var_mem_global; 1790 break; 1791 case SpvStorageClassUniformConstant: 1792 mode = vtn_variable_mode_uniform; 1793 nir_mode = nir_var_uniform; 1794 break; 1795 case SpvStorageClassPushConstant: 1796 mode = vtn_variable_mode_push_constant; 1797 nir_mode = nir_var_uniform; 1798 break; 1799 case SpvStorageClassInput: 1800 mode = vtn_variable_mode_input; 1801 nir_mode = nir_var_shader_in; 1802 break; 1803 case SpvStorageClassOutput: 1804 mode = vtn_variable_mode_output; 1805 nir_mode = nir_var_shader_out; 1806 break; 1807 case SpvStorageClassPrivate: 1808 mode = vtn_variable_mode_private; 1809 nir_mode = nir_var_shader_temp; 1810 break; 1811 case SpvStorageClassFunction: 1812 mode = vtn_variable_mode_function; 1813 nir_mode = nir_var_function_temp; 1814 break; 1815 case SpvStorageClassWorkgroup: 1816 mode = vtn_variable_mode_workgroup; 1817 nir_mode = nir_var_mem_shared; 1818 break; 1819 case SpvStorageClassAtomicCounter: 1820 mode = vtn_variable_mode_uniform; 1821 nir_mode = nir_var_uniform; 1822 break; 1823 case SpvStorageClassCrossWorkgroup: 1824 mode = vtn_variable_mode_cross_workgroup; 1825 nir_mode = nir_var_mem_global; 1826 break; 1827 case SpvStorageClassGeneric: 1828 default: 1829 vtn_fail("Unhandled variable storage class: %s (%u)", 1830 spirv_storageclass_to_string(class), class); 1831 } 1832 1833 if (nir_mode_out) 1834 *nir_mode_out = nir_mode; 1835 1836 return mode; 1837} 1838 1839nir_ssa_def * 1840vtn_pointer_to_ssa(struct vtn_builder *b, struct vtn_pointer *ptr) 1841{ 1842 if (vtn_pointer_uses_ssa_offset(b, ptr)) { 1843 /* This pointer needs to have a pointer type with actual storage */ 1844 vtn_assert(ptr->ptr_type); 1845 vtn_assert(ptr->ptr_type->type); 1846 1847 if (!ptr->offset) { 1848 /* If we don't have an offset then we must be a pointer to the variable 1849 * itself. 1850 */ 1851 vtn_assert(!ptr->offset && !ptr->block_index); 1852 1853 struct vtn_access_chain chain = { 1854 .length = 0, 1855 }; 1856 ptr = vtn_ssa_offset_pointer_dereference(b, ptr, &chain); 1857 } 1858 1859 vtn_assert(ptr->offset); 1860 if (ptr->block_index) { 1861 vtn_assert(ptr->mode == vtn_variable_mode_ubo || 1862 ptr->mode == vtn_variable_mode_ssbo); 1863 return nir_vec2(&b->nb, ptr->block_index, ptr->offset); 1864 } else { 1865 vtn_assert(ptr->mode == vtn_variable_mode_workgroup); 1866 return ptr->offset; 1867 } 1868 } else { 1869 if (vtn_pointer_is_external_block(b, ptr) && 1870 vtn_type_contains_block(b, ptr->type) && 1871 ptr->mode != vtn_variable_mode_phys_ssbo) { 1872 /* In this case, we're looking for a block index and not an actual 1873 * deref. 1874 * 1875 * For PhysicalStorageBufferEXT pointers, we don't have a block index 1876 * at all because we get the pointer directly from the client. This 1877 * assumes that there will never be a SSBO binding variable using the 1878 * PhysicalStorageBufferEXT storage class. This assumption appears 1879 * to be correct according to the Vulkan spec because the table, 1880 * "Shader Resource and Storage Class Correspondence," the only the 1881 * Uniform storage class with BufferBlock or the StorageBuffer 1882 * storage class with Block can be used. 1883 */ 1884 if (!ptr->block_index) { 1885 /* If we don't have a block_index then we must be a pointer to the 1886 * variable itself. 1887 */ 1888 vtn_assert(!ptr->deref); 1889 1890 struct vtn_access_chain chain = { 1891 .length = 0, 1892 }; 1893 ptr = vtn_nir_deref_pointer_dereference(b, ptr, &chain); 1894 } 1895 1896 return ptr->block_index; 1897 } else { 1898 return &vtn_pointer_to_deref(b, ptr)->dest.ssa; 1899 } 1900 } 1901} 1902 1903struct vtn_pointer * 1904vtn_pointer_from_ssa(struct vtn_builder *b, nir_ssa_def *ssa, 1905 struct vtn_type *ptr_type) 1906{ 1907 vtn_assert(ptr_type->base_type == vtn_base_type_pointer); 1908 1909 struct vtn_type *interface_type = ptr_type->deref; 1910 while (interface_type->base_type == vtn_base_type_array) 1911 interface_type = interface_type->array_element; 1912 1913 struct vtn_pointer *ptr = rzalloc(b, struct vtn_pointer); 1914 nir_variable_mode nir_mode; 1915 ptr->mode = vtn_storage_class_to_mode(b, ptr_type->storage_class, 1916 interface_type, &nir_mode); 1917 ptr->type = ptr_type->deref; 1918 ptr->ptr_type = ptr_type; 1919 1920 if (b->wa_glslang_179) { 1921 /* To work around https://github.com/KhronosGroup/glslang/issues/179 we 1922 * need to whack the mode because it creates a function parameter with 1923 * the Function storage class even though it's a pointer to a sampler. 1924 * If we don't do this, then NIR won't get rid of the deref_cast for us. 1925 */ 1926 if (ptr->mode == vtn_variable_mode_function && 1927 (ptr->type->base_type == vtn_base_type_sampler || 1928 ptr->type->base_type == vtn_base_type_sampled_image)) { 1929 ptr->mode = vtn_variable_mode_uniform; 1930 nir_mode = nir_var_uniform; 1931 } 1932 } 1933 1934 if (vtn_pointer_uses_ssa_offset(b, ptr)) { 1935 /* This pointer type needs to have actual storage */ 1936 vtn_assert(ptr_type->type); 1937 if (ptr->mode == vtn_variable_mode_ubo || 1938 ptr->mode == vtn_variable_mode_ssbo) { 1939 vtn_assert(ssa->num_components == 2); 1940 ptr->block_index = nir_channel(&b->nb, ssa, 0); 1941 ptr->offset = nir_channel(&b->nb, ssa, 1); 1942 } else { 1943 vtn_assert(ssa->num_components == 1); 1944 ptr->block_index = NULL; 1945 ptr->offset = ssa; 1946 } 1947 } else { 1948 const struct glsl_type *deref_type = ptr_type->deref->type; 1949 if (!vtn_pointer_is_external_block(b, ptr)) { 1950 ptr->deref = nir_build_deref_cast(&b->nb, ssa, nir_mode, 1951 deref_type, 0); 1952 } else if (vtn_type_contains_block(b, ptr->type) && 1953 ptr->mode != vtn_variable_mode_phys_ssbo) { 1954 /* This is a pointer to somewhere in an array of blocks, not a 1955 * pointer to somewhere inside the block. Set the block index 1956 * instead of making a cast. 1957 */ 1958 ptr->block_index = ssa; 1959 } else { 1960 /* This is a pointer to something internal or a pointer inside a 1961 * block. It's just a regular cast. 1962 * 1963 * For PhysicalStorageBufferEXT pointers, we don't have a block index 1964 * at all because we get the pointer directly from the client. This 1965 * assumes that there will never be a SSBO binding variable using the 1966 * PhysicalStorageBufferEXT storage class. This assumption appears 1967 * to be correct according to the Vulkan spec because the table, 1968 * "Shader Resource and Storage Class Correspondence," the only the 1969 * Uniform storage class with BufferBlock or the StorageBuffer 1970 * storage class with Block can be used. 1971 */ 1972 ptr->deref = nir_build_deref_cast(&b->nb, ssa, nir_mode, 1973 ptr_type->deref->type, 1974 ptr_type->stride); 1975 ptr->deref->dest.ssa.num_components = 1976 glsl_get_vector_elements(ptr_type->type); 1977 ptr->deref->dest.ssa.bit_size = glsl_get_bit_size(ptr_type->type); 1978 } 1979 } 1980 1981 return ptr; 1982} 1983 1984static bool 1985is_per_vertex_inout(const struct vtn_variable *var, gl_shader_stage stage) 1986{ 1987 if (var->patch || !glsl_type_is_array(var->type->type)) 1988 return false; 1989 1990 if (var->mode == vtn_variable_mode_input) { 1991 return stage == MESA_SHADER_TESS_CTRL || 1992 stage == MESA_SHADER_TESS_EVAL || 1993 stage == MESA_SHADER_GEOMETRY; 1994 } 1995 1996 if (var->mode == vtn_variable_mode_output) 1997 return stage == MESA_SHADER_TESS_CTRL; 1998 1999 return false; 2000} 2001 2002static void 2003assign_missing_member_locations(struct vtn_variable *var) 2004{ 2005 unsigned length = 2006 glsl_get_length(glsl_without_array(var->type->type)); 2007 int location = var->base_location; 2008 2009 for (unsigned i = 0; i < length; i++) { 2010 /* From the Vulkan spec: 2011 * 2012 * “If the structure type is a Block but without a Location, then each 2013 * of its members must have a Location decoration.” 2014 * 2015 */ 2016 if (var->type->block) { 2017 assert(var->base_location != -1 || 2018 var->var->members[i].location != -1); 2019 } 2020 2021 /* From the Vulkan spec: 2022 * 2023 * “Any member with its own Location decoration is assigned that 2024 * location. Each remaining member is assigned the location after the 2025 * immediately preceding member in declaration order.” 2026 */ 2027 if (var->var->members[i].location != -1) 2028 location = var->var->members[i].location; 2029 else 2030 var->var->members[i].location = location; 2031 2032 /* Below we use type instead of interface_type, because interface_type 2033 * is only available when it is a Block. This code also supports 2034 * input/outputs that are just structs 2035 */ 2036 const struct glsl_type *member_type = 2037 glsl_get_struct_field(glsl_without_array(var->type->type), i); 2038 2039 location += 2040 glsl_count_attribute_slots(member_type, 2041 false /* is_gl_vertex_input */); 2042 } 2043} 2044 2045 2046static void 2047vtn_create_variable(struct vtn_builder *b, struct vtn_value *val, 2048 struct vtn_type *ptr_type, SpvStorageClass storage_class, 2049 nir_constant *initializer) 2050{ 2051 vtn_assert(ptr_type->base_type == vtn_base_type_pointer); 2052 struct vtn_type *type = ptr_type->deref; 2053 2054 struct vtn_type *without_array = type; 2055 while(glsl_type_is_array(without_array->type)) 2056 without_array = without_array->array_element; 2057 2058 enum vtn_variable_mode mode; 2059 nir_variable_mode nir_mode; 2060 mode = vtn_storage_class_to_mode(b, storage_class, without_array, &nir_mode); 2061 2062 switch (mode) { 2063 case vtn_variable_mode_ubo: 2064 /* There's no other way to get vtn_variable_mode_ubo */ 2065 vtn_assert(without_array->block); 2066 b->shader->info.num_ubos++; 2067 break; 2068 case vtn_variable_mode_ssbo: 2069 if (storage_class == SpvStorageClassStorageBuffer && 2070 !without_array->block) { 2071 if (b->variable_pointers) { 2072 vtn_fail("Variables in the StorageBuffer storage class must " 2073 "have a struct type with the Block decoration"); 2074 } else { 2075 /* If variable pointers are not present, it's still malformed 2076 * SPIR-V but we can parse it and do the right thing anyway. 2077 * Since some of the 8-bit storage tests have bugs in this are, 2078 * just make it a warning for now. 2079 */ 2080 vtn_warn("Variables in the StorageBuffer storage class must " 2081 "have a struct type with the Block decoration"); 2082 } 2083 } 2084 b->shader->info.num_ssbos++; 2085 break; 2086 case vtn_variable_mode_uniform: 2087 if (glsl_type_is_image(without_array->type)) 2088 b->shader->info.num_images++; 2089 else if (glsl_type_is_sampler(without_array->type)) 2090 b->shader->info.num_textures++; 2091 break; 2092 case vtn_variable_mode_push_constant: 2093 b->shader->num_uniforms = vtn_type_block_size(b, type); 2094 break; 2095 2096 case vtn_variable_mode_phys_ssbo: 2097 vtn_fail("Cannot create a variable with the " 2098 "PhysicalStorageBufferEXT storage class"); 2099 break; 2100 2101 default: 2102 /* No tallying is needed */ 2103 break; 2104 } 2105 2106 struct vtn_variable *var = rzalloc(b, struct vtn_variable); 2107 var->type = type; 2108 var->mode = mode; 2109 var->base_location = -1; 2110 2111 vtn_assert(val->value_type == vtn_value_type_pointer); 2112 val->pointer = vtn_pointer_for_variable(b, var, ptr_type); 2113 2114 switch (var->mode) { 2115 case vtn_variable_mode_function: 2116 case vtn_variable_mode_private: 2117 case vtn_variable_mode_uniform: 2118 /* For these, we create the variable normally */ 2119 var->var = rzalloc(b->shader, nir_variable); 2120 var->var->name = ralloc_strdup(var->var, val->name); 2121 2122 if (storage_class == SpvStorageClassAtomicCounter) { 2123 /* Need to tweak the nir type here as at vtn_handle_type we don't 2124 * have the access to storage_class, that is the one that points us 2125 * that is an atomic uint. 2126 */ 2127 var->var->type = repair_atomic_type(var->type->type); 2128 } else { 2129 /* Private variables don't have any explicit layout but some layouts 2130 * may have leaked through due to type deduplication in the SPIR-V. 2131 */ 2132 var->var->type = var->type->type; 2133 } 2134 var->var->data.mode = nir_mode; 2135 var->var->data.location = -1; 2136 var->var->interface_type = NULL; 2137 break; 2138 2139 case vtn_variable_mode_workgroup: 2140 if (b->options->lower_workgroup_access_to_offsets) { 2141 var->shared_location = -1; 2142 } else { 2143 /* Create the variable normally */ 2144 var->var = rzalloc(b->shader, nir_variable); 2145 var->var->name = ralloc_strdup(var->var, val->name); 2146 /* Workgroup variables don't have any explicit layout but some 2147 * layouts may have leaked through due to type deduplication in the 2148 * SPIR-V. 2149 */ 2150 var->var->type = var->type->type; 2151 var->var->data.mode = nir_var_mem_shared; 2152 } 2153 break; 2154 2155 case vtn_variable_mode_input: 2156 case vtn_variable_mode_output: { 2157 /* In order to know whether or not we're a per-vertex inout, we need 2158 * the patch qualifier. This means walking the variable decorations 2159 * early before we actually create any variables. Not a big deal. 2160 * 2161 * GLSLang really likes to place decorations in the most interior 2162 * thing it possibly can. In particular, if you have a struct, it 2163 * will place the patch decorations on the struct members. This 2164 * should be handled by the variable splitting below just fine. 2165 * 2166 * If you have an array-of-struct, things get even more weird as it 2167 * will place the patch decorations on the struct even though it's 2168 * inside an array and some of the members being patch and others not 2169 * makes no sense whatsoever. Since the only sensible thing is for 2170 * it to be all or nothing, we'll call it patch if any of the members 2171 * are declared patch. 2172 */ 2173 var->patch = false; 2174 vtn_foreach_decoration(b, val, var_is_patch_cb, &var->patch); 2175 if (glsl_type_is_array(var->type->type) && 2176 glsl_type_is_struct_or_ifc(without_array->type)) { 2177 vtn_foreach_decoration(b, vtn_value(b, without_array->id, 2178 vtn_value_type_type), 2179 var_is_patch_cb, &var->patch); 2180 } 2181 2182 /* For inputs and outputs, we immediately split structures. This 2183 * is for a couple of reasons. For one, builtins may all come in 2184 * a struct and we really want those split out into separate 2185 * variables. For another, interpolation qualifiers can be 2186 * applied to members of the top-level struct ane we need to be 2187 * able to preserve that information. 2188 */ 2189 2190 struct vtn_type *per_vertex_type = var->type; 2191 if (is_per_vertex_inout(var, b->shader->info.stage)) { 2192 /* In Geometry shaders (and some tessellation), inputs come 2193 * in per-vertex arrays. However, some builtins come in 2194 * non-per-vertex, hence the need for the is_array check. In 2195 * any case, there are no non-builtin arrays allowed so this 2196 * check should be sufficient. 2197 */ 2198 per_vertex_type = var->type->array_element; 2199 } 2200 2201 var->var = rzalloc(b->shader, nir_variable); 2202 var->var->name = ralloc_strdup(var->var, val->name); 2203 /* In Vulkan, shader I/O variables don't have any explicit layout but 2204 * some layouts may have leaked through due to type deduplication in 2205 * the SPIR-V. We do, however, keep the layouts in the variable's 2206 * interface_type because we need offsets for XFB arrays of blocks. 2207 */ 2208 var->var->type = var->type->type; 2209 var->var->data.mode = nir_mode; 2210 var->var->data.patch = var->patch; 2211 2212 /* Figure out the interface block type. */ 2213 struct vtn_type *iface_type = per_vertex_type; 2214 if (var->mode == vtn_variable_mode_output && 2215 (b->shader->info.stage == MESA_SHADER_VERTEX || 2216 b->shader->info.stage == MESA_SHADER_TESS_EVAL || 2217 b->shader->info.stage == MESA_SHADER_GEOMETRY)) { 2218 /* For vertex data outputs, we can end up with arrays of blocks for 2219 * transform feedback where each array element corresponds to a 2220 * different XFB output buffer. 2221 */ 2222 while (iface_type->base_type == vtn_base_type_array) 2223 iface_type = iface_type->array_element; 2224 } 2225 if (iface_type->base_type == vtn_base_type_struct && iface_type->block) 2226 var->var->interface_type = iface_type->type; 2227 2228 if (per_vertex_type->base_type == vtn_base_type_struct && 2229 per_vertex_type->block) { 2230 /* It's a struct. Set it up as per-member. */ 2231 var->var->num_members = glsl_get_length(per_vertex_type->type); 2232 var->var->members = rzalloc_array(var->var, struct nir_variable_data, 2233 var->var->num_members); 2234 2235 for (unsigned i = 0; i < var->var->num_members; i++) { 2236 var->var->members[i].mode = nir_mode; 2237 var->var->members[i].patch = var->patch; 2238 var->var->members[i].location = -1; 2239 } 2240 } 2241 2242 /* For inputs and outputs, we need to grab locations and builtin 2243 * information from the per-vertex type. 2244 */ 2245 vtn_foreach_decoration(b, vtn_value(b, per_vertex_type->id, 2246 vtn_value_type_type), 2247 var_decoration_cb, var); 2248 break; 2249 } 2250 2251 case vtn_variable_mode_ubo: 2252 case vtn_variable_mode_ssbo: 2253 case vtn_variable_mode_push_constant: 2254 case vtn_variable_mode_cross_workgroup: 2255 /* These don't need actual variables. */ 2256 break; 2257 2258 case vtn_variable_mode_phys_ssbo: 2259 unreachable("Should have been caught before"); 2260 } 2261 2262 if (initializer) { 2263 var->var->constant_initializer = 2264 nir_constant_clone(initializer, var->var); 2265 } 2266 2267 vtn_foreach_decoration(b, val, var_decoration_cb, var); 2268 vtn_foreach_decoration(b, val, ptr_decoration_cb, val->pointer); 2269 2270 if ((var->mode == vtn_variable_mode_input || 2271 var->mode == vtn_variable_mode_output) && 2272 var->var->members) { 2273 assign_missing_member_locations(var); 2274 } 2275 2276 if (var->mode == vtn_variable_mode_uniform) { 2277 /* XXX: We still need the binding information in the nir_variable 2278 * for these. We should fix that. 2279 */ 2280 var->var->data.binding = var->binding; 2281 var->var->data.explicit_binding = var->explicit_binding; 2282 var->var->data.descriptor_set = var->descriptor_set; 2283 var->var->data.index = var->input_attachment_index; 2284 var->var->data.offset = var->offset; 2285 2286 if (glsl_type_is_image(without_array->type)) 2287 var->var->data.image.format = without_array->image_format; 2288 } 2289 2290 if (var->mode == vtn_variable_mode_function) { 2291 vtn_assert(var->var != NULL && var->var->members == NULL); 2292 nir_function_impl_add_variable(b->nb.impl, var->var); 2293 } else if (var->var) { 2294 nir_shader_add_variable(b->shader, var->var); 2295 } else { 2296 vtn_assert(vtn_pointer_is_external_block(b, val->pointer)); 2297 } 2298} 2299 2300static void 2301vtn_assert_types_equal(struct vtn_builder *b, SpvOp opcode, 2302 struct vtn_type *dst_type, 2303 struct vtn_type *src_type) 2304{ 2305 if (dst_type->id == src_type->id) 2306 return; 2307 2308 if (vtn_types_compatible(b, dst_type, src_type)) { 2309 /* Early versions of GLSLang would re-emit types unnecessarily and you 2310 * would end up with OpLoad, OpStore, or OpCopyMemory opcodes which have 2311 * mismatched source and destination types. 2312 * 2313 * https://github.com/KhronosGroup/glslang/issues/304 2314 * https://github.com/KhronosGroup/glslang/issues/307 2315 * https://bugs.freedesktop.org/show_bug.cgi?id=104338 2316 * https://bugs.freedesktop.org/show_bug.cgi?id=104424 2317 */ 2318 vtn_warn("Source and destination types of %s do not have the same " 2319 "ID (but are compatible): %u vs %u", 2320 spirv_op_to_string(opcode), dst_type->id, src_type->id); 2321 return; 2322 } 2323 2324 vtn_fail("Source and destination types of %s do not match: %s vs. %s", 2325 spirv_op_to_string(opcode), 2326 glsl_get_type_name(dst_type->type), 2327 glsl_get_type_name(src_type->type)); 2328} 2329 2330static nir_ssa_def * 2331nir_shrink_zero_pad_vec(nir_builder *b, nir_ssa_def *val, 2332 unsigned num_components) 2333{ 2334 if (val->num_components == num_components) 2335 return val; 2336 2337 nir_ssa_def *comps[NIR_MAX_VEC_COMPONENTS]; 2338 for (unsigned i = 0; i < num_components; i++) { 2339 if (i < val->num_components) 2340 comps[i] = nir_channel(b, val, i); 2341 else 2342 comps[i] = nir_imm_intN_t(b, 0, val->bit_size); 2343 } 2344 return nir_vec(b, comps, num_components); 2345} 2346 2347static nir_ssa_def * 2348nir_sloppy_bitcast(nir_builder *b, nir_ssa_def *val, 2349 const struct glsl_type *type) 2350{ 2351 const unsigned num_components = glsl_get_vector_elements(type); 2352 const unsigned bit_size = glsl_get_bit_size(type); 2353 2354 /* First, zero-pad to ensure that the value is big enough that when we 2355 * bit-cast it, we don't loose anything. 2356 */ 2357 if (val->bit_size < bit_size) { 2358 const unsigned src_num_components_needed = 2359 vtn_align_u32(val->num_components, bit_size / val->bit_size); 2360 val = nir_shrink_zero_pad_vec(b, val, src_num_components_needed); 2361 } 2362 2363 val = nir_bitcast_vector(b, val, bit_size); 2364 2365 return nir_shrink_zero_pad_vec(b, val, num_components); 2366} 2367 2368void 2369vtn_handle_variables(struct vtn_builder *b, SpvOp opcode, 2370 const uint32_t *w, unsigned count) 2371{ 2372 switch (opcode) { 2373 case SpvOpUndef: { 2374 struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_undef); 2375 val->type = vtn_value(b, w[1], vtn_value_type_type)->type; 2376 break; 2377 } 2378 2379 case SpvOpVariable: { 2380 struct vtn_type *ptr_type = vtn_value(b, w[1], vtn_value_type_type)->type; 2381 2382 struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_pointer); 2383 2384 SpvStorageClass storage_class = w[3]; 2385 nir_constant *initializer = NULL; 2386 if (count > 4) 2387 initializer = vtn_value(b, w[4], vtn_value_type_constant)->constant; 2388 2389 vtn_create_variable(b, val, ptr_type, storage_class, initializer); 2390 break; 2391 } 2392 2393 case SpvOpAccessChain: 2394 case SpvOpPtrAccessChain: 2395 case SpvOpInBoundsAccessChain: 2396 case SpvOpInBoundsPtrAccessChain: { 2397 struct vtn_access_chain *chain = vtn_access_chain_create(b, count - 4); 2398 enum gl_access_qualifier access = 0; 2399 chain->ptr_as_array = (opcode == SpvOpPtrAccessChain || opcode == SpvOpInBoundsPtrAccessChain); 2400 2401 unsigned idx = 0; 2402 for (int i = 4; i < count; i++) { 2403 struct vtn_value *link_val = vtn_untyped_value(b, w[i]); 2404 if (link_val->value_type == vtn_value_type_constant) { 2405 chain->link[idx].mode = vtn_access_mode_literal; 2406 const unsigned bit_size = glsl_get_bit_size(link_val->type->type); 2407 switch (bit_size) { 2408 case 8: 2409 chain->link[idx].id = link_val->constant->values[0][0].i8; 2410 break; 2411 case 16: 2412 chain->link[idx].id = link_val->constant->values[0][0].i16; 2413 break; 2414 case 32: 2415 chain->link[idx].id = link_val->constant->values[0][0].i32; 2416 break; 2417 case 64: 2418 chain->link[idx].id = link_val->constant->values[0][0].i64; 2419 break; 2420 default: 2421 vtn_fail("Invalid bit size: %u", bit_size); 2422 } 2423 } else { 2424 chain->link[idx].mode = vtn_access_mode_id; 2425 chain->link[idx].id = w[i]; 2426 } 2427 access |= vtn_value_access(link_val); 2428 idx++; 2429 } 2430 2431 struct vtn_type *ptr_type = vtn_value(b, w[1], vtn_value_type_type)->type; 2432 struct vtn_value *base_val = vtn_untyped_value(b, w[3]); 2433 if (base_val->value_type == vtn_value_type_sampled_image) { 2434 /* This is rather insane. SPIR-V allows you to use OpSampledImage 2435 * to combine an array of images with a single sampler to get an 2436 * array of sampled images that all share the same sampler. 2437 * Fortunately, this means that we can more-or-less ignore the 2438 * sampler when crawling the access chain, but it does leave us 2439 * with this rather awkward little special-case. 2440 */ 2441 struct vtn_value *val = 2442 vtn_push_value(b, w[2], vtn_value_type_sampled_image); 2443 val->sampled_image = ralloc(b, struct vtn_sampled_image); 2444 val->sampled_image->type = base_val->sampled_image->type; 2445 val->sampled_image->image = 2446 vtn_pointer_dereference(b, base_val->sampled_image->image, chain); 2447 val->sampled_image->sampler = base_val->sampled_image->sampler; 2448 vtn_foreach_decoration(b, val, ptr_decoration_cb, 2449 val->sampled_image->image); 2450 vtn_foreach_decoration(b, val, ptr_decoration_cb, 2451 val->sampled_image->sampler); 2452 } else { 2453 vtn_assert(base_val->value_type == vtn_value_type_pointer); 2454 struct vtn_pointer *ptr = 2455 vtn_pointer_dereference(b, base_val->pointer, chain); 2456 ptr->ptr_type = ptr_type; 2457 ptr->access |= access; 2458 vtn_push_value_pointer(b, w[2], ptr); 2459 } 2460 break; 2461 } 2462 2463 case SpvOpCopyMemory: { 2464 struct vtn_value *dest = vtn_value(b, w[1], vtn_value_type_pointer); 2465 struct vtn_value *src = vtn_value(b, w[2], vtn_value_type_pointer); 2466 2467 vtn_assert_types_equal(b, opcode, dest->type->deref, src->type->deref); 2468 2469 vtn_variable_copy(b, dest->pointer, src->pointer); 2470 break; 2471 } 2472 2473 case SpvOpLoad: { 2474 struct vtn_type *res_type = 2475 vtn_value(b, w[1], vtn_value_type_type)->type; 2476 struct vtn_value *src_val = vtn_value(b, w[3], vtn_value_type_pointer); 2477 struct vtn_pointer *src = src_val->pointer; 2478 2479 vtn_assert_types_equal(b, opcode, res_type, src_val->type->deref); 2480 2481 if (glsl_type_is_image(res_type->type) || 2482 glsl_type_is_sampler(res_type->type)) { 2483 vtn_push_value_pointer(b, w[2], src); 2484 return; 2485 } 2486 2487 vtn_push_ssa(b, w[2], res_type, vtn_variable_load(b, src)); 2488 break; 2489 } 2490 2491 case SpvOpStore: { 2492 struct vtn_value *dest_val = vtn_value(b, w[1], vtn_value_type_pointer); 2493 struct vtn_pointer *dest = dest_val->pointer; 2494 struct vtn_value *src_val = vtn_untyped_value(b, w[2]); 2495 2496 /* OpStore requires us to actually have a storage type */ 2497 vtn_fail_if(dest->type->type == NULL, 2498 "Invalid destination type for OpStore"); 2499 2500 if (glsl_get_base_type(dest->type->type) == GLSL_TYPE_BOOL && 2501 glsl_get_base_type(src_val->type->type) == GLSL_TYPE_UINT) { 2502 /* Early versions of GLSLang would use uint types for UBOs/SSBOs but 2503 * would then store them to a local variable as bool. Work around 2504 * the issue by doing an implicit conversion. 2505 * 2506 * https://github.com/KhronosGroup/glslang/issues/170 2507 * https://bugs.freedesktop.org/show_bug.cgi?id=104424 2508 */ 2509 vtn_warn("OpStore of value of type OpTypeInt to a pointer to type " 2510 "OpTypeBool. Doing an implicit conversion to work around " 2511 "the problem."); 2512 struct vtn_ssa_value *bool_ssa = 2513 vtn_create_ssa_value(b, dest->type->type); 2514 bool_ssa->def = nir_i2b(&b->nb, vtn_ssa_value(b, w[2])->def); 2515 vtn_variable_store(b, bool_ssa, dest); 2516 break; 2517 } 2518 2519 vtn_assert_types_equal(b, opcode, dest_val->type->deref, src_val->type); 2520 2521 if (glsl_type_is_sampler(dest->type->type)) { 2522 if (b->wa_glslang_179) { 2523 vtn_warn("OpStore of a sampler detected. Doing on-the-fly copy " 2524 "propagation to workaround the problem."); 2525 vtn_assert(dest->var->copy_prop_sampler == NULL); 2526 dest->var->copy_prop_sampler = 2527 vtn_value(b, w[2], vtn_value_type_pointer)->pointer; 2528 } else { 2529 vtn_fail("Vulkan does not allow OpStore of a sampler or image."); 2530 } 2531 break; 2532 } 2533 2534 struct vtn_ssa_value *src = vtn_ssa_value(b, w[2]); 2535 vtn_variable_store(b, src, dest); 2536 break; 2537 } 2538 2539 case SpvOpArrayLength: { 2540 struct vtn_pointer *ptr = 2541 vtn_value(b, w[3], vtn_value_type_pointer)->pointer; 2542 const uint32_t field = w[4]; 2543 2544 vtn_fail_if(ptr->type->base_type != vtn_base_type_struct, 2545 "OpArrayLength must take a pointer to a structure type"); 2546 vtn_fail_if(field != ptr->type->length - 1 || 2547 ptr->type->members[field]->base_type != vtn_base_type_array, 2548 "OpArrayLength must reference the last memeber of the " 2549 "structure and that must be an array"); 2550 2551 const uint32_t offset = ptr->type->offsets[field]; 2552 const uint32_t stride = ptr->type->members[field]->stride; 2553 2554 if (!ptr->block_index) { 2555 struct vtn_access_chain chain = { 2556 .length = 0, 2557 }; 2558 ptr = vtn_pointer_dereference(b, ptr, &chain); 2559 vtn_assert(ptr->block_index); 2560 } 2561 2562 nir_intrinsic_instr *instr = 2563 nir_intrinsic_instr_create(b->nb.shader, 2564 nir_intrinsic_get_buffer_size); 2565 instr->src[0] = nir_src_for_ssa(ptr->block_index); 2566 nir_ssa_dest_init(&instr->instr, &instr->dest, 1, 32, NULL); 2567 nir_builder_instr_insert(&b->nb, &instr->instr); 2568 nir_ssa_def *buf_size = &instr->dest.ssa; 2569 2570 /* array_length = max(buffer_size - offset, 0) / stride */ 2571 nir_ssa_def *array_length = 2572 nir_idiv(&b->nb, 2573 nir_imax(&b->nb, 2574 nir_isub(&b->nb, 2575 buf_size, 2576 nir_imm_int(&b->nb, offset)), 2577 nir_imm_int(&b->nb, 0u)), 2578 nir_imm_int(&b->nb, stride)); 2579 2580 struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_ssa); 2581 val->ssa = vtn_create_ssa_value(b, glsl_uint_type()); 2582 val->ssa->def = array_length; 2583 break; 2584 } 2585 2586 case SpvOpConvertPtrToU: { 2587 struct vtn_value *u_val = vtn_push_value(b, w[2], vtn_value_type_ssa); 2588 2589 vtn_fail_if(u_val->type->base_type != vtn_base_type_vector && 2590 u_val->type->base_type != vtn_base_type_scalar, 2591 "OpConvertPtrToU can only be used to cast to a vector or " 2592 "scalar type"); 2593 2594 /* The pointer will be converted to an SSA value automatically */ 2595 struct vtn_ssa_value *ptr_ssa = vtn_ssa_value(b, w[3]); 2596 2597 u_val->ssa = vtn_create_ssa_value(b, u_val->type->type); 2598 u_val->ssa->def = nir_sloppy_bitcast(&b->nb, ptr_ssa->def, u_val->type->type); 2599 u_val->ssa->access |= ptr_ssa->access; 2600 break; 2601 } 2602 2603 case SpvOpConvertUToPtr: { 2604 struct vtn_value *ptr_val = 2605 vtn_push_value(b, w[2], vtn_value_type_pointer); 2606 struct vtn_value *u_val = vtn_value(b, w[3], vtn_value_type_ssa); 2607 2608 vtn_fail_if(ptr_val->type->type == NULL, 2609 "OpConvertUToPtr can only be used on physical pointers"); 2610 2611 vtn_fail_if(u_val->type->base_type != vtn_base_type_vector && 2612 u_val->type->base_type != vtn_base_type_scalar, 2613 "OpConvertUToPtr can only be used to cast from a vector or " 2614 "scalar type"); 2615 2616 nir_ssa_def *ptr_ssa = nir_sloppy_bitcast(&b->nb, u_val->ssa->def, 2617 ptr_val->type->type); 2618 ptr_val->pointer = vtn_pointer_from_ssa(b, ptr_ssa, ptr_val->type); 2619 vtn_foreach_decoration(b, ptr_val, ptr_decoration_cb, ptr_val->pointer); 2620 ptr_val->pointer->access |= u_val->ssa->access; 2621 break; 2622 } 2623 2624 case SpvOpCopyMemorySized: 2625 default: 2626 vtn_fail_with_opcode("Unhandled opcode", opcode); 2627 } 2628} 2629