1 /* 2 * Copyright 2015 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Jason Ekstrand (jason (at) jlekstrand.net) 25 * 26 */ 27 28 #include "vtn_private.h" 29 #include "spirv_info.h" 30 #include "nir_deref.h" 31 #include <vulkan/vulkan_core.h> 32 33 static struct vtn_pointer* 34 vtn_align_pointer(struct vtn_builder *b, struct vtn_pointer *ptr, 35 unsigned alignment) 36 { 37 if (alignment == 0) 38 return ptr; 39 40 if (!util_is_power_of_two_nonzero(alignment)) { 41 vtn_warn("Provided alignment is not a power of two"); 42 alignment = 1 << (ffs(alignment) - 1); 43 } 44 45 /* If this pointer doesn't have a deref, bail. This either means we're 46 * using the old offset+alignment pointers which don't support carrying 47 * alignment information or we're a pointer that is below the block 48 * boundary in our access chain in which case alignment is meaningless. 49 */ 50 if (ptr->deref == NULL) 51 return ptr; 52 53 /* Ignore alignment information on logical pointers. This way, we don't 54 * trip up drivers with unnecessary casts. 55 */ 56 nir_address_format addr_format = vtn_mode_to_address_format(b, ptr->mode); 57 if (addr_format == nir_address_format_logical) 58 return ptr; 59 60 struct vtn_pointer *copy = ralloc(b, struct vtn_pointer); 61 *copy = *ptr; 62 copy->deref = nir_alignment_deref_cast(&b->nb, ptr->deref, alignment, 0); 63 64 return copy; 65 } 66 67 static void 68 ptr_decoration_cb(struct vtn_builder *b, struct vtn_value *val, int member, 69 const struct vtn_decoration *dec, void *void_ptr) 70 { 71 struct vtn_pointer *ptr = void_ptr; 72 73 switch (dec->decoration) { 74 case SpvDecorationNonUniformEXT: 75 ptr->access |= ACCESS_NON_UNIFORM; 76 break; 77 78 default: 79 break; 80 } 81 } 82 83 struct access_align { 84 enum gl_access_qualifier access; 85 uint32_t alignment; 86 }; 87 88 static void 89 access_align_cb(struct vtn_builder *b, struct vtn_value *val, int member, 90 const struct vtn_decoration *dec, void *void_ptr) 91 { 92 struct access_align *aa = void_ptr; 93 94 switch (dec->decoration) { 95 case SpvDecorationAlignment: 96 aa->alignment = dec->operands[0]; 97 break; 98 99 case SpvDecorationNonUniformEXT: 100 aa->access |= ACCESS_NON_UNIFORM; 101 break; 102 103 default: 104 break; 105 } 106 } 107 108 static struct vtn_pointer* 109 vtn_decorate_pointer(struct vtn_builder *b, struct vtn_value *val, 110 struct vtn_pointer *ptr) 111 { 112 struct access_align aa = { 0, }; 113 vtn_foreach_decoration(b, val, access_align_cb, &aa); 114 115 ptr = vtn_align_pointer(b, ptr, aa.alignment); 116 117 /* If we're adding access flags, make a copy of the pointer. We could 118 * probably just OR them in without doing so but this prevents us from 119 * leaking them any further than actually specified in the SPIR-V. 120 */ 121 if (aa.access & ~ptr->access) { 122 struct vtn_pointer *copy = ralloc(b, struct vtn_pointer); 123 *copy = *ptr; 124 copy->access |= aa.access; 125 return copy; 126 } 127 128 return ptr; 129 } 130 131 struct vtn_value * 132 vtn_push_pointer(struct vtn_builder *b, uint32_t value_id, 133 struct vtn_pointer *ptr) 134 { 135 struct vtn_value *val = vtn_push_value(b, value_id, vtn_value_type_pointer); 136 val->pointer = vtn_decorate_pointer(b, val, ptr); 137 return val; 138 } 139 140 void 141 vtn_copy_value(struct vtn_builder *b, uint32_t src_value_id, 142 uint32_t dst_value_id) 143 { 144 struct vtn_value *src = vtn_untyped_value(b, src_value_id); 145 struct vtn_value *dst = vtn_untyped_value(b, dst_value_id); 146 struct vtn_value src_copy = *src; 147 148 vtn_fail_if(dst->value_type != vtn_value_type_invalid, 149 "SPIR-V id %u has already been written by another instruction", 150 dst_value_id); 151 152 vtn_fail_if(dst->type->id != src->type->id, 153 "Result Type must equal Operand type"); 154 155 src_copy.name = dst->name; 156 src_copy.decoration = dst->decoration; 157 src_copy.type = dst->type; 158 *dst = src_copy; 159 160 if (dst->value_type == vtn_value_type_pointer) 161 dst->pointer = vtn_decorate_pointer(b, dst, dst->pointer); 162 } 163 164 static struct vtn_access_chain * 165 vtn_access_chain_create(struct vtn_builder *b, unsigned length) 166 { 167 struct vtn_access_chain *chain; 168 169 /* Subtract 1 from the length since there's already one built in */ 170 size_t size = sizeof(*chain) + 171 (MAX2(length, 1) - 1) * sizeof(chain->link[0]); 172 chain = rzalloc_size(b, size); 173 chain->length = length; 174 175 return chain; 176 } 177 178 static bool 179 vtn_mode_is_cross_invocation(struct vtn_builder *b, 180 enum vtn_variable_mode mode) 181 { 182 return mode == vtn_variable_mode_ssbo || 183 mode == vtn_variable_mode_ubo || 184 mode == vtn_variable_mode_phys_ssbo || 185 mode == vtn_variable_mode_push_constant || 186 mode == vtn_variable_mode_workgroup || 187 mode == vtn_variable_mode_cross_workgroup; 188 } 189 190 static bool 191 vtn_pointer_is_external_block(struct vtn_builder *b, 192 struct vtn_pointer *ptr) 193 { 194 return ptr->mode == vtn_variable_mode_ssbo || 195 ptr->mode == vtn_variable_mode_ubo || 196 ptr->mode == vtn_variable_mode_phys_ssbo; 197 } 198 199 static nir_ssa_def * 200 vtn_access_link_as_ssa(struct vtn_builder *b, struct vtn_access_link link, 201 unsigned stride, unsigned bit_size) 202 { 203 vtn_assert(stride > 0); 204 if (link.mode == vtn_access_mode_literal) { 205 return nir_imm_intN_t(&b->nb, link.id * stride, bit_size); 206 } else { 207 nir_ssa_def *ssa = vtn_ssa_value(b, link.id)->def; 208 if (ssa->bit_size != bit_size) 209 ssa = nir_i2i(&b->nb, ssa, bit_size); 210 return nir_imul_imm(&b->nb, ssa, stride); 211 } 212 } 213 214 static VkDescriptorType 215 vk_desc_type_for_mode(struct vtn_builder *b, enum vtn_variable_mode mode) 216 { 217 switch (mode) { 218 case vtn_variable_mode_ubo: 219 return VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER; 220 case vtn_variable_mode_ssbo: 221 return VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; 222 case vtn_variable_mode_accel_struct: 223 return VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR; 224 default: 225 vtn_fail("Invalid mode for vulkan_resource_index"); 226 } 227 } 228 229 static nir_ssa_def * 230 vtn_variable_resource_index(struct vtn_builder *b, struct vtn_variable *var, 231 nir_ssa_def *desc_array_index) 232 { 233 vtn_assert(b->options->environment == NIR_SPIRV_VULKAN); 234 235 if (!desc_array_index) 236 desc_array_index = nir_imm_int(&b->nb, 0); 237 238 if (b->vars_used_indirectly) { 239 vtn_assert(var->var); 240 _mesa_set_add(b->vars_used_indirectly, var->var); 241 } 242 243 nir_intrinsic_instr *instr = 244 nir_intrinsic_instr_create(b->nb.shader, 245 nir_intrinsic_vulkan_resource_index); 246 instr->src[0] = nir_src_for_ssa(desc_array_index); 247 nir_intrinsic_set_desc_set(instr, var->descriptor_set); 248 nir_intrinsic_set_binding(instr, var->binding); 249 nir_intrinsic_set_desc_type(instr, vk_desc_type_for_mode(b, var->mode)); 250 251 nir_address_format addr_format = vtn_mode_to_address_format(b, var->mode); 252 nir_ssa_dest_init(&instr->instr, &instr->dest, 253 nir_address_format_num_components(addr_format), 254 nir_address_format_bit_size(addr_format), NULL); 255 instr->num_components = instr->dest.ssa.num_components; 256 nir_builder_instr_insert(&b->nb, &instr->instr); 257 258 return &instr->dest.ssa; 259 } 260 261 static nir_ssa_def * 262 vtn_resource_reindex(struct vtn_builder *b, enum vtn_variable_mode mode, 263 nir_ssa_def *base_index, nir_ssa_def *offset_index) 264 { 265 vtn_assert(b->options->environment == NIR_SPIRV_VULKAN); 266 267 nir_intrinsic_instr *instr = 268 nir_intrinsic_instr_create(b->nb.shader, 269 nir_intrinsic_vulkan_resource_reindex); 270 instr->src[0] = nir_src_for_ssa(base_index); 271 instr->src[1] = nir_src_for_ssa(offset_index); 272 nir_intrinsic_set_desc_type(instr, vk_desc_type_for_mode(b, mode)); 273 274 nir_address_format addr_format = vtn_mode_to_address_format(b, mode); 275 nir_ssa_dest_init(&instr->instr, &instr->dest, 276 nir_address_format_num_components(addr_format), 277 nir_address_format_bit_size(addr_format), NULL); 278 instr->num_components = instr->dest.ssa.num_components; 279 nir_builder_instr_insert(&b->nb, &instr->instr); 280 281 return &instr->dest.ssa; 282 } 283 284 static nir_ssa_def * 285 vtn_descriptor_load(struct vtn_builder *b, enum vtn_variable_mode mode, 286 nir_ssa_def *desc_index) 287 { 288 vtn_assert(b->options->environment == NIR_SPIRV_VULKAN); 289 290 nir_intrinsic_instr *desc_load = 291 nir_intrinsic_instr_create(b->nb.shader, 292 nir_intrinsic_load_vulkan_descriptor); 293 desc_load->src[0] = nir_src_for_ssa(desc_index); 294 nir_intrinsic_set_desc_type(desc_load, vk_desc_type_for_mode(b, mode)); 295 296 nir_address_format addr_format = vtn_mode_to_address_format(b, mode); 297 nir_ssa_dest_init(&desc_load->instr, &desc_load->dest, 298 nir_address_format_num_components(addr_format), 299 nir_address_format_bit_size(addr_format), NULL); 300 desc_load->num_components = desc_load->dest.ssa.num_components; 301 nir_builder_instr_insert(&b->nb, &desc_load->instr); 302 303 return &desc_load->dest.ssa; 304 } 305 306 static struct vtn_pointer * 307 vtn_pointer_dereference(struct vtn_builder *b, 308 struct vtn_pointer *base, 309 struct vtn_access_chain *deref_chain) 310 { 311 struct vtn_type *type = base->type; 312 enum gl_access_qualifier access = base->access | deref_chain->access; 313 unsigned idx = 0; 314 315 nir_deref_instr *tail; 316 if (base->deref) { 317 tail = base->deref; 318 } else if (b->options->environment == NIR_SPIRV_VULKAN && 319 (vtn_pointer_is_external_block(b, base) || 320 base->mode == vtn_variable_mode_accel_struct)) { 321 nir_ssa_def *block_index = base->block_index; 322 323 /* We dereferencing an external block pointer. Correctness of this 324 * operation relies on one particular line in the SPIR-V spec, section 325 * entitled "Validation Rules for Shader Capabilities": 326 * 327 * "Block and BufferBlock decorations cannot decorate a structure 328 * type that is nested at any level inside another structure type 329 * decorated with Block or BufferBlock." 330 * 331 * This means that we can detect the point where we cross over from 332 * descriptor indexing to buffer indexing by looking for the block 333 * decorated struct type. Anything before the block decorated struct 334 * type is a descriptor indexing operation and anything after the block 335 * decorated struct is a buffer offset operation. 336 */ 337 338 /* Figure out the descriptor array index if any 339 * 340 * Some of the Vulkan CTS tests with hand-rolled SPIR-V have been known 341 * to forget the Block or BufferBlock decoration from time to time. 342 * It's more robust if we check for both !block_index and for the type 343 * to contain a block. This way there's a decent chance that arrays of 344 * UBOs/SSBOs will work correctly even if variable pointers are 345 * completley toast. 346 */ 347 nir_ssa_def *desc_arr_idx = NULL; 348 if (!block_index || vtn_type_contains_block(b, type) || 349 base->mode == vtn_variable_mode_accel_struct) { 350 /* If our type contains a block, then we're still outside the block 351 * and we need to process enough levels of dereferences to get inside 352 * of it. Same applies to acceleration structures. 353 */ 354 if (deref_chain->ptr_as_array) { 355 unsigned aoa_size = glsl_get_aoa_size(type->type); 356 desc_arr_idx = vtn_access_link_as_ssa(b, deref_chain->link[idx], 357 MAX2(aoa_size, 1), 32); 358 idx++; 359 } 360 361 for (; idx < deref_chain->length; idx++) { 362 if (type->base_type != vtn_base_type_array) { 363 vtn_assert(type->base_type == vtn_base_type_struct); 364 break; 365 } 366 367 unsigned aoa_size = glsl_get_aoa_size(type->array_element->type); 368 nir_ssa_def *arr_offset = 369 vtn_access_link_as_ssa(b, deref_chain->link[idx], 370 MAX2(aoa_size, 1), 32); 371 if (desc_arr_idx) 372 desc_arr_idx = nir_iadd(&b->nb, desc_arr_idx, arr_offset); 373 else 374 desc_arr_idx = arr_offset; 375 376 type = type->array_element; 377 access |= type->access; 378 } 379 } 380 381 if (!block_index) { 382 vtn_assert(base->var && base->type); 383 block_index = vtn_variable_resource_index(b, base->var, desc_arr_idx); 384 } else if (desc_arr_idx) { 385 block_index = vtn_resource_reindex(b, base->mode, 386 block_index, desc_arr_idx); 387 } 388 389 if (idx == deref_chain->length) { 390 /* The entire deref was consumed in finding the block index. Return 391 * a pointer which just has a block index and a later access chain 392 * will dereference deeper. 393 */ 394 struct vtn_pointer *ptr = rzalloc(b, struct vtn_pointer); 395 ptr->mode = base->mode; 396 ptr->type = type; 397 ptr->block_index = block_index; 398 ptr->access = access; 399 return ptr; 400 } 401 402 /* If we got here, there's more access chain to handle and we have the 403 * final block index. Insert a descriptor load and cast to a deref to 404 * start the deref chain. 405 */ 406 nir_ssa_def *desc = vtn_descriptor_load(b, base->mode, block_index); 407 408 assert(base->mode == vtn_variable_mode_ssbo || 409 base->mode == vtn_variable_mode_ubo); 410 nir_variable_mode nir_mode = 411 base->mode == vtn_variable_mode_ssbo ? nir_var_mem_ssbo : nir_var_mem_ubo; 412 413 tail = nir_build_deref_cast(&b->nb, desc, nir_mode, 414 vtn_type_get_nir_type(b, type, base->mode), 415 base->ptr_type->stride); 416 } else if (base->mode == vtn_variable_mode_shader_record) { 417 /* For ShaderRecordBufferKHR variables, we don't have a nir_variable. 418 * It's just a fancy handle around a pointer to the shader record for 419 * the current shader. 420 */ 421 tail = nir_build_deref_cast(&b->nb, nir_load_shader_record_ptr(&b->nb), 422 nir_var_mem_constant, 423 vtn_type_get_nir_type(b, base->type, 424 base->mode), 425 0 /* ptr_as_array stride */); 426 } else { 427 assert(base->var && base->var->var); 428 tail = nir_build_deref_var(&b->nb, base->var->var); 429 if (base->ptr_type && base->ptr_type->type) { 430 tail->dest.ssa.num_components = 431 glsl_get_vector_elements(base->ptr_type->type); 432 tail->dest.ssa.bit_size = glsl_get_bit_size(base->ptr_type->type); 433 } 434 } 435 436 if (idx == 0 && deref_chain->ptr_as_array) { 437 /* We start with a deref cast to get the stride. Hopefully, we'll be 438 * able to delete that cast eventually. 439 */ 440 tail = nir_build_deref_cast(&b->nb, &tail->dest.ssa, tail->modes, 441 tail->type, base->ptr_type->stride); 442 443 nir_ssa_def *index = vtn_access_link_as_ssa(b, deref_chain->link[0], 1, 444 tail->dest.ssa.bit_size); 445 tail = nir_build_deref_ptr_as_array(&b->nb, tail, index); 446 idx++; 447 } 448 449 for (; idx < deref_chain->length; idx++) { 450 if (glsl_type_is_struct_or_ifc(type->type)) { 451 vtn_assert(deref_chain->link[idx].mode == vtn_access_mode_literal); 452 unsigned field = deref_chain->link[idx].id; 453 tail = nir_build_deref_struct(&b->nb, tail, field); 454 type = type->members[field]; 455 } else { 456 nir_ssa_def *arr_index = 457 vtn_access_link_as_ssa(b, deref_chain->link[idx], 1, 458 tail->dest.ssa.bit_size); 459 tail = nir_build_deref_array(&b->nb, tail, arr_index); 460 type = type->array_element; 461 } 462 463 access |= type->access; 464 } 465 466 struct vtn_pointer *ptr = rzalloc(b, struct vtn_pointer); 467 ptr->mode = base->mode; 468 ptr->type = type; 469 ptr->var = base->var; 470 ptr->deref = tail; 471 ptr->access = access; 472 473 return ptr; 474 } 475 476 nir_deref_instr * 477 vtn_pointer_to_deref(struct vtn_builder *b, struct vtn_pointer *ptr) 478 { 479 if (!ptr->deref) { 480 struct vtn_access_chain chain = { 481 .length = 0, 482 }; 483 ptr = vtn_pointer_dereference(b, ptr, &chain); 484 } 485 486 return ptr->deref; 487 } 488 489 static void 490 _vtn_local_load_store(struct vtn_builder *b, bool load, nir_deref_instr *deref, 491 struct vtn_ssa_value *inout, 492 enum gl_access_qualifier access) 493 { 494 if (glsl_type_is_vector_or_scalar(deref->type)) { 495 if (load) { 496 inout->def = nir_load_deref_with_access(&b->nb, deref, access); 497 } else { 498 nir_store_deref_with_access(&b->nb, deref, inout->def, ~0, access); 499 } 500 } else if (glsl_type_is_array(deref->type) || 501 glsl_type_is_matrix(deref->type)) { 502 unsigned elems = glsl_get_length(deref->type); 503 for (unsigned i = 0; i < elems; i++) { 504 nir_deref_instr *child = 505 nir_build_deref_array_imm(&b->nb, deref, i); 506 _vtn_local_load_store(b, load, child, inout->elems[i], access); 507 } 508 } else { 509 vtn_assert(glsl_type_is_struct_or_ifc(deref->type)); 510 unsigned elems = glsl_get_length(deref->type); 511 for (unsigned i = 0; i < elems; i++) { 512 nir_deref_instr *child = nir_build_deref_struct(&b->nb, deref, i); 513 _vtn_local_load_store(b, load, child, inout->elems[i], access); 514 } 515 } 516 } 517 518 nir_deref_instr * 519 vtn_nir_deref(struct vtn_builder *b, uint32_t id) 520 { 521 struct vtn_pointer *ptr = vtn_pointer(b, id); 522 return vtn_pointer_to_deref(b, ptr); 523 } 524 525 /* 526 * Gets the NIR-level deref tail, which may have as a child an array deref 527 * selecting which component due to OpAccessChain supporting per-component 528 * indexing in SPIR-V. 529 */ 530 static nir_deref_instr * 531 get_deref_tail(nir_deref_instr *deref) 532 { 533 if (deref->deref_type != nir_deref_type_array) 534 return deref; 535 536 nir_deref_instr *parent = 537 nir_instr_as_deref(deref->parent.ssa->parent_instr); 538 539 if (glsl_type_is_vector(parent->type)) 540 return parent; 541 else 542 return deref; 543 } 544 545 struct vtn_ssa_value * 546 vtn_local_load(struct vtn_builder *b, nir_deref_instr *src, 547 enum gl_access_qualifier access) 548 { 549 nir_deref_instr *src_tail = get_deref_tail(src); 550 struct vtn_ssa_value *val = vtn_create_ssa_value(b, src_tail->type); 551 _vtn_local_load_store(b, true, src_tail, val, access); 552 553 if (src_tail != src) { 554 val->type = src->type; 555 val->def = nir_vector_extract(&b->nb, val->def, src->arr.index.ssa); 556 } 557 558 return val; 559 } 560 561 void 562 vtn_local_store(struct vtn_builder *b, struct vtn_ssa_value *src, 563 nir_deref_instr *dest, enum gl_access_qualifier access) 564 { 565 nir_deref_instr *dest_tail = get_deref_tail(dest); 566 567 if (dest_tail != dest) { 568 struct vtn_ssa_value *val = vtn_create_ssa_value(b, dest_tail->type); 569 _vtn_local_load_store(b, true, dest_tail, val, access); 570 571 val->def = nir_vector_insert(&b->nb, val->def, src->def, 572 dest->arr.index.ssa); 573 _vtn_local_load_store(b, false, dest_tail, val, access); 574 } else { 575 _vtn_local_load_store(b, false, dest_tail, src, access); 576 } 577 } 578 579 static nir_ssa_def * 580 vtn_pointer_to_descriptor(struct vtn_builder *b, struct vtn_pointer *ptr) 581 { 582 assert(ptr->mode == vtn_variable_mode_accel_struct); 583 if (!ptr->block_index) { 584 struct vtn_access_chain chain = { 585 .length = 0, 586 }; 587 ptr = vtn_pointer_dereference(b, ptr, &chain); 588 } 589 590 vtn_assert(ptr->deref == NULL && ptr->block_index != NULL); 591 return vtn_descriptor_load(b, ptr->mode, ptr->block_index); 592 } 593 594 static void 595 _vtn_variable_load_store(struct vtn_builder *b, bool load, 596 struct vtn_pointer *ptr, 597 enum gl_access_qualifier access, 598 struct vtn_ssa_value **inout) 599 { 600 if (ptr->mode == vtn_variable_mode_uniform) { 601 if (ptr->type->base_type == vtn_base_type_image || 602 ptr->type->base_type == vtn_base_type_sampler) { 603 /* See also our handling of OpTypeSampler and OpTypeImage */ 604 vtn_assert(load); 605 (*inout)->def = vtn_pointer_to_ssa(b, ptr); 606 return; 607 } else if (ptr->type->base_type == vtn_base_type_sampled_image) { 608 /* See also our handling of OpTypeSampledImage */ 609 vtn_assert(load); 610 struct vtn_sampled_image si = { 611 .image = vtn_pointer_to_deref(b, ptr), 612 .sampler = vtn_pointer_to_deref(b, ptr), 613 }; 614 (*inout)->def = vtn_sampled_image_to_nir_ssa(b, si); 615 return; 616 } 617 } else if (ptr->mode == vtn_variable_mode_accel_struct) { 618 vtn_assert(load); 619 (*inout)->def = vtn_pointer_to_descriptor(b, ptr); 620 return; 621 } 622 623 enum glsl_base_type base_type = glsl_get_base_type(ptr->type->type); 624 switch (base_type) { 625 case GLSL_TYPE_UINT: 626 case GLSL_TYPE_INT: 627 case GLSL_TYPE_UINT16: 628 case GLSL_TYPE_INT16: 629 case GLSL_TYPE_UINT8: 630 case GLSL_TYPE_INT8: 631 case GLSL_TYPE_UINT64: 632 case GLSL_TYPE_INT64: 633 case GLSL_TYPE_FLOAT: 634 case GLSL_TYPE_FLOAT16: 635 case GLSL_TYPE_BOOL: 636 case GLSL_TYPE_DOUBLE: 637 if (glsl_type_is_vector_or_scalar(ptr->type->type)) { 638 /* We hit a vector or scalar; go ahead and emit the load[s] */ 639 nir_deref_instr *deref = vtn_pointer_to_deref(b, ptr); 640 if (vtn_mode_is_cross_invocation(b, ptr->mode)) { 641 /* If it's cross-invocation, we call nir_load/store_deref 642 * directly. The vtn_local_load/store helpers are too clever and 643 * do magic to avoid array derefs of vectors. That magic is both 644 * less efficient than the direct load/store and, in the case of 645 * stores, is broken because it creates a race condition if two 646 * threads are writing to different components of the same vector 647 * due to the load+insert+store it uses to emulate the array 648 * deref. 649 */ 650 if (load) { 651 (*inout)->def = nir_load_deref_with_access(&b->nb, deref, 652 ptr->type->access | access); 653 } else { 654 nir_store_deref_with_access(&b->nb, deref, (*inout)->def, ~0, 655 ptr->type->access | access); 656 } 657 } else { 658 if (load) { 659 *inout = vtn_local_load(b, deref, ptr->type->access | access); 660 } else { 661 vtn_local_store(b, *inout, deref, ptr->type->access | access); 662 } 663 } 664 return; 665 } 666 FALLTHROUGH; 667 668 case GLSL_TYPE_INTERFACE: 669 case GLSL_TYPE_ARRAY: 670 case GLSL_TYPE_STRUCT: { 671 unsigned elems = glsl_get_length(ptr->type->type); 672 struct vtn_access_chain chain = { 673 .length = 1, 674 .link = { 675 { .mode = vtn_access_mode_literal, }, 676 } 677 }; 678 for (unsigned i = 0; i < elems; i++) { 679 chain.link[0].id = i; 680 struct vtn_pointer *elem = vtn_pointer_dereference(b, ptr, &chain); 681 _vtn_variable_load_store(b, load, elem, ptr->type->access | access, 682 &(*inout)->elems[i]); 683 } 684 return; 685 } 686 687 default: 688 vtn_fail("Invalid access chain type"); 689 } 690 } 691 692 struct vtn_ssa_value * 693 vtn_variable_load(struct vtn_builder *b, struct vtn_pointer *src, 694 enum gl_access_qualifier access) 695 { 696 struct vtn_ssa_value *val = vtn_create_ssa_value(b, src->type->type); 697 _vtn_variable_load_store(b, true, src, src->access | access, &val); 698 return val; 699 } 700 701 void 702 vtn_variable_store(struct vtn_builder *b, struct vtn_ssa_value *src, 703 struct vtn_pointer *dest, enum gl_access_qualifier access) 704 { 705 _vtn_variable_load_store(b, false, dest, dest->access | access, &src); 706 } 707 708 static void 709 _vtn_variable_copy(struct vtn_builder *b, struct vtn_pointer *dest, 710 struct vtn_pointer *src, enum gl_access_qualifier dest_access, 711 enum gl_access_qualifier src_access) 712 { 713 vtn_assert(glsl_get_bare_type(src->type->type) == 714 glsl_get_bare_type(dest->type->type)); 715 enum glsl_base_type base_type = glsl_get_base_type(src->type->type); 716 switch (base_type) { 717 case GLSL_TYPE_UINT: 718 case GLSL_TYPE_INT: 719 case GLSL_TYPE_UINT16: 720 case GLSL_TYPE_INT16: 721 case GLSL_TYPE_UINT8: 722 case GLSL_TYPE_INT8: 723 case GLSL_TYPE_UINT64: 724 case GLSL_TYPE_INT64: 725 case GLSL_TYPE_FLOAT: 726 case GLSL_TYPE_FLOAT16: 727 case GLSL_TYPE_DOUBLE: 728 case GLSL_TYPE_BOOL: 729 /* At this point, we have a scalar, vector, or matrix so we know that 730 * there cannot be any structure splitting still in the way. By 731 * stopping at the matrix level rather than the vector level, we 732 * ensure that matrices get loaded in the optimal way even if they 733 * are storred row-major in a UBO. 734 */ 735 vtn_variable_store(b, vtn_variable_load(b, src, src_access), dest, dest_access); 736 return; 737 738 case GLSL_TYPE_INTERFACE: 739 case GLSL_TYPE_ARRAY: 740 case GLSL_TYPE_STRUCT: { 741 struct vtn_access_chain chain = { 742 .length = 1, 743 .link = { 744 { .mode = vtn_access_mode_literal, }, 745 } 746 }; 747 unsigned elems = glsl_get_length(src->type->type); 748 for (unsigned i = 0; i < elems; i++) { 749 chain.link[0].id = i; 750 struct vtn_pointer *src_elem = 751 vtn_pointer_dereference(b, src, &chain); 752 struct vtn_pointer *dest_elem = 753 vtn_pointer_dereference(b, dest, &chain); 754 755 _vtn_variable_copy(b, dest_elem, src_elem, dest_access, src_access); 756 } 757 return; 758 } 759 760 default: 761 vtn_fail("Invalid access chain type"); 762 } 763 } 764 765 static void 766 vtn_variable_copy(struct vtn_builder *b, struct vtn_pointer *dest, 767 struct vtn_pointer *src, enum gl_access_qualifier dest_access, 768 enum gl_access_qualifier src_access) 769 { 770 /* TODO: At some point, we should add a special-case for when we can 771 * just emit a copy_var intrinsic. 772 */ 773 _vtn_variable_copy(b, dest, src, dest_access, src_access); 774 } 775 776 static void 777 set_mode_system_value(struct vtn_builder *b, nir_variable_mode *mode) 778 { 779 vtn_assert(*mode == nir_var_system_value || *mode == nir_var_shader_in); 780 *mode = nir_var_system_value; 781 } 782 783 static void 784 vtn_get_builtin_location(struct vtn_builder *b, 785 SpvBuiltIn builtin, int *location, 786 nir_variable_mode *mode) 787 { 788 switch (builtin) { 789 case SpvBuiltInPosition: 790 case SpvBuiltInPositionPerViewNV: 791 *location = VARYING_SLOT_POS; 792 break; 793 case SpvBuiltInPointSize: 794 *location = VARYING_SLOT_PSIZ; 795 break; 796 case SpvBuiltInClipDistance: 797 case SpvBuiltInClipDistancePerViewNV: 798 *location = VARYING_SLOT_CLIP_DIST0; 799 break; 800 case SpvBuiltInCullDistance: 801 case SpvBuiltInCullDistancePerViewNV: 802 *location = VARYING_SLOT_CULL_DIST0; 803 break; 804 case SpvBuiltInVertexId: 805 case SpvBuiltInVertexIndex: 806 /* The Vulkan spec defines VertexIndex to be non-zero-based and doesn't 807 * allow VertexId. The ARB_gl_spirv spec defines VertexId to be the 808 * same as gl_VertexID, which is non-zero-based, and removes 809 * VertexIndex. Since they're both defined to be non-zero-based, we use 810 * SYSTEM_VALUE_VERTEX_ID for both. 811 */ 812 *location = SYSTEM_VALUE_VERTEX_ID; 813 set_mode_system_value(b, mode); 814 break; 815 case SpvBuiltInInstanceIndex: 816 *location = SYSTEM_VALUE_INSTANCE_INDEX; 817 set_mode_system_value(b, mode); 818 break; 819 case SpvBuiltInInstanceId: 820 *location = SYSTEM_VALUE_INSTANCE_ID; 821 set_mode_system_value(b, mode); 822 break; 823 case SpvBuiltInPrimitiveId: 824 if (b->shader->info.stage == MESA_SHADER_FRAGMENT) { 825 vtn_assert(*mode == nir_var_shader_in); 826 *location = VARYING_SLOT_PRIMITIVE_ID; 827 } else if (*mode == nir_var_shader_out) { 828 *location = VARYING_SLOT_PRIMITIVE_ID; 829 } else { 830 *location = SYSTEM_VALUE_PRIMITIVE_ID; 831 set_mode_system_value(b, mode); 832 } 833 break; 834 case SpvBuiltInInvocationId: 835 *location = SYSTEM_VALUE_INVOCATION_ID; 836 set_mode_system_value(b, mode); 837 break; 838 case SpvBuiltInLayer: 839 *location = VARYING_SLOT_LAYER; 840 if (b->shader->info.stage == MESA_SHADER_FRAGMENT) 841 *mode = nir_var_shader_in; 842 else if (b->shader->info.stage == MESA_SHADER_GEOMETRY) 843 *mode = nir_var_shader_out; 844 else if (b->options && b->options->caps.shader_viewport_index_layer && 845 (b->shader->info.stage == MESA_SHADER_VERTEX || 846 b->shader->info.stage == MESA_SHADER_TESS_EVAL || 847 b->shader->info.stage == MESA_SHADER_MESH)) 848 *mode = nir_var_shader_out; 849 else 850 vtn_fail("invalid stage for SpvBuiltInLayer"); 851 break; 852 case SpvBuiltInViewportIndex: 853 *location = VARYING_SLOT_VIEWPORT; 854 if (b->shader->info.stage == MESA_SHADER_GEOMETRY) 855 *mode = nir_var_shader_out; 856 else if (b->options && b->options->caps.shader_viewport_index_layer && 857 (b->shader->info.stage == MESA_SHADER_VERTEX || 858 b->shader->info.stage == MESA_SHADER_TESS_EVAL || 859 b->shader->info.stage == MESA_SHADER_MESH)) 860 *mode = nir_var_shader_out; 861 else if (b->shader->info.stage == MESA_SHADER_FRAGMENT) 862 *mode = nir_var_shader_in; 863 else 864 vtn_fail("invalid stage for SpvBuiltInViewportIndex"); 865 break; 866 case SpvBuiltInTessLevelOuter: 867 *location = VARYING_SLOT_TESS_LEVEL_OUTER; 868 break; 869 case SpvBuiltInTessLevelInner: 870 *location = VARYING_SLOT_TESS_LEVEL_INNER; 871 break; 872 case SpvBuiltInTessCoord: 873 *location = SYSTEM_VALUE_TESS_COORD; 874 set_mode_system_value(b, mode); 875 break; 876 case SpvBuiltInPatchVertices: 877 *location = SYSTEM_VALUE_VERTICES_IN; 878 set_mode_system_value(b, mode); 879 break; 880 case SpvBuiltInFragCoord: 881 vtn_assert(*mode == nir_var_shader_in); 882 *mode = nir_var_system_value; 883 *location = SYSTEM_VALUE_FRAG_COORD; 884 break; 885 case SpvBuiltInPointCoord: 886 vtn_assert(*mode == nir_var_shader_in); 887 set_mode_system_value(b, mode); 888 *location = SYSTEM_VALUE_POINT_COORD; 889 break; 890 case SpvBuiltInFrontFacing: 891 *location = SYSTEM_VALUE_FRONT_FACE; 892 set_mode_system_value(b, mode); 893 break; 894 case SpvBuiltInSampleId: 895 *location = SYSTEM_VALUE_SAMPLE_ID; 896 set_mode_system_value(b, mode); 897 break; 898 case SpvBuiltInSamplePosition: 899 *location = SYSTEM_VALUE_SAMPLE_POS; 900 set_mode_system_value(b, mode); 901 break; 902 case SpvBuiltInSampleMask: 903 if (*mode == nir_var_shader_out) { 904 *location = FRAG_RESULT_SAMPLE_MASK; 905 } else { 906 *location = SYSTEM_VALUE_SAMPLE_MASK_IN; 907 set_mode_system_value(b, mode); 908 } 909 break; 910 case SpvBuiltInFragDepth: 911 *location = FRAG_RESULT_DEPTH; 912 vtn_assert(*mode == nir_var_shader_out); 913 break; 914 case SpvBuiltInHelperInvocation: 915 *location = SYSTEM_VALUE_HELPER_INVOCATION; 916 set_mode_system_value(b, mode); 917 break; 918 case SpvBuiltInNumWorkgroups: 919 *location = SYSTEM_VALUE_NUM_WORKGROUPS; 920 set_mode_system_value(b, mode); 921 break; 922 case SpvBuiltInWorkgroupSize: 923 case SpvBuiltInEnqueuedWorkgroupSize: 924 *location = SYSTEM_VALUE_WORKGROUP_SIZE; 925 set_mode_system_value(b, mode); 926 break; 927 case SpvBuiltInWorkgroupId: 928 *location = SYSTEM_VALUE_WORKGROUP_ID; 929 set_mode_system_value(b, mode); 930 break; 931 case SpvBuiltInLocalInvocationId: 932 *location = SYSTEM_VALUE_LOCAL_INVOCATION_ID; 933 set_mode_system_value(b, mode); 934 break; 935 case SpvBuiltInLocalInvocationIndex: 936 *location = SYSTEM_VALUE_LOCAL_INVOCATION_INDEX; 937 set_mode_system_value(b, mode); 938 break; 939 case SpvBuiltInGlobalInvocationId: 940 *location = SYSTEM_VALUE_GLOBAL_INVOCATION_ID; 941 set_mode_system_value(b, mode); 942 break; 943 case SpvBuiltInGlobalLinearId: 944 *location = SYSTEM_VALUE_GLOBAL_INVOCATION_INDEX; 945 set_mode_system_value(b, mode); 946 break; 947 case SpvBuiltInGlobalOffset: 948 *location = SYSTEM_VALUE_BASE_GLOBAL_INVOCATION_ID; 949 set_mode_system_value(b, mode); 950 break; 951 case SpvBuiltInBaseVertex: 952 /* OpenGL gl_BaseVertex (SYSTEM_VALUE_BASE_VERTEX) is not the same 953 * semantic as Vulkan BaseVertex (SYSTEM_VALUE_FIRST_VERTEX). 954 */ 955 if (b->options->environment == NIR_SPIRV_OPENGL) 956 *location = SYSTEM_VALUE_BASE_VERTEX; 957 else 958 *location = SYSTEM_VALUE_FIRST_VERTEX; 959 set_mode_system_value(b, mode); 960 break; 961 case SpvBuiltInBaseInstance: 962 *location = SYSTEM_VALUE_BASE_INSTANCE; 963 set_mode_system_value(b, mode); 964 break; 965 case SpvBuiltInDrawIndex: 966 *location = SYSTEM_VALUE_DRAW_ID; 967 set_mode_system_value(b, mode); 968 break; 969 case SpvBuiltInSubgroupSize: 970 *location = SYSTEM_VALUE_SUBGROUP_SIZE; 971 set_mode_system_value(b, mode); 972 break; 973 case SpvBuiltInSubgroupId: 974 *location = SYSTEM_VALUE_SUBGROUP_ID; 975 set_mode_system_value(b, mode); 976 break; 977 case SpvBuiltInSubgroupLocalInvocationId: 978 *location = SYSTEM_VALUE_SUBGROUP_INVOCATION; 979 set_mode_system_value(b, mode); 980 break; 981 case SpvBuiltInNumSubgroups: 982 *location = SYSTEM_VALUE_NUM_SUBGROUPS; 983 set_mode_system_value(b, mode); 984 break; 985 case SpvBuiltInDeviceIndex: 986 *location = SYSTEM_VALUE_DEVICE_INDEX; 987 set_mode_system_value(b, mode); 988 break; 989 case SpvBuiltInViewIndex: 990 if (b->options && b->options->view_index_is_input) { 991 *location = VARYING_SLOT_VIEW_INDEX; 992 vtn_assert(*mode == nir_var_shader_in); 993 } else { 994 *location = SYSTEM_VALUE_VIEW_INDEX; 995 set_mode_system_value(b, mode); 996 } 997 break; 998 case SpvBuiltInSubgroupEqMask: 999 *location = SYSTEM_VALUE_SUBGROUP_EQ_MASK, 1000 set_mode_system_value(b, mode); 1001 break; 1002 case SpvBuiltInSubgroupGeMask: 1003 *location = SYSTEM_VALUE_SUBGROUP_GE_MASK, 1004 set_mode_system_value(b, mode); 1005 break; 1006 case SpvBuiltInSubgroupGtMask: 1007 *location = SYSTEM_VALUE_SUBGROUP_GT_MASK, 1008 set_mode_system_value(b, mode); 1009 break; 1010 case SpvBuiltInSubgroupLeMask: 1011 *location = SYSTEM_VALUE_SUBGROUP_LE_MASK, 1012 set_mode_system_value(b, mode); 1013 break; 1014 case SpvBuiltInSubgroupLtMask: 1015 *location = SYSTEM_VALUE_SUBGROUP_LT_MASK, 1016 set_mode_system_value(b, mode); 1017 break; 1018 case SpvBuiltInFragStencilRefEXT: 1019 *location = FRAG_RESULT_STENCIL; 1020 vtn_assert(*mode == nir_var_shader_out); 1021 break; 1022 case SpvBuiltInWorkDim: 1023 *location = SYSTEM_VALUE_WORK_DIM; 1024 set_mode_system_value(b, mode); 1025 break; 1026 case SpvBuiltInGlobalSize: 1027 *location = SYSTEM_VALUE_GLOBAL_GROUP_SIZE; 1028 set_mode_system_value(b, mode); 1029 break; 1030 case SpvBuiltInBaryCoordNoPerspAMD: 1031 *location = SYSTEM_VALUE_BARYCENTRIC_LINEAR_PIXEL; 1032 set_mode_system_value(b, mode); 1033 break; 1034 case SpvBuiltInBaryCoordNoPerspCentroidAMD: 1035 *location = SYSTEM_VALUE_BARYCENTRIC_LINEAR_CENTROID; 1036 set_mode_system_value(b, mode); 1037 break; 1038 case SpvBuiltInBaryCoordNoPerspSampleAMD: 1039 *location = SYSTEM_VALUE_BARYCENTRIC_LINEAR_SAMPLE; 1040 set_mode_system_value(b, mode); 1041 break; 1042 case SpvBuiltInBaryCoordSmoothAMD: 1043 *location = SYSTEM_VALUE_BARYCENTRIC_PERSP_PIXEL; 1044 set_mode_system_value(b, mode); 1045 break; 1046 case SpvBuiltInBaryCoordSmoothCentroidAMD: 1047 *location = SYSTEM_VALUE_BARYCENTRIC_PERSP_CENTROID; 1048 set_mode_system_value(b, mode); 1049 break; 1050 case SpvBuiltInBaryCoordSmoothSampleAMD: 1051 *location = SYSTEM_VALUE_BARYCENTRIC_PERSP_SAMPLE; 1052 set_mode_system_value(b, mode); 1053 break; 1054 case SpvBuiltInBaryCoordPullModelAMD: 1055 *location = SYSTEM_VALUE_BARYCENTRIC_PULL_MODEL; 1056 set_mode_system_value(b, mode); 1057 break; 1058 case SpvBuiltInLaunchIdKHR: 1059 *location = SYSTEM_VALUE_RAY_LAUNCH_ID; 1060 set_mode_system_value(b, mode); 1061 break; 1062 case SpvBuiltInLaunchSizeKHR: 1063 *location = SYSTEM_VALUE_RAY_LAUNCH_SIZE; 1064 set_mode_system_value(b, mode); 1065 break; 1066 case SpvBuiltInWorldRayOriginKHR: 1067 *location = SYSTEM_VALUE_RAY_WORLD_ORIGIN; 1068 set_mode_system_value(b, mode); 1069 break; 1070 case SpvBuiltInWorldRayDirectionKHR: 1071 *location = SYSTEM_VALUE_RAY_WORLD_DIRECTION; 1072 set_mode_system_value(b, mode); 1073 break; 1074 case SpvBuiltInObjectRayOriginKHR: 1075 *location = SYSTEM_VALUE_RAY_OBJECT_ORIGIN; 1076 set_mode_system_value(b, mode); 1077 break; 1078 case SpvBuiltInObjectRayDirectionKHR: 1079 *location = SYSTEM_VALUE_RAY_OBJECT_DIRECTION; 1080 set_mode_system_value(b, mode); 1081 break; 1082 case SpvBuiltInObjectToWorldKHR: 1083 *location = SYSTEM_VALUE_RAY_OBJECT_TO_WORLD; 1084 set_mode_system_value(b, mode); 1085 break; 1086 case SpvBuiltInWorldToObjectKHR: 1087 *location = SYSTEM_VALUE_RAY_WORLD_TO_OBJECT; 1088 set_mode_system_value(b, mode); 1089 break; 1090 case SpvBuiltInRayTminKHR: 1091 *location = SYSTEM_VALUE_RAY_T_MIN; 1092 set_mode_system_value(b, mode); 1093 break; 1094 case SpvBuiltInRayTmaxKHR: 1095 case SpvBuiltInHitTNV: 1096 *location = SYSTEM_VALUE_RAY_T_MAX; 1097 set_mode_system_value(b, mode); 1098 break; 1099 case SpvBuiltInInstanceCustomIndexKHR: 1100 *location = SYSTEM_VALUE_RAY_INSTANCE_CUSTOM_INDEX; 1101 set_mode_system_value(b, mode); 1102 break; 1103 case SpvBuiltInHitKindKHR: 1104 *location = SYSTEM_VALUE_RAY_HIT_KIND; 1105 set_mode_system_value(b, mode); 1106 break; 1107 case SpvBuiltInIncomingRayFlagsKHR: 1108 *location = SYSTEM_VALUE_RAY_FLAGS; 1109 set_mode_system_value(b, mode); 1110 break; 1111 case SpvBuiltInRayGeometryIndexKHR: 1112 *location = SYSTEM_VALUE_RAY_GEOMETRY_INDEX; 1113 set_mode_system_value(b, mode); 1114 break; 1115 case SpvBuiltInShadingRateKHR: 1116 *location = SYSTEM_VALUE_FRAG_SHADING_RATE; 1117 set_mode_system_value(b, mode); 1118 break; 1119 case SpvBuiltInPrimitiveShadingRateKHR: 1120 if (b->shader->info.stage == MESA_SHADER_VERTEX || 1121 b->shader->info.stage == MESA_SHADER_GEOMETRY) { 1122 *location = VARYING_SLOT_PRIMITIVE_SHADING_RATE; 1123 *mode = nir_var_shader_out; 1124 } else { 1125 vtn_fail("invalid stage for SpvBuiltInPrimitiveShadingRateKHR"); 1126 } 1127 break; 1128 case SpvBuiltInPrimitiveCountNV: 1129 *location = VARYING_SLOT_PRIMITIVE_COUNT; 1130 break; 1131 case SpvBuiltInPrimitiveIndicesNV: 1132 *location = VARYING_SLOT_PRIMITIVE_INDICES; 1133 break; 1134 case SpvBuiltInTaskCountNV: 1135 *location = VARYING_SLOT_TASK_COUNT; 1136 break; 1137 default: 1138 vtn_fail("Unsupported builtin: %s (%u)", 1139 spirv_builtin_to_string(builtin), builtin); 1140 } 1141 } 1142 1143 static void 1144 apply_var_decoration(struct vtn_builder *b, 1145 struct nir_variable_data *var_data, 1146 const struct vtn_decoration *dec) 1147 { 1148 switch (dec->decoration) { 1149 case SpvDecorationRelaxedPrecision: 1150 var_data->precision = GLSL_PRECISION_MEDIUM; 1151 break; 1152 case SpvDecorationNoPerspective: 1153 var_data->interpolation = INTERP_MODE_NOPERSPECTIVE; 1154 break; 1155 case SpvDecorationFlat: 1156 var_data->interpolation = INTERP_MODE_FLAT; 1157 break; 1158 case SpvDecorationExplicitInterpAMD: 1159 var_data->interpolation = INTERP_MODE_EXPLICIT; 1160 break; 1161 case SpvDecorationCentroid: 1162 var_data->centroid = true; 1163 break; 1164 case SpvDecorationSample: 1165 var_data->sample = true; 1166 break; 1167 case SpvDecorationInvariant: 1168 var_data->invariant = true; 1169 break; 1170 case SpvDecorationConstant: 1171 var_data->read_only = true; 1172 break; 1173 case SpvDecorationNonReadable: 1174 var_data->access |= ACCESS_NON_READABLE; 1175 break; 1176 case SpvDecorationNonWritable: 1177 var_data->read_only = true; 1178 var_data->access |= ACCESS_NON_WRITEABLE; 1179 break; 1180 case SpvDecorationRestrict: 1181 var_data->access |= ACCESS_RESTRICT; 1182 break; 1183 case SpvDecorationAliased: 1184 var_data->access &= ~ACCESS_RESTRICT; 1185 break; 1186 case SpvDecorationVolatile: 1187 var_data->access |= ACCESS_VOLATILE; 1188 break; 1189 case SpvDecorationCoherent: 1190 var_data->access |= ACCESS_COHERENT; 1191 break; 1192 case SpvDecorationComponent: 1193 var_data->location_frac = dec->operands[0]; 1194 break; 1195 case SpvDecorationIndex: 1196 var_data->index = dec->operands[0]; 1197 break; 1198 case SpvDecorationBuiltIn: { 1199 SpvBuiltIn builtin = dec->operands[0]; 1200 1201 nir_variable_mode mode = var_data->mode; 1202 vtn_get_builtin_location(b, builtin, &var_data->location, &mode); 1203 var_data->mode = mode; 1204 1205 switch (builtin) { 1206 case SpvBuiltInTessLevelOuter: 1207 case SpvBuiltInTessLevelInner: 1208 case SpvBuiltInClipDistance: 1209 case SpvBuiltInCullDistance: 1210 var_data->compact = true; 1211 break; 1212 default: 1213 break; 1214 } 1215 1216 break; 1217 } 1218 1219 case SpvDecorationSpecId: 1220 case SpvDecorationRowMajor: 1221 case SpvDecorationColMajor: 1222 case SpvDecorationMatrixStride: 1223 case SpvDecorationUniform: 1224 case SpvDecorationUniformId: 1225 case SpvDecorationLinkageAttributes: 1226 break; /* Do nothing with these here */ 1227 1228 case SpvDecorationPatch: 1229 var_data->patch = true; 1230 break; 1231 1232 case SpvDecorationLocation: 1233 vtn_fail("Should be handled earlier by var_decoration_cb()"); 1234 1235 case SpvDecorationBlock: 1236 case SpvDecorationBufferBlock: 1237 case SpvDecorationArrayStride: 1238 case SpvDecorationGLSLShared: 1239 case SpvDecorationGLSLPacked: 1240 break; /* These can apply to a type but we don't care about them */ 1241 1242 case SpvDecorationBinding: 1243 case SpvDecorationDescriptorSet: 1244 case SpvDecorationNoContraction: 1245 case SpvDecorationInputAttachmentIndex: 1246 vtn_warn("Decoration not allowed for variable or structure member: %s", 1247 spirv_decoration_to_string(dec->decoration)); 1248 break; 1249 1250 case SpvDecorationXfbBuffer: 1251 var_data->explicit_xfb_buffer = true; 1252 var_data->xfb.buffer = dec->operands[0]; 1253 var_data->always_active_io = true; 1254 break; 1255 case SpvDecorationXfbStride: 1256 var_data->explicit_xfb_stride = true; 1257 var_data->xfb.stride = dec->operands[0]; 1258 break; 1259 case SpvDecorationOffset: 1260 var_data->explicit_offset = true; 1261 var_data->offset = dec->operands[0]; 1262 break; 1263 1264 case SpvDecorationStream: 1265 var_data->stream = dec->operands[0]; 1266 break; 1267 1268 case SpvDecorationCPacked: 1269 case SpvDecorationSaturatedConversion: 1270 case SpvDecorationFuncParamAttr: 1271 case SpvDecorationFPRoundingMode: 1272 case SpvDecorationFPFastMathMode: 1273 case SpvDecorationAlignment: 1274 if (b->shader->info.stage != MESA_SHADER_KERNEL) { 1275 vtn_warn("Decoration only allowed for CL-style kernels: %s", 1276 spirv_decoration_to_string(dec->decoration)); 1277 } 1278 break; 1279 1280 case SpvDecorationUserSemantic: 1281 case SpvDecorationUserTypeGOOGLE: 1282 /* User semantic decorations can safely be ignored by the driver. */ 1283 break; 1284 1285 case SpvDecorationRestrictPointerEXT: 1286 case SpvDecorationAliasedPointerEXT: 1287 /* TODO: We should actually plumb alias information through NIR. */ 1288 break; 1289 1290 case SpvDecorationPerPrimitiveNV: 1291 vtn_fail_if( 1292 !(b->shader->info.stage == MESA_SHADER_MESH && var_data->mode == nir_var_shader_out) && 1293 !(b->shader->info.stage == MESA_SHADER_FRAGMENT && var_data->mode == nir_var_shader_in), 1294 "PerPrimitiveNV decoration only allowed for Mesh shader outputs or Fragment shader inputs"); 1295 var_data->per_primitive = true; 1296 break; 1297 1298 case SpvDecorationPerTaskNV: 1299 vtn_fail_if( 1300 !(b->shader->info.stage == MESA_SHADER_TASK && var_data->mode == nir_var_shader_out) && 1301 !(b->shader->info.stage == MESA_SHADER_MESH && var_data->mode == nir_var_shader_in), 1302 "PerTaskNV decoration only allowed for Task shader outputs or Mesh shader inputs"); 1303 /* Don't set anything, because this decoration is implied by being a 1304 * non-builtin Task Output or Mesh Input. 1305 */ 1306 break; 1307 1308 case SpvDecorationPerViewNV: 1309 vtn_fail_if(b->shader->info.stage != MESA_SHADER_MESH, 1310 "PerViewNV decoration only allowed in Mesh shaders"); 1311 var_data->per_view = true; 1312 break; 1313 1314 default: 1315 vtn_fail_with_decoration("Unhandled decoration", dec->decoration); 1316 } 1317 } 1318 1319 static void 1320 gather_var_kind_cb(struct vtn_builder *b, struct vtn_value *val, int member, 1321 const struct vtn_decoration *dec, void *void_var) 1322 { 1323 struct vtn_variable *vtn_var = void_var; 1324 switch (dec->decoration) { 1325 case SpvDecorationPatch: 1326 vtn_var->var->data.patch = true; 1327 break; 1328 case SpvDecorationPerPrimitiveNV: 1329 vtn_var->var->data.per_primitive = true; 1330 break; 1331 case SpvDecorationBuiltIn: 1332 if (b->shader->info.stage == MESA_SHADER_MESH) { 1333 SpvBuiltIn builtin = dec->operands[0]; 1334 switch (builtin) { 1335 case SpvBuiltInPrimitiveIndicesNV: 1336 vtn_var->var->data.per_primitive = true; 1337 break; 1338 default: 1339 /* Nothing to do. */ 1340 break; 1341 } 1342 } 1343 break; 1344 default: 1345 /* Nothing to do. */ 1346 break; 1347 } 1348 } 1349 1350 static void 1351 var_decoration_cb(struct vtn_builder *b, struct vtn_value *val, int member, 1352 const struct vtn_decoration *dec, void *void_var) 1353 { 1354 struct vtn_variable *vtn_var = void_var; 1355 1356 /* Handle decorations that apply to a vtn_variable as a whole */ 1357 switch (dec->decoration) { 1358 case SpvDecorationBinding: 1359 vtn_var->binding = dec->operands[0]; 1360 vtn_var->explicit_binding = true; 1361 return; 1362 case SpvDecorationDescriptorSet: 1363 vtn_var->descriptor_set = dec->operands[0]; 1364 return; 1365 case SpvDecorationInputAttachmentIndex: 1366 vtn_var->input_attachment_index = dec->operands[0]; 1367 return; 1368 case SpvDecorationPatch: 1369 vtn_var->var->data.patch = true; 1370 break; 1371 case SpvDecorationOffset: 1372 vtn_var->offset = dec->operands[0]; 1373 break; 1374 case SpvDecorationNonWritable: 1375 vtn_var->access |= ACCESS_NON_WRITEABLE; 1376 break; 1377 case SpvDecorationNonReadable: 1378 vtn_var->access |= ACCESS_NON_READABLE; 1379 break; 1380 case SpvDecorationVolatile: 1381 vtn_var->access |= ACCESS_VOLATILE; 1382 break; 1383 case SpvDecorationCoherent: 1384 vtn_var->access |= ACCESS_COHERENT; 1385 break; 1386 case SpvDecorationCounterBuffer: 1387 /* Counter buffer decorations can safely be ignored by the driver. */ 1388 return; 1389 default: 1390 break; 1391 } 1392 1393 if (val->value_type == vtn_value_type_pointer) { 1394 assert(val->pointer->var == void_var); 1395 assert(member == -1); 1396 } else { 1397 assert(val->value_type == vtn_value_type_type); 1398 } 1399 1400 /* Location is odd. If applied to a split structure, we have to walk the 1401 * whole thing and accumulate the location. It's easier to handle as a 1402 * special case. 1403 */ 1404 if (dec->decoration == SpvDecorationLocation) { 1405 unsigned location = dec->operands[0]; 1406 if (b->shader->info.stage == MESA_SHADER_FRAGMENT && 1407 vtn_var->mode == vtn_variable_mode_output) { 1408 location += FRAG_RESULT_DATA0; 1409 } else if (b->shader->info.stage == MESA_SHADER_VERTEX && 1410 vtn_var->mode == vtn_variable_mode_input) { 1411 location += VERT_ATTRIB_GENERIC0; 1412 } else if (vtn_var->mode == vtn_variable_mode_input || 1413 vtn_var->mode == vtn_variable_mode_output) { 1414 location += vtn_var->var->data.patch ? VARYING_SLOT_PATCH0 : VARYING_SLOT_VAR0; 1415 } else if (vtn_var->mode == vtn_variable_mode_call_data || 1416 vtn_var->mode == vtn_variable_mode_ray_payload) { 1417 /* This location is fine as-is */ 1418 } else if (vtn_var->mode != vtn_variable_mode_uniform) { 1419 vtn_warn("Location must be on input, output, uniform, sampler or " 1420 "image variable"); 1421 return; 1422 } 1423 1424 if (vtn_var->var->num_members == 0) { 1425 /* This handles the member and lone variable cases */ 1426 vtn_var->var->data.location = location; 1427 } else { 1428 /* This handles the structure member case */ 1429 assert(vtn_var->var->members); 1430 1431 if (member == -1) 1432 vtn_var->base_location = location; 1433 else 1434 vtn_var->var->members[member].location = location; 1435 } 1436 1437 return; 1438 } else { 1439 if (vtn_var->var) { 1440 if (vtn_var->var->num_members == 0) { 1441 /* We call this function on types as well as variables and not all 1442 * struct types get split so we can end up having stray member 1443 * decorations; just ignore them. 1444 */ 1445 if (member == -1) 1446 apply_var_decoration(b, &vtn_var->var->data, dec); 1447 } else if (member >= 0) { 1448 /* Member decorations must come from a type */ 1449 assert(val->value_type == vtn_value_type_type); 1450 apply_var_decoration(b, &vtn_var->var->members[member], dec); 1451 } else { 1452 unsigned length = 1453 glsl_get_length(glsl_without_array(vtn_var->type->type)); 1454 for (unsigned i = 0; i < length; i++) 1455 apply_var_decoration(b, &vtn_var->var->members[i], dec); 1456 } 1457 } else { 1458 /* A few variables, those with external storage, have no actual 1459 * nir_variables associated with them. Fortunately, all decorations 1460 * we care about for those variables are on the type only. 1461 */ 1462 vtn_assert(vtn_var->mode == vtn_variable_mode_ubo || 1463 vtn_var->mode == vtn_variable_mode_ssbo || 1464 vtn_var->mode == vtn_variable_mode_push_constant); 1465 } 1466 } 1467 } 1468 1469 enum vtn_variable_mode 1470 vtn_storage_class_to_mode(struct vtn_builder *b, 1471 SpvStorageClass class, 1472 struct vtn_type *interface_type, 1473 nir_variable_mode *nir_mode_out) 1474 { 1475 enum vtn_variable_mode mode; 1476 nir_variable_mode nir_mode; 1477 switch (class) { 1478 case SpvStorageClassUniform: 1479 /* Assume it's an UBO if we lack the interface_type. */ 1480 if (!interface_type || interface_type->block) { 1481 mode = vtn_variable_mode_ubo; 1482 nir_mode = nir_var_mem_ubo; 1483 } else if (interface_type->buffer_block) { 1484 mode = vtn_variable_mode_ssbo; 1485 nir_mode = nir_var_mem_ssbo; 1486 } else { 1487 /* Default-block uniforms, coming from gl_spirv */ 1488 mode = vtn_variable_mode_uniform; 1489 nir_mode = nir_var_uniform; 1490 } 1491 break; 1492 case SpvStorageClassStorageBuffer: 1493 mode = vtn_variable_mode_ssbo; 1494 nir_mode = nir_var_mem_ssbo; 1495 break; 1496 case SpvStorageClassPhysicalStorageBuffer: 1497 mode = vtn_variable_mode_phys_ssbo; 1498 nir_mode = nir_var_mem_global; 1499 break; 1500 case SpvStorageClassUniformConstant: 1501 if (b->shader->info.stage == MESA_SHADER_KERNEL) { 1502 mode = vtn_variable_mode_constant; 1503 nir_mode = nir_var_mem_constant; 1504 } else { 1505 /* interface_type is only NULL when OpTypeForwardPointer is used and 1506 * OpTypeForwardPointer cannot be used with the UniformConstant 1507 * storage class. 1508 */ 1509 assert(interface_type != NULL); 1510 interface_type = vtn_type_without_array(interface_type); 1511 if (interface_type->base_type == vtn_base_type_accel_struct) { 1512 mode = vtn_variable_mode_accel_struct; 1513 nir_mode = nir_var_uniform; 1514 } else { 1515 mode = vtn_variable_mode_uniform; 1516 nir_mode = nir_var_uniform; 1517 } 1518 } 1519 break; 1520 case SpvStorageClassPushConstant: 1521 mode = vtn_variable_mode_push_constant; 1522 nir_mode = nir_var_mem_push_const; 1523 break; 1524 case SpvStorageClassInput: 1525 mode = vtn_variable_mode_input; 1526 nir_mode = nir_var_shader_in; 1527 break; 1528 case SpvStorageClassOutput: 1529 mode = vtn_variable_mode_output; 1530 nir_mode = nir_var_shader_out; 1531 break; 1532 case SpvStorageClassPrivate: 1533 mode = vtn_variable_mode_private; 1534 nir_mode = nir_var_shader_temp; 1535 break; 1536 case SpvStorageClassFunction: 1537 mode = vtn_variable_mode_function; 1538 nir_mode = nir_var_function_temp; 1539 break; 1540 case SpvStorageClassWorkgroup: 1541 mode = vtn_variable_mode_workgroup; 1542 nir_mode = nir_var_mem_shared; 1543 break; 1544 case SpvStorageClassAtomicCounter: 1545 mode = vtn_variable_mode_atomic_counter; 1546 nir_mode = nir_var_uniform; 1547 break; 1548 case SpvStorageClassCrossWorkgroup: 1549 mode = vtn_variable_mode_cross_workgroup; 1550 nir_mode = nir_var_mem_global; 1551 break; 1552 case SpvStorageClassImage: 1553 mode = vtn_variable_mode_image; 1554 nir_mode = nir_var_mem_ubo; 1555 break; 1556 case SpvStorageClassCallableDataKHR: 1557 mode = vtn_variable_mode_call_data; 1558 nir_mode = nir_var_shader_temp; 1559 break; 1560 case SpvStorageClassIncomingCallableDataKHR: 1561 mode = vtn_variable_mode_call_data_in; 1562 nir_mode = nir_var_shader_call_data; 1563 break; 1564 case SpvStorageClassRayPayloadKHR: 1565 mode = vtn_variable_mode_ray_payload; 1566 nir_mode = nir_var_shader_temp; 1567 break; 1568 case SpvStorageClassIncomingRayPayloadKHR: 1569 mode = vtn_variable_mode_ray_payload_in; 1570 nir_mode = nir_var_shader_call_data; 1571 break; 1572 case SpvStorageClassHitAttributeKHR: 1573 mode = vtn_variable_mode_hit_attrib; 1574 nir_mode = nir_var_ray_hit_attrib; 1575 break; 1576 case SpvStorageClassShaderRecordBufferKHR: 1577 mode = vtn_variable_mode_shader_record; 1578 nir_mode = nir_var_mem_constant; 1579 break; 1580 1581 case SpvStorageClassGeneric: 1582 mode = vtn_variable_mode_generic; 1583 nir_mode = nir_var_mem_generic; 1584 break; 1585 default: 1586 vtn_fail("Unhandled variable storage class: %s (%u)", 1587 spirv_storageclass_to_string(class), class); 1588 } 1589 1590 if (nir_mode_out) 1591 *nir_mode_out = nir_mode; 1592 1593 return mode; 1594 } 1595 1596 nir_address_format 1597 vtn_mode_to_address_format(struct vtn_builder *b, enum vtn_variable_mode mode) 1598 { 1599 switch (mode) { 1600 case vtn_variable_mode_ubo: 1601 return b->options->ubo_addr_format; 1602 1603 case vtn_variable_mode_ssbo: 1604 return b->options->ssbo_addr_format; 1605 1606 case vtn_variable_mode_phys_ssbo: 1607 return b->options->phys_ssbo_addr_format; 1608 1609 case vtn_variable_mode_push_constant: 1610 return b->options->push_const_addr_format; 1611 1612 case vtn_variable_mode_workgroup: 1613 return b->options->shared_addr_format; 1614 1615 case vtn_variable_mode_generic: 1616 case vtn_variable_mode_cross_workgroup: 1617 return b->options->global_addr_format; 1618 1619 case vtn_variable_mode_shader_record: 1620 case vtn_variable_mode_constant: 1621 return b->options->constant_addr_format; 1622 1623 case vtn_variable_mode_accel_struct: 1624 return nir_address_format_64bit_global; 1625 1626 case vtn_variable_mode_function: 1627 if (b->physical_ptrs) 1628 return b->options->temp_addr_format; 1629 FALLTHROUGH; 1630 1631 case vtn_variable_mode_private: 1632 case vtn_variable_mode_uniform: 1633 case vtn_variable_mode_atomic_counter: 1634 case vtn_variable_mode_input: 1635 case vtn_variable_mode_output: 1636 case vtn_variable_mode_image: 1637 case vtn_variable_mode_call_data: 1638 case vtn_variable_mode_call_data_in: 1639 case vtn_variable_mode_ray_payload: 1640 case vtn_variable_mode_ray_payload_in: 1641 case vtn_variable_mode_hit_attrib: 1642 return nir_address_format_logical; 1643 } 1644 1645 unreachable("Invalid variable mode"); 1646 } 1647 1648 nir_ssa_def * 1649 vtn_pointer_to_ssa(struct vtn_builder *b, struct vtn_pointer *ptr) 1650 { 1651 if ((vtn_pointer_is_external_block(b, ptr) && 1652 vtn_type_contains_block(b, ptr->type) && 1653 ptr->mode != vtn_variable_mode_phys_ssbo) || 1654 ptr->mode == vtn_variable_mode_accel_struct) { 1655 /* In this case, we're looking for a block index and not an actual 1656 * deref. 1657 * 1658 * For PhysicalStorageBuffer pointers, we don't have a block index 1659 * at all because we get the pointer directly from the client. This 1660 * assumes that there will never be a SSBO binding variable using the 1661 * PhysicalStorageBuffer storage class. This assumption appears 1662 * to be correct according to the Vulkan spec because the table, 1663 * "Shader Resource and Storage Class Correspondence," the only the 1664 * Uniform storage class with BufferBlock or the StorageBuffer 1665 * storage class with Block can be used. 1666 */ 1667 if (!ptr->block_index) { 1668 /* If we don't have a block_index then we must be a pointer to the 1669 * variable itself. 1670 */ 1671 vtn_assert(!ptr->deref); 1672 1673 struct vtn_access_chain chain = { 1674 .length = 0, 1675 }; 1676 ptr = vtn_pointer_dereference(b, ptr, &chain); 1677 } 1678 1679 return ptr->block_index; 1680 } else { 1681 return &vtn_pointer_to_deref(b, ptr)->dest.ssa; 1682 } 1683 } 1684 1685 struct vtn_pointer * 1686 vtn_pointer_from_ssa(struct vtn_builder *b, nir_ssa_def *ssa, 1687 struct vtn_type *ptr_type) 1688 { 1689 vtn_assert(ptr_type->base_type == vtn_base_type_pointer); 1690 1691 struct vtn_pointer *ptr = rzalloc(b, struct vtn_pointer); 1692 struct vtn_type *without_array = 1693 vtn_type_without_array(ptr_type->deref); 1694 1695 nir_variable_mode nir_mode; 1696 ptr->mode = vtn_storage_class_to_mode(b, ptr_type->storage_class, 1697 without_array, &nir_mode); 1698 ptr->type = ptr_type->deref; 1699 ptr->ptr_type = ptr_type; 1700 1701 const struct glsl_type *deref_type = 1702 vtn_type_get_nir_type(b, ptr_type->deref, ptr->mode); 1703 if (!vtn_pointer_is_external_block(b, ptr) && 1704 ptr->mode != vtn_variable_mode_accel_struct) { 1705 ptr->deref = nir_build_deref_cast(&b->nb, ssa, nir_mode, 1706 deref_type, ptr_type->stride); 1707 } else if ((vtn_type_contains_block(b, ptr->type) && 1708 ptr->mode != vtn_variable_mode_phys_ssbo) || 1709 ptr->mode == vtn_variable_mode_accel_struct) { 1710 /* This is a pointer to somewhere in an array of blocks, not a 1711 * pointer to somewhere inside the block. Set the block index 1712 * instead of making a cast. 1713 */ 1714 ptr->block_index = ssa; 1715 } else { 1716 /* This is a pointer to something internal or a pointer inside a 1717 * block. It's just a regular cast. 1718 * 1719 * For PhysicalStorageBuffer pointers, we don't have a block index 1720 * at all because we get the pointer directly from the client. This 1721 * assumes that there will never be a SSBO binding variable using the 1722 * PhysicalStorageBuffer storage class. This assumption appears 1723 * to be correct according to the Vulkan spec because the table, 1724 * "Shader Resource and Storage Class Correspondence," the only the 1725 * Uniform storage class with BufferBlock or the StorageBuffer 1726 * storage class with Block can be used. 1727 */ 1728 ptr->deref = nir_build_deref_cast(&b->nb, ssa, nir_mode, 1729 deref_type, ptr_type->stride); 1730 ptr->deref->dest.ssa.num_components = 1731 glsl_get_vector_elements(ptr_type->type); 1732 ptr->deref->dest.ssa.bit_size = glsl_get_bit_size(ptr_type->type); 1733 } 1734 1735 return ptr; 1736 } 1737 1738 static void 1739 assign_missing_member_locations(struct vtn_variable *var) 1740 { 1741 unsigned length = 1742 glsl_get_length(glsl_without_array(var->type->type)); 1743 int location = var->base_location; 1744 1745 for (unsigned i = 0; i < length; i++) { 1746 /* From the Vulkan spec: 1747 * 1748 * If the structure type is a Block but without a Location, then each 1749 * of its members must have a Location decoration. 1750 * 1751 */ 1752 if (var->type->block) { 1753 assert(var->base_location != -1 || 1754 var->var->members[i].location != -1); 1755 } 1756 1757 /* From the Vulkan spec: 1758 * 1759 * Any member with its own Location decoration is assigned that 1760 * location. Each remaining member is assigned the location after the 1761 * immediately preceding member in declaration order. 1762 */ 1763 if (var->var->members[i].location != -1) 1764 location = var->var->members[i].location; 1765 else 1766 var->var->members[i].location = location; 1767 1768 /* Below we use type instead of interface_type, because interface_type 1769 * is only available when it is a Block. This code also supports 1770 * input/outputs that are just structs 1771 */ 1772 const struct glsl_type *member_type = 1773 glsl_get_struct_field(glsl_without_array(var->type->type), i); 1774 1775 location += 1776 glsl_count_attribute_slots(member_type, 1777 false /* is_gl_vertex_input */); 1778 } 1779 } 1780 1781 nir_deref_instr * 1782 vtn_get_call_payload_for_location(struct vtn_builder *b, uint32_t location_id) 1783 { 1784 uint32_t location = vtn_constant_uint(b, location_id); 1785 nir_foreach_variable_with_modes(var, b->nb.shader, nir_var_shader_temp) { 1786 if (var->data.explicit_location && 1787 var->data.location == location) 1788 return nir_build_deref_var(&b->nb, var); 1789 } 1790 vtn_fail("Couldn't find variable with a storage class of CallableDataKHR " 1791 "or RayPayloadKHR and location %d", location); 1792 } 1793 1794 static void 1795 vtn_create_variable(struct vtn_builder *b, struct vtn_value *val, 1796 struct vtn_type *ptr_type, SpvStorageClass storage_class, 1797 struct vtn_value *initializer) 1798 { 1799 vtn_assert(ptr_type->base_type == vtn_base_type_pointer); 1800 struct vtn_type *type = ptr_type->deref; 1801 1802 struct vtn_type *without_array = vtn_type_without_array(ptr_type->deref); 1803 1804 enum vtn_variable_mode mode; 1805 nir_variable_mode nir_mode; 1806 mode = vtn_storage_class_to_mode(b, storage_class, without_array, &nir_mode); 1807 1808 switch (mode) { 1809 case vtn_variable_mode_ubo: 1810 /* There's no other way to get vtn_variable_mode_ubo */ 1811 vtn_assert(without_array->block); 1812 break; 1813 case vtn_variable_mode_ssbo: 1814 if (storage_class == SpvStorageClassStorageBuffer && 1815 !without_array->block) { 1816 if (b->variable_pointers) { 1817 vtn_fail("Variables in the StorageBuffer storage class must " 1818 "have a struct type with the Block decoration"); 1819 } else { 1820 /* If variable pointers are not present, it's still malformed 1821 * SPIR-V but we can parse it and do the right thing anyway. 1822 * Since some of the 8-bit storage tests have bugs in this are, 1823 * just make it a warning for now. 1824 */ 1825 vtn_warn("Variables in the StorageBuffer storage class must " 1826 "have a struct type with the Block decoration"); 1827 } 1828 } 1829 break; 1830 1831 case vtn_variable_mode_generic: 1832 vtn_fail("Cannot create a variable with the Generic storage class"); 1833 break; 1834 1835 case vtn_variable_mode_image: 1836 vtn_fail("Cannot create a variable with the Image storage class"); 1837 break; 1838 1839 case vtn_variable_mode_phys_ssbo: 1840 vtn_fail("Cannot create a variable with the " 1841 "PhysicalStorageBuffer storage class"); 1842 break; 1843 1844 default: 1845 /* No tallying is needed */ 1846 break; 1847 } 1848 1849 struct vtn_variable *var = rzalloc(b, struct vtn_variable); 1850 var->type = type; 1851 var->mode = mode; 1852 var->base_location = -1; 1853 1854 val->pointer = rzalloc(b, struct vtn_pointer); 1855 val->pointer->mode = var->mode; 1856 val->pointer->type = var->type; 1857 val->pointer->ptr_type = ptr_type; 1858 val->pointer->var = var; 1859 val->pointer->access = var->type->access; 1860 1861 switch (var->mode) { 1862 case vtn_variable_mode_function: 1863 case vtn_variable_mode_private: 1864 case vtn_variable_mode_uniform: 1865 case vtn_variable_mode_atomic_counter: 1866 case vtn_variable_mode_constant: 1867 case vtn_variable_mode_call_data: 1868 case vtn_variable_mode_call_data_in: 1869 case vtn_variable_mode_ray_payload: 1870 case vtn_variable_mode_ray_payload_in: 1871 case vtn_variable_mode_hit_attrib: 1872 /* For these, we create the variable normally */ 1873 var->var = rzalloc(b->shader, nir_variable); 1874 var->var->name = ralloc_strdup(var->var, val->name); 1875 var->var->type = vtn_type_get_nir_type(b, var->type, var->mode); 1876 1877 /* This is a total hack but we need some way to flag variables which are 1878 * going to be call payloads. See get_call_payload_deref. 1879 */ 1880 if (storage_class == SpvStorageClassCallableDataKHR || 1881 storage_class == SpvStorageClassRayPayloadKHR) 1882 var->var->data.explicit_location = true; 1883 1884 var->var->data.mode = nir_mode; 1885 var->var->data.location = -1; 1886 var->var->interface_type = NULL; 1887 break; 1888 1889 case vtn_variable_mode_ubo: 1890 case vtn_variable_mode_ssbo: 1891 case vtn_variable_mode_push_constant: 1892 case vtn_variable_mode_accel_struct: 1893 case vtn_variable_mode_shader_record: 1894 var->var = rzalloc(b->shader, nir_variable); 1895 var->var->name = ralloc_strdup(var->var, val->name); 1896 1897 var->var->type = vtn_type_get_nir_type(b, var->type, var->mode); 1898 var->var->interface_type = var->var->type; 1899 1900 var->var->data.mode = nir_mode; 1901 var->var->data.location = -1; 1902 var->var->data.driver_location = 0; 1903 var->var->data.access = var->type->access; 1904 break; 1905 1906 case vtn_variable_mode_workgroup: 1907 case vtn_variable_mode_cross_workgroup: 1908 /* Create the variable normally */ 1909 var->var = rzalloc(b->shader, nir_variable); 1910 var->var->name = ralloc_strdup(var->var, val->name); 1911 var->var->type = vtn_type_get_nir_type(b, var->type, var->mode); 1912 var->var->data.mode = nir_mode; 1913 break; 1914 1915 case vtn_variable_mode_input: 1916 case vtn_variable_mode_output: { 1917 var->var = rzalloc(b->shader, nir_variable); 1918 var->var->name = ralloc_strdup(var->var, val->name); 1919 var->var->type = vtn_type_get_nir_type(b, var->type, var->mode); 1920 var->var->data.mode = nir_mode; 1921 1922 /* In order to know whether or not we're a per-vertex inout, we need 1923 * the patch qualifier. This means walking the variable decorations 1924 * early before we actually create any variables. Not a big deal. 1925 * 1926 * GLSLang really likes to place decorations in the most interior 1927 * thing it possibly can. In particular, if you have a struct, it 1928 * will place the patch decorations on the struct members. This 1929 * should be handled by the variable splitting below just fine. 1930 * 1931 * If you have an array-of-struct, things get even more weird as it 1932 * will place the patch decorations on the struct even though it's 1933 * inside an array and some of the members being patch and others not 1934 * makes no sense whatsoever. Since the only sensible thing is for 1935 * it to be all or nothing, we'll call it patch if any of the members 1936 * are declared patch. 1937 */ 1938 vtn_foreach_decoration(b, val, gather_var_kind_cb, var); 1939 if (glsl_type_is_array(var->type->type) && 1940 glsl_type_is_struct_or_ifc(without_array->type)) { 1941 vtn_foreach_decoration(b, vtn_value(b, without_array->id, 1942 vtn_value_type_type), 1943 gather_var_kind_cb, var); 1944 } 1945 1946 struct vtn_type *per_vertex_type = var->type; 1947 if (nir_is_arrayed_io(var->var, b->shader->info.stage)) 1948 per_vertex_type = var->type->array_element; 1949 1950 /* Figure out the interface block type. */ 1951 struct vtn_type *iface_type = per_vertex_type; 1952 if (var->mode == vtn_variable_mode_output && 1953 (b->shader->info.stage == MESA_SHADER_VERTEX || 1954 b->shader->info.stage == MESA_SHADER_TESS_EVAL || 1955 b->shader->info.stage == MESA_SHADER_GEOMETRY)) { 1956 /* For vertex data outputs, we can end up with arrays of blocks for 1957 * transform feedback where each array element corresponds to a 1958 * different XFB output buffer. 1959 */ 1960 while (iface_type->base_type == vtn_base_type_array) 1961 iface_type = iface_type->array_element; 1962 } 1963 if (iface_type->base_type == vtn_base_type_struct && iface_type->block) 1964 var->var->interface_type = vtn_type_get_nir_type(b, iface_type, 1965 var->mode); 1966 1967 /* If it's a block, set it up as per-member so can be splitted later by 1968 * nir_split_per_member_structs. 1969 * 1970 * This is for a couple of reasons. For one, builtins may all come in a 1971 * block and we really want those split out into separate variables. 1972 * For another, interpolation qualifiers can be applied to members of 1973 * the top-level struct and we need to be able to preserve that 1974 * information. 1975 */ 1976 if (per_vertex_type->base_type == vtn_base_type_struct && 1977 per_vertex_type->block) { 1978 var->var->num_members = glsl_get_length(per_vertex_type->type); 1979 var->var->members = rzalloc_array(var->var, struct nir_variable_data, 1980 var->var->num_members); 1981 1982 for (unsigned i = 0; i < var->var->num_members; i++) { 1983 var->var->members[i].mode = nir_mode; 1984 var->var->members[i].patch = var->var->data.patch; 1985 var->var->members[i].location = -1; 1986 } 1987 } 1988 1989 /* For inputs and outputs, we need to grab locations and builtin 1990 * information from the per-vertex type. 1991 */ 1992 vtn_foreach_decoration(b, vtn_value(b, per_vertex_type->id, 1993 vtn_value_type_type), 1994 var_decoration_cb, var); 1995 1996 /* PerTask I/O is always a single block without any Location, so 1997 * initialize the base_location of the block and let 1998 * assign_missing_member_locations() do the rest. 1999 */ 2000 if ((b->shader->info.stage == MESA_SHADER_TASK && var->mode == vtn_variable_mode_output) || 2001 (b->shader->info.stage == MESA_SHADER_MESH && var->mode == vtn_variable_mode_input)) { 2002 if (var->type->block) 2003 var->base_location = VARYING_SLOT_VAR0; 2004 } 2005 2006 break; 2007 } 2008 2009 case vtn_variable_mode_image: 2010 case vtn_variable_mode_phys_ssbo: 2011 case vtn_variable_mode_generic: 2012 unreachable("Should have been caught before"); 2013 } 2014 2015 /* Ignore incorrectly generated Undef initializers. */ 2016 if (b->wa_llvm_spirv_ignore_workgroup_initializer && 2017 initializer && 2018 storage_class == SpvStorageClassWorkgroup) 2019 initializer = NULL; 2020 2021 /* Only initialize variable when there is an initializer and it's not 2022 * undef. 2023 */ 2024 if (initializer && !initializer->is_undef_constant) { 2025 switch (storage_class) { 2026 case SpvStorageClassWorkgroup: 2027 /* VK_KHR_zero_initialize_workgroup_memory. */ 2028 vtn_fail_if(b->options->environment != NIR_SPIRV_VULKAN, 2029 "Only Vulkan supports variable initializer " 2030 "for Workgroup variable %u", 2031 vtn_id_for_value(b, val)); 2032 vtn_fail_if(initializer->value_type != vtn_value_type_constant || 2033 !initializer->is_null_constant, 2034 "Workgroup variable %u can only have OpConstantNull " 2035 "as initializer, but have %u instead", 2036 vtn_id_for_value(b, val), 2037 vtn_id_for_value(b, initializer)); 2038 b->shader->info.zero_initialize_shared_memory = true; 2039 break; 2040 2041 case SpvStorageClassUniformConstant: 2042 vtn_fail_if(b->options->environment != NIR_SPIRV_OPENGL && 2043 b->options->environment != NIR_SPIRV_OPENCL, 2044 "Only OpenGL and OpenCL support variable initializer " 2045 "for UniformConstant variable %u\n", 2046 vtn_id_for_value(b, val)); 2047 vtn_fail_if(initializer->value_type != vtn_value_type_constant, 2048 "UniformConstant variable %u can only have a constant " 2049 "initializer, but have %u instead", 2050 vtn_id_for_value(b, val), 2051 vtn_id_for_value(b, initializer)); 2052 break; 2053 2054 case SpvStorageClassOutput: 2055 case SpvStorageClassPrivate: 2056 vtn_assert(b->options->environment != NIR_SPIRV_OPENCL); 2057 /* These can have any initializer. */ 2058 break; 2059 2060 case SpvStorageClassFunction: 2061 /* These can have any initializer. */ 2062 break; 2063 2064 case SpvStorageClassCrossWorkgroup: 2065 vtn_assert(b->options->environment == NIR_SPIRV_OPENCL); 2066 vtn_fail("Initializer for CrossWorkgroup variable %u " 2067 "not yet supported in Mesa.", 2068 vtn_id_for_value(b, val)); 2069 break; 2070 2071 default: { 2072 const enum nir_spirv_execution_environment env = 2073 b->options->environment; 2074 const char *env_name = 2075 env == NIR_SPIRV_VULKAN ? "Vulkan" : 2076 env == NIR_SPIRV_OPENCL ? "OpenCL" : 2077 env == NIR_SPIRV_OPENGL ? "OpenGL" : 2078 NULL; 2079 vtn_assert(env_name); 2080 vtn_fail("In %s, any OpVariable with an Initializer operand " 2081 "must have %s%s%s, or Function as " 2082 "its Storage Class operand. Variable %u has an " 2083 "Initializer but its Storage Class is %s.", 2084 env_name, 2085 env == NIR_SPIRV_VULKAN ? "Private, Output, Workgroup" : "", 2086 env == NIR_SPIRV_OPENCL ? "CrossWorkgroup, UniformConstant" : "", 2087 env == NIR_SPIRV_OPENGL ? "Private, Output, UniformConstant" : "", 2088 vtn_id_for_value(b, val), 2089 spirv_storageclass_to_string(storage_class)); 2090 } 2091 } 2092 2093 switch (initializer->value_type) { 2094 case vtn_value_type_constant: 2095 var->var->constant_initializer = 2096 nir_constant_clone(initializer->constant, var->var); 2097 break; 2098 case vtn_value_type_pointer: 2099 var->var->pointer_initializer = initializer->pointer->var->var; 2100 break; 2101 default: 2102 vtn_fail("SPIR-V variable initializer %u must be constant or pointer", 2103 vtn_id_for_value(b, initializer)); 2104 } 2105 } 2106 2107 if (var->mode == vtn_variable_mode_uniform || 2108 var->mode == vtn_variable_mode_ssbo) { 2109 /* SSBOs and images are assumed to not alias in the Simple, GLSL and Vulkan memory models */ 2110 var->var->data.access |= b->mem_model != SpvMemoryModelOpenCL ? ACCESS_RESTRICT : 0; 2111 } 2112 2113 vtn_foreach_decoration(b, val, var_decoration_cb, var); 2114 vtn_foreach_decoration(b, val, ptr_decoration_cb, val->pointer); 2115 2116 /* Propagate access flags from the OpVariable decorations. */ 2117 val->pointer->access |= var->access; 2118 2119 if ((var->mode == vtn_variable_mode_input || 2120 var->mode == vtn_variable_mode_output) && 2121 var->var->members) { 2122 assign_missing_member_locations(var); 2123 } 2124 2125 if (var->mode == vtn_variable_mode_uniform || 2126 var->mode == vtn_variable_mode_ubo || 2127 var->mode == vtn_variable_mode_ssbo || 2128 var->mode == vtn_variable_mode_atomic_counter) { 2129 /* XXX: We still need the binding information in the nir_variable 2130 * for these. We should fix that. 2131 */ 2132 var->var->data.binding = var->binding; 2133 var->var->data.explicit_binding = var->explicit_binding; 2134 var->var->data.descriptor_set = var->descriptor_set; 2135 var->var->data.index = var->input_attachment_index; 2136 var->var->data.offset = var->offset; 2137 2138 if (glsl_type_is_image(glsl_without_array(var->var->type))) 2139 var->var->data.image.format = without_array->image_format; 2140 } 2141 2142 if (var->mode == vtn_variable_mode_function) { 2143 vtn_assert(var->var != NULL && var->var->members == NULL); 2144 nir_function_impl_add_variable(b->nb.impl, var->var); 2145 } else if (var->var) { 2146 nir_shader_add_variable(b->shader, var->var); 2147 } else { 2148 vtn_assert(vtn_pointer_is_external_block(b, val->pointer) || 2149 var->mode == vtn_variable_mode_accel_struct || 2150 var->mode == vtn_variable_mode_shader_record); 2151 } 2152 } 2153 2154 static void 2155 vtn_assert_types_equal(struct vtn_builder *b, SpvOp opcode, 2156 struct vtn_type *dst_type, 2157 struct vtn_type *src_type) 2158 { 2159 if (dst_type->id == src_type->id) 2160 return; 2161 2162 if (vtn_types_compatible(b, dst_type, src_type)) { 2163 /* Early versions of GLSLang would re-emit types unnecessarily and you 2164 * would end up with OpLoad, OpStore, or OpCopyMemory opcodes which have 2165 * mismatched source and destination types. 2166 * 2167 * https://github.com/KhronosGroup/glslang/issues/304 2168 * https://github.com/KhronosGroup/glslang/issues/307 2169 * https://bugs.freedesktop.org/show_bug.cgi?id=104338 2170 * https://bugs.freedesktop.org/show_bug.cgi?id=104424 2171 */ 2172 vtn_warn("Source and destination types of %s do not have the same " 2173 "ID (but are compatible): %u vs %u", 2174 spirv_op_to_string(opcode), dst_type->id, src_type->id); 2175 return; 2176 } 2177 2178 vtn_fail("Source and destination types of %s do not match: %s vs. %s", 2179 spirv_op_to_string(opcode), 2180 glsl_get_type_name(dst_type->type), 2181 glsl_get_type_name(src_type->type)); 2182 } 2183 2184 static nir_ssa_def * 2185 nir_shrink_zero_pad_vec(nir_builder *b, nir_ssa_def *val, 2186 unsigned num_components) 2187 { 2188 if (val->num_components == num_components) 2189 return val; 2190 2191 nir_ssa_def *comps[NIR_MAX_VEC_COMPONENTS]; 2192 for (unsigned i = 0; i < num_components; i++) { 2193 if (i < val->num_components) 2194 comps[i] = nir_channel(b, val, i); 2195 else 2196 comps[i] = nir_imm_intN_t(b, 0, val->bit_size); 2197 } 2198 return nir_vec(b, comps, num_components); 2199 } 2200 2201 static nir_ssa_def * 2202 nir_sloppy_bitcast(nir_builder *b, nir_ssa_def *val, 2203 const struct glsl_type *type) 2204 { 2205 const unsigned num_components = glsl_get_vector_elements(type); 2206 const unsigned bit_size = glsl_get_bit_size(type); 2207 2208 /* First, zero-pad to ensure that the value is big enough that when we 2209 * bit-cast it, we don't loose anything. 2210 */ 2211 if (val->bit_size < bit_size) { 2212 const unsigned src_num_components_needed = 2213 vtn_align_u32(val->num_components, bit_size / val->bit_size); 2214 val = nir_shrink_zero_pad_vec(b, val, src_num_components_needed); 2215 } 2216 2217 val = nir_bitcast_vector(b, val, bit_size); 2218 2219 return nir_shrink_zero_pad_vec(b, val, num_components); 2220 } 2221 2222 static bool 2223 vtn_get_mem_operands(struct vtn_builder *b, const uint32_t *w, unsigned count, 2224 unsigned *idx, SpvMemoryAccessMask *access, unsigned *alignment, 2225 SpvScope *dest_scope, SpvScope *src_scope) 2226 { 2227 *access = 0; 2228 *alignment = 0; 2229 if (*idx >= count) 2230 return false; 2231 2232 *access = w[(*idx)++]; 2233 if (*access & SpvMemoryAccessAlignedMask) { 2234 vtn_assert(*idx < count); 2235 *alignment = w[(*idx)++]; 2236 } 2237 2238 if (*access & SpvMemoryAccessMakePointerAvailableMask) { 2239 vtn_assert(*idx < count); 2240 vtn_assert(dest_scope); 2241 *dest_scope = vtn_constant_uint(b, w[(*idx)++]); 2242 } 2243 2244 if (*access & SpvMemoryAccessMakePointerVisibleMask) { 2245 vtn_assert(*idx < count); 2246 vtn_assert(src_scope); 2247 *src_scope = vtn_constant_uint(b, w[(*idx)++]); 2248 } 2249 2250 return true; 2251 } 2252 2253 static enum gl_access_qualifier 2254 spv_access_to_gl_access(SpvMemoryAccessMask access) 2255 { 2256 unsigned result = 0; 2257 2258 if (access & SpvMemoryAccessVolatileMask) 2259 result |= ACCESS_VOLATILE; 2260 if (access & SpvMemoryAccessNontemporalMask) 2261 result |= ACCESS_STREAM_CACHE_POLICY; 2262 2263 return result; 2264 } 2265 2266 2267 SpvMemorySemanticsMask 2268 vtn_mode_to_memory_semantics(enum vtn_variable_mode mode) 2269 { 2270 switch (mode) { 2271 case vtn_variable_mode_ssbo: 2272 case vtn_variable_mode_phys_ssbo: 2273 return SpvMemorySemanticsUniformMemoryMask; 2274 case vtn_variable_mode_workgroup: 2275 return SpvMemorySemanticsWorkgroupMemoryMask; 2276 case vtn_variable_mode_cross_workgroup: 2277 return SpvMemorySemanticsCrossWorkgroupMemoryMask; 2278 case vtn_variable_mode_atomic_counter: 2279 return SpvMemorySemanticsAtomicCounterMemoryMask; 2280 case vtn_variable_mode_image: 2281 return SpvMemorySemanticsImageMemoryMask; 2282 case vtn_variable_mode_output: 2283 return SpvMemorySemanticsOutputMemoryMask; 2284 default: 2285 return SpvMemorySemanticsMaskNone; 2286 } 2287 } 2288 2289 static void 2290 vtn_emit_make_visible_barrier(struct vtn_builder *b, SpvMemoryAccessMask access, 2291 SpvScope scope, enum vtn_variable_mode mode) 2292 { 2293 if (!(access & SpvMemoryAccessMakePointerVisibleMask)) 2294 return; 2295 2296 vtn_emit_memory_barrier(b, scope, SpvMemorySemanticsMakeVisibleMask | 2297 SpvMemorySemanticsAcquireMask | 2298 vtn_mode_to_memory_semantics(mode)); 2299 } 2300 2301 static void 2302 vtn_emit_make_available_barrier(struct vtn_builder *b, SpvMemoryAccessMask access, 2303 SpvScope scope, enum vtn_variable_mode mode) 2304 { 2305 if (!(access & SpvMemoryAccessMakePointerAvailableMask)) 2306 return; 2307 2308 vtn_emit_memory_barrier(b, scope, SpvMemorySemanticsMakeAvailableMask | 2309 SpvMemorySemanticsReleaseMask | 2310 vtn_mode_to_memory_semantics(mode)); 2311 } 2312 2313 static void 2314 ptr_nonuniform_workaround_cb(struct vtn_builder *b, struct vtn_value *val, 2315 int member, const struct vtn_decoration *dec, void *void_ptr) 2316 { 2317 enum gl_access_qualifier *access = void_ptr; 2318 2319 switch (dec->decoration) { 2320 case SpvDecorationNonUniformEXT: 2321 *access |= ACCESS_NON_UNIFORM; 2322 break; 2323 2324 default: 2325 break; 2326 } 2327 } 2328 2329 void 2330 vtn_handle_variables(struct vtn_builder *b, SpvOp opcode, 2331 const uint32_t *w, unsigned count) 2332 { 2333 switch (opcode) { 2334 case SpvOpUndef: { 2335 struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_undef); 2336 val->type = vtn_get_type(b, w[1]); 2337 val->is_undef_constant = true; 2338 break; 2339 } 2340 2341 case SpvOpVariable: { 2342 struct vtn_type *ptr_type = vtn_get_type(b, w[1]); 2343 2344 SpvStorageClass storage_class = w[3]; 2345 2346 const bool is_global = storage_class != SpvStorageClassFunction; 2347 const bool is_io = storage_class == SpvStorageClassInput || 2348 storage_class == SpvStorageClassOutput; 2349 2350 /* Skip global variables that are not used by the entrypoint. Before 2351 * SPIR-V 1.4 the interface is only used for I/O variables, so extra 2352 * variables will still need to be removed later. 2353 */ 2354 if (!b->options->create_library && 2355 (is_io || (b->version >= 0x10400 && is_global))) { 2356 if (!bsearch(&w[2], b->interface_ids, b->interface_ids_count, 4, cmp_uint32_t)) 2357 break; 2358 } 2359 2360 struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_pointer); 2361 struct vtn_value *initializer = count > 4 ? vtn_untyped_value(b, w[4]) : NULL; 2362 2363 vtn_create_variable(b, val, ptr_type, storage_class, initializer); 2364 2365 break; 2366 } 2367 2368 case SpvOpConstantSampler: { 2369 /* Synthesize a pointer-to-sampler type, create a variable of that type, 2370 * and give the variable a constant initializer with the sampler params */ 2371 struct vtn_type *sampler_type = vtn_value(b, w[1], vtn_value_type_type)->type; 2372 struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_pointer); 2373 2374 struct vtn_type *ptr_type = rzalloc(b, struct vtn_type); 2375 ptr_type = rzalloc(b, struct vtn_type); 2376 ptr_type->base_type = vtn_base_type_pointer; 2377 ptr_type->deref = sampler_type; 2378 ptr_type->storage_class = SpvStorageClassUniform; 2379 2380 ptr_type->type = nir_address_format_to_glsl_type( 2381 vtn_mode_to_address_format(b, vtn_variable_mode_function)); 2382 2383 vtn_create_variable(b, val, ptr_type, ptr_type->storage_class, NULL); 2384 2385 nir_variable *nir_var = val->pointer->var->var; 2386 nir_var->data.sampler.is_inline_sampler = true; 2387 nir_var->data.sampler.addressing_mode = w[3]; 2388 nir_var->data.sampler.normalized_coordinates = w[4]; 2389 nir_var->data.sampler.filter_mode = w[5]; 2390 2391 break; 2392 } 2393 2394 case SpvOpAccessChain: 2395 case SpvOpPtrAccessChain: 2396 case SpvOpInBoundsAccessChain: 2397 case SpvOpInBoundsPtrAccessChain: { 2398 struct vtn_access_chain *chain = vtn_access_chain_create(b, count - 4); 2399 enum gl_access_qualifier access = 0; 2400 chain->ptr_as_array = (opcode == SpvOpPtrAccessChain || opcode == SpvOpInBoundsPtrAccessChain); 2401 2402 unsigned idx = 0; 2403 for (int i = 4; i < count; i++) { 2404 struct vtn_value *link_val = vtn_untyped_value(b, w[i]); 2405 if (link_val->value_type == vtn_value_type_constant) { 2406 chain->link[idx].mode = vtn_access_mode_literal; 2407 chain->link[idx].id = vtn_constant_int(b, w[i]); 2408 } else { 2409 chain->link[idx].mode = vtn_access_mode_id; 2410 chain->link[idx].id = w[i]; 2411 } 2412 2413 /* Workaround for https://gitlab.freedesktop.org/mesa/mesa/-/issues/3406 */ 2414 vtn_foreach_decoration(b, link_val, ptr_nonuniform_workaround_cb, &access); 2415 2416 idx++; 2417 } 2418 2419 struct vtn_type *ptr_type = vtn_get_type(b, w[1]); 2420 2421 struct vtn_pointer *base = vtn_pointer(b, w[3]); 2422 2423 /* Workaround for https://gitlab.freedesktop.org/mesa/mesa/-/issues/3406 */ 2424 access |= base->access & ACCESS_NON_UNIFORM; 2425 2426 struct vtn_pointer *ptr = vtn_pointer_dereference(b, base, chain); 2427 ptr->ptr_type = ptr_type; 2428 ptr->access |= access; 2429 vtn_push_pointer(b, w[2], ptr); 2430 break; 2431 } 2432 2433 case SpvOpCopyMemory: { 2434 struct vtn_value *dest_val = vtn_pointer_value(b, w[1]); 2435 struct vtn_value *src_val = vtn_pointer_value(b, w[2]); 2436 struct vtn_pointer *dest = vtn_value_to_pointer(b, dest_val); 2437 struct vtn_pointer *src = vtn_value_to_pointer(b, src_val); 2438 2439 vtn_assert_types_equal(b, opcode, dest_val->type->deref, 2440 src_val->type->deref); 2441 2442 unsigned idx = 3, dest_alignment, src_alignment; 2443 SpvMemoryAccessMask dest_access, src_access; 2444 SpvScope dest_scope, src_scope; 2445 vtn_get_mem_operands(b, w, count, &idx, &dest_access, &dest_alignment, 2446 &dest_scope, &src_scope); 2447 if (!vtn_get_mem_operands(b, w, count, &idx, &src_access, &src_alignment, 2448 NULL, &src_scope)) { 2449 src_alignment = dest_alignment; 2450 src_access = dest_access; 2451 } 2452 src = vtn_align_pointer(b, src, src_alignment); 2453 dest = vtn_align_pointer(b, dest, dest_alignment); 2454 2455 vtn_emit_make_visible_barrier(b, src_access, src_scope, src->mode); 2456 2457 vtn_variable_copy(b, dest, src, 2458 spv_access_to_gl_access(dest_access), 2459 spv_access_to_gl_access(src_access)); 2460 2461 vtn_emit_make_available_barrier(b, dest_access, dest_scope, dest->mode); 2462 break; 2463 } 2464 2465 case SpvOpCopyMemorySized: { 2466 struct vtn_value *dest_val = vtn_pointer_value(b, w[1]); 2467 struct vtn_value *src_val = vtn_pointer_value(b, w[2]); 2468 nir_ssa_def *size = vtn_get_nir_ssa(b, w[3]); 2469 struct vtn_pointer *dest = vtn_value_to_pointer(b, dest_val); 2470 struct vtn_pointer *src = vtn_value_to_pointer(b, src_val); 2471 2472 unsigned idx = 4, dest_alignment, src_alignment; 2473 SpvMemoryAccessMask dest_access, src_access; 2474 SpvScope dest_scope, src_scope; 2475 vtn_get_mem_operands(b, w, count, &idx, &dest_access, &dest_alignment, 2476 &dest_scope, &src_scope); 2477 if (!vtn_get_mem_operands(b, w, count, &idx, &src_access, &src_alignment, 2478 NULL, &src_scope)) { 2479 src_alignment = dest_alignment; 2480 src_access = dest_access; 2481 } 2482 src = vtn_align_pointer(b, src, src_alignment); 2483 dest = vtn_align_pointer(b, dest, dest_alignment); 2484 2485 vtn_emit_make_visible_barrier(b, src_access, src_scope, src->mode); 2486 2487 nir_memcpy_deref_with_access(&b->nb, 2488 vtn_pointer_to_deref(b, dest), 2489 vtn_pointer_to_deref(b, src), 2490 size, 2491 spv_access_to_gl_access(dest_access), 2492 spv_access_to_gl_access(src_access)); 2493 2494 vtn_emit_make_available_barrier(b, dest_access, dest_scope, dest->mode); 2495 break; 2496 } 2497 2498 case SpvOpLoad: { 2499 struct vtn_type *res_type = vtn_get_type(b, w[1]); 2500 struct vtn_value *src_val = vtn_value(b, w[3], vtn_value_type_pointer); 2501 struct vtn_pointer *src = vtn_value_to_pointer(b, src_val); 2502 2503 vtn_assert_types_equal(b, opcode, res_type, src_val->type->deref); 2504 2505 unsigned idx = 4, alignment; 2506 SpvMemoryAccessMask access; 2507 SpvScope scope; 2508 vtn_get_mem_operands(b, w, count, &idx, &access, &alignment, NULL, &scope); 2509 src = vtn_align_pointer(b, src, alignment); 2510 2511 vtn_emit_make_visible_barrier(b, access, scope, src->mode); 2512 2513 vtn_push_ssa_value(b, w[2], vtn_variable_load(b, src, spv_access_to_gl_access(access))); 2514 break; 2515 } 2516 2517 case SpvOpStore: { 2518 struct vtn_value *dest_val = vtn_pointer_value(b, w[1]); 2519 struct vtn_pointer *dest = vtn_value_to_pointer(b, dest_val); 2520 struct vtn_value *src_val = vtn_untyped_value(b, w[2]); 2521 2522 /* OpStore requires us to actually have a storage type */ 2523 vtn_fail_if(dest->type->type == NULL, 2524 "Invalid destination type for OpStore"); 2525 2526 if (glsl_get_base_type(dest->type->type) == GLSL_TYPE_BOOL && 2527 glsl_get_base_type(src_val->type->type) == GLSL_TYPE_UINT) { 2528 /* Early versions of GLSLang would use uint types for UBOs/SSBOs but 2529 * would then store them to a local variable as bool. Work around 2530 * the issue by doing an implicit conversion. 2531 * 2532 * https://github.com/KhronosGroup/glslang/issues/170 2533 * https://bugs.freedesktop.org/show_bug.cgi?id=104424 2534 */ 2535 vtn_warn("OpStore of value of type OpTypeInt to a pointer to type " 2536 "OpTypeBool. Doing an implicit conversion to work around " 2537 "the problem."); 2538 struct vtn_ssa_value *bool_ssa = 2539 vtn_create_ssa_value(b, dest->type->type); 2540 bool_ssa->def = nir_i2b(&b->nb, vtn_ssa_value(b, w[2])->def); 2541 vtn_variable_store(b, bool_ssa, dest, 0); 2542 break; 2543 } 2544 2545 vtn_assert_types_equal(b, opcode, dest_val->type->deref, src_val->type); 2546 2547 unsigned idx = 3, alignment; 2548 SpvMemoryAccessMask access; 2549 SpvScope scope; 2550 vtn_get_mem_operands(b, w, count, &idx, &access, &alignment, &scope, NULL); 2551 dest = vtn_align_pointer(b, dest, alignment); 2552 2553 struct vtn_ssa_value *src = vtn_ssa_value(b, w[2]); 2554 vtn_variable_store(b, src, dest, spv_access_to_gl_access(access)); 2555 2556 vtn_emit_make_available_barrier(b, access, scope, dest->mode); 2557 break; 2558 } 2559 2560 case SpvOpArrayLength: { 2561 struct vtn_pointer *ptr = vtn_pointer(b, w[3]); 2562 const uint32_t field = w[4]; 2563 2564 vtn_fail_if(ptr->type->base_type != vtn_base_type_struct, 2565 "OpArrayLength must take a pointer to a structure type"); 2566 vtn_fail_if(field != ptr->type->length - 1 || 2567 ptr->type->members[field]->base_type != vtn_base_type_array, 2568 "OpArrayLength must reference the last memeber of the " 2569 "structure and that must be an array"); 2570 2571 if (b->options->use_deref_buffer_array_length) { 2572 struct vtn_access_chain chain = { 2573 .length = 1, 2574 .link = { 2575 { .mode = vtn_access_mode_literal, .id = field }, 2576 } 2577 }; 2578 struct vtn_pointer *array = vtn_pointer_dereference(b, ptr, &chain); 2579 2580 nir_ssa_def *array_length = 2581 nir_build_deref_buffer_array_length(&b->nb, 32, 2582 vtn_pointer_to_ssa(b, array), 2583 .access=ptr->access | ptr->type->access); 2584 2585 vtn_push_nir_ssa(b, w[2], array_length); 2586 } else { 2587 const uint32_t offset = ptr->type->offsets[field]; 2588 const uint32_t stride = ptr->type->members[field]->stride; 2589 2590 if (!ptr->block_index) { 2591 struct vtn_access_chain chain = { 2592 .length = 0, 2593 }; 2594 ptr = vtn_pointer_dereference(b, ptr, &chain); 2595 vtn_assert(ptr->block_index); 2596 } 2597 2598 nir_ssa_def *buf_size = nir_get_ssbo_size(&b->nb, ptr->block_index, 2599 .access=ptr->access | ptr->type->access); 2600 2601 /* array_length = max(buffer_size - offset, 0) / stride */ 2602 nir_ssa_def *array_length = 2603 nir_idiv(&b->nb, 2604 nir_imax(&b->nb, 2605 nir_isub(&b->nb, 2606 buf_size, 2607 nir_imm_int(&b->nb, offset)), 2608 nir_imm_int(&b->nb, 0u)), 2609 nir_imm_int(&b->nb, stride)); 2610 2611 vtn_push_nir_ssa(b, w[2], array_length); 2612 } 2613 break; 2614 } 2615 2616 case SpvOpConvertPtrToU: { 2617 struct vtn_type *u_type = vtn_get_type(b, w[1]); 2618 struct vtn_type *ptr_type = vtn_get_value_type(b, w[3]); 2619 2620 vtn_fail_if(ptr_type->base_type != vtn_base_type_pointer || 2621 ptr_type->type == NULL, 2622 "OpConvertPtrToU can only be used on physical pointers"); 2623 2624 vtn_fail_if(u_type->base_type != vtn_base_type_vector && 2625 u_type->base_type != vtn_base_type_scalar, 2626 "OpConvertPtrToU can only be used to cast to a vector or " 2627 "scalar type"); 2628 2629 /* The pointer will be converted to an SSA value automatically */ 2630 nir_ssa_def *ptr = vtn_get_nir_ssa(b, w[3]); 2631 nir_ssa_def *u = nir_sloppy_bitcast(&b->nb, ptr, u_type->type); 2632 vtn_push_nir_ssa(b, w[2], u); 2633 break; 2634 } 2635 2636 case SpvOpConvertUToPtr: { 2637 struct vtn_type *ptr_type = vtn_get_type(b, w[1]); 2638 struct vtn_type *u_type = vtn_get_value_type(b, w[3]); 2639 2640 vtn_fail_if(ptr_type->base_type != vtn_base_type_pointer || 2641 ptr_type->type == NULL, 2642 "OpConvertUToPtr can only be used on physical pointers"); 2643 2644 vtn_fail_if(u_type->base_type != vtn_base_type_vector && 2645 u_type->base_type != vtn_base_type_scalar, 2646 "OpConvertUToPtr can only be used to cast from a vector or " 2647 "scalar type"); 2648 2649 nir_ssa_def *u = vtn_get_nir_ssa(b, w[3]); 2650 nir_ssa_def *ptr = nir_sloppy_bitcast(&b->nb, u, ptr_type->type); 2651 vtn_push_pointer(b, w[2], vtn_pointer_from_ssa(b, ptr, ptr_type)); 2652 break; 2653 } 2654 2655 case SpvOpGenericCastToPtrExplicit: { 2656 struct vtn_type *dst_type = vtn_get_type(b, w[1]); 2657 struct vtn_type *src_type = vtn_get_value_type(b, w[3]); 2658 SpvStorageClass storage_class = w[4]; 2659 2660 vtn_fail_if(dst_type->base_type != vtn_base_type_pointer || 2661 dst_type->storage_class != storage_class, 2662 "Result type of an SpvOpGenericCastToPtrExplicit must be " 2663 "an OpTypePointer. Its Storage Class must match the " 2664 "storage class specified in the instruction"); 2665 2666 vtn_fail_if(src_type->base_type != vtn_base_type_pointer || 2667 src_type->deref->id != dst_type->deref->id, 2668 "Source pointer of an SpvOpGenericCastToPtrExplicit must " 2669 "have a type of OpTypePointer whose Type is the same as " 2670 "the Type of Result Type"); 2671 2672 vtn_fail_if(src_type->storage_class != SpvStorageClassGeneric, 2673 "Source pointer of an SpvOpGenericCastToPtrExplicit must " 2674 "point to the Generic Storage Class."); 2675 2676 vtn_fail_if(storage_class != SpvStorageClassWorkgroup && 2677 storage_class != SpvStorageClassCrossWorkgroup && 2678 storage_class != SpvStorageClassFunction, 2679 "Storage must be one of the following literal values from " 2680 "Storage Class: Workgroup, CrossWorkgroup, or Function."); 2681 2682 nir_deref_instr *src_deref = vtn_nir_deref(b, w[3]); 2683 2684 nir_variable_mode nir_mode; 2685 enum vtn_variable_mode mode = 2686 vtn_storage_class_to_mode(b, storage_class, dst_type->deref, &nir_mode); 2687 nir_address_format addr_format = vtn_mode_to_address_format(b, mode); 2688 2689 nir_ssa_def *null_value = 2690 nir_build_imm(&b->nb, nir_address_format_num_components(addr_format), 2691 nir_address_format_bit_size(addr_format), 2692 nir_address_format_null_value(addr_format)); 2693 2694 nir_ssa_def *valid = nir_build_deref_mode_is(&b->nb, 1, &src_deref->dest.ssa, nir_mode); 2695 vtn_push_nir_ssa(b, w[2], nir_bcsel(&b->nb, valid, 2696 &src_deref->dest.ssa, 2697 null_value)); 2698 break; 2699 } 2700 2701 case SpvOpGenericPtrMemSemantics: { 2702 struct vtn_type *dst_type = vtn_get_type(b, w[1]); 2703 struct vtn_type *src_type = vtn_get_value_type(b, w[3]); 2704 2705 vtn_fail_if(dst_type->base_type != vtn_base_type_scalar || 2706 dst_type->type != glsl_uint_type(), 2707 "Result type of an SpvOpGenericPtrMemSemantics must be " 2708 "an OpTypeInt with 32-bit Width and 0 Signedness."); 2709 2710 vtn_fail_if(src_type->base_type != vtn_base_type_pointer || 2711 src_type->storage_class != SpvStorageClassGeneric, 2712 "Source pointer of an SpvOpGenericPtrMemSemantics must " 2713 "point to the Generic Storage Class"); 2714 2715 nir_deref_instr *src_deref = vtn_nir_deref(b, w[3]); 2716 2717 nir_ssa_def *global_bit = 2718 nir_bcsel(&b->nb, nir_build_deref_mode_is(&b->nb, 1, &src_deref->dest.ssa, 2719 nir_var_mem_global), 2720 nir_imm_int(&b->nb, SpvMemorySemanticsCrossWorkgroupMemoryMask), 2721 nir_imm_int(&b->nb, 0)); 2722 2723 nir_ssa_def *shared_bit = 2724 nir_bcsel(&b->nb, nir_build_deref_mode_is(&b->nb, 1, &src_deref->dest.ssa, 2725 nir_var_mem_shared), 2726 nir_imm_int(&b->nb, SpvMemorySemanticsWorkgroupMemoryMask), 2727 nir_imm_int(&b->nb, 0)); 2728 2729 vtn_push_nir_ssa(b, w[2], nir_iand(&b->nb, global_bit, shared_bit)); 2730 break; 2731 } 2732 2733 case SpvOpSubgroupBlockReadINTEL: { 2734 struct vtn_type *res_type = vtn_get_type(b, w[1]); 2735 nir_deref_instr *src = vtn_nir_deref(b, w[3]); 2736 2737 nir_intrinsic_instr *load = 2738 nir_intrinsic_instr_create(b->nb.shader, 2739 nir_intrinsic_load_deref_block_intel); 2740 load->src[0] = nir_src_for_ssa(&src->dest.ssa); 2741 nir_ssa_dest_init_for_type(&load->instr, &load->dest, 2742 res_type->type, NULL); 2743 load->num_components = load->dest.ssa.num_components; 2744 nir_builder_instr_insert(&b->nb, &load->instr); 2745 2746 vtn_push_nir_ssa(b, w[2], &load->dest.ssa); 2747 break; 2748 } 2749 2750 case SpvOpSubgroupBlockWriteINTEL: { 2751 nir_deref_instr *dest = vtn_nir_deref(b, w[1]); 2752 nir_ssa_def *data = vtn_ssa_value(b, w[2])->def; 2753 2754 nir_intrinsic_instr *store = 2755 nir_intrinsic_instr_create(b->nb.shader, 2756 nir_intrinsic_store_deref_block_intel); 2757 store->src[0] = nir_src_for_ssa(&dest->dest.ssa); 2758 store->src[1] = nir_src_for_ssa(data); 2759 store->num_components = data->num_components; 2760 nir_builder_instr_insert(&b->nb, &store->instr); 2761 break; 2762 } 2763 2764 case SpvOpConvertUToAccelerationStructureKHR: { 2765 struct vtn_type *as_type = vtn_get_type(b, w[1]); 2766 struct vtn_type *u_type = vtn_get_value_type(b, w[3]); 2767 vtn_fail_if(!((u_type->base_type == vtn_base_type_vector && 2768 u_type->type == glsl_vector_type(GLSL_TYPE_UINT, 2)) || 2769 (u_type->base_type == vtn_base_type_scalar && 2770 u_type->type == glsl_uint64_t_type())), 2771 "OpConvertUToAccelerationStructure may only be used to " 2772 "cast from a 64-bit scalar integer or a 2-component vector " 2773 "of 32-bit integers"); 2774 vtn_fail_if(as_type->base_type != vtn_base_type_accel_struct, 2775 "The result type of an OpConvertUToAccelerationStructure " 2776 "must be OpTypeAccelerationStructure"); 2777 2778 nir_ssa_def *u = vtn_get_nir_ssa(b, w[3]); 2779 vtn_push_nir_ssa(b, w[2], nir_sloppy_bitcast(&b->nb, u, as_type->type)); 2780 break; 2781 } 2782 2783 default: 2784 vtn_fail_with_opcode("Unhandled opcode", opcode); 2785 } 2786 } 2787
Indexes created Sat Jul 25 00:23:55 UTC 2026