1/* 2 * Copyright © 2021 Valve Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Mike Blumenkrantz <michael.blumenkrantz@gmail.com> 25 */ 26#include "tgsi/tgsi_from_mesa.h" 27 28 29 30#include "zink_context.h" 31#include "zink_compiler.h" 32#include "zink_descriptors.h" 33#include "zink_program.h" 34#include "zink_resource.h" 35#include "zink_screen.h" 36 37#define MAX_LAZY_DESCRIPTORS (ZINK_DEFAULT_MAX_DESCS / 10) 38 39struct zink_descriptor_data_lazy { 40 struct zink_descriptor_data base; 41 VkDescriptorUpdateTemplateEntry push_entries[PIPE_SHADER_TYPES]; //gfx+fbfetch 42 VkDescriptorUpdateTemplateEntry compute_push_entry; 43 bool push_state_changed[2]; //gfx, compute 44 uint8_t state_changed[2]; //gfx, compute 45}; 46 47struct zink_descriptor_pool { 48 VkDescriptorPool pool; 49 VkDescriptorSet sets[MAX_LAZY_DESCRIPTORS]; 50 unsigned set_idx; 51 unsigned sets_alloc; 52}; 53 54struct zink_batch_descriptor_data_lazy { 55 struct zink_batch_descriptor_data base; 56 struct util_dynarray overflowed_pools; 57 struct hash_table pools[ZINK_DESCRIPTOR_TYPES]; 58 struct zink_descriptor_pool *push_pool[2]; 59 struct zink_program *pg[2]; //gfx, compute 60 uint32_t compat_id[2]; 61 VkDescriptorSetLayout dsl[2][ZINK_DESCRIPTOR_TYPES]; 62 VkDescriptorSet sets[2][ZINK_DESCRIPTOR_TYPES + 1]; 63 unsigned push_usage[2]; 64 bool has_fbfetch; 65}; 66 67ALWAYS_INLINE static struct zink_descriptor_data_lazy * 68dd_lazy(struct zink_context *ctx) 69{ 70 return (struct zink_descriptor_data_lazy*)ctx->dd; 71} 72 73ALWAYS_INLINE static struct zink_batch_descriptor_data_lazy * 74bdd_lazy(struct zink_batch_state *bs) 75{ 76 return (struct zink_batch_descriptor_data_lazy*)bs->dd; 77} 78 79static void 80init_template_entry(struct zink_shader *shader, enum zink_descriptor_type type, 81 unsigned idx, unsigned offset, VkDescriptorUpdateTemplateEntry *entry, unsigned *entry_idx, bool flatten_dynamic) 82{ 83 int index = shader->bindings[type][idx].index; 84 enum pipe_shader_type stage = pipe_shader_type_from_mesa(shader->nir->info.stage); 85 entry->dstArrayElement = 0; 86 entry->dstBinding = shader->bindings[type][idx].binding; 87 if (shader->bindings[type][idx].type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC && flatten_dynamic) 88 /* filter out DYNAMIC type here */ 89 entry->descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER; 90 else 91 entry->descriptorType = shader->bindings[type][idx].type; 92 switch (shader->bindings[type][idx].type) { 93 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER: 94 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC: 95 entry->descriptorCount = 1; 96 entry->offset = offsetof(struct zink_context, di.ubos[stage][index + offset]); 97 entry->stride = sizeof(VkDescriptorBufferInfo); 98 break; 99 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER: 100 entry->descriptorCount = shader->bindings[type][idx].size; 101 entry->offset = offsetof(struct zink_context, di.textures[stage][index + offset]); 102 entry->stride = sizeof(VkDescriptorImageInfo); 103 break; 104 case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER: 105 entry->descriptorCount = shader->bindings[type][idx].size; 106 entry->offset = offsetof(struct zink_context, di.tbos[stage][index + offset]); 107 entry->stride = sizeof(VkBufferView); 108 break; 109 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER: 110 entry->descriptorCount = 1; 111 entry->offset = offsetof(struct zink_context, di.ssbos[stage][index + offset]); 112 entry->stride = sizeof(VkDescriptorBufferInfo); 113 break; 114 case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE: 115 entry->descriptorCount = shader->bindings[type][idx].size; 116 entry->offset = offsetof(struct zink_context, di.images[stage][index + offset]); 117 entry->stride = sizeof(VkDescriptorImageInfo); 118 break; 119 case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER: 120 entry->descriptorCount = shader->bindings[type][idx].size; 121 entry->offset = offsetof(struct zink_context, di.texel_images[stage][index + offset]); 122 entry->stride = sizeof(VkBufferView); 123 break; 124 default: 125 unreachable("unknown type"); 126 } 127 (*entry_idx)++; 128} 129 130bool 131zink_descriptor_program_init_lazy(struct zink_context *ctx, struct zink_program *pg) 132{ 133 struct zink_screen *screen = zink_screen(ctx->base.screen); 134 VkDescriptorSetLayoutBinding bindings[ZINK_DESCRIPTOR_TYPES][PIPE_SHADER_TYPES * 32]; 135 VkDescriptorUpdateTemplateEntry entries[ZINK_DESCRIPTOR_TYPES][PIPE_SHADER_TYPES * 32]; 136 unsigned num_bindings[ZINK_DESCRIPTOR_TYPES] = {0}; 137 uint8_t has_bindings = 0; 138 unsigned push_count = 0; 139 140 struct zink_shader **stages; 141 if (pg->is_compute) 142 stages = &((struct zink_compute_program*)pg)->shader; 143 else 144 stages = ((struct zink_gfx_program*)pg)->shaders; 145 146 if (!pg->dd) 147 pg->dd = (void*)rzalloc(pg, struct zink_program_descriptor_data); 148 if (!pg->dd) 149 return false; 150 151 if (!pg->is_compute && stages[PIPE_SHADER_FRAGMENT]->nir->info.fs.uses_fbfetch_output) { 152 zink_descriptor_util_init_fbfetch(ctx); 153 push_count = 1; 154 pg->dd->fbfetch = true; 155 } 156 157 unsigned entry_idx[ZINK_DESCRIPTOR_TYPES] = {0}; 158 159 unsigned num_shaders = pg->is_compute ? 1 : ZINK_SHADER_COUNT; 160 bool have_push = screen->info.have_KHR_push_descriptor; 161 for (int i = 0; i < num_shaders; i++) { 162 struct zink_shader *shader = stages[i]; 163 if (!shader) 164 continue; 165 166 enum pipe_shader_type stage = pipe_shader_type_from_mesa(shader->nir->info.stage); 167 VkShaderStageFlagBits stage_flags = zink_shader_stage(stage); 168 for (int j = 0; j < ZINK_DESCRIPTOR_TYPES; j++) { 169 for (int k = 0; k < shader->num_bindings[j]; k++) { 170 /* dynamic ubos handled in push */ 171 if (shader->bindings[j][k].type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC) { 172 pg->dd->push_usage |= BITFIELD64_BIT(stage); 173 174 push_count++; 175 continue; 176 } 177 178 assert(num_bindings[j] < ARRAY_SIZE(bindings[j])); 179 VkDescriptorSetLayoutBinding *binding = &bindings[j][num_bindings[j]]; 180 binding->binding = shader->bindings[j][k].binding; 181 binding->descriptorType = shader->bindings[j][k].type; 182 binding->descriptorCount = shader->bindings[j][k].size; 183 binding->stageFlags = stage_flags; 184 binding->pImmutableSamplers = NULL; 185 186 enum zink_descriptor_size_index idx = zink_vktype_to_size_idx(shader->bindings[j][k].type); 187 pg->dd->sizes[idx].descriptorCount += shader->bindings[j][k].size; 188 pg->dd->sizes[idx].type = shader->bindings[j][k].type; 189 switch (shader->bindings[j][k].type) { 190 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER: 191 case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE: 192 case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER: 193 case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER: 194 init_template_entry(shader, j, k, 0, &entries[j][entry_idx[j]], &entry_idx[j], screen->descriptor_mode == ZINK_DESCRIPTOR_MODE_LAZY); 195 break; 196 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER: 197 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER: 198 for (unsigned l = 0; l < shader->bindings[j][k].size; l++) 199 init_template_entry(shader, j, k, l, &entries[j][entry_idx[j]], &entry_idx[j], screen->descriptor_mode == ZINK_DESCRIPTOR_MODE_LAZY); 200 break; 201 default: 202 break; 203 } 204 num_bindings[j]++; 205 has_bindings |= BITFIELD_BIT(j); 206 } 207 } 208 pg->dd->bindless |= shader->bindless; 209 } 210 if (pg->dd->bindless) 211 zink_descriptors_init_bindless(ctx); 212 pg->dd->binding_usage = has_bindings; 213 if (!has_bindings && !push_count) { 214 ralloc_free(pg->dd); 215 pg->dd = NULL; 216 217 pg->layout = zink_pipeline_layout_create(screen, pg, &pg->compat_id); 218 return !!pg->layout; 219 } 220 221 pg->dsl[pg->num_dsl++] = push_count ? ctx->dd->push_dsl[pg->is_compute]->layout : ctx->dd->dummy_dsl->layout; 222 if (has_bindings) { 223 u_foreach_bit(type, has_bindings) { 224 for (unsigned i = 0; i < type; i++) { 225 /* push set is always 0 */ 226 if (!pg->dsl[i + 1]) { 227 /* inject a null dsl */ 228 pg->dsl[pg->num_dsl++] = ctx->dd->dummy_dsl->layout; 229 pg->dd->binding_usage |= BITFIELD_BIT(i); 230 } 231 } 232 pg->dd->layouts[pg->num_dsl] = zink_descriptor_util_layout_get(ctx, type, bindings[type], num_bindings[type], &pg->dd->layout_key[type]); 233 pg->dd->layout_key[type]->use_count++; 234 pg->dsl[pg->num_dsl] = pg->dd->layouts[pg->num_dsl]->layout; 235 pg->num_dsl++; 236 } 237 for (unsigned i = 0; i < ARRAY_SIZE(pg->dd->sizes); i++) 238 pg->dd->sizes[i].descriptorCount *= screen->descriptor_mode == ZINK_DESCRIPTOR_MODE_LAZY ? MAX_LAZY_DESCRIPTORS : ZINK_DEFAULT_MAX_DESCS; 239 } 240 /* TODO: make this dynamic? */ 241 if (pg->dd->bindless) { 242 pg->num_dsl = ZINK_DESCRIPTOR_BINDLESS + 1; 243 pg->dsl[ZINK_DESCRIPTOR_BINDLESS] = ctx->dd->bindless_layout; 244 for (unsigned i = 0; i < ZINK_DESCRIPTOR_BINDLESS; i++) { 245 if (!pg->dsl[i]) { 246 /* inject a null dsl */ 247 pg->dsl[i] = ctx->dd->dummy_dsl->layout; 248 if (i != ZINK_DESCRIPTOR_TYPES) 249 pg->dd->binding_usage |= BITFIELD_BIT(i); 250 } 251 } 252 } 253 254 pg->layout = zink_pipeline_layout_create(screen, pg, &pg->compat_id); 255 if (!pg->layout) 256 return false; 257 if (!screen->info.have_KHR_descriptor_update_template || screen->descriptor_mode == ZINK_DESCRIPTOR_MODE_NOTEMPLATES) 258 return true; 259 260 VkDescriptorUpdateTemplateCreateInfo template[ZINK_DESCRIPTOR_TYPES + 1] = {0}; 261 /* type of template */ 262 VkDescriptorUpdateTemplateType types[ZINK_DESCRIPTOR_TYPES + 1] = {VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_DESCRIPTOR_SET}; 263 if (have_push && screen->descriptor_mode == ZINK_DESCRIPTOR_MODE_LAZY) 264 types[0] = VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_PUSH_DESCRIPTORS_KHR; 265 266 /* number of descriptors in template */ 267 unsigned wd_count[ZINK_DESCRIPTOR_TYPES + 1]; 268 if (push_count) 269 wd_count[0] = pg->is_compute ? 1 : (ZINK_SHADER_COUNT + !!ctx->dd->has_fbfetch); 270 for (unsigned i = 0; i < ZINK_DESCRIPTOR_TYPES; i++) 271 wd_count[i + 1] = pg->dd->layout_key[i] ? pg->dd->layout_key[i]->num_descriptors : 0; 272 273 VkDescriptorUpdateTemplateEntry *push_entries[2] = { 274 dd_lazy(ctx)->push_entries, 275 &dd_lazy(ctx)->compute_push_entry, 276 }; 277 for (unsigned i = 0; i < pg->num_dsl; i++) { 278 bool is_push = i == 0; 279 /* no need for empty templates */ 280 if (pg->dsl[i] == ctx->dd->dummy_dsl->layout || 281 pg->dsl[i] == ctx->dd->bindless_layout || 282 (!is_push && pg->dd->layouts[i]->desc_template)) 283 continue; 284 template[i].sType = VK_STRUCTURE_TYPE_DESCRIPTOR_UPDATE_TEMPLATE_CREATE_INFO; 285 assert(wd_count[i]); 286 template[i].descriptorUpdateEntryCount = wd_count[i]; 287 if (is_push) 288 template[i].pDescriptorUpdateEntries = push_entries[pg->is_compute]; 289 else 290 template[i].pDescriptorUpdateEntries = entries[i - 1]; 291 template[i].templateType = types[i]; 292 template[i].descriptorSetLayout = pg->dsl[i]; 293 template[i].pipelineBindPoint = pg->is_compute ? VK_PIPELINE_BIND_POINT_COMPUTE : VK_PIPELINE_BIND_POINT_GRAPHICS; 294 template[i].pipelineLayout = pg->layout; 295 template[i].set = i; 296 VkDescriptorUpdateTemplateKHR t; 297 if (VKSCR(CreateDescriptorUpdateTemplate)(screen->dev, &template[i], NULL, &t) != VK_SUCCESS) 298 return false; 299 if (is_push) 300 pg->dd->push_template = t; 301 else 302 pg->dd->layouts[i]->desc_template = t; 303 } 304 return true; 305} 306 307void 308zink_descriptor_program_deinit_lazy(struct zink_screen *screen, struct zink_program *pg) 309{ 310 for (unsigned i = 0; pg->num_dsl && i < ZINK_DESCRIPTOR_TYPES; i++) { 311 if (pg->dd->layout_key[i]) 312 pg->dd->layout_key[i]->use_count--; 313 } 314 if (pg->dd && pg->dd->push_template) 315 VKSCR(DestroyDescriptorUpdateTemplate)(screen->dev, pg->dd->push_template, NULL); 316 ralloc_free(pg->dd); 317} 318 319static VkDescriptorPool 320create_pool(struct zink_screen *screen, unsigned num_type_sizes, VkDescriptorPoolSize *sizes, unsigned flags) 321{ 322 VkDescriptorPool pool; 323 VkDescriptorPoolCreateInfo dpci = {0}; 324 dpci.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO; 325 dpci.pPoolSizes = sizes; 326 dpci.poolSizeCount = num_type_sizes; 327 dpci.flags = flags; 328 dpci.maxSets = MAX_LAZY_DESCRIPTORS; 329 if (VKSCR(CreateDescriptorPool)(screen->dev, &dpci, 0, &pool) != VK_SUCCESS) { 330 debug_printf("vkCreateDescriptorPool failed\n"); 331 return VK_NULL_HANDLE; 332 } 333 return pool; 334} 335 336static struct zink_descriptor_pool * 337get_descriptor_pool_lazy(struct zink_context *ctx, struct zink_program *pg, enum zink_descriptor_type type, struct zink_batch_descriptor_data_lazy *bdd, bool is_compute); 338 339static struct zink_descriptor_pool * 340check_pool_alloc(struct zink_context *ctx, struct zink_descriptor_pool *pool, struct hash_entry *he, struct zink_program *pg, 341 enum zink_descriptor_type type, struct zink_batch_descriptor_data_lazy *bdd, bool is_compute) 342{ 343 struct zink_screen *screen = zink_screen(ctx->base.screen); 344 /* allocate up to $current * 10, e.g., 10 -> 100 or 100 -> 1000 */ 345 if (pool->set_idx == pool->sets_alloc) { 346 unsigned sets_to_alloc = MIN2(MIN2(MAX2(pool->sets_alloc * 10, 10), MAX_LAZY_DESCRIPTORS) - pool->sets_alloc, 100); 347 if (!sets_to_alloc) { 348 /* overflowed pool: queue for deletion on next reset */ 349 util_dynarray_append(&bdd->overflowed_pools, struct zink_descriptor_pool*, pool); 350 _mesa_hash_table_remove(&bdd->pools[type], he); 351 return get_descriptor_pool_lazy(ctx, pg, type, bdd, is_compute); 352 } 353 if (!zink_descriptor_util_alloc_sets(screen, pg->dsl[type + 1], 354 pool->pool, &pool->sets[pool->sets_alloc], sets_to_alloc)) 355 return NULL; 356 pool->sets_alloc += sets_to_alloc; 357 } 358 return pool; 359} 360 361static struct zink_descriptor_pool * 362create_push_pool(struct zink_screen *screen, struct zink_batch_descriptor_data_lazy *bdd, bool is_compute, bool has_fbfetch) 363{ 364 struct zink_descriptor_pool *pool = rzalloc(bdd, struct zink_descriptor_pool); 365 VkDescriptorPoolSize sizes[2]; 366 sizes[0].type = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER; 367 if (is_compute) 368 sizes[0].descriptorCount = MAX_LAZY_DESCRIPTORS; 369 else { 370 sizes[0].descriptorCount = ZINK_SHADER_COUNT * MAX_LAZY_DESCRIPTORS; 371 sizes[1].type = VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT; 372 sizes[1].descriptorCount = MAX_LAZY_DESCRIPTORS; 373 } 374 pool->pool = create_pool(screen, !is_compute && has_fbfetch ? 2 : 1, sizes, 0); 375 return pool; 376} 377 378static struct zink_descriptor_pool * 379check_push_pool_alloc(struct zink_context *ctx, struct zink_descriptor_pool *pool, struct zink_batch_descriptor_data_lazy *bdd, bool is_compute) 380{ 381 struct zink_screen *screen = zink_screen(ctx->base.screen); 382 /* allocate up to $current * 10, e.g., 10 -> 100 or 100 -> 1000 */ 383 if (pool->set_idx == pool->sets_alloc || unlikely(ctx->dd->has_fbfetch != bdd->has_fbfetch)) { 384 unsigned sets_to_alloc = MIN2(MIN2(MAX2(pool->sets_alloc * 10, 10), MAX_LAZY_DESCRIPTORS) - pool->sets_alloc, 100); 385 if (!sets_to_alloc || unlikely(ctx->dd->has_fbfetch != bdd->has_fbfetch)) { 386 /* overflowed pool: queue for deletion on next reset */ 387 util_dynarray_append(&bdd->overflowed_pools, struct zink_descriptor_pool*, pool); 388 bdd->push_pool[is_compute] = create_push_pool(screen, bdd, is_compute, ctx->dd->has_fbfetch); 389 bdd->has_fbfetch = ctx->dd->has_fbfetch; 390 return check_push_pool_alloc(ctx, bdd->push_pool[is_compute], bdd, is_compute); 391 } 392 if (!zink_descriptor_util_alloc_sets(screen, ctx->dd->push_dsl[is_compute]->layout, 393 pool->pool, &pool->sets[pool->sets_alloc], sets_to_alloc)) 394 return NULL; 395 pool->sets_alloc += sets_to_alloc; 396 } 397 return pool; 398} 399 400static struct zink_descriptor_pool * 401get_descriptor_pool_lazy(struct zink_context *ctx, struct zink_program *pg, enum zink_descriptor_type type, struct zink_batch_descriptor_data_lazy *bdd, bool is_compute) 402{ 403 struct zink_screen *screen = zink_screen(ctx->base.screen); 404 struct hash_entry *he = _mesa_hash_table_search(&bdd->pools[type], pg->dd->layout_key[type]); 405 struct zink_descriptor_pool *pool; 406 if (he) { 407 pool = he->data; 408 return check_pool_alloc(ctx, pool, he, pg, type, bdd, is_compute); 409 } 410 pool = rzalloc(bdd, struct zink_descriptor_pool); 411 if (!pool) 412 return NULL; 413 unsigned idx = zink_descriptor_type_to_size_idx(type); 414 VkDescriptorPoolSize *size = &pg->dd->sizes[idx]; 415 /* this is a sampler/image set with no images only texels */ 416 if (!size->descriptorCount) 417 size++; 418 pool->pool = create_pool(screen, zink_descriptor_program_num_sizes(pg, type), size, 0); 419 if (!pool->pool) { 420 ralloc_free(pool); 421 return NULL; 422 } 423 _mesa_hash_table_insert(&bdd->pools[type], pg->dd->layout_key[type], pool); 424 return check_pool_alloc(ctx, pool, he, pg, type, bdd, is_compute); 425} 426 427ALWAYS_INLINE static VkDescriptorSet 428get_descriptor_set_lazy(struct zink_descriptor_pool *pool) 429{ 430 if (!pool) 431 return VK_NULL_HANDLE; 432 433 assert(pool->set_idx < pool->sets_alloc); 434 return pool->sets[pool->set_idx++]; 435} 436 437static bool 438populate_sets(struct zink_context *ctx, struct zink_batch_descriptor_data_lazy *bdd, 439 struct zink_program *pg, uint8_t *changed_sets, VkDescriptorSet *sets) 440{ 441 u_foreach_bit(type, *changed_sets) { 442 if (pg->dd->layout_key[type]) { 443 struct zink_descriptor_pool *pool = get_descriptor_pool_lazy(ctx, pg, type, bdd, pg->is_compute); 444 sets[type] = get_descriptor_set_lazy(pool); 445 } else 446 sets[type] = ctx->dd->dummy_set; 447 if (!sets[type]) 448 return false; 449 } 450 return true; 451} 452 453void 454zink_descriptor_set_update_lazy(struct zink_context *ctx, struct zink_program *pg, enum zink_descriptor_type type, VkDescriptorSet set) 455{ 456 struct zink_screen *screen = zink_screen(ctx->base.screen); 457 VKCTX(UpdateDescriptorSetWithTemplate)(screen->dev, set, pg->dd->layouts[type + 1]->desc_template, ctx); 458} 459 460void 461zink_descriptors_update_lazy_masked(struct zink_context *ctx, bool is_compute, uint8_t changed_sets, uint8_t bind_sets) 462{ 463 struct zink_screen *screen = zink_screen(ctx->base.screen); 464 struct zink_batch_state *bs = ctx->batch.state; 465 struct zink_batch_descriptor_data_lazy *bdd = bdd_lazy(bs); 466 struct zink_program *pg = is_compute ? &ctx->curr_compute->base : &ctx->curr_program->base; 467 VkDescriptorSet desc_sets[ZINK_DESCRIPTOR_TYPES]; 468 if (!pg->dd->binding_usage || (!changed_sets && !bind_sets)) 469 return; 470 471 if (!populate_sets(ctx, bdd, pg, &changed_sets, desc_sets)) { 472 debug_printf("ZINK: couldn't get descriptor sets!\n"); 473 return; 474 } 475 /* no flushing allowed */ 476 assert(ctx->batch.state == bs); 477 478 u_foreach_bit(type, changed_sets) { 479 assert(type + 1 < pg->num_dsl); 480 if (pg->dd->layout_key[type]) { 481 VKSCR(UpdateDescriptorSetWithTemplate)(screen->dev, desc_sets[type], pg->dd->layouts[type + 1]->desc_template, ctx); 482 VKSCR(CmdBindDescriptorSets)(bs->cmdbuf, 483 is_compute ? VK_PIPELINE_BIND_POINT_COMPUTE : VK_PIPELINE_BIND_POINT_GRAPHICS, 484 /* set index incremented by 1 to account for push set */ 485 pg->layout, type + 1, 1, &desc_sets[type], 486 0, NULL); 487 bdd->sets[is_compute][type + 1] = desc_sets[type]; 488 } 489 } 490 u_foreach_bit(type, bind_sets & ~changed_sets) { 491 if (!pg->dd->layout_key[type]) 492 bdd->sets[is_compute][type + 1] = ctx->dd->dummy_set; 493 assert(bdd->sets[is_compute][type + 1]); 494 VKSCR(CmdBindDescriptorSets)(bs->cmdbuf, 495 is_compute ? VK_PIPELINE_BIND_POINT_COMPUTE : VK_PIPELINE_BIND_POINT_GRAPHICS, 496 /* set index incremented by 1 to account for push set */ 497 pg->layout, type + 1, 1, &bdd->sets[is_compute][type + 1], 498 0, NULL); 499 } 500} 501 502/* only called by cached manager for fbfetch handling */ 503VkDescriptorSet 504zink_descriptors_alloc_lazy_push(struct zink_context *ctx) 505{ 506 struct zink_batch_state *bs = ctx->batch.state; 507 struct zink_batch_descriptor_data_lazy *bdd = bdd_lazy(bs); 508 struct zink_screen *screen = zink_screen(ctx->base.screen); 509 VkDescriptorSet push_set = VK_NULL_HANDLE; 510 if (!bdd->push_pool[0]) { 511 bdd->push_pool[0] = create_push_pool(screen, bdd, false, true); 512 bdd->has_fbfetch = true; 513 } 514 struct zink_descriptor_pool *pool = check_push_pool_alloc(ctx, bdd->push_pool[0], bdd, false); 515 push_set = get_descriptor_set_lazy(pool); 516 if (!push_set) 517 mesa_loge("ZINK: failed to get push descriptor set!"); 518 return push_set; 519} 520 521void 522zink_descriptors_update_lazy(struct zink_context *ctx, bool is_compute) 523{ 524 struct zink_batch_state *bs = ctx->batch.state; 525 struct zink_batch_descriptor_data_lazy *bdd = bdd_lazy(bs); 526 struct zink_program *pg = is_compute ? &ctx->curr_compute->base : &ctx->curr_program->base; 527 struct zink_screen *screen = zink_screen(ctx->base.screen); 528 bool have_KHR_push_descriptor = screen->info.have_KHR_push_descriptor; 529 530 bool batch_changed = !bdd->pg[is_compute]; 531 if (batch_changed) { 532 /* update all sets and bind null sets */ 533 dd_lazy(ctx)->state_changed[is_compute] = pg->dd->binding_usage; 534 dd_lazy(ctx)->push_state_changed[is_compute] = !!pg->dd->push_usage; 535 } 536 537 if (pg != bdd->pg[is_compute]) { 538 /* if we don't already know that we have to update all sets, 539 * check to see if any dsls changed 540 * 541 * also always update the dsl pointers on program change 542 */ 543 for (unsigned i = 0; i < ARRAY_SIZE(bdd->dsl[is_compute]); i++) { 544 /* push set is already detected, start at 1 */ 545 if (bdd->dsl[is_compute][i] != pg->dsl[i + 1]) 546 dd_lazy(ctx)->state_changed[is_compute] |= BITFIELD_BIT(i); 547 bdd->dsl[is_compute][i] = pg->dsl[i + 1]; 548 } 549 dd_lazy(ctx)->push_state_changed[is_compute] |= bdd->push_usage[is_compute] != pg->dd->push_usage; 550 bdd->push_usage[is_compute] = pg->dd->push_usage; 551 } 552 553 uint8_t changed_sets = pg->dd->binding_usage & dd_lazy(ctx)->state_changed[is_compute]; 554 bool need_push = pg->dd->push_usage && 555 (dd_lazy(ctx)->push_state_changed[is_compute] || batch_changed); 556 VkDescriptorSet push_set = VK_NULL_HANDLE; 557 if (need_push && !have_KHR_push_descriptor) { 558 struct zink_descriptor_pool *pool = check_push_pool_alloc(ctx, bdd->push_pool[pg->is_compute], bdd, pg->is_compute); 559 push_set = get_descriptor_set_lazy(pool); 560 if (!push_set) { 561 mesa_loge("ZINK: failed to get push descriptor set!"); 562 /* just jam something in to avoid a hang */ 563 push_set = ctx->dd->dummy_set; 564 } 565 } 566 /* 567 * when binding a pipeline, the pipeline can correctly access any previously bound 568 * descriptor sets which were bound with compatible pipeline layouts 569 * VK 14.2.2 570 */ 571 uint8_t bind_sets = bdd->pg[is_compute] && bdd->compat_id[is_compute] == pg->compat_id ? 0 : pg->dd->binding_usage; 572 if (pg->dd->push_usage && (dd_lazy(ctx)->push_state_changed[is_compute] || bind_sets)) { 573 if (have_KHR_push_descriptor) { 574 if (dd_lazy(ctx)->push_state_changed[is_compute]) 575 VKCTX(CmdPushDescriptorSetWithTemplateKHR)(bs->cmdbuf, pg->dd->push_template, 576 pg->layout, 0, ctx); 577 } else { 578 if (dd_lazy(ctx)->push_state_changed[is_compute]) { 579 VKCTX(UpdateDescriptorSetWithTemplate)(screen->dev, push_set, pg->dd->push_template, ctx); 580 bdd->sets[is_compute][0] = push_set; 581 } 582 assert(push_set || bdd->sets[is_compute][0]); 583 VKCTX(CmdBindDescriptorSets)(bs->cmdbuf, 584 is_compute ? VK_PIPELINE_BIND_POINT_COMPUTE : VK_PIPELINE_BIND_POINT_GRAPHICS, 585 pg->layout, 0, 1, push_set ? &push_set : &bdd->sets[is_compute][0], 586 0, NULL); 587 } 588 dd_lazy(ctx)->push_state_changed[is_compute] = false; 589 } else if (dd_lazy(ctx)->push_state_changed[is_compute] || bind_sets) { 590 VKCTX(CmdBindDescriptorSets)(bs->cmdbuf, 591 is_compute ? VK_PIPELINE_BIND_POINT_COMPUTE : VK_PIPELINE_BIND_POINT_GRAPHICS, 592 pg->layout, 0, 1, &ctx->dd->dummy_set, 593 0, NULL); 594 dd_lazy(ctx)->push_state_changed[is_compute] = false; 595 } 596 zink_descriptors_update_lazy_masked(ctx, is_compute, changed_sets, bind_sets); 597 if (pg->dd->bindless && unlikely(!ctx->dd->bindless_bound)) { 598 VKCTX(CmdBindDescriptorSets)(ctx->batch.state->cmdbuf, is_compute ? VK_PIPELINE_BIND_POINT_COMPUTE : VK_PIPELINE_BIND_POINT_GRAPHICS, 599 pg->layout, ZINK_DESCRIPTOR_BINDLESS, 1, &ctx->dd->bindless_set, 600 0, NULL); 601 ctx->dd->bindless_bound = true; 602 } 603 bdd->pg[is_compute] = pg; 604 ctx->dd->pg[is_compute] = pg; 605 bdd->compat_id[is_compute] = pg->compat_id; 606 dd_lazy(ctx)->state_changed[is_compute] = false; 607} 608 609void 610zink_context_invalidate_descriptor_state_lazy(struct zink_context *ctx, enum pipe_shader_type shader, enum zink_descriptor_type type, unsigned start, unsigned count) 611{ 612 if (type == ZINK_DESCRIPTOR_TYPE_UBO && !start) 613 dd_lazy(ctx)->push_state_changed[shader == PIPE_SHADER_COMPUTE] = true; 614 else 615 dd_lazy(ctx)->state_changed[shader == PIPE_SHADER_COMPUTE] |= BITFIELD_BIT(type); 616} 617 618void 619zink_batch_descriptor_deinit_lazy(struct zink_screen *screen, struct zink_batch_state *bs) 620{ 621 if (!bs->dd) 622 return; 623 struct zink_batch_descriptor_data_lazy *bdd = bdd_lazy(bs); 624 if (screen->info.have_KHR_descriptor_update_template) { 625 for (unsigned i = 0; i < ZINK_DESCRIPTOR_TYPES; i++) { 626 hash_table_foreach(&bdd->pools[i], entry) { 627 struct zink_descriptor_pool *pool = (void*)entry->data; 628 VKSCR(DestroyDescriptorPool)(screen->dev, pool->pool, NULL); 629 } 630 } 631 if (bdd->push_pool[0]) 632 VKSCR(DestroyDescriptorPool)(screen->dev, bdd->push_pool[0]->pool, NULL); 633 if (bdd->push_pool[1]) 634 VKSCR(DestroyDescriptorPool)(screen->dev, bdd->push_pool[1]->pool, NULL); 635 } 636 ralloc_free(bs->dd); 637} 638 639static void 640pool_destroy(struct zink_screen *screen, struct zink_descriptor_pool *pool) 641{ 642 VKSCR(DestroyDescriptorPool)(screen->dev, pool->pool, NULL); 643 ralloc_free(pool); 644} 645 646void 647zink_batch_descriptor_reset_lazy(struct zink_screen *screen, struct zink_batch_state *bs) 648{ 649 if (!screen->info.have_KHR_descriptor_update_template) 650 return; 651 struct zink_batch_descriptor_data_lazy *bdd = bdd_lazy(bs); 652 for (unsigned i = 0; i < ZINK_DESCRIPTOR_TYPES; i++) { 653 hash_table_foreach(&bdd->pools[i], entry) { 654 const struct zink_descriptor_layout_key *key = entry->key; 655 struct zink_descriptor_pool *pool = (void*)entry->data; 656 if (key->use_count) 657 pool->set_idx = 0; 658 else { 659 pool_destroy(screen, pool); 660 _mesa_hash_table_remove(&bdd->pools[i], entry); 661 } 662 } 663 } 664 for (unsigned i = 0; i < 2; i++) { 665 bdd->pg[i] = NULL; 666 if (bdd->push_pool[i]) 667 bdd->push_pool[i]->set_idx = 0; 668 } 669 while (util_dynarray_num_elements(&bdd->overflowed_pools, struct zink_descriptor_pool*)) { 670 struct zink_descriptor_pool *pool = util_dynarray_pop(&bdd->overflowed_pools, struct zink_descriptor_pool*); 671 pool_destroy(screen, pool); 672 } 673} 674 675bool 676zink_batch_descriptor_init_lazy(struct zink_screen *screen, struct zink_batch_state *bs) 677{ 678 bs->dd = (void*)rzalloc(bs, struct zink_batch_descriptor_data_lazy); 679 if (!bs->dd) 680 return false; 681 if (!screen->info.have_KHR_descriptor_update_template) 682 return true; 683 struct zink_batch_descriptor_data_lazy *bdd = bdd_lazy(bs); 684 for (unsigned i = 0; i < ZINK_DESCRIPTOR_TYPES; i++) { 685 if (!_mesa_hash_table_init(&bdd->pools[i], bs->dd, _mesa_hash_pointer, _mesa_key_pointer_equal)) 686 return false; 687 } 688 util_dynarray_init(&bdd->overflowed_pools, bs->dd); 689 if (!screen->info.have_KHR_push_descriptor) { 690 bdd->push_pool[0] = create_push_pool(screen, bdd, false, false); 691 bdd->push_pool[1] = create_push_pool(screen, bdd, true, false); 692 } 693 return true; 694} 695 696static void 697init_push_template_entry(VkDescriptorUpdateTemplateEntry *entry, unsigned i) 698{ 699 entry->dstBinding = tgsi_processor_to_shader_stage(i); 700 entry->descriptorCount = 1; 701 entry->descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER; 702 entry->offset = offsetof(struct zink_context, di.ubos[i][0]); 703 entry->stride = sizeof(VkDescriptorBufferInfo); 704} 705 706bool 707zink_descriptors_init_lazy(struct zink_context *ctx) 708{ 709 struct zink_screen *screen = zink_screen(ctx->base.screen); 710 ctx->dd = (void*)rzalloc(ctx, struct zink_descriptor_data_lazy); 711 if (!ctx->dd) 712 return false; 713 714 if (screen->descriptor_mode == ZINK_DESCRIPTOR_MODE_NOTEMPLATES) 715 printf("ZINK: CACHED/NOTEMPLATES DESCRIPTORS\n"); 716 else if (screen->info.have_KHR_descriptor_update_template) { 717 for (unsigned i = 0; i < ZINK_SHADER_COUNT; i++) { 718 VkDescriptorUpdateTemplateEntry *entry = &dd_lazy(ctx)->push_entries[i]; 719 init_push_template_entry(entry, i); 720 } 721 init_push_template_entry(&dd_lazy(ctx)->compute_push_entry, PIPE_SHADER_COMPUTE); 722 VkDescriptorUpdateTemplateEntry *entry = &dd_lazy(ctx)->push_entries[ZINK_SHADER_COUNT]; //fbfetch 723 entry->dstBinding = ZINK_FBFETCH_BINDING; 724 entry->descriptorCount = 1; 725 entry->descriptorType = VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT; 726 entry->offset = offsetof(struct zink_context, di.fbfetch); 727 entry->stride = sizeof(VkDescriptorImageInfo); 728 if (screen->descriptor_mode == ZINK_DESCRIPTOR_MODE_LAZY) 729 printf("ZINK: USING LAZY DESCRIPTORS\n"); 730 } 731 struct zink_descriptor_layout_key *layout_key; 732 if (!zink_descriptor_util_push_layouts_get(ctx, ctx->dd->push_dsl, ctx->dd->push_layout_keys)) 733 return false; 734 735 ctx->dd->dummy_dsl = zink_descriptor_util_layout_get(ctx, 0, NULL, 0, &layout_key); 736 if (!ctx->dd->dummy_dsl) 737 return false; 738 VkDescriptorPoolSize null_size = {VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 1}; 739 ctx->dd->dummy_pool = create_pool(screen, 1, &null_size, 0); 740 zink_descriptor_util_alloc_sets(screen, ctx->dd->dummy_dsl->layout, 741 ctx->dd->dummy_pool, &ctx->dd->dummy_set, 1); 742 zink_descriptor_util_init_null_set(ctx, ctx->dd->dummy_set); 743 744 return true; 745} 746 747void 748zink_descriptors_deinit_lazy(struct zink_context *ctx) 749{ 750 if (ctx->dd) { 751 struct zink_screen *screen = zink_screen(ctx->base.screen); 752 if (ctx->dd->dummy_pool) 753 VKSCR(DestroyDescriptorPool)(screen->dev, ctx->dd->dummy_pool, NULL); 754 if (ctx->dd->push_dsl[0]) 755 VKSCR(DestroyDescriptorSetLayout)(screen->dev, ctx->dd->push_dsl[0]->layout, NULL); 756 if (ctx->dd->push_dsl[1]) 757 VKSCR(DestroyDescriptorSetLayout)(screen->dev, ctx->dd->push_dsl[1]->layout, NULL); 758 } 759 ralloc_free(ctx->dd); 760} 761