radv_meta_blit2d.c revision 7ec681f3
1/* 2 * Copyright © 2016 Red Hat 3 * 4 * based on anv driver: 5 * Copyright © 2016 Intel Corporation 6 * 7 * Permission is hereby granted, free of charge, to any person obtaining a 8 * copy of this software and associated documentation files (the "Software"), 9 * to deal in the Software without restriction, including without limitation 10 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 11 * and/or sell copies of the Software, and to permit persons to whom the 12 * Software is furnished to do so, subject to the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the next 15 * paragraph) shall be included in all copies or substantial portions of the 16 * Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 21 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 23 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 24 * IN THE SOFTWARE. 25 */ 26 27#include "nir/nir_builder.h" 28#include "radv_meta.h" 29#include "vk_format.h" 30 31enum blit2d_src_type { 32 BLIT2D_SRC_TYPE_IMAGE, 33 BLIT2D_SRC_TYPE_IMAGE_3D, 34 BLIT2D_SRC_TYPE_BUFFER, 35 BLIT2D_NUM_SRC_TYPES, 36}; 37 38static VkResult blit2d_init_color_pipeline(struct radv_device *device, 39 enum blit2d_src_type src_type, VkFormat format, 40 uint32_t log2_samples); 41 42static VkResult blit2d_init_depth_only_pipeline(struct radv_device *device, 43 enum blit2d_src_type src_type, 44 uint32_t log2_samples); 45 46static VkResult blit2d_init_stencil_only_pipeline(struct radv_device *device, 47 enum blit2d_src_type src_type, 48 uint32_t log2_samples); 49 50static void 51create_iview(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *surf, 52 struct radv_image_view *iview, VkFormat depth_format, VkImageAspectFlagBits aspects) 53{ 54 VkFormat format; 55 VkImageViewType view_type = cmd_buffer->device->physical_device->rad_info.chip_class < GFX9 56 ? VK_IMAGE_VIEW_TYPE_2D 57 : radv_meta_get_view_type(surf->image); 58 59 if (depth_format) 60 format = depth_format; 61 else 62 format = surf->format; 63 64 radv_image_view_init(iview, cmd_buffer->device, 65 &(VkImageViewCreateInfo){ 66 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, 67 .image = radv_image_to_handle(surf->image), 68 .viewType = view_type, 69 .format = format, 70 .subresourceRange = {.aspectMask = aspects, 71 .baseMipLevel = surf->level, 72 .levelCount = 1, 73 .baseArrayLayer = surf->layer, 74 .layerCount = 1}, 75 }, 76 NULL); 77} 78 79static void 80create_bview(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_buffer *src, 81 struct radv_buffer_view *bview, VkFormat depth_format) 82{ 83 VkFormat format; 84 85 if (depth_format) 86 format = depth_format; 87 else 88 format = src->format; 89 radv_buffer_view_init(bview, cmd_buffer->device, 90 &(VkBufferViewCreateInfo){ 91 .sType = VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO, 92 .flags = 0, 93 .buffer = radv_buffer_to_handle(src->buffer), 94 .format = format, 95 .offset = src->offset, 96 .range = VK_WHOLE_SIZE, 97 }); 98} 99 100struct blit2d_src_temps { 101 struct radv_image_view iview; 102 struct radv_buffer_view bview; 103}; 104 105static void 106blit2d_bind_src(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *src_img, 107 struct radv_meta_blit2d_buffer *src_buf, struct blit2d_src_temps *tmp, 108 enum blit2d_src_type src_type, VkFormat depth_format, VkImageAspectFlagBits aspects, 109 uint32_t log2_samples) 110{ 111 struct radv_device *device = cmd_buffer->device; 112 113 if (src_type == BLIT2D_SRC_TYPE_BUFFER) { 114 create_bview(cmd_buffer, src_buf, &tmp->bview, depth_format); 115 116 radv_meta_push_descriptor_set( 117 cmd_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, 118 device->meta_state.blit2d[log2_samples].p_layouts[src_type], 0, /* set */ 119 1, /* descriptorWriteCount */ 120 (VkWriteDescriptorSet[]){ 121 {.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, 122 .dstBinding = 0, 123 .dstArrayElement = 0, 124 .descriptorCount = 1, 125 .descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER, 126 .pTexelBufferView = (VkBufferView[]){radv_buffer_view_to_handle(&tmp->bview)}}}); 127 128 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer), 129 device->meta_state.blit2d[log2_samples].p_layouts[src_type], 130 VK_SHADER_STAGE_FRAGMENT_BIT, 16, 4, &src_buf->pitch); 131 } else { 132 create_iview(cmd_buffer, src_img, &tmp->iview, depth_format, aspects); 133 134 if (src_type == BLIT2D_SRC_TYPE_IMAGE_3D) 135 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer), 136 device->meta_state.blit2d[log2_samples].p_layouts[src_type], 137 VK_SHADER_STAGE_FRAGMENT_BIT, 16, 4, &src_img->layer); 138 139 radv_meta_push_descriptor_set( 140 cmd_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, 141 device->meta_state.blit2d[log2_samples].p_layouts[src_type], 0, /* set */ 142 1, /* descriptorWriteCount */ 143 (VkWriteDescriptorSet[]){{.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, 144 .dstBinding = 0, 145 .dstArrayElement = 0, 146 .descriptorCount = 1, 147 .descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, 148 .pImageInfo = (VkDescriptorImageInfo[]){ 149 { 150 .sampler = VK_NULL_HANDLE, 151 .imageView = radv_image_view_to_handle(&tmp->iview), 152 .imageLayout = VK_IMAGE_LAYOUT_GENERAL, 153 }, 154 }}}); 155 } 156} 157 158struct blit2d_dst_temps { 159 VkImage image; 160 struct radv_image_view iview; 161 VkFramebuffer fb; 162}; 163 164static void 165blit2d_bind_dst(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *dst, 166 uint32_t width, uint32_t height, VkFormat depth_format, 167 struct blit2d_dst_temps *tmp, VkImageAspectFlagBits aspects) 168{ 169 create_iview(cmd_buffer, dst, &tmp->iview, depth_format, aspects); 170 171 radv_CreateFramebuffer( 172 radv_device_to_handle(cmd_buffer->device), 173 &(VkFramebufferCreateInfo){.sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO, 174 .attachmentCount = 1, 175 .pAttachments = 176 (VkImageView[]){ 177 radv_image_view_to_handle(&tmp->iview), 178 }, 179 .width = width, 180 .height = height, 181 .layers = 1}, 182 &cmd_buffer->pool->alloc, &tmp->fb); 183} 184 185static void 186bind_pipeline(struct radv_cmd_buffer *cmd_buffer, enum blit2d_src_type src_type, unsigned fs_key, 187 uint32_t log2_samples) 188{ 189 VkPipeline pipeline = 190 cmd_buffer->device->meta_state.blit2d[log2_samples].pipelines[src_type][fs_key]; 191 192 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_GRAPHICS, 193 pipeline); 194} 195 196static void 197bind_depth_pipeline(struct radv_cmd_buffer *cmd_buffer, enum blit2d_src_type src_type, 198 uint32_t log2_samples) 199{ 200 VkPipeline pipeline = 201 cmd_buffer->device->meta_state.blit2d[log2_samples].depth_only_pipeline[src_type]; 202 203 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_GRAPHICS, 204 pipeline); 205} 206 207static void 208bind_stencil_pipeline(struct radv_cmd_buffer *cmd_buffer, enum blit2d_src_type src_type, 209 uint32_t log2_samples) 210{ 211 VkPipeline pipeline = 212 cmd_buffer->device->meta_state.blit2d[log2_samples].stencil_only_pipeline[src_type]; 213 214 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_GRAPHICS, 215 pipeline); 216} 217 218static void 219radv_meta_blit2d_normal_dst(struct radv_cmd_buffer *cmd_buffer, 220 struct radv_meta_blit2d_surf *src_img, 221 struct radv_meta_blit2d_buffer *src_buf, 222 struct radv_meta_blit2d_surf *dst, unsigned num_rects, 223 struct radv_meta_blit2d_rect *rects, enum blit2d_src_type src_type, 224 uint32_t log2_samples) 225{ 226 struct radv_device *device = cmd_buffer->device; 227 228 for (unsigned r = 0; r < num_rects; ++r) { 229 u_foreach_bit(i, dst->aspect_mask) 230 { 231 unsigned aspect_mask = 1u << i; 232 unsigned src_aspect_mask = aspect_mask; 233 VkFormat depth_format = 0; 234 if (aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT) 235 depth_format = vk_format_stencil_only(dst->image->vk_format); 236 else if (aspect_mask == VK_IMAGE_ASPECT_DEPTH_BIT) 237 depth_format = vk_format_depth_only(dst->image->vk_format); 238 else if (src_img) 239 src_aspect_mask = src_img->aspect_mask; 240 241 struct blit2d_src_temps src_temps; 242 blit2d_bind_src(cmd_buffer, src_img, src_buf, &src_temps, src_type, depth_format, 243 src_aspect_mask, log2_samples); 244 245 struct blit2d_dst_temps dst_temps; 246 blit2d_bind_dst(cmd_buffer, dst, rects[r].dst_x + rects[r].width, 247 rects[r].dst_y + rects[r].height, depth_format, &dst_temps, aspect_mask); 248 249 float vertex_push_constants[4] = { 250 rects[r].src_x, 251 rects[r].src_y, 252 rects[r].src_x + rects[r].width, 253 rects[r].src_y + rects[r].height, 254 }; 255 256 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer), 257 device->meta_state.blit2d[log2_samples].p_layouts[src_type], 258 VK_SHADER_STAGE_VERTEX_BIT, 0, 16, vertex_push_constants); 259 260 if (aspect_mask == VK_IMAGE_ASPECT_COLOR_BIT || 261 aspect_mask == VK_IMAGE_ASPECT_PLANE_0_BIT || 262 aspect_mask == VK_IMAGE_ASPECT_PLANE_1_BIT || 263 aspect_mask == VK_IMAGE_ASPECT_PLANE_2_BIT) { 264 unsigned fs_key = radv_format_meta_fs_key(device, dst_temps.iview.vk_format); 265 unsigned dst_layout = radv_meta_dst_layout_from_layout(dst->current_layout); 266 267 if (device->meta_state.blit2d[log2_samples].pipelines[src_type][fs_key] == 268 VK_NULL_HANDLE) { 269 VkResult ret = blit2d_init_color_pipeline( 270 device, src_type, radv_fs_key_format_exemplars[fs_key], log2_samples); 271 if (ret != VK_SUCCESS) { 272 cmd_buffer->record_result = ret; 273 goto fail_pipeline; 274 } 275 } 276 277 radv_cmd_buffer_begin_render_pass( 278 cmd_buffer, 279 &(VkRenderPassBeginInfo){ 280 .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO, 281 .renderPass = device->meta_state.blit2d_render_passes[fs_key][dst_layout], 282 .framebuffer = dst_temps.fb, 283 .renderArea = 284 { 285 .offset = 286 { 287 rects[r].dst_x, 288 rects[r].dst_y, 289 }, 290 .extent = {rects[r].width, rects[r].height}, 291 }, 292 .clearValueCount = 0, 293 .pClearValues = NULL, 294 }, 295 &(struct radv_extra_render_pass_begin_info){.disable_dcc = 296 dst->disable_compression}); 297 298 radv_cmd_buffer_set_subpass(cmd_buffer, &cmd_buffer->state.pass->subpasses[0]); 299 300 bind_pipeline(cmd_buffer, src_type, fs_key, log2_samples); 301 } else if (aspect_mask == VK_IMAGE_ASPECT_DEPTH_BIT) { 302 enum radv_blit_ds_layout ds_layout = radv_meta_blit_ds_to_type(dst->current_layout); 303 304 if (device->meta_state.blit2d[log2_samples].depth_only_pipeline[src_type] == 305 VK_NULL_HANDLE) { 306 VkResult ret = blit2d_init_depth_only_pipeline(device, src_type, log2_samples); 307 if (ret != VK_SUCCESS) { 308 cmd_buffer->record_result = ret; 309 goto fail_pipeline; 310 } 311 } 312 313 radv_cmd_buffer_begin_render_pass( 314 cmd_buffer, 315 &(VkRenderPassBeginInfo){ 316 .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO, 317 .renderPass = device->meta_state.blit2d_depth_only_rp[ds_layout], 318 .framebuffer = dst_temps.fb, 319 .renderArea = 320 { 321 .offset = 322 { 323 rects[r].dst_x, 324 rects[r].dst_y, 325 }, 326 .extent = {rects[r].width, rects[r].height}, 327 }, 328 .clearValueCount = 0, 329 .pClearValues = NULL, 330 }, 331 NULL); 332 333 radv_cmd_buffer_set_subpass(cmd_buffer, &cmd_buffer->state.pass->subpasses[0]); 334 335 bind_depth_pipeline(cmd_buffer, src_type, log2_samples); 336 337 } else if (aspect_mask == VK_IMAGE_ASPECT_STENCIL_BIT) { 338 enum radv_blit_ds_layout ds_layout = radv_meta_blit_ds_to_type(dst->current_layout); 339 340 if (device->meta_state.blit2d[log2_samples].stencil_only_pipeline[src_type] == 341 VK_NULL_HANDLE) { 342 VkResult ret = blit2d_init_stencil_only_pipeline(device, src_type, log2_samples); 343 if (ret != VK_SUCCESS) { 344 cmd_buffer->record_result = ret; 345 goto fail_pipeline; 346 } 347 } 348 349 radv_cmd_buffer_begin_render_pass( 350 cmd_buffer, 351 &(VkRenderPassBeginInfo){ 352 .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO, 353 .renderPass = device->meta_state.blit2d_stencil_only_rp[ds_layout], 354 .framebuffer = dst_temps.fb, 355 .renderArea = 356 { 357 .offset = 358 { 359 rects[r].dst_x, 360 rects[r].dst_y, 361 }, 362 .extent = {rects[r].width, rects[r].height}, 363 }, 364 .clearValueCount = 0, 365 .pClearValues = NULL, 366 }, 367 NULL); 368 369 radv_cmd_buffer_set_subpass(cmd_buffer, &cmd_buffer->state.pass->subpasses[0]); 370 371 bind_stencil_pipeline(cmd_buffer, src_type, log2_samples); 372 } else 373 unreachable("Processing blit2d with multiple aspects."); 374 375 radv_CmdSetViewport(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1, 376 &(VkViewport){.x = rects[r].dst_x, 377 .y = rects[r].dst_y, 378 .width = rects[r].width, 379 .height = rects[r].height, 380 .minDepth = 0.0f, 381 .maxDepth = 1.0f}); 382 383 radv_CmdSetScissor(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1, 384 &(VkRect2D){ 385 .offset = (VkOffset2D){rects[r].dst_x, rects[r].dst_y}, 386 .extent = (VkExtent2D){rects[r].width, rects[r].height}, 387 }); 388 389 radv_CmdDraw(radv_cmd_buffer_to_handle(cmd_buffer), 3, 1, 0, 0); 390 radv_cmd_buffer_end_render_pass(cmd_buffer); 391 392 fail_pipeline: 393 /* At the point where we emit the draw call, all data from the 394 * descriptor sets, etc. has been used. We are free to delete it. 395 */ 396 radv_DestroyFramebuffer(radv_device_to_handle(device), dst_temps.fb, 397 &cmd_buffer->pool->alloc); 398 399 if (src_type == BLIT2D_SRC_TYPE_BUFFER) 400 radv_buffer_view_finish(&src_temps.bview); 401 else 402 radv_image_view_finish(&dst_temps.iview); 403 404 radv_image_view_finish(&dst_temps.iview); 405 } 406 } 407} 408 409void 410radv_meta_blit2d(struct radv_cmd_buffer *cmd_buffer, struct radv_meta_blit2d_surf *src_img, 411 struct radv_meta_blit2d_buffer *src_buf, struct radv_meta_blit2d_surf *dst, 412 unsigned num_rects, struct radv_meta_blit2d_rect *rects) 413{ 414 bool use_3d = cmd_buffer->device->physical_device->rad_info.chip_class >= GFX9 && 415 (src_img && src_img->image->type == VK_IMAGE_TYPE_3D); 416 enum blit2d_src_type src_type = src_buf ? BLIT2D_SRC_TYPE_BUFFER 417 : use_3d ? BLIT2D_SRC_TYPE_IMAGE_3D 418 : BLIT2D_SRC_TYPE_IMAGE; 419 radv_meta_blit2d_normal_dst(cmd_buffer, src_img, src_buf, dst, num_rects, rects, src_type, 420 src_img ? util_logbase2(src_img->image->info.samples) : 0); 421} 422 423static nir_shader * 424build_nir_vertex_shader(void) 425{ 426 const struct glsl_type *vec4 = glsl_vec4_type(); 427 const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2); 428 nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_VERTEX, NULL, "meta_blit2d_vs"); 429 430 nir_variable *pos_out = nir_variable_create(b.shader, nir_var_shader_out, vec4, "gl_Position"); 431 pos_out->data.location = VARYING_SLOT_POS; 432 433 nir_variable *tex_pos_out = nir_variable_create(b.shader, nir_var_shader_out, vec2, "v_tex_pos"); 434 tex_pos_out->data.location = VARYING_SLOT_VAR0; 435 tex_pos_out->data.interpolation = INTERP_MODE_SMOOTH; 436 437 nir_ssa_def *outvec = radv_meta_gen_rect_vertices(&b); 438 nir_store_var(&b, pos_out, outvec, 0xf); 439 440 nir_ssa_def *src_box = nir_load_push_constant(&b, 4, 32, nir_imm_int(&b, 0), .range = 16); 441 nir_ssa_def *vertex_id = nir_load_vertex_id_zero_base(&b); 442 443 /* vertex 0 - src_x, src_y */ 444 /* vertex 1 - src_x, src_y+h */ 445 /* vertex 2 - src_x+w, src_y */ 446 /* so channel 0 is vertex_id != 2 ? src_x : src_x + w 447 channel 1 is vertex id != 1 ? src_y : src_y + w */ 448 449 nir_ssa_def *c0cmp = nir_ine(&b, vertex_id, nir_imm_int(&b, 2)); 450 nir_ssa_def *c1cmp = nir_ine(&b, vertex_id, nir_imm_int(&b, 1)); 451 452 nir_ssa_def *comp[2]; 453 comp[0] = nir_bcsel(&b, c0cmp, nir_channel(&b, src_box, 0), nir_channel(&b, src_box, 2)); 454 455 comp[1] = nir_bcsel(&b, c1cmp, nir_channel(&b, src_box, 1), nir_channel(&b, src_box, 3)); 456 nir_ssa_def *out_tex_vec = nir_vec(&b, comp, 2); 457 nir_store_var(&b, tex_pos_out, out_tex_vec, 0x3); 458 return b.shader; 459} 460 461typedef nir_ssa_def *(*texel_fetch_build_func)(struct nir_builder *, struct radv_device *, 462 nir_ssa_def *, bool, bool); 463 464static nir_ssa_def * 465build_nir_texel_fetch(struct nir_builder *b, struct radv_device *device, nir_ssa_def *tex_pos, 466 bool is_3d, bool is_multisampled) 467{ 468 enum glsl_sampler_dim dim = is_3d ? GLSL_SAMPLER_DIM_3D 469 : is_multisampled ? GLSL_SAMPLER_DIM_MS 470 : GLSL_SAMPLER_DIM_2D; 471 const struct glsl_type *sampler_type = glsl_sampler_type(dim, false, false, GLSL_TYPE_UINT); 472 nir_variable *sampler = nir_variable_create(b->shader, nir_var_uniform, sampler_type, "s_tex"); 473 sampler->data.descriptor_set = 0; 474 sampler->data.binding = 0; 475 476 nir_ssa_def *tex_pos_3d = NULL; 477 nir_ssa_def *sample_idx = NULL; 478 if (is_3d) { 479 nir_ssa_def *layer = 480 nir_load_push_constant(b, 1, 32, nir_imm_int(b, 0), .base = 16, .range = 4); 481 482 nir_ssa_def *chans[3]; 483 chans[0] = nir_channel(b, tex_pos, 0); 484 chans[1] = nir_channel(b, tex_pos, 1); 485 chans[2] = layer; 486 tex_pos_3d = nir_vec(b, chans, 3); 487 } 488 if (is_multisampled) { 489 sample_idx = nir_load_sample_id(b); 490 } 491 492 nir_ssa_def *tex_deref = &nir_build_deref_var(b, sampler)->dest.ssa; 493 494 nir_tex_instr *tex = nir_tex_instr_create(b->shader, is_multisampled ? 4 : 3); 495 tex->sampler_dim = dim; 496 tex->op = is_multisampled ? nir_texop_txf_ms : nir_texop_txf; 497 tex->src[0].src_type = nir_tex_src_coord; 498 tex->src[0].src = nir_src_for_ssa(is_3d ? tex_pos_3d : tex_pos); 499 tex->src[1].src_type = is_multisampled ? nir_tex_src_ms_index : nir_tex_src_lod; 500 tex->src[1].src = nir_src_for_ssa(is_multisampled ? sample_idx : nir_imm_int(b, 0)); 501 tex->src[2].src_type = nir_tex_src_texture_deref; 502 tex->src[2].src = nir_src_for_ssa(tex_deref); 503 if (is_multisampled) { 504 tex->src[3].src_type = nir_tex_src_lod; 505 tex->src[3].src = nir_src_for_ssa(nir_imm_int(b, 0)); 506 } 507 tex->dest_type = nir_type_uint32; 508 tex->is_array = false; 509 tex->coord_components = is_3d ? 3 : 2; 510 511 nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex"); 512 nir_builder_instr_insert(b, &tex->instr); 513 514 return &tex->dest.ssa; 515} 516 517static nir_ssa_def * 518build_nir_buffer_fetch(struct nir_builder *b, struct radv_device *device, nir_ssa_def *tex_pos, 519 bool is_3d, bool is_multisampled) 520{ 521 const struct glsl_type *sampler_type = 522 glsl_sampler_type(GLSL_SAMPLER_DIM_BUF, false, false, GLSL_TYPE_UINT); 523 nir_variable *sampler = nir_variable_create(b->shader, nir_var_uniform, sampler_type, "s_tex"); 524 sampler->data.descriptor_set = 0; 525 sampler->data.binding = 0; 526 527 nir_ssa_def *width = nir_load_push_constant(b, 1, 32, nir_imm_int(b, 0), .base = 16, .range = 4); 528 529 nir_ssa_def *pos_x = nir_channel(b, tex_pos, 0); 530 nir_ssa_def *pos_y = nir_channel(b, tex_pos, 1); 531 pos_y = nir_imul(b, pos_y, width); 532 pos_x = nir_iadd(b, pos_x, pos_y); 533 534 nir_ssa_def *tex_deref = &nir_build_deref_var(b, sampler)->dest.ssa; 535 536 nir_tex_instr *tex = nir_tex_instr_create(b->shader, 2); 537 tex->sampler_dim = GLSL_SAMPLER_DIM_BUF; 538 tex->op = nir_texop_txf; 539 tex->src[0].src_type = nir_tex_src_coord; 540 tex->src[0].src = nir_src_for_ssa(pos_x); 541 tex->src[1].src_type = nir_tex_src_texture_deref; 542 tex->src[1].src = nir_src_for_ssa(tex_deref); 543 tex->dest_type = nir_type_uint32; 544 tex->is_array = false; 545 tex->coord_components = 1; 546 547 nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex"); 548 nir_builder_instr_insert(b, &tex->instr); 549 550 return &tex->dest.ssa; 551} 552 553static const VkPipelineVertexInputStateCreateInfo normal_vi_create_info = { 554 .sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO, 555 .vertexBindingDescriptionCount = 0, 556 .vertexAttributeDescriptionCount = 0, 557}; 558 559static nir_shader * 560build_nir_copy_fragment_shader(struct radv_device *device, texel_fetch_build_func txf_func, 561 const char *name, bool is_3d, bool is_multisampled) 562{ 563 const struct glsl_type *vec4 = glsl_vec4_type(); 564 const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2); 565 nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_FRAGMENT, NULL, "%s", name); 566 567 nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in, vec2, "v_tex_pos"); 568 tex_pos_in->data.location = VARYING_SLOT_VAR0; 569 570 nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out, vec4, "f_color"); 571 color_out->data.location = FRAG_RESULT_DATA0; 572 573 nir_ssa_def *pos_int = nir_f2i32(&b, nir_load_var(&b, tex_pos_in)); 574 nir_ssa_def *tex_pos = nir_channels(&b, pos_int, 0x3); 575 576 nir_ssa_def *color = txf_func(&b, device, tex_pos, is_3d, is_multisampled); 577 nir_store_var(&b, color_out, color, 0xf); 578 579 return b.shader; 580} 581 582static nir_shader * 583build_nir_copy_fragment_shader_depth(struct radv_device *device, texel_fetch_build_func txf_func, 584 const char *name, bool is_3d, bool is_multisampled) 585{ 586 const struct glsl_type *vec4 = glsl_vec4_type(); 587 const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2); 588 nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_FRAGMENT, NULL, "%s", name); 589 590 nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in, vec2, "v_tex_pos"); 591 tex_pos_in->data.location = VARYING_SLOT_VAR0; 592 593 nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out, vec4, "f_color"); 594 color_out->data.location = FRAG_RESULT_DEPTH; 595 596 nir_ssa_def *pos_int = nir_f2i32(&b, nir_load_var(&b, tex_pos_in)); 597 nir_ssa_def *tex_pos = nir_channels(&b, pos_int, 0x3); 598 599 nir_ssa_def *color = txf_func(&b, device, tex_pos, is_3d, is_multisampled); 600 nir_store_var(&b, color_out, color, 0x1); 601 602 return b.shader; 603} 604 605static nir_shader * 606build_nir_copy_fragment_shader_stencil(struct radv_device *device, texel_fetch_build_func txf_func, 607 const char *name, bool is_3d, bool is_multisampled) 608{ 609 const struct glsl_type *vec4 = glsl_vec4_type(); 610 const struct glsl_type *vec2 = glsl_vector_type(GLSL_TYPE_FLOAT, 2); 611 nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_FRAGMENT, NULL, "%s", name); 612 613 nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in, vec2, "v_tex_pos"); 614 tex_pos_in->data.location = VARYING_SLOT_VAR0; 615 616 nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out, vec4, "f_color"); 617 color_out->data.location = FRAG_RESULT_STENCIL; 618 619 nir_ssa_def *pos_int = nir_f2i32(&b, nir_load_var(&b, tex_pos_in)); 620 nir_ssa_def *tex_pos = nir_channels(&b, pos_int, 0x3); 621 622 nir_ssa_def *color = txf_func(&b, device, tex_pos, is_3d, is_multisampled); 623 nir_store_var(&b, color_out, color, 0x1); 624 625 return b.shader; 626} 627 628void 629radv_device_finish_meta_blit2d_state(struct radv_device *device) 630{ 631 struct radv_meta_state *state = &device->meta_state; 632 633 for (unsigned j = 0; j < NUM_META_FS_KEYS; ++j) { 634 for (unsigned k = 0; k < RADV_META_DST_LAYOUT_COUNT; ++k) { 635 radv_DestroyRenderPass(radv_device_to_handle(device), state->blit2d_render_passes[j][k], 636 &state->alloc); 637 } 638 } 639 640 for (enum radv_blit_ds_layout j = RADV_BLIT_DS_LAYOUT_TILE_ENABLE; j < RADV_BLIT_DS_LAYOUT_COUNT; 641 j++) { 642 radv_DestroyRenderPass(radv_device_to_handle(device), state->blit2d_depth_only_rp[j], 643 &state->alloc); 644 radv_DestroyRenderPass(radv_device_to_handle(device), state->blit2d_stencil_only_rp[j], 645 &state->alloc); 646 } 647 648 for (unsigned log2_samples = 0; log2_samples < MAX_SAMPLES_LOG2; ++log2_samples) { 649 for (unsigned src = 0; src < BLIT2D_NUM_SRC_TYPES; src++) { 650 radv_DestroyPipelineLayout(radv_device_to_handle(device), 651 state->blit2d[log2_samples].p_layouts[src], &state->alloc); 652 radv_DestroyDescriptorSetLayout(radv_device_to_handle(device), 653 state->blit2d[log2_samples].ds_layouts[src], 654 &state->alloc); 655 656 for (unsigned j = 0; j < NUM_META_FS_KEYS; ++j) { 657 radv_DestroyPipeline(radv_device_to_handle(device), 658 state->blit2d[log2_samples].pipelines[src][j], &state->alloc); 659 } 660 661 radv_DestroyPipeline(radv_device_to_handle(device), 662 state->blit2d[log2_samples].depth_only_pipeline[src], &state->alloc); 663 radv_DestroyPipeline(radv_device_to_handle(device), 664 state->blit2d[log2_samples].stencil_only_pipeline[src], 665 &state->alloc); 666 } 667 } 668} 669 670static VkResult 671blit2d_init_color_pipeline(struct radv_device *device, enum blit2d_src_type src_type, 672 VkFormat format, uint32_t log2_samples) 673{ 674 VkResult result; 675 unsigned fs_key = radv_format_meta_fs_key(device, format); 676 const char *name; 677 678 mtx_lock(&device->meta_state.mtx); 679 if (device->meta_state.blit2d[log2_samples].pipelines[src_type][fs_key]) { 680 mtx_unlock(&device->meta_state.mtx); 681 return VK_SUCCESS; 682 } 683 684 texel_fetch_build_func src_func; 685 switch (src_type) { 686 case BLIT2D_SRC_TYPE_IMAGE: 687 src_func = build_nir_texel_fetch; 688 name = "meta_blit2d_image_fs"; 689 break; 690 case BLIT2D_SRC_TYPE_IMAGE_3D: 691 src_func = build_nir_texel_fetch; 692 name = "meta_blit3d_image_fs"; 693 break; 694 case BLIT2D_SRC_TYPE_BUFFER: 695 src_func = build_nir_buffer_fetch; 696 name = "meta_blit2d_buffer_fs"; 697 break; 698 default: 699 unreachable("unknown blit src type\n"); 700 break; 701 } 702 703 const VkPipelineVertexInputStateCreateInfo *vi_create_info; 704 nir_shader *fs = build_nir_copy_fragment_shader( 705 device, src_func, name, src_type == BLIT2D_SRC_TYPE_IMAGE_3D, log2_samples > 0); 706 nir_shader *vs = build_nir_vertex_shader(); 707 708 vi_create_info = &normal_vi_create_info; 709 710 VkPipelineShaderStageCreateInfo pipeline_shader_stages[] = { 711 {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, 712 .stage = VK_SHADER_STAGE_VERTEX_BIT, 713 .module = vk_shader_module_handle_from_nir(vs), 714 .pName = "main", 715 .pSpecializationInfo = NULL}, 716 {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, 717 .stage = VK_SHADER_STAGE_FRAGMENT_BIT, 718 .module = vk_shader_module_handle_from_nir(fs), 719 .pName = "main", 720 .pSpecializationInfo = NULL}, 721 }; 722 723 for (unsigned dst_layout = 0; dst_layout < RADV_META_DST_LAYOUT_COUNT; ++dst_layout) { 724 if (!device->meta_state.blit2d_render_passes[fs_key][dst_layout]) { 725 VkImageLayout layout = radv_meta_dst_layout_to_layout(dst_layout); 726 727 result = radv_CreateRenderPass2( 728 radv_device_to_handle(device), 729 &(VkRenderPassCreateInfo2){ 730 .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO_2, 731 .attachmentCount = 1, 732 .pAttachments = 733 &(VkAttachmentDescription2){ 734 .sType = VK_STRUCTURE_TYPE_ATTACHMENT_DESCRIPTION_2, 735 .format = format, 736 .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD, 737 .storeOp = VK_ATTACHMENT_STORE_OP_STORE, 738 .initialLayout = layout, 739 .finalLayout = layout, 740 }, 741 .subpassCount = 1, 742 .pSubpasses = 743 &(VkSubpassDescription2){ 744 .sType = VK_STRUCTURE_TYPE_SUBPASS_DESCRIPTION_2, 745 .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS, 746 .inputAttachmentCount = 0, 747 .colorAttachmentCount = 1, 748 .pColorAttachments = 749 &(VkAttachmentReference2){ 750 .sType = VK_STRUCTURE_TYPE_ATTACHMENT_REFERENCE_2, 751 .attachment = 0, 752 .layout = layout, 753 }, 754 .pResolveAttachments = NULL, 755 .pDepthStencilAttachment = 756 &(VkAttachmentReference2){ 757 .sType = VK_STRUCTURE_TYPE_ATTACHMENT_REFERENCE_2, 758 .attachment = VK_ATTACHMENT_UNUSED, 759 .layout = layout, 760 }, 761 .preserveAttachmentCount = 0, 762 .pPreserveAttachments = NULL, 763 }, 764 .dependencyCount = 2, 765 .pDependencies = 766 (VkSubpassDependency2[]){{.sType = VK_STRUCTURE_TYPE_SUBPASS_DEPENDENCY_2, 767 .srcSubpass = VK_SUBPASS_EXTERNAL, 768 .dstSubpass = 0, 769 .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, 770 .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 771 .srcAccessMask = 0, 772 .dstAccessMask = 0, 773 .dependencyFlags = 0}, 774 {.sType = VK_STRUCTURE_TYPE_SUBPASS_DEPENDENCY_2, 775 .srcSubpass = 0, 776 .dstSubpass = VK_SUBPASS_EXTERNAL, 777 .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, 778 .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 779 .srcAccessMask = 0, 780 .dstAccessMask = 0, 781 .dependencyFlags = 0}}, 782 }, 783 &device->meta_state.alloc, 784 &device->meta_state.blit2d_render_passes[fs_key][dst_layout]); 785 } 786 } 787 788 const VkGraphicsPipelineCreateInfo vk_pipeline_info = { 789 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, 790 .stageCount = ARRAY_SIZE(pipeline_shader_stages), 791 .pStages = pipeline_shader_stages, 792 .pVertexInputState = vi_create_info, 793 .pInputAssemblyState = 794 &(VkPipelineInputAssemblyStateCreateInfo){ 795 .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO, 796 .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP, 797 .primitiveRestartEnable = false, 798 }, 799 .pViewportState = 800 &(VkPipelineViewportStateCreateInfo){ 801 .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO, 802 .viewportCount = 1, 803 .scissorCount = 1, 804 }, 805 .pRasterizationState = 806 &(VkPipelineRasterizationStateCreateInfo){ 807 .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO, 808 .rasterizerDiscardEnable = false, 809 .polygonMode = VK_POLYGON_MODE_FILL, 810 .cullMode = VK_CULL_MODE_NONE, 811 .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE}, 812 .pMultisampleState = 813 &(VkPipelineMultisampleStateCreateInfo){ 814 .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO, 815 .rasterizationSamples = 1 << log2_samples, 816 .sampleShadingEnable = log2_samples > 1, 817 .minSampleShading = 1.0, 818 .pSampleMask = (VkSampleMask[]){UINT32_MAX}, 819 }, 820 .pColorBlendState = 821 &(VkPipelineColorBlendStateCreateInfo){ 822 .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO, 823 .attachmentCount = 1, 824 .pAttachments = 825 (VkPipelineColorBlendAttachmentState[]){ 826 {.colorWriteMask = VK_COLOR_COMPONENT_A_BIT | VK_COLOR_COMPONENT_R_BIT | 827 VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT}, 828 }}, 829 .pDynamicState = 830 &(VkPipelineDynamicStateCreateInfo){ 831 .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO, 832 .dynamicStateCount = 9, 833 .pDynamicStates = 834 (VkDynamicState[]){ 835 VK_DYNAMIC_STATE_VIEWPORT, 836 VK_DYNAMIC_STATE_SCISSOR, 837 VK_DYNAMIC_STATE_LINE_WIDTH, 838 VK_DYNAMIC_STATE_DEPTH_BIAS, 839 VK_DYNAMIC_STATE_BLEND_CONSTANTS, 840 VK_DYNAMIC_STATE_DEPTH_BOUNDS, 841 VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK, 842 VK_DYNAMIC_STATE_STENCIL_WRITE_MASK, 843 VK_DYNAMIC_STATE_STENCIL_REFERENCE, 844 }, 845 }, 846 .flags = 0, 847 .layout = device->meta_state.blit2d[log2_samples].p_layouts[src_type], 848 .renderPass = device->meta_state.blit2d_render_passes[fs_key][0], 849 .subpass = 0, 850 }; 851 852 const struct radv_graphics_pipeline_create_info radv_pipeline_info = {.use_rectlist = true}; 853 854 result = radv_graphics_pipeline_create( 855 radv_device_to_handle(device), radv_pipeline_cache_to_handle(&device->meta_state.cache), 856 &vk_pipeline_info, &radv_pipeline_info, &device->meta_state.alloc, 857 &device->meta_state.blit2d[log2_samples].pipelines[src_type][fs_key]); 858 859 ralloc_free(vs); 860 ralloc_free(fs); 861 862 mtx_unlock(&device->meta_state.mtx); 863 return result; 864} 865 866static VkResult 867blit2d_init_depth_only_pipeline(struct radv_device *device, enum blit2d_src_type src_type, 868 uint32_t log2_samples) 869{ 870 VkResult result; 871 const char *name; 872 873 mtx_lock(&device->meta_state.mtx); 874 if (device->meta_state.blit2d[log2_samples].depth_only_pipeline[src_type]) { 875 mtx_unlock(&device->meta_state.mtx); 876 return VK_SUCCESS; 877 } 878 879 texel_fetch_build_func src_func; 880 switch (src_type) { 881 case BLIT2D_SRC_TYPE_IMAGE: 882 src_func = build_nir_texel_fetch; 883 name = "meta_blit2d_depth_image_fs"; 884 break; 885 case BLIT2D_SRC_TYPE_IMAGE_3D: 886 src_func = build_nir_texel_fetch; 887 name = "meta_blit3d_depth_image_fs"; 888 break; 889 case BLIT2D_SRC_TYPE_BUFFER: 890 src_func = build_nir_buffer_fetch; 891 name = "meta_blit2d_depth_buffer_fs"; 892 break; 893 default: 894 unreachable("unknown blit src type\n"); 895 break; 896 } 897 898 const VkPipelineVertexInputStateCreateInfo *vi_create_info; 899 nir_shader *fs = build_nir_copy_fragment_shader_depth( 900 device, src_func, name, src_type == BLIT2D_SRC_TYPE_IMAGE_3D, log2_samples > 0); 901 nir_shader *vs = build_nir_vertex_shader(); 902 903 vi_create_info = &normal_vi_create_info; 904 905 VkPipelineShaderStageCreateInfo pipeline_shader_stages[] = { 906 {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, 907 .stage = VK_SHADER_STAGE_VERTEX_BIT, 908 .module = vk_shader_module_handle_from_nir(vs), 909 .pName = "main", 910 .pSpecializationInfo = NULL}, 911 {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, 912 .stage = VK_SHADER_STAGE_FRAGMENT_BIT, 913 .module = vk_shader_module_handle_from_nir(fs), 914 .pName = "main", 915 .pSpecializationInfo = NULL}, 916 }; 917 918 for (enum radv_blit_ds_layout ds_layout = RADV_BLIT_DS_LAYOUT_TILE_ENABLE; 919 ds_layout < RADV_BLIT_DS_LAYOUT_COUNT; ds_layout++) { 920 if (!device->meta_state.blit2d_depth_only_rp[ds_layout]) { 921 VkImageLayout layout = radv_meta_blit_ds_to_layout(ds_layout); 922 result = radv_CreateRenderPass2( 923 radv_device_to_handle(device), 924 &(VkRenderPassCreateInfo2){ 925 .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO_2, 926 .attachmentCount = 1, 927 .pAttachments = 928 &(VkAttachmentDescription2){ 929 .sType = VK_STRUCTURE_TYPE_ATTACHMENT_DESCRIPTION_2, 930 .format = VK_FORMAT_D32_SFLOAT, 931 .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD, 932 .storeOp = VK_ATTACHMENT_STORE_OP_STORE, 933 .initialLayout = layout, 934 .finalLayout = layout, 935 }, 936 .subpassCount = 1, 937 .pSubpasses = 938 &(VkSubpassDescription2){ 939 .sType = VK_STRUCTURE_TYPE_SUBPASS_DESCRIPTION_2, 940 .inputAttachmentCount = 0, 941 .colorAttachmentCount = 0, 942 .pColorAttachments = NULL, 943 .pResolveAttachments = NULL, 944 .pDepthStencilAttachment = 945 &(VkAttachmentReference2){ 946 .sType = VK_STRUCTURE_TYPE_ATTACHMENT_REFERENCE_2, 947 .attachment = 0, 948 .layout = layout, 949 }, 950 .preserveAttachmentCount = 0, 951 .pPreserveAttachments = NULL, 952 }, 953 .dependencyCount = 2, 954 .pDependencies = 955 (VkSubpassDependency2[]){{.sType = VK_STRUCTURE_TYPE_SUBPASS_DEPENDENCY_2, 956 .srcSubpass = VK_SUBPASS_EXTERNAL, 957 .dstSubpass = 0, 958 .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, 959 .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 960 .srcAccessMask = 0, 961 .dstAccessMask = 0, 962 .dependencyFlags = 0}, 963 {.sType = VK_STRUCTURE_TYPE_SUBPASS_DEPENDENCY_2, 964 .srcSubpass = 0, 965 .dstSubpass = VK_SUBPASS_EXTERNAL, 966 .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, 967 .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 968 .srcAccessMask = 0, 969 .dstAccessMask = 0, 970 .dependencyFlags = 0}}, 971 }, 972 &device->meta_state.alloc, &device->meta_state.blit2d_depth_only_rp[ds_layout]); 973 } 974 } 975 976 const VkGraphicsPipelineCreateInfo vk_pipeline_info = { 977 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, 978 .stageCount = ARRAY_SIZE(pipeline_shader_stages), 979 .pStages = pipeline_shader_stages, 980 .pVertexInputState = vi_create_info, 981 .pInputAssemblyState = 982 &(VkPipelineInputAssemblyStateCreateInfo){ 983 .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO, 984 .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP, 985 .primitiveRestartEnable = false, 986 }, 987 .pViewportState = 988 &(VkPipelineViewportStateCreateInfo){ 989 .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO, 990 .viewportCount = 1, 991 .scissorCount = 1, 992 }, 993 .pRasterizationState = 994 &(VkPipelineRasterizationStateCreateInfo){ 995 .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO, 996 .rasterizerDiscardEnable = false, 997 .polygonMode = VK_POLYGON_MODE_FILL, 998 .cullMode = VK_CULL_MODE_NONE, 999 .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE}, 1000 .pMultisampleState = 1001 &(VkPipelineMultisampleStateCreateInfo){ 1002 .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO, 1003 .rasterizationSamples = 1 << log2_samples, 1004 .sampleShadingEnable = false, 1005 .pSampleMask = (VkSampleMask[]){UINT32_MAX}, 1006 }, 1007 .pColorBlendState = 1008 &(VkPipelineColorBlendStateCreateInfo){ 1009 .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO, 1010 .attachmentCount = 0, 1011 .pAttachments = NULL, 1012 }, 1013 .pDepthStencilState = 1014 &(VkPipelineDepthStencilStateCreateInfo){ 1015 .sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO, 1016 .depthTestEnable = true, 1017 .depthWriteEnable = true, 1018 .depthCompareOp = VK_COMPARE_OP_ALWAYS, 1019 }, 1020 .pDynamicState = 1021 &(VkPipelineDynamicStateCreateInfo){ 1022 .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO, 1023 .dynamicStateCount = 9, 1024 .pDynamicStates = 1025 (VkDynamicState[]){ 1026 VK_DYNAMIC_STATE_VIEWPORT, 1027 VK_DYNAMIC_STATE_SCISSOR, 1028 VK_DYNAMIC_STATE_LINE_WIDTH, 1029 VK_DYNAMIC_STATE_DEPTH_BIAS, 1030 VK_DYNAMIC_STATE_BLEND_CONSTANTS, 1031 VK_DYNAMIC_STATE_DEPTH_BOUNDS, 1032 VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK, 1033 VK_DYNAMIC_STATE_STENCIL_WRITE_MASK, 1034 VK_DYNAMIC_STATE_STENCIL_REFERENCE, 1035 }, 1036 }, 1037 .flags = 0, 1038 .layout = device->meta_state.blit2d[log2_samples].p_layouts[src_type], 1039 .renderPass = device->meta_state.blit2d_depth_only_rp[0], 1040 .subpass = 0, 1041 }; 1042 1043 const struct radv_graphics_pipeline_create_info radv_pipeline_info = {.use_rectlist = true}; 1044 1045 result = radv_graphics_pipeline_create( 1046 radv_device_to_handle(device), radv_pipeline_cache_to_handle(&device->meta_state.cache), 1047 &vk_pipeline_info, &radv_pipeline_info, &device->meta_state.alloc, 1048 &device->meta_state.blit2d[log2_samples].depth_only_pipeline[src_type]); 1049 1050 ralloc_free(vs); 1051 ralloc_free(fs); 1052 1053 mtx_unlock(&device->meta_state.mtx); 1054 return result; 1055} 1056 1057static VkResult 1058blit2d_init_stencil_only_pipeline(struct radv_device *device, enum blit2d_src_type src_type, 1059 uint32_t log2_samples) 1060{ 1061 VkResult result; 1062 const char *name; 1063 1064 mtx_lock(&device->meta_state.mtx); 1065 if (device->meta_state.blit2d[log2_samples].stencil_only_pipeline[src_type]) { 1066 mtx_unlock(&device->meta_state.mtx); 1067 return VK_SUCCESS; 1068 } 1069 1070 texel_fetch_build_func src_func; 1071 switch (src_type) { 1072 case BLIT2D_SRC_TYPE_IMAGE: 1073 src_func = build_nir_texel_fetch; 1074 name = "meta_blit2d_stencil_image_fs"; 1075 break; 1076 case BLIT2D_SRC_TYPE_IMAGE_3D: 1077 src_func = build_nir_texel_fetch; 1078 name = "meta_blit3d_stencil_image_fs"; 1079 break; 1080 case BLIT2D_SRC_TYPE_BUFFER: 1081 src_func = build_nir_buffer_fetch; 1082 name = "meta_blit2d_stencil_buffer_fs"; 1083 break; 1084 default: 1085 unreachable("unknown blit src type\n"); 1086 break; 1087 } 1088 1089 const VkPipelineVertexInputStateCreateInfo *vi_create_info; 1090 nir_shader *fs = build_nir_copy_fragment_shader_stencil( 1091 device, src_func, name, src_type == BLIT2D_SRC_TYPE_IMAGE_3D, log2_samples > 0); 1092 nir_shader *vs = build_nir_vertex_shader(); 1093 1094 vi_create_info = &normal_vi_create_info; 1095 1096 VkPipelineShaderStageCreateInfo pipeline_shader_stages[] = { 1097 {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, 1098 .stage = VK_SHADER_STAGE_VERTEX_BIT, 1099 .module = vk_shader_module_handle_from_nir(vs), 1100 .pName = "main", 1101 .pSpecializationInfo = NULL}, 1102 {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, 1103 .stage = VK_SHADER_STAGE_FRAGMENT_BIT, 1104 .module = vk_shader_module_handle_from_nir(fs), 1105 .pName = "main", 1106 .pSpecializationInfo = NULL}, 1107 }; 1108 1109 for (enum radv_blit_ds_layout ds_layout = RADV_BLIT_DS_LAYOUT_TILE_ENABLE; 1110 ds_layout < RADV_BLIT_DS_LAYOUT_COUNT; ds_layout++) { 1111 if (!device->meta_state.blit2d_stencil_only_rp[ds_layout]) { 1112 VkImageLayout layout = radv_meta_blit_ds_to_layout(ds_layout); 1113 result = radv_CreateRenderPass2( 1114 radv_device_to_handle(device), 1115 &(VkRenderPassCreateInfo2){ 1116 .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO_2, 1117 .attachmentCount = 1, 1118 .pAttachments = 1119 &(VkAttachmentDescription2){ 1120 .sType = VK_STRUCTURE_TYPE_ATTACHMENT_DESCRIPTION_2, 1121 .format = VK_FORMAT_S8_UINT, 1122 .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD, 1123 .storeOp = VK_ATTACHMENT_STORE_OP_STORE, 1124 .initialLayout = layout, 1125 .finalLayout = layout, 1126 }, 1127 .subpassCount = 1, 1128 .pSubpasses = 1129 &(VkSubpassDescription2){ 1130 .sType = VK_STRUCTURE_TYPE_SUBPASS_DESCRIPTION_2, 1131 .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS, 1132 .inputAttachmentCount = 0, 1133 .colorAttachmentCount = 0, 1134 .pColorAttachments = NULL, 1135 .pResolveAttachments = NULL, 1136 .pDepthStencilAttachment = 1137 &(VkAttachmentReference2){ 1138 .sType = VK_STRUCTURE_TYPE_ATTACHMENT_REFERENCE_2, 1139 .attachment = 0, 1140 .layout = layout, 1141 }, 1142 .preserveAttachmentCount = 0, 1143 .pPreserveAttachments = NULL, 1144 }, 1145 .dependencyCount = 2, 1146 .pDependencies = 1147 (VkSubpassDependency2[]){{.sType = VK_STRUCTURE_TYPE_SUBPASS_DEPENDENCY_2, 1148 .srcSubpass = VK_SUBPASS_EXTERNAL, 1149 .dstSubpass = 0, 1150 .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, 1151 .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 1152 .srcAccessMask = 0, 1153 .dstAccessMask = 0, 1154 .dependencyFlags = 0}, 1155 {.sType = VK_STRUCTURE_TYPE_SUBPASS_DEPENDENCY_2, 1156 .srcSubpass = 0, 1157 .dstSubpass = VK_SUBPASS_EXTERNAL, 1158 .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, 1159 .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 1160 .srcAccessMask = 0, 1161 .dstAccessMask = 0, 1162 .dependencyFlags = 0}}, 1163 }, 1164 &device->meta_state.alloc, &device->meta_state.blit2d_stencil_only_rp[ds_layout]); 1165 } 1166 } 1167 1168 const VkGraphicsPipelineCreateInfo vk_pipeline_info = { 1169 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, 1170 .stageCount = ARRAY_SIZE(pipeline_shader_stages), 1171 .pStages = pipeline_shader_stages, 1172 .pVertexInputState = vi_create_info, 1173 .pInputAssemblyState = 1174 &(VkPipelineInputAssemblyStateCreateInfo){ 1175 .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO, 1176 .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP, 1177 .primitiveRestartEnable = false, 1178 }, 1179 .pViewportState = 1180 &(VkPipelineViewportStateCreateInfo){ 1181 .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO, 1182 .viewportCount = 1, 1183 .scissorCount = 1, 1184 }, 1185 .pRasterizationState = 1186 &(VkPipelineRasterizationStateCreateInfo){ 1187 .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO, 1188 .rasterizerDiscardEnable = false, 1189 .polygonMode = VK_POLYGON_MODE_FILL, 1190 .cullMode = VK_CULL_MODE_NONE, 1191 .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE}, 1192 .pMultisampleState = 1193 &(VkPipelineMultisampleStateCreateInfo){ 1194 .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO, 1195 .rasterizationSamples = 1 << log2_samples, 1196 .sampleShadingEnable = false, 1197 .pSampleMask = (VkSampleMask[]){UINT32_MAX}, 1198 }, 1199 .pColorBlendState = 1200 &(VkPipelineColorBlendStateCreateInfo){ 1201 .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO, 1202 .attachmentCount = 0, 1203 .pAttachments = NULL, 1204 }, 1205 .pDepthStencilState = 1206 &(VkPipelineDepthStencilStateCreateInfo){ 1207 .sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO, 1208 .depthTestEnable = false, 1209 .depthWriteEnable = false, 1210 .stencilTestEnable = true, 1211 .front = {.failOp = VK_STENCIL_OP_REPLACE, 1212 .passOp = VK_STENCIL_OP_REPLACE, 1213 .depthFailOp = VK_STENCIL_OP_REPLACE, 1214 .compareOp = VK_COMPARE_OP_ALWAYS, 1215 .compareMask = 0xff, 1216 .writeMask = 0xff, 1217 .reference = 0}, 1218 .back = {.failOp = VK_STENCIL_OP_REPLACE, 1219 .passOp = VK_STENCIL_OP_REPLACE, 1220 .depthFailOp = VK_STENCIL_OP_REPLACE, 1221 .compareOp = VK_COMPARE_OP_ALWAYS, 1222 .compareMask = 0xff, 1223 .writeMask = 0xff, 1224 .reference = 0}, 1225 .depthCompareOp = VK_COMPARE_OP_ALWAYS, 1226 }, 1227 .pDynamicState = 1228 &(VkPipelineDynamicStateCreateInfo){ 1229 .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO, 1230 .dynamicStateCount = 6, 1231 .pDynamicStates = 1232 (VkDynamicState[]){ 1233 VK_DYNAMIC_STATE_VIEWPORT, 1234 VK_DYNAMIC_STATE_SCISSOR, 1235 VK_DYNAMIC_STATE_LINE_WIDTH, 1236 VK_DYNAMIC_STATE_DEPTH_BIAS, 1237 VK_DYNAMIC_STATE_BLEND_CONSTANTS, 1238 VK_DYNAMIC_STATE_DEPTH_BOUNDS, 1239 }, 1240 }, 1241 .flags = 0, 1242 .layout = device->meta_state.blit2d[log2_samples].p_layouts[src_type], 1243 .renderPass = device->meta_state.blit2d_stencil_only_rp[0], 1244 .subpass = 0, 1245 }; 1246 1247 const struct radv_graphics_pipeline_create_info radv_pipeline_info = {.use_rectlist = true}; 1248 1249 result = radv_graphics_pipeline_create( 1250 radv_device_to_handle(device), radv_pipeline_cache_to_handle(&device->meta_state.cache), 1251 &vk_pipeline_info, &radv_pipeline_info, &device->meta_state.alloc, 1252 &device->meta_state.blit2d[log2_samples].stencil_only_pipeline[src_type]); 1253 1254 ralloc_free(vs); 1255 ralloc_free(fs); 1256 1257 mtx_unlock(&device->meta_state.mtx); 1258 return result; 1259} 1260 1261static VkResult 1262meta_blit2d_create_pipe_layout(struct radv_device *device, int idx, uint32_t log2_samples) 1263{ 1264 VkResult result; 1265 VkDescriptorType desc_type = (idx == BLIT2D_SRC_TYPE_BUFFER) 1266 ? VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER 1267 : VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE; 1268 const VkPushConstantRange push_constant_ranges[] = { 1269 {VK_SHADER_STAGE_VERTEX_BIT, 0, 16}, 1270 {VK_SHADER_STAGE_FRAGMENT_BIT, 16, 4}, 1271 }; 1272 int num_push_constant_range = (idx != BLIT2D_SRC_TYPE_IMAGE || log2_samples > 0) ? 2 : 1; 1273 1274 result = radv_CreateDescriptorSetLayout( 1275 radv_device_to_handle(device), 1276 &(VkDescriptorSetLayoutCreateInfo){ 1277 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, 1278 .flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR, 1279 .bindingCount = 1, 1280 .pBindings = 1281 (VkDescriptorSetLayoutBinding[]){ 1282 {.binding = 0, 1283 .descriptorType = desc_type, 1284 .descriptorCount = 1, 1285 .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT, 1286 .pImmutableSamplers = NULL}, 1287 }}, 1288 &device->meta_state.alloc, &device->meta_state.blit2d[log2_samples].ds_layouts[idx]); 1289 if (result != VK_SUCCESS) 1290 goto fail; 1291 1292 result = radv_CreatePipelineLayout( 1293 radv_device_to_handle(device), 1294 &(VkPipelineLayoutCreateInfo){ 1295 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, 1296 .setLayoutCount = 1, 1297 .pSetLayouts = &device->meta_state.blit2d[log2_samples].ds_layouts[idx], 1298 .pushConstantRangeCount = num_push_constant_range, 1299 .pPushConstantRanges = push_constant_ranges, 1300 }, 1301 &device->meta_state.alloc, &device->meta_state.blit2d[log2_samples].p_layouts[idx]); 1302 if (result != VK_SUCCESS) 1303 goto fail; 1304 return VK_SUCCESS; 1305fail: 1306 return result; 1307} 1308 1309VkResult 1310radv_device_init_meta_blit2d_state(struct radv_device *device, bool on_demand) 1311{ 1312 VkResult result; 1313 bool create_3d = device->physical_device->rad_info.chip_class >= GFX9; 1314 1315 for (unsigned log2_samples = 0; log2_samples < MAX_SAMPLES_LOG2; log2_samples++) { 1316 for (unsigned src = 0; src < BLIT2D_NUM_SRC_TYPES; src++) { 1317 if (src == BLIT2D_SRC_TYPE_IMAGE_3D && !create_3d) 1318 continue; 1319 1320 /* Don't need to handle copies between buffers and multisample images. */ 1321 if (src == BLIT2D_SRC_TYPE_BUFFER && log2_samples > 0) 1322 continue; 1323 1324 /* There are no multisampled 3D images. */ 1325 if (src == BLIT2D_SRC_TYPE_IMAGE_3D && log2_samples > 0) 1326 continue; 1327 1328 result = meta_blit2d_create_pipe_layout(device, src, log2_samples); 1329 if (result != VK_SUCCESS) 1330 goto fail; 1331 1332 if (on_demand) 1333 continue; 1334 1335 for (unsigned j = 0; j < NUM_META_FS_KEYS; ++j) { 1336 result = blit2d_init_color_pipeline(device, src, radv_fs_key_format_exemplars[j], 1337 log2_samples); 1338 if (result != VK_SUCCESS) 1339 goto fail; 1340 } 1341 1342 result = blit2d_init_depth_only_pipeline(device, src, log2_samples); 1343 if (result != VK_SUCCESS) 1344 goto fail; 1345 1346 result = blit2d_init_stencil_only_pipeline(device, src, log2_samples); 1347 if (result != VK_SUCCESS) 1348 goto fail; 1349 } 1350 } 1351 1352 return VK_SUCCESS; 1353 1354fail: 1355 radv_device_finish_meta_blit2d_state(device); 1356 return result; 1357} 1358