1/* 2 * Copyright © 2016 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 */ 23 24#include <assert.h> 25#include <stdbool.h> 26 27#include "nir/nir_builder.h" 28#include "radv_meta.h" 29#include "radv_private.h" 30#include "sid.h" 31#include "vk_format.h" 32 33/* emit 0, 0, 0, 1 */ 34static nir_shader * 35build_nir_fs(void) 36{ 37 const struct glsl_type *vec4 = glsl_vec4_type(); 38 nir_variable *f_color; /* vec4, fragment output color */ 39 40 nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_FRAGMENT, NULL, "meta_resolve_fs"); 41 42 f_color = nir_variable_create(b.shader, nir_var_shader_out, vec4, "f_color"); 43 f_color->data.location = FRAG_RESULT_DATA0; 44 nir_store_var(&b, f_color, nir_imm_vec4(&b, 0.0, 0.0, 0.0, 1.0), 0xf); 45 46 return b.shader; 47} 48 49static VkResult 50create_pass(struct radv_device *device, VkFormat vk_format, VkRenderPass *pass) 51{ 52 VkResult result; 53 VkDevice device_h = radv_device_to_handle(device); 54 const VkAllocationCallbacks *alloc = &device->meta_state.alloc; 55 VkAttachmentDescription2 attachments[2]; 56 int i; 57 58 for (i = 0; i < 2; i++) { 59 attachments[i].sType = VK_STRUCTURE_TYPE_ATTACHMENT_DESCRIPTION_2; 60 attachments[i].pNext = NULL; 61 attachments[i].format = vk_format; 62 attachments[i].samples = 1; 63 attachments[i].loadOp = VK_ATTACHMENT_LOAD_OP_LOAD; 64 attachments[i].storeOp = VK_ATTACHMENT_STORE_OP_STORE; 65 } 66 attachments[0].initialLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; 67 attachments[0].finalLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; 68 attachments[1].initialLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; 69 attachments[1].finalLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; 70 71 result = radv_CreateRenderPass2( 72 device_h, 73 &(VkRenderPassCreateInfo2){ 74 .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO_2, 75 .attachmentCount = 2, 76 .pAttachments = attachments, 77 .subpassCount = 1, 78 .pSubpasses = 79 &(VkSubpassDescription2){ 80 .sType = VK_STRUCTURE_TYPE_SUBPASS_DESCRIPTION_2, 81 .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS, 82 .inputAttachmentCount = 0, 83 .colorAttachmentCount = 2, 84 .pColorAttachments = 85 (VkAttachmentReference2[]){ 86 { 87 .sType = VK_STRUCTURE_TYPE_ATTACHMENT_REFERENCE_2, 88 .attachment = 0, 89 .layout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, 90 }, 91 { 92 .sType = VK_STRUCTURE_TYPE_ATTACHMENT_REFERENCE_2, 93 .attachment = 1, 94 .layout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, 95 }, 96 }, 97 .pResolveAttachments = NULL, 98 .pDepthStencilAttachment = 99 &(VkAttachmentReference2){ 100 .sType = VK_STRUCTURE_TYPE_ATTACHMENT_REFERENCE_2, 101 .attachment = VK_ATTACHMENT_UNUSED, 102 }, 103 .preserveAttachmentCount = 0, 104 .pPreserveAttachments = NULL, 105 }, 106 .dependencyCount = 2, 107 .pDependencies = 108 (VkSubpassDependency2[]){{.sType = VK_STRUCTURE_TYPE_SUBPASS_DEPENDENCY_2, 109 .srcSubpass = VK_SUBPASS_EXTERNAL, 110 .dstSubpass = 0, 111 .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, 112 .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 113 .srcAccessMask = 0, 114 .dstAccessMask = 0, 115 .dependencyFlags = 0}, 116 {.sType = VK_STRUCTURE_TYPE_SUBPASS_DEPENDENCY_2, 117 .srcSubpass = 0, 118 .dstSubpass = VK_SUBPASS_EXTERNAL, 119 .srcStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, 120 .dstStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 121 .srcAccessMask = 0, 122 .dstAccessMask = 0, 123 .dependencyFlags = 0}}, 124 }, 125 alloc, pass); 126 127 return result; 128} 129 130static VkResult 131create_pipeline(struct radv_device *device, VkShaderModule vs_module_h, VkPipeline *pipeline, 132 VkRenderPass pass) 133{ 134 VkResult result; 135 VkDevice device_h = radv_device_to_handle(device); 136 137 nir_shader *fs_module = build_nir_fs(); 138 if (!fs_module) { 139 /* XXX: Need more accurate error */ 140 result = VK_ERROR_OUT_OF_HOST_MEMORY; 141 goto cleanup; 142 } 143 144 VkPipelineLayoutCreateInfo pl_create_info = { 145 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, 146 .setLayoutCount = 0, 147 .pSetLayouts = NULL, 148 .pushConstantRangeCount = 0, 149 .pPushConstantRanges = NULL, 150 }; 151 152 if (!device->meta_state.resolve.p_layout) { 153 result = 154 radv_CreatePipelineLayout(radv_device_to_handle(device), &pl_create_info, 155 &device->meta_state.alloc, &device->meta_state.resolve.p_layout); 156 if (result != VK_SUCCESS) 157 goto cleanup; 158 } 159 160 result = radv_graphics_pipeline_create( 161 device_h, radv_pipeline_cache_to_handle(&device->meta_state.cache), 162 &(VkGraphicsPipelineCreateInfo){ 163 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, 164 .stageCount = 2, 165 .pStages = 166 (VkPipelineShaderStageCreateInfo[]){ 167 { 168 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, 169 .stage = VK_SHADER_STAGE_VERTEX_BIT, 170 .module = vs_module_h, 171 .pName = "main", 172 }, 173 { 174 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, 175 .stage = VK_SHADER_STAGE_FRAGMENT_BIT, 176 .module = vk_shader_module_handle_from_nir(fs_module), 177 .pName = "main", 178 }, 179 }, 180 .pVertexInputState = 181 &(VkPipelineVertexInputStateCreateInfo){ 182 .sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO, 183 .vertexBindingDescriptionCount = 0, 184 .vertexAttributeDescriptionCount = 0, 185 }, 186 .pInputAssemblyState = 187 &(VkPipelineInputAssemblyStateCreateInfo){ 188 .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO, 189 .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP, 190 .primitiveRestartEnable = false, 191 }, 192 .pViewportState = 193 &(VkPipelineViewportStateCreateInfo){ 194 .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO, 195 .viewportCount = 1, 196 .scissorCount = 1, 197 }, 198 .pRasterizationState = 199 &(VkPipelineRasterizationStateCreateInfo){ 200 .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO, 201 .depthClampEnable = false, 202 .rasterizerDiscardEnable = false, 203 .polygonMode = VK_POLYGON_MODE_FILL, 204 .cullMode = VK_CULL_MODE_NONE, 205 .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE, 206 }, 207 .pMultisampleState = 208 &(VkPipelineMultisampleStateCreateInfo){ 209 .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO, 210 .rasterizationSamples = 1, 211 .sampleShadingEnable = false, 212 .pSampleMask = NULL, 213 .alphaToCoverageEnable = false, 214 .alphaToOneEnable = false, 215 }, 216 .pColorBlendState = 217 &(VkPipelineColorBlendStateCreateInfo){ 218 .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO, 219 .logicOpEnable = false, 220 .attachmentCount = 2, 221 .pAttachments = 222 (VkPipelineColorBlendAttachmentState[]){ 223 { 224 .colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | 225 VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT, 226 }, 227 { 228 .colorWriteMask = 0, 229 230 }}, 231 }, 232 .pDynamicState = 233 &(VkPipelineDynamicStateCreateInfo){ 234 .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO, 235 .dynamicStateCount = 2, 236 .pDynamicStates = 237 (VkDynamicState[]){ 238 VK_DYNAMIC_STATE_VIEWPORT, 239 VK_DYNAMIC_STATE_SCISSOR, 240 }, 241 }, 242 .layout = device->meta_state.resolve.p_layout, 243 .renderPass = pass, 244 .subpass = 0, 245 }, 246 &(struct radv_graphics_pipeline_create_info){ 247 .use_rectlist = true, 248 .custom_blend_mode = V_028808_CB_RESOLVE, 249 }, 250 &device->meta_state.alloc, pipeline); 251 if (result != VK_SUCCESS) 252 goto cleanup; 253 254 goto cleanup; 255 256cleanup: 257 ralloc_free(fs_module); 258 return result; 259} 260 261void 262radv_device_finish_meta_resolve_state(struct radv_device *device) 263{ 264 struct radv_meta_state *state = &device->meta_state; 265 266 for (uint32_t j = 0; j < NUM_META_FS_KEYS; j++) { 267 radv_DestroyRenderPass(radv_device_to_handle(device), state->resolve.pass[j], &state->alloc); 268 radv_DestroyPipeline(radv_device_to_handle(device), state->resolve.pipeline[j], 269 &state->alloc); 270 } 271 radv_DestroyPipelineLayout(radv_device_to_handle(device), state->resolve.p_layout, 272 &state->alloc); 273} 274 275VkResult 276radv_device_init_meta_resolve_state(struct radv_device *device, bool on_demand) 277{ 278 if (on_demand) 279 return VK_SUCCESS; 280 281 VkResult res = VK_SUCCESS; 282 struct radv_meta_state *state = &device->meta_state; 283 nir_shader *vs_module = radv_meta_build_nir_vs_generate_vertices(); 284 if (!vs_module) { 285 /* XXX: Need more accurate error */ 286 res = VK_ERROR_OUT_OF_HOST_MEMORY; 287 goto fail; 288 } 289 290 for (uint32_t i = 0; i < NUM_META_FS_KEYS; ++i) { 291 VkFormat format = radv_fs_key_format_exemplars[i]; 292 unsigned fs_key = radv_format_meta_fs_key(device, format); 293 res = create_pass(device, format, &state->resolve.pass[fs_key]); 294 if (res != VK_SUCCESS) 295 goto fail; 296 297 VkShaderModule vs_module_h = vk_shader_module_handle_from_nir(vs_module); 298 res = create_pipeline(device, vs_module_h, &state->resolve.pipeline[fs_key], 299 state->resolve.pass[fs_key]); 300 if (res != VK_SUCCESS) 301 goto fail; 302 } 303 304 goto cleanup; 305 306fail: 307 radv_device_finish_meta_resolve_state(device); 308 309cleanup: 310 ralloc_free(vs_module); 311 312 return res; 313} 314 315static void 316emit_resolve(struct radv_cmd_buffer *cmd_buffer, const struct radv_image *src_image, 317 const struct radv_image *dst_image, VkFormat vk_format, const VkOffset2D *dest_offset, 318 const VkExtent2D *resolve_extent) 319{ 320 struct radv_device *device = cmd_buffer->device; 321 VkCommandBuffer cmd_buffer_h = radv_cmd_buffer_to_handle(cmd_buffer); 322 unsigned fs_key = radv_format_meta_fs_key(device, vk_format); 323 324 cmd_buffer->state.flush_bits |= 325 radv_src_access_flush(cmd_buffer, VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, src_image) | 326 radv_dst_access_flush(cmd_buffer, VK_ACCESS_COLOR_ATTACHMENT_READ_BIT, src_image) | 327 radv_dst_access_flush(cmd_buffer, VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, dst_image); 328 329 radv_CmdBindPipeline(cmd_buffer_h, VK_PIPELINE_BIND_POINT_GRAPHICS, 330 device->meta_state.resolve.pipeline[fs_key]); 331 332 radv_CmdSetViewport(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1, 333 &(VkViewport){.x = dest_offset->x, 334 .y = dest_offset->y, 335 .width = resolve_extent->width, 336 .height = resolve_extent->height, 337 .minDepth = 0.0f, 338 .maxDepth = 1.0f}); 339 340 radv_CmdSetScissor(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1, 341 &(VkRect2D){ 342 .offset = *dest_offset, 343 .extent = *resolve_extent, 344 }); 345 346 radv_CmdDraw(cmd_buffer_h, 3, 1, 0, 0); 347 cmd_buffer->state.flush_bits |= 348 radv_src_access_flush(cmd_buffer, VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, dst_image); 349} 350 351enum radv_resolve_method { 352 RESOLVE_HW, 353 RESOLVE_COMPUTE, 354 RESOLVE_FRAGMENT, 355}; 356 357static bool 358image_hw_resolve_compat(const struct radv_device *device, struct radv_image *src_image, 359 struct radv_image *dst_image) 360{ 361 if (device->physical_device->rad_info.chip_class >= GFX9) { 362 return dst_image->planes[0].surface.u.gfx9.swizzle_mode == 363 src_image->planes[0].surface.u.gfx9.swizzle_mode; 364 } else { 365 return dst_image->planes[0].surface.micro_tile_mode == 366 src_image->planes[0].surface.micro_tile_mode; 367 } 368} 369 370static void 371radv_pick_resolve_method_images(struct radv_device *device, struct radv_image *src_image, 372 VkFormat src_format, struct radv_image *dest_image, 373 unsigned dest_level, VkImageLayout dest_image_layout, 374 bool dest_render_loop, struct radv_cmd_buffer *cmd_buffer, 375 enum radv_resolve_method *method) 376 377{ 378 uint32_t queue_mask = radv_image_queue_family_mask(dest_image, cmd_buffer->queue_family_index, 379 cmd_buffer->queue_family_index); 380 381 if (vk_format_is_color(src_format)) { 382 /* Using the fragment resolve path is currently a hint to 383 * avoid decompressing DCC for partial resolves and 384 * re-initialize it after resolving using compute. 385 * TODO: Add support for layered and int to the fragment path. 386 */ 387 if (radv_layout_dcc_compressed(device, dest_image, dest_level, dest_image_layout, 388 dest_render_loop, queue_mask)) { 389 *method = RESOLVE_FRAGMENT; 390 } else if (!image_hw_resolve_compat(device, src_image, dest_image)) { 391 /* The micro tile mode only needs to match for the HW 392 * resolve path which is the default path for non-DCC 393 * resolves. 394 */ 395 *method = RESOLVE_COMPUTE; 396 } 397 398 if (src_format == VK_FORMAT_R16G16_UNORM || src_format == VK_FORMAT_R16G16_SNORM) 399 *method = RESOLVE_COMPUTE; 400 else if (vk_format_is_int(src_format)) 401 *method = RESOLVE_COMPUTE; 402 else if (src_image->info.array_size > 1 || dest_image->info.array_size > 1) 403 *method = RESOLVE_COMPUTE; 404 } else { 405 if (src_image->info.array_size > 1 || dest_image->info.array_size > 1) 406 *method = RESOLVE_COMPUTE; 407 else 408 *method = RESOLVE_FRAGMENT; 409 } 410} 411 412static VkResult 413build_resolve_pipeline(struct radv_device *device, unsigned fs_key) 414{ 415 VkResult result = VK_SUCCESS; 416 417 if (device->meta_state.resolve.pipeline[fs_key]) 418 return result; 419 420 mtx_lock(&device->meta_state.mtx); 421 if (device->meta_state.resolve.pipeline[fs_key]) { 422 mtx_unlock(&device->meta_state.mtx); 423 return result; 424 } 425 426 nir_shader *vs_module = radv_meta_build_nir_vs_generate_vertices(); 427 428 result = create_pass(device, radv_fs_key_format_exemplars[fs_key], 429 &device->meta_state.resolve.pass[fs_key]); 430 if (result != VK_SUCCESS) 431 goto fail; 432 433 VkShaderModule vs_module_h = vk_shader_module_handle_from_nir(vs_module); 434 result = create_pipeline(device, vs_module_h, &device->meta_state.resolve.pipeline[fs_key], 435 device->meta_state.resolve.pass[fs_key]); 436 437fail: 438 ralloc_free(vs_module); 439 mtx_unlock(&device->meta_state.mtx); 440 return result; 441} 442 443static void 444radv_meta_resolve_hardware_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image, 445 VkImageLayout src_image_layout, struct radv_image *dst_image, 446 VkImageLayout dst_image_layout, const VkImageResolve2KHR *region) 447{ 448 struct radv_device *device = cmd_buffer->device; 449 struct radv_meta_saved_state saved_state; 450 451 radv_meta_save(&saved_state, cmd_buffer, RADV_META_SAVE_GRAPHICS_PIPELINE); 452 453 assert(src_image->info.samples > 1); 454 assert(dst_image->info.samples == 1); 455 456 unsigned fs_key = radv_format_meta_fs_key(device, dst_image->vk_format); 457 458 /* From the Vulkan 1.0 spec: 459 * 460 * - The aspectMask member of srcSubresource and dstSubresource must 461 * only contain VK_IMAGE_ASPECT_COLOR_BIT 462 * 463 * - The layerCount member of srcSubresource and dstSubresource must 464 * match 465 */ 466 assert(region->srcSubresource.aspectMask == VK_IMAGE_ASPECT_COLOR_BIT); 467 assert(region->dstSubresource.aspectMask == VK_IMAGE_ASPECT_COLOR_BIT); 468 assert(region->srcSubresource.layerCount == region->dstSubresource.layerCount); 469 470 const uint32_t src_base_layer = 471 radv_meta_get_iview_layer(src_image, ®ion->srcSubresource, ®ion->srcOffset); 472 473 const uint32_t dst_base_layer = 474 radv_meta_get_iview_layer(dst_image, ®ion->dstSubresource, ®ion->dstOffset); 475 476 /** 477 * From Vulkan 1.0.6 spec: 18.6 Resolving Multisample Images 478 * 479 * extent is the size in texels of the source image to resolve in width, 480 * height and depth. 1D images use only x and width. 2D images use x, y, 481 * width and height. 3D images use x, y, z, width, height and depth. 482 * 483 * srcOffset and dstOffset select the initial x, y, and z offsets in 484 * texels of the sub-regions of the source and destination image data. 485 * extent is the size in texels of the source image to resolve in width, 486 * height and depth. 1D images use only x and width. 2D images use x, y, 487 * width and height. 3D images use x, y, z, width, height and depth. 488 */ 489 const struct VkExtent3D extent = radv_sanitize_image_extent(src_image->type, region->extent); 490 const struct VkOffset3D dstOffset = 491 radv_sanitize_image_offset(dst_image->type, region->dstOffset); 492 493 uint32_t queue_mask = radv_image_queue_family_mask(dst_image, cmd_buffer->queue_family_index, 494 cmd_buffer->queue_family_index); 495 496 if (radv_layout_dcc_compressed(cmd_buffer->device, dst_image, region->dstSubresource.mipLevel, 497 dst_image_layout, false, queue_mask)) { 498 VkImageSubresourceRange range = { 499 .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, 500 .baseMipLevel = region->dstSubresource.mipLevel, 501 .levelCount = 1, 502 .baseArrayLayer = dst_base_layer, 503 .layerCount = region->dstSubresource.layerCount, 504 }; 505 506 cmd_buffer->state.flush_bits |= radv_init_dcc(cmd_buffer, dst_image, &range, 0xffffffff); 507 } 508 509 for (uint32_t layer = 0; layer < region->srcSubresource.layerCount; ++layer) { 510 511 VkResult ret = build_resolve_pipeline(device, fs_key); 512 if (ret != VK_SUCCESS) { 513 cmd_buffer->record_result = ret; 514 break; 515 } 516 517 struct radv_image_view src_iview; 518 radv_image_view_init(&src_iview, cmd_buffer->device, 519 &(VkImageViewCreateInfo){ 520 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, 521 .image = radv_image_to_handle(src_image), 522 .viewType = radv_meta_get_view_type(src_image), 523 .format = src_image->vk_format, 524 .subresourceRange = 525 { 526 .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, 527 .baseMipLevel = region->srcSubresource.mipLevel, 528 .levelCount = 1, 529 .baseArrayLayer = src_base_layer + layer, 530 .layerCount = 1, 531 }, 532 }, 533 NULL); 534 535 struct radv_image_view dst_iview; 536 radv_image_view_init(&dst_iview, cmd_buffer->device, 537 &(VkImageViewCreateInfo){ 538 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, 539 .image = radv_image_to_handle(dst_image), 540 .viewType = radv_meta_get_view_type(dst_image), 541 .format = dst_image->vk_format, 542 .subresourceRange = 543 { 544 .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, 545 .baseMipLevel = region->dstSubresource.mipLevel, 546 .levelCount = 1, 547 .baseArrayLayer = dst_base_layer + layer, 548 .layerCount = 1, 549 }, 550 }, 551 NULL); 552 553 VkFramebuffer fb_h; 554 radv_CreateFramebuffer( 555 radv_device_to_handle(device), 556 &(VkFramebufferCreateInfo){ 557 .sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO, 558 .attachmentCount = 2, 559 .pAttachments = 560 (VkImageView[]){ 561 radv_image_view_to_handle(&src_iview), 562 radv_image_view_to_handle(&dst_iview), 563 }, 564 .width = radv_minify(dst_image->info.width, region->dstSubresource.mipLevel), 565 .height = radv_minify(dst_image->info.height, region->dstSubresource.mipLevel), 566 .layers = 1}, 567 &cmd_buffer->pool->alloc, &fb_h); 568 569 radv_cmd_buffer_begin_render_pass(cmd_buffer, 570 &(VkRenderPassBeginInfo){ 571 .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO, 572 .renderPass = device->meta_state.resolve.pass[fs_key], 573 .framebuffer = fb_h, 574 .renderArea = {.offset = 575 { 576 dstOffset.x, 577 dstOffset.y, 578 }, 579 .extent = 580 { 581 extent.width, 582 extent.height, 583 }}, 584 .clearValueCount = 0, 585 .pClearValues = NULL, 586 }, 587 NULL); 588 589 radv_cmd_buffer_set_subpass(cmd_buffer, &cmd_buffer->state.pass->subpasses[0]); 590 591 emit_resolve(cmd_buffer, src_image, dst_image, dst_iview.vk_format, 592 &(VkOffset2D){ 593 .x = dstOffset.x, 594 .y = dstOffset.y, 595 }, 596 &(VkExtent2D){ 597 .width = extent.width, 598 .height = extent.height, 599 }); 600 601 radv_cmd_buffer_end_render_pass(cmd_buffer); 602 603 radv_image_view_finish(&src_iview); 604 radv_image_view_finish(&dst_iview); 605 radv_DestroyFramebuffer(radv_device_to_handle(device), fb_h, &cmd_buffer->pool->alloc); 606 } 607 608 radv_meta_restore(&saved_state, cmd_buffer); 609} 610 611static void 612resolve_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image, 613 VkImageLayout src_image_layout, struct radv_image *dst_image, 614 VkImageLayout dst_image_layout, const VkImageResolve2KHR *region, 615 enum radv_resolve_method resolve_method) 616{ 617 switch (resolve_method) { 618 case RESOLVE_HW: 619 radv_meta_resolve_hardware_image(cmd_buffer, src_image, src_image_layout, dst_image, 620 dst_image_layout, region); 621 break; 622 case RESOLVE_FRAGMENT: 623 radv_meta_resolve_fragment_image(cmd_buffer, src_image, src_image_layout, dst_image, 624 dst_image_layout, region); 625 break; 626 case RESOLVE_COMPUTE: 627 radv_meta_resolve_compute_image(cmd_buffer, src_image, src_image->vk_format, src_image_layout, 628 dst_image, dst_image->vk_format, dst_image_layout, region); 629 break; 630 default: 631 assert(!"Invalid resolve method selected"); 632 } 633} 634 635void 636radv_CmdResolveImage2KHR(VkCommandBuffer commandBuffer, 637 const VkResolveImageInfo2KHR *pResolveImageInfo) 638{ 639 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); 640 RADV_FROM_HANDLE(radv_image, src_image, pResolveImageInfo->srcImage); 641 RADV_FROM_HANDLE(radv_image, dst_image, pResolveImageInfo->dstImage); 642 VkImageLayout src_image_layout = pResolveImageInfo->srcImageLayout; 643 VkImageLayout dst_image_layout = pResolveImageInfo->dstImageLayout; 644 enum radv_resolve_method resolve_method = RESOLVE_HW; 645 /* we can use the hw resolve only for single full resolves */ 646 if (pResolveImageInfo->regionCount == 1) { 647 if (pResolveImageInfo->pRegions[0].srcOffset.x || 648 pResolveImageInfo->pRegions[0].srcOffset.y || pResolveImageInfo->pRegions[0].srcOffset.z) 649 resolve_method = RESOLVE_COMPUTE; 650 if (pResolveImageInfo->pRegions[0].dstOffset.x || 651 pResolveImageInfo->pRegions[0].dstOffset.y || pResolveImageInfo->pRegions[0].dstOffset.z) 652 resolve_method = RESOLVE_COMPUTE; 653 654 if (pResolveImageInfo->pRegions[0].extent.width != src_image->info.width || 655 pResolveImageInfo->pRegions[0].extent.height != src_image->info.height || 656 pResolveImageInfo->pRegions[0].extent.depth != src_image->info.depth) 657 resolve_method = RESOLVE_COMPUTE; 658 } else 659 resolve_method = RESOLVE_COMPUTE; 660 661 for (uint32_t r = 0; r < pResolveImageInfo->regionCount; r++) { 662 const VkImageResolve2KHR *region = &pResolveImageInfo->pRegions[r]; 663 664 radv_pick_resolve_method_images(cmd_buffer->device, src_image, src_image->vk_format, dst_image, 665 region->dstSubresource.mipLevel, dst_image_layout, false, 666 cmd_buffer, &resolve_method); 667 668 resolve_image(cmd_buffer, src_image, src_image_layout, dst_image, dst_image_layout, region, 669 resolve_method); 670 } 671} 672 673static void 674radv_cmd_buffer_resolve_subpass_hw(struct radv_cmd_buffer *cmd_buffer) 675{ 676 struct radv_framebuffer *fb = cmd_buffer->state.framebuffer; 677 const struct radv_subpass *subpass = cmd_buffer->state.subpass; 678 struct radv_meta_saved_state saved_state; 679 680 radv_meta_save(&saved_state, cmd_buffer, RADV_META_SAVE_GRAPHICS_PIPELINE); 681 682 for (uint32_t i = 0; i < subpass->color_count; ++i) { 683 struct radv_subpass_attachment src_att = subpass->color_attachments[i]; 684 struct radv_subpass_attachment dest_att = subpass->resolve_attachments[i]; 685 686 if (dest_att.attachment == VK_ATTACHMENT_UNUSED) 687 continue; 688 689 struct radv_image_view *src_iview = cmd_buffer->state.attachments[src_att.attachment].iview; 690 struct radv_image *src_img = src_iview->image; 691 692 struct radv_image_view *dest_iview = cmd_buffer->state.attachments[dest_att.attachment].iview; 693 struct radv_image *dst_img = dest_iview->image; 694 VkImageLayout dst_image_layout = cmd_buffer->state.attachments[dest_att.attachment].current_layout; 695 696 uint32_t queue_mask = radv_image_queue_family_mask(dst_img, cmd_buffer->queue_family_index, 697 cmd_buffer->queue_family_index); 698 699 if (radv_layout_dcc_compressed(cmd_buffer->device, dst_img, dest_iview->base_mip, 700 dst_image_layout, false, queue_mask)) { 701 VkImageSubresourceRange range = { 702 .aspectMask = dest_iview->aspect_mask, 703 .baseMipLevel = dest_iview->base_mip, 704 .levelCount = dest_iview->level_count, 705 .baseArrayLayer = dest_iview->base_layer, 706 .layerCount = dest_iview->layer_count, 707 }; 708 709 cmd_buffer->state.flush_bits |= radv_init_dcc(cmd_buffer, dst_img, &range, 0xffffffff); 710 cmd_buffer->state.attachments[dest_att.attachment].current_layout = 711 VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; 712 } 713 714 struct radv_subpass resolve_subpass = { 715 .color_count = 2, 716 .color_attachments = (struct radv_subpass_attachment[]){src_att, dest_att}, 717 .depth_stencil_attachment = NULL, 718 }; 719 720 radv_cmd_buffer_set_subpass(cmd_buffer, &resolve_subpass); 721 722 VkResult ret = build_resolve_pipeline( 723 cmd_buffer->device, radv_format_meta_fs_key(cmd_buffer->device, dest_iview->vk_format)); 724 if (ret != VK_SUCCESS) { 725 cmd_buffer->record_result = ret; 726 continue; 727 } 728 729 emit_resolve(cmd_buffer, src_img, dst_img, dest_iview->vk_format, &(VkOffset2D){0, 0}, 730 &(VkExtent2D){fb->width, fb->height}); 731 } 732 733 radv_cmd_buffer_restore_subpass(cmd_buffer, subpass); 734 735 radv_meta_restore(&saved_state, cmd_buffer); 736} 737 738/** 739 * Emit any needed resolves for the current subpass. 740 */ 741void 742radv_cmd_buffer_resolve_subpass(struct radv_cmd_buffer *cmd_buffer) 743{ 744 const struct radv_subpass *subpass = cmd_buffer->state.subpass; 745 enum radv_resolve_method resolve_method = RESOLVE_HW; 746 747 if (!subpass->has_color_resolve && !subpass->ds_resolve_attachment) 748 return; 749 750 radv_describe_begin_render_pass_resolve(cmd_buffer); 751 752 if (subpass->ds_resolve_attachment) { 753 struct radv_subpass_attachment src_att = *subpass->depth_stencil_attachment; 754 struct radv_subpass_attachment dst_att = *subpass->ds_resolve_attachment; 755 struct radv_image_view *src_iview = cmd_buffer->state.attachments[src_att.attachment].iview; 756 struct radv_image_view *dst_iview = cmd_buffer->state.attachments[dst_att.attachment].iview; 757 758 /* Make sure to not clear the depth/stencil attachment after resolves. */ 759 cmd_buffer->state.attachments[dst_att.attachment].pending_clear_aspects = 0; 760 761 radv_pick_resolve_method_images(cmd_buffer->device, src_iview->image, src_iview->vk_format, 762 dst_iview->image, dst_iview->base_mip, dst_att.layout, 763 dst_att.in_render_loop, cmd_buffer, &resolve_method); 764 765 if ((src_iview->aspect_mask & VK_IMAGE_ASPECT_DEPTH_BIT) && 766 subpass->depth_resolve_mode != VK_RESOLVE_MODE_NONE_KHR) { 767 if (resolve_method == RESOLVE_FRAGMENT) { 768 radv_depth_stencil_resolve_subpass_fs(cmd_buffer, VK_IMAGE_ASPECT_DEPTH_BIT, 769 subpass->depth_resolve_mode); 770 } else { 771 assert(resolve_method == RESOLVE_COMPUTE); 772 radv_depth_stencil_resolve_subpass_cs(cmd_buffer, VK_IMAGE_ASPECT_DEPTH_BIT, 773 subpass->depth_resolve_mode); 774 } 775 } 776 777 if ((src_iview->aspect_mask & VK_IMAGE_ASPECT_STENCIL_BIT) && 778 subpass->stencil_resolve_mode != VK_RESOLVE_MODE_NONE_KHR) { 779 if (resolve_method == RESOLVE_FRAGMENT) { 780 radv_depth_stencil_resolve_subpass_fs(cmd_buffer, VK_IMAGE_ASPECT_STENCIL_BIT, 781 subpass->stencil_resolve_mode); 782 } else { 783 assert(resolve_method == RESOLVE_COMPUTE); 784 radv_depth_stencil_resolve_subpass_cs(cmd_buffer, VK_IMAGE_ASPECT_STENCIL_BIT, 785 subpass->stencil_resolve_mode); 786 } 787 } 788 789 /* From the Vulkan spec 1.2.165: 790 * 791 * "VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT specifies 792 * write access to a color, resolve, or depth/stencil 793 * resolve attachment during a render pass or via 794 * certain subpass load and store operations." 795 * 796 * Yes, it's counterintuitive but it makes sense because ds 797 * resolve operations happen late at the end of the subpass. 798 * 799 * That said, RADV is wrong because it executes the subpass 800 * end barrier *before* any subpass resolves instead of after. 801 * 802 * TODO: Fix this properly by executing subpass end barriers 803 * after subpass resolves. 804 */ 805 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_DB; 806 if (radv_image_has_htile(dst_iview->image)) 807 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_DB_META; 808 } 809 810 if (subpass->has_color_resolve) { 811 for (uint32_t i = 0; i < subpass->color_count; ++i) { 812 struct radv_subpass_attachment src_att = subpass->color_attachments[i]; 813 struct radv_subpass_attachment dest_att = subpass->resolve_attachments[i]; 814 815 if (dest_att.attachment == VK_ATTACHMENT_UNUSED) 816 continue; 817 818 /* Make sure to not clear color attachments after resolves. */ 819 cmd_buffer->state.attachments[dest_att.attachment].pending_clear_aspects = 0; 820 821 struct radv_image_view *dst_iview = 822 cmd_buffer->state.attachments[dest_att.attachment].iview; 823 struct radv_image *dst_img = dst_iview->image; 824 struct radv_image_view *src_iview = 825 cmd_buffer->state.attachments[src_att.attachment].iview; 826 struct radv_image *src_img = src_iview->image; 827 828 radv_pick_resolve_method_images(cmd_buffer->device, src_img, src_iview->vk_format, dst_img, 829 dst_iview->base_mip, dest_att.layout, 830 dest_att.in_render_loop, cmd_buffer, &resolve_method); 831 832 if (resolve_method == RESOLVE_FRAGMENT) { 833 break; 834 } 835 } 836 837 switch (resolve_method) { 838 case RESOLVE_HW: 839 radv_cmd_buffer_resolve_subpass_hw(cmd_buffer); 840 break; 841 case RESOLVE_COMPUTE: 842 radv_cmd_buffer_resolve_subpass_cs(cmd_buffer); 843 break; 844 case RESOLVE_FRAGMENT: 845 radv_cmd_buffer_resolve_subpass_fs(cmd_buffer); 846 break; 847 default: 848 unreachable("Invalid resolve method"); 849 } 850 } 851 852 radv_describe_end_render_pass_resolve(cmd_buffer); 853} 854 855/** 856 * Decompress CMask/FMask before resolving a multisampled source image inside a 857 * subpass. 858 */ 859void 860radv_decompress_resolve_subpass_src(struct radv_cmd_buffer *cmd_buffer) 861{ 862 const struct radv_subpass *subpass = cmd_buffer->state.subpass; 863 struct radv_framebuffer *fb = cmd_buffer->state.framebuffer; 864 uint32_t layer_count = fb->layers; 865 866 if (subpass->view_mask) 867 layer_count = util_last_bit(subpass->view_mask); 868 869 for (uint32_t i = 0; i < subpass->color_count; ++i) { 870 struct radv_subpass_attachment src_att = subpass->color_attachments[i]; 871 struct radv_subpass_attachment dest_att = subpass->resolve_attachments[i]; 872 873 if (dest_att.attachment == VK_ATTACHMENT_UNUSED) 874 continue; 875 876 struct radv_image_view *src_iview = cmd_buffer->state.attachments[src_att.attachment].iview; 877 struct radv_image *src_image = src_iview->image; 878 879 VkImageResolve2KHR region = {0}; 880 region.sType = VK_STRUCTURE_TYPE_IMAGE_RESOLVE_2_KHR; 881 region.srcSubresource.aspectMask = src_iview->aspect_mask; 882 region.srcSubresource.mipLevel = 0; 883 region.srcSubresource.baseArrayLayer = src_iview->base_layer; 884 region.srcSubresource.layerCount = layer_count; 885 886 radv_decompress_resolve_src(cmd_buffer, src_image, src_att.layout, ®ion); 887 } 888} 889 890static struct radv_sample_locations_state * 891radv_get_resolve_sample_locations(struct radv_cmd_buffer *cmd_buffer) 892{ 893 struct radv_cmd_state *state = &cmd_buffer->state; 894 uint32_t subpass_id = radv_get_subpass_id(cmd_buffer); 895 896 for (uint32_t i = 0; i < state->num_subpass_sample_locs; i++) { 897 if (state->subpass_sample_locs[i].subpass_idx == subpass_id) 898 return &state->subpass_sample_locs[i].sample_location; 899 } 900 901 return NULL; 902} 903 904/** 905 * Decompress CMask/FMask before resolving a multisampled source image. 906 */ 907void 908radv_decompress_resolve_src(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image, 909 VkImageLayout src_image_layout, const VkImageResolve2KHR *region) 910{ 911 const uint32_t src_base_layer = 912 radv_meta_get_iview_layer(src_image, ®ion->srcSubresource, ®ion->srcOffset); 913 914 VkImageMemoryBarrier barrier = {0}; 915 barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; 916 barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; 917 barrier.oldLayout = src_image_layout; 918 barrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; 919 barrier.image = radv_image_to_handle(src_image); 920 barrier.subresourceRange = (VkImageSubresourceRange){ 921 .aspectMask = region->srcSubresource.aspectMask, 922 .baseMipLevel = region->srcSubresource.mipLevel, 923 .levelCount = 1, 924 .baseArrayLayer = src_base_layer, 925 .layerCount = region->srcSubresource.layerCount, 926 }; 927 928 if (src_image->flags & VK_IMAGE_CREATE_SAMPLE_LOCATIONS_COMPATIBLE_DEPTH_BIT_EXT) { 929 /* If the depth/stencil image uses different sample 930 * locations, we need them during HTILE decompressions. 931 */ 932 struct radv_sample_locations_state *sample_locs = 933 radv_get_resolve_sample_locations(cmd_buffer); 934 935 barrier.pNext = &(VkSampleLocationsInfoEXT){ 936 .sType = VK_STRUCTURE_TYPE_SAMPLE_LOCATIONS_INFO_EXT, 937 .sampleLocationsPerPixel = sample_locs->per_pixel, 938 .sampleLocationGridSize = sample_locs->grid_size, 939 .sampleLocationsCount = sample_locs->count, 940 .pSampleLocations = sample_locs->locations, 941 }; 942 } 943 944 radv_CmdPipelineBarrier(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, 945 VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, false, 0, NULL, 0, NULL, 1, 946 &barrier); 947} 948