anv_blorp.c revision 01e04c3f
1/* 2 * Copyright © 2016 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 */ 23 24#include "anv_private.h" 25 26static bool 27lookup_blorp_shader(struct blorp_context *blorp, 28 const void *key, uint32_t key_size, 29 uint32_t *kernel_out, void *prog_data_out) 30{ 31 struct anv_device *device = blorp->driver_ctx; 32 33 /* The default cache must be a real cache */ 34 assert(device->default_pipeline_cache.cache); 35 36 struct anv_shader_bin *bin = 37 anv_pipeline_cache_search(&device->default_pipeline_cache, key, key_size); 38 if (!bin) 39 return false; 40 41 /* The cache already has a reference and it's not going anywhere so there 42 * is no need to hold a second reference. 43 */ 44 anv_shader_bin_unref(device, bin); 45 46 *kernel_out = bin->kernel.offset; 47 *(const struct brw_stage_prog_data **)prog_data_out = bin->prog_data; 48 49 return true; 50} 51 52static bool 53upload_blorp_shader(struct blorp_context *blorp, 54 const void *key, uint32_t key_size, 55 const void *kernel, uint32_t kernel_size, 56 const struct brw_stage_prog_data *prog_data, 57 uint32_t prog_data_size, 58 uint32_t *kernel_out, void *prog_data_out) 59{ 60 struct anv_device *device = blorp->driver_ctx; 61 62 /* The blorp cache must be a real cache */ 63 assert(device->default_pipeline_cache.cache); 64 65 struct anv_pipeline_bind_map bind_map = { 66 .surface_count = 0, 67 .sampler_count = 0, 68 }; 69 70 struct anv_shader_bin *bin = 71 anv_pipeline_cache_upload_kernel(&device->default_pipeline_cache, 72 key, key_size, kernel, kernel_size, 73 NULL, 0, 74 prog_data, prog_data_size, &bind_map); 75 76 if (!bin) 77 return false; 78 79 /* The cache already has a reference and it's not going anywhere so there 80 * is no need to hold a second reference. 81 */ 82 anv_shader_bin_unref(device, bin); 83 84 *kernel_out = bin->kernel.offset; 85 *(const struct brw_stage_prog_data **)prog_data_out = bin->prog_data; 86 87 return true; 88} 89 90void 91anv_device_init_blorp(struct anv_device *device) 92{ 93 blorp_init(&device->blorp, device, &device->isl_dev); 94 device->blorp.compiler = device->instance->physicalDevice.compiler; 95 device->blorp.lookup_shader = lookup_blorp_shader; 96 device->blorp.upload_shader = upload_blorp_shader; 97 switch (device->info.gen) { 98 case 7: 99 if (device->info.is_haswell) { 100 device->blorp.exec = gen75_blorp_exec; 101 } else { 102 device->blorp.exec = gen7_blorp_exec; 103 } 104 break; 105 case 8: 106 device->blorp.exec = gen8_blorp_exec; 107 break; 108 case 9: 109 device->blorp.exec = gen9_blorp_exec; 110 break; 111 case 10: 112 device->blorp.exec = gen10_blorp_exec; 113 break; 114 case 11: 115 device->blorp.exec = gen11_blorp_exec; 116 break; 117 default: 118 unreachable("Unknown hardware generation"); 119 } 120} 121 122void 123anv_device_finish_blorp(struct anv_device *device) 124{ 125 blorp_finish(&device->blorp); 126} 127 128static void 129get_blorp_surf_for_anv_buffer(struct anv_device *device, 130 struct anv_buffer *buffer, uint64_t offset, 131 uint32_t width, uint32_t height, 132 uint32_t row_pitch, enum isl_format format, 133 struct blorp_surf *blorp_surf, 134 struct isl_surf *isl_surf) 135{ 136 const struct isl_format_layout *fmtl = 137 isl_format_get_layout(format); 138 bool ok UNUSED; 139 140 /* ASTC is the only format which doesn't support linear layouts. 141 * Create an equivalently sized surface with ISL to get around this. 142 */ 143 if (fmtl->txc == ISL_TXC_ASTC) { 144 /* Use an equivalently sized format */ 145 format = ISL_FORMAT_R32G32B32A32_UINT; 146 assert(fmtl->bpb == isl_format_get_layout(format)->bpb); 147 148 /* Shrink the dimensions for the new format */ 149 width = DIV_ROUND_UP(width, fmtl->bw); 150 height = DIV_ROUND_UP(height, fmtl->bh); 151 } 152 153 *blorp_surf = (struct blorp_surf) { 154 .surf = isl_surf, 155 .addr = { 156 .buffer = buffer->address.bo, 157 .offset = buffer->address.offset + offset, 158 .mocs = anv_mocs_for_bo(device, buffer->address.bo), 159 }, 160 }; 161 162 ok = isl_surf_init(&device->isl_dev, isl_surf, 163 .dim = ISL_SURF_DIM_2D, 164 .format = format, 165 .width = width, 166 .height = height, 167 .depth = 1, 168 .levels = 1, 169 .array_len = 1, 170 .samples = 1, 171 .row_pitch_B = row_pitch, 172 .usage = ISL_SURF_USAGE_TEXTURE_BIT | 173 ISL_SURF_USAGE_RENDER_TARGET_BIT, 174 .tiling_flags = ISL_TILING_LINEAR_BIT); 175 assert(ok); 176} 177 178/* Pick something high enough that it won't be used in core and low enough it 179 * will never map to an extension. 180 */ 181#define ANV_IMAGE_LAYOUT_EXPLICIT_AUX (VkImageLayout)10000000 182 183static struct blorp_address 184anv_to_blorp_address(struct anv_address addr) 185{ 186 return (struct blorp_address) { 187 .buffer = addr.bo, 188 .offset = addr.offset, 189 }; 190} 191 192static void 193get_blorp_surf_for_anv_image(const struct anv_device *device, 194 const struct anv_image *image, 195 VkImageAspectFlags aspect, 196 VkImageLayout layout, 197 enum isl_aux_usage aux_usage, 198 struct blorp_surf *blorp_surf) 199{ 200 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect); 201 202 if (layout != ANV_IMAGE_LAYOUT_EXPLICIT_AUX) 203 aux_usage = anv_layout_to_aux_usage(&device->info, image, aspect, layout); 204 205 const struct anv_surface *surface = &image->planes[plane].surface; 206 *blorp_surf = (struct blorp_surf) { 207 .surf = &surface->isl, 208 .addr = { 209 .buffer = image->planes[plane].address.bo, 210 .offset = image->planes[plane].address.offset + surface->offset, 211 .mocs = anv_mocs_for_bo(device, image->planes[plane].address.bo), 212 }, 213 }; 214 215 if (aux_usage != ISL_AUX_USAGE_NONE) { 216 const struct anv_surface *aux_surface = &image->planes[plane].aux_surface; 217 blorp_surf->aux_surf = &aux_surface->isl, 218 blorp_surf->aux_addr = (struct blorp_address) { 219 .buffer = image->planes[plane].address.bo, 220 .offset = image->planes[plane].address.offset + aux_surface->offset, 221 .mocs = anv_mocs_for_bo(device, image->planes[plane].address.bo), 222 }; 223 blorp_surf->aux_usage = aux_usage; 224 225 /* If we're doing a partial resolve, then we need the indirect clear 226 * color. If we are doing a fast clear and want to store/update the 227 * clear color, we also pass the address to blorp, otherwise it will only 228 * stomp the CCS to a particular value and won't care about format or 229 * clear value 230 */ 231 if (aspect & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) { 232 const struct anv_address clear_color_addr = 233 anv_image_get_clear_color_addr(device, image, aspect); 234 blorp_surf->clear_color_addr = anv_to_blorp_address(clear_color_addr); 235 } else if (aspect & VK_IMAGE_ASPECT_DEPTH_BIT 236 && device->info.gen >= 10) { 237 /* Vulkan always clears to 1.0. On gen < 10, we set that directly in 238 * the state packet. For gen >= 10, must provide the clear value in a 239 * buffer. We have a single global buffer that stores the 1.0 value. 240 */ 241 const struct anv_address clear_color_addr = (struct anv_address) { 242 .bo = (struct anv_bo *)&device->hiz_clear_bo 243 }; 244 blorp_surf->clear_color_addr = anv_to_blorp_address(clear_color_addr); 245 } 246 } 247} 248 249void anv_CmdCopyImage( 250 VkCommandBuffer commandBuffer, 251 VkImage srcImage, 252 VkImageLayout srcImageLayout, 253 VkImage dstImage, 254 VkImageLayout dstImageLayout, 255 uint32_t regionCount, 256 const VkImageCopy* pRegions) 257{ 258 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer); 259 ANV_FROM_HANDLE(anv_image, src_image, srcImage); 260 ANV_FROM_HANDLE(anv_image, dst_image, dstImage); 261 262 struct blorp_batch batch; 263 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0); 264 265 for (unsigned r = 0; r < regionCount; r++) { 266 VkOffset3D srcOffset = 267 anv_sanitize_image_offset(src_image->type, pRegions[r].srcOffset); 268 VkOffset3D dstOffset = 269 anv_sanitize_image_offset(dst_image->type, pRegions[r].dstOffset); 270 VkExtent3D extent = 271 anv_sanitize_image_extent(src_image->type, pRegions[r].extent); 272 273 const uint32_t dst_level = pRegions[r].dstSubresource.mipLevel; 274 unsigned dst_base_layer, layer_count; 275 if (dst_image->type == VK_IMAGE_TYPE_3D) { 276 dst_base_layer = pRegions[r].dstOffset.z; 277 layer_count = pRegions[r].extent.depth; 278 } else { 279 dst_base_layer = pRegions[r].dstSubresource.baseArrayLayer; 280 layer_count = 281 anv_get_layerCount(dst_image, &pRegions[r].dstSubresource); 282 } 283 284 const uint32_t src_level = pRegions[r].srcSubresource.mipLevel; 285 unsigned src_base_layer; 286 if (src_image->type == VK_IMAGE_TYPE_3D) { 287 src_base_layer = pRegions[r].srcOffset.z; 288 } else { 289 src_base_layer = pRegions[r].srcSubresource.baseArrayLayer; 290 assert(layer_count == 291 anv_get_layerCount(src_image, &pRegions[r].srcSubresource)); 292 } 293 294 VkImageAspectFlags src_mask = pRegions[r].srcSubresource.aspectMask, 295 dst_mask = pRegions[r].dstSubresource.aspectMask; 296 297 assert(anv_image_aspects_compatible(src_mask, dst_mask)); 298 299 if (util_bitcount(src_mask) > 1) { 300 uint32_t aspect_bit; 301 anv_foreach_image_aspect_bit(aspect_bit, src_image, src_mask) { 302 struct blorp_surf src_surf, dst_surf; 303 get_blorp_surf_for_anv_image(cmd_buffer->device, 304 src_image, 1UL << aspect_bit, 305 srcImageLayout, ISL_AUX_USAGE_NONE, 306 &src_surf); 307 get_blorp_surf_for_anv_image(cmd_buffer->device, 308 dst_image, 1UL << aspect_bit, 309 dstImageLayout, ISL_AUX_USAGE_NONE, 310 &dst_surf); 311 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image, 312 1UL << aspect_bit, 313 dst_surf.aux_usage, dst_level, 314 dst_base_layer, layer_count); 315 316 for (unsigned i = 0; i < layer_count; i++) { 317 blorp_copy(&batch, &src_surf, src_level, src_base_layer + i, 318 &dst_surf, dst_level, dst_base_layer + i, 319 srcOffset.x, srcOffset.y, 320 dstOffset.x, dstOffset.y, 321 extent.width, extent.height); 322 } 323 } 324 } else { 325 struct blorp_surf src_surf, dst_surf; 326 get_blorp_surf_for_anv_image(cmd_buffer->device, src_image, src_mask, 327 srcImageLayout, ISL_AUX_USAGE_NONE, 328 &src_surf); 329 get_blorp_surf_for_anv_image(cmd_buffer->device, dst_image, dst_mask, 330 dstImageLayout, ISL_AUX_USAGE_NONE, 331 &dst_surf); 332 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image, dst_mask, 333 dst_surf.aux_usage, dst_level, 334 dst_base_layer, layer_count); 335 336 for (unsigned i = 0; i < layer_count; i++) { 337 blorp_copy(&batch, &src_surf, src_level, src_base_layer + i, 338 &dst_surf, dst_level, dst_base_layer + i, 339 srcOffset.x, srcOffset.y, 340 dstOffset.x, dstOffset.y, 341 extent.width, extent.height); 342 } 343 } 344 } 345 346 blorp_batch_finish(&batch); 347} 348 349static void 350copy_buffer_to_image(struct anv_cmd_buffer *cmd_buffer, 351 struct anv_buffer *anv_buffer, 352 struct anv_image *anv_image, 353 VkImageLayout image_layout, 354 uint32_t regionCount, 355 const VkBufferImageCopy* pRegions, 356 bool buffer_to_image) 357{ 358 struct blorp_batch batch; 359 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0); 360 361 struct { 362 struct blorp_surf surf; 363 uint32_t level; 364 VkOffset3D offset; 365 } image, buffer, *src, *dst; 366 367 buffer.level = 0; 368 buffer.offset = (VkOffset3D) { 0, 0, 0 }; 369 370 if (buffer_to_image) { 371 src = &buffer; 372 dst = ℑ 373 } else { 374 src = ℑ 375 dst = &buffer; 376 } 377 378 for (unsigned r = 0; r < regionCount; r++) { 379 const VkImageAspectFlags aspect = pRegions[r].imageSubresource.aspectMask; 380 381 get_blorp_surf_for_anv_image(cmd_buffer->device, anv_image, aspect, 382 image_layout, ISL_AUX_USAGE_NONE, 383 &image.surf); 384 image.offset = 385 anv_sanitize_image_offset(anv_image->type, pRegions[r].imageOffset); 386 image.level = pRegions[r].imageSubresource.mipLevel; 387 388 VkExtent3D extent = 389 anv_sanitize_image_extent(anv_image->type, pRegions[r].imageExtent); 390 if (anv_image->type != VK_IMAGE_TYPE_3D) { 391 image.offset.z = pRegions[r].imageSubresource.baseArrayLayer; 392 extent.depth = 393 anv_get_layerCount(anv_image, &pRegions[r].imageSubresource); 394 } 395 396 const enum isl_format buffer_format = 397 anv_get_isl_format(&cmd_buffer->device->info, anv_image->vk_format, 398 aspect, VK_IMAGE_TILING_LINEAR); 399 400 const VkExtent3D bufferImageExtent = { 401 .width = pRegions[r].bufferRowLength ? 402 pRegions[r].bufferRowLength : extent.width, 403 .height = pRegions[r].bufferImageHeight ? 404 pRegions[r].bufferImageHeight : extent.height, 405 }; 406 407 const struct isl_format_layout *buffer_fmtl = 408 isl_format_get_layout(buffer_format); 409 410 const uint32_t buffer_row_pitch = 411 DIV_ROUND_UP(bufferImageExtent.width, buffer_fmtl->bw) * 412 (buffer_fmtl->bpb / 8); 413 414 const uint32_t buffer_layer_stride = 415 DIV_ROUND_UP(bufferImageExtent.height, buffer_fmtl->bh) * 416 buffer_row_pitch; 417 418 struct isl_surf buffer_isl_surf; 419 get_blorp_surf_for_anv_buffer(cmd_buffer->device, 420 anv_buffer, pRegions[r].bufferOffset, 421 extent.width, extent.height, 422 buffer_row_pitch, buffer_format, 423 &buffer.surf, &buffer_isl_surf); 424 425 if (&image == dst) { 426 anv_cmd_buffer_mark_image_written(cmd_buffer, anv_image, 427 aspect, dst->surf.aux_usage, 428 dst->level, 429 dst->offset.z, extent.depth); 430 } 431 432 for (unsigned z = 0; z < extent.depth; z++) { 433 blorp_copy(&batch, &src->surf, src->level, src->offset.z, 434 &dst->surf, dst->level, dst->offset.z, 435 src->offset.x, src->offset.y, dst->offset.x, dst->offset.y, 436 extent.width, extent.height); 437 438 image.offset.z++; 439 buffer.surf.addr.offset += buffer_layer_stride; 440 } 441 } 442 443 blorp_batch_finish(&batch); 444} 445 446void anv_CmdCopyBufferToImage( 447 VkCommandBuffer commandBuffer, 448 VkBuffer srcBuffer, 449 VkImage dstImage, 450 VkImageLayout dstImageLayout, 451 uint32_t regionCount, 452 const VkBufferImageCopy* pRegions) 453{ 454 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer); 455 ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer); 456 ANV_FROM_HANDLE(anv_image, dst_image, dstImage); 457 458 copy_buffer_to_image(cmd_buffer, src_buffer, dst_image, dstImageLayout, 459 regionCount, pRegions, true); 460} 461 462void anv_CmdCopyImageToBuffer( 463 VkCommandBuffer commandBuffer, 464 VkImage srcImage, 465 VkImageLayout srcImageLayout, 466 VkBuffer dstBuffer, 467 uint32_t regionCount, 468 const VkBufferImageCopy* pRegions) 469{ 470 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer); 471 ANV_FROM_HANDLE(anv_image, src_image, srcImage); 472 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer); 473 474 copy_buffer_to_image(cmd_buffer, dst_buffer, src_image, srcImageLayout, 475 regionCount, pRegions, false); 476} 477 478static bool 479flip_coords(unsigned *src0, unsigned *src1, unsigned *dst0, unsigned *dst1) 480{ 481 bool flip = false; 482 if (*src0 > *src1) { 483 unsigned tmp = *src0; 484 *src0 = *src1; 485 *src1 = tmp; 486 flip = !flip; 487 } 488 489 if (*dst0 > *dst1) { 490 unsigned tmp = *dst0; 491 *dst0 = *dst1; 492 *dst1 = tmp; 493 flip = !flip; 494 } 495 496 return flip; 497} 498 499void anv_CmdBlitImage( 500 VkCommandBuffer commandBuffer, 501 VkImage srcImage, 502 VkImageLayout srcImageLayout, 503 VkImage dstImage, 504 VkImageLayout dstImageLayout, 505 uint32_t regionCount, 506 const VkImageBlit* pRegions, 507 VkFilter filter) 508 509{ 510 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer); 511 ANV_FROM_HANDLE(anv_image, src_image, srcImage); 512 ANV_FROM_HANDLE(anv_image, dst_image, dstImage); 513 514 struct blorp_surf src, dst; 515 516 enum blorp_filter blorp_filter; 517 switch (filter) { 518 case VK_FILTER_NEAREST: 519 blorp_filter = BLORP_FILTER_NEAREST; 520 break; 521 case VK_FILTER_LINEAR: 522 blorp_filter = BLORP_FILTER_BILINEAR; 523 break; 524 default: 525 unreachable("Invalid filter"); 526 } 527 528 struct blorp_batch batch; 529 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0); 530 531 for (unsigned r = 0; r < regionCount; r++) { 532 const VkImageSubresourceLayers *src_res = &pRegions[r].srcSubresource; 533 const VkImageSubresourceLayers *dst_res = &pRegions[r].dstSubresource; 534 535 assert(anv_image_aspects_compatible(src_res->aspectMask, 536 dst_res->aspectMask)); 537 538 uint32_t aspect_bit; 539 anv_foreach_image_aspect_bit(aspect_bit, src_image, src_res->aspectMask) { 540 get_blorp_surf_for_anv_image(cmd_buffer->device, 541 src_image, 1U << aspect_bit, 542 srcImageLayout, ISL_AUX_USAGE_NONE, &src); 543 get_blorp_surf_for_anv_image(cmd_buffer->device, 544 dst_image, 1U << aspect_bit, 545 dstImageLayout, ISL_AUX_USAGE_NONE, &dst); 546 547 struct anv_format_plane src_format = 548 anv_get_format_plane(&cmd_buffer->device->info, src_image->vk_format, 549 1U << aspect_bit, src_image->tiling); 550 struct anv_format_plane dst_format = 551 anv_get_format_plane(&cmd_buffer->device->info, dst_image->vk_format, 552 1U << aspect_bit, dst_image->tiling); 553 554 unsigned dst_start, dst_end; 555 if (dst_image->type == VK_IMAGE_TYPE_3D) { 556 assert(dst_res->baseArrayLayer == 0); 557 dst_start = pRegions[r].dstOffsets[0].z; 558 dst_end = pRegions[r].dstOffsets[1].z; 559 } else { 560 dst_start = dst_res->baseArrayLayer; 561 dst_end = dst_start + anv_get_layerCount(dst_image, dst_res); 562 } 563 564 unsigned src_start, src_end; 565 if (src_image->type == VK_IMAGE_TYPE_3D) { 566 assert(src_res->baseArrayLayer == 0); 567 src_start = pRegions[r].srcOffsets[0].z; 568 src_end = pRegions[r].srcOffsets[1].z; 569 } else { 570 src_start = src_res->baseArrayLayer; 571 src_end = src_start + anv_get_layerCount(src_image, src_res); 572 } 573 574 bool flip_z = flip_coords(&src_start, &src_end, &dst_start, &dst_end); 575 float src_z_step = (float)(src_end + 1 - src_start) / 576 (float)(dst_end + 1 - dst_start); 577 578 if (flip_z) { 579 src_start = src_end; 580 src_z_step *= -1; 581 } 582 583 unsigned src_x0 = pRegions[r].srcOffsets[0].x; 584 unsigned src_x1 = pRegions[r].srcOffsets[1].x; 585 unsigned dst_x0 = pRegions[r].dstOffsets[0].x; 586 unsigned dst_x1 = pRegions[r].dstOffsets[1].x; 587 bool flip_x = flip_coords(&src_x0, &src_x1, &dst_x0, &dst_x1); 588 589 unsigned src_y0 = pRegions[r].srcOffsets[0].y; 590 unsigned src_y1 = pRegions[r].srcOffsets[1].y; 591 unsigned dst_y0 = pRegions[r].dstOffsets[0].y; 592 unsigned dst_y1 = pRegions[r].dstOffsets[1].y; 593 bool flip_y = flip_coords(&src_y0, &src_y1, &dst_y0, &dst_y1); 594 595 const unsigned num_layers = dst_end - dst_start; 596 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image, 597 1U << aspect_bit, 598 dst.aux_usage, 599 dst_res->mipLevel, 600 dst_start, num_layers); 601 602 for (unsigned i = 0; i < num_layers; i++) { 603 unsigned dst_z = dst_start + i; 604 unsigned src_z = src_start + i * src_z_step; 605 606 blorp_blit(&batch, &src, src_res->mipLevel, src_z, 607 src_format.isl_format, src_format.swizzle, 608 &dst, dst_res->mipLevel, dst_z, 609 dst_format.isl_format, dst_format.swizzle, 610 src_x0, src_y0, src_x1, src_y1, 611 dst_x0, dst_y0, dst_x1, dst_y1, 612 blorp_filter, flip_x, flip_y); 613 } 614 } 615 } 616 617 blorp_batch_finish(&batch); 618} 619 620static enum isl_format 621isl_format_for_size(unsigned size_B) 622{ 623 switch (size_B) { 624 case 4: return ISL_FORMAT_R32_UINT; 625 case 8: return ISL_FORMAT_R32G32_UINT; 626 case 16: return ISL_FORMAT_R32G32B32A32_UINT; 627 default: 628 unreachable("Not a power-of-two format size"); 629 } 630} 631 632/** 633 * Returns the greatest common divisor of a and b that is a power of two. 634 */ 635static uint64_t 636gcd_pow2_u64(uint64_t a, uint64_t b) 637{ 638 assert(a > 0 || b > 0); 639 640 unsigned a_log2 = ffsll(a) - 1; 641 unsigned b_log2 = ffsll(b) - 1; 642 643 /* If either a or b is 0, then a_log2 or b_log2 till be UINT_MAX in which 644 * case, the MIN2() will take the other one. If both are 0 then we will 645 * hit the assert above. 646 */ 647 return 1 << MIN2(a_log2, b_log2); 648} 649 650/* This is maximum possible width/height our HW can handle */ 651#define MAX_SURFACE_DIM (1ull << 14) 652 653void anv_CmdCopyBuffer( 654 VkCommandBuffer commandBuffer, 655 VkBuffer srcBuffer, 656 VkBuffer dstBuffer, 657 uint32_t regionCount, 658 const VkBufferCopy* pRegions) 659{ 660 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer); 661 ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer); 662 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer); 663 664 struct blorp_batch batch; 665 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0); 666 667 for (unsigned r = 0; r < regionCount; r++) { 668 struct blorp_address src = { 669 .buffer = src_buffer->address.bo, 670 .offset = src_buffer->address.offset + pRegions[r].srcOffset, 671 .mocs = anv_mocs_for_bo(cmd_buffer->device, src_buffer->address.bo), 672 }; 673 struct blorp_address dst = { 674 .buffer = dst_buffer->address.bo, 675 .offset = dst_buffer->address.offset + pRegions[r].dstOffset, 676 .mocs = anv_mocs_for_bo(cmd_buffer->device, dst_buffer->address.bo), 677 }; 678 679 blorp_buffer_copy(&batch, src, dst, pRegions[r].size); 680 } 681 682 blorp_batch_finish(&batch); 683} 684 685void anv_CmdUpdateBuffer( 686 VkCommandBuffer commandBuffer, 687 VkBuffer dstBuffer, 688 VkDeviceSize dstOffset, 689 VkDeviceSize dataSize, 690 const void* pData) 691{ 692 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer); 693 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer); 694 695 struct blorp_batch batch; 696 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0); 697 698 /* We can't quite grab a full block because the state stream needs a 699 * little data at the top to build its linked list. 700 */ 701 const uint32_t max_update_size = 702 cmd_buffer->device->dynamic_state_pool.block_size - 64; 703 704 assert(max_update_size < MAX_SURFACE_DIM * 4); 705 706 /* We're about to read data that was written from the CPU. Flush the 707 * texture cache so we don't get anything stale. 708 */ 709 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT; 710 711 while (dataSize) { 712 const uint32_t copy_size = MIN2(dataSize, max_update_size); 713 714 struct anv_state tmp_data = 715 anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, copy_size, 64); 716 717 memcpy(tmp_data.map, pData, copy_size); 718 719 anv_state_flush(cmd_buffer->device, tmp_data); 720 721 struct blorp_address src = { 722 .buffer = &cmd_buffer->device->dynamic_state_pool.block_pool.bo, 723 .offset = tmp_data.offset, 724 .mocs = cmd_buffer->device->default_mocs, 725 }; 726 struct blorp_address dst = { 727 .buffer = dst_buffer->address.bo, 728 .offset = dst_buffer->address.offset + dstOffset, 729 .mocs = anv_mocs_for_bo(cmd_buffer->device, dst_buffer->address.bo), 730 }; 731 732 blorp_buffer_copy(&batch, src, dst, copy_size); 733 734 dataSize -= copy_size; 735 dstOffset += copy_size; 736 pData = (void *)pData + copy_size; 737 } 738 739 blorp_batch_finish(&batch); 740} 741 742void anv_CmdFillBuffer( 743 VkCommandBuffer commandBuffer, 744 VkBuffer dstBuffer, 745 VkDeviceSize dstOffset, 746 VkDeviceSize fillSize, 747 uint32_t data) 748{ 749 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer); 750 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer); 751 struct blorp_surf surf; 752 struct isl_surf isl_surf; 753 754 struct blorp_batch batch; 755 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0); 756 757 fillSize = anv_buffer_get_range(dst_buffer, dstOffset, fillSize); 758 759 /* From the Vulkan spec: 760 * 761 * "size is the number of bytes to fill, and must be either a multiple 762 * of 4, or VK_WHOLE_SIZE to fill the range from offset to the end of 763 * the buffer. If VK_WHOLE_SIZE is used and the remaining size of the 764 * buffer is not a multiple of 4, then the nearest smaller multiple is 765 * used." 766 */ 767 fillSize &= ~3ull; 768 769 /* First, we compute the biggest format that can be used with the 770 * given offsets and size. 771 */ 772 int bs = 16; 773 bs = gcd_pow2_u64(bs, dstOffset); 774 bs = gcd_pow2_u64(bs, fillSize); 775 enum isl_format isl_format = isl_format_for_size(bs); 776 777 union isl_color_value color = { 778 .u32 = { data, data, data, data }, 779 }; 780 781 const uint64_t max_fill_size = MAX_SURFACE_DIM * MAX_SURFACE_DIM * bs; 782 while (fillSize >= max_fill_size) { 783 get_blorp_surf_for_anv_buffer(cmd_buffer->device, 784 dst_buffer, dstOffset, 785 MAX_SURFACE_DIM, MAX_SURFACE_DIM, 786 MAX_SURFACE_DIM * bs, isl_format, 787 &surf, &isl_surf); 788 789 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY, 790 0, 0, 1, 0, 0, MAX_SURFACE_DIM, MAX_SURFACE_DIM, 791 color, NULL); 792 fillSize -= max_fill_size; 793 dstOffset += max_fill_size; 794 } 795 796 uint64_t height = fillSize / (MAX_SURFACE_DIM * bs); 797 assert(height < MAX_SURFACE_DIM); 798 if (height != 0) { 799 const uint64_t rect_fill_size = height * MAX_SURFACE_DIM * bs; 800 get_blorp_surf_for_anv_buffer(cmd_buffer->device, 801 dst_buffer, dstOffset, 802 MAX_SURFACE_DIM, height, 803 MAX_SURFACE_DIM * bs, isl_format, 804 &surf, &isl_surf); 805 806 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY, 807 0, 0, 1, 0, 0, MAX_SURFACE_DIM, height, 808 color, NULL); 809 fillSize -= rect_fill_size; 810 dstOffset += rect_fill_size; 811 } 812 813 if (fillSize != 0) { 814 const uint32_t width = fillSize / bs; 815 get_blorp_surf_for_anv_buffer(cmd_buffer->device, 816 dst_buffer, dstOffset, 817 width, 1, 818 width * bs, isl_format, 819 &surf, &isl_surf); 820 821 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY, 822 0, 0, 1, 0, 0, width, 1, 823 color, NULL); 824 } 825 826 blorp_batch_finish(&batch); 827} 828 829void anv_CmdClearColorImage( 830 VkCommandBuffer commandBuffer, 831 VkImage _image, 832 VkImageLayout imageLayout, 833 const VkClearColorValue* pColor, 834 uint32_t rangeCount, 835 const VkImageSubresourceRange* pRanges) 836{ 837 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer); 838 ANV_FROM_HANDLE(anv_image, image, _image); 839 840 static const bool color_write_disable[4] = { false, false, false, false }; 841 842 struct blorp_batch batch; 843 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0); 844 845 846 for (unsigned r = 0; r < rangeCount; r++) { 847 if (pRanges[r].aspectMask == 0) 848 continue; 849 850 assert(pRanges[r].aspectMask & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV); 851 852 struct blorp_surf surf; 853 get_blorp_surf_for_anv_image(cmd_buffer->device, 854 image, pRanges[r].aspectMask, 855 imageLayout, ISL_AUX_USAGE_NONE, &surf); 856 857 struct anv_format_plane src_format = 858 anv_get_format_plane(&cmd_buffer->device->info, image->vk_format, 859 VK_IMAGE_ASPECT_COLOR_BIT, image->tiling); 860 861 unsigned base_layer = pRanges[r].baseArrayLayer; 862 unsigned layer_count = anv_get_layerCount(image, &pRanges[r]); 863 864 for (unsigned i = 0; i < anv_get_levelCount(image, &pRanges[r]); i++) { 865 const unsigned level = pRanges[r].baseMipLevel + i; 866 const unsigned level_width = anv_minify(image->extent.width, level); 867 const unsigned level_height = anv_minify(image->extent.height, level); 868 869 if (image->type == VK_IMAGE_TYPE_3D) { 870 base_layer = 0; 871 layer_count = anv_minify(image->extent.depth, level); 872 } 873 874 anv_cmd_buffer_mark_image_written(cmd_buffer, image, 875 pRanges[r].aspectMask, 876 surf.aux_usage, level, 877 base_layer, layer_count); 878 879 blorp_clear(&batch, &surf, 880 src_format.isl_format, src_format.swizzle, 881 level, base_layer, layer_count, 882 0, 0, level_width, level_height, 883 vk_to_isl_color(*pColor), color_write_disable); 884 } 885 } 886 887 blorp_batch_finish(&batch); 888} 889 890void anv_CmdClearDepthStencilImage( 891 VkCommandBuffer commandBuffer, 892 VkImage image_h, 893 VkImageLayout imageLayout, 894 const VkClearDepthStencilValue* pDepthStencil, 895 uint32_t rangeCount, 896 const VkImageSubresourceRange* pRanges) 897{ 898 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer); 899 ANV_FROM_HANDLE(anv_image, image, image_h); 900 901 struct blorp_batch batch; 902 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0); 903 904 struct blorp_surf depth, stencil; 905 if (image->aspects & VK_IMAGE_ASPECT_DEPTH_BIT) { 906 get_blorp_surf_for_anv_image(cmd_buffer->device, 907 image, VK_IMAGE_ASPECT_DEPTH_BIT, 908 imageLayout, ISL_AUX_USAGE_NONE, &depth); 909 } else { 910 memset(&depth, 0, sizeof(depth)); 911 } 912 913 if (image->aspects & VK_IMAGE_ASPECT_STENCIL_BIT) { 914 get_blorp_surf_for_anv_image(cmd_buffer->device, 915 image, VK_IMAGE_ASPECT_STENCIL_BIT, 916 imageLayout, ISL_AUX_USAGE_NONE, &stencil); 917 } else { 918 memset(&stencil, 0, sizeof(stencil)); 919 } 920 921 for (unsigned r = 0; r < rangeCount; r++) { 922 if (pRanges[r].aspectMask == 0) 923 continue; 924 925 bool clear_depth = pRanges[r].aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT; 926 bool clear_stencil = pRanges[r].aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT; 927 928 unsigned base_layer = pRanges[r].baseArrayLayer; 929 unsigned layer_count = anv_get_layerCount(image, &pRanges[r]); 930 931 for (unsigned i = 0; i < anv_get_levelCount(image, &pRanges[r]); i++) { 932 const unsigned level = pRanges[r].baseMipLevel + i; 933 const unsigned level_width = anv_minify(image->extent.width, level); 934 const unsigned level_height = anv_minify(image->extent.height, level); 935 936 if (image->type == VK_IMAGE_TYPE_3D) 937 layer_count = anv_minify(image->extent.depth, level); 938 939 blorp_clear_depth_stencil(&batch, &depth, &stencil, 940 level, base_layer, layer_count, 941 0, 0, level_width, level_height, 942 clear_depth, pDepthStencil->depth, 943 clear_stencil ? 0xff : 0, 944 pDepthStencil->stencil); 945 } 946 } 947 948 blorp_batch_finish(&batch); 949} 950 951VkResult 952anv_cmd_buffer_alloc_blorp_binding_table(struct anv_cmd_buffer *cmd_buffer, 953 uint32_t num_entries, 954 uint32_t *state_offset, 955 struct anv_state *bt_state) 956{ 957 *bt_state = anv_cmd_buffer_alloc_binding_table(cmd_buffer, num_entries, 958 state_offset); 959 if (bt_state->map == NULL) { 960 /* We ran out of space. Grab a new binding table block. */ 961 VkResult result = anv_cmd_buffer_new_binding_table_block(cmd_buffer); 962 if (result != VK_SUCCESS) 963 return result; 964 965 /* Re-emit state base addresses so we get the new surface state base 966 * address before we start emitting binding tables etc. 967 */ 968 anv_cmd_buffer_emit_state_base_address(cmd_buffer); 969 970 *bt_state = anv_cmd_buffer_alloc_binding_table(cmd_buffer, num_entries, 971 state_offset); 972 assert(bt_state->map != NULL); 973 } 974 975 return VK_SUCCESS; 976} 977 978static VkResult 979binding_table_for_surface_state(struct anv_cmd_buffer *cmd_buffer, 980 struct anv_state surface_state, 981 uint32_t *bt_offset) 982{ 983 uint32_t state_offset; 984 struct anv_state bt_state; 985 986 VkResult result = 987 anv_cmd_buffer_alloc_blorp_binding_table(cmd_buffer, 1, &state_offset, 988 &bt_state); 989 if (result != VK_SUCCESS) 990 return result; 991 992 uint32_t *bt_map = bt_state.map; 993 bt_map[0] = surface_state.offset + state_offset; 994 995 *bt_offset = bt_state.offset; 996 return VK_SUCCESS; 997} 998 999static void 1000clear_color_attachment(struct anv_cmd_buffer *cmd_buffer, 1001 struct blorp_batch *batch, 1002 const VkClearAttachment *attachment, 1003 uint32_t rectCount, const VkClearRect *pRects) 1004{ 1005 const struct anv_subpass *subpass = cmd_buffer->state.subpass; 1006 const uint32_t color_att = attachment->colorAttachment; 1007 const uint32_t att_idx = subpass->color_attachments[color_att].attachment; 1008 1009 if (att_idx == VK_ATTACHMENT_UNUSED) 1010 return; 1011 1012 struct anv_render_pass_attachment *pass_att = 1013 &cmd_buffer->state.pass->attachments[att_idx]; 1014 struct anv_attachment_state *att_state = 1015 &cmd_buffer->state.attachments[att_idx]; 1016 1017 uint32_t binding_table; 1018 VkResult result = 1019 binding_table_for_surface_state(cmd_buffer, att_state->color.state, 1020 &binding_table); 1021 if (result != VK_SUCCESS) 1022 return; 1023 1024 union isl_color_value clear_color = 1025 vk_to_isl_color(attachment->clearValue.color); 1026 1027 /* If multiview is enabled we ignore baseArrayLayer and layerCount */ 1028 if (subpass->view_mask) { 1029 uint32_t view_idx; 1030 for_each_bit(view_idx, subpass->view_mask) { 1031 for (uint32_t r = 0; r < rectCount; ++r) { 1032 const VkOffset2D offset = pRects[r].rect.offset; 1033 const VkExtent2D extent = pRects[r].rect.extent; 1034 blorp_clear_attachments(batch, binding_table, 1035 ISL_FORMAT_UNSUPPORTED, pass_att->samples, 1036 view_idx, 1, 1037 offset.x, offset.y, 1038 offset.x + extent.width, 1039 offset.y + extent.height, 1040 true, clear_color, false, 0.0f, 0, 0); 1041 } 1042 } 1043 return; 1044 } 1045 1046 for (uint32_t r = 0; r < rectCount; ++r) { 1047 const VkOffset2D offset = pRects[r].rect.offset; 1048 const VkExtent2D extent = pRects[r].rect.extent; 1049 assert(pRects[r].layerCount != VK_REMAINING_ARRAY_LAYERS); 1050 blorp_clear_attachments(batch, binding_table, 1051 ISL_FORMAT_UNSUPPORTED, pass_att->samples, 1052 pRects[r].baseArrayLayer, 1053 pRects[r].layerCount, 1054 offset.x, offset.y, 1055 offset.x + extent.width, offset.y + extent.height, 1056 true, clear_color, false, 0.0f, 0, 0); 1057 } 1058} 1059 1060static void 1061clear_depth_stencil_attachment(struct anv_cmd_buffer *cmd_buffer, 1062 struct blorp_batch *batch, 1063 const VkClearAttachment *attachment, 1064 uint32_t rectCount, const VkClearRect *pRects) 1065{ 1066 static const union isl_color_value color_value = { .u32 = { 0, } }; 1067 const struct anv_subpass *subpass = cmd_buffer->state.subpass; 1068 const uint32_t att_idx = subpass->depth_stencil_attachment->attachment; 1069 1070 if (att_idx == VK_ATTACHMENT_UNUSED) 1071 return; 1072 1073 struct anv_render_pass_attachment *pass_att = 1074 &cmd_buffer->state.pass->attachments[att_idx]; 1075 1076 bool clear_depth = attachment->aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT; 1077 bool clear_stencil = attachment->aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT; 1078 1079 enum isl_format depth_format = ISL_FORMAT_UNSUPPORTED; 1080 if (clear_depth) { 1081 depth_format = anv_get_isl_format(&cmd_buffer->device->info, 1082 pass_att->format, 1083 VK_IMAGE_ASPECT_DEPTH_BIT, 1084 VK_IMAGE_TILING_OPTIMAL); 1085 } 1086 1087 uint32_t binding_table; 1088 VkResult result = 1089 binding_table_for_surface_state(cmd_buffer, 1090 cmd_buffer->state.null_surface_state, 1091 &binding_table); 1092 if (result != VK_SUCCESS) 1093 return; 1094 1095 /* If multiview is enabled we ignore baseArrayLayer and layerCount */ 1096 if (subpass->view_mask) { 1097 uint32_t view_idx; 1098 for_each_bit(view_idx, subpass->view_mask) { 1099 for (uint32_t r = 0; r < rectCount; ++r) { 1100 const VkOffset2D offset = pRects[r].rect.offset; 1101 const VkExtent2D extent = pRects[r].rect.extent; 1102 VkClearDepthStencilValue value = attachment->clearValue.depthStencil; 1103 blorp_clear_attachments(batch, binding_table, 1104 depth_format, pass_att->samples, 1105 view_idx, 1, 1106 offset.x, offset.y, 1107 offset.x + extent.width, 1108 offset.y + extent.height, 1109 false, color_value, 1110 clear_depth, value.depth, 1111 clear_stencil ? 0xff : 0, value.stencil); 1112 } 1113 } 1114 return; 1115 } 1116 1117 for (uint32_t r = 0; r < rectCount; ++r) { 1118 const VkOffset2D offset = pRects[r].rect.offset; 1119 const VkExtent2D extent = pRects[r].rect.extent; 1120 VkClearDepthStencilValue value = attachment->clearValue.depthStencil; 1121 assert(pRects[r].layerCount != VK_REMAINING_ARRAY_LAYERS); 1122 blorp_clear_attachments(batch, binding_table, 1123 depth_format, pass_att->samples, 1124 pRects[r].baseArrayLayer, 1125 pRects[r].layerCount, 1126 offset.x, offset.y, 1127 offset.x + extent.width, offset.y + extent.height, 1128 false, color_value, 1129 clear_depth, value.depth, 1130 clear_stencil ? 0xff : 0, value.stencil); 1131 } 1132} 1133 1134void anv_CmdClearAttachments( 1135 VkCommandBuffer commandBuffer, 1136 uint32_t attachmentCount, 1137 const VkClearAttachment* pAttachments, 1138 uint32_t rectCount, 1139 const VkClearRect* pRects) 1140{ 1141 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer); 1142 1143 /* Because this gets called within a render pass, we tell blorp not to 1144 * trash our depth and stencil buffers. 1145 */ 1146 struct blorp_batch batch; 1147 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 1148 BLORP_BATCH_NO_EMIT_DEPTH_STENCIL); 1149 1150 for (uint32_t a = 0; a < attachmentCount; ++a) { 1151 if (pAttachments[a].aspectMask & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) { 1152 assert(pAttachments[a].aspectMask == VK_IMAGE_ASPECT_COLOR_BIT); 1153 clear_color_attachment(cmd_buffer, &batch, 1154 &pAttachments[a], 1155 rectCount, pRects); 1156 } else { 1157 clear_depth_stencil_attachment(cmd_buffer, &batch, 1158 &pAttachments[a], 1159 rectCount, pRects); 1160 } 1161 } 1162 1163 blorp_batch_finish(&batch); 1164} 1165 1166enum subpass_stage { 1167 SUBPASS_STAGE_LOAD, 1168 SUBPASS_STAGE_DRAW, 1169 SUBPASS_STAGE_RESOLVE, 1170}; 1171 1172static void 1173resolve_surface(struct blorp_batch *batch, 1174 struct blorp_surf *src_surf, 1175 uint32_t src_level, uint32_t src_layer, 1176 struct blorp_surf *dst_surf, 1177 uint32_t dst_level, uint32_t dst_layer, 1178 uint32_t src_x, uint32_t src_y, uint32_t dst_x, uint32_t dst_y, 1179 uint32_t width, uint32_t height, 1180 enum blorp_filter filter) 1181{ 1182 blorp_blit(batch, 1183 src_surf, src_level, src_layer, 1184 ISL_FORMAT_UNSUPPORTED, ISL_SWIZZLE_IDENTITY, 1185 dst_surf, dst_level, dst_layer, 1186 ISL_FORMAT_UNSUPPORTED, ISL_SWIZZLE_IDENTITY, 1187 src_x, src_y, src_x + width, src_y + height, 1188 dst_x, dst_y, dst_x + width, dst_y + height, 1189 filter, false, false); 1190} 1191 1192static void 1193resolve_image(struct anv_device *device, 1194 struct blorp_batch *batch, 1195 const struct anv_image *src_image, 1196 VkImageLayout src_image_layout, 1197 uint32_t src_level, uint32_t src_layer, 1198 const struct anv_image *dst_image, 1199 VkImageLayout dst_image_layout, 1200 uint32_t dst_level, uint32_t dst_layer, 1201 VkImageAspectFlags aspect_mask, 1202 uint32_t src_x, uint32_t src_y, uint32_t dst_x, uint32_t dst_y, 1203 uint32_t width, uint32_t height) 1204{ 1205 struct anv_cmd_buffer *cmd_buffer = batch->driver_batch; 1206 1207 assert(src_image->type == VK_IMAGE_TYPE_2D); 1208 assert(src_image->samples > 1); 1209 assert(dst_image->type == VK_IMAGE_TYPE_2D); 1210 assert(dst_image->samples == 1); 1211 assert(src_image->n_planes == dst_image->n_planes); 1212 1213 uint32_t aspect_bit; 1214 1215 anv_foreach_image_aspect_bit(aspect_bit, src_image, aspect_mask) { 1216 struct blorp_surf src_surf, dst_surf; 1217 get_blorp_surf_for_anv_image(device, src_image, 1UL << aspect_bit, 1218 src_image_layout, ISL_AUX_USAGE_NONE, 1219 &src_surf); 1220 get_blorp_surf_for_anv_image(device, dst_image, 1UL << aspect_bit, 1221 dst_image_layout, ISL_AUX_USAGE_NONE, 1222 &dst_surf); 1223 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image, 1224 1UL << aspect_bit, 1225 dst_surf.aux_usage, 1226 dst_level, dst_layer, 1); 1227 1228 enum blorp_filter filter; 1229 if ((src_surf.surf->usage & ISL_SURF_USAGE_DEPTH_BIT) || 1230 (src_surf.surf->usage & ISL_SURF_USAGE_STENCIL_BIT) || 1231 isl_format_has_int_channel(src_surf.surf->format)) { 1232 filter = BLORP_FILTER_SAMPLE_0; 1233 } else { 1234 filter = BLORP_FILTER_AVERAGE; 1235 } 1236 1237 assert(!src_image->format->can_ycbcr); 1238 assert(!dst_image->format->can_ycbcr); 1239 1240 resolve_surface(batch, 1241 &src_surf, src_level, src_layer, 1242 &dst_surf, dst_level, dst_layer, 1243 src_x, src_y, dst_x, dst_y, width, height, filter); 1244 } 1245} 1246 1247void anv_CmdResolveImage( 1248 VkCommandBuffer commandBuffer, 1249 VkImage srcImage, 1250 VkImageLayout srcImageLayout, 1251 VkImage dstImage, 1252 VkImageLayout dstImageLayout, 1253 uint32_t regionCount, 1254 const VkImageResolve* pRegions) 1255{ 1256 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer); 1257 ANV_FROM_HANDLE(anv_image, src_image, srcImage); 1258 ANV_FROM_HANDLE(anv_image, dst_image, dstImage); 1259 1260 struct blorp_batch batch; 1261 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0); 1262 1263 for (uint32_t r = 0; r < regionCount; r++) { 1264 assert(pRegions[r].srcSubresource.aspectMask == 1265 pRegions[r].dstSubresource.aspectMask); 1266 assert(anv_get_layerCount(src_image, &pRegions[r].srcSubresource) == 1267 anv_get_layerCount(dst_image, &pRegions[r].dstSubresource)); 1268 1269 const uint32_t layer_count = 1270 anv_get_layerCount(dst_image, &pRegions[r].dstSubresource); 1271 1272 VkImageAspectFlags src_mask = pRegions[r].srcSubresource.aspectMask, 1273 dst_mask = pRegions[r].dstSubresource.aspectMask; 1274 1275 assert(anv_image_aspects_compatible(src_mask, dst_mask)); 1276 1277 for (uint32_t layer = 0; layer < layer_count; layer++) { 1278 resolve_image(cmd_buffer->device, &batch, 1279 src_image, srcImageLayout, 1280 pRegions[r].srcSubresource.mipLevel, 1281 pRegions[r].srcSubresource.baseArrayLayer + layer, 1282 dst_image, dstImageLayout, 1283 pRegions[r].dstSubresource.mipLevel, 1284 pRegions[r].dstSubresource.baseArrayLayer + layer, 1285 pRegions[r].dstSubresource.aspectMask, 1286 pRegions[r].srcOffset.x, pRegions[r].srcOffset.y, 1287 pRegions[r].dstOffset.x, pRegions[r].dstOffset.y, 1288 pRegions[r].extent.width, pRegions[r].extent.height); 1289 } 1290 } 1291 1292 blorp_batch_finish(&batch); 1293} 1294 1295static enum isl_aux_usage 1296fast_clear_aux_usage(const struct anv_image *image, 1297 VkImageAspectFlagBits aspect) 1298{ 1299 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect); 1300 if (image->planes[plane].aux_usage == ISL_AUX_USAGE_NONE) 1301 return ISL_AUX_USAGE_CCS_D; 1302 else 1303 return image->planes[plane].aux_usage; 1304} 1305 1306void 1307anv_cmd_buffer_resolve_subpass(struct anv_cmd_buffer *cmd_buffer) 1308{ 1309 struct anv_framebuffer *fb = cmd_buffer->state.framebuffer; 1310 struct anv_subpass *subpass = cmd_buffer->state.subpass; 1311 1312 if (subpass->has_resolve) { 1313 struct blorp_batch batch; 1314 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0); 1315 1316 /* We are about to do some MSAA resolves. We need to flush so that the 1317 * result of writes to the MSAA color attachments show up in the sampler 1318 * when we blit to the single-sampled resolve target. 1319 */ 1320 cmd_buffer->state.pending_pipe_bits |= 1321 ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT | 1322 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT; 1323 1324 for (uint32_t i = 0; i < subpass->color_count; ++i) { 1325 uint32_t src_att = subpass->color_attachments[i].attachment; 1326 uint32_t dst_att = subpass->resolve_attachments[i].attachment; 1327 1328 if (dst_att == VK_ATTACHMENT_UNUSED) 1329 continue; 1330 1331 assert(src_att < cmd_buffer->state.pass->attachment_count); 1332 assert(dst_att < cmd_buffer->state.pass->attachment_count); 1333 1334 if (cmd_buffer->state.attachments[dst_att].pending_clear_aspects) { 1335 /* From the Vulkan 1.0 spec: 1336 * 1337 * If the first use of an attachment in a render pass is as a 1338 * resolve attachment, then the loadOp is effectively ignored 1339 * as the resolve is guaranteed to overwrite all pixels in the 1340 * render area. 1341 */ 1342 cmd_buffer->state.attachments[dst_att].pending_clear_aspects = 0; 1343 } 1344 1345 struct anv_image_view *src_iview = fb->attachments[src_att]; 1346 struct anv_image_view *dst_iview = fb->attachments[dst_att]; 1347 1348 enum isl_aux_usage src_aux_usage = 1349 cmd_buffer->state.attachments[src_att].aux_usage; 1350 enum isl_aux_usage dst_aux_usage = 1351 cmd_buffer->state.attachments[dst_att].aux_usage; 1352 1353 const VkRect2D render_area = cmd_buffer->state.render_area; 1354 1355 assert(src_iview->aspect_mask == VK_IMAGE_ASPECT_COLOR_BIT && 1356 dst_iview->aspect_mask == VK_IMAGE_ASPECT_COLOR_BIT); 1357 1358 enum blorp_filter filter; 1359 if (isl_format_has_int_channel(src_iview->planes[0].isl.format)) { 1360 filter = BLORP_FILTER_SAMPLE_0; 1361 } else { 1362 filter = BLORP_FILTER_AVERAGE; 1363 } 1364 1365 struct blorp_surf src_surf, dst_surf; 1366 get_blorp_surf_for_anv_image(cmd_buffer->device, src_iview->image, 1367 VK_IMAGE_ASPECT_COLOR_BIT, 1368 ANV_IMAGE_LAYOUT_EXPLICIT_AUX, 1369 src_aux_usage, &src_surf); 1370 if (src_aux_usage == ISL_AUX_USAGE_MCS) { 1371 src_surf.clear_color_addr = anv_to_blorp_address( 1372 anv_image_get_clear_color_addr(cmd_buffer->device, 1373 src_iview->image, 1374 VK_IMAGE_ASPECT_COLOR_BIT)); 1375 } 1376 get_blorp_surf_for_anv_image(cmd_buffer->device, dst_iview->image, 1377 VK_IMAGE_ASPECT_COLOR_BIT, 1378 ANV_IMAGE_LAYOUT_EXPLICIT_AUX, 1379 dst_aux_usage, &dst_surf); 1380 1381 uint32_t base_src_layer = src_iview->planes[0].isl.base_array_layer; 1382 uint32_t base_dst_layer = dst_iview->planes[0].isl.base_array_layer; 1383 1384 assert(src_iview->planes[0].isl.array_len >= fb->layers); 1385 assert(dst_iview->planes[0].isl.array_len >= fb->layers); 1386 1387 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_iview->image, 1388 VK_IMAGE_ASPECT_COLOR_BIT, 1389 dst_surf.aux_usage, 1390 dst_iview->planes[0].isl.base_level, 1391 base_dst_layer, fb->layers); 1392 1393 assert(!src_iview->image->format->can_ycbcr); 1394 assert(!dst_iview->image->format->can_ycbcr); 1395 1396 for (uint32_t i = 0; i < fb->layers; i++) { 1397 resolve_surface(&batch, 1398 &src_surf, 1399 src_iview->planes[0].isl.base_level, 1400 base_src_layer + i, 1401 &dst_surf, 1402 dst_iview->planes[0].isl.base_level, 1403 base_dst_layer + i, 1404 render_area.offset.x, render_area.offset.y, 1405 render_area.offset.x, render_area.offset.y, 1406 render_area.extent.width, render_area.extent.height, 1407 filter); 1408 } 1409 } 1410 1411 blorp_batch_finish(&batch); 1412 } 1413} 1414 1415void 1416anv_image_copy_to_shadow(struct anv_cmd_buffer *cmd_buffer, 1417 const struct anv_image *image, 1418 uint32_t base_level, uint32_t level_count, 1419 uint32_t base_layer, uint32_t layer_count) 1420{ 1421 struct blorp_batch batch; 1422 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0); 1423 1424 assert(image->aspects == VK_IMAGE_ASPECT_COLOR_BIT && image->n_planes == 1); 1425 1426 struct blorp_surf surf; 1427 get_blorp_surf_for_anv_image(cmd_buffer->device, 1428 image, VK_IMAGE_ASPECT_COLOR_BIT, 1429 VK_IMAGE_LAYOUT_GENERAL, 1430 ISL_AUX_USAGE_NONE, &surf); 1431 assert(surf.aux_usage == ISL_AUX_USAGE_NONE); 1432 1433 struct blorp_surf shadow_surf = { 1434 .surf = &image->planes[0].shadow_surface.isl, 1435 .addr = { 1436 .buffer = image->planes[0].address.bo, 1437 .offset = image->planes[0].address.offset + 1438 image->planes[0].shadow_surface.offset, 1439 .mocs = anv_mocs_for_bo(cmd_buffer->device, 1440 image->planes[0].address.bo), 1441 }, 1442 }; 1443 1444 for (uint32_t l = 0; l < level_count; l++) { 1445 const uint32_t level = base_level + l; 1446 1447 const VkExtent3D extent = { 1448 .width = anv_minify(image->extent.width, level), 1449 .height = anv_minify(image->extent.height, level), 1450 .depth = anv_minify(image->extent.depth, level), 1451 }; 1452 1453 if (image->type == VK_IMAGE_TYPE_3D) 1454 layer_count = extent.depth; 1455 1456 for (uint32_t a = 0; a < layer_count; a++) { 1457 const uint32_t layer = base_layer + a; 1458 1459 blorp_copy(&batch, &surf, level, layer, 1460 &shadow_surf, level, layer, 1461 0, 0, 0, 0, extent.width, extent.height); 1462 } 1463 } 1464 1465 blorp_batch_finish(&batch); 1466} 1467 1468void 1469anv_image_clear_color(struct anv_cmd_buffer *cmd_buffer, 1470 const struct anv_image *image, 1471 VkImageAspectFlagBits aspect, 1472 enum isl_aux_usage aux_usage, 1473 enum isl_format format, struct isl_swizzle swizzle, 1474 uint32_t level, uint32_t base_layer, uint32_t layer_count, 1475 VkRect2D area, union isl_color_value clear_color) 1476{ 1477 assert(image->aspects == VK_IMAGE_ASPECT_COLOR_BIT); 1478 1479 /* We don't support planar images with multisampling yet */ 1480 assert(image->n_planes == 1); 1481 1482 struct blorp_batch batch; 1483 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0); 1484 1485 struct blorp_surf surf; 1486 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect, 1487 ANV_IMAGE_LAYOUT_EXPLICIT_AUX, 1488 aux_usage, &surf); 1489 anv_cmd_buffer_mark_image_written(cmd_buffer, image, aspect, aux_usage, 1490 level, base_layer, layer_count); 1491 1492 blorp_clear(&batch, &surf, format, anv_swizzle_for_render(swizzle), 1493 level, base_layer, layer_count, 1494 area.offset.x, area.offset.y, 1495 area.offset.x + area.extent.width, 1496 area.offset.y + area.extent.height, 1497 clear_color, NULL); 1498 1499 blorp_batch_finish(&batch); 1500} 1501 1502void 1503anv_image_clear_depth_stencil(struct anv_cmd_buffer *cmd_buffer, 1504 const struct anv_image *image, 1505 VkImageAspectFlags aspects, 1506 enum isl_aux_usage depth_aux_usage, 1507 uint32_t level, 1508 uint32_t base_layer, uint32_t layer_count, 1509 VkRect2D area, 1510 float depth_value, uint8_t stencil_value) 1511{ 1512 assert(image->aspects & (VK_IMAGE_ASPECT_DEPTH_BIT | 1513 VK_IMAGE_ASPECT_STENCIL_BIT)); 1514 1515 struct blorp_batch batch; 1516 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0); 1517 1518 struct blorp_surf depth = {}; 1519 if (aspects & VK_IMAGE_ASPECT_DEPTH_BIT) { 1520 get_blorp_surf_for_anv_image(cmd_buffer->device, 1521 image, VK_IMAGE_ASPECT_DEPTH_BIT, 1522 ANV_IMAGE_LAYOUT_EXPLICIT_AUX, 1523 depth_aux_usage, &depth); 1524 depth.clear_color.f32[0] = ANV_HZ_FC_VAL; 1525 } 1526 1527 struct blorp_surf stencil = {}; 1528 if (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) { 1529 get_blorp_surf_for_anv_image(cmd_buffer->device, 1530 image, VK_IMAGE_ASPECT_STENCIL_BIT, 1531 ANV_IMAGE_LAYOUT_EXPLICIT_AUX, 1532 ISL_AUX_USAGE_NONE, &stencil); 1533 } 1534 1535 blorp_clear_depth_stencil(&batch, &depth, &stencil, 1536 level, base_layer, layer_count, 1537 area.offset.x, area.offset.y, 1538 area.offset.x + area.extent.width, 1539 area.offset.y + area.extent.height, 1540 aspects & VK_IMAGE_ASPECT_DEPTH_BIT, 1541 depth_value, 1542 (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) ? 0xff : 0, 1543 stencil_value); 1544 1545 blorp_batch_finish(&batch); 1546} 1547 1548void 1549anv_image_hiz_op(struct anv_cmd_buffer *cmd_buffer, 1550 const struct anv_image *image, 1551 VkImageAspectFlagBits aspect, uint32_t level, 1552 uint32_t base_layer, uint32_t layer_count, 1553 enum isl_aux_op hiz_op) 1554{ 1555 assert(aspect == VK_IMAGE_ASPECT_DEPTH_BIT); 1556 assert(base_layer + layer_count <= anv_image_aux_layers(image, aspect, level)); 1557 assert(anv_image_aspect_to_plane(image->aspects, 1558 VK_IMAGE_ASPECT_DEPTH_BIT) == 0); 1559 1560 struct blorp_batch batch; 1561 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0); 1562 1563 struct blorp_surf surf; 1564 get_blorp_surf_for_anv_image(cmd_buffer->device, 1565 image, VK_IMAGE_ASPECT_DEPTH_BIT, 1566 ANV_IMAGE_LAYOUT_EXPLICIT_AUX, 1567 ISL_AUX_USAGE_HIZ, &surf); 1568 surf.clear_color.f32[0] = ANV_HZ_FC_VAL; 1569 1570 blorp_hiz_op(&batch, &surf, level, base_layer, layer_count, hiz_op); 1571 1572 blorp_batch_finish(&batch); 1573} 1574 1575void 1576anv_image_hiz_clear(struct anv_cmd_buffer *cmd_buffer, 1577 const struct anv_image *image, 1578 VkImageAspectFlags aspects, 1579 uint32_t level, 1580 uint32_t base_layer, uint32_t layer_count, 1581 VkRect2D area, uint8_t stencil_value) 1582{ 1583 assert(image->aspects & (VK_IMAGE_ASPECT_DEPTH_BIT | 1584 VK_IMAGE_ASPECT_STENCIL_BIT)); 1585 1586 struct blorp_batch batch; 1587 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0); 1588 1589 struct blorp_surf depth = {}; 1590 if (aspects & VK_IMAGE_ASPECT_DEPTH_BIT) { 1591 assert(base_layer + layer_count <= 1592 anv_image_aux_layers(image, VK_IMAGE_ASPECT_DEPTH_BIT, level)); 1593 get_blorp_surf_for_anv_image(cmd_buffer->device, 1594 image, VK_IMAGE_ASPECT_DEPTH_BIT, 1595 ANV_IMAGE_LAYOUT_EXPLICIT_AUX, 1596 ISL_AUX_USAGE_HIZ, &depth); 1597 depth.clear_color.f32[0] = ANV_HZ_FC_VAL; 1598 } 1599 1600 struct blorp_surf stencil = {}; 1601 if (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) { 1602 get_blorp_surf_for_anv_image(cmd_buffer->device, 1603 image, VK_IMAGE_ASPECT_STENCIL_BIT, 1604 ANV_IMAGE_LAYOUT_EXPLICIT_AUX, 1605 ISL_AUX_USAGE_NONE, &stencil); 1606 } 1607 1608 /* From the Sky Lake PRM Volume 7, "Depth Buffer Clear": 1609 * 1610 * "The following is required when performing a depth buffer clear with 1611 * using the WM_STATE or 3DSTATE_WM: 1612 * 1613 * * If other rendering operations have preceded this clear, a 1614 * PIPE_CONTROL with depth cache flush enabled, Depth Stall bit 1615 * enabled must be issued before the rectangle primitive used for 1616 * the depth buffer clear operation. 1617 * * [...]" 1618 * 1619 * Even though the PRM only says that this is required if using 3DSTATE_WM 1620 * and a 3DPRIMITIVE, the GPU appears to also need this to avoid occasional 1621 * hangs when doing a clear with WM_HZ_OP. 1622 */ 1623 cmd_buffer->state.pending_pipe_bits |= 1624 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT | ANV_PIPE_DEPTH_STALL_BIT; 1625 1626 blorp_hiz_clear_depth_stencil(&batch, &depth, &stencil, 1627 level, base_layer, layer_count, 1628 area.offset.x, area.offset.y, 1629 area.offset.x + area.extent.width, 1630 area.offset.y + area.extent.height, 1631 aspects & VK_IMAGE_ASPECT_DEPTH_BIT, 1632 ANV_HZ_FC_VAL, 1633 aspects & VK_IMAGE_ASPECT_STENCIL_BIT, 1634 stencil_value); 1635 1636 blorp_batch_finish(&batch); 1637 1638 /* From the SKL PRM, Depth Buffer Clear: 1639 * 1640 * "Depth Buffer Clear Workaround 1641 * 1642 * Depth buffer clear pass using any of the methods (WM_STATE, 1643 * 3DSTATE_WM or 3DSTATE_WM_HZ_OP) must be followed by a PIPE_CONTROL 1644 * command with DEPTH_STALL bit and Depth FLUSH bits “set” before 1645 * starting to render. DepthStall and DepthFlush are not needed between 1646 * consecutive depth clear passes nor is it required if the depth-clear 1647 * pass was done with “full_surf_clear” bit set in the 1648 * 3DSTATE_WM_HZ_OP." 1649 * 1650 * Even though the PRM provides a bunch of conditions under which this is 1651 * supposedly unnecessary, we choose to perform the flush unconditionally 1652 * just to be safe. 1653 */ 1654 cmd_buffer->state.pending_pipe_bits |= 1655 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT | ANV_PIPE_DEPTH_STALL_BIT; 1656} 1657 1658void 1659anv_image_mcs_op(struct anv_cmd_buffer *cmd_buffer, 1660 const struct anv_image *image, 1661 enum isl_format format, 1662 VkImageAspectFlagBits aspect, 1663 uint32_t base_layer, uint32_t layer_count, 1664 enum isl_aux_op mcs_op, union isl_color_value *clear_value, 1665 bool predicate) 1666{ 1667 assert(image->aspects == VK_IMAGE_ASPECT_COLOR_BIT); 1668 assert(image->samples > 1); 1669 assert(base_layer + layer_count <= anv_image_aux_layers(image, aspect, 0)); 1670 1671 /* Multisampling with multi-planar formats is not supported */ 1672 assert(image->n_planes == 1); 1673 1674 struct blorp_batch batch; 1675 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 1676 predicate ? BLORP_BATCH_PREDICATE_ENABLE : 0); 1677 1678 struct blorp_surf surf; 1679 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect, 1680 ANV_IMAGE_LAYOUT_EXPLICIT_AUX, 1681 ISL_AUX_USAGE_MCS, &surf); 1682 1683 /* Blorp will store the clear color for us if we provide the clear color 1684 * address and we are doing a fast clear. So we save the clear value into 1685 * the blorp surface. However, in some situations we want to do a fast clear 1686 * without changing the clear value stored in the state buffer. For those 1687 * cases, we set the clear color address pointer to NULL, so blorp will not 1688 * try to store a garbage color. 1689 */ 1690 if (mcs_op == ISL_AUX_OP_FAST_CLEAR) { 1691 if (clear_value) 1692 surf.clear_color = *clear_value; 1693 else 1694 surf.clear_color_addr.buffer = NULL; 1695 } 1696 1697 /* From the Sky Lake PRM Vol. 7, "Render Target Fast Clear": 1698 * 1699 * "After Render target fast clear, pipe-control with color cache 1700 * write-flush must be issued before sending any DRAW commands on 1701 * that render target." 1702 * 1703 * This comment is a bit cryptic and doesn't really tell you what's going 1704 * or what's really needed. It appears that fast clear ops are not 1705 * properly synchronized with other drawing. This means that we cannot 1706 * have a fast clear operation in the pipe at the same time as other 1707 * regular drawing operations. We need to use a PIPE_CONTROL to ensure 1708 * that the contents of the previous draw hit the render target before we 1709 * resolve and then use a second PIPE_CONTROL after the resolve to ensure 1710 * that it is completed before any additional drawing occurs. 1711 */ 1712 cmd_buffer->state.pending_pipe_bits |= 1713 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT; 1714 1715 switch (mcs_op) { 1716 case ISL_AUX_OP_FAST_CLEAR: 1717 blorp_fast_clear(&batch, &surf, format, 1718 0, base_layer, layer_count, 1719 0, 0, image->extent.width, image->extent.height); 1720 break; 1721 case ISL_AUX_OP_PARTIAL_RESOLVE: 1722 blorp_mcs_partial_resolve(&batch, &surf, format, 1723 base_layer, layer_count); 1724 break; 1725 case ISL_AUX_OP_FULL_RESOLVE: 1726 case ISL_AUX_OP_AMBIGUATE: 1727 default: 1728 unreachable("Unsupported MCS operation"); 1729 } 1730 1731 cmd_buffer->state.pending_pipe_bits |= 1732 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT; 1733 1734 blorp_batch_finish(&batch); 1735} 1736 1737void 1738anv_image_ccs_op(struct anv_cmd_buffer *cmd_buffer, 1739 const struct anv_image *image, 1740 enum isl_format format, 1741 VkImageAspectFlagBits aspect, uint32_t level, 1742 uint32_t base_layer, uint32_t layer_count, 1743 enum isl_aux_op ccs_op, union isl_color_value *clear_value, 1744 bool predicate) 1745{ 1746 assert(image->aspects & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV); 1747 assert(image->samples == 1); 1748 assert(level < anv_image_aux_levels(image, aspect)); 1749 /* Multi-LOD YcBcR is not allowed */ 1750 assert(image->n_planes == 1 || level == 0); 1751 assert(base_layer + layer_count <= 1752 anv_image_aux_layers(image, aspect, level)); 1753 1754 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect); 1755 uint32_t width_div = image->format->planes[plane].denominator_scales[0]; 1756 uint32_t height_div = image->format->planes[plane].denominator_scales[1]; 1757 uint32_t level_width = anv_minify(image->extent.width, level) / width_div; 1758 uint32_t level_height = anv_minify(image->extent.height, level) / height_div; 1759 1760 struct blorp_batch batch; 1761 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 1762 predicate ? BLORP_BATCH_PREDICATE_ENABLE : 0); 1763 1764 struct blorp_surf surf; 1765 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect, 1766 ANV_IMAGE_LAYOUT_EXPLICIT_AUX, 1767 fast_clear_aux_usage(image, aspect), 1768 &surf); 1769 1770 /* Blorp will store the clear color for us if we provide the clear color 1771 * address and we are doing a fast clear. So we save the clear value into 1772 * the blorp surface. However, in some situations we want to do a fast clear 1773 * without changing the clear value stored in the state buffer. For those 1774 * cases, we set the clear color address pointer to NULL, so blorp will not 1775 * try to store a garbage color. 1776 */ 1777 if (ccs_op == ISL_AUX_OP_FAST_CLEAR) { 1778 if (clear_value) 1779 surf.clear_color = *clear_value; 1780 else 1781 surf.clear_color_addr.buffer = NULL; 1782 } 1783 1784 /* From the Sky Lake PRM Vol. 7, "Render Target Fast Clear": 1785 * 1786 * "After Render target fast clear, pipe-control with color cache 1787 * write-flush must be issued before sending any DRAW commands on 1788 * that render target." 1789 * 1790 * This comment is a bit cryptic and doesn't really tell you what's going 1791 * or what's really needed. It appears that fast clear ops are not 1792 * properly synchronized with other drawing. This means that we cannot 1793 * have a fast clear operation in the pipe at the same time as other 1794 * regular drawing operations. We need to use a PIPE_CONTROL to ensure 1795 * that the contents of the previous draw hit the render target before we 1796 * resolve and then use a second PIPE_CONTROL after the resolve to ensure 1797 * that it is completed before any additional drawing occurs. 1798 */ 1799 cmd_buffer->state.pending_pipe_bits |= 1800 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT; 1801 1802 switch (ccs_op) { 1803 case ISL_AUX_OP_FAST_CLEAR: 1804 blorp_fast_clear(&batch, &surf, format, 1805 level, base_layer, layer_count, 1806 0, 0, level_width, level_height); 1807 break; 1808 case ISL_AUX_OP_FULL_RESOLVE: 1809 case ISL_AUX_OP_PARTIAL_RESOLVE: 1810 blorp_ccs_resolve(&batch, &surf, level, base_layer, layer_count, 1811 format, ccs_op); 1812 break; 1813 case ISL_AUX_OP_AMBIGUATE: 1814 for (uint32_t a = 0; a < layer_count; a++) { 1815 const uint32_t layer = base_layer + a; 1816 blorp_ccs_ambiguate(&batch, &surf, level, layer); 1817 } 1818 break; 1819 default: 1820 unreachable("Unsupported CCS operation"); 1821 } 1822 1823 cmd_buffer->state.pending_pipe_bits |= 1824 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT; 1825 1826 blorp_batch_finish(&batch); 1827} 1828