1/* 2 * Copyright © 2016 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 */ 23 24#include "anv_private.h" 25 26static bool 27lookup_blorp_shader(struct blorp_batch *batch, 28 const void *key, uint32_t key_size, 29 uint32_t *kernel_out, void *prog_data_out) 30{ 31 struct blorp_context *blorp = batch->blorp; 32 struct anv_device *device = blorp->driver_ctx; 33 34 /* The default cache must be a real cache */ 35 assert(device->default_pipeline_cache.cache); 36 37 struct anv_shader_bin *bin = 38 anv_pipeline_cache_search(&device->default_pipeline_cache, key, key_size); 39 if (!bin) 40 return false; 41 42 /* The cache already has a reference and it's not going anywhere so there 43 * is no need to hold a second reference. 44 */ 45 anv_shader_bin_unref(device, bin); 46 47 *kernel_out = bin->kernel.offset; 48 *(const struct brw_stage_prog_data **)prog_data_out = bin->prog_data; 49 50 return true; 51} 52 53static bool 54upload_blorp_shader(struct blorp_batch *batch, 55 const void *key, uint32_t key_size, 56 const void *kernel, uint32_t kernel_size, 57 const struct brw_stage_prog_data *prog_data, 58 uint32_t prog_data_size, 59 uint32_t *kernel_out, void *prog_data_out) 60{ 61 struct blorp_context *blorp = batch->blorp; 62 struct anv_device *device = blorp->driver_ctx; 63 64 /* The blorp cache must be a real cache */ 65 assert(device->default_pipeline_cache.cache); 66 67 struct anv_pipeline_bind_map bind_map = { 68 .surface_count = 0, 69 .sampler_count = 0, 70 }; 71 72 struct anv_shader_bin *bin = 73 anv_pipeline_cache_upload_kernel(&device->default_pipeline_cache, 74 key, key_size, kernel, kernel_size, 75 NULL, 0, 76 prog_data, prog_data_size, 77 NULL, &bind_map); 78 79 if (!bin) 80 return false; 81 82 /* The cache already has a reference and it's not going anywhere so there 83 * is no need to hold a second reference. 84 */ 85 anv_shader_bin_unref(device, bin); 86 87 *kernel_out = bin->kernel.offset; 88 *(const struct brw_stage_prog_data **)prog_data_out = bin->prog_data; 89 90 return true; 91} 92 93void 94anv_device_init_blorp(struct anv_device *device) 95{ 96 blorp_init(&device->blorp, device, &device->isl_dev); 97 device->blorp.compiler = device->instance->physicalDevice.compiler; 98 device->blorp.lookup_shader = lookup_blorp_shader; 99 device->blorp.upload_shader = upload_blorp_shader; 100 switch (device->info.gen) { 101 case 7: 102 if (device->info.is_haswell) { 103 device->blorp.exec = gen75_blorp_exec; 104 } else { 105 device->blorp.exec = gen7_blorp_exec; 106 } 107 break; 108 case 8: 109 device->blorp.exec = gen8_blorp_exec; 110 break; 111 case 9: 112 device->blorp.exec = gen9_blorp_exec; 113 break; 114 case 10: 115 device->blorp.exec = gen10_blorp_exec; 116 break; 117 case 11: 118 device->blorp.exec = gen11_blorp_exec; 119 break; 120 default: 121 unreachable("Unknown hardware generation"); 122 } 123} 124 125void 126anv_device_finish_blorp(struct anv_device *device) 127{ 128 blorp_finish(&device->blorp); 129} 130 131static void 132get_blorp_surf_for_anv_buffer(struct anv_device *device, 133 struct anv_buffer *buffer, uint64_t offset, 134 uint32_t width, uint32_t height, 135 uint32_t row_pitch, enum isl_format format, 136 struct blorp_surf *blorp_surf, 137 struct isl_surf *isl_surf) 138{ 139 const struct isl_format_layout *fmtl = 140 isl_format_get_layout(format); 141 bool ok UNUSED; 142 143 /* ASTC is the only format which doesn't support linear layouts. 144 * Create an equivalently sized surface with ISL to get around this. 145 */ 146 if (fmtl->txc == ISL_TXC_ASTC) { 147 /* Use an equivalently sized format */ 148 format = ISL_FORMAT_R32G32B32A32_UINT; 149 assert(fmtl->bpb == isl_format_get_layout(format)->bpb); 150 151 /* Shrink the dimensions for the new format */ 152 width = DIV_ROUND_UP(width, fmtl->bw); 153 height = DIV_ROUND_UP(height, fmtl->bh); 154 } 155 156 *blorp_surf = (struct blorp_surf) { 157 .surf = isl_surf, 158 .addr = { 159 .buffer = buffer->address.bo, 160 .offset = buffer->address.offset + offset, 161 .mocs = anv_mocs_for_bo(device, buffer->address.bo), 162 }, 163 }; 164 165 ok = isl_surf_init(&device->isl_dev, isl_surf, 166 .dim = ISL_SURF_DIM_2D, 167 .format = format, 168 .width = width, 169 .height = height, 170 .depth = 1, 171 .levels = 1, 172 .array_len = 1, 173 .samples = 1, 174 .row_pitch_B = row_pitch, 175 .usage = ISL_SURF_USAGE_TEXTURE_BIT | 176 ISL_SURF_USAGE_RENDER_TARGET_BIT, 177 .tiling_flags = ISL_TILING_LINEAR_BIT); 178 assert(ok); 179} 180 181/* Pick something high enough that it won't be used in core and low enough it 182 * will never map to an extension. 183 */ 184#define ANV_IMAGE_LAYOUT_EXPLICIT_AUX (VkImageLayout)10000000 185 186static struct blorp_address 187anv_to_blorp_address(struct anv_address addr) 188{ 189 return (struct blorp_address) { 190 .buffer = addr.bo, 191 .offset = addr.offset, 192 }; 193} 194 195static void 196get_blorp_surf_for_anv_image(const struct anv_device *device, 197 const struct anv_image *image, 198 VkImageAspectFlags aspect, 199 VkImageLayout layout, 200 enum isl_aux_usage aux_usage, 201 struct blorp_surf *blorp_surf) 202{ 203 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect); 204 205 if (layout != ANV_IMAGE_LAYOUT_EXPLICIT_AUX) 206 aux_usage = anv_layout_to_aux_usage(&device->info, image, aspect, layout); 207 208 const struct anv_surface *surface = &image->planes[plane].surface; 209 *blorp_surf = (struct blorp_surf) { 210 .surf = &surface->isl, 211 .addr = { 212 .buffer = image->planes[plane].address.bo, 213 .offset = image->planes[plane].address.offset + surface->offset, 214 .mocs = anv_mocs_for_bo(device, image->planes[plane].address.bo), 215 }, 216 }; 217 218 if (aux_usage != ISL_AUX_USAGE_NONE) { 219 const struct anv_surface *aux_surface = &image->planes[plane].aux_surface; 220 blorp_surf->aux_surf = &aux_surface->isl, 221 blorp_surf->aux_addr = (struct blorp_address) { 222 .buffer = image->planes[plane].address.bo, 223 .offset = image->planes[plane].address.offset + aux_surface->offset, 224 .mocs = anv_mocs_for_bo(device, image->planes[plane].address.bo), 225 }; 226 blorp_surf->aux_usage = aux_usage; 227 228 /* If we're doing a partial resolve, then we need the indirect clear 229 * color. If we are doing a fast clear and want to store/update the 230 * clear color, we also pass the address to blorp, otherwise it will only 231 * stomp the CCS to a particular value and won't care about format or 232 * clear value 233 */ 234 if (aspect & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) { 235 const struct anv_address clear_color_addr = 236 anv_image_get_clear_color_addr(device, image, aspect); 237 blorp_surf->clear_color_addr = anv_to_blorp_address(clear_color_addr); 238 } else if (aspect & VK_IMAGE_ASPECT_DEPTH_BIT 239 && device->info.gen >= 10) { 240 /* Vulkan always clears to 1.0. On gen < 10, we set that directly in 241 * the state packet. For gen >= 10, must provide the clear value in a 242 * buffer. We have a single global buffer that stores the 1.0 value. 243 */ 244 const struct anv_address clear_color_addr = (struct anv_address) { 245 .bo = (struct anv_bo *)&device->hiz_clear_bo 246 }; 247 blorp_surf->clear_color_addr = anv_to_blorp_address(clear_color_addr); 248 } 249 } 250} 251 252void anv_CmdCopyImage( 253 VkCommandBuffer commandBuffer, 254 VkImage srcImage, 255 VkImageLayout srcImageLayout, 256 VkImage dstImage, 257 VkImageLayout dstImageLayout, 258 uint32_t regionCount, 259 const VkImageCopy* pRegions) 260{ 261 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer); 262 ANV_FROM_HANDLE(anv_image, src_image, srcImage); 263 ANV_FROM_HANDLE(anv_image, dst_image, dstImage); 264 265 struct blorp_batch batch; 266 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0); 267 268 for (unsigned r = 0; r < regionCount; r++) { 269 VkOffset3D srcOffset = 270 anv_sanitize_image_offset(src_image->type, pRegions[r].srcOffset); 271 VkOffset3D dstOffset = 272 anv_sanitize_image_offset(dst_image->type, pRegions[r].dstOffset); 273 VkExtent3D extent = 274 anv_sanitize_image_extent(src_image->type, pRegions[r].extent); 275 276 const uint32_t dst_level = pRegions[r].dstSubresource.mipLevel; 277 unsigned dst_base_layer, layer_count; 278 if (dst_image->type == VK_IMAGE_TYPE_3D) { 279 dst_base_layer = pRegions[r].dstOffset.z; 280 layer_count = pRegions[r].extent.depth; 281 } else { 282 dst_base_layer = pRegions[r].dstSubresource.baseArrayLayer; 283 layer_count = 284 anv_get_layerCount(dst_image, &pRegions[r].dstSubresource); 285 } 286 287 const uint32_t src_level = pRegions[r].srcSubresource.mipLevel; 288 unsigned src_base_layer; 289 if (src_image->type == VK_IMAGE_TYPE_3D) { 290 src_base_layer = pRegions[r].srcOffset.z; 291 } else { 292 src_base_layer = pRegions[r].srcSubresource.baseArrayLayer; 293 assert(layer_count == 294 anv_get_layerCount(src_image, &pRegions[r].srcSubresource)); 295 } 296 297 VkImageAspectFlags src_mask = pRegions[r].srcSubresource.aspectMask, 298 dst_mask = pRegions[r].dstSubresource.aspectMask; 299 300 assert(anv_image_aspects_compatible(src_mask, dst_mask)); 301 302 if (util_bitcount(src_mask) > 1) { 303 uint32_t aspect_bit; 304 anv_foreach_image_aspect_bit(aspect_bit, src_image, src_mask) { 305 struct blorp_surf src_surf, dst_surf; 306 get_blorp_surf_for_anv_image(cmd_buffer->device, 307 src_image, 1UL << aspect_bit, 308 srcImageLayout, ISL_AUX_USAGE_NONE, 309 &src_surf); 310 get_blorp_surf_for_anv_image(cmd_buffer->device, 311 dst_image, 1UL << aspect_bit, 312 dstImageLayout, ISL_AUX_USAGE_NONE, 313 &dst_surf); 314 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image, 315 1UL << aspect_bit, 316 dst_surf.aux_usage, dst_level, 317 dst_base_layer, layer_count); 318 319 for (unsigned i = 0; i < layer_count; i++) { 320 blorp_copy(&batch, &src_surf, src_level, src_base_layer + i, 321 &dst_surf, dst_level, dst_base_layer + i, 322 srcOffset.x, srcOffset.y, 323 dstOffset.x, dstOffset.y, 324 extent.width, extent.height); 325 } 326 } 327 } else { 328 struct blorp_surf src_surf, dst_surf; 329 get_blorp_surf_for_anv_image(cmd_buffer->device, src_image, src_mask, 330 srcImageLayout, ISL_AUX_USAGE_NONE, 331 &src_surf); 332 get_blorp_surf_for_anv_image(cmd_buffer->device, dst_image, dst_mask, 333 dstImageLayout, ISL_AUX_USAGE_NONE, 334 &dst_surf); 335 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image, dst_mask, 336 dst_surf.aux_usage, dst_level, 337 dst_base_layer, layer_count); 338 339 for (unsigned i = 0; i < layer_count; i++) { 340 blorp_copy(&batch, &src_surf, src_level, src_base_layer + i, 341 &dst_surf, dst_level, dst_base_layer + i, 342 srcOffset.x, srcOffset.y, 343 dstOffset.x, dstOffset.y, 344 extent.width, extent.height); 345 } 346 } 347 } 348 349 blorp_batch_finish(&batch); 350} 351 352static void 353copy_buffer_to_image(struct anv_cmd_buffer *cmd_buffer, 354 struct anv_buffer *anv_buffer, 355 struct anv_image *anv_image, 356 VkImageLayout image_layout, 357 uint32_t regionCount, 358 const VkBufferImageCopy* pRegions, 359 bool buffer_to_image) 360{ 361 struct blorp_batch batch; 362 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0); 363 364 struct { 365 struct blorp_surf surf; 366 uint32_t level; 367 VkOffset3D offset; 368 } image, buffer, *src, *dst; 369 370 buffer.level = 0; 371 buffer.offset = (VkOffset3D) { 0, 0, 0 }; 372 373 if (buffer_to_image) { 374 src = &buffer; 375 dst = ℑ 376 } else { 377 src = ℑ 378 dst = &buffer; 379 } 380 381 for (unsigned r = 0; r < regionCount; r++) { 382 const VkImageAspectFlags aspect = pRegions[r].imageSubresource.aspectMask; 383 384 get_blorp_surf_for_anv_image(cmd_buffer->device, anv_image, aspect, 385 image_layout, ISL_AUX_USAGE_NONE, 386 &image.surf); 387 image.offset = 388 anv_sanitize_image_offset(anv_image->type, pRegions[r].imageOffset); 389 image.level = pRegions[r].imageSubresource.mipLevel; 390 391 VkExtent3D extent = 392 anv_sanitize_image_extent(anv_image->type, pRegions[r].imageExtent); 393 if (anv_image->type != VK_IMAGE_TYPE_3D) { 394 image.offset.z = pRegions[r].imageSubresource.baseArrayLayer; 395 extent.depth = 396 anv_get_layerCount(anv_image, &pRegions[r].imageSubresource); 397 } 398 399 const enum isl_format buffer_format = 400 anv_get_isl_format(&cmd_buffer->device->info, anv_image->vk_format, 401 aspect, VK_IMAGE_TILING_LINEAR); 402 403 const VkExtent3D bufferImageExtent = { 404 .width = pRegions[r].bufferRowLength ? 405 pRegions[r].bufferRowLength : extent.width, 406 .height = pRegions[r].bufferImageHeight ? 407 pRegions[r].bufferImageHeight : extent.height, 408 }; 409 410 const struct isl_format_layout *buffer_fmtl = 411 isl_format_get_layout(buffer_format); 412 413 const uint32_t buffer_row_pitch = 414 DIV_ROUND_UP(bufferImageExtent.width, buffer_fmtl->bw) * 415 (buffer_fmtl->bpb / 8); 416 417 const uint32_t buffer_layer_stride = 418 DIV_ROUND_UP(bufferImageExtent.height, buffer_fmtl->bh) * 419 buffer_row_pitch; 420 421 struct isl_surf buffer_isl_surf; 422 get_blorp_surf_for_anv_buffer(cmd_buffer->device, 423 anv_buffer, pRegions[r].bufferOffset, 424 extent.width, extent.height, 425 buffer_row_pitch, buffer_format, 426 &buffer.surf, &buffer_isl_surf); 427 428 if (&image == dst) { 429 anv_cmd_buffer_mark_image_written(cmd_buffer, anv_image, 430 aspect, dst->surf.aux_usage, 431 dst->level, 432 dst->offset.z, extent.depth); 433 } 434 435 for (unsigned z = 0; z < extent.depth; z++) { 436 blorp_copy(&batch, &src->surf, src->level, src->offset.z, 437 &dst->surf, dst->level, dst->offset.z, 438 src->offset.x, src->offset.y, dst->offset.x, dst->offset.y, 439 extent.width, extent.height); 440 441 image.offset.z++; 442 buffer.surf.addr.offset += buffer_layer_stride; 443 } 444 } 445 446 blorp_batch_finish(&batch); 447} 448 449void anv_CmdCopyBufferToImage( 450 VkCommandBuffer commandBuffer, 451 VkBuffer srcBuffer, 452 VkImage dstImage, 453 VkImageLayout dstImageLayout, 454 uint32_t regionCount, 455 const VkBufferImageCopy* pRegions) 456{ 457 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer); 458 ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer); 459 ANV_FROM_HANDLE(anv_image, dst_image, dstImage); 460 461 copy_buffer_to_image(cmd_buffer, src_buffer, dst_image, dstImageLayout, 462 regionCount, pRegions, true); 463} 464 465void anv_CmdCopyImageToBuffer( 466 VkCommandBuffer commandBuffer, 467 VkImage srcImage, 468 VkImageLayout srcImageLayout, 469 VkBuffer dstBuffer, 470 uint32_t regionCount, 471 const VkBufferImageCopy* pRegions) 472{ 473 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer); 474 ANV_FROM_HANDLE(anv_image, src_image, srcImage); 475 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer); 476 477 copy_buffer_to_image(cmd_buffer, dst_buffer, src_image, srcImageLayout, 478 regionCount, pRegions, false); 479 480 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES; 481} 482 483static bool 484flip_coords(unsigned *src0, unsigned *src1, unsigned *dst0, unsigned *dst1) 485{ 486 bool flip = false; 487 if (*src0 > *src1) { 488 unsigned tmp = *src0; 489 *src0 = *src1; 490 *src1 = tmp; 491 flip = !flip; 492 } 493 494 if (*dst0 > *dst1) { 495 unsigned tmp = *dst0; 496 *dst0 = *dst1; 497 *dst1 = tmp; 498 flip = !flip; 499 } 500 501 return flip; 502} 503 504void anv_CmdBlitImage( 505 VkCommandBuffer commandBuffer, 506 VkImage srcImage, 507 VkImageLayout srcImageLayout, 508 VkImage dstImage, 509 VkImageLayout dstImageLayout, 510 uint32_t regionCount, 511 const VkImageBlit* pRegions, 512 VkFilter filter) 513 514{ 515 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer); 516 ANV_FROM_HANDLE(anv_image, src_image, srcImage); 517 ANV_FROM_HANDLE(anv_image, dst_image, dstImage); 518 519 struct blorp_surf src, dst; 520 521 enum blorp_filter blorp_filter; 522 switch (filter) { 523 case VK_FILTER_NEAREST: 524 blorp_filter = BLORP_FILTER_NEAREST; 525 break; 526 case VK_FILTER_LINEAR: 527 blorp_filter = BLORP_FILTER_BILINEAR; 528 break; 529 default: 530 unreachable("Invalid filter"); 531 } 532 533 struct blorp_batch batch; 534 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0); 535 536 for (unsigned r = 0; r < regionCount; r++) { 537 const VkImageSubresourceLayers *src_res = &pRegions[r].srcSubresource; 538 const VkImageSubresourceLayers *dst_res = &pRegions[r].dstSubresource; 539 540 assert(anv_image_aspects_compatible(src_res->aspectMask, 541 dst_res->aspectMask)); 542 543 uint32_t aspect_bit; 544 anv_foreach_image_aspect_bit(aspect_bit, src_image, src_res->aspectMask) { 545 get_blorp_surf_for_anv_image(cmd_buffer->device, 546 src_image, 1U << aspect_bit, 547 srcImageLayout, ISL_AUX_USAGE_NONE, &src); 548 get_blorp_surf_for_anv_image(cmd_buffer->device, 549 dst_image, 1U << aspect_bit, 550 dstImageLayout, ISL_AUX_USAGE_NONE, &dst); 551 552 struct anv_format_plane src_format = 553 anv_get_format_plane(&cmd_buffer->device->info, src_image->vk_format, 554 1U << aspect_bit, src_image->tiling); 555 struct anv_format_plane dst_format = 556 anv_get_format_plane(&cmd_buffer->device->info, dst_image->vk_format, 557 1U << aspect_bit, dst_image->tiling); 558 559 unsigned dst_start, dst_end; 560 if (dst_image->type == VK_IMAGE_TYPE_3D) { 561 assert(dst_res->baseArrayLayer == 0); 562 dst_start = pRegions[r].dstOffsets[0].z; 563 dst_end = pRegions[r].dstOffsets[1].z; 564 } else { 565 dst_start = dst_res->baseArrayLayer; 566 dst_end = dst_start + anv_get_layerCount(dst_image, dst_res); 567 } 568 569 unsigned src_start, src_end; 570 if (src_image->type == VK_IMAGE_TYPE_3D) { 571 assert(src_res->baseArrayLayer == 0); 572 src_start = pRegions[r].srcOffsets[0].z; 573 src_end = pRegions[r].srcOffsets[1].z; 574 } else { 575 src_start = src_res->baseArrayLayer; 576 src_end = src_start + anv_get_layerCount(src_image, src_res); 577 } 578 579 bool flip_z = flip_coords(&src_start, &src_end, &dst_start, &dst_end); 580 float src_z_step = (float)(src_end + 1 - src_start) / 581 (float)(dst_end + 1 - dst_start); 582 583 if (flip_z) { 584 src_start = src_end; 585 src_z_step *= -1; 586 } 587 588 unsigned src_x0 = pRegions[r].srcOffsets[0].x; 589 unsigned src_x1 = pRegions[r].srcOffsets[1].x; 590 unsigned dst_x0 = pRegions[r].dstOffsets[0].x; 591 unsigned dst_x1 = pRegions[r].dstOffsets[1].x; 592 bool flip_x = flip_coords(&src_x0, &src_x1, &dst_x0, &dst_x1); 593 594 unsigned src_y0 = pRegions[r].srcOffsets[0].y; 595 unsigned src_y1 = pRegions[r].srcOffsets[1].y; 596 unsigned dst_y0 = pRegions[r].dstOffsets[0].y; 597 unsigned dst_y1 = pRegions[r].dstOffsets[1].y; 598 bool flip_y = flip_coords(&src_y0, &src_y1, &dst_y0, &dst_y1); 599 600 const unsigned num_layers = dst_end - dst_start; 601 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image, 602 1U << aspect_bit, 603 dst.aux_usage, 604 dst_res->mipLevel, 605 dst_start, num_layers); 606 607 for (unsigned i = 0; i < num_layers; i++) { 608 unsigned dst_z = dst_start + i; 609 unsigned src_z = src_start + i * src_z_step; 610 611 blorp_blit(&batch, &src, src_res->mipLevel, src_z, 612 src_format.isl_format, src_format.swizzle, 613 &dst, dst_res->mipLevel, dst_z, 614 dst_format.isl_format, dst_format.swizzle, 615 src_x0, src_y0, src_x1, src_y1, 616 dst_x0, dst_y0, dst_x1, dst_y1, 617 blorp_filter, flip_x, flip_y); 618 } 619 } 620 } 621 622 blorp_batch_finish(&batch); 623} 624 625static enum isl_format 626isl_format_for_size(unsigned size_B) 627{ 628 switch (size_B) { 629 case 4: return ISL_FORMAT_R32_UINT; 630 case 8: return ISL_FORMAT_R32G32_UINT; 631 case 16: return ISL_FORMAT_R32G32B32A32_UINT; 632 default: 633 unreachable("Not a power-of-two format size"); 634 } 635} 636 637/** 638 * Returns the greatest common divisor of a and b that is a power of two. 639 */ 640static uint64_t 641gcd_pow2_u64(uint64_t a, uint64_t b) 642{ 643 assert(a > 0 || b > 0); 644 645 unsigned a_log2 = ffsll(a) - 1; 646 unsigned b_log2 = ffsll(b) - 1; 647 648 /* If either a or b is 0, then a_log2 or b_log2 till be UINT_MAX in which 649 * case, the MIN2() will take the other one. If both are 0 then we will 650 * hit the assert above. 651 */ 652 return 1 << MIN2(a_log2, b_log2); 653} 654 655/* This is maximum possible width/height our HW can handle */ 656#define MAX_SURFACE_DIM (1ull << 14) 657 658void anv_CmdCopyBuffer( 659 VkCommandBuffer commandBuffer, 660 VkBuffer srcBuffer, 661 VkBuffer dstBuffer, 662 uint32_t regionCount, 663 const VkBufferCopy* pRegions) 664{ 665 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer); 666 ANV_FROM_HANDLE(anv_buffer, src_buffer, srcBuffer); 667 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer); 668 669 struct blorp_batch batch; 670 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0); 671 672 for (unsigned r = 0; r < regionCount; r++) { 673 struct blorp_address src = { 674 .buffer = src_buffer->address.bo, 675 .offset = src_buffer->address.offset + pRegions[r].srcOffset, 676 .mocs = anv_mocs_for_bo(cmd_buffer->device, src_buffer->address.bo), 677 }; 678 struct blorp_address dst = { 679 .buffer = dst_buffer->address.bo, 680 .offset = dst_buffer->address.offset + pRegions[r].dstOffset, 681 .mocs = anv_mocs_for_bo(cmd_buffer->device, dst_buffer->address.bo), 682 }; 683 684 blorp_buffer_copy(&batch, src, dst, pRegions[r].size); 685 } 686 687 blorp_batch_finish(&batch); 688 689 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES; 690} 691 692void anv_CmdUpdateBuffer( 693 VkCommandBuffer commandBuffer, 694 VkBuffer dstBuffer, 695 VkDeviceSize dstOffset, 696 VkDeviceSize dataSize, 697 const void* pData) 698{ 699 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer); 700 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer); 701 702 struct blorp_batch batch; 703 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0); 704 705 /* We can't quite grab a full block because the state stream needs a 706 * little data at the top to build its linked list. 707 */ 708 const uint32_t max_update_size = 709 cmd_buffer->device->dynamic_state_pool.block_size - 64; 710 711 assert(max_update_size < MAX_SURFACE_DIM * 4); 712 713 /* We're about to read data that was written from the CPU. Flush the 714 * texture cache so we don't get anything stale. 715 */ 716 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_TEXTURE_CACHE_INVALIDATE_BIT; 717 718 while (dataSize) { 719 const uint32_t copy_size = MIN2(dataSize, max_update_size); 720 721 struct anv_state tmp_data = 722 anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, copy_size, 64); 723 724 memcpy(tmp_data.map, pData, copy_size); 725 726 struct blorp_address src = { 727 .buffer = cmd_buffer->device->dynamic_state_pool.block_pool.bo, 728 .offset = tmp_data.offset, 729 .mocs = cmd_buffer->device->default_mocs, 730 }; 731 struct blorp_address dst = { 732 .buffer = dst_buffer->address.bo, 733 .offset = dst_buffer->address.offset + dstOffset, 734 .mocs = anv_mocs_for_bo(cmd_buffer->device, dst_buffer->address.bo), 735 }; 736 737 blorp_buffer_copy(&batch, src, dst, copy_size); 738 739 dataSize -= copy_size; 740 dstOffset += copy_size; 741 pData = (void *)pData + copy_size; 742 } 743 744 blorp_batch_finish(&batch); 745 746 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES; 747} 748 749void anv_CmdFillBuffer( 750 VkCommandBuffer commandBuffer, 751 VkBuffer dstBuffer, 752 VkDeviceSize dstOffset, 753 VkDeviceSize fillSize, 754 uint32_t data) 755{ 756 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer); 757 ANV_FROM_HANDLE(anv_buffer, dst_buffer, dstBuffer); 758 struct blorp_surf surf; 759 struct isl_surf isl_surf; 760 761 struct blorp_batch batch; 762 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0); 763 764 fillSize = anv_buffer_get_range(dst_buffer, dstOffset, fillSize); 765 766 /* From the Vulkan spec: 767 * 768 * "size is the number of bytes to fill, and must be either a multiple 769 * of 4, or VK_WHOLE_SIZE to fill the range from offset to the end of 770 * the buffer. If VK_WHOLE_SIZE is used and the remaining size of the 771 * buffer is not a multiple of 4, then the nearest smaller multiple is 772 * used." 773 */ 774 fillSize &= ~3ull; 775 776 /* First, we compute the biggest format that can be used with the 777 * given offsets and size. 778 */ 779 int bs = 16; 780 bs = gcd_pow2_u64(bs, dstOffset); 781 bs = gcd_pow2_u64(bs, fillSize); 782 enum isl_format isl_format = isl_format_for_size(bs); 783 784 union isl_color_value color = { 785 .u32 = { data, data, data, data }, 786 }; 787 788 const uint64_t max_fill_size = MAX_SURFACE_DIM * MAX_SURFACE_DIM * bs; 789 while (fillSize >= max_fill_size) { 790 get_blorp_surf_for_anv_buffer(cmd_buffer->device, 791 dst_buffer, dstOffset, 792 MAX_SURFACE_DIM, MAX_SURFACE_DIM, 793 MAX_SURFACE_DIM * bs, isl_format, 794 &surf, &isl_surf); 795 796 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY, 797 0, 0, 1, 0, 0, MAX_SURFACE_DIM, MAX_SURFACE_DIM, 798 color, NULL); 799 fillSize -= max_fill_size; 800 dstOffset += max_fill_size; 801 } 802 803 uint64_t height = fillSize / (MAX_SURFACE_DIM * bs); 804 assert(height < MAX_SURFACE_DIM); 805 if (height != 0) { 806 const uint64_t rect_fill_size = height * MAX_SURFACE_DIM * bs; 807 get_blorp_surf_for_anv_buffer(cmd_buffer->device, 808 dst_buffer, dstOffset, 809 MAX_SURFACE_DIM, height, 810 MAX_SURFACE_DIM * bs, isl_format, 811 &surf, &isl_surf); 812 813 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY, 814 0, 0, 1, 0, 0, MAX_SURFACE_DIM, height, 815 color, NULL); 816 fillSize -= rect_fill_size; 817 dstOffset += rect_fill_size; 818 } 819 820 if (fillSize != 0) { 821 const uint32_t width = fillSize / bs; 822 get_blorp_surf_for_anv_buffer(cmd_buffer->device, 823 dst_buffer, dstOffset, 824 width, 1, 825 width * bs, isl_format, 826 &surf, &isl_surf); 827 828 blorp_clear(&batch, &surf, isl_format, ISL_SWIZZLE_IDENTITY, 829 0, 0, 1, 0, 0, width, 1, 830 color, NULL); 831 } 832 833 blorp_batch_finish(&batch); 834 835 cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_RENDER_TARGET_BUFFER_WRITES; 836} 837 838void anv_CmdClearColorImage( 839 VkCommandBuffer commandBuffer, 840 VkImage _image, 841 VkImageLayout imageLayout, 842 const VkClearColorValue* pColor, 843 uint32_t rangeCount, 844 const VkImageSubresourceRange* pRanges) 845{ 846 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer); 847 ANV_FROM_HANDLE(anv_image, image, _image); 848 849 static const bool color_write_disable[4] = { false, false, false, false }; 850 851 struct blorp_batch batch; 852 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0); 853 854 855 for (unsigned r = 0; r < rangeCount; r++) { 856 if (pRanges[r].aspectMask == 0) 857 continue; 858 859 assert(pRanges[r].aspectMask & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV); 860 861 struct blorp_surf surf; 862 get_blorp_surf_for_anv_image(cmd_buffer->device, 863 image, pRanges[r].aspectMask, 864 imageLayout, ISL_AUX_USAGE_NONE, &surf); 865 866 struct anv_format_plane src_format = 867 anv_get_format_plane(&cmd_buffer->device->info, image->vk_format, 868 VK_IMAGE_ASPECT_COLOR_BIT, image->tiling); 869 870 unsigned base_layer = pRanges[r].baseArrayLayer; 871 unsigned layer_count = anv_get_layerCount(image, &pRanges[r]); 872 873 for (unsigned i = 0; i < anv_get_levelCount(image, &pRanges[r]); i++) { 874 const unsigned level = pRanges[r].baseMipLevel + i; 875 const unsigned level_width = anv_minify(image->extent.width, level); 876 const unsigned level_height = anv_minify(image->extent.height, level); 877 878 if (image->type == VK_IMAGE_TYPE_3D) { 879 base_layer = 0; 880 layer_count = anv_minify(image->extent.depth, level); 881 } 882 883 anv_cmd_buffer_mark_image_written(cmd_buffer, image, 884 pRanges[r].aspectMask, 885 surf.aux_usage, level, 886 base_layer, layer_count); 887 888 blorp_clear(&batch, &surf, 889 src_format.isl_format, src_format.swizzle, 890 level, base_layer, layer_count, 891 0, 0, level_width, level_height, 892 vk_to_isl_color(*pColor), color_write_disable); 893 } 894 } 895 896 blorp_batch_finish(&batch); 897} 898 899void anv_CmdClearDepthStencilImage( 900 VkCommandBuffer commandBuffer, 901 VkImage image_h, 902 VkImageLayout imageLayout, 903 const VkClearDepthStencilValue* pDepthStencil, 904 uint32_t rangeCount, 905 const VkImageSubresourceRange* pRanges) 906{ 907 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer); 908 ANV_FROM_HANDLE(anv_image, image, image_h); 909 910 struct blorp_batch batch; 911 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0); 912 913 struct blorp_surf depth, stencil; 914 if (image->aspects & VK_IMAGE_ASPECT_DEPTH_BIT) { 915 get_blorp_surf_for_anv_image(cmd_buffer->device, 916 image, VK_IMAGE_ASPECT_DEPTH_BIT, 917 imageLayout, ISL_AUX_USAGE_NONE, &depth); 918 } else { 919 memset(&depth, 0, sizeof(depth)); 920 } 921 922 if (image->aspects & VK_IMAGE_ASPECT_STENCIL_BIT) { 923 get_blorp_surf_for_anv_image(cmd_buffer->device, 924 image, VK_IMAGE_ASPECT_STENCIL_BIT, 925 imageLayout, ISL_AUX_USAGE_NONE, &stencil); 926 } else { 927 memset(&stencil, 0, sizeof(stencil)); 928 } 929 930 for (unsigned r = 0; r < rangeCount; r++) { 931 if (pRanges[r].aspectMask == 0) 932 continue; 933 934 bool clear_depth = pRanges[r].aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT; 935 bool clear_stencil = pRanges[r].aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT; 936 937 unsigned base_layer = pRanges[r].baseArrayLayer; 938 unsigned layer_count = anv_get_layerCount(image, &pRanges[r]); 939 940 for (unsigned i = 0; i < anv_get_levelCount(image, &pRanges[r]); i++) { 941 const unsigned level = pRanges[r].baseMipLevel + i; 942 const unsigned level_width = anv_minify(image->extent.width, level); 943 const unsigned level_height = anv_minify(image->extent.height, level); 944 945 if (image->type == VK_IMAGE_TYPE_3D) 946 layer_count = anv_minify(image->extent.depth, level); 947 948 blorp_clear_depth_stencil(&batch, &depth, &stencil, 949 level, base_layer, layer_count, 950 0, 0, level_width, level_height, 951 clear_depth, pDepthStencil->depth, 952 clear_stencil ? 0xff : 0, 953 pDepthStencil->stencil); 954 } 955 } 956 957 blorp_batch_finish(&batch); 958} 959 960VkResult 961anv_cmd_buffer_alloc_blorp_binding_table(struct anv_cmd_buffer *cmd_buffer, 962 uint32_t num_entries, 963 uint32_t *state_offset, 964 struct anv_state *bt_state) 965{ 966 *bt_state = anv_cmd_buffer_alloc_binding_table(cmd_buffer, num_entries, 967 state_offset); 968 if (bt_state->map == NULL) { 969 /* We ran out of space. Grab a new binding table block. */ 970 VkResult result = anv_cmd_buffer_new_binding_table_block(cmd_buffer); 971 if (result != VK_SUCCESS) 972 return result; 973 974 /* Re-emit state base addresses so we get the new surface state base 975 * address before we start emitting binding tables etc. 976 */ 977 anv_cmd_buffer_emit_state_base_address(cmd_buffer); 978 979 *bt_state = anv_cmd_buffer_alloc_binding_table(cmd_buffer, num_entries, 980 state_offset); 981 assert(bt_state->map != NULL); 982 } 983 984 return VK_SUCCESS; 985} 986 987static VkResult 988binding_table_for_surface_state(struct anv_cmd_buffer *cmd_buffer, 989 struct anv_state surface_state, 990 uint32_t *bt_offset) 991{ 992 uint32_t state_offset; 993 struct anv_state bt_state; 994 995 VkResult result = 996 anv_cmd_buffer_alloc_blorp_binding_table(cmd_buffer, 1, &state_offset, 997 &bt_state); 998 if (result != VK_SUCCESS) 999 return result; 1000 1001 uint32_t *bt_map = bt_state.map; 1002 bt_map[0] = surface_state.offset + state_offset; 1003 1004 *bt_offset = bt_state.offset; 1005 return VK_SUCCESS; 1006} 1007 1008static void 1009clear_color_attachment(struct anv_cmd_buffer *cmd_buffer, 1010 struct blorp_batch *batch, 1011 const VkClearAttachment *attachment, 1012 uint32_t rectCount, const VkClearRect *pRects) 1013{ 1014 const struct anv_subpass *subpass = cmd_buffer->state.subpass; 1015 const uint32_t color_att = attachment->colorAttachment; 1016 assert(color_att < subpass->color_count); 1017 const uint32_t att_idx = subpass->color_attachments[color_att].attachment; 1018 1019 if (att_idx == VK_ATTACHMENT_UNUSED) 1020 return; 1021 1022 struct anv_render_pass_attachment *pass_att = 1023 &cmd_buffer->state.pass->attachments[att_idx]; 1024 struct anv_attachment_state *att_state = 1025 &cmd_buffer->state.attachments[att_idx]; 1026 1027 uint32_t binding_table; 1028 VkResult result = 1029 binding_table_for_surface_state(cmd_buffer, att_state->color.state, 1030 &binding_table); 1031 if (result != VK_SUCCESS) 1032 return; 1033 1034 union isl_color_value clear_color = 1035 vk_to_isl_color(attachment->clearValue.color); 1036 1037 /* If multiview is enabled we ignore baseArrayLayer and layerCount */ 1038 if (subpass->view_mask) { 1039 uint32_t view_idx; 1040 for_each_bit(view_idx, subpass->view_mask) { 1041 for (uint32_t r = 0; r < rectCount; ++r) { 1042 const VkOffset2D offset = pRects[r].rect.offset; 1043 const VkExtent2D extent = pRects[r].rect.extent; 1044 blorp_clear_attachments(batch, binding_table, 1045 ISL_FORMAT_UNSUPPORTED, pass_att->samples, 1046 view_idx, 1, 1047 offset.x, offset.y, 1048 offset.x + extent.width, 1049 offset.y + extent.height, 1050 true, clear_color, false, 0.0f, 0, 0); 1051 } 1052 } 1053 return; 1054 } 1055 1056 for (uint32_t r = 0; r < rectCount; ++r) { 1057 const VkOffset2D offset = pRects[r].rect.offset; 1058 const VkExtent2D extent = pRects[r].rect.extent; 1059 assert(pRects[r].layerCount != VK_REMAINING_ARRAY_LAYERS); 1060 blorp_clear_attachments(batch, binding_table, 1061 ISL_FORMAT_UNSUPPORTED, pass_att->samples, 1062 pRects[r].baseArrayLayer, 1063 pRects[r].layerCount, 1064 offset.x, offset.y, 1065 offset.x + extent.width, offset.y + extent.height, 1066 true, clear_color, false, 0.0f, 0, 0); 1067 } 1068} 1069 1070static void 1071clear_depth_stencil_attachment(struct anv_cmd_buffer *cmd_buffer, 1072 struct blorp_batch *batch, 1073 const VkClearAttachment *attachment, 1074 uint32_t rectCount, const VkClearRect *pRects) 1075{ 1076 static const union isl_color_value color_value = { .u32 = { 0, } }; 1077 const struct anv_subpass *subpass = cmd_buffer->state.subpass; 1078 if (!subpass->depth_stencil_attachment) 1079 return; 1080 1081 const uint32_t att_idx = subpass->depth_stencil_attachment->attachment; 1082 assert(att_idx != VK_ATTACHMENT_UNUSED); 1083 struct anv_render_pass_attachment *pass_att = 1084 &cmd_buffer->state.pass->attachments[att_idx]; 1085 1086 bool clear_depth = attachment->aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT; 1087 bool clear_stencil = attachment->aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT; 1088 1089 enum isl_format depth_format = ISL_FORMAT_UNSUPPORTED; 1090 if (clear_depth) { 1091 depth_format = anv_get_isl_format(&cmd_buffer->device->info, 1092 pass_att->format, 1093 VK_IMAGE_ASPECT_DEPTH_BIT, 1094 VK_IMAGE_TILING_OPTIMAL); 1095 } 1096 1097 uint32_t binding_table; 1098 VkResult result = 1099 binding_table_for_surface_state(cmd_buffer, 1100 cmd_buffer->state.null_surface_state, 1101 &binding_table); 1102 if (result != VK_SUCCESS) 1103 return; 1104 1105 /* If multiview is enabled we ignore baseArrayLayer and layerCount */ 1106 if (subpass->view_mask) { 1107 uint32_t view_idx; 1108 for_each_bit(view_idx, subpass->view_mask) { 1109 for (uint32_t r = 0; r < rectCount; ++r) { 1110 const VkOffset2D offset = pRects[r].rect.offset; 1111 const VkExtent2D extent = pRects[r].rect.extent; 1112 VkClearDepthStencilValue value = attachment->clearValue.depthStencil; 1113 blorp_clear_attachments(batch, binding_table, 1114 depth_format, pass_att->samples, 1115 view_idx, 1, 1116 offset.x, offset.y, 1117 offset.x + extent.width, 1118 offset.y + extent.height, 1119 false, color_value, 1120 clear_depth, value.depth, 1121 clear_stencil ? 0xff : 0, value.stencil); 1122 } 1123 } 1124 return; 1125 } 1126 1127 for (uint32_t r = 0; r < rectCount; ++r) { 1128 const VkOffset2D offset = pRects[r].rect.offset; 1129 const VkExtent2D extent = pRects[r].rect.extent; 1130 VkClearDepthStencilValue value = attachment->clearValue.depthStencil; 1131 assert(pRects[r].layerCount != VK_REMAINING_ARRAY_LAYERS); 1132 blorp_clear_attachments(batch, binding_table, 1133 depth_format, pass_att->samples, 1134 pRects[r].baseArrayLayer, 1135 pRects[r].layerCount, 1136 offset.x, offset.y, 1137 offset.x + extent.width, offset.y + extent.height, 1138 false, color_value, 1139 clear_depth, value.depth, 1140 clear_stencil ? 0xff : 0, value.stencil); 1141 } 1142} 1143 1144void anv_CmdClearAttachments( 1145 VkCommandBuffer commandBuffer, 1146 uint32_t attachmentCount, 1147 const VkClearAttachment* pAttachments, 1148 uint32_t rectCount, 1149 const VkClearRect* pRects) 1150{ 1151 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer); 1152 1153 /* Because this gets called within a render pass, we tell blorp not to 1154 * trash our depth and stencil buffers. 1155 */ 1156 struct blorp_batch batch; 1157 enum blorp_batch_flags flags = BLORP_BATCH_NO_EMIT_DEPTH_STENCIL; 1158 if (cmd_buffer->state.conditional_render_enabled) { 1159 anv_cmd_emit_conditional_render_predicate(cmd_buffer); 1160 flags |= BLORP_BATCH_PREDICATE_ENABLE; 1161 } 1162 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, flags); 1163 1164 for (uint32_t a = 0; a < attachmentCount; ++a) { 1165 if (pAttachments[a].aspectMask & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV) { 1166 assert(pAttachments[a].aspectMask == VK_IMAGE_ASPECT_COLOR_BIT); 1167 clear_color_attachment(cmd_buffer, &batch, 1168 &pAttachments[a], 1169 rectCount, pRects); 1170 } else { 1171 clear_depth_stencil_attachment(cmd_buffer, &batch, 1172 &pAttachments[a], 1173 rectCount, pRects); 1174 } 1175 } 1176 1177 blorp_batch_finish(&batch); 1178} 1179 1180enum subpass_stage { 1181 SUBPASS_STAGE_LOAD, 1182 SUBPASS_STAGE_DRAW, 1183 SUBPASS_STAGE_RESOLVE, 1184}; 1185 1186void 1187anv_image_msaa_resolve(struct anv_cmd_buffer *cmd_buffer, 1188 const struct anv_image *src_image, 1189 enum isl_aux_usage src_aux_usage, 1190 uint32_t src_level, uint32_t src_base_layer, 1191 const struct anv_image *dst_image, 1192 enum isl_aux_usage dst_aux_usage, 1193 uint32_t dst_level, uint32_t dst_base_layer, 1194 VkImageAspectFlagBits aspect, 1195 uint32_t src_x, uint32_t src_y, 1196 uint32_t dst_x, uint32_t dst_y, 1197 uint32_t width, uint32_t height, 1198 uint32_t layer_count, 1199 enum blorp_filter filter) 1200{ 1201 struct blorp_batch batch; 1202 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0); 1203 1204 assert(src_image->type == VK_IMAGE_TYPE_2D); 1205 assert(src_image->samples > 1); 1206 assert(dst_image->type == VK_IMAGE_TYPE_2D); 1207 assert(dst_image->samples == 1); 1208 assert(src_image->n_planes == dst_image->n_planes); 1209 assert(!src_image->format->can_ycbcr); 1210 assert(!dst_image->format->can_ycbcr); 1211 1212 struct blorp_surf src_surf, dst_surf; 1213 get_blorp_surf_for_anv_image(cmd_buffer->device, src_image, aspect, 1214 ANV_IMAGE_LAYOUT_EXPLICIT_AUX, 1215 src_aux_usage, &src_surf); 1216 if (src_aux_usage == ISL_AUX_USAGE_MCS) { 1217 src_surf.clear_color_addr = anv_to_blorp_address( 1218 anv_image_get_clear_color_addr(cmd_buffer->device, src_image, 1219 VK_IMAGE_ASPECT_COLOR_BIT)); 1220 } 1221 get_blorp_surf_for_anv_image(cmd_buffer->device, dst_image, aspect, 1222 ANV_IMAGE_LAYOUT_EXPLICIT_AUX, 1223 dst_aux_usage, &dst_surf); 1224 anv_cmd_buffer_mark_image_written(cmd_buffer, dst_image, 1225 aspect, dst_aux_usage, 1226 dst_level, dst_base_layer, layer_count); 1227 1228 if (filter == BLORP_FILTER_NONE) { 1229 /* If no explicit filter is provided, then it's implied by the type of 1230 * the source image. 1231 */ 1232 if ((src_surf.surf->usage & ISL_SURF_USAGE_DEPTH_BIT) || 1233 (src_surf.surf->usage & ISL_SURF_USAGE_STENCIL_BIT) || 1234 isl_format_has_int_channel(src_surf.surf->format)) { 1235 filter = BLORP_FILTER_SAMPLE_0; 1236 } else { 1237 filter = BLORP_FILTER_AVERAGE; 1238 } 1239 } 1240 1241 for (uint32_t l = 0; l < layer_count; l++) { 1242 blorp_blit(&batch, 1243 &src_surf, src_level, src_base_layer + l, 1244 ISL_FORMAT_UNSUPPORTED, ISL_SWIZZLE_IDENTITY, 1245 &dst_surf, dst_level, dst_base_layer + l, 1246 ISL_FORMAT_UNSUPPORTED, ISL_SWIZZLE_IDENTITY, 1247 src_x, src_y, src_x + width, src_y + height, 1248 dst_x, dst_y, dst_x + width, dst_y + height, 1249 filter, false, false); 1250 } 1251 1252 blorp_batch_finish(&batch); 1253} 1254 1255void anv_CmdResolveImage( 1256 VkCommandBuffer commandBuffer, 1257 VkImage srcImage, 1258 VkImageLayout srcImageLayout, 1259 VkImage dstImage, 1260 VkImageLayout dstImageLayout, 1261 uint32_t regionCount, 1262 const VkImageResolve* pRegions) 1263{ 1264 ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer); 1265 ANV_FROM_HANDLE(anv_image, src_image, srcImage); 1266 ANV_FROM_HANDLE(anv_image, dst_image, dstImage); 1267 1268 assert(!src_image->format->can_ycbcr); 1269 1270 for (uint32_t r = 0; r < regionCount; r++) { 1271 assert(pRegions[r].srcSubresource.aspectMask == 1272 pRegions[r].dstSubresource.aspectMask); 1273 assert(anv_get_layerCount(src_image, &pRegions[r].srcSubresource) == 1274 anv_get_layerCount(dst_image, &pRegions[r].dstSubresource)); 1275 1276 const uint32_t layer_count = 1277 anv_get_layerCount(dst_image, &pRegions[r].dstSubresource); 1278 1279 uint32_t aspect_bit; 1280 anv_foreach_image_aspect_bit(aspect_bit, src_image, 1281 pRegions[r].srcSubresource.aspectMask) { 1282 enum isl_aux_usage src_aux_usage = 1283 anv_layout_to_aux_usage(&cmd_buffer->device->info, src_image, 1284 (1 << aspect_bit), srcImageLayout); 1285 enum isl_aux_usage dst_aux_usage = 1286 anv_layout_to_aux_usage(&cmd_buffer->device->info, dst_image, 1287 (1 << aspect_bit), dstImageLayout); 1288 1289 anv_image_msaa_resolve(cmd_buffer, 1290 src_image, src_aux_usage, 1291 pRegions[r].srcSubresource.mipLevel, 1292 pRegions[r].srcSubresource.baseArrayLayer, 1293 dst_image, dst_aux_usage, 1294 pRegions[r].dstSubresource.mipLevel, 1295 pRegions[r].dstSubresource.baseArrayLayer, 1296 (1 << aspect_bit), 1297 pRegions[r].srcOffset.x, 1298 pRegions[r].srcOffset.y, 1299 pRegions[r].dstOffset.x, 1300 pRegions[r].dstOffset.y, 1301 pRegions[r].extent.width, 1302 pRegions[r].extent.height, 1303 layer_count, BLORP_FILTER_NONE); 1304 } 1305 } 1306} 1307 1308static enum isl_aux_usage 1309fast_clear_aux_usage(const struct anv_image *image, 1310 VkImageAspectFlagBits aspect) 1311{ 1312 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect); 1313 if (image->planes[plane].aux_usage == ISL_AUX_USAGE_NONE) 1314 return ISL_AUX_USAGE_CCS_D; 1315 else 1316 return image->planes[plane].aux_usage; 1317} 1318 1319void 1320anv_image_copy_to_shadow(struct anv_cmd_buffer *cmd_buffer, 1321 const struct anv_image *image, 1322 uint32_t base_level, uint32_t level_count, 1323 uint32_t base_layer, uint32_t layer_count) 1324{ 1325 struct blorp_batch batch; 1326 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0); 1327 1328 assert(image->aspects == VK_IMAGE_ASPECT_COLOR_BIT && image->n_planes == 1); 1329 1330 struct blorp_surf surf; 1331 get_blorp_surf_for_anv_image(cmd_buffer->device, 1332 image, VK_IMAGE_ASPECT_COLOR_BIT, 1333 VK_IMAGE_LAYOUT_GENERAL, 1334 ISL_AUX_USAGE_NONE, &surf); 1335 assert(surf.aux_usage == ISL_AUX_USAGE_NONE); 1336 1337 struct blorp_surf shadow_surf = { 1338 .surf = &image->planes[0].shadow_surface.isl, 1339 .addr = { 1340 .buffer = image->planes[0].address.bo, 1341 .offset = image->planes[0].address.offset + 1342 image->planes[0].shadow_surface.offset, 1343 .mocs = anv_mocs_for_bo(cmd_buffer->device, 1344 image->planes[0].address.bo), 1345 }, 1346 }; 1347 1348 for (uint32_t l = 0; l < level_count; l++) { 1349 const uint32_t level = base_level + l; 1350 1351 const VkExtent3D extent = { 1352 .width = anv_minify(image->extent.width, level), 1353 .height = anv_minify(image->extent.height, level), 1354 .depth = anv_minify(image->extent.depth, level), 1355 }; 1356 1357 if (image->type == VK_IMAGE_TYPE_3D) 1358 layer_count = extent.depth; 1359 1360 for (uint32_t a = 0; a < layer_count; a++) { 1361 const uint32_t layer = base_layer + a; 1362 1363 blorp_copy(&batch, &surf, level, layer, 1364 &shadow_surf, level, layer, 1365 0, 0, 0, 0, extent.width, extent.height); 1366 } 1367 } 1368 1369 blorp_batch_finish(&batch); 1370} 1371 1372void 1373anv_image_clear_color(struct anv_cmd_buffer *cmd_buffer, 1374 const struct anv_image *image, 1375 VkImageAspectFlagBits aspect, 1376 enum isl_aux_usage aux_usage, 1377 enum isl_format format, struct isl_swizzle swizzle, 1378 uint32_t level, uint32_t base_layer, uint32_t layer_count, 1379 VkRect2D area, union isl_color_value clear_color) 1380{ 1381 assert(image->aspects == VK_IMAGE_ASPECT_COLOR_BIT); 1382 1383 /* We don't support planar images with multisampling yet */ 1384 assert(image->n_planes == 1); 1385 1386 struct blorp_batch batch; 1387 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0); 1388 1389 struct blorp_surf surf; 1390 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect, 1391 ANV_IMAGE_LAYOUT_EXPLICIT_AUX, 1392 aux_usage, &surf); 1393 anv_cmd_buffer_mark_image_written(cmd_buffer, image, aspect, aux_usage, 1394 level, base_layer, layer_count); 1395 1396 blorp_clear(&batch, &surf, format, anv_swizzle_for_render(swizzle), 1397 level, base_layer, layer_count, 1398 area.offset.x, area.offset.y, 1399 area.offset.x + area.extent.width, 1400 area.offset.y + area.extent.height, 1401 clear_color, NULL); 1402 1403 blorp_batch_finish(&batch); 1404} 1405 1406void 1407anv_image_clear_depth_stencil(struct anv_cmd_buffer *cmd_buffer, 1408 const struct anv_image *image, 1409 VkImageAspectFlags aspects, 1410 enum isl_aux_usage depth_aux_usage, 1411 uint32_t level, 1412 uint32_t base_layer, uint32_t layer_count, 1413 VkRect2D area, 1414 float depth_value, uint8_t stencil_value) 1415{ 1416 assert(image->aspects & (VK_IMAGE_ASPECT_DEPTH_BIT | 1417 VK_IMAGE_ASPECT_STENCIL_BIT)); 1418 1419 struct blorp_batch batch; 1420 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0); 1421 1422 struct blorp_surf depth = {}; 1423 if (aspects & VK_IMAGE_ASPECT_DEPTH_BIT) { 1424 get_blorp_surf_for_anv_image(cmd_buffer->device, 1425 image, VK_IMAGE_ASPECT_DEPTH_BIT, 1426 ANV_IMAGE_LAYOUT_EXPLICIT_AUX, 1427 depth_aux_usage, &depth); 1428 depth.clear_color.f32[0] = ANV_HZ_FC_VAL; 1429 } 1430 1431 struct blorp_surf stencil = {}; 1432 if (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) { 1433 get_blorp_surf_for_anv_image(cmd_buffer->device, 1434 image, VK_IMAGE_ASPECT_STENCIL_BIT, 1435 ANV_IMAGE_LAYOUT_EXPLICIT_AUX, 1436 ISL_AUX_USAGE_NONE, &stencil); 1437 } 1438 1439 blorp_clear_depth_stencil(&batch, &depth, &stencil, 1440 level, base_layer, layer_count, 1441 area.offset.x, area.offset.y, 1442 area.offset.x + area.extent.width, 1443 area.offset.y + area.extent.height, 1444 aspects & VK_IMAGE_ASPECT_DEPTH_BIT, 1445 depth_value, 1446 (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) ? 0xff : 0, 1447 stencil_value); 1448 1449 blorp_batch_finish(&batch); 1450} 1451 1452void 1453anv_image_hiz_op(struct anv_cmd_buffer *cmd_buffer, 1454 const struct anv_image *image, 1455 VkImageAspectFlagBits aspect, uint32_t level, 1456 uint32_t base_layer, uint32_t layer_count, 1457 enum isl_aux_op hiz_op) 1458{ 1459 assert(aspect == VK_IMAGE_ASPECT_DEPTH_BIT); 1460 assert(base_layer + layer_count <= anv_image_aux_layers(image, aspect, level)); 1461 assert(anv_image_aspect_to_plane(image->aspects, 1462 VK_IMAGE_ASPECT_DEPTH_BIT) == 0); 1463 1464 struct blorp_batch batch; 1465 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0); 1466 1467 struct blorp_surf surf; 1468 get_blorp_surf_for_anv_image(cmd_buffer->device, 1469 image, VK_IMAGE_ASPECT_DEPTH_BIT, 1470 ANV_IMAGE_LAYOUT_EXPLICIT_AUX, 1471 ISL_AUX_USAGE_HIZ, &surf); 1472 surf.clear_color.f32[0] = ANV_HZ_FC_VAL; 1473 1474 blorp_hiz_op(&batch, &surf, level, base_layer, layer_count, hiz_op); 1475 1476 blorp_batch_finish(&batch); 1477} 1478 1479void 1480anv_image_hiz_clear(struct anv_cmd_buffer *cmd_buffer, 1481 const struct anv_image *image, 1482 VkImageAspectFlags aspects, 1483 uint32_t level, 1484 uint32_t base_layer, uint32_t layer_count, 1485 VkRect2D area, uint8_t stencil_value) 1486{ 1487 assert(image->aspects & (VK_IMAGE_ASPECT_DEPTH_BIT | 1488 VK_IMAGE_ASPECT_STENCIL_BIT)); 1489 1490 struct blorp_batch batch; 1491 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 0); 1492 1493 struct blorp_surf depth = {}; 1494 if (aspects & VK_IMAGE_ASPECT_DEPTH_BIT) { 1495 assert(base_layer + layer_count <= 1496 anv_image_aux_layers(image, VK_IMAGE_ASPECT_DEPTH_BIT, level)); 1497 get_blorp_surf_for_anv_image(cmd_buffer->device, 1498 image, VK_IMAGE_ASPECT_DEPTH_BIT, 1499 ANV_IMAGE_LAYOUT_EXPLICIT_AUX, 1500 ISL_AUX_USAGE_HIZ, &depth); 1501 depth.clear_color.f32[0] = ANV_HZ_FC_VAL; 1502 } 1503 1504 struct blorp_surf stencil = {}; 1505 if (aspects & VK_IMAGE_ASPECT_STENCIL_BIT) { 1506 get_blorp_surf_for_anv_image(cmd_buffer->device, 1507 image, VK_IMAGE_ASPECT_STENCIL_BIT, 1508 ANV_IMAGE_LAYOUT_EXPLICIT_AUX, 1509 ISL_AUX_USAGE_NONE, &stencil); 1510 } 1511 1512 /* From the Sky Lake PRM Volume 7, "Depth Buffer Clear": 1513 * 1514 * "The following is required when performing a depth buffer clear with 1515 * using the WM_STATE or 3DSTATE_WM: 1516 * 1517 * * If other rendering operations have preceded this clear, a 1518 * PIPE_CONTROL with depth cache flush enabled, Depth Stall bit 1519 * enabled must be issued before the rectangle primitive used for 1520 * the depth buffer clear operation. 1521 * * [...]" 1522 * 1523 * Even though the PRM only says that this is required if using 3DSTATE_WM 1524 * and a 3DPRIMITIVE, the GPU appears to also need this to avoid occasional 1525 * hangs when doing a clear with WM_HZ_OP. 1526 */ 1527 cmd_buffer->state.pending_pipe_bits |= 1528 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT | ANV_PIPE_DEPTH_STALL_BIT; 1529 1530 blorp_hiz_clear_depth_stencil(&batch, &depth, &stencil, 1531 level, base_layer, layer_count, 1532 area.offset.x, area.offset.y, 1533 area.offset.x + area.extent.width, 1534 area.offset.y + area.extent.height, 1535 aspects & VK_IMAGE_ASPECT_DEPTH_BIT, 1536 ANV_HZ_FC_VAL, 1537 aspects & VK_IMAGE_ASPECT_STENCIL_BIT, 1538 stencil_value); 1539 1540 blorp_batch_finish(&batch); 1541 1542 /* From the SKL PRM, Depth Buffer Clear: 1543 * 1544 * "Depth Buffer Clear Workaround 1545 * 1546 * Depth buffer clear pass using any of the methods (WM_STATE, 1547 * 3DSTATE_WM or 3DSTATE_WM_HZ_OP) must be followed by a PIPE_CONTROL 1548 * command with DEPTH_STALL bit and Depth FLUSH bits “set” before 1549 * starting to render. DepthStall and DepthFlush are not needed between 1550 * consecutive depth clear passes nor is it required if the depth-clear 1551 * pass was done with “full_surf_clear” bit set in the 1552 * 3DSTATE_WM_HZ_OP." 1553 * 1554 * Even though the PRM provides a bunch of conditions under which this is 1555 * supposedly unnecessary, we choose to perform the flush unconditionally 1556 * just to be safe. 1557 */ 1558 cmd_buffer->state.pending_pipe_bits |= 1559 ANV_PIPE_DEPTH_CACHE_FLUSH_BIT | ANV_PIPE_DEPTH_STALL_BIT; 1560} 1561 1562void 1563anv_image_mcs_op(struct anv_cmd_buffer *cmd_buffer, 1564 const struct anv_image *image, 1565 enum isl_format format, 1566 VkImageAspectFlagBits aspect, 1567 uint32_t base_layer, uint32_t layer_count, 1568 enum isl_aux_op mcs_op, union isl_color_value *clear_value, 1569 bool predicate) 1570{ 1571 assert(image->aspects == VK_IMAGE_ASPECT_COLOR_BIT); 1572 assert(image->samples > 1); 1573 assert(base_layer + layer_count <= anv_image_aux_layers(image, aspect, 0)); 1574 1575 /* Multisampling with multi-planar formats is not supported */ 1576 assert(image->n_planes == 1); 1577 1578 struct blorp_batch batch; 1579 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 1580 predicate ? BLORP_BATCH_PREDICATE_ENABLE : 0); 1581 1582 struct blorp_surf surf; 1583 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect, 1584 ANV_IMAGE_LAYOUT_EXPLICIT_AUX, 1585 ISL_AUX_USAGE_MCS, &surf); 1586 1587 /* Blorp will store the clear color for us if we provide the clear color 1588 * address and we are doing a fast clear. So we save the clear value into 1589 * the blorp surface. However, in some situations we want to do a fast clear 1590 * without changing the clear value stored in the state buffer. For those 1591 * cases, we set the clear color address pointer to NULL, so blorp will not 1592 * try to store a garbage color. 1593 */ 1594 if (mcs_op == ISL_AUX_OP_FAST_CLEAR) { 1595 if (clear_value) 1596 surf.clear_color = *clear_value; 1597 else 1598 surf.clear_color_addr.buffer = NULL; 1599 } 1600 1601 /* From the Sky Lake PRM Vol. 7, "Render Target Fast Clear": 1602 * 1603 * "After Render target fast clear, pipe-control with color cache 1604 * write-flush must be issued before sending any DRAW commands on 1605 * that render target." 1606 * 1607 * This comment is a bit cryptic and doesn't really tell you what's going 1608 * or what's really needed. It appears that fast clear ops are not 1609 * properly synchronized with other drawing. This means that we cannot 1610 * have a fast clear operation in the pipe at the same time as other 1611 * regular drawing operations. We need to use a PIPE_CONTROL to ensure 1612 * that the contents of the previous draw hit the render target before we 1613 * resolve and then use a second PIPE_CONTROL after the resolve to ensure 1614 * that it is completed before any additional drawing occurs. 1615 */ 1616 cmd_buffer->state.pending_pipe_bits |= 1617 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT; 1618 1619 switch (mcs_op) { 1620 case ISL_AUX_OP_FAST_CLEAR: 1621 blorp_fast_clear(&batch, &surf, format, 1622 0, base_layer, layer_count, 1623 0, 0, image->extent.width, image->extent.height); 1624 break; 1625 case ISL_AUX_OP_PARTIAL_RESOLVE: 1626 blorp_mcs_partial_resolve(&batch, &surf, format, 1627 base_layer, layer_count); 1628 break; 1629 case ISL_AUX_OP_FULL_RESOLVE: 1630 case ISL_AUX_OP_AMBIGUATE: 1631 default: 1632 unreachable("Unsupported MCS operation"); 1633 } 1634 1635 cmd_buffer->state.pending_pipe_bits |= 1636 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT; 1637 1638 blorp_batch_finish(&batch); 1639} 1640 1641void 1642anv_image_ccs_op(struct anv_cmd_buffer *cmd_buffer, 1643 const struct anv_image *image, 1644 enum isl_format format, 1645 VkImageAspectFlagBits aspect, uint32_t level, 1646 uint32_t base_layer, uint32_t layer_count, 1647 enum isl_aux_op ccs_op, union isl_color_value *clear_value, 1648 bool predicate) 1649{ 1650 assert(image->aspects & VK_IMAGE_ASPECT_ANY_COLOR_BIT_ANV); 1651 assert(image->samples == 1); 1652 assert(level < anv_image_aux_levels(image, aspect)); 1653 /* Multi-LOD YcBcR is not allowed */ 1654 assert(image->n_planes == 1 || level == 0); 1655 assert(base_layer + layer_count <= 1656 anv_image_aux_layers(image, aspect, level)); 1657 1658 uint32_t plane = anv_image_aspect_to_plane(image->aspects, aspect); 1659 uint32_t width_div = image->format->planes[plane].denominator_scales[0]; 1660 uint32_t height_div = image->format->planes[plane].denominator_scales[1]; 1661 uint32_t level_width = anv_minify(image->extent.width, level) / width_div; 1662 uint32_t level_height = anv_minify(image->extent.height, level) / height_div; 1663 1664 struct blorp_batch batch; 1665 blorp_batch_init(&cmd_buffer->device->blorp, &batch, cmd_buffer, 1666 predicate ? BLORP_BATCH_PREDICATE_ENABLE : 0); 1667 1668 struct blorp_surf surf; 1669 get_blorp_surf_for_anv_image(cmd_buffer->device, image, aspect, 1670 ANV_IMAGE_LAYOUT_EXPLICIT_AUX, 1671 fast_clear_aux_usage(image, aspect), 1672 &surf); 1673 1674 /* Blorp will store the clear color for us if we provide the clear color 1675 * address and we are doing a fast clear. So we save the clear value into 1676 * the blorp surface. However, in some situations we want to do a fast clear 1677 * without changing the clear value stored in the state buffer. For those 1678 * cases, we set the clear color address pointer to NULL, so blorp will not 1679 * try to store a garbage color. 1680 */ 1681 if (ccs_op == ISL_AUX_OP_FAST_CLEAR) { 1682 if (clear_value) 1683 surf.clear_color = *clear_value; 1684 else 1685 surf.clear_color_addr.buffer = NULL; 1686 } 1687 1688 /* From the Sky Lake PRM Vol. 7, "Render Target Fast Clear": 1689 * 1690 * "After Render target fast clear, pipe-control with color cache 1691 * write-flush must be issued before sending any DRAW commands on 1692 * that render target." 1693 * 1694 * This comment is a bit cryptic and doesn't really tell you what's going 1695 * or what's really needed. It appears that fast clear ops are not 1696 * properly synchronized with other drawing. This means that we cannot 1697 * have a fast clear operation in the pipe at the same time as other 1698 * regular drawing operations. We need to use a PIPE_CONTROL to ensure 1699 * that the contents of the previous draw hit the render target before we 1700 * resolve and then use a second PIPE_CONTROL after the resolve to ensure 1701 * that it is completed before any additional drawing occurs. 1702 */ 1703 cmd_buffer->state.pending_pipe_bits |= 1704 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT; 1705 1706 switch (ccs_op) { 1707 case ISL_AUX_OP_FAST_CLEAR: 1708 blorp_fast_clear(&batch, &surf, format, 1709 level, base_layer, layer_count, 1710 0, 0, level_width, level_height); 1711 break; 1712 case ISL_AUX_OP_FULL_RESOLVE: 1713 case ISL_AUX_OP_PARTIAL_RESOLVE: 1714 blorp_ccs_resolve(&batch, &surf, level, base_layer, layer_count, 1715 format, ccs_op); 1716 break; 1717 case ISL_AUX_OP_AMBIGUATE: 1718 for (uint32_t a = 0; a < layer_count; a++) { 1719 const uint32_t layer = base_layer + a; 1720 blorp_ccs_ambiguate(&batch, &surf, level, layer); 1721 } 1722 break; 1723 default: 1724 unreachable("Unsupported CCS operation"); 1725 } 1726 1727 cmd_buffer->state.pending_pipe_bits |= 1728 ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT | ANV_PIPE_CS_STALL_BIT; 1729 1730 blorp_batch_finish(&batch); 1731} 1732