1/* 2 * Copyright © 2017 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included 12 * in all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 15 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 * DEALINGS IN THE SOFTWARE. 21 */ 22 23#include <stdio.h> 24#include "pipe/p_defines.h" 25#include "pipe/p_state.h" 26#include "pipe/p_context.h" 27#include "pipe/p_screen.h" 28#include "util/format/u_format.h" 29#include "util/u_inlines.h" 30#include "util/ralloc.h" 31#include "intel/blorp/blorp.h" 32#include "iris_context.h" 33#include "iris_resource.h" 34#include "iris_screen.h" 35 36/** 37 * Helper function for handling mirror image blits. 38 * 39 * If coord0 > coord1, swap them and return "true" (mirrored). 40 */ 41static bool 42apply_mirror(float *coord0, float *coord1) 43{ 44 if (*coord0 > *coord1) { 45 float tmp = *coord0; 46 *coord0 = *coord1; 47 *coord1 = tmp; 48 return true; 49 } 50 return false; 51} 52 53/** 54 * Compute the number of pixels to clip for each side of a rect 55 * 56 * \param x0 The rect's left coordinate 57 * \param y0 The rect's bottom coordinate 58 * \param x1 The rect's right coordinate 59 * \param y1 The rect's top coordinate 60 * \param min_x The clipping region's left coordinate 61 * \param min_y The clipping region's bottom coordinate 62 * \param max_x The clipping region's right coordinate 63 * \param max_y The clipping region's top coordinate 64 * \param clipped_x0 The number of pixels to clip from the left side 65 * \param clipped_y0 The number of pixels to clip from the bottom side 66 * \param clipped_x1 The number of pixels to clip from the right side 67 * \param clipped_y1 The number of pixels to clip from the top side 68 * 69 * \return false if we clip everything away, true otherwise 70 */ 71static inline bool 72compute_pixels_clipped(float x0, float y0, float x1, float y1, 73 float min_x, float min_y, float max_x, float max_y, 74 float *clipped_x0, float *clipped_y0, 75 float *clipped_x1, float *clipped_y1) 76{ 77 /* If we are going to clip everything away, stop. */ 78 if (!(min_x <= max_x && 79 min_y <= max_y && 80 x0 <= max_x && 81 y0 <= max_y && 82 min_x <= x1 && 83 min_y <= y1 && 84 x0 <= x1 && 85 y0 <= y1)) { 86 return false; 87 } 88 89 if (x0 < min_x) 90 *clipped_x0 = min_x - x0; 91 else 92 *clipped_x0 = 0; 93 if (max_x < x1) 94 *clipped_x1 = x1 - max_x; 95 else 96 *clipped_x1 = 0; 97 98 if (y0 < min_y) 99 *clipped_y0 = min_y - y0; 100 else 101 *clipped_y0 = 0; 102 if (max_y < y1) 103 *clipped_y1 = y1 - max_y; 104 else 105 *clipped_y1 = 0; 106 107 return true; 108} 109 110/** 111 * Clips a coordinate (left, right, top or bottom) for the src or dst rect 112 * (whichever requires the largest clip) and adjusts the coordinate 113 * for the other rect accordingly. 114 * 115 * \param mirror true if mirroring is required 116 * \param src the source rect coordinate (for example src_x0) 117 * \param dst0 the dst rect coordinate (for example dst_x0) 118 * \param dst1 the opposite dst rect coordinate (for example dst_x1) 119 * \param clipped_dst0 number of pixels to clip from the dst coordinate 120 * \param clipped_dst1 number of pixels to clip from the opposite dst coordinate 121 * \param scale the src vs dst scale involved for that coordinate 122 * \param is_left_or_bottom true if we are clipping the left or bottom sides 123 * of the rect. 124 */ 125static void 126clip_coordinates(bool mirror, 127 float *src, float *dst0, float *dst1, 128 float clipped_dst0, 129 float clipped_dst1, 130 float scale, 131 bool is_left_or_bottom) 132{ 133 /* When clipping we need to add or subtract pixels from the original 134 * coordinates depending on whether we are acting on the left/bottom 135 * or right/top sides of the rect respectively. We assume we have to 136 * add them in the code below, and multiply by -1 when we should 137 * subtract. 138 */ 139 int mult = is_left_or_bottom ? 1 : -1; 140 141 if (!mirror) { 142 *dst0 += clipped_dst0 * mult; 143 *src += clipped_dst0 * scale * mult; 144 } else { 145 *dst1 -= clipped_dst1 * mult; 146 *src += clipped_dst1 * scale * mult; 147 } 148} 149 150/** 151 * Apply a scissor rectangle to blit coordinates. 152 * 153 * Returns true if the blit was entirely scissored away. 154 */ 155static bool 156apply_blit_scissor(const struct pipe_scissor_state *scissor, 157 float *src_x0, float *src_y0, 158 float *src_x1, float *src_y1, 159 float *dst_x0, float *dst_y0, 160 float *dst_x1, float *dst_y1, 161 bool mirror_x, bool mirror_y) 162{ 163 float clip_dst_x0, clip_dst_x1, clip_dst_y0, clip_dst_y1; 164 165 /* Compute number of pixels to scissor away. */ 166 if (!compute_pixels_clipped(*dst_x0, *dst_y0, *dst_x1, *dst_y1, 167 scissor->minx, scissor->miny, 168 scissor->maxx, scissor->maxy, 169 &clip_dst_x0, &clip_dst_y0, 170 &clip_dst_x1, &clip_dst_y1)) 171 return true; 172 173 // XXX: comments assume source clipping, which we don't do 174 175 /* When clipping any of the two rects we need to adjust the coordinates 176 * in the other rect considering the scaling factor involved. To obtain 177 * the best precision we want to make sure that we only clip once per 178 * side to avoid accumulating errors due to the scaling adjustment. 179 * 180 * For example, if src_x0 and dst_x0 need both to be clipped we want to 181 * avoid the situation where we clip src_x0 first, then adjust dst_x0 182 * accordingly but then we realize that the resulting dst_x0 still needs 183 * to be clipped, so we clip dst_x0 and adjust src_x0 again. Because we are 184 * applying scaling factors to adjust the coordinates in each clipping 185 * pass we lose some precision and that can affect the results of the 186 * blorp blit operation slightly. What we want to do here is detect the 187 * rect that we should clip first for each side so that when we adjust 188 * the other rect we ensure the resulting coordinate does not need to be 189 * clipped again. 190 * 191 * The code below implements this by comparing the number of pixels that 192 * we need to clip for each side of both rects considering the scales 193 * involved. For example, clip_src_x0 represents the number of pixels 194 * to be clipped for the src rect's left side, so if clip_src_x0 = 5, 195 * clip_dst_x0 = 4 and scale_x = 2 it means that we are clipping more 196 * from the dst rect so we should clip dst_x0 only and adjust src_x0. 197 * This is because clipping 4 pixels in the dst is equivalent to 198 * clipping 4 * 2 = 8 > 5 in the src. 199 */ 200 201 if (*src_x0 == *src_x1 || *src_y0 == *src_y1 202 || *dst_x0 == *dst_x1 || *dst_y0 == *dst_y1) 203 return true; 204 205 float scale_x = (float) (*src_x1 - *src_x0) / (*dst_x1 - *dst_x0); 206 float scale_y = (float) (*src_y1 - *src_y0) / (*dst_y1 - *dst_y0); 207 208 /* Clip left side */ 209 clip_coordinates(mirror_x, src_x0, dst_x0, dst_x1, 210 clip_dst_x0, clip_dst_x1, scale_x, true); 211 212 /* Clip right side */ 213 clip_coordinates(mirror_x, src_x1, dst_x1, dst_x0, 214 clip_dst_x1, clip_dst_x0, scale_x, false); 215 216 /* Clip bottom side */ 217 clip_coordinates(mirror_y, src_y0, dst_y0, dst_y1, 218 clip_dst_y0, clip_dst_y1, scale_y, true); 219 220 /* Clip top side */ 221 clip_coordinates(mirror_y, src_y1, dst_y1, dst_y0, 222 clip_dst_y1, clip_dst_y0, scale_y, false); 223 224 /* Check for invalid bounds 225 * Can't blit for 0-dimensions 226 */ 227 return *src_x0 == *src_x1 || *src_y0 == *src_y1 228 || *dst_x0 == *dst_x1 || *dst_y0 == *dst_y1; 229} 230 231void 232iris_blorp_surf_for_resource(struct isl_device *isl_dev, 233 struct blorp_surf *surf, 234 struct pipe_resource *p_res, 235 enum isl_aux_usage aux_usage, 236 unsigned level, 237 bool is_render_target) 238{ 239 struct iris_resource *res = (void *) p_res; 240 241 *surf = (struct blorp_surf) { 242 .surf = &res->surf, 243 .addr = (struct blorp_address) { 244 .buffer = res->bo, 245 .offset = res->offset, 246 .reloc_flags = is_render_target ? EXEC_OBJECT_WRITE : 0, 247 .mocs = iris_mocs(res->bo, isl_dev, 248 is_render_target ? ISL_SURF_USAGE_RENDER_TARGET_BIT 249 : ISL_SURF_USAGE_TEXTURE_BIT), 250 }, 251 .aux_usage = aux_usage, 252 }; 253 254 if (aux_usage != ISL_AUX_USAGE_NONE) { 255 surf->aux_surf = &res->aux.surf; 256 surf->aux_addr = (struct blorp_address) { 257 .buffer = res->aux.bo, 258 .offset = res->aux.offset, 259 .reloc_flags = is_render_target ? EXEC_OBJECT_WRITE : 0, 260 .mocs = iris_mocs(res->bo, isl_dev, 0), 261 }; 262 surf->clear_color = res->aux.clear_color; 263 surf->clear_color_addr = (struct blorp_address) { 264 .buffer = res->aux.clear_color_bo, 265 .offset = res->aux.clear_color_offset, 266 .reloc_flags = 0, 267 .mocs = iris_mocs(res->aux.clear_color_bo, isl_dev, 0), 268 }; 269 } 270} 271 272static bool 273is_astc(enum isl_format format) 274{ 275 return format != ISL_FORMAT_UNSUPPORTED && 276 isl_format_get_layout(format)->txc == ISL_TXC_ASTC; 277} 278 279static void 280tex_cache_flush_hack(struct iris_batch *batch, 281 enum isl_format view_format, 282 enum isl_format surf_format) 283{ 284 const struct intel_device_info *devinfo = &batch->screen->devinfo; 285 286 /* The WaSamplerCacheFlushBetweenRedescribedSurfaceReads workaround says: 287 * 288 * "Currently Sampler assumes that a surface would not have two 289 * different format associate with it. It will not properly cache 290 * the different views in the MT cache, causing a data corruption." 291 * 292 * We may need to handle this for texture views in general someday, but 293 * for now we handle it here, as it hurts copies and blits particularly 294 * badly because they ofter reinterpret formats. 295 * 296 * If the BO hasn't been referenced yet this batch, we assume that the 297 * texture cache doesn't contain any relevant data nor need flushing. 298 * 299 * Icelake (Gfx11+) claims to fix this issue, but seems to still have 300 * issues with ASTC formats. 301 */ 302 bool need_flush = devinfo->ver >= 11 ? 303 is_astc(surf_format) != is_astc(view_format) : 304 view_format != surf_format; 305 if (!need_flush) 306 return; 307 308 const char *reason = 309 "workaround: WaSamplerCacheFlushBetweenRedescribedSurfaceReads"; 310 311 iris_emit_pipe_control_flush(batch, reason, PIPE_CONTROL_CS_STALL); 312 iris_emit_pipe_control_flush(batch, reason, 313 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE); 314} 315 316static struct iris_resource * 317iris_resource_for_aspect(struct pipe_resource *p_res, unsigned pipe_mask) 318{ 319 if (pipe_mask == PIPE_MASK_S) { 320 struct iris_resource *junk, *s_res; 321 iris_get_depth_stencil_resources(p_res, &junk, &s_res); 322 return s_res; 323 } else { 324 return (struct iris_resource *)p_res; 325 } 326} 327 328static enum pipe_format 329pipe_format_for_aspect(enum pipe_format format, unsigned pipe_mask) 330{ 331 if (pipe_mask == PIPE_MASK_S) { 332 return util_format_stencil_only(format); 333 } else if (pipe_mask == PIPE_MASK_Z) { 334 return util_format_get_depth_only(format); 335 } else { 336 return format; 337 } 338} 339 340static bool 341clear_color_is_fully_zero(const struct iris_resource *res) 342{ 343 return !res->aux.clear_color_unknown && 344 res->aux.clear_color.u32[0] == 0 && 345 res->aux.clear_color.u32[1] == 0 && 346 res->aux.clear_color.u32[2] == 0 && 347 res->aux.clear_color.u32[3] == 0; 348} 349 350/** 351 * The pipe->blit() driver hook. 352 * 353 * This performs a blit between two surfaces, which copies data but may 354 * also perform format conversion, scaling, flipping, and so on. 355 */ 356static void 357iris_blit(struct pipe_context *ctx, const struct pipe_blit_info *info) 358{ 359 struct iris_context *ice = (void *) ctx; 360 struct iris_screen *screen = (struct iris_screen *)ctx->screen; 361 const struct intel_device_info *devinfo = &screen->devinfo; 362 struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER]; 363 enum blorp_batch_flags blorp_flags = 0; 364 365 /* We don't support color masking. */ 366 assert((info->mask & PIPE_MASK_RGBA) == PIPE_MASK_RGBA || 367 (info->mask & PIPE_MASK_RGBA) == 0); 368 369 if (info->render_condition_enable) { 370 if (ice->state.predicate == IRIS_PREDICATE_STATE_DONT_RENDER) 371 return; 372 373 if (ice->state.predicate == IRIS_PREDICATE_STATE_USE_BIT) 374 blorp_flags |= BLORP_BATCH_PREDICATE_ENABLE; 375 } 376 377 float src_x0 = info->src.box.x; 378 float src_x1 = info->src.box.x + info->src.box.width; 379 float src_y0 = info->src.box.y; 380 float src_y1 = info->src.box.y + info->src.box.height; 381 float dst_x0 = info->dst.box.x; 382 float dst_x1 = info->dst.box.x + info->dst.box.width; 383 float dst_y0 = info->dst.box.y; 384 float dst_y1 = info->dst.box.y + info->dst.box.height; 385 bool mirror_x = apply_mirror(&src_x0, &src_x1); 386 bool mirror_y = apply_mirror(&src_y0, &src_y1); 387 enum blorp_filter filter; 388 389 if (info->scissor_enable) { 390 bool noop = apply_blit_scissor(&info->scissor, 391 &src_x0, &src_y0, &src_x1, &src_y1, 392 &dst_x0, &dst_y0, &dst_x1, &dst_y1, 393 mirror_x, mirror_y); 394 if (noop) 395 return; 396 } 397 398 if (abs(info->dst.box.width) == abs(info->src.box.width) && 399 abs(info->dst.box.height) == abs(info->src.box.height)) { 400 if (info->src.resource->nr_samples > 1 && 401 info->dst.resource->nr_samples <= 1) { 402 /* The OpenGL ES 3.2 specification, section 16.2.1, says: 403 * 404 * "If the read framebuffer is multisampled (its effective 405 * value of SAMPLE_BUFFERS is one) and the draw framebuffer 406 * is not (its value of SAMPLE_BUFFERS is zero), the samples 407 * corresponding to each pixel location in the source are 408 * converted to a single sample before being written to the 409 * destination. The filter parameter is ignored. If the 410 * source formats are integer types or stencil values, a 411 * single sample’s value is selected for each pixel. If the 412 * source formats are floating-point or normalized types, 413 * the sample values for each pixel are resolved in an 414 * implementation-dependent manner. If the source formats 415 * are depth values, sample values are resolved in an 416 * implementation-dependent manner where the result will be 417 * between the minimum and maximum depth values in the pixel." 418 * 419 * When selecting a single sample, we always choose sample 0. 420 */ 421 if (util_format_is_depth_or_stencil(info->src.format) || 422 util_format_is_pure_integer(info->src.format)) { 423 filter = BLORP_FILTER_SAMPLE_0; 424 } else { 425 filter = BLORP_FILTER_AVERAGE; 426 } 427 } else { 428 /* The OpenGL 4.6 specification, section 18.3.1, says: 429 * 430 * "If the source and destination dimensions are identical, 431 * no filtering is applied." 432 * 433 * Using BLORP_FILTER_NONE will also handle the upsample case by 434 * replicating the one value in the source to all values in the 435 * destination. 436 */ 437 filter = BLORP_FILTER_NONE; 438 } 439 } else if (info->filter == PIPE_TEX_FILTER_LINEAR) { 440 filter = BLORP_FILTER_BILINEAR; 441 } else { 442 filter = BLORP_FILTER_NEAREST; 443 } 444 445 struct blorp_batch blorp_batch; 446 blorp_batch_init(&ice->blorp, &blorp_batch, batch, blorp_flags); 447 448 float src_z_step = (float)info->src.box.depth / (float)info->dst.box.depth; 449 450 /* There is no interpolation to the pixel center during rendering, so 451 * add the 0.5 offset ourselves here. 452 */ 453 float depth_center_offset = 0; 454 if (info->src.resource->target == PIPE_TEXTURE_3D) 455 depth_center_offset = 0.5 / info->dst.box.depth * info->src.box.depth; 456 457 /* Perform a blit for each aspect requested by the caller. PIPE_MASK_R is 458 * used to represent the color aspect. */ 459 unsigned aspect_mask = info->mask & (PIPE_MASK_R | PIPE_MASK_ZS); 460 while (aspect_mask) { 461 unsigned aspect = 1 << u_bit_scan(&aspect_mask); 462 463 struct iris_resource *src_res = 464 iris_resource_for_aspect(info->src.resource, aspect); 465 struct iris_resource *dst_res = 466 iris_resource_for_aspect(info->dst.resource, aspect); 467 468 enum pipe_format src_pfmt = 469 pipe_format_for_aspect(info->src.format, aspect); 470 enum pipe_format dst_pfmt = 471 pipe_format_for_aspect(info->dst.format, aspect); 472 473 struct iris_format_info src_fmt = 474 iris_format_for_usage(devinfo, src_pfmt, ISL_SURF_USAGE_TEXTURE_BIT); 475 enum isl_aux_usage src_aux_usage = 476 iris_resource_texture_aux_usage(ice, src_res, src_fmt.fmt); 477 478 iris_resource_prepare_texture(ice, src_res, src_fmt.fmt, 479 info->src.level, 1, info->src.box.z, 480 info->src.box.depth); 481 iris_emit_buffer_barrier_for(batch, src_res->bo, 482 IRIS_DOMAIN_OTHER_READ); 483 484 struct iris_format_info dst_fmt = 485 iris_format_for_usage(devinfo, dst_pfmt, 486 ISL_SURF_USAGE_RENDER_TARGET_BIT); 487 enum isl_aux_usage dst_aux_usage = 488 iris_resource_render_aux_usage(ice, dst_res, info->dst.level, 489 dst_fmt.fmt, false); 490 491 struct blorp_surf src_surf, dst_surf; 492 iris_blorp_surf_for_resource(&screen->isl_dev, &src_surf, 493 &src_res->base.b, src_aux_usage, 494 info->src.level, false); 495 iris_blorp_surf_for_resource(&screen->isl_dev, &dst_surf, 496 &dst_res->base.b, dst_aux_usage, 497 info->dst.level, true); 498 499 iris_resource_prepare_render(ice, dst_res, info->dst.level, 500 info->dst.box.z, info->dst.box.depth, 501 dst_aux_usage); 502 iris_emit_buffer_barrier_for(batch, dst_res->bo, 503 IRIS_DOMAIN_RENDER_WRITE); 504 505 if (iris_batch_references(batch, src_res->bo)) 506 tex_cache_flush_hack(batch, src_fmt.fmt, src_res->surf.format); 507 508 if (dst_res->base.b.target == PIPE_BUFFER) { 509 util_range_add(&dst_res->base.b, &dst_res->valid_buffer_range, 510 dst_x0, dst_x1); 511 } 512 513 for (int slice = 0; slice < info->dst.box.depth; slice++) { 514 unsigned dst_z = info->dst.box.z + slice; 515 float src_z = info->src.box.z + slice * src_z_step + 516 depth_center_offset; 517 518 iris_batch_maybe_flush(batch, 1500); 519 iris_batch_sync_region_start(batch); 520 521 blorp_blit(&blorp_batch, 522 &src_surf, info->src.level, src_z, 523 src_fmt.fmt, src_fmt.swizzle, 524 &dst_surf, info->dst.level, dst_z, 525 dst_fmt.fmt, dst_fmt.swizzle, 526 src_x0, src_y0, src_x1, src_y1, 527 dst_x0, dst_y0, dst_x1, dst_y1, 528 filter, mirror_x, mirror_y); 529 530 iris_batch_sync_region_end(batch); 531 } 532 533 tex_cache_flush_hack(batch, src_fmt.fmt, src_res->surf.format); 534 535 iris_resource_finish_render(ice, dst_res, info->dst.level, 536 info->dst.box.z, info->dst.box.depth, 537 dst_aux_usage); 538 } 539 540 blorp_batch_finish(&blorp_batch); 541 542 iris_flush_and_dirty_for_history(ice, batch, (struct iris_resource *) 543 info->dst.resource, 544 PIPE_CONTROL_RENDER_TARGET_FLUSH, 545 "cache history: post-blit"); 546} 547 548static void 549get_copy_region_aux_settings(struct iris_context *ice, 550 struct iris_resource *res, 551 unsigned level, 552 enum isl_aux_usage *out_aux_usage, 553 bool *out_clear_supported, 554 bool is_render_target) 555{ 556 struct iris_screen *screen = (void *) ice->ctx.screen; 557 struct intel_device_info *devinfo = &screen->devinfo; 558 559 switch (res->aux.usage) { 560 case ISL_AUX_USAGE_HIZ: 561 case ISL_AUX_USAGE_HIZ_CCS: 562 case ISL_AUX_USAGE_HIZ_CCS_WT: 563 case ISL_AUX_USAGE_STC_CCS: 564 if (is_render_target) { 565 *out_aux_usage = iris_resource_render_aux_usage(ice, res, level, 566 res->surf.format, 567 false); 568 } else { 569 *out_aux_usage = iris_resource_texture_aux_usage(ice, res, 570 res->surf.format); 571 } 572 *out_clear_supported = isl_aux_usage_has_fast_clears(*out_aux_usage); 573 break; 574 case ISL_AUX_USAGE_MCS: 575 case ISL_AUX_USAGE_MCS_CCS: 576 if (!is_render_target && 577 !iris_can_sample_mcs_with_clear(devinfo, res)) { 578 *out_aux_usage = res->aux.usage; 579 *out_clear_supported = false; 580 break; 581 } 582 FALLTHROUGH; 583 case ISL_AUX_USAGE_CCS_E: 584 case ISL_AUX_USAGE_GFX12_CCS_E: 585 *out_aux_usage = res->aux.usage; 586 587 /* blorp_copy may reinterpret the surface format and has limited support 588 * for adjusting the clear color, so clear support may only be enabled 589 * in some cases: 590 * 591 * - On gfx11+, the clear color is indirect and comes in two forms: a 592 * 32bpc representation used for rendering and a pixel representation 593 * used for sampling. blorp_copy doesn't change indirect clear colors, 594 * so clears are only supported in the sampling case. 595 * 596 * - A clear color of zeroes holds the same meaning regardless of the 597 * format. Although it could avoid more resolves, we don't use 598 * isl_color_value_is_zero because the surface format used by 599 * blorp_copy isn't guaranteed to access the same components as the 600 * original format (e.g. A8_UNORM/R8_UINT). 601 */ 602 *out_clear_supported = (devinfo->ver >= 11 && !is_render_target) || 603 clear_color_is_fully_zero(res); 604 break; 605 default: 606 *out_aux_usage = ISL_AUX_USAGE_NONE; 607 *out_clear_supported = false; 608 break; 609 } 610} 611 612/** 613 * Perform a GPU-based raw memory copy between compatible view classes. 614 * 615 * Does not perform any flushing - the new data may still be left in the 616 * render cache, and old data may remain in other caches. 617 * 618 * Wraps blorp_copy() and blorp_buffer_copy(). 619 */ 620void 621iris_copy_region(struct blorp_context *blorp, 622 struct iris_batch *batch, 623 struct pipe_resource *dst, 624 unsigned dst_level, 625 unsigned dstx, unsigned dsty, unsigned dstz, 626 struct pipe_resource *src, 627 unsigned src_level, 628 const struct pipe_box *src_box) 629{ 630 struct blorp_batch blorp_batch; 631 struct iris_context *ice = blorp->driver_ctx; 632 struct iris_screen *screen = (void *) ice->ctx.screen; 633 struct iris_resource *src_res = (void *) src; 634 struct iris_resource *dst_res = (void *) dst; 635 636 enum isl_aux_usage src_aux_usage, dst_aux_usage; 637 bool src_clear_supported, dst_clear_supported; 638 get_copy_region_aux_settings(ice, src_res, src_level, &src_aux_usage, 639 &src_clear_supported, false); 640 get_copy_region_aux_settings(ice, dst_res, dst_level, &dst_aux_usage, 641 &dst_clear_supported, true); 642 643 if (iris_batch_references(batch, src_res->bo)) 644 tex_cache_flush_hack(batch, ISL_FORMAT_UNSUPPORTED, src_res->surf.format); 645 646 if (dst->target == PIPE_BUFFER) 647 util_range_add(&dst_res->base.b, &dst_res->valid_buffer_range, dstx, dstx + src_box->width); 648 649 if (dst->target == PIPE_BUFFER && src->target == PIPE_BUFFER) { 650 struct blorp_address src_addr = { 651 .buffer = iris_resource_bo(src), .offset = src_box->x, 652 .mocs = iris_mocs(src_res->bo, &screen->isl_dev, 653 ISL_SURF_USAGE_RENDER_TARGET_BIT), 654 }; 655 struct blorp_address dst_addr = { 656 .buffer = iris_resource_bo(dst), .offset = dstx, 657 .reloc_flags = EXEC_OBJECT_WRITE, 658 .mocs = iris_mocs(dst_res->bo, &screen->isl_dev, 659 ISL_SURF_USAGE_TEXTURE_BIT), 660 }; 661 662 iris_emit_buffer_barrier_for(batch, iris_resource_bo(src), 663 IRIS_DOMAIN_OTHER_READ); 664 iris_emit_buffer_barrier_for(batch, iris_resource_bo(dst), 665 IRIS_DOMAIN_RENDER_WRITE); 666 667 iris_batch_maybe_flush(batch, 1500); 668 669 iris_batch_sync_region_start(batch); 670 blorp_batch_init(&ice->blorp, &blorp_batch, batch, 0); 671 blorp_buffer_copy(&blorp_batch, src_addr, dst_addr, src_box->width); 672 blorp_batch_finish(&blorp_batch); 673 iris_batch_sync_region_end(batch); 674 } else { 675 // XXX: what about one surface being a buffer and not the other? 676 677 struct blorp_surf src_surf, dst_surf; 678 iris_blorp_surf_for_resource(&screen->isl_dev, &src_surf, 679 src, src_aux_usage, src_level, false); 680 iris_blorp_surf_for_resource(&screen->isl_dev, &dst_surf, 681 dst, dst_aux_usage, dst_level, true); 682 683 iris_resource_prepare_access(ice, src_res, src_level, 1, 684 src_box->z, src_box->depth, 685 src_aux_usage, src_clear_supported); 686 iris_resource_prepare_access(ice, dst_res, dst_level, 1, 687 dstz, src_box->depth, 688 dst_aux_usage, dst_clear_supported); 689 690 iris_emit_buffer_barrier_for(batch, iris_resource_bo(src), 691 IRIS_DOMAIN_OTHER_READ); 692 iris_emit_buffer_barrier_for(batch, iris_resource_bo(dst), 693 IRIS_DOMAIN_RENDER_WRITE); 694 695 blorp_batch_init(&ice->blorp, &blorp_batch, batch, 0); 696 697 for (int slice = 0; slice < src_box->depth; slice++) { 698 iris_batch_maybe_flush(batch, 1500); 699 700 iris_batch_sync_region_start(batch); 701 blorp_copy(&blorp_batch, &src_surf, src_level, src_box->z + slice, 702 &dst_surf, dst_level, dstz + slice, 703 src_box->x, src_box->y, dstx, dsty, 704 src_box->width, src_box->height); 705 iris_batch_sync_region_end(batch); 706 } 707 blorp_batch_finish(&blorp_batch); 708 709 iris_resource_finish_write(ice, dst_res, dst_level, dstz, 710 src_box->depth, dst_aux_usage); 711 } 712 713 tex_cache_flush_hack(batch, ISL_FORMAT_UNSUPPORTED, src_res->surf.format); 714} 715 716/** 717 * The pipe->resource_copy_region() driver hook. 718 * 719 * This implements ARB_copy_image semantics - a raw memory copy between 720 * compatible view classes. 721 */ 722static void 723iris_resource_copy_region(struct pipe_context *ctx, 724 struct pipe_resource *p_dst, 725 unsigned dst_level, 726 unsigned dstx, unsigned dsty, unsigned dstz, 727 struct pipe_resource *p_src, 728 unsigned src_level, 729 const struct pipe_box *src_box) 730{ 731 struct iris_context *ice = (void *) ctx; 732 struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER]; 733 734 iris_copy_region(&ice->blorp, batch, p_dst, dst_level, dstx, dsty, dstz, 735 p_src, src_level, src_box); 736 737 if (util_format_is_depth_and_stencil(p_dst->format) && 738 util_format_has_stencil(util_format_description(p_src->format))) { 739 struct iris_resource *junk, *s_src_res, *s_dst_res; 740 iris_get_depth_stencil_resources(p_src, &junk, &s_src_res); 741 iris_get_depth_stencil_resources(p_dst, &junk, &s_dst_res); 742 743 iris_copy_region(&ice->blorp, batch, &s_dst_res->base.b, dst_level, dstx, 744 dsty, dstz, &s_src_res->base.b, src_level, src_box); 745 } 746 747 iris_flush_and_dirty_for_history(ice, batch, (struct iris_resource *)p_dst, 748 PIPE_CONTROL_RENDER_TARGET_FLUSH, 749 "cache history: post copy_region"); 750} 751 752void 753iris_init_blit_functions(struct pipe_context *ctx) 754{ 755 ctx->blit = iris_blit; 756 ctx->resource_copy_region = iris_resource_copy_region; 757} 758