1/* 2 * Copyright © 2017 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included 12 * in all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 15 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 * DEALINGS IN THE SOFTWARE. 21 */ 22 23/** 24 * @file iris_resource.c 25 * 26 * Resources are images, buffers, and other objects used by the GPU. 27 * 28 * XXX: explain resources 29 */ 30 31#include <stdio.h> 32#include <errno.h> 33#include "pipe/p_defines.h" 34#include "pipe/p_state.h" 35#include "pipe/p_context.h" 36#include "pipe/p_screen.h" 37#include "util/os_memory.h" 38#include "util/u_cpu_detect.h" 39#include "util/u_inlines.h" 40#include "util/u_format.h" 41#include "util/u_threaded_context.h" 42#include "util/u_transfer.h" 43#include "util/u_transfer_helper.h" 44#include "util/u_upload_mgr.h" 45#include "util/ralloc.h" 46#include "iris_batch.h" 47#include "iris_context.h" 48#include "iris_resource.h" 49#include "iris_screen.h" 50#include "intel/dev/gen_debug.h" 51#include "isl/isl.h" 52#include "drm-uapi/drm_fourcc.h" 53#include "drm-uapi/i915_drm.h" 54 55enum modifier_priority { 56 MODIFIER_PRIORITY_INVALID = 0, 57 MODIFIER_PRIORITY_LINEAR, 58 MODIFIER_PRIORITY_X, 59 MODIFIER_PRIORITY_Y, 60 MODIFIER_PRIORITY_Y_CCS, 61}; 62 63static const uint64_t priority_to_modifier[] = { 64 [MODIFIER_PRIORITY_INVALID] = DRM_FORMAT_MOD_INVALID, 65 [MODIFIER_PRIORITY_LINEAR] = DRM_FORMAT_MOD_LINEAR, 66 [MODIFIER_PRIORITY_X] = I915_FORMAT_MOD_X_TILED, 67 [MODIFIER_PRIORITY_Y] = I915_FORMAT_MOD_Y_TILED, 68 [MODIFIER_PRIORITY_Y_CCS] = I915_FORMAT_MOD_Y_TILED_CCS, 69}; 70 71static bool 72modifier_is_supported(const struct gen_device_info *devinfo, 73 uint64_t modifier) 74{ 75 /* XXX: do something real */ 76 switch (modifier) { 77 case I915_FORMAT_MOD_Y_TILED: 78 case I915_FORMAT_MOD_X_TILED: 79 case DRM_FORMAT_MOD_LINEAR: 80 return true; 81 case I915_FORMAT_MOD_Y_TILED_CCS: 82 case DRM_FORMAT_MOD_INVALID: 83 default: 84 return false; 85 } 86} 87 88static uint64_t 89select_best_modifier(struct gen_device_info *devinfo, 90 const uint64_t *modifiers, 91 int count) 92{ 93 enum modifier_priority prio = MODIFIER_PRIORITY_INVALID; 94 95 for (int i = 0; i < count; i++) { 96 if (!modifier_is_supported(devinfo, modifiers[i])) 97 continue; 98 99 switch (modifiers[i]) { 100 case I915_FORMAT_MOD_Y_TILED_CCS: 101 prio = MAX2(prio, MODIFIER_PRIORITY_Y_CCS); 102 break; 103 case I915_FORMAT_MOD_Y_TILED: 104 prio = MAX2(prio, MODIFIER_PRIORITY_Y); 105 break; 106 case I915_FORMAT_MOD_X_TILED: 107 prio = MAX2(prio, MODIFIER_PRIORITY_X); 108 break; 109 case DRM_FORMAT_MOD_LINEAR: 110 prio = MAX2(prio, MODIFIER_PRIORITY_LINEAR); 111 break; 112 case DRM_FORMAT_MOD_INVALID: 113 default: 114 break; 115 } 116 } 117 118 return priority_to_modifier[prio]; 119} 120 121static enum isl_surf_dim 122target_to_isl_surf_dim(enum pipe_texture_target target) 123{ 124 switch (target) { 125 case PIPE_BUFFER: 126 case PIPE_TEXTURE_1D: 127 case PIPE_TEXTURE_1D_ARRAY: 128 return ISL_SURF_DIM_1D; 129 case PIPE_TEXTURE_2D: 130 case PIPE_TEXTURE_CUBE: 131 case PIPE_TEXTURE_RECT: 132 case PIPE_TEXTURE_2D_ARRAY: 133 case PIPE_TEXTURE_CUBE_ARRAY: 134 return ISL_SURF_DIM_2D; 135 case PIPE_TEXTURE_3D: 136 return ISL_SURF_DIM_3D; 137 case PIPE_MAX_TEXTURE_TYPES: 138 break; 139 } 140 unreachable("invalid texture type"); 141} 142 143static void 144iris_query_dmabuf_modifiers(struct pipe_screen *pscreen, 145 enum pipe_format pfmt, 146 int max, 147 uint64_t *modifiers, 148 unsigned int *external_only, 149 int *count) 150{ 151 struct iris_screen *screen = (void *) pscreen; 152 const struct gen_device_info *devinfo = &screen->devinfo; 153 154 uint64_t all_modifiers[] = { 155 DRM_FORMAT_MOD_LINEAR, 156 I915_FORMAT_MOD_X_TILED, 157 I915_FORMAT_MOD_Y_TILED, 158 // XXX: (broken) I915_FORMAT_MOD_Y_TILED_CCS, 159 }; 160 161 int supported_mods = 0; 162 163 for (int i = 0; i < ARRAY_SIZE(all_modifiers); i++) { 164 if (!modifier_is_supported(devinfo, all_modifiers[i])) 165 continue; 166 167 if (supported_mods < max) { 168 if (modifiers) 169 modifiers[supported_mods] = all_modifiers[i]; 170 171 if (external_only) 172 external_only[supported_mods] = util_format_is_yuv(pfmt); 173 } 174 175 supported_mods++; 176 } 177 178 *count = supported_mods; 179} 180 181static isl_surf_usage_flags_t 182pipe_bind_to_isl_usage(unsigned bindings) 183{ 184 isl_surf_usage_flags_t usage = 0; 185 186 if (bindings & PIPE_BIND_RENDER_TARGET) 187 usage |= ISL_SURF_USAGE_RENDER_TARGET_BIT; 188 189 if (bindings & PIPE_BIND_SAMPLER_VIEW) 190 usage |= ISL_SURF_USAGE_TEXTURE_BIT; 191 192 if (bindings & (PIPE_BIND_SHADER_IMAGE | PIPE_BIND_SHADER_BUFFER)) 193 usage |= ISL_SURF_USAGE_STORAGE_BIT; 194 195 if (bindings & PIPE_BIND_DISPLAY_TARGET) 196 usage |= ISL_SURF_USAGE_DISPLAY_BIT; 197 198 return usage; 199} 200 201struct pipe_resource * 202iris_resource_get_separate_stencil(struct pipe_resource *p_res) 203{ 204 /* For packed depth-stencil, we treat depth as the primary resource 205 * and store S8 as the "second plane" resource. 206 */ 207 return p_res->next; 208} 209 210static void 211iris_resource_set_separate_stencil(struct pipe_resource *p_res, 212 struct pipe_resource *stencil) 213{ 214 assert(util_format_has_depth(util_format_description(p_res->format))); 215 pipe_resource_reference(&p_res->next, stencil); 216} 217 218void 219iris_get_depth_stencil_resources(struct pipe_resource *res, 220 struct iris_resource **out_z, 221 struct iris_resource **out_s) 222{ 223 if (!res) { 224 *out_z = NULL; 225 *out_s = NULL; 226 return; 227 } 228 229 if (res->format != PIPE_FORMAT_S8_UINT) { 230 *out_z = (void *) res; 231 *out_s = (void *) iris_resource_get_separate_stencil(res); 232 } else { 233 *out_z = NULL; 234 *out_s = (void *) res; 235 } 236} 237 238void 239iris_resource_disable_aux(struct iris_resource *res) 240{ 241 iris_bo_unreference(res->aux.bo); 242 iris_bo_unreference(res->aux.clear_color_bo); 243 free(res->aux.state); 244 245 res->aux.usage = ISL_AUX_USAGE_NONE; 246 res->aux.possible_usages = 1 << ISL_AUX_USAGE_NONE; 247 res->aux.sampler_usages = 1 << ISL_AUX_USAGE_NONE; 248 res->aux.surf.size_B = 0; 249 res->aux.bo = NULL; 250 res->aux.clear_color_bo = NULL; 251 res->aux.state = NULL; 252} 253 254static void 255iris_resource_destroy(struct pipe_screen *screen, 256 struct pipe_resource *resource) 257{ 258 struct iris_resource *res = (struct iris_resource *)resource; 259 260 if (resource->target == PIPE_BUFFER) 261 util_range_destroy(&res->valid_buffer_range); 262 263 iris_resource_disable_aux(res); 264 265 iris_bo_unreference(res->bo); 266 free(res); 267} 268 269static struct iris_resource * 270iris_alloc_resource(struct pipe_screen *pscreen, 271 const struct pipe_resource *templ) 272{ 273 struct iris_resource *res = calloc(1, sizeof(struct iris_resource)); 274 if (!res) 275 return NULL; 276 277 res->base = *templ; 278 res->base.screen = pscreen; 279 pipe_reference_init(&res->base.reference, 1); 280 281 res->aux.possible_usages = 1 << ISL_AUX_USAGE_NONE; 282 res->aux.sampler_usages = 1 << ISL_AUX_USAGE_NONE; 283 284 if (templ->target == PIPE_BUFFER) 285 util_range_init(&res->valid_buffer_range); 286 287 return res; 288} 289 290unsigned 291iris_get_num_logical_layers(const struct iris_resource *res, unsigned level) 292{ 293 if (res->surf.dim == ISL_SURF_DIM_3D) 294 return minify(res->surf.logical_level0_px.depth, level); 295 else 296 return res->surf.logical_level0_px.array_len; 297} 298 299static enum isl_aux_state ** 300create_aux_state_map(struct iris_resource *res, enum isl_aux_state initial) 301{ 302 uint32_t total_slices = 0; 303 for (uint32_t level = 0; level < res->surf.levels; level++) 304 total_slices += iris_get_num_logical_layers(res, level); 305 306 const size_t per_level_array_size = 307 res->surf.levels * sizeof(enum isl_aux_state *); 308 309 /* We're going to allocate a single chunk of data for both the per-level 310 * reference array and the arrays of aux_state. This makes cleanup 311 * significantly easier. 312 */ 313 const size_t total_size = 314 per_level_array_size + total_slices * sizeof(enum isl_aux_state); 315 316 void *data = malloc(total_size); 317 if (!data) 318 return NULL; 319 320 enum isl_aux_state **per_level_arr = data; 321 enum isl_aux_state *s = data + per_level_array_size; 322 for (uint32_t level = 0; level < res->surf.levels; level++) { 323 per_level_arr[level] = s; 324 const unsigned level_layers = iris_get_num_logical_layers(res, level); 325 for (uint32_t a = 0; a < level_layers; a++) 326 *(s++) = initial; 327 } 328 assert((void *)s == data + total_size); 329 330 return per_level_arr; 331} 332 333/** 334 * Allocate the initial aux surface for a resource based on aux.usage 335 */ 336static bool 337iris_resource_alloc_aux(struct iris_screen *screen, struct iris_resource *res) 338{ 339 struct isl_device *isl_dev = &screen->isl_dev; 340 enum isl_aux_state initial_state; 341 UNUSED bool ok = false; 342 uint8_t memset_value = 0; 343 uint32_t alloc_flags = 0; 344 const struct gen_device_info *devinfo = &screen->devinfo; 345 const unsigned clear_color_state_size = devinfo->gen >= 10 ? 346 screen->isl_dev.ss.clear_color_state_size : 347 (devinfo->gen >= 9 ? screen->isl_dev.ss.clear_value_size : 0); 348 349 assert(!res->aux.bo); 350 351 switch (res->aux.usage) { 352 case ISL_AUX_USAGE_NONE: 353 res->aux.surf.size_B = 0; 354 break; 355 case ISL_AUX_USAGE_HIZ: 356 initial_state = ISL_AUX_STATE_AUX_INVALID; 357 memset_value = 0; 358 ok = isl_surf_get_hiz_surf(isl_dev, &res->surf, &res->aux.surf); 359 break; 360 case ISL_AUX_USAGE_MCS: 361 /* The Ivybridge PRM, Vol 2 Part 1 p326 says: 362 * 363 * "When MCS buffer is enabled and bound to MSRT, it is required 364 * that it is cleared prior to any rendering." 365 * 366 * Since we only use the MCS buffer for rendering, we just clear it 367 * immediately on allocation. The clear value for MCS buffers is all 368 * 1's, so we simply memset it to 0xff. 369 */ 370 initial_state = ISL_AUX_STATE_CLEAR; 371 memset_value = 0xFF; 372 ok = isl_surf_get_mcs_surf(isl_dev, &res->surf, &res->aux.surf); 373 break; 374 case ISL_AUX_USAGE_CCS_D: 375 case ISL_AUX_USAGE_CCS_E: 376 /* When CCS_E is used, we need to ensure that the CCS starts off in 377 * a valid state. From the Sky Lake PRM, "MCS Buffer for Render 378 * Target(s)": 379 * 380 * "If Software wants to enable Color Compression without Fast 381 * clear, Software needs to initialize MCS with zeros." 382 * 383 * A CCS value of 0 indicates that the corresponding block is in the 384 * pass-through state which is what we want. 385 * 386 * For CCS_D, do the same thing. On Gen9+, this avoids having any 387 * undefined bits in the aux buffer. 388 */ 389 initial_state = ISL_AUX_STATE_PASS_THROUGH; 390 alloc_flags |= BO_ALLOC_ZEROED; 391 ok = isl_surf_get_ccs_surf(isl_dev, &res->surf, &res->aux.surf, 0); 392 break; 393 } 394 395 /* No work is needed for a zero-sized auxiliary buffer. */ 396 if (res->aux.surf.size_B == 0) 397 return true; 398 399 /* Assert that ISL gave us a valid aux surf */ 400 assert(ok); 401 402 /* Create the aux_state for the auxiliary buffer. */ 403 res->aux.state = create_aux_state_map(res, initial_state); 404 if (!res->aux.state) 405 return false; 406 407 uint64_t size = res->aux.surf.size_B; 408 409 /* Allocate space in the buffer for storing the clear color. On modern 410 * platforms (gen > 9), we can read it directly from such buffer. 411 * 412 * On gen <= 9, we are going to store the clear color on the buffer 413 * anyways, and copy it back to the surface state during state emission. 414 */ 415 res->aux.clear_color_offset = size; 416 size += clear_color_state_size; 417 418 /* Allocate the auxiliary buffer. ISL has stricter set of alignment rules 419 * the drm allocator. Therefore, one can pass the ISL dimensions in terms 420 * of bytes instead of trying to recalculate based on different format 421 * block sizes. 422 */ 423 res->aux.bo = iris_bo_alloc_tiled(screen->bufmgr, "aux buffer", size, 424 IRIS_MEMZONE_OTHER, I915_TILING_Y, 425 res->aux.surf.row_pitch_B, alloc_flags); 426 if (!res->aux.bo) { 427 return false; 428 } 429 430 if (!(alloc_flags & BO_ALLOC_ZEROED)) { 431 void *map = iris_bo_map(NULL, res->aux.bo, MAP_WRITE | MAP_RAW); 432 433 if (!map) { 434 iris_resource_disable_aux(res); 435 return false; 436 } 437 438 if (memset_value != 0) 439 memset(map, memset_value, res->aux.surf.size_B); 440 441 /* Zero the indirect clear color to match ::fast_clear_color. */ 442 memset((char *)map + res->aux.clear_color_offset, 0, 443 clear_color_state_size); 444 445 iris_bo_unmap(res->aux.bo); 446 } 447 448 if (clear_color_state_size > 0) { 449 res->aux.clear_color_bo = res->aux.bo; 450 iris_bo_reference(res->aux.clear_color_bo); 451 } 452 453 if (res->aux.usage == ISL_AUX_USAGE_HIZ) { 454 for (unsigned level = 0; level < res->surf.levels; ++level) { 455 uint32_t width = u_minify(res->surf.phys_level0_sa.width, level); 456 uint32_t height = u_minify(res->surf.phys_level0_sa.height, level); 457 458 /* Disable HiZ for LOD > 0 unless the width/height are 8x4 aligned. 459 * For LOD == 0, we can grow the dimensions to make it work. 460 */ 461 if (level == 0 || ((width & 7) == 0 && (height & 3) == 0)) 462 res->aux.has_hiz |= 1 << level; 463 } 464 } 465 466 return true; 467} 468 469static bool 470supports_mcs(const struct isl_surf *surf) 471{ 472 /* MCS compression only applies to multisampled resources. */ 473 if (surf->samples <= 1) 474 return false; 475 476 /* See isl_surf_get_mcs_surf for details. */ 477 if (surf->samples == 16 && surf->logical_level0_px.width > 8192) 478 return false; 479 480 /* Depth and stencil buffers use the IMS (interleaved) layout. */ 481 if (isl_surf_usage_is_depth_or_stencil(surf->usage)) 482 return false; 483 484 return true; 485} 486 487static bool 488supports_ccs(const struct gen_device_info *devinfo, 489 const struct isl_surf *surf) 490{ 491 /* Gen9+ only supports CCS for Y-tiled buffers. */ 492 if (surf->tiling != ISL_TILING_Y0) 493 return false; 494 495 /* CCS only supports singlesampled resources. */ 496 if (surf->samples > 1) 497 return false; 498 499 /* The PRM doesn't say this explicitly, but fast-clears don't appear to 500 * work for 3D textures until Gen9 where the layout of 3D textures changes 501 * to match 2D array textures. 502 */ 503 if (devinfo->gen < 9 && surf->dim != ISL_SURF_DIM_2D) 504 return false; 505 506 /* Note: still need to check the format! */ 507 508 return true; 509} 510 511static struct pipe_resource * 512iris_resource_create_for_buffer(struct pipe_screen *pscreen, 513 const struct pipe_resource *templ) 514{ 515 struct iris_screen *screen = (struct iris_screen *)pscreen; 516 struct iris_resource *res = iris_alloc_resource(pscreen, templ); 517 518 assert(templ->target == PIPE_BUFFER); 519 assert(templ->height0 <= 1); 520 assert(templ->depth0 <= 1); 521 assert(templ->format == PIPE_FORMAT_NONE || 522 util_format_get_blocksize(templ->format) == 1); 523 524 res->internal_format = templ->format; 525 res->surf.tiling = ISL_TILING_LINEAR; 526 527 enum iris_memory_zone memzone = IRIS_MEMZONE_OTHER; 528 const char *name = templ->target == PIPE_BUFFER ? "buffer" : "miptree"; 529 if (templ->flags & IRIS_RESOURCE_FLAG_SHADER_MEMZONE) { 530 memzone = IRIS_MEMZONE_SHADER; 531 name = "shader kernels"; 532 } else if (templ->flags & IRIS_RESOURCE_FLAG_SURFACE_MEMZONE) { 533 memzone = IRIS_MEMZONE_SURFACE; 534 name = "surface state"; 535 } else if (templ->flags & IRIS_RESOURCE_FLAG_DYNAMIC_MEMZONE) { 536 memzone = IRIS_MEMZONE_DYNAMIC; 537 name = "dynamic state"; 538 } 539 540 res->bo = iris_bo_alloc(screen->bufmgr, name, templ->width0, memzone); 541 if (!res->bo) { 542 iris_resource_destroy(pscreen, &res->base); 543 return NULL; 544 } 545 546 return &res->base; 547} 548 549static struct pipe_resource * 550iris_resource_create_with_modifiers(struct pipe_screen *pscreen, 551 const struct pipe_resource *templ, 552 const uint64_t *modifiers, 553 int modifiers_count) 554{ 555 struct iris_screen *screen = (struct iris_screen *)pscreen; 556 struct gen_device_info *devinfo = &screen->devinfo; 557 struct iris_resource *res = iris_alloc_resource(pscreen, templ); 558 559 if (!res) 560 return NULL; 561 562 const struct util_format_description *format_desc = 563 util_format_description(templ->format); 564 const bool has_depth = util_format_has_depth(format_desc); 565 uint64_t modifier = 566 select_best_modifier(devinfo, modifiers, modifiers_count); 567 568 isl_tiling_flags_t tiling_flags = ISL_TILING_ANY_MASK; 569 570 if (modifier != DRM_FORMAT_MOD_INVALID) { 571 res->mod_info = isl_drm_modifier_get_info(modifier); 572 573 tiling_flags = 1 << res->mod_info->tiling; 574 } else { 575 if (modifiers_count > 0) { 576 fprintf(stderr, "Unsupported modifier, resource creation failed.\n"); 577 return NULL; 578 } 579 580 /* No modifiers - we can select our own tiling. */ 581 582 if (has_depth) { 583 /* Depth must be Y-tiled */ 584 tiling_flags = ISL_TILING_Y0_BIT; 585 } else if (templ->format == PIPE_FORMAT_S8_UINT) { 586 /* Stencil must be W-tiled */ 587 tiling_flags = ISL_TILING_W_BIT; 588 } else if (templ->target == PIPE_BUFFER || 589 templ->target == PIPE_TEXTURE_1D || 590 templ->target == PIPE_TEXTURE_1D_ARRAY) { 591 /* Use linear for buffers and 1D textures */ 592 tiling_flags = ISL_TILING_LINEAR_BIT; 593 } 594 595 /* Use linear for staging buffers */ 596 if (templ->usage == PIPE_USAGE_STAGING || 597 templ->bind & (PIPE_BIND_LINEAR | PIPE_BIND_CURSOR) ) 598 tiling_flags = ISL_TILING_LINEAR_BIT; 599 } 600 601 isl_surf_usage_flags_t usage = pipe_bind_to_isl_usage(templ->bind); 602 603 if (templ->target == PIPE_TEXTURE_CUBE || 604 templ->target == PIPE_TEXTURE_CUBE_ARRAY) 605 usage |= ISL_SURF_USAGE_CUBE_BIT; 606 607 if (templ->usage != PIPE_USAGE_STAGING) { 608 if (templ->format == PIPE_FORMAT_S8_UINT) 609 usage |= ISL_SURF_USAGE_STENCIL_BIT; 610 else if (has_depth) 611 usage |= ISL_SURF_USAGE_DEPTH_BIT; 612 } 613 614 enum pipe_format pfmt = templ->format; 615 res->internal_format = pfmt; 616 617 /* Should be handled by u_transfer_helper */ 618 assert(!util_format_is_depth_and_stencil(pfmt)); 619 620 struct iris_format_info fmt = iris_format_for_usage(devinfo, pfmt, usage); 621 assert(fmt.fmt != ISL_FORMAT_UNSUPPORTED); 622 623 UNUSED const bool isl_surf_created_successfully = 624 isl_surf_init(&screen->isl_dev, &res->surf, 625 .dim = target_to_isl_surf_dim(templ->target), 626 .format = fmt.fmt, 627 .width = templ->width0, 628 .height = templ->height0, 629 .depth = templ->depth0, 630 .levels = templ->last_level + 1, 631 .array_len = templ->array_size, 632 .samples = MAX2(templ->nr_samples, 1), 633 .min_alignment_B = 0, 634 .row_pitch_B = 0, 635 .usage = usage, 636 .tiling_flags = tiling_flags); 637 assert(isl_surf_created_successfully); 638 639 if (res->mod_info) { 640 res->aux.possible_usages |= 1 << res->mod_info->aux_usage; 641 } else if (supports_mcs(&res->surf)) { 642 res->aux.possible_usages |= 1 << ISL_AUX_USAGE_MCS; 643 } else if (has_depth) { 644 if (likely(!(INTEL_DEBUG & DEBUG_NO_HIZ))) 645 res->aux.possible_usages |= 1 << ISL_AUX_USAGE_HIZ; 646 } else if (likely(!(INTEL_DEBUG & DEBUG_NO_RBC)) && 647 supports_ccs(devinfo, &res->surf)) { 648 if (isl_format_supports_ccs_e(devinfo, res->surf.format)) 649 res->aux.possible_usages |= 1 << ISL_AUX_USAGE_CCS_E; 650 651 if (isl_format_supports_ccs_d(devinfo, res->surf.format)) 652 res->aux.possible_usages |= 1 << ISL_AUX_USAGE_CCS_D; 653 } 654 655 res->aux.usage = util_last_bit(res->aux.possible_usages) - 1; 656 657 res->aux.sampler_usages = res->aux.possible_usages; 658 659 /* We don't always support sampling with hiz. But when we do, it must be 660 * single sampled. 661 */ 662 if (!devinfo->has_sample_with_hiz || res->surf.samples > 1) { 663 res->aux.sampler_usages &= ~(1 << ISL_AUX_USAGE_HIZ); 664 } 665 666 const char *name = "miptree"; 667 enum iris_memory_zone memzone = IRIS_MEMZONE_OTHER; 668 669 unsigned int flags = 0; 670 if (templ->usage == PIPE_USAGE_STAGING) 671 flags |= BO_ALLOC_COHERENT; 672 673 /* These are for u_upload_mgr buffers only */ 674 assert(!(templ->flags & (IRIS_RESOURCE_FLAG_SHADER_MEMZONE | 675 IRIS_RESOURCE_FLAG_SURFACE_MEMZONE | 676 IRIS_RESOURCE_FLAG_DYNAMIC_MEMZONE))); 677 678 res->bo = iris_bo_alloc_tiled(screen->bufmgr, name, res->surf.size_B, 679 memzone, 680 isl_tiling_to_i915_tiling(res->surf.tiling), 681 res->surf.row_pitch_B, flags); 682 683 if (!res->bo) 684 goto fail; 685 686 if (!iris_resource_alloc_aux(screen, res)) 687 goto fail; 688 689 return &res->base; 690 691fail: 692 fprintf(stderr, "XXX: resource creation failed\n"); 693 iris_resource_destroy(pscreen, &res->base); 694 return NULL; 695 696} 697 698static struct pipe_resource * 699iris_resource_create(struct pipe_screen *pscreen, 700 const struct pipe_resource *templ) 701{ 702 if (templ->target == PIPE_BUFFER) 703 return iris_resource_create_for_buffer(pscreen, templ); 704 else 705 return iris_resource_create_with_modifiers(pscreen, templ, NULL, 0); 706} 707 708static uint64_t 709tiling_to_modifier(uint32_t tiling) 710{ 711 static const uint64_t map[] = { 712 [I915_TILING_NONE] = DRM_FORMAT_MOD_LINEAR, 713 [I915_TILING_X] = I915_FORMAT_MOD_X_TILED, 714 [I915_TILING_Y] = I915_FORMAT_MOD_Y_TILED, 715 }; 716 717 assert(tiling < ARRAY_SIZE(map)); 718 719 return map[tiling]; 720} 721 722static struct pipe_resource * 723iris_resource_from_user_memory(struct pipe_screen *pscreen, 724 const struct pipe_resource *templ, 725 void *user_memory) 726{ 727 struct iris_screen *screen = (struct iris_screen *)pscreen; 728 struct iris_bufmgr *bufmgr = screen->bufmgr; 729 struct iris_resource *res = iris_alloc_resource(pscreen, templ); 730 if (!res) 731 return NULL; 732 733 assert(templ->target == PIPE_BUFFER); 734 735 res->internal_format = templ->format; 736 res->bo = iris_bo_create_userptr(bufmgr, "user", 737 user_memory, templ->width0, 738 IRIS_MEMZONE_OTHER); 739 if (!res->bo) { 740 free(res); 741 return NULL; 742 } 743 744 util_range_add(&res->valid_buffer_range, 0, templ->width0); 745 746 return &res->base; 747} 748 749static struct pipe_resource * 750iris_resource_from_handle(struct pipe_screen *pscreen, 751 const struct pipe_resource *templ, 752 struct winsys_handle *whandle, 753 unsigned usage) 754{ 755 struct iris_screen *screen = (struct iris_screen *)pscreen; 756 struct gen_device_info *devinfo = &screen->devinfo; 757 struct iris_bufmgr *bufmgr = screen->bufmgr; 758 struct iris_resource *res = iris_alloc_resource(pscreen, templ); 759 if (!res) 760 return NULL; 761 762 if (whandle->offset != 0) { 763 dbg_printf("Attempt to import unsupported winsys offset %u\n", 764 whandle->offset); 765 goto fail; 766 } 767 768 switch (whandle->type) { 769 case WINSYS_HANDLE_TYPE_FD: 770 res->bo = iris_bo_import_dmabuf(bufmgr, whandle->handle); 771 break; 772 case WINSYS_HANDLE_TYPE_SHARED: 773 res->bo = iris_bo_gem_create_from_name(bufmgr, "winsys image", 774 whandle->handle); 775 break; 776 default: 777 unreachable("invalid winsys handle type"); 778 } 779 if (!res->bo) 780 return NULL; 781 782 uint64_t modifier = whandle->modifier; 783 if (modifier == DRM_FORMAT_MOD_INVALID) { 784 modifier = tiling_to_modifier(res->bo->tiling_mode); 785 } 786 res->mod_info = isl_drm_modifier_get_info(modifier); 787 assert(res->mod_info); 788 789 isl_surf_usage_flags_t isl_usage = pipe_bind_to_isl_usage(templ->bind); 790 791 const struct iris_format_info fmt = 792 iris_format_for_usage(devinfo, templ->format, isl_usage); 793 res->internal_format = templ->format; 794 795 if (templ->target == PIPE_BUFFER) { 796 res->surf.tiling = ISL_TILING_LINEAR; 797 } else { 798 isl_surf_init(&screen->isl_dev, &res->surf, 799 .dim = target_to_isl_surf_dim(templ->target), 800 .format = fmt.fmt, 801 .width = templ->width0, 802 .height = templ->height0, 803 .depth = templ->depth0, 804 .levels = templ->last_level + 1, 805 .array_len = templ->array_size, 806 .samples = MAX2(templ->nr_samples, 1), 807 .min_alignment_B = 0, 808 .row_pitch_B = whandle->stride, 809 .usage = isl_usage, 810 .tiling_flags = 1 << res->mod_info->tiling); 811 812 assert(res->bo->tiling_mode == 813 isl_tiling_to_i915_tiling(res->surf.tiling)); 814 815 // XXX: create_ccs_buf_for_image? 816 if (!iris_resource_alloc_aux(screen, res)) 817 goto fail; 818 } 819 820 return &res->base; 821 822fail: 823 iris_resource_destroy(pscreen, &res->base); 824 return NULL; 825} 826 827static void 828iris_flush_resource(struct pipe_context *ctx, struct pipe_resource *resource) 829{ 830 struct iris_context *ice = (struct iris_context *)ctx; 831 struct iris_batch *render_batch = &ice->batches[IRIS_BATCH_RENDER]; 832 struct iris_resource *res = (void *) resource; 833 const struct isl_drm_modifier_info *mod = res->mod_info; 834 835 iris_resource_prepare_access(ice, render_batch, res, 836 0, INTEL_REMAINING_LEVELS, 837 0, INTEL_REMAINING_LAYERS, 838 mod ? mod->aux_usage : ISL_AUX_USAGE_NONE, 839 mod ? mod->supports_clear_color : false); 840} 841 842static boolean 843iris_resource_get_handle(struct pipe_screen *pscreen, 844 struct pipe_context *ctx, 845 struct pipe_resource *resource, 846 struct winsys_handle *whandle, 847 unsigned usage) 848{ 849 struct iris_resource *res = (struct iris_resource *)resource; 850 851 /* Disable aux usage if explicit flush not set and this is the 852 * first time we are dealing with this resource. 853 */ 854 if ((!(usage & PIPE_HANDLE_USAGE_EXPLICIT_FLUSH) && res->aux.usage != 0)) { 855 if (p_atomic_read(&resource->reference.count) == 1) 856 iris_resource_disable_aux(res); 857 } 858 859 /* If this is a buffer, stride should be 0 - no need to special case */ 860 whandle->stride = res->surf.row_pitch_B; 861 whandle->modifier = 862 res->mod_info ? res->mod_info->modifier 863 : tiling_to_modifier(res->bo->tiling_mode); 864 865#ifndef NDEBUG 866 enum isl_aux_usage allowed_usage = 867 res->mod_info ? res->mod_info->aux_usage : ISL_AUX_USAGE_NONE; 868 869 if (res->aux.usage != allowed_usage) { 870 enum isl_aux_state aux_state = iris_resource_get_aux_state(res, 0, 0); 871 assert(aux_state == ISL_AUX_STATE_RESOLVED || 872 aux_state == ISL_AUX_STATE_PASS_THROUGH); 873 } 874#endif 875 876 switch (whandle->type) { 877 case WINSYS_HANDLE_TYPE_SHARED: 878 return iris_bo_flink(res->bo, &whandle->handle) == 0; 879 case WINSYS_HANDLE_TYPE_KMS: 880 whandle->handle = iris_bo_export_gem_handle(res->bo); 881 return true; 882 case WINSYS_HANDLE_TYPE_FD: 883 return iris_bo_export_dmabuf(res->bo, (int *) &whandle->handle) == 0; 884 } 885 886 return false; 887} 888 889static bool 890resource_is_busy(struct iris_context *ice, 891 struct iris_resource *res) 892{ 893 bool busy = iris_bo_busy(res->bo); 894 895 for (int i = 0; i < IRIS_BATCH_COUNT; i++) 896 busy |= iris_batch_references(&ice->batches[i], res->bo); 897 898 return busy; 899} 900 901static void 902iris_invalidate_resource(struct pipe_context *ctx, 903 struct pipe_resource *resource) 904{ 905 struct iris_screen *screen = (void *) ctx->screen; 906 struct iris_context *ice = (void *) ctx; 907 struct iris_resource *res = (void *) resource; 908 909 if (resource->target != PIPE_BUFFER) 910 return; 911 912 if (!resource_is_busy(ice, res)) { 913 /* The resource is idle, so just mark that it contains no data and 914 * keep using the same underlying buffer object. 915 */ 916 util_range_set_empty(&res->valid_buffer_range); 917 return; 918 } 919 920 /* Otherwise, try and replace the backing storage with a new BO. */ 921 922 /* We can't reallocate memory we didn't allocate in the first place. */ 923 if (res->bo->userptr) 924 return; 925 926 // XXX: We should support this. 927 if (res->bind_history & PIPE_BIND_STREAM_OUTPUT) 928 return; 929 930 struct iris_bo *old_bo = res->bo; 931 struct iris_bo *new_bo = 932 iris_bo_alloc(screen->bufmgr, res->bo->name, resource->width0, 933 iris_memzone_for_address(old_bo->gtt_offset)); 934 if (!new_bo) 935 return; 936 937 /* Swap out the backing storage */ 938 res->bo = new_bo; 939 940 /* Rebind the buffer, replacing any state referring to the old BO's 941 * address, and marking state dirty so it's reemitted. 942 */ 943 ice->vtbl.rebind_buffer(ice, res, old_bo->gtt_offset); 944 945 util_range_set_empty(&res->valid_buffer_range); 946 947 iris_bo_unreference(old_bo); 948} 949 950static void 951iris_flush_staging_region(struct pipe_transfer *xfer, 952 const struct pipe_box *flush_box) 953{ 954 if (!(xfer->usage & PIPE_TRANSFER_WRITE)) 955 return; 956 957 struct iris_transfer *map = (void *) xfer; 958 959 struct pipe_box src_box = *flush_box; 960 961 /* Account for extra alignment padding in staging buffer */ 962 if (xfer->resource->target == PIPE_BUFFER) 963 src_box.x += xfer->box.x % IRIS_MAP_BUFFER_ALIGNMENT; 964 965 struct pipe_box dst_box = (struct pipe_box) { 966 .x = xfer->box.x + flush_box->x, 967 .y = xfer->box.y + flush_box->y, 968 .z = xfer->box.z + flush_box->z, 969 .width = flush_box->width, 970 .height = flush_box->height, 971 .depth = flush_box->depth, 972 }; 973 974 iris_copy_region(map->blorp, map->batch, xfer->resource, xfer->level, 975 dst_box.x, dst_box.y, dst_box.z, map->staging, 0, 976 &src_box); 977} 978 979static void 980iris_unmap_copy_region(struct iris_transfer *map) 981{ 982 iris_resource_destroy(map->staging->screen, map->staging); 983 984 map->ptr = NULL; 985} 986 987static void 988iris_map_copy_region(struct iris_transfer *map) 989{ 990 struct pipe_screen *pscreen = &map->batch->screen->base; 991 struct pipe_transfer *xfer = &map->base; 992 struct pipe_box *box = &xfer->box; 993 struct iris_resource *res = (void *) xfer->resource; 994 995 unsigned extra = xfer->resource->target == PIPE_BUFFER ? 996 box->x % IRIS_MAP_BUFFER_ALIGNMENT : 0; 997 998 struct pipe_resource templ = (struct pipe_resource) { 999 .usage = PIPE_USAGE_STAGING, 1000 .width0 = box->width + extra, 1001 .height0 = box->height, 1002 .depth0 = 1, 1003 .nr_samples = xfer->resource->nr_samples, 1004 .nr_storage_samples = xfer->resource->nr_storage_samples, 1005 .array_size = box->depth, 1006 .format = res->internal_format, 1007 }; 1008 1009 if (xfer->resource->target == PIPE_BUFFER) 1010 templ.target = PIPE_BUFFER; 1011 else if (templ.array_size > 1) 1012 templ.target = PIPE_TEXTURE_2D_ARRAY; 1013 else 1014 templ.target = PIPE_TEXTURE_2D; 1015 1016 map->staging = iris_resource_create(pscreen, &templ); 1017 assert(map->staging); 1018 1019 if (templ.target != PIPE_BUFFER) { 1020 struct isl_surf *surf = &((struct iris_resource *) map->staging)->surf; 1021 xfer->stride = isl_surf_get_row_pitch_B(surf); 1022 xfer->layer_stride = isl_surf_get_array_pitch(surf); 1023 } 1024 1025 if (!(xfer->usage & PIPE_TRANSFER_DISCARD_RANGE)) { 1026 iris_copy_region(map->blorp, map->batch, map->staging, 0, extra, 0, 0, 1027 xfer->resource, xfer->level, box); 1028 /* Ensure writes to the staging BO land before we map it below. */ 1029 iris_emit_pipe_control_flush(map->batch, 1030 PIPE_CONTROL_RENDER_TARGET_FLUSH | 1031 PIPE_CONTROL_CS_STALL); 1032 } 1033 1034 struct iris_bo *staging_bo = iris_resource_bo(map->staging); 1035 1036 if (iris_batch_references(map->batch, staging_bo)) 1037 iris_batch_flush(map->batch); 1038 1039 map->ptr = 1040 iris_bo_map(map->dbg, staging_bo, xfer->usage & MAP_FLAGS) + extra; 1041 1042 map->unmap = iris_unmap_copy_region; 1043} 1044 1045static void 1046get_image_offset_el(const struct isl_surf *surf, unsigned level, unsigned z, 1047 unsigned *out_x0_el, unsigned *out_y0_el) 1048{ 1049 if (surf->dim == ISL_SURF_DIM_3D) { 1050 isl_surf_get_image_offset_el(surf, level, 0, z, out_x0_el, out_y0_el); 1051 } else { 1052 isl_surf_get_image_offset_el(surf, level, z, 0, out_x0_el, out_y0_el); 1053 } 1054} 1055 1056/** 1057 * Get pointer offset into stencil buffer. 1058 * 1059 * The stencil buffer is W tiled. Since the GTT is incapable of W fencing, we 1060 * must decode the tile's layout in software. 1061 * 1062 * See 1063 * - PRM, 2011 Sandy Bridge, Volume 1, Part 2, Section 4.5.2.1 W-Major Tile 1064 * Format. 1065 * - PRM, 2011 Sandy Bridge, Volume 1, Part 2, Section 4.5.3 Tiling Algorithm 1066 * 1067 * Even though the returned offset is always positive, the return type is 1068 * signed due to 1069 * commit e8b1c6d6f55f5be3bef25084fdd8b6127517e137 1070 * mesa: Fix return type of _mesa_get_format_bytes() (#37351) 1071 */ 1072static intptr_t 1073s8_offset(uint32_t stride, uint32_t x, uint32_t y, bool swizzled) 1074{ 1075 uint32_t tile_size = 4096; 1076 uint32_t tile_width = 64; 1077 uint32_t tile_height = 64; 1078 uint32_t row_size = 64 * stride / 2; /* Two rows are interleaved. */ 1079 1080 uint32_t tile_x = x / tile_width; 1081 uint32_t tile_y = y / tile_height; 1082 1083 /* The byte's address relative to the tile's base addres. */ 1084 uint32_t byte_x = x % tile_width; 1085 uint32_t byte_y = y % tile_height; 1086 1087 uintptr_t u = tile_y * row_size 1088 + tile_x * tile_size 1089 + 512 * (byte_x / 8) 1090 + 64 * (byte_y / 8) 1091 + 32 * ((byte_y / 4) % 2) 1092 + 16 * ((byte_x / 4) % 2) 1093 + 8 * ((byte_y / 2) % 2) 1094 + 4 * ((byte_x / 2) % 2) 1095 + 2 * (byte_y % 2) 1096 + 1 * (byte_x % 2); 1097 1098 if (swizzled) { 1099 /* adjust for bit6 swizzling */ 1100 if (((byte_x / 8) % 2) == 1) { 1101 if (((byte_y / 8) % 2) == 0) { 1102 u += 64; 1103 } else { 1104 u -= 64; 1105 } 1106 } 1107 } 1108 1109 return u; 1110} 1111 1112static void 1113iris_unmap_s8(struct iris_transfer *map) 1114{ 1115 struct pipe_transfer *xfer = &map->base; 1116 const struct pipe_box *box = &xfer->box; 1117 struct iris_resource *res = (struct iris_resource *) xfer->resource; 1118 struct isl_surf *surf = &res->surf; 1119 const bool has_swizzling = false; 1120 1121 if (xfer->usage & PIPE_TRANSFER_WRITE) { 1122 uint8_t *untiled_s8_map = map->ptr; 1123 uint8_t *tiled_s8_map = 1124 iris_bo_map(map->dbg, res->bo, (xfer->usage | MAP_RAW) & MAP_FLAGS); 1125 1126 for (int s = 0; s < box->depth; s++) { 1127 unsigned x0_el, y0_el; 1128 get_image_offset_el(surf, xfer->level, box->z + s, &x0_el, &y0_el); 1129 1130 for (uint32_t y = 0; y < box->height; y++) { 1131 for (uint32_t x = 0; x < box->width; x++) { 1132 ptrdiff_t offset = s8_offset(surf->row_pitch_B, 1133 x0_el + box->x + x, 1134 y0_el + box->y + y, 1135 has_swizzling); 1136 tiled_s8_map[offset] = 1137 untiled_s8_map[s * xfer->layer_stride + y * xfer->stride + x]; 1138 } 1139 } 1140 } 1141 } 1142 1143 free(map->buffer); 1144} 1145 1146static void 1147iris_map_s8(struct iris_transfer *map) 1148{ 1149 struct pipe_transfer *xfer = &map->base; 1150 const struct pipe_box *box = &xfer->box; 1151 struct iris_resource *res = (struct iris_resource *) xfer->resource; 1152 struct isl_surf *surf = &res->surf; 1153 1154 xfer->stride = surf->row_pitch_B; 1155 xfer->layer_stride = xfer->stride * box->height; 1156 1157 /* The tiling and detiling functions require that the linear buffer has 1158 * a 16-byte alignment (that is, its `x0` is 16-byte aligned). Here we 1159 * over-allocate the linear buffer to get the proper alignment. 1160 */ 1161 map->buffer = map->ptr = malloc(xfer->layer_stride * box->depth); 1162 assert(map->buffer); 1163 1164 const bool has_swizzling = false; 1165 1166 /* One of either READ_BIT or WRITE_BIT or both is set. READ_BIT implies no 1167 * INVALIDATE_RANGE_BIT. WRITE_BIT needs the original values read in unless 1168 * invalidate is set, since we'll be writing the whole rectangle from our 1169 * temporary buffer back out. 1170 */ 1171 if (!(xfer->usage & PIPE_TRANSFER_DISCARD_RANGE)) { 1172 uint8_t *untiled_s8_map = map->ptr; 1173 uint8_t *tiled_s8_map = 1174 iris_bo_map(map->dbg, res->bo, (xfer->usage | MAP_RAW) & MAP_FLAGS); 1175 1176 for (int s = 0; s < box->depth; s++) { 1177 unsigned x0_el, y0_el; 1178 get_image_offset_el(surf, xfer->level, box->z + s, &x0_el, &y0_el); 1179 1180 for (uint32_t y = 0; y < box->height; y++) { 1181 for (uint32_t x = 0; x < box->width; x++) { 1182 ptrdiff_t offset = s8_offset(surf->row_pitch_B, 1183 x0_el + box->x + x, 1184 y0_el + box->y + y, 1185 has_swizzling); 1186 untiled_s8_map[s * xfer->layer_stride + y * xfer->stride + x] = 1187 tiled_s8_map[offset]; 1188 } 1189 } 1190 } 1191 } 1192 1193 map->unmap = iris_unmap_s8; 1194} 1195 1196/* Compute extent parameters for use with tiled_memcpy functions. 1197 * xs are in units of bytes and ys are in units of strides. 1198 */ 1199static inline void 1200tile_extents(const struct isl_surf *surf, 1201 const struct pipe_box *box, 1202 unsigned level, int z, 1203 unsigned *x1_B, unsigned *x2_B, 1204 unsigned *y1_el, unsigned *y2_el) 1205{ 1206 const struct isl_format_layout *fmtl = isl_format_get_layout(surf->format); 1207 const unsigned cpp = fmtl->bpb / 8; 1208 1209 assert(box->x % fmtl->bw == 0); 1210 assert(box->y % fmtl->bh == 0); 1211 1212 unsigned x0_el, y0_el; 1213 get_image_offset_el(surf, level, box->z + z, &x0_el, &y0_el); 1214 1215 *x1_B = (box->x / fmtl->bw + x0_el) * cpp; 1216 *y1_el = box->y / fmtl->bh + y0_el; 1217 *x2_B = (DIV_ROUND_UP(box->x + box->width, fmtl->bw) + x0_el) * cpp; 1218 *y2_el = DIV_ROUND_UP(box->y + box->height, fmtl->bh) + y0_el; 1219} 1220 1221static void 1222iris_unmap_tiled_memcpy(struct iris_transfer *map) 1223{ 1224 struct pipe_transfer *xfer = &map->base; 1225 const struct pipe_box *box = &xfer->box; 1226 struct iris_resource *res = (struct iris_resource *) xfer->resource; 1227 struct isl_surf *surf = &res->surf; 1228 1229 const bool has_swizzling = false; 1230 1231 if (xfer->usage & PIPE_TRANSFER_WRITE) { 1232 char *dst = 1233 iris_bo_map(map->dbg, res->bo, (xfer->usage | MAP_RAW) & MAP_FLAGS); 1234 1235 for (int s = 0; s < box->depth; s++) { 1236 unsigned x1, x2, y1, y2; 1237 tile_extents(surf, box, xfer->level, s, &x1, &x2, &y1, &y2); 1238 1239 void *ptr = map->ptr + s * xfer->layer_stride; 1240 1241 isl_memcpy_linear_to_tiled(x1, x2, y1, y2, dst, ptr, 1242 surf->row_pitch_B, xfer->stride, 1243 has_swizzling, surf->tiling, ISL_MEMCPY); 1244 } 1245 } 1246 os_free_aligned(map->buffer); 1247 map->buffer = map->ptr = NULL; 1248} 1249 1250static void 1251iris_map_tiled_memcpy(struct iris_transfer *map) 1252{ 1253 struct pipe_transfer *xfer = &map->base; 1254 const struct pipe_box *box = &xfer->box; 1255 struct iris_resource *res = (struct iris_resource *) xfer->resource; 1256 struct isl_surf *surf = &res->surf; 1257 1258 xfer->stride = ALIGN(surf->row_pitch_B, 16); 1259 xfer->layer_stride = xfer->stride * box->height; 1260 1261 unsigned x1, x2, y1, y2; 1262 tile_extents(surf, box, xfer->level, 0, &x1, &x2, &y1, &y2); 1263 1264 /* The tiling and detiling functions require that the linear buffer has 1265 * a 16-byte alignment (that is, its `x0` is 16-byte aligned). Here we 1266 * over-allocate the linear buffer to get the proper alignment. 1267 */ 1268 map->buffer = 1269 os_malloc_aligned(xfer->layer_stride * box->depth, 16); 1270 assert(map->buffer); 1271 map->ptr = (char *)map->buffer + (x1 & 0xf); 1272 1273 const bool has_swizzling = false; 1274 1275 // XXX: PIPE_TRANSFER_READ? 1276 if (!(xfer->usage & PIPE_TRANSFER_DISCARD_RANGE)) { 1277 char *src = 1278 iris_bo_map(map->dbg, res->bo, (xfer->usage | MAP_RAW) & MAP_FLAGS); 1279 1280 for (int s = 0; s < box->depth; s++) { 1281 unsigned x1, x2, y1, y2; 1282 tile_extents(surf, box, xfer->level, s, &x1, &x2, &y1, &y2); 1283 1284 /* Use 's' rather than 'box->z' to rebase the first slice to 0. */ 1285 void *ptr = map->ptr + s * xfer->layer_stride; 1286 1287 isl_memcpy_tiled_to_linear(x1, x2, y1, y2, ptr, src, xfer->stride, 1288 surf->row_pitch_B, has_swizzling, 1289 surf->tiling, ISL_MEMCPY_STREAMING_LOAD); 1290 } 1291 } 1292 1293 map->unmap = iris_unmap_tiled_memcpy; 1294} 1295 1296static void 1297iris_map_direct(struct iris_transfer *map) 1298{ 1299 struct pipe_transfer *xfer = &map->base; 1300 struct pipe_box *box = &xfer->box; 1301 struct iris_resource *res = (struct iris_resource *) xfer->resource; 1302 1303 void *ptr = iris_bo_map(map->dbg, res->bo, xfer->usage & MAP_FLAGS); 1304 1305 if (res->base.target == PIPE_BUFFER) { 1306 xfer->stride = 0; 1307 xfer->layer_stride = 0; 1308 1309 map->ptr = ptr + box->x; 1310 } else { 1311 struct isl_surf *surf = &res->surf; 1312 const struct isl_format_layout *fmtl = 1313 isl_format_get_layout(surf->format); 1314 const unsigned cpp = fmtl->bpb / 8; 1315 unsigned x0_el, y0_el; 1316 1317 get_image_offset_el(surf, xfer->level, box->z, &x0_el, &y0_el); 1318 1319 xfer->stride = isl_surf_get_row_pitch_B(surf); 1320 xfer->layer_stride = isl_surf_get_array_pitch(surf); 1321 1322 map->ptr = ptr + (y0_el + box->y) * xfer->stride + (x0_el + box->x) * cpp; 1323 } 1324} 1325 1326static bool 1327can_promote_to_async(const struct iris_resource *res, 1328 const struct pipe_box *box, 1329 enum pipe_transfer_usage usage) 1330{ 1331 /* If we're writing to a section of the buffer that hasn't even been 1332 * initialized with useful data, then we can safely promote this write 1333 * to be unsynchronized. This helps the common pattern of appending data. 1334 */ 1335 return res->base.target == PIPE_BUFFER && (usage & PIPE_TRANSFER_WRITE) && 1336 !(usage & TC_TRANSFER_MAP_NO_INFER_UNSYNCHRONIZED) && 1337 !util_ranges_intersect(&res->valid_buffer_range, box->x, 1338 box->x + box->width); 1339} 1340 1341static void * 1342iris_transfer_map(struct pipe_context *ctx, 1343 struct pipe_resource *resource, 1344 unsigned level, 1345 enum pipe_transfer_usage usage, 1346 const struct pipe_box *box, 1347 struct pipe_transfer **ptransfer) 1348{ 1349 struct iris_context *ice = (struct iris_context *)ctx; 1350 struct iris_resource *res = (struct iris_resource *)resource; 1351 struct isl_surf *surf = &res->surf; 1352 1353 if (usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE) { 1354 /* Replace the backing storage with a fresh buffer for non-async maps */ 1355 if (!(usage & (PIPE_TRANSFER_UNSYNCHRONIZED | 1356 TC_TRANSFER_MAP_NO_INVALIDATE))) 1357 iris_invalidate_resource(ctx, resource); 1358 1359 /* If we can discard the whole resource, we can discard the range. */ 1360 usage |= PIPE_TRANSFER_DISCARD_RANGE; 1361 } 1362 1363 bool map_would_stall = false; 1364 1365 if (resource->target != PIPE_BUFFER) { 1366 iris_resource_access_raw(ice, &ice->batches[IRIS_BATCH_RENDER], res, 1367 level, box->z, box->depth, 1368 usage & PIPE_TRANSFER_WRITE); 1369 } 1370 1371 if (!(usage & PIPE_TRANSFER_UNSYNCHRONIZED) && 1372 can_promote_to_async(res, box, usage)) { 1373 usage |= PIPE_TRANSFER_UNSYNCHRONIZED; 1374 } 1375 1376 if (!(usage & PIPE_TRANSFER_UNSYNCHRONIZED)) { 1377 map_would_stall = resource_is_busy(ice, res); 1378 1379 if (map_would_stall && (usage & PIPE_TRANSFER_DONTBLOCK) && 1380 (usage & PIPE_TRANSFER_MAP_DIRECTLY)) 1381 return NULL; 1382 } 1383 1384 if (surf->tiling != ISL_TILING_LINEAR && 1385 (usage & PIPE_TRANSFER_MAP_DIRECTLY)) 1386 return NULL; 1387 1388 struct iris_transfer *map = slab_alloc(&ice->transfer_pool); 1389 struct pipe_transfer *xfer = &map->base; 1390 1391 if (!map) 1392 return NULL; 1393 1394 memset(map, 0, sizeof(*map)); 1395 map->dbg = &ice->dbg; 1396 1397 pipe_resource_reference(&xfer->resource, resource); 1398 xfer->level = level; 1399 xfer->usage = usage; 1400 xfer->box = *box; 1401 *ptransfer = xfer; 1402 1403 if (usage & PIPE_TRANSFER_WRITE) 1404 util_range_add(&res->valid_buffer_range, box->x, box->x + box->width); 1405 1406 /* Avoid using GPU copies for persistent/coherent buffers, as the idea 1407 * there is to access them simultaneously on the CPU & GPU. This also 1408 * avoids trying to use GPU copies for our u_upload_mgr buffers which 1409 * contain state we're constructing for a GPU draw call, which would 1410 * kill us with infinite stack recursion. 1411 */ 1412 bool no_gpu = usage & (PIPE_TRANSFER_PERSISTENT | 1413 PIPE_TRANSFER_COHERENT | 1414 PIPE_TRANSFER_MAP_DIRECTLY); 1415 1416 /* GPU copies are not useful for buffer reads. Instead of stalling to 1417 * read from the original buffer, we'd simply copy it to a temporary... 1418 * then stall (a bit longer) to read from that buffer. 1419 * 1420 * Images are less clear-cut. Color resolves are destructive, removing 1421 * the underlying compression, so we'd rather blit the data to a linear 1422 * temporary and map that, to avoid the resolve. (It might be better to 1423 * a tiled temporary and use the tiled_memcpy paths...) 1424 */ 1425 if (!(usage & PIPE_TRANSFER_DISCARD_RANGE) && 1426 res->aux.usage != ISL_AUX_USAGE_CCS_E && 1427 res->aux.usage != ISL_AUX_USAGE_CCS_D) { 1428 no_gpu = true; 1429 } 1430 1431 const struct isl_format_layout *fmtl = isl_format_get_layout(surf->format); 1432 if (fmtl->txc == ISL_TXC_ASTC) 1433 no_gpu = true; 1434 1435 if ((map_would_stall || res->aux.usage == ISL_AUX_USAGE_CCS_E) && !no_gpu) { 1436 /* If we need a synchronous mapping and the resource is busy, 1437 * we copy to/from a linear temporary buffer using the GPU. 1438 */ 1439 map->batch = &ice->batches[IRIS_BATCH_RENDER]; 1440 map->blorp = &ice->blorp; 1441 iris_map_copy_region(map); 1442 } else { 1443 /* Otherwise we're free to map on the CPU. Flush if needed. */ 1444 if (!(usage & PIPE_TRANSFER_UNSYNCHRONIZED)) { 1445 for (int i = 0; i < IRIS_BATCH_COUNT; i++) { 1446 if (iris_batch_references(&ice->batches[i], res->bo)) 1447 iris_batch_flush(&ice->batches[i]); 1448 } 1449 } 1450 1451 if (surf->tiling == ISL_TILING_W) { 1452 /* TODO: Teach iris_map_tiled_memcpy about W-tiling... */ 1453 iris_map_s8(map); 1454 } else if (surf->tiling != ISL_TILING_LINEAR) { 1455 iris_map_tiled_memcpy(map); 1456 } else { 1457 iris_map_direct(map); 1458 } 1459 } 1460 1461 return map->ptr; 1462} 1463 1464static void 1465iris_transfer_flush_region(struct pipe_context *ctx, 1466 struct pipe_transfer *xfer, 1467 const struct pipe_box *box) 1468{ 1469 struct iris_context *ice = (struct iris_context *)ctx; 1470 struct iris_resource *res = (struct iris_resource *) xfer->resource; 1471 struct iris_transfer *map = (void *) xfer; 1472 1473 if (map->staging) 1474 iris_flush_staging_region(xfer, box); 1475 1476 for (int i = 0; i < IRIS_BATCH_COUNT; i++) { 1477 if (ice->batches[i].contains_draw || 1478 ice->batches[i].cache.render->entries) { 1479 iris_batch_maybe_flush(&ice->batches[i], 24); 1480 iris_flush_and_dirty_for_history(ice, &ice->batches[i], res); 1481 } 1482 } 1483 1484 /* Make sure we flag constants dirty even if there's no need to emit 1485 * any PIPE_CONTROLs to a batch. 1486 */ 1487 iris_dirty_for_history(ice, res); 1488} 1489 1490static void 1491iris_transfer_unmap(struct pipe_context *ctx, struct pipe_transfer *xfer) 1492{ 1493 struct iris_context *ice = (struct iris_context *)ctx; 1494 struct iris_transfer *map = (void *) xfer; 1495 1496 if (!(xfer->usage & PIPE_TRANSFER_FLUSH_EXPLICIT)) { 1497 struct pipe_box flush_box = { 1498 .x = 0, .y = 0, .z = 0, 1499 .width = xfer->box.width, 1500 .height = xfer->box.height, 1501 .depth = xfer->box.depth, 1502 }; 1503 iris_transfer_flush_region(ctx, xfer, &flush_box); 1504 } 1505 1506 if (map->unmap) 1507 map->unmap(map); 1508 1509 pipe_resource_reference(&xfer->resource, NULL); 1510 slab_free(&ice->transfer_pool, map); 1511} 1512 1513/** 1514 * Mark state dirty that needs to be re-emitted when a resource is written. 1515 */ 1516void 1517iris_dirty_for_history(struct iris_context *ice, 1518 struct iris_resource *res) 1519{ 1520 uint64_t dirty = 0ull; 1521 1522 if (res->bind_history & PIPE_BIND_CONSTANT_BUFFER) { 1523 dirty |= IRIS_DIRTY_CONSTANTS_VS | 1524 IRIS_DIRTY_CONSTANTS_TCS | 1525 IRIS_DIRTY_CONSTANTS_TES | 1526 IRIS_DIRTY_CONSTANTS_GS | 1527 IRIS_DIRTY_CONSTANTS_FS | 1528 IRIS_DIRTY_CONSTANTS_CS | 1529 IRIS_ALL_DIRTY_BINDINGS; 1530 } 1531 1532 ice->state.dirty |= dirty; 1533} 1534 1535/** 1536 * Produce a set of PIPE_CONTROL bits which ensure data written to a 1537 * resource becomes visible, and any stale read cache data is invalidated. 1538 */ 1539uint32_t 1540iris_flush_bits_for_history(struct iris_resource *res) 1541{ 1542 uint32_t flush = PIPE_CONTROL_CS_STALL; 1543 1544 if (res->bind_history & PIPE_BIND_CONSTANT_BUFFER) { 1545 flush |= PIPE_CONTROL_CONST_CACHE_INVALIDATE | 1546 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE; 1547 } 1548 1549 if (res->bind_history & PIPE_BIND_SAMPLER_VIEW) 1550 flush |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE; 1551 1552 if (res->bind_history & (PIPE_BIND_VERTEX_BUFFER | PIPE_BIND_INDEX_BUFFER)) 1553 flush |= PIPE_CONTROL_VF_CACHE_INVALIDATE; 1554 1555 if (res->bind_history & (PIPE_BIND_SHADER_BUFFER | PIPE_BIND_SHADER_IMAGE)) 1556 flush |= PIPE_CONTROL_DATA_CACHE_FLUSH; 1557 1558 return flush; 1559} 1560 1561void 1562iris_flush_and_dirty_for_history(struct iris_context *ice, 1563 struct iris_batch *batch, 1564 struct iris_resource *res) 1565{ 1566 if (res->base.target != PIPE_BUFFER) 1567 return; 1568 1569 uint32_t flush = iris_flush_bits_for_history(res); 1570 1571 /* We've likely used the rendering engine (i.e. BLORP) to write to this 1572 * surface. Flush the render cache so the data actually lands. 1573 */ 1574 if (batch->name != IRIS_BATCH_COMPUTE) 1575 flush |= PIPE_CONTROL_RENDER_TARGET_FLUSH; 1576 1577 iris_emit_pipe_control_flush(batch, flush); 1578 1579 iris_dirty_for_history(ice, res); 1580} 1581 1582bool 1583iris_resource_set_clear_color(struct iris_context *ice, 1584 struct iris_resource *res, 1585 union isl_color_value color) 1586{ 1587 if (memcmp(&res->aux.clear_color, &color, sizeof(color)) != 0) { 1588 res->aux.clear_color = color; 1589 return true; 1590 } 1591 1592 return false; 1593} 1594 1595union isl_color_value 1596iris_resource_get_clear_color(const struct iris_resource *res, 1597 struct iris_bo **clear_color_bo, 1598 uint64_t *clear_color_offset) 1599{ 1600 assert(res->aux.bo); 1601 1602 if (clear_color_bo) 1603 *clear_color_bo = res->aux.clear_color_bo; 1604 if (clear_color_offset) 1605 *clear_color_offset = res->aux.clear_color_offset; 1606 return res->aux.clear_color; 1607} 1608 1609static enum pipe_format 1610iris_resource_get_internal_format(struct pipe_resource *p_res) 1611{ 1612 struct iris_resource *res = (void *) p_res; 1613 return res->internal_format; 1614} 1615 1616static const struct u_transfer_vtbl transfer_vtbl = { 1617 .resource_create = iris_resource_create, 1618 .resource_destroy = iris_resource_destroy, 1619 .transfer_map = iris_transfer_map, 1620 .transfer_unmap = iris_transfer_unmap, 1621 .transfer_flush_region = iris_transfer_flush_region, 1622 .get_internal_format = iris_resource_get_internal_format, 1623 .set_stencil = iris_resource_set_separate_stencil, 1624 .get_stencil = iris_resource_get_separate_stencil, 1625}; 1626 1627void 1628iris_init_screen_resource_functions(struct pipe_screen *pscreen) 1629{ 1630 pscreen->query_dmabuf_modifiers = iris_query_dmabuf_modifiers; 1631 pscreen->resource_create_with_modifiers = 1632 iris_resource_create_with_modifiers; 1633 pscreen->resource_create = u_transfer_helper_resource_create; 1634 pscreen->resource_from_user_memory = iris_resource_from_user_memory; 1635 pscreen->resource_from_handle = iris_resource_from_handle; 1636 pscreen->resource_get_handle = iris_resource_get_handle; 1637 pscreen->resource_destroy = u_transfer_helper_resource_destroy; 1638 pscreen->transfer_helper = 1639 u_transfer_helper_create(&transfer_vtbl, true, true, false, true); 1640} 1641 1642void 1643iris_init_resource_functions(struct pipe_context *ctx) 1644{ 1645 ctx->flush_resource = iris_flush_resource; 1646 ctx->invalidate_resource = iris_invalidate_resource; 1647 ctx->transfer_map = u_transfer_helper_transfer_map; 1648 ctx->transfer_flush_region = u_transfer_helper_transfer_flush_region; 1649 ctx->transfer_unmap = u_transfer_helper_transfer_unmap; 1650 ctx->buffer_subdata = u_default_buffer_subdata; 1651 ctx->texture_subdata = u_default_texture_subdata; 1652} 1653