1/* 2 * Copyright © 2014-2017 Broadcom 3 * Copyright (C) 2012 Rob Clark <robclark@freedesktop.org> 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice (including the next 13 * paragraph) shall be included in all copies or substantial portions of the 14 * Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 22 * IN THE SOFTWARE. 23 */ 24 25#include "pipe/p_defines.h" 26#include "util/u_memory.h" 27#include "util/format/u_format.h" 28#include "util/u_inlines.h" 29#include "util/u_surface.h" 30#include "util/u_transfer_helper.h" 31#include "util/u_upload_mgr.h" 32#include "util/format/u_format_zs.h" 33#include "util/u_drm.h" 34 35#include "drm-uapi/drm_fourcc.h" 36#include "v3d_screen.h" 37#include "v3d_context.h" 38#include "v3d_resource.h" 39#include "broadcom/cle/v3d_packet_v33_pack.h" 40 41static void 42v3d_debug_resource_layout(struct v3d_resource *rsc, const char *caller) 43{ 44 if (!(unlikely(V3D_DEBUG & V3D_DEBUG_SURFACE))) 45 return; 46 47 struct pipe_resource *prsc = &rsc->base; 48 49 if (prsc->target == PIPE_BUFFER) { 50 fprintf(stderr, 51 "rsc %s %p (format %s), %dx%d buffer @0x%08x-0x%08x\n", 52 caller, rsc, 53 util_format_short_name(prsc->format), 54 prsc->width0, prsc->height0, 55 rsc->bo->offset, 56 rsc->bo->offset + rsc->bo->size - 1); 57 return; 58 } 59 60 static const char *const tiling_descriptions[] = { 61 [V3D_TILING_RASTER] = "R", 62 [V3D_TILING_LINEARTILE] = "LT", 63 [V3D_TILING_UBLINEAR_1_COLUMN] = "UB1", 64 [V3D_TILING_UBLINEAR_2_COLUMN] = "UB2", 65 [V3D_TILING_UIF_NO_XOR] = "UIF", 66 [V3D_TILING_UIF_XOR] = "UIF^", 67 }; 68 69 for (int i = 0; i <= prsc->last_level; i++) { 70 struct v3d_resource_slice *slice = &rsc->slices[i]; 71 72 int level_width = slice->stride / rsc->cpp; 73 int level_height = slice->padded_height; 74 int level_depth = 75 u_minify(util_next_power_of_two(prsc->depth0), i); 76 77 fprintf(stderr, 78 "rsc %s %p (format %s), %dx%d: " 79 "level %d (%s) %dx%dx%d -> %dx%dx%d, stride %d@0x%08x\n", 80 caller, rsc, 81 util_format_short_name(prsc->format), 82 prsc->width0, prsc->height0, 83 i, tiling_descriptions[slice->tiling], 84 u_minify(prsc->width0, i), 85 u_minify(prsc->height0, i), 86 u_minify(prsc->depth0, i), 87 level_width, 88 level_height, 89 level_depth, 90 slice->stride, 91 rsc->bo->offset + slice->offset); 92 } 93} 94 95static bool 96v3d_resource_bo_alloc(struct v3d_resource *rsc) 97{ 98 struct pipe_resource *prsc = &rsc->base; 99 struct pipe_screen *pscreen = prsc->screen; 100 struct v3d_bo *bo; 101 102 bo = v3d_bo_alloc(v3d_screen(pscreen), rsc->size, "resource"); 103 if (bo) { 104 v3d_bo_unreference(&rsc->bo); 105 rsc->bo = bo; 106 v3d_debug_resource_layout(rsc, "alloc"); 107 return true; 108 } else { 109 return false; 110 } 111} 112 113static void 114v3d_resource_transfer_unmap(struct pipe_context *pctx, 115 struct pipe_transfer *ptrans) 116{ 117 struct v3d_context *v3d = v3d_context(pctx); 118 struct v3d_transfer *trans = v3d_transfer(ptrans); 119 120 if (trans->map) { 121 struct v3d_resource *rsc = v3d_resource(ptrans->resource); 122 struct v3d_resource_slice *slice = &rsc->slices[ptrans->level]; 123 124 if (ptrans->usage & PIPE_MAP_WRITE) { 125 for (int z = 0; z < ptrans->box.depth; z++) { 126 void *dst = rsc->bo->map + 127 v3d_layer_offset(&rsc->base, 128 ptrans->level, 129 ptrans->box.z + z); 130 v3d_store_tiled_image(dst, 131 slice->stride, 132 (trans->map + 133 ptrans->stride * 134 ptrans->box.height * z), 135 ptrans->stride, 136 slice->tiling, rsc->cpp, 137 slice->padded_height, 138 &ptrans->box); 139 } 140 } 141 free(trans->map); 142 } 143 144 pipe_resource_reference(&ptrans->resource, NULL); 145 slab_free(&v3d->transfer_pool, ptrans); 146} 147 148static void 149rebind_sampler_views(struct v3d_context *v3d, 150 struct v3d_resource *rsc) 151{ 152 for (int st = 0; st < PIPE_SHADER_TYPES; st++) { 153 struct v3d_texture_stateobj *tex = v3d->tex + st; 154 155 for (unsigned i = 0; i < tex->num_textures; i++) { 156 struct pipe_sampler_view *psview = tex->textures[i]; 157 158 if (psview->texture != &rsc->base) 159 continue; 160 161 struct v3d_sampler_view *sview = 162 v3d_sampler_view(psview); 163 164 v3d_create_texture_shader_state_bo(v3d, sview); 165 166 v3d_flag_dirty_sampler_state(v3d, st); 167 } 168 } 169} 170 171static void 172v3d_map_usage_prep(struct pipe_context *pctx, 173 struct pipe_resource *prsc, 174 unsigned usage) 175{ 176 struct v3d_context *v3d = v3d_context(pctx); 177 struct v3d_resource *rsc = v3d_resource(prsc); 178 179 if (usage & PIPE_MAP_DISCARD_WHOLE_RESOURCE) { 180 if (v3d_resource_bo_alloc(rsc)) { 181 /* If it might be bound as one of our vertex buffers 182 * or UBOs, make sure we re-emit vertex buffer state 183 * or uniforms. 184 */ 185 if (prsc->bind & PIPE_BIND_VERTEX_BUFFER) 186 v3d->dirty |= V3D_DIRTY_VTXBUF; 187 if (prsc->bind & PIPE_BIND_CONSTANT_BUFFER) 188 v3d->dirty |= V3D_DIRTY_CONSTBUF; 189 if (prsc->bind & PIPE_BIND_SAMPLER_VIEW) 190 rebind_sampler_views(v3d, rsc); 191 } else { 192 /* If we failed to reallocate, flush users so that we 193 * don't violate any syncing requirements. 194 */ 195 v3d_flush_jobs_reading_resource(v3d, prsc, 196 V3D_FLUSH_DEFAULT, 197 false); 198 } 199 } else if (!(usage & PIPE_MAP_UNSYNCHRONIZED)) { 200 /* If we're writing and the buffer is being used by the CL, we 201 * have to flush the CL first. If we're only reading, we need 202 * to flush if the CL has written our buffer. 203 */ 204 if (usage & PIPE_MAP_WRITE) { 205 v3d_flush_jobs_reading_resource(v3d, prsc, 206 V3D_FLUSH_ALWAYS, 207 false); 208 } else { 209 v3d_flush_jobs_writing_resource(v3d, prsc, 210 V3D_FLUSH_ALWAYS, 211 false); 212 } 213 } 214 215 if (usage & PIPE_MAP_WRITE) { 216 rsc->writes++; 217 rsc->initialized_buffers = ~0; 218 } 219} 220 221static void * 222v3d_resource_transfer_map(struct pipe_context *pctx, 223 struct pipe_resource *prsc, 224 unsigned level, unsigned usage, 225 const struct pipe_box *box, 226 struct pipe_transfer **pptrans) 227{ 228 struct v3d_context *v3d = v3d_context(pctx); 229 struct v3d_resource *rsc = v3d_resource(prsc); 230 struct v3d_transfer *trans; 231 struct pipe_transfer *ptrans; 232 enum pipe_format format = prsc->format; 233 char *buf; 234 235 /* MSAA maps should have been handled by u_transfer_helper. */ 236 assert(prsc->nr_samples <= 1); 237 238 /* Upgrade DISCARD_RANGE to WHOLE_RESOURCE if the whole resource is 239 * being mapped. 240 */ 241 if ((usage & PIPE_MAP_DISCARD_RANGE) && 242 !(usage & PIPE_MAP_UNSYNCHRONIZED) && 243 !(prsc->flags & PIPE_RESOURCE_FLAG_MAP_PERSISTENT) && 244 prsc->last_level == 0 && 245 prsc->width0 == box->width && 246 prsc->height0 == box->height && 247 prsc->depth0 == box->depth && 248 prsc->array_size == 1 && 249 rsc->bo->private) { 250 usage |= PIPE_MAP_DISCARD_WHOLE_RESOURCE; 251 } 252 253 v3d_map_usage_prep(pctx, prsc, usage); 254 255 trans = slab_alloc(&v3d->transfer_pool); 256 if (!trans) 257 return NULL; 258 259 /* XXX: Handle DONTBLOCK, DISCARD_RANGE, PERSISTENT, COHERENT. */ 260 261 /* slab_alloc_st() doesn't zero: */ 262 memset(trans, 0, sizeof(*trans)); 263 ptrans = &trans->base; 264 265 pipe_resource_reference(&ptrans->resource, prsc); 266 ptrans->level = level; 267 ptrans->usage = usage; 268 ptrans->box = *box; 269 270 /* Note that the current kernel implementation is synchronous, so no 271 * need to do syncing stuff here yet. 272 */ 273 274 if (usage & PIPE_MAP_UNSYNCHRONIZED) 275 buf = v3d_bo_map_unsynchronized(rsc->bo); 276 else 277 buf = v3d_bo_map(rsc->bo); 278 if (!buf) { 279 fprintf(stderr, "Failed to map bo\n"); 280 goto fail; 281 } 282 283 *pptrans = ptrans; 284 285 /* Our load/store routines work on entire compressed blocks. */ 286 ptrans->box.x /= util_format_get_blockwidth(format); 287 ptrans->box.y /= util_format_get_blockheight(format); 288 ptrans->box.width = DIV_ROUND_UP(ptrans->box.width, 289 util_format_get_blockwidth(format)); 290 ptrans->box.height = DIV_ROUND_UP(ptrans->box.height, 291 util_format_get_blockheight(format)); 292 293 struct v3d_resource_slice *slice = &rsc->slices[level]; 294 if (rsc->tiled) { 295 /* No direct mappings of tiled, since we need to manually 296 * tile/untile. 297 */ 298 if (usage & PIPE_MAP_DIRECTLY) 299 return NULL; 300 301 ptrans->stride = ptrans->box.width * rsc->cpp; 302 ptrans->layer_stride = ptrans->stride * ptrans->box.height; 303 304 trans->map = malloc(ptrans->layer_stride * ptrans->box.depth); 305 306 if (usage & PIPE_MAP_READ) { 307 for (int z = 0; z < ptrans->box.depth; z++) { 308 void *src = rsc->bo->map + 309 v3d_layer_offset(&rsc->base, 310 ptrans->level, 311 ptrans->box.z + z); 312 v3d_load_tiled_image((trans->map + 313 ptrans->stride * 314 ptrans->box.height * z), 315 ptrans->stride, 316 src, 317 slice->stride, 318 slice->tiling, rsc->cpp, 319 slice->padded_height, 320 &ptrans->box); 321 } 322 } 323 return trans->map; 324 } else { 325 ptrans->stride = slice->stride; 326 ptrans->layer_stride = rsc->cube_map_stride; 327 328 return buf + slice->offset + 329 ptrans->box.y * ptrans->stride + 330 ptrans->box.x * rsc->cpp + 331 ptrans->box.z * rsc->cube_map_stride; 332 } 333 334 335fail: 336 v3d_resource_transfer_unmap(pctx, ptrans); 337 return NULL; 338} 339 340static void 341v3d_texture_subdata(struct pipe_context *pctx, 342 struct pipe_resource *prsc, 343 unsigned level, 344 unsigned usage, 345 const struct pipe_box *box, 346 const void *data, 347 unsigned stride, 348 unsigned layer_stride) 349{ 350 struct v3d_resource *rsc = v3d_resource(prsc); 351 struct v3d_resource_slice *slice = &rsc->slices[level]; 352 353 /* For a direct mapping, we can just take the u_transfer path. */ 354 if (!rsc->tiled) { 355 return u_default_texture_subdata(pctx, prsc, level, usage, box, 356 data, stride, layer_stride); 357 } 358 359 /* Otherwise, map and store the texture data directly into the tiled 360 * texture. Note that gallium's texture_subdata may be called with 361 * obvious usage flags missing! 362 */ 363 v3d_map_usage_prep(pctx, prsc, usage | (PIPE_MAP_WRITE | 364 PIPE_MAP_DISCARD_RANGE)); 365 366 void *buf; 367 if (usage & PIPE_MAP_UNSYNCHRONIZED) 368 buf = v3d_bo_map_unsynchronized(rsc->bo); 369 else 370 buf = v3d_bo_map(rsc->bo); 371 372 for (int i = 0; i < box->depth; i++) { 373 v3d_store_tiled_image(buf + 374 v3d_layer_offset(&rsc->base, 375 level, 376 box->z + i), 377 slice->stride, 378 (void *)data + layer_stride * i, 379 stride, 380 slice->tiling, rsc->cpp, slice->padded_height, 381 box); 382 } 383} 384 385static void 386v3d_resource_destroy(struct pipe_screen *pscreen, 387 struct pipe_resource *prsc) 388{ 389 struct v3d_screen *screen = v3d_screen(pscreen); 390 struct v3d_resource *rsc = v3d_resource(prsc); 391 392 if (rsc->scanout) 393 renderonly_scanout_destroy(rsc->scanout, screen->ro); 394 395 v3d_bo_unreference(&rsc->bo); 396 free(rsc); 397} 398 399static uint64_t 400v3d_resource_modifier(struct v3d_resource *rsc) 401{ 402 if (rsc->tiled) { 403 /* A shared tiled buffer should always be allocated as UIF, 404 * not UBLINEAR or LT. 405 */ 406 assert(rsc->slices[0].tiling == V3D_TILING_UIF_XOR || 407 rsc->slices[0].tiling == V3D_TILING_UIF_NO_XOR); 408 return DRM_FORMAT_MOD_BROADCOM_UIF; 409 } else { 410 return DRM_FORMAT_MOD_LINEAR; 411 } 412} 413 414static bool 415v3d_resource_get_handle(struct pipe_screen *pscreen, 416 struct pipe_context *pctx, 417 struct pipe_resource *prsc, 418 struct winsys_handle *whandle, 419 unsigned usage) 420{ 421 struct v3d_screen *screen = v3d_screen(pscreen); 422 struct v3d_resource *rsc = v3d_resource(prsc); 423 struct v3d_bo *bo = rsc->bo; 424 425 whandle->stride = rsc->slices[0].stride; 426 whandle->offset = 0; 427 whandle->modifier = v3d_resource_modifier(rsc); 428 429 /* If we're passing some reference to our BO out to some other part of 430 * the system, then we can't do any optimizations about only us being 431 * the ones seeing it (like BO caching). 432 */ 433 bo->private = false; 434 435 switch (whandle->type) { 436 case WINSYS_HANDLE_TYPE_SHARED: 437 return v3d_bo_flink(bo, &whandle->handle); 438 case WINSYS_HANDLE_TYPE_KMS: 439 if (screen->ro) { 440 if (renderonly_get_handle(rsc->scanout, whandle)) { 441 whandle->stride = rsc->slices[0].stride; 442 return true; 443 } 444 return false; 445 } 446 whandle->handle = bo->handle; 447 return true; 448 case WINSYS_HANDLE_TYPE_FD: 449 whandle->handle = v3d_bo_get_dmabuf(bo); 450 return whandle->handle != -1; 451 } 452 453 return false; 454} 455 456static bool 457v3d_resource_get_param(struct pipe_screen *pscreen, 458 struct pipe_context *pctx, struct pipe_resource *prsc, 459 unsigned plane, unsigned layer, unsigned level, 460 enum pipe_resource_param param, 461 unsigned usage, uint64_t *value) 462{ 463 struct v3d_resource *rsc = v3d_resource(prsc); 464 465 switch (param) { 466 case PIPE_RESOURCE_PARAM_STRIDE: 467 *value = rsc->slices[level].stride; 468 return true; 469 case PIPE_RESOURCE_PARAM_OFFSET: 470 *value = 0; 471 return true; 472 case PIPE_RESOURCE_PARAM_MODIFIER: 473 *value = v3d_resource_modifier(rsc); 474 return true; 475 default: 476 return false; 477 } 478} 479 480#define PAGE_UB_ROWS (V3D_UIFCFG_PAGE_SIZE / V3D_UIFBLOCK_ROW_SIZE) 481#define PAGE_UB_ROWS_TIMES_1_5 ((PAGE_UB_ROWS * 3) >> 1) 482#define PAGE_CACHE_UB_ROWS (V3D_PAGE_CACHE_SIZE / V3D_UIFBLOCK_ROW_SIZE) 483#define PAGE_CACHE_MINUS_1_5_UB_ROWS (PAGE_CACHE_UB_ROWS - PAGE_UB_ROWS_TIMES_1_5) 484 485/** 486 * Computes the HW's UIFblock padding for a given height/cpp. 487 * 488 * The goal of the padding is to keep pages of the same color (bank number) at 489 * least half a page away from each other vertically when crossing between 490 * between columns of UIF blocks. 491 */ 492static uint32_t 493v3d_get_ub_pad(struct v3d_resource *rsc, uint32_t height) 494{ 495 uint32_t utile_h = v3d_utile_height(rsc->cpp); 496 uint32_t uif_block_h = utile_h * 2; 497 uint32_t height_ub = height / uif_block_h; 498 499 uint32_t height_offset_in_pc = height_ub % PAGE_CACHE_UB_ROWS; 500 501 /* For the perfectly-aligned-for-UIF-XOR case, don't add any pad. */ 502 if (height_offset_in_pc == 0) 503 return 0; 504 505 /* Try padding up to where we're offset by at least half a page. */ 506 if (height_offset_in_pc < PAGE_UB_ROWS_TIMES_1_5) { 507 /* If we fit entirely in the page cache, don't pad. */ 508 if (height_ub < PAGE_CACHE_UB_ROWS) 509 return 0; 510 else 511 return PAGE_UB_ROWS_TIMES_1_5 - height_offset_in_pc; 512 } 513 514 /* If we're close to being aligned to page cache size, then round up 515 * and rely on XOR. 516 */ 517 if (height_offset_in_pc > PAGE_CACHE_MINUS_1_5_UB_ROWS) 518 return PAGE_CACHE_UB_ROWS - height_offset_in_pc; 519 520 /* Otherwise, we're far enough away (top and bottom) to not need any 521 * padding. 522 */ 523 return 0; 524} 525 526static void 527v3d_setup_slices(struct v3d_resource *rsc, uint32_t winsys_stride, 528 bool uif_top) 529{ 530 struct pipe_resource *prsc = &rsc->base; 531 uint32_t width = prsc->width0; 532 uint32_t height = prsc->height0; 533 uint32_t depth = prsc->depth0; 534 /* Note that power-of-two padding is based on level 1. These are not 535 * equivalent to just util_next_power_of_two(dimension), because at a 536 * level 0 dimension of 9, the level 1 power-of-two padded value is 4, 537 * not 8. 538 */ 539 uint32_t pot_width = 2 * util_next_power_of_two(u_minify(width, 1)); 540 uint32_t pot_height = 2 * util_next_power_of_two(u_minify(height, 1)); 541 uint32_t pot_depth = 2 * util_next_power_of_two(u_minify(depth, 1)); 542 uint32_t offset = 0; 543 uint32_t utile_w = v3d_utile_width(rsc->cpp); 544 uint32_t utile_h = v3d_utile_height(rsc->cpp); 545 uint32_t uif_block_w = utile_w * 2; 546 uint32_t uif_block_h = utile_h * 2; 547 uint32_t block_width = util_format_get_blockwidth(prsc->format); 548 uint32_t block_height = util_format_get_blockheight(prsc->format); 549 bool msaa = prsc->nr_samples > 1; 550 551 /* MSAA textures/renderbuffers are always laid out as single-level 552 * UIF. 553 */ 554 uif_top |= msaa; 555 556 /* Check some easy mistakes to make in a resource_create() call that 557 * will break our setup. 558 */ 559 assert(prsc->array_size != 0); 560 assert(prsc->depth0 != 0); 561 562 for (int i = prsc->last_level; i >= 0; i--) { 563 struct v3d_resource_slice *slice = &rsc->slices[i]; 564 565 uint32_t level_width, level_height, level_depth; 566 if (i < 2) { 567 level_width = u_minify(width, i); 568 level_height = u_minify(height, i); 569 } else { 570 level_width = u_minify(pot_width, i); 571 level_height = u_minify(pot_height, i); 572 } 573 if (i < 1) 574 level_depth = u_minify(depth, i); 575 else 576 level_depth = u_minify(pot_depth, i); 577 578 if (msaa) { 579 level_width *= 2; 580 level_height *= 2; 581 } 582 583 level_width = DIV_ROUND_UP(level_width, block_width); 584 level_height = DIV_ROUND_UP(level_height, block_height); 585 586 if (!rsc->tiled) { 587 slice->tiling = V3D_TILING_RASTER; 588 if (prsc->target == PIPE_TEXTURE_1D) 589 level_width = align(level_width, 64 / rsc->cpp); 590 } else { 591 if ((i != 0 || !uif_top) && 592 (level_width <= utile_w || 593 level_height <= utile_h)) { 594 slice->tiling = V3D_TILING_LINEARTILE; 595 level_width = align(level_width, utile_w); 596 level_height = align(level_height, utile_h); 597 } else if ((i != 0 || !uif_top) && 598 level_width <= uif_block_w) { 599 slice->tiling = V3D_TILING_UBLINEAR_1_COLUMN; 600 level_width = align(level_width, uif_block_w); 601 level_height = align(level_height, uif_block_h); 602 } else if ((i != 0 || !uif_top) && 603 level_width <= 2 * uif_block_w) { 604 slice->tiling = V3D_TILING_UBLINEAR_2_COLUMN; 605 level_width = align(level_width, 2 * uif_block_w); 606 level_height = align(level_height, uif_block_h); 607 } else { 608 /* We align the width to a 4-block column of 609 * UIF blocks, but we only align height to UIF 610 * blocks. 611 */ 612 level_width = align(level_width, 613 4 * uif_block_w); 614 level_height = align(level_height, 615 uif_block_h); 616 617 slice->ub_pad = v3d_get_ub_pad(rsc, 618 level_height); 619 level_height += slice->ub_pad * uif_block_h; 620 621 /* If the padding set us to to be aligned to 622 * the page cache size, then the HW will use 623 * the XOR bit on odd columns to get us 624 * perfectly misaligned 625 */ 626 if ((level_height / uif_block_h) % 627 (V3D_PAGE_CACHE_SIZE / 628 V3D_UIFBLOCK_ROW_SIZE) == 0) { 629 slice->tiling = V3D_TILING_UIF_XOR; 630 } else { 631 slice->tiling = V3D_TILING_UIF_NO_XOR; 632 } 633 } 634 } 635 636 slice->offset = offset; 637 if (winsys_stride) 638 slice->stride = winsys_stride; 639 else 640 slice->stride = level_width * rsc->cpp; 641 slice->padded_height = level_height; 642 slice->size = level_height * slice->stride; 643 644 uint32_t slice_total_size = slice->size * level_depth; 645 646 /* The HW aligns level 1's base to a page if any of level 1 or 647 * below could be UIF XOR. The lower levels then inherit the 648 * alignment for as long as necessary, thanks to being power of 649 * two aligned. 650 */ 651 if (i == 1 && 652 level_width > 4 * uif_block_w && 653 level_height > PAGE_CACHE_MINUS_1_5_UB_ROWS * uif_block_h) { 654 slice_total_size = align(slice_total_size, 655 V3D_UIFCFG_PAGE_SIZE); 656 } 657 658 offset += slice_total_size; 659 660 } 661 rsc->size = offset; 662 663 /* UIF/UBLINEAR levels need to be aligned to UIF-blocks, and LT only 664 * needs to be aligned to utile boundaries. Since tiles are laid out 665 * from small to big in memory, we need to align the later UIF slices 666 * to UIF blocks, if they were preceded by non-UIF-block-aligned LT 667 * slices. 668 * 669 * We additionally align to 4k, which improves UIF XOR performance. 670 */ 671 uint32_t page_align_offset = (align(rsc->slices[0].offset, 4096) - 672 rsc->slices[0].offset); 673 if (page_align_offset) { 674 rsc->size += page_align_offset; 675 for (int i = 0; i <= prsc->last_level; i++) 676 rsc->slices[i].offset += page_align_offset; 677 } 678 679 /* Arrays and cube textures have a stride which is the distance from 680 * one full mipmap tree to the next (64b aligned). For 3D textures, 681 * we need to program the stride between slices of miplevel 0. 682 */ 683 if (prsc->target != PIPE_TEXTURE_3D) { 684 rsc->cube_map_stride = align(rsc->slices[0].offset + 685 rsc->slices[0].size, 64); 686 rsc->size += rsc->cube_map_stride * (prsc->array_size - 1); 687 } else { 688 rsc->cube_map_stride = rsc->slices[0].size; 689 } 690} 691 692uint32_t 693v3d_layer_offset(struct pipe_resource *prsc, uint32_t level, uint32_t layer) 694{ 695 struct v3d_resource *rsc = v3d_resource(prsc); 696 struct v3d_resource_slice *slice = &rsc->slices[level]; 697 698 if (prsc->target == PIPE_TEXTURE_3D) 699 return slice->offset + layer * slice->size; 700 else 701 return slice->offset + layer * rsc->cube_map_stride; 702} 703 704static struct v3d_resource * 705v3d_resource_setup(struct pipe_screen *pscreen, 706 const struct pipe_resource *tmpl) 707{ 708 struct v3d_screen *screen = v3d_screen(pscreen); 709 struct v3d_resource *rsc = CALLOC_STRUCT(v3d_resource); 710 if (!rsc) 711 return NULL; 712 struct pipe_resource *prsc = &rsc->base; 713 714 *prsc = *tmpl; 715 716 pipe_reference_init(&prsc->reference, 1); 717 prsc->screen = pscreen; 718 719 if (prsc->nr_samples <= 1 || 720 screen->devinfo.ver >= 40 || 721 util_format_is_depth_or_stencil(prsc->format)) { 722 rsc->cpp = util_format_get_blocksize(prsc->format); 723 if (screen->devinfo.ver < 40 && prsc->nr_samples > 1) 724 rsc->cpp *= prsc->nr_samples; 725 } else { 726 assert(v3d_rt_format_supported(&screen->devinfo, prsc->format)); 727 uint32_t output_image_format = 728 v3d_get_rt_format(&screen->devinfo, prsc->format); 729 uint32_t internal_type; 730 uint32_t internal_bpp; 731 v3d_get_internal_type_bpp_for_output_format(&screen->devinfo, 732 output_image_format, 733 &internal_type, 734 &internal_bpp); 735 switch (internal_bpp) { 736 case V3D_INTERNAL_BPP_32: 737 rsc->cpp = 4; 738 break; 739 case V3D_INTERNAL_BPP_64: 740 rsc->cpp = 8; 741 break; 742 case V3D_INTERNAL_BPP_128: 743 rsc->cpp = 16; 744 break; 745 } 746 } 747 748 assert(rsc->cpp); 749 750 return rsc; 751} 752 753static struct pipe_resource * 754v3d_resource_create_with_modifiers(struct pipe_screen *pscreen, 755 const struct pipe_resource *tmpl, 756 const uint64_t *modifiers, 757 int count) 758{ 759 struct v3d_screen *screen = v3d_screen(pscreen); 760 761 bool linear_ok = drm_find_modifier(DRM_FORMAT_MOD_LINEAR, modifiers, count); 762 struct v3d_resource *rsc = v3d_resource_setup(pscreen, tmpl); 763 struct pipe_resource *prsc = &rsc->base; 764 /* Use a tiled layout if we can, for better 3D performance. */ 765 bool should_tile = true; 766 767 /* VBOs/PBOs are untiled (and 1 height). */ 768 if (tmpl->target == PIPE_BUFFER) 769 should_tile = false; 770 771 /* Cursors are always linear, and the user can request linear as well. 772 */ 773 if (tmpl->bind & (PIPE_BIND_LINEAR | PIPE_BIND_CURSOR)) 774 should_tile = false; 775 776 /* 1D and 1D_ARRAY textures are always raster-order. */ 777 if (tmpl->target == PIPE_TEXTURE_1D || 778 tmpl->target == PIPE_TEXTURE_1D_ARRAY) 779 should_tile = false; 780 781 /* Scanout BOs for simulator need to be linear for interaction with 782 * i965. 783 */ 784 if (using_v3d_simulator && 785 tmpl->bind & (PIPE_BIND_SHARED | PIPE_BIND_SCANOUT)) 786 should_tile = false; 787 788 /* If using the old-school SCANOUT flag, we don't know what the screen 789 * might support other than linear. Just force linear. 790 */ 791 if (tmpl->bind & PIPE_BIND_SCANOUT) 792 should_tile = false; 793 794 /* No user-specified modifier; determine our own. */ 795 if (count == 1 && modifiers[0] == DRM_FORMAT_MOD_INVALID) { 796 linear_ok = true; 797 rsc->tiled = should_tile; 798 } else if (should_tile && 799 drm_find_modifier(DRM_FORMAT_MOD_BROADCOM_UIF, 800 modifiers, count)) { 801 rsc->tiled = true; 802 } else if (linear_ok) { 803 rsc->tiled = false; 804 } else { 805 fprintf(stderr, "Unsupported modifier requested\n"); 806 goto fail; 807 } 808 809 rsc->internal_format = prsc->format; 810 811 v3d_setup_slices(rsc, 0, tmpl->bind & PIPE_BIND_SHARED); 812 813 /* If we're in a renderonly setup, use the other device to perform our 814 * allocation and just import it to v3d. The other device may be 815 * using CMA, and V3D can import from CMA but doesn't do CMA 816 * allocations on its own. 817 * 818 * We always allocate this way for SHARED, because get_handle will 819 * need a resource on the display fd. 820 */ 821 if (screen->ro && (tmpl->bind & (PIPE_BIND_SCANOUT | 822 PIPE_BIND_SHARED))) { 823 struct winsys_handle handle; 824 struct pipe_resource scanout_tmpl = { 825 .target = prsc->target, 826 .format = PIPE_FORMAT_RGBA8888_UNORM, 827 .width0 = 1024, /* one page */ 828 .height0 = align(rsc->size, 4096) / 4096, 829 .depth0 = 1, 830 .array_size = 1, 831 }; 832 833 rsc->scanout = 834 renderonly_scanout_for_resource(&scanout_tmpl, 835 screen->ro, 836 &handle); 837 838 if (!rsc->scanout) { 839 fprintf(stderr, "Failed to create scanout resource\n"); 840 goto fail; 841 } 842 assert(handle.type == WINSYS_HANDLE_TYPE_FD); 843 rsc->bo = v3d_bo_open_dmabuf(screen, handle.handle); 844 close(handle.handle); 845 846 if (!rsc->bo) 847 goto fail; 848 849 v3d_debug_resource_layout(rsc, "renderonly"); 850 851 return prsc; 852 } else { 853 if (!v3d_resource_bo_alloc(rsc)) 854 goto fail; 855 } 856 857 return prsc; 858fail: 859 v3d_resource_destroy(pscreen, prsc); 860 return NULL; 861} 862 863struct pipe_resource * 864v3d_resource_create(struct pipe_screen *pscreen, 865 const struct pipe_resource *tmpl) 866{ 867 const uint64_t mod = DRM_FORMAT_MOD_INVALID; 868 return v3d_resource_create_with_modifiers(pscreen, tmpl, &mod, 1); 869} 870 871static struct pipe_resource * 872v3d_resource_from_handle(struct pipe_screen *pscreen, 873 const struct pipe_resource *tmpl, 874 struct winsys_handle *whandle, 875 unsigned usage) 876{ 877 struct v3d_screen *screen = v3d_screen(pscreen); 878 struct v3d_resource *rsc = v3d_resource_setup(pscreen, tmpl); 879 struct pipe_resource *prsc = &rsc->base; 880 struct v3d_resource_slice *slice = &rsc->slices[0]; 881 882 if (!rsc) 883 return NULL; 884 885 switch (whandle->modifier) { 886 case DRM_FORMAT_MOD_LINEAR: 887 rsc->tiled = false; 888 break; 889 case DRM_FORMAT_MOD_BROADCOM_UIF: 890 rsc->tiled = true; 891 break; 892 case DRM_FORMAT_MOD_INVALID: 893 rsc->tiled = screen->ro == NULL; 894 break; 895 default: 896 switch(fourcc_mod_broadcom_mod(whandle->modifier)) { 897 case DRM_FORMAT_MOD_BROADCOM_SAND128: 898 rsc->tiled = false; 899 rsc->sand_col128_stride = 900 fourcc_mod_broadcom_param(whandle->modifier); 901 break; 902 default: 903 fprintf(stderr, 904 "Attempt to import unsupported modifier 0x%llx\n", 905 (long long)whandle->modifier); 906 goto fail; 907 } 908 } 909 910 switch (whandle->type) { 911 case WINSYS_HANDLE_TYPE_SHARED: 912 rsc->bo = v3d_bo_open_name(screen, whandle->handle); 913 break; 914 case WINSYS_HANDLE_TYPE_FD: 915 rsc->bo = v3d_bo_open_dmabuf(screen, whandle->handle); 916 break; 917 default: 918 fprintf(stderr, 919 "Attempt to import unsupported handle type %d\n", 920 whandle->type); 921 goto fail; 922 } 923 924 if (!rsc->bo) 925 goto fail; 926 927 rsc->internal_format = prsc->format; 928 929 v3d_setup_slices(rsc, whandle->stride, true); 930 v3d_debug_resource_layout(rsc, "import"); 931 932 if (whandle->offset != 0) { 933 if (rsc->tiled) { 934 fprintf(stderr, 935 "Attempt to import unsupported winsys offset %u\n", 936 whandle->offset); 937 goto fail; 938 } 939 rsc->slices[0].offset += whandle->offset; 940 941 if (rsc->slices[0].offset + rsc->slices[0].size > 942 rsc->bo->size) { 943 fprintf(stderr, "Attempt to import " 944 "with overflowing offset (%d + %d > %d)\n", 945 whandle->offset, 946 rsc->slices[0].size, 947 rsc->bo->size); 948 goto fail; 949 } 950 } 951 952 if (screen->ro) { 953 /* Make sure that renderonly has a handle to our buffer in the 954 * display's fd, so that a later renderonly_get_handle() 955 * returns correct handles or GEM names. 956 */ 957 rsc->scanout = 958 renderonly_create_gpu_import_for_resource(prsc, 959 screen->ro, 960 NULL); 961 } 962 963 if (rsc->tiled && whandle->stride != slice->stride) { 964 static bool warned = false; 965 if (!warned) { 966 warned = true; 967 fprintf(stderr, 968 "Attempting to import %dx%d %s with " 969 "unsupported stride %d instead of %d\n", 970 prsc->width0, prsc->height0, 971 util_format_short_name(prsc->format), 972 whandle->stride, 973 slice->stride); 974 } 975 goto fail; 976 } else if (!rsc->tiled) { 977 slice->stride = whandle->stride; 978 } 979 980 return prsc; 981 982fail: 983 v3d_resource_destroy(pscreen, prsc); 984 return NULL; 985} 986 987void 988v3d_update_shadow_texture(struct pipe_context *pctx, 989 struct pipe_sampler_view *pview) 990{ 991 struct v3d_context *v3d = v3d_context(pctx); 992 struct v3d_sampler_view *view = v3d_sampler_view(pview); 993 struct v3d_resource *shadow = v3d_resource(view->texture); 994 struct v3d_resource *orig = v3d_resource(pview->texture); 995 996 assert(view->texture != pview->texture); 997 998 if (shadow->writes == orig->writes && orig->bo->private) 999 return; 1000 1001 perf_debug("Updating %dx%d@%d shadow for linear texture\n", 1002 orig->base.width0, orig->base.height0, 1003 pview->u.tex.first_level); 1004 1005 for (int i = 0; i <= shadow->base.last_level; i++) { 1006 unsigned width = u_minify(shadow->base.width0, i); 1007 unsigned height = u_minify(shadow->base.height0, i); 1008 struct pipe_blit_info info = { 1009 .dst = { 1010 .resource = &shadow->base, 1011 .level = i, 1012 .box = { 1013 .x = 0, 1014 .y = 0, 1015 .z = 0, 1016 .width = width, 1017 .height = height, 1018 .depth = 1, 1019 }, 1020 .format = shadow->base.format, 1021 }, 1022 .src = { 1023 .resource = &orig->base, 1024 .level = pview->u.tex.first_level + i, 1025 .box = { 1026 .x = 0, 1027 .y = 0, 1028 .z = 0, 1029 .width = width, 1030 .height = height, 1031 .depth = 1, 1032 }, 1033 .format = orig->base.format, 1034 }, 1035 .mask = util_format_get_mask(orig->base.format), 1036 }; 1037 pctx->blit(pctx, &info); 1038 } 1039 1040 shadow->writes = orig->writes; 1041} 1042 1043static struct pipe_surface * 1044v3d_create_surface(struct pipe_context *pctx, 1045 struct pipe_resource *ptex, 1046 const struct pipe_surface *surf_tmpl) 1047{ 1048 struct v3d_context *v3d = v3d_context(pctx); 1049 struct v3d_screen *screen = v3d->screen; 1050 struct v3d_surface *surface = CALLOC_STRUCT(v3d_surface); 1051 struct v3d_resource *rsc = v3d_resource(ptex); 1052 1053 if (!surface) 1054 return NULL; 1055 1056 struct pipe_surface *psurf = &surface->base; 1057 unsigned level = surf_tmpl->u.tex.level; 1058 struct v3d_resource_slice *slice = &rsc->slices[level]; 1059 1060 pipe_reference_init(&psurf->reference, 1); 1061 pipe_resource_reference(&psurf->texture, ptex); 1062 1063 psurf->context = pctx; 1064 psurf->format = surf_tmpl->format; 1065 psurf->width = u_minify(ptex->width0, level); 1066 psurf->height = u_minify(ptex->height0, level); 1067 psurf->u.tex.level = level; 1068 psurf->u.tex.first_layer = surf_tmpl->u.tex.first_layer; 1069 psurf->u.tex.last_layer = surf_tmpl->u.tex.last_layer; 1070 1071 surface->offset = v3d_layer_offset(ptex, level, 1072 psurf->u.tex.first_layer); 1073 surface->tiling = slice->tiling; 1074 1075 surface->format = v3d_get_rt_format(&screen->devinfo, psurf->format); 1076 1077 const struct util_format_description *desc = 1078 util_format_description(psurf->format); 1079 1080 surface->swap_rb = (desc->swizzle[0] == PIPE_SWIZZLE_Z && 1081 psurf->format != PIPE_FORMAT_B5G6R5_UNORM); 1082 1083 if (util_format_is_depth_or_stencil(psurf->format)) { 1084 switch (psurf->format) { 1085 case PIPE_FORMAT_Z16_UNORM: 1086 surface->internal_type = V3D_INTERNAL_TYPE_DEPTH_16; 1087 break; 1088 case PIPE_FORMAT_Z32_FLOAT: 1089 case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT: 1090 surface->internal_type = V3D_INTERNAL_TYPE_DEPTH_32F; 1091 break; 1092 default: 1093 surface->internal_type = V3D_INTERNAL_TYPE_DEPTH_24; 1094 } 1095 } else { 1096 uint32_t bpp, type; 1097 v3d_get_internal_type_bpp_for_output_format(&screen->devinfo, 1098 surface->format, 1099 &type, &bpp); 1100 surface->internal_type = type; 1101 surface->internal_bpp = bpp; 1102 } 1103 1104 if (surface->tiling == V3D_TILING_UIF_NO_XOR || 1105 surface->tiling == V3D_TILING_UIF_XOR) { 1106 surface->padded_height_of_output_image_in_uif_blocks = 1107 (slice->padded_height / 1108 (2 * v3d_utile_height(rsc->cpp))); 1109 } 1110 1111 if (rsc->separate_stencil) { 1112 surface->separate_stencil = 1113 v3d_create_surface(pctx, &rsc->separate_stencil->base, 1114 surf_tmpl); 1115 } 1116 1117 return &surface->base; 1118} 1119 1120static void 1121v3d_surface_destroy(struct pipe_context *pctx, struct pipe_surface *psurf) 1122{ 1123 struct v3d_surface *surf = v3d_surface(psurf); 1124 1125 if (surf->separate_stencil) 1126 pipe_surface_reference(&surf->separate_stencil, NULL); 1127 1128 pipe_resource_reference(&psurf->texture, NULL); 1129 FREE(psurf); 1130} 1131 1132static void 1133v3d_flush_resource(struct pipe_context *pctx, struct pipe_resource *resource) 1134{ 1135 /* All calls to flush_resource are followed by a flush of the context, 1136 * so there's nothing to do. 1137 */ 1138} 1139 1140static enum pipe_format 1141v3d_resource_get_internal_format(struct pipe_resource *prsc) 1142{ 1143 return v3d_resource(prsc)->internal_format; 1144} 1145 1146static void 1147v3d_resource_set_stencil(struct pipe_resource *prsc, 1148 struct pipe_resource *stencil) 1149{ 1150 v3d_resource(prsc)->separate_stencil = v3d_resource(stencil); 1151} 1152 1153static struct pipe_resource * 1154v3d_resource_get_stencil(struct pipe_resource *prsc) 1155{ 1156 struct v3d_resource *rsc = v3d_resource(prsc); 1157 1158 return &rsc->separate_stencil->base; 1159} 1160 1161static const struct u_transfer_vtbl transfer_vtbl = { 1162 .resource_create = v3d_resource_create, 1163 .resource_destroy = v3d_resource_destroy, 1164 .transfer_map = v3d_resource_transfer_map, 1165 .transfer_unmap = v3d_resource_transfer_unmap, 1166 .transfer_flush_region = u_default_transfer_flush_region, 1167 .get_internal_format = v3d_resource_get_internal_format, 1168 .set_stencil = v3d_resource_set_stencil, 1169 .get_stencil = v3d_resource_get_stencil, 1170}; 1171 1172void 1173v3d_resource_screen_init(struct pipe_screen *pscreen) 1174{ 1175 pscreen->resource_create_with_modifiers = 1176 v3d_resource_create_with_modifiers; 1177 pscreen->resource_create = u_transfer_helper_resource_create; 1178 pscreen->resource_from_handle = v3d_resource_from_handle; 1179 pscreen->resource_get_handle = v3d_resource_get_handle; 1180 pscreen->resource_get_param = v3d_resource_get_param; 1181 pscreen->resource_destroy = u_transfer_helper_resource_destroy; 1182 pscreen->transfer_helper = u_transfer_helper_create(&transfer_vtbl, 1183 true, false, 1184 true, true); 1185} 1186 1187void 1188v3d_resource_context_init(struct pipe_context *pctx) 1189{ 1190 pctx->buffer_map = u_transfer_helper_transfer_map; 1191 pctx->texture_map = u_transfer_helper_transfer_map; 1192 pctx->transfer_flush_region = u_transfer_helper_transfer_flush_region; 1193 pctx->buffer_unmap = u_transfer_helper_transfer_unmap; 1194 pctx->texture_unmap = u_transfer_helper_transfer_unmap; 1195 pctx->buffer_subdata = u_default_buffer_subdata; 1196 pctx->texture_subdata = v3d_texture_subdata; 1197 pctx->create_surface = v3d_create_surface; 1198 pctx->surface_destroy = v3d_surface_destroy; 1199 pctx->resource_copy_region = util_resource_copy_region; 1200 pctx->blit = v3d_blit; 1201 pctx->generate_mipmap = v3d_generate_mipmap; 1202 pctx->flush_resource = v3d_flush_resource; 1203} 1204