1/* 2 * Copyright (C) 2017 Rob Clark <robclark@freedesktop.org> 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 * SOFTWARE. 22 * 23 * Authors: 24 * Rob Clark <robclark@freedesktop.org> 25 */ 26 27#include "freedreno_blitter.h" 28#include "freedreno_resource.h" 29 30#include "fd5_blitter.h" 31#include "fd5_format.h" 32#include "fd5_emit.h" 33 34/* Make sure none of the requested dimensions extend beyond the size of the 35 * resource. Not entirely sure why this happens, but sometimes it does, and 36 * w/ 2d blt doesn't have wrap modes like a sampler, so force those cases 37 * back to u_blitter 38 */ 39static bool 40ok_dims(const struct pipe_resource *r, const struct pipe_box *b, int lvl) 41{ 42 return (b->x >= 0) && (b->x + b->width <= u_minify(r->width0, lvl)) && 43 (b->y >= 0) && (b->y + b->height <= u_minify(r->height0, lvl)) && 44 (b->z >= 0) && (b->z + b->depth <= u_minify(r->depth0, lvl)); 45} 46 47/* Not sure if format restrictions differ for src and dst, or if 48 * they only matter when src fmt != dst fmt.. but there appear to 49 * be *some* limitations so let's just start blacklisting stuff that 50 * piglit complains about 51 */ 52static bool 53ok_format(enum pipe_format fmt) 54{ 55 if (util_format_is_compressed(fmt)) 56 return false; 57 58 switch (fmt) { 59 case PIPE_FORMAT_R10G10B10A2_SSCALED: 60 case PIPE_FORMAT_R10G10B10A2_SNORM: 61 case PIPE_FORMAT_B10G10R10A2_USCALED: 62 case PIPE_FORMAT_B10G10R10A2_SSCALED: 63 case PIPE_FORMAT_B10G10R10A2_SNORM: 64 case PIPE_FORMAT_R10G10B10A2_UNORM: 65 case PIPE_FORMAT_R10G10B10A2_USCALED: 66 case PIPE_FORMAT_B10G10R10A2_UNORM: 67 case PIPE_FORMAT_R10SG10SB10SA2U_NORM: 68 case PIPE_FORMAT_B10G10R10A2_UINT: 69 case PIPE_FORMAT_R10G10B10A2_UINT: 70 return false; 71 default: 72 break; 73 } 74 75 if (fd5_pipe2color(fmt) == ~0) 76 return false; 77 78 return true; 79} 80 81static bool 82can_do_blit(const struct pipe_blit_info *info) 83{ 84 /* I think we can do scaling, but not in z dimension since that would 85 * require blending.. 86 */ 87 if (info->dst.box.depth != info->src.box.depth) 88 return false; 89 90 if (!ok_format(info->dst.format)) 91 return false; 92 93 if (!ok_format(info->src.format)) 94 return false; 95 96 /* hw ignores {SRC,DST}_INFO.COLOR_SWAP if {SRC,DST}_INFO.TILE_MODE 97 * is set (not linear). We can kind of get around that when tiling/ 98 * untiling by setting both src and dst COLOR_SWAP=WZYX, but that 99 * means the formats must match: 100 */ 101 if ((fd_resource(info->dst.resource)->tile_mode || 102 fd_resource(info->src.resource)->tile_mode) && 103 info->dst.format != info->src.format) 104 return false; 105 106 /* until we figure out a few more registers: */ 107 if ((info->dst.box.width != info->src.box.width) || 108 (info->dst.box.height != info->src.box.height)) 109 return false; 110 111 /* src box can be inverted, which we don't support.. dst box cannot: */ 112 if ((info->src.box.width < 0) || (info->src.box.height < 0)) 113 return false; 114 115 if (!ok_dims(info->src.resource, &info->src.box, info->src.level)) 116 return false; 117 118 if (!ok_dims(info->dst.resource, &info->dst.box, info->dst.level)) 119 return false; 120 121 debug_assert(info->dst.box.width >= 0); 122 debug_assert(info->dst.box.height >= 0); 123 debug_assert(info->dst.box.depth >= 0); 124 125 if ((info->dst.resource->nr_samples > 1) || 126 (info->src.resource->nr_samples > 1)) 127 return false; 128 129 if (info->scissor_enable) 130 return false; 131 132 if (info->window_rectangle_include) 133 return false; 134 135 if (info->render_condition_enable) 136 return false; 137 138 if (info->alpha_blend) 139 return false; 140 141 if (info->filter != PIPE_TEX_FILTER_NEAREST) 142 return false; 143 144 if (info->mask != util_format_get_mask(info->src.format)) 145 return false; 146 147 if (info->mask != util_format_get_mask(info->dst.format)) 148 return false; 149 150 return true; 151} 152 153static void 154emit_setup(struct fd_ringbuffer *ring) 155{ 156 OUT_PKT7(ring, CP_EVENT_WRITE, 1); 157 OUT_RING(ring, LRZ_FLUSH); 158 159 OUT_PKT7(ring, CP_SKIP_IB2_ENABLE_GLOBAL, 1); 160 OUT_RING(ring, 0x0); 161 162 OUT_PKT4(ring, REG_A5XX_PC_POWER_CNTL, 1); 163 OUT_RING(ring, 0x00000003); /* PC_POWER_CNTL */ 164 165 OUT_PKT4(ring, REG_A5XX_VFD_POWER_CNTL, 1); 166 OUT_RING(ring, 0x00000003); /* VFD_POWER_CNTL */ 167 168 /* 0x10000000 for BYPASS.. 0x7c13c080 for GMEM: */ 169 OUT_WFI5(ring); 170 OUT_PKT4(ring, REG_A5XX_RB_CCU_CNTL, 1); 171 OUT_RING(ring, 0x10000000); /* RB_CCU_CNTL */ 172 173 OUT_PKT4(ring, REG_A5XX_RB_RENDER_CNTL, 1); 174 OUT_RING(ring, 0x00000008); 175 176 OUT_PKT4(ring, REG_A5XX_UNKNOWN_2100, 1); 177 OUT_RING(ring, 0x86000000); /* UNKNOWN_2100 */ 178 179 OUT_PKT4(ring, REG_A5XX_UNKNOWN_2180, 1); 180 OUT_RING(ring, 0x86000000); /* UNKNOWN_2180 */ 181 182 OUT_PKT4(ring, REG_A5XX_UNKNOWN_2184, 1); 183 OUT_RING(ring, 0x00000009); /* UNKNOWN_2184 */ 184 185 OUT_PKT4(ring, REG_A5XX_RB_CNTL, 1); 186 OUT_RING(ring, A5XX_RB_CNTL_BYPASS); 187 188 OUT_PKT4(ring, REG_A5XX_RB_MODE_CNTL, 1); 189 OUT_RING(ring, 0x00000004); /* RB_MODE_CNTL */ 190 191 OUT_PKT4(ring, REG_A5XX_SP_MODE_CNTL, 1); 192 OUT_RING(ring, 0x0000000c); /* SP_MODE_CNTL */ 193 194 OUT_PKT4(ring, REG_A5XX_TPL1_MODE_CNTL, 1); 195 OUT_RING(ring, 0x00000344); /* TPL1_MODE_CNTL */ 196 197 OUT_PKT4(ring, REG_A5XX_HLSQ_MODE_CNTL, 1); 198 OUT_RING(ring, 0x00000002); /* HLSQ_MODE_CNTL */ 199 200 OUT_PKT4(ring, REG_A5XX_GRAS_CL_CNTL, 1); 201 OUT_RING(ring, 0x00000181); /* GRAS_CL_CNTL */ 202} 203 204/* buffers need to be handled specially since x/width can exceed the bounds 205 * supported by hw.. if necessary decompose into (potentially) two 2D blits 206 */ 207static void 208emit_blit_buffer(struct fd_ringbuffer *ring, const struct pipe_blit_info *info) 209{ 210 const struct pipe_box *sbox = &info->src.box; 211 const struct pipe_box *dbox = &info->dst.box; 212 struct fd_resource *src, *dst; 213 unsigned sshift, dshift; 214 215 src = fd_resource(info->src.resource); 216 dst = fd_resource(info->dst.resource); 217 218 debug_assert(src->cpp == 1); 219 debug_assert(dst->cpp == 1); 220 debug_assert(info->src.resource->format == info->dst.resource->format); 221 debug_assert((sbox->y == 0) && (sbox->height == 1)); 222 debug_assert((dbox->y == 0) && (dbox->height == 1)); 223 debug_assert((sbox->z == 0) && (sbox->depth == 1)); 224 debug_assert((dbox->z == 0) && (dbox->depth == 1)); 225 debug_assert(sbox->width == dbox->width); 226 debug_assert(info->src.level == 0); 227 debug_assert(info->dst.level == 0); 228 229 /* 230 * Buffers can have dimensions bigger than max width, remap into 231 * multiple 1d blits to fit within max dimension 232 * 233 * Note that blob uses .ARRAY_PITCH=128 for blitting buffers, which 234 * seems to prevent overfetch related faults. Not quite sure what 235 * the deal is there. 236 * 237 * Low 6 bits of SRC/DST addresses need to be zero (ie. address 238 * aligned to 64) so we need to shift src/dst x1/x2 to make up the 239 * difference. On top of already splitting up the blit so width 240 * isn't > 16k. 241 * 242 * We perhaps could do a bit better, if src and dst are aligned but 243 * in the worst case this means we have to split the copy up into 244 * 16k (0x4000) minus 64 (0x40). 245 */ 246 247 sshift = sbox->x & 0x3f; 248 dshift = dbox->x & 0x3f; 249 250 for (unsigned off = 0; off < sbox->width; off += (0x4000 - 0x40)) { 251 unsigned soff, doff, w, p; 252 253 soff = (sbox->x + off) & ~0x3f; 254 doff = (dbox->x + off) & ~0x3f; 255 256 w = MIN2(sbox->width - off, (0x4000 - 0x40)); 257 p = align(w, 64); 258 259 debug_assert((soff + w) <= fd_bo_size(src->bo)); 260 debug_assert((doff + w) <= fd_bo_size(dst->bo)); 261 262 OUT_PKT7(ring, CP_SET_RENDER_MODE, 1); 263 OUT_RING(ring, CP_SET_RENDER_MODE_0_MODE(BLIT2D)); 264 265 /* 266 * Emit source: 267 */ 268 OUT_PKT4(ring, REG_A5XX_RB_2D_SRC_INFO, 9); 269 OUT_RING(ring, A5XX_RB_2D_SRC_INFO_COLOR_FORMAT(RB5_R8_UNORM) | 270 A5XX_RB_2D_SRC_INFO_TILE_MODE(TILE5_LINEAR) | 271 A5XX_RB_2D_SRC_INFO_COLOR_SWAP(WZYX)); 272 OUT_RELOC(ring, src->bo, soff, 0, 0); /* RB_2D_SRC_LO/HI */ 273 OUT_RING(ring, A5XX_RB_2D_SRC_SIZE_PITCH(p) | 274 A5XX_RB_2D_SRC_SIZE_ARRAY_PITCH(128)); 275 OUT_RING(ring, 0x00000000); 276 OUT_RING(ring, 0x00000000); 277 OUT_RING(ring, 0x00000000); 278 OUT_RING(ring, 0x00000000); 279 OUT_RING(ring, 0x00000000); 280 281 OUT_PKT4(ring, REG_A5XX_GRAS_2D_SRC_INFO, 1); 282 OUT_RING(ring, A5XX_GRAS_2D_SRC_INFO_COLOR_FORMAT(RB5_R8_UNORM) | 283 A5XX_GRAS_2D_SRC_INFO_COLOR_SWAP(WZYX)); 284 285 /* 286 * Emit destination: 287 */ 288 OUT_PKT4(ring, REG_A5XX_RB_2D_DST_INFO, 9); 289 OUT_RING(ring, A5XX_RB_2D_DST_INFO_COLOR_FORMAT(RB5_R8_UNORM) | 290 A5XX_RB_2D_DST_INFO_TILE_MODE(TILE5_LINEAR) | 291 A5XX_RB_2D_DST_INFO_COLOR_SWAP(WZYX)); 292 OUT_RELOCW(ring, dst->bo, doff, 0, 0); /* RB_2D_DST_LO/HI */ 293 OUT_RING(ring, A5XX_RB_2D_DST_SIZE_PITCH(p) | 294 A5XX_RB_2D_DST_SIZE_ARRAY_PITCH(128)); 295 OUT_RING(ring, 0x00000000); 296 OUT_RING(ring, 0x00000000); 297 OUT_RING(ring, 0x00000000); 298 OUT_RING(ring, 0x00000000); 299 OUT_RING(ring, 0x00000000); 300 301 OUT_PKT4(ring, REG_A5XX_GRAS_2D_DST_INFO, 1); 302 OUT_RING(ring, A5XX_GRAS_2D_DST_INFO_COLOR_FORMAT(RB5_R8_UNORM) | 303 A5XX_GRAS_2D_DST_INFO_COLOR_SWAP(WZYX)); 304 305 /* 306 * Blit command: 307 */ 308 OUT_PKT7(ring, CP_BLIT, 5); 309 OUT_RING(ring, CP_BLIT_0_OP(BLIT_OP_COPY)); 310 OUT_RING(ring, CP_BLIT_1_SRC_X1(sshift) | CP_BLIT_1_SRC_Y1(0)); 311 OUT_RING(ring, CP_BLIT_2_SRC_X2(sshift+w-1) | CP_BLIT_2_SRC_Y2(0)); 312 OUT_RING(ring, CP_BLIT_3_DST_X1(dshift) | CP_BLIT_3_DST_Y1(0)); 313 OUT_RING(ring, CP_BLIT_4_DST_X2(dshift+w-1) | CP_BLIT_4_DST_Y2(0)); 314 315 OUT_PKT7(ring, CP_SET_RENDER_MODE, 1); 316 OUT_RING(ring, CP_SET_RENDER_MODE_0_MODE(END2D)); 317 318 OUT_WFI5(ring); 319 } 320} 321 322static void 323emit_blit(struct fd_ringbuffer *ring, const struct pipe_blit_info *info) 324{ 325 const struct pipe_box *sbox = &info->src.box; 326 const struct pipe_box *dbox = &info->dst.box; 327 struct fd_resource *src, *dst; 328 struct fd_resource_slice *sslice, *dslice; 329 enum a5xx_color_fmt sfmt, dfmt; 330 enum a5xx_tile_mode stile, dtile; 331 enum a3xx_color_swap sswap, dswap; 332 unsigned ssize, dsize, spitch, dpitch; 333 unsigned sx1, sy1, sx2, sy2; 334 unsigned dx1, dy1, dx2, dy2; 335 336 src = fd_resource(info->src.resource); 337 dst = fd_resource(info->dst.resource); 338 339 sslice = fd_resource_slice(src, info->src.level); 340 dslice = fd_resource_slice(dst, info->dst.level); 341 342 sfmt = fd5_pipe2color(info->src.format); 343 dfmt = fd5_pipe2color(info->dst.format); 344 345 stile = fd_resource_level_linear(info->src.resource, info->src.level) ? 346 TILE5_LINEAR : src->tile_mode; 347 dtile = fd_resource_level_linear(info->dst.resource, info->dst.level) ? 348 TILE5_LINEAR : dst->tile_mode; 349 350 sswap = fd5_pipe2swap(info->src.format); 351 dswap = fd5_pipe2swap(info->dst.format); 352 353 spitch = sslice->pitch * src->cpp; 354 dpitch = dslice->pitch * dst->cpp; 355 356 /* if dtile, then dswap ignored by hw, and likewise if stile then sswap 357 * ignored by hw.. but in this case we have already rejected the blit 358 * if src and dst formats differ, so juse use WZYX for both src and 359 * dst swap mode (so we don't change component order) 360 */ 361 if (stile || dtile) { 362 debug_assert(info->src.format == info->dst.format); 363 sswap = dswap = WZYX; 364 } 365 366 sx1 = sbox->x; 367 sy1 = sbox->y; 368 sx2 = sbox->x + sbox->width - 1; 369 sy2 = sbox->y + sbox->height - 1; 370 371 dx1 = dbox->x; 372 dy1 = dbox->y; 373 dx2 = dbox->x + dbox->width - 1; 374 dy2 = dbox->y + dbox->height - 1; 375 376 if (info->src.resource->target == PIPE_TEXTURE_3D) 377 ssize = sslice->size0; 378 else 379 ssize = src->layer_size; 380 381 if (info->dst.resource->target == PIPE_TEXTURE_3D) 382 dsize = dslice->size0; 383 else 384 dsize = dst->layer_size; 385 386 for (unsigned i = 0; i < info->dst.box.depth; i++) { 387 unsigned soff = fd_resource_offset(src, info->src.level, sbox->z + i); 388 unsigned doff = fd_resource_offset(dst, info->dst.level, dbox->z + i); 389 390 debug_assert((soff + (sbox->height * spitch)) <= fd_bo_size(src->bo)); 391 debug_assert((doff + (dbox->height * dpitch)) <= fd_bo_size(dst->bo)); 392 393 OUT_PKT7(ring, CP_SET_RENDER_MODE, 1); 394 OUT_RING(ring, CP_SET_RENDER_MODE_0_MODE(BLIT2D)); 395 396 /* 397 * Emit source: 398 */ 399 OUT_PKT4(ring, REG_A5XX_RB_2D_SRC_INFO, 9); 400 OUT_RING(ring, A5XX_RB_2D_SRC_INFO_COLOR_FORMAT(sfmt) | 401 A5XX_RB_2D_SRC_INFO_TILE_MODE(stile) | 402 A5XX_RB_2D_SRC_INFO_COLOR_SWAP(sswap)); 403 OUT_RELOC(ring, src->bo, soff, 0, 0); /* RB_2D_SRC_LO/HI */ 404 OUT_RING(ring, A5XX_RB_2D_SRC_SIZE_PITCH(spitch) | 405 A5XX_RB_2D_SRC_SIZE_ARRAY_PITCH(ssize)); 406 OUT_RING(ring, 0x00000000); 407 OUT_RING(ring, 0x00000000); 408 OUT_RING(ring, 0x00000000); 409 OUT_RING(ring, 0x00000000); 410 OUT_RING(ring, 0x00000000); 411 412 OUT_PKT4(ring, REG_A5XX_GRAS_2D_SRC_INFO, 1); 413 OUT_RING(ring, A5XX_GRAS_2D_SRC_INFO_COLOR_FORMAT(sfmt) | 414 A5XX_GRAS_2D_SRC_INFO_TILE_MODE(stile) | 415 A5XX_GRAS_2D_SRC_INFO_COLOR_SWAP(sswap)); 416 417 /* 418 * Emit destination: 419 */ 420 OUT_PKT4(ring, REG_A5XX_RB_2D_DST_INFO, 9); 421 OUT_RING(ring, A5XX_RB_2D_DST_INFO_COLOR_FORMAT(dfmt) | 422 A5XX_RB_2D_DST_INFO_TILE_MODE(dtile) | 423 A5XX_RB_2D_DST_INFO_COLOR_SWAP(dswap)); 424 OUT_RELOCW(ring, dst->bo, doff, 0, 0); /* RB_2D_DST_LO/HI */ 425 OUT_RING(ring, A5XX_RB_2D_DST_SIZE_PITCH(dpitch) | 426 A5XX_RB_2D_DST_SIZE_ARRAY_PITCH(dsize)); 427 OUT_RING(ring, 0x00000000); 428 OUT_RING(ring, 0x00000000); 429 OUT_RING(ring, 0x00000000); 430 OUT_RING(ring, 0x00000000); 431 OUT_RING(ring, 0x00000000); 432 433 OUT_PKT4(ring, REG_A5XX_GRAS_2D_DST_INFO, 1); 434 OUT_RING(ring, A5XX_GRAS_2D_DST_INFO_COLOR_FORMAT(dfmt) | 435 A5XX_GRAS_2D_DST_INFO_TILE_MODE(dtile) | 436 A5XX_GRAS_2D_DST_INFO_COLOR_SWAP(dswap)); 437 438 /* 439 * Blit command: 440 */ 441 OUT_PKT7(ring, CP_BLIT, 5); 442 OUT_RING(ring, CP_BLIT_0_OP(BLIT_OP_COPY)); 443 OUT_RING(ring, CP_BLIT_1_SRC_X1(sx1) | CP_BLIT_1_SRC_Y1(sy1)); 444 OUT_RING(ring, CP_BLIT_2_SRC_X2(sx2) | CP_BLIT_2_SRC_Y2(sy2)); 445 OUT_RING(ring, CP_BLIT_3_DST_X1(dx1) | CP_BLIT_3_DST_Y1(dy1)); 446 OUT_RING(ring, CP_BLIT_4_DST_X2(dx2) | CP_BLIT_4_DST_Y2(dy2)); 447 448 OUT_PKT7(ring, CP_SET_RENDER_MODE, 1); 449 OUT_RING(ring, CP_SET_RENDER_MODE_0_MODE(END2D)); 450 } 451} 452 453bool 454fd5_blitter_blit(struct fd_context *ctx, const struct pipe_blit_info *info) 455{ 456 struct fd_batch *batch; 457 458 if (!can_do_blit(info)) { 459 return false; 460 } 461 462 batch = fd_bc_alloc_batch(&ctx->screen->batch_cache, ctx, true); 463 464 fd5_emit_restore(batch, batch->draw); 465 fd5_emit_lrz_flush(batch->draw); 466 467 emit_setup(batch->draw); 468 469 if ((info->src.resource->target == PIPE_BUFFER) && 470 (info->dst.resource->target == PIPE_BUFFER)) { 471 assert(fd_resource(info->src.resource)->tile_mode == TILE5_LINEAR); 472 assert(fd_resource(info->dst.resource)->tile_mode == TILE5_LINEAR); 473 emit_blit_buffer(batch->draw, info); 474 } else { 475 /* I don't *think* we need to handle blits between buffer <-> !buffer */ 476 debug_assert(info->src.resource->target != PIPE_BUFFER); 477 debug_assert(info->dst.resource->target != PIPE_BUFFER); 478 emit_blit(batch->draw, info); 479 } 480 481 fd_resource(info->dst.resource)->valid = true; 482 batch->needs_flush = true; 483 484 fd_batch_flush(batch, false, false); 485 fd_batch_reference(&batch, NULL); 486 487 return true; 488} 489 490unsigned 491fd5_tile_mode(const struct pipe_resource *tmpl) 492{ 493 /* basically just has to be a format we can blit, so uploads/downloads 494 * via linear staging buffer works: 495 */ 496 if (ok_format(tmpl->format)) 497 return TILE5_3; 498 499 return TILE5_LINEAR; 500} 501