radeon_evergreen_cs.c revision 1.1.1.1
1 /* $NetBSD: radeon_evergreen_cs.c,v 1.1.1.1 2021/12/18 20:15:48 riastradh Exp $ */ 2 3 /* 4 * Copyright 2010 Advanced Micro Devices, Inc. 5 * Copyright 2008 Red Hat Inc. 6 * Copyright 2009 Jerome Glisse. 7 * 8 * Permission is hereby granted, free of charge, to any person obtaining a 9 * copy of this software and associated documentation files (the "Software"), 10 * to deal in the Software without restriction, including without limitation 11 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 12 * and/or sell copies of the Software, and to permit persons to whom the 13 * Software is furnished to do so, subject to the following conditions: 14 * 15 * The above copyright notice and this permission notice shall be included in 16 * all copies or substantial portions of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 21 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 22 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 23 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 24 * OTHER DEALINGS IN THE SOFTWARE. 25 * 26 * Authors: Dave Airlie 27 * Alex Deucher 28 * Jerome Glisse 29 */ 30 31 #include <sys/cdefs.h> 32 __KERNEL_RCSID(0, "$NetBSD: radeon_evergreen_cs.c,v 1.1.1.1 2021/12/18 20:15:48 riastradh Exp $"); 33 34 #include "radeon.h" 35 #include "radeon_asic.h" 36 #include "evergreend.h" 37 #include "evergreen_reg_safe.h" 38 #include "cayman_reg_safe.h" 39 40 #define MAX(a,b) (((a)>(b))?(a):(b)) 41 #define MIN(a,b) (((a)<(b))?(a):(b)) 42 43 #define REG_SAFE_BM_SIZE ARRAY_SIZE(evergreen_reg_safe_bm) 44 45 int r600_dma_cs_next_reloc(struct radeon_cs_parser *p, 46 struct radeon_bo_list **cs_reloc); 47 struct evergreen_cs_track { 48 u32 group_size; 49 u32 nbanks; 50 u32 npipes; 51 u32 row_size; 52 /* value we track */ 53 u32 nsamples; /* unused */ 54 struct radeon_bo *cb_color_bo[12]; 55 u32 cb_color_bo_offset[12]; 56 struct radeon_bo *cb_color_fmask_bo[8]; /* unused */ 57 struct radeon_bo *cb_color_cmask_bo[8]; /* unused */ 58 u32 cb_color_info[12]; 59 u32 cb_color_view[12]; 60 u32 cb_color_pitch[12]; 61 u32 cb_color_slice[12]; 62 u32 cb_color_slice_idx[12]; 63 u32 cb_color_attrib[12]; 64 u32 cb_color_cmask_slice[8];/* unused */ 65 u32 cb_color_fmask_slice[8];/* unused */ 66 u32 cb_target_mask; 67 u32 cb_shader_mask; /* unused */ 68 u32 vgt_strmout_config; 69 u32 vgt_strmout_buffer_config; 70 struct radeon_bo *vgt_strmout_bo[4]; 71 u32 vgt_strmout_bo_offset[4]; 72 u32 vgt_strmout_size[4]; 73 u32 db_depth_control; 74 u32 db_depth_view; 75 u32 db_depth_slice; 76 u32 db_depth_size; 77 u32 db_z_info; 78 u32 db_z_read_offset; 79 u32 db_z_write_offset; 80 struct radeon_bo *db_z_read_bo; 81 struct radeon_bo *db_z_write_bo; 82 u32 db_s_info; 83 u32 db_s_read_offset; 84 u32 db_s_write_offset; 85 struct radeon_bo *db_s_read_bo; 86 struct radeon_bo *db_s_write_bo; 87 bool sx_misc_kill_all_prims; 88 bool cb_dirty; 89 bool db_dirty; 90 bool streamout_dirty; 91 u32 htile_offset; 92 u32 htile_surface; 93 struct radeon_bo *htile_bo; 94 unsigned long indirect_draw_buffer_size; 95 const unsigned *reg_safe_bm; 96 }; 97 98 static u32 evergreen_cs_get_aray_mode(u32 tiling_flags) 99 { 100 if (tiling_flags & RADEON_TILING_MACRO) 101 return ARRAY_2D_TILED_THIN1; 102 else if (tiling_flags & RADEON_TILING_MICRO) 103 return ARRAY_1D_TILED_THIN1; 104 else 105 return ARRAY_LINEAR_GENERAL; 106 } 107 108 static u32 evergreen_cs_get_num_banks(u32 nbanks) 109 { 110 switch (nbanks) { 111 case 2: 112 return ADDR_SURF_2_BANK; 113 case 4: 114 return ADDR_SURF_4_BANK; 115 case 8: 116 default: 117 return ADDR_SURF_8_BANK; 118 case 16: 119 return ADDR_SURF_16_BANK; 120 } 121 } 122 123 static void evergreen_cs_track_init(struct evergreen_cs_track *track) 124 { 125 int i; 126 127 for (i = 0; i < 8; i++) { 128 track->cb_color_fmask_bo[i] = NULL; 129 track->cb_color_cmask_bo[i] = NULL; 130 track->cb_color_cmask_slice[i] = 0; 131 track->cb_color_fmask_slice[i] = 0; 132 } 133 134 for (i = 0; i < 12; i++) { 135 track->cb_color_bo[i] = NULL; 136 track->cb_color_bo_offset[i] = 0xFFFFFFFF; 137 track->cb_color_info[i] = 0; 138 track->cb_color_view[i] = 0xFFFFFFFF; 139 track->cb_color_pitch[i] = 0; 140 track->cb_color_slice[i] = 0xfffffff; 141 track->cb_color_slice_idx[i] = 0; 142 } 143 track->cb_target_mask = 0xFFFFFFFF; 144 track->cb_shader_mask = 0xFFFFFFFF; 145 track->cb_dirty = true; 146 147 track->db_depth_slice = 0xffffffff; 148 track->db_depth_view = 0xFFFFC000; 149 track->db_depth_size = 0xFFFFFFFF; 150 track->db_depth_control = 0xFFFFFFFF; 151 track->db_z_info = 0xFFFFFFFF; 152 track->db_z_read_offset = 0xFFFFFFFF; 153 track->db_z_write_offset = 0xFFFFFFFF; 154 track->db_z_read_bo = NULL; 155 track->db_z_write_bo = NULL; 156 track->db_s_info = 0xFFFFFFFF; 157 track->db_s_read_offset = 0xFFFFFFFF; 158 track->db_s_write_offset = 0xFFFFFFFF; 159 track->db_s_read_bo = NULL; 160 track->db_s_write_bo = NULL; 161 track->db_dirty = true; 162 track->htile_bo = NULL; 163 track->htile_offset = 0xFFFFFFFF; 164 track->htile_surface = 0; 165 166 for (i = 0; i < 4; i++) { 167 track->vgt_strmout_size[i] = 0; 168 track->vgt_strmout_bo[i] = NULL; 169 track->vgt_strmout_bo_offset[i] = 0xFFFFFFFF; 170 } 171 track->streamout_dirty = true; 172 track->sx_misc_kill_all_prims = false; 173 } 174 175 struct eg_surface { 176 /* value gathered from cs */ 177 unsigned nbx; 178 unsigned nby; 179 unsigned format; 180 unsigned mode; 181 unsigned nbanks; 182 unsigned bankw; 183 unsigned bankh; 184 unsigned tsplit; 185 unsigned mtilea; 186 unsigned nsamples; 187 /* output value */ 188 unsigned bpe; 189 unsigned layer_size; 190 unsigned palign; 191 unsigned halign; 192 unsigned long base_align; 193 }; 194 195 static int evergreen_surface_check_linear(struct radeon_cs_parser *p, 196 struct eg_surface *surf, 197 const char *prefix) 198 { 199 surf->layer_size = surf->nbx * surf->nby * surf->bpe * surf->nsamples; 200 surf->base_align = surf->bpe; 201 surf->palign = 1; 202 surf->halign = 1; 203 return 0; 204 } 205 206 static int evergreen_surface_check_linear_aligned(struct radeon_cs_parser *p, 207 struct eg_surface *surf, 208 const char *prefix) 209 { 210 struct evergreen_cs_track *track = p->track; 211 unsigned palign; 212 213 palign = MAX(64, track->group_size / surf->bpe); 214 surf->layer_size = surf->nbx * surf->nby * surf->bpe * surf->nsamples; 215 surf->base_align = track->group_size; 216 surf->palign = palign; 217 surf->halign = 1; 218 if (surf->nbx & (palign - 1)) { 219 if (prefix) { 220 dev_warn(p->dev, "%s:%d %s pitch %d invalid must be aligned with %d\n", 221 __func__, __LINE__, prefix, surf->nbx, palign); 222 } 223 return -EINVAL; 224 } 225 return 0; 226 } 227 228 static int evergreen_surface_check_1d(struct radeon_cs_parser *p, 229 struct eg_surface *surf, 230 const char *prefix) 231 { 232 struct evergreen_cs_track *track = p->track; 233 unsigned palign; 234 235 palign = track->group_size / (8 * surf->bpe * surf->nsamples); 236 palign = MAX(8, palign); 237 surf->layer_size = surf->nbx * surf->nby * surf->bpe; 238 surf->base_align = track->group_size; 239 surf->palign = palign; 240 surf->halign = 8; 241 if ((surf->nbx & (palign - 1))) { 242 if (prefix) { 243 dev_warn(p->dev, "%s:%d %s pitch %d invalid must be aligned with %d (%d %d %d)\n", 244 __func__, __LINE__, prefix, surf->nbx, palign, 245 track->group_size, surf->bpe, surf->nsamples); 246 } 247 return -EINVAL; 248 } 249 if ((surf->nby & (8 - 1))) { 250 if (prefix) { 251 dev_warn(p->dev, "%s:%d %s height %d invalid must be aligned with 8\n", 252 __func__, __LINE__, prefix, surf->nby); 253 } 254 return -EINVAL; 255 } 256 return 0; 257 } 258 259 static int evergreen_surface_check_2d(struct radeon_cs_parser *p, 260 struct eg_surface *surf, 261 const char *prefix) 262 { 263 struct evergreen_cs_track *track = p->track; 264 unsigned palign, halign, tileb, slice_pt; 265 unsigned mtile_pr, mtile_ps, mtileb; 266 267 tileb = 64 * surf->bpe * surf->nsamples; 268 slice_pt = 1; 269 if (tileb > surf->tsplit) { 270 slice_pt = tileb / surf->tsplit; 271 } 272 tileb = tileb / slice_pt; 273 /* macro tile width & height */ 274 palign = (8 * surf->bankw * track->npipes) * surf->mtilea; 275 halign = (8 * surf->bankh * surf->nbanks) / surf->mtilea; 276 mtileb = (palign / 8) * (halign / 8) * tileb; 277 mtile_pr = surf->nbx / palign; 278 mtile_ps = (mtile_pr * surf->nby) / halign; 279 surf->layer_size = mtile_ps * mtileb * slice_pt; 280 surf->base_align = (palign / 8) * (halign / 8) * tileb; 281 surf->palign = palign; 282 surf->halign = halign; 283 284 if ((surf->nbx & (palign - 1))) { 285 if (prefix) { 286 dev_warn(p->dev, "%s:%d %s pitch %d invalid must be aligned with %d\n", 287 __func__, __LINE__, prefix, surf->nbx, palign); 288 } 289 return -EINVAL; 290 } 291 if ((surf->nby & (halign - 1))) { 292 if (prefix) { 293 dev_warn(p->dev, "%s:%d %s height %d invalid must be aligned with %d\n", 294 __func__, __LINE__, prefix, surf->nby, halign); 295 } 296 return -EINVAL; 297 } 298 299 return 0; 300 } 301 302 static int evergreen_surface_check(struct radeon_cs_parser *p, 303 struct eg_surface *surf, 304 const char *prefix) 305 { 306 /* some common value computed here */ 307 surf->bpe = r600_fmt_get_blocksize(surf->format); 308 309 switch (surf->mode) { 310 case ARRAY_LINEAR_GENERAL: 311 return evergreen_surface_check_linear(p, surf, prefix); 312 case ARRAY_LINEAR_ALIGNED: 313 return evergreen_surface_check_linear_aligned(p, surf, prefix); 314 case ARRAY_1D_TILED_THIN1: 315 return evergreen_surface_check_1d(p, surf, prefix); 316 case ARRAY_2D_TILED_THIN1: 317 return evergreen_surface_check_2d(p, surf, prefix); 318 default: 319 dev_warn(p->dev, "%s:%d %s invalid array mode %d\n", 320 __func__, __LINE__, prefix, surf->mode); 321 return -EINVAL; 322 } 323 return -EINVAL; 324 } 325 326 static int evergreen_surface_value_conv_check(struct radeon_cs_parser *p, 327 struct eg_surface *surf, 328 const char *prefix) 329 { 330 switch (surf->mode) { 331 case ARRAY_2D_TILED_THIN1: 332 break; 333 case ARRAY_LINEAR_GENERAL: 334 case ARRAY_LINEAR_ALIGNED: 335 case ARRAY_1D_TILED_THIN1: 336 return 0; 337 default: 338 dev_warn(p->dev, "%s:%d %s invalid array mode %d\n", 339 __func__, __LINE__, prefix, surf->mode); 340 return -EINVAL; 341 } 342 343 switch (surf->nbanks) { 344 case 0: surf->nbanks = 2; break; 345 case 1: surf->nbanks = 4; break; 346 case 2: surf->nbanks = 8; break; 347 case 3: surf->nbanks = 16; break; 348 default: 349 dev_warn(p->dev, "%s:%d %s invalid number of banks %d\n", 350 __func__, __LINE__, prefix, surf->nbanks); 351 return -EINVAL; 352 } 353 switch (surf->bankw) { 354 case 0: surf->bankw = 1; break; 355 case 1: surf->bankw = 2; break; 356 case 2: surf->bankw = 4; break; 357 case 3: surf->bankw = 8; break; 358 default: 359 dev_warn(p->dev, "%s:%d %s invalid bankw %d\n", 360 __func__, __LINE__, prefix, surf->bankw); 361 return -EINVAL; 362 } 363 switch (surf->bankh) { 364 case 0: surf->bankh = 1; break; 365 case 1: surf->bankh = 2; break; 366 case 2: surf->bankh = 4; break; 367 case 3: surf->bankh = 8; break; 368 default: 369 dev_warn(p->dev, "%s:%d %s invalid bankh %d\n", 370 __func__, __LINE__, prefix, surf->bankh); 371 return -EINVAL; 372 } 373 switch (surf->mtilea) { 374 case 0: surf->mtilea = 1; break; 375 case 1: surf->mtilea = 2; break; 376 case 2: surf->mtilea = 4; break; 377 case 3: surf->mtilea = 8; break; 378 default: 379 dev_warn(p->dev, "%s:%d %s invalid macro tile aspect %d\n", 380 __func__, __LINE__, prefix, surf->mtilea); 381 return -EINVAL; 382 } 383 switch (surf->tsplit) { 384 case 0: surf->tsplit = 64; break; 385 case 1: surf->tsplit = 128; break; 386 case 2: surf->tsplit = 256; break; 387 case 3: surf->tsplit = 512; break; 388 case 4: surf->tsplit = 1024; break; 389 case 5: surf->tsplit = 2048; break; 390 case 6: surf->tsplit = 4096; break; 391 default: 392 dev_warn(p->dev, "%s:%d %s invalid tile split %d\n", 393 __func__, __LINE__, prefix, surf->tsplit); 394 return -EINVAL; 395 } 396 return 0; 397 } 398 399 static int evergreen_cs_track_validate_cb(struct radeon_cs_parser *p, unsigned id) 400 { 401 struct evergreen_cs_track *track = p->track; 402 struct eg_surface surf; 403 unsigned pitch, slice, mslice; 404 unsigned long offset; 405 int r; 406 407 mslice = G_028C6C_SLICE_MAX(track->cb_color_view[id]) + 1; 408 pitch = track->cb_color_pitch[id]; 409 slice = track->cb_color_slice[id]; 410 surf.nbx = (pitch + 1) * 8; 411 surf.nby = ((slice + 1) * 64) / surf.nbx; 412 surf.mode = G_028C70_ARRAY_MODE(track->cb_color_info[id]); 413 surf.format = G_028C70_FORMAT(track->cb_color_info[id]); 414 surf.tsplit = G_028C74_TILE_SPLIT(track->cb_color_attrib[id]); 415 surf.nbanks = G_028C74_NUM_BANKS(track->cb_color_attrib[id]); 416 surf.bankw = G_028C74_BANK_WIDTH(track->cb_color_attrib[id]); 417 surf.bankh = G_028C74_BANK_HEIGHT(track->cb_color_attrib[id]); 418 surf.mtilea = G_028C74_MACRO_TILE_ASPECT(track->cb_color_attrib[id]); 419 surf.nsamples = 1; 420 421 if (!r600_fmt_is_valid_color(surf.format)) { 422 dev_warn(p->dev, "%s:%d cb invalid format %d for %d (0x%08x)\n", 423 __func__, __LINE__, surf.format, 424 id, track->cb_color_info[id]); 425 return -EINVAL; 426 } 427 428 r = evergreen_surface_value_conv_check(p, &surf, "cb"); 429 if (r) { 430 return r; 431 } 432 433 r = evergreen_surface_check(p, &surf, "cb"); 434 if (r) { 435 dev_warn(p->dev, "%s:%d cb[%d] invalid (0x%08x 0x%08x 0x%08x 0x%08x)\n", 436 __func__, __LINE__, id, track->cb_color_pitch[id], 437 track->cb_color_slice[id], track->cb_color_attrib[id], 438 track->cb_color_info[id]); 439 return r; 440 } 441 442 offset = track->cb_color_bo_offset[id] << 8; 443 if (offset & (surf.base_align - 1)) { 444 dev_warn(p->dev, "%s:%d cb[%d] bo base %ld not aligned with %ld\n", 445 __func__, __LINE__, id, offset, surf.base_align); 446 return -EINVAL; 447 } 448 449 offset += surf.layer_size * mslice; 450 if (offset > radeon_bo_size(track->cb_color_bo[id])) { 451 /* old ddx are broken they allocate bo with w*h*bpp but 452 * program slice with ALIGN(h, 8), catch this and patch 453 * command stream. 454 */ 455 if (!surf.mode) { 456 uint32_t *ib = p->ib.ptr; 457 unsigned long tmp, nby, bsize, size, min = 0; 458 459 /* find the height the ddx wants */ 460 if (surf.nby > 8) { 461 min = surf.nby - 8; 462 } 463 bsize = radeon_bo_size(track->cb_color_bo[id]); 464 tmp = track->cb_color_bo_offset[id] << 8; 465 for (nby = surf.nby; nby > min; nby--) { 466 size = nby * surf.nbx * surf.bpe * surf.nsamples; 467 if ((tmp + size * mslice) <= bsize) { 468 break; 469 } 470 } 471 if (nby > min) { 472 surf.nby = nby; 473 slice = ((nby * surf.nbx) / 64) - 1; 474 if (!evergreen_surface_check(p, &surf, "cb")) { 475 /* check if this one works */ 476 tmp += surf.layer_size * mslice; 477 if (tmp <= bsize) { 478 ib[track->cb_color_slice_idx[id]] = slice; 479 goto old_ddx_ok; 480 } 481 } 482 } 483 } 484 dev_warn(p->dev, "%s:%d cb[%d] bo too small (layer size %d, " 485 "offset %d, max layer %d, bo size %ld, slice %d)\n", 486 __func__, __LINE__, id, surf.layer_size, 487 track->cb_color_bo_offset[id] << 8, mslice, 488 radeon_bo_size(track->cb_color_bo[id]), slice); 489 dev_warn(p->dev, "%s:%d problematic surf: (%d %d) (%d %d %d %d %d %d %d)\n", 490 __func__, __LINE__, surf.nbx, surf.nby, 491 surf.mode, surf.bpe, surf.nsamples, 492 surf.bankw, surf.bankh, 493 surf.tsplit, surf.mtilea); 494 return -EINVAL; 495 } 496 old_ddx_ok: 497 498 return 0; 499 } 500 501 static int evergreen_cs_track_validate_htile(struct radeon_cs_parser *p, 502 unsigned nbx, unsigned nby) 503 { 504 struct evergreen_cs_track *track = p->track; 505 unsigned long size; 506 507 if (track->htile_bo == NULL) { 508 dev_warn(p->dev, "%s:%d htile enabled without htile surface 0x%08x\n", 509 __func__, __LINE__, track->db_z_info); 510 return -EINVAL; 511 } 512 513 if (G_028ABC_LINEAR(track->htile_surface)) { 514 /* pitch must be 16 htiles aligned == 16 * 8 pixel aligned */ 515 nbx = round_up(nbx, 16 * 8); 516 /* height is npipes htiles aligned == npipes * 8 pixel aligned */ 517 nby = round_up(nby, track->npipes * 8); 518 } else { 519 /* always assume 8x8 htile */ 520 /* align is htile align * 8, htile align vary according to 521 * number of pipe and tile width and nby 522 */ 523 switch (track->npipes) { 524 case 8: 525 /* HTILE_WIDTH = 8 & HTILE_HEIGHT = 8*/ 526 nbx = round_up(nbx, 64 * 8); 527 nby = round_up(nby, 64 * 8); 528 break; 529 case 4: 530 /* HTILE_WIDTH = 8 & HTILE_HEIGHT = 8*/ 531 nbx = round_up(nbx, 64 * 8); 532 nby = round_up(nby, 32 * 8); 533 break; 534 case 2: 535 /* HTILE_WIDTH = 8 & HTILE_HEIGHT = 8*/ 536 nbx = round_up(nbx, 32 * 8); 537 nby = round_up(nby, 32 * 8); 538 break; 539 case 1: 540 /* HTILE_WIDTH = 8 & HTILE_HEIGHT = 8*/ 541 nbx = round_up(nbx, 32 * 8); 542 nby = round_up(nby, 16 * 8); 543 break; 544 default: 545 dev_warn(p->dev, "%s:%d invalid num pipes %d\n", 546 __func__, __LINE__, track->npipes); 547 return -EINVAL; 548 } 549 } 550 /* compute number of htile */ 551 nbx = nbx >> 3; 552 nby = nby >> 3; 553 /* size must be aligned on npipes * 2K boundary */ 554 size = roundup(nbx * nby * 4, track->npipes * (2 << 10)); 555 size += track->htile_offset; 556 557 if (size > radeon_bo_size(track->htile_bo)) { 558 dev_warn(p->dev, "%s:%d htile surface too small %ld for %ld (%d %d)\n", 559 __func__, __LINE__, radeon_bo_size(track->htile_bo), 560 size, nbx, nby); 561 return -EINVAL; 562 } 563 return 0; 564 } 565 566 static int evergreen_cs_track_validate_stencil(struct radeon_cs_parser *p) 567 { 568 struct evergreen_cs_track *track = p->track; 569 struct eg_surface surf; 570 unsigned pitch, slice, mslice; 571 unsigned long offset; 572 int r; 573 574 mslice = G_028008_SLICE_MAX(track->db_depth_view) + 1; 575 pitch = G_028058_PITCH_TILE_MAX(track->db_depth_size); 576 slice = track->db_depth_slice; 577 surf.nbx = (pitch + 1) * 8; 578 surf.nby = ((slice + 1) * 64) / surf.nbx; 579 surf.mode = G_028040_ARRAY_MODE(track->db_z_info); 580 surf.format = G_028044_FORMAT(track->db_s_info); 581 surf.tsplit = G_028044_TILE_SPLIT(track->db_s_info); 582 surf.nbanks = G_028040_NUM_BANKS(track->db_z_info); 583 surf.bankw = G_028040_BANK_WIDTH(track->db_z_info); 584 surf.bankh = G_028040_BANK_HEIGHT(track->db_z_info); 585 surf.mtilea = G_028040_MACRO_TILE_ASPECT(track->db_z_info); 586 surf.nsamples = 1; 587 588 if (surf.format != 1) { 589 dev_warn(p->dev, "%s:%d stencil invalid format %d\n", 590 __func__, __LINE__, surf.format); 591 return -EINVAL; 592 } 593 /* replace by color format so we can use same code */ 594 surf.format = V_028C70_COLOR_8; 595 596 r = evergreen_surface_value_conv_check(p, &surf, "stencil"); 597 if (r) { 598 return r; 599 } 600 601 r = evergreen_surface_check(p, &surf, NULL); 602 if (r) { 603 /* old userspace doesn't compute proper depth/stencil alignment 604 * check that alignment against a bigger byte per elements and 605 * only report if that alignment is wrong too. 606 */ 607 surf.format = V_028C70_COLOR_8_8_8_8; 608 r = evergreen_surface_check(p, &surf, "stencil"); 609 if (r) { 610 dev_warn(p->dev, "%s:%d stencil invalid (0x%08x 0x%08x 0x%08x 0x%08x)\n", 611 __func__, __LINE__, track->db_depth_size, 612 track->db_depth_slice, track->db_s_info, track->db_z_info); 613 } 614 return r; 615 } 616 617 offset = track->db_s_read_offset << 8; 618 if (offset & (surf.base_align - 1)) { 619 dev_warn(p->dev, "%s:%d stencil read bo base %ld not aligned with %ld\n", 620 __func__, __LINE__, offset, surf.base_align); 621 return -EINVAL; 622 } 623 offset += surf.layer_size * mslice; 624 if (offset > radeon_bo_size(track->db_s_read_bo)) { 625 dev_warn(p->dev, "%s:%d stencil read bo too small (layer size %d, " 626 "offset %ld, max layer %d, bo size %ld)\n", 627 __func__, __LINE__, surf.layer_size, 628 (unsigned long)track->db_s_read_offset << 8, mslice, 629 radeon_bo_size(track->db_s_read_bo)); 630 dev_warn(p->dev, "%s:%d stencil invalid (0x%08x 0x%08x 0x%08x 0x%08x)\n", 631 __func__, __LINE__, track->db_depth_size, 632 track->db_depth_slice, track->db_s_info, track->db_z_info); 633 return -EINVAL; 634 } 635 636 offset = track->db_s_write_offset << 8; 637 if (offset & (surf.base_align - 1)) { 638 dev_warn(p->dev, "%s:%d stencil write bo base %ld not aligned with %ld\n", 639 __func__, __LINE__, offset, surf.base_align); 640 return -EINVAL; 641 } 642 offset += surf.layer_size * mslice; 643 if (offset > radeon_bo_size(track->db_s_write_bo)) { 644 dev_warn(p->dev, "%s:%d stencil write bo too small (layer size %d, " 645 "offset %ld, max layer %d, bo size %ld)\n", 646 __func__, __LINE__, surf.layer_size, 647 (unsigned long)track->db_s_write_offset << 8, mslice, 648 radeon_bo_size(track->db_s_write_bo)); 649 return -EINVAL; 650 } 651 652 /* hyperz */ 653 if (G_028040_TILE_SURFACE_ENABLE(track->db_z_info)) { 654 r = evergreen_cs_track_validate_htile(p, surf.nbx, surf.nby); 655 if (r) { 656 return r; 657 } 658 } 659 660 return 0; 661 } 662 663 static int evergreen_cs_track_validate_depth(struct radeon_cs_parser *p) 664 { 665 struct evergreen_cs_track *track = p->track; 666 struct eg_surface surf; 667 unsigned pitch, slice, mslice; 668 unsigned long offset; 669 int r; 670 671 mslice = G_028008_SLICE_MAX(track->db_depth_view) + 1; 672 pitch = G_028058_PITCH_TILE_MAX(track->db_depth_size); 673 slice = track->db_depth_slice; 674 surf.nbx = (pitch + 1) * 8; 675 surf.nby = ((slice + 1) * 64) / surf.nbx; 676 surf.mode = G_028040_ARRAY_MODE(track->db_z_info); 677 surf.format = G_028040_FORMAT(track->db_z_info); 678 surf.tsplit = G_028040_TILE_SPLIT(track->db_z_info); 679 surf.nbanks = G_028040_NUM_BANKS(track->db_z_info); 680 surf.bankw = G_028040_BANK_WIDTH(track->db_z_info); 681 surf.bankh = G_028040_BANK_HEIGHT(track->db_z_info); 682 surf.mtilea = G_028040_MACRO_TILE_ASPECT(track->db_z_info); 683 surf.nsamples = 1; 684 685 switch (surf.format) { 686 case V_028040_Z_16: 687 surf.format = V_028C70_COLOR_16; 688 break; 689 case V_028040_Z_24: 690 case V_028040_Z_32_FLOAT: 691 surf.format = V_028C70_COLOR_8_8_8_8; 692 break; 693 default: 694 dev_warn(p->dev, "%s:%d depth invalid format %d\n", 695 __func__, __LINE__, surf.format); 696 return -EINVAL; 697 } 698 699 r = evergreen_surface_value_conv_check(p, &surf, "depth"); 700 if (r) { 701 dev_warn(p->dev, "%s:%d depth invalid (0x%08x 0x%08x 0x%08x)\n", 702 __func__, __LINE__, track->db_depth_size, 703 track->db_depth_slice, track->db_z_info); 704 return r; 705 } 706 707 r = evergreen_surface_check(p, &surf, "depth"); 708 if (r) { 709 dev_warn(p->dev, "%s:%d depth invalid (0x%08x 0x%08x 0x%08x)\n", 710 __func__, __LINE__, track->db_depth_size, 711 track->db_depth_slice, track->db_z_info); 712 return r; 713 } 714 715 offset = track->db_z_read_offset << 8; 716 if (offset & (surf.base_align - 1)) { 717 dev_warn(p->dev, "%s:%d stencil read bo base %ld not aligned with %ld\n", 718 __func__, __LINE__, offset, surf.base_align); 719 return -EINVAL; 720 } 721 offset += surf.layer_size * mslice; 722 if (offset > radeon_bo_size(track->db_z_read_bo)) { 723 dev_warn(p->dev, "%s:%d depth read bo too small (layer size %d, " 724 "offset %ld, max layer %d, bo size %ld)\n", 725 __func__, __LINE__, surf.layer_size, 726 (unsigned long)track->db_z_read_offset << 8, mslice, 727 radeon_bo_size(track->db_z_read_bo)); 728 return -EINVAL; 729 } 730 731 offset = track->db_z_write_offset << 8; 732 if (offset & (surf.base_align - 1)) { 733 dev_warn(p->dev, "%s:%d stencil write bo base %ld not aligned with %ld\n", 734 __func__, __LINE__, offset, surf.base_align); 735 return -EINVAL; 736 } 737 offset += surf.layer_size * mslice; 738 if (offset > radeon_bo_size(track->db_z_write_bo)) { 739 dev_warn(p->dev, "%s:%d depth write bo too small (layer size %d, " 740 "offset %ld, max layer %d, bo size %ld)\n", 741 __func__, __LINE__, surf.layer_size, 742 (unsigned long)track->db_z_write_offset << 8, mslice, 743 radeon_bo_size(track->db_z_write_bo)); 744 return -EINVAL; 745 } 746 747 /* hyperz */ 748 if (G_028040_TILE_SURFACE_ENABLE(track->db_z_info)) { 749 r = evergreen_cs_track_validate_htile(p, surf.nbx, surf.nby); 750 if (r) { 751 return r; 752 } 753 } 754 755 return 0; 756 } 757 758 static int evergreen_cs_track_validate_texture(struct radeon_cs_parser *p, 759 struct radeon_bo *texture, 760 struct radeon_bo *mipmap, 761 unsigned idx) 762 { 763 struct eg_surface surf; 764 unsigned long toffset, moffset; 765 unsigned dim, llevel, mslice, width, height, depth, i; 766 u32 texdw[8]; 767 int r; 768 769 texdw[0] = radeon_get_ib_value(p, idx + 0); 770 texdw[1] = radeon_get_ib_value(p, idx + 1); 771 texdw[2] = radeon_get_ib_value(p, idx + 2); 772 texdw[3] = radeon_get_ib_value(p, idx + 3); 773 texdw[4] = radeon_get_ib_value(p, idx + 4); 774 texdw[5] = radeon_get_ib_value(p, idx + 5); 775 texdw[6] = radeon_get_ib_value(p, idx + 6); 776 texdw[7] = radeon_get_ib_value(p, idx + 7); 777 dim = G_030000_DIM(texdw[0]); 778 llevel = G_030014_LAST_LEVEL(texdw[5]); 779 mslice = G_030014_LAST_ARRAY(texdw[5]) + 1; 780 width = G_030000_TEX_WIDTH(texdw[0]) + 1; 781 height = G_030004_TEX_HEIGHT(texdw[1]) + 1; 782 depth = G_030004_TEX_DEPTH(texdw[1]) + 1; 783 surf.format = G_03001C_DATA_FORMAT(texdw[7]); 784 surf.nbx = (G_030000_PITCH(texdw[0]) + 1) * 8; 785 surf.nbx = r600_fmt_get_nblocksx(surf.format, surf.nbx); 786 surf.nby = r600_fmt_get_nblocksy(surf.format, height); 787 surf.mode = G_030004_ARRAY_MODE(texdw[1]); 788 surf.tsplit = G_030018_TILE_SPLIT(texdw[6]); 789 surf.nbanks = G_03001C_NUM_BANKS(texdw[7]); 790 surf.bankw = G_03001C_BANK_WIDTH(texdw[7]); 791 surf.bankh = G_03001C_BANK_HEIGHT(texdw[7]); 792 surf.mtilea = G_03001C_MACRO_TILE_ASPECT(texdw[7]); 793 surf.nsamples = 1; 794 toffset = texdw[2] << 8; 795 moffset = texdw[3] << 8; 796 797 if (!r600_fmt_is_valid_texture(surf.format, p->family)) { 798 dev_warn(p->dev, "%s:%d texture invalid format %d\n", 799 __func__, __LINE__, surf.format); 800 return -EINVAL; 801 } 802 switch (dim) { 803 case V_030000_SQ_TEX_DIM_1D: 804 case V_030000_SQ_TEX_DIM_2D: 805 case V_030000_SQ_TEX_DIM_CUBEMAP: 806 case V_030000_SQ_TEX_DIM_1D_ARRAY: 807 case V_030000_SQ_TEX_DIM_2D_ARRAY: 808 depth = 1; 809 break; 810 case V_030000_SQ_TEX_DIM_2D_MSAA: 811 case V_030000_SQ_TEX_DIM_2D_ARRAY_MSAA: 812 surf.nsamples = 1 << llevel; 813 llevel = 0; 814 depth = 1; 815 break; 816 case V_030000_SQ_TEX_DIM_3D: 817 break; 818 default: 819 dev_warn(p->dev, "%s:%d texture invalid dimension %d\n", 820 __func__, __LINE__, dim); 821 return -EINVAL; 822 } 823 824 r = evergreen_surface_value_conv_check(p, &surf, "texture"); 825 if (r) { 826 return r; 827 } 828 829 /* align height */ 830 evergreen_surface_check(p, &surf, NULL); 831 surf.nby = ALIGN(surf.nby, surf.halign); 832 833 r = evergreen_surface_check(p, &surf, "texture"); 834 if (r) { 835 dev_warn(p->dev, "%s:%d texture invalid 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n", 836 __func__, __LINE__, texdw[0], texdw[1], texdw[4], 837 texdw[5], texdw[6], texdw[7]); 838 return r; 839 } 840 841 /* check texture size */ 842 if (toffset & (surf.base_align - 1)) { 843 dev_warn(p->dev, "%s:%d texture bo base %ld not aligned with %ld\n", 844 __func__, __LINE__, toffset, surf.base_align); 845 return -EINVAL; 846 } 847 if (surf.nsamples <= 1 && moffset & (surf.base_align - 1)) { 848 dev_warn(p->dev, "%s:%d mipmap bo base %ld not aligned with %ld\n", 849 __func__, __LINE__, moffset, surf.base_align); 850 return -EINVAL; 851 } 852 if (dim == SQ_TEX_DIM_3D) { 853 toffset += surf.layer_size * depth; 854 } else { 855 toffset += surf.layer_size * mslice; 856 } 857 if (toffset > radeon_bo_size(texture)) { 858 dev_warn(p->dev, "%s:%d texture bo too small (layer size %d, " 859 "offset %ld, max layer %d, depth %d, bo size %ld) (%d %d)\n", 860 __func__, __LINE__, surf.layer_size, 861 (unsigned long)texdw[2] << 8, mslice, 862 depth, radeon_bo_size(texture), 863 surf.nbx, surf.nby); 864 return -EINVAL; 865 } 866 867 if (!mipmap) { 868 if (llevel) { 869 dev_warn(p->dev, "%s:%i got NULL MIP_ADDRESS relocation\n", 870 __func__, __LINE__); 871 return -EINVAL; 872 } else { 873 return 0; /* everything's ok */ 874 } 875 } 876 877 /* check mipmap size */ 878 for (i = 1; i <= llevel; i++) { 879 unsigned w, h, d; 880 881 w = r600_mip_minify(width, i); 882 h = r600_mip_minify(height, i); 883 d = r600_mip_minify(depth, i); 884 surf.nbx = r600_fmt_get_nblocksx(surf.format, w); 885 surf.nby = r600_fmt_get_nblocksy(surf.format, h); 886 887 switch (surf.mode) { 888 case ARRAY_2D_TILED_THIN1: 889 if (surf.nbx < surf.palign || surf.nby < surf.halign) { 890 surf.mode = ARRAY_1D_TILED_THIN1; 891 } 892 /* recompute alignment */ 893 evergreen_surface_check(p, &surf, NULL); 894 break; 895 case ARRAY_LINEAR_GENERAL: 896 case ARRAY_LINEAR_ALIGNED: 897 case ARRAY_1D_TILED_THIN1: 898 break; 899 default: 900 dev_warn(p->dev, "%s:%d invalid array mode %d\n", 901 __func__, __LINE__, surf.mode); 902 return -EINVAL; 903 } 904 surf.nbx = ALIGN(surf.nbx, surf.palign); 905 surf.nby = ALIGN(surf.nby, surf.halign); 906 907 r = evergreen_surface_check(p, &surf, "mipmap"); 908 if (r) { 909 return r; 910 } 911 912 if (dim == SQ_TEX_DIM_3D) { 913 moffset += surf.layer_size * d; 914 } else { 915 moffset += surf.layer_size * mslice; 916 } 917 if (moffset > radeon_bo_size(mipmap)) { 918 dev_warn(p->dev, "%s:%d mipmap [%d] bo too small (layer size %d, " 919 "offset %ld, coffset %ld, max layer %d, depth %d, " 920 "bo size %ld) level0 (%d %d %d)\n", 921 __func__, __LINE__, i, surf.layer_size, 922 (unsigned long)texdw[3] << 8, moffset, mslice, 923 d, radeon_bo_size(mipmap), 924 width, height, depth); 925 dev_warn(p->dev, "%s:%d problematic surf: (%d %d) (%d %d %d %d %d %d %d)\n", 926 __func__, __LINE__, surf.nbx, surf.nby, 927 surf.mode, surf.bpe, surf.nsamples, 928 surf.bankw, surf.bankh, 929 surf.tsplit, surf.mtilea); 930 return -EINVAL; 931 } 932 } 933 934 return 0; 935 } 936 937 static int evergreen_cs_track_check(struct radeon_cs_parser *p) 938 { 939 struct evergreen_cs_track *track = p->track; 940 unsigned tmp, i; 941 int r; 942 unsigned buffer_mask = 0; 943 944 /* check streamout */ 945 if (track->streamout_dirty && track->vgt_strmout_config) { 946 for (i = 0; i < 4; i++) { 947 if (track->vgt_strmout_config & (1 << i)) { 948 buffer_mask |= (track->vgt_strmout_buffer_config >> (i * 4)) & 0xf; 949 } 950 } 951 952 for (i = 0; i < 4; i++) { 953 if (buffer_mask & (1 << i)) { 954 if (track->vgt_strmout_bo[i]) { 955 u64 offset = (u64)track->vgt_strmout_bo_offset[i] + 956 (u64)track->vgt_strmout_size[i]; 957 if (offset > radeon_bo_size(track->vgt_strmout_bo[i])) { 958 DRM_ERROR("streamout %d bo too small: 0x%llx, 0x%lx\n", 959 i, offset, 960 radeon_bo_size(track->vgt_strmout_bo[i])); 961 return -EINVAL; 962 } 963 } else { 964 dev_warn(p->dev, "No buffer for streamout %d\n", i); 965 return -EINVAL; 966 } 967 } 968 } 969 track->streamout_dirty = false; 970 } 971 972 if (track->sx_misc_kill_all_prims) 973 return 0; 974 975 /* check that we have a cb for each enabled target 976 */ 977 if (track->cb_dirty) { 978 tmp = track->cb_target_mask; 979 for (i = 0; i < 8; i++) { 980 u32 format = G_028C70_FORMAT(track->cb_color_info[i]); 981 982 if (format != V_028C70_COLOR_INVALID && 983 (tmp >> (i * 4)) & 0xF) { 984 /* at least one component is enabled */ 985 if (track->cb_color_bo[i] == NULL) { 986 dev_warn(p->dev, "%s:%d mask 0x%08X | 0x%08X no cb for %d\n", 987 __func__, __LINE__, track->cb_target_mask, track->cb_shader_mask, i); 988 return -EINVAL; 989 } 990 /* check cb */ 991 r = evergreen_cs_track_validate_cb(p, i); 992 if (r) { 993 return r; 994 } 995 } 996 } 997 track->cb_dirty = false; 998 } 999 1000 if (track->db_dirty) { 1001 /* Check stencil buffer */ 1002 if (G_028044_FORMAT(track->db_s_info) != V_028044_STENCIL_INVALID && 1003 G_028800_STENCIL_ENABLE(track->db_depth_control)) { 1004 r = evergreen_cs_track_validate_stencil(p); 1005 if (r) 1006 return r; 1007 } 1008 /* Check depth buffer */ 1009 if (G_028040_FORMAT(track->db_z_info) != V_028040_Z_INVALID && 1010 G_028800_Z_ENABLE(track->db_depth_control)) { 1011 r = evergreen_cs_track_validate_depth(p); 1012 if (r) 1013 return r; 1014 } 1015 track->db_dirty = false; 1016 } 1017 1018 return 0; 1019 } 1020 1021 /** 1022 * evergreen_cs_packet_parse_vline() - parse userspace VLINE packet 1023 * @parser: parser structure holding parsing context. 1024 * 1025 * This is an Evergreen(+)-specific function for parsing VLINE packets. 1026 * Real work is done by r600_cs_common_vline_parse function. 1027 * Here we just set up ASIC-specific register table and call 1028 * the common implementation function. 1029 */ 1030 static int evergreen_cs_packet_parse_vline(struct radeon_cs_parser *p) 1031 { 1032 1033 static uint32_t vline_start_end[6] = { 1034 EVERGREEN_VLINE_START_END + EVERGREEN_CRTC0_REGISTER_OFFSET, 1035 EVERGREEN_VLINE_START_END + EVERGREEN_CRTC1_REGISTER_OFFSET, 1036 EVERGREEN_VLINE_START_END + EVERGREEN_CRTC2_REGISTER_OFFSET, 1037 EVERGREEN_VLINE_START_END + EVERGREEN_CRTC3_REGISTER_OFFSET, 1038 EVERGREEN_VLINE_START_END + EVERGREEN_CRTC4_REGISTER_OFFSET, 1039 EVERGREEN_VLINE_START_END + EVERGREEN_CRTC5_REGISTER_OFFSET 1040 }; 1041 static uint32_t vline_status[6] = { 1042 EVERGREEN_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, 1043 EVERGREEN_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, 1044 EVERGREEN_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, 1045 EVERGREEN_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, 1046 EVERGREEN_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, 1047 EVERGREEN_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET 1048 }; 1049 1050 return r600_cs_common_vline_parse(p, vline_start_end, vline_status); 1051 } 1052 1053 static int evergreen_packet0_check(struct radeon_cs_parser *p, 1054 struct radeon_cs_packet *pkt, 1055 unsigned idx, unsigned reg) 1056 { 1057 int r; 1058 1059 switch (reg) { 1060 case EVERGREEN_VLINE_START_END: 1061 r = evergreen_cs_packet_parse_vline(p); 1062 if (r) { 1063 DRM_ERROR("No reloc for ib[%d]=0x%04X\n", 1064 idx, reg); 1065 return r; 1066 } 1067 break; 1068 default: 1069 pr_err("Forbidden register 0x%04X in cs at %d\n", reg, idx); 1070 return -EINVAL; 1071 } 1072 return 0; 1073 } 1074 1075 static int evergreen_cs_parse_packet0(struct radeon_cs_parser *p, 1076 struct radeon_cs_packet *pkt) 1077 { 1078 unsigned reg, i; 1079 unsigned idx; 1080 int r; 1081 1082 idx = pkt->idx + 1; 1083 reg = pkt->reg; 1084 for (i = 0; i <= pkt->count; i++, idx++, reg += 4) { 1085 r = evergreen_packet0_check(p, pkt, idx, reg); 1086 if (r) { 1087 return r; 1088 } 1089 } 1090 return 0; 1091 } 1092 1093 /** 1094 * evergreen_cs_handle_reg() - process registers that need special handling. 1095 * @parser: parser structure holding parsing context 1096 * @reg: register we are testing 1097 * @idx: index into the cs buffer 1098 */ 1099 static int evergreen_cs_handle_reg(struct radeon_cs_parser *p, u32 reg, u32 idx) 1100 { 1101 struct evergreen_cs_track *track = (struct evergreen_cs_track *)p->track; 1102 struct radeon_bo_list *reloc; 1103 u32 tmp, *ib; 1104 int r; 1105 1106 ib = p->ib.ptr; 1107 switch (reg) { 1108 /* force following reg to 0 in an attempt to disable out buffer 1109 * which will need us to better understand how it works to perform 1110 * security check on it (Jerome) 1111 */ 1112 case SQ_ESGS_RING_SIZE: 1113 case SQ_GSVS_RING_SIZE: 1114 case SQ_ESTMP_RING_SIZE: 1115 case SQ_GSTMP_RING_SIZE: 1116 case SQ_HSTMP_RING_SIZE: 1117 case SQ_LSTMP_RING_SIZE: 1118 case SQ_PSTMP_RING_SIZE: 1119 case SQ_VSTMP_RING_SIZE: 1120 case SQ_ESGS_RING_ITEMSIZE: 1121 case SQ_ESTMP_RING_ITEMSIZE: 1122 case SQ_GSTMP_RING_ITEMSIZE: 1123 case SQ_GSVS_RING_ITEMSIZE: 1124 case SQ_GS_VERT_ITEMSIZE: 1125 case SQ_GS_VERT_ITEMSIZE_1: 1126 case SQ_GS_VERT_ITEMSIZE_2: 1127 case SQ_GS_VERT_ITEMSIZE_3: 1128 case SQ_GSVS_RING_OFFSET_1: 1129 case SQ_GSVS_RING_OFFSET_2: 1130 case SQ_GSVS_RING_OFFSET_3: 1131 case SQ_HSTMP_RING_ITEMSIZE: 1132 case SQ_LSTMP_RING_ITEMSIZE: 1133 case SQ_PSTMP_RING_ITEMSIZE: 1134 case SQ_VSTMP_RING_ITEMSIZE: 1135 case VGT_TF_RING_SIZE: 1136 /* get value to populate the IB don't remove */ 1137 /*tmp =radeon_get_ib_value(p, idx); 1138 ib[idx] = 0;*/ 1139 break; 1140 case SQ_ESGS_RING_BASE: 1141 case SQ_GSVS_RING_BASE: 1142 case SQ_ESTMP_RING_BASE: 1143 case SQ_GSTMP_RING_BASE: 1144 case SQ_HSTMP_RING_BASE: 1145 case SQ_LSTMP_RING_BASE: 1146 case SQ_PSTMP_RING_BASE: 1147 case SQ_VSTMP_RING_BASE: 1148 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 1149 if (r) { 1150 dev_warn(p->dev, "bad SET_CONTEXT_REG " 1151 "0x%04X\n", reg); 1152 return -EINVAL; 1153 } 1154 ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); 1155 break; 1156 case DB_DEPTH_CONTROL: 1157 track->db_depth_control = radeon_get_ib_value(p, idx); 1158 track->db_dirty = true; 1159 break; 1160 case CAYMAN_DB_EQAA: 1161 if (p->rdev->family < CHIP_CAYMAN) { 1162 dev_warn(p->dev, "bad SET_CONTEXT_REG " 1163 "0x%04X\n", reg); 1164 return -EINVAL; 1165 } 1166 break; 1167 case CAYMAN_DB_DEPTH_INFO: 1168 if (p->rdev->family < CHIP_CAYMAN) { 1169 dev_warn(p->dev, "bad SET_CONTEXT_REG " 1170 "0x%04X\n", reg); 1171 return -EINVAL; 1172 } 1173 break; 1174 case DB_Z_INFO: 1175 track->db_z_info = radeon_get_ib_value(p, idx); 1176 if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) { 1177 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 1178 if (r) { 1179 dev_warn(p->dev, "bad SET_CONTEXT_REG " 1180 "0x%04X\n", reg); 1181 return -EINVAL; 1182 } 1183 ib[idx] &= ~Z_ARRAY_MODE(0xf); 1184 track->db_z_info &= ~Z_ARRAY_MODE(0xf); 1185 ib[idx] |= Z_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->tiling_flags)); 1186 track->db_z_info |= Z_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->tiling_flags)); 1187 if (reloc->tiling_flags & RADEON_TILING_MACRO) { 1188 unsigned bankw, bankh, mtaspect, tile_split; 1189 1190 evergreen_tiling_fields(reloc->tiling_flags, 1191 &bankw, &bankh, &mtaspect, 1192 &tile_split); 1193 ib[idx] |= DB_NUM_BANKS(evergreen_cs_get_num_banks(track->nbanks)); 1194 ib[idx] |= DB_TILE_SPLIT(tile_split) | 1195 DB_BANK_WIDTH(bankw) | 1196 DB_BANK_HEIGHT(bankh) | 1197 DB_MACRO_TILE_ASPECT(mtaspect); 1198 } 1199 } 1200 track->db_dirty = true; 1201 break; 1202 case DB_STENCIL_INFO: 1203 track->db_s_info = radeon_get_ib_value(p, idx); 1204 track->db_dirty = true; 1205 break; 1206 case DB_DEPTH_VIEW: 1207 track->db_depth_view = radeon_get_ib_value(p, idx); 1208 track->db_dirty = true; 1209 break; 1210 case DB_DEPTH_SIZE: 1211 track->db_depth_size = radeon_get_ib_value(p, idx); 1212 track->db_dirty = true; 1213 break; 1214 case R_02805C_DB_DEPTH_SLICE: 1215 track->db_depth_slice = radeon_get_ib_value(p, idx); 1216 track->db_dirty = true; 1217 break; 1218 case DB_Z_READ_BASE: 1219 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 1220 if (r) { 1221 dev_warn(p->dev, "bad SET_CONTEXT_REG " 1222 "0x%04X\n", reg); 1223 return -EINVAL; 1224 } 1225 track->db_z_read_offset = radeon_get_ib_value(p, idx); 1226 ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); 1227 track->db_z_read_bo = reloc->robj; 1228 track->db_dirty = true; 1229 break; 1230 case DB_Z_WRITE_BASE: 1231 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 1232 if (r) { 1233 dev_warn(p->dev, "bad SET_CONTEXT_REG " 1234 "0x%04X\n", reg); 1235 return -EINVAL; 1236 } 1237 track->db_z_write_offset = radeon_get_ib_value(p, idx); 1238 ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); 1239 track->db_z_write_bo = reloc->robj; 1240 track->db_dirty = true; 1241 break; 1242 case DB_STENCIL_READ_BASE: 1243 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 1244 if (r) { 1245 dev_warn(p->dev, "bad SET_CONTEXT_REG " 1246 "0x%04X\n", reg); 1247 return -EINVAL; 1248 } 1249 track->db_s_read_offset = radeon_get_ib_value(p, idx); 1250 ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); 1251 track->db_s_read_bo = reloc->robj; 1252 track->db_dirty = true; 1253 break; 1254 case DB_STENCIL_WRITE_BASE: 1255 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 1256 if (r) { 1257 dev_warn(p->dev, "bad SET_CONTEXT_REG " 1258 "0x%04X\n", reg); 1259 return -EINVAL; 1260 } 1261 track->db_s_write_offset = radeon_get_ib_value(p, idx); 1262 ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); 1263 track->db_s_write_bo = reloc->robj; 1264 track->db_dirty = true; 1265 break; 1266 case VGT_STRMOUT_CONFIG: 1267 track->vgt_strmout_config = radeon_get_ib_value(p, idx); 1268 track->streamout_dirty = true; 1269 break; 1270 case VGT_STRMOUT_BUFFER_CONFIG: 1271 track->vgt_strmout_buffer_config = radeon_get_ib_value(p, idx); 1272 track->streamout_dirty = true; 1273 break; 1274 case VGT_STRMOUT_BUFFER_BASE_0: 1275 case VGT_STRMOUT_BUFFER_BASE_1: 1276 case VGT_STRMOUT_BUFFER_BASE_2: 1277 case VGT_STRMOUT_BUFFER_BASE_3: 1278 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 1279 if (r) { 1280 dev_warn(p->dev, "bad SET_CONTEXT_REG " 1281 "0x%04X\n", reg); 1282 return -EINVAL; 1283 } 1284 tmp = (reg - VGT_STRMOUT_BUFFER_BASE_0) / 16; 1285 track->vgt_strmout_bo_offset[tmp] = radeon_get_ib_value(p, idx) << 8; 1286 ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); 1287 track->vgt_strmout_bo[tmp] = reloc->robj; 1288 track->streamout_dirty = true; 1289 break; 1290 case VGT_STRMOUT_BUFFER_SIZE_0: 1291 case VGT_STRMOUT_BUFFER_SIZE_1: 1292 case VGT_STRMOUT_BUFFER_SIZE_2: 1293 case VGT_STRMOUT_BUFFER_SIZE_3: 1294 tmp = (reg - VGT_STRMOUT_BUFFER_SIZE_0) / 16; 1295 /* size in register is DWs, convert to bytes */ 1296 track->vgt_strmout_size[tmp] = radeon_get_ib_value(p, idx) * 4; 1297 track->streamout_dirty = true; 1298 break; 1299 case CP_COHER_BASE: 1300 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 1301 if (r) { 1302 dev_warn(p->dev, "missing reloc for CP_COHER_BASE " 1303 "0x%04X\n", reg); 1304 return -EINVAL; 1305 } 1306 ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); 1307 break; 1308 case CB_TARGET_MASK: 1309 track->cb_target_mask = radeon_get_ib_value(p, idx); 1310 track->cb_dirty = true; 1311 break; 1312 case CB_SHADER_MASK: 1313 track->cb_shader_mask = radeon_get_ib_value(p, idx); 1314 track->cb_dirty = true; 1315 break; 1316 case PA_SC_AA_CONFIG: 1317 if (p->rdev->family >= CHIP_CAYMAN) { 1318 dev_warn(p->dev, "bad SET_CONTEXT_REG " 1319 "0x%04X\n", reg); 1320 return -EINVAL; 1321 } 1322 tmp = radeon_get_ib_value(p, idx) & MSAA_NUM_SAMPLES_MASK; 1323 track->nsamples = 1 << tmp; 1324 break; 1325 case CAYMAN_PA_SC_AA_CONFIG: 1326 if (p->rdev->family < CHIP_CAYMAN) { 1327 dev_warn(p->dev, "bad SET_CONTEXT_REG " 1328 "0x%04X\n", reg); 1329 return -EINVAL; 1330 } 1331 tmp = radeon_get_ib_value(p, idx) & CAYMAN_MSAA_NUM_SAMPLES_MASK; 1332 track->nsamples = 1 << tmp; 1333 break; 1334 case CB_COLOR0_VIEW: 1335 case CB_COLOR1_VIEW: 1336 case CB_COLOR2_VIEW: 1337 case CB_COLOR3_VIEW: 1338 case CB_COLOR4_VIEW: 1339 case CB_COLOR5_VIEW: 1340 case CB_COLOR6_VIEW: 1341 case CB_COLOR7_VIEW: 1342 tmp = (reg - CB_COLOR0_VIEW) / 0x3c; 1343 track->cb_color_view[tmp] = radeon_get_ib_value(p, idx); 1344 track->cb_dirty = true; 1345 break; 1346 case CB_COLOR8_VIEW: 1347 case CB_COLOR9_VIEW: 1348 case CB_COLOR10_VIEW: 1349 case CB_COLOR11_VIEW: 1350 tmp = ((reg - CB_COLOR8_VIEW) / 0x1c) + 8; 1351 track->cb_color_view[tmp] = radeon_get_ib_value(p, idx); 1352 track->cb_dirty = true; 1353 break; 1354 case CB_COLOR0_INFO: 1355 case CB_COLOR1_INFO: 1356 case CB_COLOR2_INFO: 1357 case CB_COLOR3_INFO: 1358 case CB_COLOR4_INFO: 1359 case CB_COLOR5_INFO: 1360 case CB_COLOR6_INFO: 1361 case CB_COLOR7_INFO: 1362 tmp = (reg - CB_COLOR0_INFO) / 0x3c; 1363 track->cb_color_info[tmp] = radeon_get_ib_value(p, idx); 1364 if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) { 1365 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 1366 if (r) { 1367 dev_warn(p->dev, "bad SET_CONTEXT_REG " 1368 "0x%04X\n", reg); 1369 return -EINVAL; 1370 } 1371 ib[idx] |= CB_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->tiling_flags)); 1372 track->cb_color_info[tmp] |= CB_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->tiling_flags)); 1373 } 1374 track->cb_dirty = true; 1375 break; 1376 case CB_COLOR8_INFO: 1377 case CB_COLOR9_INFO: 1378 case CB_COLOR10_INFO: 1379 case CB_COLOR11_INFO: 1380 tmp = ((reg - CB_COLOR8_INFO) / 0x1c) + 8; 1381 track->cb_color_info[tmp] = radeon_get_ib_value(p, idx); 1382 if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) { 1383 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 1384 if (r) { 1385 dev_warn(p->dev, "bad SET_CONTEXT_REG " 1386 "0x%04X\n", reg); 1387 return -EINVAL; 1388 } 1389 ib[idx] |= CB_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->tiling_flags)); 1390 track->cb_color_info[tmp] |= CB_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->tiling_flags)); 1391 } 1392 track->cb_dirty = true; 1393 break; 1394 case CB_COLOR0_PITCH: 1395 case CB_COLOR1_PITCH: 1396 case CB_COLOR2_PITCH: 1397 case CB_COLOR3_PITCH: 1398 case CB_COLOR4_PITCH: 1399 case CB_COLOR5_PITCH: 1400 case CB_COLOR6_PITCH: 1401 case CB_COLOR7_PITCH: 1402 tmp = (reg - CB_COLOR0_PITCH) / 0x3c; 1403 track->cb_color_pitch[tmp] = radeon_get_ib_value(p, idx); 1404 track->cb_dirty = true; 1405 break; 1406 case CB_COLOR8_PITCH: 1407 case CB_COLOR9_PITCH: 1408 case CB_COLOR10_PITCH: 1409 case CB_COLOR11_PITCH: 1410 tmp = ((reg - CB_COLOR8_PITCH) / 0x1c) + 8; 1411 track->cb_color_pitch[tmp] = radeon_get_ib_value(p, idx); 1412 track->cb_dirty = true; 1413 break; 1414 case CB_COLOR0_SLICE: 1415 case CB_COLOR1_SLICE: 1416 case CB_COLOR2_SLICE: 1417 case CB_COLOR3_SLICE: 1418 case CB_COLOR4_SLICE: 1419 case CB_COLOR5_SLICE: 1420 case CB_COLOR6_SLICE: 1421 case CB_COLOR7_SLICE: 1422 tmp = (reg - CB_COLOR0_SLICE) / 0x3c; 1423 track->cb_color_slice[tmp] = radeon_get_ib_value(p, idx); 1424 track->cb_color_slice_idx[tmp] = idx; 1425 track->cb_dirty = true; 1426 break; 1427 case CB_COLOR8_SLICE: 1428 case CB_COLOR9_SLICE: 1429 case CB_COLOR10_SLICE: 1430 case CB_COLOR11_SLICE: 1431 tmp = ((reg - CB_COLOR8_SLICE) / 0x1c) + 8; 1432 track->cb_color_slice[tmp] = radeon_get_ib_value(p, idx); 1433 track->cb_color_slice_idx[tmp] = idx; 1434 track->cb_dirty = true; 1435 break; 1436 case CB_COLOR0_ATTRIB: 1437 case CB_COLOR1_ATTRIB: 1438 case CB_COLOR2_ATTRIB: 1439 case CB_COLOR3_ATTRIB: 1440 case CB_COLOR4_ATTRIB: 1441 case CB_COLOR5_ATTRIB: 1442 case CB_COLOR6_ATTRIB: 1443 case CB_COLOR7_ATTRIB: 1444 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 1445 if (r) { 1446 dev_warn(p->dev, "bad SET_CONTEXT_REG " 1447 "0x%04X\n", reg); 1448 return -EINVAL; 1449 } 1450 if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) { 1451 if (reloc->tiling_flags & RADEON_TILING_MACRO) { 1452 unsigned bankw, bankh, mtaspect, tile_split; 1453 1454 evergreen_tiling_fields(reloc->tiling_flags, 1455 &bankw, &bankh, &mtaspect, 1456 &tile_split); 1457 ib[idx] |= CB_NUM_BANKS(evergreen_cs_get_num_banks(track->nbanks)); 1458 ib[idx] |= CB_TILE_SPLIT(tile_split) | 1459 CB_BANK_WIDTH(bankw) | 1460 CB_BANK_HEIGHT(bankh) | 1461 CB_MACRO_TILE_ASPECT(mtaspect); 1462 } 1463 } 1464 tmp = ((reg - CB_COLOR0_ATTRIB) / 0x3c); 1465 track->cb_color_attrib[tmp] = ib[idx]; 1466 track->cb_dirty = true; 1467 break; 1468 case CB_COLOR8_ATTRIB: 1469 case CB_COLOR9_ATTRIB: 1470 case CB_COLOR10_ATTRIB: 1471 case CB_COLOR11_ATTRIB: 1472 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 1473 if (r) { 1474 dev_warn(p->dev, "bad SET_CONTEXT_REG " 1475 "0x%04X\n", reg); 1476 return -EINVAL; 1477 } 1478 if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) { 1479 if (reloc->tiling_flags & RADEON_TILING_MACRO) { 1480 unsigned bankw, bankh, mtaspect, tile_split; 1481 1482 evergreen_tiling_fields(reloc->tiling_flags, 1483 &bankw, &bankh, &mtaspect, 1484 &tile_split); 1485 ib[idx] |= CB_NUM_BANKS(evergreen_cs_get_num_banks(track->nbanks)); 1486 ib[idx] |= CB_TILE_SPLIT(tile_split) | 1487 CB_BANK_WIDTH(bankw) | 1488 CB_BANK_HEIGHT(bankh) | 1489 CB_MACRO_TILE_ASPECT(mtaspect); 1490 } 1491 } 1492 tmp = ((reg - CB_COLOR8_ATTRIB) / 0x1c) + 8; 1493 track->cb_color_attrib[tmp] = ib[idx]; 1494 track->cb_dirty = true; 1495 break; 1496 case CB_COLOR0_FMASK: 1497 case CB_COLOR1_FMASK: 1498 case CB_COLOR2_FMASK: 1499 case CB_COLOR3_FMASK: 1500 case CB_COLOR4_FMASK: 1501 case CB_COLOR5_FMASK: 1502 case CB_COLOR6_FMASK: 1503 case CB_COLOR7_FMASK: 1504 tmp = (reg - CB_COLOR0_FMASK) / 0x3c; 1505 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 1506 if (r) { 1507 dev_err(p->dev, "bad SET_CONTEXT_REG 0x%04X\n", reg); 1508 return -EINVAL; 1509 } 1510 ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); 1511 track->cb_color_fmask_bo[tmp] = reloc->robj; 1512 break; 1513 case CB_COLOR0_CMASK: 1514 case CB_COLOR1_CMASK: 1515 case CB_COLOR2_CMASK: 1516 case CB_COLOR3_CMASK: 1517 case CB_COLOR4_CMASK: 1518 case CB_COLOR5_CMASK: 1519 case CB_COLOR6_CMASK: 1520 case CB_COLOR7_CMASK: 1521 tmp = (reg - CB_COLOR0_CMASK) / 0x3c; 1522 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 1523 if (r) { 1524 dev_err(p->dev, "bad SET_CONTEXT_REG 0x%04X\n", reg); 1525 return -EINVAL; 1526 } 1527 ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); 1528 track->cb_color_cmask_bo[tmp] = reloc->robj; 1529 break; 1530 case CB_COLOR0_FMASK_SLICE: 1531 case CB_COLOR1_FMASK_SLICE: 1532 case CB_COLOR2_FMASK_SLICE: 1533 case CB_COLOR3_FMASK_SLICE: 1534 case CB_COLOR4_FMASK_SLICE: 1535 case CB_COLOR5_FMASK_SLICE: 1536 case CB_COLOR6_FMASK_SLICE: 1537 case CB_COLOR7_FMASK_SLICE: 1538 tmp = (reg - CB_COLOR0_FMASK_SLICE) / 0x3c; 1539 track->cb_color_fmask_slice[tmp] = radeon_get_ib_value(p, idx); 1540 break; 1541 case CB_COLOR0_CMASK_SLICE: 1542 case CB_COLOR1_CMASK_SLICE: 1543 case CB_COLOR2_CMASK_SLICE: 1544 case CB_COLOR3_CMASK_SLICE: 1545 case CB_COLOR4_CMASK_SLICE: 1546 case CB_COLOR5_CMASK_SLICE: 1547 case CB_COLOR6_CMASK_SLICE: 1548 case CB_COLOR7_CMASK_SLICE: 1549 tmp = (reg - CB_COLOR0_CMASK_SLICE) / 0x3c; 1550 track->cb_color_cmask_slice[tmp] = radeon_get_ib_value(p, idx); 1551 break; 1552 case CB_COLOR0_BASE: 1553 case CB_COLOR1_BASE: 1554 case CB_COLOR2_BASE: 1555 case CB_COLOR3_BASE: 1556 case CB_COLOR4_BASE: 1557 case CB_COLOR5_BASE: 1558 case CB_COLOR6_BASE: 1559 case CB_COLOR7_BASE: 1560 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 1561 if (r) { 1562 dev_warn(p->dev, "bad SET_CONTEXT_REG " 1563 "0x%04X\n", reg); 1564 return -EINVAL; 1565 } 1566 tmp = (reg - CB_COLOR0_BASE) / 0x3c; 1567 track->cb_color_bo_offset[tmp] = radeon_get_ib_value(p, idx); 1568 ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); 1569 track->cb_color_bo[tmp] = reloc->robj; 1570 track->cb_dirty = true; 1571 break; 1572 case CB_COLOR8_BASE: 1573 case CB_COLOR9_BASE: 1574 case CB_COLOR10_BASE: 1575 case CB_COLOR11_BASE: 1576 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 1577 if (r) { 1578 dev_warn(p->dev, "bad SET_CONTEXT_REG " 1579 "0x%04X\n", reg); 1580 return -EINVAL; 1581 } 1582 tmp = ((reg - CB_COLOR8_BASE) / 0x1c) + 8; 1583 track->cb_color_bo_offset[tmp] = radeon_get_ib_value(p, idx); 1584 ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); 1585 track->cb_color_bo[tmp] = reloc->robj; 1586 track->cb_dirty = true; 1587 break; 1588 case DB_HTILE_DATA_BASE: 1589 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 1590 if (r) { 1591 dev_warn(p->dev, "bad SET_CONTEXT_REG " 1592 "0x%04X\n", reg); 1593 return -EINVAL; 1594 } 1595 track->htile_offset = radeon_get_ib_value(p, idx); 1596 ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); 1597 track->htile_bo = reloc->robj; 1598 track->db_dirty = true; 1599 break; 1600 case DB_HTILE_SURFACE: 1601 /* 8x8 only */ 1602 track->htile_surface = radeon_get_ib_value(p, idx); 1603 /* force 8x8 htile width and height */ 1604 ib[idx] |= 3; 1605 track->db_dirty = true; 1606 break; 1607 case CB_IMMED0_BASE: 1608 case CB_IMMED1_BASE: 1609 case CB_IMMED2_BASE: 1610 case CB_IMMED3_BASE: 1611 case CB_IMMED4_BASE: 1612 case CB_IMMED5_BASE: 1613 case CB_IMMED6_BASE: 1614 case CB_IMMED7_BASE: 1615 case CB_IMMED8_BASE: 1616 case CB_IMMED9_BASE: 1617 case CB_IMMED10_BASE: 1618 case CB_IMMED11_BASE: 1619 case SQ_PGM_START_FS: 1620 case SQ_PGM_START_ES: 1621 case SQ_PGM_START_VS: 1622 case SQ_PGM_START_GS: 1623 case SQ_PGM_START_PS: 1624 case SQ_PGM_START_HS: 1625 case SQ_PGM_START_LS: 1626 case SQ_CONST_MEM_BASE: 1627 case SQ_ALU_CONST_CACHE_GS_0: 1628 case SQ_ALU_CONST_CACHE_GS_1: 1629 case SQ_ALU_CONST_CACHE_GS_2: 1630 case SQ_ALU_CONST_CACHE_GS_3: 1631 case SQ_ALU_CONST_CACHE_GS_4: 1632 case SQ_ALU_CONST_CACHE_GS_5: 1633 case SQ_ALU_CONST_CACHE_GS_6: 1634 case SQ_ALU_CONST_CACHE_GS_7: 1635 case SQ_ALU_CONST_CACHE_GS_8: 1636 case SQ_ALU_CONST_CACHE_GS_9: 1637 case SQ_ALU_CONST_CACHE_GS_10: 1638 case SQ_ALU_CONST_CACHE_GS_11: 1639 case SQ_ALU_CONST_CACHE_GS_12: 1640 case SQ_ALU_CONST_CACHE_GS_13: 1641 case SQ_ALU_CONST_CACHE_GS_14: 1642 case SQ_ALU_CONST_CACHE_GS_15: 1643 case SQ_ALU_CONST_CACHE_PS_0: 1644 case SQ_ALU_CONST_CACHE_PS_1: 1645 case SQ_ALU_CONST_CACHE_PS_2: 1646 case SQ_ALU_CONST_CACHE_PS_3: 1647 case SQ_ALU_CONST_CACHE_PS_4: 1648 case SQ_ALU_CONST_CACHE_PS_5: 1649 case SQ_ALU_CONST_CACHE_PS_6: 1650 case SQ_ALU_CONST_CACHE_PS_7: 1651 case SQ_ALU_CONST_CACHE_PS_8: 1652 case SQ_ALU_CONST_CACHE_PS_9: 1653 case SQ_ALU_CONST_CACHE_PS_10: 1654 case SQ_ALU_CONST_CACHE_PS_11: 1655 case SQ_ALU_CONST_CACHE_PS_12: 1656 case SQ_ALU_CONST_CACHE_PS_13: 1657 case SQ_ALU_CONST_CACHE_PS_14: 1658 case SQ_ALU_CONST_CACHE_PS_15: 1659 case SQ_ALU_CONST_CACHE_VS_0: 1660 case SQ_ALU_CONST_CACHE_VS_1: 1661 case SQ_ALU_CONST_CACHE_VS_2: 1662 case SQ_ALU_CONST_CACHE_VS_3: 1663 case SQ_ALU_CONST_CACHE_VS_4: 1664 case SQ_ALU_CONST_CACHE_VS_5: 1665 case SQ_ALU_CONST_CACHE_VS_6: 1666 case SQ_ALU_CONST_CACHE_VS_7: 1667 case SQ_ALU_CONST_CACHE_VS_8: 1668 case SQ_ALU_CONST_CACHE_VS_9: 1669 case SQ_ALU_CONST_CACHE_VS_10: 1670 case SQ_ALU_CONST_CACHE_VS_11: 1671 case SQ_ALU_CONST_CACHE_VS_12: 1672 case SQ_ALU_CONST_CACHE_VS_13: 1673 case SQ_ALU_CONST_CACHE_VS_14: 1674 case SQ_ALU_CONST_CACHE_VS_15: 1675 case SQ_ALU_CONST_CACHE_HS_0: 1676 case SQ_ALU_CONST_CACHE_HS_1: 1677 case SQ_ALU_CONST_CACHE_HS_2: 1678 case SQ_ALU_CONST_CACHE_HS_3: 1679 case SQ_ALU_CONST_CACHE_HS_4: 1680 case SQ_ALU_CONST_CACHE_HS_5: 1681 case SQ_ALU_CONST_CACHE_HS_6: 1682 case SQ_ALU_CONST_CACHE_HS_7: 1683 case SQ_ALU_CONST_CACHE_HS_8: 1684 case SQ_ALU_CONST_CACHE_HS_9: 1685 case SQ_ALU_CONST_CACHE_HS_10: 1686 case SQ_ALU_CONST_CACHE_HS_11: 1687 case SQ_ALU_CONST_CACHE_HS_12: 1688 case SQ_ALU_CONST_CACHE_HS_13: 1689 case SQ_ALU_CONST_CACHE_HS_14: 1690 case SQ_ALU_CONST_CACHE_HS_15: 1691 case SQ_ALU_CONST_CACHE_LS_0: 1692 case SQ_ALU_CONST_CACHE_LS_1: 1693 case SQ_ALU_CONST_CACHE_LS_2: 1694 case SQ_ALU_CONST_CACHE_LS_3: 1695 case SQ_ALU_CONST_CACHE_LS_4: 1696 case SQ_ALU_CONST_CACHE_LS_5: 1697 case SQ_ALU_CONST_CACHE_LS_6: 1698 case SQ_ALU_CONST_CACHE_LS_7: 1699 case SQ_ALU_CONST_CACHE_LS_8: 1700 case SQ_ALU_CONST_CACHE_LS_9: 1701 case SQ_ALU_CONST_CACHE_LS_10: 1702 case SQ_ALU_CONST_CACHE_LS_11: 1703 case SQ_ALU_CONST_CACHE_LS_12: 1704 case SQ_ALU_CONST_CACHE_LS_13: 1705 case SQ_ALU_CONST_CACHE_LS_14: 1706 case SQ_ALU_CONST_CACHE_LS_15: 1707 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 1708 if (r) { 1709 dev_warn(p->dev, "bad SET_CONTEXT_REG " 1710 "0x%04X\n", reg); 1711 return -EINVAL; 1712 } 1713 ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); 1714 break; 1715 case SX_MEMORY_EXPORT_BASE: 1716 if (p->rdev->family >= CHIP_CAYMAN) { 1717 dev_warn(p->dev, "bad SET_CONFIG_REG " 1718 "0x%04X\n", reg); 1719 return -EINVAL; 1720 } 1721 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 1722 if (r) { 1723 dev_warn(p->dev, "bad SET_CONFIG_REG " 1724 "0x%04X\n", reg); 1725 return -EINVAL; 1726 } 1727 ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); 1728 break; 1729 case CAYMAN_SX_SCATTER_EXPORT_BASE: 1730 if (p->rdev->family < CHIP_CAYMAN) { 1731 dev_warn(p->dev, "bad SET_CONTEXT_REG " 1732 "0x%04X\n", reg); 1733 return -EINVAL; 1734 } 1735 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 1736 if (r) { 1737 dev_warn(p->dev, "bad SET_CONTEXT_REG " 1738 "0x%04X\n", reg); 1739 return -EINVAL; 1740 } 1741 ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); 1742 break; 1743 case SX_MISC: 1744 track->sx_misc_kill_all_prims = (radeon_get_ib_value(p, idx) & 0x1) != 0; 1745 break; 1746 default: 1747 dev_warn(p->dev, "forbidden register 0x%08x at %d\n", reg, idx); 1748 return -EINVAL; 1749 } 1750 return 0; 1751 } 1752 1753 /** 1754 * evergreen_is_safe_reg() - check if register is authorized or not 1755 * @parser: parser structure holding parsing context 1756 * @reg: register we are testing 1757 * 1758 * This function will test against reg_safe_bm and return true 1759 * if register is safe or false otherwise. 1760 */ 1761 static inline bool evergreen_is_safe_reg(struct radeon_cs_parser *p, u32 reg) 1762 { 1763 struct evergreen_cs_track *track = p->track; 1764 u32 m, i; 1765 1766 i = (reg >> 7); 1767 if (unlikely(i >= REG_SAFE_BM_SIZE)) { 1768 return false; 1769 } 1770 m = 1 << ((reg >> 2) & 31); 1771 if (!(track->reg_safe_bm[i] & m)) 1772 return true; 1773 1774 return false; 1775 } 1776 1777 static int evergreen_packet3_check(struct radeon_cs_parser *p, 1778 struct radeon_cs_packet *pkt) 1779 { 1780 struct radeon_bo_list *reloc; 1781 struct evergreen_cs_track *track; 1782 uint32_t *ib; 1783 unsigned idx; 1784 unsigned i; 1785 unsigned start_reg, end_reg, reg; 1786 int r; 1787 u32 idx_value; 1788 1789 track = (struct evergreen_cs_track *)p->track; 1790 ib = p->ib.ptr; 1791 idx = pkt->idx + 1; 1792 idx_value = radeon_get_ib_value(p, idx); 1793 1794 switch (pkt->opcode) { 1795 case PACKET3_SET_PREDICATION: 1796 { 1797 int pred_op; 1798 int tmp; 1799 uint64_t offset; 1800 1801 if (pkt->count != 1) { 1802 DRM_ERROR("bad SET PREDICATION\n"); 1803 return -EINVAL; 1804 } 1805 1806 tmp = radeon_get_ib_value(p, idx + 1); 1807 pred_op = (tmp >> 16) & 0x7; 1808 1809 /* for the clear predicate operation */ 1810 if (pred_op == 0) 1811 return 0; 1812 1813 if (pred_op > 2) { 1814 DRM_ERROR("bad SET PREDICATION operation %d\n", pred_op); 1815 return -EINVAL; 1816 } 1817 1818 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 1819 if (r) { 1820 DRM_ERROR("bad SET PREDICATION\n"); 1821 return -EINVAL; 1822 } 1823 1824 offset = reloc->gpu_offset + 1825 (idx_value & 0xfffffff0) + 1826 ((u64)(tmp & 0xff) << 32); 1827 1828 ib[idx + 0] = offset; 1829 ib[idx + 1] = (tmp & 0xffffff00) | (upper_32_bits(offset) & 0xff); 1830 } 1831 break; 1832 case PACKET3_CONTEXT_CONTROL: 1833 if (pkt->count != 1) { 1834 DRM_ERROR("bad CONTEXT_CONTROL\n"); 1835 return -EINVAL; 1836 } 1837 break; 1838 case PACKET3_INDEX_TYPE: 1839 case PACKET3_NUM_INSTANCES: 1840 case PACKET3_CLEAR_STATE: 1841 if (pkt->count) { 1842 DRM_ERROR("bad INDEX_TYPE/NUM_INSTANCES/CLEAR_STATE\n"); 1843 return -EINVAL; 1844 } 1845 break; 1846 case CAYMAN_PACKET3_DEALLOC_STATE: 1847 if (p->rdev->family < CHIP_CAYMAN) { 1848 DRM_ERROR("bad PACKET3_DEALLOC_STATE\n"); 1849 return -EINVAL; 1850 } 1851 if (pkt->count) { 1852 DRM_ERROR("bad INDEX_TYPE/NUM_INSTANCES/CLEAR_STATE\n"); 1853 return -EINVAL; 1854 } 1855 break; 1856 case PACKET3_INDEX_BASE: 1857 { 1858 uint64_t offset; 1859 1860 if (pkt->count != 1) { 1861 DRM_ERROR("bad INDEX_BASE\n"); 1862 return -EINVAL; 1863 } 1864 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 1865 if (r) { 1866 DRM_ERROR("bad INDEX_BASE\n"); 1867 return -EINVAL; 1868 } 1869 1870 offset = reloc->gpu_offset + 1871 idx_value + 1872 ((u64)(radeon_get_ib_value(p, idx+1) & 0xff) << 32); 1873 1874 ib[idx+0] = offset; 1875 ib[idx+1] = upper_32_bits(offset) & 0xff; 1876 1877 r = evergreen_cs_track_check(p); 1878 if (r) { 1879 dev_warn(p->dev, "%s:%d invalid cmd stream\n", __func__, __LINE__); 1880 return r; 1881 } 1882 break; 1883 } 1884 case PACKET3_INDEX_BUFFER_SIZE: 1885 { 1886 if (pkt->count != 0) { 1887 DRM_ERROR("bad INDEX_BUFFER_SIZE\n"); 1888 return -EINVAL; 1889 } 1890 break; 1891 } 1892 case PACKET3_DRAW_INDEX: 1893 { 1894 uint64_t offset; 1895 if (pkt->count != 3) { 1896 DRM_ERROR("bad DRAW_INDEX\n"); 1897 return -EINVAL; 1898 } 1899 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 1900 if (r) { 1901 DRM_ERROR("bad DRAW_INDEX\n"); 1902 return -EINVAL; 1903 } 1904 1905 offset = reloc->gpu_offset + 1906 idx_value + 1907 ((u64)(radeon_get_ib_value(p, idx+1) & 0xff) << 32); 1908 1909 ib[idx+0] = offset; 1910 ib[idx+1] = upper_32_bits(offset) & 0xff; 1911 1912 r = evergreen_cs_track_check(p); 1913 if (r) { 1914 dev_warn(p->dev, "%s:%d invalid cmd stream\n", __func__, __LINE__); 1915 return r; 1916 } 1917 break; 1918 } 1919 case PACKET3_DRAW_INDEX_2: 1920 { 1921 uint64_t offset; 1922 1923 if (pkt->count != 4) { 1924 DRM_ERROR("bad DRAW_INDEX_2\n"); 1925 return -EINVAL; 1926 } 1927 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 1928 if (r) { 1929 DRM_ERROR("bad DRAW_INDEX_2\n"); 1930 return -EINVAL; 1931 } 1932 1933 offset = reloc->gpu_offset + 1934 radeon_get_ib_value(p, idx+1) + 1935 ((u64)(radeon_get_ib_value(p, idx+2) & 0xff) << 32); 1936 1937 ib[idx+1] = offset; 1938 ib[idx+2] = upper_32_bits(offset) & 0xff; 1939 1940 r = evergreen_cs_track_check(p); 1941 if (r) { 1942 dev_warn(p->dev, "%s:%d invalid cmd stream\n", __func__, __LINE__); 1943 return r; 1944 } 1945 break; 1946 } 1947 case PACKET3_DRAW_INDEX_AUTO: 1948 if (pkt->count != 1) { 1949 DRM_ERROR("bad DRAW_INDEX_AUTO\n"); 1950 return -EINVAL; 1951 } 1952 r = evergreen_cs_track_check(p); 1953 if (r) { 1954 dev_warn(p->dev, "%s:%d invalid cmd stream %d\n", __func__, __LINE__, idx); 1955 return r; 1956 } 1957 break; 1958 case PACKET3_DRAW_INDEX_MULTI_AUTO: 1959 if (pkt->count != 2) { 1960 DRM_ERROR("bad DRAW_INDEX_MULTI_AUTO\n"); 1961 return -EINVAL; 1962 } 1963 r = evergreen_cs_track_check(p); 1964 if (r) { 1965 dev_warn(p->dev, "%s:%d invalid cmd stream %d\n", __func__, __LINE__, idx); 1966 return r; 1967 } 1968 break; 1969 case PACKET3_DRAW_INDEX_IMMD: 1970 if (pkt->count < 2) { 1971 DRM_ERROR("bad DRAW_INDEX_IMMD\n"); 1972 return -EINVAL; 1973 } 1974 r = evergreen_cs_track_check(p); 1975 if (r) { 1976 dev_warn(p->dev, "%s:%d invalid cmd stream\n", __func__, __LINE__); 1977 return r; 1978 } 1979 break; 1980 case PACKET3_DRAW_INDEX_OFFSET: 1981 if (pkt->count != 2) { 1982 DRM_ERROR("bad DRAW_INDEX_OFFSET\n"); 1983 return -EINVAL; 1984 } 1985 r = evergreen_cs_track_check(p); 1986 if (r) { 1987 dev_warn(p->dev, "%s:%d invalid cmd stream\n", __func__, __LINE__); 1988 return r; 1989 } 1990 break; 1991 case PACKET3_DRAW_INDEX_OFFSET_2: 1992 if (pkt->count != 3) { 1993 DRM_ERROR("bad DRAW_INDEX_OFFSET_2\n"); 1994 return -EINVAL; 1995 } 1996 r = evergreen_cs_track_check(p); 1997 if (r) { 1998 dev_warn(p->dev, "%s:%d invalid cmd stream\n", __func__, __LINE__); 1999 return r; 2000 } 2001 break; 2002 case PACKET3_SET_BASE: 2003 { 2004 /* 2005 DW 1 HEADER Header of the packet. Shader_Type in bit 1 of the Header will correspond to the shader type of the Load, see Type-3 Packet. 2006 2 BASE_INDEX Bits [3:0] BASE_INDEX - Base Index specifies which base address is specified in the last two DWs. 2007 0001: DX11 Draw_Index_Indirect Patch Table Base: Base address for Draw_Index_Indirect data. 2008 3 ADDRESS_LO Bits [31:3] - Lower bits of QWORD-Aligned Address. Bits [2:0] - Reserved 2009 4 ADDRESS_HI Bits [31:8] - Reserved. Bits [7:0] - Upper bits of Address [47:32] 2010 */ 2011 if (pkt->count != 2) { 2012 DRM_ERROR("bad SET_BASE\n"); 2013 return -EINVAL; 2014 } 2015 2016 /* currently only supporting setting indirect draw buffer base address */ 2017 if (idx_value != 1) { 2018 DRM_ERROR("bad SET_BASE\n"); 2019 return -EINVAL; 2020 } 2021 2022 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 2023 if (r) { 2024 DRM_ERROR("bad SET_BASE\n"); 2025 return -EINVAL; 2026 } 2027 2028 track->indirect_draw_buffer_size = radeon_bo_size(reloc->robj); 2029 2030 ib[idx+1] = reloc->gpu_offset; 2031 ib[idx+2] = upper_32_bits(reloc->gpu_offset) & 0xff; 2032 2033 break; 2034 } 2035 case PACKET3_DRAW_INDIRECT: 2036 case PACKET3_DRAW_INDEX_INDIRECT: 2037 { 2038 u64 size = pkt->opcode == PACKET3_DRAW_INDIRECT ? 16 : 20; 2039 2040 /* 2041 DW 1 HEADER 2042 2 DATA_OFFSET Bits [31:0] + byte aligned offset where the required data structure starts. Bits 1:0 are zero 2043 3 DRAW_INITIATOR Draw Initiator Register. Written to the VGT_DRAW_INITIATOR register for the assigned context 2044 */ 2045 if (pkt->count != 1) { 2046 DRM_ERROR("bad DRAW_INDIRECT\n"); 2047 return -EINVAL; 2048 } 2049 2050 if (idx_value + size > track->indirect_draw_buffer_size) { 2051 dev_warn(p->dev, "DRAW_INDIRECT buffer too small %u + %llu > %lu\n", 2052 idx_value, size, track->indirect_draw_buffer_size); 2053 return -EINVAL; 2054 } 2055 2056 r = evergreen_cs_track_check(p); 2057 if (r) { 2058 dev_warn(p->dev, "%s:%d invalid cmd stream\n", __func__, __LINE__); 2059 return r; 2060 } 2061 break; 2062 } 2063 case PACKET3_DISPATCH_DIRECT: 2064 if (pkt->count != 3) { 2065 DRM_ERROR("bad DISPATCH_DIRECT\n"); 2066 return -EINVAL; 2067 } 2068 r = evergreen_cs_track_check(p); 2069 if (r) { 2070 dev_warn(p->dev, "%s:%d invalid cmd stream %d\n", __func__, __LINE__, idx); 2071 return r; 2072 } 2073 break; 2074 case PACKET3_DISPATCH_INDIRECT: 2075 if (pkt->count != 1) { 2076 DRM_ERROR("bad DISPATCH_INDIRECT\n"); 2077 return -EINVAL; 2078 } 2079 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 2080 if (r) { 2081 DRM_ERROR("bad DISPATCH_INDIRECT\n"); 2082 return -EINVAL; 2083 } 2084 ib[idx+0] = idx_value + (u32)(reloc->gpu_offset & 0xffffffff); 2085 r = evergreen_cs_track_check(p); 2086 if (r) { 2087 dev_warn(p->dev, "%s:%d invalid cmd stream\n", __func__, __LINE__); 2088 return r; 2089 } 2090 break; 2091 case PACKET3_WAIT_REG_MEM: 2092 if (pkt->count != 5) { 2093 DRM_ERROR("bad WAIT_REG_MEM\n"); 2094 return -EINVAL; 2095 } 2096 /* bit 4 is reg (0) or mem (1) */ 2097 if (idx_value & 0x10) { 2098 uint64_t offset; 2099 2100 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 2101 if (r) { 2102 DRM_ERROR("bad WAIT_REG_MEM\n"); 2103 return -EINVAL; 2104 } 2105 2106 offset = reloc->gpu_offset + 2107 (radeon_get_ib_value(p, idx+1) & 0xfffffffc) + 2108 ((u64)(radeon_get_ib_value(p, idx+2) & 0xff) << 32); 2109 2110 ib[idx+1] = (ib[idx+1] & 0x3) | (offset & 0xfffffffc); 2111 ib[idx+2] = upper_32_bits(offset) & 0xff; 2112 } else if (idx_value & 0x100) { 2113 DRM_ERROR("cannot use PFP on REG wait\n"); 2114 return -EINVAL; 2115 } 2116 break; 2117 case PACKET3_CP_DMA: 2118 { 2119 u32 command, size, info; 2120 u64 offset, tmp; 2121 if (pkt->count != 4) { 2122 DRM_ERROR("bad CP DMA\n"); 2123 return -EINVAL; 2124 } 2125 command = radeon_get_ib_value(p, idx+4); 2126 size = command & 0x1fffff; 2127 info = radeon_get_ib_value(p, idx+1); 2128 if ((((info & 0x60000000) >> 29) != 0) || /* src = GDS or DATA */ 2129 (((info & 0x00300000) >> 20) != 0) || /* dst = GDS */ 2130 ((((info & 0x00300000) >> 20) == 0) && 2131 (command & PACKET3_CP_DMA_CMD_DAS)) || /* dst = register */ 2132 ((((info & 0x60000000) >> 29) == 0) && 2133 (command & PACKET3_CP_DMA_CMD_SAS))) { /* src = register */ 2134 /* non mem to mem copies requires dw aligned count */ 2135 if (size % 4) { 2136 DRM_ERROR("CP DMA command requires dw count alignment\n"); 2137 return -EINVAL; 2138 } 2139 } 2140 if (command & PACKET3_CP_DMA_CMD_SAS) { 2141 /* src address space is register */ 2142 /* GDS is ok */ 2143 if (((info & 0x60000000) >> 29) != 1) { 2144 DRM_ERROR("CP DMA SAS not supported\n"); 2145 return -EINVAL; 2146 } 2147 } else { 2148 if (command & PACKET3_CP_DMA_CMD_SAIC) { 2149 DRM_ERROR("CP DMA SAIC only supported for registers\n"); 2150 return -EINVAL; 2151 } 2152 /* src address space is memory */ 2153 if (((info & 0x60000000) >> 29) == 0) { 2154 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 2155 if (r) { 2156 DRM_ERROR("bad CP DMA SRC\n"); 2157 return -EINVAL; 2158 } 2159 2160 tmp = radeon_get_ib_value(p, idx) + 2161 ((u64)(radeon_get_ib_value(p, idx+1) & 0xff) << 32); 2162 2163 offset = reloc->gpu_offset + tmp; 2164 2165 if ((tmp + size) > radeon_bo_size(reloc->robj)) { 2166 dev_warn(p->dev, "CP DMA src buffer too small (%llu %lu)\n", 2167 tmp + size, radeon_bo_size(reloc->robj)); 2168 return -EINVAL; 2169 } 2170 2171 ib[idx] = offset; 2172 ib[idx+1] = (ib[idx+1] & 0xffffff00) | (upper_32_bits(offset) & 0xff); 2173 } else if (((info & 0x60000000) >> 29) != 2) { 2174 DRM_ERROR("bad CP DMA SRC_SEL\n"); 2175 return -EINVAL; 2176 } 2177 } 2178 if (command & PACKET3_CP_DMA_CMD_DAS) { 2179 /* dst address space is register */ 2180 /* GDS is ok */ 2181 if (((info & 0x00300000) >> 20) != 1) { 2182 DRM_ERROR("CP DMA DAS not supported\n"); 2183 return -EINVAL; 2184 } 2185 } else { 2186 /* dst address space is memory */ 2187 if (command & PACKET3_CP_DMA_CMD_DAIC) { 2188 DRM_ERROR("CP DMA DAIC only supported for registers\n"); 2189 return -EINVAL; 2190 } 2191 if (((info & 0x00300000) >> 20) == 0) { 2192 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 2193 if (r) { 2194 DRM_ERROR("bad CP DMA DST\n"); 2195 return -EINVAL; 2196 } 2197 2198 tmp = radeon_get_ib_value(p, idx+2) + 2199 ((u64)(radeon_get_ib_value(p, idx+3) & 0xff) << 32); 2200 2201 offset = reloc->gpu_offset + tmp; 2202 2203 if ((tmp + size) > radeon_bo_size(reloc->robj)) { 2204 dev_warn(p->dev, "CP DMA dst buffer too small (%llu %lu)\n", 2205 tmp + size, radeon_bo_size(reloc->robj)); 2206 return -EINVAL; 2207 } 2208 2209 ib[idx+2] = offset; 2210 ib[idx+3] = upper_32_bits(offset) & 0xff; 2211 } else { 2212 DRM_ERROR("bad CP DMA DST_SEL\n"); 2213 return -EINVAL; 2214 } 2215 } 2216 break; 2217 } 2218 case PACKET3_PFP_SYNC_ME: 2219 if (pkt->count) { 2220 DRM_ERROR("bad PFP_SYNC_ME\n"); 2221 return -EINVAL; 2222 } 2223 break; 2224 case PACKET3_SURFACE_SYNC: 2225 if (pkt->count != 3) { 2226 DRM_ERROR("bad SURFACE_SYNC\n"); 2227 return -EINVAL; 2228 } 2229 /* 0xffffffff/0x0 is flush all cache flag */ 2230 if (radeon_get_ib_value(p, idx + 1) != 0xffffffff || 2231 radeon_get_ib_value(p, idx + 2) != 0) { 2232 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 2233 if (r) { 2234 DRM_ERROR("bad SURFACE_SYNC\n"); 2235 return -EINVAL; 2236 } 2237 ib[idx+2] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff); 2238 } 2239 break; 2240 case PACKET3_EVENT_WRITE: 2241 if (pkt->count != 2 && pkt->count != 0) { 2242 DRM_ERROR("bad EVENT_WRITE\n"); 2243 return -EINVAL; 2244 } 2245 if (pkt->count) { 2246 uint64_t offset; 2247 2248 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 2249 if (r) { 2250 DRM_ERROR("bad EVENT_WRITE\n"); 2251 return -EINVAL; 2252 } 2253 offset = reloc->gpu_offset + 2254 (radeon_get_ib_value(p, idx+1) & 0xfffffff8) + 2255 ((u64)(radeon_get_ib_value(p, idx+2) & 0xff) << 32); 2256 2257 ib[idx+1] = offset & 0xfffffff8; 2258 ib[idx+2] = upper_32_bits(offset) & 0xff; 2259 } 2260 break; 2261 case PACKET3_EVENT_WRITE_EOP: 2262 { 2263 uint64_t offset; 2264 2265 if (pkt->count != 4) { 2266 DRM_ERROR("bad EVENT_WRITE_EOP\n"); 2267 return -EINVAL; 2268 } 2269 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 2270 if (r) { 2271 DRM_ERROR("bad EVENT_WRITE_EOP\n"); 2272 return -EINVAL; 2273 } 2274 2275 offset = reloc->gpu_offset + 2276 (radeon_get_ib_value(p, idx+1) & 0xfffffffc) + 2277 ((u64)(radeon_get_ib_value(p, idx+2) & 0xff) << 32); 2278 2279 ib[idx+1] = offset & 0xfffffffc; 2280 ib[idx+2] = (ib[idx+2] & 0xffffff00) | (upper_32_bits(offset) & 0xff); 2281 break; 2282 } 2283 case PACKET3_EVENT_WRITE_EOS: 2284 { 2285 uint64_t offset; 2286 2287 if (pkt->count != 3) { 2288 DRM_ERROR("bad EVENT_WRITE_EOS\n"); 2289 return -EINVAL; 2290 } 2291 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 2292 if (r) { 2293 DRM_ERROR("bad EVENT_WRITE_EOS\n"); 2294 return -EINVAL; 2295 } 2296 2297 offset = reloc->gpu_offset + 2298 (radeon_get_ib_value(p, idx+1) & 0xfffffffc) + 2299 ((u64)(radeon_get_ib_value(p, idx+2) & 0xff) << 32); 2300 2301 ib[idx+1] = offset & 0xfffffffc; 2302 ib[idx+2] = (ib[idx+2] & 0xffffff00) | (upper_32_bits(offset) & 0xff); 2303 break; 2304 } 2305 case PACKET3_SET_CONFIG_REG: 2306 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START; 2307 end_reg = 4 * pkt->count + start_reg - 4; 2308 if ((start_reg < PACKET3_SET_CONFIG_REG_START) || 2309 (start_reg >= PACKET3_SET_CONFIG_REG_END) || 2310 (end_reg >= PACKET3_SET_CONFIG_REG_END)) { 2311 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n"); 2312 return -EINVAL; 2313 } 2314 for (reg = start_reg, idx++; reg <= end_reg; reg += 4, idx++) { 2315 if (evergreen_is_safe_reg(p, reg)) 2316 continue; 2317 r = evergreen_cs_handle_reg(p, reg, idx); 2318 if (r) 2319 return r; 2320 } 2321 break; 2322 case PACKET3_SET_CONTEXT_REG: 2323 start_reg = (idx_value << 2) + PACKET3_SET_CONTEXT_REG_START; 2324 end_reg = 4 * pkt->count + start_reg - 4; 2325 if ((start_reg < PACKET3_SET_CONTEXT_REG_START) || 2326 (start_reg >= PACKET3_SET_CONTEXT_REG_END) || 2327 (end_reg >= PACKET3_SET_CONTEXT_REG_END)) { 2328 DRM_ERROR("bad PACKET3_SET_CONTEXT_REG\n"); 2329 return -EINVAL; 2330 } 2331 for (reg = start_reg, idx++; reg <= end_reg; reg += 4, idx++) { 2332 if (evergreen_is_safe_reg(p, reg)) 2333 continue; 2334 r = evergreen_cs_handle_reg(p, reg, idx); 2335 if (r) 2336 return r; 2337 } 2338 break; 2339 case PACKET3_SET_RESOURCE: 2340 if (pkt->count % 8) { 2341 DRM_ERROR("bad SET_RESOURCE\n"); 2342 return -EINVAL; 2343 } 2344 start_reg = (idx_value << 2) + PACKET3_SET_RESOURCE_START; 2345 end_reg = 4 * pkt->count + start_reg - 4; 2346 if ((start_reg < PACKET3_SET_RESOURCE_START) || 2347 (start_reg >= PACKET3_SET_RESOURCE_END) || 2348 (end_reg >= PACKET3_SET_RESOURCE_END)) { 2349 DRM_ERROR("bad SET_RESOURCE\n"); 2350 return -EINVAL; 2351 } 2352 for (i = 0; i < (pkt->count / 8); i++) { 2353 struct radeon_bo *texture, *mipmap; 2354 u32 toffset, moffset; 2355 u32 size, offset, mip_address, tex_dim; 2356 2357 switch (G__SQ_CONSTANT_TYPE(radeon_get_ib_value(p, idx+1+(i*8)+7))) { 2358 case SQ_TEX_VTX_VALID_TEXTURE: 2359 /* tex base */ 2360 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 2361 if (r) { 2362 DRM_ERROR("bad SET_RESOURCE (tex)\n"); 2363 return -EINVAL; 2364 } 2365 if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) { 2366 ib[idx+1+(i*8)+1] |= 2367 TEX_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->tiling_flags)); 2368 if (reloc->tiling_flags & RADEON_TILING_MACRO) { 2369 unsigned bankw, bankh, mtaspect, tile_split; 2370 2371 evergreen_tiling_fields(reloc->tiling_flags, 2372 &bankw, &bankh, &mtaspect, 2373 &tile_split); 2374 ib[idx+1+(i*8)+6] |= TEX_TILE_SPLIT(tile_split); 2375 ib[idx+1+(i*8)+7] |= 2376 TEX_BANK_WIDTH(bankw) | 2377 TEX_BANK_HEIGHT(bankh) | 2378 MACRO_TILE_ASPECT(mtaspect) | 2379 TEX_NUM_BANKS(evergreen_cs_get_num_banks(track->nbanks)); 2380 } 2381 } 2382 texture = reloc->robj; 2383 toffset = (u32)((reloc->gpu_offset >> 8) & 0xffffffff); 2384 2385 /* tex mip base */ 2386 tex_dim = ib[idx+1+(i*8)+0] & 0x7; 2387 mip_address = ib[idx+1+(i*8)+3]; 2388 2389 if ((tex_dim == SQ_TEX_DIM_2D_MSAA || tex_dim == SQ_TEX_DIM_2D_ARRAY_MSAA) && 2390 !mip_address && 2391 !radeon_cs_packet_next_is_pkt3_nop(p)) { 2392 /* MIP_ADDRESS should point to FMASK for an MSAA texture. 2393 * It should be 0 if FMASK is disabled. */ 2394 moffset = 0; 2395 mipmap = NULL; 2396 } else { 2397 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 2398 if (r) { 2399 DRM_ERROR("bad SET_RESOURCE (tex)\n"); 2400 return -EINVAL; 2401 } 2402 moffset = (u32)((reloc->gpu_offset >> 8) & 0xffffffff); 2403 mipmap = reloc->robj; 2404 } 2405 2406 r = evergreen_cs_track_validate_texture(p, texture, mipmap, idx+1+(i*8)); 2407 if (r) 2408 return r; 2409 ib[idx+1+(i*8)+2] += toffset; 2410 ib[idx+1+(i*8)+3] += moffset; 2411 break; 2412 case SQ_TEX_VTX_VALID_BUFFER: 2413 { 2414 uint64_t offset64; 2415 /* vtx base */ 2416 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 2417 if (r) { 2418 DRM_ERROR("bad SET_RESOURCE (vtx)\n"); 2419 return -EINVAL; 2420 } 2421 offset = radeon_get_ib_value(p, idx+1+(i*8)+0); 2422 size = radeon_get_ib_value(p, idx+1+(i*8)+1); 2423 if (p->rdev && (size + offset) > radeon_bo_size(reloc->robj)) { 2424 /* force size to size of the buffer */ 2425 dev_warn_ratelimited(p->dev, "vbo resource seems too big for the bo\n"); 2426 ib[idx+1+(i*8)+1] = radeon_bo_size(reloc->robj) - offset; 2427 } 2428 2429 offset64 = reloc->gpu_offset + offset; 2430 ib[idx+1+(i*8)+0] = offset64; 2431 ib[idx+1+(i*8)+2] = (ib[idx+1+(i*8)+2] & 0xffffff00) | 2432 (upper_32_bits(offset64) & 0xff); 2433 break; 2434 } 2435 case SQ_TEX_VTX_INVALID_TEXTURE: 2436 case SQ_TEX_VTX_INVALID_BUFFER: 2437 default: 2438 DRM_ERROR("bad SET_RESOURCE\n"); 2439 return -EINVAL; 2440 } 2441 } 2442 break; 2443 case PACKET3_SET_ALU_CONST: 2444 /* XXX fix me ALU const buffers only */ 2445 break; 2446 case PACKET3_SET_BOOL_CONST: 2447 start_reg = (idx_value << 2) + PACKET3_SET_BOOL_CONST_START; 2448 end_reg = 4 * pkt->count + start_reg - 4; 2449 if ((start_reg < PACKET3_SET_BOOL_CONST_START) || 2450 (start_reg >= PACKET3_SET_BOOL_CONST_END) || 2451 (end_reg >= PACKET3_SET_BOOL_CONST_END)) { 2452 DRM_ERROR("bad SET_BOOL_CONST\n"); 2453 return -EINVAL; 2454 } 2455 break; 2456 case PACKET3_SET_LOOP_CONST: 2457 start_reg = (idx_value << 2) + PACKET3_SET_LOOP_CONST_START; 2458 end_reg = 4 * pkt->count + start_reg - 4; 2459 if ((start_reg < PACKET3_SET_LOOP_CONST_START) || 2460 (start_reg >= PACKET3_SET_LOOP_CONST_END) || 2461 (end_reg >= PACKET3_SET_LOOP_CONST_END)) { 2462 DRM_ERROR("bad SET_LOOP_CONST\n"); 2463 return -EINVAL; 2464 } 2465 break; 2466 case PACKET3_SET_CTL_CONST: 2467 start_reg = (idx_value << 2) + PACKET3_SET_CTL_CONST_START; 2468 end_reg = 4 * pkt->count + start_reg - 4; 2469 if ((start_reg < PACKET3_SET_CTL_CONST_START) || 2470 (start_reg >= PACKET3_SET_CTL_CONST_END) || 2471 (end_reg >= PACKET3_SET_CTL_CONST_END)) { 2472 DRM_ERROR("bad SET_CTL_CONST\n"); 2473 return -EINVAL; 2474 } 2475 break; 2476 case PACKET3_SET_SAMPLER: 2477 if (pkt->count % 3) { 2478 DRM_ERROR("bad SET_SAMPLER\n"); 2479 return -EINVAL; 2480 } 2481 start_reg = (idx_value << 2) + PACKET3_SET_SAMPLER_START; 2482 end_reg = 4 * pkt->count + start_reg - 4; 2483 if ((start_reg < PACKET3_SET_SAMPLER_START) || 2484 (start_reg >= PACKET3_SET_SAMPLER_END) || 2485 (end_reg >= PACKET3_SET_SAMPLER_END)) { 2486 DRM_ERROR("bad SET_SAMPLER\n"); 2487 return -EINVAL; 2488 } 2489 break; 2490 case PACKET3_STRMOUT_BUFFER_UPDATE: 2491 if (pkt->count != 4) { 2492 DRM_ERROR("bad STRMOUT_BUFFER_UPDATE (invalid count)\n"); 2493 return -EINVAL; 2494 } 2495 /* Updating memory at DST_ADDRESS. */ 2496 if (idx_value & 0x1) { 2497 u64 offset; 2498 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 2499 if (r) { 2500 DRM_ERROR("bad STRMOUT_BUFFER_UPDATE (missing dst reloc)\n"); 2501 return -EINVAL; 2502 } 2503 offset = radeon_get_ib_value(p, idx+1); 2504 offset += ((u64)(radeon_get_ib_value(p, idx+2) & 0xff)) << 32; 2505 if ((offset + 4) > radeon_bo_size(reloc->robj)) { 2506 DRM_ERROR("bad STRMOUT_BUFFER_UPDATE dst bo too small: 0x%llx, 0x%lx\n", 2507 offset + 4, radeon_bo_size(reloc->robj)); 2508 return -EINVAL; 2509 } 2510 offset += reloc->gpu_offset; 2511 ib[idx+1] = offset; 2512 ib[idx+2] = upper_32_bits(offset) & 0xff; 2513 } 2514 /* Reading data from SRC_ADDRESS. */ 2515 if (((idx_value >> 1) & 0x3) == 2) { 2516 u64 offset; 2517 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 2518 if (r) { 2519 DRM_ERROR("bad STRMOUT_BUFFER_UPDATE (missing src reloc)\n"); 2520 return -EINVAL; 2521 } 2522 offset = radeon_get_ib_value(p, idx+3); 2523 offset += ((u64)(radeon_get_ib_value(p, idx+4) & 0xff)) << 32; 2524 if ((offset + 4) > radeon_bo_size(reloc->robj)) { 2525 DRM_ERROR("bad STRMOUT_BUFFER_UPDATE src bo too small: 0x%llx, 0x%lx\n", 2526 offset + 4, radeon_bo_size(reloc->robj)); 2527 return -EINVAL; 2528 } 2529 offset += reloc->gpu_offset; 2530 ib[idx+3] = offset; 2531 ib[idx+4] = upper_32_bits(offset) & 0xff; 2532 } 2533 break; 2534 case PACKET3_MEM_WRITE: 2535 { 2536 u64 offset; 2537 2538 if (pkt->count != 3) { 2539 DRM_ERROR("bad MEM_WRITE (invalid count)\n"); 2540 return -EINVAL; 2541 } 2542 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 2543 if (r) { 2544 DRM_ERROR("bad MEM_WRITE (missing reloc)\n"); 2545 return -EINVAL; 2546 } 2547 offset = radeon_get_ib_value(p, idx+0); 2548 offset += ((u64)(radeon_get_ib_value(p, idx+1) & 0xff)) << 32UL; 2549 if (offset & 0x7) { 2550 DRM_ERROR("bad MEM_WRITE (address not qwords aligned)\n"); 2551 return -EINVAL; 2552 } 2553 if ((offset + 8) > radeon_bo_size(reloc->robj)) { 2554 DRM_ERROR("bad MEM_WRITE bo too small: 0x%llx, 0x%lx\n", 2555 offset + 8, radeon_bo_size(reloc->robj)); 2556 return -EINVAL; 2557 } 2558 offset += reloc->gpu_offset; 2559 ib[idx+0] = offset; 2560 ib[idx+1] = upper_32_bits(offset) & 0xff; 2561 break; 2562 } 2563 case PACKET3_COPY_DW: 2564 if (pkt->count != 4) { 2565 DRM_ERROR("bad COPY_DW (invalid count)\n"); 2566 return -EINVAL; 2567 } 2568 if (idx_value & 0x1) { 2569 u64 offset; 2570 /* SRC is memory. */ 2571 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 2572 if (r) { 2573 DRM_ERROR("bad COPY_DW (missing src reloc)\n"); 2574 return -EINVAL; 2575 } 2576 offset = radeon_get_ib_value(p, idx+1); 2577 offset += ((u64)(radeon_get_ib_value(p, idx+2) & 0xff)) << 32; 2578 if ((offset + 4) > radeon_bo_size(reloc->robj)) { 2579 DRM_ERROR("bad COPY_DW src bo too small: 0x%llx, 0x%lx\n", 2580 offset + 4, radeon_bo_size(reloc->robj)); 2581 return -EINVAL; 2582 } 2583 offset += reloc->gpu_offset; 2584 ib[idx+1] = offset; 2585 ib[idx+2] = upper_32_bits(offset) & 0xff; 2586 } else { 2587 /* SRC is a reg. */ 2588 reg = radeon_get_ib_value(p, idx+1) << 2; 2589 if (!evergreen_is_safe_reg(p, reg)) { 2590 dev_warn(p->dev, "forbidden register 0x%08x at %d\n", 2591 reg, idx + 1); 2592 return -EINVAL; 2593 } 2594 } 2595 if (idx_value & 0x2) { 2596 u64 offset; 2597 /* DST is memory. */ 2598 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 2599 if (r) { 2600 DRM_ERROR("bad COPY_DW (missing dst reloc)\n"); 2601 return -EINVAL; 2602 } 2603 offset = radeon_get_ib_value(p, idx+3); 2604 offset += ((u64)(radeon_get_ib_value(p, idx+4) & 0xff)) << 32; 2605 if ((offset + 4) > radeon_bo_size(reloc->robj)) { 2606 DRM_ERROR("bad COPY_DW dst bo too small: 0x%llx, 0x%lx\n", 2607 offset + 4, radeon_bo_size(reloc->robj)); 2608 return -EINVAL; 2609 } 2610 offset += reloc->gpu_offset; 2611 ib[idx+3] = offset; 2612 ib[idx+4] = upper_32_bits(offset) & 0xff; 2613 } else { 2614 /* DST is a reg. */ 2615 reg = radeon_get_ib_value(p, idx+3) << 2; 2616 if (!evergreen_is_safe_reg(p, reg)) { 2617 dev_warn(p->dev, "forbidden register 0x%08x at %d\n", 2618 reg, idx + 3); 2619 return -EINVAL; 2620 } 2621 } 2622 break; 2623 case PACKET3_SET_APPEND_CNT: 2624 { 2625 uint32_t areg; 2626 uint32_t allowed_reg_base; 2627 uint32_t source_sel; 2628 if (pkt->count != 2) { 2629 DRM_ERROR("bad SET_APPEND_CNT (invalid count)\n"); 2630 return -EINVAL; 2631 } 2632 2633 allowed_reg_base = GDS_APPEND_COUNT_0; 2634 allowed_reg_base -= PACKET3_SET_CONTEXT_REG_START; 2635 allowed_reg_base >>= 2; 2636 2637 areg = idx_value >> 16; 2638 if (areg < allowed_reg_base || areg > (allowed_reg_base + 11)) { 2639 dev_warn(p->dev, "forbidden register for append cnt 0x%08x at %d\n", 2640 areg, idx); 2641 return -EINVAL; 2642 } 2643 2644 source_sel = G_PACKET3_SET_APPEND_CNT_SRC_SELECT(idx_value); 2645 if (source_sel == PACKET3_SAC_SRC_SEL_MEM) { 2646 uint64_t offset; 2647 uint32_t swap; 2648 r = radeon_cs_packet_next_reloc(p, &reloc, 0); 2649 if (r) { 2650 DRM_ERROR("bad SET_APPEND_CNT (missing reloc)\n"); 2651 return -EINVAL; 2652 } 2653 offset = radeon_get_ib_value(p, idx + 1); 2654 swap = offset & 0x3; 2655 offset &= ~0x3; 2656 2657 offset += ((u64)(radeon_get_ib_value(p, idx + 2) & 0xff)) << 32; 2658 2659 offset += reloc->gpu_offset; 2660 ib[idx+1] = (offset & 0xfffffffc) | swap; 2661 ib[idx+2] = upper_32_bits(offset) & 0xff; 2662 } else { 2663 DRM_ERROR("bad SET_APPEND_CNT (unsupported operation)\n"); 2664 return -EINVAL; 2665 } 2666 break; 2667 } 2668 case PACKET3_NOP: 2669 break; 2670 default: 2671 DRM_ERROR("Packet3 opcode %x not supported\n", pkt->opcode); 2672 return -EINVAL; 2673 } 2674 return 0; 2675 } 2676 2677 int evergreen_cs_parse(struct radeon_cs_parser *p) 2678 { 2679 struct radeon_cs_packet pkt; 2680 struct evergreen_cs_track *track; 2681 u32 tmp; 2682 int r; 2683 2684 if (p->track == NULL) { 2685 /* initialize tracker, we are in kms */ 2686 track = kzalloc(sizeof(*track), GFP_KERNEL); 2687 if (track == NULL) 2688 return -ENOMEM; 2689 evergreen_cs_track_init(track); 2690 if (p->rdev->family >= CHIP_CAYMAN) { 2691 tmp = p->rdev->config.cayman.tile_config; 2692 track->reg_safe_bm = cayman_reg_safe_bm; 2693 } else { 2694 tmp = p->rdev->config.evergreen.tile_config; 2695 track->reg_safe_bm = evergreen_reg_safe_bm; 2696 } 2697 BUILD_BUG_ON(ARRAY_SIZE(cayman_reg_safe_bm) != REG_SAFE_BM_SIZE); 2698 BUILD_BUG_ON(ARRAY_SIZE(evergreen_reg_safe_bm) != REG_SAFE_BM_SIZE); 2699 switch (tmp & 0xf) { 2700 case 0: 2701 track->npipes = 1; 2702 break; 2703 case 1: 2704 default: 2705 track->npipes = 2; 2706 break; 2707 case 2: 2708 track->npipes = 4; 2709 break; 2710 case 3: 2711 track->npipes = 8; 2712 break; 2713 } 2714 2715 switch ((tmp & 0xf0) >> 4) { 2716 case 0: 2717 track->nbanks = 4; 2718 break; 2719 case 1: 2720 default: 2721 track->nbanks = 8; 2722 break; 2723 case 2: 2724 track->nbanks = 16; 2725 break; 2726 } 2727 2728 switch ((tmp & 0xf00) >> 8) { 2729 case 0: 2730 track->group_size = 256; 2731 break; 2732 case 1: 2733 default: 2734 track->group_size = 512; 2735 break; 2736 } 2737 2738 switch ((tmp & 0xf000) >> 12) { 2739 case 0: 2740 track->row_size = 1; 2741 break; 2742 case 1: 2743 default: 2744 track->row_size = 2; 2745 break; 2746 case 2: 2747 track->row_size = 4; 2748 break; 2749 } 2750 2751 p->track = track; 2752 } 2753 do { 2754 r = radeon_cs_packet_parse(p, &pkt, p->idx); 2755 if (r) { 2756 kfree(p->track); 2757 p->track = NULL; 2758 return r; 2759 } 2760 p->idx += pkt.count + 2; 2761 switch (pkt.type) { 2762 case RADEON_PACKET_TYPE0: 2763 r = evergreen_cs_parse_packet0(p, &pkt); 2764 break; 2765 case RADEON_PACKET_TYPE2: 2766 break; 2767 case RADEON_PACKET_TYPE3: 2768 r = evergreen_packet3_check(p, &pkt); 2769 break; 2770 default: 2771 DRM_ERROR("Unknown packet type %d !\n", pkt.type); 2772 kfree(p->track); 2773 p->track = NULL; 2774 return -EINVAL; 2775 } 2776 if (r) { 2777 kfree(p->track); 2778 p->track = NULL; 2779 return r; 2780 } 2781 } while (p->idx < p->chunk_ib->length_dw); 2782 #if 0 2783 for (r = 0; r < p->ib.length_dw; r++) { 2784 pr_info("%05d 0x%08X\n", r, p->ib.ptr[r]); 2785 mdelay(1); 2786 } 2787 #endif 2788 kfree(p->track); 2789 p->track = NULL; 2790 return 0; 2791 } 2792 2793 /** 2794 * evergreen_dma_cs_parse() - parse the DMA IB 2795 * @p: parser structure holding parsing context. 2796 * 2797 * Parses the DMA IB from the CS ioctl and updates 2798 * the GPU addresses based on the reloc information and 2799 * checks for errors. (Evergreen-Cayman) 2800 * Returns 0 for success and an error on failure. 2801 **/ 2802 int evergreen_dma_cs_parse(struct radeon_cs_parser *p) 2803 { 2804 struct radeon_cs_chunk *ib_chunk = p->chunk_ib; 2805 struct radeon_bo_list *src_reloc, *dst_reloc, *dst2_reloc; 2806 u32 header, cmd, count, sub_cmd; 2807 uint32_t *ib = p->ib.ptr; 2808 u32 idx; 2809 u64 src_offset, dst_offset, dst2_offset; 2810 int r; 2811 2812 do { 2813 if (p->idx >= ib_chunk->length_dw) { 2814 DRM_ERROR("Can not parse packet at %d after CS end %d !\n", 2815 p->idx, ib_chunk->length_dw); 2816 return -EINVAL; 2817 } 2818 idx = p->idx; 2819 header = radeon_get_ib_value(p, idx); 2820 cmd = GET_DMA_CMD(header); 2821 count = GET_DMA_COUNT(header); 2822 sub_cmd = GET_DMA_SUB_CMD(header); 2823 2824 switch (cmd) { 2825 case DMA_PACKET_WRITE: 2826 r = r600_dma_cs_next_reloc(p, &dst_reloc); 2827 if (r) { 2828 DRM_ERROR("bad DMA_PACKET_WRITE\n"); 2829 return -EINVAL; 2830 } 2831 switch (sub_cmd) { 2832 /* tiled */ 2833 case 8: 2834 dst_offset = radeon_get_ib_value(p, idx+1); 2835 dst_offset <<= 8; 2836 2837 ib[idx+1] += (u32)(dst_reloc->gpu_offset >> 8); 2838 p->idx += count + 7; 2839 break; 2840 /* linear */ 2841 case 0: 2842 dst_offset = radeon_get_ib_value(p, idx+1); 2843 dst_offset |= ((u64)(radeon_get_ib_value(p, idx+2) & 0xff)) << 32; 2844 2845 ib[idx+1] += (u32)(dst_reloc->gpu_offset & 0xfffffffc); 2846 ib[idx+2] += upper_32_bits(dst_reloc->gpu_offset) & 0xff; 2847 p->idx += count + 3; 2848 break; 2849 default: 2850 DRM_ERROR("bad DMA_PACKET_WRITE [%6d] 0x%08x sub cmd is not 0 or 8\n", idx, header); 2851 return -EINVAL; 2852 } 2853 if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) { 2854 dev_warn(p->dev, "DMA write buffer too small (%llu %lu)\n", 2855 dst_offset, radeon_bo_size(dst_reloc->robj)); 2856 return -EINVAL; 2857 } 2858 break; 2859 case DMA_PACKET_COPY: 2860 r = r600_dma_cs_next_reloc(p, &src_reloc); 2861 if (r) { 2862 DRM_ERROR("bad DMA_PACKET_COPY\n"); 2863 return -EINVAL; 2864 } 2865 r = r600_dma_cs_next_reloc(p, &dst_reloc); 2866 if (r) { 2867 DRM_ERROR("bad DMA_PACKET_COPY\n"); 2868 return -EINVAL; 2869 } 2870 switch (sub_cmd) { 2871 /* Copy L2L, DW aligned */ 2872 case 0x00: 2873 /* L2L, dw */ 2874 src_offset = radeon_get_ib_value(p, idx+2); 2875 src_offset |= ((u64)(radeon_get_ib_value(p, idx+4) & 0xff)) << 32; 2876 dst_offset = radeon_get_ib_value(p, idx+1); 2877 dst_offset |= ((u64)(radeon_get_ib_value(p, idx+3) & 0xff)) << 32; 2878 if ((src_offset + (count * 4)) > radeon_bo_size(src_reloc->robj)) { 2879 dev_warn(p->dev, "DMA L2L, dw src buffer too small (%llu %lu)\n", 2880 src_offset + (count * 4), radeon_bo_size(src_reloc->robj)); 2881 return -EINVAL; 2882 } 2883 if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) { 2884 dev_warn(p->dev, "DMA L2L, dw dst buffer too small (%llu %lu)\n", 2885 dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj)); 2886 return -EINVAL; 2887 } 2888 ib[idx+1] += (u32)(dst_reloc->gpu_offset & 0xfffffffc); 2889 ib[idx+2] += (u32)(src_reloc->gpu_offset & 0xfffffffc); 2890 ib[idx+3] += upper_32_bits(dst_reloc->gpu_offset) & 0xff; 2891 ib[idx+4] += upper_32_bits(src_reloc->gpu_offset) & 0xff; 2892 p->idx += 5; 2893 break; 2894 /* Copy L2T/T2L */ 2895 case 0x08: 2896 /* detile bit */ 2897 if (radeon_get_ib_value(p, idx + 2) & (1 << 31)) { 2898 /* tiled src, linear dst */ 2899 src_offset = radeon_get_ib_value(p, idx+1); 2900 src_offset <<= 8; 2901 ib[idx+1] += (u32)(src_reloc->gpu_offset >> 8); 2902 2903 dst_offset = radeon_get_ib_value(p, idx + 7); 2904 dst_offset |= ((u64)(radeon_get_ib_value(p, idx+8) & 0xff)) << 32; 2905 ib[idx+7] += (u32)(dst_reloc->gpu_offset & 0xfffffffc); 2906 ib[idx+8] += upper_32_bits(dst_reloc->gpu_offset) & 0xff; 2907 } else { 2908 /* linear src, tiled dst */ 2909 src_offset = radeon_get_ib_value(p, idx+7); 2910 src_offset |= ((u64)(radeon_get_ib_value(p, idx+8) & 0xff)) << 32; 2911 ib[idx+7] += (u32)(src_reloc->gpu_offset & 0xfffffffc); 2912 ib[idx+8] += upper_32_bits(src_reloc->gpu_offset) & 0xff; 2913 2914 dst_offset = radeon_get_ib_value(p, idx+1); 2915 dst_offset <<= 8; 2916 ib[idx+1] += (u32)(dst_reloc->gpu_offset >> 8); 2917 } 2918 if ((src_offset + (count * 4)) > radeon_bo_size(src_reloc->robj)) { 2919 dev_warn(p->dev, "DMA L2T, src buffer too small (%llu %lu)\n", 2920 src_offset + (count * 4), radeon_bo_size(src_reloc->robj)); 2921 return -EINVAL; 2922 } 2923 if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) { 2924 dev_warn(p->dev, "DMA L2T, dst buffer too small (%llu %lu)\n", 2925 dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj)); 2926 return -EINVAL; 2927 } 2928 p->idx += 9; 2929 break; 2930 /* Copy L2L, byte aligned */ 2931 case 0x40: 2932 /* L2L, byte */ 2933 src_offset = radeon_get_ib_value(p, idx+2); 2934 src_offset |= ((u64)(radeon_get_ib_value(p, idx+4) & 0xff)) << 32; 2935 dst_offset = radeon_get_ib_value(p, idx+1); 2936 dst_offset |= ((u64)(radeon_get_ib_value(p, idx+3) & 0xff)) << 32; 2937 if ((src_offset + count) > radeon_bo_size(src_reloc->robj)) { 2938 dev_warn(p->dev, "DMA L2L, byte src buffer too small (%llu %lu)\n", 2939 src_offset + count, radeon_bo_size(src_reloc->robj)); 2940 return -EINVAL; 2941 } 2942 if ((dst_offset + count) > radeon_bo_size(dst_reloc->robj)) { 2943 dev_warn(p->dev, "DMA L2L, byte dst buffer too small (%llu %lu)\n", 2944 dst_offset + count, radeon_bo_size(dst_reloc->robj)); 2945 return -EINVAL; 2946 } 2947 ib[idx+1] += (u32)(dst_reloc->gpu_offset & 0xffffffff); 2948 ib[idx+2] += (u32)(src_reloc->gpu_offset & 0xffffffff); 2949 ib[idx+3] += upper_32_bits(dst_reloc->gpu_offset) & 0xff; 2950 ib[idx+4] += upper_32_bits(src_reloc->gpu_offset) & 0xff; 2951 p->idx += 5; 2952 break; 2953 /* Copy L2L, partial */ 2954 case 0x41: 2955 /* L2L, partial */ 2956 if (p->family < CHIP_CAYMAN) { 2957 DRM_ERROR("L2L Partial is cayman only !\n"); 2958 return -EINVAL; 2959 } 2960 ib[idx+1] += (u32)(src_reloc->gpu_offset & 0xffffffff); 2961 ib[idx+2] += upper_32_bits(src_reloc->gpu_offset) & 0xff; 2962 ib[idx+4] += (u32)(dst_reloc->gpu_offset & 0xffffffff); 2963 ib[idx+5] += upper_32_bits(dst_reloc->gpu_offset) & 0xff; 2964 2965 p->idx += 9; 2966 break; 2967 /* Copy L2L, DW aligned, broadcast */ 2968 case 0x44: 2969 /* L2L, dw, broadcast */ 2970 r = r600_dma_cs_next_reloc(p, &dst2_reloc); 2971 if (r) { 2972 DRM_ERROR("bad L2L, dw, broadcast DMA_PACKET_COPY\n"); 2973 return -EINVAL; 2974 } 2975 dst_offset = radeon_get_ib_value(p, idx+1); 2976 dst_offset |= ((u64)(radeon_get_ib_value(p, idx+4) & 0xff)) << 32; 2977 dst2_offset = radeon_get_ib_value(p, idx+2); 2978 dst2_offset |= ((u64)(radeon_get_ib_value(p, idx+5) & 0xff)) << 32; 2979 src_offset = radeon_get_ib_value(p, idx+3); 2980 src_offset |= ((u64)(radeon_get_ib_value(p, idx+6) & 0xff)) << 32; 2981 if ((src_offset + (count * 4)) > radeon_bo_size(src_reloc->robj)) { 2982 dev_warn(p->dev, "DMA L2L, dw, broadcast src buffer too small (%llu %lu)\n", 2983 src_offset + (count * 4), radeon_bo_size(src_reloc->robj)); 2984 return -EINVAL; 2985 } 2986 if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) { 2987 dev_warn(p->dev, "DMA L2L, dw, broadcast dst buffer too small (%llu %lu)\n", 2988 dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj)); 2989 return -EINVAL; 2990 } 2991 if ((dst2_offset + (count * 4)) > radeon_bo_size(dst2_reloc->robj)) { 2992 dev_warn(p->dev, "DMA L2L, dw, broadcast dst2 buffer too small (%llu %lu)\n", 2993 dst2_offset + (count * 4), radeon_bo_size(dst2_reloc->robj)); 2994 return -EINVAL; 2995 } 2996 ib[idx+1] += (u32)(dst_reloc->gpu_offset & 0xfffffffc); 2997 ib[idx+2] += (u32)(dst2_reloc->gpu_offset & 0xfffffffc); 2998 ib[idx+3] += (u32)(src_reloc->gpu_offset & 0xfffffffc); 2999 ib[idx+4] += upper_32_bits(dst_reloc->gpu_offset) & 0xff; 3000 ib[idx+5] += upper_32_bits(dst2_reloc->gpu_offset) & 0xff; 3001 ib[idx+6] += upper_32_bits(src_reloc->gpu_offset) & 0xff; 3002 p->idx += 7; 3003 break; 3004 /* Copy L2T Frame to Field */ 3005 case 0x48: 3006 if (radeon_get_ib_value(p, idx + 2) & (1 << 31)) { 3007 DRM_ERROR("bad L2T, frame to fields DMA_PACKET_COPY\n"); 3008 return -EINVAL; 3009 } 3010 r = r600_dma_cs_next_reloc(p, &dst2_reloc); 3011 if (r) { 3012 DRM_ERROR("bad L2T, frame to fields DMA_PACKET_COPY\n"); 3013 return -EINVAL; 3014 } 3015 dst_offset = radeon_get_ib_value(p, idx+1); 3016 dst_offset <<= 8; 3017 dst2_offset = radeon_get_ib_value(p, idx+2); 3018 dst2_offset <<= 8; 3019 src_offset = radeon_get_ib_value(p, idx+8); 3020 src_offset |= ((u64)(radeon_get_ib_value(p, idx+9) & 0xff)) << 32; 3021 if ((src_offset + (count * 4)) > radeon_bo_size(src_reloc->robj)) { 3022 dev_warn(p->dev, "DMA L2T, frame to fields src buffer too small (%llu %lu)\n", 3023 src_offset + (count * 4), radeon_bo_size(src_reloc->robj)); 3024 return -EINVAL; 3025 } 3026 if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) { 3027 dev_warn(p->dev, "DMA L2T, frame to fields buffer too small (%llu %lu)\n", 3028 dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj)); 3029 return -EINVAL; 3030 } 3031 if ((dst2_offset + (count * 4)) > radeon_bo_size(dst2_reloc->robj)) { 3032 dev_warn(p->dev, "DMA L2T, frame to fields buffer too small (%llu %lu)\n", 3033 dst2_offset + (count * 4), radeon_bo_size(dst2_reloc->robj)); 3034 return -EINVAL; 3035 } 3036 ib[idx+1] += (u32)(dst_reloc->gpu_offset >> 8); 3037 ib[idx+2] += (u32)(dst2_reloc->gpu_offset >> 8); 3038 ib[idx+8] += (u32)(src_reloc->gpu_offset & 0xfffffffc); 3039 ib[idx+9] += upper_32_bits(src_reloc->gpu_offset) & 0xff; 3040 p->idx += 10; 3041 break; 3042 /* Copy L2T/T2L, partial */ 3043 case 0x49: 3044 /* L2T, T2L partial */ 3045 if (p->family < CHIP_CAYMAN) { 3046 DRM_ERROR("L2T, T2L Partial is cayman only !\n"); 3047 return -EINVAL; 3048 } 3049 /* detile bit */ 3050 if (radeon_get_ib_value(p, idx + 2) & (1 << 31)) { 3051 /* tiled src, linear dst */ 3052 ib[idx+1] += (u32)(src_reloc->gpu_offset >> 8); 3053 3054 ib[idx+7] += (u32)(dst_reloc->gpu_offset & 0xfffffffc); 3055 ib[idx+8] += upper_32_bits(dst_reloc->gpu_offset) & 0xff; 3056 } else { 3057 /* linear src, tiled dst */ 3058 ib[idx+7] += (u32)(src_reloc->gpu_offset & 0xfffffffc); 3059 ib[idx+8] += upper_32_bits(src_reloc->gpu_offset) & 0xff; 3060 3061 ib[idx+1] += (u32)(dst_reloc->gpu_offset >> 8); 3062 } 3063 p->idx += 12; 3064 break; 3065 /* Copy L2T broadcast */ 3066 case 0x4b: 3067 /* L2T, broadcast */ 3068 if (radeon_get_ib_value(p, idx + 2) & (1 << 31)) { 3069 DRM_ERROR("bad L2T, broadcast DMA_PACKET_COPY\n"); 3070 return -EINVAL; 3071 } 3072 r = r600_dma_cs_next_reloc(p, &dst2_reloc); 3073 if (r) { 3074 DRM_ERROR("bad L2T, broadcast DMA_PACKET_COPY\n"); 3075 return -EINVAL; 3076 } 3077 dst_offset = radeon_get_ib_value(p, idx+1); 3078 dst_offset <<= 8; 3079 dst2_offset = radeon_get_ib_value(p, idx+2); 3080 dst2_offset <<= 8; 3081 src_offset = radeon_get_ib_value(p, idx+8); 3082 src_offset |= ((u64)(radeon_get_ib_value(p, idx+9) & 0xff)) << 32; 3083 if ((src_offset + (count * 4)) > radeon_bo_size(src_reloc->robj)) { 3084 dev_warn(p->dev, "DMA L2T, broadcast src buffer too small (%llu %lu)\n", 3085 src_offset + (count * 4), radeon_bo_size(src_reloc->robj)); 3086 return -EINVAL; 3087 } 3088 if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) { 3089 dev_warn(p->dev, "DMA L2T, broadcast dst buffer too small (%llu %lu)\n", 3090 dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj)); 3091 return -EINVAL; 3092 } 3093 if ((dst2_offset + (count * 4)) > radeon_bo_size(dst2_reloc->robj)) { 3094 dev_warn(p->dev, "DMA L2T, broadcast dst2 buffer too small (%llu %lu)\n", 3095 dst2_offset + (count * 4), radeon_bo_size(dst2_reloc->robj)); 3096 return -EINVAL; 3097 } 3098 ib[idx+1] += (u32)(dst_reloc->gpu_offset >> 8); 3099 ib[idx+2] += (u32)(dst2_reloc->gpu_offset >> 8); 3100 ib[idx+8] += (u32)(src_reloc->gpu_offset & 0xfffffffc); 3101 ib[idx+9] += upper_32_bits(src_reloc->gpu_offset) & 0xff; 3102 p->idx += 10; 3103 break; 3104 /* Copy L2T/T2L (tile units) */ 3105 case 0x4c: 3106 /* L2T, T2L */ 3107 /* detile bit */ 3108 if (radeon_get_ib_value(p, idx + 2) & (1 << 31)) { 3109 /* tiled src, linear dst */ 3110 src_offset = radeon_get_ib_value(p, idx+1); 3111 src_offset <<= 8; 3112 ib[idx+1] += (u32)(src_reloc->gpu_offset >> 8); 3113 3114 dst_offset = radeon_get_ib_value(p, idx+7); 3115 dst_offset |= ((u64)(radeon_get_ib_value(p, idx+8) & 0xff)) << 32; 3116 ib[idx+7] += (u32)(dst_reloc->gpu_offset & 0xfffffffc); 3117 ib[idx+8] += upper_32_bits(dst_reloc->gpu_offset) & 0xff; 3118 } else { 3119 /* linear src, tiled dst */ 3120 src_offset = radeon_get_ib_value(p, idx+7); 3121 src_offset |= ((u64)(radeon_get_ib_value(p, idx+8) & 0xff)) << 32; 3122 ib[idx+7] += (u32)(src_reloc->gpu_offset & 0xfffffffc); 3123 ib[idx+8] += upper_32_bits(src_reloc->gpu_offset) & 0xff; 3124 3125 dst_offset = radeon_get_ib_value(p, idx+1); 3126 dst_offset <<= 8; 3127 ib[idx+1] += (u32)(dst_reloc->gpu_offset >> 8); 3128 } 3129 if ((src_offset + (count * 4)) > radeon_bo_size(src_reloc->robj)) { 3130 dev_warn(p->dev, "DMA L2T, T2L src buffer too small (%llu %lu)\n", 3131 src_offset + (count * 4), radeon_bo_size(src_reloc->robj)); 3132 return -EINVAL; 3133 } 3134 if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) { 3135 dev_warn(p->dev, "DMA L2T, T2L dst buffer too small (%llu %lu)\n", 3136 dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj)); 3137 return -EINVAL; 3138 } 3139 p->idx += 9; 3140 break; 3141 /* Copy T2T, partial (tile units) */ 3142 case 0x4d: 3143 /* T2T partial */ 3144 if (p->family < CHIP_CAYMAN) { 3145 DRM_ERROR("L2T, T2L Partial is cayman only !\n"); 3146 return -EINVAL; 3147 } 3148 ib[idx+1] += (u32)(src_reloc->gpu_offset >> 8); 3149 ib[idx+4] += (u32)(dst_reloc->gpu_offset >> 8); 3150 p->idx += 13; 3151 break; 3152 /* Copy L2T broadcast (tile units) */ 3153 case 0x4f: 3154 /* L2T, broadcast */ 3155 if (radeon_get_ib_value(p, idx + 2) & (1 << 31)) { 3156 DRM_ERROR("bad L2T, broadcast DMA_PACKET_COPY\n"); 3157 return -EINVAL; 3158 } 3159 r = r600_dma_cs_next_reloc(p, &dst2_reloc); 3160 if (r) { 3161 DRM_ERROR("bad L2T, broadcast DMA_PACKET_COPY\n"); 3162 return -EINVAL; 3163 } 3164 dst_offset = radeon_get_ib_value(p, idx+1); 3165 dst_offset <<= 8; 3166 dst2_offset = radeon_get_ib_value(p, idx+2); 3167 dst2_offset <<= 8; 3168 src_offset = radeon_get_ib_value(p, idx+8); 3169 src_offset |= ((u64)(radeon_get_ib_value(p, idx+9) & 0xff)) << 32; 3170 if ((src_offset + (count * 4)) > radeon_bo_size(src_reloc->robj)) { 3171 dev_warn(p->dev, "DMA L2T, broadcast src buffer too small (%llu %lu)\n", 3172 src_offset + (count * 4), radeon_bo_size(src_reloc->robj)); 3173 return -EINVAL; 3174 } 3175 if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) { 3176 dev_warn(p->dev, "DMA L2T, broadcast dst buffer too small (%llu %lu)\n", 3177 dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj)); 3178 return -EINVAL; 3179 } 3180 if ((dst2_offset + (count * 4)) > radeon_bo_size(dst2_reloc->robj)) { 3181 dev_warn(p->dev, "DMA L2T, broadcast dst2 buffer too small (%llu %lu)\n", 3182 dst2_offset + (count * 4), radeon_bo_size(dst2_reloc->robj)); 3183 return -EINVAL; 3184 } 3185 ib[idx+1] += (u32)(dst_reloc->gpu_offset >> 8); 3186 ib[idx+2] += (u32)(dst2_reloc->gpu_offset >> 8); 3187 ib[idx+8] += (u32)(src_reloc->gpu_offset & 0xfffffffc); 3188 ib[idx+9] += upper_32_bits(src_reloc->gpu_offset) & 0xff; 3189 p->idx += 10; 3190 break; 3191 default: 3192 DRM_ERROR("bad DMA_PACKET_COPY [%6d] 0x%08x invalid sub cmd\n", idx, header); 3193 return -EINVAL; 3194 } 3195 break; 3196 case DMA_PACKET_CONSTANT_FILL: 3197 r = r600_dma_cs_next_reloc(p, &dst_reloc); 3198 if (r) { 3199 DRM_ERROR("bad DMA_PACKET_CONSTANT_FILL\n"); 3200 return -EINVAL; 3201 } 3202 dst_offset = radeon_get_ib_value(p, idx+1); 3203 dst_offset |= ((u64)(radeon_get_ib_value(p, idx+3) & 0x00ff0000)) << 16; 3204 if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) { 3205 dev_warn(p->dev, "DMA constant fill buffer too small (%llu %lu)\n", 3206 dst_offset, radeon_bo_size(dst_reloc->robj)); 3207 return -EINVAL; 3208 } 3209 ib[idx+1] += (u32)(dst_reloc->gpu_offset & 0xfffffffc); 3210 ib[idx+3] += (upper_32_bits(dst_reloc->gpu_offset) << 16) & 0x00ff0000; 3211 p->idx += 4; 3212 break; 3213 case DMA_PACKET_NOP: 3214 p->idx += 1; 3215 break; 3216 default: 3217 DRM_ERROR("Unknown packet type %d at %d !\n", cmd, idx); 3218 return -EINVAL; 3219 } 3220 } while (p->idx < p->chunk_ib->length_dw); 3221 #if 0 3222 for (r = 0; r < p->ib->length_dw; r++) { 3223 pr_info("%05d 0x%08X\n", r, p->ib.ptr[r]); 3224 mdelay(1); 3225 } 3226 #endif 3227 return 0; 3228 } 3229 3230 /* vm parser */ 3231 static bool evergreen_vm_reg_valid(u32 reg) 3232 { 3233 /* context regs are fine */ 3234 if (reg >= 0x28000) 3235 return true; 3236 3237 /* check config regs */ 3238 switch (reg) { 3239 case WAIT_UNTIL: 3240 case GRBM_GFX_INDEX: 3241 case CP_STRMOUT_CNTL: 3242 case CP_COHER_CNTL: 3243 case CP_COHER_SIZE: 3244 case VGT_VTX_VECT_EJECT_REG: 3245 case VGT_CACHE_INVALIDATION: 3246 case VGT_GS_VERTEX_REUSE: 3247 case VGT_PRIMITIVE_TYPE: 3248 case VGT_INDEX_TYPE: 3249 case VGT_NUM_INDICES: 3250 case VGT_NUM_INSTANCES: 3251 case VGT_COMPUTE_DIM_X: 3252 case VGT_COMPUTE_DIM_Y: 3253 case VGT_COMPUTE_DIM_Z: 3254 case VGT_COMPUTE_START_X: 3255 case VGT_COMPUTE_START_Y: 3256 case VGT_COMPUTE_START_Z: 3257 case VGT_COMPUTE_INDEX: 3258 case VGT_COMPUTE_THREAD_GROUP_SIZE: 3259 case VGT_HS_OFFCHIP_PARAM: 3260 case PA_CL_ENHANCE: 3261 case PA_SU_LINE_STIPPLE_VALUE: 3262 case PA_SC_LINE_STIPPLE_STATE: 3263 case PA_SC_ENHANCE: 3264 case SQ_DYN_GPR_CNTL_PS_FLUSH_REQ: 3265 case SQ_DYN_GPR_SIMD_LOCK_EN: 3266 case SQ_CONFIG: 3267 case SQ_GPR_RESOURCE_MGMT_1: 3268 case SQ_GLOBAL_GPR_RESOURCE_MGMT_1: 3269 case SQ_GLOBAL_GPR_RESOURCE_MGMT_2: 3270 case SQ_CONST_MEM_BASE: 3271 case SQ_STATIC_THREAD_MGMT_1: 3272 case SQ_STATIC_THREAD_MGMT_2: 3273 case SQ_STATIC_THREAD_MGMT_3: 3274 case SPI_CONFIG_CNTL: 3275 case SPI_CONFIG_CNTL_1: 3276 case TA_CNTL_AUX: 3277 case DB_DEBUG: 3278 case DB_DEBUG2: 3279 case DB_DEBUG3: 3280 case DB_DEBUG4: 3281 case DB_WATERMARKS: 3282 case TD_PS_BORDER_COLOR_INDEX: 3283 case TD_PS_BORDER_COLOR_RED: 3284 case TD_PS_BORDER_COLOR_GREEN: 3285 case TD_PS_BORDER_COLOR_BLUE: 3286 case TD_PS_BORDER_COLOR_ALPHA: 3287 case TD_VS_BORDER_COLOR_INDEX: 3288 case TD_VS_BORDER_COLOR_RED: 3289 case TD_VS_BORDER_COLOR_GREEN: 3290 case TD_VS_BORDER_COLOR_BLUE: 3291 case TD_VS_BORDER_COLOR_ALPHA: 3292 case TD_GS_BORDER_COLOR_INDEX: 3293 case TD_GS_BORDER_COLOR_RED: 3294 case TD_GS_BORDER_COLOR_GREEN: 3295 case TD_GS_BORDER_COLOR_BLUE: 3296 case TD_GS_BORDER_COLOR_ALPHA: 3297 case TD_HS_BORDER_COLOR_INDEX: 3298 case TD_HS_BORDER_COLOR_RED: 3299 case TD_HS_BORDER_COLOR_GREEN: 3300 case TD_HS_BORDER_COLOR_BLUE: 3301 case TD_HS_BORDER_COLOR_ALPHA: 3302 case TD_LS_BORDER_COLOR_INDEX: 3303 case TD_LS_BORDER_COLOR_RED: 3304 case TD_LS_BORDER_COLOR_GREEN: 3305 case TD_LS_BORDER_COLOR_BLUE: 3306 case TD_LS_BORDER_COLOR_ALPHA: 3307 case TD_CS_BORDER_COLOR_INDEX: 3308 case TD_CS_BORDER_COLOR_RED: 3309 case TD_CS_BORDER_COLOR_GREEN: 3310 case TD_CS_BORDER_COLOR_BLUE: 3311 case TD_CS_BORDER_COLOR_ALPHA: 3312 case SQ_ESGS_RING_SIZE: 3313 case SQ_GSVS_RING_SIZE: 3314 case SQ_ESTMP_RING_SIZE: 3315 case SQ_GSTMP_RING_SIZE: 3316 case SQ_HSTMP_RING_SIZE: 3317 case SQ_LSTMP_RING_SIZE: 3318 case SQ_PSTMP_RING_SIZE: 3319 case SQ_VSTMP_RING_SIZE: 3320 case SQ_ESGS_RING_ITEMSIZE: 3321 case SQ_ESTMP_RING_ITEMSIZE: 3322 case SQ_GSTMP_RING_ITEMSIZE: 3323 case SQ_GSVS_RING_ITEMSIZE: 3324 case SQ_GS_VERT_ITEMSIZE: 3325 case SQ_GS_VERT_ITEMSIZE_1: 3326 case SQ_GS_VERT_ITEMSIZE_2: 3327 case SQ_GS_VERT_ITEMSIZE_3: 3328 case SQ_GSVS_RING_OFFSET_1: 3329 case SQ_GSVS_RING_OFFSET_2: 3330 case SQ_GSVS_RING_OFFSET_3: 3331 case SQ_HSTMP_RING_ITEMSIZE: 3332 case SQ_LSTMP_RING_ITEMSIZE: 3333 case SQ_PSTMP_RING_ITEMSIZE: 3334 case SQ_VSTMP_RING_ITEMSIZE: 3335 case VGT_TF_RING_SIZE: 3336 case SQ_ESGS_RING_BASE: 3337 case SQ_GSVS_RING_BASE: 3338 case SQ_ESTMP_RING_BASE: 3339 case SQ_GSTMP_RING_BASE: 3340 case SQ_HSTMP_RING_BASE: 3341 case SQ_LSTMP_RING_BASE: 3342 case SQ_PSTMP_RING_BASE: 3343 case SQ_VSTMP_RING_BASE: 3344 case CAYMAN_VGT_OFFCHIP_LDS_BASE: 3345 case CAYMAN_SQ_EX_ALLOC_TABLE_SLOTS: 3346 return true; 3347 default: 3348 DRM_ERROR("Invalid register 0x%x in CS\n", reg); 3349 return false; 3350 } 3351 } 3352 3353 static int evergreen_vm_packet3_check(struct radeon_device *rdev, 3354 u32 *ib, struct radeon_cs_packet *pkt) 3355 { 3356 u32 idx = pkt->idx + 1; 3357 u32 idx_value = ib[idx]; 3358 u32 start_reg, end_reg, reg, i; 3359 u32 command, info; 3360 3361 switch (pkt->opcode) { 3362 case PACKET3_NOP: 3363 break; 3364 case PACKET3_SET_BASE: 3365 if (idx_value != 1) { 3366 DRM_ERROR("bad SET_BASE"); 3367 return -EINVAL; 3368 } 3369 break; 3370 case PACKET3_CLEAR_STATE: 3371 case PACKET3_INDEX_BUFFER_SIZE: 3372 case PACKET3_DISPATCH_DIRECT: 3373 case PACKET3_DISPATCH_INDIRECT: 3374 case PACKET3_MODE_CONTROL: 3375 case PACKET3_SET_PREDICATION: 3376 case PACKET3_COND_EXEC: 3377 case PACKET3_PRED_EXEC: 3378 case PACKET3_DRAW_INDIRECT: 3379 case PACKET3_DRAW_INDEX_INDIRECT: 3380 case PACKET3_INDEX_BASE: 3381 case PACKET3_DRAW_INDEX_2: 3382 case PACKET3_CONTEXT_CONTROL: 3383 case PACKET3_DRAW_INDEX_OFFSET: 3384 case PACKET3_INDEX_TYPE: 3385 case PACKET3_DRAW_INDEX: 3386 case PACKET3_DRAW_INDEX_AUTO: 3387 case PACKET3_DRAW_INDEX_IMMD: 3388 case PACKET3_NUM_INSTANCES: 3389 case PACKET3_DRAW_INDEX_MULTI_AUTO: 3390 case PACKET3_STRMOUT_BUFFER_UPDATE: 3391 case PACKET3_DRAW_INDEX_OFFSET_2: 3392 case PACKET3_DRAW_INDEX_MULTI_ELEMENT: 3393 case PACKET3_MPEG_INDEX: 3394 case PACKET3_WAIT_REG_MEM: 3395 case PACKET3_MEM_WRITE: 3396 case PACKET3_PFP_SYNC_ME: 3397 case PACKET3_SURFACE_SYNC: 3398 case PACKET3_EVENT_WRITE: 3399 case PACKET3_EVENT_WRITE_EOP: 3400 case PACKET3_EVENT_WRITE_EOS: 3401 case PACKET3_SET_CONTEXT_REG: 3402 case PACKET3_SET_BOOL_CONST: 3403 case PACKET3_SET_LOOP_CONST: 3404 case PACKET3_SET_RESOURCE: 3405 case PACKET3_SET_SAMPLER: 3406 case PACKET3_SET_CTL_CONST: 3407 case PACKET3_SET_RESOURCE_OFFSET: 3408 case PACKET3_SET_CONTEXT_REG_INDIRECT: 3409 case PACKET3_SET_RESOURCE_INDIRECT: 3410 case CAYMAN_PACKET3_DEALLOC_STATE: 3411 break; 3412 case PACKET3_COND_WRITE: 3413 if (idx_value & 0x100) { 3414 reg = ib[idx + 5] * 4; 3415 if (!evergreen_vm_reg_valid(reg)) 3416 return -EINVAL; 3417 } 3418 break; 3419 case PACKET3_COPY_DW: 3420 if (idx_value & 0x2) { 3421 reg = ib[idx + 3] * 4; 3422 if (!evergreen_vm_reg_valid(reg)) 3423 return -EINVAL; 3424 } 3425 break; 3426 case PACKET3_SET_CONFIG_REG: 3427 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START; 3428 end_reg = 4 * pkt->count + start_reg - 4; 3429 if ((start_reg < PACKET3_SET_CONFIG_REG_START) || 3430 (start_reg >= PACKET3_SET_CONFIG_REG_END) || 3431 (end_reg >= PACKET3_SET_CONFIG_REG_END)) { 3432 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n"); 3433 return -EINVAL; 3434 } 3435 for (i = 0; i < pkt->count; i++) { 3436 reg = start_reg + (4 * i); 3437 if (!evergreen_vm_reg_valid(reg)) 3438 return -EINVAL; 3439 } 3440 break; 3441 case PACKET3_CP_DMA: 3442 command = ib[idx + 4]; 3443 info = ib[idx + 1]; 3444 if ((((info & 0x60000000) >> 29) != 0) || /* src = GDS or DATA */ 3445 (((info & 0x00300000) >> 20) != 0) || /* dst = GDS */ 3446 ((((info & 0x00300000) >> 20) == 0) && 3447 (command & PACKET3_CP_DMA_CMD_DAS)) || /* dst = register */ 3448 ((((info & 0x60000000) >> 29) == 0) && 3449 (command & PACKET3_CP_DMA_CMD_SAS))) { /* src = register */ 3450 /* non mem to mem copies requires dw aligned count */ 3451 if ((command & 0x1fffff) % 4) { 3452 DRM_ERROR("CP DMA command requires dw count alignment\n"); 3453 return -EINVAL; 3454 } 3455 } 3456 if (command & PACKET3_CP_DMA_CMD_SAS) { 3457 /* src address space is register */ 3458 if (((info & 0x60000000) >> 29) == 0) { 3459 start_reg = idx_value << 2; 3460 if (command & PACKET3_CP_DMA_CMD_SAIC) { 3461 reg = start_reg; 3462 if (!evergreen_vm_reg_valid(reg)) { 3463 DRM_ERROR("CP DMA Bad SRC register\n"); 3464 return -EINVAL; 3465 } 3466 } else { 3467 for (i = 0; i < (command & 0x1fffff); i++) { 3468 reg = start_reg + (4 * i); 3469 if (!evergreen_vm_reg_valid(reg)) { 3470 DRM_ERROR("CP DMA Bad SRC register\n"); 3471 return -EINVAL; 3472 } 3473 } 3474 } 3475 } 3476 } 3477 if (command & PACKET3_CP_DMA_CMD_DAS) { 3478 /* dst address space is register */ 3479 if (((info & 0x00300000) >> 20) == 0) { 3480 start_reg = ib[idx + 2]; 3481 if (command & PACKET3_CP_DMA_CMD_DAIC) { 3482 reg = start_reg; 3483 if (!evergreen_vm_reg_valid(reg)) { 3484 DRM_ERROR("CP DMA Bad DST register\n"); 3485 return -EINVAL; 3486 } 3487 } else { 3488 for (i = 0; i < (command & 0x1fffff); i++) { 3489 reg = start_reg + (4 * i); 3490 if (!evergreen_vm_reg_valid(reg)) { 3491 DRM_ERROR("CP DMA Bad DST register\n"); 3492 return -EINVAL; 3493 } 3494 } 3495 } 3496 } 3497 } 3498 break; 3499 case PACKET3_SET_APPEND_CNT: { 3500 uint32_t areg; 3501 uint32_t allowed_reg_base; 3502 3503 if (pkt->count != 2) { 3504 DRM_ERROR("bad SET_APPEND_CNT (invalid count)\n"); 3505 return -EINVAL; 3506 } 3507 3508 allowed_reg_base = GDS_APPEND_COUNT_0; 3509 allowed_reg_base -= PACKET3_SET_CONTEXT_REG_START; 3510 allowed_reg_base >>= 2; 3511 3512 areg = idx_value >> 16; 3513 if (areg < allowed_reg_base || areg > (allowed_reg_base + 11)) { 3514 DRM_ERROR("forbidden register for append cnt 0x%08x at %d\n", 3515 areg, idx); 3516 return -EINVAL; 3517 } 3518 break; 3519 } 3520 default: 3521 return -EINVAL; 3522 } 3523 return 0; 3524 } 3525 3526 int evergreen_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib) 3527 { 3528 int ret = 0; 3529 u32 idx = 0; 3530 struct radeon_cs_packet pkt; 3531 3532 do { 3533 pkt.idx = idx; 3534 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]); 3535 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]); 3536 pkt.one_reg_wr = 0; 3537 switch (pkt.type) { 3538 case RADEON_PACKET_TYPE0: 3539 dev_err(rdev->dev, "Packet0 not allowed!\n"); 3540 ret = -EINVAL; 3541 break; 3542 case RADEON_PACKET_TYPE2: 3543 idx += 1; 3544 break; 3545 case RADEON_PACKET_TYPE3: 3546 pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]); 3547 ret = evergreen_vm_packet3_check(rdev, ib->ptr, &pkt); 3548 idx += pkt.count + 2; 3549 break; 3550 default: 3551 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type); 3552 ret = -EINVAL; 3553 break; 3554 } 3555 if (ret) 3556 break; 3557 } while (idx < ib->length_dw); 3558 3559 return ret; 3560 } 3561 3562 /** 3563 * evergreen_dma_ib_parse() - parse the DMA IB for VM 3564 * @rdev: radeon_device pointer 3565 * @ib: radeon_ib pointer 3566 * 3567 * Parses the DMA IB from the VM CS ioctl 3568 * checks for errors. (Cayman-SI) 3569 * Returns 0 for success and an error on failure. 3570 **/ 3571 int evergreen_dma_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib) 3572 { 3573 u32 idx = 0; 3574 u32 header, cmd, count, sub_cmd; 3575 3576 do { 3577 header = ib->ptr[idx]; 3578 cmd = GET_DMA_CMD(header); 3579 count = GET_DMA_COUNT(header); 3580 sub_cmd = GET_DMA_SUB_CMD(header); 3581 3582 switch (cmd) { 3583 case DMA_PACKET_WRITE: 3584 switch (sub_cmd) { 3585 /* tiled */ 3586 case 8: 3587 idx += count + 7; 3588 break; 3589 /* linear */ 3590 case 0: 3591 idx += count + 3; 3592 break; 3593 default: 3594 DRM_ERROR("bad DMA_PACKET_WRITE [%6d] 0x%08x sub cmd is not 0 or 8\n", idx, ib->ptr[idx]); 3595 return -EINVAL; 3596 } 3597 break; 3598 case DMA_PACKET_COPY: 3599 switch (sub_cmd) { 3600 /* Copy L2L, DW aligned */ 3601 case 0x00: 3602 idx += 5; 3603 break; 3604 /* Copy L2T/T2L */ 3605 case 0x08: 3606 idx += 9; 3607 break; 3608 /* Copy L2L, byte aligned */ 3609 case 0x40: 3610 idx += 5; 3611 break; 3612 /* Copy L2L, partial */ 3613 case 0x41: 3614 idx += 9; 3615 break; 3616 /* Copy L2L, DW aligned, broadcast */ 3617 case 0x44: 3618 idx += 7; 3619 break; 3620 /* Copy L2T Frame to Field */ 3621 case 0x48: 3622 idx += 10; 3623 break; 3624 /* Copy L2T/T2L, partial */ 3625 case 0x49: 3626 idx += 12; 3627 break; 3628 /* Copy L2T broadcast */ 3629 case 0x4b: 3630 idx += 10; 3631 break; 3632 /* Copy L2T/T2L (tile units) */ 3633 case 0x4c: 3634 idx += 9; 3635 break; 3636 /* Copy T2T, partial (tile units) */ 3637 case 0x4d: 3638 idx += 13; 3639 break; 3640 /* Copy L2T broadcast (tile units) */ 3641 case 0x4f: 3642 idx += 10; 3643 break; 3644 default: 3645 DRM_ERROR("bad DMA_PACKET_COPY [%6d] 0x%08x invalid sub cmd\n", idx, ib->ptr[idx]); 3646 return -EINVAL; 3647 } 3648 break; 3649 case DMA_PACKET_CONSTANT_FILL: 3650 idx += 4; 3651 break; 3652 case DMA_PACKET_NOP: 3653 idx += 1; 3654 break; 3655 default: 3656 DRM_ERROR("Unknown packet type %d at %d !\n", cmd, idx); 3657 return -EINVAL; 3658 } 3659 } while (idx < ib->length_dw); 3660 3661 return 0; 3662 } 3663
Indexes created Sat Oct 25 10:09:55 GMT 2025