1 /* 2 * Copyright 2013 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 * SOFTWARE. 22 * 23 * Authors: Marek Olk <maraeo (at) gmail.com> 24 * 25 */ 26 27 #include "r600_pipe_common.h" 28 #include "r600_cs.h" 29 #include "tgsi/tgsi_parse.h" 30 #include "util/list.h" 31 #include "util/u_draw_quad.h" 32 #include "util/u_memory.h" 33 #include "util/u_format_s3tc.h" 34 #include "util/u_upload_mgr.h" 35 #include "util/os_time.h" 36 #include "vl/vl_decoder.h" 37 #include "vl/vl_video_buffer.h" 38 #include "radeon_video.h" 39 #include <inttypes.h> 40 #include <sys/utsname.h> 41 42 #ifndef HAVE_LLVM 43 #define HAVE_LLVM 0 44 #endif 45 46 #if HAVE_LLVM 47 #include <llvm-c/TargetMachine.h> 48 #endif 49 50 struct r600_multi_fence { 51 struct pipe_reference reference; 52 struct pipe_fence_handle *gfx; 53 struct pipe_fence_handle *sdma; 54 55 /* If the context wasn't flushed at fence creation, this is non-NULL. */ 56 struct { 57 struct r600_common_context *ctx; 58 unsigned ib_index; 59 } gfx_unflushed; 60 }; 61 62 /* 63 * shader binary helpers. 64 */ 65 void radeon_shader_binary_init(struct ac_shader_binary *b) 66 { 67 memset(b, 0, sizeof(*b)); 68 } 69 70 void radeon_shader_binary_clean(struct ac_shader_binary *b) 71 { 72 if (!b) 73 return; 74 FREE(b->code); 75 FREE(b->config); 76 FREE(b->rodata); 77 FREE(b->global_symbol_offsets); 78 FREE(b->relocs); 79 FREE(b->disasm_string); 80 FREE(b->llvm_ir_string); 81 } 82 83 /* 84 * pipe_context 85 */ 86 87 /** 88 * Write an EOP event. 89 * 90 * \param event EVENT_TYPE_* 91 * \param event_flags Optional cache flush flags (TC) 92 * \param data_sel 1 = fence, 3 = timestamp 93 * \param buf Buffer 94 * \param va GPU address 95 * \param old_value Previous fence value (for a bug workaround) 96 * \param new_value Fence value to write for this event. 97 */ 98 void r600_gfx_write_event_eop(struct r600_common_context *ctx, 99 unsigned event, unsigned event_flags, 100 unsigned data_sel, 101 struct r600_resource *buf, uint64_t va, 102 uint32_t new_fence, unsigned query_type) 103 { 104 struct radeon_cmdbuf *cs = ctx->gfx.cs; 105 unsigned op = EVENT_TYPE(event) | 106 EVENT_INDEX(5) | 107 event_flags; 108 unsigned sel = EOP_DATA_SEL(data_sel); 109 110 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE_EOP, 4, 0)); 111 radeon_emit(cs, op); 112 radeon_emit(cs, va); 113 radeon_emit(cs, ((va >> 32) & 0xffff) | sel); 114 radeon_emit(cs, new_fence); /* immediate data */ 115 radeon_emit(cs, 0); /* unused */ 116 117 if (buf) 118 r600_emit_reloc(ctx, &ctx->gfx, buf, RADEON_USAGE_WRITE, 119 RADEON_PRIO_QUERY); 120 } 121 122 unsigned r600_gfx_write_fence_dwords(struct r600_common_screen *screen) 123 { 124 unsigned dwords = 6; 125 126 if (!screen->info.r600_has_virtual_memory) 127 dwords += 2; 128 129 return dwords; 130 } 131 132 void r600_gfx_wait_fence(struct r600_common_context *ctx, 133 struct r600_resource *buf, 134 uint64_t va, uint32_t ref, uint32_t mask) 135 { 136 struct radeon_cmdbuf *cs = ctx->gfx.cs; 137 138 radeon_emit(cs, PKT3(PKT3_WAIT_REG_MEM, 5, 0)); 139 radeon_emit(cs, WAIT_REG_MEM_EQUAL | WAIT_REG_MEM_MEM_SPACE(1)); 140 radeon_emit(cs, va); 141 radeon_emit(cs, va >> 32); 142 radeon_emit(cs, ref); /* reference value */ 143 radeon_emit(cs, mask); /* mask */ 144 radeon_emit(cs, 4); /* poll interval */ 145 146 if (buf) 147 r600_emit_reloc(ctx, &ctx->gfx, buf, RADEON_USAGE_READ, 148 RADEON_PRIO_QUERY); 149 } 150 151 void r600_draw_rectangle(struct blitter_context *blitter, 152 void *vertex_elements_cso, 153 blitter_get_vs_func get_vs, 154 int x1, int y1, int x2, int y2, 155 float depth, unsigned num_instances, 156 enum blitter_attrib_type type, 157 const union blitter_attrib *attrib) 158 { 159 struct r600_common_context *rctx = 160 (struct r600_common_context*)util_blitter_get_pipe(blitter); 161 struct pipe_viewport_state viewport; 162 struct pipe_resource *buf = NULL; 163 unsigned offset = 0; 164 float *vb; 165 166 rctx->b.bind_vertex_elements_state(&rctx->b, vertex_elements_cso); 167 rctx->b.bind_vs_state(&rctx->b, get_vs(blitter)); 168 169 /* Some operations (like color resolve on r6xx) don't work 170 * with the conventional primitive types. 171 * One that works is PT_RECTLIST, which we use here. */ 172 173 /* setup viewport */ 174 viewport.scale[0] = 1.0f; 175 viewport.scale[1] = 1.0f; 176 viewport.scale[2] = 1.0f; 177 viewport.translate[0] = 0.0f; 178 viewport.translate[1] = 0.0f; 179 viewport.translate[2] = 0.0f; 180 rctx->b.set_viewport_states(&rctx->b, 0, 1, &viewport); 181 182 /* Upload vertices. The hw rectangle has only 3 vertices, 183 * The 4th one is derived from the first 3. 184 * The vertex specification should match u_blitter's vertex element state. */ 185 u_upload_alloc(rctx->b.stream_uploader, 0, sizeof(float) * 24, 186 rctx->screen->info.tcc_cache_line_size, 187 &offset, &buf, (void**)&vb); 188 if (!buf) 189 return; 190 191 vb[0] = x1; 192 vb[1] = y1; 193 vb[2] = depth; 194 vb[3] = 1; 195 196 vb[8] = x1; 197 vb[9] = y2; 198 vb[10] = depth; 199 vb[11] = 1; 200 201 vb[16] = x2; 202 vb[17] = y1; 203 vb[18] = depth; 204 vb[19] = 1; 205 206 switch (type) { 207 case UTIL_BLITTER_ATTRIB_COLOR: 208 memcpy(vb+4, attrib->color, sizeof(float)*4); 209 memcpy(vb+12, attrib->color, sizeof(float)*4); 210 memcpy(vb+20, attrib->color, sizeof(float)*4); 211 break; 212 case UTIL_BLITTER_ATTRIB_TEXCOORD_XYZW: 213 case UTIL_BLITTER_ATTRIB_TEXCOORD_XY: 214 vb[6] = vb[14] = vb[22] = attrib->texcoord.z; 215 vb[7] = vb[15] = vb[23] = attrib->texcoord.w; 216 /* fall through */ 217 vb[4] = attrib->texcoord.x1; 218 vb[5] = attrib->texcoord.y1; 219 vb[12] = attrib->texcoord.x1; 220 vb[13] = attrib->texcoord.y2; 221 vb[20] = attrib->texcoord.x2; 222 vb[21] = attrib->texcoord.y1; 223 break; 224 default:; /* Nothing to do. */ 225 } 226 227 /* draw */ 228 struct pipe_vertex_buffer vbuffer = {}; 229 vbuffer.buffer.resource = buf; 230 vbuffer.stride = 2 * 4 * sizeof(float); /* vertex size */ 231 vbuffer.buffer_offset = offset; 232 233 rctx->b.set_vertex_buffers(&rctx->b, blitter->vb_slot, 1, &vbuffer); 234 util_draw_arrays_instanced(&rctx->b, R600_PRIM_RECTANGLE_LIST, 0, 3, 235 0, num_instances); 236 pipe_resource_reference(&buf, NULL); 237 } 238 239 static void r600_dma_emit_wait_idle(struct r600_common_context *rctx) 240 { 241 struct radeon_cmdbuf *cs = rctx->dma.cs; 242 243 if (rctx->chip_class >= EVERGREEN) 244 radeon_emit(cs, 0xf0000000); /* NOP */ 245 else { 246 /* TODO: R600-R700 should use the FENCE packet. 247 * CS checker support is required. */ 248 } 249 } 250 251 void r600_need_dma_space(struct r600_common_context *ctx, unsigned num_dw, 252 struct r600_resource *dst, struct r600_resource *src) 253 { 254 uint64_t vram = ctx->dma.cs->used_vram; 255 uint64_t gtt = ctx->dma.cs->used_gart; 256 257 if (dst) { 258 vram += dst->vram_usage; 259 gtt += dst->gart_usage; 260 } 261 if (src) { 262 vram += src->vram_usage; 263 gtt += src->gart_usage; 264 } 265 266 /* Flush the GFX IB if DMA depends on it. */ 267 if (radeon_emitted(ctx->gfx.cs, ctx->initial_gfx_cs_size) && 268 ((dst && 269 ctx->ws->cs_is_buffer_referenced(ctx->gfx.cs, dst->buf, 270 RADEON_USAGE_READWRITE)) || 271 (src && 272 ctx->ws->cs_is_buffer_referenced(ctx->gfx.cs, src->buf, 273 RADEON_USAGE_WRITE)))) 274 ctx->gfx.flush(ctx, PIPE_FLUSH_ASYNC, NULL); 275 276 /* Flush if there's not enough space, or if the memory usage per IB 277 * is too large. 278 * 279 * IBs using too little memory are limited by the IB submission overhead. 280 * IBs using too much memory are limited by the kernel/TTM overhead. 281 * Too long IBs create CPU-GPU pipeline bubbles and add latency. 282 * 283 * This heuristic makes sure that DMA requests are executed 284 * very soon after the call is made and lowers memory usage. 285 * It improves texture upload performance by keeping the DMA 286 * engine busy while uploads are being submitted. 287 */ 288 num_dw++; /* for emit_wait_idle below */ 289 if (!ctx->ws->cs_check_space(ctx->dma.cs, num_dw) || 290 ctx->dma.cs->used_vram + ctx->dma.cs->used_gart > 64 * 1024 * 1024 || 291 !radeon_cs_memory_below_limit(ctx->screen, ctx->dma.cs, vram, gtt)) { 292 ctx->dma.flush(ctx, PIPE_FLUSH_ASYNC, NULL); 293 assert((num_dw + ctx->dma.cs->current.cdw) <= ctx->dma.cs->current.max_dw); 294 } 295 296 /* Wait for idle if either buffer has been used in the IB before to 297 * prevent read-after-write hazards. 298 */ 299 if ((dst && 300 ctx->ws->cs_is_buffer_referenced(ctx->dma.cs, dst->buf, 301 RADEON_USAGE_READWRITE)) || 302 (src && 303 ctx->ws->cs_is_buffer_referenced(ctx->dma.cs, src->buf, 304 RADEON_USAGE_WRITE))) 305 r600_dma_emit_wait_idle(ctx); 306 307 /* If GPUVM is not supported, the CS checker needs 2 entries 308 * in the buffer list per packet, which has to be done manually. 309 */ 310 if (ctx->screen->info.r600_has_virtual_memory) { 311 if (dst) 312 radeon_add_to_buffer_list(ctx, &ctx->dma, dst, 313 RADEON_USAGE_WRITE, 0); 314 if (src) 315 radeon_add_to_buffer_list(ctx, &ctx->dma, src, 316 RADEON_USAGE_READ, 0); 317 } 318 319 /* this function is called before all DMA calls, so increment this. */ 320 ctx->num_dma_calls++; 321 } 322 323 void r600_preflush_suspend_features(struct r600_common_context *ctx) 324 { 325 /* suspend queries */ 326 if (!LIST_IS_EMPTY(&ctx->active_queries)) 327 r600_suspend_queries(ctx); 328 329 ctx->streamout.suspended = false; 330 if (ctx->streamout.begin_emitted) { 331 r600_emit_streamout_end(ctx); 332 ctx->streamout.suspended = true; 333 } 334 } 335 336 void r600_postflush_resume_features(struct r600_common_context *ctx) 337 { 338 if (ctx->streamout.suspended) { 339 ctx->streamout.append_bitmask = ctx->streamout.enabled_mask; 340 r600_streamout_buffers_dirty(ctx); 341 } 342 343 /* resume queries */ 344 if (!LIST_IS_EMPTY(&ctx->active_queries)) 345 r600_resume_queries(ctx); 346 } 347 348 static void r600_add_fence_dependency(struct r600_common_context *rctx, 349 struct pipe_fence_handle *fence) 350 { 351 struct radeon_winsys *ws = rctx->ws; 352 353 if (rctx->dma.cs) 354 ws->cs_add_fence_dependency(rctx->dma.cs, fence); 355 ws->cs_add_fence_dependency(rctx->gfx.cs, fence); 356 } 357 358 static void r600_fence_server_sync(struct pipe_context *ctx, 359 struct pipe_fence_handle *fence) 360 { 361 struct r600_common_context *rctx = (struct r600_common_context *)ctx; 362 struct r600_multi_fence *rfence = (struct r600_multi_fence *)fence; 363 364 /* Only amdgpu needs to handle fence dependencies (for fence imports). 365 * radeon synchronizes all rings by default and will not implement 366 * fence imports. 367 */ 368 if (rctx->screen->info.drm_major == 2) 369 return; 370 371 /* Only imported fences need to be handled by fence_server_sync, 372 * because the winsys handles synchronizations automatically for BOs 373 * within the process. 374 * 375 * Simply skip unflushed fences here, and the winsys will drop no-op 376 * dependencies (i.e. dependencies within the same ring). 377 */ 378 if (rfence->gfx_unflushed.ctx) 379 return; 380 381 /* All unflushed commands will not start execution before 382 * this fence dependency is signalled. 383 * 384 * Should we flush the context to allow more GPU parallelism? 385 */ 386 if (rfence->sdma) 387 r600_add_fence_dependency(rctx, rfence->sdma); 388 if (rfence->gfx) 389 r600_add_fence_dependency(rctx, rfence->gfx); 390 } 391 392 static void r600_flush_from_st(struct pipe_context *ctx, 393 struct pipe_fence_handle **fence, 394 unsigned flags) 395 { 396 struct pipe_screen *screen = ctx->screen; 397 struct r600_common_context *rctx = (struct r600_common_context *)ctx; 398 struct radeon_winsys *ws = rctx->ws; 399 struct pipe_fence_handle *gfx_fence = NULL; 400 struct pipe_fence_handle *sdma_fence = NULL; 401 bool deferred_fence = false; 402 unsigned rflags = PIPE_FLUSH_ASYNC; 403 404 if (flags & PIPE_FLUSH_END_OF_FRAME) 405 rflags |= PIPE_FLUSH_END_OF_FRAME; 406 407 /* DMA IBs are preambles to gfx IBs, therefore must be flushed first. */ 408 if (rctx->dma.cs) 409 rctx->dma.flush(rctx, rflags, fence ? &sdma_fence : NULL); 410 411 if (!radeon_emitted(rctx->gfx.cs, rctx->initial_gfx_cs_size)) { 412 if (fence) 413 ws->fence_reference(&gfx_fence, rctx->last_gfx_fence); 414 if (!(flags & PIPE_FLUSH_DEFERRED)) 415 ws->cs_sync_flush(rctx->gfx.cs); 416 } else { 417 /* Instead of flushing, create a deferred fence. Constraints: 418 * - The state tracker must allow a deferred flush. 419 * - The state tracker must request a fence. 420 * Thread safety in fence_finish must be ensured by the state tracker. 421 */ 422 if (flags & PIPE_FLUSH_DEFERRED && fence) { 423 gfx_fence = rctx->ws->cs_get_next_fence(rctx->gfx.cs); 424 deferred_fence = true; 425 } else { 426 rctx->gfx.flush(rctx, rflags, fence ? &gfx_fence : NULL); 427 } 428 } 429 430 /* Both engines can signal out of order, so we need to keep both fences. */ 431 if (fence) { 432 struct r600_multi_fence *multi_fence = 433 CALLOC_STRUCT(r600_multi_fence); 434 if (!multi_fence) { 435 ws->fence_reference(&sdma_fence, NULL); 436 ws->fence_reference(&gfx_fence, NULL); 437 goto finish; 438 } 439 440 multi_fence->reference.count = 1; 441 /* If both fences are NULL, fence_finish will always return true. */ 442 multi_fence->gfx = gfx_fence; 443 multi_fence->sdma = sdma_fence; 444 445 if (deferred_fence) { 446 multi_fence->gfx_unflushed.ctx = rctx; 447 multi_fence->gfx_unflushed.ib_index = rctx->num_gfx_cs_flushes; 448 } 449 450 screen->fence_reference(screen, fence, NULL); 451 *fence = (struct pipe_fence_handle*)multi_fence; 452 } 453 finish: 454 if (!(flags & PIPE_FLUSH_DEFERRED)) { 455 if (rctx->dma.cs) 456 ws->cs_sync_flush(rctx->dma.cs); 457 ws->cs_sync_flush(rctx->gfx.cs); 458 } 459 } 460 461 static void r600_flush_dma_ring(void *ctx, unsigned flags, 462 struct pipe_fence_handle **fence) 463 { 464 struct r600_common_context *rctx = (struct r600_common_context *)ctx; 465 struct radeon_cmdbuf *cs = rctx->dma.cs; 466 struct radeon_saved_cs saved; 467 bool check_vm = 468 (rctx->screen->debug_flags & DBG_CHECK_VM) && 469 rctx->check_vm_faults; 470 471 if (!radeon_emitted(cs, 0)) { 472 if (fence) 473 rctx->ws->fence_reference(fence, rctx->last_sdma_fence); 474 return; 475 } 476 477 if (check_vm) 478 radeon_save_cs(rctx->ws, cs, &saved, true); 479 480 rctx->ws->cs_flush(cs, flags, &rctx->last_sdma_fence); 481 if (fence) 482 rctx->ws->fence_reference(fence, rctx->last_sdma_fence); 483 484 if (check_vm) { 485 /* Use conservative timeout 800ms, after which we won't wait any 486 * longer and assume the GPU is hung. 487 */ 488 rctx->ws->fence_wait(rctx->ws, rctx->last_sdma_fence, 800*1000*1000); 489 490 rctx->check_vm_faults(rctx, &saved, RING_DMA); 491 radeon_clear_saved_cs(&saved); 492 } 493 } 494 495 /** 496 * Store a linearized copy of all chunks of \p cs together with the buffer 497 * list in \p saved. 498 */ 499 void radeon_save_cs(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, 500 struct radeon_saved_cs *saved, bool get_buffer_list) 501 { 502 uint32_t *buf; 503 unsigned i; 504 505 /* Save the IB chunks. */ 506 saved->num_dw = cs->prev_dw + cs->current.cdw; 507 saved->ib = MALLOC(4 * saved->num_dw); 508 if (!saved->ib) 509 goto oom; 510 511 buf = saved->ib; 512 for (i = 0; i < cs->num_prev; ++i) { 513 memcpy(buf, cs->prev[i].buf, cs->prev[i].cdw * 4); 514 buf += cs->prev[i].cdw; 515 } 516 memcpy(buf, cs->current.buf, cs->current.cdw * 4); 517 518 if (!get_buffer_list) 519 return; 520 521 /* Save the buffer list. */ 522 saved->bo_count = ws->cs_get_buffer_list(cs, NULL); 523 saved->bo_list = CALLOC(saved->bo_count, 524 sizeof(saved->bo_list[0])); 525 if (!saved->bo_list) { 526 FREE(saved->ib); 527 goto oom; 528 } 529 ws->cs_get_buffer_list(cs, saved->bo_list); 530 531 return; 532 533 oom: 534 fprintf(stderr, "%s: out of memory\n", __func__); 535 memset(saved, 0, sizeof(*saved)); 536 } 537 538 void radeon_clear_saved_cs(struct radeon_saved_cs *saved) 539 { 540 FREE(saved->ib); 541 FREE(saved->bo_list); 542 543 memset(saved, 0, sizeof(*saved)); 544 } 545 546 static enum pipe_reset_status r600_get_reset_status(struct pipe_context *ctx) 547 { 548 struct r600_common_context *rctx = (struct r600_common_context *)ctx; 549 unsigned latest = rctx->ws->query_value(rctx->ws, 550 RADEON_GPU_RESET_COUNTER); 551 552 if (rctx->gpu_reset_counter == latest) 553 return PIPE_NO_RESET; 554 555 rctx->gpu_reset_counter = latest; 556 return PIPE_UNKNOWN_CONTEXT_RESET; 557 } 558 559 static void r600_set_debug_callback(struct pipe_context *ctx, 560 const struct pipe_debug_callback *cb) 561 { 562 struct r600_common_context *rctx = (struct r600_common_context *)ctx; 563 564 if (cb) 565 rctx->debug = *cb; 566 else 567 memset(&rctx->debug, 0, sizeof(rctx->debug)); 568 } 569 570 static void r600_set_device_reset_callback(struct pipe_context *ctx, 571 const struct pipe_device_reset_callback *cb) 572 { 573 struct r600_common_context *rctx = (struct r600_common_context *)ctx; 574 575 if (cb) 576 rctx->device_reset_callback = *cb; 577 else 578 memset(&rctx->device_reset_callback, 0, 579 sizeof(rctx->device_reset_callback)); 580 } 581 582 bool r600_check_device_reset(struct r600_common_context *rctx) 583 { 584 enum pipe_reset_status status; 585 586 if (!rctx->device_reset_callback.reset) 587 return false; 588 589 if (!rctx->b.get_device_reset_status) 590 return false; 591 592 status = rctx->b.get_device_reset_status(&rctx->b); 593 if (status == PIPE_NO_RESET) 594 return false; 595 596 rctx->device_reset_callback.reset(rctx->device_reset_callback.data, status); 597 return true; 598 } 599 600 static void r600_dma_clear_buffer_fallback(struct pipe_context *ctx, 601 struct pipe_resource *dst, 602 uint64_t offset, uint64_t size, 603 unsigned value) 604 { 605 struct r600_common_context *rctx = (struct r600_common_context *)ctx; 606 607 rctx->clear_buffer(ctx, dst, offset, size, value, R600_COHERENCY_NONE); 608 } 609 610 static bool r600_resource_commit(struct pipe_context *pctx, 611 struct pipe_resource *resource, 612 unsigned level, struct pipe_box *box, 613 bool commit) 614 { 615 struct r600_common_context *ctx = (struct r600_common_context *)pctx; 616 struct r600_resource *res = r600_resource(resource); 617 618 /* 619 * Since buffer commitment changes cannot be pipelined, we need to 620 * (a) flush any pending commands that refer to the buffer we're about 621 * to change, and 622 * (b) wait for threaded submit to finish, including those that were 623 * triggered by some other, earlier operation. 624 */ 625 if (radeon_emitted(ctx->gfx.cs, ctx->initial_gfx_cs_size) && 626 ctx->ws->cs_is_buffer_referenced(ctx->gfx.cs, 627 res->buf, RADEON_USAGE_READWRITE)) { 628 ctx->gfx.flush(ctx, PIPE_FLUSH_ASYNC, NULL); 629 } 630 if (radeon_emitted(ctx->dma.cs, 0) && 631 ctx->ws->cs_is_buffer_referenced(ctx->dma.cs, 632 res->buf, RADEON_USAGE_READWRITE)) { 633 ctx->dma.flush(ctx, PIPE_FLUSH_ASYNC, NULL); 634 } 635 636 ctx->ws->cs_sync_flush(ctx->dma.cs); 637 ctx->ws->cs_sync_flush(ctx->gfx.cs); 638 639 assert(resource->target == PIPE_BUFFER); 640 641 return ctx->ws->buffer_commit(res->buf, box->x, box->width, commit); 642 } 643 644 bool r600_common_context_init(struct r600_common_context *rctx, 645 struct r600_common_screen *rscreen, 646 unsigned context_flags) 647 { 648 slab_create_child(&rctx->pool_transfers, &rscreen->pool_transfers); 649 slab_create_child(&rctx->pool_transfers_unsync, &rscreen->pool_transfers); 650 651 rctx->screen = rscreen; 652 rctx->ws = rscreen->ws; 653 rctx->family = rscreen->family; 654 rctx->chip_class = rscreen->chip_class; 655 656 rctx->b.invalidate_resource = r600_invalidate_resource; 657 rctx->b.resource_commit = r600_resource_commit; 658 rctx->b.transfer_map = u_transfer_map_vtbl; 659 rctx->b.transfer_flush_region = u_transfer_flush_region_vtbl; 660 rctx->b.transfer_unmap = u_transfer_unmap_vtbl; 661 rctx->b.texture_subdata = u_default_texture_subdata; 662 rctx->b.flush = r600_flush_from_st; 663 rctx->b.set_debug_callback = r600_set_debug_callback; 664 rctx->b.fence_server_sync = r600_fence_server_sync; 665 rctx->dma_clear_buffer = r600_dma_clear_buffer_fallback; 666 667 /* evergreen_compute.c has a special codepath for global buffers. 668 * Everything else can use the direct path. 669 */ 670 if ((rscreen->chip_class == EVERGREEN || rscreen->chip_class == CAYMAN) && 671 (context_flags & PIPE_CONTEXT_COMPUTE_ONLY)) 672 rctx->b.buffer_subdata = u_default_buffer_subdata; 673 else 674 rctx->b.buffer_subdata = r600_buffer_subdata; 675 676 if (rscreen->info.drm_major == 2 && rscreen->info.drm_minor >= 43) { 677 rctx->b.get_device_reset_status = r600_get_reset_status; 678 rctx->gpu_reset_counter = 679 rctx->ws->query_value(rctx->ws, 680 RADEON_GPU_RESET_COUNTER); 681 } 682 683 rctx->b.set_device_reset_callback = r600_set_device_reset_callback; 684 685 r600_init_context_texture_functions(rctx); 686 r600_init_viewport_functions(rctx); 687 r600_streamout_init(rctx); 688 r600_query_init(rctx); 689 cayman_init_msaa(&rctx->b); 690 691 rctx->allocator_zeroed_memory = 692 u_suballocator_create(&rctx->b, rscreen->info.gart_page_size, 693 0, PIPE_USAGE_DEFAULT, 0, true); 694 if (!rctx->allocator_zeroed_memory) 695 return false; 696 697 rctx->b.stream_uploader = u_upload_create(&rctx->b, 1024 * 1024, 698 0, PIPE_USAGE_STREAM, 0); 699 if (!rctx->b.stream_uploader) 700 return false; 701 702 rctx->b.const_uploader = u_upload_create(&rctx->b, 128 * 1024, 703 0, PIPE_USAGE_DEFAULT, 0); 704 if (!rctx->b.const_uploader) 705 return false; 706 707 rctx->ctx = rctx->ws->ctx_create(rctx->ws); 708 if (!rctx->ctx) 709 return false; 710 711 if (rscreen->info.num_sdma_rings && !(rscreen->debug_flags & DBG_NO_ASYNC_DMA)) { 712 rctx->dma.cs = rctx->ws->cs_create(rctx->ctx, RING_DMA, 713 r600_flush_dma_ring, 714 rctx, false); 715 rctx->dma.flush = r600_flush_dma_ring; 716 } 717 718 return true; 719 } 720 721 void r600_common_context_cleanup(struct r600_common_context *rctx) 722 { 723 if (rctx->query_result_shader) 724 rctx->b.delete_compute_state(&rctx->b, rctx->query_result_shader); 725 726 if (rctx->gfx.cs) 727 rctx->ws->cs_destroy(rctx->gfx.cs); 728 if (rctx->dma.cs) 729 rctx->ws->cs_destroy(rctx->dma.cs); 730 if (rctx->ctx) 731 rctx->ws->ctx_destroy(rctx->ctx); 732 733 if (rctx->b.stream_uploader) 734 u_upload_destroy(rctx->b.stream_uploader); 735 if (rctx->b.const_uploader) 736 u_upload_destroy(rctx->b.const_uploader); 737 738 slab_destroy_child(&rctx->pool_transfers); 739 slab_destroy_child(&rctx->pool_transfers_unsync); 740 741 if (rctx->allocator_zeroed_memory) { 742 u_suballocator_destroy(rctx->allocator_zeroed_memory); 743 } 744 rctx->ws->fence_reference(&rctx->last_gfx_fence, NULL); 745 rctx->ws->fence_reference(&rctx->last_sdma_fence, NULL); 746 r600_resource_reference(&rctx->eop_bug_scratch, NULL); 747 } 748 749 /* 750 * pipe_screen 751 */ 752 753 static const struct debug_named_value common_debug_options[] = { 754 /* logging */ 755 { "tex", DBG_TEX, "Print texture info" }, 756 { "nir", DBG_NIR, "Enable experimental NIR shaders" }, 757 { "compute", DBG_COMPUTE, "Print compute info" }, 758 { "vm", DBG_VM, "Print virtual addresses when creating resources" }, 759 { "info", DBG_INFO, "Print driver information" }, 760 761 /* shaders */ 762 { "fs", DBG_FS, "Print fetch shaders" }, 763 { "vs", DBG_VS, "Print vertex shaders" }, 764 { "gs", DBG_GS, "Print geometry shaders" }, 765 { "ps", DBG_PS, "Print pixel shaders" }, 766 { "cs", DBG_CS, "Print compute shaders" }, 767 { "tcs", DBG_TCS, "Print tessellation control shaders" }, 768 { "tes", DBG_TES, "Print tessellation evaluation shaders" }, 769 { "noir", DBG_NO_IR, "Don't print the LLVM IR"}, 770 { "notgsi", DBG_NO_TGSI, "Don't print the TGSI"}, 771 { "noasm", DBG_NO_ASM, "Don't print disassembled shaders"}, 772 { "preoptir", DBG_PREOPT_IR, "Print the LLVM IR before initial optimizations" }, 773 { "checkir", DBG_CHECK_IR, "Enable additional sanity checks on shader IR" }, 774 { "nooptvariant", DBG_NO_OPT_VARIANT, "Disable compiling optimized shader variants." }, 775 776 { "testdma", DBG_TEST_DMA, "Invoke SDMA tests and exit." }, 777 { "testvmfaultcp", DBG_TEST_VMFAULT_CP, "Invoke a CP VM fault test and exit." }, 778 { "testvmfaultsdma", DBG_TEST_VMFAULT_SDMA, "Invoke a SDMA VM fault test and exit." }, 779 { "testvmfaultshader", DBG_TEST_VMFAULT_SHADER, "Invoke a shader VM fault test and exit." }, 780 781 /* features */ 782 { "nodma", DBG_NO_ASYNC_DMA, "Disable asynchronous DMA" }, 783 { "nohyperz", DBG_NO_HYPERZ, "Disable Hyper-Z" }, 784 /* GL uses the word INVALIDATE, gallium uses the word DISCARD */ 785 { "noinvalrange", DBG_NO_DISCARD_RANGE, "Disable handling of INVALIDATE_RANGE map flags" }, 786 { "no2d", DBG_NO_2D_TILING, "Disable 2D tiling" }, 787 { "notiling", DBG_NO_TILING, "Disable tiling" }, 788 { "switch_on_eop", DBG_SWITCH_ON_EOP, "Program WD/IA to switch on end-of-packet." }, 789 { "forcedma", DBG_FORCE_DMA, "Use asynchronous DMA for all operations when possible." }, 790 { "precompile", DBG_PRECOMPILE, "Compile one shader variant at shader creation." }, 791 { "nowc", DBG_NO_WC, "Disable GTT write combining" }, 792 { "check_vm", DBG_CHECK_VM, "Check VM faults and dump debug info." }, 793 { "unsafemath", DBG_UNSAFE_MATH, "Enable unsafe math shader optimizations" }, 794 795 DEBUG_NAMED_VALUE_END /* must be last */ 796 }; 797 798 static const char* r600_get_vendor(struct pipe_screen* pscreen) 799 { 800 return "X.Org"; 801 } 802 803 static const char* r600_get_device_vendor(struct pipe_screen* pscreen) 804 { 805 return "AMD"; 806 } 807 808 static const char *r600_get_family_name(const struct r600_common_screen *rscreen) 809 { 810 switch (rscreen->info.family) { 811 case CHIP_R600: return "AMD R600"; 812 case CHIP_RV610: return "AMD RV610"; 813 case CHIP_RV630: return "AMD RV630"; 814 case CHIP_RV670: return "AMD RV670"; 815 case CHIP_RV620: return "AMD RV620"; 816 case CHIP_RV635: return "AMD RV635"; 817 case CHIP_RS780: return "AMD RS780"; 818 case CHIP_RS880: return "AMD RS880"; 819 case CHIP_RV770: return "AMD RV770"; 820 case CHIP_RV730: return "AMD RV730"; 821 case CHIP_RV710: return "AMD RV710"; 822 case CHIP_RV740: return "AMD RV740"; 823 case CHIP_CEDAR: return "AMD CEDAR"; 824 case CHIP_REDWOOD: return "AMD REDWOOD"; 825 case CHIP_JUNIPER: return "AMD JUNIPER"; 826 case CHIP_CYPRESS: return "AMD CYPRESS"; 827 case CHIP_HEMLOCK: return "AMD HEMLOCK"; 828 case CHIP_PALM: return "AMD PALM"; 829 case CHIP_SUMO: return "AMD SUMO"; 830 case CHIP_SUMO2: return "AMD SUMO2"; 831 case CHIP_BARTS: return "AMD BARTS"; 832 case CHIP_TURKS: return "AMD TURKS"; 833 case CHIP_CAICOS: return "AMD CAICOS"; 834 case CHIP_CAYMAN: return "AMD CAYMAN"; 835 case CHIP_ARUBA: return "AMD ARUBA"; 836 default: return "AMD unknown"; 837 } 838 } 839 840 static void r600_disk_cache_create(struct r600_common_screen *rscreen) 841 { 842 /* Don't use the cache if shader dumping is enabled. */ 843 if (rscreen->debug_flags & DBG_ALL_SHADERS) 844 return; 845 846 struct mesa_sha1 ctx; 847 unsigned char sha1[20]; 848 char cache_id[20 * 2 + 1]; 849 850 _mesa_sha1_init(&ctx); 851 if (!disk_cache_get_function_identifier(r600_disk_cache_create, 852 &ctx)) 853 return; 854 855 _mesa_sha1_final(&ctx, sha1); 856 disk_cache_format_hex_id(cache_id, sha1, 20 * 2); 857 858 /* These flags affect shader compilation. */ 859 uint64_t shader_debug_flags = 860 rscreen->debug_flags & 861 (DBG_FS_CORRECT_DERIVS_AFTER_KILL | 862 DBG_UNSAFE_MATH); 863 864 rscreen->disk_shader_cache = 865 disk_cache_create(r600_get_family_name(rscreen), 866 cache_id, 867 shader_debug_flags); 868 } 869 870 static struct disk_cache *r600_get_disk_shader_cache(struct pipe_screen *pscreen) 871 { 872 struct r600_common_screen *rscreen = (struct r600_common_screen*)pscreen; 873 return rscreen->disk_shader_cache; 874 } 875 876 static const char* r600_get_name(struct pipe_screen* pscreen) 877 { 878 struct r600_common_screen *rscreen = (struct r600_common_screen*)pscreen; 879 880 return rscreen->renderer_string; 881 } 882 883 static float r600_get_paramf(struct pipe_screen* pscreen, 884 enum pipe_capf param) 885 { 886 struct r600_common_screen *rscreen = (struct r600_common_screen *)pscreen; 887 888 switch (param) { 889 case PIPE_CAPF_MAX_LINE_WIDTH: 890 case PIPE_CAPF_MAX_LINE_WIDTH_AA: 891 case PIPE_CAPF_MAX_POINT_WIDTH: 892 case PIPE_CAPF_MAX_POINT_WIDTH_AA: 893 if (rscreen->family >= CHIP_CEDAR) 894 return 16384.0f; 895 else 896 return 8192.0f; 897 case PIPE_CAPF_MAX_TEXTURE_ANISOTROPY: 898 return 16.0f; 899 case PIPE_CAPF_MAX_TEXTURE_LOD_BIAS: 900 return 16.0f; 901 case PIPE_CAPF_MIN_CONSERVATIVE_RASTER_DILATE: 902 case PIPE_CAPF_MAX_CONSERVATIVE_RASTER_DILATE: 903 case PIPE_CAPF_CONSERVATIVE_RASTER_DILATE_GRANULARITY: 904 return 0.0f; 905 } 906 return 0.0f; 907 } 908 909 static int r600_get_video_param(struct pipe_screen *screen, 910 enum pipe_video_profile profile, 911 enum pipe_video_entrypoint entrypoint, 912 enum pipe_video_cap param) 913 { 914 switch (param) { 915 case PIPE_VIDEO_CAP_SUPPORTED: 916 return vl_profile_supported(screen, profile, entrypoint); 917 case PIPE_VIDEO_CAP_NPOT_TEXTURES: 918 return 1; 919 case PIPE_VIDEO_CAP_MAX_WIDTH: 920 case PIPE_VIDEO_CAP_MAX_HEIGHT: 921 return vl_video_buffer_max_size(screen); 922 case PIPE_VIDEO_CAP_PREFERED_FORMAT: 923 return PIPE_FORMAT_NV12; 924 case PIPE_VIDEO_CAP_PREFERS_INTERLACED: 925 return false; 926 case PIPE_VIDEO_CAP_SUPPORTS_INTERLACED: 927 return false; 928 case PIPE_VIDEO_CAP_SUPPORTS_PROGRESSIVE: 929 return true; 930 case PIPE_VIDEO_CAP_MAX_LEVEL: 931 return vl_level_supported(screen, profile); 932 default: 933 return 0; 934 } 935 } 936 937 const char *r600_get_llvm_processor_name(enum radeon_family family) 938 { 939 switch (family) { 940 case CHIP_R600: 941 case CHIP_RV630: 942 case CHIP_RV635: 943 case CHIP_RV670: 944 return "r600"; 945 case CHIP_RV610: 946 case CHIP_RV620: 947 case CHIP_RS780: 948 case CHIP_RS880: 949 return "rs880"; 950 case CHIP_RV710: 951 return "rv710"; 952 case CHIP_RV730: 953 return "rv730"; 954 case CHIP_RV740: 955 case CHIP_RV770: 956 return "rv770"; 957 case CHIP_PALM: 958 case CHIP_CEDAR: 959 return "cedar"; 960 case CHIP_SUMO: 961 case CHIP_SUMO2: 962 return "sumo"; 963 case CHIP_REDWOOD: 964 return "redwood"; 965 case CHIP_JUNIPER: 966 return "juniper"; 967 case CHIP_HEMLOCK: 968 case CHIP_CYPRESS: 969 return "cypress"; 970 case CHIP_BARTS: 971 return "barts"; 972 case CHIP_TURKS: 973 return "turks"; 974 case CHIP_CAICOS: 975 return "caicos"; 976 case CHIP_CAYMAN: 977 case CHIP_ARUBA: 978 return "cayman"; 979 980 default: 981 return ""; 982 } 983 } 984 985 static unsigned get_max_threads_per_block(struct r600_common_screen *screen, 986 enum pipe_shader_ir ir_type) 987 { 988 if (ir_type != PIPE_SHADER_IR_TGSI) 989 return 256; 990 if (screen->chip_class >= EVERGREEN) 991 return 1024; 992 return 256; 993 } 994 995 static int r600_get_compute_param(struct pipe_screen *screen, 996 enum pipe_shader_ir ir_type, 997 enum pipe_compute_cap param, 998 void *ret) 999 { 1000 struct r600_common_screen *rscreen = (struct r600_common_screen *)screen; 1001 1002 //TODO: select these params by asic 1003 switch (param) { 1004 case PIPE_COMPUTE_CAP_IR_TARGET: { 1005 const char *gpu; 1006 const char *triple = "r600--"; 1007 gpu = r600_get_llvm_processor_name(rscreen->family); 1008 if (ret) { 1009 sprintf(ret, "%s-%s", gpu, triple); 1010 } 1011 /* +2 for dash and terminating NIL byte */ 1012 return (strlen(triple) + strlen(gpu) + 2) * sizeof(char); 1013 } 1014 case PIPE_COMPUTE_CAP_GRID_DIMENSION: 1015 if (ret) { 1016 uint64_t *grid_dimension = ret; 1017 grid_dimension[0] = 3; 1018 } 1019 return 1 * sizeof(uint64_t); 1020 1021 case PIPE_COMPUTE_CAP_MAX_GRID_SIZE: 1022 if (ret) { 1023 uint64_t *grid_size = ret; 1024 grid_size[0] = 65535; 1025 grid_size[1] = 65535; 1026 grid_size[2] = 65535; 1027 } 1028 return 3 * sizeof(uint64_t) ; 1029 1030 case PIPE_COMPUTE_CAP_MAX_BLOCK_SIZE: 1031 if (ret) { 1032 uint64_t *block_size = ret; 1033 unsigned threads_per_block = get_max_threads_per_block(rscreen, ir_type); 1034 block_size[0] = threads_per_block; 1035 block_size[1] = threads_per_block; 1036 block_size[2] = threads_per_block; 1037 } 1038 return 3 * sizeof(uint64_t); 1039 1040 case PIPE_COMPUTE_CAP_MAX_THREADS_PER_BLOCK: 1041 if (ret) { 1042 uint64_t *max_threads_per_block = ret; 1043 *max_threads_per_block = get_max_threads_per_block(rscreen, ir_type); 1044 } 1045 return sizeof(uint64_t); 1046 case PIPE_COMPUTE_CAP_ADDRESS_BITS: 1047 if (ret) { 1048 uint32_t *address_bits = ret; 1049 address_bits[0] = 32; 1050 } 1051 return 1 * sizeof(uint32_t); 1052 1053 case PIPE_COMPUTE_CAP_MAX_GLOBAL_SIZE: 1054 if (ret) { 1055 uint64_t *max_global_size = ret; 1056 uint64_t max_mem_alloc_size; 1057 1058 r600_get_compute_param(screen, ir_type, 1059 PIPE_COMPUTE_CAP_MAX_MEM_ALLOC_SIZE, 1060 &max_mem_alloc_size); 1061 1062 /* In OpenCL, the MAX_MEM_ALLOC_SIZE must be at least 1063 * 1/4 of the MAX_GLOBAL_SIZE. Since the 1064 * MAX_MEM_ALLOC_SIZE is fixed for older kernels, 1065 * make sure we never report more than 1066 * 4 * MAX_MEM_ALLOC_SIZE. 1067 */ 1068 *max_global_size = MIN2(4 * max_mem_alloc_size, 1069 MAX2(rscreen->info.gart_size, 1070 rscreen->info.vram_size)); 1071 } 1072 return sizeof(uint64_t); 1073 1074 case PIPE_COMPUTE_CAP_MAX_LOCAL_SIZE: 1075 if (ret) { 1076 uint64_t *max_local_size = ret; 1077 /* Value reported by the closed source driver. */ 1078 *max_local_size = 32768; 1079 } 1080 return sizeof(uint64_t); 1081 1082 case PIPE_COMPUTE_CAP_MAX_INPUT_SIZE: 1083 if (ret) { 1084 uint64_t *max_input_size = ret; 1085 /* Value reported by the closed source driver. */ 1086 *max_input_size = 1024; 1087 } 1088 return sizeof(uint64_t); 1089 1090 case PIPE_COMPUTE_CAP_MAX_MEM_ALLOC_SIZE: 1091 if (ret) { 1092 uint64_t *max_mem_alloc_size = ret; 1093 1094 *max_mem_alloc_size = rscreen->info.max_alloc_size; 1095 } 1096 return sizeof(uint64_t); 1097 1098 case PIPE_COMPUTE_CAP_MAX_CLOCK_FREQUENCY: 1099 if (ret) { 1100 uint32_t *max_clock_frequency = ret; 1101 *max_clock_frequency = rscreen->info.max_shader_clock; 1102 } 1103 return sizeof(uint32_t); 1104 1105 case PIPE_COMPUTE_CAP_MAX_COMPUTE_UNITS: 1106 if (ret) { 1107 uint32_t *max_compute_units = ret; 1108 *max_compute_units = rscreen->info.num_good_compute_units; 1109 } 1110 return sizeof(uint32_t); 1111 1112 case PIPE_COMPUTE_CAP_IMAGES_SUPPORTED: 1113 if (ret) { 1114 uint32_t *images_supported = ret; 1115 *images_supported = 0; 1116 } 1117 return sizeof(uint32_t); 1118 case PIPE_COMPUTE_CAP_MAX_PRIVATE_SIZE: 1119 break; /* unused */ 1120 case PIPE_COMPUTE_CAP_SUBGROUP_SIZE: 1121 if (ret) { 1122 uint32_t *subgroup_size = ret; 1123 *subgroup_size = r600_wavefront_size(rscreen->family); 1124 } 1125 return sizeof(uint32_t); 1126 case PIPE_COMPUTE_CAP_MAX_VARIABLE_THREADS_PER_BLOCK: 1127 if (ret) { 1128 uint64_t *max_variable_threads_per_block = ret; 1129 *max_variable_threads_per_block = 0; 1130 } 1131 return sizeof(uint64_t); 1132 } 1133 1134 fprintf(stderr, "unknown PIPE_COMPUTE_CAP %d\n", param); 1135 return 0; 1136 } 1137 1138 static uint64_t r600_get_timestamp(struct pipe_screen *screen) 1139 { 1140 struct r600_common_screen *rscreen = (struct r600_common_screen*)screen; 1141 1142 return 1000000 * rscreen->ws->query_value(rscreen->ws, RADEON_TIMESTAMP) / 1143 rscreen->info.clock_crystal_freq; 1144 } 1145 1146 static void r600_fence_reference(struct pipe_screen *screen, 1147 struct pipe_fence_handle **dst, 1148 struct pipe_fence_handle *src) 1149 { 1150 struct radeon_winsys *ws = ((struct r600_common_screen*)screen)->ws; 1151 struct r600_multi_fence **rdst = (struct r600_multi_fence **)dst; 1152 struct r600_multi_fence *rsrc = (struct r600_multi_fence *)src; 1153 1154 if (pipe_reference(&(*rdst)->reference, &rsrc->reference)) { 1155 ws->fence_reference(&(*rdst)->gfx, NULL); 1156 ws->fence_reference(&(*rdst)->sdma, NULL); 1157 FREE(*rdst); 1158 } 1159 *rdst = rsrc; 1160 } 1161 1162 static boolean r600_fence_finish(struct pipe_screen *screen, 1163 struct pipe_context *ctx, 1164 struct pipe_fence_handle *fence, 1165 uint64_t timeout) 1166 { 1167 struct radeon_winsys *rws = ((struct r600_common_screen*)screen)->ws; 1168 struct r600_multi_fence *rfence = (struct r600_multi_fence *)fence; 1169 struct r600_common_context *rctx; 1170 int64_t abs_timeout = os_time_get_absolute_timeout(timeout); 1171 1172 ctx = threaded_context_unwrap_sync(ctx); 1173 rctx = ctx ? (struct r600_common_context*)ctx : NULL; 1174 1175 if (rfence->sdma) { 1176 if (!rws->fence_wait(rws, rfence->sdma, timeout)) 1177 return false; 1178 1179 /* Recompute the timeout after waiting. */ 1180 if (timeout && timeout != PIPE_TIMEOUT_INFINITE) { 1181 int64_t time = os_time_get_nano(); 1182 timeout = abs_timeout > time ? abs_timeout - time : 0; 1183 } 1184 } 1185 1186 if (!rfence->gfx) 1187 return true; 1188 1189 /* Flush the gfx IB if it hasn't been flushed yet. */ 1190 if (rctx && 1191 rfence->gfx_unflushed.ctx == rctx && 1192 rfence->gfx_unflushed.ib_index == rctx->num_gfx_cs_flushes) { 1193 rctx->gfx.flush(rctx, timeout ? 0 : PIPE_FLUSH_ASYNC, NULL); 1194 rfence->gfx_unflushed.ctx = NULL; 1195 1196 if (!timeout) 1197 return false; 1198 1199 /* Recompute the timeout after all that. */ 1200 if (timeout && timeout != PIPE_TIMEOUT_INFINITE) { 1201 int64_t time = os_time_get_nano(); 1202 timeout = abs_timeout > time ? abs_timeout - time : 0; 1203 } 1204 } 1205 1206 return rws->fence_wait(rws, rfence->gfx, timeout); 1207 } 1208 1209 static void r600_query_memory_info(struct pipe_screen *screen, 1210 struct pipe_memory_info *info) 1211 { 1212 struct r600_common_screen *rscreen = (struct r600_common_screen*)screen; 1213 struct radeon_winsys *ws = rscreen->ws; 1214 unsigned vram_usage, gtt_usage; 1215 1216 info->total_device_memory = rscreen->info.vram_size / 1024; 1217 info->total_staging_memory = rscreen->info.gart_size / 1024; 1218 1219 /* The real TTM memory usage is somewhat random, because: 1220 * 1221 * 1) TTM delays freeing memory, because it can only free it after 1222 * fences expire. 1223 * 1224 * 2) The memory usage can be really low if big VRAM evictions are 1225 * taking place, but the real usage is well above the size of VRAM. 1226 * 1227 * Instead, return statistics of this process. 1228 */ 1229 vram_usage = ws->query_value(ws, RADEON_REQUESTED_VRAM_MEMORY) / 1024; 1230 gtt_usage = ws->query_value(ws, RADEON_REQUESTED_GTT_MEMORY) / 1024; 1231 1232 info->avail_device_memory = 1233 vram_usage <= info->total_device_memory ? 1234 info->total_device_memory - vram_usage : 0; 1235 info->avail_staging_memory = 1236 gtt_usage <= info->total_staging_memory ? 1237 info->total_staging_memory - gtt_usage : 0; 1238 1239 info->device_memory_evicted = 1240 ws->query_value(ws, RADEON_NUM_BYTES_MOVED) / 1024; 1241 1242 if (rscreen->info.drm_major == 3 && rscreen->info.drm_minor >= 4) 1243 info->nr_device_memory_evictions = 1244 ws->query_value(ws, RADEON_NUM_EVICTIONS); 1245 else 1246 /* Just return the number of evicted 64KB pages. */ 1247 info->nr_device_memory_evictions = info->device_memory_evicted / 64; 1248 } 1249 1250 struct pipe_resource *r600_resource_create_common(struct pipe_screen *screen, 1251 const struct pipe_resource *templ) 1252 { 1253 if (templ->target == PIPE_BUFFER) { 1254 return r600_buffer_create(screen, templ, 256); 1255 } else { 1256 return r600_texture_create(screen, templ); 1257 } 1258 } 1259 1260 bool r600_common_screen_init(struct r600_common_screen *rscreen, 1261 struct radeon_winsys *ws) 1262 { 1263 char family_name[32] = {}, kernel_version[128] = {}; 1264 struct utsname uname_data; 1265 const char *chip_name; 1266 1267 ws->query_info(ws, &rscreen->info); 1268 rscreen->ws = ws; 1269 1270 chip_name = r600_get_family_name(rscreen); 1271 1272 if (uname(&uname_data) == 0) 1273 snprintf(kernel_version, sizeof(kernel_version), 1274 " / %s", uname_data.release); 1275 1276 snprintf(rscreen->renderer_string, sizeof(rscreen->renderer_string), 1277 "%s (%sDRM %i.%i.%i%s" 1278 #if HAVE_LLVM > 0 1279 ", LLVM " MESA_LLVM_VERSION_STRING 1280 #endif 1281 ")", 1282 chip_name, family_name, rscreen->info.drm_major, 1283 rscreen->info.drm_minor, rscreen->info.drm_patchlevel, 1284 kernel_version); 1285 1286 rscreen->b.get_name = r600_get_name; 1287 rscreen->b.get_vendor = r600_get_vendor; 1288 rscreen->b.get_device_vendor = r600_get_device_vendor; 1289 rscreen->b.get_disk_shader_cache = r600_get_disk_shader_cache; 1290 rscreen->b.get_compute_param = r600_get_compute_param; 1291 rscreen->b.get_paramf = r600_get_paramf; 1292 rscreen->b.get_timestamp = r600_get_timestamp; 1293 rscreen->b.fence_finish = r600_fence_finish; 1294 rscreen->b.fence_reference = r600_fence_reference; 1295 rscreen->b.resource_destroy = u_resource_destroy_vtbl; 1296 rscreen->b.resource_from_user_memory = r600_buffer_from_user_memory; 1297 rscreen->b.query_memory_info = r600_query_memory_info; 1298 1299 if (rscreen->info.has_hw_decode) { 1300 rscreen->b.get_video_param = rvid_get_video_param; 1301 rscreen->b.is_video_format_supported = rvid_is_format_supported; 1302 } else { 1303 rscreen->b.get_video_param = r600_get_video_param; 1304 rscreen->b.is_video_format_supported = vl_video_buffer_is_format_supported; 1305 } 1306 1307 r600_init_screen_texture_functions(rscreen); 1308 r600_init_screen_query_functions(rscreen); 1309 1310 rscreen->family = rscreen->info.family; 1311 rscreen->chip_class = rscreen->info.chip_class; 1312 rscreen->debug_flags |= debug_get_flags_option("R600_DEBUG", common_debug_options, 0); 1313 1314 r600_disk_cache_create(rscreen); 1315 1316 slab_create_parent(&rscreen->pool_transfers, sizeof(struct r600_transfer), 64); 1317 1318 rscreen->force_aniso = MIN2(16, debug_get_num_option("R600_TEX_ANISO", -1)); 1319 if (rscreen->force_aniso >= 0) { 1320 printf("radeon: Forcing anisotropy filter to %ix\n", 1321 /* round down to a power of two */ 1322 1 << util_logbase2(rscreen->force_aniso)); 1323 } 1324 1325 (void) mtx_init(&rscreen->aux_context_lock, mtx_plain); 1326 (void) mtx_init(&rscreen->gpu_load_mutex, mtx_plain); 1327 1328 if (rscreen->debug_flags & DBG_INFO) { 1329 printf("pci (domain:bus:dev.func): %04x:%02x:%02x.%x\n", 1330 rscreen->info.pci_domain, rscreen->info.pci_bus, 1331 rscreen->info.pci_dev, rscreen->info.pci_func); 1332 printf("pci_id = 0x%x\n", rscreen->info.pci_id); 1333 printf("family = %i (%s)\n", rscreen->info.family, 1334 r600_get_family_name(rscreen)); 1335 printf("chip_class = %i\n", rscreen->info.chip_class); 1336 printf("pte_fragment_size = %u\n", rscreen->info.pte_fragment_size); 1337 printf("gart_page_size = %u\n", rscreen->info.gart_page_size); 1338 printf("gart_size = %i MB\n", (int)DIV_ROUND_UP(rscreen->info.gart_size, 1024*1024)); 1339 printf("vram_size = %i MB\n", (int)DIV_ROUND_UP(rscreen->info.vram_size, 1024*1024)); 1340 printf("vram_vis_size = %i MB\n", (int)DIV_ROUND_UP(rscreen->info.vram_vis_size, 1024*1024)); 1341 printf("max_alloc_size = %i MB\n", 1342 (int)DIV_ROUND_UP(rscreen->info.max_alloc_size, 1024*1024)); 1343 printf("min_alloc_size = %u\n", rscreen->info.min_alloc_size); 1344 printf("has_dedicated_vram = %u\n", rscreen->info.has_dedicated_vram); 1345 printf("r600_has_virtual_memory = %i\n", rscreen->info.r600_has_virtual_memory); 1346 printf("gfx_ib_pad_with_type2 = %i\n", rscreen->info.gfx_ib_pad_with_type2); 1347 printf("has_hw_decode = %u\n", rscreen->info.has_hw_decode); 1348 printf("num_sdma_rings = %i\n", rscreen->info.num_sdma_rings); 1349 printf("num_compute_rings = %u\n", rscreen->info.num_compute_rings); 1350 printf("uvd_fw_version = %u\n", rscreen->info.uvd_fw_version); 1351 printf("vce_fw_version = %u\n", rscreen->info.vce_fw_version); 1352 printf("me_fw_version = %i\n", rscreen->info.me_fw_version); 1353 printf("pfp_fw_version = %i\n", rscreen->info.pfp_fw_version); 1354 printf("ce_fw_version = %i\n", rscreen->info.ce_fw_version); 1355 printf("vce_harvest_config = %i\n", rscreen->info.vce_harvest_config); 1356 printf("clock_crystal_freq = %i\n", rscreen->info.clock_crystal_freq); 1357 printf("tcc_cache_line_size = %u\n", rscreen->info.tcc_cache_line_size); 1358 printf("drm = %i.%i.%i\n", rscreen->info.drm_major, 1359 rscreen->info.drm_minor, rscreen->info.drm_patchlevel); 1360 printf("has_userptr = %i\n", rscreen->info.has_userptr); 1361 printf("has_syncobj = %u\n", rscreen->info.has_syncobj); 1362 1363 printf("r600_max_quad_pipes = %i\n", rscreen->info.r600_max_quad_pipes); 1364 printf("max_shader_clock = %i\n", rscreen->info.max_shader_clock); 1365 printf("num_good_compute_units = %i\n", rscreen->info.num_good_compute_units); 1366 printf("max_se = %i\n", rscreen->info.max_se); 1367 printf("max_sh_per_se = %i\n", rscreen->info.max_sh_per_se); 1368 1369 printf("r600_gb_backend_map = %i\n", rscreen->info.r600_gb_backend_map); 1370 printf("r600_gb_backend_map_valid = %i\n", rscreen->info.r600_gb_backend_map_valid); 1371 printf("r600_num_banks = %i\n", rscreen->info.r600_num_banks); 1372 printf("num_render_backends = %i\n", rscreen->info.num_render_backends); 1373 printf("num_tile_pipes = %i\n", rscreen->info.num_tile_pipes); 1374 printf("pipe_interleave_bytes = %i\n", rscreen->info.pipe_interleave_bytes); 1375 printf("enabled_rb_mask = 0x%x\n", rscreen->info.enabled_rb_mask); 1376 printf("max_alignment = %u\n", (unsigned)rscreen->info.max_alignment); 1377 } 1378 return true; 1379 } 1380 1381 void r600_destroy_common_screen(struct r600_common_screen *rscreen) 1382 { 1383 r600_perfcounters_destroy(rscreen); 1384 r600_gpu_load_kill_thread(rscreen); 1385 1386 mtx_destroy(&rscreen->gpu_load_mutex); 1387 mtx_destroy(&rscreen->aux_context_lock); 1388 rscreen->aux_context->destroy(rscreen->aux_context); 1389 1390 slab_destroy_parent(&rscreen->pool_transfers); 1391 1392 disk_cache_destroy(rscreen->disk_shader_cache); 1393 rscreen->ws->destroy(rscreen->ws); 1394 FREE(rscreen); 1395 } 1396 1397 bool r600_can_dump_shader(struct r600_common_screen *rscreen, 1398 unsigned processor) 1399 { 1400 return rscreen->debug_flags & (1 << processor); 1401 } 1402 1403 bool r600_extra_shader_checks(struct r600_common_screen *rscreen, unsigned processor) 1404 { 1405 return (rscreen->debug_flags & DBG_CHECK_IR) || 1406 r600_can_dump_shader(rscreen, processor); 1407 } 1408 1409 void r600_screen_clear_buffer(struct r600_common_screen *rscreen, struct pipe_resource *dst, 1410 uint64_t offset, uint64_t size, unsigned value) 1411 { 1412 struct r600_common_context *rctx = (struct r600_common_context*)rscreen->aux_context; 1413 1414 mtx_lock(&rscreen->aux_context_lock); 1415 rctx->dma_clear_buffer(&rctx->b, dst, offset, size, value); 1416 rscreen->aux_context->flush(rscreen->aux_context, NULL, 0); 1417 mtx_unlock(&rscreen->aux_context_lock); 1418 } 1419
Indexes created Sat Aug 01 00:24:56 UTC 2026