si_query.c revision b8e80941
1/* 2 * Copyright 2010 Jerome Glisse <glisse@freedesktop.org> 3 * Copyright 2014 Marek Olšák <marek.olsak@amd.com> 4 * Copyright 2018 Advanced Micro Devices, Inc. 5 * All Rights Reserved. 6 * 7 * Permission is hereby granted, free of charge, to any person obtaining a 8 * copy of this software and associated documentation files (the "Software"), 9 * to deal in the Software without restriction, including without limitation 10 * on the rights to use, copy, modify, merge, publish, distribute, sub 11 * license, and/or sell copies of the Software, and to permit persons to whom 12 * the Software is furnished to do so, subject to the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the next 15 * paragraph) shall be included in all copies or substantial portions of the 16 * Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM, 22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24 * USE OR OTHER DEALINGS IN THE SOFTWARE. 25 */ 26 27#include "si_pipe.h" 28#include "si_query.h" 29#include "util/u_memory.h" 30#include "util/u_upload_mgr.h" 31#include "util/os_time.h" 32#include "util/u_suballoc.h" 33#include "amd/common/sid.h" 34 35#define SI_MAX_STREAMS 4 36 37static const struct si_query_ops query_hw_ops; 38 39struct si_hw_query_params { 40 unsigned start_offset; 41 unsigned end_offset; 42 unsigned fence_offset; 43 unsigned pair_stride; 44 unsigned pair_count; 45}; 46 47/* Queries without buffer handling or suspend/resume. */ 48struct si_query_sw { 49 struct si_query b; 50 51 uint64_t begin_result; 52 uint64_t end_result; 53 54 uint64_t begin_time; 55 uint64_t end_time; 56 57 /* Fence for GPU_FINISHED. */ 58 struct pipe_fence_handle *fence; 59}; 60 61static void si_query_sw_destroy(struct si_screen *sscreen, 62 struct si_query *squery) 63{ 64 struct si_query_sw *query = (struct si_query_sw *)squery; 65 66 sscreen->b.fence_reference(&sscreen->b, &query->fence, NULL); 67 FREE(query); 68} 69 70static enum radeon_value_id winsys_id_from_type(unsigned type) 71{ 72 switch (type) { 73 case SI_QUERY_REQUESTED_VRAM: return RADEON_REQUESTED_VRAM_MEMORY; 74 case SI_QUERY_REQUESTED_GTT: return RADEON_REQUESTED_GTT_MEMORY; 75 case SI_QUERY_MAPPED_VRAM: return RADEON_MAPPED_VRAM; 76 case SI_QUERY_MAPPED_GTT: return RADEON_MAPPED_GTT; 77 case SI_QUERY_BUFFER_WAIT_TIME: return RADEON_BUFFER_WAIT_TIME_NS; 78 case SI_QUERY_NUM_MAPPED_BUFFERS: return RADEON_NUM_MAPPED_BUFFERS; 79 case SI_QUERY_NUM_GFX_IBS: return RADEON_NUM_GFX_IBS; 80 case SI_QUERY_NUM_SDMA_IBS: return RADEON_NUM_SDMA_IBS; 81 case SI_QUERY_GFX_BO_LIST_SIZE: return RADEON_GFX_BO_LIST_COUNTER; 82 case SI_QUERY_GFX_IB_SIZE: return RADEON_GFX_IB_SIZE_COUNTER; 83 case SI_QUERY_NUM_BYTES_MOVED: return RADEON_NUM_BYTES_MOVED; 84 case SI_QUERY_NUM_EVICTIONS: return RADEON_NUM_EVICTIONS; 85 case SI_QUERY_NUM_VRAM_CPU_PAGE_FAULTS: return RADEON_NUM_VRAM_CPU_PAGE_FAULTS; 86 case SI_QUERY_VRAM_USAGE: return RADEON_VRAM_USAGE; 87 case SI_QUERY_VRAM_VIS_USAGE: return RADEON_VRAM_VIS_USAGE; 88 case SI_QUERY_GTT_USAGE: return RADEON_GTT_USAGE; 89 case SI_QUERY_GPU_TEMPERATURE: return RADEON_GPU_TEMPERATURE; 90 case SI_QUERY_CURRENT_GPU_SCLK: return RADEON_CURRENT_SCLK; 91 case SI_QUERY_CURRENT_GPU_MCLK: return RADEON_CURRENT_MCLK; 92 case SI_QUERY_CS_THREAD_BUSY: return RADEON_CS_THREAD_TIME; 93 default: unreachable("query type does not correspond to winsys id"); 94 } 95} 96 97static int64_t si_finish_dma_get_cpu_time(struct si_context *sctx) 98{ 99 struct pipe_fence_handle *fence = NULL; 100 101 si_flush_dma_cs(sctx, 0, &fence); 102 if (fence) { 103 sctx->ws->fence_wait(sctx->ws, fence, PIPE_TIMEOUT_INFINITE); 104 sctx->ws->fence_reference(&fence, NULL); 105 } 106 107 return os_time_get_nano(); 108} 109 110static bool si_query_sw_begin(struct si_context *sctx, 111 struct si_query *squery) 112{ 113 struct si_query_sw *query = (struct si_query_sw *)squery; 114 enum radeon_value_id ws_id; 115 116 switch(query->b.type) { 117 case PIPE_QUERY_TIMESTAMP_DISJOINT: 118 case PIPE_QUERY_GPU_FINISHED: 119 break; 120 case SI_QUERY_TIME_ELAPSED_SDMA_SI: 121 query->begin_result = si_finish_dma_get_cpu_time(sctx); 122 break; 123 case SI_QUERY_DRAW_CALLS: 124 query->begin_result = sctx->num_draw_calls; 125 break; 126 case SI_QUERY_DECOMPRESS_CALLS: 127 query->begin_result = sctx->num_decompress_calls; 128 break; 129 case SI_QUERY_MRT_DRAW_CALLS: 130 query->begin_result = sctx->num_mrt_draw_calls; 131 break; 132 case SI_QUERY_PRIM_RESTART_CALLS: 133 query->begin_result = sctx->num_prim_restart_calls; 134 break; 135 case SI_QUERY_SPILL_DRAW_CALLS: 136 query->begin_result = sctx->num_spill_draw_calls; 137 break; 138 case SI_QUERY_COMPUTE_CALLS: 139 query->begin_result = sctx->num_compute_calls; 140 break; 141 case SI_QUERY_SPILL_COMPUTE_CALLS: 142 query->begin_result = sctx->num_spill_compute_calls; 143 break; 144 case SI_QUERY_DMA_CALLS: 145 query->begin_result = sctx->num_dma_calls; 146 break; 147 case SI_QUERY_CP_DMA_CALLS: 148 query->begin_result = sctx->num_cp_dma_calls; 149 break; 150 case SI_QUERY_NUM_VS_FLUSHES: 151 query->begin_result = sctx->num_vs_flushes; 152 break; 153 case SI_QUERY_NUM_PS_FLUSHES: 154 query->begin_result = sctx->num_ps_flushes; 155 break; 156 case SI_QUERY_NUM_CS_FLUSHES: 157 query->begin_result = sctx->num_cs_flushes; 158 break; 159 case SI_QUERY_NUM_CB_CACHE_FLUSHES: 160 query->begin_result = sctx->num_cb_cache_flushes; 161 break; 162 case SI_QUERY_NUM_DB_CACHE_FLUSHES: 163 query->begin_result = sctx->num_db_cache_flushes; 164 break; 165 case SI_QUERY_NUM_L2_INVALIDATES: 166 query->begin_result = sctx->num_L2_invalidates; 167 break; 168 case SI_QUERY_NUM_L2_WRITEBACKS: 169 query->begin_result = sctx->num_L2_writebacks; 170 break; 171 case SI_QUERY_NUM_RESIDENT_HANDLES: 172 query->begin_result = sctx->num_resident_handles; 173 break; 174 case SI_QUERY_TC_OFFLOADED_SLOTS: 175 query->begin_result = sctx->tc ? sctx->tc->num_offloaded_slots : 0; 176 break; 177 case SI_QUERY_TC_DIRECT_SLOTS: 178 query->begin_result = sctx->tc ? sctx->tc->num_direct_slots : 0; 179 break; 180 case SI_QUERY_TC_NUM_SYNCS: 181 query->begin_result = sctx->tc ? sctx->tc->num_syncs : 0; 182 break; 183 case SI_QUERY_REQUESTED_VRAM: 184 case SI_QUERY_REQUESTED_GTT: 185 case SI_QUERY_MAPPED_VRAM: 186 case SI_QUERY_MAPPED_GTT: 187 case SI_QUERY_VRAM_USAGE: 188 case SI_QUERY_VRAM_VIS_USAGE: 189 case SI_QUERY_GTT_USAGE: 190 case SI_QUERY_GPU_TEMPERATURE: 191 case SI_QUERY_CURRENT_GPU_SCLK: 192 case SI_QUERY_CURRENT_GPU_MCLK: 193 case SI_QUERY_BACK_BUFFER_PS_DRAW_RATIO: 194 case SI_QUERY_NUM_MAPPED_BUFFERS: 195 query->begin_result = 0; 196 break; 197 case SI_QUERY_BUFFER_WAIT_TIME: 198 case SI_QUERY_GFX_IB_SIZE: 199 case SI_QUERY_NUM_GFX_IBS: 200 case SI_QUERY_NUM_SDMA_IBS: 201 case SI_QUERY_NUM_BYTES_MOVED: 202 case SI_QUERY_NUM_EVICTIONS: 203 case SI_QUERY_NUM_VRAM_CPU_PAGE_FAULTS: { 204 enum radeon_value_id ws_id = winsys_id_from_type(query->b.type); 205 query->begin_result = sctx->ws->query_value(sctx->ws, ws_id); 206 break; 207 } 208 case SI_QUERY_GFX_BO_LIST_SIZE: 209 ws_id = winsys_id_from_type(query->b.type); 210 query->begin_result = sctx->ws->query_value(sctx->ws, ws_id); 211 query->begin_time = sctx->ws->query_value(sctx->ws, 212 RADEON_NUM_GFX_IBS); 213 break; 214 case SI_QUERY_CS_THREAD_BUSY: 215 ws_id = winsys_id_from_type(query->b.type); 216 query->begin_result = sctx->ws->query_value(sctx->ws, ws_id); 217 query->begin_time = os_time_get_nano(); 218 break; 219 case SI_QUERY_GALLIUM_THREAD_BUSY: 220 query->begin_result = 221 sctx->tc ? util_queue_get_thread_time_nano(&sctx->tc->queue, 0) : 0; 222 query->begin_time = os_time_get_nano(); 223 break; 224 case SI_QUERY_GPU_LOAD: 225 case SI_QUERY_GPU_SHADERS_BUSY: 226 case SI_QUERY_GPU_TA_BUSY: 227 case SI_QUERY_GPU_GDS_BUSY: 228 case SI_QUERY_GPU_VGT_BUSY: 229 case SI_QUERY_GPU_IA_BUSY: 230 case SI_QUERY_GPU_SX_BUSY: 231 case SI_QUERY_GPU_WD_BUSY: 232 case SI_QUERY_GPU_BCI_BUSY: 233 case SI_QUERY_GPU_SC_BUSY: 234 case SI_QUERY_GPU_PA_BUSY: 235 case SI_QUERY_GPU_DB_BUSY: 236 case SI_QUERY_GPU_CP_BUSY: 237 case SI_QUERY_GPU_CB_BUSY: 238 case SI_QUERY_GPU_SDMA_BUSY: 239 case SI_QUERY_GPU_PFP_BUSY: 240 case SI_QUERY_GPU_MEQ_BUSY: 241 case SI_QUERY_GPU_ME_BUSY: 242 case SI_QUERY_GPU_SURF_SYNC_BUSY: 243 case SI_QUERY_GPU_CP_DMA_BUSY: 244 case SI_QUERY_GPU_SCRATCH_RAM_BUSY: 245 query->begin_result = si_begin_counter(sctx->screen, 246 query->b.type); 247 break; 248 case SI_QUERY_NUM_COMPILATIONS: 249 query->begin_result = p_atomic_read(&sctx->screen->num_compilations); 250 break; 251 case SI_QUERY_NUM_SHADERS_CREATED: 252 query->begin_result = p_atomic_read(&sctx->screen->num_shaders_created); 253 break; 254 case SI_QUERY_NUM_SHADER_CACHE_HITS: 255 query->begin_result = 256 p_atomic_read(&sctx->screen->num_shader_cache_hits); 257 break; 258 case SI_QUERY_GPIN_ASIC_ID: 259 case SI_QUERY_GPIN_NUM_SIMD: 260 case SI_QUERY_GPIN_NUM_RB: 261 case SI_QUERY_GPIN_NUM_SPI: 262 case SI_QUERY_GPIN_NUM_SE: 263 break; 264 default: 265 unreachable("si_query_sw_begin: bad query type"); 266 } 267 268 return true; 269} 270 271static bool si_query_sw_end(struct si_context *sctx, 272 struct si_query *squery) 273{ 274 struct si_query_sw *query = (struct si_query_sw *)squery; 275 enum radeon_value_id ws_id; 276 277 switch(query->b.type) { 278 case PIPE_QUERY_TIMESTAMP_DISJOINT: 279 break; 280 case PIPE_QUERY_GPU_FINISHED: 281 sctx->b.flush(&sctx->b, &query->fence, PIPE_FLUSH_DEFERRED); 282 break; 283 case SI_QUERY_TIME_ELAPSED_SDMA_SI: 284 query->end_result = si_finish_dma_get_cpu_time(sctx); 285 break; 286 case SI_QUERY_DRAW_CALLS: 287 query->end_result = sctx->num_draw_calls; 288 break; 289 case SI_QUERY_DECOMPRESS_CALLS: 290 query->end_result = sctx->num_decompress_calls; 291 break; 292 case SI_QUERY_MRT_DRAW_CALLS: 293 query->end_result = sctx->num_mrt_draw_calls; 294 break; 295 case SI_QUERY_PRIM_RESTART_CALLS: 296 query->end_result = sctx->num_prim_restart_calls; 297 break; 298 case SI_QUERY_SPILL_DRAW_CALLS: 299 query->end_result = sctx->num_spill_draw_calls; 300 break; 301 case SI_QUERY_COMPUTE_CALLS: 302 query->end_result = sctx->num_compute_calls; 303 break; 304 case SI_QUERY_SPILL_COMPUTE_CALLS: 305 query->end_result = sctx->num_spill_compute_calls; 306 break; 307 case SI_QUERY_DMA_CALLS: 308 query->end_result = sctx->num_dma_calls; 309 break; 310 case SI_QUERY_CP_DMA_CALLS: 311 query->end_result = sctx->num_cp_dma_calls; 312 break; 313 case SI_QUERY_NUM_VS_FLUSHES: 314 query->end_result = sctx->num_vs_flushes; 315 break; 316 case SI_QUERY_NUM_PS_FLUSHES: 317 query->end_result = sctx->num_ps_flushes; 318 break; 319 case SI_QUERY_NUM_CS_FLUSHES: 320 query->end_result = sctx->num_cs_flushes; 321 break; 322 case SI_QUERY_NUM_CB_CACHE_FLUSHES: 323 query->end_result = sctx->num_cb_cache_flushes; 324 break; 325 case SI_QUERY_NUM_DB_CACHE_FLUSHES: 326 query->end_result = sctx->num_db_cache_flushes; 327 break; 328 case SI_QUERY_NUM_L2_INVALIDATES: 329 query->end_result = sctx->num_L2_invalidates; 330 break; 331 case SI_QUERY_NUM_L2_WRITEBACKS: 332 query->end_result = sctx->num_L2_writebacks; 333 break; 334 case SI_QUERY_NUM_RESIDENT_HANDLES: 335 query->end_result = sctx->num_resident_handles; 336 break; 337 case SI_QUERY_TC_OFFLOADED_SLOTS: 338 query->end_result = sctx->tc ? sctx->tc->num_offloaded_slots : 0; 339 break; 340 case SI_QUERY_TC_DIRECT_SLOTS: 341 query->end_result = sctx->tc ? sctx->tc->num_direct_slots : 0; 342 break; 343 case SI_QUERY_TC_NUM_SYNCS: 344 query->end_result = sctx->tc ? sctx->tc->num_syncs : 0; 345 break; 346 case SI_QUERY_REQUESTED_VRAM: 347 case SI_QUERY_REQUESTED_GTT: 348 case SI_QUERY_MAPPED_VRAM: 349 case SI_QUERY_MAPPED_GTT: 350 case SI_QUERY_VRAM_USAGE: 351 case SI_QUERY_VRAM_VIS_USAGE: 352 case SI_QUERY_GTT_USAGE: 353 case SI_QUERY_GPU_TEMPERATURE: 354 case SI_QUERY_CURRENT_GPU_SCLK: 355 case SI_QUERY_CURRENT_GPU_MCLK: 356 case SI_QUERY_BUFFER_WAIT_TIME: 357 case SI_QUERY_GFX_IB_SIZE: 358 case SI_QUERY_NUM_MAPPED_BUFFERS: 359 case SI_QUERY_NUM_GFX_IBS: 360 case SI_QUERY_NUM_SDMA_IBS: 361 case SI_QUERY_NUM_BYTES_MOVED: 362 case SI_QUERY_NUM_EVICTIONS: 363 case SI_QUERY_NUM_VRAM_CPU_PAGE_FAULTS: { 364 enum radeon_value_id ws_id = winsys_id_from_type(query->b.type); 365 query->end_result = sctx->ws->query_value(sctx->ws, ws_id); 366 break; 367 } 368 case SI_QUERY_GFX_BO_LIST_SIZE: 369 ws_id = winsys_id_from_type(query->b.type); 370 query->end_result = sctx->ws->query_value(sctx->ws, ws_id); 371 query->end_time = sctx->ws->query_value(sctx->ws, 372 RADEON_NUM_GFX_IBS); 373 break; 374 case SI_QUERY_CS_THREAD_BUSY: 375 ws_id = winsys_id_from_type(query->b.type); 376 query->end_result = sctx->ws->query_value(sctx->ws, ws_id); 377 query->end_time = os_time_get_nano(); 378 break; 379 case SI_QUERY_GALLIUM_THREAD_BUSY: 380 query->end_result = 381 sctx->tc ? util_queue_get_thread_time_nano(&sctx->tc->queue, 0) : 0; 382 query->end_time = os_time_get_nano(); 383 break; 384 case SI_QUERY_GPU_LOAD: 385 case SI_QUERY_GPU_SHADERS_BUSY: 386 case SI_QUERY_GPU_TA_BUSY: 387 case SI_QUERY_GPU_GDS_BUSY: 388 case SI_QUERY_GPU_VGT_BUSY: 389 case SI_QUERY_GPU_IA_BUSY: 390 case SI_QUERY_GPU_SX_BUSY: 391 case SI_QUERY_GPU_WD_BUSY: 392 case SI_QUERY_GPU_BCI_BUSY: 393 case SI_QUERY_GPU_SC_BUSY: 394 case SI_QUERY_GPU_PA_BUSY: 395 case SI_QUERY_GPU_DB_BUSY: 396 case SI_QUERY_GPU_CP_BUSY: 397 case SI_QUERY_GPU_CB_BUSY: 398 case SI_QUERY_GPU_SDMA_BUSY: 399 case SI_QUERY_GPU_PFP_BUSY: 400 case SI_QUERY_GPU_MEQ_BUSY: 401 case SI_QUERY_GPU_ME_BUSY: 402 case SI_QUERY_GPU_SURF_SYNC_BUSY: 403 case SI_QUERY_GPU_CP_DMA_BUSY: 404 case SI_QUERY_GPU_SCRATCH_RAM_BUSY: 405 query->end_result = si_end_counter(sctx->screen, 406 query->b.type, 407 query->begin_result); 408 query->begin_result = 0; 409 break; 410 case SI_QUERY_NUM_COMPILATIONS: 411 query->end_result = p_atomic_read(&sctx->screen->num_compilations); 412 break; 413 case SI_QUERY_NUM_SHADERS_CREATED: 414 query->end_result = p_atomic_read(&sctx->screen->num_shaders_created); 415 break; 416 case SI_QUERY_BACK_BUFFER_PS_DRAW_RATIO: 417 query->end_result = sctx->last_tex_ps_draw_ratio; 418 break; 419 case SI_QUERY_NUM_SHADER_CACHE_HITS: 420 query->end_result = 421 p_atomic_read(&sctx->screen->num_shader_cache_hits); 422 break; 423 case SI_QUERY_GPIN_ASIC_ID: 424 case SI_QUERY_GPIN_NUM_SIMD: 425 case SI_QUERY_GPIN_NUM_RB: 426 case SI_QUERY_GPIN_NUM_SPI: 427 case SI_QUERY_GPIN_NUM_SE: 428 break; 429 default: 430 unreachable("si_query_sw_end: bad query type"); 431 } 432 433 return true; 434} 435 436static bool si_query_sw_get_result(struct si_context *sctx, 437 struct si_query *squery, 438 bool wait, 439 union pipe_query_result *result) 440{ 441 struct si_query_sw *query = (struct si_query_sw *)squery; 442 443 switch (query->b.type) { 444 case PIPE_QUERY_TIMESTAMP_DISJOINT: 445 /* Convert from cycles per millisecond to cycles per second (Hz). */ 446 result->timestamp_disjoint.frequency = 447 (uint64_t)sctx->screen->info.clock_crystal_freq * 1000; 448 result->timestamp_disjoint.disjoint = false; 449 return true; 450 case PIPE_QUERY_GPU_FINISHED: { 451 struct pipe_screen *screen = sctx->b.screen; 452 struct pipe_context *ctx = squery->b.flushed ? NULL : &sctx->b; 453 454 result->b = screen->fence_finish(screen, ctx, query->fence, 455 wait ? PIPE_TIMEOUT_INFINITE : 0); 456 return result->b; 457 } 458 459 case SI_QUERY_GFX_BO_LIST_SIZE: 460 result->u64 = (query->end_result - query->begin_result) / 461 (query->end_time - query->begin_time); 462 return true; 463 case SI_QUERY_CS_THREAD_BUSY: 464 case SI_QUERY_GALLIUM_THREAD_BUSY: 465 result->u64 = (query->end_result - query->begin_result) * 100 / 466 (query->end_time - query->begin_time); 467 return true; 468 case SI_QUERY_GPIN_ASIC_ID: 469 result->u32 = 0; 470 return true; 471 case SI_QUERY_GPIN_NUM_SIMD: 472 result->u32 = sctx->screen->info.num_good_compute_units; 473 return true; 474 case SI_QUERY_GPIN_NUM_RB: 475 result->u32 = sctx->screen->info.num_render_backends; 476 return true; 477 case SI_QUERY_GPIN_NUM_SPI: 478 result->u32 = 1; /* all supported chips have one SPI per SE */ 479 return true; 480 case SI_QUERY_GPIN_NUM_SE: 481 result->u32 = sctx->screen->info.max_se; 482 return true; 483 } 484 485 result->u64 = query->end_result - query->begin_result; 486 487 switch (query->b.type) { 488 case SI_QUERY_BUFFER_WAIT_TIME: 489 case SI_QUERY_GPU_TEMPERATURE: 490 result->u64 /= 1000; 491 break; 492 case SI_QUERY_CURRENT_GPU_SCLK: 493 case SI_QUERY_CURRENT_GPU_MCLK: 494 result->u64 *= 1000000; 495 break; 496 } 497 498 return true; 499} 500 501 502static const struct si_query_ops sw_query_ops = { 503 .destroy = si_query_sw_destroy, 504 .begin = si_query_sw_begin, 505 .end = si_query_sw_end, 506 .get_result = si_query_sw_get_result, 507 .get_result_resource = NULL 508}; 509 510static struct pipe_query *si_query_sw_create(unsigned query_type) 511{ 512 struct si_query_sw *query; 513 514 query = CALLOC_STRUCT(si_query_sw); 515 if (!query) 516 return NULL; 517 518 query->b.type = query_type; 519 query->b.ops = &sw_query_ops; 520 521 return (struct pipe_query *)query; 522} 523 524void si_query_buffer_destroy(struct si_screen *sscreen, struct si_query_buffer *buffer) 525{ 526 struct si_query_buffer *prev = buffer->previous; 527 528 /* Release all query buffers. */ 529 while (prev) { 530 struct si_query_buffer *qbuf = prev; 531 prev = prev->previous; 532 si_resource_reference(&qbuf->buf, NULL); 533 FREE(qbuf); 534 } 535 536 si_resource_reference(&buffer->buf, NULL); 537} 538 539void si_query_buffer_reset(struct si_context *sctx, struct si_query_buffer *buffer) 540{ 541 /* Discard all query buffers except for the oldest. */ 542 while (buffer->previous) { 543 struct si_query_buffer *qbuf = buffer->previous; 544 buffer->previous = qbuf->previous; 545 546 si_resource_reference(&buffer->buf, NULL); 547 buffer->buf = qbuf->buf; /* move ownership */ 548 FREE(qbuf); 549 } 550 buffer->results_end = 0; 551 552 if (!buffer->buf) 553 return; 554 555 /* Discard even the oldest buffer if it can't be mapped without a stall. */ 556 if (si_rings_is_buffer_referenced(sctx, buffer->buf->buf, RADEON_USAGE_READWRITE) || 557 !sctx->ws->buffer_wait(buffer->buf->buf, 0, RADEON_USAGE_READWRITE)) { 558 si_resource_reference(&buffer->buf, NULL); 559 } else { 560 buffer->unprepared = true; 561 } 562} 563 564bool si_query_buffer_alloc(struct si_context *sctx, struct si_query_buffer *buffer, 565 bool (*prepare_buffer)(struct si_context *, struct si_query_buffer*), 566 unsigned size) 567{ 568 bool unprepared = buffer->unprepared; 569 buffer->unprepared = false; 570 571 if (!buffer->buf || buffer->results_end + size > buffer->buf->b.b.width0) { 572 if (buffer->buf) { 573 struct si_query_buffer *qbuf = MALLOC_STRUCT(si_query_buffer); 574 memcpy(qbuf, buffer, sizeof(*qbuf)); 575 buffer->previous = qbuf; 576 } 577 buffer->results_end = 0; 578 579 /* Queries are normally read by the CPU after 580 * being written by the gpu, hence staging is probably a good 581 * usage pattern. 582 */ 583 struct si_screen *screen = sctx->screen; 584 unsigned buf_size = MAX2(size, screen->info.min_alloc_size); 585 buffer->buf = si_resource( 586 pipe_buffer_create(&screen->b, 0, PIPE_USAGE_STAGING, buf_size)); 587 if (unlikely(!buffer->buf)) 588 return false; 589 unprepared = true; 590 } 591 592 if (unprepared && prepare_buffer) { 593 if (unlikely(!prepare_buffer(sctx, buffer))) { 594 si_resource_reference(&buffer->buf, NULL); 595 return false; 596 } 597 } 598 599 return true; 600} 601 602 603void si_query_hw_destroy(struct si_screen *sscreen, 604 struct si_query *squery) 605{ 606 struct si_query_hw *query = (struct si_query_hw *)squery; 607 608 si_query_buffer_destroy(sscreen, &query->buffer); 609 si_resource_reference(&query->workaround_buf, NULL); 610 FREE(squery); 611} 612 613static bool si_query_hw_prepare_buffer(struct si_context *sctx, 614 struct si_query_buffer *qbuf) 615{ 616 static const struct si_query_hw si_query_hw_s; 617 struct si_query_hw *query = container_of(qbuf, &si_query_hw_s, buffer); 618 struct si_screen *screen = sctx->screen; 619 620 /* The caller ensures that the buffer is currently unused by the GPU. */ 621 uint32_t *results = screen->ws->buffer_map(qbuf->buf->buf, NULL, 622 PIPE_TRANSFER_WRITE | 623 PIPE_TRANSFER_UNSYNCHRONIZED); 624 if (!results) 625 return false; 626 627 memset(results, 0, qbuf->buf->b.b.width0); 628 629 if (query->b.type == PIPE_QUERY_OCCLUSION_COUNTER || 630 query->b.type == PIPE_QUERY_OCCLUSION_PREDICATE || 631 query->b.type == PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE) { 632 unsigned max_rbs = screen->info.num_render_backends; 633 unsigned enabled_rb_mask = screen->info.enabled_rb_mask; 634 unsigned num_results; 635 unsigned i, j; 636 637 /* Set top bits for unused backends. */ 638 num_results = qbuf->buf->b.b.width0 / query->result_size; 639 for (j = 0; j < num_results; j++) { 640 for (i = 0; i < max_rbs; i++) { 641 if (!(enabled_rb_mask & (1<<i))) { 642 results[(i * 4)+1] = 0x80000000; 643 results[(i * 4)+3] = 0x80000000; 644 } 645 } 646 results += 4 * max_rbs; 647 } 648 } 649 650 return true; 651} 652 653static void si_query_hw_get_result_resource(struct si_context *sctx, 654 struct si_query *squery, 655 bool wait, 656 enum pipe_query_value_type result_type, 657 int index, 658 struct pipe_resource *resource, 659 unsigned offset); 660 661static void si_query_hw_do_emit_start(struct si_context *sctx, 662 struct si_query_hw *query, 663 struct si_resource *buffer, 664 uint64_t va); 665static void si_query_hw_do_emit_stop(struct si_context *sctx, 666 struct si_query_hw *query, 667 struct si_resource *buffer, 668 uint64_t va); 669static void si_query_hw_add_result(struct si_screen *sscreen, 670 struct si_query_hw *, void *buffer, 671 union pipe_query_result *result); 672static void si_query_hw_clear_result(struct si_query_hw *, 673 union pipe_query_result *); 674 675static struct si_query_hw_ops query_hw_default_hw_ops = { 676 .prepare_buffer = si_query_hw_prepare_buffer, 677 .emit_start = si_query_hw_do_emit_start, 678 .emit_stop = si_query_hw_do_emit_stop, 679 .clear_result = si_query_hw_clear_result, 680 .add_result = si_query_hw_add_result, 681}; 682 683static struct pipe_query *si_query_hw_create(struct si_screen *sscreen, 684 unsigned query_type, 685 unsigned index) 686{ 687 struct si_query_hw *query = CALLOC_STRUCT(si_query_hw); 688 if (!query) 689 return NULL; 690 691 query->b.type = query_type; 692 query->b.ops = &query_hw_ops; 693 query->ops = &query_hw_default_hw_ops; 694 695 switch (query_type) { 696 case PIPE_QUERY_OCCLUSION_COUNTER: 697 case PIPE_QUERY_OCCLUSION_PREDICATE: 698 case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE: 699 query->result_size = 16 * sscreen->info.num_render_backends; 700 query->result_size += 16; /* for the fence + alignment */ 701 query->b.num_cs_dw_suspend = 6 + si_cp_write_fence_dwords(sscreen); 702 break; 703 case SI_QUERY_TIME_ELAPSED_SDMA: 704 /* GET_GLOBAL_TIMESTAMP only works if the offset is a multiple of 32. */ 705 query->result_size = 64; 706 break; 707 case PIPE_QUERY_TIME_ELAPSED: 708 query->result_size = 24; 709 query->b.num_cs_dw_suspend = 8 + si_cp_write_fence_dwords(sscreen); 710 break; 711 case PIPE_QUERY_TIMESTAMP: 712 query->result_size = 16; 713 query->b.num_cs_dw_suspend = 8 + si_cp_write_fence_dwords(sscreen); 714 query->flags = SI_QUERY_HW_FLAG_NO_START; 715 break; 716 case PIPE_QUERY_PRIMITIVES_EMITTED: 717 case PIPE_QUERY_PRIMITIVES_GENERATED: 718 case PIPE_QUERY_SO_STATISTICS: 719 case PIPE_QUERY_SO_OVERFLOW_PREDICATE: 720 /* NumPrimitivesWritten, PrimitiveStorageNeeded. */ 721 query->result_size = 32; 722 query->b.num_cs_dw_suspend = 6; 723 query->stream = index; 724 break; 725 case PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE: 726 /* NumPrimitivesWritten, PrimitiveStorageNeeded. */ 727 query->result_size = 32 * SI_MAX_STREAMS; 728 query->b.num_cs_dw_suspend = 6 * SI_MAX_STREAMS; 729 break; 730 case PIPE_QUERY_PIPELINE_STATISTICS: 731 /* 11 values on GCN. */ 732 query->result_size = 11 * 16; 733 query->result_size += 8; /* for the fence + alignment */ 734 query->b.num_cs_dw_suspend = 6 + si_cp_write_fence_dwords(sscreen); 735 break; 736 default: 737 assert(0); 738 FREE(query); 739 return NULL; 740 } 741 742 return (struct pipe_query *)query; 743} 744 745static void si_update_occlusion_query_state(struct si_context *sctx, 746 unsigned type, int diff) 747{ 748 if (type == PIPE_QUERY_OCCLUSION_COUNTER || 749 type == PIPE_QUERY_OCCLUSION_PREDICATE || 750 type == PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE) { 751 bool old_enable = sctx->num_occlusion_queries != 0; 752 bool old_perfect_enable = 753 sctx->num_perfect_occlusion_queries != 0; 754 bool enable, perfect_enable; 755 756 sctx->num_occlusion_queries += diff; 757 assert(sctx->num_occlusion_queries >= 0); 758 759 if (type != PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE) { 760 sctx->num_perfect_occlusion_queries += diff; 761 assert(sctx->num_perfect_occlusion_queries >= 0); 762 } 763 764 enable = sctx->num_occlusion_queries != 0; 765 perfect_enable = sctx->num_perfect_occlusion_queries != 0; 766 767 if (enable != old_enable || perfect_enable != old_perfect_enable) { 768 si_set_occlusion_query_state(sctx, old_perfect_enable); 769 } 770 } 771} 772 773static unsigned event_type_for_stream(unsigned stream) 774{ 775 switch (stream) { 776 default: 777 case 0: return V_028A90_SAMPLE_STREAMOUTSTATS; 778 case 1: return V_028A90_SAMPLE_STREAMOUTSTATS1; 779 case 2: return V_028A90_SAMPLE_STREAMOUTSTATS2; 780 case 3: return V_028A90_SAMPLE_STREAMOUTSTATS3; 781 } 782} 783 784static void emit_sample_streamout(struct radeon_cmdbuf *cs, uint64_t va, 785 unsigned stream) 786{ 787 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 2, 0)); 788 radeon_emit(cs, EVENT_TYPE(event_type_for_stream(stream)) | EVENT_INDEX(3)); 789 radeon_emit(cs, va); 790 radeon_emit(cs, va >> 32); 791} 792 793static void si_query_hw_do_emit_start(struct si_context *sctx, 794 struct si_query_hw *query, 795 struct si_resource *buffer, 796 uint64_t va) 797{ 798 struct radeon_cmdbuf *cs = sctx->gfx_cs; 799 800 switch (query->b.type) { 801 case SI_QUERY_TIME_ELAPSED_SDMA: 802 si_dma_emit_timestamp(sctx, buffer, va - buffer->gpu_address); 803 return; 804 case PIPE_QUERY_OCCLUSION_COUNTER: 805 case PIPE_QUERY_OCCLUSION_PREDICATE: 806 case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE: 807 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 2, 0)); 808 radeon_emit(cs, EVENT_TYPE(V_028A90_ZPASS_DONE) | EVENT_INDEX(1)); 809 radeon_emit(cs, va); 810 radeon_emit(cs, va >> 32); 811 break; 812 case PIPE_QUERY_PRIMITIVES_EMITTED: 813 case PIPE_QUERY_PRIMITIVES_GENERATED: 814 case PIPE_QUERY_SO_STATISTICS: 815 case PIPE_QUERY_SO_OVERFLOW_PREDICATE: 816 emit_sample_streamout(cs, va, query->stream); 817 break; 818 case PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE: 819 for (unsigned stream = 0; stream < SI_MAX_STREAMS; ++stream) 820 emit_sample_streamout(cs, va + 32 * stream, stream); 821 break; 822 case PIPE_QUERY_TIME_ELAPSED: 823 si_cp_release_mem(sctx, V_028A90_BOTTOM_OF_PIPE_TS, 0, 824 EOP_DST_SEL_MEM, EOP_INT_SEL_NONE, 825 EOP_DATA_SEL_TIMESTAMP, NULL, va, 826 0, query->b.type); 827 break; 828 case PIPE_QUERY_PIPELINE_STATISTICS: 829 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 2, 0)); 830 radeon_emit(cs, EVENT_TYPE(V_028A90_SAMPLE_PIPELINESTAT) | EVENT_INDEX(2)); 831 radeon_emit(cs, va); 832 radeon_emit(cs, va >> 32); 833 break; 834 default: 835 assert(0); 836 } 837 radeon_add_to_buffer_list(sctx, sctx->gfx_cs, query->buffer.buf, RADEON_USAGE_WRITE, 838 RADEON_PRIO_QUERY); 839} 840 841static void si_query_hw_emit_start(struct si_context *sctx, 842 struct si_query_hw *query) 843{ 844 uint64_t va; 845 846 if (!si_query_buffer_alloc(sctx, &query->buffer, query->ops->prepare_buffer, 847 query->result_size)) 848 return; 849 850 si_update_occlusion_query_state(sctx, query->b.type, 1); 851 si_update_prims_generated_query_state(sctx, query->b.type, 1); 852 853 if (query->b.type != SI_QUERY_TIME_ELAPSED_SDMA) 854 si_need_gfx_cs_space(sctx); 855 856 va = query->buffer.buf->gpu_address + query->buffer.results_end; 857 query->ops->emit_start(sctx, query, query->buffer.buf, va); 858} 859 860static void si_query_hw_do_emit_stop(struct si_context *sctx, 861 struct si_query_hw *query, 862 struct si_resource *buffer, 863 uint64_t va) 864{ 865 struct radeon_cmdbuf *cs = sctx->gfx_cs; 866 uint64_t fence_va = 0; 867 868 switch (query->b.type) { 869 case SI_QUERY_TIME_ELAPSED_SDMA: 870 si_dma_emit_timestamp(sctx, buffer, va + 32 - buffer->gpu_address); 871 return; 872 case PIPE_QUERY_OCCLUSION_COUNTER: 873 case PIPE_QUERY_OCCLUSION_PREDICATE: 874 case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE: 875 va += 8; 876 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 2, 0)); 877 radeon_emit(cs, EVENT_TYPE(V_028A90_ZPASS_DONE) | EVENT_INDEX(1)); 878 radeon_emit(cs, va); 879 radeon_emit(cs, va >> 32); 880 881 fence_va = va + sctx->screen->info.num_render_backends * 16 - 8; 882 break; 883 case PIPE_QUERY_PRIMITIVES_EMITTED: 884 case PIPE_QUERY_PRIMITIVES_GENERATED: 885 case PIPE_QUERY_SO_STATISTICS: 886 case PIPE_QUERY_SO_OVERFLOW_PREDICATE: 887 va += 16; 888 emit_sample_streamout(cs, va, query->stream); 889 break; 890 case PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE: 891 va += 16; 892 for (unsigned stream = 0; stream < SI_MAX_STREAMS; ++stream) 893 emit_sample_streamout(cs, va + 32 * stream, stream); 894 break; 895 case PIPE_QUERY_TIME_ELAPSED: 896 va += 8; 897 /* fall through */ 898 case PIPE_QUERY_TIMESTAMP: 899 si_cp_release_mem(sctx, V_028A90_BOTTOM_OF_PIPE_TS, 0, 900 EOP_DST_SEL_MEM, EOP_INT_SEL_NONE, 901 EOP_DATA_SEL_TIMESTAMP, NULL, va, 902 0, query->b.type); 903 fence_va = va + 8; 904 break; 905 case PIPE_QUERY_PIPELINE_STATISTICS: { 906 unsigned sample_size = (query->result_size - 8) / 2; 907 908 va += sample_size; 909 radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 2, 0)); 910 radeon_emit(cs, EVENT_TYPE(V_028A90_SAMPLE_PIPELINESTAT) | EVENT_INDEX(2)); 911 radeon_emit(cs, va); 912 radeon_emit(cs, va >> 32); 913 914 fence_va = va + sample_size; 915 break; 916 } 917 default: 918 assert(0); 919 } 920 radeon_add_to_buffer_list(sctx, sctx->gfx_cs, query->buffer.buf, RADEON_USAGE_WRITE, 921 RADEON_PRIO_QUERY); 922 923 if (fence_va) { 924 si_cp_release_mem(sctx, V_028A90_BOTTOM_OF_PIPE_TS, 0, 925 EOP_DST_SEL_MEM, EOP_INT_SEL_NONE, 926 EOP_DATA_SEL_VALUE_32BIT, 927 query->buffer.buf, fence_va, 0x80000000, 928 query->b.type); 929 } 930} 931 932static void si_query_hw_emit_stop(struct si_context *sctx, 933 struct si_query_hw *query) 934{ 935 uint64_t va; 936 937 /* The queries which need begin already called this in begin_query. */ 938 if (query->flags & SI_QUERY_HW_FLAG_NO_START) { 939 si_need_gfx_cs_space(sctx); 940 if (!si_query_buffer_alloc(sctx, &query->buffer, query->ops->prepare_buffer, 941 query->result_size)) 942 return; 943 } 944 945 if (!query->buffer.buf) 946 return; // previous buffer allocation failure 947 948 /* emit end query */ 949 va = query->buffer.buf->gpu_address + query->buffer.results_end; 950 951 query->ops->emit_stop(sctx, query, query->buffer.buf, va); 952 953 query->buffer.results_end += query->result_size; 954 955 si_update_occlusion_query_state(sctx, query->b.type, -1); 956 si_update_prims_generated_query_state(sctx, query->b.type, -1); 957} 958 959static void emit_set_predicate(struct si_context *ctx, 960 struct si_resource *buf, uint64_t va, 961 uint32_t op) 962{ 963 struct radeon_cmdbuf *cs = ctx->gfx_cs; 964 965 if (ctx->chip_class >= GFX9) { 966 radeon_emit(cs, PKT3(PKT3_SET_PREDICATION, 2, 0)); 967 radeon_emit(cs, op); 968 radeon_emit(cs, va); 969 radeon_emit(cs, va >> 32); 970 } else { 971 radeon_emit(cs, PKT3(PKT3_SET_PREDICATION, 1, 0)); 972 radeon_emit(cs, va); 973 radeon_emit(cs, op | ((va >> 32) & 0xFF)); 974 } 975 radeon_add_to_buffer_list(ctx, ctx->gfx_cs, buf, RADEON_USAGE_READ, 976 RADEON_PRIO_QUERY); 977} 978 979static void si_emit_query_predication(struct si_context *ctx) 980{ 981 struct si_query_hw *query = (struct si_query_hw *)ctx->render_cond; 982 struct si_query_buffer *qbuf; 983 uint32_t op; 984 bool flag_wait, invert; 985 986 if (!query) 987 return; 988 989 invert = ctx->render_cond_invert; 990 flag_wait = ctx->render_cond_mode == PIPE_RENDER_COND_WAIT || 991 ctx->render_cond_mode == PIPE_RENDER_COND_BY_REGION_WAIT; 992 993 if (query->workaround_buf) { 994 op = PRED_OP(PREDICATION_OP_BOOL64); 995 } else { 996 switch (query->b.type) { 997 case PIPE_QUERY_OCCLUSION_COUNTER: 998 case PIPE_QUERY_OCCLUSION_PREDICATE: 999 case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE: 1000 op = PRED_OP(PREDICATION_OP_ZPASS); 1001 break; 1002 case PIPE_QUERY_SO_OVERFLOW_PREDICATE: 1003 case PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE: 1004 op = PRED_OP(PREDICATION_OP_PRIMCOUNT); 1005 invert = !invert; 1006 break; 1007 default: 1008 assert(0); 1009 return; 1010 } 1011 } 1012 1013 /* if true then invert, see GL_ARB_conditional_render_inverted */ 1014 if (invert) 1015 op |= PREDICATION_DRAW_NOT_VISIBLE; /* Draw if not visible or overflow */ 1016 else 1017 op |= PREDICATION_DRAW_VISIBLE; /* Draw if visible or no overflow */ 1018 1019 /* Use the value written by compute shader as a workaround. Note that 1020 * the wait flag does not apply in this predication mode. 1021 * 1022 * The shader outputs the result value to L2. Workarounds only affect VI 1023 * and later, where the CP reads data from L2, so we don't need an 1024 * additional flush. 1025 */ 1026 if (query->workaround_buf) { 1027 uint64_t va = query->workaround_buf->gpu_address + query->workaround_offset; 1028 emit_set_predicate(ctx, query->workaround_buf, va, op); 1029 return; 1030 } 1031 1032 op |= flag_wait ? PREDICATION_HINT_WAIT : PREDICATION_HINT_NOWAIT_DRAW; 1033 1034 /* emit predicate packets for all data blocks */ 1035 for (qbuf = &query->buffer; qbuf; qbuf = qbuf->previous) { 1036 unsigned results_base = 0; 1037 uint64_t va_base = qbuf->buf->gpu_address; 1038 1039 while (results_base < qbuf->results_end) { 1040 uint64_t va = va_base + results_base; 1041 1042 if (query->b.type == PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE) { 1043 for (unsigned stream = 0; stream < SI_MAX_STREAMS; ++stream) { 1044 emit_set_predicate(ctx, qbuf->buf, va + 32 * stream, op); 1045 1046 /* set CONTINUE bit for all packets except the first */ 1047 op |= PREDICATION_CONTINUE; 1048 } 1049 } else { 1050 emit_set_predicate(ctx, qbuf->buf, va, op); 1051 op |= PREDICATION_CONTINUE; 1052 } 1053 1054 results_base += query->result_size; 1055 } 1056 } 1057} 1058 1059static struct pipe_query *si_create_query(struct pipe_context *ctx, unsigned query_type, unsigned index) 1060{ 1061 struct si_screen *sscreen = 1062 (struct si_screen *)ctx->screen; 1063 1064 if (query_type == PIPE_QUERY_TIMESTAMP_DISJOINT || 1065 query_type == PIPE_QUERY_GPU_FINISHED || 1066 (query_type >= PIPE_QUERY_DRIVER_SPECIFIC && 1067 query_type != SI_QUERY_TIME_ELAPSED_SDMA)) 1068 return si_query_sw_create(query_type); 1069 1070 return si_query_hw_create(sscreen, query_type, index); 1071} 1072 1073static void si_destroy_query(struct pipe_context *ctx, struct pipe_query *query) 1074{ 1075 struct si_context *sctx = (struct si_context *)ctx; 1076 struct si_query *squery = (struct si_query *)query; 1077 1078 squery->ops->destroy(sctx->screen, squery); 1079} 1080 1081static boolean si_begin_query(struct pipe_context *ctx, 1082 struct pipe_query *query) 1083{ 1084 struct si_context *sctx = (struct si_context *)ctx; 1085 struct si_query *squery = (struct si_query *)query; 1086 1087 return squery->ops->begin(sctx, squery); 1088} 1089 1090bool si_query_hw_begin(struct si_context *sctx, 1091 struct si_query *squery) 1092{ 1093 struct si_query_hw *query = (struct si_query_hw *)squery; 1094 1095 if (query->flags & SI_QUERY_HW_FLAG_NO_START) { 1096 assert(0); 1097 return false; 1098 } 1099 1100 if (!(query->flags & SI_QUERY_HW_FLAG_BEGIN_RESUMES)) 1101 si_query_buffer_reset(sctx, &query->buffer); 1102 1103 si_resource_reference(&query->workaround_buf, NULL); 1104 1105 si_query_hw_emit_start(sctx, query); 1106 if (!query->buffer.buf) 1107 return false; 1108 1109 LIST_ADDTAIL(&query->b.active_list, &sctx->active_queries); 1110 sctx->num_cs_dw_queries_suspend += query->b.num_cs_dw_suspend; 1111 return true; 1112} 1113 1114static bool si_end_query(struct pipe_context *ctx, struct pipe_query *query) 1115{ 1116 struct si_context *sctx = (struct si_context *)ctx; 1117 struct si_query *squery = (struct si_query *)query; 1118 1119 return squery->ops->end(sctx, squery); 1120} 1121 1122bool si_query_hw_end(struct si_context *sctx, 1123 struct si_query *squery) 1124{ 1125 struct si_query_hw *query = (struct si_query_hw *)squery; 1126 1127 if (query->flags & SI_QUERY_HW_FLAG_NO_START) 1128 si_query_buffer_reset(sctx, &query->buffer); 1129 1130 si_query_hw_emit_stop(sctx, query); 1131 1132 if (!(query->flags & SI_QUERY_HW_FLAG_NO_START)) { 1133 LIST_DELINIT(&query->b.active_list); 1134 sctx->num_cs_dw_queries_suspend -= query->b.num_cs_dw_suspend; 1135 } 1136 1137 if (!query->buffer.buf) 1138 return false; 1139 1140 return true; 1141} 1142 1143static void si_get_hw_query_params(struct si_context *sctx, 1144 struct si_query_hw *squery, int index, 1145 struct si_hw_query_params *params) 1146{ 1147 unsigned max_rbs = sctx->screen->info.num_render_backends; 1148 1149 params->pair_stride = 0; 1150 params->pair_count = 1; 1151 1152 switch (squery->b.type) { 1153 case PIPE_QUERY_OCCLUSION_COUNTER: 1154 case PIPE_QUERY_OCCLUSION_PREDICATE: 1155 case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE: 1156 params->start_offset = 0; 1157 params->end_offset = 8; 1158 params->fence_offset = max_rbs * 16; 1159 params->pair_stride = 16; 1160 params->pair_count = max_rbs; 1161 break; 1162 case PIPE_QUERY_TIME_ELAPSED: 1163 params->start_offset = 0; 1164 params->end_offset = 8; 1165 params->fence_offset = 16; 1166 break; 1167 case PIPE_QUERY_TIMESTAMP: 1168 params->start_offset = 0; 1169 params->end_offset = 0; 1170 params->fence_offset = 8; 1171 break; 1172 case PIPE_QUERY_PRIMITIVES_EMITTED: 1173 params->start_offset = 8; 1174 params->end_offset = 24; 1175 params->fence_offset = params->end_offset + 4; 1176 break; 1177 case PIPE_QUERY_PRIMITIVES_GENERATED: 1178 params->start_offset = 0; 1179 params->end_offset = 16; 1180 params->fence_offset = params->end_offset + 4; 1181 break; 1182 case PIPE_QUERY_SO_STATISTICS: 1183 params->start_offset = 8 - index * 8; 1184 params->end_offset = 24 - index * 8; 1185 params->fence_offset = params->end_offset + 4; 1186 break; 1187 case PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE: 1188 params->pair_count = SI_MAX_STREAMS; 1189 params->pair_stride = 32; 1190 case PIPE_QUERY_SO_OVERFLOW_PREDICATE: 1191 params->start_offset = 0; 1192 params->end_offset = 16; 1193 1194 /* We can re-use the high dword of the last 64-bit value as a 1195 * fence: it is initialized as 0, and the high bit is set by 1196 * the write of the streamout stats event. 1197 */ 1198 params->fence_offset = squery->result_size - 4; 1199 break; 1200 case PIPE_QUERY_PIPELINE_STATISTICS: 1201 { 1202 static const unsigned offsets[] = {56, 48, 24, 32, 40, 16, 8, 0, 64, 72, 80}; 1203 params->start_offset = offsets[index]; 1204 params->end_offset = 88 + offsets[index]; 1205 params->fence_offset = 2 * 88; 1206 break; 1207 } 1208 default: 1209 unreachable("si_get_hw_query_params unsupported"); 1210 } 1211} 1212 1213static unsigned si_query_read_result(void *map, unsigned start_index, unsigned end_index, 1214 bool test_status_bit) 1215{ 1216 uint32_t *current_result = (uint32_t*)map; 1217 uint64_t start, end; 1218 1219 start = (uint64_t)current_result[start_index] | 1220 (uint64_t)current_result[start_index+1] << 32; 1221 end = (uint64_t)current_result[end_index] | 1222 (uint64_t)current_result[end_index+1] << 32; 1223 1224 if (!test_status_bit || 1225 ((start & 0x8000000000000000UL) && (end & 0x8000000000000000UL))) { 1226 return end - start; 1227 } 1228 return 0; 1229} 1230 1231static void si_query_hw_add_result(struct si_screen *sscreen, 1232 struct si_query_hw *query, 1233 void *buffer, 1234 union pipe_query_result *result) 1235{ 1236 unsigned max_rbs = sscreen->info.num_render_backends; 1237 1238 switch (query->b.type) { 1239 case PIPE_QUERY_OCCLUSION_COUNTER: { 1240 for (unsigned i = 0; i < max_rbs; ++i) { 1241 unsigned results_base = i * 16; 1242 result->u64 += 1243 si_query_read_result(buffer + results_base, 0, 2, true); 1244 } 1245 break; 1246 } 1247 case PIPE_QUERY_OCCLUSION_PREDICATE: 1248 case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE: { 1249 for (unsigned i = 0; i < max_rbs; ++i) { 1250 unsigned results_base = i * 16; 1251 result->b = result->b || 1252 si_query_read_result(buffer + results_base, 0, 2, true) != 0; 1253 } 1254 break; 1255 } 1256 case PIPE_QUERY_TIME_ELAPSED: 1257 result->u64 += si_query_read_result(buffer, 0, 2, false); 1258 break; 1259 case SI_QUERY_TIME_ELAPSED_SDMA: 1260 result->u64 += si_query_read_result(buffer, 0, 32/4, false); 1261 break; 1262 case PIPE_QUERY_TIMESTAMP: 1263 result->u64 = *(uint64_t*)buffer; 1264 break; 1265 case PIPE_QUERY_PRIMITIVES_EMITTED: 1266 /* SAMPLE_STREAMOUTSTATS stores this structure: 1267 * { 1268 * u64 NumPrimitivesWritten; 1269 * u64 PrimitiveStorageNeeded; 1270 * } 1271 * We only need NumPrimitivesWritten here. */ 1272 result->u64 += si_query_read_result(buffer, 2, 6, true); 1273 break; 1274 case PIPE_QUERY_PRIMITIVES_GENERATED: 1275 /* Here we read PrimitiveStorageNeeded. */ 1276 result->u64 += si_query_read_result(buffer, 0, 4, true); 1277 break; 1278 case PIPE_QUERY_SO_STATISTICS: 1279 result->so_statistics.num_primitives_written += 1280 si_query_read_result(buffer, 2, 6, true); 1281 result->so_statistics.primitives_storage_needed += 1282 si_query_read_result(buffer, 0, 4, true); 1283 break; 1284 case PIPE_QUERY_SO_OVERFLOW_PREDICATE: 1285 result->b = result->b || 1286 si_query_read_result(buffer, 2, 6, true) != 1287 si_query_read_result(buffer, 0, 4, true); 1288 break; 1289 case PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE: 1290 for (unsigned stream = 0; stream < SI_MAX_STREAMS; ++stream) { 1291 result->b = result->b || 1292 si_query_read_result(buffer, 2, 6, true) != 1293 si_query_read_result(buffer, 0, 4, true); 1294 buffer = (char *)buffer + 32; 1295 } 1296 break; 1297 case PIPE_QUERY_PIPELINE_STATISTICS: 1298 result->pipeline_statistics.ps_invocations += 1299 si_query_read_result(buffer, 0, 22, false); 1300 result->pipeline_statistics.c_primitives += 1301 si_query_read_result(buffer, 2, 24, false); 1302 result->pipeline_statistics.c_invocations += 1303 si_query_read_result(buffer, 4, 26, false); 1304 result->pipeline_statistics.vs_invocations += 1305 si_query_read_result(buffer, 6, 28, false); 1306 result->pipeline_statistics.gs_invocations += 1307 si_query_read_result(buffer, 8, 30, false); 1308 result->pipeline_statistics.gs_primitives += 1309 si_query_read_result(buffer, 10, 32, false); 1310 result->pipeline_statistics.ia_primitives += 1311 si_query_read_result(buffer, 12, 34, false); 1312 result->pipeline_statistics.ia_vertices += 1313 si_query_read_result(buffer, 14, 36, false); 1314 result->pipeline_statistics.hs_invocations += 1315 si_query_read_result(buffer, 16, 38, false); 1316 result->pipeline_statistics.ds_invocations += 1317 si_query_read_result(buffer, 18, 40, false); 1318 result->pipeline_statistics.cs_invocations += 1319 si_query_read_result(buffer, 20, 42, false); 1320#if 0 /* for testing */ 1321 printf("Pipeline stats: IA verts=%llu, IA prims=%llu, VS=%llu, HS=%llu, " 1322 "DS=%llu, GS=%llu, GS prims=%llu, Clipper=%llu, " 1323 "Clipper prims=%llu, PS=%llu, CS=%llu\n", 1324 result->pipeline_statistics.ia_vertices, 1325 result->pipeline_statistics.ia_primitives, 1326 result->pipeline_statistics.vs_invocations, 1327 result->pipeline_statistics.hs_invocations, 1328 result->pipeline_statistics.ds_invocations, 1329 result->pipeline_statistics.gs_invocations, 1330 result->pipeline_statistics.gs_primitives, 1331 result->pipeline_statistics.c_invocations, 1332 result->pipeline_statistics.c_primitives, 1333 result->pipeline_statistics.ps_invocations, 1334 result->pipeline_statistics.cs_invocations); 1335#endif 1336 break; 1337 default: 1338 assert(0); 1339 } 1340} 1341 1342void si_query_hw_suspend(struct si_context *sctx, struct si_query *query) 1343{ 1344 si_query_hw_emit_stop(sctx, (struct si_query_hw *)query); 1345} 1346 1347void si_query_hw_resume(struct si_context *sctx, struct si_query *query) 1348{ 1349 si_query_hw_emit_start(sctx, (struct si_query_hw *)query); 1350} 1351 1352static const struct si_query_ops query_hw_ops = { 1353 .destroy = si_query_hw_destroy, 1354 .begin = si_query_hw_begin, 1355 .end = si_query_hw_end, 1356 .get_result = si_query_hw_get_result, 1357 .get_result_resource = si_query_hw_get_result_resource, 1358 1359 .suspend = si_query_hw_suspend, 1360 .resume = si_query_hw_resume, 1361}; 1362 1363static boolean si_get_query_result(struct pipe_context *ctx, 1364 struct pipe_query *query, boolean wait, 1365 union pipe_query_result *result) 1366{ 1367 struct si_context *sctx = (struct si_context *)ctx; 1368 struct si_query *squery = (struct si_query *)query; 1369 1370 return squery->ops->get_result(sctx, squery, wait, result); 1371} 1372 1373static void si_get_query_result_resource(struct pipe_context *ctx, 1374 struct pipe_query *query, 1375 boolean wait, 1376 enum pipe_query_value_type result_type, 1377 int index, 1378 struct pipe_resource *resource, 1379 unsigned offset) 1380{ 1381 struct si_context *sctx = (struct si_context *)ctx; 1382 struct si_query *squery = (struct si_query *)query; 1383 1384 squery->ops->get_result_resource(sctx, squery, wait, result_type, index, 1385 resource, offset); 1386} 1387 1388static void si_query_hw_clear_result(struct si_query_hw *query, 1389 union pipe_query_result *result) 1390{ 1391 util_query_clear_result(result, query->b.type); 1392} 1393 1394bool si_query_hw_get_result(struct si_context *sctx, 1395 struct si_query *squery, 1396 bool wait, union pipe_query_result *result) 1397{ 1398 struct si_screen *sscreen = sctx->screen; 1399 struct si_query_hw *query = (struct si_query_hw *)squery; 1400 struct si_query_buffer *qbuf; 1401 1402 query->ops->clear_result(query, result); 1403 1404 for (qbuf = &query->buffer; qbuf; qbuf = qbuf->previous) { 1405 unsigned usage = PIPE_TRANSFER_READ | 1406 (wait ? 0 : PIPE_TRANSFER_DONTBLOCK); 1407 unsigned results_base = 0; 1408 void *map; 1409 1410 if (squery->b.flushed) 1411 map = sctx->ws->buffer_map(qbuf->buf->buf, NULL, usage); 1412 else 1413 map = si_buffer_map_sync_with_rings(sctx, qbuf->buf, usage); 1414 1415 if (!map) 1416 return false; 1417 1418 while (results_base != qbuf->results_end) { 1419 query->ops->add_result(sscreen, query, map + results_base, 1420 result); 1421 results_base += query->result_size; 1422 } 1423 } 1424 1425 /* Convert the time to expected units. */ 1426 if (squery->type == PIPE_QUERY_TIME_ELAPSED || 1427 squery->type == SI_QUERY_TIME_ELAPSED_SDMA || 1428 squery->type == PIPE_QUERY_TIMESTAMP) { 1429 result->u64 = (1000000 * result->u64) / sscreen->info.clock_crystal_freq; 1430 } 1431 return true; 1432} 1433 1434static void si_restore_qbo_state(struct si_context *sctx, 1435 struct si_qbo_state *st) 1436{ 1437 sctx->b.bind_compute_state(&sctx->b, st->saved_compute); 1438 1439 sctx->b.set_constant_buffer(&sctx->b, PIPE_SHADER_COMPUTE, 0, &st->saved_const0); 1440 pipe_resource_reference(&st->saved_const0.buffer, NULL); 1441 1442 sctx->b.set_shader_buffers(&sctx->b, PIPE_SHADER_COMPUTE, 0, 3, st->saved_ssbo, 1443 st->saved_ssbo_writable_mask); 1444 for (unsigned i = 0; i < 3; ++i) 1445 pipe_resource_reference(&st->saved_ssbo[i].buffer, NULL); 1446} 1447 1448static void si_query_hw_get_result_resource(struct si_context *sctx, 1449 struct si_query *squery, 1450 bool wait, 1451 enum pipe_query_value_type result_type, 1452 int index, 1453 struct pipe_resource *resource, 1454 unsigned offset) 1455{ 1456 struct si_query_hw *query = (struct si_query_hw *)squery; 1457 struct si_query_buffer *qbuf; 1458 struct si_query_buffer *qbuf_prev; 1459 struct pipe_resource *tmp_buffer = NULL; 1460 unsigned tmp_buffer_offset = 0; 1461 struct si_qbo_state saved_state = {}; 1462 struct pipe_grid_info grid = {}; 1463 struct pipe_constant_buffer constant_buffer = {}; 1464 struct pipe_shader_buffer ssbo[3]; 1465 struct si_hw_query_params params; 1466 struct { 1467 uint32_t end_offset; 1468 uint32_t result_stride; 1469 uint32_t result_count; 1470 uint32_t config; 1471 uint32_t fence_offset; 1472 uint32_t pair_stride; 1473 uint32_t pair_count; 1474 } consts; 1475 1476 if (!sctx->query_result_shader) { 1477 sctx->query_result_shader = si_create_query_result_cs(sctx); 1478 if (!sctx->query_result_shader) 1479 return; 1480 } 1481 1482 if (query->buffer.previous) { 1483 u_suballocator_alloc(sctx->allocator_zeroed_memory, 16, 16, 1484 &tmp_buffer_offset, &tmp_buffer); 1485 if (!tmp_buffer) 1486 return; 1487 } 1488 1489 si_save_qbo_state(sctx, &saved_state); 1490 1491 si_get_hw_query_params(sctx, query, index >= 0 ? index : 0, ¶ms); 1492 consts.end_offset = params.end_offset - params.start_offset; 1493 consts.fence_offset = params.fence_offset - params.start_offset; 1494 consts.result_stride = query->result_size; 1495 consts.pair_stride = params.pair_stride; 1496 consts.pair_count = params.pair_count; 1497 1498 constant_buffer.buffer_size = sizeof(consts); 1499 constant_buffer.user_buffer = &consts; 1500 1501 ssbo[1].buffer = tmp_buffer; 1502 ssbo[1].buffer_offset = tmp_buffer_offset; 1503 ssbo[1].buffer_size = 16; 1504 1505 ssbo[2] = ssbo[1]; 1506 1507 sctx->b.bind_compute_state(&sctx->b, sctx->query_result_shader); 1508 1509 grid.block[0] = 1; 1510 grid.block[1] = 1; 1511 grid.block[2] = 1; 1512 grid.grid[0] = 1; 1513 grid.grid[1] = 1; 1514 grid.grid[2] = 1; 1515 1516 consts.config = 0; 1517 if (index < 0) 1518 consts.config |= 4; 1519 if (query->b.type == PIPE_QUERY_OCCLUSION_PREDICATE || 1520 query->b.type == PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE) 1521 consts.config |= 8; 1522 else if (query->b.type == PIPE_QUERY_SO_OVERFLOW_PREDICATE || 1523 query->b.type == PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE) 1524 consts.config |= 8 | 256; 1525 else if (query->b.type == PIPE_QUERY_TIMESTAMP || 1526 query->b.type == PIPE_QUERY_TIME_ELAPSED) 1527 consts.config |= 32; 1528 1529 switch (result_type) { 1530 case PIPE_QUERY_TYPE_U64: 1531 case PIPE_QUERY_TYPE_I64: 1532 consts.config |= 64; 1533 break; 1534 case PIPE_QUERY_TYPE_I32: 1535 consts.config |= 128; 1536 break; 1537 case PIPE_QUERY_TYPE_U32: 1538 break; 1539 } 1540 1541 sctx->flags |= sctx->screen->barrier_flags.cp_to_L2; 1542 1543 for (qbuf = &query->buffer; qbuf; qbuf = qbuf_prev) { 1544 if (query->b.type != PIPE_QUERY_TIMESTAMP) { 1545 qbuf_prev = qbuf->previous; 1546 consts.result_count = qbuf->results_end / query->result_size; 1547 consts.config &= ~3; 1548 if (qbuf != &query->buffer) 1549 consts.config |= 1; 1550 if (qbuf->previous) 1551 consts.config |= 2; 1552 } else { 1553 /* Only read the last timestamp. */ 1554 qbuf_prev = NULL; 1555 consts.result_count = 0; 1556 consts.config |= 16; 1557 params.start_offset += qbuf->results_end - query->result_size; 1558 } 1559 1560 sctx->b.set_constant_buffer(&sctx->b, PIPE_SHADER_COMPUTE, 0, &constant_buffer); 1561 1562 ssbo[0].buffer = &qbuf->buf->b.b; 1563 ssbo[0].buffer_offset = params.start_offset; 1564 ssbo[0].buffer_size = qbuf->results_end - params.start_offset; 1565 1566 if (!qbuf->previous) { 1567 ssbo[2].buffer = resource; 1568 ssbo[2].buffer_offset = offset; 1569 ssbo[2].buffer_size = 8; 1570 1571 si_resource(resource)->TC_L2_dirty = true; 1572 } 1573 1574 sctx->b.set_shader_buffers(&sctx->b, PIPE_SHADER_COMPUTE, 0, 3, ssbo, 1575 1 << 2); 1576 1577 if (wait && qbuf == &query->buffer) { 1578 uint64_t va; 1579 1580 /* Wait for result availability. Wait only for readiness 1581 * of the last entry, since the fence writes should be 1582 * serialized in the CP. 1583 */ 1584 va = qbuf->buf->gpu_address + qbuf->results_end - query->result_size; 1585 va += params.fence_offset; 1586 1587 si_cp_wait_mem(sctx, sctx->gfx_cs, va, 0x80000000, 1588 0x80000000, WAIT_REG_MEM_EQUAL); 1589 } 1590 1591 sctx->b.launch_grid(&sctx->b, &grid); 1592 sctx->flags |= SI_CONTEXT_CS_PARTIAL_FLUSH; 1593 } 1594 1595 si_restore_qbo_state(sctx, &saved_state); 1596 pipe_resource_reference(&tmp_buffer, NULL); 1597} 1598 1599static void si_render_condition(struct pipe_context *ctx, 1600 struct pipe_query *query, 1601 boolean condition, 1602 enum pipe_render_cond_flag mode) 1603{ 1604 struct si_context *sctx = (struct si_context *)ctx; 1605 struct si_query_hw *squery = (struct si_query_hw *)query; 1606 struct si_atom *atom = &sctx->atoms.s.render_cond; 1607 1608 if (query) { 1609 bool needs_workaround = false; 1610 1611 /* There was a firmware regression in VI which causes successive 1612 * SET_PREDICATION packets to give the wrong answer for 1613 * non-inverted stream overflow predication. 1614 */ 1615 if (((sctx->chip_class == VI && sctx->screen->info.pfp_fw_feature < 49) || 1616 (sctx->chip_class == GFX9 && sctx->screen->info.pfp_fw_feature < 38)) && 1617 !condition && 1618 (squery->b.type == PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE || 1619 (squery->b.type == PIPE_QUERY_SO_OVERFLOW_PREDICATE && 1620 (squery->buffer.previous || 1621 squery->buffer.results_end > squery->result_size)))) { 1622 needs_workaround = true; 1623 } 1624 1625 if (needs_workaround && !squery->workaround_buf) { 1626 bool old_force_off = sctx->render_cond_force_off; 1627 sctx->render_cond_force_off = true; 1628 1629 u_suballocator_alloc( 1630 sctx->allocator_zeroed_memory, 8, 8, 1631 &squery->workaround_offset, 1632 (struct pipe_resource **)&squery->workaround_buf); 1633 1634 /* Reset to NULL to avoid a redundant SET_PREDICATION 1635 * from launching the compute grid. 1636 */ 1637 sctx->render_cond = NULL; 1638 1639 ctx->get_query_result_resource( 1640 ctx, query, true, PIPE_QUERY_TYPE_U64, 0, 1641 &squery->workaround_buf->b.b, squery->workaround_offset); 1642 1643 /* Settings this in the render cond atom is too late, 1644 * so set it here. */ 1645 sctx->flags |= sctx->screen->barrier_flags.L2_to_cp | 1646 SI_CONTEXT_FLUSH_FOR_RENDER_COND; 1647 1648 sctx->render_cond_force_off = old_force_off; 1649 } 1650 } 1651 1652 sctx->render_cond = query; 1653 sctx->render_cond_invert = condition; 1654 sctx->render_cond_mode = mode; 1655 1656 si_set_atom_dirty(sctx, atom, query != NULL); 1657} 1658 1659void si_suspend_queries(struct si_context *sctx) 1660{ 1661 struct si_query *query; 1662 1663 LIST_FOR_EACH_ENTRY(query, &sctx->active_queries, active_list) 1664 query->ops->suspend(sctx, query); 1665} 1666 1667void si_resume_queries(struct si_context *sctx) 1668{ 1669 struct si_query *query; 1670 1671 /* Check CS space here. Resuming must not be interrupted by flushes. */ 1672 si_need_gfx_cs_space(sctx); 1673 1674 LIST_FOR_EACH_ENTRY(query, &sctx->active_queries, active_list) 1675 query->ops->resume(sctx, query); 1676} 1677 1678#define XFULL(name_, query_type_, type_, result_type_, group_id_) \ 1679 { \ 1680 .name = name_, \ 1681 .query_type = SI_QUERY_##query_type_, \ 1682 .type = PIPE_DRIVER_QUERY_TYPE_##type_, \ 1683 .result_type = PIPE_DRIVER_QUERY_RESULT_TYPE_##result_type_, \ 1684 .group_id = group_id_ \ 1685 } 1686 1687#define X(name_, query_type_, type_, result_type_) \ 1688 XFULL(name_, query_type_, type_, result_type_, ~(unsigned)0) 1689 1690#define XG(group_, name_, query_type_, type_, result_type_) \ 1691 XFULL(name_, query_type_, type_, result_type_, SI_QUERY_GROUP_##group_) 1692 1693static struct pipe_driver_query_info si_driver_query_list[] = { 1694 X("num-compilations", NUM_COMPILATIONS, UINT64, CUMULATIVE), 1695 X("num-shaders-created", NUM_SHADERS_CREATED, UINT64, CUMULATIVE), 1696 X("num-shader-cache-hits", NUM_SHADER_CACHE_HITS, UINT64, CUMULATIVE), 1697 X("draw-calls", DRAW_CALLS, UINT64, AVERAGE), 1698 X("decompress-calls", DECOMPRESS_CALLS, UINT64, AVERAGE), 1699 X("MRT-draw-calls", MRT_DRAW_CALLS, UINT64, AVERAGE), 1700 X("prim-restart-calls", PRIM_RESTART_CALLS, UINT64, AVERAGE), 1701 X("spill-draw-calls", SPILL_DRAW_CALLS, UINT64, AVERAGE), 1702 X("compute-calls", COMPUTE_CALLS, UINT64, AVERAGE), 1703 X("spill-compute-calls", SPILL_COMPUTE_CALLS, UINT64, AVERAGE), 1704 X("dma-calls", DMA_CALLS, UINT64, AVERAGE), 1705 X("cp-dma-calls", CP_DMA_CALLS, UINT64, AVERAGE), 1706 X("num-vs-flushes", NUM_VS_FLUSHES, UINT64, AVERAGE), 1707 X("num-ps-flushes", NUM_PS_FLUSHES, UINT64, AVERAGE), 1708 X("num-cs-flushes", NUM_CS_FLUSHES, UINT64, AVERAGE), 1709 X("num-CB-cache-flushes", NUM_CB_CACHE_FLUSHES, UINT64, AVERAGE), 1710 X("num-DB-cache-flushes", NUM_DB_CACHE_FLUSHES, UINT64, AVERAGE), 1711 X("num-L2-invalidates", NUM_L2_INVALIDATES, UINT64, AVERAGE), 1712 X("num-L2-writebacks", NUM_L2_WRITEBACKS, UINT64, AVERAGE), 1713 X("num-resident-handles", NUM_RESIDENT_HANDLES, UINT64, AVERAGE), 1714 X("tc-offloaded-slots", TC_OFFLOADED_SLOTS, UINT64, AVERAGE), 1715 X("tc-direct-slots", TC_DIRECT_SLOTS, UINT64, AVERAGE), 1716 X("tc-num-syncs", TC_NUM_SYNCS, UINT64, AVERAGE), 1717 X("CS-thread-busy", CS_THREAD_BUSY, UINT64, AVERAGE), 1718 X("gallium-thread-busy", GALLIUM_THREAD_BUSY, UINT64, AVERAGE), 1719 X("requested-VRAM", REQUESTED_VRAM, BYTES, AVERAGE), 1720 X("requested-GTT", REQUESTED_GTT, BYTES, AVERAGE), 1721 X("mapped-VRAM", MAPPED_VRAM, BYTES, AVERAGE), 1722 X("mapped-GTT", MAPPED_GTT, BYTES, AVERAGE), 1723 X("buffer-wait-time", BUFFER_WAIT_TIME, MICROSECONDS, CUMULATIVE), 1724 X("num-mapped-buffers", NUM_MAPPED_BUFFERS, UINT64, AVERAGE), 1725 X("num-GFX-IBs", NUM_GFX_IBS, UINT64, AVERAGE), 1726 X("num-SDMA-IBs", NUM_SDMA_IBS, UINT64, AVERAGE), 1727 X("GFX-BO-list-size", GFX_BO_LIST_SIZE, UINT64, AVERAGE), 1728 X("GFX-IB-size", GFX_IB_SIZE, UINT64, AVERAGE), 1729 X("num-bytes-moved", NUM_BYTES_MOVED, BYTES, CUMULATIVE), 1730 X("num-evictions", NUM_EVICTIONS, UINT64, CUMULATIVE), 1731 X("VRAM-CPU-page-faults", NUM_VRAM_CPU_PAGE_FAULTS, UINT64, CUMULATIVE), 1732 X("VRAM-usage", VRAM_USAGE, BYTES, AVERAGE), 1733 X("VRAM-vis-usage", VRAM_VIS_USAGE, BYTES, AVERAGE), 1734 X("GTT-usage", GTT_USAGE, BYTES, AVERAGE), 1735 X("back-buffer-ps-draw-ratio", BACK_BUFFER_PS_DRAW_RATIO, UINT64, AVERAGE), 1736 1737 /* GPIN queries are for the benefit of old versions of GPUPerfStudio, 1738 * which use it as a fallback path to detect the GPU type. 1739 * 1740 * Note: The names of these queries are significant for GPUPerfStudio 1741 * (and possibly their order as well). */ 1742 XG(GPIN, "GPIN_000", GPIN_ASIC_ID, UINT, AVERAGE), 1743 XG(GPIN, "GPIN_001", GPIN_NUM_SIMD, UINT, AVERAGE), 1744 XG(GPIN, "GPIN_002", GPIN_NUM_RB, UINT, AVERAGE), 1745 XG(GPIN, "GPIN_003", GPIN_NUM_SPI, UINT, AVERAGE), 1746 XG(GPIN, "GPIN_004", GPIN_NUM_SE, UINT, AVERAGE), 1747 1748 X("temperature", GPU_TEMPERATURE, UINT64, AVERAGE), 1749 X("shader-clock", CURRENT_GPU_SCLK, HZ, AVERAGE), 1750 X("memory-clock", CURRENT_GPU_MCLK, HZ, AVERAGE), 1751 1752 /* The following queries must be at the end of the list because their 1753 * availability is adjusted dynamically based on the DRM version. */ 1754 X("GPU-load", GPU_LOAD, UINT64, AVERAGE), 1755 X("GPU-shaders-busy", GPU_SHADERS_BUSY, UINT64, AVERAGE), 1756 X("GPU-ta-busy", GPU_TA_BUSY, UINT64, AVERAGE), 1757 X("GPU-gds-busy", GPU_GDS_BUSY, UINT64, AVERAGE), 1758 X("GPU-vgt-busy", GPU_VGT_BUSY, UINT64, AVERAGE), 1759 X("GPU-ia-busy", GPU_IA_BUSY, UINT64, AVERAGE), 1760 X("GPU-sx-busy", GPU_SX_BUSY, UINT64, AVERAGE), 1761 X("GPU-wd-busy", GPU_WD_BUSY, UINT64, AVERAGE), 1762 X("GPU-bci-busy", GPU_BCI_BUSY, UINT64, AVERAGE), 1763 X("GPU-sc-busy", GPU_SC_BUSY, UINT64, AVERAGE), 1764 X("GPU-pa-busy", GPU_PA_BUSY, UINT64, AVERAGE), 1765 X("GPU-db-busy", GPU_DB_BUSY, UINT64, AVERAGE), 1766 X("GPU-cp-busy", GPU_CP_BUSY, UINT64, AVERAGE), 1767 X("GPU-cb-busy", GPU_CB_BUSY, UINT64, AVERAGE), 1768 1769 /* SRBM_STATUS2 */ 1770 X("GPU-sdma-busy", GPU_SDMA_BUSY, UINT64, AVERAGE), 1771 1772 /* CP_STAT */ 1773 X("GPU-pfp-busy", GPU_PFP_BUSY, UINT64, AVERAGE), 1774 X("GPU-meq-busy", GPU_MEQ_BUSY, UINT64, AVERAGE), 1775 X("GPU-me-busy", GPU_ME_BUSY, UINT64, AVERAGE), 1776 X("GPU-surf-sync-busy", GPU_SURF_SYNC_BUSY, UINT64, AVERAGE), 1777 X("GPU-cp-dma-busy", GPU_CP_DMA_BUSY, UINT64, AVERAGE), 1778 X("GPU-scratch-ram-busy", GPU_SCRATCH_RAM_BUSY, UINT64, AVERAGE), 1779}; 1780 1781#undef X 1782#undef XG 1783#undef XFULL 1784 1785static unsigned si_get_num_queries(struct si_screen *sscreen) 1786{ 1787 /* amdgpu */ 1788 if (sscreen->info.drm_major == 3) { 1789 if (sscreen->info.chip_class >= VI) 1790 return ARRAY_SIZE(si_driver_query_list); 1791 else 1792 return ARRAY_SIZE(si_driver_query_list) - 7; 1793 } 1794 1795 /* radeon */ 1796 if (sscreen->info.has_read_registers_query) { 1797 if (sscreen->info.chip_class == CIK) 1798 return ARRAY_SIZE(si_driver_query_list) - 6; 1799 else 1800 return ARRAY_SIZE(si_driver_query_list) - 7; 1801 } 1802 1803 return ARRAY_SIZE(si_driver_query_list) - 21; 1804} 1805 1806static int si_get_driver_query_info(struct pipe_screen *screen, 1807 unsigned index, 1808 struct pipe_driver_query_info *info) 1809{ 1810 struct si_screen *sscreen = (struct si_screen*)screen; 1811 unsigned num_queries = si_get_num_queries(sscreen); 1812 1813 if (!info) { 1814 unsigned num_perfcounters = 1815 si_get_perfcounter_info(sscreen, 0, NULL); 1816 1817 return num_queries + num_perfcounters; 1818 } 1819 1820 if (index >= num_queries) 1821 return si_get_perfcounter_info(sscreen, index - num_queries, info); 1822 1823 *info = si_driver_query_list[index]; 1824 1825 switch (info->query_type) { 1826 case SI_QUERY_REQUESTED_VRAM: 1827 case SI_QUERY_VRAM_USAGE: 1828 case SI_QUERY_MAPPED_VRAM: 1829 info->max_value.u64 = sscreen->info.vram_size; 1830 break; 1831 case SI_QUERY_REQUESTED_GTT: 1832 case SI_QUERY_GTT_USAGE: 1833 case SI_QUERY_MAPPED_GTT: 1834 info->max_value.u64 = sscreen->info.gart_size; 1835 break; 1836 case SI_QUERY_GPU_TEMPERATURE: 1837 info->max_value.u64 = 125; 1838 break; 1839 case SI_QUERY_VRAM_VIS_USAGE: 1840 info->max_value.u64 = sscreen->info.vram_vis_size; 1841 break; 1842 } 1843 1844 if (info->group_id != ~(unsigned)0 && sscreen->perfcounters) 1845 info->group_id += sscreen->perfcounters->num_groups; 1846 1847 return 1; 1848} 1849 1850/* Note: Unfortunately, GPUPerfStudio hardcodes the order of hardware 1851 * performance counter groups, so be careful when changing this and related 1852 * functions. 1853 */ 1854static int si_get_driver_query_group_info(struct pipe_screen *screen, 1855 unsigned index, 1856 struct pipe_driver_query_group_info *info) 1857{ 1858 struct si_screen *sscreen = (struct si_screen *)screen; 1859 unsigned num_pc_groups = 0; 1860 1861 if (sscreen->perfcounters) 1862 num_pc_groups = sscreen->perfcounters->num_groups; 1863 1864 if (!info) 1865 return num_pc_groups + SI_NUM_SW_QUERY_GROUPS; 1866 1867 if (index < num_pc_groups) 1868 return si_get_perfcounter_group_info(sscreen, index, info); 1869 1870 index -= num_pc_groups; 1871 if (index >= SI_NUM_SW_QUERY_GROUPS) 1872 return 0; 1873 1874 info->name = "GPIN"; 1875 info->max_active_queries = 5; 1876 info->num_queries = 5; 1877 return 1; 1878} 1879 1880void si_init_query_functions(struct si_context *sctx) 1881{ 1882 sctx->b.create_query = si_create_query; 1883 sctx->b.create_batch_query = si_create_batch_query; 1884 sctx->b.destroy_query = si_destroy_query; 1885 sctx->b.begin_query = si_begin_query; 1886 sctx->b.end_query = si_end_query; 1887 sctx->b.get_query_result = si_get_query_result; 1888 sctx->b.get_query_result_resource = si_get_query_result_resource; 1889 sctx->atoms.s.render_cond.emit = si_emit_query_predication; 1890 1891 if (((struct si_screen*)sctx->b.screen)->info.num_render_backends > 0) 1892 sctx->b.render_condition = si_render_condition; 1893 1894 LIST_INITHEAD(&sctx->active_queries); 1895} 1896 1897void si_init_screen_query_functions(struct si_screen *sscreen) 1898{ 1899 sscreen->b.get_driver_query_info = si_get_driver_query_info; 1900 sscreen->b.get_driver_query_group_info = si_get_driver_query_group_info; 1901} 1902