1/*
2 * Copyright 2015 Advanced Micro Devices, Inc.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24
25/* The GPU load is measured as follows.
26 *
27 * There is a thread which samples the GRBM_STATUS register at a certain
28 * frequency and the "busy" or "idle" counter is incremented based on
29 * whether the GUI_ACTIVE bit is set or not.
30 *
31 * Then, the user can sample the counters twice and calculate the average
32 * GPU load between the two samples.
33 */
34
35#include "radeonsi/si_pipe.h"
36#include "radeonsi/si_query.h"
37#include "util/os_time.h"
38
39/* For good accuracy at 1000 fps or lower. This will be inaccurate for higher
40 * fps (there are too few samples per frame). */
41#define SAMPLES_PER_SEC 10000
42
43#define GRBM_STATUS		0x8010
44#define TA_BUSY(x)		(((x) >> 14) & 0x1)
45#define GDS_BUSY(x)		(((x) >> 15) & 0x1)
46#define VGT_BUSY(x)		(((x) >> 17) & 0x1)
47#define IA_BUSY(x)		(((x) >> 19) & 0x1)
48#define SX_BUSY(x)		(((x) >> 20) & 0x1)
49#define WD_BUSY(x)		(((x) >> 21) & 0x1)
50#define SPI_BUSY(x)		(((x) >> 22) & 0x1)
51#define BCI_BUSY(x)		(((x) >> 23) & 0x1)
52#define SC_BUSY(x)		(((x) >> 24) & 0x1)
53#define PA_BUSY(x)		(((x) >> 25) & 0x1)
54#define DB_BUSY(x)		(((x) >> 26) & 0x1)
55#define CP_BUSY(x)		(((x) >> 29) & 0x1)
56#define CB_BUSY(x)		(((x) >> 30) & 0x1)
57#define GUI_ACTIVE(x)		(((x) >> 31) & 0x1)
58
59#define SRBM_STATUS2		0x0e4c
60#define SDMA_BUSY(x)		(((x) >> 5) & 0x1)
61
62#define CP_STAT                 0x8680
63#define PFP_BUSY(x)		(((x) >> 15) & 0x1)
64#define MEQ_BUSY(x)		(((x) >> 16) & 0x1)
65#define ME_BUSY(x)		(((x) >> 17) & 0x1)
66#define SURFACE_SYNC_BUSY(x)	(((x) >> 21) & 0x1)
67#define DMA_BUSY(x)		(((x) >> 22) & 0x1)
68#define SCRATCH_RAM_BUSY(x)	(((x) >> 24) & 0x1)
69
70#define IDENTITY(x) x
71
72#define UPDATE_COUNTER(field, mask)					\
73	do {								\
74		if (mask(value))					\
75			p_atomic_inc(&counters->named.field.busy);	\
76		else							\
77			p_atomic_inc(&counters->named.field.idle);	\
78	} while (0)
79
80static void si_update_mmio_counters(struct si_screen *sscreen,
81				    union si_mmio_counters *counters)
82{
83	uint32_t value = 0;
84	bool gui_busy, sdma_busy = false;
85
86	/* GRBM_STATUS */
87	sscreen->ws->read_registers(sscreen->ws, GRBM_STATUS, 1, &value);
88
89	UPDATE_COUNTER(ta, TA_BUSY);
90	UPDATE_COUNTER(gds, GDS_BUSY);
91	UPDATE_COUNTER(vgt, VGT_BUSY);
92	UPDATE_COUNTER(ia, IA_BUSY);
93	UPDATE_COUNTER(sx, SX_BUSY);
94	UPDATE_COUNTER(wd, WD_BUSY);
95	UPDATE_COUNTER(spi, SPI_BUSY);
96	UPDATE_COUNTER(bci, BCI_BUSY);
97	UPDATE_COUNTER(sc, SC_BUSY);
98	UPDATE_COUNTER(pa, PA_BUSY);
99	UPDATE_COUNTER(db, DB_BUSY);
100	UPDATE_COUNTER(cp, CP_BUSY);
101	UPDATE_COUNTER(cb, CB_BUSY);
102	UPDATE_COUNTER(gui, GUI_ACTIVE);
103	gui_busy = GUI_ACTIVE(value);
104
105	if (sscreen->info.chip_class == CIK || sscreen->info.chip_class == VI) {
106		/* SRBM_STATUS2 */
107		sscreen->ws->read_registers(sscreen->ws, SRBM_STATUS2, 1, &value);
108
109		UPDATE_COUNTER(sdma, SDMA_BUSY);
110		sdma_busy = SDMA_BUSY(value);
111	}
112
113	if (sscreen->info.chip_class >= VI) {
114		/* CP_STAT */
115		sscreen->ws->read_registers(sscreen->ws, CP_STAT, 1, &value);
116
117		UPDATE_COUNTER(pfp, PFP_BUSY);
118		UPDATE_COUNTER(meq, MEQ_BUSY);
119		UPDATE_COUNTER(me, ME_BUSY);
120		UPDATE_COUNTER(surf_sync, SURFACE_SYNC_BUSY);
121		UPDATE_COUNTER(cp_dma, DMA_BUSY);
122		UPDATE_COUNTER(scratch_ram, SCRATCH_RAM_BUSY);
123	}
124
125	value = gui_busy || sdma_busy;
126	UPDATE_COUNTER(gpu, IDENTITY);
127}
128
129#undef UPDATE_COUNTER
130
131static int
132si_gpu_load_thread(void *param)
133{
134	struct si_screen *sscreen = (struct si_screen*)param;
135	const int period_us = 1000000 / SAMPLES_PER_SEC;
136	int sleep_us = period_us;
137	int64_t cur_time, last_time = os_time_get();
138
139	while (!p_atomic_read(&sscreen->gpu_load_stop_thread)) {
140		if (sleep_us)
141			os_time_sleep(sleep_us);
142
143		/* Make sure we sleep the ideal amount of time to match
144		 * the expected frequency. */
145		cur_time = os_time_get();
146
147		if (os_time_timeout(last_time, last_time + period_us,
148				    cur_time))
149			sleep_us = MAX2(sleep_us - 1, 1);
150		else
151			sleep_us += 1;
152
153		/*printf("Hz: %.1f\n", 1000000.0 / (cur_time - last_time));*/
154		last_time = cur_time;
155
156		/* Update the counters. */
157		si_update_mmio_counters(sscreen, &sscreen->mmio_counters);
158	}
159	p_atomic_dec(&sscreen->gpu_load_stop_thread);
160	return 0;
161}
162
163void si_gpu_load_kill_thread(struct si_screen *sscreen)
164{
165	if (!sscreen->gpu_load_thread)
166		return;
167
168	p_atomic_inc(&sscreen->gpu_load_stop_thread);
169	thrd_join(sscreen->gpu_load_thread, NULL);
170	sscreen->gpu_load_thread = 0;
171}
172
173static uint64_t si_read_mmio_counter(struct si_screen *sscreen,
174				     unsigned busy_index)
175{
176	/* Start the thread if needed. */
177	if (!sscreen->gpu_load_thread) {
178		mtx_lock(&sscreen->gpu_load_mutex);
179		/* Check again inside the mutex. */
180		if (!sscreen->gpu_load_thread)
181			sscreen->gpu_load_thread =
182				u_thread_create(si_gpu_load_thread, sscreen);
183		mtx_unlock(&sscreen->gpu_load_mutex);
184	}
185
186	unsigned busy = p_atomic_read(&sscreen->mmio_counters.array[busy_index]);
187	unsigned idle = p_atomic_read(&sscreen->mmio_counters.array[busy_index + 1]);
188
189	return busy | ((uint64_t)idle << 32);
190}
191
192static unsigned si_end_mmio_counter(struct si_screen *sscreen,
193				    uint64_t begin, unsigned busy_index)
194{
195	uint64_t end = si_read_mmio_counter(sscreen, busy_index);
196	unsigned busy = (end & 0xffffffff) - (begin & 0xffffffff);
197	unsigned idle = (end >> 32) - (begin >> 32);
198
199	/* Calculate the % of time the busy counter was being incremented.
200	 *
201	 * If no counters were incremented, return the current counter status.
202	 * It's for the case when the load is queried faster than
203	 * the counters are updated.
204	 */
205	if (idle || busy) {
206		return busy*100 / (busy + idle);
207	} else {
208		union si_mmio_counters counters;
209
210		memset(&counters, 0, sizeof(counters));
211		si_update_mmio_counters(sscreen, &counters);
212		return counters.array[busy_index] ? 100 : 0;
213	}
214}
215
216#define BUSY_INDEX(sscreen, field) (&sscreen->mmio_counters.named.field.busy - \
217				    sscreen->mmio_counters.array)
218
219static unsigned busy_index_from_type(struct si_screen *sscreen,
220				     unsigned type)
221{
222	switch (type) {
223	case SI_QUERY_GPU_LOAD:
224		return BUSY_INDEX(sscreen, gpu);
225	case SI_QUERY_GPU_SHADERS_BUSY:
226		return BUSY_INDEX(sscreen, spi);
227	case SI_QUERY_GPU_TA_BUSY:
228		return BUSY_INDEX(sscreen, ta);
229	case SI_QUERY_GPU_GDS_BUSY:
230		return BUSY_INDEX(sscreen, gds);
231	case SI_QUERY_GPU_VGT_BUSY:
232		return BUSY_INDEX(sscreen, vgt);
233	case SI_QUERY_GPU_IA_BUSY:
234		return BUSY_INDEX(sscreen, ia);
235	case SI_QUERY_GPU_SX_BUSY:
236		return BUSY_INDEX(sscreen, sx);
237	case SI_QUERY_GPU_WD_BUSY:
238		return BUSY_INDEX(sscreen, wd);
239	case SI_QUERY_GPU_BCI_BUSY:
240		return BUSY_INDEX(sscreen, bci);
241	case SI_QUERY_GPU_SC_BUSY:
242		return BUSY_INDEX(sscreen, sc);
243	case SI_QUERY_GPU_PA_BUSY:
244		return BUSY_INDEX(sscreen, pa);
245	case SI_QUERY_GPU_DB_BUSY:
246		return BUSY_INDEX(sscreen, db);
247	case SI_QUERY_GPU_CP_BUSY:
248		return BUSY_INDEX(sscreen, cp);
249	case SI_QUERY_GPU_CB_BUSY:
250		return BUSY_INDEX(sscreen, cb);
251	case SI_QUERY_GPU_SDMA_BUSY:
252		return BUSY_INDEX(sscreen, sdma);
253	case SI_QUERY_GPU_PFP_BUSY:
254		return BUSY_INDEX(sscreen, pfp);
255	case SI_QUERY_GPU_MEQ_BUSY:
256		return BUSY_INDEX(sscreen, meq);
257	case SI_QUERY_GPU_ME_BUSY:
258		return BUSY_INDEX(sscreen, me);
259	case SI_QUERY_GPU_SURF_SYNC_BUSY:
260		return BUSY_INDEX(sscreen, surf_sync);
261	case SI_QUERY_GPU_CP_DMA_BUSY:
262		return BUSY_INDEX(sscreen, cp_dma);
263	case SI_QUERY_GPU_SCRATCH_RAM_BUSY:
264		return BUSY_INDEX(sscreen, scratch_ram);
265	default:
266		unreachable("invalid query type");
267	}
268}
269
270uint64_t si_begin_counter(struct si_screen *sscreen, unsigned type)
271{
272	unsigned busy_index = busy_index_from_type(sscreen, type);
273	return si_read_mmio_counter(sscreen, busy_index);
274}
275
276unsigned si_end_counter(struct si_screen *sscreen, unsigned type,
277			uint64_t begin)
278{
279	unsigned busy_index = busy_index_from_type(sscreen, type);
280	return si_end_mmio_counter(sscreen, begin, busy_index);
281}
282