si_shader.h revision 9f464c52 
1/*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * on the rights to use, copy, modify, merge, publish, distribute, sub
9 * license, and/or sell copies of the Software, and to permit persons to whom
10 * the Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
20 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
21 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
22 * USE OR OTHER DEALINGS IN THE SOFTWARE.
23 */
24
25/* The compiler middle-end architecture: Explaining (non-)monolithic shaders
26 * -------------------------------------------------------------------------
27 *
28 * Typically, there is one-to-one correspondence between API and HW shaders,
29 * that is, for every API shader, there is exactly one shader binary in
30 * the driver.
31 *
32 * The problem with that is that we also have to emulate some API states
33 * (e.g. alpha-test, and many others) in shaders too. The two obvious ways
34 * to deal with it are:
35 * - each shader has multiple variants for each combination of emulated states,
36 *   and the variants are compiled on demand, possibly relying on a shader
37 *   cache for good performance
38 * - patch shaders at the binary level
39 *
40 * This driver uses something completely different. The emulated states are
41 * usually implemented at the beginning or end of shaders. Therefore, we can
42 * split the shader into 3 parts:
43 * - prolog part (shader code dependent on states)
44 * - main part (the API shader)
45 * - epilog part (shader code dependent on states)
46 *
47 * Each part is compiled as a separate shader and the final binaries are
48 * concatenated. This type of shader is called non-monolithic, because it
49 * consists of multiple independent binaries. Creating a new shader variant
50 * is therefore only a concatenation of shader parts (binaries) and doesn't
51 * involve any compilation. The main shader parts are the only parts that are
52 * compiled when applications create shader objects. The prolog and epilog
53 * parts are compiled on the first use and saved, so that their binaries can
54 * be reused by many other shaders.
55 *
56 * One of the roles of the prolog part is to compute vertex buffer addresses
57 * for vertex shaders. A few of the roles of the epilog part are color buffer
58 * format conversions in pixel shaders that we have to do manually, and write
59 * tessellation factors in tessellation control shaders. The prolog and epilog
60 * have many other important responsibilities in various shader stages.
61 * They don't just "emulate legacy stuff".
62 *
63 * Monolithic shaders are shaders where the parts are combined before LLVM
64 * compilation, and the whole thing is compiled and optimized as one unit with
65 * one binary on the output. The result is the same as the non-monolithic
66 * shader, but the final code can be better, because LLVM can optimize across
67 * all shader parts. Monolithic shaders aren't usually used except for these
68 * special cases:
69 *
70 * 1) Some rarely-used states require modification of the main shader part
71 *    itself, and in such cases, only the monolithic shader variant is
72 *    compiled, and that's always done on the first use.
73 *
74 * 2) When we do cross-stage optimizations for separate shader objects and
75 *    e.g. eliminate unused shader varyings, the resulting optimized shader
76 *    variants are always compiled as monolithic shaders, and always
77 *    asynchronously (i.e. not stalling ongoing rendering). We call them
78 *    "optimized monolithic" shaders. The important property here is that
79 *    the non-monolithic unoptimized shader variant is always available for use
80 *    when the asynchronous compilation of the optimized shader is not done
81 *    yet.
82 *
83 * Starting with GFX9 chips, some shader stages are merged, and the number of
84 * shader parts per shader increased. The complete new list of shader parts is:
85 * - 1st shader: prolog part
86 * - 1st shader: main part
87 * - 2nd shader: prolog part
88 * - 2nd shader: main part
89 * - 2nd shader: epilog part
90 */
91
92/* How linking shader inputs and outputs between vertex, tessellation, and
93 * geometry shaders works.
94 *
95 * Inputs and outputs between shaders are stored in a buffer. This buffer
96 * lives in LDS (typical case for tessellation), but it can also live
97 * in memory (ESGS). Each input or output has a fixed location within a vertex.
98 * The highest used input or output determines the stride between vertices.
99 *
100 * Since GS and tessellation are only possible in the OpenGL core profile,
101 * only these semantics are valid for per-vertex data:
102 *
103 *   Name             Location
104 *
105 *   POSITION         0
106 *   PSIZE            1
107 *   CLIPDIST0..1     2..3
108 *   CULLDIST0..1     (not implemented)
109 *   GENERIC0..31     4..35
110 *
111 * For example, a shader only writing GENERIC0 has the output stride of 5.
112 *
113 * Only these semantics are valid for per-patch data:
114 *
115 *   Name             Location
116 *
117 *   TESSOUTER        0
118 *   TESSINNER        1
119 *   PATCH0..29       2..31
120 *
121 * That's how independent shaders agree on input and output locations.
122 * The si_shader_io_get_unique_index function assigns the locations.
123 *
124 * For tessellation, other required information for calculating the input and
125 * output addresses like the vertex stride, the patch stride, and the offsets
126 * where per-vertex and per-patch data start, is passed to the shader via
127 * user data SGPRs. The offsets and strides are calculated at draw time and
128 * aren't available at compile time.
129 */
130
131#ifndef SI_SHADER_H
132#define SI_SHADER_H
133
134#include <llvm-c/Core.h> /* LLVMModuleRef */
135#include <llvm-c/TargetMachine.h>
136#include "tgsi/tgsi_scan.h"
137#include "util/u_inlines.h"
138#include "util/u_queue.h"
139
140#include "ac_binary.h"
141#include "ac_llvm_build.h"
142#include "ac_llvm_util.h"
143
144#include <stdio.h>
145
146struct nir_shader;
147struct si_shader;
148struct si_context;
149
150#define SI_MAX_ATTRIBS		16
151#define SI_MAX_VS_OUTPUTS	40
152
153/* Shader IO unique indices are supported for TGSI_SEMANTIC_GENERIC with an
154 * index smaller than this.
155 */
156#define SI_MAX_IO_GENERIC       43
157
158/* SGPR user data indices */
159enum {
160	SI_SGPR_RW_BUFFERS,  /* rings (& stream-out, VS only) */
161	SI_SGPR_BINDLESS_SAMPLERS_AND_IMAGES,
162	SI_SGPR_CONST_AND_SHADER_BUFFERS, /* or just a constant buffer 0 pointer */
163	SI_SGPR_SAMPLERS_AND_IMAGES,
164	SI_NUM_RESOURCE_SGPRS,
165
166	/* API VS, TES without GS, GS copy shader */
167	SI_SGPR_VS_STATE_BITS = SI_NUM_RESOURCE_SGPRS,
168	SI_NUM_VS_STATE_RESOURCE_SGPRS,
169
170	/* all VS variants */
171	SI_SGPR_BASE_VERTEX = SI_NUM_VS_STATE_RESOURCE_SGPRS,
172	SI_SGPR_START_INSTANCE,
173	SI_SGPR_DRAWID,
174	SI_VS_NUM_USER_SGPR,
175
176	SI_SGPR_VS_BLIT_DATA = SI_SGPR_CONST_AND_SHADER_BUFFERS,
177
178	/* TES */
179	SI_SGPR_TES_OFFCHIP_LAYOUT = SI_NUM_VS_STATE_RESOURCE_SGPRS,
180	SI_SGPR_TES_OFFCHIP_ADDR,
181	SI_TES_NUM_USER_SGPR,
182
183	/* GFX6-8: TCS only */
184	GFX6_SGPR_TCS_OFFCHIP_LAYOUT = SI_NUM_RESOURCE_SGPRS,
185	GFX6_SGPR_TCS_OUT_OFFSETS,
186	GFX6_SGPR_TCS_OUT_LAYOUT,
187	GFX6_SGPR_TCS_IN_LAYOUT,
188	GFX6_TCS_NUM_USER_SGPR,
189
190	/* GFX9: Merged shaders. */
191	/* 2ND_CONST_AND_SHADER_BUFFERS is set in USER_DATA_ADDR_LO (SGPR0). */
192	/* 2ND_SAMPLERS_AND_IMAGES is set in USER_DATA_ADDR_HI (SGPR1). */
193	GFX9_MERGED_NUM_USER_SGPR = SI_VS_NUM_USER_SGPR,
194
195	/* GFX9: Merged LS-HS (VS-TCS) only. */
196	GFX9_SGPR_TCS_OFFCHIP_LAYOUT = GFX9_MERGED_NUM_USER_SGPR,
197	GFX9_SGPR_TCS_OUT_OFFSETS,
198	GFX9_SGPR_TCS_OUT_LAYOUT,
199	GFX9_TCS_NUM_USER_SGPR,
200
201	/* GS limits */
202	GFX6_GS_NUM_USER_SGPR = SI_NUM_RESOURCE_SGPRS,
203	GFX9_VSGS_NUM_USER_SGPR = SI_VS_NUM_USER_SGPR,
204	GFX9_TESGS_NUM_USER_SGPR = SI_TES_NUM_USER_SGPR,
205	SI_GSCOPY_NUM_USER_SGPR = SI_NUM_VS_STATE_RESOURCE_SGPRS,
206
207	/* PS only */
208	SI_SGPR_ALPHA_REF	= SI_NUM_RESOURCE_SGPRS,
209	SI_PS_NUM_USER_SGPR,
210};
211
212/* LLVM function parameter indices */
213enum {
214	SI_NUM_RESOURCE_PARAMS = 4,
215
216	/* PS only parameters */
217	SI_PARAM_ALPHA_REF = SI_NUM_RESOURCE_PARAMS,
218	SI_PARAM_PRIM_MASK,
219	SI_PARAM_PERSP_SAMPLE,
220	SI_PARAM_PERSP_CENTER,
221	SI_PARAM_PERSP_CENTROID,
222	SI_PARAM_PERSP_PULL_MODEL,
223	SI_PARAM_LINEAR_SAMPLE,
224	SI_PARAM_LINEAR_CENTER,
225	SI_PARAM_LINEAR_CENTROID,
226	SI_PARAM_LINE_STIPPLE_TEX,
227	SI_PARAM_POS_X_FLOAT,
228	SI_PARAM_POS_Y_FLOAT,
229	SI_PARAM_POS_Z_FLOAT,
230	SI_PARAM_POS_W_FLOAT,
231	SI_PARAM_FRONT_FACE,
232	SI_PARAM_ANCILLARY,
233	SI_PARAM_SAMPLE_COVERAGE,
234	SI_PARAM_POS_FIXED_PT,
235
236	SI_NUM_PARAMS = SI_PARAM_POS_FIXED_PT + 9, /* +8 for COLOR[0..1] */
237};
238
239/* Fields of driver-defined VS state SGPR. */
240/* Clamp vertex color output (only used in VS as VS). */
241#define S_VS_STATE_CLAMP_VERTEX_COLOR(x)	(((unsigned)(x) & 0x1) << 0)
242#define C_VS_STATE_CLAMP_VERTEX_COLOR		0xFFFFFFFE
243#define S_VS_STATE_INDEXED(x)			(((unsigned)(x) & 0x1) << 1)
244#define C_VS_STATE_INDEXED			0xFFFFFFFD
245#define S_VS_STATE_LS_OUT_PATCH_SIZE(x)		(((unsigned)(x) & 0x1FFF) << 8)
246#define C_VS_STATE_LS_OUT_PATCH_SIZE		0xFFE000FF
247#define S_VS_STATE_LS_OUT_VERTEX_SIZE(x)	(((unsigned)(x) & 0xFF) << 24)
248#define C_VS_STATE_LS_OUT_VERTEX_SIZE		0x00FFFFFF
249
250/* SI-specific system values. */
251enum {
252	/* Values from set_tess_state. */
253	TGSI_SEMANTIC_DEFAULT_TESSOUTER_SI = TGSI_SEMANTIC_COUNT,
254	TGSI_SEMANTIC_DEFAULT_TESSINNER_SI,
255
256	/* Up to 4 dwords in user SGPRs for compute shaders. */
257	TGSI_SEMANTIC_CS_USER_DATA,
258};
259
260enum {
261	/* Use a property enum that CS wouldn't use. */
262	TGSI_PROPERTY_CS_LOCAL_SIZE = TGSI_PROPERTY_FS_COORD_ORIGIN,
263
264	/* The number of used user data dwords in the range [1, 4]. */
265	TGSI_PROPERTY_CS_USER_DATA_DWORDS = TGSI_PROPERTY_FS_COORD_PIXEL_CENTER,
266
267	/* Use a property enum that VS wouldn't use. */
268	TGSI_PROPERTY_VS_BLIT_SGPRS = TGSI_PROPERTY_FS_COORD_ORIGIN,
269
270	/* These represent the number of SGPRs the shader uses. */
271	SI_VS_BLIT_SGPRS_POS = 3,
272	SI_VS_BLIT_SGPRS_POS_COLOR = 7,
273	SI_VS_BLIT_SGPRS_POS_TEXCOORD = 9,
274};
275
276/* For VS shader key fix_fetch. */
277enum {
278	SI_FIX_FETCH_NONE = 0,
279	SI_FIX_FETCH_A2_SNORM,
280	SI_FIX_FETCH_A2_SSCALED,
281	SI_FIX_FETCH_A2_SINT,
282	SI_FIX_FETCH_RGBA_32_UNORM,
283	SI_FIX_FETCH_RGBX_32_UNORM,
284	SI_FIX_FETCH_RGBA_32_SNORM,
285	SI_FIX_FETCH_RGBX_32_SNORM,
286	SI_FIX_FETCH_RGBA_32_USCALED,
287	SI_FIX_FETCH_RGBA_32_SSCALED,
288	SI_FIX_FETCH_RGBA_32_FIXED,
289	SI_FIX_FETCH_RGBX_32_FIXED,
290	SI_FIX_FETCH_RG_64_FLOAT,
291	SI_FIX_FETCH_RGB_64_FLOAT,
292	SI_FIX_FETCH_RGBA_64_FLOAT,
293	SI_FIX_FETCH_RGB_8,	/* A = 1.0 */
294	SI_FIX_FETCH_RGB_8_INT,	/* A = 1 */
295	SI_FIX_FETCH_RGB_16,
296	SI_FIX_FETCH_RGB_16_INT,
297};
298
299struct si_shader;
300
301/* State of the context creating the shader object. */
302struct si_compiler_ctx_state {
303	/* Should only be used by si_init_shader_selector_async and
304	 * si_build_shader_variant if thread_index == -1 (non-threaded). */
305	struct ac_llvm_compiler		*compiler;
306
307	/* Used if thread_index == -1 or if debug.async is true. */
308	struct pipe_debug_callback	debug;
309
310	/* Used for creating the log string for gallium/ddebug. */
311	bool				is_debug_context;
312};
313
314/* A shader selector is a gallium CSO and contains shader variants and
315 * binaries for one TGSI program. This can be shared by multiple contexts.
316 */
317struct si_shader_selector {
318	struct pipe_reference	reference;
319	struct si_screen	*screen;
320	struct util_queue_fence ready;
321	struct si_compiler_ctx_state compiler_ctx_state;
322
323	mtx_t		mutex;
324	struct si_shader	*first_variant; /* immutable after the first variant */
325	struct si_shader	*last_variant; /* mutable */
326
327	/* The compiled TGSI shader expecting a prolog and/or epilog (not
328	 * uploaded to a buffer).
329	 */
330	struct si_shader	*main_shader_part;
331	struct si_shader	*main_shader_part_ls; /* as_ls is set in the key */
332	struct si_shader	*main_shader_part_es; /* as_es is set in the key */
333
334	struct si_shader	*gs_copy_shader;
335
336	struct tgsi_token       *tokens;
337	struct nir_shader       *nir;
338	struct pipe_stream_output_info  so;
339	struct tgsi_shader_info		info;
340	struct tgsi_tessctrl_info	tcs_info;
341
342	/* PIPE_SHADER_[VERTEX|FRAGMENT|...] */
343	unsigned	type;
344	bool		vs_needs_prolog;
345	bool		force_correct_derivs_after_kill;
346	unsigned	pa_cl_vs_out_cntl;
347	ubyte		clipdist_mask;
348	ubyte		culldist_mask;
349
350	/* ES parameters. */
351	unsigned	esgs_itemsize; /* vertex stride */
352	unsigned	lshs_vertex_stride;
353
354	/* GS parameters. */
355	unsigned	gs_input_verts_per_prim;
356	unsigned	gs_output_prim;
357	unsigned	gs_max_out_vertices;
358	unsigned	gs_num_invocations;
359	unsigned	max_gs_stream; /* count - 1 */
360	unsigned	gsvs_vertex_size;
361	unsigned	max_gsvs_emit_size;
362	unsigned	enabled_streamout_buffer_mask;
363
364	/* PS parameters. */
365	unsigned	color_attr_index[2];
366	unsigned	db_shader_control;
367	/* Set 0xf or 0x0 (4 bits) per each written output.
368	 * ANDed with spi_shader_col_format.
369	 */
370	unsigned	colors_written_4bit;
371
372	uint64_t	outputs_written_before_ps; /* "get_unique_index" bits */
373	uint64_t	outputs_written;	/* "get_unique_index" bits */
374	uint32_t	patch_outputs_written;	/* "get_unique_index_patch" bits */
375
376	uint64_t	inputs_read;		/* "get_unique_index" bits */
377
378	/* bitmasks of used descriptor slots */
379	uint32_t	active_const_and_shader_buffers;
380	uint64_t	active_samplers_and_images;
381};
382
383/* Valid shader configurations:
384 *
385 * API shaders       VS | TCS | TES | GS |pass| PS
386 * are compiled as:     |     |     |    |thru|
387 *                      |     |     |    |    |
388 * Only VS & PS:     VS |     |     |    |    | PS
389 * GFX6 - with GS:   ES |     |     | GS | VS | PS
390 *      - with tess: LS | HS  | VS  |    |    | PS
391 *      - with both: LS | HS  | ES  | GS | VS | PS
392 * GFX9 - with GS:   -> |     |     | GS | VS | PS
393 *      - with tess: -> | HS  | VS  |    |    | PS
394 *      - with both: -> | HS  | ->  | GS | VS | PS
395 *
396 * -> = merged with the next stage
397 */
398
399/* Use the byte alignment for all following structure members for optimal
400 * shader key memory footprint.
401 */
402#pragma pack(push, 1)
403
404/* Common VS bits between the shader key and the prolog key. */
405struct si_vs_prolog_bits {
406	/* - If neither "is_one" nor "is_fetched" has a bit set, the instance
407	 *   divisor is 0.
408	 * - If "is_one" has a bit set, the instance divisor is 1.
409	 * - If "is_fetched" has a bit set, the instance divisor will be loaded
410	 *   from the constant buffer.
411	 */
412	uint16_t	instance_divisor_is_one;     /* bitmask of inputs */
413	uint16_t	instance_divisor_is_fetched; /* bitmask of inputs */
414	unsigned	ls_vgpr_fix:1;
415};
416
417/* Common TCS bits between the shader key and the epilog key. */
418struct si_tcs_epilog_bits {
419	unsigned	prim_mode:3;
420	unsigned	invoc0_tess_factors_are_def:1;
421	unsigned	tes_reads_tess_factors:1;
422};
423
424struct si_gs_prolog_bits {
425	unsigned	tri_strip_adj_fix:1;
426	unsigned	gfx9_prev_is_vs:1;
427};
428
429/* Common PS bits between the shader key and the prolog key. */
430struct si_ps_prolog_bits {
431	unsigned	color_two_side:1;
432	unsigned	flatshade_colors:1;
433	unsigned	poly_stipple:1;
434	unsigned	force_persp_sample_interp:1;
435	unsigned	force_linear_sample_interp:1;
436	unsigned	force_persp_center_interp:1;
437	unsigned	force_linear_center_interp:1;
438	unsigned	bc_optimize_for_persp:1;
439	unsigned	bc_optimize_for_linear:1;
440	unsigned	samplemask_log_ps_iter:3;
441};
442
443/* Common PS bits between the shader key and the epilog key. */
444struct si_ps_epilog_bits {
445	unsigned	spi_shader_col_format;
446	unsigned	color_is_int8:8;
447	unsigned	color_is_int10:8;
448	unsigned	last_cbuf:3;
449	unsigned	alpha_func:3;
450	unsigned	alpha_to_one:1;
451	unsigned	poly_line_smoothing:1;
452	unsigned	clamp_color:1;
453};
454
455union si_shader_part_key {
456	struct {
457		struct si_vs_prolog_bits states;
458		unsigned	num_input_sgprs:6;
459		/* For merged stages such as LS-HS, HS input VGPRs are first. */
460		unsigned	num_merged_next_stage_vgprs:3;
461		unsigned	last_input:4;
462		unsigned	as_ls:1;
463		unsigned	as_es:1;
464		/* Prologs for monolithic shaders shouldn't set EXEC. */
465		unsigned	is_monolithic:1;
466	} vs_prolog;
467	struct {
468		struct si_tcs_epilog_bits states;
469	} tcs_epilog;
470	struct {
471		struct si_gs_prolog_bits states;
472		/* Prologs of monolithic shaders shouldn't set EXEC. */
473		unsigned	is_monolithic:1;
474	} gs_prolog;
475	struct {
476		struct si_ps_prolog_bits states;
477		unsigned	num_input_sgprs:6;
478		unsigned	num_input_vgprs:5;
479		/* Color interpolation and two-side color selection. */
480		unsigned	colors_read:8; /* color input components read */
481		unsigned	num_interp_inputs:5; /* BCOLOR is at this location */
482		unsigned	face_vgpr_index:5;
483		unsigned	ancillary_vgpr_index:5;
484		unsigned	wqm:1;
485		char		color_attr_index[2];
486		signed char	color_interp_vgpr_index[2]; /* -1 == constant */
487	} ps_prolog;
488	struct {
489		struct si_ps_epilog_bits states;
490		unsigned	colors_written:8;
491		unsigned	writes_z:1;
492		unsigned	writes_stencil:1;
493		unsigned	writes_samplemask:1;
494	} ps_epilog;
495};
496
497struct si_shader_key {
498	/* Prolog and epilog flags. */
499	union {
500		struct {
501			struct si_vs_prolog_bits prolog;
502		} vs;
503		struct {
504			struct si_vs_prolog_bits ls_prolog; /* for merged LS-HS */
505			struct si_shader_selector *ls;   /* for merged LS-HS */
506			struct si_tcs_epilog_bits epilog;
507		} tcs; /* tessellation control shader */
508		struct {
509			struct si_vs_prolog_bits vs_prolog; /* for merged ES-GS */
510			struct si_shader_selector *es;   /* for merged ES-GS */
511			struct si_gs_prolog_bits prolog;
512		} gs;
513		struct {
514			struct si_ps_prolog_bits prolog;
515			struct si_ps_epilog_bits epilog;
516		} ps;
517	} part;
518
519	/* These two are initially set according to the NEXT_SHADER property,
520	 * or guessed if the property doesn't seem correct.
521	 */
522	unsigned as_es:1; /* export shader, which precedes GS */
523	unsigned as_ls:1; /* local shader, which precedes TCS */
524
525	/* Flags for monolithic compilation only. */
526	struct {
527		/* One byte for every input: SI_FIX_FETCH_* enums. */
528		uint8_t		vs_fix_fetch[SI_MAX_ATTRIBS];
529
530		union {
531			uint64_t	ff_tcs_inputs_to_copy; /* for fixed-func TCS */
532			/* When PS needs PrimID and GS is disabled. */
533			unsigned	vs_export_prim_id:1;
534			struct {
535				unsigned interpolate_at_sample_force_center:1;
536				unsigned fbfetch_msaa;
537				unsigned fbfetch_is_1D;
538				unsigned fbfetch_layered;
539			} ps;
540		} u;
541	} mono;
542
543	/* Optimization flags for asynchronous compilation only. */
544	struct {
545		/* For HW VS (it can be VS, TES, GS) */
546		uint64_t	kill_outputs; /* "get_unique_index" bits */
547		unsigned	clip_disable:1;
548
549		/* For shaders where monolithic variants have better code.
550		 *
551		 * This is a flag that has no effect on code generation,
552		 * but forces monolithic shaders to be used as soon as
553		 * possible, because it's in the "opt" group.
554		 */
555		unsigned	prefer_mono:1;
556	} opt;
557};
558
559/* Restore the pack alignment to default. */
560#pragma pack(pop)
561
562struct si_shader_config {
563	unsigned			num_sgprs;
564	unsigned			num_vgprs;
565	unsigned			spilled_sgprs;
566	unsigned			spilled_vgprs;
567	unsigned			private_mem_vgprs;
568	unsigned			lds_size;
569	unsigned			max_simd_waves;
570	unsigned			spi_ps_input_ena;
571	unsigned			spi_ps_input_addr;
572	unsigned			float_mode;
573	unsigned			scratch_bytes_per_wave;
574	unsigned			rsrc1;
575	unsigned			rsrc2;
576};
577
578/* GCN-specific shader info. */
579struct si_shader_info {
580	ubyte			vs_output_param_offset[SI_MAX_VS_OUTPUTS];
581	ubyte			num_input_sgprs;
582	ubyte			num_input_vgprs;
583	signed char		face_vgpr_index;
584	signed char		ancillary_vgpr_index;
585	bool			uses_instanceid;
586	ubyte			nr_pos_exports;
587	ubyte			nr_param_exports;
588};
589
590struct si_shader {
591	struct si_compiler_ctx_state	compiler_ctx_state;
592
593	struct si_shader_selector	*selector;
594	struct si_shader_selector	*previous_stage_sel; /* for refcounting */
595	struct si_shader		*next_variant;
596
597	struct si_shader_part		*prolog;
598	struct si_shader		*previous_stage; /* for GFX9 */
599	struct si_shader_part		*prolog2;
600	struct si_shader_part		*epilog;
601
602	struct si_pm4_state		*pm4;
603	struct si_resource		*bo;
604	struct si_resource		*scratch_bo;
605	struct si_shader_key		key;
606	struct util_queue_fence		ready;
607	bool				compilation_failed;
608	bool				is_monolithic;
609	bool				is_optimized;
610	bool				is_binary_shared;
611	bool				is_gs_copy_shader;
612
613	/* The following data is all that's needed for binary shaders. */
614	struct ac_shader_binary	binary;
615	struct si_shader_config		config;
616	struct si_shader_info		info;
617
618	/* Shader key + LLVM IR + disassembly + statistics.
619	 * Generated for debug contexts only.
620	 */
621	char				*shader_log;
622	size_t				shader_log_size;
623
624	/* For save precompute context registers values. */
625	union {
626		struct {
627			unsigned	vgt_gsvs_ring_offset_1;
628			unsigned	vgt_gsvs_ring_offset_2;
629			unsigned	vgt_gsvs_ring_offset_3;
630			unsigned	vgt_gs_out_prim_type;
631			unsigned	vgt_gsvs_ring_itemsize;
632			unsigned	vgt_gs_max_vert_out;
633			unsigned	vgt_gs_vert_itemsize;
634			unsigned	vgt_gs_vert_itemsize_1;
635			unsigned	vgt_gs_vert_itemsize_2;
636			unsigned	vgt_gs_vert_itemsize_3;
637			unsigned	vgt_gs_instance_cnt;
638			unsigned	vgt_gs_onchip_cntl;
639			unsigned	vgt_gs_max_prims_per_subgroup;
640			unsigned	vgt_esgs_ring_itemsize;
641		} gs;
642
643		struct {
644			unsigned	vgt_gs_mode;
645			unsigned	vgt_primitiveid_en;
646			unsigned	vgt_reuse_off;
647			unsigned	spi_vs_out_config;
648			unsigned	spi_shader_pos_format;
649			unsigned	pa_cl_vte_cntl;
650		} vs;
651
652		struct {
653			unsigned	spi_ps_input_ena;
654			unsigned	spi_ps_input_addr;
655			unsigned	spi_baryc_cntl;
656			unsigned	spi_ps_in_control;
657			unsigned	spi_shader_z_format;
658			unsigned	spi_shader_col_format;
659			unsigned	cb_shader_mask;
660		} ps;
661	} ctx_reg;
662
663	/*For save precompute registers value */
664	unsigned vgt_tf_param; /* VGT_TF_PARAM */
665	unsigned vgt_vertex_reuse_block_cntl; /* VGT_VERTEX_REUSE_BLOCK_CNTL */
666};
667
668struct si_shader_part {
669	struct si_shader_part *next;
670	union si_shader_part_key key;
671	struct ac_shader_binary binary;
672	struct si_shader_config config;
673};
674
675/* si_shader.c */
676struct si_shader *
677si_generate_gs_copy_shader(struct si_screen *sscreen,
678			   struct ac_llvm_compiler *compiler,
679			   struct si_shader_selector *gs_selector,
680			   struct pipe_debug_callback *debug);
681int si_compile_tgsi_shader(struct si_screen *sscreen,
682			   struct ac_llvm_compiler *compiler,
683			   struct si_shader *shader,
684			   struct pipe_debug_callback *debug);
685int si_shader_create(struct si_screen *sscreen, struct ac_llvm_compiler *compiler,
686		     struct si_shader *shader,
687		     struct pipe_debug_callback *debug);
688void si_shader_destroy(struct si_shader *shader);
689unsigned si_shader_io_get_unique_index_patch(unsigned semantic_name, unsigned index);
690unsigned si_shader_io_get_unique_index(unsigned semantic_name, unsigned index,
691				       unsigned is_varying);
692int si_shader_binary_upload(struct si_screen *sscreen, struct si_shader *shader);
693void si_shader_dump(struct si_screen *sscreen, const struct si_shader *shader,
694		    struct pipe_debug_callback *debug, unsigned processor,
695		    FILE *f, bool check_debug_option);
696void si_shader_dump_stats_for_shader_db(const struct si_shader *shader,
697					struct pipe_debug_callback *debug);
698void si_multiwave_lds_size_workaround(struct si_screen *sscreen,
699				      unsigned *lds_size);
700void si_shader_apply_scratch_relocs(struct si_shader *shader,
701				    uint64_t scratch_va);
702void si_shader_binary_read_config(struct ac_shader_binary *binary,
703				  struct si_shader_config *conf,
704				  unsigned symbol_offset);
705const char *si_get_shader_name(const struct si_shader *shader, unsigned processor);
706
707/* si_shader_nir.c */
708void si_nir_scan_shader(const struct nir_shader *nir,
709			struct tgsi_shader_info *info);
710void si_nir_scan_tess_ctrl(const struct nir_shader *nir,
711			   struct tgsi_tessctrl_info *out);
712void si_lower_nir(struct si_shader_selector *sel);
713void si_nir_opts(struct nir_shader *nir);
714
715/* Inline helpers. */
716
717/* Return the pointer to the main shader part's pointer. */
718static inline struct si_shader **
719si_get_main_shader_part(struct si_shader_selector *sel,
720			struct si_shader_key *key)
721{
722	if (key->as_ls)
723		return &sel->main_shader_part_ls;
724	if (key->as_es)
725		return &sel->main_shader_part_es;
726	return &sel->main_shader_part;
727}
728
729static inline bool
730si_shader_uses_bindless_samplers(struct si_shader_selector *selector)
731{
732	return selector ? selector->info.uses_bindless_samplers : false;
733}
734
735static inline bool
736si_shader_uses_bindless_images(struct si_shader_selector *selector)
737{
738	return selector ? selector->info.uses_bindless_images : false;
739}
740
741void si_destroy_shader_selector(struct si_context *sctx,
742			        struct si_shader_selector *sel);
743
744static inline void
745si_shader_selector_reference(struct si_context *sctx,
746			     struct si_shader_selector **dst,
747			     struct si_shader_selector *src)
748{
749	if (pipe_reference(&(*dst)->reference, &src->reference))
750		si_destroy_shader_selector(sctx, *dst);
751
752	*dst = src;
753}
754
755#endif
756