1/*
2 * Copyright (C) 2018-2020 Collabora, Ltd.
3 * Copyright (C) 2019-2020 Icecream95
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24
25#include "pan_ir.h"
26#include "compiler/nir/nir_builder.h"
27
28/* Midgard can write all of color, depth and stencil in a single writeout
29 * operation, so we merge depth/stencil stores with color stores.
30 * If there are no color stores, we add a write to the "depth RT".
31 *
32 * For Bifrost, we want these combined so we can properly order
33 * +ZS_EMIT with respect to +ATEST and +BLEND, as well as combining
34 * depth/stencil stores into a single +ZS_EMIT op.
35 */
36bool
37pan_nir_lower_zs_store(nir_shader *nir)
38{
39        if (nir->info.stage != MESA_SHADER_FRAGMENT)
40                return false;
41
42        nir_variable *z_var = NULL, *s_var = NULL;
43
44        nir_foreach_shader_out_variable(var, nir) {
45                if (var->data.location == FRAG_RESULT_DEPTH)
46                        z_var = var;
47                else if (var->data.location == FRAG_RESULT_STENCIL)
48                        s_var = var;
49        }
50
51        if (!z_var && !s_var)
52                return false;
53
54        bool progress = false;
55
56        nir_foreach_function(function, nir) {
57                if (!function->impl) continue;
58
59                nir_intrinsic_instr *z_store = NULL, *s_store = NULL;
60
61                nir_foreach_block(block, function->impl) {
62                        nir_foreach_instr_safe(instr, block) {
63                                if (instr->type != nir_instr_type_intrinsic)
64                                        continue;
65
66                                nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
67                                if (intr->intrinsic != nir_intrinsic_store_output)
68                                        continue;
69
70                                if (z_var && nir_intrinsic_base(intr) == z_var->data.driver_location) {
71                                        assert(!z_store);
72                                        z_store = intr;
73                                }
74
75                                if (s_var && nir_intrinsic_base(intr) == s_var->data.driver_location) {
76                                        assert(!s_store);
77                                        s_store = intr;
78                                }
79                        }
80                }
81
82                if (!z_store && !s_store) continue;
83
84                bool replaced = false;
85
86                nir_foreach_block(block, function->impl) {
87                        nir_foreach_instr_safe(instr, block) {
88                                if (instr->type != nir_instr_type_intrinsic)
89                                        continue;
90
91                                nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
92                                if (intr->intrinsic != nir_intrinsic_store_output)
93                                        continue;
94
95                                const nir_variable *var = nir_find_variable_with_driver_location(nir, nir_var_shader_out, nir_intrinsic_base(intr));
96                                assert(var);
97
98                                if (var->data.location < FRAG_RESULT_DATA0)
99                                        continue;
100
101                                if (var->data.index)
102                                        continue;
103
104                                assert(nir_src_is_const(intr->src[1]) && "no indirect outputs");
105
106                                nir_builder b;
107                                nir_builder_init(&b, function->impl);
108
109                                assert(!z_store || z_store->instr.block == instr->block);
110                                assert(!s_store || s_store->instr.block == instr->block);
111                                b.cursor = nir_after_block_before_jump(instr->block);
112
113                                nir_intrinsic_instr *combined_store;
114                                combined_store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_store_combined_output_pan);
115
116                                combined_store->num_components = intr->src[0].ssa->num_components;
117
118                                nir_intrinsic_set_base(combined_store, nir_intrinsic_base(intr));
119                                nir_intrinsic_set_src_type(combined_store, nir_intrinsic_src_type(intr));
120
121                                unsigned writeout = PAN_WRITEOUT_C;
122                                if (z_store)
123                                        writeout |= PAN_WRITEOUT_Z;
124                                if (s_store)
125                                        writeout |= PAN_WRITEOUT_S;
126
127                                nir_intrinsic_set_component(combined_store, writeout);
128
129                                struct nir_ssa_def *zero = nir_imm_int(&b, 0);
130
131                                struct nir_ssa_def *src[4] = {
132                                   intr->src[0].ssa,
133                                   intr->src[1].ssa,
134                                   z_store ? z_store->src[0].ssa : zero,
135                                   s_store ? s_store->src[0].ssa : zero,
136                                };
137
138                                for (int i = 0; i < 4; ++i)
139                                   combined_store->src[i] = nir_src_for_ssa(src[i]);
140
141                                nir_builder_instr_insert(&b, &combined_store->instr);
142
143                                nir_instr_remove(instr);
144
145                                replaced = true;
146                        }
147                }
148
149                /* Insert a store to the depth RT (0xff) if needed */
150                if (!replaced) {
151                        nir_builder b;
152                        nir_builder_init(&b, function->impl);
153
154                        nir_block *block = NULL;
155                        if (z_store && s_store)
156                                assert(z_store->instr.block == s_store->instr.block);
157
158                        if (z_store)
159                                block = z_store->instr.block;
160                        else
161                                block = s_store->instr.block;
162
163                        b.cursor = nir_after_block_before_jump(block);
164
165                        nir_intrinsic_instr *combined_store;
166                        combined_store = nir_intrinsic_instr_create(b.shader, nir_intrinsic_store_combined_output_pan);
167
168                        combined_store->num_components = 4;
169
170                        unsigned base;
171                        if (z_store)
172                                base = nir_intrinsic_base(z_store);
173                        else
174                                base = nir_intrinsic_base(s_store);
175                        nir_intrinsic_set_base(combined_store, base);
176                        nir_intrinsic_set_src_type(combined_store, nir_type_float32);
177
178                        unsigned writeout = 0;
179                        if (z_store)
180                                writeout |= PAN_WRITEOUT_Z;
181                        if (s_store)
182                                writeout |= PAN_WRITEOUT_S;
183
184                        nir_intrinsic_set_component(combined_store, writeout);
185
186                        struct nir_ssa_def *zero = nir_imm_int(&b, 0);
187
188                        struct nir_ssa_def *src[4] = {
189                                nir_imm_vec4(&b, 0, 0, 0, 0),
190                                zero,
191                                z_store ? z_store->src[0].ssa : zero,
192                                s_store ? s_store->src[0].ssa : zero,
193                        };
194
195                        for (int i = 0; i < 4; ++i)
196                                combined_store->src[i] = nir_src_for_ssa(src[i]);
197
198                        nir_builder_instr_insert(&b, &combined_store->instr);
199                }
200
201                if (z_store)
202                        nir_instr_remove(&z_store->instr);
203
204                if (s_store)
205                        nir_instr_remove(&s_store->instr);
206
207                nir_metadata_preserve(function->impl, nir_metadata_block_index | nir_metadata_dominance);
208                progress = true;
209        }
210
211        return progress;
212}
213
214/* Real writeout stores, which break execution, need to be moved to after
215 * dual-source stores, which are just standard register writes. */
216bool
217pan_nir_reorder_writeout(nir_shader *nir)
218{
219        bool progress = false;
220
221        nir_foreach_function(function, nir) {
222                if (!function->impl) continue;
223
224                nir_foreach_block(block, function->impl) {
225                        nir_instr *last_writeout = NULL;
226
227                        nir_foreach_instr_reverse_safe(instr, block) {
228                                if (instr->type != nir_instr_type_intrinsic)
229                                        continue;
230
231                                nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
232                                if (intr->intrinsic != nir_intrinsic_store_output)
233                                        continue;
234
235                                const nir_variable *var = nir_find_variable_with_driver_location(nir, nir_var_shader_out, nir_intrinsic_base(intr));
236
237                                if (var->data.index) {
238                                        if (!last_writeout)
239                                                last_writeout = instr;
240                                        continue;
241                                }
242
243                                if (!last_writeout)
244                                        continue;
245
246                                /* This is a real store, so move it to after dual-source stores */
247                                exec_node_remove(&instr->node);
248                                exec_node_insert_after(&last_writeout->node, &instr->node);
249
250                                progress = true;
251                        }
252                }
253        }
254
255        return progress;
256}
257