1/*
2 * Copyright © 2019 Broadcom
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24/**
25 * Implements lowering for logical operations.
26 *
27 * V3D doesn't have any hardware support for logic ops.  Instead, you read the
28 * current contents of the destination from the tile buffer, then do math using
29 * your output color and that destination value, and update the output color
30 * appropriately.
31 */
32
33#include "util/format/u_format.h"
34#include "compiler/nir/nir_builder.h"
35#include "compiler/nir/nir_format_convert.h"
36#include "v3d_compiler.h"
37
38
39typedef nir_ssa_def *(*nir_pack_func)(nir_builder *b, nir_ssa_def *c);
40typedef nir_ssa_def *(*nir_unpack_func)(nir_builder *b, nir_ssa_def *c);
41
42static bool
43logicop_depends_on_dst_color(int logicop_func)
44{
45        switch (logicop_func) {
46        case PIPE_LOGICOP_SET:
47        case PIPE_LOGICOP_CLEAR:
48        case PIPE_LOGICOP_COPY:
49        case PIPE_LOGICOP_COPY_INVERTED:
50                return false;
51        default:
52                return true;
53        }
54}
55
56static nir_ssa_def *
57v3d_logicop(nir_builder *b, int logicop_func,
58            nir_ssa_def *src, nir_ssa_def *dst)
59{
60        switch (logicop_func) {
61        case PIPE_LOGICOP_CLEAR:
62                return nir_imm_int(b, 0);
63        case PIPE_LOGICOP_NOR:
64                return nir_inot(b, nir_ior(b, src, dst));
65        case PIPE_LOGICOP_AND_INVERTED:
66                return nir_iand(b, nir_inot(b, src), dst);
67        case PIPE_LOGICOP_COPY_INVERTED:
68                return nir_inot(b, src);
69        case PIPE_LOGICOP_AND_REVERSE:
70                return nir_iand(b, src, nir_inot(b, dst));
71        case PIPE_LOGICOP_INVERT:
72                return nir_inot(b, dst);
73        case PIPE_LOGICOP_XOR:
74                return nir_ixor(b, src, dst);
75        case PIPE_LOGICOP_NAND:
76                return nir_inot(b, nir_iand(b, src, dst));
77        case PIPE_LOGICOP_AND:
78                return nir_iand(b, src, dst);
79        case PIPE_LOGICOP_EQUIV:
80                return nir_inot(b, nir_ixor(b, src, dst));
81        case PIPE_LOGICOP_NOOP:
82                return dst;
83        case PIPE_LOGICOP_OR_INVERTED:
84                return nir_ior(b, nir_inot(b, src), dst);
85        case PIPE_LOGICOP_OR_REVERSE:
86                return nir_ior(b, src, nir_inot(b, dst));
87        case PIPE_LOGICOP_OR:
88                return nir_ior(b, src, dst);
89        case PIPE_LOGICOP_SET:
90                return nir_imm_int(b, ~0);
91        default:
92                fprintf(stderr, "Unknown logic op %d\n", logicop_func);
93                FALLTHROUGH;
94        case PIPE_LOGICOP_COPY:
95                return src;
96        }
97}
98
99static nir_ssa_def *
100v3d_nir_get_swizzled_channel(nir_builder *b, nir_ssa_def **srcs, int swiz)
101{
102        switch (swiz) {
103        default:
104        case PIPE_SWIZZLE_NONE:
105                fprintf(stderr, "warning: unknown swizzle\n");
106                FALLTHROUGH;
107        case PIPE_SWIZZLE_0:
108                return nir_imm_float(b, 0.0);
109        case PIPE_SWIZZLE_1:
110                return nir_imm_float(b, 1.0);
111        case PIPE_SWIZZLE_X:
112        case PIPE_SWIZZLE_Y:
113        case PIPE_SWIZZLE_Z:
114        case PIPE_SWIZZLE_W:
115                return srcs[swiz];
116        }
117}
118
119static nir_ssa_def *
120v3d_nir_swizzle_and_pack(nir_builder *b, nir_ssa_def **chans,
121                         const uint8_t *swiz, nir_pack_func pack_func)
122{
123        nir_ssa_def *c[4];
124        for (int i = 0; i < 4; i++)
125                c[i] = v3d_nir_get_swizzled_channel(b, chans, swiz[i]);
126
127        return pack_func(b, nir_vec4(b, c[0], c[1], c[2], c[3]));
128}
129
130static nir_ssa_def *
131v3d_nir_unpack_and_swizzle(nir_builder *b, nir_ssa_def *packed,
132                           const uint8_t *swiz, nir_unpack_func unpack_func)
133{
134        nir_ssa_def *unpacked = unpack_func(b, packed);
135
136        nir_ssa_def *unpacked_chans[4];
137        for (int i = 0; i < 4; i++)
138                unpacked_chans[i] = nir_channel(b, unpacked, i);
139
140        nir_ssa_def *c[4];
141        for (int i = 0; i < 4; i++)
142                c[i] = v3d_nir_get_swizzled_channel(b, unpacked_chans, swiz[i]);
143
144        return nir_vec4(b, c[0], c[1], c[2], c[3]);
145}
146
147static nir_ssa_def *
148pack_unorm_rgb10a2(nir_builder *b, nir_ssa_def *c)
149{
150        static const unsigned bits[4] = { 10, 10, 10, 2 };
151        nir_ssa_def *unorm = nir_format_float_to_unorm(b, c, bits);
152
153        nir_ssa_def *chans[4];
154        for (int i = 0; i < 4; i++)
155                chans[i] = nir_channel(b, unorm, i);
156
157        nir_ssa_def *result = nir_mov(b, chans[0]);
158        int offset = bits[0];
159        for (int i = 1; i < 4; i++) {
160                nir_ssa_def *shifted_chan =
161                        nir_ishl(b, chans[i], nir_imm_int(b, offset));
162                result = nir_ior(b, result, shifted_chan);
163                offset += bits[i];
164        }
165        return result;
166}
167
168static nir_ssa_def *
169unpack_unorm_rgb10a2(nir_builder *b, nir_ssa_def *c)
170{
171        static const unsigned bits[4] = { 10, 10, 10, 2 };
172        const unsigned masks[4] = { BITFIELD_MASK(bits[0]),
173                                    BITFIELD_MASK(bits[1]),
174                                    BITFIELD_MASK(bits[2]),
175                                    BITFIELD_MASK(bits[3]) };
176
177        nir_ssa_def *chans[4];
178        for (int i = 0; i < 4; i++) {
179                nir_ssa_def *unorm = nir_iand(b, c, nir_imm_int(b, masks[i]));
180                chans[i] = nir_format_unorm_to_float(b, unorm, &bits[i]);
181                c = nir_ushr(b, c, nir_imm_int(b, bits[i]));
182        }
183
184        return nir_vec4(b, chans[0], chans[1], chans[2], chans[3]);
185}
186
187static const uint8_t *
188v3d_get_format_swizzle_for_rt(struct v3d_compile *c, int rt)
189{
190        static const uint8_t ident[4] = { 0, 1, 2, 3 };
191
192        /* We will automatically swap R and B channels for BGRA formats
193         * on tile loads and stores (see 'swap_rb' field in v3d_resource) so
194         * we want to treat these surfaces as if they were regular RGBA formats.
195         */
196        if (c->fs_key->color_fmt[rt].swizzle[0] == 2 &&
197            c->fs_key->color_fmt[rt].format != PIPE_FORMAT_B5G6R5_UNORM) {
198                return ident;
199        } else {
200                return  c->fs_key->color_fmt[rt].swizzle;
201        }
202}
203
204static nir_ssa_def *
205v3d_nir_get_tlb_color(nir_builder *b, struct v3d_compile *c, int rt, int sample)
206{
207        uint32_t num_components =
208                util_format_get_nr_components(c->fs_key->color_fmt[rt].format);
209
210        nir_ssa_def *color[4];
211        for (int i = 0; i < 4; i++) {
212                if (i < num_components) {
213                        color[i] =
214                                nir_load_tlb_color_v3d(b, 1, 32, nir_imm_int(b, rt),
215                                                       .base = sample,
216                                                       .component = i);
217                } else {
218                        /* These will be DCEd */
219                        color[i] = nir_imm_int(b, 0);
220                }
221        }
222        return nir_vec4(b, color[0], color[1], color[2], color[3]);
223}
224
225static nir_ssa_def *
226v3d_emit_logic_op_raw(struct v3d_compile *c, nir_builder *b,
227                      nir_ssa_def **src_chans, nir_ssa_def **dst_chans,
228                      int rt, int sample)
229{
230        const uint8_t *fmt_swz = v3d_get_format_swizzle_for_rt(c, rt);
231
232        nir_ssa_def *op_res[4];
233        for (int i = 0; i < 4; i++) {
234                nir_ssa_def *src = src_chans[i];
235                nir_ssa_def *dst =
236                        v3d_nir_get_swizzled_channel(b, dst_chans, fmt_swz[i]);
237                op_res[i] = v3d_logicop(b, c->fs_key->logicop_func, src, dst);
238
239                /* In Vulkan we configure our integer RTs to clamp, so we need
240                 * to ignore result bits that don't fit in the destination RT
241                 * component size.
242                 */
243                if (c->key->environment == V3D_ENVIRONMENT_VULKAN) {
244                        uint32_t bits =
245                                util_format_get_component_bits(
246                                        c->fs_key->color_fmt[rt].format,
247                                        UTIL_FORMAT_COLORSPACE_RGB, i);
248                        if (bits > 0 && bits < 32) {
249                                nir_ssa_def *mask =
250                                        nir_imm_int(b, (1u << bits) - 1);
251                                op_res[i] = nir_iand(b, op_res[i], mask);
252                        }
253                }
254        }
255
256        nir_ssa_def *r[4];
257        for (int i = 0; i < 4; i++)
258                r[i] = v3d_nir_get_swizzled_channel(b, op_res, fmt_swz[i]);
259
260        return nir_vec4(b, r[0], r[1], r[2], r[3]);
261}
262
263static nir_ssa_def *
264v3d_emit_logic_op_unorm(struct v3d_compile *c, nir_builder *b,
265                        nir_ssa_def **src_chans, nir_ssa_def **dst_chans,
266                        int rt, int sample,
267                        nir_pack_func pack_func, nir_unpack_func unpack_func)
268{
269        static const uint8_t src_swz[4] = { 0, 1, 2, 3 };
270        nir_ssa_def *packed_src =
271                v3d_nir_swizzle_and_pack(b, src_chans, src_swz, pack_func);
272
273        const uint8_t *fmt_swz = v3d_get_format_swizzle_for_rt(c, rt);
274        nir_ssa_def *packed_dst =
275                v3d_nir_swizzle_and_pack(b, dst_chans, fmt_swz, pack_func);
276
277        nir_ssa_def *packed_result =
278                v3d_logicop(b, c->fs_key->logicop_func, packed_src, packed_dst);
279
280        return v3d_nir_unpack_and_swizzle(b, packed_result, fmt_swz, unpack_func);
281}
282
283static nir_ssa_def *
284v3d_nir_emit_logic_op(struct v3d_compile *c, nir_builder *b,
285                      nir_ssa_def *src, int rt, int sample)
286{
287        nir_ssa_def *dst = v3d_nir_get_tlb_color(b, c, rt, sample);
288
289        nir_ssa_def *src_chans[4], *dst_chans[4];
290        for (unsigned i = 0; i < 4; i++) {
291                src_chans[i] = nir_channel(b, src, i);
292                dst_chans[i] = nir_channel(b, dst, i);
293        }
294
295        if (c->fs_key->color_fmt[rt].format == PIPE_FORMAT_R10G10B10A2_UNORM) {
296                return v3d_emit_logic_op_unorm(
297                        c, b, src_chans, dst_chans, rt, 0,
298                        pack_unorm_rgb10a2, unpack_unorm_rgb10a2);
299        }
300
301        if (util_format_is_unorm(c->fs_key->color_fmt[rt].format)) {
302                return v3d_emit_logic_op_unorm(
303                        c, b, src_chans, dst_chans, rt, 0,
304                        nir_pack_unorm_4x8, nir_unpack_unorm_4x8);
305        }
306
307        return v3d_emit_logic_op_raw(c, b, src_chans, dst_chans, rt, 0);
308}
309
310static void
311v3d_emit_ms_output(nir_builder *b,
312                   nir_ssa_def *color, nir_src *offset,
313                   nir_alu_type type, int rt, int sample)
314{
315        nir_store_tlb_sample_color_v3d(b, color, nir_imm_int(b, rt), .base = sample, .component = 0, .src_type = type);
316}
317
318static void
319v3d_nir_lower_logic_op_instr(struct v3d_compile *c,
320                             nir_builder *b,
321                             nir_intrinsic_instr *intr,
322                             int rt)
323{
324        nir_ssa_def *frag_color = intr->src[0].ssa;
325
326
327        const int logic_op = c->fs_key->logicop_func;
328        if (c->fs_key->msaa && logicop_depends_on_dst_color(logic_op)) {
329                c->msaa_per_sample_output = true;
330
331                nir_src *offset = &intr->src[1];
332                nir_alu_type type = nir_intrinsic_src_type(intr);
333                for (int i = 0; i < V3D_MAX_SAMPLES; i++) {
334                        nir_ssa_def *sample =
335                                v3d_nir_emit_logic_op(c, b, frag_color, rt, i);
336
337                        v3d_emit_ms_output(b, sample, offset, type, rt, i);
338                }
339
340                nir_instr_remove(&intr->instr);
341        } else {
342                nir_ssa_def *result =
343                        v3d_nir_emit_logic_op(c, b, frag_color, rt, 0);
344
345                nir_instr_rewrite_src(&intr->instr, &intr->src[0],
346                                      nir_src_for_ssa(result));
347                intr->num_components = result->num_components;
348        }
349}
350
351static bool
352v3d_nir_lower_logic_ops_block(nir_block *block, struct v3d_compile *c)
353{
354        nir_foreach_instr_safe(instr, block) {
355                if (instr->type != nir_instr_type_intrinsic)
356                        continue;
357
358                nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
359                if (intr->intrinsic != nir_intrinsic_store_output)
360                        continue;
361
362                nir_foreach_shader_out_variable(var, c->s) {
363                        const int driver_loc = var->data.driver_location;
364                        if (driver_loc != nir_intrinsic_base(intr))
365                                continue;
366
367                        const int loc = var->data.location;
368                        if (loc != FRAG_RESULT_COLOR &&
369                            (loc < FRAG_RESULT_DATA0 ||
370                             loc >= FRAG_RESULT_DATA0 + V3D_MAX_DRAW_BUFFERS)) {
371                                continue;
372                        }
373
374                        /* Logic operations do not apply on floating point or
375                         * sRGB enabled render targets.
376                         */
377                        const int rt = driver_loc;
378                        assert(rt < V3D_MAX_DRAW_BUFFERS);
379
380                        const enum pipe_format format =
381                                c->fs_key->color_fmt[rt].format;
382                        if (util_format_is_float(format) ||
383                            util_format_is_srgb(format)) {
384                                continue;
385                        }
386
387                        nir_function_impl *impl =
388                                nir_cf_node_get_function(&block->cf_node);
389                        nir_builder b;
390                        nir_builder_init(&b, impl);
391                        b.cursor = nir_before_instr(&intr->instr);
392                        v3d_nir_lower_logic_op_instr(c, &b, intr, rt);
393                }
394        }
395
396        return true;
397}
398
399void
400v3d_nir_lower_logic_ops(nir_shader *s, struct v3d_compile *c)
401{
402        /* Nothing to do if logic op is 'copy src to dst' or if logic ops are
403         * disabled (we set the logic op to copy in that case).
404         */
405        if (c->fs_key->logicop_func == PIPE_LOGICOP_COPY)
406                return;
407
408        nir_foreach_function(function, s) {
409                if (function->impl) {
410                        nir_foreach_block(block, function->impl)
411                                v3d_nir_lower_logic_ops_block(block, c);
412
413                        nir_metadata_preserve(function->impl,
414                                              nir_metadata_block_index |
415                                              nir_metadata_dominance);
416                }
417        }
418}
419