1/*
2 * Copyright © 2018 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24#include "nir.h"
25#include "linker_util.h"
26#include "gl_nir_linker.h"
27#include "compiler/glsl/ir_uniform.h" /* for gl_uniform_storage */
28#include "main/context.h"
29
30/**
31 * This file do the common link for GLSL atomic counter uniforms, using NIR,
32 * instead of IR as the counter-part glsl/link_uniforms.cpp
33 */
34
35struct active_atomic_counter_uniform {
36   unsigned loc;
37   nir_variable *var;
38};
39
40struct active_atomic_buffer {
41   struct active_atomic_counter_uniform *uniforms;
42   unsigned num_uniforms;
43   unsigned uniform_buffer_size;
44   unsigned stage_counter_references[MESA_SHADER_STAGES];
45   unsigned size;
46};
47
48static void
49add_atomic_counter(const void *ctx,
50                   struct active_atomic_buffer *buffer,
51                   unsigned uniform_loc,
52                   nir_variable *var)
53{
54   if (buffer->num_uniforms >= buffer->uniform_buffer_size) {
55      if (buffer->uniform_buffer_size == 0)
56         buffer->uniform_buffer_size = 1;
57      else
58         buffer->uniform_buffer_size *= 2;
59      buffer->uniforms = reralloc(ctx,
60                                  buffer->uniforms,
61                                  struct active_atomic_counter_uniform,
62                                  buffer->uniform_buffer_size);
63   }
64
65   struct active_atomic_counter_uniform *uniform =
66      buffer->uniforms + buffer->num_uniforms;
67   uniform->loc = uniform_loc;
68   uniform->var = var;
69   buffer->num_uniforms++;
70}
71
72static void
73process_atomic_variable(const struct glsl_type *t,
74                        struct gl_shader_program *prog,
75                        unsigned *uniform_loc,
76                        nir_variable *var,
77                        struct active_atomic_buffer *buffers,
78                        unsigned *num_buffers,
79                        int *offset,
80                        unsigned shader_stage)
81{
82   /* FIXME: Arrays of arrays get counted separately. For example:
83    * x1[3][3][2] = 9 uniforms, 18 atomic counters
84    * x2[3][2]    = 3 uniforms, 6 atomic counters
85    * x3[2]       = 1 uniform, 2 atomic counters
86    *
87    * However this code marks all the counters as active even when they
88    * might not be used.
89    */
90   if (glsl_type_is_array(t) &&
91       glsl_type_is_array(glsl_get_array_element(t))) {
92      for (unsigned i = 0; i < glsl_get_length(t); i++) {
93         process_atomic_variable(glsl_get_array_element(t),
94                                 prog,
95                                 uniform_loc,
96                                 var,
97                                 buffers, num_buffers,
98                                 offset,
99                                 shader_stage);
100      }
101   } else {
102      struct active_atomic_buffer *buf = buffers + var->data.binding;
103      struct gl_uniform_storage *const storage =
104         &prog->data->UniformStorage[*uniform_loc];
105
106      /* If this is the first time the buffer is used, increment
107       * the counter of buffers used.
108       */
109      if (buf->size == 0)
110         (*num_buffers)++;
111
112      add_atomic_counter(buffers, /* ctx */
113                         buf,
114                         *uniform_loc,
115                         var);
116
117      /* When checking for atomic counters we should count every member in
118       * an array as an atomic counter reference.
119       */
120      if (glsl_type_is_array(t))
121         buf->stage_counter_references[shader_stage] += glsl_get_length(t);
122      else
123         buf->stage_counter_references[shader_stage]++;
124      buf->size = MAX2(buf->size, *offset + glsl_atomic_size(t));
125
126      storage->offset = *offset;
127      *offset += glsl_atomic_size(t);
128
129      (*uniform_loc)++;
130   }
131}
132
133static struct active_atomic_buffer *
134find_active_atomic_counters(struct gl_context *ctx,
135                            struct gl_shader_program *prog,
136                            unsigned *num_buffers)
137{
138   struct active_atomic_buffer *buffers =
139      rzalloc_array(NULL, /* ctx */
140                    struct active_atomic_buffer,
141                    ctx->Const.MaxAtomicBufferBindings);
142   *num_buffers = 0;
143
144   for (unsigned i = 0; i < MESA_SHADER_STAGES; ++i) {
145      struct gl_linked_shader *sh = prog->_LinkedShaders[i];
146      if (sh == NULL)
147         continue;
148
149      nir_shader *nir = sh->Program->nir;
150
151      nir_foreach_uniform_variable(var, nir) {
152         if (!glsl_contains_atomic(var->type))
153            continue;
154
155         int offset = var->data.offset;
156         unsigned uniform_loc = var->data.location;
157
158         process_atomic_variable(var->type,
159                                 prog,
160                                 &uniform_loc,
161                                 var,
162                                 buffers,
163                                 num_buffers,
164                                 &offset,
165                                 i);
166      }
167   }
168
169   return buffers;
170}
171
172static bool
173check_atomic_counters_overlap(const nir_variable *x, const nir_variable *y)
174{
175   return ((x->data.offset >= y->data.offset &&
176            x->data.offset < y->data.offset + glsl_atomic_size(y->type)) ||
177           (y->data.offset >= x->data.offset &&
178            y->data.offset < x->data.offset + glsl_atomic_size(x->type)));
179}
180
181static int
182cmp_active_counter_offsets(const void *a, const void *b)
183{
184   const struct active_atomic_counter_uniform *const first =
185      (struct active_atomic_counter_uniform *) a;
186   const struct active_atomic_counter_uniform *const second =
187      (struct active_atomic_counter_uniform *) b;
188
189   return first->var->data.offset - second->var->data.offset;
190}
191
192void
193gl_nir_link_assign_atomic_counter_resources(struct gl_context *ctx,
194                                            struct gl_shader_program *prog)
195{
196   unsigned num_buffers;
197   unsigned num_atomic_buffers[MESA_SHADER_STAGES] = {0};
198   struct active_atomic_buffer *abs =
199      find_active_atomic_counters(ctx, prog, &num_buffers);
200
201   prog->data->AtomicBuffers =
202      rzalloc_array(prog->data, struct gl_active_atomic_buffer, num_buffers);
203   prog->data->NumAtomicBuffers = num_buffers;
204
205   unsigned buffer_idx = 0;
206   for (unsigned binding = 0;
207        binding < ctx->Const.MaxAtomicBufferBindings;
208        binding++) {
209
210      /* If the binding was not used, skip.
211       */
212      if (abs[binding].size == 0)
213         continue;
214
215      struct active_atomic_buffer *ab = abs + binding;
216      struct gl_active_atomic_buffer *mab =
217         prog->data->AtomicBuffers + buffer_idx;
218
219      /* Assign buffer-specific fields. */
220      mab->Binding = binding;
221      mab->MinimumSize = ab->size;
222      mab->Uniforms = rzalloc_array(prog->data->AtomicBuffers, GLuint,
223                                    ab->num_uniforms);
224      mab->NumUniforms = ab->num_uniforms;
225
226      /* Assign counter-specific fields. */
227      for (unsigned j = 0; j < ab->num_uniforms; j++) {
228         nir_variable *var = ab->uniforms[j].var;
229         struct gl_uniform_storage *storage =
230            &prog->data->UniformStorage[ab->uniforms[j].loc];
231
232         mab->Uniforms[j] = ab->uniforms[j].loc;
233
234         storage->atomic_buffer_index = buffer_idx;
235         storage->offset = var->data.offset;
236         if (glsl_type_is_array(var->type)) {
237            const struct glsl_type *without_array =
238               glsl_without_array(var->type);
239            storage->array_stride = glsl_atomic_size(without_array);
240         } else {
241            storage->array_stride = 0;
242         }
243         if (!glsl_type_is_matrix(var->type))
244            storage->matrix_stride = 0;
245      }
246
247      /* Assign stage-specific fields. */
248      for (unsigned stage = 0; stage < MESA_SHADER_STAGES; ++stage) {
249         if (ab->stage_counter_references[stage]) {
250            mab->StageReferences[stage] = GL_TRUE;
251            num_atomic_buffers[stage]++;
252         } else {
253            mab->StageReferences[stage] = GL_FALSE;
254         }
255      }
256
257      buffer_idx++;
258   }
259
260   /* Store a list pointers to atomic buffers per stage and store the index
261    * to the intra-stage buffer list in uniform storage.
262    */
263   for (unsigned stage = 0; stage < MESA_SHADER_STAGES; ++stage) {
264      if (prog->_LinkedShaders[stage] == NULL ||
265          num_atomic_buffers[stage] <= 0)
266         continue;
267
268      struct gl_program *gl_prog = prog->_LinkedShaders[stage]->Program;
269      gl_prog->info.num_abos = num_atomic_buffers[stage];
270      gl_prog->sh.AtomicBuffers =
271         rzalloc_array(gl_prog,
272                       struct gl_active_atomic_buffer *,
273                       num_atomic_buffers[stage]);
274
275      gl_prog->nir->info.num_abos = num_atomic_buffers[stage];
276
277      unsigned intra_stage_idx = 0;
278      for (unsigned i = 0; i < num_buffers; i++) {
279         struct gl_active_atomic_buffer *atomic_buffer =
280            &prog->data->AtomicBuffers[i];
281         if (!atomic_buffer->StageReferences[stage])
282            continue;
283
284         gl_prog->sh.AtomicBuffers[intra_stage_idx] = atomic_buffer;
285
286         for (unsigned u = 0; u < atomic_buffer->NumUniforms; u++) {
287            GLuint uniform_loc = atomic_buffer->Uniforms[u];
288            struct gl_opaque_uniform_index *opaque =
289               prog->data->UniformStorage[uniform_loc].opaque + stage;
290            opaque->index = intra_stage_idx;
291            opaque->active = true;
292         }
293
294         intra_stage_idx++;
295      }
296   }
297
298   assert(buffer_idx == num_buffers);
299
300   ralloc_free(abs);
301}
302
303void
304gl_nir_link_check_atomic_counter_resources(struct gl_context *ctx,
305                                           struct gl_shader_program *prog)
306{
307   unsigned num_buffers;
308   struct active_atomic_buffer *abs =
309      find_active_atomic_counters(ctx, prog, &num_buffers);
310   unsigned atomic_counters[MESA_SHADER_STAGES] = {0};
311   unsigned atomic_buffers[MESA_SHADER_STAGES] = {0};
312   unsigned total_atomic_counters = 0;
313   unsigned total_atomic_buffers = 0;
314
315   /* Sum the required resources.  Note that this counts buffers and
316    * counters referenced by several shader stages multiple times
317    * against the combined limit -- That's the behavior the spec
318    * requires.
319    */
320   for (unsigned i = 0; i < ctx->Const.MaxAtomicBufferBindings; i++) {
321      if (abs[i].size == 0)
322         continue;
323
324      qsort(abs[i].uniforms, abs[i].num_uniforms,
325            sizeof(struct active_atomic_counter_uniform),
326            cmp_active_counter_offsets);
327
328      for (unsigned j = 1; j < abs[i].num_uniforms; j++) {
329         /* If an overlapping counter found, it must be a reference to the
330          * same counter from a different shader stage.
331          */
332         if (check_atomic_counters_overlap(abs[i].uniforms[j-1].var,
333                                           abs[i].uniforms[j].var)
334             && strcmp(abs[i].uniforms[j-1].var->name,
335                       abs[i].uniforms[j].var->name) != 0) {
336            linker_error(prog, "Atomic counter %s declared at offset %d "
337                         "which is already in use.",
338                         abs[i].uniforms[j].var->name,
339                         abs[i].uniforms[j].var->data.offset);
340         }
341      }
342
343      for (unsigned j = 0; j < MESA_SHADER_STAGES; ++j) {
344         const unsigned n = abs[i].stage_counter_references[j];
345
346         if (n) {
347            atomic_counters[j] += n;
348            total_atomic_counters += n;
349            atomic_buffers[j]++;
350            total_atomic_buffers++;
351         }
352      }
353   }
354
355   /* Check that they are within the supported limits. */
356   for (unsigned i = 0; i < MESA_SHADER_STAGES; i++) {
357      if (atomic_counters[i] > ctx->Const.Program[i].MaxAtomicCounters)
358         linker_error(prog, "Too many %s shader atomic counters",
359                      _mesa_shader_stage_to_string(i));
360
361      if (atomic_buffers[i] > ctx->Const.Program[i].MaxAtomicBuffers)
362         linker_error(prog, "Too many %s shader atomic counter buffers",
363                      _mesa_shader_stage_to_string(i));
364   }
365
366   if (total_atomic_counters > ctx->Const.MaxCombinedAtomicCounters)
367      linker_error(prog, "Too many combined atomic counters");
368
369   if (total_atomic_buffers > ctx->Const.MaxCombinedAtomicBuffers)
370      linker_error(prog, "Too many combined atomic buffers");
371
372   ralloc_free(abs);
373}
374