1/*
2 * Copyright 2017 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 */
23
24#include "glspirv.h"
25#include "errors.h"
26#include "shaderobj.h"
27#include "mtypes.h"
28
29#include "compiler/nir/nir.h"
30#include "compiler/spirv/nir_spirv.h"
31
32#include "program/program.h"
33
34#include "util/u_atomic.h"
35
36void
37_mesa_spirv_module_reference(struct gl_spirv_module **dest,
38                             struct gl_spirv_module *src)
39{
40   struct gl_spirv_module *old = *dest;
41
42   if (old && p_atomic_dec_zero(&old->RefCount))
43      free(old);
44
45   *dest = src;
46
47   if (src)
48      p_atomic_inc(&src->RefCount);
49}
50
51void
52_mesa_shader_spirv_data_reference(struct gl_shader_spirv_data **dest,
53                                  struct gl_shader_spirv_data *src)
54{
55   struct gl_shader_spirv_data *old = *dest;
56
57   if (old && p_atomic_dec_zero(&old->RefCount)) {
58      _mesa_spirv_module_reference(&(*dest)->SpirVModule, NULL);
59      ralloc_free(old);
60   }
61
62   *dest = src;
63
64   if (src)
65      p_atomic_inc(&src->RefCount);
66}
67
68void
69_mesa_spirv_shader_binary(struct gl_context *ctx,
70                          unsigned n, struct gl_shader **shaders,
71                          const void* binary, size_t length)
72{
73   struct gl_spirv_module *module;
74   struct gl_shader_spirv_data *spirv_data;
75
76   module = malloc(sizeof(*module) + length);
77   if (!module) {
78      _mesa_error(ctx, GL_OUT_OF_MEMORY, "glShaderBinary");
79      return;
80   }
81
82   p_atomic_set(&module->RefCount, 0);
83   module->Length = length;
84   memcpy(&module->Binary[0], binary, length);
85
86   for (int i = 0; i < n; ++i) {
87      struct gl_shader *sh = shaders[i];
88
89      spirv_data = rzalloc(NULL, struct gl_shader_spirv_data);
90      _mesa_shader_spirv_data_reference(&sh->spirv_data, spirv_data);
91      _mesa_spirv_module_reference(&spirv_data->SpirVModule, module);
92
93      sh->CompileStatus = COMPILE_FAILURE;
94
95      free((void *)sh->Source);
96      sh->Source = NULL;
97      free((void *)sh->FallbackSource);
98      sh->FallbackSource = NULL;
99
100      ralloc_free(sh->ir);
101      sh->ir = NULL;
102      ralloc_free(sh->symbols);
103      sh->symbols = NULL;
104   }
105}
106
107/**
108 * This is the equivalent to compiler/glsl/linker.cpp::link_shaders()
109 * but for SPIR-V programs.
110 *
111 * This method just creates the gl_linked_shader structs with a reference to
112 * the SPIR-V data collected during previous steps.
113 *
114 * The real linking happens later in the driver-specifc call LinkShader().
115 * This is so backends can implement different linking strategies for
116 * SPIR-V programs.
117 */
118void
119_mesa_spirv_link_shaders(struct gl_context *ctx, struct gl_shader_program *prog)
120{
121   prog->data->LinkStatus = LINKING_SUCCESS;
122   prog->data->Validated = false;
123
124   for (unsigned i = 0; i < prog->NumShaders; i++) {
125      struct gl_shader *shader = prog->Shaders[i];
126      gl_shader_stage shader_type = shader->Stage;
127
128      /* We only support one shader per stage. The gl_spirv spec doesn't seem
129       * to prevent this, but the way the API is designed, requiring all shaders
130       * to be specialized with an entry point, makes supporting this quite
131       * undefined.
132       *
133       * TODO: Turn this into a proper error once the spec bug
134       * <https://gitlab.khronos.org/opengl/API/issues/58> is resolved.
135       */
136      if (prog->_LinkedShaders[shader_type]) {
137         ralloc_strcat(&prog->data->InfoLog,
138                       "\nError trying to link more than one SPIR-V shader "
139                       "per stage.\n");
140         prog->data->LinkStatus = LINKING_FAILURE;
141         return;
142      }
143
144      assert(shader->spirv_data);
145
146      struct gl_linked_shader *linked = rzalloc(NULL, struct gl_linked_shader);
147      linked->Stage = shader_type;
148
149      /* Create program and attach it to the linked shader */
150      struct gl_program *gl_prog =
151         ctx->Driver.NewProgram(ctx,
152                                _mesa_shader_stage_to_program(shader_type),
153                                prog->Name, false);
154      if (!gl_prog) {
155         prog->data->LinkStatus = LINKING_FAILURE;
156         _mesa_delete_linked_shader(ctx, linked);
157         return;
158      }
159
160      _mesa_reference_shader_program_data(ctx,
161                                          &gl_prog->sh.data,
162                                          prog->data);
163
164      /* Don't use _mesa_reference_program() just take ownership */
165      linked->Program = gl_prog;
166
167      /* Reference the SPIR-V data from shader to the linked shader */
168      _mesa_shader_spirv_data_reference(&linked->spirv_data,
169                                        shader->spirv_data);
170
171      prog->_LinkedShaders[shader_type] = linked;
172      prog->data->linked_stages |= 1 << shader_type;
173   }
174
175   int last_vert_stage =
176      util_last_bit(prog->data->linked_stages &
177                    ((1 << (MESA_SHADER_GEOMETRY + 1)) - 1));
178
179   if (last_vert_stage)
180      prog->last_vert_prog = prog->_LinkedShaders[last_vert_stage - 1]->Program;
181}
182
183nir_shader *
184_mesa_spirv_to_nir(struct gl_context *ctx,
185                   const struct gl_shader_program *prog,
186                   gl_shader_stage stage,
187                   const nir_shader_compiler_options *options)
188{
189   nir_shader *nir = NULL;
190
191   struct gl_linked_shader *linked_shader = prog->_LinkedShaders[stage];
192   assert (linked_shader);
193
194   struct gl_shader_spirv_data *spirv_data = linked_shader->spirv_data;
195   assert(spirv_data);
196
197   struct gl_spirv_module *spirv_module = spirv_data->SpirVModule;
198   assert (spirv_module != NULL);
199
200   const char *entry_point_name = spirv_data->SpirVEntryPoint;
201   assert(entry_point_name);
202
203   struct nir_spirv_specialization *spec_entries =
204      calloc(sizeof(*spec_entries),
205             spirv_data->NumSpecializationConstants);
206
207   for (unsigned i = 0; i < spirv_data->NumSpecializationConstants; ++i) {
208      spec_entries[i].id = spirv_data->SpecializationConstantsIndex[i];
209      spec_entries[i].data32 = spirv_data->SpecializationConstantsValue[i];
210      spec_entries[i].defined_on_module = false;
211   }
212
213   const struct spirv_to_nir_options spirv_options = {
214      .environment = NIR_SPIRV_OPENGL,
215      .lower_workgroup_access_to_offsets = true,
216      .lower_ubo_ssbo_access_to_offsets = true,
217      .caps = ctx->Const.SpirVCapabilities
218   };
219
220   nir_function *entry_point =
221      spirv_to_nir((const uint32_t *) &spirv_module->Binary[0],
222                   spirv_module->Length / 4,
223                   spec_entries, spirv_data->NumSpecializationConstants,
224                   stage, entry_point_name,
225                   &spirv_options,
226                   options);
227   free(spec_entries);
228
229   assert (entry_point);
230   nir = entry_point->shader;
231   assert(nir->info.stage == stage);
232
233   nir->options = options;
234
235   nir->info.name =
236      ralloc_asprintf(nir, "SPIRV:%s:%d",
237                      _mesa_shader_stage_to_abbrev(nir->info.stage),
238                      prog->Name);
239   nir_validate_shader(nir, "after spirv_to_nir");
240
241   nir->info.separate_shader = linked_shader->Program->info.separate_shader;
242
243   /* We have to lower away local constant initializers right before we
244    * inline functions.  That way they get properly initialized at the top
245    * of the function and not at the top of its caller.
246    */
247   NIR_PASS_V(nir, nir_lower_constant_initializers, nir_var_function_temp);
248   NIR_PASS_V(nir, nir_lower_returns);
249   NIR_PASS_V(nir, nir_inline_functions);
250   NIR_PASS_V(nir, nir_opt_deref);
251
252   /* Pick off the single entrypoint that we want */
253   foreach_list_typed_safe(nir_function, func, node, &nir->functions) {
254      if (func != entry_point)
255         exec_node_remove(&func->node);
256   }
257   assert(exec_list_length(&nir->functions) == 1);
258
259   /* Split member structs.  We do this before lower_io_to_temporaries so that
260    * it doesn't lower system values to temporaries by accident.
261    */
262   NIR_PASS_V(nir, nir_split_var_copies);
263   NIR_PASS_V(nir, nir_split_per_member_structs);
264
265   if (nir->info.stage == MESA_SHADER_VERTEX)
266      nir_remap_dual_slot_attributes(nir, &linked_shader->Program->DualSlotInputs);
267
268   return nir;
269}
270
271void GLAPIENTRY
272_mesa_SpecializeShaderARB(GLuint shader,
273                          const GLchar *pEntryPoint,
274                          GLuint numSpecializationConstants,
275                          const GLuint *pConstantIndex,
276                          const GLuint *pConstantValue)
277{
278   GET_CURRENT_CONTEXT(ctx);
279   struct gl_shader *sh;
280   bool has_entry_point;
281   struct nir_spirv_specialization *spec_entries = NULL;
282
283   if (!ctx->Extensions.ARB_gl_spirv) {
284      _mesa_error(ctx, GL_INVALID_OPERATION, "glSpecializeShaderARB");
285      return;
286   }
287
288   sh = _mesa_lookup_shader_err(ctx, shader, "glSpecializeShaderARB");
289   if (!sh)
290      return;
291
292   if (!sh->spirv_data) {
293      _mesa_error(ctx, GL_INVALID_OPERATION,
294                  "glSpecializeShaderARB(not SPIR-V)");
295      return;
296   }
297
298   if (sh->CompileStatus) {
299      _mesa_error(ctx, GL_INVALID_OPERATION,
300                  "glSpecializeShaderARB(already specialized)");
301      return;
302   }
303
304   struct gl_shader_spirv_data *spirv_data = sh->spirv_data;
305
306   /* From the GL_ARB_gl_spirv spec:
307    *
308    *    "The OpenGL API expects the SPIR-V module to have already been
309    *     validated, and can return an error if it discovers anything invalid
310    *     in the module. An invalid SPIR-V module is allowed to result in
311    *     undefined behavior."
312    *
313    * However, the following errors still need to be detected (from the same
314    * spec):
315    *
316    *    "INVALID_VALUE is generated if <pEntryPoint> does not name a valid
317    *     entry point for <shader>.
318    *
319    *     INVALID_VALUE is generated if any element of <pConstantIndex>
320    *     refers to a specialization constant that does not exist in the
321    *     shader module contained in <shader>."
322    *
323    * We cannot flag those errors a-priori because detecting them requires
324    * parsing the module. However, flagging them during specialization is okay,
325    * since it makes no difference in terms of application-visible state.
326    */
327   spec_entries = calloc(sizeof(*spec_entries), numSpecializationConstants);
328
329   for (unsigned i = 0; i < numSpecializationConstants; ++i) {
330      spec_entries[i].id = pConstantIndex[i];
331      spec_entries[i].data32 = pConstantValue[i];
332      spec_entries[i].defined_on_module = false;
333   }
334
335   has_entry_point =
336      gl_spirv_validation((uint32_t *)&spirv_data->SpirVModule->Binary[0],
337                          spirv_data->SpirVModule->Length / 4,
338                          spec_entries, numSpecializationConstants,
339                          sh->Stage, pEntryPoint);
340
341   /* See previous spec comment */
342   if (!has_entry_point) {
343      _mesa_error(ctx, GL_INVALID_VALUE,
344                  "glSpecializeShaderARB(\"%s\" is not a valid entry point"
345                  " for shader)", pEntryPoint);
346      goto end;
347   }
348
349   for (unsigned i = 0; i < numSpecializationConstants; ++i) {
350      if (spec_entries[i].defined_on_module == false) {
351         _mesa_error(ctx, GL_INVALID_VALUE,
352                     "glSpecializeShaderARB(constant \"%i\" does not exist "
353                     "in shader)", spec_entries[i].id);
354         goto end;
355      }
356   }
357
358   spirv_data->SpirVEntryPoint = ralloc_strdup(spirv_data, pEntryPoint);
359
360   /* Note that we didn't make a real compilation of the module (spirv_to_nir),
361    * but just checked some error conditions. Real "compilation" will be done
362    * later, upon linking.
363    */
364   sh->CompileStatus = COMPILE_SUCCESS;
365
366   spirv_data->NumSpecializationConstants = numSpecializationConstants;
367   spirv_data->SpecializationConstantsIndex =
368      rzalloc_array_size(spirv_data, sizeof(GLuint),
369                         numSpecializationConstants);
370   spirv_data->SpecializationConstantsValue =
371      rzalloc_array_size(spirv_data, sizeof(GLuint),
372                         numSpecializationConstants);
373   for (unsigned i = 0; i < numSpecializationConstants; ++i) {
374      spirv_data->SpecializationConstantsIndex[i] = pConstantIndex[i];
375      spirv_data->SpecializationConstantsValue[i] = pConstantValue[i];
376   }
377
378 end:
379   free(spec_entries);
380}
381