1/**************************************************************************
2 *
3 * Copyright 2009 VMware, Inc.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28
29/**
30 * @file
31 * Helpers for emiting intrinsic calls.
32 *
33 * LLVM vanilla IR doesn't represent all basic arithmetic operations we care
34 * about, and it is often necessary to resort target-specific intrinsics for
35 * performance, convenience.
36 *
37 * Ideally we would like to stay away from target specific intrinsics and
38 * move all the instruction selection logic into upstream LLVM where it belongs.
39 *
40 * These functions are also used for calling C functions provided by us from
41 * generated LLVM code.
42 *
43 * @author Jose Fonseca <jfonseca@vmware.com>
44 */
45
46
47#include "util/u_debug.h"
48#include "util/u_string.h"
49#include "util/bitscan.h"
50
51#include "lp_bld_const.h"
52#include "lp_bld_intr.h"
53#include "lp_bld_type.h"
54#include "lp_bld_pack.h"
55#include "lp_bld_debug.h"
56
57
58void
59lp_format_intrinsic(char *name,
60                    size_t size,
61                    const char *name_root,
62                    LLVMTypeRef type)
63{
64   unsigned length = 0;
65   unsigned width;
66   char c;
67
68   LLVMTypeKind kind = LLVMGetTypeKind(type);
69   if (kind == LLVMVectorTypeKind) {
70      length = LLVMGetVectorSize(type);
71      type = LLVMGetElementType(type);
72      kind = LLVMGetTypeKind(type);
73   }
74
75   switch (kind) {
76   case LLVMIntegerTypeKind:
77      c = 'i';
78      width = LLVMGetIntTypeWidth(type);
79      break;
80   case LLVMFloatTypeKind:
81      c = 'f';
82      width = 32;
83      break;
84   case LLVMDoubleTypeKind:
85      c = 'f';
86      width = 64;
87      break;
88   default:
89      unreachable("unexpected LLVMTypeKind");
90   }
91
92   if (length) {
93      util_snprintf(name, size, "%s.v%u%c%u", name_root, length, c, width);
94   } else {
95      util_snprintf(name, size, "%s.%c%u", name_root, c, width);
96   }
97}
98
99
100LLVMValueRef
101lp_declare_intrinsic(LLVMModuleRef module,
102                     const char *name,
103                     LLVMTypeRef ret_type,
104                     LLVMTypeRef *arg_types,
105                     unsigned num_args)
106{
107   LLVMTypeRef function_type;
108   LLVMValueRef function;
109
110   assert(!LLVMGetNamedFunction(module, name));
111
112   function_type = LLVMFunctionType(ret_type, arg_types, num_args, 0);
113   function = LLVMAddFunction(module, name, function_type);
114
115   LLVMSetFunctionCallConv(function, LLVMCCallConv);
116   LLVMSetLinkage(function, LLVMExternalLinkage);
117
118   assert(LLVMIsDeclaration(function));
119
120   return function;
121}
122
123
124#if HAVE_LLVM < 0x0400
125static LLVMAttribute lp_attr_to_llvm_attr(enum lp_func_attr attr)
126{
127   switch (attr) {
128   case LP_FUNC_ATTR_ALWAYSINLINE: return LLVMAlwaysInlineAttribute;
129   case LP_FUNC_ATTR_INREG: return LLVMInRegAttribute;
130   case LP_FUNC_ATTR_NOALIAS: return LLVMNoAliasAttribute;
131   case LP_FUNC_ATTR_NOUNWIND: return LLVMNoUnwindAttribute;
132   case LP_FUNC_ATTR_READNONE: return LLVMReadNoneAttribute;
133   case LP_FUNC_ATTR_READONLY: return LLVMReadOnlyAttribute;
134   default:
135      _debug_printf("Unhandled function attribute: %x\n", attr);
136      return 0;
137   }
138}
139
140#else
141
142static const char *attr_to_str(enum lp_func_attr attr)
143{
144   switch (attr) {
145   case LP_FUNC_ATTR_ALWAYSINLINE: return "alwaysinline";
146   case LP_FUNC_ATTR_INREG: return "inreg";
147   case LP_FUNC_ATTR_NOALIAS: return "noalias";
148   case LP_FUNC_ATTR_NOUNWIND: return "nounwind";
149   case LP_FUNC_ATTR_READNONE: return "readnone";
150   case LP_FUNC_ATTR_READONLY: return "readonly";
151   case LP_FUNC_ATTR_WRITEONLY: return "writeonly";
152   case LP_FUNC_ATTR_INACCESSIBLE_MEM_ONLY: return "inaccessiblememonly";
153   case LP_FUNC_ATTR_CONVERGENT: return "convergent";
154   default:
155      _debug_printf("Unhandled function attribute: %x\n", attr);
156      return 0;
157   }
158}
159
160#endif
161
162void
163lp_add_function_attr(LLVMValueRef function_or_call,
164                     int attr_idx, enum lp_func_attr attr)
165{
166
167#if HAVE_LLVM < 0x0400
168   LLVMAttribute llvm_attr = lp_attr_to_llvm_attr(attr);
169   if (LLVMIsAFunction(function_or_call)) {
170      if (attr_idx == -1) {
171         LLVMAddFunctionAttr(function_or_call, llvm_attr);
172      } else {
173         LLVMAddAttribute(LLVMGetParam(function_or_call, attr_idx - 1), llvm_attr);
174      }
175   } else {
176      LLVMAddInstrAttribute(function_or_call, attr_idx, llvm_attr);
177   }
178#else
179
180   LLVMModuleRef module;
181   if (LLVMIsAFunction(function_or_call)) {
182      module = LLVMGetGlobalParent(function_or_call);
183   } else {
184      LLVMBasicBlockRef bb = LLVMGetInstructionParent(function_or_call);
185      LLVMValueRef function = LLVMGetBasicBlockParent(bb);
186      module = LLVMGetGlobalParent(function);
187   }
188   LLVMContextRef ctx = LLVMGetModuleContext(module);
189
190   const char *attr_name = attr_to_str(attr);
191   unsigned kind_id = LLVMGetEnumAttributeKindForName(attr_name,
192                                                      strlen(attr_name));
193   LLVMAttributeRef llvm_attr = LLVMCreateEnumAttribute(ctx, kind_id, 0);
194
195   if (LLVMIsAFunction(function_or_call))
196      LLVMAddAttributeAtIndex(function_or_call, attr_idx, llvm_attr);
197   else
198      LLVMAddCallSiteAttribute(function_or_call, attr_idx, llvm_attr);
199#endif
200}
201
202static void
203lp_add_func_attributes(LLVMValueRef function, unsigned attrib_mask)
204{
205   /* NoUnwind indicates that the intrinsic never raises a C++ exception.
206    * Set it for all intrinsics.
207    */
208   attrib_mask |= LP_FUNC_ATTR_NOUNWIND;
209   attrib_mask &= ~LP_FUNC_ATTR_LEGACY;
210
211   while (attrib_mask) {
212      enum lp_func_attr attr = 1u << u_bit_scan(&attrib_mask);
213      lp_add_function_attr(function, -1, attr);
214   }
215}
216
217LLVMValueRef
218lp_build_intrinsic(LLVMBuilderRef builder,
219                   const char *name,
220                   LLVMTypeRef ret_type,
221                   LLVMValueRef *args,
222                   unsigned num_args,
223                   unsigned attr_mask)
224{
225   LLVMModuleRef module = LLVMGetGlobalParent(LLVMGetBasicBlockParent(LLVMGetInsertBlock(builder)));
226   LLVMValueRef function, call;
227   bool set_callsite_attrs = HAVE_LLVM >= 0x0400 &&
228                             !(attr_mask & LP_FUNC_ATTR_LEGACY);
229
230   function = LLVMGetNamedFunction(module, name);
231   if(!function) {
232      LLVMTypeRef arg_types[LP_MAX_FUNC_ARGS];
233      unsigned i;
234
235      assert(num_args <= LP_MAX_FUNC_ARGS);
236
237      for(i = 0; i < num_args; ++i) {
238         assert(args[i]);
239         arg_types[i] = LLVMTypeOf(args[i]);
240      }
241
242      function = lp_declare_intrinsic(module, name, ret_type, arg_types, num_args);
243
244      /*
245       * If llvm removes an intrinsic we use, we'll hit this abort (rather
246       * than a call to address zero in the jited code).
247       */
248      if (LLVMGetIntrinsicID(function) == 0) {
249         _debug_printf("llvm (version 0x%x) found no intrinsic for %s, going to crash...\n",
250                HAVE_LLVM, name);
251         abort();
252      }
253
254      if (!set_callsite_attrs)
255         lp_add_func_attributes(function, attr_mask);
256
257      if (gallivm_debug & GALLIVM_DEBUG_IR) {
258         lp_debug_dump_value(function);
259      }
260   }
261
262   call = LLVMBuildCall(builder, function, args, num_args, "");
263   if (set_callsite_attrs)
264      lp_add_func_attributes(call, attr_mask);
265   return call;
266}
267
268
269LLVMValueRef
270lp_build_intrinsic_unary(LLVMBuilderRef builder,
271                         const char *name,
272                         LLVMTypeRef ret_type,
273                         LLVMValueRef a)
274{
275   return lp_build_intrinsic(builder, name, ret_type, &a, 1, 0);
276}
277
278
279LLVMValueRef
280lp_build_intrinsic_binary(LLVMBuilderRef builder,
281                          const char *name,
282                          LLVMTypeRef ret_type,
283                          LLVMValueRef a,
284                          LLVMValueRef b)
285{
286   LLVMValueRef args[2];
287
288   args[0] = a;
289   args[1] = b;
290
291   return lp_build_intrinsic(builder, name, ret_type, args, 2, 0);
292}
293
294
295/**
296 * Call intrinsic with arguments adapted to intrinsic vector length.
297 *
298 * Split vectors which are too large for the hw, or expand them if they
299 * are too small, so a caller calling a function which might use intrinsics
300 * doesn't need to do splitting/expansion on its own.
301 * This only supports intrinsics where src and dst types match.
302 */
303LLVMValueRef
304lp_build_intrinsic_binary_anylength(struct gallivm_state *gallivm,
305                                    const char *name,
306                                    struct lp_type src_type,
307                                    unsigned intr_size,
308                                    LLVMValueRef a,
309                                    LLVMValueRef b)
310{
311   unsigned i;
312   struct lp_type intrin_type = src_type;
313   LLVMBuilderRef builder = gallivm->builder;
314   LLVMValueRef i32undef = LLVMGetUndef(LLVMInt32TypeInContext(gallivm->context));
315   LLVMValueRef anative, bnative;
316   unsigned intrin_length = intr_size / src_type.width;
317
318   intrin_type.length = intrin_length;
319
320   if (intrin_length > src_type.length) {
321      LLVMValueRef elems[LP_MAX_VECTOR_LENGTH];
322      LLVMValueRef constvec, tmp;
323
324      for (i = 0; i < src_type.length; i++) {
325         elems[i] = lp_build_const_int32(gallivm, i);
326      }
327      for (; i < intrin_length; i++) {
328         elems[i] = i32undef;
329      }
330      if (src_type.length == 1) {
331         LLVMTypeRef elem_type = lp_build_elem_type(gallivm, intrin_type);
332         a = LLVMBuildBitCast(builder, a, LLVMVectorType(elem_type, 1), "");
333         b = LLVMBuildBitCast(builder, b, LLVMVectorType(elem_type, 1), "");
334      }
335      constvec = LLVMConstVector(elems, intrin_length);
336      anative = LLVMBuildShuffleVector(builder, a, a, constvec, "");
337      bnative = LLVMBuildShuffleVector(builder, b, b, constvec, "");
338      tmp = lp_build_intrinsic_binary(builder, name,
339                                      lp_build_vec_type(gallivm, intrin_type),
340                                      anative, bnative);
341      if (src_type.length > 1) {
342         constvec = LLVMConstVector(elems, src_type.length);
343         return LLVMBuildShuffleVector(builder, tmp, tmp, constvec, "");
344      }
345      else {
346         return LLVMBuildExtractElement(builder, tmp, elems[0], "");
347      }
348   }
349   else if (intrin_length < src_type.length) {
350      unsigned num_vec = src_type.length / intrin_length;
351      LLVMValueRef tmp[LP_MAX_VECTOR_LENGTH];
352
353      /* don't support arbitrary size here as this is so yuck */
354      if (src_type.length % intrin_length) {
355         /* FIXME: This is something which should be supported
356          * but there doesn't seem to be any need for it currently
357          * so crash and burn.
358          */
359         debug_printf("%s: should handle arbitrary vector size\n",
360                      __FUNCTION__);
361         assert(0);
362         return NULL;
363      }
364
365      for (i = 0; i < num_vec; i++) {
366         anative = lp_build_extract_range(gallivm, a, i*intrin_length,
367                                        intrin_length);
368         bnative = lp_build_extract_range(gallivm, b, i*intrin_length,
369                                        intrin_length);
370         tmp[i] = lp_build_intrinsic_binary(builder, name,
371                                            lp_build_vec_type(gallivm, intrin_type),
372                                            anative, bnative);
373      }
374      return lp_build_concat(gallivm, tmp, intrin_type, num_vec);
375   }
376   else {
377      return lp_build_intrinsic_binary(builder, name,
378                                       lp_build_vec_type(gallivm, src_type),
379                                       a, b);
380   }
381}
382
383
384LLVMValueRef
385lp_build_intrinsic_map(struct gallivm_state *gallivm,
386                       const char *name,
387                       LLVMTypeRef ret_type,
388                       LLVMValueRef *args,
389                       unsigned num_args)
390{
391   LLVMBuilderRef builder = gallivm->builder;
392   LLVMTypeRef ret_elem_type = LLVMGetElementType(ret_type);
393   unsigned n = LLVMGetVectorSize(ret_type);
394   unsigned i, j;
395   LLVMValueRef res;
396
397   assert(num_args <= LP_MAX_FUNC_ARGS);
398
399   res = LLVMGetUndef(ret_type);
400   for(i = 0; i < n; ++i) {
401      LLVMValueRef index = lp_build_const_int32(gallivm, i);
402      LLVMValueRef arg_elems[LP_MAX_FUNC_ARGS];
403      LLVMValueRef res_elem;
404      for(j = 0; j < num_args; ++j)
405         arg_elems[j] = LLVMBuildExtractElement(builder, args[j], index, "");
406      res_elem = lp_build_intrinsic(builder, name, ret_elem_type, arg_elems, num_args, 0);
407      res = LLVMBuildInsertElement(builder, res, res_elem, index, "");
408   }
409
410   return res;
411}
412
413
414LLVMValueRef
415lp_build_intrinsic_map_unary(struct gallivm_state *gallivm,
416                             const char *name,
417                             LLVMTypeRef ret_type,
418                             LLVMValueRef a)
419{
420   return lp_build_intrinsic_map(gallivm, name, ret_type, &a, 1);
421}
422
423
424LLVMValueRef
425lp_build_intrinsic_map_binary(struct gallivm_state *gallivm,
426                              const char *name,
427                              LLVMTypeRef ret_type,
428                              LLVMValueRef a,
429                              LLVMValueRef b)
430{
431   LLVMValueRef args[2];
432
433   args[0] = a;
434   args[1] = b;
435
436   return lp_build_intrinsic_map(gallivm, name, ret_type, args, 2);
437}
438
439
440