1/************************************************************************** 2 * 3 * Copyright 2009 VMware, Inc. 4 * Copyright 2007 VMware, Inc. 5 * All Rights Reserved. 6 * 7 * Permission is hereby granted, free of charge, to any person obtaining a 8 * copy of this software and associated documentation files (the 9 * "Software"), to deal in the Software without restriction, including 10 * without limitation the rights to use, copy, modify, merge, publish, 11 * distribute, sub license, and/or sell copies of the Software, and to 12 * permit persons to whom the Software is furnished to do so, subject to 13 * the following conditions: 14 * 15 * The above copyright notice and this permission notice (including the 16 * next paragraph) shall be included in all copies or substantial portions 17 * of the Software. 18 * 19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 20 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 21 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. 22 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR 23 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 24 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 25 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 26 * 27 **************************************************************************/ 28 29/** 30 * @file 31 * Code generate the whole fragment pipeline. 32 * 33 * The fragment pipeline consists of the following stages: 34 * - early depth test 35 * - fragment shader 36 * - alpha test 37 * - depth/stencil test 38 * - blending 39 * 40 * This file has only the glue to assemble the fragment pipeline. The actual 41 * plumbing of converting Gallium state into LLVM IR is done elsewhere, in the 42 * lp_bld_*.[ch] files, and in a complete generic and reusable way. Here we 43 * muster the LLVM JIT execution engine to create a function that follows an 44 * established binary interface and that can be called from C directly. 45 * 46 * A big source of complexity here is that we often want to run different 47 * stages with different precisions and data types and precisions. For example, 48 * the fragment shader needs typically to be done in floats, but the 49 * depth/stencil test and blending is better done in the type that most closely 50 * matches the depth/stencil and color buffer respectively. 51 * 52 * Since the width of a SIMD vector register stays the same regardless of the 53 * element type, different types imply different number of elements, so we must 54 * code generate more instances of the stages with larger types to be able to 55 * feed/consume the stages with smaller types. 56 * 57 * @author Jose Fonseca <jfonseca@vmware.com> 58 */ 59 60#include <limits.h> 61#include "pipe/p_defines.h" 62#include "util/u_inlines.h" 63#include "util/u_memory.h" 64#include "util/u_pointer.h" 65#include "util/format/u_format.h" 66#include "util/u_dump.h" 67#include "util/u_string.h" 68#include "util/simple_list.h" 69#include "util/u_dual_blend.h" 70#include "util/u_upload_mgr.h" 71#include "util/os_time.h" 72#include "pipe/p_shader_tokens.h" 73#include "draw/draw_context.h" 74#include "tgsi/tgsi_dump.h" 75#include "tgsi/tgsi_scan.h" 76#include "tgsi/tgsi_parse.h" 77#include "gallivm/lp_bld_type.h" 78#include "gallivm/lp_bld_const.h" 79#include "gallivm/lp_bld_conv.h" 80#include "gallivm/lp_bld_init.h" 81#include "gallivm/lp_bld_intr.h" 82#include "gallivm/lp_bld_logic.h" 83#include "gallivm/lp_bld_tgsi.h" 84#include "gallivm/lp_bld_nir.h" 85#include "gallivm/lp_bld_swizzle.h" 86#include "gallivm/lp_bld_flow.h" 87#include "gallivm/lp_bld_debug.h" 88#include "gallivm/lp_bld_arit.h" 89#include "gallivm/lp_bld_bitarit.h" 90#include "gallivm/lp_bld_pack.h" 91#include "gallivm/lp_bld_format.h" 92#include "gallivm/lp_bld_quad.h" 93#include "gallivm/lp_bld_gather.h" 94 95#include "lp_bld_alpha.h" 96#include "lp_bld_blend.h" 97#include "lp_bld_depth.h" 98#include "lp_bld_interp.h" 99#include "lp_context.h" 100#include "lp_debug.h" 101#include "lp_perf.h" 102#include "lp_setup.h" 103#include "lp_state.h" 104#include "lp_tex_sample.h" 105#include "lp_flush.h" 106#include "lp_state_fs.h" 107#include "lp_rast.h" 108#include "nir/nir_to_tgsi_info.h" 109 110#include "lp_screen.h" 111#include "compiler/nir/nir_serialize.h" 112#include "util/mesa-sha1.h" 113/** Fragment shader number (for debugging) */ 114static unsigned fs_no = 0; 115 116static void 117load_unswizzled_block(struct gallivm_state *gallivm, 118 LLVMValueRef base_ptr, 119 LLVMValueRef stride, 120 unsigned block_width, 121 unsigned block_height, 122 LLVMValueRef* dst, 123 struct lp_type dst_type, 124 unsigned dst_count, 125 unsigned dst_alignment, 126 LLVMValueRef x_offset, 127 LLVMValueRef y_offset, 128 bool fb_fetch_twiddle); 129/** 130 * Checks if a format description is an arithmetic format 131 * 132 * A format which has irregular channel sizes such as R3_G3_B2 or R5_G6_B5. 133 */ 134static inline boolean 135is_arithmetic_format(const struct util_format_description *format_desc) 136{ 137 boolean arith = false; 138 unsigned i; 139 140 for (i = 0; i < format_desc->nr_channels; ++i) { 141 arith |= format_desc->channel[i].size != format_desc->channel[0].size; 142 arith |= (format_desc->channel[i].size % 8) != 0; 143 } 144 145 return arith; 146} 147 148/** 149 * Checks if this format requires special handling due to required expansion 150 * to floats for blending, and furthermore has "natural" packed AoS -> unpacked 151 * SoA conversion. 152 */ 153static inline boolean 154format_expands_to_float_soa(const struct util_format_description *format_desc) 155{ 156 if (format_desc->format == PIPE_FORMAT_R11G11B10_FLOAT || 157 format_desc->colorspace == UTIL_FORMAT_COLORSPACE_SRGB) { 158 return true; 159 } 160 return false; 161} 162 163 164/** 165 * Retrieves the type representing the memory layout for a format 166 * 167 * e.g. RGBA16F = 4x half-float and R3G3B2 = 1x byte 168 */ 169static inline void 170lp_mem_type_from_format_desc(const struct util_format_description *format_desc, 171 struct lp_type* type) 172{ 173 unsigned i; 174 unsigned chan; 175 176 if (format_expands_to_float_soa(format_desc)) { 177 /* just make this a uint with width of block */ 178 type->floating = false; 179 type->fixed = false; 180 type->sign = false; 181 type->norm = false; 182 type->width = format_desc->block.bits; 183 type->length = 1; 184 return; 185 } 186 187 for (i = 0; i < 4; i++) 188 if (format_desc->channel[i].type != UTIL_FORMAT_TYPE_VOID) 189 break; 190 chan = i; 191 192 memset(type, 0, sizeof(struct lp_type)); 193 type->floating = format_desc->channel[chan].type == UTIL_FORMAT_TYPE_FLOAT; 194 type->fixed = format_desc->channel[chan].type == UTIL_FORMAT_TYPE_FIXED; 195 type->sign = format_desc->channel[chan].type != UTIL_FORMAT_TYPE_UNSIGNED; 196 type->norm = format_desc->channel[chan].normalized; 197 198 if (is_arithmetic_format(format_desc)) { 199 type->width = 0; 200 type->length = 1; 201 202 for (i = 0; i < format_desc->nr_channels; ++i) { 203 type->width += format_desc->channel[i].size; 204 } 205 } else { 206 type->width = format_desc->channel[chan].size; 207 type->length = format_desc->nr_channels; 208 } 209} 210 211/** 212 * Expand the relevant bits of mask_input to a n*4-dword mask for the 213 * n*four pixels in n 2x2 quads. This will set the n*four elements of the 214 * quad mask vector to 0 or ~0. 215 * Grouping is 01, 23 for 2 quad mode hence only 0 and 2 are valid 216 * quad arguments with fs length 8. 217 * 218 * \param first_quad which quad(s) of the quad group to test, in [0,3] 219 * \param mask_input bitwise mask for the whole 4x4 stamp 220 */ 221static LLVMValueRef 222generate_quad_mask(struct gallivm_state *gallivm, 223 struct lp_type fs_type, 224 unsigned first_quad, 225 unsigned sample, 226 LLVMValueRef mask_input) /* int64 */ 227{ 228 LLVMBuilderRef builder = gallivm->builder; 229 struct lp_type mask_type; 230 LLVMTypeRef i32t = LLVMInt32TypeInContext(gallivm->context); 231 LLVMValueRef bits[16]; 232 LLVMValueRef mask, bits_vec; 233 int shift, i; 234 235 /* 236 * XXX: We'll need a different path for 16 x u8 237 */ 238 assert(fs_type.width == 32); 239 assert(fs_type.length <= ARRAY_SIZE(bits)); 240 mask_type = lp_int_type(fs_type); 241 242 /* 243 * mask_input >>= (quad * 4) 244 */ 245 switch (first_quad) { 246 case 0: 247 shift = 0; 248 break; 249 case 1: 250 assert(fs_type.length == 4); 251 shift = 2; 252 break; 253 case 2: 254 shift = 8; 255 break; 256 case 3: 257 assert(fs_type.length == 4); 258 shift = 10; 259 break; 260 default: 261 assert(0); 262 shift = 0; 263 } 264 265 mask_input = LLVMBuildLShr(builder, mask_input, lp_build_const_int64(gallivm, 16 * sample), ""); 266 mask_input = LLVMBuildTrunc(builder, mask_input, 267 i32t, ""); 268 mask_input = LLVMBuildAnd(builder, mask_input, lp_build_const_int32(gallivm, 0xffff), ""); 269 270 mask_input = LLVMBuildLShr(builder, 271 mask_input, 272 LLVMConstInt(i32t, shift, 0), 273 ""); 274 275 /* 276 * mask = { mask_input & (1 << i), for i in [0,3] } 277 */ 278 mask = lp_build_broadcast(gallivm, 279 lp_build_vec_type(gallivm, mask_type), 280 mask_input); 281 282 for (i = 0; i < fs_type.length / 4; i++) { 283 unsigned j = 2 * (i % 2) + (i / 2) * 8; 284 bits[4*i + 0] = LLVMConstInt(i32t, 1ULL << (j + 0), 0); 285 bits[4*i + 1] = LLVMConstInt(i32t, 1ULL << (j + 1), 0); 286 bits[4*i + 2] = LLVMConstInt(i32t, 1ULL << (j + 4), 0); 287 bits[4*i + 3] = LLVMConstInt(i32t, 1ULL << (j + 5), 0); 288 } 289 bits_vec = LLVMConstVector(bits, fs_type.length); 290 mask = LLVMBuildAnd(builder, mask, bits_vec, ""); 291 292 /* 293 * mask = mask == bits ? ~0 : 0 294 */ 295 mask = lp_build_compare(gallivm, 296 mask_type, PIPE_FUNC_EQUAL, 297 mask, bits_vec); 298 299 return mask; 300} 301 302 303#define EARLY_DEPTH_TEST 0x1 304#define LATE_DEPTH_TEST 0x2 305#define EARLY_DEPTH_WRITE 0x4 306#define LATE_DEPTH_WRITE 0x8 307 308static int 309find_output_by_semantic( const struct tgsi_shader_info *info, 310 unsigned semantic, 311 unsigned index ) 312{ 313 int i; 314 315 for (i = 0; i < info->num_outputs; i++) 316 if (info->output_semantic_name[i] == semantic && 317 info->output_semantic_index[i] == index) 318 return i; 319 320 return -1; 321} 322 323 324/** 325 * Fetch the specified lp_jit_viewport structure for a given viewport_index. 326 */ 327static LLVMValueRef 328lp_llvm_viewport(LLVMValueRef context_ptr, 329 struct gallivm_state *gallivm, 330 LLVMValueRef viewport_index) 331{ 332 LLVMBuilderRef builder = gallivm->builder; 333 LLVMValueRef ptr; 334 LLVMValueRef res; 335 struct lp_type viewport_type = 336 lp_type_float_vec(32, 32 * LP_JIT_VIEWPORT_NUM_FIELDS); 337 338 ptr = lp_jit_context_viewports(gallivm, context_ptr); 339 ptr = LLVMBuildPointerCast(builder, ptr, 340 LLVMPointerType(lp_build_vec_type(gallivm, viewport_type), 0), ""); 341 342 res = lp_build_pointer_get(builder, ptr, viewport_index); 343 344 return res; 345} 346 347 348static LLVMValueRef 349lp_build_depth_clamp(struct gallivm_state *gallivm, 350 LLVMBuilderRef builder, 351 struct lp_type type, 352 LLVMValueRef context_ptr, 353 LLVMValueRef thread_data_ptr, 354 LLVMValueRef z) 355{ 356 LLVMValueRef viewport, min_depth, max_depth; 357 LLVMValueRef viewport_index; 358 struct lp_build_context f32_bld; 359 360 assert(type.floating); 361 lp_build_context_init(&f32_bld, gallivm, type); 362 363 /* 364 * Assumes clamping of the viewport index will occur in setup/gs. Value 365 * is passed through the rasterization stage via lp_rast_shader_inputs. 366 * 367 * See: draw_clamp_viewport_idx and lp_clamp_viewport_idx for clamping 368 * semantics. 369 */ 370 viewport_index = lp_jit_thread_data_raster_state_viewport_index(gallivm, 371 thread_data_ptr); 372 373 /* 374 * Load the min and max depth from the lp_jit_context.viewports 375 * array of lp_jit_viewport structures. 376 */ 377 viewport = lp_llvm_viewport(context_ptr, gallivm, viewport_index); 378 379 /* viewports[viewport_index].min_depth */ 380 min_depth = LLVMBuildExtractElement(builder, viewport, 381 lp_build_const_int32(gallivm, LP_JIT_VIEWPORT_MIN_DEPTH), ""); 382 min_depth = lp_build_broadcast_scalar(&f32_bld, min_depth); 383 384 /* viewports[viewport_index].max_depth */ 385 max_depth = LLVMBuildExtractElement(builder, viewport, 386 lp_build_const_int32(gallivm, LP_JIT_VIEWPORT_MAX_DEPTH), ""); 387 max_depth = lp_build_broadcast_scalar(&f32_bld, max_depth); 388 389 /* 390 * Clamp to the min and max depth values for the given viewport. 391 */ 392 return lp_build_clamp(&f32_bld, z, min_depth, max_depth); 393} 394 395static void 396lp_build_sample_alpha_to_coverage(struct gallivm_state *gallivm, 397 struct lp_type type, 398 unsigned coverage_samples, 399 LLVMValueRef num_loop, 400 LLVMValueRef loop_counter, 401 LLVMValueRef coverage_mask_store, 402 LLVMValueRef alpha) 403{ 404 struct lp_build_context bld; 405 LLVMBuilderRef builder = gallivm->builder; 406 float step = 1.0 / coverage_samples; 407 408 lp_build_context_init(&bld, gallivm, type); 409 for (unsigned s = 0; s < coverage_samples; s++) { 410 LLVMValueRef alpha_ref_value = lp_build_const_vec(gallivm, type, step * s); 411 LLVMValueRef test = lp_build_cmp(&bld, PIPE_FUNC_GREATER, alpha, alpha_ref_value); 412 413 LLVMValueRef s_mask_idx = LLVMBuildMul(builder, lp_build_const_int32(gallivm, s), num_loop, ""); 414 s_mask_idx = LLVMBuildAdd(builder, s_mask_idx, loop_counter, ""); 415 LLVMValueRef s_mask_ptr = LLVMBuildGEP(builder, coverage_mask_store, &s_mask_idx, 1, ""); 416 LLVMValueRef s_mask = LLVMBuildLoad(builder, s_mask_ptr, ""); 417 s_mask = LLVMBuildAnd(builder, s_mask, test, ""); 418 LLVMBuildStore(builder, s_mask, s_mask_ptr); 419 } 420}; 421 422struct lp_build_fs_llvm_iface { 423 struct lp_build_fs_iface base; 424 struct lp_build_interp_soa_context *interp; 425 struct lp_build_for_loop_state *loop_state; 426 LLVMValueRef mask_store; 427 LLVMValueRef sample_id; 428 LLVMValueRef color_ptr_ptr; 429 LLVMValueRef color_stride_ptr; 430 LLVMValueRef color_sample_stride_ptr; 431 const struct lp_fragment_shader_variant_key *key; 432}; 433 434static LLVMValueRef fs_interp(const struct lp_build_fs_iface *iface, 435 struct lp_build_context *bld, 436 unsigned attrib, unsigned chan, 437 bool centroid, bool sample, 438 LLVMValueRef attrib_indir, 439 LLVMValueRef offsets[2]) 440{ 441 struct lp_build_fs_llvm_iface *fs_iface = (struct lp_build_fs_llvm_iface *)iface; 442 struct lp_build_interp_soa_context *interp = fs_iface->interp; 443 unsigned loc = TGSI_INTERPOLATE_LOC_CENTER; 444 if (centroid) 445 loc = TGSI_INTERPOLATE_LOC_CENTROID; 446 if (sample) 447 loc = TGSI_INTERPOLATE_LOC_SAMPLE; 448 449 return lp_build_interp_soa(interp, bld->gallivm, fs_iface->loop_state->counter, 450 fs_iface->mask_store, 451 attrib, chan, loc, attrib_indir, offsets); 452} 453 454static void fs_fb_fetch(const struct lp_build_fs_iface *iface, 455 struct lp_build_context *bld, 456 int location, 457 LLVMValueRef result[4]) 458{ 459 assert(location >= FRAG_RESULT_DATA0 && location <= FRAG_RESULT_DATA7); 460 const int cbuf = location - FRAG_RESULT_DATA0; 461 462 struct lp_build_fs_llvm_iface *fs_iface = (struct lp_build_fs_llvm_iface *)iface; 463 struct gallivm_state *gallivm = bld->gallivm; 464 LLVMBuilderRef builder = gallivm->builder; 465 const struct lp_fragment_shader_variant_key *key = fs_iface->key; 466 LLVMValueRef index = lp_build_const_int32(gallivm, cbuf); 467 LLVMValueRef color_ptr = LLVMBuildLoad(builder, LLVMBuildGEP(builder, fs_iface->color_ptr_ptr, &index, 1, ""), ""); 468 LLVMValueRef stride = LLVMBuildLoad(builder, LLVMBuildGEP(builder, fs_iface->color_stride_ptr, &index, 1, ""), ""); 469 470 LLVMValueRef dst[4 * 4]; 471 enum pipe_format cbuf_format = key->cbuf_format[cbuf]; 472 const struct util_format_description* out_format_desc = util_format_description(cbuf_format); 473 struct lp_type dst_type; 474 unsigned block_size = bld->type.length; 475 unsigned block_height = key->resource_1d ? 1 : 2; 476 unsigned block_width = block_size / block_height; 477 478 lp_mem_type_from_format_desc(out_format_desc, &dst_type); 479 480 struct lp_type blend_type; 481 memset(&blend_type, 0, sizeof blend_type); 482 blend_type.floating = FALSE; /* values are integers */ 483 blend_type.sign = FALSE; /* values are unsigned */ 484 blend_type.norm = TRUE; /* values are in [0,1] or [-1,1] */ 485 blend_type.width = 8; /* 8-bit ubyte values */ 486 blend_type.length = 16; /* 16 elements per vector */ 487 488 uint32_t dst_alignment; 489 /* 490 * Compute the alignment of the destination pointer in bytes 491 * We fetch 1-4 pixels, if the format has pot alignment then those fetches 492 * are always aligned by MIN2(16, fetch_width) except for buffers (not 493 * 1d tex but can't distinguish here) so need to stick with per-pixel 494 * alignment in this case. 495 */ 496 if (key->resource_1d) { 497 dst_alignment = (out_format_desc->block.bits + 7)/(out_format_desc->block.width * 8); 498 } 499 else { 500 dst_alignment = dst_type.length * dst_type.width / 8; 501 } 502 /* Force power-of-two alignment by extracting only the least-significant-bit */ 503 dst_alignment = 1 << (ffs(dst_alignment) - 1); 504 /* 505 * Resource base and stride pointers are aligned to 16 bytes, so that's 506 * the maximum alignment we can guarantee 507 */ 508 dst_alignment = MIN2(16, dst_alignment); 509 510 LLVMTypeRef blend_vec_type = lp_build_vec_type(gallivm, blend_type); 511 color_ptr = LLVMBuildBitCast(builder, color_ptr, LLVMPointerType(blend_vec_type, 0), ""); 512 513 if (key->multisample) { 514 LLVMValueRef sample_stride = LLVMBuildLoad(builder, 515 LLVMBuildGEP(builder, fs_iface->color_sample_stride_ptr, 516 &index, 1, ""), ""); 517 LLVMValueRef sample_offset = LLVMBuildMul(builder, sample_stride, fs_iface->sample_id, ""); 518 color_ptr = LLVMBuildGEP(builder, color_ptr, &sample_offset, 1, ""); 519 } 520 /* fragment shader executes on 4x4 blocks. depending on vector width it can execute 2 or 4 iterations. 521 * only move to the next row once the top row has completed 8 wide 1 iteration, 4 wide 2 iterations */ 522 LLVMValueRef x_offset = NULL, y_offset = NULL; 523 if (!key->resource_1d) { 524 LLVMValueRef counter = fs_iface->loop_state->counter; 525 526 if (block_size == 4) { 527 x_offset = LLVMBuildShl(builder, 528 LLVMBuildAnd(builder, fs_iface->loop_state->counter, lp_build_const_int32(gallivm, 1), ""), 529 lp_build_const_int32(gallivm, 1), ""); 530 counter = LLVMBuildLShr(builder, fs_iface->loop_state->counter, lp_build_const_int32(gallivm, 1), ""); 531 } 532 y_offset = LLVMBuildMul(builder, counter, lp_build_const_int32(gallivm, 2), ""); 533 } 534 load_unswizzled_block(gallivm, color_ptr, stride, block_width, block_height, dst, dst_type, block_size, dst_alignment, x_offset, y_offset, true); 535 536 for (unsigned i = 0; i < block_size; i++) { 537 dst[i] = LLVMBuildBitCast(builder, dst[i], LLVMInt32TypeInContext(gallivm->context), ""); 538 } 539 LLVMValueRef packed = lp_build_gather_values(gallivm, dst, block_size); 540 541 struct lp_type texel_type = bld->type; 542 if (out_format_desc->colorspace == UTIL_FORMAT_COLORSPACE_RGB && 543 out_format_desc->channel[0].pure_integer) { 544 if (out_format_desc->channel[0].type == UTIL_FORMAT_TYPE_SIGNED) { 545 texel_type = lp_type_int_vec(bld->type.width, bld->type.width * bld->type.length); 546 } 547 else if (out_format_desc->channel[0].type == UTIL_FORMAT_TYPE_UNSIGNED) { 548 texel_type = lp_type_uint_vec(bld->type.width, bld->type.width * bld->type.length); 549 } 550 } 551 lp_build_unpack_rgba_soa(gallivm, out_format_desc, 552 texel_type, 553 packed, result); 554} 555 556/** 557 * Generate the fragment shader, depth/stencil test, and alpha tests. 558 */ 559static void 560generate_fs_loop(struct gallivm_state *gallivm, 561 struct lp_fragment_shader *shader, 562 const struct lp_fragment_shader_variant_key *key, 563 LLVMBuilderRef builder, 564 struct lp_type type, 565 LLVMValueRef context_ptr, 566 LLVMValueRef sample_pos_array, 567 LLVMValueRef num_loop, 568 struct lp_build_interp_soa_context *interp, 569 const struct lp_build_sampler_soa *sampler, 570 const struct lp_build_image_soa *image, 571 LLVMValueRef mask_store, 572 LLVMValueRef (*out_color)[4], 573 LLVMValueRef depth_base_ptr, 574 LLVMValueRef depth_stride, 575 LLVMValueRef depth_sample_stride, 576 LLVMValueRef color_ptr_ptr, 577 LLVMValueRef color_stride_ptr, 578 LLVMValueRef color_sample_stride_ptr, 579 LLVMValueRef facing, 580 LLVMValueRef thread_data_ptr) 581{ 582 const struct util_format_description *zs_format_desc = NULL; 583 const struct tgsi_token *tokens = shader->base.tokens; 584 struct lp_type int_type = lp_int_type(type); 585 LLVMTypeRef vec_type, int_vec_type; 586 LLVMValueRef mask_ptr = NULL, mask_val = NULL; 587 LLVMValueRef consts_ptr, num_consts_ptr; 588 LLVMValueRef ssbo_ptr, num_ssbo_ptr; 589 LLVMValueRef z; 590 LLVMValueRef z_value, s_value; 591 LLVMValueRef z_fb, s_fb; 592 LLVMValueRef depth_ptr; 593 LLVMValueRef stencil_refs[2]; 594 LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][TGSI_NUM_CHANNELS]; 595 LLVMValueRef zs_samples = lp_build_const_int32(gallivm, key->zsbuf_nr_samples); 596 LLVMValueRef z_out = NULL, s_out = NULL; 597 struct lp_build_for_loop_state loop_state, sample_loop_state = {0}; 598 struct lp_build_mask_context mask; 599 /* 600 * TODO: figure out if simple_shader optimization is really worthwile to 601 * keep. Disabled because it may hide some real bugs in the (depth/stencil) 602 * code since tests tend to take another codepath than real shaders. 603 */ 604 boolean simple_shader = (shader->info.base.file_count[TGSI_FILE_SAMPLER] == 0 && 605 shader->info.base.num_inputs < 3 && 606 shader->info.base.num_instructions < 8) && 0; 607 const boolean dual_source_blend = key->blend.rt[0].blend_enable && 608 util_blend_state_is_dual(&key->blend, 0); 609 const bool post_depth_coverage = shader->info.base.properties[TGSI_PROPERTY_FS_POST_DEPTH_COVERAGE]; 610 unsigned attrib; 611 unsigned chan; 612 unsigned cbuf; 613 unsigned depth_mode; 614 615 struct lp_bld_tgsi_system_values system_values; 616 617 memset(&system_values, 0, sizeof(system_values)); 618 619 /* truncate then sign extend. */ 620 system_values.front_facing = LLVMBuildTrunc(gallivm->builder, facing, LLVMInt1TypeInContext(gallivm->context), ""); 621 system_values.front_facing = LLVMBuildSExt(gallivm->builder, system_values.front_facing, LLVMInt32TypeInContext(gallivm->context), ""); 622 system_values.view_index = lp_jit_thread_data_raster_state_view_index(gallivm, 623 thread_data_ptr); 624 if (key->depth.enabled || 625 key->stencil[0].enabled) { 626 627 zs_format_desc = util_format_description(key->zsbuf_format); 628 assert(zs_format_desc); 629 630 if (shader->info.base.properties[TGSI_PROPERTY_FS_EARLY_DEPTH_STENCIL]) 631 depth_mode = EARLY_DEPTH_TEST | EARLY_DEPTH_WRITE; 632 else if (!shader->info.base.writes_z && !shader->info.base.writes_stencil) { 633 if (shader->info.base.writes_memory) 634 depth_mode = LATE_DEPTH_TEST | LATE_DEPTH_WRITE; 635 else if (key->alpha.enabled || 636 key->blend.alpha_to_coverage || 637 shader->info.base.uses_kill || 638 shader->info.base.writes_samplemask) { 639 /* With alpha test and kill, can do the depth test early 640 * and hopefully eliminate some quads. But need to do a 641 * special deferred depth write once the final mask value 642 * is known. This only works though if there's either no 643 * stencil test or the stencil value isn't written. 644 */ 645 if (key->stencil[0].enabled && (key->stencil[0].writemask || 646 (key->stencil[1].enabled && 647 key->stencil[1].writemask))) 648 depth_mode = LATE_DEPTH_TEST | LATE_DEPTH_WRITE; 649 else 650 depth_mode = EARLY_DEPTH_TEST | LATE_DEPTH_WRITE; 651 } 652 else 653 depth_mode = EARLY_DEPTH_TEST | EARLY_DEPTH_WRITE; 654 } 655 else { 656 depth_mode = LATE_DEPTH_TEST | LATE_DEPTH_WRITE; 657 } 658 659 if (!(key->depth.enabled && key->depth.writemask) && 660 !(key->stencil[0].enabled && (key->stencil[0].writemask || 661 (key->stencil[1].enabled && 662 key->stencil[1].writemask)))) 663 depth_mode &= ~(LATE_DEPTH_WRITE | EARLY_DEPTH_WRITE); 664 } 665 else { 666 depth_mode = 0; 667 } 668 669 vec_type = lp_build_vec_type(gallivm, type); 670 int_vec_type = lp_build_vec_type(gallivm, int_type); 671 672 stencil_refs[0] = lp_jit_context_stencil_ref_front_value(gallivm, context_ptr); 673 stencil_refs[1] = lp_jit_context_stencil_ref_back_value(gallivm, context_ptr); 674 /* convert scalar stencil refs into vectors */ 675 stencil_refs[0] = lp_build_broadcast(gallivm, int_vec_type, stencil_refs[0]); 676 stencil_refs[1] = lp_build_broadcast(gallivm, int_vec_type, stencil_refs[1]); 677 678 consts_ptr = lp_jit_context_constants(gallivm, context_ptr); 679 num_consts_ptr = lp_jit_context_num_constants(gallivm, context_ptr); 680 681 ssbo_ptr = lp_jit_context_ssbos(gallivm, context_ptr); 682 num_ssbo_ptr = lp_jit_context_num_ssbos(gallivm, context_ptr); 683 684 memset(outputs, 0, sizeof outputs); 685 686 /* Allocate color storage for each fragment sample */ 687 LLVMValueRef color_store_size = num_loop; 688 if (key->min_samples > 1) 689 color_store_size = LLVMBuildMul(builder, num_loop, lp_build_const_int32(gallivm, key->min_samples), ""); 690 691 for(cbuf = 0; cbuf < key->nr_cbufs; cbuf++) { 692 for(chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) { 693 out_color[cbuf][chan] = lp_build_array_alloca(gallivm, 694 lp_build_vec_type(gallivm, 695 type), 696 color_store_size, "color"); 697 } 698 } 699 if (dual_source_blend) { 700 assert(key->nr_cbufs <= 1); 701 for(chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) { 702 out_color[1][chan] = lp_build_array_alloca(gallivm, 703 lp_build_vec_type(gallivm, 704 type), 705 color_store_size, "color1"); 706 } 707 } 708 if (shader->info.base.writes_z) { 709 z_out = lp_build_array_alloca(gallivm, 710 lp_build_vec_type(gallivm, type), 711 color_store_size, "depth"); 712 } 713 714 if (shader->info.base.writes_stencil) { 715 s_out = lp_build_array_alloca(gallivm, 716 lp_build_vec_type(gallivm, type), 717 color_store_size, "depth"); 718 } 719 720 lp_build_for_loop_begin(&loop_state, gallivm, 721 lp_build_const_int32(gallivm, 0), 722 LLVMIntULT, 723 num_loop, 724 lp_build_const_int32(gallivm, 1)); 725 726 LLVMValueRef sample_mask_in; 727 if (key->multisample) { 728 sample_mask_in = lp_build_const_int_vec(gallivm, type, 0); 729 /* create shader execution mask by combining all sample masks. */ 730 for (unsigned s = 0; s < key->coverage_samples; s++) { 731 LLVMValueRef s_mask_idx = LLVMBuildMul(builder, num_loop, lp_build_const_int32(gallivm, s), ""); 732 s_mask_idx = LLVMBuildAdd(builder, s_mask_idx, loop_state.counter, ""); 733 LLVMValueRef s_mask = lp_build_pointer_get(builder, mask_store, s_mask_idx); 734 if (s == 0) 735 mask_val = s_mask; 736 else 737 mask_val = LLVMBuildOr(builder, s_mask, mask_val, ""); 738 739 LLVMValueRef mask_in = LLVMBuildAnd(builder, s_mask, lp_build_const_int_vec(gallivm, type, (1ll << s)), ""); 740 sample_mask_in = LLVMBuildOr(builder, sample_mask_in, mask_in, ""); 741 } 742 } else { 743 sample_mask_in = lp_build_const_int_vec(gallivm, type, 1); 744 mask_ptr = LLVMBuildGEP(builder, mask_store, 745 &loop_state.counter, 1, "mask_ptr"); 746 mask_val = LLVMBuildLoad(builder, mask_ptr, ""); 747 748 LLVMValueRef mask_in = LLVMBuildAnd(builder, mask_val, lp_build_const_int_vec(gallivm, type, 1), ""); 749 sample_mask_in = LLVMBuildOr(builder, sample_mask_in, mask_in, ""); 750 } 751 752 /* 'mask' will control execution based on quad's pixel alive/killed state */ 753 lp_build_mask_begin(&mask, gallivm, type, mask_val); 754 755 if (!(depth_mode & EARLY_DEPTH_TEST) && !simple_shader) 756 lp_build_mask_check(&mask); 757 758 /* Create storage for recombining sample masks after early Z pass. */ 759 LLVMValueRef s_mask_or = lp_build_alloca(gallivm, lp_build_int_vec_type(gallivm, type), "cov_mask_early_depth"); 760 LLVMBuildStore(builder, LLVMConstNull(lp_build_int_vec_type(gallivm, type)), s_mask_or); 761 762 /* Create storage for post depth sample mask */ 763 LLVMValueRef post_depth_sample_mask_in = NULL; 764 if (post_depth_coverage) 765 post_depth_sample_mask_in = lp_build_alloca(gallivm, int_vec_type, "post_depth_sample_mask_in"); 766 767 LLVMValueRef s_mask = NULL, s_mask_ptr = NULL; 768 LLVMValueRef z_sample_value_store = NULL, s_sample_value_store = NULL; 769 LLVMValueRef z_fb_store = NULL, s_fb_store = NULL; 770 LLVMTypeRef z_type = NULL, z_fb_type = NULL; 771 772 /* Run early depth once per sample */ 773 if (key->multisample) { 774 775 if (zs_format_desc) { 776 struct lp_type zs_type = lp_depth_type(zs_format_desc, type.length); 777 struct lp_type z_type = zs_type; 778 struct lp_type s_type = zs_type; 779 if (zs_format_desc->block.bits < type.width) 780 z_type.width = type.width; 781 if (zs_format_desc->block.bits == 8) 782 s_type.width = type.width; 783 784 else if (zs_format_desc->block.bits > 32) { 785 z_type.width = z_type.width / 2; 786 s_type.width = s_type.width / 2; 787 s_type.floating = 0; 788 } 789 z_sample_value_store = lp_build_array_alloca(gallivm, lp_build_int_vec_type(gallivm, type), 790 zs_samples, "z_sample_store"); 791 s_sample_value_store = lp_build_array_alloca(gallivm, lp_build_int_vec_type(gallivm, type), 792 zs_samples, "s_sample_store"); 793 z_fb_store = lp_build_array_alloca(gallivm, lp_build_vec_type(gallivm, z_type), 794 zs_samples, "z_fb_store"); 795 s_fb_store = lp_build_array_alloca(gallivm, lp_build_vec_type(gallivm, s_type), 796 zs_samples, "s_fb_store"); 797 } 798 lp_build_for_loop_begin(&sample_loop_state, gallivm, 799 lp_build_const_int32(gallivm, 0), 800 LLVMIntULT, lp_build_const_int32(gallivm, key->coverage_samples), 801 lp_build_const_int32(gallivm, 1)); 802 803 LLVMValueRef s_mask_idx = LLVMBuildMul(builder, sample_loop_state.counter, num_loop, ""); 804 s_mask_idx = LLVMBuildAdd(builder, s_mask_idx, loop_state.counter, ""); 805 s_mask_ptr = LLVMBuildGEP(builder, mask_store, &s_mask_idx, 1, ""); 806 807 s_mask = LLVMBuildLoad(builder, s_mask_ptr, ""); 808 s_mask = LLVMBuildAnd(builder, s_mask, mask_val, ""); 809 } 810 811 812 /* for multisample Z needs to be interpolated at sample points for testing. */ 813 lp_build_interp_soa_update_pos_dyn(interp, gallivm, loop_state.counter, key->multisample ? sample_loop_state.counter : NULL); 814 z = interp->pos[2]; 815 816 depth_ptr = depth_base_ptr; 817 if (key->multisample) { 818 LLVMValueRef sample_offset = LLVMBuildMul(builder, sample_loop_state.counter, depth_sample_stride, ""); 819 depth_ptr = LLVMBuildGEP(builder, depth_ptr, &sample_offset, 1, ""); 820 } 821 822 if (depth_mode & EARLY_DEPTH_TEST) { 823 /* 824 * Clamp according to ARB_depth_clamp semantics. 825 */ 826 if (key->depth_clamp) { 827 z = lp_build_depth_clamp(gallivm, builder, type, context_ptr, 828 thread_data_ptr, z); 829 } 830 lp_build_depth_stencil_load_swizzled(gallivm, type, 831 zs_format_desc, key->resource_1d, 832 depth_ptr, depth_stride, 833 &z_fb, &s_fb, loop_state.counter); 834 lp_build_depth_stencil_test(gallivm, 835 &key->depth, 836 key->stencil, 837 type, 838 zs_format_desc, 839 key->multisample ? NULL : &mask, 840 &s_mask, 841 stencil_refs, 842 z, z_fb, s_fb, 843 facing, 844 &z_value, &s_value, 845 !simple_shader && !key->multisample); 846 847 if (depth_mode & EARLY_DEPTH_WRITE) { 848 lp_build_depth_stencil_write_swizzled(gallivm, type, 849 zs_format_desc, key->resource_1d, 850 NULL, NULL, NULL, loop_state.counter, 851 depth_ptr, depth_stride, 852 z_value, s_value); 853 } 854 /* 855 * Note mask check if stencil is enabled must be after ds write not after 856 * stencil test otherwise new stencil values may not get written if all 857 * fragments got killed by depth/stencil test. 858 */ 859 if (!simple_shader && key->stencil[0].enabled && !key->multisample) 860 lp_build_mask_check(&mask); 861 862 if (key->multisample) { 863 z_fb_type = LLVMTypeOf(z_fb); 864 z_type = LLVMTypeOf(z_value); 865 lp_build_pointer_set(builder, z_sample_value_store, sample_loop_state.counter, LLVMBuildBitCast(builder, z_value, lp_build_int_vec_type(gallivm, type), "")); 866 lp_build_pointer_set(builder, s_sample_value_store, sample_loop_state.counter, LLVMBuildBitCast(builder, s_value, lp_build_int_vec_type(gallivm, type), "")); 867 lp_build_pointer_set(builder, z_fb_store, sample_loop_state.counter, z_fb); 868 lp_build_pointer_set(builder, s_fb_store, sample_loop_state.counter, s_fb); 869 } 870 } 871 872 if (key->multisample) { 873 /* 874 * Store the post-early Z coverage mask. 875 * Recombine the resulting coverage masks post early Z into the fragment 876 * shader execution mask. 877 */ 878 LLVMValueRef tmp_s_mask_or = LLVMBuildLoad(builder, s_mask_or, ""); 879 tmp_s_mask_or = LLVMBuildOr(builder, tmp_s_mask_or, s_mask, ""); 880 LLVMBuildStore(builder, tmp_s_mask_or, s_mask_or); 881 882 if (post_depth_coverage) { 883 LLVMValueRef mask_bit_idx = LLVMBuildShl(builder, lp_build_const_int32(gallivm, 1), sample_loop_state.counter, ""); 884 LLVMValueRef post_depth_mask_in = LLVMBuildLoad(builder, post_depth_sample_mask_in, ""); 885 mask_bit_idx = LLVMBuildAnd(builder, s_mask, lp_build_broadcast(gallivm, int_vec_type, mask_bit_idx), ""); 886 post_depth_mask_in = LLVMBuildOr(builder, post_depth_mask_in, mask_bit_idx, ""); 887 LLVMBuildStore(builder, post_depth_mask_in, post_depth_sample_mask_in); 888 } 889 890 LLVMBuildStore(builder, s_mask, s_mask_ptr); 891 892 lp_build_for_loop_end(&sample_loop_state); 893 894 /* recombined all the coverage masks in the shader exec mask. */ 895 tmp_s_mask_or = LLVMBuildLoad(builder, s_mask_or, ""); 896 lp_build_mask_update(&mask, tmp_s_mask_or); 897 898 if (key->min_samples == 1) { 899 /* for multisample Z needs to be re interpolated at pixel center */ 900 lp_build_interp_soa_update_pos_dyn(interp, gallivm, loop_state.counter, NULL); 901 z = interp->pos[2]; 902 lp_build_mask_update(&mask, tmp_s_mask_or); 903 } 904 } else { 905 if (post_depth_coverage) { 906 LLVMValueRef post_depth_mask_in = LLVMBuildAnd(builder, lp_build_mask_value(&mask), lp_build_const_int_vec(gallivm, type, 1), ""); 907 LLVMBuildStore(builder, post_depth_mask_in, post_depth_sample_mask_in); 908 } 909 } 910 911 LLVMValueRef out_sample_mask_storage = NULL; 912 if (shader->info.base.writes_samplemask) { 913 out_sample_mask_storage = lp_build_alloca(gallivm, int_vec_type, "write_mask"); 914 if (key->min_samples > 1) 915 LLVMBuildStore(builder, LLVMConstNull(int_vec_type), out_sample_mask_storage); 916 } 917 918 if (post_depth_coverage) { 919 system_values.sample_mask_in = LLVMBuildLoad(builder, post_depth_sample_mask_in, ""); 920 } 921 else 922 system_values.sample_mask_in = sample_mask_in; 923 if (key->multisample && key->min_samples > 1) { 924 lp_build_for_loop_begin(&sample_loop_state, gallivm, 925 lp_build_const_int32(gallivm, 0), 926 LLVMIntULT, 927 lp_build_const_int32(gallivm, key->min_samples), 928 lp_build_const_int32(gallivm, 1)); 929 930 LLVMValueRef s_mask_idx = LLVMBuildMul(builder, sample_loop_state.counter, num_loop, ""); 931 s_mask_idx = LLVMBuildAdd(builder, s_mask_idx, loop_state.counter, ""); 932 s_mask_ptr = LLVMBuildGEP(builder, mask_store, &s_mask_idx, 1, ""); 933 s_mask = LLVMBuildLoad(builder, s_mask_ptr, ""); 934 lp_build_mask_force(&mask, s_mask); 935 lp_build_interp_soa_update_pos_dyn(interp, gallivm, loop_state.counter, sample_loop_state.counter); 936 system_values.sample_id = sample_loop_state.counter; 937 system_values.sample_mask_in = LLVMBuildAnd(builder, system_values.sample_mask_in, 938 lp_build_broadcast(gallivm, int_vec_type, 939 LLVMBuildShl(builder, lp_build_const_int32(gallivm, 1), sample_loop_state.counter, "")), ""); 940 } else { 941 system_values.sample_id = lp_build_const_int32(gallivm, 0); 942 943 } 944 system_values.sample_pos = sample_pos_array; 945 946 lp_build_interp_soa_update_inputs_dyn(interp, gallivm, loop_state.counter, mask_store, sample_loop_state.counter); 947 948 struct lp_build_fs_llvm_iface fs_iface = { 949 .base.interp_fn = fs_interp, 950 .base.fb_fetch = fs_fb_fetch, 951 .interp = interp, 952 .loop_state = &loop_state, 953 .sample_id = system_values.sample_id, 954 .mask_store = mask_store, 955 .color_ptr_ptr = color_ptr_ptr, 956 .color_stride_ptr = color_stride_ptr, 957 .color_sample_stride_ptr = color_sample_stride_ptr, 958 .key = key, 959 }; 960 961 struct lp_build_tgsi_params params; 962 memset(¶ms, 0, sizeof(params)); 963 964 params.type = type; 965 params.mask = &mask; 966 params.fs_iface = &fs_iface.base; 967 params.consts_ptr = consts_ptr; 968 params.const_sizes_ptr = num_consts_ptr; 969 params.system_values = &system_values; 970 params.inputs = interp->inputs; 971 params.context_ptr = context_ptr; 972 params.thread_data_ptr = thread_data_ptr; 973 params.sampler = sampler; 974 params.info = &shader->info.base; 975 params.ssbo_ptr = ssbo_ptr; 976 params.ssbo_sizes_ptr = num_ssbo_ptr; 977 params.image = image; 978 params.aniso_filter_table = lp_jit_context_aniso_filter_table(gallivm, context_ptr); 979 980 /* Build the actual shader */ 981 if (shader->base.type == PIPE_SHADER_IR_TGSI) 982 lp_build_tgsi_soa(gallivm, tokens, ¶ms, 983 outputs); 984 else 985 lp_build_nir_soa(gallivm, shader->base.ir.nir, ¶ms, 986 outputs); 987 988 /* Alpha test */ 989 if (key->alpha.enabled) { 990 int color0 = find_output_by_semantic(&shader->info.base, 991 TGSI_SEMANTIC_COLOR, 992 0); 993 994 if (color0 != -1 && outputs[color0][3]) { 995 const struct util_format_description *cbuf_format_desc; 996 LLVMValueRef alpha = LLVMBuildLoad(builder, outputs[color0][3], "alpha"); 997 LLVMValueRef alpha_ref_value; 998 999 alpha_ref_value = lp_jit_context_alpha_ref_value(gallivm, context_ptr); 1000 alpha_ref_value = lp_build_broadcast(gallivm, vec_type, alpha_ref_value); 1001 1002 cbuf_format_desc = util_format_description(key->cbuf_format[0]); 1003 1004 lp_build_alpha_test(gallivm, key->alpha.func, type, cbuf_format_desc, 1005 &mask, alpha, alpha_ref_value, 1006 (depth_mode & LATE_DEPTH_TEST) != 0); 1007 } 1008 } 1009 1010 /* Emulate Alpha to Coverage with Alpha test */ 1011 if (key->blend.alpha_to_coverage) { 1012 int color0 = find_output_by_semantic(&shader->info.base, 1013 TGSI_SEMANTIC_COLOR, 1014 0); 1015 1016 if (color0 != -1 && outputs[color0][3]) { 1017 LLVMValueRef alpha = LLVMBuildLoad(builder, outputs[color0][3], "alpha"); 1018 1019 if (!key->multisample) { 1020 lp_build_alpha_to_coverage(gallivm, type, 1021 &mask, alpha, 1022 (depth_mode & LATE_DEPTH_TEST) != 0); 1023 } else { 1024 lp_build_sample_alpha_to_coverage(gallivm, type, key->coverage_samples, num_loop, 1025 loop_state.counter, 1026 mask_store, alpha); 1027 } 1028 } 1029 } 1030 if (key->blend.alpha_to_one && key->multisample) { 1031 for (attrib = 0; attrib < shader->info.base.num_outputs; ++attrib) { 1032 unsigned cbuf = shader->info.base.output_semantic_index[attrib]; 1033 if ((shader->info.base.output_semantic_name[attrib] == TGSI_SEMANTIC_COLOR) && 1034 ((cbuf < key->nr_cbufs) || (cbuf == 1 && dual_source_blend))) 1035 if (outputs[cbuf][3]) { 1036 LLVMBuildStore(builder, lp_build_const_vec(gallivm, type, 1.0), outputs[cbuf][3]); 1037 } 1038 } 1039 } 1040 if (shader->info.base.writes_samplemask) { 1041 LLVMValueRef output_smask = NULL; 1042 int smaski = find_output_by_semantic(&shader->info.base, 1043 TGSI_SEMANTIC_SAMPLEMASK, 1044 0); 1045 struct lp_build_context smask_bld; 1046 lp_build_context_init(&smask_bld, gallivm, int_type); 1047 1048 assert(smaski >= 0); 1049 output_smask = LLVMBuildLoad(builder, outputs[smaski][0], "smask"); 1050 output_smask = LLVMBuildBitCast(builder, output_smask, smask_bld.vec_type, ""); 1051 if (!key->multisample && key->no_ms_sample_mask_out) { 1052 output_smask = lp_build_and(&smask_bld, output_smask, smask_bld.one); 1053 output_smask = lp_build_cmp(&smask_bld, PIPE_FUNC_NOTEQUAL, output_smask, smask_bld.zero); 1054 lp_build_mask_update(&mask, output_smask); 1055 } 1056 1057 if (key->min_samples > 1) { 1058 /* only the bit corresponding to this sample is to be used. */ 1059 LLVMValueRef tmp_mask = LLVMBuildLoad(builder, out_sample_mask_storage, "tmp_mask"); 1060 LLVMValueRef out_smask_idx = LLVMBuildShl(builder, lp_build_const_int32(gallivm, 1), sample_loop_state.counter, ""); 1061 LLVMValueRef smask_bit = LLVMBuildAnd(builder, output_smask, lp_build_broadcast(gallivm, int_vec_type, out_smask_idx), ""); 1062 output_smask = LLVMBuildOr(builder, tmp_mask, smask_bit, ""); 1063 } 1064 1065 LLVMBuildStore(builder, output_smask, out_sample_mask_storage); 1066 } 1067 1068 if (shader->info.base.writes_z) { 1069 int pos0 = find_output_by_semantic(&shader->info.base, 1070 TGSI_SEMANTIC_POSITION, 1071 0); 1072 LLVMValueRef out = LLVMBuildLoad(builder, outputs[pos0][2], ""); 1073 LLVMValueRef idx = loop_state.counter; 1074 if (key->min_samples > 1) 1075 idx = LLVMBuildAdd(builder, idx, 1076 LLVMBuildMul(builder, sample_loop_state.counter, num_loop, ""), ""); 1077 LLVMValueRef ptr = LLVMBuildGEP(builder, z_out, &idx, 1, ""); 1078 LLVMBuildStore(builder, out, ptr); 1079 } 1080 1081 if (shader->info.base.writes_stencil) { 1082 int sten_out = find_output_by_semantic(&shader->info.base, 1083 TGSI_SEMANTIC_STENCIL, 1084 0); 1085 LLVMValueRef out = LLVMBuildLoad(builder, outputs[sten_out][1], "output.s"); 1086 LLVMValueRef idx = loop_state.counter; 1087 if (key->min_samples > 1) 1088 idx = LLVMBuildAdd(builder, idx, 1089 LLVMBuildMul(builder, sample_loop_state.counter, num_loop, ""), ""); 1090 LLVMValueRef ptr = LLVMBuildGEP(builder, s_out, &idx, 1, ""); 1091 LLVMBuildStore(builder, out, ptr); 1092 } 1093 1094 1095 /* Color write - per fragment sample */ 1096 for (attrib = 0; attrib < shader->info.base.num_outputs; ++attrib) 1097 { 1098 unsigned cbuf = shader->info.base.output_semantic_index[attrib]; 1099 if ((shader->info.base.output_semantic_name[attrib] == TGSI_SEMANTIC_COLOR) && 1100 ((cbuf < key->nr_cbufs) || (cbuf == 1 && dual_source_blend))) 1101 { 1102 for(chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) { 1103 if(outputs[attrib][chan]) { 1104 /* XXX: just initialize outputs to point at colors[] and 1105 * skip this. 1106 */ 1107 LLVMValueRef out = LLVMBuildLoad(builder, outputs[attrib][chan], ""); 1108 LLVMValueRef color_ptr; 1109 LLVMValueRef color_idx = loop_state.counter; 1110 if (key->min_samples > 1) 1111 color_idx = LLVMBuildAdd(builder, color_idx, 1112 LLVMBuildMul(builder, sample_loop_state.counter, num_loop, ""), ""); 1113 color_ptr = LLVMBuildGEP(builder, out_color[cbuf][chan], 1114 &color_idx, 1, ""); 1115 lp_build_name(out, "color%u.%c", attrib, "rgba"[chan]); 1116 LLVMBuildStore(builder, out, color_ptr); 1117 } 1118 } 1119 } 1120 } 1121 1122 if (key->multisample && key->min_samples > 1) { 1123 LLVMBuildStore(builder, lp_build_mask_value(&mask), s_mask_ptr); 1124 lp_build_for_loop_end(&sample_loop_state); 1125 } 1126 1127 if (key->multisample) { 1128 /* execute depth test for each sample */ 1129 lp_build_for_loop_begin(&sample_loop_state, gallivm, 1130 lp_build_const_int32(gallivm, 0), 1131 LLVMIntULT, lp_build_const_int32(gallivm, key->coverage_samples), 1132 lp_build_const_int32(gallivm, 1)); 1133 1134 /* load the per-sample coverage mask */ 1135 LLVMValueRef s_mask_idx = LLVMBuildMul(builder, sample_loop_state.counter, num_loop, ""); 1136 s_mask_idx = LLVMBuildAdd(builder, s_mask_idx, loop_state.counter, ""); 1137 s_mask_ptr = LLVMBuildGEP(builder, mask_store, &s_mask_idx, 1, ""); 1138 1139 /* combine the execution mask post fragment shader with the coverage mask. */ 1140 s_mask = LLVMBuildLoad(builder, s_mask_ptr, ""); 1141 if (key->min_samples == 1) 1142 s_mask = LLVMBuildAnd(builder, s_mask, lp_build_mask_value(&mask), ""); 1143 1144 /* if the shader writes sample mask use that */ 1145 if (shader->info.base.writes_samplemask) { 1146 LLVMValueRef out_smask_idx = LLVMBuildShl(builder, lp_build_const_int32(gallivm, 1), sample_loop_state.counter, ""); 1147 out_smask_idx = lp_build_broadcast(gallivm, int_vec_type, out_smask_idx); 1148 LLVMValueRef output_smask = LLVMBuildLoad(builder, out_sample_mask_storage, ""); 1149 LLVMValueRef smask_bit = LLVMBuildAnd(builder, output_smask, out_smask_idx, ""); 1150 LLVMValueRef cmp = LLVMBuildICmp(builder, LLVMIntNE, smask_bit, lp_build_const_int_vec(gallivm, int_type, 0), ""); 1151 smask_bit = LLVMBuildSExt(builder, cmp, int_vec_type, ""); 1152 1153 s_mask = LLVMBuildAnd(builder, s_mask, smask_bit, ""); 1154 } 1155 } 1156 1157 depth_ptr = depth_base_ptr; 1158 if (key->multisample) { 1159 LLVMValueRef sample_offset = LLVMBuildMul(builder, sample_loop_state.counter, depth_sample_stride, ""); 1160 depth_ptr = LLVMBuildGEP(builder, depth_ptr, &sample_offset, 1, ""); 1161 } 1162 1163 /* Late Z test */ 1164 if (depth_mode & LATE_DEPTH_TEST) { 1165 if (shader->info.base.writes_z) { 1166 LLVMValueRef idx = loop_state.counter; 1167 if (key->min_samples > 1) 1168 idx = LLVMBuildAdd(builder, idx, 1169 LLVMBuildMul(builder, sample_loop_state.counter, num_loop, ""), ""); 1170 LLVMValueRef ptr = LLVMBuildGEP(builder, z_out, &idx, 1, ""); 1171 z = LLVMBuildLoad(builder, ptr, "output.z"); 1172 } else { 1173 if (key->multisample) { 1174 lp_build_interp_soa_update_pos_dyn(interp, gallivm, loop_state.counter, key->multisample ? sample_loop_state.counter : NULL); 1175 z = interp->pos[2]; 1176 } 1177 } 1178 1179 /* 1180 * Clamp according to ARB_depth_clamp semantics. 1181 */ 1182 if (key->depth_clamp) { 1183 z = lp_build_depth_clamp(gallivm, builder, type, context_ptr, 1184 thread_data_ptr, z); 1185 } else { 1186 struct lp_build_context f32_bld; 1187 lp_build_context_init(&f32_bld, gallivm, type); 1188 z = lp_build_clamp(&f32_bld, z, 1189 lp_build_const_vec(gallivm, type, 0.0), 1190 lp_build_const_vec(gallivm, type, 1.0)); 1191 } 1192 1193 if (shader->info.base.writes_stencil) { 1194 LLVMValueRef idx = loop_state.counter; 1195 if (key->min_samples > 1) 1196 idx = LLVMBuildAdd(builder, idx, 1197 LLVMBuildMul(builder, sample_loop_state.counter, num_loop, ""), ""); 1198 LLVMValueRef ptr = LLVMBuildGEP(builder, s_out, &idx, 1, ""); 1199 stencil_refs[0] = LLVMBuildLoad(builder, ptr, "output.s"); 1200 /* there's only one value, and spec says to discard additional bits */ 1201 LLVMValueRef s_max_mask = lp_build_const_int_vec(gallivm, int_type, 255); 1202 stencil_refs[0] = LLVMBuildBitCast(builder, stencil_refs[0], int_vec_type, ""); 1203 stencil_refs[0] = LLVMBuildAnd(builder, stencil_refs[0], s_max_mask, ""); 1204 stencil_refs[1] = stencil_refs[0]; 1205 } 1206 1207 lp_build_depth_stencil_load_swizzled(gallivm, type, 1208 zs_format_desc, key->resource_1d, 1209 depth_ptr, depth_stride, 1210 &z_fb, &s_fb, loop_state.counter); 1211 1212 lp_build_depth_stencil_test(gallivm, 1213 &key->depth, 1214 key->stencil, 1215 type, 1216 zs_format_desc, 1217 key->multisample ? NULL : &mask, 1218 &s_mask, 1219 stencil_refs, 1220 z, z_fb, s_fb, 1221 facing, 1222 &z_value, &s_value, 1223 !simple_shader); 1224 /* Late Z write */ 1225 if (depth_mode & LATE_DEPTH_WRITE) { 1226 lp_build_depth_stencil_write_swizzled(gallivm, type, 1227 zs_format_desc, key->resource_1d, 1228 NULL, NULL, NULL, loop_state.counter, 1229 depth_ptr, depth_stride, 1230 z_value, s_value); 1231 } 1232 } 1233 else if ((depth_mode & EARLY_DEPTH_TEST) && 1234 (depth_mode & LATE_DEPTH_WRITE)) 1235 { 1236 /* Need to apply a reduced mask to the depth write. Reload the 1237 * depth value, update from zs_value with the new mask value and 1238 * write that out. 1239 */ 1240 if (key->multisample) { 1241 z_value = LLVMBuildBitCast(builder, lp_build_pointer_get(builder, z_sample_value_store, sample_loop_state.counter), z_type, "");; 1242 s_value = lp_build_pointer_get(builder, s_sample_value_store, sample_loop_state.counter); 1243 z_fb = LLVMBuildBitCast(builder, lp_build_pointer_get(builder, z_fb_store, sample_loop_state.counter), z_fb_type, ""); 1244 s_fb = lp_build_pointer_get(builder, s_fb_store, sample_loop_state.counter); 1245 } 1246 lp_build_depth_stencil_write_swizzled(gallivm, type, 1247 zs_format_desc, key->resource_1d, 1248 key->multisample ? s_mask : lp_build_mask_value(&mask), z_fb, s_fb, loop_state.counter, 1249 depth_ptr, depth_stride, 1250 z_value, s_value); 1251 } 1252 1253 if (key->occlusion_count) { 1254 LLVMValueRef counter = lp_jit_thread_data_counter(gallivm, thread_data_ptr); 1255 lp_build_name(counter, "counter"); 1256 1257 lp_build_occlusion_count(gallivm, type, 1258 key->multisample ? s_mask : lp_build_mask_value(&mask), counter); 1259 } 1260 1261 if (key->multisample) { 1262 /* store the sample mask for this loop */ 1263 LLVMBuildStore(builder, s_mask, s_mask_ptr); 1264 lp_build_for_loop_end(&sample_loop_state); 1265 } 1266 1267 mask_val = lp_build_mask_end(&mask); 1268 if (!key->multisample) 1269 LLVMBuildStore(builder, mask_val, mask_ptr); 1270 lp_build_for_loop_end(&loop_state); 1271} 1272 1273 1274/** 1275 * This function will reorder pixels from the fragment shader SoA to memory layout AoS 1276 * 1277 * Fragment Shader outputs pixels in small 2x2 blocks 1278 * e.g. (0, 0), (1, 0), (0, 1), (1, 1) ; (2, 0) ... 1279 * 1280 * However in memory pixels are stored in rows 1281 * e.g. (0, 0), (1, 0), (2, 0), (3, 0) ; (0, 1) ... 1282 * 1283 * @param type fragment shader type (4x or 8x float) 1284 * @param num_fs number of fs_src 1285 * @param is_1d whether we're outputting to a 1d resource 1286 * @param dst_channels number of output channels 1287 * @param fs_src output from fragment shader 1288 * @param dst pointer to store result 1289 * @param pad_inline is channel padding inline or at end of row 1290 * @return the number of dsts 1291 */ 1292static int 1293generate_fs_twiddle(struct gallivm_state *gallivm, 1294 struct lp_type type, 1295 unsigned num_fs, 1296 unsigned dst_channels, 1297 LLVMValueRef fs_src[][4], 1298 LLVMValueRef* dst, 1299 bool pad_inline) 1300{ 1301 LLVMValueRef src[16]; 1302 1303 bool swizzle_pad; 1304 bool twiddle; 1305 bool split; 1306 1307 unsigned pixels = type.length / 4; 1308 unsigned reorder_group; 1309 unsigned src_channels; 1310 unsigned src_count; 1311 unsigned i; 1312 1313 src_channels = dst_channels < 3 ? dst_channels : 4; 1314 src_count = num_fs * src_channels; 1315 1316 assert(pixels == 2 || pixels == 1); 1317 assert(num_fs * src_channels <= ARRAY_SIZE(src)); 1318 1319 /* 1320 * Transpose from SoA -> AoS 1321 */ 1322 for (i = 0; i < num_fs; ++i) { 1323 lp_build_transpose_aos_n(gallivm, type, &fs_src[i][0], src_channels, &src[i * src_channels]); 1324 } 1325 1326 /* 1327 * Pick transformation options 1328 */ 1329 swizzle_pad = false; 1330 twiddle = false; 1331 split = false; 1332 reorder_group = 0; 1333 1334 if (dst_channels == 1) { 1335 twiddle = true; 1336 1337 if (pixels == 2) { 1338 split = true; 1339 } 1340 } else if (dst_channels == 2) { 1341 if (pixels == 1) { 1342 reorder_group = 1; 1343 } 1344 } else if (dst_channels > 2) { 1345 if (pixels == 1) { 1346 reorder_group = 2; 1347 } else { 1348 twiddle = true; 1349 } 1350 1351 if (!pad_inline && dst_channels == 3 && pixels > 1) { 1352 swizzle_pad = true; 1353 } 1354 } 1355 1356 /* 1357 * Split the src in half 1358 */ 1359 if (split) { 1360 for (i = num_fs; i > 0; --i) { 1361 src[(i - 1)*2 + 1] = lp_build_extract_range(gallivm, src[i - 1], 4, 4); 1362 src[(i - 1)*2 + 0] = lp_build_extract_range(gallivm, src[i - 1], 0, 4); 1363 } 1364 1365 src_count *= 2; 1366 type.length = 4; 1367 } 1368 1369 /* 1370 * Ensure pixels are in memory order 1371 */ 1372 if (reorder_group) { 1373 /* Twiddle pixels by reordering the array, e.g.: 1374 * 1375 * src_count = 8 -> 0 2 1 3 4 6 5 7 1376 * src_count = 16 -> 0 1 4 5 2 3 6 7 8 9 12 13 10 11 14 15 1377 */ 1378 const unsigned reorder_sw[] = { 0, 2, 1, 3 }; 1379 1380 for (i = 0; i < src_count; ++i) { 1381 unsigned group = i / reorder_group; 1382 unsigned block = (group / 4) * 4 * reorder_group; 1383 unsigned j = block + (reorder_sw[group % 4] * reorder_group) + (i % reorder_group); 1384 dst[i] = src[j]; 1385 } 1386 } else if (twiddle) { 1387 /* Twiddle pixels across elements of array */ 1388 /* 1389 * XXX: we should avoid this in some cases, but would need to tell 1390 * lp_build_conv to reorder (or deal with it ourselves). 1391 */ 1392 lp_bld_quad_twiddle(gallivm, type, src, src_count, dst); 1393 } else { 1394 /* Do nothing */ 1395 memcpy(dst, src, sizeof(LLVMValueRef) * src_count); 1396 } 1397 1398 /* 1399 * Moves any padding between pixels to the end 1400 * e.g. RGBXRGBX -> RGBRGBXX 1401 */ 1402 if (swizzle_pad) { 1403 unsigned char swizzles[16]; 1404 unsigned elems = pixels * dst_channels; 1405 1406 for (i = 0; i < type.length; ++i) { 1407 if (i < elems) 1408 swizzles[i] = i % dst_channels + (i / dst_channels) * 4; 1409 else 1410 swizzles[i] = LP_BLD_SWIZZLE_DONTCARE; 1411 } 1412 1413 for (i = 0; i < src_count; ++i) { 1414 dst[i] = lp_build_swizzle_aos_n(gallivm, dst[i], swizzles, type.length, type.length); 1415 } 1416 } 1417 1418 return src_count; 1419} 1420 1421 1422/* 1423 * Untwiddle and transpose, much like the above. 1424 * However, this is after conversion, so we get packed vectors. 1425 * At this time only handle 4x16i8 rgba / 2x16i8 rg / 1x16i8 r data, 1426 * the vectors will look like: 1427 * r0r1r4r5r2r3r6r7r8r9r12... (albeit color channels may 1428 * be swizzled here). Extending to 16bit should be trivial. 1429 * Should also be extended to handle twice wide vectors with AVX2... 1430 */ 1431static void 1432fs_twiddle_transpose(struct gallivm_state *gallivm, 1433 struct lp_type type, 1434 LLVMValueRef *src, 1435 unsigned src_count, 1436 LLVMValueRef *dst) 1437{ 1438 unsigned i, j; 1439 struct lp_type type64, type16, type32; 1440 LLVMTypeRef type64_t, type8_t, type16_t, type32_t; 1441 LLVMBuilderRef builder = gallivm->builder; 1442 LLVMValueRef tmp[4], shuf[8]; 1443 for (j = 0; j < 2; j++) { 1444 shuf[j*4 + 0] = lp_build_const_int32(gallivm, j*4 + 0); 1445 shuf[j*4 + 1] = lp_build_const_int32(gallivm, j*4 + 2); 1446 shuf[j*4 + 2] = lp_build_const_int32(gallivm, j*4 + 1); 1447 shuf[j*4 + 3] = lp_build_const_int32(gallivm, j*4 + 3); 1448 } 1449 1450 assert(src_count == 4 || src_count == 2 || src_count == 1); 1451 assert(type.width == 8); 1452 assert(type.length == 16); 1453 1454 type8_t = lp_build_vec_type(gallivm, type); 1455 1456 type64 = type; 1457 type64.length /= 8; 1458 type64.width *= 8; 1459 type64_t = lp_build_vec_type(gallivm, type64); 1460 1461 type16 = type; 1462 type16.length /= 2; 1463 type16.width *= 2; 1464 type16_t = lp_build_vec_type(gallivm, type16); 1465 1466 type32 = type; 1467 type32.length /= 4; 1468 type32.width *= 4; 1469 type32_t = lp_build_vec_type(gallivm, type32); 1470 1471 lp_build_transpose_aos_n(gallivm, type, src, src_count, tmp); 1472 1473 if (src_count == 1) { 1474 /* transpose was no-op, just untwiddle */ 1475 LLVMValueRef shuf_vec; 1476 shuf_vec = LLVMConstVector(shuf, 8); 1477 tmp[0] = LLVMBuildBitCast(builder, src[0], type16_t, ""); 1478 tmp[0] = LLVMBuildShuffleVector(builder, tmp[0], tmp[0], shuf_vec, ""); 1479 dst[0] = LLVMBuildBitCast(builder, tmp[0], type8_t, ""); 1480 } else if (src_count == 2) { 1481 LLVMValueRef shuf_vec; 1482 shuf_vec = LLVMConstVector(shuf, 4); 1483 1484 for (i = 0; i < 2; i++) { 1485 tmp[i] = LLVMBuildBitCast(builder, tmp[i], type32_t, ""); 1486 tmp[i] = LLVMBuildShuffleVector(builder, tmp[i], tmp[i], shuf_vec, ""); 1487 dst[i] = LLVMBuildBitCast(builder, tmp[i], type8_t, ""); 1488 } 1489 } else { 1490 for (j = 0; j < 2; j++) { 1491 LLVMValueRef lo, hi, lo2, hi2; 1492 /* 1493 * Note that if we only really have 3 valid channels (rgb) 1494 * and we don't need alpha we could substitute a undef here 1495 * for the respective channel (causing llvm to drop conversion 1496 * for alpha). 1497 */ 1498 /* we now have rgba0rgba1rgba4rgba5 etc, untwiddle */ 1499 lo2 = LLVMBuildBitCast(builder, tmp[j*2], type64_t, ""); 1500 hi2 = LLVMBuildBitCast(builder, tmp[j*2 + 1], type64_t, ""); 1501 lo = lp_build_interleave2(gallivm, type64, lo2, hi2, 0); 1502 hi = lp_build_interleave2(gallivm, type64, lo2, hi2, 1); 1503 dst[j*2] = LLVMBuildBitCast(builder, lo, type8_t, ""); 1504 dst[j*2 + 1] = LLVMBuildBitCast(builder, hi, type8_t, ""); 1505 } 1506 } 1507} 1508 1509 1510/** 1511 * Load an unswizzled block of pixels from memory 1512 */ 1513static void 1514load_unswizzled_block(struct gallivm_state *gallivm, 1515 LLVMValueRef base_ptr, 1516 LLVMValueRef stride, 1517 unsigned block_width, 1518 unsigned block_height, 1519 LLVMValueRef* dst, 1520 struct lp_type dst_type, 1521 unsigned dst_count, 1522 unsigned dst_alignment, 1523 LLVMValueRef x_offset, 1524 LLVMValueRef y_offset, 1525 bool fb_fetch_twiddle) 1526{ 1527 LLVMBuilderRef builder = gallivm->builder; 1528 unsigned row_size = dst_count / block_height; 1529 unsigned i; 1530 1531 /* Ensure block exactly fits into dst */ 1532 assert((block_width * block_height) % dst_count == 0); 1533 1534 for (i = 0; i < dst_count; ++i) { 1535 unsigned x = i % row_size; 1536 unsigned y = i / row_size; 1537 1538 if (block_height == 2 && dst_count == 8 && fb_fetch_twiddle) { 1539 /* remap the raw slots into the fragment shader execution mode. */ 1540 /* this math took me way too long to work out, I'm sure it's overkill. */ 1541 x = (i & 1) + ((i >> 2) << 1); 1542 y = (i & 2) >> 1; 1543 } 1544 1545 LLVMValueRef x_val; 1546 if (x_offset) { 1547 x_val = lp_build_const_int32(gallivm, x); 1548 if (x_offset) 1549 x_val = LLVMBuildAdd(builder, x_val, x_offset, ""); 1550 x_val = LLVMBuildMul(builder, x_val, lp_build_const_int32(gallivm, (dst_type.width / 8) * dst_type.length), ""); 1551 } else 1552 x_val = lp_build_const_int32(gallivm, x * (dst_type.width / 8) * dst_type.length); 1553 1554 LLVMValueRef bx = x_val; 1555 1556 LLVMValueRef y_val = lp_build_const_int32(gallivm, y); 1557 if (y_offset) 1558 y_val = LLVMBuildAdd(builder, y_val, y_offset, ""); 1559 LLVMValueRef by = LLVMBuildMul(builder, y_val, stride, ""); 1560 1561 LLVMValueRef gep[2]; 1562 LLVMValueRef dst_ptr; 1563 1564 gep[0] = lp_build_const_int32(gallivm, 0); 1565 gep[1] = LLVMBuildAdd(builder, bx, by, ""); 1566 1567 dst_ptr = LLVMBuildGEP(builder, base_ptr, gep, 2, ""); 1568 dst_ptr = LLVMBuildBitCast(builder, dst_ptr, 1569 LLVMPointerType(lp_build_vec_type(gallivm, dst_type), 0), ""); 1570 1571 dst[i] = LLVMBuildLoad(builder, dst_ptr, ""); 1572 1573 LLVMSetAlignment(dst[i], dst_alignment); 1574 } 1575} 1576 1577 1578/** 1579 * Store an unswizzled block of pixels to memory 1580 */ 1581static void 1582store_unswizzled_block(struct gallivm_state *gallivm, 1583 LLVMValueRef base_ptr, 1584 LLVMValueRef stride, 1585 unsigned block_width, 1586 unsigned block_height, 1587 LLVMValueRef* src, 1588 struct lp_type src_type, 1589 unsigned src_count, 1590 unsigned src_alignment) 1591{ 1592 LLVMBuilderRef builder = gallivm->builder; 1593 unsigned row_size = src_count / block_height; 1594 unsigned i; 1595 1596 /* Ensure src exactly fits into block */ 1597 assert((block_width * block_height) % src_count == 0); 1598 1599 for (i = 0; i < src_count; ++i) { 1600 unsigned x = i % row_size; 1601 unsigned y = i / row_size; 1602 1603 LLVMValueRef bx = lp_build_const_int32(gallivm, x * (src_type.width / 8) * src_type.length); 1604 LLVMValueRef by = LLVMBuildMul(builder, lp_build_const_int32(gallivm, y), stride, ""); 1605 1606 LLVMValueRef gep[2]; 1607 LLVMValueRef src_ptr; 1608 1609 gep[0] = lp_build_const_int32(gallivm, 0); 1610 gep[1] = LLVMBuildAdd(builder, bx, by, ""); 1611 1612 src_ptr = LLVMBuildGEP(builder, base_ptr, gep, 2, ""); 1613 src_ptr = LLVMBuildBitCast(builder, src_ptr, 1614 LLVMPointerType(lp_build_vec_type(gallivm, src_type), 0), ""); 1615 1616 src_ptr = LLVMBuildStore(builder, src[i], src_ptr); 1617 1618 LLVMSetAlignment(src_ptr, src_alignment); 1619 } 1620} 1621 1622 1623 1624/** 1625 * Retrieves the type for a format which is usable in the blending code. 1626 * 1627 * e.g. RGBA16F = 4x float, R3G3B2 = 3x byte 1628 */ 1629static inline void 1630lp_blend_type_from_format_desc(const struct util_format_description *format_desc, 1631 struct lp_type* type) 1632{ 1633 unsigned i; 1634 unsigned chan; 1635 1636 if (format_expands_to_float_soa(format_desc)) { 1637 /* always use ordinary floats for blending */ 1638 type->floating = true; 1639 type->fixed = false; 1640 type->sign = true; 1641 type->norm = false; 1642 type->width = 32; 1643 type->length = 4; 1644 return; 1645 } 1646 1647 for (i = 0; i < 4; i++) 1648 if (format_desc->channel[i].type != UTIL_FORMAT_TYPE_VOID) 1649 break; 1650 chan = i; 1651 1652 memset(type, 0, sizeof(struct lp_type)); 1653 type->floating = format_desc->channel[chan].type == UTIL_FORMAT_TYPE_FLOAT; 1654 type->fixed = format_desc->channel[chan].type == UTIL_FORMAT_TYPE_FIXED; 1655 type->sign = format_desc->channel[chan].type != UTIL_FORMAT_TYPE_UNSIGNED; 1656 type->norm = format_desc->channel[chan].normalized; 1657 type->width = format_desc->channel[chan].size; 1658 type->length = format_desc->nr_channels; 1659 1660 for (i = 1; i < format_desc->nr_channels; ++i) { 1661 if (format_desc->channel[i].size > type->width) 1662 type->width = format_desc->channel[i].size; 1663 } 1664 1665 if (type->floating) { 1666 type->width = 32; 1667 } else { 1668 if (type->width <= 8) { 1669 type->width = 8; 1670 } else if (type->width <= 16) { 1671 type->width = 16; 1672 } else { 1673 type->width = 32; 1674 } 1675 } 1676 1677 if (is_arithmetic_format(format_desc) && type->length == 3) { 1678 type->length = 4; 1679 } 1680} 1681 1682 1683/** 1684 * Scale a normalized value from src_bits to dst_bits. 1685 * 1686 * The exact calculation is 1687 * 1688 * dst = iround(src * dst_mask / src_mask) 1689 * 1690 * or with integer rounding 1691 * 1692 * dst = src * (2*dst_mask + sign(src)*src_mask) / (2*src_mask) 1693 * 1694 * where 1695 * 1696 * src_mask = (1 << src_bits) - 1 1697 * dst_mask = (1 << dst_bits) - 1 1698 * 1699 * but we try to avoid division and multiplication through shifts. 1700 */ 1701static inline LLVMValueRef 1702scale_bits(struct gallivm_state *gallivm, 1703 int src_bits, 1704 int dst_bits, 1705 LLVMValueRef src, 1706 struct lp_type src_type) 1707{ 1708 LLVMBuilderRef builder = gallivm->builder; 1709 LLVMValueRef result = src; 1710 1711 if (dst_bits < src_bits) { 1712 int delta_bits = src_bits - dst_bits; 1713 1714 if (delta_bits <= dst_bits) { 1715 1716 if (dst_bits == 4) { 1717 struct lp_type flt_type = lp_type_float_vec(32, src_type.length * 32); 1718 1719 result = lp_build_unsigned_norm_to_float(gallivm, src_bits, flt_type, src); 1720 result = lp_build_clamped_float_to_unsigned_norm(gallivm, flt_type, dst_bits, result); 1721 return result; 1722 } 1723 1724 /* 1725 * Approximate the rescaling with a single shift. 1726 * 1727 * This gives the wrong rounding. 1728 */ 1729 1730 result = LLVMBuildLShr(builder, 1731 src, 1732 lp_build_const_int_vec(gallivm, src_type, delta_bits), 1733 ""); 1734 1735 } else { 1736 /* 1737 * Try more accurate rescaling. 1738 */ 1739 1740 /* 1741 * Drop the least significant bits to make space for the multiplication. 1742 * 1743 * XXX: A better approach would be to use a wider integer type as intermediate. But 1744 * this is enough to convert alpha from 16bits -> 2 when rendering to 1745 * PIPE_FORMAT_R10G10B10A2_UNORM. 1746 */ 1747 result = LLVMBuildLShr(builder, 1748 src, 1749 lp_build_const_int_vec(gallivm, src_type, dst_bits), 1750 ""); 1751 1752 1753 result = LLVMBuildMul(builder, 1754 result, 1755 lp_build_const_int_vec(gallivm, src_type, (1LL << dst_bits) - 1), 1756 ""); 1757 1758 /* 1759 * Add a rounding term before the division. 1760 * 1761 * TODO: Handle signed integers too. 1762 */ 1763 if (!src_type.sign) { 1764 result = LLVMBuildAdd(builder, 1765 result, 1766 lp_build_const_int_vec(gallivm, src_type, (1LL << (delta_bits - 1))), 1767 ""); 1768 } 1769 1770 /* 1771 * Approximate the division by src_mask with a src_bits shift. 1772 * 1773 * Given the src has already been shifted by dst_bits, all we need 1774 * to do is to shift by the difference. 1775 */ 1776 1777 result = LLVMBuildLShr(builder, 1778 result, 1779 lp_build_const_int_vec(gallivm, src_type, delta_bits), 1780 ""); 1781 } 1782 1783 } else if (dst_bits > src_bits) { 1784 /* Scale up bits */ 1785 int db = dst_bits - src_bits; 1786 1787 /* Shift left by difference in bits */ 1788 result = LLVMBuildShl(builder, 1789 src, 1790 lp_build_const_int_vec(gallivm, src_type, db), 1791 ""); 1792 1793 if (db <= src_bits) { 1794 /* Enough bits in src to fill the remainder */ 1795 LLVMValueRef lower = LLVMBuildLShr(builder, 1796 src, 1797 lp_build_const_int_vec(gallivm, src_type, src_bits - db), 1798 ""); 1799 1800 result = LLVMBuildOr(builder, result, lower, ""); 1801 } else if (db > src_bits) { 1802 /* Need to repeatedly copy src bits to fill remainder in dst */ 1803 unsigned n; 1804 1805 for (n = src_bits; n < dst_bits; n *= 2) { 1806 LLVMValueRef shuv = lp_build_const_int_vec(gallivm, src_type, n); 1807 1808 result = LLVMBuildOr(builder, 1809 result, 1810 LLVMBuildLShr(builder, result, shuv, ""), 1811 ""); 1812 } 1813 } 1814 } 1815 1816 return result; 1817} 1818 1819/** 1820 * If RT is a smallfloat (needing denorms) format 1821 */ 1822static inline int 1823have_smallfloat_format(struct lp_type dst_type, 1824 enum pipe_format format) 1825{ 1826 return ((dst_type.floating && dst_type.width != 32) || 1827 /* due to format handling hacks this format doesn't have floating set 1828 * here (and actually has width set to 32 too) so special case this. */ 1829 (format == PIPE_FORMAT_R11G11B10_FLOAT)); 1830} 1831 1832 1833/** 1834 * Convert from memory format to blending format 1835 * 1836 * e.g. GL_R3G3B2 is 1 byte in memory but 3 bytes for blending 1837 */ 1838static void 1839convert_to_blend_type(struct gallivm_state *gallivm, 1840 unsigned block_size, 1841 const struct util_format_description *src_fmt, 1842 struct lp_type src_type, 1843 struct lp_type dst_type, 1844 LLVMValueRef* src, // and dst 1845 unsigned num_srcs) 1846{ 1847 LLVMValueRef *dst = src; 1848 LLVMBuilderRef builder = gallivm->builder; 1849 struct lp_type blend_type; 1850 struct lp_type mem_type; 1851 unsigned i, j; 1852 unsigned pixels = block_size / num_srcs; 1853 bool is_arith; 1854 1855 /* 1856 * full custom path for packed floats and srgb formats - none of the later 1857 * functions would do anything useful, and given the lp_type representation they 1858 * can't be fixed. Should really have some SoA blend path for these kind of 1859 * formats rather than hacking them in here. 1860 */ 1861 if (format_expands_to_float_soa(src_fmt)) { 1862 LLVMValueRef tmpsrc[4]; 1863 /* 1864 * This is pretty suboptimal for this case blending in SoA would be much 1865 * better, since conversion gets us SoA values so need to convert back. 1866 */ 1867 assert(src_type.width == 32 || src_type.width == 16); 1868 assert(dst_type.floating); 1869 assert(dst_type.width == 32); 1870 assert(dst_type.length % 4 == 0); 1871 assert(num_srcs % 4 == 0); 1872 1873 if (src_type.width == 16) { 1874 /* expand 4x16bit values to 4x32bit */ 1875 struct lp_type type32x4 = src_type; 1876 LLVMTypeRef ltype32x4; 1877 unsigned num_fetch = dst_type.length == 8 ? num_srcs / 2 : num_srcs / 4; 1878 type32x4.width = 32; 1879 ltype32x4 = lp_build_vec_type(gallivm, type32x4); 1880 for (i = 0; i < num_fetch; i++) { 1881 src[i] = LLVMBuildZExt(builder, src[i], ltype32x4, ""); 1882 } 1883 src_type.width = 32; 1884 } 1885 for (i = 0; i < 4; i++) { 1886 tmpsrc[i] = src[i]; 1887 } 1888 for (i = 0; i < num_srcs / 4; i++) { 1889 LLVMValueRef tmpsoa[4]; 1890 LLVMValueRef tmps = tmpsrc[i]; 1891 if (dst_type.length == 8) { 1892 LLVMValueRef shuffles[8]; 1893 unsigned j; 1894 /* fetch was 4 values but need 8-wide output values */ 1895 tmps = lp_build_concat(gallivm, &tmpsrc[i * 2], src_type, 2); 1896 /* 1897 * for 8-wide aos transpose would give us wrong order not matching 1898 * incoming converted fs values and mask. ARGH. 1899 */ 1900 for (j = 0; j < 4; j++) { 1901 shuffles[j] = lp_build_const_int32(gallivm, j * 2); 1902 shuffles[j + 4] = lp_build_const_int32(gallivm, j * 2 + 1); 1903 } 1904 tmps = LLVMBuildShuffleVector(builder, tmps, tmps, 1905 LLVMConstVector(shuffles, 8), ""); 1906 } 1907 if (src_fmt->format == PIPE_FORMAT_R11G11B10_FLOAT) { 1908 lp_build_r11g11b10_to_float(gallivm, tmps, tmpsoa); 1909 } 1910 else { 1911 lp_build_unpack_rgba_soa(gallivm, src_fmt, dst_type, tmps, tmpsoa); 1912 } 1913 lp_build_transpose_aos(gallivm, dst_type, tmpsoa, &src[i * 4]); 1914 } 1915 return; 1916 } 1917 1918 lp_mem_type_from_format_desc(src_fmt, &mem_type); 1919 lp_blend_type_from_format_desc(src_fmt, &blend_type); 1920 1921 /* Is the format arithmetic */ 1922 is_arith = blend_type.length * blend_type.width != mem_type.width * mem_type.length; 1923 is_arith &= !(mem_type.width == 16 && mem_type.floating); 1924 1925 /* Pad if necessary */ 1926 if (!is_arith && src_type.length < dst_type.length) { 1927 for (i = 0; i < num_srcs; ++i) { 1928 dst[i] = lp_build_pad_vector(gallivm, src[i], dst_type.length); 1929 } 1930 1931 src_type.length = dst_type.length; 1932 } 1933 1934 /* Special case for half-floats */ 1935 if (mem_type.width == 16 && mem_type.floating) { 1936 assert(blend_type.width == 32 && blend_type.floating); 1937 lp_build_conv_auto(gallivm, src_type, &dst_type, dst, num_srcs, dst); 1938 is_arith = false; 1939 } 1940 1941 if (!is_arith) { 1942 return; 1943 } 1944 1945 src_type.width = blend_type.width * blend_type.length; 1946 blend_type.length *= pixels; 1947 src_type.length *= pixels / (src_type.length / mem_type.length); 1948 1949 for (i = 0; i < num_srcs; ++i) { 1950 LLVMValueRef chans[4]; 1951 LLVMValueRef res = NULL; 1952 1953 dst[i] = LLVMBuildZExt(builder, src[i], lp_build_vec_type(gallivm, src_type), ""); 1954 1955 for (j = 0; j < src_fmt->nr_channels; ++j) { 1956 unsigned mask = 0; 1957 unsigned sa = src_fmt->channel[j].shift; 1958#if UTIL_ARCH_LITTLE_ENDIAN 1959 unsigned from_lsb = j; 1960#else 1961 unsigned from_lsb = src_fmt->nr_channels - j - 1; 1962#endif 1963 1964 mask = (1 << src_fmt->channel[j].size) - 1; 1965 1966 /* Extract bits from source */ 1967 chans[j] = LLVMBuildLShr(builder, 1968 dst[i], 1969 lp_build_const_int_vec(gallivm, src_type, sa), 1970 ""); 1971 1972 chans[j] = LLVMBuildAnd(builder, 1973 chans[j], 1974 lp_build_const_int_vec(gallivm, src_type, mask), 1975 ""); 1976 1977 /* Scale bits */ 1978 if (src_type.norm) { 1979 chans[j] = scale_bits(gallivm, src_fmt->channel[j].size, 1980 blend_type.width, chans[j], src_type); 1981 } 1982 1983 /* Insert bits into correct position */ 1984 chans[j] = LLVMBuildShl(builder, 1985 chans[j], 1986 lp_build_const_int_vec(gallivm, src_type, from_lsb * blend_type.width), 1987 ""); 1988 1989 if (j == 0) { 1990 res = chans[j]; 1991 } else { 1992 res = LLVMBuildOr(builder, res, chans[j], ""); 1993 } 1994 } 1995 1996 dst[i] = LLVMBuildBitCast(builder, res, lp_build_vec_type(gallivm, blend_type), ""); 1997 } 1998} 1999 2000 2001/** 2002 * Convert from blending format to memory format 2003 * 2004 * e.g. GL_R3G3B2 is 3 bytes for blending but 1 byte in memory 2005 */ 2006static void 2007convert_from_blend_type(struct gallivm_state *gallivm, 2008 unsigned block_size, 2009 const struct util_format_description *src_fmt, 2010 struct lp_type src_type, 2011 struct lp_type dst_type, 2012 LLVMValueRef* src, // and dst 2013 unsigned num_srcs) 2014{ 2015 LLVMValueRef* dst = src; 2016 unsigned i, j, k; 2017 struct lp_type mem_type; 2018 struct lp_type blend_type; 2019 LLVMBuilderRef builder = gallivm->builder; 2020 unsigned pixels = block_size / num_srcs; 2021 bool is_arith; 2022 2023 /* 2024 * full custom path for packed floats and srgb formats - none of the later 2025 * functions would do anything useful, and given the lp_type representation they 2026 * can't be fixed. Should really have some SoA blend path for these kind of 2027 * formats rather than hacking them in here. 2028 */ 2029 if (format_expands_to_float_soa(src_fmt)) { 2030 /* 2031 * This is pretty suboptimal for this case blending in SoA would be much 2032 * better - we need to transpose the AoS values back to SoA values for 2033 * conversion/packing. 2034 */ 2035 assert(src_type.floating); 2036 assert(src_type.width == 32); 2037 assert(src_type.length % 4 == 0); 2038 assert(dst_type.width == 32 || dst_type.width == 16); 2039 2040 for (i = 0; i < num_srcs / 4; i++) { 2041 LLVMValueRef tmpsoa[4], tmpdst; 2042 lp_build_transpose_aos(gallivm, src_type, &src[i * 4], tmpsoa); 2043 /* really really need SoA here */ 2044 2045 if (src_fmt->format == PIPE_FORMAT_R11G11B10_FLOAT) { 2046 tmpdst = lp_build_float_to_r11g11b10(gallivm, tmpsoa); 2047 } 2048 else { 2049 tmpdst = lp_build_float_to_srgb_packed(gallivm, src_fmt, 2050 src_type, tmpsoa); 2051 } 2052 2053 if (src_type.length == 8) { 2054 LLVMValueRef tmpaos, shuffles[8]; 2055 unsigned j; 2056 /* 2057 * for 8-wide aos transpose has given us wrong order not matching 2058 * output order. HMPF. Also need to split the output values manually. 2059 */ 2060 for (j = 0; j < 4; j++) { 2061 shuffles[j * 2] = lp_build_const_int32(gallivm, j); 2062 shuffles[j * 2 + 1] = lp_build_const_int32(gallivm, j + 4); 2063 } 2064 tmpaos = LLVMBuildShuffleVector(builder, tmpdst, tmpdst, 2065 LLVMConstVector(shuffles, 8), ""); 2066 src[i * 2] = lp_build_extract_range(gallivm, tmpaos, 0, 4); 2067 src[i * 2 + 1] = lp_build_extract_range(gallivm, tmpaos, 4, 4); 2068 } 2069 else { 2070 src[i] = tmpdst; 2071 } 2072 } 2073 if (dst_type.width == 16) { 2074 struct lp_type type16x8 = dst_type; 2075 struct lp_type type32x4 = dst_type; 2076 LLVMTypeRef ltype16x4, ltypei64, ltypei128; 2077 unsigned num_fetch = src_type.length == 8 ? num_srcs / 2 : num_srcs / 4; 2078 type16x8.length = 8; 2079 type32x4.width = 32; 2080 ltypei128 = LLVMIntTypeInContext(gallivm->context, 128); 2081 ltypei64 = LLVMIntTypeInContext(gallivm->context, 64); 2082 ltype16x4 = lp_build_vec_type(gallivm, dst_type); 2083 /* We could do vector truncation but it doesn't generate very good code */ 2084 for (i = 0; i < num_fetch; i++) { 2085 src[i] = lp_build_pack2(gallivm, type32x4, type16x8, 2086 src[i], lp_build_zero(gallivm, type32x4)); 2087 src[i] = LLVMBuildBitCast(builder, src[i], ltypei128, ""); 2088 src[i] = LLVMBuildTrunc(builder, src[i], ltypei64, ""); 2089 src[i] = LLVMBuildBitCast(builder, src[i], ltype16x4, ""); 2090 } 2091 } 2092 return; 2093 } 2094 2095 lp_mem_type_from_format_desc(src_fmt, &mem_type); 2096 lp_blend_type_from_format_desc(src_fmt, &blend_type); 2097 2098 is_arith = (blend_type.length * blend_type.width != mem_type.width * mem_type.length); 2099 2100 /* Special case for half-floats */ 2101 if (mem_type.width == 16 && mem_type.floating) { 2102 int length = dst_type.length; 2103 assert(blend_type.width == 32 && blend_type.floating); 2104 2105 dst_type.length = src_type.length; 2106 2107 lp_build_conv_auto(gallivm, src_type, &dst_type, dst, num_srcs, dst); 2108 2109 dst_type.length = length; 2110 is_arith = false; 2111 } 2112 2113 /* Remove any padding */ 2114 if (!is_arith && (src_type.length % mem_type.length)) { 2115 src_type.length -= (src_type.length % mem_type.length); 2116 2117 for (i = 0; i < num_srcs; ++i) { 2118 dst[i] = lp_build_extract_range(gallivm, dst[i], 0, src_type.length); 2119 } 2120 } 2121 2122 /* No bit arithmetic to do */ 2123 if (!is_arith) { 2124 return; 2125 } 2126 2127 src_type.length = pixels; 2128 src_type.width = blend_type.length * blend_type.width; 2129 dst_type.length = pixels; 2130 2131 for (i = 0; i < num_srcs; ++i) { 2132 LLVMValueRef chans[4]; 2133 LLVMValueRef res = NULL; 2134 2135 dst[i] = LLVMBuildBitCast(builder, src[i], lp_build_vec_type(gallivm, src_type), ""); 2136 2137 for (j = 0; j < src_fmt->nr_channels; ++j) { 2138 unsigned mask = 0; 2139 unsigned sa = src_fmt->channel[j].shift; 2140 unsigned sz_a = src_fmt->channel[j].size; 2141#if UTIL_ARCH_LITTLE_ENDIAN 2142 unsigned from_lsb = j; 2143#else 2144 unsigned from_lsb = src_fmt->nr_channels - j - 1; 2145#endif 2146 2147 assert(blend_type.width > src_fmt->channel[j].size); 2148 2149 for (k = 0; k < blend_type.width; ++k) { 2150 mask |= 1 << k; 2151 } 2152 2153 /* Extract bits */ 2154 chans[j] = LLVMBuildLShr(builder, 2155 dst[i], 2156 lp_build_const_int_vec(gallivm, src_type, 2157 from_lsb * blend_type.width), 2158 ""); 2159 2160 chans[j] = LLVMBuildAnd(builder, 2161 chans[j], 2162 lp_build_const_int_vec(gallivm, src_type, mask), 2163 ""); 2164 2165 /* Scale down bits */ 2166 if (src_type.norm) { 2167 chans[j] = scale_bits(gallivm, blend_type.width, 2168 src_fmt->channel[j].size, chans[j], src_type); 2169 } else if (!src_type.floating && sz_a < blend_type.width) { 2170 LLVMValueRef mask_val = lp_build_const_int_vec(gallivm, src_type, (1UL << sz_a) - 1); 2171 LLVMValueRef mask = LLVMBuildICmp(builder, LLVMIntUGT, chans[j], mask_val, ""); 2172 chans[j] = LLVMBuildSelect(builder, mask, mask_val, chans[j], ""); 2173 } 2174 2175 /* Insert bits */ 2176 chans[j] = LLVMBuildShl(builder, 2177 chans[j], 2178 lp_build_const_int_vec(gallivm, src_type, sa), 2179 ""); 2180 2181 sa += src_fmt->channel[j].size; 2182 2183 if (j == 0) { 2184 res = chans[j]; 2185 } else { 2186 res = LLVMBuildOr(builder, res, chans[j], ""); 2187 } 2188 } 2189 2190 assert (dst_type.width != 24); 2191 2192 dst[i] = LLVMBuildTrunc(builder, res, lp_build_vec_type(gallivm, dst_type), ""); 2193 } 2194} 2195 2196 2197/** 2198 * Convert alpha to same blend type as src 2199 */ 2200static void 2201convert_alpha(struct gallivm_state *gallivm, 2202 struct lp_type row_type, 2203 struct lp_type alpha_type, 2204 const unsigned block_size, 2205 const unsigned block_height, 2206 const unsigned src_count, 2207 const unsigned dst_channels, 2208 const bool pad_inline, 2209 LLVMValueRef* src_alpha) 2210{ 2211 LLVMBuilderRef builder = gallivm->builder; 2212 unsigned i, j; 2213 unsigned length = row_type.length; 2214 row_type.length = alpha_type.length; 2215 2216 /* Twiddle the alpha to match pixels */ 2217 lp_bld_quad_twiddle(gallivm, alpha_type, src_alpha, block_height, src_alpha); 2218 2219 /* 2220 * TODO this should use single lp_build_conv call for 2221 * src_count == 1 && dst_channels == 1 case (dropping the concat below) 2222 */ 2223 for (i = 0; i < block_height; ++i) { 2224 lp_build_conv(gallivm, alpha_type, row_type, &src_alpha[i], 1, &src_alpha[i], 1); 2225 } 2226 2227 alpha_type = row_type; 2228 row_type.length = length; 2229 2230 /* If only one channel we can only need the single alpha value per pixel */ 2231 if (src_count == 1 && dst_channels == 1) { 2232 2233 lp_build_concat_n(gallivm, alpha_type, src_alpha, block_height, src_alpha, src_count); 2234 } else { 2235 /* If there are more srcs than rows then we need to split alpha up */ 2236 if (src_count > block_height) { 2237 for (i = src_count; i > 0; --i) { 2238 unsigned pixels = block_size / src_count; 2239 unsigned idx = i - 1; 2240 2241 src_alpha[idx] = lp_build_extract_range(gallivm, src_alpha[(idx * pixels) / 4], 2242 (idx * pixels) % 4, pixels); 2243 } 2244 } 2245 2246 /* If there is a src for each pixel broadcast the alpha across whole row */ 2247 if (src_count == block_size) { 2248 for (i = 0; i < src_count; ++i) { 2249 src_alpha[i] = lp_build_broadcast(gallivm, 2250 lp_build_vec_type(gallivm, row_type), src_alpha[i]); 2251 } 2252 } else { 2253 unsigned pixels = block_size / src_count; 2254 unsigned channels = pad_inline ? TGSI_NUM_CHANNELS : dst_channels; 2255 unsigned alpha_span = 1; 2256 LLVMValueRef shuffles[LP_MAX_VECTOR_LENGTH]; 2257 2258 /* Check if we need 2 src_alphas for our shuffles */ 2259 if (pixels > alpha_type.length) { 2260 alpha_span = 2; 2261 } 2262 2263 /* Broadcast alpha across all channels, e.g. a1a2 to a1a1a1a1a2a2a2a2 */ 2264 for (j = 0; j < row_type.length; ++j) { 2265 if (j < pixels * channels) { 2266 shuffles[j] = lp_build_const_int32(gallivm, j / channels); 2267 } else { 2268 shuffles[j] = LLVMGetUndef(LLVMInt32TypeInContext(gallivm->context)); 2269 } 2270 } 2271 2272 for (i = 0; i < src_count; ++i) { 2273 unsigned idx1 = i, idx2 = i; 2274 2275 if (alpha_span > 1){ 2276 idx1 *= alpha_span; 2277 idx2 = idx1 + 1; 2278 } 2279 2280 src_alpha[i] = LLVMBuildShuffleVector(builder, 2281 src_alpha[idx1], 2282 src_alpha[idx2], 2283 LLVMConstVector(shuffles, row_type.length), 2284 ""); 2285 } 2286 } 2287 } 2288} 2289 2290 2291/** 2292 * Generates the blend function for unswizzled colour buffers 2293 * Also generates the read & write from colour buffer 2294 */ 2295static void 2296generate_unswizzled_blend(struct gallivm_state *gallivm, 2297 unsigned rt, 2298 struct lp_fragment_shader_variant *variant, 2299 enum pipe_format out_format, 2300 unsigned int num_fs, 2301 struct lp_type fs_type, 2302 LLVMValueRef* fs_mask, 2303 LLVMValueRef fs_out_color[PIPE_MAX_COLOR_BUFS][TGSI_NUM_CHANNELS][4], 2304 LLVMValueRef context_ptr, 2305 LLVMValueRef color_ptr, 2306 LLVMValueRef stride, 2307 unsigned partial_mask, 2308 boolean do_branch) 2309{ 2310 const unsigned alpha_channel = 3; 2311 const unsigned block_width = LP_RASTER_BLOCK_SIZE; 2312 const unsigned block_height = LP_RASTER_BLOCK_SIZE; 2313 const unsigned block_size = block_width * block_height; 2314 const unsigned lp_integer_vector_width = 128; 2315 2316 LLVMBuilderRef builder = gallivm->builder; 2317 LLVMValueRef fs_src[4][TGSI_NUM_CHANNELS]; 2318 LLVMValueRef fs_src1[4][TGSI_NUM_CHANNELS]; 2319 LLVMValueRef src_alpha[4 * 4]; 2320 LLVMValueRef src1_alpha[4 * 4] = { NULL }; 2321 LLVMValueRef src_mask[4 * 4]; 2322 LLVMValueRef src[4 * 4]; 2323 LLVMValueRef src1[4 * 4]; 2324 LLVMValueRef dst[4 * 4]; 2325 LLVMValueRef blend_color; 2326 LLVMValueRef blend_alpha; 2327 LLVMValueRef i32_zero; 2328 LLVMValueRef check_mask; 2329 LLVMValueRef undef_src_val; 2330 2331 struct lp_build_mask_context mask_ctx; 2332 struct lp_type mask_type; 2333 struct lp_type blend_type; 2334 struct lp_type row_type; 2335 struct lp_type dst_type; 2336 struct lp_type ls_type; 2337 2338 unsigned char swizzle[TGSI_NUM_CHANNELS]; 2339 unsigned vector_width; 2340 unsigned src_channels = TGSI_NUM_CHANNELS; 2341 unsigned dst_channels; 2342 unsigned dst_count; 2343 unsigned src_count; 2344 unsigned i, j; 2345 2346 const struct util_format_description* out_format_desc = util_format_description(out_format); 2347 2348 unsigned dst_alignment; 2349 2350 bool pad_inline = is_arithmetic_format(out_format_desc); 2351 bool has_alpha = false; 2352 const boolean dual_source_blend = variant->key.blend.rt[0].blend_enable && 2353 util_blend_state_is_dual(&variant->key.blend, 0); 2354 2355 const boolean is_1d = variant->key.resource_1d; 2356 boolean twiddle_after_convert = FALSE; 2357 unsigned num_fullblock_fs = is_1d ? 2 * num_fs : num_fs; 2358 LLVMValueRef fpstate = 0; 2359 2360 /* Get type from output format */ 2361 lp_blend_type_from_format_desc(out_format_desc, &row_type); 2362 lp_mem_type_from_format_desc(out_format_desc, &dst_type); 2363 2364 /* 2365 * Technically this code should go into lp_build_smallfloat_to_float 2366 * and lp_build_float_to_smallfloat but due to the 2367 * http://llvm.org/bugs/show_bug.cgi?id=6393 2368 * llvm reorders the mxcsr intrinsics in a way that breaks the code. 2369 * So the ordering is important here and there shouldn't be any 2370 * llvm ir instrunctions in this function before 2371 * this, otherwise half-float format conversions won't work 2372 * (again due to llvm bug #6393). 2373 */ 2374 if (have_smallfloat_format(dst_type, out_format)) { 2375 /* We need to make sure that denorms are ok for half float 2376 conversions */ 2377 fpstate = lp_build_fpstate_get(gallivm); 2378 lp_build_fpstate_set_denorms_zero(gallivm, FALSE); 2379 } 2380 2381 mask_type = lp_int32_vec4_type(); 2382 mask_type.length = fs_type.length; 2383 2384 for (i = num_fs; i < num_fullblock_fs; i++) { 2385 fs_mask[i] = lp_build_zero(gallivm, mask_type); 2386 } 2387 2388 /* Do not bother executing code when mask is empty.. */ 2389 if (do_branch) { 2390 check_mask = LLVMConstNull(lp_build_int_vec_type(gallivm, mask_type)); 2391 2392 for (i = 0; i < num_fullblock_fs; ++i) { 2393 check_mask = LLVMBuildOr(builder, check_mask, fs_mask[i], ""); 2394 } 2395 2396 lp_build_mask_begin(&mask_ctx, gallivm, mask_type, check_mask); 2397 lp_build_mask_check(&mask_ctx); 2398 } 2399 2400 partial_mask |= !variant->opaque; 2401 i32_zero = lp_build_const_int32(gallivm, 0); 2402 2403 undef_src_val = lp_build_undef(gallivm, fs_type); 2404 2405 row_type.length = fs_type.length; 2406 vector_width = dst_type.floating ? lp_native_vector_width : lp_integer_vector_width; 2407 2408 /* Compute correct swizzle and count channels */ 2409 memset(swizzle, LP_BLD_SWIZZLE_DONTCARE, TGSI_NUM_CHANNELS); 2410 dst_channels = 0; 2411 2412 for (i = 0; i < TGSI_NUM_CHANNELS; ++i) { 2413 /* Ensure channel is used */ 2414 if (out_format_desc->swizzle[i] >= TGSI_NUM_CHANNELS) { 2415 continue; 2416 } 2417 2418 /* Ensure not already written to (happens in case with GL_ALPHA) */ 2419 if (swizzle[out_format_desc->swizzle[i]] < TGSI_NUM_CHANNELS) { 2420 continue; 2421 } 2422 2423 /* Ensure we haven't already found all channels */ 2424 if (dst_channels >= out_format_desc->nr_channels) { 2425 continue; 2426 } 2427 2428 swizzle[out_format_desc->swizzle[i]] = i; 2429 ++dst_channels; 2430 2431 if (i == alpha_channel) { 2432 has_alpha = true; 2433 } 2434 } 2435 2436 if (format_expands_to_float_soa(out_format_desc)) { 2437 /* 2438 * the code above can't work for layout_other 2439 * for srgb it would sort of work but we short-circuit swizzles, etc. 2440 * as that is done as part of unpack / pack. 2441 */ 2442 dst_channels = 4; /* HACK: this is fake 4 really but need it due to transpose stuff later */ 2443 has_alpha = true; 2444 swizzle[0] = 0; 2445 swizzle[1] = 1; 2446 swizzle[2] = 2; 2447 swizzle[3] = 3; 2448 pad_inline = true; /* HACK: prevent rgbxrgbx->rgbrgbxx conversion later */ 2449 } 2450 2451 /* If 3 channels then pad to include alpha for 4 element transpose */ 2452 if (dst_channels == 3) { 2453 assert (!has_alpha); 2454 for (i = 0; i < TGSI_NUM_CHANNELS; i++) { 2455 if (swizzle[i] > TGSI_NUM_CHANNELS) 2456 swizzle[i] = 3; 2457 } 2458 if (out_format_desc->nr_channels == 4) { 2459 dst_channels = 4; 2460 /* 2461 * We use alpha from the color conversion, not separate one. 2462 * We had to include it for transpose, hence it will get converted 2463 * too (albeit when doing transpose after conversion, that would 2464 * no longer be the case necessarily). 2465 * (It works only with 4 channel dsts, e.g. rgbx formats, because 2466 * otherwise we really have padding, not alpha, included.) 2467 */ 2468 has_alpha = true; 2469 } 2470 } 2471 2472 /* 2473 * Load shader output 2474 */ 2475 for (i = 0; i < num_fullblock_fs; ++i) { 2476 /* Always load alpha for use in blending */ 2477 LLVMValueRef alpha; 2478 if (i < num_fs) { 2479 alpha = LLVMBuildLoad(builder, fs_out_color[rt][alpha_channel][i], ""); 2480 } 2481 else { 2482 alpha = undef_src_val; 2483 } 2484 2485 /* Load each channel */ 2486 for (j = 0; j < dst_channels; ++j) { 2487 assert(swizzle[j] < 4); 2488 if (i < num_fs) { 2489 fs_src[i][j] = LLVMBuildLoad(builder, fs_out_color[rt][swizzle[j]][i], ""); 2490 } 2491 else { 2492 fs_src[i][j] = undef_src_val; 2493 } 2494 } 2495 2496 /* If 3 channels then pad to include alpha for 4 element transpose */ 2497 /* 2498 * XXX If we include that here maybe could actually use it instead of 2499 * separate alpha for blending? 2500 * (Difficult though we actually convert pad channels, not alpha.) 2501 */ 2502 if (dst_channels == 3 && !has_alpha) { 2503 fs_src[i][3] = alpha; 2504 } 2505 2506 /* We split the row_mask and row_alpha as we want 128bit interleave */ 2507 if (fs_type.length == 8) { 2508 src_mask[i*2 + 0] = lp_build_extract_range(gallivm, fs_mask[i], 2509 0, src_channels); 2510 src_mask[i*2 + 1] = lp_build_extract_range(gallivm, fs_mask[i], 2511 src_channels, src_channels); 2512 2513 src_alpha[i*2 + 0] = lp_build_extract_range(gallivm, alpha, 0, src_channels); 2514 src_alpha[i*2 + 1] = lp_build_extract_range(gallivm, alpha, 2515 src_channels, src_channels); 2516 } else { 2517 src_mask[i] = fs_mask[i]; 2518 src_alpha[i] = alpha; 2519 } 2520 } 2521 if (dual_source_blend) { 2522 /* same as above except different src/dst, skip masks and comments... */ 2523 for (i = 0; i < num_fullblock_fs; ++i) { 2524 LLVMValueRef alpha; 2525 if (i < num_fs) { 2526 alpha = LLVMBuildLoad(builder, fs_out_color[1][alpha_channel][i], ""); 2527 } 2528 else { 2529 alpha = undef_src_val; 2530 } 2531 2532 for (j = 0; j < dst_channels; ++j) { 2533 assert(swizzle[j] < 4); 2534 if (i < num_fs) { 2535 fs_src1[i][j] = LLVMBuildLoad(builder, fs_out_color[1][swizzle[j]][i], ""); 2536 } 2537 else { 2538 fs_src1[i][j] = undef_src_val; 2539 } 2540 } 2541 if (dst_channels == 3 && !has_alpha) { 2542 fs_src1[i][3] = alpha; 2543 } 2544 if (fs_type.length == 8) { 2545 src1_alpha[i*2 + 0] = lp_build_extract_range(gallivm, alpha, 0, src_channels); 2546 src1_alpha[i*2 + 1] = lp_build_extract_range(gallivm, alpha, 2547 src_channels, src_channels); 2548 } else { 2549 src1_alpha[i] = alpha; 2550 } 2551 } 2552 } 2553 2554 if (util_format_is_pure_integer(out_format)) { 2555 /* 2556 * In this case fs_type was really ints or uints disguised as floats, 2557 * fix that up now. 2558 */ 2559 fs_type.floating = 0; 2560 fs_type.sign = dst_type.sign; 2561 for (i = 0; i < num_fullblock_fs; ++i) { 2562 for (j = 0; j < dst_channels; ++j) { 2563 fs_src[i][j] = LLVMBuildBitCast(builder, fs_src[i][j], 2564 lp_build_vec_type(gallivm, fs_type), ""); 2565 } 2566 if (dst_channels == 3 && !has_alpha) { 2567 fs_src[i][3] = LLVMBuildBitCast(builder, fs_src[i][3], 2568 lp_build_vec_type(gallivm, fs_type), ""); 2569 } 2570 } 2571 } 2572 2573 /* 2574 * We actually should generally do conversion first (for non-1d cases) 2575 * when the blend format is 8 or 16 bits. The reason is obvious, 2576 * there's 2 or 4 times less vectors to deal with for the interleave... 2577 * Albeit for the AVX (not AVX2) case there's no benefit with 16 bit 2578 * vectors (as it can do 32bit unpack with 256bit vectors, but 8/16bit 2579 * unpack only with 128bit vectors). 2580 * Note: for 16bit sizes really need matching pack conversion code 2581 */ 2582 if (!is_1d && dst_channels != 3 && dst_type.width == 8) { 2583 twiddle_after_convert = TRUE; 2584 } 2585 2586 /* 2587 * Pixel twiddle from fragment shader order to memory order 2588 */ 2589 if (!twiddle_after_convert) { 2590 src_count = generate_fs_twiddle(gallivm, fs_type, num_fullblock_fs, 2591 dst_channels, fs_src, src, pad_inline); 2592 if (dual_source_blend) { 2593 generate_fs_twiddle(gallivm, fs_type, num_fullblock_fs, dst_channels, 2594 fs_src1, src1, pad_inline); 2595 } 2596 } else { 2597 src_count = num_fullblock_fs * dst_channels; 2598 /* 2599 * We reorder things a bit here, so the cases for 4-wide and 8-wide 2600 * (AVX) turn out the same later when untwiddling/transpose (albeit 2601 * for true AVX2 path untwiddle needs to be different). 2602 * For now just order by colors first (so we can use unpack later). 2603 */ 2604 for (j = 0; j < num_fullblock_fs; j++) { 2605 for (i = 0; i < dst_channels; i++) { 2606 src[i*num_fullblock_fs + j] = fs_src[j][i]; 2607 if (dual_source_blend) { 2608 src1[i*num_fullblock_fs + j] = fs_src1[j][i]; 2609 } 2610 } 2611 } 2612 } 2613 2614 src_channels = dst_channels < 3 ? dst_channels : 4; 2615 if (src_count != num_fullblock_fs * src_channels) { 2616 unsigned ds = src_count / (num_fullblock_fs * src_channels); 2617 row_type.length /= ds; 2618 fs_type.length = row_type.length; 2619 } 2620 2621 blend_type = row_type; 2622 mask_type.length = 4; 2623 2624 /* Convert src to row_type */ 2625 if (dual_source_blend) { 2626 struct lp_type old_row_type = row_type; 2627 lp_build_conv_auto(gallivm, fs_type, &row_type, src, src_count, src); 2628 src_count = lp_build_conv_auto(gallivm, fs_type, &old_row_type, src1, src_count, src1); 2629 } 2630 else { 2631 src_count = lp_build_conv_auto(gallivm, fs_type, &row_type, src, src_count, src); 2632 } 2633 2634 /* If the rows are not an SSE vector, combine them to become SSE size! */ 2635 if ((row_type.width * row_type.length) % 128) { 2636 unsigned bits = row_type.width * row_type.length; 2637 unsigned combined; 2638 2639 assert(src_count >= (vector_width / bits)); 2640 2641 dst_count = src_count / (vector_width / bits); 2642 2643 combined = lp_build_concat_n(gallivm, row_type, src, src_count, src, dst_count); 2644 if (dual_source_blend) { 2645 lp_build_concat_n(gallivm, row_type, src1, src_count, src1, dst_count); 2646 } 2647 2648 row_type.length *= combined; 2649 src_count /= combined; 2650 2651 bits = row_type.width * row_type.length; 2652 assert(bits == 128 || bits == 256); 2653 } 2654 2655 if (twiddle_after_convert) { 2656 fs_twiddle_transpose(gallivm, row_type, src, src_count, src); 2657 if (dual_source_blend) { 2658 fs_twiddle_transpose(gallivm, row_type, src1, src_count, src1); 2659 } 2660 } 2661 2662 /* 2663 * Blend Colour conversion 2664 */ 2665 blend_color = lp_jit_context_f_blend_color(gallivm, context_ptr); 2666 blend_color = LLVMBuildPointerCast(builder, blend_color, 2667 LLVMPointerType(lp_build_vec_type(gallivm, fs_type), 0), ""); 2668 blend_color = LLVMBuildLoad(builder, LLVMBuildGEP(builder, blend_color, 2669 &i32_zero, 1, ""), ""); 2670 2671 /* Convert */ 2672 lp_build_conv(gallivm, fs_type, blend_type, &blend_color, 1, &blend_color, 1); 2673 2674 if (out_format_desc->colorspace == UTIL_FORMAT_COLORSPACE_SRGB) { 2675 /* 2676 * since blending is done with floats, there was no conversion. 2677 * However, the rules according to fixed point renderbuffers still 2678 * apply, that is we must clamp inputs to 0.0/1.0. 2679 * (This would apply to separate alpha conversion too but we currently 2680 * force has_alpha to be true.) 2681 * TODO: should skip this with "fake" blend, since post-blend conversion 2682 * will clamp anyway. 2683 * TODO: could also skip this if fragment color clamping is enabled. We 2684 * don't support it natively so it gets baked into the shader however, so 2685 * can't really tell here. 2686 */ 2687 struct lp_build_context f32_bld; 2688 assert(row_type.floating); 2689 lp_build_context_init(&f32_bld, gallivm, row_type); 2690 for (i = 0; i < src_count; i++) { 2691 src[i] = lp_build_clamp_zero_one_nanzero(&f32_bld, src[i]); 2692 } 2693 if (dual_source_blend) { 2694 for (i = 0; i < src_count; i++) { 2695 src1[i] = lp_build_clamp_zero_one_nanzero(&f32_bld, src1[i]); 2696 } 2697 } 2698 /* probably can't be different than row_type but better safe than sorry... */ 2699 lp_build_context_init(&f32_bld, gallivm, blend_type); 2700 blend_color = lp_build_clamp(&f32_bld, blend_color, f32_bld.zero, f32_bld.one); 2701 } 2702 2703 /* Extract alpha */ 2704 blend_alpha = lp_build_extract_broadcast(gallivm, blend_type, row_type, blend_color, lp_build_const_int32(gallivm, 3)); 2705 2706 /* Swizzle to appropriate channels, e.g. from RGBA to BGRA BGRA */ 2707 pad_inline &= (dst_channels * (block_size / src_count) * row_type.width) != vector_width; 2708 if (pad_inline) { 2709 /* Use all 4 channels e.g. from RGBA RGBA to RGxx RGxx */ 2710 blend_color = lp_build_swizzle_aos_n(gallivm, blend_color, swizzle, TGSI_NUM_CHANNELS, row_type.length); 2711 } else { 2712 /* Only use dst_channels e.g. RGBA RGBA to RG RG xxxx */ 2713 blend_color = lp_build_swizzle_aos_n(gallivm, blend_color, swizzle, dst_channels, row_type.length); 2714 } 2715 2716 /* 2717 * Mask conversion 2718 */ 2719 lp_bld_quad_twiddle(gallivm, mask_type, &src_mask[0], block_height, &src_mask[0]); 2720 2721 if (src_count < block_height) { 2722 lp_build_concat_n(gallivm, mask_type, src_mask, 4, src_mask, src_count); 2723 } else if (src_count > block_height) { 2724 for (i = src_count; i > 0; --i) { 2725 unsigned pixels = block_size / src_count; 2726 unsigned idx = i - 1; 2727 2728 src_mask[idx] = lp_build_extract_range(gallivm, src_mask[(idx * pixels) / 4], 2729 (idx * pixels) % 4, pixels); 2730 } 2731 } 2732 2733 assert(mask_type.width == 32); 2734 2735 for (i = 0; i < src_count; ++i) { 2736 unsigned pixels = block_size / src_count; 2737 unsigned pixel_width = row_type.width * dst_channels; 2738 2739 if (pixel_width == 24) { 2740 mask_type.width = 8; 2741 mask_type.length = vector_width / mask_type.width; 2742 } else { 2743 mask_type.length = pixels; 2744 mask_type.width = row_type.width * dst_channels; 2745 2746 /* 2747 * If mask_type width is smaller than 32bit, this doesn't quite 2748 * generate the most efficient code (could use some pack). 2749 */ 2750 src_mask[i] = LLVMBuildIntCast(builder, src_mask[i], 2751 lp_build_int_vec_type(gallivm, mask_type), ""); 2752 2753 mask_type.length *= dst_channels; 2754 mask_type.width /= dst_channels; 2755 } 2756 2757 src_mask[i] = LLVMBuildBitCast(builder, src_mask[i], 2758 lp_build_int_vec_type(gallivm, mask_type), ""); 2759 src_mask[i] = lp_build_pad_vector(gallivm, src_mask[i], row_type.length); 2760 } 2761 2762 /* 2763 * Alpha conversion 2764 */ 2765 if (!has_alpha) { 2766 struct lp_type alpha_type = fs_type; 2767 alpha_type.length = 4; 2768 convert_alpha(gallivm, row_type, alpha_type, 2769 block_size, block_height, 2770 src_count, dst_channels, 2771 pad_inline, src_alpha); 2772 if (dual_source_blend) { 2773 convert_alpha(gallivm, row_type, alpha_type, 2774 block_size, block_height, 2775 src_count, dst_channels, 2776 pad_inline, src1_alpha); 2777 } 2778 } 2779 2780 2781 /* 2782 * Load dst from memory 2783 */ 2784 if (src_count < block_height) { 2785 dst_count = block_height; 2786 } else { 2787 dst_count = src_count; 2788 } 2789 2790 dst_type.length *= block_size / dst_count; 2791 2792 if (format_expands_to_float_soa(out_format_desc)) { 2793 /* 2794 * we need multiple values at once for the conversion, so can as well 2795 * load them vectorized here too instead of concatenating later. 2796 * (Still need concatenation later for 8-wide vectors). 2797 */ 2798 dst_count = block_height; 2799 dst_type.length = block_width; 2800 } 2801 2802 /* 2803 * Compute the alignment of the destination pointer in bytes 2804 * We fetch 1-4 pixels, if the format has pot alignment then those fetches 2805 * are always aligned by MIN2(16, fetch_width) except for buffers (not 2806 * 1d tex but can't distinguish here) so need to stick with per-pixel 2807 * alignment in this case. 2808 */ 2809 if (is_1d) { 2810 dst_alignment = (out_format_desc->block.bits + 7)/(out_format_desc->block.width * 8); 2811 } 2812 else { 2813 dst_alignment = dst_type.length * dst_type.width / 8; 2814 } 2815 /* Force power-of-two alignment by extracting only the least-significant-bit */ 2816 dst_alignment = 1 << (ffs(dst_alignment) - 1); 2817 /* 2818 * Resource base and stride pointers are aligned to 16 bytes, so that's 2819 * the maximum alignment we can guarantee 2820 */ 2821 dst_alignment = MIN2(16, dst_alignment); 2822 2823 ls_type = dst_type; 2824 2825 if (dst_count > src_count) { 2826 if ((dst_type.width == 8 || dst_type.width == 16) && 2827 util_is_power_of_two_or_zero(dst_type.length) && 2828 dst_type.length * dst_type.width < 128) { 2829 /* 2830 * Never try to load values as 4xi8 which we will then 2831 * concatenate to larger vectors. This gives llvm a real 2832 * headache (the problem is the type legalizer (?) will 2833 * try to load that as 4xi8 zext to 4xi32 to fill the vector, 2834 * then the shuffles to concatenate are more or less impossible 2835 * - llvm is easily capable of generating a sequence of 32 2836 * pextrb/pinsrb instructions for that. Albeit it appears to 2837 * be fixed in llvm 4.0. So, load and concatenate with 32bit 2838 * width to avoid the trouble (16bit seems not as bad, llvm 2839 * probably recognizes the load+shuffle as only one shuffle 2840 * is necessary, but we can do just the same anyway). 2841 */ 2842 ls_type.length = dst_type.length * dst_type.width / 32; 2843 ls_type.width = 32; 2844 } 2845 } 2846 2847 if (is_1d) { 2848 load_unswizzled_block(gallivm, color_ptr, stride, block_width, 1, 2849 dst, ls_type, dst_count / 4, dst_alignment, NULL, NULL, false); 2850 for (i = dst_count / 4; i < dst_count; i++) { 2851 dst[i] = lp_build_undef(gallivm, ls_type); 2852 } 2853 2854 } 2855 else { 2856 load_unswizzled_block(gallivm, color_ptr, stride, block_width, block_height, 2857 dst, ls_type, dst_count, dst_alignment, NULL, NULL, false); 2858 } 2859 2860 2861 /* 2862 * Convert from dst/output format to src/blending format. 2863 * 2864 * This is necessary as we can only read 1 row from memory at a time, 2865 * so the minimum dst_count will ever be at this point is 4. 2866 * 2867 * With, for example, R8 format you can have all 16 pixels in a 128 bit vector, 2868 * this will take the 4 dsts and combine them into 1 src so we can perform blending 2869 * on all 16 pixels in that single vector at once. 2870 */ 2871 if (dst_count > src_count) { 2872 if (ls_type.length != dst_type.length && ls_type.length == 1) { 2873 LLVMTypeRef elem_type = lp_build_elem_type(gallivm, ls_type); 2874 LLVMTypeRef ls_vec_type = LLVMVectorType(elem_type, 1); 2875 for (i = 0; i < dst_count; i++) { 2876 dst[i] = LLVMBuildBitCast(builder, dst[i], ls_vec_type, ""); 2877 } 2878 } 2879 2880 lp_build_concat_n(gallivm, ls_type, dst, 4, dst, src_count); 2881 2882 if (ls_type.length != dst_type.length) { 2883 struct lp_type tmp_type = dst_type; 2884 tmp_type.length = dst_type.length * 4 / src_count; 2885 for (i = 0; i < src_count; i++) { 2886 dst[i] = LLVMBuildBitCast(builder, dst[i], 2887 lp_build_vec_type(gallivm, tmp_type), ""); 2888 } 2889 } 2890 } 2891 2892 /* 2893 * Blending 2894 */ 2895 /* XXX this is broken for RGB8 formats - 2896 * they get expanded from 12 to 16 elements (to include alpha) 2897 * by convert_to_blend_type then reduced to 15 instead of 12 2898 * by convert_from_blend_type (a simple fix though breaks A8...). 2899 * R16G16B16 also crashes differently however something going wrong 2900 * inside llvm handling npot vector sizes seemingly. 2901 * It seems some cleanup could be done here (like skipping conversion/blend 2902 * when not needed). 2903 */ 2904 convert_to_blend_type(gallivm, block_size, out_format_desc, dst_type, 2905 row_type, dst, src_count); 2906 2907 /* 2908 * FIXME: Really should get logic ops / masks out of generic blend / row 2909 * format. Logic ops will definitely not work on the blend float format 2910 * used for SRGB here and I think OpenGL expects this to work as expected 2911 * (that is incoming values converted to srgb then logic op applied). 2912 */ 2913 for (i = 0; i < src_count; ++i) { 2914 dst[i] = lp_build_blend_aos(gallivm, 2915 &variant->key.blend, 2916 out_format, 2917 row_type, 2918 rt, 2919 src[i], 2920 has_alpha ? NULL : src_alpha[i], 2921 src1[i], 2922 has_alpha ? NULL : src1_alpha[i], 2923 dst[i], 2924 partial_mask ? src_mask[i] : NULL, 2925 blend_color, 2926 has_alpha ? NULL : blend_alpha, 2927 swizzle, 2928 pad_inline ? 4 : dst_channels); 2929 } 2930 2931 convert_from_blend_type(gallivm, block_size, out_format_desc, 2932 row_type, dst_type, dst, src_count); 2933 2934 /* Split the blend rows back to memory rows */ 2935 if (dst_count > src_count) { 2936 row_type.length = dst_type.length * (dst_count / src_count); 2937 2938 if (src_count == 1) { 2939 dst[1] = lp_build_extract_range(gallivm, dst[0], row_type.length / 2, row_type.length / 2); 2940 dst[0] = lp_build_extract_range(gallivm, dst[0], 0, row_type.length / 2); 2941 2942 row_type.length /= 2; 2943 src_count *= 2; 2944 } 2945 2946 dst[3] = lp_build_extract_range(gallivm, dst[1], row_type.length / 2, row_type.length / 2); 2947 dst[2] = lp_build_extract_range(gallivm, dst[1], 0, row_type.length / 2); 2948 dst[1] = lp_build_extract_range(gallivm, dst[0], row_type.length / 2, row_type.length / 2); 2949 dst[0] = lp_build_extract_range(gallivm, dst[0], 0, row_type.length / 2); 2950 2951 row_type.length /= 2; 2952 src_count *= 2; 2953 } 2954 2955 /* 2956 * Store blend result to memory 2957 */ 2958 if (is_1d) { 2959 store_unswizzled_block(gallivm, color_ptr, stride, block_width, 1, 2960 dst, dst_type, dst_count / 4, dst_alignment); 2961 } 2962 else { 2963 store_unswizzled_block(gallivm, color_ptr, stride, block_width, block_height, 2964 dst, dst_type, dst_count, dst_alignment); 2965 } 2966 2967 if (have_smallfloat_format(dst_type, out_format)) { 2968 lp_build_fpstate_set(gallivm, fpstate); 2969 } 2970 2971 if (do_branch) { 2972 lp_build_mask_end(&mask_ctx); 2973 } 2974} 2975 2976 2977/** 2978 * Generate the runtime callable function for the whole fragment pipeline. 2979 * Note that the function which we generate operates on a block of 16 2980 * pixels at at time. The block contains 2x2 quads. Each quad contains 2981 * 2x2 pixels. 2982 */ 2983static void 2984generate_fragment(struct llvmpipe_context *lp, 2985 struct lp_fragment_shader *shader, 2986 struct lp_fragment_shader_variant *variant, 2987 unsigned partial_mask) 2988{ 2989 struct gallivm_state *gallivm = variant->gallivm; 2990 struct lp_fragment_shader_variant_key *key = &variant->key; 2991 struct lp_shader_input inputs[PIPE_MAX_SHADER_INPUTS]; 2992 char func_name[64]; 2993 struct lp_type fs_type; 2994 struct lp_type blend_type; 2995 LLVMTypeRef fs_elem_type; 2996 LLVMTypeRef blend_vec_type; 2997 LLVMTypeRef arg_types[15]; 2998 LLVMTypeRef func_type; 2999 LLVMTypeRef int32_type = LLVMInt32TypeInContext(gallivm->context); 3000 LLVMTypeRef int8_type = LLVMInt8TypeInContext(gallivm->context); 3001 LLVMValueRef context_ptr; 3002 LLVMValueRef x; 3003 LLVMValueRef y; 3004 LLVMValueRef a0_ptr; 3005 LLVMValueRef dadx_ptr; 3006 LLVMValueRef dady_ptr; 3007 LLVMValueRef color_ptr_ptr; 3008 LLVMValueRef stride_ptr; 3009 LLVMValueRef color_sample_stride_ptr; 3010 LLVMValueRef depth_ptr; 3011 LLVMValueRef depth_stride; 3012 LLVMValueRef depth_sample_stride; 3013 LLVMValueRef mask_input; 3014 LLVMValueRef thread_data_ptr; 3015 LLVMBasicBlockRef block; 3016 LLVMBuilderRef builder; 3017 struct lp_build_sampler_soa *sampler; 3018 struct lp_build_image_soa *image; 3019 struct lp_build_interp_soa_context interp; 3020 LLVMValueRef fs_mask[(16 / 4) * LP_MAX_SAMPLES]; 3021 LLVMValueRef fs_out_color[LP_MAX_SAMPLES][PIPE_MAX_COLOR_BUFS][TGSI_NUM_CHANNELS][16 / 4]; 3022 LLVMValueRef function; 3023 LLVMValueRef facing; 3024 unsigned num_fs; 3025 unsigned i; 3026 unsigned chan; 3027 unsigned cbuf; 3028 boolean cbuf0_write_all; 3029 const boolean dual_source_blend = key->blend.rt[0].blend_enable && 3030 util_blend_state_is_dual(&key->blend, 0); 3031 3032 assert(lp_native_vector_width / 32 >= 4); 3033 3034 /* Adjust color input interpolation according to flatshade state: 3035 */ 3036 memcpy(inputs, shader->inputs, shader->info.base.num_inputs * sizeof inputs[0]); 3037 for (i = 0; i < shader->info.base.num_inputs; i++) { 3038 if (inputs[i].interp == LP_INTERP_COLOR) { 3039 if (key->flatshade) 3040 inputs[i].interp = LP_INTERP_CONSTANT; 3041 else 3042 inputs[i].interp = LP_INTERP_PERSPECTIVE; 3043 } 3044 } 3045 3046 /* check if writes to cbuf[0] are to be copied to all cbufs */ 3047 cbuf0_write_all = 3048 shader->info.base.properties[TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS]; 3049 3050 /* TODO: actually pick these based on the fs and color buffer 3051 * characteristics. */ 3052 3053 memset(&fs_type, 0, sizeof fs_type); 3054 fs_type.floating = TRUE; /* floating point values */ 3055 fs_type.sign = TRUE; /* values are signed */ 3056 fs_type.norm = FALSE; /* values are not limited to [0,1] or [-1,1] */ 3057 fs_type.width = 32; /* 32-bit float */ 3058 fs_type.length = MIN2(lp_native_vector_width / 32, 16); /* n*4 elements per vector */ 3059 3060 memset(&blend_type, 0, sizeof blend_type); 3061 blend_type.floating = FALSE; /* values are integers */ 3062 blend_type.sign = FALSE; /* values are unsigned */ 3063 blend_type.norm = TRUE; /* values are in [0,1] or [-1,1] */ 3064 blend_type.width = 8; /* 8-bit ubyte values */ 3065 blend_type.length = 16; /* 16 elements per vector */ 3066 3067 /* 3068 * Generate the function prototype. Any change here must be reflected in 3069 * lp_jit.h's lp_jit_frag_func function pointer type, and vice-versa. 3070 */ 3071 3072 fs_elem_type = lp_build_elem_type(gallivm, fs_type); 3073 3074 blend_vec_type = lp_build_vec_type(gallivm, blend_type); 3075 3076 snprintf(func_name, sizeof(func_name), "fs_variant_%s", 3077 partial_mask ? "partial" : "whole"); 3078 3079 arg_types[0] = variant->jit_context_ptr_type; /* context */ 3080 arg_types[1] = int32_type; /* x */ 3081 arg_types[2] = int32_type; /* y */ 3082 arg_types[3] = int32_type; /* facing */ 3083 arg_types[4] = LLVMPointerType(fs_elem_type, 0); /* a0 */ 3084 arg_types[5] = LLVMPointerType(fs_elem_type, 0); /* dadx */ 3085 arg_types[6] = LLVMPointerType(fs_elem_type, 0); /* dady */ 3086 arg_types[7] = LLVMPointerType(LLVMPointerType(int8_type, 0), 0); /* color */ 3087 arg_types[8] = LLVMPointerType(int8_type, 0); /* depth */ 3088 arg_types[9] = LLVMInt64TypeInContext(gallivm->context); /* mask_input */ 3089 arg_types[10] = variant->jit_thread_data_ptr_type; /* per thread data */ 3090 arg_types[11] = LLVMPointerType(int32_type, 0); /* stride */ 3091 arg_types[12] = int32_type; /* depth_stride */ 3092 arg_types[13] = LLVMPointerType(int32_type, 0); /* color sample strides */ 3093 arg_types[14] = int32_type; /* depth sample stride */ 3094 3095 func_type = LLVMFunctionType(LLVMVoidTypeInContext(gallivm->context), 3096 arg_types, ARRAY_SIZE(arg_types), 0); 3097 3098 function = LLVMAddFunction(gallivm->module, func_name, func_type); 3099 LLVMSetFunctionCallConv(function, LLVMCCallConv); 3100 3101 variant->function[partial_mask] = function; 3102 3103 /* XXX: need to propagate noalias down into color param now we are 3104 * passing a pointer-to-pointer? 3105 */ 3106 for(i = 0; i < ARRAY_SIZE(arg_types); ++i) 3107 if(LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind) 3108 lp_add_function_attr(function, i + 1, LP_FUNC_ATTR_NOALIAS); 3109 3110 if (variant->gallivm->cache->data_size) 3111 return; 3112 3113 context_ptr = LLVMGetParam(function, 0); 3114 x = LLVMGetParam(function, 1); 3115 y = LLVMGetParam(function, 2); 3116 facing = LLVMGetParam(function, 3); 3117 a0_ptr = LLVMGetParam(function, 4); 3118 dadx_ptr = LLVMGetParam(function, 5); 3119 dady_ptr = LLVMGetParam(function, 6); 3120 color_ptr_ptr = LLVMGetParam(function, 7); 3121 depth_ptr = LLVMGetParam(function, 8); 3122 mask_input = LLVMGetParam(function, 9); 3123 thread_data_ptr = LLVMGetParam(function, 10); 3124 stride_ptr = LLVMGetParam(function, 11); 3125 depth_stride = LLVMGetParam(function, 12); 3126 color_sample_stride_ptr = LLVMGetParam(function, 13); 3127 depth_sample_stride = LLVMGetParam(function, 14); 3128 3129 lp_build_name(context_ptr, "context"); 3130 lp_build_name(x, "x"); 3131 lp_build_name(y, "y"); 3132 lp_build_name(a0_ptr, "a0"); 3133 lp_build_name(dadx_ptr, "dadx"); 3134 lp_build_name(dady_ptr, "dady"); 3135 lp_build_name(color_ptr_ptr, "color_ptr_ptr"); 3136 lp_build_name(depth_ptr, "depth"); 3137 lp_build_name(mask_input, "mask_input"); 3138 lp_build_name(thread_data_ptr, "thread_data"); 3139 lp_build_name(stride_ptr, "stride_ptr"); 3140 lp_build_name(depth_stride, "depth_stride"); 3141 lp_build_name(color_sample_stride_ptr, "color_sample_stride_ptr"); 3142 lp_build_name(depth_sample_stride, "depth_sample_stride"); 3143 3144 /* 3145 * Function body 3146 */ 3147 3148 block = LLVMAppendBasicBlockInContext(gallivm->context, function, "entry"); 3149 builder = gallivm->builder; 3150 assert(builder); 3151 LLVMPositionBuilderAtEnd(builder, block); 3152 3153 /* 3154 * Must not count ps invocations if there's a null shader. 3155 * (It would be ok to count with null shader if there's d/s tests, 3156 * but only if there's d/s buffers too, which is different 3157 * to implicit rasterization disable which must not depend 3158 * on the d/s buffers.) 3159 * Could use popcount on mask, but pixel accuracy is not required. 3160 * Could disable if there's no stats query, but maybe not worth it. 3161 */ 3162 if (shader->info.base.num_instructions > 1) { 3163 LLVMValueRef invocs, val; 3164 invocs = lp_jit_thread_data_invocations(gallivm, thread_data_ptr); 3165 val = LLVMBuildLoad(builder, invocs, ""); 3166 val = LLVMBuildAdd(builder, val, 3167 LLVMConstInt(LLVMInt64TypeInContext(gallivm->context), 1, 0), 3168 "invoc_count"); 3169 LLVMBuildStore(builder, val, invocs); 3170 } 3171 3172 /* code generated texture sampling */ 3173 sampler = lp_llvm_sampler_soa_create(lp_fs_variant_key_samplers(key), key->nr_samplers); 3174 image = lp_llvm_image_soa_create(lp_fs_variant_key_images(key), key->nr_images); 3175 3176 num_fs = 16 / fs_type.length; /* number of loops per 4x4 stamp */ 3177 /* for 1d resources only run "upper half" of stamp */ 3178 if (key->resource_1d) 3179 num_fs /= 2; 3180 3181 { 3182 LLVMValueRef num_loop = lp_build_const_int32(gallivm, num_fs); 3183 LLVMTypeRef mask_type = lp_build_int_vec_type(gallivm, fs_type); 3184 LLVMValueRef num_loop_samp = lp_build_const_int32(gallivm, num_fs * key->coverage_samples); 3185 LLVMValueRef mask_store = lp_build_array_alloca(gallivm, mask_type, 3186 num_loop_samp, "mask_store"); 3187 3188 LLVMTypeRef flt_type = LLVMFloatTypeInContext(gallivm->context); 3189 LLVMValueRef glob_sample_pos = LLVMAddGlobal(gallivm->module, LLVMArrayType(flt_type, key->coverage_samples * 2), ""); 3190 LLVMValueRef sample_pos_array; 3191 3192 if (key->multisample && key->coverage_samples == 4) { 3193 LLVMValueRef sample_pos_arr[8]; 3194 for (unsigned i = 0; i < 4; i++) { 3195 sample_pos_arr[i * 2] = LLVMConstReal(flt_type, lp_sample_pos_4x[i][0]); 3196 sample_pos_arr[i * 2 + 1] = LLVMConstReal(flt_type, lp_sample_pos_4x[i][1]); 3197 } 3198 sample_pos_array = LLVMConstArray(LLVMFloatTypeInContext(gallivm->context), sample_pos_arr, 8); 3199 } else { 3200 LLVMValueRef sample_pos_arr[2]; 3201 sample_pos_arr[0] = LLVMConstReal(flt_type, 0.5); 3202 sample_pos_arr[1] = LLVMConstReal(flt_type, 0.5); 3203 sample_pos_array = LLVMConstArray(LLVMFloatTypeInContext(gallivm->context), sample_pos_arr, 2); 3204 } 3205 LLVMSetInitializer(glob_sample_pos, sample_pos_array); 3206 3207 LLVMValueRef color_store[PIPE_MAX_COLOR_BUFS][TGSI_NUM_CHANNELS]; 3208 boolean pixel_center_integer = 3209 shader->info.base.properties[TGSI_PROPERTY_FS_COORD_PIXEL_CENTER]; 3210 3211 /* 3212 * The shader input interpolation info is not explicitely baked in the 3213 * shader key, but everything it derives from (TGSI, and flatshade) is 3214 * already included in the shader key. 3215 */ 3216 lp_build_interp_soa_init(&interp, 3217 gallivm, 3218 shader->info.base.num_inputs, 3219 inputs, 3220 pixel_center_integer, 3221 key->coverage_samples, glob_sample_pos, 3222 num_loop, 3223 key->depth_clamp, 3224 builder, fs_type, 3225 a0_ptr, dadx_ptr, dady_ptr, 3226 x, y); 3227 3228 for (i = 0; i < num_fs; i++) { 3229 if (key->multisample) { 3230 LLVMValueRef smask_val = LLVMBuildLoad(builder, lp_jit_context_sample_mask(gallivm, context_ptr), ""); 3231 3232 /* 3233 * For multisampling, extract the per-sample mask from the incoming 64-bit mask, 3234 * store to the per sample mask storage. Or all of them together to generate 3235 * the fragment shader mask. (sample shading TODO). 3236 * Take the incoming state coverage mask into account. 3237 */ 3238 for (unsigned s = 0; s < key->coverage_samples; s++) { 3239 LLVMValueRef sindexi = lp_build_const_int32(gallivm, i + (s * num_fs)); 3240 LLVMValueRef sample_mask_ptr = LLVMBuildGEP(builder, mask_store, 3241 &sindexi, 1, "sample_mask_ptr"); 3242 LLVMValueRef s_mask = generate_quad_mask(gallivm, fs_type, 3243 i*fs_type.length/4, s, mask_input); 3244 3245 LLVMValueRef smask_bit = LLVMBuildAnd(builder, smask_val, lp_build_const_int32(gallivm, (1 << s)), ""); 3246 LLVMValueRef cmp = LLVMBuildICmp(builder, LLVMIntNE, smask_bit, lp_build_const_int32(gallivm, 0), ""); 3247 smask_bit = LLVMBuildSExt(builder, cmp, int32_type, ""); 3248 smask_bit = lp_build_broadcast(gallivm, mask_type, smask_bit); 3249 3250 s_mask = LLVMBuildAnd(builder, s_mask, smask_bit, ""); 3251 LLVMBuildStore(builder, s_mask, sample_mask_ptr); 3252 } 3253 } else { 3254 LLVMValueRef mask; 3255 LLVMValueRef indexi = lp_build_const_int32(gallivm, i); 3256 LLVMValueRef mask_ptr = LLVMBuildGEP(builder, mask_store, 3257 &indexi, 1, "mask_ptr"); 3258 3259 if (partial_mask) { 3260 mask = generate_quad_mask(gallivm, fs_type, 3261 i*fs_type.length/4, 0, mask_input); 3262 } 3263 else { 3264 mask = lp_build_const_int_vec(gallivm, fs_type, ~0); 3265 } 3266 LLVMBuildStore(builder, mask, mask_ptr); 3267 } 3268 } 3269 3270 generate_fs_loop(gallivm, 3271 shader, key, 3272 builder, 3273 fs_type, 3274 context_ptr, 3275 glob_sample_pos, 3276 num_loop, 3277 &interp, 3278 sampler, 3279 image, 3280 mask_store, /* output */ 3281 color_store, 3282 depth_ptr, 3283 depth_stride, 3284 depth_sample_stride, 3285 color_ptr_ptr, 3286 stride_ptr, 3287 color_sample_stride_ptr, 3288 facing, 3289 thread_data_ptr); 3290 3291 for (i = 0; i < num_fs; i++) { 3292 LLVMValueRef ptr; 3293 for (unsigned s = 0; s < key->coverage_samples; s++) { 3294 int idx = (i + (s * num_fs)); 3295 LLVMValueRef sindexi = lp_build_const_int32(gallivm, idx); 3296 ptr = LLVMBuildGEP(builder, mask_store, &sindexi, 1, ""); 3297 3298 fs_mask[idx] = LLVMBuildLoad(builder, ptr, "smask"); 3299 } 3300 3301 for (unsigned s = 0; s < key->min_samples; s++) { 3302 /* This is fucked up need to reorganize things */ 3303 int idx = s * num_fs + i; 3304 LLVMValueRef sindexi = lp_build_const_int32(gallivm, idx); 3305 for (cbuf = 0; cbuf < key->nr_cbufs; cbuf++) { 3306 for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) { 3307 ptr = LLVMBuildGEP(builder, 3308 color_store[cbuf * !cbuf0_write_all][chan], 3309 &sindexi, 1, ""); 3310 fs_out_color[s][cbuf][chan][i] = ptr; 3311 } 3312 } 3313 if (dual_source_blend) { 3314 /* only support one dual source blend target hence always use output 1 */ 3315 for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) { 3316 ptr = LLVMBuildGEP(builder, 3317 color_store[1][chan], 3318 &sindexi, 1, ""); 3319 fs_out_color[s][1][chan][i] = ptr; 3320 } 3321 } 3322 } 3323 } 3324 } 3325 3326 sampler->destroy(sampler); 3327 image->destroy(image); 3328 /* Loop over color outputs / color buffers to do blending. 3329 */ 3330 for(cbuf = 0; cbuf < key->nr_cbufs; cbuf++) { 3331 if (key->cbuf_format[cbuf] != PIPE_FORMAT_NONE) { 3332 LLVMValueRef color_ptr; 3333 LLVMValueRef stride; 3334 LLVMValueRef sample_stride = NULL; 3335 LLVMValueRef index = lp_build_const_int32(gallivm, cbuf); 3336 3337 boolean do_branch = ((key->depth.enabled 3338 || key->stencil[0].enabled 3339 || key->alpha.enabled) 3340 && !shader->info.base.uses_kill); 3341 3342 color_ptr = LLVMBuildLoad(builder, 3343 LLVMBuildGEP(builder, color_ptr_ptr, 3344 &index, 1, ""), 3345 ""); 3346 3347 stride = LLVMBuildLoad(builder, 3348 LLVMBuildGEP(builder, stride_ptr, &index, 1, ""), 3349 ""); 3350 3351 if (key->cbuf_nr_samples[cbuf] > 1) 3352 sample_stride = LLVMBuildLoad(builder, 3353 LLVMBuildGEP(builder, color_sample_stride_ptr, 3354 &index, 1, ""), ""); 3355 3356 for (unsigned s = 0; s < key->cbuf_nr_samples[cbuf]; s++) { 3357 unsigned mask_idx = num_fs * (key->multisample ? s : 0); 3358 unsigned out_idx = key->min_samples == 1 ? 0 : s; 3359 LLVMValueRef out_ptr = color_ptr;; 3360 3361 if (sample_stride) { 3362 LLVMValueRef sample_offset = LLVMBuildMul(builder, sample_stride, lp_build_const_int32(gallivm, s), ""); 3363 out_ptr = LLVMBuildGEP(builder, out_ptr, &sample_offset, 1, ""); 3364 } 3365 out_ptr = LLVMBuildBitCast(builder, out_ptr, LLVMPointerType(blend_vec_type, 0), ""); 3366 3367 lp_build_name(out_ptr, "color_ptr%d", cbuf); 3368 3369 generate_unswizzled_blend(gallivm, cbuf, variant, 3370 key->cbuf_format[cbuf], 3371 num_fs, fs_type, &fs_mask[mask_idx], fs_out_color[out_idx], 3372 context_ptr, out_ptr, stride, 3373 partial_mask, do_branch); 3374 } 3375 } 3376 } 3377 3378 LLVMBuildRetVoid(builder); 3379 3380 gallivm_verify_function(gallivm, function); 3381} 3382 3383 3384static void 3385dump_fs_variant_key(struct lp_fragment_shader_variant_key *key) 3386{ 3387 unsigned i; 3388 3389 debug_printf("fs variant %p:\n", (void *) key); 3390 3391 if (key->flatshade) { 3392 debug_printf("flatshade = 1\n"); 3393 } 3394 if (key->multisample) { 3395 debug_printf("multisample = 1\n"); 3396 debug_printf("coverage samples = %d\n", key->coverage_samples); 3397 debug_printf("min samples = %d\n", key->min_samples); 3398 } 3399 for (i = 0; i < key->nr_cbufs; ++i) { 3400 debug_printf("cbuf_format[%u] = %s\n", i, util_format_name(key->cbuf_format[i])); 3401 debug_printf("cbuf nr_samples[%u] = %d\n", i, key->cbuf_nr_samples[i]); 3402 } 3403 if (key->depth.enabled || key->stencil[0].enabled) { 3404 debug_printf("depth.format = %s\n", util_format_name(key->zsbuf_format)); 3405 debug_printf("depth nr_samples = %d\n", key->zsbuf_nr_samples); 3406 } 3407 if (key->depth.enabled) { 3408 debug_printf("depth.func = %s\n", util_str_func(key->depth.func, TRUE)); 3409 debug_printf("depth.writemask = %u\n", key->depth.writemask); 3410 } 3411 3412 for (i = 0; i < 2; ++i) { 3413 if (key->stencil[i].enabled) { 3414 debug_printf("stencil[%u].func = %s\n", i, util_str_func(key->stencil[i].func, TRUE)); 3415 debug_printf("stencil[%u].fail_op = %s\n", i, util_str_stencil_op(key->stencil[i].fail_op, TRUE)); 3416 debug_printf("stencil[%u].zpass_op = %s\n", i, util_str_stencil_op(key->stencil[i].zpass_op, TRUE)); 3417 debug_printf("stencil[%u].zfail_op = %s\n", i, util_str_stencil_op(key->stencil[i].zfail_op, TRUE)); 3418 debug_printf("stencil[%u].valuemask = 0x%x\n", i, key->stencil[i].valuemask); 3419 debug_printf("stencil[%u].writemask = 0x%x\n", i, key->stencil[i].writemask); 3420 } 3421 } 3422 3423 if (key->alpha.enabled) { 3424 debug_printf("alpha.func = %s\n", util_str_func(key->alpha.func, TRUE)); 3425 } 3426 3427 if (key->occlusion_count) { 3428 debug_printf("occlusion_count = 1\n"); 3429 } 3430 3431 if (key->blend.logicop_enable) { 3432 debug_printf("blend.logicop_func = %s\n", util_str_logicop(key->blend.logicop_func, TRUE)); 3433 } 3434 else if (key->blend.rt[0].blend_enable) { 3435 debug_printf("blend.rgb_func = %s\n", util_str_blend_func (key->blend.rt[0].rgb_func, TRUE)); 3436 debug_printf("blend.rgb_src_factor = %s\n", util_str_blend_factor(key->blend.rt[0].rgb_src_factor, TRUE)); 3437 debug_printf("blend.rgb_dst_factor = %s\n", util_str_blend_factor(key->blend.rt[0].rgb_dst_factor, TRUE)); 3438 debug_printf("blend.alpha_func = %s\n", util_str_blend_func (key->blend.rt[0].alpha_func, TRUE)); 3439 debug_printf("blend.alpha_src_factor = %s\n", util_str_blend_factor(key->blend.rt[0].alpha_src_factor, TRUE)); 3440 debug_printf("blend.alpha_dst_factor = %s\n", util_str_blend_factor(key->blend.rt[0].alpha_dst_factor, TRUE)); 3441 } 3442 debug_printf("blend.colormask = 0x%x\n", key->blend.rt[0].colormask); 3443 if (key->blend.alpha_to_coverage) { 3444 debug_printf("blend.alpha_to_coverage is enabled\n"); 3445 } 3446 for (i = 0; i < key->nr_samplers; ++i) { 3447 const struct lp_sampler_static_state *samplers = lp_fs_variant_key_samplers(key); 3448 const struct lp_static_sampler_state *sampler = &samplers[i].sampler_state; 3449 debug_printf("sampler[%u] = \n", i); 3450 debug_printf(" .wrap = %s %s %s\n", 3451 util_str_tex_wrap(sampler->wrap_s, TRUE), 3452 util_str_tex_wrap(sampler->wrap_t, TRUE), 3453 util_str_tex_wrap(sampler->wrap_r, TRUE)); 3454 debug_printf(" .min_img_filter = %s\n", 3455 util_str_tex_filter(sampler->min_img_filter, TRUE)); 3456 debug_printf(" .min_mip_filter = %s\n", 3457 util_str_tex_mipfilter(sampler->min_mip_filter, TRUE)); 3458 debug_printf(" .mag_img_filter = %s\n", 3459 util_str_tex_filter(sampler->mag_img_filter, TRUE)); 3460 if (sampler->compare_mode != PIPE_TEX_COMPARE_NONE) 3461 debug_printf(" .compare_func = %s\n", util_str_func(sampler->compare_func, TRUE)); 3462 debug_printf(" .normalized_coords = %u\n", sampler->normalized_coords); 3463 debug_printf(" .min_max_lod_equal = %u\n", sampler->min_max_lod_equal); 3464 debug_printf(" .lod_bias_non_zero = %u\n", sampler->lod_bias_non_zero); 3465 debug_printf(" .apply_min_lod = %u\n", sampler->apply_min_lod); 3466 debug_printf(" .apply_max_lod = %u\n", sampler->apply_max_lod); 3467 debug_printf(" .reduction_mode = %u\n", sampler->reduction_mode); 3468 debug_printf(" .aniso = %u\n", sampler->aniso); 3469 } 3470 for (i = 0; i < key->nr_sampler_views; ++i) { 3471 const struct lp_sampler_static_state *samplers = lp_fs_variant_key_samplers(key); 3472 const struct lp_static_texture_state *texture = &samplers[i].texture_state; 3473 debug_printf("texture[%u] = \n", i); 3474 debug_printf(" .format = %s\n", 3475 util_format_name(texture->format)); 3476 debug_printf(" .target = %s\n", 3477 util_str_tex_target(texture->target, TRUE)); 3478 debug_printf(" .level_zero_only = %u\n", 3479 texture->level_zero_only); 3480 debug_printf(" .pot = %u %u %u\n", 3481 texture->pot_width, 3482 texture->pot_height, 3483 texture->pot_depth); 3484 } 3485 struct lp_image_static_state *images = lp_fs_variant_key_images(key); 3486 for (i = 0; i < key->nr_images; ++i) { 3487 const struct lp_static_texture_state *image = &images[i].image_state; 3488 debug_printf("image[%u] = \n", i); 3489 debug_printf(" .format = %s\n", 3490 util_format_name(image->format)); 3491 debug_printf(" .target = %s\n", 3492 util_str_tex_target(image->target, TRUE)); 3493 debug_printf(" .level_zero_only = %u\n", 3494 image->level_zero_only); 3495 debug_printf(" .pot = %u %u %u\n", 3496 image->pot_width, 3497 image->pot_height, 3498 image->pot_depth); 3499 } 3500} 3501 3502const char * 3503lp_debug_fs_kind(enum lp_fs_kind kind) 3504{ 3505 switch(kind) { 3506 case LP_FS_KIND_GENERAL: 3507 return "GENERAL"; 3508 case LP_FS_KIND_BLIT_RGBA: 3509 return "BLIT_RGBA"; 3510 case LP_FS_KIND_BLIT_RGB1: 3511 return "BLIT_RGB1"; 3512 case LP_FS_KIND_AERO_MINIFICATION: 3513 return "AERO_MINIFICATION"; 3514 case LP_FS_KIND_LLVM_LINEAR: 3515 return "LLVM_LINEAR"; 3516 default: 3517 return "unknown"; 3518 } 3519} 3520 3521void 3522lp_debug_fs_variant(struct lp_fragment_shader_variant *variant) 3523{ 3524 debug_printf("llvmpipe: Fragment shader #%u variant #%u:\n", 3525 variant->shader->no, variant->no); 3526 if (variant->shader->base.type == PIPE_SHADER_IR_TGSI) 3527 tgsi_dump(variant->shader->base.tokens, 0); 3528 else 3529 nir_print_shader(variant->shader->base.ir.nir, stderr); 3530 dump_fs_variant_key(&variant->key); 3531 debug_printf("variant->opaque = %u\n", variant->opaque); 3532 debug_printf("variant->potentially_opaque = %u\n", variant->potentially_opaque); 3533 debug_printf("variant->blit = %u\n", variant->blit); 3534 debug_printf("shader->kind = %s\n", lp_debug_fs_kind(variant->shader->kind)); 3535 debug_printf("\n"); 3536} 3537 3538static void 3539lp_fs_get_ir_cache_key(struct lp_fragment_shader_variant *variant, 3540 unsigned char ir_sha1_cache_key[20]) 3541{ 3542 struct blob blob = { 0 }; 3543 unsigned ir_size; 3544 void *ir_binary; 3545 3546 blob_init(&blob); 3547 nir_serialize(&blob, variant->shader->base.ir.nir, true); 3548 ir_binary = blob.data; 3549 ir_size = blob.size; 3550 3551 struct mesa_sha1 ctx; 3552 _mesa_sha1_init(&ctx); 3553 _mesa_sha1_update(&ctx, &variant->key, variant->shader->variant_key_size); 3554 _mesa_sha1_update(&ctx, ir_binary, ir_size); 3555 _mesa_sha1_final(&ctx, ir_sha1_cache_key); 3556 3557 blob_finish(&blob); 3558} 3559 3560/** 3561 * Generate a new fragment shader variant from the shader code and 3562 * other state indicated by the key. 3563 */ 3564static struct lp_fragment_shader_variant * 3565generate_variant(struct llvmpipe_context *lp, 3566 struct lp_fragment_shader *shader, 3567 const struct lp_fragment_shader_variant_key *key) 3568{ 3569 struct llvmpipe_screen *screen = llvmpipe_screen(lp->pipe.screen); 3570 struct lp_fragment_shader_variant *variant; 3571 const struct util_format_description *cbuf0_format_desc = NULL; 3572 boolean fullcolormask; 3573 boolean no_kill; 3574 boolean linear; 3575 char module_name[64]; 3576 unsigned char ir_sha1_cache_key[20]; 3577 struct lp_cached_code cached = { 0 }; 3578 bool needs_caching = false; 3579 variant = MALLOC(sizeof *variant + shader->variant_key_size - sizeof variant->key); 3580 if (!variant) 3581 return NULL; 3582 3583 memset(variant, 0, sizeof(*variant)); 3584 snprintf(module_name, sizeof(module_name), "fs%u_variant%u", 3585 shader->no, shader->variants_created); 3586 3587 pipe_reference_init(&variant->reference, 1); 3588 lp_fs_reference(lp, &variant->shader, shader); 3589 3590 memcpy(&variant->key, key, shader->variant_key_size); 3591 3592 if (shader->base.ir.nir) { 3593 lp_fs_get_ir_cache_key(variant, ir_sha1_cache_key); 3594 3595 lp_disk_cache_find_shader(screen, &cached, ir_sha1_cache_key); 3596 if (!cached.data_size) 3597 needs_caching = true; 3598 } 3599 variant->gallivm = gallivm_create(module_name, lp->context, &cached); 3600 if (!variant->gallivm) { 3601 FREE(variant); 3602 return NULL; 3603 } 3604 3605 variant->list_item_global.base = variant; 3606 variant->list_item_local.base = variant; 3607 variant->no = shader->variants_created++; 3608 3609 3610 3611 /* 3612 * Determine whether we are touching all channels in the color buffer. 3613 */ 3614 fullcolormask = FALSE; 3615 if (key->nr_cbufs == 1) { 3616 cbuf0_format_desc = util_format_description(key->cbuf_format[0]); 3617 fullcolormask = util_format_colormask_full(cbuf0_format_desc, key->blend.rt[0].colormask); 3618 } 3619 3620 /* The scissor is ignored here as only tiles inside the scissoring 3621 * rectangle will refer to this */ 3622 no_kill = 3623 fullcolormask && 3624 !key->stencil[0].enabled && 3625 !key->alpha.enabled && 3626 !key->multisample && 3627 !key->blend.alpha_to_coverage && 3628 !key->depth.enabled && 3629 !shader->info.base.uses_kill && 3630 !shader->info.base.writes_samplemask; 3631 3632 variant->opaque = 3633 no_kill && 3634 !key->blend.logicop_enable && 3635 !key->blend.rt[0].blend_enable 3636 ? TRUE : FALSE; 3637 3638 variant->potentially_opaque = 3639 no_kill && 3640 !key->blend.logicop_enable && 3641 key->blend.rt[0].blend_enable && 3642 key->blend.rt[0].rgb_func == PIPE_BLEND_ADD && 3643 key->blend.rt[0].rgb_dst_factor == PIPE_BLENDFACTOR_INV_SRC_ALPHA && 3644 key->blend.rt[0].alpha_func == key->blend.rt[0].rgb_func && 3645 key->blend.rt[0].alpha_dst_factor == key->blend.rt[0].rgb_dst_factor && 3646 shader->base.type == PIPE_SHADER_IR_TGSI && 3647 /* 3648 * FIXME: for NIR, all of the fields of info.xxx (except info.base) 3649 * are zeros, hence shader analysis (here and elsewhere) using these 3650 * bits cannot work and will silently fail (cbuf is the only pointer 3651 * field, hence causing a crash). 3652 */ 3653 shader->info.cbuf[0][3].file != TGSI_FILE_NULL 3654 ? TRUE : FALSE; 3655 3656 /* We only care about opaque blits for now */ 3657 if (variant->opaque && 3658 (shader->kind == LP_FS_KIND_BLIT_RGBA || 3659 shader->kind == LP_FS_KIND_BLIT_RGB1)) { 3660 unsigned target, min_img_filter, mag_img_filter, min_mip_filter; 3661 enum pipe_format texture_format; 3662 struct lp_sampler_static_state *samp0 = lp_fs_variant_key_sampler_idx(key, 0); 3663 assert(samp0); 3664 texture_format = samp0->texture_state.format; 3665 target = samp0->texture_state.target; 3666 min_img_filter = samp0->sampler_state.min_img_filter; 3667 mag_img_filter = samp0->sampler_state.mag_img_filter; 3668 if (samp0->texture_state.level_zero_only) { 3669 min_mip_filter = PIPE_TEX_MIPFILTER_NONE; 3670 } else { 3671 min_mip_filter = samp0->sampler_state.min_mip_filter; 3672 } 3673 3674 if (target == PIPE_TEXTURE_2D && 3675 min_img_filter == PIPE_TEX_FILTER_NEAREST && 3676 mag_img_filter == PIPE_TEX_FILTER_NEAREST && 3677 min_mip_filter == PIPE_TEX_MIPFILTER_NONE && 3678 ((texture_format && 3679 util_is_format_compatible(util_format_description(texture_format), 3680 cbuf0_format_desc)) || 3681 (shader->kind == LP_FS_KIND_BLIT_RGB1 && 3682 (texture_format == PIPE_FORMAT_B8G8R8A8_UNORM || 3683 texture_format == PIPE_FORMAT_B8G8R8X8_UNORM) && 3684 (key->cbuf_format[0] == PIPE_FORMAT_B8G8R8A8_UNORM || 3685 key->cbuf_format[0] == PIPE_FORMAT_B8G8R8X8_UNORM)))) 3686 variant->blit = 1; 3687 } 3688 3689 3690 /* Whether this is a candidate for the linear path */ 3691 linear = 3692 !key->stencil[0].enabled && 3693 !key->depth.enabled && 3694 !shader->info.base.uses_kill && 3695 !key->blend.logicop_enable && 3696 (key->cbuf_format[0] == PIPE_FORMAT_B8G8R8A8_UNORM || 3697 key->cbuf_format[0] == PIPE_FORMAT_B8G8R8X8_UNORM); 3698 3699 memcpy(&variant->key, key, sizeof *key); 3700 3701 if ((LP_DEBUG & DEBUG_FS) || (gallivm_debug & GALLIVM_DEBUG_IR)) { 3702 lp_debug_fs_variant(variant); 3703 } 3704 3705 llvmpipe_fs_variant_fastpath(variant); 3706 3707 lp_jit_init_types(variant); 3708 3709 if (variant->jit_function[RAST_EDGE_TEST] == NULL) 3710 generate_fragment(lp, shader, variant, RAST_EDGE_TEST); 3711 3712 if (variant->jit_function[RAST_WHOLE] == NULL) { 3713 if (variant->opaque) { 3714 /* Specialized shader, which doesn't need to read the color buffer. */ 3715 generate_fragment(lp, shader, variant, RAST_WHOLE); 3716 } 3717 } 3718 3719 if (linear) { 3720 /* Currently keeping both the old fastpaths and new linear path 3721 * active. The older code is still somewhat faster for the cases 3722 * it covers. 3723 * 3724 * XXX: consider restricting this to aero-mode only. 3725 */ 3726 if (fullcolormask && 3727 !key->alpha.enabled && 3728 !key->blend.alpha_to_coverage) { 3729 llvmpipe_fs_variant_linear_fastpath(variant); 3730 } 3731 3732 /* If the original fastpath doesn't cover this variant, try the new 3733 * code: 3734 */ 3735 if (variant->jit_linear == NULL) { 3736 if (shader->kind == LP_FS_KIND_BLIT_RGBA || 3737 shader->kind == LP_FS_KIND_BLIT_RGB1 || 3738 shader->kind == LP_FS_KIND_LLVM_LINEAR) { 3739 llvmpipe_fs_variant_linear_llvm(lp, shader, variant); 3740 } 3741 } 3742 } else { 3743 if (LP_DEBUG & DEBUG_LINEAR) { 3744 lp_debug_fs_variant(variant); 3745 debug_printf(" ----> no linear path for this variant\n"); 3746 } 3747 } 3748 3749 /* 3750 * Compile everything 3751 */ 3752 3753 gallivm_compile_module(variant->gallivm); 3754 3755 variant->nr_instrs += lp_build_count_ir_module(variant->gallivm->module); 3756 3757 if (variant->function[RAST_EDGE_TEST]) { 3758 variant->jit_function[RAST_EDGE_TEST] = (lp_jit_frag_func) 3759 gallivm_jit_function(variant->gallivm, 3760 variant->function[RAST_EDGE_TEST]); 3761 } 3762 3763 if (variant->function[RAST_WHOLE]) { 3764 variant->jit_function[RAST_WHOLE] = (lp_jit_frag_func) 3765 gallivm_jit_function(variant->gallivm, 3766 variant->function[RAST_WHOLE]); 3767 } else if (!variant->jit_function[RAST_WHOLE]) { 3768 variant->jit_function[RAST_WHOLE] = variant->jit_function[RAST_EDGE_TEST]; 3769 } 3770 3771 if (linear) { 3772 if (variant->linear_function) { 3773 variant->jit_linear_llvm = (lp_jit_linear_llvm_func) 3774 gallivm_jit_function(variant->gallivm, variant->linear_function); 3775 } 3776 3777 /* 3778 * This must be done after LLVM compilation, as it will call the JIT'ed 3779 * code to determine active inputs. 3780 */ 3781 lp_linear_check_variant(variant); 3782 } 3783 3784 if (needs_caching) { 3785 lp_disk_cache_insert_shader(screen, &cached, ir_sha1_cache_key); 3786 } 3787 3788 gallivm_free_ir(variant->gallivm); 3789 3790 return variant; 3791} 3792 3793 3794static void * 3795llvmpipe_create_fs_state(struct pipe_context *pipe, 3796 const struct pipe_shader_state *templ) 3797{ 3798 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe); 3799 struct lp_fragment_shader *shader; 3800 int nr_samplers; 3801 int nr_sampler_views; 3802 int nr_images; 3803 int i; 3804 3805 shader = CALLOC_STRUCT(lp_fragment_shader); 3806 if (!shader) 3807 return NULL; 3808 3809 pipe_reference_init(&shader->reference, 1); 3810 shader->no = fs_no++; 3811 make_empty_list(&shader->variants); 3812 3813 shader->base.type = templ->type; 3814 if (templ->type == PIPE_SHADER_IR_TGSI) { 3815 /* get/save the summary info for this shader */ 3816 lp_build_tgsi_info(templ->tokens, &shader->info); 3817 3818 /* we need to keep a local copy of the tokens */ 3819 shader->base.tokens = tgsi_dup_tokens(templ->tokens); 3820 } else { 3821 shader->base.ir.nir = templ->ir.nir; 3822 nir_tgsi_scan_shader(templ->ir.nir, &shader->info.base, true); 3823 } 3824 3825 shader->draw_data = draw_create_fragment_shader(llvmpipe->draw, templ); 3826 if (shader->draw_data == NULL) { 3827 FREE((void *) shader->base.tokens); 3828 FREE(shader); 3829 return NULL; 3830 } 3831 3832 nr_samplers = shader->info.base.file_max[TGSI_FILE_SAMPLER] + 1; 3833 nr_sampler_views = shader->info.base.file_max[TGSI_FILE_SAMPLER_VIEW] + 1; 3834 nr_images = shader->info.base.file_max[TGSI_FILE_IMAGE] + 1; 3835 shader->variant_key_size = lp_fs_variant_key_size(MAX2(nr_samplers, nr_sampler_views), nr_images); 3836 3837 for (i = 0; i < shader->info.base.num_inputs; i++) { 3838 shader->inputs[i].usage_mask = shader->info.base.input_usage_mask[i]; 3839 shader->inputs[i].location = shader->info.base.input_interpolate_loc[i]; 3840 3841 switch (shader->info.base.input_interpolate[i]) { 3842 case TGSI_INTERPOLATE_CONSTANT: 3843 shader->inputs[i].interp = LP_INTERP_CONSTANT; 3844 break; 3845 case TGSI_INTERPOLATE_LINEAR: 3846 shader->inputs[i].interp = LP_INTERP_LINEAR; 3847 break; 3848 case TGSI_INTERPOLATE_PERSPECTIVE: 3849 shader->inputs[i].interp = LP_INTERP_PERSPECTIVE; 3850 break; 3851 case TGSI_INTERPOLATE_COLOR: 3852 shader->inputs[i].interp = LP_INTERP_COLOR; 3853 break; 3854 default: 3855 assert(0); 3856 break; 3857 } 3858 3859 switch (shader->info.base.input_semantic_name[i]) { 3860 case TGSI_SEMANTIC_FACE: 3861 shader->inputs[i].interp = LP_INTERP_FACING; 3862 break; 3863 case TGSI_SEMANTIC_POSITION: 3864 /* Position was already emitted above 3865 */ 3866 shader->inputs[i].interp = LP_INTERP_POSITION; 3867 shader->inputs[i].src_index = 0; 3868 continue; 3869 } 3870 3871 /* XXX this is a completely pointless index map... */ 3872 shader->inputs[i].src_index = i+1; 3873 } 3874 3875 if (LP_DEBUG & DEBUG_TGSI && templ->type == PIPE_SHADER_IR_TGSI) { 3876 unsigned attrib; 3877 debug_printf("llvmpipe: Create fragment shader #%u %p:\n", 3878 shader->no, (void *) shader); 3879 tgsi_dump(templ->tokens, 0); 3880 debug_printf("usage masks:\n"); 3881 for (attrib = 0; attrib < shader->info.base.num_inputs; ++attrib) { 3882 unsigned usage_mask = shader->info.base.input_usage_mask[attrib]; 3883 debug_printf(" IN[%u].%s%s%s%s\n", 3884 attrib, 3885 usage_mask & TGSI_WRITEMASK_X ? "x" : "", 3886 usage_mask & TGSI_WRITEMASK_Y ? "y" : "", 3887 usage_mask & TGSI_WRITEMASK_Z ? "z" : "", 3888 usage_mask & TGSI_WRITEMASK_W ? "w" : ""); 3889 } 3890 debug_printf("\n"); 3891 } 3892 3893 /* This will put a derived copy of the tokens into shader->base.tokens */ 3894 if (templ->type == PIPE_SHADER_IR_TGSI) 3895 llvmpipe_fs_analyse(shader, templ->tokens); 3896 else 3897 shader->kind = LP_FS_KIND_GENERAL; 3898 3899 return shader; 3900} 3901 3902 3903static void 3904llvmpipe_bind_fs_state(struct pipe_context *pipe, void *fs) 3905{ 3906 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe); 3907 struct lp_fragment_shader *lp_fs = (struct lp_fragment_shader *)fs; 3908 if (llvmpipe->fs == lp_fs) 3909 return; 3910 3911 draw_bind_fragment_shader(llvmpipe->draw, 3912 (lp_fs ? lp_fs->draw_data : NULL)); 3913 3914 lp_fs_reference(llvmpipe, &llvmpipe->fs, lp_fs); 3915 3916 /* invalidate the setup link, NEW_FS will make it update */ 3917 lp_setup_set_fs_variant(llvmpipe->setup, NULL); 3918 llvmpipe->dirty |= LP_NEW_FS; 3919} 3920 3921 3922/** 3923 * Remove shader variant from two lists: the shader's variant list 3924 * and the context's variant list. 3925 */ 3926 3927static 3928void llvmpipe_remove_shader_variant(struct llvmpipe_context *lp, 3929 struct lp_fragment_shader_variant *variant) 3930{ 3931 if ((LP_DEBUG & DEBUG_FS) || (gallivm_debug & GALLIVM_DEBUG_IR)) { 3932 debug_printf("llvmpipe: del fs #%u var %u v created %u v cached %u " 3933 "v total cached %u inst %u total inst %u\n", 3934 variant->shader->no, variant->no, 3935 variant->shader->variants_created, 3936 variant->shader->variants_cached, 3937 lp->nr_fs_variants, variant->nr_instrs, lp->nr_fs_instrs); 3938 } 3939 3940 /* remove from shader's list */ 3941 remove_from_list(&variant->list_item_local); 3942 variant->shader->variants_cached--; 3943 3944 /* remove from context's list */ 3945 remove_from_list(&variant->list_item_global); 3946 lp->nr_fs_variants--; 3947 lp->nr_fs_instrs -= variant->nr_instrs; 3948} 3949 3950void 3951llvmpipe_destroy_shader_variant(struct llvmpipe_context *lp, 3952 struct lp_fragment_shader_variant *variant) 3953{ 3954 gallivm_destroy(variant->gallivm); 3955 3956 lp_fs_reference(lp, &variant->shader, NULL); 3957 3958 FREE(variant); 3959} 3960 3961void 3962llvmpipe_destroy_fs(struct llvmpipe_context *llvmpipe, 3963 struct lp_fragment_shader *shader) 3964{ 3965 /* Delete draw module's data */ 3966 draw_delete_fragment_shader(llvmpipe->draw, shader->draw_data); 3967 3968 if (shader->base.ir.nir) 3969 ralloc_free(shader->base.ir.nir); 3970 assert(shader->variants_cached == 0); 3971 FREE((void *) shader->base.tokens); 3972 FREE(shader); 3973} 3974 3975static void 3976llvmpipe_delete_fs_state(struct pipe_context *pipe, void *fs) 3977{ 3978 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe); 3979 struct lp_fragment_shader *shader = fs; 3980 struct lp_fs_variant_list_item *li; 3981 3982 /* Delete all the variants */ 3983 li = first_elem(&shader->variants); 3984 while(!at_end(&shader->variants, li)) { 3985 struct lp_fs_variant_list_item *next = next_elem(li); 3986 struct lp_fragment_shader_variant *variant; 3987 variant = li->base; 3988 llvmpipe_remove_shader_variant(llvmpipe, li->base); 3989 lp_fs_variant_reference(llvmpipe, &variant, NULL); 3990 li = next; 3991 } 3992 3993 lp_fs_reference(llvmpipe, &shader, NULL); 3994} 3995 3996static void 3997llvmpipe_set_constant_buffer(struct pipe_context *pipe, 3998 enum pipe_shader_type shader, uint index, 3999 bool take_ownership, 4000 const struct pipe_constant_buffer *cb) 4001{ 4002 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe); 4003 struct pipe_constant_buffer *constants = &llvmpipe->constants[shader][index]; 4004 4005 assert(shader < PIPE_SHADER_TYPES); 4006 assert(index < ARRAY_SIZE(llvmpipe->constants[shader])); 4007 4008 /* note: reference counting */ 4009 util_copy_constant_buffer(&llvmpipe->constants[shader][index], cb, 4010 take_ownership); 4011 4012 /* user_buffer is only valid until the next set_constant_buffer (at most, 4013 * possibly until shader deletion), so we need to upload it now to make sure 4014 * it doesn't get updated/freed out from under us. 4015 */ 4016 if (constants->user_buffer) { 4017 u_upload_data(llvmpipe->pipe.const_uploader, 0, constants->buffer_size, 16, 4018 constants->user_buffer, &constants->buffer_offset, 4019 &constants->buffer); 4020 } 4021 if (constants->buffer) { 4022 if (!(constants->buffer->bind & PIPE_BIND_CONSTANT_BUFFER)) { 4023 debug_printf("Illegal set constant without bind flag\n"); 4024 constants->buffer->bind |= PIPE_BIND_CONSTANT_BUFFER; 4025 } 4026 } 4027 4028 if (shader == PIPE_SHADER_VERTEX || 4029 shader == PIPE_SHADER_GEOMETRY || 4030 shader == PIPE_SHADER_TESS_CTRL || 4031 shader == PIPE_SHADER_TESS_EVAL) { 4032 /* Pass the constants to the 'draw' module */ 4033 const unsigned size = cb ? cb->buffer_size : 0; 4034 4035 const ubyte *data = NULL; 4036 if (constants->buffer) 4037 data = (ubyte *) llvmpipe_resource_data(constants->buffer) + constants->buffer_offset; 4038 4039 draw_set_mapped_constant_buffer(llvmpipe->draw, shader, 4040 index, data, size); 4041 } 4042 else if (shader == PIPE_SHADER_COMPUTE) 4043 llvmpipe->cs_dirty |= LP_CSNEW_CONSTANTS; 4044 else 4045 llvmpipe->dirty |= LP_NEW_FS_CONSTANTS; 4046} 4047 4048static void 4049llvmpipe_set_shader_buffers(struct pipe_context *pipe, 4050 enum pipe_shader_type shader, unsigned start_slot, 4051 unsigned count, const struct pipe_shader_buffer *buffers, 4052 unsigned writable_bitmask) 4053{ 4054 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe); 4055 unsigned i, idx; 4056 for (i = start_slot, idx = 0; i < start_slot + count; i++, idx++) { 4057 const struct pipe_shader_buffer *buffer = buffers ? &buffers[idx] : NULL; 4058 4059 util_copy_shader_buffer(&llvmpipe->ssbos[shader][i], buffer); 4060 4061 if (shader == PIPE_SHADER_VERTEX || 4062 shader == PIPE_SHADER_GEOMETRY || 4063 shader == PIPE_SHADER_TESS_CTRL || 4064 shader == PIPE_SHADER_TESS_EVAL) { 4065 const unsigned size = buffer ? buffer->buffer_size : 0; 4066 const ubyte *data = NULL; 4067 if (buffer && buffer->buffer) 4068 data = (ubyte *) llvmpipe_resource_data(buffer->buffer); 4069 if (data) 4070 data += buffer->buffer_offset; 4071 draw_set_mapped_shader_buffer(llvmpipe->draw, shader, 4072 i, data, size); 4073 } else if (shader == PIPE_SHADER_COMPUTE) { 4074 llvmpipe->cs_dirty |= LP_CSNEW_SSBOS; 4075 } else if (shader == PIPE_SHADER_FRAGMENT) { 4076 llvmpipe->dirty |= LP_NEW_FS_SSBOS; 4077 } 4078 } 4079} 4080 4081static void 4082llvmpipe_set_shader_images(struct pipe_context *pipe, 4083 enum pipe_shader_type shader, unsigned start_slot, 4084 unsigned count, unsigned unbind_num_trailing_slots, 4085 const struct pipe_image_view *images) 4086{ 4087 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe); 4088 unsigned i, idx; 4089 4090 draw_flush(llvmpipe->draw); 4091 for (i = start_slot, idx = 0; i < start_slot + count; i++, idx++) { 4092 const struct pipe_image_view *image = images ? &images[idx] : NULL; 4093 4094 util_copy_image_view(&llvmpipe->images[shader][i], image); 4095 } 4096 4097 llvmpipe->num_images[shader] = start_slot + count; 4098 if (shader == PIPE_SHADER_VERTEX || 4099 shader == PIPE_SHADER_GEOMETRY || 4100 shader == PIPE_SHADER_TESS_CTRL || 4101 shader == PIPE_SHADER_TESS_EVAL) { 4102 draw_set_images(llvmpipe->draw, 4103 shader, 4104 llvmpipe->images[shader], 4105 start_slot + count); 4106 } else if (shader == PIPE_SHADER_COMPUTE) 4107 llvmpipe->cs_dirty |= LP_CSNEW_IMAGES; 4108 else 4109 llvmpipe->dirty |= LP_NEW_FS_IMAGES; 4110 4111 if (unbind_num_trailing_slots) { 4112 llvmpipe_set_shader_images(pipe, shader, start_slot + count, 4113 unbind_num_trailing_slots, 0, NULL); 4114 } 4115} 4116 4117/** 4118 * Return the blend factor equivalent to a destination alpha of one. 4119 */ 4120static inline unsigned 4121force_dst_alpha_one(unsigned factor, boolean clamped_zero) 4122{ 4123 switch(factor) { 4124 case PIPE_BLENDFACTOR_DST_ALPHA: 4125 return PIPE_BLENDFACTOR_ONE; 4126 case PIPE_BLENDFACTOR_INV_DST_ALPHA: 4127 return PIPE_BLENDFACTOR_ZERO; 4128 case PIPE_BLENDFACTOR_SRC_ALPHA_SATURATE: 4129 if (clamped_zero) 4130 return PIPE_BLENDFACTOR_ZERO; 4131 else 4132 return PIPE_BLENDFACTOR_SRC_ALPHA_SATURATE; 4133 } 4134 4135 return factor; 4136} 4137 4138 4139/** 4140 * We need to generate several variants of the fragment pipeline to match 4141 * all the combinations of the contributing state atoms. 4142 * 4143 * TODO: there is actually no reason to tie this to context state -- the 4144 * generated code could be cached globally in the screen. 4145 */ 4146static struct lp_fragment_shader_variant_key * 4147make_variant_key(struct llvmpipe_context *lp, 4148 struct lp_fragment_shader *shader, 4149 char *store) 4150{ 4151 unsigned i; 4152 struct lp_fragment_shader_variant_key *key; 4153 4154 key = (struct lp_fragment_shader_variant_key *)store; 4155 4156 memset(key, 0, sizeof(*key)); 4157 4158 if (lp->framebuffer.zsbuf) { 4159 enum pipe_format zsbuf_format = lp->framebuffer.zsbuf->format; 4160 const struct util_format_description *zsbuf_desc = 4161 util_format_description(zsbuf_format); 4162 4163 if (lp->depth_stencil->depth_enabled && 4164 util_format_has_depth(zsbuf_desc)) { 4165 key->zsbuf_format = zsbuf_format; 4166 key->depth.enabled = lp->depth_stencil->depth_enabled; 4167 key->depth.writemask = lp->depth_stencil->depth_writemask; 4168 key->depth.func = lp->depth_stencil->depth_func; 4169 } 4170 if (lp->depth_stencil->stencil[0].enabled && 4171 util_format_has_stencil(zsbuf_desc)) { 4172 key->zsbuf_format = zsbuf_format; 4173 memcpy(&key->stencil, &lp->depth_stencil->stencil, sizeof key->stencil); 4174 } 4175 if (llvmpipe_resource_is_1d(lp->framebuffer.zsbuf->texture)) { 4176 key->resource_1d = TRUE; 4177 } 4178 key->zsbuf_nr_samples = util_res_sample_count(lp->framebuffer.zsbuf->texture); 4179 } 4180 4181 /* 4182 * Propagate the depth clamp setting from the rasterizer state. 4183 */ 4184 key->depth_clamp = lp->rasterizer->depth_clamp; 4185 4186 /* alpha test only applies if render buffer 0 is non-integer (or does not exist) */ 4187 if (!lp->framebuffer.nr_cbufs || 4188 !lp->framebuffer.cbufs[0] || 4189 !util_format_is_pure_integer(lp->framebuffer.cbufs[0]->format)) { 4190 key->alpha.enabled = lp->depth_stencil->alpha_enabled; 4191 } 4192 if(key->alpha.enabled) 4193 key->alpha.func = lp->depth_stencil->alpha_func; 4194 /* alpha.ref_value is passed in jit_context */ 4195 4196 key->flatshade = lp->rasterizer->flatshade; 4197 key->multisample = lp->rasterizer->multisample; 4198 key->no_ms_sample_mask_out = lp->rasterizer->no_ms_sample_mask_out; 4199 if (lp->active_occlusion_queries && !lp->queries_disabled) { 4200 key->occlusion_count = TRUE; 4201 } 4202 4203 memcpy(&key->blend, lp->blend, sizeof key->blend); 4204 4205 key->coverage_samples = 1; 4206 key->min_samples = 1; 4207 if (key->multisample) { 4208 key->coverage_samples = util_framebuffer_get_num_samples(&lp->framebuffer); 4209 key->min_samples = lp->min_samples == 1 ? 1 : key->coverage_samples; 4210 } 4211 key->nr_cbufs = lp->framebuffer.nr_cbufs; 4212 4213 if (!key->blend.independent_blend_enable) { 4214 /* we always need independent blend otherwise the fixups below won't work */ 4215 for (i = 1; i < key->nr_cbufs; i++) { 4216 memcpy(&key->blend.rt[i], &key->blend.rt[0], sizeof(key->blend.rt[0])); 4217 } 4218 key->blend.independent_blend_enable = 1; 4219 } 4220 4221 for (i = 0; i < lp->framebuffer.nr_cbufs; i++) { 4222 struct pipe_rt_blend_state *blend_rt = &key->blend.rt[i]; 4223 4224 if (lp->framebuffer.cbufs[i]) { 4225 enum pipe_format format = lp->framebuffer.cbufs[i]->format; 4226 const struct util_format_description *format_desc; 4227 4228 key->cbuf_format[i] = format; 4229 key->cbuf_nr_samples[i] = util_res_sample_count(lp->framebuffer.cbufs[i]->texture); 4230 4231 /* 4232 * Figure out if this is a 1d resource. Note that OpenGL allows crazy 4233 * mixing of 2d textures with height 1 and 1d textures, so make sure 4234 * we pick 1d if any cbuf or zsbuf is 1d. 4235 */ 4236 if (llvmpipe_resource_is_1d(lp->framebuffer.cbufs[i]->texture)) { 4237 key->resource_1d = TRUE; 4238 } 4239 4240 format_desc = util_format_description(format); 4241 assert(format_desc->colorspace == UTIL_FORMAT_COLORSPACE_RGB || 4242 format_desc->colorspace == UTIL_FORMAT_COLORSPACE_SRGB); 4243 4244 /* 4245 * Mask out color channels not present in the color buffer. 4246 */ 4247 blend_rt->colormask &= util_format_colormask(format_desc); 4248 4249 /* 4250 * Disable blend for integer formats. 4251 */ 4252 if (util_format_is_pure_integer(format)) { 4253 blend_rt->blend_enable = 0; 4254 } 4255 4256 /* 4257 * Our swizzled render tiles always have an alpha channel, but the 4258 * linear render target format often does not, so force here the dst 4259 * alpha to be one. 4260 * 4261 * This is not a mere optimization. Wrong results will be produced if 4262 * the dst alpha is used, the dst format does not have alpha, and the 4263 * previous rendering was not flushed from the swizzled to linear 4264 * buffer. For example, NonPowTwo DCT. 4265 * 4266 * TODO: This should be generalized to all channels for better 4267 * performance, but only alpha causes correctness issues. 4268 * 4269 * Also, force rgb/alpha func/factors match, to make AoS blending 4270 * easier. 4271 */ 4272 if (format_desc->swizzle[3] > PIPE_SWIZZLE_W || 4273 format_desc->swizzle[3] == format_desc->swizzle[0]) { 4274 /* Doesn't cover mixed snorm/unorm but can't render to them anyway */ 4275 boolean clamped_zero = !util_format_is_float(format) && 4276 !util_format_is_snorm(format); 4277 blend_rt->rgb_src_factor = 4278 force_dst_alpha_one(blend_rt->rgb_src_factor, clamped_zero); 4279 blend_rt->rgb_dst_factor = 4280 force_dst_alpha_one(blend_rt->rgb_dst_factor, clamped_zero); 4281 blend_rt->alpha_func = blend_rt->rgb_func; 4282 blend_rt->alpha_src_factor = blend_rt->rgb_src_factor; 4283 blend_rt->alpha_dst_factor = blend_rt->rgb_dst_factor; 4284 } 4285 } 4286 else { 4287 /* no color buffer for this fragment output */ 4288 key->cbuf_format[i] = PIPE_FORMAT_NONE; 4289 key->cbuf_nr_samples[i] = 0; 4290 blend_rt->colormask = 0x0; 4291 blend_rt->blend_enable = 0; 4292 } 4293 } 4294 4295 /* This value will be the same for all the variants of a given shader: 4296 */ 4297 key->nr_samplers = shader->info.base.file_max[TGSI_FILE_SAMPLER] + 1; 4298 4299 struct lp_sampler_static_state *fs_sampler; 4300 4301 fs_sampler = lp_fs_variant_key_samplers(key); 4302 4303 memset(fs_sampler, 0, MAX2(key->nr_samplers, key->nr_sampler_views) * sizeof *fs_sampler); 4304 4305 for(i = 0; i < key->nr_samplers; ++i) { 4306 if(shader->info.base.file_mask[TGSI_FILE_SAMPLER] & (1 << i)) { 4307 lp_sampler_static_sampler_state(&fs_sampler[i].sampler_state, 4308 lp->samplers[PIPE_SHADER_FRAGMENT][i]); 4309 } 4310 } 4311 4312 /* 4313 * XXX If TGSI_FILE_SAMPLER_VIEW exists assume all texture opcodes 4314 * are dx10-style? Can't really have mixed opcodes, at least not 4315 * if we want to skip the holes here (without rescanning tgsi). 4316 */ 4317 if (shader->info.base.file_max[TGSI_FILE_SAMPLER_VIEW] != -1) { 4318 key->nr_sampler_views = shader->info.base.file_max[TGSI_FILE_SAMPLER_VIEW] + 1; 4319 for(i = 0; i < key->nr_sampler_views; ++i) { 4320 /* 4321 * Note sview may exceed what's representable by file_mask. 4322 * This will still work, the only downside is that not actually 4323 * used views may be included in the shader key. 4324 */ 4325 if(shader->info.base.file_mask[TGSI_FILE_SAMPLER_VIEW] & (1u << (i & 31))) { 4326 lp_sampler_static_texture_state(&fs_sampler[i].texture_state, 4327 lp->sampler_views[PIPE_SHADER_FRAGMENT][i]); 4328 } 4329 } 4330 } 4331 else { 4332 key->nr_sampler_views = key->nr_samplers; 4333 for(i = 0; i < key->nr_sampler_views; ++i) { 4334 if(shader->info.base.file_mask[TGSI_FILE_SAMPLER] & (1 << i)) { 4335 lp_sampler_static_texture_state(&fs_sampler[i].texture_state, 4336 lp->sampler_views[PIPE_SHADER_FRAGMENT][i]); 4337 } 4338 } 4339 } 4340 4341 struct lp_image_static_state *lp_image; 4342 lp_image = lp_fs_variant_key_images(key); 4343 key->nr_images = shader->info.base.file_max[TGSI_FILE_IMAGE] + 1; 4344 for (i = 0; i < key->nr_images; ++i) { 4345 if (shader->info.base.file_mask[TGSI_FILE_IMAGE] & (1 << i)) { 4346 lp_sampler_static_texture_state_image(&lp_image[i].image_state, 4347 &lp->images[PIPE_SHADER_FRAGMENT][i]); 4348 } 4349 } 4350 4351 if (shader->kind == LP_FS_KIND_AERO_MINIFICATION) { 4352 struct lp_sampler_static_state *samp0 = lp_fs_variant_key_sampler_idx(key, 0); 4353 assert(samp0); 4354 samp0->sampler_state.min_img_filter = PIPE_TEX_FILTER_NEAREST; 4355 samp0->sampler_state.mag_img_filter = PIPE_TEX_FILTER_NEAREST; 4356 } 4357 4358 return key; 4359} 4360 4361 4362/** 4363 * Update fragment shader state. This is called just prior to drawing 4364 * something when some fragment-related state has changed. 4365 */ 4366void 4367llvmpipe_update_fs(struct llvmpipe_context *lp) 4368{ 4369 struct lp_fragment_shader *shader = lp->fs; 4370 struct lp_fragment_shader_variant_key *key; 4371 struct lp_fragment_shader_variant *variant = NULL; 4372 struct lp_fs_variant_list_item *li; 4373 char store[LP_FS_MAX_VARIANT_KEY_SIZE]; 4374 4375 key = make_variant_key(lp, shader, store); 4376 4377 /* Search the variants for one which matches the key */ 4378 li = first_elem(&shader->variants); 4379 while(!at_end(&shader->variants, li)) { 4380 if(memcmp(&li->base->key, key, shader->variant_key_size) == 0) { 4381 variant = li->base; 4382 break; 4383 } 4384 li = next_elem(li); 4385 } 4386 4387 if (variant) { 4388 /* Move this variant to the head of the list to implement LRU 4389 * deletion of shader's when we have too many. 4390 */ 4391 move_to_head(&lp->fs_variants_list, &variant->list_item_global); 4392 } 4393 else { 4394 /* variant not found, create it now */ 4395 int64_t t0, t1, dt; 4396 unsigned i; 4397 unsigned variants_to_cull; 4398 4399 if (LP_DEBUG & DEBUG_FS) { 4400 debug_printf("%u variants,\t%u instrs,\t%u instrs/variant\n", 4401 lp->nr_fs_variants, 4402 lp->nr_fs_instrs, 4403 lp->nr_fs_variants ? lp->nr_fs_instrs / lp->nr_fs_variants : 0); 4404 } 4405 4406 /* First, check if we've exceeded the max number of shader variants. 4407 * If so, free 6.25% of them (the least recently used ones). 4408 */ 4409 variants_to_cull = lp->nr_fs_variants >= LP_MAX_SHADER_VARIANTS ? LP_MAX_SHADER_VARIANTS / 16 : 0; 4410 4411 if (variants_to_cull || 4412 lp->nr_fs_instrs >= LP_MAX_SHADER_INSTRUCTIONS) { 4413 if (gallivm_debug & GALLIVM_DEBUG_PERF) { 4414 debug_printf("Evicting FS: %u fs variants,\t%u total variants," 4415 "\t%u instrs,\t%u instrs/variant\n", 4416 shader->variants_cached, 4417 lp->nr_fs_variants, lp->nr_fs_instrs, 4418 lp->nr_fs_instrs / lp->nr_fs_variants); 4419 } 4420 4421 /* 4422 * We need to re-check lp->nr_fs_variants because an arbitrarliy large 4423 * number of shader variants (potentially all of them) could be 4424 * pending for destruction on flush. 4425 */ 4426 4427 for (i = 0; i < variants_to_cull || lp->nr_fs_instrs >= LP_MAX_SHADER_INSTRUCTIONS; i++) { 4428 struct lp_fs_variant_list_item *item; 4429 if (is_empty_list(&lp->fs_variants_list)) { 4430 break; 4431 } 4432 item = last_elem(&lp->fs_variants_list); 4433 assert(item); 4434 assert(item->base); 4435 llvmpipe_remove_shader_variant(lp, item->base); 4436 struct lp_fragment_shader_variant *variant = item->base; 4437 lp_fs_variant_reference(lp, &variant, NULL); 4438 } 4439 } 4440 4441 /* 4442 * Generate the new variant. 4443 */ 4444 t0 = os_time_get(); 4445 variant = generate_variant(lp, shader, key); 4446 t1 = os_time_get(); 4447 dt = t1 - t0; 4448 LP_COUNT_ADD(llvm_compile_time, dt); 4449 LP_COUNT_ADD(nr_llvm_compiles, 2); /* emit vs. omit in/out test */ 4450 4451 /* Put the new variant into the list */ 4452 if (variant) { 4453 insert_at_head(&shader->variants, &variant->list_item_local); 4454 insert_at_head(&lp->fs_variants_list, &variant->list_item_global); 4455 lp->nr_fs_variants++; 4456 lp->nr_fs_instrs += variant->nr_instrs; 4457 shader->variants_cached++; 4458 } 4459 } 4460 4461 /* Bind this variant */ 4462 lp_setup_set_fs_variant(lp->setup, variant); 4463} 4464 4465 4466 4467 4468 4469void 4470llvmpipe_init_fs_funcs(struct llvmpipe_context *llvmpipe) 4471{ 4472 llvmpipe->pipe.create_fs_state = llvmpipe_create_fs_state; 4473 llvmpipe->pipe.bind_fs_state = llvmpipe_bind_fs_state; 4474 llvmpipe->pipe.delete_fs_state = llvmpipe_delete_fs_state; 4475 4476 llvmpipe->pipe.set_constant_buffer = llvmpipe_set_constant_buffer; 4477 4478 llvmpipe->pipe.set_shader_buffers = llvmpipe_set_shader_buffers; 4479 llvmpipe->pipe.set_shader_images = llvmpipe_set_shader_images; 4480} 4481 4482 4483