1/* 2 * Copyright © 2015 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 */ 23 24#include <assert.h> 25#include <stdbool.h> 26#include <string.h> 27#include <unistd.h> 28#include <fcntl.h> 29 30#include "anv_private.h" 31 32#include "common/intel_aux_map.h" 33#include "common/intel_sample_positions.h" 34#include "genxml/gen_macros.h" 35#include "genxml/genX_pack.h" 36 37#include "vk_util.h" 38 39/** 40 * Compute an \p n x \p m pixel hashing table usable as slice, subslice or 41 * pixel pipe hashing table. The resulting table is the cyclic repetition of 42 * a fixed pattern with periodicity equal to \p period. 43 * 44 * If \p index is specified to be equal to \p period, a 2-way hashing table 45 * will be generated such that indices 0 and 1 are returned for the following 46 * fractions of entries respectively: 47 * 48 * p_0 = ceil(period / 2) / period 49 * p_1 = floor(period / 2) / period 50 * 51 * If \p index is even and less than \p period, a 3-way hashing table will be 52 * generated such that indices 0, 1 and 2 are returned for the following 53 * fractions of entries: 54 * 55 * p_0 = (ceil(period / 2) - 1) / period 56 * p_1 = floor(period / 2) / period 57 * p_2 = 1 / period 58 * 59 * The equations above apply if \p flip is equal to 0, if it is equal to 1 p_0 60 * and p_1 will be swapped for the result. Note that in the context of pixel 61 * pipe hashing this can be always 0 on Gfx12 platforms, since the hardware 62 * transparently remaps logical indices found on the table to physical pixel 63 * pipe indices from the highest to lowest EU count. 64 */ 65UNUSED static void 66calculate_pixel_hashing_table(unsigned n, unsigned m, 67 unsigned period, unsigned index, bool flip, 68 uint32_t *p) 69{ 70 for (unsigned i = 0; i < n; i++) { 71 for (unsigned j = 0; j < m; j++) { 72 const unsigned k = (i + j) % period; 73 p[j + m * i] = (k == index ? 2 : (k & 1) ^ flip); 74 } 75 } 76} 77 78static void 79genX(emit_slice_hashing_state)(struct anv_device *device, 80 struct anv_batch *batch) 81{ 82#if GFX_VER == 11 83 assert(device->info.ppipe_subslices[2] == 0); 84 85 if (device->info.ppipe_subslices[0] == device->info.ppipe_subslices[1]) 86 return; 87 88 if (!device->slice_hash.alloc_size) { 89 unsigned size = GENX(SLICE_HASH_TABLE_length) * 4; 90 device->slice_hash = 91 anv_state_pool_alloc(&device->dynamic_state_pool, size, 64); 92 93 const bool flip = device->info.ppipe_subslices[0] < 94 device->info.ppipe_subslices[1]; 95 struct GENX(SLICE_HASH_TABLE) table; 96 calculate_pixel_hashing_table(16, 16, 3, 3, flip, table.Entry[0]); 97 98 GENX(SLICE_HASH_TABLE_pack)(NULL, device->slice_hash.map, &table); 99 } 100 101 anv_batch_emit(batch, GENX(3DSTATE_SLICE_TABLE_STATE_POINTERS), ptr) { 102 ptr.SliceHashStatePointerValid = true; 103 ptr.SliceHashTableStatePointer = device->slice_hash.offset; 104 } 105 106 anv_batch_emit(batch, GENX(3DSTATE_3D_MODE), mode) { 107 mode.SliceHashingTableEnable = true; 108 } 109#elif GFX_VERx10 == 120 110 /* For each n calculate ppipes_of[n], equal to the number of pixel pipes 111 * present with n active dual subslices. 112 */ 113 unsigned ppipes_of[3] = {}; 114 115 for (unsigned n = 0; n < ARRAY_SIZE(ppipes_of); n++) { 116 for (unsigned p = 0; p < ARRAY_SIZE(device->info.ppipe_subslices); p++) 117 ppipes_of[n] += (device->info.ppipe_subslices[p] == n); 118 } 119 120 /* Gfx12 has three pixel pipes. */ 121 assert(ppipes_of[0] + ppipes_of[1] + ppipes_of[2] == 3); 122 123 if (ppipes_of[2] == 3 || ppipes_of[0] == 2) { 124 /* All three pixel pipes have the maximum number of active dual 125 * subslices, or there is only one active pixel pipe: Nothing to do. 126 */ 127 return; 128 } 129 130 anv_batch_emit(batch, GENX(3DSTATE_SUBSLICE_HASH_TABLE), p) { 131 p.SliceHashControl[0] = TABLE_0; 132 133 if (ppipes_of[2] == 2 && ppipes_of[0] == 1) 134 calculate_pixel_hashing_table(8, 16, 2, 2, 0, p.TwoWayTableEntry[0]); 135 else if (ppipes_of[2] == 1 && ppipes_of[1] == 1 && ppipes_of[0] == 1) 136 calculate_pixel_hashing_table(8, 16, 3, 3, 0, p.TwoWayTableEntry[0]); 137 138 if (ppipes_of[2] == 2 && ppipes_of[1] == 1) 139 calculate_pixel_hashing_table(8, 16, 5, 4, 0, p.ThreeWayTableEntry[0]); 140 else if (ppipes_of[2] == 2 && ppipes_of[0] == 1) 141 calculate_pixel_hashing_table(8, 16, 2, 2, 0, p.ThreeWayTableEntry[0]); 142 else if (ppipes_of[2] == 1 && ppipes_of[1] == 1 && ppipes_of[0] == 1) 143 calculate_pixel_hashing_table(8, 16, 3, 3, 0, p.ThreeWayTableEntry[0]); 144 else 145 unreachable("Illegal fusing."); 146 } 147 148 anv_batch_emit(batch, GENX(3DSTATE_3D_MODE), p) { 149 p.SubsliceHashingTableEnable = true; 150 p.SubsliceHashingTableEnableMask = true; 151 } 152#endif 153} 154 155static VkResult 156init_render_queue_state(struct anv_queue *queue) 157{ 158 struct anv_device *device = queue->device; 159 uint32_t cmds[64]; 160 struct anv_batch batch = { 161 .start = cmds, 162 .next = cmds, 163 .end = (void *) cmds + sizeof(cmds), 164 }; 165 166 anv_batch_emit(&batch, GENX(PIPELINE_SELECT), ps) { 167#if GFX_VER >= 9 168 ps.MaskBits = GFX_VER >= 12 ? 0x13 : 3; 169 ps.MediaSamplerDOPClockGateEnable = GFX_VER >= 12; 170#endif 171 ps.PipelineSelection = _3D; 172 } 173 174#if GFX_VER == 9 175 anv_batch_write_reg(&batch, GENX(CACHE_MODE_1), cm1) { 176 cm1.FloatBlendOptimizationEnable = true; 177 cm1.FloatBlendOptimizationEnableMask = true; 178 cm1.MSCRAWHazardAvoidanceBit = true; 179 cm1.MSCRAWHazardAvoidanceBitMask = true; 180 cm1.PartialResolveDisableInVC = true; 181 cm1.PartialResolveDisableInVCMask = true; 182 } 183#endif 184 185 anv_batch_emit(&batch, GENX(3DSTATE_AA_LINE_PARAMETERS), aa); 186 187 anv_batch_emit(&batch, GENX(3DSTATE_DRAWING_RECTANGLE), rect) { 188 rect.ClippedDrawingRectangleYMin = 0; 189 rect.ClippedDrawingRectangleXMin = 0; 190 rect.ClippedDrawingRectangleYMax = UINT16_MAX; 191 rect.ClippedDrawingRectangleXMax = UINT16_MAX; 192 rect.DrawingRectangleOriginY = 0; 193 rect.DrawingRectangleOriginX = 0; 194 } 195 196#if GFX_VER >= 8 197 anv_batch_emit(&batch, GENX(3DSTATE_WM_CHROMAKEY), ck); 198 199 genX(emit_sample_pattern)(&batch, 0, NULL); 200 201 /* The BDW+ docs describe how to use the 3DSTATE_WM_HZ_OP instruction in the 202 * section titled, "Optimized Depth Buffer Clear and/or Stencil Buffer 203 * Clear." It mentions that the packet overrides GPU state for the clear 204 * operation and needs to be reset to 0s to clear the overrides. Depending 205 * on the kernel, we may not get a context with the state for this packet 206 * zeroed. Do it ourselves just in case. We've observed this to prevent a 207 * number of GPU hangs on ICL. 208 */ 209 anv_batch_emit(&batch, GENX(3DSTATE_WM_HZ_OP), hzp); 210#endif 211 212#if GFX_VER == 11 213 /* The default behavior of bit 5 "Headerless Message for Pre-emptable 214 * Contexts" in SAMPLER MODE register is set to 0, which means 215 * headerless sampler messages are not allowed for pre-emptable 216 * contexts. Set the bit 5 to 1 to allow them. 217 */ 218 anv_batch_write_reg(&batch, GENX(SAMPLER_MODE), sm) { 219 sm.HeaderlessMessageforPreemptableContexts = true; 220 sm.HeaderlessMessageforPreemptableContextsMask = true; 221 } 222 223 /* Bit 1 "Enabled Texel Offset Precision Fix" must be set in 224 * HALF_SLICE_CHICKEN7 register. 225 */ 226 anv_batch_write_reg(&batch, GENX(HALF_SLICE_CHICKEN7), hsc7) { 227 hsc7.EnabledTexelOffsetPrecisionFix = true; 228 hsc7.EnabledTexelOffsetPrecisionFixMask = true; 229 } 230 231 anv_batch_write_reg(&batch, GENX(TCCNTLREG), tcc) { 232 tcc.L3DataPartialWriteMergingEnable = true; 233 tcc.ColorZPartialWriteMergingEnable = true; 234 tcc.URBPartialWriteMergingEnable = true; 235 tcc.TCDisable = true; 236 } 237#endif 238 genX(emit_slice_hashing_state)(device, &batch); 239 240#if GFX_VER >= 11 241 /* hardware specification recommends disabling repacking for 242 * the compatibility with decompression mechanism in display controller. 243 */ 244 if (device->info.disable_ccs_repack) { 245 anv_batch_write_reg(&batch, GENX(CACHE_MODE_0), cm0) { 246 cm0.DisableRepackingforCompression = true; 247 cm0.DisableRepackingforCompressionMask = true; 248 } 249 } 250 251 /* an unknown issue is causing vs push constants to become 252 * corrupted during object-level preemption. For now, restrict 253 * to command buffer level preemption to avoid rendering 254 * corruption. 255 */ 256 anv_batch_write_reg(&batch, GENX(CS_CHICKEN1), cc1) { 257 cc1.ReplayMode = MidcmdbufferPreemption; 258 cc1.ReplayModeMask = true; 259 } 260 261#if GFX_VERx10 < 125 262#define AA_LINE_QUALITY_REG GENX(3D_CHICKEN3) 263#else 264#define AA_LINE_QUALITY_REG GENX(CHICKEN_RASTER_1) 265#endif 266 267 /* Enable the new line drawing algorithm that produces higher quality 268 * lines. 269 */ 270 anv_batch_write_reg(&batch, AA_LINE_QUALITY_REG, c3) { 271 c3.AALineQualityFix = true; 272 c3.AALineQualityFixMask = true; 273 } 274#endif 275 276#if GFX_VER == 12 277 if (device->info.has_aux_map) { 278 uint64_t aux_base_addr = intel_aux_map_get_base(device->aux_map_ctx); 279 assert(aux_base_addr % (32 * 1024) == 0); 280 anv_batch_emit(&batch, GENX(MI_LOAD_REGISTER_IMM), lri) { 281 lri.RegisterOffset = GENX(GFX_AUX_TABLE_BASE_ADDR_num); 282 lri.DataDWord = aux_base_addr & 0xffffffff; 283 } 284 anv_batch_emit(&batch, GENX(MI_LOAD_REGISTER_IMM), lri) { 285 lri.RegisterOffset = GENX(GFX_AUX_TABLE_BASE_ADDR_num) + 4; 286 lri.DataDWord = aux_base_addr >> 32; 287 } 288 } 289#endif 290 291 /* Set the "CONSTANT_BUFFER Address Offset Disable" bit, so 292 * 3DSTATE_CONSTANT_XS buffer 0 is an absolute address. 293 * 294 * This is only safe on kernels with context isolation support. 295 */ 296 if (GFX_VER >= 8 && device->physical->has_context_isolation) { 297#if GFX_VER >= 9 298 anv_batch_write_reg(&batch, GENX(CS_DEBUG_MODE2), csdm2) { 299 csdm2.CONSTANT_BUFFERAddressOffsetDisable = true; 300 csdm2.CONSTANT_BUFFERAddressOffsetDisableMask = true; 301 } 302#elif GFX_VER == 8 303 anv_batch_write_reg(&batch, GENX(INSTPM), instpm) { 304 instpm.CONSTANT_BUFFERAddressOffsetDisable = true; 305 instpm.CONSTANT_BUFFERAddressOffsetDisableMask = true; 306 } 307#endif 308 } 309 310#if GFX_VER >= 11 311 /* Starting with GFX version 11, SLM is no longer part of the L3$ config 312 * so it never changes throughout the lifetime of the VkDevice. 313 */ 314 const struct intel_l3_config *cfg = intel_get_default_l3_config(&device->info); 315 genX(emit_l3_config)(&batch, device, cfg); 316 device->l3_config = cfg; 317#endif 318 319 anv_batch_emit(&batch, GENX(MI_BATCH_BUFFER_END), bbe); 320 321 assert(batch.next <= batch.end); 322 323 return anv_queue_submit_simple_batch(queue, &batch); 324} 325 326void 327genX(init_physical_device_state)(ASSERTED struct anv_physical_device *device) 328{ 329 assert(device->info.verx10 == GFX_VERx10); 330} 331 332VkResult 333genX(init_device_state)(struct anv_device *device) 334{ 335 VkResult res; 336 337 device->slice_hash = (struct anv_state) { 0 }; 338 for (uint32_t i = 0; i < device->queue_count; i++) { 339 struct anv_queue *queue = &device->queues[i]; 340 switch (queue->family->engine_class) { 341 case I915_ENGINE_CLASS_RENDER: 342 res = init_render_queue_state(queue); 343 break; 344 default: 345 res = vk_error(device, VK_ERROR_INITIALIZATION_FAILED); 346 break; 347 } 348 if (res != VK_SUCCESS) 349 return res; 350 } 351 352 return res; 353} 354 355void 356genX(emit_l3_config)(struct anv_batch *batch, 357 const struct anv_device *device, 358 const struct intel_l3_config *cfg) 359{ 360 UNUSED const struct intel_device_info *devinfo = &device->info; 361 362#if GFX_VER >= 8 363 364#if GFX_VER >= 12 365#define L3_ALLOCATION_REG GENX(L3ALLOC) 366#define L3_ALLOCATION_REG_num GENX(L3ALLOC_num) 367#else 368#define L3_ALLOCATION_REG GENX(L3CNTLREG) 369#define L3_ALLOCATION_REG_num GENX(L3CNTLREG_num) 370#endif 371 372 anv_batch_write_reg(batch, L3_ALLOCATION_REG, l3cr) { 373 if (cfg == NULL) { 374#if GFX_VER >= 12 375 l3cr.L3FullWayAllocationEnable = true; 376#else 377 unreachable("Invalid L3$ config"); 378#endif 379 } else { 380#if GFX_VER < 11 381 l3cr.SLMEnable = cfg->n[INTEL_L3P_SLM]; 382#endif 383#if GFX_VER == 11 384 /* Wa_1406697149: Bit 9 "Error Detection Behavior Control" must be 385 * set in L3CNTLREG register. The default setting of the bit is not 386 * the desirable behavior. 387 */ 388 l3cr.ErrorDetectionBehaviorControl = true; 389 l3cr.UseFullWays = true; 390#endif /* GFX_VER == 11 */ 391 assert(cfg->n[INTEL_L3P_IS] == 0); 392 assert(cfg->n[INTEL_L3P_C] == 0); 393 assert(cfg->n[INTEL_L3P_T] == 0); 394 l3cr.URBAllocation = cfg->n[INTEL_L3P_URB]; 395 l3cr.ROAllocation = cfg->n[INTEL_L3P_RO]; 396 l3cr.DCAllocation = cfg->n[INTEL_L3P_DC]; 397 l3cr.AllAllocation = cfg->n[INTEL_L3P_ALL]; 398 } 399 } 400 401#else /* GFX_VER < 8 */ 402 403 const bool has_dc = cfg->n[INTEL_L3P_DC] || cfg->n[INTEL_L3P_ALL]; 404 const bool has_is = cfg->n[INTEL_L3P_IS] || cfg->n[INTEL_L3P_RO] || 405 cfg->n[INTEL_L3P_ALL]; 406 const bool has_c = cfg->n[INTEL_L3P_C] || cfg->n[INTEL_L3P_RO] || 407 cfg->n[INTEL_L3P_ALL]; 408 const bool has_t = cfg->n[INTEL_L3P_T] || cfg->n[INTEL_L3P_RO] || 409 cfg->n[INTEL_L3P_ALL]; 410 411 assert(!cfg->n[INTEL_L3P_ALL]); 412 413 /* When enabled SLM only uses a portion of the L3 on half of the banks, 414 * the matching space on the remaining banks has to be allocated to a 415 * client (URB for all validated configurations) set to the 416 * lower-bandwidth 2-bank address hashing mode. 417 */ 418 const bool urb_low_bw = cfg->n[INTEL_L3P_SLM] && !devinfo->is_baytrail; 419 assert(!urb_low_bw || cfg->n[INTEL_L3P_URB] == cfg->n[INTEL_L3P_SLM]); 420 421 /* Minimum number of ways that can be allocated to the URB. */ 422 const unsigned n0_urb = devinfo->is_baytrail ? 32 : 0; 423 assert(cfg->n[INTEL_L3P_URB] >= n0_urb); 424 425 anv_batch_write_reg(batch, GENX(L3SQCREG1), l3sqc) { 426 l3sqc.ConvertDC_UC = !has_dc; 427 l3sqc.ConvertIS_UC = !has_is; 428 l3sqc.ConvertC_UC = !has_c; 429 l3sqc.ConvertT_UC = !has_t; 430#if GFX_VERx10 == 75 431 l3sqc.L3SQGeneralPriorityCreditInitialization = SQGPCI_DEFAULT; 432#else 433 l3sqc.L3SQGeneralPriorityCreditInitialization = 434 devinfo->is_baytrail ? BYT_SQGPCI_DEFAULT : SQGPCI_DEFAULT; 435#endif 436 l3sqc.L3SQHighPriorityCreditInitialization = SQHPCI_DEFAULT; 437 } 438 439 anv_batch_write_reg(batch, GENX(L3CNTLREG2), l3cr2) { 440 l3cr2.SLMEnable = cfg->n[INTEL_L3P_SLM]; 441 l3cr2.URBLowBandwidth = urb_low_bw; 442 l3cr2.URBAllocation = cfg->n[INTEL_L3P_URB] - n0_urb; 443#if !GFX_VERx10 == 75 444 l3cr2.ALLAllocation = cfg->n[INTEL_L3P_ALL]; 445#endif 446 l3cr2.ROAllocation = cfg->n[INTEL_L3P_RO]; 447 l3cr2.DCAllocation = cfg->n[INTEL_L3P_DC]; 448 } 449 450 anv_batch_write_reg(batch, GENX(L3CNTLREG3), l3cr3) { 451 l3cr3.ISAllocation = cfg->n[INTEL_L3P_IS]; 452 l3cr3.ISLowBandwidth = 0; 453 l3cr3.CAllocation = cfg->n[INTEL_L3P_C]; 454 l3cr3.CLowBandwidth = 0; 455 l3cr3.TAllocation = cfg->n[INTEL_L3P_T]; 456 l3cr3.TLowBandwidth = 0; 457 } 458 459#if GFX_VERx10 == 75 460 if (device->physical->cmd_parser_version >= 4) { 461 /* Enable L3 atomics on HSW if we have a DC partition, otherwise keep 462 * them disabled to avoid crashing the system hard. 463 */ 464 anv_batch_write_reg(batch, GENX(SCRATCH1), s1) { 465 s1.L3AtomicDisable = !has_dc; 466 } 467 anv_batch_write_reg(batch, GENX(CHICKEN3), c3) { 468 c3.L3AtomicDisableMask = true; 469 c3.L3AtomicDisable = !has_dc; 470 } 471 } 472#endif /* GFX_VERx10 == 75 */ 473 474#endif /* GFX_VER < 8 */ 475} 476 477void 478genX(emit_multisample)(struct anv_batch *batch, uint32_t samples, 479 const VkSampleLocationEXT *locations) 480{ 481 anv_batch_emit(batch, GENX(3DSTATE_MULTISAMPLE), ms) { 482 ms.NumberofMultisamples = __builtin_ffs(samples) - 1; 483 484 ms.PixelLocation = CENTER; 485#if GFX_VER >= 8 486 /* The PRM says that this bit is valid only for DX9: 487 * 488 * SW can choose to set this bit only for DX9 API. DX10/OGL API's 489 * should not have any effect by setting or not setting this bit. 490 */ 491 ms.PixelPositionOffsetEnable = false; 492#else 493 494 if (locations) { 495 switch (samples) { 496 case 1: 497 INTEL_SAMPLE_POS_1X_ARRAY(ms.Sample, locations); 498 break; 499 case 2: 500 INTEL_SAMPLE_POS_2X_ARRAY(ms.Sample, locations); 501 break; 502 case 4: 503 INTEL_SAMPLE_POS_4X_ARRAY(ms.Sample, locations); 504 break; 505 case 8: 506 INTEL_SAMPLE_POS_8X_ARRAY(ms.Sample, locations); 507 break; 508 default: 509 break; 510 } 511 } else { 512 switch (samples) { 513 case 1: 514 INTEL_SAMPLE_POS_1X(ms.Sample); 515 break; 516 case 2: 517 INTEL_SAMPLE_POS_2X(ms.Sample); 518 break; 519 case 4: 520 INTEL_SAMPLE_POS_4X(ms.Sample); 521 break; 522 case 8: 523 INTEL_SAMPLE_POS_8X(ms.Sample); 524 break; 525 default: 526 break; 527 } 528 } 529#endif 530 } 531} 532 533#if GFX_VER >= 8 534void 535genX(emit_sample_pattern)(struct anv_batch *batch, uint32_t samples, 536 const VkSampleLocationEXT *locations) 537{ 538 /* See the Vulkan 1.0 spec Table 24.1 "Standard sample locations" and 539 * VkPhysicalDeviceFeatures::standardSampleLocations. 540 */ 541 anv_batch_emit(batch, GENX(3DSTATE_SAMPLE_PATTERN), sp) { 542 if (locations) { 543 /* The Skylake PRM Vol. 2a "3DSTATE_SAMPLE_PATTERN" says: 544 * 545 * "When programming the sample offsets (for NUMSAMPLES_4 or _8 546 * and MSRASTMODE_xxx_PATTERN), the order of the samples 0 to 3 547 * (or 7 for 8X, or 15 for 16X) must have monotonically increasing 548 * distance from the pixel center. This is required to get the 549 * correct centroid computation in the device." 550 * 551 * However, the Vulkan spec seems to require that the the samples 552 * occur in the order provided through the API. The standard sample 553 * patterns have the above property that they have monotonically 554 * increasing distances from the center but client-provided ones do 555 * not. As long as this only affects centroid calculations as the 556 * docs say, we should be ok because OpenGL and Vulkan only require 557 * that the centroid be some lit sample and that it's the same for 558 * all samples in a pixel; they have no requirement that it be the 559 * one closest to center. 560 */ 561 switch (samples) { 562 case 1: 563 INTEL_SAMPLE_POS_1X_ARRAY(sp._1xSample, locations); 564 break; 565 case 2: 566 INTEL_SAMPLE_POS_2X_ARRAY(sp._2xSample, locations); 567 break; 568 case 4: 569 INTEL_SAMPLE_POS_4X_ARRAY(sp._4xSample, locations); 570 break; 571 case 8: 572 INTEL_SAMPLE_POS_8X_ARRAY(sp._8xSample, locations); 573 break; 574#if GFX_VER >= 9 575 case 16: 576 INTEL_SAMPLE_POS_16X_ARRAY(sp._16xSample, locations); 577 break; 578#endif 579 default: 580 break; 581 } 582 } else { 583 INTEL_SAMPLE_POS_1X(sp._1xSample); 584 INTEL_SAMPLE_POS_2X(sp._2xSample); 585 INTEL_SAMPLE_POS_4X(sp._4xSample); 586 INTEL_SAMPLE_POS_8X(sp._8xSample); 587#if GFX_VER >= 9 588 INTEL_SAMPLE_POS_16X(sp._16xSample); 589#endif 590 } 591 } 592} 593#endif 594 595#if GFX_VER >= 11 596void 597genX(emit_shading_rate)(struct anv_batch *batch, 598 const struct anv_graphics_pipeline *pipeline, 599 struct anv_state cps_states, 600 struct anv_dynamic_state *dynamic_state) 601{ 602 const struct brw_wm_prog_data *wm_prog_data = get_wm_prog_data(pipeline); 603 const bool cps_enable = wm_prog_data && wm_prog_data->per_coarse_pixel_dispatch; 604 605#if GFX_VER == 11 606 anv_batch_emit(batch, GENX(3DSTATE_CPS), cps) { 607 cps.CoarsePixelShadingMode = cps_enable ? CPS_MODE_CONSTANT : CPS_MODE_NONE; 608 if (cps_enable) { 609 cps.MinCPSizeX = dynamic_state->fragment_shading_rate.width; 610 cps.MinCPSizeY = dynamic_state->fragment_shading_rate.height; 611 } 612 } 613#elif GFX_VER == 12 614 for (uint32_t i = 0; i < dynamic_state->viewport.count; i++) { 615 uint32_t *cps_state_dwords = 616 cps_states.map + GENX(CPS_STATE_length) * 4 * i; 617 struct GENX(CPS_STATE) cps_state = { 618 .CoarsePixelShadingMode = cps_enable ? CPS_MODE_CONSTANT : CPS_MODE_NONE, 619 }; 620 621 if (cps_enable) { 622 cps_state.MinCPSizeX = dynamic_state->fragment_shading_rate.width; 623 cps_state.MinCPSizeY = dynamic_state->fragment_shading_rate.height; 624 } 625 626 GENX(CPS_STATE_pack)(NULL, cps_state_dwords, &cps_state); 627 } 628 629 anv_batch_emit(batch, GENX(3DSTATE_CPS_POINTERS), cps) { 630 cps.CoarsePixelShadingStateArrayPointer = cps_states.offset; 631 } 632#endif 633} 634#endif /* GFX_VER >= 11 */ 635 636static uint32_t 637vk_to_intel_tex_filter(VkFilter filter, bool anisotropyEnable) 638{ 639 switch (filter) { 640 default: 641 assert(!"Invalid filter"); 642 case VK_FILTER_NEAREST: 643 return anisotropyEnable ? MAPFILTER_ANISOTROPIC : MAPFILTER_NEAREST; 644 case VK_FILTER_LINEAR: 645 return anisotropyEnable ? MAPFILTER_ANISOTROPIC : MAPFILTER_LINEAR; 646 } 647} 648 649static uint32_t 650vk_to_intel_max_anisotropy(float ratio) 651{ 652 return (anv_clamp_f(ratio, 2, 16) - 2) / 2; 653} 654 655static const uint32_t vk_to_intel_mipmap_mode[] = { 656 [VK_SAMPLER_MIPMAP_MODE_NEAREST] = MIPFILTER_NEAREST, 657 [VK_SAMPLER_MIPMAP_MODE_LINEAR] = MIPFILTER_LINEAR 658}; 659 660static const uint32_t vk_to_intel_tex_address[] = { 661 [VK_SAMPLER_ADDRESS_MODE_REPEAT] = TCM_WRAP, 662 [VK_SAMPLER_ADDRESS_MODE_MIRRORED_REPEAT] = TCM_MIRROR, 663 [VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE] = TCM_CLAMP, 664 [VK_SAMPLER_ADDRESS_MODE_MIRROR_CLAMP_TO_EDGE] = TCM_MIRROR_ONCE, 665 [VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER] = TCM_CLAMP_BORDER, 666}; 667 668/* Vulkan specifies the result of shadow comparisons as: 669 * 1 if ref <op> texel, 670 * 0 otherwise. 671 * 672 * The hardware does: 673 * 0 if texel <op> ref, 674 * 1 otherwise. 675 * 676 * So, these look a bit strange because there's both a negation 677 * and swapping of the arguments involved. 678 */ 679static const uint32_t vk_to_intel_shadow_compare_op[] = { 680 [VK_COMPARE_OP_NEVER] = PREFILTEROP_ALWAYS, 681 [VK_COMPARE_OP_LESS] = PREFILTEROP_LEQUAL, 682 [VK_COMPARE_OP_EQUAL] = PREFILTEROP_NOTEQUAL, 683 [VK_COMPARE_OP_LESS_OR_EQUAL] = PREFILTEROP_LESS, 684 [VK_COMPARE_OP_GREATER] = PREFILTEROP_GEQUAL, 685 [VK_COMPARE_OP_NOT_EQUAL] = PREFILTEROP_EQUAL, 686 [VK_COMPARE_OP_GREATER_OR_EQUAL] = PREFILTEROP_GREATER, 687 [VK_COMPARE_OP_ALWAYS] = PREFILTEROP_NEVER, 688}; 689 690#if GFX_VER >= 9 691static const uint32_t vk_to_intel_sampler_reduction_mode[] = { 692 [VK_SAMPLER_REDUCTION_MODE_WEIGHTED_AVERAGE_EXT] = STD_FILTER, 693 [VK_SAMPLER_REDUCTION_MODE_MIN_EXT] = MINIMUM, 694 [VK_SAMPLER_REDUCTION_MODE_MAX_EXT] = MAXIMUM, 695}; 696#endif 697 698VkResult genX(CreateSampler)( 699 VkDevice _device, 700 const VkSamplerCreateInfo* pCreateInfo, 701 const VkAllocationCallbacks* pAllocator, 702 VkSampler* pSampler) 703{ 704 ANV_FROM_HANDLE(anv_device, device, _device); 705 struct anv_sampler *sampler; 706 707 assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO); 708 709 sampler = vk_object_zalloc(&device->vk, pAllocator, sizeof(*sampler), 710 VK_OBJECT_TYPE_SAMPLER); 711 if (!sampler) 712 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); 713 714 sampler->n_planes = 1; 715 716 uint32_t border_color_stride = GFX_VERx10 == 75 ? 512 : 64; 717 uint32_t border_color_offset; 718 ASSERTED bool has_custom_color = false; 719 if (pCreateInfo->borderColor <= VK_BORDER_COLOR_INT_OPAQUE_WHITE) { 720 border_color_offset = device->border_colors.offset + 721 pCreateInfo->borderColor * 722 border_color_stride; 723 } else { 724 assert(GFX_VER >= 8); 725 sampler->custom_border_color = 726 anv_state_reserved_pool_alloc(&device->custom_border_colors); 727 border_color_offset = sampler->custom_border_color.offset; 728 } 729 730#if GFX_VER >= 9 731 unsigned sampler_reduction_mode = STD_FILTER; 732 bool enable_sampler_reduction = false; 733#endif 734 735 vk_foreach_struct(ext, pCreateInfo->pNext) { 736 switch (ext->sType) { 737 case VK_STRUCTURE_TYPE_SAMPLER_YCBCR_CONVERSION_INFO: { 738 VkSamplerYcbcrConversionInfo *pSamplerConversion = 739 (VkSamplerYcbcrConversionInfo *) ext; 740 ANV_FROM_HANDLE(anv_ycbcr_conversion, conversion, 741 pSamplerConversion->conversion); 742 743 /* Ignore conversion for non-YUV formats. This fulfills a requirement 744 * for clients that want to utilize same code path for images with 745 * external formats (VK_FORMAT_UNDEFINED) and "regular" RGBA images 746 * where format is known. 747 */ 748 if (conversion == NULL || !conversion->format->can_ycbcr) 749 break; 750 751 sampler->n_planes = conversion->format->n_planes; 752 sampler->conversion = conversion; 753 break; 754 } 755#if GFX_VER >= 9 756 case VK_STRUCTURE_TYPE_SAMPLER_REDUCTION_MODE_CREATE_INFO: { 757 VkSamplerReductionModeCreateInfo *sampler_reduction = 758 (VkSamplerReductionModeCreateInfo *) ext; 759 sampler_reduction_mode = 760 vk_to_intel_sampler_reduction_mode[sampler_reduction->reductionMode]; 761 enable_sampler_reduction = true; 762 break; 763 } 764#endif 765 case VK_STRUCTURE_TYPE_SAMPLER_CUSTOM_BORDER_COLOR_CREATE_INFO_EXT: { 766 VkSamplerCustomBorderColorCreateInfoEXT *custom_border_color = 767 (VkSamplerCustomBorderColorCreateInfoEXT *) ext; 768 if (sampler->custom_border_color.map == NULL) 769 break; 770 struct gfx8_border_color *cbc = sampler->custom_border_color.map; 771 if (custom_border_color->format == VK_FORMAT_B4G4R4A4_UNORM_PACK16) { 772 /* B4G4R4A4_UNORM_PACK16 is treated as R4G4B4A4_UNORM_PACK16 with 773 * a swizzle, but this does not carry over to the sampler for 774 * border colors, so we need to do the swizzle ourselves here. 775 */ 776 cbc->uint32[0] = custom_border_color->customBorderColor.uint32[2]; 777 cbc->uint32[1] = custom_border_color->customBorderColor.uint32[1]; 778 cbc->uint32[2] = custom_border_color->customBorderColor.uint32[0]; 779 cbc->uint32[3] = custom_border_color->customBorderColor.uint32[3]; 780 } else { 781 /* Both structs share the same layout, so just copy them over. */ 782 memcpy(cbc, &custom_border_color->customBorderColor, 783 sizeof(VkClearColorValue)); 784 } 785 has_custom_color = true; 786 break; 787 } 788 default: 789 anv_debug_ignored_stype(ext->sType); 790 break; 791 } 792 } 793 794 assert((sampler->custom_border_color.map == NULL) || has_custom_color); 795 796 if (device->physical->has_bindless_samplers) { 797 /* If we have bindless, allocate enough samplers. We allocate 32 bytes 798 * for each sampler instead of 16 bytes because we want all bindless 799 * samplers to be 32-byte aligned so we don't have to use indirect 800 * sampler messages on them. 801 */ 802 sampler->bindless_state = 803 anv_state_pool_alloc(&device->dynamic_state_pool, 804 sampler->n_planes * 32, 32); 805 } 806 807 for (unsigned p = 0; p < sampler->n_planes; p++) { 808 const bool plane_has_chroma = 809 sampler->conversion && sampler->conversion->format->planes[p].has_chroma; 810 const VkFilter min_filter = 811 plane_has_chroma ? sampler->conversion->chroma_filter : pCreateInfo->minFilter; 812 const VkFilter mag_filter = 813 plane_has_chroma ? sampler->conversion->chroma_filter : pCreateInfo->magFilter; 814 const bool enable_min_filter_addr_rounding = min_filter != VK_FILTER_NEAREST; 815 const bool enable_mag_filter_addr_rounding = mag_filter != VK_FILTER_NEAREST; 816 /* From Broadwell PRM, SAMPLER_STATE: 817 * "Mip Mode Filter must be set to MIPFILTER_NONE for Planar YUV surfaces." 818 */ 819 const bool isl_format_is_planar_yuv = sampler->conversion && 820 isl_format_is_yuv(sampler->conversion->format->planes[0].isl_format) && 821 isl_format_is_planar(sampler->conversion->format->planes[0].isl_format); 822 823 const uint32_t mip_filter_mode = 824 isl_format_is_planar_yuv ? 825 MIPFILTER_NONE : vk_to_intel_mipmap_mode[pCreateInfo->mipmapMode]; 826 827 struct GENX(SAMPLER_STATE) sampler_state = { 828 .SamplerDisable = false, 829 .TextureBorderColorMode = DX10OGL, 830 831#if GFX_VER >= 11 832 .CPSLODCompensationEnable = true, 833#endif 834 835#if GFX_VER >= 8 836 .LODPreClampMode = CLAMP_MODE_OGL, 837#else 838 .LODPreClampEnable = CLAMP_ENABLE_OGL, 839#endif 840 841#if GFX_VER == 8 842 .BaseMipLevel = 0.0, 843#endif 844 .MipModeFilter = mip_filter_mode, 845 .MagModeFilter = vk_to_intel_tex_filter(mag_filter, pCreateInfo->anisotropyEnable), 846 .MinModeFilter = vk_to_intel_tex_filter(min_filter, pCreateInfo->anisotropyEnable), 847 .TextureLODBias = anv_clamp_f(pCreateInfo->mipLodBias, -16, 15.996), 848 .AnisotropicAlgorithm = 849 pCreateInfo->anisotropyEnable ? EWAApproximation : LEGACY, 850 .MinLOD = anv_clamp_f(pCreateInfo->minLod, 0, 14), 851 .MaxLOD = anv_clamp_f(pCreateInfo->maxLod, 0, 14), 852 .ChromaKeyEnable = 0, 853 .ChromaKeyIndex = 0, 854 .ChromaKeyMode = 0, 855 .ShadowFunction = 856 vk_to_intel_shadow_compare_op[pCreateInfo->compareEnable ? 857 pCreateInfo->compareOp : VK_COMPARE_OP_NEVER], 858 .CubeSurfaceControlMode = OVERRIDE, 859 860 .BorderColorPointer = border_color_offset, 861 862#if GFX_VER >= 8 863 .LODClampMagnificationMode = MIPNONE, 864#endif 865 866 .MaximumAnisotropy = vk_to_intel_max_anisotropy(pCreateInfo->maxAnisotropy), 867 .RAddressMinFilterRoundingEnable = enable_min_filter_addr_rounding, 868 .RAddressMagFilterRoundingEnable = enable_mag_filter_addr_rounding, 869 .VAddressMinFilterRoundingEnable = enable_min_filter_addr_rounding, 870 .VAddressMagFilterRoundingEnable = enable_mag_filter_addr_rounding, 871 .UAddressMinFilterRoundingEnable = enable_min_filter_addr_rounding, 872 .UAddressMagFilterRoundingEnable = enable_mag_filter_addr_rounding, 873 .TrilinearFilterQuality = 0, 874 .NonnormalizedCoordinateEnable = pCreateInfo->unnormalizedCoordinates, 875 .TCXAddressControlMode = vk_to_intel_tex_address[pCreateInfo->addressModeU], 876 .TCYAddressControlMode = vk_to_intel_tex_address[pCreateInfo->addressModeV], 877 .TCZAddressControlMode = vk_to_intel_tex_address[pCreateInfo->addressModeW], 878 879#if GFX_VER >= 9 880 .ReductionType = sampler_reduction_mode, 881 .ReductionTypeEnable = enable_sampler_reduction, 882#endif 883 }; 884 885 GENX(SAMPLER_STATE_pack)(NULL, sampler->state[p], &sampler_state); 886 887 if (sampler->bindless_state.map) { 888 memcpy(sampler->bindless_state.map + p * 32, 889 sampler->state[p], GENX(SAMPLER_STATE_length) * 4); 890 } 891 } 892 893 *pSampler = anv_sampler_to_handle(sampler); 894 895 return VK_SUCCESS; 896} 897