1/*
2 * Copyright © 2015 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24#include <assert.h>
25#include <stdbool.h>
26#include <string.h>
27#include <unistd.h>
28#include <fcntl.h>
29
30#include "anv_private.h"
31
32#include "common/gen_sample_positions.h"
33#include "genxml/gen_macros.h"
34#include "genxml/genX_pack.h"
35
36#include "vk_util.h"
37
38#if GEN_GEN == 10
39/**
40 * From Gen10 Workarounds page in h/w specs:
41 * WaSampleOffsetIZ:
42 *    "Prior to the 3DSTATE_SAMPLE_PATTERN driver must ensure there are no
43 *     markers in the pipeline by programming a PIPE_CONTROL with stall."
44 */
45static void
46gen10_emit_wa_cs_stall_flush(struct anv_batch *batch)
47{
48
49   anv_batch_emit(batch, GENX(PIPE_CONTROL), pc) {
50      pc.CommandStreamerStallEnable = true;
51      pc.StallAtPixelScoreboard = true;
52   }
53}
54
55/**
56 * From Gen10 Workarounds page in h/w specs:
57 * WaSampleOffsetIZ:_cs_stall_flush
58 *    "When 3DSTATE_SAMPLE_PATTERN is programmed, driver must then issue an
59 *     MI_LOAD_REGISTER_IMM command to an offset between 0x7000 and 0x7FFF(SVL)
60 *     after the command to ensure the state has been delivered prior to any
61 *     command causing a marker in the pipeline."
62 */
63static void
64gen10_emit_wa_lri_to_cache_mode_zero(struct anv_batch *batch)
65{
66   /* Before changing the value of CACHE_MODE_0 register, GFX pipeline must
67    * be idle; i.e., full flush is required.
68    */
69   anv_batch_emit(batch, GENX(PIPE_CONTROL), pc) {
70      pc.DepthCacheFlushEnable = true;
71      pc.DCFlushEnable = true;
72      pc.RenderTargetCacheFlushEnable = true;
73      pc.InstructionCacheInvalidateEnable = true;
74      pc.StateCacheInvalidationEnable = true;
75      pc.TextureCacheInvalidationEnable = true;
76      pc.VFCacheInvalidationEnable = true;
77      pc.ConstantCacheInvalidationEnable =true;
78   }
79
80   /* Write to CACHE_MODE_0 (0x7000) */
81   uint32_t cache_mode_0 = 0;
82   anv_pack_struct(&cache_mode_0, GENX(CACHE_MODE_0));
83
84   anv_batch_emit(batch, GENX(MI_LOAD_REGISTER_IMM), lri) {
85      lri.RegisterOffset = GENX(CACHE_MODE_0_num);
86      lri.DataDWord      = cache_mode_0;
87   }
88}
89#endif
90
91VkResult
92genX(init_device_state)(struct anv_device *device)
93{
94   device->default_mocs = GENX(MOCS);
95#if GEN_GEN >= 8
96   device->external_mocs = GENX(EXTERNAL_MOCS);
97#else
98   device->external_mocs = device->default_mocs;
99#endif
100
101   struct anv_batch batch;
102
103   uint32_t cmds[64];
104   batch.start = batch.next = cmds;
105   batch.end = (void *) cmds + sizeof(cmds);
106
107   anv_batch_emit(&batch, GENX(PIPELINE_SELECT), ps) {
108#if GEN_GEN >= 9
109      ps.MaskBits = 3;
110#endif
111      ps.PipelineSelection = _3D;
112   }
113
114#if GEN_GEN == 9
115   uint32_t cache_mode_1;
116   anv_pack_struct(&cache_mode_1, GENX(CACHE_MODE_1),
117                   .FloatBlendOptimizationEnable = true,
118                   .FloatBlendOptimizationEnableMask = true,
119                   .PartialResolveDisableInVC = true,
120                   .PartialResolveDisableInVCMask = true);
121
122   anv_batch_emit(&batch, GENX(MI_LOAD_REGISTER_IMM), lri) {
123      lri.RegisterOffset = GENX(CACHE_MODE_1_num);
124      lri.DataDWord      = cache_mode_1;
125   }
126#endif
127
128   anv_batch_emit(&batch, GENX(3DSTATE_AA_LINE_PARAMETERS), aa);
129
130   anv_batch_emit(&batch, GENX(3DSTATE_DRAWING_RECTANGLE), rect) {
131      rect.ClippedDrawingRectangleYMin = 0;
132      rect.ClippedDrawingRectangleXMin = 0;
133      rect.ClippedDrawingRectangleYMax = UINT16_MAX;
134      rect.ClippedDrawingRectangleXMax = UINT16_MAX;
135      rect.DrawingRectangleOriginY = 0;
136      rect.DrawingRectangleOriginX = 0;
137   }
138
139#if GEN_GEN >= 8
140   anv_batch_emit(&batch, GENX(3DSTATE_WM_CHROMAKEY), ck);
141
142#if GEN_GEN == 10
143   gen10_emit_wa_cs_stall_flush(&batch);
144#endif
145
146   /* See the Vulkan 1.0 spec Table 24.1 "Standard sample locations" and
147    * VkPhysicalDeviceFeatures::standardSampleLocations.
148    */
149   anv_batch_emit(&batch, GENX(3DSTATE_SAMPLE_PATTERN), sp) {
150      GEN_SAMPLE_POS_1X(sp._1xSample);
151      GEN_SAMPLE_POS_2X(sp._2xSample);
152      GEN_SAMPLE_POS_4X(sp._4xSample);
153      GEN_SAMPLE_POS_8X(sp._8xSample);
154#if GEN_GEN >= 9
155      GEN_SAMPLE_POS_16X(sp._16xSample);
156#endif
157   }
158
159   /* The BDW+ docs describe how to use the 3DSTATE_WM_HZ_OP instruction in the
160    * section titled, "Optimized Depth Buffer Clear and/or Stencil Buffer
161    * Clear." It mentions that the packet overrides GPU state for the clear
162    * operation and needs to be reset to 0s to clear the overrides. Depending
163    * on the kernel, we may not get a context with the state for this packet
164    * zeroed. Do it ourselves just in case. We've observed this to prevent a
165    * number of GPU hangs on ICL.
166    */
167   anv_batch_emit(&batch, GENX(3DSTATE_WM_HZ_OP), hzp);
168#endif
169
170#if GEN_GEN == 10
171   gen10_emit_wa_lri_to_cache_mode_zero(&batch);
172#endif
173
174#if GEN_GEN == 11
175   /* The default behavior of bit 5 "Headerless Message for Pre-emptable
176    * Contexts" in SAMPLER MODE register is set to 0, which means
177    * headerless sampler messages are not allowed for pre-emptable
178    * contexts. Set the bit 5 to 1 to allow them.
179    */
180   uint32_t sampler_mode;
181   anv_pack_struct(&sampler_mode, GENX(SAMPLER_MODE),
182                   .HeaderlessMessageforPreemptableContexts = true,
183                   .HeaderlessMessageforPreemptableContextsMask = true);
184
185    anv_batch_emit(&batch, GENX(MI_LOAD_REGISTER_IMM), lri) {
186      lri.RegisterOffset = GENX(SAMPLER_MODE_num);
187      lri.DataDWord      = sampler_mode;
188   }
189
190   /* Bit 1 "Enabled Texel Offset Precision Fix" must be set in
191    * HALF_SLICE_CHICKEN7 register.
192    */
193   uint32_t half_slice_chicken7;
194   anv_pack_struct(&half_slice_chicken7, GENX(HALF_SLICE_CHICKEN7),
195                   .EnabledTexelOffsetPrecisionFix = true,
196                   .EnabledTexelOffsetPrecisionFixMask = true);
197
198    anv_batch_emit(&batch, GENX(MI_LOAD_REGISTER_IMM), lri) {
199      lri.RegisterOffset = GENX(HALF_SLICE_CHICKEN7_num);
200      lri.DataDWord      = half_slice_chicken7;
201   }
202
203   /* WaEnableStateCacheRedirectToCS:icl */
204   uint32_t slice_common_eco_chicken1;
205   anv_pack_struct(&slice_common_eco_chicken1,
206                   GENX(SLICE_COMMON_ECO_CHICKEN1),
207                   .StateCacheRedirectToCSSectionEnable = true,
208                   .StateCacheRedirectToCSSectionEnableMask = true);
209
210   anv_batch_emit(&batch, GENX(MI_LOAD_REGISTER_IMM), lri) {
211      lri.RegisterOffset = GENX(SLICE_COMMON_ECO_CHICKEN1_num);
212      lri.DataDWord      = slice_common_eco_chicken1;
213   }
214#endif
215
216   /* Set the "CONSTANT_BUFFER Address Offset Disable" bit, so
217    * 3DSTATE_CONSTANT_XS buffer 0 is an absolute address.
218    *
219    * This is only safe on kernels with context isolation support.
220    */
221   if (GEN_GEN >= 8 &&
222       device->instance->physicalDevice.has_context_isolation) {
223      UNUSED uint32_t tmp_reg;
224#if GEN_GEN >= 9
225      anv_pack_struct(&tmp_reg, GENX(CS_DEBUG_MODE2),
226                      .CONSTANT_BUFFERAddressOffsetDisable = true,
227                      .CONSTANT_BUFFERAddressOffsetDisableMask = true);
228      anv_batch_emit(&batch, GENX(MI_LOAD_REGISTER_IMM), lri) {
229         lri.RegisterOffset = GENX(CS_DEBUG_MODE2_num);
230         lri.DataDWord      = tmp_reg;
231      }
232#elif GEN_GEN == 8
233      anv_pack_struct(&tmp_reg, GENX(INSTPM),
234                      .CONSTANT_BUFFERAddressOffsetDisable = true,
235                      .CONSTANT_BUFFERAddressOffsetDisableMask = true);
236      anv_batch_emit(&batch, GENX(MI_LOAD_REGISTER_IMM), lri) {
237         lri.RegisterOffset = GENX(INSTPM_num);
238         lri.DataDWord      = tmp_reg;
239      }
240#endif
241   }
242
243   anv_batch_emit(&batch, GENX(MI_BATCH_BUFFER_END), bbe);
244
245   assert(batch.next <= batch.end);
246
247   return anv_device_submit_simple_batch(device, &batch);
248}
249
250static uint32_t
251vk_to_gen_tex_filter(VkFilter filter, bool anisotropyEnable)
252{
253   switch (filter) {
254   default:
255      assert(!"Invalid filter");
256   case VK_FILTER_NEAREST:
257      return anisotropyEnable ? MAPFILTER_ANISOTROPIC : MAPFILTER_NEAREST;
258   case VK_FILTER_LINEAR:
259      return anisotropyEnable ? MAPFILTER_ANISOTROPIC : MAPFILTER_LINEAR;
260   }
261}
262
263static uint32_t
264vk_to_gen_max_anisotropy(float ratio)
265{
266   return (anv_clamp_f(ratio, 2, 16) - 2) / 2;
267}
268
269static const uint32_t vk_to_gen_mipmap_mode[] = {
270   [VK_SAMPLER_MIPMAP_MODE_NEAREST]          = MIPFILTER_NEAREST,
271   [VK_SAMPLER_MIPMAP_MODE_LINEAR]           = MIPFILTER_LINEAR
272};
273
274static const uint32_t vk_to_gen_tex_address[] = {
275   [VK_SAMPLER_ADDRESS_MODE_REPEAT]          = TCM_WRAP,
276   [VK_SAMPLER_ADDRESS_MODE_MIRRORED_REPEAT] = TCM_MIRROR,
277   [VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE]   = TCM_CLAMP,
278   [VK_SAMPLER_ADDRESS_MODE_MIRROR_CLAMP_TO_EDGE] = TCM_MIRROR_ONCE,
279   [VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER] = TCM_CLAMP_BORDER,
280};
281
282/* Vulkan specifies the result of shadow comparisons as:
283 *     1     if   ref <op> texel,
284 *     0     otherwise.
285 *
286 * The hardware does:
287 *     0     if texel <op> ref,
288 *     1     otherwise.
289 *
290 * So, these look a bit strange because there's both a negation
291 * and swapping of the arguments involved.
292 */
293static const uint32_t vk_to_gen_shadow_compare_op[] = {
294   [VK_COMPARE_OP_NEVER]                        = PREFILTEROPALWAYS,
295   [VK_COMPARE_OP_LESS]                         = PREFILTEROPLEQUAL,
296   [VK_COMPARE_OP_EQUAL]                        = PREFILTEROPNOTEQUAL,
297   [VK_COMPARE_OP_LESS_OR_EQUAL]                = PREFILTEROPLESS,
298   [VK_COMPARE_OP_GREATER]                      = PREFILTEROPGEQUAL,
299   [VK_COMPARE_OP_NOT_EQUAL]                    = PREFILTEROPEQUAL,
300   [VK_COMPARE_OP_GREATER_OR_EQUAL]             = PREFILTEROPGREATER,
301   [VK_COMPARE_OP_ALWAYS]                       = PREFILTEROPNEVER,
302};
303
304#if GEN_GEN >= 9
305static const uint32_t vk_to_gen_sampler_reduction_mode[] = {
306   [VK_SAMPLER_REDUCTION_MODE_WEIGHTED_AVERAGE_EXT] = STD_FILTER,
307   [VK_SAMPLER_REDUCTION_MODE_MIN_EXT]              = MINIMUM,
308   [VK_SAMPLER_REDUCTION_MODE_MAX_EXT]              = MAXIMUM,
309};
310#endif
311
312VkResult genX(CreateSampler)(
313    VkDevice                                    _device,
314    const VkSamplerCreateInfo*                  pCreateInfo,
315    const VkAllocationCallbacks*                pAllocator,
316    VkSampler*                                  pSampler)
317{
318   ANV_FROM_HANDLE(anv_device, device, _device);
319   const struct anv_physical_device *pdevice =
320      &device->instance->physicalDevice;
321   struct anv_sampler *sampler;
322
323   assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO);
324
325   sampler = vk_zalloc2(&device->alloc, pAllocator, sizeof(*sampler), 8,
326                        VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
327   if (!sampler)
328      return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
329
330   sampler->n_planes = 1;
331
332   uint32_t border_color_offset = device->border_colors.offset +
333                                  pCreateInfo->borderColor * 64;
334
335#if GEN_GEN >= 9
336   unsigned sampler_reduction_mode = STD_FILTER;
337   bool enable_sampler_reduction = false;
338#endif
339
340   vk_foreach_struct(ext, pCreateInfo->pNext) {
341      switch (ext->sType) {
342      case VK_STRUCTURE_TYPE_SAMPLER_YCBCR_CONVERSION_INFO: {
343         VkSamplerYcbcrConversionInfo *pSamplerConversion =
344            (VkSamplerYcbcrConversionInfo *) ext;
345         ANV_FROM_HANDLE(anv_ycbcr_conversion, conversion,
346                         pSamplerConversion->conversion);
347
348         /* Ignore conversion for non-YUV formats. This fulfills a requirement
349          * for clients that want to utilize same code path for images with
350          * external formats (VK_FORMAT_UNDEFINED) and "regular" RGBA images
351          * where format is known.
352          */
353         if (conversion == NULL || !conversion->format->can_ycbcr)
354            break;
355
356         sampler->n_planes = conversion->format->n_planes;
357         sampler->conversion = conversion;
358         break;
359      }
360#if GEN_GEN >= 9
361      case VK_STRUCTURE_TYPE_SAMPLER_REDUCTION_MODE_CREATE_INFO_EXT: {
362         struct VkSamplerReductionModeCreateInfoEXT *sampler_reduction =
363            (struct VkSamplerReductionModeCreateInfoEXT *) ext;
364         sampler_reduction_mode =
365            vk_to_gen_sampler_reduction_mode[sampler_reduction->reductionMode];
366         enable_sampler_reduction = true;
367         break;
368      }
369#endif
370      default:
371         anv_debug_ignored_stype(ext->sType);
372         break;
373      }
374   }
375
376   if (pdevice->has_bindless_samplers) {
377      /* If we have bindless, allocate enough samplers.  We allocate 32 bytes
378       * for each sampler instead of 16 bytes because we want all bindless
379       * samplers to be 32-byte aligned so we don't have to use indirect
380       * sampler messages on them.
381       */
382      sampler->bindless_state =
383         anv_state_pool_alloc(&device->dynamic_state_pool,
384                              sampler->n_planes * 32, 32);
385   }
386
387   for (unsigned p = 0; p < sampler->n_planes; p++) {
388      const bool plane_has_chroma =
389         sampler->conversion && sampler->conversion->format->planes[p].has_chroma;
390      const VkFilter min_filter =
391         plane_has_chroma ? sampler->conversion->chroma_filter : pCreateInfo->minFilter;
392      const VkFilter mag_filter =
393         plane_has_chroma ? sampler->conversion->chroma_filter : pCreateInfo->magFilter;
394      const bool enable_min_filter_addr_rounding = min_filter != VK_FILTER_NEAREST;
395      const bool enable_mag_filter_addr_rounding = mag_filter != VK_FILTER_NEAREST;
396      /* From Broadwell PRM, SAMPLER_STATE:
397       *   "Mip Mode Filter must be set to MIPFILTER_NONE for Planar YUV surfaces."
398       */
399      const uint32_t mip_filter_mode =
400         (sampler->conversion &&
401          isl_format_is_yuv(sampler->conversion->format->planes[0].isl_format)) ?
402         MIPFILTER_NONE : vk_to_gen_mipmap_mode[pCreateInfo->mipmapMode];
403
404      struct GENX(SAMPLER_STATE) sampler_state = {
405         .SamplerDisable = false,
406         .TextureBorderColorMode = DX10OGL,
407
408#if GEN_GEN >= 8
409         .LODPreClampMode = CLAMP_MODE_OGL,
410#else
411         .LODPreClampEnable = CLAMP_ENABLE_OGL,
412#endif
413
414#if GEN_GEN == 8
415         .BaseMipLevel = 0.0,
416#endif
417         .MipModeFilter = mip_filter_mode,
418         .MagModeFilter = vk_to_gen_tex_filter(mag_filter, pCreateInfo->anisotropyEnable),
419         .MinModeFilter = vk_to_gen_tex_filter(min_filter, pCreateInfo->anisotropyEnable),
420         .TextureLODBias = anv_clamp_f(pCreateInfo->mipLodBias, -16, 15.996),
421         .AnisotropicAlgorithm = EWAApproximation,
422         .MinLOD = anv_clamp_f(pCreateInfo->minLod, 0, 14),
423         .MaxLOD = anv_clamp_f(pCreateInfo->maxLod, 0, 14),
424         .ChromaKeyEnable = 0,
425         .ChromaKeyIndex = 0,
426         .ChromaKeyMode = 0,
427         .ShadowFunction = vk_to_gen_shadow_compare_op[pCreateInfo->compareOp],
428         .CubeSurfaceControlMode = OVERRIDE,
429
430         .BorderColorPointer = border_color_offset,
431
432#if GEN_GEN >= 8
433         .LODClampMagnificationMode = MIPNONE,
434#endif
435
436         .MaximumAnisotropy = vk_to_gen_max_anisotropy(pCreateInfo->maxAnisotropy),
437         .RAddressMinFilterRoundingEnable = enable_min_filter_addr_rounding,
438         .RAddressMagFilterRoundingEnable = enable_mag_filter_addr_rounding,
439         .VAddressMinFilterRoundingEnable = enable_min_filter_addr_rounding,
440         .VAddressMagFilterRoundingEnable = enable_mag_filter_addr_rounding,
441         .UAddressMinFilterRoundingEnable = enable_min_filter_addr_rounding,
442         .UAddressMagFilterRoundingEnable = enable_mag_filter_addr_rounding,
443         .TrilinearFilterQuality = 0,
444         .NonnormalizedCoordinateEnable = pCreateInfo->unnormalizedCoordinates,
445         .TCXAddressControlMode = vk_to_gen_tex_address[pCreateInfo->addressModeU],
446         .TCYAddressControlMode = vk_to_gen_tex_address[pCreateInfo->addressModeV],
447         .TCZAddressControlMode = vk_to_gen_tex_address[pCreateInfo->addressModeW],
448
449#if GEN_GEN >= 9
450         .ReductionType = sampler_reduction_mode,
451         .ReductionTypeEnable = enable_sampler_reduction,
452#endif
453      };
454
455      GENX(SAMPLER_STATE_pack)(NULL, sampler->state[p], &sampler_state);
456
457      if (sampler->bindless_state.map) {
458         memcpy(sampler->bindless_state.map + p * 32,
459                sampler->state[p], GENX(SAMPLER_STATE_length) * 4);
460      }
461   }
462
463   *pSampler = anv_sampler_to_handle(sampler);
464
465   return VK_SUCCESS;
466}
467