1/*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24#include "anv_private.h"
25
26#include "genxml/gen_macros.h"
27#include "genxml/genX_pack.h"
28
29#include "common/gen_l3_config.h"
30
31/**
32 * This file implements some lightweight memcpy/memset operations on the GPU
33 * using a vertex buffer and streamout.
34 */
35
36/**
37 * Returns the greatest common divisor of a and b that is a power of two.
38 */
39static uint64_t
40gcd_pow2_u64(uint64_t a, uint64_t b)
41{
42   assert(a > 0 || b > 0);
43
44   unsigned a_log2 = ffsll(a) - 1;
45   unsigned b_log2 = ffsll(b) - 1;
46
47   /* If either a or b is 0, then a_log2 or b_log2 will be UINT_MAX in which
48    * case, the MIN2() will take the other one.  If both are 0 then we will
49    * hit the assert above.
50    */
51   return 1 << MIN2(a_log2, b_log2);
52}
53
54void
55genX(cmd_buffer_so_memcpy)(struct anv_cmd_buffer *cmd_buffer,
56                           struct anv_address dst, struct anv_address src,
57                           uint32_t size)
58{
59   if (size == 0)
60      return;
61
62   /* The maximum copy block size is 4 32-bit components at a time. */
63   assert(size % 4 == 0);
64   unsigned bs = gcd_pow2_u64(16, size);
65
66   enum isl_format format;
67   switch (bs) {
68   case 4:  format = ISL_FORMAT_R32_UINT;          break;
69   case 8:  format = ISL_FORMAT_R32G32_UINT;       break;
70   case 16: format = ISL_FORMAT_R32G32B32A32_UINT; break;
71   default:
72      unreachable("Invalid size");
73   }
74
75   if (!cmd_buffer->state.current_l3_config) {
76      const struct gen_l3_config *cfg =
77         gen_get_default_l3_config(&cmd_buffer->device->info);
78      genX(cmd_buffer_config_l3)(cmd_buffer, cfg);
79   }
80
81   genX(cmd_buffer_apply_pipe_flushes)(cmd_buffer);
82
83   genX(flush_pipeline_select_3d)(cmd_buffer);
84
85   uint32_t *dw;
86   dw = anv_batch_emitn(&cmd_buffer->batch, 5, GENX(3DSTATE_VERTEX_BUFFERS));
87   GENX(VERTEX_BUFFER_STATE_pack)(&cmd_buffer->batch, dw + 1,
88      &(struct GENX(VERTEX_BUFFER_STATE)) {
89         .VertexBufferIndex = 32, /* Reserved for this */
90         .AddressModifyEnable = true,
91         .BufferStartingAddress = src,
92         .BufferPitch = bs,
93         .MOCS = anv_mocs_for_bo(cmd_buffer->device, src.bo),
94#if (GEN_GEN >= 8)
95         .BufferSize = size,
96#else
97         .EndAddress = anv_address_add(src, size - 1),
98#endif
99      });
100
101   dw = anv_batch_emitn(&cmd_buffer->batch, 3, GENX(3DSTATE_VERTEX_ELEMENTS));
102   GENX(VERTEX_ELEMENT_STATE_pack)(&cmd_buffer->batch, dw + 1,
103      &(struct GENX(VERTEX_ELEMENT_STATE)) {
104         .VertexBufferIndex = 32,
105         .Valid = true,
106         .SourceElementFormat = format,
107         .SourceElementOffset = 0,
108         .Component0Control = (bs >= 4) ? VFCOMP_STORE_SRC : VFCOMP_STORE_0,
109         .Component1Control = (bs >= 8) ? VFCOMP_STORE_SRC : VFCOMP_STORE_0,
110         .Component2Control = (bs >= 12) ? VFCOMP_STORE_SRC : VFCOMP_STORE_0,
111         .Component3Control = (bs >= 16) ? VFCOMP_STORE_SRC : VFCOMP_STORE_0,
112      });
113
114#if GEN_GEN >= 8
115   anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_VF_SGVS), sgvs);
116#endif
117
118   /* Disable all shader stages */
119   anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_VS), vs);
120   anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_HS), hs);
121   anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_TE), te);
122   anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_DS), DS);
123   anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_GS), gs);
124   anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_PS), gs);
125
126   anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_SBE), sbe) {
127      sbe.VertexURBEntryReadOffset = 1;
128      sbe.NumberofSFOutputAttributes = 1;
129      sbe.VertexURBEntryReadLength = 1;
130#if GEN_GEN >= 8
131      sbe.ForceVertexURBEntryReadLength = true;
132      sbe.ForceVertexURBEntryReadOffset = true;
133#endif
134
135#if GEN_GEN >= 9
136      for (unsigned i = 0; i < 32; i++)
137         sbe.AttributeActiveComponentFormat[i] = ACF_XYZW;
138#endif
139   }
140
141   /* Emit URB setup.  We tell it that the VS is active because we want it to
142    * allocate space for the VS.  Even though one isn't run, we need VUEs to
143    * store the data that VF is going to pass to SOL.
144    */
145   const unsigned entry_size[4] = { DIV_ROUND_UP(32, 64), 1, 1, 1 };
146
147   genX(emit_urb_setup)(cmd_buffer->device, &cmd_buffer->batch,
148                        cmd_buffer->state.current_l3_config,
149                        VK_SHADER_STAGE_VERTEX_BIT, entry_size);
150
151   anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_SO_BUFFER), sob) {
152      sob.SOBufferIndex = 0;
153      sob.MOCS = anv_mocs_for_bo(cmd_buffer->device, dst.bo),
154      sob.SurfaceBaseAddress = dst;
155
156#if GEN_GEN >= 8
157      sob.SOBufferEnable = true;
158      sob.SurfaceSize = size / 4 - 1;
159#else
160      sob.SurfacePitch = bs;
161      sob.SurfaceEndAddress = anv_address_add(dst, size);
162#endif
163
164#if GEN_GEN >= 8
165      /* As SOL writes out data, it updates the SO_WRITE_OFFSET registers with
166       * the end position of the stream.  We need to reset this value to 0 at
167       * the beginning of the run or else SOL will start at the offset from
168       * the previous draw.
169       */
170      sob.StreamOffsetWriteEnable = true;
171      sob.StreamOffset = 0;
172#endif
173   }
174
175#if GEN_GEN <= 7
176   /* The hardware can do this for us on BDW+ (see above) */
177   anv_batch_emit(&cmd_buffer->batch, GENX(MI_LOAD_REGISTER_IMM), load) {
178      load.RegisterOffset = GENX(SO_WRITE_OFFSET0_num);
179      load.DataDWord = 0;
180   }
181#endif
182
183   dw = anv_batch_emitn(&cmd_buffer->batch, 5, GENX(3DSTATE_SO_DECL_LIST),
184                        .StreamtoBufferSelects0 = (1 << 0),
185                        .NumEntries0 = 1);
186   GENX(SO_DECL_ENTRY_pack)(&cmd_buffer->batch, dw + 3,
187      &(struct GENX(SO_DECL_ENTRY)) {
188         .Stream0Decl = {
189            .OutputBufferSlot = 0,
190            .RegisterIndex = 0,
191            .ComponentMask = (1 << (bs / 4)) - 1,
192         },
193      });
194
195   anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_STREAMOUT), so) {
196      so.SOFunctionEnable = true;
197      so.RenderingDisable = true;
198      so.Stream0VertexReadOffset = 0;
199      so.Stream0VertexReadLength = DIV_ROUND_UP(32, 64);
200#if GEN_GEN >= 8
201      so.Buffer0SurfacePitch = bs;
202#else
203      so.SOBufferEnable0 = true;
204#endif
205   }
206
207#if GEN_GEN >= 8
208   anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_VF_TOPOLOGY), topo) {
209      topo.PrimitiveTopologyType = _3DPRIM_POINTLIST;
210   }
211#endif
212
213   anv_batch_emit(&cmd_buffer->batch, GENX(3DSTATE_VF_STATISTICS), vf) {
214      vf.StatisticsEnable = false;
215   }
216
217   anv_batch_emit(&cmd_buffer->batch, GENX(3DPRIMITIVE), prim) {
218      prim.VertexAccessType         = SEQUENTIAL;
219      prim.PrimitiveTopologyType    = _3DPRIM_POINTLIST;
220      prim.VertexCountPerInstance   = size / bs;
221      prim.StartVertexLocation      = 0;
222      prim.InstanceCount            = 1;
223      prim.StartInstanceLocation    = 0;
224      prim.BaseVertexLocation       = 0;
225   }
226
227   cmd_buffer->state.gfx.dirty |= ANV_CMD_DIRTY_PIPELINE;
228}
229