1/*
2 * Copyright © 2017 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24#ifndef IRIS_BUFMGR_H
25#define IRIS_BUFMGR_H
26
27#include <stdbool.h>
28#include <stdint.h>
29#include <stdio.h>
30#include <sys/types.h>
31#include "util/macros.h"
32#include "util/u_atomic.h"
33#include "util/list.h"
34#include "pipe/p_defines.h"
35
36struct gen_device_info;
37struct pipe_debug_callback;
38
39/**
40 * Memory zones.  When allocating a buffer, you can request that it is
41 * placed into a specific region of the virtual address space (PPGTT).
42 *
43 * Most buffers can go anywhere (IRIS_MEMZONE_OTHER).  Some buffers are
44 * accessed via an offset from a base address.  STATE_BASE_ADDRESS has
45 * a maximum 4GB size for each region, so we need to restrict those
46 * buffers to be within 4GB of the base.  Each memory zone corresponds
47 * to a particular base address.
48 *
49 * We lay out the virtual address space as follows:
50 *
51 * - [0,   4K): Nothing            (empty page for null address)
52 * - [4K,  4G): Shaders            (Instruction Base Address)
53 * - [4G,  8G): Surfaces & Binders (Surface State Base Address, Bindless ...)
54 * - [8G, 12G): Dynamic            (Dynamic State Base Address)
55 * - [12G, *):  Other              (everything else in the full 48-bit VMA)
56 *
57 * A special buffer for border color lives at the start of the dynamic state
58 * memory zone.  This unfortunately has to be handled specially because the
59 * SAMPLER_STATE "Indirect State Pointer" field is only a 24-bit pointer.
60 *
61 * Each GL context uses a separate GEM context, which technically gives them
62 * each a separate VMA.  However, we assign address globally, so buffers will
63 * have the same address in all GEM contexts.  This lets us have a single BO
64 * field for the address, which is easy and cheap.
65 */
66enum iris_memory_zone {
67   IRIS_MEMZONE_SHADER,
68   IRIS_MEMZONE_BINDER,
69   IRIS_MEMZONE_SURFACE,
70   IRIS_MEMZONE_DYNAMIC,
71   IRIS_MEMZONE_OTHER,
72
73   IRIS_MEMZONE_BORDER_COLOR_POOL,
74};
75
76/* Intentionally exclude single buffer "zones" */
77#define IRIS_MEMZONE_COUNT (IRIS_MEMZONE_OTHER + 1)
78
79#define IRIS_BINDER_SIZE (64 * 1024)
80#define IRIS_MAX_BINDERS 100
81
82#define IRIS_MEMZONE_SHADER_START     (0ull * (1ull << 32))
83#define IRIS_MEMZONE_BINDER_START     (1ull * (1ull << 32))
84#define IRIS_MEMZONE_SURFACE_START    (IRIS_MEMZONE_BINDER_START + IRIS_MAX_BINDERS * IRIS_BINDER_SIZE)
85#define IRIS_MEMZONE_DYNAMIC_START    (2ull * (1ull << 32))
86#define IRIS_MEMZONE_OTHER_START      (3ull * (1ull << 32))
87
88#define IRIS_BORDER_COLOR_POOL_ADDRESS IRIS_MEMZONE_DYNAMIC_START
89#define IRIS_BORDER_COLOR_POOL_SIZE (64 * 1024)
90
91struct iris_bo {
92   /**
93    * Size in bytes of the buffer object.
94    *
95    * The size may be larger than the size originally requested for the
96    * allocation, such as being aligned to page size.
97    */
98   uint64_t size;
99
100   /** Buffer manager context associated with this buffer object */
101   struct iris_bufmgr *bufmgr;
102
103   /** The GEM handle for this buffer object. */
104   uint32_t gem_handle;
105
106   /**
107    * Virtual address of the buffer inside the PPGTT (Per-Process Graphics
108    * Translation Table).
109    *
110    * Although each hardware context has its own VMA, we assign BO's to the
111    * same address in all contexts, for simplicity.
112    */
113   uint64_t gtt_offset;
114
115   /**
116    * The validation list index for this buffer, or -1 when not in a batch.
117    * Note that a single buffer may be in multiple batches (contexts), and
118    * this is a global field, which refers to the last batch using the BO.
119    * It should not be considered authoritative, but can be used to avoid a
120    * linear walk of the validation list in the common case by guessing that
121    * exec_bos[bo->index] == bo and confirming whether that's the case.
122    *
123    * XXX: this is not ideal now that we have more than one batch per context,
124    * XXX: as the index will flop back and forth between the render index and
125    * XXX: compute index...
126    */
127   unsigned index;
128
129   /**
130    * Boolean of whether the GPU is definitely not accessing the buffer.
131    *
132    * This is only valid when reusable, since non-reusable
133    * buffers are those that have been shared with other
134    * processes, so we don't know their state.
135    */
136   bool idle;
137
138   int refcount;
139   const char *name;
140
141   uint64_t kflags;
142
143   /**
144    * Kenel-assigned global name for this object
145    *
146    * List contains both flink named and prime fd'd objects
147    */
148   unsigned global_name;
149
150   /**
151    * Current tiling mode
152    */
153   uint32_t tiling_mode;
154   uint32_t swizzle_mode;
155   uint32_t stride;
156
157   time_t free_time;
158
159   /** Mapped address for the buffer, saved across map/unmap cycles */
160   void *map_cpu;
161   /** GTT virtual address for the buffer, saved across map/unmap cycles */
162   void *map_gtt;
163   /** WC CPU address for the buffer, saved across map/unmap cycles */
164   void *map_wc;
165
166   /** BO cache list */
167   struct list_head head;
168
169   /**
170    * Boolean of whether this buffer can be re-used
171    */
172   bool reusable;
173
174   /**
175    * Boolean of whether this buffer has been shared with an external client.
176    */
177   bool external;
178
179   /**
180    * Boolean of whether this buffer is cache coherent
181    */
182   bool cache_coherent;
183
184   /**
185    * Boolean of whether this buffer points into user memory
186    */
187   bool userptr;
188
189   /** Pre-computed hash using _mesa_hash_pointer for cache tracking sets */
190   uint32_t hash;
191};
192
193#define BO_ALLOC_ZEROED     (1<<0)
194#define BO_ALLOC_COHERENT   (1<<1)
195
196/**
197 * Allocate a buffer object.
198 *
199 * Buffer objects are not necessarily initially mapped into CPU virtual
200 * address space or graphics device aperture.  They must be mapped
201 * using iris_bo_map() to be used by the CPU.
202 */
203struct iris_bo *iris_bo_alloc(struct iris_bufmgr *bufmgr,
204                              const char *name,
205                              uint64_t size,
206                              enum iris_memory_zone memzone);
207
208/**
209 * Allocate a tiled buffer object.
210 *
211 * Alignment for tiled objects is set automatically; the 'flags'
212 * argument provides a hint about how the object will be used initially.
213 *
214 * Valid tiling formats are:
215 *  I915_TILING_NONE
216 *  I915_TILING_X
217 *  I915_TILING_Y
218 */
219struct iris_bo *iris_bo_alloc_tiled(struct iris_bufmgr *bufmgr,
220                                    const char *name,
221                                    uint64_t size,
222                                    enum iris_memory_zone memzone,
223                                    uint32_t tiling_mode,
224                                    uint32_t pitch,
225                                    unsigned flags);
226
227struct iris_bo *
228iris_bo_create_userptr(struct iris_bufmgr *bufmgr, const char *name,
229                       void *ptr, size_t size,
230                       enum iris_memory_zone memzone);
231
232/** Takes a reference on a buffer object */
233static inline void
234iris_bo_reference(struct iris_bo *bo)
235{
236   p_atomic_inc(&bo->refcount);
237}
238
239/**
240 * Releases a reference on a buffer object, freeing the data if
241 * no references remain.
242 */
243void iris_bo_unreference(struct iris_bo *bo);
244
245#define MAP_READ          PIPE_TRANSFER_READ
246#define MAP_WRITE         PIPE_TRANSFER_WRITE
247#define MAP_ASYNC         PIPE_TRANSFER_UNSYNCHRONIZED
248#define MAP_PERSISTENT    PIPE_TRANSFER_PERSISTENT
249#define MAP_COHERENT      PIPE_TRANSFER_COHERENT
250/* internal */
251#define MAP_INTERNAL_MASK (0xff << 24)
252#define MAP_RAW           (0x01 << 24)
253
254#define MAP_FLAGS         (MAP_READ | MAP_WRITE | MAP_ASYNC | \
255                           MAP_PERSISTENT | MAP_COHERENT | MAP_INTERNAL_MASK)
256
257/**
258 * Maps the buffer into userspace.
259 *
260 * This function will block waiting for any existing execution on the
261 * buffer to complete, first.  The resulting mapping is returned.
262 */
263MUST_CHECK void *iris_bo_map(struct pipe_debug_callback *dbg,
264                             struct iris_bo *bo, unsigned flags);
265
266/**
267 * Reduces the refcount on the userspace mapping of the buffer
268 * object.
269 */
270static inline int iris_bo_unmap(struct iris_bo *bo) { return 0; }
271
272/**
273 * Waits for rendering to an object by the GPU to have completed.
274 *
275 * This is not required for any access to the BO by bo_map,
276 * bo_subdata, etc.  It is merely a way for the driver to implement
277 * glFinish.
278 */
279void iris_bo_wait_rendering(struct iris_bo *bo);
280
281/**
282 * Tears down the buffer manager instance.
283 */
284void iris_bufmgr_destroy(struct iris_bufmgr *bufmgr);
285
286/**
287 * Get the current tiling (and resulting swizzling) mode for the bo.
288 *
289 * \param buf Buffer to get tiling mode for
290 * \param tiling_mode returned tiling mode
291 * \param swizzle_mode returned swizzling mode
292 */
293int iris_bo_get_tiling(struct iris_bo *bo, uint32_t *tiling_mode,
294                      uint32_t *swizzle_mode);
295
296/**
297 * Create a visible name for a buffer which can be used by other apps
298 *
299 * \param buf Buffer to create a name for
300 * \param name Returned name
301 */
302int iris_bo_flink(struct iris_bo *bo, uint32_t *name);
303
304/**
305 * Returns 1 if mapping the buffer for write could cause the process
306 * to block, due to the object being active in the GPU.
307 */
308int iris_bo_busy(struct iris_bo *bo);
309
310/**
311 * Specify the volatility of the buffer.
312 * \param bo Buffer to create a name for
313 * \param madv The purgeable status
314 *
315 * Use I915_MADV_DONTNEED to mark the buffer as purgeable, and it will be
316 * reclaimed under memory pressure. If you subsequently require the buffer,
317 * then you must pass I915_MADV_WILLNEED to mark the buffer as required.
318 *
319 * Returns 1 if the buffer was retained, or 0 if it was discarded whilst
320 * marked as I915_MADV_DONTNEED.
321 */
322int iris_bo_madvise(struct iris_bo *bo, int madv);
323
324/* drm_bacon_bufmgr_gem.c */
325struct iris_bufmgr *iris_bufmgr_init(struct gen_device_info *devinfo, int fd);
326struct iris_bo *iris_bo_gem_create_from_name(struct iris_bufmgr *bufmgr,
327                                             const char *name,
328                                             unsigned handle);
329void iris_bufmgr_enable_reuse(struct iris_bufmgr *bufmgr);
330
331int iris_bo_wait(struct iris_bo *bo, int64_t timeout_ns);
332
333uint32_t iris_create_hw_context(struct iris_bufmgr *bufmgr);
334
335#define IRIS_CONTEXT_LOW_PRIORITY    ((I915_CONTEXT_MIN_USER_PRIORITY-1)/2)
336#define IRIS_CONTEXT_MEDIUM_PRIORITY (I915_CONTEXT_DEFAULT_PRIORITY)
337#define IRIS_CONTEXT_HIGH_PRIORITY   ((I915_CONTEXT_MAX_USER_PRIORITY+1)/2)
338
339int iris_hw_context_set_priority(struct iris_bufmgr *bufmgr,
340                                 uint32_t ctx_id, int priority);
341
342void iris_destroy_hw_context(struct iris_bufmgr *bufmgr, uint32_t ctx_id);
343
344int iris_bo_export_dmabuf(struct iris_bo *bo, int *prime_fd);
345struct iris_bo *iris_bo_import_dmabuf(struct iris_bufmgr *bufmgr, int prime_fd);
346
347uint32_t iris_bo_export_gem_handle(struct iris_bo *bo);
348
349int iris_reg_read(struct iris_bufmgr *bufmgr, uint32_t offset, uint64_t *out);
350
351int drm_ioctl(int fd, unsigned long request, void *arg);
352
353/**
354 * Returns the BO's address relative to the appropriate base address.
355 *
356 * All of our base addresses are programmed to the start of a 4GB region,
357 * so simply returning the bottom 32 bits of the BO address will give us
358 * the offset from whatever base address corresponds to that memory region.
359 */
360static inline uint32_t
361iris_bo_offset_from_base_address(struct iris_bo *bo)
362{
363   /* This only works for buffers in the memory zones corresponding to a
364    * base address - the top, unbounded memory zone doesn't have a base.
365    */
366   assert(bo->gtt_offset < IRIS_MEMZONE_OTHER_START);
367   return bo->gtt_offset;
368}
369
370enum iris_memory_zone iris_memzone_for_address(uint64_t address);
371
372#endif /* IRIS_BUFMGR_H */
373