1/*
2 * Copyright © 2011 Marek Olšák <maraeo@gmail.com>
3 * Copyright © 2015 Advanced Micro Devices, Inc.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining
7 * a copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
16 * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17 * NON-INFRINGEMENT. IN NO EVENT SHALL THE COPYRIGHT HOLDERS, AUTHORS
18 * AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 *
23 * The above copyright notice and this permission notice (including the
24 * next paragraph) shall be included in all copies or substantial portions
25 * of the Software.
26 */
27
28#ifndef AMDGPU_CS_H
29#define AMDGPU_CS_H
30
31#include "amdgpu_bo.h"
32#include "util/u_memory.h"
33#include <amdgpu_drm.h>
34
35struct amdgpu_ctx {
36   struct amdgpu_winsys *ws;
37   amdgpu_context_handle ctx;
38   amdgpu_bo_handle user_fence_bo;
39   uint64_t *user_fence_cpu_address_base;
40   int refcount;
41   unsigned initial_num_total_rejected_cs;
42   unsigned num_rejected_cs;
43};
44
45struct amdgpu_cs_buffer {
46   struct amdgpu_winsys_bo *bo;
47   union {
48      struct {
49         uint32_t priority_usage;
50      } real;
51      struct {
52         uint32_t real_idx; /* index of underlying real BO */
53      } slab;
54   } u;
55   enum radeon_bo_usage usage;
56};
57
58enum ib_type {
59   IB_MAIN,
60   IB_NUM,
61};
62
63struct amdgpu_ib {
64   struct radeon_cmdbuf base;
65
66   /* A buffer out of which new IBs are allocated. */
67   struct pb_buffer        *big_ib_buffer;
68   uint8_t                 *ib_mapped;
69   unsigned                used_ib_space;
70
71   /* The maximum seen size from cs_check_space. If the driver does
72    * cs_check_space and flush, the newly allocated IB should have at least
73    * this size.
74    */
75   unsigned                max_check_space_size;
76
77   unsigned                max_ib_size;
78   uint32_t                *ptr_ib_size;
79   bool                    ptr_ib_size_inside_ib;
80   enum ib_type            ib_type;
81};
82
83struct amdgpu_fence_list {
84   struct pipe_fence_handle    **list;
85   unsigned                    num;
86   unsigned                    max;
87};
88
89struct amdgpu_cs_context {
90   struct drm_amdgpu_cs_chunk_ib ib[IB_NUM];
91
92   /* Buffers. */
93   unsigned                    max_real_buffers;
94   unsigned                    num_real_buffers;
95   struct amdgpu_cs_buffer     *real_buffers;
96
97   unsigned                    num_slab_buffers;
98   unsigned                    max_slab_buffers;
99   struct amdgpu_cs_buffer     *slab_buffers;
100
101   unsigned                    num_sparse_buffers;
102   unsigned                    max_sparse_buffers;
103   struct amdgpu_cs_buffer     *sparse_buffers;
104
105   int                         buffer_indices_hashlist[4096];
106
107   struct amdgpu_winsys_bo     *last_added_bo;
108   unsigned                    last_added_bo_index;
109   unsigned                    last_added_bo_usage;
110   uint32_t                    last_added_bo_priority_usage;
111
112   struct amdgpu_fence_list    fence_dependencies;
113   struct amdgpu_fence_list    syncobj_dependencies;
114   struct amdgpu_fence_list    syncobj_to_signal;
115
116   struct pipe_fence_handle    *fence;
117
118   /* the error returned from cs_flush for non-async submissions */
119   int                         error_code;
120};
121
122struct amdgpu_cs {
123   struct amdgpu_ib main; /* must be first because this is inherited */
124   struct amdgpu_ctx *ctx;
125   enum ring_type ring_type;
126   struct drm_amdgpu_cs_chunk_fence fence_chunk;
127
128   /* We flip between these two CS. While one is being consumed
129    * by the kernel in another thread, the other one is being filled
130    * by the pipe driver. */
131   struct amdgpu_cs_context csc1;
132   struct amdgpu_cs_context csc2;
133   /* The currently-used CS. */
134   struct amdgpu_cs_context *csc;
135   /* The CS being currently-owned by the other thread. */
136   struct amdgpu_cs_context *cst;
137
138   /* Flush CS. */
139   void (*flush_cs)(void *ctx, unsigned flags, struct pipe_fence_handle **fence);
140   void *flush_data;
141   bool stop_exec_on_failure;
142
143   struct util_queue_fence flush_completed;
144   struct pipe_fence_handle *next_fence;
145};
146
147struct amdgpu_fence {
148   struct pipe_reference reference;
149   /* If ctx == NULL, this fence is syncobj-based. */
150   uint32_t syncobj;
151
152   struct amdgpu_winsys *ws;
153   struct amdgpu_ctx *ctx;  /* submission context */
154   struct amdgpu_cs_fence fence;
155   uint64_t *user_fence_cpu_address;
156
157   /* If the fence has been submitted. This is unsignalled for deferred fences
158    * (cs->next_fence) and while an IB is still being submitted in the submit
159    * thread. */
160   struct util_queue_fence submitted;
161
162   volatile int signalled;              /* bool (int for atomicity) */
163};
164
165static inline bool amdgpu_fence_is_syncobj(struct amdgpu_fence *fence)
166{
167   return fence->ctx == NULL;
168}
169
170static inline void amdgpu_ctx_unref(struct amdgpu_ctx *ctx)
171{
172   if (p_atomic_dec_zero(&ctx->refcount)) {
173      amdgpu_cs_ctx_free(ctx->ctx);
174      amdgpu_bo_free(ctx->user_fence_bo);
175      FREE(ctx);
176   }
177}
178
179static inline void amdgpu_fence_reference(struct pipe_fence_handle **dst,
180                                          struct pipe_fence_handle *src)
181{
182   struct amdgpu_fence **adst = (struct amdgpu_fence **)dst;
183   struct amdgpu_fence *asrc = (struct amdgpu_fence *)src;
184
185   if (pipe_reference(&(*adst)->reference, &asrc->reference)) {
186      struct amdgpu_fence *fence = *adst;
187
188      if (amdgpu_fence_is_syncobj(fence))
189         amdgpu_cs_destroy_syncobj(fence->ws->dev, fence->syncobj);
190      else
191         amdgpu_ctx_unref(fence->ctx);
192
193      util_queue_fence_destroy(&fence->submitted);
194      FREE(fence);
195   }
196   *adst = asrc;
197}
198
199int amdgpu_lookup_buffer(struct amdgpu_cs_context *cs, struct amdgpu_winsys_bo *bo);
200
201static inline struct amdgpu_ib *
202amdgpu_ib(struct radeon_cmdbuf *base)
203{
204   return (struct amdgpu_ib *)base;
205}
206
207static inline struct amdgpu_cs *
208amdgpu_cs(struct radeon_cmdbuf *base)
209{
210   assert(amdgpu_ib(base)->ib_type == IB_MAIN);
211   return (struct amdgpu_cs*)base;
212}
213
214#define get_container(member_ptr, container_type, container_member) \
215   (container_type *)((char *)(member_ptr) - offsetof(container_type, container_member))
216
217static inline struct amdgpu_cs *
218amdgpu_cs_from_ib(struct amdgpu_ib *ib)
219{
220   switch (ib->ib_type) {
221   case IB_MAIN:
222      return get_container(ib, struct amdgpu_cs, main);
223   default:
224      unreachable("bad ib_type");
225   }
226}
227
228static inline bool
229amdgpu_bo_is_referenced_by_cs(struct amdgpu_cs *cs,
230                              struct amdgpu_winsys_bo *bo)
231{
232   int num_refs = bo->num_cs_references;
233   return num_refs == bo->ws->num_cs ||
234         (num_refs && amdgpu_lookup_buffer(cs->csc, bo) != -1);
235}
236
237static inline bool
238amdgpu_bo_is_referenced_by_cs_with_usage(struct amdgpu_cs *cs,
239                                         struct amdgpu_winsys_bo *bo,
240                                         enum radeon_bo_usage usage)
241{
242   int index;
243   struct amdgpu_cs_buffer *buffer;
244
245   if (!bo->num_cs_references)
246      return false;
247
248   index = amdgpu_lookup_buffer(cs->csc, bo);
249   if (index == -1)
250      return false;
251
252   buffer = bo->bo ? &cs->csc->real_buffers[index] :
253            bo->sparse ? &cs->csc->sparse_buffers[index] :
254            &cs->csc->slab_buffers[index];
255
256   return (buffer->usage & usage) != 0;
257}
258
259static inline bool
260amdgpu_bo_is_referenced_by_any_cs(struct amdgpu_winsys_bo *bo)
261{
262   return bo->num_cs_references != 0;
263}
264
265bool amdgpu_fence_wait(struct pipe_fence_handle *fence, uint64_t timeout,
266                       bool absolute);
267void amdgpu_add_fences(struct amdgpu_winsys_bo *bo,
268                       unsigned num_fences,
269                       struct pipe_fence_handle **fences);
270void amdgpu_cs_sync_flush(struct radeon_cmdbuf *rcs);
271void amdgpu_cs_init_functions(struct amdgpu_winsys *ws);
272void amdgpu_cs_submit_ib(void *job, int thread_index);
273
274#endif
275