1/*
2 * Copyright 2010 Christoph Bumiller
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23#include "nouveau_screen.h"
24#include "nouveau_winsys.h"
25#include "nouveau_fence.h"
26#include "util/os_time.h"
27
28#ifdef PIPE_OS_UNIX
29#include <sched.h>
30#endif
31
32bool
33nouveau_fence_new(struct nouveau_screen *screen, struct nouveau_fence **fence)
34{
35   *fence = CALLOC_STRUCT(nouveau_fence);
36   if (!*fence)
37      return false;
38
39   (*fence)->screen = screen;
40   (*fence)->ref = 1;
41   LIST_INITHEAD(&(*fence)->work);
42
43   return true;
44}
45
46static void
47nouveau_fence_trigger_work(struct nouveau_fence *fence)
48{
49   struct nouveau_fence_work *work, *tmp;
50
51   LIST_FOR_EACH_ENTRY_SAFE(work, tmp, &fence->work, list) {
52      work->func(work->data);
53      LIST_DEL(&work->list);
54      FREE(work);
55   }
56}
57
58void
59nouveau_fence_emit(struct nouveau_fence *fence)
60{
61   struct nouveau_screen *screen = fence->screen;
62
63   assert(fence->state == NOUVEAU_FENCE_STATE_AVAILABLE);
64
65   /* set this now, so that if fence.emit triggers a flush we don't recurse */
66   fence->state = NOUVEAU_FENCE_STATE_EMITTING;
67
68   ++fence->ref;
69
70   if (screen->fence.tail)
71      screen->fence.tail->next = fence;
72   else
73      screen->fence.head = fence;
74
75   screen->fence.tail = fence;
76
77   screen->fence.emit(&screen->base, &fence->sequence);
78
79   assert(fence->state == NOUVEAU_FENCE_STATE_EMITTING);
80   fence->state = NOUVEAU_FENCE_STATE_EMITTED;
81}
82
83void
84nouveau_fence_del(struct nouveau_fence *fence)
85{
86   struct nouveau_fence *it;
87   struct nouveau_screen *screen = fence->screen;
88
89   if (fence->state == NOUVEAU_FENCE_STATE_EMITTED ||
90       fence->state == NOUVEAU_FENCE_STATE_FLUSHED) {
91      if (fence == screen->fence.head) {
92         screen->fence.head = fence->next;
93         if (!screen->fence.head)
94            screen->fence.tail = NULL;
95      } else {
96         for (it = screen->fence.head; it && it->next != fence; it = it->next);
97         it->next = fence->next;
98         if (screen->fence.tail == fence)
99            screen->fence.tail = it;
100      }
101   }
102
103   if (!LIST_IS_EMPTY(&fence->work)) {
104      debug_printf("WARNING: deleting fence with work still pending !\n");
105      nouveau_fence_trigger_work(fence);
106   }
107
108   FREE(fence);
109}
110
111void
112nouveau_fence_update(struct nouveau_screen *screen, bool flushed)
113{
114   struct nouveau_fence *fence;
115   struct nouveau_fence *next = NULL;
116   u32 sequence = screen->fence.update(&screen->base);
117
118   if (screen->fence.sequence_ack == sequence)
119      return;
120   screen->fence.sequence_ack = sequence;
121
122   for (fence = screen->fence.head; fence; fence = next) {
123      next = fence->next;
124      sequence = fence->sequence;
125
126      fence->state = NOUVEAU_FENCE_STATE_SIGNALLED;
127
128      nouveau_fence_trigger_work(fence);
129      nouveau_fence_ref(NULL, &fence);
130
131      if (sequence == screen->fence.sequence_ack)
132         break;
133   }
134   screen->fence.head = next;
135   if (!next)
136      screen->fence.tail = NULL;
137
138   if (flushed) {
139      for (fence = next; fence; fence = fence->next)
140         if (fence->state == NOUVEAU_FENCE_STATE_EMITTED)
141            fence->state = NOUVEAU_FENCE_STATE_FLUSHED;
142   }
143}
144
145#define NOUVEAU_FENCE_MAX_SPINS (1 << 31)
146
147bool
148nouveau_fence_signalled(struct nouveau_fence *fence)
149{
150   struct nouveau_screen *screen = fence->screen;
151
152   if (fence->state == NOUVEAU_FENCE_STATE_SIGNALLED)
153      return true;
154
155   if (fence->state >= NOUVEAU_FENCE_STATE_EMITTED)
156      nouveau_fence_update(screen, false);
157
158   return fence->state == NOUVEAU_FENCE_STATE_SIGNALLED;
159}
160
161static bool
162nouveau_fence_kick(struct nouveau_fence *fence)
163{
164   struct nouveau_screen *screen = fence->screen;
165
166   /* wtf, someone is waiting on a fence in flush_notify handler? */
167   assert(fence->state != NOUVEAU_FENCE_STATE_EMITTING);
168
169   if (fence->state < NOUVEAU_FENCE_STATE_EMITTED) {
170      PUSH_SPACE(screen->pushbuf, 8);
171      /* The space allocation might trigger a flush, which could emit the
172       * current fence. So check again.
173       */
174      if (fence->state < NOUVEAU_FENCE_STATE_EMITTED)
175         nouveau_fence_emit(fence);
176   }
177
178   if (fence->state < NOUVEAU_FENCE_STATE_FLUSHED)
179      if (nouveau_pushbuf_kick(screen->pushbuf, screen->pushbuf->channel))
180         return false;
181
182   if (fence == screen->fence.current)
183      nouveau_fence_next(screen);
184
185   nouveau_fence_update(screen, false);
186
187   return true;
188}
189
190bool
191nouveau_fence_wait(struct nouveau_fence *fence, struct pipe_debug_callback *debug)
192{
193   struct nouveau_screen *screen = fence->screen;
194   uint32_t spins = 0;
195   int64_t start = 0;
196
197   if (debug && debug->debug_message)
198      start = os_time_get_nano();
199
200   if (!nouveau_fence_kick(fence))
201      return false;
202
203   do {
204      if (fence->state == NOUVEAU_FENCE_STATE_SIGNALLED) {
205         if (debug && debug->debug_message)
206            pipe_debug_message(debug, PERF_INFO,
207                               "stalled %.3f ms waiting for fence",
208                               (os_time_get_nano() - start) / 1000000.f);
209         return true;
210      }
211      if (!spins)
212         NOUVEAU_DRV_STAT(screen, any_non_kernel_fence_sync_count, 1);
213      spins++;
214#ifdef PIPE_OS_UNIX
215      if (!(spins % 8)) /* donate a few cycles */
216         sched_yield();
217#endif
218
219      nouveau_fence_update(screen, false);
220   } while (spins < NOUVEAU_FENCE_MAX_SPINS);
221
222   debug_printf("Wait on fence %u (ack = %u, next = %u) timed out !\n",
223                fence->sequence,
224                screen->fence.sequence_ack, screen->fence.sequence);
225
226   return false;
227}
228
229void
230nouveau_fence_next(struct nouveau_screen *screen)
231{
232   if (screen->fence.current->state < NOUVEAU_FENCE_STATE_EMITTING) {
233      if (screen->fence.current->ref > 1)
234         nouveau_fence_emit(screen->fence.current);
235      else
236         return;
237   }
238
239   nouveau_fence_ref(NULL, &screen->fence.current);
240
241   nouveau_fence_new(screen, &screen->fence.current);
242}
243
244void
245nouveau_fence_unref_bo(void *data)
246{
247   struct nouveau_bo *bo = data;
248
249   nouveau_bo_ref(NULL, &bo);
250}
251
252bool
253nouveau_fence_work(struct nouveau_fence *fence,
254                   void (*func)(void *), void *data)
255{
256   struct nouveau_fence_work *work;
257
258   if (!fence || fence->state == NOUVEAU_FENCE_STATE_SIGNALLED) {
259      func(data);
260      return true;
261   }
262
263   work = CALLOC_STRUCT(nouveau_fence_work);
264   if (!work)
265      return false;
266   work->func = func;
267   work->data = data;
268   LIST_ADD(&work->list, &fence->work);
269   p_atomic_inc(&fence->work_count);
270   if (fence->work_count > 64)
271      nouveau_fence_kick(fence);
272   return true;
273}
274