Lines Matching refs:fence
42 #include <linux/dma-fence.h>
51 SDT_PROBE_DEFINE1(sdt, drm, fence, init,
52 "struct dma_fence *"/*fence*/);
53 SDT_PROBE_DEFINE1(sdt, drm, fence, reset,
54 "struct dma_fence *"/*fence*/);
55 SDT_PROBE_DEFINE1(sdt, drm, fence, release,
56 "struct dma_fence *"/*fence*/);
57 SDT_PROBE_DEFINE1(sdt, drm, fence, free,
58 "struct dma_fence *"/*fence*/);
59 SDT_PROBE_DEFINE1(sdt, drm, fence, destroy,
60 "struct dma_fence *"/*fence*/);
62 SDT_PROBE_DEFINE1(sdt, drm, fence, enable_signaling,
63 "struct dma_fence *"/*fence*/);
64 SDT_PROBE_DEFINE2(sdt, drm, fence, add_callback,
65 "struct dma_fence *"/*fence*/,
67 SDT_PROBE_DEFINE2(sdt, drm, fence, remove_callback,
68 "struct dma_fence *"/*fence*/,
70 SDT_PROBE_DEFINE2(sdt, drm, fence, callback,
71 "struct dma_fence *"/*fence*/,
73 SDT_PROBE_DEFINE1(sdt, drm, fence, test,
74 "struct dma_fence *"/*fence*/);
75 SDT_PROBE_DEFINE2(sdt, drm, fence, set_error,
76 "struct dma_fence *"/*fence*/,
78 SDT_PROBE_DEFINE1(sdt, drm, fence, signal,
79 "struct dma_fence *"/*fence*/);
81 SDT_PROBE_DEFINE3(sdt, drm, fence, wait_start,
82 "struct dma_fence *"/*fence*/,
85 SDT_PROBE_DEFINE2(sdt, drm, fence, wait_done,
86 "struct dma_fence *"/*fence*/,
100 struct dma_fence fence;
106 KASSERT(f == &dma_fence_stub.fence);
114 KASSERT(f == &dma_fence_stub.fence);
135 dma_fence_init(&dma_fence_stub.fence, &dma_fence_stub_ops,
137 error = dma_fence_signal(&dma_fence_stub.fence);
145 dma_fence_put(&dma_fence_stub.fence);
150 * dma_fence_referenced_p(fence)
152 * True if fence has a positive reference count. True after
154 * false. The fence must have been initialized and must not have
158 dma_fence_referenced_p(struct dma_fence *fence)
161 KASSERTMSG(fence->f_magic != FENCE_MAGIC_BAD, "fence %p", fence);
162 KASSERTMSG(fence->f_magic == FENCE_MAGIC_GOOD, "fence %p", fence);
164 return kref_referenced_p(&fence->refcount);
168 * dma_fence_init(fence, ops, lock, context, seqno)
170 * Initialize fence. Caller should call dma_fence_destroy when
174 dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops,
178 kref_init(&fence->refcount);
179 fence->lock = lock;
180 fence->flags = 0;
181 fence->context = context;
182 fence->seqno = seqno;
183 fence->ops = ops;
184 fence->error = 0;
185 TAILQ_INIT(&fence->f_callbacks);
186 cv_init(&fence->f_cv, "dmafence");
189 fence->f_magic = FENCE_MAGIC_GOOD;
192 SDT_PROBE1(sdt, drm, fence, init, fence);
196 * dma_fence_reset(fence)
198 * Ensure fence is in a quiescent state. Allowed either for newly
205 dma_fence_reset(struct dma_fence *fence, const struct dma_fence_ops *ops,
209 KASSERTMSG(fence->f_magic != FENCE_MAGIC_BAD, "fence %p", fence);
210 KASSERTMSG(fence->f_magic == FENCE_MAGIC_GOOD, "fence %p", fence);
211 KASSERT(kref_read(&fence->refcount) == 0 ||
212 kref_read(&fence->refcount) == 1);
213 KASSERT(TAILQ_EMPTY(&fence->f_callbacks));
214 KASSERT(fence->lock == lock);
215 KASSERT(fence->ops == ops);
217 kref_init(&fence->refcount);
218 fence->flags = 0;
219 fence->context = context;
220 fence->seqno = seqno;
221 fence->error = 0;
223 SDT_PROBE1(sdt, drm, fence, reset, fence);
227 * dma_fence_destroy(fence)
230 * to be used after a fence release callback.
235 dma_fence_destroy(struct dma_fence *fence)
238 KASSERT(!dma_fence_referenced_p(fence));
240 SDT_PROBE1(sdt, drm, fence, destroy, fence);
243 fence->f_magic = FENCE_MAGIC_BAD;
246 KASSERT(TAILQ_EMPTY(&fence->f_callbacks));
247 cv_destroy(&fence->f_cv);
253 struct dma_fence *fence = container_of(rcu, struct dma_fence, rcu);
255 KASSERT(!dma_fence_referenced_p(fence));
257 dma_fence_destroy(fence);
258 kfree(fence);
262 * dma_fence_free(fence)
264 * Schedule fence to be destroyed and then freed with kfree after
267 * is meant to be used after a fence release callback.
274 dma_fence_free(struct dma_fence *fence)
277 KASSERT(!dma_fence_referenced_p(fence));
279 SDT_PROBE1(sdt, drm, fence, free, fence);
281 call_rcu(&fence->rcu, &dma_fence_free_cb);
313 * according to the given fence ops.
315 * - For fence ops with 64-bit sequence numbers, this is simply
318 * - For fence ops with 32-bit sequence numbers, this is defined
335 * True if the sequence number of fence a is later than the
336 * sequence number of fence b. Since sequence numbers wrap
338 * fence a is no more than INT_MAX past the sequence number of
339 * fence b.
348 KASSERTMSG(a->f_magic != FENCE_MAGIC_BAD, "fence %p", a);
349 KASSERTMSG(a->f_magic == FENCE_MAGIC_GOOD, "fence %p", a);
350 KASSERTMSG(b->f_magic != FENCE_MAGIC_BAD, "fence %p", b);
351 KASSERTMSG(b->f_magic == FENCE_MAGIC_GOOD, "fence %p", b);
362 * Return a dma fence that is always already signalled.
368 return dma_fence_get(&dma_fence_stub.fence);
372 * dma_fence_get(fence)
374 * Acquire a reference to fence and return it, or return NULL if
375 * fence is NULL. The fence, if nonnull, must not be being
379 dma_fence_get(struct dma_fence *fence)
382 if (fence == NULL)
385 KASSERTMSG(fence->f_magic != FENCE_MAGIC_BAD, "fence %p", fence);
386 KASSERTMSG(fence->f_magic == FENCE_MAGIC_GOOD, "fence %p", fence);
388 kref_get(&fence->refcount);
389 return fence;
393 * dma_fence_get_rcu(fence)
395 * Attempt to acquire a reference to a fence that may be about to
396 * be destroyed, during a read section. Return the fence on
397 * success, or NULL on failure. The fence must be nonnull.
400 dma_fence_get_rcu(struct dma_fence *fence)
404 KASSERTMSG(fence->f_magic != FENCE_MAGIC_BAD, "fence %p", fence);
405 KASSERTMSG(fence->f_magic == FENCE_MAGIC_GOOD, "fence %p", fence);
406 if (!kref_get_unless_zero(&fence->refcount))
408 return fence;
414 * Attempt to acquire a reference to the fence *fencep, which may
423 struct dma_fence *fence;
427 * Load the fence, ensuring we observe the fully initialized
430 if ((fence = atomic_load_consume(fencep)) == NULL)
434 if (!dma_fence_get_rcu(fence))
438 * Confirm that it's still the same fence. If not, release it
441 if (fence != atomic_load_relaxed(fencep)) {
442 dma_fence_put(fence);
447 KASSERT(dma_fence_referenced_p(fence));
448 return fence;
454 struct dma_fence *fence = container_of(refcount, struct dma_fence,
457 KASSERTMSG(TAILQ_EMPTY(&fence->f_callbacks),
458 "fence %p has pending callbacks", fence);
459 KASSERT(!dma_fence_referenced_p(fence));
461 SDT_PROBE1(sdt, drm, fence, release, fence);
463 if (fence->ops->release)
464 (*fence->ops->release)(fence);
466 dma_fence_free(fence);
470 * dma_fence_put(fence)
472 * Release a reference to fence. If this was the last one, call
473 * the fence's release callback.
476 dma_fence_put(struct dma_fence *fence)
479 if (fence == NULL)
481 KASSERT(dma_fence_referenced_p(fence));
482 kref_put(&fence->refcount, &dma_fence_release);
486 * dma_fence_ensure_signal_enabled(fence)
488 * Internal subroutine. If the fence was already signalled,
490 * has not been called yet, call it. If fails, signal the fence
494 * Caller must hold the fence's lock.
497 dma_fence_ensure_signal_enabled(struct dma_fence *fence)
501 KASSERT(dma_fence_referenced_p(fence));
502 KASSERT(spin_is_locked(fence->lock));
506 &fence->flags);
508 /* If the fence was already signalled, fail with -ENOENT. */
509 if (fence->flags & (1u << DMA_FENCE_FLAG_SIGNALED_BIT))
516 if (!already_enabled && fence->ops->enable_signaling) {
517 SDT_PROBE1(sdt, drm, fence, enable_signaling, fence);
518 if (!(*fence->ops->enable_signaling)(fence)) {
520 dma_fence_signal_locked(fence);
530 * dma_fence_add_callback(fence, fcb, fn)
532 * If fence has been signalled, return -ENOENT. If the enable
534 * fails, return -ENOENT. Otherwise, arrange to call fn(fence,
537 * The fence uses memory allocated by the caller in fcb from the
542 dma_fence_add_callback(struct dma_fence *fence, struct dma_fence_cb *fcb,
547 KASSERT(dma_fence_referenced_p(fence));
550 if (atomic_load_relaxed(&fence->flags) &
557 spin_lock(fence->lock);
560 ret = dma_fence_ensure_signal_enabled(fence);
565 SDT_PROBE2(sdt, drm, fence, add_callback, fence, fcb);
567 TAILQ_INSERT_TAIL(&fence->f_callbacks, fcb, fcb_entry);
572 out1: spin_unlock(fence->lock);
581 * dma_fence_remove_callback(fence, fcb)
583 * Remove the callback fcb from fence. Return true if it was
586 * dma_fence_add_callback(fence, fcb).
589 dma_fence_remove_callback(struct dma_fence *fence, struct dma_fence_cb *fcb)
593 KASSERT(dma_fence_referenced_p(fence));
595 spin_lock(fence->lock);
598 SDT_PROBE2(sdt, drm, fence, remove_callback, fence, fcb);
599 TAILQ_REMOVE(&fence->f_callbacks, fcb, fcb_entry);
602 spin_unlock(fence->lock);
608 * dma_fence_enable_sw_signaling(fence)
610 * If it hasn't been called yet and the fence hasn't been
611 * signalled yet, call the fence's enable_sw_signaling callback.
613 fence.
616 dma_fence_enable_sw_signaling(struct dma_fence *fence)
619 KASSERT(dma_fence_referenced_p(fence));
621 spin_lock(fence->lock);
622 if ((fence->flags & (1u << DMA_FENCE_FLAG_SIGNALED_BIT)) == 0)
623 (void)dma_fence_ensure_signal_enabled(fence);
624 spin_unlock(fence->lock);
628 * dma_fence_is_signaled(fence)
630 * Test whether the fence has been signalled. If it has been
637 dma_fence_is_signaled(struct dma_fence *fence)
641 KASSERT(dma_fence_referenced_p(fence));
643 spin_lock(fence->lock);
644 signaled = dma_fence_is_signaled_locked(fence);
645 spin_unlock(fence->lock);
651 * dma_fence_is_signaled_locked(fence)
653 * Test whether the fence has been signalled. Like
654 * dma_fence_is_signaleed, but caller already holds the fence's lock.
657 dma_fence_is_signaled_locked(struct dma_fence *fence)
660 KASSERT(dma_fence_referenced_p(fence));
661 KASSERT(spin_is_locked(fence->lock));
664 if (fence->flags & (1u << DMA_FENCE_FLAG_SIGNALED_BIT))
668 if (fence->ops->signaled) {
669 SDT_PROBE1(sdt, drm, fence, test, fence);
670 if ((*fence->ops->signaled)(fence)) {
676 dma_fence_signal_locked(fence);
685 * dma_fence_set_error(fence, error)
688 * waiter to learn about success or failure of the fence.
691 dma_fence_set_error(struct dma_fence *fence, int error)
694 KASSERTMSG(fence->f_magic != FENCE_MAGIC_BAD, "fence %p", fence);
695 KASSERTMSG(fence->f_magic == FENCE_MAGIC_GOOD, "fence %p", fence);
696 KASSERT((atomic_load_relaxed(&fence->flags) &
701 SDT_PROBE2(sdt, drm, fence, set_error, fence, error);
702 fence->error = error;
706 * dma_fence_get_status(fence)
708 * Return 0 if fence has yet to be signalled, 1 if it has been
713 dma_fence_get_status(struct dma_fence *fence)
717 KASSERTMSG(fence->f_magic != FENCE_MAGIC_BAD, "fence %p", fence);
718 KASSERTMSG(fence->f_magic == FENCE_MAGIC_GOOD, "fence %p", fence);
720 spin_lock(fence->lock);
721 if (!dma_fence_is_signaled_locked(fence)) {
723 } else if (fence->error) {
724 ret = fence->error;
729 spin_unlock(fence->lock);
735 * dma_fence_signal(fence)
737 * Signal the fence. If it has already been signalled, return
744 dma_fence_signal(struct dma_fence *fence)
748 KASSERT(dma_fence_referenced_p(fence));
750 spin_lock(fence->lock);
751 ret = dma_fence_signal_locked(fence);
752 spin_unlock(fence->lock);
758 * dma_fence_signal_locked(fence)
760 * Signal the fence. Like dma_fence_signal, but caller already
761 * holds the fence's lock.
764 dma_fence_signal_locked(struct dma_fence *fence)
768 KASSERT(dma_fence_referenced_p(fence));
769 KASSERT(spin_is_locked(fence->lock));
772 if (test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
775 SDT_PROBE1(sdt, drm, fence, signal, fence);
778 fence->timestamp = ktime_get();
779 set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);
782 cv_broadcast(&fence->f_cv);
785 TAILQ_FOREACH_SAFE(fcb, &fence->f_callbacks, fcb_entry, next) {
786 SDT_PROBE2(sdt, drm, fence, callback, fence, fcb);
787 TAILQ_REMOVE(&fence->f_callbacks, fcb, fcb_entry);
789 (*fcb->func)(fence, fcb);
807 wait_any_cb(struct dma_fence *fence, struct dma_fence_cb *fcb)
811 KASSERT(dma_fence_referenced_p(fence));
820 * dma_fence_wait_any_timeout(fence, nfences, intr, timeout, ip)
866 cv_init(&common.cv, "fence");
978 * dma_fence_wait_timeout(fence, intr, timeout)
980 * Wait until fence is signalled; or until interrupt, if intr is
983 * timeout, or positive number of ticks remaining if the fence is
984 * signalled before the timeout. Works by calling the fence wait
991 dma_fence_wait_timeout(struct dma_fence *fence, bool intr, long timeout)
995 KASSERT(dma_fence_referenced_p(fence));
999 SDT_PROBE3(sdt, drm, fence, wait_start, fence, intr, timeout);
1000 if (fence->ops->wait)
1001 ret = (*fence->ops->wait)(fence, intr, timeout);
1003 ret = dma_fence_default_wait(fence, intr, timeout);
1004 SDT_PROBE2(sdt, drm, fence, wait_done, fence, ret);
1010 * dma_fence_wait(fence, intr)
1012 * Wait until fence is signalled; or until interrupt, if intr is
1014 * on any other error, zero on sucess. Works by calling the fence
1018 dma_fence_wait(struct dma_fence *fence, bool intr)
1022 KASSERT(dma_fence_referenced_p(fence));
1024 ret = dma_fence_wait_timeout(fence, intr, MAX_SCHEDULE_TIMEOUT);
1033 * dma_fence_default_wait(fence, intr, timeout)
1035 * Default implementation of fence wait callback using a condition
1036 * variable. If the fence is already signalled, return timeout,
1039 * if the fence had been signalled. Otherwise, wait on the
1044 dma_fence_default_wait(struct dma_fence *fence, bool intr, long timeout)
1047 kmutex_t *lock = &fence->lock->sl_lock;
1050 KASSERT(dma_fence_referenced_p(fence));
1055 if (atomic_load_relaxed(&fence->flags) &
1060 spin_lock(fence->lock);
1063 if (dma_fence_ensure_signal_enabled(fence) != 0) {
1082 while (!(fence->flags & (1u << DMA_FENCE_FLAG_SIGNALED_BIT))) {
1098 ret = -cv_timedwait_sig(&fence->f_cv, lock,
1101 ret = -cv_wait_sig(&fence->f_cv, lock);
1105 ret = -cv_timedwait(&fence->f_cv, lock,
1108 cv_wait(&fence->f_cv, lock);
1139 if (fence->flags & (1u << DMA_FENCE_FLAG_SIGNALED_BIT)) {
1147 spin_unlock(fence->lock);
1152 * __dma_fence_signal(fence)
1154 * Set fence's signalled bit, without waking waiters yet. Return
1158 __dma_fence_signal(struct dma_fence *fence)
1161 KASSERTMSG(fence->f_magic != FENCE_MAGIC_BAD, "fence %p", fence);
1162 KASSERTMSG(fence->f_magic == FENCE_MAGIC_GOOD, "fence %p", fence);
1164 if (test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
1171 * __dma_fence_signal_wake(fence)
1173 * Set fence's timestamp and wake fence's waiters. Caller must
1178 __dma_fence_signal_wake(struct dma_fence *fence, ktime_t timestamp)
1182 KASSERTMSG(fence->f_magic != FENCE_MAGIC_BAD, "fence %p", fence);
1183 KASSERTMSG(fence->f_magic == FENCE_MAGIC_GOOD, "fence %p", fence);
1185 spin_lock(fence->lock);
1187 KASSERT(fence->flags & DMA_FENCE_FLAG_SIGNALED_BIT);
1189 SDT_PROBE1(sdt, drm, fence, signal, fence);
1192 fence->timestamp = timestamp;
1193 set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);
1196 cv_broadcast(&fence->f_cv);
1199 TAILQ_FOREACH_SAFE(fcb, &fence->f_callbacks, fcb_entry, next) {
1200 TAILQ_REMOVE(&fence->f_callbacks, fcb, fcb_entry);
1202 (*fcb->func)(fence, fcb);
1205 spin_unlock(fence->lock);
Indexes created Thu Nov 06 07:09:58 GMT 2025