1/*
2 * Copyright © 2015 Intel
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24#ifndef _SIMPLE_MTX_H
25#define _SIMPLE_MTX_H
26
27#include "util/futex.h"
28#include "util/macros.h"
29
30#include "c11/threads.h"
31
32#if UTIL_FUTEX_SUPPORTED
33
34#if defined(HAVE_VALGRIND) && !defined(NDEBUG)
35#  include <valgrind.h>
36#  include <helgrind.h>
37#  define HG(x) x
38#else
39#  define HG(x)
40#endif
41
42/* mtx_t - Fast, simple mutex
43 *
44 * While modern pthread mutexes are very fast (implemented using futex), they
45 * still incur a call to an external DSO and overhead of the generality and
46 * features of pthread mutexes.  Most mutexes in mesa only needs lock/unlock,
47 * and the idea here is that we can inline the atomic operation and make the
48 * fast case just two intructions.  Mutexes are subtle and finicky to
49 * implement, so we carefully copy the implementation from Ulrich Dreppers
50 * well-written and well-reviewed paper:
51 *
52 *   "Futexes Are Tricky"
53 *   http://www.akkadia.org/drepper/futex.pdf
54 *
55 * We implement "mutex3", which gives us a mutex that has no syscalls on
56 * uncontended lock or unlock.  Further, the uncontended case boils down to a
57 * locked cmpxchg and an untaken branch, the uncontended unlock is just a
58 * locked decr and an untaken branch.  We use __builtin_expect() to indicate
59 * that contention is unlikely so that gcc will put the contention code out of
60 * the main code flow.
61 *
62 * A fast mutex only supports lock/unlock, can't be recursive or used with
63 * condition variables.
64 */
65
66typedef struct {
67   uint32_t val;
68} simple_mtx_t;
69
70#define _SIMPLE_MTX_INITIALIZER_NP { 0 }
71
72#define _SIMPLE_MTX_INVALID_VALUE 0xd0d0d0d0
73
74static inline void
75simple_mtx_init(simple_mtx_t *mtx, ASSERTED int type)
76{
77   assert(type == mtx_plain);
78
79   mtx->val = 0;
80
81   HG(ANNOTATE_RWLOCK_CREATE(mtx));
82}
83
84static inline void
85simple_mtx_destroy(ASSERTED simple_mtx_t *mtx)
86{
87   HG(ANNOTATE_RWLOCK_DESTROY(mtx));
88#ifndef NDEBUG
89   mtx->val = _SIMPLE_MTX_INVALID_VALUE;
90#endif
91}
92
93static inline void
94simple_mtx_lock(simple_mtx_t *mtx)
95{
96   uint32_t c;
97
98   c = __sync_val_compare_and_swap(&mtx->val, 0, 1);
99
100   assert(c != _SIMPLE_MTX_INVALID_VALUE);
101
102   if (__builtin_expect(c != 0, 0)) {
103      if (c != 2)
104         c = __sync_lock_test_and_set(&mtx->val, 2);
105      while (c != 0) {
106         futex_wait(&mtx->val, 2, NULL);
107         c = __sync_lock_test_and_set(&mtx->val, 2);
108      }
109   }
110
111   HG(ANNOTATE_RWLOCK_ACQUIRED(mtx, 1));
112}
113
114static inline void
115simple_mtx_unlock(simple_mtx_t *mtx)
116{
117   uint32_t c;
118
119   HG(ANNOTATE_RWLOCK_RELEASED(mtx, 1));
120
121   c = __sync_fetch_and_sub(&mtx->val, 1);
122
123   assert(c != _SIMPLE_MTX_INVALID_VALUE);
124
125   if (__builtin_expect(c != 1, 0)) {
126      mtx->val = 0;
127      futex_wake(&mtx->val, 1);
128   }
129}
130
131static inline void
132simple_mtx_assert_locked(simple_mtx_t *mtx)
133{
134   assert(mtx->val);
135}
136
137#else
138
139typedef mtx_t simple_mtx_t;
140
141#define _SIMPLE_MTX_INITIALIZER_NP _MTX_INITIALIZER_NP
142
143static inline void
144simple_mtx_init(simple_mtx_t *mtx, int type)
145{
146   mtx_init(mtx, type);
147}
148
149static inline void
150simple_mtx_destroy(simple_mtx_t *mtx)
151{
152   mtx_destroy(mtx);
153}
154
155static inline void
156simple_mtx_lock(simple_mtx_t *mtx)
157{
158   mtx_lock(mtx);
159}
160
161static inline void
162simple_mtx_unlock(simple_mtx_t *mtx)
163{
164   mtx_unlock(mtx);
165}
166
167static inline void
168simple_mtx_assert_locked(simple_mtx_t *mtx)
169{
170#ifdef DEBUG
171   /* NOTE: this would not work for recursive mutexes, but
172    * mtx_t doesn't support those
173    */
174   int ret = mtx_trylock(mtx);
175   assert(ret == thrd_busy);
176   if (ret == thrd_success)
177      mtx_unlock(mtx);
178#else
179   (void)mtx;
180#endif
181}
182
183#endif
184
185#endif
186