subr_pcu.c revision 1.3 1 1.3 matt /* $NetBSD: subr_pcu.c,v 1.3 2011/02/19 20:19:54 matt Exp $ */
2 1.1 rmind
3 1.1 rmind /*-
4 1.1 rmind * Copyright (c) 2011 The NetBSD Foundation, Inc.
5 1.1 rmind * All rights reserved.
6 1.1 rmind *
7 1.1 rmind * This code is derived from software contributed to The NetBSD Foundation
8 1.1 rmind * by Mindaugas Rasiukevicius.
9 1.1 rmind *
10 1.1 rmind * Redistribution and use in source and binary forms, with or without
11 1.1 rmind * modification, are permitted provided that the following conditions
12 1.1 rmind * are met:
13 1.1 rmind * 1. Redistributions of source code must retain the above copyright
14 1.1 rmind * notice, this list of conditions and the following disclaimer.
15 1.1 rmind * 2. Redistributions in binary form must reproduce the above copyright
16 1.1 rmind * notice, this list of conditions and the following disclaimer in the
17 1.1 rmind * documentation and/or other materials provided with the distribution.
18 1.1 rmind *
19 1.1 rmind * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 1.1 rmind * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 1.1 rmind * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 1.1 rmind * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 1.1 rmind * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 1.1 rmind * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 1.1 rmind * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 1.1 rmind * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 1.1 rmind * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 1.1 rmind * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 1.1 rmind * POSSIBILITY OF SUCH DAMAGE.
30 1.1 rmind */
31 1.1 rmind
32 1.1 rmind /*
33 1.1 rmind * Per CPU Unit (PCU) - is an interface to manage synchronization of any
34 1.1 rmind * per CPU context (unit) tied with LWP context. Typical use: FPU state.
35 1.1 rmind *
36 1.1 rmind * Concurrency notes:
37 1.1 rmind *
38 1.1 rmind * PCU state may be loaded only by the current LWP, that is, curlwp.
39 1.1 rmind * Therefore, only LWP itself can set a CPU for lwp_t::l_pcu_cpu[id].
40 1.1 rmind *
41 1.1 rmind * Request for a PCU release can be from owner LWP (whether PCU state
42 1.1 rmind * is on current CPU or remote CPU) or any other LWP running on that
43 1.1 rmind * CPU (in such case, owner LWP is on a remote CPU or sleeping).
44 1.1 rmind *
45 1.1 rmind * In any case, PCU state can only be changed from the running CPU.
46 1.1 rmind * If said PCU state is on the remote CPU, a cross-call will be sent
47 1.1 rmind * by the owner LWP. Therefore struct cpu_info::ci_pcu_curlwp[id]
48 1.1 rmind * may only be changed by current CPU, and lwp_t::l_pcu_cpu[id] may
49 1.1 rmind * only be unset by the CPU which has PCU state loaded.
50 1.1 rmind *
51 1.1 rmind * There is a race condition: LWP may have a PCU state on a remote CPU,
52 1.1 rmind * which it requests to be released via cross-call. At the same time,
53 1.1 rmind * other LWP on remote CPU might release existing PCU state and load
54 1.1 rmind * its own one. Cross-call may arrive after this and release different
55 1.1 rmind * PCU state than intended. In such case, such LWP would re-load its
56 1.1 rmind * PCU state again.
57 1.1 rmind */
58 1.1 rmind
59 1.1 rmind #include <sys/cdefs.h>
60 1.3 matt __KERNEL_RCSID(0, "$NetBSD: subr_pcu.c,v 1.3 2011/02/19 20:19:54 matt Exp $");
61 1.1 rmind
62 1.1 rmind #include <sys/param.h>
63 1.1 rmind #include <sys/cpu.h>
64 1.1 rmind #include <sys/lwp.h>
65 1.1 rmind #include <sys/pcu.h>
66 1.1 rmind #include <sys/xcall.h>
67 1.1 rmind
68 1.3 matt #if PCU_UNIT_COUNT > 0
69 1.3 matt
70 1.1 rmind #define PCU_SAVE 0x01 /* Save PCU state to the LWP. */
71 1.1 rmind #define PCU_RELEASE 0x02 /* Release PCU state on the CPU. */
72 1.1 rmind
73 1.1 rmind #if 0
74 1.1 rmind /*
75 1.1 rmind * pcu_init_lwp: initialize PCU structures for LWP.
76 1.1 rmind */
77 1.1 rmind void
78 1.1 rmind pcu_init_lwp(lwp_t *l)
79 1.1 rmind {
80 1.1 rmind
81 1.1 rmind memset(l->l_pcu_cpu, 0, sizeof(uint32_t) * PCU_UNIT_COUNT);
82 1.1 rmind l->l_pcu_used = 0;
83 1.1 rmind }
84 1.1 rmind #endif
85 1.1 rmind
86 1.1 rmind /*
87 1.1 rmind * pcu_cpu_op: save/release PCU state on the current CPU.
88 1.1 rmind *
89 1.1 rmind * => Must be called at IPL_SOFTCLOCK or from the soft-interrupt.
90 1.1 rmind */
91 1.1 rmind static void
92 1.1 rmind pcu_cpu_op(const pcu_ops_t *pcu, const int flags)
93 1.1 rmind {
94 1.1 rmind const u_int id = pcu->pcu_id;
95 1.1 rmind struct cpu_info *ci = curcpu();
96 1.1 rmind lwp_t *l = ci->ci_pcu_curlwp[id];
97 1.1 rmind
98 1.1 rmind /* If no state - nothing to do. */
99 1.1 rmind if (l == NULL) {
100 1.1 rmind return;
101 1.1 rmind }
102 1.1 rmind if (flags & PCU_SAVE) {
103 1.2 matt pcu->pcu_state_save(l, (flags & PCU_RELEASE) != 0);
104 1.1 rmind }
105 1.1 rmind if (flags & PCU_RELEASE) {
106 1.1 rmind ci->ci_pcu_curlwp[id] = NULL;
107 1.1 rmind l->l_pcu_cpu[id] = NULL;
108 1.1 rmind }
109 1.1 rmind }
110 1.1 rmind
111 1.1 rmind /*
112 1.1 rmind * pcu_lwp_op: perform PCU state save, release or both operations on LWP.
113 1.1 rmind */
114 1.1 rmind static void
115 1.1 rmind pcu_lwp_op(const pcu_ops_t *pcu, lwp_t *l, int flags)
116 1.1 rmind {
117 1.1 rmind const u_int id = pcu->pcu_id;
118 1.1 rmind struct cpu_info *ci;
119 1.1 rmind uint64_t where;
120 1.1 rmind int s;
121 1.1 rmind
122 1.1 rmind /*
123 1.1 rmind * Caller should have re-checked if there is any state to manage.
124 1.1 rmind * Block the interrupts and inspect again, since cross-call sent
125 1.1 rmind * by remote CPU could have changed the state.
126 1.1 rmind */
127 1.1 rmind s = splsoftclock();
128 1.1 rmind ci = l->l_pcu_cpu[id];
129 1.1 rmind if (ci == curcpu()) {
130 1.1 rmind /*
131 1.1 rmind * State is on the current CPU - just perform the operations.
132 1.1 rmind */
133 1.1 rmind KASSERT(ci->ci_pcu_curlwp[id] == l);
134 1.1 rmind
135 1.1 rmind if (flags & PCU_SAVE) {
136 1.2 matt pcu->pcu_state_save(l, (flags & PCU_RELEASE) != 0);
137 1.1 rmind }
138 1.1 rmind if (flags & PCU_RELEASE) {
139 1.1 rmind ci->ci_pcu_curlwp[id] = NULL;
140 1.1 rmind l->l_pcu_cpu[id] = NULL;
141 1.1 rmind }
142 1.1 rmind splx(s);
143 1.1 rmind return;
144 1.1 rmind }
145 1.1 rmind splx(s);
146 1.1 rmind
147 1.1 rmind if (__predict_false(ci == NULL)) {
148 1.1 rmind /* Cross-call has won the race - no state to manage. */
149 1.1 rmind return;
150 1.1 rmind }
151 1.1 rmind
152 1.1 rmind /*
153 1.1 rmind * State is on the remote CPU - perform the operations there.
154 1.1 rmind * Note: there is a race condition; see description in the top.
155 1.1 rmind */
156 1.1 rmind where = xc_unicast(XC_HIGHPRI, (xcfunc_t)pcu_cpu_op,
157 1.1 rmind __UNCONST(pcu), (void *)(uintptr_t)flags, ci);
158 1.1 rmind xc_wait(where);
159 1.1 rmind
160 1.1 rmind KASSERT((flags & PCU_RELEASE) == 0 || l->l_pcu_cpu[id] == NULL);
161 1.1 rmind }
162 1.1 rmind
163 1.1 rmind /*
164 1.1 rmind * pcu_load: load/initialize the PCU state of current LWP on current CPU.
165 1.1 rmind */
166 1.1 rmind void
167 1.1 rmind pcu_load(const pcu_ops_t *pcu)
168 1.1 rmind {
169 1.1 rmind const u_int id = pcu->pcu_id;
170 1.1 rmind struct cpu_info *ci, *curci;
171 1.1 rmind lwp_t *l = curlwp;
172 1.1 rmind uint64_t where;
173 1.1 rmind int s;
174 1.1 rmind
175 1.1 rmind KASSERT(!cpu_intr_p() && !cpu_softintr_p());
176 1.1 rmind
177 1.1 rmind s = splsoftclock();
178 1.1 rmind curci = curcpu();
179 1.1 rmind ci = l->l_pcu_cpu[id];
180 1.1 rmind
181 1.1 rmind /* Does this CPU already have our PCU state loaded? */
182 1.1 rmind if (ci == curci) {
183 1.1 rmind KASSERT(curci->ci_pcu_curlwp[id] == l);
184 1.1 rmind splx(s);
185 1.1 rmind return;
186 1.1 rmind }
187 1.1 rmind
188 1.1 rmind /* If PCU state of this LWP is on the remote CPU - save it there. */
189 1.1 rmind if (ci) {
190 1.1 rmind splx(s);
191 1.1 rmind /* Note: there is a race; see description in the top. */
192 1.1 rmind where = xc_unicast(XC_HIGHPRI, (xcfunc_t)pcu_cpu_op,
193 1.1 rmind __UNCONST(pcu), (void *)(PCU_SAVE | PCU_RELEASE), ci);
194 1.1 rmind xc_wait(where);
195 1.1 rmind
196 1.1 rmind /* Enter IPL_SOFTCLOCK and re-fetch the current CPU. */
197 1.1 rmind s = splsoftclock();
198 1.1 rmind curci = curcpu();
199 1.1 rmind }
200 1.1 rmind KASSERT(l->l_pcu_cpu[id] == NULL);
201 1.1 rmind
202 1.1 rmind /* Save the PCU state on the current CPU, if there is any. */
203 1.1 rmind pcu_cpu_op(pcu, PCU_SAVE | PCU_RELEASE);
204 1.1 rmind KASSERT(curci->ci_pcu_curlwp[id] == NULL);
205 1.1 rmind
206 1.1 rmind /*
207 1.1 rmind * Finally, load the state for this LWP on this CPU. Indicate to
208 1.1 rmind * load function whether PCU was used before. Note the usage.
209 1.1 rmind */
210 1.1 rmind pcu->pcu_state_load(l, ((1 << id) & l->l_pcu_used) != 0);
211 1.1 rmind curci->ci_pcu_curlwp[id] = l;
212 1.1 rmind l->l_pcu_cpu[id] = curci;
213 1.1 rmind l->l_pcu_used |= (1 << id);
214 1.1 rmind splx(s);
215 1.1 rmind }
216 1.1 rmind
217 1.1 rmind /*
218 1.1 rmind * pcu_discard: discard the PCU state of current LWP.
219 1.1 rmind */
220 1.1 rmind void
221 1.1 rmind pcu_discard(const pcu_ops_t *pcu)
222 1.1 rmind {
223 1.1 rmind const u_int id = pcu->pcu_id;
224 1.1 rmind lwp_t *l = curlwp;
225 1.1 rmind
226 1.1 rmind KASSERT(!cpu_intr_p() && !cpu_softintr_p());
227 1.1 rmind
228 1.1 rmind if (__predict_true(l->l_pcu_cpu[id] == NULL)) {
229 1.1 rmind return;
230 1.1 rmind }
231 1.1 rmind pcu_lwp_op(pcu, l, PCU_RELEASE);
232 1.1 rmind l->l_pcu_used &= ~(1 << id);
233 1.1 rmind }
234 1.1 rmind
235 1.1 rmind /*
236 1.1 rmind * pcu_save_lwp: save PCU state to the given LWP.
237 1.1 rmind */
238 1.1 rmind void
239 1.1 rmind pcu_save_lwp(const pcu_ops_t *pcu, lwp_t *l)
240 1.1 rmind {
241 1.1 rmind const u_int id = pcu->pcu_id;
242 1.1 rmind
243 1.1 rmind KASSERT(!cpu_intr_p() && !cpu_softintr_p());
244 1.1 rmind
245 1.1 rmind if (__predict_true(l->l_pcu_cpu[id] == NULL)) {
246 1.1 rmind return;
247 1.1 rmind }
248 1.1 rmind pcu_lwp_op(pcu, l, PCU_SAVE | PCU_RELEASE);
249 1.1 rmind }
250 1.1 rmind
251 1.1 rmind /*
252 1.1 rmind * pcu_used: return true if PCU was used (pcu_load() case) by the LWP.
253 1.1 rmind */
254 1.1 rmind bool
255 1.1 rmind pcu_used(const pcu_ops_t *pcu, lwp_t *l)
256 1.1 rmind {
257 1.1 rmind const u_int id = pcu->pcu_id;
258 1.1 rmind
259 1.1 rmind return l->l_pcu_used & (1 << id);
260 1.1 rmind }
261 1.3 matt
262 1.3 matt #endif /* PCU_UNIT_COUNT > 0 */
263