1 /* $NetBSD: locks_up.c,v 1.14 2023/11/02 10:31:55 martin Exp $ */ 2 3 /* 4 * Copyright (c) 2010 Antti Kantee. All Rights Reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 16 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 17 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 21 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 /* 29 * Virtual uniprocessor rump kernel version of locks. Since the entire 30 * kernel is running on only one CPU in the system, there is no need 31 * to perform slow cache-coherent MP locking operations. This speeds 32 * up things quite dramatically and is a good example of that two 33 * disjoint kernels running simultaneously in an MP system can be 34 * massively faster than one with fine-grained locking. 35 */ 36 37 #include <sys/cdefs.h> 38 __KERNEL_RCSID(0, "$NetBSD: locks_up.c,v 1.14 2023/11/02 10:31:55 martin Exp $"); 39 40 #include <sys/param.h> 41 #include <sys/kernel.h> 42 #include <sys/kmem.h> 43 #include <sys/mutex.h> 44 #include <sys/rwlock.h> 45 46 #include <rump-sys/kern.h> 47 48 #include <rump/rumpuser.h> 49 50 struct upmtx { 51 struct lwp *upm_owner; 52 int upm_wanted; 53 struct rumpuser_cv *upm_rucv; 54 }; 55 #define UPMTX(mtx) struct upmtx *upm = *(struct upmtx **)mtx 56 57 static inline void 58 checkncpu(void) 59 { 60 61 if (__predict_false(ncpu != 1)) 62 panic("UP lock implementation requires RUMP_NCPU == 1"); 63 } 64 65 void 66 mutex_init(kmutex_t *mtx, kmutex_type_t type, int ipl) 67 { 68 struct upmtx *upm; 69 70 CTASSERT(sizeof(kmutex_t) >= sizeof(void *)); 71 checkncpu(); 72 73 /* 74 * In uniprocessor locking we don't need to differentiate 75 * between spin mutexes and adaptive ones. We could 76 * replace mutex_enter() with a NOP for spin mutexes, but 77 * not bothering with that for now. 78 */ 79 80 /* 81 * XXX: pool_cache would be nice, but not easily possible, 82 * as pool cache init wants to call mutex_init() ... 83 */ 84 upm = rump_hypermalloc(sizeof(*upm), 0, true, "mutex_init"); 85 memset(upm, 0, sizeof(*upm)); 86 rumpuser_cv_init(&upm->upm_rucv); 87 memcpy(mtx, &upm, sizeof(void *)); 88 } 89 90 void 91 mutex_destroy(kmutex_t *mtx) 92 { 93 UPMTX(mtx); 94 95 KASSERT(upm->upm_owner == NULL); 96 KASSERT(upm->upm_wanted == 0); 97 rumpuser_cv_destroy(upm->upm_rucv); 98 rump_hyperfree(upm, sizeof(*upm)); 99 } 100 101 void 102 mutex_enter(kmutex_t *mtx) 103 { 104 UPMTX(mtx); 105 106 /* fastpath? */ 107 if (mutex_tryenter(mtx)) 108 return; 109 110 /* 111 * No? bummer, do it the slow and painful way then. 112 */ 113 upm->upm_wanted++; 114 while (!mutex_tryenter(mtx)) { 115 rump_schedlock_cv_wait(upm->upm_rucv); 116 } 117 upm->upm_wanted--; 118 119 KASSERT(upm->upm_wanted >= 0); 120 } 121 122 void 123 mutex_spin_enter(kmutex_t *mtx) 124 { 125 126 mutex_enter(mtx); 127 } 128 129 int 130 mutex_tryenter(kmutex_t *mtx) 131 { 132 UPMTX(mtx); 133 134 if (upm->upm_owner) 135 return 0; 136 137 upm->upm_owner = curlwp; 138 return 1; 139 } 140 141 void 142 mutex_exit(kmutex_t *mtx) 143 { 144 UPMTX(mtx); 145 146 if (upm->upm_wanted) { 147 rumpuser_cv_signal(upm->upm_rucv); /* CPU is our interlock */ 148 } 149 upm->upm_owner = NULL; 150 } 151 152 void 153 mutex_spin_exit(kmutex_t *mtx) 154 { 155 156 mutex_exit(mtx); 157 } 158 159 int 160 mutex_owned(kmutex_t *mtx) 161 { 162 UPMTX(mtx); 163 164 return upm->upm_owner == curlwp; 165 } 166 167 struct uprw { 168 struct lwp *uprw_owner; 169 int uprw_readers; 170 uint16_t uprw_rwant; 171 uint16_t uprw_wwant; 172 struct rumpuser_cv *uprw_rucv_reader; 173 struct rumpuser_cv *uprw_rucv_writer; 174 }; 175 176 #define UPRW(rw) struct uprw *uprw = *(struct uprw **)rw 177 178 /* reader/writer locks */ 179 180 void 181 rw_init(krwlock_t *rw) 182 { 183 struct uprw *uprw; 184 185 CTASSERT(sizeof(krwlock_t) >= sizeof(void *)); 186 checkncpu(); 187 188 uprw = rump_hypermalloc(sizeof(*uprw), 0, true, "rwinit"); 189 memset(uprw, 0, sizeof(*uprw)); 190 rumpuser_cv_init(&uprw->uprw_rucv_reader); 191 rumpuser_cv_init(&uprw->uprw_rucv_writer); 192 memcpy(rw, &uprw, sizeof(void *)); 193 } 194 195 void 196 rw_destroy(krwlock_t *rw) 197 { 198 UPRW(rw); 199 200 rumpuser_cv_destroy(uprw->uprw_rucv_reader); 201 rumpuser_cv_destroy(uprw->uprw_rucv_writer); 202 rump_hyperfree(uprw, sizeof(*uprw)); 203 } 204 205 /* take rwlock. prefer writers over readers (see rw_tryenter and rw_exit) */ 206 void 207 rw_enter(krwlock_t *rw, const krw_t op) 208 { 209 UPRW(rw); 210 struct rumpuser_cv *rucv; 211 uint16_t *wp; 212 213 if (rw_tryenter(rw, op)) 214 return; 215 216 /* lagpath */ 217 if (op == RW_READER) { 218 rucv = uprw->uprw_rucv_reader; 219 wp = &uprw->uprw_rwant; 220 } else { 221 rucv = uprw->uprw_rucv_writer; 222 wp = &uprw->uprw_wwant; 223 } 224 225 (*wp)++; 226 while (!rw_tryenter(rw, op)) { 227 rump_schedlock_cv_wait(rucv); 228 } 229 (*wp)--; 230 } 231 232 int 233 rw_tryenter(krwlock_t *rw, const krw_t op) 234 { 235 UPRW(rw); 236 237 switch (op) { 238 case RW_READER: 239 if (uprw->uprw_owner == NULL && uprw->uprw_wwant == 0) { 240 uprw->uprw_readers++; 241 return 1; 242 } 243 break; 244 case RW_WRITER: 245 if (uprw->uprw_owner == NULL && uprw->uprw_readers == 0) { 246 uprw->uprw_owner = curlwp; 247 return 1; 248 } 249 break; 250 } 251 252 return 0; 253 } 254 255 void 256 rw_exit(krwlock_t *rw) 257 { 258 UPRW(rw); 259 260 if (uprw->uprw_readers > 0) { 261 uprw->uprw_readers--; 262 } else { 263 KASSERT(uprw->uprw_owner == curlwp); 264 uprw->uprw_owner = NULL; 265 } 266 267 if (uprw->uprw_wwant) { 268 rumpuser_cv_signal(uprw->uprw_rucv_writer); 269 } else if (uprw->uprw_rwant) { 270 rumpuser_cv_signal(uprw->uprw_rucv_reader); 271 } 272 } 273 274 int 275 rw_tryupgrade(krwlock_t *rw) 276 { 277 UPRW(rw); 278 279 if (uprw->uprw_readers == 1 && uprw->uprw_owner == NULL) { 280 uprw->uprw_readers = 0; 281 uprw->uprw_owner = curlwp; 282 return 1; 283 } else { 284 return 0; 285 } 286 } 287 288 int 289 rw_write_held(krwlock_t *rw) 290 { 291 UPRW(rw); 292 293 return uprw->uprw_owner == curlwp; 294 } 295 296 int 297 rw_read_held(krwlock_t *rw) 298 { 299 UPRW(rw); 300 301 return uprw->uprw_readers > 0; 302 } 303 304 int 305 rw_lock_held(krwlock_t *rw) 306 { 307 UPRW(rw); 308 309 return uprw->uprw_owner || uprw->uprw_readers; 310 } 311 312 krw_t 313 rw_lock_op(krwlock_t *rw) 314 { 315 316 return rw_write_held(rw) ? RW_WRITER : RW_READER; 317 } 318 319 /* 320 * Condvars are almost the same as in the MP case except that we 321 * use the scheduler mutex as the pthread interlock instead of the 322 * mutex associated with the condvar. 323 */ 324 325 #define RUMPCV(cv) (*(struct rumpuser_cv **)(cv)) 326 327 void 328 cv_init(kcondvar_t *cv, const char *msg) 329 { 330 331 CTASSERT(sizeof(kcondvar_t) >= sizeof(void *)); 332 checkncpu(); 333 334 rumpuser_cv_init((struct rumpuser_cv **)cv); 335 } 336 337 void 338 cv_destroy(kcondvar_t *cv) 339 { 340 341 rumpuser_cv_destroy(RUMPCV(cv)); 342 } 343 344 void 345 cv_wait(kcondvar_t *cv, kmutex_t *mtx) 346 { 347 #ifdef DIAGNOSTIC 348 UPMTX(mtx); 349 KASSERT(upm->upm_owner == curlwp); 350 351 if (rump_threads == 0) 352 panic("cv_wait without threads"); 353 #endif 354 355 /* 356 * NOTE: we must atomically release the *CPU* here, i.e. 357 * nothing between mutex_exit and entering rumpuser condwait 358 * may preempt us from the virtual CPU. 359 */ 360 mutex_exit(mtx); 361 rump_schedlock_cv_wait(RUMPCV(cv)); 362 mutex_enter(mtx); 363 } 364 365 int 366 cv_wait_sig(kcondvar_t *cv, kmutex_t *mtx) 367 { 368 369 cv_wait(cv, mtx); 370 return 0; 371 } 372 373 int 374 cv_timedwait(kcondvar_t *cv, kmutex_t *mtx, int ticks) 375 { 376 struct timespec ts; 377 378 #ifdef DIAGNOSTIC 379 UPMTX(mtx); 380 KASSERT(upm->upm_owner == curlwp); 381 #endif 382 383 ts.tv_sec = ticks / hz; 384 ts.tv_nsec = (ticks % hz) * (1000000000/hz); 385 386 if (ticks == 0) { 387 cv_wait(cv, mtx); 388 return 0; 389 } else { 390 int rv; 391 mutex_exit(mtx); 392 rv = rump_schedlock_cv_timedwait(RUMPCV(cv), &ts); 393 mutex_enter(mtx); 394 if (rv) 395 return EWOULDBLOCK; 396 else 397 return 0; 398 } 399 } 400 401 int 402 cv_timedwait_sig(kcondvar_t *cv, kmutex_t *mtx, int ticks) 403 { 404 405 return cv_timedwait(cv, mtx, ticks); 406 } 407 408 void 409 cv_signal(kcondvar_t *cv) 410 { 411 412 /* CPU == interlock */ 413 rumpuser_cv_signal(RUMPCV(cv)); 414 } 415 416 void 417 cv_broadcast(kcondvar_t *cv) 418 { 419 420 /* CPU == interlock */ 421 rumpuser_cv_broadcast(RUMPCV(cv)); 422 } 423 424 bool 425 cv_has_waiters(kcondvar_t *cv) 426 { 427 int n; 428 429 rumpuser_cv_has_waiters(RUMPCV(cv), &n); 430 431 return n > 0; 432 } 433 434 /* this is not much of an attempt, but ... */ 435 bool 436 cv_is_valid(kcondvar_t *cv) 437 { 438 439 return RUMPCV(cv) != NULL; 440 } 441
Indexes created Sat Sep 20 22:09:52 GMT 2025