lockstat.c revision 1.5.2.2
1 /* $NetBSD: lockstat.c,v 1.5.2.2 2006/12/30 20:47:50 yamt Exp $ */ 2 3 /*- 4 * Copyright (c) 2006 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed by the NetBSD 21 * Foundation, Inc. and its contributors. 22 * 4. Neither the name of The NetBSD Foundation nor the names of its 23 * contributors may be used to endorse or promote products derived 24 * from this software without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 * POSSIBILITY OF SUCH DAMAGE. 37 */ 38 39 /* 40 * Lock statistics driver, providing kernel support for the lockstat(8) 41 * command. 42 * 43 * XXX Timings for contention on sleep locks are currently incorrect. 44 */ 45 46 #include <sys/cdefs.h> 47 __KERNEL_RCSID(0, "$NetBSD: lockstat.c,v 1.5.2.2 2006/12/30 20:47:50 yamt Exp $"); 48 49 #include <sys/types.h> 50 #include <sys/param.h> 51 #include <sys/lock.h> 52 #include <sys/proc.h> 53 #include <sys/resourcevar.h> 54 #include <sys/systm.h> 55 #include <sys/kernel.h> 56 #include <sys/malloc.h> 57 #include <sys/conf.h> 58 #include <sys/syslog.h> 59 60 #include <dev/lockstat.h> 61 62 #ifndef __HAVE_CPU_COUNTER 63 #error CPU counters not available 64 #endif 65 66 #if LONG_BIT == 64 67 #define LOCKSTAT_HASH_SHIFT 3 68 #elif LONG_BIT == 32 69 #define LOCKSTAT_HASH_SHIFT 2 70 #endif 71 72 #define LOCKSTAT_MINBUFS 100 73 #define LOCKSTAT_DEFBUFS 1000 74 #define LOCKSTAT_MAXBUFS 10000 75 76 #define LOCKSTAT_HASH_SIZE 64 77 #define LOCKSTAT_HASH_MASK (LOCKSTAT_HASH_SIZE - 1) 78 #define LOCKSTAT_HASH(key) \ 79 ((key >> LOCKSTAT_HASH_SHIFT) & LOCKSTAT_HASH_MASK) 80 81 typedef struct lscpu { 82 SLIST_HEAD(, lsbuf) lc_free; 83 u_int lc_overflow; 84 LIST_HEAD(lslist, lsbuf) lc_hash[LOCKSTAT_HASH_SIZE]; 85 } lscpu_t; 86 87 typedef struct lslist lslist_t; 88 89 void lockstatattach(int); 90 void lockstat_start(lsenable_t *); 91 int lockstat_alloc(lsenable_t *); 92 void lockstat_init_tables(lsenable_t *); 93 int lockstat_stop(lsdisable_t *); 94 void lockstat_free(void); 95 96 dev_type_open(lockstat_open); 97 dev_type_close(lockstat_close); 98 dev_type_read(lockstat_read); 99 dev_type_ioctl(lockstat_ioctl); 100 101 /* Protected against write by lockstat_lock(). Used by lockstat_event(). */ 102 volatile u_int lockstat_enabled; 103 uintptr_t lockstat_csstart; 104 uintptr_t lockstat_csend; 105 uintptr_t lockstat_csmask; 106 uintptr_t lockstat_lockstart; 107 uintptr_t lockstat_lockend; 108 109 /* Protected by lockstat_lock(). */ 110 struct simplelock lockstat_slock; 111 lsbuf_t *lockstat_baseb; 112 size_t lockstat_sizeb; 113 int lockstat_busy; 114 int lockstat_devopen; 115 struct timespec lockstat_stime; 116 117 const struct cdevsw lockstat_cdevsw = { 118 lockstat_open, lockstat_close, lockstat_read, nowrite, lockstat_ioctl, 119 nostop, notty, nopoll, nommap, nokqfilter, 0 120 }; 121 122 MALLOC_DEFINE(M_LOCKSTAT, "lockstat", "lockstat event buffers"); 123 124 /* 125 * Called when the pseudo-driver is attached. 126 */ 127 void 128 lockstatattach(int nunits) 129 { 130 131 (void)nunits; 132 133 __cpu_simple_lock_init(&lockstat_slock.lock_data); 134 } 135 136 /* 137 * Grab the global lock. If busy is set, we want to block out operations on 138 * the control device. 139 */ 140 static inline int 141 lockstat_lock(int busy) 142 { 143 144 if (!__cpu_simple_lock_try(&lockstat_slock.lock_data)) 145 return (EBUSY); 146 if (busy) { 147 if (lockstat_busy) { 148 __cpu_simple_unlock(&lockstat_slock.lock_data); 149 return (EBUSY); 150 } 151 lockstat_busy = 1; 152 } 153 KASSERT(lockstat_busy); 154 155 return 0; 156 } 157 158 /* 159 * Release the global lock. If unbusy is set, we want to allow new 160 * operations on the control device. 161 */ 162 static inline void 163 lockstat_unlock(int unbusy) 164 { 165 166 KASSERT(lockstat_busy); 167 if (unbusy) 168 lockstat_busy = 0; 169 __cpu_simple_unlock(&lockstat_slock.lock_data); 170 } 171 172 /* 173 * Prepare the per-CPU tables for use, or clear down tables when tracing is 174 * stopped. 175 */ 176 void 177 lockstat_init_tables(lsenable_t *le) 178 { 179 int i, ncpu, per, slop, cpuno; 180 CPU_INFO_ITERATOR cii; 181 struct cpu_info *ci; 182 lscpu_t *lc; 183 lsbuf_t *lb; 184 185 KASSERT(!lockstat_enabled); 186 187 ncpu = 0; 188 for (CPU_INFO_FOREACH(cii, ci)) { 189 if (ci->ci_lockstat != NULL) { 190 free(ci->ci_lockstat, M_LOCKSTAT); 191 ci->ci_lockstat = NULL; 192 } 193 ncpu++; 194 } 195 196 if (le == NULL) 197 return; 198 199 lb = lockstat_baseb; 200 per = le->le_nbufs / ncpu; 201 slop = le->le_nbufs - (per * ncpu); 202 cpuno = 0; 203 for (CPU_INFO_FOREACH(cii, ci)) { 204 lc = malloc(sizeof(*lc), M_LOCKSTAT, M_WAITOK); 205 lc->lc_overflow = 0; 206 ci->ci_lockstat = lc; 207 208 SLIST_INIT(&lc->lc_free); 209 for (i = 0; i < LOCKSTAT_HASH_SIZE; i++) 210 LIST_INIT(&lc->lc_hash[i]); 211 212 for (i = per; i != 0; i--, lb++) { 213 lb->lb_cpu = (uint16_t)cpuno; 214 SLIST_INSERT_HEAD(&lc->lc_free, lb, lb_chain.slist); 215 } 216 if (--slop > 0) { 217 lb->lb_cpu = (uint16_t)cpuno; 218 SLIST_INSERT_HEAD(&lc->lc_free, lb, lb_chain.slist); 219 lb++; 220 } 221 cpuno++; 222 } 223 } 224 225 /* 226 * Start collecting lock statistics. 227 */ 228 void 229 lockstat_start(lsenable_t *le) 230 { 231 232 KASSERT(!lockstat_enabled); 233 234 lockstat_init_tables(le); 235 236 if ((le->le_flags & LE_CALLSITE) != 0) 237 lockstat_csmask = (uintptr_t)-1LL; 238 else 239 lockstat_csmask = 0; 240 241 lockstat_csstart = le->le_csstart; 242 lockstat_csend = le->le_csend; 243 lockstat_lockstart = le->le_lockstart; 244 lockstat_lockend = le->le_lockend; 245 #ifdef notyet 246 mb_memory(); 247 #else 248 lockstat_unlock(0); 249 (void)lockstat_lock(0); 250 #endif 251 252 getnanotime(&lockstat_stime); 253 lockstat_enabled = le->le_mask; 254 #ifdef notyet 255 mb_write(); 256 #else 257 lockstat_unlock(0); 258 (void)lockstat_lock(0); 259 #endif 260 } 261 262 /* 263 * Stop collecting lock statistics. 264 */ 265 int 266 lockstat_stop(lsdisable_t *ld) 267 { 268 CPU_INFO_ITERATOR cii; 269 struct cpu_info *ci; 270 u_int cpuno, overflow; 271 struct timespec ts; 272 int error; 273 274 KASSERT(lockstat_enabled); 275 276 /* 277 * Set enabled false, force a write barrier, and wait for other CPUs 278 * to exit lockstat_event(). 279 */ 280 lockstat_enabled = 0; 281 lockstat_unlock(0); 282 getnanotime(&ts); 283 tsleep(&lockstat_stop, PPAUSE, "lockstat", mstohz(10)); 284 (void)lockstat_lock(0); 285 286 /* 287 * Did we run out of buffers while tracing? 288 */ 289 overflow = 0; 290 for (CPU_INFO_FOREACH(cii, ci)) 291 overflow += ((lscpu_t *)ci->ci_lockstat)->lc_overflow; 292 293 if (overflow != 0) { 294 error = EOVERFLOW; 295 log(LOG_NOTICE, "lockstat: %d buffer allocations failed\n", 296 overflow); 297 } else 298 error = 0; 299 300 lockstat_init_tables(NULL); 301 302 if (ld == NULL) 303 return (error); 304 305 /* 306 * Fill out the disable struct for the caller. 307 */ 308 timespecsub(&ts, &lockstat_stime, &ld->ld_time); 309 ld->ld_size = lockstat_sizeb; 310 311 cpuno = 0; 312 for (CPU_INFO_FOREACH(cii, ci)) { 313 if (cpuno > sizeof(ld->ld_freq) / sizeof(ld->ld_freq[0])) { 314 log(LOG_WARNING, "lockstat: too many CPUs\n"); 315 break; 316 } 317 ld->ld_freq[cpuno++] = cpu_frequency(ci); 318 } 319 320 return (error); 321 } 322 323 /* 324 * Allocate buffers for lockstat_start(). 325 */ 326 int 327 lockstat_alloc(lsenable_t *le) 328 { 329 lsbuf_t *lb; 330 size_t sz; 331 332 KASSERT(!lockstat_enabled); 333 lockstat_free(); 334 335 sz = sizeof(*lb) * le->le_nbufs; 336 337 lockstat_unlock(0); 338 lb = malloc(sz, M_LOCKSTAT, M_WAITOK | M_ZERO); 339 (void)lockstat_lock(0); 340 341 if (lb == NULL) 342 return (ENOMEM); 343 344 KASSERT(!lockstat_enabled); 345 KASSERT(lockstat_baseb == NULL); 346 lockstat_sizeb = sz; 347 lockstat_baseb = lb; 348 349 return (0); 350 } 351 352 /* 353 * Free allocated buffers after tracing has stopped. 354 */ 355 void 356 lockstat_free(void) 357 { 358 359 KASSERT(!lockstat_enabled); 360 361 if (lockstat_baseb != NULL) { 362 free(lockstat_baseb, M_LOCKSTAT); 363 lockstat_baseb = NULL; 364 } 365 } 366 367 /* 368 * Main entry point from lock primatives. 369 */ 370 void 371 lockstat_event(uintptr_t lock, uintptr_t callsite, u_int flags, u_int count, 372 uint64_t time) 373 { 374 lslist_t *ll; 375 lscpu_t *lc; 376 lsbuf_t *lb; 377 u_int event; 378 int s; 379 380 if ((flags & lockstat_enabled) != flags || count == 0) 381 return; 382 if (lock < lockstat_lockstart || lock > lockstat_lockend) 383 return; 384 if (callsite < lockstat_csstart || callsite > lockstat_csend) 385 return; 386 387 callsite &= lockstat_csmask; 388 389 /* 390 * Find the table for this lock+callsite pair, and try to locate a 391 * buffer with the same key. 392 */ 393 lc = curcpu()->ci_lockstat; 394 ll = &lc->lc_hash[LOCKSTAT_HASH(lock ^ callsite)]; 395 event = (flags & LB_EVENT_MASK) - 1; 396 s = spllock(); 397 398 LIST_FOREACH(lb, ll, lb_chain.list) { 399 if (lb->lb_lock == lock && lb->lb_callsite == callsite) 400 break; 401 } 402 403 if (lb != NULL) { 404 /* 405 * We found a record. Move it to the front of the list, as 406 * we're likely to hit it again soon. 407 */ 408 if (lb != LIST_FIRST(ll)) { 409 LIST_REMOVE(lb, lb_chain.list); 410 LIST_INSERT_HEAD(ll, lb, lb_chain.list); 411 } 412 lb->lb_counts[event] += count; 413 lb->lb_times[event] += time; 414 } else if ((lb = SLIST_FIRST(&lc->lc_free)) != NULL) { 415 /* 416 * Pinch a new buffer and fill it out. 417 */ 418 SLIST_REMOVE_HEAD(&lc->lc_free, lb_chain.slist); 419 LIST_INSERT_HEAD(ll, lb, lb_chain.list); 420 lb->lb_flags = (uint16_t)flags; 421 lb->lb_lock = lock; 422 lb->lb_callsite = callsite; 423 lb->lb_counts[event] = count; 424 lb->lb_times[event] = time; 425 } else { 426 /* 427 * We didn't find a buffer and there were none free. 428 * lockstat_stop() will notice later on and report the 429 * error. 430 */ 431 lc->lc_overflow++; 432 } 433 434 splx(s); 435 } 436 437 /* 438 * Accept an open() on /dev/lockstat. 439 */ 440 int 441 lockstat_open(dev_t dev, int flag, int mode, 442 struct lwp *l) 443 { 444 int error; 445 446 if ((error = lockstat_lock(1)) != 0) 447 return error; 448 449 if (lockstat_devopen) 450 error = EBUSY; 451 else { 452 lockstat_devopen = 1; 453 error = 0; 454 } 455 456 lockstat_unlock(1); 457 458 return error; 459 } 460 461 /* 462 * Accept the last close() on /dev/lockstat. 463 */ 464 int 465 lockstat_close(dev_t dev, int flag, int mode, 466 struct lwp *l) 467 { 468 int error; 469 470 if ((error = lockstat_lock(1)) == 0) { 471 if (lockstat_enabled) 472 (void)lockstat_stop(NULL); 473 lockstat_free(); 474 lockstat_devopen = 0; 475 lockstat_unlock(1); 476 } 477 478 return error; 479 } 480 481 /* 482 * Handle control operations. 483 */ 484 int 485 lockstat_ioctl(dev_t dev, u_long cmd, caddr_t data, 486 int flag, struct lwp *l) 487 { 488 lsenable_t *le; 489 int error; 490 491 if ((error = lockstat_lock(1)) != 0) 492 return error; 493 494 switch (cmd) { 495 case IOC_LOCKSTAT_GVERSION: 496 *(int *)data = LS_VERSION; 497 error = 0; 498 break; 499 500 case IOC_LOCKSTAT_ENABLE: 501 le = (lsenable_t *)data; 502 503 if (!cpu_hascounter()) { 504 error = ENODEV; 505 break; 506 } 507 if (lockstat_enabled) { 508 error = EBUSY; 509 break; 510 } 511 512 /* 513 * Sanitize the arguments passed in and set up filtering. 514 */ 515 if (le->le_nbufs == 0) 516 le->le_nbufs = LOCKSTAT_DEFBUFS; 517 else if (le->le_nbufs > LOCKSTAT_MAXBUFS || 518 le->le_nbufs < LOCKSTAT_MINBUFS) { 519 error = EINVAL; 520 break; 521 } 522 if ((le->le_flags & LE_ONE_CALLSITE) == 0) { 523 le->le_csstart = 0; 524 le->le_csend = le->le_csstart - 1; 525 } 526 if ((le->le_flags & LE_ONE_LOCK) == 0) { 527 le->le_lockstart = 0; 528 le->le_lockend = le->le_lockstart - 1; 529 } 530 if ((le->le_mask & LB_EVENT_MASK) == 0) 531 return (EINVAL); 532 if ((le->le_mask & LB_LOCK_MASK) == 0) 533 return (EINVAL); 534 535 /* 536 * Start tracing. 537 */ 538 if ((error = lockstat_alloc(le)) == 0) 539 lockstat_start(le); 540 break; 541 542 case IOC_LOCKSTAT_DISABLE: 543 if (!lockstat_enabled) 544 error = EINVAL; 545 else 546 error = lockstat_stop((lsdisable_t *)data); 547 break; 548 549 default: 550 error = ENOTTY; 551 break; 552 } 553 554 lockstat_unlock(1); 555 return error; 556 } 557 558 /* 559 * Copy buffers out to user-space. 560 */ 561 int 562 lockstat_read(dev_t dev, struct uio *uio, int flag) 563 { 564 int error; 565 566 if ((error = lockstat_lock(1)) != 0) 567 return (error); 568 569 if (lockstat_enabled) { 570 lockstat_unlock(1); 571 return (EBUSY); 572 } 573 574 lockstat_unlock(0); 575 error = uiomove(lockstat_baseb, lockstat_sizeb, uio); 576 lockstat_lock(0); 577 578 lockstat_unlock(1); 579 580 return (error); 581 } 582
Indexes created Thu Oct 02 01:09:59 GMT 2025