lockstat.c revision 1.2.2.3
1 /* $NetBSD: lockstat.c,v 1.2.2.3 2006/12/29 20:27:43 ad Exp $ */ 2 3 /*- 4 * Copyright (c) 2006 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed by the NetBSD 21 * Foundation, Inc. and its contributors. 22 * 4. Neither the name of The NetBSD Foundation nor the names of its 23 * contributors may be used to endorse or promote products derived 24 * from this software without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 * POSSIBILITY OF SUCH DAMAGE. 37 */ 38 39 /* 40 * Lock statistics driver, providing kernel support for the lockstat(8) 41 * command. 42 */ 43 44 #include <sys/cdefs.h> 45 __KERNEL_RCSID(0, "$NetBSD: lockstat.c,v 1.2.2.3 2006/12/29 20:27:43 ad Exp $"); 46 47 #include <sys/types.h> 48 #include <sys/param.h> 49 #include <sys/lock.h> 50 #include <sys/proc.h> 51 #include <sys/resourcevar.h> 52 #include <sys/systm.h> 53 #include <sys/kernel.h> 54 #include <sys/malloc.h> 55 #include <sys/conf.h> 56 #include <sys/syslog.h> 57 58 #include <dev/lockstat.h> 59 60 #ifndef __HAVE_CPU_COUNTER 61 #error CPU counters not available 62 #endif 63 64 #if LONG_BIT == 64 65 #define LOCKSTAT_HASH_SHIFT 3 66 #elif LONG_BIT == 32 67 #define LOCKSTAT_HASH_SHIFT 2 68 #endif 69 70 #define LOCKSTAT_MINBUFS 100 71 #define LOCKSTAT_DEFBUFS 1000 72 #define LOCKSTAT_MAXBUFS 10000 73 74 #define LOCKSTAT_HASH_SIZE 64 75 #define LOCKSTAT_HASH_MASK (LOCKSTAT_HASH_SIZE - 1) 76 #define LOCKSTAT_HASH(key) \ 77 ((key >> LOCKSTAT_HASH_SHIFT) & LOCKSTAT_HASH_MASK) 78 79 typedef struct lscpu { 80 SLIST_HEAD(, lsbuf) lc_free; 81 u_int lc_overflow; 82 LIST_HEAD(lslist, lsbuf) lc_hash[LOCKSTAT_HASH_SIZE]; 83 } lscpu_t; 84 85 typedef struct lslist lslist_t; 86 87 void lockstatattach(int); 88 void lockstat_start(lsenable_t *); 89 int lockstat_alloc(lsenable_t *); 90 void lockstat_init_tables(lsenable_t *); 91 int lockstat_stop(lsdisable_t *); 92 void lockstat_free(void); 93 94 dev_type_open(lockstat_open); 95 dev_type_close(lockstat_close); 96 dev_type_read(lockstat_read); 97 dev_type_ioctl(lockstat_ioctl); 98 99 /* Protected against write by lockstat_lock(). Used by lockstat_event(). */ 100 volatile u_int lockstat_enabled; 101 uintptr_t lockstat_csstart; 102 uintptr_t lockstat_csend; 103 uintptr_t lockstat_csmask; 104 uintptr_t lockstat_lockstart; 105 uintptr_t lockstat_lockend; 106 107 /* Protected by lockstat_lock(). */ 108 struct simplelock lockstat_slock; 109 lsbuf_t *lockstat_baseb; 110 size_t lockstat_sizeb; 111 int lockstat_busy; 112 int lockstat_devopen; 113 struct timespec lockstat_stime; 114 115 const struct cdevsw lockstat_cdevsw = { 116 lockstat_open, lockstat_close, lockstat_read, nowrite, lockstat_ioctl, 117 nostop, notty, nopoll, nommap, nokqfilter, 0 118 }; 119 120 MALLOC_DEFINE(M_LOCKSTAT, "lockstat", "lockstat event buffers"); 121 122 /* 123 * Called when the pseudo-driver is attached. 124 */ 125 void 126 lockstatattach(int nunits) 127 { 128 129 (void)nunits; 130 131 __cpu_simple_lock_init(&lockstat_slock.lock_data); 132 } 133 134 /* 135 * Grab the global lock. If busy is set, we want to block out operations on 136 * the control device. 137 */ 138 static inline int 139 lockstat_lock(int busy) 140 { 141 142 if (!__cpu_simple_lock_try(&lockstat_slock.lock_data)) 143 return (EBUSY); 144 if (busy) { 145 if (lockstat_busy) { 146 __cpu_simple_unlock(&lockstat_slock.lock_data); 147 return (EBUSY); 148 } 149 lockstat_busy = 1; 150 } 151 KASSERT(lockstat_busy); 152 153 return 0; 154 } 155 156 /* 157 * Release the global lock. If unbusy is set, we want to allow new 158 * operations on the control device. 159 */ 160 static inline void 161 lockstat_unlock(int unbusy) 162 { 163 164 KASSERT(lockstat_busy); 165 if (unbusy) 166 lockstat_busy = 0; 167 __cpu_simple_unlock(&lockstat_slock.lock_data); 168 } 169 170 /* 171 * Prepare the per-CPU tables for use, or clear down tables when tracing is 172 * stopped. 173 */ 174 void 175 lockstat_init_tables(lsenable_t *le) 176 { 177 int i, ncpu, per, slop, cpuno; 178 CPU_INFO_ITERATOR cii; 179 struct cpu_info *ci; 180 lscpu_t *lc; 181 lsbuf_t *lb; 182 183 KASSERT(!lockstat_enabled); 184 185 ncpu = 0; 186 for (CPU_INFO_FOREACH(cii, ci)) { 187 if (ci->ci_lockstat != NULL) { 188 free(ci->ci_lockstat, M_LOCKSTAT); 189 ci->ci_lockstat = NULL; 190 } 191 ncpu++; 192 } 193 194 if (le == NULL) 195 return; 196 197 lb = lockstat_baseb; 198 per = le->le_nbufs / ncpu; 199 slop = le->le_nbufs - (per * ncpu); 200 cpuno = 0; 201 for (CPU_INFO_FOREACH(cii, ci)) { 202 lc = malloc(sizeof(*lc), M_LOCKSTAT, M_WAITOK); 203 lc->lc_overflow = 0; 204 ci->ci_lockstat = lc; 205 206 SLIST_INIT(&lc->lc_free); 207 for (i = 0; i < LOCKSTAT_HASH_SIZE; i++) 208 LIST_INIT(&lc->lc_hash[i]); 209 210 for (i = per; i != 0; i--, lb++) { 211 lb->lb_cpu = (uint16_t)cpuno; 212 SLIST_INSERT_HEAD(&lc->lc_free, lb, lb_chain.slist); 213 } 214 if (--slop > 0) { 215 lb->lb_cpu = (uint16_t)cpuno; 216 SLIST_INSERT_HEAD(&lc->lc_free, lb, lb_chain.slist); 217 lb++; 218 } 219 cpuno++; 220 } 221 } 222 223 /* 224 * Start collecting lock statistics. 225 */ 226 void 227 lockstat_start(lsenable_t *le) 228 { 229 230 KASSERT(!lockstat_enabled); 231 232 lockstat_init_tables(le); 233 234 if ((le->le_flags & LE_CALLSITE) != 0) 235 lockstat_csmask = (uintptr_t)-1LL; 236 else 237 lockstat_csmask = 0; 238 239 lockstat_csstart = le->le_csstart; 240 lockstat_csend = le->le_csend; 241 lockstat_lockstart = le->le_lockstart; 242 lockstat_lockend = le->le_lockend; 243 mb_memory(); 244 245 getnanotime(&lockstat_stime); 246 lockstat_enabled = le->le_mask; 247 mb_write(); 248 } 249 250 /* 251 * Stop collecting lock statistics. 252 */ 253 int 254 lockstat_stop(lsdisable_t *ld) 255 { 256 CPU_INFO_ITERATOR cii; 257 struct cpu_info *ci; 258 u_int cpuno, overflow; 259 struct timespec ts; 260 int error; 261 262 KASSERT(lockstat_enabled); 263 264 /* 265 * Set enabled false, force a write barrier, and wait for other CPUs 266 * to exit lockstat_event(). 267 */ 268 lockstat_enabled = 0; 269 lockstat_unlock(0); 270 getnanotime(&ts); 271 tsleep(&lockstat_stop, PPAUSE, "lockstat", mstohz(10)); 272 (void)lockstat_lock(0); 273 274 /* 275 * Did we run out of buffers while tracing? 276 */ 277 overflow = 0; 278 for (CPU_INFO_FOREACH(cii, ci)) 279 overflow += ((lscpu_t *)ci->ci_lockstat)->lc_overflow; 280 281 if (overflow != 0) { 282 error = EOVERFLOW; 283 log(LOG_NOTICE, "lockstat: %d buffer allocations failed\n", 284 overflow); 285 } else 286 error = 0; 287 288 lockstat_init_tables(NULL); 289 290 if (ld == NULL) 291 return (error); 292 293 /* 294 * Fill out the disable struct for the caller. 295 */ 296 timespecsub(&ts, &lockstat_stime, &ld->ld_time); 297 ld->ld_size = lockstat_sizeb; 298 299 cpuno = 0; 300 for (CPU_INFO_FOREACH(cii, ci)) { 301 if (cpuno > sizeof(ld->ld_freq) / sizeof(ld->ld_freq[0])) { 302 log(LOG_WARNING, "lockstat: too many CPUs\n"); 303 break; 304 } 305 ld->ld_freq[cpuno++] = cpu_frequency(ci); 306 } 307 308 return (error); 309 } 310 311 /* 312 * Allocate buffers for lockstat_start(). 313 */ 314 int 315 lockstat_alloc(lsenable_t *le) 316 { 317 lsbuf_t *lb; 318 size_t sz; 319 320 KASSERT(!lockstat_enabled); 321 lockstat_free(); 322 323 sz = sizeof(*lb) * le->le_nbufs; 324 325 lockstat_unlock(0); 326 lb = malloc(sz, M_LOCKSTAT, M_WAITOK | M_ZERO); 327 (void)lockstat_lock(0); 328 329 if (lb == NULL) 330 return (ENOMEM); 331 332 KASSERT(!lockstat_enabled); 333 KASSERT(lockstat_baseb == NULL); 334 lockstat_sizeb = sz; 335 lockstat_baseb = lb; 336 337 return (0); 338 } 339 340 /* 341 * Free allocated buffers after tracing has stopped. 342 */ 343 void 344 lockstat_free(void) 345 { 346 347 KASSERT(!lockstat_enabled); 348 349 if (lockstat_baseb != NULL) { 350 free(lockstat_baseb, M_LOCKSTAT); 351 lockstat_baseb = NULL; 352 } 353 } 354 355 /* 356 * Main entry point from lock primatives. 357 */ 358 void 359 lockstat_event(uintptr_t lock, uintptr_t callsite, u_int flags, u_int count, 360 uint64_t time) 361 { 362 lslist_t *ll; 363 lscpu_t *lc; 364 lsbuf_t *lb; 365 u_int event; 366 int s; 367 368 if ((flags & lockstat_enabled) != flags || count == 0) 369 return; 370 if (lock < lockstat_lockstart || lock > lockstat_lockend) 371 return; 372 if (callsite < lockstat_csstart || callsite > lockstat_csend) 373 return; 374 375 callsite &= lockstat_csmask; 376 377 /* 378 * Find the table for this lock+callsite pair, and try to locate a 379 * buffer with the same key. 380 */ 381 lc = curcpu()->ci_lockstat; 382 ll = &lc->lc_hash[LOCKSTAT_HASH(lock ^ callsite)]; 383 event = (flags & LB_EVENT_MASK) - 1; 384 s = spllock(); 385 386 LIST_FOREACH(lb, ll, lb_chain.list) { 387 if (lb->lb_lock == lock && lb->lb_callsite == callsite) 388 break; 389 } 390 391 if (lb != NULL) { 392 /* 393 * We found a record. Move it to the front of the list, as 394 * we're likely to hit it again soon. 395 */ 396 if (lb != LIST_FIRST(ll)) { 397 LIST_REMOVE(lb, lb_chain.list); 398 LIST_INSERT_HEAD(ll, lb, lb_chain.list); 399 } 400 lb->lb_counts[event] += count; 401 lb->lb_times[event] += time; 402 } else if ((lb = SLIST_FIRST(&lc->lc_free)) != NULL) { 403 /* 404 * Pinch a new buffer and fill it out. 405 */ 406 SLIST_REMOVE_HEAD(&lc->lc_free, lb_chain.slist); 407 LIST_INSERT_HEAD(ll, lb, lb_chain.list); 408 lb->lb_flags = (uint16_t)flags; 409 lb->lb_lock = lock; 410 lb->lb_callsite = callsite; 411 lb->lb_counts[event] = count; 412 lb->lb_times[event] = time; 413 } else { 414 /* 415 * We didn't find a buffer and there were none free. 416 * lockstat_stop() will notice later on and report the 417 * error. 418 */ 419 lc->lc_overflow++; 420 } 421 422 splx(s); 423 } 424 425 /* 426 * Accept an open() on /dev/lockstat. 427 */ 428 int 429 lockstat_open(dev_t dev, int flag, int mode, 430 struct lwp *l) 431 { 432 int error; 433 434 if ((error = lockstat_lock(1)) != 0) 435 return error; 436 437 if (lockstat_devopen) 438 error = EBUSY; 439 else { 440 lockstat_devopen = 1; 441 error = 0; 442 } 443 444 lockstat_unlock(1); 445 446 return error; 447 } 448 449 /* 450 * Accept the last close() on /dev/lockstat. 451 */ 452 int 453 lockstat_close(dev_t dev, int flag, int mode, 454 struct lwp *l) 455 { 456 int error; 457 458 if ((error = lockstat_lock(1)) == 0) { 459 if (lockstat_enabled) 460 (void)lockstat_stop(NULL); 461 lockstat_free(); 462 lockstat_devopen = 0; 463 lockstat_unlock(1); 464 } 465 466 return error; 467 } 468 469 /* 470 * Handle control operations. 471 */ 472 int 473 lockstat_ioctl(dev_t dev, u_long cmd, caddr_t data, 474 int flag, struct lwp *l) 475 { 476 lsenable_t *le; 477 int error; 478 479 if ((error = lockstat_lock(1)) != 0) 480 return error; 481 482 switch (cmd) { 483 case IOC_LOCKSTAT_GVERSION: 484 *(int *)data = LS_VERSION; 485 error = 0; 486 break; 487 488 case IOC_LOCKSTAT_ENABLE: 489 le = (lsenable_t *)data; 490 491 if (!cpu_hascounter()) { 492 error = ENODEV; 493 break; 494 } 495 if (lockstat_enabled) { 496 error = EBUSY; 497 break; 498 } 499 500 /* 501 * Sanitize the arguments passed in and set up filtering. 502 */ 503 if (le->le_nbufs == 0) 504 le->le_nbufs = LOCKSTAT_DEFBUFS; 505 else if (le->le_nbufs > LOCKSTAT_MAXBUFS || 506 le->le_nbufs < LOCKSTAT_MINBUFS) { 507 error = EINVAL; 508 break; 509 } 510 if ((le->le_flags & LE_ONE_CALLSITE) == 0) { 511 le->le_csstart = 0; 512 le->le_csend = le->le_csstart - 1; 513 } 514 if ((le->le_flags & LE_ONE_LOCK) == 0) { 515 le->le_lockstart = 0; 516 le->le_lockend = le->le_lockstart - 1; 517 } 518 if ((le->le_mask & LB_EVENT_MASK) == 0) 519 return (EINVAL); 520 if ((le->le_mask & LB_LOCK_MASK) == 0) 521 return (EINVAL); 522 523 /* 524 * Start tracing. 525 */ 526 if ((error = lockstat_alloc(le)) == 0) 527 lockstat_start(le); 528 break; 529 530 case IOC_LOCKSTAT_DISABLE: 531 if (!lockstat_enabled) 532 error = EINVAL; 533 else 534 error = lockstat_stop((lsdisable_t *)data); 535 break; 536 537 default: 538 error = ENOTTY; 539 break; 540 } 541 542 lockstat_unlock(1); 543 return error; 544 } 545 546 /* 547 * Copy buffers out to user-space. 548 */ 549 int 550 lockstat_read(dev_t dev, struct uio *uio, int flag) 551 { 552 int error; 553 554 if ((error = lockstat_lock(1)) != 0) 555 return (error); 556 557 if (lockstat_enabled) { 558 lockstat_unlock(1); 559 return (EBUSY); 560 } 561 562 lockstat_unlock(0); 563 error = uiomove(lockstat_baseb, lockstat_sizeb, uio); 564 lockstat_lock(0); 565 566 lockstat_unlock(1); 567 568 return (error); 569 } 570
Indexes created Thu Oct 02 01:09:59 GMT 2025