lockstat.c revision 1.2.2.2
1 /* $NetBSD: lockstat.c,v 1.2.2.2 2006/11/18 21:34:03 ad Exp $ */ 2 3 /*- 4 * Copyright (c) 2006 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed by the NetBSD 21 * Foundation, Inc. and its contributors. 22 * 4. Neither the name of The NetBSD Foundation nor the names of its 23 * contributors may be used to endorse or promote products derived 24 * from this software without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 * POSSIBILITY OF SUCH DAMAGE. 37 */ 38 39 /* 40 * Lock statistics driver, providing kernel support for the lockstat(8) 41 * command. 42 */ 43 44 #include <sys/cdefs.h> 45 __KERNEL_RCSID(0, "$NetBSD: lockstat.c,v 1.2.2.2 2006/11/18 21:34:03 ad Exp $"); 46 47 #include <sys/types.h> 48 #include <sys/param.h> 49 #include <sys/lock.h> 50 #include <sys/proc.h> 51 #include <sys/resourcevar.h> 52 #include <sys/systm.h> 53 #include <sys/kernel.h> 54 #include <sys/malloc.h> 55 #include <sys/conf.h> 56 #include <sys/syslog.h> 57 58 #include <dev/lockstat.h> 59 60 #ifndef __HAVE_CPU_COUNTER 61 #error CPU counters not available 62 #endif 63 64 #if LONG_BIT == 64 65 #define LOCKSTAT_HASH_SHIFT 3 66 #elif LONG_BIT == 32 67 #define LOCKSTAT_HASH_SHIFT 2 68 #endif 69 70 #define LOCKSTAT_MINBUFS 100 71 #define LOCKSTAT_DEFBUFS 1000 72 #define LOCKSTAT_MAXBUFS 10000 73 74 #define LOCKSTAT_HASH_SIZE 64 75 #define LOCKSTAT_HASH_MASK (LOCKSTAT_HASH_SIZE - 1) 76 #define LOCKSTAT_HASH(key) \ 77 ((key >> LOCKSTAT_HASH_SHIFT) & LOCKSTAT_HASH_MASK) 78 79 typedef struct lscpu { 80 SLIST_HEAD(, lsbuf) lc_free; 81 u_int lc_overflow; 82 LIST_HEAD(lslist, lsbuf) lc_hash[LOCKSTAT_HASH_SIZE]; 83 } lscpu_t; 84 85 typedef struct lslist lslist_t; 86 87 void lockstatattach(int); 88 void lockstat_start(lsenable_t *); 89 int lockstat_alloc(lsenable_t *); 90 void lockstat_init_tables(lsenable_t *); 91 int lockstat_stop(lsdisable_t *); 92 void lockstat_free(void); 93 94 dev_type_open(lockstat_open); 95 dev_type_close(lockstat_close); 96 dev_type_read(lockstat_read); 97 dev_type_ioctl(lockstat_ioctl); 98 99 /* Protected against write by lockstat_lock(). Used by lockstat_event(). */ 100 volatile u_int lockstat_enabled; 101 uintptr_t lockstat_csstart; 102 uintptr_t lockstat_csend; 103 uintptr_t lockstat_csmask; 104 uintptr_t lockstat_lockaddr; 105 106 /* Protected by lockstat_lock(). */ 107 struct simplelock lockstat_slock; 108 lsbuf_t *lockstat_baseb; 109 size_t lockstat_sizeb; 110 int lockstat_busy; 111 int lockstat_devopen; 112 struct timespec lockstat_stime; 113 114 const struct cdevsw lockstat_cdevsw = { 115 lockstat_open, lockstat_close, lockstat_read, nowrite, lockstat_ioctl, 116 nostop, notty, nopoll, nommap, nokqfilter, 0 117 }; 118 119 MALLOC_DEFINE(M_LOCKSTAT, "lockstat", "lockstat event buffers"); 120 121 /* 122 * Called when the pseudo-driver is attached. 123 */ 124 void 125 lockstatattach(int nunits) 126 { 127 128 (void)nunits; 129 130 __cpu_simple_lock_init(&lockstat_slock.lock_data); 131 } 132 133 /* 134 * Grab the global lock. If busy is set, we want to block out operations on 135 * the control device. 136 */ 137 static inline int 138 lockstat_lock(int busy) 139 { 140 141 if (!__cpu_simple_lock_try(&lockstat_slock.lock_data)) 142 return (EBUSY); 143 if (busy) { 144 if (lockstat_busy) { 145 __cpu_simple_unlock(&lockstat_slock.lock_data); 146 return (EBUSY); 147 } 148 lockstat_busy = 1; 149 } 150 KASSERT(lockstat_busy); 151 152 return 0; 153 } 154 155 /* 156 * Release the global lock. If unbusy is set, we want to allow new 157 * operations on the control device. 158 */ 159 static inline void 160 lockstat_unlock(int unbusy) 161 { 162 163 KASSERT(lockstat_busy); 164 if (unbusy) 165 lockstat_busy = 0; 166 __cpu_simple_unlock(&lockstat_slock.lock_data); 167 } 168 169 /* 170 * Prepare the per-CPU tables for use, or clear down tables when tracing is 171 * stopped. 172 */ 173 void 174 lockstat_init_tables(lsenable_t *le) 175 { 176 int i, ncpu, per, slop, cpuno; 177 CPU_INFO_ITERATOR cii; 178 struct cpu_info *ci; 179 lscpu_t *lc; 180 lsbuf_t *lb; 181 182 KASSERT(!lockstat_enabled); 183 184 ncpu = 0; 185 for (CPU_INFO_FOREACH(cii, ci)) { 186 if (ci->ci_lockstat != NULL) { 187 free(ci->ci_lockstat, M_LOCKSTAT); 188 ci->ci_lockstat = NULL; 189 } 190 ncpu++; 191 } 192 193 if (le == NULL) 194 return; 195 196 lb = lockstat_baseb; 197 per = le->le_nbufs / ncpu; 198 slop = le->le_nbufs - (per * ncpu); 199 cpuno = 0; 200 for (CPU_INFO_FOREACH(cii, ci)) { 201 lc = malloc(sizeof(*lc), M_LOCKSTAT, M_WAITOK); 202 lc->lc_overflow = 0; 203 ci->ci_lockstat = lc; 204 205 SLIST_INIT(&lc->lc_free); 206 for (i = 0; i < LOCKSTAT_HASH_SIZE; i++) 207 LIST_INIT(&lc->lc_hash[i]); 208 209 for (i = per; i != 0; i--, lb++) { 210 lb->lb_cpu = (uint16_t)cpuno; 211 SLIST_INSERT_HEAD(&lc->lc_free, lb, lb_chain.slist); 212 } 213 if (--slop > 0) { 214 lb->lb_cpu = (uint16_t)cpuno; 215 SLIST_INSERT_HEAD(&lc->lc_free, lb, lb_chain.slist); 216 lb++; 217 } 218 cpuno++; 219 } 220 } 221 222 /* 223 * Start collecting lock statistics. 224 */ 225 void 226 lockstat_start(lsenable_t *le) 227 { 228 229 KASSERT(!lockstat_enabled); 230 231 lockstat_init_tables(le); 232 233 if ((le->le_flags & LE_CALLSITE) != 0) 234 lockstat_csmask = (uintptr_t)-1LL; 235 else 236 lockstat_csmask = 0; 237 238 lockstat_csstart = le->le_csstart; 239 lockstat_csend = le->le_csend; 240 lockstat_lockaddr = le->le_lock; 241 mb_write(); 242 243 getnanotime(&lockstat_stime); 244 lockstat_enabled = le->le_mask; 245 mb_write(); 246 } 247 248 /* 249 * Stop collecting lock statistics. 250 */ 251 int 252 lockstat_stop(lsdisable_t *ld) 253 { 254 CPU_INFO_ITERATOR cii; 255 struct cpu_info *ci; 256 u_int cpuno, overflow; 257 struct timespec ts; 258 int error; 259 260 KASSERT(lockstat_enabled); 261 262 /* 263 * Set enabled false, force a write barrier, and wait for other CPUs 264 * to exit lockstat_event(). 265 */ 266 lockstat_enabled = 0; 267 lockstat_unlock(0); 268 getnanotime(&ts); 269 tsleep(&lockstat_stop, PPAUSE, "lockstat", mstohz(10)); 270 (void)lockstat_lock(0); 271 272 /* 273 * Did we run out of buffers while tracing? 274 */ 275 overflow = 0; 276 for (CPU_INFO_FOREACH(cii, ci)) 277 overflow += ((lscpu_t *)ci->ci_lockstat)->lc_overflow; 278 279 if (overflow != 0) { 280 error = EOVERFLOW; 281 log(LOG_NOTICE, "lockstat: %d buffer allocations failed\n", 282 overflow); 283 } else 284 error = 0; 285 286 lockstat_init_tables(NULL); 287 288 if (ld == NULL) 289 return (error); 290 291 /* 292 * Fill out the disable struct for the caller. 293 */ 294 timespecsub(&ts, &lockstat_stime, &ld->ld_time); 295 ld->ld_size = lockstat_sizeb; 296 297 cpuno = 0; 298 for (CPU_INFO_FOREACH(cii, ci)) { 299 if (cpuno > sizeof(ld->ld_freq) / sizeof(ld->ld_freq[0])) { 300 log(LOG_WARNING, "lockstat: too many CPUs\n"); 301 break; 302 } 303 ld->ld_freq[cpuno++] = cpu_frequency(ci); 304 } 305 306 return (error); 307 } 308 309 /* 310 * Allocate buffers for lockstat_start(). 311 */ 312 int 313 lockstat_alloc(lsenable_t *le) 314 { 315 lsbuf_t *lb; 316 size_t sz; 317 318 KASSERT(!lockstat_enabled); 319 lockstat_free(); 320 321 sz = sizeof(*lb) * le->le_nbufs; 322 323 lockstat_unlock(0); 324 lb = malloc(sz, M_LOCKSTAT, M_WAITOK | M_ZERO); 325 (void)lockstat_lock(0); 326 327 if (lb == NULL) 328 return (ENOMEM); 329 330 KASSERT(!lockstat_enabled); 331 KASSERT(lockstat_baseb == NULL); 332 lockstat_sizeb = sz; 333 lockstat_baseb = lb; 334 335 return (0); 336 } 337 338 /* 339 * Free allocated buffers after tracing has stopped. 340 */ 341 void 342 lockstat_free(void) 343 { 344 345 KASSERT(!lockstat_enabled); 346 347 if (lockstat_baseb != NULL) { 348 free(lockstat_baseb, M_LOCKSTAT); 349 lockstat_baseb = NULL; 350 } 351 } 352 353 /* 354 * Main entry point from lock primatives. 355 */ 356 void 357 lockstat_event(uintptr_t lock, uintptr_t callsite, u_int flags, u_int count, 358 uint64_t time) 359 { 360 lslist_t *ll; 361 lscpu_t *lc; 362 lsbuf_t *lb; 363 u_int event; 364 int s; 365 366 if ((flags & lockstat_enabled) != flags || count == 0) 367 return; 368 if (lockstat_lockaddr != 0 && lock != lockstat_lockaddr) 369 return; 370 if (callsite < lockstat_csstart || callsite > lockstat_csend) 371 return; 372 373 callsite &= lockstat_csmask; 374 375 /* 376 * Find the table for this lock+callsite pair, and try to locate a 377 * buffer with the same key. 378 */ 379 lc = curcpu()->ci_lockstat; 380 ll = &lc->lc_hash[LOCKSTAT_HASH(lock ^ callsite)]; 381 event = (flags & LB_EVENT_MASK) - 1; 382 s = spllock(); 383 384 LIST_FOREACH(lb, ll, lb_chain.list) { 385 if (lb->lb_lock == lock && lb->lb_callsite == callsite) 386 break; 387 } 388 389 if (lb != NULL) { 390 /* 391 * We found a record. Move it to the front of the list, as 392 * we're likely to hit it again soon. 393 */ 394 if (lb != LIST_FIRST(ll)) { 395 LIST_REMOVE(lb, lb_chain.list); 396 LIST_INSERT_HEAD(ll, lb, lb_chain.list); 397 } 398 lb->lb_counts[event] += count; 399 lb->lb_times[event] += time; 400 } else if ((lb = SLIST_FIRST(&lc->lc_free)) != NULL) { 401 /* 402 * Pinch a new buffer and fill it out. 403 */ 404 SLIST_REMOVE_HEAD(&lc->lc_free, lb_chain.slist); 405 LIST_INSERT_HEAD(ll, lb, lb_chain.list); 406 lb->lb_flags = (uint16_t)flags; 407 lb->lb_lock = lock; 408 lb->lb_callsite = callsite; 409 lb->lb_counts[event] = count; 410 lb->lb_times[event] = time; 411 } else { 412 /* 413 * We didn't find a buffer and there were none free. 414 * lockstat_stop() will notice later on and report the 415 * error. 416 */ 417 lc->lc_overflow++; 418 } 419 420 splx(s); 421 } 422 423 /* 424 * Accept an open() on /dev/lockstat. 425 */ 426 int 427 lockstat_open(dev_t dev, int flag, int mode, 428 struct lwp *l) 429 { 430 int error; 431 432 if ((error = lockstat_lock(1)) != 0) 433 return error; 434 435 if (lockstat_devopen) 436 error = EBUSY; 437 else { 438 lockstat_devopen = 1; 439 error = 0; 440 } 441 442 lockstat_unlock(1); 443 444 return error; 445 } 446 447 /* 448 * Accept the last close() on /dev/lockstat. 449 */ 450 int 451 lockstat_close(dev_t dev, int flag, int mode, 452 struct lwp *l) 453 { 454 int error; 455 456 if ((error = lockstat_lock(1)) == 0) { 457 if (lockstat_enabled) 458 (void)lockstat_stop(NULL); 459 lockstat_free(); 460 lockstat_devopen = 0; 461 lockstat_unlock(1); 462 } 463 464 return error; 465 } 466 467 /* 468 * Handle control operations. 469 */ 470 int 471 lockstat_ioctl(dev_t dev, u_long cmd, caddr_t data, 472 int flag, struct lwp *l) 473 { 474 lsenable_t *le; 475 int error; 476 477 if ((error = lockstat_lock(1)) != 0) 478 return error; 479 480 switch (cmd) { 481 case IOC_LOCKSTAT_GVERSION: 482 *(int *)data = LS_VERSION; 483 error = 0; 484 break; 485 486 case IOC_LOCKSTAT_ENABLE: 487 le = (lsenable_t *)data; 488 489 if (!cpu_hascounter()) { 490 error = ENODEV; 491 break; 492 } 493 if (lockstat_enabled) { 494 error = EBUSY; 495 break; 496 } 497 498 /* 499 * Sanitize the arguments passed in and set up filtering. 500 */ 501 if (le->le_nbufs == 0) 502 le->le_nbufs = LOCKSTAT_DEFBUFS; 503 else if (le->le_nbufs > LOCKSTAT_MAXBUFS || 504 le->le_nbufs < LOCKSTAT_MINBUFS) { 505 error = EINVAL; 506 break; 507 } 508 if ((le->le_flags & LE_ONE_CALLSITE) == 0) { 509 le->le_csstart = 0; 510 le->le_csend = le->le_csstart - 1; 511 } 512 if ((le->le_flags & LE_ONE_LOCK) == 0) 513 le->le_lock = 0; 514 if ((le->le_mask & LB_EVENT_MASK) == 0) 515 return (EINVAL); 516 if ((le->le_mask & LB_LOCK_MASK) == 0) 517 return (EINVAL); 518 519 /* 520 * Start tracing. 521 */ 522 if ((error = lockstat_alloc(le)) == 0) 523 lockstat_start(le); 524 break; 525 526 case IOC_LOCKSTAT_DISABLE: 527 if (!lockstat_enabled) 528 error = EINVAL; 529 else 530 error = lockstat_stop((lsdisable_t *)data); 531 break; 532 533 default: 534 error = ENOTTY; 535 break; 536 } 537 538 lockstat_unlock(1); 539 return error; 540 } 541 542 /* 543 * Copy buffers out to user-space. 544 */ 545 int 546 lockstat_read(dev_t dev, struct uio *uio, int flag) 547 { 548 int error; 549 550 if ((error = lockstat_lock(1)) != 0) 551 return (error); 552 553 if (lockstat_enabled) { 554 lockstat_unlock(1); 555 return (EBUSY); 556 } 557 558 lockstat_unlock(0); 559 error = uiomove(lockstat_baseb, lockstat_sizeb, uio); 560 lockstat_lock(0); 561 562 lockstat_unlock(1); 563 564 return (error); 565 } 566
Indexes created Thu Oct 02 01:09:59 GMT 2025