lockstat.c revision 1.2.4.2
1 /* $NetBSD: lockstat.c,v 1.2.4.2 2006/09/09 02:49:09 rpaulo Exp $ */ 2 3 /*- 4 * Copyright (c) 2006 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed by the NetBSD 21 * Foundation, Inc. and its contributors. 22 * 4. Neither the name of The NetBSD Foundation nor the names of its 23 * contributors may be used to endorse or promote products derived 24 * from this software without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 * POSSIBILITY OF SUCH DAMAGE. 37 */ 38 39 /* 40 * Lock statistics driver, providing kernel support for the lockstat(8) 41 * command. 42 */ 43 44 #include <sys/cdefs.h> 45 __KERNEL_RCSID(0, "$NetBSD: lockstat.c,v 1.2.4.2 2006/09/09 02:49:09 rpaulo Exp $"); 46 47 #include <sys/types.h> 48 #include <sys/param.h> 49 #include <sys/lock.h> 50 #include <sys/proc.h> 51 #include <sys/resourcevar.h> 52 #include <sys/systm.h> 53 #include <sys/kernel.h> 54 #include <sys/malloc.h> 55 #include <sys/conf.h> 56 #include <sys/syslog.h> 57 58 #include <dev/lockstat.h> 59 60 #ifndef __HAVE_CPU_COUNTER 61 #error CPU counters not available 62 #endif 63 64 #if LONG_BIT == 64 65 #define LOCKSTAT_HASH_SHIFT 3 66 #elif LONG_BIT == 32 67 #define LOCKSTAT_HASH_SHIFT 2 68 #endif 69 70 #define LOCKSTAT_MINBUFS 100 71 #define LOCKSTAT_DEFBUFS 1000 72 #define LOCKSTAT_MAXBUFS 10000 73 74 #define LOCKSTAT_HASH_SIZE 64 75 #define LOCKSTAT_HASH_MASK (LOCKSTAT_HASH_SIZE - 1) 76 #define LOCKSTAT_HASH(key) \ 77 ((key >> LOCKSTAT_HASH_SHIFT) & LOCKSTAT_HASH_MASK) 78 79 typedef struct lscpu { 80 SLIST_HEAD(, lsbuf) lc_free; 81 u_int lc_overflow; 82 LIST_HEAD(lslist, lsbuf) lc_hash[LOCKSTAT_HASH_SIZE]; 83 } lscpu_t; 84 85 typedef struct lslist lslist_t; 86 87 void lockstatattach(int); 88 void lockstat_start(lsenable_t *); 89 int lockstat_alloc(lsenable_t *); 90 void lockstat_init_tables(lsenable_t *); 91 int lockstat_stop(lsdisable_t *); 92 void lockstat_free(void); 93 94 dev_type_open(lockstat_open); 95 dev_type_close(lockstat_close); 96 dev_type_read(lockstat_read); 97 dev_type_ioctl(lockstat_ioctl); 98 99 /* Protected against write by lockstat_lock(). Used by lockstat_event(). */ 100 volatile u_int lockstat_enabled; 101 uintptr_t lockstat_csstart; 102 uintptr_t lockstat_csend; 103 uintptr_t lockstat_csmask; 104 uintptr_t lockstat_lockaddr; 105 106 /* Protected by lockstat_lock(). */ 107 struct simplelock lockstat_slock; 108 lsbuf_t *lockstat_baseb; 109 size_t lockstat_sizeb; 110 int lockstat_busy; 111 int lockstat_devopen; 112 struct timespec lockstat_stime; 113 114 const struct cdevsw lockstat_cdevsw = { 115 lockstat_open, lockstat_close, lockstat_read, nowrite, lockstat_ioctl, 116 nostop, notty, nopoll, nommap, nokqfilter, 0 117 }; 118 119 MALLOC_DEFINE(M_LOCKSTAT, "lockstat", "lockstat event buffers"); 120 121 /* 122 * Called when the pseudo-driver is attached. 123 */ 124 void 125 lockstatattach(int nunits) 126 { 127 128 (void)nunits; 129 130 __cpu_simple_lock_init(&lockstat_slock.lock_data); 131 } 132 133 /* 134 * Grab the global lock. If busy is set, we want to block out operations on 135 * the control device. 136 */ 137 static inline int 138 lockstat_lock(int busy) 139 { 140 141 if (!__cpu_simple_lock_try(&lockstat_slock.lock_data)) 142 return (EBUSY); 143 if (busy) { 144 if (lockstat_busy) { 145 __cpu_simple_unlock(&lockstat_slock.lock_data); 146 return (EBUSY); 147 } 148 lockstat_busy = 1; 149 } 150 KASSERT(lockstat_busy); 151 152 return 0; 153 } 154 155 /* 156 * Release the global lock. If unbusy is set, we want to allow new 157 * operations on the control device. 158 */ 159 static inline void 160 lockstat_unlock(int unbusy) 161 { 162 163 KASSERT(lockstat_busy); 164 if (unbusy) 165 lockstat_busy = 0; 166 __cpu_simple_unlock(&lockstat_slock.lock_data); 167 } 168 169 /* 170 * Prepare the per-CPU tables for use, or clear down tables when tracing is 171 * stopped. 172 */ 173 void 174 lockstat_init_tables(lsenable_t *le) 175 { 176 int i, ncpu, per, slop, cpuno; 177 CPU_INFO_ITERATOR cii; 178 struct cpu_info *ci; 179 lscpu_t *lc; 180 lsbuf_t *lb; 181 182 KASSERT(!lockstat_enabled); 183 184 ncpu = 0; 185 for (CPU_INFO_FOREACH(cii, ci)) { 186 if (ci->ci_lockstat != NULL) { 187 free(ci->ci_lockstat, M_LOCKSTAT); 188 ci->ci_lockstat = NULL; 189 } 190 ncpu++; 191 } 192 193 if (le == NULL) 194 return; 195 196 lb = lockstat_baseb; 197 per = le->le_nbufs / ncpu; 198 slop = le->le_nbufs - (per * ncpu); 199 cpuno = 0; 200 for (CPU_INFO_FOREACH(cii, ci)) { 201 lc = malloc(sizeof(*lc), M_LOCKSTAT, M_WAITOK); 202 lc->lc_overflow = 0; 203 ci->ci_lockstat = lc; 204 205 SLIST_INIT(&lc->lc_free); 206 for (i = 0; i < LOCKSTAT_HASH_SIZE; i++) 207 LIST_INIT(&lc->lc_hash[i]); 208 209 for (i = per; i != 0; i--, lb++) { 210 lb->lb_cpu = (uint16_t)cpuno; 211 SLIST_INSERT_HEAD(&lc->lc_free, lb, lb_chain.slist); 212 } 213 if (--slop > 0) { 214 lb->lb_cpu = (uint16_t)cpuno; 215 SLIST_INSERT_HEAD(&lc->lc_free, lb, lb_chain.slist); 216 lb++; 217 } 218 cpuno++; 219 } 220 } 221 222 /* 223 * Start collecting lock statistics. 224 */ 225 void 226 lockstat_start(lsenable_t *le) 227 { 228 229 KASSERT(!lockstat_enabled); 230 231 lockstat_init_tables(le); 232 233 if ((le->le_flags & LE_CALLSITE) != 0) 234 lockstat_csmask = (uintptr_t)-1LL; 235 else 236 lockstat_csmask = 0; 237 238 lockstat_csstart = le->le_csstart; 239 lockstat_csend = le->le_csend; 240 lockstat_lockaddr = le->le_lock; 241 242 /* 243 * Force a write barrier. XXX This may not be sufficient.. 244 */ 245 lockstat_unlock(0); 246 tsleep(&lockstat_start, PPAUSE, "lockstat", mstohz(10)); 247 (void)lockstat_lock(0); 248 249 getnanotime(&lockstat_stime); 250 lockstat_enabled = le->le_mask; 251 lockstat_unlock(0); 252 (void)lockstat_lock(0); 253 } 254 255 /* 256 * Stop collecting lock statistics. 257 */ 258 int 259 lockstat_stop(lsdisable_t *ld) 260 { 261 CPU_INFO_ITERATOR cii; 262 struct cpu_info *ci; 263 u_int cpuno, overflow; 264 struct timespec ts; 265 int error; 266 267 KASSERT(lockstat_enabled); 268 269 /* 270 * Set enabled false, force a write barrier, and wait for other CPUs 271 * to exit lockstat_event(). XXX This may not be sufficient.. 272 */ 273 lockstat_enabled = 0; 274 lockstat_unlock(0); 275 getnanotime(&ts); 276 tsleep(&lockstat_stop, PPAUSE, "lockstat", mstohz(10)); 277 (void)lockstat_lock(0); 278 279 /* 280 * Did we run out of buffers while tracing? 281 */ 282 overflow = 0; 283 for (CPU_INFO_FOREACH(cii, ci)) 284 overflow += ((lscpu_t *)ci->ci_lockstat)->lc_overflow; 285 286 if (overflow != 0) { 287 error = EOVERFLOW; 288 log(LOG_NOTICE, "lockstat: %d buffer allocations failed\n", 289 overflow); 290 } else 291 error = 0; 292 293 lockstat_init_tables(NULL); 294 295 if (ld == NULL) 296 return (error); 297 298 /* 299 * Fill out the disable struct for the caller. 300 */ 301 timespecsub(&ts, &lockstat_stime, &ld->ld_time); 302 ld->ld_size = lockstat_sizeb; 303 304 cpuno = 0; 305 for (CPU_INFO_FOREACH(cii, ci)) { 306 if (cpuno > sizeof(ld->ld_freq) / sizeof(ld->ld_freq[0])) { 307 log(LOG_WARNING, "lockstat: too many CPUs\n"); 308 break; 309 } 310 ld->ld_freq[cpuno++] = cpu_frequency(ci); 311 } 312 313 return (error); 314 } 315 316 /* 317 * Allocate buffers for lockstat_start(). 318 */ 319 int 320 lockstat_alloc(lsenable_t *le) 321 { 322 lsbuf_t *lb; 323 size_t sz; 324 325 KASSERT(!lockstat_enabled); 326 lockstat_free(); 327 328 sz = sizeof(*lb) * le->le_nbufs; 329 330 lockstat_unlock(0); 331 lb = malloc(sz, M_LOCKSTAT, M_WAITOK | M_ZERO); 332 (void)lockstat_lock(0); 333 334 if (lb == NULL) 335 return (ENOMEM); 336 337 KASSERT(!lockstat_enabled); 338 KASSERT(lockstat_baseb == NULL); 339 lockstat_sizeb = sz; 340 lockstat_baseb = lb; 341 342 return (0); 343 } 344 345 /* 346 * Free allocated buffers after tracing has stopped. 347 */ 348 void 349 lockstat_free(void) 350 { 351 352 KASSERT(!lockstat_enabled); 353 354 if (lockstat_baseb != NULL) { 355 free(lockstat_baseb, M_LOCKSTAT); 356 lockstat_baseb = NULL; 357 } 358 } 359 360 /* 361 * Main entry point from lock primatives. 362 */ 363 void 364 lockstat_event(uintptr_t lock, uintptr_t callsite, u_int flags, u_int count, 365 uint64_t time) 366 { 367 lslist_t *ll; 368 lscpu_t *lc; 369 lsbuf_t *lb; 370 u_int event; 371 int s; 372 373 if ((flags & lockstat_enabled) != flags || count == 0) 374 return; 375 if (lockstat_lockaddr != 0 && lock != lockstat_lockaddr) 376 return; 377 if (callsite < lockstat_csstart || callsite > lockstat_csend) 378 return; 379 380 callsite &= lockstat_csmask; 381 382 /* 383 * Find the table for this lock+callsite pair, and try to locate a 384 * buffer with the same key. 385 */ 386 lc = curcpu()->ci_lockstat; 387 ll = &lc->lc_hash[LOCKSTAT_HASH(lock ^ callsite)]; 388 event = (flags & LB_EVENT_MASK) - 1; 389 s = spllock(); 390 391 LIST_FOREACH(lb, ll, lb_chain.list) { 392 if (lb->lb_lock == lock && lb->lb_callsite == callsite) 393 break; 394 } 395 396 if (lb != NULL) { 397 /* 398 * We found a record. Move it to the front of the list, as 399 * we're likely to hit it again soon. 400 */ 401 if (lb != LIST_FIRST(ll)) { 402 LIST_REMOVE(lb, lb_chain.list); 403 LIST_INSERT_HEAD(ll, lb, lb_chain.list); 404 } 405 lb->lb_counts[event] += count; 406 lb->lb_times[event] += time; 407 } else if ((lb = SLIST_FIRST(&lc->lc_free)) != NULL) { 408 /* 409 * Pinch a new buffer and fill it out. 410 */ 411 SLIST_REMOVE_HEAD(&lc->lc_free, lb_chain.slist); 412 LIST_INSERT_HEAD(ll, lb, lb_chain.list); 413 lb->lb_flags = (uint16_t)flags; 414 lb->lb_lock = lock; 415 lb->lb_callsite = callsite; 416 lb->lb_counts[event] = count; 417 lb->lb_times[event] = time; 418 } else { 419 /* 420 * We didn't find a buffer and there were none free. 421 * lockstat_stop() will notice later on and report the 422 * error. 423 */ 424 lc->lc_overflow++; 425 } 426 427 splx(s); 428 } 429 430 /* 431 * Accept an open() on /dev/lockstat. 432 */ 433 int 434 lockstat_open(dev_t dev, int flag, int mode, struct lwp *l) 435 { 436 int error; 437 438 if ((error = lockstat_lock(1)) != 0) 439 return error; 440 441 if (lockstat_devopen) 442 error = EBUSY; 443 else { 444 lockstat_devopen = 1; 445 error = 0; 446 } 447 448 lockstat_unlock(1); 449 450 return error; 451 } 452 453 /* 454 * Accept the last close() on /dev/lockstat. 455 */ 456 int 457 lockstat_close(dev_t dev, int flag, int mode, struct lwp *l) 458 { 459 int error; 460 461 if ((error = lockstat_lock(1)) == 0) { 462 if (lockstat_enabled) 463 (void)lockstat_stop(NULL); 464 lockstat_free(); 465 lockstat_devopen = 0; 466 lockstat_unlock(1); 467 } 468 469 return error; 470 } 471 472 /* 473 * Handle control operations. 474 */ 475 int 476 lockstat_ioctl(dev_t dev, u_long cmd, caddr_t data, int flag, struct lwp *l) 477 { 478 lsenable_t *le; 479 int error; 480 481 if ((error = lockstat_lock(1)) != 0) 482 return error; 483 484 switch (cmd) { 485 case IOC_LOCKSTAT_GVERSION: 486 *(int *)data = LS_VERSION; 487 error = 0; 488 break; 489 490 case IOC_LOCKSTAT_ENABLE: 491 le = (lsenable_t *)data; 492 493 if (!cpu_hascounter()) { 494 error = ENODEV; 495 break; 496 } 497 if (lockstat_enabled) { 498 error = EBUSY; 499 break; 500 } 501 502 /* 503 * Sanitize the arguments passed in and set up filtering. 504 */ 505 if (le->le_nbufs == 0) 506 le->le_nbufs = LOCKSTAT_DEFBUFS; 507 else if (le->le_nbufs > LOCKSTAT_MAXBUFS || 508 le->le_nbufs < LOCKSTAT_MINBUFS) { 509 error = EINVAL; 510 break; 511 } 512 if ((le->le_flags & LE_ONE_CALLSITE) == 0) { 513 le->le_csstart = 0; 514 le->le_csend = le->le_csstart - 1; 515 } 516 if ((le->le_flags & LE_ONE_LOCK) == 0) 517 le->le_lock = 0; 518 if ((le->le_mask & LB_EVENT_MASK) == 0) 519 return (EINVAL); 520 if ((le->le_mask & LB_LOCK_MASK) == 0) 521 return (EINVAL); 522 523 /* 524 * Start tracing. 525 */ 526 if ((error = lockstat_alloc(le)) == 0) 527 lockstat_start(le); 528 break; 529 530 case IOC_LOCKSTAT_DISABLE: 531 if (!lockstat_enabled) 532 error = EINVAL; 533 else 534 error = lockstat_stop((lsdisable_t *)data); 535 break; 536 537 default: 538 error = ENOTTY; 539 break; 540 } 541 542 lockstat_unlock(1); 543 return error; 544 } 545 546 /* 547 * Copy buffers out to user-space. 548 */ 549 int 550 lockstat_read(dev_t dev, struct uio *uio, int flag) 551 { 552 int error; 553 554 if ((error = lockstat_lock(1)) != 0) 555 return (error); 556 557 if (lockstat_enabled) { 558 lockstat_unlock(1); 559 return (EBUSY); 560 } 561 562 lockstat_unlock(0); 563 error = uiomove(lockstat_baseb, lockstat_sizeb, uio); 564 lockstat_lock(0); 565 566 lockstat_unlock(1); 567 568 return (error); 569 } 570
Indexes created Thu Oct 02 01:09:59 GMT 2025