Home | History | Annotate | Line # | Download | only in dev
lockstat.c revision 1.6 
      1 /*	$NetBSD: lockstat.c,v 1.6 2007/02/09 21:55:26 ad Exp $	*/
      2 
      3 /*-
      4  * Copyright (c) 2006 The NetBSD Foundation, Inc.
      5  * All rights reserved.
      6  *
      7  * This code is derived from software contributed to The NetBSD Foundation
      8  * by Andrew Doran.
      9  *
     10  * Redistribution and use in source and binary forms, with or without
     11  * modification, are permitted provided that the following conditions
     12  * are met:
     13  * 1. Redistributions of source code must retain the above copyright
     14  *    notice, this list of conditions and the following disclaimer.
     15  * 2. Redistributions in binary form must reproduce the above copyright
     16  *    notice, this list of conditions and the following disclaimer in the
     17  *    documentation and/or other materials provided with the distribution.
     18  * 3. All advertising materials mentioning features or use of this software
     19  *    must display the following acknowledgement:
     20  *	This product includes software developed by the NetBSD
     21  *	Foundation, Inc. and its contributors.
     22  * 4. Neither the name of The NetBSD Foundation nor the names of its
     23  *    contributors may be used to endorse or promote products derived
     24  *    from this software without specific prior written permission.
     25  *
     26  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     27  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     28  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     29  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     30  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     31  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     32  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     33  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     34  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     35  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     36  * POSSIBILITY OF SUCH DAMAGE.
     37  */
     38 
     39 /*
     40  * Lock statistics driver, providing kernel support for the lockstat(8)
     41  * command.
     42  *
     43  * XXX Timings for contention on sleep locks are currently incorrect.
     44  */
     45 
     46 #include <sys/cdefs.h>
     47 __KERNEL_RCSID(0, "$NetBSD: lockstat.c,v 1.6 2007/02/09 21:55:26 ad Exp $");
     48 
     49 #include <sys/types.h>
     50 #include <sys/param.h>
     51 #include <sys/lock.h>
     52 #include <sys/proc.h>
     53 #include <sys/resourcevar.h>
     54 #include <sys/systm.h>
     55 #include <sys/kernel.h>
     56 #include <sys/malloc.h>
     57 #include <sys/conf.h>
     58 #include <sys/syslog.h>
     59 
     60 #include <dev/lockstat.h>
     61 
     62 #ifndef __HAVE_CPU_COUNTER
     63 #error CPU counters not available
     64 #endif
     65 
     66 #if LONG_BIT == 64
     67 #define	LOCKSTAT_HASH_SHIFT	3
     68 #elif LONG_BIT == 32
     69 #define	LOCKSTAT_HASH_SHIFT	2
     70 #endif
     71 
     72 #define	LOCKSTAT_MINBUFS	100
     73 #define	LOCKSTAT_DEFBUFS	1000
     74 #define	LOCKSTAT_MAXBUFS	10000
     75 
     76 #define	LOCKSTAT_HASH_SIZE	64
     77 #define	LOCKSTAT_HASH_MASK	(LOCKSTAT_HASH_SIZE - 1)
     78 #define	LOCKSTAT_HASH(key)	\
     79 	((key >> LOCKSTAT_HASH_SHIFT) & LOCKSTAT_HASH_MASK)
     80 
     81 typedef struct lscpu {
     82 	SLIST_HEAD(, lsbuf)	lc_free;
     83 	u_int			lc_overflow;
     84 	LIST_HEAD(lslist, lsbuf) lc_hash[LOCKSTAT_HASH_SIZE];
     85 } lscpu_t;
     86 
     87 typedef struct lslist lslist_t;
     88 
     89 void	lockstatattach(int);
     90 void	lockstat_start(lsenable_t *);
     91 int	lockstat_alloc(lsenable_t *);
     92 void	lockstat_init_tables(lsenable_t *);
     93 int	lockstat_stop(lsdisable_t *);
     94 void	lockstat_free(void);
     95 
     96 dev_type_open(lockstat_open);
     97 dev_type_close(lockstat_close);
     98 dev_type_read(lockstat_read);
     99 dev_type_ioctl(lockstat_ioctl);
    100 
    101 /* Protected against write by lockstat_lock().  Used by lockstat_event(). */
    102 volatile u_int	lockstat_enabled;
    103 uintptr_t	lockstat_csstart;
    104 uintptr_t	lockstat_csend;
    105 uintptr_t	lockstat_csmask;
    106 uintptr_t	lockstat_lockstart;
    107 uintptr_t	lockstat_lockend;
    108 
    109 /* Protected by lockstat_lock(). */
    110 struct simplelock lockstat_slock;
    111 lsbuf_t		*lockstat_baseb;
    112 size_t		lockstat_sizeb;
    113 int		lockstat_busy;
    114 int		lockstat_devopen;
    115 struct timespec	lockstat_stime;
    116 
    117 const struct cdevsw lockstat_cdevsw = {
    118 	lockstat_open, lockstat_close, lockstat_read, nowrite, lockstat_ioctl,
    119 	nostop, notty, nopoll, nommap, nokqfilter, 0
    120 };
    121 
    122 MALLOC_DEFINE(M_LOCKSTAT, "lockstat", "lockstat event buffers");
    123 
    124 /*
    125  * Called when the pseudo-driver is attached.
    126  */
    127 void
    128 lockstatattach(int nunits)
    129 {
    130 
    131 	(void)nunits;
    132 
    133 	__cpu_simple_lock_init(&lockstat_slock.lock_data);
    134 }
    135 
    136 /*
    137  * Grab the global lock.  If busy is set, we want to block out operations on
    138  * the control device.
    139  */
    140 static inline int
    141 lockstat_lock(int busy)
    142 {
    143 
    144 	if (!__cpu_simple_lock_try(&lockstat_slock.lock_data))
    145 		return (EBUSY);
    146 	if (busy) {
    147 		if (lockstat_busy) {
    148 			__cpu_simple_unlock(&lockstat_slock.lock_data);
    149 			return (EBUSY);
    150 		}
    151 		lockstat_busy = 1;
    152 	}
    153 	KASSERT(lockstat_busy);
    154 
    155 	return 0;
    156 }
    157 
    158 /*
    159  * Release the global lock.  If unbusy is set, we want to allow new
    160  * operations on the control device.
    161  */
    162 static inline void
    163 lockstat_unlock(int unbusy)
    164 {
    165 
    166 	KASSERT(lockstat_busy);
    167 	if (unbusy)
    168 		lockstat_busy = 0;
    169 	__cpu_simple_unlock(&lockstat_slock.lock_data);
    170 }
    171 
    172 /*
    173  * Prepare the per-CPU tables for use, or clear down tables when tracing is
    174  * stopped.
    175  */
    176 void
    177 lockstat_init_tables(lsenable_t *le)
    178 {
    179 	int i, ncpu, per, slop, cpuno;
    180 	CPU_INFO_ITERATOR cii;
    181 	struct cpu_info *ci;
    182 	lscpu_t *lc;
    183 	lsbuf_t *lb;
    184 
    185 	KASSERT(!lockstat_enabled);
    186 
    187 	ncpu = 0;
    188 	for (CPU_INFO_FOREACH(cii, ci)) {
    189 		if (ci->ci_lockstat != NULL) {
    190 			free(ci->ci_lockstat, M_LOCKSTAT);
    191 			ci->ci_lockstat = NULL;
    192 		}
    193 		ncpu++;
    194 	}
    195 
    196 	if (le == NULL)
    197 		return;
    198 
    199 	lb = lockstat_baseb;
    200 	per = le->le_nbufs / ncpu;
    201 	slop = le->le_nbufs - (per * ncpu);
    202 	cpuno = 0;
    203 	for (CPU_INFO_FOREACH(cii, ci)) {
    204 		lc = malloc(sizeof(*lc), M_LOCKSTAT, M_WAITOK);
    205 		lc->lc_overflow = 0;
    206 		ci->ci_lockstat = lc;
    207 
    208 		SLIST_INIT(&lc->lc_free);
    209 		for (i = 0; i < LOCKSTAT_HASH_SIZE; i++)
    210 			LIST_INIT(&lc->lc_hash[i]);
    211 
    212 		for (i = per; i != 0; i--, lb++) {
    213 			lb->lb_cpu = (uint16_t)cpuno;
    214 			SLIST_INSERT_HEAD(&lc->lc_free, lb, lb_chain.slist);
    215 		}
    216 		if (--slop > 0) {
    217 			lb->lb_cpu = (uint16_t)cpuno;
    218 			SLIST_INSERT_HEAD(&lc->lc_free, lb, lb_chain.slist);
    219 			lb++;
    220 		}
    221 		cpuno++;
    222 	}
    223 }
    224 
    225 /*
    226  * Start collecting lock statistics.
    227  */
    228 void
    229 lockstat_start(lsenable_t *le)
    230 {
    231 
    232 	KASSERT(!lockstat_enabled);
    233 
    234 	lockstat_init_tables(le);
    235 
    236 	if ((le->le_flags & LE_CALLSITE) != 0)
    237 		lockstat_csmask = (uintptr_t)-1LL;
    238 	else
    239 		lockstat_csmask = 0;
    240 
    241 	lockstat_csstart = le->le_csstart;
    242 	lockstat_csend = le->le_csend;
    243 	lockstat_lockstart = le->le_lockstart;
    244 	lockstat_lockstart = le->le_lockstart;
    245 	lockstat_lockend = le->le_lockend;
    246 	mb_memory();
    247 	getnanotime(&lockstat_stime);
    248 	lockstat_enabled = le->le_mask;
    249 	mb_write();
    250 }
    251 
    252 /*
    253  * Stop collecting lock statistics.
    254  */
    255 int
    256 lockstat_stop(lsdisable_t *ld)
    257 {
    258 	CPU_INFO_ITERATOR cii;
    259 	struct cpu_info *ci;
    260 	u_int cpuno, overflow;
    261 	struct timespec ts;
    262 	int error;
    263 
    264 	KASSERT(lockstat_enabled);
    265 
    266 	/*
    267 	 * Set enabled false, force a write barrier, and wait for other CPUs
    268 	 * to exit lockstat_event().
    269 	 */
    270 	lockstat_enabled = 0;
    271 	lockstat_unlock(0);
    272 	getnanotime(&ts);
    273 	tsleep(&lockstat_stop, PPAUSE, "lockstat", mstohz(10));
    274  	(void)lockstat_lock(0);
    275 
    276 	/*
    277 	 * Did we run out of buffers while tracing?
    278 	 */
    279 	overflow = 0;
    280 	for (CPU_INFO_FOREACH(cii, ci))
    281 		overflow += ((lscpu_t *)ci->ci_lockstat)->lc_overflow;
    282 
    283 	if (overflow != 0) {
    284 		error = EOVERFLOW;
    285 		log(LOG_NOTICE, "lockstat: %d buffer allocations failed\n",
    286 		    overflow);
    287 	} else
    288 		error = 0;
    289 
    290 	lockstat_init_tables(NULL);
    291 
    292 	if (ld == NULL)
    293 		return (error);
    294 
    295 	/*
    296 	 * Fill out the disable struct for the caller.
    297 	 */
    298 	timespecsub(&ts, &lockstat_stime, &ld->ld_time);
    299 	ld->ld_size = lockstat_sizeb;
    300 
    301 	cpuno = 0;
    302 	for (CPU_INFO_FOREACH(cii, ci)) {
    303 		if (cpuno > sizeof(ld->ld_freq) / sizeof(ld->ld_freq[0])) {
    304 			log(LOG_WARNING, "lockstat: too many CPUs\n");
    305 			break;
    306 		}
    307 		ld->ld_freq[cpuno++] = cpu_frequency(ci);
    308 	}
    309 
    310 	return (error);
    311 }
    312 
    313 /*
    314  * Allocate buffers for lockstat_start().
    315  */
    316 int
    317 lockstat_alloc(lsenable_t *le)
    318 {
    319 	lsbuf_t *lb;
    320 	size_t sz;
    321 
    322 	KASSERT(!lockstat_enabled);
    323 	lockstat_free();
    324 
    325 	sz = sizeof(*lb) * le->le_nbufs;
    326 
    327 	lockstat_unlock(0);
    328 	lb = malloc(sz, M_LOCKSTAT, M_WAITOK | M_ZERO);
    329 	(void)lockstat_lock(0);
    330 
    331 	if (lb == NULL)
    332 		return (ENOMEM);
    333 
    334 	KASSERT(!lockstat_enabled);
    335 	KASSERT(lockstat_baseb == NULL);
    336 	lockstat_sizeb = sz;
    337 	lockstat_baseb = lb;
    338 
    339 	return (0);
    340 }
    341 
    342 /*
    343  * Free allocated buffers after tracing has stopped.
    344  */
    345 void
    346 lockstat_free(void)
    347 {
    348 
    349 	KASSERT(!lockstat_enabled);
    350 
    351 	if (lockstat_baseb != NULL) {
    352 		free(lockstat_baseb, M_LOCKSTAT);
    353 		lockstat_baseb = NULL;
    354 	}
    355 }
    356 
    357 /*
    358  * Main entry point from lock primatives.
    359  */
    360 void
    361 lockstat_event(uintptr_t lock, uintptr_t callsite, u_int flags, u_int count,
    362 	       uint64_t cycles)
    363 {
    364 	lslist_t *ll;
    365 	lscpu_t *lc;
    366 	lsbuf_t *lb;
    367 	u_int event;
    368 	int s;
    369 
    370 	if ((flags & lockstat_enabled) != flags || count == 0)
    371 		return;
    372 	if (lock < lockstat_lockstart || lock > lockstat_lockend)
    373 		return;
    374 	if (callsite < lockstat_csstart || callsite > lockstat_csend)
    375 		return;
    376 
    377 	callsite &= lockstat_csmask;
    378 
    379 	/*
    380 	 * Find the table for this lock+callsite pair, and try to locate a
    381 	 * buffer with the same key.
    382 	 */
    383 	lc = curcpu()->ci_lockstat;
    384 	ll = &lc->lc_hash[LOCKSTAT_HASH(lock ^ callsite)];
    385 	event = (flags & LB_EVENT_MASK) - 1;
    386 	s = spllock();
    387 
    388 	LIST_FOREACH(lb, ll, lb_chain.list) {
    389 		if (lb->lb_lock == lock && lb->lb_callsite == callsite)
    390 			break;
    391 	}
    392 
    393 	if (lb != NULL) {
    394 		/*
    395 		 * We found a record.  Move it to the front of the list, as
    396 		 * we're likely to hit it again soon.
    397 		 */
    398 		if (lb != LIST_FIRST(ll)) {
    399 			LIST_REMOVE(lb, lb_chain.list);
    400 			LIST_INSERT_HEAD(ll, lb, lb_chain.list);
    401 		}
    402 		lb->lb_counts[event] += count;
    403 		lb->lb_times[event] += cycles;
    404 	} else if ((lb = SLIST_FIRST(&lc->lc_free)) != NULL) {
    405 		/*
    406 		 * Pinch a new buffer and fill it out.
    407 		 */
    408 		SLIST_REMOVE_HEAD(&lc->lc_free, lb_chain.slist);
    409 		LIST_INSERT_HEAD(ll, lb, lb_chain.list);
    410 		lb->lb_flags = (uint16_t)flags;
    411 		lb->lb_lock = lock;
    412 		lb->lb_callsite = callsite;
    413 		lb->lb_counts[event] = count;
    414 		lb->lb_times[event] = cycles;
    415 	} else {
    416 		/*
    417 		 * We didn't find a buffer and there were none free.
    418 		 * lockstat_stop() will notice later on and report the
    419 		 * error.
    420 		 */
    421 		 lc->lc_overflow++;
    422 	}
    423 
    424 	splx(s);
    425 }
    426 
    427 /*
    428  * Accept an open() on /dev/lockstat.
    429  */
    430 int
    431 lockstat_open(dev_t dev, int flag, int mode,
    432 	struct lwp *l)
    433 {
    434 	int error;
    435 
    436 	if ((error = lockstat_lock(1)) != 0)
    437 		return error;
    438 
    439 	if (lockstat_devopen)
    440 		error = EBUSY;
    441 	else {
    442 		lockstat_devopen = 1;
    443 		error = 0;
    444 	}
    445 
    446 	lockstat_unlock(1);
    447 
    448 	return error;
    449 }
    450 
    451 /*
    452  * Accept the last close() on /dev/lockstat.
    453  */
    454 int
    455 lockstat_close(dev_t dev, int flag, int mode,
    456 	struct lwp *l)
    457 {
    458 	int error;
    459 
    460 	if ((error = lockstat_lock(1)) == 0) {
    461 		if (lockstat_enabled)
    462 			(void)lockstat_stop(NULL);
    463 		lockstat_free();
    464 		lockstat_devopen = 0;
    465 		lockstat_unlock(1);
    466 	}
    467 
    468 	return error;
    469 }
    470 
    471 /*
    472  * Handle control operations.
    473  */
    474 int
    475 lockstat_ioctl(dev_t dev, u_long cmd, caddr_t data,
    476 	int flag, struct lwp *l)
    477 {
    478 	lsenable_t *le;
    479 	int error;
    480 
    481 	if ((error = lockstat_lock(1)) != 0)
    482 		return error;
    483 
    484 	switch (cmd) {
    485 	case IOC_LOCKSTAT_GVERSION:
    486 		*(int *)data = LS_VERSION;
    487 		error = 0;
    488 		break;
    489 
    490 	case IOC_LOCKSTAT_ENABLE:
    491 		le = (lsenable_t *)data;
    492 
    493 		if (!cpu_hascounter()) {
    494 			error = ENODEV;
    495 			break;
    496 		}
    497 		if (lockstat_enabled) {
    498 			error = EBUSY;
    499 			break;
    500 		}
    501 
    502 		/*
    503 		 * Sanitize the arguments passed in and set up filtering.
    504 		 */
    505 		if (le->le_nbufs == 0)
    506 			le->le_nbufs = LOCKSTAT_DEFBUFS;
    507 		else if (le->le_nbufs > LOCKSTAT_MAXBUFS ||
    508 		    le->le_nbufs < LOCKSTAT_MINBUFS) {
    509 			error = EINVAL;
    510 			break;
    511 		}
    512 		if ((le->le_flags & LE_ONE_CALLSITE) == 0) {
    513 			le->le_csstart = 0;
    514 			le->le_csend = le->le_csstart - 1;
    515 		}
    516 		if ((le->le_flags & LE_ONE_LOCK) == 0) {
    517 			le->le_lockstart = 0;
    518 			le->le_lockend = le->le_lockstart - 1;
    519 		}
    520 		if ((le->le_mask & LB_EVENT_MASK) == 0)
    521 			return (EINVAL);
    522 		if ((le->le_mask & LB_LOCK_MASK) == 0)
    523 			return (EINVAL);
    524 
    525 		/*
    526 		 * Start tracing.
    527 		 */
    528 		if ((error = lockstat_alloc(le)) == 0)
    529 			lockstat_start(le);
    530 		break;
    531 
    532 	case IOC_LOCKSTAT_DISABLE:
    533 		if (!lockstat_enabled)
    534 			error = EINVAL;
    535 		else
    536 			error = lockstat_stop((lsdisable_t *)data);
    537 		break;
    538 
    539 	default:
    540 		error = ENOTTY;
    541 		break;
    542 	}
    543 
    544 	lockstat_unlock(1);
    545 	return error;
    546 }
    547 
    548 /*
    549  * Copy buffers out to user-space.
    550  */
    551 int
    552 lockstat_read(dev_t dev, struct uio *uio, int flag)
    553 {
    554 	int error;
    555 
    556 	if ((error = lockstat_lock(1)) != 0)
    557 		return (error);
    558 
    559 	if (lockstat_enabled) {
    560 		lockstat_unlock(1);
    561 		return (EBUSY);
    562 	}
    563 
    564 	lockstat_unlock(0);
    565 	error = uiomove(lockstat_baseb, lockstat_sizeb, uio);
    566 	lockstat_lock(0);
    567 
    568 	lockstat_unlock(1);
    569 
    570 	return (error);
    571 }
    572