Home | History | Annotate | Line # | Download | only in dev
lockstat.c revision 1.7 
      1 /*	$NetBSD: lockstat.c,v 1.7 2007/02/15 20:32:47 ad Exp $	*/
      2 
      3 /*-
      4  * Copyright (c) 2006 The NetBSD Foundation, Inc.
      5  * All rights reserved.
      6  *
      7  * This code is derived from software contributed to The NetBSD Foundation
      8  * by Andrew Doran.
      9  *
     10  * Redistribution and use in source and binary forms, with or without
     11  * modification, are permitted provided that the following conditions
     12  * are met:
     13  * 1. Redistributions of source code must retain the above copyright
     14  *    notice, this list of conditions and the following disclaimer.
     15  * 2. Redistributions in binary form must reproduce the above copyright
     16  *    notice, this list of conditions and the following disclaimer in the
     17  *    documentation and/or other materials provided with the distribution.
     18  * 3. All advertising materials mentioning features or use of this software
     19  *    must display the following acknowledgement:
     20  *	This product includes software developed by the NetBSD
     21  *	Foundation, Inc. and its contributors.
     22  * 4. Neither the name of The NetBSD Foundation nor the names of its
     23  *    contributors may be used to endorse or promote products derived
     24  *    from this software without specific prior written permission.
     25  *
     26  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     27  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     28  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     29  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     30  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     31  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     32  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     33  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     34  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     35  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     36  * POSSIBILITY OF SUCH DAMAGE.
     37  */
     38 
     39 /*
     40  * Lock statistics driver, providing kernel support for the lockstat(8)
     41  * command.
     42  *
     43  * XXX Timings for contention on sleep locks are currently incorrect.
     44  */
     45 
     46 #include <sys/cdefs.h>
     47 __KERNEL_RCSID(0, "$NetBSD: lockstat.c,v 1.7 2007/02/15 20:32:47 ad Exp $");
     48 
     49 #include <sys/types.h>
     50 #include <sys/param.h>
     51 #include <sys/lock.h>
     52 #include <sys/proc.h>
     53 #include <sys/resourcevar.h>
     54 #include <sys/systm.h>
     55 #include <sys/kernel.h>
     56 #include <sys/malloc.h>
     57 #include <sys/conf.h>
     58 #include <sys/syslog.h>
     59 
     60 #include <dev/lockstat.h>
     61 
     62 #ifndef __HAVE_CPU_COUNTER
     63 #error CPU counters not available
     64 #endif
     65 
     66 #if LONG_BIT == 64
     67 #define	LOCKSTAT_HASH_SHIFT	3
     68 #elif LONG_BIT == 32
     69 #define	LOCKSTAT_HASH_SHIFT	2
     70 #endif
     71 
     72 #define	LOCKSTAT_MINBUFS	100
     73 #define	LOCKSTAT_DEFBUFS	1000
     74 #define	LOCKSTAT_MAXBUFS	10000
     75 
     76 #define	LOCKSTAT_HASH_SIZE	64
     77 #define	LOCKSTAT_HASH_MASK	(LOCKSTAT_HASH_SIZE - 1)
     78 #define	LOCKSTAT_HASH(key)	\
     79 	((key >> LOCKSTAT_HASH_SHIFT) & LOCKSTAT_HASH_MASK)
     80 
     81 typedef struct lscpu {
     82 	SLIST_HEAD(, lsbuf)	lc_free;
     83 	u_int			lc_overflow;
     84 	LIST_HEAD(lslist, lsbuf) lc_hash[LOCKSTAT_HASH_SIZE];
     85 } lscpu_t;
     86 
     87 typedef struct lslist lslist_t;
     88 
     89 void	lockstatattach(int);
     90 void	lockstat_start(lsenable_t *);
     91 int	lockstat_alloc(lsenable_t *);
     92 void	lockstat_init_tables(lsenable_t *);
     93 int	lockstat_stop(lsdisable_t *);
     94 void	lockstat_free(void);
     95 
     96 dev_type_open(lockstat_open);
     97 dev_type_close(lockstat_close);
     98 dev_type_read(lockstat_read);
     99 dev_type_ioctl(lockstat_ioctl);
    100 
    101 /* Protected against write by lockstat_lock().  Used by lockstat_event(). */
    102 volatile u_int	lockstat_enabled;
    103 uintptr_t	lockstat_csstart;
    104 uintptr_t	lockstat_csend;
    105 uintptr_t	lockstat_csmask;
    106 uintptr_t	lockstat_lockstart;
    107 uintptr_t	lockstat_lockend;
    108 
    109 /* Protected by lockstat_lock(). */
    110 struct simplelock lockstat_slock;
    111 lsbuf_t		*lockstat_baseb;
    112 size_t		lockstat_sizeb;
    113 int		lockstat_busy;
    114 int		lockstat_devopen;
    115 struct timespec	lockstat_stime;
    116 
    117 const struct cdevsw lockstat_cdevsw = {
    118 	lockstat_open, lockstat_close, lockstat_read, nowrite, lockstat_ioctl,
    119 	nostop, notty, nopoll, nommap, nokqfilter, 0
    120 };
    121 
    122 MALLOC_DEFINE(M_LOCKSTAT, "lockstat", "lockstat event buffers");
    123 
    124 /*
    125  * Called when the pseudo-driver is attached.
    126  */
    127 void
    128 lockstatattach(int nunits)
    129 {
    130 
    131 	(void)nunits;
    132 
    133 	__cpu_simple_lock_init(&lockstat_slock.lock_data);
    134 }
    135 
    136 /*
    137  * Grab the global lock.  If busy is set, we want to block out operations on
    138  * the control device.
    139  */
    140 static inline int
    141 lockstat_lock(int busy)
    142 {
    143 
    144 	if (!__cpu_simple_lock_try(&lockstat_slock.lock_data))
    145 		return (EBUSY);
    146 	if (busy) {
    147 		if (lockstat_busy) {
    148 			__cpu_simple_unlock(&lockstat_slock.lock_data);
    149 			return (EBUSY);
    150 		}
    151 		lockstat_busy = 1;
    152 	}
    153 	KASSERT(lockstat_busy);
    154 
    155 	return 0;
    156 }
    157 
    158 /*
    159  * Release the global lock.  If unbusy is set, we want to allow new
    160  * operations on the control device.
    161  */
    162 static inline void
    163 lockstat_unlock(int unbusy)
    164 {
    165 
    166 	KASSERT(lockstat_busy);
    167 	if (unbusy)
    168 		lockstat_busy = 0;
    169 	__cpu_simple_unlock(&lockstat_slock.lock_data);
    170 }
    171 
    172 /*
    173  * Prepare the per-CPU tables for use, or clear down tables when tracing is
    174  * stopped.
    175  */
    176 void
    177 lockstat_init_tables(lsenable_t *le)
    178 {
    179 	int i, per, slop, cpuno;
    180 	CPU_INFO_ITERATOR cii;
    181 	struct cpu_info *ci;
    182 	lscpu_t *lc;
    183 	lsbuf_t *lb;
    184 
    185 	KASSERT(!lockstat_enabled);
    186 
    187 	for (CPU_INFO_FOREACH(cii, ci)) {
    188 		if (ci->ci_lockstat != NULL) {
    189 			free(ci->ci_lockstat, M_LOCKSTAT);
    190 			ci->ci_lockstat = NULL;
    191 		}
    192 	}
    193 
    194 	if (le == NULL)
    195 		return;
    196 
    197 	lb = lockstat_baseb;
    198 	per = le->le_nbufs / ncpu;
    199 	slop = le->le_nbufs - (per * ncpu);
    200 	cpuno = 0;
    201 	for (CPU_INFO_FOREACH(cii, ci)) {
    202 		lc = malloc(sizeof(*lc), M_LOCKSTAT, M_WAITOK);
    203 		lc->lc_overflow = 0;
    204 		ci->ci_lockstat = lc;
    205 
    206 		SLIST_INIT(&lc->lc_free);
    207 		for (i = 0; i < LOCKSTAT_HASH_SIZE; i++)
    208 			LIST_INIT(&lc->lc_hash[i]);
    209 
    210 		for (i = per; i != 0; i--, lb++) {
    211 			lb->lb_cpu = (uint16_t)cpuno;
    212 			SLIST_INSERT_HEAD(&lc->lc_free, lb, lb_chain.slist);
    213 		}
    214 		if (--slop > 0) {
    215 			lb->lb_cpu = (uint16_t)cpuno;
    216 			SLIST_INSERT_HEAD(&lc->lc_free, lb, lb_chain.slist);
    217 			lb++;
    218 		}
    219 		cpuno++;
    220 	}
    221 }
    222 
    223 /*
    224  * Start collecting lock statistics.
    225  */
    226 void
    227 lockstat_start(lsenable_t *le)
    228 {
    229 
    230 	KASSERT(!lockstat_enabled);
    231 
    232 	lockstat_init_tables(le);
    233 
    234 	if ((le->le_flags & LE_CALLSITE) != 0)
    235 		lockstat_csmask = (uintptr_t)-1LL;
    236 	else
    237 		lockstat_csmask = 0;
    238 
    239 	lockstat_csstart = le->le_csstart;
    240 	lockstat_csend = le->le_csend;
    241 	lockstat_lockstart = le->le_lockstart;
    242 	lockstat_lockstart = le->le_lockstart;
    243 	lockstat_lockend = le->le_lockend;
    244 	mb_memory();
    245 	getnanotime(&lockstat_stime);
    246 	lockstat_enabled = le->le_mask;
    247 	mb_write();
    248 }
    249 
    250 /*
    251  * Stop collecting lock statistics.
    252  */
    253 int
    254 lockstat_stop(lsdisable_t *ld)
    255 {
    256 	CPU_INFO_ITERATOR cii;
    257 	struct cpu_info *ci;
    258 	u_int cpuno, overflow;
    259 	struct timespec ts;
    260 	int error;
    261 
    262 	KASSERT(lockstat_enabled);
    263 
    264 	/*
    265 	 * Set enabled false, force a write barrier, and wait for other CPUs
    266 	 * to exit lockstat_event().
    267 	 */
    268 	lockstat_enabled = 0;
    269 	lockstat_unlock(0);
    270 	getnanotime(&ts);
    271 	tsleep(&lockstat_stop, PPAUSE, "lockstat", mstohz(10));
    272  	(void)lockstat_lock(0);
    273 
    274 	/*
    275 	 * Did we run out of buffers while tracing?
    276 	 */
    277 	overflow = 0;
    278 	for (CPU_INFO_FOREACH(cii, ci))
    279 		overflow += ((lscpu_t *)ci->ci_lockstat)->lc_overflow;
    280 
    281 	if (overflow != 0) {
    282 		error = EOVERFLOW;
    283 		log(LOG_NOTICE, "lockstat: %d buffer allocations failed\n",
    284 		    overflow);
    285 	} else
    286 		error = 0;
    287 
    288 	lockstat_init_tables(NULL);
    289 
    290 	if (ld == NULL)
    291 		return (error);
    292 
    293 	/*
    294 	 * Fill out the disable struct for the caller.
    295 	 */
    296 	timespecsub(&ts, &lockstat_stime, &ld->ld_time);
    297 	ld->ld_size = lockstat_sizeb;
    298 
    299 	cpuno = 0;
    300 	for (CPU_INFO_FOREACH(cii, ci)) {
    301 		if (cpuno > sizeof(ld->ld_freq) / sizeof(ld->ld_freq[0])) {
    302 			log(LOG_WARNING, "lockstat: too many CPUs\n");
    303 			break;
    304 		}
    305 		ld->ld_freq[cpuno++] = cpu_frequency(ci);
    306 	}
    307 
    308 	return (error);
    309 }
    310 
    311 /*
    312  * Allocate buffers for lockstat_start().
    313  */
    314 int
    315 lockstat_alloc(lsenable_t *le)
    316 {
    317 	lsbuf_t *lb;
    318 	size_t sz;
    319 
    320 	KASSERT(!lockstat_enabled);
    321 	lockstat_free();
    322 
    323 	sz = sizeof(*lb) * le->le_nbufs;
    324 
    325 	lockstat_unlock(0);
    326 	lb = malloc(sz, M_LOCKSTAT, M_WAITOK | M_ZERO);
    327 	(void)lockstat_lock(0);
    328 
    329 	if (lb == NULL)
    330 		return (ENOMEM);
    331 
    332 	KASSERT(!lockstat_enabled);
    333 	KASSERT(lockstat_baseb == NULL);
    334 	lockstat_sizeb = sz;
    335 	lockstat_baseb = lb;
    336 
    337 	return (0);
    338 }
    339 
    340 /*
    341  * Free allocated buffers after tracing has stopped.
    342  */
    343 void
    344 lockstat_free(void)
    345 {
    346 
    347 	KASSERT(!lockstat_enabled);
    348 
    349 	if (lockstat_baseb != NULL) {
    350 		free(lockstat_baseb, M_LOCKSTAT);
    351 		lockstat_baseb = NULL;
    352 	}
    353 }
    354 
    355 /*
    356  * Main entry point from lock primatives.
    357  */
    358 void
    359 lockstat_event(uintptr_t lock, uintptr_t callsite, u_int flags, u_int count,
    360 	       uint64_t cycles)
    361 {
    362 	lslist_t *ll;
    363 	lscpu_t *lc;
    364 	lsbuf_t *lb;
    365 	u_int event;
    366 	int s;
    367 
    368 	if ((flags & lockstat_enabled) != flags || count == 0)
    369 		return;
    370 	if (lock < lockstat_lockstart || lock > lockstat_lockend)
    371 		return;
    372 	if (callsite < lockstat_csstart || callsite > lockstat_csend)
    373 		return;
    374 
    375 	callsite &= lockstat_csmask;
    376 
    377 	/*
    378 	 * Find the table for this lock+callsite pair, and try to locate a
    379 	 * buffer with the same key.
    380 	 */
    381 	lc = curcpu()->ci_lockstat;
    382 	ll = &lc->lc_hash[LOCKSTAT_HASH(lock ^ callsite)];
    383 	event = (flags & LB_EVENT_MASK) - 1;
    384 	s = spllock();
    385 
    386 	LIST_FOREACH(lb, ll, lb_chain.list) {
    387 		if (lb->lb_lock == lock && lb->lb_callsite == callsite)
    388 			break;
    389 	}
    390 
    391 	if (lb != NULL) {
    392 		/*
    393 		 * We found a record.  Move it to the front of the list, as
    394 		 * we're likely to hit it again soon.
    395 		 */
    396 		if (lb != LIST_FIRST(ll)) {
    397 			LIST_REMOVE(lb, lb_chain.list);
    398 			LIST_INSERT_HEAD(ll, lb, lb_chain.list);
    399 		}
    400 		lb->lb_counts[event] += count;
    401 		lb->lb_times[event] += cycles;
    402 	} else if ((lb = SLIST_FIRST(&lc->lc_free)) != NULL) {
    403 		/*
    404 		 * Pinch a new buffer and fill it out.
    405 		 */
    406 		SLIST_REMOVE_HEAD(&lc->lc_free, lb_chain.slist);
    407 		LIST_INSERT_HEAD(ll, lb, lb_chain.list);
    408 		lb->lb_flags = (uint16_t)flags;
    409 		lb->lb_lock = lock;
    410 		lb->lb_callsite = callsite;
    411 		lb->lb_counts[event] = count;
    412 		lb->lb_times[event] = cycles;
    413 	} else {
    414 		/*
    415 		 * We didn't find a buffer and there were none free.
    416 		 * lockstat_stop() will notice later on and report the
    417 		 * error.
    418 		 */
    419 		 lc->lc_overflow++;
    420 	}
    421 
    422 	splx(s);
    423 }
    424 
    425 /*
    426  * Accept an open() on /dev/lockstat.
    427  */
    428 int
    429 lockstat_open(dev_t dev, int flag, int mode,
    430 	struct lwp *l)
    431 {
    432 	int error;
    433 
    434 	if ((error = lockstat_lock(1)) != 0)
    435 		return error;
    436 
    437 	if (lockstat_devopen)
    438 		error = EBUSY;
    439 	else {
    440 		lockstat_devopen = 1;
    441 		error = 0;
    442 	}
    443 
    444 	lockstat_unlock(1);
    445 
    446 	return error;
    447 }
    448 
    449 /*
    450  * Accept the last close() on /dev/lockstat.
    451  */
    452 int
    453 lockstat_close(dev_t dev, int flag, int mode,
    454 	struct lwp *l)
    455 {
    456 	int error;
    457 
    458 	if ((error = lockstat_lock(1)) == 0) {
    459 		if (lockstat_enabled)
    460 			(void)lockstat_stop(NULL);
    461 		lockstat_free();
    462 		lockstat_devopen = 0;
    463 		lockstat_unlock(1);
    464 	}
    465 
    466 	return error;
    467 }
    468 
    469 /*
    470  * Handle control operations.
    471  */
    472 int
    473 lockstat_ioctl(dev_t dev, u_long cmd, caddr_t data,
    474 	int flag, struct lwp *l)
    475 {
    476 	lsenable_t *le;
    477 	int error;
    478 
    479 	if ((error = lockstat_lock(1)) != 0)
    480 		return error;
    481 
    482 	switch (cmd) {
    483 	case IOC_LOCKSTAT_GVERSION:
    484 		*(int *)data = LS_VERSION;
    485 		error = 0;
    486 		break;
    487 
    488 	case IOC_LOCKSTAT_ENABLE:
    489 		le = (lsenable_t *)data;
    490 
    491 		if (!cpu_hascounter()) {
    492 			error = ENODEV;
    493 			break;
    494 		}
    495 		if (lockstat_enabled) {
    496 			error = EBUSY;
    497 			break;
    498 		}
    499 
    500 		/*
    501 		 * Sanitize the arguments passed in and set up filtering.
    502 		 */
    503 		if (le->le_nbufs == 0)
    504 			le->le_nbufs = LOCKSTAT_DEFBUFS;
    505 		else if (le->le_nbufs > LOCKSTAT_MAXBUFS ||
    506 		    le->le_nbufs < LOCKSTAT_MINBUFS) {
    507 			error = EINVAL;
    508 			break;
    509 		}
    510 		if ((le->le_flags & LE_ONE_CALLSITE) == 0) {
    511 			le->le_csstart = 0;
    512 			le->le_csend = le->le_csstart - 1;
    513 		}
    514 		if ((le->le_flags & LE_ONE_LOCK) == 0) {
    515 			le->le_lockstart = 0;
    516 			le->le_lockend = le->le_lockstart - 1;
    517 		}
    518 		if ((le->le_mask & LB_EVENT_MASK) == 0)
    519 			return (EINVAL);
    520 		if ((le->le_mask & LB_LOCK_MASK) == 0)
    521 			return (EINVAL);
    522 
    523 		/*
    524 		 * Start tracing.
    525 		 */
    526 		if ((error = lockstat_alloc(le)) == 0)
    527 			lockstat_start(le);
    528 		break;
    529 
    530 	case IOC_LOCKSTAT_DISABLE:
    531 		if (!lockstat_enabled)
    532 			error = EINVAL;
    533 		else
    534 			error = lockstat_stop((lsdisable_t *)data);
    535 		break;
    536 
    537 	default:
    538 		error = ENOTTY;
    539 		break;
    540 	}
    541 
    542 	lockstat_unlock(1);
    543 	return error;
    544 }
    545 
    546 /*
    547  * Copy buffers out to user-space.
    548  */
    549 int
    550 lockstat_read(dev_t dev, struct uio *uio, int flag)
    551 {
    552 	int error;
    553 
    554 	if ((error = lockstat_lock(1)) != 0)
    555 		return (error);
    556 
    557 	if (lockstat_enabled) {
    558 		lockstat_unlock(1);
    559 		return (EBUSY);
    560 	}
    561 
    562 	lockstat_unlock(0);
    563 	error = uiomove(lockstat_baseb, lockstat_sizeb, uio);
    564 	lockstat_lock(0);
    565 
    566 	lockstat_unlock(1);
    567 
    568 	return (error);
    569 }
    570