Home | History | Annotate | Line # | Download | only in dev
lockstat.c revision 1.5 
      1 /*	$NetBSD: lockstat.c,v 1.5 2006/12/25 11:57:40 ad Exp $	*/
      2 
      3 /*-
      4  * Copyright (c) 2006 The NetBSD Foundation, Inc.
      5  * All rights reserved.
      6  *
      7  * This code is derived from software contributed to The NetBSD Foundation
      8  * by Andrew Doran.
      9  *
     10  * Redistribution and use in source and binary forms, with or without
     11  * modification, are permitted provided that the following conditions
     12  * are met:
     13  * 1. Redistributions of source code must retain the above copyright
     14  *    notice, this list of conditions and the following disclaimer.
     15  * 2. Redistributions in binary form must reproduce the above copyright
     16  *    notice, this list of conditions and the following disclaimer in the
     17  *    documentation and/or other materials provided with the distribution.
     18  * 3. All advertising materials mentioning features or use of this software
     19  *    must display the following acknowledgement:
     20  *	This product includes software developed by the NetBSD
     21  *	Foundation, Inc. and its contributors.
     22  * 4. Neither the name of The NetBSD Foundation nor the names of its
     23  *    contributors may be used to endorse or promote products derived
     24  *    from this software without specific prior written permission.
     25  *
     26  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     27  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     28  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     29  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     30  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     31  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     32  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     33  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     34  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     35  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     36  * POSSIBILITY OF SUCH DAMAGE.
     37  */
     38 
     39 /*
     40  * Lock statistics driver, providing kernel support for the lockstat(8)
     41  * command.
     42  *
     43  * XXX Timings for contention on sleep locks are currently incorrect.
     44  */
     45 
     46 #include <sys/cdefs.h>
     47 __KERNEL_RCSID(0, "$NetBSD: lockstat.c,v 1.5 2006/12/25 11:57:40 ad Exp $");
     48 
     49 #include <sys/types.h>
     50 #include <sys/param.h>
     51 #include <sys/lock.h>
     52 #include <sys/proc.h>
     53 #include <sys/resourcevar.h>
     54 #include <sys/systm.h>
     55 #include <sys/kernel.h>
     56 #include <sys/malloc.h>
     57 #include <sys/conf.h>
     58 #include <sys/syslog.h>
     59 
     60 #include <dev/lockstat.h>
     61 
     62 #ifndef __HAVE_CPU_COUNTER
     63 #error CPU counters not available
     64 #endif
     65 
     66 #if LONG_BIT == 64
     67 #define	LOCKSTAT_HASH_SHIFT	3
     68 #elif LONG_BIT == 32
     69 #define	LOCKSTAT_HASH_SHIFT	2
     70 #endif
     71 
     72 #define	LOCKSTAT_MINBUFS	100
     73 #define	LOCKSTAT_DEFBUFS	1000
     74 #define	LOCKSTAT_MAXBUFS	10000
     75 
     76 #define	LOCKSTAT_HASH_SIZE	64
     77 #define	LOCKSTAT_HASH_MASK	(LOCKSTAT_HASH_SIZE - 1)
     78 #define	LOCKSTAT_HASH(key)	\
     79 	((key >> LOCKSTAT_HASH_SHIFT) & LOCKSTAT_HASH_MASK)
     80 
     81 typedef struct lscpu {
     82 	SLIST_HEAD(, lsbuf)	lc_free;
     83 	u_int			lc_overflow;
     84 	LIST_HEAD(lslist, lsbuf) lc_hash[LOCKSTAT_HASH_SIZE];
     85 } lscpu_t;
     86 
     87 typedef struct lslist lslist_t;
     88 
     89 void	lockstatattach(int);
     90 void	lockstat_start(lsenable_t *);
     91 int	lockstat_alloc(lsenable_t *);
     92 void	lockstat_init_tables(lsenable_t *);
     93 int	lockstat_stop(lsdisable_t *);
     94 void	lockstat_free(void);
     95 
     96 dev_type_open(lockstat_open);
     97 dev_type_close(lockstat_close);
     98 dev_type_read(lockstat_read);
     99 dev_type_ioctl(lockstat_ioctl);
    100 
    101 /* Protected against write by lockstat_lock().  Used by lockstat_event(). */
    102 volatile u_int	lockstat_enabled;
    103 uintptr_t	lockstat_csstart;
    104 uintptr_t	lockstat_csend;
    105 uintptr_t	lockstat_csmask;
    106 uintptr_t	lockstat_lockstart;
    107 uintptr_t	lockstat_lockend;
    108 
    109 /* Protected by lockstat_lock(). */
    110 struct simplelock lockstat_slock;
    111 lsbuf_t		*lockstat_baseb;
    112 size_t		lockstat_sizeb;
    113 int		lockstat_busy;
    114 int		lockstat_devopen;
    115 struct timespec	lockstat_stime;
    116 
    117 const struct cdevsw lockstat_cdevsw = {
    118 	lockstat_open, lockstat_close, lockstat_read, nowrite, lockstat_ioctl,
    119 	nostop, notty, nopoll, nommap, nokqfilter, 0
    120 };
    121 
    122 MALLOC_DEFINE(M_LOCKSTAT, "lockstat", "lockstat event buffers");
    123 
    124 /*
    125  * Called when the pseudo-driver is attached.
    126  */
    127 void
    128 lockstatattach(int nunits)
    129 {
    130 
    131 	(void)nunits;
    132 
    133 	__cpu_simple_lock_init(&lockstat_slock.lock_data);
    134 }
    135 
    136 /*
    137  * Grab the global lock.  If busy is set, we want to block out operations on
    138  * the control device.
    139  */
    140 static inline int
    141 lockstat_lock(int busy)
    142 {
    143 
    144 	if (!__cpu_simple_lock_try(&lockstat_slock.lock_data))
    145 		return (EBUSY);
    146 	if (busy) {
    147 		if (lockstat_busy) {
    148 			__cpu_simple_unlock(&lockstat_slock.lock_data);
    149 			return (EBUSY);
    150 		}
    151 		lockstat_busy = 1;
    152 	}
    153 	KASSERT(lockstat_busy);
    154 
    155 	return 0;
    156 }
    157 
    158 /*
    159  * Release the global lock.  If unbusy is set, we want to allow new
    160  * operations on the control device.
    161  */
    162 static inline void
    163 lockstat_unlock(int unbusy)
    164 {
    165 
    166 	KASSERT(lockstat_busy);
    167 	if (unbusy)
    168 		lockstat_busy = 0;
    169 	__cpu_simple_unlock(&lockstat_slock.lock_data);
    170 }
    171 
    172 /*
    173  * Prepare the per-CPU tables for use, or clear down tables when tracing is
    174  * stopped.
    175  */
    176 void
    177 lockstat_init_tables(lsenable_t *le)
    178 {
    179 	int i, ncpu, per, slop, cpuno;
    180 	CPU_INFO_ITERATOR cii;
    181 	struct cpu_info *ci;
    182 	lscpu_t *lc;
    183 	lsbuf_t *lb;
    184 
    185 	KASSERT(!lockstat_enabled);
    186 
    187 	ncpu = 0;
    188 	for (CPU_INFO_FOREACH(cii, ci)) {
    189 		if (ci->ci_lockstat != NULL) {
    190 			free(ci->ci_lockstat, M_LOCKSTAT);
    191 			ci->ci_lockstat = NULL;
    192 		}
    193 		ncpu++;
    194 	}
    195 
    196 	if (le == NULL)
    197 		return;
    198 
    199 	lb = lockstat_baseb;
    200 	per = le->le_nbufs / ncpu;
    201 	slop = le->le_nbufs - (per * ncpu);
    202 	cpuno = 0;
    203 	for (CPU_INFO_FOREACH(cii, ci)) {
    204 		lc = malloc(sizeof(*lc), M_LOCKSTAT, M_WAITOK);
    205 		lc->lc_overflow = 0;
    206 		ci->ci_lockstat = lc;
    207 
    208 		SLIST_INIT(&lc->lc_free);
    209 		for (i = 0; i < LOCKSTAT_HASH_SIZE; i++)
    210 			LIST_INIT(&lc->lc_hash[i]);
    211 
    212 		for (i = per; i != 0; i--, lb++) {
    213 			lb->lb_cpu = (uint16_t)cpuno;
    214 			SLIST_INSERT_HEAD(&lc->lc_free, lb, lb_chain.slist);
    215 		}
    216 		if (--slop > 0) {
    217 			lb->lb_cpu = (uint16_t)cpuno;
    218 			SLIST_INSERT_HEAD(&lc->lc_free, lb, lb_chain.slist);
    219 			lb++;
    220 		}
    221 		cpuno++;
    222 	}
    223 }
    224 
    225 /*
    226  * Start collecting lock statistics.
    227  */
    228 void
    229 lockstat_start(lsenable_t *le)
    230 {
    231 
    232 	KASSERT(!lockstat_enabled);
    233 
    234 	lockstat_init_tables(le);
    235 
    236 	if ((le->le_flags & LE_CALLSITE) != 0)
    237 		lockstat_csmask = (uintptr_t)-1LL;
    238 	else
    239 		lockstat_csmask = 0;
    240 
    241 	lockstat_csstart = le->le_csstart;
    242 	lockstat_csend = le->le_csend;
    243 	lockstat_lockstart = le->le_lockstart;
    244 	lockstat_lockend = le->le_lockend;
    245 #ifdef notyet
    246 	mb_memory();
    247 #else
    248 	lockstat_unlock(0);
    249 	(void)lockstat_lock(0);
    250 #endif
    251 
    252 	getnanotime(&lockstat_stime);
    253 	lockstat_enabled = le->le_mask;
    254 #ifdef notyet
    255 	mb_write();
    256 #else
    257 	lockstat_unlock(0);
    258 	(void)lockstat_lock(0);
    259 #endif
    260 }
    261 
    262 /*
    263  * Stop collecting lock statistics.
    264  */
    265 int
    266 lockstat_stop(lsdisable_t *ld)
    267 {
    268 	CPU_INFO_ITERATOR cii;
    269 	struct cpu_info *ci;
    270 	u_int cpuno, overflow;
    271 	struct timespec ts;
    272 	int error;
    273 
    274 	KASSERT(lockstat_enabled);
    275 
    276 	/*
    277 	 * Set enabled false, force a write barrier, and wait for other CPUs
    278 	 * to exit lockstat_event().
    279 	 */
    280 	lockstat_enabled = 0;
    281 	lockstat_unlock(0);
    282 	getnanotime(&ts);
    283 	tsleep(&lockstat_stop, PPAUSE, "lockstat", mstohz(10));
    284  	(void)lockstat_lock(0);
    285 
    286 	/*
    287 	 * Did we run out of buffers while tracing?
    288 	 */
    289 	overflow = 0;
    290 	for (CPU_INFO_FOREACH(cii, ci))
    291 		overflow += ((lscpu_t *)ci->ci_lockstat)->lc_overflow;
    292 
    293 	if (overflow != 0) {
    294 		error = EOVERFLOW;
    295 		log(LOG_NOTICE, "lockstat: %d buffer allocations failed\n",
    296 		    overflow);
    297 	} else
    298 		error = 0;
    299 
    300 	lockstat_init_tables(NULL);
    301 
    302 	if (ld == NULL)
    303 		return (error);
    304 
    305 	/*
    306 	 * Fill out the disable struct for the caller.
    307 	 */
    308 	timespecsub(&ts, &lockstat_stime, &ld->ld_time);
    309 	ld->ld_size = lockstat_sizeb;
    310 
    311 	cpuno = 0;
    312 	for (CPU_INFO_FOREACH(cii, ci)) {
    313 		if (cpuno > sizeof(ld->ld_freq) / sizeof(ld->ld_freq[0])) {
    314 			log(LOG_WARNING, "lockstat: too many CPUs\n");
    315 			break;
    316 		}
    317 		ld->ld_freq[cpuno++] = cpu_frequency(ci);
    318 	}
    319 
    320 	return (error);
    321 }
    322 
    323 /*
    324  * Allocate buffers for lockstat_start().
    325  */
    326 int
    327 lockstat_alloc(lsenable_t *le)
    328 {
    329 	lsbuf_t *lb;
    330 	size_t sz;
    331 
    332 	KASSERT(!lockstat_enabled);
    333 	lockstat_free();
    334 
    335 	sz = sizeof(*lb) * le->le_nbufs;
    336 
    337 	lockstat_unlock(0);
    338 	lb = malloc(sz, M_LOCKSTAT, M_WAITOK | M_ZERO);
    339 	(void)lockstat_lock(0);
    340 
    341 	if (lb == NULL)
    342 		return (ENOMEM);
    343 
    344 	KASSERT(!lockstat_enabled);
    345 	KASSERT(lockstat_baseb == NULL);
    346 	lockstat_sizeb = sz;
    347 	lockstat_baseb = lb;
    348 
    349 	return (0);
    350 }
    351 
    352 /*
    353  * Free allocated buffers after tracing has stopped.
    354  */
    355 void
    356 lockstat_free(void)
    357 {
    358 
    359 	KASSERT(!lockstat_enabled);
    360 
    361 	if (lockstat_baseb != NULL) {
    362 		free(lockstat_baseb, M_LOCKSTAT);
    363 		lockstat_baseb = NULL;
    364 	}
    365 }
    366 
    367 /*
    368  * Main entry point from lock primatives.
    369  */
    370 void
    371 lockstat_event(uintptr_t lock, uintptr_t callsite, u_int flags, u_int count,
    372 	       uint64_t time)
    373 {
    374 	lslist_t *ll;
    375 	lscpu_t *lc;
    376 	lsbuf_t *lb;
    377 	u_int event;
    378 	int s;
    379 
    380 	if ((flags & lockstat_enabled) != flags || count == 0)
    381 		return;
    382 	if (lock < lockstat_lockstart || lock > lockstat_lockend)
    383 		return;
    384 	if (callsite < lockstat_csstart || callsite > lockstat_csend)
    385 		return;
    386 
    387 	callsite &= lockstat_csmask;
    388 
    389 	/*
    390 	 * Find the table for this lock+callsite pair, and try to locate a
    391 	 * buffer with the same key.
    392 	 */
    393 	lc = curcpu()->ci_lockstat;
    394 	ll = &lc->lc_hash[LOCKSTAT_HASH(lock ^ callsite)];
    395 	event = (flags & LB_EVENT_MASK) - 1;
    396 	s = spllock();
    397 
    398 	LIST_FOREACH(lb, ll, lb_chain.list) {
    399 		if (lb->lb_lock == lock && lb->lb_callsite == callsite)
    400 			break;
    401 	}
    402 
    403 	if (lb != NULL) {
    404 		/*
    405 		 * We found a record.  Move it to the front of the list, as
    406 		 * we're likely to hit it again soon.
    407 		 */
    408 		if (lb != LIST_FIRST(ll)) {
    409 			LIST_REMOVE(lb, lb_chain.list);
    410 			LIST_INSERT_HEAD(ll, lb, lb_chain.list);
    411 		}
    412 		lb->lb_counts[event] += count;
    413 		lb->lb_times[event] += time;
    414 	} else if ((lb = SLIST_FIRST(&lc->lc_free)) != NULL) {
    415 		/*
    416 		 * Pinch a new buffer and fill it out.
    417 		 */
    418 		SLIST_REMOVE_HEAD(&lc->lc_free, lb_chain.slist);
    419 		LIST_INSERT_HEAD(ll, lb, lb_chain.list);
    420 		lb->lb_flags = (uint16_t)flags;
    421 		lb->lb_lock = lock;
    422 		lb->lb_callsite = callsite;
    423 		lb->lb_counts[event] = count;
    424 		lb->lb_times[event] = time;
    425 	} else {
    426 		/*
    427 		 * We didn't find a buffer and there were none free.
    428 		 * lockstat_stop() will notice later on and report the
    429 		 * error.
    430 		 */
    431 		 lc->lc_overflow++;
    432 	}
    433 
    434 	splx(s);
    435 }
    436 
    437 /*
    438  * Accept an open() on /dev/lockstat.
    439  */
    440 int
    441 lockstat_open(dev_t dev, int flag, int mode,
    442 	struct lwp *l)
    443 {
    444 	int error;
    445 
    446 	if ((error = lockstat_lock(1)) != 0)
    447 		return error;
    448 
    449 	if (lockstat_devopen)
    450 		error = EBUSY;
    451 	else {
    452 		lockstat_devopen = 1;
    453 		error = 0;
    454 	}
    455 
    456 	lockstat_unlock(1);
    457 
    458 	return error;
    459 }
    460 
    461 /*
    462  * Accept the last close() on /dev/lockstat.
    463  */
    464 int
    465 lockstat_close(dev_t dev, int flag, int mode,
    466 	struct lwp *l)
    467 {
    468 	int error;
    469 
    470 	if ((error = lockstat_lock(1)) == 0) {
    471 		if (lockstat_enabled)
    472 			(void)lockstat_stop(NULL);
    473 		lockstat_free();
    474 		lockstat_devopen = 0;
    475 		lockstat_unlock(1);
    476 	}
    477 
    478 	return error;
    479 }
    480 
    481 /*
    482  * Handle control operations.
    483  */
    484 int
    485 lockstat_ioctl(dev_t dev, u_long cmd, caddr_t data,
    486 	int flag, struct lwp *l)
    487 {
    488 	lsenable_t *le;
    489 	int error;
    490 
    491 	if ((error = lockstat_lock(1)) != 0)
    492 		return error;
    493 
    494 	switch (cmd) {
    495 	case IOC_LOCKSTAT_GVERSION:
    496 		*(int *)data = LS_VERSION;
    497 		error = 0;
    498 		break;
    499 
    500 	case IOC_LOCKSTAT_ENABLE:
    501 		le = (lsenable_t *)data;
    502 
    503 		if (!cpu_hascounter()) {
    504 			error = ENODEV;
    505 			break;
    506 		}
    507 		if (lockstat_enabled) {
    508 			error = EBUSY;
    509 			break;
    510 		}
    511 
    512 		/*
    513 		 * Sanitize the arguments passed in and set up filtering.
    514 		 */
    515 		if (le->le_nbufs == 0)
    516 			le->le_nbufs = LOCKSTAT_DEFBUFS;
    517 		else if (le->le_nbufs > LOCKSTAT_MAXBUFS ||
    518 		    le->le_nbufs < LOCKSTAT_MINBUFS) {
    519 			error = EINVAL;
    520 			break;
    521 		}
    522 		if ((le->le_flags & LE_ONE_CALLSITE) == 0) {
    523 			le->le_csstart = 0;
    524 			le->le_csend = le->le_csstart - 1;
    525 		}
    526 		if ((le->le_flags & LE_ONE_LOCK) == 0) {
    527 			le->le_lockstart = 0;
    528 			le->le_lockend = le->le_lockstart - 1;
    529 		}
    530 		if ((le->le_mask & LB_EVENT_MASK) == 0)
    531 			return (EINVAL);
    532 		if ((le->le_mask & LB_LOCK_MASK) == 0)
    533 			return (EINVAL);
    534 
    535 		/*
    536 		 * Start tracing.
    537 		 */
    538 		if ((error = lockstat_alloc(le)) == 0)
    539 			lockstat_start(le);
    540 		break;
    541 
    542 	case IOC_LOCKSTAT_DISABLE:
    543 		if (!lockstat_enabled)
    544 			error = EINVAL;
    545 		else
    546 			error = lockstat_stop((lsdisable_t *)data);
    547 		break;
    548 
    549 	default:
    550 		error = ENOTTY;
    551 		break;
    552 	}
    553 
    554 	lockstat_unlock(1);
    555 	return error;
    556 }
    557 
    558 /*
    559  * Copy buffers out to user-space.
    560  */
    561 int
    562 lockstat_read(dev_t dev, struct uio *uio, int flag)
    563 {
    564 	int error;
    565 
    566 	if ((error = lockstat_lock(1)) != 0)
    567 		return (error);
    568 
    569 	if (lockstat_enabled) {
    570 		lockstat_unlock(1);
    571 		return (EBUSY);
    572 	}
    573 
    574 	lockstat_unlock(0);
    575 	error = uiomove(lockstat_baseb, lockstat_sizeb, uio);
    576 	lockstat_lock(0);
    577 
    578 	lockstat_unlock(1);
    579 
    580 	return (error);
    581 }
    582