Home | History | Annotate | Line # | Download | only in dev
lockstat.c revision 1.24 
      1 /*	$NetBSD: lockstat.c,v 1.24 2015/08/20 14:40:17 christos Exp $	*/
      2 
      3 /*-
      4  * Copyright (c) 2006, 2007 The NetBSD Foundation, Inc.
      5  * All rights reserved.
      6  *
      7  * This code is derived from software contributed to The NetBSD Foundation
      8  * by Andrew Doran.
      9  *
     10  * Redistribution and use in source and binary forms, with or without
     11  * modification, are permitted provided that the following conditions
     12  * are met:
     13  * 1. Redistributions of source code must retain the above copyright
     14  *    notice, this list of conditions and the following disclaimer.
     15  * 2. Redistributions in binary form must reproduce the above copyright
     16  *    notice, this list of conditions and the following disclaimer in the
     17  *    documentation and/or other materials provided with the distribution.
     18  *
     19  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     21  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     22  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     23  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     29  * POSSIBILITY OF SUCH DAMAGE.
     30  */
     31 
     32 /*
     33  * Lock statistics driver, providing kernel support for the lockstat(8)
     34  * command.
     35  *
     36  * We use a global lock word (lockstat_lock) to track device opens.
     37  * Only one thread can hold the device at a time, providing a global lock.
     38  *
     39  * XXX Timings for contention on sleep locks are currently incorrect.
     40  */
     41 
     42 #include <sys/cdefs.h>
     43 __KERNEL_RCSID(0, "$NetBSD: lockstat.c,v 1.24 2015/08/20 14:40:17 christos Exp $");
     44 
     45 #include <sys/types.h>
     46 #include <sys/param.h>
     47 #include <sys/proc.h>
     48 #include <sys/resourcevar.h>
     49 #include <sys/systm.h>
     50 #include <sys/kernel.h>
     51 #include <sys/kmem.h>
     52 #include <sys/conf.h>
     53 #include <sys/cpu.h>
     54 #include <sys/syslog.h>
     55 #include <sys/atomic.h>
     56 
     57 #include <dev/lockstat.h>
     58 
     59 #include <machine/lock.h>
     60 
     61 #include "ioconf.h"
     62 
     63 #ifndef __HAVE_CPU_COUNTER
     64 #error CPU counters not available
     65 #endif
     66 
     67 #if LONG_BIT == 64
     68 #define	LOCKSTAT_HASH_SHIFT	3
     69 #elif LONG_BIT == 32
     70 #define	LOCKSTAT_HASH_SHIFT	2
     71 #endif
     72 
     73 #define	LOCKSTAT_MINBUFS	1000
     74 #define	LOCKSTAT_DEFBUFS	10000
     75 #define	LOCKSTAT_MAXBUFS	1000000
     76 
     77 #define	LOCKSTAT_HASH_SIZE	128
     78 #define	LOCKSTAT_HASH_MASK	(LOCKSTAT_HASH_SIZE - 1)
     79 #define	LOCKSTAT_HASH(key)	\
     80 	((key >> LOCKSTAT_HASH_SHIFT) & LOCKSTAT_HASH_MASK)
     81 
     82 typedef struct lscpu {
     83 	SLIST_HEAD(, lsbuf)	lc_free;
     84 	u_int			lc_overflow;
     85 	LIST_HEAD(lslist, lsbuf) lc_hash[LOCKSTAT_HASH_SIZE];
     86 } lscpu_t;
     87 
     88 typedef struct lslist lslist_t;
     89 
     90 void	lockstat_start(lsenable_t *);
     91 int	lockstat_alloc(lsenable_t *);
     92 void	lockstat_init_tables(lsenable_t *);
     93 int	lockstat_stop(lsdisable_t *);
     94 void	lockstat_free(void);
     95 
     96 dev_type_open(lockstat_open);
     97 dev_type_close(lockstat_close);
     98 dev_type_read(lockstat_read);
     99 dev_type_ioctl(lockstat_ioctl);
    100 
    101 volatile u_int	lockstat_enabled;
    102 volatile u_int	lockstat_dev_enabled;
    103 uintptr_t	lockstat_csstart;
    104 uintptr_t	lockstat_csend;
    105 uintptr_t	lockstat_csmask;
    106 uintptr_t	lockstat_lamask;
    107 uintptr_t	lockstat_lockstart;
    108 uintptr_t	lockstat_lockend;
    109 __cpu_simple_lock_t lockstat_lock;
    110 lwp_t		*lockstat_lwp;
    111 lsbuf_t		*lockstat_baseb;
    112 size_t		lockstat_sizeb;
    113 int		lockstat_busy;
    114 struct timespec	lockstat_stime;
    115 
    116 #ifdef KDTRACE_HOOKS
    117 volatile u_int lockstat_dtrace_enabled;
    118 CTASSERT(LB_NEVENT <= 3);
    119 CTASSERT(LB_NLOCK <= (7 << LB_LOCK_SHIFT));
    120 void
    121 lockstat_probe_stub(uint32_t id, uintptr_t lock, uintptr_t callsite,
    122     uintptr_t flags, uintptr_t count, uintptr_t cycles)
    123 {
    124 }
    125 
    126 uint32_t	lockstat_probemap[LS_NPROBES];
    127 void		(*lockstat_probe_func)(uint32_t, uintptr_t, uintptr_t,
    128 		    uintptr_t, uintptr_t, uintptr_t) = &lockstat_probe_stub;
    129 #endif
    130 
    131 const struct cdevsw lockstat_cdevsw = {
    132 	.d_open = lockstat_open,
    133 	.d_close = lockstat_close,
    134 	.d_read = lockstat_read,
    135 	.d_write = nowrite,
    136 	.d_ioctl = lockstat_ioctl,
    137 	.d_stop = nostop,
    138 	.d_tty = notty,
    139 	.d_poll = nopoll,
    140 	.d_mmap = nommap,
    141 	.d_kqfilter = nokqfilter,
    142 	.d_discard = nodiscard,
    143 	.d_flag = D_OTHER | D_MPSAFE
    144 };
    145 
    146 /*
    147  * Called when the pseudo-driver is attached.
    148  */
    149 void
    150 lockstatattach(int nunits)
    151 {
    152 
    153 	(void)nunits;
    154 
    155 	__cpu_simple_lock_init(&lockstat_lock);
    156 }
    157 
    158 /*
    159  * Prepare the per-CPU tables for use, or clear down tables when tracing is
    160  * stopped.
    161  */
    162 void
    163 lockstat_init_tables(lsenable_t *le)
    164 {
    165 	int i, per, slop, cpuno;
    166 	CPU_INFO_ITERATOR cii;
    167 	struct cpu_info *ci;
    168 	lscpu_t *lc;
    169 	lsbuf_t *lb;
    170 
    171 	/* coverity[assert_side_effect] */
    172 	KASSERT(!lockstat_dev_enabled);
    173 
    174 	for (CPU_INFO_FOREACH(cii, ci)) {
    175 		if (ci->ci_lockstat != NULL) {
    176 			kmem_free(ci->ci_lockstat, sizeof(lscpu_t));
    177 			ci->ci_lockstat = NULL;
    178 		}
    179 	}
    180 
    181 	if (le == NULL)
    182 		return;
    183 
    184 	lb = lockstat_baseb;
    185 	per = le->le_nbufs / ncpu;
    186 	slop = le->le_nbufs - (per * ncpu);
    187 	cpuno = 0;
    188 	for (CPU_INFO_FOREACH(cii, ci)) {
    189 		lc = kmem_alloc(sizeof(*lc), KM_SLEEP);
    190 		lc->lc_overflow = 0;
    191 		ci->ci_lockstat = lc;
    192 
    193 		SLIST_INIT(&lc->lc_free);
    194 		for (i = 0; i < LOCKSTAT_HASH_SIZE; i++)
    195 			LIST_INIT(&lc->lc_hash[i]);
    196 
    197 		for (i = per; i != 0; i--, lb++) {
    198 			lb->lb_cpu = (uint16_t)cpuno;
    199 			SLIST_INSERT_HEAD(&lc->lc_free, lb, lb_chain.slist);
    200 		}
    201 		if (--slop > 0) {
    202 			lb->lb_cpu = (uint16_t)cpuno;
    203 			SLIST_INSERT_HEAD(&lc->lc_free, lb, lb_chain.slist);
    204 			lb++;
    205 		}
    206 		cpuno++;
    207 	}
    208 }
    209 
    210 /*
    211  * Start collecting lock statistics.
    212  */
    213 void
    214 lockstat_start(lsenable_t *le)
    215 {
    216 
    217 	/* coverity[assert_side_effect] */
    218 	KASSERT(!lockstat_dev_enabled);
    219 
    220 	lockstat_init_tables(le);
    221 
    222 	if ((le->le_flags & LE_CALLSITE) != 0)
    223 		lockstat_csmask = (uintptr_t)-1LL;
    224 	else
    225 		lockstat_csmask = 0;
    226 
    227 	if ((le->le_flags & LE_LOCK) != 0)
    228 		lockstat_lamask = (uintptr_t)-1LL;
    229 	else
    230 		lockstat_lamask = 0;
    231 
    232 	lockstat_csstart = le->le_csstart;
    233 	lockstat_csend = le->le_csend;
    234 	lockstat_lockstart = le->le_lockstart;
    235 	lockstat_lockstart = le->le_lockstart;
    236 	lockstat_lockend = le->le_lockend;
    237 	membar_sync();
    238 	getnanotime(&lockstat_stime);
    239 	lockstat_dev_enabled = le->le_mask;
    240 	LOCKSTAT_ENABLED_UPDATE();
    241 }
    242 
    243 /*
    244  * Stop collecting lock statistics.
    245  */
    246 int
    247 lockstat_stop(lsdisable_t *ld)
    248 {
    249 	CPU_INFO_ITERATOR cii;
    250 	struct cpu_info *ci;
    251 	u_int cpuno, overflow;
    252 	struct timespec ts;
    253 	int error;
    254 	lwp_t *l;
    255 
    256 	/* coverity[assert_side_effect] */
    257 	KASSERT(lockstat_dev_enabled);
    258 
    259 	/*
    260 	 * Set enabled false, force a write barrier, and wait for other CPUs
    261 	 * to exit lockstat_event().
    262 	 */
    263 	lockstat_dev_enabled = 0;
    264 	LOCKSTAT_ENABLED_UPDATE();
    265 	getnanotime(&ts);
    266 	tsleep(&lockstat_stop, PPAUSE, "lockstat", mstohz(10));
    267 
    268 	/*
    269 	 * Did we run out of buffers while tracing?
    270 	 */
    271 	overflow = 0;
    272 	for (CPU_INFO_FOREACH(cii, ci))
    273 		overflow += ((lscpu_t *)ci->ci_lockstat)->lc_overflow;
    274 
    275 	if (overflow != 0) {
    276 		error = EOVERFLOW;
    277 		log(LOG_NOTICE, "lockstat: %d buffer allocations failed\n",
    278 		    overflow);
    279 	} else
    280 		error = 0;
    281 
    282 	lockstat_init_tables(NULL);
    283 
    284 	/* Run through all LWPs and clear the slate for the next run. */
    285 	mutex_enter(proc_lock);
    286 	LIST_FOREACH(l, &alllwp, l_list) {
    287 		l->l_pfailaddr = 0;
    288 		l->l_pfailtime = 0;
    289 		l->l_pfaillock = 0;
    290 	}
    291 	mutex_exit(proc_lock);
    292 
    293 	if (ld == NULL)
    294 		return error;
    295 
    296 	/*
    297 	 * Fill out the disable struct for the caller.
    298 	 */
    299 	timespecsub(&ts, &lockstat_stime, &ld->ld_time);
    300 	ld->ld_size = lockstat_sizeb;
    301 
    302 	cpuno = 0;
    303 	for (CPU_INFO_FOREACH(cii, ci)) {
    304 		if (cpuno >= sizeof(ld->ld_freq) / sizeof(ld->ld_freq[0])) {
    305 			log(LOG_WARNING, "lockstat: too many CPUs\n");
    306 			break;
    307 		}
    308 		ld->ld_freq[cpuno++] = cpu_frequency(ci);
    309 	}
    310 
    311 	return error;
    312 }
    313 
    314 /*
    315  * Allocate buffers for lockstat_start().
    316  */
    317 int
    318 lockstat_alloc(lsenable_t *le)
    319 {
    320 	lsbuf_t *lb;
    321 	size_t sz;
    322 
    323 	/* coverity[assert_side_effect] */
    324 	KASSERT(!lockstat_dev_enabled);
    325 	lockstat_free();
    326 
    327 	sz = sizeof(*lb) * le->le_nbufs;
    328 
    329 	lb = kmem_zalloc(sz, KM_SLEEP);
    330 	if (lb == NULL)
    331 		return (ENOMEM);
    332 
    333 	/* coverity[assert_side_effect] */
    334 	KASSERT(!lockstat_dev_enabled);
    335 	KASSERT(lockstat_baseb == NULL);
    336 	lockstat_sizeb = sz;
    337 	lockstat_baseb = lb;
    338 
    339 	return (0);
    340 }
    341 
    342 /*
    343  * Free allocated buffers after tracing has stopped.
    344  */
    345 void
    346 lockstat_free(void)
    347 {
    348 
    349 	/* coverity[assert_side_effect] */
    350 	KASSERT(!lockstat_dev_enabled);
    351 
    352 	if (lockstat_baseb != NULL) {
    353 		kmem_free(lockstat_baseb, lockstat_sizeb);
    354 		lockstat_baseb = NULL;
    355 	}
    356 }
    357 
    358 /*
    359  * Main entry point from lock primatives.
    360  */
    361 void
    362 lockstat_event(uintptr_t lock, uintptr_t callsite, u_int flags, u_int count,
    363 	       uint64_t cycles)
    364 {
    365 	lslist_t *ll;
    366 	lscpu_t *lc;
    367 	lsbuf_t *lb;
    368 	u_int event;
    369 	int s;
    370 
    371 #ifdef KDTRACE_HOOKS
    372 	uint32_t id;
    373 	CTASSERT((LS_NPROBES & (LS_NPROBES - 1)) == 0);
    374 	if ((id = lockstat_probemap[LS_COMPRESS(flags)]) != 0)
    375 		(*lockstat_probe_func)(id, lock, callsite, flags, count,
    376 		    cycles);
    377 #endif
    378 
    379 	if ((flags & lockstat_dev_enabled) != flags || count == 0)
    380 		return;
    381 	if (lock < lockstat_lockstart || lock > lockstat_lockend)
    382 		return;
    383 	if (callsite < lockstat_csstart || callsite > lockstat_csend)
    384 		return;
    385 
    386 	callsite &= lockstat_csmask;
    387 	lock &= lockstat_lamask;
    388 
    389 	/*
    390 	 * Find the table for this lock+callsite pair, and try to locate a
    391 	 * buffer with the same key.
    392 	 */
    393 	s = splhigh();
    394 	lc = curcpu()->ci_lockstat;
    395 	ll = &lc->lc_hash[LOCKSTAT_HASH(lock ^ callsite)];
    396 	event = (flags & LB_EVENT_MASK) - 1;
    397 
    398 	LIST_FOREACH(lb, ll, lb_chain.list) {
    399 		if (lb->lb_lock == lock && lb->lb_callsite == callsite)
    400 			break;
    401 	}
    402 
    403 	if (lb != NULL) {
    404 		/*
    405 		 * We found a record.  Move it to the front of the list, as
    406 		 * we're likely to hit it again soon.
    407 		 */
    408 		if (lb != LIST_FIRST(ll)) {
    409 			LIST_REMOVE(lb, lb_chain.list);
    410 			LIST_INSERT_HEAD(ll, lb, lb_chain.list);
    411 		}
    412 		lb->lb_counts[event] += count;
    413 		lb->lb_times[event] += cycles;
    414 	} else if ((lb = SLIST_FIRST(&lc->lc_free)) != NULL) {
    415 		/*
    416 		 * Pinch a new buffer and fill it out.
    417 		 */
    418 		SLIST_REMOVE_HEAD(&lc->lc_free, lb_chain.slist);
    419 		LIST_INSERT_HEAD(ll, lb, lb_chain.list);
    420 		lb->lb_flags = (uint16_t)flags;
    421 		lb->lb_lock = lock;
    422 		lb->lb_callsite = callsite;
    423 		lb->lb_counts[event] = count;
    424 		lb->lb_times[event] = cycles;
    425 	} else {
    426 		/*
    427 		 * We didn't find a buffer and there were none free.
    428 		 * lockstat_stop() will notice later on and report the
    429 		 * error.
    430 		 */
    431 		 lc->lc_overflow++;
    432 	}
    433 
    434 	splx(s);
    435 }
    436 
    437 /*
    438  * Accept an open() on /dev/lockstat.
    439  */
    440 int
    441 lockstat_open(dev_t dev, int flag, int mode, lwp_t *l)
    442 {
    443 
    444 	if (!__cpu_simple_lock_try(&lockstat_lock))
    445 		return EBUSY;
    446 	lockstat_lwp = curlwp;
    447 	return 0;
    448 }
    449 
    450 /*
    451  * Accept the last close() on /dev/lockstat.
    452  */
    453 int
    454 lockstat_close(dev_t dev, int flag, int mode, lwp_t *l)
    455 {
    456 
    457 	lockstat_lwp = NULL;
    458 	__cpu_simple_unlock(&lockstat_lock);
    459 	return 0;
    460 }
    461 
    462 /*
    463  * Handle control operations.
    464  */
    465 int
    466 lockstat_ioctl(dev_t dev, u_long cmd, void *data, int flag, lwp_t *l)
    467 {
    468 	lsenable_t *le;
    469 	int error;
    470 
    471 	if (lockstat_lwp != curlwp)
    472 		return EBUSY;
    473 
    474 	switch (cmd) {
    475 	case IOC_LOCKSTAT_GVERSION:
    476 		*(int *)data = LS_VERSION;
    477 		error = 0;
    478 		break;
    479 
    480 	case IOC_LOCKSTAT_ENABLE:
    481 		le = (lsenable_t *)data;
    482 
    483 		if (!cpu_hascounter()) {
    484 			error = ENODEV;
    485 			break;
    486 		}
    487 		if (lockstat_dev_enabled) {
    488 			error = EBUSY;
    489 			break;
    490 		}
    491 
    492 		/*
    493 		 * Sanitize the arguments passed in and set up filtering.
    494 		 */
    495 		if (le->le_nbufs == 0)
    496 			le->le_nbufs = LOCKSTAT_DEFBUFS;
    497 		else if (le->le_nbufs > LOCKSTAT_MAXBUFS ||
    498 		    le->le_nbufs < LOCKSTAT_MINBUFS) {
    499 			error = EINVAL;
    500 			break;
    501 		}
    502 		if ((le->le_flags & LE_ONE_CALLSITE) == 0) {
    503 			le->le_csstart = 0;
    504 			le->le_csend = le->le_csstart - 1;
    505 		}
    506 		if ((le->le_flags & LE_ONE_LOCK) == 0) {
    507 			le->le_lockstart = 0;
    508 			le->le_lockend = le->le_lockstart - 1;
    509 		}
    510 		if ((le->le_mask & LB_EVENT_MASK) == 0)
    511 			return EINVAL;
    512 		if ((le->le_mask & LB_LOCK_MASK) == 0)
    513 			return EINVAL;
    514 
    515 		/*
    516 		 * Start tracing.
    517 		 */
    518 		if ((error = lockstat_alloc(le)) == 0)
    519 			lockstat_start(le);
    520 		break;
    521 
    522 	case IOC_LOCKSTAT_DISABLE:
    523 		if (!lockstat_dev_enabled)
    524 			error = EINVAL;
    525 		else
    526 			error = lockstat_stop((lsdisable_t *)data);
    527 		break;
    528 
    529 	default:
    530 		error = ENOTTY;
    531 		break;
    532 	}
    533 
    534 	return error;
    535 }
    536 
    537 /*
    538  * Copy buffers out to user-space.
    539  */
    540 int
    541 lockstat_read(dev_t dev, struct uio *uio, int flag)
    542 {
    543 
    544 	if (curlwp != lockstat_lwp || lockstat_dev_enabled)
    545 		return EBUSY;
    546 	return uiomove(lockstat_baseb, lockstat_sizeb, uio);
    547 }
    548