Home | History | Annotate | Line # | Download | only in lockstat
main.c revision 1.12 
      1 /*	$NetBSD: main.c,v 1.12 2008/04/04 17:25:09 ad Exp $	*/
      2 
      3 /*-
      4  * Copyright (c) 2006, 2007 The NetBSD Foundation, Inc.
      5  * All rights reserved.
      6  *
      7  * This code is derived from software contributed to The NetBSD Foundation
      8  * by Andrew Doran.
      9  *
     10  * Redistribution and use in source and binary forms, with or without
     11  * modification, are permitted provided that the following conditions
     12  * are met:
     13  * 1. Redistributions of source code must retain the above copyright
     14  *    notice, this list of conditions and the following disclaimer.
     15  * 2. Redistributions in binary form must reproduce the above copyright
     16  *    notice, this list of conditions and the following disclaimer in the
     17  *    documentation and/or other materials provided with the distribution.
     18  * 3. All advertising materials mentioning features or use of this software
     19  *    must display the following acknowledgement:
     20  *	This product includes software developed by the NetBSD
     21  *	Foundation, Inc. and its contributors.
     22  * 4. Neither the name of The NetBSD Foundation nor the names of its
     23  *    contributors may be used to endorse or promote products derived
     24  *    from this software without specific prior written permission.
     25  *
     26  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     27  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     28  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     29  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     30  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     31  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     32  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     33  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     34  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     35  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     36  * POSSIBILITY OF SUCH DAMAGE.
     37  */
     38 
     39 /*
     40  * TODO:
     41  *
     42  * - Tracking of times for sleep locks is broken.
     43  * - Need better analysis and tracking of events.
     44  * - Shouldn't have to parse the namelist here.  We should use something like
     45  *   FreeBSD's libelf.
     46  * - The way the namelist is searched sucks, is it worth doing something
     47  *   better?
     48  */
     49 
     50 #include <sys/cdefs.h>
     51 #ifndef lint
     52 __RCSID("$NetBSD: main.c,v 1.12 2008/04/04 17:25:09 ad Exp $");
     53 #endif /* not lint */
     54 
     55 #include <sys/types.h>
     56 #include <sys/param.h>
     57 #include <sys/time.h>
     58 #include <sys/fcntl.h>
     59 #include <sys/ioctl.h>
     60 #include <sys/wait.h>
     61 #include <sys/signal.h>
     62 #include <sys/sysctl.h>
     63 
     64 #include <dev/lockstat.h>
     65 
     66 #include <stdio.h>
     67 #include <stdlib.h>
     68 #include <string.h>
     69 #include <limits.h>
     70 #include <unistd.h>
     71 #include <err.h>
     72 #include <paths.h>
     73 #include <util.h>
     74 #include <ctype.h>
     75 #include <errno.h>
     76 #include <stdbool.h>
     77 
     78 #include "extern.h"
     79 
     80 #define	_PATH_DEV_LOCKSTAT	"/dev/lockstat"
     81 
     82 #define	MILLI	1000.0
     83 #define	MICRO	1000000.0
     84 #define	NANO	1000000000.0
     85 #define	PICO	1000000000000.0
     86 
     87 TAILQ_HEAD(lock_head, lockstruct);
     88 typedef struct lock_head locklist_t;
     89 TAILQ_HEAD(buf_head, lsbuf);
     90 typedef struct buf_head buflist_t;
     91 
     92 typedef struct lockstruct {
     93 	TAILQ_ENTRY(lockstruct)	chain;
     94 	buflist_t		bufs;
     95 	buflist_t		tosort;
     96 	uintptr_t		lock;
     97  	double			time;
     98 	uint32_t		count;
     99 	u_int			flags;
    100 	u_int			nbufs;
    101 	char			name[NAME_SIZE];
    102 } lock_t;
    103 
    104 typedef struct name {
    105 	const char	*name;
    106 	int		mask;
    107 } name_t;
    108 
    109 const name_t locknames[] = {
    110 	{ "adaptive_mutex", LB_ADAPTIVE_MUTEX },
    111 	{ "spin_mutex", LB_SPIN_MUTEX },
    112 	{ "rwlock", LB_RWLOCK },
    113 	{ "kernel_lock", LB_KERNEL_LOCK },
    114 	{ NULL, 0 }
    115 };
    116 
    117 const name_t eventnames[] = {
    118 	{ "spin", LB_SPIN },
    119 	{ "sleep_exclusive", LB_SLEEP1 },
    120 	{ "sleep_shared", LB_SLEEP2 },
    121 	{ NULL, 0 },
    122 };
    123 
    124 const name_t alltypes[] = {
    125 	{ "Adaptive mutex spin", LB_ADAPTIVE_MUTEX | LB_SPIN },
    126 	{ "Adaptive mutex sleep", LB_ADAPTIVE_MUTEX | LB_SLEEP1 },
    127 	{ "Spin mutex spin", LB_SPIN_MUTEX | LB_SPIN },
    128 	{ "RW lock sleep (writer)", LB_RWLOCK | LB_SLEEP1 },
    129 	{ "RW lock sleep (reader)", LB_RWLOCK | LB_SLEEP2 },
    130 	{ "RW lock spin", LB_RWLOCK | LB_SPIN },
    131 	{ "Kernel lock spin", LB_KERNEL_LOCK | LB_SPIN },
    132 	{ NULL, 0 }
    133 };
    134 
    135 locklist_t	locklist;
    136 locklist_t	freelist;
    137 locklist_t	sortlist;
    138 
    139 lsbuf_t		*bufs;
    140 lsdisable_t	ld;
    141 bool		lflag;
    142 bool		fflag;
    143 int		nbufs;
    144 bool		cflag;
    145 int		lsfd;
    146 int		displayed;
    147 int		bin64;
    148 double		tscale;
    149 double		cscale;
    150 double		cpuscale[sizeof(ld.ld_freq) / sizeof(ld.ld_freq[0])];
    151 FILE		*outfp;
    152 
    153 void	findsym(findsym_t, char *, uintptr_t *, uintptr_t *, bool);
    154 void	spawn(int, char **);
    155 void	display(int, const char *name);
    156 void	listnames(const name_t *);
    157 void	collapse(bool, bool);
    158 int	matchname(const name_t *, char *);
    159 void	makelists(int, int);
    160 void	nullsig(int);
    161 void	usage(void);
    162 int	ncpu(void);
    163 lock_t	*morelocks(void);
    164 
    165 int
    166 main(int argc, char **argv)
    167 {
    168 	int eventtype, locktype, ch, nlfd, fd, i;
    169 	bool sflag, pflag, mflag, Mflag;
    170 	const char *nlistf, *outf;
    171 	char *lockname, *funcname;
    172 	const name_t *name;
    173 	lsenable_t le;
    174 	double ms;
    175 	char *p;
    176 
    177 	nlistf = NULL;
    178 	outf = NULL;
    179 	lockname = NULL;
    180 	funcname = NULL;
    181 	eventtype = -1;
    182 	locktype = -1;
    183 	nbufs = 0;
    184 	sflag = false;
    185 	pflag = false;
    186 	mflag = false;
    187 	Mflag = false;
    188 
    189 	while ((ch = getopt(argc, argv, "E:F:L:MN:T:b:ceflmo:pst")) != -1)
    190 		switch (ch) {
    191 		case 'E':
    192 			eventtype = matchname(eventnames, optarg);
    193 			break;
    194 		case 'F':
    195 			funcname = optarg;
    196 			break;
    197 		case 'L':
    198 			lockname = optarg;
    199 			break;
    200 		case 'N':
    201 			nlistf = optarg;
    202 			break;
    203 		case 'T':
    204 			locktype = matchname(locknames, optarg);
    205 			break;
    206 		case 'b':
    207 			nbufs = (int)strtol(optarg, &p, 0);
    208 			if (!isdigit((u_int)*optarg) || *p != '\0')
    209 				usage();
    210 			break;
    211 		case 'c':
    212 			cflag = true;
    213 			break;
    214 		case 'e':
    215 			listnames(eventnames);
    216 			break;
    217 		case 'f':
    218 			fflag = true;
    219 			break;
    220 		case 'l':
    221 			lflag = true;
    222 			break;
    223 		case 'm':
    224 			mflag = true;
    225 			break;
    226 		case 'M':
    227 			Mflag = true;
    228 			break;
    229 		case 'o':
    230 			outf = optarg;
    231 			break;
    232 		case 'p':
    233 			pflag = true;
    234 			break;
    235 		case 's':
    236 			sflag = true;
    237 			break;
    238 		case 't':
    239 			listnames(locknames);
    240 			break;
    241 		default:
    242 			usage();
    243 		}
    244 	argc -= optind;
    245 	argv += optind;
    246 
    247 	if (*argv == NULL)
    248 		usage();
    249 
    250 	if (outf) {
    251 		fd = open(outf, O_WRONLY | O_CREAT | O_TRUNC, 0600);
    252 		if (fd == -1)
    253 			err(EXIT_FAILURE, "opening %s", outf);
    254 		outfp = fdopen(fd, "w");
    255 	} else
    256 		outfp = stdout;
    257 
    258 	/*
    259 	 * Find the name list for resolving symbol names, and load it into
    260 	 * memory.
    261 	 */
    262 	if (nlistf == NULL) {
    263 		nlfd = open(_PATH_KSYMS, O_RDONLY);
    264 		nlistf = getbootfile();
    265 	} else
    266 		nlfd = -1;
    267 	if (nlfd == -1) {
    268 		if ((nlfd = open(nlistf, O_RDONLY)) < 0)
    269 			err(EXIT_FAILURE, "cannot open " _PATH_KSYMS " or %s",
    270 			    nlistf);
    271 	}
    272 	if (loadsym32(nlfd) != 0) {
    273 		if (loadsym64(nlfd) != 0)
    274 			errx(EXIT_FAILURE, "unable to load symbol table");
    275 		bin64 = 1;
    276 	}
    277 	close(nlfd);
    278 
    279 	memset(&le, 0, sizeof(le));
    280 	le.le_nbufs = nbufs;
    281 
    282 	/*
    283 	 * Set up initial filtering.
    284 	 */
    285 	if (lockname != NULL) {
    286 		findsym(LOCK_BYNAME, lockname, &le.le_lockstart,
    287 		    &le.le_lockend, true);
    288 		le.le_flags |= LE_ONE_LOCK;
    289 	}
    290 	if (!lflag)
    291 		le.le_flags |= LE_CALLSITE;
    292 	if (!fflag)
    293 		le.le_flags |= LE_LOCK;
    294 	if (funcname != NULL) {
    295 		if (lflag)
    296 			usage();
    297 		findsym(FUNC_BYNAME, funcname, &le.le_csstart, &le.le_csend, true);
    298 		le.le_flags |= LE_ONE_CALLSITE;
    299 	}
    300 	le.le_mask = (eventtype & LB_EVENT_MASK) | (locktype & LB_LOCK_MASK);
    301 
    302 	/*
    303 	 * Start tracing.
    304 	 */
    305 	if ((lsfd = open(_PATH_DEV_LOCKSTAT, O_RDONLY)) < 0)
    306 		err(EXIT_FAILURE, "cannot open " _PATH_DEV_LOCKSTAT);
    307 	if (ioctl(lsfd, IOC_LOCKSTAT_GVERSION, &ch) < 0)
    308 		err(EXIT_FAILURE, "ioctl");
    309 	if (ch != LS_VERSION)
    310 		errx(EXIT_FAILURE,
    311 		    "incompatible lockstat interface version (%d, kernel %d)",
    312 			LS_VERSION, ch);
    313 	if (ioctl(lsfd, IOC_LOCKSTAT_ENABLE, &le))
    314 		err(EXIT_FAILURE, "cannot enable tracing");
    315 
    316 	/*
    317 	 * Execute the traced program.
    318 	 */
    319 	spawn(argc, argv);
    320 
    321 	/*
    322 	 * Stop tracing, and read the trace buffers from the kernel.
    323 	 */
    324 	if (ioctl(lsfd, IOC_LOCKSTAT_DISABLE, &ld) == -1) {
    325 		if (errno == EOVERFLOW) {
    326 			warnx("overflowed available kernel trace buffers");
    327 			exit(EXIT_FAILURE);
    328 		}
    329 		err(EXIT_FAILURE, "cannot disable tracing");
    330 	}
    331 	if ((bufs = malloc(ld.ld_size)) == NULL)
    332 		err(EXIT_FAILURE, "cannot allocate memory for user buffers");
    333 	if (read(lsfd, bufs, ld.ld_size) != ld.ld_size)
    334 		err(EXIT_FAILURE, "reading from " _PATH_DEV_LOCKSTAT);
    335 	if (close(lsfd))
    336 		err(EXIT_FAILURE, "close(" _PATH_DEV_LOCKSTAT ")");
    337 
    338 	/*
    339 	 * Figure out how to scale the results.  For internal use we convert
    340 	 * all times from CPU frequency based to picoseconds, and values are
    341 	 * eventually displayed in ms.
    342 	 */
    343 	for (i = 0; i < sizeof(ld.ld_freq) / sizeof(ld.ld_freq[0]); i++)
    344 		if (ld.ld_freq[i] != 0)
    345 			cpuscale[i] = PICO / ld.ld_freq[i];
    346 	ms = ld.ld_time.tv_sec * MILLI + ld.ld_time.tv_nsec / MICRO;
    347 	if (pflag)
    348 		cscale = 1.0 / ncpu();
    349 	else
    350 		cscale = 1.0;
    351 	cscale *= (sflag ? MILLI / ms : 1.0);
    352 	tscale = cscale / NANO;
    353 	nbufs = (int)(ld.ld_size / sizeof(lsbuf_t));
    354 
    355 	TAILQ_INIT(&locklist);
    356 	TAILQ_INIT(&sortlist);
    357 	TAILQ_INIT(&freelist);
    358 
    359 	if ((mflag | Mflag) != 0)
    360 		collapse(mflag, Mflag);
    361 
    362 	/*
    363 	 * Display the results.
    364 	 */
    365 	fprintf(outfp, "Elapsed time: %.2f seconds.", ms / MILLI);
    366 	if (sflag || pflag) {
    367 		fprintf(outfp, " Displaying ");
    368 		if (pflag)
    369 			fprintf(outfp, "per-CPU ");
    370 		if (sflag)
    371 			fprintf(outfp, "per-second ");
    372 		fprintf(outfp, "averages.");
    373 	}
    374 	putc('\n', outfp);
    375 
    376 	for (name = alltypes; name->name != NULL; name++) {
    377 		if (eventtype != -1 &&
    378 		    (name->mask & LB_EVENT_MASK) != eventtype)
    379 			continue;
    380 		if (locktype != -1 &&
    381 		    (name->mask & LB_LOCK_MASK) != locktype)
    382 			continue;
    383 
    384 		display(name->mask, name->name);
    385 	}
    386 
    387 	if (displayed == 0)
    388 		fprintf(outfp, "None of the selected events were recorded.\n");
    389 	exit(EXIT_SUCCESS);
    390 }
    391 
    392 void
    393 usage(void)
    394 {
    395 
    396 	fprintf(stderr,
    397 	    "%s: usage:\n"
    398 	    "%s [options] <command>\n\n"
    399 	    "-b nbuf\t\tset number of event buffers to allocate\n"
    400 	    "-c\t\treport percentage of total events by count, not time\n"
    401 	    "-E event\t\tdisplay only one type of event\n"
    402 	    "-e\t\tlist event types\n"
    403 	    "-F func\t\tlimit trace to one function\n"
    404 	    "-f\t\ttrace only by function\n"
    405 	    "-L lock\t\tlimit trace to one lock (name, or address)\n"
    406 	    "-l\t\ttrace only by lock\n"
    407 	    "-M\t\tmerge lock addresses within unique objects\n"
    408 	    "-m\t\tmerge call sites within unique functions\n"
    409 	    "-N nlist\tspecify name list file\n"
    410 	    "-o file\t\tsend output to named file, not stdout\n"
    411 	    "-p\t\tshow average count/time per CPU, not total\n"
    412 	    "-s\t\tshow average count/time per second, not total\n"
    413 	    "-T type\t\tdisplay only one type of lock\n"
    414 	    "-t\t\tlist lock types\n",
    415 	    getprogname(), getprogname());
    416 
    417 	exit(EXIT_FAILURE);
    418 }
    419 
    420 void
    421 nullsig(int junk)
    422 {
    423 
    424 	(void)junk;
    425 }
    426 
    427 void
    428 listnames(const name_t *name)
    429 {
    430 
    431 	for (; name->name != NULL; name++)
    432 		printf("%s\n", name->name);
    433 
    434 	exit(EXIT_SUCCESS);
    435 }
    436 
    437 int
    438 matchname(const name_t *name, char *string)
    439 {
    440 	int empty, mask;
    441 	char *sp;
    442 
    443 	empty = 1;
    444 	mask = 0;
    445 
    446 	while ((sp = strsep(&string, ",")) != NULL) {
    447 		if (*sp == '\0')
    448 			usage();
    449 
    450 		for (; name->name != NULL; name++) {
    451 			if (strcasecmp(name->name, sp) == 0) {
    452 				mask |= name->mask;
    453 				break;
    454 			}
    455 		}
    456 		if (name->name == NULL)
    457 			errx(EXIT_FAILURE, "unknown identifier `%s'", sp);
    458 		empty = 0;
    459 	}
    460 
    461 	if (empty)
    462 		usage();
    463 
    464 	return mask;
    465 }
    466 
    467 /*
    468  * Return the number of CPUs in the running system.
    469  */
    470 int
    471 ncpu(void)
    472 {
    473 	int rv, mib[2];
    474 	size_t varlen;
    475 
    476 	mib[0] = CTL_HW;
    477 	mib[1] = HW_NCPU;
    478 	varlen = sizeof(rv);
    479 	if (sysctl(mib, 2, &rv, &varlen, NULL, (size_t)0) < 0)
    480 		rv = 1;
    481 
    482 	return (rv);
    483 }
    484 
    485 /*
    486  * Call into the ELF parser and look up a symbol by name or by address.
    487  */
    488 void
    489 findsym(findsym_t find, char *name, uintptr_t *start, uintptr_t *end, bool chg)
    490 {
    491 	uintptr_t tend, sa, ea;
    492 	char *p;
    493 	int rv;
    494 
    495 	if (!chg) {
    496 		sa = *start;
    497 		start = &sa;
    498 		end = &ea;
    499 	}
    500 
    501 	if (end == NULL)
    502 		end = &tend;
    503 
    504 	if (find == LOCK_BYNAME) {
    505 		if (isdigit((u_int)name[0])) {
    506 			*start = (uintptr_t)strtoul(name, &p, 0);
    507 			if (*p == '\0')
    508 				return;
    509 		}
    510 	}
    511 
    512 	if (bin64)
    513 		rv = findsym64(find, name, start, end);
    514 	else
    515 		rv = findsym32(find, name, start, end);
    516 
    517 	if (find == FUNC_BYNAME || find == LOCK_BYNAME) {
    518 		if (rv == -1)
    519 			errx(EXIT_FAILURE, "unable to find symbol `%s'", name);
    520 		return;
    521 	}
    522 
    523 	if (rv == -1)
    524 		snprintf(name, NAME_SIZE, "%016lx", (long)*start);
    525 }
    526 
    527 /*
    528  * Fork off the child process and wait for it to complete.  We trap SIGINT
    529  * so that the caller can use Ctrl-C to stop tracing early and still get
    530  * useful results.
    531  */
    532 void
    533 spawn(int argc, char **argv)
    534 {
    535 	pid_t pid;
    536 
    537 	switch (pid = fork()) {
    538 	case 0:
    539 		close(lsfd);
    540 		if (execvp(argv[0], argv) == -1)
    541 			err(EXIT_FAILURE, "cannot exec");
    542 		break;
    543 	case -1:
    544 		err(EXIT_FAILURE, "cannot fork to exec");
    545 		break;
    546 	default:
    547 		signal(SIGINT, nullsig);
    548 		wait(NULL);
    549 		signal(SIGINT, SIG_DFL);
    550 		break;
    551 	}
    552 }
    553 
    554 /*
    555  * Allocate a new block of lock_t structures.
    556  */
    557 lock_t *
    558 morelocks(void)
    559 {
    560 	const static int batch = 32;
    561 	lock_t *l, *lp, *max;
    562 
    563 	l = (lock_t *)malloc(sizeof(*l) * batch);
    564 
    565 	for (lp = l, max = l + batch; lp < max; lp++)
    566 		TAILQ_INSERT_TAIL(&freelist, lp, chain);
    567 
    568 	return l;
    569 }
    570 
    571 /*
    572  * Collapse addresses from unique objects.
    573  */
    574 void
    575 collapse(bool func, bool lock)
    576 {
    577 	lsbuf_t *lb, *max;
    578 
    579 	for (lb = bufs, max = bufs + nbufs; lb < max; lb++) {
    580 		if (func && lb->lb_callsite != 0) {
    581 			findsym(FUNC_BYADDR, NULL, &lb->lb_callsite, NULL,
    582 			    true);
    583 		}
    584 		if (lock && lb->lb_lock != 0) {
    585 			findsym(LOCK_BYADDR, NULL, &lb->lb_lock, NULL,
    586 			    true);
    587 		}
    588 	}
    589 }
    590 
    591 /*
    592  * From the kernel supplied data, construct two dimensional lists of locks
    593  * and event buffers, indexed by lock type and sorted by event type.
    594  */
    595 void
    596 makelists(int mask, int event)
    597 {
    598 	lsbuf_t *lb, *lb2, *max;
    599 	lock_t *l, *l2;
    600 	int type;
    601 
    602 	/*
    603 	 * Recycle lock_t structures from the last run.
    604 	 */
    605 	while ((l = TAILQ_FIRST(&locklist)) != NULL) {
    606 		TAILQ_REMOVE(&locklist, l, chain);
    607 		TAILQ_INSERT_HEAD(&freelist, l, chain);
    608 	}
    609 
    610 	type = mask & LB_LOCK_MASK;
    611 
    612 	for (lb = bufs, max = bufs + nbufs; lb < max; lb++) {
    613 		if ((lb->lb_flags & LB_LOCK_MASK) != type ||
    614 		    lb->lb_counts[event] == 0)
    615 			continue;
    616 
    617 		/*
    618 		 * Look for a record descibing this lock, and allocate a
    619 		 * new one if needed.
    620 		 */
    621 		TAILQ_FOREACH(l, &sortlist, chain) {
    622 			if (l->lock == lb->lb_lock)
    623 				break;
    624 		}
    625 		if (l == NULL) {
    626 			if ((l = TAILQ_FIRST(&freelist)) == NULL)
    627 				l = morelocks();
    628 			TAILQ_REMOVE(&freelist, l, chain);
    629 			l->flags = lb->lb_flags;
    630 			l->lock = lb->lb_lock;
    631 			l->nbufs = 0;
    632 			l->name[0] = '\0';
    633 			l->count = 0;
    634 			l->time = 0;
    635 			TAILQ_INIT(&l->tosort);
    636 			TAILQ_INIT(&l->bufs);
    637 			TAILQ_INSERT_TAIL(&sortlist, l, chain);
    638 		}
    639 
    640 		/*
    641 		 * Scale the time values per buffer and summarise
    642 		 * times+counts per lock.
    643 		 */
    644 		lb->lb_times[event] *= cpuscale[lb->lb_cpu];
    645 		l->count += lb->lb_counts[event];
    646 		l->time += lb->lb_times[event];
    647 
    648 		/*
    649 		 * Merge same lock+callsite pairs from multiple CPUs
    650 		 * together.
    651 		 */
    652 		TAILQ_FOREACH(lb2, &l->tosort, lb_chain.tailq) {
    653 			if (lb->lb_callsite == lb2->lb_callsite)
    654 				break;
    655 		}
    656 		if (lb2 != NULL) {
    657 			lb2->lb_counts[event] += lb->lb_counts[event];
    658 			lb2->lb_times[event] += lb->lb_times[event];
    659 		} else {
    660 			TAILQ_INSERT_HEAD(&l->tosort, lb, lb_chain.tailq);
    661 			l->nbufs++;
    662 		}
    663 	}
    664 
    665 	/*
    666 	 * Now sort the lists.
    667 	 */
    668 	while ((l = TAILQ_FIRST(&sortlist)) != NULL) {
    669 		TAILQ_REMOVE(&sortlist, l, chain);
    670 
    671 		/*
    672 		 * Sort the buffers into the per-lock list.
    673 		 */
    674 		while ((lb = TAILQ_FIRST(&l->tosort)) != NULL) {
    675 			TAILQ_REMOVE(&l->tosort, lb, lb_chain.tailq);
    676 
    677 			lb2 = TAILQ_FIRST(&l->bufs);
    678 			while (lb2 != NULL) {
    679 				if (cflag) {
    680 					if (lb->lb_counts[event] >
    681 					    lb2->lb_counts[event])
    682 						break;
    683 				} else if (lb->lb_times[event] >
    684 				    lb2->lb_times[event])
    685 					break;
    686 				lb2 = TAILQ_NEXT(lb2, lb_chain.tailq);
    687 			}
    688 			if (lb2 == NULL)
    689 				TAILQ_INSERT_TAIL(&l->bufs, lb,
    690 				    lb_chain.tailq);
    691 			else
    692 				TAILQ_INSERT_BEFORE(lb2, lb, lb_chain.tailq);
    693 		}
    694 
    695 		/*
    696 		 * Sort this lock into the per-type list, based on the
    697 		 * totals per lock.
    698 		 */
    699 		l2 = TAILQ_FIRST(&locklist);
    700 		while (l2 != NULL) {
    701 			if (cflag) {
    702 				if (l->count > l2->count)
    703 					break;
    704 			} else if (l->time > l2->time)
    705 				break;
    706 			l2 = TAILQ_NEXT(l2, chain);
    707 		}
    708 		if (l2 == NULL)
    709 			TAILQ_INSERT_TAIL(&locklist, l, chain);
    710 		else
    711 			TAILQ_INSERT_BEFORE(l2, l, chain);
    712 	}
    713 }
    714 
    715 /*
    716  * Display a summary table for one lock type / event type pair.
    717  */
    718 void
    719 display(int mask, const char *name)
    720 {
    721 	lock_t *l;
    722 	lsbuf_t *lb;
    723 	double pcscale, metric;
    724 	char fname[NAME_SIZE];
    725 	int event;
    726 
    727 	event = (mask & LB_EVENT_MASK) - 1;
    728 	makelists(mask, event);
    729 
    730 	if (TAILQ_EMPTY(&locklist))
    731 		return;
    732 
    733 	fprintf(outfp, "\n-- %s\n\n"
    734 	    "Total%%  Count   Time/ms          Lock                       Caller\n"
    735 	    "------ ------- --------- ---------------------- ------------------------------\n",
    736 	    name);
    737 
    738 	/*
    739 	 * Sum up all events for this type of lock + event.
    740 	 */
    741 	pcscale = 0;
    742 	TAILQ_FOREACH(l, &locklist, chain) {
    743 		if (cflag)
    744 			pcscale += l->count;
    745 		else
    746 			pcscale += l->time;
    747 		displayed++;
    748 	}
    749 	if (pcscale == 0)
    750 		pcscale = 100;
    751 	else
    752 		pcscale = (100.0 / pcscale);
    753 
    754 	/*
    755 	 * For each lock, print a summary total, followed by a breakdown by
    756 	 * caller.
    757 	 */
    758 	TAILQ_FOREACH(l, &locklist, chain) {
    759 		if (cflag)
    760 			metric = l->count;
    761 		else
    762 			metric = l->time;
    763 		metric *= pcscale;
    764 
    765 		if (l->name[0] == '\0')
    766 			findsym(LOCK_BYADDR, l->name, &l->lock, NULL, false);
    767 
    768 		if (lflag || l->nbufs > 1)
    769 			fprintf(outfp, "%6.2f %7d %9.2f %-22s <all>\n",
    770 			    metric, (int)(l->count * cscale),
    771 			    l->time * tscale, l->name);
    772 
    773 		if (lflag)
    774 			continue;
    775 
    776 		TAILQ_FOREACH(lb, &l->bufs, lb_chain.tailq) {
    777 			if (cflag)
    778 				metric = lb->lb_counts[event];
    779 			else
    780 				metric = lb->lb_times[event];
    781 			metric *= pcscale;
    782 
    783 			findsym(FUNC_BYADDR, fname, &lb->lb_callsite, NULL,
    784 			    false);
    785 			fprintf(outfp, "%6.2f %7d %9.2f %-22s %s\n",
    786 			    metric, (int)(lb->lb_counts[event] * cscale),
    787 			    lb->lb_times[event] * tscale, l->name, fname);
    788 		}
    789 	}
    790 }
    791