Home | History | Annotate | Line # | Download | only in lockstat
main.c revision 1.9 
      1 /*	$NetBSD: main.c,v 1.9 2007/07/14 13:30:44 ad Exp $	*/
      2 
      3 /*-
      4  * Copyright (c) 2006, 2007 The NetBSD Foundation, Inc.
      5  * All rights reserved.
      6  *
      7  * This code is derived from software contributed to The NetBSD Foundation
      8  * by Andrew Doran.
      9  *
     10  * Redistribution and use in source and binary forms, with or without
     11  * modification, are permitted provided that the following conditions
     12  * are met:
     13  * 1. Redistributions of source code must retain the above copyright
     14  *    notice, this list of conditions and the following disclaimer.
     15  * 2. Redistributions in binary form must reproduce the above copyright
     16  *    notice, this list of conditions and the following disclaimer in the
     17  *    documentation and/or other materials provided with the distribution.
     18  * 3. All advertising materials mentioning features or use of this software
     19  *    must display the following acknowledgement:
     20  *	This product includes software developed by the NetBSD
     21  *	Foundation, Inc. and its contributors.
     22  * 4. Neither the name of The NetBSD Foundation nor the names of its
     23  *    contributors may be used to endorse or promote products derived
     24  *    from this software without specific prior written permission.
     25  *
     26  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     27  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     28  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     29  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     30  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     31  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     32  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     33  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     34  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     35  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     36  * POSSIBILITY OF SUCH DAMAGE.
     37  */
     38 
     39 /*
     40  * TODO:
     41  *
     42  * - Tracking of times for sleep locks is broken.
     43  * - Need better analysis and tracking of events.
     44  * - Shouldn't have to parse the namelist here.  We should use something like
     45  *   FreeBSD's libelf.
     46  * - The way the namelist is searched sucks, is it worth doing something
     47  *   better?
     48  */
     49 
     50 #include <sys/cdefs.h>
     51 #ifndef lint
     52 __RCSID("$NetBSD: main.c,v 1.9 2007/07/14 13:30:44 ad Exp $");
     53 #endif /* not lint */
     54 
     55 #include <sys/types.h>
     56 #include <sys/param.h>
     57 #include <sys/time.h>
     58 #include <sys/fcntl.h>
     59 #include <sys/ioctl.h>
     60 #include <sys/wait.h>
     61 #include <sys/signal.h>
     62 #include <sys/sysctl.h>
     63 
     64 #include <dev/lockstat.h>
     65 
     66 #include <stdio.h>
     67 #include <stdlib.h>
     68 #include <string.h>
     69 #include <limits.h>
     70 #include <unistd.h>
     71 #include <err.h>
     72 #include <paths.h>
     73 #include <util.h>
     74 #include <ctype.h>
     75 #include <errno.h>
     76 #include <stdbool.h>
     77 
     78 #include "extern.h"
     79 
     80 #define	_PATH_DEV_LOCKSTAT	"/dev/lockstat"
     81 
     82 #define	MILLI	1000.0
     83 #define	MICRO	1000000.0
     84 #define	NANO	1000000000.0
     85 #define	PICO	1000000000000.0
     86 
     87 TAILQ_HEAD(lock_head, lockstruct);
     88 typedef struct lock_head locklist_t;
     89 TAILQ_HEAD(buf_head, lsbuf);
     90 typedef struct buf_head buflist_t;
     91 
     92 typedef struct lockstruct {
     93 	TAILQ_ENTRY(lockstruct)	chain;
     94 	buflist_t		bufs;
     95 	buflist_t		tosort;
     96 	uintptr_t		lock;
     97  	double			time;
     98 	uint32_t		count;
     99 	u_int			flags;
    100 	u_int			nbufs;
    101 	char			name[NAME_SIZE];
    102 } lock_t;
    103 
    104 typedef struct name {
    105 	const char	*name;
    106 	int		mask;
    107 } name_t;
    108 
    109 const name_t locknames[] = {
    110 	{ "adaptive_mutex", LB_ADAPTIVE_MUTEX },
    111 	{ "spin_mutex", LB_SPIN_MUTEX },
    112 	{ "lockmgr", LB_LOCKMGR },
    113 	{ "rwlock", LB_RWLOCK },
    114 	{ "kernel_lock", LB_KERNEL_LOCK },
    115 	{ NULL, 0 }
    116 };
    117 
    118 const name_t eventnames[] = {
    119 	{ "spin", LB_SPIN },
    120 	{ "sleep_exclusive", LB_SLEEP1 },
    121 	{ "sleep_shared", LB_SLEEP2 },
    122 	{ NULL, 0 },
    123 };
    124 
    125 const name_t alltypes[] = {
    126 	{ "Adaptive mutex spin", LB_ADAPTIVE_MUTEX | LB_SPIN },
    127 	{ "Adaptive mutex sleep", LB_ADAPTIVE_MUTEX | LB_SLEEP1 },
    128 	{ "Spin mutex spin", LB_SPIN_MUTEX | LB_SPIN },
    129 	{ "lockmgr sleep", LB_LOCKMGR | LB_SLEEP1 },
    130 	{ "RW lock sleep (writer)", LB_RWLOCK | LB_SLEEP1 },
    131 	{ "RW lock sleep (reader)", LB_RWLOCK | LB_SLEEP2 },
    132 	{ "Kernel lock spin", LB_KERNEL_LOCK | LB_SPIN },
    133 	{ NULL, 0 }
    134 };
    135 
    136 locklist_t	locklist;
    137 locklist_t	freelist;
    138 locklist_t	sortlist;
    139 
    140 lsbuf_t		*bufs;
    141 lsdisable_t	ld;
    142 bool		lflag;
    143 bool		fflag;
    144 int		nbufs;
    145 bool		cflag;
    146 int		lsfd;
    147 int		displayed;
    148 int		bin64;
    149 double		tscale;
    150 double		cscale;
    151 double		cpuscale[sizeof(ld.ld_freq) / sizeof(ld.ld_freq[0])];
    152 FILE		*outfp;
    153 
    154 void	findsym(findsym_t, char *, uintptr_t *, uintptr_t *, bool);
    155 void	spawn(int, char **);
    156 void	display(int, const char *name);
    157 void	listnames(const name_t *);
    158 void	collapse(bool, bool);
    159 int	matchname(const name_t *, char *);
    160 void	makelists(int, int);
    161 void	nullsig(int);
    162 void	usage(void);
    163 int	ncpu(void);
    164 lock_t	*morelocks(void);
    165 
    166 int
    167 main(int argc, char **argv)
    168 {
    169 	int eventtype, locktype, ch, nlfd, fd, i;
    170 	bool sflag, pflag, mflag, Mflag;
    171 	const char *nlistf, *outf;
    172 	char *lockname, *funcname;
    173 	const name_t *name;
    174 	lsenable_t le;
    175 	double ms;
    176 	char *p;
    177 
    178 	nlistf = NULL;
    179 	outf = NULL;
    180 	lockname = NULL;
    181 	funcname = NULL;
    182 	eventtype = -1;
    183 	locktype = -1;
    184 	nbufs = 0;
    185 	sflag = false;
    186 	pflag = false;
    187 	mflag = false;
    188 	Mflag = false;
    189 
    190 	while ((ch = getopt(argc, argv, "E:F:L:MN:T:b:ceflmo:pst")) != -1)
    191 		switch (ch) {
    192 		case 'E':
    193 			eventtype = matchname(eventnames, optarg);
    194 			break;
    195 		case 'F':
    196 			funcname = optarg;
    197 			break;
    198 		case 'L':
    199 			lockname = optarg;
    200 			break;
    201 		case 'N':
    202 			nlistf = optarg;
    203 			break;
    204 		case 'T':
    205 			locktype = matchname(locknames, optarg);
    206 			break;
    207 		case 'b':
    208 			nbufs = (int)strtol(optarg, &p, 0);
    209 			if (!isdigit((u_int)*optarg) || *p != '\0')
    210 				usage();
    211 			break;
    212 		case 'c':
    213 			cflag = true;
    214 			break;
    215 		case 'e':
    216 			listnames(eventnames);
    217 			break;
    218 		case 'f':
    219 			fflag = true;
    220 			break;
    221 		case 'l':
    222 			lflag = true;
    223 			break;
    224 		case 'm':
    225 			mflag = true;
    226 			break;
    227 		case 'M':
    228 			Mflag = true;
    229 			break;
    230 		case 'o':
    231 			outf = optarg;
    232 			break;
    233 		case 'p':
    234 			pflag = true;
    235 			break;
    236 		case 's':
    237 			sflag = true;
    238 			break;
    239 		case 't':
    240 			listnames(locknames);
    241 			break;
    242 		default:
    243 			usage();
    244 		}
    245 	argc -= optind;
    246 	argv += optind;
    247 
    248 	if (*argv == NULL)
    249 		usage();
    250 
    251 	if (outf) {
    252 		fd = open(outf, O_WRONLY | O_CREAT | O_TRUNC, 0600);
    253 		if (fd == -1)
    254 			err(EXIT_FAILURE, "opening %s", outf);
    255 		outfp = fdopen(fd, "w");
    256 	} else
    257 		outfp = stdout;
    258 
    259 	/*
    260 	 * Find the name list for resolving symbol names, and load it into
    261 	 * memory.
    262 	 */
    263 	if (nlistf == NULL) {
    264 		nlfd = open(_PATH_KSYMS, O_RDONLY);
    265 		nlistf = getbootfile();
    266 	} else
    267 		nlfd = -1;
    268 	if (nlfd == -1) {
    269 		if ((nlfd = open(nlistf, O_RDONLY)) < 0)
    270 			err(EXIT_FAILURE, "cannot open " _PATH_KSYMS " or %s",
    271 			    nlistf);
    272 	}
    273 	if (loadsym32(nlfd) != 0) {
    274 		if (loadsym64(nlfd) != 0)
    275 			errx(EXIT_FAILURE, "unable to load symbol table");
    276 		bin64 = 1;
    277 	}
    278 	close(nlfd);
    279 
    280 	memset(&le, 0, sizeof(le));
    281 	le.le_nbufs = nbufs;
    282 
    283 	/*
    284 	 * Set up initial filtering.
    285 	 */
    286 	if (lockname != NULL) {
    287 		findsym(LOCK_BYNAME, lockname, &le.le_lockstart,
    288 		    &le.le_lockend, true);
    289 		le.le_flags |= LE_ONE_LOCK;
    290 	}
    291 	if (!lflag)
    292 		le.le_flags |= LE_CALLSITE;
    293 	if (!fflag)
    294 		le.le_flags |= LE_LOCK;
    295 	if (funcname != NULL) {
    296 		if (lflag)
    297 			usage();
    298 		findsym(FUNC_BYNAME, funcname, &le.le_csstart, &le.le_csend, true);
    299 		le.le_flags |= LE_ONE_CALLSITE;
    300 	}
    301 	le.le_mask = (eventtype & LB_EVENT_MASK) | (locktype & LB_LOCK_MASK);
    302 
    303 	/*
    304 	 * Start tracing.
    305 	 */
    306 	if ((lsfd = open(_PATH_DEV_LOCKSTAT, O_RDONLY)) < 0)
    307 		err(EXIT_FAILURE, "cannot open " _PATH_DEV_LOCKSTAT);
    308 	if (ioctl(lsfd, IOC_LOCKSTAT_GVERSION, &ch) < 0)
    309 		err(EXIT_FAILURE, "ioctl");
    310 	if (ch != LS_VERSION)
    311 		errx(EXIT_FAILURE,
    312 		    "incompatible lockstat interface version (%d, kernel %d)",
    313 			LS_VERSION, ch);
    314 	if (ioctl(lsfd, IOC_LOCKSTAT_ENABLE, &le))
    315 		err(EXIT_FAILURE, "cannot enable tracing");
    316 
    317 	/*
    318 	 * Execute the traced program.
    319 	 */
    320 	spawn(argc, argv);
    321 
    322 	/*
    323 	 * Stop tracing, and read the trace buffers from the kernel.
    324 	 */
    325 	if (ioctl(lsfd, IOC_LOCKSTAT_DISABLE, &ld) == -1) {
    326 		if (errno == EOVERFLOW) {
    327 			warnx("overflowed available kernel trace buffers");
    328 			exit(EXIT_FAILURE);
    329 		}
    330 		err(EXIT_FAILURE, "cannot disable tracing");
    331 	}
    332 	if ((bufs = malloc(ld.ld_size)) == NULL)
    333 		err(EXIT_FAILURE, "cannot allocate memory for user buffers");
    334 	if (read(lsfd, bufs, ld.ld_size) != ld.ld_size)
    335 		err(EXIT_FAILURE, "reading from " _PATH_DEV_LOCKSTAT);
    336 	if (close(lsfd))
    337 		err(EXIT_FAILURE, "close(" _PATH_DEV_LOCKSTAT ")");
    338 
    339 	/*
    340 	 * Figure out how to scale the results.  For internal use we convert
    341 	 * all times from CPU frequency based to picoseconds, and values are
    342 	 * eventually displayed in ms.
    343 	 */
    344 	for (i = 0; i < sizeof(ld.ld_freq) / sizeof(ld.ld_freq[0]); i++)
    345 		if (ld.ld_freq[i] != 0)
    346 			cpuscale[i] = PICO / ld.ld_freq[i];
    347 	ms = ld.ld_time.tv_sec * MILLI + ld.ld_time.tv_nsec / MICRO;
    348 	if (pflag)
    349 		cscale = 1.0 / ncpu();
    350 	else
    351 		cscale = 1.0;
    352 	cscale *= (sflag ? MILLI / ms : 1.0);
    353 	tscale = cscale / NANO;
    354 	nbufs = (int)(ld.ld_size / sizeof(lsbuf_t));
    355 
    356 	TAILQ_INIT(&locklist);
    357 	TAILQ_INIT(&sortlist);
    358 	TAILQ_INIT(&freelist);
    359 
    360 	if ((mflag | Mflag) != 0)
    361 		collapse(mflag, Mflag);
    362 
    363 	/*
    364 	 * Display the results.
    365 	 */
    366 	fprintf(outfp, "Elapsed time: %.2f seconds.", ms / MILLI);
    367 	if (sflag || pflag) {
    368 		fprintf(outfp, " Displaying ");
    369 		if (pflag)
    370 			fprintf(outfp, "per-CPU ");
    371 		if (sflag)
    372 			fprintf(outfp, "per-second ");
    373 		fprintf(outfp, "averages.");
    374 	}
    375 	putc('\n', outfp);
    376 
    377 	for (name = alltypes; name->name != NULL; name++) {
    378 		if (eventtype != -1 &&
    379 		    (name->mask & LB_EVENT_MASK) != eventtype)
    380 			continue;
    381 		if (locktype != -1 &&
    382 		    (name->mask & LB_LOCK_MASK) != locktype)
    383 			continue;
    384 
    385 		display(name->mask, name->name);
    386 	}
    387 
    388 	if (displayed == 0)
    389 		fprintf(outfp, "None of the selected events were recorded.\n");
    390 	exit(EXIT_SUCCESS);
    391 }
    392 
    393 void
    394 usage(void)
    395 {
    396 
    397 	fprintf(stderr,
    398 	    "%s: usage:\n"
    399 	    "%s [options] <command>\n\n"
    400 	    "-b nbuf\t\tset number of event buffers to allocate\n"
    401 	    "-c\t\treport percentage of total events by count, not time\n"
    402 	    "-E evt\t\tdisplay only one type of event\n"
    403 	    "-e\t\tlist event types\n"
    404 	    "-f\t\ttrace only by function\n"
    405 	    "-F func\t\tlimit trace to one function\n"
    406 	    "-L lock\t\tlimit trace to one lock (name, or address)\n"
    407 	    "-l\t\ttrace only by lock\n"
    408 	    "-m\t\tmerge call sites within unique functions\n"
    409 	    "-M\t\tmerge lock addresses within unique objects\n"
    410 	    "-N nlist\tspecify name list file\n"
    411 	    "-o file\t\tsend output to named file, not stdout\n"
    412 	    "-p\t\tshow average count/time per CPU, not total\n"
    413 	    "-s\t\tshow average count/time per second, not total\n"
    414 	    "-T type\t\tdisplay only one type of lock\n"
    415 	    "-t\t\tlist lock types\n",
    416 	    getprogname(), getprogname());
    417 
    418 	exit(EXIT_FAILURE);
    419 }
    420 
    421 void
    422 nullsig(int junk)
    423 {
    424 
    425 	(void)junk;
    426 }
    427 
    428 void
    429 listnames(const name_t *name)
    430 {
    431 
    432 	for (; name->name != NULL; name++)
    433 		printf("%s\n", name->name);
    434 
    435 	exit(EXIT_SUCCESS);
    436 }
    437 
    438 int
    439 matchname(const name_t *name, char *string)
    440 {
    441 	int empty, mask;
    442 	char *sp;
    443 
    444 	empty = 1;
    445 	mask = 0;
    446 
    447 	while ((sp = strsep(&string, ",")) != NULL) {
    448 		if (*sp == '\0')
    449 			usage();
    450 
    451 		for (; name->name != NULL; name++) {
    452 			if (strcasecmp(name->name, sp) == 0) {
    453 				mask |= name->mask;
    454 				break;
    455 			}
    456 		}
    457 		if (name->name == NULL)
    458 			errx(EXIT_FAILURE, "unknown identifier `%s'", sp);
    459 		empty = 0;
    460 	}
    461 
    462 	if (empty)
    463 		usage();
    464 
    465 	return mask;
    466 }
    467 
    468 /*
    469  * Return the number of CPUs in the running system.
    470  */
    471 int
    472 ncpu(void)
    473 {
    474 	int rv, mib[2];
    475 	size_t varlen;
    476 
    477 	mib[0] = CTL_HW;
    478 	mib[1] = HW_NCPU;
    479 	varlen = sizeof(rv);
    480 	if (sysctl(mib, 2, &rv, &varlen, NULL, (size_t)0) < 0)
    481 		rv = 1;
    482 
    483 	return (rv);
    484 }
    485 
    486 /*
    487  * Call into the ELF parser and look up a symbol by name or by address.
    488  */
    489 void
    490 findsym(findsym_t find, char *name, uintptr_t *start, uintptr_t *end, bool chg)
    491 {
    492 	uintptr_t tend, sa, ea;
    493 	char *p;
    494 	int rv;
    495 
    496 	if (!chg) {
    497 		sa = *start;
    498 		start = &sa;
    499 		end = &ea;
    500 	}
    501 
    502 	if (end == NULL)
    503 		end = &tend;
    504 
    505 	if (find == LOCK_BYNAME) {
    506 		if (isdigit((u_int)name[0])) {
    507 			*start = (uintptr_t)strtoul(name, &p, 0);
    508 			if (*p == '\0')
    509 				return;
    510 		}
    511 	}
    512 
    513 	if (bin64)
    514 		rv = findsym64(find, name, start, end);
    515 	else
    516 		rv = findsym32(find, name, start, end);
    517 
    518 	if (find == FUNC_BYNAME || find == LOCK_BYNAME) {
    519 		if (rv == -1)
    520 			errx(EXIT_FAILURE, "unable to find symbol `%s'", name);
    521 		return;
    522 	}
    523 
    524 	if (rv == -1)
    525 		snprintf(name, NAME_SIZE, "%016lx", (long)*start);
    526 }
    527 
    528 /*
    529  * Fork off the child process and wait for it to complete.  We trap SIGINT
    530  * so that the caller can use Ctrl-C to stop tracing early and still get
    531  * useful results.
    532  */
    533 void
    534 spawn(int argc, char **argv)
    535 {
    536 	pid_t pid;
    537 
    538 	switch (pid = fork()) {
    539 	case 0:
    540 		close(lsfd);
    541 		if (execvp(argv[0], argv) == -1)
    542 			err(EXIT_FAILURE, "cannot exec");
    543 		break;
    544 	case -1:
    545 		err(EXIT_FAILURE, "cannot fork to exec");
    546 		break;
    547 	default:
    548 		signal(SIGINT, nullsig);
    549 		wait(NULL);
    550 		signal(SIGINT, SIG_DFL);
    551 		break;
    552 	}
    553 }
    554 
    555 /*
    556  * Allocate a new block of lock_t structures.
    557  */
    558 lock_t *
    559 morelocks(void)
    560 {
    561 	const static int batch = 32;
    562 	lock_t *l, *lp, *max;
    563 
    564 	l = (lock_t *)malloc(sizeof(*l) * batch);
    565 
    566 	for (lp = l, max = l + batch; lp < max; lp++)
    567 		TAILQ_INSERT_TAIL(&freelist, lp, chain);
    568 
    569 	return l;
    570 }
    571 
    572 /*
    573  * Collapse addresses from unique objects.
    574  */
    575 void
    576 collapse(bool func, bool lock)
    577 {
    578 	lsbuf_t *lb, *max;
    579 
    580 	for (lb = bufs, max = bufs + nbufs; lb < max; lb++) {
    581 		if (func && lb->lb_callsite != 0) {
    582 			findsym(FUNC_BYADDR, NULL, &lb->lb_callsite, NULL,
    583 			    true);
    584 		}
    585 		if (lock && lb->lb_lock != 0) {
    586 			findsym(LOCK_BYADDR, NULL, &lb->lb_lock, NULL,
    587 			    true);
    588 		}
    589 	}
    590 }
    591 
    592 /*
    593  * From the kernel supplied data, construct two dimensional lists of locks
    594  * and event buffers, indexed by lock type and sorted by event type.
    595  */
    596 void
    597 makelists(int mask, int event)
    598 {
    599 	lsbuf_t *lb, *lb2, *max;
    600 	lock_t *l, *l2;
    601 	int type;
    602 
    603 	/*
    604 	 * Recycle lock_t structures from the last run.
    605 	 */
    606 	while ((l = TAILQ_FIRST(&locklist)) != NULL) {
    607 		TAILQ_REMOVE(&locklist, l, chain);
    608 		TAILQ_INSERT_HEAD(&freelist, l, chain);
    609 	}
    610 
    611 	type = mask & LB_LOCK_MASK;
    612 
    613 	for (lb = bufs, max = bufs + nbufs; lb < max; lb++) {
    614 		if ((lb->lb_flags & LB_LOCK_MASK) != type ||
    615 		    lb->lb_counts[event] == 0)
    616 			continue;
    617 
    618 		/*
    619 		 * Look for a record descibing this lock, and allocate a
    620 		 * new one if needed.
    621 		 */
    622 		TAILQ_FOREACH(l, &sortlist, chain) {
    623 			if (l->lock == lb->lb_lock)
    624 				break;
    625 		}
    626 		if (l == NULL) {
    627 			if ((l = TAILQ_FIRST(&freelist)) == NULL)
    628 				l = morelocks();
    629 			TAILQ_REMOVE(&freelist, l, chain);
    630 			l->flags = lb->lb_flags;
    631 			l->lock = lb->lb_lock;
    632 			l->nbufs = 0;
    633 			l->name[0] = '\0';
    634 			l->count = 0;
    635 			l->time = 0;
    636 			TAILQ_INIT(&l->tosort);
    637 			TAILQ_INIT(&l->bufs);
    638 			TAILQ_INSERT_TAIL(&sortlist, l, chain);
    639 		}
    640 
    641 		/*
    642 		 * Scale the time values per buffer and summarise
    643 		 * times+counts per lock.
    644 		 */
    645 		lb->lb_times[event] *= cpuscale[lb->lb_cpu];
    646 		l->count += lb->lb_counts[event];
    647 		l->time += lb->lb_times[event];
    648 
    649 		/*
    650 		 * Merge same lock+callsite pairs from multiple CPUs
    651 		 * together.
    652 		 */
    653 		TAILQ_FOREACH(lb2, &l->tosort, lb_chain.tailq) {
    654 			if (lb->lb_callsite == lb2->lb_callsite)
    655 				break;
    656 		}
    657 		if (lb2 != NULL) {
    658 			lb2->lb_counts[event] += lb->lb_counts[event];
    659 			lb2->lb_times[event] += lb->lb_times[event];
    660 		} else {
    661 			TAILQ_INSERT_HEAD(&l->tosort, lb, lb_chain.tailq);
    662 			l->nbufs++;
    663 		}
    664 	}
    665 
    666 	/*
    667 	 * Now sort the lists.
    668 	 */
    669 	while ((l = TAILQ_FIRST(&sortlist)) != NULL) {
    670 		TAILQ_REMOVE(&sortlist, l, chain);
    671 
    672 		/*
    673 		 * Sort the buffers into the per-lock list.
    674 		 */
    675 		while ((lb = TAILQ_FIRST(&l->tosort)) != NULL) {
    676 			TAILQ_REMOVE(&l->tosort, lb, lb_chain.tailq);
    677 
    678 			lb2 = TAILQ_FIRST(&l->bufs);
    679 			while (lb2 != NULL) {
    680 				if (cflag) {
    681 					if (lb->lb_counts[event] >
    682 					    lb2->lb_counts[event])
    683 						break;
    684 				} else if (lb->lb_times[event] >
    685 				    lb2->lb_times[event])
    686 					break;
    687 				lb2 = TAILQ_NEXT(lb2, lb_chain.tailq);
    688 			}
    689 			if (lb2 == NULL)
    690 				TAILQ_INSERT_TAIL(&l->bufs, lb,
    691 				    lb_chain.tailq);
    692 			else
    693 				TAILQ_INSERT_BEFORE(lb2, lb, lb_chain.tailq);
    694 		}
    695 
    696 		/*
    697 		 * Sort this lock into the per-type list, based on the
    698 		 * totals per lock.
    699 		 */
    700 		l2 = TAILQ_FIRST(&locklist);
    701 		while (l2 != NULL) {
    702 			if (cflag) {
    703 				if (l->count > l2->count)
    704 					break;
    705 			} else if (l->time > l2->time)
    706 				break;
    707 			l2 = TAILQ_NEXT(l2, chain);
    708 		}
    709 		if (l2 == NULL)
    710 			TAILQ_INSERT_TAIL(&locklist, l, chain);
    711 		else
    712 			TAILQ_INSERT_BEFORE(l2, l, chain);
    713 	}
    714 }
    715 
    716 /*
    717  * Display a summary table for one lock type / event type pair.
    718  */
    719 void
    720 display(int mask, const char *name)
    721 {
    722 	lock_t *l;
    723 	lsbuf_t *lb;
    724 	double pcscale, metric;
    725 	char fname[NAME_SIZE];
    726 	int event;
    727 
    728 	event = (mask & LB_EVENT_MASK) - 1;
    729 	makelists(mask, event);
    730 
    731 	if (TAILQ_EMPTY(&locklist))
    732 		return;
    733 
    734 	fprintf(outfp, "\n-- %s\n\n"
    735 	    "Total%%  Count   Time/ms          Lock                       Caller\n"
    736 	    "------ ------- --------- ---------------------- ------------------------------\n",
    737 	    name);
    738 
    739 	/*
    740 	 * Sum up all events for this type of lock + event.
    741 	 */
    742 	pcscale = 0;
    743 	TAILQ_FOREACH(l, &locklist, chain) {
    744 		if (cflag)
    745 			pcscale += l->count;
    746 		else
    747 			pcscale += l->time;
    748 		displayed++;
    749 	}
    750 	if (pcscale == 0)
    751 		pcscale = 100;
    752 	else
    753 		pcscale = (100.0 / pcscale);
    754 
    755 	/*
    756 	 * For each lock, print a summary total, followed by a breakdown by
    757 	 * caller.
    758 	 */
    759 	TAILQ_FOREACH(l, &locklist, chain) {
    760 		if (cflag)
    761 			metric = l->count;
    762 		else
    763 			metric = l->time;
    764 		metric *= pcscale;
    765 
    766 		if (l->name[0] == '\0')
    767 			findsym(LOCK_BYADDR, l->name, &l->lock, NULL, false);
    768 
    769 		if (lflag || l->nbufs > 1)
    770 			fprintf(outfp, "%6.2f %7d %9.2f %-22s <all>\n",
    771 			    metric, (int)(l->count * cscale),
    772 			    l->time * tscale, l->name);
    773 
    774 		if (lflag)
    775 			continue;
    776 
    777 		TAILQ_FOREACH(lb, &l->bufs, lb_chain.tailq) {
    778 			if (cflag)
    779 				metric = lb->lb_counts[event];
    780 			else
    781 				metric = lb->lb_times[event];
    782 			metric *= pcscale;
    783 
    784 			findsym(FUNC_BYADDR, fname, &lb->lb_callsite, NULL,
    785 			    false);
    786 			fprintf(outfp, "%6.2f %7d %9.2f %-22s %s\n",
    787 			    metric, (int)(lb->lb_counts[event] * cscale),
    788 			    lb->lb_times[event] * tscale, l->name, fname);
    789 		}
    790 	}
    791 }
    792