Home | History | Annotate | Line # | Download | only in sys
t_futex_ops.c revision 1.1
      1 /* $NetBSD: t_futex_ops.c,v 1.1 2020/04/26 18:53:33 thorpej Exp $ */
      2 
      3 /*-
      4  * Copyright (c) 2019, 2020 The NetBSD Foundation, Inc.
      5  * All rights reserved.
      6  *
      7  * Redistribution and use in source and binary forms, with or without
      8  * modification, are permitted provided that the following conditions
      9  * are met:
     10  * 1. Redistributions of source code must retain the above copyright
     11  *    notice, this list of conditions and the following disclaimer.
     12  * 2. Redistributions in binary form must reproduce the above copyright
     13  *    notice, this list of conditions and the following disclaimer in the
     14  *    documentation and/or other materials provided with the distribution.
     15  *
     16  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     17  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     18  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     19  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     20  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     21  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     22  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     23  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     24  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     25  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     26  * POSSIBILITY OF SUCH DAMAGE.
     27  */
     28 
     29 #include <sys/cdefs.h>
     30 __COPYRIGHT("@(#) Copyright (c) 2019, 2020\
     31  The NetBSD Foundation, inc. All rights reserved.");
     32 __RCSID("$NetBSD: t_futex_ops.c,v 1.1 2020/04/26 18:53:33 thorpej Exp $");
     33 
     34 #include <sys/fcntl.h>
     35 #include <sys/mman.h>
     36 #include <sys/wait.h>
     37 #include <atomic.h>
     38 #include <errno.h>
     39 #include <lwp.h>
     40 #include <stdlib.h>
     41 #include <stdio.h>
     42 #include <time.h>
     43 #include <limits.h>
     44 #include <unistd.h>
     45 
     46 #include <atf-c.h>
     47 
     48 #include <libc/include/futex_private.h>
     49 
     50 #define	LOAD(x)		(*(volatile int *)(x))
     51 #define	STORE(x, y)	*(volatile int *)(x) = (y)
     52 
     53 #if 0
     54 #define	DPRINTF(x)	printf x
     55 #else
     56 #define	DPRINTF(x)	__nothing
     57 #endif
     58 
     59 #define	STACK_SIZE	65536
     60 
     61 static volatile int futex_word;
     62 static volatile int futex_word1;
     63 
     64 static volatile unsigned int nlwps_running;
     65 
     66 struct lwp_data {
     67 	ucontext_t	context;
     68 	void		(*func)(void *);
     69 	void		*stack_base;
     70 	lwpid_t		lwpid;
     71 	pid_t		child;
     72 	lwpid_t		threadid;
     73 	int		wait_op;
     74 	int		op_flags;
     75 	int		bitset;
     76 	volatile int	*futex_ptr;
     77 	volatile int	*error_ptr;
     78 	int		block_val;
     79 
     80 	void		(*exit_func)(void);
     81 
     82 	int		futex_error;
     83 };
     84 
     85 #define	WAITER_LWP0		0
     86 #define	WAITER_LWP1		1
     87 #define	WAITER_LWP2		2
     88 #define	WAITER_LWP3		3
     89 #define	WAITER_LWP4		4
     90 #define	WAITER_LWP5		5
     91 #define	NLWPS			6
     92 
     93 struct lwp_data lwp_data[NLWPS];
     94 
     95 static const char *bs_path = "t_futex_ops_backing_store";
     96 static int bs_fd = -1;
     97 static int *bs_addr = MAP_FAILED;
     98 static void *bs_source_buffer = NULL;
     99 static void *bs_verify_buffer = NULL;
    100 static long bs_pagesize;
    101 
    102 static void
    103 create_lwp_waiter(struct lwp_data *d)
    104 {
    105 	ATF_REQUIRE(_lwp_create(&d->context, 0, &d->lwpid) == 0);
    106 }
    107 
    108 static void
    109 exit_lwp_waiter(void)
    110 {
    111 	_lwp_exit();
    112 }
    113 
    114 static void
    115 reap_lwp_waiter(struct lwp_data *d)
    116 {
    117 	ATF_REQUIRE(_lwp_wait(d->lwpid, NULL) == 0);
    118 }
    119 
    120 static void
    121 create_proc_waiter(struct lwp_data *d)
    122 {
    123 	pid_t pid;
    124 
    125 	ATF_REQUIRE((pid = fork()) != -1);
    126 	if (pid == 0) {
    127 		(*d->func)(d);
    128 		_exit(666);		/* backstop */
    129 	} else
    130 		d->child = pid;
    131 }
    132 
    133 static void
    134 exit_proc_waiter(void)
    135 {
    136 	_exit(0);
    137 }
    138 
    139 static void
    140 reap_proc_waiter(struct lwp_data *d)
    141 {
    142 	int status;
    143 
    144 	ATF_REQUIRE(waitpid(d->child, &status, 0) == d->child);
    145 	ATF_REQUIRE(WIFEXITED(status));
    146 	ATF_REQUIRE(WEXITSTATUS(status) == 0);
    147 }
    148 
    149 static void
    150 setup_lwp_context(struct lwp_data *d, void (*func)(void *))
    151 {
    152 
    153 	memset(d, 0, sizeof(*d));
    154 	d->stack_base = mmap(NULL, STACK_SIZE, PROT_READ | PROT_WRITE,
    155 	    MAP_ANON | MAP_STACK | MAP_PRIVATE, -1, 0);
    156 	ATF_REQUIRE(d->stack_base != MAP_FAILED);
    157 	_lwp_makecontext(&d->context, func, d, NULL, d->stack_base, STACK_SIZE);
    158 	d->threadid = 0;
    159 	d->func = func;
    160 }
    161 
    162 static void
    163 simple_test_waiter_lwp(void *arg)
    164 {
    165 	struct lwp_data *d = arg;
    166 
    167 	d->threadid = _lwp_self();
    168 
    169 	atomic_inc_uint(&nlwps_running);
    170 	membar_sync();
    171 
    172 	if (__futex(d->futex_ptr, d->wait_op | d->op_flags,
    173 		    d->block_val, NULL, NULL, 0, d->bitset) == -1) {
    174 		d->futex_error = errno;
    175 		_lwp_exit();
    176 	} else {
    177 		d->futex_error = 0;
    178 	}
    179 
    180 	membar_sync();
    181 	atomic_dec_uint(&nlwps_running);
    182 
    183 	_lwp_exit();
    184 }
    185 
    186 static bool
    187 verify_zero_bs(void)
    188 {
    189 
    190 	if (bs_verify_buffer == NULL) {
    191 		bs_verify_buffer = malloc(bs_pagesize);
    192 		ATF_REQUIRE(bs_verify_buffer != NULL);
    193 	}
    194 
    195 	ATF_REQUIRE(pread(bs_fd, bs_verify_buffer,
    196 			  bs_pagesize, 0) == bs_pagesize);
    197 
    198 	return (memcmp(bs_verify_buffer, bs_source_buffer, bs_pagesize) == 0);
    199 }
    200 
    201 static void
    202 create_bs(int map_flags)
    203 {
    204 
    205 	bs_pagesize = sysconf(_SC_PAGESIZE);
    206 	ATF_REQUIRE(bs_pagesize > 0);
    207 
    208 	if ((map_flags & (MAP_FILE | MAP_ANON)) == MAP_FILE) {
    209 		bs_source_buffer = calloc(1, bs_pagesize);
    210 		ATF_REQUIRE(bs_source_buffer != NULL);
    211 
    212 		bs_fd = open(bs_path, O_RDWR | O_CREAT | O_EXCL, 0644);
    213 		ATF_REQUIRE(bs_fd != -1);
    214 
    215 		ATF_REQUIRE(pwrite(bs_fd, bs_source_buffer,
    216 				   bs_pagesize, 0) == bs_pagesize);
    217 		ATF_REQUIRE(verify_zero_bs());
    218 	}
    219 
    220 	bs_addr = mmap(NULL, bs_pagesize, PROT_READ | PROT_WRITE,
    221 		       map_flags | MAP_HASSEMAPHORE, bs_fd, 0);
    222 	ATF_REQUIRE(bs_addr != MAP_FAILED);
    223 }
    224 
    225 static void
    226 cleanup_bs(void)
    227 {
    228 
    229 	if (bs_fd != -1) {
    230 		(void) close(bs_fd);
    231 		bs_fd = -1;
    232 		(void) unlink(bs_path);
    233 	}
    234 	if (bs_source_buffer != NULL) {
    235 		free(bs_source_buffer);
    236 		bs_source_buffer = NULL;
    237 	}
    238 	if (bs_verify_buffer != NULL) {
    239 		free(bs_verify_buffer);
    240 		bs_verify_buffer = NULL;
    241 	}
    242 	if (bs_addr != MAP_FAILED) {
    243 		munmap(bs_addr, bs_pagesize);
    244 		bs_addr = MAP_FAILED;
    245 	}
    246 }
    247 
    248 static void
    249 do_cleanup(void)
    250 {
    251 	int i;
    252 
    253 	for (i = 0; i < NLWPS; i++) {
    254 		struct lwp_data *d = &lwp_data[i];
    255 		if (d->stack_base != NULL && d->stack_base != MAP_FAILED) {
    256 			(void) munmap(d->stack_base, STACK_SIZE);
    257 		}
    258 	}
    259 	memset(lwp_data, 0, sizeof(lwp_data));
    260 	STORE(&futex_word, 0);
    261 	STORE(&futex_word1, 0);
    262 	nlwps_running = 0;
    263 
    264 	cleanup_bs();
    265 }
    266 
    267 /*****************************************************************************/
    268 
    269 static void
    270 wait_wake_test_waiter_lwp(void *arg)
    271 {
    272 	struct lwp_data *d = arg;
    273 
    274 	d->threadid = _lwp_self();
    275 
    276 	STORE(d->futex_ptr, 1);
    277 	membar_sync();
    278 
    279 	/* This will block because *futex_ptr == 1. */
    280 	if (__futex(d->futex_ptr, FUTEX_WAIT | d->op_flags,
    281 		    1, NULL, NULL, 0, 0) == -1) {
    282 		STORE(d->error_ptr, errno);
    283 		(*d->exit_func)();
    284 	} else {
    285 		STORE(d->error_ptr, 0);
    286 	}
    287 
    288 	do {
    289 		membar_sync();
    290 		sleep(1);
    291 	} while (LOAD(d->futex_ptr) != 0);
    292 
    293 	STORE(d->futex_ptr, 2);
    294 	membar_sync();
    295 
    296 	do {
    297 		membar_sync();
    298 		sleep(1);
    299 	} while (LOAD(d->futex_ptr) != 3);
    300 
    301 	/* This will not block because futex_word != 666. */
    302 	if (__futex(d->futex_ptr, FUTEX_WAIT | d->op_flags,
    303 		    666, NULL, NULL, 0, 0) == -1) {
    304 		/* This SHOULD be EAGAIN. */
    305 		STORE(d->error_ptr, errno);
    306 	}
    307 
    308 	STORE(d->futex_ptr, 4);
    309 	membar_sync();
    310 
    311 	(*d->exit_func)();
    312 }
    313 
    314 static void
    315 do_futex_wait_wake_test(volatile int *futex_ptr, volatile int *error_ptr,
    316 			void (*create_func)(struct lwp_data *),
    317 			void (*exit_func)(void),
    318 			void (*reap_func)(struct lwp_data *),
    319 			int flags)
    320 {
    321 	struct lwp_data *wlwp = &lwp_data[WAITER_LWP0];
    322 	int tries;
    323 
    324 	if (error_ptr == NULL)
    325 		error_ptr = &wlwp->futex_error;
    326 
    327 	if (create_func == NULL)
    328 		create_func = create_lwp_waiter;
    329 	if (exit_func == NULL)
    330 		exit_func = exit_lwp_waiter;
    331 	if (reap_func == NULL)
    332 		reap_func = reap_lwp_waiter;
    333 
    334 	setup_lwp_context(wlwp, wait_wake_test_waiter_lwp);
    335 
    336 	DPRINTF(("futex_basic_wait_wake: testing with flags 0x%x\n", flags));
    337 	wlwp->op_flags = flags;
    338 	wlwp->error_ptr = error_ptr;
    339 	STORE(error_ptr, -1);
    340 	wlwp->futex_ptr = futex_ptr;
    341 	STORE(futex_ptr, 0);
    342 	wlwp->exit_func = exit_func;
    343 	membar_sync();
    344 
    345 	DPRINTF(("futex_basic_wait_wake: creating watier LWP\n"));
    346 	(*create_func)(wlwp);
    347 
    348 	DPRINTF(("futex_basic_wait_wake: waiting for LWP %d to enter futex\n",
    349 	    wlwp->lwpid));
    350 	for (tries = 0; tries < 5; tries++) {
    351 		membar_sync();
    352 		if (LOAD(futex_ptr) == 1)
    353 			break;
    354 		sleep(1);
    355 	}
    356 	membar_sync();
    357 	ATF_REQUIRE(LOAD(futex_ptr) == 1);
    358 
    359 	/*
    360 	 * If the LWP is blocked in the futex, it will not have yet
    361 	 * modified *error_ptr.
    362 	 */
    363 	DPRINTF(("futex_basic_wait_wake: checking for successful wait (%d)\n",
    364 	    LOAD(error_ptr)));
    365 	for (tries = 0; tries < 5; tries++) {
    366 		membar_sync();
    367 		if (LOAD(error_ptr) == -1)
    368 			break;
    369 		sleep(1);
    370 	}
    371 	membar_sync();
    372 	ATF_REQUIRE(LOAD(error_ptr) == -1);
    373 
    374 	/* Make sure invalid #wakes in rejected. */
    375 	ATF_REQUIRE_ERRNO(EINVAL,
    376 	    __futex(futex_ptr, FUTEX_WAKE | flags,
    377 		    -1, NULL, NULL, 0, 0) == -1);
    378 
    379 	DPRINTF(("futex_basic_wait_wake: waking 1 waiter\n"));
    380 	ATF_REQUIRE(__futex(futex_ptr, FUTEX_WAKE | flags,
    381 			    1, NULL, NULL, 0, 0) == 1);
    382 
    383 	DPRINTF(("futex_basic_wait_wake: checking for successful wake (%d)\n",
    384 	    LOAD(error_ptr)));
    385 	for (tries = 0; tries < 5; tries++) {
    386 		membar_sync();
    387 		if (LOAD(error_ptr) == 0)
    388 			break;
    389 		sleep(1);
    390 	}
    391 	membar_sync();
    392 	ATF_REQUIRE(LOAD(error_ptr) == 0);
    393 
    394 	STORE(futex_ptr, 0);
    395 	membar_sync();
    396 
    397 	DPRINTF(("futex_basic_wait_wake: waiting for LWP to advance (2)\n"));
    398 	for (tries = 0; tries < 5; tries++) {
    399 		membar_sync();
    400 		if (LOAD(futex_ptr) == 2)
    401 			break;
    402 		sleep(1);
    403 	}
    404 	membar_sync();
    405 	ATF_REQUIRE(LOAD(futex_ptr) == 2);
    406 
    407 	STORE(futex_ptr, 3);
    408 	membar_sync();
    409 
    410 	DPRINTF(("futex_basic_wait_wake: waiting for LWP to advance (4)\n"));
    411 	for (tries = 0; tries < 5; tries++) {
    412 		membar_sync();
    413 		if (LOAD(futex_ptr) == 4)
    414 			break;
    415 		sleep(1);
    416 	}
    417 	membar_sync();
    418 	ATF_REQUIRE(LOAD(futex_ptr) == 4);
    419 
    420 	DPRINTF(("futex_basic_wait_wake: checking for expected EGAIN\n"));
    421 	ATF_REQUIRE(LOAD(error_ptr) == EAGAIN);
    422 
    423 	DPRINTF(("futex_basic_wait_wake: reaping LWP %d\n", wlwp->lwpid));
    424 	(*reap_func)(wlwp);
    425 }
    426 
    427 ATF_TC_WITH_CLEANUP(futex_basic_wait_wake_private);
    428 ATF_TC_HEAD(futex_basic_wait_wake_private, tc)
    429 {
    430 	atf_tc_set_md_var(tc, "descr",
    431 	    "tests basic futex WAIT + WAKE operations (PRIVATE)");
    432 }
    433 ATF_TC_BODY(futex_basic_wait_wake_private, tc)
    434 {
    435 	do_futex_wait_wake_test(&futex_word, NULL,
    436 				NULL, NULL, NULL,
    437 				FUTEX_PRIVATE_FLAG);
    438 }
    439 ATF_TC_CLEANUP(futex_basic_wait_wake_private, tc)
    440 {
    441 	do_cleanup();
    442 }
    443 
    444 ATF_TC_WITH_CLEANUP(futex_basic_wait_wake_shared);
    445 ATF_TC_HEAD(futex_basic_wait_wake_shared, tc)
    446 {
    447 	atf_tc_set_md_var(tc, "descr",
    448 	    "tests basic futex WAIT + WAKE operations (SHARED)");
    449 }
    450 ATF_TC_BODY(futex_basic_wait_wake_shared, tc)
    451 {
    452 	do_futex_wait_wake_test(&futex_word, NULL,
    453 				NULL, NULL, NULL,
    454 				0);
    455 }
    456 ATF_TC_CLEANUP(futex_basic_wait_wake_shared, tc)
    457 {
    458 	do_cleanup();
    459 }
    460 
    461 ATF_TC_WITH_CLEANUP(futex_wait_wake_anon_bs_private);
    462 ATF_TC_HEAD(futex_wait_wake_anon_bs_private, tc)
    463 {
    464 	atf_tc_set_md_var(tc, "descr",
    465 	    "tests futex WAIT + WAKE operations (MAP_ANON + PRIVATE)");
    466 }
    467 ATF_TC_BODY(futex_wait_wake_anon_bs_private, tc)
    468 {
    469 	create_bs(MAP_ANON | MAP_PRIVATE);
    470 	do_futex_wait_wake_test(&bs_addr[0], NULL,
    471 				NULL, NULL, NULL,
    472 				FUTEX_PRIVATE_FLAG);
    473 }
    474 ATF_TC_CLEANUP(futex_wait_wake_anon_bs_private, tc)
    475 {
    476 	do_cleanup();
    477 }
    478 
    479 ATF_TC_WITH_CLEANUP(futex_wait_wake_anon_bs_shared);
    480 ATF_TC_HEAD(futex_wait_wake_anon_bs_shared, tc)
    481 {
    482 	atf_tc_set_md_var(tc, "descr",
    483 	    "tests futex WAIT + WAKE operations (MAP_ANON + SHARED)");
    484 }
    485 ATF_TC_BODY(futex_wait_wake_anon_bs_shared, tc)
    486 {
    487 	create_bs(MAP_ANON | MAP_PRIVATE);
    488 	do_futex_wait_wake_test(&bs_addr[0], NULL,
    489 				NULL, NULL, NULL,
    490 				0);
    491 }
    492 ATF_TC_CLEANUP(futex_wait_wake_anon_bs_shared, tc)
    493 {
    494 	do_cleanup();
    495 }
    496 
    497 ATF_TC_WITH_CLEANUP(futex_wait_wake_file_bs_private);
    498 ATF_TC_HEAD(futex_wait_wake_file_bs_private, tc)
    499 {
    500 	atf_tc_set_md_var(tc, "descr",
    501 	    "tests futex WAIT + WAKE operations (MAP_FILE + PRIVATE)");
    502 }
    503 ATF_TC_BODY(futex_wait_wake_file_bs_private, tc)
    504 {
    505 	/*
    506 	 * This combination (non-COW mapped file + PRIVATE futex)
    507 	 * doesn't really make sense, but we should make sure it
    508 	 * works as expected.
    509 	 */
    510 	create_bs(MAP_FILE | MAP_SHARED);
    511 	do_futex_wait_wake_test(&bs_addr[0], NULL,
    512 				NULL, NULL, NULL,
    513 				FUTEX_PRIVATE_FLAG);
    514 	ATF_REQUIRE(! verify_zero_bs());
    515 }
    516 ATF_TC_CLEANUP(futex_wait_wake_file_bs_private, tc)
    517 {
    518 	do_cleanup();
    519 }
    520 
    521 ATF_TC_WITH_CLEANUP(futex_wait_wake_file_bs_cow_private);
    522 ATF_TC_HEAD(futex_wait_wake_file_bs_cow_private, tc)
    523 {
    524 	atf_tc_set_md_var(tc, "descr",
    525 	    "tests futex WAIT + WAKE operations (MAP_FILE COW + PRIVATE)");
    526 }
    527 ATF_TC_BODY(futex_wait_wake_file_bs_cow_private, tc)
    528 {
    529 	create_bs(MAP_FILE | MAP_PRIVATE);
    530 	do_futex_wait_wake_test(&bs_addr[0], NULL,
    531 				NULL, NULL, NULL,
    532 				FUTEX_PRIVATE_FLAG);
    533 	ATF_REQUIRE(verify_zero_bs());
    534 }
    535 ATF_TC_CLEANUP(futex_wait_wake_file_bs_cow_private, tc)
    536 {
    537 	do_cleanup();
    538 }
    539 
    540 ATF_TC_WITH_CLEANUP(futex_wait_wake_file_bs_shared);
    541 ATF_TC_HEAD(futex_wait_wake_file_bs_shared, tc)
    542 {
    543 	atf_tc_set_md_var(tc, "descr",
    544 	    "tests futex WAIT + WAKE operations (MAP_FILE + SHARED)");
    545 }
    546 ATF_TC_BODY(futex_wait_wake_file_bs_shared, tc)
    547 {
    548 	create_bs(MAP_FILE | MAP_SHARED);
    549 	do_futex_wait_wake_test(&bs_addr[0], NULL,
    550 				NULL, NULL, NULL,
    551 				0);
    552 	ATF_REQUIRE(! verify_zero_bs());
    553 }
    554 ATF_TC_CLEANUP(futex_wait_wake_file_bs_shared, tc)
    555 {
    556 	do_cleanup();
    557 }
    558 
    559 ATF_TC_WITH_CLEANUP(futex_wait_wake_file_bs_cow_shared);
    560 ATF_TC_HEAD(futex_wait_wake_file_bs_cow_shared, tc)
    561 {
    562 	atf_tc_set_md_var(tc, "descr",
    563 	    "tests futex WAIT + WAKE operations (MAP_FILE COW + SHARED)");
    564 }
    565 ATF_TC_BODY(futex_wait_wake_file_bs_cow_shared, tc)
    566 {
    567 	/*
    568 	 * This combination (COW mapped file + SHARED futex)
    569 	 * doesn't really make sense, but we should make sure it
    570 	 * works as expected.
    571 	 */
    572 	create_bs(MAP_FILE | MAP_PRIVATE);
    573 	do_futex_wait_wake_test(&bs_addr[0], NULL,
    574 				NULL, NULL, NULL,
    575 				0);
    576 	ATF_REQUIRE(verify_zero_bs());
    577 }
    578 ATF_TC_CLEANUP(futex_wait_wake_file_bs_cow_shared, tc)
    579 {
    580 	do_cleanup();
    581 }
    582 
    583 ATF_TC_WITH_CLEANUP(futex_wait_wake_anon_bs_shared_proc);
    584 ATF_TC_HEAD(futex_wait_wake_anon_bs_shared_proc, tc)
    585 {
    586 	atf_tc_set_md_var(tc, "descr",
    587 	    "tests multiproc futex WAIT + WAKE operations (MAP_ANON + SHARED)");
    588 }
    589 ATF_TC_BODY(futex_wait_wake_anon_bs_shared_proc, tc)
    590 {
    591 	create_bs(MAP_ANON | MAP_SHARED);
    592 	do_futex_wait_wake_test(&bs_addr[0], &bs_addr[1],
    593 				create_proc_waiter,
    594 				exit_proc_waiter,
    595 				reap_proc_waiter,
    596 				0);
    597 }
    598 ATF_TC_CLEANUP(futex_wait_wake_anon_bs_shared_proc, tc)
    599 {
    600 	do_cleanup();
    601 }
    602 
    603 ATF_TC_WITH_CLEANUP(futex_wait_wake_file_bs_shared_proc);
    604 ATF_TC_HEAD(futex_wait_wake_file_bs_shared_proc, tc)
    605 {
    606 	atf_tc_set_md_var(tc, "descr",
    607 	    "tests multiproc futex WAIT + WAKE operations (MAP_ANON + SHARED)");
    608 }
    609 ATF_TC_BODY(futex_wait_wake_file_bs_shared_proc, tc)
    610 {
    611 	create_bs(MAP_FILE | MAP_SHARED);
    612 	do_futex_wait_wake_test(&bs_addr[0], &bs_addr[1],
    613 				create_proc_waiter,
    614 				exit_proc_waiter,
    615 				reap_proc_waiter,
    616 				0);
    617 }
    618 ATF_TC_CLEANUP(futex_wait_wake_file_bs_shared_proc, tc)
    619 {
    620 	do_cleanup();
    621 }
    622 
    623 /*****************************************************************************/
    624 
    625 ATF_TC(futex_wait_pointless_bitset);
    626 ATF_TC_HEAD(futex_wait_pointless_bitset, tc)
    627 {
    628 	atf_tc_set_md_var(tc, "descr",
    629 	    "tests basic futex WAIT + WAKE operations (SHARED)");
    630 }
    631 ATF_TC_BODY(futex_wait_pointless_bitset, tc)
    632 {
    633 
    634 	futex_word = 1;
    635 	/* This won't block because no bits are set. */
    636 	ATF_REQUIRE(__futex(&futex_word, FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG,
    637 			    1, NULL, NULL, 0, 0) == 0);
    638 }
    639 
    640 static void
    641 do_futex_wait_wake_bitset_test(int flags)
    642 {
    643 	struct lwp_data *wlwp0 = &lwp_data[WAITER_LWP0];
    644 	struct lwp_data *wlwp1 = &lwp_data[WAITER_LWP1];
    645 	int i, tries;
    646 
    647 	for (i = WAITER_LWP0; i <= WAITER_LWP1; i++) {
    648 		setup_lwp_context(&lwp_data[i], simple_test_waiter_lwp);
    649 		lwp_data[i].op_flags = flags;
    650 		lwp_data[i].futex_error = -1;
    651 		lwp_data[i].bitset = __BIT(i);
    652 		lwp_data[i].wait_op = FUTEX_WAIT_BITSET;
    653 		lwp_data[i].futex_ptr = &futex_word;
    654 		lwp_data[i].block_val = 1;
    655 	}
    656 
    657 	STORE(&futex_word, 1);
    658 	membar_sync();
    659 
    660 	ATF_REQUIRE(_lwp_create(&wlwp0->context, 0, &wlwp0->lwpid) == 0);
    661 	ATF_REQUIRE(_lwp_create(&wlwp1->context, 0, &wlwp1->lwpid) == 0);
    662 
    663 	for (tries = 0; tries < 5; tries++) {
    664 		membar_sync();
    665 		if (nlwps_running == 2)
    666 			break;
    667 		sleep(1);
    668 	}
    669 	membar_sync();
    670 	ATF_REQUIRE_EQ_MSG(nlwps_running, 2, "waiters failed to start");
    671 
    672 	/* Ensure they're blocked. */
    673 	ATF_REQUIRE(wlwp0->futex_error == -1);
    674 	ATF_REQUIRE(wlwp1->futex_error == -1);
    675 
    676 	/* Make sure invalid #wakes in rejected. */
    677 	ATF_REQUIRE_ERRNO(EINVAL,
    678 	    __futex(&futex_word, FUTEX_WAKE_BITSET | flags,
    679 		    -1, NULL, NULL, 0, 0) == -1);
    680 
    681 	/* This should result in no wakeups because no bits are set. */
    682 	ATF_REQUIRE(__futex(&futex_word, FUTEX_WAKE_BITSET | flags,
    683 			    INT_MAX, NULL, NULL, 0, 0) == 0);
    684 
    685 	/* This should result in no wakeups because the wrongs bits are set. */
    686 	ATF_REQUIRE(__futex(&futex_word, FUTEX_WAKE_BITSET | flags,
    687 			    INT_MAX, NULL, NULL, 0,
    688 			    ~(wlwp0->bitset | wlwp1->bitset)) == 0);
    689 
    690 	/* Trust, but verify. */
    691 	sleep(1);
    692 	for (tries = 0; tries < 5; tries++) {
    693 		membar_sync();
    694 		if (nlwps_running == 2)
    695 			break;
    696 		sleep(1);
    697 	}
    698 	membar_sync();
    699 	ATF_REQUIRE_EQ_MSG(nlwps_running, 2, "waiters exited unexpectedly");
    700 
    701 	/* Wake up the first LWP. */
    702 	ATF_REQUIRE(__futex(&futex_word, FUTEX_WAKE_BITSET | flags,
    703 			    INT_MAX, NULL, NULL, 0,
    704 			    wlwp0->bitset) == 1);
    705 	sleep(1);
    706 	for (tries = 0; tries < 5; tries++) {
    707 		membar_sync();
    708 		if (nlwps_running == 1)
    709 			break;
    710 		sleep(1);
    711 	}
    712 	membar_sync();
    713 	ATF_REQUIRE(nlwps_running == 1);
    714 	ATF_REQUIRE(wlwp0->futex_error == 0);
    715 	ATF_REQUIRE(_lwp_wait(wlwp0->lwpid, NULL) == 0);
    716 
    717 	/* Wake up the second LWP. */
    718 	ATF_REQUIRE(__futex(&futex_word, FUTEX_WAKE_BITSET | flags,
    719 			    INT_MAX, NULL, NULL, 0,
    720 			    wlwp1->bitset) == 1);
    721 	sleep(1);
    722 	for (tries = 0; tries < 5; tries++) {
    723 		membar_sync();
    724 		if (nlwps_running == 0)
    725 			break;
    726 		sleep(1);
    727 	}
    728 	membar_sync();
    729 	ATF_REQUIRE(nlwps_running == 0);
    730 	ATF_REQUIRE(wlwp1->futex_error == 0);
    731 	ATF_REQUIRE(_lwp_wait(wlwp1->lwpid, NULL) == 0);
    732 }
    733 
    734 ATF_TC_WITH_CLEANUP(futex_wait_wake_bitset);
    735 ATF_TC_HEAD(futex_wait_wake_bitset, tc)
    736 {
    737 	atf_tc_set_md_var(tc, "descr",
    738 	    "tests futex WAIT_BITSET + WAKE_BITSET operations");
    739 }
    740 ATF_TC_BODY(futex_wait_wake_bitset, tc)
    741 {
    742 	do_futex_wait_wake_bitset_test(FUTEX_PRIVATE_FLAG);
    743 }
    744 ATF_TC_CLEANUP(futex_wait_wake_bitset, tc)
    745 {
    746 	do_cleanup();
    747 }
    748 
    749 /*****************************************************************************/
    750 
    751 static void
    752 do_futex_requeue_test(int flags, int op)
    753 {
    754 	struct lwp_data *wlwp0 = &lwp_data[WAITER_LWP0];
    755 	struct lwp_data *wlwp1 = &lwp_data[WAITER_LWP1];
    756 	struct lwp_data *wlwp2 = &lwp_data[WAITER_LWP2];
    757 	struct lwp_data *wlwp3 = &lwp_data[WAITER_LWP3];
    758 	const int good_val3 = (op == FUTEX_CMP_REQUEUE) ?   1 : 0;
    759 	const int bad_val3  = (op == FUTEX_CMP_REQUEUE) ? 666 : 0;
    760 	int i, tries;
    761 
    762 	for (i = WAITER_LWP0; i <= WAITER_LWP3; i++) {
    763 		setup_lwp_context(&lwp_data[i], simple_test_waiter_lwp);
    764 		lwp_data[i].op_flags = flags;
    765 		lwp_data[i].futex_error = -1;
    766 		lwp_data[i].futex_ptr = &futex_word;
    767 		lwp_data[i].block_val = 1;
    768 		lwp_data[i].bitset = 0;
    769 		lwp_data[i].wait_op = FUTEX_WAIT;
    770 	}
    771 
    772 	STORE(&futex_word, 1);
    773 	STORE(&futex_word1, 1);
    774 	membar_sync();
    775 
    776 	ATF_REQUIRE(_lwp_create(&wlwp0->context, 0, &wlwp0->lwpid) == 0);
    777 	ATF_REQUIRE(_lwp_create(&wlwp1->context, 0, &wlwp1->lwpid) == 0);
    778 	ATF_REQUIRE(_lwp_create(&wlwp2->context, 0, &wlwp2->lwpid) == 0);
    779 	ATF_REQUIRE(_lwp_create(&wlwp3->context, 0, &wlwp3->lwpid) == 0);
    780 
    781 	for (tries = 0; tries < 5; tries++) {
    782 		membar_sync();
    783 		if (nlwps_running == 4)
    784 			break;
    785 		sleep(1);
    786 	}
    787 	membar_sync();
    788 	ATF_REQUIRE_EQ_MSG(nlwps_running, 4, "waiters failed to start");
    789 
    790 	/* Ensure they're blocked. */
    791 	ATF_REQUIRE(wlwp0->futex_error == -1);
    792 	ATF_REQUIRE(wlwp1->futex_error == -1);
    793 	ATF_REQUIRE(wlwp2->futex_error == -1);
    794 	ATF_REQUIRE(wlwp3->futex_error == -1);
    795 
    796 	/* Make sure invalid #wakes and #requeues are rejected. */
    797 	ATF_REQUIRE_ERRNO(EINVAL,
    798 	    __futex(&futex_word, op | flags,
    799 		    -1, NULL, &futex_word1, INT_MAX, bad_val3) == -1);
    800 
    801 	ATF_REQUIRE_ERRNO(EINVAL,
    802 	    __futex(&futex_word, op | flags,
    803 		    0, NULL, &futex_word1, -1, bad_val3) == -1);
    804 
    805 	/*
    806 	 * FUTEX 0: 4 LWPs
    807 	 * FUTEX 1: 0 LWPs
    808 	 */
    809 
    810 	if (op == FUTEX_CMP_REQUEUE) {
    811 		/* This should fail because the futex_word value is 1. */
    812 		ATF_REQUIRE_ERRNO(EAGAIN,
    813 		    __futex(&futex_word, op | flags,
    814 			    0, NULL, &futex_word1, INT_MAX, bad_val3) == -1);
    815 	}
    816 
    817 	/*
    818 	 * FUTEX 0: 4 LWPs
    819 	 * FUTEX 1: 0 LWPs
    820 	 */
    821 
    822 	/* Move all waiters from 0 to 1. */
    823 	ATF_REQUIRE(__futex(&futex_word, op | flags,
    824 			    0, NULL, &futex_word1, INT_MAX, good_val3) == 0);
    825 
    826 	/*
    827 	 * FUTEX 0: 0 LWPs
    828 	 * FUTEX 1: 4 LWPs
    829 	 */
    830 
    831 	if (op == FUTEX_CMP_REQUEUE) {
    832 		/* This should fail because the futex_word1 value is 1. */
    833 		ATF_REQUIRE_ERRNO(EAGAIN,
    834 		    __futex(&futex_word1, op | flags,
    835 			    1, NULL, &futex_word, 1, bad_val3) == -1);
    836 	}
    837 
    838 	/*
    839 	 * FUTEX 0: 0 LWPs
    840 	 * FUTEX 1: 4 LWPs
    841 	 */
    842 
    843 	/* Wake one waiter on 1, move one waiter to 0. */
    844 	ATF_REQUIRE(__futex(&futex_word1, op | flags,
    845 			    1, NULL, &futex_word, 1, good_val3) == 1);
    846 
    847 	/*
    848 	 * FUTEX 0: 1 LWP
    849 	 * FUTEX 1: 2 LWPs
    850 	 */
    851 
    852 	/* Wake all waiters on 0 (should be 1). */
    853 	ATF_REQUIRE(__futex(&futex_word, FUTEX_WAKE | flags,
    854 			    INT_MAX, NULL, NULL, 0, 0) == 1);
    855 
    856 	/* Wake all waiters on 1 (should be 2). */
    857 	ATF_REQUIRE(__futex(&futex_word1, FUTEX_WAKE | flags,
    858 			    INT_MAX, NULL, NULL, 0, 0) == 2);
    859 
    860 	/* Trust, but verify. */
    861 	sleep(1);
    862 	for (tries = 0; tries < 5; tries++) {
    863 		membar_sync();
    864 		if (nlwps_running == 0)
    865 			break;
    866 		sleep(1);
    867 	}
    868 	membar_sync();
    869 	ATF_REQUIRE_EQ_MSG(nlwps_running, 0, "waiters failed to exit");
    870 
    871 	ATF_REQUIRE(_lwp_wait(wlwp0->lwpid, NULL) == 0);
    872 	ATF_REQUIRE(_lwp_wait(wlwp1->lwpid, NULL) == 0);
    873 	ATF_REQUIRE(_lwp_wait(wlwp2->lwpid, NULL) == 0);
    874 	ATF_REQUIRE(_lwp_wait(wlwp3->lwpid, NULL) == 0);
    875 }
    876 
    877 ATF_TC_WITH_CLEANUP(futex_requeue);
    878 ATF_TC_HEAD(futex_requeue, tc)
    879 {
    880 	atf_tc_set_md_var(tc, "descr",
    881 	    "tests futex REQUEUE operations");
    882 }
    883 ATF_TC_BODY(futex_requeue, tc)
    884 {
    885 	do_futex_requeue_test(FUTEX_PRIVATE_FLAG, FUTEX_REQUEUE);
    886 }
    887 ATF_TC_CLEANUP(futex_requeue, tc)
    888 {
    889 	do_cleanup();
    890 }
    891 
    892 ATF_TC_WITH_CLEANUP(futex_cmp_requeue);
    893 ATF_TC_HEAD(futex_cmp_requeue, tc)
    894 {
    895 	atf_tc_set_md_var(tc, "descr",
    896 	    "tests futex CMP_REQUEUE operations");
    897 }
    898 ATF_TC_BODY(futex_cmp_requeue, tc)
    899 {
    900 	do_futex_requeue_test(FUTEX_PRIVATE_FLAG, FUTEX_CMP_REQUEUE);
    901 }
    902 ATF_TC_CLEANUP(futex_cmp_requeue, tc)
    903 {
    904 	do_cleanup();
    905 }
    906 
    907 /*****************************************************************************/
    908 
    909 static void
    910 do_futex_wake_op_op_test(int flags)
    911 {
    912 	int op;
    913 
    914 	futex_word = 0;
    915 	futex_word1 = 0;
    916 
    917 	/*
    918 	 * The op= operations should work even if there are no waiters.
    919 	 */
    920 
    921 	/*
    922 	 * Because these operations use both futex addresses, exercise
    923 	 * rejecting unaligned futex addresses here.
    924 	 */
    925 	op = FUTEX_OP(FUTEX_OP_SET, 1, FUTEX_OP_CMP_EQ, 0);
    926 	ATF_REQUIRE_ERRNO(EINVAL,
    927 	    __futex((int *)1, FUTEX_WAKE_OP | flags,
    928 		    0, NULL, &futex_word1, 0, op) == -1);
    929 	ATF_REQUIRE(futex_word1 == 0);
    930 
    931 	ATF_REQUIRE_ERRNO(EINVAL,
    932 	    __futex(&futex_word, FUTEX_WAKE_OP | flags,
    933 		    0, NULL, (int *)1, 0, op) == -1);
    934 	ATF_REQUIRE(futex_word == 0);
    935 
    936 	/* Check unmapped uaddr2 handling, too. */
    937 	ATF_REQUIRE_ERRNO(EFAULT,
    938 	    __futex(&futex_word, FUTEX_WAKE_OP | flags,
    939 		    0, NULL, NULL, 0, op) == -1);
    940 	ATF_REQUIRE(futex_word == 0);
    941 
    942 	op = FUTEX_OP(FUTEX_OP_SET, 1, FUTEX_OP_CMP_EQ, 0);
    943 	ATF_REQUIRE(__futex(&futex_word, FUTEX_WAKE_OP | flags,
    944 			    0, NULL, &futex_word1, 0, op) == 0);
    945 	ATF_REQUIRE(futex_word1 == 1);
    946 
    947 	op = FUTEX_OP(FUTEX_OP_ADD, 1, FUTEX_OP_CMP_EQ, 0);
    948 	ATF_REQUIRE(__futex(&futex_word, FUTEX_WAKE_OP | flags,
    949 			    0, NULL, &futex_word1, 0, op) == 0);
    950 	ATF_REQUIRE(futex_word1 == 2);
    951 
    952 	op = FUTEX_OP(FUTEX_OP_OR, 2, FUTEX_OP_CMP_EQ, 0);
    953 	ATF_REQUIRE(__futex(&futex_word, FUTEX_WAKE_OP | flags,
    954 			    0, NULL, &futex_word1, 0, op) == 0);
    955 	ATF_REQUIRE(futex_word1 == 2);
    956 
    957 	/* This should fail because of invalid shift value 32. */
    958 	op = FUTEX_OP(FUTEX_OP_OR | FUTEX_OP_OPARG_SHIFT, 32,
    959 		      FUTEX_OP_CMP_EQ, 0);
    960 	ATF_REQUIRE_ERRNO(EINVAL,
    961 	    __futex(&futex_word, FUTEX_WAKE_OP | flags,
    962 		    0, NULL, &futex_word1, 0, op) == -1);
    963 	ATF_REQUIRE(futex_word1 == 2);
    964 
    965 	op = FUTEX_OP(FUTEX_OP_OR | FUTEX_OP_OPARG_SHIFT, 31,
    966 		      FUTEX_OP_CMP_EQ, 0);
    967 	ATF_REQUIRE(__futex(&futex_word, FUTEX_WAKE_OP | flags,
    968 			    0, NULL, &futex_word1, 0, op) == 0);
    969 	ATF_REQUIRE(futex_word1 == (int)0x80000002);
    970 
    971 	op = FUTEX_OP(FUTEX_OP_ANDN | FUTEX_OP_OPARG_SHIFT, 31,
    972 		      FUTEX_OP_CMP_EQ, 0);
    973 	ATF_REQUIRE(__futex(&futex_word, FUTEX_WAKE_OP | flags,
    974 			    0, NULL, &futex_word1, 0, op) == 0);
    975 	ATF_REQUIRE(futex_word1 == 2);
    976 
    977 	op = FUTEX_OP(FUTEX_OP_XOR, 2, FUTEX_OP_CMP_EQ, 0);
    978 	ATF_REQUIRE(__futex(&futex_word, FUTEX_WAKE_OP | flags,
    979 			    0, NULL, &futex_word1, 0, op) == 0);
    980 	ATF_REQUIRE(futex_word1 == 0);
    981 }
    982 
    983 ATF_TC_WITH_CLEANUP(futex_wake_op_op);
    984 ATF_TC_HEAD(futex_wake_op_op, tc)
    985 {
    986 	atf_tc_set_md_var(tc, "descr",
    987 	    "tests futex WAKE_OP OP operations");
    988 }
    989 ATF_TC_BODY(futex_wake_op_op, tc)
    990 {
    991 	do_futex_wake_op_op_test(FUTEX_PRIVATE_FLAG);
    992 }
    993 ATF_TC_CLEANUP(futex_wake_op_op, tc)
    994 {
    995 	do_cleanup();
    996 }
    997 
    998 static void
    999 create_wake_op_test_lwps(int flags)
   1000 {
   1001 	int i;
   1002 
   1003 	futex_word1 = 0;
   1004 	membar_sync();
   1005 
   1006 	for (i = WAITER_LWP0; i <= WAITER_LWP5; i++) {
   1007 		setup_lwp_context(&lwp_data[i], simple_test_waiter_lwp);
   1008 		lwp_data[i].op_flags = flags;
   1009 		lwp_data[i].futex_error = -1;
   1010 		lwp_data[i].futex_ptr = &futex_word1;
   1011 		lwp_data[i].block_val = 0;
   1012 		lwp_data[i].bitset = 0;
   1013 		lwp_data[i].wait_op = FUTEX_WAIT;
   1014 		ATF_REQUIRE(_lwp_create(&lwp_data[i].context, 0,
   1015 					&lwp_data[i].lwpid) == 0);
   1016 	}
   1017 
   1018 	for (i = 0; i < 5; i++) {
   1019 		membar_sync();
   1020 		if (nlwps_running == 6)
   1021 			break;
   1022 		sleep(1);
   1023 	}
   1024 	membar_sync();
   1025 	ATF_REQUIRE_EQ_MSG(nlwps_running, 6, "waiters failed to start");
   1026 
   1027 	/* Ensure they're blocked. */
   1028 	for (i = WAITER_LWP0; i <= WAITER_LWP5; i++) {
   1029 		ATF_REQUIRE(lwp_data[i].futex_error == -1);
   1030 	}
   1031 }
   1032 
   1033 static void
   1034 reap_wake_op_test_lwps(void)
   1035 {
   1036 	int i;
   1037 
   1038 	for (i = WAITER_LWP0; i <= WAITER_LWP5; i++) {
   1039 		ATF_REQUIRE(_lwp_wait(lwp_data[i].lwpid, NULL) == 0);
   1040 	}
   1041 }
   1042 
   1043 static void
   1044 do_futex_wake_op_cmp_test(int flags)
   1045 {
   1046 	int tries, op;
   1047 
   1048 	futex_word = 0;
   1049 	membar_sync();
   1050 
   1051 	/*
   1052 	 * Verify and negative and positive for each individual
   1053 	 * compare.
   1054 	 */
   1055 
   1056 	create_wake_op_test_lwps(flags);
   1057 
   1058 	/* #LWPs = 6 */
   1059 	op = FUTEX_OP(FUTEX_OP_SET, 0, FUTEX_OP_CMP_EQ, 1);
   1060 	ATF_REQUIRE(__futex(&futex_word, FUTEX_WAKE_OP | flags,
   1061 			    0, NULL, &futex_word1, 1, op) == 0);
   1062 	ATF_REQUIRE(futex_word1 == 0);
   1063 
   1064 	op = FUTEX_OP(FUTEX_OP_SET, 1, FUTEX_OP_CMP_EQ, 0);
   1065 	ATF_REQUIRE(__futex(&futex_word, FUTEX_WAKE_OP | flags,
   1066 			    0, NULL, &futex_word1, 1, op) == 1);
   1067 	ATF_REQUIRE(futex_word1 == 1);
   1068 
   1069 	/* #LWPs = 5 */
   1070 	op = FUTEX_OP(FUTEX_OP_SET, 1, FUTEX_OP_CMP_NE, 1);
   1071 	ATF_REQUIRE(__futex(&futex_word, FUTEX_WAKE_OP | flags,
   1072 			    0, NULL, &futex_word1, 1, op) == 0);
   1073 	ATF_REQUIRE(futex_word1 == 1);
   1074 
   1075 	op = FUTEX_OP(FUTEX_OP_SET, 2, FUTEX_OP_CMP_NE, 2);
   1076 	ATF_REQUIRE(__futex(&futex_word, FUTEX_WAKE_OP | flags,
   1077 			    0, NULL, &futex_word1, 1, op) == 1);
   1078 	ATF_REQUIRE(futex_word1 == 2);
   1079 
   1080 	/* #LWPs = 4 */
   1081 	op = FUTEX_OP(FUTEX_OP_SET, 2, FUTEX_OP_CMP_LT, 2);
   1082 	ATF_REQUIRE(__futex(&futex_word, FUTEX_WAKE_OP | flags,
   1083 			    0, NULL, &futex_word1, 1, op) == 0);
   1084 	ATF_REQUIRE(futex_word1 == 2);
   1085 
   1086 	op = FUTEX_OP(FUTEX_OP_SET, 2, FUTEX_OP_CMP_LT, 3);
   1087 	ATF_REQUIRE(__futex(&futex_word, FUTEX_WAKE_OP | flags,
   1088 			    0, NULL, &futex_word1, 1, op) == 1);
   1089 	ATF_REQUIRE(futex_word1 == 2);
   1090 
   1091 	/* #LWPs = 3 */
   1092 	op = FUTEX_OP(FUTEX_OP_SET, 1, FUTEX_OP_CMP_LE, 1);
   1093 	ATF_REQUIRE(__futex(&futex_word, FUTEX_WAKE_OP | flags,
   1094 			    0, NULL, &futex_word1, 1, op) == 0);
   1095 	ATF_REQUIRE(futex_word1 == 1);
   1096 
   1097 	op = FUTEX_OP(FUTEX_OP_SET, 1, FUTEX_OP_CMP_LE, 1);
   1098 	ATF_REQUIRE(__futex(&futex_word, FUTEX_WAKE_OP | flags,
   1099 			    0, NULL, &futex_word1, 1, op) == 1);
   1100 	ATF_REQUIRE(futex_word1 == 1);
   1101 
   1102 	/* #LWPs = 2 */
   1103 	op = FUTEX_OP(FUTEX_OP_SET, 3, FUTEX_OP_CMP_GT, 3);
   1104 	ATF_REQUIRE(__futex(&futex_word, FUTEX_WAKE_OP | flags,
   1105 			    0, NULL, &futex_word1, 1, op) == 0);
   1106 	ATF_REQUIRE(futex_word1 == 3);
   1107 
   1108 	op = FUTEX_OP(FUTEX_OP_SET, 2, FUTEX_OP_CMP_GT, 2);
   1109 	ATF_REQUIRE(__futex(&futex_word, FUTEX_WAKE_OP | flags,
   1110 			    0, NULL, &futex_word1, 1, op) == 1);
   1111 	ATF_REQUIRE(futex_word1 == 2);
   1112 
   1113 	/* #LWPs = 1 */
   1114 	op = FUTEX_OP(FUTEX_OP_SET, 3, FUTEX_OP_CMP_GE, 4);
   1115 	ATF_REQUIRE(__futex(&futex_word, FUTEX_WAKE_OP | flags,
   1116 			    0, NULL, &futex_word1, 1, op) == 0);
   1117 	ATF_REQUIRE(futex_word1 == 3);
   1118 
   1119 	op = FUTEX_OP(FUTEX_OP_SET, 2, FUTEX_OP_CMP_GE, 3);
   1120 	ATF_REQUIRE(__futex(&futex_word, FUTEX_WAKE_OP | flags,
   1121 			    0, NULL, &futex_word1, 1, op) == 1);
   1122 	ATF_REQUIRE(futex_word1 == 2);
   1123 
   1124 	/* #LWPs = 0 */
   1125 
   1126 	/* Trust, but verify. */
   1127 	sleep(1);
   1128 	for (tries = 0; tries < 5; tries++) {
   1129 		membar_sync();
   1130 		if (nlwps_running == 0)
   1131 			break;
   1132 		sleep(1);
   1133 	}
   1134 	membar_sync();
   1135 	ATF_REQUIRE_EQ_MSG(nlwps_running, 0, "waiters failed to exit");
   1136 
   1137 	reap_wake_op_test_lwps();
   1138 
   1139 	/*
   1140 	 * Verify wakes on uaddr work even if the uaddr2 comparison
   1141 	 * fails.
   1142 	 */
   1143 
   1144 	create_wake_op_test_lwps(flags);
   1145 
   1146 	/* #LWPs = 6 */
   1147 	ATF_REQUIRE(futex_word == 0);
   1148 	op = FUTEX_OP(FUTEX_OP_SET, 0, FUTEX_OP_CMP_EQ, 666);
   1149 	ATF_REQUIRE(__futex(&futex_word1, FUTEX_WAKE_OP | flags,
   1150 			    INT_MAX, NULL, &futex_word, 0, op) == 6);
   1151 	ATF_REQUIRE(futex_word == 0);
   1152 
   1153 	/* #LWPs = 0 */
   1154 
   1155 	/* Trust, but verify. */
   1156 	sleep(1);
   1157 	for (tries = 0; tries < 5; tries++) {
   1158 		membar_sync();
   1159 		if (nlwps_running == 0)
   1160 			break;
   1161 		sleep(1);
   1162 	}
   1163 	membar_sync();
   1164 	ATF_REQUIRE_EQ_MSG(nlwps_running, 0, "waiters failed to exit");
   1165 
   1166 	reap_wake_op_test_lwps();
   1167 }
   1168 
   1169 ATF_TC_WITH_CLEANUP(futex_wake_op_cmp);
   1170 ATF_TC_HEAD(futex_wake_op_cmp, tc)
   1171 {
   1172 	atf_tc_set_md_var(tc, "descr",
   1173 	    "tests futex WAKE_OP CMP operations");
   1174 }
   1175 ATF_TC_BODY(futex_wake_op_cmp, tc)
   1176 {
   1177 	do_futex_wake_op_cmp_test(FUTEX_PRIVATE_FLAG);
   1178 }
   1179 ATF_TC_CLEANUP(futex_wake_op_cmp, tc)
   1180 {
   1181 	do_cleanup();
   1182 }
   1183 
   1184 /*****************************************************************************/
   1185 
   1186 static void
   1187 do_futex_wait_timeout(bool relative, clockid_t clock)
   1188 {
   1189 	struct timespec ts;
   1190 	struct timespec deadline;
   1191 	int op = relative ? FUTEX_WAIT : FUTEX_WAIT_BITSET;
   1192 
   1193 	if (clock == CLOCK_REALTIME)
   1194 		op |= FUTEX_CLOCK_REALTIME;
   1195 
   1196 	ATF_REQUIRE(clock_gettime(clock, &deadline) == 0);
   1197 	deadline.tv_sec += 2;
   1198 	if (relative) {
   1199 		ts.tv_sec = 2;
   1200 		ts.tv_nsec = 0;
   1201 	} else {
   1202 		ts = deadline;
   1203 	}
   1204 
   1205 	futex_word = 1;
   1206 	ATF_REQUIRE_ERRNO(ETIMEDOUT,
   1207 	    __futex(&futex_word, op | FUTEX_PRIVATE_FLAG,
   1208 		    1, &ts, NULL, 0, FUTEX_BITSET_MATCH_ANY) == -1);
   1209 
   1210 	/* Can't reliably check CLOCK_REALTIME in the presence of NTP. */
   1211 	if (clock != CLOCK_REALTIME) {
   1212 		ATF_REQUIRE(clock_gettime(clock, &ts) == 0);
   1213 		ATF_REQUIRE(ts.tv_sec >= deadline.tv_sec);
   1214 		ATF_REQUIRE(ts.tv_sec > deadline.tv_sec ||
   1215 			    ts.tv_nsec >= deadline.tv_nsec);
   1216 	}
   1217 }
   1218 
   1219 ATF_TC(futex_wait_timeout_relative);
   1220 ATF_TC_HEAD(futex_wait_timeout_relative, tc)
   1221 {
   1222 	atf_tc_set_md_var(tc, "descr",
   1223 	    "tests futex WAIT with relative timeout");
   1224 }
   1225 ATF_TC_BODY(futex_wait_timeout_relative, tc)
   1226 {
   1227 	do_futex_wait_timeout(true, CLOCK_MONOTONIC);
   1228 }
   1229 
   1230 ATF_TC(futex_wait_timeout_relative_rt);
   1231 ATF_TC_HEAD(futex_wait_timeout_relative_rt, tc)
   1232 {
   1233 	atf_tc_set_md_var(tc, "descr",
   1234 	    "tests futex WAIT with relative timeout (REALTIME)");
   1235 }
   1236 ATF_TC_BODY(futex_wait_timeout_relative_rt, tc)
   1237 {
   1238 	do_futex_wait_timeout(true, CLOCK_REALTIME);
   1239 }
   1240 
   1241 ATF_TC(futex_wait_timeout_deadline);
   1242 ATF_TC_HEAD(futex_wait_timeout_deadline, tc)
   1243 {
   1244 	atf_tc_set_md_var(tc, "descr",
   1245 	    "tests futex WAIT with absolute deadline");
   1246 }
   1247 ATF_TC_BODY(futex_wait_timeout_deadline, tc)
   1248 {
   1249 	do_futex_wait_timeout(false, CLOCK_MONOTONIC);
   1250 }
   1251 
   1252 ATF_TC(futex_wait_timeout_deadline_rt);
   1253 ATF_TC_HEAD(futex_wait_timeout_deadline_rt, tc)
   1254 {
   1255 	atf_tc_set_md_var(tc, "descr",
   1256 	    "tests futex WAIT with absolute deadline (REALTIME)");
   1257 }
   1258 ATF_TC_BODY(futex_wait_timeout_deadline_rt, tc)
   1259 {
   1260 	do_futex_wait_timeout(false, CLOCK_REALTIME);
   1261 }
   1262 
   1263 /*****************************************************************************/
   1264 
   1265 ATF_TP_ADD_TCS(tp)
   1266 {
   1267 	ATF_TP_ADD_TC(tp, futex_basic_wait_wake_private);
   1268 	ATF_TP_ADD_TC(tp, futex_basic_wait_wake_shared);
   1269 	ATF_TP_ADD_TC(tp, futex_wait_wake_anon_bs_private);
   1270 	ATF_TP_ADD_TC(tp, futex_wait_wake_anon_bs_shared);
   1271 	ATF_TP_ADD_TC(tp, futex_wait_wake_file_bs_private);
   1272 	ATF_TP_ADD_TC(tp, futex_wait_wake_file_bs_shared);
   1273 	ATF_TP_ADD_TC(tp, futex_wait_wake_file_bs_cow_private);
   1274 	ATF_TP_ADD_TC(tp, futex_wait_wake_file_bs_cow_shared);
   1275 
   1276 	ATF_TP_ADD_TC(tp, futex_wait_wake_anon_bs_shared_proc);
   1277 	ATF_TP_ADD_TC(tp, futex_wait_wake_file_bs_shared_proc);
   1278 
   1279 	ATF_TP_ADD_TC(tp, futex_wait_pointless_bitset);
   1280 	ATF_TP_ADD_TC(tp, futex_wait_wake_bitset);
   1281 
   1282 	ATF_TP_ADD_TC(tp, futex_wait_timeout_relative);
   1283 	ATF_TP_ADD_TC(tp, futex_wait_timeout_relative_rt);
   1284 	ATF_TP_ADD_TC(tp, futex_wait_timeout_deadline);
   1285 	ATF_TP_ADD_TC(tp, futex_wait_timeout_deadline_rt);
   1286 
   1287 	ATF_TP_ADD_TC(tp, futex_requeue);
   1288 	ATF_TP_ADD_TC(tp, futex_cmp_requeue);
   1289 
   1290 	ATF_TP_ADD_TC(tp, futex_wake_op_op);
   1291 	ATF_TP_ADD_TC(tp, futex_wake_op_cmp);
   1292 
   1293 	return atf_no_error();
   1294 }
   1295