Home | History | Annotate | Line # | Download | only in rumpvfs
rumpblk.c revision 1.23 
      1 /*	$NetBSD: rumpblk.c,v 1.23 2009/04/30 17:43:07 pooka Exp $	*/
      2 
      3 /*
      4  * Copyright (c) 2009 Antti Kantee.  All Rights Reserved.
      5  *
      6  * Development of this software was supported by the
      7  * Finnish Cultural Foundation.
      8  *
      9  * Redistribution and use in source and binary forms, with or without
     10  * modification, are permitted provided that the following conditions
     11  * are met:
     12  * 1. Redistributions of source code must retain the above copyright
     13  *    notice, this list of conditions and the following disclaimer.
     14  * 2. Redistributions in binary form must reproduce the above copyright
     15  *    notice, this list of conditions and the following disclaimer in the
     16  *    documentation and/or other materials provided with the distribution.
     17  *
     18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
     19  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
     20  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
     21  * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
     22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
     24  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     28  * SUCH DAMAGE.
     29  */
     30 
     31 /*
     32  * Block device emulation.  Presents a block device interface and
     33  * uses rumpuser system calls to satisfy I/O requests.
     34  *
     35  * We provide fault injection.  The driver can be made to fail
     36  * I/O occasionally.
     37  *
     38  * The driver also provides an optimization for regular files by
     39  * using memory-mapped I/O.  This avoids kernel access for every
     40  * I/O operation.  It also gives finer-grained control of how to
     41  * flush data.  Additionally, in case the rump kernel dumps core,
     42  * we get way less carnage.
     43  */
     44 
     45 #include <sys/cdefs.h>
     46 __KERNEL_RCSID(0, "$NetBSD: rumpblk.c,v 1.23 2009/04/30 17:43:07 pooka Exp $");
     47 
     48 #include <sys/param.h>
     49 #include <sys/buf.h>
     50 #include <sys/conf.h>
     51 #include <sys/condvar.h>
     52 #include <sys/disklabel.h>
     53 #include <sys/evcnt.h>
     54 #include <sys/fcntl.h>
     55 #include <sys/kmem.h>
     56 #include <sys/malloc.h>
     57 #include <sys/queue.h>
     58 #include <sys/stat.h>
     59 
     60 #include <rump/rumpuser.h>
     61 
     62 #include "rump_private.h"
     63 #include "rump_vfs_private.h"
     64 
     65 #if 0
     66 #define DPRINTF(x) printf x
     67 #else
     68 #define DPRINTF(x)
     69 #endif
     70 
     71 /* Default: 16 x 1MB windows */
     72 unsigned memwinsize = (1<<20);
     73 unsigned memwincnt = 16;
     74 
     75 #define STARTWIN(off)		((off) & ~(memwinsize-1))
     76 #define INWIN(win,off)		((win)->win_off == STARTWIN(off))
     77 #define WINSIZE(rblk, win)	(MIN((rblk->rblk_size-win->win_off),memwinsize))
     78 #define WINVALID(win)		((win)->win_off != (off_t)-1)
     79 #define WINVALIDATE(win)	((win)->win_off = (off_t)-1)
     80 struct blkwin {
     81 	off_t win_off;
     82 	void *win_mem;
     83 	int win_refcnt;
     84 
     85 	TAILQ_ENTRY(blkwin) win_lru;
     86 };
     87 
     88 #define RUMPBLK_SIZE 16
     89 static struct rblkdev {
     90 	char *rblk_path;
     91 	int rblk_fd;
     92 #ifdef HAS_ODIRECT
     93 	int rblk_dfd;
     94 #endif
     95 
     96 	/* for mmap */
     97 	int rblk_mmflags;
     98 	kmutex_t rblk_memmtx;
     99 	kcondvar_t rblk_memcv;
    100 	TAILQ_HEAD(winlru, blkwin) rblk_lruq;
    101 	size_t rblk_size;
    102 	bool rblk_waiting;
    103 
    104 	struct partition *rblk_curpi;
    105 	struct partition rblk_pi;
    106 	struct disklabel rblk_dl;
    107 } minors[RUMPBLK_SIZE];
    108 
    109 static struct evcnt ev_io_total;
    110 static struct evcnt ev_io_async;
    111 
    112 static struct evcnt ev_memblk_hits;
    113 static struct evcnt ev_memblk_busy;
    114 
    115 static struct evcnt ev_bwrite_total;
    116 static struct evcnt ev_bwrite_async;
    117 static struct evcnt ev_bread_total;
    118 
    119 dev_type_open(rumpblk_open);
    120 dev_type_close(rumpblk_close);
    121 dev_type_read(rumpblk_read);
    122 dev_type_write(rumpblk_write);
    123 dev_type_ioctl(rumpblk_ioctl);
    124 dev_type_strategy(rumpblk_strategy);
    125 dev_type_strategy(rumpblk_strategy_fail);
    126 dev_type_dump(rumpblk_dump);
    127 dev_type_size(rumpblk_size);
    128 
    129 static const struct bdevsw rumpblk_bdevsw = {
    130 	rumpblk_open, rumpblk_close, rumpblk_strategy, rumpblk_ioctl,
    131 	nodump, nosize, D_DISK
    132 };
    133 
    134 static const struct bdevsw rumpblk_bdevsw_fail = {
    135 	rumpblk_open, rumpblk_close, rumpblk_strategy_fail, rumpblk_ioctl,
    136 	nodump, nosize, D_DISK
    137 };
    138 
    139 static const struct cdevsw rumpblk_cdevsw = {
    140 	rumpblk_open, rumpblk_close, rumpblk_read, rumpblk_write,
    141 	rumpblk_ioctl, nostop, notty, nopoll, nommap, nokqfilter, D_DISK
    142 };
    143 
    144 /* fail every n out of BLKFAIL_MAX */
    145 #define BLKFAIL_MAX 10000
    146 static int blkfail;
    147 static unsigned randstate;
    148 
    149 static struct blkwin *
    150 getwindow(struct rblkdev *rblk, off_t off, int *wsize, int *error)
    151 {
    152 	struct blkwin *win;
    153 
    154 	mutex_enter(&rblk->rblk_memmtx);
    155  retry:
    156 	/* search for window */
    157 	TAILQ_FOREACH(win, &rblk->rblk_lruq, win_lru) {
    158 		if (INWIN(win, off) && WINVALID(win))
    159 			break;
    160 	}
    161 
    162 	/* found?  return */
    163 	if (win) {
    164 		ev_memblk_hits.ev_count++;
    165 		TAILQ_REMOVE(&rblk->rblk_lruq, win, win_lru);
    166 		goto good;
    167 	}
    168 
    169 	/*
    170 	 * Else, create new window.  If the least recently used is not
    171 	 * currently in use, reuse that.  Otherwise we need to wait.
    172 	 */
    173 	win = TAILQ_LAST(&rblk->rblk_lruq, winlru);
    174 	if (win->win_refcnt == 0) {
    175 		TAILQ_REMOVE(&rblk->rblk_lruq, win, win_lru);
    176 		mutex_exit(&rblk->rblk_memmtx);
    177 
    178 		if (WINVALID(win)) {
    179 			DPRINTF(("win %p, unmap mem %p, off 0x%" PRIx64 "\n",
    180 			    win, win->win_mem, win->win_off));
    181 			rumpuser_unmap(win->win_mem, WINSIZE(rblk, win));
    182 			WINVALIDATE(win);
    183 		}
    184 
    185 		win->win_off = STARTWIN(off);
    186 		win->win_mem = rumpuser_filemmap(rblk->rblk_fd, win->win_off,
    187 		    WINSIZE(rblk, win), rblk->rblk_mmflags, error);
    188 		DPRINTF(("win %p, off 0x%" PRIx64 ", mem %p\n",
    189 		    win, win->win_off, win->win_mem));
    190 
    191 		mutex_enter(&rblk->rblk_memmtx);
    192 		if (win->win_mem == NULL) {
    193 			WINVALIDATE(win);
    194 			TAILQ_INSERT_TAIL(&rblk->rblk_lruq, win, win_lru);
    195 			mutex_exit(&rblk->rblk_memmtx);
    196 			return NULL;
    197 		}
    198 	} else {
    199 		DPRINTF(("memwin wait\n"));
    200 		ev_memblk_busy.ev_count++;
    201 
    202 		rblk->rblk_waiting = true;
    203 		cv_wait(&rblk->rblk_memcv, &rblk->rblk_memmtx);
    204 		goto retry;
    205 	}
    206 
    207  good:
    208 	KASSERT(win);
    209 	win->win_refcnt++;
    210 	TAILQ_INSERT_HEAD(&rblk->rblk_lruq, win, win_lru);
    211 	mutex_exit(&rblk->rblk_memmtx);
    212 	*wsize = MIN(*wsize, memwinsize - (off-win->win_off));
    213 	KASSERT(*wsize);
    214 
    215 	return win;
    216 }
    217 
    218 static void
    219 putwindow(struct rblkdev *rblk, struct blkwin *win)
    220 {
    221 
    222 	mutex_enter(&rblk->rblk_memmtx);
    223 	if (--win->win_refcnt == 0 && rblk->rblk_waiting) {
    224 		rblk->rblk_waiting = false;
    225 		cv_signal(&rblk->rblk_memcv);
    226 	}
    227 	KASSERT(win->win_refcnt >= 0);
    228 	mutex_exit(&rblk->rblk_memmtx);
    229 }
    230 
    231 static void
    232 wincleanup(struct rblkdev *rblk)
    233 {
    234 	struct blkwin *win;
    235 
    236 	while ((win = TAILQ_FIRST(&rblk->rblk_lruq)) != NULL) {
    237 		TAILQ_REMOVE(&rblk->rblk_lruq, win, win_lru);
    238 		if (WINVALID(win)) {
    239 			DPRINTF(("cleanup win %p addr %p\n",
    240 			    win, win->win_mem));
    241 			rumpuser_unmap(win->win_mem, WINSIZE(rblk, win));
    242 		}
    243 		kmem_free(win, sizeof(*win));
    244 	}
    245 	rblk->rblk_mmflags = 0;
    246 }
    247 
    248 int
    249 rumpblk_init(void)
    250 {
    251 	char buf[64];
    252 	int rumpblk = RUMPBLK;
    253 	unsigned tmp;
    254 	int error, i;
    255 
    256 	if (rumpuser_getenv("RUMP_BLKFAIL", buf, sizeof(buf), &error) == 0) {
    257 		blkfail = strtoul(buf, NULL, 10);
    258 		/* fail everything */
    259 		if (blkfail > BLKFAIL_MAX)
    260 			blkfail = BLKFAIL_MAX;
    261 		if (rumpuser_getenv("RUMP_BLKFAIL_SEED", buf, sizeof(buf),
    262 		    &error) == 0) {
    263 			randstate = strtoul(buf, NULL, 10);
    264 		} else {
    265 			randstate = arc4random();
    266 		}
    267 		printf("rumpblk: FAULT INJECTION ACTIVE! fail %d/%d. "
    268 		    "seed %u\n", blkfail, BLKFAIL_MAX, randstate);
    269 	} else {
    270 		blkfail = 0;
    271 	}
    272 
    273 	if (rumpuser_getenv("RUMP_BLKWINSIZE", buf, sizeof(buf), &error) == 0) {
    274 		printf("rumpblk: ");
    275 		tmp = strtoul(buf, NULL, 10);
    276 		if (tmp && !(tmp & (tmp-1)))
    277 			memwinsize = tmp;
    278 		else
    279 			printf("invalid RUMP_BLKWINSIZE %d, ", tmp);
    280 		printf("using %d for memwinsize\n", memwinsize);
    281 	}
    282 	if (rumpuser_getenv("RUMP_BLKWINCOUNT", buf, sizeof(buf), &error) == 0){
    283 		printf("rumpblk: ");
    284 		tmp = strtoul(buf, NULL, 10);
    285 		if (tmp)
    286 			memwincnt = tmp;
    287 		else
    288 			printf("invalid RUMP_BLKWINCOUNT %d, ", tmp);
    289 		printf("using %d for memwincount\n", memwincnt);
    290 	}
    291 
    292 	memset(minors, 0, sizeof(minors));
    293 	for (i = 0; i < RUMPBLK_SIZE; i++) {
    294 		mutex_init(&minors[i].rblk_memmtx, MUTEX_DEFAULT, IPL_NONE);
    295 		cv_init(&minors[i].rblk_memcv, "rblkmcv");
    296 	}
    297 
    298 	evcnt_attach_dynamic(&ev_io_total, EVCNT_TYPE_MISC, NULL,
    299 	    "rumpblk", "rumpblk I/O reqs");
    300 	evcnt_attach_dynamic(&ev_io_async, EVCNT_TYPE_MISC, NULL,
    301 	    "rumpblk", "rumpblk async I/O");
    302 
    303 	evcnt_attach_dynamic(&ev_bread_total, EVCNT_TYPE_MISC, NULL,
    304 	    "rumpblk", "rumpblk bytes read");
    305 	evcnt_attach_dynamic(&ev_bwrite_total, EVCNT_TYPE_MISC, NULL,
    306 	    "rumpblk", "rumpblk bytes written");
    307 	evcnt_attach_dynamic(&ev_bwrite_async, EVCNT_TYPE_MISC, NULL,
    308 	    "rumpblk", "rumpblk bytes written async");
    309 
    310 	evcnt_attach_dynamic(&ev_memblk_hits, EVCNT_TYPE_MISC, NULL,
    311 	    "rumpblk", "memblk window hits");
    312 	evcnt_attach_dynamic(&ev_memblk_busy, EVCNT_TYPE_MISC, NULL,
    313 	    "rumpblk", "memblk all windows busy");
    314 
    315 	if (blkfail) {
    316 		return devsw_attach("rumpblk", &rumpblk_bdevsw_fail, &rumpblk,
    317 		    &rumpblk_cdevsw, &rumpblk);
    318 	} else {
    319 		return devsw_attach("rumpblk", &rumpblk_bdevsw, &rumpblk,
    320 		    &rumpblk_cdevsw, &rumpblk);
    321 	}
    322 }
    323 
    324 int
    325 rumpblk_register(const char *path)
    326 {
    327 	size_t len;
    328 	int i;
    329 
    330 	for (i = 0; i < RUMPBLK_SIZE; i++)
    331 		if (minors[i].rblk_path && strcmp(minors[i].rblk_path, path)==0)
    332 			return i;
    333 
    334 	for (i = 0; i < RUMPBLK_SIZE; i++)
    335 		if (minors[i].rblk_path == NULL)
    336 			break;
    337 	if (i == RUMPBLK_SIZE)
    338 		return -1;
    339 
    340 	len = strlen(path);
    341 	minors[i].rblk_path = malloc(len + 1, M_TEMP, M_WAITOK);
    342 	strcpy(minors[i].rblk_path, path);
    343 	minors[i].rblk_fd = -1;
    344 	return i;
    345 }
    346 
    347 int
    348 rumpblk_open(dev_t dev, int flag, int fmt, struct lwp *l)
    349 {
    350 	struct rblkdev *rblk = &minors[minor(dev)];
    351 	uint64_t fsize;
    352 	int ft, dummy;
    353 	int error, fd;
    354 
    355 	KASSERT(rblk->rblk_fd == -1);
    356 	fd = rumpuser_open(rblk->rblk_path, OFLAGS(flag), &error);
    357 	if (error)
    358 		return error;
    359 
    360 	if (rumpuser_getfileinfo(rblk->rblk_path, &fsize, &ft, &error) == -1) {
    361 		rumpuser_close(fd, &dummy);
    362 		return error;
    363 	}
    364 
    365 #ifdef HAS_ODIRECT
    366 	rblk->rblk_dfd = rumpuser_open(rblk->rblk_path,
    367 	    OFLAGS(flag) | O_DIRECT, &error);
    368 	if (error)
    369 		return error;
    370 #endif
    371 
    372 	if (ft == RUMPUSER_FT_REG) {
    373 		struct blkwin *win;
    374 		int i, winsize;
    375 
    376 		/*
    377 		 * Use mmap to access a regular file.  Allocate and
    378 		 * cache initial windows here.  Failure to allocate one
    379 		 * means fallback to read/write i/o.
    380 		 */
    381 
    382 		rblk->rblk_mmflags = 0;
    383 		if (flag & FREAD)
    384 			rblk->rblk_mmflags |= RUMPUSER_FILEMMAP_READ;
    385 		if (flag & FWRITE) {
    386 			rblk->rblk_mmflags |= RUMPUSER_FILEMMAP_WRITE;
    387 			rblk->rblk_mmflags |= RUMPUSER_FILEMMAP_SHARED;
    388 		}
    389 
    390 		TAILQ_INIT(&rblk->rblk_lruq);
    391 		rblk->rblk_size = fsize;
    392 		rblk->rblk_fd = fd;
    393 
    394 		for (i = 0; i < memwincnt && i * memwinsize < fsize; i++) {
    395 			win = kmem_zalloc(sizeof(*win), KM_SLEEP);
    396 			WINVALIDATE(win);
    397 			TAILQ_INSERT_TAIL(&rblk->rblk_lruq, win, win_lru);
    398 
    399 			/*
    400 			 * Allocate first windows.  Here we just generally
    401 			 * make sure a) we can mmap at all b) we have the
    402 			 * necessary VA available
    403 			 */
    404 			winsize = 1;
    405 			win = getwindow(rblk, i*memwinsize, &winsize, &error);
    406 			if (win) {
    407 				putwindow(rblk, win);
    408 			} else {
    409 				wincleanup(rblk);
    410 				break;
    411 			}
    412 		}
    413 
    414 		memset(&rblk->rblk_dl, 0, sizeof(rblk->rblk_dl));
    415 		rblk->rblk_pi.p_size = fsize >> DEV_BSHIFT;
    416 		rblk->rblk_dl.d_secsize = DEV_BSIZE;
    417 		rblk->rblk_curpi = &rblk->rblk_pi;
    418 	} else {
    419 		if (rumpuser_ioctl(fd, DIOCGDINFO, &rblk->rblk_dl,
    420 		    &error) == -1) {
    421 			KASSERT(error);
    422 			rumpuser_close(fd, &dummy);
    423 			return error;
    424 		}
    425 
    426 		rblk->rblk_fd = fd;
    427 		rblk->rblk_curpi = &rblk->rblk_dl.d_partitions[0];
    428 	}
    429 
    430 	KASSERT(rblk->rblk_fd != -1);
    431 	return 0;
    432 }
    433 
    434 int
    435 rumpblk_close(dev_t dev, int flag, int fmt, struct lwp *l)
    436 {
    437 	struct rblkdev *rblk = &minors[minor(dev)];
    438 	int dummy;
    439 
    440 	if (rblk->rblk_mmflags)
    441 		wincleanup(rblk);
    442 	rumpuser_fsync(rblk->rblk_fd, &dummy);
    443 	rumpuser_close(rblk->rblk_fd, &dummy);
    444 	rblk->rblk_fd = -1;
    445 
    446 	return 0;
    447 }
    448 
    449 int
    450 rumpblk_ioctl(dev_t dev, u_long xfer, void *addr, int flag, struct lwp *l)
    451 {
    452 	struct rblkdev *rblk = &minors[minor(dev)];
    453 	int rv, error;
    454 
    455 	if (xfer == DIOCGPART) {
    456 		struct partinfo *pi = (struct partinfo *)addr;
    457 
    458 		pi->part = rblk->rblk_curpi;
    459 		pi->disklab = &rblk->rblk_dl;
    460 
    461 		return 0;
    462 	}
    463 
    464 	rv = rumpuser_ioctl(rblk->rblk_fd, xfer, addr, &error);
    465 	if (rv == -1)
    466 		return error;
    467 
    468 	return 0;
    469 }
    470 
    471 int
    472 rumpblk_read(dev_t dev, struct uio *uio, int flags)
    473 {
    474 
    475 	panic("%s: unimplemented", __func__);
    476 }
    477 
    478 int
    479 rumpblk_write(dev_t dev, struct uio *uio, int flags)
    480 {
    481 
    482 	panic("%s: unimplemented", __func__);
    483 }
    484 
    485 static void
    486 dostrategy(struct buf *bp)
    487 {
    488 	struct rblkdev *rblk = &minors[minor(bp->b_dev)];
    489 	off_t off;
    490 	int async = bp->b_flags & B_ASYNC;
    491 	int error;
    492 
    493 	/* collect statistics */
    494 	ev_io_total.ev_count++;
    495 	if (async)
    496 		ev_io_async.ev_count++;
    497 	if (BUF_ISWRITE(bp)) {
    498 		ev_bwrite_total.ev_count += bp->b_bcount;
    499 		if (async)
    500 			ev_bwrite_async.ev_count += bp->b_bcount;
    501 	} else {
    502 		ev_bread_total.ev_count++;
    503 	}
    504 
    505 	off = bp->b_blkno << DEV_BSHIFT;
    506 	/*
    507 	 * Do bounds checking if we're working on a file.  Otherwise
    508 	 * invalid file systems might attempt to read beyond EOF.  This
    509 	 * is bad(tm) especially on mmapped images.  This is essentially
    510 	 * the kernel bounds_check() routines.
    511 	 */
    512 	if (rblk->rblk_size && off + bp->b_bcount > rblk->rblk_size) {
    513 		int64_t sz = rblk->rblk_size - off;
    514 
    515 		/* EOF */
    516 		if (sz == 0) {
    517 			rump_biodone(bp, 0, 0);
    518 			return;
    519 		}
    520 		/* beyond EOF ==> error */
    521 		if (sz < 0) {
    522 			rump_biodone(bp, 0, EINVAL);
    523 			return;
    524 		}
    525 
    526 		/* truncate to device size */
    527 		bp->b_bcount = sz;
    528 	}
    529 
    530 	DPRINTF(("rumpblk_strategy: 0x%x bytes %s off 0x%" PRIx64
    531 	    " (0x%" PRIx64 " - 0x%" PRIx64 "), %ssync\n",
    532 	    bp->b_bcount, BUF_ISREAD(bp) ? "READ" : "WRITE",
    533 	    off, off, (off + bp->b_bcount), async ? "a" : ""));
    534 
    535 	/* mmap?  handle here and return */
    536 	if (rblk->rblk_mmflags) {
    537 		struct blkwin *win;
    538 		int winsize, iodone;
    539 		uint8_t *ioaddr, *bufaddr;
    540 
    541 		for (iodone = 0; iodone < bp->b_bcount;
    542 		    iodone += winsize, off += winsize) {
    543 			winsize = bp->b_bcount - iodone;
    544 			win = getwindow(rblk, off, &winsize, &error);
    545 			if (win == NULL) {
    546 				rump_biodone(bp, iodone, error);
    547 				return;
    548 			}
    549 
    550 			ioaddr = (uint8_t *)win->win_mem + (off-STARTWIN(off));
    551 			bufaddr = (uint8_t *)bp->b_data + iodone;
    552 
    553 			DPRINTF(("strat: %p off 0x%" PRIx64
    554 			    ", ioaddr %p (%p)/buf %p\n", win,
    555 			    win->win_off, ioaddr, win->win_mem, bufaddr));
    556 			if (BUF_ISREAD(bp)) {
    557 				memcpy(bufaddr, ioaddr, winsize);
    558 			} else {
    559 				memcpy(ioaddr, bufaddr, winsize);
    560 			}
    561 
    562 			/* synchronous write, sync bits back to disk */
    563 			if (BUF_ISWRITE(bp) && !async) {
    564 				rumpuser_memsync(ioaddr, winsize, &error);
    565 			}
    566 			putwindow(rblk, win);
    567 		}
    568 
    569 		rump_biodone(bp, bp->b_bcount, 0);
    570 		return;
    571 	}
    572 
    573 	/*
    574 	 * Do I/O.  We have different paths for async and sync I/O.
    575 	 * Async I/O is done by passing a request to rumpuser where
    576 	 * it is executed.  The rumpuser routine then calls
    577 	 * biodone() to signal any waiters in the kernel.  I/O's are
    578 	 * executed in series.  Technically executing them in parallel
    579 	 * would produce better results, but then we'd need either
    580 	 * more threads or posix aio.  Maybe worth investigating
    581 	 * this later.
    582 	 *
    583 	 * Using bufq here might be a good idea.
    584 	 */
    585 
    586 	if (rump_threads) {
    587 		struct rumpuser_aio *rua;
    588 		int op, fd;
    589 
    590 		fd = rblk->rblk_fd;
    591 		if (BUF_ISREAD(bp)) {
    592 			op = RUA_OP_READ;
    593 		} else {
    594 			op = RUA_OP_WRITE;
    595 			if (!async) {
    596 				/* O_DIRECT not fully automatic yet */
    597 #ifdef HAS_ODIRECT
    598 				if ((off & (DEV_BSIZE-1)) == 0
    599 				    && ((intptr_t)bp->b_data&(DEV_BSIZE-1)) == 0
    600 				    && (bp->b_bcount & (DEV_BSIZE-1)) == 0)
    601 					fd = rblk->rblk_dfd;
    602 				else
    603 #endif
    604 					op |= RUA_OP_SYNC;
    605 			}
    606 		}
    607 
    608 		rumpuser_mutex_enter(&rumpuser_aio_mtx);
    609 		while ((rumpuser_aio_head+1) % N_AIOS == rumpuser_aio_tail) {
    610 			rumpuser_cv_wait(&rumpuser_aio_cv, &rumpuser_aio_mtx);
    611 		}
    612 
    613 		rua = &rumpuser_aios[rumpuser_aio_head];
    614 		KASSERT(rua->rua_bp == NULL);
    615 		rua->rua_fd = fd;
    616 		rua->rua_data = bp->b_data;
    617 		rua->rua_dlen = bp->b_bcount;
    618 		rua->rua_off = off;
    619 		rua->rua_bp = bp;
    620 		rua->rua_op = op;
    621 
    622 		/* insert into queue & signal */
    623 		rumpuser_aio_head = (rumpuser_aio_head+1) % N_AIOS;
    624 		rumpuser_cv_signal(&rumpuser_aio_cv);
    625 		rumpuser_mutex_exit(&rumpuser_aio_mtx);
    626 	} else {
    627 		if (BUF_ISREAD(bp)) {
    628 			rumpuser_read_bio(rblk->rblk_fd, bp->b_data,
    629 			    bp->b_bcount, off, rump_biodone, bp);
    630 		} else {
    631 			rumpuser_write_bio(rblk->rblk_fd, bp->b_data,
    632 			    bp->b_bcount, off, rump_biodone, bp);
    633 		}
    634 		if (BUF_ISWRITE(bp) && !async)
    635 			rumpuser_fsync(rblk->rblk_fd, &error);
    636 	}
    637 }
    638 
    639 void
    640 rumpblk_strategy(struct buf *bp)
    641 {
    642 
    643 	dostrategy(bp);
    644 }
    645 
    646 /*
    647  * Simple random number generator.  This is private so that we can
    648  * very repeatedly control which blocks will fail.
    649  *
    650  * <mlelstv> pooka, rand()
    651  * <mlelstv> [paste]
    652  */
    653 static unsigned
    654 gimmerand(void)
    655 {
    656 
    657 	return (randstate = randstate * 1103515245 + 12345) % (0x80000000L);
    658 }
    659 
    660 /*
    661  * Block device with very simple fault injection.  Fails every
    662  * n out of BLKFAIL_MAX I/O with EIO.  n is determined by the env
    663  * variable RUMP_BLKFAIL.
    664  */
    665 void
    666 rumpblk_strategy_fail(struct buf *bp)
    667 {
    668 
    669 	if (gimmerand() % BLKFAIL_MAX >= blkfail) {
    670 		dostrategy(bp);
    671 	} else {
    672 		printf("block fault injection: failing I/O on block %lld\n",
    673 		    (long long)bp->b_blkno);
    674 		bp->b_error = EIO;
    675 		biodone(bp);
    676 	}
    677 }
    678