Home | History | Annotate | Line # | Download | only in rumpvfs
rumpblk.c revision 1.21
      1 /*	$NetBSD: rumpblk.c,v 1.21 2009/04/27 18:36:43 pooka Exp $	*/
      2 
      3 /*
      4  * Copyright (c) 2009 Antti Kantee.  All Rights Reserved.
      5  *
      6  * Development of this software was supported by the
      7  * Finnish Cultural Foundation.
      8  *
      9  * Redistribution and use in source and binary forms, with or without
     10  * modification, are permitted provided that the following conditions
     11  * are met:
     12  * 1. Redistributions of source code must retain the above copyright
     13  *    notice, this list of conditions and the following disclaimer.
     14  * 2. Redistributions in binary form must reproduce the above copyright
     15  *    notice, this list of conditions and the following disclaimer in the
     16  *    documentation and/or other materials provided with the distribution.
     17  *
     18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
     19  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
     20  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
     21  * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
     22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
     24  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     28  * SUCH DAMAGE.
     29  */
     30 
     31 /*
     32  * Block device emulation.  Presents a block device interface and
     33  * uses rumpuser system calls to satisfy I/O requests.
     34  *
     35  * We provide fault injection.  The driver can be made to fail
     36  * I/O occasionally.
     37  *
     38  * The driver also provides an optimization for regular files by
     39  * using memory-mapped I/O.  This avoids kernel access for every
     40  * I/O operation.  It also gives finer-grained control of how to
     41  * flush data.  Additionally, in case the rump kernel dumps core,
     42  * we get way less carnage.
     43  */
     44 
     45 #include <sys/cdefs.h>
     46 __KERNEL_RCSID(0, "$NetBSD: rumpblk.c,v 1.21 2009/04/27 18:36:43 pooka Exp $");
     47 
     48 #include <sys/param.h>
     49 #include <sys/buf.h>
     50 #include <sys/conf.h>
     51 #include <sys/condvar.h>
     52 #include <sys/disklabel.h>
     53 #include <sys/evcnt.h>
     54 #include <sys/fcntl.h>
     55 #include <sys/kmem.h>
     56 #include <sys/malloc.h>
     57 #include <sys/queue.h>
     58 #include <sys/stat.h>
     59 
     60 #include <rump/rumpuser.h>
     61 
     62 #include "rump_private.h"
     63 #include "rump_vfs_private.h"
     64 
     65 #if 0
     66 #define DPRINTF(x) printf x
     67 #else
     68 #define DPRINTF(x)
     69 #endif
     70 
     71 /* Default: 16 x 1MB windows */
     72 unsigned memwinsize = (1<<20);
     73 unsigned memwincnt = 16;
     74 
     75 #define STARTWIN(off)		((off) & ~(memwinsize-1))
     76 #define INWIN(win,off)		((win)->win_off == STARTWIN(off))
     77 #define WINSIZE(rblk, win)	(MIN((rblk->rblk_size-win->win_off),memwinsize))
     78 #define WINVALID(win)		((win)->win_off != (off_t)-1)
     79 #define WINVALIDATE(win)	((win)->win_off = (off_t)-1)
     80 struct blkwin {
     81 	off_t win_off;
     82 	void *win_mem;
     83 	int win_refcnt;
     84 
     85 	TAILQ_ENTRY(blkwin) win_lru;
     86 };
     87 
     88 #define RUMPBLK_SIZE 16
     89 static struct rblkdev {
     90 	char *rblk_path;
     91 	int rblk_fd;
     92 #ifdef HAS_ODIRECT
     93 	int rblk_dfd;
     94 #endif
     95 
     96 	/* for mmap */
     97 	int rblk_mmflags;
     98 	kmutex_t rblk_memmtx;
     99 	kcondvar_t rblk_memcv;
    100 	TAILQ_HEAD(winlru, blkwin) rblk_lruq;
    101 	size_t rblk_size;
    102 	bool rblk_waiting;
    103 
    104 	struct partition *rblk_curpi;
    105 	struct partition rblk_pi;
    106 	struct disklabel rblk_dl;
    107 } minors[RUMPBLK_SIZE];
    108 
    109 static struct evcnt ev_io_total;
    110 static struct evcnt ev_io_async;
    111 
    112 static struct evcnt ev_memblk_hits;
    113 static struct evcnt ev_memblk_busy;
    114 
    115 static struct evcnt ev_bwrite_total;
    116 static struct evcnt ev_bwrite_async;
    117 static struct evcnt ev_bread_total;
    118 
    119 dev_type_open(rumpblk_open);
    120 dev_type_close(rumpblk_close);
    121 dev_type_read(rumpblk_read);
    122 dev_type_write(rumpblk_write);
    123 dev_type_ioctl(rumpblk_ioctl);
    124 dev_type_strategy(rumpblk_strategy);
    125 dev_type_strategy(rumpblk_strategy_fail);
    126 dev_type_dump(rumpblk_dump);
    127 dev_type_size(rumpblk_size);
    128 
    129 static const struct bdevsw rumpblk_bdevsw = {
    130 	rumpblk_open, rumpblk_close, rumpblk_strategy, rumpblk_ioctl,
    131 	nodump, nosize, D_DISK
    132 };
    133 
    134 static const struct bdevsw rumpblk_bdevsw_fail = {
    135 	rumpblk_open, rumpblk_close, rumpblk_strategy_fail, rumpblk_ioctl,
    136 	nodump, nosize, D_DISK
    137 };
    138 
    139 static const struct cdevsw rumpblk_cdevsw = {
    140 	rumpblk_open, rumpblk_close, rumpblk_read, rumpblk_write,
    141 	rumpblk_ioctl, nostop, notty, nopoll, nommap, nokqfilter, D_DISK
    142 };
    143 
    144 /* fail every n out of BLKFAIL_MAX */
    145 #define BLKFAIL_MAX 10000
    146 static int blkfail;
    147 static unsigned randstate;
    148 
    149 static struct blkwin *
    150 getwindow(struct rblkdev *rblk, off_t off, int *wsize, int *error)
    151 {
    152 	struct blkwin *win;
    153 
    154 	mutex_enter(&rblk->rblk_memmtx);
    155  retry:
    156 	/* search for window */
    157 	TAILQ_FOREACH(win, &rblk->rblk_lruq, win_lru) {
    158 		if (INWIN(win, off) && WINVALID(win))
    159 			break;
    160 	}
    161 
    162 	/* found?  return */
    163 	if (win) {
    164 		ev_memblk_hits.ev_count++;
    165 		TAILQ_REMOVE(&rblk->rblk_lruq, win, win_lru);
    166 		goto good;
    167 	}
    168 
    169 	/*
    170 	 * Else, create new window.  If the least recently used is not
    171 	 * currently in use, reuse that.  Otherwise we need to wait.
    172 	 */
    173 	win = TAILQ_LAST(&rblk->rblk_lruq, winlru);
    174 	if (win->win_refcnt == 0) {
    175 		TAILQ_REMOVE(&rblk->rblk_lruq, win, win_lru);
    176 		mutex_exit(&rblk->rblk_memmtx);
    177 
    178 		if (WINVALID(win)) {
    179 			DPRINTF(("win %p, unmap mem %p, off 0x%" PRIx64 "\n",
    180 			    win, win->win_mem, win->win_off));
    181 			rumpuser_unmap(win->win_mem, WINSIZE(rblk, win));
    182 			WINVALIDATE(win);
    183 		}
    184 
    185 		win->win_off = STARTWIN(off);
    186 		win->win_mem = rumpuser_filemmap(rblk->rblk_fd, win->win_off,
    187 		    WINSIZE(rblk, win), rblk->rblk_mmflags, error);
    188 		DPRINTF(("win %p, off 0x%" PRIx64 ", mem %p\n",
    189 		    win, win->win_off, win->win_mem));
    190 
    191 		mutex_enter(&rblk->rblk_memmtx);
    192 		if (win->win_mem == NULL) {
    193 			WINVALIDATE(win);
    194 			TAILQ_INSERT_TAIL(&rblk->rblk_lruq, win, win_lru);
    195 			mutex_exit(&rblk->rblk_memmtx);
    196 			return NULL;
    197 		}
    198 	} else {
    199 		DPRINTF(("memwin wait\n"));
    200 		ev_memblk_busy.ev_count++;
    201 
    202 		rblk->rblk_waiting = true;
    203 		cv_wait(&rblk->rblk_memcv, &rblk->rblk_memmtx);
    204 		goto retry;
    205 	}
    206 
    207  good:
    208 	KASSERT(win);
    209 	win->win_refcnt++;
    210 	TAILQ_INSERT_HEAD(&rblk->rblk_lruq, win, win_lru);
    211 	mutex_exit(&rblk->rblk_memmtx);
    212 	*wsize = MIN(*wsize, memwinsize - (off-win->win_off));
    213 	KASSERT(*wsize);
    214 
    215 	return win;
    216 }
    217 
    218 static void
    219 putwindow(struct rblkdev *rblk, struct blkwin *win)
    220 {
    221 
    222 	mutex_enter(&rblk->rblk_memmtx);
    223 	if (--win->win_refcnt == 0 && rblk->rblk_waiting) {
    224 		rblk->rblk_waiting = false;
    225 		cv_signal(&rblk->rblk_memcv);
    226 	}
    227 	KASSERT(win->win_refcnt >= 0);
    228 	mutex_exit(&rblk->rblk_memmtx);
    229 }
    230 
    231 static void
    232 wincleanup(struct rblkdev *rblk)
    233 {
    234 	struct blkwin *win;
    235 
    236 	while ((win = TAILQ_FIRST(&rblk->rblk_lruq)) != NULL) {
    237 		TAILQ_REMOVE(&rblk->rblk_lruq, win, win_lru);
    238 		if (WINVALID(win)) {
    239 			DPRINTF(("cleanup win %p addr %p\n",
    240 			    win, win->win_mem));
    241 			rumpuser_unmap(win->win_mem, WINSIZE(rblk, win));
    242 		}
    243 		kmem_free(win, sizeof(*win));
    244 	}
    245 	rblk->rblk_mmflags = 0;
    246 }
    247 
    248 int
    249 rumpblk_init(void)
    250 {
    251 	char buf[64];
    252 	int rumpblk = RUMPBLK;
    253 	unsigned tmp;
    254 	int error, i;
    255 
    256 	if (rumpuser_getenv("RUMP_BLKFAIL", buf, sizeof(buf), &error) == 0) {
    257 		blkfail = strtoul(buf, NULL, 10);
    258 		/* fail everything */
    259 		if (blkfail > BLKFAIL_MAX)
    260 			blkfail = BLKFAIL_MAX;
    261 		if (rumpuser_getenv("RUMP_BLKFAIL_SEED", buf, sizeof(buf),
    262 		    &error) == 0) {
    263 			randstate = strtoul(buf, NULL, 10);
    264 		} else {
    265 			randstate = arc4random(); /* XXX: not enough entropy */
    266 		}
    267 		printf("rumpblk: FAULT INJECTION ACTIVE!  every %d out of"
    268 		    " %d I/O will fail.  key %u\n", blkfail, BLKFAIL_MAX,
    269 		    randstate);
    270 	} else {
    271 		blkfail = 0;
    272 	}
    273 
    274 	if (rumpuser_getenv("RUMP_BLKWINSIZE", buf, sizeof(buf), &error) == 0) {
    275 		printf("rumpblk: ");
    276 		tmp = strtoul(buf, NULL, 10);
    277 		if (tmp && !(tmp & (tmp-1)))
    278 			memwinsize = tmp;
    279 		else
    280 			printf("invalid RUMP_BLKWINSIZE %d, ", tmp);
    281 		printf("using %d for memwinsize\n", memwinsize);
    282 	}
    283 	if (rumpuser_getenv("RUMP_BLKWINCOUNT", buf, sizeof(buf), &error) == 0){
    284 		printf("rumpblk: ");
    285 		tmp = strtoul(buf, NULL, 10);
    286 		if (tmp)
    287 			memwincnt = tmp;
    288 		else
    289 			printf("invalid RUMP_BLKWINCOUNT %d, ", tmp);
    290 		printf("using %d for memwincount\n", memwincnt);
    291 	}
    292 
    293 	memset(minors, 0, sizeof(minors));
    294 	for (i = 0; i < RUMPBLK_SIZE; i++) {
    295 		mutex_init(&minors[i].rblk_memmtx, MUTEX_DEFAULT, IPL_NONE);
    296 		cv_init(&minors[i].rblk_memcv, "rblkmcv");
    297 	}
    298 
    299 	evcnt_attach_dynamic(&ev_io_total, EVCNT_TYPE_MISC, NULL,
    300 	    "rumpblk", "rumpblk I/O reqs");
    301 	evcnt_attach_dynamic(&ev_io_async, EVCNT_TYPE_MISC, NULL,
    302 	    "rumpblk", "rumpblk async I/O");
    303 
    304 	evcnt_attach_dynamic(&ev_bread_total, EVCNT_TYPE_MISC, NULL,
    305 	    "rumpblk", "rumpblk bytes read");
    306 	evcnt_attach_dynamic(&ev_bwrite_total, EVCNT_TYPE_MISC, NULL,
    307 	    "rumpblk", "rumpblk bytes written");
    308 	evcnt_attach_dynamic(&ev_bwrite_async, EVCNT_TYPE_MISC, NULL,
    309 	    "rumpblk", "rumpblk bytes written async");
    310 
    311 	evcnt_attach_dynamic(&ev_memblk_hits, EVCNT_TYPE_MISC, NULL,
    312 	    "rumpblk", "memblk window hits");
    313 	evcnt_attach_dynamic(&ev_memblk_busy, EVCNT_TYPE_MISC, NULL,
    314 	    "rumpblk", "memblk all windows busy");
    315 
    316 	if (blkfail) {
    317 		return devsw_attach("rumpblk", &rumpblk_bdevsw_fail, &rumpblk,
    318 		    &rumpblk_cdevsw, &rumpblk);
    319 	} else {
    320 		return devsw_attach("rumpblk", &rumpblk_bdevsw, &rumpblk,
    321 		    &rumpblk_cdevsw, &rumpblk);
    322 	}
    323 }
    324 
    325 int
    326 rumpblk_register(const char *path)
    327 {
    328 	size_t len;
    329 	int i;
    330 
    331 	for (i = 0; i < RUMPBLK_SIZE; i++)
    332 		if (minors[i].rblk_path && strcmp(minors[i].rblk_path, path)==0)
    333 			return i;
    334 
    335 	for (i = 0; i < RUMPBLK_SIZE; i++)
    336 		if (minors[i].rblk_path == NULL)
    337 			break;
    338 	if (i == RUMPBLK_SIZE)
    339 		return -1;
    340 
    341 	len = strlen(path);
    342 	minors[i].rblk_path = malloc(len + 1, M_TEMP, M_WAITOK);
    343 	strcpy(minors[i].rblk_path, path);
    344 	minors[i].rblk_fd = -1;
    345 	return i;
    346 }
    347 
    348 int
    349 rumpblk_open(dev_t dev, int flag, int fmt, struct lwp *l)
    350 {
    351 	struct rblkdev *rblk = &minors[minor(dev)];
    352 	uint64_t fsize;
    353 	int ft, dummy;
    354 	int error, fd;
    355 
    356 	KASSERT(rblk->rblk_fd == -1);
    357 	fd = rumpuser_open(rblk->rblk_path, OFLAGS(flag), &error);
    358 	if (error)
    359 		return error;
    360 
    361 	if (rumpuser_getfileinfo(rblk->rblk_path, &fsize, &ft, &error) == -1) {
    362 		rumpuser_close(fd, &dummy);
    363 		return error;
    364 	}
    365 
    366 #ifdef HAS_ODIRECT
    367 	rblk->rblk_dfd = rumpuser_open(rblk->rblk_path,
    368 	    OFLAGS(flag) | O_DIRECT, &error);
    369 	if (error)
    370 		return error;
    371 #endif
    372 
    373 	if (ft == RUMPUSER_FT_REG) {
    374 		struct blkwin *win;
    375 		int i, winsize;
    376 
    377 		/*
    378 		 * Use mmap to access a regular file.  Allocate and
    379 		 * cache initial windows here.  Failure to allocate one
    380 		 * means fallback to read/write i/o.
    381 		 */
    382 
    383 		rblk->rblk_mmflags = 0;
    384 		if (flag & FREAD)
    385 			rblk->rblk_mmflags |= RUMPUSER_FILEMMAP_READ;
    386 		if (flag & FWRITE) {
    387 			rblk->rblk_mmflags |= RUMPUSER_FILEMMAP_WRITE;
    388 			rblk->rblk_mmflags |= RUMPUSER_FILEMMAP_SHARED;
    389 		}
    390 
    391 		TAILQ_INIT(&rblk->rblk_lruq);
    392 		rblk->rblk_size = fsize;
    393 		rblk->rblk_fd = fd;
    394 
    395 		for (i = 0; i < memwincnt && i * memwinsize < fsize; i++) {
    396 			win = kmem_zalloc(sizeof(*win), KM_SLEEP);
    397 			WINVALIDATE(win);
    398 			TAILQ_INSERT_TAIL(&rblk->rblk_lruq, win, win_lru);
    399 
    400 			/*
    401 			 * Allocate first windows.  Here we just generally
    402 			 * make sure a) we can mmap at all b) we have the
    403 			 * necessary VA available
    404 			 */
    405 			winsize = 1;
    406 			win = getwindow(rblk, i*memwinsize, &winsize, &error);
    407 			if (win) {
    408 				putwindow(rblk, win);
    409 			} else {
    410 				wincleanup(rblk);
    411 				break;
    412 			}
    413 		}
    414 
    415 		memset(&rblk->rblk_dl, 0, sizeof(rblk->rblk_dl));
    416 		rblk->rblk_pi.p_size = fsize >> DEV_BSHIFT;
    417 		rblk->rblk_dl.d_secsize = DEV_BSIZE;
    418 		rblk->rblk_curpi = &rblk->rblk_pi;
    419 	} else {
    420 		if (rumpuser_ioctl(fd, DIOCGDINFO, &rblk->rblk_dl,
    421 		    &error) == -1) {
    422 			KASSERT(error);
    423 			rumpuser_close(fd, &dummy);
    424 			return error;
    425 		}
    426 
    427 		rblk->rblk_fd = fd;
    428 		rblk->rblk_curpi = &rblk->rblk_dl.d_partitions[0];
    429 	}
    430 
    431 	KASSERT(rblk->rblk_fd != -1);
    432 	return 0;
    433 }
    434 
    435 int
    436 rumpblk_close(dev_t dev, int flag, int fmt, struct lwp *l)
    437 {
    438 	struct rblkdev *rblk = &minors[minor(dev)];
    439 	int dummy;
    440 
    441 	if (rblk->rblk_mmflags)
    442 		wincleanup(rblk);
    443 	rumpuser_fsync(rblk->rblk_fd, &dummy);
    444 	rumpuser_close(rblk->rblk_fd, &dummy);
    445 	rblk->rblk_fd = -1;
    446 
    447 	return 0;
    448 }
    449 
    450 int
    451 rumpblk_ioctl(dev_t dev, u_long xfer, void *addr, int flag, struct lwp *l)
    452 {
    453 	struct rblkdev *rblk = &minors[minor(dev)];
    454 	int rv, error;
    455 
    456 	if (xfer == DIOCGPART) {
    457 		struct partinfo *pi = (struct partinfo *)addr;
    458 
    459 		pi->part = rblk->rblk_curpi;
    460 		pi->disklab = &rblk->rblk_dl;
    461 
    462 		return 0;
    463 	}
    464 
    465 	rv = rumpuser_ioctl(rblk->rblk_fd, xfer, addr, &error);
    466 	if (rv == -1)
    467 		return error;
    468 
    469 	return 0;
    470 }
    471 
    472 int
    473 rumpblk_read(dev_t dev, struct uio *uio, int flags)
    474 {
    475 
    476 	panic("%s: unimplemented", __func__);
    477 }
    478 
    479 int
    480 rumpblk_write(dev_t dev, struct uio *uio, int flags)
    481 {
    482 
    483 	panic("%s: unimplemented", __func__);
    484 }
    485 
    486 static void
    487 dostrategy(struct buf *bp)
    488 {
    489 	struct rblkdev *rblk = &minors[minor(bp->b_dev)];
    490 	off_t off;
    491 	int async = bp->b_flags & B_ASYNC;
    492 	int error;
    493 
    494 	/* collect statistics */
    495 	ev_io_total.ev_count++;
    496 	if (async)
    497 		ev_io_async.ev_count++;
    498 	if (BUF_ISWRITE(bp)) {
    499 		ev_bwrite_total.ev_count += bp->b_bcount;
    500 		if (async)
    501 			ev_bwrite_async.ev_count += bp->b_bcount;
    502 	} else {
    503 		ev_bread_total.ev_count++;
    504 	}
    505 
    506 	off = bp->b_blkno << DEV_BSHIFT;
    507 	/*
    508 	 * Do bounds checking if we're working on a file.  Otherwise
    509 	 * invalid file systems might attempt to read beyond EOF.  This
    510 	 * is bad(tm) especially on mmapped images.  This is essentially
    511 	 * the kernel bounds_check() routines.
    512 	 */
    513 	if (rblk->rblk_size && off + bp->b_bcount > rblk->rblk_size) {
    514 		int64_t sz = rblk->rblk_size - off;
    515 
    516 		/* EOF */
    517 		if (sz == 0) {
    518 			rump_biodone(bp, 0, 0);
    519 			return;
    520 		}
    521 		/* beyond EOF ==> error */
    522 		if (sz < 0) {
    523 			rump_biodone(bp, 0, EINVAL);
    524 			return;
    525 		}
    526 
    527 		/* truncate to device size */
    528 		bp->b_bcount = sz;
    529 	}
    530 
    531 	DPRINTF(("rumpblk_strategy: 0x%x bytes %s off 0x%" PRIx64
    532 	    " (0x%" PRIx64 " - 0x%" PRIx64 "), %ssync\n",
    533 	    bp->b_bcount, BUF_ISREAD(bp) ? "READ" : "WRITE",
    534 	    off, off, (off + bp->b_bcount), async ? "a" : ""));
    535 
    536 	/* mmap?  handle here and return */
    537 	if (rblk->rblk_mmflags) {
    538 		struct blkwin *win;
    539 		int winsize, iodone;
    540 		uint8_t *ioaddr, *bufaddr;
    541 
    542 		for (iodone = 0; iodone < bp->b_bcount;
    543 		    iodone += winsize, off += winsize) {
    544 			winsize = bp->b_bcount - iodone;
    545 			win = getwindow(rblk, off, &winsize, &error);
    546 			if (win == NULL) {
    547 				rump_biodone(bp, iodone, error);
    548 				return;
    549 			}
    550 
    551 			ioaddr = (uint8_t *)win->win_mem + (off-STARTWIN(off));
    552 			bufaddr = (uint8_t *)bp->b_data + iodone;
    553 
    554 			DPRINTF(("strat: %p off 0x%" PRIx64
    555 			    ", ioaddr %p (%p)/buf %p\n", win,
    556 			    win->win_off, ioaddr, win->win_mem, bufaddr));
    557 			if (BUF_ISREAD(bp)) {
    558 				memcpy(bufaddr, ioaddr, winsize);
    559 			} else {
    560 				memcpy(ioaddr, bufaddr, winsize);
    561 			}
    562 
    563 			/* synchronous write, sync bits back to disk */
    564 			if (BUF_ISWRITE(bp) && !async) {
    565 				rumpuser_memsync(ioaddr, winsize, &error);
    566 			}
    567 			putwindow(rblk, win);
    568 		}
    569 
    570 		rump_biodone(bp, bp->b_bcount, 0);
    571 		return;
    572 	}
    573 
    574 	/*
    575 	 * Do I/O.  We have different paths for async and sync I/O.
    576 	 * Async I/O is done by passing a request to rumpuser where
    577 	 * it is executed.  The rumpuser routine then calls
    578 	 * biodone() to signal any waiters in the kernel.  I/O's are
    579 	 * executed in series.  Technically executing them in parallel
    580 	 * would produce better results, but then we'd need either
    581 	 * more threads or posix aio.  Maybe worth investigating
    582 	 * this later.
    583 	 *
    584 	 * Using bufq here might be a good idea.
    585 	 */
    586 
    587 	if (rump_threads) {
    588 		struct rumpuser_aio *rua;
    589 		int op, fd;
    590 
    591 		fd = rblk->rblk_fd;
    592 		if (BUF_ISREAD(bp)) {
    593 			op = RUA_OP_READ;
    594 		} else {
    595 			op = RUA_OP_WRITE;
    596 			if (!async) {
    597 				/* O_DIRECT not fully automatic yet */
    598 #ifdef HAS_ODIRECT
    599 				if ((off & (DEV_BSIZE-1)) == 0
    600 				    && ((intptr_t)bp->b_data&(DEV_BSIZE-1)) == 0
    601 				    && (bp->b_bcount & (DEV_BSIZE-1)) == 0)
    602 					fd = rblk->rblk_dfd;
    603 				else
    604 #endif
    605 					op |= RUA_OP_SYNC;
    606 			}
    607 		}
    608 
    609 		rumpuser_mutex_enter(&rumpuser_aio_mtx);
    610 		while ((rumpuser_aio_head+1) % N_AIOS == rumpuser_aio_tail) {
    611 			rumpuser_cv_wait(&rumpuser_aio_cv, &rumpuser_aio_mtx);
    612 		}
    613 
    614 		rua = &rumpuser_aios[rumpuser_aio_head];
    615 		KASSERT(rua->rua_bp == NULL);
    616 		rua->rua_fd = fd;
    617 		rua->rua_data = bp->b_data;
    618 		rua->rua_dlen = bp->b_bcount;
    619 		rua->rua_off = off;
    620 		rua->rua_bp = bp;
    621 		rua->rua_op = op;
    622 
    623 		/* insert into queue & signal */
    624 		rumpuser_aio_head = (rumpuser_aio_head+1) % N_AIOS;
    625 		rumpuser_cv_signal(&rumpuser_aio_cv);
    626 		rumpuser_mutex_exit(&rumpuser_aio_mtx);
    627 	} else {
    628 		if (BUF_ISREAD(bp)) {
    629 			rumpuser_read_bio(rblk->rblk_fd, bp->b_data,
    630 			    bp->b_bcount, off, rump_biodone, bp);
    631 		} else {
    632 			rumpuser_write_bio(rblk->rblk_fd, bp->b_data,
    633 			    bp->b_bcount, off, rump_biodone, bp);
    634 		}
    635 		if (BUF_ISWRITE(bp) && !async)
    636 			rumpuser_fsync(rblk->rblk_fd, &error);
    637 	}
    638 }
    639 
    640 void
    641 rumpblk_strategy(struct buf *bp)
    642 {
    643 
    644 	dostrategy(bp);
    645 }
    646 
    647 /*
    648  * Simple random number generator.  This is private so that we can
    649  * very repeatedly control which blocks will fail.
    650  *
    651  * <mlelstv> pooka, rand()
    652  * <mlelstv> [paste]
    653  */
    654 static unsigned
    655 gimmerand(void)
    656 {
    657 
    658 	return (randstate = randstate * 1103515245 + 12345) % (0x80000000L);
    659 }
    660 
    661 /*
    662  * Block device with very simple fault injection.  Fails every
    663  * n out of BLKFAIL_MAX I/O with EIO.  n is determined by the env
    664  * variable RUMP_BLKFAIL.
    665  */
    666 void
    667 rumpblk_strategy_fail(struct buf *bp)
    668 {
    669 
    670 	if (gimmerand() % BLKFAIL_MAX >= blkfail) {
    671 		dostrategy(bp);
    672 	} else {
    673 		printf("block fault injection: failing I/O on block %lld\n",
    674 		    (long long)bp->b_blkno);
    675 		bp->b_error = EIO;
    676 		biodone(bp);
    677 	}
    678 }
    679