Home | History | Annotate | Line # | Download | only in rumpvfs
rumpblk.c revision 1.28 
      1 /*	$NetBSD: rumpblk.c,v 1.28 2009/10/07 09:23:03 pooka Exp $	*/
      2 
      3 /*
      4  * Copyright (c) 2009 Antti Kantee.  All Rights Reserved.
      5  *
      6  * Development of this software was supported by the
      7  * Finnish Cultural Foundation.
      8  *
      9  * Redistribution and use in source and binary forms, with or without
     10  * modification, are permitted provided that the following conditions
     11  * are met:
     12  * 1. Redistributions of source code must retain the above copyright
     13  *    notice, this list of conditions and the following disclaimer.
     14  * 2. Redistributions in binary form must reproduce the above copyright
     15  *    notice, this list of conditions and the following disclaimer in the
     16  *    documentation and/or other materials provided with the distribution.
     17  *
     18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
     19  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
     20  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
     21  * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
     22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
     24  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     26  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     27  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     28  * SUCH DAMAGE.
     29  */
     30 
     31 /*
     32  * Block device emulation.  Presents a block device interface and
     33  * uses rumpuser system calls to satisfy I/O requests.
     34  *
     35  * We provide fault injection.  The driver can be made to fail
     36  * I/O occasionally.
     37  *
     38  * The driver also provides an optimization for regular files by
     39  * using memory-mapped I/O.  This avoids kernel access for every
     40  * I/O operation.  It also gives finer-grained control of how to
     41  * flush data.  Additionally, in case the rump kernel dumps core,
     42  * we get way less carnage.
     43  */
     44 
     45 #include <sys/cdefs.h>
     46 __KERNEL_RCSID(0, "$NetBSD: rumpblk.c,v 1.28 2009/10/07 09:23:03 pooka Exp $");
     47 
     48 #include <sys/param.h>
     49 #include <sys/buf.h>
     50 #include <sys/conf.h>
     51 #include <sys/condvar.h>
     52 #include <sys/disklabel.h>
     53 #include <sys/evcnt.h>
     54 #include <sys/fcntl.h>
     55 #include <sys/kmem.h>
     56 #include <sys/malloc.h>
     57 #include <sys/queue.h>
     58 #include <sys/stat.h>
     59 
     60 #include <rump/rumpuser.h>
     61 
     62 #include "rump_private.h"
     63 #include "rump_vfs_private.h"
     64 
     65 #if 0
     66 #define DPRINTF(x) printf x
     67 #else
     68 #define DPRINTF(x)
     69 #endif
     70 
     71 /* Default: 16 x 1MB windows */
     72 unsigned memwinsize = (1<<20);
     73 unsigned memwincnt = 16;
     74 
     75 #define STARTWIN(off)		((off) & ~(memwinsize-1))
     76 #define INWIN(win,off)		((win)->win_off == STARTWIN(off))
     77 #define WINSIZE(rblk, win)	(MIN((rblk->rblk_size-win->win_off),memwinsize))
     78 #define WINVALID(win)		((win)->win_off != (off_t)-1)
     79 #define WINVALIDATE(win)	((win)->win_off = (off_t)-1)
     80 struct blkwin {
     81 	off_t win_off;
     82 	void *win_mem;
     83 	int win_refcnt;
     84 
     85 	TAILQ_ENTRY(blkwin) win_lru;
     86 };
     87 
     88 #define RUMPBLK_SIZE 16
     89 static struct rblkdev {
     90 	char *rblk_path;
     91 	int rblk_fd;
     92 	int rblk_opencnt;
     93 #ifdef HAS_ODIRECT
     94 	int rblk_dfd;
     95 #endif
     96 	uint64_t rblk_size;
     97 	uint64_t rblk_hostoffset;
     98 	int rblk_ftype;
     99 
    100 	/* for mmap */
    101 	int rblk_mmflags;
    102 	kmutex_t rblk_memmtx;
    103 	kcondvar_t rblk_memcv;
    104 	TAILQ_HEAD(winlru, blkwin) rblk_lruq;
    105 	bool rblk_waiting;
    106 
    107 	struct partition *rblk_curpi;
    108 	struct partition rblk_pi;
    109 	struct disklabel rblk_dl;
    110 } minors[RUMPBLK_SIZE];
    111 
    112 static struct evcnt ev_io_total;
    113 static struct evcnt ev_io_async;
    114 
    115 static struct evcnt ev_memblk_hits;
    116 static struct evcnt ev_memblk_busy;
    117 
    118 static struct evcnt ev_bwrite_total;
    119 static struct evcnt ev_bwrite_async;
    120 static struct evcnt ev_bread_total;
    121 
    122 dev_type_open(rumpblk_open);
    123 dev_type_close(rumpblk_close);
    124 dev_type_read(rumpblk_read);
    125 dev_type_write(rumpblk_write);
    126 dev_type_ioctl(rumpblk_ioctl);
    127 dev_type_strategy(rumpblk_strategy);
    128 dev_type_strategy(rumpblk_strategy_fail);
    129 dev_type_dump(rumpblk_dump);
    130 dev_type_size(rumpblk_size);
    131 
    132 static const struct bdevsw rumpblk_bdevsw = {
    133 	rumpblk_open, rumpblk_close, rumpblk_strategy, rumpblk_ioctl,
    134 	nodump, nosize, D_DISK
    135 };
    136 
    137 static const struct bdevsw rumpblk_bdevsw_fail = {
    138 	rumpblk_open, rumpblk_close, rumpblk_strategy_fail, rumpblk_ioctl,
    139 	nodump, nosize, D_DISK
    140 };
    141 
    142 static const struct cdevsw rumpblk_cdevsw = {
    143 	rumpblk_open, rumpblk_close, rumpblk_read, rumpblk_write,
    144 	rumpblk_ioctl, nostop, notty, nopoll, nommap, nokqfilter, D_DISK
    145 };
    146 
    147 /* fail every n out of BLKFAIL_MAX */
    148 #define BLKFAIL_MAX 10000
    149 static int blkfail;
    150 static unsigned randstate;
    151 static kmutex_t rumpblk_lock;
    152 
    153 static struct blkwin *
    154 getwindow(struct rblkdev *rblk, off_t off, int *wsize, int *error)
    155 {
    156 	struct blkwin *win;
    157 
    158 	mutex_enter(&rblk->rblk_memmtx);
    159  retry:
    160 	/* search for window */
    161 	TAILQ_FOREACH(win, &rblk->rblk_lruq, win_lru) {
    162 		if (INWIN(win, off) && WINVALID(win))
    163 			break;
    164 	}
    165 
    166 	/* found?  return */
    167 	if (win) {
    168 		ev_memblk_hits.ev_count++;
    169 		TAILQ_REMOVE(&rblk->rblk_lruq, win, win_lru);
    170 		goto good;
    171 	}
    172 
    173 	/*
    174 	 * Else, create new window.  If the least recently used is not
    175 	 * currently in use, reuse that.  Otherwise we need to wait.
    176 	 */
    177 	win = TAILQ_LAST(&rblk->rblk_lruq, winlru);
    178 	if (win->win_refcnt == 0) {
    179 		TAILQ_REMOVE(&rblk->rblk_lruq, win, win_lru);
    180 		mutex_exit(&rblk->rblk_memmtx);
    181 
    182 		if (WINVALID(win)) {
    183 			DPRINTF(("win %p, unmap mem %p, off 0x%" PRIx64 "\n",
    184 			    win, win->win_mem, win->win_off));
    185 			rumpuser_unmap(win->win_mem, WINSIZE(rblk, win));
    186 			WINVALIDATE(win);
    187 		}
    188 
    189 		win->win_off = STARTWIN(off);
    190 		win->win_mem = rumpuser_filemmap(rblk->rblk_fd, win->win_off,
    191 		    WINSIZE(rblk, win), rblk->rblk_mmflags, error);
    192 		DPRINTF(("win %p, off 0x%" PRIx64 ", mem %p\n",
    193 		    win, win->win_off, win->win_mem));
    194 
    195 		mutex_enter(&rblk->rblk_memmtx);
    196 		if (win->win_mem == NULL) {
    197 			WINVALIDATE(win);
    198 			TAILQ_INSERT_TAIL(&rblk->rblk_lruq, win, win_lru);
    199 			mutex_exit(&rblk->rblk_memmtx);
    200 			return NULL;
    201 		}
    202 	} else {
    203 		DPRINTF(("memwin wait\n"));
    204 		ev_memblk_busy.ev_count++;
    205 
    206 		rblk->rblk_waiting = true;
    207 		cv_wait(&rblk->rblk_memcv, &rblk->rblk_memmtx);
    208 		goto retry;
    209 	}
    210 
    211  good:
    212 	KASSERT(win);
    213 	win->win_refcnt++;
    214 	TAILQ_INSERT_HEAD(&rblk->rblk_lruq, win, win_lru);
    215 	mutex_exit(&rblk->rblk_memmtx);
    216 	*wsize = MIN(*wsize, memwinsize - (off-win->win_off));
    217 	KASSERT(*wsize);
    218 
    219 	return win;
    220 }
    221 
    222 static void
    223 putwindow(struct rblkdev *rblk, struct blkwin *win)
    224 {
    225 
    226 	mutex_enter(&rblk->rblk_memmtx);
    227 	if (--win->win_refcnt == 0 && rblk->rblk_waiting) {
    228 		rblk->rblk_waiting = false;
    229 		cv_signal(&rblk->rblk_memcv);
    230 	}
    231 	KASSERT(win->win_refcnt >= 0);
    232 	mutex_exit(&rblk->rblk_memmtx);
    233 }
    234 
    235 static void
    236 wincleanup(struct rblkdev *rblk)
    237 {
    238 	struct blkwin *win;
    239 
    240 	while ((win = TAILQ_FIRST(&rblk->rblk_lruq)) != NULL) {
    241 		TAILQ_REMOVE(&rblk->rblk_lruq, win, win_lru);
    242 		if (WINVALID(win)) {
    243 			DPRINTF(("cleanup win %p addr %p\n",
    244 			    win, win->win_mem));
    245 			rumpuser_unmap(win->win_mem, WINSIZE(rblk, win));
    246 		}
    247 		kmem_free(win, sizeof(*win));
    248 	}
    249 	rblk->rblk_mmflags = 0;
    250 }
    251 
    252 int
    253 rumpblk_init(void)
    254 {
    255 	char buf[64];
    256 	int rumpblk = RUMPBLK;
    257 	unsigned tmp;
    258 	int error, i;
    259 
    260 	mutex_init(&rumpblk_lock, MUTEX_DEFAULT, IPL_NONE);
    261 
    262 	if (rumpuser_getenv("RUMP_BLKFAIL", buf, sizeof(buf), &error) == 0) {
    263 		blkfail = strtoul(buf, NULL, 10);
    264 		/* fail everything */
    265 		if (blkfail > BLKFAIL_MAX)
    266 			blkfail = BLKFAIL_MAX;
    267 		if (rumpuser_getenv("RUMP_BLKFAIL_SEED", buf, sizeof(buf),
    268 		    &error) == 0) {
    269 			randstate = strtoul(buf, NULL, 10);
    270 		} else {
    271 			randstate = arc4random();
    272 		}
    273 		printf("rumpblk: FAULT INJECTION ACTIVE! fail %d/%d. "
    274 		    "seed %u\n", blkfail, BLKFAIL_MAX, randstate);
    275 	} else {
    276 		blkfail = 0;
    277 	}
    278 
    279 	if (rumpuser_getenv("RUMP_BLKWINSIZE", buf, sizeof(buf), &error) == 0) {
    280 		printf("rumpblk: ");
    281 		tmp = strtoul(buf, NULL, 10);
    282 		if (tmp && !(tmp & (tmp-1)))
    283 			memwinsize = tmp;
    284 		else
    285 			printf("invalid RUMP_BLKWINSIZE %d, ", tmp);
    286 		printf("using %d for memwinsize\n", memwinsize);
    287 	}
    288 	if (rumpuser_getenv("RUMP_BLKWINCOUNT", buf, sizeof(buf), &error) == 0){
    289 		printf("rumpblk: ");
    290 		tmp = strtoul(buf, NULL, 10);
    291 		if (tmp)
    292 			memwincnt = tmp;
    293 		else
    294 			printf("invalid RUMP_BLKWINCOUNT %d, ", tmp);
    295 		printf("using %d for memwincount\n", memwincnt);
    296 	}
    297 
    298 	memset(minors, 0, sizeof(minors));
    299 	for (i = 0; i < RUMPBLK_SIZE; i++) {
    300 		mutex_init(&minors[i].rblk_memmtx, MUTEX_DEFAULT, IPL_NONE);
    301 		cv_init(&minors[i].rblk_memcv, "rblkmcv");
    302 	}
    303 
    304 	evcnt_attach_dynamic(&ev_io_total, EVCNT_TYPE_MISC, NULL,
    305 	    "rumpblk", "rumpblk I/O reqs");
    306 	evcnt_attach_dynamic(&ev_io_async, EVCNT_TYPE_MISC, NULL,
    307 	    "rumpblk", "rumpblk async I/O");
    308 
    309 	evcnt_attach_dynamic(&ev_bread_total, EVCNT_TYPE_MISC, NULL,
    310 	    "rumpblk", "rumpblk bytes read");
    311 	evcnt_attach_dynamic(&ev_bwrite_total, EVCNT_TYPE_MISC, NULL,
    312 	    "rumpblk", "rumpblk bytes written");
    313 	evcnt_attach_dynamic(&ev_bwrite_async, EVCNT_TYPE_MISC, NULL,
    314 	    "rumpblk", "rumpblk bytes written async");
    315 
    316 	evcnt_attach_dynamic(&ev_memblk_hits, EVCNT_TYPE_MISC, NULL,
    317 	    "rumpblk", "memblk window hits");
    318 	evcnt_attach_dynamic(&ev_memblk_busy, EVCNT_TYPE_MISC, NULL,
    319 	    "rumpblk", "memblk all windows busy");
    320 
    321 	if (blkfail) {
    322 		return devsw_attach("rumpblk", &rumpblk_bdevsw_fail, &rumpblk,
    323 		    &rumpblk_cdevsw, &rumpblk);
    324 	} else {
    325 		return devsw_attach("rumpblk", &rumpblk_bdevsw, &rumpblk,
    326 		    &rumpblk_cdevsw, &rumpblk);
    327 	}
    328 }
    329 
    330 /* XXX: no deregister */
    331 int
    332 rumpblk_register(const char *path, devminor_t *dmin,
    333 	uint64_t offset, uint64_t size)
    334 {
    335 	struct rblkdev *rblk;
    336 	uint64_t flen;
    337 	size_t len;
    338 	int ftype, error, dummy, i;
    339 	int fd;
    340 
    341 	/* devices might not report correct size unless they're open */
    342 	fd = rumpuser_open(path, O_RDONLY, &error);
    343 	if (fd == -1)
    344 		return error;
    345 	rumpuser_getfileinfo(path, &flen, &ftype, &error);
    346 	rumpuser_close(fd, &dummy);
    347 	if (error)
    348 		return error;
    349 
    350 	/* verify host file is of supported type */
    351 	if (!(ftype == RUMPUSER_FT_REG
    352 	   || ftype == RUMPUSER_FT_BLK
    353 	   || ftype == RUMPUSER_FT_CHR))
    354 		return EINVAL;
    355 
    356 	mutex_enter(&rumpblk_lock);
    357 	for (i = 0; i < RUMPBLK_SIZE; i++) {
    358 		if (minors[i].rblk_path&&strcmp(minors[i].rblk_path, path)==0) {
    359 			mutex_exit(&rumpblk_lock);
    360 			*dmin = i;
    361 			return 0;
    362 		}
    363 	}
    364 
    365 	for (i = 0; i < RUMPBLK_SIZE; i++)
    366 		if (minors[i].rblk_path == NULL)
    367 			break;
    368 	if (i == RUMPBLK_SIZE) {
    369 		mutex_exit(&rumpblk_lock);
    370 		return EBUSY;
    371 	}
    372 
    373 	rblk = &minors[i];
    374 	len = strlen(path);
    375 	rblk->rblk_path = malloc(len + 1, M_TEMP, M_WAITOK);
    376 	strcpy(rblk->rblk_path, path);
    377 	rblk->rblk_fd = -1;
    378 	rblk->rblk_hostoffset = offset;
    379 	if (size == RUMPBLK_SIZENOTSET) {
    380 		KASSERT(size + offset <= flen);
    381 		rblk->rblk_size = size;
    382 	} else {
    383 		KASSERT(offset < flen);
    384 		rblk->rblk_size = flen - offset;
    385 	}
    386 	rblk->rblk_ftype = ftype;
    387 	mutex_exit(&rumpblk_lock);
    388 
    389 	*dmin = i;
    390 	return 0;
    391 }
    392 
    393 int
    394 rumpblk_open(dev_t dev, int flag, int fmt, struct lwp *l)
    395 {
    396 	struct rblkdev *rblk = &minors[minor(dev)];
    397 	uint64_t fsize, off;
    398 	int dummy;
    399 	int error, fd;
    400 
    401 	if (rblk->rblk_path == NULL)
    402 		return ENXIO;
    403 
    404 	if (rblk->rblk_fd != -1)
    405 		return 0; /* XXX: refcount, open mode */
    406 	fd = rumpuser_open(rblk->rblk_path, OFLAGS(flag), &error);
    407 	if (error)
    408 		return error;
    409 
    410 #ifdef HAS_ODIRECT
    411 	rblk->rblk_dfd = rumpuser_open(rblk->rblk_path,
    412 	    OFLAGS(flag) | O_DIRECT, &error);
    413 	if (error)
    414 		return error;
    415 #endif
    416 
    417 	fsize = rblk->rblk_size;
    418 	off = rblk->rblk_hostoffset;
    419 
    420 	if (rblk->rblk_ftype == RUMPUSER_FT_REG) {
    421 		struct blkwin *win;
    422 		int i, winsize;
    423 
    424 		/*
    425 		 * Use mmap to access a regular file.  Allocate and
    426 		 * cache initial windows here.  Failure to allocate one
    427 		 * means fallback to read/write i/o.
    428 		 */
    429 
    430 		rblk->rblk_mmflags = 0;
    431 		if (flag & FREAD)
    432 			rblk->rblk_mmflags |= RUMPUSER_FILEMMAP_READ;
    433 		if (flag & FWRITE) {
    434 			rblk->rblk_mmflags |= RUMPUSER_FILEMMAP_WRITE;
    435 			rblk->rblk_mmflags |= RUMPUSER_FILEMMAP_SHARED;
    436 		}
    437 
    438 		TAILQ_INIT(&rblk->rblk_lruq);
    439 		rblk->rblk_fd = fd;
    440 
    441 		for (i = 0; i < memwincnt && off + i*memwinsize < fsize; i++) {
    442 			win = kmem_zalloc(sizeof(*win), KM_SLEEP);
    443 			WINVALIDATE(win);
    444 			TAILQ_INSERT_TAIL(&rblk->rblk_lruq, win, win_lru);
    445 
    446 			/*
    447 			 * Allocate first windows.  Here we just generally
    448 			 * make sure a) we can mmap at all b) we have the
    449 			 * necessary VA available
    450 			 */
    451 			winsize = 1;
    452 			win = getwindow(rblk, off + i*memwinsize, &winsize,
    453 			    &error);
    454 			if (win) {
    455 				putwindow(rblk, win);
    456 			} else {
    457 				wincleanup(rblk);
    458 				break;
    459 			}
    460 		}
    461 
    462 		memset(&rblk->rblk_dl, 0, sizeof(rblk->rblk_dl));
    463 		rblk->rblk_pi.p_size = fsize >> DEV_BSHIFT;
    464 		rblk->rblk_dl.d_secsize = DEV_BSIZE;
    465 		rblk->rblk_curpi = &rblk->rblk_pi;
    466 	} else {
    467 		if (rumpuser_ioctl(fd, DIOCGDINFO, &rblk->rblk_dl,
    468 		    &error) == -1) {
    469 			KASSERT(error);
    470 			rumpuser_close(fd, &dummy);
    471 			return error;
    472 		}
    473 
    474 		rblk->rblk_fd = fd;
    475 		rblk->rblk_curpi = &rblk->rblk_dl.d_partitions[0];
    476 	}
    477 
    478 	KASSERT(rblk->rblk_fd != -1);
    479 	return 0;
    480 }
    481 
    482 int
    483 rumpblk_close(dev_t dev, int flag, int fmt, struct lwp *l)
    484 {
    485 	struct rblkdev *rblk = &minors[minor(dev)];
    486 	int dummy;
    487 
    488 	if (rblk->rblk_mmflags)
    489 		wincleanup(rblk);
    490 	rumpuser_fsync(rblk->rblk_fd, &dummy);
    491 	rumpuser_close(rblk->rblk_fd, &dummy);
    492 	rblk->rblk_fd = -1;
    493 
    494 	return 0;
    495 }
    496 
    497 int
    498 rumpblk_ioctl(dev_t dev, u_long xfer, void *addr, int flag, struct lwp *l)
    499 {
    500 	struct rblkdev *rblk = &minors[minor(dev)];
    501 	int rv, error;
    502 
    503 	if (xfer == DIOCGPART) {
    504 		struct partinfo *pi = (struct partinfo *)addr;
    505 
    506 		pi->part = rblk->rblk_curpi;
    507 		pi->disklab = &rblk->rblk_dl;
    508 
    509 		return 0;
    510 	}
    511 
    512 	rv = rumpuser_ioctl(rblk->rblk_fd, xfer, addr, &error);
    513 	if (rv == -1)
    514 		return error;
    515 
    516 	return 0;
    517 }
    518 
    519 static int
    520 do_physio(dev_t dev, struct uio *uio, int which)
    521 {
    522 	void (*strat)(struct buf *);
    523 
    524 	if (blkfail)
    525 		strat = rumpblk_strategy_fail;
    526 	else
    527 		strat = rumpblk_strategy;
    528 
    529 	return physio(strat, NULL, dev, which, minphys, uio);
    530 }
    531 
    532 int
    533 rumpblk_read(dev_t dev, struct uio *uio, int flags)
    534 {
    535 
    536 	return do_physio(dev, uio, B_READ);
    537 }
    538 
    539 int
    540 rumpblk_write(dev_t dev, struct uio *uio, int flags)
    541 {
    542 
    543 	return do_physio(dev, uio, B_WRITE);
    544 }
    545 
    546 static void
    547 dostrategy(struct buf *bp)
    548 {
    549 	struct rblkdev *rblk = &minors[minor(bp->b_dev)];
    550 	off_t off;
    551 	int async = bp->b_flags & B_ASYNC;
    552 	int error;
    553 
    554 	/* collect statistics */
    555 	ev_io_total.ev_count++;
    556 	if (async)
    557 		ev_io_async.ev_count++;
    558 	if (BUF_ISWRITE(bp)) {
    559 		ev_bwrite_total.ev_count += bp->b_bcount;
    560 		if (async)
    561 			ev_bwrite_async.ev_count += bp->b_bcount;
    562 	} else {
    563 		ev_bread_total.ev_count++;
    564 	}
    565 
    566 	off = bp->b_blkno << DEV_BSHIFT;
    567 	off += rblk->rblk_hostoffset;
    568 	/*
    569 	 * Do bounds checking if we're working on a file.  Otherwise
    570 	 * invalid file systems might attempt to read beyond EOF.  This
    571 	 * is bad(tm) especially on mmapped images.  This is essentially
    572 	 * the kernel bounds_check() routines.
    573 	 */
    574 	if (off + bp->b_bcount > rblk->rblk_size) {
    575 		int64_t sz = rblk->rblk_size - off;
    576 
    577 		/* EOF */
    578 		if (sz == 0) {
    579 			rump_biodone(bp, 0, 0);
    580 			return;
    581 		}
    582 		/* beyond EOF ==> error */
    583 		if (sz < 0) {
    584 			rump_biodone(bp, 0, EINVAL);
    585 			return;
    586 		}
    587 
    588 		/* truncate to device size */
    589 		bp->b_bcount = sz;
    590 	}
    591 
    592 	DPRINTF(("rumpblk_strategy: 0x%x bytes %s off 0x%" PRIx64
    593 	    " (0x%" PRIx64 " - 0x%" PRIx64 "), %ssync\n",
    594 	    bp->b_bcount, BUF_ISREAD(bp) ? "READ" : "WRITE",
    595 	    off, off, (off + bp->b_bcount), async ? "a" : ""));
    596 
    597 	/* mmap?  handle here and return */
    598 	if (rblk->rblk_mmflags) {
    599 		struct blkwin *win;
    600 		int winsize, iodone;
    601 		uint8_t *ioaddr, *bufaddr;
    602 
    603 		for (iodone = 0; iodone < bp->b_bcount;
    604 		    iodone += winsize, off += winsize) {
    605 			winsize = bp->b_bcount - iodone;
    606 			win = getwindow(rblk, off, &winsize, &error);
    607 			if (win == NULL) {
    608 				rump_biodone(bp, iodone, error);
    609 				return;
    610 			}
    611 
    612 			ioaddr = (uint8_t *)win->win_mem + (off-STARTWIN(off));
    613 			bufaddr = (uint8_t *)bp->b_data + iodone;
    614 
    615 			DPRINTF(("strat: %p off 0x%" PRIx64
    616 			    ", ioaddr %p (%p)/buf %p\n", win,
    617 			    win->win_off, ioaddr, win->win_mem, bufaddr));
    618 			if (BUF_ISREAD(bp)) {
    619 				memcpy(bufaddr, ioaddr, winsize);
    620 			} else {
    621 				memcpy(ioaddr, bufaddr, winsize);
    622 			}
    623 
    624 			/* synchronous write, sync bits back to disk */
    625 			if (BUF_ISWRITE(bp) && !async) {
    626 				rumpuser_memsync(ioaddr, winsize, &error);
    627 			}
    628 			putwindow(rblk, win);
    629 		}
    630 
    631 		rump_biodone(bp, bp->b_bcount, 0);
    632 		return;
    633 	}
    634 
    635 	/*
    636 	 * Do I/O.  We have different paths for async and sync I/O.
    637 	 * Async I/O is done by passing a request to rumpuser where
    638 	 * it is executed.  The rumpuser routine then calls
    639 	 * biodone() to signal any waiters in the kernel.  I/O's are
    640 	 * executed in series.  Technically executing them in parallel
    641 	 * would produce better results, but then we'd need either
    642 	 * more threads or posix aio.  Maybe worth investigating
    643 	 * this later.
    644 	 *
    645 	 * Using bufq here might be a good idea.
    646 	 */
    647 
    648 	if (rump_threads) {
    649 		struct rumpuser_aio *rua;
    650 		int op, fd;
    651 
    652 		fd = rblk->rblk_fd;
    653 		if (BUF_ISREAD(bp)) {
    654 			op = RUA_OP_READ;
    655 		} else {
    656 			op = RUA_OP_WRITE;
    657 			if (!async) {
    658 				/* O_DIRECT not fully automatic yet */
    659 #ifdef HAS_ODIRECT
    660 				if ((off & (DEV_BSIZE-1)) == 0
    661 				    && ((intptr_t)bp->b_data&(DEV_BSIZE-1)) == 0
    662 				    && (bp->b_bcount & (DEV_BSIZE-1)) == 0)
    663 					fd = rblk->rblk_dfd;
    664 				else
    665 #endif
    666 					op |= RUA_OP_SYNC;
    667 			}
    668 		}
    669 
    670 		rumpuser_mutex_enter(&rumpuser_aio_mtx);
    671 		while ((rumpuser_aio_head+1) % N_AIOS == rumpuser_aio_tail) {
    672 			rumpuser_cv_wait(&rumpuser_aio_cv, &rumpuser_aio_mtx);
    673 		}
    674 
    675 		rua = &rumpuser_aios[rumpuser_aio_head];
    676 		KASSERT(rua->rua_bp == NULL);
    677 		rua->rua_fd = fd;
    678 		rua->rua_data = bp->b_data;
    679 		rua->rua_dlen = bp->b_bcount;
    680 		rua->rua_off = off;
    681 		rua->rua_bp = bp;
    682 		rua->rua_op = op;
    683 
    684 		/* insert into queue & signal */
    685 		rumpuser_aio_head = (rumpuser_aio_head+1) % N_AIOS;
    686 		rumpuser_cv_signal(&rumpuser_aio_cv);
    687 		rumpuser_mutex_exit(&rumpuser_aio_mtx);
    688 	} else {
    689 		if (BUF_ISREAD(bp)) {
    690 			rumpuser_read_bio(rblk->rblk_fd, bp->b_data,
    691 			    bp->b_bcount, off, rump_biodone, bp);
    692 		} else {
    693 			rumpuser_write_bio(rblk->rblk_fd, bp->b_data,
    694 			    bp->b_bcount, off, rump_biodone, bp);
    695 		}
    696 		if (BUF_ISWRITE(bp) && !async)
    697 			rumpuser_fsync(rblk->rblk_fd, &error);
    698 	}
    699 }
    700 
    701 void
    702 rumpblk_strategy(struct buf *bp)
    703 {
    704 
    705 	dostrategy(bp);
    706 }
    707 
    708 /*
    709  * Simple random number generator.  This is private so that we can
    710  * very repeatedly control which blocks will fail.
    711  *
    712  * <mlelstv> pooka, rand()
    713  * <mlelstv> [paste]
    714  */
    715 static unsigned
    716 gimmerand(void)
    717 {
    718 
    719 	return (randstate = randstate * 1103515245 + 12345) % (0x80000000L);
    720 }
    721 
    722 /*
    723  * Block device with very simple fault injection.  Fails every
    724  * n out of BLKFAIL_MAX I/O with EIO.  n is determined by the env
    725  * variable RUMP_BLKFAIL.
    726  */
    727 void
    728 rumpblk_strategy_fail(struct buf *bp)
    729 {
    730 
    731 	if (gimmerand() % BLKFAIL_MAX >= blkfail) {
    732 		dostrategy(bp);
    733 	} else {
    734 		printf("block fault injection: failing I/O on block %lld\n",
    735 		    (long long)bp->b_blkno);
    736 		bp->b_error = EIO;
    737 		biodone(bp);
    738 	}
    739 }
    740