Home | History | Annotate | Line # | Download | only in rumpvfs
rumpblk.c revision 1.27
      1  1.27   pooka /*	$NetBSD: rumpblk.c,v 1.27 2009/10/07 09:17:54 pooka Exp $	*/
      2   1.1   pooka 
      3   1.1   pooka /*
      4   1.1   pooka  * Copyright (c) 2009 Antti Kantee.  All Rights Reserved.
      5   1.1   pooka  *
      6   1.1   pooka  * Development of this software was supported by the
      7   1.1   pooka  * Finnish Cultural Foundation.
      8   1.1   pooka  *
      9   1.1   pooka  * Redistribution and use in source and binary forms, with or without
     10   1.1   pooka  * modification, are permitted provided that the following conditions
     11   1.1   pooka  * are met:
     12   1.1   pooka  * 1. Redistributions of source code must retain the above copyright
     13   1.1   pooka  *    notice, this list of conditions and the following disclaimer.
     14   1.1   pooka  * 2. Redistributions in binary form must reproduce the above copyright
     15   1.1   pooka  *    notice, this list of conditions and the following disclaimer in the
     16   1.1   pooka  *    documentation and/or other materials provided with the distribution.
     17   1.1   pooka  *
     18   1.1   pooka  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
     19   1.1   pooka  * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
     20   1.1   pooka  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
     21   1.1   pooka  * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
     22   1.1   pooka  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     23   1.1   pooka  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
     24   1.1   pooka  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     25   1.1   pooka  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     26   1.1   pooka  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     27   1.1   pooka  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     28   1.1   pooka  * SUCH DAMAGE.
     29   1.1   pooka  */
     30   1.1   pooka 
     31   1.1   pooka /*
     32   1.1   pooka  * Block device emulation.  Presents a block device interface and
     33   1.1   pooka  * uses rumpuser system calls to satisfy I/O requests.
     34  1.16   pooka  *
     35  1.16   pooka  * We provide fault injection.  The driver can be made to fail
     36  1.16   pooka  * I/O occasionally.
     37  1.16   pooka  *
     38  1.16   pooka  * The driver also provides an optimization for regular files by
     39  1.16   pooka  * using memory-mapped I/O.  This avoids kernel access for every
     40  1.16   pooka  * I/O operation.  It also gives finer-grained control of how to
     41  1.16   pooka  * flush data.  Additionally, in case the rump kernel dumps core,
     42  1.16   pooka  * we get way less carnage.
     43   1.1   pooka  */
     44   1.1   pooka 
     45   1.1   pooka #include <sys/cdefs.h>
     46  1.27   pooka __KERNEL_RCSID(0, "$NetBSD: rumpblk.c,v 1.27 2009/10/07 09:17:54 pooka Exp $");
     47   1.1   pooka 
     48   1.1   pooka #include <sys/param.h>
     49   1.1   pooka #include <sys/buf.h>
     50   1.1   pooka #include <sys/conf.h>
     51  1.16   pooka #include <sys/condvar.h>
     52   1.1   pooka #include <sys/disklabel.h>
     53  1.18   pooka #include <sys/evcnt.h>
     54   1.1   pooka #include <sys/fcntl.h>
     55   1.1   pooka #include <sys/kmem.h>
     56   1.1   pooka #include <sys/malloc.h>
     57  1.16   pooka #include <sys/queue.h>
     58   1.1   pooka #include <sys/stat.h>
     59   1.1   pooka 
     60   1.1   pooka #include <rump/rumpuser.h>
     61   1.1   pooka 
     62   1.1   pooka #include "rump_private.h"
     63   1.1   pooka #include "rump_vfs_private.h"
     64   1.1   pooka 
     65  1.16   pooka #if 0
     66  1.16   pooka #define DPRINTF(x) printf x
     67  1.16   pooka #else
     68  1.16   pooka #define DPRINTF(x)
     69  1.16   pooka #endif
     70  1.16   pooka 
     71  1.19   pooka /* Default: 16 x 1MB windows */
     72  1.19   pooka unsigned memwinsize = (1<<20);
     73  1.19   pooka unsigned memwincnt = 16;
     74  1.19   pooka 
     75  1.19   pooka #define STARTWIN(off)		((off) & ~(memwinsize-1))
     76  1.16   pooka #define INWIN(win,off)		((win)->win_off == STARTWIN(off))
     77  1.19   pooka #define WINSIZE(rblk, win)	(MIN((rblk->rblk_size-win->win_off),memwinsize))
     78  1.16   pooka #define WINVALID(win)		((win)->win_off != (off_t)-1)
     79  1.16   pooka #define WINVALIDATE(win)	((win)->win_off = (off_t)-1)
     80  1.16   pooka struct blkwin {
     81  1.16   pooka 	off_t win_off;
     82  1.16   pooka 	void *win_mem;
     83  1.16   pooka 	int win_refcnt;
     84  1.16   pooka 
     85  1.16   pooka 	TAILQ_ENTRY(blkwin) win_lru;
     86  1.16   pooka };
     87  1.16   pooka 
     88   1.1   pooka #define RUMPBLK_SIZE 16
     89   1.1   pooka static struct rblkdev {
     90   1.1   pooka 	char *rblk_path;
     91   1.1   pooka 	int rblk_fd;
     92  1.25   pooka 	int rblk_opencnt;
     93  1.20   pooka #ifdef HAS_ODIRECT
     94  1.20   pooka 	int rblk_dfd;
     95  1.20   pooka #endif
     96  1.26   pooka 	uint64_t rblk_size;
     97  1.27   pooka 	uint64_t rblk_hostoffset;
     98  1.27   pooka 	int rblk_ftype;
     99  1.16   pooka 
    100  1.16   pooka 	/* for mmap */
    101  1.16   pooka 	int rblk_mmflags;
    102  1.16   pooka 	kmutex_t rblk_memmtx;
    103  1.16   pooka 	kcondvar_t rblk_memcv;
    104  1.16   pooka 	TAILQ_HEAD(winlru, blkwin) rblk_lruq;
    105  1.16   pooka 	bool rblk_waiting;
    106   1.1   pooka 
    107   1.1   pooka 	struct partition *rblk_curpi;
    108   1.1   pooka 	struct partition rblk_pi;
    109   1.1   pooka 	struct disklabel rblk_dl;
    110   1.1   pooka } minors[RUMPBLK_SIZE];
    111   1.1   pooka 
    112  1.20   pooka static struct evcnt ev_io_total;
    113  1.20   pooka static struct evcnt ev_io_async;
    114  1.20   pooka 
    115  1.20   pooka static struct evcnt ev_memblk_hits;
    116  1.20   pooka static struct evcnt ev_memblk_busy;
    117  1.20   pooka 
    118  1.20   pooka static struct evcnt ev_bwrite_total;
    119  1.20   pooka static struct evcnt ev_bwrite_async;
    120  1.20   pooka static struct evcnt ev_bread_total;
    121  1.18   pooka 
    122   1.1   pooka dev_type_open(rumpblk_open);
    123   1.1   pooka dev_type_close(rumpblk_close);
    124   1.1   pooka dev_type_read(rumpblk_read);
    125   1.1   pooka dev_type_write(rumpblk_write);
    126   1.1   pooka dev_type_ioctl(rumpblk_ioctl);
    127   1.1   pooka dev_type_strategy(rumpblk_strategy);
    128   1.3   pooka dev_type_strategy(rumpblk_strategy_fail);
    129   1.1   pooka dev_type_dump(rumpblk_dump);
    130   1.1   pooka dev_type_size(rumpblk_size);
    131   1.1   pooka 
    132   1.1   pooka static const struct bdevsw rumpblk_bdevsw = {
    133   1.1   pooka 	rumpblk_open, rumpblk_close, rumpblk_strategy, rumpblk_ioctl,
    134   1.1   pooka 	nodump, nosize, D_DISK
    135   1.1   pooka };
    136   1.1   pooka 
    137   1.3   pooka static const struct bdevsw rumpblk_bdevsw_fail = {
    138   1.3   pooka 	rumpblk_open, rumpblk_close, rumpblk_strategy_fail, rumpblk_ioctl,
    139   1.3   pooka 	nodump, nosize, D_DISK
    140   1.3   pooka };
    141   1.3   pooka 
    142   1.1   pooka static const struct cdevsw rumpblk_cdevsw = {
    143   1.1   pooka 	rumpblk_open, rumpblk_close, rumpblk_read, rumpblk_write,
    144   1.1   pooka 	rumpblk_ioctl, nostop, notty, nopoll, nommap, nokqfilter, D_DISK
    145   1.1   pooka };
    146   1.1   pooka 
    147   1.3   pooka /* fail every n out of BLKFAIL_MAX */
    148   1.3   pooka #define BLKFAIL_MAX 10000
    149   1.3   pooka static int blkfail;
    150   1.3   pooka static unsigned randstate;
    151  1.24   pooka static kmutex_t rumpblk_lock;
    152   1.1   pooka 
    153  1.16   pooka static struct blkwin *
    154  1.16   pooka getwindow(struct rblkdev *rblk, off_t off, int *wsize, int *error)
    155  1.16   pooka {
    156  1.16   pooka 	struct blkwin *win;
    157  1.16   pooka 
    158  1.16   pooka 	mutex_enter(&rblk->rblk_memmtx);
    159  1.16   pooka  retry:
    160  1.16   pooka 	/* search for window */
    161  1.16   pooka 	TAILQ_FOREACH(win, &rblk->rblk_lruq, win_lru) {
    162  1.16   pooka 		if (INWIN(win, off) && WINVALID(win))
    163  1.16   pooka 			break;
    164  1.16   pooka 	}
    165  1.16   pooka 
    166  1.16   pooka 	/* found?  return */
    167  1.16   pooka 	if (win) {
    168  1.20   pooka 		ev_memblk_hits.ev_count++;
    169  1.16   pooka 		TAILQ_REMOVE(&rblk->rblk_lruq, win, win_lru);
    170  1.16   pooka 		goto good;
    171  1.16   pooka 	}
    172  1.16   pooka 
    173  1.16   pooka 	/*
    174  1.16   pooka 	 * Else, create new window.  If the least recently used is not
    175  1.16   pooka 	 * currently in use, reuse that.  Otherwise we need to wait.
    176  1.16   pooka 	 */
    177  1.16   pooka 	win = TAILQ_LAST(&rblk->rblk_lruq, winlru);
    178  1.16   pooka 	if (win->win_refcnt == 0) {
    179  1.16   pooka 		TAILQ_REMOVE(&rblk->rblk_lruq, win, win_lru);
    180  1.16   pooka 		mutex_exit(&rblk->rblk_memmtx);
    181  1.16   pooka 
    182  1.16   pooka 		if (WINVALID(win)) {
    183  1.16   pooka 			DPRINTF(("win %p, unmap mem %p, off 0x%" PRIx64 "\n",
    184  1.16   pooka 			    win, win->win_mem, win->win_off));
    185  1.16   pooka 			rumpuser_unmap(win->win_mem, WINSIZE(rblk, win));
    186  1.16   pooka 			WINVALIDATE(win);
    187  1.16   pooka 		}
    188  1.16   pooka 
    189  1.16   pooka 		win->win_off = STARTWIN(off);
    190  1.16   pooka 		win->win_mem = rumpuser_filemmap(rblk->rblk_fd, win->win_off,
    191  1.16   pooka 		    WINSIZE(rblk, win), rblk->rblk_mmflags, error);
    192  1.16   pooka 		DPRINTF(("win %p, off 0x%" PRIx64 ", mem %p\n",
    193  1.16   pooka 		    win, win->win_off, win->win_mem));
    194  1.16   pooka 
    195  1.16   pooka 		mutex_enter(&rblk->rblk_memmtx);
    196  1.16   pooka 		if (win->win_mem == NULL) {
    197  1.16   pooka 			WINVALIDATE(win);
    198  1.16   pooka 			TAILQ_INSERT_TAIL(&rblk->rblk_lruq, win, win_lru);
    199  1.16   pooka 			mutex_exit(&rblk->rblk_memmtx);
    200  1.16   pooka 			return NULL;
    201  1.16   pooka 		}
    202  1.16   pooka 	} else {
    203  1.16   pooka 		DPRINTF(("memwin wait\n"));
    204  1.20   pooka 		ev_memblk_busy.ev_count++;
    205  1.18   pooka 
    206  1.16   pooka 		rblk->rblk_waiting = true;
    207  1.16   pooka 		cv_wait(&rblk->rblk_memcv, &rblk->rblk_memmtx);
    208  1.16   pooka 		goto retry;
    209  1.16   pooka 	}
    210  1.16   pooka 
    211  1.16   pooka  good:
    212  1.16   pooka 	KASSERT(win);
    213  1.16   pooka 	win->win_refcnt++;
    214  1.16   pooka 	TAILQ_INSERT_HEAD(&rblk->rblk_lruq, win, win_lru);
    215  1.16   pooka 	mutex_exit(&rblk->rblk_memmtx);
    216  1.19   pooka 	*wsize = MIN(*wsize, memwinsize - (off-win->win_off));
    217  1.16   pooka 	KASSERT(*wsize);
    218  1.16   pooka 
    219  1.16   pooka 	return win;
    220  1.16   pooka }
    221  1.16   pooka 
    222  1.16   pooka static void
    223  1.16   pooka putwindow(struct rblkdev *rblk, struct blkwin *win)
    224  1.16   pooka {
    225  1.16   pooka 
    226  1.16   pooka 	mutex_enter(&rblk->rblk_memmtx);
    227  1.16   pooka 	if (--win->win_refcnt == 0 && rblk->rblk_waiting) {
    228  1.16   pooka 		rblk->rblk_waiting = false;
    229  1.16   pooka 		cv_signal(&rblk->rblk_memcv);
    230  1.16   pooka 	}
    231  1.16   pooka 	KASSERT(win->win_refcnt >= 0);
    232  1.16   pooka 	mutex_exit(&rblk->rblk_memmtx);
    233  1.16   pooka }
    234  1.16   pooka 
    235  1.16   pooka static void
    236  1.16   pooka wincleanup(struct rblkdev *rblk)
    237  1.16   pooka {
    238  1.16   pooka 	struct blkwin *win;
    239  1.16   pooka 
    240  1.16   pooka 	while ((win = TAILQ_FIRST(&rblk->rblk_lruq)) != NULL) {
    241  1.16   pooka 		TAILQ_REMOVE(&rblk->rblk_lruq, win, win_lru);
    242  1.16   pooka 		if (WINVALID(win)) {
    243  1.16   pooka 			DPRINTF(("cleanup win %p addr %p\n",
    244  1.16   pooka 			    win, win->win_mem));
    245  1.16   pooka 			rumpuser_unmap(win->win_mem, WINSIZE(rblk, win));
    246  1.16   pooka 		}
    247  1.16   pooka 		kmem_free(win, sizeof(*win));
    248  1.16   pooka 	}
    249  1.16   pooka 	rblk->rblk_mmflags = 0;
    250  1.16   pooka }
    251  1.16   pooka 
    252   1.1   pooka int
    253   1.8  cegger rumpblk_init(void)
    254   1.1   pooka {
    255   1.3   pooka 	char buf[64];
    256   1.1   pooka 	int rumpblk = RUMPBLK;
    257  1.19   pooka 	unsigned tmp;
    258  1.16   pooka 	int error, i;
    259   1.3   pooka 
    260  1.24   pooka 	mutex_init(&rumpblk_lock, MUTEX_DEFAULT, IPL_NONE);
    261  1.24   pooka 
    262   1.3   pooka 	if (rumpuser_getenv("RUMP_BLKFAIL", buf, sizeof(buf), &error) == 0) {
    263   1.3   pooka 		blkfail = strtoul(buf, NULL, 10);
    264   1.3   pooka 		/* fail everything */
    265   1.3   pooka 		if (blkfail > BLKFAIL_MAX)
    266   1.3   pooka 			blkfail = BLKFAIL_MAX;
    267   1.3   pooka 		if (rumpuser_getenv("RUMP_BLKFAIL_SEED", buf, sizeof(buf),
    268   1.3   pooka 		    &error) == 0) {
    269   1.3   pooka 			randstate = strtoul(buf, NULL, 10);
    270   1.3   pooka 		} else {
    271  1.23   pooka 			randstate = arc4random();
    272   1.3   pooka 		}
    273  1.22   pooka 		printf("rumpblk: FAULT INJECTION ACTIVE! fail %d/%d. "
    274  1.22   pooka 		    "seed %u\n", blkfail, BLKFAIL_MAX, randstate);
    275   1.3   pooka 	} else {
    276   1.3   pooka 		blkfail = 0;
    277   1.3   pooka 	}
    278   1.1   pooka 
    279  1.19   pooka 	if (rumpuser_getenv("RUMP_BLKWINSIZE", buf, sizeof(buf), &error) == 0) {
    280  1.19   pooka 		printf("rumpblk: ");
    281  1.19   pooka 		tmp = strtoul(buf, NULL, 10);
    282  1.19   pooka 		if (tmp && !(tmp & (tmp-1)))
    283  1.19   pooka 			memwinsize = tmp;
    284  1.19   pooka 		else
    285  1.19   pooka 			printf("invalid RUMP_BLKWINSIZE %d, ", tmp);
    286  1.19   pooka 		printf("using %d for memwinsize\n", memwinsize);
    287  1.19   pooka 	}
    288  1.19   pooka 	if (rumpuser_getenv("RUMP_BLKWINCOUNT", buf, sizeof(buf), &error) == 0){
    289  1.19   pooka 		printf("rumpblk: ");
    290  1.19   pooka 		tmp = strtoul(buf, NULL, 10);
    291  1.19   pooka 		if (tmp)
    292  1.19   pooka 			memwincnt = tmp;
    293  1.19   pooka 		else
    294  1.19   pooka 			printf("invalid RUMP_BLKWINCOUNT %d, ", tmp);
    295  1.19   pooka 		printf("using %d for memwincount\n", memwincnt);
    296  1.19   pooka 	}
    297  1.19   pooka 
    298  1.16   pooka 	memset(minors, 0, sizeof(minors));
    299  1.16   pooka 	for (i = 0; i < RUMPBLK_SIZE; i++) {
    300  1.16   pooka 		mutex_init(&minors[i].rblk_memmtx, MUTEX_DEFAULT, IPL_NONE);
    301  1.16   pooka 		cv_init(&minors[i].rblk_memcv, "rblkmcv");
    302  1.16   pooka 	}
    303  1.16   pooka 
    304  1.20   pooka 	evcnt_attach_dynamic(&ev_io_total, EVCNT_TYPE_MISC, NULL,
    305  1.20   pooka 	    "rumpblk", "rumpblk I/O reqs");
    306  1.20   pooka 	evcnt_attach_dynamic(&ev_io_async, EVCNT_TYPE_MISC, NULL,
    307  1.20   pooka 	    "rumpblk", "rumpblk async I/O");
    308  1.20   pooka 
    309  1.20   pooka 	evcnt_attach_dynamic(&ev_bread_total, EVCNT_TYPE_MISC, NULL,
    310  1.20   pooka 	    "rumpblk", "rumpblk bytes read");
    311  1.20   pooka 	evcnt_attach_dynamic(&ev_bwrite_total, EVCNT_TYPE_MISC, NULL,
    312  1.20   pooka 	    "rumpblk", "rumpblk bytes written");
    313  1.20   pooka 	evcnt_attach_dynamic(&ev_bwrite_async, EVCNT_TYPE_MISC, NULL,
    314  1.20   pooka 	    "rumpblk", "rumpblk bytes written async");
    315  1.20   pooka 
    316  1.20   pooka 	evcnt_attach_dynamic(&ev_memblk_hits, EVCNT_TYPE_MISC, NULL,
    317  1.18   pooka 	    "rumpblk", "memblk window hits");
    318  1.20   pooka 	evcnt_attach_dynamic(&ev_memblk_busy, EVCNT_TYPE_MISC, NULL,
    319  1.18   pooka 	    "rumpblk", "memblk all windows busy");
    320  1.18   pooka 
    321   1.3   pooka 	if (blkfail) {
    322   1.3   pooka 		return devsw_attach("rumpblk", &rumpblk_bdevsw_fail, &rumpblk,
    323   1.3   pooka 		    &rumpblk_cdevsw, &rumpblk);
    324   1.3   pooka 	} else {
    325   1.3   pooka 		return devsw_attach("rumpblk", &rumpblk_bdevsw, &rumpblk,
    326   1.3   pooka 		    &rumpblk_cdevsw, &rumpblk);
    327   1.3   pooka 	}
    328   1.1   pooka }
    329   1.1   pooka 
    330  1.24   pooka /* XXX: no deregister */
    331   1.1   pooka int
    332  1.27   pooka rumpblk_register(const char *path, devminor_t *dmin,
    333  1.27   pooka 	uint64_t offset, uint64_t size)
    334   1.1   pooka {
    335  1.27   pooka 	struct rblkdev *rblk;
    336  1.24   pooka 	uint64_t flen;
    337   1.1   pooka 	size_t len;
    338  1.27   pooka 	int ftype, error, dummy, i;
    339  1.27   pooka 	int fd;
    340  1.24   pooka 
    341  1.27   pooka 	/* devices might not report correct size unless they're open */
    342  1.27   pooka 	fd = rumpuser_open(path, O_RDONLY, &error);
    343  1.27   pooka 	if (fd == -1)
    344  1.24   pooka 		return error;
    345  1.27   pooka 	rumpuser_getfileinfo(path, &flen, &ftype, &error);
    346  1.27   pooka 	rumpuser_close(fd, &dummy);
    347  1.27   pooka 	if (error)
    348  1.27   pooka 		return error;
    349  1.27   pooka 
    350  1.24   pooka 	/* verify host file is of supported type */
    351  1.24   pooka 	if (!(ftype == RUMPUSER_FT_REG
    352  1.24   pooka 	   || ftype == RUMPUSER_FT_BLK
    353  1.24   pooka 	   || ftype == RUMPUSER_FT_CHR))
    354  1.24   pooka 		return EINVAL;
    355   1.1   pooka 
    356  1.24   pooka 	mutex_enter(&rumpblk_lock);
    357  1.24   pooka 	for (i = 0; i < RUMPBLK_SIZE; i++) {
    358  1.24   pooka 		if (minors[i].rblk_path&&strcmp(minors[i].rblk_path, path)==0) {
    359  1.24   pooka 			mutex_exit(&rumpblk_lock);
    360  1.24   pooka 			*dmin = i;
    361  1.24   pooka 			return 0;
    362  1.24   pooka 		}
    363  1.24   pooka 	}
    364   1.1   pooka 
    365   1.1   pooka 	for (i = 0; i < RUMPBLK_SIZE; i++)
    366   1.1   pooka 		if (minors[i].rblk_path == NULL)
    367   1.1   pooka 			break;
    368  1.24   pooka 	if (i == RUMPBLK_SIZE) {
    369  1.24   pooka 		mutex_exit(&rumpblk_lock);
    370  1.24   pooka 		return EBUSY;
    371  1.24   pooka 	}
    372   1.1   pooka 
    373  1.27   pooka 	rblk = &minors[i];
    374   1.1   pooka 	len = strlen(path);
    375  1.27   pooka 	rblk->rblk_path = malloc(len + 1, M_TEMP, M_WAITOK);
    376  1.27   pooka 	strcpy(rblk->rblk_path, path);
    377  1.27   pooka 	rblk->rblk_fd = -1;
    378  1.27   pooka 	rblk->rblk_hostoffset = offset;
    379  1.27   pooka 	if (size == RUMPBLK_SIZENOTSET) {
    380  1.27   pooka 		KASSERT(size + offset <= flen);
    381  1.27   pooka 		rblk->rblk_size = size;
    382  1.27   pooka 	} else {
    383  1.27   pooka 		KASSERT(offset < flen);
    384  1.27   pooka 		rblk->rblk_size = flen - offset;
    385  1.27   pooka 	}
    386  1.27   pooka 	rblk->rblk_ftype = ftype;
    387  1.24   pooka 	mutex_exit(&rumpblk_lock);
    388  1.24   pooka 
    389  1.24   pooka 	*dmin = i;
    390  1.24   pooka 	return 0;
    391   1.1   pooka }
    392   1.1   pooka 
    393   1.1   pooka int
    394   1.1   pooka rumpblk_open(dev_t dev, int flag, int fmt, struct lwp *l)
    395   1.1   pooka {
    396   1.1   pooka 	struct rblkdev *rblk = &minors[minor(dev)];
    397   1.5   pooka 	uint64_t fsize;
    398  1.27   pooka 	int dummy;
    399   1.1   pooka 	int error, fd;
    400   1.1   pooka 
    401  1.27   pooka 	if (rblk->rblk_path == NULL)
    402  1.27   pooka 		return ENXIO;
    403  1.27   pooka 
    404  1.25   pooka 	if (rblk->rblk_fd != -1)
    405  1.25   pooka 		return 0; /* XXX: refcount, open mode */
    406   1.1   pooka 	fd = rumpuser_open(rblk->rblk_path, OFLAGS(flag), &error);
    407   1.1   pooka 	if (error)
    408   1.1   pooka 		return error;
    409   1.1   pooka 
    410  1.20   pooka #ifdef HAS_ODIRECT
    411  1.20   pooka 	rblk->rblk_dfd = rumpuser_open(rblk->rblk_path,
    412  1.20   pooka 	    OFLAGS(flag) | O_DIRECT, &error);
    413  1.20   pooka 	if (error)
    414  1.20   pooka 		return error;
    415  1.20   pooka #endif
    416  1.20   pooka 
    417  1.27   pooka 	if (rblk->rblk_ftype == RUMPUSER_FT_REG) {
    418  1.16   pooka 		struct blkwin *win;
    419  1.16   pooka 		int i, winsize;
    420  1.16   pooka 
    421   1.1   pooka 		/*
    422  1.16   pooka 		 * Use mmap to access a regular file.  Allocate and
    423  1.16   pooka 		 * cache initial windows here.  Failure to allocate one
    424  1.16   pooka 		 * means fallback to read/write i/o.
    425   1.1   pooka 		 */
    426   1.9   pooka 
    427  1.16   pooka 		rblk->rblk_mmflags = 0;
    428  1.16   pooka 		if (flag & FREAD)
    429  1.16   pooka 			rblk->rblk_mmflags |= RUMPUSER_FILEMMAP_READ;
    430  1.16   pooka 		if (flag & FWRITE) {
    431  1.16   pooka 			rblk->rblk_mmflags |= RUMPUSER_FILEMMAP_WRITE;
    432  1.16   pooka 			rblk->rblk_mmflags |= RUMPUSER_FILEMMAP_SHARED;
    433  1.16   pooka 		}
    434  1.16   pooka 
    435  1.16   pooka 		TAILQ_INIT(&rblk->rblk_lruq);
    436  1.16   pooka 		rblk->rblk_fd = fd;
    437  1.16   pooka 
    438  1.19   pooka 		for (i = 0; i < memwincnt && i * memwinsize < fsize; i++) {
    439  1.16   pooka 			win = kmem_zalloc(sizeof(*win), KM_SLEEP);
    440  1.16   pooka 			WINVALIDATE(win);
    441  1.16   pooka 			TAILQ_INSERT_TAIL(&rblk->rblk_lruq, win, win_lru);
    442  1.16   pooka 
    443  1.16   pooka 			/*
    444  1.16   pooka 			 * Allocate first windows.  Here we just generally
    445  1.16   pooka 			 * make sure a) we can mmap at all b) we have the
    446  1.16   pooka 			 * necessary VA available
    447  1.16   pooka 			 */
    448  1.16   pooka 			winsize = 1;
    449  1.19   pooka 			win = getwindow(rblk, i*memwinsize, &winsize, &error);
    450  1.16   pooka 			if (win) {
    451  1.16   pooka 				putwindow(rblk, win);
    452  1.16   pooka 			} else {
    453  1.16   pooka 				wincleanup(rblk);
    454  1.16   pooka 				break;
    455  1.12   pooka 			}
    456   1.9   pooka 		}
    457   1.9   pooka 
    458   1.1   pooka 		memset(&rblk->rblk_dl, 0, sizeof(rblk->rblk_dl));
    459   1.9   pooka 		rblk->rblk_pi.p_size = fsize >> DEV_BSHIFT;
    460   1.9   pooka 		rblk->rblk_dl.d_secsize = DEV_BSIZE;
    461   1.9   pooka 		rblk->rblk_curpi = &rblk->rblk_pi;
    462   1.9   pooka 	} else {
    463  1.13   pooka 		if (rumpuser_ioctl(fd, DIOCGDINFO, &rblk->rblk_dl,
    464  1.17   pooka 		    &error) == -1) {
    465  1.17   pooka 			KASSERT(error);
    466   1.1   pooka 			rumpuser_close(fd, &dummy);
    467   1.1   pooka 			return error;
    468   1.1   pooka 		}
    469   1.9   pooka 
    470  1.16   pooka 		rblk->rblk_fd = fd;
    471   1.9   pooka 		rblk->rblk_curpi = &rblk->rblk_dl.d_partitions[0];
    472   1.1   pooka 	}
    473   1.1   pooka 
    474  1.16   pooka 	KASSERT(rblk->rblk_fd != -1);
    475   1.1   pooka 	return 0;
    476   1.1   pooka }
    477   1.1   pooka 
    478   1.1   pooka int
    479   1.1   pooka rumpblk_close(dev_t dev, int flag, int fmt, struct lwp *l)
    480   1.1   pooka {
    481   1.1   pooka 	struct rblkdev *rblk = &minors[minor(dev)];
    482   1.1   pooka 	int dummy;
    483   1.1   pooka 
    484  1.16   pooka 	if (rblk->rblk_mmflags)
    485  1.16   pooka 		wincleanup(rblk);
    486  1.16   pooka 	rumpuser_fsync(rblk->rblk_fd, &dummy);
    487   1.1   pooka 	rumpuser_close(rblk->rblk_fd, &dummy);
    488   1.1   pooka 	rblk->rblk_fd = -1;
    489   1.1   pooka 
    490   1.1   pooka 	return 0;
    491   1.1   pooka }
    492   1.1   pooka 
    493   1.1   pooka int
    494   1.1   pooka rumpblk_ioctl(dev_t dev, u_long xfer, void *addr, int flag, struct lwp *l)
    495   1.1   pooka {
    496   1.1   pooka 	struct rblkdev *rblk = &minors[minor(dev)];
    497   1.1   pooka 	int rv, error;
    498   1.1   pooka 
    499   1.1   pooka 	if (xfer == DIOCGPART) {
    500   1.1   pooka 		struct partinfo *pi = (struct partinfo *)addr;
    501   1.1   pooka 
    502   1.1   pooka 		pi->part = rblk->rblk_curpi;
    503   1.1   pooka 		pi->disklab = &rblk->rblk_dl;
    504   1.1   pooka 
    505   1.1   pooka 		return 0;
    506   1.1   pooka 	}
    507   1.1   pooka 
    508   1.1   pooka 	rv = rumpuser_ioctl(rblk->rblk_fd, xfer, addr, &error);
    509   1.1   pooka 	if (rv == -1)
    510   1.1   pooka 		return error;
    511   1.1   pooka 
    512   1.1   pooka 	return 0;
    513   1.1   pooka }
    514   1.1   pooka 
    515  1.25   pooka static int
    516  1.25   pooka do_physio(dev_t dev, struct uio *uio, int which)
    517  1.25   pooka {
    518  1.25   pooka 	void (*strat)(struct buf *);
    519  1.25   pooka 
    520  1.25   pooka 	if (blkfail)
    521  1.25   pooka 		strat = rumpblk_strategy_fail;
    522  1.25   pooka 	else
    523  1.25   pooka 		strat = rumpblk_strategy;
    524  1.25   pooka 
    525  1.25   pooka 	return physio(strat, NULL, dev, which, minphys, uio);
    526  1.25   pooka }
    527  1.25   pooka 
    528   1.1   pooka int
    529   1.1   pooka rumpblk_read(dev_t dev, struct uio *uio, int flags)
    530   1.1   pooka {
    531   1.1   pooka 
    532  1.25   pooka 	return do_physio(dev, uio, B_READ);
    533   1.1   pooka }
    534   1.1   pooka 
    535   1.1   pooka int
    536   1.1   pooka rumpblk_write(dev_t dev, struct uio *uio, int flags)
    537   1.1   pooka {
    538   1.1   pooka 
    539  1.25   pooka 	return do_physio(dev, uio, B_WRITE);
    540   1.1   pooka }
    541   1.1   pooka 
    542   1.3   pooka static void
    543   1.3   pooka dostrategy(struct buf *bp)
    544   1.1   pooka {
    545   1.1   pooka 	struct rblkdev *rblk = &minors[minor(bp->b_dev)];
    546   1.1   pooka 	off_t off;
    547  1.21   pooka 	int async = bp->b_flags & B_ASYNC;
    548  1.21   pooka 	int error;
    549   1.1   pooka 
    550  1.20   pooka 	/* collect statistics */
    551  1.20   pooka 	ev_io_total.ev_count++;
    552  1.20   pooka 	if (async)
    553  1.20   pooka 		ev_io_async.ev_count++;
    554  1.20   pooka 	if (BUF_ISWRITE(bp)) {
    555  1.20   pooka 		ev_bwrite_total.ev_count += bp->b_bcount;
    556  1.20   pooka 		if (async)
    557  1.20   pooka 			ev_bwrite_async.ev_count += bp->b_bcount;
    558  1.20   pooka 	} else {
    559  1.20   pooka 		ev_bread_total.ev_count++;
    560  1.20   pooka 	}
    561  1.20   pooka 
    562   1.1   pooka 	off = bp->b_blkno << DEV_BSHIFT;
    563  1.27   pooka 	off += rblk->rblk_hostoffset;
    564  1.11   pooka 	/*
    565  1.11   pooka 	 * Do bounds checking if we're working on a file.  Otherwise
    566  1.11   pooka 	 * invalid file systems might attempt to read beyond EOF.  This
    567  1.11   pooka 	 * is bad(tm) especially on mmapped images.  This is essentially
    568  1.11   pooka 	 * the kernel bounds_check() routines.
    569  1.11   pooka 	 */
    570  1.27   pooka 	if (off + bp->b_bcount > rblk->rblk_size) {
    571  1.11   pooka 		int64_t sz = rblk->rblk_size - off;
    572  1.11   pooka 
    573  1.11   pooka 		/* EOF */
    574  1.11   pooka 		if (sz == 0) {
    575  1.11   pooka 			rump_biodone(bp, 0, 0);
    576  1.11   pooka 			return;
    577  1.11   pooka 		}
    578  1.11   pooka 		/* beyond EOF ==> error */
    579  1.11   pooka 		if (sz < 0) {
    580  1.11   pooka 			rump_biodone(bp, 0, EINVAL);
    581  1.11   pooka 			return;
    582  1.11   pooka 		}
    583  1.11   pooka 
    584  1.11   pooka 		/* truncate to device size */
    585  1.11   pooka 		bp->b_bcount = sz;
    586  1.11   pooka 	}
    587  1.11   pooka 
    588   1.1   pooka 	DPRINTF(("rumpblk_strategy: 0x%x bytes %s off 0x%" PRIx64
    589  1.20   pooka 	    " (0x%" PRIx64 " - 0x%" PRIx64 "), %ssync\n",
    590  1.16   pooka 	    bp->b_bcount, BUF_ISREAD(bp) ? "READ" : "WRITE",
    591  1.20   pooka 	    off, off, (off + bp->b_bcount), async ? "a" : ""));
    592   1.1   pooka 
    593  1.16   pooka 	/* mmap?  handle here and return */
    594  1.16   pooka 	if (rblk->rblk_mmflags) {
    595  1.16   pooka 		struct blkwin *win;
    596  1.16   pooka 		int winsize, iodone;
    597  1.16   pooka 		uint8_t *ioaddr, *bufaddr;
    598  1.16   pooka 
    599  1.16   pooka 		for (iodone = 0; iodone < bp->b_bcount;
    600  1.16   pooka 		    iodone += winsize, off += winsize) {
    601  1.16   pooka 			winsize = bp->b_bcount - iodone;
    602  1.16   pooka 			win = getwindow(rblk, off, &winsize, &error);
    603  1.16   pooka 			if (win == NULL) {
    604  1.16   pooka 				rump_biodone(bp, iodone, error);
    605  1.16   pooka 				return;
    606  1.16   pooka 			}
    607  1.16   pooka 
    608  1.16   pooka 			ioaddr = (uint8_t *)win->win_mem + (off-STARTWIN(off));
    609  1.16   pooka 			bufaddr = (uint8_t *)bp->b_data + iodone;
    610  1.11   pooka 
    611  1.16   pooka 			DPRINTF(("strat: %p off 0x%" PRIx64
    612  1.16   pooka 			    ", ioaddr %p (%p)/buf %p\n", win,
    613  1.16   pooka 			    win->win_off, ioaddr, win->win_mem, bufaddr));
    614  1.16   pooka 			if (BUF_ISREAD(bp)) {
    615  1.16   pooka 				memcpy(bufaddr, ioaddr, winsize);
    616  1.16   pooka 			} else {
    617  1.16   pooka 				memcpy(ioaddr, bufaddr, winsize);
    618  1.16   pooka 			}
    619  1.16   pooka 
    620  1.16   pooka 			/* synchronous write, sync bits back to disk */
    621  1.16   pooka 			if (BUF_ISWRITE(bp) && !async) {
    622  1.16   pooka 				rumpuser_memsync(ioaddr, winsize, &error);
    623  1.16   pooka 			}
    624  1.16   pooka 			putwindow(rblk, win);
    625   1.9   pooka 		}
    626   1.9   pooka 
    627   1.9   pooka 		rump_biodone(bp, bp->b_bcount, 0);
    628   1.9   pooka 		return;
    629   1.9   pooka 	}
    630   1.9   pooka 
    631   1.1   pooka 	/*
    632   1.1   pooka 	 * Do I/O.  We have different paths for async and sync I/O.
    633   1.1   pooka 	 * Async I/O is done by passing a request to rumpuser where
    634   1.1   pooka 	 * it is executed.  The rumpuser routine then calls
    635   1.1   pooka 	 * biodone() to signal any waiters in the kernel.  I/O's are
    636   1.1   pooka 	 * executed in series.  Technically executing them in parallel
    637   1.1   pooka 	 * would produce better results, but then we'd need either
    638   1.1   pooka 	 * more threads or posix aio.  Maybe worth investigating
    639   1.1   pooka 	 * this later.
    640  1.14   pooka 	 *
    641  1.14   pooka 	 * Using bufq here might be a good idea.
    642   1.1   pooka 	 */
    643  1.20   pooka 
    644  1.13   pooka 	if (rump_threads) {
    645   1.1   pooka 		struct rumpuser_aio *rua;
    646  1.20   pooka 		int op, fd;
    647  1.20   pooka 
    648  1.20   pooka 		fd = rblk->rblk_fd;
    649  1.20   pooka 		if (BUF_ISREAD(bp)) {
    650  1.20   pooka 			op = RUA_OP_READ;
    651  1.20   pooka 		} else {
    652  1.20   pooka 			op = RUA_OP_WRITE;
    653  1.20   pooka 			if (!async) {
    654  1.20   pooka 				/* O_DIRECT not fully automatic yet */
    655  1.20   pooka #ifdef HAS_ODIRECT
    656  1.20   pooka 				if ((off & (DEV_BSIZE-1)) == 0
    657  1.20   pooka 				    && ((intptr_t)bp->b_data&(DEV_BSIZE-1)) == 0
    658  1.20   pooka 				    && (bp->b_bcount & (DEV_BSIZE-1)) == 0)
    659  1.20   pooka 					fd = rblk->rblk_dfd;
    660  1.20   pooka 				else
    661  1.20   pooka #endif
    662  1.20   pooka 					op |= RUA_OP_SYNC;
    663  1.20   pooka 			}
    664  1.20   pooka 		}
    665   1.1   pooka 
    666   1.1   pooka 		rumpuser_mutex_enter(&rumpuser_aio_mtx);
    667  1.20   pooka 		while ((rumpuser_aio_head+1) % N_AIOS == rumpuser_aio_tail) {
    668  1.13   pooka 			rumpuser_cv_wait(&rumpuser_aio_cv, &rumpuser_aio_mtx);
    669  1.20   pooka 		}
    670   1.1   pooka 
    671   1.2   pooka 		rua = &rumpuser_aios[rumpuser_aio_head];
    672   1.2   pooka 		KASSERT(rua->rua_bp == NULL);
    673  1.20   pooka 		rua->rua_fd = fd;
    674   1.2   pooka 		rua->rua_data = bp->b_data;
    675   1.2   pooka 		rua->rua_dlen = bp->b_bcount;
    676   1.2   pooka 		rua->rua_off = off;
    677   1.2   pooka 		rua->rua_bp = bp;
    678  1.20   pooka 		rua->rua_op = op;
    679   1.2   pooka 
    680   1.1   pooka 		/* insert into queue & signal */
    681   1.6   pooka 		rumpuser_aio_head = (rumpuser_aio_head+1) % N_AIOS;
    682   1.1   pooka 		rumpuser_cv_signal(&rumpuser_aio_cv);
    683   1.1   pooka 		rumpuser_mutex_exit(&rumpuser_aio_mtx);
    684   1.1   pooka 	} else {
    685   1.1   pooka 		if (BUF_ISREAD(bp)) {
    686   1.1   pooka 			rumpuser_read_bio(rblk->rblk_fd, bp->b_data,
    687   1.1   pooka 			    bp->b_bcount, off, rump_biodone, bp);
    688   1.1   pooka 		} else {
    689   1.1   pooka 			rumpuser_write_bio(rblk->rblk_fd, bp->b_data,
    690   1.1   pooka 			    bp->b_bcount, off, rump_biodone, bp);
    691   1.1   pooka 		}
    692  1.20   pooka 		if (BUF_ISWRITE(bp) && !async)
    693  1.20   pooka 			rumpuser_fsync(rblk->rblk_fd, &error);
    694   1.1   pooka 	}
    695   1.1   pooka }
    696   1.3   pooka 
    697   1.3   pooka void
    698   1.3   pooka rumpblk_strategy(struct buf *bp)
    699   1.3   pooka {
    700   1.3   pooka 
    701   1.3   pooka 	dostrategy(bp);
    702   1.3   pooka }
    703   1.3   pooka 
    704   1.3   pooka /*
    705   1.4   pooka  * Simple random number generator.  This is private so that we can
    706   1.4   pooka  * very repeatedly control which blocks will fail.
    707   1.4   pooka  *
    708   1.3   pooka  * <mlelstv> pooka, rand()
    709   1.3   pooka  * <mlelstv> [paste]
    710   1.3   pooka  */
    711   1.3   pooka static unsigned
    712   1.3   pooka gimmerand(void)
    713   1.3   pooka {
    714   1.3   pooka 
    715   1.3   pooka 	return (randstate = randstate * 1103515245 + 12345) % (0x80000000L);
    716   1.3   pooka }
    717   1.3   pooka 
    718   1.3   pooka /*
    719   1.3   pooka  * Block device with very simple fault injection.  Fails every
    720   1.3   pooka  * n out of BLKFAIL_MAX I/O with EIO.  n is determined by the env
    721   1.3   pooka  * variable RUMP_BLKFAIL.
    722   1.3   pooka  */
    723   1.3   pooka void
    724   1.3   pooka rumpblk_strategy_fail(struct buf *bp)
    725   1.3   pooka {
    726   1.3   pooka 
    727   1.3   pooka 	if (gimmerand() % BLKFAIL_MAX >= blkfail) {
    728   1.3   pooka 		dostrategy(bp);
    729   1.3   pooka 	} else {
    730   1.3   pooka 		printf("block fault injection: failing I/O on block %lld\n",
    731   1.3   pooka 		    (long long)bp->b_blkno);
    732   1.3   pooka 		bp->b_error = EIO;
    733   1.3   pooka 		biodone(bp);
    734   1.3   pooka 	}
    735   1.3   pooka }
    736