rumpblk.c revision 1.27 1 1.27 pooka /* $NetBSD: rumpblk.c,v 1.27 2009/10/07 09:17:54 pooka Exp $ */
2 1.1 pooka
3 1.1 pooka /*
4 1.1 pooka * Copyright (c) 2009 Antti Kantee. All Rights Reserved.
5 1.1 pooka *
6 1.1 pooka * Development of this software was supported by the
7 1.1 pooka * Finnish Cultural Foundation.
8 1.1 pooka *
9 1.1 pooka * Redistribution and use in source and binary forms, with or without
10 1.1 pooka * modification, are permitted provided that the following conditions
11 1.1 pooka * are met:
12 1.1 pooka * 1. Redistributions of source code must retain the above copyright
13 1.1 pooka * notice, this list of conditions and the following disclaimer.
14 1.1 pooka * 2. Redistributions in binary form must reproduce the above copyright
15 1.1 pooka * notice, this list of conditions and the following disclaimer in the
16 1.1 pooka * documentation and/or other materials provided with the distribution.
17 1.1 pooka *
18 1.1 pooka * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
19 1.1 pooka * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 1.1 pooka * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21 1.1 pooka * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 1.1 pooka * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 1.1 pooka * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24 1.1 pooka * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 1.1 pooka * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 1.1 pooka * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 1.1 pooka * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 1.1 pooka * SUCH DAMAGE.
29 1.1 pooka */
30 1.1 pooka
31 1.1 pooka /*
32 1.1 pooka * Block device emulation. Presents a block device interface and
33 1.1 pooka * uses rumpuser system calls to satisfy I/O requests.
34 1.16 pooka *
35 1.16 pooka * We provide fault injection. The driver can be made to fail
36 1.16 pooka * I/O occasionally.
37 1.16 pooka *
38 1.16 pooka * The driver also provides an optimization for regular files by
39 1.16 pooka * using memory-mapped I/O. This avoids kernel access for every
40 1.16 pooka * I/O operation. It also gives finer-grained control of how to
41 1.16 pooka * flush data. Additionally, in case the rump kernel dumps core,
42 1.16 pooka * we get way less carnage.
43 1.1 pooka */
44 1.1 pooka
45 1.1 pooka #include <sys/cdefs.h>
46 1.27 pooka __KERNEL_RCSID(0, "$NetBSD: rumpblk.c,v 1.27 2009/10/07 09:17:54 pooka Exp $");
47 1.1 pooka
48 1.1 pooka #include <sys/param.h>
49 1.1 pooka #include <sys/buf.h>
50 1.1 pooka #include <sys/conf.h>
51 1.16 pooka #include <sys/condvar.h>
52 1.1 pooka #include <sys/disklabel.h>
53 1.18 pooka #include <sys/evcnt.h>
54 1.1 pooka #include <sys/fcntl.h>
55 1.1 pooka #include <sys/kmem.h>
56 1.1 pooka #include <sys/malloc.h>
57 1.16 pooka #include <sys/queue.h>
58 1.1 pooka #include <sys/stat.h>
59 1.1 pooka
60 1.1 pooka #include <rump/rumpuser.h>
61 1.1 pooka
62 1.1 pooka #include "rump_private.h"
63 1.1 pooka #include "rump_vfs_private.h"
64 1.1 pooka
65 1.16 pooka #if 0
66 1.16 pooka #define DPRINTF(x) printf x
67 1.16 pooka #else
68 1.16 pooka #define DPRINTF(x)
69 1.16 pooka #endif
70 1.16 pooka
71 1.19 pooka /* Default: 16 x 1MB windows */
72 1.19 pooka unsigned memwinsize = (1<<20);
73 1.19 pooka unsigned memwincnt = 16;
74 1.19 pooka
75 1.19 pooka #define STARTWIN(off) ((off) & ~(memwinsize-1))
76 1.16 pooka #define INWIN(win,off) ((win)->win_off == STARTWIN(off))
77 1.19 pooka #define WINSIZE(rblk, win) (MIN((rblk->rblk_size-win->win_off),memwinsize))
78 1.16 pooka #define WINVALID(win) ((win)->win_off != (off_t)-1)
79 1.16 pooka #define WINVALIDATE(win) ((win)->win_off = (off_t)-1)
80 1.16 pooka struct blkwin {
81 1.16 pooka off_t win_off;
82 1.16 pooka void *win_mem;
83 1.16 pooka int win_refcnt;
84 1.16 pooka
85 1.16 pooka TAILQ_ENTRY(blkwin) win_lru;
86 1.16 pooka };
87 1.16 pooka
88 1.1 pooka #define RUMPBLK_SIZE 16
89 1.1 pooka static struct rblkdev {
90 1.1 pooka char *rblk_path;
91 1.1 pooka int rblk_fd;
92 1.25 pooka int rblk_opencnt;
93 1.20 pooka #ifdef HAS_ODIRECT
94 1.20 pooka int rblk_dfd;
95 1.20 pooka #endif
96 1.26 pooka uint64_t rblk_size;
97 1.27 pooka uint64_t rblk_hostoffset;
98 1.27 pooka int rblk_ftype;
99 1.16 pooka
100 1.16 pooka /* for mmap */
101 1.16 pooka int rblk_mmflags;
102 1.16 pooka kmutex_t rblk_memmtx;
103 1.16 pooka kcondvar_t rblk_memcv;
104 1.16 pooka TAILQ_HEAD(winlru, blkwin) rblk_lruq;
105 1.16 pooka bool rblk_waiting;
106 1.1 pooka
107 1.1 pooka struct partition *rblk_curpi;
108 1.1 pooka struct partition rblk_pi;
109 1.1 pooka struct disklabel rblk_dl;
110 1.1 pooka } minors[RUMPBLK_SIZE];
111 1.1 pooka
112 1.20 pooka static struct evcnt ev_io_total;
113 1.20 pooka static struct evcnt ev_io_async;
114 1.20 pooka
115 1.20 pooka static struct evcnt ev_memblk_hits;
116 1.20 pooka static struct evcnt ev_memblk_busy;
117 1.20 pooka
118 1.20 pooka static struct evcnt ev_bwrite_total;
119 1.20 pooka static struct evcnt ev_bwrite_async;
120 1.20 pooka static struct evcnt ev_bread_total;
121 1.18 pooka
122 1.1 pooka dev_type_open(rumpblk_open);
123 1.1 pooka dev_type_close(rumpblk_close);
124 1.1 pooka dev_type_read(rumpblk_read);
125 1.1 pooka dev_type_write(rumpblk_write);
126 1.1 pooka dev_type_ioctl(rumpblk_ioctl);
127 1.1 pooka dev_type_strategy(rumpblk_strategy);
128 1.3 pooka dev_type_strategy(rumpblk_strategy_fail);
129 1.1 pooka dev_type_dump(rumpblk_dump);
130 1.1 pooka dev_type_size(rumpblk_size);
131 1.1 pooka
132 1.1 pooka static const struct bdevsw rumpblk_bdevsw = {
133 1.1 pooka rumpblk_open, rumpblk_close, rumpblk_strategy, rumpblk_ioctl,
134 1.1 pooka nodump, nosize, D_DISK
135 1.1 pooka };
136 1.1 pooka
137 1.3 pooka static const struct bdevsw rumpblk_bdevsw_fail = {
138 1.3 pooka rumpblk_open, rumpblk_close, rumpblk_strategy_fail, rumpblk_ioctl,
139 1.3 pooka nodump, nosize, D_DISK
140 1.3 pooka };
141 1.3 pooka
142 1.1 pooka static const struct cdevsw rumpblk_cdevsw = {
143 1.1 pooka rumpblk_open, rumpblk_close, rumpblk_read, rumpblk_write,
144 1.1 pooka rumpblk_ioctl, nostop, notty, nopoll, nommap, nokqfilter, D_DISK
145 1.1 pooka };
146 1.1 pooka
147 1.3 pooka /* fail every n out of BLKFAIL_MAX */
148 1.3 pooka #define BLKFAIL_MAX 10000
149 1.3 pooka static int blkfail;
150 1.3 pooka static unsigned randstate;
151 1.24 pooka static kmutex_t rumpblk_lock;
152 1.1 pooka
153 1.16 pooka static struct blkwin *
154 1.16 pooka getwindow(struct rblkdev *rblk, off_t off, int *wsize, int *error)
155 1.16 pooka {
156 1.16 pooka struct blkwin *win;
157 1.16 pooka
158 1.16 pooka mutex_enter(&rblk->rblk_memmtx);
159 1.16 pooka retry:
160 1.16 pooka /* search for window */
161 1.16 pooka TAILQ_FOREACH(win, &rblk->rblk_lruq, win_lru) {
162 1.16 pooka if (INWIN(win, off) && WINVALID(win))
163 1.16 pooka break;
164 1.16 pooka }
165 1.16 pooka
166 1.16 pooka /* found? return */
167 1.16 pooka if (win) {
168 1.20 pooka ev_memblk_hits.ev_count++;
169 1.16 pooka TAILQ_REMOVE(&rblk->rblk_lruq, win, win_lru);
170 1.16 pooka goto good;
171 1.16 pooka }
172 1.16 pooka
173 1.16 pooka /*
174 1.16 pooka * Else, create new window. If the least recently used is not
175 1.16 pooka * currently in use, reuse that. Otherwise we need to wait.
176 1.16 pooka */
177 1.16 pooka win = TAILQ_LAST(&rblk->rblk_lruq, winlru);
178 1.16 pooka if (win->win_refcnt == 0) {
179 1.16 pooka TAILQ_REMOVE(&rblk->rblk_lruq, win, win_lru);
180 1.16 pooka mutex_exit(&rblk->rblk_memmtx);
181 1.16 pooka
182 1.16 pooka if (WINVALID(win)) {
183 1.16 pooka DPRINTF(("win %p, unmap mem %p, off 0x%" PRIx64 "\n",
184 1.16 pooka win, win->win_mem, win->win_off));
185 1.16 pooka rumpuser_unmap(win->win_mem, WINSIZE(rblk, win));
186 1.16 pooka WINVALIDATE(win);
187 1.16 pooka }
188 1.16 pooka
189 1.16 pooka win->win_off = STARTWIN(off);
190 1.16 pooka win->win_mem = rumpuser_filemmap(rblk->rblk_fd, win->win_off,
191 1.16 pooka WINSIZE(rblk, win), rblk->rblk_mmflags, error);
192 1.16 pooka DPRINTF(("win %p, off 0x%" PRIx64 ", mem %p\n",
193 1.16 pooka win, win->win_off, win->win_mem));
194 1.16 pooka
195 1.16 pooka mutex_enter(&rblk->rblk_memmtx);
196 1.16 pooka if (win->win_mem == NULL) {
197 1.16 pooka WINVALIDATE(win);
198 1.16 pooka TAILQ_INSERT_TAIL(&rblk->rblk_lruq, win, win_lru);
199 1.16 pooka mutex_exit(&rblk->rblk_memmtx);
200 1.16 pooka return NULL;
201 1.16 pooka }
202 1.16 pooka } else {
203 1.16 pooka DPRINTF(("memwin wait\n"));
204 1.20 pooka ev_memblk_busy.ev_count++;
205 1.18 pooka
206 1.16 pooka rblk->rblk_waiting = true;
207 1.16 pooka cv_wait(&rblk->rblk_memcv, &rblk->rblk_memmtx);
208 1.16 pooka goto retry;
209 1.16 pooka }
210 1.16 pooka
211 1.16 pooka good:
212 1.16 pooka KASSERT(win);
213 1.16 pooka win->win_refcnt++;
214 1.16 pooka TAILQ_INSERT_HEAD(&rblk->rblk_lruq, win, win_lru);
215 1.16 pooka mutex_exit(&rblk->rblk_memmtx);
216 1.19 pooka *wsize = MIN(*wsize, memwinsize - (off-win->win_off));
217 1.16 pooka KASSERT(*wsize);
218 1.16 pooka
219 1.16 pooka return win;
220 1.16 pooka }
221 1.16 pooka
222 1.16 pooka static void
223 1.16 pooka putwindow(struct rblkdev *rblk, struct blkwin *win)
224 1.16 pooka {
225 1.16 pooka
226 1.16 pooka mutex_enter(&rblk->rblk_memmtx);
227 1.16 pooka if (--win->win_refcnt == 0 && rblk->rblk_waiting) {
228 1.16 pooka rblk->rblk_waiting = false;
229 1.16 pooka cv_signal(&rblk->rblk_memcv);
230 1.16 pooka }
231 1.16 pooka KASSERT(win->win_refcnt >= 0);
232 1.16 pooka mutex_exit(&rblk->rblk_memmtx);
233 1.16 pooka }
234 1.16 pooka
235 1.16 pooka static void
236 1.16 pooka wincleanup(struct rblkdev *rblk)
237 1.16 pooka {
238 1.16 pooka struct blkwin *win;
239 1.16 pooka
240 1.16 pooka while ((win = TAILQ_FIRST(&rblk->rblk_lruq)) != NULL) {
241 1.16 pooka TAILQ_REMOVE(&rblk->rblk_lruq, win, win_lru);
242 1.16 pooka if (WINVALID(win)) {
243 1.16 pooka DPRINTF(("cleanup win %p addr %p\n",
244 1.16 pooka win, win->win_mem));
245 1.16 pooka rumpuser_unmap(win->win_mem, WINSIZE(rblk, win));
246 1.16 pooka }
247 1.16 pooka kmem_free(win, sizeof(*win));
248 1.16 pooka }
249 1.16 pooka rblk->rblk_mmflags = 0;
250 1.16 pooka }
251 1.16 pooka
252 1.1 pooka int
253 1.8 cegger rumpblk_init(void)
254 1.1 pooka {
255 1.3 pooka char buf[64];
256 1.1 pooka int rumpblk = RUMPBLK;
257 1.19 pooka unsigned tmp;
258 1.16 pooka int error, i;
259 1.3 pooka
260 1.24 pooka mutex_init(&rumpblk_lock, MUTEX_DEFAULT, IPL_NONE);
261 1.24 pooka
262 1.3 pooka if (rumpuser_getenv("RUMP_BLKFAIL", buf, sizeof(buf), &error) == 0) {
263 1.3 pooka blkfail = strtoul(buf, NULL, 10);
264 1.3 pooka /* fail everything */
265 1.3 pooka if (blkfail > BLKFAIL_MAX)
266 1.3 pooka blkfail = BLKFAIL_MAX;
267 1.3 pooka if (rumpuser_getenv("RUMP_BLKFAIL_SEED", buf, sizeof(buf),
268 1.3 pooka &error) == 0) {
269 1.3 pooka randstate = strtoul(buf, NULL, 10);
270 1.3 pooka } else {
271 1.23 pooka randstate = arc4random();
272 1.3 pooka }
273 1.22 pooka printf("rumpblk: FAULT INJECTION ACTIVE! fail %d/%d. "
274 1.22 pooka "seed %u\n", blkfail, BLKFAIL_MAX, randstate);
275 1.3 pooka } else {
276 1.3 pooka blkfail = 0;
277 1.3 pooka }
278 1.1 pooka
279 1.19 pooka if (rumpuser_getenv("RUMP_BLKWINSIZE", buf, sizeof(buf), &error) == 0) {
280 1.19 pooka printf("rumpblk: ");
281 1.19 pooka tmp = strtoul(buf, NULL, 10);
282 1.19 pooka if (tmp && !(tmp & (tmp-1)))
283 1.19 pooka memwinsize = tmp;
284 1.19 pooka else
285 1.19 pooka printf("invalid RUMP_BLKWINSIZE %d, ", tmp);
286 1.19 pooka printf("using %d for memwinsize\n", memwinsize);
287 1.19 pooka }
288 1.19 pooka if (rumpuser_getenv("RUMP_BLKWINCOUNT", buf, sizeof(buf), &error) == 0){
289 1.19 pooka printf("rumpblk: ");
290 1.19 pooka tmp = strtoul(buf, NULL, 10);
291 1.19 pooka if (tmp)
292 1.19 pooka memwincnt = tmp;
293 1.19 pooka else
294 1.19 pooka printf("invalid RUMP_BLKWINCOUNT %d, ", tmp);
295 1.19 pooka printf("using %d for memwincount\n", memwincnt);
296 1.19 pooka }
297 1.19 pooka
298 1.16 pooka memset(minors, 0, sizeof(minors));
299 1.16 pooka for (i = 0; i < RUMPBLK_SIZE; i++) {
300 1.16 pooka mutex_init(&minors[i].rblk_memmtx, MUTEX_DEFAULT, IPL_NONE);
301 1.16 pooka cv_init(&minors[i].rblk_memcv, "rblkmcv");
302 1.16 pooka }
303 1.16 pooka
304 1.20 pooka evcnt_attach_dynamic(&ev_io_total, EVCNT_TYPE_MISC, NULL,
305 1.20 pooka "rumpblk", "rumpblk I/O reqs");
306 1.20 pooka evcnt_attach_dynamic(&ev_io_async, EVCNT_TYPE_MISC, NULL,
307 1.20 pooka "rumpblk", "rumpblk async I/O");
308 1.20 pooka
309 1.20 pooka evcnt_attach_dynamic(&ev_bread_total, EVCNT_TYPE_MISC, NULL,
310 1.20 pooka "rumpblk", "rumpblk bytes read");
311 1.20 pooka evcnt_attach_dynamic(&ev_bwrite_total, EVCNT_TYPE_MISC, NULL,
312 1.20 pooka "rumpblk", "rumpblk bytes written");
313 1.20 pooka evcnt_attach_dynamic(&ev_bwrite_async, EVCNT_TYPE_MISC, NULL,
314 1.20 pooka "rumpblk", "rumpblk bytes written async");
315 1.20 pooka
316 1.20 pooka evcnt_attach_dynamic(&ev_memblk_hits, EVCNT_TYPE_MISC, NULL,
317 1.18 pooka "rumpblk", "memblk window hits");
318 1.20 pooka evcnt_attach_dynamic(&ev_memblk_busy, EVCNT_TYPE_MISC, NULL,
319 1.18 pooka "rumpblk", "memblk all windows busy");
320 1.18 pooka
321 1.3 pooka if (blkfail) {
322 1.3 pooka return devsw_attach("rumpblk", &rumpblk_bdevsw_fail, &rumpblk,
323 1.3 pooka &rumpblk_cdevsw, &rumpblk);
324 1.3 pooka } else {
325 1.3 pooka return devsw_attach("rumpblk", &rumpblk_bdevsw, &rumpblk,
326 1.3 pooka &rumpblk_cdevsw, &rumpblk);
327 1.3 pooka }
328 1.1 pooka }
329 1.1 pooka
330 1.24 pooka /* XXX: no deregister */
331 1.1 pooka int
332 1.27 pooka rumpblk_register(const char *path, devminor_t *dmin,
333 1.27 pooka uint64_t offset, uint64_t size)
334 1.1 pooka {
335 1.27 pooka struct rblkdev *rblk;
336 1.24 pooka uint64_t flen;
337 1.1 pooka size_t len;
338 1.27 pooka int ftype, error, dummy, i;
339 1.27 pooka int fd;
340 1.24 pooka
341 1.27 pooka /* devices might not report correct size unless they're open */
342 1.27 pooka fd = rumpuser_open(path, O_RDONLY, &error);
343 1.27 pooka if (fd == -1)
344 1.24 pooka return error;
345 1.27 pooka rumpuser_getfileinfo(path, &flen, &ftype, &error);
346 1.27 pooka rumpuser_close(fd, &dummy);
347 1.27 pooka if (error)
348 1.27 pooka return error;
349 1.27 pooka
350 1.24 pooka /* verify host file is of supported type */
351 1.24 pooka if (!(ftype == RUMPUSER_FT_REG
352 1.24 pooka || ftype == RUMPUSER_FT_BLK
353 1.24 pooka || ftype == RUMPUSER_FT_CHR))
354 1.24 pooka return EINVAL;
355 1.1 pooka
356 1.24 pooka mutex_enter(&rumpblk_lock);
357 1.24 pooka for (i = 0; i < RUMPBLK_SIZE; i++) {
358 1.24 pooka if (minors[i].rblk_path&&strcmp(minors[i].rblk_path, path)==0) {
359 1.24 pooka mutex_exit(&rumpblk_lock);
360 1.24 pooka *dmin = i;
361 1.24 pooka return 0;
362 1.24 pooka }
363 1.24 pooka }
364 1.1 pooka
365 1.1 pooka for (i = 0; i < RUMPBLK_SIZE; i++)
366 1.1 pooka if (minors[i].rblk_path == NULL)
367 1.1 pooka break;
368 1.24 pooka if (i == RUMPBLK_SIZE) {
369 1.24 pooka mutex_exit(&rumpblk_lock);
370 1.24 pooka return EBUSY;
371 1.24 pooka }
372 1.1 pooka
373 1.27 pooka rblk = &minors[i];
374 1.1 pooka len = strlen(path);
375 1.27 pooka rblk->rblk_path = malloc(len + 1, M_TEMP, M_WAITOK);
376 1.27 pooka strcpy(rblk->rblk_path, path);
377 1.27 pooka rblk->rblk_fd = -1;
378 1.27 pooka rblk->rblk_hostoffset = offset;
379 1.27 pooka if (size == RUMPBLK_SIZENOTSET) {
380 1.27 pooka KASSERT(size + offset <= flen);
381 1.27 pooka rblk->rblk_size = size;
382 1.27 pooka } else {
383 1.27 pooka KASSERT(offset < flen);
384 1.27 pooka rblk->rblk_size = flen - offset;
385 1.27 pooka }
386 1.27 pooka rblk->rblk_ftype = ftype;
387 1.24 pooka mutex_exit(&rumpblk_lock);
388 1.24 pooka
389 1.24 pooka *dmin = i;
390 1.24 pooka return 0;
391 1.1 pooka }
392 1.1 pooka
393 1.1 pooka int
394 1.1 pooka rumpblk_open(dev_t dev, int flag, int fmt, struct lwp *l)
395 1.1 pooka {
396 1.1 pooka struct rblkdev *rblk = &minors[minor(dev)];
397 1.5 pooka uint64_t fsize;
398 1.27 pooka int dummy;
399 1.1 pooka int error, fd;
400 1.1 pooka
401 1.27 pooka if (rblk->rblk_path == NULL)
402 1.27 pooka return ENXIO;
403 1.27 pooka
404 1.25 pooka if (rblk->rblk_fd != -1)
405 1.25 pooka return 0; /* XXX: refcount, open mode */
406 1.1 pooka fd = rumpuser_open(rblk->rblk_path, OFLAGS(flag), &error);
407 1.1 pooka if (error)
408 1.1 pooka return error;
409 1.1 pooka
410 1.20 pooka #ifdef HAS_ODIRECT
411 1.20 pooka rblk->rblk_dfd = rumpuser_open(rblk->rblk_path,
412 1.20 pooka OFLAGS(flag) | O_DIRECT, &error);
413 1.20 pooka if (error)
414 1.20 pooka return error;
415 1.20 pooka #endif
416 1.20 pooka
417 1.27 pooka if (rblk->rblk_ftype == RUMPUSER_FT_REG) {
418 1.16 pooka struct blkwin *win;
419 1.16 pooka int i, winsize;
420 1.16 pooka
421 1.1 pooka /*
422 1.16 pooka * Use mmap to access a regular file. Allocate and
423 1.16 pooka * cache initial windows here. Failure to allocate one
424 1.16 pooka * means fallback to read/write i/o.
425 1.1 pooka */
426 1.9 pooka
427 1.16 pooka rblk->rblk_mmflags = 0;
428 1.16 pooka if (flag & FREAD)
429 1.16 pooka rblk->rblk_mmflags |= RUMPUSER_FILEMMAP_READ;
430 1.16 pooka if (flag & FWRITE) {
431 1.16 pooka rblk->rblk_mmflags |= RUMPUSER_FILEMMAP_WRITE;
432 1.16 pooka rblk->rblk_mmflags |= RUMPUSER_FILEMMAP_SHARED;
433 1.16 pooka }
434 1.16 pooka
435 1.16 pooka TAILQ_INIT(&rblk->rblk_lruq);
436 1.16 pooka rblk->rblk_fd = fd;
437 1.16 pooka
438 1.19 pooka for (i = 0; i < memwincnt && i * memwinsize < fsize; i++) {
439 1.16 pooka win = kmem_zalloc(sizeof(*win), KM_SLEEP);
440 1.16 pooka WINVALIDATE(win);
441 1.16 pooka TAILQ_INSERT_TAIL(&rblk->rblk_lruq, win, win_lru);
442 1.16 pooka
443 1.16 pooka /*
444 1.16 pooka * Allocate first windows. Here we just generally
445 1.16 pooka * make sure a) we can mmap at all b) we have the
446 1.16 pooka * necessary VA available
447 1.16 pooka */
448 1.16 pooka winsize = 1;
449 1.19 pooka win = getwindow(rblk, i*memwinsize, &winsize, &error);
450 1.16 pooka if (win) {
451 1.16 pooka putwindow(rblk, win);
452 1.16 pooka } else {
453 1.16 pooka wincleanup(rblk);
454 1.16 pooka break;
455 1.12 pooka }
456 1.9 pooka }
457 1.9 pooka
458 1.1 pooka memset(&rblk->rblk_dl, 0, sizeof(rblk->rblk_dl));
459 1.9 pooka rblk->rblk_pi.p_size = fsize >> DEV_BSHIFT;
460 1.9 pooka rblk->rblk_dl.d_secsize = DEV_BSIZE;
461 1.9 pooka rblk->rblk_curpi = &rblk->rblk_pi;
462 1.9 pooka } else {
463 1.13 pooka if (rumpuser_ioctl(fd, DIOCGDINFO, &rblk->rblk_dl,
464 1.17 pooka &error) == -1) {
465 1.17 pooka KASSERT(error);
466 1.1 pooka rumpuser_close(fd, &dummy);
467 1.1 pooka return error;
468 1.1 pooka }
469 1.9 pooka
470 1.16 pooka rblk->rblk_fd = fd;
471 1.9 pooka rblk->rblk_curpi = &rblk->rblk_dl.d_partitions[0];
472 1.1 pooka }
473 1.1 pooka
474 1.16 pooka KASSERT(rblk->rblk_fd != -1);
475 1.1 pooka return 0;
476 1.1 pooka }
477 1.1 pooka
478 1.1 pooka int
479 1.1 pooka rumpblk_close(dev_t dev, int flag, int fmt, struct lwp *l)
480 1.1 pooka {
481 1.1 pooka struct rblkdev *rblk = &minors[minor(dev)];
482 1.1 pooka int dummy;
483 1.1 pooka
484 1.16 pooka if (rblk->rblk_mmflags)
485 1.16 pooka wincleanup(rblk);
486 1.16 pooka rumpuser_fsync(rblk->rblk_fd, &dummy);
487 1.1 pooka rumpuser_close(rblk->rblk_fd, &dummy);
488 1.1 pooka rblk->rblk_fd = -1;
489 1.1 pooka
490 1.1 pooka return 0;
491 1.1 pooka }
492 1.1 pooka
493 1.1 pooka int
494 1.1 pooka rumpblk_ioctl(dev_t dev, u_long xfer, void *addr, int flag, struct lwp *l)
495 1.1 pooka {
496 1.1 pooka struct rblkdev *rblk = &minors[minor(dev)];
497 1.1 pooka int rv, error;
498 1.1 pooka
499 1.1 pooka if (xfer == DIOCGPART) {
500 1.1 pooka struct partinfo *pi = (struct partinfo *)addr;
501 1.1 pooka
502 1.1 pooka pi->part = rblk->rblk_curpi;
503 1.1 pooka pi->disklab = &rblk->rblk_dl;
504 1.1 pooka
505 1.1 pooka return 0;
506 1.1 pooka }
507 1.1 pooka
508 1.1 pooka rv = rumpuser_ioctl(rblk->rblk_fd, xfer, addr, &error);
509 1.1 pooka if (rv == -1)
510 1.1 pooka return error;
511 1.1 pooka
512 1.1 pooka return 0;
513 1.1 pooka }
514 1.1 pooka
515 1.25 pooka static int
516 1.25 pooka do_physio(dev_t dev, struct uio *uio, int which)
517 1.25 pooka {
518 1.25 pooka void (*strat)(struct buf *);
519 1.25 pooka
520 1.25 pooka if (blkfail)
521 1.25 pooka strat = rumpblk_strategy_fail;
522 1.25 pooka else
523 1.25 pooka strat = rumpblk_strategy;
524 1.25 pooka
525 1.25 pooka return physio(strat, NULL, dev, which, minphys, uio);
526 1.25 pooka }
527 1.25 pooka
528 1.1 pooka int
529 1.1 pooka rumpblk_read(dev_t dev, struct uio *uio, int flags)
530 1.1 pooka {
531 1.1 pooka
532 1.25 pooka return do_physio(dev, uio, B_READ);
533 1.1 pooka }
534 1.1 pooka
535 1.1 pooka int
536 1.1 pooka rumpblk_write(dev_t dev, struct uio *uio, int flags)
537 1.1 pooka {
538 1.1 pooka
539 1.25 pooka return do_physio(dev, uio, B_WRITE);
540 1.1 pooka }
541 1.1 pooka
542 1.3 pooka static void
543 1.3 pooka dostrategy(struct buf *bp)
544 1.1 pooka {
545 1.1 pooka struct rblkdev *rblk = &minors[minor(bp->b_dev)];
546 1.1 pooka off_t off;
547 1.21 pooka int async = bp->b_flags & B_ASYNC;
548 1.21 pooka int error;
549 1.1 pooka
550 1.20 pooka /* collect statistics */
551 1.20 pooka ev_io_total.ev_count++;
552 1.20 pooka if (async)
553 1.20 pooka ev_io_async.ev_count++;
554 1.20 pooka if (BUF_ISWRITE(bp)) {
555 1.20 pooka ev_bwrite_total.ev_count += bp->b_bcount;
556 1.20 pooka if (async)
557 1.20 pooka ev_bwrite_async.ev_count += bp->b_bcount;
558 1.20 pooka } else {
559 1.20 pooka ev_bread_total.ev_count++;
560 1.20 pooka }
561 1.20 pooka
562 1.1 pooka off = bp->b_blkno << DEV_BSHIFT;
563 1.27 pooka off += rblk->rblk_hostoffset;
564 1.11 pooka /*
565 1.11 pooka * Do bounds checking if we're working on a file. Otherwise
566 1.11 pooka * invalid file systems might attempt to read beyond EOF. This
567 1.11 pooka * is bad(tm) especially on mmapped images. This is essentially
568 1.11 pooka * the kernel bounds_check() routines.
569 1.11 pooka */
570 1.27 pooka if (off + bp->b_bcount > rblk->rblk_size) {
571 1.11 pooka int64_t sz = rblk->rblk_size - off;
572 1.11 pooka
573 1.11 pooka /* EOF */
574 1.11 pooka if (sz == 0) {
575 1.11 pooka rump_biodone(bp, 0, 0);
576 1.11 pooka return;
577 1.11 pooka }
578 1.11 pooka /* beyond EOF ==> error */
579 1.11 pooka if (sz < 0) {
580 1.11 pooka rump_biodone(bp, 0, EINVAL);
581 1.11 pooka return;
582 1.11 pooka }
583 1.11 pooka
584 1.11 pooka /* truncate to device size */
585 1.11 pooka bp->b_bcount = sz;
586 1.11 pooka }
587 1.11 pooka
588 1.1 pooka DPRINTF(("rumpblk_strategy: 0x%x bytes %s off 0x%" PRIx64
589 1.20 pooka " (0x%" PRIx64 " - 0x%" PRIx64 "), %ssync\n",
590 1.16 pooka bp->b_bcount, BUF_ISREAD(bp) ? "READ" : "WRITE",
591 1.20 pooka off, off, (off + bp->b_bcount), async ? "a" : ""));
592 1.1 pooka
593 1.16 pooka /* mmap? handle here and return */
594 1.16 pooka if (rblk->rblk_mmflags) {
595 1.16 pooka struct blkwin *win;
596 1.16 pooka int winsize, iodone;
597 1.16 pooka uint8_t *ioaddr, *bufaddr;
598 1.16 pooka
599 1.16 pooka for (iodone = 0; iodone < bp->b_bcount;
600 1.16 pooka iodone += winsize, off += winsize) {
601 1.16 pooka winsize = bp->b_bcount - iodone;
602 1.16 pooka win = getwindow(rblk, off, &winsize, &error);
603 1.16 pooka if (win == NULL) {
604 1.16 pooka rump_biodone(bp, iodone, error);
605 1.16 pooka return;
606 1.16 pooka }
607 1.16 pooka
608 1.16 pooka ioaddr = (uint8_t *)win->win_mem + (off-STARTWIN(off));
609 1.16 pooka bufaddr = (uint8_t *)bp->b_data + iodone;
610 1.11 pooka
611 1.16 pooka DPRINTF(("strat: %p off 0x%" PRIx64
612 1.16 pooka ", ioaddr %p (%p)/buf %p\n", win,
613 1.16 pooka win->win_off, ioaddr, win->win_mem, bufaddr));
614 1.16 pooka if (BUF_ISREAD(bp)) {
615 1.16 pooka memcpy(bufaddr, ioaddr, winsize);
616 1.16 pooka } else {
617 1.16 pooka memcpy(ioaddr, bufaddr, winsize);
618 1.16 pooka }
619 1.16 pooka
620 1.16 pooka /* synchronous write, sync bits back to disk */
621 1.16 pooka if (BUF_ISWRITE(bp) && !async) {
622 1.16 pooka rumpuser_memsync(ioaddr, winsize, &error);
623 1.16 pooka }
624 1.16 pooka putwindow(rblk, win);
625 1.9 pooka }
626 1.9 pooka
627 1.9 pooka rump_biodone(bp, bp->b_bcount, 0);
628 1.9 pooka return;
629 1.9 pooka }
630 1.9 pooka
631 1.1 pooka /*
632 1.1 pooka * Do I/O. We have different paths for async and sync I/O.
633 1.1 pooka * Async I/O is done by passing a request to rumpuser where
634 1.1 pooka * it is executed. The rumpuser routine then calls
635 1.1 pooka * biodone() to signal any waiters in the kernel. I/O's are
636 1.1 pooka * executed in series. Technically executing them in parallel
637 1.1 pooka * would produce better results, but then we'd need either
638 1.1 pooka * more threads or posix aio. Maybe worth investigating
639 1.1 pooka * this later.
640 1.14 pooka *
641 1.14 pooka * Using bufq here might be a good idea.
642 1.1 pooka */
643 1.20 pooka
644 1.13 pooka if (rump_threads) {
645 1.1 pooka struct rumpuser_aio *rua;
646 1.20 pooka int op, fd;
647 1.20 pooka
648 1.20 pooka fd = rblk->rblk_fd;
649 1.20 pooka if (BUF_ISREAD(bp)) {
650 1.20 pooka op = RUA_OP_READ;
651 1.20 pooka } else {
652 1.20 pooka op = RUA_OP_WRITE;
653 1.20 pooka if (!async) {
654 1.20 pooka /* O_DIRECT not fully automatic yet */
655 1.20 pooka #ifdef HAS_ODIRECT
656 1.20 pooka if ((off & (DEV_BSIZE-1)) == 0
657 1.20 pooka && ((intptr_t)bp->b_data&(DEV_BSIZE-1)) == 0
658 1.20 pooka && (bp->b_bcount & (DEV_BSIZE-1)) == 0)
659 1.20 pooka fd = rblk->rblk_dfd;
660 1.20 pooka else
661 1.20 pooka #endif
662 1.20 pooka op |= RUA_OP_SYNC;
663 1.20 pooka }
664 1.20 pooka }
665 1.1 pooka
666 1.1 pooka rumpuser_mutex_enter(&rumpuser_aio_mtx);
667 1.20 pooka while ((rumpuser_aio_head+1) % N_AIOS == rumpuser_aio_tail) {
668 1.13 pooka rumpuser_cv_wait(&rumpuser_aio_cv, &rumpuser_aio_mtx);
669 1.20 pooka }
670 1.1 pooka
671 1.2 pooka rua = &rumpuser_aios[rumpuser_aio_head];
672 1.2 pooka KASSERT(rua->rua_bp == NULL);
673 1.20 pooka rua->rua_fd = fd;
674 1.2 pooka rua->rua_data = bp->b_data;
675 1.2 pooka rua->rua_dlen = bp->b_bcount;
676 1.2 pooka rua->rua_off = off;
677 1.2 pooka rua->rua_bp = bp;
678 1.20 pooka rua->rua_op = op;
679 1.2 pooka
680 1.1 pooka /* insert into queue & signal */
681 1.6 pooka rumpuser_aio_head = (rumpuser_aio_head+1) % N_AIOS;
682 1.1 pooka rumpuser_cv_signal(&rumpuser_aio_cv);
683 1.1 pooka rumpuser_mutex_exit(&rumpuser_aio_mtx);
684 1.1 pooka } else {
685 1.1 pooka if (BUF_ISREAD(bp)) {
686 1.1 pooka rumpuser_read_bio(rblk->rblk_fd, bp->b_data,
687 1.1 pooka bp->b_bcount, off, rump_biodone, bp);
688 1.1 pooka } else {
689 1.1 pooka rumpuser_write_bio(rblk->rblk_fd, bp->b_data,
690 1.1 pooka bp->b_bcount, off, rump_biodone, bp);
691 1.1 pooka }
692 1.20 pooka if (BUF_ISWRITE(bp) && !async)
693 1.20 pooka rumpuser_fsync(rblk->rblk_fd, &error);
694 1.1 pooka }
695 1.1 pooka }
696 1.3 pooka
697 1.3 pooka void
698 1.3 pooka rumpblk_strategy(struct buf *bp)
699 1.3 pooka {
700 1.3 pooka
701 1.3 pooka dostrategy(bp);
702 1.3 pooka }
703 1.3 pooka
704 1.3 pooka /*
705 1.4 pooka * Simple random number generator. This is private so that we can
706 1.4 pooka * very repeatedly control which blocks will fail.
707 1.4 pooka *
708 1.3 pooka * <mlelstv> pooka, rand()
709 1.3 pooka * <mlelstv> [paste]
710 1.3 pooka */
711 1.3 pooka static unsigned
712 1.3 pooka gimmerand(void)
713 1.3 pooka {
714 1.3 pooka
715 1.3 pooka return (randstate = randstate * 1103515245 + 12345) % (0x80000000L);
716 1.3 pooka }
717 1.3 pooka
718 1.3 pooka /*
719 1.3 pooka * Block device with very simple fault injection. Fails every
720 1.3 pooka * n out of BLKFAIL_MAX I/O with EIO. n is determined by the env
721 1.3 pooka * variable RUMP_BLKFAIL.
722 1.3 pooka */
723 1.3 pooka void
724 1.3 pooka rumpblk_strategy_fail(struct buf *bp)
725 1.3 pooka {
726 1.3 pooka
727 1.3 pooka if (gimmerand() % BLKFAIL_MAX >= blkfail) {
728 1.3 pooka dostrategy(bp);
729 1.3 pooka } else {
730 1.3 pooka printf("block fault injection: failing I/O on block %lld\n",
731 1.3 pooka (long long)bp->b_blkno);
732 1.3 pooka bp->b_error = EIO;
733 1.3 pooka biodone(bp);
734 1.3 pooka }
735 1.3 pooka }
736