kern_physio.c revision 1.93 1 1.93 pooka /* $NetBSD: kern_physio.c,v 1.93 2015/04/21 10:54:52 pooka Exp $ */
2 1.20 cgd
3 1.20 cgd /*-
4 1.20 cgd * Copyright (c) 1982, 1986, 1990, 1993
5 1.20 cgd * The Regents of the University of California. All rights reserved.
6 1.20 cgd * (c) UNIX System Laboratories, Inc.
7 1.20 cgd * All or some portions of this file are derived from material licensed
8 1.20 cgd * to the University of California by American Telephone and Telegraph
9 1.20 cgd * Co. or Unix System Laboratories, Inc. and are reproduced herein with
10 1.20 cgd * the permission of UNIX System Laboratories, Inc.
11 1.20 cgd *
12 1.20 cgd * Redistribution and use in source and binary forms, with or without
13 1.20 cgd * modification, are permitted provided that the following conditions
14 1.20 cgd * are met:
15 1.20 cgd * 1. Redistributions of source code must retain the above copyright
16 1.20 cgd * notice, this list of conditions and the following disclaimer.
17 1.20 cgd * 2. Redistributions in binary form must reproduce the above copyright
18 1.20 cgd * notice, this list of conditions and the following disclaimer in the
19 1.20 cgd * documentation and/or other materials provided with the distribution.
20 1.57 agc * 3. Neither the name of the University nor the names of its contributors
21 1.57 agc * may be used to endorse or promote products derived from this software
22 1.57 agc * without specific prior written permission.
23 1.57 agc *
24 1.57 agc * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
25 1.57 agc * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 1.57 agc * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 1.57 agc * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
28 1.57 agc * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29 1.57 agc * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
30 1.57 agc * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31 1.57 agc * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32 1.57 agc * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
33 1.57 agc * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 1.57 agc * SUCH DAMAGE.
35 1.57 agc *
36 1.57 agc * @(#)kern_physio.c 8.1 (Berkeley) 6/10/93
37 1.57 agc */
38 1.57 agc
39 1.57 agc /*-
40 1.57 agc * Copyright (c) 1994 Christopher G. Demetriou
41 1.57 agc *
42 1.57 agc * Redistribution and use in source and binary forms, with or without
43 1.57 agc * modification, are permitted provided that the following conditions
44 1.57 agc * are met:
45 1.57 agc * 1. Redistributions of source code must retain the above copyright
46 1.57 agc * notice, this list of conditions and the following disclaimer.
47 1.57 agc * 2. Redistributions in binary form must reproduce the above copyright
48 1.57 agc * notice, this list of conditions and the following disclaimer in the
49 1.57 agc * documentation and/or other materials provided with the distribution.
50 1.20 cgd * 3. All advertising materials mentioning features or use of this software
51 1.20 cgd * must display the following acknowledgement:
52 1.20 cgd * This product includes software developed by the University of
53 1.20 cgd * California, Berkeley and its contributors.
54 1.20 cgd * 4. Neither the name of the University nor the names of its contributors
55 1.20 cgd * may be used to endorse or promote products derived from this software
56 1.20 cgd * without specific prior written permission.
57 1.20 cgd *
58 1.20 cgd * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
59 1.20 cgd * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
60 1.20 cgd * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
61 1.20 cgd * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
62 1.20 cgd * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
63 1.20 cgd * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
64 1.20 cgd * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
65 1.20 cgd * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
66 1.20 cgd * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
67 1.20 cgd * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
68 1.20 cgd * SUCH DAMAGE.
69 1.20 cgd *
70 1.20 cgd * @(#)kern_physio.c 8.1 (Berkeley) 6/10/93
71 1.20 cgd */
72 1.51 lukem
73 1.51 lukem #include <sys/cdefs.h>
74 1.93 pooka __KERNEL_RCSID(0, "$NetBSD: kern_physio.c,v 1.93 2015/04/21 10:54:52 pooka Exp $");
75 1.30 mrg
76 1.20 cgd #include <sys/param.h>
77 1.20 cgd #include <sys/systm.h>
78 1.20 cgd #include <sys/buf.h>
79 1.20 cgd #include <sys/proc.h>
80 1.62 yamt #include <sys/once.h>
81 1.62 yamt #include <sys/workqueue.h>
82 1.84 hannken #include <sys/kmem.h>
83 1.26 christos
84 1.29 mrg #include <uvm/uvm_extern.h>
85 1.29 mrg
86 1.62 yamt ONCE_DECL(physio_initialized);
87 1.62 yamt struct workqueue *physio_workqueue;
88 1.62 yamt
89 1.90 ad int physio_concurrency = 16;
90 1.20 cgd
91 1.70 yamt /* #define PHYSIO_DEBUG */
92 1.70 yamt #if defined(PHYSIO_DEBUG)
93 1.70 yamt #define DPRINTF(a) printf a
94 1.70 yamt #else /* defined(PHYSIO_DEBUG) */
95 1.70 yamt #define DPRINTF(a) /* nothing */
96 1.70 yamt #endif /* defined(PHYSIO_DEBUG) */
97 1.70 yamt
98 1.84 hannken struct physio_stat {
99 1.84 hannken int ps_running;
100 1.84 hannken int ps_error;
101 1.84 hannken int ps_failed;
102 1.84 hannken off_t ps_endoffset;
103 1.88 hannken buf_t *ps_orig_bp;
104 1.84 hannken kmutex_t ps_lock;
105 1.84 hannken kcondvar_t ps_cv;
106 1.84 hannken };
107 1.84 hannken
108 1.62 yamt static void
109 1.76 yamt physio_done(struct work *wk, void *dummy)
110 1.62 yamt {
111 1.62 yamt struct buf *bp = (void *)wk;
112 1.62 yamt size_t todo = bp->b_bufsize;
113 1.66 yamt size_t done = bp->b_bcount - bp->b_resid;
114 1.84 hannken struct physio_stat *ps = bp->b_private;
115 1.91 hannken bool is_iobuf;
116 1.62 yamt
117 1.62 yamt KASSERT(&bp->b_work == wk);
118 1.62 yamt KASSERT(bp->b_bcount <= todo);
119 1.62 yamt KASSERT(bp->b_resid <= bp->b_bcount);
120 1.62 yamt KASSERT((bp->b_flags & B_PHYS) != 0);
121 1.62 yamt KASSERT(dummy == NULL);
122 1.62 yamt
123 1.62 yamt vunmapbuf(bp, todo);
124 1.74 chs uvm_vsunlock(bp->b_proc->p_vmspace, bp->b_data, todo);
125 1.62 yamt
126 1.84 hannken mutex_enter(&ps->ps_lock);
127 1.91 hannken is_iobuf = (bp != ps->ps_orig_bp);
128 1.66 yamt if (__predict_false(done != todo)) {
129 1.66 yamt off_t endoffset = dbtob(bp->b_blkno) + done;
130 1.66 yamt
131 1.70 yamt /*
132 1.70 yamt * we got an error or hit EOM.
133 1.70 yamt *
134 1.70 yamt * we only care about the first one.
135 1.70 yamt * ie. the one at the lowest offset.
136 1.70 yamt */
137 1.70 yamt
138 1.84 hannken KASSERT(ps->ps_endoffset != endoffset);
139 1.70 yamt DPRINTF(("%s: error=%d at %" PRIu64 " - %" PRIu64
140 1.70 yamt ", blkno=%" PRIu64 ", bcount=%d, flags=0x%x\n",
141 1.70 yamt __func__, bp->b_error, dbtob(bp->b_blkno), endoffset,
142 1.70 yamt bp->b_blkno, bp->b_bcount, bp->b_flags));
143 1.70 yamt
144 1.84 hannken if (ps->ps_endoffset == -1 || endoffset < ps->ps_endoffset) {
145 1.84 hannken DPRINTF(("%s: ps=%p, error %d -> %d, endoff %" PRIu64
146 1.70 yamt " -> %" PRIu64 "\n",
147 1.84 hannken __func__, ps,
148 1.84 hannken ps->ps_error, bp->b_error,
149 1.84 hannken ps->ps_endoffset, endoffset));
150 1.66 yamt
151 1.84 hannken ps->ps_endoffset = endoffset;
152 1.84 hannken ps->ps_error = bp->b_error;
153 1.62 yamt }
154 1.84 hannken ps->ps_failed++;
155 1.70 yamt } else {
156 1.83 ad KASSERT(bp->b_error == 0);
157 1.62 yamt }
158 1.67 yamt
159 1.84 hannken ps->ps_running--;
160 1.84 hannken cv_signal(&ps->ps_cv);
161 1.84 hannken mutex_exit(&ps->ps_lock);
162 1.62 yamt
163 1.91 hannken if (is_iobuf)
164 1.88 hannken putiobuf(bp);
165 1.62 yamt }
166 1.62 yamt
167 1.62 yamt static void
168 1.62 yamt physio_biodone(struct buf *bp)
169 1.62 yamt {
170 1.63 yamt #if defined(DIAGNOSTIC)
171 1.84 hannken struct physio_stat *ps = bp->b_private;
172 1.62 yamt size_t todo = bp->b_bufsize;
173 1.89 bouyer size_t done = bp->b_bcount - bp->b_resid;
174 1.62 yamt
175 1.84 hannken KASSERT(ps->ps_running > 0);
176 1.62 yamt KASSERT(bp->b_bcount <= todo);
177 1.62 yamt KASSERT(bp->b_resid <= bp->b_bcount);
178 1.89 bouyer if (done == todo)
179 1.89 bouyer KASSERT(bp->b_error == 0);
180 1.63 yamt #endif /* defined(DIAGNOSTIC) */
181 1.62 yamt
182 1.82 rmind workqueue_enqueue(physio_workqueue, &bp->b_work, NULL);
183 1.62 yamt }
184 1.62 yamt
185 1.84 hannken static void
186 1.84 hannken physio_wait(struct physio_stat *ps, int n)
187 1.62 yamt {
188 1.62 yamt
189 1.84 hannken KASSERT(mutex_owned(&ps->ps_lock));
190 1.62 yamt
191 1.84 hannken while (ps->ps_running > n)
192 1.84 hannken cv_wait(&ps->ps_cv, &ps->ps_lock);
193 1.62 yamt }
194 1.62 yamt
195 1.72 yamt static int
196 1.62 yamt physio_init(void)
197 1.62 yamt {
198 1.72 yamt int error;
199 1.62 yamt
200 1.62 yamt KASSERT(physio_workqueue == NULL);
201 1.62 yamt
202 1.72 yamt error = workqueue_create(&physio_workqueue, "physiod",
203 1.86 ad physio_done, NULL, PRI_BIO, IPL_BIO, WQ_MPSAFE);
204 1.72 yamt
205 1.72 yamt return error;
206 1.62 yamt }
207 1.62 yamt
208 1.20 cgd /*
209 1.20 cgd * Do "physical I/O" on behalf of a user. "Physical I/O" is I/O directly
210 1.20 cgd * from the raw device to user buffers, and bypasses the buffer cache.
211 1.20 cgd */
212 1.20 cgd int
213 1.62 yamt physio(void (*strategy)(struct buf *), struct buf *obp, dev_t dev, int flags,
214 1.61 thorpej void (*min_phys)(struct buf *), struct uio *uio)
215 1.20 cgd {
216 1.20 cgd struct iovec *iovp;
217 1.54 thorpej struct lwp *l = curlwp;
218 1.54 thorpej struct proc *p = l->l_proc;
219 1.86 ad int i, error;
220 1.62 yamt struct buf *bp = NULL;
221 1.84 hannken struct physio_stat *ps;
222 1.90 ad int concurrency = physio_concurrency - 1;
223 1.62 yamt
224 1.72 yamt error = RUN_ONCE(&physio_initialized, physio_init);
225 1.72 yamt if (__predict_false(error != 0)) {
226 1.72 yamt return error;
227 1.72 yamt }
228 1.20 cgd
229 1.70 yamt DPRINTF(("%s: called: off=%" PRIu64 ", resid=%zu\n",
230 1.70 yamt __func__, uio->uio_offset, uio->uio_resid));
231 1.70 yamt
232 1.53 hannken flags &= B_READ | B_WRITE;
233 1.20 cgd
234 1.93 pooka ps = kmem_zalloc(sizeof(*ps), KM_SLEEP);
235 1.84 hannken /* ps->ps_running = 0; */
236 1.84 hannken /* ps->ps_error = 0; */
237 1.84 hannken /* ps->ps_failed = 0; */
238 1.88 hannken ps->ps_orig_bp = obp;
239 1.84 hannken ps->ps_endoffset = -1;
240 1.84 hannken mutex_init(&ps->ps_lock, MUTEX_DEFAULT, IPL_NONE);
241 1.84 hannken cv_init(&ps->ps_cv, "physio");
242 1.84 hannken
243 1.20 cgd /* Make sure we have a buffer, creating one if necessary. */
244 1.62 yamt if (obp != NULL) {
245 1.86 ad mutex_enter(&bufcache_lock);
246 1.88 hannken /* Mark it busy, so nobody else will use it. */
247 1.87 ad while (bbusy(obp, false, 0, NULL) == EPASSTHROUGH)
248 1.86 ad ;
249 1.86 ad mutex_exit(&bufcache_lock);
250 1.65 yamt concurrency = 0; /* see "XXXkludge" comment below */
251 1.29 mrg }
252 1.20 cgd
253 1.20 cgd for (i = 0; i < uio->uio_iovcnt; i++) {
254 1.79 thorpej bool sync = true;
255 1.68 yamt
256 1.20 cgd iovp = &uio->uio_iov[i];
257 1.20 cgd while (iovp->iov_len > 0) {
258 1.68 yamt size_t todo;
259 1.69 yamt vaddr_t endp;
260 1.68 yamt
261 1.84 hannken mutex_enter(&ps->ps_lock);
262 1.84 hannken if (ps->ps_failed != 0) {
263 1.62 yamt goto done_locked;
264 1.62 yamt }
265 1.84 hannken physio_wait(ps, sync ? 0 : concurrency);
266 1.84 hannken mutex_exit(&ps->ps_lock);
267 1.65 yamt if (obp != NULL) {
268 1.65 yamt /*
269 1.65 yamt * XXXkludge
270 1.65 yamt * some drivers use "obp" as an identifier.
271 1.65 yamt */
272 1.65 yamt bp = obp;
273 1.65 yamt } else {
274 1.88 hannken bp = getiobuf(NULL, true);
275 1.88 hannken bp->b_cflags = BC_BUSY;
276 1.65 yamt }
277 1.62 yamt bp->b_dev = dev;
278 1.62 yamt bp->b_proc = p;
279 1.84 hannken bp->b_private = ps;
280 1.47 chs
281 1.20 cgd /*
282 1.90 ad * Mrk the buffer busy for physical I/O. Also set
283 1.90 ad * B_PHYS because it's an I/O to user memory, and
284 1.90 ad * B_RAW because B_RAW is to be "set by physio for
285 1.90 ad * raw transfers".
286 1.20 cgd */
287 1.86 ad bp->b_oflags = 0;
288 1.88 hannken bp->b_cflags = BC_BUSY;
289 1.86 ad bp->b_flags = flags | B_PHYS | B_RAW;
290 1.62 yamt bp->b_iodone = physio_biodone;
291 1.20 cgd
292 1.90 ad /* Set up the buffer for a maximum-sized transfer. */
293 1.20 cgd bp->b_blkno = btodb(uio->uio_offset);
294 1.62 yamt if (dbtob(bp->b_blkno) != uio->uio_offset) {
295 1.62 yamt error = EINVAL;
296 1.62 yamt goto done;
297 1.62 yamt }
298 1.62 yamt bp->b_bcount = MIN(MAXPHYS, iovp->iov_len);
299 1.20 cgd bp->b_data = iovp->iov_base;
300 1.24 mycroft
301 1.20 cgd /*
302 1.90 ad * Call minphys to bound the transfer size,
303 1.20 cgd * and remember the amount of data to transfer,
304 1.20 cgd * for later comparison.
305 1.20 cgd */
306 1.59 simonb (*min_phys)(bp);
307 1.69 yamt todo = bp->b_bufsize = bp->b_bcount;
308 1.62 yamt #if defined(DIAGNOSTIC)
309 1.69 yamt if (todo > MAXPHYS)
310 1.62 yamt panic("todo(%zu) > MAXPHYS; minphys broken",
311 1.69 yamt todo);
312 1.62 yamt #endif /* defined(DIAGNOSTIC) */
313 1.20 cgd
314 1.79 thorpej sync = false;
315 1.69 yamt endp = (vaddr_t)bp->b_data + todo;
316 1.69 yamt if (trunc_page(endp) != endp) {
317 1.69 yamt /*
318 1.90 ad * Following requests can overlap.
319 1.69 yamt * note that uvm_vslock does round_page.
320 1.69 yamt */
321 1.79 thorpej sync = true;
322 1.68 yamt }
323 1.68 yamt
324 1.20 cgd /*
325 1.90 ad * Lock the part of the user address space involved
326 1.92 pooka * in the transfer.
327 1.20 cgd */
328 1.74 chs error = uvm_vslock(p->p_vmspace, bp->b_data, todo,
329 1.62 yamt (flags & B_READ) ? VM_PROT_WRITE : VM_PROT_READ);
330 1.47 chs if (error) {
331 1.62 yamt goto done;
332 1.37 thorpej }
333 1.92 pooka
334 1.92 pooka /*
335 1.92 pooka * Beware vmapbuf(); if succesful it clobbers
336 1.92 pooka * b_data and saves it in b_saveaddr.
337 1.92 pooka * However, vunmapbuf() restores b_data.
338 1.92 pooka */
339 1.92 pooka if ((error = vmapbuf(bp, todo)) != 0) {
340 1.92 pooka uvm_vsunlock(p->p_vmspace, bp->b_data, todo);
341 1.92 pooka goto done;
342 1.92 pooka }
343 1.20 cgd
344 1.58 yamt BIO_SETPRIO(bp, BPRIO_TIMECRITICAL);
345 1.58 yamt
346 1.84 hannken mutex_enter(&ps->ps_lock);
347 1.84 hannken ps->ps_running++;
348 1.84 hannken mutex_exit(&ps->ps_lock);
349 1.62 yamt
350 1.90 ad /* Call strategy to start the transfer. */
351 1.20 cgd (*strategy)(bp);
352 1.62 yamt bp = NULL;
353 1.20 cgd
354 1.62 yamt iovp->iov_len -= todo;
355 1.80 christos iovp->iov_base = (char *)iovp->iov_base + todo;
356 1.62 yamt uio->uio_offset += todo;
357 1.62 yamt uio->uio_resid -= todo;
358 1.20 cgd }
359 1.20 cgd }
360 1.20 cgd
361 1.20 cgd done:
362 1.84 hannken mutex_enter(&ps->ps_lock);
363 1.62 yamt done_locked:
364 1.84 hannken physio_wait(ps, 0);
365 1.84 hannken mutex_exit(&ps->ps_lock);
366 1.66 yamt
367 1.84 hannken if (ps->ps_failed != 0) {
368 1.70 yamt off_t delta;
369 1.70 yamt
370 1.84 hannken delta = uio->uio_offset - ps->ps_endoffset;
371 1.70 yamt KASSERT(delta > 0);
372 1.70 yamt uio->uio_resid += delta;
373 1.84 hannken /* uio->uio_offset = ps->ps_endoffset; */
374 1.66 yamt } else {
375 1.84 hannken KASSERT(ps->ps_endoffset == -1);
376 1.62 yamt }
377 1.88 hannken if (bp != NULL && bp != obp) {
378 1.88 hannken putiobuf(bp);
379 1.62 yamt }
380 1.62 yamt if (error == 0) {
381 1.84 hannken error = ps->ps_error;
382 1.62 yamt }
383 1.84 hannken mutex_destroy(&ps->ps_lock);
384 1.84 hannken cv_destroy(&ps->ps_cv);
385 1.84 hannken kmem_free(ps, sizeof(*ps));
386 1.62 yamt
387 1.20 cgd /*
388 1.90 ad * Clean up the state of the buffer. Remember if somebody wants
389 1.90 ad * it, so we can wake them up below. Also, if we had to steal it,
390 1.90 ad * give it back.
391 1.20 cgd */
392 1.62 yamt if (obp != NULL) {
393 1.86 ad KASSERT((obp->b_cflags & BC_BUSY) != 0);
394 1.65 yamt
395 1.20 cgd /*
396 1.90 ad * If another process is waiting for the raw I/O buffer,
397 1.90 ad * wake up processes waiting to do physical I/O;
398 1.20 cgd */
399 1.86 ad mutex_enter(&bufcache_lock);
400 1.88 hannken obp->b_cflags &= ~(BC_BUSY | BC_WANTED);
401 1.86 ad obp->b_flags &= ~(B_PHYS | B_RAW);
402 1.86 ad obp->b_iodone = NULL;
403 1.86 ad cv_broadcast(&obp->b_busy);
404 1.86 ad mutex_exit(&bufcache_lock);
405 1.20 cgd }
406 1.20 cgd
407 1.70 yamt DPRINTF(("%s: done: off=%" PRIu64 ", resid=%zu\n",
408 1.70 yamt __func__, uio->uio_offset, uio->uio_resid));
409 1.70 yamt
410 1.62 yamt return error;
411 1.20 cgd }
412 1.20 cgd
413 1.20 cgd /*
414 1.90 ad * A minphys() routine is called by physio() to adjust the size of each
415 1.90 ad * I/O transfer before the latter is passed to the strategy routine.
416 1.20 cgd *
417 1.90 ad * This minphys() is a default that must be called to enforce limits
418 1.90 ad * that are applicable to all devices, because of limitations in the
419 1.90 ad * kernel or the hardware platform.
420 1.20 cgd */
421 1.24 mycroft void
422 1.61 thorpej minphys(struct buf *bp)
423 1.20 cgd {
424 1.20 cgd
425 1.24 mycroft if (bp->b_bcount > MAXPHYS)
426 1.24 mycroft bp->b_bcount = MAXPHYS;
427 1.20 cgd }
428