genfs_io.c revision 1.39 1 1.39 pooka /* $NetBSD: genfs_io.c,v 1.39 2010/08/19 02:10:02 pooka Exp $ */
2 1.1 pooka
3 1.1 pooka /*
4 1.1 pooka * Copyright (c) 1982, 1986, 1989, 1993
5 1.1 pooka * The Regents of the University of California. All rights reserved.
6 1.1 pooka *
7 1.1 pooka * Redistribution and use in source and binary forms, with or without
8 1.1 pooka * modification, are permitted provided that the following conditions
9 1.1 pooka * are met:
10 1.1 pooka * 1. Redistributions of source code must retain the above copyright
11 1.1 pooka * notice, this list of conditions and the following disclaimer.
12 1.1 pooka * 2. Redistributions in binary form must reproduce the above copyright
13 1.1 pooka * notice, this list of conditions and the following disclaimer in the
14 1.1 pooka * documentation and/or other materials provided with the distribution.
15 1.1 pooka * 3. Neither the name of the University nor the names of its contributors
16 1.1 pooka * may be used to endorse or promote products derived from this software
17 1.1 pooka * without specific prior written permission.
18 1.1 pooka *
19 1.1 pooka * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
20 1.1 pooka * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 1.1 pooka * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 1.1 pooka * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
23 1.1 pooka * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 1.1 pooka * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 1.1 pooka * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 1.1 pooka * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 1.1 pooka * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 1.1 pooka * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 1.1 pooka * SUCH DAMAGE.
30 1.1 pooka *
31 1.1 pooka */
32 1.1 pooka
33 1.1 pooka #include <sys/cdefs.h>
34 1.39 pooka __KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.39 2010/08/19 02:10:02 pooka Exp $");
35 1.1 pooka
36 1.1 pooka #include <sys/param.h>
37 1.1 pooka #include <sys/systm.h>
38 1.1 pooka #include <sys/proc.h>
39 1.1 pooka #include <sys/kernel.h>
40 1.1 pooka #include <sys/mount.h>
41 1.1 pooka #include <sys/namei.h>
42 1.1 pooka #include <sys/vnode.h>
43 1.1 pooka #include <sys/fcntl.h>
44 1.1 pooka #include <sys/kmem.h>
45 1.1 pooka #include <sys/poll.h>
46 1.1 pooka #include <sys/mman.h>
47 1.1 pooka #include <sys/file.h>
48 1.1 pooka #include <sys/kauth.h>
49 1.1 pooka #include <sys/fstrans.h>
50 1.15 pooka #include <sys/buf.h>
51 1.1 pooka
52 1.1 pooka #include <miscfs/genfs/genfs.h>
53 1.1 pooka #include <miscfs/genfs/genfs_node.h>
54 1.1 pooka #include <miscfs/specfs/specdev.h>
55 1.1 pooka
56 1.1 pooka #include <uvm/uvm.h>
57 1.1 pooka #include <uvm/uvm_pager.h>
58 1.1 pooka
59 1.1 pooka static int genfs_do_directio(struct vmspace *, vaddr_t, size_t, struct vnode *,
60 1.1 pooka off_t, enum uio_rw);
61 1.1 pooka static void genfs_dio_iodone(struct buf *);
62 1.1 pooka
63 1.1 pooka static int genfs_do_io(struct vnode *, off_t, vaddr_t, size_t, int, enum uio_rw,
64 1.1 pooka void (*)(struct buf *));
65 1.38 chs static void genfs_rel_pages(struct vm_page **, int);
66 1.38 chs static void genfs_markdirty(struct vnode *);
67 1.1 pooka
68 1.1 pooka int genfs_maxdio = MAXPHYS;
69 1.1 pooka
70 1.38 chs static void
71 1.1 pooka genfs_rel_pages(struct vm_page **pgs, int npages)
72 1.1 pooka {
73 1.1 pooka int i;
74 1.1 pooka
75 1.1 pooka for (i = 0; i < npages; i++) {
76 1.1 pooka struct vm_page *pg = pgs[i];
77 1.1 pooka
78 1.1 pooka if (pg == NULL || pg == PGO_DONTCARE)
79 1.1 pooka continue;
80 1.1 pooka if (pg->flags & PG_FAKE) {
81 1.1 pooka pg->flags |= PG_RELEASED;
82 1.1 pooka }
83 1.1 pooka }
84 1.2 ad mutex_enter(&uvm_pageqlock);
85 1.1 pooka uvm_page_unbusy(pgs, npages);
86 1.2 ad mutex_exit(&uvm_pageqlock);
87 1.1 pooka }
88 1.1 pooka
89 1.38 chs static void
90 1.38 chs genfs_markdirty(struct vnode *vp)
91 1.38 chs {
92 1.38 chs struct genfs_node * const gp = VTOG(vp);
93 1.38 chs
94 1.38 chs KASSERT(mutex_owned(&vp->v_interlock));
95 1.38 chs gp->g_dirtygen++;
96 1.38 chs if ((vp->v_iflag & VI_ONWORKLST) == 0) {
97 1.38 chs vn_syncer_add_to_worklist(vp, filedelay);
98 1.38 chs }
99 1.38 chs if ((vp->v_iflag & (VI_WRMAP|VI_WRMAPDIRTY)) == VI_WRMAP) {
100 1.38 chs vp->v_iflag |= VI_WRMAPDIRTY;
101 1.38 chs }
102 1.38 chs }
103 1.38 chs
104 1.1 pooka /*
105 1.1 pooka * generic VM getpages routine.
106 1.1 pooka * Return PG_BUSY pages for the given range,
107 1.1 pooka * reading from backing store if necessary.
108 1.1 pooka */
109 1.1 pooka
110 1.1 pooka int
111 1.1 pooka genfs_getpages(void *v)
112 1.1 pooka {
113 1.1 pooka struct vop_getpages_args /* {
114 1.1 pooka struct vnode *a_vp;
115 1.1 pooka voff_t a_offset;
116 1.1 pooka struct vm_page **a_m;
117 1.1 pooka int *a_count;
118 1.1 pooka int a_centeridx;
119 1.1 pooka vm_prot_t a_access_type;
120 1.1 pooka int a_advice;
121 1.1 pooka int a_flags;
122 1.22 uebayasi } */ * const ap = v;
123 1.1 pooka
124 1.24 uebayasi off_t diskeof, memeof;
125 1.31 uebayasi int i, error, npages;
126 1.10 yamt const int flags = ap->a_flags;
127 1.22 uebayasi struct vnode * const vp = ap->a_vp;
128 1.22 uebayasi struct genfs_node * const gp = VTOG(vp);
129 1.22 uebayasi struct uvm_object * const uobj = &vp->v_uobj;
130 1.31 uebayasi kauth_cred_t const cred = curlwp->l_cred; /* XXXUBC curlwp */
131 1.10 yamt const bool async = (flags & PGO_SYNCIO) == 0;
132 1.35 uebayasi const bool memwrite = (ap->a_access_type & VM_PROT_WRITE) != 0;
133 1.1 pooka bool has_trans = false;
134 1.10 yamt const bool overwrite = (flags & PGO_OVERWRITE) != 0;
135 1.35 uebayasi const bool blockalloc = memwrite && (flags & PGO_NOBLOCKALLOC) == 0;
136 1.1 pooka UVMHIST_FUNC("genfs_getpages"); UVMHIST_CALLED(ubchist);
137 1.1 pooka
138 1.1 pooka UVMHIST_LOG(ubchist, "vp %p off 0x%x/%x count %d",
139 1.1 pooka vp, ap->a_offset >> 32, ap->a_offset, *ap->a_count);
140 1.1 pooka
141 1.1 pooka KASSERT(vp->v_type == VREG || vp->v_type == VDIR ||
142 1.1 pooka vp->v_type == VLNK || vp->v_type == VBLK);
143 1.1 pooka
144 1.1 pooka startover:
145 1.1 pooka error = 0;
146 1.27 uebayasi const voff_t origvsize = vp->v_size;
147 1.27 uebayasi const off_t origoffset = ap->a_offset;
148 1.29 uebayasi const int orignpages = *ap->a_count;
149 1.33 uebayasi
150 1.1 pooka GOP_SIZE(vp, origvsize, &diskeof, 0);
151 1.1 pooka if (flags & PGO_PASTEOF) {
152 1.24 uebayasi off_t newsize;
153 1.1 pooka #if defined(DIAGNOSTIC)
154 1.1 pooka off_t writeeof;
155 1.1 pooka #endif /* defined(DIAGNOSTIC) */
156 1.1 pooka
157 1.1 pooka newsize = MAX(origvsize,
158 1.1 pooka origoffset + (orignpages << PAGE_SHIFT));
159 1.1 pooka GOP_SIZE(vp, newsize, &memeof, GOP_SIZE_MEM);
160 1.1 pooka #if defined(DIAGNOSTIC)
161 1.1 pooka GOP_SIZE(vp, vp->v_writesize, &writeeof, GOP_SIZE_MEM);
162 1.1 pooka if (newsize > round_page(writeeof)) {
163 1.39 pooka panic("%s: past eof: %" PRId64 " vs. %" PRId64,
164 1.39 pooka __func__, newsize, round_page(writeeof));
165 1.1 pooka }
166 1.1 pooka #endif /* defined(DIAGNOSTIC) */
167 1.1 pooka } else {
168 1.1 pooka GOP_SIZE(vp, origvsize, &memeof, GOP_SIZE_MEM);
169 1.1 pooka }
170 1.1 pooka KASSERT(ap->a_centeridx >= 0 || ap->a_centeridx <= orignpages);
171 1.1 pooka KASSERT((origoffset & (PAGE_SIZE - 1)) == 0 && origoffset >= 0);
172 1.1 pooka KASSERT(orignpages > 0);
173 1.1 pooka
174 1.1 pooka /*
175 1.1 pooka * Bounds-check the request.
176 1.1 pooka */
177 1.1 pooka
178 1.1 pooka if (origoffset + (ap->a_centeridx << PAGE_SHIFT) >= memeof) {
179 1.1 pooka if ((flags & PGO_LOCKED) == 0) {
180 1.2 ad mutex_exit(&uobj->vmobjlock);
181 1.1 pooka }
182 1.1 pooka UVMHIST_LOG(ubchist, "off 0x%x count %d goes past EOF 0x%x",
183 1.1 pooka origoffset, *ap->a_count, memeof,0);
184 1.1 pooka error = EINVAL;
185 1.1 pooka goto out_err;
186 1.1 pooka }
187 1.1 pooka
188 1.1 pooka /* uobj is locked */
189 1.1 pooka
190 1.1 pooka if ((flags & PGO_NOTIMESTAMP) == 0 &&
191 1.1 pooka (vp->v_type != VBLK ||
192 1.1 pooka (vp->v_mount->mnt_flag & MNT_NODEVMTIME) == 0)) {
193 1.1 pooka int updflags = 0;
194 1.1 pooka
195 1.1 pooka if ((vp->v_mount->mnt_flag & MNT_NOATIME) == 0) {
196 1.1 pooka updflags = GOP_UPDATE_ACCESSED;
197 1.1 pooka }
198 1.35 uebayasi if (memwrite) {
199 1.1 pooka updflags |= GOP_UPDATE_MODIFIED;
200 1.1 pooka }
201 1.1 pooka if (updflags != 0) {
202 1.1 pooka GOP_MARKUPDATE(vp, updflags);
203 1.1 pooka }
204 1.1 pooka }
205 1.1 pooka
206 1.1 pooka /*
207 1.1 pooka * For PGO_LOCKED requests, just return whatever's in memory.
208 1.1 pooka */
209 1.1 pooka
210 1.1 pooka if (flags & PGO_LOCKED) {
211 1.1 pooka int nfound;
212 1.31 uebayasi struct vm_page *pg;
213 1.1 pooka
214 1.1 pooka npages = *ap->a_count;
215 1.1 pooka #if defined(DEBUG)
216 1.1 pooka for (i = 0; i < npages; i++) {
217 1.1 pooka pg = ap->a_m[i];
218 1.1 pooka KASSERT(pg == NULL || pg == PGO_DONTCARE);
219 1.1 pooka }
220 1.1 pooka #endif /* defined(DEBUG) */
221 1.1 pooka nfound = uvn_findpages(uobj, origoffset, &npages,
222 1.35 uebayasi ap->a_m, UFP_NOWAIT|UFP_NOALLOC|(memwrite ? UFP_NORDONLY : 0));
223 1.1 pooka KASSERT(npages == *ap->a_count);
224 1.1 pooka if (nfound == 0) {
225 1.1 pooka error = EBUSY;
226 1.1 pooka goto out_err;
227 1.1 pooka }
228 1.23 uebayasi if (!genfs_node_rdtrylock(vp)) {
229 1.1 pooka genfs_rel_pages(ap->a_m, npages);
230 1.1 pooka
231 1.1 pooka /*
232 1.1 pooka * restore the array.
233 1.1 pooka */
234 1.1 pooka
235 1.1 pooka for (i = 0; i < npages; i++) {
236 1.1 pooka pg = ap->a_m[i];
237 1.1 pooka
238 1.1 pooka if (pg != NULL || pg != PGO_DONTCARE) {
239 1.1 pooka ap->a_m[i] = NULL;
240 1.1 pooka }
241 1.1 pooka }
242 1.1 pooka } else {
243 1.23 uebayasi genfs_node_unlock(vp);
244 1.1 pooka }
245 1.1 pooka error = (ap->a_m[ap->a_centeridx] == NULL ? EBUSY : 0);
246 1.38 chs if (error == 0 && memwrite) {
247 1.38 chs genfs_markdirty(vp);
248 1.38 chs }
249 1.1 pooka goto out_err;
250 1.1 pooka }
251 1.2 ad mutex_exit(&uobj->vmobjlock);
252 1.1 pooka
253 1.1 pooka /*
254 1.1 pooka * find the requested pages and make some simple checks.
255 1.1 pooka * leave space in the page array for a whole block.
256 1.1 pooka */
257 1.1 pooka
258 1.27 uebayasi const int fs_bshift = (vp->v_type != VBLK) ?
259 1.27 uebayasi vp->v_mount->mnt_fs_bshift : DEV_BSHIFT;
260 1.27 uebayasi const int dev_bshift = (vp->v_type != VBLK) ?
261 1.27 uebayasi vp->v_mount->mnt_dev_bshift : DEV_BSHIFT;
262 1.27 uebayasi const int fs_bsize = 1 << fs_bshift;
263 1.30 uebayasi #define blk_mask (fs_bsize - 1)
264 1.30 uebayasi #define trunc_blk(x) ((x) & ~blk_mask)
265 1.30 uebayasi #define round_blk(x) (((x) + blk_mask) & ~blk_mask)
266 1.1 pooka
267 1.29 uebayasi const int orignmempages = MIN(orignpages,
268 1.1 pooka round_page(memeof - origoffset) >> PAGE_SHIFT);
269 1.29 uebayasi npages = orignmempages;
270 1.30 uebayasi const off_t startoffset = trunc_blk(origoffset);
271 1.30 uebayasi const off_t endoffset = MIN(
272 1.30 uebayasi round_page(round_blk(origoffset + (npages << PAGE_SHIFT))),
273 1.30 uebayasi round_page(memeof));
274 1.31 uebayasi const int ridx = (origoffset - startoffset) >> PAGE_SHIFT;
275 1.1 pooka
276 1.33 uebayasi const int pgs_size = sizeof(struct vm_page *) *
277 1.1 pooka ((endoffset - startoffset) >> PAGE_SHIFT);
278 1.33 uebayasi struct vm_page **pgs, *pgs_onstack[UBC_MAX_PAGES];
279 1.31 uebayasi
280 1.1 pooka if (pgs_size > sizeof(pgs_onstack)) {
281 1.1 pooka pgs = kmem_zalloc(pgs_size, async ? KM_NOSLEEP : KM_SLEEP);
282 1.1 pooka if (pgs == NULL) {
283 1.1 pooka pgs = pgs_onstack;
284 1.1 pooka error = ENOMEM;
285 1.32 uebayasi goto out_err;
286 1.1 pooka }
287 1.1 pooka } else {
288 1.14 christos pgs = pgs_onstack;
289 1.14 christos (void)memset(pgs, 0, pgs_size);
290 1.1 pooka }
291 1.14 christos
292 1.1 pooka UVMHIST_LOG(ubchist, "ridx %d npages %d startoff %ld endoff %ld",
293 1.1 pooka ridx, npages, startoffset, endoffset);
294 1.1 pooka
295 1.1 pooka if (!has_trans) {
296 1.1 pooka fstrans_start(vp->v_mount, FSTRANS_SHARED);
297 1.1 pooka has_trans = true;
298 1.1 pooka }
299 1.1 pooka
300 1.1 pooka /*
301 1.1 pooka * hold g_glock to prevent a race with truncate.
302 1.1 pooka *
303 1.1 pooka * check if our idea of v_size is still valid.
304 1.1 pooka */
305 1.1 pooka
306 1.1 pooka if (blockalloc) {
307 1.1 pooka rw_enter(&gp->g_glock, RW_WRITER);
308 1.1 pooka } else {
309 1.1 pooka rw_enter(&gp->g_glock, RW_READER);
310 1.1 pooka }
311 1.2 ad mutex_enter(&uobj->vmobjlock);
312 1.1 pooka if (vp->v_size < origvsize) {
313 1.23 uebayasi genfs_node_unlock(vp);
314 1.1 pooka if (pgs != pgs_onstack)
315 1.1 pooka kmem_free(pgs, pgs_size);
316 1.1 pooka goto startover;
317 1.1 pooka }
318 1.1 pooka
319 1.1 pooka if (uvn_findpages(uobj, origoffset, &npages, &pgs[ridx],
320 1.29 uebayasi async ? UFP_NOWAIT : UFP_ALL) != orignmempages) {
321 1.23 uebayasi genfs_node_unlock(vp);
322 1.1 pooka KASSERT(async != 0);
323 1.29 uebayasi genfs_rel_pages(&pgs[ridx], orignmempages);
324 1.2 ad mutex_exit(&uobj->vmobjlock);
325 1.1 pooka error = EBUSY;
326 1.33 uebayasi goto out_err_free;
327 1.1 pooka }
328 1.1 pooka
329 1.1 pooka /*
330 1.1 pooka * if the pages are already resident, just return them.
331 1.1 pooka */
332 1.1 pooka
333 1.1 pooka for (i = 0; i < npages; i++) {
334 1.31 uebayasi struct vm_page *pg = pgs[ridx + i];
335 1.1 pooka
336 1.31 uebayasi if ((pg->flags & PG_FAKE) ||
337 1.31 uebayasi (blockalloc && (pg->flags & PG_RDONLY))) {
338 1.1 pooka break;
339 1.1 pooka }
340 1.1 pooka }
341 1.1 pooka if (i == npages) {
342 1.23 uebayasi genfs_node_unlock(vp);
343 1.1 pooka UVMHIST_LOG(ubchist, "returning cached pages", 0,0,0,0);
344 1.1 pooka npages += ridx;
345 1.1 pooka goto out;
346 1.1 pooka }
347 1.1 pooka
348 1.1 pooka /*
349 1.1 pooka * if PGO_OVERWRITE is set, don't bother reading the pages.
350 1.1 pooka */
351 1.1 pooka
352 1.1 pooka if (overwrite) {
353 1.23 uebayasi genfs_node_unlock(vp);
354 1.1 pooka UVMHIST_LOG(ubchist, "PGO_OVERWRITE",0,0,0,0);
355 1.1 pooka
356 1.1 pooka for (i = 0; i < npages; i++) {
357 1.31 uebayasi struct vm_page *pg = pgs[ridx + i];
358 1.1 pooka
359 1.31 uebayasi pg->flags &= ~(PG_RDONLY|PG_CLEAN);
360 1.1 pooka }
361 1.1 pooka npages += ridx;
362 1.1 pooka goto out;
363 1.1 pooka }
364 1.1 pooka
365 1.1 pooka /*
366 1.1 pooka * the page wasn't resident and we're not overwriting,
367 1.1 pooka * so we're going to have to do some i/o.
368 1.1 pooka * find any additional pages needed to cover the expanded range.
369 1.1 pooka */
370 1.1 pooka
371 1.1 pooka npages = (endoffset - startoffset) >> PAGE_SHIFT;
372 1.29 uebayasi if (startoffset != origoffset || npages != orignmempages) {
373 1.31 uebayasi int npgs;
374 1.1 pooka
375 1.1 pooka /*
376 1.1 pooka * we need to avoid deadlocks caused by locking
377 1.1 pooka * additional pages at lower offsets than pages we
378 1.1 pooka * already have locked. unlock them all and start over.
379 1.1 pooka */
380 1.1 pooka
381 1.29 uebayasi genfs_rel_pages(&pgs[ridx], orignmempages);
382 1.1 pooka memset(pgs, 0, pgs_size);
383 1.1 pooka
384 1.1 pooka UVMHIST_LOG(ubchist, "reset npages start 0x%x end 0x%x",
385 1.1 pooka startoffset, endoffset, 0,0);
386 1.1 pooka npgs = npages;
387 1.1 pooka if (uvn_findpages(uobj, startoffset, &npgs, pgs,
388 1.1 pooka async ? UFP_NOWAIT : UFP_ALL) != npages) {
389 1.23 uebayasi genfs_node_unlock(vp);
390 1.1 pooka KASSERT(async != 0);
391 1.1 pooka genfs_rel_pages(pgs, npages);
392 1.2 ad mutex_exit(&uobj->vmobjlock);
393 1.1 pooka error = EBUSY;
394 1.33 uebayasi goto out_err_free;
395 1.1 pooka }
396 1.1 pooka }
397 1.34 uebayasi
398 1.2 ad mutex_exit(&uobj->vmobjlock);
399 1.1 pooka
400 1.34 uebayasi {
401 1.34 uebayasi size_t bytes, iobytes, tailstart, tailbytes, totalbytes, skipbytes;
402 1.34 uebayasi vaddr_t kva;
403 1.34 uebayasi struct buf *bp, *mbp;
404 1.34 uebayasi bool sawhole = false;
405 1.34 uebayasi
406 1.1 pooka /*
407 1.1 pooka * read the desired page(s).
408 1.1 pooka */
409 1.1 pooka
410 1.1 pooka totalbytes = npages << PAGE_SHIFT;
411 1.1 pooka bytes = MIN(totalbytes, MAX(diskeof - startoffset, 0));
412 1.1 pooka tailbytes = totalbytes - bytes;
413 1.1 pooka skipbytes = 0;
414 1.1 pooka
415 1.1 pooka kva = uvm_pagermapin(pgs, npages,
416 1.1 pooka UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK);
417 1.1 pooka
418 1.2 ad mbp = getiobuf(vp, true);
419 1.1 pooka mbp->b_bufsize = totalbytes;
420 1.1 pooka mbp->b_data = (void *)kva;
421 1.1 pooka mbp->b_resid = mbp->b_bcount = bytes;
422 1.2 ad mbp->b_cflags = BC_BUSY;
423 1.2 ad if (async) {
424 1.2 ad mbp->b_flags = B_READ | B_ASYNC;
425 1.2 ad mbp->b_iodone = uvm_aio_biodone;
426 1.2 ad } else {
427 1.2 ad mbp->b_flags = B_READ;
428 1.2 ad mbp->b_iodone = NULL;
429 1.2 ad }
430 1.1 pooka if (async)
431 1.1 pooka BIO_SETPRIO(mbp, BPRIO_TIMELIMITED);
432 1.1 pooka else
433 1.1 pooka BIO_SETPRIO(mbp, BPRIO_TIMECRITICAL);
434 1.1 pooka
435 1.1 pooka /*
436 1.1 pooka * if EOF is in the middle of the range, zero the part past EOF.
437 1.1 pooka * skip over pages which are not PG_FAKE since in that case they have
438 1.1 pooka * valid data that we need to preserve.
439 1.1 pooka */
440 1.1 pooka
441 1.1 pooka tailstart = bytes;
442 1.1 pooka while (tailbytes > 0) {
443 1.1 pooka const int len = PAGE_SIZE - (tailstart & PAGE_MASK);
444 1.1 pooka
445 1.1 pooka KASSERT(len <= tailbytes);
446 1.1 pooka if ((pgs[tailstart >> PAGE_SHIFT]->flags & PG_FAKE) != 0) {
447 1.1 pooka memset((void *)(kva + tailstart), 0, len);
448 1.1 pooka UVMHIST_LOG(ubchist, "tailbytes %p 0x%x 0x%x",
449 1.1 pooka kva, tailstart, len, 0);
450 1.1 pooka }
451 1.1 pooka tailstart += len;
452 1.1 pooka tailbytes -= len;
453 1.1 pooka }
454 1.1 pooka
455 1.1 pooka /*
456 1.1 pooka * now loop over the pages, reading as needed.
457 1.1 pooka */
458 1.1 pooka
459 1.1 pooka bp = NULL;
460 1.28 uebayasi off_t offset;
461 1.28 uebayasi for (offset = startoffset;
462 1.1 pooka bytes > 0;
463 1.1 pooka offset += iobytes, bytes -= iobytes) {
464 1.30 uebayasi int run;
465 1.25 uebayasi daddr_t lbn, blkno;
466 1.24 uebayasi int pidx;
467 1.26 uebayasi struct vnode *devvp;
468 1.1 pooka
469 1.1 pooka /*
470 1.1 pooka * skip pages which don't need to be read.
471 1.1 pooka */
472 1.1 pooka
473 1.1 pooka pidx = (offset - startoffset) >> PAGE_SHIFT;
474 1.1 pooka while ((pgs[pidx]->flags & PG_FAKE) == 0) {
475 1.1 pooka size_t b;
476 1.1 pooka
477 1.1 pooka KASSERT((offset & (PAGE_SIZE - 1)) == 0);
478 1.1 pooka if ((pgs[pidx]->flags & PG_RDONLY)) {
479 1.1 pooka sawhole = true;
480 1.1 pooka }
481 1.1 pooka b = MIN(PAGE_SIZE, bytes);
482 1.1 pooka offset += b;
483 1.1 pooka bytes -= b;
484 1.1 pooka skipbytes += b;
485 1.1 pooka pidx++;
486 1.1 pooka UVMHIST_LOG(ubchist, "skipping, new offset 0x%x",
487 1.1 pooka offset, 0,0,0);
488 1.1 pooka if (bytes == 0) {
489 1.1 pooka goto loopdone;
490 1.1 pooka }
491 1.1 pooka }
492 1.1 pooka
493 1.1 pooka /*
494 1.1 pooka * bmap the file to find out the blkno to read from and
495 1.1 pooka * how much we can read in one i/o. if bmap returns an error,
496 1.1 pooka * skip the rest of the top-level i/o.
497 1.1 pooka */
498 1.1 pooka
499 1.1 pooka lbn = offset >> fs_bshift;
500 1.1 pooka error = VOP_BMAP(vp, lbn, &devvp, &blkno, &run);
501 1.1 pooka if (error) {
502 1.1 pooka UVMHIST_LOG(ubchist, "VOP_BMAP lbn 0x%x -> %d\n",
503 1.36 uebayasi lbn,error,0,0);
504 1.1 pooka skipbytes += bytes;
505 1.36 uebayasi bytes = 0;
506 1.1 pooka goto loopdone;
507 1.1 pooka }
508 1.1 pooka
509 1.1 pooka /*
510 1.1 pooka * see how many pages can be read with this i/o.
511 1.1 pooka * reduce the i/o size if necessary to avoid
512 1.1 pooka * overwriting pages with valid data.
513 1.1 pooka */
514 1.1 pooka
515 1.1 pooka iobytes = MIN((((off_t)lbn + 1 + run) << fs_bshift) - offset,
516 1.1 pooka bytes);
517 1.1 pooka if (offset + iobytes > round_page(offset)) {
518 1.24 uebayasi int pcount;
519 1.24 uebayasi
520 1.1 pooka pcount = 1;
521 1.1 pooka while (pidx + pcount < npages &&
522 1.1 pooka pgs[pidx + pcount]->flags & PG_FAKE) {
523 1.1 pooka pcount++;
524 1.1 pooka }
525 1.1 pooka iobytes = MIN(iobytes, (pcount << PAGE_SHIFT) -
526 1.1 pooka (offset - trunc_page(offset)));
527 1.1 pooka }
528 1.1 pooka
529 1.1 pooka /*
530 1.1 pooka * if this block isn't allocated, zero it instead of
531 1.1 pooka * reading it. unless we are going to allocate blocks,
532 1.1 pooka * mark the pages we zeroed PG_RDONLY.
533 1.1 pooka */
534 1.1 pooka
535 1.36 uebayasi if (blkno == (daddr_t)-1) {
536 1.1 pooka int holepages = (round_page(offset + iobytes) -
537 1.1 pooka trunc_page(offset)) >> PAGE_SHIFT;
538 1.1 pooka UVMHIST_LOG(ubchist, "lbn 0x%x -> HOLE", lbn,0,0,0);
539 1.1 pooka
540 1.1 pooka sawhole = true;
541 1.1 pooka memset((char *)kva + (offset - startoffset), 0,
542 1.1 pooka iobytes);
543 1.1 pooka skipbytes += iobytes;
544 1.1 pooka
545 1.1 pooka for (i = 0; i < holepages; i++) {
546 1.35 uebayasi if (memwrite) {
547 1.1 pooka pgs[pidx + i]->flags &= ~PG_CLEAN;
548 1.1 pooka }
549 1.1 pooka if (!blockalloc) {
550 1.1 pooka pgs[pidx + i]->flags |= PG_RDONLY;
551 1.1 pooka }
552 1.1 pooka }
553 1.1 pooka continue;
554 1.1 pooka }
555 1.1 pooka
556 1.1 pooka /*
557 1.1 pooka * allocate a sub-buf for this piece of the i/o
558 1.1 pooka * (or just use mbp if there's only 1 piece),
559 1.1 pooka * and start it going.
560 1.1 pooka */
561 1.1 pooka
562 1.1 pooka if (offset == startoffset && iobytes == bytes) {
563 1.1 pooka bp = mbp;
564 1.1 pooka } else {
565 1.36 uebayasi UVMHIST_LOG(ubchist, "vp %p bp %p num now %d",
566 1.36 uebayasi vp, bp, vp->v_numoutput, 0);
567 1.2 ad bp = getiobuf(vp, true);
568 1.1 pooka nestiobuf_setup(mbp, bp, offset - startoffset, iobytes);
569 1.1 pooka }
570 1.1 pooka bp->b_lblkno = 0;
571 1.1 pooka
572 1.1 pooka /* adjust physical blkno for partial blocks */
573 1.1 pooka bp->b_blkno = blkno + ((offset - ((off_t)lbn << fs_bshift)) >>
574 1.1 pooka dev_bshift);
575 1.1 pooka
576 1.1 pooka UVMHIST_LOG(ubchist,
577 1.1 pooka "bp %p offset 0x%x bcount 0x%x blkno 0x%x",
578 1.36 uebayasi bp, offset, bp->b_bcount, bp->b_blkno);
579 1.1 pooka
580 1.1 pooka VOP_STRATEGY(devvp, bp);
581 1.1 pooka }
582 1.1 pooka
583 1.1 pooka loopdone:
584 1.1 pooka nestiobuf_done(mbp, skipbytes, error);
585 1.1 pooka if (async) {
586 1.1 pooka UVMHIST_LOG(ubchist, "returning 0 (async)",0,0,0,0);
587 1.23 uebayasi genfs_node_unlock(vp);
588 1.1 pooka error = 0;
589 1.33 uebayasi goto out_err_free;
590 1.1 pooka }
591 1.1 pooka if (bp != NULL) {
592 1.1 pooka error = biowait(mbp);
593 1.1 pooka }
594 1.1 pooka
595 1.19 rmind /* Remove the mapping (make KVA available as soon as possible) */
596 1.19 rmind uvm_pagermapout(kva, npages);
597 1.19 rmind
598 1.1 pooka /*
599 1.1 pooka * if this we encountered a hole then we have to do a little more work.
600 1.1 pooka * for read faults, we marked the page PG_RDONLY so that future
601 1.1 pooka * write accesses to the page will fault again.
602 1.1 pooka * for write faults, we must make sure that the backing store for
603 1.1 pooka * the page is completely allocated while the pages are locked.
604 1.1 pooka */
605 1.1 pooka
606 1.1 pooka if (!error && sawhole && blockalloc) {
607 1.9 simonb /*
608 1.9 simonb * XXX: This assumes that we come here only via
609 1.9 simonb * the mmio path
610 1.9 simonb */
611 1.11 yamt if (vp->v_mount->mnt_wapbl) {
612 1.9 simonb error = WAPBL_BEGIN(vp->v_mount);
613 1.9 simonb }
614 1.9 simonb
615 1.9 simonb if (!error) {
616 1.9 simonb error = GOP_ALLOC(vp, startoffset,
617 1.9 simonb npages << PAGE_SHIFT, 0, cred);
618 1.11 yamt if (vp->v_mount->mnt_wapbl) {
619 1.9 simonb WAPBL_END(vp->v_mount);
620 1.9 simonb }
621 1.9 simonb }
622 1.9 simonb
623 1.1 pooka UVMHIST_LOG(ubchist, "gop_alloc off 0x%x/0x%x -> %d",
624 1.1 pooka startoffset, npages << PAGE_SHIFT, error,0);
625 1.1 pooka if (!error) {
626 1.1 pooka for (i = 0; i < npages; i++) {
627 1.31 uebayasi struct vm_page *pg = pgs[i];
628 1.31 uebayasi
629 1.31 uebayasi if (pg == NULL) {
630 1.1 pooka continue;
631 1.1 pooka }
632 1.31 uebayasi pg->flags &= ~(PG_CLEAN|PG_RDONLY);
633 1.1 pooka UVMHIST_LOG(ubchist, "mark dirty pg %p",
634 1.31 uebayasi pg,0,0,0);
635 1.1 pooka }
636 1.1 pooka }
637 1.1 pooka }
638 1.23 uebayasi genfs_node_unlock(vp);
639 1.18 rmind
640 1.18 rmind putiobuf(mbp);
641 1.34 uebayasi }
642 1.18 rmind
643 1.2 ad mutex_enter(&uobj->vmobjlock);
644 1.1 pooka
645 1.1 pooka /*
646 1.1 pooka * we're almost done! release the pages...
647 1.1 pooka * for errors, we free the pages.
648 1.1 pooka * otherwise we activate them and mark them as valid and clean.
649 1.1 pooka * also, unbusy pages that were not actually requested.
650 1.1 pooka */
651 1.1 pooka
652 1.1 pooka if (error) {
653 1.1 pooka for (i = 0; i < npages; i++) {
654 1.31 uebayasi struct vm_page *pg = pgs[i];
655 1.31 uebayasi
656 1.31 uebayasi if (pg == NULL) {
657 1.1 pooka continue;
658 1.1 pooka }
659 1.1 pooka UVMHIST_LOG(ubchist, "examining pg %p flags 0x%x",
660 1.31 uebayasi pg, pg->flags, 0,0);
661 1.31 uebayasi if (pg->flags & PG_FAKE) {
662 1.31 uebayasi pg->flags |= PG_RELEASED;
663 1.1 pooka }
664 1.1 pooka }
665 1.2 ad mutex_enter(&uvm_pageqlock);
666 1.1 pooka uvm_page_unbusy(pgs, npages);
667 1.2 ad mutex_exit(&uvm_pageqlock);
668 1.2 ad mutex_exit(&uobj->vmobjlock);
669 1.1 pooka UVMHIST_LOG(ubchist, "returning error %d", error,0,0,0);
670 1.33 uebayasi goto out_err_free;
671 1.1 pooka }
672 1.1 pooka
673 1.1 pooka out:
674 1.1 pooka UVMHIST_LOG(ubchist, "succeeding, npages %d", npages,0,0,0);
675 1.1 pooka error = 0;
676 1.2 ad mutex_enter(&uvm_pageqlock);
677 1.1 pooka for (i = 0; i < npages; i++) {
678 1.31 uebayasi struct vm_page *pg = pgs[i];
679 1.1 pooka if (pg == NULL) {
680 1.1 pooka continue;
681 1.1 pooka }
682 1.1 pooka UVMHIST_LOG(ubchist, "examining pg %p flags 0x%x",
683 1.1 pooka pg, pg->flags, 0,0);
684 1.1 pooka if (pg->flags & PG_FAKE && !overwrite) {
685 1.1 pooka pg->flags &= ~(PG_FAKE);
686 1.1 pooka pmap_clear_modify(pgs[i]);
687 1.1 pooka }
688 1.35 uebayasi KASSERT(!memwrite || !blockalloc || (pg->flags & PG_RDONLY) == 0);
689 1.29 uebayasi if (i < ridx || i >= ridx + orignmempages || async) {
690 1.1 pooka UVMHIST_LOG(ubchist, "unbusy pg %p offset 0x%x",
691 1.1 pooka pg, pg->offset,0,0);
692 1.1 pooka if (pg->flags & PG_WANTED) {
693 1.1 pooka wakeup(pg);
694 1.1 pooka }
695 1.1 pooka if (pg->flags & PG_FAKE) {
696 1.1 pooka KASSERT(overwrite);
697 1.1 pooka uvm_pagezero(pg);
698 1.1 pooka }
699 1.1 pooka if (pg->flags & PG_RELEASED) {
700 1.1 pooka uvm_pagefree(pg);
701 1.1 pooka continue;
702 1.1 pooka }
703 1.1 pooka uvm_pageenqueue(pg);
704 1.1 pooka pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE);
705 1.1 pooka UVM_PAGE_OWN(pg, NULL);
706 1.1 pooka }
707 1.1 pooka }
708 1.2 ad mutex_exit(&uvm_pageqlock);
709 1.38 chs if (memwrite) {
710 1.38 chs genfs_markdirty(vp);
711 1.38 chs }
712 1.2 ad mutex_exit(&uobj->vmobjlock);
713 1.1 pooka if (ap->a_m != NULL) {
714 1.1 pooka memcpy(ap->a_m, &pgs[ridx],
715 1.29 uebayasi orignmempages * sizeof(struct vm_page *));
716 1.1 pooka }
717 1.1 pooka
718 1.33 uebayasi out_err_free:
719 1.14 christos if (pgs != NULL && pgs != pgs_onstack)
720 1.1 pooka kmem_free(pgs, pgs_size);
721 1.33 uebayasi out_err:
722 1.1 pooka if (has_trans)
723 1.1 pooka fstrans_done(vp->v_mount);
724 1.38 chs return error;
725 1.1 pooka }
726 1.1 pooka
727 1.1 pooka /*
728 1.1 pooka * generic VM putpages routine.
729 1.1 pooka * Write the given range of pages to backing store.
730 1.1 pooka *
731 1.1 pooka * => "offhi == 0" means flush all pages at or after "offlo".
732 1.1 pooka * => object should be locked by caller. we return with the
733 1.1 pooka * object unlocked.
734 1.1 pooka * => if PGO_CLEANIT or PGO_SYNCIO is set, we may block (due to I/O).
735 1.1 pooka * thus, a caller might want to unlock higher level resources
736 1.1 pooka * (e.g. vm_map) before calling flush.
737 1.1 pooka * => if neither PGO_CLEANIT nor PGO_SYNCIO is set, we will not block
738 1.1 pooka * => if PGO_ALLPAGES is set, then all pages in the object will be processed.
739 1.1 pooka * => NOTE: we rely on the fact that the object's memq is a TAILQ and
740 1.1 pooka * that new pages are inserted on the tail end of the list. thus,
741 1.1 pooka * we can make a complete pass through the object in one go by starting
742 1.1 pooka * at the head and working towards the tail (new pages are put in
743 1.1 pooka * front of us).
744 1.1 pooka * => NOTE: we are allowed to lock the page queues, so the caller
745 1.1 pooka * must not be holding the page queue lock.
746 1.1 pooka *
747 1.1 pooka * note on "cleaning" object and PG_BUSY pages:
748 1.1 pooka * this routine is holding the lock on the object. the only time
749 1.1 pooka * that it can run into a PG_BUSY page that it does not own is if
750 1.1 pooka * some other process has started I/O on the page (e.g. either
751 1.1 pooka * a pagein, or a pageout). if the PG_BUSY page is being paged
752 1.1 pooka * in, then it can not be dirty (!PG_CLEAN) because no one has
753 1.1 pooka * had a chance to modify it yet. if the PG_BUSY page is being
754 1.1 pooka * paged out then it means that someone else has already started
755 1.1 pooka * cleaning the page for us (how nice!). in this case, if we
756 1.1 pooka * have syncio specified, then after we make our pass through the
757 1.1 pooka * object we need to wait for the other PG_BUSY pages to clear
758 1.1 pooka * off (i.e. we need to do an iosync). also note that once a
759 1.1 pooka * page is PG_BUSY it must stay in its object until it is un-busyed.
760 1.1 pooka *
761 1.1 pooka * note on page traversal:
762 1.1 pooka * we can traverse the pages in an object either by going down the
763 1.1 pooka * linked list in "uobj->memq", or we can go over the address range
764 1.1 pooka * by page doing hash table lookups for each address. depending
765 1.1 pooka * on how many pages are in the object it may be cheaper to do one
766 1.1 pooka * or the other. we set "by_list" to true if we are using memq.
767 1.1 pooka * if the cost of a hash lookup was equal to the cost of the list
768 1.1 pooka * traversal we could compare the number of pages in the start->stop
769 1.1 pooka * range to the total number of pages in the object. however, it
770 1.1 pooka * seems that a hash table lookup is more expensive than the linked
771 1.1 pooka * list traversal, so we multiply the number of pages in the
772 1.1 pooka * range by an estimate of the relatively higher cost of the hash lookup.
773 1.1 pooka */
774 1.1 pooka
775 1.1 pooka int
776 1.1 pooka genfs_putpages(void *v)
777 1.1 pooka {
778 1.1 pooka struct vop_putpages_args /* {
779 1.1 pooka struct vnode *a_vp;
780 1.1 pooka voff_t a_offlo;
781 1.1 pooka voff_t a_offhi;
782 1.1 pooka int a_flags;
783 1.22 uebayasi } */ * const ap = v;
784 1.1 pooka
785 1.1 pooka return genfs_do_putpages(ap->a_vp, ap->a_offlo, ap->a_offhi,
786 1.1 pooka ap->a_flags, NULL);
787 1.1 pooka }
788 1.1 pooka
789 1.1 pooka int
790 1.4 yamt genfs_do_putpages(struct vnode *vp, off_t startoff, off_t endoff,
791 1.4 yamt int origflags, struct vm_page **busypg)
792 1.1 pooka {
793 1.22 uebayasi struct uvm_object * const uobj = &vp->v_uobj;
794 1.22 uebayasi kmutex_t * const slock = &uobj->vmobjlock;
795 1.1 pooka off_t off;
796 1.1 pooka /* Even for strange MAXPHYS, the shift rounds down to a page */
797 1.1 pooka #define maxpages (MAXPHYS >> PAGE_SHIFT)
798 1.2 ad int i, error, npages, nback;
799 1.1 pooka int freeflag;
800 1.1 pooka struct vm_page *pgs[maxpages], *pg, *nextpg, *tpg, curmp, endmp;
801 1.1 pooka bool wasclean, by_list, needs_clean, yld;
802 1.4 yamt bool async = (origflags & PGO_SYNCIO) == 0;
803 1.1 pooka bool pagedaemon = curlwp == uvm.pagedaemon_lwp;
804 1.22 uebayasi struct lwp * const l = curlwp ? curlwp : &lwp0;
805 1.22 uebayasi struct genfs_node * const gp = VTOG(vp);
806 1.4 yamt int flags;
807 1.1 pooka int dirtygen;
808 1.4 yamt bool modified;
809 1.12 hannken bool need_wapbl;
810 1.4 yamt bool has_trans;
811 1.1 pooka bool cleanall;
812 1.4 yamt bool onworklst;
813 1.1 pooka
814 1.1 pooka UVMHIST_FUNC("genfs_putpages"); UVMHIST_CALLED(ubchist);
815 1.1 pooka
816 1.4 yamt KASSERT(origflags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE));
817 1.1 pooka KASSERT((startoff & PAGE_MASK) == 0 && (endoff & PAGE_MASK) == 0);
818 1.1 pooka KASSERT(startoff < endoff || endoff == 0);
819 1.1 pooka
820 1.1 pooka UVMHIST_LOG(ubchist, "vp %p pages %d off 0x%x len 0x%x",
821 1.1 pooka vp, uobj->uo_npages, startoff, endoff - startoff);
822 1.1 pooka
823 1.6 hannken has_trans = false;
824 1.12 hannken need_wapbl = (!pagedaemon && vp->v_mount && vp->v_mount->mnt_wapbl &&
825 1.12 hannken (origflags & PGO_JOURNALLOCKED) == 0);
826 1.6 hannken
827 1.4 yamt retry:
828 1.4 yamt modified = false;
829 1.4 yamt flags = origflags;
830 1.1 pooka KASSERT((vp->v_iflag & VI_ONWORKLST) != 0 ||
831 1.1 pooka (vp->v_iflag & VI_WRMAPDIRTY) == 0);
832 1.1 pooka if (uobj->uo_npages == 0) {
833 1.1 pooka if (vp->v_iflag & VI_ONWORKLST) {
834 1.1 pooka vp->v_iflag &= ~VI_WRMAPDIRTY;
835 1.1 pooka if (LIST_FIRST(&vp->v_dirtyblkhd) == NULL)
836 1.1 pooka vn_syncer_remove_from_worklist(vp);
837 1.1 pooka }
838 1.12 hannken if (has_trans) {
839 1.12 hannken if (need_wapbl)
840 1.12 hannken WAPBL_END(vp->v_mount);
841 1.6 hannken fstrans_done(vp->v_mount);
842 1.12 hannken }
843 1.2 ad mutex_exit(slock);
844 1.1 pooka return (0);
845 1.1 pooka }
846 1.1 pooka
847 1.1 pooka /*
848 1.1 pooka * the vnode has pages, set up to process the request.
849 1.1 pooka */
850 1.1 pooka
851 1.6 hannken if (!has_trans && (flags & PGO_CLEANIT) != 0) {
852 1.2 ad mutex_exit(slock);
853 1.1 pooka if (pagedaemon) {
854 1.1 pooka error = fstrans_start_nowait(vp->v_mount, FSTRANS_LAZY);
855 1.1 pooka if (error)
856 1.1 pooka return error;
857 1.1 pooka } else
858 1.1 pooka fstrans_start(vp->v_mount, FSTRANS_LAZY);
859 1.12 hannken if (need_wapbl) {
860 1.12 hannken error = WAPBL_BEGIN(vp->v_mount);
861 1.12 hannken if (error) {
862 1.12 hannken fstrans_done(vp->v_mount);
863 1.12 hannken return error;
864 1.12 hannken }
865 1.12 hannken }
866 1.1 pooka has_trans = true;
867 1.2 ad mutex_enter(slock);
868 1.6 hannken goto retry;
869 1.1 pooka }
870 1.1 pooka
871 1.1 pooka error = 0;
872 1.1 pooka wasclean = (vp->v_numoutput == 0);
873 1.1 pooka off = startoff;
874 1.1 pooka if (endoff == 0 || flags & PGO_ALLPAGES) {
875 1.1 pooka endoff = trunc_page(LLONG_MAX);
876 1.1 pooka }
877 1.1 pooka by_list = (uobj->uo_npages <=
878 1.17 yamt ((endoff - startoff) >> PAGE_SHIFT) * UVM_PAGE_TREE_PENALTY);
879 1.1 pooka
880 1.1 pooka #if !defined(DEBUG)
881 1.1 pooka /*
882 1.1 pooka * if this vnode is known not to have dirty pages,
883 1.1 pooka * don't bother to clean it out.
884 1.1 pooka */
885 1.1 pooka
886 1.1 pooka if ((vp->v_iflag & VI_ONWORKLST) == 0) {
887 1.1 pooka if ((flags & (PGO_FREE|PGO_DEACTIVATE)) == 0) {
888 1.1 pooka goto skip_scan;
889 1.1 pooka }
890 1.1 pooka flags &= ~PGO_CLEANIT;
891 1.1 pooka }
892 1.1 pooka #endif /* !defined(DEBUG) */
893 1.1 pooka
894 1.1 pooka /*
895 1.1 pooka * start the loop. when scanning by list, hold the last page
896 1.1 pooka * in the list before we start. pages allocated after we start
897 1.1 pooka * will be added to the end of the list, so we can stop at the
898 1.1 pooka * current last page.
899 1.1 pooka */
900 1.1 pooka
901 1.1 pooka cleanall = (flags & PGO_CLEANIT) != 0 && wasclean &&
902 1.1 pooka startoff == 0 && endoff == trunc_page(LLONG_MAX) &&
903 1.1 pooka (vp->v_iflag & VI_ONWORKLST) != 0;
904 1.1 pooka dirtygen = gp->g_dirtygen;
905 1.1 pooka freeflag = pagedaemon ? PG_PAGEOUT : PG_RELEASED;
906 1.1 pooka if (by_list) {
907 1.37 hannken curmp.flags = PG_MARKER;
908 1.37 hannken endmp.flags = PG_MARKER;
909 1.1 pooka pg = TAILQ_FIRST(&uobj->memq);
910 1.8 ad TAILQ_INSERT_TAIL(&uobj->memq, &endmp, listq.queue);
911 1.1 pooka } else {
912 1.1 pooka pg = uvm_pagelookup(uobj, off);
913 1.1 pooka }
914 1.1 pooka nextpg = NULL;
915 1.1 pooka while (by_list || off < endoff) {
916 1.1 pooka
917 1.1 pooka /*
918 1.1 pooka * if the current page is not interesting, move on to the next.
919 1.1 pooka */
920 1.1 pooka
921 1.37 hannken KASSERT(pg == NULL || pg->uobject == uobj ||
922 1.37 hannken (pg->flags & PG_MARKER) != 0);
923 1.1 pooka KASSERT(pg == NULL ||
924 1.1 pooka (pg->flags & (PG_RELEASED|PG_PAGEOUT)) == 0 ||
925 1.37 hannken (pg->flags & (PG_BUSY|PG_MARKER)) != 0);
926 1.1 pooka if (by_list) {
927 1.1 pooka if (pg == &endmp) {
928 1.1 pooka break;
929 1.1 pooka }
930 1.37 hannken if (pg->flags & PG_MARKER) {
931 1.37 hannken pg = TAILQ_NEXT(pg, listq.queue);
932 1.37 hannken continue;
933 1.37 hannken }
934 1.1 pooka if (pg->offset < startoff || pg->offset >= endoff ||
935 1.1 pooka pg->flags & (PG_RELEASED|PG_PAGEOUT)) {
936 1.1 pooka if (pg->flags & (PG_RELEASED|PG_PAGEOUT)) {
937 1.1 pooka wasclean = false;
938 1.1 pooka }
939 1.8 ad pg = TAILQ_NEXT(pg, listq.queue);
940 1.1 pooka continue;
941 1.1 pooka }
942 1.1 pooka off = pg->offset;
943 1.1 pooka } else if (pg == NULL || pg->flags & (PG_RELEASED|PG_PAGEOUT)) {
944 1.1 pooka if (pg != NULL) {
945 1.1 pooka wasclean = false;
946 1.1 pooka }
947 1.1 pooka off += PAGE_SIZE;
948 1.1 pooka if (off < endoff) {
949 1.1 pooka pg = uvm_pagelookup(uobj, off);
950 1.1 pooka }
951 1.1 pooka continue;
952 1.1 pooka }
953 1.1 pooka
954 1.1 pooka /*
955 1.1 pooka * if the current page needs to be cleaned and it's busy,
956 1.1 pooka * wait for it to become unbusy.
957 1.1 pooka */
958 1.1 pooka
959 1.1 pooka yld = (l->l_cpu->ci_schedstate.spc_flags &
960 1.1 pooka SPCF_SHOULDYIELD) && !pagedaemon;
961 1.1 pooka if (pg->flags & PG_BUSY || yld) {
962 1.1 pooka UVMHIST_LOG(ubchist, "busy %p", pg,0,0,0);
963 1.1 pooka if (flags & PGO_BUSYFAIL && pg->flags & PG_BUSY) {
964 1.1 pooka UVMHIST_LOG(ubchist, "busyfail %p", pg, 0,0,0);
965 1.1 pooka error = EDEADLK;
966 1.1 pooka if (busypg != NULL)
967 1.1 pooka *busypg = pg;
968 1.1 pooka break;
969 1.1 pooka }
970 1.1 pooka if (pagedaemon) {
971 1.1 pooka /*
972 1.1 pooka * someone has taken the page while we
973 1.1 pooka * dropped the lock for fstrans_start.
974 1.1 pooka */
975 1.1 pooka break;
976 1.1 pooka }
977 1.1 pooka if (by_list) {
978 1.8 ad TAILQ_INSERT_BEFORE(pg, &curmp, listq.queue);
979 1.1 pooka UVMHIST_LOG(ubchist, "curmp next %p",
980 1.8 ad TAILQ_NEXT(&curmp, listq.queue), 0,0,0);
981 1.1 pooka }
982 1.1 pooka if (yld) {
983 1.2 ad mutex_exit(slock);
984 1.1 pooka preempt();
985 1.2 ad mutex_enter(slock);
986 1.1 pooka } else {
987 1.1 pooka pg->flags |= PG_WANTED;
988 1.1 pooka UVM_UNLOCK_AND_WAIT(pg, slock, 0, "genput", 0);
989 1.2 ad mutex_enter(slock);
990 1.1 pooka }
991 1.1 pooka if (by_list) {
992 1.1 pooka UVMHIST_LOG(ubchist, "after next %p",
993 1.8 ad TAILQ_NEXT(&curmp, listq.queue), 0,0,0);
994 1.8 ad pg = TAILQ_NEXT(&curmp, listq.queue);
995 1.8 ad TAILQ_REMOVE(&uobj->memq, &curmp, listq.queue);
996 1.1 pooka } else {
997 1.1 pooka pg = uvm_pagelookup(uobj, off);
998 1.1 pooka }
999 1.1 pooka continue;
1000 1.1 pooka }
1001 1.1 pooka
1002 1.1 pooka /*
1003 1.1 pooka * if we're freeing, remove all mappings of the page now.
1004 1.1 pooka * if we're cleaning, check if the page is needs to be cleaned.
1005 1.1 pooka */
1006 1.1 pooka
1007 1.1 pooka if (flags & PGO_FREE) {
1008 1.1 pooka pmap_page_protect(pg, VM_PROT_NONE);
1009 1.1 pooka } else if (flags & PGO_CLEANIT) {
1010 1.1 pooka
1011 1.1 pooka /*
1012 1.1 pooka * if we still have some hope to pull this vnode off
1013 1.1 pooka * from the syncer queue, write-protect the page.
1014 1.1 pooka */
1015 1.1 pooka
1016 1.1 pooka if (cleanall && wasclean &&
1017 1.1 pooka gp->g_dirtygen == dirtygen) {
1018 1.1 pooka
1019 1.1 pooka /*
1020 1.1 pooka * uobj pages get wired only by uvm_fault
1021 1.1 pooka * where uobj is locked.
1022 1.1 pooka */
1023 1.1 pooka
1024 1.1 pooka if (pg->wire_count == 0) {
1025 1.1 pooka pmap_page_protect(pg,
1026 1.1 pooka VM_PROT_READ|VM_PROT_EXECUTE);
1027 1.1 pooka } else {
1028 1.1 pooka cleanall = false;
1029 1.1 pooka }
1030 1.1 pooka }
1031 1.1 pooka }
1032 1.1 pooka
1033 1.1 pooka if (flags & PGO_CLEANIT) {
1034 1.1 pooka needs_clean = pmap_clear_modify(pg) ||
1035 1.1 pooka (pg->flags & PG_CLEAN) == 0;
1036 1.1 pooka pg->flags |= PG_CLEAN;
1037 1.1 pooka } else {
1038 1.1 pooka needs_clean = false;
1039 1.1 pooka }
1040 1.1 pooka
1041 1.1 pooka /*
1042 1.1 pooka * if we're cleaning, build a cluster.
1043 1.1 pooka * the cluster will consist of pages which are currently dirty,
1044 1.1 pooka * but they will be returned to us marked clean.
1045 1.1 pooka * if not cleaning, just operate on the one page.
1046 1.1 pooka */
1047 1.1 pooka
1048 1.1 pooka if (needs_clean) {
1049 1.1 pooka KDASSERT((vp->v_iflag & VI_ONWORKLST));
1050 1.1 pooka wasclean = false;
1051 1.1 pooka memset(pgs, 0, sizeof(pgs));
1052 1.1 pooka pg->flags |= PG_BUSY;
1053 1.1 pooka UVM_PAGE_OWN(pg, "genfs_putpages");
1054 1.1 pooka
1055 1.1 pooka /*
1056 1.1 pooka * first look backward.
1057 1.1 pooka */
1058 1.1 pooka
1059 1.1 pooka npages = MIN(maxpages >> 1, off >> PAGE_SHIFT);
1060 1.1 pooka nback = npages;
1061 1.1 pooka uvn_findpages(uobj, off - PAGE_SIZE, &nback, &pgs[0],
1062 1.1 pooka UFP_NOWAIT|UFP_NOALLOC|UFP_DIRTYONLY|UFP_BACKWARD);
1063 1.1 pooka if (nback) {
1064 1.1 pooka memmove(&pgs[0], &pgs[npages - nback],
1065 1.1 pooka nback * sizeof(pgs[0]));
1066 1.1 pooka if (npages - nback < nback)
1067 1.1 pooka memset(&pgs[nback], 0,
1068 1.1 pooka (npages - nback) * sizeof(pgs[0]));
1069 1.1 pooka else
1070 1.1 pooka memset(&pgs[npages - nback], 0,
1071 1.1 pooka nback * sizeof(pgs[0]));
1072 1.1 pooka }
1073 1.1 pooka
1074 1.1 pooka /*
1075 1.1 pooka * then plug in our page of interest.
1076 1.1 pooka */
1077 1.1 pooka
1078 1.1 pooka pgs[nback] = pg;
1079 1.1 pooka
1080 1.1 pooka /*
1081 1.1 pooka * then look forward to fill in the remaining space in
1082 1.1 pooka * the array of pages.
1083 1.1 pooka */
1084 1.1 pooka
1085 1.1 pooka npages = maxpages - nback - 1;
1086 1.1 pooka uvn_findpages(uobj, off + PAGE_SIZE, &npages,
1087 1.1 pooka &pgs[nback + 1],
1088 1.1 pooka UFP_NOWAIT|UFP_NOALLOC|UFP_DIRTYONLY);
1089 1.1 pooka npages += nback + 1;
1090 1.1 pooka } else {
1091 1.1 pooka pgs[0] = pg;
1092 1.1 pooka npages = 1;
1093 1.1 pooka nback = 0;
1094 1.1 pooka }
1095 1.1 pooka
1096 1.1 pooka /*
1097 1.1 pooka * apply FREE or DEACTIVATE options if requested.
1098 1.1 pooka */
1099 1.1 pooka
1100 1.1 pooka if (flags & (PGO_DEACTIVATE|PGO_FREE)) {
1101 1.2 ad mutex_enter(&uvm_pageqlock);
1102 1.1 pooka }
1103 1.1 pooka for (i = 0; i < npages; i++) {
1104 1.1 pooka tpg = pgs[i];
1105 1.1 pooka KASSERT(tpg->uobject == uobj);
1106 1.8 ad if (by_list && tpg == TAILQ_NEXT(pg, listq.queue))
1107 1.1 pooka pg = tpg;
1108 1.1 pooka if (tpg->offset < startoff || tpg->offset >= endoff)
1109 1.1 pooka continue;
1110 1.1 pooka if (flags & PGO_DEACTIVATE && tpg->wire_count == 0) {
1111 1.1 pooka uvm_pagedeactivate(tpg);
1112 1.1 pooka } else if (flags & PGO_FREE) {
1113 1.1 pooka pmap_page_protect(tpg, VM_PROT_NONE);
1114 1.1 pooka if (tpg->flags & PG_BUSY) {
1115 1.1 pooka tpg->flags |= freeflag;
1116 1.1 pooka if (pagedaemon) {
1117 1.2 ad uvm_pageout_start(1);
1118 1.1 pooka uvm_pagedequeue(tpg);
1119 1.1 pooka }
1120 1.1 pooka } else {
1121 1.1 pooka
1122 1.1 pooka /*
1123 1.1 pooka * ``page is not busy''
1124 1.1 pooka * implies that npages is 1
1125 1.1 pooka * and needs_clean is false.
1126 1.1 pooka */
1127 1.1 pooka
1128 1.8 ad nextpg = TAILQ_NEXT(tpg, listq.queue);
1129 1.1 pooka uvm_pagefree(tpg);
1130 1.1 pooka if (pagedaemon)
1131 1.1 pooka uvmexp.pdfreed++;
1132 1.1 pooka }
1133 1.1 pooka }
1134 1.1 pooka }
1135 1.1 pooka if (flags & (PGO_DEACTIVATE|PGO_FREE)) {
1136 1.2 ad mutex_exit(&uvm_pageqlock);
1137 1.1 pooka }
1138 1.1 pooka if (needs_clean) {
1139 1.1 pooka modified = true;
1140 1.1 pooka
1141 1.1 pooka /*
1142 1.1 pooka * start the i/o. if we're traversing by list,
1143 1.1 pooka * keep our place in the list with a marker page.
1144 1.1 pooka */
1145 1.1 pooka
1146 1.1 pooka if (by_list) {
1147 1.1 pooka TAILQ_INSERT_AFTER(&uobj->memq, pg, &curmp,
1148 1.8 ad listq.queue);
1149 1.1 pooka }
1150 1.2 ad mutex_exit(slock);
1151 1.1 pooka error = GOP_WRITE(vp, pgs, npages, flags);
1152 1.2 ad mutex_enter(slock);
1153 1.1 pooka if (by_list) {
1154 1.8 ad pg = TAILQ_NEXT(&curmp, listq.queue);
1155 1.8 ad TAILQ_REMOVE(&uobj->memq, &curmp, listq.queue);
1156 1.1 pooka }
1157 1.1 pooka if (error) {
1158 1.1 pooka break;
1159 1.1 pooka }
1160 1.1 pooka if (by_list) {
1161 1.1 pooka continue;
1162 1.1 pooka }
1163 1.1 pooka }
1164 1.1 pooka
1165 1.1 pooka /*
1166 1.1 pooka * find the next page and continue if there was no error.
1167 1.1 pooka */
1168 1.1 pooka
1169 1.1 pooka if (by_list) {
1170 1.1 pooka if (nextpg) {
1171 1.1 pooka pg = nextpg;
1172 1.1 pooka nextpg = NULL;
1173 1.1 pooka } else {
1174 1.8 ad pg = TAILQ_NEXT(pg, listq.queue);
1175 1.1 pooka }
1176 1.1 pooka } else {
1177 1.1 pooka off += (npages - nback) << PAGE_SHIFT;
1178 1.1 pooka if (off < endoff) {
1179 1.1 pooka pg = uvm_pagelookup(uobj, off);
1180 1.1 pooka }
1181 1.1 pooka }
1182 1.1 pooka }
1183 1.1 pooka if (by_list) {
1184 1.8 ad TAILQ_REMOVE(&uobj->memq, &endmp, listq.queue);
1185 1.1 pooka }
1186 1.1 pooka
1187 1.1 pooka if (modified && (vp->v_iflag & VI_WRMAPDIRTY) != 0 &&
1188 1.1 pooka (vp->v_type != VBLK ||
1189 1.1 pooka (vp->v_mount->mnt_flag & MNT_NODEVMTIME) == 0)) {
1190 1.1 pooka GOP_MARKUPDATE(vp, GOP_UPDATE_MODIFIED);
1191 1.1 pooka }
1192 1.1 pooka
1193 1.1 pooka /*
1194 1.1 pooka * if we're cleaning and there was nothing to clean,
1195 1.1 pooka * take us off the syncer list. if we started any i/o
1196 1.1 pooka * and we're doing sync i/o, wait for all writes to finish.
1197 1.1 pooka */
1198 1.1 pooka
1199 1.1 pooka if (cleanall && wasclean && gp->g_dirtygen == dirtygen &&
1200 1.1 pooka (vp->v_iflag & VI_ONWORKLST) != 0) {
1201 1.5 yamt #if defined(DEBUG)
1202 1.8 ad TAILQ_FOREACH(pg, &uobj->memq, listq.queue) {
1203 1.37 hannken if ((pg->flags & PG_MARKER) != 0) {
1204 1.37 hannken continue;
1205 1.37 hannken }
1206 1.5 yamt if ((pg->flags & PG_CLEAN) == 0) {
1207 1.5 yamt printf("%s: %p: !CLEAN\n", __func__, pg);
1208 1.5 yamt }
1209 1.5 yamt if (pmap_is_modified(pg)) {
1210 1.5 yamt printf("%s: %p: modified\n", __func__, pg);
1211 1.5 yamt }
1212 1.5 yamt }
1213 1.5 yamt #endif /* defined(DEBUG) */
1214 1.1 pooka vp->v_iflag &= ~VI_WRMAPDIRTY;
1215 1.1 pooka if (LIST_FIRST(&vp->v_dirtyblkhd) == NULL)
1216 1.1 pooka vn_syncer_remove_from_worklist(vp);
1217 1.1 pooka }
1218 1.1 pooka
1219 1.1 pooka #if !defined(DEBUG)
1220 1.1 pooka skip_scan:
1221 1.1 pooka #endif /* !defined(DEBUG) */
1222 1.2 ad
1223 1.2 ad /* Wait for output to complete. */
1224 1.2 ad if (!wasclean && !async && vp->v_numoutput != 0) {
1225 1.2 ad while (vp->v_numoutput != 0)
1226 1.2 ad cv_wait(&vp->v_cv, slock);
1227 1.1 pooka }
1228 1.4 yamt onworklst = (vp->v_iflag & VI_ONWORKLST) != 0;
1229 1.2 ad mutex_exit(slock);
1230 1.1 pooka
1231 1.4 yamt if ((flags & PGO_RECLAIM) != 0 && onworklst) {
1232 1.4 yamt /*
1233 1.4 yamt * in the case of PGO_RECLAIM, ensure to make the vnode clean.
1234 1.4 yamt * retrying is not a big deal because, in many cases,
1235 1.4 yamt * uobj->uo_npages is already 0 here.
1236 1.4 yamt */
1237 1.4 yamt mutex_enter(slock);
1238 1.4 yamt goto retry;
1239 1.4 yamt }
1240 1.4 yamt
1241 1.12 hannken if (has_trans) {
1242 1.12 hannken if (need_wapbl)
1243 1.12 hannken WAPBL_END(vp->v_mount);
1244 1.6 hannken fstrans_done(vp->v_mount);
1245 1.12 hannken }
1246 1.6 hannken
1247 1.1 pooka return (error);
1248 1.1 pooka }
1249 1.1 pooka
1250 1.1 pooka int
1251 1.1 pooka genfs_gop_write(struct vnode *vp, struct vm_page **pgs, int npages, int flags)
1252 1.1 pooka {
1253 1.1 pooka off_t off;
1254 1.1 pooka vaddr_t kva;
1255 1.1 pooka size_t len;
1256 1.1 pooka int error;
1257 1.1 pooka UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist);
1258 1.1 pooka
1259 1.1 pooka UVMHIST_LOG(ubchist, "vp %p pgs %p npages %d flags 0x%x",
1260 1.1 pooka vp, pgs, npages, flags);
1261 1.1 pooka
1262 1.1 pooka off = pgs[0]->offset;
1263 1.1 pooka kva = uvm_pagermapin(pgs, npages,
1264 1.1 pooka UVMPAGER_MAPIN_WRITE | UVMPAGER_MAPIN_WAITOK);
1265 1.1 pooka len = npages << PAGE_SHIFT;
1266 1.1 pooka
1267 1.1 pooka error = genfs_do_io(vp, off, kva, len, flags, UIO_WRITE,
1268 1.1 pooka uvm_aio_biodone);
1269 1.1 pooka
1270 1.1 pooka return error;
1271 1.1 pooka }
1272 1.1 pooka
1273 1.7 reinoud int
1274 1.7 reinoud genfs_gop_write_rwmap(struct vnode *vp, struct vm_page **pgs, int npages, int flags)
1275 1.7 reinoud {
1276 1.7 reinoud off_t off;
1277 1.7 reinoud vaddr_t kva;
1278 1.7 reinoud size_t len;
1279 1.7 reinoud int error;
1280 1.7 reinoud UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist);
1281 1.7 reinoud
1282 1.7 reinoud UVMHIST_LOG(ubchist, "vp %p pgs %p npages %d flags 0x%x",
1283 1.7 reinoud vp, pgs, npages, flags);
1284 1.7 reinoud
1285 1.7 reinoud off = pgs[0]->offset;
1286 1.7 reinoud kva = uvm_pagermapin(pgs, npages,
1287 1.7 reinoud UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK);
1288 1.7 reinoud len = npages << PAGE_SHIFT;
1289 1.7 reinoud
1290 1.7 reinoud error = genfs_do_io(vp, off, kva, len, flags, UIO_WRITE,
1291 1.7 reinoud uvm_aio_biodone);
1292 1.7 reinoud
1293 1.7 reinoud return error;
1294 1.7 reinoud }
1295 1.7 reinoud
1296 1.1 pooka /*
1297 1.1 pooka * Backend routine for doing I/O to vnode pages. Pages are already locked
1298 1.1 pooka * and mapped into kernel memory. Here we just look up the underlying
1299 1.1 pooka * device block addresses and call the strategy routine.
1300 1.1 pooka */
1301 1.1 pooka
1302 1.1 pooka static int
1303 1.1 pooka genfs_do_io(struct vnode *vp, off_t off, vaddr_t kva, size_t len, int flags,
1304 1.1 pooka enum uio_rw rw, void (*iodone)(struct buf *))
1305 1.1 pooka {
1306 1.36 uebayasi int s, error;
1307 1.1 pooka int fs_bshift, dev_bshift;
1308 1.1 pooka off_t eof, offset, startoffset;
1309 1.1 pooka size_t bytes, iobytes, skipbytes;
1310 1.1 pooka struct buf *mbp, *bp;
1311 1.35 uebayasi const bool async = (flags & PGO_SYNCIO) == 0;
1312 1.35 uebayasi const bool iowrite = rw == UIO_WRITE;
1313 1.35 uebayasi const int brw = iowrite ? B_WRITE : B_READ;
1314 1.1 pooka UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist);
1315 1.1 pooka
1316 1.1 pooka UVMHIST_LOG(ubchist, "vp %p kva %p len 0x%x flags 0x%x",
1317 1.1 pooka vp, kva, len, flags);
1318 1.1 pooka
1319 1.1 pooka KASSERT(vp->v_size <= vp->v_writesize);
1320 1.1 pooka GOP_SIZE(vp, vp->v_writesize, &eof, 0);
1321 1.1 pooka if (vp->v_type != VBLK) {
1322 1.1 pooka fs_bshift = vp->v_mount->mnt_fs_bshift;
1323 1.1 pooka dev_bshift = vp->v_mount->mnt_dev_bshift;
1324 1.1 pooka } else {
1325 1.1 pooka fs_bshift = DEV_BSHIFT;
1326 1.1 pooka dev_bshift = DEV_BSHIFT;
1327 1.1 pooka }
1328 1.1 pooka error = 0;
1329 1.1 pooka startoffset = off;
1330 1.1 pooka bytes = MIN(len, eof - startoffset);
1331 1.1 pooka skipbytes = 0;
1332 1.1 pooka KASSERT(bytes != 0);
1333 1.1 pooka
1334 1.35 uebayasi if (iowrite) {
1335 1.2 ad mutex_enter(&vp->v_interlock);
1336 1.1 pooka vp->v_numoutput += 2;
1337 1.2 ad mutex_exit(&vp->v_interlock);
1338 1.1 pooka }
1339 1.2 ad mbp = getiobuf(vp, true);
1340 1.1 pooka UVMHIST_LOG(ubchist, "vp %p mbp %p num now %d bytes 0x%x",
1341 1.1 pooka vp, mbp, vp->v_numoutput, bytes);
1342 1.1 pooka mbp->b_bufsize = len;
1343 1.1 pooka mbp->b_data = (void *)kva;
1344 1.1 pooka mbp->b_resid = mbp->b_bcount = bytes;
1345 1.2 ad mbp->b_cflags = BC_BUSY | BC_AGE;
1346 1.2 ad if (async) {
1347 1.2 ad mbp->b_flags = brw | B_ASYNC;
1348 1.2 ad mbp->b_iodone = iodone;
1349 1.2 ad } else {
1350 1.2 ad mbp->b_flags = brw;
1351 1.2 ad mbp->b_iodone = NULL;
1352 1.2 ad }
1353 1.1 pooka if (curlwp == uvm.pagedaemon_lwp)
1354 1.1 pooka BIO_SETPRIO(mbp, BPRIO_TIMELIMITED);
1355 1.1 pooka else if (async)
1356 1.1 pooka BIO_SETPRIO(mbp, BPRIO_TIMENONCRITICAL);
1357 1.1 pooka else
1358 1.1 pooka BIO_SETPRIO(mbp, BPRIO_TIMECRITICAL);
1359 1.1 pooka
1360 1.1 pooka bp = NULL;
1361 1.1 pooka for (offset = startoffset;
1362 1.1 pooka bytes > 0;
1363 1.1 pooka offset += iobytes, bytes -= iobytes) {
1364 1.36 uebayasi int run;
1365 1.36 uebayasi daddr_t lbn, blkno;
1366 1.36 uebayasi struct vnode *devvp;
1367 1.36 uebayasi
1368 1.36 uebayasi /*
1369 1.36 uebayasi * bmap the file to find out the blkno to read from and
1370 1.36 uebayasi * how much we can read in one i/o. if bmap returns an error,
1371 1.36 uebayasi * skip the rest of the top-level i/o.
1372 1.36 uebayasi */
1373 1.36 uebayasi
1374 1.1 pooka lbn = offset >> fs_bshift;
1375 1.1 pooka error = VOP_BMAP(vp, lbn, &devvp, &blkno, &run);
1376 1.1 pooka if (error) {
1377 1.36 uebayasi UVMHIST_LOG(ubchist, "VOP_BMAP lbn 0x%x -> %d\n",
1378 1.36 uebayasi lbn,error,0,0);
1379 1.1 pooka skipbytes += bytes;
1380 1.1 pooka bytes = 0;
1381 1.36 uebayasi goto loopdone;
1382 1.1 pooka }
1383 1.1 pooka
1384 1.36 uebayasi /*
1385 1.36 uebayasi * see how many pages can be read with this i/o.
1386 1.36 uebayasi * reduce the i/o size if necessary to avoid
1387 1.36 uebayasi * overwriting pages with valid data.
1388 1.36 uebayasi */
1389 1.36 uebayasi
1390 1.1 pooka iobytes = MIN((((off_t)lbn + 1 + run) << fs_bshift) - offset,
1391 1.1 pooka bytes);
1392 1.36 uebayasi
1393 1.36 uebayasi /*
1394 1.36 uebayasi * if this block isn't allocated, zero it instead of
1395 1.36 uebayasi * reading it. unless we are going to allocate blocks,
1396 1.36 uebayasi * mark the pages we zeroed PG_RDONLY.
1397 1.36 uebayasi */
1398 1.36 uebayasi
1399 1.1 pooka if (blkno == (daddr_t)-1) {
1400 1.35 uebayasi if (!iowrite) {
1401 1.1 pooka memset((char *)kva + (offset - startoffset), 0,
1402 1.36 uebayasi iobytes);
1403 1.1 pooka }
1404 1.1 pooka skipbytes += iobytes;
1405 1.1 pooka continue;
1406 1.1 pooka }
1407 1.1 pooka
1408 1.36 uebayasi /*
1409 1.36 uebayasi * allocate a sub-buf for this piece of the i/o
1410 1.36 uebayasi * (or just use mbp if there's only 1 piece),
1411 1.36 uebayasi * and start it going.
1412 1.36 uebayasi */
1413 1.36 uebayasi
1414 1.1 pooka if (offset == startoffset && iobytes == bytes) {
1415 1.1 pooka bp = mbp;
1416 1.1 pooka } else {
1417 1.1 pooka UVMHIST_LOG(ubchist, "vp %p bp %p num now %d",
1418 1.1 pooka vp, bp, vp->v_numoutput, 0);
1419 1.2 ad bp = getiobuf(vp, true);
1420 1.1 pooka nestiobuf_setup(mbp, bp, offset - startoffset, iobytes);
1421 1.1 pooka }
1422 1.1 pooka bp->b_lblkno = 0;
1423 1.1 pooka
1424 1.1 pooka /* adjust physical blkno for partial blocks */
1425 1.1 pooka bp->b_blkno = blkno + ((offset - ((off_t)lbn << fs_bshift)) >>
1426 1.1 pooka dev_bshift);
1427 1.36 uebayasi
1428 1.1 pooka UVMHIST_LOG(ubchist,
1429 1.36 uebayasi "bp %p offset 0x%x bcount 0x%x blkno 0x%x",
1430 1.36 uebayasi bp, offset, bp->b_bcount, bp->b_blkno);
1431 1.1 pooka
1432 1.1 pooka VOP_STRATEGY(devvp, bp);
1433 1.1 pooka }
1434 1.36 uebayasi
1435 1.36 uebayasi loopdone:
1436 1.1 pooka if (skipbytes) {
1437 1.1 pooka UVMHIST_LOG(ubchist, "skipbytes %d", skipbytes, 0,0,0);
1438 1.1 pooka }
1439 1.1 pooka nestiobuf_done(mbp, skipbytes, error);
1440 1.1 pooka if (async) {
1441 1.1 pooka UVMHIST_LOG(ubchist, "returning 0 (async)", 0,0,0,0);
1442 1.1 pooka return (0);
1443 1.1 pooka }
1444 1.1 pooka UVMHIST_LOG(ubchist, "waiting for mbp %p", mbp,0,0,0);
1445 1.1 pooka error = biowait(mbp);
1446 1.1 pooka s = splbio();
1447 1.1 pooka (*iodone)(mbp);
1448 1.1 pooka splx(s);
1449 1.1 pooka UVMHIST_LOG(ubchist, "returning, error %d", error,0,0,0);
1450 1.1 pooka return (error);
1451 1.1 pooka }
1452 1.1 pooka
1453 1.1 pooka int
1454 1.1 pooka genfs_compat_getpages(void *v)
1455 1.1 pooka {
1456 1.1 pooka struct vop_getpages_args /* {
1457 1.1 pooka struct vnode *a_vp;
1458 1.1 pooka voff_t a_offset;
1459 1.1 pooka struct vm_page **a_m;
1460 1.1 pooka int *a_count;
1461 1.1 pooka int a_centeridx;
1462 1.1 pooka vm_prot_t a_access_type;
1463 1.1 pooka int a_advice;
1464 1.1 pooka int a_flags;
1465 1.1 pooka } */ *ap = v;
1466 1.1 pooka
1467 1.1 pooka off_t origoffset;
1468 1.1 pooka struct vnode *vp = ap->a_vp;
1469 1.1 pooka struct uvm_object *uobj = &vp->v_uobj;
1470 1.1 pooka struct vm_page *pg, **pgs;
1471 1.1 pooka vaddr_t kva;
1472 1.1 pooka int i, error, orignpages, npages;
1473 1.1 pooka struct iovec iov;
1474 1.1 pooka struct uio uio;
1475 1.1 pooka kauth_cred_t cred = curlwp->l_cred;
1476 1.35 uebayasi const bool memwrite = (ap->a_access_type & VM_PROT_WRITE) != 0;
1477 1.1 pooka
1478 1.1 pooka error = 0;
1479 1.1 pooka origoffset = ap->a_offset;
1480 1.1 pooka orignpages = *ap->a_count;
1481 1.1 pooka pgs = ap->a_m;
1482 1.1 pooka
1483 1.1 pooka if (ap->a_flags & PGO_LOCKED) {
1484 1.1 pooka uvn_findpages(uobj, origoffset, ap->a_count, ap->a_m,
1485 1.35 uebayasi UFP_NOWAIT|UFP_NOALLOC| (memwrite ? UFP_NORDONLY : 0));
1486 1.1 pooka
1487 1.38 chs error = ap->a_m[ap->a_centeridx] == NULL ? EBUSY : 0;
1488 1.38 chs if (error == 0 && memwrite) {
1489 1.38 chs genfs_markdirty(vp);
1490 1.38 chs }
1491 1.38 chs return error;
1492 1.1 pooka }
1493 1.1 pooka if (origoffset + (ap->a_centeridx << PAGE_SHIFT) >= vp->v_size) {
1494 1.2 ad mutex_exit(&uobj->vmobjlock);
1495 1.38 chs return EINVAL;
1496 1.1 pooka }
1497 1.1 pooka if ((ap->a_flags & PGO_SYNCIO) == 0) {
1498 1.2 ad mutex_exit(&uobj->vmobjlock);
1499 1.1 pooka return 0;
1500 1.1 pooka }
1501 1.1 pooka npages = orignpages;
1502 1.1 pooka uvn_findpages(uobj, origoffset, &npages, pgs, UFP_ALL);
1503 1.2 ad mutex_exit(&uobj->vmobjlock);
1504 1.1 pooka kva = uvm_pagermapin(pgs, npages,
1505 1.1 pooka UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK);
1506 1.1 pooka for (i = 0; i < npages; i++) {
1507 1.1 pooka pg = pgs[i];
1508 1.1 pooka if ((pg->flags & PG_FAKE) == 0) {
1509 1.1 pooka continue;
1510 1.1 pooka }
1511 1.1 pooka iov.iov_base = (char *)kva + (i << PAGE_SHIFT);
1512 1.1 pooka iov.iov_len = PAGE_SIZE;
1513 1.1 pooka uio.uio_iov = &iov;
1514 1.1 pooka uio.uio_iovcnt = 1;
1515 1.1 pooka uio.uio_offset = origoffset + (i << PAGE_SHIFT);
1516 1.1 pooka uio.uio_rw = UIO_READ;
1517 1.1 pooka uio.uio_resid = PAGE_SIZE;
1518 1.1 pooka UIO_SETUP_SYSSPACE(&uio);
1519 1.1 pooka /* XXX vn_lock */
1520 1.1 pooka error = VOP_READ(vp, &uio, 0, cred);
1521 1.1 pooka if (error) {
1522 1.1 pooka break;
1523 1.1 pooka }
1524 1.1 pooka if (uio.uio_resid) {
1525 1.1 pooka memset(iov.iov_base, 0, uio.uio_resid);
1526 1.1 pooka }
1527 1.1 pooka }
1528 1.1 pooka uvm_pagermapout(kva, npages);
1529 1.2 ad mutex_enter(&uobj->vmobjlock);
1530 1.2 ad mutex_enter(&uvm_pageqlock);
1531 1.1 pooka for (i = 0; i < npages; i++) {
1532 1.1 pooka pg = pgs[i];
1533 1.1 pooka if (error && (pg->flags & PG_FAKE) != 0) {
1534 1.1 pooka pg->flags |= PG_RELEASED;
1535 1.1 pooka } else {
1536 1.1 pooka pmap_clear_modify(pg);
1537 1.1 pooka uvm_pageactivate(pg);
1538 1.1 pooka }
1539 1.1 pooka }
1540 1.1 pooka if (error) {
1541 1.1 pooka uvm_page_unbusy(pgs, npages);
1542 1.1 pooka }
1543 1.2 ad mutex_exit(&uvm_pageqlock);
1544 1.38 chs if (error == 0 && memwrite) {
1545 1.38 chs genfs_markdirty(vp);
1546 1.38 chs }
1547 1.2 ad mutex_exit(&uobj->vmobjlock);
1548 1.38 chs return error;
1549 1.1 pooka }
1550 1.1 pooka
1551 1.1 pooka int
1552 1.1 pooka genfs_compat_gop_write(struct vnode *vp, struct vm_page **pgs, int npages,
1553 1.1 pooka int flags)
1554 1.1 pooka {
1555 1.1 pooka off_t offset;
1556 1.1 pooka struct iovec iov;
1557 1.1 pooka struct uio uio;
1558 1.1 pooka kauth_cred_t cred = curlwp->l_cred;
1559 1.1 pooka struct buf *bp;
1560 1.1 pooka vaddr_t kva;
1561 1.2 ad int error;
1562 1.1 pooka
1563 1.1 pooka offset = pgs[0]->offset;
1564 1.1 pooka kva = uvm_pagermapin(pgs, npages,
1565 1.1 pooka UVMPAGER_MAPIN_WRITE | UVMPAGER_MAPIN_WAITOK);
1566 1.1 pooka
1567 1.1 pooka iov.iov_base = (void *)kva;
1568 1.1 pooka iov.iov_len = npages << PAGE_SHIFT;
1569 1.1 pooka uio.uio_iov = &iov;
1570 1.1 pooka uio.uio_iovcnt = 1;
1571 1.1 pooka uio.uio_offset = offset;
1572 1.1 pooka uio.uio_rw = UIO_WRITE;
1573 1.1 pooka uio.uio_resid = npages << PAGE_SHIFT;
1574 1.1 pooka UIO_SETUP_SYSSPACE(&uio);
1575 1.1 pooka /* XXX vn_lock */
1576 1.1 pooka error = VOP_WRITE(vp, &uio, 0, cred);
1577 1.1 pooka
1578 1.2 ad mutex_enter(&vp->v_interlock);
1579 1.2 ad vp->v_numoutput++;
1580 1.2 ad mutex_exit(&vp->v_interlock);
1581 1.1 pooka
1582 1.2 ad bp = getiobuf(vp, true);
1583 1.2 ad bp->b_cflags = BC_BUSY | BC_AGE;
1584 1.1 pooka bp->b_lblkno = offset >> vp->v_mount->mnt_fs_bshift;
1585 1.1 pooka bp->b_data = (char *)kva;
1586 1.1 pooka bp->b_bcount = npages << PAGE_SHIFT;
1587 1.1 pooka bp->b_bufsize = npages << PAGE_SHIFT;
1588 1.1 pooka bp->b_resid = 0;
1589 1.1 pooka bp->b_error = error;
1590 1.1 pooka uvm_aio_aiodone(bp);
1591 1.1 pooka return (error);
1592 1.1 pooka }
1593 1.1 pooka
1594 1.1 pooka /*
1595 1.1 pooka * Process a uio using direct I/O. If we reach a part of the request
1596 1.1 pooka * which cannot be processed in this fashion for some reason, just return.
1597 1.1 pooka * The caller must handle some additional part of the request using
1598 1.1 pooka * buffered I/O before trying direct I/O again.
1599 1.1 pooka */
1600 1.1 pooka
1601 1.1 pooka void
1602 1.1 pooka genfs_directio(struct vnode *vp, struct uio *uio, int ioflag)
1603 1.1 pooka {
1604 1.1 pooka struct vmspace *vs;
1605 1.1 pooka struct iovec *iov;
1606 1.1 pooka vaddr_t va;
1607 1.1 pooka size_t len;
1608 1.1 pooka const int mask = DEV_BSIZE - 1;
1609 1.1 pooka int error;
1610 1.16 joerg bool need_wapbl = (vp->v_mount && vp->v_mount->mnt_wapbl &&
1611 1.16 joerg (ioflag & IO_JOURNALLOCKED) == 0);
1612 1.1 pooka
1613 1.1 pooka /*
1614 1.1 pooka * We only support direct I/O to user space for now.
1615 1.1 pooka */
1616 1.1 pooka
1617 1.1 pooka if (VMSPACE_IS_KERNEL_P(uio->uio_vmspace)) {
1618 1.1 pooka return;
1619 1.1 pooka }
1620 1.1 pooka
1621 1.1 pooka /*
1622 1.1 pooka * If the vnode is mapped, we would need to get the getpages lock
1623 1.1 pooka * to stabilize the bmap, but then we would get into trouble whil e
1624 1.1 pooka * locking the pages if the pages belong to this same vnode (or a
1625 1.1 pooka * multi-vnode cascade to the same effect). Just fall back to
1626 1.1 pooka * buffered I/O if the vnode is mapped to avoid this mess.
1627 1.1 pooka */
1628 1.1 pooka
1629 1.1 pooka if (vp->v_vflag & VV_MAPPED) {
1630 1.1 pooka return;
1631 1.1 pooka }
1632 1.1 pooka
1633 1.16 joerg if (need_wapbl) {
1634 1.13 hannken error = WAPBL_BEGIN(vp->v_mount);
1635 1.13 hannken if (error)
1636 1.13 hannken return;
1637 1.13 hannken }
1638 1.13 hannken
1639 1.1 pooka /*
1640 1.1 pooka * Do as much of the uio as possible with direct I/O.
1641 1.1 pooka */
1642 1.1 pooka
1643 1.1 pooka vs = uio->uio_vmspace;
1644 1.1 pooka while (uio->uio_resid) {
1645 1.1 pooka iov = uio->uio_iov;
1646 1.1 pooka if (iov->iov_len == 0) {
1647 1.1 pooka uio->uio_iov++;
1648 1.1 pooka uio->uio_iovcnt--;
1649 1.1 pooka continue;
1650 1.1 pooka }
1651 1.1 pooka va = (vaddr_t)iov->iov_base;
1652 1.1 pooka len = MIN(iov->iov_len, genfs_maxdio);
1653 1.1 pooka len &= ~mask;
1654 1.1 pooka
1655 1.1 pooka /*
1656 1.1 pooka * If the next chunk is smaller than DEV_BSIZE or extends past
1657 1.1 pooka * the current EOF, then fall back to buffered I/O.
1658 1.1 pooka */
1659 1.1 pooka
1660 1.1 pooka if (len == 0 || uio->uio_offset + len > vp->v_size) {
1661 1.13 hannken break;
1662 1.1 pooka }
1663 1.1 pooka
1664 1.1 pooka /*
1665 1.1 pooka * Check alignment. The file offset must be at least
1666 1.1 pooka * sector-aligned. The exact constraint on memory alignment
1667 1.1 pooka * is very hardware-dependent, but requiring sector-aligned
1668 1.1 pooka * addresses there too is safe.
1669 1.1 pooka */
1670 1.1 pooka
1671 1.1 pooka if (uio->uio_offset & mask || va & mask) {
1672 1.13 hannken break;
1673 1.1 pooka }
1674 1.1 pooka error = genfs_do_directio(vs, va, len, vp, uio->uio_offset,
1675 1.1 pooka uio->uio_rw);
1676 1.1 pooka if (error) {
1677 1.1 pooka break;
1678 1.1 pooka }
1679 1.1 pooka iov->iov_base = (char *)iov->iov_base + len;
1680 1.1 pooka iov->iov_len -= len;
1681 1.1 pooka uio->uio_offset += len;
1682 1.1 pooka uio->uio_resid -= len;
1683 1.1 pooka }
1684 1.13 hannken
1685 1.16 joerg if (need_wapbl)
1686 1.13 hannken WAPBL_END(vp->v_mount);
1687 1.1 pooka }
1688 1.1 pooka
1689 1.1 pooka /*
1690 1.1 pooka * Iodone routine for direct I/O. We don't do much here since the request is
1691 1.1 pooka * always synchronous, so the caller will do most of the work after biowait().
1692 1.1 pooka */
1693 1.1 pooka
1694 1.1 pooka static void
1695 1.1 pooka genfs_dio_iodone(struct buf *bp)
1696 1.1 pooka {
1697 1.1 pooka
1698 1.1 pooka KASSERT((bp->b_flags & B_ASYNC) == 0);
1699 1.2 ad if ((bp->b_flags & B_READ) == 0 && (bp->b_cflags & BC_AGE) != 0) {
1700 1.2 ad mutex_enter(bp->b_objlock);
1701 1.1 pooka vwakeup(bp);
1702 1.2 ad mutex_exit(bp->b_objlock);
1703 1.1 pooka }
1704 1.1 pooka putiobuf(bp);
1705 1.1 pooka }
1706 1.1 pooka
1707 1.1 pooka /*
1708 1.1 pooka * Process one chunk of a direct I/O request.
1709 1.1 pooka */
1710 1.1 pooka
1711 1.1 pooka static int
1712 1.1 pooka genfs_do_directio(struct vmspace *vs, vaddr_t uva, size_t len, struct vnode *vp,
1713 1.1 pooka off_t off, enum uio_rw rw)
1714 1.1 pooka {
1715 1.1 pooka struct vm_map *map;
1716 1.1 pooka struct pmap *upm, *kpm;
1717 1.1 pooka size_t klen = round_page(uva + len) - trunc_page(uva);
1718 1.1 pooka off_t spoff, epoff;
1719 1.1 pooka vaddr_t kva, puva;
1720 1.1 pooka paddr_t pa;
1721 1.1 pooka vm_prot_t prot;
1722 1.1 pooka int error, rv, poff, koff;
1723 1.13 hannken const int pgoflags = PGO_CLEANIT | PGO_SYNCIO | PGO_JOURNALLOCKED |
1724 1.1 pooka (rw == UIO_WRITE ? PGO_FREE : 0);
1725 1.1 pooka
1726 1.1 pooka /*
1727 1.1 pooka * For writes, verify that this range of the file already has fully
1728 1.1 pooka * allocated backing store. If there are any holes, just punt and
1729 1.1 pooka * make the caller take the buffered write path.
1730 1.1 pooka */
1731 1.1 pooka
1732 1.1 pooka if (rw == UIO_WRITE) {
1733 1.1 pooka daddr_t lbn, elbn, blkno;
1734 1.1 pooka int bsize, bshift, run;
1735 1.1 pooka
1736 1.1 pooka bshift = vp->v_mount->mnt_fs_bshift;
1737 1.1 pooka bsize = 1 << bshift;
1738 1.1 pooka lbn = off >> bshift;
1739 1.1 pooka elbn = (off + len + bsize - 1) >> bshift;
1740 1.1 pooka while (lbn < elbn) {
1741 1.1 pooka error = VOP_BMAP(vp, lbn, NULL, &blkno, &run);
1742 1.1 pooka if (error) {
1743 1.1 pooka return error;
1744 1.1 pooka }
1745 1.1 pooka if (blkno == (daddr_t)-1) {
1746 1.1 pooka return ENOSPC;
1747 1.1 pooka }
1748 1.1 pooka lbn += 1 + run;
1749 1.1 pooka }
1750 1.1 pooka }
1751 1.1 pooka
1752 1.1 pooka /*
1753 1.1 pooka * Flush any cached pages for parts of the file that we're about to
1754 1.1 pooka * access. If we're writing, invalidate pages as well.
1755 1.1 pooka */
1756 1.1 pooka
1757 1.1 pooka spoff = trunc_page(off);
1758 1.1 pooka epoff = round_page(off + len);
1759 1.2 ad mutex_enter(&vp->v_interlock);
1760 1.1 pooka error = VOP_PUTPAGES(vp, spoff, epoff, pgoflags);
1761 1.1 pooka if (error) {
1762 1.1 pooka return error;
1763 1.1 pooka }
1764 1.1 pooka
1765 1.1 pooka /*
1766 1.1 pooka * Wire the user pages and remap them into kernel memory.
1767 1.1 pooka */
1768 1.1 pooka
1769 1.1 pooka prot = rw == UIO_READ ? VM_PROT_READ | VM_PROT_WRITE : VM_PROT_READ;
1770 1.1 pooka error = uvm_vslock(vs, (void *)uva, len, prot);
1771 1.1 pooka if (error) {
1772 1.1 pooka return error;
1773 1.1 pooka }
1774 1.1 pooka
1775 1.1 pooka map = &vs->vm_map;
1776 1.1 pooka upm = vm_map_pmap(map);
1777 1.1 pooka kpm = vm_map_pmap(kernel_map);
1778 1.1 pooka kva = uvm_km_alloc(kernel_map, klen, 0,
1779 1.1 pooka UVM_KMF_VAONLY | UVM_KMF_WAITVA);
1780 1.1 pooka puva = trunc_page(uva);
1781 1.1 pooka for (poff = 0; poff < klen; poff += PAGE_SIZE) {
1782 1.1 pooka rv = pmap_extract(upm, puva + poff, &pa);
1783 1.1 pooka KASSERT(rv);
1784 1.1 pooka pmap_enter(kpm, kva + poff, pa, prot, prot | PMAP_WIRED);
1785 1.1 pooka }
1786 1.1 pooka pmap_update(kpm);
1787 1.1 pooka
1788 1.1 pooka /*
1789 1.1 pooka * Do the I/O.
1790 1.1 pooka */
1791 1.1 pooka
1792 1.1 pooka koff = uva - trunc_page(uva);
1793 1.1 pooka error = genfs_do_io(vp, off, kva + koff, len, PGO_SYNCIO, rw,
1794 1.1 pooka genfs_dio_iodone);
1795 1.1 pooka
1796 1.1 pooka /*
1797 1.1 pooka * Tear down the kernel mapping.
1798 1.1 pooka */
1799 1.1 pooka
1800 1.1 pooka pmap_remove(kpm, kva, kva + klen);
1801 1.1 pooka pmap_update(kpm);
1802 1.1 pooka uvm_km_free(kernel_map, kva, klen, UVM_KMF_VAONLY);
1803 1.1 pooka
1804 1.1 pooka /*
1805 1.1 pooka * Unwire the user pages.
1806 1.1 pooka */
1807 1.1 pooka
1808 1.1 pooka uvm_vsunlock(vs, (void *)uva, len);
1809 1.1 pooka return error;
1810 1.1 pooka }
1811 1.2 ad
1812