uvm_fault.c revision 1.180.4.1 1 1.180.4.1 bouyer /* $NetBSD: uvm_fault.c,v 1.180.4.1 2011/02/08 16:20:06 bouyer Exp $ */
2 1.1 mrg
3 1.1 mrg /*
4 1.1 mrg * Copyright (c) 1997 Charles D. Cranor and Washington University.
5 1.1 mrg * All rights reserved.
6 1.1 mrg *
7 1.1 mrg * Redistribution and use in source and binary forms, with or without
8 1.1 mrg * modification, are permitted provided that the following conditions
9 1.1 mrg * are met:
10 1.1 mrg * 1. Redistributions of source code must retain the above copyright
11 1.1 mrg * notice, this list of conditions and the following disclaimer.
12 1.1 mrg * 2. Redistributions in binary form must reproduce the above copyright
13 1.1 mrg * notice, this list of conditions and the following disclaimer in the
14 1.1 mrg * documentation and/or other materials provided with the distribution.
15 1.1 mrg *
16 1.1 mrg * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 1.1 mrg * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 1.1 mrg * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 1.1 mrg * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 1.1 mrg * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 1.1 mrg * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 1.1 mrg * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 1.1 mrg * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 1.1 mrg * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 1.1 mrg * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 1.4 mrg *
27 1.4 mrg * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp
28 1.1 mrg */
29 1.1 mrg
30 1.1 mrg /*
31 1.1 mrg * uvm_fault.c: fault handler
32 1.1 mrg */
33 1.71 lukem
34 1.71 lukem #include <sys/cdefs.h>
35 1.180.4.1 bouyer __KERNEL_RCSID(0, "$NetBSD: uvm_fault.c,v 1.180.4.1 2011/02/08 16:20:06 bouyer Exp $");
36 1.71 lukem
37 1.71 lukem #include "opt_uvmhist.h"
38 1.1 mrg
39 1.1 mrg #include <sys/param.h>
40 1.1 mrg #include <sys/systm.h>
41 1.1 mrg #include <sys/kernel.h>
42 1.1 mrg #include <sys/proc.h>
43 1.1 mrg #include <sys/malloc.h>
44 1.1 mrg #include <sys/mman.h>
45 1.1 mrg
46 1.1 mrg #include <uvm/uvm.h>
47 1.1 mrg
48 1.1 mrg /*
49 1.1 mrg *
50 1.1 mrg * a word on page faults:
51 1.1 mrg *
52 1.1 mrg * types of page faults we handle:
53 1.1 mrg *
54 1.1 mrg * CASE 1: upper layer faults CASE 2: lower layer faults
55 1.1 mrg *
56 1.1 mrg * CASE 1A CASE 1B CASE 2A CASE 2B
57 1.1 mrg * read/write1 write>1 read/write +-cow_write/zero
58 1.63 chs * | | | |
59 1.1 mrg * +--|--+ +--|--+ +-----+ + | + | +-----+
60 1.127 uebayasi * amap | V | | ---------> new | | | | ^ |
61 1.1 mrg * +-----+ +-----+ +-----+ + | + | +--|--+
62 1.1 mrg * | | |
63 1.1 mrg * +-----+ +-----+ +--|--+ | +--|--+
64 1.127 uebayasi * uobj | d/c | | d/c | | V | +----+ |
65 1.1 mrg * +-----+ +-----+ +-----+ +-----+
66 1.1 mrg *
67 1.1 mrg * d/c = don't care
68 1.63 chs *
69 1.1 mrg * case [0]: layerless fault
70 1.1 mrg * no amap or uobj is present. this is an error.
71 1.1 mrg *
72 1.1 mrg * case [1]: upper layer fault [anon active]
73 1.1 mrg * 1A: [read] or [write with anon->an_ref == 1]
74 1.127 uebayasi * I/O takes place in upper level anon and uobj is not touched.
75 1.1 mrg * 1B: [write with anon->an_ref > 1]
76 1.1 mrg * new anon is alloc'd and data is copied off ["COW"]
77 1.1 mrg *
78 1.1 mrg * case [2]: lower layer fault [uobj]
79 1.1 mrg * 2A: [read on non-NULL uobj] or [write to non-copy_on_write area]
80 1.1 mrg * I/O takes place directly in object.
81 1.1 mrg * 2B: [write to copy_on_write] or [read on NULL uobj]
82 1.63 chs * data is "promoted" from uobj to a new anon.
83 1.1 mrg * if uobj is null, then we zero fill.
84 1.1 mrg *
85 1.1 mrg * we follow the standard UVM locking protocol ordering:
86 1.1 mrg *
87 1.63 chs * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ)
88 1.1 mrg * we hold a PG_BUSY page if we unlock for I/O
89 1.1 mrg *
90 1.1 mrg *
91 1.1 mrg * the code is structured as follows:
92 1.63 chs *
93 1.1 mrg * - init the "IN" params in the ufi structure
94 1.177 yamt * ReFault: (ERESTART returned to the loop in uvm_fault_internal)
95 1.1 mrg * - do lookups [locks maps], check protection, handle needs_copy
96 1.1 mrg * - check for case 0 fault (error)
97 1.1 mrg * - establish "range" of fault
98 1.1 mrg * - if we have an amap lock it and extract the anons
99 1.1 mrg * - if sequential advice deactivate pages behind us
100 1.1 mrg * - at the same time check pmap for unmapped areas and anon for pages
101 1.1 mrg * that we could map in (and do map it if found)
102 1.1 mrg * - check object for resident pages that we could map in
103 1.1 mrg * - if (case 2) goto Case2
104 1.1 mrg * - >>> handle case 1
105 1.1 mrg * - ensure source anon is resident in RAM
106 1.1 mrg * - if case 1B alloc new anon and copy from source
107 1.1 mrg * - map the correct page in
108 1.1 mrg * Case2:
109 1.1 mrg * - >>> handle case 2
110 1.1 mrg * - ensure source page is resident (if uobj)
111 1.1 mrg * - if case 2B alloc new anon and copy from source (could be zero
112 1.1 mrg * fill if uobj == NULL)
113 1.1 mrg * - map the correct page in
114 1.1 mrg * - done!
115 1.1 mrg *
116 1.1 mrg * note on paging:
117 1.1 mrg * if we have to do I/O we place a PG_BUSY page in the correct object,
118 1.1 mrg * unlock everything, and do the I/O. when I/O is done we must reverify
119 1.1 mrg * the state of the world before assuming that our data structures are
120 1.1 mrg * valid. [because mappings could change while the map is unlocked]
121 1.1 mrg *
122 1.1 mrg * alternative 1: unbusy the page in question and restart the page fault
123 1.1 mrg * from the top (ReFault). this is easy but does not take advantage
124 1.63 chs * of the information that we already have from our previous lookup,
125 1.1 mrg * although it is possible that the "hints" in the vm_map will help here.
126 1.1 mrg *
127 1.1 mrg * alternative 2: the system already keeps track of a "version" number of
128 1.1 mrg * a map. [i.e. every time you write-lock a map (e.g. to change a
129 1.1 mrg * mapping) you bump the version number up by one...] so, we can save
130 1.1 mrg * the version number of the map before we release the lock and start I/O.
131 1.1 mrg * then when I/O is done we can relock and check the version numbers
132 1.1 mrg * to see if anything changed. this might save us some over 1 because
133 1.1 mrg * we don't have to unbusy the page and may be less compares(?).
134 1.1 mrg *
135 1.1 mrg * alternative 3: put in backpointers or a way to "hold" part of a map
136 1.1 mrg * in place while I/O is in progress. this could be complex to
137 1.1 mrg * implement (especially with structures like amap that can be referenced
138 1.1 mrg * by multiple map entries, and figuring out what should wait could be
139 1.1 mrg * complex as well...).
140 1.1 mrg *
141 1.125 ad * we use alternative 2. given that we are multi-threaded now we may want
142 1.125 ad * to reconsider the choice.
143 1.1 mrg */
144 1.1 mrg
145 1.1 mrg /*
146 1.1 mrg * local data structures
147 1.1 mrg */
148 1.1 mrg
149 1.1 mrg struct uvm_advice {
150 1.7 mrg int advice;
151 1.7 mrg int nback;
152 1.7 mrg int nforw;
153 1.1 mrg };
154 1.1 mrg
155 1.1 mrg /*
156 1.1 mrg * page range array:
157 1.63 chs * note: index in array must match "advice" value
158 1.1 mrg * XXX: borrowed numbers from freebsd. do they work well for us?
159 1.1 mrg */
160 1.1 mrg
161 1.95 thorpej static const struct uvm_advice uvmadvice[] = {
162 1.7 mrg { MADV_NORMAL, 3, 4 },
163 1.7 mrg { MADV_RANDOM, 0, 0 },
164 1.7 mrg { MADV_SEQUENTIAL, 8, 7},
165 1.1 mrg };
166 1.1 mrg
167 1.69 chs #define UVM_MAXRANGE 16 /* must be MAX() of nback+nforw+1 */
168 1.1 mrg
169 1.1 mrg /*
170 1.1 mrg * private prototypes
171 1.1 mrg */
172 1.1 mrg
173 1.1 mrg /*
174 1.1 mrg * inline functions
175 1.1 mrg */
176 1.1 mrg
177 1.1 mrg /*
178 1.1 mrg * uvmfault_anonflush: try and deactivate pages in specified anons
179 1.1 mrg *
180 1.1 mrg * => does not have to deactivate page if it is busy
181 1.1 mrg */
182 1.1 mrg
183 1.103 perry static inline void
184 1.95 thorpej uvmfault_anonflush(struct vm_anon **anons, int n)
185 1.1 mrg {
186 1.7 mrg int lcv;
187 1.7 mrg struct vm_page *pg;
188 1.63 chs
189 1.163 uebayasi for (lcv = 0; lcv < n; lcv++) {
190 1.7 mrg if (anons[lcv] == NULL)
191 1.7 mrg continue;
192 1.122 ad mutex_enter(&anons[lcv]->an_lock);
193 1.94 yamt pg = anons[lcv]->an_page;
194 1.117 yamt if (pg && (pg->flags & PG_BUSY) == 0) {
195 1.122 ad mutex_enter(&uvm_pageqlock);
196 1.7 mrg if (pg->wire_count == 0) {
197 1.7 mrg uvm_pagedeactivate(pg);
198 1.7 mrg }
199 1.122 ad mutex_exit(&uvm_pageqlock);
200 1.7 mrg }
201 1.122 ad mutex_exit(&anons[lcv]->an_lock);
202 1.7 mrg }
203 1.1 mrg }
204 1.1 mrg
205 1.1 mrg /*
206 1.1 mrg * normal functions
207 1.1 mrg */
208 1.1 mrg
209 1.1 mrg /*
210 1.1 mrg * uvmfault_amapcopy: clear "needs_copy" in a map.
211 1.1 mrg *
212 1.1 mrg * => called with VM data structures unlocked (usually, see below)
213 1.1 mrg * => we get a write lock on the maps and clear needs_copy for a VA
214 1.1 mrg * => if we are out of RAM we sleep (waiting for more)
215 1.1 mrg */
216 1.1 mrg
217 1.7 mrg static void
218 1.95 thorpej uvmfault_amapcopy(struct uvm_faultinfo *ufi)
219 1.1 mrg {
220 1.69 chs for (;;) {
221 1.1 mrg
222 1.7 mrg /*
223 1.7 mrg * no mapping? give up.
224 1.7 mrg */
225 1.1 mrg
226 1.119 thorpej if (uvmfault_lookup(ufi, true) == false)
227 1.7 mrg return;
228 1.1 mrg
229 1.7 mrg /*
230 1.7 mrg * copy if needed.
231 1.7 mrg */
232 1.1 mrg
233 1.7 mrg if (UVM_ET_ISNEEDSCOPY(ufi->entry))
234 1.108 yamt amap_copy(ufi->map, ufi->entry, AMAP_COPY_NOWAIT,
235 1.13 chuck ufi->orig_rvaddr, ufi->orig_rvaddr + 1);
236 1.1 mrg
237 1.7 mrg /*
238 1.7 mrg * didn't work? must be out of RAM. unlock and sleep.
239 1.7 mrg */
240 1.7 mrg
241 1.7 mrg if (UVM_ET_ISNEEDSCOPY(ufi->entry)) {
242 1.119 thorpej uvmfault_unlockmaps(ufi, true);
243 1.7 mrg uvm_wait("fltamapcopy");
244 1.7 mrg continue;
245 1.7 mrg }
246 1.7 mrg
247 1.7 mrg /*
248 1.7 mrg * got it! unlock and return.
249 1.7 mrg */
250 1.63 chs
251 1.119 thorpej uvmfault_unlockmaps(ufi, true);
252 1.7 mrg return;
253 1.7 mrg }
254 1.7 mrg /*NOTREACHED*/
255 1.1 mrg }
256 1.1 mrg
257 1.1 mrg /*
258 1.1 mrg * uvmfault_anonget: get data in an anon into a non-busy, non-released
259 1.1 mrg * page in that anon.
260 1.1 mrg *
261 1.1 mrg * => maps, amap, and anon locked by caller.
262 1.57 chs * => if we fail (result != 0) we unlock everything.
263 1.1 mrg * => if we are successful, we return with everything still locked.
264 1.1 mrg * => we don't move the page on the queues [gets moved later]
265 1.1 mrg * => if we allocate a new page [we_own], it gets put on the queues.
266 1.1 mrg * either way, the result is that the page is on the queues at return time
267 1.1 mrg * => for pages which are on loan from a uvm_object (and thus are not
268 1.1 mrg * owned by the anon): if successful, we return with the owning object
269 1.1 mrg * locked. the caller must unlock this object when it unlocks everything
270 1.1 mrg * else.
271 1.1 mrg */
272 1.1 mrg
273 1.47 chs int
274 1.95 thorpej uvmfault_anonget(struct uvm_faultinfo *ufi, struct vm_amap *amap,
275 1.95 thorpej struct vm_anon *anon)
276 1.7 mrg {
277 1.118 thorpej bool we_own; /* we own anon's page? */
278 1.118 thorpej bool locked; /* did we relock? */
279 1.7 mrg struct vm_page *pg;
280 1.58 chs int error;
281 1.7 mrg UVMHIST_FUNC("uvmfault_anonget"); UVMHIST_CALLED(maphist);
282 1.7 mrg
283 1.122 ad KASSERT(mutex_owned(&anon->an_lock));
284 1.53 thorpej
285 1.58 chs error = 0;
286 1.9 chuck uvmexp.fltanget++;
287 1.9 chuck /* bump rusage counters */
288 1.94 yamt if (anon->an_page)
289 1.124 ad curlwp->l_ru.ru_minflt++;
290 1.9 chuck else
291 1.124 ad curlwp->l_ru.ru_majflt++;
292 1.7 mrg
293 1.63 chs /*
294 1.7 mrg * loop until we get it, or fail.
295 1.7 mrg */
296 1.7 mrg
297 1.69 chs for (;;) {
298 1.119 thorpej we_own = false; /* true if we set PG_BUSY on a page */
299 1.94 yamt pg = anon->an_page;
300 1.1 mrg
301 1.7 mrg /*
302 1.7 mrg * if there is a resident page and it is loaned, then anon
303 1.7 mrg * may not own it. call out to uvm_anon_lockpage() to ensure
304 1.7 mrg * the real owner of the page has been identified and locked.
305 1.7 mrg */
306 1.7 mrg
307 1.7 mrg if (pg && pg->loan_count)
308 1.13 chuck pg = uvm_anon_lockloanpg(anon);
309 1.7 mrg
310 1.7 mrg /*
311 1.7 mrg * page there? make sure it is not busy/released.
312 1.7 mrg */
313 1.7 mrg
314 1.7 mrg if (pg) {
315 1.7 mrg
316 1.7 mrg /*
317 1.7 mrg * at this point, if the page has a uobject [meaning
318 1.7 mrg * we have it on loan], then that uobject is locked
319 1.7 mrg * by us! if the page is busy, we drop all the
320 1.7 mrg * locks (including uobject) and try again.
321 1.7 mrg */
322 1.7 mrg
323 1.69 chs if ((pg->flags & PG_BUSY) == 0) {
324 1.7 mrg UVMHIST_LOG(maphist, "<- OK",0,0,0,0);
325 1.57 chs return (0);
326 1.7 mrg }
327 1.7 mrg pg->flags |= PG_WANTED;
328 1.7 mrg uvmexp.fltpgwait++;
329 1.7 mrg
330 1.7 mrg /*
331 1.7 mrg * the last unlock must be an atomic unlock+wait on
332 1.7 mrg * the owner of page
333 1.7 mrg */
334 1.69 chs
335 1.7 mrg if (pg->uobject) { /* owner is uobject ? */
336 1.7 mrg uvmfault_unlockall(ufi, amap, NULL, anon);
337 1.7 mrg UVMHIST_LOG(maphist, " unlock+wait on uobj",0,
338 1.7 mrg 0,0,0);
339 1.7 mrg UVM_UNLOCK_AND_WAIT(pg,
340 1.7 mrg &pg->uobject->vmobjlock,
341 1.119 thorpej false, "anonget1",0);
342 1.7 mrg } else {
343 1.7 mrg /* anon owns page */
344 1.7 mrg uvmfault_unlockall(ufi, amap, NULL, NULL);
345 1.7 mrg UVMHIST_LOG(maphist, " unlock+wait on anon",0,
346 1.7 mrg 0,0,0);
347 1.7 mrg UVM_UNLOCK_AND_WAIT(pg,&anon->an_lock,0,
348 1.7 mrg "anonget2",0);
349 1.7 mrg }
350 1.7 mrg } else {
351 1.101 yamt #if defined(VMSWAP)
352 1.63 chs
353 1.7 mrg /*
354 1.7 mrg * no page, we must try and bring it in.
355 1.7 mrg */
356 1.69 chs
357 1.180 enami pg = uvm_pagealloc(NULL,
358 1.180 enami ufi != NULL ? ufi->orig_rvaddr : 0,
359 1.180 enami anon, UVM_FLAG_COLORMATCH);
360 1.7 mrg if (pg == NULL) { /* out of RAM. */
361 1.7 mrg uvmfault_unlockall(ufi, amap, NULL, anon);
362 1.7 mrg uvmexp.fltnoram++;
363 1.7 mrg UVMHIST_LOG(maphist, " noram -- UVM_WAIT",0,
364 1.7 mrg 0,0,0);
365 1.93 yamt if (!uvm_reclaimable()) {
366 1.93 yamt return ENOMEM;
367 1.93 yamt }
368 1.7 mrg uvm_wait("flt_noram1");
369 1.7 mrg } else {
370 1.7 mrg /* we set the PG_BUSY bit */
371 1.119 thorpej we_own = true;
372 1.7 mrg uvmfault_unlockall(ufi, amap, NULL, anon);
373 1.7 mrg
374 1.7 mrg /*
375 1.7 mrg * we are passing a PG_BUSY+PG_FAKE+PG_CLEAN
376 1.7 mrg * page into the uvm_swap_get function with
377 1.18 chuck * all data structures unlocked. note that
378 1.18 chuck * it is ok to read an_swslot here because
379 1.18 chuck * we hold PG_BUSY on the page.
380 1.7 mrg */
381 1.7 mrg uvmexp.pageins++;
382 1.58 chs error = uvm_swap_get(pg, anon->an_swslot,
383 1.7 mrg PGO_SYNCIO);
384 1.7 mrg
385 1.7 mrg /*
386 1.7 mrg * we clean up after the i/o below in the
387 1.7 mrg * "we_own" case
388 1.7 mrg */
389 1.7 mrg }
390 1.101 yamt #else /* defined(VMSWAP) */
391 1.101 yamt panic("%s: no page", __func__);
392 1.101 yamt #endif /* defined(VMSWAP) */
393 1.7 mrg }
394 1.7 mrg
395 1.7 mrg /*
396 1.7 mrg * now relock and try again
397 1.7 mrg */
398 1.7 mrg
399 1.7 mrg locked = uvmfault_relock(ufi);
400 1.47 chs if (locked && amap != NULL) {
401 1.19 chuck amap_lock(amap);
402 1.7 mrg }
403 1.7 mrg if (locked || we_own)
404 1.122 ad mutex_enter(&anon->an_lock);
405 1.7 mrg
406 1.7 mrg /*
407 1.7 mrg * if we own the page (i.e. we set PG_BUSY), then we need
408 1.7 mrg * to clean up after the I/O. there are three cases to
409 1.7 mrg * consider:
410 1.7 mrg * [1] page released during I/O: free anon and ReFault.
411 1.63 chs * [2] I/O not OK. free the page and cause the fault
412 1.7 mrg * to fail.
413 1.7 mrg * [3] I/O OK! activate the page and sync with the
414 1.7 mrg * non-we_own case (i.e. drop anon lock if not locked).
415 1.7 mrg */
416 1.63 chs
417 1.7 mrg if (we_own) {
418 1.101 yamt #if defined(VMSWAP)
419 1.7 mrg if (pg->flags & PG_WANTED) {
420 1.63 chs wakeup(pg);
421 1.7 mrg }
422 1.58 chs if (error) {
423 1.1 mrg
424 1.47 chs /*
425 1.47 chs * remove the swap slot from the anon
426 1.47 chs * and mark the anon as having no real slot.
427 1.47 chs * don't free the swap slot, thus preventing
428 1.47 chs * it from being used again.
429 1.47 chs */
430 1.69 chs
431 1.84 pk if (anon->an_swslot > 0)
432 1.84 pk uvm_swap_markbad(anon->an_swslot, 1);
433 1.47 chs anon->an_swslot = SWSLOT_BAD;
434 1.47 chs
435 1.88 yamt if ((pg->flags & PG_RELEASED) != 0)
436 1.88 yamt goto released;
437 1.88 yamt
438 1.47 chs /*
439 1.7 mrg * note: page was never !PG_BUSY, so it
440 1.7 mrg * can't be mapped and thus no need to
441 1.7 mrg * pmap_page_protect it...
442 1.7 mrg */
443 1.69 chs
444 1.122 ad mutex_enter(&uvm_pageqlock);
445 1.7 mrg uvm_pagefree(pg);
446 1.122 ad mutex_exit(&uvm_pageqlock);
447 1.7 mrg
448 1.7 mrg if (locked)
449 1.7 mrg uvmfault_unlockall(ufi, amap, NULL,
450 1.7 mrg anon);
451 1.7 mrg else
452 1.122 ad mutex_exit(&anon->an_lock);
453 1.7 mrg UVMHIST_LOG(maphist, "<- ERROR", 0,0,0,0);
454 1.58 chs return error;
455 1.7 mrg }
456 1.63 chs
457 1.88 yamt if ((pg->flags & PG_RELEASED) != 0) {
458 1.88 yamt released:
459 1.88 yamt KASSERT(anon->an_ref == 0);
460 1.88 yamt
461 1.88 yamt /*
462 1.88 yamt * released while we unlocked amap.
463 1.88 yamt */
464 1.88 yamt
465 1.88 yamt if (locked)
466 1.88 yamt uvmfault_unlockall(ufi, amap, NULL,
467 1.88 yamt NULL);
468 1.88 yamt
469 1.88 yamt uvm_anon_release(anon);
470 1.88 yamt
471 1.88 yamt if (error) {
472 1.88 yamt UVMHIST_LOG(maphist,
473 1.88 yamt "<- ERROR/RELEASED", 0,0,0,0);
474 1.88 yamt return error;
475 1.88 yamt }
476 1.88 yamt
477 1.88 yamt UVMHIST_LOG(maphist, "<- RELEASED", 0,0,0,0);
478 1.88 yamt return ERESTART;
479 1.88 yamt }
480 1.88 yamt
481 1.7 mrg /*
482 1.69 chs * we've successfully read the page, activate it.
483 1.7 mrg */
484 1.69 chs
485 1.122 ad mutex_enter(&uvm_pageqlock);
486 1.7 mrg uvm_pageactivate(pg);
487 1.122 ad mutex_exit(&uvm_pageqlock);
488 1.69 chs pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE);
489 1.69 chs UVM_PAGE_OWN(pg, NULL);
490 1.7 mrg if (!locked)
491 1.122 ad mutex_exit(&anon->an_lock);
492 1.101 yamt #else /* defined(VMSWAP) */
493 1.101 yamt panic("%s: we_own", __func__);
494 1.101 yamt #endif /* defined(VMSWAP) */
495 1.7 mrg }
496 1.7 mrg
497 1.7 mrg /*
498 1.7 mrg * we were not able to relock. restart fault.
499 1.7 mrg */
500 1.7 mrg
501 1.7 mrg if (!locked) {
502 1.7 mrg UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0);
503 1.57 chs return (ERESTART);
504 1.7 mrg }
505 1.7 mrg
506 1.7 mrg /*
507 1.7 mrg * verify no one has touched the amap and moved the anon on us.
508 1.7 mrg */
509 1.1 mrg
510 1.47 chs if (ufi != NULL &&
511 1.63 chs amap_lookup(&ufi->entry->aref,
512 1.47 chs ufi->orig_rvaddr - ufi->entry->start) != anon) {
513 1.63 chs
514 1.7 mrg uvmfault_unlockall(ufi, amap, NULL, anon);
515 1.7 mrg UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0);
516 1.57 chs return (ERESTART);
517 1.7 mrg }
518 1.63 chs
519 1.7 mrg /*
520 1.63 chs * try it again!
521 1.7 mrg */
522 1.1 mrg
523 1.7 mrg uvmexp.fltanretry++;
524 1.7 mrg continue;
525 1.69 chs }
526 1.7 mrg /*NOTREACHED*/
527 1.1 mrg }
528 1.1 mrg
529 1.1 mrg /*
530 1.106 yamt * uvmfault_promote: promote data to a new anon. used for 1B and 2B.
531 1.106 yamt *
532 1.106 yamt * 1. allocate an anon and a page.
533 1.106 yamt * 2. fill its contents.
534 1.106 yamt * 3. put it into amap.
535 1.106 yamt *
536 1.106 yamt * => if we fail (result != 0) we unlock everything.
537 1.106 yamt * => on success, return a new locked anon via 'nanon'.
538 1.106 yamt * (*nanon)->an_page will be a resident, locked, dirty page.
539 1.106 yamt */
540 1.106 yamt
541 1.106 yamt static int
542 1.106 yamt uvmfault_promote(struct uvm_faultinfo *ufi,
543 1.106 yamt struct vm_anon *oanon,
544 1.106 yamt struct vm_page *uobjpage,
545 1.106 yamt struct vm_anon **nanon, /* OUT: allocated anon */
546 1.106 yamt struct vm_anon **spare)
547 1.106 yamt {
548 1.106 yamt struct vm_amap *amap = ufi->entry->aref.ar_amap;
549 1.106 yamt struct uvm_object *uobj;
550 1.106 yamt struct vm_anon *anon;
551 1.106 yamt struct vm_page *pg;
552 1.106 yamt struct vm_page *opg;
553 1.106 yamt int error;
554 1.106 yamt UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
555 1.106 yamt
556 1.106 yamt if (oanon) {
557 1.106 yamt /* anon COW */
558 1.106 yamt opg = oanon->an_page;
559 1.106 yamt KASSERT(opg != NULL);
560 1.106 yamt KASSERT(opg->uobject == NULL || opg->loan_count > 0);
561 1.106 yamt } else if (uobjpage != PGO_DONTCARE) {
562 1.106 yamt /* object-backed COW */
563 1.106 yamt opg = uobjpage;
564 1.106 yamt } else {
565 1.106 yamt /* ZFOD */
566 1.106 yamt opg = NULL;
567 1.106 yamt }
568 1.106 yamt if (opg != NULL) {
569 1.106 yamt uobj = opg->uobject;
570 1.106 yamt } else {
571 1.106 yamt uobj = NULL;
572 1.106 yamt }
573 1.106 yamt
574 1.106 yamt KASSERT(amap != NULL);
575 1.106 yamt KASSERT(uobjpage != NULL);
576 1.106 yamt KASSERT(uobjpage == PGO_DONTCARE || (uobjpage->flags & PG_BUSY) != 0);
577 1.120 ad KASSERT(mutex_owned(&amap->am_l));
578 1.122 ad KASSERT(oanon == NULL || mutex_owned(&oanon->an_lock));
579 1.122 ad KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock));
580 1.122 ad #if 0
581 1.122 ad KASSERT(*spare == NULL || !mutex_owned(&(*spare)->an_lock));
582 1.122 ad #endif
583 1.106 yamt
584 1.106 yamt if (*spare != NULL) {
585 1.106 yamt anon = *spare;
586 1.106 yamt *spare = NULL;
587 1.122 ad mutex_enter(&anon->an_lock);
588 1.106 yamt } else if (ufi->map != kernel_map) {
589 1.106 yamt anon = uvm_analloc();
590 1.106 yamt } else {
591 1.106 yamt UVMHIST_LOG(maphist, "kernel_map, unlock and retry", 0,0,0,0);
592 1.106 yamt
593 1.106 yamt /*
594 1.106 yamt * we can't allocate anons with kernel_map locked.
595 1.106 yamt */
596 1.106 yamt
597 1.106 yamt uvm_page_unbusy(&uobjpage, 1);
598 1.106 yamt uvmfault_unlockall(ufi, amap, uobj, oanon);
599 1.106 yamt
600 1.106 yamt *spare = uvm_analloc();
601 1.106 yamt if (*spare == NULL) {
602 1.106 yamt goto nomem;
603 1.106 yamt }
604 1.122 ad mutex_exit(&(*spare)->an_lock);
605 1.106 yamt error = ERESTART;
606 1.106 yamt goto done;
607 1.106 yamt }
608 1.106 yamt if (anon) {
609 1.106 yamt
610 1.106 yamt /*
611 1.106 yamt * The new anon is locked.
612 1.106 yamt *
613 1.106 yamt * if opg == NULL, we want a zero'd, dirty page,
614 1.106 yamt * so have uvm_pagealloc() do that for us.
615 1.106 yamt */
616 1.106 yamt
617 1.179 matt pg = uvm_pagealloc(NULL, ufi->orig_rvaddr, anon,
618 1.179 matt UVM_FLAG_COLORMATCH | (opg == NULL ? UVM_PGA_ZERO : 0));
619 1.106 yamt } else {
620 1.106 yamt pg = NULL;
621 1.106 yamt }
622 1.106 yamt
623 1.106 yamt /*
624 1.106 yamt * out of memory resources?
625 1.106 yamt */
626 1.106 yamt
627 1.106 yamt if (pg == NULL) {
628 1.106 yamt /* save anon for the next try. */
629 1.106 yamt if (anon != NULL) {
630 1.122 ad mutex_exit(&anon->an_lock);
631 1.106 yamt *spare = anon;
632 1.106 yamt }
633 1.106 yamt
634 1.106 yamt /* unlock and fail ... */
635 1.106 yamt uvm_page_unbusy(&uobjpage, 1);
636 1.106 yamt uvmfault_unlockall(ufi, amap, uobj, oanon);
637 1.106 yamt nomem:
638 1.106 yamt if (!uvm_reclaimable()) {
639 1.106 yamt UVMHIST_LOG(maphist, "out of VM", 0,0,0,0);
640 1.106 yamt uvmexp.fltnoanon++;
641 1.106 yamt error = ENOMEM;
642 1.106 yamt goto done;
643 1.106 yamt }
644 1.106 yamt
645 1.106 yamt UVMHIST_LOG(maphist, "out of RAM, waiting for more", 0,0,0,0);
646 1.106 yamt uvmexp.fltnoram++;
647 1.106 yamt uvm_wait("flt_noram5");
648 1.106 yamt error = ERESTART;
649 1.106 yamt goto done;
650 1.106 yamt }
651 1.106 yamt
652 1.106 yamt /* copy page [pg now dirty] */
653 1.106 yamt if (opg) {
654 1.106 yamt uvm_pagecopy(opg, pg);
655 1.106 yamt }
656 1.106 yamt
657 1.106 yamt amap_add(&ufi->entry->aref, ufi->orig_rvaddr - ufi->entry->start, anon,
658 1.106 yamt oanon != NULL);
659 1.106 yamt
660 1.106 yamt *nanon = anon;
661 1.106 yamt error = 0;
662 1.106 yamt done:
663 1.106 yamt return error;
664 1.106 yamt }
665 1.106 yamt
666 1.106 yamt
667 1.106 yamt /*
668 1.1 mrg * F A U L T - m a i n e n t r y p o i n t
669 1.1 mrg */
670 1.1 mrg
671 1.1 mrg /*
672 1.1 mrg * uvm_fault: page fault handler
673 1.1 mrg *
674 1.1 mrg * => called from MD code to resolve a page fault
675 1.63 chs * => VM data structures usually should be unlocked. however, it is
676 1.1 mrg * possible to call here with the main map locked if the caller
677 1.1 mrg * gets a write lock, sets it recusive, and then calls us (c.f.
678 1.1 mrg * uvm_map_pageable). this should be avoided because it keeps
679 1.1 mrg * the map locked off during I/O.
680 1.66 thorpej * => MUST NEVER BE CALLED IN INTERRUPT CONTEXT
681 1.1 mrg */
682 1.1 mrg
683 1.24 mycroft #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \
684 1.24 mycroft ~VM_PROT_WRITE : VM_PROT_ALL)
685 1.24 mycroft
686 1.110 drochner /* fault_flag values passed from uvm_fault_wire to uvm_fault_internal */
687 1.130 uebayasi #define UVM_FAULT_WIRE (1 << 0)
688 1.130 uebayasi #define UVM_FAULT_MAXPROT (1 << 1)
689 1.110 drochner
690 1.140 uebayasi struct uvm_faultctx {
691 1.140 uebayasi vm_prot_t access_type;
692 1.140 uebayasi vm_prot_t enter_prot;
693 1.150 uebayasi vaddr_t startva;
694 1.150 uebayasi int npages;
695 1.150 uebayasi int centeridx;
696 1.150 uebayasi struct vm_anon *anon_spare;
697 1.146 uebayasi bool wire_mapping;
698 1.140 uebayasi bool narrow;
699 1.146 uebayasi bool wire_paging;
700 1.140 uebayasi bool cow_now;
701 1.168 uebayasi bool promote;
702 1.140 uebayasi };
703 1.140 uebayasi
704 1.163 uebayasi static inline int uvm_fault_check(
705 1.163 uebayasi struct uvm_faultinfo *, struct uvm_faultctx *,
706 1.177 yamt struct vm_anon ***, bool);
707 1.163 uebayasi
708 1.163 uebayasi static int uvm_fault_upper(
709 1.163 uebayasi struct uvm_faultinfo *, struct uvm_faultctx *,
710 1.163 uebayasi struct vm_anon **);
711 1.163 uebayasi static inline int uvm_fault_upper_lookup(
712 1.177 yamt struct uvm_faultinfo *, const struct uvm_faultctx *,
713 1.163 uebayasi struct vm_anon **, struct vm_page **);
714 1.163 uebayasi static inline void uvm_fault_upper_neighbor(
715 1.177 yamt struct uvm_faultinfo *, const struct uvm_faultctx *,
716 1.163 uebayasi vaddr_t, struct vm_page *, bool);
717 1.163 uebayasi static inline int uvm_fault_upper_loan(
718 1.163 uebayasi struct uvm_faultinfo *, struct uvm_faultctx *,
719 1.163 uebayasi struct vm_anon *, struct uvm_object **);
720 1.163 uebayasi static inline int uvm_fault_upper_promote(
721 1.163 uebayasi struct uvm_faultinfo *, struct uvm_faultctx *,
722 1.163 uebayasi struct uvm_object *, struct vm_anon *);
723 1.163 uebayasi static inline int uvm_fault_upper_direct(
724 1.163 uebayasi struct uvm_faultinfo *, struct uvm_faultctx *,
725 1.163 uebayasi struct uvm_object *, struct vm_anon *);
726 1.163 uebayasi static int uvm_fault_upper_enter(
727 1.177 yamt struct uvm_faultinfo *, const struct uvm_faultctx *,
728 1.163 uebayasi struct uvm_object *, struct vm_anon *,
729 1.163 uebayasi struct vm_page *, struct vm_anon *);
730 1.169 uebayasi static inline void uvm_fault_upper_done(
731 1.177 yamt struct uvm_faultinfo *, const struct uvm_faultctx *,
732 1.177 yamt struct vm_anon *, struct vm_page *);
733 1.163 uebayasi
734 1.163 uebayasi static int uvm_fault_lower(
735 1.163 uebayasi struct uvm_faultinfo *, struct uvm_faultctx *,
736 1.163 uebayasi struct vm_page **);
737 1.173 uebayasi static inline void uvm_fault_lower_lookup(
738 1.177 yamt struct uvm_faultinfo *, const struct uvm_faultctx *,
739 1.163 uebayasi struct vm_page **);
740 1.163 uebayasi static inline void uvm_fault_lower_neighbor(
741 1.177 yamt struct uvm_faultinfo *, const struct uvm_faultctx *,
742 1.163 uebayasi vaddr_t, struct vm_page *, bool);
743 1.163 uebayasi static inline int uvm_fault_lower_io(
744 1.177 yamt struct uvm_faultinfo *, const struct uvm_faultctx *,
745 1.163 uebayasi struct uvm_object **, struct vm_page **);
746 1.163 uebayasi static inline int uvm_fault_lower_direct(
747 1.163 uebayasi struct uvm_faultinfo *, struct uvm_faultctx *,
748 1.163 uebayasi struct uvm_object *, struct vm_page *);
749 1.163 uebayasi static inline int uvm_fault_lower_direct_loan(
750 1.163 uebayasi struct uvm_faultinfo *, struct uvm_faultctx *,
751 1.163 uebayasi struct uvm_object *, struct vm_page **,
752 1.163 uebayasi struct vm_page **);
753 1.163 uebayasi static inline int uvm_fault_lower_promote(
754 1.163 uebayasi struct uvm_faultinfo *, struct uvm_faultctx *,
755 1.163 uebayasi struct uvm_object *, struct vm_page *);
756 1.163 uebayasi static int uvm_fault_lower_enter(
757 1.177 yamt struct uvm_faultinfo *, const struct uvm_faultctx *,
758 1.163 uebayasi struct uvm_object *,
759 1.163 uebayasi struct vm_anon *, struct vm_page *,
760 1.163 uebayasi struct vm_page *);
761 1.169 uebayasi static inline void uvm_fault_lower_done(
762 1.177 yamt struct uvm_faultinfo *, const struct uvm_faultctx *,
763 1.177 yamt struct uvm_object *, struct vm_page *);
764 1.138 uebayasi
765 1.7 mrg int
766 1.110 drochner uvm_fault_internal(struct vm_map *orig_map, vaddr_t vaddr,
767 1.110 drochner vm_prot_t access_type, int fault_flag)
768 1.1 mrg {
769 1.7 mrg struct uvm_faultinfo ufi;
770 1.140 uebayasi struct uvm_faultctx flt = {
771 1.140 uebayasi .access_type = access_type,
772 1.146 uebayasi
773 1.146 uebayasi /* don't look for neighborhood * pages on "wire" fault */
774 1.146 uebayasi .narrow = (fault_flag & UVM_FAULT_WIRE) != 0,
775 1.146 uebayasi
776 1.146 uebayasi /* "wire" fault causes wiring of both mapping and paging */
777 1.146 uebayasi .wire_mapping = (fault_flag & UVM_FAULT_WIRE) != 0,
778 1.146 uebayasi .wire_paging = (fault_flag & UVM_FAULT_WIRE) != 0,
779 1.140 uebayasi };
780 1.177 yamt const bool maxprot = (fault_flag & UVM_FAULT_MAXPROT) != 0;
781 1.137 uebayasi struct vm_anon *anons_store[UVM_MAXRANGE], **anons;
782 1.141 uebayasi struct vm_page *pages_store[UVM_MAXRANGE], **pages;
783 1.140 uebayasi int error;
784 1.7 mrg UVMHIST_FUNC("uvm_fault"); UVMHIST_CALLED(maphist);
785 1.1 mrg
786 1.110 drochner UVMHIST_LOG(maphist, "(map=0x%x, vaddr=0x%x, at=%d, ff=%d)",
787 1.110 drochner orig_map, vaddr, access_type, fault_flag);
788 1.1 mrg
789 1.178 matt curcpu()->ci_data.cpu_nfault++;
790 1.7 mrg
791 1.7 mrg /*
792 1.7 mrg * init the IN parameters in the ufi
793 1.7 mrg */
794 1.1 mrg
795 1.7 mrg ufi.orig_map = orig_map;
796 1.7 mrg ufi.orig_rvaddr = trunc_page(vaddr);
797 1.7 mrg ufi.orig_size = PAGE_SIZE; /* can't get any smaller than this */
798 1.7 mrg
799 1.142 uebayasi error = ERESTART;
800 1.142 uebayasi while (error == ERESTART) {
801 1.143 uebayasi anons = anons_store;
802 1.143 uebayasi pages = pages_store;
803 1.1 mrg
804 1.177 yamt error = uvm_fault_check(&ufi, &flt, &anons, maxprot);
805 1.143 uebayasi if (error != 0)
806 1.143 uebayasi continue;
807 1.141 uebayasi
808 1.143 uebayasi error = uvm_fault_upper_lookup(&ufi, &flt, anons, pages);
809 1.143 uebayasi if (error != 0)
810 1.143 uebayasi continue;
811 1.138 uebayasi
812 1.144 uebayasi if (pages[flt.centeridx] == PGO_DONTCARE)
813 1.148 uebayasi error = uvm_fault_upper(&ufi, &flt, anons);
814 1.167 uebayasi else {
815 1.177 yamt struct uvm_object * const uobj =
816 1.177 yamt ufi.entry->object.uvm_obj;
817 1.167 uebayasi
818 1.167 uebayasi if (uobj && uobj->pgops->pgo_fault != NULL) {
819 1.173 uebayasi /*
820 1.173 uebayasi * invoke "special" fault routine.
821 1.173 uebayasi */
822 1.167 uebayasi mutex_enter(&uobj->vmobjlock);
823 1.173 uebayasi /* locked: maps(read), amap(if there), uobj */
824 1.173 uebayasi error = uobj->pgops->pgo_fault(&ufi,
825 1.173 uebayasi flt.startva, pages, flt.npages,
826 1.173 uebayasi flt.centeridx, flt.access_type,
827 1.173 uebayasi PGO_LOCKED|PGO_SYNCIO);
828 1.167 uebayasi
829 1.177 yamt /*
830 1.177 yamt * locked: nothing, pgo_fault has unlocked
831 1.177 yamt * everything
832 1.177 yamt */
833 1.167 uebayasi
834 1.167 uebayasi /*
835 1.177 yamt * object fault routine responsible for
836 1.177 yamt * pmap_update().
837 1.167 uebayasi */
838 1.167 uebayasi } else {
839 1.167 uebayasi error = uvm_fault_lower(&ufi, &flt, pages);
840 1.167 uebayasi }
841 1.167 uebayasi }
842 1.142 uebayasi }
843 1.138 uebayasi
844 1.140 uebayasi if (flt.anon_spare != NULL) {
845 1.140 uebayasi flt.anon_spare->an_ref--;
846 1.140 uebayasi uvm_anfree(flt.anon_spare);
847 1.138 uebayasi }
848 1.138 uebayasi return error;
849 1.141 uebayasi }
850 1.138 uebayasi
851 1.173 uebayasi /*
852 1.173 uebayasi * uvm_fault_check: check prot, handle needs-copy, etc.
853 1.173 uebayasi *
854 1.173 uebayasi * 1. lookup entry.
855 1.173 uebayasi * 2. check protection.
856 1.173 uebayasi * 3. adjust fault condition (mainly for simulated fault).
857 1.173 uebayasi * 4. handle needs-copy (lazy amap copy).
858 1.173 uebayasi * 5. establish range of interest for neighbor fault (aka pre-fault).
859 1.173 uebayasi * 6. look up anons (if amap exists).
860 1.173 uebayasi * 7. flush pages (if MADV_SEQUENTIAL)
861 1.173 uebayasi *
862 1.173 uebayasi * => called with nothing locked.
863 1.173 uebayasi * => if we fail (result != 0) we unlock everything.
864 1.177 yamt * => initialize/adjust many members of flt.
865 1.173 uebayasi */
866 1.173 uebayasi
867 1.144 uebayasi static int
868 1.141 uebayasi uvm_fault_check(
869 1.141 uebayasi struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
870 1.177 yamt struct vm_anon ***ranons, bool maxprot)
871 1.141 uebayasi {
872 1.141 uebayasi struct vm_amap *amap;
873 1.141 uebayasi struct uvm_object *uobj;
874 1.137 uebayasi vm_prot_t check_prot;
875 1.137 uebayasi int nback, nforw;
876 1.164 mlelstv UVMHIST_FUNC("uvm_fault_check"); UVMHIST_CALLED(maphist);
877 1.137 uebayasi
878 1.7 mrg /*
879 1.7 mrg * lookup and lock the maps
880 1.7 mrg */
881 1.7 mrg
882 1.141 uebayasi if (uvmfault_lookup(ufi, false) == false) {
883 1.177 yamt UVMHIST_LOG(maphist, "<- no mapping @ 0x%x", ufi->orig_rvaddr,
884 1.177 yamt 0,0,0);
885 1.141 uebayasi return EFAULT;
886 1.7 mrg }
887 1.7 mrg /* locked: maps(read) */
888 1.7 mrg
889 1.61 thorpej #ifdef DIAGNOSTIC
890 1.141 uebayasi if ((ufi->map->flags & VM_MAP_PAGEABLE) == 0) {
891 1.61 thorpej printf("Page fault on non-pageable map:\n");
892 1.141 uebayasi printf("ufi->map = %p\n", ufi->map);
893 1.141 uebayasi printf("ufi->orig_map = %p\n", ufi->orig_map);
894 1.141 uebayasi printf("ufi->orig_rvaddr = 0x%lx\n", (u_long) ufi->orig_rvaddr);
895 1.141 uebayasi panic("uvm_fault: (ufi->map->flags & VM_MAP_PAGEABLE) == 0");
896 1.61 thorpej }
897 1.61 thorpej #endif
898 1.58 chs
899 1.7 mrg /*
900 1.7 mrg * check protection
901 1.7 mrg */
902 1.7 mrg
903 1.177 yamt check_prot = maxprot ?
904 1.141 uebayasi ufi->entry->max_protection : ufi->entry->protection;
905 1.141 uebayasi if ((check_prot & flt->access_type) != flt->access_type) {
906 1.7 mrg UVMHIST_LOG(maphist,
907 1.7 mrg "<- protection failure (prot=0x%x, access=0x%x)",
908 1.141 uebayasi ufi->entry->protection, flt->access_type, 0, 0);
909 1.141 uebayasi uvmfault_unlockmaps(ufi, false);
910 1.141 uebayasi return EACCES;
911 1.7 mrg }
912 1.7 mrg
913 1.7 mrg /*
914 1.7 mrg * "enter_prot" is the protection we want to enter the page in at.
915 1.7 mrg * for certain pages (e.g. copy-on-write pages) this protection can
916 1.141 uebayasi * be more strict than ufi->entry->protection. "wired" means either
917 1.7 mrg * the entry is wired or we are fault-wiring the pg.
918 1.7 mrg */
919 1.7 mrg
920 1.141 uebayasi flt->enter_prot = ufi->entry->protection;
921 1.146 uebayasi if (VM_MAPENT_ISWIRED(ufi->entry))
922 1.146 uebayasi flt->wire_mapping = true;
923 1.146 uebayasi
924 1.146 uebayasi if (flt->wire_mapping) {
925 1.141 uebayasi flt->access_type = flt->enter_prot; /* full access for wired */
926 1.141 uebayasi flt->cow_now = (check_prot & VM_PROT_WRITE) != 0;
927 1.73 chs } else {
928 1.141 uebayasi flt->cow_now = (flt->access_type & VM_PROT_WRITE) != 0;
929 1.73 chs }
930 1.7 mrg
931 1.168 uebayasi flt->promote = false;
932 1.168 uebayasi
933 1.7 mrg /*
934 1.7 mrg * handle "needs_copy" case. if we need to copy the amap we will
935 1.7 mrg * have to drop our readlock and relock it with a write lock. (we
936 1.7 mrg * need a write lock to change anything in a map entry [e.g.
937 1.7 mrg * needs_copy]).
938 1.7 mrg */
939 1.7 mrg
940 1.141 uebayasi if (UVM_ET_ISNEEDSCOPY(ufi->entry)) {
941 1.141 uebayasi if (flt->cow_now || (ufi->entry->object.uvm_obj == NULL)) {
942 1.177 yamt KASSERT(!maxprot);
943 1.7 mrg /* need to clear */
944 1.7 mrg UVMHIST_LOG(maphist,
945 1.7 mrg " need to clear needs_copy and refault",0,0,0,0);
946 1.141 uebayasi uvmfault_unlockmaps(ufi, false);
947 1.141 uebayasi uvmfault_amapcopy(ufi);
948 1.7 mrg uvmexp.fltamcopy++;
949 1.141 uebayasi return ERESTART;
950 1.7 mrg
951 1.7 mrg } else {
952 1.7 mrg
953 1.7 mrg /*
954 1.7 mrg * ensure that we pmap_enter page R/O since
955 1.7 mrg * needs_copy is still true
956 1.7 mrg */
957 1.72 chs
958 1.141 uebayasi flt->enter_prot &= ~VM_PROT_WRITE;
959 1.7 mrg }
960 1.7 mrg }
961 1.7 mrg
962 1.7 mrg /*
963 1.7 mrg * identify the players
964 1.7 mrg */
965 1.7 mrg
966 1.141 uebayasi amap = ufi->entry->aref.ar_amap; /* upper layer */
967 1.141 uebayasi uobj = ufi->entry->object.uvm_obj; /* lower layer */
968 1.7 mrg
969 1.7 mrg /*
970 1.7 mrg * check for a case 0 fault. if nothing backing the entry then
971 1.7 mrg * error now.
972 1.7 mrg */
973 1.7 mrg
974 1.7 mrg if (amap == NULL && uobj == NULL) {
975 1.141 uebayasi uvmfault_unlockmaps(ufi, false);
976 1.7 mrg UVMHIST_LOG(maphist,"<- no backing store, no overlay",0,0,0,0);
977 1.141 uebayasi return EFAULT;
978 1.7 mrg }
979 1.1 mrg
980 1.7 mrg /*
981 1.7 mrg * establish range of interest based on advice from mapper
982 1.7 mrg * and then clip to fit map entry. note that we only want
983 1.63 chs * to do this the first time through the fault. if we
984 1.7 mrg * ReFault we will disable this by setting "narrow" to true.
985 1.7 mrg */
986 1.1 mrg
987 1.141 uebayasi if (flt->narrow == false) {
988 1.7 mrg
989 1.7 mrg /* wide fault (!narrow) */
990 1.141 uebayasi KASSERT(uvmadvice[ufi->entry->advice].advice ==
991 1.141 uebayasi ufi->entry->advice);
992 1.141 uebayasi nback = MIN(uvmadvice[ufi->entry->advice].nback,
993 1.177 yamt (ufi->orig_rvaddr - ufi->entry->start) >> PAGE_SHIFT);
994 1.141 uebayasi flt->startva = ufi->orig_rvaddr - (nback << PAGE_SHIFT);
995 1.7 mrg /*
996 1.7 mrg * note: "-1" because we don't want to count the
997 1.7 mrg * faulting page as forw
998 1.7 mrg */
999 1.177 yamt nforw = MIN(uvmadvice[ufi->entry->advice].nforw,
1000 1.177 yamt ((ufi->entry->end - ufi->orig_rvaddr) >>
1001 1.177 yamt PAGE_SHIFT) - 1);
1002 1.141 uebayasi flt->npages = nback + nforw + 1;
1003 1.141 uebayasi flt->centeridx = nback;
1004 1.7 mrg
1005 1.141 uebayasi flt->narrow = true; /* ensure only once per-fault */
1006 1.7 mrg
1007 1.7 mrg } else {
1008 1.63 chs
1009 1.7 mrg /* narrow fault! */
1010 1.7 mrg nback = nforw = 0;
1011 1.141 uebayasi flt->startva = ufi->orig_rvaddr;
1012 1.141 uebayasi flt->npages = 1;
1013 1.141 uebayasi flt->centeridx = 0;
1014 1.1 mrg
1015 1.7 mrg }
1016 1.131 uebayasi /* offset from entry's start to pgs' start */
1017 1.141 uebayasi const voff_t eoff = flt->startva - ufi->entry->start;
1018 1.1 mrg
1019 1.7 mrg /* locked: maps(read) */
1020 1.13 chuck UVMHIST_LOG(maphist, " narrow=%d, back=%d, forw=%d, startva=0x%x",
1021 1.141 uebayasi flt->narrow, nback, nforw, flt->startva);
1022 1.141 uebayasi UVMHIST_LOG(maphist, " entry=0x%x, amap=0x%x, obj=0x%x", ufi->entry,
1023 1.16 chs amap, uobj, 0);
1024 1.1 mrg
1025 1.7 mrg /*
1026 1.7 mrg * if we've got an amap, lock it and extract current anons.
1027 1.7 mrg */
1028 1.7 mrg
1029 1.7 mrg if (amap) {
1030 1.19 chuck amap_lock(amap);
1031 1.141 uebayasi amap_lookups(&ufi->entry->aref, eoff, *ranons, flt->npages);
1032 1.7 mrg } else {
1033 1.141 uebayasi *ranons = NULL; /* to be safe */
1034 1.7 mrg }
1035 1.7 mrg
1036 1.7 mrg /* locked: maps(read), amap(if there) */
1037 1.120 ad KASSERT(amap == NULL || mutex_owned(&amap->am_l));
1038 1.7 mrg
1039 1.7 mrg /*
1040 1.7 mrg * for MADV_SEQUENTIAL mappings we want to deactivate the back pages
1041 1.7 mrg * now and then forget about them (for the rest of the fault).
1042 1.7 mrg */
1043 1.7 mrg
1044 1.141 uebayasi if (ufi->entry->advice == MADV_SEQUENTIAL && nback != 0) {
1045 1.7 mrg
1046 1.7 mrg UVMHIST_LOG(maphist, " MADV_SEQUENTIAL: flushing backpages",
1047 1.7 mrg 0,0,0,0);
1048 1.7 mrg /* flush back-page anons? */
1049 1.63 chs if (amap)
1050 1.141 uebayasi uvmfault_anonflush(*ranons, nback);
1051 1.7 mrg
1052 1.7 mrg /* flush object? */
1053 1.7 mrg if (uobj) {
1054 1.137 uebayasi voff_t uoff;
1055 1.137 uebayasi
1056 1.141 uebayasi uoff = ufi->entry->offset + eoff;
1057 1.122 ad mutex_enter(&uobj->vmobjlock);
1058 1.90 yamt (void) (uobj->pgops->pgo_put)(uobj, uoff, uoff +
1059 1.15 chs (nback << PAGE_SHIFT), PGO_DEACTIVATE);
1060 1.7 mrg }
1061 1.7 mrg
1062 1.7 mrg /* now forget about the backpages */
1063 1.7 mrg if (amap)
1064 1.141 uebayasi *ranons += nback;
1065 1.141 uebayasi flt->startva += (nback << PAGE_SHIFT);
1066 1.141 uebayasi flt->npages -= nback;
1067 1.141 uebayasi flt->centeridx = 0;
1068 1.7 mrg }
1069 1.137 uebayasi /*
1070 1.137 uebayasi * => startva is fixed
1071 1.137 uebayasi * => npages is fixed
1072 1.137 uebayasi */
1073 1.177 yamt KASSERT(flt->startva <= ufi->orig_rvaddr);
1074 1.177 yamt KASSERT(ufi->orig_rvaddr + ufi->orig_size <=
1075 1.177 yamt flt->startva + (flt->npages << PAGE_SHIFT));
1076 1.141 uebayasi return 0;
1077 1.141 uebayasi }
1078 1.141 uebayasi
1079 1.173 uebayasi /*
1080 1.173 uebayasi * uvm_fault_upper_lookup: look up existing h/w mapping and amap.
1081 1.173 uebayasi *
1082 1.173 uebayasi * iterate range of interest:
1083 1.173 uebayasi * 1. check if h/w mapping exists. if yes, we don't care
1084 1.173 uebayasi * 2. check if anon exists. if not, page is lower.
1085 1.173 uebayasi * 3. if anon exists, enter h/w mapping for neighbors.
1086 1.173 uebayasi *
1087 1.173 uebayasi * => called with amap locked (if exists).
1088 1.173 uebayasi */
1089 1.173 uebayasi
1090 1.144 uebayasi static int
1091 1.141 uebayasi uvm_fault_upper_lookup(
1092 1.177 yamt struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
1093 1.141 uebayasi struct vm_anon **anons, struct vm_page **pages)
1094 1.141 uebayasi {
1095 1.141 uebayasi struct vm_amap *amap = ufi->entry->aref.ar_amap;
1096 1.137 uebayasi int lcv;
1097 1.137 uebayasi vaddr_t currva;
1098 1.144 uebayasi bool shadowed;
1099 1.164 mlelstv UVMHIST_FUNC("uvm_fault_upper_lookup"); UVMHIST_CALLED(maphist);
1100 1.7 mrg
1101 1.7 mrg /* locked: maps(read), amap(if there) */
1102 1.120 ad KASSERT(amap == NULL || mutex_owned(&amap->am_l));
1103 1.1 mrg
1104 1.7 mrg /*
1105 1.7 mrg * map in the backpages and frontpages we found in the amap in hopes
1106 1.7 mrg * of preventing future faults. we also init the pages[] array as
1107 1.7 mrg * we go.
1108 1.7 mrg */
1109 1.7 mrg
1110 1.141 uebayasi currva = flt->startva;
1111 1.144 uebayasi shadowed = false;
1112 1.163 uebayasi for (lcv = 0; lcv < flt->npages; lcv++, currva += PAGE_SIZE) {
1113 1.7 mrg /*
1114 1.177 yamt * don't play with VAs that are already mapped
1115 1.177 yamt * (except for center)
1116 1.7 mrg */
1117 1.141 uebayasi if (lcv != flt->centeridx &&
1118 1.141 uebayasi pmap_extract(ufi->orig_map->pmap, currva, NULL)) {
1119 1.52 chs pages[lcv] = PGO_DONTCARE;
1120 1.52 chs continue;
1121 1.7 mrg }
1122 1.7 mrg
1123 1.7 mrg /*
1124 1.7 mrg * unmapped or center page. check if any anon at this level.
1125 1.7 mrg */
1126 1.7 mrg if (amap == NULL || anons[lcv] == NULL) {
1127 1.7 mrg pages[lcv] = NULL;
1128 1.7 mrg continue;
1129 1.7 mrg }
1130 1.7 mrg
1131 1.7 mrg /*
1132 1.7 mrg * check for present page and map if possible. re-activate it.
1133 1.7 mrg */
1134 1.7 mrg
1135 1.7 mrg pages[lcv] = PGO_DONTCARE;
1136 1.177 yamt if (lcv == flt->centeridx) { /* save center for later! */
1137 1.144 uebayasi shadowed = true;
1138 1.151 uebayasi } else {
1139 1.161 uebayasi struct vm_anon *anon = anons[lcv];
1140 1.161 uebayasi
1141 1.161 uebayasi mutex_enter(&anon->an_lock);
1142 1.172 uebayasi struct vm_page *pg = anon->an_page;
1143 1.172 uebayasi
1144 1.172 uebayasi /* ignore loaned and busy pages */
1145 1.172 uebayasi if (pg != NULL && pg->loan_count == 0 &&
1146 1.172 uebayasi (pg->flags & PG_BUSY) == 0)
1147 1.172 uebayasi uvm_fault_upper_neighbor(ufi, flt, currva,
1148 1.172 uebayasi pg, anon->an_ref > 1);
1149 1.161 uebayasi mutex_exit(&anon->an_lock);
1150 1.7 mrg }
1151 1.151 uebayasi }
1152 1.151 uebayasi
1153 1.160 uebayasi /* locked: maps(read), amap(if there) */
1154 1.160 uebayasi KASSERT(amap == NULL || mutex_owned(&amap->am_l));
1155 1.160 uebayasi /* (shadowed == true) if there is an anon at the faulting address */
1156 1.160 uebayasi UVMHIST_LOG(maphist, " shadowed=%d, will_get=%d", shadowed,
1157 1.164 mlelstv (ufi->entry->object.uvm_obj && shadowed != false),0,0);
1158 1.160 uebayasi
1159 1.160 uebayasi /*
1160 1.160 uebayasi * note that if we are really short of RAM we could sleep in the above
1161 1.160 uebayasi * call to pmap_enter with everything locked. bad?
1162 1.160 uebayasi *
1163 1.160 uebayasi * XXX Actually, that is bad; pmap_enter() should just fail in that
1164 1.160 uebayasi * XXX case. --thorpej
1165 1.160 uebayasi */
1166 1.151 uebayasi
1167 1.151 uebayasi return 0;
1168 1.151 uebayasi }
1169 1.151 uebayasi
1170 1.173 uebayasi /*
1171 1.173 uebayasi * uvm_fault_upper_neighbor: enter single lower neighbor page.
1172 1.173 uebayasi *
1173 1.173 uebayasi * => called with amap and anon locked.
1174 1.173 uebayasi */
1175 1.173 uebayasi
1176 1.151 uebayasi static void
1177 1.163 uebayasi uvm_fault_upper_neighbor(
1178 1.177 yamt struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
1179 1.161 uebayasi vaddr_t currva, struct vm_page *pg, bool readonly)
1180 1.151 uebayasi {
1181 1.164 mlelstv UVMHIST_FUNC("uvm_fault_upper_neighbor"); UVMHIST_CALLED(maphist);
1182 1.151 uebayasi
1183 1.173 uebayasi /* locked: amap, anon */
1184 1.173 uebayasi
1185 1.152 uebayasi mutex_enter(&uvm_pageqlock);
1186 1.161 uebayasi uvm_pageenqueue(pg);
1187 1.152 uebayasi mutex_exit(&uvm_pageqlock);
1188 1.152 uebayasi UVMHIST_LOG(maphist,
1189 1.152 uebayasi " MAPPING: n anon: pm=0x%x, va=0x%x, pg=0x%x",
1190 1.161 uebayasi ufi->orig_map->pmap, currva, pg, 0);
1191 1.152 uebayasi uvmexp.fltnamap++;
1192 1.152 uebayasi
1193 1.152 uebayasi /*
1194 1.161 uebayasi * Since this page isn't the page that's actually faulting,
1195 1.161 uebayasi * ignore pmap_enter() failures; it's not critical that we
1196 1.161 uebayasi * enter these right now.
1197 1.152 uebayasi */
1198 1.152 uebayasi
1199 1.152 uebayasi (void) pmap_enter(ufi->orig_map->pmap, currva,
1200 1.161 uebayasi VM_PAGE_TO_PHYS(pg),
1201 1.161 uebayasi readonly ? (flt->enter_prot & ~VM_PROT_WRITE) :
1202 1.152 uebayasi flt->enter_prot,
1203 1.154 uebayasi PMAP_CANFAIL | (flt->wire_mapping ? PMAP_WIRED : 0));
1204 1.52 chs
1205 1.152 uebayasi pmap_update(ufi->orig_map->pmap);
1206 1.151 uebayasi }
1207 1.151 uebayasi
1208 1.173 uebayasi /*
1209 1.173 uebayasi * uvm_fault_upper: handle upper fault.
1210 1.173 uebayasi *
1211 1.173 uebayasi * 1. acquire anon lock.
1212 1.173 uebayasi * 2. get anon. let uvmfault_anonget do the dirty work.
1213 1.173 uebayasi * 3. handle loan.
1214 1.173 uebayasi * 4. dispatch direct or promote handlers.
1215 1.173 uebayasi */
1216 1.134 uebayasi
1217 1.138 uebayasi static int
1218 1.138 uebayasi uvm_fault_upper(
1219 1.140 uebayasi struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1220 1.148 uebayasi struct vm_anon **anons)
1221 1.138 uebayasi {
1222 1.148 uebayasi struct vm_amap * const amap = ufi->entry->aref.ar_amap;
1223 1.148 uebayasi struct vm_anon * const anon = anons[flt->centeridx];
1224 1.148 uebayasi struct uvm_object *uobj;
1225 1.138 uebayasi int error;
1226 1.164 mlelstv UVMHIST_FUNC("uvm_fault_upper"); UVMHIST_CALLED(maphist);
1227 1.137 uebayasi
1228 1.7 mrg /* locked: maps(read), amap */
1229 1.133 uebayasi KASSERT(mutex_owned(&amap->am_l));
1230 1.7 mrg
1231 1.7 mrg /*
1232 1.7 mrg * handle case 1: fault on an anon in our amap
1233 1.7 mrg */
1234 1.7 mrg
1235 1.7 mrg UVMHIST_LOG(maphist, " case 1 fault: anon=0x%x", anon, 0,0,0);
1236 1.122 ad mutex_enter(&anon->an_lock);
1237 1.7 mrg
1238 1.7 mrg /* locked: maps(read), amap, anon */
1239 1.120 ad KASSERT(mutex_owned(&amap->am_l));
1240 1.122 ad KASSERT(mutex_owned(&anon->an_lock));
1241 1.7 mrg
1242 1.7 mrg /*
1243 1.7 mrg * no matter if we have case 1A or case 1B we are going to need to
1244 1.7 mrg * have the anon's memory resident. ensure that now.
1245 1.7 mrg */
1246 1.7 mrg
1247 1.7 mrg /*
1248 1.47 chs * let uvmfault_anonget do the dirty work.
1249 1.51 thorpej * if it fails (!OK) it will unlock everything for us.
1250 1.47 chs * if it succeeds, locks are still valid and locked.
1251 1.7 mrg * also, if it is OK, then the anon's page is on the queues.
1252 1.7 mrg * if the page is on loan from a uvm_object, then anonget will
1253 1.7 mrg * lock that object for us if it does not fail.
1254 1.7 mrg */
1255 1.7 mrg
1256 1.138 uebayasi error = uvmfault_anonget(ufi, amap, anon);
1257 1.58 chs switch (error) {
1258 1.57 chs case 0:
1259 1.63 chs break;
1260 1.7 mrg
1261 1.57 chs case ERESTART:
1262 1.139 uebayasi return ERESTART;
1263 1.7 mrg
1264 1.57 chs case EAGAIN:
1265 1.128 pooka kpause("fltagain1", false, hz/2, NULL);
1266 1.139 uebayasi return ERESTART;
1267 1.51 thorpej
1268 1.51 thorpej default:
1269 1.138 uebayasi return error;
1270 1.1 mrg }
1271 1.7 mrg
1272 1.7 mrg /*
1273 1.7 mrg * uobj is non null if the page is on loan from an object (i.e. uobj)
1274 1.7 mrg */
1275 1.7 mrg
1276 1.94 yamt uobj = anon->an_page->uobject; /* locked by anonget if !NULL */
1277 1.7 mrg
1278 1.7 mrg /* locked: maps(read), amap, anon, uobj(if one) */
1279 1.120 ad KASSERT(mutex_owned(&amap->am_l));
1280 1.122 ad KASSERT(mutex_owned(&anon->an_lock));
1281 1.122 ad KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock));
1282 1.7 mrg
1283 1.7 mrg /*
1284 1.63 chs * special handling for loaned pages
1285 1.7 mrg */
1286 1.52 chs
1287 1.94 yamt if (anon->an_page->loan_count) {
1288 1.148 uebayasi error = uvm_fault_upper_loan(ufi, flt, anon, &uobj);
1289 1.148 uebayasi if (error != 0)
1290 1.148 uebayasi return error;
1291 1.148 uebayasi }
1292 1.160 uebayasi
1293 1.160 uebayasi /*
1294 1.160 uebayasi * if we are case 1B then we will need to allocate a new blank
1295 1.160 uebayasi * anon to transfer the data into. note that we have a lock
1296 1.160 uebayasi * on anon, so no one can busy or release the page until we are done.
1297 1.160 uebayasi * also note that the ref count can't drop to zero here because
1298 1.160 uebayasi * it is > 1 and we are only dropping one ref.
1299 1.160 uebayasi *
1300 1.160 uebayasi * in the (hopefully very rare) case that we are out of RAM we
1301 1.160 uebayasi * will unlock, wait for more RAM, and refault.
1302 1.160 uebayasi *
1303 1.160 uebayasi * if we are out of anon VM we kill the process (XXX: could wait?).
1304 1.160 uebayasi */
1305 1.160 uebayasi
1306 1.160 uebayasi if (flt->cow_now && anon->an_ref > 1) {
1307 1.168 uebayasi flt->promote = true;
1308 1.160 uebayasi error = uvm_fault_upper_promote(ufi, flt, uobj, anon);
1309 1.160 uebayasi } else {
1310 1.160 uebayasi error = uvm_fault_upper_direct(ufi, flt, uobj, anon);
1311 1.160 uebayasi }
1312 1.160 uebayasi return error;
1313 1.148 uebayasi }
1314 1.148 uebayasi
1315 1.173 uebayasi /*
1316 1.173 uebayasi * uvm_fault_upper_loan: handle loaned upper page.
1317 1.173 uebayasi *
1318 1.177 yamt * 1. if not cow'ing now, simply adjust flt->enter_prot.
1319 1.173 uebayasi * 2. if cow'ing now, and if ref count is 1, break loan.
1320 1.173 uebayasi */
1321 1.173 uebayasi
1322 1.148 uebayasi static int
1323 1.148 uebayasi uvm_fault_upper_loan(
1324 1.148 uebayasi struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1325 1.148 uebayasi struct vm_anon *anon, struct uvm_object **ruobj)
1326 1.148 uebayasi {
1327 1.149 uebayasi struct vm_amap * const amap = ufi->entry->aref.ar_amap;
1328 1.151 uebayasi int error = 0;
1329 1.173 uebayasi UVMHIST_FUNC("uvm_fault_upper_loan"); UVMHIST_CALLED(maphist);
1330 1.149 uebayasi
1331 1.149 uebayasi if (!flt->cow_now) {
1332 1.7 mrg
1333 1.149 uebayasi /*
1334 1.149 uebayasi * for read faults on loaned pages we just cap the
1335 1.149 uebayasi * protection at read-only.
1336 1.149 uebayasi */
1337 1.63 chs
1338 1.149 uebayasi flt->enter_prot = flt->enter_prot & ~VM_PROT_WRITE;
1339 1.7 mrg
1340 1.149 uebayasi } else {
1341 1.149 uebayasi /*
1342 1.149 uebayasi * note that we can't allow writes into a loaned page!
1343 1.149 uebayasi *
1344 1.149 uebayasi * if we have a write fault on a loaned page in an
1345 1.149 uebayasi * anon then we need to look at the anon's ref count.
1346 1.149 uebayasi * if it is greater than one then we are going to do
1347 1.149 uebayasi * a normal copy-on-write fault into a new anon (this
1348 1.149 uebayasi * is not a problem). however, if the reference count
1349 1.149 uebayasi * is one (a case where we would normally allow a
1350 1.149 uebayasi * write directly to the page) then we need to kill
1351 1.149 uebayasi * the loan before we continue.
1352 1.149 uebayasi */
1353 1.149 uebayasi
1354 1.149 uebayasi /* >1 case is already ok */
1355 1.149 uebayasi if (anon->an_ref == 1) {
1356 1.155 uebayasi error = uvm_loanbreak_anon(anon, *ruobj);
1357 1.151 uebayasi if (error != 0) {
1358 1.151 uebayasi uvmfault_unlockall(ufi, amap, *ruobj, anon);
1359 1.151 uebayasi uvm_wait("flt_noram2");
1360 1.151 uebayasi return ERESTART;
1361 1.151 uebayasi }
1362 1.155 uebayasi /* if we were a loan reciever uobj is gone */
1363 1.155 uebayasi if (*ruobj)
1364 1.155 uebayasi *ruobj = NULL;
1365 1.151 uebayasi }
1366 1.151 uebayasi }
1367 1.151 uebayasi return error;
1368 1.151 uebayasi }
1369 1.151 uebayasi
1370 1.173 uebayasi /*
1371 1.173 uebayasi * uvm_fault_upper_promote: promote upper page.
1372 1.173 uebayasi *
1373 1.173 uebayasi * 1. call uvmfault_promote.
1374 1.173 uebayasi * 2. enqueue page.
1375 1.173 uebayasi * 3. deref.
1376 1.173 uebayasi * 4. pass page to uvm_fault_upper_enter.
1377 1.173 uebayasi */
1378 1.173 uebayasi
1379 1.148 uebayasi static int
1380 1.148 uebayasi uvm_fault_upper_promote(
1381 1.148 uebayasi struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1382 1.148 uebayasi struct uvm_object *uobj, struct vm_anon *anon)
1383 1.148 uebayasi {
1384 1.149 uebayasi struct vm_anon * const oanon = anon;
1385 1.149 uebayasi struct vm_page *pg;
1386 1.149 uebayasi int error;
1387 1.164 mlelstv UVMHIST_FUNC("uvm_fault_upper_promote"); UVMHIST_CALLED(maphist);
1388 1.149 uebayasi
1389 1.149 uebayasi UVMHIST_LOG(maphist, " case 1B: COW fault",0,0,0,0);
1390 1.149 uebayasi uvmexp.flt_acow++;
1391 1.149 uebayasi
1392 1.177 yamt error = uvmfault_promote(ufi, oanon, PGO_DONTCARE, &anon,
1393 1.177 yamt &flt->anon_spare);
1394 1.149 uebayasi switch (error) {
1395 1.149 uebayasi case 0:
1396 1.149 uebayasi break;
1397 1.149 uebayasi case ERESTART:
1398 1.149 uebayasi return ERESTART;
1399 1.149 uebayasi default:
1400 1.149 uebayasi return error;
1401 1.149 uebayasi }
1402 1.7 mrg
1403 1.149 uebayasi pg = anon->an_page;
1404 1.149 uebayasi mutex_enter(&uvm_pageqlock);
1405 1.149 uebayasi uvm_pageactivate(pg);
1406 1.149 uebayasi mutex_exit(&uvm_pageqlock);
1407 1.149 uebayasi pg->flags &= ~(PG_BUSY|PG_FAKE);
1408 1.149 uebayasi UVM_PAGE_OWN(pg, NULL);
1409 1.7 mrg
1410 1.149 uebayasi /* deref: can not drop to zero here by defn! */
1411 1.149 uebayasi oanon->an_ref--;
1412 1.53 thorpej
1413 1.149 uebayasi /*
1414 1.149 uebayasi * note: oanon is still locked, as is the new anon. we
1415 1.149 uebayasi * need to check for this later when we unlock oanon; if
1416 1.149 uebayasi * oanon != anon, we'll have to unlock anon, too.
1417 1.149 uebayasi */
1418 1.7 mrg
1419 1.149 uebayasi return uvm_fault_upper_enter(ufi, flt, uobj, anon, pg, oanon);
1420 1.148 uebayasi }
1421 1.148 uebayasi
1422 1.173 uebayasi /*
1423 1.173 uebayasi * uvm_fault_upper_direct: handle direct fault.
1424 1.173 uebayasi */
1425 1.173 uebayasi
1426 1.148 uebayasi static int
1427 1.148 uebayasi uvm_fault_upper_direct(
1428 1.148 uebayasi struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1429 1.148 uebayasi struct uvm_object *uobj, struct vm_anon *anon)
1430 1.148 uebayasi {
1431 1.149 uebayasi struct vm_anon * const oanon = anon;
1432 1.149 uebayasi struct vm_page *pg;
1433 1.173 uebayasi UVMHIST_FUNC("uvm_fault_upper_direct"); UVMHIST_CALLED(maphist);
1434 1.52 chs
1435 1.149 uebayasi uvmexp.flt_anon++;
1436 1.149 uebayasi pg = anon->an_page;
1437 1.149 uebayasi if (anon->an_ref > 1) /* disallow writes to ref > 1 anons */
1438 1.149 uebayasi flt->enter_prot = flt->enter_prot & ~VM_PROT_WRITE;
1439 1.7 mrg
1440 1.149 uebayasi return uvm_fault_upper_enter(ufi, flt, uobj, anon, pg, oanon);
1441 1.148 uebayasi }
1442 1.148 uebayasi
1443 1.173 uebayasi /*
1444 1.173 uebayasi * uvm_fault_upper_enter: enter h/w mapping of upper page.
1445 1.173 uebayasi */
1446 1.173 uebayasi
1447 1.148 uebayasi static int
1448 1.148 uebayasi uvm_fault_upper_enter(
1449 1.177 yamt struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
1450 1.148 uebayasi struct uvm_object *uobj, struct vm_anon *anon, struct vm_page *pg,
1451 1.148 uebayasi struct vm_anon *oanon)
1452 1.148 uebayasi {
1453 1.148 uebayasi struct vm_amap * const amap = ufi->entry->aref.ar_amap;
1454 1.164 mlelstv UVMHIST_FUNC("uvm_fault_upper_enter"); UVMHIST_CALLED(maphist);
1455 1.7 mrg
1456 1.173 uebayasi /* locked: maps(read), amap, oanon, anon(if different from oanon) */
1457 1.120 ad KASSERT(mutex_owned(&amap->am_l));
1458 1.122 ad KASSERT(mutex_owned(&anon->an_lock));
1459 1.122 ad KASSERT(mutex_owned(&oanon->an_lock));
1460 1.7 mrg
1461 1.7 mrg /*
1462 1.69 chs * now map the page in.
1463 1.7 mrg */
1464 1.7 mrg
1465 1.177 yamt UVMHIST_LOG(maphist,
1466 1.177 yamt " MAPPING: anon: pm=0x%x, va=0x%x, pg=0x%x, promote=%d",
1467 1.168 uebayasi ufi->orig_map->pmap, ufi->orig_rvaddr, pg, flt->promote);
1468 1.177 yamt if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr,
1469 1.177 yamt VM_PAGE_TO_PHYS(pg),
1470 1.177 yamt flt->enter_prot, flt->access_type | PMAP_CANFAIL |
1471 1.177 yamt (flt->wire_mapping ? PMAP_WIRED : 0)) != 0) {
1472 1.69 chs
1473 1.46 thorpej /*
1474 1.46 thorpej * No need to undo what we did; we can simply think of
1475 1.46 thorpej * this as the pmap throwing away the mapping information.
1476 1.46 thorpej *
1477 1.46 thorpej * We do, however, have to go through the ReFault path,
1478 1.46 thorpej * as the map may change while we're asleep.
1479 1.46 thorpej */
1480 1.69 chs
1481 1.53 thorpej if (anon != oanon)
1482 1.122 ad mutex_exit(&anon->an_lock);
1483 1.138 uebayasi uvmfault_unlockall(ufi, amap, uobj, oanon);
1484 1.92 yamt if (!uvm_reclaimable()) {
1485 1.46 thorpej UVMHIST_LOG(maphist,
1486 1.46 thorpej "<- failed. out of VM",0,0,0,0);
1487 1.46 thorpej /* XXX instrumentation */
1488 1.148 uebayasi return ENOMEM;
1489 1.46 thorpej }
1490 1.46 thorpej /* XXX instrumentation */
1491 1.46 thorpej uvm_wait("flt_pmfail1");
1492 1.139 uebayasi return ERESTART;
1493 1.46 thorpej }
1494 1.7 mrg
1495 1.177 yamt uvm_fault_upper_done(ufi, flt, anon, pg);
1496 1.169 uebayasi
1497 1.169 uebayasi /*
1498 1.169 uebayasi * done case 1! finish up by unlocking everything and returning success
1499 1.169 uebayasi */
1500 1.169 uebayasi
1501 1.175 rmind if (anon != oanon) {
1502 1.169 uebayasi mutex_exit(&anon->an_lock);
1503 1.175 rmind }
1504 1.175 rmind pmap_update(ufi->orig_map->pmap);
1505 1.169 uebayasi uvmfault_unlockall(ufi, amap, uobj, oanon);
1506 1.169 uebayasi return 0;
1507 1.148 uebayasi }
1508 1.148 uebayasi
1509 1.173 uebayasi /*
1510 1.173 uebayasi * uvm_fault_upper_done: queue upper center page.
1511 1.173 uebayasi */
1512 1.173 uebayasi
1513 1.169 uebayasi static void
1514 1.148 uebayasi uvm_fault_upper_done(
1515 1.177 yamt struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
1516 1.177 yamt struct vm_anon *anon, struct vm_page *pg)
1517 1.148 uebayasi {
1518 1.174 rmind const bool wire_paging = flt->wire_paging;
1519 1.174 rmind
1520 1.173 uebayasi UVMHIST_FUNC("uvm_fault_upper_done"); UVMHIST_CALLED(maphist);
1521 1.148 uebayasi
1522 1.7 mrg /*
1523 1.46 thorpej * ... update the page queues.
1524 1.7 mrg */
1525 1.7 mrg
1526 1.122 ad mutex_enter(&uvm_pageqlock);
1527 1.174 rmind if (wire_paging) {
1528 1.8 chuck uvm_pagewire(pg);
1529 1.29 chs
1530 1.29 chs /*
1531 1.29 chs * since the now-wired page cannot be paged out,
1532 1.29 chs * release its swap resources for others to use.
1533 1.29 chs * since an anon with no swap cannot be PG_CLEAN,
1534 1.29 chs * clear its clean flag now.
1535 1.29 chs */
1536 1.29 chs
1537 1.29 chs pg->flags &= ~(PG_CLEAN);
1538 1.174 rmind
1539 1.7 mrg } else {
1540 1.7 mrg uvm_pageactivate(pg);
1541 1.7 mrg }
1542 1.122 ad mutex_exit(&uvm_pageqlock);
1543 1.174 rmind
1544 1.174 rmind if (wire_paging) {
1545 1.174 rmind uvm_anon_dropswap(anon);
1546 1.174 rmind }
1547 1.138 uebayasi }
1548 1.1 mrg
1549 1.173 uebayasi /*
1550 1.173 uebayasi * uvm_fault_lower: handle lower fault.
1551 1.173 uebayasi *
1552 1.173 uebayasi * 1. check uobj
1553 1.173 uebayasi * 1.1. if null, ZFOD.
1554 1.173 uebayasi * 1.2. if not null, look up unnmapped neighbor pages.
1555 1.173 uebayasi * 2. for center page, check if promote.
1556 1.173 uebayasi * 2.1. ZFOD always needs promotion.
1557 1.173 uebayasi * 2.2. other uobjs, when entry is marked COW (usually MAP_PRIVATE vnode).
1558 1.173 uebayasi * 3. if uobj is not ZFOD and page is not found, do i/o.
1559 1.173 uebayasi * 4. dispatch either direct / promote fault.
1560 1.173 uebayasi */
1561 1.173 uebayasi
1562 1.138 uebayasi static int
1563 1.173 uebayasi uvm_fault_lower(
1564 1.140 uebayasi struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1565 1.173 uebayasi struct vm_page **pages)
1566 1.138 uebayasi {
1567 1.148 uebayasi #ifdef DIAGNOSTIC
1568 1.173 uebayasi struct vm_amap *amap = ufi->entry->aref.ar_amap;
1569 1.148 uebayasi #endif
1570 1.173 uebayasi struct uvm_object *uobj = ufi->entry->object.uvm_obj;
1571 1.173 uebayasi struct vm_page *uobjpage;
1572 1.138 uebayasi int error;
1573 1.173 uebayasi UVMHIST_FUNC("uvm_fault_lower"); UVMHIST_CALLED(maphist);
1574 1.173 uebayasi
1575 1.173 uebayasi /* locked: maps(read), amap(if there), uobj(if !null) */
1576 1.137 uebayasi
1577 1.7 mrg /*
1578 1.173 uebayasi * now, if the desired page is not shadowed by the amap and we have
1579 1.173 uebayasi * a backing object that does not have a special fault routine, then
1580 1.173 uebayasi * we ask (with pgo_get) the object for resident pages that we care
1581 1.173 uebayasi * about and attempt to map them in. we do not let pgo_get block
1582 1.173 uebayasi * (PGO_LOCKED).
1583 1.173 uebayasi */
1584 1.173 uebayasi
1585 1.173 uebayasi if (uobj == NULL) {
1586 1.173 uebayasi /* zero fill; don't care neighbor pages */
1587 1.173 uebayasi uobjpage = NULL;
1588 1.173 uebayasi } else {
1589 1.173 uebayasi uvm_fault_lower_lookup(ufi, flt, pages);
1590 1.173 uebayasi uobjpage = pages[flt->centeridx];
1591 1.173 uebayasi }
1592 1.173 uebayasi
1593 1.173 uebayasi /*
1594 1.173 uebayasi * note that at this point we are done with any front or back pages.
1595 1.173 uebayasi * we are now going to focus on the center page (i.e. the one we've
1596 1.173 uebayasi * faulted on). if we have faulted on the upper (anon) layer
1597 1.173 uebayasi * [i.e. case 1], then the anon we want is anons[centeridx] (we have
1598 1.173 uebayasi * not touched it yet). if we have faulted on the bottom (uobj)
1599 1.173 uebayasi * layer [i.e. case 2] and the page was both present and available,
1600 1.173 uebayasi * then we've got a pointer to it as "uobjpage" and we've already
1601 1.173 uebayasi * made it BUSY.
1602 1.7 mrg */
1603 1.7 mrg
1604 1.7 mrg /*
1605 1.7 mrg * locked:
1606 1.7 mrg * maps(read), amap(if there), uobj(if !null), uobjpage(if !null)
1607 1.7 mrg */
1608 1.120 ad KASSERT(amap == NULL || mutex_owned(&amap->am_l));
1609 1.122 ad KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock));
1610 1.120 ad KASSERT(uobjpage == NULL || (uobjpage->flags & PG_BUSY) != 0);
1611 1.7 mrg
1612 1.7 mrg /*
1613 1.7 mrg * note that uobjpage can not be PGO_DONTCARE at this point. we now
1614 1.7 mrg * set uobjpage to PGO_DONTCARE if we are doing a zero fill. if we
1615 1.7 mrg * have a backing object, check and see if we are going to promote
1616 1.7 mrg * the data up to an anon during the fault.
1617 1.7 mrg */
1618 1.7 mrg
1619 1.7 mrg if (uobj == NULL) {
1620 1.63 chs uobjpage = PGO_DONTCARE;
1621 1.168 uebayasi flt->promote = true; /* always need anon here */
1622 1.7 mrg } else {
1623 1.52 chs KASSERT(uobjpage != PGO_DONTCARE);
1624 1.168 uebayasi flt->promote = flt->cow_now && UVM_ET_ISCOPYONWRITE(ufi->entry);
1625 1.7 mrg }
1626 1.7 mrg UVMHIST_LOG(maphist, " case 2 fault: promote=%d, zfill=%d",
1627 1.168 uebayasi flt->promote, (uobj == NULL), 0,0);
1628 1.1 mrg
1629 1.7 mrg /*
1630 1.9 chuck * if uobjpage is not null then we do not need to do I/O to get the
1631 1.9 chuck * uobjpage.
1632 1.9 chuck *
1633 1.63 chs * if uobjpage is null, then we need to unlock and ask the pager to
1634 1.7 mrg * get the data for us. once we have the data, we need to reverify
1635 1.7 mrg * the state the world. we are currently not holding any resources.
1636 1.7 mrg */
1637 1.1 mrg
1638 1.9 chuck if (uobjpage) {
1639 1.9 chuck /* update rusage counters */
1640 1.124 ad curlwp->l_ru.ru_minflt++;
1641 1.9 chuck } else {
1642 1.163 uebayasi error = uvm_fault_lower_io(ufi, flt, &uobj, &uobjpage);
1643 1.148 uebayasi if (error != 0)
1644 1.148 uebayasi return error;
1645 1.148 uebayasi }
1646 1.160 uebayasi
1647 1.160 uebayasi /*
1648 1.160 uebayasi * locked:
1649 1.160 uebayasi * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj)
1650 1.160 uebayasi */
1651 1.160 uebayasi KASSERT(amap == NULL || mutex_owned(&amap->am_l));
1652 1.160 uebayasi KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock));
1653 1.160 uebayasi KASSERT(uobj == NULL || (uobjpage->flags & PG_BUSY) != 0);
1654 1.160 uebayasi
1655 1.160 uebayasi /*
1656 1.160 uebayasi * notes:
1657 1.160 uebayasi * - at this point uobjpage can not be NULL
1658 1.160 uebayasi * - at this point uobjpage can not be PG_RELEASED (since we checked
1659 1.160 uebayasi * for it above)
1660 1.160 uebayasi * - at this point uobjpage could be PG_WANTED (handle later)
1661 1.160 uebayasi */
1662 1.160 uebayasi
1663 1.177 yamt KASSERT(uobjpage != NULL);
1664 1.160 uebayasi KASSERT(uobj == NULL || uobj == uobjpage->uobject);
1665 1.160 uebayasi KASSERT(uobj == NULL || !UVM_OBJ_IS_CLEAN(uobjpage->uobject) ||
1666 1.160 uebayasi (uobjpage->flags & PG_CLEAN) != 0);
1667 1.160 uebayasi
1668 1.177 yamt if (!flt->promote) {
1669 1.163 uebayasi error = uvm_fault_lower_direct(ufi, flt, uobj, uobjpage);
1670 1.160 uebayasi } else {
1671 1.163 uebayasi error = uvm_fault_lower_promote(ufi, flt, uobj, uobjpage);
1672 1.160 uebayasi }
1673 1.160 uebayasi return error;
1674 1.148 uebayasi }
1675 1.148 uebayasi
1676 1.173 uebayasi /*
1677 1.173 uebayasi * uvm_fault_lower_lookup: look up on-memory uobj pages.
1678 1.173 uebayasi *
1679 1.173 uebayasi * 1. get on-memory pages.
1680 1.173 uebayasi * 2. if failed, give up (get only center page later).
1681 1.173 uebayasi * 3. if succeeded, enter h/w mapping of neighbor pages.
1682 1.173 uebayasi */
1683 1.173 uebayasi
1684 1.173 uebayasi static void
1685 1.173 uebayasi uvm_fault_lower_lookup(
1686 1.177 yamt struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
1687 1.173 uebayasi struct vm_page **pages)
1688 1.173 uebayasi {
1689 1.173 uebayasi struct uvm_object *uobj = ufi->entry->object.uvm_obj;
1690 1.173 uebayasi int lcv, gotpages;
1691 1.173 uebayasi vaddr_t currva;
1692 1.173 uebayasi UVMHIST_FUNC("uvm_fault_lower_lookup"); UVMHIST_CALLED(maphist);
1693 1.173 uebayasi
1694 1.173 uebayasi mutex_enter(&uobj->vmobjlock);
1695 1.173 uebayasi /* locked: maps(read), amap(if there), uobj */
1696 1.173 uebayasi /*
1697 1.173 uebayasi * the following call to pgo_get does _not_ change locking state
1698 1.173 uebayasi */
1699 1.173 uebayasi
1700 1.173 uebayasi uvmexp.fltlget++;
1701 1.173 uebayasi gotpages = flt->npages;
1702 1.173 uebayasi (void) uobj->pgops->pgo_get(uobj,
1703 1.173 uebayasi ufi->entry->offset + flt->startva - ufi->entry->start,
1704 1.173 uebayasi pages, &gotpages, flt->centeridx,
1705 1.173 uebayasi flt->access_type & MASK(ufi->entry), ufi->entry->advice, PGO_LOCKED);
1706 1.173 uebayasi
1707 1.173 uebayasi /*
1708 1.173 uebayasi * check for pages to map, if we got any
1709 1.173 uebayasi */
1710 1.173 uebayasi
1711 1.173 uebayasi if (gotpages == 0) {
1712 1.173 uebayasi pages[flt->centeridx] = NULL;
1713 1.173 uebayasi return;
1714 1.173 uebayasi }
1715 1.173 uebayasi
1716 1.173 uebayasi currva = flt->startva;
1717 1.173 uebayasi for (lcv = 0; lcv < flt->npages; lcv++, currva += PAGE_SIZE) {
1718 1.173 uebayasi struct vm_page *curpg;
1719 1.173 uebayasi
1720 1.173 uebayasi curpg = pages[lcv];
1721 1.173 uebayasi if (curpg == NULL || curpg == PGO_DONTCARE) {
1722 1.173 uebayasi continue;
1723 1.173 uebayasi }
1724 1.173 uebayasi KASSERT(curpg->uobject == uobj);
1725 1.173 uebayasi
1726 1.173 uebayasi /*
1727 1.173 uebayasi * if center page is resident and not PG_BUSY|PG_RELEASED
1728 1.173 uebayasi * then pgo_get made it PG_BUSY for us and gave us a handle
1729 1.173 uebayasi * to it.
1730 1.173 uebayasi */
1731 1.173 uebayasi
1732 1.173 uebayasi if (lcv == flt->centeridx) {
1733 1.173 uebayasi UVMHIST_LOG(maphist, " got uobjpage "
1734 1.173 uebayasi "(0x%x) with locked get",
1735 1.173 uebayasi curpg, 0,0,0);
1736 1.173 uebayasi } else {
1737 1.173 uebayasi bool readonly = (curpg->flags & PG_RDONLY)
1738 1.173 uebayasi || (curpg->loan_count > 0)
1739 1.173 uebayasi || UVM_OBJ_NEEDS_WRITEFAULT(curpg->uobject);
1740 1.173 uebayasi
1741 1.173 uebayasi uvm_fault_lower_neighbor(ufi, flt,
1742 1.173 uebayasi currva, curpg, readonly);
1743 1.173 uebayasi }
1744 1.173 uebayasi }
1745 1.173 uebayasi pmap_update(ufi->orig_map->pmap);
1746 1.173 uebayasi }
1747 1.173 uebayasi
1748 1.173 uebayasi /*
1749 1.173 uebayasi * uvm_fault_lower_neighbor: enter h/w mapping of lower neighbor page.
1750 1.173 uebayasi */
1751 1.173 uebayasi
1752 1.173 uebayasi static void
1753 1.173 uebayasi uvm_fault_lower_neighbor(
1754 1.177 yamt struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
1755 1.173 uebayasi vaddr_t currva, struct vm_page *pg, bool readonly)
1756 1.173 uebayasi {
1757 1.173 uebayasi UVMHIST_FUNC("uvm_fault_lower_neighor"); UVMHIST_CALLED(maphist);
1758 1.173 uebayasi
1759 1.173 uebayasi /* locked: maps(read), amap(if there), uobj */
1760 1.173 uebayasi
1761 1.173 uebayasi /*
1762 1.173 uebayasi * calling pgo_get with PGO_LOCKED returns us pages which
1763 1.173 uebayasi * are neither busy nor released, so we don't need to check
1764 1.173 uebayasi * for this. we can just directly enter the pages.
1765 1.173 uebayasi */
1766 1.173 uebayasi
1767 1.173 uebayasi mutex_enter(&uvm_pageqlock);
1768 1.173 uebayasi uvm_pageenqueue(pg);
1769 1.173 uebayasi mutex_exit(&uvm_pageqlock);
1770 1.173 uebayasi UVMHIST_LOG(maphist,
1771 1.173 uebayasi " MAPPING: n obj: pm=0x%x, va=0x%x, pg=0x%x",
1772 1.173 uebayasi ufi->orig_map->pmap, currva, pg, 0);
1773 1.173 uebayasi uvmexp.fltnomap++;
1774 1.173 uebayasi
1775 1.173 uebayasi /*
1776 1.173 uebayasi * Since this page isn't the page that's actually faulting,
1777 1.173 uebayasi * ignore pmap_enter() failures; it's not critical that we
1778 1.173 uebayasi * enter these right now.
1779 1.173 uebayasi * NOTE: page can't be PG_WANTED or PG_RELEASED because we've
1780 1.173 uebayasi * held the lock the whole time we've had the handle.
1781 1.173 uebayasi */
1782 1.173 uebayasi KASSERT((pg->flags & PG_PAGEOUT) == 0);
1783 1.173 uebayasi KASSERT((pg->flags & PG_RELEASED) == 0);
1784 1.173 uebayasi KASSERT((pg->flags & PG_WANTED) == 0);
1785 1.173 uebayasi KASSERT(!UVM_OBJ_IS_CLEAN(pg->uobject) ||
1786 1.173 uebayasi (pg->flags & PG_CLEAN) != 0);
1787 1.173 uebayasi pg->flags &= ~(PG_BUSY);
1788 1.173 uebayasi UVM_PAGE_OWN(pg, NULL);
1789 1.173 uebayasi
1790 1.173 uebayasi (void) pmap_enter(ufi->orig_map->pmap, currva,
1791 1.173 uebayasi VM_PAGE_TO_PHYS(pg),
1792 1.173 uebayasi readonly ? (flt->enter_prot & ~VM_PROT_WRITE) :
1793 1.173 uebayasi flt->enter_prot & MASK(ufi->entry),
1794 1.173 uebayasi PMAP_CANFAIL | (flt->wire_mapping ? PMAP_WIRED : 0));
1795 1.173 uebayasi }
1796 1.173 uebayasi
1797 1.173 uebayasi /*
1798 1.173 uebayasi * uvm_fault_lower_io: get lower page from backing store.
1799 1.173 uebayasi *
1800 1.173 uebayasi * 1. unlock everything, because i/o will block.
1801 1.173 uebayasi * 2. call pgo_get.
1802 1.173 uebayasi * 3. if failed, recover.
1803 1.173 uebayasi * 4. if succeeded, relock everything and verify things.
1804 1.173 uebayasi */
1805 1.173 uebayasi
1806 1.148 uebayasi static int
1807 1.163 uebayasi uvm_fault_lower_io(
1808 1.177 yamt struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
1809 1.156 uebayasi struct uvm_object **ruobj, struct vm_page **ruobjpage)
1810 1.148 uebayasi {
1811 1.149 uebayasi struct vm_amap * const amap = ufi->entry->aref.ar_amap;
1812 1.156 uebayasi struct uvm_object *uobj = *ruobj;
1813 1.158 uebayasi struct vm_page *pg;
1814 1.149 uebayasi bool locked;
1815 1.149 uebayasi int gotpages;
1816 1.149 uebayasi int error;
1817 1.149 uebayasi voff_t uoff;
1818 1.164 mlelstv UVMHIST_FUNC("uvm_fault_lower_io"); UVMHIST_CALLED(maphist);
1819 1.149 uebayasi
1820 1.149 uebayasi /* update rusage counters */
1821 1.149 uebayasi curlwp->l_ru.ru_majflt++;
1822 1.137 uebayasi
1823 1.149 uebayasi /* locked: maps(read), amap(if there), uobj */
1824 1.149 uebayasi uvmfault_unlockall(ufi, amap, NULL, NULL);
1825 1.149 uebayasi /* locked: uobj */
1826 1.63 chs
1827 1.149 uebayasi uvmexp.fltget++;
1828 1.149 uebayasi gotpages = 1;
1829 1.166 mlelstv pg = NULL;
1830 1.149 uebayasi uoff = (ufi->orig_rvaddr - ufi->entry->start) + ufi->entry->offset;
1831 1.158 uebayasi error = uobj->pgops->pgo_get(uobj, uoff, &pg, &gotpages,
1832 1.149 uebayasi 0, flt->access_type & MASK(ufi->entry), ufi->entry->advice,
1833 1.149 uebayasi PGO_SYNCIO);
1834 1.158 uebayasi /* locked: pg(if no error) */
1835 1.52 chs
1836 1.149 uebayasi /*
1837 1.149 uebayasi * recover from I/O
1838 1.149 uebayasi */
1839 1.1 mrg
1840 1.149 uebayasi if (error) {
1841 1.149 uebayasi if (error == EAGAIN) {
1842 1.149 uebayasi UVMHIST_LOG(maphist,
1843 1.149 uebayasi " pgo_get says TRY AGAIN!",0,0,0,0);
1844 1.149 uebayasi kpause("fltagain2", false, hz/2, NULL);
1845 1.149 uebayasi return ERESTART;
1846 1.149 uebayasi }
1847 1.1 mrg
1848 1.139 uebayasi #if 0
1849 1.149 uebayasi KASSERT(error != ERESTART);
1850 1.139 uebayasi #else
1851 1.149 uebayasi /* XXXUEBS don't re-fault? */
1852 1.149 uebayasi if (error == ERESTART)
1853 1.149 uebayasi error = EIO;
1854 1.139 uebayasi #endif
1855 1.139 uebayasi
1856 1.149 uebayasi UVMHIST_LOG(maphist, "<- pgo_get failed (code %d)",
1857 1.149 uebayasi error, 0,0,0);
1858 1.149 uebayasi return error;
1859 1.149 uebayasi }
1860 1.7 mrg
1861 1.158 uebayasi /* locked: pg */
1862 1.7 mrg
1863 1.165 mlelstv KASSERT((pg->flags & PG_BUSY) != 0);
1864 1.165 mlelstv
1865 1.149 uebayasi mutex_enter(&uvm_pageqlock);
1866 1.158 uebayasi uvm_pageactivate(pg);
1867 1.149 uebayasi mutex_exit(&uvm_pageqlock);
1868 1.69 chs
1869 1.149 uebayasi /*
1870 1.149 uebayasi * re-verify the state of the world by first trying to relock
1871 1.149 uebayasi * the maps. always relock the object.
1872 1.149 uebayasi */
1873 1.7 mrg
1874 1.149 uebayasi locked = uvmfault_relock(ufi);
1875 1.149 uebayasi if (locked && amap)
1876 1.149 uebayasi amap_lock(amap);
1877 1.156 uebayasi
1878 1.156 uebayasi /* might be changed */
1879 1.158 uebayasi uobj = pg->uobject;
1880 1.156 uebayasi
1881 1.149 uebayasi mutex_enter(&uobj->vmobjlock);
1882 1.63 chs
1883 1.158 uebayasi /* locked(locked): maps(read), amap(if !null), uobj, pg */
1884 1.158 uebayasi /* locked(!locked): uobj, pg */
1885 1.7 mrg
1886 1.149 uebayasi /*
1887 1.149 uebayasi * verify that the page has not be released and re-verify
1888 1.149 uebayasi * that amap slot is still free. if there is a problem,
1889 1.149 uebayasi * we unlock and clean up.
1890 1.149 uebayasi */
1891 1.7 mrg
1892 1.158 uebayasi if ((pg->flags & PG_RELEASED) != 0 ||
1893 1.158 uebayasi (locked && amap && amap_lookup(&ufi->entry->aref,
1894 1.149 uebayasi ufi->orig_rvaddr - ufi->entry->start))) {
1895 1.149 uebayasi if (locked)
1896 1.149 uebayasi uvmfault_unlockall(ufi, amap, NULL, NULL);
1897 1.149 uebayasi locked = false;
1898 1.149 uebayasi }
1899 1.7 mrg
1900 1.149 uebayasi /*
1901 1.149 uebayasi * didn't get the lock? release the page and retry.
1902 1.149 uebayasi */
1903 1.7 mrg
1904 1.149 uebayasi if (locked == false) {
1905 1.149 uebayasi UVMHIST_LOG(maphist,
1906 1.149 uebayasi " wasn't able to relock after fault: retry",
1907 1.149 uebayasi 0,0,0,0);
1908 1.158 uebayasi if (pg->flags & PG_WANTED) {
1909 1.158 uebayasi wakeup(pg);
1910 1.158 uebayasi }
1911 1.158 uebayasi if (pg->flags & PG_RELEASED) {
1912 1.149 uebayasi uvmexp.fltpgrele++;
1913 1.158 uebayasi uvm_pagefree(pg);
1914 1.157 uebayasi mutex_exit(&uobj->vmobjlock);
1915 1.139 uebayasi return ERESTART;
1916 1.7 mrg }
1917 1.158 uebayasi pg->flags &= ~(PG_BUSY|PG_WANTED);
1918 1.158 uebayasi UVM_PAGE_OWN(pg, NULL);
1919 1.149 uebayasi mutex_exit(&uobj->vmobjlock);
1920 1.149 uebayasi return ERESTART;
1921 1.149 uebayasi }
1922 1.7 mrg
1923 1.149 uebayasi /*
1924 1.158 uebayasi * we have the data in pg which is busy and
1925 1.149 uebayasi * not released. we are holding object lock (so the page
1926 1.149 uebayasi * can't be released on us).
1927 1.149 uebayasi */
1928 1.7 mrg
1929 1.158 uebayasi /* locked: maps(read), amap(if !null), uobj, pg */
1930 1.148 uebayasi
1931 1.156 uebayasi *ruobj = uobj;
1932 1.158 uebayasi *ruobjpage = pg;
1933 1.148 uebayasi return 0;
1934 1.148 uebayasi }
1935 1.148 uebayasi
1936 1.173 uebayasi /*
1937 1.173 uebayasi * uvm_fault_lower_direct: fault lower center page
1938 1.173 uebayasi *
1939 1.177 yamt * 1. adjust flt->enter_prot.
1940 1.173 uebayasi * 2. if page is loaned, resolve.
1941 1.173 uebayasi */
1942 1.173 uebayasi
1943 1.148 uebayasi int
1944 1.163 uebayasi uvm_fault_lower_direct(
1945 1.148 uebayasi struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1946 1.156 uebayasi struct uvm_object *uobj, struct vm_page *uobjpage)
1947 1.148 uebayasi {
1948 1.149 uebayasi struct vm_page *pg;
1949 1.173 uebayasi UVMHIST_FUNC("uvm_fault_lower_direct"); UVMHIST_CALLED(maphist);
1950 1.149 uebayasi
1951 1.149 uebayasi /*
1952 1.149 uebayasi * we are not promoting. if the mapping is COW ensure that we
1953 1.149 uebayasi * don't give more access than we should (e.g. when doing a read
1954 1.149 uebayasi * fault on a COPYONWRITE mapping we want to map the COW page in
1955 1.149 uebayasi * R/O even though the entry protection could be R/W).
1956 1.149 uebayasi *
1957 1.149 uebayasi * set "pg" to the page we want to map in (uobjpage, usually)
1958 1.149 uebayasi */
1959 1.1 mrg
1960 1.149 uebayasi uvmexp.flt_obj++;
1961 1.149 uebayasi if (UVM_ET_ISCOPYONWRITE(ufi->entry) ||
1962 1.149 uebayasi UVM_OBJ_NEEDS_WRITEFAULT(uobjpage->uobject))
1963 1.149 uebayasi flt->enter_prot &= ~VM_PROT_WRITE;
1964 1.149 uebayasi pg = uobjpage; /* map in the actual object */
1965 1.7 mrg
1966 1.149 uebayasi KASSERT(uobjpage != PGO_DONTCARE);
1967 1.7 mrg
1968 1.149 uebayasi /*
1969 1.149 uebayasi * we are faulting directly on the page. be careful
1970 1.149 uebayasi * about writing to loaned pages...
1971 1.149 uebayasi */
1972 1.149 uebayasi
1973 1.149 uebayasi if (uobjpage->loan_count) {
1974 1.163 uebayasi uvm_fault_lower_direct_loan(ufi, flt, uobj, &pg, &uobjpage);
1975 1.151 uebayasi }
1976 1.151 uebayasi KASSERT(pg == uobjpage);
1977 1.151 uebayasi
1978 1.163 uebayasi return uvm_fault_lower_enter(ufi, flt, uobj, NULL, pg, uobjpage);
1979 1.151 uebayasi }
1980 1.151 uebayasi
1981 1.173 uebayasi /*
1982 1.173 uebayasi * uvm_fault_lower_direct_loan: resolve loaned page.
1983 1.173 uebayasi *
1984 1.177 yamt * 1. if not cow'ing, adjust flt->enter_prot.
1985 1.173 uebayasi * 2. if cow'ing, break loan.
1986 1.173 uebayasi */
1987 1.173 uebayasi
1988 1.151 uebayasi static int
1989 1.163 uebayasi uvm_fault_lower_direct_loan(
1990 1.151 uebayasi struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1991 1.177 yamt struct uvm_object *uobj, struct vm_page **rpg,
1992 1.177 yamt struct vm_page **ruobjpage)
1993 1.151 uebayasi {
1994 1.152 uebayasi struct vm_amap * const amap = ufi->entry->aref.ar_amap;
1995 1.152 uebayasi struct vm_page *pg;
1996 1.152 uebayasi struct vm_page *uobjpage = *ruobjpage;
1997 1.164 mlelstv UVMHIST_FUNC("uvm_fault_lower_direct_loan"); UVMHIST_CALLED(maphist);
1998 1.152 uebayasi
1999 1.152 uebayasi if (!flt->cow_now) {
2000 1.152 uebayasi /* read fault: cap the protection at readonly */
2001 1.152 uebayasi /* cap! */
2002 1.152 uebayasi flt->enter_prot = flt->enter_prot & ~VM_PROT_WRITE;
2003 1.152 uebayasi } else {
2004 1.152 uebayasi /* write fault: must break the loan here */
2005 1.152 uebayasi
2006 1.152 uebayasi pg = uvm_loanbreak(uobjpage);
2007 1.152 uebayasi if (pg == NULL) {
2008 1.152 uebayasi
2009 1.152 uebayasi /*
2010 1.152 uebayasi * drop ownership of page, it can't be released
2011 1.152 uebayasi */
2012 1.152 uebayasi
2013 1.152 uebayasi if (uobjpage->flags & PG_WANTED)
2014 1.152 uebayasi wakeup(uobjpage);
2015 1.152 uebayasi uobjpage->flags &= ~(PG_BUSY|PG_WANTED);
2016 1.152 uebayasi UVM_PAGE_OWN(uobjpage, NULL);
2017 1.152 uebayasi
2018 1.152 uebayasi uvmfault_unlockall(ufi, amap, uobj, NULL);
2019 1.152 uebayasi UVMHIST_LOG(maphist,
2020 1.152 uebayasi " out of RAM breaking loan, waiting",
2021 1.152 uebayasi 0,0,0,0);
2022 1.152 uebayasi uvmexp.fltnoram++;
2023 1.152 uebayasi uvm_wait("flt_noram4");
2024 1.152 uebayasi return ERESTART;
2025 1.69 chs }
2026 1.152 uebayasi *rpg = pg;
2027 1.152 uebayasi *ruobjpage = pg;
2028 1.152 uebayasi }
2029 1.152 uebayasi return 0;
2030 1.148 uebayasi }
2031 1.148 uebayasi
2032 1.173 uebayasi /*
2033 1.173 uebayasi * uvm_fault_lower_promote: promote lower page.
2034 1.173 uebayasi *
2035 1.173 uebayasi * 1. call uvmfault_promote.
2036 1.173 uebayasi * 2. fill in data.
2037 1.173 uebayasi * 3. if not ZFOD, dispose old page.
2038 1.173 uebayasi */
2039 1.173 uebayasi
2040 1.148 uebayasi int
2041 1.163 uebayasi uvm_fault_lower_promote(
2042 1.148 uebayasi struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
2043 1.156 uebayasi struct uvm_object *uobj, struct vm_page *uobjpage)
2044 1.148 uebayasi {
2045 1.149 uebayasi struct vm_amap * const amap = ufi->entry->aref.ar_amap;
2046 1.149 uebayasi struct vm_anon *anon;
2047 1.149 uebayasi struct vm_page *pg;
2048 1.149 uebayasi int error;
2049 1.164 mlelstv UVMHIST_FUNC("uvm_fault_lower_promote"); UVMHIST_CALLED(maphist);
2050 1.63 chs
2051 1.149 uebayasi /*
2052 1.149 uebayasi * if we are going to promote the data to an anon we
2053 1.149 uebayasi * allocate a blank anon here and plug it into our amap.
2054 1.149 uebayasi */
2055 1.1 mrg #if DIAGNOSTIC
2056 1.149 uebayasi if (amap == NULL)
2057 1.149 uebayasi panic("uvm_fault: want to promote data, but no anon");
2058 1.1 mrg #endif
2059 1.149 uebayasi error = uvmfault_promote(ufi, NULL, uobjpage,
2060 1.149 uebayasi &anon, &flt->anon_spare);
2061 1.149 uebayasi switch (error) {
2062 1.149 uebayasi case 0:
2063 1.149 uebayasi break;
2064 1.149 uebayasi case ERESTART:
2065 1.149 uebayasi return ERESTART;
2066 1.149 uebayasi default:
2067 1.149 uebayasi return error;
2068 1.149 uebayasi }
2069 1.149 uebayasi
2070 1.149 uebayasi pg = anon->an_page;
2071 1.149 uebayasi
2072 1.149 uebayasi /*
2073 1.149 uebayasi * fill in the data
2074 1.149 uebayasi */
2075 1.105 yamt
2076 1.149 uebayasi if (uobjpage != PGO_DONTCARE) {
2077 1.149 uebayasi uvmexp.flt_prcopy++;
2078 1.1 mrg
2079 1.7 mrg /*
2080 1.149 uebayasi * promote to shared amap? make sure all sharing
2081 1.149 uebayasi * procs see it
2082 1.7 mrg */
2083 1.7 mrg
2084 1.149 uebayasi if ((amap_flags(amap) & AMAP_SHARED) != 0) {
2085 1.149 uebayasi pmap_page_protect(uobjpage, VM_PROT_NONE);
2086 1.7 mrg /*
2087 1.149 uebayasi * XXX: PAGE MIGHT BE WIRED!
2088 1.7 mrg */
2089 1.149 uebayasi }
2090 1.69 chs
2091 1.149 uebayasi /*
2092 1.149 uebayasi * dispose of uobjpage. it can't be PG_RELEASED
2093 1.149 uebayasi * since we still hold the object lock.
2094 1.149 uebayasi * drop handle to uobj as well.
2095 1.149 uebayasi */
2096 1.149 uebayasi
2097 1.149 uebayasi if (uobjpage->flags & PG_WANTED)
2098 1.149 uebayasi /* still have the obj lock */
2099 1.149 uebayasi wakeup(uobjpage);
2100 1.149 uebayasi uobjpage->flags &= ~(PG_BUSY|PG_WANTED);
2101 1.149 uebayasi UVM_PAGE_OWN(uobjpage, NULL);
2102 1.149 uebayasi mutex_exit(&uobj->vmobjlock);
2103 1.149 uebayasi uobj = NULL;
2104 1.149 uebayasi
2105 1.149 uebayasi UVMHIST_LOG(maphist,
2106 1.149 uebayasi " promote uobjpage 0x%x to anon/page 0x%x/0x%x",
2107 1.149 uebayasi uobjpage, anon, pg, 0);
2108 1.63 chs
2109 1.149 uebayasi } else {
2110 1.149 uebayasi uvmexp.flt_przero++;
2111 1.7 mrg
2112 1.149 uebayasi /*
2113 1.149 uebayasi * Page is zero'd and marked dirty by
2114 1.149 uebayasi * uvmfault_promote().
2115 1.149 uebayasi */
2116 1.52 chs
2117 1.149 uebayasi UVMHIST_LOG(maphist," zero fill anon/page 0x%x/0%x",
2118 1.149 uebayasi anon, pg, 0, 0);
2119 1.149 uebayasi }
2120 1.148 uebayasi
2121 1.163 uebayasi return uvm_fault_lower_enter(ufi, flt, uobj, anon, pg, uobjpage);
2122 1.148 uebayasi }
2123 1.148 uebayasi
2124 1.173 uebayasi /*
2125 1.173 uebayasi * uvm_fault_lower_enter: enter h/w mapping of lower page.
2126 1.173 uebayasi */
2127 1.173 uebayasi
2128 1.148 uebayasi int
2129 1.163 uebayasi uvm_fault_lower_enter(
2130 1.177 yamt struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
2131 1.148 uebayasi struct uvm_object *uobj,
2132 1.148 uebayasi struct vm_anon *anon, struct vm_page *pg, struct vm_page *uobjpage)
2133 1.148 uebayasi {
2134 1.148 uebayasi struct vm_amap * const amap = ufi->entry->aref.ar_amap;
2135 1.148 uebayasi int error;
2136 1.164 mlelstv UVMHIST_FUNC("uvm_fault_lower_enter"); UVMHIST_CALLED(maphist);
2137 1.7 mrg
2138 1.7 mrg /*
2139 1.7 mrg * locked:
2140 1.53 thorpej * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj),
2141 1.53 thorpej * anon(if !null), pg(if anon)
2142 1.7 mrg *
2143 1.7 mrg * note: pg is either the uobjpage or the new page in the new anon
2144 1.7 mrg */
2145 1.120 ad KASSERT(amap == NULL || mutex_owned(&amap->am_l));
2146 1.122 ad KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock));
2147 1.120 ad KASSERT(uobj == NULL || (uobjpage->flags & PG_BUSY) != 0);
2148 1.122 ad KASSERT(anon == NULL || mutex_owned(&anon->an_lock));
2149 1.120 ad KASSERT((pg->flags & PG_BUSY) != 0);
2150 1.7 mrg
2151 1.7 mrg /*
2152 1.7 mrg * all resources are present. we can now map it in and free our
2153 1.7 mrg * resources.
2154 1.7 mrg */
2155 1.7 mrg
2156 1.7 mrg UVMHIST_LOG(maphist,
2157 1.168 uebayasi " MAPPING: case2: pm=0x%x, va=0x%x, pg=0x%x, promote=%d",
2158 1.168 uebayasi ufi->orig_map->pmap, ufi->orig_rvaddr, pg, flt->promote);
2159 1.140 uebayasi KASSERT((flt->access_type & VM_PROT_WRITE) == 0 ||
2160 1.75 chs (pg->flags & PG_RDONLY) == 0);
2161 1.177 yamt if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr,
2162 1.177 yamt VM_PAGE_TO_PHYS(pg),
2163 1.177 yamt (pg->flags & PG_RDONLY) != 0 ?
2164 1.177 yamt flt->enter_prot & ~VM_PROT_WRITE : flt->enter_prot,
2165 1.177 yamt flt->access_type | PMAP_CANFAIL |
2166 1.177 yamt (flt->wire_mapping ? PMAP_WIRED : 0)) != 0) {
2167 1.52 chs
2168 1.46 thorpej /*
2169 1.46 thorpej * No need to undo what we did; we can simply think of
2170 1.46 thorpej * this as the pmap throwing away the mapping information.
2171 1.46 thorpej *
2172 1.46 thorpej * We do, however, have to go through the ReFault path,
2173 1.46 thorpej * as the map may change while we're asleep.
2174 1.46 thorpej */
2175 1.52 chs
2176 1.46 thorpej if (pg->flags & PG_WANTED)
2177 1.69 chs wakeup(pg);
2178 1.46 thorpej
2179 1.63 chs /*
2180 1.46 thorpej * note that pg can't be PG_RELEASED since we did not drop
2181 1.46 thorpej * the object lock since the last time we checked.
2182 1.46 thorpej */
2183 1.111 yamt KASSERT((pg->flags & PG_RELEASED) == 0);
2184 1.63 chs
2185 1.46 thorpej pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED);
2186 1.46 thorpej UVM_PAGE_OWN(pg, NULL);
2187 1.171 uebayasi
2188 1.138 uebayasi uvmfault_unlockall(ufi, amap, uobj, anon);
2189 1.92 yamt if (!uvm_reclaimable()) {
2190 1.46 thorpej UVMHIST_LOG(maphist,
2191 1.46 thorpej "<- failed. out of VM",0,0,0,0);
2192 1.46 thorpej /* XXX instrumentation */
2193 1.106 yamt error = ENOMEM;
2194 1.138 uebayasi return error;
2195 1.46 thorpej }
2196 1.46 thorpej /* XXX instrumentation */
2197 1.46 thorpej uvm_wait("flt_pmfail2");
2198 1.139 uebayasi return ERESTART;
2199 1.46 thorpej }
2200 1.1 mrg
2201 1.177 yamt uvm_fault_lower_done(ufi, flt, uobj, pg);
2202 1.177 yamt
2203 1.177 yamt /*
2204 1.177 yamt * note that pg can't be PG_RELEASED since we did not drop the object
2205 1.177 yamt * lock since the last time we checked.
2206 1.177 yamt */
2207 1.177 yamt KASSERT((pg->flags & PG_RELEASED) == 0);
2208 1.177 yamt if (pg->flags & PG_WANTED)
2209 1.177 yamt wakeup(pg);
2210 1.177 yamt pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED);
2211 1.177 yamt UVM_PAGE_OWN(pg, NULL);
2212 1.169 uebayasi
2213 1.175 rmind pmap_update(ufi->orig_map->pmap);
2214 1.169 uebayasi uvmfault_unlockall(ufi, amap, uobj, anon);
2215 1.175 rmind
2216 1.169 uebayasi UVMHIST_LOG(maphist, "<- done (SUCCESS!)",0,0,0,0);
2217 1.169 uebayasi return 0;
2218 1.148 uebayasi }
2219 1.148 uebayasi
2220 1.173 uebayasi /*
2221 1.173 uebayasi * uvm_fault_lower_done: queue lower center page.
2222 1.173 uebayasi */
2223 1.173 uebayasi
2224 1.169 uebayasi void
2225 1.163 uebayasi uvm_fault_lower_done(
2226 1.177 yamt struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
2227 1.177 yamt struct uvm_object *uobj, struct vm_page *pg)
2228 1.148 uebayasi {
2229 1.174 rmind bool dropswap = false;
2230 1.174 rmind
2231 1.164 mlelstv UVMHIST_FUNC("uvm_fault_lower_done"); UVMHIST_CALLED(maphist);
2232 1.148 uebayasi
2233 1.122 ad mutex_enter(&uvm_pageqlock);
2234 1.146 uebayasi if (flt->wire_paging) {
2235 1.8 chuck uvm_pagewire(pg);
2236 1.22 chs if (pg->pqflags & PQ_AOBJ) {
2237 1.29 chs
2238 1.29 chs /*
2239 1.29 chs * since the now-wired page cannot be paged out,
2240 1.29 chs * release its swap resources for others to use.
2241 1.29 chs * since an aobj page with no swap cannot be PG_CLEAN,
2242 1.29 chs * clear its clean flag now.
2243 1.29 chs */
2244 1.29 chs
2245 1.113 christos KASSERT(uobj != NULL);
2246 1.29 chs pg->flags &= ~(PG_CLEAN);
2247 1.174 rmind dropswap = true;
2248 1.22 chs }
2249 1.7 mrg } else {
2250 1.7 mrg uvm_pageactivate(pg);
2251 1.7 mrg }
2252 1.122 ad mutex_exit(&uvm_pageqlock);
2253 1.171 uebayasi
2254 1.174 rmind if (dropswap) {
2255 1.174 rmind uao_dropswap(uobj, pg->offset >> PAGE_SHIFT);
2256 1.174 rmind }
2257 1.1 mrg }
2258 1.1 mrg
2259 1.110 drochner
2260 1.1 mrg /*
2261 1.1 mrg * uvm_fault_wire: wire down a range of virtual addresses in a map.
2262 1.1 mrg *
2263 1.36 thorpej * => map may be read-locked by caller, but MUST NOT be write-locked.
2264 1.36 thorpej * => if map is read-locked, any operations which may cause map to
2265 1.36 thorpej * be write-locked in uvm_fault() must be taken care of by
2266 1.36 thorpej * the caller. See uvm_map_pageable().
2267 1.1 mrg */
2268 1.1 mrg
2269 1.7 mrg int
2270 1.95 thorpej uvm_fault_wire(struct vm_map *map, vaddr_t start, vaddr_t end,
2271 1.130 uebayasi vm_prot_t access_type, int maxprot)
2272 1.7 mrg {
2273 1.12 eeh vaddr_t va;
2274 1.58 chs int error;
2275 1.7 mrg
2276 1.7 mrg /*
2277 1.47 chs * now fault it in a page at a time. if the fault fails then we have
2278 1.63 chs * to undo what we have done. note that in uvm_fault VM_PROT_NONE
2279 1.47 chs * is replaced with the max protection if fault_type is VM_FAULT_WIRE.
2280 1.7 mrg */
2281 1.1 mrg
2282 1.65 chs /*
2283 1.65 chs * XXX work around overflowing a vaddr_t. this prevents us from
2284 1.65 chs * wiring the last page in the address space, though.
2285 1.65 chs */
2286 1.65 chs if (start > end) {
2287 1.65 chs return EFAULT;
2288 1.65 chs }
2289 1.65 chs
2290 1.163 uebayasi for (va = start; va < end; va += PAGE_SIZE) {
2291 1.110 drochner error = uvm_fault_internal(map, va, access_type,
2292 1.177 yamt (maxprot ? UVM_FAULT_MAXPROT : 0) | UVM_FAULT_WIRE);
2293 1.58 chs if (error) {
2294 1.7 mrg if (va != start) {
2295 1.31 thorpej uvm_fault_unwire(map, start, va);
2296 1.7 mrg }
2297 1.58 chs return error;
2298 1.7 mrg }
2299 1.7 mrg }
2300 1.58 chs return 0;
2301 1.1 mrg }
2302 1.1 mrg
2303 1.1 mrg /*
2304 1.1 mrg * uvm_fault_unwire(): unwire range of virtual space.
2305 1.1 mrg */
2306 1.1 mrg
2307 1.7 mrg void
2308 1.95 thorpej uvm_fault_unwire(struct vm_map *map, vaddr_t start, vaddr_t end)
2309 1.36 thorpej {
2310 1.36 thorpej vm_map_lock_read(map);
2311 1.36 thorpej uvm_fault_unwire_locked(map, start, end);
2312 1.36 thorpej vm_map_unlock_read(map);
2313 1.36 thorpej }
2314 1.36 thorpej
2315 1.36 thorpej /*
2316 1.36 thorpej * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire().
2317 1.36 thorpej *
2318 1.36 thorpej * => map must be at least read-locked.
2319 1.36 thorpej */
2320 1.36 thorpej
2321 1.36 thorpej void
2322 1.95 thorpej uvm_fault_unwire_locked(struct vm_map *map, vaddr_t start, vaddr_t end)
2323 1.7 mrg {
2324 1.64 chs struct vm_map_entry *entry;
2325 1.31 thorpej pmap_t pmap = vm_map_pmap(map);
2326 1.42 thorpej vaddr_t va;
2327 1.12 eeh paddr_t pa;
2328 1.42 thorpej struct vm_page *pg;
2329 1.31 thorpej
2330 1.52 chs KASSERT((map->flags & VM_MAP_INTRSAFE) == 0);
2331 1.7 mrg
2332 1.7 mrg /*
2333 1.7 mrg * we assume that the area we are unwiring has actually been wired
2334 1.7 mrg * in the first place. this means that we should be able to extract
2335 1.7 mrg * the PAs from the pmap. we also lock out the page daemon so that
2336 1.7 mrg * we can call uvm_pageunwire.
2337 1.7 mrg */
2338 1.37 thorpej
2339 1.122 ad mutex_enter(&uvm_pageqlock);
2340 1.7 mrg
2341 1.37 thorpej /*
2342 1.37 thorpej * find the beginning map entry for the region.
2343 1.37 thorpej */
2344 1.74 chs
2345 1.56 chs KASSERT(start >= vm_map_min(map) && end <= vm_map_max(map));
2346 1.119 thorpej if (uvm_map_lookup_entry(map, start, &entry) == false)
2347 1.37 thorpej panic("uvm_fault_unwire_locked: address not in map");
2348 1.37 thorpej
2349 1.69 chs for (va = start; va < end; va += PAGE_SIZE) {
2350 1.119 thorpej if (pmap_extract(pmap, va, &pa) == false)
2351 1.74 chs continue;
2352 1.42 thorpej
2353 1.42 thorpej /*
2354 1.74 chs * find the map entry for the current address.
2355 1.42 thorpej */
2356 1.56 chs
2357 1.56 chs KASSERT(va >= entry->start);
2358 1.74 chs while (va >= entry->end) {
2359 1.56 chs KASSERT(entry->next != &map->header &&
2360 1.56 chs entry->next->start <= entry->end);
2361 1.42 thorpej entry = entry->next;
2362 1.42 thorpej }
2363 1.37 thorpej
2364 1.42 thorpej /*
2365 1.42 thorpej * if the entry is no longer wired, tell the pmap.
2366 1.42 thorpej */
2367 1.74 chs
2368 1.42 thorpej if (VM_MAPENT_ISWIRED(entry) == 0)
2369 1.42 thorpej pmap_unwire(pmap, va);
2370 1.42 thorpej
2371 1.42 thorpej pg = PHYS_TO_VM_PAGE(pa);
2372 1.42 thorpej if (pg)
2373 1.42 thorpej uvm_pageunwire(pg);
2374 1.7 mrg }
2375 1.1 mrg
2376 1.122 ad mutex_exit(&uvm_pageqlock);
2377 1.1 mrg }
2378