uvm_fault.c revision 1.190.2.2 1 /* $NetBSD: uvm_fault.c,v 1.190.2.2 2011/11/14 14:23:16 yamt Exp $ */
2
3 /*
4 * Copyright (c) 1997 Charles D. Cranor and Washington University.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 *
27 * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp
28 */
29
30 /*
31 * uvm_fault.c: fault handler
32 */
33
34 #include <sys/cdefs.h>
35 __KERNEL_RCSID(0, "$NetBSD: uvm_fault.c,v 1.190.2.2 2011/11/14 14:23:16 yamt Exp $");
36
37 #include "opt_uvmhist.h"
38
39 #include <sys/param.h>
40 #include <sys/systm.h>
41 #include <sys/kernel.h>
42 #include <sys/mman.h>
43
44 #include <uvm/uvm.h>
45
46 /*
47 *
48 * a word on page faults:
49 *
50 * types of page faults we handle:
51 *
52 * CASE 1: upper layer faults CASE 2: lower layer faults
53 *
54 * CASE 1A CASE 1B CASE 2A CASE 2B
55 * read/write1 write>1 read/write +-cow_write/zero
56 * | | | |
57 * +--|--+ +--|--+ +-----+ + | + | +-----+
58 * amap | V | | ---------> new | | | | ^ |
59 * +-----+ +-----+ +-----+ + | + | +--|--+
60 * | | |
61 * +-----+ +-----+ +--|--+ | +--|--+
62 * uobj | d/c | | d/c | | V | +----+ |
63 * +-----+ +-----+ +-----+ +-----+
64 *
65 * d/c = don't care
66 *
67 * case [0]: layerless fault
68 * no amap or uobj is present. this is an error.
69 *
70 * case [1]: upper layer fault [anon active]
71 * 1A: [read] or [write with anon->an_ref == 1]
72 * I/O takes place in upper level anon and uobj is not touched.
73 * 1B: [write with anon->an_ref > 1]
74 * new anon is alloc'd and data is copied off ["COW"]
75 *
76 * case [2]: lower layer fault [uobj]
77 * 2A: [read on non-NULL uobj] or [write to non-copy_on_write area]
78 * I/O takes place directly in object.
79 * 2B: [write to copy_on_write] or [read on NULL uobj]
80 * data is "promoted" from uobj to a new anon.
81 * if uobj is null, then we zero fill.
82 *
83 * we follow the standard UVM locking protocol ordering:
84 *
85 * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ)
86 * we hold a PG_BUSY page if we unlock for I/O
87 *
88 *
89 * the code is structured as follows:
90 *
91 * - init the "IN" params in the ufi structure
92 * ReFault: (ERESTART returned to the loop in uvm_fault_internal)
93 * - do lookups [locks maps], check protection, handle needs_copy
94 * - check for case 0 fault (error)
95 * - establish "range" of fault
96 * - if we have an amap lock it and extract the anons
97 * - if sequential advice deactivate pages behind us
98 * - at the same time check pmap for unmapped areas and anon for pages
99 * that we could map in (and do map it if found)
100 * - check object for resident pages that we could map in
101 * - if (case 2) goto Case2
102 * - >>> handle case 1
103 * - ensure source anon is resident in RAM
104 * - if case 1B alloc new anon and copy from source
105 * - map the correct page in
106 * Case2:
107 * - >>> handle case 2
108 * - ensure source page is resident (if uobj)
109 * - if case 2B alloc new anon and copy from source (could be zero
110 * fill if uobj == NULL)
111 * - map the correct page in
112 * - done!
113 *
114 * note on paging:
115 * if we have to do I/O we place a PG_BUSY page in the correct object,
116 * unlock everything, and do the I/O. when I/O is done we must reverify
117 * the state of the world before assuming that our data structures are
118 * valid. [because mappings could change while the map is unlocked]
119 *
120 * alternative 1: unbusy the page in question and restart the page fault
121 * from the top (ReFault). this is easy but does not take advantage
122 * of the information that we already have from our previous lookup,
123 * although it is possible that the "hints" in the vm_map will help here.
124 *
125 * alternative 2: the system already keeps track of a "version" number of
126 * a map. [i.e. every time you write-lock a map (e.g. to change a
127 * mapping) you bump the version number up by one...] so, we can save
128 * the version number of the map before we release the lock and start I/O.
129 * then when I/O is done we can relock and check the version numbers
130 * to see if anything changed. this might save us some over 1 because
131 * we don't have to unbusy the page and may be less compares(?).
132 *
133 * alternative 3: put in backpointers or a way to "hold" part of a map
134 * in place while I/O is in progress. this could be complex to
135 * implement (especially with structures like amap that can be referenced
136 * by multiple map entries, and figuring out what should wait could be
137 * complex as well...).
138 *
139 * we use alternative 2. given that we are multi-threaded now we may want
140 * to reconsider the choice.
141 */
142
143 /*
144 * local data structures
145 */
146
147 struct uvm_advice {
148 int advice;
149 int nback;
150 int nforw;
151 };
152
153 /*
154 * page range array:
155 * note: index in array must match "advice" value
156 * XXX: borrowed numbers from freebsd. do they work well for us?
157 */
158
159 static const struct uvm_advice uvmadvice[] = {
160 { UVM_ADV_NORMAL, 3, 4 },
161 { UVM_ADV_RANDOM, 0, 0 },
162 { UVM_ADV_SEQUENTIAL, 8, 7},
163 };
164
165 #define UVM_MAXRANGE 16 /* must be MAX() of nback+nforw+1 */
166
167 /*
168 * private prototypes
169 */
170
171 /*
172 * inline functions
173 */
174
175 /*
176 * uvmfault_anonflush: try and deactivate pages in specified anons
177 *
178 * => does not have to deactivate page if it is busy
179 */
180
181 static inline void
182 uvmfault_anonflush(struct vm_anon **anons, int n)
183 {
184 int lcv;
185 struct vm_page *pg;
186
187 for (lcv = 0; lcv < n; lcv++) {
188 if (anons[lcv] == NULL)
189 continue;
190 KASSERT(mutex_owned(anons[lcv]->an_lock));
191 pg = anons[lcv]->an_page;
192 if (pg && (pg->flags & PG_BUSY) == 0) {
193 mutex_enter(&uvm_pageqlock);
194 if (pg->wire_count == 0) {
195 uvm_pagedeactivate(pg);
196 }
197 mutex_exit(&uvm_pageqlock);
198 }
199 }
200 }
201
202 /*
203 * normal functions
204 */
205
206 /*
207 * uvmfault_amapcopy: clear "needs_copy" in a map.
208 *
209 * => called with VM data structures unlocked (usually, see below)
210 * => we get a write lock on the maps and clear needs_copy for a VA
211 * => if we are out of RAM we sleep (waiting for more)
212 */
213
214 static void
215 uvmfault_amapcopy(struct uvm_faultinfo *ufi)
216 {
217 for (;;) {
218
219 /*
220 * no mapping? give up.
221 */
222
223 if (uvmfault_lookup(ufi, true) == false)
224 return;
225
226 /*
227 * copy if needed.
228 */
229
230 if (UVM_ET_ISNEEDSCOPY(ufi->entry))
231 amap_copy(ufi->map, ufi->entry, AMAP_COPY_NOWAIT,
232 ufi->orig_rvaddr, ufi->orig_rvaddr + 1);
233
234 /*
235 * didn't work? must be out of RAM. unlock and sleep.
236 */
237
238 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) {
239 uvmfault_unlockmaps(ufi, true);
240 uvm_wait("fltamapcopy");
241 continue;
242 }
243
244 /*
245 * got it! unlock and return.
246 */
247
248 uvmfault_unlockmaps(ufi, true);
249 return;
250 }
251 /*NOTREACHED*/
252 }
253
254 /*
255 * uvmfault_anonget: get data in an anon into a non-busy, non-released
256 * page in that anon.
257 *
258 * => Map, amap and thus anon should be locked by caller.
259 * => If we fail, we unlock everything and error is returned.
260 * => If we are successful, return with everything still locked.
261 * => We do not move the page on the queues [gets moved later]. If we
262 * allocate a new page [we_own], it gets put on the queues. Either way,
263 * the result is that the page is on the queues at return time
264 * => For pages which are on loan from a uvm_object (and thus are not owned
265 * by the anon): if successful, return with the owning object locked.
266 * The caller must unlock this object when it unlocks everything else.
267 */
268
269 int
270 uvmfault_anonget(struct uvm_faultinfo *ufi, struct vm_amap *amap,
271 struct vm_anon *anon)
272 {
273 struct vm_page *pg;
274 int error;
275
276 UVMHIST_FUNC("uvmfault_anonget"); UVMHIST_CALLED(maphist);
277 KASSERT(mutex_owned(anon->an_lock));
278 KASSERT(anon->an_lock == amap->am_lock);
279
280 /* Increment the counters.*/
281 uvmexp.fltanget++;
282 if (anon->an_page) {
283 curlwp->l_ru.ru_minflt++;
284 } else {
285 curlwp->l_ru.ru_majflt++;
286 }
287 error = 0;
288
289 /*
290 * Loop until we get the anon data, or fail.
291 */
292
293 for (;;) {
294 bool we_own, locked;
295 /*
296 * Note: 'we_own' will become true if we set PG_BUSY on a page.
297 */
298 we_own = false;
299 pg = anon->an_page;
300
301 /*
302 * If there is a resident page and it is loaned, then anon
303 * may not own it. Call out to uvm_anon_lockloanpg() to
304 * identify and lock the real owner of the page.
305 */
306
307 if (pg && pg->loan_count)
308 pg = uvm_anon_lockloanpg(anon);
309
310 /*
311 * Is page resident? Make sure it is not busy/released.
312 */
313
314 if (pg) {
315
316 /*
317 * at this point, if the page has a uobject [meaning
318 * we have it on loan], then that uobject is locked
319 * by us! if the page is busy, we drop all the
320 * locks (including uobject) and try again.
321 */
322
323 if ((pg->flags & PG_BUSY) == 0) {
324 UVMHIST_LOG(maphist, "<- OK",0,0,0,0);
325 return 0;
326 }
327 pg->flags |= PG_WANTED;
328 uvmexp.fltpgwait++;
329
330 /*
331 * The last unlock must be an atomic unlock and wait
332 * on the owner of page.
333 */
334
335 if (pg->uobject) {
336 /* Owner of page is UVM object. */
337 uvmfault_unlockall(ufi, amap, NULL);
338 UVMHIST_LOG(maphist, " unlock+wait on uobj",0,
339 0,0,0);
340 UVM_UNLOCK_AND_WAIT(pg,
341 pg->uobject->vmobjlock,
342 false, "anonget1", 0);
343 } else {
344 /* Owner of page is anon. */
345 uvmfault_unlockall(ufi, NULL, NULL);
346 UVMHIST_LOG(maphist, " unlock+wait on anon",0,
347 0,0,0);
348 UVM_UNLOCK_AND_WAIT(pg, anon->an_lock,
349 false, "anonget2", 0);
350 }
351 } else {
352 #if defined(VMSWAP)
353 /*
354 * No page, therefore allocate one.
355 */
356
357 pg = uvm_pagealloc(NULL,
358 ufi != NULL ? ufi->orig_rvaddr : 0,
359 anon, ufi != NULL ? UVM_FLAG_COLORMATCH : 0);
360 if (pg == NULL) {
361 /* Out of memory. Wait a little. */
362 uvmfault_unlockall(ufi, amap, NULL);
363 uvmexp.fltnoram++;
364 UVMHIST_LOG(maphist, " noram -- UVM_WAIT",0,
365 0,0,0);
366 if (!uvm_reclaimable()) {
367 return ENOMEM;
368 }
369 uvm_wait("flt_noram1");
370 } else {
371 /* PG_BUSY bit is set. */
372 we_own = true;
373 uvmfault_unlockall(ufi, amap, NULL);
374
375 /*
376 * Pass a PG_BUSY+PG_FAKE clean page into
377 * the uvm_swap_get() function with all data
378 * structures unlocked. Note that it is OK
379 * to read an_swslot here, because we hold
380 * PG_BUSY on the page.
381 */
382 uvmexp.pageins++;
383 error = uvm_swap_get(pg, anon->an_swslot,
384 PGO_SYNCIO);
385
386 /*
387 * We clean up after the I/O below in the
388 * 'we_own' case.
389 */
390 }
391 #else
392 panic("%s: no page", __func__);
393 #endif /* defined(VMSWAP) */
394 }
395
396 /*
397 * Re-lock the map and anon.
398 */
399
400 locked = uvmfault_relock(ufi);
401 if (locked || we_own) {
402 mutex_enter(anon->an_lock);
403 }
404
405 /*
406 * If we own the page (i.e. we set PG_BUSY), then we need
407 * to clean up after the I/O. There are three cases to
408 * consider:
409 *
410 * 1) Page was released during I/O: free anon and ReFault.
411 * 2) I/O not OK. Free the page and cause the fault to fail.
412 * 3) I/O OK! Activate the page and sync with the non-we_own
413 * case (i.e. drop anon lock if not locked).
414 */
415
416 if (we_own) {
417 #if defined(VMSWAP)
418 if (pg->flags & PG_WANTED) {
419 wakeup(pg);
420 }
421 if (error) {
422
423 /*
424 * Remove the swap slot from the anon and
425 * mark the anon as having no real slot.
426 * Do not free the swap slot, thus preventing
427 * it from being used again.
428 */
429
430 if (anon->an_swslot > 0) {
431 uvm_swap_markbad(anon->an_swslot, 1);
432 }
433 anon->an_swslot = SWSLOT_BAD;
434
435 if ((pg->flags & PG_RELEASED) != 0) {
436 goto released;
437 }
438
439 /*
440 * Note: page was never !PG_BUSY, so it
441 * cannot be mapped and thus no need to
442 * pmap_page_protect() it.
443 */
444
445 mutex_enter(&uvm_pageqlock);
446 uvm_pagefree(pg);
447 mutex_exit(&uvm_pageqlock);
448
449 if (locked) {
450 uvmfault_unlockall(ufi, NULL, NULL);
451 }
452 mutex_exit(anon->an_lock);
453 UVMHIST_LOG(maphist, "<- ERROR", 0,0,0,0);
454 return error;
455 }
456
457 if ((pg->flags & PG_RELEASED) != 0) {
458 released:
459 KASSERT(anon->an_ref == 0);
460
461 /*
462 * Released while we had unlocked amap.
463 */
464
465 if (locked) {
466 uvmfault_unlockall(ufi, NULL, NULL);
467 }
468 uvm_anon_release(anon);
469
470 if (error) {
471 UVMHIST_LOG(maphist,
472 "<- ERROR/RELEASED", 0,0,0,0);
473 return error;
474 }
475
476 UVMHIST_LOG(maphist, "<- RELEASED", 0,0,0,0);
477 return ERESTART;
478 }
479
480 /*
481 * We have successfully read the page, activate it.
482 */
483
484 mutex_enter(&uvm_pageqlock);
485 uvm_pageactivate(pg);
486 mutex_exit(&uvm_pageqlock);
487 pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE);
488 uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_UNKNOWN);
489 UVM_PAGE_OWN(pg, NULL);
490 #else
491 panic("%s: we_own", __func__);
492 #endif /* defined(VMSWAP) */
493 }
494
495 /*
496 * We were not able to re-lock the map - restart the fault.
497 */
498
499 if (!locked) {
500 if (we_own) {
501 mutex_exit(anon->an_lock);
502 }
503 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0);
504 return ERESTART;
505 }
506
507 /*
508 * Verify that no one has touched the amap and moved
509 * the anon on us.
510 */
511
512 if (ufi != NULL && amap_lookup(&ufi->entry->aref,
513 ufi->orig_rvaddr - ufi->entry->start) != anon) {
514
515 uvmfault_unlockall(ufi, amap, NULL);
516 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0);
517 return ERESTART;
518 }
519
520 /*
521 * Retry..
522 */
523
524 uvmexp.fltanretry++;
525 continue;
526 }
527 /*NOTREACHED*/
528 }
529
530 /*
531 * uvmfault_promote: promote data to a new anon. used for 1B and 2B.
532 *
533 * 1. allocate an anon and a page.
534 * 2. fill its contents.
535 * 3. put it into amap.
536 *
537 * => if we fail (result != 0) we unlock everything.
538 * => on success, return a new locked anon via 'nanon'.
539 * (*nanon)->an_page will be a resident, locked, dirty page.
540 * => it's caller's responsibility to put the promoted nanon->an_page to the
541 * page queue.
542 */
543
544 static int
545 uvmfault_promote(struct uvm_faultinfo *ufi,
546 struct vm_anon *oanon,
547 struct vm_page *uobjpage,
548 struct vm_anon **nanon, /* OUT: allocated anon */
549 struct vm_anon **spare)
550 {
551 struct vm_amap *amap = ufi->entry->aref.ar_amap;
552 struct uvm_object *uobj;
553 struct vm_anon *anon;
554 struct vm_page *pg;
555 struct vm_page *opg;
556 int error;
557 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
558
559 if (oanon) {
560 /* anon COW */
561 opg = oanon->an_page;
562 KASSERT(opg != NULL);
563 KASSERT(opg->uobject == NULL || opg->loan_count > 0);
564 } else if (uobjpage != PGO_DONTCARE) {
565 /* object-backed COW */
566 opg = uobjpage;
567 } else {
568 /* ZFOD */
569 opg = NULL;
570 }
571 if (opg != NULL) {
572 uobj = opg->uobject;
573 } else {
574 uobj = NULL;
575 }
576
577 KASSERT(amap != NULL);
578 KASSERT(uobjpage != NULL);
579 KASSERT(uobjpage == PGO_DONTCARE || (uobjpage->flags & PG_BUSY) != 0);
580 KASSERT(mutex_owned(amap->am_lock));
581 KASSERT(oanon == NULL || amap->am_lock == oanon->an_lock);
582 KASSERT(uobj == NULL || mutex_owned(uobj->vmobjlock));
583
584 if (*spare != NULL) {
585 anon = *spare;
586 *spare = NULL;
587 } else if (ufi->map != kernel_map) {
588 anon = uvm_analloc();
589 } else {
590 UVMHIST_LOG(maphist, "kernel_map, unlock and retry", 0,0,0,0);
591
592 /*
593 * we can't allocate anons with kernel_map locked.
594 */
595
596 uvm_page_unbusy(&uobjpage, 1);
597 uvmfault_unlockall(ufi, amap, uobj);
598
599 *spare = uvm_analloc();
600 if (*spare == NULL) {
601 goto nomem;
602 }
603 KASSERT((*spare)->an_lock == NULL);
604 error = ERESTART;
605 goto done;
606 }
607 if (anon) {
608
609 /*
610 * The new anon is locked.
611 *
612 * if opg == NULL, we want a zero'd, dirty page,
613 * so have uvm_pagealloc() do that for us.
614 */
615
616 KASSERT(anon->an_lock == NULL);
617 anon->an_lock = amap->am_lock;
618 pg = uvm_pagealloc(NULL, ufi->orig_rvaddr, anon,
619 UVM_FLAG_COLORMATCH | (opg == NULL ? UVM_PGA_ZERO : 0));
620 if (pg == NULL) {
621 anon->an_lock = NULL;
622 }
623 } else {
624 pg = NULL;
625 }
626
627 /*
628 * out of memory resources?
629 */
630
631 if (pg == NULL) {
632 /* save anon for the next try. */
633 if (anon != NULL) {
634 *spare = anon;
635 }
636
637 /* unlock and fail ... */
638 uvm_page_unbusy(&uobjpage, 1);
639 uvmfault_unlockall(ufi, amap, uobj);
640 nomem:
641 if (!uvm_reclaimable()) {
642 UVMHIST_LOG(maphist, "out of VM", 0,0,0,0);
643 uvmexp.fltnoanon++;
644 error = ENOMEM;
645 goto done;
646 }
647
648 UVMHIST_LOG(maphist, "out of RAM, waiting for more", 0,0,0,0);
649 uvmexp.fltnoram++;
650 uvm_wait("flt_noram5");
651 error = ERESTART;
652 goto done;
653 }
654
655 /* copy page [pg now dirty] */
656 if (opg) {
657 uvm_pagecopy(opg, pg);
658 }
659 KASSERT(uvm_pagegetdirty(pg) == UVM_PAGE_STATUS_DIRTY);
660
661 amap_add(&ufi->entry->aref, ufi->orig_rvaddr - ufi->entry->start, anon,
662 oanon != NULL);
663
664 *nanon = anon;
665 error = 0;
666 done:
667 return error;
668 }
669
670
671 /*
672 * F A U L T - m a i n e n t r y p o i n t
673 */
674
675 /*
676 * uvm_fault: page fault handler
677 *
678 * => called from MD code to resolve a page fault
679 * => VM data structures usually should be unlocked. however, it is
680 * possible to call here with the main map locked if the caller
681 * gets a write lock, sets it recusive, and then calls us (c.f.
682 * uvm_map_pageable). this should be avoided because it keeps
683 * the map locked off during I/O.
684 * => MUST NEVER BE CALLED IN INTERRUPT CONTEXT
685 */
686
687 #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \
688 ~VM_PROT_WRITE : VM_PROT_ALL)
689
690 /* fault_flag values passed from uvm_fault_wire to uvm_fault_internal */
691 #define UVM_FAULT_WIRE (1 << 0)
692 #define UVM_FAULT_MAXPROT (1 << 1)
693
694 struct uvm_faultctx {
695 vm_prot_t access_type;
696 vm_prot_t enter_prot;
697 vaddr_t startva;
698 int npages;
699 int centeridx;
700 struct vm_anon *anon_spare;
701 bool wire_mapping;
702 bool narrow;
703 bool wire_paging;
704 bool cow_now;
705 bool promote;
706 };
707
708 static inline int uvm_fault_check(
709 struct uvm_faultinfo *, struct uvm_faultctx *,
710 struct vm_anon ***, bool);
711
712 static int uvm_fault_upper(
713 struct uvm_faultinfo *, struct uvm_faultctx *,
714 struct vm_anon **);
715 static inline int uvm_fault_upper_lookup(
716 struct uvm_faultinfo *, const struct uvm_faultctx *,
717 struct vm_anon **, struct vm_page **);
718 static inline void uvm_fault_upper_neighbor(
719 struct uvm_faultinfo *, const struct uvm_faultctx *,
720 vaddr_t, struct vm_page *, bool);
721 static inline int uvm_fault_upper_loan(
722 struct uvm_faultinfo *, struct uvm_faultctx *,
723 struct vm_anon *, struct uvm_object **);
724 static inline int uvm_fault_upper_promote(
725 struct uvm_faultinfo *, struct uvm_faultctx *,
726 struct uvm_object *, struct vm_anon *);
727 static inline int uvm_fault_upper_direct(
728 struct uvm_faultinfo *, struct uvm_faultctx *,
729 struct uvm_object *, struct vm_anon *);
730 static int uvm_fault_upper_enter(
731 struct uvm_faultinfo *, const struct uvm_faultctx *,
732 struct uvm_object *, struct vm_anon *,
733 struct vm_page *, struct vm_anon *);
734 static inline void uvm_fault_upper_done(
735 struct uvm_faultinfo *, const struct uvm_faultctx *,
736 struct vm_anon *, struct vm_page *);
737
738 static int uvm_fault_lower(
739 struct uvm_faultinfo *, struct uvm_faultctx *,
740 struct vm_page **);
741 static inline void uvm_fault_lower_lookup(
742 struct uvm_faultinfo *, const struct uvm_faultctx *,
743 struct vm_page **);
744 static inline void uvm_fault_lower_neighbor(
745 struct uvm_faultinfo *, const struct uvm_faultctx *,
746 vaddr_t, struct vm_page *);
747 static inline int uvm_fault_lower_io(
748 struct uvm_faultinfo *, const struct uvm_faultctx *,
749 struct uvm_object **, struct vm_page **);
750 static inline int uvm_fault_lower_direct(
751 struct uvm_faultinfo *, struct uvm_faultctx *,
752 struct uvm_object *, struct vm_page *);
753 static inline int uvm_fault_lower_direct_loan(
754 struct uvm_faultinfo *, struct uvm_faultctx *,
755 struct uvm_object *, struct vm_page **,
756 struct vm_page **);
757 static inline int uvm_fault_lower_promote(
758 struct uvm_faultinfo *, struct uvm_faultctx *,
759 struct uvm_object *, struct vm_page *);
760 static int uvm_fault_lower_enter(
761 struct uvm_faultinfo *, const struct uvm_faultctx *,
762 struct uvm_object *,
763 struct vm_anon *, struct vm_page *);
764 static inline void uvm_fault_lower_done(
765 struct uvm_faultinfo *, const struct uvm_faultctx *,
766 struct uvm_object *, struct vm_page *);
767
768 int
769 uvm_fault_internal(struct vm_map *orig_map, vaddr_t vaddr,
770 vm_prot_t access_type, int fault_flag)
771 {
772 struct uvm_faultinfo ufi;
773 struct uvm_faultctx flt = {
774 .access_type = access_type,
775
776 /* don't look for neighborhood * pages on "wire" fault */
777 .narrow = (fault_flag & UVM_FAULT_WIRE) != 0,
778
779 /* "wire" fault causes wiring of both mapping and paging */
780 .wire_mapping = (fault_flag & UVM_FAULT_WIRE) != 0,
781 .wire_paging = (fault_flag & UVM_FAULT_WIRE) != 0,
782 };
783 const bool maxprot = (fault_flag & UVM_FAULT_MAXPROT) != 0;
784 struct vm_anon *anons_store[UVM_MAXRANGE], **anons;
785 struct vm_page *pages_store[UVM_MAXRANGE], **pages;
786 int error;
787 UVMHIST_FUNC("uvm_fault"); UVMHIST_CALLED(maphist);
788
789 UVMHIST_LOG(maphist, "(map=0x%x, vaddr=0x%x, at=%d, ff=%d)",
790 orig_map, vaddr, access_type, fault_flag);
791
792 curcpu()->ci_data.cpu_nfault++;
793
794 /*
795 * init the IN parameters in the ufi
796 */
797
798 ufi.orig_map = orig_map;
799 ufi.orig_rvaddr = trunc_page(vaddr);
800 ufi.orig_size = PAGE_SIZE; /* can't get any smaller than this */
801
802 error = ERESTART;
803 while (error == ERESTART) { /* ReFault: */
804 anons = anons_store;
805 pages = pages_store;
806
807 error = uvm_fault_check(&ufi, &flt, &anons, maxprot);
808 if (error != 0)
809 continue;
810
811 error = uvm_fault_upper_lookup(&ufi, &flt, anons, pages);
812 if (error != 0)
813 continue;
814
815 if (pages[flt.centeridx] == PGO_DONTCARE)
816 error = uvm_fault_upper(&ufi, &flt, anons);
817 else {
818 struct uvm_object * const uobj =
819 ufi.entry->object.uvm_obj;
820
821 if (uobj && uobj->pgops->pgo_fault != NULL) {
822 /*
823 * invoke "special" fault routine.
824 */
825 mutex_enter(uobj->vmobjlock);
826 /* locked: maps(read), amap(if there), uobj */
827 error = uobj->pgops->pgo_fault(&ufi,
828 flt.startva, pages, flt.npages,
829 flt.centeridx, flt.access_type,
830 PGO_LOCKED|PGO_SYNCIO);
831
832 /*
833 * locked: nothing, pgo_fault has unlocked
834 * everything
835 */
836
837 /*
838 * object fault routine responsible for
839 * pmap_update().
840 */
841 } else {
842 error = uvm_fault_lower(&ufi, &flt, pages);
843 }
844 }
845 }
846
847 if (flt.anon_spare != NULL) {
848 flt.anon_spare->an_ref--;
849 KASSERT(flt.anon_spare->an_ref == 0);
850 KASSERT(flt.anon_spare->an_lock == NULL);
851 uvm_anon_free(flt.anon_spare);
852 }
853 return error;
854 }
855
856 /*
857 * uvm_fault_check: check prot, handle needs-copy, etc.
858 *
859 * 1. lookup entry.
860 * 2. check protection.
861 * 3. adjust fault condition (mainly for simulated fault).
862 * 4. handle needs-copy (lazy amap copy).
863 * 5. establish range of interest for neighbor fault (aka pre-fault).
864 * 6. look up anons (if amap exists).
865 * 7. flush pages (if MADV_SEQUENTIAL)
866 *
867 * => called with nothing locked.
868 * => if we fail (result != 0) we unlock everything.
869 * => initialize/adjust many members of flt.
870 */
871
872 static int
873 uvm_fault_check(
874 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
875 struct vm_anon ***ranons, bool maxprot)
876 {
877 struct vm_amap *amap;
878 struct uvm_object *uobj;
879 vm_prot_t check_prot;
880 int nback, nforw;
881 UVMHIST_FUNC("uvm_fault_check"); UVMHIST_CALLED(maphist);
882
883 /*
884 * lookup and lock the maps
885 */
886
887 if (uvmfault_lookup(ufi, false) == false) {
888 UVMHIST_LOG(maphist, "<- no mapping @ 0x%x", ufi->orig_rvaddr,
889 0,0,0);
890 return EFAULT;
891 }
892 /* locked: maps(read) */
893
894 #ifdef DIAGNOSTIC
895 if ((ufi->map->flags & VM_MAP_PAGEABLE) == 0) {
896 printf("Page fault on non-pageable map:\n");
897 printf("ufi->map = %p\n", ufi->map);
898 printf("ufi->orig_map = %p\n", ufi->orig_map);
899 printf("ufi->orig_rvaddr = 0x%lx\n", (u_long) ufi->orig_rvaddr);
900 panic("uvm_fault: (ufi->map->flags & VM_MAP_PAGEABLE) == 0");
901 }
902 #endif
903
904 /*
905 * check protection
906 */
907
908 check_prot = maxprot ?
909 ufi->entry->max_protection : ufi->entry->protection;
910 if ((check_prot & flt->access_type) != flt->access_type) {
911 UVMHIST_LOG(maphist,
912 "<- protection failure (prot=0x%x, access=0x%x)",
913 ufi->entry->protection, flt->access_type, 0, 0);
914 uvmfault_unlockmaps(ufi, false);
915 return EACCES;
916 }
917
918 /*
919 * "enter_prot" is the protection we want to enter the page in at.
920 * for certain pages (e.g. copy-on-write pages) this protection can
921 * be more strict than ufi->entry->protection. "wired" means either
922 * the entry is wired or we are fault-wiring the pg.
923 */
924
925 flt->enter_prot = ufi->entry->protection;
926 if (VM_MAPENT_ISWIRED(ufi->entry))
927 flt->wire_mapping = true;
928
929 if (flt->wire_mapping) {
930 flt->access_type = flt->enter_prot; /* full access for wired */
931 flt->cow_now = (check_prot & VM_PROT_WRITE) != 0;
932 } else {
933 flt->cow_now = (flt->access_type & VM_PROT_WRITE) != 0;
934 }
935
936 flt->promote = false;
937
938 /*
939 * handle "needs_copy" case. if we need to copy the amap we will
940 * have to drop our readlock and relock it with a write lock. (we
941 * need a write lock to change anything in a map entry [e.g.
942 * needs_copy]).
943 */
944
945 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) {
946 if (flt->cow_now || (ufi->entry->object.uvm_obj == NULL)) {
947 KASSERT(!maxprot);
948 /* need to clear */
949 UVMHIST_LOG(maphist,
950 " need to clear needs_copy and refault",0,0,0,0);
951 uvmfault_unlockmaps(ufi, false);
952 uvmfault_amapcopy(ufi);
953 uvmexp.fltamcopy++;
954 return ERESTART;
955
956 } else {
957
958 /*
959 * ensure that we pmap_enter page R/O since
960 * needs_copy is still true
961 */
962
963 flt->enter_prot &= ~VM_PROT_WRITE;
964 }
965 }
966
967 /*
968 * identify the players
969 */
970
971 amap = ufi->entry->aref.ar_amap; /* upper layer */
972 uobj = ufi->entry->object.uvm_obj; /* lower layer */
973
974 /*
975 * check for a case 0 fault. if nothing backing the entry then
976 * error now.
977 */
978
979 if (amap == NULL && uobj == NULL) {
980 uvmfault_unlockmaps(ufi, false);
981 UVMHIST_LOG(maphist,"<- no backing store, no overlay",0,0,0,0);
982 return EFAULT;
983 }
984
985 /*
986 * establish range of interest based on advice from mapper
987 * and then clip to fit map entry. note that we only want
988 * to do this the first time through the fault. if we
989 * ReFault we will disable this by setting "narrow" to true.
990 */
991
992 if (flt->narrow == false) {
993
994 /* wide fault (!narrow) */
995 KASSERT(uvmadvice[ufi->entry->advice].advice ==
996 ufi->entry->advice);
997 nback = MIN(uvmadvice[ufi->entry->advice].nback,
998 (ufi->orig_rvaddr - ufi->entry->start) >> PAGE_SHIFT);
999 flt->startva = ufi->orig_rvaddr - (nback << PAGE_SHIFT);
1000 /*
1001 * note: "-1" because we don't want to count the
1002 * faulting page as forw
1003 */
1004 nforw = MIN(uvmadvice[ufi->entry->advice].nforw,
1005 ((ufi->entry->end - ufi->orig_rvaddr) >>
1006 PAGE_SHIFT) - 1);
1007 flt->npages = nback + nforw + 1;
1008 flt->centeridx = nback;
1009
1010 flt->narrow = true; /* ensure only once per-fault */
1011
1012 } else {
1013
1014 /* narrow fault! */
1015 nback = nforw = 0;
1016 flt->startva = ufi->orig_rvaddr;
1017 flt->npages = 1;
1018 flt->centeridx = 0;
1019
1020 }
1021 /* offset from entry's start to pgs' start */
1022 const voff_t eoff = flt->startva - ufi->entry->start;
1023
1024 /* locked: maps(read) */
1025 UVMHIST_LOG(maphist, " narrow=%d, back=%d, forw=%d, startva=0x%x",
1026 flt->narrow, nback, nforw, flt->startva);
1027 UVMHIST_LOG(maphist, " entry=0x%x, amap=0x%x, obj=0x%x", ufi->entry,
1028 amap, uobj, 0);
1029
1030 /*
1031 * if we've got an amap, lock it and extract current anons.
1032 */
1033
1034 if (amap) {
1035 amap_lock(amap);
1036 amap_lookups(&ufi->entry->aref, eoff, *ranons, flt->npages);
1037 } else {
1038 *ranons = NULL; /* to be safe */
1039 }
1040
1041 /* locked: maps(read), amap(if there) */
1042 KASSERT(amap == NULL || mutex_owned(amap->am_lock));
1043
1044 /*
1045 * for MADV_SEQUENTIAL mappings we want to deactivate the back pages
1046 * now and then forget about them (for the rest of the fault).
1047 */
1048
1049 if (ufi->entry->advice == MADV_SEQUENTIAL && nback != 0) {
1050
1051 UVMHIST_LOG(maphist, " MADV_SEQUENTIAL: flushing backpages",
1052 0,0,0,0);
1053 /* flush back-page anons? */
1054 if (amap)
1055 uvmfault_anonflush(*ranons, nback);
1056
1057 /* flush object? */
1058 if (uobj) {
1059 voff_t uoff;
1060
1061 uoff = ufi->entry->offset + eoff;
1062 mutex_enter(uobj->vmobjlock);
1063 (void) (uobj->pgops->pgo_put)(uobj, uoff, uoff +
1064 (nback << PAGE_SHIFT), PGO_DEACTIVATE);
1065 }
1066
1067 /* now forget about the backpages */
1068 if (amap)
1069 *ranons += nback;
1070 flt->startva += (nback << PAGE_SHIFT);
1071 flt->npages -= nback;
1072 flt->centeridx = 0;
1073 }
1074 /*
1075 * => startva is fixed
1076 * => npages is fixed
1077 */
1078 KASSERT(flt->startva <= ufi->orig_rvaddr);
1079 KASSERT(ufi->orig_rvaddr + ufi->orig_size <=
1080 flt->startva + (flt->npages << PAGE_SHIFT));
1081 return 0;
1082 }
1083
1084 /*
1085 * uvm_fault_upper_lookup: look up existing h/w mapping and amap.
1086 *
1087 * iterate range of interest:
1088 * 1. check if h/w mapping exists. if yes, we don't care
1089 * 2. check if anon exists. if not, page is lower.
1090 * 3. if anon exists, enter h/w mapping for neighbors.
1091 *
1092 * => called with amap locked (if exists).
1093 */
1094
1095 static int
1096 uvm_fault_upper_lookup(
1097 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
1098 struct vm_anon **anons, struct vm_page **pages)
1099 {
1100 struct vm_amap *amap = ufi->entry->aref.ar_amap;
1101 int lcv;
1102 vaddr_t currva;
1103 bool shadowed;
1104 UVMHIST_FUNC("uvm_fault_upper_lookup"); UVMHIST_CALLED(maphist);
1105
1106 /* locked: maps(read), amap(if there) */
1107 KASSERT(amap == NULL || mutex_owned(amap->am_lock));
1108
1109 /*
1110 * map in the backpages and frontpages we found in the amap in hopes
1111 * of preventing future faults. we also init the pages[] array as
1112 * we go.
1113 */
1114
1115 currva = flt->startva;
1116 shadowed = false;
1117 for (lcv = 0; lcv < flt->npages; lcv++, currva += PAGE_SIZE) {
1118 /*
1119 * don't play with VAs that are already mapped
1120 * (except for center)
1121 */
1122 if (lcv != flt->centeridx &&
1123 pmap_extract(ufi->orig_map->pmap, currva, NULL)) {
1124 pages[lcv] = PGO_DONTCARE;
1125 continue;
1126 }
1127
1128 /*
1129 * unmapped or center page. check if any anon at this level.
1130 */
1131 if (amap == NULL || anons[lcv] == NULL) {
1132 pages[lcv] = NULL;
1133 continue;
1134 }
1135
1136 /*
1137 * check for present page and map if possible. re-activate it.
1138 */
1139
1140 pages[lcv] = PGO_DONTCARE;
1141 if (lcv == flt->centeridx) { /* save center for later! */
1142 shadowed = true;
1143 continue;
1144 }
1145
1146 struct vm_anon *anon = anons[lcv];
1147 struct vm_page *pg = anon->an_page;
1148
1149 KASSERT(anon->an_lock == amap->am_lock);
1150
1151 /* Ignore loaned and busy pages. */
1152 if (pg && pg->loan_count == 0 && (pg->flags & PG_BUSY) == 0) {
1153 uvm_fault_upper_neighbor(ufi, flt, currva,
1154 pg, anon->an_ref > 1);
1155 }
1156 }
1157
1158 /* locked: maps(read), amap(if there) */
1159 KASSERT(amap == NULL || mutex_owned(amap->am_lock));
1160 /* (shadowed == true) if there is an anon at the faulting address */
1161 UVMHIST_LOG(maphist, " shadowed=%d, will_get=%d", shadowed,
1162 (ufi->entry->object.uvm_obj && shadowed != false),0,0);
1163
1164 /*
1165 * note that if we are really short of RAM we could sleep in the above
1166 * call to pmap_enter with everything locked. bad?
1167 *
1168 * XXX Actually, that is bad; pmap_enter() should just fail in that
1169 * XXX case. --thorpej
1170 */
1171
1172 return 0;
1173 }
1174
1175 /*
1176 * uvm_fault_upper_neighbor: enter single lower neighbor page.
1177 *
1178 * => called with amap and anon locked.
1179 */
1180
1181 static void
1182 uvm_fault_upper_neighbor(
1183 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
1184 vaddr_t currva, struct vm_page *pg, bool readonly)
1185 {
1186 UVMHIST_FUNC("uvm_fault_upper_neighbor"); UVMHIST_CALLED(maphist);
1187
1188 /* locked: amap, anon */
1189 KASSERT(pg->uobject == NULL);
1190 KASSERT(pg->uanon != NULL);
1191 KASSERT(mutex_owned(pg->uanon->an_lock));
1192 KASSERT(uvm_pagegetdirty(pg) != UVM_PAGE_STATUS_CLEAN);
1193 mutex_enter(&uvm_pageqlock);
1194 uvm_pageenqueue(pg);
1195 mutex_exit(&uvm_pageqlock);
1196 UVMHIST_LOG(maphist,
1197 " MAPPING: n anon: pm=0x%x, va=0x%x, pg=0x%x",
1198 ufi->orig_map->pmap, currva, pg, 0);
1199 uvmexp.fltnamap++;
1200
1201 /*
1202 * Since this page isn't the page that's actually faulting,
1203 * ignore pmap_enter() failures; it's not critical that we
1204 * enter these right now.
1205 */
1206
1207 (void) pmap_enter(ufi->orig_map->pmap, currva,
1208 VM_PAGE_TO_PHYS(pg),
1209 readonly ? (flt->enter_prot & ~VM_PROT_WRITE) :
1210 flt->enter_prot,
1211 PMAP_CANFAIL | (flt->wire_mapping ? PMAP_WIRED : 0));
1212
1213 pmap_update(ufi->orig_map->pmap);
1214 }
1215
1216 /*
1217 * uvm_fault_upper: handle upper fault.
1218 *
1219 * 1. acquire anon lock.
1220 * 2. get anon. let uvmfault_anonget do the dirty work.
1221 * 3. handle loan.
1222 * 4. dispatch direct or promote handlers.
1223 */
1224
1225 static int
1226 uvm_fault_upper(
1227 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1228 struct vm_anon **anons)
1229 {
1230 struct vm_amap * const amap = ufi->entry->aref.ar_amap;
1231 struct vm_anon * const anon = anons[flt->centeridx];
1232 struct uvm_object *uobj;
1233 int error;
1234 UVMHIST_FUNC("uvm_fault_upper"); UVMHIST_CALLED(maphist);
1235
1236 /* locked: maps(read), amap, anon */
1237 KASSERT(mutex_owned(amap->am_lock));
1238 KASSERT(anon->an_lock == amap->am_lock);
1239
1240 /*
1241 * handle case 1: fault on an anon in our amap
1242 */
1243
1244 UVMHIST_LOG(maphist, " case 1 fault: anon=0x%x", anon, 0,0,0);
1245
1246 /*
1247 * no matter if we have case 1A or case 1B we are going to need to
1248 * have the anon's memory resident. ensure that now.
1249 */
1250
1251 /*
1252 * let uvmfault_anonget do the dirty work.
1253 * if it fails (!OK) it will unlock everything for us.
1254 * if it succeeds, locks are still valid and locked.
1255 * also, if it is OK, then the anon's page is on the queues.
1256 * if the page is on loan from a uvm_object, then anonget will
1257 * lock that object for us if it does not fail.
1258 */
1259
1260 error = uvmfault_anonget(ufi, amap, anon);
1261 switch (error) {
1262 case 0:
1263 break;
1264
1265 case ERESTART:
1266 return ERESTART;
1267
1268 case EAGAIN:
1269 kpause("fltagain1", false, hz/2, NULL);
1270 return ERESTART;
1271
1272 default:
1273 return error;
1274 }
1275
1276 /*
1277 * uobj is non null if the page is on loan from an object (i.e. uobj)
1278 */
1279
1280 uobj = anon->an_page->uobject; /* locked by anonget if !NULL */
1281
1282 /* locked: maps(read), amap, anon, uobj(if one) */
1283 KASSERT(mutex_owned(amap->am_lock));
1284 KASSERT(anon->an_lock == amap->am_lock);
1285 KASSERT(uobj == NULL || mutex_owned(uobj->vmobjlock));
1286
1287 /*
1288 * special handling for loaned pages
1289 */
1290
1291 if (anon->an_page->loan_count) {
1292 error = uvm_fault_upper_loan(ufi, flt, anon, &uobj);
1293 if (error != 0)
1294 return error;
1295 }
1296
1297 /*
1298 * if we are case 1B then we will need to allocate a new blank
1299 * anon to transfer the data into. note that we have a lock
1300 * on anon, so no one can busy or release the page until we are done.
1301 * also note that the ref count can't drop to zero here because
1302 * it is > 1 and we are only dropping one ref.
1303 *
1304 * in the (hopefully very rare) case that we are out of RAM we
1305 * will unlock, wait for more RAM, and refault.
1306 *
1307 * if we are out of anon VM we kill the process (XXX: could wait?).
1308 */
1309
1310 if (flt->cow_now && anon->an_ref > 1) {
1311 flt->promote = true;
1312 error = uvm_fault_upper_promote(ufi, flt, uobj, anon);
1313 } else {
1314 error = uvm_fault_upper_direct(ufi, flt, uobj, anon);
1315 }
1316 return error;
1317 }
1318
1319 /*
1320 * uvm_fault_upper_loan: handle loaned upper page.
1321 *
1322 * 1. if not cow'ing now, simply adjust flt->enter_prot.
1323 * 2. if cow'ing now, and if ref count is 1, break loan.
1324 */
1325
1326 static int
1327 uvm_fault_upper_loan(
1328 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1329 struct vm_anon *anon, struct uvm_object **ruobj)
1330 {
1331 struct vm_amap * const amap = ufi->entry->aref.ar_amap;
1332 int error = 0;
1333 UVMHIST_FUNC("uvm_fault_upper_loan"); UVMHIST_CALLED(maphist);
1334
1335 if (!flt->cow_now) {
1336
1337 /*
1338 * for read faults on loaned pages we just cap the
1339 * protection at read-only.
1340 */
1341
1342 flt->enter_prot = flt->enter_prot & ~VM_PROT_WRITE;
1343
1344 } else {
1345 /*
1346 * note that we can't allow writes into a loaned page!
1347 *
1348 * if we have a write fault on a loaned page in an
1349 * anon then we need to look at the anon's ref count.
1350 * if it is greater than one then we are going to do
1351 * a normal copy-on-write fault into a new anon (this
1352 * is not a problem). however, if the reference count
1353 * is one (a case where we would normally allow a
1354 * write directly to the page) then we need to kill
1355 * the loan before we continue.
1356 */
1357
1358 /* >1 case is already ok */
1359 if (anon->an_ref == 1) {
1360 error = uvm_loanbreak_anon(anon, *ruobj);
1361 if (error != 0) {
1362 uvmfault_unlockall(ufi, amap, *ruobj);
1363 uvm_wait("flt_noram2");
1364 return ERESTART;
1365 }
1366 /* if we were a loan reciever uobj is gone */
1367 if (*ruobj)
1368 *ruobj = NULL;
1369 }
1370 }
1371 return error;
1372 }
1373
1374 /*
1375 * uvm_fault_upper_promote: promote upper page.
1376 *
1377 * 1. call uvmfault_promote.
1378 * 2. enqueue page.
1379 * 3. deref.
1380 * 4. pass page to uvm_fault_upper_enter.
1381 */
1382
1383 static int
1384 uvm_fault_upper_promote(
1385 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1386 struct uvm_object *uobj, struct vm_anon *anon)
1387 {
1388 struct vm_anon * const oanon = anon;
1389 struct vm_page *pg;
1390 int error;
1391 UVMHIST_FUNC("uvm_fault_upper_promote"); UVMHIST_CALLED(maphist);
1392
1393 UVMHIST_LOG(maphist, " case 1B: COW fault",0,0,0,0);
1394 uvmexp.flt_acow++;
1395
1396 error = uvmfault_promote(ufi, oanon, PGO_DONTCARE, &anon,
1397 &flt->anon_spare);
1398 switch (error) {
1399 case 0:
1400 break;
1401 case ERESTART:
1402 return ERESTART;
1403 default:
1404 return error;
1405 }
1406
1407 KASSERT(anon == NULL || anon->an_lock == oanon->an_lock);
1408
1409 pg = anon->an_page;
1410 mutex_enter(&uvm_pageqlock);
1411 uvm_pageenqueue(pg); /* uvm_fault_upper_done will activate the page */
1412 mutex_exit(&uvm_pageqlock);
1413 pg->flags &= ~(PG_BUSY|PG_FAKE);
1414 UVM_PAGE_OWN(pg, NULL);
1415
1416 /* deref: can not drop to zero here by defn! */
1417 KASSERT(oanon->an_ref > 1);
1418 oanon->an_ref--;
1419
1420 /*
1421 * note: oanon is still locked, as is the new anon. we
1422 * need to check for this later when we unlock oanon; if
1423 * oanon != anon, we'll have to unlock anon, too.
1424 */
1425
1426 return uvm_fault_upper_enter(ufi, flt, uobj, anon, pg, oanon);
1427 }
1428
1429 /*
1430 * uvm_fault_upper_direct: handle direct fault.
1431 */
1432
1433 static int
1434 uvm_fault_upper_direct(
1435 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1436 struct uvm_object *uobj, struct vm_anon *anon)
1437 {
1438 struct vm_anon * const oanon = anon;
1439 struct vm_page *pg;
1440 UVMHIST_FUNC("uvm_fault_upper_direct"); UVMHIST_CALLED(maphist);
1441
1442 uvmexp.flt_anon++;
1443 pg = anon->an_page;
1444 if (anon->an_ref > 1) /* disallow writes to ref > 1 anons */
1445 flt->enter_prot = flt->enter_prot & ~VM_PROT_WRITE;
1446
1447 return uvm_fault_upper_enter(ufi, flt, uobj, anon, pg, oanon);
1448 }
1449
1450 /*
1451 * uvm_fault_upper_enter: enter h/w mapping of upper page.
1452 */
1453
1454 static int
1455 uvm_fault_upper_enter(
1456 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
1457 struct uvm_object *uobj, struct vm_anon *anon, struct vm_page *pg,
1458 struct vm_anon *oanon)
1459 {
1460 struct vm_amap * const amap = ufi->entry->aref.ar_amap;
1461 UVMHIST_FUNC("uvm_fault_upper_enter"); UVMHIST_CALLED(maphist);
1462
1463 /* locked: maps(read), amap, oanon, anon(if different from oanon) */
1464 KASSERT(mutex_owned(amap->am_lock));
1465 KASSERT(anon->an_lock == amap->am_lock);
1466 KASSERT(oanon->an_lock == amap->am_lock);
1467 KASSERT(uobj == NULL || mutex_owned(uobj->vmobjlock));
1468 KASSERT(uvm_pagegetdirty(pg) != UVM_PAGE_STATUS_CLEAN);
1469
1470 /*
1471 * now map the page in.
1472 */
1473
1474 UVMHIST_LOG(maphist,
1475 " MAPPING: anon: pm=0x%x, va=0x%x, pg=0x%x, promote=%d",
1476 ufi->orig_map->pmap, ufi->orig_rvaddr, pg, flt->promote);
1477 if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr,
1478 VM_PAGE_TO_PHYS(pg),
1479 flt->enter_prot, flt->access_type | PMAP_CANFAIL |
1480 (flt->wire_mapping ? PMAP_WIRED : 0)) != 0) {
1481
1482 /*
1483 * No need to undo what we did; we can simply think of
1484 * this as the pmap throwing away the mapping information.
1485 *
1486 * We do, however, have to go through the ReFault path,
1487 * as the map may change while we're asleep.
1488 */
1489
1490 uvmfault_unlockall(ufi, amap, uobj);
1491 if (!uvm_reclaimable()) {
1492 UVMHIST_LOG(maphist,
1493 "<- failed. out of VM",0,0,0,0);
1494 /* XXX instrumentation */
1495 return ENOMEM;
1496 }
1497 /* XXX instrumentation */
1498 uvm_wait("flt_pmfail1");
1499 return ERESTART;
1500 }
1501
1502 uvm_fault_upper_done(ufi, flt, anon, pg);
1503
1504 /*
1505 * done case 1! finish up by unlocking everything and returning success
1506 */
1507
1508 pmap_update(ufi->orig_map->pmap);
1509 uvmfault_unlockall(ufi, amap, uobj);
1510 return 0;
1511 }
1512
1513 /*
1514 * uvm_fault_upper_done: queue upper center page.
1515 */
1516
1517 static void
1518 uvm_fault_upper_done(
1519 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
1520 struct vm_anon *anon, struct vm_page *pg)
1521 {
1522 const bool wire_paging = flt->wire_paging;
1523
1524 UVMHIST_FUNC("uvm_fault_upper_done"); UVMHIST_CALLED(maphist);
1525
1526 /*
1527 * ... update the page queues.
1528 */
1529
1530 mutex_enter(&uvm_pageqlock);
1531 if (wire_paging) {
1532 uvm_pagewire(pg);
1533
1534 /*
1535 * since the now-wired page cannot be paged out,
1536 * release its swap resources for others to use.
1537 * since an anon with no swap cannot be clean,
1538 * mark it dirty now.
1539 */
1540
1541 uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY);
1542 } else {
1543 uvm_pageactivate(pg);
1544 }
1545 mutex_exit(&uvm_pageqlock);
1546
1547 if (wire_paging) {
1548 uvm_anon_dropswap(anon);
1549 }
1550 }
1551
1552 /*
1553 * uvm_fault_lower: handle lower fault.
1554 *
1555 * 1. check uobj
1556 * 1.1. if null, ZFOD.
1557 * 1.2. if not null, look up unnmapped neighbor pages.
1558 * 2. for center page, check if promote.
1559 * 2.1. ZFOD always needs promotion.
1560 * 2.2. other uobjs, when entry is marked COW (usually MAP_PRIVATE vnode).
1561 * 3. if uobj is not ZFOD and page is not found, do i/o.
1562 * 4. dispatch either direct / promote fault.
1563 */
1564
1565 static int
1566 uvm_fault_lower(
1567 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1568 struct vm_page **pages)
1569 {
1570 #ifdef DIAGNOSTIC
1571 struct vm_amap *amap = ufi->entry->aref.ar_amap;
1572 #endif
1573 struct uvm_object *uobj = ufi->entry->object.uvm_obj;
1574 struct vm_page *uobjpage;
1575 int error;
1576 UVMHIST_FUNC("uvm_fault_lower"); UVMHIST_CALLED(maphist);
1577
1578 /*
1579 * now, if the desired page is not shadowed by the amap and we have
1580 * a backing object that does not have a special fault routine, then
1581 * we ask (with pgo_get) the object for resident pages that we care
1582 * about and attempt to map them in. we do not let pgo_get block
1583 * (PGO_LOCKED).
1584 */
1585
1586 if (uobj == NULL) {
1587 /* zero fill; don't care neighbor pages */
1588 uobjpage = NULL;
1589 } else {
1590 uvm_fault_lower_lookup(ufi, flt, pages);
1591 uobjpage = pages[flt->centeridx];
1592 }
1593
1594 /*
1595 * note that at this point we are done with any front or back pages.
1596 * we are now going to focus on the center page (i.e. the one we've
1597 * faulted on). if we have faulted on the upper (anon) layer
1598 * [i.e. case 1], then the anon we want is anons[centeridx] (we have
1599 * not touched it yet). if we have faulted on the bottom (uobj)
1600 * layer [i.e. case 2] and the page was both present and available,
1601 * then we've got a pointer to it as "uobjpage" and we've already
1602 * made it BUSY.
1603 */
1604
1605 /*
1606 * locked:
1607 * maps(read), amap(if there), uobj(if !null), uobjpage(if !null)
1608 */
1609 KASSERT(amap == NULL || mutex_owned(amap->am_lock));
1610 KASSERT(uobj == NULL || mutex_owned(uobj->vmobjlock));
1611 KASSERT(uobjpage == NULL || (uobjpage->flags & PG_BUSY) != 0);
1612
1613 /*
1614 * note that uobjpage can not be PGO_DONTCARE at this point. we now
1615 * set uobjpage to PGO_DONTCARE if we are doing a zero fill. if we
1616 * have a backing object, check and see if we are going to promote
1617 * the data up to an anon during the fault.
1618 */
1619
1620 if (uobj == NULL) {
1621 uobjpage = PGO_DONTCARE;
1622 flt->promote = true; /* always need anon here */
1623 } else {
1624 KASSERT(uobjpage != PGO_DONTCARE);
1625 flt->promote = flt->cow_now && UVM_ET_ISCOPYONWRITE(ufi->entry);
1626 }
1627 UVMHIST_LOG(maphist, " case 2 fault: promote=%d, zfill=%d",
1628 flt->promote, (uobj == NULL), 0,0);
1629
1630 /*
1631 * if uobjpage is not null then we do not need to do I/O to get the
1632 * uobjpage.
1633 *
1634 * if uobjpage is null, then we need to unlock and ask the pager to
1635 * get the data for us. once we have the data, we need to reverify
1636 * the state the world. we are currently not holding any resources.
1637 */
1638
1639 if (uobjpage) {
1640 /* update rusage counters */
1641 curlwp->l_ru.ru_minflt++;
1642 } else {
1643 error = uvm_fault_lower_io(ufi, flt, &uobj, &uobjpage);
1644 if (error != 0)
1645 return error;
1646 }
1647
1648 /*
1649 * locked:
1650 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj)
1651 */
1652 KASSERT(amap == NULL || mutex_owned(amap->am_lock));
1653 KASSERT(uobj == NULL || mutex_owned(uobj->vmobjlock));
1654 KASSERT(uobj == NULL || (uobjpage->flags & PG_BUSY) != 0);
1655
1656 /*
1657 * notes:
1658 * - at this point uobjpage can not be NULL
1659 * - at this point uobjpage can not be PG_RELEASED (since we checked
1660 * for it above)
1661 * - at this point uobjpage could be PG_WANTED (handle later)
1662 */
1663
1664 KASSERT(uobjpage != NULL);
1665 KASSERT(uobj == NULL || uobj == uobjpage->uobject);
1666 KASSERT(uobj == NULL || !UVM_OBJ_IS_CLEAN(uobjpage->uobject) ||
1667 uvm_pagegetdirty(uobjpage) == UVM_PAGE_STATUS_CLEAN);
1668
1669 if (!flt->promote) {
1670 error = uvm_fault_lower_direct(ufi, flt, uobj, uobjpage);
1671 } else {
1672 error = uvm_fault_lower_promote(ufi, flt, uobj, uobjpage);
1673 }
1674 return error;
1675 }
1676
1677 /*
1678 * uvm_fault_lower_lookup: look up on-memory uobj pages.
1679 *
1680 * 1. get on-memory pages.
1681 * 2. if failed, give up (get only center page later).
1682 * 3. if succeeded, enter h/w mapping of neighbor pages.
1683 */
1684
1685 static void
1686 uvm_fault_lower_lookup(
1687 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
1688 struct vm_page **pages)
1689 {
1690 struct uvm_object *uobj = ufi->entry->object.uvm_obj;
1691 int lcv, gotpages;
1692 vaddr_t currva;
1693 UVMHIST_FUNC("uvm_fault_lower_lookup"); UVMHIST_CALLED(maphist);
1694
1695 mutex_enter(uobj->vmobjlock);
1696 /* Locked: maps(read), amap(if there), uobj */
1697
1698 uvmexp.fltlget++;
1699 gotpages = flt->npages;
1700 (void) uobj->pgops->pgo_get(uobj,
1701 ufi->entry->offset + flt->startva - ufi->entry->start,
1702 pages, &gotpages, flt->centeridx,
1703 flt->access_type & MASK(ufi->entry), ufi->entry->advice, PGO_LOCKED);
1704
1705 KASSERT(mutex_owned(uobj->vmobjlock));
1706
1707 /*
1708 * check for pages to map, if we got any
1709 */
1710
1711 if (gotpages == 0) {
1712 pages[flt->centeridx] = NULL;
1713 return;
1714 }
1715
1716 currva = flt->startva;
1717 for (lcv = 0; lcv < flt->npages; lcv++, currva += PAGE_SIZE) {
1718 struct vm_page *curpg;
1719
1720 curpg = pages[lcv];
1721 if (curpg == NULL || curpg == PGO_DONTCARE) {
1722 continue;
1723 }
1724 KASSERT(curpg->uobject == uobj);
1725
1726 /*
1727 * if center page is resident and not PG_BUSY|PG_RELEASED
1728 * then pgo_get made it PG_BUSY for us and gave us a handle
1729 * to it.
1730 */
1731
1732 if (lcv == flt->centeridx) {
1733 UVMHIST_LOG(maphist, " got uobjpage "
1734 "(0x%x) with locked get",
1735 curpg, 0,0,0);
1736 } else {
1737 uvm_fault_lower_neighbor(ufi, flt, currva, curpg);
1738 }
1739 }
1740 pmap_update(ufi->orig_map->pmap);
1741 }
1742
1743 /*
1744 * uvm_fault_lower_neighbor: enter h/w mapping of lower neighbor page.
1745 */
1746
1747 static void
1748 uvm_fault_lower_neighbor(
1749 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
1750 vaddr_t currva, struct vm_page *pg)
1751 {
1752 const bool readonly = uvm_pagereadonly_p(pg) || pg->loan_count > 0;
1753 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
1754
1755 /* locked: maps(read), amap(if there), uobj */
1756
1757 /*
1758 * calling pgo_get with PGO_LOCKED returns us pages which
1759 * are neither busy nor released, so we don't need to check
1760 * for this. we can just directly enter the pages.
1761 */
1762
1763 mutex_enter(&uvm_pageqlock);
1764 uvm_pageenqueue(pg);
1765 mutex_exit(&uvm_pageqlock);
1766 UVMHIST_LOG(maphist,
1767 " MAPPING: n obj: pm=0x%x, va=0x%x, pg=0x%x",
1768 ufi->orig_map->pmap, currva, pg, 0);
1769 uvmexp.fltnomap++;
1770
1771 /*
1772 * Since this page isn't the page that's actually faulting,
1773 * ignore pmap_enter() failures; it's not critical that we
1774 * enter these right now.
1775 * NOTE: page can't be PG_WANTED or PG_RELEASED because we've
1776 * held the lock the whole time we've had the handle.
1777 */
1778 KASSERT((pg->flags & PG_PAGEOUT) == 0);
1779 KASSERT((pg->flags & PG_RELEASED) == 0);
1780 KASSERT((pg->flags & PG_WANTED) == 0);
1781 KASSERT(!UVM_OBJ_IS_CLEAN(pg->uobject) ||
1782 uvm_pagegetdirty(pg) == UVM_PAGE_STATUS_CLEAN);
1783 pg->flags &= ~(PG_BUSY);
1784 UVM_PAGE_OWN(pg, NULL);
1785
1786 KASSERT(mutex_owned(pg->uobject->vmobjlock));
1787 (void) pmap_enter(ufi->orig_map->pmap, currva,
1788 VM_PAGE_TO_PHYS(pg),
1789 readonly ? (flt->enter_prot & ~VM_PROT_WRITE) :
1790 flt->enter_prot & MASK(ufi->entry),
1791 PMAP_CANFAIL | (flt->wire_mapping ? PMAP_WIRED : 0));
1792 }
1793
1794 /*
1795 * uvm_fault_lower_io: get lower page from backing store.
1796 *
1797 * 1. unlock everything, because i/o will block.
1798 * 2. call pgo_get.
1799 * 3. if failed, recover.
1800 * 4. if succeeded, relock everything and verify things.
1801 */
1802
1803 static int
1804 uvm_fault_lower_io(
1805 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
1806 struct uvm_object **ruobj, struct vm_page **ruobjpage)
1807 {
1808 struct vm_amap * const amap = ufi->entry->aref.ar_amap;
1809 struct uvm_object *uobj = *ruobj;
1810 struct vm_page *pg;
1811 bool locked;
1812 int gotpages;
1813 int error;
1814 voff_t uoff;
1815 UVMHIST_FUNC("uvm_fault_lower_io"); UVMHIST_CALLED(maphist);
1816
1817 /* update rusage counters */
1818 curlwp->l_ru.ru_majflt++;
1819
1820 /* Locked: maps(read), amap(if there), uobj */
1821 uvmfault_unlockall(ufi, amap, NULL);
1822
1823 /* Locked: uobj */
1824 KASSERT(uobj == NULL || mutex_owned(uobj->vmobjlock));
1825
1826 uvmexp.fltget++;
1827 gotpages = 1;
1828 pg = NULL;
1829 uoff = (ufi->orig_rvaddr - ufi->entry->start) + ufi->entry->offset;
1830 error = uobj->pgops->pgo_get(uobj, uoff, &pg, &gotpages,
1831 0, flt->access_type & MASK(ufi->entry), ufi->entry->advice,
1832 PGO_SYNCIO);
1833 /* locked: pg(if no error) */
1834
1835 /*
1836 * recover from I/O
1837 */
1838
1839 if (error) {
1840 if (error == EAGAIN) {
1841 UVMHIST_LOG(maphist,
1842 " pgo_get says TRY AGAIN!",0,0,0,0);
1843 kpause("fltagain2", false, hz/2, NULL);
1844 return ERESTART;
1845 }
1846
1847 #if 0
1848 KASSERT(error != ERESTART);
1849 #else
1850 /* XXXUEBS don't re-fault? */
1851 if (error == ERESTART)
1852 error = EIO;
1853 #endif
1854
1855 UVMHIST_LOG(maphist, "<- pgo_get failed (code %d)",
1856 error, 0,0,0);
1857 return error;
1858 }
1859
1860 /*
1861 * re-verify the state of the world by first trying to relock
1862 * the maps. always relock the object.
1863 */
1864
1865 locked = uvmfault_relock(ufi);
1866 if (locked && amap)
1867 amap_lock(amap);
1868
1869 /* might be changed */
1870 uobj = pg->uobject;
1871
1872 mutex_enter(uobj->vmobjlock);
1873 KASSERT((pg->flags & PG_BUSY) != 0);
1874
1875 mutex_enter(&uvm_pageqlock);
1876 uvm_pageactivate(pg);
1877 mutex_exit(&uvm_pageqlock);
1878
1879 /* locked(locked): maps(read), amap(if !null), uobj, pg */
1880 /* locked(!locked): uobj, pg */
1881
1882 /*
1883 * verify that the page has not be released and re-verify
1884 * that amap slot is still free. if there is a problem,
1885 * we unlock and clean up.
1886 */
1887
1888 if ((pg->flags & PG_RELEASED) != 0 ||
1889 (locked && amap && amap_lookup(&ufi->entry->aref,
1890 ufi->orig_rvaddr - ufi->entry->start))) {
1891 if (locked)
1892 uvmfault_unlockall(ufi, amap, NULL);
1893 locked = false;
1894 }
1895
1896 /*
1897 * didn't get the lock? release the page and retry.
1898 */
1899
1900 if (locked == false) {
1901 UVMHIST_LOG(maphist,
1902 " wasn't able to relock after fault: retry",
1903 0,0,0,0);
1904 if (pg->flags & PG_WANTED) {
1905 wakeup(pg);
1906 }
1907 if ((pg->flags & PG_RELEASED) == 0) {
1908 pg->flags &= ~(PG_BUSY | PG_WANTED);
1909 UVM_PAGE_OWN(pg, NULL);
1910 } else {
1911 uvmexp.fltpgrele++;
1912 uvm_pagefree(pg);
1913 }
1914 mutex_exit(uobj->vmobjlock);
1915 return ERESTART;
1916 }
1917
1918 /*
1919 * we have the data in pg which is busy and
1920 * not released. we are holding object lock (so the page
1921 * can't be released on us).
1922 */
1923
1924 /* locked: maps(read), amap(if !null), uobj, pg */
1925
1926 *ruobj = uobj;
1927 *ruobjpage = pg;
1928 return 0;
1929 }
1930
1931 /*
1932 * uvm_fault_lower_direct: fault lower center page
1933 *
1934 * 1. adjust flt->enter_prot.
1935 * 2. if page is loaned, resolve.
1936 */
1937
1938 int
1939 uvm_fault_lower_direct(
1940 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1941 struct uvm_object *uobj, struct vm_page *uobjpage)
1942 {
1943 struct vm_page *pg;
1944 UVMHIST_FUNC("uvm_fault_lower_direct"); UVMHIST_CALLED(maphist);
1945
1946 /*
1947 * we are not promoting. if the mapping is COW ensure that we
1948 * don't give more access than we should (e.g. when doing a read
1949 * fault on a COPYONWRITE mapping we want to map the COW page in
1950 * R/O even though the entry protection could be R/W).
1951 *
1952 * set "pg" to the page we want to map in (uobjpage, usually)
1953 */
1954
1955 uvmexp.flt_obj++;
1956 if (UVM_ET_ISCOPYONWRITE(ufi->entry))
1957 flt->enter_prot &= ~VM_PROT_WRITE;
1958 pg = uobjpage; /* map in the actual object */
1959
1960 KASSERT(uobjpage != PGO_DONTCARE);
1961
1962 /*
1963 * we are faulting directly on the page. be careful
1964 * about writing to loaned pages...
1965 */
1966
1967 if (uobjpage->loan_count) {
1968 uvm_fault_lower_direct_loan(ufi, flt, uobj, &pg, &uobjpage);
1969 }
1970 KASSERT(pg == uobjpage);
1971
1972 KASSERT(uobj == NULL || (uobjpage->flags & PG_BUSY) != 0);
1973 return uvm_fault_lower_enter(ufi, flt, uobj, NULL, pg);
1974 }
1975
1976 /*
1977 * uvm_fault_lower_direct_loan: resolve loaned page.
1978 *
1979 * 1. if not cow'ing, adjust flt->enter_prot.
1980 * 2. if cow'ing, break loan.
1981 */
1982
1983 static int
1984 uvm_fault_lower_direct_loan(
1985 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
1986 struct uvm_object *uobj, struct vm_page **rpg,
1987 struct vm_page **ruobjpage)
1988 {
1989 struct vm_amap * const amap = ufi->entry->aref.ar_amap;
1990 struct vm_page *pg;
1991 struct vm_page *uobjpage = *ruobjpage;
1992 UVMHIST_FUNC("uvm_fault_lower_direct_loan"); UVMHIST_CALLED(maphist);
1993
1994 if (!flt->cow_now) {
1995 /* read fault: cap the protection at readonly */
1996 /* cap! */
1997 flt->enter_prot = flt->enter_prot & ~VM_PROT_WRITE;
1998 } else {
1999 /* write fault: must break the loan here */
2000
2001 pg = uvm_loanbreak(uobjpage);
2002 if (pg == NULL) {
2003
2004 /*
2005 * drop ownership of page, it can't be released
2006 */
2007
2008 if (uobjpage->flags & PG_WANTED)
2009 wakeup(uobjpage);
2010 uobjpage->flags &= ~(PG_BUSY|PG_WANTED);
2011 UVM_PAGE_OWN(uobjpage, NULL);
2012
2013 uvmfault_unlockall(ufi, amap, uobj);
2014 UVMHIST_LOG(maphist,
2015 " out of RAM breaking loan, waiting",
2016 0,0,0,0);
2017 uvmexp.fltnoram++;
2018 uvm_wait("flt_noram4");
2019 return ERESTART;
2020 }
2021 *rpg = pg;
2022 *ruobjpage = pg;
2023 }
2024 return 0;
2025 }
2026
2027 /*
2028 * uvm_fault_lower_promote: promote lower page.
2029 *
2030 * 1. call uvmfault_promote.
2031 * 2. fill in data.
2032 * 3. if not ZFOD, dispose old page.
2033 */
2034
2035 int
2036 uvm_fault_lower_promote(
2037 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
2038 struct uvm_object *uobj, struct vm_page *uobjpage)
2039 {
2040 struct vm_amap * const amap = ufi->entry->aref.ar_amap;
2041 struct vm_anon *anon;
2042 struct vm_page *pg;
2043 int error;
2044 UVMHIST_FUNC("uvm_fault_lower_promote"); UVMHIST_CALLED(maphist);
2045
2046 KASSERT(amap != NULL);
2047
2048 /*
2049 * If we are going to promote the data to an anon we
2050 * allocate a blank anon here and plug it into our amap.
2051 */
2052 error = uvmfault_promote(ufi, NULL, uobjpage,
2053 &anon, &flt->anon_spare);
2054 switch (error) {
2055 case 0:
2056 break;
2057 case ERESTART:
2058 return ERESTART;
2059 default:
2060 return error;
2061 }
2062
2063 pg = anon->an_page;
2064
2065 /*
2066 * Fill in the data.
2067 */
2068 KASSERT(uobj == NULL || (uobjpage->flags & PG_BUSY) != 0);
2069
2070 if (uobjpage != PGO_DONTCARE) {
2071 uvmexp.flt_prcopy++;
2072
2073 /*
2074 * promote to shared amap? make sure all sharing
2075 * procs see it
2076 */
2077
2078 if ((amap_flags(amap) & AMAP_SHARED) != 0) {
2079 pmap_page_protect(uobjpage, VM_PROT_NONE);
2080 /*
2081 * XXX: PAGE MIGHT BE WIRED!
2082 */
2083 }
2084
2085 /*
2086 * dispose of uobjpage. it can't be PG_RELEASED
2087 * since we still hold the object lock.
2088 */
2089
2090 if (uobjpage->flags & PG_WANTED) {
2091 /* still have the obj lock */
2092 wakeup(uobjpage);
2093 }
2094 uobjpage->flags &= ~(PG_BUSY|PG_WANTED);
2095 UVM_PAGE_OWN(uobjpage, NULL);
2096
2097 UVMHIST_LOG(maphist,
2098 " promote uobjpage 0x%x to anon/page 0x%x/0x%x",
2099 uobjpage, anon, pg, 0);
2100
2101 } else {
2102 uvmexp.flt_przero++;
2103
2104 /*
2105 * Page is zero'd and marked dirty by
2106 * uvmfault_promote().
2107 */
2108
2109 UVMHIST_LOG(maphist," zero fill anon/page 0x%x/0%x",
2110 anon, pg, 0, 0);
2111 }
2112
2113 return uvm_fault_lower_enter(ufi, flt, uobj, anon, pg);
2114 }
2115
2116 /*
2117 * uvm_fault_lower_enter: enter h/w mapping of lower page or anon page promoted
2118 * from the lower page.
2119 */
2120
2121 int
2122 uvm_fault_lower_enter(
2123 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
2124 struct uvm_object *uobj,
2125 struct vm_anon *anon, struct vm_page *pg)
2126 {
2127 struct vm_amap * const amap = ufi->entry->aref.ar_amap;
2128 const bool readonly = uvm_pagereadonly_p(pg);
2129 int error;
2130 UVMHIST_FUNC("uvm_fault_lower_enter"); UVMHIST_CALLED(maphist);
2131
2132 /*
2133 * Locked:
2134 *
2135 * maps(read), amap(if !null), uobj(if !null),
2136 * anon(if !null), pg(if anon), unlock_uobj(if !null)
2137 *
2138 * Note: pg is either the uobjpage or the new page in the new anon.
2139 */
2140 KASSERT(amap == NULL || mutex_owned(amap->am_lock));
2141 KASSERT(uobj == NULL || mutex_owned(uobj->vmobjlock));
2142 KASSERT(anon == NULL || anon->an_lock == amap->am_lock);
2143 KASSERT((pg->flags & PG_BUSY) != 0);
2144
2145 /*
2146 * all resources are present. we can now map it in and free our
2147 * resources.
2148 */
2149
2150 UVMHIST_LOG(maphist,
2151 " MAPPING: case2: pm=0x%x, va=0x%x, pg=0x%x, promote=%d",
2152 ufi->orig_map->pmap, ufi->orig_rvaddr, pg, flt->promote);
2153 KASSERTMSG((flt->access_type & VM_PROT_WRITE) == 0 || !readonly,
2154 "promote=%u cow_now=%u access_type=%x enter_prot=%x cow=%u "
2155 "entry=%p map=%p orig_rvaddr=%p pg=%p",
2156 flt->promote, flt->cow_now, flt->access_type, flt->enter_prot,
2157 UVM_ET_ISCOPYONWRITE(ufi->entry), ufi->entry, ufi->orig_map,
2158 (void *)ufi->orig_rvaddr, pg);
2159 KASSERT((flt->access_type & VM_PROT_WRITE) == 0 || !readonly);
2160 if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr,
2161 VM_PAGE_TO_PHYS(pg),
2162 readonly ? flt->enter_prot & ~VM_PROT_WRITE : flt->enter_prot,
2163 flt->access_type | PMAP_CANFAIL |
2164 (flt->wire_mapping ? PMAP_WIRED : 0)) != 0) {
2165
2166 /*
2167 * No need to undo what we did; we can simply think of
2168 * this as the pmap throwing away the mapping information.
2169 *
2170 * We do, however, have to go through the ReFault path,
2171 * as the map may change while we're asleep.
2172 */
2173
2174 /*
2175 * ensure that the page is queued in the case that
2176 * we just promoted the page.
2177 */
2178
2179 mutex_enter(&uvm_pageqlock);
2180 uvm_pageenqueue(pg);
2181 mutex_exit(&uvm_pageqlock);
2182
2183 if (pg->flags & PG_WANTED)
2184 wakeup(pg);
2185
2186 /*
2187 * note that pg can't be PG_RELEASED since we did not drop
2188 * the object lock since the last time we checked.
2189 */
2190 KASSERT((pg->flags & PG_RELEASED) == 0);
2191
2192 pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED);
2193 UVM_PAGE_OWN(pg, NULL);
2194
2195 uvmfault_unlockall(ufi, amap, uobj);
2196 if (!uvm_reclaimable()) {
2197 UVMHIST_LOG(maphist,
2198 "<- failed. out of VM",0,0,0,0);
2199 /* XXX instrumentation */
2200 error = ENOMEM;
2201 return error;
2202 }
2203 /* XXX instrumentation */
2204 uvm_wait("flt_pmfail2");
2205 return ERESTART;
2206 }
2207
2208 uvm_fault_lower_done(ufi, flt, uobj, pg);
2209
2210 /*
2211 * note that pg can't be PG_RELEASED since we did not drop the object
2212 * lock since the last time we checked.
2213 */
2214 KASSERT((pg->flags & PG_RELEASED) == 0);
2215 if (pg->flags & PG_WANTED)
2216 wakeup(pg);
2217 pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED);
2218 UVM_PAGE_OWN(pg, NULL);
2219
2220 pmap_update(ufi->orig_map->pmap);
2221 uvmfault_unlockall(ufi, amap, uobj);
2222
2223 UVMHIST_LOG(maphist, "<- done (SUCCESS!)",0,0,0,0);
2224 return 0;
2225 }
2226
2227 /*
2228 * uvm_fault_lower_done: queue lower center page.
2229 */
2230
2231 void
2232 uvm_fault_lower_done(
2233 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
2234 struct uvm_object *uobj, struct vm_page *pg)
2235 {
2236 bool dropswap = false;
2237
2238 UVMHIST_FUNC("uvm_fault_lower_done"); UVMHIST_CALLED(maphist);
2239
2240 mutex_enter(&uvm_pageqlock);
2241 if (flt->wire_paging) {
2242 uvm_pagewire(pg);
2243 if (pg->pqflags & PQ_AOBJ) {
2244
2245 /*
2246 * since the now-wired page cannot be paged out,
2247 * release its swap resources for others to use.
2248 * since an aobj page with no swap cannot be clean,
2249 * mark it dirty now.
2250 */
2251
2252 KASSERT(uobj != NULL);
2253 uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY);
2254 dropswap = true;
2255 }
2256 } else {
2257 uvm_pageactivate(pg);
2258 }
2259 mutex_exit(&uvm_pageqlock);
2260
2261 if (dropswap) {
2262 uao_dropswap(uobj, pg->offset >> PAGE_SHIFT);
2263 }
2264 }
2265
2266
2267 /*
2268 * uvm_fault_wire: wire down a range of virtual addresses in a map.
2269 *
2270 * => map may be read-locked by caller, but MUST NOT be write-locked.
2271 * => if map is read-locked, any operations which may cause map to
2272 * be write-locked in uvm_fault() must be taken care of by
2273 * the caller. See uvm_map_pageable().
2274 */
2275
2276 int
2277 uvm_fault_wire(struct vm_map *map, vaddr_t start, vaddr_t end,
2278 vm_prot_t access_type, int maxprot)
2279 {
2280 vaddr_t va;
2281 int error;
2282
2283 /*
2284 * now fault it in a page at a time. if the fault fails then we have
2285 * to undo what we have done. note that in uvm_fault VM_PROT_NONE
2286 * is replaced with the max protection if fault_type is VM_FAULT_WIRE.
2287 */
2288
2289 /*
2290 * XXX work around overflowing a vaddr_t. this prevents us from
2291 * wiring the last page in the address space, though.
2292 */
2293 if (start > end) {
2294 return EFAULT;
2295 }
2296
2297 for (va = start; va < end; va += PAGE_SIZE) {
2298 error = uvm_fault_internal(map, va, access_type,
2299 (maxprot ? UVM_FAULT_MAXPROT : 0) | UVM_FAULT_WIRE);
2300 if (error) {
2301 if (va != start) {
2302 uvm_fault_unwire(map, start, va);
2303 }
2304 return error;
2305 }
2306 }
2307 return 0;
2308 }
2309
2310 /*
2311 * uvm_fault_unwire(): unwire range of virtual space.
2312 */
2313
2314 void
2315 uvm_fault_unwire(struct vm_map *map, vaddr_t start, vaddr_t end)
2316 {
2317 vm_map_lock_read(map);
2318 uvm_fault_unwire_locked(map, start, end);
2319 vm_map_unlock_read(map);
2320 }
2321
2322 /*
2323 * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire().
2324 *
2325 * => map must be at least read-locked.
2326 */
2327
2328 void
2329 uvm_fault_unwire_locked(struct vm_map *map, vaddr_t start, vaddr_t end)
2330 {
2331 struct vm_map_entry *entry, *oentry;
2332 pmap_t pmap = vm_map_pmap(map);
2333 vaddr_t va;
2334 paddr_t pa;
2335 struct vm_page *pg;
2336
2337 KASSERT((map->flags & VM_MAP_INTRSAFE) == 0);
2338
2339 /*
2340 * we assume that the area we are unwiring has actually been wired
2341 * in the first place. this means that we should be able to extract
2342 * the PAs from the pmap. we also lock out the page daemon so that
2343 * we can call uvm_pageunwire.
2344 */
2345
2346 /*
2347 * find the beginning map entry for the region.
2348 */
2349
2350 KASSERT(start >= vm_map_min(map) && end <= vm_map_max(map));
2351 if (uvm_map_lookup_entry(map, start, &entry) == false)
2352 panic("uvm_fault_unwire_locked: address not in map");
2353
2354 oentry = NULL;
2355 for (va = start; va < end; va += PAGE_SIZE) {
2356 if (pmap_extract(pmap, va, &pa) == false)
2357 continue;
2358
2359 /*
2360 * find the map entry for the current address.
2361 */
2362
2363 KASSERT(va >= entry->start);
2364 while (va >= entry->end) {
2365 KASSERT(entry->next != &map->header &&
2366 entry->next->start <= entry->end);
2367 entry = entry->next;
2368 }
2369
2370 /*
2371 * lock it.
2372 */
2373
2374 if (entry != oentry) {
2375 if (oentry != NULL) {
2376 mutex_exit(&uvm_pageqlock);
2377 uvm_map_unlock_entry(oentry);
2378 }
2379 uvm_map_lock_entry(entry);
2380 mutex_enter(&uvm_pageqlock);
2381 oentry = entry;
2382 }
2383
2384 /*
2385 * if the entry is no longer wired, tell the pmap.
2386 */
2387
2388 if (VM_MAPENT_ISWIRED(entry) == 0)
2389 pmap_unwire(pmap, va);
2390
2391 pg = PHYS_TO_VM_PAGE(pa);
2392 if (pg)
2393 uvm_pageunwire(pg);
2394 }
2395
2396 if (oentry != NULL) {
2397 mutex_exit(&uvm_pageqlock);
2398 uvm_map_unlock_entry(entry);
2399 }
2400 }
2401