uvm_fault.c revision 1.56.2.2 1 /* $NetBSD: uvm_fault.c,v 1.56.2.2 2001/04/09 01:59:13 nathanw Exp $ */
2
3 /*
4 *
5 * Copyright (c) 1997 Charles D. Cranor and Washington University.
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 3. All advertising materials mentioning features or use of this software
17 * must display the following acknowledgement:
18 * This product includes software developed by Charles D. Cranor and
19 * Washington University.
20 * 4. The name of the author may not be used to endorse or promote products
21 * derived from this software without specific prior written permission.
22 *
23 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
24 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
25 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
26 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
27 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
28 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
32 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33 *
34 * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp
35 */
36
37 #include "opt_uvmhist.h"
38
39 /*
40 * uvm_fault.c: fault handler
41 */
42
43 #include <sys/param.h>
44 #include <sys/systm.h>
45 #include <sys/kernel.h>
46 #include <sys/lwp.h>
47 #include <sys/proc.h>
48 #include <sys/malloc.h>
49 #include <sys/mman.h>
50 #include <sys/user.h>
51
52 #include <uvm/uvm.h>
53
54 /*
55 *
56 * a word on page faults:
57 *
58 * types of page faults we handle:
59 *
60 * CASE 1: upper layer faults CASE 2: lower layer faults
61 *
62 * CASE 1A CASE 1B CASE 2A CASE 2B
63 * read/write1 write>1 read/write +-cow_write/zero
64 * | | | |
65 * +--|--+ +--|--+ +-----+ + | + | +-----+
66 * amap | V | | ----------->new| | | | ^ |
67 * +-----+ +-----+ +-----+ + | + | +--|--+
68 * | | |
69 * +-----+ +-----+ +--|--+ | +--|--+
70 * uobj | d/c | | d/c | | V | +----| |
71 * +-----+ +-----+ +-----+ +-----+
72 *
73 * d/c = don't care
74 *
75 * case [0]: layerless fault
76 * no amap or uobj is present. this is an error.
77 *
78 * case [1]: upper layer fault [anon active]
79 * 1A: [read] or [write with anon->an_ref == 1]
80 * I/O takes place in top level anon and uobj is not touched.
81 * 1B: [write with anon->an_ref > 1]
82 * new anon is alloc'd and data is copied off ["COW"]
83 *
84 * case [2]: lower layer fault [uobj]
85 * 2A: [read on non-NULL uobj] or [write to non-copy_on_write area]
86 * I/O takes place directly in object.
87 * 2B: [write to copy_on_write] or [read on NULL uobj]
88 * data is "promoted" from uobj to a new anon.
89 * if uobj is null, then we zero fill.
90 *
91 * we follow the standard UVM locking protocol ordering:
92 *
93 * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ)
94 * we hold a PG_BUSY page if we unlock for I/O
95 *
96 *
97 * the code is structured as follows:
98 *
99 * - init the "IN" params in the ufi structure
100 * ReFault:
101 * - do lookups [locks maps], check protection, handle needs_copy
102 * - check for case 0 fault (error)
103 * - establish "range" of fault
104 * - if we have an amap lock it and extract the anons
105 * - if sequential advice deactivate pages behind us
106 * - at the same time check pmap for unmapped areas and anon for pages
107 * that we could map in (and do map it if found)
108 * - check object for resident pages that we could map in
109 * - if (case 2) goto Case2
110 * - >>> handle case 1
111 * - ensure source anon is resident in RAM
112 * - if case 1B alloc new anon and copy from source
113 * - map the correct page in
114 * Case2:
115 * - >>> handle case 2
116 * - ensure source page is resident (if uobj)
117 * - if case 2B alloc new anon and copy from source (could be zero
118 * fill if uobj == NULL)
119 * - map the correct page in
120 * - done!
121 *
122 * note on paging:
123 * if we have to do I/O we place a PG_BUSY page in the correct object,
124 * unlock everything, and do the I/O. when I/O is done we must reverify
125 * the state of the world before assuming that our data structures are
126 * valid. [because mappings could change while the map is unlocked]
127 *
128 * alternative 1: unbusy the page in question and restart the page fault
129 * from the top (ReFault). this is easy but does not take advantage
130 * of the information that we already have from our previous lookup,
131 * although it is possible that the "hints" in the vm_map will help here.
132 *
133 * alternative 2: the system already keeps track of a "version" number of
134 * a map. [i.e. every time you write-lock a map (e.g. to change a
135 * mapping) you bump the version number up by one...] so, we can save
136 * the version number of the map before we release the lock and start I/O.
137 * then when I/O is done we can relock and check the version numbers
138 * to see if anything changed. this might save us some over 1 because
139 * we don't have to unbusy the page and may be less compares(?).
140 *
141 * alternative 3: put in backpointers or a way to "hold" part of a map
142 * in place while I/O is in progress. this could be complex to
143 * implement (especially with structures like amap that can be referenced
144 * by multiple map entries, and figuring out what should wait could be
145 * complex as well...).
146 *
147 * given that we are not currently multiprocessor or multithreaded we might
148 * as well choose alternative 2 now. maybe alternative 3 would be useful
149 * in the future. XXX keep in mind for future consideration//rechecking.
150 */
151
152 /*
153 * local data structures
154 */
155
156 struct uvm_advice {
157 int advice;
158 int nback;
159 int nforw;
160 };
161
162 /*
163 * page range array:
164 * note: index in array must match "advice" value
165 * XXX: borrowed numbers from freebsd. do they work well for us?
166 */
167
168 static struct uvm_advice uvmadvice[] = {
169 { MADV_NORMAL, 3, 4 },
170 { MADV_RANDOM, 0, 0 },
171 { MADV_SEQUENTIAL, 8, 7},
172 };
173
174 #define UVM_MAXRANGE 16 /* must be max() of nback+nforw+1 */
175
176 /*
177 * private prototypes
178 */
179
180 static void uvmfault_amapcopy __P((struct uvm_faultinfo *));
181 static __inline void uvmfault_anonflush __P((struct vm_anon **, int));
182
183 /*
184 * inline functions
185 */
186
187 /*
188 * uvmfault_anonflush: try and deactivate pages in specified anons
189 *
190 * => does not have to deactivate page if it is busy
191 */
192
193 static __inline void
194 uvmfault_anonflush(anons, n)
195 struct vm_anon **anons;
196 int n;
197 {
198 int lcv;
199 struct vm_page *pg;
200
201 for (lcv = 0 ; lcv < n ; lcv++) {
202 if (anons[lcv] == NULL)
203 continue;
204 simple_lock(&anons[lcv]->an_lock);
205 pg = anons[lcv]->u.an_page;
206 if (pg && (pg->flags & PG_BUSY) == 0 && pg->loan_count == 0) {
207 uvm_lock_pageq();
208 if (pg->wire_count == 0) {
209 pmap_clear_reference(pg);
210 uvm_pagedeactivate(pg);
211 }
212 uvm_unlock_pageq();
213 }
214 simple_unlock(&anons[lcv]->an_lock);
215 }
216 }
217
218 /*
219 * normal functions
220 */
221
222 /*
223 * uvmfault_amapcopy: clear "needs_copy" in a map.
224 *
225 * => called with VM data structures unlocked (usually, see below)
226 * => we get a write lock on the maps and clear needs_copy for a VA
227 * => if we are out of RAM we sleep (waiting for more)
228 */
229
230 static void
231 uvmfault_amapcopy(ufi)
232 struct uvm_faultinfo *ufi;
233 {
234
235 /*
236 * while we haven't done the job
237 */
238
239 while (1) {
240
241 /*
242 * no mapping? give up.
243 */
244
245 if (uvmfault_lookup(ufi, TRUE) == FALSE)
246 return;
247
248 /*
249 * copy if needed.
250 */
251
252 if (UVM_ET_ISNEEDSCOPY(ufi->entry))
253 amap_copy(ufi->map, ufi->entry, M_NOWAIT, TRUE,
254 ufi->orig_rvaddr, ufi->orig_rvaddr + 1);
255
256 /*
257 * didn't work? must be out of RAM. unlock and sleep.
258 */
259
260 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) {
261 uvmfault_unlockmaps(ufi, TRUE);
262 uvm_wait("fltamapcopy");
263 continue;
264 }
265
266 /*
267 * got it! unlock and return.
268 */
269
270 uvmfault_unlockmaps(ufi, TRUE);
271 return;
272 }
273 /*NOTREACHED*/
274 }
275
276 /*
277 * uvmfault_anonget: get data in an anon into a non-busy, non-released
278 * page in that anon.
279 *
280 * => maps, amap, and anon locked by caller.
281 * => if we fail (result != 0) we unlock everything.
282 * => if we are successful, we return with everything still locked.
283 * => we don't move the page on the queues [gets moved later]
284 * => if we allocate a new page [we_own], it gets put on the queues.
285 * either way, the result is that the page is on the queues at return time
286 * => for pages which are on loan from a uvm_object (and thus are not
287 * owned by the anon): if successful, we return with the owning object
288 * locked. the caller must unlock this object when it unlocks everything
289 * else.
290 */
291
292 int
293 uvmfault_anonget(ufi, amap, anon)
294 struct uvm_faultinfo *ufi;
295 struct vm_amap *amap;
296 struct vm_anon *anon;
297 {
298 boolean_t we_own; /* we own anon's page? */
299 boolean_t locked; /* did we relock? */
300 struct vm_page *pg;
301 int error;
302 UVMHIST_FUNC("uvmfault_anonget"); UVMHIST_CALLED(maphist);
303
304 LOCK_ASSERT(simple_lock_held(&anon->an_lock));
305
306 error = 0;
307 uvmexp.fltanget++;
308 /* bump rusage counters */
309 if (anon->u.an_page)
310 curproc->l_proc->p_stats->p_ru.ru_minflt++;
311 else
312 curproc->l_proc->p_stats->p_ru.ru_majflt++;
313
314 /*
315 * loop until we get it, or fail.
316 */
317
318 while (1) {
319
320 we_own = FALSE; /* TRUE if we set PG_BUSY on a page */
321 pg = anon->u.an_page;
322
323 /*
324 * if there is a resident page and it is loaned, then anon
325 * may not own it. call out to uvm_anon_lockpage() to ensure
326 * the real owner of the page has been identified and locked.
327 */
328
329 if (pg && pg->loan_count)
330 pg = uvm_anon_lockloanpg(anon);
331
332 /*
333 * page there? make sure it is not busy/released.
334 */
335
336 if (pg) {
337
338 /*
339 * at this point, if the page has a uobject [meaning
340 * we have it on loan], then that uobject is locked
341 * by us! if the page is busy, we drop all the
342 * locks (including uobject) and try again.
343 */
344
345 if ((pg->flags & (PG_BUSY|PG_RELEASED)) == 0) {
346 UVMHIST_LOG(maphist, "<- OK",0,0,0,0);
347 return (0);
348 }
349 pg->flags |= PG_WANTED;
350 uvmexp.fltpgwait++;
351
352 /*
353 * the last unlock must be an atomic unlock+wait on
354 * the owner of page
355 */
356 if (pg->uobject) { /* owner is uobject ? */
357 uvmfault_unlockall(ufi, amap, NULL, anon);
358 UVMHIST_LOG(maphist, " unlock+wait on uobj",0,
359 0,0,0);
360 UVM_UNLOCK_AND_WAIT(pg,
361 &pg->uobject->vmobjlock,
362 FALSE, "anonget1",0);
363 } else {
364 /* anon owns page */
365 uvmfault_unlockall(ufi, amap, NULL, NULL);
366 UVMHIST_LOG(maphist, " unlock+wait on anon",0,
367 0,0,0);
368 UVM_UNLOCK_AND_WAIT(pg,&anon->an_lock,0,
369 "anonget2",0);
370 }
371 /* ready to relock and try again */
372
373 } else {
374
375 /*
376 * no page, we must try and bring it in.
377 */
378 pg = uvm_pagealloc(NULL, 0, anon, 0);
379
380 if (pg == NULL) { /* out of RAM. */
381
382 uvmfault_unlockall(ufi, amap, NULL, anon);
383 uvmexp.fltnoram++;
384 UVMHIST_LOG(maphist, " noram -- UVM_WAIT",0,
385 0,0,0);
386 uvm_wait("flt_noram1");
387 /* ready to relock and try again */
388
389 } else {
390
391 /* we set the PG_BUSY bit */
392 we_own = TRUE;
393 uvmfault_unlockall(ufi, amap, NULL, anon);
394
395 /*
396 * we are passing a PG_BUSY+PG_FAKE+PG_CLEAN
397 * page into the uvm_swap_get function with
398 * all data structures unlocked. note that
399 * it is ok to read an_swslot here because
400 * we hold PG_BUSY on the page.
401 */
402 uvmexp.pageins++;
403 error = uvm_swap_get(pg, anon->an_swslot,
404 PGO_SYNCIO);
405
406 /*
407 * we clean up after the i/o below in the
408 * "we_own" case
409 */
410 /* ready to relock and try again */
411 }
412 }
413
414 /*
415 * now relock and try again
416 */
417
418 locked = uvmfault_relock(ufi);
419 if (locked && amap != NULL) {
420 amap_lock(amap);
421 }
422 if (locked || we_own)
423 simple_lock(&anon->an_lock);
424
425 /*
426 * if we own the page (i.e. we set PG_BUSY), then we need
427 * to clean up after the I/O. there are three cases to
428 * consider:
429 * [1] page released during I/O: free anon and ReFault.
430 * [2] I/O not OK. free the page and cause the fault
431 * to fail.
432 * [3] I/O OK! activate the page and sync with the
433 * non-we_own case (i.e. drop anon lock if not locked).
434 */
435
436 if (we_own) {
437
438 if (pg->flags & PG_WANTED) {
439 /* still holding object lock */
440 wakeup(pg);
441 }
442 /* un-busy! */
443 pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE);
444 UVM_PAGE_OWN(pg, NULL);
445
446 /*
447 * if we were RELEASED during I/O, then our anon is
448 * no longer part of an amap. we need to free the
449 * anon and try again.
450 */
451 if (pg->flags & PG_RELEASED) {
452 pmap_page_protect(pg, VM_PROT_NONE);
453 simple_unlock(&anon->an_lock);
454 uvm_anfree(anon); /* frees page for us */
455 if (locked)
456 uvmfault_unlockall(ufi, amap, NULL,
457 NULL);
458 uvmexp.fltpgrele++;
459 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0);
460 return (ERESTART); /* refault! */
461 }
462
463 if (error) {
464 /* remove page from anon */
465 anon->u.an_page = NULL;
466
467 /*
468 * remove the swap slot from the anon
469 * and mark the anon as having no real slot.
470 * don't free the swap slot, thus preventing
471 * it from being used again.
472 */
473 uvm_swap_markbad(anon->an_swslot, 1);
474 anon->an_swslot = SWSLOT_BAD;
475
476 /*
477 * note: page was never !PG_BUSY, so it
478 * can't be mapped and thus no need to
479 * pmap_page_protect it...
480 */
481 uvm_lock_pageq();
482 uvm_pagefree(pg);
483 uvm_unlock_pageq();
484
485 if (locked)
486 uvmfault_unlockall(ufi, amap, NULL,
487 anon);
488 else
489 simple_unlock(&anon->an_lock);
490 UVMHIST_LOG(maphist, "<- ERROR", 0,0,0,0);
491 return error;
492 }
493
494 /*
495 * must be OK, clear modify (already PG_CLEAN)
496 * and activate
497 */
498 pmap_clear_modify(pg);
499 uvm_lock_pageq();
500 uvm_pageactivate(pg);
501 uvm_unlock_pageq();
502 if (!locked)
503 simple_unlock(&anon->an_lock);
504 }
505
506 /*
507 * we were not able to relock. restart fault.
508 */
509
510 if (!locked) {
511 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0);
512 return (ERESTART);
513 }
514
515 /*
516 * verify no one has touched the amap and moved the anon on us.
517 */
518
519 if (ufi != NULL &&
520 amap_lookup(&ufi->entry->aref,
521 ufi->orig_rvaddr - ufi->entry->start) != anon) {
522
523 uvmfault_unlockall(ufi, amap, NULL, anon);
524 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0);
525 return (ERESTART);
526 }
527
528 /*
529 * try it again!
530 */
531
532 uvmexp.fltanretry++;
533 continue;
534
535 } /* while (1) */
536
537 /*NOTREACHED*/
538 }
539
540 /*
541 * F A U L T - m a i n e n t r y p o i n t
542 */
543
544 /*
545 * uvm_fault: page fault handler
546 *
547 * => called from MD code to resolve a page fault
548 * => VM data structures usually should be unlocked. however, it is
549 * possible to call here with the main map locked if the caller
550 * gets a write lock, sets it recusive, and then calls us (c.f.
551 * uvm_map_pageable). this should be avoided because it keeps
552 * the map locked off during I/O.
553 */
554
555 #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \
556 ~VM_PROT_WRITE : VM_PROT_ALL)
557
558 int
559 uvm_fault(orig_map, vaddr, fault_type, access_type)
560 vm_map_t orig_map;
561 vaddr_t vaddr;
562 vm_fault_t fault_type;
563 vm_prot_t access_type;
564 {
565 struct uvm_faultinfo ufi;
566 vm_prot_t enter_prot;
567 boolean_t wired, narrow, promote, locked, shadowed;
568 int npages, nback, nforw, centeridx, error, lcv, gotpages;
569 vaddr_t startva, objaddr, currva, offset, uoff;
570 paddr_t pa;
571 struct vm_amap *amap;
572 struct uvm_object *uobj;
573 struct vm_anon *anons_store[UVM_MAXRANGE], **anons, *anon, *oanon;
574 struct vm_page *pages[UVM_MAXRANGE], *pg, *uobjpage;
575 UVMHIST_FUNC("uvm_fault"); UVMHIST_CALLED(maphist);
576
577 UVMHIST_LOG(maphist, "(map=0x%x, vaddr=0x%x, ft=%d, at=%d)",
578 orig_map, vaddr, fault_type, access_type);
579
580 anon = NULL;
581 pg = NULL;
582
583 uvmexp.faults++; /* XXX: locking? */
584
585 /*
586 * init the IN parameters in the ufi
587 */
588
589 ufi.orig_map = orig_map;
590 ufi.orig_rvaddr = trunc_page(vaddr);
591 ufi.orig_size = PAGE_SIZE; /* can't get any smaller than this */
592 if (fault_type == VM_FAULT_WIRE)
593 narrow = TRUE; /* don't look for neighborhood
594 * pages on wire */
595 else
596 narrow = FALSE; /* normal fault */
597
598 /*
599 * before we do anything else, if this is a fault on a kernel
600 * address, check to see if the address is managed by an
601 * interrupt-safe map. If it is, we fail immediately. Intrsafe
602 * maps are never pageable, and this approach avoids an evil
603 * locking mess.
604 */
605
606 if (orig_map == kernel_map && uvmfault_check_intrsafe(&ufi)) {
607 UVMHIST_LOG(maphist, "<- VA 0x%lx in intrsafe map %p",
608 ufi.orig_rvaddr, ufi.map, 0, 0);
609 return EFAULT;
610 }
611
612 /*
613 * "goto ReFault" means restart the page fault from ground zero.
614 */
615 ReFault:
616
617 /*
618 * lookup and lock the maps
619 */
620
621 if (uvmfault_lookup(&ufi, FALSE) == FALSE) {
622 UVMHIST_LOG(maphist, "<- no mapping @ 0x%x", vaddr, 0,0,0);
623 return (EFAULT);
624 }
625 /* locked: maps(read) */
626
627 KASSERT(ufi.map->flags & VM_MAP_PAGEABLE);
628
629 /*
630 * check protection
631 */
632
633 if ((ufi.entry->protection & access_type) != access_type) {
634 UVMHIST_LOG(maphist,
635 "<- protection failure (prot=0x%x, access=0x%x)",
636 ufi.entry->protection, access_type, 0, 0);
637 uvmfault_unlockmaps(&ufi, FALSE);
638 return EACCES;
639 }
640
641 /*
642 * "enter_prot" is the protection we want to enter the page in at.
643 * for certain pages (e.g. copy-on-write pages) this protection can
644 * be more strict than ufi.entry->protection. "wired" means either
645 * the entry is wired or we are fault-wiring the pg.
646 */
647
648 enter_prot = ufi.entry->protection;
649 wired = VM_MAPENT_ISWIRED(ufi.entry) || (fault_type == VM_FAULT_WIRE);
650 if (wired)
651 access_type = enter_prot; /* full access for wired */
652
653 /*
654 * handle "needs_copy" case. if we need to copy the amap we will
655 * have to drop our readlock and relock it with a write lock. (we
656 * need a write lock to change anything in a map entry [e.g.
657 * needs_copy]).
658 */
659
660 if (UVM_ET_ISNEEDSCOPY(ufi.entry)) {
661 if ((access_type & VM_PROT_WRITE) ||
662 (ufi.entry->object.uvm_obj == NULL)) {
663 /* need to clear */
664 UVMHIST_LOG(maphist,
665 " need to clear needs_copy and refault",0,0,0,0);
666 uvmfault_unlockmaps(&ufi, FALSE);
667 uvmfault_amapcopy(&ufi);
668 uvmexp.fltamcopy++;
669 goto ReFault;
670
671 } else {
672
673 /*
674 * ensure that we pmap_enter page R/O since
675 * needs_copy is still true
676 */
677 enter_prot &= ~VM_PROT_WRITE;
678
679 }
680 }
681
682 /*
683 * identify the players
684 */
685
686 amap = ufi.entry->aref.ar_amap; /* top layer */
687 uobj = ufi.entry->object.uvm_obj; /* bottom layer */
688
689 /*
690 * check for a case 0 fault. if nothing backing the entry then
691 * error now.
692 */
693
694 if (amap == NULL && uobj == NULL) {
695 uvmfault_unlockmaps(&ufi, FALSE);
696 UVMHIST_LOG(maphist,"<- no backing store, no overlay",0,0,0,0);
697 return (EFAULT);
698 }
699
700 /*
701 * establish range of interest based on advice from mapper
702 * and then clip to fit map entry. note that we only want
703 * to do this the first time through the fault. if we
704 * ReFault we will disable this by setting "narrow" to true.
705 */
706
707 if (narrow == FALSE) {
708
709 /* wide fault (!narrow) */
710 KASSERT(uvmadvice[ufi.entry->advice].advice ==
711 ufi.entry->advice);
712 nback = min(uvmadvice[ufi.entry->advice].nback,
713 (ufi.orig_rvaddr - ufi.entry->start) >> PAGE_SHIFT);
714 startva = ufi.orig_rvaddr - (nback << PAGE_SHIFT);
715 nforw = min(uvmadvice[ufi.entry->advice].nforw,
716 ((ufi.entry->end - ufi.orig_rvaddr) >>
717 PAGE_SHIFT) - 1);
718 /*
719 * note: "-1" because we don't want to count the
720 * faulting page as forw
721 */
722 npages = nback + nforw + 1;
723 centeridx = nback;
724
725 narrow = TRUE; /* ensure only once per-fault */
726
727 } else {
728
729 /* narrow fault! */
730 nback = nforw = 0;
731 startva = ufi.orig_rvaddr;
732 npages = 1;
733 centeridx = 0;
734
735 }
736
737 /* locked: maps(read) */
738 UVMHIST_LOG(maphist, " narrow=%d, back=%d, forw=%d, startva=0x%x",
739 narrow, nback, nforw, startva);
740 UVMHIST_LOG(maphist, " entry=0x%x, amap=0x%x, obj=0x%x", ufi.entry,
741 amap, uobj, 0);
742
743 /*
744 * if we've got an amap, lock it and extract current anons.
745 */
746
747 if (amap) {
748 amap_lock(amap);
749 anons = anons_store;
750 amap_lookups(&ufi.entry->aref, startva - ufi.entry->start,
751 anons, npages);
752 } else {
753 anons = NULL; /* to be safe */
754 }
755
756 /* locked: maps(read), amap(if there) */
757
758 /*
759 * for MADV_SEQUENTIAL mappings we want to deactivate the back pages
760 * now and then forget about them (for the rest of the fault).
761 */
762
763 if (ufi.entry->advice == MADV_SEQUENTIAL) {
764
765 UVMHIST_LOG(maphist, " MADV_SEQUENTIAL: flushing backpages",
766 0,0,0,0);
767 /* flush back-page anons? */
768 if (amap)
769 uvmfault_anonflush(anons, nback);
770
771 /* flush object? */
772 if (uobj) {
773 objaddr =
774 (startva - ufi.entry->start) + ufi.entry->offset;
775 simple_lock(&uobj->vmobjlock);
776 (void) uobj->pgops->pgo_flush(uobj, objaddr, objaddr +
777 (nback << PAGE_SHIFT), PGO_DEACTIVATE);
778 simple_unlock(&uobj->vmobjlock);
779 }
780
781 /* now forget about the backpages */
782 if (amap)
783 anons += nback;
784 startva += (nback << PAGE_SHIFT);
785 npages -= nback;
786 nback = centeridx = 0;
787 }
788
789 /* locked: maps(read), amap(if there) */
790
791 /*
792 * map in the backpages and frontpages we found in the amap in hopes
793 * of preventing future faults. we also init the pages[] array as
794 * we go.
795 */
796
797 currva = startva;
798 shadowed = FALSE;
799 for (lcv = 0 ; lcv < npages ; lcv++, currva += PAGE_SIZE) {
800
801 /*
802 * dont play with VAs that are already mapped
803 * except for center)
804 */
805 if (lcv != centeridx &&
806 pmap_extract(ufi.orig_map->pmap, currva, &pa)) {
807 pages[lcv] = PGO_DONTCARE;
808 continue;
809 }
810
811 /*
812 * unmapped or center page. check if any anon at this level.
813 */
814 if (amap == NULL || anons[lcv] == NULL) {
815 pages[lcv] = NULL;
816 continue;
817 }
818
819 /*
820 * check for present page and map if possible. re-activate it.
821 */
822
823 pages[lcv] = PGO_DONTCARE;
824 if (lcv == centeridx) { /* save center for later! */
825 shadowed = TRUE;
826 continue;
827 }
828 anon = anons[lcv];
829 simple_lock(&anon->an_lock);
830 /* ignore loaned pages */
831 if (anon->u.an_page && anon->u.an_page->loan_count == 0 &&
832 (anon->u.an_page->flags & (PG_RELEASED|PG_BUSY)) == 0) {
833 uvm_lock_pageq();
834 uvm_pageactivate(anon->u.an_page); /* reactivate */
835 uvm_unlock_pageq();
836 UVMHIST_LOG(maphist,
837 " MAPPING: n anon: pm=0x%x, va=0x%x, pg=0x%x",
838 ufi.orig_map->pmap, currva, anon->u.an_page, 0);
839 uvmexp.fltnamap++;
840
841 /*
842 * Since this isn't the page that's actually faulting,
843 * ignore pmap_enter() failures; it's not critical
844 * that we enter these right now.
845 */
846
847 (void) pmap_enter(ufi.orig_map->pmap, currva,
848 VM_PAGE_TO_PHYS(anon->u.an_page),
849 (anon->an_ref > 1) ? (enter_prot & ~VM_PROT_WRITE) :
850 enter_prot,
851 PMAP_CANFAIL |
852 (VM_MAPENT_ISWIRED(ufi.entry) ? PMAP_WIRED : 0));
853 }
854 simple_unlock(&anon->an_lock);
855 }
856
857 /* locked: maps(read), amap(if there) */
858 /* (shadowed == TRUE) if there is an anon at the faulting address */
859 UVMHIST_LOG(maphist, " shadowed=%d, will_get=%d", shadowed,
860 (uobj && shadowed == FALSE),0,0);
861
862 /*
863 * note that if we are really short of RAM we could sleep in the above
864 * call to pmap_enter with everything locked. bad?
865 *
866 * XXX Actually, that is bad; pmap_enter() should just fail in that
867 * XXX case. --thorpej
868 */
869
870 /*
871 * if the desired page is not shadowed by the amap and we have a
872 * backing object, then we check to see if the backing object would
873 * prefer to handle the fault itself (rather than letting us do it
874 * with the usual pgo_get hook). the backing object signals this by
875 * providing a pgo_fault routine.
876 */
877
878 if (uobj && shadowed == FALSE && uobj->pgops->pgo_fault != NULL) {
879 simple_lock(&uobj->vmobjlock);
880
881 /* locked: maps(read), amap (if there), uobj */
882 error = uobj->pgops->pgo_fault(&ufi, startva, pages, npages,
883 centeridx, fault_type, access_type, PGO_LOCKED|PGO_SYNCIO);
884
885 /* locked: nothing, pgo_fault has unlocked everything */
886
887 if (error == ERESTART)
888 goto ReFault; /* try again! */
889 return error;
890 }
891
892 /*
893 * now, if the desired page is not shadowed by the amap and we have
894 * a backing object that does not have a special fault routine, then
895 * we ask (with pgo_get) the object for resident pages that we care
896 * about and attempt to map them in. we do not let pgo_get block
897 * (PGO_LOCKED).
898 *
899 * ("get" has the option of doing a pmap_enter for us)
900 */
901
902 if (uobj && shadowed == FALSE) {
903 simple_lock(&uobj->vmobjlock);
904
905 /* locked (!shadowed): maps(read), amap (if there), uobj */
906 /*
907 * the following call to pgo_get does _not_ change locking state
908 */
909
910 uvmexp.fltlget++;
911 gotpages = npages;
912 (void) uobj->pgops->pgo_get(uobj, ufi.entry->offset +
913 (startva - ufi.entry->start),
914 pages, &gotpages, centeridx,
915 access_type & MASK(ufi.entry),
916 ufi.entry->advice, PGO_LOCKED);
917
918 /*
919 * check for pages to map, if we got any
920 */
921
922 uobjpage = NULL;
923
924 if (gotpages) {
925 currva = startva;
926 for (lcv = 0 ; lcv < npages ;
927 lcv++, currva += PAGE_SIZE) {
928
929 if (pages[lcv] == NULL ||
930 pages[lcv] == PGO_DONTCARE)
931 continue;
932
933 KASSERT((pages[lcv]->flags & PG_RELEASED) == 0);
934
935 /*
936 * if center page is resident and not
937 * PG_BUSY|PG_RELEASED then pgo_get
938 * made it PG_BUSY for us and gave
939 * us a handle to it. remember this
940 * page as "uobjpage." (for later use).
941 */
942
943 if (lcv == centeridx) {
944 uobjpage = pages[lcv];
945 UVMHIST_LOG(maphist, " got uobjpage "
946 "(0x%x) with locked get",
947 uobjpage, 0,0,0);
948 continue;
949 }
950
951 /*
952 * note: calling pgo_get with locked data
953 * structures returns us pages which are
954 * neither busy nor released, so we don't
955 * need to check for this. we can just
956 * directly enter the page (after moving it
957 * to the head of the active queue [useful?]).
958 */
959
960 uvm_lock_pageq();
961 uvm_pageactivate(pages[lcv]); /* reactivate */
962 uvm_unlock_pageq();
963 UVMHIST_LOG(maphist,
964 " MAPPING: n obj: pm=0x%x, va=0x%x, pg=0x%x",
965 ufi.orig_map->pmap, currva, pages[lcv], 0);
966 uvmexp.fltnomap++;
967
968 /*
969 * Since this page isn't the page that's
970 * actually fauling, ignore pmap_enter()
971 * failures; it's not critical that we
972 * enter these right now.
973 */
974
975 (void) pmap_enter(ufi.orig_map->pmap, currva,
976 VM_PAGE_TO_PHYS(pages[lcv]),
977 pages[lcv]->flags & PG_RDONLY ?
978 VM_PROT_READ : enter_prot & MASK(ufi.entry),
979 PMAP_CANFAIL |
980 (wired ? PMAP_WIRED : 0));
981
982 /*
983 * NOTE: page can't be PG_WANTED or PG_RELEASED
984 * because we've held the lock the whole time
985 * we've had the handle.
986 */
987
988 pages[lcv]->flags &= ~(PG_BUSY); /* un-busy! */
989 UVM_PAGE_OWN(pages[lcv], NULL);
990 } /* for "lcv" loop */
991 } /* "gotpages" != 0 */
992 /* note: object still _locked_ */
993 } else {
994 uobjpage = NULL;
995 }
996
997 /* locked (shadowed): maps(read), amap */
998 /* locked (!shadowed): maps(read), amap(if there),
999 uobj(if !null), uobjpage(if !null) */
1000
1001 /*
1002 * note that at this point we are done with any front or back pages.
1003 * we are now going to focus on the center page (i.e. the one we've
1004 * faulted on). if we have faulted on the top (anon) layer
1005 * [i.e. case 1], then the anon we want is anons[centeridx] (we have
1006 * not touched it yet). if we have faulted on the bottom (uobj)
1007 * layer [i.e. case 2] and the page was both present and available,
1008 * then we've got a pointer to it as "uobjpage" and we've already
1009 * made it BUSY.
1010 */
1011
1012 /*
1013 * there are four possible cases we must address: 1A, 1B, 2A, and 2B
1014 */
1015
1016 /*
1017 * redirect case 2: if we are not shadowed, go to case 2.
1018 */
1019
1020 if (shadowed == FALSE)
1021 goto Case2;
1022
1023 /* locked: maps(read), amap */
1024
1025 /*
1026 * handle case 1: fault on an anon in our amap
1027 */
1028
1029 anon = anons[centeridx];
1030 UVMHIST_LOG(maphist, " case 1 fault: anon=0x%x", anon, 0,0,0);
1031 simple_lock(&anon->an_lock);
1032
1033 /* locked: maps(read), amap, anon */
1034
1035 /*
1036 * no matter if we have case 1A or case 1B we are going to need to
1037 * have the anon's memory resident. ensure that now.
1038 */
1039
1040 /*
1041 * let uvmfault_anonget do the dirty work.
1042 * if it fails (!OK) it will unlock everything for us.
1043 * if it succeeds, locks are still valid and locked.
1044 * also, if it is OK, then the anon's page is on the queues.
1045 * if the page is on loan from a uvm_object, then anonget will
1046 * lock that object for us if it does not fail.
1047 */
1048
1049 error = uvmfault_anonget(&ufi, amap, anon);
1050 switch (error) {
1051 case 0:
1052 break;
1053
1054 case ERESTART:
1055 goto ReFault;
1056
1057 case EAGAIN:
1058 tsleep(&lbolt, PVM, "fltagain1", 0);
1059 goto ReFault;
1060
1061 default:
1062 return error;
1063 }
1064
1065 /*
1066 * uobj is non null if the page is on loan from an object (i.e. uobj)
1067 */
1068
1069 uobj = anon->u.an_page->uobject; /* locked by anonget if !NULL */
1070
1071 /* locked: maps(read), amap, anon, uobj(if one) */
1072
1073 /*
1074 * special handling for loaned pages
1075 */
1076
1077 if (anon->u.an_page->loan_count) {
1078
1079 if ((access_type & VM_PROT_WRITE) == 0) {
1080
1081 /*
1082 * for read faults on loaned pages we just cap the
1083 * protection at read-only.
1084 */
1085
1086 enter_prot = enter_prot & ~VM_PROT_WRITE;
1087
1088 } else {
1089 /*
1090 * note that we can't allow writes into a loaned page!
1091 *
1092 * if we have a write fault on a loaned page in an
1093 * anon then we need to look at the anon's ref count.
1094 * if it is greater than one then we are going to do
1095 * a normal copy-on-write fault into a new anon (this
1096 * is not a problem). however, if the reference count
1097 * is one (a case where we would normally allow a
1098 * write directly to the page) then we need to kill
1099 * the loan before we continue.
1100 */
1101
1102 /* >1 case is already ok */
1103 if (anon->an_ref == 1) {
1104
1105 /* get new un-owned replacement page */
1106 pg = uvm_pagealloc(NULL, 0, NULL, 0);
1107 if (pg == NULL) {
1108 uvmfault_unlockall(&ufi, amap, uobj,
1109 anon);
1110 uvm_wait("flt_noram2");
1111 goto ReFault;
1112 }
1113
1114 /*
1115 * copy data, kill loan, and drop uobj lock
1116 * (if any)
1117 */
1118 /* copy old -> new */
1119 uvm_pagecopy(anon->u.an_page, pg);
1120
1121 /* force reload */
1122 pmap_page_protect(anon->u.an_page,
1123 VM_PROT_NONE);
1124 uvm_lock_pageq(); /* KILL loan */
1125 if (uobj)
1126 /* if we were loaning */
1127 anon->u.an_page->loan_count--;
1128 anon->u.an_page->uanon = NULL;
1129 /* in case we owned */
1130 anon->u.an_page->pqflags &= ~PQ_ANON;
1131 uvm_unlock_pageq();
1132 if (uobj) {
1133 simple_unlock(&uobj->vmobjlock);
1134 uobj = NULL;
1135 }
1136
1137 /* install new page in anon */
1138 anon->u.an_page = pg;
1139 pg->uanon = anon;
1140 pg->pqflags |= PQ_ANON;
1141 pg->flags &= ~(PG_BUSY|PG_FAKE);
1142 UVM_PAGE_OWN(pg, NULL);
1143
1144 /* done! */
1145 } /* ref == 1 */
1146 } /* write fault */
1147 } /* loan count */
1148
1149 /*
1150 * if we are case 1B then we will need to allocate a new blank
1151 * anon to transfer the data into. note that we have a lock
1152 * on anon, so no one can busy or release the page until we are done.
1153 * also note that the ref count can't drop to zero here because
1154 * it is > 1 and we are only dropping one ref.
1155 *
1156 * in the (hopefully very rare) case that we are out of RAM we
1157 * will unlock, wait for more RAM, and refault.
1158 *
1159 * if we are out of anon VM we kill the process (XXX: could wait?).
1160 */
1161
1162 if ((access_type & VM_PROT_WRITE) != 0 && anon->an_ref > 1) {
1163
1164 UVMHIST_LOG(maphist, " case 1B: COW fault",0,0,0,0);
1165 uvmexp.flt_acow++;
1166 oanon = anon; /* oanon = old, locked anon */
1167 anon = uvm_analloc();
1168 if (anon) {
1169 /* new anon is locked! */
1170 pg = uvm_pagealloc(NULL, 0, anon, 0);
1171 }
1172
1173 /* check for out of RAM */
1174 if (anon == NULL || pg == NULL) {
1175 if (anon) {
1176 anon->an_ref--;
1177 simple_unlock(&anon->an_lock);
1178 uvm_anfree(anon);
1179 }
1180 uvmfault_unlockall(&ufi, amap, uobj, oanon);
1181 KASSERT(uvmexp.swpgonly <= uvmexp.swpages);
1182 if (anon == NULL || uvmexp.swpgonly == uvmexp.swpages) {
1183 UVMHIST_LOG(maphist,
1184 "<- failed. out of VM",0,0,0,0);
1185 uvmexp.fltnoanon++;
1186 return ENOMEM;
1187 }
1188
1189 uvmexp.fltnoram++;
1190 uvm_wait("flt_noram3"); /* out of RAM, wait for more */
1191 goto ReFault;
1192 }
1193
1194 /* got all resources, replace anon with nanon */
1195
1196 uvm_pagecopy(oanon->u.an_page, pg); /* pg now !PG_CLEAN */
1197 pg->flags &= ~(PG_BUSY|PG_FAKE); /* un-busy! new page */
1198 UVM_PAGE_OWN(pg, NULL);
1199 amap_add(&ufi.entry->aref, ufi.orig_rvaddr - ufi.entry->start,
1200 anon, 1);
1201
1202 /* deref: can not drop to zero here by defn! */
1203 oanon->an_ref--;
1204
1205 /*
1206 * note: oanon is still locked, as is the new anon. we
1207 * need to check for this later when we unlock oanon; if
1208 * oanon != anon, we'll have to unlock anon, too.
1209 */
1210
1211 } else {
1212
1213 uvmexp.flt_anon++;
1214 oanon = anon; /* old, locked anon is same as anon */
1215 pg = anon->u.an_page;
1216 if (anon->an_ref > 1) /* disallow writes to ref > 1 anons */
1217 enter_prot = enter_prot & ~VM_PROT_WRITE;
1218
1219 }
1220
1221 /* locked: maps(read), amap, oanon, anon (if different from oanon) */
1222
1223 /*
1224 * now map the page in ...
1225 * XXX: old fault unlocks object before pmap_enter. this seems
1226 * suspect since some other thread could blast the page out from
1227 * under us between the unlock and the pmap_enter.
1228 */
1229
1230 UVMHIST_LOG(maphist, " MAPPING: anon: pm=0x%x, va=0x%x, pg=0x%x",
1231 ufi.orig_map->pmap, ufi.orig_rvaddr, pg, 0);
1232 if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg),
1233 enter_prot, access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0))
1234 != 0) {
1235 /*
1236 * No need to undo what we did; we can simply think of
1237 * this as the pmap throwing away the mapping information.
1238 *
1239 * We do, however, have to go through the ReFault path,
1240 * as the map may change while we're asleep.
1241 */
1242 if (anon != oanon)
1243 simple_unlock(&anon->an_lock);
1244 uvmfault_unlockall(&ufi, amap, uobj, oanon);
1245 KASSERT(uvmexp.swpgonly <= uvmexp.swpages);
1246 if (uvmexp.swpgonly == uvmexp.swpages) {
1247 UVMHIST_LOG(maphist,
1248 "<- failed. out of VM",0,0,0,0);
1249 /* XXX instrumentation */
1250 return ENOMEM;
1251 }
1252 /* XXX instrumentation */
1253 uvm_wait("flt_pmfail1");
1254 goto ReFault;
1255 }
1256
1257 /*
1258 * ... update the page queues.
1259 */
1260
1261 uvm_lock_pageq();
1262
1263 if (fault_type == VM_FAULT_WIRE) {
1264 uvm_pagewire(pg);
1265
1266 /*
1267 * since the now-wired page cannot be paged out,
1268 * release its swap resources for others to use.
1269 * since an anon with no swap cannot be PG_CLEAN,
1270 * clear its clean flag now.
1271 */
1272
1273 pg->flags &= ~(PG_CLEAN);
1274 uvm_anon_dropswap(anon);
1275 } else {
1276 /* activate it */
1277 uvm_pageactivate(pg);
1278 }
1279
1280 uvm_unlock_pageq();
1281
1282 /*
1283 * done case 1! finish up by unlocking everything and returning success
1284 */
1285
1286 if (anon != oanon)
1287 simple_unlock(&anon->an_lock);
1288 uvmfault_unlockall(&ufi, amap, uobj, oanon);
1289 return 0;
1290
1291
1292 Case2:
1293 /*
1294 * handle case 2: faulting on backing object or zero fill
1295 */
1296
1297 /*
1298 * locked:
1299 * maps(read), amap(if there), uobj(if !null), uobjpage(if !null)
1300 */
1301
1302 /*
1303 * note that uobjpage can not be PGO_DONTCARE at this point. we now
1304 * set uobjpage to PGO_DONTCARE if we are doing a zero fill. if we
1305 * have a backing object, check and see if we are going to promote
1306 * the data up to an anon during the fault.
1307 */
1308
1309 if (uobj == NULL) {
1310 uobjpage = PGO_DONTCARE;
1311 promote = TRUE; /* always need anon here */
1312 } else {
1313 KASSERT(uobjpage != PGO_DONTCARE);
1314 promote = (access_type & VM_PROT_WRITE) &&
1315 UVM_ET_ISCOPYONWRITE(ufi.entry);
1316 }
1317 UVMHIST_LOG(maphist, " case 2 fault: promote=%d, zfill=%d",
1318 promote, (uobj == NULL), 0,0);
1319
1320 /*
1321 * if uobjpage is not null then we do not need to do I/O to get the
1322 * uobjpage.
1323 *
1324 * if uobjpage is null, then we need to unlock and ask the pager to
1325 * get the data for us. once we have the data, we need to reverify
1326 * the state the world. we are currently not holding any resources.
1327 */
1328
1329 if (uobjpage) {
1330 /* update rusage counters */
1331 curproc->l_proc->p_stats->p_ru.ru_minflt++;
1332 } else {
1333 /* update rusage counters */
1334 curproc->l_proc->p_stats->p_ru.ru_majflt++;
1335
1336 /* locked: maps(read), amap(if there), uobj */
1337 uvmfault_unlockall(&ufi, amap, NULL, NULL);
1338 /* locked: uobj */
1339
1340 uvmexp.fltget++;
1341 gotpages = 1;
1342 uoff = (ufi.orig_rvaddr - ufi.entry->start) + ufi.entry->offset;
1343 error = uobj->pgops->pgo_get(uobj, uoff, &uobjpage, &gotpages,
1344 0, access_type & MASK(ufi.entry), ufi.entry->advice,
1345 PGO_SYNCIO);
1346
1347 /* locked: uobjpage(if no error) */
1348
1349 /*
1350 * recover from I/O
1351 */
1352
1353 if (error) {
1354 if (error == EAGAIN) {
1355 UVMHIST_LOG(maphist,
1356 " pgo_get says TRY AGAIN!",0,0,0,0);
1357 tsleep(&lbolt, PVM, "fltagain2", 0);
1358 goto ReFault;
1359 }
1360
1361 UVMHIST_LOG(maphist, "<- pgo_get failed (code %d)",
1362 error, 0,0,0);
1363 return error;
1364 }
1365
1366 /* locked: uobjpage */
1367
1368 /*
1369 * re-verify the state of the world by first trying to relock
1370 * the maps. always relock the object.
1371 */
1372
1373 locked = uvmfault_relock(&ufi);
1374 if (locked && amap)
1375 amap_lock(amap);
1376 simple_lock(&uobj->vmobjlock);
1377
1378 /* locked(locked): maps(read), amap(if !null), uobj, uobjpage */
1379 /* locked(!locked): uobj, uobjpage */
1380
1381 /*
1382 * verify that the page has not be released and re-verify
1383 * that amap slot is still free. if there is a problem,
1384 * we unlock and clean up.
1385 */
1386
1387 if ((uobjpage->flags & PG_RELEASED) != 0 ||
1388 (locked && amap &&
1389 amap_lookup(&ufi.entry->aref,
1390 ufi.orig_rvaddr - ufi.entry->start))) {
1391 if (locked)
1392 uvmfault_unlockall(&ufi, amap, NULL, NULL);
1393 locked = FALSE;
1394 }
1395
1396 /*
1397 * didn't get the lock? release the page and retry.
1398 */
1399
1400 if (locked == FALSE) {
1401
1402 UVMHIST_LOG(maphist,
1403 " wasn't able to relock after fault: retry",
1404 0,0,0,0);
1405 if (uobjpage->flags & PG_WANTED)
1406 /* still holding object lock */
1407 wakeup(uobjpage);
1408
1409 if (uobjpage->flags & PG_RELEASED) {
1410 uvmexp.fltpgrele++;
1411 KASSERT(uobj->pgops->pgo_releasepg != NULL);
1412
1413 /* frees page */
1414 if (uobj->pgops->pgo_releasepg(uobjpage,NULL))
1415 /* unlock if still alive */
1416 simple_unlock(&uobj->vmobjlock);
1417 goto ReFault;
1418 }
1419
1420 uvm_lock_pageq();
1421 /* make sure it is in queues */
1422 uvm_pageactivate(uobjpage);
1423
1424 uvm_unlock_pageq();
1425 uobjpage->flags &= ~(PG_BUSY|PG_WANTED);
1426 UVM_PAGE_OWN(uobjpage, NULL);
1427 simple_unlock(&uobj->vmobjlock);
1428 goto ReFault;
1429
1430 }
1431
1432 /*
1433 * we have the data in uobjpage which is PG_BUSY and
1434 * !PG_RELEASED. we are holding object lock (so the page
1435 * can't be released on us).
1436 */
1437
1438 /* locked: maps(read), amap(if !null), uobj, uobjpage */
1439 }
1440
1441 /*
1442 * locked:
1443 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj)
1444 */
1445
1446 /*
1447 * notes:
1448 * - at this point uobjpage can not be NULL
1449 * - at this point uobjpage can not be PG_RELEASED (since we checked
1450 * for it above)
1451 * - at this point uobjpage could be PG_WANTED (handle later)
1452 */
1453
1454 if (promote == FALSE) {
1455
1456 /*
1457 * we are not promoting. if the mapping is COW ensure that we
1458 * don't give more access than we should (e.g. when doing a read
1459 * fault on a COPYONWRITE mapping we want to map the COW page in
1460 * R/O even though the entry protection could be R/W).
1461 *
1462 * set "pg" to the page we want to map in (uobjpage, usually)
1463 */
1464
1465 /* no anon in this case. */
1466 anon = NULL;
1467
1468 uvmexp.flt_obj++;
1469 if (UVM_ET_ISCOPYONWRITE(ufi.entry))
1470 enter_prot &= ~VM_PROT_WRITE;
1471 pg = uobjpage; /* map in the actual object */
1472
1473 /* assert(uobjpage != PGO_DONTCARE) */
1474
1475 /*
1476 * we are faulting directly on the page. be careful
1477 * about writing to loaned pages...
1478 */
1479 if (uobjpage->loan_count) {
1480
1481 if ((access_type & VM_PROT_WRITE) == 0) {
1482 /* read fault: cap the protection at readonly */
1483 /* cap! */
1484 enter_prot = enter_prot & ~VM_PROT_WRITE;
1485 } else {
1486 /* write fault: must break the loan here */
1487
1488 /* alloc new un-owned page */
1489 pg = uvm_pagealloc(NULL, 0, NULL, 0);
1490
1491 if (pg == NULL) {
1492 /*
1493 * drop ownership of page, it can't
1494 * be released
1495 */
1496 if (uobjpage->flags & PG_WANTED)
1497 wakeup(uobjpage);
1498 uobjpage->flags &= ~(PG_BUSY|PG_WANTED);
1499 UVM_PAGE_OWN(uobjpage, NULL);
1500
1501 uvm_lock_pageq();
1502 /* activate: we will need it later */
1503 uvm_pageactivate(uobjpage);
1504
1505 uvm_unlock_pageq();
1506 uvmfault_unlockall(&ufi, amap, uobj,
1507 NULL);
1508 UVMHIST_LOG(maphist,
1509 " out of RAM breaking loan, waiting",
1510 0,0,0,0);
1511 uvmexp.fltnoram++;
1512 uvm_wait("flt_noram4");
1513 goto ReFault;
1514 }
1515
1516 /*
1517 * copy the data from the old page to the new
1518 * one and clear the fake/clean flags on the
1519 * new page (keep it busy). force a reload
1520 * of the old page by clearing it from all
1521 * pmaps. then lock the page queues to
1522 * rename the pages.
1523 */
1524 uvm_pagecopy(uobjpage, pg); /* old -> new */
1525 pg->flags &= ~(PG_FAKE|PG_CLEAN);
1526 pmap_page_protect(uobjpage, VM_PROT_NONE);
1527 if (uobjpage->flags & PG_WANTED)
1528 wakeup(uobjpage);
1529 /* uobj still locked */
1530 uobjpage->flags &= ~(PG_WANTED|PG_BUSY);
1531 UVM_PAGE_OWN(uobjpage, NULL);
1532
1533 uvm_lock_pageq();
1534 offset = uobjpage->offset;
1535 /* remove old page */
1536 uvm_pagerealloc(uobjpage, NULL, 0);
1537
1538 /*
1539 * at this point we have absolutely no
1540 * control over uobjpage
1541 */
1542 /* install new page */
1543 uvm_pagerealloc(pg, uobj, offset);
1544 uvm_unlock_pageq();
1545
1546 /*
1547 * done! loan is broken and "pg" is
1548 * PG_BUSY. it can now replace uobjpage.
1549 */
1550
1551 uobjpage = pg;
1552
1553 } /* write fault case */
1554 } /* if loan_count */
1555
1556 } else {
1557
1558 /*
1559 * if we are going to promote the data to an anon we
1560 * allocate a blank anon here and plug it into our amap.
1561 */
1562 #if DIAGNOSTIC
1563 if (amap == NULL)
1564 panic("uvm_fault: want to promote data, but no anon");
1565 #endif
1566
1567 anon = uvm_analloc();
1568 if (anon) {
1569 /*
1570 * The new anon is locked.
1571 *
1572 * In `Fill in data...' below, if
1573 * uobjpage == PGO_DONTCARE, we want
1574 * a zero'd, dirty page, so have
1575 * uvm_pagealloc() do that for us.
1576 */
1577 pg = uvm_pagealloc(NULL, 0, anon,
1578 (uobjpage == PGO_DONTCARE) ? UVM_PGA_ZERO : 0);
1579 }
1580
1581 /*
1582 * out of memory resources?
1583 */
1584 if (anon == NULL || pg == NULL) {
1585
1586 if (anon != NULL) {
1587 anon->an_ref--;
1588 simple_unlock(&anon->an_lock);
1589 uvm_anfree(anon);
1590 }
1591
1592 /*
1593 * arg! must unbusy our page and fail or sleep.
1594 */
1595 if (uobjpage != PGO_DONTCARE) {
1596 if (uobjpage->flags & PG_WANTED)
1597 /* still holding object lock */
1598 wakeup(uobjpage);
1599
1600 uvm_lock_pageq();
1601 uvm_pageactivate(uobjpage);
1602 uvm_unlock_pageq();
1603 uobjpage->flags &= ~(PG_BUSY|PG_WANTED);
1604 UVM_PAGE_OWN(uobjpage, NULL);
1605 }
1606
1607 /* unlock and fail ... */
1608 uvmfault_unlockall(&ufi, amap, uobj, NULL);
1609 KASSERT(uvmexp.swpgonly <= uvmexp.swpages);
1610 if (anon == NULL || uvmexp.swpgonly == uvmexp.swpages) {
1611 UVMHIST_LOG(maphist, " promote: out of VM",
1612 0,0,0,0);
1613 uvmexp.fltnoanon++;
1614 return ENOMEM;
1615 }
1616
1617 UVMHIST_LOG(maphist, " out of RAM, waiting for more",
1618 0,0,0,0);
1619 uvmexp.fltnoram++;
1620 uvm_wait("flt_noram5");
1621 goto ReFault;
1622 }
1623
1624 /*
1625 * fill in the data
1626 */
1627
1628 if (uobjpage != PGO_DONTCARE) {
1629 uvmexp.flt_prcopy++;
1630 /* copy page [pg now dirty] */
1631 uvm_pagecopy(uobjpage, pg);
1632
1633 /*
1634 * promote to shared amap? make sure all sharing
1635 * procs see it
1636 */
1637 if ((amap_flags(amap) & AMAP_SHARED) != 0) {
1638 pmap_page_protect(uobjpage, VM_PROT_NONE);
1639 }
1640
1641 /*
1642 * dispose of uobjpage. it can't be PG_RELEASED
1643 * since we still hold the object lock.
1644 * drop handle to uobj as well.
1645 */
1646
1647 if (uobjpage->flags & PG_WANTED)
1648 /* still have the obj lock */
1649 wakeup(uobjpage);
1650 uobjpage->flags &= ~(PG_BUSY|PG_WANTED);
1651 UVM_PAGE_OWN(uobjpage, NULL);
1652 uvm_lock_pageq();
1653 uvm_pageactivate(uobjpage);
1654 uvm_unlock_pageq();
1655 simple_unlock(&uobj->vmobjlock);
1656 uobj = NULL;
1657
1658 UVMHIST_LOG(maphist,
1659 " promote uobjpage 0x%x to anon/page 0x%x/0x%x",
1660 uobjpage, anon, pg, 0);
1661
1662 } else {
1663 uvmexp.flt_przero++;
1664 /*
1665 * Page is zero'd and marked dirty by uvm_pagealloc()
1666 * above.
1667 */
1668 UVMHIST_LOG(maphist," zero fill anon/page 0x%x/0%x",
1669 anon, pg, 0, 0);
1670 }
1671
1672 amap_add(&ufi.entry->aref, ufi.orig_rvaddr - ufi.entry->start,
1673 anon, 0);
1674 }
1675
1676 /*
1677 * locked:
1678 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj),
1679 * anon(if !null), pg(if anon)
1680 *
1681 * note: pg is either the uobjpage or the new page in the new anon
1682 */
1683
1684 /*
1685 * all resources are present. we can now map it in and free our
1686 * resources.
1687 */
1688
1689 UVMHIST_LOG(maphist,
1690 " MAPPING: case2: pm=0x%x, va=0x%x, pg=0x%x, promote=%d",
1691 ufi.orig_map->pmap, ufi.orig_rvaddr, pg, promote);
1692 KASSERT(access_type == VM_PROT_READ || (pg->flags & PG_RDONLY) == 0);
1693 if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg),
1694 pg->flags & PG_RDONLY ? VM_PROT_READ : enter_prot,
1695 access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0)) != 0) {
1696
1697 /*
1698 * No need to undo what we did; we can simply think of
1699 * this as the pmap throwing away the mapping information.
1700 *
1701 * We do, however, have to go through the ReFault path,
1702 * as the map may change while we're asleep.
1703 */
1704
1705 if (pg->flags & PG_WANTED)
1706 wakeup(pg); /* lock still held */
1707
1708 /*
1709 * note that pg can't be PG_RELEASED since we did not drop
1710 * the object lock since the last time we checked.
1711 */
1712
1713 pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED);
1714 UVM_PAGE_OWN(pg, NULL);
1715 uvmfault_unlockall(&ufi, amap, uobj, anon);
1716 KASSERT(uvmexp.swpgonly <= uvmexp.swpages);
1717 if (uvmexp.swpgonly == uvmexp.swpages) {
1718 UVMHIST_LOG(maphist,
1719 "<- failed. out of VM",0,0,0,0);
1720 /* XXX instrumentation */
1721 return ENOMEM;
1722 }
1723 /* XXX instrumentation */
1724 uvm_wait("flt_pmfail2");
1725 goto ReFault;
1726 }
1727
1728 uvm_lock_pageq();
1729
1730 if (fault_type == VM_FAULT_WIRE) {
1731 uvm_pagewire(pg);
1732 if (pg->pqflags & PQ_AOBJ) {
1733
1734 /*
1735 * since the now-wired page cannot be paged out,
1736 * release its swap resources for others to use.
1737 * since an aobj page with no swap cannot be PG_CLEAN,
1738 * clear its clean flag now.
1739 */
1740
1741 pg->flags &= ~(PG_CLEAN);
1742 uao_dropswap(uobj, pg->offset >> PAGE_SHIFT);
1743 }
1744 } else {
1745 /* activate it */
1746 uvm_pageactivate(pg);
1747 }
1748 uvm_unlock_pageq();
1749
1750 if (pg->flags & PG_WANTED)
1751 wakeup(pg); /* lock still held */
1752
1753 /*
1754 * note that pg can't be PG_RELEASED since we did not drop the object
1755 * lock since the last time we checked.
1756 */
1757
1758 pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED);
1759 UVM_PAGE_OWN(pg, NULL);
1760 uvmfault_unlockall(&ufi, amap, uobj, anon);
1761
1762 UVMHIST_LOG(maphist, "<- done (SUCCESS!)",0,0,0,0);
1763 return 0;
1764 }
1765
1766
1767 /*
1768 * uvm_fault_wire: wire down a range of virtual addresses in a map.
1769 *
1770 * => map may be read-locked by caller, but MUST NOT be write-locked.
1771 * => if map is read-locked, any operations which may cause map to
1772 * be write-locked in uvm_fault() must be taken care of by
1773 * the caller. See uvm_map_pageable().
1774 */
1775
1776 int
1777 uvm_fault_wire(map, start, end, access_type)
1778 vm_map_t map;
1779 vaddr_t start, end;
1780 vm_prot_t access_type;
1781 {
1782 vaddr_t va;
1783 pmap_t pmap;
1784 int error;
1785
1786 pmap = vm_map_pmap(map);
1787
1788 /*
1789 * now fault it in a page at a time. if the fault fails then we have
1790 * to undo what we have done. note that in uvm_fault VM_PROT_NONE
1791 * is replaced with the max protection if fault_type is VM_FAULT_WIRE.
1792 */
1793
1794 for (va = start ; va < end ; va += PAGE_SIZE) {
1795 error = uvm_fault(map, va, VM_FAULT_WIRE, access_type);
1796 if (error) {
1797 if (va != start) {
1798 uvm_fault_unwire(map, start, va);
1799 }
1800 return error;
1801 }
1802 }
1803
1804 return 0;
1805 }
1806
1807 /*
1808 * uvm_fault_unwire(): unwire range of virtual space.
1809 */
1810
1811 void
1812 uvm_fault_unwire(map, start, end)
1813 vm_map_t map;
1814 vaddr_t start, end;
1815 {
1816
1817 vm_map_lock_read(map);
1818 uvm_fault_unwire_locked(map, start, end);
1819 vm_map_unlock_read(map);
1820 }
1821
1822 /*
1823 * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire().
1824 *
1825 * => map must be at least read-locked.
1826 */
1827
1828 void
1829 uvm_fault_unwire_locked(map, start, end)
1830 vm_map_t map;
1831 vaddr_t start, end;
1832 {
1833 vm_map_entry_t entry;
1834 pmap_t pmap = vm_map_pmap(map);
1835 vaddr_t va;
1836 paddr_t pa;
1837 struct vm_page *pg;
1838
1839 KASSERT((map->flags & VM_MAP_INTRSAFE) == 0);
1840
1841 /*
1842 * we assume that the area we are unwiring has actually been wired
1843 * in the first place. this means that we should be able to extract
1844 * the PAs from the pmap. we also lock out the page daemon so that
1845 * we can call uvm_pageunwire.
1846 */
1847
1848 uvm_lock_pageq();
1849
1850 /*
1851 * find the beginning map entry for the region.
1852 */
1853 KASSERT(start >= vm_map_min(map) && end <= vm_map_max(map));
1854 if (uvm_map_lookup_entry(map, start, &entry) == FALSE)
1855 panic("uvm_fault_unwire_locked: address not in map");
1856
1857 for (va = start; va < end ; va += PAGE_SIZE) {
1858 if (pmap_extract(pmap, va, &pa) == FALSE)
1859 panic("uvm_fault_unwire_locked: unwiring "
1860 "non-wired memory");
1861
1862 /*
1863 * make sure the current entry is for the address we're
1864 * dealing with. if not, grab the next entry.
1865 */
1866
1867 KASSERT(va >= entry->start);
1868 if (va >= entry->end) {
1869 KASSERT(entry->next != &map->header &&
1870 entry->next->start <= entry->end);
1871 entry = entry->next;
1872 }
1873
1874 /*
1875 * if the entry is no longer wired, tell the pmap.
1876 */
1877 if (VM_MAPENT_ISWIRED(entry) == 0)
1878 pmap_unwire(pmap, va);
1879
1880 pg = PHYS_TO_VM_PAGE(pa);
1881 if (pg)
1882 uvm_pageunwire(pg);
1883 }
1884
1885 uvm_unlock_pageq();
1886 }
1887