uvm_pager.c revision 1.23.4.1 1 /* $NetBSD: uvm_pager.c,v 1.23.4.1 1999/11/15 00:42:30 fvdl Exp $ */
2
3 /*
4 *
5 * Copyright (c) 1997 Charles D. Cranor and Washington University.
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 3. All advertising materials mentioning features or use of this software
17 * must display the following acknowledgement:
18 * This product includes software developed by Charles D. Cranor and
19 * Washington University.
20 * 4. The name of the author may not be used to endorse or promote products
21 * derived from this software without specific prior written permission.
22 *
23 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
24 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
25 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
26 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
27 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
28 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
32 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33 *
34 * from: Id: uvm_pager.c,v 1.1.2.23 1998/02/02 20:38:06 chuck Exp
35 */
36
37 #include "opt_uvmhist.h"
38
39 /*
40 * uvm_pager.c: generic functions used to assist the pagers.
41 */
42
43 #include <sys/param.h>
44 #include <sys/systm.h>
45 #include <sys/proc.h>
46 #include <sys/malloc.h>
47
48 #include <vm/vm.h>
49 #include <vm/vm_page.h>
50 #include <vm/vm_kern.h>
51
52 #define UVM_PAGER
53 #include <uvm/uvm.h>
54
55 /*
56 * list of uvm pagers in the system
57 */
58
59 extern struct uvm_pagerops aobj_pager;
60 extern struct uvm_pagerops uvm_deviceops;
61 extern struct uvm_pagerops uvm_vnodeops;
62
63 struct uvm_pagerops *uvmpagerops[] = {
64 &aobj_pager,
65 &uvm_deviceops,
66 &uvm_vnodeops,
67 };
68
69 /*
70 * the pager map: provides KVA for I/O
71 */
72
73 #define PAGER_MAP_SIZE (4 * 1024 * 1024)
74 vm_map_t pager_map; /* XXX */
75 simple_lock_data_t pager_map_wanted_lock;
76 boolean_t pager_map_wanted; /* locked by pager map */
77
78
79 /*
80 * uvm_pager_init: init pagers (at boot time)
81 */
82
83 void
84 uvm_pager_init()
85 {
86 int lcv;
87
88 /*
89 * init pager map
90 */
91
92 pager_map = uvm_km_suballoc(kernel_map, &uvm.pager_sva, &uvm.pager_eva,
93 PAGER_MAP_SIZE, 0, FALSE, NULL);
94 simple_lock_init(&pager_map_wanted_lock);
95 pager_map_wanted = FALSE;
96
97 /*
98 * init ASYNC I/O queue
99 */
100
101 TAILQ_INIT(&uvm.aio_done);
102
103 /*
104 * call pager init functions
105 */
106 for (lcv = 0 ; lcv < sizeof(uvmpagerops)/sizeof(struct uvm_pagerops *);
107 lcv++) {
108 if (uvmpagerops[lcv]->pgo_init)
109 uvmpagerops[lcv]->pgo_init();
110 }
111 }
112
113 /*
114 * uvm_pagermapin: map pages into KVA (pager_map) for I/O that needs mappings
115 *
116 * we basically just map in a blank map entry to reserve the space in the
117 * map and then use pmap_enter() to put the mappings in by hand.
118 *
119 * XXX It would be nice to know the direction of the I/O, so that we can
120 * XXX map only what is necessary.
121 */
122
123 vaddr_t
124 uvm_pagermapin(pps, npages, aiop, waitf)
125 struct vm_page **pps;
126 int npages;
127 struct uvm_aiodesc **aiop; /* OUT */
128 int waitf;
129 {
130 vsize_t size;
131 vaddr_t kva;
132 struct uvm_aiodesc *aio;
133 vaddr_t cva;
134 struct vm_page *pp;
135 UVMHIST_FUNC("uvm_pagermapin"); UVMHIST_CALLED(maphist);
136
137 UVMHIST_LOG(maphist,"(pps=0x%x, npages=%d, aiop=0x%x, waitf=%d)",
138 pps, npages, aiop, waitf);
139
140 ReStart:
141 if (aiop) {
142 MALLOC(aio, struct uvm_aiodesc *, sizeof(*aio), M_TEMP, waitf);
143 if (aio == NULL)
144 return(0);
145 *aiop = aio;
146 } else {
147 aio = NULL;
148 }
149
150 size = npages << PAGE_SHIFT;
151 kva = NULL; /* let system choose VA */
152
153 if (uvm_map(pager_map, &kva, size, NULL,
154 UVM_UNKNOWN_OFFSET, UVM_FLAG_NOMERGE) != KERN_SUCCESS) {
155 if (waitf == M_NOWAIT) {
156 if (aio)
157 FREE(aio, M_TEMP);
158 UVMHIST_LOG(maphist,"<- NOWAIT failed", 0,0,0,0);
159 return(NULL);
160 }
161 simple_lock(&pager_map_wanted_lock);
162 pager_map_wanted = TRUE;
163 UVMHIST_LOG(maphist, " SLEEPING on pager_map",0,0,0,0);
164 UVM_UNLOCK_AND_WAIT(pager_map, &pager_map_wanted_lock, FALSE,
165 "pager_map",0);
166 goto ReStart;
167 }
168
169 /* got it */
170 for (cva = kva ; size != 0 ; size -= PAGE_SIZE, cva += PAGE_SIZE) {
171 pp = *pps++;
172 #ifdef DEBUG
173 if ((pp->flags & PG_BUSY) == 0)
174 panic("uvm_pagermapin: page not busy");
175 #endif
176
177 /*
178 * XXX VM_PROT_DEFAULT includes VM_PROT_EXEC; is that
179 * XXX really necessary? It could lead to unnecessary
180 * XXX instruction cache flushes.
181 */
182 pmap_enter(vm_map_pmap(pager_map), cva, VM_PAGE_TO_PHYS(pp),
183 VM_PROT_DEFAULT, PMAP_WIRED | VM_PROT_READ | VM_PROT_WRITE);
184 }
185
186 UVMHIST_LOG(maphist, "<- done (KVA=0x%x)", kva,0,0,0);
187 return(kva);
188 }
189
190 /*
191 * uvm_pagermapout: remove pager_map mapping
192 *
193 * we remove our mappings by hand and then remove the mapping (waking
194 * up anyone wanting space).
195 */
196
197 void
198 uvm_pagermapout(kva, npages)
199 vaddr_t kva;
200 int npages;
201 {
202 vsize_t size = npages << PAGE_SHIFT;
203 vm_map_entry_t entries;
204 UVMHIST_FUNC("uvm_pagermapout"); UVMHIST_CALLED(maphist);
205
206 UVMHIST_LOG(maphist, " (kva=0x%x, npages=%d)", kva, npages,0,0);
207
208 /*
209 * duplicate uvm_unmap, but add in pager_map_wanted handling.
210 */
211
212 vm_map_lock(pager_map);
213 (void) uvm_unmap_remove(pager_map, kva, kva + size, &entries);
214 simple_lock(&pager_map_wanted_lock);
215 if (pager_map_wanted) {
216 pager_map_wanted = FALSE;
217 wakeup(pager_map);
218 }
219 simple_unlock(&pager_map_wanted_lock);
220 vm_map_unlock(pager_map);
221 if (entries)
222 uvm_unmap_detach(entries, 0);
223
224 UVMHIST_LOG(maphist,"<- done",0,0,0,0);
225 }
226
227 /*
228 * uvm_mk_pcluster
229 *
230 * generic "make 'pager put' cluster" function. a pager can either
231 * [1] set pgo_mk_pcluster to NULL (never cluster), [2] set it to this
232 * generic function, or [3] set it to a pager specific function.
233 *
234 * => caller must lock object _and_ pagequeues (since we need to look
235 * at active vs. inactive bits, etc.)
236 * => caller must make center page busy and write-protect it
237 * => we mark all cluster pages busy for the caller
238 * => the caller must unbusy all pages (and check wanted/released
239 * status if it drops the object lock)
240 * => flags:
241 * PGO_ALLPAGES: all pages in object are valid targets
242 * !PGO_ALLPAGES: use "lo" and "hi" to limit range of cluster
243 * PGO_DOACTCLUST: include active pages in cluster.
244 * NOTE: the caller should clear PG_CLEANCHK bits if PGO_DOACTCLUST.
245 * PG_CLEANCHK is only a hint, but clearing will help reduce
246 * the number of calls we make to the pmap layer.
247 */
248
249 struct vm_page **
250 uvm_mk_pcluster(uobj, pps, npages, center, flags, mlo, mhi)
251 struct uvm_object *uobj; /* IN */
252 struct vm_page **pps, *center; /* IN/OUT, IN */
253 int *npages, flags; /* IN/OUT, IN */
254 vaddr_t mlo, mhi; /* IN (if !PGO_ALLPAGES) */
255 {
256 struct vm_page **ppsp, *pclust;
257 vaddr_t lo, hi, curoff;
258 int center_idx, forward;
259 UVMHIST_FUNC("uvm_mk_pcluster"); UVMHIST_CALLED(maphist);
260
261 /*
262 * center page should already be busy and write protected. XXX:
263 * suppose page is wired? if we lock, then a process could
264 * fault/block on it. if we don't lock, a process could write the
265 * pages in the middle of an I/O. (consider an msync()). let's
266 * lock it for now (better to delay than corrupt data?).
267 */
268
269 /*
270 * get cluster boundaries, check sanity, and apply our limits as well.
271 */
272
273 uobj->pgops->pgo_cluster(uobj, center->offset, &lo, &hi);
274 if ((flags & PGO_ALLPAGES) == 0) {
275 if (lo < mlo)
276 lo = mlo;
277 if (hi > mhi)
278 hi = mhi;
279 }
280 if ((hi - lo) >> PAGE_SHIFT > *npages) { /* pps too small, bail out! */
281 #ifdef DIAGNOSTIC
282 printf("uvm_mk_pcluster: provided page array too small (fixed)\n");
283 #endif
284 pps[0] = center;
285 *npages = 1;
286 return(pps);
287 }
288
289 /*
290 * now determine the center and attempt to cluster around the
291 * edges
292 */
293
294 center_idx = (center->offset - lo) >> PAGE_SHIFT;
295 pps[center_idx] = center; /* plug in the center page */
296 ppsp = &pps[center_idx];
297 *npages = 1;
298
299 /*
300 * attempt to cluster around the left [backward], and then
301 * the right side [forward].
302 *
303 * note that for inactive pages (pages that have been deactivated)
304 * there are no valid mappings and PG_CLEAN should be up to date.
305 * [i.e. there is no need to query the pmap with pmap_is_modified
306 * since there are no mappings].
307 */
308
309 for (forward = 0 ; forward <= 1 ; forward++) {
310
311 curoff = center->offset + (forward ? PAGE_SIZE : -PAGE_SIZE);
312 for ( ;(forward == 0 && curoff >= lo) ||
313 (forward && curoff < hi);
314 curoff += (forward ? 1 : -1) << PAGE_SHIFT) {
315
316 pclust = uvm_pagelookup(uobj, curoff); /* lookup page */
317 if (pclust == NULL)
318 break; /* no page */
319 /* handle active pages */
320 /* NOTE: inactive pages don't have pmap mappings */
321 if ((pclust->pqflags & PQ_INACTIVE) == 0) {
322 if ((flags & PGO_DOACTCLUST) == 0)
323 /* dont want mapped pages at all */
324 break;
325
326 /* make sure "clean" bit is sync'd */
327 if ((pclust->flags & PG_CLEANCHK) == 0) {
328 if ((pclust->flags & (PG_CLEAN|PG_BUSY))
329 == PG_CLEAN &&
330 pmap_is_modified(pclust))
331 pclust->flags &= ~PG_CLEAN;
332
333 /* now checked */
334 pclust->flags |= PG_CLEANCHK;
335 }
336 }
337 /* is page available for cleaning and does it need it */
338 if ((pclust->flags & (PG_CLEAN|PG_BUSY)) != 0)
339 break; /* page is already clean or is busy */
340
341 /* yes! enroll the page in our array */
342 pclust->flags |= PG_BUSY; /* busy! */
343 UVM_PAGE_OWN(pclust, "uvm_mk_pcluster");
344 /* XXX: protect wired page? see above comment. */
345 pmap_page_protect(pclust, VM_PROT_READ);
346 if (!forward) {
347 ppsp--; /* back up one page */
348 *ppsp = pclust;
349 } else {
350 /* move forward one page */
351 ppsp[*npages] = pclust;
352 }
353 *npages = *npages + 1;
354 }
355 }
356
357 /*
358 * done! return the cluster array to the caller!!!
359 */
360
361 UVMHIST_LOG(maphist, "<- done",0,0,0,0);
362 return(ppsp);
363 }
364
365
366 /*
367 * uvm_shareprot: generic share protect routine
368 *
369 * => caller must lock map entry's map
370 * => caller must lock object pointed to by map entry
371 */
372
373 void
374 uvm_shareprot(entry, prot)
375 vm_map_entry_t entry;
376 vm_prot_t prot;
377 {
378 struct uvm_object *uobj = entry->object.uvm_obj;
379 struct vm_page *pp;
380 vaddr_t start, stop;
381 UVMHIST_FUNC("uvm_shareprot"); UVMHIST_CALLED(maphist);
382
383 if (UVM_ET_ISSUBMAP(entry))
384 panic("uvm_shareprot: non-object attached");
385
386 start = entry->offset;
387 stop = start + (entry->end - entry->start);
388
389 /*
390 * traverse list of pages in object. if page in range, pmap_prot it
391 */
392
393 for (pp = uobj->memq.tqh_first ; pp != NULL ; pp = pp->listq.tqe_next) {
394 if (pp->offset >= start && pp->offset < stop)
395 pmap_page_protect(pp, prot);
396 }
397 UVMHIST_LOG(maphist, "<- done",0,0,0,0);
398 }
399
400 /*
401 * uvm_pager_put: high level pageout routine
402 *
403 * we want to pageout page "pg" to backing store, clustering if
404 * possible.
405 *
406 * => page queues must be locked by caller
407 * => if page is not swap-backed, then "uobj" points to the object
408 * backing it. this object should be locked by the caller.
409 * => if page is swap-backed, then "uobj" should be NULL.
410 * => "pg" should be PG_BUSY (by caller), and !PG_CLEAN
411 * for swap-backed memory, "pg" can be NULL if there is no page
412 * of interest [sometimes the case for the pagedaemon]
413 * => "ppsp_ptr" should point to an array of npages vm_page pointers
414 * for possible cluster building
415 * => flags (first two for non-swap-backed pages)
416 * PGO_ALLPAGES: all pages in uobj are valid targets
417 * PGO_DOACTCLUST: include "PQ_ACTIVE" pages as valid targets
418 * PGO_SYNCIO: do SYNC I/O (no async)
419 * PGO_PDFREECLUST: pagedaemon: drop cluster on successful I/O
420 * => start/stop: if (uobj && !PGO_ALLPAGES) limit targets to this range
421 * if (!uobj) start is the (daddr_t) of the starting swapblk
422 * => return state:
423 * 1. we return the VM_PAGER status code of the pageout
424 * 2. we return with the page queues unlocked
425 * 3. if (uobj != NULL) [!swap_backed] we return with
426 * uobj locked _only_ if PGO_PDFREECLUST is set
427 * AND result != VM_PAGER_PEND. in all other cases
428 * we return with uobj unlocked. [this is a hack
429 * that allows the pagedaemon to save one lock/unlock
430 * pair in the !swap_backed case since we have to
431 * lock the uobj to drop the cluster anyway]
432 * 4. on errors we always drop the cluster. thus, if we return
433 * !PEND, !OK, then the caller only has to worry about
434 * un-busying the main page (not the cluster pages).
435 * 5. on success, if !PGO_PDFREECLUST, we return the cluster
436 * with all pages busy (caller must un-busy and check
437 * wanted/released flags).
438 */
439
440 int
441 uvm_pager_put(uobj, pg, ppsp_ptr, npages, flags, start, stop)
442 struct uvm_object *uobj; /* IN */
443 struct vm_page *pg, ***ppsp_ptr;/* IN, IN/OUT */
444 int *npages; /* IN/OUT */
445 int flags; /* IN */
446 vaddr_t start, stop; /* IN, IN */
447 {
448 int result;
449 daddr_t swblk;
450 struct vm_page **ppsp = *ppsp_ptr;
451
452 /*
453 * note that uobj is null if we are doing a swap-backed pageout.
454 * note that uobj is !null if we are doing normal object pageout.
455 * note that the page queues must be locked to cluster.
456 */
457
458 if (uobj) { /* if !swap-backed */
459
460 /*
461 * attempt to build a cluster for pageout using its
462 * make-put-cluster function (if it has one).
463 */
464
465 if (uobj->pgops->pgo_mk_pcluster) {
466 ppsp = uobj->pgops->pgo_mk_pcluster(uobj, ppsp,
467 npages, pg, flags, start, stop);
468 *ppsp_ptr = ppsp; /* update caller's pointer */
469 } else {
470 ppsp[0] = pg;
471 *npages = 1;
472 }
473
474 swblk = 0; /* XXX: keep gcc happy */
475
476 } else {
477
478 /*
479 * for swap-backed pageout, the caller (the pagedaemon) has
480 * already built the cluster for us. the starting swap
481 * block we are writing to has been passed in as "start."
482 * "pg" could be NULL if there is no page we are especially
483 * interested in (in which case the whole cluster gets dropped
484 * in the event of an error or a sync "done").
485 */
486 swblk = (daddr_t) start;
487 /* ppsp and npages should be ok */
488 }
489
490 /* now that we've clustered we can unlock the page queues */
491 uvm_unlock_pageq();
492
493 /*
494 * now attempt the I/O. if we have a failure and we are
495 * clustered, we will drop the cluster and try again.
496 */
497
498 ReTry:
499 if (uobj) {
500 /* object is locked */
501 result = uobj->pgops->pgo_put(uobj, ppsp, *npages,
502 flags & PGO_SYNCIO);
503 /* object is now unlocked */
504 } else {
505 /* nothing locked */
506 result = uvm_swap_put(swblk, ppsp, *npages, flags & PGO_SYNCIO);
507 /* nothing locked */
508 }
509
510 /*
511 * we have attempted the I/O.
512 *
513 * if the I/O was a success then:
514 * if !PGO_PDFREECLUST, we return the cluster to the
515 * caller (who must un-busy all pages)
516 * else we un-busy cluster pages for the pagedaemon
517 *
518 * if I/O is pending (async i/o) then we return the pending code.
519 * [in this case the async i/o done function must clean up when
520 * i/o is done...]
521 */
522
523 if (result == VM_PAGER_PEND || result == VM_PAGER_OK) {
524 if (result == VM_PAGER_OK && (flags & PGO_PDFREECLUST)) {
525 /*
526 * drop cluster and relock object (only if I/O is
527 * not pending)
528 */
529 if (uobj)
530 /* required for dropcluster */
531 simple_lock(&uobj->vmobjlock);
532 if (*npages > 1 || pg == NULL)
533 uvm_pager_dropcluster(uobj, pg, ppsp, npages,
534 PGO_PDFREECLUST, 0);
535 /* if (uobj): object still locked, as per
536 * return-state item #3 */
537 }
538 return (result);
539 }
540
541 /*
542 * a pager error occured. if we have clustered, we drop the
543 * cluster and try again.
544 */
545
546 if (*npages > 1 || pg == NULL) {
547 if (uobj)
548 simple_lock(&uobj->vmobjlock);
549 uvm_pager_dropcluster(uobj, pg, ppsp, npages, PGO_REALLOCSWAP,
550 swblk);
551 if (pg != NULL)
552 goto ReTry;
553 }
554
555 /*
556 * a pager error occured (even after dropping the cluster, if there
557 * was one). give up! the caller only has one page ("pg")
558 * to worry about.
559 */
560
561 if (uobj && (flags & PGO_PDFREECLUST) != 0)
562 simple_lock(&uobj->vmobjlock);
563 return(result);
564 }
565
566 /*
567 * uvm_pager_dropcluster: drop a cluster we have built (because we
568 * got an error, or, if PGO_PDFREECLUST we are un-busying the
569 * cluster pages on behalf of the pagedaemon).
570 *
571 * => uobj, if non-null, is a non-swap-backed object that is
572 * locked by the caller. we return with this object still
573 * locked.
574 * => page queues are not locked
575 * => pg is our page of interest (the one we clustered around, can be null)
576 * => ppsp/npages is our current cluster
577 * => flags: PGO_PDFREECLUST: pageout was a success: un-busy cluster
578 * pages on behalf of the pagedaemon.
579 * PGO_REALLOCSWAP: drop previously allocated swap slots for
580 * clustered swap-backed pages (except for "pg" if !NULL)
581 * "swblk" is the start of swap alloc (e.g. for ppsp[0])
582 * [only meaningful if swap-backed (uobj == NULL)]
583 */
584
585 void
586 uvm_pager_dropcluster(uobj, pg, ppsp, npages, flags, swblk)
587 struct uvm_object *uobj; /* IN */
588 struct vm_page *pg, **ppsp; /* IN, IN/OUT */
589 int *npages; /* IN/OUT */
590 int flags;
591 int swblk; /* valid if
592 (uobj == NULL && PGO_REALLOCSWAP) */
593 {
594 int lcv;
595 boolean_t obj_is_alive;
596 struct uvm_object *saved_uobj;
597
598 /*
599 * if we need to reallocate swap space for the cluster we are dropping
600 * (true if swap-backed and PGO_REALLOCSWAP) then free the old
601 * allocation now. save a block for "pg" if it is non-NULL.
602 *
603 * note that we will zap the object's pointer to swap in the "for" loop
604 * below...
605 */
606
607 if (uobj == NULL && (flags & PGO_REALLOCSWAP)) {
608 if (pg)
609 uvm_swap_free(swblk + 1, *npages - 1);
610 else
611 uvm_swap_free(swblk, *npages);
612 }
613
614 /*
615 * drop all pages but "pg"
616 */
617
618 for (lcv = 0 ; lcv < *npages ; lcv++) {
619
620 if (ppsp[lcv] == pg) /* skip "pg" */
621 continue;
622
623 /*
624 * if swap-backed, gain lock on object that owns page. note
625 * that PQ_ANON bit can't change as long as we are holding
626 * the PG_BUSY bit (so there is no need to lock the page
627 * queues to test it).
628 *
629 * once we have the lock, dispose of the pointer to swap, if
630 * requested
631 */
632 if (!uobj) {
633 if (ppsp[lcv]->pqflags & PQ_ANON) {
634 simple_lock(&ppsp[lcv]->uanon->an_lock);
635 if (flags & PGO_REALLOCSWAP)
636 /* zap swap block */
637 ppsp[lcv]->uanon->an_swslot = 0;
638 } else {
639 simple_lock(&ppsp[lcv]->uobject->vmobjlock);
640 if (flags & PGO_REALLOCSWAP)
641 uao_set_swslot(ppsp[lcv]->uobject,
642 ppsp[lcv]->offset >> PAGE_SHIFT, 0);
643 }
644 }
645
646 /* did someone want the page while we had it busy-locked? */
647 if (ppsp[lcv]->flags & PG_WANTED)
648 /* still holding obj lock */
649 wakeup(ppsp[lcv]);
650
651 /* if page was released, release it. otherwise un-busy it */
652 if (ppsp[lcv]->flags & PG_RELEASED) {
653
654 if (ppsp[lcv]->pqflags & PQ_ANON) {
655 /* so that anfree will free */
656 ppsp[lcv]->flags &= ~(PG_BUSY);
657 UVM_PAGE_OWN(ppsp[lcv], NULL);
658
659 pmap_page_protect(ppsp[lcv], VM_PROT_NONE);
660 simple_unlock(&ppsp[lcv]->uanon->an_lock);
661 /* kills anon and frees pg */
662 uvm_anfree(ppsp[lcv]->uanon);
663
664 continue;
665 }
666
667 /*
668 * pgo_releasepg will dump the page for us
669 */
670
671 #ifdef DIAGNOSTIC
672 if (ppsp[lcv]->uobject->pgops->pgo_releasepg == NULL)
673 panic("uvm_pager_dropcluster: no releasepg "
674 "function");
675 #endif
676 saved_uobj = ppsp[lcv]->uobject;
677 obj_is_alive =
678 saved_uobj->pgops->pgo_releasepg(ppsp[lcv], NULL);
679
680 #ifdef DIAGNOSTIC
681 /* for normal objects, "pg" is still PG_BUSY by us,
682 * so obj can't die */
683 if (uobj && !obj_is_alive)
684 panic("uvm_pager_dropcluster: object died "
685 "with active page");
686 #endif
687 /* only unlock the object if it is still alive... */
688 if (obj_is_alive && saved_uobj != uobj)
689 simple_unlock(&saved_uobj->vmobjlock);
690
691 /*
692 * XXXCDC: suppose uobj died in the pgo_releasepg?
693 * how pass that
694 * info up to caller. we are currently ignoring it...
695 */
696
697 continue; /* next page */
698
699 } else {
700 ppsp[lcv]->flags &= ~(PG_BUSY|PG_WANTED);
701 UVM_PAGE_OWN(ppsp[lcv], NULL);
702 }
703
704 /*
705 * if we are operating on behalf of the pagedaemon and we
706 * had a successful pageout update the page!
707 */
708 if (flags & PGO_PDFREECLUST) {
709 pmap_clear_reference(ppsp[lcv]);
710 pmap_clear_modify(ppsp[lcv]);
711 ppsp[lcv]->flags |= PG_CLEAN;
712 }
713
714 /* if anonymous cluster, unlock object and move on */
715 if (!uobj) {
716 if (ppsp[lcv]->pqflags & PQ_ANON)
717 simple_unlock(&ppsp[lcv]->uanon->an_lock);
718 else
719 simple_unlock(&ppsp[lcv]->uobject->vmobjlock);
720 }
721
722 }
723
724 /*
725 * drop to a cluster of 1 page ("pg") if requested
726 */
727
728 if (pg && (flags & PGO_PDFREECLUST) == 0) {
729 /*
730 * if we are not a successful pageout, we make a 1 page cluster.
731 */
732 ppsp[0] = pg;
733 *npages = 1;
734
735 /*
736 * assign new swap block to new cluster, if anon backed
737 */
738 if (uobj == NULL && (flags & PGO_REALLOCSWAP)) {
739 if (pg->pqflags & PQ_ANON) {
740 simple_lock(&pg->uanon->an_lock);
741 pg->uanon->an_swslot = swblk; /* reassign */
742 simple_unlock(&pg->uanon->an_lock);
743 } else {
744 simple_lock(&pg->uobject->vmobjlock);
745 uao_set_swslot(pg->uobject,
746 pg->offset >> PAGE_SHIFT, swblk);
747 simple_unlock(&pg->uobject->vmobjlock);
748 }
749 }
750 }
751 }
752