uvm_anon.c revision 1.73 1 1.73 ad /* $NetBSD: uvm_anon.c,v 1.73 2020/02/23 21:36:29 ad Exp $ */
2 1.1 chuck
3 1.1 chuck /*
4 1.1 chuck * Copyright (c) 1997 Charles D. Cranor and Washington University.
5 1.1 chuck * All rights reserved.
6 1.1 chuck *
7 1.1 chuck * Redistribution and use in source and binary forms, with or without
8 1.1 chuck * modification, are permitted provided that the following conditions
9 1.1 chuck * are met:
10 1.1 chuck * 1. Redistributions of source code must retain the above copyright
11 1.1 chuck * notice, this list of conditions and the following disclaimer.
12 1.1 chuck * 2. Redistributions in binary form must reproduce the above copyright
13 1.1 chuck * notice, this list of conditions and the following disclaimer in the
14 1.1 chuck * documentation and/or other materials provided with the distribution.
15 1.1 chuck *
16 1.1 chuck * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 1.1 chuck * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 1.1 chuck * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 1.1 chuck * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 1.1 chuck * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 1.1 chuck * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 1.1 chuck * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 1.1 chuck * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 1.1 chuck * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 1.1 chuck * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 1.1 chuck */
27 1.1 chuck
28 1.1 chuck /*
29 1.1 chuck * uvm_anon.c: uvm anon ops
30 1.1 chuck */
31 1.21 lukem
32 1.21 lukem #include <sys/cdefs.h>
33 1.73 ad __KERNEL_RCSID(0, "$NetBSD: uvm_anon.c,v 1.73 2020/02/23 21:36:29 ad Exp $");
34 1.1 chuck
35 1.1 chuck #include "opt_uvmhist.h"
36 1.1 chuck
37 1.1 chuck #include <sys/param.h>
38 1.1 chuck #include <sys/systm.h>
39 1.1 chuck #include <sys/pool.h>
40 1.5 chs #include <sys/kernel.h>
41 1.67 uwe #include <sys/atomic.h>
42 1.1 chuck
43 1.1 chuck #include <uvm/uvm.h>
44 1.1 chuck #include <uvm/uvm_swap.h>
45 1.39 yamt #include <uvm/uvm_pdpolicy.h>
46 1.1 chuck
47 1.55 rmind static struct pool_cache uvm_anon_cache;
48 1.5 chs
49 1.55 rmind static int uvm_anon_ctor(void *, void *, int);
50 1.5 chs
51 1.1 chuck void
52 1.34 thorpej uvm_anon_init(void)
53 1.1 chuck {
54 1.5 chs
55 1.49 ad pool_cache_bootstrap(&uvm_anon_cache, sizeof(struct vm_anon), 0, 0,
56 1.49 ad PR_LARGECACHE, "anonpl", NULL, IPL_NONE, uvm_anon_ctor,
57 1.55 rmind NULL, NULL);
58 1.1 chuck }
59 1.1 chuck
60 1.33 yamt static int
61 1.41 yamt uvm_anon_ctor(void *arg, void *object, int flags)
62 1.1 chuck {
63 1.33 yamt struct vm_anon *anon = object;
64 1.1 chuck
65 1.33 yamt anon->an_ref = 0;
66 1.59 rmind anon->an_lock = NULL;
67 1.33 yamt anon->an_page = NULL;
68 1.37 yamt #if defined(VMSWAP)
69 1.33 yamt anon->an_swslot = 0;
70 1.55 rmind #endif
71 1.11 chs return 0;
72 1.1 chuck }
73 1.1 chuck
74 1.1 chuck /*
75 1.55 rmind * uvm_analloc: allocate a new anon.
76 1.13 thorpej *
77 1.55 rmind * => anon will have no lock associated.
78 1.1 chuck */
79 1.1 chuck struct vm_anon *
80 1.34 thorpej uvm_analloc(void)
81 1.1 chuck {
82 1.33 yamt struct vm_anon *anon;
83 1.1 chuck
84 1.47 ad anon = pool_cache_get(&uvm_anon_cache, PR_NOWAIT);
85 1.33 yamt if (anon) {
86 1.33 yamt KASSERT(anon->an_ref == 0);
87 1.59 rmind KASSERT(anon->an_lock == NULL);
88 1.33 yamt KASSERT(anon->an_page == NULL);
89 1.37 yamt #if defined(VMSWAP)
90 1.33 yamt KASSERT(anon->an_swslot == 0);
91 1.55 rmind #endif
92 1.33 yamt anon->an_ref = 1;
93 1.1 chuck }
94 1.33 yamt return anon;
95 1.1 chuck }
96 1.1 chuck
97 1.1 chuck /*
98 1.69 ad * uvm_anon_dispose: free any resident page or swap resources of anon.
99 1.1 chuck *
100 1.55 rmind * => anon must be removed from the amap (if anon was in an amap).
101 1.59 rmind * => amap must be locked; we may drop and re-acquire the lock here.
102 1.1 chuck */
103 1.60 rmind static bool
104 1.59 rmind uvm_anon_dispose(struct vm_anon *anon)
105 1.1 chuck {
106 1.55 rmind struct vm_page *pg = anon->an_page;
107 1.55 rmind
108 1.59 rmind UVMHIST_FUNC("uvm_anon_dispose"); UVMHIST_CALLED(maphist);
109 1.64 pgoyette UVMHIST_LOG(maphist,"(anon=0x%#jx)", (uintptr_t)anon, 0,0,0);
110 1.1 chuck
111 1.72 ad KASSERT(rw_write_held(anon->an_lock));
112 1.12 thorpej
113 1.1 chuck /*
114 1.66 ad * Dispose the page, if it is resident.
115 1.1 chuck */
116 1.1 chuck
117 1.66 ad if (pg) {
118 1.54 rmind KASSERT(anon->an_lock != NULL);
119 1.1 chuck
120 1.66 ad /*
121 1.66 ad * If there is a resident page and it is loaned, then anon
122 1.66 ad * may not own it. Call out to uvm_anon_lockloanpg() to
123 1.66 ad * identify and lock the real owner of the page.
124 1.66 ad */
125 1.1 chuck
126 1.66 ad if (pg->loan_count) {
127 1.66 ad pg = uvm_anon_lockloanpg(anon);
128 1.66 ad }
129 1.1 chuck
130 1.1 chuck /*
131 1.55 rmind * If the page is owned by a UVM object (now locked),
132 1.55 rmind * then kill the loan on the page rather than free it,
133 1.55 rmind * and release the object lock.
134 1.1 chuck */
135 1.1 chuck
136 1.1 chuck if (pg->uobject) {
137 1.69 ad mutex_enter(&pg->interlock);
138 1.10 chs KASSERT(pg->loan_count > 0);
139 1.1 chuck pg->loan_count--;
140 1.1 chuck pg->uanon = NULL;
141 1.69 ad mutex_exit(&pg->interlock);
142 1.72 ad rw_exit(pg->uobject->vmobjlock);
143 1.1 chuck } else {
144 1.1 chuck
145 1.1 chuck /*
146 1.55 rmind * If page has no UVM object, then anon is the owner,
147 1.55 rmind * and it is already locked.
148 1.1 chuck */
149 1.1 chuck
150 1.18 chs KASSERT((pg->flags & PG_RELEASED) == 0);
151 1.18 chs pmap_page_protect(pg, VM_PROT_NONE);
152 1.29 yamt
153 1.29 yamt /*
154 1.55 rmind * If the page is busy, mark it as PG_RELEASED, so
155 1.55 rmind * that uvm_anon_release(9) would release it later.
156 1.29 yamt */
157 1.29 yamt
158 1.29 yamt if (pg->flags & PG_BUSY) {
159 1.29 yamt pg->flags |= PG_RELEASED;
160 1.72 ad rw_obj_hold(anon->an_lock);
161 1.60 rmind return false;
162 1.17 chs }
163 1.69 ad uvm_pagefree(pg);
164 1.69 ad UVMHIST_LOG(maphist, "anon 0x%#jx, page 0x%#jx: "
165 1.69 ad "freed now!", (uintptr_t)anon, (uintptr_t)pg,
166 1.69 ad 0, 0);
167 1.1 chuck }
168 1.1 chuck }
169 1.55 rmind
170 1.37 yamt #if defined(VMSWAP)
171 1.26 pk if (pg == NULL && anon->an_swslot > 0) {
172 1.55 rmind /* This page is no longer only in swap. */
173 1.20 chs KASSERT(uvmexp.swpgonly > 0);
174 1.65 ad atomic_dec_uint(&uvmexp.swpgonly);
175 1.20 chs }
176 1.55 rmind #endif
177 1.1 chuck
178 1.69 ad /*
179 1.69 ad * Free any swap resources, leave a page replacement hint.
180 1.69 ad */
181 1.69 ad
182 1.69 ad uvm_anon_dropswap(anon);
183 1.69 ad uvmpdpol_anfree(anon);
184 1.59 rmind UVMHIST_LOG(maphist,"<- done!",0,0,0,0);
185 1.60 rmind return true;
186 1.59 rmind }
187 1.59 rmind
188 1.59 rmind /*
189 1.59 rmind * uvm_anon_free: free a single anon.
190 1.59 rmind *
191 1.59 rmind * => anon must be already disposed.
192 1.59 rmind */
193 1.59 rmind void
194 1.59 rmind uvm_anon_free(struct vm_anon *anon)
195 1.59 rmind {
196 1.39 yamt
197 1.59 rmind KASSERT(anon->an_ref == 0);
198 1.59 rmind KASSERT(anon->an_lock == NULL);
199 1.33 yamt KASSERT(anon->an_page == NULL);
200 1.37 yamt #if defined(VMSWAP)
201 1.31 yamt KASSERT(anon->an_swslot == 0);
202 1.55 rmind #endif
203 1.47 ad pool_cache_put(&uvm_anon_cache, anon);
204 1.2 chs }
205 1.2 chs
206 1.55 rmind /*
207 1.59 rmind * uvm_anon_freelst: free a linked list of anon structures.
208 1.59 rmind *
209 1.66 ad * => amap must be locked, we will unlock it.
210 1.55 rmind */
211 1.54 rmind void
212 1.59 rmind uvm_anon_freelst(struct vm_amap *amap, struct vm_anon *anonlst)
213 1.54 rmind {
214 1.69 ad struct vm_anon *next;
215 1.59 rmind
216 1.68 chs UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
217 1.68 chs
218 1.72 ad KASSERT(rw_write_held(amap->am_lock));
219 1.66 ad
220 1.69 ad for (; anonlst != NULL; anonlst = next) {
221 1.69 ad next = anonlst->an_link;
222 1.69 ad /* Note: clears an_ref as well. */
223 1.69 ad anonlst->an_link = NULL;
224 1.69 ad if (uvm_anon_dispose(anonlst)) {
225 1.69 ad anonlst->an_lock = NULL;
226 1.69 ad uvm_anon_free(anonlst);
227 1.60 rmind }
228 1.59 rmind }
229 1.59 rmind amap_unlock(amap);
230 1.54 rmind }
231 1.54 rmind
232 1.2 chs /*
233 1.55 rmind * uvm_anon_lockloanpg: given a locked anon, lock its resident page owner.
234 1.1 chuck *
235 1.1 chuck * => anon is locked by caller
236 1.1 chuck * => on return: anon is locked
237 1.1 chuck * if there is a resident page:
238 1.1 chuck * if it has a uobject, it is locked by us
239 1.1 chuck * if it is ownerless, we take over as owner
240 1.1 chuck * we return the resident page (it can change during
241 1.1 chuck * this function)
242 1.1 chuck * => note that the only time an anon has an ownerless resident page
243 1.1 chuck * is if the page was loaned from a uvm_object and the uvm_object
244 1.1 chuck * disowned it
245 1.1 chuck * => this only needs to be called when you want to do an operation
246 1.1 chuck * on an anon's resident page and that page has a non-zero loan
247 1.1 chuck * count.
248 1.1 chuck */
249 1.1 chuck struct vm_page *
250 1.34 thorpej uvm_anon_lockloanpg(struct vm_anon *anon)
251 1.1 chuck {
252 1.1 chuck struct vm_page *pg;
253 1.72 ad krw_t op;
254 1.1 chuck
255 1.72 ad KASSERT(rw_lock_held(anon->an_lock));
256 1.12 thorpej
257 1.1 chuck /*
258 1.1 chuck * loop while we have a resident page that has a non-zero loan count.
259 1.1 chuck * if we successfully get our lock, we will "break" the loop.
260 1.1 chuck * note that the test for pg->loan_count is not protected -- this
261 1.1 chuck * may produce false positive results. note that a false positive
262 1.1 chuck * result may cause us to do more work than we need to, but it will
263 1.1 chuck * not produce an incorrect result.
264 1.1 chuck */
265 1.1 chuck
266 1.33 yamt while (((pg = anon->an_page) != NULL) && pg->loan_count != 0) {
267 1.69 ad mutex_enter(&pg->interlock);
268 1.1 chuck if (pg->uobject) {
269 1.1 chuck /*
270 1.1 chuck * if we didn't get a lock (try lock failed), then we
271 1.1 chuck * toggle our anon lock and try again
272 1.1 chuck */
273 1.1 chuck
274 1.72 ad if (!rw_tryenter(pg->uobject->vmobjlock, RW_WRITER)) {
275 1.1 chuck /*
276 1.1 chuck * someone locking the object has a chance to
277 1.1 chuck * lock us right now
278 1.54 rmind *
279 1.54 rmind * XXX Better than yielding but inadequate.
280 1.1 chuck */
281 1.69 ad mutex_exit(&pg->interlock);
282 1.73 ad op = rw_lock_op(anon->an_lock);
283 1.72 ad rw_exit(anon->an_lock);
284 1.72 ad kpause("lkloanpg", false, 1, NULL);
285 1.72 ad rw_enter(anon->an_lock, op);
286 1.10 chs continue;
287 1.1 chuck }
288 1.1 chuck }
289 1.1 chuck
290 1.1 chuck /*
291 1.55 rmind * If page is un-owned i.e. the object dropped its ownership,
292 1.55 rmind * then we have to take the ownership.
293 1.1 chuck */
294 1.1 chuck
295 1.69 ad if (pg->uobject == NULL && (pg->flags & PG_ANON) == 0) {
296 1.69 ad pg->flags |= PG_ANON;
297 1.18 chs pg->loan_count--;
298 1.1 chuck }
299 1.69 ad mutex_exit(&pg->interlock);
300 1.1 chuck break;
301 1.1 chuck }
302 1.55 rmind return pg;
303 1.5 chs }
304 1.5 chs
305 1.37 yamt #if defined(VMSWAP)
306 1.37 yamt
307 1.5 chs /*
308 1.55 rmind * uvm_anon_pagein: fetch an anon's page.
309 1.5 chs *
310 1.5 chs * => anon must be locked, and is unlocked upon return.
311 1.43 thorpej * => returns true if pagein was aborted due to lack of memory.
312 1.5 chs */
313 1.5 chs
314 1.42 thorpej bool
315 1.57 rmind uvm_anon_pagein(struct vm_amap *amap, struct vm_anon *anon)
316 1.5 chs {
317 1.5 chs struct vm_page *pg;
318 1.5 chs struct uvm_object *uobj;
319 1.8 thorpej
320 1.72 ad KASSERT(rw_write_held(anon->an_lock));
321 1.57 rmind KASSERT(anon->an_lock == amap->am_lock);
322 1.12 thorpej
323 1.8 thorpej /*
324 1.55 rmind * Get the page of the anon.
325 1.8 thorpej */
326 1.5 chs
327 1.57 rmind switch (uvmfault_anonget(NULL, amap, anon)) {
328 1.16 chs case 0:
329 1.55 rmind /* Success - we have the page. */
330 1.72 ad KASSERT(rw_write_held(anon->an_lock));
331 1.5 chs break;
332 1.16 chs case EIO:
333 1.16 chs case ERESTART:
334 1.5 chs /*
335 1.55 rmind * Nothing more to do on errors. ERESTART means that the
336 1.55 rmind * anon was freed.
337 1.5 chs */
338 1.43 thorpej return false;
339 1.25 pk default:
340 1.43 thorpej return true;
341 1.5 chs }
342 1.5 chs
343 1.5 chs /*
344 1.55 rmind * Mark the page as dirty, clear its swslot and un-busy it.
345 1.5 chs */
346 1.5 chs
347 1.33 yamt pg = anon->an_page;
348 1.5 chs uobj = pg->uobject;
349 1.55 rmind if (anon->an_swslot > 0) {
350 1.23 pk uvm_swap_free(anon->an_swslot, 1);
351 1.55 rmind }
352 1.5 chs anon->an_swslot = 0;
353 1.71 ad uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY);
354 1.5 chs
355 1.5 chs /*
356 1.55 rmind * Deactivate the page (to put it on a page queue).
357 1.5 chs */
358 1.5 chs
359 1.70 ad uvm_pagelock(pg);
360 1.69 ad uvm_pagedeactivate(pg);
361 1.70 ad uvm_pageunlock(pg);
362 1.25 pk if (pg->flags & PG_WANTED) {
363 1.55 rmind pg->flags &= ~PG_WANTED;
364 1.25 pk wakeup(pg);
365 1.25 pk }
366 1.5 chs
367 1.72 ad rw_exit(anon->an_lock);
368 1.5 chs if (uobj) {
369 1.72 ad rw_exit(uobj->vmobjlock);
370 1.5 chs }
371 1.43 thorpej return false;
372 1.1 chuck }
373 1.29 yamt
374 1.55 rmind /*
375 1.55 rmind * uvm_anon_dropswap: release any swap resources from this anon.
376 1.55 rmind *
377 1.55 rmind * => anon must be locked or have a reference count of 0.
378 1.55 rmind */
379 1.55 rmind void
380 1.55 rmind uvm_anon_dropswap(struct vm_anon *anon)
381 1.55 rmind {
382 1.55 rmind UVMHIST_FUNC("uvm_anon_dropswap"); UVMHIST_CALLED(maphist);
383 1.55 rmind
384 1.55 rmind if (anon->an_swslot == 0)
385 1.55 rmind return;
386 1.55 rmind
387 1.64 pgoyette UVMHIST_LOG(maphist,"freeing swap for anon %#jx, paged to swslot 0x%jx",
388 1.64 pgoyette (uintptr_t)anon, anon->an_swslot, 0, 0);
389 1.55 rmind uvm_swap_free(anon->an_swslot, 1);
390 1.55 rmind anon->an_swslot = 0;
391 1.55 rmind }
392 1.55 rmind
393 1.55 rmind #endif
394 1.37 yamt
395 1.29 yamt /*
396 1.29 yamt * uvm_anon_release: release an anon and its page.
397 1.29 yamt *
398 1.55 rmind * => anon should not have any references.
399 1.55 rmind * => anon must be locked.
400 1.29 yamt */
401 1.29 yamt
402 1.29 yamt void
403 1.34 thorpej uvm_anon_release(struct vm_anon *anon)
404 1.29 yamt {
405 1.33 yamt struct vm_page *pg = anon->an_page;
406 1.63 martin bool success __diagused;
407 1.29 yamt
408 1.72 ad KASSERT(rw_write_held(anon->an_lock));
409 1.29 yamt KASSERT(pg != NULL);
410 1.29 yamt KASSERT((pg->flags & PG_RELEASED) != 0);
411 1.29 yamt KASSERT((pg->flags & PG_BUSY) != 0);
412 1.29 yamt KASSERT(pg->uobject == NULL);
413 1.29 yamt KASSERT(pg->uanon == anon);
414 1.29 yamt KASSERT(pg->loan_count == 0);
415 1.29 yamt KASSERT(anon->an_ref == 0);
416 1.29 yamt
417 1.29 yamt uvm_pagefree(pg);
418 1.62 yamt KASSERT(anon->an_page == NULL);
419 1.62 yamt /* dispose should succeed as no one can reach this anon anymore. */
420 1.62 yamt success = uvm_anon_dispose(anon);
421 1.62 yamt KASSERT(success);
422 1.72 ad rw_exit(anon->an_lock);
423 1.59 rmind /* Note: extra reference is held for PG_RELEASED case. */
424 1.72 ad rw_obj_free(anon->an_lock);
425 1.59 rmind anon->an_lock = NULL;
426 1.59 rmind uvm_anon_free(anon);
427 1.29 yamt }
428