uvm_anon.c revision 1.65 1 1.65 ad /* $NetBSD: uvm_anon.c,v 1.65 2019/12/01 14:40:31 ad Exp $ */
2 1.1 chuck
3 1.1 chuck /*
4 1.1 chuck * Copyright (c) 1997 Charles D. Cranor and Washington University.
5 1.1 chuck * All rights reserved.
6 1.1 chuck *
7 1.1 chuck * Redistribution and use in source and binary forms, with or without
8 1.1 chuck * modification, are permitted provided that the following conditions
9 1.1 chuck * are met:
10 1.1 chuck * 1. Redistributions of source code must retain the above copyright
11 1.1 chuck * notice, this list of conditions and the following disclaimer.
12 1.1 chuck * 2. Redistributions in binary form must reproduce the above copyright
13 1.1 chuck * notice, this list of conditions and the following disclaimer in the
14 1.1 chuck * documentation and/or other materials provided with the distribution.
15 1.1 chuck *
16 1.1 chuck * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 1.1 chuck * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 1.1 chuck * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 1.1 chuck * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 1.1 chuck * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 1.1 chuck * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 1.1 chuck * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 1.1 chuck * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 1.1 chuck * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 1.1 chuck * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 1.1 chuck */
27 1.1 chuck
28 1.1 chuck /*
29 1.1 chuck * uvm_anon.c: uvm anon ops
30 1.1 chuck */
31 1.21 lukem
32 1.21 lukem #include <sys/cdefs.h>
33 1.65 ad __KERNEL_RCSID(0, "$NetBSD: uvm_anon.c,v 1.65 2019/12/01 14:40:31 ad Exp $");
34 1.1 chuck
35 1.1 chuck #include "opt_uvmhist.h"
36 1.1 chuck
37 1.1 chuck #include <sys/param.h>
38 1.1 chuck #include <sys/systm.h>
39 1.1 chuck #include <sys/pool.h>
40 1.5 chs #include <sys/kernel.h>
41 1.1 chuck
42 1.1 chuck #include <uvm/uvm.h>
43 1.1 chuck #include <uvm/uvm_swap.h>
44 1.39 yamt #include <uvm/uvm_pdpolicy.h>
45 1.1 chuck
46 1.55 rmind static struct pool_cache uvm_anon_cache;
47 1.5 chs
48 1.55 rmind static int uvm_anon_ctor(void *, void *, int);
49 1.5 chs
50 1.1 chuck void
51 1.34 thorpej uvm_anon_init(void)
52 1.1 chuck {
53 1.5 chs
54 1.49 ad pool_cache_bootstrap(&uvm_anon_cache, sizeof(struct vm_anon), 0, 0,
55 1.49 ad PR_LARGECACHE, "anonpl", NULL, IPL_NONE, uvm_anon_ctor,
56 1.55 rmind NULL, NULL);
57 1.1 chuck }
58 1.1 chuck
59 1.33 yamt static int
60 1.41 yamt uvm_anon_ctor(void *arg, void *object, int flags)
61 1.1 chuck {
62 1.33 yamt struct vm_anon *anon = object;
63 1.1 chuck
64 1.33 yamt anon->an_ref = 0;
65 1.59 rmind anon->an_lock = NULL;
66 1.33 yamt anon->an_page = NULL;
67 1.37 yamt #if defined(VMSWAP)
68 1.33 yamt anon->an_swslot = 0;
69 1.55 rmind #endif
70 1.11 chs return 0;
71 1.1 chuck }
72 1.1 chuck
73 1.1 chuck /*
74 1.55 rmind * uvm_analloc: allocate a new anon.
75 1.13 thorpej *
76 1.55 rmind * => anon will have no lock associated.
77 1.1 chuck */
78 1.1 chuck struct vm_anon *
79 1.34 thorpej uvm_analloc(void)
80 1.1 chuck {
81 1.33 yamt struct vm_anon *anon;
82 1.1 chuck
83 1.47 ad anon = pool_cache_get(&uvm_anon_cache, PR_NOWAIT);
84 1.33 yamt if (anon) {
85 1.33 yamt KASSERT(anon->an_ref == 0);
86 1.59 rmind KASSERT(anon->an_lock == NULL);
87 1.33 yamt KASSERT(anon->an_page == NULL);
88 1.37 yamt #if defined(VMSWAP)
89 1.33 yamt KASSERT(anon->an_swslot == 0);
90 1.55 rmind #endif
91 1.33 yamt anon->an_ref = 1;
92 1.1 chuck }
93 1.33 yamt return anon;
94 1.1 chuck }
95 1.1 chuck
96 1.1 chuck /*
97 1.59 rmind * uvm_anon_dispose: free any resident page or swap resources of anon.
98 1.1 chuck *
99 1.55 rmind * => anon must be removed from the amap (if anon was in an amap).
100 1.59 rmind * => amap must be locked; we may drop and re-acquire the lock here.
101 1.1 chuck */
102 1.60 rmind static bool
103 1.59 rmind uvm_anon_dispose(struct vm_anon *anon)
104 1.1 chuck {
105 1.55 rmind struct vm_page *pg = anon->an_page;
106 1.55 rmind
107 1.59 rmind UVMHIST_FUNC("uvm_anon_dispose"); UVMHIST_CALLED(maphist);
108 1.64 pgoyette UVMHIST_LOG(maphist,"(anon=0x%#jx)", (uintptr_t)anon, 0,0,0);
109 1.1 chuck
110 1.59 rmind KASSERT(mutex_owned(anon->an_lock));
111 1.12 thorpej
112 1.1 chuck /*
113 1.55 rmind * If there is a resident page and it is loaned, then anon may not
114 1.55 rmind * own it. Call out to uvm_anon_lockloanpg() to identify and lock
115 1.55 rmind * the real owner of the page.
116 1.1 chuck */
117 1.1 chuck
118 1.22 chs if (pg && pg->loan_count) {
119 1.54 rmind KASSERT(anon->an_lock != NULL);
120 1.1 chuck pg = uvm_anon_lockloanpg(anon);
121 1.22 chs }
122 1.1 chuck
123 1.1 chuck /*
124 1.55 rmind * Dispose the page, if it is resident.
125 1.1 chuck */
126 1.1 chuck
127 1.1 chuck if (pg) {
128 1.54 rmind KASSERT(anon->an_lock != NULL);
129 1.1 chuck
130 1.1 chuck /*
131 1.55 rmind * If the page is owned by a UVM object (now locked),
132 1.55 rmind * then kill the loan on the page rather than free it,
133 1.55 rmind * and release the object lock.
134 1.1 chuck */
135 1.1 chuck
136 1.1 chuck if (pg->uobject) {
137 1.50 ad mutex_enter(&uvm_pageqlock);
138 1.10 chs KASSERT(pg->loan_count > 0);
139 1.1 chuck pg->loan_count--;
140 1.1 chuck pg->uanon = NULL;
141 1.50 ad mutex_exit(&uvm_pageqlock);
142 1.54 rmind mutex_exit(pg->uobject->vmobjlock);
143 1.1 chuck } else {
144 1.1 chuck
145 1.1 chuck /*
146 1.55 rmind * If page has no UVM object, then anon is the owner,
147 1.55 rmind * and it is already locked.
148 1.1 chuck */
149 1.1 chuck
150 1.18 chs KASSERT((pg->flags & PG_RELEASED) == 0);
151 1.18 chs pmap_page_protect(pg, VM_PROT_NONE);
152 1.29 yamt
153 1.29 yamt /*
154 1.55 rmind * If the page is busy, mark it as PG_RELEASED, so
155 1.55 rmind * that uvm_anon_release(9) would release it later.
156 1.29 yamt */
157 1.29 yamt
158 1.29 yamt if (pg->flags & PG_BUSY) {
159 1.29 yamt pg->flags |= PG_RELEASED;
160 1.58 yamt mutex_obj_hold(anon->an_lock);
161 1.60 rmind return false;
162 1.17 chs }
163 1.50 ad mutex_enter(&uvm_pageqlock);
164 1.29 yamt uvm_pagefree(pg);
165 1.50 ad mutex_exit(&uvm_pageqlock);
166 1.64 pgoyette UVMHIST_LOG(maphist, "anon 0x%#jx, page 0x%#jx: "
167 1.64 pgoyette "freed now!", (uintptr_t)anon, (uintptr_t)pg,
168 1.64 pgoyette 0, 0);
169 1.1 chuck }
170 1.1 chuck }
171 1.55 rmind
172 1.37 yamt #if defined(VMSWAP)
173 1.26 pk if (pg == NULL && anon->an_swslot > 0) {
174 1.55 rmind /* This page is no longer only in swap. */
175 1.20 chs KASSERT(uvmexp.swpgonly > 0);
176 1.65 ad atomic_dec_uint(&uvmexp.swpgonly);
177 1.20 chs }
178 1.55 rmind #endif
179 1.1 chuck
180 1.1 chuck /*
181 1.59 rmind * Free any swap resources, leave a page replacement hint.
182 1.1 chuck */
183 1.18 chs
184 1.2 chs uvm_anon_dropswap(anon);
185 1.39 yamt uvmpdpol_anfree(anon);
186 1.59 rmind UVMHIST_LOG(maphist,"<- done!",0,0,0,0);
187 1.60 rmind return true;
188 1.59 rmind }
189 1.59 rmind
190 1.59 rmind /*
191 1.59 rmind * uvm_anon_free: free a single anon.
192 1.59 rmind *
193 1.59 rmind * => anon must be already disposed.
194 1.59 rmind */
195 1.59 rmind void
196 1.59 rmind uvm_anon_free(struct vm_anon *anon)
197 1.59 rmind {
198 1.39 yamt
199 1.59 rmind KASSERT(anon->an_ref == 0);
200 1.59 rmind KASSERT(anon->an_lock == NULL);
201 1.33 yamt KASSERT(anon->an_page == NULL);
202 1.37 yamt #if defined(VMSWAP)
203 1.31 yamt KASSERT(anon->an_swslot == 0);
204 1.55 rmind #endif
205 1.47 ad pool_cache_put(&uvm_anon_cache, anon);
206 1.2 chs }
207 1.2 chs
208 1.55 rmind /*
209 1.59 rmind * uvm_anon_freelst: free a linked list of anon structures.
210 1.59 rmind *
211 1.59 rmind * => anon must be locked, we will unlock it.
212 1.55 rmind */
213 1.54 rmind void
214 1.59 rmind uvm_anon_freelst(struct vm_amap *amap, struct vm_anon *anonlst)
215 1.54 rmind {
216 1.61 yamt struct vm_anon *anon;
217 1.61 yamt struct vm_anon **anonp = &anonlst;
218 1.59 rmind
219 1.59 rmind KASSERT(mutex_owned(amap->am_lock));
220 1.61 yamt while ((anon = *anonp) != NULL) {
221 1.60 rmind if (!uvm_anon_dispose(anon)) {
222 1.60 rmind /* Do not free this anon. */
223 1.61 yamt *anonp = anon->an_link;
224 1.61 yamt /* Note: clears an_ref as well. */
225 1.61 yamt anon->an_link = NULL;
226 1.61 yamt } else {
227 1.61 yamt anonp = &anon->an_link;
228 1.60 rmind }
229 1.59 rmind }
230 1.59 rmind amap_unlock(amap);
231 1.59 rmind
232 1.59 rmind while (anonlst) {
233 1.59 rmind anon = anonlst->an_link;
234 1.59 rmind /* Note: clears an_ref as well. */
235 1.59 rmind anonlst->an_link = NULL;
236 1.59 rmind anonlst->an_lock = NULL;
237 1.59 rmind uvm_anon_free(anonlst);
238 1.59 rmind anonlst = anon;
239 1.54 rmind }
240 1.54 rmind }
241 1.54 rmind
242 1.2 chs /*
243 1.55 rmind * uvm_anon_lockloanpg: given a locked anon, lock its resident page owner.
244 1.1 chuck *
245 1.1 chuck * => anon is locked by caller
246 1.1 chuck * => on return: anon is locked
247 1.1 chuck * if there is a resident page:
248 1.1 chuck * if it has a uobject, it is locked by us
249 1.1 chuck * if it is ownerless, we take over as owner
250 1.1 chuck * we return the resident page (it can change during
251 1.1 chuck * this function)
252 1.1 chuck * => note that the only time an anon has an ownerless resident page
253 1.1 chuck * is if the page was loaned from a uvm_object and the uvm_object
254 1.1 chuck * disowned it
255 1.1 chuck * => this only needs to be called when you want to do an operation
256 1.1 chuck * on an anon's resident page and that page has a non-zero loan
257 1.1 chuck * count.
258 1.1 chuck */
259 1.1 chuck struct vm_page *
260 1.34 thorpej uvm_anon_lockloanpg(struct vm_anon *anon)
261 1.1 chuck {
262 1.1 chuck struct vm_page *pg;
263 1.43 thorpej bool locked = false;
264 1.1 chuck
265 1.54 rmind KASSERT(mutex_owned(anon->an_lock));
266 1.12 thorpej
267 1.1 chuck /*
268 1.1 chuck * loop while we have a resident page that has a non-zero loan count.
269 1.1 chuck * if we successfully get our lock, we will "break" the loop.
270 1.1 chuck * note that the test for pg->loan_count is not protected -- this
271 1.1 chuck * may produce false positive results. note that a false positive
272 1.1 chuck * result may cause us to do more work than we need to, but it will
273 1.1 chuck * not produce an incorrect result.
274 1.1 chuck */
275 1.1 chuck
276 1.33 yamt while (((pg = anon->an_page) != NULL) && pg->loan_count != 0) {
277 1.1 chuck
278 1.1 chuck /*
279 1.1 chuck * quickly check to see if the page has an object before
280 1.1 chuck * bothering to lock the page queues. this may also produce
281 1.1 chuck * a false positive result, but that's ok because we do a real
282 1.1 chuck * check after that.
283 1.1 chuck */
284 1.1 chuck
285 1.1 chuck if (pg->uobject) {
286 1.50 ad mutex_enter(&uvm_pageqlock);
287 1.18 chs if (pg->uobject) {
288 1.1 chuck locked =
289 1.54 rmind mutex_tryenter(pg->uobject->vmobjlock);
290 1.1 chuck } else {
291 1.1 chuck /* object disowned before we got PQ lock */
292 1.43 thorpej locked = true;
293 1.1 chuck }
294 1.50 ad mutex_exit(&uvm_pageqlock);
295 1.1 chuck
296 1.1 chuck /*
297 1.1 chuck * if we didn't get a lock (try lock failed), then we
298 1.1 chuck * toggle our anon lock and try again
299 1.1 chuck */
300 1.1 chuck
301 1.1 chuck if (!locked) {
302 1.1 chuck /*
303 1.1 chuck * someone locking the object has a chance to
304 1.1 chuck * lock us right now
305 1.54 rmind *
306 1.54 rmind * XXX Better than yielding but inadequate.
307 1.1 chuck */
308 1.54 rmind kpause("livelock", false, 1, anon->an_lock);
309 1.10 chs continue;
310 1.1 chuck }
311 1.1 chuck }
312 1.1 chuck
313 1.1 chuck /*
314 1.55 rmind * If page is un-owned i.e. the object dropped its ownership,
315 1.55 rmind * then we have to take the ownership.
316 1.1 chuck */
317 1.1 chuck
318 1.1 chuck if (pg->uobject == NULL && (pg->pqflags & PQ_ANON) == 0) {
319 1.50 ad mutex_enter(&uvm_pageqlock);
320 1.18 chs pg->pqflags |= PQ_ANON;
321 1.18 chs pg->loan_count--;
322 1.50 ad mutex_exit(&uvm_pageqlock);
323 1.1 chuck }
324 1.1 chuck break;
325 1.1 chuck }
326 1.55 rmind return pg;
327 1.5 chs }
328 1.5 chs
329 1.37 yamt #if defined(VMSWAP)
330 1.37 yamt
331 1.5 chs /*
332 1.55 rmind * uvm_anon_pagein: fetch an anon's page.
333 1.5 chs *
334 1.5 chs * => anon must be locked, and is unlocked upon return.
335 1.43 thorpej * => returns true if pagein was aborted due to lack of memory.
336 1.5 chs */
337 1.5 chs
338 1.42 thorpej bool
339 1.57 rmind uvm_anon_pagein(struct vm_amap *amap, struct vm_anon *anon)
340 1.5 chs {
341 1.5 chs struct vm_page *pg;
342 1.5 chs struct uvm_object *uobj;
343 1.8 thorpej
344 1.54 rmind KASSERT(mutex_owned(anon->an_lock));
345 1.57 rmind KASSERT(anon->an_lock == amap->am_lock);
346 1.12 thorpej
347 1.8 thorpej /*
348 1.55 rmind * Get the page of the anon.
349 1.8 thorpej */
350 1.5 chs
351 1.57 rmind switch (uvmfault_anonget(NULL, amap, anon)) {
352 1.16 chs case 0:
353 1.55 rmind /* Success - we have the page. */
354 1.55 rmind KASSERT(mutex_owned(anon->an_lock));
355 1.5 chs break;
356 1.16 chs case EIO:
357 1.16 chs case ERESTART:
358 1.5 chs /*
359 1.55 rmind * Nothing more to do on errors. ERESTART means that the
360 1.55 rmind * anon was freed.
361 1.5 chs */
362 1.43 thorpej return false;
363 1.25 pk default:
364 1.43 thorpej return true;
365 1.5 chs }
366 1.5 chs
367 1.5 chs /*
368 1.55 rmind * Mark the page as dirty, clear its swslot and un-busy it.
369 1.5 chs */
370 1.5 chs
371 1.33 yamt pg = anon->an_page;
372 1.5 chs uobj = pg->uobject;
373 1.55 rmind if (anon->an_swslot > 0) {
374 1.23 pk uvm_swap_free(anon->an_swslot, 1);
375 1.55 rmind }
376 1.5 chs anon->an_swslot = 0;
377 1.55 rmind pg->flags &= ~PG_CLEAN;
378 1.5 chs
379 1.5 chs /*
380 1.55 rmind * Deactivate the page (to put it on a page queue).
381 1.5 chs */
382 1.5 chs
383 1.50 ad mutex_enter(&uvm_pageqlock);
384 1.55 rmind if (pg->wire_count == 0) {
385 1.24 pk uvm_pagedeactivate(pg);
386 1.55 rmind }
387 1.50 ad mutex_exit(&uvm_pageqlock);
388 1.25 pk
389 1.25 pk if (pg->flags & PG_WANTED) {
390 1.55 rmind pg->flags &= ~PG_WANTED;
391 1.25 pk wakeup(pg);
392 1.25 pk }
393 1.5 chs
394 1.54 rmind mutex_exit(anon->an_lock);
395 1.5 chs if (uobj) {
396 1.54 rmind mutex_exit(uobj->vmobjlock);
397 1.5 chs }
398 1.43 thorpej return false;
399 1.1 chuck }
400 1.29 yamt
401 1.55 rmind /*
402 1.55 rmind * uvm_anon_dropswap: release any swap resources from this anon.
403 1.55 rmind *
404 1.55 rmind * => anon must be locked or have a reference count of 0.
405 1.55 rmind */
406 1.55 rmind void
407 1.55 rmind uvm_anon_dropswap(struct vm_anon *anon)
408 1.55 rmind {
409 1.55 rmind UVMHIST_FUNC("uvm_anon_dropswap"); UVMHIST_CALLED(maphist);
410 1.55 rmind
411 1.55 rmind if (anon->an_swslot == 0)
412 1.55 rmind return;
413 1.55 rmind
414 1.64 pgoyette UVMHIST_LOG(maphist,"freeing swap for anon %#jx, paged to swslot 0x%jx",
415 1.64 pgoyette (uintptr_t)anon, anon->an_swslot, 0, 0);
416 1.55 rmind uvm_swap_free(anon->an_swslot, 1);
417 1.55 rmind anon->an_swslot = 0;
418 1.55 rmind }
419 1.55 rmind
420 1.55 rmind #endif
421 1.37 yamt
422 1.29 yamt /*
423 1.29 yamt * uvm_anon_release: release an anon and its page.
424 1.29 yamt *
425 1.55 rmind * => anon should not have any references.
426 1.55 rmind * => anon must be locked.
427 1.29 yamt */
428 1.29 yamt
429 1.29 yamt void
430 1.34 thorpej uvm_anon_release(struct vm_anon *anon)
431 1.29 yamt {
432 1.33 yamt struct vm_page *pg = anon->an_page;
433 1.63 martin bool success __diagused;
434 1.29 yamt
435 1.59 rmind KASSERT(mutex_owned(anon->an_lock));
436 1.29 yamt KASSERT(pg != NULL);
437 1.29 yamt KASSERT((pg->flags & PG_RELEASED) != 0);
438 1.29 yamt KASSERT((pg->flags & PG_BUSY) != 0);
439 1.29 yamt KASSERT(pg->uobject == NULL);
440 1.29 yamt KASSERT(pg->uanon == anon);
441 1.29 yamt KASSERT(pg->loan_count == 0);
442 1.29 yamt KASSERT(anon->an_ref == 0);
443 1.29 yamt
444 1.50 ad mutex_enter(&uvm_pageqlock);
445 1.29 yamt uvm_pagefree(pg);
446 1.50 ad mutex_exit(&uvm_pageqlock);
447 1.62 yamt KASSERT(anon->an_page == NULL);
448 1.62 yamt /* dispose should succeed as no one can reach this anon anymore. */
449 1.62 yamt success = uvm_anon_dispose(anon);
450 1.62 yamt KASSERT(success);
451 1.59 rmind mutex_exit(anon->an_lock);
452 1.59 rmind /* Note: extra reference is held for PG_RELEASED case. */
453 1.62 yamt mutex_obj_free(anon->an_lock);
454 1.59 rmind anon->an_lock = NULL;
455 1.59 rmind uvm_anon_free(anon);
456 1.29 yamt }
457