uvm_object.c revision 1.24 1 1.24 chs /* $NetBSD: uvm_object.c,v 1.24 2020/08/14 09:06:15 chs Exp $ */
2 1.1 yamt
3 1.1 yamt /*
4 1.17 ad * Copyright (c) 2006, 2010, 2019 The NetBSD Foundation, Inc.
5 1.1 yamt * All rights reserved.
6 1.1 yamt *
7 1.3 rmind * This code is derived from software contributed to The NetBSD Foundation
8 1.3 rmind * by Mindaugas Rasiukevicius.
9 1.3 rmind *
10 1.1 yamt * Redistribution and use in source and binary forms, with or without
11 1.1 yamt * modification, are permitted provided that the following conditions
12 1.1 yamt * are met:
13 1.1 yamt * 1. Redistributions of source code must retain the above copyright
14 1.1 yamt * notice, this list of conditions and the following disclaimer.
15 1.1 yamt * 2. Redistributions in binary form must reproduce the above copyright
16 1.1 yamt * notice, this list of conditions and the following disclaimer in the
17 1.1 yamt * documentation and/or other materials provided with the distribution.
18 1.1 yamt *
19 1.1 yamt * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 1.1 yamt * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 1.1 yamt * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 1.1 yamt * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 1.1 yamt * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 1.1 yamt * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 1.1 yamt * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 1.1 yamt * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 1.1 yamt * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 1.1 yamt * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 1.1 yamt * POSSIBILITY OF SUCH DAMAGE.
30 1.1 yamt */
31 1.1 yamt
32 1.1 yamt /*
33 1.1 yamt * uvm_object.c: operate with memory objects
34 1.1 yamt *
35 1.1 yamt * TODO:
36 1.1 yamt * 1. Support PG_RELEASED-using objects
37 1.1 yamt */
38 1.1 yamt
39 1.1 yamt #include <sys/cdefs.h>
40 1.24 chs __KERNEL_RCSID(0, "$NetBSD: uvm_object.c,v 1.24 2020/08/14 09:06:15 chs Exp $");
41 1.1 yamt
42 1.13 pooka #ifdef _KERNEL_OPT
43 1.7 thorpej #include "opt_ddb.h"
44 1.13 pooka #endif
45 1.1 yamt
46 1.1 yamt #include <sys/param.h>
47 1.21 ad #include <sys/rwlock.h>
48 1.8 rmind #include <sys/queue.h>
49 1.1 yamt
50 1.1 yamt #include <uvm/uvm.h>
51 1.7 thorpej #include <uvm/uvm_ddb.h>
52 1.18 ad #include <uvm/uvm_page_array.h>
53 1.1 yamt
54 1.8 rmind /* Page count to fetch per single step. */
55 1.8 rmind #define FETCH_PAGECOUNT 16
56 1.8 rmind
57 1.8 rmind /*
58 1.8 rmind * uvm_obj_init: initialize UVM memory object.
59 1.8 rmind */
60 1.8 rmind void
61 1.8 rmind uvm_obj_init(struct uvm_object *uo, const struct uvm_pagerops *ops,
62 1.8 rmind bool alock, u_int refs)
63 1.8 rmind {
64 1.8 rmind
65 1.15 mrg #if 0 /* notyet */
66 1.14 mrg KASSERT(ops);
67 1.15 mrg #endif
68 1.8 rmind if (alock) {
69 1.8 rmind /* Allocate and assign a lock. */
70 1.21 ad uo->vmobjlock = rw_obj_alloc();
71 1.8 rmind } else {
72 1.8 rmind /* The lock will need to be set via uvm_obj_setlock(). */
73 1.8 rmind uo->vmobjlock = NULL;
74 1.8 rmind }
75 1.8 rmind uo->pgops = ops;
76 1.8 rmind LIST_INIT(&uo->uo_ubc);
77 1.8 rmind uo->uo_npages = 0;
78 1.8 rmind uo->uo_refs = refs;
79 1.17 ad radix_tree_init_tree(&uo->uo_pages);
80 1.8 rmind }
81 1.8 rmind
82 1.8 rmind /*
83 1.8 rmind * uvm_obj_destroy: destroy UVM memory object.
84 1.8 rmind */
85 1.8 rmind void
86 1.8 rmind uvm_obj_destroy(struct uvm_object *uo, bool dlock)
87 1.8 rmind {
88 1.8 rmind
89 1.17 ad KASSERT(radix_tree_empty_tree_p(&uo->uo_pages));
90 1.8 rmind
91 1.10 rmind /* Purge any UBC entries associated with this object. */
92 1.10 rmind ubc_purge(uo);
93 1.10 rmind
94 1.8 rmind /* Destroy the lock, if requested. */
95 1.8 rmind if (dlock) {
96 1.21 ad rw_obj_free(uo->vmobjlock);
97 1.8 rmind }
98 1.17 ad radix_tree_fini_tree(&uo->uo_pages);
99 1.8 rmind }
100 1.8 rmind
101 1.8 rmind /*
102 1.8 rmind * uvm_obj_setlock: assign a vmobjlock to the UVM object.
103 1.8 rmind *
104 1.8 rmind * => Caller is responsible to ensure that UVM objects is not use.
105 1.8 rmind * => Only dynamic lock may be previously set. We drop the reference then.
106 1.8 rmind */
107 1.8 rmind void
108 1.21 ad uvm_obj_setlock(struct uvm_object *uo, krwlock_t *lockptr)
109 1.8 rmind {
110 1.21 ad krwlock_t *olockptr = uo->vmobjlock;
111 1.8 rmind
112 1.8 rmind if (olockptr) {
113 1.8 rmind /* Drop the reference on the old lock. */
114 1.21 ad rw_obj_free(olockptr);
115 1.8 rmind }
116 1.8 rmind if (lockptr == NULL) {
117 1.8 rmind /* If new lock is not passed - allocate default one. */
118 1.21 ad lockptr = rw_obj_alloc();
119 1.8 rmind }
120 1.8 rmind uo->vmobjlock = lockptr;
121 1.8 rmind }
122 1.1 yamt
123 1.1 yamt /*
124 1.8 rmind * uvm_obj_wirepages: wire the pages of entire UVM object.
125 1.1 yamt *
126 1.1 yamt * => NOTE: this function should only be used for types of objects
127 1.1 yamt * where PG_RELEASED flag is never set (aobj objects)
128 1.1 yamt * => caller must pass page-aligned start and end values
129 1.1 yamt */
130 1.1 yamt int
131 1.11 christos uvm_obj_wirepages(struct uvm_object *uobj, off_t start, off_t end,
132 1.11 christos struct pglist *list)
133 1.1 yamt {
134 1.1 yamt int i, npages, error;
135 1.1 yamt struct vm_page *pgs[FETCH_PAGECOUNT], *pg = NULL;
136 1.1 yamt off_t offset = start, left;
137 1.1 yamt
138 1.1 yamt left = (end - start) >> PAGE_SHIFT;
139 1.1 yamt
140 1.21 ad rw_enter(uobj->vmobjlock, RW_WRITER);
141 1.1 yamt while (left) {
142 1.1 yamt
143 1.1 yamt npages = MIN(FETCH_PAGECOUNT, left);
144 1.1 yamt
145 1.1 yamt /* Get the pages */
146 1.1 yamt memset(pgs, 0, sizeof(pgs));
147 1.1 yamt error = (*uobj->pgops->pgo_get)(uobj, offset, pgs, &npages, 0,
148 1.1 yamt VM_PROT_READ | VM_PROT_WRITE, UVM_ADV_SEQUENTIAL,
149 1.22 ad PGO_SYNCIO);
150 1.1 yamt
151 1.1 yamt if (error)
152 1.1 yamt goto error;
153 1.1 yamt
154 1.21 ad rw_enter(uobj->vmobjlock, RW_WRITER);
155 1.1 yamt for (i = 0; i < npages; i++) {
156 1.1 yamt
157 1.1 yamt KASSERT(pgs[i] != NULL);
158 1.1 yamt KASSERT(!(pgs[i]->flags & PG_RELEASED));
159 1.1 yamt
160 1.1 yamt /*
161 1.1 yamt * Loan break
162 1.1 yamt */
163 1.1 yamt if (pgs[i]->loan_count) {
164 1.1 yamt while (pgs[i]->loan_count) {
165 1.1 yamt pg = uvm_loanbreak(pgs[i]);
166 1.1 yamt if (!pg) {
167 1.21 ad rw_exit(uobj->vmobjlock);
168 1.1 yamt uvm_wait("uobjwirepg");
169 1.21 ad rw_enter(uobj->vmobjlock, RW_WRITER);
170 1.1 yamt continue;
171 1.1 yamt }
172 1.1 yamt }
173 1.1 yamt pgs[i] = pg;
174 1.1 yamt }
175 1.1 yamt
176 1.16 ad if (pgs[i]->flags & PG_AOBJ) {
177 1.20 ad uvm_pagemarkdirty(pgs[i],
178 1.20 ad UVM_PAGE_STATUS_DIRTY);
179 1.1 yamt uao_dropswap(uobj, i);
180 1.1 yamt }
181 1.1 yamt }
182 1.1 yamt
183 1.1 yamt /* Wire the pages */
184 1.1 yamt for (i = 0; i < npages; i++) {
185 1.19 ad uvm_pagelock(pgs[i]);
186 1.1 yamt uvm_pagewire(pgs[i]);
187 1.19 ad uvm_pageunlock(pgs[i]);
188 1.11 christos if (list != NULL)
189 1.11 christos TAILQ_INSERT_TAIL(list, pgs[i], pageq.queue);
190 1.1 yamt }
191 1.1 yamt
192 1.1 yamt /* Unbusy the pages */
193 1.1 yamt uvm_page_unbusy(pgs, npages);
194 1.1 yamt
195 1.1 yamt left -= npages;
196 1.1 yamt offset += npages << PAGE_SHIFT;
197 1.1 yamt }
198 1.21 ad rw_exit(uobj->vmobjlock);
199 1.1 yamt
200 1.1 yamt return 0;
201 1.1 yamt
202 1.1 yamt error:
203 1.1 yamt /* Unwire the pages which has been wired */
204 1.8 rmind uvm_obj_unwirepages(uobj, start, offset);
205 1.1 yamt
206 1.1 yamt return error;
207 1.1 yamt }
208 1.1 yamt
209 1.1 yamt /*
210 1.8 rmind * uvm_obj_unwirepages: unwire the pages of entire UVM object.
211 1.1 yamt *
212 1.1 yamt * => NOTE: this function should only be used for types of objects
213 1.1 yamt * where PG_RELEASED flag is never set
214 1.1 yamt * => caller must pass page-aligned start and end values
215 1.1 yamt */
216 1.1 yamt void
217 1.8 rmind uvm_obj_unwirepages(struct uvm_object *uobj, off_t start, off_t end)
218 1.1 yamt {
219 1.1 yamt struct vm_page *pg;
220 1.1 yamt off_t offset;
221 1.1 yamt
222 1.21 ad rw_enter(uobj->vmobjlock, RW_WRITER);
223 1.1 yamt for (offset = start; offset < end; offset += PAGE_SIZE) {
224 1.1 yamt pg = uvm_pagelookup(uobj, offset);
225 1.1 yamt
226 1.1 yamt KASSERT(pg != NULL);
227 1.1 yamt KASSERT(!(pg->flags & PG_RELEASED));
228 1.1 yamt
229 1.19 ad uvm_pagelock(pg);
230 1.1 yamt uvm_pageunwire(pg);
231 1.19 ad uvm_pageunlock(pg);
232 1.1 yamt }
233 1.21 ad rw_exit(uobj->vmobjlock);
234 1.1 yamt }
235 1.7 thorpej
236 1.24 chs static inline bool
237 1.24 chs uvm_obj_notag_p(struct uvm_object *uobj, int tag)
238 1.24 chs {
239 1.24 chs
240 1.24 chs KASSERT(rw_lock_held(uobj->vmobjlock));
241 1.24 chs return radix_tree_empty_tagged_tree_p(&uobj->uo_pages, tag);
242 1.24 chs }
243 1.24 chs
244 1.24 chs bool
245 1.24 chs uvm_obj_clean_p(struct uvm_object *uobj)
246 1.24 chs {
247 1.24 chs
248 1.24 chs return uvm_obj_notag_p(uobj, UVM_PAGE_DIRTY_TAG);
249 1.24 chs }
250 1.24 chs
251 1.24 chs bool
252 1.24 chs uvm_obj_nowriteback_p(struct uvm_object *uobj)
253 1.24 chs {
254 1.24 chs
255 1.24 chs return uvm_obj_notag_p(uobj, UVM_PAGE_WRITEBACK_TAG);
256 1.24 chs }
257 1.24 chs
258 1.24 chs static inline bool
259 1.24 chs uvm_obj_page_tag_p(struct vm_page *pg, int tag)
260 1.24 chs {
261 1.24 chs struct uvm_object *uobj = pg->uobject;
262 1.24 chs int pgidx = pg->offset >> PAGE_SHIFT;
263 1.24 chs
264 1.24 chs KASSERT(uobj != NULL);
265 1.24 chs KASSERT(rw_lock_held(uobj->vmobjlock));
266 1.24 chs return radix_tree_get_tag(&uobj->uo_pages, pgidx, tag) != 0;
267 1.24 chs }
268 1.24 chs
269 1.24 chs static inline void
270 1.24 chs uvm_obj_page_set_tag(struct vm_page *pg, int tag)
271 1.24 chs {
272 1.24 chs struct uvm_object *uobj = pg->uobject;
273 1.24 chs int pgidx = pg->offset >> PAGE_SHIFT;
274 1.24 chs
275 1.24 chs KASSERT(uobj != NULL);
276 1.24 chs KASSERT(rw_write_held(uobj->vmobjlock));
277 1.24 chs radix_tree_set_tag(&uobj->uo_pages, pgidx, tag);
278 1.24 chs }
279 1.24 chs
280 1.24 chs static inline void
281 1.24 chs uvm_obj_page_clear_tag(struct vm_page *pg, int tag)
282 1.24 chs {
283 1.24 chs struct uvm_object *uobj = pg->uobject;
284 1.24 chs int pgidx = pg->offset >> PAGE_SHIFT;
285 1.24 chs
286 1.24 chs KASSERT(uobj != NULL);
287 1.24 chs KASSERT(rw_write_held(uobj->vmobjlock));
288 1.24 chs radix_tree_clear_tag(&uobj->uo_pages, pgidx, tag);
289 1.24 chs }
290 1.24 chs
291 1.24 chs bool
292 1.24 chs uvm_obj_page_dirty_p(struct vm_page *pg)
293 1.24 chs {
294 1.24 chs
295 1.24 chs return uvm_obj_page_tag_p(pg, UVM_PAGE_DIRTY_TAG);
296 1.24 chs }
297 1.24 chs
298 1.24 chs void
299 1.24 chs uvm_obj_page_set_dirty(struct vm_page *pg)
300 1.24 chs {
301 1.24 chs
302 1.24 chs uvm_obj_page_set_tag(pg, UVM_PAGE_DIRTY_TAG);
303 1.24 chs }
304 1.24 chs
305 1.24 chs void
306 1.24 chs uvm_obj_page_clear_dirty(struct vm_page *pg)
307 1.24 chs {
308 1.24 chs
309 1.24 chs uvm_obj_page_clear_tag(pg, UVM_PAGE_DIRTY_TAG);
310 1.24 chs }
311 1.24 chs
312 1.24 chs bool
313 1.24 chs uvm_obj_page_writeback_p(struct vm_page *pg)
314 1.24 chs {
315 1.24 chs
316 1.24 chs return uvm_obj_page_tag_p(pg, UVM_PAGE_WRITEBACK_TAG);
317 1.24 chs }
318 1.24 chs
319 1.24 chs void
320 1.24 chs uvm_obj_page_set_writeback(struct vm_page *pg)
321 1.24 chs {
322 1.24 chs
323 1.24 chs uvm_obj_page_set_tag(pg, UVM_PAGE_WRITEBACK_TAG);
324 1.24 chs }
325 1.24 chs
326 1.24 chs void
327 1.24 chs uvm_obj_page_clear_writeback(struct vm_page *pg)
328 1.24 chs {
329 1.24 chs
330 1.24 chs uvm_obj_page_clear_tag(pg, UVM_PAGE_WRITEBACK_TAG);
331 1.24 chs }
332 1.24 chs
333 1.12 pooka #if defined(DDB) || defined(DEBUGPRINT)
334 1.7 thorpej
335 1.7 thorpej /*
336 1.7 thorpej * uvm_object_printit: actually prints the object
337 1.7 thorpej */
338 1.7 thorpej void
339 1.7 thorpej uvm_object_printit(struct uvm_object *uobj, bool full,
340 1.7 thorpej void (*pr)(const char *, ...))
341 1.7 thorpej {
342 1.18 ad struct uvm_page_array a;
343 1.7 thorpej struct vm_page *pg;
344 1.7 thorpej int cnt = 0;
345 1.18 ad voff_t off;
346 1.7 thorpej
347 1.7 thorpej (*pr)("OBJECT %p: locked=%d, pgops=%p, npages=%d, ",
348 1.21 ad uobj, rw_write_held(uobj->vmobjlock), uobj->pgops, uobj->uo_npages);
349 1.7 thorpej if (UVM_OBJ_IS_KERN_OBJECT(uobj))
350 1.7 thorpej (*pr)("refs=<SYSTEM>\n");
351 1.7 thorpej else
352 1.7 thorpej (*pr)("refs=%d\n", uobj->uo_refs);
353 1.7 thorpej
354 1.7 thorpej if (!full) {
355 1.7 thorpej return;
356 1.7 thorpej }
357 1.7 thorpej (*pr)(" PAGES <pg,offset>:\n ");
358 1.23 ad uvm_page_array_init(&a, uobj, 0);
359 1.18 ad off = 0;
360 1.23 ad while ((pg = uvm_page_array_fill_and_peek(&a, off, 0)) != NULL) {
361 1.7 thorpej cnt++;
362 1.7 thorpej (*pr)("<%p,0x%llx> ", pg, (long long)pg->offset);
363 1.7 thorpej if ((cnt % 3) == 0) {
364 1.7 thorpej (*pr)("\n ");
365 1.7 thorpej }
366 1.18 ad off = pg->offset + PAGE_SIZE;
367 1.18 ad uvm_page_array_advance(&a);
368 1.7 thorpej }
369 1.7 thorpej if ((cnt % 3) != 0) {
370 1.7 thorpej (*pr)("\n");
371 1.7 thorpej }
372 1.18 ad uvm_page_array_fini(&a);
373 1.7 thorpej }
374 1.7 thorpej
375 1.7 thorpej #endif /* DDB || DEBUGPRINT */
376