1 /* $NetBSD: uvm_object.c,v 1.26 2026/08/23 22:08:41 riastradh Exp $ */ 2 3 /* 4 * Copyright (c) 2006, 2010, 2019 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Mindaugas Rasiukevicius. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * uvm_object.c: operate with memory objects 34 * 35 * TODO: 36 * 1. Support PG_RELEASED-using objects 37 */ 38 39 #include <sys/cdefs.h> 40 __KERNEL_RCSID(0, "$NetBSD: uvm_object.c,v 1.26 2026/08/23 22:08:41 riastradh Exp $"); 41 42 #ifdef _KERNEL_OPT 43 #include "opt_ddb.h" 44 #endif 45 46 #include <sys/param.h> 47 #include <sys/rwlock.h> 48 #include <sys/queue.h> 49 50 #include <uvm/uvm.h> 51 #include <uvm/uvm_ddb.h> 52 #include <uvm/uvm_page_array.h> 53 54 /* Page count to fetch per single step. */ 55 #define FETCH_PAGECOUNT 16 56 57 /* 58 * uvm_obj_init: initialize UVM memory object. 59 */ 60 void 61 uvm_obj_init(struct uvm_object *uo, const struct uvm_pagerops *ops, 62 bool alock, u_int refs) 63 { 64 65 #if 0 /* notyet */ 66 KASSERT(ops); 67 #endif 68 if (alock) { 69 /* Allocate and assign a lock. */ 70 uo->vmobjlock = rw_obj_alloc(); 71 } else { 72 /* The lock will need to be set via uvm_obj_setlock(). */ 73 uo->vmobjlock = NULL; 74 } 75 uo->pgops = ops; 76 LIST_INIT(&uo->uo_ubc); 77 uo->uo_npages = 0; 78 uo->uo_refs = refs; 79 radix_tree_init_tree(&uo->uo_pages); 80 } 81 82 /* 83 * uvm_obj_destroy: destroy UVM memory object. 84 */ 85 void 86 uvm_obj_destroy(struct uvm_object *uo, bool dlock) 87 { 88 89 KASSERT(radix_tree_empty_tree_p(&uo->uo_pages)); 90 91 /* Purge any UBC entries associated with this object. */ 92 ubc_purge(uo); 93 94 /* Destroy the lock, if requested. */ 95 if (dlock) { 96 rw_obj_free(uo->vmobjlock); 97 } 98 radix_tree_fini_tree(&uo->uo_pages); 99 } 100 101 /* 102 * uvm_obj_setlock: assign a vmobjlock to the UVM object. 103 * 104 * => Caller is responsible to ensure that UVM objects is not use. 105 * => Only dynamic lock may be previously set. We drop the reference then. 106 */ 107 void 108 uvm_obj_setlock(struct uvm_object *uo, krwlock_t *lockptr) 109 { 110 krwlock_t *olockptr = uo->vmobjlock; 111 112 if (olockptr) { 113 /* Drop the reference on the old lock. */ 114 rw_obj_free(olockptr); 115 } 116 if (lockptr == NULL) { 117 /* If new lock is not passed - allocate default one. */ 118 lockptr = rw_obj_alloc(); 119 } 120 uo->vmobjlock = lockptr; 121 } 122 123 /* 124 * uvm_obj_wirepages: wire the pages of entire UVM object. 125 * 126 * => NOTE: this function should only be used for types of objects 127 * where PG_RELEASED flag is never set (aobj objects) 128 * => caller must pass page-aligned start and end values 129 */ 130 int 131 uvm_obj_wirepages(struct uvm_object *uobj, off_t start, off_t end, 132 struct pglist *list) 133 { 134 int i, npages, error; 135 struct vm_page *pgs[FETCH_PAGECOUNT], *pg = NULL; 136 off_t offset = start, left; 137 138 left = (end - start) >> PAGE_SHIFT; 139 140 rw_enter(uobj->vmobjlock, RW_WRITER); 141 while (left) { 142 143 npages = MIN(FETCH_PAGECOUNT, left); 144 145 /* Get the pages */ 146 memset(pgs, 0, sizeof(pgs)); 147 error = (*uobj->pgops->pgo_get)(uobj, offset, pgs, &npages, 0, 148 VM_PROT_READ | VM_PROT_WRITE, UVM_ADV_SEQUENTIAL, 149 PGO_SYNCIO); 150 151 if (error) 152 goto error; 153 154 rw_enter(uobj->vmobjlock, RW_WRITER); 155 for (i = 0; i < npages; i++) { 156 157 KASSERT(pgs[i] != NULL); 158 KASSERT(!(pgs[i]->flags & PG_RELEASED)); 159 160 /* 161 * Loan break 162 */ 163 if (pgs[i]->loan_count) { 164 while (pgs[i]->loan_count) { 165 uint64_t ticket = 166 uvm_wait_prepare(); 167 168 pg = uvm_loanbreak(pgs[i]); 169 if (!pg) { 170 rw_exit(uobj->vmobjlock); 171 uvm_wait("uobjwirepg", ticket); 172 rw_enter(uobj->vmobjlock, 173 RW_WRITER); 174 continue; 175 } 176 } 177 pgs[i] = pg; 178 } 179 180 if (pgs[i]->flags & PG_AOBJ) { 181 uvm_pagemarkdirty(pgs[i], 182 UVM_PAGE_STATUS_DIRTY); 183 uao_dropswap(uobj, i); 184 } 185 } 186 187 /* Wire the pages */ 188 for (i = 0; i < npages; i++) { 189 uvm_pagelock(pgs[i]); 190 uvm_pagewire(pgs[i]); 191 uvm_pageunlock(pgs[i]); 192 if (list != NULL) 193 TAILQ_INSERT_TAIL(list, pgs[i], pageq.queue); 194 } 195 196 /* Unbusy the pages */ 197 uvm_page_unbusy(pgs, npages); 198 199 left -= npages; 200 offset += npages << PAGE_SHIFT; 201 } 202 rw_exit(uobj->vmobjlock); 203 204 return 0; 205 206 error: 207 /* Unwire the pages which has been wired */ 208 uvm_obj_unwirepages(uobj, start, offset); 209 210 return error; 211 } 212 213 /* 214 * uvm_obj_unwirepages: unwire the pages of entire UVM object. 215 * 216 * => NOTE: this function should only be used for types of objects 217 * where PG_RELEASED flag is never set 218 * => caller must pass page-aligned start and end values 219 */ 220 void 221 uvm_obj_unwirepages(struct uvm_object *uobj, off_t start, off_t end) 222 { 223 struct vm_page *pg; 224 off_t offset; 225 226 rw_enter(uobj->vmobjlock, RW_WRITER); 227 for (offset = start; offset < end; offset += PAGE_SIZE) { 228 pg = uvm_pagelookup(uobj, offset); 229 230 KASSERT(pg != NULL); 231 KASSERT(!(pg->flags & PG_RELEASED)); 232 233 uvm_pagelock(pg); 234 uvm_pageunwire(pg); 235 uvm_pageunlock(pg); 236 } 237 rw_exit(uobj->vmobjlock); 238 } 239 240 static inline bool 241 uvm_obj_notag_p(struct uvm_object *uobj, int tag) 242 { 243 244 KASSERT(rw_lock_held(uobj->vmobjlock)); 245 return radix_tree_empty_tagged_tree_p(&uobj->uo_pages, tag); 246 } 247 248 bool 249 uvm_obj_clean_p(struct uvm_object *uobj) 250 { 251 252 return uvm_obj_notag_p(uobj, UVM_PAGE_DIRTY_TAG); 253 } 254 255 bool 256 uvm_obj_nowriteback_p(struct uvm_object *uobj) 257 { 258 259 return uvm_obj_notag_p(uobj, UVM_PAGE_WRITEBACK_TAG); 260 } 261 262 static inline bool 263 uvm_obj_page_tag_p(struct vm_page *pg, int tag) 264 { 265 struct uvm_object *uobj = pg->uobject; 266 uint64_t pgidx = pg->offset >> PAGE_SHIFT; 267 268 KASSERT(uobj != NULL); 269 KASSERT(rw_lock_held(uobj->vmobjlock)); 270 return radix_tree_get_tag(&uobj->uo_pages, pgidx, tag) != 0; 271 } 272 273 static inline void 274 uvm_obj_page_set_tag(struct vm_page *pg, int tag) 275 { 276 struct uvm_object *uobj = pg->uobject; 277 uint64_t pgidx = pg->offset >> PAGE_SHIFT; 278 279 KASSERT(uobj != NULL); 280 KASSERT(rw_write_held(uobj->vmobjlock)); 281 radix_tree_set_tag(&uobj->uo_pages, pgidx, tag); 282 } 283 284 static inline void 285 uvm_obj_page_clear_tag(struct vm_page *pg, int tag) 286 { 287 struct uvm_object *uobj = pg->uobject; 288 uint64_t pgidx = pg->offset >> PAGE_SHIFT; 289 290 KASSERT(uobj != NULL); 291 KASSERT(rw_write_held(uobj->vmobjlock)); 292 radix_tree_clear_tag(&uobj->uo_pages, pgidx, tag); 293 } 294 295 bool 296 uvm_obj_page_dirty_p(struct vm_page *pg) 297 { 298 299 return uvm_obj_page_tag_p(pg, UVM_PAGE_DIRTY_TAG); 300 } 301 302 void 303 uvm_obj_page_set_dirty(struct vm_page *pg) 304 { 305 306 uvm_obj_page_set_tag(pg, UVM_PAGE_DIRTY_TAG); 307 } 308 309 void 310 uvm_obj_page_clear_dirty(struct vm_page *pg) 311 { 312 313 uvm_obj_page_clear_tag(pg, UVM_PAGE_DIRTY_TAG); 314 } 315 316 bool 317 uvm_obj_page_writeback_p(struct vm_page *pg) 318 { 319 320 return uvm_obj_page_tag_p(pg, UVM_PAGE_WRITEBACK_TAG); 321 } 322 323 void 324 uvm_obj_page_set_writeback(struct vm_page *pg) 325 { 326 327 uvm_obj_page_set_tag(pg, UVM_PAGE_WRITEBACK_TAG); 328 } 329 330 void 331 uvm_obj_page_clear_writeback(struct vm_page *pg) 332 { 333 334 uvm_obj_page_clear_tag(pg, UVM_PAGE_WRITEBACK_TAG); 335 } 336 337 #if defined(DDB) || defined(DEBUGPRINT) 338 339 /* 340 * uvm_object_printit: actually prints the object 341 */ 342 void 343 uvm_object_printit(struct uvm_object *uobj, bool full, 344 void (*pr)(const char *, ...)) 345 { 346 struct uvm_page_array a; 347 struct vm_page *pg; 348 int cnt = 0; 349 voff_t off; 350 351 (*pr)("OBJECT %p: locked=%d, pgops=%p, npages=%d, ", 352 uobj, rw_write_held(uobj->vmobjlock), uobj->pgops, uobj->uo_npages); 353 if (UVM_OBJ_IS_KERN_OBJECT(uobj)) 354 (*pr)("refs=<SYSTEM>\n"); 355 else 356 (*pr)("refs=%d\n", uobj->uo_refs); 357 358 if (!full) { 359 return; 360 } 361 (*pr)(" PAGES <pg,offset>:\n "); 362 uvm_page_array_init(&a, uobj, 0); 363 off = 0; 364 while ((pg = uvm_page_array_fill_and_peek(&a, off, 0)) != NULL) { 365 cnt++; 366 (*pr)("<%p,0x%llx> ", pg, (long long)pg->offset); 367 if ((cnt % 3) == 0) { 368 (*pr)("\n "); 369 } 370 off = pg->offset + PAGE_SIZE; 371 uvm_page_array_advance(&a); 372 } 373 if ((cnt % 3) != 0) { 374 (*pr)("\n"); 375 } 376 uvm_page_array_fini(&a); 377 } 378 379 #endif /* DDB || DEBUGPRINT */ 380
Indexes created Fri Sep 25 00:27:33 UTC 2026