uvm_map.c revision 1.40 1 /* $NetBSD: uvm_map.c,v 1.40 1999/05/20 23:03:23 thorpej Exp $ */
2
3 /*
4 * Copyright (c) 1997 Charles D. Cranor and Washington University.
5 * Copyright (c) 1991, 1993, The Regents of the University of California.
6 *
7 * All rights reserved.
8 *
9 * This code is derived from software contributed to Berkeley by
10 * The Mach Operating System project at Carnegie-Mellon University.
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 * 1. Redistributions of source code must retain the above copyright
16 * notice, this list of conditions and the following disclaimer.
17 * 2. Redistributions in binary form must reproduce the above copyright
18 * notice, this list of conditions and the following disclaimer in the
19 * documentation and/or other materials provided with the distribution.
20 * 3. All advertising materials mentioning features or use of this software
21 * must display the following acknowledgement:
22 * This product includes software developed by Charles D. Cranor,
23 * Washington University, the University of California, Berkeley and
24 * its contributors.
25 * 4. Neither the name of the University nor the names of its contributors
26 * may be used to endorse or promote products derived from this software
27 * without specific prior written permission.
28 *
29 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
30 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
31 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
32 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
33 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
34 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
35 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
36 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
37 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
38 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
39 * SUCH DAMAGE.
40 *
41 * @(#)vm_map.c 8.3 (Berkeley) 1/12/94
42 * from: Id: uvm_map.c,v 1.1.2.27 1998/02/07 01:16:54 chs Exp
43 *
44 *
45 * Copyright (c) 1987, 1990 Carnegie-Mellon University.
46 * All rights reserved.
47 *
48 * Permission to use, copy, modify and distribute this software and
49 * its documentation is hereby granted, provided that both the copyright
50 * notice and this permission notice appear in all copies of the
51 * software, derivative works or modified versions, and any portions
52 * thereof, and that both notices appear in supporting documentation.
53 *
54 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
55 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
56 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
57 *
58 * Carnegie Mellon requests users of this software to return to
59 *
60 * Software Distribution Coordinator or Software.Distribution (at) CS.CMU.EDU
61 * School of Computer Science
62 * Carnegie Mellon University
63 * Pittsburgh PA 15213-3890
64 *
65 * any improvements or extensions that they make and grant Carnegie the
66 * rights to redistribute these changes.
67 */
68
69 #include "opt_ddb.h"
70 #include "opt_uvmhist.h"
71 #include "opt_pmap_new.h"
72 #include "opt_sysv.h"
73
74 /*
75 * uvm_map.c: uvm map operations
76 */
77
78 #include <sys/param.h>
79 #include <sys/systm.h>
80 #include <sys/mman.h>
81 #include <sys/proc.h>
82 #include <sys/malloc.h>
83 #include <sys/pool.h>
84
85 #ifdef SYSVSHM
86 #include <sys/shm.h>
87 #endif
88
89 #include <vm/vm.h>
90 #include <vm/vm_page.h>
91 #include <vm/vm_kern.h>
92
93 #define UVM_MAP
94 #include <uvm/uvm.h>
95
96 #ifdef DDB
97 #include <uvm/uvm_ddb.h>
98 #endif
99
100
101 struct uvm_cnt uvm_map_call, map_backmerge, map_forwmerge;
102 struct uvm_cnt uvm_mlk_call, uvm_mlk_hint;
103
104 /*
105 * pool for vmspace structures.
106 */
107
108 struct pool uvm_vmspace_pool;
109
110 /*
111 * pool for dynamically-allocated map entries.
112 */
113
114 struct pool uvm_map_entry_pool;
115
116 #ifdef PMAP_GROWKERNEL
117 /*
118 * This global represents the end of the kernel virtual address
119 * space. If we want to exceed this, we must grow the kernel
120 * virtual address space dynamically.
121 *
122 * Note, this variable is locked by kernel_map's lock.
123 */
124 vaddr_t uvm_maxkaddr;
125 #endif
126
127 /*
128 * macros
129 */
130
131 /*
132 * uvm_map_entry_link: insert entry into a map
133 *
134 * => map must be locked
135 */
136 #define uvm_map_entry_link(map, after_where, entry) do { \
137 (map)->nentries++; \
138 (entry)->prev = (after_where); \
139 (entry)->next = (after_where)->next; \
140 (entry)->prev->next = (entry); \
141 (entry)->next->prev = (entry); \
142 } while (0)
143
144 /*
145 * uvm_map_entry_unlink: remove entry from a map
146 *
147 * => map must be locked
148 */
149 #define uvm_map_entry_unlink(map, entry) do { \
150 (map)->nentries--; \
151 (entry)->next->prev = (entry)->prev; \
152 (entry)->prev->next = (entry)->next; \
153 } while (0)
154
155 /*
156 * SAVE_HINT: saves the specified entry as the hint for future lookups.
157 *
158 * => map need not be locked (protected by hint_lock).
159 */
160 #define SAVE_HINT(map,value) do { \
161 simple_lock(&(map)->hint_lock); \
162 (map)->hint = (value); \
163 simple_unlock(&(map)->hint_lock); \
164 } while (0)
165
166 /*
167 * VM_MAP_RANGE_CHECK: check and correct range
168 *
169 * => map must at least be read locked
170 */
171
172 #define VM_MAP_RANGE_CHECK(map, start, end) do { \
173 if (start < vm_map_min(map)) \
174 start = vm_map_min(map); \
175 if (end > vm_map_max(map)) \
176 end = vm_map_max(map); \
177 if (start > end) \
178 start = end; \
179 } while (0)
180
181 /*
182 * local prototypes
183 */
184
185 static vm_map_entry_t uvm_mapent_alloc __P((vm_map_t));
186 static void uvm_mapent_copy __P((vm_map_entry_t,vm_map_entry_t));
187 static void uvm_mapent_free __P((vm_map_entry_t));
188 static void uvm_map_entry_unwire __P((vm_map_t, vm_map_entry_t));
189
190 /*
191 * local inlines
192 */
193
194 /*
195 * uvm_mapent_alloc: allocate a map entry
196 *
197 * => XXX: static pool for kernel map?
198 */
199
200 static __inline vm_map_entry_t
201 uvm_mapent_alloc(map)
202 vm_map_t map;
203 {
204 vm_map_entry_t me;
205 int s;
206 UVMHIST_FUNC("uvm_mapent_alloc");
207 UVMHIST_CALLED(maphist);
208
209 if (map->entries_pageable) {
210 me = pool_get(&uvm_map_entry_pool, PR_WAITOK);
211 me->flags = 0;
212 /* me can't be null, wait ok */
213
214 } else {
215 s = splimp(); /* protect kentry_free list with splimp */
216 simple_lock(&uvm.kentry_lock);
217 me = uvm.kentry_free;
218 if (me) uvm.kentry_free = me->next;
219 simple_unlock(&uvm.kentry_lock);
220 splx(s);
221 if (!me)
222 panic("mapent_alloc: out of kernel map entries, check MAX_KMAPENT");
223 me->flags = UVM_MAP_STATIC;
224 }
225
226 UVMHIST_LOG(maphist, "<- new entry=0x%x [pageable=%d]",
227 me, map->entries_pageable, 0, 0);
228 return(me);
229
230 }
231
232 /*
233 * uvm_mapent_free: free map entry
234 *
235 * => XXX: static pool for kernel map?
236 */
237
238 static __inline void
239 uvm_mapent_free(me)
240 vm_map_entry_t me;
241 {
242 int s;
243 UVMHIST_FUNC("uvm_mapent_free");
244 UVMHIST_CALLED(maphist);
245 UVMHIST_LOG(maphist,"<- freeing map entry=0x%x [flags=%d]",
246 me, me->flags, 0, 0);
247 if ((me->flags & UVM_MAP_STATIC) == 0) {
248 pool_put(&uvm_map_entry_pool, me);
249 } else {
250 s = splimp(); /* protect kentry_free list with splimp */
251 simple_lock(&uvm.kentry_lock);
252 me->next = uvm.kentry_free;
253 uvm.kentry_free = me;
254 simple_unlock(&uvm.kentry_lock);
255 splx(s);
256 }
257 }
258
259 /*
260 * uvm_mapent_copy: copy a map entry, preserving flags
261 */
262
263 static __inline void
264 uvm_mapent_copy(src, dst)
265 vm_map_entry_t src;
266 vm_map_entry_t dst;
267 {
268
269 memcpy(dst, src, ((char *)&src->uvm_map_entry_stop_copy) - ((char*)src));
270 }
271
272 /*
273 * uvm_map_entry_unwire: unwire a map entry
274 *
275 * => map should be locked by caller
276 */
277
278 static __inline void
279 uvm_map_entry_unwire(map, entry)
280 vm_map_t map;
281 vm_map_entry_t entry;
282 {
283
284 uvm_fault_unwire(map->pmap, entry->start, entry->end);
285 entry->wired_count = 0;
286 }
287
288 /*
289 * uvm_map_init: init mapping system at boot time. note that we allocate
290 * and init the static pool of vm_map_entry_t's for the kernel here.
291 */
292
293 void
294 uvm_map_init()
295 {
296 static struct vm_map_entry kernel_map_entry[MAX_KMAPENT];
297 #if defined(UVMHIST)
298 static struct uvm_history_ent maphistbuf[100];
299 static struct uvm_history_ent pdhistbuf[100];
300 #endif
301 int lcv;
302
303 /*
304 * first, init logging system.
305 */
306
307 UVMHIST_FUNC("uvm_map_init");
308 UVMHIST_INIT_STATIC(maphist, maphistbuf);
309 UVMHIST_INIT_STATIC(pdhist, pdhistbuf);
310 UVMHIST_CALLED(maphist);
311 UVMHIST_LOG(maphist,"<starting uvm map system>", 0, 0, 0, 0);
312 UVMCNT_INIT(uvm_map_call, UVMCNT_CNT, 0,
313 "# uvm_map() successful calls", 0);
314 UVMCNT_INIT(map_backmerge, UVMCNT_CNT, 0, "# uvm_map() back merges", 0);
315 UVMCNT_INIT(map_forwmerge, UVMCNT_CNT, 0, "# uvm_map() missed forward",
316 0);
317 UVMCNT_INIT(uvm_mlk_call, UVMCNT_CNT, 0, "# map lookup calls", 0);
318 UVMCNT_INIT(uvm_mlk_hint, UVMCNT_CNT, 0, "# map lookup hint hits", 0);
319
320 /*
321 * now set up static pool of kernel map entrys ...
322 */
323
324 simple_lock_init(&uvm.kentry_lock);
325 uvm.kentry_free = NULL;
326 for (lcv = 0 ; lcv < MAX_KMAPENT ; lcv++) {
327 kernel_map_entry[lcv].next = uvm.kentry_free;
328 uvm.kentry_free = &kernel_map_entry[lcv];
329 }
330
331 /*
332 * initialize the map-related pools.
333 */
334 pool_init(&uvm_vmspace_pool, sizeof(struct vmspace),
335 0, 0, 0, "vmsppl", 0,
336 pool_page_alloc_nointr, pool_page_free_nointr, M_VMMAP);
337 pool_init(&uvm_map_entry_pool, sizeof(struct vm_map_entry),
338 0, 0, 0, "vmmpepl", 0,
339 pool_page_alloc_nointr, pool_page_free_nointr, M_VMMAP);
340 }
341
342 /*
343 * clippers
344 */
345
346 /*
347 * uvm_map_clip_start: ensure that the entry begins at or after
348 * the starting address, if it doesn't we split the entry.
349 *
350 * => caller should use UVM_MAP_CLIP_START macro rather than calling
351 * this directly
352 * => map must be locked by caller
353 */
354
355 void uvm_map_clip_start(map, entry, start)
356 vm_map_t map;
357 vm_map_entry_t entry;
358 vaddr_t start;
359 {
360 vm_map_entry_t new_entry;
361 vaddr_t new_adj;
362
363 /* uvm_map_simplify_entry(map, entry); */ /* XXX */
364
365 /*
366 * Split off the front portion. note that we must insert the new
367 * entry BEFORE this one, so that this entry has the specified
368 * starting address.
369 */
370
371 new_entry = uvm_mapent_alloc(map);
372 uvm_mapent_copy(entry, new_entry); /* entry -> new_entry */
373
374 new_entry->end = start;
375 new_adj = start - new_entry->start;
376 if (entry->object.uvm_obj)
377 entry->offset += new_adj; /* shift start over */
378 entry->start = start;
379
380 if (new_entry->aref.ar_amap) {
381 amap_splitref(&new_entry->aref, &entry->aref, new_adj);
382 }
383
384 uvm_map_entry_link(map, entry->prev, new_entry);
385
386 if (UVM_ET_ISSUBMAP(entry)) {
387 /* ... unlikely to happen, but play it safe */
388 uvm_map_reference(new_entry->object.sub_map);
389 } else {
390 if (UVM_ET_ISOBJ(entry) &&
391 entry->object.uvm_obj->pgops &&
392 entry->object.uvm_obj->pgops->pgo_reference)
393 entry->object.uvm_obj->pgops->pgo_reference(
394 entry->object.uvm_obj);
395 }
396 }
397
398 /*
399 * uvm_map_clip_end: ensure that the entry ends at or before
400 * the ending address, if it does't we split the reference
401 *
402 * => caller should use UVM_MAP_CLIP_END macro rather than calling
403 * this directly
404 * => map must be locked by caller
405 */
406
407 void
408 uvm_map_clip_end(map, entry, end)
409 vm_map_t map;
410 vm_map_entry_t entry;
411 vaddr_t end;
412 {
413 vm_map_entry_t new_entry;
414 vaddr_t new_adj; /* #bytes we move start forward */
415
416 /*
417 * Create a new entry and insert it
418 * AFTER the specified entry
419 */
420
421 new_entry = uvm_mapent_alloc(map);
422 uvm_mapent_copy(entry, new_entry); /* entry -> new_entry */
423
424 new_entry->start = entry->end = end;
425 new_adj = end - entry->start;
426 if (new_entry->object.uvm_obj)
427 new_entry->offset += new_adj;
428
429 if (entry->aref.ar_amap)
430 amap_splitref(&entry->aref, &new_entry->aref, new_adj);
431
432 uvm_map_entry_link(map, entry, new_entry);
433
434 if (UVM_ET_ISSUBMAP(entry)) {
435 /* ... unlikely to happen, but play it safe */
436 uvm_map_reference(new_entry->object.sub_map);
437 } else {
438 if (UVM_ET_ISOBJ(entry) &&
439 entry->object.uvm_obj->pgops &&
440 entry->object.uvm_obj->pgops->pgo_reference)
441 entry->object.uvm_obj->pgops->pgo_reference(
442 entry->object.uvm_obj);
443 }
444 }
445
446
447 /*
448 * M A P - m a i n e n t r y p o i n t
449 */
450 /*
451 * uvm_map: establish a valid mapping in a map
452 *
453 * => assume startp is page aligned.
454 * => assume size is a multiple of PAGE_SIZE.
455 * => assume sys_mmap provides enough of a "hint" to have us skip
456 * over text/data/bss area.
457 * => map must be unlocked (we will lock it)
458 * => <uobj,uoffset> value meanings (4 cases):
459 * [1] <NULL,uoffset> == uoffset is a hint for PMAP_PREFER
460 * [2] <NULL,UVM_UNKNOWN_OFFSET> == don't PMAP_PREFER
461 * [3] <uobj,uoffset> == normal mapping
462 * [4] <uobj,UVM_UNKNOWN_OFFSET> == uvm_map finds offset based on VA
463 *
464 * case [4] is for kernel mappings where we don't know the offset until
465 * we've found a virtual address. note that kernel object offsets are
466 * always relative to vm_map_min(kernel_map).
467 * => XXXCDC: need way to map in external amap?
468 */
469
470 int
471 uvm_map(map, startp, size, uobj, uoffset, flags)
472 vm_map_t map;
473 vaddr_t *startp; /* IN/OUT */
474 vsize_t size;
475 struct uvm_object *uobj;
476 vaddr_t uoffset;
477 uvm_flag_t flags;
478 {
479 vm_map_entry_t prev_entry, new_entry;
480 vm_prot_t prot = UVM_PROTECTION(flags), maxprot =
481 UVM_MAXPROTECTION(flags);
482 vm_inherit_t inherit = UVM_INHERIT(flags);
483 int advice = UVM_ADVICE(flags);
484 UVMHIST_FUNC("uvm_map");
485 UVMHIST_CALLED(maphist);
486
487 UVMHIST_LOG(maphist, "(map=0x%x, *startp=0x%x, size=%d, flags=0x%x)",
488 map, *startp, size, flags);
489 UVMHIST_LOG(maphist, " uobj/offset 0x%x/%d", uobj, uoffset,0,0);
490
491 /*
492 * step 0: sanity check of protection code
493 */
494
495 if ((prot & maxprot) != prot) {
496 UVMHIST_LOG(maphist, "<- prot. failure: prot=0x%x, max=0x%x",
497 prot, maxprot,0,0);
498 return(KERN_PROTECTION_FAILURE);
499 }
500
501 /*
502 * step 1: figure out where to put new VM range
503 */
504
505 if (vm_map_lock_try(map) == FALSE) {
506 if (flags & UVM_FLAG_TRYLOCK)
507 return(KERN_FAILURE);
508 vm_map_lock(map); /* could sleep here */
509 }
510 if ((prev_entry = uvm_map_findspace(map, *startp, size, startp,
511 uobj, uoffset, flags & UVM_FLAG_FIXED)) == NULL) {
512 UVMHIST_LOG(maphist,"<- uvm_map_findspace failed!",0,0,0,0);
513 vm_map_unlock(map);
514 return (KERN_NO_SPACE);
515 }
516
517 #ifdef PMAP_GROWKERNEL
518 {
519 /*
520 * If the kernel pmap can't map the requested space,
521 * then allocate more resources for it.
522 */
523 if (map == kernel_map && uvm_maxkaddr < (*startp + size))
524 uvm_maxkaddr = pmap_growkernel(*startp + size);
525 }
526 #endif
527
528 UVMCNT_INCR(uvm_map_call);
529
530 /*
531 * if uobj is null, then uoffset is either a VAC hint for PMAP_PREFER
532 * [typically from uvm_map_reserve] or it is UVM_UNKNOWN_OFFSET. in
533 * either case we want to zero it before storing it in the map entry
534 * (because it looks strange and confusing when debugging...)
535 *
536 * if uobj is not null
537 * if uoffset is not UVM_UNKNOWN_OFFSET then we have a normal mapping
538 * and we do not need to change uoffset.
539 * if uoffset is UVM_UNKNOWN_OFFSET then we need to find the offset
540 * now (based on the starting address of the map). this case is
541 * for kernel object mappings where we don't know the offset until
542 * the virtual address is found (with uvm_map_findspace). the
543 * offset is the distance we are from the start of the map.
544 */
545
546 if (uobj == NULL) {
547 uoffset = 0;
548 } else {
549 if (uoffset == UVM_UNKNOWN_OFFSET) {
550 #ifdef DIAGNOSTIC
551 if (uobj->uo_refs != UVM_OBJ_KERN)
552 panic("uvm_map: unknown offset with non-kernel object");
553 #endif
554 uoffset = *startp - vm_map_min(kernel_map);
555 }
556 }
557
558 /*
559 * step 2: try and insert in map by extending previous entry, if
560 * possible
561 * XXX: we don't try and pull back the next entry. might be useful
562 * for a stack, but we are currently allocating our stack in advance.
563 */
564
565 if ((flags & UVM_FLAG_NOMERGE) == 0 &&
566 prev_entry->end == *startp && prev_entry != &map->header &&
567 prev_entry->object.uvm_obj == uobj) {
568
569 if (uobj && prev_entry->offset +
570 (prev_entry->end - prev_entry->start) != uoffset)
571 goto step3;
572
573 if (UVM_ET_ISSUBMAP(prev_entry))
574 goto step3;
575
576 if (prev_entry->protection != prot ||
577 prev_entry->max_protection != maxprot)
578 goto step3;
579
580 if (prev_entry->inheritance != inherit ||
581 prev_entry->advice != advice)
582 goto step3;
583
584 /* wired_count's must match (new area is unwired) */
585 if (prev_entry->wired_count)
586 goto step3;
587
588 /*
589 * can't extend a shared amap. note: no need to lock amap to
590 * look at refs since we don't care about its exact value.
591 * if it is one (i.e. we have only reference) it will stay there
592 */
593
594 if (prev_entry->aref.ar_amap &&
595 amap_refs(prev_entry->aref.ar_amap) != 1) {
596 goto step3;
597 }
598
599 /* got it! */
600
601 UVMCNT_INCR(map_backmerge);
602 UVMHIST_LOG(maphist," starting back merge", 0, 0, 0, 0);
603
604 /*
605 * drop our reference to uobj since we are extending a reference
606 * that we already have (the ref count can not drop to zero).
607 */
608 if (uobj && uobj->pgops->pgo_detach)
609 uobj->pgops->pgo_detach(uobj);
610
611 if (prev_entry->aref.ar_amap) {
612 amap_extend(prev_entry, size);
613 }
614
615 prev_entry->end += size;
616 map->size += size;
617
618 UVMHIST_LOG(maphist,"<- done (via backmerge)!", 0, 0, 0, 0);
619 vm_map_unlock(map);
620 return (KERN_SUCCESS);
621
622 }
623 step3:
624 UVMHIST_LOG(maphist," allocating new map entry", 0, 0, 0, 0);
625
626 /*
627 * check for possible forward merge (which we don't do) and count
628 * the number of times we missed a *possible* chance to merge more
629 */
630
631 if ((flags & UVM_FLAG_NOMERGE) == 0 &&
632 prev_entry->next != &map->header &&
633 prev_entry->next->start == (*startp + size))
634 UVMCNT_INCR(map_forwmerge);
635
636 /*
637 * step 3: allocate new entry and link it in
638 */
639
640 new_entry = uvm_mapent_alloc(map);
641 new_entry->start = *startp;
642 new_entry->end = new_entry->start + size;
643 new_entry->object.uvm_obj = uobj;
644 new_entry->offset = uoffset;
645
646 if (uobj)
647 new_entry->etype = UVM_ET_OBJ;
648 else
649 new_entry->etype = 0;
650
651 if (flags & UVM_FLAG_COPYONW) {
652 new_entry->etype |= UVM_ET_COPYONWRITE;
653 if ((flags & UVM_FLAG_OVERLAY) == 0)
654 new_entry->etype |= UVM_ET_NEEDSCOPY;
655 }
656
657 new_entry->protection = prot;
658 new_entry->max_protection = maxprot;
659 new_entry->inheritance = inherit;
660 new_entry->wired_count = 0;
661 new_entry->advice = advice;
662 if (flags & UVM_FLAG_OVERLAY) {
663 /*
664 * to_add: for BSS we overallocate a little since we
665 * are likely to extend
666 */
667 vaddr_t to_add = (flags & UVM_FLAG_AMAPPAD) ?
668 UVM_AMAP_CHUNK << PAGE_SHIFT : 0;
669 struct vm_amap *amap = amap_alloc(size, to_add, M_WAITOK);
670 new_entry->aref.ar_pageoff = 0;
671 new_entry->aref.ar_amap = amap;
672 } else {
673 new_entry->aref.ar_amap = NULL;
674 }
675
676 uvm_map_entry_link(map, prev_entry, new_entry);
677
678 map->size += size;
679
680 /*
681 * Update the free space hint
682 */
683
684 if ((map->first_free == prev_entry) &&
685 (prev_entry->end >= new_entry->start))
686 map->first_free = new_entry;
687
688 UVMHIST_LOG(maphist,"<- done!", 0, 0, 0, 0);
689 vm_map_unlock(map);
690 return(KERN_SUCCESS);
691 }
692
693 /*
694 * uvm_map_lookup_entry: find map entry at or before an address
695 *
696 * => map must at least be read-locked by caller
697 * => entry is returned in "entry"
698 * => return value is true if address is in the returned entry
699 */
700
701 boolean_t
702 uvm_map_lookup_entry(map, address, entry)
703 vm_map_t map;
704 vaddr_t address;
705 vm_map_entry_t *entry; /* OUT */
706 {
707 vm_map_entry_t cur;
708 vm_map_entry_t last;
709 UVMHIST_FUNC("uvm_map_lookup_entry");
710 UVMHIST_CALLED(maphist);
711
712 UVMHIST_LOG(maphist,"(map=0x%x,addr=0x%x,ent=0x%x)",
713 map, address, entry, 0);
714
715 /*
716 * start looking either from the head of the
717 * list, or from the hint.
718 */
719
720 simple_lock(&map->hint_lock);
721 cur = map->hint;
722 simple_unlock(&map->hint_lock);
723
724 if (cur == &map->header)
725 cur = cur->next;
726
727 UVMCNT_INCR(uvm_mlk_call);
728 if (address >= cur->start) {
729 /*
730 * go from hint to end of list.
731 *
732 * but first, make a quick check to see if
733 * we are already looking at the entry we
734 * want (which is usually the case).
735 * note also that we don't need to save the hint
736 * here... it is the same hint (unless we are
737 * at the header, in which case the hint didn't
738 * buy us anything anyway).
739 */
740 last = &map->header;
741 if ((cur != last) && (cur->end > address)) {
742 UVMCNT_INCR(uvm_mlk_hint);
743 *entry = cur;
744 UVMHIST_LOG(maphist,"<- got it via hint (0x%x)",
745 cur, 0, 0, 0);
746 return (TRUE);
747 }
748 } else {
749 /*
750 * go from start to hint, *inclusively*
751 */
752 last = cur->next;
753 cur = map->header.next;
754 }
755
756 /*
757 * search linearly
758 */
759
760 while (cur != last) {
761 if (cur->end > address) {
762 if (address >= cur->start) {
763 /*
764 * save this lookup for future
765 * hints, and return
766 */
767
768 *entry = cur;
769 SAVE_HINT(map, cur);
770 UVMHIST_LOG(maphist,"<- search got it (0x%x)",
771 cur, 0, 0, 0);
772 return (TRUE);
773 }
774 break;
775 }
776 cur = cur->next;
777 }
778 *entry = cur->prev;
779 SAVE_HINT(map, *entry);
780 UVMHIST_LOG(maphist,"<- failed!",0,0,0,0);
781 return (FALSE);
782 }
783
784
785 /*
786 * uvm_map_findspace: find "length" sized space in "map".
787 *
788 * => "hint" is a hint about where we want it, unless fixed is true
789 * (in which case we insist on using "hint").
790 * => "result" is VA returned
791 * => uobj/uoffset are to be used to handle VAC alignment, if required
792 * => caller must at least have read-locked map
793 * => returns NULL on failure, or pointer to prev. map entry if success
794 * => note this is a cross between the old vm_map_findspace and vm_map_find
795 */
796
797 vm_map_entry_t
798 uvm_map_findspace(map, hint, length, result, uobj, uoffset, fixed)
799 vm_map_t map;
800 vaddr_t hint;
801 vsize_t length;
802 vaddr_t *result; /* OUT */
803 struct uvm_object *uobj;
804 vaddr_t uoffset;
805 boolean_t fixed;
806 {
807 vm_map_entry_t entry, next, tmp;
808 vaddr_t end;
809 UVMHIST_FUNC("uvm_map_findspace");
810 UVMHIST_CALLED(maphist);
811
812 UVMHIST_LOG(maphist, "(map=0x%x, hint=0x%x, len=%d, fixed=%d)",
813 map, hint, length, fixed);
814
815 if (hint < map->min_offset) { /* check ranges ... */
816 if (fixed) {
817 UVMHIST_LOG(maphist,"<- VA below map range",0,0,0,0);
818 return(NULL);
819 }
820 hint = map->min_offset;
821 }
822 if (hint > map->max_offset) {
823 UVMHIST_LOG(maphist,"<- VA 0x%x > range [0x%x->0x%x]",
824 hint, map->min_offset, map->max_offset, 0);
825 return(NULL);
826 }
827
828 /*
829 * Look for the first possible address; if there's already
830 * something at this address, we have to start after it.
831 */
832
833 if (!fixed && hint == map->min_offset) {
834 if ((entry = map->first_free) != &map->header)
835 hint = entry->end;
836 } else {
837 if (uvm_map_lookup_entry(map, hint, &tmp)) {
838 /* "hint" address already in use ... */
839 if (fixed) {
840 UVMHIST_LOG(maphist,"<- fixed & VA in use",
841 0, 0, 0, 0);
842 return(NULL);
843 }
844 hint = tmp->end;
845 }
846 entry = tmp;
847 }
848
849 /*
850 * Look through the rest of the map, trying to fit a new region in
851 * the gap between existing regions, or after the very last region.
852 * note: entry->end = base VA of current gap,
853 * next->start = VA of end of current gap
854 */
855 for (;; hint = (entry = next)->end) {
856 /*
857 * Find the end of the proposed new region. Be sure we didn't
858 * go beyond the end of the map, or wrap around the address;
859 * if so, we lose. Otherwise, if this is the last entry, or
860 * if the proposed new region fits before the next entry, we
861 * win.
862 */
863
864 #ifdef PMAP_PREFER
865 /*
866 * push hint forward as needed to avoid VAC alias problems.
867 * we only do this if a valid offset is specified.
868 */
869 if (!fixed && uoffset != UVM_UNKNOWN_OFFSET)
870 PMAP_PREFER(uoffset, &hint);
871 #endif
872 end = hint + length;
873 if (end > map->max_offset || end < hint) {
874 UVMHIST_LOG(maphist,"<- failed (off end)", 0,0,0,0);
875 return (NULL);
876 }
877 next = entry->next;
878 if (next == &map->header || next->start >= end)
879 break;
880 if (fixed) {
881 UVMHIST_LOG(maphist,"<- fixed mapping failed", 0,0,0,0);
882 return(NULL); /* only one shot at it ... */
883 }
884 }
885 SAVE_HINT(map, entry);
886 *result = hint;
887 UVMHIST_LOG(maphist,"<- got it! (result=0x%x)", hint, 0,0,0);
888 return (entry);
889 }
890
891 /*
892 * U N M A P - m a i n h e l p e r f u n c t i o n s
893 */
894
895 /*
896 * uvm_unmap_remove: remove mappings from a vm_map (from "start" up to "stop")
897 *
898 * => caller must check alignment and size
899 * => map must be locked by caller
900 * => we return a list of map entries that we've remove from the map
901 * in "entry_list"
902 */
903
904 int
905 uvm_unmap_remove(map, start, end, entry_list)
906 vm_map_t map;
907 vaddr_t start,end;
908 vm_map_entry_t *entry_list; /* OUT */
909 {
910 vm_map_entry_t entry, first_entry, next;
911 vaddr_t len;
912 UVMHIST_FUNC("uvm_unmap_remove");
913 UVMHIST_CALLED(maphist);
914
915 UVMHIST_LOG(maphist,"(map=0x%x, start=0x%x, end=0x%x)",
916 map, start, end, 0);
917
918 VM_MAP_RANGE_CHECK(map, start, end);
919
920 /*
921 * find first entry
922 */
923 if (uvm_map_lookup_entry(map, start, &first_entry) == TRUE) {
924 /* clip and go... */
925 entry = first_entry;
926 UVM_MAP_CLIP_START(map, entry, start);
927 /* critical! prevents stale hint */
928 SAVE_HINT(map, entry->prev);
929
930 } else {
931 entry = first_entry->next;
932 }
933
934 /*
935 * Save the free space hint
936 */
937
938 if (map->first_free->start >= start)
939 map->first_free = entry->prev;
940
941 /*
942 * note: we now re-use first_entry for a different task. we remove
943 * a number of map entries from the map and save them in a linked
944 * list headed by "first_entry". once we remove them from the map
945 * the caller should unlock the map and drop the references to the
946 * backing objects [c.f. uvm_unmap_detach]. the object is to
947 * seperate unmapping from reference dropping. why?
948 * [1] the map has to be locked for unmapping
949 * [2] the map need not be locked for reference dropping
950 * [3] dropping references may trigger pager I/O, and if we hit
951 * a pager that does synchronous I/O we may have to wait for it.
952 * [4] we would like all waiting for I/O to occur with maps unlocked
953 * so that we don't block other threads.
954 */
955 first_entry = NULL;
956 *entry_list = NULL; /* to be safe */
957
958 /*
959 * break up the area into map entry sized regions and unmap. note
960 * that all mappings have to be removed before we can even consider
961 * dropping references to amaps or VM objects (otherwise we could end
962 * up with a mapping to a page on the free list which would be very bad)
963 */
964
965 while ((entry != &map->header) && (entry->start < end)) {
966
967 UVM_MAP_CLIP_END(map, entry, end);
968 next = entry->next;
969 len = entry->end - entry->start;
970
971 /*
972 * unwire before removing addresses from the pmap; otherwise
973 * unwiring will put the entries back into the pmap (XXX).
974 */
975
976 if (entry->wired_count)
977 uvm_map_entry_unwire(map, entry);
978
979 /*
980 * special case: handle mappings to anonymous kernel objects.
981 * we want to free these pages right away...
982 */
983 if (UVM_ET_ISOBJ(entry) &&
984 entry->object.uvm_obj->uo_refs == UVM_OBJ_KERN) {
985
986 #ifdef DIAGNOSTIC
987 if (vm_map_pmap(map) != pmap_kernel())
988 panic("uvm_unmap_remove: kernel object mapped by non-kernel map");
989 #endif
990
991 /*
992 * note: kernel object mappings are currently used in
993 * two ways:
994 * [1] "normal" mappings of pages in the kernel object
995 * [2] uvm_km_valloc'd allocations in which we
996 * pmap_enter in some non-kernel-object page
997 * (e.g. vmapbuf).
998 *
999 * for case [1], we need to remove the mapping from
1000 * the pmap and then remove the page from the kernel
1001 * object (because, once pages in a kernel object are
1002 * unmapped they are no longer needed, unlike, say,
1003 * a vnode where you might want the data to persist
1004 * until flushed out of a queue).
1005 *
1006 * for case [2], we need to remove the mapping from
1007 * the pmap. there shouldn't be any pages at the
1008 * specified offset in the kernel object [but it
1009 * doesn't hurt to call uvm_km_pgremove just to be
1010 * safe?]
1011 *
1012 * uvm_km_pgremove currently does the following:
1013 * for pages in the kernel object in range:
1014 * - pmap_page_protect them out of all pmaps
1015 * - uvm_pagefree the page
1016 *
1017 * note that in case [1] the pmap_page_protect call
1018 * in uvm_km_pgremove may very well be redundant
1019 * because we have already removed the mappings
1020 * beforehand with pmap_remove (or pmap_kremove).
1021 * in the PMAP_NEW case, the pmap_page_protect call
1022 * may not do anything, since PMAP_NEW allows the
1023 * kernel to enter/remove kernel mappings without
1024 * bothing to keep track of the mappings (e.g. via
1025 * pv_entry lists). XXX: because of this, in the
1026 * future we should consider removing the
1027 * pmap_page_protect from uvm_km_pgremove some time
1028 * in the future.
1029 */
1030
1031 /*
1032 * remove mappings from pmap
1033 */
1034 #if defined(PMAP_NEW)
1035 pmap_kremove(entry->start, len);
1036 #else
1037 pmap_remove(pmap_kernel(), entry->start,
1038 entry->start+len);
1039 #endif
1040
1041 /*
1042 * remove pages from a kernel object (offsets are
1043 * always relative to vm_map_min(kernel_map)).
1044 */
1045 uvm_km_pgremove(entry->object.uvm_obj,
1046 entry->start - vm_map_min(kernel_map),
1047 entry->end - vm_map_min(kernel_map));
1048
1049 /*
1050 * null out kernel_object reference, we've just
1051 * dropped it
1052 */
1053 entry->etype &= ~UVM_ET_OBJ;
1054 entry->object.uvm_obj = NULL; /* to be safe */
1055
1056 } else {
1057 /*
1058 * remove mappings the standard way.
1059 */
1060 pmap_remove(map->pmap, entry->start, entry->end);
1061 }
1062
1063 /*
1064 * remove entry from map and put it on our list of entries
1065 * that we've nuked. then go do next entry.
1066 */
1067 UVMHIST_LOG(maphist, " removed map entry 0x%x", entry, 0, 0,0);
1068 uvm_map_entry_unlink(map, entry);
1069 map->size -= len;
1070 entry->next = first_entry;
1071 first_entry = entry;
1072 entry = next; /* next entry, please */
1073 }
1074
1075 /*
1076 * now we've cleaned up the map and are ready for the caller to drop
1077 * references to the mapped objects.
1078 */
1079
1080 *entry_list = first_entry;
1081 UVMHIST_LOG(maphist,"<- done!", 0, 0, 0, 0);
1082 return(KERN_SUCCESS);
1083 }
1084
1085 /*
1086 * uvm_unmap_detach: drop references in a chain of map entries
1087 *
1088 * => we will free the map entries as we traverse the list.
1089 */
1090
1091 void
1092 uvm_unmap_detach(first_entry, amap_unref_flags)
1093 vm_map_entry_t first_entry;
1094 int amap_unref_flags;
1095 {
1096 vm_map_entry_t next_entry;
1097 UVMHIST_FUNC("uvm_unmap_detach"); UVMHIST_CALLED(maphist);
1098
1099 while (first_entry) {
1100
1101 #ifdef DIAGNOSTIC
1102 /*
1103 * sanity check
1104 */
1105 /* was part of vm_map_entry_delete() */
1106 if (first_entry->wired_count)
1107 panic("unmap: still wired!");
1108 #endif
1109
1110 UVMHIST_LOG(maphist,
1111 " detach 0x%x: amap=0x%x, obj=0x%x, submap?=%d",
1112 first_entry, first_entry->aref.ar_amap,
1113 first_entry->object.uvm_obj,
1114 UVM_ET_ISSUBMAP(first_entry));
1115
1116 /*
1117 * drop reference to amap, if we've got one
1118 */
1119
1120 if (first_entry->aref.ar_amap)
1121 amap_unref(first_entry, amap_unref_flags);
1122
1123 /*
1124 * drop reference to our backing object, if we've got one
1125 */
1126
1127 if (UVM_ET_ISSUBMAP(first_entry)) {
1128 /* ... unlikely to happen, but play it safe */
1129 uvm_map_deallocate(first_entry->object.sub_map);
1130 } else {
1131 if (UVM_ET_ISOBJ(first_entry) &&
1132 first_entry->object.uvm_obj->pgops->pgo_detach)
1133 first_entry->object.uvm_obj->pgops->
1134 pgo_detach(first_entry->object.uvm_obj);
1135 }
1136
1137 /*
1138 * next entry
1139 */
1140 next_entry = first_entry->next;
1141 uvm_mapent_free(first_entry);
1142 first_entry = next_entry;
1143 }
1144
1145 /*
1146 * done!
1147 */
1148 UVMHIST_LOG(maphist, "<- done", 0,0,0,0);
1149 return;
1150 }
1151
1152 /*
1153 * E X T R A C T I O N F U N C T I O N S
1154 */
1155
1156 /*
1157 * uvm_map_reserve: reserve space in a vm_map for future use.
1158 *
1159 * => we reserve space in a map by putting a dummy map entry in the
1160 * map (dummy means obj=NULL, amap=NULL, prot=VM_PROT_NONE)
1161 * => map should be unlocked (we will write lock it)
1162 * => we return true if we were able to reserve space
1163 * => XXXCDC: should be inline?
1164 */
1165
1166 int
1167 uvm_map_reserve(map, size, offset, raddr)
1168 vm_map_t map;
1169 vsize_t size;
1170 vaddr_t offset; /* hint for pmap_prefer */
1171 vaddr_t *raddr; /* OUT: reserved VA */
1172 {
1173 UVMHIST_FUNC("uvm_map_reserve"); UVMHIST_CALLED(maphist);
1174
1175 UVMHIST_LOG(maphist, "(map=0x%x, size=0x%x, offset=0x%x,addr=0x%x)",
1176 map,size,offset,raddr);
1177
1178 size = round_page(size);
1179 if (*raddr < vm_map_min(map))
1180 *raddr = vm_map_min(map); /* hint */
1181
1182 /*
1183 * reserve some virtual space.
1184 */
1185
1186 if (uvm_map(map, raddr, size, NULL, offset,
1187 UVM_MAPFLAG(UVM_PROT_NONE, UVM_PROT_NONE, UVM_INH_NONE,
1188 UVM_ADV_RANDOM, UVM_FLAG_NOMERGE)) != KERN_SUCCESS) {
1189 UVMHIST_LOG(maphist, "<- done (no VM)", 0,0,0,0);
1190 return (FALSE);
1191 }
1192
1193 UVMHIST_LOG(maphist, "<- done (*raddr=0x%x)", *raddr,0,0,0);
1194 return (TRUE);
1195 }
1196
1197 /*
1198 * uvm_map_replace: replace a reserved (blank) area of memory with
1199 * real mappings.
1200 *
1201 * => caller must WRITE-LOCK the map
1202 * => we return TRUE if replacement was a success
1203 * => we expect the newents chain to have nnewents entrys on it and
1204 * we expect newents->prev to point to the last entry on the list
1205 * => note newents is allowed to be NULL
1206 */
1207
1208 int
1209 uvm_map_replace(map, start, end, newents, nnewents)
1210 struct vm_map *map;
1211 vaddr_t start, end;
1212 vm_map_entry_t newents;
1213 int nnewents;
1214 {
1215 vm_map_entry_t oldent, last;
1216 UVMHIST_FUNC("uvm_map_replace");
1217 UVMHIST_CALLED(maphist);
1218
1219 /*
1220 * first find the blank map entry at the specified address
1221 */
1222
1223 if (!uvm_map_lookup_entry(map, start, &oldent)) {
1224 return(FALSE);
1225 }
1226
1227 /*
1228 * check to make sure we have a proper blank entry
1229 */
1230
1231 if (oldent->start != start || oldent->end != end ||
1232 oldent->object.uvm_obj != NULL || oldent->aref.ar_amap != NULL) {
1233 return (FALSE);
1234 }
1235
1236 #ifdef DIAGNOSTIC
1237 /*
1238 * sanity check the newents chain
1239 */
1240 {
1241 vm_map_entry_t tmpent = newents;
1242 int nent = 0;
1243 vaddr_t cur = start;
1244
1245 while (tmpent) {
1246 nent++;
1247 if (tmpent->start < cur)
1248 panic("uvm_map_replace1");
1249 if (tmpent->start > tmpent->end || tmpent->end > end) {
1250 printf("tmpent->start=0x%lx, tmpent->end=0x%lx, end=0x%lx\n",
1251 tmpent->start, tmpent->end, end);
1252 panic("uvm_map_replace2");
1253 }
1254 cur = tmpent->end;
1255 if (tmpent->next) {
1256 if (tmpent->next->prev != tmpent)
1257 panic("uvm_map_replace3");
1258 } else {
1259 if (newents->prev != tmpent)
1260 panic("uvm_map_replace4");
1261 }
1262 tmpent = tmpent->next;
1263 }
1264 if (nent != nnewents)
1265 panic("uvm_map_replace5");
1266 }
1267 #endif
1268
1269 /*
1270 * map entry is a valid blank! replace it. (this does all the
1271 * work of map entry link/unlink...).
1272 */
1273
1274 if (newents) {
1275
1276 last = newents->prev; /* we expect this */
1277
1278 /* critical: flush stale hints out of map */
1279 SAVE_HINT(map, newents);
1280 if (map->first_free == oldent)
1281 map->first_free = last;
1282
1283 last->next = oldent->next;
1284 last->next->prev = last;
1285 newents->prev = oldent->prev;
1286 newents->prev->next = newents;
1287 map->nentries = map->nentries + (nnewents - 1);
1288
1289 } else {
1290
1291 /* critical: flush stale hints out of map */
1292 SAVE_HINT(map, oldent->prev);
1293 if (map->first_free == oldent)
1294 map->first_free = oldent->prev;
1295
1296 /* NULL list of new entries: just remove the old one */
1297 uvm_map_entry_unlink(map, oldent);
1298 }
1299
1300
1301 /*
1302 * now we can free the old blank entry, unlock the map and return.
1303 */
1304
1305 uvm_mapent_free(oldent);
1306 return(TRUE);
1307 }
1308
1309 /*
1310 * uvm_map_extract: extract a mapping from a map and put it somewhere
1311 * (maybe removing the old mapping)
1312 *
1313 * => maps should be unlocked (we will write lock them)
1314 * => returns 0 on success, error code otherwise
1315 * => start must be page aligned
1316 * => len must be page sized
1317 * => flags:
1318 * UVM_EXTRACT_REMOVE: remove mappings from srcmap
1319 * UVM_EXTRACT_CONTIG: abort if unmapped area (advisory only)
1320 * UVM_EXTRACT_QREF: for a temporary extraction do quick obj refs
1321 * UVM_EXTRACT_FIXPROT: set prot to maxprot as we go
1322 * >>>NOTE: if you set REMOVE, you are not allowed to use CONTIG or QREF!<<<
1323 * >>>NOTE: QREF's must be unmapped via the QREF path, thus should only
1324 * be used from within the kernel in a kernel level map <<<
1325 */
1326
1327 int
1328 uvm_map_extract(srcmap, start, len, dstmap, dstaddrp, flags)
1329 vm_map_t srcmap, dstmap;
1330 vaddr_t start, *dstaddrp;
1331 vsize_t len;
1332 int flags;
1333 {
1334 vaddr_t dstaddr, end, newend, oldoffset, fudge, orig_fudge,
1335 oldstart;
1336 vm_map_entry_t chain, endchain, entry, orig_entry, newentry, deadentry;
1337 vm_map_entry_t oldentry;
1338 vsize_t elen;
1339 int nchain, error, copy_ok;
1340 UVMHIST_FUNC("uvm_map_extract"); UVMHIST_CALLED(maphist);
1341 UVMHIST_LOG(maphist,"(srcmap=0x%x,start=0x%x, len=0x%x", srcmap, start,
1342 len,0);
1343 UVMHIST_LOG(maphist," ...,dstmap=0x%x, flags=0x%x)", dstmap,flags,0,0);
1344
1345 #ifdef DIAGNOSTIC
1346 /*
1347 * step 0: sanity check: start must be on a page boundary, length
1348 * must be page sized. can't ask for CONTIG/QREF if you asked for
1349 * REMOVE.
1350 */
1351 if ((start & PAGE_MASK) || (len & PAGE_MASK))
1352 panic("uvm_map_extract1");
1353 if (flags & UVM_EXTRACT_REMOVE)
1354 if (flags & (UVM_EXTRACT_CONTIG|UVM_EXTRACT_QREF))
1355 panic("uvm_map_extract2");
1356 #endif
1357
1358
1359 /*
1360 * step 1: reserve space in the target map for the extracted area
1361 */
1362
1363 dstaddr = *dstaddrp;
1364 if (uvm_map_reserve(dstmap, len, start, &dstaddr) == FALSE)
1365 return(ENOMEM);
1366 *dstaddrp = dstaddr; /* pass address back to caller */
1367 UVMHIST_LOG(maphist, " dstaddr=0x%x", dstaddr,0,0,0);
1368
1369
1370 /*
1371 * step 2: setup for the extraction process loop by init'ing the
1372 * map entry chain, locking src map, and looking up the first useful
1373 * entry in the map.
1374 */
1375
1376 end = start + len;
1377 newend = dstaddr + len;
1378 chain = endchain = NULL;
1379 nchain = 0;
1380 vm_map_lock(srcmap);
1381
1382 if (uvm_map_lookup_entry(srcmap, start, &entry)) {
1383
1384 /* "start" is within an entry */
1385 if (flags & UVM_EXTRACT_QREF) {
1386 /*
1387 * for quick references we don't clip the entry, so
1388 * the entry may map space "before" the starting
1389 * virtual address... this is the "fudge" factor
1390 * (which can be non-zero only the first time
1391 * through the "while" loop in step 3).
1392 */
1393 fudge = start - entry->start;
1394 } else {
1395 /*
1396 * normal reference: we clip the map to fit (thus
1397 * fudge is zero)
1398 */
1399 UVM_MAP_CLIP_START(srcmap, entry, start);
1400 SAVE_HINT(srcmap, entry->prev);
1401 fudge = 0;
1402 }
1403
1404 } else {
1405
1406 /* "start" is not within an entry ... skip to next entry */
1407 if (flags & UVM_EXTRACT_CONTIG) {
1408 error = EINVAL;
1409 goto bad; /* definite hole here ... */
1410 }
1411
1412 entry = entry->next;
1413 fudge = 0;
1414 }
1415 /* save values from srcmap for step 6 */
1416 orig_entry = entry;
1417 orig_fudge = fudge;
1418
1419
1420 /*
1421 * step 3: now start looping through the map entries, extracting
1422 * as we go.
1423 */
1424
1425 while (entry->start < end && entry != &srcmap->header) {
1426
1427 /* if we are not doing a quick reference, clip it */
1428 if ((flags & UVM_EXTRACT_QREF) == 0)
1429 UVM_MAP_CLIP_END(srcmap, entry, end);
1430
1431 /* clear needs_copy (allow chunking) */
1432 if (UVM_ET_ISNEEDSCOPY(entry)) {
1433 if (fudge)
1434 oldstart = entry->start;
1435 else
1436 oldstart = 0; /* XXX: gcc */
1437 amap_copy(srcmap, entry, M_NOWAIT, TRUE, start, end);
1438 if (UVM_ET_ISNEEDSCOPY(entry)) { /* failed? */
1439 error = ENOMEM;
1440 goto bad;
1441 }
1442 /* amap_copy could clip (during chunk)! update fudge */
1443 if (fudge) {
1444 fudge = fudge - (entry->start - oldstart);
1445 orig_fudge = fudge;
1446 }
1447 }
1448
1449 /* calculate the offset of this from "start" */
1450 oldoffset = (entry->start + fudge) - start;
1451
1452 /* allocate a new map entry */
1453 newentry = uvm_mapent_alloc(dstmap);
1454 if (newentry == NULL) {
1455 error = ENOMEM;
1456 goto bad;
1457 }
1458
1459 /* set up new map entry */
1460 newentry->next = NULL;
1461 newentry->prev = endchain;
1462 newentry->start = dstaddr + oldoffset;
1463 newentry->end =
1464 newentry->start + (entry->end - (entry->start + fudge));
1465 if (newentry->end > newend || newentry->end < newentry->start)
1466 newentry->end = newend;
1467 newentry->object.uvm_obj = entry->object.uvm_obj;
1468 if (newentry->object.uvm_obj) {
1469 if (newentry->object.uvm_obj->pgops->pgo_reference)
1470 newentry->object.uvm_obj->pgops->
1471 pgo_reference(newentry->object.uvm_obj);
1472 newentry->offset = entry->offset + fudge;
1473 } else {
1474 newentry->offset = 0;
1475 }
1476 newentry->etype = entry->etype;
1477 newentry->protection = (flags & UVM_EXTRACT_FIXPROT) ?
1478 entry->max_protection : entry->protection;
1479 newentry->max_protection = entry->max_protection;
1480 newentry->inheritance = entry->inheritance;
1481 newentry->wired_count = 0;
1482 newentry->aref.ar_amap = entry->aref.ar_amap;
1483 if (newentry->aref.ar_amap) {
1484 newentry->aref.ar_pageoff =
1485 entry->aref.ar_pageoff + (fudge >> PAGE_SHIFT);
1486 amap_ref(newentry, AMAP_SHARED |
1487 ((flags & UVM_EXTRACT_QREF) ? AMAP_REFALL : 0));
1488 } else {
1489 newentry->aref.ar_pageoff = 0;
1490 }
1491 newentry->advice = entry->advice;
1492
1493 /* now link it on the chain */
1494 nchain++;
1495 if (endchain == NULL) {
1496 chain = endchain = newentry;
1497 } else {
1498 endchain->next = newentry;
1499 endchain = newentry;
1500 }
1501
1502 /* end of 'while' loop! */
1503 if ((flags & UVM_EXTRACT_CONTIG) && entry->end < end &&
1504 (entry->next == &srcmap->header ||
1505 entry->next->start != entry->end)) {
1506 error = EINVAL;
1507 goto bad;
1508 }
1509 entry = entry->next;
1510 fudge = 0;
1511 }
1512
1513
1514 /*
1515 * step 4: close off chain (in format expected by uvm_map_replace)
1516 */
1517
1518 if (chain)
1519 chain->prev = endchain;
1520
1521
1522 /*
1523 * step 5: attempt to lock the dest map so we can pmap_copy.
1524 * note usage of copy_ok:
1525 * 1 => dstmap locked, pmap_copy ok, and we "replace" here (step 5)
1526 * 0 => dstmap unlocked, NO pmap_copy, and we will "replace" in step 7
1527 */
1528
1529 if (srcmap == dstmap || vm_map_lock_try(dstmap) == TRUE) {
1530
1531 copy_ok = 1;
1532 if (!uvm_map_replace(dstmap, dstaddr, dstaddr+len, chain,
1533 nchain)) {
1534 if (srcmap != dstmap)
1535 vm_map_unlock(dstmap);
1536 error = EIO;
1537 goto bad;
1538 }
1539
1540 } else {
1541
1542 copy_ok = 0;
1543 /* replace defered until step 7 */
1544
1545 }
1546
1547
1548 /*
1549 * step 6: traverse the srcmap a second time to do the following:
1550 * - if we got a lock on the dstmap do pmap_copy
1551 * - if UVM_EXTRACT_REMOVE remove the entries
1552 * we make use of orig_entry and orig_fudge (saved in step 2)
1553 */
1554
1555 if (copy_ok || (flags & UVM_EXTRACT_REMOVE)) {
1556
1557 /* purge possible stale hints from srcmap */
1558 if (flags & UVM_EXTRACT_REMOVE) {
1559 SAVE_HINT(srcmap, orig_entry->prev);
1560 if (srcmap->first_free->start >= start)
1561 srcmap->first_free = orig_entry->prev;
1562 }
1563
1564 entry = orig_entry;
1565 fudge = orig_fudge;
1566 deadentry = NULL; /* for UVM_EXTRACT_REMOVE */
1567
1568 while (entry->start < end && entry != &srcmap->header) {
1569
1570 if (copy_ok) {
1571 oldoffset = (entry->start + fudge) - start;
1572 elen = min(end, entry->end) - (entry->start + fudge);
1573 pmap_copy(dstmap->pmap, srcmap->pmap, dstaddr + oldoffset,
1574 elen, entry->start + fudge);
1575 }
1576
1577 /* we advance "entry" in the following if statement */
1578 if (flags & UVM_EXTRACT_REMOVE) {
1579 pmap_remove(srcmap->pmap, entry->start,
1580 entry->end);
1581 oldentry = entry; /* save entry */
1582 entry = entry->next; /* advance */
1583 uvm_map_entry_unlink(srcmap, oldentry);
1584 /* add to dead list */
1585 oldentry->next = deadentry;
1586 deadentry = oldentry;
1587 } else {
1588 entry = entry->next; /* advance */
1589 }
1590
1591 /* end of 'while' loop */
1592 fudge = 0;
1593 }
1594
1595 /*
1596 * unlock dstmap. we will dispose of deadentry in
1597 * step 7 if needed
1598 */
1599 if (copy_ok && srcmap != dstmap)
1600 vm_map_unlock(dstmap);
1601
1602 }
1603 else
1604 deadentry = NULL; /* XXX: gcc */
1605
1606 /*
1607 * step 7: we are done with the source map, unlock. if copy_ok
1608 * is 0 then we have not replaced the dummy mapping in dstmap yet
1609 * and we need to do so now.
1610 */
1611
1612 vm_map_unlock(srcmap);
1613 if ((flags & UVM_EXTRACT_REMOVE) && deadentry)
1614 uvm_unmap_detach(deadentry, 0); /* dispose of old entries */
1615
1616 /* now do the replacement if we didn't do it in step 5 */
1617 if (copy_ok == 0) {
1618 vm_map_lock(dstmap);
1619 error = uvm_map_replace(dstmap, dstaddr, dstaddr+len, chain,
1620 nchain);
1621 vm_map_unlock(dstmap);
1622
1623 if (error == FALSE) {
1624 error = EIO;
1625 goto bad2;
1626 }
1627 }
1628
1629 /*
1630 * done!
1631 */
1632 return(0);
1633
1634 /*
1635 * bad: failure recovery
1636 */
1637 bad:
1638 vm_map_unlock(srcmap);
1639 bad2: /* src already unlocked */
1640 if (chain)
1641 uvm_unmap_detach(chain,
1642 (flags & UVM_EXTRACT_QREF) ? AMAP_REFALL : 0);
1643 uvm_unmap(dstmap, dstaddr, dstaddr+len); /* ??? */
1644 return(error);
1645 }
1646
1647 /* end of extraction functions */
1648
1649 /*
1650 * uvm_map_submap: punch down part of a map into a submap
1651 *
1652 * => only the kernel_map is allowed to be submapped
1653 * => the purpose of submapping is to break up the locking granularity
1654 * of a larger map
1655 * => the range specified must have been mapped previously with a uvm_map()
1656 * call [with uobj==NULL] to create a blank map entry in the main map.
1657 * [And it had better still be blank!]
1658 * => maps which contain submaps should never be copied or forked.
1659 * => to remove a submap, use uvm_unmap() on the main map
1660 * and then uvm_map_deallocate() the submap.
1661 * => main map must be unlocked.
1662 * => submap must have been init'd and have a zero reference count.
1663 * [need not be locked as we don't actually reference it]
1664 */
1665
1666 int
1667 uvm_map_submap(map, start, end, submap)
1668 vm_map_t map, submap;
1669 vaddr_t start, end;
1670 {
1671 vm_map_entry_t entry;
1672 int result;
1673 UVMHIST_FUNC("uvm_map_submap"); UVMHIST_CALLED(maphist);
1674
1675 vm_map_lock(map);
1676
1677 VM_MAP_RANGE_CHECK(map, start, end);
1678
1679 if (uvm_map_lookup_entry(map, start, &entry)) {
1680 UVM_MAP_CLIP_START(map, entry, start);
1681 UVM_MAP_CLIP_END(map, entry, end); /* to be safe */
1682 }
1683 else {
1684 entry = NULL;
1685 }
1686
1687 if (entry != NULL &&
1688 entry->start == start && entry->end == end &&
1689 entry->object.uvm_obj == NULL && entry->aref.ar_amap == NULL &&
1690 !UVM_ET_ISCOPYONWRITE(entry) && !UVM_ET_ISNEEDSCOPY(entry)) {
1691
1692 /*
1693 * doit!
1694 */
1695 entry->etype |= UVM_ET_SUBMAP;
1696 entry->object.sub_map = submap;
1697 entry->offset = 0;
1698 uvm_map_reference(submap);
1699 result = KERN_SUCCESS;
1700 } else {
1701 result = KERN_INVALID_ARGUMENT;
1702 }
1703 vm_map_unlock(map);
1704
1705 return(result);
1706 }
1707
1708
1709 /*
1710 * uvm_map_protect: change map protection
1711 *
1712 * => set_max means set max_protection.
1713 * => map must be unlocked.
1714 */
1715
1716 #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \
1717 ~VM_PROT_WRITE : VM_PROT_ALL)
1718 #define max(a,b) ((a) > (b) ? (a) : (b))
1719
1720 int
1721 uvm_map_protect(map, start, end, new_prot, set_max)
1722 vm_map_t map;
1723 vaddr_t start, end;
1724 vm_prot_t new_prot;
1725 boolean_t set_max;
1726 {
1727 vm_map_entry_t current, entry;
1728 UVMHIST_FUNC("uvm_map_protect"); UVMHIST_CALLED(maphist);
1729 UVMHIST_LOG(maphist,"(map=0x%x,start=0x%x,end=0x%x,new_prot=0x%x)",
1730 map, start, end, new_prot);
1731
1732 vm_map_lock(map);
1733
1734 VM_MAP_RANGE_CHECK(map, start, end);
1735
1736 if (uvm_map_lookup_entry(map, start, &entry)) {
1737 UVM_MAP_CLIP_START(map, entry, start);
1738 } else {
1739 entry = entry->next;
1740 }
1741
1742 /*
1743 * make a first pass to check for protection violations.
1744 */
1745
1746 current = entry;
1747 while ((current != &map->header) && (current->start < end)) {
1748 if (UVM_ET_ISSUBMAP(current))
1749 return (KERN_INVALID_ARGUMENT);
1750 if ((new_prot & current->max_protection) != new_prot) {
1751 vm_map_unlock(map);
1752 return (KERN_PROTECTION_FAILURE);
1753 }
1754 current = current->next;
1755 }
1756
1757 /* go back and fix up protections (no need to clip this time). */
1758
1759 current = entry;
1760
1761 while ((current != &map->header) && (current->start < end)) {
1762 vm_prot_t old_prot;
1763
1764 UVM_MAP_CLIP_END(map, current, end);
1765
1766 old_prot = current->protection;
1767 if (set_max)
1768 current->protection =
1769 (current->max_protection = new_prot) & old_prot;
1770 else
1771 current->protection = new_prot;
1772
1773 /*
1774 * update physical map if necessary. worry about copy-on-write
1775 * here -- CHECK THIS XXX
1776 */
1777
1778 if (current->protection != old_prot) {
1779
1780 /* update pmap! */
1781 pmap_protect(map->pmap, current->start, current->end,
1782 current->protection & MASK(entry));
1783
1784 }
1785 current = current->next;
1786 }
1787
1788 vm_map_unlock(map);
1789 UVMHIST_LOG(maphist, "<- done",0,0,0,0);
1790 return(KERN_SUCCESS);
1791 }
1792
1793 #undef max
1794 #undef MASK
1795
1796 /*
1797 * uvm_map_inherit: set inheritance code for range of addrs in map.
1798 *
1799 * => map must be unlocked
1800 * => note that the inherit code is used during a "fork". see fork
1801 * code for details.
1802 */
1803
1804 int
1805 uvm_map_inherit(map, start, end, new_inheritance)
1806 vm_map_t map;
1807 vaddr_t start;
1808 vaddr_t end;
1809 vm_inherit_t new_inheritance;
1810 {
1811 vm_map_entry_t entry, temp_entry;
1812 UVMHIST_FUNC("uvm_map_inherit"); UVMHIST_CALLED(maphist);
1813 UVMHIST_LOG(maphist,"(map=0x%x,start=0x%x,end=0x%x,new_inh=0x%x)",
1814 map, start, end, new_inheritance);
1815
1816 switch (new_inheritance) {
1817 case VM_INHERIT_NONE:
1818 case VM_INHERIT_COPY:
1819 case VM_INHERIT_SHARE:
1820 break;
1821 default:
1822 UVMHIST_LOG(maphist,"<- done (INVALID ARG)",0,0,0,0);
1823 return (KERN_INVALID_ARGUMENT);
1824 }
1825
1826 vm_map_lock(map);
1827
1828 VM_MAP_RANGE_CHECK(map, start, end);
1829
1830 if (uvm_map_lookup_entry(map, start, &temp_entry)) {
1831 entry = temp_entry;
1832 UVM_MAP_CLIP_START(map, entry, start);
1833 } else {
1834 entry = temp_entry->next;
1835 }
1836
1837 while ((entry != &map->header) && (entry->start < end)) {
1838 UVM_MAP_CLIP_END(map, entry, end);
1839
1840 entry->inheritance = new_inheritance;
1841
1842 entry = entry->next;
1843 }
1844
1845 vm_map_unlock(map);
1846 UVMHIST_LOG(maphist,"<- done (OK)",0,0,0,0);
1847 return(KERN_SUCCESS);
1848 }
1849
1850 /*
1851 * uvm_map_pageable: sets the pageability of a range in a map.
1852 *
1853 * => regions sepcified as not pageable require lock-down (wired) memory
1854 * and page tables.
1855 * => map must not be locked.
1856 * => XXXCDC: check this and try and clean it up.
1857 */
1858
1859 int
1860 uvm_map_pageable(map, start, end, new_pageable)
1861 vm_map_t map;
1862 vaddr_t start, end;
1863 boolean_t new_pageable;
1864 {
1865 vm_map_entry_t entry, start_entry;
1866 vaddr_t failed = 0;
1867 int rv;
1868 UVMHIST_FUNC("uvm_map_pageable"); UVMHIST_CALLED(maphist);
1869 UVMHIST_LOG(maphist,"(map=0x%x,start=0x%x,end=0x%x,new_pageable=0x%x)",
1870 map, start, end, new_pageable);
1871
1872 vm_map_lock(map);
1873 VM_MAP_RANGE_CHECK(map, start, end);
1874
1875 /*
1876 * only one pageability change may take place at one time, since
1877 * uvm_fault_wire assumes it will be called only once for each
1878 * wiring/unwiring. therefore, we have to make sure we're actually
1879 * changing the pageability for the entire region. we do so before
1880 * making any changes.
1881 */
1882
1883 if (uvm_map_lookup_entry(map, start, &start_entry) == FALSE) {
1884 vm_map_unlock(map);
1885
1886 UVMHIST_LOG(maphist,"<- done (INVALID ARG)",0,0,0,0);
1887 return (KERN_INVALID_ADDRESS);
1888 }
1889 entry = start_entry;
1890
1891 /*
1892 * handle wiring and unwiring seperately.
1893 */
1894
1895 if (new_pageable) { /* unwire */
1896
1897 UVM_MAP_CLIP_START(map, entry, start);
1898
1899 /*
1900 * unwiring. first ensure that the range to be unwired is
1901 * really wired down and that there are no holes.
1902 */
1903 while ((entry != &map->header) && (entry->start < end)) {
1904
1905 if (entry->wired_count == 0 ||
1906 (entry->end < end &&
1907 (entry->next == &map->header ||
1908 entry->next->start > entry->end))) {
1909 vm_map_unlock(map);
1910 UVMHIST_LOG(maphist,
1911 "<- done (INVALID UNWIRE ARG)",0,0,0,0);
1912 return (KERN_INVALID_ARGUMENT);
1913 }
1914 entry = entry->next;
1915 }
1916
1917 /*
1918 * now decrement the wiring count for each region. if a region
1919 * becomes completely unwired, unwire its physical pages and
1920 * mappings.
1921 */
1922 #if 0 /* not necessary: uvm_fault_unwire does not lock */
1923 lock_set_recursive(&map->lock);
1924 #endif /* XXXCDC */
1925
1926 entry = start_entry;
1927 while ((entry != &map->header) && (entry->start < end)) {
1928 UVM_MAP_CLIP_END(map, entry, end);
1929
1930 entry->wired_count--;
1931 if (entry->wired_count == 0)
1932 uvm_map_entry_unwire(map, entry);
1933
1934 entry = entry->next;
1935 }
1936 #if 0 /* XXXCDC: not necessary, see above */
1937 lock_clear_recursive(&map->lock);
1938 #endif
1939 vm_map_unlock(map);
1940 UVMHIST_LOG(maphist,"<- done (OK UNWIRE)",0,0,0,0);
1941 return(KERN_SUCCESS);
1942
1943 /*
1944 * end of unwire case!
1945 */
1946 }
1947
1948 /*
1949 * wire case: in two passes [XXXCDC: ugly block of code here]
1950 *
1951 * 1: holding the write lock, we create any anonymous maps that need
1952 * to be created. then we clip each map entry to the region to
1953 * be wired and increment its wiring count.
1954 *
1955 * 2: we downgrade to a read lock, and call uvm_fault_wire to fault
1956 * in the pages for any newly wired area (wired_count is 1).
1957 *
1958 * downgrading to a read lock for uvm_fault_wire avoids a possible
1959 * deadlock with another thread that may have faulted on one of
1960 * the pages to be wired (it would mark the page busy, blocking
1961 * us, then in turn block on the map lock that we hold). because
1962 * of problems in the recursive lock package, we cannot upgrade
1963 * to a write lock in vm_map_lookup. thus, any actions that
1964 * require the write lock must be done beforehand. because we
1965 * keep the read lock on the map, the copy-on-write status of the
1966 * entries we modify here cannot change.
1967 */
1968
1969 while ((entry != &map->header) && (entry->start < end)) {
1970
1971 if (entry->wired_count == 0) { /* not already wired? */
1972
1973 /*
1974 * perform actions of vm_map_lookup that need the
1975 * write lock on the map: create an anonymous map
1976 * for a copy-on-write region, or an anonymous map
1977 * for a zero-fill region. (XXXCDC: submap case
1978 * ok?)
1979 */
1980
1981 if (!UVM_ET_ISSUBMAP(entry)) { /* not submap */
1982 /*
1983 * XXXCDC: protection vs. max_protection??
1984 * (wirefault uses max?)
1985 * XXXCDC: used to do it always if
1986 * uvm_obj == NULL (wrong?)
1987 */
1988 if ( UVM_ET_ISNEEDSCOPY(entry) &&
1989 (entry->protection & VM_PROT_WRITE) != 0) {
1990 amap_copy(map, entry, M_WAITOK, TRUE,
1991 start, end);
1992 /* XXXCDC: wait OK? */
1993 }
1994 }
1995 } /* wired_count == 0 */
1996 UVM_MAP_CLIP_START(map, entry, start);
1997 UVM_MAP_CLIP_END(map, entry, end);
1998 entry->wired_count++;
1999
2000 /*
2001 * Check for holes
2002 */
2003 if (entry->end < end && (entry->next == &map->header ||
2004 entry->next->start > entry->end)) {
2005 /*
2006 * found one. amap creation actions do not need to
2007 * be undone, but the wired counts need to be restored.
2008 */
2009 while (entry != &map->header && entry->end > start) {
2010 entry->wired_count--;
2011 entry = entry->prev;
2012 }
2013 vm_map_unlock(map);
2014 UVMHIST_LOG(maphist,"<- done (INVALID WIRE)",0,0,0,0);
2015 return (KERN_INVALID_ARGUMENT);
2016 }
2017 entry = entry->next;
2018 }
2019
2020 /*
2021 * Pass 2.
2022 */
2023 /*
2024 * HACK HACK HACK HACK
2025 *
2026 * if we are wiring in the kernel map or a submap of it, unlock the
2027 * map to avoid deadlocks. we trust that the kernel threads are
2028 * well-behaved, and therefore will not do anything destructive to
2029 * this region of the map while we have it unlocked. we cannot
2030 * trust user threads to do the same.
2031 *
2032 * HACK HACK HACK HACK
2033 */
2034 if (vm_map_pmap(map) == pmap_kernel()) {
2035 vm_map_unlock(map); /* trust me ... */
2036 } else {
2037 vm_map_set_recursive(&map->lock);
2038 lockmgr(&map->lock, LK_DOWNGRADE, (void *)0);
2039 }
2040
2041 rv = 0;
2042 entry = start_entry;
2043 while (entry != &map->header && entry->start < end) {
2044 /*
2045 * if uvm_fault_wire fails for any page we need to undo what has
2046 * been done. we decrement the wiring count for those pages
2047 * which have not yet been wired (now) and unwire those that
2048 * have * (later).
2049 *
2050 * XXX this violates the locking protocol on the map, needs to
2051 * be fixed. [because we only have a read lock on map we
2052 * shouldn't be changing wired_count?]
2053 */
2054 if (rv) {
2055 entry->wired_count--;
2056 } else if (entry->wired_count == 1) {
2057 rv = uvm_fault_wire(map, entry->start, entry->end);
2058 if (rv) {
2059 failed = entry->start;
2060 entry->wired_count--;
2061 }
2062 }
2063 entry = entry->next;
2064 }
2065
2066 if (vm_map_pmap(map) == pmap_kernel()) {
2067 vm_map_lock(map); /* relock */
2068 } else {
2069 vm_map_clear_recursive(&map->lock);
2070 }
2071
2072 if (rv) { /* failed? */
2073 vm_map_unlock(map);
2074 (void) uvm_map_pageable(map, start, failed, TRUE);
2075 UVMHIST_LOG(maphist, "<- done (RV=%d)", rv,0,0,0);
2076 return(rv);
2077 }
2078 vm_map_unlock(map);
2079
2080 UVMHIST_LOG(maphist,"<- done (OK WIRE)",0,0,0,0);
2081 return(KERN_SUCCESS);
2082 }
2083
2084 /*
2085 * uvm_map_clean: push dirty pages off to backing store.
2086 *
2087 * => valid flags:
2088 * if (flags & PGO_SYNCIO): dirty pages are written synchronously
2089 * if (flags & PGO_DEACTIVATE): any cached pages are deactivated after clean
2090 * if (flags & PGO_FREE): any cached pages are freed after clean
2091 * => returns an error if any part of the specified range isn't mapped
2092 * => never a need to flush amap layer since the anonymous memory has
2093 * no permanent home...
2094 * => called from sys_msync()
2095 * => caller must not write-lock map (read OK).
2096 * => we may sleep while cleaning if SYNCIO [with map read-locked]
2097 */
2098
2099 int
2100 uvm_map_clean(map, start, end, flags)
2101 vm_map_t map;
2102 vaddr_t start, end;
2103 int flags;
2104 {
2105 vm_map_entry_t current;
2106 vm_map_entry_t entry;
2107 vsize_t size;
2108 struct uvm_object *object;
2109 vaddr_t offset;
2110 UVMHIST_FUNC("uvm_map_clean"); UVMHIST_CALLED(maphist);
2111 UVMHIST_LOG(maphist,"(map=0x%x,start=0x%x,end=0x%x,flags=0x%x)",
2112 map, start, end, flags);
2113
2114 vm_map_lock_read(map);
2115 VM_MAP_RANGE_CHECK(map, start, end);
2116 if (!uvm_map_lookup_entry(map, start, &entry)) {
2117 vm_map_unlock_read(map);
2118 return(KERN_INVALID_ADDRESS);
2119 }
2120
2121 /*
2122 * Make a first pass to check for holes.
2123 */
2124 for (current = entry; current->start < end; current = current->next) {
2125 if (UVM_ET_ISSUBMAP(current)) {
2126 vm_map_unlock_read(map);
2127 return (KERN_INVALID_ARGUMENT);
2128 }
2129 if (end > current->end && (current->next == &map->header ||
2130 current->end != current->next->start)) {
2131 vm_map_unlock_read(map);
2132 return (KERN_INVALID_ADDRESS);
2133 }
2134 }
2135
2136 /*
2137 * add "cleanit" flag to flags (for generic flush routine).
2138 * then make a second pass, cleaning/uncaching pages from
2139 * the indicated objects as we go.
2140 */
2141 flags = flags | PGO_CLEANIT;
2142 for (current = entry; current->start < end; current = current->next) {
2143 offset = current->offset + (start - current->start);
2144 size = (end <= current->end ? end : current->end) - start;
2145
2146 /*
2147 * get object/offset. can't be submap (checked above).
2148 */
2149 object = current->object.uvm_obj;
2150 simple_lock(&object->vmobjlock);
2151
2152 /*
2153 * flush pages if we've got a valid backing object.
2154 * note that object is locked.
2155 * XXX should we continue on an error?
2156 */
2157
2158 if (object && object->pgops) {
2159 if (!object->pgops->pgo_flush(object, offset,
2160 offset+size, flags)) {
2161 simple_unlock(&object->vmobjlock);
2162 vm_map_unlock_read(map);
2163 return (KERN_FAILURE);
2164 }
2165 }
2166 simple_unlock(&object->vmobjlock);
2167 start += size;
2168 }
2169 vm_map_unlock_read(map);
2170 return(KERN_SUCCESS);
2171 }
2172
2173
2174 /*
2175 * uvm_map_checkprot: check protection in map
2176 *
2177 * => must allow specified protection in a fully allocated region.
2178 * => map must be read or write locked by caller.
2179 */
2180
2181 boolean_t
2182 uvm_map_checkprot(map, start, end, protection)
2183 vm_map_t map;
2184 vaddr_t start, end;
2185 vm_prot_t protection;
2186 {
2187 vm_map_entry_t entry;
2188 vm_map_entry_t tmp_entry;
2189
2190 if (!uvm_map_lookup_entry(map, start, &tmp_entry)) {
2191 return(FALSE);
2192 }
2193
2194 entry = tmp_entry;
2195
2196 while (start < end) {
2197 if (entry == &map->header) {
2198 return(FALSE);
2199 }
2200
2201 /*
2202 * no holes allowed
2203 */
2204
2205 if (start < entry->start) {
2206 return(FALSE);
2207 }
2208
2209 /*
2210 * check protection associated with entry
2211 */
2212
2213 if ((entry->protection & protection) != protection) {
2214 return(FALSE);
2215 }
2216
2217 /* go to next entry */
2218
2219 start = entry->end;
2220 entry = entry->next;
2221 }
2222 return(TRUE);
2223 }
2224
2225 /*
2226 * uvmspace_alloc: allocate a vmspace structure.
2227 *
2228 * - structure includes vm_map and pmap
2229 * - XXX: no locking on this structure
2230 * - refcnt set to 1, rest must be init'd by caller
2231 */
2232 struct vmspace *
2233 uvmspace_alloc(min, max, pageable)
2234 vaddr_t min, max;
2235 int pageable;
2236 {
2237 struct vmspace *vm;
2238 UVMHIST_FUNC("uvmspace_alloc"); UVMHIST_CALLED(maphist);
2239
2240 vm = pool_get(&uvm_vmspace_pool, PR_WAITOK);
2241 uvmspace_init(vm, NULL, min, max, pageable);
2242 UVMHIST_LOG(maphist,"<- done (vm=0x%x)", vm,0,0,0);
2243 return (vm);
2244 }
2245
2246 /*
2247 * uvmspace_init: initialize a vmspace structure.
2248 *
2249 * - XXX: no locking on this structure
2250 * - refcnt set to 1, rest must me init'd by caller
2251 */
2252 void
2253 uvmspace_init(vm, pmap, min, max, pageable)
2254 struct vmspace *vm;
2255 struct pmap *pmap;
2256 vaddr_t min, max;
2257 boolean_t pageable;
2258 {
2259 UVMHIST_FUNC("uvmspace_init"); UVMHIST_CALLED(maphist);
2260
2261 memset(vm, 0, sizeof(*vm));
2262
2263 uvm_map_setup(&vm->vm_map, min, max, pageable);
2264
2265 if (pmap)
2266 pmap_reference(pmap);
2267 else
2268 #if defined(PMAP_NEW)
2269 pmap = pmap_create();
2270 #else
2271 pmap = pmap_create(0);
2272 #endif
2273 vm->vm_map.pmap = pmap;
2274
2275 vm->vm_refcnt = 1;
2276 UVMHIST_LOG(maphist,"<- done",0,0,0,0);
2277 }
2278
2279 /*
2280 * uvmspace_share: share a vmspace between two proceses
2281 *
2282 * - XXX: no locking on vmspace
2283 * - used for vfork, threads(?)
2284 */
2285
2286 void
2287 uvmspace_share(p1, p2)
2288 struct proc *p1, *p2;
2289 {
2290 p2->p_vmspace = p1->p_vmspace;
2291 p1->p_vmspace->vm_refcnt++;
2292 }
2293
2294 /*
2295 * uvmspace_unshare: ensure that process "p" has its own, unshared, vmspace
2296 *
2297 * - XXX: no locking on vmspace
2298 */
2299
2300 void
2301 uvmspace_unshare(p)
2302 struct proc *p;
2303 {
2304 struct vmspace *nvm, *ovm = p->p_vmspace;
2305 int s;
2306
2307 if (ovm->vm_refcnt == 1)
2308 /* nothing to do: vmspace isn't shared in the first place */
2309 return;
2310
2311 /* make a new vmspace, still holding old one */
2312 nvm = uvmspace_fork(ovm);
2313
2314 s = splhigh(); /* make this `atomic' */
2315 pmap_deactivate(p); /* unbind old vmspace */
2316 p->p_vmspace = nvm;
2317 pmap_activate(p); /* switch to new vmspace */
2318 splx(s); /* end of critical section */
2319
2320 uvmspace_free(ovm); /* drop reference to old vmspace */
2321 }
2322
2323 /*
2324 * uvmspace_exec: the process wants to exec a new program
2325 *
2326 * - XXX: no locking on vmspace
2327 */
2328
2329 void
2330 uvmspace_exec(p)
2331 struct proc *p;
2332 {
2333 struct vmspace *nvm, *ovm = p->p_vmspace;
2334 vm_map_t map = &ovm->vm_map;
2335 int s;
2336
2337 #ifdef sparc
2338 /* XXX cgd 960926: the sparc #ifdef should be a MD hook */
2339 kill_user_windows(p); /* before stack addresses go away */
2340 #endif
2341
2342 /*
2343 * see if more than one process is using this vmspace...
2344 */
2345
2346 if (ovm->vm_refcnt == 1) {
2347
2348 /*
2349 * if p is the only process using its vmspace then we can safely
2350 * recycle that vmspace for the program that is being exec'd.
2351 */
2352
2353 #ifdef SYSVSHM
2354 /*
2355 * SYSV SHM semantics require us to kill all segments on an exec
2356 */
2357 if (ovm->vm_shm)
2358 shmexit(ovm);
2359 #endif
2360
2361 /*
2362 * now unmap the old program
2363 */
2364 uvm_unmap(map, VM_MIN_ADDRESS, VM_MAXUSER_ADDRESS);
2365
2366 } else {
2367
2368 /*
2369 * p's vmspace is being shared, so we can't reuse it for p since
2370 * it is still being used for others. allocate a new vmspace
2371 * for p
2372 */
2373 nvm = uvmspace_alloc(map->min_offset, map->max_offset,
2374 map->entries_pageable);
2375
2376 #if (defined(i386) || defined(pc532)) && !defined(PMAP_NEW)
2377 /*
2378 * allocate zero fill area in the new vmspace's map for user
2379 * page tables for ports that have old style pmaps that keep
2380 * user page tables in the top part of the process' address
2381 * space.
2382 *
2383 * XXXCDC: this should go away once all pmaps are fixed
2384 */
2385 {
2386 vaddr_t addr = VM_MAXUSER_ADDRESS;
2387 if (uvm_map(&nvm->vm_map, &addr, VM_MAX_ADDRESS - addr,
2388 NULL, UVM_UNKNOWN_OFFSET, UVM_MAPFLAG(UVM_PROT_ALL,
2389 UVM_PROT_ALL, UVM_INH_NONE, UVM_ADV_NORMAL,
2390 UVM_FLAG_FIXED|UVM_FLAG_COPYONW)) != KERN_SUCCESS)
2391 panic("vm_allocate of PT page area failed");
2392 }
2393 #endif
2394
2395 /*
2396 * install new vmspace and drop our ref to the old one.
2397 */
2398
2399 s = splhigh();
2400 pmap_deactivate(p);
2401 p->p_vmspace = nvm;
2402 pmap_activate(p);
2403 splx(s);
2404
2405 uvmspace_free(ovm);
2406 }
2407 }
2408
2409 /*
2410 * uvmspace_free: free a vmspace data structure
2411 *
2412 * - XXX: no locking on vmspace
2413 */
2414
2415 void
2416 uvmspace_free(vm)
2417 struct vmspace *vm;
2418 {
2419 vm_map_entry_t dead_entries;
2420 UVMHIST_FUNC("uvmspace_free"); UVMHIST_CALLED(maphist);
2421
2422 UVMHIST_LOG(maphist,"(vm=0x%x) ref=%d", vm, vm->vm_refcnt,0,0);
2423 if (--vm->vm_refcnt == 0) {
2424 /*
2425 * lock the map, to wait out all other references to it. delete
2426 * all of the mappings and pages they hold, then call the pmap
2427 * module to reclaim anything left.
2428 */
2429 vm_map_lock(&vm->vm_map);
2430 if (vm->vm_map.nentries) {
2431 (void)uvm_unmap_remove(&vm->vm_map,
2432 vm->vm_map.min_offset, vm->vm_map.max_offset,
2433 &dead_entries);
2434 if (dead_entries != NULL)
2435 uvm_unmap_detach(dead_entries, 0);
2436 }
2437 pmap_destroy(vm->vm_map.pmap);
2438 vm->vm_map.pmap = NULL;
2439 pool_put(&uvm_vmspace_pool, vm);
2440 }
2441 UVMHIST_LOG(maphist,"<- done", 0,0,0,0);
2442 }
2443
2444 /*
2445 * F O R K - m a i n e n t r y p o i n t
2446 */
2447 /*
2448 * uvmspace_fork: fork a process' main map
2449 *
2450 * => create a new vmspace for child process from parent.
2451 * => parent's map must not be locked.
2452 */
2453
2454 struct vmspace *
2455 uvmspace_fork(vm1)
2456 struct vmspace *vm1;
2457 {
2458 struct vmspace *vm2;
2459 vm_map_t old_map = &vm1->vm_map;
2460 vm_map_t new_map;
2461 vm_map_entry_t old_entry;
2462 vm_map_entry_t new_entry;
2463 pmap_t new_pmap;
2464 boolean_t protect_child;
2465 UVMHIST_FUNC("uvmspace_fork"); UVMHIST_CALLED(maphist);
2466
2467 #if (defined(i386) || defined(pc532)) && !defined(PMAP_NEW)
2468 /*
2469 * avoid copying any of the parent's pagetables or other per-process
2470 * objects that reside in the map by marking all of them non-inheritable
2471 * XXXCDC: should go away
2472 */
2473 (void) uvm_map_inherit(old_map, VM_MAXUSER_ADDRESS, VM_MAX_ADDRESS,
2474 VM_INHERIT_NONE);
2475 #endif
2476
2477 vm_map_lock(old_map);
2478
2479 vm2 = uvmspace_alloc(old_map->min_offset, old_map->max_offset,
2480 old_map->entries_pageable);
2481 memcpy(&vm2->vm_startcopy, &vm1->vm_startcopy,
2482 (caddr_t) (vm1 + 1) - (caddr_t) &vm1->vm_startcopy);
2483 new_map = &vm2->vm_map; /* XXX */
2484 new_pmap = new_map->pmap;
2485
2486 old_entry = old_map->header.next;
2487
2488 /*
2489 * go entry-by-entry
2490 */
2491
2492 while (old_entry != &old_map->header) {
2493
2494 /*
2495 * first, some sanity checks on the old entry
2496 */
2497 if (UVM_ET_ISSUBMAP(old_entry))
2498 panic("fork: encountered a submap during fork (illegal)");
2499
2500 if (!UVM_ET_ISCOPYONWRITE(old_entry) &&
2501 UVM_ET_ISNEEDSCOPY(old_entry))
2502 panic("fork: non-copy_on_write map entry marked needs_copy (illegal)");
2503
2504
2505 switch (old_entry->inheritance) {
2506 case VM_INHERIT_NONE:
2507 /*
2508 * drop the mapping
2509 */
2510 break;
2511
2512 case VM_INHERIT_SHARE:
2513 /*
2514 * share the mapping: this means we want the old and
2515 * new entries to share amaps and backing objects.
2516 */
2517
2518 /*
2519 * if the old_entry needs a new amap (due to prev fork)
2520 * then we need to allocate it now so that we have
2521 * something we own to share with the new_entry. [in
2522 * other words, we need to clear needs_copy]
2523 */
2524
2525 if (UVM_ET_ISNEEDSCOPY(old_entry)) {
2526 /* get our own amap, clears needs_copy */
2527 amap_copy(old_map, old_entry, M_WAITOK, FALSE,
2528 0, 0);
2529 /* XXXCDC: WAITOK??? */
2530 }
2531
2532 new_entry = uvm_mapent_alloc(new_map);
2533 /* old_entry -> new_entry */
2534 uvm_mapent_copy(old_entry, new_entry);
2535
2536 /* new pmap has nothing wired in it */
2537 new_entry->wired_count = 0;
2538
2539 /*
2540 * gain reference to object backing the map (can't
2541 * be a submap, already checked this case).
2542 */
2543 if (new_entry->aref.ar_amap)
2544 /* share reference */
2545 amap_ref(new_entry, AMAP_SHARED);
2546
2547 if (new_entry->object.uvm_obj &&
2548 new_entry->object.uvm_obj->pgops->pgo_reference)
2549 new_entry->object.uvm_obj->
2550 pgops->pgo_reference(
2551 new_entry->object.uvm_obj);
2552
2553 /* insert entry at end of new_map's entry list */
2554 uvm_map_entry_link(new_map, new_map->header.prev,
2555 new_entry);
2556
2557 /*
2558 * pmap_copy the mappings: this routine is optional
2559 * but if it is there it will reduce the number of
2560 * page faults in the new proc.
2561 */
2562
2563 pmap_copy(new_pmap, old_map->pmap, new_entry->start,
2564 (old_entry->end - old_entry->start),
2565 old_entry->start);
2566
2567 break;
2568
2569 case VM_INHERIT_COPY:
2570
2571 /*
2572 * copy-on-write the mapping (using mmap's
2573 * MAP_PRIVATE semantics)
2574 *
2575 * allocate new_entry, adjust reference counts.
2576 * (note that new references are read-only).
2577 */
2578
2579 new_entry = uvm_mapent_alloc(new_map);
2580 /* old_entry -> new_entry */
2581 uvm_mapent_copy(old_entry, new_entry);
2582
2583 if (new_entry->aref.ar_amap)
2584 amap_ref(new_entry, 0);
2585
2586 if (new_entry->object.uvm_obj &&
2587 new_entry->object.uvm_obj->pgops->pgo_reference)
2588 new_entry->object.uvm_obj->pgops->pgo_reference
2589 (new_entry->object.uvm_obj);
2590
2591 /* new pmap has nothing wired in it */
2592 new_entry->wired_count = 0;
2593
2594 new_entry->etype |=
2595 (UVM_ET_COPYONWRITE|UVM_ET_NEEDSCOPY);
2596 uvm_map_entry_link(new_map, new_map->header.prev,
2597 new_entry);
2598
2599 /*
2600 * the new entry will need an amap. it will either
2601 * need to be copied from the old entry or created
2602 * from scratch (if the old entry does not have an
2603 * amap). can we defer this process until later
2604 * (by setting "needs_copy") or do we need to copy
2605 * the amap now?
2606 *
2607 * we must copy the amap now if any of the following
2608 * conditions hold:
2609 * 1. the old entry has an amap and that amap is
2610 * being shared. this means that the old (parent)
2611 * process is sharing the amap with another
2612 * process. if we do not clear needs_copy here
2613 * we will end up in a situation where both the
2614 * parent and child process are refering to the
2615 * same amap with "needs_copy" set. if the
2616 * parent write-faults, the fault routine will
2617 * clear "needs_copy" in the parent by allocating
2618 * a new amap. this is wrong because the
2619 * parent is supposed to be sharing the old amap
2620 * and the new amap will break that.
2621 *
2622 * 2. if the old entry has an amap and a non-zero
2623 * wire count then we are going to have to call
2624 * amap_cow_now to avoid page faults in the
2625 * parent process. since amap_cow_now requires
2626 * "needs_copy" to be clear we might as well
2627 * clear it here as well.
2628 *
2629 */
2630
2631 if (old_entry->aref.ar_amap != NULL) {
2632
2633 if ((amap_flags(old_entry->aref.ar_amap) &
2634 AMAP_SHARED) != 0 ||
2635 old_entry->wired_count != 0) {
2636
2637 amap_copy(new_map, new_entry, M_WAITOK, FALSE,
2638 0, 0);
2639 /* XXXCDC: M_WAITOK ... ok? */
2640 }
2641 }
2642
2643 /*
2644 * if the parent's entry is wired down, then the
2645 * parent process does not want page faults on
2646 * access to that memory. this means that we
2647 * cannot do copy-on-write because we can't write
2648 * protect the old entry. in this case we
2649 * resolve all copy-on-write faults now, using
2650 * amap_cow_now. note that we have already
2651 * allocated any needed amap (above).
2652 */
2653
2654 if (old_entry->wired_count != 0) {
2655
2656 /*
2657 * resolve all copy-on-write faults now
2658 * (note that there is nothing to do if
2659 * the old mapping does not have an amap).
2660 * XXX: is it worthwhile to bother with pmap_copy
2661 * in this case?
2662 */
2663 if (old_entry->aref.ar_amap)
2664 amap_cow_now(new_map, new_entry);
2665
2666 } else {
2667
2668 /*
2669 * setup mappings to trigger copy-on-write faults
2670 * we must write-protect the parent if it has
2671 * an amap and it is not already "needs_copy"...
2672 * if it is already "needs_copy" then the parent
2673 * has already been write-protected by a previous
2674 * fork operation.
2675 *
2676 * if we do not write-protect the parent, then
2677 * we must be sure to write-protect the child
2678 * after the pmap_copy() operation.
2679 *
2680 * XXX: pmap_copy should have some way of telling
2681 * us that it didn't do anything so we can avoid
2682 * calling pmap_protect needlessly.
2683 */
2684
2685 if (old_entry->aref.ar_amap) {
2686
2687 if (!UVM_ET_ISNEEDSCOPY(old_entry)) {
2688 if (old_entry->max_protection & VM_PROT_WRITE) {
2689 pmap_protect(old_map->pmap,
2690 old_entry->start,
2691 old_entry->end,
2692 old_entry->protection &
2693 ~VM_PROT_WRITE);
2694 }
2695 old_entry->etype |= UVM_ET_NEEDSCOPY;
2696 }
2697
2698 /*
2699 * parent must now be write-protected
2700 */
2701 protect_child = FALSE;
2702 } else {
2703
2704 /*
2705 * we only need to protect the child if the
2706 * parent has write access.
2707 */
2708 if (old_entry->max_protection & VM_PROT_WRITE)
2709 protect_child = TRUE;
2710 else
2711 protect_child = FALSE;
2712
2713 }
2714
2715 /*
2716 * copy the mappings
2717 * XXX: need a way to tell if this does anything
2718 */
2719
2720 pmap_copy(new_pmap, old_map->pmap,
2721 new_entry->start,
2722 (old_entry->end - old_entry->start),
2723 old_entry->start);
2724
2725 /*
2726 * protect the child's mappings if necessary
2727 */
2728 if (protect_child) {
2729 pmap_protect(new_pmap, new_entry->start,
2730 new_entry->end,
2731 new_entry->protection &
2732 ~VM_PROT_WRITE);
2733 }
2734
2735 }
2736 break;
2737 } /* end of switch statement */
2738 old_entry = old_entry->next;
2739 }
2740
2741 new_map->size = old_map->size;
2742 vm_map_unlock(old_map);
2743
2744 #if (defined(i386) || defined(pc532)) && !defined(PMAP_NEW)
2745 /*
2746 * allocate zero fill area in the new vmspace's map for user
2747 * page tables for ports that have old style pmaps that keep
2748 * user page tables in the top part of the process' address
2749 * space.
2750 *
2751 * XXXCDC: this should go away once all pmaps are fixed
2752 */
2753 {
2754 vaddr_t addr = VM_MAXUSER_ADDRESS;
2755 if (uvm_map(new_map, &addr, VM_MAX_ADDRESS - addr, NULL,
2756 UVM_UNKNOWN_OFFSET, UVM_MAPFLAG(UVM_PROT_ALL,
2757 UVM_PROT_ALL, UVM_INH_NONE, UVM_ADV_NORMAL,
2758 UVM_FLAG_FIXED|UVM_FLAG_COPYONW)) != KERN_SUCCESS)
2759 panic("vm_allocate of PT page area failed");
2760 }
2761 #endif
2762
2763 #ifdef SYSVSHM
2764 if (vm1->vm_shm)
2765 shmfork(vm1, vm2);
2766 #endif
2767
2768 #ifdef PMAP_FORK
2769 pmap_fork(vm1->vm_map.pmap, vm2->vm_map.pmap);
2770 #endif
2771
2772 UVMHIST_LOG(maphist,"<- done",0,0,0,0);
2773 return(vm2);
2774 }
2775
2776
2777 #if defined(DDB)
2778
2779 /*
2780 * DDB hooks
2781 */
2782
2783 /*
2784 * uvm_map_print: print out a map
2785 */
2786
2787 void
2788 uvm_map_print(map, full)
2789 vm_map_t map;
2790 boolean_t full;
2791 {
2792
2793 uvm_map_printit(map, full, printf);
2794 }
2795
2796 /*
2797 * uvm_map_printit: actually prints the map
2798 */
2799
2800 void
2801 uvm_map_printit(map, full, pr)
2802 vm_map_t map;
2803 boolean_t full;
2804 void (*pr) __P((const char *, ...));
2805 {
2806 vm_map_entry_t entry;
2807
2808 (*pr)("MAP %p: [0x%lx->0x%lx]\n", map, map->min_offset,map->max_offset);
2809 (*pr)("\t#ent=%d, sz=%d, ref=%d, version=%d\n",
2810 map->nentries, map->size, map->ref_count, map->timestamp);
2811 #ifdef pmap_resident_count
2812 (*pr)("\tpmap=%p(resident=%d)\n", map->pmap,
2813 pmap_resident_count(map->pmap));
2814 #else
2815 /* XXXCDC: this should be required ... */
2816 (*pr)("\tpmap=%p(resident=<<NOT SUPPORTED!!!>>)\n", map->pmap);
2817 #endif
2818 if (!full)
2819 return;
2820 for (entry = map->header.next; entry != &map->header;
2821 entry = entry->next) {
2822 (*pr)(" - %p: 0x%lx->0x%lx: obj=%p/0x%x, amap=%p/%d\n",
2823 entry, entry->start, entry->end, entry->object.uvm_obj,
2824 entry->offset, entry->aref.ar_amap, entry->aref.ar_pageoff);
2825 (*pr)(
2826 "\tsubmap=%c, cow=%c, nc=%c, prot(max)=%d/%d, inh=%d, wc=%d, adv=%d\n",
2827 (entry->etype & UVM_ET_SUBMAP) ? 'T' : 'F',
2828 (entry->etype & UVM_ET_COPYONWRITE) ? 'T' : 'F',
2829 (entry->etype & UVM_ET_NEEDSCOPY) ? 'T' : 'F',
2830 entry->protection, entry->max_protection,
2831 entry->inheritance, entry->wired_count, entry->advice);
2832 }
2833 }
2834
2835 /*
2836 * uvm_object_print: print out an object
2837 */
2838
2839 void
2840 uvm_object_print(uobj, full)
2841 struct uvm_object *uobj;
2842 boolean_t full;
2843 {
2844
2845 uvm_object_printit(uobj, full, printf);
2846 }
2847
2848 /*
2849 * uvm_object_printit: actually prints the object
2850 */
2851
2852 void
2853 uvm_object_printit(uobj, full, pr)
2854 struct uvm_object *uobj;
2855 boolean_t full;
2856 void (*pr) __P((const char *, ...));
2857 {
2858 struct vm_page *pg;
2859 int cnt = 0;
2860
2861 (*pr)("OBJECT %p: pgops=%p, npages=%d, ", uobj, uobj->pgops,
2862 uobj->uo_npages);
2863 if (uobj->uo_refs == UVM_OBJ_KERN)
2864 (*pr)("refs=<SYSTEM>\n");
2865 else
2866 (*pr)("refs=%d\n", uobj->uo_refs);
2867
2868 if (!full) return;
2869 (*pr)(" PAGES <pg,offset>:\n ");
2870 for (pg = uobj->memq.tqh_first ; pg ; pg = pg->listq.tqe_next, cnt++) {
2871 (*pr)("<%p,0x%lx> ", pg, pg->offset);
2872 if ((cnt % 3) == 2) (*pr)("\n ");
2873 }
2874 if ((cnt % 3) != 2) (*pr)("\n");
2875 }
2876
2877 /*
2878 * uvm_page_print: print out a page
2879 */
2880
2881 void
2882 uvm_page_print(pg, full)
2883 struct vm_page *pg;
2884 boolean_t full;
2885 {
2886
2887 uvm_page_printit(pg, full, printf);
2888 }
2889
2890 /*
2891 * uvm_page_printit: actually print the page
2892 */
2893
2894 void
2895 uvm_page_printit(pg, full, pr)
2896 struct vm_page *pg;
2897 boolean_t full;
2898 void (*pr) __P((const char *, ...));
2899 {
2900 struct vm_page *lcv;
2901 struct uvm_object *uobj;
2902 struct pglist *pgl;
2903
2904 (*pr)("PAGE %p:\n", pg);
2905 (*pr)(" flags=0x%x, pqflags=0x%x, vers=%d, wire_count=%d, pa=0x%lx\n",
2906 pg->flags, pg->pqflags, pg->version, pg->wire_count, (long)pg->phys_addr);
2907 (*pr)(" uobject=%p, uanon=%p, offset=0x%lx loan_count=%d\n",
2908 pg->uobject, pg->uanon, pg->offset, pg->loan_count);
2909 #if defined(UVM_PAGE_TRKOWN)
2910 if (pg->flags & PG_BUSY)
2911 (*pr)(" owning process = %d, tag=%s\n",
2912 pg->owner, pg->owner_tag);
2913 else
2914 (*pr)(" page not busy, no owner\n");
2915 #else
2916 (*pr)(" [page ownership tracking disabled]\n");
2917 #endif
2918
2919 if (!full)
2920 return;
2921
2922 /* cross-verify object/anon */
2923 if ((pg->pqflags & PQ_FREE) == 0) {
2924 if (pg->pqflags & PQ_ANON) {
2925 if (pg->uanon == NULL || pg->uanon->u.an_page != pg)
2926 (*pr)(" >>> ANON DOES NOT POINT HERE <<< (%p)\n",
2927 (pg->uanon) ? pg->uanon->u.an_page : NULL);
2928 else
2929 (*pr)(" anon backpointer is OK\n");
2930 } else {
2931 uobj = pg->uobject;
2932 if (uobj) {
2933 (*pr)(" checking object list\n");
2934 for (lcv = uobj->memq.tqh_first ; lcv ;
2935 lcv = lcv->listq.tqe_next) {
2936 if (lcv == pg) break;
2937 }
2938 if (lcv)
2939 (*pr)(" page found on object list\n");
2940 else
2941 (*pr)(" >>> PAGE NOT FOUND ON OBJECT LIST! <<<\n");
2942 }
2943 }
2944 }
2945
2946 /* cross-verify page queue */
2947 if (pg->pqflags & PQ_FREE)
2948 pgl = &uvm.page_free[uvm_page_lookup_freelist(pg)];
2949 else if (pg->pqflags & PQ_INACTIVE)
2950 pgl = (pg->pqflags & PQ_SWAPBACKED) ?
2951 &uvm.page_inactive_swp : &uvm.page_inactive_obj;
2952 else if (pg->pqflags & PQ_ACTIVE)
2953 pgl = &uvm.page_active;
2954 else
2955 pgl = NULL;
2956
2957 if (pgl) {
2958 (*pr)(" checking pageq list\n");
2959 for (lcv = pgl->tqh_first ; lcv ; lcv = lcv->pageq.tqe_next) {
2960 if (lcv == pg) break;
2961 }
2962 if (lcv)
2963 (*pr)(" page found on pageq list\n");
2964 else
2965 (*pr)(" >>> PAGE NOT FOUND ON PAGEQ LIST! <<<\n");
2966 }
2967 }
2968 #endif
2969