uvm_map.c revision 1.272 1 1.272 yamt /* $NetBSD: uvm_map.c,v 1.272 2009/08/01 15:30:33 yamt Exp $ */
2 1.1 mrg
3 1.98 chs /*
4 1.1 mrg * Copyright (c) 1997 Charles D. Cranor and Washington University.
5 1.98 chs * Copyright (c) 1991, 1993, The Regents of the University of California.
6 1.1 mrg *
7 1.1 mrg * All rights reserved.
8 1.1 mrg *
9 1.1 mrg * This code is derived from software contributed to Berkeley by
10 1.1 mrg * The Mach Operating System project at Carnegie-Mellon University.
11 1.1 mrg *
12 1.1 mrg * Redistribution and use in source and binary forms, with or without
13 1.1 mrg * modification, are permitted provided that the following conditions
14 1.1 mrg * are met:
15 1.1 mrg * 1. Redistributions of source code must retain the above copyright
16 1.1 mrg * notice, this list of conditions and the following disclaimer.
17 1.1 mrg * 2. Redistributions in binary form must reproduce the above copyright
18 1.1 mrg * notice, this list of conditions and the following disclaimer in the
19 1.1 mrg * documentation and/or other materials provided with the distribution.
20 1.1 mrg * 3. All advertising materials mentioning features or use of this software
21 1.1 mrg * must display the following acknowledgement:
22 1.1 mrg * This product includes software developed by Charles D. Cranor,
23 1.98 chs * Washington University, the University of California, Berkeley and
24 1.1 mrg * its contributors.
25 1.1 mrg * 4. Neither the name of the University nor the names of its contributors
26 1.1 mrg * may be used to endorse or promote products derived from this software
27 1.1 mrg * without specific prior written permission.
28 1.1 mrg *
29 1.1 mrg * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
30 1.1 mrg * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
31 1.1 mrg * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
32 1.1 mrg * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
33 1.1 mrg * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
34 1.1 mrg * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
35 1.1 mrg * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
36 1.1 mrg * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
37 1.1 mrg * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
38 1.1 mrg * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
39 1.1 mrg * SUCH DAMAGE.
40 1.1 mrg *
41 1.1 mrg * @(#)vm_map.c 8.3 (Berkeley) 1/12/94
42 1.3 mrg * from: Id: uvm_map.c,v 1.1.2.27 1998/02/07 01:16:54 chs Exp
43 1.1 mrg *
44 1.1 mrg *
45 1.1 mrg * Copyright (c) 1987, 1990 Carnegie-Mellon University.
46 1.1 mrg * All rights reserved.
47 1.98 chs *
48 1.1 mrg * Permission to use, copy, modify and distribute this software and
49 1.1 mrg * its documentation is hereby granted, provided that both the copyright
50 1.1 mrg * notice and this permission notice appear in all copies of the
51 1.1 mrg * software, derivative works or modified versions, and any portions
52 1.1 mrg * thereof, and that both notices appear in supporting documentation.
53 1.98 chs *
54 1.98 chs * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
55 1.98 chs * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
56 1.1 mrg * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
57 1.98 chs *
58 1.1 mrg * Carnegie Mellon requests users of this software to return to
59 1.1 mrg *
60 1.1 mrg * Software Distribution Coordinator or Software.Distribution (at) CS.CMU.EDU
61 1.1 mrg * School of Computer Science
62 1.1 mrg * Carnegie Mellon University
63 1.1 mrg * Pittsburgh PA 15213-3890
64 1.1 mrg *
65 1.1 mrg * any improvements or extensions that they make and grant Carnegie the
66 1.1 mrg * rights to redistribute these changes.
67 1.1 mrg */
68 1.1 mrg
69 1.114 lukem /*
70 1.114 lukem * uvm_map.c: uvm map operations
71 1.114 lukem */
72 1.114 lukem
73 1.114 lukem #include <sys/cdefs.h>
74 1.272 yamt __KERNEL_RCSID(0, "$NetBSD: uvm_map.c,v 1.272 2009/08/01 15:30:33 yamt Exp $");
75 1.114 lukem
76 1.21 jonathan #include "opt_ddb.h"
77 1.6 mrg #include "opt_uvmhist.h"
78 1.169 petrov #include "opt_uvm.h"
79 1.31 tron #include "opt_sysv.h"
80 1.1 mrg
81 1.1 mrg #include <sys/param.h>
82 1.1 mrg #include <sys/systm.h>
83 1.1 mrg #include <sys/mman.h>
84 1.1 mrg #include <sys/proc.h>
85 1.1 mrg #include <sys/malloc.h>
86 1.25 thorpej #include <sys/pool.h>
87 1.104 chs #include <sys/kernel.h>
88 1.112 thorpej #include <sys/mount.h>
89 1.109 thorpej #include <sys/vnode.h>
90 1.244 yamt #include <sys/lockdebug.h>
91 1.248 ad #include <sys/atomic.h>
92 1.1 mrg
93 1.1 mrg #ifdef SYSVSHM
94 1.1 mrg #include <sys/shm.h>
95 1.1 mrg #endif
96 1.1 mrg
97 1.1 mrg #include <uvm/uvm.h>
98 1.271 yamt #include <uvm/uvm_readahead.h>
99 1.21 jonathan
100 1.270 pooka #if defined(DDB) || defined(DEBUGPRINT)
101 1.21 jonathan #include <uvm/uvm_ddb.h>
102 1.21 jonathan #endif
103 1.21 jonathan
104 1.258 ad #if !defined(UVMMAP_COUNTERS)
105 1.207 yamt
106 1.207 yamt #define UVMMAP_EVCNT_DEFINE(name) /* nothing */
107 1.207 yamt #define UVMMAP_EVCNT_INCR(ev) /* nothing */
108 1.207 yamt #define UVMMAP_EVCNT_DECR(ev) /* nothing */
109 1.207 yamt
110 1.207 yamt #else /* defined(UVMMAP_NOCOUNTERS) */
111 1.207 yamt
112 1.228 yamt #include <sys/evcnt.h>
113 1.207 yamt #define UVMMAP_EVCNT_DEFINE(name) \
114 1.207 yamt struct evcnt uvmmap_evcnt_##name = EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, \
115 1.207 yamt "uvmmap", #name); \
116 1.207 yamt EVCNT_ATTACH_STATIC(uvmmap_evcnt_##name);
117 1.207 yamt #define UVMMAP_EVCNT_INCR(ev) uvmmap_evcnt_##ev.ev_count++
118 1.207 yamt #define UVMMAP_EVCNT_DECR(ev) uvmmap_evcnt_##ev.ev_count--
119 1.207 yamt
120 1.207 yamt #endif /* defined(UVMMAP_NOCOUNTERS) */
121 1.207 yamt
122 1.207 yamt UVMMAP_EVCNT_DEFINE(ubackmerge)
123 1.207 yamt UVMMAP_EVCNT_DEFINE(uforwmerge)
124 1.207 yamt UVMMAP_EVCNT_DEFINE(ubimerge)
125 1.207 yamt UVMMAP_EVCNT_DEFINE(unomerge)
126 1.207 yamt UVMMAP_EVCNT_DEFINE(kbackmerge)
127 1.207 yamt UVMMAP_EVCNT_DEFINE(kforwmerge)
128 1.207 yamt UVMMAP_EVCNT_DEFINE(kbimerge)
129 1.207 yamt UVMMAP_EVCNT_DEFINE(knomerge)
130 1.207 yamt UVMMAP_EVCNT_DEFINE(map_call)
131 1.207 yamt UVMMAP_EVCNT_DEFINE(mlk_call)
132 1.207 yamt UVMMAP_EVCNT_DEFINE(mlk_hint)
133 1.263 matt UVMMAP_EVCNT_DEFINE(mlk_list)
134 1.263 matt UVMMAP_EVCNT_DEFINE(mlk_tree)
135 1.263 matt UVMMAP_EVCNT_DEFINE(mlk_treeloop)
136 1.263 matt UVMMAP_EVCNT_DEFINE(mlk_listloop)
137 1.169 petrov
138 1.208 yamt UVMMAP_EVCNT_DEFINE(uke_alloc)
139 1.208 yamt UVMMAP_EVCNT_DEFINE(uke_free)
140 1.208 yamt UVMMAP_EVCNT_DEFINE(ukh_alloc)
141 1.208 yamt UVMMAP_EVCNT_DEFINE(ukh_free)
142 1.208 yamt
143 1.87 enami const char vmmapbsy[] = "vmmapbsy";
144 1.1 mrg
145 1.1 mrg /*
146 1.248 ad * cache for vmspace structures.
147 1.25 thorpej */
148 1.25 thorpej
149 1.248 ad static struct pool_cache uvm_vmspace_cache;
150 1.25 thorpej
151 1.26 thorpej /*
152 1.248 ad * cache for dynamically-allocated map entries.
153 1.26 thorpej */
154 1.26 thorpej
155 1.248 ad static struct pool_cache uvm_map_entry_cache;
156 1.130 thorpej
157 1.130 thorpej MALLOC_DEFINE(M_VMMAP, "VM map", "VM map structures");
158 1.130 thorpej MALLOC_DEFINE(M_VMPMAP, "VM pmap", "VM pmap");
159 1.25 thorpej
160 1.40 thorpej #ifdef PMAP_GROWKERNEL
161 1.40 thorpej /*
162 1.40 thorpej * This global represents the end of the kernel virtual address
163 1.40 thorpej * space. If we want to exceed this, we must grow the kernel
164 1.40 thorpej * virtual address space dynamically.
165 1.40 thorpej *
166 1.40 thorpej * Note, this variable is locked by kernel_map's lock.
167 1.40 thorpej */
168 1.40 thorpej vaddr_t uvm_maxkaddr;
169 1.40 thorpej #endif
170 1.40 thorpej
171 1.25 thorpej /*
172 1.1 mrg * macros
173 1.1 mrg */
174 1.1 mrg
175 1.1 mrg /*
176 1.174 yamt * VM_MAP_USE_KMAPENT: determine if uvm_kmapent_alloc/free is used
177 1.174 yamt * for the vm_map.
178 1.174 yamt */
179 1.174 yamt extern struct vm_map *pager_map; /* XXX */
180 1.219 yamt #define VM_MAP_USE_KMAPENT_FLAGS(flags) \
181 1.219 yamt (((flags) & VM_MAP_INTRSAFE) != 0)
182 1.174 yamt #define VM_MAP_USE_KMAPENT(map) \
183 1.219 yamt (VM_MAP_USE_KMAPENT_FLAGS((map)->flags) || (map) == kernel_map)
184 1.174 yamt
185 1.174 yamt /*
186 1.194 yamt * UVM_ET_ISCOMPATIBLE: check some requirements for map entry merging
187 1.194 yamt */
188 1.194 yamt
189 1.194 yamt #define UVM_ET_ISCOMPATIBLE(ent, type, uobj, meflags, \
190 1.194 yamt prot, maxprot, inh, adv, wire) \
191 1.194 yamt ((ent)->etype == (type) && \
192 1.194 yamt (((ent)->flags ^ (meflags)) & (UVM_MAP_NOMERGE | UVM_MAP_QUANTUM)) \
193 1.194 yamt == 0 && \
194 1.194 yamt (ent)->object.uvm_obj == (uobj) && \
195 1.194 yamt (ent)->protection == (prot) && \
196 1.194 yamt (ent)->max_protection == (maxprot) && \
197 1.194 yamt (ent)->inheritance == (inh) && \
198 1.194 yamt (ent)->advice == (adv) && \
199 1.194 yamt (ent)->wired_count == (wire))
200 1.194 yamt
201 1.194 yamt /*
202 1.1 mrg * uvm_map_entry_link: insert entry into a map
203 1.1 mrg *
204 1.1 mrg * => map must be locked
205 1.1 mrg */
206 1.10 mrg #define uvm_map_entry_link(map, after_where, entry) do { \
207 1.218 yamt uvm_mapent_check(entry); \
208 1.10 mrg (map)->nentries++; \
209 1.10 mrg (entry)->prev = (after_where); \
210 1.10 mrg (entry)->next = (after_where)->next; \
211 1.10 mrg (entry)->prev->next = (entry); \
212 1.10 mrg (entry)->next->prev = (entry); \
213 1.144 yamt uvm_rb_insert((map), (entry)); \
214 1.124 perry } while (/*CONSTCOND*/ 0)
215 1.10 mrg
216 1.1 mrg /*
217 1.1 mrg * uvm_map_entry_unlink: remove entry from a map
218 1.1 mrg *
219 1.1 mrg * => map must be locked
220 1.1 mrg */
221 1.10 mrg #define uvm_map_entry_unlink(map, entry) do { \
222 1.221 yamt KASSERT((entry) != (map)->first_free); \
223 1.221 yamt KASSERT((entry) != (map)->hint); \
224 1.218 yamt uvm_mapent_check(entry); \
225 1.10 mrg (map)->nentries--; \
226 1.10 mrg (entry)->next->prev = (entry)->prev; \
227 1.10 mrg (entry)->prev->next = (entry)->next; \
228 1.144 yamt uvm_rb_remove((map), (entry)); \
229 1.124 perry } while (/*CONSTCOND*/ 0)
230 1.1 mrg
231 1.1 mrg /*
232 1.1 mrg * SAVE_HINT: saves the specified entry as the hint for future lookups.
233 1.1 mrg *
234 1.248 ad * => map need not be locked.
235 1.1 mrg */
236 1.248 ad #define SAVE_HINT(map, check, value) do { \
237 1.258 ad if ((map)->hint == (check)) \
238 1.258 ad (map)->hint = (value); \
239 1.124 perry } while (/*CONSTCOND*/ 0)
240 1.1 mrg
241 1.1 mrg /*
242 1.221 yamt * clear_hints: ensure that hints don't point to the entry.
243 1.221 yamt *
244 1.221 yamt * => map must be write-locked.
245 1.221 yamt */
246 1.221 yamt static void
247 1.221 yamt clear_hints(struct vm_map *map, struct vm_map_entry *ent)
248 1.221 yamt {
249 1.221 yamt
250 1.221 yamt SAVE_HINT(map, ent, ent->prev);
251 1.221 yamt if (map->first_free == ent) {
252 1.221 yamt map->first_free = ent->prev;
253 1.221 yamt }
254 1.221 yamt }
255 1.221 yamt
256 1.221 yamt /*
257 1.1 mrg * VM_MAP_RANGE_CHECK: check and correct range
258 1.1 mrg *
259 1.1 mrg * => map must at least be read locked
260 1.1 mrg */
261 1.1 mrg
262 1.10 mrg #define VM_MAP_RANGE_CHECK(map, start, end) do { \
263 1.139 enami if (start < vm_map_min(map)) \
264 1.139 enami start = vm_map_min(map); \
265 1.139 enami if (end > vm_map_max(map)) \
266 1.139 enami end = vm_map_max(map); \
267 1.139 enami if (start > end) \
268 1.139 enami start = end; \
269 1.124 perry } while (/*CONSTCOND*/ 0)
270 1.1 mrg
271 1.1 mrg /*
272 1.1 mrg * local prototypes
273 1.1 mrg */
274 1.1 mrg
275 1.138 enami static struct vm_map_entry *
276 1.138 enami uvm_mapent_alloc(struct vm_map *, int);
277 1.174 yamt static struct vm_map_entry *
278 1.174 yamt uvm_mapent_alloc_split(struct vm_map *,
279 1.174 yamt const struct vm_map_entry *, int,
280 1.174 yamt struct uvm_mapent_reservation *);
281 1.138 enami static void uvm_mapent_copy(struct vm_map_entry *, struct vm_map_entry *);
282 1.138 enami static void uvm_mapent_free(struct vm_map_entry *);
283 1.218 yamt #if defined(DEBUG)
284 1.218 yamt static void _uvm_mapent_check(const struct vm_map_entry *, const char *,
285 1.218 yamt int);
286 1.218 yamt #define uvm_mapent_check(map) _uvm_mapent_check(map, __FILE__, __LINE__)
287 1.218 yamt #else /* defined(DEBUG) */
288 1.218 yamt #define uvm_mapent_check(e) /* nothing */
289 1.218 yamt #endif /* defined(DEBUG) */
290 1.174 yamt static struct vm_map_entry *
291 1.174 yamt uvm_kmapent_alloc(struct vm_map *, int);
292 1.174 yamt static void uvm_kmapent_free(struct vm_map_entry *);
293 1.219 yamt static vsize_t uvm_kmapent_overhead(vsize_t);
294 1.219 yamt
295 1.138 enami static void uvm_map_entry_unwire(struct vm_map *, struct vm_map_entry *);
296 1.138 enami static void uvm_map_reference_amap(struct vm_map_entry *, int);
297 1.140 enami static int uvm_map_space_avail(vaddr_t *, vsize_t, voff_t, vsize_t, int,
298 1.140 enami struct vm_map_entry *);
299 1.138 enami static void uvm_map_unreference_amap(struct vm_map_entry *, int);
300 1.1 mrg
301 1.222 yamt int _uvm_map_sanity(struct vm_map *);
302 1.222 yamt int _uvm_tree_sanity(struct vm_map *);
303 1.263 matt static vsize_t uvm_rb_maxgap(const struct vm_map_entry *);
304 1.144 yamt
305 1.263 matt CTASSERT(offsetof(struct vm_map_entry, rb_node) == 0);
306 1.263 matt #define ROOT_ENTRY(map) ((struct vm_map_entry *)(map)->rb_tree.rbt_root)
307 1.263 matt #define LEFT_ENTRY(entry) ((struct vm_map_entry *)(entry)->rb_node.rb_left)
308 1.263 matt #define RIGHT_ENTRY(entry) ((struct vm_map_entry *)(entry)->rb_node.rb_right)
309 1.263 matt #define PARENT_ENTRY(map, entry) \
310 1.263 matt (ROOT_ENTRY(map) == (entry) \
311 1.263 matt ? NULL \
312 1.263 matt : (struct vm_map_entry *)RB_FATHER(&(entry)->rb_node))
313 1.263 matt
314 1.263 matt static int
315 1.263 matt uvm_map_compare_nodes(const struct rb_node *nparent,
316 1.263 matt const struct rb_node *nkey)
317 1.144 yamt {
318 1.263 matt const struct vm_map_entry *eparent = (const void *) nparent;
319 1.263 matt const struct vm_map_entry *ekey = (const void *) nkey;
320 1.144 yamt
321 1.263 matt KASSERT(eparent->start < ekey->start || eparent->start >= ekey->end);
322 1.263 matt KASSERT(ekey->start < eparent->start || ekey->start >= eparent->end);
323 1.164 junyoung
324 1.263 matt if (ekey->start < eparent->start)
325 1.263 matt return -1;
326 1.263 matt if (ekey->start >= eparent->end)
327 1.263 matt return 1;
328 1.263 matt return 0;
329 1.144 yamt }
330 1.144 yamt
331 1.263 matt static int
332 1.263 matt uvm_map_compare_key(const struct rb_node *nparent, const void *vkey)
333 1.144 yamt {
334 1.263 matt const struct vm_map_entry *eparent = (const void *) nparent;
335 1.263 matt const vaddr_t va = *(const vaddr_t *) vkey;
336 1.144 yamt
337 1.263 matt if (va < eparent->start)
338 1.263 matt return -1;
339 1.263 matt if (va >= eparent->end)
340 1.263 matt return 1;
341 1.263 matt return 0;
342 1.144 yamt }
343 1.144 yamt
344 1.263 matt static const struct rb_tree_ops uvm_map_tree_ops = {
345 1.263 matt .rbto_compare_nodes = uvm_map_compare_nodes,
346 1.263 matt .rbto_compare_key = uvm_map_compare_key,
347 1.263 matt };
348 1.144 yamt
349 1.206 perry static inline vsize_t
350 1.263 matt uvm_rb_gap(const struct vm_map_entry *entry)
351 1.144 yamt {
352 1.144 yamt KASSERT(entry->next != NULL);
353 1.144 yamt return entry->next->start - entry->end;
354 1.144 yamt }
355 1.144 yamt
356 1.144 yamt static vsize_t
357 1.263 matt uvm_rb_maxgap(const struct vm_map_entry *entry)
358 1.144 yamt {
359 1.263 matt struct vm_map_entry *child;
360 1.263 matt vsize_t maxgap = entry->gap;
361 1.144 yamt
362 1.263 matt /*
363 1.263 matt * We need maxgap to be the largest gap of us or any of our
364 1.263 matt * descendents. Since each of our children's maxgap is the
365 1.263 matt * cached value of their largest gap of themselves or their
366 1.263 matt * descendents, we can just use that value and avoid recursing
367 1.263 matt * down the tree to calculate it.
368 1.263 matt */
369 1.263 matt if ((child = LEFT_ENTRY(entry)) != NULL && maxgap < child->maxgap)
370 1.263 matt maxgap = child->maxgap;
371 1.144 yamt
372 1.263 matt if ((child = RIGHT_ENTRY(entry)) != NULL && maxgap < child->maxgap)
373 1.263 matt maxgap = child->maxgap;
374 1.144 yamt
375 1.263 matt return maxgap;
376 1.144 yamt }
377 1.144 yamt
378 1.263 matt static void
379 1.144 yamt uvm_rb_fixup(struct vm_map *map, struct vm_map_entry *entry)
380 1.144 yamt {
381 1.263 matt struct vm_map_entry *parent;
382 1.263 matt
383 1.263 matt KASSERT(entry->gap == uvm_rb_gap(entry));
384 1.263 matt entry->maxgap = uvm_rb_maxgap(entry);
385 1.263 matt
386 1.263 matt while ((parent = PARENT_ENTRY(map, entry)) != NULL) {
387 1.263 matt struct vm_map_entry *brother;
388 1.263 matt vsize_t maxgap = parent->gap;
389 1.263 matt
390 1.263 matt KDASSERT(parent->gap == uvm_rb_gap(parent));
391 1.263 matt if (maxgap < entry->maxgap)
392 1.263 matt maxgap = entry->maxgap;
393 1.263 matt /*
394 1.263 matt * Since we work our towards the root, we know entry's maxgap
395 1.263 matt * value is ok but its brothers may now be out-of-date due
396 1.263 matt * rebalancing. So refresh it.
397 1.263 matt */
398 1.263 matt brother = (struct vm_map_entry *)parent->rb_node.rb_nodes[RB_POSITION(&entry->rb_node) ^ RB_DIR_OTHER];
399 1.263 matt if (brother != NULL) {
400 1.263 matt KDASSERT(brother->gap == uvm_rb_gap(brother));
401 1.263 matt brother->maxgap = uvm_rb_maxgap(brother);
402 1.263 matt if (maxgap < brother->maxgap)
403 1.263 matt maxgap = brother->maxgap;
404 1.263 matt }
405 1.263 matt
406 1.263 matt parent->maxgap = maxgap;
407 1.263 matt entry = parent;
408 1.263 matt }
409 1.144 yamt }
410 1.144 yamt
411 1.203 thorpej static void
412 1.144 yamt uvm_rb_insert(struct vm_map *map, struct vm_map_entry *entry)
413 1.144 yamt {
414 1.263 matt entry->gap = entry->maxgap = uvm_rb_gap(entry);
415 1.263 matt if (entry->prev != &map->header)
416 1.263 matt entry->prev->gap = uvm_rb_gap(entry->prev);
417 1.144 yamt
418 1.263 matt if (!rb_tree_insert_node(&map->rb_tree, &entry->rb_node))
419 1.263 matt panic("uvm_rb_insert: map %p: duplicate entry?", map);
420 1.263 matt
421 1.263 matt /*
422 1.263 matt * If the previous entry is not our immediate left child, then it's an
423 1.263 matt * ancestor and will be fixed up on the way to the root. We don't
424 1.263 matt * have to check entry->prev against &map->header since &map->header
425 1.263 matt * will never be in the tree.
426 1.263 matt */
427 1.263 matt uvm_rb_fixup(map,
428 1.263 matt LEFT_ENTRY(entry) == entry->prev ? entry->prev : entry);
429 1.144 yamt }
430 1.144 yamt
431 1.203 thorpej static void
432 1.144 yamt uvm_rb_remove(struct vm_map *map, struct vm_map_entry *entry)
433 1.144 yamt {
434 1.263 matt struct vm_map_entry *prev_parent = NULL, *next_parent = NULL;
435 1.144 yamt
436 1.263 matt /*
437 1.263 matt * If we are removing an interior node, then an adjacent node will
438 1.263 matt * be used to replace its position in the tree. Therefore we will
439 1.263 matt * need to fixup the tree starting at the parent of the replacement
440 1.263 matt * node. So record their parents for later use.
441 1.263 matt */
442 1.144 yamt if (entry->prev != &map->header)
443 1.263 matt prev_parent = PARENT_ENTRY(map, entry->prev);
444 1.263 matt if (entry->next != &map->header)
445 1.263 matt next_parent = PARENT_ENTRY(map, entry->next);
446 1.263 matt
447 1.263 matt rb_tree_remove_node(&map->rb_tree, &entry->rb_node);
448 1.263 matt
449 1.263 matt /*
450 1.263 matt * If the previous node has a new parent, fixup the tree starting
451 1.263 matt * at the previous node's old parent.
452 1.263 matt */
453 1.263 matt if (entry->prev != &map->header) {
454 1.263 matt /*
455 1.263 matt * Update the previous entry's gap due to our absence.
456 1.263 matt */
457 1.263 matt entry->prev->gap = uvm_rb_gap(entry->prev);
458 1.144 yamt uvm_rb_fixup(map, entry->prev);
459 1.263 matt if (prev_parent != NULL
460 1.263 matt && prev_parent != entry
461 1.263 matt && prev_parent != PARENT_ENTRY(map, entry->prev))
462 1.263 matt uvm_rb_fixup(map, prev_parent);
463 1.263 matt }
464 1.263 matt
465 1.263 matt /*
466 1.263 matt * If the next node has a new parent, fixup the tree starting
467 1.263 matt * at the next node's old parent.
468 1.263 matt */
469 1.263 matt if (entry->next != &map->header) {
470 1.263 matt uvm_rb_fixup(map, entry->next);
471 1.263 matt if (next_parent != NULL
472 1.263 matt && next_parent != entry
473 1.263 matt && next_parent != PARENT_ENTRY(map, entry->next))
474 1.263 matt uvm_rb_fixup(map, next_parent);
475 1.263 matt }
476 1.144 yamt }
477 1.144 yamt
478 1.222 yamt #if defined(DEBUG)
479 1.222 yamt int uvm_debug_check_map = 0;
480 1.159 yamt int uvm_debug_check_rbtree = 0;
481 1.222 yamt #define uvm_map_check(map, name) \
482 1.222 yamt _uvm_map_check((map), (name), __FILE__, __LINE__)
483 1.222 yamt static void
484 1.222 yamt _uvm_map_check(struct vm_map *map, const char *name,
485 1.222 yamt const char *file, int line)
486 1.222 yamt {
487 1.222 yamt
488 1.222 yamt if ((uvm_debug_check_map && _uvm_map_sanity(map)) ||
489 1.222 yamt (uvm_debug_check_rbtree && _uvm_tree_sanity(map))) {
490 1.222 yamt panic("uvm_map_check failed: \"%s\" map=%p (%s:%d)",
491 1.222 yamt name, map, file, line);
492 1.222 yamt }
493 1.222 yamt }
494 1.222 yamt #else /* defined(DEBUG) */
495 1.222 yamt #define uvm_map_check(map, name) /* nothing */
496 1.222 yamt #endif /* defined(DEBUG) */
497 1.222 yamt
498 1.222 yamt #if defined(DEBUG) || defined(DDB)
499 1.222 yamt int
500 1.222 yamt _uvm_map_sanity(struct vm_map *map)
501 1.222 yamt {
502 1.234 thorpej bool first_free_found = false;
503 1.234 thorpej bool hint_found = false;
504 1.222 yamt const struct vm_map_entry *e;
505 1.272 yamt struct vm_map_entry *hint = map->hint;
506 1.222 yamt
507 1.222 yamt e = &map->header;
508 1.222 yamt for (;;) {
509 1.222 yamt if (map->first_free == e) {
510 1.234 thorpej first_free_found = true;
511 1.222 yamt } else if (!first_free_found && e->next->start > e->end) {
512 1.222 yamt printf("first_free %p should be %p\n",
513 1.222 yamt map->first_free, e);
514 1.222 yamt return -1;
515 1.222 yamt }
516 1.272 yamt if (hint == e) {
517 1.234 thorpej hint_found = true;
518 1.222 yamt }
519 1.222 yamt
520 1.222 yamt e = e->next;
521 1.222 yamt if (e == &map->header) {
522 1.222 yamt break;
523 1.222 yamt }
524 1.222 yamt }
525 1.222 yamt if (!first_free_found) {
526 1.222 yamt printf("stale first_free\n");
527 1.222 yamt return -1;
528 1.222 yamt }
529 1.222 yamt if (!hint_found) {
530 1.222 yamt printf("stale hint\n");
531 1.222 yamt return -1;
532 1.222 yamt }
533 1.222 yamt return 0;
534 1.222 yamt }
535 1.144 yamt
536 1.144 yamt int
537 1.222 yamt _uvm_tree_sanity(struct vm_map *map)
538 1.144 yamt {
539 1.144 yamt struct vm_map_entry *tmp, *trtmp;
540 1.144 yamt int n = 0, i = 1;
541 1.144 yamt
542 1.263 matt for (tmp = map->header.next; tmp != &map->header; tmp = tmp->next) {
543 1.263 matt if (tmp->gap != uvm_rb_gap(tmp)) {
544 1.263 matt printf("%d/%d gap %lx != %lx %s\n",
545 1.222 yamt n + 1, map->nentries,
546 1.263 matt (ulong)tmp->gap, (ulong)uvm_rb_gap(tmp),
547 1.144 yamt tmp->next == &map->header ? "(last)" : "");
548 1.144 yamt goto error;
549 1.144 yamt }
550 1.263 matt /*
551 1.263 matt * If any entries are out of order, tmp->gap will be unsigned
552 1.263 matt * and will likely exceed the size of the map.
553 1.263 matt */
554 1.263 matt KASSERT(tmp->gap < map->size);
555 1.263 matt n++;
556 1.263 matt }
557 1.263 matt
558 1.263 matt if (n != map->nentries) {
559 1.263 matt printf("nentries: %d vs %d\n", n, map->nentries);
560 1.263 matt goto error;
561 1.144 yamt }
562 1.263 matt
563 1.144 yamt trtmp = NULL;
564 1.263 matt for (tmp = map->header.next; tmp != &map->header; tmp = tmp->next) {
565 1.263 matt if (tmp->maxgap != uvm_rb_maxgap(tmp)) {
566 1.263 matt printf("maxgap %lx != %lx\n",
567 1.263 matt (ulong)tmp->maxgap,
568 1.263 matt (ulong)uvm_rb_maxgap(tmp));
569 1.144 yamt goto error;
570 1.144 yamt }
571 1.144 yamt if (trtmp != NULL && trtmp->start >= tmp->start) {
572 1.222 yamt printf("corrupt: 0x%lx >= 0x%lx\n",
573 1.222 yamt trtmp->start, tmp->start);
574 1.144 yamt goto error;
575 1.144 yamt }
576 1.144 yamt
577 1.144 yamt trtmp = tmp;
578 1.144 yamt }
579 1.144 yamt
580 1.263 matt for (tmp = map->header.next; tmp != &map->header;
581 1.144 yamt tmp = tmp->next, i++) {
582 1.263 matt trtmp = (void *) rb_tree_iterate(&map->rb_tree, &tmp->rb_node,
583 1.263 matt RB_DIR_LEFT);
584 1.263 matt if (trtmp == NULL)
585 1.263 matt trtmp = &map->header;
586 1.263 matt if (tmp->prev != trtmp) {
587 1.263 matt printf("lookup: %d: %p->prev=%p: %p\n",
588 1.263 matt i, tmp, tmp->prev, trtmp);
589 1.263 matt goto error;
590 1.263 matt }
591 1.263 matt trtmp = (void *) rb_tree_iterate(&map->rb_tree, &tmp->rb_node,
592 1.263 matt RB_DIR_RIGHT);
593 1.263 matt if (trtmp == NULL)
594 1.263 matt trtmp = &map->header;
595 1.263 matt if (tmp->next != trtmp) {
596 1.263 matt printf("lookup: %d: %p->next=%p: %p\n",
597 1.263 matt i, tmp, tmp->next, trtmp);
598 1.263 matt goto error;
599 1.263 matt }
600 1.263 matt trtmp = (void *)rb_tree_find_node(&map->rb_tree, &tmp->start);
601 1.144 yamt if (trtmp != tmp) {
602 1.222 yamt printf("lookup: %d: %p - %p: %p\n", i, tmp, trtmp,
603 1.263 matt PARENT_ENTRY(map, tmp));
604 1.144 yamt goto error;
605 1.144 yamt }
606 1.144 yamt }
607 1.144 yamt
608 1.144 yamt return (0);
609 1.144 yamt error:
610 1.144 yamt return (-1);
611 1.144 yamt }
612 1.222 yamt #endif /* defined(DEBUG) || defined(DDB) */
613 1.144 yamt
614 1.201 dsl #ifdef DIAGNOSTIC
615 1.203 thorpej static struct vm_map *uvm_kmapent_map(struct vm_map_entry *);
616 1.201 dsl #endif
617 1.174 yamt
618 1.1 mrg /*
619 1.238 ad * vm_map_lock: acquire an exclusive (write) lock on a map.
620 1.238 ad *
621 1.238 ad * => Note that "intrsafe" maps use only exclusive, spin locks.
622 1.238 ad *
623 1.238 ad * => The locking protocol provides for guaranteed upgrade from shared ->
624 1.238 ad * exclusive by whichever thread currently has the map marked busy.
625 1.238 ad * See "LOCKING PROTOCOL NOTES" in uvm_map.h. This is horrible; among
626 1.238 ad * other problems, it defeats any fairness guarantees provided by RW
627 1.238 ad * locks.
628 1.238 ad */
629 1.238 ad
630 1.238 ad void
631 1.238 ad vm_map_lock(struct vm_map *map)
632 1.238 ad {
633 1.238 ad
634 1.238 ad if ((map->flags & VM_MAP_INTRSAFE) != 0) {
635 1.238 ad mutex_spin_enter(&map->mutex);
636 1.238 ad return;
637 1.238 ad }
638 1.238 ad
639 1.238 ad for (;;) {
640 1.238 ad rw_enter(&map->lock, RW_WRITER);
641 1.238 ad if (map->busy == NULL)
642 1.238 ad break;
643 1.249 yamt if (map->busy == curlwp)
644 1.249 yamt break;
645 1.238 ad mutex_enter(&map->misc_lock);
646 1.238 ad rw_exit(&map->lock);
647 1.248 ad if (map->busy != NULL)
648 1.248 ad cv_wait(&map->cv, &map->misc_lock);
649 1.238 ad mutex_exit(&map->misc_lock);
650 1.238 ad }
651 1.238 ad
652 1.238 ad map->timestamp++;
653 1.238 ad }
654 1.238 ad
655 1.238 ad /*
656 1.238 ad * vm_map_lock_try: try to lock a map, failing if it is already locked.
657 1.238 ad */
658 1.238 ad
659 1.238 ad bool
660 1.238 ad vm_map_lock_try(struct vm_map *map)
661 1.238 ad {
662 1.238 ad
663 1.238 ad if ((map->flags & VM_MAP_INTRSAFE) != 0)
664 1.238 ad return mutex_tryenter(&map->mutex);
665 1.238 ad if (!rw_tryenter(&map->lock, RW_WRITER))
666 1.238 ad return false;
667 1.238 ad if (map->busy != NULL) {
668 1.238 ad rw_exit(&map->lock);
669 1.238 ad return false;
670 1.238 ad }
671 1.238 ad
672 1.238 ad map->timestamp++;
673 1.238 ad return true;
674 1.238 ad }
675 1.238 ad
676 1.238 ad /*
677 1.238 ad * vm_map_unlock: release an exclusive lock on a map.
678 1.238 ad */
679 1.238 ad
680 1.238 ad void
681 1.238 ad vm_map_unlock(struct vm_map *map)
682 1.238 ad {
683 1.238 ad
684 1.238 ad if ((map->flags & VM_MAP_INTRSAFE) != 0)
685 1.238 ad mutex_spin_exit(&map->mutex);
686 1.238 ad else {
687 1.238 ad KASSERT(rw_write_held(&map->lock));
688 1.249 yamt KASSERT(map->busy == NULL || map->busy == curlwp);
689 1.238 ad rw_exit(&map->lock);
690 1.238 ad }
691 1.238 ad }
692 1.238 ad
693 1.238 ad /*
694 1.238 ad * vm_map_unbusy: mark the map as unbusy, and wake any waiters that
695 1.238 ad * want an exclusive lock.
696 1.238 ad */
697 1.238 ad
698 1.238 ad void
699 1.238 ad vm_map_unbusy(struct vm_map *map)
700 1.238 ad {
701 1.238 ad
702 1.238 ad KASSERT(map->busy == curlwp);
703 1.238 ad
704 1.238 ad /*
705 1.238 ad * Safe to clear 'busy' and 'waiters' with only a read lock held:
706 1.238 ad *
707 1.238 ad * o they can only be set with a write lock held
708 1.238 ad * o writers are blocked out with a read or write hold
709 1.238 ad * o at any time, only one thread owns the set of values
710 1.238 ad */
711 1.248 ad mutex_enter(&map->misc_lock);
712 1.238 ad map->busy = NULL;
713 1.238 ad cv_broadcast(&map->cv);
714 1.238 ad mutex_exit(&map->misc_lock);
715 1.238 ad }
716 1.238 ad
717 1.238 ad /*
718 1.248 ad * vm_map_lock_read: acquire a shared (read) lock on a map.
719 1.248 ad */
720 1.248 ad
721 1.248 ad void
722 1.248 ad vm_map_lock_read(struct vm_map *map)
723 1.248 ad {
724 1.248 ad
725 1.248 ad KASSERT((map->flags & VM_MAP_INTRSAFE) == 0);
726 1.248 ad
727 1.248 ad rw_enter(&map->lock, RW_READER);
728 1.248 ad }
729 1.248 ad
730 1.248 ad /*
731 1.248 ad * vm_map_unlock_read: release a shared lock on a map.
732 1.248 ad */
733 1.248 ad
734 1.248 ad void
735 1.248 ad vm_map_unlock_read(struct vm_map *map)
736 1.248 ad {
737 1.248 ad
738 1.248 ad KASSERT((map->flags & VM_MAP_INTRSAFE) == 0);
739 1.248 ad
740 1.248 ad rw_exit(&map->lock);
741 1.248 ad }
742 1.248 ad
743 1.248 ad /*
744 1.248 ad * vm_map_busy: mark a map as busy.
745 1.248 ad *
746 1.248 ad * => the caller must hold the map write locked
747 1.248 ad */
748 1.248 ad
749 1.248 ad void
750 1.248 ad vm_map_busy(struct vm_map *map)
751 1.248 ad {
752 1.248 ad
753 1.248 ad KASSERT(rw_write_held(&map->lock));
754 1.248 ad KASSERT(map->busy == NULL);
755 1.248 ad
756 1.248 ad map->busy = curlwp;
757 1.248 ad }
758 1.248 ad
759 1.248 ad /*
760 1.248 ad * vm_map_locked_p: return true if the map is write locked.
761 1.269 yamt *
762 1.269 yamt * => only for debug purposes like KASSERTs.
763 1.269 yamt * => should not be used to verify that a map is not locked.
764 1.248 ad */
765 1.248 ad
766 1.248 ad bool
767 1.248 ad vm_map_locked_p(struct vm_map *map)
768 1.248 ad {
769 1.248 ad
770 1.248 ad if ((map->flags & VM_MAP_INTRSAFE) != 0) {
771 1.248 ad return mutex_owned(&map->mutex);
772 1.248 ad } else {
773 1.248 ad return rw_write_held(&map->lock);
774 1.248 ad }
775 1.248 ad }
776 1.248 ad
777 1.248 ad /*
778 1.1 mrg * uvm_mapent_alloc: allocate a map entry
779 1.1 mrg */
780 1.1 mrg
781 1.203 thorpej static struct vm_map_entry *
782 1.138 enami uvm_mapent_alloc(struct vm_map *map, int flags)
783 1.10 mrg {
784 1.99 chs struct vm_map_entry *me;
785 1.127 thorpej int pflags = (flags & UVM_FLAG_NOWAIT) ? PR_NOWAIT : PR_WAITOK;
786 1.104 chs UVMHIST_FUNC("uvm_mapent_alloc"); UVMHIST_CALLED(maphist);
787 1.1 mrg
788 1.174 yamt if (VM_MAP_USE_KMAPENT(map)) {
789 1.174 yamt me = uvm_kmapent_alloc(map, flags);
790 1.104 chs } else {
791 1.248 ad me = pool_cache_get(&uvm_map_entry_cache, pflags);
792 1.126 bouyer if (__predict_false(me == NULL))
793 1.126 bouyer return NULL;
794 1.104 chs me->flags = 0;
795 1.10 mrg }
796 1.1 mrg
797 1.104 chs UVMHIST_LOG(maphist, "<- new entry=0x%x [kentry=%d]", me,
798 1.104 chs ((map->flags & VM_MAP_INTRSAFE) != 0 || map == kernel_map), 0, 0);
799 1.139 enami return (me);
800 1.1 mrg }
801 1.1 mrg
802 1.1 mrg /*
803 1.174 yamt * uvm_mapent_alloc_split: allocate a map entry for clipping.
804 1.248 ad *
805 1.248 ad * => map must be locked by caller if UVM_MAP_QUANTUM is set.
806 1.174 yamt */
807 1.174 yamt
808 1.203 thorpej static struct vm_map_entry *
809 1.174 yamt uvm_mapent_alloc_split(struct vm_map *map,
810 1.174 yamt const struct vm_map_entry *old_entry, int flags,
811 1.232 yamt struct uvm_mapent_reservation *umr)
812 1.174 yamt {
813 1.174 yamt struct vm_map_entry *me;
814 1.174 yamt
815 1.174 yamt KASSERT(!VM_MAP_USE_KMAPENT(map) ||
816 1.174 yamt (old_entry->flags & UVM_MAP_QUANTUM) || !UMR_EMPTY(umr));
817 1.174 yamt
818 1.174 yamt if (old_entry->flags & UVM_MAP_QUANTUM) {
819 1.175 yamt struct vm_map_kernel *vmk = vm_map_to_kernel(map);
820 1.174 yamt
821 1.248 ad KASSERT(vm_map_locked_p(map));
822 1.175 yamt me = vmk->vmk_merged_entries;
823 1.174 yamt KASSERT(me);
824 1.175 yamt vmk->vmk_merged_entries = me->next;
825 1.174 yamt KASSERT(me->flags & UVM_MAP_QUANTUM);
826 1.174 yamt } else {
827 1.174 yamt me = uvm_mapent_alloc(map, flags);
828 1.174 yamt }
829 1.174 yamt
830 1.174 yamt return me;
831 1.174 yamt }
832 1.174 yamt
833 1.174 yamt /*
834 1.1 mrg * uvm_mapent_free: free map entry
835 1.1 mrg */
836 1.1 mrg
837 1.203 thorpej static void
838 1.138 enami uvm_mapent_free(struct vm_map_entry *me)
839 1.1 mrg {
840 1.104 chs UVMHIST_FUNC("uvm_mapent_free"); UVMHIST_CALLED(maphist);
841 1.104 chs
842 1.98 chs UVMHIST_LOG(maphist,"<- freeing map entry=0x%x [flags=%d]",
843 1.1 mrg me, me->flags, 0, 0);
844 1.174 yamt if (me->flags & UVM_MAP_KERNEL) {
845 1.174 yamt uvm_kmapent_free(me);
846 1.174 yamt } else {
847 1.248 ad pool_cache_put(&uvm_map_entry_cache, me);
848 1.174 yamt }
849 1.174 yamt }
850 1.174 yamt
851 1.174 yamt /*
852 1.194 yamt * uvm_mapent_free_merged: free merged map entry
853 1.174 yamt *
854 1.174 yamt * => keep the entry if needed.
855 1.194 yamt * => caller shouldn't hold map locked if VM_MAP_USE_KMAPENT(map) is true.
856 1.248 ad * => map should be locked if UVM_MAP_QUANTUM is set.
857 1.174 yamt */
858 1.174 yamt
859 1.203 thorpej static void
860 1.181 yamt uvm_mapent_free_merged(struct vm_map *map, struct vm_map_entry *me)
861 1.174 yamt {
862 1.174 yamt
863 1.181 yamt KASSERT(!(me->flags & UVM_MAP_KERNEL) || uvm_kmapent_map(me) == map);
864 1.181 yamt
865 1.174 yamt if (me->flags & UVM_MAP_QUANTUM) {
866 1.174 yamt /*
867 1.174 yamt * keep this entry for later splitting.
868 1.174 yamt */
869 1.175 yamt struct vm_map_kernel *vmk;
870 1.174 yamt
871 1.248 ad KASSERT(vm_map_locked_p(map));
872 1.181 yamt KASSERT(VM_MAP_IS_KERNEL(map));
873 1.181 yamt KASSERT(!VM_MAP_USE_KMAPENT(map) ||
874 1.181 yamt (me->flags & UVM_MAP_KERNEL));
875 1.174 yamt
876 1.175 yamt vmk = vm_map_to_kernel(map);
877 1.175 yamt me->next = vmk->vmk_merged_entries;
878 1.175 yamt vmk->vmk_merged_entries = me;
879 1.104 chs } else {
880 1.174 yamt uvm_mapent_free(me);
881 1.10 mrg }
882 1.1 mrg }
883 1.1 mrg
884 1.1 mrg /*
885 1.1 mrg * uvm_mapent_copy: copy a map entry, preserving flags
886 1.1 mrg */
887 1.1 mrg
888 1.206 perry static inline void
889 1.138 enami uvm_mapent_copy(struct vm_map_entry *src, struct vm_map_entry *dst)
890 1.10 mrg {
891 1.139 enami
892 1.106 chs memcpy(dst, src, ((char *)&src->uvm_map_entry_stop_copy) -
893 1.139 enami ((char *)src));
894 1.1 mrg }
895 1.1 mrg
896 1.219 yamt /*
897 1.219 yamt * uvm_mapent_overhead: calculate maximum kva overhead necessary for
898 1.219 yamt * map entries.
899 1.219 yamt *
900 1.219 yamt * => size and flags are the same as uvm_km_suballoc's ones.
901 1.219 yamt */
902 1.219 yamt
903 1.219 yamt vsize_t
904 1.219 yamt uvm_mapent_overhead(vsize_t size, int flags)
905 1.219 yamt {
906 1.219 yamt
907 1.219 yamt if (VM_MAP_USE_KMAPENT_FLAGS(flags)) {
908 1.219 yamt return uvm_kmapent_overhead(size);
909 1.219 yamt }
910 1.219 yamt return 0;
911 1.219 yamt }
912 1.219 yamt
913 1.218 yamt #if defined(DEBUG)
914 1.218 yamt static void
915 1.218 yamt _uvm_mapent_check(const struct vm_map_entry *entry, const char *file, int line)
916 1.218 yamt {
917 1.218 yamt
918 1.218 yamt if (entry->start >= entry->end) {
919 1.218 yamt goto bad;
920 1.218 yamt }
921 1.218 yamt if (UVM_ET_ISOBJ(entry)) {
922 1.218 yamt if (entry->object.uvm_obj == NULL) {
923 1.218 yamt goto bad;
924 1.218 yamt }
925 1.218 yamt } else if (UVM_ET_ISSUBMAP(entry)) {
926 1.218 yamt if (entry->object.sub_map == NULL) {
927 1.218 yamt goto bad;
928 1.218 yamt }
929 1.218 yamt } else {
930 1.218 yamt if (entry->object.uvm_obj != NULL ||
931 1.218 yamt entry->object.sub_map != NULL) {
932 1.218 yamt goto bad;
933 1.218 yamt }
934 1.218 yamt }
935 1.218 yamt if (!UVM_ET_ISOBJ(entry)) {
936 1.218 yamt if (entry->offset != 0) {
937 1.218 yamt goto bad;
938 1.218 yamt }
939 1.218 yamt }
940 1.218 yamt
941 1.218 yamt return;
942 1.218 yamt
943 1.218 yamt bad:
944 1.218 yamt panic("%s: bad entry %p (%s:%d)", __func__, entry, file, line);
945 1.218 yamt }
946 1.218 yamt #endif /* defined(DEBUG) */
947 1.218 yamt
948 1.1 mrg /*
949 1.1 mrg * uvm_map_entry_unwire: unwire a map entry
950 1.1 mrg *
951 1.1 mrg * => map should be locked by caller
952 1.1 mrg */
953 1.1 mrg
954 1.206 perry static inline void
955 1.138 enami uvm_map_entry_unwire(struct vm_map *map, struct vm_map_entry *entry)
956 1.10 mrg {
957 1.139 enami
958 1.10 mrg entry->wired_count = 0;
959 1.57 thorpej uvm_fault_unwire_locked(map, entry->start, entry->end);
960 1.1 mrg }
961 1.1 mrg
962 1.85 chs
963 1.85 chs /*
964 1.85 chs * wrapper for calling amap_ref()
965 1.85 chs */
966 1.206 perry static inline void
967 1.138 enami uvm_map_reference_amap(struct vm_map_entry *entry, int flags)
968 1.85 chs {
969 1.139 enami
970 1.99 chs amap_ref(entry->aref.ar_amap, entry->aref.ar_pageoff,
971 1.139 enami (entry->end - entry->start) >> PAGE_SHIFT, flags);
972 1.85 chs }
973 1.85 chs
974 1.85 chs
975 1.85 chs /*
976 1.98 chs * wrapper for calling amap_unref()
977 1.85 chs */
978 1.206 perry static inline void
979 1.138 enami uvm_map_unreference_amap(struct vm_map_entry *entry, int flags)
980 1.85 chs {
981 1.139 enami
982 1.99 chs amap_unref(entry->aref.ar_amap, entry->aref.ar_pageoff,
983 1.139 enami (entry->end - entry->start) >> PAGE_SHIFT, flags);
984 1.85 chs }
985 1.85 chs
986 1.85 chs
987 1.1 mrg /*
988 1.248 ad * uvm_map_init: init mapping system at boot time.
989 1.1 mrg */
990 1.1 mrg
991 1.10 mrg void
992 1.138 enami uvm_map_init(void)
993 1.1 mrg {
994 1.1 mrg #if defined(UVMHIST)
995 1.10 mrg static struct uvm_history_ent maphistbuf[100];
996 1.10 mrg static struct uvm_history_ent pdhistbuf[100];
997 1.1 mrg #endif
998 1.10 mrg
999 1.10 mrg /*
1000 1.10 mrg * first, init logging system.
1001 1.10 mrg */
1002 1.1 mrg
1003 1.10 mrg UVMHIST_FUNC("uvm_map_init");
1004 1.10 mrg UVMHIST_INIT_STATIC(maphist, maphistbuf);
1005 1.10 mrg UVMHIST_INIT_STATIC(pdhist, pdhistbuf);
1006 1.10 mrg UVMHIST_CALLED(maphist);
1007 1.10 mrg UVMHIST_LOG(maphist,"<starting uvm map system>", 0, 0, 0, 0);
1008 1.10 mrg
1009 1.10 mrg /*
1010 1.174 yamt * initialize the global lock for kernel map entry.
1011 1.10 mrg */
1012 1.10 mrg
1013 1.238 ad mutex_init(&uvm_kentry_lock, MUTEX_DRIVER, IPL_VM);
1014 1.248 ad
1015 1.248 ad /*
1016 1.248 ad * initialize caches.
1017 1.248 ad */
1018 1.248 ad
1019 1.248 ad pool_cache_bootstrap(&uvm_map_entry_cache, sizeof(struct vm_map_entry),
1020 1.248 ad 0, 0, 0, "vmmpepl", NULL, IPL_NONE, NULL, NULL, NULL);
1021 1.248 ad pool_cache_bootstrap(&uvm_vmspace_cache, sizeof(struct vmspace),
1022 1.248 ad 0, 0, 0, "vmsppl", NULL, IPL_NONE, NULL, NULL, NULL);
1023 1.1 mrg }
1024 1.1 mrg
1025 1.1 mrg /*
1026 1.1 mrg * clippers
1027 1.1 mrg */
1028 1.1 mrg
1029 1.1 mrg /*
1030 1.218 yamt * uvm_mapent_splitadj: adjust map entries for splitting, after uvm_mapent_copy.
1031 1.218 yamt */
1032 1.218 yamt
1033 1.218 yamt static void
1034 1.218 yamt uvm_mapent_splitadj(struct vm_map_entry *entry1, struct vm_map_entry *entry2,
1035 1.218 yamt vaddr_t splitat)
1036 1.218 yamt {
1037 1.218 yamt vaddr_t adj;
1038 1.218 yamt
1039 1.218 yamt KASSERT(entry1->start < splitat);
1040 1.218 yamt KASSERT(splitat < entry1->end);
1041 1.218 yamt
1042 1.218 yamt adj = splitat - entry1->start;
1043 1.218 yamt entry1->end = entry2->start = splitat;
1044 1.218 yamt
1045 1.218 yamt if (entry1->aref.ar_amap) {
1046 1.218 yamt amap_splitref(&entry1->aref, &entry2->aref, adj);
1047 1.218 yamt }
1048 1.218 yamt if (UVM_ET_ISSUBMAP(entry1)) {
1049 1.218 yamt /* ... unlikely to happen, but play it safe */
1050 1.218 yamt uvm_map_reference(entry1->object.sub_map);
1051 1.218 yamt } else if (UVM_ET_ISOBJ(entry1)) {
1052 1.218 yamt KASSERT(entry1->object.uvm_obj != NULL); /* suppress coverity */
1053 1.218 yamt entry2->offset += adj;
1054 1.218 yamt if (entry1->object.uvm_obj->pgops &&
1055 1.218 yamt entry1->object.uvm_obj->pgops->pgo_reference)
1056 1.218 yamt entry1->object.uvm_obj->pgops->pgo_reference(
1057 1.218 yamt entry1->object.uvm_obj);
1058 1.218 yamt }
1059 1.218 yamt }
1060 1.218 yamt
1061 1.218 yamt /*
1062 1.1 mrg * uvm_map_clip_start: ensure that the entry begins at or after
1063 1.1 mrg * the starting address, if it doesn't we split the entry.
1064 1.98 chs *
1065 1.1 mrg * => caller should use UVM_MAP_CLIP_START macro rather than calling
1066 1.1 mrg * this directly
1067 1.1 mrg * => map must be locked by caller
1068 1.1 mrg */
1069 1.1 mrg
1070 1.99 chs void
1071 1.138 enami uvm_map_clip_start(struct vm_map *map, struct vm_map_entry *entry,
1072 1.174 yamt vaddr_t start, struct uvm_mapent_reservation *umr)
1073 1.1 mrg {
1074 1.99 chs struct vm_map_entry *new_entry;
1075 1.1 mrg
1076 1.1 mrg /* uvm_map_simplify_entry(map, entry); */ /* XXX */
1077 1.1 mrg
1078 1.222 yamt uvm_map_check(map, "clip_start entry");
1079 1.218 yamt uvm_mapent_check(entry);
1080 1.144 yamt
1081 1.10 mrg /*
1082 1.10 mrg * Split off the front portion. note that we must insert the new
1083 1.10 mrg * entry BEFORE this one, so that this entry has the specified
1084 1.1 mrg * starting address.
1085 1.10 mrg */
1086 1.174 yamt new_entry = uvm_mapent_alloc_split(map, entry, 0, umr);
1087 1.1 mrg uvm_mapent_copy(entry, new_entry); /* entry -> new_entry */
1088 1.218 yamt uvm_mapent_splitadj(new_entry, entry, start);
1089 1.10 mrg uvm_map_entry_link(map, entry->prev, new_entry);
1090 1.85 chs
1091 1.222 yamt uvm_map_check(map, "clip_start leave");
1092 1.1 mrg }
1093 1.1 mrg
1094 1.1 mrg /*
1095 1.1 mrg * uvm_map_clip_end: ensure that the entry ends at or before
1096 1.1 mrg * the ending address, if it does't we split the reference
1097 1.98 chs *
1098 1.1 mrg * => caller should use UVM_MAP_CLIP_END macro rather than calling
1099 1.1 mrg * this directly
1100 1.1 mrg * => map must be locked by caller
1101 1.1 mrg */
1102 1.1 mrg
1103 1.10 mrg void
1104 1.174 yamt uvm_map_clip_end(struct vm_map *map, struct vm_map_entry *entry, vaddr_t end,
1105 1.174 yamt struct uvm_mapent_reservation *umr)
1106 1.1 mrg {
1107 1.218 yamt struct vm_map_entry *new_entry;
1108 1.1 mrg
1109 1.222 yamt uvm_map_check(map, "clip_end entry");
1110 1.218 yamt uvm_mapent_check(entry);
1111 1.174 yamt
1112 1.1 mrg /*
1113 1.1 mrg * Create a new entry and insert it
1114 1.1 mrg * AFTER the specified entry
1115 1.1 mrg */
1116 1.174 yamt new_entry = uvm_mapent_alloc_split(map, entry, 0, umr);
1117 1.1 mrg uvm_mapent_copy(entry, new_entry); /* entry -> new_entry */
1118 1.218 yamt uvm_mapent_splitadj(entry, new_entry, end);
1119 1.1 mrg uvm_map_entry_link(map, entry, new_entry);
1120 1.1 mrg
1121 1.222 yamt uvm_map_check(map, "clip_end leave");
1122 1.1 mrg }
1123 1.1 mrg
1124 1.226 yamt static void
1125 1.226 yamt vm_map_drain(struct vm_map *map, uvm_flag_t flags)
1126 1.226 yamt {
1127 1.226 yamt
1128 1.226 yamt if (!VM_MAP_IS_KERNEL(map)) {
1129 1.226 yamt return;
1130 1.226 yamt }
1131 1.226 yamt
1132 1.226 yamt uvm_km_va_drain(map, flags);
1133 1.226 yamt }
1134 1.1 mrg
1135 1.1 mrg /*
1136 1.1 mrg * M A P - m a i n e n t r y p o i n t
1137 1.1 mrg */
1138 1.1 mrg /*
1139 1.1 mrg * uvm_map: establish a valid mapping in a map
1140 1.1 mrg *
1141 1.1 mrg * => assume startp is page aligned.
1142 1.1 mrg * => assume size is a multiple of PAGE_SIZE.
1143 1.1 mrg * => assume sys_mmap provides enough of a "hint" to have us skip
1144 1.1 mrg * over text/data/bss area.
1145 1.1 mrg * => map must be unlocked (we will lock it)
1146 1.1 mrg * => <uobj,uoffset> value meanings (4 cases):
1147 1.139 enami * [1] <NULL,uoffset> == uoffset is a hint for PMAP_PREFER
1148 1.1 mrg * [2] <NULL,UVM_UNKNOWN_OFFSET> == don't PMAP_PREFER
1149 1.1 mrg * [3] <uobj,uoffset> == normal mapping
1150 1.1 mrg * [4] <uobj,UVM_UNKNOWN_OFFSET> == uvm_map finds offset based on VA
1151 1.98 chs *
1152 1.1 mrg * case [4] is for kernel mappings where we don't know the offset until
1153 1.8 chuck * we've found a virtual address. note that kernel object offsets are
1154 1.8 chuck * always relative to vm_map_min(kernel_map).
1155 1.81 thorpej *
1156 1.165 yamt * => if `align' is non-zero, we align the virtual address to the specified
1157 1.165 yamt * alignment.
1158 1.165 yamt * this is provided as a mechanism for large pages.
1159 1.81 thorpej *
1160 1.1 mrg * => XXXCDC: need way to map in external amap?
1161 1.1 mrg */
1162 1.1 mrg
1163 1.10 mrg int
1164 1.138 enami uvm_map(struct vm_map *map, vaddr_t *startp /* IN/OUT */, vsize_t size,
1165 1.138 enami struct uvm_object *uobj, voff_t uoffset, vsize_t align, uvm_flag_t flags)
1166 1.10 mrg {
1167 1.174 yamt struct uvm_map_args args;
1168 1.174 yamt struct vm_map_entry *new_entry;
1169 1.174 yamt int error;
1170 1.174 yamt
1171 1.181 yamt KASSERT((flags & UVM_FLAG_QUANTUM) == 0 || VM_MAP_IS_KERNEL(map));
1172 1.187 yamt KASSERT((size & PAGE_MASK) == 0);
1173 1.174 yamt
1174 1.174 yamt /*
1175 1.174 yamt * for pager_map, allocate the new entry first to avoid sleeping
1176 1.174 yamt * for memory while we have the map locked.
1177 1.174 yamt *
1178 1.261 skrll * Also, because we allocate entries for in-kernel maps
1179 1.174 yamt * a bit differently (cf. uvm_kmapent_alloc/free), we need to
1180 1.174 yamt * allocate them before locking the map.
1181 1.174 yamt */
1182 1.174 yamt
1183 1.174 yamt new_entry = NULL;
1184 1.181 yamt if (VM_MAP_USE_KMAPENT(map) || (flags & UVM_FLAG_QUANTUM) ||
1185 1.181 yamt map == pager_map) {
1186 1.174 yamt new_entry = uvm_mapent_alloc(map, (flags & UVM_FLAG_NOWAIT));
1187 1.174 yamt if (__predict_false(new_entry == NULL))
1188 1.174 yamt return ENOMEM;
1189 1.174 yamt if (flags & UVM_FLAG_QUANTUM)
1190 1.174 yamt new_entry->flags |= UVM_MAP_QUANTUM;
1191 1.174 yamt }
1192 1.174 yamt if (map == pager_map)
1193 1.174 yamt flags |= UVM_FLAG_NOMERGE;
1194 1.174 yamt
1195 1.174 yamt error = uvm_map_prepare(map, *startp, size, uobj, uoffset, align,
1196 1.174 yamt flags, &args);
1197 1.174 yamt if (!error) {
1198 1.174 yamt error = uvm_map_enter(map, &args, new_entry);
1199 1.174 yamt *startp = args.uma_start;
1200 1.189 yamt } else if (new_entry) {
1201 1.189 yamt uvm_mapent_free(new_entry);
1202 1.174 yamt }
1203 1.174 yamt
1204 1.187 yamt #if defined(DEBUG)
1205 1.187 yamt if (!error && VM_MAP_IS_KERNEL(map)) {
1206 1.264 ad uvm_km_check_empty(map, *startp, *startp + size);
1207 1.187 yamt }
1208 1.187 yamt #endif /* defined(DEBUG) */
1209 1.187 yamt
1210 1.174 yamt return error;
1211 1.174 yamt }
1212 1.174 yamt
1213 1.174 yamt int
1214 1.174 yamt uvm_map_prepare(struct vm_map *map, vaddr_t start, vsize_t size,
1215 1.174 yamt struct uvm_object *uobj, voff_t uoffset, vsize_t align, uvm_flag_t flags,
1216 1.174 yamt struct uvm_map_args *args)
1217 1.174 yamt {
1218 1.174 yamt struct vm_map_entry *prev_entry;
1219 1.174 yamt vm_prot_t prot = UVM_PROTECTION(flags);
1220 1.174 yamt vm_prot_t maxprot = UVM_MAXPROTECTION(flags);
1221 1.174 yamt
1222 1.174 yamt UVMHIST_FUNC("uvm_map_prepare");
1223 1.10 mrg UVMHIST_CALLED(maphist);
1224 1.1 mrg
1225 1.174 yamt UVMHIST_LOG(maphist, "(map=0x%x, start=0x%x, size=%d, flags=0x%x)",
1226 1.174 yamt map, start, size, flags);
1227 1.10 mrg UVMHIST_LOG(maphist, " uobj/offset 0x%x/%d", uobj, uoffset,0,0);
1228 1.107 chs
1229 1.107 chs /*
1230 1.107 chs * detect a popular device driver bug.
1231 1.107 chs */
1232 1.107 chs
1233 1.139 enami KASSERT(doing_shutdown || curlwp != NULL ||
1234 1.129 christos (map->flags & VM_MAP_INTRSAFE));
1235 1.1 mrg
1236 1.10 mrg /*
1237 1.144 yamt * zero-sized mapping doesn't make any sense.
1238 1.144 yamt */
1239 1.144 yamt KASSERT(size > 0);
1240 1.144 yamt
1241 1.180 yamt KASSERT((~flags & (UVM_FLAG_NOWAIT | UVM_FLAG_WAITVA)) != 0);
1242 1.180 yamt
1243 1.222 yamt uvm_map_check(map, "map entry");
1244 1.144 yamt
1245 1.144 yamt /*
1246 1.106 chs * check sanity of protection code
1247 1.10 mrg */
1248 1.1 mrg
1249 1.10 mrg if ((prot & maxprot) != prot) {
1250 1.98 chs UVMHIST_LOG(maphist, "<- prot. failure: prot=0x%x, max=0x%x",
1251 1.10 mrg prot, maxprot,0,0);
1252 1.94 chs return EACCES;
1253 1.10 mrg }
1254 1.1 mrg
1255 1.10 mrg /*
1256 1.106 chs * figure out where to put new VM range
1257 1.10 mrg */
1258 1.1 mrg
1259 1.180 yamt retry:
1260 1.234 thorpej if (vm_map_lock_try(map) == false) {
1261 1.248 ad if ((flags & UVM_FLAG_TRYLOCK) != 0 &&
1262 1.248 ad (map->flags & VM_MAP_INTRSAFE) == 0) {
1263 1.94 chs return EAGAIN;
1264 1.106 chs }
1265 1.10 mrg vm_map_lock(map); /* could sleep here */
1266 1.10 mrg }
1267 1.226 yamt prev_entry = uvm_map_findspace(map, start, size, &start,
1268 1.226 yamt uobj, uoffset, align, flags);
1269 1.226 yamt if (prev_entry == NULL) {
1270 1.180 yamt unsigned int timestamp;
1271 1.180 yamt
1272 1.180 yamt timestamp = map->timestamp;
1273 1.180 yamt UVMHIST_LOG(maphist,"waiting va timestamp=0x%x",
1274 1.180 yamt timestamp,0,0,0);
1275 1.180 yamt map->flags |= VM_MAP_WANTVA;
1276 1.10 mrg vm_map_unlock(map);
1277 1.180 yamt
1278 1.180 yamt /*
1279 1.226 yamt * try to reclaim kva and wait until someone does unmap.
1280 1.238 ad * fragile locking here, so we awaken every second to
1281 1.238 ad * recheck the condition.
1282 1.180 yamt */
1283 1.180 yamt
1284 1.226 yamt vm_map_drain(map, flags);
1285 1.226 yamt
1286 1.238 ad mutex_enter(&map->misc_lock);
1287 1.180 yamt while ((map->flags & VM_MAP_WANTVA) != 0 &&
1288 1.180 yamt map->timestamp == timestamp) {
1289 1.226 yamt if ((flags & UVM_FLAG_WAITVA) == 0) {
1290 1.238 ad mutex_exit(&map->misc_lock);
1291 1.226 yamt UVMHIST_LOG(maphist,
1292 1.226 yamt "<- uvm_map_findspace failed!", 0,0,0,0);
1293 1.226 yamt return ENOMEM;
1294 1.226 yamt } else {
1295 1.238 ad cv_timedwait(&map->cv, &map->misc_lock, hz);
1296 1.226 yamt }
1297 1.180 yamt }
1298 1.238 ad mutex_exit(&map->misc_lock);
1299 1.180 yamt goto retry;
1300 1.10 mrg }
1301 1.1 mrg
1302 1.40 thorpej #ifdef PMAP_GROWKERNEL
1303 1.152 simonb /*
1304 1.152 simonb * If the kernel pmap can't map the requested space,
1305 1.152 simonb * then allocate more resources for it.
1306 1.152 simonb */
1307 1.229 yamt if (map == kernel_map && uvm_maxkaddr < (start + size))
1308 1.229 yamt uvm_maxkaddr = pmap_growkernel(start + size);
1309 1.10 mrg #endif
1310 1.10 mrg
1311 1.207 yamt UVMMAP_EVCNT_INCR(map_call);
1312 1.10 mrg
1313 1.10 mrg /*
1314 1.10 mrg * if uobj is null, then uoffset is either a VAC hint for PMAP_PREFER
1315 1.98 chs * [typically from uvm_map_reserve] or it is UVM_UNKNOWN_OFFSET. in
1316 1.98 chs * either case we want to zero it before storing it in the map entry
1317 1.10 mrg * (because it looks strange and confusing when debugging...)
1318 1.98 chs *
1319 1.98 chs * if uobj is not null
1320 1.10 mrg * if uoffset is not UVM_UNKNOWN_OFFSET then we have a normal mapping
1321 1.10 mrg * and we do not need to change uoffset.
1322 1.10 mrg * if uoffset is UVM_UNKNOWN_OFFSET then we need to find the offset
1323 1.10 mrg * now (based on the starting address of the map). this case is
1324 1.10 mrg * for kernel object mappings where we don't know the offset until
1325 1.10 mrg * the virtual address is found (with uvm_map_findspace). the
1326 1.10 mrg * offset is the distance we are from the start of the map.
1327 1.10 mrg */
1328 1.10 mrg
1329 1.10 mrg if (uobj == NULL) {
1330 1.10 mrg uoffset = 0;
1331 1.10 mrg } else {
1332 1.10 mrg if (uoffset == UVM_UNKNOWN_OFFSET) {
1333 1.85 chs KASSERT(UVM_OBJ_IS_KERN_OBJECT(uobj));
1334 1.174 yamt uoffset = start - vm_map_min(kernel_map);
1335 1.10 mrg }
1336 1.10 mrg }
1337 1.10 mrg
1338 1.174 yamt args->uma_flags = flags;
1339 1.174 yamt args->uma_prev = prev_entry;
1340 1.174 yamt args->uma_start = start;
1341 1.174 yamt args->uma_size = size;
1342 1.174 yamt args->uma_uobj = uobj;
1343 1.174 yamt args->uma_uoffset = uoffset;
1344 1.174 yamt
1345 1.174 yamt return 0;
1346 1.174 yamt }
1347 1.174 yamt
1348 1.174 yamt int
1349 1.174 yamt uvm_map_enter(struct vm_map *map, const struct uvm_map_args *args,
1350 1.174 yamt struct vm_map_entry *new_entry)
1351 1.174 yamt {
1352 1.174 yamt struct vm_map_entry *prev_entry = args->uma_prev;
1353 1.174 yamt struct vm_map_entry *dead = NULL;
1354 1.174 yamt
1355 1.174 yamt const uvm_flag_t flags = args->uma_flags;
1356 1.174 yamt const vm_prot_t prot = UVM_PROTECTION(flags);
1357 1.174 yamt const vm_prot_t maxprot = UVM_MAXPROTECTION(flags);
1358 1.174 yamt const vm_inherit_t inherit = UVM_INHERIT(flags);
1359 1.174 yamt const int amapwaitflag = (flags & UVM_FLAG_NOWAIT) ?
1360 1.174 yamt AMAP_EXTEND_NOWAIT : 0;
1361 1.174 yamt const int advice = UVM_ADVICE(flags);
1362 1.174 yamt const int meflagval = (flags & UVM_FLAG_QUANTUM) ?
1363 1.174 yamt UVM_MAP_QUANTUM : 0;
1364 1.174 yamt
1365 1.174 yamt vaddr_t start = args->uma_start;
1366 1.174 yamt vsize_t size = args->uma_size;
1367 1.174 yamt struct uvm_object *uobj = args->uma_uobj;
1368 1.174 yamt voff_t uoffset = args->uma_uoffset;
1369 1.174 yamt
1370 1.174 yamt const int kmap = (vm_map_pmap(map) == pmap_kernel());
1371 1.174 yamt int merged = 0;
1372 1.174 yamt int error;
1373 1.176 yamt int newetype;
1374 1.174 yamt
1375 1.174 yamt UVMHIST_FUNC("uvm_map_enter");
1376 1.174 yamt UVMHIST_CALLED(maphist);
1377 1.174 yamt
1378 1.174 yamt UVMHIST_LOG(maphist, "(map=0x%x, start=0x%x, size=%d, flags=0x%x)",
1379 1.174 yamt map, start, size, flags);
1380 1.174 yamt UVMHIST_LOG(maphist, " uobj/offset 0x%x/%d", uobj, uoffset,0,0);
1381 1.174 yamt
1382 1.221 yamt KASSERT(map->hint == prev_entry); /* bimerge case assumes this */
1383 1.221 yamt
1384 1.174 yamt if (flags & UVM_FLAG_QUANTUM) {
1385 1.174 yamt KASSERT(new_entry);
1386 1.174 yamt KASSERT(new_entry->flags & UVM_MAP_QUANTUM);
1387 1.174 yamt }
1388 1.174 yamt
1389 1.176 yamt if (uobj)
1390 1.176 yamt newetype = UVM_ET_OBJ;
1391 1.176 yamt else
1392 1.176 yamt newetype = 0;
1393 1.176 yamt
1394 1.176 yamt if (flags & UVM_FLAG_COPYONW) {
1395 1.176 yamt newetype |= UVM_ET_COPYONWRITE;
1396 1.176 yamt if ((flags & UVM_FLAG_OVERLAY) == 0)
1397 1.176 yamt newetype |= UVM_ET_NEEDSCOPY;
1398 1.176 yamt }
1399 1.176 yamt
1400 1.10 mrg /*
1401 1.106 chs * try and insert in map by extending previous entry, if possible.
1402 1.10 mrg * XXX: we don't try and pull back the next entry. might be useful
1403 1.10 mrg * for a stack, but we are currently allocating our stack in advance.
1404 1.10 mrg */
1405 1.10 mrg
1406 1.121 atatat if (flags & UVM_FLAG_NOMERGE)
1407 1.121 atatat goto nomerge;
1408 1.121 atatat
1409 1.194 yamt if (prev_entry->end == start &&
1410 1.121 atatat prev_entry != &map->header &&
1411 1.194 yamt UVM_ET_ISCOMPATIBLE(prev_entry, newetype, uobj, meflagval,
1412 1.194 yamt prot, maxprot, inherit, advice, 0)) {
1413 1.161 matt
1414 1.10 mrg if (uobj && prev_entry->offset +
1415 1.10 mrg (prev_entry->end - prev_entry->start) != uoffset)
1416 1.121 atatat goto forwardmerge;
1417 1.10 mrg
1418 1.10 mrg /*
1419 1.98 chs * can't extend a shared amap. note: no need to lock amap to
1420 1.34 chuck * look at refs since we don't care about its exact value.
1421 1.10 mrg * if it is one (i.e. we have only reference) it will stay there
1422 1.10 mrg */
1423 1.85 chs
1424 1.10 mrg if (prev_entry->aref.ar_amap &&
1425 1.34 chuck amap_refs(prev_entry->aref.ar_amap) != 1) {
1426 1.121 atatat goto forwardmerge;
1427 1.10 mrg }
1428 1.85 chs
1429 1.119 chs if (prev_entry->aref.ar_amap) {
1430 1.139 enami error = amap_extend(prev_entry, size,
1431 1.126 bouyer amapwaitflag | AMAP_EXTEND_FORWARDS);
1432 1.174 yamt if (error)
1433 1.191 yamt goto nomerge;
1434 1.119 chs }
1435 1.10 mrg
1436 1.258 ad if (kmap) {
1437 1.207 yamt UVMMAP_EVCNT_INCR(kbackmerge);
1438 1.258 ad } else {
1439 1.207 yamt UVMMAP_EVCNT_INCR(ubackmerge);
1440 1.258 ad }
1441 1.10 mrg UVMHIST_LOG(maphist," starting back merge", 0, 0, 0, 0);
1442 1.10 mrg
1443 1.10 mrg /*
1444 1.10 mrg * drop our reference to uobj since we are extending a reference
1445 1.10 mrg * that we already have (the ref count can not drop to zero).
1446 1.10 mrg */
1447 1.119 chs
1448 1.10 mrg if (uobj && uobj->pgops->pgo_detach)
1449 1.10 mrg uobj->pgops->pgo_detach(uobj);
1450 1.10 mrg
1451 1.263 matt /*
1452 1.263 matt * Now that we've merged the entries, note that we've grown
1453 1.263 matt * and our gap has shrunk. Then fix the tree.
1454 1.263 matt */
1455 1.10 mrg prev_entry->end += size;
1456 1.263 matt prev_entry->gap -= size;
1457 1.145 yamt uvm_rb_fixup(map, prev_entry);
1458 1.145 yamt
1459 1.222 yamt uvm_map_check(map, "map backmerged");
1460 1.10 mrg
1461 1.10 mrg UVMHIST_LOG(maphist,"<- done (via backmerge)!", 0, 0, 0, 0);
1462 1.121 atatat merged++;
1463 1.106 chs }
1464 1.10 mrg
1465 1.121 atatat forwardmerge:
1466 1.194 yamt if (prev_entry->next->start == (start + size) &&
1467 1.121 atatat prev_entry->next != &map->header &&
1468 1.194 yamt UVM_ET_ISCOMPATIBLE(prev_entry->next, newetype, uobj, meflagval,
1469 1.194 yamt prot, maxprot, inherit, advice, 0)) {
1470 1.161 matt
1471 1.121 atatat if (uobj && prev_entry->next->offset != uoffset + size)
1472 1.121 atatat goto nomerge;
1473 1.121 atatat
1474 1.121 atatat /*
1475 1.121 atatat * can't extend a shared amap. note: no need to lock amap to
1476 1.121 atatat * look at refs since we don't care about its exact value.
1477 1.122 atatat * if it is one (i.e. we have only reference) it will stay there.
1478 1.122 atatat *
1479 1.122 atatat * note that we also can't merge two amaps, so if we
1480 1.122 atatat * merged with the previous entry which has an amap,
1481 1.122 atatat * and the next entry also has an amap, we give up.
1482 1.122 atatat *
1483 1.125 atatat * Interesting cases:
1484 1.125 atatat * amap, new, amap -> give up second merge (single fwd extend)
1485 1.125 atatat * amap, new, none -> double forward extend (extend again here)
1486 1.125 atatat * none, new, amap -> double backward extend (done here)
1487 1.125 atatat * uobj, new, amap -> single backward extend (done here)
1488 1.125 atatat *
1489 1.122 atatat * XXX should we attempt to deal with someone refilling
1490 1.122 atatat * the deallocated region between two entries that are
1491 1.122 atatat * backed by the same amap (ie, arefs is 2, "prev" and
1492 1.122 atatat * "next" refer to it, and adding this allocation will
1493 1.122 atatat * close the hole, thus restoring arefs to 1 and
1494 1.122 atatat * deallocating the "next" vm_map_entry)? -- @@@
1495 1.121 atatat */
1496 1.121 atatat
1497 1.121 atatat if (prev_entry->next->aref.ar_amap &&
1498 1.122 atatat (amap_refs(prev_entry->next->aref.ar_amap) != 1 ||
1499 1.122 atatat (merged && prev_entry->aref.ar_amap))) {
1500 1.121 atatat goto nomerge;
1501 1.121 atatat }
1502 1.121 atatat
1503 1.122 atatat if (merged) {
1504 1.123 atatat /*
1505 1.123 atatat * Try to extend the amap of the previous entry to
1506 1.123 atatat * cover the next entry as well. If it doesn't work
1507 1.123 atatat * just skip on, don't actually give up, since we've
1508 1.123 atatat * already completed the back merge.
1509 1.123 atatat */
1510 1.125 atatat if (prev_entry->aref.ar_amap) {
1511 1.125 atatat if (amap_extend(prev_entry,
1512 1.125 atatat prev_entry->next->end -
1513 1.125 atatat prev_entry->next->start,
1514 1.126 bouyer amapwaitflag | AMAP_EXTEND_FORWARDS))
1515 1.142 enami goto nomerge;
1516 1.125 atatat }
1517 1.125 atatat
1518 1.125 atatat /*
1519 1.125 atatat * Try to extend the amap of the *next* entry
1520 1.125 atatat * back to cover the new allocation *and* the
1521 1.125 atatat * previous entry as well (the previous merge
1522 1.125 atatat * didn't have an amap already otherwise we
1523 1.125 atatat * wouldn't be checking here for an amap). If
1524 1.125 atatat * it doesn't work just skip on, again, don't
1525 1.125 atatat * actually give up, since we've already
1526 1.125 atatat * completed the back merge.
1527 1.125 atatat */
1528 1.125 atatat else if (prev_entry->next->aref.ar_amap) {
1529 1.125 atatat if (amap_extend(prev_entry->next,
1530 1.125 atatat prev_entry->end -
1531 1.141 atatat prev_entry->start,
1532 1.126 bouyer amapwaitflag | AMAP_EXTEND_BACKWARDS))
1533 1.142 enami goto nomerge;
1534 1.125 atatat }
1535 1.125 atatat } else {
1536 1.125 atatat /*
1537 1.125 atatat * Pull the next entry's amap backwards to cover this
1538 1.125 atatat * new allocation.
1539 1.125 atatat */
1540 1.125 atatat if (prev_entry->next->aref.ar_amap) {
1541 1.125 atatat error = amap_extend(prev_entry->next, size,
1542 1.126 bouyer amapwaitflag | AMAP_EXTEND_BACKWARDS);
1543 1.174 yamt if (error)
1544 1.191 yamt goto nomerge;
1545 1.125 atatat }
1546 1.122 atatat }
1547 1.122 atatat
1548 1.121 atatat if (merged) {
1549 1.121 atatat if (kmap) {
1550 1.207 yamt UVMMAP_EVCNT_DECR(kbackmerge);
1551 1.207 yamt UVMMAP_EVCNT_INCR(kbimerge);
1552 1.121 atatat } else {
1553 1.207 yamt UVMMAP_EVCNT_DECR(ubackmerge);
1554 1.207 yamt UVMMAP_EVCNT_INCR(ubimerge);
1555 1.121 atatat }
1556 1.122 atatat } else {
1557 1.258 ad if (kmap) {
1558 1.207 yamt UVMMAP_EVCNT_INCR(kforwmerge);
1559 1.258 ad } else {
1560 1.207 yamt UVMMAP_EVCNT_INCR(uforwmerge);
1561 1.258 ad }
1562 1.121 atatat }
1563 1.121 atatat UVMHIST_LOG(maphist," starting forward merge", 0, 0, 0, 0);
1564 1.10 mrg
1565 1.121 atatat /*
1566 1.121 atatat * drop our reference to uobj since we are extending a reference
1567 1.121 atatat * that we already have (the ref count can not drop to zero).
1568 1.121 atatat * (if merged, we've already detached)
1569 1.121 atatat */
1570 1.121 atatat if (uobj && uobj->pgops->pgo_detach && !merged)
1571 1.121 atatat uobj->pgops->pgo_detach(uobj);
1572 1.1 mrg
1573 1.121 atatat if (merged) {
1574 1.174 yamt dead = prev_entry->next;
1575 1.121 atatat prev_entry->end = dead->end;
1576 1.121 atatat uvm_map_entry_unlink(map, dead);
1577 1.125 atatat if (dead->aref.ar_amap != NULL) {
1578 1.125 atatat prev_entry->aref = dead->aref;
1579 1.125 atatat dead->aref.ar_amap = NULL;
1580 1.125 atatat }
1581 1.121 atatat } else {
1582 1.121 atatat prev_entry->next->start -= size;
1583 1.263 matt if (prev_entry != &map->header) {
1584 1.263 matt prev_entry->gap -= size;
1585 1.263 matt KASSERT(prev_entry->gap == uvm_rb_gap(prev_entry));
1586 1.145 yamt uvm_rb_fixup(map, prev_entry);
1587 1.263 matt }
1588 1.121 atatat if (uobj)
1589 1.121 atatat prev_entry->next->offset = uoffset;
1590 1.121 atatat }
1591 1.145 yamt
1592 1.222 yamt uvm_map_check(map, "map forwardmerged");
1593 1.1 mrg
1594 1.121 atatat UVMHIST_LOG(maphist,"<- done forwardmerge", 0, 0, 0, 0);
1595 1.121 atatat merged++;
1596 1.106 chs }
1597 1.121 atatat
1598 1.121 atatat nomerge:
1599 1.121 atatat if (!merged) {
1600 1.121 atatat UVMHIST_LOG(maphist," allocating new map entry", 0, 0, 0, 0);
1601 1.258 ad if (kmap) {
1602 1.207 yamt UVMMAP_EVCNT_INCR(knomerge);
1603 1.258 ad } else {
1604 1.207 yamt UVMMAP_EVCNT_INCR(unomerge);
1605 1.258 ad }
1606 1.106 chs
1607 1.10 mrg /*
1608 1.121 atatat * allocate new entry and link it in.
1609 1.10 mrg */
1610 1.106 chs
1611 1.121 atatat if (new_entry == NULL) {
1612 1.126 bouyer new_entry = uvm_mapent_alloc(map,
1613 1.127 thorpej (flags & UVM_FLAG_NOWAIT));
1614 1.126 bouyer if (__predict_false(new_entry == NULL)) {
1615 1.174 yamt error = ENOMEM;
1616 1.174 yamt goto done;
1617 1.126 bouyer }
1618 1.121 atatat }
1619 1.174 yamt new_entry->start = start;
1620 1.121 atatat new_entry->end = new_entry->start + size;
1621 1.121 atatat new_entry->object.uvm_obj = uobj;
1622 1.121 atatat new_entry->offset = uoffset;
1623 1.121 atatat
1624 1.176 yamt new_entry->etype = newetype;
1625 1.121 atatat
1626 1.161 matt if (flags & UVM_FLAG_NOMERGE) {
1627 1.161 matt new_entry->flags |= UVM_MAP_NOMERGE;
1628 1.161 matt }
1629 1.121 atatat
1630 1.121 atatat new_entry->protection = prot;
1631 1.121 atatat new_entry->max_protection = maxprot;
1632 1.121 atatat new_entry->inheritance = inherit;
1633 1.121 atatat new_entry->wired_count = 0;
1634 1.121 atatat new_entry->advice = advice;
1635 1.121 atatat if (flags & UVM_FLAG_OVERLAY) {
1636 1.121 atatat
1637 1.121 atatat /*
1638 1.121 atatat * to_add: for BSS we overallocate a little since we
1639 1.121 atatat * are likely to extend
1640 1.121 atatat */
1641 1.121 atatat
1642 1.121 atatat vaddr_t to_add = (flags & UVM_FLAG_AMAPPAD) ?
1643 1.121 atatat UVM_AMAP_CHUNK << PAGE_SHIFT : 0;
1644 1.126 bouyer struct vm_amap *amap = amap_alloc(size, to_add,
1645 1.227 yamt (flags & UVM_FLAG_NOWAIT));
1646 1.126 bouyer if (__predict_false(amap == NULL)) {
1647 1.174 yamt error = ENOMEM;
1648 1.174 yamt goto done;
1649 1.126 bouyer }
1650 1.121 atatat new_entry->aref.ar_pageoff = 0;
1651 1.121 atatat new_entry->aref.ar_amap = amap;
1652 1.121 atatat } else {
1653 1.121 atatat new_entry->aref.ar_pageoff = 0;
1654 1.121 atatat new_entry->aref.ar_amap = NULL;
1655 1.121 atatat }
1656 1.121 atatat uvm_map_entry_link(map, prev_entry, new_entry);
1657 1.1 mrg
1658 1.121 atatat /*
1659 1.121 atatat * Update the free space hint
1660 1.121 atatat */
1661 1.10 mrg
1662 1.121 atatat if ((map->first_free == prev_entry) &&
1663 1.121 atatat (prev_entry->end >= new_entry->start))
1664 1.121 atatat map->first_free = new_entry;
1665 1.174 yamt
1666 1.174 yamt new_entry = NULL;
1667 1.121 atatat }
1668 1.10 mrg
1669 1.146 yamt map->size += size;
1670 1.146 yamt
1671 1.10 mrg UVMHIST_LOG(maphist,"<- done!", 0, 0, 0, 0);
1672 1.174 yamt
1673 1.174 yamt error = 0;
1674 1.174 yamt done:
1675 1.248 ad if ((flags & UVM_FLAG_QUANTUM) == 0) {
1676 1.248 ad /*
1677 1.248 ad * vmk_merged_entries is locked by the map's lock.
1678 1.248 ad */
1679 1.248 ad vm_map_unlock(map);
1680 1.248 ad }
1681 1.248 ad if (new_entry && error == 0) {
1682 1.248 ad KDASSERT(merged);
1683 1.248 ad uvm_mapent_free_merged(map, new_entry);
1684 1.248 ad new_entry = NULL;
1685 1.174 yamt }
1686 1.174 yamt if (dead) {
1687 1.174 yamt KDASSERT(merged);
1688 1.181 yamt uvm_mapent_free_merged(map, dead);
1689 1.174 yamt }
1690 1.248 ad if ((flags & UVM_FLAG_QUANTUM) != 0) {
1691 1.248 ad vm_map_unlock(map);
1692 1.248 ad }
1693 1.248 ad if (new_entry != NULL) {
1694 1.248 ad uvm_mapent_free(new_entry);
1695 1.248 ad }
1696 1.174 yamt return error;
1697 1.1 mrg }
1698 1.1 mrg
1699 1.1 mrg /*
1700 1.247 yamt * uvm_map_lookup_entry_bytree: lookup an entry in tree
1701 1.247 yamt */
1702 1.247 yamt
1703 1.263 matt static inline bool
1704 1.247 yamt uvm_map_lookup_entry_bytree(struct vm_map *map, vaddr_t address,
1705 1.247 yamt struct vm_map_entry **entry /* OUT */)
1706 1.247 yamt {
1707 1.247 yamt struct vm_map_entry *prev = &map->header;
1708 1.263 matt struct vm_map_entry *cur = ROOT_ENTRY(map);
1709 1.247 yamt
1710 1.247 yamt while (cur) {
1711 1.263 matt UVMMAP_EVCNT_INCR(mlk_treeloop);
1712 1.247 yamt if (address >= cur->start) {
1713 1.247 yamt if (address < cur->end) {
1714 1.247 yamt *entry = cur;
1715 1.247 yamt return true;
1716 1.247 yamt }
1717 1.247 yamt prev = cur;
1718 1.263 matt cur = RIGHT_ENTRY(cur);
1719 1.247 yamt } else
1720 1.263 matt cur = LEFT_ENTRY(cur);
1721 1.247 yamt }
1722 1.247 yamt *entry = prev;
1723 1.247 yamt return false;
1724 1.247 yamt }
1725 1.247 yamt
1726 1.247 yamt /*
1727 1.1 mrg * uvm_map_lookup_entry: find map entry at or before an address
1728 1.1 mrg *
1729 1.1 mrg * => map must at least be read-locked by caller
1730 1.1 mrg * => entry is returned in "entry"
1731 1.1 mrg * => return value is true if address is in the returned entry
1732 1.1 mrg */
1733 1.1 mrg
1734 1.233 thorpej bool
1735 1.138 enami uvm_map_lookup_entry(struct vm_map *map, vaddr_t address,
1736 1.138 enami struct vm_map_entry **entry /* OUT */)
1737 1.1 mrg {
1738 1.99 chs struct vm_map_entry *cur;
1739 1.234 thorpej bool use_tree = false;
1740 1.1 mrg UVMHIST_FUNC("uvm_map_lookup_entry");
1741 1.1 mrg UVMHIST_CALLED(maphist);
1742 1.1 mrg
1743 1.1 mrg UVMHIST_LOG(maphist,"(map=0x%x,addr=0x%x,ent=0x%x)",
1744 1.10 mrg map, address, entry, 0);
1745 1.1 mrg
1746 1.1 mrg /*
1747 1.10 mrg * start looking either from the head of the
1748 1.10 mrg * list, or from the hint.
1749 1.1 mrg */
1750 1.1 mrg
1751 1.1 mrg cur = map->hint;
1752 1.1 mrg
1753 1.1 mrg if (cur == &map->header)
1754 1.1 mrg cur = cur->next;
1755 1.1 mrg
1756 1.207 yamt UVMMAP_EVCNT_INCR(mlk_call);
1757 1.1 mrg if (address >= cur->start) {
1758 1.99 chs
1759 1.139 enami /*
1760 1.10 mrg * go from hint to end of list.
1761 1.1 mrg *
1762 1.10 mrg * but first, make a quick check to see if
1763 1.10 mrg * we are already looking at the entry we
1764 1.10 mrg * want (which is usually the case).
1765 1.10 mrg * note also that we don't need to save the hint
1766 1.10 mrg * here... it is the same hint (unless we are
1767 1.10 mrg * at the header, in which case the hint didn't
1768 1.10 mrg * buy us anything anyway).
1769 1.1 mrg */
1770 1.99 chs
1771 1.144 yamt if (cur != &map->header && cur->end > address) {
1772 1.207 yamt UVMMAP_EVCNT_INCR(mlk_hint);
1773 1.1 mrg *entry = cur;
1774 1.1 mrg UVMHIST_LOG(maphist,"<- got it via hint (0x%x)",
1775 1.10 mrg cur, 0, 0, 0);
1776 1.218 yamt uvm_mapent_check(*entry);
1777 1.234 thorpej return (true);
1778 1.1 mrg }
1779 1.144 yamt
1780 1.263 matt if (map->nentries > 15)
1781 1.234 thorpej use_tree = true;
1782 1.10 mrg } else {
1783 1.99 chs
1784 1.139 enami /*
1785 1.144 yamt * invalid hint. use tree.
1786 1.1 mrg */
1787 1.234 thorpej use_tree = true;
1788 1.144 yamt }
1789 1.144 yamt
1790 1.222 yamt uvm_map_check(map, __func__);
1791 1.144 yamt
1792 1.144 yamt if (use_tree) {
1793 1.144 yamt /*
1794 1.144 yamt * Simple lookup in the tree. Happens when the hint is
1795 1.144 yamt * invalid, or nentries reach a threshold.
1796 1.144 yamt */
1797 1.263 matt UVMMAP_EVCNT_INCR(mlk_tree);
1798 1.247 yamt if (uvm_map_lookup_entry_bytree(map, address, entry)) {
1799 1.247 yamt goto got;
1800 1.247 yamt } else {
1801 1.247 yamt goto failed;
1802 1.144 yamt }
1803 1.1 mrg }
1804 1.1 mrg
1805 1.1 mrg /*
1806 1.10 mrg * search linearly
1807 1.1 mrg */
1808 1.1 mrg
1809 1.263 matt UVMMAP_EVCNT_INCR(mlk_list);
1810 1.144 yamt while (cur != &map->header) {
1811 1.263 matt UVMMAP_EVCNT_INCR(mlk_listloop);
1812 1.1 mrg if (cur->end > address) {
1813 1.1 mrg if (address >= cur->start) {
1814 1.139 enami /*
1815 1.10 mrg * save this lookup for future
1816 1.10 mrg * hints, and return
1817 1.1 mrg */
1818 1.1 mrg
1819 1.1 mrg *entry = cur;
1820 1.144 yamt got:
1821 1.144 yamt SAVE_HINT(map, map->hint, *entry);
1822 1.1 mrg UVMHIST_LOG(maphist,"<- search got it (0x%x)",
1823 1.10 mrg cur, 0, 0, 0);
1824 1.144 yamt KDASSERT((*entry)->start <= address);
1825 1.144 yamt KDASSERT(address < (*entry)->end);
1826 1.218 yamt uvm_mapent_check(*entry);
1827 1.234 thorpej return (true);
1828 1.1 mrg }
1829 1.1 mrg break;
1830 1.1 mrg }
1831 1.1 mrg cur = cur->next;
1832 1.1 mrg }
1833 1.1 mrg *entry = cur->prev;
1834 1.144 yamt failed:
1835 1.82 thorpej SAVE_HINT(map, map->hint, *entry);
1836 1.1 mrg UVMHIST_LOG(maphist,"<- failed!",0,0,0,0);
1837 1.147 yamt KDASSERT((*entry) == &map->header || (*entry)->end <= address);
1838 1.144 yamt KDASSERT((*entry)->next == &map->header ||
1839 1.144 yamt address < (*entry)->next->start);
1840 1.234 thorpej return (false);
1841 1.1 mrg }
1842 1.1 mrg
1843 1.1 mrg /*
1844 1.140 enami * See if the range between start and start + length fits in the gap
1845 1.140 enami * entry->next->start and entry->end. Returns 1 if fits, 0 if doesn't
1846 1.140 enami * fit, and -1 address wraps around.
1847 1.140 enami */
1848 1.203 thorpej static int
1849 1.232 yamt uvm_map_space_avail(vaddr_t *start, vsize_t length, voff_t uoffset,
1850 1.140 enami vsize_t align, int topdown, struct vm_map_entry *entry)
1851 1.140 enami {
1852 1.140 enami vaddr_t end;
1853 1.140 enami
1854 1.140 enami #ifdef PMAP_PREFER
1855 1.140 enami /*
1856 1.140 enami * push start address forward as needed to avoid VAC alias problems.
1857 1.140 enami * we only do this if a valid offset is specified.
1858 1.140 enami */
1859 1.140 enami
1860 1.140 enami if (uoffset != UVM_UNKNOWN_OFFSET)
1861 1.182 atatat PMAP_PREFER(uoffset, start, length, topdown);
1862 1.140 enami #endif
1863 1.140 enami if (align != 0) {
1864 1.140 enami if ((*start & (align - 1)) != 0) {
1865 1.140 enami if (topdown)
1866 1.140 enami *start &= ~(align - 1);
1867 1.140 enami else
1868 1.140 enami *start = roundup(*start, align);
1869 1.140 enami }
1870 1.140 enami /*
1871 1.140 enami * XXX Should we PMAP_PREFER() here again?
1872 1.182 atatat * eh...i think we're okay
1873 1.140 enami */
1874 1.140 enami }
1875 1.140 enami
1876 1.140 enami /*
1877 1.140 enami * Find the end of the proposed new region. Be sure we didn't
1878 1.140 enami * wrap around the address; if so, we lose. Otherwise, if the
1879 1.140 enami * proposed new region fits before the next entry, we win.
1880 1.140 enami */
1881 1.140 enami
1882 1.140 enami end = *start + length;
1883 1.140 enami if (end < *start)
1884 1.140 enami return (-1);
1885 1.140 enami
1886 1.140 enami if (entry->next->start >= end && *start >= entry->end)
1887 1.140 enami return (1);
1888 1.140 enami
1889 1.140 enami return (0);
1890 1.140 enami }
1891 1.140 enami
1892 1.140 enami /*
1893 1.1 mrg * uvm_map_findspace: find "length" sized space in "map".
1894 1.1 mrg *
1895 1.167 junyoung * => "hint" is a hint about where we want it, unless UVM_FLAG_FIXED is
1896 1.167 junyoung * set in "flags" (in which case we insist on using "hint").
1897 1.1 mrg * => "result" is VA returned
1898 1.1 mrg * => uobj/uoffset are to be used to handle VAC alignment, if required
1899 1.167 junyoung * => if "align" is non-zero, we attempt to align to that value.
1900 1.1 mrg * => caller must at least have read-locked map
1901 1.1 mrg * => returns NULL on failure, or pointer to prev. map entry if success
1902 1.1 mrg * => note this is a cross between the old vm_map_findspace and vm_map_find
1903 1.1 mrg */
1904 1.1 mrg
1905 1.99 chs struct vm_map_entry *
1906 1.138 enami uvm_map_findspace(struct vm_map *map, vaddr_t hint, vsize_t length,
1907 1.232 yamt vaddr_t *result /* OUT */, struct uvm_object *uobj, voff_t uoffset,
1908 1.138 enami vsize_t align, int flags)
1909 1.1 mrg {
1910 1.140 enami struct vm_map_entry *entry;
1911 1.144 yamt struct vm_map_entry *child, *prev, *tmp;
1912 1.140 enami vaddr_t orig_hint;
1913 1.131 atatat const int topdown = map->flags & VM_MAP_TOPDOWN;
1914 1.1 mrg UVMHIST_FUNC("uvm_map_findspace");
1915 1.1 mrg UVMHIST_CALLED(maphist);
1916 1.1 mrg
1917 1.98 chs UVMHIST_LOG(maphist, "(map=0x%x, hint=0x%x, len=%d, flags=0x%x)",
1918 1.140 enami map, hint, length, flags);
1919 1.85 chs KASSERT((align & (align - 1)) == 0);
1920 1.85 chs KASSERT((flags & UVM_FLAG_FIXED) == 0 || align == 0);
1921 1.81 thorpej
1922 1.222 yamt uvm_map_check(map, "map_findspace entry");
1923 1.144 yamt
1924 1.81 thorpej /*
1925 1.81 thorpej * remember the original hint. if we are aligning, then we
1926 1.81 thorpej * may have to try again with no alignment constraint if
1927 1.81 thorpej * we fail the first time.
1928 1.81 thorpej */
1929 1.85 chs
1930 1.81 thorpej orig_hint = hint;
1931 1.184 chs if (hint < vm_map_min(map)) { /* check ranges ... */
1932 1.81 thorpej if (flags & UVM_FLAG_FIXED) {
1933 1.1 mrg UVMHIST_LOG(maphist,"<- VA below map range",0,0,0,0);
1934 1.139 enami return (NULL);
1935 1.1 mrg }
1936 1.184 chs hint = vm_map_min(map);
1937 1.1 mrg }
1938 1.184 chs if (hint > vm_map_max(map)) {
1939 1.1 mrg UVMHIST_LOG(maphist,"<- VA 0x%x > range [0x%x->0x%x]",
1940 1.184 chs hint, vm_map_min(map), vm_map_max(map), 0);
1941 1.139 enami return (NULL);
1942 1.1 mrg }
1943 1.1 mrg
1944 1.1 mrg /*
1945 1.1 mrg * Look for the first possible address; if there's already
1946 1.1 mrg * something at this address, we have to start after it.
1947 1.1 mrg */
1948 1.1 mrg
1949 1.131 atatat /*
1950 1.131 atatat * @@@: there are four, no, eight cases to consider.
1951 1.131 atatat *
1952 1.131 atatat * 0: found, fixed, bottom up -> fail
1953 1.131 atatat * 1: found, fixed, top down -> fail
1954 1.140 enami * 2: found, not fixed, bottom up -> start after entry->end,
1955 1.140 enami * loop up
1956 1.140 enami * 3: found, not fixed, top down -> start before entry->start,
1957 1.140 enami * loop down
1958 1.140 enami * 4: not found, fixed, bottom up -> check entry->next->start, fail
1959 1.140 enami * 5: not found, fixed, top down -> check entry->next->start, fail
1960 1.140 enami * 6: not found, not fixed, bottom up -> check entry->next->start,
1961 1.140 enami * loop up
1962 1.140 enami * 7: not found, not fixed, top down -> check entry->next->start,
1963 1.140 enami * loop down
1964 1.131 atatat *
1965 1.131 atatat * as you can see, it reduces to roughly five cases, and that
1966 1.131 atatat * adding top down mapping only adds one unique case (without
1967 1.131 atatat * it, there would be four cases).
1968 1.131 atatat */
1969 1.131 atatat
1970 1.184 chs if ((flags & UVM_FLAG_FIXED) == 0 && hint == vm_map_min(map)) {
1971 1.140 enami entry = map->first_free;
1972 1.1 mrg } else {
1973 1.140 enami if (uvm_map_lookup_entry(map, hint, &entry)) {
1974 1.1 mrg /* "hint" address already in use ... */
1975 1.81 thorpej if (flags & UVM_FLAG_FIXED) {
1976 1.140 enami UVMHIST_LOG(maphist, "<- fixed & VA in use",
1977 1.10 mrg 0, 0, 0, 0);
1978 1.139 enami return (NULL);
1979 1.1 mrg }
1980 1.140 enami if (topdown)
1981 1.140 enami /* Start from lower gap. */
1982 1.140 enami entry = entry->prev;
1983 1.140 enami } else if (flags & UVM_FLAG_FIXED) {
1984 1.140 enami if (entry->next->start >= hint + length &&
1985 1.140 enami hint + length > hint)
1986 1.140 enami goto found;
1987 1.140 enami
1988 1.140 enami /* "hint" address is gap but too small */
1989 1.140 enami UVMHIST_LOG(maphist, "<- fixed mapping failed",
1990 1.140 enami 0, 0, 0, 0);
1991 1.140 enami return (NULL); /* only one shot at it ... */
1992 1.140 enami } else {
1993 1.140 enami /*
1994 1.140 enami * See if given hint fits in this gap.
1995 1.140 enami */
1996 1.140 enami switch (uvm_map_space_avail(&hint, length,
1997 1.140 enami uoffset, align, topdown, entry)) {
1998 1.140 enami case 1:
1999 1.140 enami goto found;
2000 1.140 enami case -1:
2001 1.140 enami goto wraparound;
2002 1.140 enami }
2003 1.140 enami
2004 1.148 yamt if (topdown) {
2005 1.140 enami /*
2006 1.140 enami * Still there is a chance to fit
2007 1.140 enami * if hint > entry->end.
2008 1.140 enami */
2009 1.148 yamt } else {
2010 1.168 junyoung /* Start from higher gap. */
2011 1.148 yamt entry = entry->next;
2012 1.148 yamt if (entry == &map->header)
2013 1.148 yamt goto notfound;
2014 1.140 enami goto nextgap;
2015 1.148 yamt }
2016 1.1 mrg }
2017 1.1 mrg }
2018 1.1 mrg
2019 1.1 mrg /*
2020 1.144 yamt * Note that all UVM_FLAGS_FIXED case is already handled.
2021 1.144 yamt */
2022 1.144 yamt KDASSERT((flags & UVM_FLAG_FIXED) == 0);
2023 1.144 yamt
2024 1.144 yamt /* Try to find the space in the red-black tree */
2025 1.144 yamt
2026 1.144 yamt /* Check slot before any entry */
2027 1.144 yamt hint = topdown ? entry->next->start - length : entry->end;
2028 1.144 yamt switch (uvm_map_space_avail(&hint, length, uoffset, align,
2029 1.144 yamt topdown, entry)) {
2030 1.144 yamt case 1:
2031 1.144 yamt goto found;
2032 1.144 yamt case -1:
2033 1.144 yamt goto wraparound;
2034 1.144 yamt }
2035 1.144 yamt
2036 1.144 yamt nextgap:
2037 1.148 yamt KDASSERT((flags & UVM_FLAG_FIXED) == 0);
2038 1.144 yamt /* If there is not enough space in the whole tree, we fail */
2039 1.263 matt tmp = ROOT_ENTRY(map);
2040 1.263 matt if (tmp == NULL || tmp->maxgap < length)
2041 1.144 yamt goto notfound;
2042 1.144 yamt
2043 1.144 yamt prev = NULL; /* previous candidate */
2044 1.144 yamt
2045 1.144 yamt /* Find an entry close to hint that has enough space */
2046 1.144 yamt for (; tmp;) {
2047 1.263 matt KASSERT(tmp->next->start == tmp->end + tmp->gap);
2048 1.144 yamt if (topdown) {
2049 1.144 yamt if (tmp->next->start < hint + length &&
2050 1.144 yamt (prev == NULL || tmp->end > prev->end)) {
2051 1.263 matt if (tmp->gap >= length)
2052 1.144 yamt prev = tmp;
2053 1.263 matt else if ((child = LEFT_ENTRY(tmp)) != NULL
2054 1.263 matt && child->maxgap >= length)
2055 1.144 yamt prev = tmp;
2056 1.144 yamt }
2057 1.144 yamt } else {
2058 1.144 yamt if (tmp->end >= hint &&
2059 1.144 yamt (prev == NULL || tmp->end < prev->end)) {
2060 1.263 matt if (tmp->gap >= length)
2061 1.144 yamt prev = tmp;
2062 1.263 matt else if ((child = RIGHT_ENTRY(tmp)) != NULL
2063 1.263 matt && child->maxgap >= length)
2064 1.144 yamt prev = tmp;
2065 1.144 yamt }
2066 1.144 yamt }
2067 1.144 yamt if (tmp->next->start < hint + length)
2068 1.263 matt child = RIGHT_ENTRY(tmp);
2069 1.144 yamt else if (tmp->end > hint)
2070 1.263 matt child = LEFT_ENTRY(tmp);
2071 1.144 yamt else {
2072 1.263 matt if (tmp->gap >= length)
2073 1.144 yamt break;
2074 1.144 yamt if (topdown)
2075 1.263 matt child = LEFT_ENTRY(tmp);
2076 1.144 yamt else
2077 1.263 matt child = RIGHT_ENTRY(tmp);
2078 1.144 yamt }
2079 1.263 matt if (child == NULL || child->maxgap < length)
2080 1.144 yamt break;
2081 1.144 yamt tmp = child;
2082 1.144 yamt }
2083 1.144 yamt
2084 1.148 yamt if (tmp != NULL && tmp->start < hint && hint < tmp->next->start) {
2085 1.164 junyoung /*
2086 1.144 yamt * Check if the entry that we found satifies the
2087 1.144 yamt * space requirement
2088 1.144 yamt */
2089 1.148 yamt if (topdown) {
2090 1.149 yamt if (hint > tmp->next->start - length)
2091 1.149 yamt hint = tmp->next->start - length;
2092 1.148 yamt } else {
2093 1.149 yamt if (hint < tmp->end)
2094 1.149 yamt hint = tmp->end;
2095 1.148 yamt }
2096 1.148 yamt switch (uvm_map_space_avail(&hint, length, uoffset, align,
2097 1.148 yamt topdown, tmp)) {
2098 1.148 yamt case 1:
2099 1.144 yamt entry = tmp;
2100 1.144 yamt goto found;
2101 1.148 yamt case -1:
2102 1.148 yamt goto wraparound;
2103 1.144 yamt }
2104 1.263 matt if (tmp->gap >= length)
2105 1.144 yamt goto listsearch;
2106 1.144 yamt }
2107 1.144 yamt if (prev == NULL)
2108 1.144 yamt goto notfound;
2109 1.144 yamt
2110 1.148 yamt if (topdown) {
2111 1.150 yamt KASSERT(orig_hint >= prev->next->start - length ||
2112 1.148 yamt prev->next->start - length > prev->next->start);
2113 1.148 yamt hint = prev->next->start - length;
2114 1.148 yamt } else {
2115 1.150 yamt KASSERT(orig_hint <= prev->end);
2116 1.148 yamt hint = prev->end;
2117 1.148 yamt }
2118 1.148 yamt switch (uvm_map_space_avail(&hint, length, uoffset, align,
2119 1.148 yamt topdown, prev)) {
2120 1.148 yamt case 1:
2121 1.144 yamt entry = prev;
2122 1.144 yamt goto found;
2123 1.148 yamt case -1:
2124 1.148 yamt goto wraparound;
2125 1.144 yamt }
2126 1.263 matt if (prev->gap >= length)
2127 1.144 yamt goto listsearch;
2128 1.164 junyoung
2129 1.144 yamt if (topdown)
2130 1.263 matt tmp = LEFT_ENTRY(prev);
2131 1.144 yamt else
2132 1.263 matt tmp = RIGHT_ENTRY(prev);
2133 1.144 yamt for (;;) {
2134 1.263 matt KASSERT(tmp && tmp->maxgap >= length);
2135 1.144 yamt if (topdown)
2136 1.263 matt child = RIGHT_ENTRY(tmp);
2137 1.144 yamt else
2138 1.263 matt child = LEFT_ENTRY(tmp);
2139 1.263 matt if (child && child->maxgap >= length) {
2140 1.144 yamt tmp = child;
2141 1.144 yamt continue;
2142 1.144 yamt }
2143 1.263 matt if (tmp->gap >= length)
2144 1.144 yamt break;
2145 1.144 yamt if (topdown)
2146 1.263 matt tmp = LEFT_ENTRY(tmp);
2147 1.144 yamt else
2148 1.263 matt tmp = RIGHT_ENTRY(tmp);
2149 1.144 yamt }
2150 1.164 junyoung
2151 1.148 yamt if (topdown) {
2152 1.150 yamt KASSERT(orig_hint >= tmp->next->start - length ||
2153 1.148 yamt tmp->next->start - length > tmp->next->start);
2154 1.148 yamt hint = tmp->next->start - length;
2155 1.148 yamt } else {
2156 1.150 yamt KASSERT(orig_hint <= tmp->end);
2157 1.148 yamt hint = tmp->end;
2158 1.148 yamt }
2159 1.144 yamt switch (uvm_map_space_avail(&hint, length, uoffset, align,
2160 1.144 yamt topdown, tmp)) {
2161 1.144 yamt case 1:
2162 1.144 yamt entry = tmp;
2163 1.144 yamt goto found;
2164 1.148 yamt case -1:
2165 1.148 yamt goto wraparound;
2166 1.144 yamt }
2167 1.144 yamt
2168 1.164 junyoung /*
2169 1.144 yamt * The tree fails to find an entry because of offset or alignment
2170 1.144 yamt * restrictions. Search the list instead.
2171 1.144 yamt */
2172 1.144 yamt listsearch:
2173 1.144 yamt /*
2174 1.1 mrg * Look through the rest of the map, trying to fit a new region in
2175 1.1 mrg * the gap between existing regions, or after the very last region.
2176 1.140 enami * note: entry->end = base VA of current gap,
2177 1.140 enami * entry->next->start = VA of end of current gap
2178 1.1 mrg */
2179 1.99 chs
2180 1.140 enami for (;;) {
2181 1.140 enami /* Update hint for current gap. */
2182 1.140 enami hint = topdown ? entry->next->start - length : entry->end;
2183 1.140 enami
2184 1.140 enami /* See if it fits. */
2185 1.140 enami switch (uvm_map_space_avail(&hint, length, uoffset, align,
2186 1.140 enami topdown, entry)) {
2187 1.140 enami case 1:
2188 1.140 enami goto found;
2189 1.140 enami case -1:
2190 1.140 enami goto wraparound;
2191 1.140 enami }
2192 1.140 enami
2193 1.140 enami /* Advance to next/previous gap */
2194 1.140 enami if (topdown) {
2195 1.140 enami if (entry == &map->header) {
2196 1.140 enami UVMHIST_LOG(maphist, "<- failed (off start)",
2197 1.140 enami 0,0,0,0);
2198 1.140 enami goto notfound;
2199 1.134 matt }
2200 1.140 enami entry = entry->prev;
2201 1.140 enami } else {
2202 1.140 enami entry = entry->next;
2203 1.140 enami if (entry == &map->header) {
2204 1.140 enami UVMHIST_LOG(maphist, "<- failed (off end)",
2205 1.81 thorpej 0,0,0,0);
2206 1.140 enami goto notfound;
2207 1.81 thorpej }
2208 1.1 mrg }
2209 1.1 mrg }
2210 1.140 enami
2211 1.140 enami found:
2212 1.82 thorpej SAVE_HINT(map, map->hint, entry);
2213 1.1 mrg *result = hint;
2214 1.1 mrg UVMHIST_LOG(maphist,"<- got it! (result=0x%x)", hint, 0,0,0);
2215 1.148 yamt KASSERT( topdown || hint >= orig_hint);
2216 1.148 yamt KASSERT(!topdown || hint <= orig_hint);
2217 1.144 yamt KASSERT(entry->end <= hint);
2218 1.144 yamt KASSERT(hint + length <= entry->next->start);
2219 1.1 mrg return (entry);
2220 1.140 enami
2221 1.140 enami wraparound:
2222 1.140 enami UVMHIST_LOG(maphist, "<- failed (wrap around)", 0,0,0,0);
2223 1.140 enami
2224 1.165 yamt return (NULL);
2225 1.165 yamt
2226 1.140 enami notfound:
2227 1.165 yamt UVMHIST_LOG(maphist, "<- failed (notfound)", 0,0,0,0);
2228 1.165 yamt
2229 1.140 enami return (NULL);
2230 1.1 mrg }
2231 1.1 mrg
2232 1.1 mrg /*
2233 1.1 mrg * U N M A P - m a i n h e l p e r f u n c t i o n s
2234 1.1 mrg */
2235 1.1 mrg
2236 1.1 mrg /*
2237 1.1 mrg * uvm_unmap_remove: remove mappings from a vm_map (from "start" up to "stop")
2238 1.1 mrg *
2239 1.98 chs * => caller must check alignment and size
2240 1.1 mrg * => map must be locked by caller
2241 1.1 mrg * => we return a list of map entries that we've remove from the map
2242 1.1 mrg * in "entry_list"
2243 1.1 mrg */
2244 1.1 mrg
2245 1.94 chs void
2246 1.138 enami uvm_unmap_remove(struct vm_map *map, vaddr_t start, vaddr_t end,
2247 1.174 yamt struct vm_map_entry **entry_list /* OUT */,
2248 1.187 yamt struct uvm_mapent_reservation *umr, int flags)
2249 1.10 mrg {
2250 1.99 chs struct vm_map_entry *entry, *first_entry, *next;
2251 1.24 eeh vaddr_t len;
2252 1.99 chs UVMHIST_FUNC("uvm_unmap_remove"); UVMHIST_CALLED(maphist);
2253 1.10 mrg
2254 1.10 mrg UVMHIST_LOG(maphist,"(map=0x%x, start=0x%x, end=0x%x)",
2255 1.10 mrg map, start, end, 0);
2256 1.10 mrg VM_MAP_RANGE_CHECK(map, start, end);
2257 1.10 mrg
2258 1.222 yamt uvm_map_check(map, "unmap_remove entry");
2259 1.144 yamt
2260 1.10 mrg /*
2261 1.10 mrg * find first entry
2262 1.10 mrg */
2263 1.99 chs
2264 1.234 thorpej if (uvm_map_lookup_entry(map, start, &first_entry) == true) {
2265 1.29 chuck /* clip and go... */
2266 1.10 mrg entry = first_entry;
2267 1.174 yamt UVM_MAP_CLIP_START(map, entry, start, umr);
2268 1.10 mrg /* critical! prevents stale hint */
2269 1.82 thorpej SAVE_HINT(map, entry, entry->prev);
2270 1.10 mrg } else {
2271 1.10 mrg entry = first_entry->next;
2272 1.10 mrg }
2273 1.10 mrg
2274 1.10 mrg /*
2275 1.10 mrg * Save the free space hint
2276 1.10 mrg */
2277 1.10 mrg
2278 1.220 yamt if (map->first_free != &map->header && map->first_free->start >= start)
2279 1.10 mrg map->first_free = entry->prev;
2280 1.10 mrg
2281 1.10 mrg /*
2282 1.10 mrg * note: we now re-use first_entry for a different task. we remove
2283 1.10 mrg * a number of map entries from the map and save them in a linked
2284 1.10 mrg * list headed by "first_entry". once we remove them from the map
2285 1.10 mrg * the caller should unlock the map and drop the references to the
2286 1.10 mrg * backing objects [c.f. uvm_unmap_detach]. the object is to
2287 1.100 wiz * separate unmapping from reference dropping. why?
2288 1.10 mrg * [1] the map has to be locked for unmapping
2289 1.10 mrg * [2] the map need not be locked for reference dropping
2290 1.10 mrg * [3] dropping references may trigger pager I/O, and if we hit
2291 1.10 mrg * a pager that does synchronous I/O we may have to wait for it.
2292 1.10 mrg * [4] we would like all waiting for I/O to occur with maps unlocked
2293 1.98 chs * so that we don't block other threads.
2294 1.10 mrg */
2295 1.99 chs
2296 1.10 mrg first_entry = NULL;
2297 1.106 chs *entry_list = NULL;
2298 1.10 mrg
2299 1.10 mrg /*
2300 1.98 chs * break up the area into map entry sized regions and unmap. note
2301 1.10 mrg * that all mappings have to be removed before we can even consider
2302 1.10 mrg * dropping references to amaps or VM objects (otherwise we could end
2303 1.10 mrg * up with a mapping to a page on the free list which would be very bad)
2304 1.10 mrg */
2305 1.10 mrg
2306 1.10 mrg while ((entry != &map->header) && (entry->start < end)) {
2307 1.174 yamt KASSERT((entry->flags & UVM_MAP_FIRST) == 0);
2308 1.174 yamt
2309 1.174 yamt UVM_MAP_CLIP_END(map, entry, end, umr);
2310 1.10 mrg next = entry->next;
2311 1.10 mrg len = entry->end - entry->start;
2312 1.81 thorpej
2313 1.10 mrg /*
2314 1.10 mrg * unwire before removing addresses from the pmap; otherwise
2315 1.10 mrg * unwiring will put the entries back into the pmap (XXX).
2316 1.10 mrg */
2317 1.1 mrg
2318 1.106 chs if (VM_MAPENT_ISWIRED(entry)) {
2319 1.10 mrg uvm_map_entry_unwire(map, entry);
2320 1.106 chs }
2321 1.187 yamt if (flags & UVM_FLAG_VAONLY) {
2322 1.187 yamt
2323 1.187 yamt /* nothing */
2324 1.187 yamt
2325 1.187 yamt } else if ((map->flags & VM_MAP_PAGEABLE) == 0) {
2326 1.10 mrg
2327 1.106 chs /*
2328 1.106 chs * if the map is non-pageable, any pages mapped there
2329 1.106 chs * must be wired and entered with pmap_kenter_pa(),
2330 1.106 chs * and we should free any such pages immediately.
2331 1.106 chs * this is mostly used for kmem_map and mb_map.
2332 1.106 chs */
2333 1.99 chs
2334 1.174 yamt if ((entry->flags & UVM_MAP_KMAPENT) == 0) {
2335 1.264 ad uvm_km_pgremove_intrsafe(map, entry->start,
2336 1.174 yamt entry->end);
2337 1.174 yamt pmap_kremove(entry->start, len);
2338 1.174 yamt }
2339 1.106 chs } else if (UVM_ET_ISOBJ(entry) &&
2340 1.106 chs UVM_OBJ_IS_KERN_OBJECT(entry->object.uvm_obj)) {
2341 1.85 chs KASSERT(vm_map_pmap(map) == pmap_kernel());
2342 1.1 mrg
2343 1.10 mrg /*
2344 1.10 mrg * note: kernel object mappings are currently used in
2345 1.10 mrg * two ways:
2346 1.10 mrg * [1] "normal" mappings of pages in the kernel object
2347 1.10 mrg * [2] uvm_km_valloc'd allocations in which we
2348 1.10 mrg * pmap_enter in some non-kernel-object page
2349 1.10 mrg * (e.g. vmapbuf).
2350 1.10 mrg *
2351 1.10 mrg * for case [1], we need to remove the mapping from
2352 1.10 mrg * the pmap and then remove the page from the kernel
2353 1.10 mrg * object (because, once pages in a kernel object are
2354 1.10 mrg * unmapped they are no longer needed, unlike, say,
2355 1.10 mrg * a vnode where you might want the data to persist
2356 1.10 mrg * until flushed out of a queue).
2357 1.10 mrg *
2358 1.10 mrg * for case [2], we need to remove the mapping from
2359 1.10 mrg * the pmap. there shouldn't be any pages at the
2360 1.10 mrg * specified offset in the kernel object [but it
2361 1.10 mrg * doesn't hurt to call uvm_km_pgremove just to be
2362 1.10 mrg * safe?]
2363 1.10 mrg *
2364 1.98 chs * uvm_km_pgremove currently does the following:
2365 1.98 chs * for pages in the kernel object in range:
2366 1.43 thorpej * - drops the swap slot
2367 1.10 mrg * - uvm_pagefree the page
2368 1.10 mrg */
2369 1.10 mrg
2370 1.10 mrg /*
2371 1.43 thorpej * remove mappings from pmap and drop the pages
2372 1.43 thorpej * from the object. offsets are always relative
2373 1.43 thorpej * to vm_map_min(kernel_map).
2374 1.10 mrg */
2375 1.99 chs
2376 1.106 chs pmap_remove(pmap_kernel(), entry->start,
2377 1.106 chs entry->start + len);
2378 1.187 yamt uvm_km_pgremove(entry->start, entry->end);
2379 1.10 mrg
2380 1.10 mrg /*
2381 1.10 mrg * null out kernel_object reference, we've just
2382 1.10 mrg * dropped it
2383 1.10 mrg */
2384 1.99 chs
2385 1.10 mrg entry->etype &= ~UVM_ET_OBJ;
2386 1.106 chs entry->object.uvm_obj = NULL;
2387 1.106 chs } else if (UVM_ET_ISOBJ(entry) || entry->aref.ar_amap) {
2388 1.99 chs
2389 1.29 chuck /*
2390 1.139 enami * remove mappings the standard way.
2391 1.139 enami */
2392 1.99 chs
2393 1.29 chuck pmap_remove(map->pmap, entry->start, entry->end);
2394 1.10 mrg }
2395 1.10 mrg
2396 1.177 yamt #if defined(DEBUG)
2397 1.177 yamt if ((entry->flags & UVM_MAP_KMAPENT) == 0) {
2398 1.177 yamt
2399 1.177 yamt /*
2400 1.177 yamt * check if there's remaining mapping,
2401 1.177 yamt * which is a bug in caller.
2402 1.177 yamt */
2403 1.177 yamt
2404 1.177 yamt vaddr_t va;
2405 1.177 yamt for (va = entry->start; va < entry->end;
2406 1.177 yamt va += PAGE_SIZE) {
2407 1.177 yamt if (pmap_extract(vm_map_pmap(map), va, NULL)) {
2408 1.177 yamt panic("uvm_unmap_remove: has mapping");
2409 1.177 yamt }
2410 1.177 yamt }
2411 1.187 yamt
2412 1.187 yamt if (VM_MAP_IS_KERNEL(map)) {
2413 1.264 ad uvm_km_check_empty(map, entry->start,
2414 1.264 ad entry->end);
2415 1.187 yamt }
2416 1.177 yamt }
2417 1.177 yamt #endif /* defined(DEBUG) */
2418 1.177 yamt
2419 1.10 mrg /*
2420 1.98 chs * remove entry from map and put it on our list of entries
2421 1.106 chs * that we've nuked. then go to next entry.
2422 1.10 mrg */
2423 1.99 chs
2424 1.10 mrg UVMHIST_LOG(maphist, " removed map entry 0x%x", entry, 0, 0,0);
2425 1.82 thorpej
2426 1.82 thorpej /* critical! prevents stale hint */
2427 1.82 thorpej SAVE_HINT(map, entry, entry->prev);
2428 1.82 thorpej
2429 1.10 mrg uvm_map_entry_unlink(map, entry);
2430 1.146 yamt KASSERT(map->size >= len);
2431 1.10 mrg map->size -= len;
2432 1.131 atatat entry->prev = NULL;
2433 1.10 mrg entry->next = first_entry;
2434 1.10 mrg first_entry = entry;
2435 1.106 chs entry = next;
2436 1.10 mrg }
2437 1.120 chs if ((map->flags & VM_MAP_DYING) == 0) {
2438 1.120 chs pmap_update(vm_map_pmap(map));
2439 1.120 chs }
2440 1.10 mrg
2441 1.222 yamt uvm_map_check(map, "unmap_remove leave");
2442 1.144 yamt
2443 1.10 mrg /*
2444 1.10 mrg * now we've cleaned up the map and are ready for the caller to drop
2445 1.98 chs * references to the mapped objects.
2446 1.10 mrg */
2447 1.10 mrg
2448 1.10 mrg *entry_list = first_entry;
2449 1.10 mrg UVMHIST_LOG(maphist,"<- done!", 0, 0, 0, 0);
2450 1.180 yamt
2451 1.180 yamt if (map->flags & VM_MAP_WANTVA) {
2452 1.238 ad mutex_enter(&map->misc_lock);
2453 1.180 yamt map->flags &= ~VM_MAP_WANTVA;
2454 1.238 ad cv_broadcast(&map->cv);
2455 1.238 ad mutex_exit(&map->misc_lock);
2456 1.180 yamt }
2457 1.1 mrg }
2458 1.1 mrg
2459 1.1 mrg /*
2460 1.1 mrg * uvm_unmap_detach: drop references in a chain of map entries
2461 1.1 mrg *
2462 1.1 mrg * => we will free the map entries as we traverse the list.
2463 1.1 mrg */
2464 1.1 mrg
2465 1.10 mrg void
2466 1.138 enami uvm_unmap_detach(struct vm_map_entry *first_entry, int flags)
2467 1.1 mrg {
2468 1.99 chs struct vm_map_entry *next_entry;
2469 1.10 mrg UVMHIST_FUNC("uvm_unmap_detach"); UVMHIST_CALLED(maphist);
2470 1.1 mrg
2471 1.10 mrg while (first_entry) {
2472 1.85 chs KASSERT(!VM_MAPENT_ISWIRED(first_entry));
2473 1.10 mrg UVMHIST_LOG(maphist,
2474 1.98 chs " detach 0x%x: amap=0x%x, obj=0x%x, submap?=%d",
2475 1.98 chs first_entry, first_entry->aref.ar_amap,
2476 1.29 chuck first_entry->object.uvm_obj,
2477 1.29 chuck UVM_ET_ISSUBMAP(first_entry));
2478 1.1 mrg
2479 1.10 mrg /*
2480 1.10 mrg * drop reference to amap, if we've got one
2481 1.10 mrg */
2482 1.10 mrg
2483 1.10 mrg if (first_entry->aref.ar_amap)
2484 1.85 chs uvm_map_unreference_amap(first_entry, flags);
2485 1.10 mrg
2486 1.10 mrg /*
2487 1.10 mrg * drop reference to our backing object, if we've got one
2488 1.10 mrg */
2489 1.85 chs
2490 1.120 chs KASSERT(!UVM_ET_ISSUBMAP(first_entry));
2491 1.120 chs if (UVM_ET_ISOBJ(first_entry) &&
2492 1.120 chs first_entry->object.uvm_obj->pgops->pgo_detach) {
2493 1.120 chs (*first_entry->object.uvm_obj->pgops->pgo_detach)
2494 1.120 chs (first_entry->object.uvm_obj);
2495 1.10 mrg }
2496 1.10 mrg next_entry = first_entry->next;
2497 1.10 mrg uvm_mapent_free(first_entry);
2498 1.10 mrg first_entry = next_entry;
2499 1.10 mrg }
2500 1.10 mrg UVMHIST_LOG(maphist, "<- done", 0,0,0,0);
2501 1.1 mrg }
2502 1.1 mrg
2503 1.1 mrg /*
2504 1.1 mrg * E X T R A C T I O N F U N C T I O N S
2505 1.1 mrg */
2506 1.1 mrg
2507 1.98 chs /*
2508 1.1 mrg * uvm_map_reserve: reserve space in a vm_map for future use.
2509 1.1 mrg *
2510 1.98 chs * => we reserve space in a map by putting a dummy map entry in the
2511 1.1 mrg * map (dummy means obj=NULL, amap=NULL, prot=VM_PROT_NONE)
2512 1.1 mrg * => map should be unlocked (we will write lock it)
2513 1.1 mrg * => we return true if we were able to reserve space
2514 1.1 mrg * => XXXCDC: should be inline?
2515 1.1 mrg */
2516 1.1 mrg
2517 1.10 mrg int
2518 1.138 enami uvm_map_reserve(struct vm_map *map, vsize_t size,
2519 1.138 enami vaddr_t offset /* hint for pmap_prefer */,
2520 1.243 yamt vsize_t align /* alignment */,
2521 1.210 yamt vaddr_t *raddr /* IN:hint, OUT: reserved VA */,
2522 1.210 yamt uvm_flag_t flags /* UVM_FLAG_FIXED or 0 */)
2523 1.1 mrg {
2524 1.98 chs UVMHIST_FUNC("uvm_map_reserve"); UVMHIST_CALLED(maphist);
2525 1.85 chs
2526 1.10 mrg UVMHIST_LOG(maphist, "(map=0x%x, size=0x%x, offset=0x%x,addr=0x%x)",
2527 1.139 enami map,size,offset,raddr);
2528 1.85 chs
2529 1.10 mrg size = round_page(size);
2530 1.85 chs
2531 1.10 mrg /*
2532 1.10 mrg * reserve some virtual space.
2533 1.10 mrg */
2534 1.85 chs
2535 1.243 yamt if (uvm_map(map, raddr, size, NULL, offset, align,
2536 1.10 mrg UVM_MAPFLAG(UVM_PROT_NONE, UVM_PROT_NONE, UVM_INH_NONE,
2537 1.210 yamt UVM_ADV_RANDOM, UVM_FLAG_NOMERGE|flags)) != 0) {
2538 1.10 mrg UVMHIST_LOG(maphist, "<- done (no VM)", 0,0,0,0);
2539 1.234 thorpej return (false);
2540 1.98 chs }
2541 1.85 chs
2542 1.10 mrg UVMHIST_LOG(maphist, "<- done (*raddr=0x%x)", *raddr,0,0,0);
2543 1.234 thorpej return (true);
2544 1.1 mrg }
2545 1.1 mrg
2546 1.1 mrg /*
2547 1.98 chs * uvm_map_replace: replace a reserved (blank) area of memory with
2548 1.1 mrg * real mappings.
2549 1.1 mrg *
2550 1.98 chs * => caller must WRITE-LOCK the map
2551 1.234 thorpej * => we return true if replacement was a success
2552 1.1 mrg * => we expect the newents chain to have nnewents entrys on it and
2553 1.1 mrg * we expect newents->prev to point to the last entry on the list
2554 1.1 mrg * => note newents is allowed to be NULL
2555 1.1 mrg */
2556 1.1 mrg
2557 1.10 mrg int
2558 1.138 enami uvm_map_replace(struct vm_map *map, vaddr_t start, vaddr_t end,
2559 1.253 yamt struct vm_map_entry *newents, int nnewents, struct vm_map_entry **oldentryp)
2560 1.10 mrg {
2561 1.99 chs struct vm_map_entry *oldent, *last;
2562 1.1 mrg
2563 1.222 yamt uvm_map_check(map, "map_replace entry");
2564 1.144 yamt
2565 1.10 mrg /*
2566 1.10 mrg * first find the blank map entry at the specified address
2567 1.10 mrg */
2568 1.85 chs
2569 1.10 mrg if (!uvm_map_lookup_entry(map, start, &oldent)) {
2570 1.234 thorpej return (false);
2571 1.10 mrg }
2572 1.85 chs
2573 1.10 mrg /*
2574 1.10 mrg * check to make sure we have a proper blank entry
2575 1.10 mrg */
2576 1.1 mrg
2577 1.210 yamt if (end < oldent->end && !VM_MAP_USE_KMAPENT(map)) {
2578 1.210 yamt UVM_MAP_CLIP_END(map, oldent, end, NULL);
2579 1.210 yamt }
2580 1.98 chs if (oldent->start != start || oldent->end != end ||
2581 1.10 mrg oldent->object.uvm_obj != NULL || oldent->aref.ar_amap != NULL) {
2582 1.234 thorpej return (false);
2583 1.10 mrg }
2584 1.1 mrg
2585 1.1 mrg #ifdef DIAGNOSTIC
2586 1.99 chs
2587 1.10 mrg /*
2588 1.10 mrg * sanity check the newents chain
2589 1.10 mrg */
2590 1.99 chs
2591 1.10 mrg {
2592 1.99 chs struct vm_map_entry *tmpent = newents;
2593 1.10 mrg int nent = 0;
2594 1.24 eeh vaddr_t cur = start;
2595 1.10 mrg
2596 1.10 mrg while (tmpent) {
2597 1.10 mrg nent++;
2598 1.10 mrg if (tmpent->start < cur)
2599 1.10 mrg panic("uvm_map_replace1");
2600 1.10 mrg if (tmpent->start > tmpent->end || tmpent->end > end) {
2601 1.10 mrg printf("tmpent->start=0x%lx, tmpent->end=0x%lx, end=0x%lx\n",
2602 1.10 mrg tmpent->start, tmpent->end, end);
2603 1.10 mrg panic("uvm_map_replace2");
2604 1.10 mrg }
2605 1.10 mrg cur = tmpent->end;
2606 1.10 mrg if (tmpent->next) {
2607 1.10 mrg if (tmpent->next->prev != tmpent)
2608 1.10 mrg panic("uvm_map_replace3");
2609 1.10 mrg } else {
2610 1.10 mrg if (newents->prev != tmpent)
2611 1.10 mrg panic("uvm_map_replace4");
2612 1.10 mrg }
2613 1.10 mrg tmpent = tmpent->next;
2614 1.10 mrg }
2615 1.10 mrg if (nent != nnewents)
2616 1.10 mrg panic("uvm_map_replace5");
2617 1.10 mrg }
2618 1.10 mrg #endif
2619 1.10 mrg
2620 1.10 mrg /*
2621 1.10 mrg * map entry is a valid blank! replace it. (this does all the
2622 1.10 mrg * work of map entry link/unlink...).
2623 1.10 mrg */
2624 1.10 mrg
2625 1.10 mrg if (newents) {
2626 1.99 chs last = newents->prev;
2627 1.10 mrg
2628 1.10 mrg /* critical: flush stale hints out of map */
2629 1.82 thorpej SAVE_HINT(map, map->hint, newents);
2630 1.10 mrg if (map->first_free == oldent)
2631 1.10 mrg map->first_free = last;
2632 1.10 mrg
2633 1.10 mrg last->next = oldent->next;
2634 1.10 mrg last->next->prev = last;
2635 1.144 yamt
2636 1.144 yamt /* Fix RB tree */
2637 1.144 yamt uvm_rb_remove(map, oldent);
2638 1.144 yamt
2639 1.10 mrg newents->prev = oldent->prev;
2640 1.10 mrg newents->prev->next = newents;
2641 1.10 mrg map->nentries = map->nentries + (nnewents - 1);
2642 1.10 mrg
2643 1.144 yamt /* Fixup the RB tree */
2644 1.144 yamt {
2645 1.144 yamt int i;
2646 1.144 yamt struct vm_map_entry *tmp;
2647 1.144 yamt
2648 1.144 yamt tmp = newents;
2649 1.144 yamt for (i = 0; i < nnewents && tmp; i++) {
2650 1.144 yamt uvm_rb_insert(map, tmp);
2651 1.144 yamt tmp = tmp->next;
2652 1.144 yamt }
2653 1.144 yamt }
2654 1.10 mrg } else {
2655 1.10 mrg /* NULL list of new entries: just remove the old one */
2656 1.221 yamt clear_hints(map, oldent);
2657 1.10 mrg uvm_map_entry_unlink(map, oldent);
2658 1.10 mrg }
2659 1.10 mrg
2660 1.222 yamt uvm_map_check(map, "map_replace leave");
2661 1.10 mrg
2662 1.10 mrg /*
2663 1.209 yamt * now we can free the old blank entry and return.
2664 1.10 mrg */
2665 1.1 mrg
2666 1.253 yamt *oldentryp = oldent;
2667 1.234 thorpej return (true);
2668 1.1 mrg }
2669 1.1 mrg
2670 1.1 mrg /*
2671 1.1 mrg * uvm_map_extract: extract a mapping from a map and put it somewhere
2672 1.1 mrg * (maybe removing the old mapping)
2673 1.1 mrg *
2674 1.1 mrg * => maps should be unlocked (we will write lock them)
2675 1.1 mrg * => returns 0 on success, error code otherwise
2676 1.1 mrg * => start must be page aligned
2677 1.1 mrg * => len must be page sized
2678 1.1 mrg * => flags:
2679 1.1 mrg * UVM_EXTRACT_REMOVE: remove mappings from srcmap
2680 1.1 mrg * UVM_EXTRACT_CONTIG: abort if unmapped area (advisory only)
2681 1.1 mrg * UVM_EXTRACT_QREF: for a temporary extraction do quick obj refs
2682 1.1 mrg * UVM_EXTRACT_FIXPROT: set prot to maxprot as we go
2683 1.1 mrg * >>>NOTE: if you set REMOVE, you are not allowed to use CONTIG or QREF!<<<
2684 1.1 mrg * >>>NOTE: QREF's must be unmapped via the QREF path, thus should only
2685 1.1 mrg * be used from within the kernel in a kernel level map <<<
2686 1.1 mrg */
2687 1.1 mrg
2688 1.10 mrg int
2689 1.138 enami uvm_map_extract(struct vm_map *srcmap, vaddr_t start, vsize_t len,
2690 1.138 enami struct vm_map *dstmap, vaddr_t *dstaddrp, int flags)
2691 1.10 mrg {
2692 1.163 mycroft vaddr_t dstaddr, end, newend, oldoffset, fudge, orig_fudge;
2693 1.99 chs struct vm_map_entry *chain, *endchain, *entry, *orig_entry, *newentry,
2694 1.99 chs *deadentry, *oldentry;
2695 1.253 yamt struct vm_map_entry *resentry = NULL; /* a dummy reservation entry */
2696 1.24 eeh vsize_t elen;
2697 1.10 mrg int nchain, error, copy_ok;
2698 1.10 mrg UVMHIST_FUNC("uvm_map_extract"); UVMHIST_CALLED(maphist);
2699 1.85 chs
2700 1.10 mrg UVMHIST_LOG(maphist,"(srcmap=0x%x,start=0x%x, len=0x%x", srcmap, start,
2701 1.10 mrg len,0);
2702 1.10 mrg UVMHIST_LOG(maphist," ...,dstmap=0x%x, flags=0x%x)", dstmap,flags,0,0);
2703 1.10 mrg
2704 1.222 yamt uvm_map_check(srcmap, "map_extract src enter");
2705 1.222 yamt uvm_map_check(dstmap, "map_extract dst enter");
2706 1.144 yamt
2707 1.10 mrg /*
2708 1.10 mrg * step 0: sanity check: start must be on a page boundary, length
2709 1.10 mrg * must be page sized. can't ask for CONTIG/QREF if you asked for
2710 1.10 mrg * REMOVE.
2711 1.10 mrg */
2712 1.10 mrg
2713 1.85 chs KASSERT((start & PAGE_MASK) == 0 && (len & PAGE_MASK) == 0);
2714 1.85 chs KASSERT((flags & UVM_EXTRACT_REMOVE) == 0 ||
2715 1.85 chs (flags & (UVM_EXTRACT_CONTIG|UVM_EXTRACT_QREF)) == 0);
2716 1.10 mrg
2717 1.10 mrg /*
2718 1.10 mrg * step 1: reserve space in the target map for the extracted area
2719 1.10 mrg */
2720 1.10 mrg
2721 1.210 yamt if ((flags & UVM_EXTRACT_RESERVED) == 0) {
2722 1.210 yamt dstaddr = vm_map_min(dstmap);
2723 1.210 yamt if (!uvm_map_reserve(dstmap, len, start, 0, &dstaddr, 0))
2724 1.210 yamt return (ENOMEM);
2725 1.210 yamt *dstaddrp = dstaddr; /* pass address back to caller */
2726 1.210 yamt UVMHIST_LOG(maphist, " dstaddr=0x%x", dstaddr,0,0,0);
2727 1.210 yamt } else {
2728 1.210 yamt dstaddr = *dstaddrp;
2729 1.210 yamt }
2730 1.10 mrg
2731 1.10 mrg /*
2732 1.98 chs * step 2: setup for the extraction process loop by init'ing the
2733 1.10 mrg * map entry chain, locking src map, and looking up the first useful
2734 1.10 mrg * entry in the map.
2735 1.10 mrg */
2736 1.1 mrg
2737 1.10 mrg end = start + len;
2738 1.10 mrg newend = dstaddr + len;
2739 1.10 mrg chain = endchain = NULL;
2740 1.10 mrg nchain = 0;
2741 1.10 mrg vm_map_lock(srcmap);
2742 1.10 mrg
2743 1.10 mrg if (uvm_map_lookup_entry(srcmap, start, &entry)) {
2744 1.10 mrg
2745 1.10 mrg /* "start" is within an entry */
2746 1.10 mrg if (flags & UVM_EXTRACT_QREF) {
2747 1.85 chs
2748 1.10 mrg /*
2749 1.10 mrg * for quick references we don't clip the entry, so
2750 1.10 mrg * the entry may map space "before" the starting
2751 1.10 mrg * virtual address... this is the "fudge" factor
2752 1.10 mrg * (which can be non-zero only the first time
2753 1.10 mrg * through the "while" loop in step 3).
2754 1.10 mrg */
2755 1.85 chs
2756 1.10 mrg fudge = start - entry->start;
2757 1.10 mrg } else {
2758 1.85 chs
2759 1.10 mrg /*
2760 1.10 mrg * normal reference: we clip the map to fit (thus
2761 1.10 mrg * fudge is zero)
2762 1.10 mrg */
2763 1.85 chs
2764 1.174 yamt UVM_MAP_CLIP_START(srcmap, entry, start, NULL);
2765 1.82 thorpej SAVE_HINT(srcmap, srcmap->hint, entry->prev);
2766 1.10 mrg fudge = 0;
2767 1.10 mrg }
2768 1.85 chs } else {
2769 1.1 mrg
2770 1.10 mrg /* "start" is not within an entry ... skip to next entry */
2771 1.10 mrg if (flags & UVM_EXTRACT_CONTIG) {
2772 1.10 mrg error = EINVAL;
2773 1.10 mrg goto bad; /* definite hole here ... */
2774 1.10 mrg }
2775 1.1 mrg
2776 1.10 mrg entry = entry->next;
2777 1.10 mrg fudge = 0;
2778 1.10 mrg }
2779 1.85 chs
2780 1.10 mrg /* save values from srcmap for step 6 */
2781 1.10 mrg orig_entry = entry;
2782 1.10 mrg orig_fudge = fudge;
2783 1.1 mrg
2784 1.10 mrg /*
2785 1.10 mrg * step 3: now start looping through the map entries, extracting
2786 1.10 mrg * as we go.
2787 1.10 mrg */
2788 1.1 mrg
2789 1.10 mrg while (entry->start < end && entry != &srcmap->header) {
2790 1.85 chs
2791 1.10 mrg /* if we are not doing a quick reference, clip it */
2792 1.10 mrg if ((flags & UVM_EXTRACT_QREF) == 0)
2793 1.174 yamt UVM_MAP_CLIP_END(srcmap, entry, end, NULL);
2794 1.10 mrg
2795 1.10 mrg /* clear needs_copy (allow chunking) */
2796 1.10 mrg if (UVM_ET_ISNEEDSCOPY(entry)) {
2797 1.212 yamt amap_copy(srcmap, entry,
2798 1.212 yamt AMAP_COPY_NOWAIT|AMAP_COPY_NOMERGE, start, end);
2799 1.10 mrg if (UVM_ET_ISNEEDSCOPY(entry)) { /* failed? */
2800 1.10 mrg error = ENOMEM;
2801 1.10 mrg goto bad;
2802 1.10 mrg }
2803 1.85 chs
2804 1.10 mrg /* amap_copy could clip (during chunk)! update fudge */
2805 1.10 mrg if (fudge) {
2806 1.163 mycroft fudge = start - entry->start;
2807 1.10 mrg orig_fudge = fudge;
2808 1.10 mrg }
2809 1.10 mrg }
2810 1.1 mrg
2811 1.10 mrg /* calculate the offset of this from "start" */
2812 1.10 mrg oldoffset = (entry->start + fudge) - start;
2813 1.1 mrg
2814 1.10 mrg /* allocate a new map entry */
2815 1.126 bouyer newentry = uvm_mapent_alloc(dstmap, 0);
2816 1.10 mrg if (newentry == NULL) {
2817 1.10 mrg error = ENOMEM;
2818 1.10 mrg goto bad;
2819 1.10 mrg }
2820 1.10 mrg
2821 1.10 mrg /* set up new map entry */
2822 1.10 mrg newentry->next = NULL;
2823 1.10 mrg newentry->prev = endchain;
2824 1.10 mrg newentry->start = dstaddr + oldoffset;
2825 1.10 mrg newentry->end =
2826 1.10 mrg newentry->start + (entry->end - (entry->start + fudge));
2827 1.37 chs if (newentry->end > newend || newentry->end < newentry->start)
2828 1.10 mrg newentry->end = newend;
2829 1.10 mrg newentry->object.uvm_obj = entry->object.uvm_obj;
2830 1.10 mrg if (newentry->object.uvm_obj) {
2831 1.10 mrg if (newentry->object.uvm_obj->pgops->pgo_reference)
2832 1.10 mrg newentry->object.uvm_obj->pgops->
2833 1.10 mrg pgo_reference(newentry->object.uvm_obj);
2834 1.10 mrg newentry->offset = entry->offset + fudge;
2835 1.10 mrg } else {
2836 1.10 mrg newentry->offset = 0;
2837 1.10 mrg }
2838 1.10 mrg newentry->etype = entry->etype;
2839 1.98 chs newentry->protection = (flags & UVM_EXTRACT_FIXPROT) ?
2840 1.98 chs entry->max_protection : entry->protection;
2841 1.10 mrg newentry->max_protection = entry->max_protection;
2842 1.10 mrg newentry->inheritance = entry->inheritance;
2843 1.10 mrg newentry->wired_count = 0;
2844 1.10 mrg newentry->aref.ar_amap = entry->aref.ar_amap;
2845 1.10 mrg if (newentry->aref.ar_amap) {
2846 1.34 chuck newentry->aref.ar_pageoff =
2847 1.34 chuck entry->aref.ar_pageoff + (fudge >> PAGE_SHIFT);
2848 1.85 chs uvm_map_reference_amap(newentry, AMAP_SHARED |
2849 1.10 mrg ((flags & UVM_EXTRACT_QREF) ? AMAP_REFALL : 0));
2850 1.10 mrg } else {
2851 1.34 chuck newentry->aref.ar_pageoff = 0;
2852 1.10 mrg }
2853 1.10 mrg newentry->advice = entry->advice;
2854 1.245 yamt if ((flags & UVM_EXTRACT_QREF) != 0) {
2855 1.245 yamt newentry->flags |= UVM_MAP_NOMERGE;
2856 1.245 yamt }
2857 1.10 mrg
2858 1.10 mrg /* now link it on the chain */
2859 1.10 mrg nchain++;
2860 1.10 mrg if (endchain == NULL) {
2861 1.10 mrg chain = endchain = newentry;
2862 1.10 mrg } else {
2863 1.10 mrg endchain->next = newentry;
2864 1.10 mrg endchain = newentry;
2865 1.10 mrg }
2866 1.10 mrg
2867 1.10 mrg /* end of 'while' loop! */
2868 1.98 chs if ((flags & UVM_EXTRACT_CONTIG) && entry->end < end &&
2869 1.10 mrg (entry->next == &srcmap->header ||
2870 1.10 mrg entry->next->start != entry->end)) {
2871 1.10 mrg error = EINVAL;
2872 1.10 mrg goto bad;
2873 1.10 mrg }
2874 1.10 mrg entry = entry->next;
2875 1.10 mrg fudge = 0;
2876 1.10 mrg }
2877 1.10 mrg
2878 1.10 mrg /*
2879 1.10 mrg * step 4: close off chain (in format expected by uvm_map_replace)
2880 1.10 mrg */
2881 1.10 mrg
2882 1.10 mrg if (chain)
2883 1.10 mrg chain->prev = endchain;
2884 1.10 mrg
2885 1.10 mrg /*
2886 1.10 mrg * step 5: attempt to lock the dest map so we can pmap_copy.
2887 1.98 chs * note usage of copy_ok:
2888 1.10 mrg * 1 => dstmap locked, pmap_copy ok, and we "replace" here (step 5)
2889 1.10 mrg * 0 => dstmap unlocked, NO pmap_copy, and we will "replace" in step 7
2890 1.10 mrg */
2891 1.85 chs
2892 1.234 thorpej if (srcmap == dstmap || vm_map_lock_try(dstmap) == true) {
2893 1.10 mrg copy_ok = 1;
2894 1.10 mrg if (!uvm_map_replace(dstmap, dstaddr, dstaddr+len, chain,
2895 1.253 yamt nchain, &resentry)) {
2896 1.10 mrg if (srcmap != dstmap)
2897 1.10 mrg vm_map_unlock(dstmap);
2898 1.10 mrg error = EIO;
2899 1.10 mrg goto bad;
2900 1.10 mrg }
2901 1.10 mrg } else {
2902 1.10 mrg copy_ok = 0;
2903 1.10 mrg /* replace defered until step 7 */
2904 1.10 mrg }
2905 1.10 mrg
2906 1.10 mrg /*
2907 1.10 mrg * step 6: traverse the srcmap a second time to do the following:
2908 1.10 mrg * - if we got a lock on the dstmap do pmap_copy
2909 1.10 mrg * - if UVM_EXTRACT_REMOVE remove the entries
2910 1.10 mrg * we make use of orig_entry and orig_fudge (saved in step 2)
2911 1.10 mrg */
2912 1.10 mrg
2913 1.10 mrg if (copy_ok || (flags & UVM_EXTRACT_REMOVE)) {
2914 1.10 mrg
2915 1.10 mrg /* purge possible stale hints from srcmap */
2916 1.10 mrg if (flags & UVM_EXTRACT_REMOVE) {
2917 1.82 thorpej SAVE_HINT(srcmap, srcmap->hint, orig_entry->prev);
2918 1.220 yamt if (srcmap->first_free != &srcmap->header &&
2919 1.220 yamt srcmap->first_free->start >= start)
2920 1.10 mrg srcmap->first_free = orig_entry->prev;
2921 1.10 mrg }
2922 1.10 mrg
2923 1.10 mrg entry = orig_entry;
2924 1.10 mrg fudge = orig_fudge;
2925 1.10 mrg deadentry = NULL; /* for UVM_EXTRACT_REMOVE */
2926 1.10 mrg
2927 1.10 mrg while (entry->start < end && entry != &srcmap->header) {
2928 1.10 mrg if (copy_ok) {
2929 1.74 thorpej oldoffset = (entry->start + fudge) - start;
2930 1.90 chs elen = MIN(end, entry->end) -
2931 1.74 thorpej (entry->start + fudge);
2932 1.74 thorpej pmap_copy(dstmap->pmap, srcmap->pmap,
2933 1.74 thorpej dstaddr + oldoffset, elen,
2934 1.74 thorpej entry->start + fudge);
2935 1.10 mrg }
2936 1.10 mrg
2937 1.74 thorpej /* we advance "entry" in the following if statement */
2938 1.10 mrg if (flags & UVM_EXTRACT_REMOVE) {
2939 1.98 chs pmap_remove(srcmap->pmap, entry->start,
2940 1.20 chuck entry->end);
2941 1.139 enami oldentry = entry; /* save entry */
2942 1.139 enami entry = entry->next; /* advance */
2943 1.20 chuck uvm_map_entry_unlink(srcmap, oldentry);
2944 1.20 chuck /* add to dead list */
2945 1.20 chuck oldentry->next = deadentry;
2946 1.20 chuck deadentry = oldentry;
2947 1.139 enami } else {
2948 1.139 enami entry = entry->next; /* advance */
2949 1.10 mrg }
2950 1.10 mrg
2951 1.10 mrg /* end of 'while' loop */
2952 1.10 mrg fudge = 0;
2953 1.10 mrg }
2954 1.105 chris pmap_update(srcmap->pmap);
2955 1.10 mrg
2956 1.10 mrg /*
2957 1.10 mrg * unlock dstmap. we will dispose of deadentry in
2958 1.10 mrg * step 7 if needed
2959 1.10 mrg */
2960 1.85 chs
2961 1.10 mrg if (copy_ok && srcmap != dstmap)
2962 1.10 mrg vm_map_unlock(dstmap);
2963 1.10 mrg
2964 1.99 chs } else {
2965 1.99 chs deadentry = NULL;
2966 1.10 mrg }
2967 1.10 mrg
2968 1.10 mrg /*
2969 1.10 mrg * step 7: we are done with the source map, unlock. if copy_ok
2970 1.10 mrg * is 0 then we have not replaced the dummy mapping in dstmap yet
2971 1.10 mrg * and we need to do so now.
2972 1.10 mrg */
2973 1.10 mrg
2974 1.10 mrg vm_map_unlock(srcmap);
2975 1.10 mrg if ((flags & UVM_EXTRACT_REMOVE) && deadentry)
2976 1.10 mrg uvm_unmap_detach(deadentry, 0); /* dispose of old entries */
2977 1.10 mrg
2978 1.10 mrg /* now do the replacement if we didn't do it in step 5 */
2979 1.10 mrg if (copy_ok == 0) {
2980 1.10 mrg vm_map_lock(dstmap);
2981 1.10 mrg error = uvm_map_replace(dstmap, dstaddr, dstaddr+len, chain,
2982 1.253 yamt nchain, &resentry);
2983 1.10 mrg vm_map_unlock(dstmap);
2984 1.10 mrg
2985 1.234 thorpej if (error == false) {
2986 1.10 mrg error = EIO;
2987 1.10 mrg goto bad2;
2988 1.10 mrg }
2989 1.10 mrg }
2990 1.144 yamt
2991 1.253 yamt if (resentry != NULL)
2992 1.253 yamt uvm_mapent_free(resentry);
2993 1.253 yamt
2994 1.222 yamt uvm_map_check(srcmap, "map_extract src leave");
2995 1.222 yamt uvm_map_check(dstmap, "map_extract dst leave");
2996 1.144 yamt
2997 1.139 enami return (0);
2998 1.10 mrg
2999 1.10 mrg /*
3000 1.10 mrg * bad: failure recovery
3001 1.10 mrg */
3002 1.10 mrg bad:
3003 1.10 mrg vm_map_unlock(srcmap);
3004 1.10 mrg bad2: /* src already unlocked */
3005 1.10 mrg if (chain)
3006 1.10 mrg uvm_unmap_detach(chain,
3007 1.10 mrg (flags & UVM_EXTRACT_QREF) ? AMAP_REFALL : 0);
3008 1.144 yamt
3009 1.253 yamt if (resentry != NULL)
3010 1.253 yamt uvm_mapent_free(resentry);
3011 1.253 yamt
3012 1.222 yamt uvm_map_check(srcmap, "map_extract src err leave");
3013 1.222 yamt uvm_map_check(dstmap, "map_extract dst err leave");
3014 1.144 yamt
3015 1.210 yamt if ((flags & UVM_EXTRACT_RESERVED) == 0) {
3016 1.210 yamt uvm_unmap(dstmap, dstaddr, dstaddr+len); /* ??? */
3017 1.210 yamt }
3018 1.139 enami return (error);
3019 1.10 mrg }
3020 1.10 mrg
3021 1.10 mrg /* end of extraction functions */
3022 1.1 mrg
3023 1.1 mrg /*
3024 1.1 mrg * uvm_map_submap: punch down part of a map into a submap
3025 1.1 mrg *
3026 1.1 mrg * => only the kernel_map is allowed to be submapped
3027 1.1 mrg * => the purpose of submapping is to break up the locking granularity
3028 1.1 mrg * of a larger map
3029 1.1 mrg * => the range specified must have been mapped previously with a uvm_map()
3030 1.1 mrg * call [with uobj==NULL] to create a blank map entry in the main map.
3031 1.1 mrg * [And it had better still be blank!]
3032 1.1 mrg * => maps which contain submaps should never be copied or forked.
3033 1.98 chs * => to remove a submap, use uvm_unmap() on the main map
3034 1.1 mrg * and then uvm_map_deallocate() the submap.
3035 1.1 mrg * => main map must be unlocked.
3036 1.1 mrg * => submap must have been init'd and have a zero reference count.
3037 1.1 mrg * [need not be locked as we don't actually reference it]
3038 1.1 mrg */
3039 1.85 chs
3040 1.10 mrg int
3041 1.138 enami uvm_map_submap(struct vm_map *map, vaddr_t start, vaddr_t end,
3042 1.138 enami struct vm_map *submap)
3043 1.10 mrg {
3044 1.99 chs struct vm_map_entry *entry;
3045 1.174 yamt struct uvm_mapent_reservation umr;
3046 1.94 chs int error;
3047 1.1 mrg
3048 1.174 yamt uvm_mapent_reserve(map, &umr, 2, 0);
3049 1.174 yamt
3050 1.10 mrg vm_map_lock(map);
3051 1.85 chs VM_MAP_RANGE_CHECK(map, start, end);
3052 1.1 mrg
3053 1.10 mrg if (uvm_map_lookup_entry(map, start, &entry)) {
3054 1.174 yamt UVM_MAP_CLIP_START(map, entry, start, &umr);
3055 1.174 yamt UVM_MAP_CLIP_END(map, entry, end, &umr); /* to be safe */
3056 1.94 chs } else {
3057 1.10 mrg entry = NULL;
3058 1.10 mrg }
3059 1.1 mrg
3060 1.98 chs if (entry != NULL &&
3061 1.10 mrg entry->start == start && entry->end == end &&
3062 1.10 mrg entry->object.uvm_obj == NULL && entry->aref.ar_amap == NULL &&
3063 1.10 mrg !UVM_ET_ISCOPYONWRITE(entry) && !UVM_ET_ISNEEDSCOPY(entry)) {
3064 1.29 chuck entry->etype |= UVM_ET_SUBMAP;
3065 1.10 mrg entry->object.sub_map = submap;
3066 1.10 mrg entry->offset = 0;
3067 1.10 mrg uvm_map_reference(submap);
3068 1.94 chs error = 0;
3069 1.10 mrg } else {
3070 1.94 chs error = EINVAL;
3071 1.10 mrg }
3072 1.10 mrg vm_map_unlock(map);
3073 1.174 yamt
3074 1.174 yamt uvm_mapent_unreserve(map, &umr);
3075 1.174 yamt
3076 1.94 chs return error;
3077 1.1 mrg }
3078 1.1 mrg
3079 1.175 yamt /*
3080 1.175 yamt * uvm_map_setup_kernel: init in-kernel map
3081 1.175 yamt *
3082 1.175 yamt * => map must not be in service yet.
3083 1.175 yamt */
3084 1.175 yamt
3085 1.175 yamt void
3086 1.175 yamt uvm_map_setup_kernel(struct vm_map_kernel *map,
3087 1.199 christos vaddr_t vmin, vaddr_t vmax, int flags)
3088 1.175 yamt {
3089 1.175 yamt
3090 1.199 christos uvm_map_setup(&map->vmk_map, vmin, vmax, flags);
3091 1.237 ad callback_head_init(&map->vmk_reclaim_callback, IPL_VM);
3092 1.175 yamt LIST_INIT(&map->vmk_kentry_free);
3093 1.175 yamt map->vmk_merged_entries = NULL;
3094 1.175 yamt }
3095 1.175 yamt
3096 1.1 mrg
3097 1.1 mrg /*
3098 1.1 mrg * uvm_map_protect: change map protection
3099 1.1 mrg *
3100 1.1 mrg * => set_max means set max_protection.
3101 1.1 mrg * => map must be unlocked.
3102 1.1 mrg */
3103 1.1 mrg
3104 1.139 enami #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \
3105 1.36 mycroft ~VM_PROT_WRITE : VM_PROT_ALL)
3106 1.1 mrg
3107 1.10 mrg int
3108 1.138 enami uvm_map_protect(struct vm_map *map, vaddr_t start, vaddr_t end,
3109 1.233 thorpej vm_prot_t new_prot, bool set_max)
3110 1.10 mrg {
3111 1.99 chs struct vm_map_entry *current, *entry;
3112 1.94 chs int error = 0;
3113 1.10 mrg UVMHIST_FUNC("uvm_map_protect"); UVMHIST_CALLED(maphist);
3114 1.10 mrg UVMHIST_LOG(maphist,"(map=0x%x,start=0x%x,end=0x%x,new_prot=0x%x)",
3115 1.85 chs map, start, end, new_prot);
3116 1.85 chs
3117 1.10 mrg vm_map_lock(map);
3118 1.10 mrg VM_MAP_RANGE_CHECK(map, start, end);
3119 1.10 mrg if (uvm_map_lookup_entry(map, start, &entry)) {
3120 1.174 yamt UVM_MAP_CLIP_START(map, entry, start, NULL);
3121 1.10 mrg } else {
3122 1.10 mrg entry = entry->next;
3123 1.10 mrg }
3124 1.10 mrg
3125 1.1 mrg /*
3126 1.10 mrg * make a first pass to check for protection violations.
3127 1.1 mrg */
3128 1.1 mrg
3129 1.10 mrg current = entry;
3130 1.10 mrg while ((current != &map->header) && (current->start < end)) {
3131 1.65 thorpej if (UVM_ET_ISSUBMAP(current)) {
3132 1.94 chs error = EINVAL;
3133 1.65 thorpej goto out;
3134 1.65 thorpej }
3135 1.10 mrg if ((new_prot & current->max_protection) != new_prot) {
3136 1.94 chs error = EACCES;
3137 1.65 thorpej goto out;
3138 1.112 thorpej }
3139 1.112 thorpej /*
3140 1.112 thorpej * Don't allow VM_PROT_EXECUTE to be set on entries that
3141 1.112 thorpej * point to vnodes that are associated with a NOEXEC file
3142 1.112 thorpej * system.
3143 1.112 thorpej */
3144 1.112 thorpej if (UVM_ET_ISOBJ(current) &&
3145 1.112 thorpej UVM_OBJ_IS_VNODE(current->object.uvm_obj)) {
3146 1.112 thorpej struct vnode *vp =
3147 1.112 thorpej (struct vnode *) current->object.uvm_obj;
3148 1.112 thorpej
3149 1.112 thorpej if ((new_prot & VM_PROT_EXECUTE) != 0 &&
3150 1.112 thorpej (vp->v_mount->mnt_flag & MNT_NOEXEC) != 0) {
3151 1.112 thorpej error = EACCES;
3152 1.112 thorpej goto out;
3153 1.112 thorpej }
3154 1.10 mrg }
3155 1.224 elad
3156 1.65 thorpej current = current->next;
3157 1.10 mrg }
3158 1.10 mrg
3159 1.10 mrg /* go back and fix up protections (no need to clip this time). */
3160 1.10 mrg
3161 1.10 mrg current = entry;
3162 1.10 mrg while ((current != &map->header) && (current->start < end)) {
3163 1.10 mrg vm_prot_t old_prot;
3164 1.85 chs
3165 1.174 yamt UVM_MAP_CLIP_END(map, current, end, NULL);
3166 1.10 mrg old_prot = current->protection;
3167 1.10 mrg if (set_max)
3168 1.10 mrg current->protection =
3169 1.10 mrg (current->max_protection = new_prot) & old_prot;
3170 1.10 mrg else
3171 1.10 mrg current->protection = new_prot;
3172 1.10 mrg
3173 1.10 mrg /*
3174 1.98 chs * update physical map if necessary. worry about copy-on-write
3175 1.10 mrg * here -- CHECK THIS XXX
3176 1.10 mrg */
3177 1.10 mrg
3178 1.10 mrg if (current->protection != old_prot) {
3179 1.29 chuck /* update pmap! */
3180 1.29 chuck pmap_protect(map->pmap, current->start, current->end,
3181 1.29 chuck current->protection & MASK(entry));
3182 1.109 thorpej
3183 1.109 thorpej /*
3184 1.109 thorpej * If this entry points at a vnode, and the
3185 1.109 thorpej * protection includes VM_PROT_EXECUTE, mark
3186 1.111 thorpej * the vnode as VEXECMAP.
3187 1.109 thorpej */
3188 1.109 thorpej if (UVM_ET_ISOBJ(current)) {
3189 1.109 thorpej struct uvm_object *uobj =
3190 1.109 thorpej current->object.uvm_obj;
3191 1.109 thorpej
3192 1.109 thorpej if (UVM_OBJ_IS_VNODE(uobj) &&
3193 1.241 ad (current->protection & VM_PROT_EXECUTE)) {
3194 1.110 thorpej vn_markexec((struct vnode *) uobj);
3195 1.241 ad }
3196 1.109 thorpej }
3197 1.65 thorpej }
3198 1.10 mrg
3199 1.65 thorpej /*
3200 1.65 thorpej * If the map is configured to lock any future mappings,
3201 1.65 thorpej * wire this entry now if the old protection was VM_PROT_NONE
3202 1.65 thorpej * and the new protection is not VM_PROT_NONE.
3203 1.65 thorpej */
3204 1.65 thorpej
3205 1.65 thorpej if ((map->flags & VM_MAP_WIREFUTURE) != 0 &&
3206 1.65 thorpej VM_MAPENT_ISWIRED(entry) == 0 &&
3207 1.65 thorpej old_prot == VM_PROT_NONE &&
3208 1.65 thorpej new_prot != VM_PROT_NONE) {
3209 1.65 thorpej if (uvm_map_pageable(map, entry->start,
3210 1.234 thorpej entry->end, false,
3211 1.94 chs UVM_LK_ENTER|UVM_LK_EXIT) != 0) {
3212 1.99 chs
3213 1.65 thorpej /*
3214 1.65 thorpej * If locking the entry fails, remember the
3215 1.65 thorpej * error if it's the first one. Note we
3216 1.65 thorpej * still continue setting the protection in
3217 1.94 chs * the map, but will return the error
3218 1.94 chs * condition regardless.
3219 1.65 thorpej *
3220 1.65 thorpej * XXX Ignore what the actual error is,
3221 1.65 thorpej * XXX just call it a resource shortage
3222 1.65 thorpej * XXX so that it doesn't get confused
3223 1.65 thorpej * XXX what uvm_map_protect() itself would
3224 1.65 thorpej * XXX normally return.
3225 1.65 thorpej */
3226 1.99 chs
3227 1.94 chs error = ENOMEM;
3228 1.65 thorpej }
3229 1.10 mrg }
3230 1.10 mrg current = current->next;
3231 1.10 mrg }
3232 1.105 chris pmap_update(map->pmap);
3233 1.85 chs
3234 1.65 thorpej out:
3235 1.10 mrg vm_map_unlock(map);
3236 1.174 yamt
3237 1.94 chs UVMHIST_LOG(maphist, "<- done, error=%d",error,0,0,0);
3238 1.94 chs return error;
3239 1.1 mrg }
3240 1.1 mrg
3241 1.1 mrg #undef MASK
3242 1.1 mrg
3243 1.98 chs /*
3244 1.1 mrg * uvm_map_inherit: set inheritance code for range of addrs in map.
3245 1.1 mrg *
3246 1.1 mrg * => map must be unlocked
3247 1.1 mrg * => note that the inherit code is used during a "fork". see fork
3248 1.1 mrg * code for details.
3249 1.1 mrg */
3250 1.1 mrg
3251 1.10 mrg int
3252 1.138 enami uvm_map_inherit(struct vm_map *map, vaddr_t start, vaddr_t end,
3253 1.138 enami vm_inherit_t new_inheritance)
3254 1.10 mrg {
3255 1.99 chs struct vm_map_entry *entry, *temp_entry;
3256 1.10 mrg UVMHIST_FUNC("uvm_map_inherit"); UVMHIST_CALLED(maphist);
3257 1.10 mrg UVMHIST_LOG(maphist,"(map=0x%x,start=0x%x,end=0x%x,new_inh=0x%x)",
3258 1.10 mrg map, start, end, new_inheritance);
3259 1.10 mrg
3260 1.10 mrg switch (new_inheritance) {
3261 1.80 wiz case MAP_INHERIT_NONE:
3262 1.80 wiz case MAP_INHERIT_COPY:
3263 1.80 wiz case MAP_INHERIT_SHARE:
3264 1.10 mrg break;
3265 1.10 mrg default:
3266 1.10 mrg UVMHIST_LOG(maphist,"<- done (INVALID ARG)",0,0,0,0);
3267 1.94 chs return EINVAL;
3268 1.10 mrg }
3269 1.1 mrg
3270 1.10 mrg vm_map_lock(map);
3271 1.10 mrg VM_MAP_RANGE_CHECK(map, start, end);
3272 1.10 mrg if (uvm_map_lookup_entry(map, start, &temp_entry)) {
3273 1.10 mrg entry = temp_entry;
3274 1.174 yamt UVM_MAP_CLIP_START(map, entry, start, NULL);
3275 1.10 mrg } else {
3276 1.10 mrg entry = temp_entry->next;
3277 1.10 mrg }
3278 1.10 mrg while ((entry != &map->header) && (entry->start < end)) {
3279 1.174 yamt UVM_MAP_CLIP_END(map, entry, end, NULL);
3280 1.10 mrg entry->inheritance = new_inheritance;
3281 1.10 mrg entry = entry->next;
3282 1.10 mrg }
3283 1.10 mrg vm_map_unlock(map);
3284 1.10 mrg UVMHIST_LOG(maphist,"<- done (OK)",0,0,0,0);
3285 1.94 chs return 0;
3286 1.41 mrg }
3287 1.41 mrg
3288 1.98 chs /*
3289 1.41 mrg * uvm_map_advice: set advice code for range of addrs in map.
3290 1.41 mrg *
3291 1.41 mrg * => map must be unlocked
3292 1.41 mrg */
3293 1.41 mrg
3294 1.41 mrg int
3295 1.138 enami uvm_map_advice(struct vm_map *map, vaddr_t start, vaddr_t end, int new_advice)
3296 1.41 mrg {
3297 1.99 chs struct vm_map_entry *entry, *temp_entry;
3298 1.41 mrg UVMHIST_FUNC("uvm_map_advice"); UVMHIST_CALLED(maphist);
3299 1.41 mrg UVMHIST_LOG(maphist,"(map=0x%x,start=0x%x,end=0x%x,new_adv=0x%x)",
3300 1.41 mrg map, start, end, new_advice);
3301 1.41 mrg
3302 1.41 mrg vm_map_lock(map);
3303 1.41 mrg VM_MAP_RANGE_CHECK(map, start, end);
3304 1.41 mrg if (uvm_map_lookup_entry(map, start, &temp_entry)) {
3305 1.41 mrg entry = temp_entry;
3306 1.174 yamt UVM_MAP_CLIP_START(map, entry, start, NULL);
3307 1.41 mrg } else {
3308 1.41 mrg entry = temp_entry->next;
3309 1.41 mrg }
3310 1.61 thorpej
3311 1.61 thorpej /*
3312 1.61 thorpej * XXXJRT: disallow holes?
3313 1.61 thorpej */
3314 1.61 thorpej
3315 1.41 mrg while ((entry != &map->header) && (entry->start < end)) {
3316 1.174 yamt UVM_MAP_CLIP_END(map, entry, end, NULL);
3317 1.41 mrg
3318 1.41 mrg switch (new_advice) {
3319 1.41 mrg case MADV_NORMAL:
3320 1.41 mrg case MADV_RANDOM:
3321 1.41 mrg case MADV_SEQUENTIAL:
3322 1.41 mrg /* nothing special here */
3323 1.41 mrg break;
3324 1.41 mrg
3325 1.41 mrg default:
3326 1.50 mrg vm_map_unlock(map);
3327 1.41 mrg UVMHIST_LOG(maphist,"<- done (INVALID ARG)",0,0,0,0);
3328 1.94 chs return EINVAL;
3329 1.41 mrg }
3330 1.41 mrg entry->advice = new_advice;
3331 1.41 mrg entry = entry->next;
3332 1.41 mrg }
3333 1.41 mrg
3334 1.41 mrg vm_map_unlock(map);
3335 1.41 mrg UVMHIST_LOG(maphist,"<- done (OK)",0,0,0,0);
3336 1.94 chs return 0;
3337 1.1 mrg }
3338 1.1 mrg
3339 1.1 mrg /*
3340 1.271 yamt * uvm_map_willneed: apply MADV_WILLNEED
3341 1.271 yamt */
3342 1.271 yamt
3343 1.271 yamt int
3344 1.271 yamt uvm_map_willneed(struct vm_map *map, vaddr_t start, vaddr_t end)
3345 1.271 yamt {
3346 1.271 yamt struct vm_map_entry *entry;
3347 1.271 yamt UVMHIST_FUNC("uvm_map_willneed"); UVMHIST_CALLED(maphist);
3348 1.271 yamt UVMHIST_LOG(maphist,"(map=0x%lx,start=0x%lx,end=0x%lx)",
3349 1.271 yamt map, start, end, 0);
3350 1.271 yamt
3351 1.271 yamt vm_map_lock_read(map);
3352 1.271 yamt VM_MAP_RANGE_CHECK(map, start, end);
3353 1.271 yamt if (!uvm_map_lookup_entry(map, start, &entry)) {
3354 1.271 yamt entry = entry->next;
3355 1.271 yamt }
3356 1.271 yamt while (entry->start < end) {
3357 1.271 yamt struct vm_amap * const amap = entry->aref.ar_amap;
3358 1.271 yamt struct uvm_object * const uobj = entry->object.uvm_obj;
3359 1.271 yamt
3360 1.271 yamt KASSERT(entry != &map->header);
3361 1.271 yamt KASSERT(start < entry->end);
3362 1.271 yamt /*
3363 1.271 yamt * XXX IMPLEMENT ME.
3364 1.271 yamt * Should invent a "weak" mode for uvm_fault()
3365 1.271 yamt * which would only do the PGO_LOCKED pgo_get().
3366 1.271 yamt *
3367 1.271 yamt * for now, we handle only the easy but common case.
3368 1.271 yamt */
3369 1.271 yamt if (UVM_ET_ISOBJ(entry) && amap == NULL && uobj != NULL) {
3370 1.271 yamt off_t offset;
3371 1.271 yamt off_t size;
3372 1.271 yamt
3373 1.271 yamt offset = entry->offset;
3374 1.271 yamt if (start < entry->start) {
3375 1.271 yamt offset += entry->start - start;
3376 1.271 yamt }
3377 1.271 yamt size = entry->offset + (entry->end - entry->start);
3378 1.271 yamt if (entry->end < end) {
3379 1.271 yamt size -= end - entry->end;
3380 1.271 yamt }
3381 1.271 yamt uvm_readahead(uobj, offset, size);
3382 1.271 yamt }
3383 1.271 yamt entry = entry->next;
3384 1.271 yamt }
3385 1.271 yamt vm_map_unlock_read(map);
3386 1.271 yamt UVMHIST_LOG(maphist,"<- done (OK)",0,0,0,0);
3387 1.271 yamt return 0;
3388 1.271 yamt }
3389 1.271 yamt
3390 1.271 yamt /*
3391 1.1 mrg * uvm_map_pageable: sets the pageability of a range in a map.
3392 1.1 mrg *
3393 1.56 thorpej * => wires map entries. should not be used for transient page locking.
3394 1.56 thorpej * for that, use uvm_fault_wire()/uvm_fault_unwire() (see uvm_vslock()).
3395 1.216 drochner * => regions specified as not pageable require lock-down (wired) memory
3396 1.1 mrg * and page tables.
3397 1.59 thorpej * => map must never be read-locked
3398 1.234 thorpej * => if islocked is true, map is already write-locked
3399 1.59 thorpej * => we always unlock the map, since we must downgrade to a read-lock
3400 1.59 thorpej * to call uvm_fault_wire()
3401 1.1 mrg * => XXXCDC: check this and try and clean it up.
3402 1.1 mrg */
3403 1.1 mrg
3404 1.19 kleink int
3405 1.138 enami uvm_map_pageable(struct vm_map *map, vaddr_t start, vaddr_t end,
3406 1.233 thorpej bool new_pageable, int lockflags)
3407 1.1 mrg {
3408 1.99 chs struct vm_map_entry *entry, *start_entry, *failed_entry;
3409 1.10 mrg int rv;
3410 1.60 thorpej #ifdef DIAGNOSTIC
3411 1.60 thorpej u_int timestamp_save;
3412 1.60 thorpej #endif
3413 1.10 mrg UVMHIST_FUNC("uvm_map_pageable"); UVMHIST_CALLED(maphist);
3414 1.10 mrg UVMHIST_LOG(maphist,"(map=0x%x,start=0x%x,end=0x%x,new_pageable=0x%x)",
3415 1.85 chs map, start, end, new_pageable);
3416 1.85 chs KASSERT(map->flags & VM_MAP_PAGEABLE);
3417 1.45 thorpej
3418 1.64 thorpej if ((lockflags & UVM_LK_ENTER) == 0)
3419 1.59 thorpej vm_map_lock(map);
3420 1.10 mrg VM_MAP_RANGE_CHECK(map, start, end);
3421 1.10 mrg
3422 1.98 chs /*
3423 1.10 mrg * only one pageability change may take place at one time, since
3424 1.10 mrg * uvm_fault_wire assumes it will be called only once for each
3425 1.10 mrg * wiring/unwiring. therefore, we have to make sure we're actually
3426 1.10 mrg * changing the pageability for the entire region. we do so before
3427 1.98 chs * making any changes.
3428 1.10 mrg */
3429 1.10 mrg
3430 1.234 thorpej if (uvm_map_lookup_entry(map, start, &start_entry) == false) {
3431 1.64 thorpej if ((lockflags & UVM_LK_EXIT) == 0)
3432 1.64 thorpej vm_map_unlock(map);
3433 1.85 chs
3434 1.94 chs UVMHIST_LOG(maphist,"<- done (fault)",0,0,0,0);
3435 1.94 chs return EFAULT;
3436 1.10 mrg }
3437 1.10 mrg entry = start_entry;
3438 1.10 mrg
3439 1.98 chs /*
3440 1.100 wiz * handle wiring and unwiring separately.
3441 1.10 mrg */
3442 1.1 mrg
3443 1.56 thorpej if (new_pageable) { /* unwire */
3444 1.174 yamt UVM_MAP_CLIP_START(map, entry, start, NULL);
3445 1.85 chs
3446 1.10 mrg /*
3447 1.10 mrg * unwiring. first ensure that the range to be unwired is
3448 1.98 chs * really wired down and that there are no holes.
3449 1.10 mrg */
3450 1.85 chs
3451 1.10 mrg while ((entry != &map->header) && (entry->start < end)) {
3452 1.10 mrg if (entry->wired_count == 0 ||
3453 1.10 mrg (entry->end < end &&
3454 1.55 thorpej (entry->next == &map->header ||
3455 1.55 thorpej entry->next->start > entry->end))) {
3456 1.64 thorpej if ((lockflags & UVM_LK_EXIT) == 0)
3457 1.64 thorpej vm_map_unlock(map);
3458 1.94 chs UVMHIST_LOG(maphist, "<- done (INVAL)",0,0,0,0);
3459 1.94 chs return EINVAL;
3460 1.10 mrg }
3461 1.10 mrg entry = entry->next;
3462 1.10 mrg }
3463 1.10 mrg
3464 1.98 chs /*
3465 1.56 thorpej * POSIX 1003.1b - a single munlock call unlocks a region,
3466 1.56 thorpej * regardless of the number of mlock calls made on that
3467 1.56 thorpej * region.
3468 1.10 mrg */
3469 1.85 chs
3470 1.10 mrg entry = start_entry;
3471 1.10 mrg while ((entry != &map->header) && (entry->start < end)) {
3472 1.174 yamt UVM_MAP_CLIP_END(map, entry, end, NULL);
3473 1.56 thorpej if (VM_MAPENT_ISWIRED(entry))
3474 1.10 mrg uvm_map_entry_unwire(map, entry);
3475 1.10 mrg entry = entry->next;
3476 1.10 mrg }
3477 1.64 thorpej if ((lockflags & UVM_LK_EXIT) == 0)
3478 1.64 thorpej vm_map_unlock(map);
3479 1.10 mrg UVMHIST_LOG(maphist,"<- done (OK UNWIRE)",0,0,0,0);
3480 1.94 chs return 0;
3481 1.10 mrg }
3482 1.10 mrg
3483 1.10 mrg /*
3484 1.10 mrg * wire case: in two passes [XXXCDC: ugly block of code here]
3485 1.10 mrg *
3486 1.10 mrg * 1: holding the write lock, we create any anonymous maps that need
3487 1.10 mrg * to be created. then we clip each map entry to the region to
3488 1.98 chs * be wired and increment its wiring count.
3489 1.10 mrg *
3490 1.10 mrg * 2: we downgrade to a read lock, and call uvm_fault_wire to fault
3491 1.56 thorpej * in the pages for any newly wired area (wired_count == 1).
3492 1.10 mrg *
3493 1.10 mrg * downgrading to a read lock for uvm_fault_wire avoids a possible
3494 1.10 mrg * deadlock with another thread that may have faulted on one of
3495 1.10 mrg * the pages to be wired (it would mark the page busy, blocking
3496 1.10 mrg * us, then in turn block on the map lock that we hold). because
3497 1.10 mrg * of problems in the recursive lock package, we cannot upgrade
3498 1.10 mrg * to a write lock in vm_map_lookup. thus, any actions that
3499 1.10 mrg * require the write lock must be done beforehand. because we
3500 1.10 mrg * keep the read lock on the map, the copy-on-write status of the
3501 1.10 mrg * entries we modify here cannot change.
3502 1.10 mrg */
3503 1.10 mrg
3504 1.10 mrg while ((entry != &map->header) && (entry->start < end)) {
3505 1.55 thorpej if (VM_MAPENT_ISWIRED(entry) == 0) { /* not already wired? */
3506 1.85 chs
3507 1.85 chs /*
3508 1.10 mrg * perform actions of vm_map_lookup that need the
3509 1.10 mrg * write lock on the map: create an anonymous map
3510 1.10 mrg * for a copy-on-write region, or an anonymous map
3511 1.29 chuck * for a zero-fill region. (XXXCDC: submap case
3512 1.29 chuck * ok?)
3513 1.10 mrg */
3514 1.85 chs
3515 1.29 chuck if (!UVM_ET_ISSUBMAP(entry)) { /* not submap */
3516 1.98 chs if (UVM_ET_ISNEEDSCOPY(entry) &&
3517 1.117 chs ((entry->max_protection & VM_PROT_WRITE) ||
3518 1.54 thorpej (entry->object.uvm_obj == NULL))) {
3519 1.212 yamt amap_copy(map, entry, 0, start, end);
3520 1.10 mrg /* XXXCDC: wait OK? */
3521 1.10 mrg }
3522 1.10 mrg }
3523 1.55 thorpej }
3524 1.174 yamt UVM_MAP_CLIP_START(map, entry, start, NULL);
3525 1.174 yamt UVM_MAP_CLIP_END(map, entry, end, NULL);
3526 1.10 mrg entry->wired_count++;
3527 1.10 mrg
3528 1.10 mrg /*
3529 1.98 chs * Check for holes
3530 1.10 mrg */
3531 1.85 chs
3532 1.54 thorpej if (entry->protection == VM_PROT_NONE ||
3533 1.54 thorpej (entry->end < end &&
3534 1.54 thorpej (entry->next == &map->header ||
3535 1.54 thorpej entry->next->start > entry->end))) {
3536 1.85 chs
3537 1.10 mrg /*
3538 1.10 mrg * found one. amap creation actions do not need to
3539 1.98 chs * be undone, but the wired counts need to be restored.
3540 1.10 mrg */
3541 1.85 chs
3542 1.10 mrg while (entry != &map->header && entry->end > start) {
3543 1.10 mrg entry->wired_count--;
3544 1.10 mrg entry = entry->prev;
3545 1.10 mrg }
3546 1.64 thorpej if ((lockflags & UVM_LK_EXIT) == 0)
3547 1.64 thorpej vm_map_unlock(map);
3548 1.10 mrg UVMHIST_LOG(maphist,"<- done (INVALID WIRE)",0,0,0,0);
3549 1.94 chs return EINVAL;
3550 1.10 mrg }
3551 1.10 mrg entry = entry->next;
3552 1.10 mrg }
3553 1.10 mrg
3554 1.10 mrg /*
3555 1.10 mrg * Pass 2.
3556 1.10 mrg */
3557 1.51 thorpej
3558 1.60 thorpej #ifdef DIAGNOSTIC
3559 1.60 thorpej timestamp_save = map->timestamp;
3560 1.60 thorpej #endif
3561 1.60 thorpej vm_map_busy(map);
3562 1.249 yamt vm_map_unlock(map);
3563 1.10 mrg
3564 1.10 mrg rv = 0;
3565 1.10 mrg entry = start_entry;
3566 1.10 mrg while (entry != &map->header && entry->start < end) {
3567 1.51 thorpej if (entry->wired_count == 1) {
3568 1.44 thorpej rv = uvm_fault_wire(map, entry->start, entry->end,
3569 1.216 drochner entry->max_protection, 1);
3570 1.10 mrg if (rv) {
3571 1.94 chs
3572 1.51 thorpej /*
3573 1.51 thorpej * wiring failed. break out of the loop.
3574 1.51 thorpej * we'll clean up the map below, once we
3575 1.51 thorpej * have a write lock again.
3576 1.51 thorpej */
3577 1.94 chs
3578 1.51 thorpej break;
3579 1.10 mrg }
3580 1.10 mrg }
3581 1.10 mrg entry = entry->next;
3582 1.10 mrg }
3583 1.10 mrg
3584 1.139 enami if (rv) { /* failed? */
3585 1.85 chs
3586 1.52 thorpej /*
3587 1.52 thorpej * Get back to an exclusive (write) lock.
3588 1.52 thorpej */
3589 1.85 chs
3590 1.249 yamt vm_map_lock(map);
3591 1.60 thorpej vm_map_unbusy(map);
3592 1.60 thorpej
3593 1.60 thorpej #ifdef DIAGNOSTIC
3594 1.252 yamt if (timestamp_save + 1 != map->timestamp)
3595 1.60 thorpej panic("uvm_map_pageable: stale map");
3596 1.60 thorpej #endif
3597 1.10 mrg
3598 1.51 thorpej /*
3599 1.51 thorpej * first drop the wiring count on all the entries
3600 1.51 thorpej * which haven't actually been wired yet.
3601 1.51 thorpej */
3602 1.85 chs
3603 1.54 thorpej failed_entry = entry;
3604 1.54 thorpej while (entry != &map->header && entry->start < end) {
3605 1.51 thorpej entry->wired_count--;
3606 1.54 thorpej entry = entry->next;
3607 1.54 thorpej }
3608 1.51 thorpej
3609 1.51 thorpej /*
3610 1.54 thorpej * now, unwire all the entries that were successfully
3611 1.54 thorpej * wired above.
3612 1.51 thorpej */
3613 1.85 chs
3614 1.54 thorpej entry = start_entry;
3615 1.54 thorpej while (entry != failed_entry) {
3616 1.54 thorpej entry->wired_count--;
3617 1.55 thorpej if (VM_MAPENT_ISWIRED(entry) == 0)
3618 1.54 thorpej uvm_map_entry_unwire(map, entry);
3619 1.54 thorpej entry = entry->next;
3620 1.54 thorpej }
3621 1.64 thorpej if ((lockflags & UVM_LK_EXIT) == 0)
3622 1.64 thorpej vm_map_unlock(map);
3623 1.10 mrg UVMHIST_LOG(maphist, "<- done (RV=%d)", rv,0,0,0);
3624 1.139 enami return (rv);
3625 1.10 mrg }
3626 1.51 thorpej
3627 1.64 thorpej if ((lockflags & UVM_LK_EXIT) == 0) {
3628 1.64 thorpej vm_map_unbusy(map);
3629 1.64 thorpej } else {
3630 1.85 chs
3631 1.64 thorpej /*
3632 1.64 thorpej * Get back to an exclusive (write) lock.
3633 1.64 thorpej */
3634 1.85 chs
3635 1.249 yamt vm_map_lock(map);
3636 1.64 thorpej vm_map_unbusy(map);
3637 1.64 thorpej }
3638 1.64 thorpej
3639 1.10 mrg UVMHIST_LOG(maphist,"<- done (OK WIRE)",0,0,0,0);
3640 1.94 chs return 0;
3641 1.1 mrg }
3642 1.1 mrg
3643 1.1 mrg /*
3644 1.54 thorpej * uvm_map_pageable_all: special case of uvm_map_pageable - affects
3645 1.54 thorpej * all mapped regions.
3646 1.54 thorpej *
3647 1.54 thorpej * => map must not be locked.
3648 1.54 thorpej * => if no flags are specified, all regions are unwired.
3649 1.54 thorpej * => XXXJRT: has some of the same problems as uvm_map_pageable() above.
3650 1.54 thorpej */
3651 1.54 thorpej
3652 1.54 thorpej int
3653 1.138 enami uvm_map_pageable_all(struct vm_map *map, int flags, vsize_t limit)
3654 1.54 thorpej {
3655 1.99 chs struct vm_map_entry *entry, *failed_entry;
3656 1.54 thorpej vsize_t size;
3657 1.54 thorpej int rv;
3658 1.60 thorpej #ifdef DIAGNOSTIC
3659 1.60 thorpej u_int timestamp_save;
3660 1.60 thorpej #endif
3661 1.54 thorpej UVMHIST_FUNC("uvm_map_pageable_all"); UVMHIST_CALLED(maphist);
3662 1.54 thorpej UVMHIST_LOG(maphist,"(map=0x%x,flags=0x%x)", map, flags, 0, 0);
3663 1.54 thorpej
3664 1.85 chs KASSERT(map->flags & VM_MAP_PAGEABLE);
3665 1.54 thorpej
3666 1.54 thorpej vm_map_lock(map);
3667 1.54 thorpej
3668 1.54 thorpej /*
3669 1.54 thorpej * handle wiring and unwiring separately.
3670 1.54 thorpej */
3671 1.54 thorpej
3672 1.54 thorpej if (flags == 0) { /* unwire */
3673 1.99 chs
3674 1.54 thorpej /*
3675 1.56 thorpej * POSIX 1003.1b -- munlockall unlocks all regions,
3676 1.56 thorpej * regardless of how many times mlockall has been called.
3677 1.54 thorpej */
3678 1.99 chs
3679 1.54 thorpej for (entry = map->header.next; entry != &map->header;
3680 1.54 thorpej entry = entry->next) {
3681 1.56 thorpej if (VM_MAPENT_ISWIRED(entry))
3682 1.56 thorpej uvm_map_entry_unwire(map, entry);
3683 1.54 thorpej }
3684 1.238 ad map->flags &= ~VM_MAP_WIREFUTURE;
3685 1.54 thorpej vm_map_unlock(map);
3686 1.54 thorpej UVMHIST_LOG(maphist,"<- done (OK UNWIRE)",0,0,0,0);
3687 1.94 chs return 0;
3688 1.54 thorpej }
3689 1.54 thorpej
3690 1.54 thorpej if (flags & MCL_FUTURE) {
3691 1.99 chs
3692 1.54 thorpej /*
3693 1.54 thorpej * must wire all future mappings; remember this.
3694 1.54 thorpej */
3695 1.99 chs
3696 1.238 ad map->flags |= VM_MAP_WIREFUTURE;
3697 1.54 thorpej }
3698 1.54 thorpej
3699 1.54 thorpej if ((flags & MCL_CURRENT) == 0) {
3700 1.99 chs
3701 1.54 thorpej /*
3702 1.54 thorpej * no more work to do!
3703 1.54 thorpej */
3704 1.99 chs
3705 1.54 thorpej UVMHIST_LOG(maphist,"<- done (OK no wire)",0,0,0,0);
3706 1.54 thorpej vm_map_unlock(map);
3707 1.94 chs return 0;
3708 1.54 thorpej }
3709 1.54 thorpej
3710 1.54 thorpej /*
3711 1.54 thorpej * wire case: in three passes [XXXCDC: ugly block of code here]
3712 1.54 thorpej *
3713 1.54 thorpej * 1: holding the write lock, count all pages mapped by non-wired
3714 1.54 thorpej * entries. if this would cause us to go over our limit, we fail.
3715 1.54 thorpej *
3716 1.54 thorpej * 2: still holding the write lock, we create any anonymous maps that
3717 1.54 thorpej * need to be created. then we increment its wiring count.
3718 1.54 thorpej *
3719 1.54 thorpej * 3: we downgrade to a read lock, and call uvm_fault_wire to fault
3720 1.56 thorpej * in the pages for any newly wired area (wired_count == 1).
3721 1.54 thorpej *
3722 1.54 thorpej * downgrading to a read lock for uvm_fault_wire avoids a possible
3723 1.54 thorpej * deadlock with another thread that may have faulted on one of
3724 1.54 thorpej * the pages to be wired (it would mark the page busy, blocking
3725 1.54 thorpej * us, then in turn block on the map lock that we hold). because
3726 1.54 thorpej * of problems in the recursive lock package, we cannot upgrade
3727 1.54 thorpej * to a write lock in vm_map_lookup. thus, any actions that
3728 1.54 thorpej * require the write lock must be done beforehand. because we
3729 1.54 thorpej * keep the read lock on the map, the copy-on-write status of the
3730 1.54 thorpej * entries we modify here cannot change.
3731 1.54 thorpej */
3732 1.54 thorpej
3733 1.54 thorpej for (size = 0, entry = map->header.next; entry != &map->header;
3734 1.54 thorpej entry = entry->next) {
3735 1.54 thorpej if (entry->protection != VM_PROT_NONE &&
3736 1.55 thorpej VM_MAPENT_ISWIRED(entry) == 0) { /* not already wired? */
3737 1.54 thorpej size += entry->end - entry->start;
3738 1.54 thorpej }
3739 1.54 thorpej }
3740 1.54 thorpej
3741 1.54 thorpej if (atop(size) + uvmexp.wired > uvmexp.wiredmax) {
3742 1.54 thorpej vm_map_unlock(map);
3743 1.94 chs return ENOMEM;
3744 1.54 thorpej }
3745 1.54 thorpej
3746 1.54 thorpej if (limit != 0 &&
3747 1.54 thorpej (size + ptoa(pmap_wired_count(vm_map_pmap(map))) > limit)) {
3748 1.54 thorpej vm_map_unlock(map);
3749 1.94 chs return ENOMEM;
3750 1.54 thorpej }
3751 1.54 thorpej
3752 1.54 thorpej /*
3753 1.54 thorpej * Pass 2.
3754 1.54 thorpej */
3755 1.54 thorpej
3756 1.54 thorpej for (entry = map->header.next; entry != &map->header;
3757 1.54 thorpej entry = entry->next) {
3758 1.54 thorpej if (entry->protection == VM_PROT_NONE)
3759 1.54 thorpej continue;
3760 1.55 thorpej if (VM_MAPENT_ISWIRED(entry) == 0) { /* not already wired? */
3761 1.99 chs
3762 1.54 thorpej /*
3763 1.54 thorpej * perform actions of vm_map_lookup that need the
3764 1.54 thorpej * write lock on the map: create an anonymous map
3765 1.54 thorpej * for a copy-on-write region, or an anonymous map
3766 1.54 thorpej * for a zero-fill region. (XXXCDC: submap case
3767 1.54 thorpej * ok?)
3768 1.54 thorpej */
3769 1.99 chs
3770 1.54 thorpej if (!UVM_ET_ISSUBMAP(entry)) { /* not submap */
3771 1.98 chs if (UVM_ET_ISNEEDSCOPY(entry) &&
3772 1.117 chs ((entry->max_protection & VM_PROT_WRITE) ||
3773 1.54 thorpej (entry->object.uvm_obj == NULL))) {
3774 1.212 yamt amap_copy(map, entry, 0, entry->start,
3775 1.212 yamt entry->end);
3776 1.54 thorpej /* XXXCDC: wait OK? */
3777 1.54 thorpej }
3778 1.54 thorpej }
3779 1.55 thorpej }
3780 1.54 thorpej entry->wired_count++;
3781 1.54 thorpej }
3782 1.54 thorpej
3783 1.54 thorpej /*
3784 1.54 thorpej * Pass 3.
3785 1.54 thorpej */
3786 1.54 thorpej
3787 1.60 thorpej #ifdef DIAGNOSTIC
3788 1.60 thorpej timestamp_save = map->timestamp;
3789 1.60 thorpej #endif
3790 1.60 thorpej vm_map_busy(map);
3791 1.249 yamt vm_map_unlock(map);
3792 1.54 thorpej
3793 1.94 chs rv = 0;
3794 1.54 thorpej for (entry = map->header.next; entry != &map->header;
3795 1.54 thorpej entry = entry->next) {
3796 1.54 thorpej if (entry->wired_count == 1) {
3797 1.54 thorpej rv = uvm_fault_wire(map, entry->start, entry->end,
3798 1.216 drochner entry->max_protection, 1);
3799 1.54 thorpej if (rv) {
3800 1.99 chs
3801 1.54 thorpej /*
3802 1.54 thorpej * wiring failed. break out of the loop.
3803 1.54 thorpej * we'll clean up the map below, once we
3804 1.54 thorpej * have a write lock again.
3805 1.54 thorpej */
3806 1.99 chs
3807 1.54 thorpej break;
3808 1.54 thorpej }
3809 1.54 thorpej }
3810 1.54 thorpej }
3811 1.54 thorpej
3812 1.99 chs if (rv) {
3813 1.99 chs
3814 1.54 thorpej /*
3815 1.54 thorpej * Get back an exclusive (write) lock.
3816 1.54 thorpej */
3817 1.99 chs
3818 1.249 yamt vm_map_lock(map);
3819 1.60 thorpej vm_map_unbusy(map);
3820 1.60 thorpej
3821 1.60 thorpej #ifdef DIAGNOSTIC
3822 1.252 yamt if (timestamp_save + 1 != map->timestamp)
3823 1.60 thorpej panic("uvm_map_pageable_all: stale map");
3824 1.60 thorpej #endif
3825 1.54 thorpej
3826 1.54 thorpej /*
3827 1.54 thorpej * first drop the wiring count on all the entries
3828 1.54 thorpej * which haven't actually been wired yet.
3829 1.67 thorpej *
3830 1.67 thorpej * Skip VM_PROT_NONE entries like we did above.
3831 1.54 thorpej */
3832 1.99 chs
3833 1.54 thorpej failed_entry = entry;
3834 1.54 thorpej for (/* nothing */; entry != &map->header;
3835 1.67 thorpej entry = entry->next) {
3836 1.67 thorpej if (entry->protection == VM_PROT_NONE)
3837 1.67 thorpej continue;
3838 1.54 thorpej entry->wired_count--;
3839 1.67 thorpej }
3840 1.54 thorpej
3841 1.54 thorpej /*
3842 1.54 thorpej * now, unwire all the entries that were successfully
3843 1.54 thorpej * wired above.
3844 1.67 thorpej *
3845 1.67 thorpej * Skip VM_PROT_NONE entries like we did above.
3846 1.54 thorpej */
3847 1.99 chs
3848 1.54 thorpej for (entry = map->header.next; entry != failed_entry;
3849 1.54 thorpej entry = entry->next) {
3850 1.67 thorpej if (entry->protection == VM_PROT_NONE)
3851 1.67 thorpej continue;
3852 1.54 thorpej entry->wired_count--;
3853 1.67 thorpej if (VM_MAPENT_ISWIRED(entry))
3854 1.54 thorpej uvm_map_entry_unwire(map, entry);
3855 1.54 thorpej }
3856 1.54 thorpej vm_map_unlock(map);
3857 1.54 thorpej UVMHIST_LOG(maphist,"<- done (RV=%d)", rv,0,0,0);
3858 1.54 thorpej return (rv);
3859 1.54 thorpej }
3860 1.54 thorpej
3861 1.60 thorpej vm_map_unbusy(map);
3862 1.54 thorpej
3863 1.54 thorpej UVMHIST_LOG(maphist,"<- done (OK WIRE)",0,0,0,0);
3864 1.94 chs return 0;
3865 1.54 thorpej }
3866 1.54 thorpej
3867 1.54 thorpej /*
3868 1.61 thorpej * uvm_map_clean: clean out a map range
3869 1.1 mrg *
3870 1.1 mrg * => valid flags:
3871 1.61 thorpej * if (flags & PGO_CLEANIT): dirty pages are cleaned first
3872 1.1 mrg * if (flags & PGO_SYNCIO): dirty pages are written synchronously
3873 1.1 mrg * if (flags & PGO_DEACTIVATE): any cached pages are deactivated after clean
3874 1.1 mrg * if (flags & PGO_FREE): any cached pages are freed after clean
3875 1.1 mrg * => returns an error if any part of the specified range isn't mapped
3876 1.98 chs * => never a need to flush amap layer since the anonymous memory has
3877 1.61 thorpej * no permanent home, but may deactivate pages there
3878 1.61 thorpej * => called from sys_msync() and sys_madvise()
3879 1.1 mrg * => caller must not write-lock map (read OK).
3880 1.1 mrg * => we may sleep while cleaning if SYNCIO [with map read-locked]
3881 1.1 mrg */
3882 1.1 mrg
3883 1.10 mrg int
3884 1.138 enami uvm_map_clean(struct vm_map *map, vaddr_t start, vaddr_t end, int flags)
3885 1.10 mrg {
3886 1.99 chs struct vm_map_entry *current, *entry;
3887 1.61 thorpej struct uvm_object *uobj;
3888 1.61 thorpej struct vm_amap *amap;
3889 1.61 thorpej struct vm_anon *anon;
3890 1.61 thorpej struct vm_page *pg;
3891 1.61 thorpej vaddr_t offset;
3892 1.24 eeh vsize_t size;
3893 1.188 dbj voff_t uoff;
3894 1.106 chs int error, refs;
3895 1.10 mrg UVMHIST_FUNC("uvm_map_clean"); UVMHIST_CALLED(maphist);
3896 1.85 chs
3897 1.10 mrg UVMHIST_LOG(maphist,"(map=0x%x,start=0x%x,end=0x%x,flags=0x%x)",
3898 1.85 chs map, start, end, flags);
3899 1.85 chs KASSERT((flags & (PGO_FREE|PGO_DEACTIVATE)) !=
3900 1.85 chs (PGO_FREE|PGO_DEACTIVATE));
3901 1.61 thorpej
3902 1.10 mrg vm_map_lock_read(map);
3903 1.10 mrg VM_MAP_RANGE_CHECK(map, start, end);
3904 1.234 thorpej if (uvm_map_lookup_entry(map, start, &entry) == false) {
3905 1.10 mrg vm_map_unlock_read(map);
3906 1.94 chs return EFAULT;
3907 1.10 mrg }
3908 1.10 mrg
3909 1.10 mrg /*
3910 1.186 chs * Make a first pass to check for holes and wiring problems.
3911 1.10 mrg */
3912 1.85 chs
3913 1.10 mrg for (current = entry; current->start < end; current = current->next) {
3914 1.10 mrg if (UVM_ET_ISSUBMAP(current)) {
3915 1.10 mrg vm_map_unlock_read(map);
3916 1.94 chs return EINVAL;
3917 1.10 mrg }
3918 1.186 chs if ((flags & PGO_FREE) != 0 && VM_MAPENT_ISWIRED(entry)) {
3919 1.186 chs vm_map_unlock_read(map);
3920 1.186 chs return EBUSY;
3921 1.186 chs }
3922 1.90 chs if (end <= current->end) {
3923 1.90 chs break;
3924 1.90 chs }
3925 1.90 chs if (current->end != current->next->start) {
3926 1.10 mrg vm_map_unlock_read(map);
3927 1.94 chs return EFAULT;
3928 1.10 mrg }
3929 1.10 mrg }
3930 1.10 mrg
3931 1.94 chs error = 0;
3932 1.90 chs for (current = entry; start < end; current = current->next) {
3933 1.61 thorpej amap = current->aref.ar_amap; /* top layer */
3934 1.61 thorpej uobj = current->object.uvm_obj; /* bottom layer */
3935 1.85 chs KASSERT(start >= current->start);
3936 1.1 mrg
3937 1.10 mrg /*
3938 1.61 thorpej * No amap cleaning necessary if:
3939 1.61 thorpej *
3940 1.61 thorpej * (1) There's no amap.
3941 1.61 thorpej *
3942 1.61 thorpej * (2) We're not deactivating or freeing pages.
3943 1.10 mrg */
3944 1.85 chs
3945 1.90 chs if (amap == NULL || (flags & (PGO_DEACTIVATE|PGO_FREE)) == 0)
3946 1.61 thorpej goto flush_object;
3947 1.61 thorpej
3948 1.61 thorpej amap_lock(amap);
3949 1.61 thorpej offset = start - current->start;
3950 1.90 chs size = MIN(end, current->end) - start;
3951 1.90 chs for ( ; size != 0; size -= PAGE_SIZE, offset += PAGE_SIZE) {
3952 1.61 thorpej anon = amap_lookup(¤t->aref, offset);
3953 1.61 thorpej if (anon == NULL)
3954 1.61 thorpej continue;
3955 1.61 thorpej
3956 1.248 ad mutex_enter(&anon->an_lock);
3957 1.192 yamt pg = anon->an_page;
3958 1.63 thorpej if (pg == NULL) {
3959 1.248 ad mutex_exit(&anon->an_lock);
3960 1.63 thorpej continue;
3961 1.63 thorpej }
3962 1.63 thorpej
3963 1.61 thorpej switch (flags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE)) {
3964 1.85 chs
3965 1.61 thorpej /*
3966 1.115 chs * In these first 3 cases, we just deactivate the page.
3967 1.61 thorpej */
3968 1.85 chs
3969 1.61 thorpej case PGO_CLEANIT|PGO_FREE:
3970 1.61 thorpej case PGO_CLEANIT|PGO_DEACTIVATE:
3971 1.61 thorpej case PGO_DEACTIVATE:
3972 1.68 thorpej deactivate_it:
3973 1.61 thorpej /*
3974 1.115 chs * skip the page if it's loaned or wired,
3975 1.115 chs * since it shouldn't be on a paging queue
3976 1.115 chs * at all in these cases.
3977 1.61 thorpej */
3978 1.85 chs
3979 1.248 ad mutex_enter(&uvm_pageqlock);
3980 1.115 chs if (pg->loan_count != 0 ||
3981 1.115 chs pg->wire_count != 0) {
3982 1.248 ad mutex_exit(&uvm_pageqlock);
3983 1.248 ad mutex_exit(&anon->an_lock);
3984 1.61 thorpej continue;
3985 1.61 thorpej }
3986 1.85 chs KASSERT(pg->uanon == anon);
3987 1.61 thorpej uvm_pagedeactivate(pg);
3988 1.248 ad mutex_exit(&uvm_pageqlock);
3989 1.248 ad mutex_exit(&anon->an_lock);
3990 1.61 thorpej continue;
3991 1.61 thorpej
3992 1.61 thorpej case PGO_FREE:
3993 1.85 chs
3994 1.68 thorpej /*
3995 1.68 thorpej * If there are multiple references to
3996 1.68 thorpej * the amap, just deactivate the page.
3997 1.68 thorpej */
3998 1.85 chs
3999 1.68 thorpej if (amap_refs(amap) > 1)
4000 1.68 thorpej goto deactivate_it;
4001 1.68 thorpej
4002 1.115 chs /* skip the page if it's wired */
4003 1.62 thorpej if (pg->wire_count != 0) {
4004 1.248 ad mutex_exit(&anon->an_lock);
4005 1.62 thorpej continue;
4006 1.62 thorpej }
4007 1.66 thorpej amap_unadd(¤t->aref, offset);
4008 1.61 thorpej refs = --anon->an_ref;
4009 1.248 ad mutex_exit(&anon->an_lock);
4010 1.61 thorpej if (refs == 0)
4011 1.61 thorpej uvm_anfree(anon);
4012 1.61 thorpej continue;
4013 1.61 thorpej }
4014 1.61 thorpej }
4015 1.61 thorpej amap_unlock(amap);
4016 1.1 mrg
4017 1.61 thorpej flush_object:
4018 1.10 mrg /*
4019 1.33 chuck * flush pages if we've got a valid backing object.
4020 1.116 chs * note that we must always clean object pages before
4021 1.116 chs * freeing them since otherwise we could reveal stale
4022 1.116 chs * data from files.
4023 1.10 mrg */
4024 1.1 mrg
4025 1.188 dbj uoff = current->offset + (start - current->start);
4026 1.90 chs size = MIN(end, current->end) - start;
4027 1.61 thorpej if (uobj != NULL) {
4028 1.248 ad mutex_enter(&uobj->vmobjlock);
4029 1.136 thorpej if (uobj->pgops->pgo_put != NULL)
4030 1.188 dbj error = (uobj->pgops->pgo_put)(uobj, uoff,
4031 1.188 dbj uoff + size, flags | PGO_CLEANIT);
4032 1.136 thorpej else
4033 1.136 thorpej error = 0;
4034 1.10 mrg }
4035 1.10 mrg start += size;
4036 1.10 mrg }
4037 1.1 mrg vm_map_unlock_read(map);
4038 1.98 chs return (error);
4039 1.1 mrg }
4040 1.1 mrg
4041 1.1 mrg
4042 1.1 mrg /*
4043 1.1 mrg * uvm_map_checkprot: check protection in map
4044 1.1 mrg *
4045 1.1 mrg * => must allow specified protection in a fully allocated region.
4046 1.1 mrg * => map must be read or write locked by caller.
4047 1.1 mrg */
4048 1.1 mrg
4049 1.233 thorpej bool
4050 1.138 enami uvm_map_checkprot(struct vm_map *map, vaddr_t start, vaddr_t end,
4051 1.138 enami vm_prot_t protection)
4052 1.10 mrg {
4053 1.99 chs struct vm_map_entry *entry;
4054 1.99 chs struct vm_map_entry *tmp_entry;
4055 1.10 mrg
4056 1.94 chs if (!uvm_map_lookup_entry(map, start, &tmp_entry)) {
4057 1.234 thorpej return (false);
4058 1.94 chs }
4059 1.94 chs entry = tmp_entry;
4060 1.94 chs while (start < end) {
4061 1.94 chs if (entry == &map->header) {
4062 1.234 thorpej return (false);
4063 1.94 chs }
4064 1.85 chs
4065 1.10 mrg /*
4066 1.10 mrg * no holes allowed
4067 1.10 mrg */
4068 1.10 mrg
4069 1.94 chs if (start < entry->start) {
4070 1.234 thorpej return (false);
4071 1.94 chs }
4072 1.10 mrg
4073 1.10 mrg /*
4074 1.10 mrg * check protection associated with entry
4075 1.10 mrg */
4076 1.1 mrg
4077 1.94 chs if ((entry->protection & protection) != protection) {
4078 1.234 thorpej return (false);
4079 1.94 chs }
4080 1.94 chs start = entry->end;
4081 1.94 chs entry = entry->next;
4082 1.94 chs }
4083 1.234 thorpej return (true);
4084 1.1 mrg }
4085 1.1 mrg
4086 1.1 mrg /*
4087 1.1 mrg * uvmspace_alloc: allocate a vmspace structure.
4088 1.1 mrg *
4089 1.1 mrg * - structure includes vm_map and pmap
4090 1.1 mrg * - XXX: no locking on this structure
4091 1.1 mrg * - refcnt set to 1, rest must be init'd by caller
4092 1.1 mrg */
4093 1.10 mrg struct vmspace *
4094 1.199 christos uvmspace_alloc(vaddr_t vmin, vaddr_t vmax)
4095 1.10 mrg {
4096 1.10 mrg struct vmspace *vm;
4097 1.10 mrg UVMHIST_FUNC("uvmspace_alloc"); UVMHIST_CALLED(maphist);
4098 1.10 mrg
4099 1.248 ad vm = pool_cache_get(&uvm_vmspace_cache, PR_WAITOK);
4100 1.199 christos uvmspace_init(vm, NULL, vmin, vmax);
4101 1.15 thorpej UVMHIST_LOG(maphist,"<- done (vm=0x%x)", vm,0,0,0);
4102 1.15 thorpej return (vm);
4103 1.15 thorpej }
4104 1.15 thorpej
4105 1.15 thorpej /*
4106 1.15 thorpej * uvmspace_init: initialize a vmspace structure.
4107 1.15 thorpej *
4108 1.15 thorpej * - XXX: no locking on this structure
4109 1.132 matt * - refcnt set to 1, rest must be init'd by caller
4110 1.15 thorpej */
4111 1.15 thorpej void
4112 1.199 christos uvmspace_init(struct vmspace *vm, struct pmap *pmap, vaddr_t vmin, vaddr_t vmax)
4113 1.15 thorpej {
4114 1.15 thorpej UVMHIST_FUNC("uvmspace_init"); UVMHIST_CALLED(maphist);
4115 1.15 thorpej
4116 1.23 perry memset(vm, 0, sizeof(*vm));
4117 1.199 christos uvm_map_setup(&vm->vm_map, vmin, vmax, VM_MAP_PAGEABLE
4118 1.131 atatat #ifdef __USING_TOPDOWN_VM
4119 1.131 atatat | VM_MAP_TOPDOWN
4120 1.131 atatat #endif
4121 1.131 atatat );
4122 1.15 thorpej if (pmap)
4123 1.15 thorpej pmap_reference(pmap);
4124 1.15 thorpej else
4125 1.15 thorpej pmap = pmap_create();
4126 1.15 thorpej vm->vm_map.pmap = pmap;
4127 1.10 mrg vm->vm_refcnt = 1;
4128 1.15 thorpej UVMHIST_LOG(maphist,"<- done",0,0,0,0);
4129 1.1 mrg }
4130 1.1 mrg
4131 1.1 mrg /*
4132 1.168 junyoung * uvmspace_share: share a vmspace between two processes
4133 1.1 mrg *
4134 1.1 mrg * - used for vfork, threads(?)
4135 1.1 mrg */
4136 1.1 mrg
4137 1.10 mrg void
4138 1.138 enami uvmspace_share(struct proc *p1, struct proc *p2)
4139 1.1 mrg {
4140 1.139 enami
4141 1.215 yamt uvmspace_addref(p1->p_vmspace);
4142 1.10 mrg p2->p_vmspace = p1->p_vmspace;
4143 1.1 mrg }
4144 1.1 mrg
4145 1.1 mrg /*
4146 1.1 mrg * uvmspace_unshare: ensure that process "p" has its own, unshared, vmspace
4147 1.1 mrg *
4148 1.1 mrg * - XXX: no locking on vmspace
4149 1.1 mrg */
4150 1.1 mrg
4151 1.10 mrg void
4152 1.138 enami uvmspace_unshare(struct lwp *l)
4153 1.10 mrg {
4154 1.128 thorpej struct proc *p = l->l_proc;
4155 1.10 mrg struct vmspace *nvm, *ovm = p->p_vmspace;
4156 1.85 chs
4157 1.10 mrg if (ovm->vm_refcnt == 1)
4158 1.10 mrg /* nothing to do: vmspace isn't shared in the first place */
4159 1.10 mrg return;
4160 1.85 chs
4161 1.10 mrg /* make a new vmspace, still holding old one */
4162 1.10 mrg nvm = uvmspace_fork(ovm);
4163 1.10 mrg
4164 1.254 ad kpreempt_disable();
4165 1.128 thorpej pmap_deactivate(l); /* unbind old vmspace */
4166 1.98 chs p->p_vmspace = nvm;
4167 1.128 thorpej pmap_activate(l); /* switch to new vmspace */
4168 1.254 ad kpreempt_enable();
4169 1.13 thorpej
4170 1.10 mrg uvmspace_free(ovm); /* drop reference to old vmspace */
4171 1.1 mrg }
4172 1.1 mrg
4173 1.1 mrg /*
4174 1.1 mrg * uvmspace_exec: the process wants to exec a new program
4175 1.1 mrg */
4176 1.1 mrg
4177 1.10 mrg void
4178 1.138 enami uvmspace_exec(struct lwp *l, vaddr_t start, vaddr_t end)
4179 1.1 mrg {
4180 1.128 thorpej struct proc *p = l->l_proc;
4181 1.10 mrg struct vmspace *nvm, *ovm = p->p_vmspace;
4182 1.99 chs struct vm_map *map = &ovm->vm_map;
4183 1.1 mrg
4184 1.71 chs #ifdef __sparc__
4185 1.10 mrg /* XXX cgd 960926: the sparc #ifdef should be a MD hook */
4186 1.128 thorpej kill_user_windows(l); /* before stack addresses go away */
4187 1.1 mrg #endif
4188 1.1 mrg
4189 1.10 mrg /*
4190 1.10 mrg * see if more than one process is using this vmspace...
4191 1.10 mrg */
4192 1.1 mrg
4193 1.10 mrg if (ovm->vm_refcnt == 1) {
4194 1.1 mrg
4195 1.10 mrg /*
4196 1.10 mrg * if p is the only process using its vmspace then we can safely
4197 1.10 mrg * recycle that vmspace for the program that is being exec'd.
4198 1.10 mrg */
4199 1.1 mrg
4200 1.1 mrg #ifdef SYSVSHM
4201 1.10 mrg /*
4202 1.10 mrg * SYSV SHM semantics require us to kill all segments on an exec
4203 1.10 mrg */
4204 1.99 chs
4205 1.10 mrg if (ovm->vm_shm)
4206 1.10 mrg shmexit(ovm);
4207 1.10 mrg #endif
4208 1.54 thorpej
4209 1.54 thorpej /*
4210 1.54 thorpej * POSIX 1003.1b -- "lock future mappings" is revoked
4211 1.54 thorpej * when a process execs another program image.
4212 1.54 thorpej */
4213 1.99 chs
4214 1.238 ad map->flags &= ~VM_MAP_WIREFUTURE;
4215 1.10 mrg
4216 1.10 mrg /*
4217 1.10 mrg * now unmap the old program
4218 1.10 mrg */
4219 1.99 chs
4220 1.120 chs pmap_remove_all(map->pmap);
4221 1.184 chs uvm_unmap(map, vm_map_min(map), vm_map_max(map));
4222 1.144 yamt KASSERT(map->header.prev == &map->header);
4223 1.144 yamt KASSERT(map->nentries == 0);
4224 1.93 eeh
4225 1.93 eeh /*
4226 1.93 eeh * resize the map
4227 1.93 eeh */
4228 1.99 chs
4229 1.184 chs vm_map_setmin(map, start);
4230 1.184 chs vm_map_setmax(map, end);
4231 1.10 mrg } else {
4232 1.10 mrg
4233 1.10 mrg /*
4234 1.10 mrg * p's vmspace is being shared, so we can't reuse it for p since
4235 1.10 mrg * it is still being used for others. allocate a new vmspace
4236 1.10 mrg * for p
4237 1.10 mrg */
4238 1.99 chs
4239 1.101 chs nvm = uvmspace_alloc(start, end);
4240 1.1 mrg
4241 1.10 mrg /*
4242 1.10 mrg * install new vmspace and drop our ref to the old one.
4243 1.10 mrg */
4244 1.10 mrg
4245 1.254 ad kpreempt_disable();
4246 1.128 thorpej pmap_deactivate(l);
4247 1.10 mrg p->p_vmspace = nvm;
4248 1.128 thorpej pmap_activate(l);
4249 1.254 ad kpreempt_enable();
4250 1.13 thorpej
4251 1.10 mrg uvmspace_free(ovm);
4252 1.10 mrg }
4253 1.1 mrg }
4254 1.1 mrg
4255 1.1 mrg /*
4256 1.215 yamt * uvmspace_addref: add a referece to a vmspace.
4257 1.215 yamt */
4258 1.215 yamt
4259 1.215 yamt void
4260 1.215 yamt uvmspace_addref(struct vmspace *vm)
4261 1.215 yamt {
4262 1.215 yamt struct vm_map *map = &vm->vm_map;
4263 1.215 yamt
4264 1.215 yamt KASSERT((map->flags & VM_MAP_DYING) == 0);
4265 1.215 yamt
4266 1.238 ad mutex_enter(&map->misc_lock);
4267 1.215 yamt KASSERT(vm->vm_refcnt > 0);
4268 1.215 yamt vm->vm_refcnt++;
4269 1.238 ad mutex_exit(&map->misc_lock);
4270 1.215 yamt }
4271 1.215 yamt
4272 1.215 yamt /*
4273 1.1 mrg * uvmspace_free: free a vmspace data structure
4274 1.1 mrg */
4275 1.1 mrg
4276 1.10 mrg void
4277 1.138 enami uvmspace_free(struct vmspace *vm)
4278 1.1 mrg {
4279 1.99 chs struct vm_map_entry *dead_entries;
4280 1.171 pk struct vm_map *map = &vm->vm_map;
4281 1.172 he int n;
4282 1.172 he
4283 1.10 mrg UVMHIST_FUNC("uvmspace_free"); UVMHIST_CALLED(maphist);
4284 1.1 mrg
4285 1.10 mrg UVMHIST_LOG(maphist,"(vm=0x%x) ref=%d", vm, vm->vm_refcnt,0,0);
4286 1.238 ad mutex_enter(&map->misc_lock);
4287 1.172 he n = --vm->vm_refcnt;
4288 1.238 ad mutex_exit(&map->misc_lock);
4289 1.171 pk if (n > 0)
4290 1.120 chs return;
4291 1.99 chs
4292 1.120 chs /*
4293 1.120 chs * at this point, there should be no other references to the map.
4294 1.120 chs * delete all of the mappings, then destroy the pmap.
4295 1.120 chs */
4296 1.99 chs
4297 1.120 chs map->flags |= VM_MAP_DYING;
4298 1.120 chs pmap_remove_all(map->pmap);
4299 1.92 thorpej #ifdef SYSVSHM
4300 1.120 chs /* Get rid of any SYSV shared memory segments. */
4301 1.120 chs if (vm->vm_shm != NULL)
4302 1.120 chs shmexit(vm);
4303 1.92 thorpej #endif
4304 1.120 chs if (map->nentries) {
4305 1.184 chs uvm_unmap_remove(map, vm_map_min(map), vm_map_max(map),
4306 1.187 yamt &dead_entries, NULL, 0);
4307 1.120 chs if (dead_entries != NULL)
4308 1.120 chs uvm_unmap_detach(dead_entries, 0);
4309 1.10 mrg }
4310 1.146 yamt KASSERT(map->nentries == 0);
4311 1.146 yamt KASSERT(map->size == 0);
4312 1.239 ad mutex_destroy(&map->misc_lock);
4313 1.239 ad mutex_destroy(&map->mutex);
4314 1.239 ad rw_destroy(&map->lock);
4315 1.255 ad cv_destroy(&map->cv);
4316 1.120 chs pmap_destroy(map->pmap);
4317 1.248 ad pool_cache_put(&uvm_vmspace_cache, vm);
4318 1.1 mrg }
4319 1.1 mrg
4320 1.1 mrg /*
4321 1.1 mrg * F O R K - m a i n e n t r y p o i n t
4322 1.1 mrg */
4323 1.1 mrg /*
4324 1.1 mrg * uvmspace_fork: fork a process' main map
4325 1.1 mrg *
4326 1.1 mrg * => create a new vmspace for child process from parent.
4327 1.1 mrg * => parent's map must not be locked.
4328 1.1 mrg */
4329 1.1 mrg
4330 1.10 mrg struct vmspace *
4331 1.138 enami uvmspace_fork(struct vmspace *vm1)
4332 1.10 mrg {
4333 1.10 mrg struct vmspace *vm2;
4334 1.99 chs struct vm_map *old_map = &vm1->vm_map;
4335 1.99 chs struct vm_map *new_map;
4336 1.99 chs struct vm_map_entry *old_entry;
4337 1.99 chs struct vm_map_entry *new_entry;
4338 1.10 mrg UVMHIST_FUNC("uvmspace_fork"); UVMHIST_CALLED(maphist);
4339 1.1 mrg
4340 1.10 mrg vm_map_lock(old_map);
4341 1.1 mrg
4342 1.184 chs vm2 = uvmspace_alloc(vm_map_min(old_map), vm_map_max(old_map));
4343 1.23 perry memcpy(&vm2->vm_startcopy, &vm1->vm_startcopy,
4344 1.235 christos (char *) (vm1 + 1) - (char *) &vm1->vm_startcopy);
4345 1.10 mrg new_map = &vm2->vm_map; /* XXX */
4346 1.10 mrg
4347 1.10 mrg old_entry = old_map->header.next;
4348 1.162 pooka new_map->size = old_map->size;
4349 1.10 mrg
4350 1.10 mrg /*
4351 1.10 mrg * go entry-by-entry
4352 1.10 mrg */
4353 1.1 mrg
4354 1.10 mrg while (old_entry != &old_map->header) {
4355 1.1 mrg
4356 1.10 mrg /*
4357 1.10 mrg * first, some sanity checks on the old entry
4358 1.10 mrg */
4359 1.99 chs
4360 1.94 chs KASSERT(!UVM_ET_ISSUBMAP(old_entry));
4361 1.94 chs KASSERT(UVM_ET_ISCOPYONWRITE(old_entry) ||
4362 1.94 chs !UVM_ET_ISNEEDSCOPY(old_entry));
4363 1.1 mrg
4364 1.10 mrg switch (old_entry->inheritance) {
4365 1.80 wiz case MAP_INHERIT_NONE:
4366 1.99 chs
4367 1.10 mrg /*
4368 1.162 pooka * drop the mapping, modify size
4369 1.10 mrg */
4370 1.162 pooka new_map->size -= old_entry->end - old_entry->start;
4371 1.10 mrg break;
4372 1.10 mrg
4373 1.80 wiz case MAP_INHERIT_SHARE:
4374 1.99 chs
4375 1.10 mrg /*
4376 1.10 mrg * share the mapping: this means we want the old and
4377 1.10 mrg * new entries to share amaps and backing objects.
4378 1.10 mrg */
4379 1.10 mrg /*
4380 1.10 mrg * if the old_entry needs a new amap (due to prev fork)
4381 1.10 mrg * then we need to allocate it now so that we have
4382 1.10 mrg * something we own to share with the new_entry. [in
4383 1.10 mrg * other words, we need to clear needs_copy]
4384 1.10 mrg */
4385 1.10 mrg
4386 1.10 mrg if (UVM_ET_ISNEEDSCOPY(old_entry)) {
4387 1.10 mrg /* get our own amap, clears needs_copy */
4388 1.212 yamt amap_copy(old_map, old_entry, AMAP_COPY_NOCHUNK,
4389 1.98 chs 0, 0);
4390 1.10 mrg /* XXXCDC: WAITOK??? */
4391 1.10 mrg }
4392 1.10 mrg
4393 1.126 bouyer new_entry = uvm_mapent_alloc(new_map, 0);
4394 1.10 mrg /* old_entry -> new_entry */
4395 1.10 mrg uvm_mapent_copy(old_entry, new_entry);
4396 1.10 mrg
4397 1.10 mrg /* new pmap has nothing wired in it */
4398 1.10 mrg new_entry->wired_count = 0;
4399 1.10 mrg
4400 1.10 mrg /*
4401 1.29 chuck * gain reference to object backing the map (can't
4402 1.29 chuck * be a submap, already checked this case).
4403 1.10 mrg */
4404 1.99 chs
4405 1.10 mrg if (new_entry->aref.ar_amap)
4406 1.85 chs uvm_map_reference_amap(new_entry, AMAP_SHARED);
4407 1.10 mrg
4408 1.10 mrg if (new_entry->object.uvm_obj &&
4409 1.10 mrg new_entry->object.uvm_obj->pgops->pgo_reference)
4410 1.10 mrg new_entry->object.uvm_obj->
4411 1.10 mrg pgops->pgo_reference(
4412 1.10 mrg new_entry->object.uvm_obj);
4413 1.10 mrg
4414 1.10 mrg /* insert entry at end of new_map's entry list */
4415 1.10 mrg uvm_map_entry_link(new_map, new_map->header.prev,
4416 1.10 mrg new_entry);
4417 1.10 mrg
4418 1.10 mrg break;
4419 1.10 mrg
4420 1.80 wiz case MAP_INHERIT_COPY:
4421 1.10 mrg
4422 1.10 mrg /*
4423 1.10 mrg * copy-on-write the mapping (using mmap's
4424 1.10 mrg * MAP_PRIVATE semantics)
4425 1.29 chuck *
4426 1.98 chs * allocate new_entry, adjust reference counts.
4427 1.29 chuck * (note that new references are read-only).
4428 1.10 mrg */
4429 1.10 mrg
4430 1.126 bouyer new_entry = uvm_mapent_alloc(new_map, 0);
4431 1.10 mrg /* old_entry -> new_entry */
4432 1.10 mrg uvm_mapent_copy(old_entry, new_entry);
4433 1.10 mrg
4434 1.10 mrg if (new_entry->aref.ar_amap)
4435 1.85 chs uvm_map_reference_amap(new_entry, 0);
4436 1.10 mrg
4437 1.10 mrg if (new_entry->object.uvm_obj &&
4438 1.10 mrg new_entry->object.uvm_obj->pgops->pgo_reference)
4439 1.10 mrg new_entry->object.uvm_obj->pgops->pgo_reference
4440 1.10 mrg (new_entry->object.uvm_obj);
4441 1.10 mrg
4442 1.10 mrg /* new pmap has nothing wired in it */
4443 1.10 mrg new_entry->wired_count = 0;
4444 1.10 mrg
4445 1.10 mrg new_entry->etype |=
4446 1.10 mrg (UVM_ET_COPYONWRITE|UVM_ET_NEEDSCOPY);
4447 1.10 mrg uvm_map_entry_link(new_map, new_map->header.prev,
4448 1.10 mrg new_entry);
4449 1.85 chs
4450 1.14 chuck /*
4451 1.10 mrg * the new entry will need an amap. it will either
4452 1.10 mrg * need to be copied from the old entry or created
4453 1.14 chuck * from scratch (if the old entry does not have an
4454 1.14 chuck * amap). can we defer this process until later
4455 1.14 chuck * (by setting "needs_copy") or do we need to copy
4456 1.14 chuck * the amap now?
4457 1.10 mrg *
4458 1.14 chuck * we must copy the amap now if any of the following
4459 1.10 mrg * conditions hold:
4460 1.14 chuck * 1. the old entry has an amap and that amap is
4461 1.14 chuck * being shared. this means that the old (parent)
4462 1.98 chs * process is sharing the amap with another
4463 1.14 chuck * process. if we do not clear needs_copy here
4464 1.14 chuck * we will end up in a situation where both the
4465 1.14 chuck * parent and child process are refering to the
4466 1.98 chs * same amap with "needs_copy" set. if the
4467 1.14 chuck * parent write-faults, the fault routine will
4468 1.14 chuck * clear "needs_copy" in the parent by allocating
4469 1.98 chs * a new amap. this is wrong because the
4470 1.14 chuck * parent is supposed to be sharing the old amap
4471 1.14 chuck * and the new amap will break that.
4472 1.10 mrg *
4473 1.14 chuck * 2. if the old entry has an amap and a non-zero
4474 1.14 chuck * wire count then we are going to have to call
4475 1.98 chs * amap_cow_now to avoid page faults in the
4476 1.14 chuck * parent process. since amap_cow_now requires
4477 1.14 chuck * "needs_copy" to be clear we might as well
4478 1.14 chuck * clear it here as well.
4479 1.10 mrg *
4480 1.10 mrg */
4481 1.10 mrg
4482 1.14 chuck if (old_entry->aref.ar_amap != NULL) {
4483 1.99 chs if ((amap_flags(old_entry->aref.ar_amap) &
4484 1.99 chs AMAP_SHARED) != 0 ||
4485 1.99 chs VM_MAPENT_ISWIRED(old_entry)) {
4486 1.99 chs
4487 1.212 yamt amap_copy(new_map, new_entry,
4488 1.212 yamt AMAP_COPY_NOCHUNK, 0, 0);
4489 1.99 chs /* XXXCDC: M_WAITOK ... ok? */
4490 1.99 chs }
4491 1.10 mrg }
4492 1.85 chs
4493 1.10 mrg /*
4494 1.14 chuck * if the parent's entry is wired down, then the
4495 1.14 chuck * parent process does not want page faults on
4496 1.14 chuck * access to that memory. this means that we
4497 1.14 chuck * cannot do copy-on-write because we can't write
4498 1.14 chuck * protect the old entry. in this case we
4499 1.14 chuck * resolve all copy-on-write faults now, using
4500 1.14 chuck * amap_cow_now. note that we have already
4501 1.14 chuck * allocated any needed amap (above).
4502 1.10 mrg */
4503 1.10 mrg
4504 1.55 thorpej if (VM_MAPENT_ISWIRED(old_entry)) {
4505 1.1 mrg
4506 1.98 chs /*
4507 1.14 chuck * resolve all copy-on-write faults now
4508 1.98 chs * (note that there is nothing to do if
4509 1.14 chuck * the old mapping does not have an amap).
4510 1.14 chuck */
4511 1.14 chuck if (old_entry->aref.ar_amap)
4512 1.14 chuck amap_cow_now(new_map, new_entry);
4513 1.14 chuck
4514 1.98 chs } else {
4515 1.14 chuck
4516 1.14 chuck /*
4517 1.14 chuck * setup mappings to trigger copy-on-write faults
4518 1.14 chuck * we must write-protect the parent if it has
4519 1.14 chuck * an amap and it is not already "needs_copy"...
4520 1.14 chuck * if it is already "needs_copy" then the parent
4521 1.14 chuck * has already been write-protected by a previous
4522 1.14 chuck * fork operation.
4523 1.14 chuck */
4524 1.14 chuck
4525 1.113 chs if (old_entry->aref.ar_amap &&
4526 1.113 chs !UVM_ET_ISNEEDSCOPY(old_entry)) {
4527 1.14 chuck if (old_entry->max_protection & VM_PROT_WRITE) {
4528 1.14 chuck pmap_protect(old_map->pmap,
4529 1.14 chuck old_entry->start,
4530 1.14 chuck old_entry->end,
4531 1.14 chuck old_entry->protection &
4532 1.14 chuck ~VM_PROT_WRITE);
4533 1.14 chuck }
4534 1.14 chuck old_entry->etype |= UVM_ET_NEEDSCOPY;
4535 1.14 chuck }
4536 1.10 mrg }
4537 1.10 mrg break;
4538 1.14 chuck } /* end of switch statement */
4539 1.10 mrg old_entry = old_entry->next;
4540 1.1 mrg }
4541 1.1 mrg
4542 1.268 ad pmap_update(old_map->pmap);
4543 1.98 chs vm_map_unlock(old_map);
4544 1.1 mrg
4545 1.1 mrg #ifdef SYSVSHM
4546 1.10 mrg if (vm1->vm_shm)
4547 1.10 mrg shmfork(vm1, vm2);
4548 1.39 thorpej #endif
4549 1.39 thorpej
4550 1.39 thorpej #ifdef PMAP_FORK
4551 1.39 thorpej pmap_fork(vm1->vm_map.pmap, vm2->vm_map.pmap);
4552 1.1 mrg #endif
4553 1.1 mrg
4554 1.10 mrg UVMHIST_LOG(maphist,"<- done",0,0,0,0);
4555 1.139 enami return (vm2);
4556 1.1 mrg }
4557 1.1 mrg
4558 1.1 mrg
4559 1.174 yamt /*
4560 1.174 yamt * in-kernel map entry allocation.
4561 1.174 yamt */
4562 1.174 yamt
4563 1.174 yamt struct uvm_kmapent_hdr {
4564 1.174 yamt LIST_ENTRY(uvm_kmapent_hdr) ukh_listq;
4565 1.174 yamt int ukh_nused;
4566 1.174 yamt struct vm_map_entry *ukh_freelist;
4567 1.174 yamt struct vm_map *ukh_map;
4568 1.178 yamt struct vm_map_entry ukh_entries[0];
4569 1.174 yamt };
4570 1.174 yamt
4571 1.174 yamt #define UVM_KMAPENT_CHUNK \
4572 1.174 yamt ((PAGE_SIZE - sizeof(struct uvm_kmapent_hdr)) \
4573 1.174 yamt / sizeof(struct vm_map_entry))
4574 1.174 yamt
4575 1.174 yamt #define UVM_KHDR_FIND(entry) \
4576 1.174 yamt ((struct uvm_kmapent_hdr *)(((vaddr_t)entry) & ~PAGE_MASK))
4577 1.174 yamt
4578 1.174 yamt
4579 1.201 dsl #ifdef DIAGNOSTIC
4580 1.203 thorpej static struct vm_map *
4581 1.174 yamt uvm_kmapent_map(struct vm_map_entry *entry)
4582 1.174 yamt {
4583 1.174 yamt const struct uvm_kmapent_hdr *ukh;
4584 1.174 yamt
4585 1.174 yamt ukh = UVM_KHDR_FIND(entry);
4586 1.174 yamt return ukh->ukh_map;
4587 1.174 yamt }
4588 1.201 dsl #endif
4589 1.174 yamt
4590 1.206 perry static inline struct vm_map_entry *
4591 1.174 yamt uvm_kmapent_get(struct uvm_kmapent_hdr *ukh)
4592 1.174 yamt {
4593 1.174 yamt struct vm_map_entry *entry;
4594 1.174 yamt
4595 1.174 yamt KASSERT(ukh->ukh_nused <= UVM_KMAPENT_CHUNK);
4596 1.174 yamt KASSERT(ukh->ukh_nused >= 0);
4597 1.174 yamt
4598 1.174 yamt entry = ukh->ukh_freelist;
4599 1.174 yamt if (entry) {
4600 1.174 yamt KASSERT((entry->flags & (UVM_MAP_KERNEL | UVM_MAP_KMAPENT))
4601 1.174 yamt == UVM_MAP_KERNEL);
4602 1.174 yamt ukh->ukh_freelist = entry->next;
4603 1.174 yamt ukh->ukh_nused++;
4604 1.174 yamt KASSERT(ukh->ukh_nused <= UVM_KMAPENT_CHUNK);
4605 1.174 yamt } else {
4606 1.174 yamt KASSERT(ukh->ukh_nused == UVM_KMAPENT_CHUNK);
4607 1.174 yamt }
4608 1.174 yamt
4609 1.174 yamt return entry;
4610 1.174 yamt }
4611 1.174 yamt
4612 1.206 perry static inline void
4613 1.174 yamt uvm_kmapent_put(struct uvm_kmapent_hdr *ukh, struct vm_map_entry *entry)
4614 1.174 yamt {
4615 1.174 yamt
4616 1.174 yamt KASSERT((entry->flags & (UVM_MAP_KERNEL | UVM_MAP_KMAPENT))
4617 1.174 yamt == UVM_MAP_KERNEL);
4618 1.174 yamt KASSERT(ukh->ukh_nused <= UVM_KMAPENT_CHUNK);
4619 1.174 yamt KASSERT(ukh->ukh_nused > 0);
4620 1.174 yamt KASSERT(ukh->ukh_freelist != NULL ||
4621 1.174 yamt ukh->ukh_nused == UVM_KMAPENT_CHUNK);
4622 1.174 yamt KASSERT(ukh->ukh_freelist == NULL ||
4623 1.174 yamt ukh->ukh_nused < UVM_KMAPENT_CHUNK);
4624 1.174 yamt
4625 1.174 yamt ukh->ukh_nused--;
4626 1.174 yamt entry->next = ukh->ukh_freelist;
4627 1.174 yamt ukh->ukh_freelist = entry;
4628 1.174 yamt }
4629 1.174 yamt
4630 1.174 yamt /*
4631 1.174 yamt * uvm_kmapent_alloc: allocate a map entry for in-kernel map
4632 1.174 yamt */
4633 1.174 yamt
4634 1.174 yamt static struct vm_map_entry *
4635 1.174 yamt uvm_kmapent_alloc(struct vm_map *map, int flags)
4636 1.174 yamt {
4637 1.174 yamt struct vm_page *pg;
4638 1.174 yamt struct uvm_map_args args;
4639 1.174 yamt struct uvm_kmapent_hdr *ukh;
4640 1.174 yamt struct vm_map_entry *entry;
4641 1.174 yamt uvm_flag_t mapflags = UVM_MAPFLAG(UVM_PROT_ALL, UVM_PROT_ALL,
4642 1.174 yamt UVM_INH_NONE, UVM_ADV_RANDOM, flags | UVM_FLAG_NOMERGE);
4643 1.174 yamt vaddr_t va;
4644 1.174 yamt int error;
4645 1.174 yamt int i;
4646 1.174 yamt
4647 1.174 yamt KDASSERT(UVM_KMAPENT_CHUNK > 2);
4648 1.174 yamt KDASSERT(kernel_map != NULL);
4649 1.174 yamt KASSERT(vm_map_pmap(map) == pmap_kernel());
4650 1.174 yamt
4651 1.208 yamt UVMMAP_EVCNT_INCR(uke_alloc);
4652 1.174 yamt entry = NULL;
4653 1.174 yamt again:
4654 1.174 yamt /*
4655 1.174 yamt * try to grab an entry from freelist.
4656 1.174 yamt */
4657 1.238 ad mutex_spin_enter(&uvm_kentry_lock);
4658 1.175 yamt ukh = LIST_FIRST(&vm_map_to_kernel(map)->vmk_kentry_free);
4659 1.174 yamt if (ukh) {
4660 1.174 yamt entry = uvm_kmapent_get(ukh);
4661 1.174 yamt if (ukh->ukh_nused == UVM_KMAPENT_CHUNK)
4662 1.174 yamt LIST_REMOVE(ukh, ukh_listq);
4663 1.174 yamt }
4664 1.238 ad mutex_spin_exit(&uvm_kentry_lock);
4665 1.174 yamt
4666 1.174 yamt if (entry)
4667 1.174 yamt return entry;
4668 1.174 yamt
4669 1.174 yamt /*
4670 1.174 yamt * there's no free entry for this vm_map.
4671 1.174 yamt * now we need to allocate some vm_map_entry.
4672 1.174 yamt * for simplicity, always allocate one page chunk of them at once.
4673 1.174 yamt */
4674 1.174 yamt
4675 1.265 ad pg = uvm_pagealloc(NULL, 0, NULL,
4676 1.265 ad (flags & UVM_KMF_NOWAIT) != 0 ? UVM_PGA_USERESERVE : 0);
4677 1.174 yamt if (__predict_false(pg == NULL)) {
4678 1.174 yamt if (flags & UVM_FLAG_NOWAIT)
4679 1.174 yamt return NULL;
4680 1.174 yamt uvm_wait("kme_alloc");
4681 1.174 yamt goto again;
4682 1.174 yamt }
4683 1.174 yamt
4684 1.242 skrll error = uvm_map_prepare(map, 0, PAGE_SIZE, NULL, UVM_UNKNOWN_OFFSET,
4685 1.242 skrll 0, mapflags, &args);
4686 1.174 yamt if (error) {
4687 1.174 yamt uvm_pagefree(pg);
4688 1.174 yamt return NULL;
4689 1.174 yamt }
4690 1.174 yamt
4691 1.174 yamt va = args.uma_start;
4692 1.174 yamt
4693 1.262 matt pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg),
4694 1.262 matt VM_PROT_READ|VM_PROT_WRITE|PMAP_KMPAGE);
4695 1.174 yamt pmap_update(vm_map_pmap(map));
4696 1.174 yamt
4697 1.174 yamt ukh = (void *)va;
4698 1.174 yamt
4699 1.174 yamt /*
4700 1.174 yamt * use the first entry for ukh itsself.
4701 1.174 yamt */
4702 1.174 yamt
4703 1.174 yamt entry = &ukh->ukh_entries[0];
4704 1.174 yamt entry->flags = UVM_MAP_KERNEL | UVM_MAP_KMAPENT;
4705 1.174 yamt error = uvm_map_enter(map, &args, entry);
4706 1.174 yamt KASSERT(error == 0);
4707 1.174 yamt
4708 1.174 yamt ukh->ukh_nused = UVM_KMAPENT_CHUNK;
4709 1.174 yamt ukh->ukh_map = map;
4710 1.174 yamt ukh->ukh_freelist = NULL;
4711 1.174 yamt for (i = UVM_KMAPENT_CHUNK - 1; i >= 2; i--) {
4712 1.199 christos struct vm_map_entry *xentry = &ukh->ukh_entries[i];
4713 1.174 yamt
4714 1.199 christos xentry->flags = UVM_MAP_KERNEL;
4715 1.199 christos uvm_kmapent_put(ukh, xentry);
4716 1.174 yamt }
4717 1.174 yamt KASSERT(ukh->ukh_nused == 2);
4718 1.174 yamt
4719 1.238 ad mutex_spin_enter(&uvm_kentry_lock);
4720 1.175 yamt LIST_INSERT_HEAD(&vm_map_to_kernel(map)->vmk_kentry_free,
4721 1.175 yamt ukh, ukh_listq);
4722 1.238 ad mutex_spin_exit(&uvm_kentry_lock);
4723 1.174 yamt
4724 1.174 yamt /*
4725 1.174 yamt * return second entry.
4726 1.174 yamt */
4727 1.174 yamt
4728 1.174 yamt entry = &ukh->ukh_entries[1];
4729 1.174 yamt entry->flags = UVM_MAP_KERNEL;
4730 1.208 yamt UVMMAP_EVCNT_INCR(ukh_alloc);
4731 1.174 yamt return entry;
4732 1.174 yamt }
4733 1.174 yamt
4734 1.174 yamt /*
4735 1.174 yamt * uvm_mapent_free: free map entry for in-kernel map
4736 1.174 yamt */
4737 1.174 yamt
4738 1.174 yamt static void
4739 1.174 yamt uvm_kmapent_free(struct vm_map_entry *entry)
4740 1.174 yamt {
4741 1.174 yamt struct uvm_kmapent_hdr *ukh;
4742 1.174 yamt struct vm_page *pg;
4743 1.174 yamt struct vm_map *map;
4744 1.174 yamt struct pmap *pmap;
4745 1.174 yamt vaddr_t va;
4746 1.174 yamt paddr_t pa;
4747 1.174 yamt struct vm_map_entry *deadentry;
4748 1.174 yamt
4749 1.208 yamt UVMMAP_EVCNT_INCR(uke_free);
4750 1.174 yamt ukh = UVM_KHDR_FIND(entry);
4751 1.174 yamt map = ukh->ukh_map;
4752 1.174 yamt
4753 1.238 ad mutex_spin_enter(&uvm_kentry_lock);
4754 1.174 yamt uvm_kmapent_put(ukh, entry);
4755 1.174 yamt if (ukh->ukh_nused > 1) {
4756 1.174 yamt if (ukh->ukh_nused == UVM_KMAPENT_CHUNK - 1)
4757 1.175 yamt LIST_INSERT_HEAD(
4758 1.175 yamt &vm_map_to_kernel(map)->vmk_kentry_free,
4759 1.175 yamt ukh, ukh_listq);
4760 1.238 ad mutex_spin_exit(&uvm_kentry_lock);
4761 1.174 yamt return;
4762 1.174 yamt }
4763 1.174 yamt
4764 1.174 yamt /*
4765 1.174 yamt * now we can free this ukh.
4766 1.174 yamt *
4767 1.174 yamt * however, keep an empty ukh to avoid ping-pong.
4768 1.174 yamt */
4769 1.174 yamt
4770 1.175 yamt if (LIST_FIRST(&vm_map_to_kernel(map)->vmk_kentry_free) == ukh &&
4771 1.174 yamt LIST_NEXT(ukh, ukh_listq) == NULL) {
4772 1.238 ad mutex_spin_exit(&uvm_kentry_lock);
4773 1.174 yamt return;
4774 1.174 yamt }
4775 1.174 yamt LIST_REMOVE(ukh, ukh_listq);
4776 1.238 ad mutex_spin_exit(&uvm_kentry_lock);
4777 1.174 yamt
4778 1.174 yamt KASSERT(ukh->ukh_nused == 1);
4779 1.174 yamt
4780 1.174 yamt /*
4781 1.174 yamt * remove map entry for ukh itsself.
4782 1.174 yamt */
4783 1.174 yamt
4784 1.174 yamt va = (vaddr_t)ukh;
4785 1.174 yamt KASSERT((va & PAGE_MASK) == 0);
4786 1.198 yamt vm_map_lock(map);
4787 1.187 yamt uvm_unmap_remove(map, va, va + PAGE_SIZE, &deadentry, NULL, 0);
4788 1.174 yamt KASSERT(deadentry->flags & UVM_MAP_KERNEL);
4789 1.174 yamt KASSERT(deadentry->flags & UVM_MAP_KMAPENT);
4790 1.174 yamt KASSERT(deadentry->next == NULL);
4791 1.174 yamt KASSERT(deadentry == &ukh->ukh_entries[0]);
4792 1.174 yamt
4793 1.174 yamt /*
4794 1.174 yamt * unmap the page from pmap and free it.
4795 1.174 yamt */
4796 1.174 yamt
4797 1.174 yamt pmap = vm_map_pmap(map);
4798 1.174 yamt KASSERT(pmap == pmap_kernel());
4799 1.174 yamt if (!pmap_extract(pmap, va, &pa))
4800 1.174 yamt panic("%s: no mapping", __func__);
4801 1.174 yamt pmap_kremove(va, PAGE_SIZE);
4802 1.251 chris pmap_update(vm_map_pmap(map));
4803 1.198 yamt vm_map_unlock(map);
4804 1.174 yamt pg = PHYS_TO_VM_PAGE(pa);
4805 1.174 yamt uvm_pagefree(pg);
4806 1.208 yamt UVMMAP_EVCNT_INCR(ukh_free);
4807 1.174 yamt }
4808 1.174 yamt
4809 1.219 yamt static vsize_t
4810 1.219 yamt uvm_kmapent_overhead(vsize_t size)
4811 1.219 yamt {
4812 1.219 yamt
4813 1.219 yamt /*
4814 1.219 yamt * - the max number of unmerged entries is howmany(size, PAGE_SIZE)
4815 1.219 yamt * as the min allocation unit is PAGE_SIZE.
4816 1.219 yamt * - UVM_KMAPENT_CHUNK "kmapent"s are allocated from a page.
4817 1.219 yamt * one of them are used to map the page itself.
4818 1.219 yamt */
4819 1.219 yamt
4820 1.219 yamt return howmany(howmany(size, PAGE_SIZE), (UVM_KMAPENT_CHUNK - 1)) *
4821 1.219 yamt PAGE_SIZE;
4822 1.219 yamt }
4823 1.219 yamt
4824 1.174 yamt /*
4825 1.174 yamt * map entry reservation
4826 1.174 yamt */
4827 1.174 yamt
4828 1.174 yamt /*
4829 1.174 yamt * uvm_mapent_reserve: reserve map entries for clipping before locking map.
4830 1.174 yamt *
4831 1.174 yamt * => needed when unmapping entries allocated without UVM_FLAG_QUANTUM.
4832 1.174 yamt * => caller shouldn't hold map locked.
4833 1.174 yamt */
4834 1.174 yamt int
4835 1.174 yamt uvm_mapent_reserve(struct vm_map *map, struct uvm_mapent_reservation *umr,
4836 1.174 yamt int nentries, int flags)
4837 1.174 yamt {
4838 1.174 yamt
4839 1.174 yamt umr->umr_nentries = 0;
4840 1.174 yamt
4841 1.179 yamt if ((flags & UVM_FLAG_QUANTUM) != 0)
4842 1.179 yamt return 0;
4843 1.179 yamt
4844 1.174 yamt if (!VM_MAP_USE_KMAPENT(map))
4845 1.174 yamt return 0;
4846 1.174 yamt
4847 1.174 yamt while (nentries--) {
4848 1.174 yamt struct vm_map_entry *ent;
4849 1.174 yamt ent = uvm_kmapent_alloc(map, flags);
4850 1.174 yamt if (!ent) {
4851 1.174 yamt uvm_mapent_unreserve(map, umr);
4852 1.174 yamt return ENOMEM;
4853 1.174 yamt }
4854 1.174 yamt UMR_PUTENTRY(umr, ent);
4855 1.174 yamt }
4856 1.174 yamt
4857 1.174 yamt return 0;
4858 1.174 yamt }
4859 1.174 yamt
4860 1.174 yamt /*
4861 1.174 yamt * uvm_mapent_unreserve:
4862 1.174 yamt *
4863 1.174 yamt * => caller shouldn't hold map locked.
4864 1.174 yamt * => never fail or sleep.
4865 1.174 yamt */
4866 1.174 yamt void
4867 1.232 yamt uvm_mapent_unreserve(struct vm_map *map, struct uvm_mapent_reservation *umr)
4868 1.174 yamt {
4869 1.174 yamt
4870 1.174 yamt while (!UMR_EMPTY(umr))
4871 1.174 yamt uvm_kmapent_free(UMR_GETENTRY(umr));
4872 1.174 yamt }
4873 1.174 yamt
4874 1.194 yamt /*
4875 1.194 yamt * uvm_mapent_trymerge: try to merge an entry with its neighbors.
4876 1.194 yamt *
4877 1.194 yamt * => called with map locked.
4878 1.194 yamt * => return non zero if successfully merged.
4879 1.194 yamt */
4880 1.194 yamt
4881 1.194 yamt int
4882 1.194 yamt uvm_mapent_trymerge(struct vm_map *map, struct vm_map_entry *entry, int flags)
4883 1.194 yamt {
4884 1.194 yamt struct uvm_object *uobj;
4885 1.194 yamt struct vm_map_entry *next;
4886 1.194 yamt struct vm_map_entry *prev;
4887 1.195 yamt vsize_t size;
4888 1.194 yamt int merged = 0;
4889 1.233 thorpej bool copying;
4890 1.194 yamt int newetype;
4891 1.194 yamt
4892 1.194 yamt if (VM_MAP_USE_KMAPENT(map)) {
4893 1.194 yamt return 0;
4894 1.194 yamt }
4895 1.194 yamt if (entry->aref.ar_amap != NULL) {
4896 1.194 yamt return 0;
4897 1.194 yamt }
4898 1.194 yamt if ((entry->flags & UVM_MAP_NOMERGE) != 0) {
4899 1.194 yamt return 0;
4900 1.194 yamt }
4901 1.194 yamt
4902 1.194 yamt uobj = entry->object.uvm_obj;
4903 1.195 yamt size = entry->end - entry->start;
4904 1.194 yamt copying = (flags & UVM_MERGE_COPYING) != 0;
4905 1.194 yamt newetype = copying ? (entry->etype & ~UVM_ET_NEEDSCOPY) : entry->etype;
4906 1.194 yamt
4907 1.194 yamt next = entry->next;
4908 1.194 yamt if (next != &map->header &&
4909 1.194 yamt next->start == entry->end &&
4910 1.194 yamt ((copying && next->aref.ar_amap != NULL &&
4911 1.194 yamt amap_refs(next->aref.ar_amap) == 1) ||
4912 1.194 yamt (!copying && next->aref.ar_amap == NULL)) &&
4913 1.194 yamt UVM_ET_ISCOMPATIBLE(next, newetype,
4914 1.194 yamt uobj, entry->flags, entry->protection,
4915 1.194 yamt entry->max_protection, entry->inheritance, entry->advice,
4916 1.195 yamt entry->wired_count) &&
4917 1.195 yamt (uobj == NULL || entry->offset + size == next->offset)) {
4918 1.194 yamt int error;
4919 1.194 yamt
4920 1.194 yamt if (copying) {
4921 1.195 yamt error = amap_extend(next, size,
4922 1.194 yamt AMAP_EXTEND_NOWAIT|AMAP_EXTEND_BACKWARDS);
4923 1.194 yamt } else {
4924 1.194 yamt error = 0;
4925 1.194 yamt }
4926 1.194 yamt if (error == 0) {
4927 1.197 yamt if (uobj) {
4928 1.197 yamt if (uobj->pgops->pgo_detach) {
4929 1.197 yamt uobj->pgops->pgo_detach(uobj);
4930 1.197 yamt }
4931 1.194 yamt }
4932 1.194 yamt
4933 1.194 yamt entry->end = next->end;
4934 1.221 yamt clear_hints(map, next);
4935 1.194 yamt uvm_map_entry_unlink(map, next);
4936 1.194 yamt if (copying) {
4937 1.194 yamt entry->aref = next->aref;
4938 1.194 yamt entry->etype &= ~UVM_ET_NEEDSCOPY;
4939 1.194 yamt }
4940 1.222 yamt uvm_map_check(map, "trymerge forwardmerge");
4941 1.194 yamt uvm_mapent_free_merged(map, next);
4942 1.194 yamt merged++;
4943 1.194 yamt }
4944 1.194 yamt }
4945 1.194 yamt
4946 1.194 yamt prev = entry->prev;
4947 1.194 yamt if (prev != &map->header &&
4948 1.194 yamt prev->end == entry->start &&
4949 1.194 yamt ((copying && !merged && prev->aref.ar_amap != NULL &&
4950 1.194 yamt amap_refs(prev->aref.ar_amap) == 1) ||
4951 1.194 yamt (!copying && prev->aref.ar_amap == NULL)) &&
4952 1.194 yamt UVM_ET_ISCOMPATIBLE(prev, newetype,
4953 1.194 yamt uobj, entry->flags, entry->protection,
4954 1.194 yamt entry->max_protection, entry->inheritance, entry->advice,
4955 1.195 yamt entry->wired_count) &&
4956 1.196 yamt (uobj == NULL ||
4957 1.196 yamt prev->offset + prev->end - prev->start == entry->offset)) {
4958 1.194 yamt int error;
4959 1.194 yamt
4960 1.194 yamt if (copying) {
4961 1.195 yamt error = amap_extend(prev, size,
4962 1.194 yamt AMAP_EXTEND_NOWAIT|AMAP_EXTEND_FORWARDS);
4963 1.194 yamt } else {
4964 1.194 yamt error = 0;
4965 1.194 yamt }
4966 1.194 yamt if (error == 0) {
4967 1.197 yamt if (uobj) {
4968 1.197 yamt if (uobj->pgops->pgo_detach) {
4969 1.197 yamt uobj->pgops->pgo_detach(uobj);
4970 1.197 yamt }
4971 1.197 yamt entry->offset = prev->offset;
4972 1.194 yamt }
4973 1.194 yamt
4974 1.194 yamt entry->start = prev->start;
4975 1.221 yamt clear_hints(map, prev);
4976 1.194 yamt uvm_map_entry_unlink(map, prev);
4977 1.194 yamt if (copying) {
4978 1.194 yamt entry->aref = prev->aref;
4979 1.194 yamt entry->etype &= ~UVM_ET_NEEDSCOPY;
4980 1.194 yamt }
4981 1.222 yamt uvm_map_check(map, "trymerge backmerge");
4982 1.194 yamt uvm_mapent_free_merged(map, prev);
4983 1.194 yamt merged++;
4984 1.194 yamt }
4985 1.194 yamt }
4986 1.194 yamt
4987 1.194 yamt return merged;
4988 1.194 yamt }
4989 1.194 yamt
4990 1.270 pooka #if defined(DDB) || defined(DEBUGPRINT)
4991 1.1 mrg
4992 1.1 mrg /*
4993 1.1 mrg * DDB hooks
4994 1.1 mrg */
4995 1.1 mrg
4996 1.1 mrg /*
4997 1.1 mrg * uvm_map_printit: actually prints the map
4998 1.1 mrg */
4999 1.1 mrg
5000 1.10 mrg void
5001 1.233 thorpej uvm_map_printit(struct vm_map *map, bool full,
5002 1.138 enami void (*pr)(const char *, ...))
5003 1.10 mrg {
5004 1.99 chs struct vm_map_entry *entry;
5005 1.10 mrg
5006 1.184 chs (*pr)("MAP %p: [0x%lx->0x%lx]\n", map, vm_map_min(map),
5007 1.184 chs vm_map_max(map));
5008 1.53 thorpej (*pr)("\t#ent=%d, sz=%d, ref=%d, version=%d, flags=0x%x\n",
5009 1.53 thorpej map->nentries, map->size, map->ref_count, map->timestamp,
5010 1.53 thorpej map->flags);
5011 1.173 yamt (*pr)("\tpmap=%p(resident=%ld, wired=%ld)\n", map->pmap,
5012 1.173 yamt pmap_resident_count(map->pmap), pmap_wired_count(map->pmap));
5013 1.10 mrg if (!full)
5014 1.10 mrg return;
5015 1.10 mrg for (entry = map->header.next; entry != &map->header;
5016 1.10 mrg entry = entry->next) {
5017 1.70 kleink (*pr)(" - %p: 0x%lx->0x%lx: obj=%p/0x%llx, amap=%p/%d\n",
5018 1.10 mrg entry, entry->start, entry->end, entry->object.uvm_obj,
5019 1.85 chs (long long)entry->offset, entry->aref.ar_amap,
5020 1.85 chs entry->aref.ar_pageoff);
5021 1.10 mrg (*pr)(
5022 1.85 chs "\tsubmap=%c, cow=%c, nc=%c, prot(max)=%d/%d, inh=%d, "
5023 1.85 chs "wc=%d, adv=%d\n",
5024 1.10 mrg (entry->etype & UVM_ET_SUBMAP) ? 'T' : 'F',
5025 1.98 chs (entry->etype & UVM_ET_COPYONWRITE) ? 'T' : 'F',
5026 1.10 mrg (entry->etype & UVM_ET_NEEDSCOPY) ? 'T' : 'F',
5027 1.10 mrg entry->protection, entry->max_protection,
5028 1.10 mrg entry->inheritance, entry->wired_count, entry->advice);
5029 1.10 mrg }
5030 1.98 chs }
5031 1.1 mrg
5032 1.1 mrg /*
5033 1.1 mrg * uvm_object_printit: actually prints the object
5034 1.1 mrg */
5035 1.1 mrg
5036 1.10 mrg void
5037 1.233 thorpej uvm_object_printit(struct uvm_object *uobj, bool full,
5038 1.138 enami void (*pr)(const char *, ...))
5039 1.10 mrg {
5040 1.10 mrg struct vm_page *pg;
5041 1.10 mrg int cnt = 0;
5042 1.10 mrg
5043 1.71 chs (*pr)("OBJECT %p: locked=%d, pgops=%p, npages=%d, ",
5044 1.248 ad uobj, mutex_owned(&uobj->vmobjlock), uobj->pgops, uobj->uo_npages);
5045 1.42 thorpej if (UVM_OBJ_IS_KERN_OBJECT(uobj))
5046 1.10 mrg (*pr)("refs=<SYSTEM>\n");
5047 1.10 mrg else
5048 1.10 mrg (*pr)("refs=%d\n", uobj->uo_refs);
5049 1.10 mrg
5050 1.71 chs if (!full) {
5051 1.71 chs return;
5052 1.71 chs }
5053 1.10 mrg (*pr)(" PAGES <pg,offset>:\n ");
5054 1.257 ad TAILQ_FOREACH(pg, &uobj->memq, listq.queue) {
5055 1.106 chs cnt++;
5056 1.85 chs (*pr)("<%p,0x%llx> ", pg, (long long)pg->offset);
5057 1.106 chs if ((cnt % 3) == 0) {
5058 1.71 chs (*pr)("\n ");
5059 1.71 chs }
5060 1.71 chs }
5061 1.106 chs if ((cnt % 3) != 0) {
5062 1.71 chs (*pr)("\n");
5063 1.10 mrg }
5064 1.98 chs }
5065 1.1 mrg
5066 1.1 mrg /*
5067 1.1 mrg * uvm_page_printit: actually print the page
5068 1.1 mrg */
5069 1.1 mrg
5070 1.228 yamt static const char page_flagbits[] = UVM_PGFLAGBITS;
5071 1.228 yamt static const char page_pqflagbits[] = UVM_PQFLAGBITS;
5072 1.86 chs
5073 1.10 mrg void
5074 1.233 thorpej uvm_page_printit(struct vm_page *pg, bool full,
5075 1.138 enami void (*pr)(const char *, ...))
5076 1.10 mrg {
5077 1.85 chs struct vm_page *tpg;
5078 1.10 mrg struct uvm_object *uobj;
5079 1.257 ad struct pgflist *pgl;
5080 1.71 chs char pgbuf[128];
5081 1.71 chs char pqbuf[128];
5082 1.1 mrg
5083 1.10 mrg (*pr)("PAGE %p:\n", pg);
5084 1.266 christos snprintb(pgbuf, sizeof(pgbuf), page_flagbits, pg->flags);
5085 1.266 christos snprintb(pqbuf, sizeof(pqbuf), page_pqflagbits, pg->pqflags);
5086 1.106 chs (*pr)(" flags=%s, pqflags=%s, wire_count=%d, pa=0x%lx\n",
5087 1.137 yamt pgbuf, pqbuf, pg->wire_count, (long)VM_PAGE_TO_PHYS(pg));
5088 1.86 chs (*pr)(" uobject=%p, uanon=%p, offset=0x%llx loan_count=%d\n",
5089 1.86 chs pg->uobject, pg->uanon, (long long)pg->offset, pg->loan_count);
5090 1.1 mrg #if defined(UVM_PAGE_TRKOWN)
5091 1.10 mrg if (pg->flags & PG_BUSY)
5092 1.10 mrg (*pr)(" owning process = %d, tag=%s\n",
5093 1.10 mrg pg->owner, pg->owner_tag);
5094 1.10 mrg else
5095 1.10 mrg (*pr)(" page not busy, no owner\n");
5096 1.1 mrg #else
5097 1.10 mrg (*pr)(" [page ownership tracking disabled]\n");
5098 1.1 mrg #endif
5099 1.1 mrg
5100 1.10 mrg if (!full)
5101 1.10 mrg return;
5102 1.10 mrg
5103 1.10 mrg /* cross-verify object/anon */
5104 1.10 mrg if ((pg->pqflags & PQ_FREE) == 0) {
5105 1.10 mrg if (pg->pqflags & PQ_ANON) {
5106 1.192 yamt if (pg->uanon == NULL || pg->uanon->an_page != pg)
5107 1.85 chs (*pr)(" >>> ANON DOES NOT POINT HERE <<< (%p)\n",
5108 1.192 yamt (pg->uanon) ? pg->uanon->an_page : NULL);
5109 1.10 mrg else
5110 1.10 mrg (*pr)(" anon backpointer is OK\n");
5111 1.10 mrg } else {
5112 1.10 mrg uobj = pg->uobject;
5113 1.10 mrg if (uobj) {
5114 1.10 mrg (*pr)(" checking object list\n");
5115 1.257 ad TAILQ_FOREACH(tpg, &uobj->memq, listq.queue) {
5116 1.85 chs if (tpg == pg) {
5117 1.85 chs break;
5118 1.85 chs }
5119 1.10 mrg }
5120 1.85 chs if (tpg)
5121 1.10 mrg (*pr)(" page found on object list\n");
5122 1.10 mrg else
5123 1.10 mrg (*pr)(" >>> PAGE NOT FOUND ON OBJECT LIST! <<<\n");
5124 1.10 mrg }
5125 1.10 mrg }
5126 1.10 mrg }
5127 1.1 mrg
5128 1.10 mrg /* cross-verify page queue */
5129 1.73 thorpej if (pg->pqflags & PQ_FREE) {
5130 1.73 thorpej int fl = uvm_page_lookup_freelist(pg);
5131 1.96 thorpej int color = VM_PGCOLOR_BUCKET(pg);
5132 1.96 thorpej pgl = &uvm.page_free[fl].pgfl_buckets[color].pgfl_queues[
5133 1.96 thorpej ((pg)->flags & PG_ZERO) ? PGFL_ZEROS : PGFL_UNKNOWN];
5134 1.139 enami } else {
5135 1.10 mrg pgl = NULL;
5136 1.85 chs }
5137 1.10 mrg
5138 1.10 mrg if (pgl) {
5139 1.10 mrg (*pr)(" checking pageq list\n");
5140 1.257 ad LIST_FOREACH(tpg, pgl, pageq.list) {
5141 1.85 chs if (tpg == pg) {
5142 1.85 chs break;
5143 1.85 chs }
5144 1.10 mrg }
5145 1.85 chs if (tpg)
5146 1.10 mrg (*pr)(" page found on pageq list\n");
5147 1.10 mrg else
5148 1.10 mrg (*pr)(" >>> PAGE NOT FOUND ON PAGEQ LIST! <<<\n");
5149 1.10 mrg }
5150 1.1 mrg }
5151 1.213 bjh21
5152 1.213 bjh21 /*
5153 1.213 bjh21 * uvm_pages_printthem - print a summary of all managed pages
5154 1.213 bjh21 */
5155 1.213 bjh21
5156 1.213 bjh21 void
5157 1.213 bjh21 uvm_page_printall(void (*pr)(const char *, ...))
5158 1.213 bjh21 {
5159 1.213 bjh21 unsigned i;
5160 1.213 bjh21 struct vm_page *pg;
5161 1.213 bjh21
5162 1.231 uwe (*pr)("%18s %4s %4s %18s %18s"
5163 1.214 bjh21 #ifdef UVM_PAGE_TRKOWN
5164 1.214 bjh21 " OWNER"
5165 1.214 bjh21 #endif
5166 1.214 bjh21 "\n", "PAGE", "FLAG", "PQ", "UOBJECT", "UANON");
5167 1.213 bjh21 for (i = 0; i < vm_nphysseg; i++) {
5168 1.213 bjh21 for (pg = vm_physmem[i].pgs; pg <= vm_physmem[i].lastpg; pg++) {
5169 1.231 uwe (*pr)("%18p %04x %04x %18p %18p",
5170 1.213 bjh21 pg, pg->flags, pg->pqflags, pg->uobject,
5171 1.213 bjh21 pg->uanon);
5172 1.214 bjh21 #ifdef UVM_PAGE_TRKOWN
5173 1.214 bjh21 if (pg->flags & PG_BUSY)
5174 1.214 bjh21 (*pr)(" %d [%s]", pg->owner, pg->owner_tag);
5175 1.214 bjh21 #endif
5176 1.214 bjh21 (*pr)("\n");
5177 1.213 bjh21 }
5178 1.213 bjh21 }
5179 1.213 bjh21 }
5180 1.213 bjh21
5181 1.1 mrg #endif
5182 1.211 yamt
5183 1.211 yamt /*
5184 1.211 yamt * uvm_map_create: create map
5185 1.211 yamt */
5186 1.211 yamt
5187 1.211 yamt struct vm_map *
5188 1.211 yamt uvm_map_create(pmap_t pmap, vaddr_t vmin, vaddr_t vmax, int flags)
5189 1.211 yamt {
5190 1.211 yamt struct vm_map *result;
5191 1.211 yamt
5192 1.267 cegger result = malloc(sizeof(struct vm_map), M_VMMAP, M_WAITOK);
5193 1.211 yamt uvm_map_setup(result, vmin, vmax, flags);
5194 1.211 yamt result->pmap = pmap;
5195 1.211 yamt return(result);
5196 1.211 yamt }
5197 1.211 yamt
5198 1.211 yamt /*
5199 1.211 yamt * uvm_map_setup: init map
5200 1.211 yamt *
5201 1.211 yamt * => map must not be in service yet.
5202 1.211 yamt */
5203 1.211 yamt
5204 1.211 yamt void
5205 1.211 yamt uvm_map_setup(struct vm_map *map, vaddr_t vmin, vaddr_t vmax, int flags)
5206 1.211 yamt {
5207 1.238 ad int ipl;
5208 1.211 yamt
5209 1.263 matt rb_tree_init(&map->rb_tree, &uvm_map_tree_ops);
5210 1.211 yamt map->header.next = map->header.prev = &map->header;
5211 1.211 yamt map->nentries = 0;
5212 1.211 yamt map->size = 0;
5213 1.211 yamt map->ref_count = 1;
5214 1.211 yamt vm_map_setmin(map, vmin);
5215 1.211 yamt vm_map_setmax(map, vmax);
5216 1.211 yamt map->flags = flags;
5217 1.211 yamt map->first_free = &map->header;
5218 1.211 yamt map->hint = &map->header;
5219 1.211 yamt map->timestamp = 0;
5220 1.238 ad map->busy = NULL;
5221 1.238 ad
5222 1.238 ad if ((flags & VM_MAP_INTRSAFE) != 0) {
5223 1.238 ad ipl = IPL_VM;
5224 1.238 ad } else {
5225 1.238 ad ipl = IPL_NONE;
5226 1.238 ad }
5227 1.238 ad
5228 1.240 ad rw_init(&map->lock);
5229 1.238 ad cv_init(&map->cv, "vm_map");
5230 1.238 ad mutex_init(&map->misc_lock, MUTEX_DRIVER, ipl);
5231 1.238 ad mutex_init(&map->mutex, MUTEX_DRIVER, ipl);
5232 1.211 yamt }
5233 1.211 yamt
5234 1.211 yamt
5235 1.211 yamt /*
5236 1.211 yamt * U N M A P - m a i n e n t r y p o i n t
5237 1.211 yamt */
5238 1.211 yamt
5239 1.211 yamt /*
5240 1.211 yamt * uvm_unmap1: remove mappings from a vm_map (from "start" up to "stop")
5241 1.211 yamt *
5242 1.211 yamt * => caller must check alignment and size
5243 1.211 yamt * => map must be unlocked (we will lock it)
5244 1.211 yamt * => flags is UVM_FLAG_QUANTUM or 0.
5245 1.211 yamt */
5246 1.211 yamt
5247 1.211 yamt void
5248 1.211 yamt uvm_unmap1(struct vm_map *map, vaddr_t start, vaddr_t end, int flags)
5249 1.211 yamt {
5250 1.211 yamt struct vm_map_entry *dead_entries;
5251 1.211 yamt struct uvm_mapent_reservation umr;
5252 1.211 yamt UVMHIST_FUNC("uvm_unmap"); UVMHIST_CALLED(maphist);
5253 1.211 yamt
5254 1.211 yamt UVMHIST_LOG(maphist, " (map=0x%x, start=0x%x, end=0x%x)",
5255 1.211 yamt map, start, end, 0);
5256 1.246 xtraeme if (map == kernel_map) {
5257 1.244 yamt LOCKDEBUG_MEM_CHECK((void *)start, end - start);
5258 1.246 xtraeme }
5259 1.211 yamt /*
5260 1.211 yamt * work now done by helper functions. wipe the pmap's and then
5261 1.211 yamt * detach from the dead entries...
5262 1.211 yamt */
5263 1.211 yamt uvm_mapent_reserve(map, &umr, 2, flags);
5264 1.211 yamt vm_map_lock(map);
5265 1.211 yamt uvm_unmap_remove(map, start, end, &dead_entries, &umr, flags);
5266 1.211 yamt vm_map_unlock(map);
5267 1.211 yamt uvm_mapent_unreserve(map, &umr);
5268 1.211 yamt
5269 1.211 yamt if (dead_entries != NULL)
5270 1.211 yamt uvm_unmap_detach(dead_entries, 0);
5271 1.211 yamt
5272 1.211 yamt UVMHIST_LOG(maphist, "<- done", 0,0,0,0);
5273 1.211 yamt }
5274 1.211 yamt
5275 1.211 yamt
5276 1.211 yamt /*
5277 1.211 yamt * uvm_map_reference: add reference to a map
5278 1.211 yamt *
5279 1.238 ad * => map need not be locked (we use misc_lock).
5280 1.211 yamt */
5281 1.211 yamt
5282 1.211 yamt void
5283 1.211 yamt uvm_map_reference(struct vm_map *map)
5284 1.211 yamt {
5285 1.238 ad mutex_enter(&map->misc_lock);
5286 1.211 yamt map->ref_count++;
5287 1.238 ad mutex_exit(&map->misc_lock);
5288 1.211 yamt }
5289 1.211 yamt
5290 1.211 yamt struct vm_map_kernel *
5291 1.211 yamt vm_map_to_kernel(struct vm_map *map)
5292 1.211 yamt {
5293 1.211 yamt
5294 1.211 yamt KASSERT(VM_MAP_IS_KERNEL(map));
5295 1.211 yamt
5296 1.211 yamt return (struct vm_map_kernel *)map;
5297 1.211 yamt }
5298 1.226 yamt
5299 1.233 thorpej bool
5300 1.226 yamt vm_map_starved_p(struct vm_map *map)
5301 1.226 yamt {
5302 1.226 yamt
5303 1.226 yamt if ((map->flags & VM_MAP_WANTVA) != 0) {
5304 1.234 thorpej return true;
5305 1.226 yamt }
5306 1.226 yamt /* XXX */
5307 1.226 yamt if ((vm_map_max(map) - vm_map_min(map)) / 16 * 15 < map->size) {
5308 1.234 thorpej return true;
5309 1.226 yamt }
5310 1.234 thorpej return false;
5311 1.226 yamt }
5312 1.247 yamt
5313 1.270 pooka #if defined(DDB) || defined(DEBUGPRINT)
5314 1.247 yamt void
5315 1.247 yamt uvm_whatis(uintptr_t addr, void (*pr)(const char *, ...))
5316 1.247 yamt {
5317 1.247 yamt struct vm_map *map;
5318 1.247 yamt
5319 1.247 yamt for (map = kernel_map;;) {
5320 1.247 yamt struct vm_map_entry *entry;
5321 1.247 yamt
5322 1.247 yamt if (!uvm_map_lookup_entry_bytree(map, (vaddr_t)addr, &entry)) {
5323 1.247 yamt break;
5324 1.247 yamt }
5325 1.247 yamt (*pr)("%p is %p+%zu from VMMAP %p\n",
5326 1.247 yamt (void *)addr, (void *)entry->start,
5327 1.247 yamt (size_t)(addr - (uintptr_t)entry->start), map);
5328 1.247 yamt if (!UVM_ET_ISSUBMAP(entry)) {
5329 1.247 yamt break;
5330 1.247 yamt }
5331 1.247 yamt map = entry->object.sub_map;
5332 1.247 yamt }
5333 1.247 yamt }
5334 1.270 pooka #endif /* defined(DDB) || defined(DEBUGPRINT) */
5335