uvm_pglist.c revision 1.14 1 1.14 thorpej /* $NetBSD: uvm_pglist.c,v 1.14 2001/04/29 04:23:21 thorpej Exp $ */
2 1.2 thorpej
3 1.1 mrg /*-
4 1.1 mrg * Copyright (c) 1997 The NetBSD Foundation, Inc.
5 1.1 mrg * All rights reserved.
6 1.1 mrg *
7 1.1 mrg * This code is derived from software contributed to The NetBSD Foundation
8 1.1 mrg * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
9 1.1 mrg * NASA Ames Research Center.
10 1.1 mrg *
11 1.1 mrg * Redistribution and use in source and binary forms, with or without
12 1.1 mrg * modification, are permitted provided that the following conditions
13 1.1 mrg * are met:
14 1.1 mrg * 1. Redistributions of source code must retain the above copyright
15 1.1 mrg * notice, this list of conditions and the following disclaimer.
16 1.1 mrg * 2. Redistributions in binary form must reproduce the above copyright
17 1.1 mrg * notice, this list of conditions and the following disclaimer in the
18 1.1 mrg * documentation and/or other materials provided with the distribution.
19 1.1 mrg * 3. All advertising materials mentioning features or use of this software
20 1.1 mrg * must display the following acknowledgement:
21 1.1 mrg * This product includes software developed by the NetBSD
22 1.1 mrg * Foundation, Inc. and its contributors.
23 1.1 mrg * 4. Neither the name of The NetBSD Foundation nor the names of its
24 1.1 mrg * contributors may be used to endorse or promote products derived
25 1.1 mrg * from this software without specific prior written permission.
26 1.1 mrg *
27 1.1 mrg * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
28 1.1 mrg * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
29 1.1 mrg * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
30 1.1 mrg * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
31 1.1 mrg * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
32 1.1 mrg * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
33 1.1 mrg * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
34 1.1 mrg * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
35 1.1 mrg * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
36 1.1 mrg * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
37 1.1 mrg * POSSIBILITY OF SUCH DAMAGE.
38 1.1 mrg */
39 1.1 mrg
40 1.1 mrg /*
41 1.1 mrg * uvm_pglist.c: pglist functions
42 1.1 mrg */
43 1.1 mrg
44 1.1 mrg #include <sys/param.h>
45 1.1 mrg #include <sys/systm.h>
46 1.1 mrg #include <sys/malloc.h>
47 1.1 mrg #include <sys/proc.h>
48 1.1 mrg
49 1.1 mrg #include <uvm/uvm.h>
50 1.1 mrg
51 1.1 mrg #ifdef VM_PAGE_ALLOC_MEMORY_STATS
52 1.1 mrg #define STAT_INCR(v) (v)++
53 1.1 mrg #define STAT_DECR(v) do { \
54 1.1 mrg if ((v) == 0) \
55 1.1 mrg printf("%s:%d -- Already 0!\n", __FILE__, __LINE__); \
56 1.1 mrg else \
57 1.1 mrg (v)--; \
58 1.1 mrg } while (0)
59 1.1 mrg u_long uvm_pglistalloc_npages;
60 1.1 mrg #else
61 1.1 mrg #define STAT_INCR(v)
62 1.1 mrg #define STAT_DECR(v)
63 1.1 mrg #endif
64 1.1 mrg
65 1.1 mrg /*
66 1.1 mrg * uvm_pglistalloc: allocate a list of pages
67 1.1 mrg *
68 1.1 mrg * => allocated pages are placed at the tail of rlist. rlist is
69 1.1 mrg * assumed to be properly initialized by caller.
70 1.1 mrg * => returns 0 on success or errno on failure
71 1.1 mrg * => XXX: implementation allocates only a single segment, also
72 1.1 mrg * might be able to better advantage of vm_physeg[].
73 1.1 mrg * => doesn't take into account clean non-busy pages on inactive list
74 1.1 mrg * that could be used(?)
75 1.1 mrg * => params:
76 1.1 mrg * size the size of the allocation, rounded to page size.
77 1.1 mrg * low the low address of the allowed allocation range.
78 1.1 mrg * high the high address of the allowed allocation range.
79 1.1 mrg * alignment memory must be aligned to this power-of-two boundary.
80 1.1 mrg * boundary no segment in the allocation may cross this
81 1.1 mrg * power-of-two boundary (relative to zero).
82 1.1 mrg */
83 1.1 mrg
84 1.3 mrg int
85 1.3 mrg uvm_pglistalloc(size, low, high, alignment, boundary, rlist, nsegs, waitok)
86 1.6 eeh psize_t size;
87 1.6 eeh paddr_t low, high, alignment, boundary;
88 1.3 mrg struct pglist *rlist;
89 1.3 mrg int nsegs, waitok;
90 1.1 mrg {
91 1.6 eeh paddr_t try, idxpa, lastidxpa;
92 1.3 mrg int psi;
93 1.3 mrg struct vm_page *pgs;
94 1.14 thorpej int s, tryidx, idx, pgflidx, end, error, free_list, color;
95 1.3 mrg vm_page_t m;
96 1.3 mrg u_long pagemask;
97 1.1 mrg #ifdef DEBUG
98 1.3 mrg vm_page_t tp;
99 1.1 mrg #endif
100 1.1 mrg
101 1.13 chs KASSERT((alignment & (alignment - 1)) == 0);
102 1.13 chs KASSERT((boundary & (boundary - 1)) == 0);
103 1.3 mrg
104 1.1 mrg /*
105 1.3 mrg * Our allocations are always page granularity, so our alignment
106 1.3 mrg * must be, too.
107 1.1 mrg */
108 1.3 mrg if (alignment < PAGE_SIZE)
109 1.3 mrg alignment = PAGE_SIZE;
110 1.3 mrg
111 1.3 mrg size = round_page(size);
112 1.3 mrg try = roundup(low, alignment);
113 1.1 mrg
114 1.3 mrg if (boundary != 0 && boundary < size)
115 1.3 mrg return (EINVAL);
116 1.1 mrg
117 1.3 mrg pagemask = ~(boundary - 1);
118 1.1 mrg
119 1.3 mrg /* Default to "lose". */
120 1.3 mrg error = ENOMEM;
121 1.3 mrg
122 1.3 mrg /*
123 1.3 mrg * Block all memory allocation and lock the free list.
124 1.3 mrg */
125 1.12 chs s = uvm_lock_fpageq();
126 1.3 mrg
127 1.3 mrg /* Are there even any free pages? */
128 1.12 chs if (uvmexp.free <= (uvmexp.reserve_pagedaemon + uvmexp.reserve_kernel))
129 1.3 mrg goto out;
130 1.3 mrg
131 1.3 mrg for (;; try += alignment) {
132 1.3 mrg if (try + size > high) {
133 1.12 chs
134 1.3 mrg /*
135 1.3 mrg * We've run past the allowable range.
136 1.3 mrg */
137 1.12 chs
138 1.3 mrg goto out;
139 1.3 mrg }
140 1.3 mrg
141 1.3 mrg /*
142 1.3 mrg * Make sure this is a managed physical page.
143 1.3 mrg */
144 1.3 mrg
145 1.3 mrg if ((psi = vm_physseg_find(atop(try), &idx)) == -1)
146 1.3 mrg continue; /* managed? */
147 1.3 mrg if (vm_physseg_find(atop(try + size), NULL) != psi)
148 1.3 mrg continue; /* end must be in this segment */
149 1.3 mrg
150 1.3 mrg tryidx = idx;
151 1.3 mrg end = idx + (size / PAGE_SIZE);
152 1.3 mrg pgs = vm_physmem[psi].pgs;
153 1.3 mrg
154 1.3 mrg /*
155 1.3 mrg * Found a suitable starting page. See of the range is free.
156 1.3 mrg */
157 1.12 chs
158 1.3 mrg for (; idx < end; idx++) {
159 1.3 mrg if (VM_PAGE_IS_FREE(&pgs[idx]) == 0) {
160 1.3 mrg break;
161 1.3 mrg }
162 1.3 mrg idxpa = VM_PAGE_TO_PHYS(&pgs[idx]);
163 1.3 mrg if (idx > tryidx) {
164 1.3 mrg lastidxpa = VM_PAGE_TO_PHYS(&pgs[idx - 1]);
165 1.12 chs if ((lastidxpa + PAGE_SIZE) != idxpa) {
166 1.3 mrg
167 1.3 mrg /*
168 1.3 mrg * Region not contiguous.
169 1.3 mrg */
170 1.12 chs
171 1.3 mrg break;
172 1.3 mrg }
173 1.3 mrg if (boundary != 0 &&
174 1.3 mrg ((lastidxpa ^ idxpa) & pagemask) != 0) {
175 1.12 chs
176 1.3 mrg /*
177 1.3 mrg * Region crosses boundary.
178 1.3 mrg */
179 1.12 chs
180 1.3 mrg break;
181 1.3 mrg }
182 1.3 mrg }
183 1.3 mrg }
184 1.3 mrg if (idx == end) {
185 1.3 mrg break;
186 1.3 mrg }
187 1.1 mrg }
188 1.1 mrg
189 1.9 thorpej #if PGFL_NQUEUES != 2
190 1.9 thorpej #error uvm_pglistalloc needs to be updated
191 1.9 thorpej #endif
192 1.9 thorpej
193 1.3 mrg /*
194 1.3 mrg * we have a chunk of memory that conforms to the requested constraints.
195 1.3 mrg */
196 1.3 mrg idx = tryidx;
197 1.3 mrg while (idx < end) {
198 1.3 mrg m = &pgs[idx];
199 1.5 thorpej free_list = uvm_page_lookup_freelist(m);
200 1.14 thorpej color = VM_PGCOLOR_BUCKET(m);
201 1.9 thorpej pgflidx = (m->flags & PG_ZERO) ? PGFL_ZEROS : PGFL_UNKNOWN;
202 1.1 mrg #ifdef DEBUG
203 1.9 thorpej for (tp = TAILQ_FIRST(&uvm.page_free[
204 1.14 thorpej free_list].pgfl_buckets[color].pgfl_queues[pgflidx]);
205 1.9 thorpej tp != NULL;
206 1.9 thorpej tp = TAILQ_NEXT(tp, pageq)) {
207 1.3 mrg if (tp == m)
208 1.3 mrg break;
209 1.3 mrg }
210 1.3 mrg if (tp == NULL)
211 1.3 mrg panic("uvm_pglistalloc: page not on freelist");
212 1.1 mrg #endif
213 1.14 thorpej TAILQ_REMOVE(&uvm.page_free[free_list].pgfl_buckets[
214 1.14 thorpej color].pgfl_queues[pgflidx], m, pageq);
215 1.3 mrg uvmexp.free--;
216 1.9 thorpej if (m->flags & PG_ZERO)
217 1.9 thorpej uvmexp.zeropages--;
218 1.3 mrg m->flags = PG_CLEAN;
219 1.3 mrg m->pqflags = 0;
220 1.3 mrg m->uobject = NULL;
221 1.3 mrg m->uanon = NULL;
222 1.12 chs m->version++;
223 1.3 mrg TAILQ_INSERT_TAIL(rlist, m, pageq);
224 1.3 mrg idx++;
225 1.3 mrg STAT_INCR(uvm_pglistalloc_npages);
226 1.3 mrg }
227 1.3 mrg error = 0;
228 1.1 mrg
229 1.1 mrg out:
230 1.3 mrg /*
231 1.3 mrg * check to see if we need to generate some free pages waking
232 1.3 mrg * the pagedaemon.
233 1.3 mrg */
234 1.3 mrg
235 1.12 chs if (uvmexp.free + uvmexp.paging < uvmexp.freemin ||
236 1.12 chs (uvmexp.free + uvmexp.paging < uvmexp.freetarg &&
237 1.12 chs uvmexp.inactive < uvmexp.inactarg)) {
238 1.8 thorpej wakeup(&uvm.pagedaemon);
239 1.12 chs }
240 1.12 chs
241 1.12 chs uvm_unlock_fpageq(s);
242 1.1 mrg
243 1.3 mrg return (error);
244 1.1 mrg }
245 1.1 mrg
246 1.1 mrg /*
247 1.1 mrg * uvm_pglistfree: free a list of pages
248 1.1 mrg *
249 1.1 mrg * => pages should already be unmapped
250 1.1 mrg */
251 1.1 mrg
252 1.3 mrg void
253 1.3 mrg uvm_pglistfree(list)
254 1.3 mrg struct pglist *list;
255 1.1 mrg {
256 1.3 mrg vm_page_t m;
257 1.3 mrg int s;
258 1.1 mrg
259 1.3 mrg /*
260 1.3 mrg * Block all memory allocation and lock the free list.
261 1.3 mrg */
262 1.7 thorpej s = uvm_lock_fpageq();
263 1.1 mrg
264 1.13 chs while ((m = TAILQ_FIRST(list)) != NULL) {
265 1.13 chs KASSERT((m->pqflags & (PQ_ACTIVE|PQ_INACTIVE)) == 0);
266 1.3 mrg TAILQ_REMOVE(list, m, pageq);
267 1.3 mrg m->pqflags = PQ_FREE;
268 1.9 thorpej TAILQ_INSERT_TAIL(&uvm.page_free[
269 1.14 thorpej uvm_page_lookup_freelist(m)].pgfl_buckets[
270 1.14 thorpej VM_PGCOLOR_BUCKET(m)].pgfl_queues[PGFL_UNKNOWN], m, pageq);
271 1.3 mrg uvmexp.free++;
272 1.9 thorpej if (uvmexp.zeropages < UVM_PAGEZERO_TARGET)
273 1.9 thorpej uvm.page_idle_zero = vm_page_zero_enable;
274 1.3 mrg STAT_DECR(uvm_pglistalloc_npages);
275 1.3 mrg }
276 1.1 mrg
277 1.7 thorpej uvm_unlock_fpageq(s);
278 1.1 mrg }
279