pmap.c revision 1.38
1 /* $NetBSD: pmap.c,v 1.38 2006/08/05 21:26:49 sanjayl Exp $ */ 2 /*- 3 * Copyright (c) 2001 The NetBSD Foundation, Inc. 4 * All rights reserved. 5 * 6 * This code is derived from software contributed to The NetBSD Foundation 7 * by Matt Thomas <matt (at) 3am-software.com> of Allegro Networks, Inc. 8 * 9 * Support for PPC64 Bridge mode added by Sanjay Lal <sanjayl (at) kymasys.com> 10 * of Kyma Systems LLC. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. All advertising materials mentioning features or use of this software 21 * must display the following acknowledgement: 22 * This product includes software developed by the NetBSD 23 * Foundation, Inc. and its contributors. 24 * 4. Neither the name of The NetBSD Foundation nor the names of its 25 * contributors may be used to endorse or promote products derived 26 * from this software without specific prior written permission. 27 * 28 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 29 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 30 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 31 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 32 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 33 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 34 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 35 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 36 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 37 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 38 * POSSIBILITY OF SUCH DAMAGE. 39 */ 40 41 /* 42 * Copyright (C) 1995, 1996 Wolfgang Solfrank. 43 * Copyright (C) 1995, 1996 TooLs GmbH. 44 * All rights reserved. 45 * 46 * Redistribution and use in source and binary forms, with or without 47 * modification, are permitted provided that the following conditions 48 * are met: 49 * 1. Redistributions of source code must retain the above copyright 50 * notice, this list of conditions and the following disclaimer. 51 * 2. Redistributions in binary form must reproduce the above copyright 52 * notice, this list of conditions and the following disclaimer in the 53 * documentation and/or other materials provided with the distribution. 54 * 3. All advertising materials mentioning features or use of this software 55 * must display the following acknowledgement: 56 * This product includes software developed by TooLs GmbH. 57 * 4. The name of TooLs GmbH may not be used to endorse or promote products 58 * derived from this software without specific prior written permission. 59 * 60 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR 61 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 62 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 63 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 64 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 65 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 66 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 67 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 68 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 69 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 70 */ 71 72 #include <sys/cdefs.h> 73 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.38 2006/08/05 21:26:49 sanjayl Exp $"); 74 75 #include "opt_ppcarch.h" 76 #include "opt_altivec.h" 77 #include "opt_pmap.h" 78 #include <sys/param.h> 79 #include <sys/malloc.h> 80 #include <sys/proc.h> 81 #include <sys/user.h> 82 #include <sys/pool.h> 83 #include <sys/queue.h> 84 #include <sys/device.h> /* for evcnt */ 85 #include <sys/systm.h> 86 87 #if __NetBSD_Version__ < 105010000 88 #include <vm/vm.h> 89 #include <vm/vm_kern.h> 90 #define splvm() splimp() 91 #endif 92 93 #include <uvm/uvm.h> 94 95 #include <machine/pcb.h> 96 #include <machine/powerpc.h> 97 #include <powerpc/spr.h> 98 #include <powerpc/oea/sr_601.h> 99 #include <powerpc/bat.h> 100 #include <powerpc/stdarg.h> 101 102 #if defined(DEBUG) || defined(PMAPCHECK) 103 #define STATIC 104 #else 105 #define STATIC static 106 #endif 107 108 #ifdef ALTIVEC 109 int pmap_use_altivec; 110 #endif 111 112 volatile struct pteg *pmap_pteg_table; 113 unsigned int pmap_pteg_cnt; 114 unsigned int pmap_pteg_mask; 115 #ifdef PMAP_MEMLIMIT 116 paddr_t pmap_memlimit = PMAP_MEMLIMIT; 117 #else 118 paddr_t pmap_memlimit = -PAGE_SIZE; /* there is no limit */ 119 #endif 120 121 struct pmap kernel_pmap_; 122 unsigned int pmap_pages_stolen; 123 u_long pmap_pte_valid; 124 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 125 u_long pmap_pvo_enter_depth; 126 u_long pmap_pvo_remove_depth; 127 #endif 128 129 int physmem; 130 #ifndef MSGBUFADDR 131 extern paddr_t msgbuf_paddr; 132 #endif 133 134 static struct mem_region *mem, *avail; 135 static u_int mem_cnt, avail_cnt; 136 137 #ifdef __HAVE_PMAP_PHYSSEG 138 /* 139 * This is a cache of referenced/modified bits. 140 * Bits herein are shifted by ATTRSHFT. 141 */ 142 #define ATTR_SHFT 4 143 struct pmap_physseg pmap_physseg; 144 #endif 145 146 /* 147 * The following structure is aligned to 32 bytes 148 */ 149 struct pvo_entry { 150 LIST_ENTRY(pvo_entry) pvo_vlink; /* Link to common virt page */ 151 TAILQ_ENTRY(pvo_entry) pvo_olink; /* Link to overflow entry */ 152 struct pte pvo_pte; /* Prebuilt PTE */ 153 pmap_t pvo_pmap; /* ptr to owning pmap */ 154 vaddr_t pvo_vaddr; /* VA of entry */ 155 #define PVO_PTEGIDX_MASK 0x0007 /* which PTEG slot */ 156 #define PVO_PTEGIDX_VALID 0x0008 /* slot is valid */ 157 #define PVO_WIRED 0x0010 /* PVO entry is wired */ 158 #define PVO_MANAGED 0x0020 /* PVO e. for managed page */ 159 #define PVO_EXECUTABLE 0x0040 /* PVO e. for executable page */ 160 #define PVO_ENTER_INSERT 0 /* PVO has been removed */ 161 #define PVO_SPILL_UNSET 1 /* PVO has been evicted */ 162 #define PVO_SPILL_SET 2 /* PVO has been spilled */ 163 #define PVO_SPILL_INSERT 3 /* PVO has been inserted */ 164 #define PVO_PMAP_PAGE_PROTECT 4 /* PVO has changed */ 165 #define PVO_PMAP_PROTECT 5 /* PVO has changed */ 166 #define PVO_REMOVE 6 /* PVO has been removed */ 167 #define PVO_WHERE_MASK 15 168 #define PVO_WHERE_SHFT 8 169 } __attribute__ ((aligned (32))); 170 #define PVO_VADDR(pvo) ((pvo)->pvo_vaddr & ~ADDR_POFF) 171 #define PVO_ISEXECUTABLE(pvo) ((pvo)->pvo_vaddr & PVO_EXECUTABLE) 172 #define PVO_PTEGIDX_GET(pvo) ((pvo)->pvo_vaddr & PVO_PTEGIDX_MASK) 173 #define PVO_PTEGIDX_ISSET(pvo) ((pvo)->pvo_vaddr & PVO_PTEGIDX_VALID) 174 #define PVO_PTEGIDX_CLR(pvo) \ 175 ((void)((pvo)->pvo_vaddr &= ~(PVO_PTEGIDX_VALID|PVO_PTEGIDX_MASK))) 176 #define PVO_PTEGIDX_SET(pvo,i) \ 177 ((void)((pvo)->pvo_vaddr |= (i)|PVO_PTEGIDX_VALID)) 178 #define PVO_WHERE(pvo,w) \ 179 ((pvo)->pvo_vaddr &= ~(PVO_WHERE_MASK << PVO_WHERE_SHFT), \ 180 (pvo)->pvo_vaddr |= ((PVO_ ## w) << PVO_WHERE_SHFT)) 181 182 TAILQ_HEAD(pvo_tqhead, pvo_entry); 183 struct pvo_tqhead *pmap_pvo_table; /* pvo entries by ptegroup index */ 184 struct pvo_head pmap_pvo_kunmanaged = LIST_HEAD_INITIALIZER(pmap_pvo_kunmanaged); /* list of unmanaged pages */ 185 struct pvo_head pmap_pvo_unmanaged = LIST_HEAD_INITIALIZER(pmap_pvo_unmanaged); /* list of unmanaged pages */ 186 187 struct pool pmap_pool; /* pool for pmap structures */ 188 struct pool pmap_upvo_pool; /* pool for pvo entries for unmanaged pages */ 189 struct pool pmap_mpvo_pool; /* pool for pvo entries for managed pages */ 190 191 /* 192 * We keep a cache of unmanaged pages to be used for pvo entries for 193 * unmanaged pages. 194 */ 195 struct pvo_page { 196 SIMPLEQ_ENTRY(pvo_page) pvop_link; 197 }; 198 SIMPLEQ_HEAD(pvop_head, pvo_page); 199 struct pvop_head pmap_upvop_head = SIMPLEQ_HEAD_INITIALIZER(pmap_upvop_head); 200 struct pvop_head pmap_mpvop_head = SIMPLEQ_HEAD_INITIALIZER(pmap_mpvop_head); 201 u_long pmap_upvop_free; 202 u_long pmap_upvop_maxfree; 203 u_long pmap_mpvop_free; 204 u_long pmap_mpvop_maxfree; 205 206 STATIC void *pmap_pool_ualloc(struct pool *, int); 207 STATIC void *pmap_pool_malloc(struct pool *, int); 208 209 STATIC void pmap_pool_ufree(struct pool *, void *); 210 STATIC void pmap_pool_mfree(struct pool *, void *); 211 212 static struct pool_allocator pmap_pool_mallocator = { 213 pmap_pool_malloc, pmap_pool_mfree, 0, 214 }; 215 216 static struct pool_allocator pmap_pool_uallocator = { 217 pmap_pool_ualloc, pmap_pool_ufree, 0, 218 }; 219 220 #if defined(DEBUG) || defined(PMAPCHECK) || defined(DDB) 221 void pmap_pte_print(volatile struct pte *); 222 void pmap_pteg_check(void); 223 void pmap_pteg_dist(void); 224 void pmap_print_pte(pmap_t, vaddr_t); 225 void pmap_print_mmuregs(void); 226 #endif 227 228 #if defined(DEBUG) || defined(PMAPCHECK) 229 #ifdef PMAPCHECK 230 int pmapcheck = 1; 231 #else 232 int pmapcheck = 0; 233 #endif 234 void pmap_pvo_verify(void); 235 STATIC void pmap_pvo_check(const struct pvo_entry *); 236 #define PMAP_PVO_CHECK(pvo) \ 237 do { \ 238 if (pmapcheck) \ 239 pmap_pvo_check(pvo); \ 240 } while (0) 241 #else 242 #define PMAP_PVO_CHECK(pvo) do { } while (/*CONSTCOND*/0) 243 #endif 244 STATIC int pmap_pte_insert(int, struct pte *); 245 STATIC int pmap_pvo_enter(pmap_t, struct pool *, struct pvo_head *, 246 vaddr_t, paddr_t, register_t, int); 247 STATIC void pmap_pvo_remove(struct pvo_entry *, int, struct pvo_head *); 248 STATIC void pmap_pvo_free(struct pvo_entry *); 249 STATIC void pmap_pvo_free_list(struct pvo_head *); 250 STATIC struct pvo_entry *pmap_pvo_find_va(pmap_t, vaddr_t, int *); 251 STATIC volatile struct pte *pmap_pvo_to_pte(const struct pvo_entry *, int); 252 STATIC struct pvo_entry *pmap_pvo_reclaim(struct pmap *); 253 STATIC void pvo_set_exec(struct pvo_entry *); 254 STATIC void pvo_clear_exec(struct pvo_entry *); 255 256 STATIC void tlbia(void); 257 258 STATIC void pmap_release(pmap_t); 259 STATIC void *pmap_boot_find_memory(psize_t, psize_t, int); 260 261 static uint32_t pmap_pvo_reclaim_nextidx; 262 #ifdef DEBUG 263 static int pmap_pvo_reclaim_debugctr; 264 #endif 265 266 #define VSID_NBPW (sizeof(uint32_t) * 8) 267 static uint32_t pmap_vsid_bitmap[NPMAPS / VSID_NBPW]; 268 269 static int pmap_initialized; 270 271 #if defined(DEBUG) || defined(PMAPDEBUG) 272 #define PMAPDEBUG_BOOT 0x0001 273 #define PMAPDEBUG_PTE 0x0002 274 #define PMAPDEBUG_EXEC 0x0008 275 #define PMAPDEBUG_PVOENTER 0x0010 276 #define PMAPDEBUG_PVOREMOVE 0x0020 277 #define PMAPDEBUG_ACTIVATE 0x0100 278 #define PMAPDEBUG_CREATE 0x0200 279 #define PMAPDEBUG_ENTER 0x1000 280 #define PMAPDEBUG_KENTER 0x2000 281 #define PMAPDEBUG_KREMOVE 0x4000 282 #define PMAPDEBUG_REMOVE 0x8000 283 284 unsigned int pmapdebug = 0; 285 286 # define DPRINTF(x) printf x 287 # define DPRINTFN(n, x) if (pmapdebug & PMAPDEBUG_ ## n) printf x 288 #else 289 # define DPRINTF(x) 290 # define DPRINTFN(n, x) 291 #endif 292 293 294 #ifdef PMAPCOUNTERS 295 #define PMAPCOUNT(ev) ((pmap_evcnt_ ## ev).ev_count++) 296 #define PMAPCOUNT2(ev) ((ev).ev_count++) 297 298 struct evcnt pmap_evcnt_mappings = 299 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 300 "pmap", "pages mapped"); 301 struct evcnt pmap_evcnt_unmappings = 302 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_mappings, 303 "pmap", "pages unmapped"); 304 305 struct evcnt pmap_evcnt_kernel_mappings = 306 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 307 "pmap", "kernel pages mapped"); 308 struct evcnt pmap_evcnt_kernel_unmappings = 309 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_kernel_mappings, 310 "pmap", "kernel pages unmapped"); 311 312 struct evcnt pmap_evcnt_mappings_replaced = 313 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 314 "pmap", "page mappings replaced"); 315 316 struct evcnt pmap_evcnt_exec_mappings = 317 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_mappings, 318 "pmap", "exec pages mapped"); 319 struct evcnt pmap_evcnt_exec_cached = 320 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_mappings, 321 "pmap", "exec pages cached"); 322 323 struct evcnt pmap_evcnt_exec_synced = 324 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_exec_mappings, 325 "pmap", "exec pages synced"); 326 struct evcnt pmap_evcnt_exec_synced_clear_modify = 327 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_exec_mappings, 328 "pmap", "exec pages synced (CM)"); 329 struct evcnt pmap_evcnt_exec_synced_pvo_remove = 330 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_exec_mappings, 331 "pmap", "exec pages synced (PR)"); 332 333 struct evcnt pmap_evcnt_exec_uncached_page_protect = 334 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_exec_mappings, 335 "pmap", "exec pages uncached (PP)"); 336 struct evcnt pmap_evcnt_exec_uncached_clear_modify = 337 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_exec_mappings, 338 "pmap", "exec pages uncached (CM)"); 339 struct evcnt pmap_evcnt_exec_uncached_zero_page = 340 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_exec_mappings, 341 "pmap", "exec pages uncached (ZP)"); 342 struct evcnt pmap_evcnt_exec_uncached_copy_page = 343 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_exec_mappings, 344 "pmap", "exec pages uncached (CP)"); 345 struct evcnt pmap_evcnt_exec_uncached_pvo_remove = 346 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_exec_mappings, 347 "pmap", "exec pages uncached (PR)"); 348 349 struct evcnt pmap_evcnt_updates = 350 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 351 "pmap", "updates"); 352 struct evcnt pmap_evcnt_collects = 353 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 354 "pmap", "collects"); 355 struct evcnt pmap_evcnt_copies = 356 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 357 "pmap", "copies"); 358 359 struct evcnt pmap_evcnt_ptes_spilled = 360 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 361 "pmap", "ptes spilled from overflow"); 362 struct evcnt pmap_evcnt_ptes_unspilled = 363 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 364 "pmap", "ptes not spilled"); 365 struct evcnt pmap_evcnt_ptes_evicted = 366 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 367 "pmap", "ptes evicted"); 368 369 struct evcnt pmap_evcnt_ptes_primary[8] = { 370 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 371 "pmap", "ptes added at primary[0]"), 372 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 373 "pmap", "ptes added at primary[1]"), 374 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 375 "pmap", "ptes added at primary[2]"), 376 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 377 "pmap", "ptes added at primary[3]"), 378 379 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 380 "pmap", "ptes added at primary[4]"), 381 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 382 "pmap", "ptes added at primary[5]"), 383 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 384 "pmap", "ptes added at primary[6]"), 385 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 386 "pmap", "ptes added at primary[7]"), 387 }; 388 struct evcnt pmap_evcnt_ptes_secondary[8] = { 389 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 390 "pmap", "ptes added at secondary[0]"), 391 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 392 "pmap", "ptes added at secondary[1]"), 393 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 394 "pmap", "ptes added at secondary[2]"), 395 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 396 "pmap", "ptes added at secondary[3]"), 397 398 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 399 "pmap", "ptes added at secondary[4]"), 400 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 401 "pmap", "ptes added at secondary[5]"), 402 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 403 "pmap", "ptes added at secondary[6]"), 404 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 405 "pmap", "ptes added at secondary[7]"), 406 }; 407 struct evcnt pmap_evcnt_ptes_removed = 408 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 409 "pmap", "ptes removed"); 410 struct evcnt pmap_evcnt_ptes_changed = 411 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 412 "pmap", "ptes changed"); 413 struct evcnt pmap_evcnt_pvos_reclaimed = 414 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 415 "pmap", "pvos reclaimed"); 416 struct evcnt pmap_evcnt_pvos_failed = 417 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 418 "pmap", "pvo allocation failures"); 419 420 /* 421 * From pmap_subr.c 422 */ 423 extern struct evcnt pmap_evcnt_zeroed_pages; 424 extern struct evcnt pmap_evcnt_copied_pages; 425 extern struct evcnt pmap_evcnt_idlezeroed_pages; 426 427 EVCNT_ATTACH_STATIC(pmap_evcnt_mappings); 428 EVCNT_ATTACH_STATIC(pmap_evcnt_mappings_replaced); 429 EVCNT_ATTACH_STATIC(pmap_evcnt_unmappings); 430 431 EVCNT_ATTACH_STATIC(pmap_evcnt_kernel_mappings); 432 EVCNT_ATTACH_STATIC(pmap_evcnt_kernel_unmappings); 433 434 EVCNT_ATTACH_STATIC(pmap_evcnt_exec_mappings); 435 EVCNT_ATTACH_STATIC(pmap_evcnt_exec_cached); 436 EVCNT_ATTACH_STATIC(pmap_evcnt_exec_synced); 437 EVCNT_ATTACH_STATIC(pmap_evcnt_exec_synced_clear_modify); 438 EVCNT_ATTACH_STATIC(pmap_evcnt_exec_synced_pvo_remove); 439 440 EVCNT_ATTACH_STATIC(pmap_evcnt_exec_uncached_page_protect); 441 EVCNT_ATTACH_STATIC(pmap_evcnt_exec_uncached_clear_modify); 442 EVCNT_ATTACH_STATIC(pmap_evcnt_exec_uncached_zero_page); 443 EVCNT_ATTACH_STATIC(pmap_evcnt_exec_uncached_copy_page); 444 EVCNT_ATTACH_STATIC(pmap_evcnt_exec_uncached_pvo_remove); 445 446 EVCNT_ATTACH_STATIC(pmap_evcnt_zeroed_pages); 447 EVCNT_ATTACH_STATIC(pmap_evcnt_copied_pages); 448 EVCNT_ATTACH_STATIC(pmap_evcnt_idlezeroed_pages); 449 450 EVCNT_ATTACH_STATIC(pmap_evcnt_updates); 451 EVCNT_ATTACH_STATIC(pmap_evcnt_collects); 452 EVCNT_ATTACH_STATIC(pmap_evcnt_copies); 453 454 EVCNT_ATTACH_STATIC(pmap_evcnt_ptes_spilled); 455 EVCNT_ATTACH_STATIC(pmap_evcnt_ptes_unspilled); 456 EVCNT_ATTACH_STATIC(pmap_evcnt_ptes_evicted); 457 EVCNT_ATTACH_STATIC(pmap_evcnt_ptes_removed); 458 EVCNT_ATTACH_STATIC(pmap_evcnt_ptes_changed); 459 460 EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_primary, 0); 461 EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_primary, 1); 462 EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_primary, 2); 463 EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_primary, 3); 464 EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_primary, 4); 465 EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_primary, 5); 466 EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_primary, 6); 467 EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_primary, 7); 468 EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_secondary, 0); 469 EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_secondary, 1); 470 EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_secondary, 2); 471 EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_secondary, 3); 472 EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_secondary, 4); 473 EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_secondary, 5); 474 EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_secondary, 6); 475 EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_secondary, 7); 476 477 EVCNT_ATTACH_STATIC(pmap_evcnt_pvos_reclaimed); 478 EVCNT_ATTACH_STATIC(pmap_evcnt_pvos_failed); 479 #else 480 #define PMAPCOUNT(ev) ((void) 0) 481 #define PMAPCOUNT2(ev) ((void) 0) 482 #endif 483 484 #define TLBIE(va) __asm volatile("tlbie %0" :: "r"(va)) 485 486 /* XXXSL: this needs to be moved to assembler */ 487 #define TLBIEL(va) __asm __volatile("tlbie %0" :: "r"(va)) 488 489 #define TLBSYNC() __asm volatile("tlbsync") 490 #define SYNC() __asm volatile("sync") 491 #define EIEIO() __asm volatile("eieio") 492 #define MFMSR() mfmsr() 493 #define MTMSR(psl) mtmsr(psl) 494 #define MFPVR() mfpvr() 495 #define MFSRIN(va) mfsrin(va) 496 #define MFTB() mfrtcltbl() 497 498 #if defined (PPC_OEA) || defined (PPC_OEA64_BRIDGE) 499 static inline register_t 500 mfsrin(vaddr_t va) 501 { 502 register_t sr; 503 __asm volatile ("mfsrin %0,%1" : "=r"(sr) : "r"(va)); 504 return sr; 505 } 506 #endif /* PPC_OEA*/ 507 508 #if defined (PPC_OEA64_BRIDGE) 509 extern void mfmsr64 (register64_t *result); 510 #endif /* PPC_OEA64_BRIDGE */ 511 512 513 static inline register_t 514 pmap_interrupts_off(void) 515 { 516 register_t msr = MFMSR(); 517 if (msr & PSL_EE) 518 MTMSR(msr & ~PSL_EE); 519 return msr; 520 } 521 522 static void 523 pmap_interrupts_restore(register_t msr) 524 { 525 if (msr & PSL_EE) 526 MTMSR(msr); 527 } 528 529 static inline u_int32_t 530 mfrtcltbl(void) 531 { 532 533 if ((MFPVR() >> 16) == MPC601) 534 return (mfrtcl() >> 7); 535 else 536 return (mftbl()); 537 } 538 539 /* 540 * These small routines may have to be replaced, 541 * if/when we support processors other that the 604. 542 */ 543 544 void 545 tlbia(void) 546 { 547 caddr_t i; 548 549 SYNC(); 550 #if defined(PPC_OEA) 551 /* 552 * Why not use "tlbia"? Because not all processors implement it. 553 * 554 * This needs to be a per-CPU callback to do the appropriate thing 555 * for the CPU. XXX 556 */ 557 for (i = 0; i < (caddr_t)0x00040000; i += 0x00001000) { 558 TLBIE(i); 559 EIEIO(); 560 SYNC(); 561 } 562 #elif defined (PPC_OEA64) || defined (PPC_OEA64_BRIDGE) 563 printf("Invalidating ALL TLB entries......\n"); 564 /* This is specifically for the 970, 970UM v1.6 pp. 140. */ 565 for (i = 0; i <= (caddr_t)0xFF000; i += 0x00001000) { 566 TLBIEL(i); 567 EIEIO(); 568 SYNC(); 569 } 570 #endif 571 TLBSYNC(); 572 SYNC(); 573 } 574 575 static inline register_t 576 va_to_vsid(const struct pmap *pm, vaddr_t addr) 577 { 578 #if defined (PPC_OEA) || defined (PPC_OEA64_BRIDGE) 579 return (pm->pm_sr[addr >> ADDR_SR_SHFT] & SR_VSID) >> SR_VSID_SHFT; 580 #else /* PPC_OEA64 */ 581 #if 0 582 const struct ste *ste; 583 register_t hash; 584 int i; 585 586 hash = (addr >> ADDR_ESID_SHFT) & ADDR_ESID_HASH; 587 588 /* 589 * Try the primary group first 590 */ 591 ste = pm->pm_stes[hash].stes; 592 for (i = 0; i < 8; i++, ste++) { 593 if (ste->ste_hi & STE_V) && 594 (addr & ~(ADDR_POFF|ADDR_PIDX)) == (ste->ste_hi & STE_ESID)) 595 return ste; 596 } 597 598 /* 599 * Then the secondary group. 600 */ 601 ste = pm->pm_stes[hash ^ ADDR_ESID_HASH].stes; 602 for (i = 0; i < 8; i++, ste++) { 603 if (ste->ste_hi & STE_V) && 604 (addr & ~(ADDR_POFF|ADDR_PIDX)) == (ste->ste_hi & STE_ESID)) 605 return addr; 606 } 607 608 return NULL; 609 #else 610 /* 611 * Rather than searching the STE groups for the VSID, we know 612 * how we generate that from the ESID and so do that. 613 */ 614 return VSID_MAKE(addr >> ADDR_SR_SHFT, pm->pm_vsid) >> SR_VSID_SHFT; 615 #endif 616 #endif /* PPC_OEA */ 617 } 618 619 static inline register_t 620 va_to_pteg(const struct pmap *pm, vaddr_t addr) 621 { 622 register_t hash; 623 624 hash = va_to_vsid(pm, addr) ^ ((addr & ADDR_PIDX) >> ADDR_PIDX_SHFT); 625 return hash & pmap_pteg_mask; 626 } 627 628 #if defined(DEBUG) || defined(PMAPCHECK) || defined(DDB) 629 /* 630 * Given a PTE in the page table, calculate the VADDR that hashes to it. 631 * The only bit of magic is that the top 4 bits of the address doesn't 632 * technically exist in the PTE. But we know we reserved 4 bits of the 633 * VSID for it so that's how we get it. 634 */ 635 static vaddr_t 636 pmap_pte_to_va(volatile const struct pte *pt) 637 { 638 vaddr_t va; 639 uintptr_t ptaddr = (uintptr_t) pt; 640 641 if (pt->pte_hi & PTE_HID) 642 ptaddr ^= (pmap_pteg_mask * sizeof(struct pteg)); 643 644 /* PPC Bits 10-19 PPC64 Bits 42-51 */ 645 #if defined(PPC_OEA) 646 va = ((pt->pte_hi >> PTE_VSID_SHFT) ^ (ptaddr / sizeof(struct pteg))) & 0x3ff; 647 #elif defined (PPC_OEA64) || defined (PPC_OEA64_BRIDGE) 648 va = ((pt->pte_hi >> PTE_VSID_SHFT) ^ (ptaddr / sizeof(struct pteg))) & 0x7ff; 649 #endif 650 va <<= ADDR_PIDX_SHFT; 651 652 /* PPC Bits 4-9 PPC64 Bits 36-41 */ 653 va |= (pt->pte_hi & PTE_API) << ADDR_API_SHFT; 654 655 #if defined(PPC_OEA64) 656 /* PPC63 Bits 0-35 */ 657 /* va |= VSID_TO_SR(pt->pte_hi >> PTE_VSID_SHFT) << ADDR_SR_SHFT; */ 658 #elif defined(PPC_OEA) || defined(PPC_OEA64_BRIDGE) 659 /* PPC Bits 0-3 */ 660 va |= VSID_TO_SR(pt->pte_hi >> PTE_VSID_SHFT) << ADDR_SR_SHFT; 661 #endif 662 663 return va; 664 } 665 #endif 666 667 static inline struct pvo_head * 668 pa_to_pvoh(paddr_t pa, struct vm_page **pg_p) 669 { 670 #ifdef __HAVE_VM_PAGE_MD 671 struct vm_page *pg; 672 673 pg = PHYS_TO_VM_PAGE(pa); 674 if (pg_p != NULL) 675 *pg_p = pg; 676 if (pg == NULL) 677 return &pmap_pvo_unmanaged; 678 return &pg->mdpage.mdpg_pvoh; 679 #endif 680 #ifdef __HAVE_PMAP_PHYSSEG 681 int bank, pg; 682 683 bank = vm_physseg_find(atop(pa), &pg); 684 if (pg_p != NULL) 685 *pg_p = pg; 686 if (bank == -1) 687 return &pmap_pvo_unmanaged; 688 return &vm_physmem[bank].pmseg.pvoh[pg]; 689 #endif 690 } 691 692 static inline struct pvo_head * 693 vm_page_to_pvoh(struct vm_page *pg) 694 { 695 #ifdef __HAVE_VM_PAGE_MD 696 return &pg->mdpage.mdpg_pvoh; 697 #endif 698 #ifdef __HAVE_PMAP_PHYSSEG 699 return pa_to_pvoh(VM_PAGE_TO_PHYS(pg), NULL); 700 #endif 701 } 702 703 704 #ifdef __HAVE_PMAP_PHYSSEG 705 static inline char * 706 pa_to_attr(paddr_t pa) 707 { 708 int bank, pg; 709 710 bank = vm_physseg_find(atop(pa), &pg); 711 if (bank == -1) 712 return NULL; 713 return &vm_physmem[bank].pmseg.attrs[pg]; 714 } 715 #endif 716 717 static inline void 718 pmap_attr_clear(struct vm_page *pg, int ptebit) 719 { 720 #ifdef __HAVE_PMAP_PHYSSEG 721 *pa_to_attr(VM_PAGE_TO_PHYS(pg)) &= ~(ptebit >> ATTR_SHFT); 722 #endif 723 #ifdef __HAVE_VM_PAGE_MD 724 pg->mdpage.mdpg_attrs &= ~ptebit; 725 #endif 726 } 727 728 static inline int 729 pmap_attr_fetch(struct vm_page *pg) 730 { 731 #ifdef __HAVE_PMAP_PHYSSEG 732 return *pa_to_attr(VM_PAGE_TO_PHYS(pg)) << ATTR_SHFT; 733 #endif 734 #ifdef __HAVE_VM_PAGE_MD 735 return pg->mdpage.mdpg_attrs; 736 #endif 737 } 738 739 static inline void 740 pmap_attr_save(struct vm_page *pg, int ptebit) 741 { 742 #ifdef __HAVE_PMAP_PHYSSEG 743 *pa_to_attr(VM_PAGE_TO_PHYS(pg)) |= (ptebit >> ATTR_SHFT); 744 #endif 745 #ifdef __HAVE_VM_PAGE_MD 746 pg->mdpage.mdpg_attrs |= ptebit; 747 #endif 748 } 749 750 static inline int 751 pmap_pte_compare(const volatile struct pte *pt, const struct pte *pvo_pt) 752 { 753 if (pt->pte_hi == pvo_pt->pte_hi 754 #if 0 755 && ((pt->pte_lo ^ pvo_pt->pte_lo) & 756 ~(PTE_REF|PTE_CHG)) == 0 757 #endif 758 ) 759 return 1; 760 return 0; 761 } 762 763 static inline void 764 pmap_pte_create(struct pte *pt, const struct pmap *pm, vaddr_t va, register_t pte_lo) 765 { 766 /* 767 * Construct the PTE. Default to IMB initially. Valid bit 768 * only gets set when the real pte is set in memory. 769 * 770 * Note: Don't set the valid bit for correct operation of tlb update. 771 */ 772 #if defined(PPC_OEA) 773 pt->pte_hi = (va_to_vsid(pm, va) << PTE_VSID_SHFT) 774 | (((va & ADDR_PIDX) >> (ADDR_API_SHFT - PTE_API_SHFT)) & PTE_API); 775 pt->pte_lo = pte_lo; 776 #elif defined (PPC_OEA64_BRIDGE) 777 pt->pte_hi = ((u_int64_t)va_to_vsid(pm, va) << PTE_VSID_SHFT) 778 | (((va & ADDR_PIDX) >> (ADDR_API_SHFT - PTE_API_SHFT)) & PTE_API); 779 pt->pte_lo = (u_int64_t) pte_lo; 780 #elif defined (PPC_OEA64) 781 #error PPC_OEA64 not supported 782 #endif /* PPC_OEA */ 783 } 784 785 static inline void 786 pmap_pte_synch(volatile struct pte *pt, struct pte *pvo_pt) 787 { 788 pvo_pt->pte_lo |= pt->pte_lo & (PTE_REF|PTE_CHG); 789 } 790 791 static inline void 792 pmap_pte_clear(volatile struct pte *pt, vaddr_t va, int ptebit) 793 { 794 /* 795 * As shown in Section 7.6.3.2.3 796 */ 797 pt->pte_lo &= ~ptebit; 798 TLBIE(va); 799 SYNC(); 800 EIEIO(); 801 TLBSYNC(); 802 SYNC(); 803 } 804 805 static inline void 806 pmap_pte_set(volatile struct pte *pt, struct pte *pvo_pt) 807 { 808 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 809 if (pvo_pt->pte_hi & PTE_VALID) 810 panic("pte_set: setting an already valid pte %p", pvo_pt); 811 #endif 812 pvo_pt->pte_hi |= PTE_VALID; 813 814 /* 815 * Update the PTE as defined in section 7.6.3.1 816 * Note that the REF/CHG bits are from pvo_pt and thus should 817 * have been saved so this routine can restore them (if desired). 818 */ 819 pt->pte_lo = pvo_pt->pte_lo; 820 EIEIO(); 821 pt->pte_hi = pvo_pt->pte_hi; 822 TLBSYNC(); 823 SYNC(); 824 pmap_pte_valid++; 825 } 826 827 static inline void 828 pmap_pte_unset(volatile struct pte *pt, struct pte *pvo_pt, vaddr_t va) 829 { 830 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 831 if ((pvo_pt->pte_hi & PTE_VALID) == 0) 832 panic("pte_unset: attempt to unset an inactive pte#1 %p/%p", pvo_pt, pt); 833 if ((pt->pte_hi & PTE_VALID) == 0) 834 panic("pte_unset: attempt to unset an inactive pte#2 %p/%p", pvo_pt, pt); 835 #endif 836 837 pvo_pt->pte_hi &= ~PTE_VALID; 838 /* 839 * Force the ref & chg bits back into the PTEs. 840 */ 841 SYNC(); 842 /* 843 * Invalidate the pte ... (Section 7.6.3.3) 844 */ 845 pt->pte_hi &= ~PTE_VALID; 846 SYNC(); 847 TLBIE(va); 848 SYNC(); 849 EIEIO(); 850 TLBSYNC(); 851 SYNC(); 852 /* 853 * Save the ref & chg bits ... 854 */ 855 pmap_pte_synch(pt, pvo_pt); 856 pmap_pte_valid--; 857 } 858 859 static inline void 860 pmap_pte_change(volatile struct pte *pt, struct pte *pvo_pt, vaddr_t va) 861 { 862 /* 863 * Invalidate the PTE 864 */ 865 pmap_pte_unset(pt, pvo_pt, va); 866 pmap_pte_set(pt, pvo_pt); 867 } 868 869 /* 870 * Try to insert the PTE @ *pvo_pt into the pmap_pteg_table at ptegidx 871 * (either primary or secondary location). 872 * 873 * Note: both the destination and source PTEs must not have PTE_VALID set. 874 */ 875 876 STATIC int 877 pmap_pte_insert(int ptegidx, struct pte *pvo_pt) 878 { 879 volatile struct pte *pt; 880 int i; 881 882 #if defined(DEBUG) 883 #if defined (PPC_OEA) 884 DPRINTFN(PTE, ("pmap_pte_insert: idx 0x%x, pte 0x%x 0x%x\n", 885 ptegidx, (unsigned int) pvo_pt->pte_hi, (unsigned int) pvo_pt->pte_lo)); 886 #elif defined (PPC_OEA64_BRIDGE) 887 DPRINTFN(PTE, ("pmap_pte_insert: idx 0x%x, pte 0x%016llx 0x%016llx\n", 888 ptegidx, (unsigned long long) pvo_pt->pte_hi, 889 (unsigned long long) pvo_pt->pte_lo)); 890 891 #endif 892 #endif 893 /* 894 * First try primary hash. 895 */ 896 for (pt = pmap_pteg_table[ptegidx].pt, i = 0; i < 8; i++, pt++) { 897 if ((pt->pte_hi & PTE_VALID) == 0) { 898 pvo_pt->pte_hi &= ~PTE_HID; 899 pmap_pte_set(pt, pvo_pt); 900 return i; 901 } 902 } 903 904 /* 905 * Now try secondary hash. 906 */ 907 ptegidx ^= pmap_pteg_mask; 908 for (pt = pmap_pteg_table[ptegidx].pt, i = 0; i < 8; i++, pt++) { 909 if ((pt->pte_hi & PTE_VALID) == 0) { 910 pvo_pt->pte_hi |= PTE_HID; 911 pmap_pte_set(pt, pvo_pt); 912 return i; 913 } 914 } 915 return -1; 916 } 917 918 /* 919 * Spill handler. 920 * 921 * Tries to spill a page table entry from the overflow area. 922 * This runs in either real mode (if dealing with a exception spill) 923 * or virtual mode when dealing with manually spilling one of the 924 * kernel's pte entries. In either case, interrupts are already 925 * disabled. 926 */ 927 928 int 929 pmap_pte_spill(struct pmap *pm, vaddr_t addr, boolean_t exec) 930 { 931 struct pvo_entry *source_pvo, *victim_pvo, *next_pvo; 932 struct pvo_entry *pvo; 933 /* XXX: gcc -- vpvoh is always set at either *1* or *2* */ 934 struct pvo_tqhead *pvoh, *vpvoh = NULL; 935 int ptegidx, i, j; 936 volatile struct pteg *pteg; 937 volatile struct pte *pt; 938 939 ptegidx = va_to_pteg(pm, addr); 940 941 /* 942 * Have to substitute some entry. Use the primary hash for this. 943 * Use low bits of timebase as random generator. Make sure we are 944 * not picking a kernel pte for replacement. 945 */ 946 pteg = &pmap_pteg_table[ptegidx]; 947 i = MFTB() & 7; 948 for (j = 0; j < 8; j++) { 949 pt = &pteg->pt[i]; 950 if ((pt->pte_hi & PTE_VALID) == 0 || 951 VSID_TO_HASH((pt->pte_hi & PTE_VSID) >> PTE_VSID_SHFT) 952 != KERNEL_VSIDBITS) 953 break; 954 i = (i + 1) & 7; 955 } 956 KASSERT(j < 8); 957 958 source_pvo = NULL; 959 victim_pvo = NULL; 960 pvoh = &pmap_pvo_table[ptegidx]; 961 TAILQ_FOREACH(pvo, pvoh, pvo_olink) { 962 963 /* 964 * We need to find pvo entry for this address... 965 */ 966 PMAP_PVO_CHECK(pvo); /* sanity check */ 967 968 /* 969 * If we haven't found the source and we come to a PVO with 970 * a valid PTE, then we know we can't find it because all 971 * evicted PVOs always are first in the list. 972 */ 973 if (source_pvo == NULL && (pvo->pvo_pte.pte_hi & PTE_VALID)) 974 break; 975 if (source_pvo == NULL && pm == pvo->pvo_pmap && 976 addr == PVO_VADDR(pvo)) { 977 978 /* 979 * Now we have found the entry to be spilled into the 980 * pteg. Attempt to insert it into the page table. 981 */ 982 j = pmap_pte_insert(ptegidx, &pvo->pvo_pte); 983 if (j >= 0) { 984 PVO_PTEGIDX_SET(pvo, j); 985 PMAP_PVO_CHECK(pvo); /* sanity check */ 986 PVO_WHERE(pvo, SPILL_INSERT); 987 pvo->pvo_pmap->pm_evictions--; 988 PMAPCOUNT(ptes_spilled); 989 PMAPCOUNT2(((pvo->pvo_pte.pte_hi & PTE_HID) 990 ? pmap_evcnt_ptes_secondary 991 : pmap_evcnt_ptes_primary)[j]); 992 993 /* 994 * Since we keep the evicted entries at the 995 * from of the PVO list, we need move this 996 * (now resident) PVO after the evicted 997 * entries. 998 */ 999 next_pvo = TAILQ_NEXT(pvo, pvo_olink); 1000 1001 /* 1002 * If we don't have to move (either we were the 1003 * last entry or the next entry was valid), 1004 * don't change our position. Otherwise 1005 * move ourselves to the tail of the queue. 1006 */ 1007 if (next_pvo != NULL && 1008 !(next_pvo->pvo_pte.pte_hi & PTE_VALID)) { 1009 TAILQ_REMOVE(pvoh, pvo, pvo_olink); 1010 TAILQ_INSERT_TAIL(pvoh, pvo, pvo_olink); 1011 } 1012 return 1; 1013 } 1014 source_pvo = pvo; 1015 if (exec && !PVO_ISEXECUTABLE(source_pvo)) { 1016 return 0; 1017 } 1018 if (victim_pvo != NULL) 1019 break; 1020 } 1021 1022 /* 1023 * We also need the pvo entry of the victim we are replacing 1024 * so save the R & C bits of the PTE. 1025 */ 1026 if ((pt->pte_hi & PTE_HID) == 0 && victim_pvo == NULL && 1027 pmap_pte_compare(pt, &pvo->pvo_pte)) { 1028 vpvoh = pvoh; /* *1* */ 1029 victim_pvo = pvo; 1030 if (source_pvo != NULL) 1031 break; 1032 } 1033 } 1034 1035 if (source_pvo == NULL) { 1036 PMAPCOUNT(ptes_unspilled); 1037 return 0; 1038 } 1039 1040 if (victim_pvo == NULL) { 1041 if ((pt->pte_hi & PTE_HID) == 0) 1042 panic("pmap_pte_spill: victim p-pte (%p) has " 1043 "no pvo entry!", pt); 1044 1045 /* 1046 * If this is a secondary PTE, we need to search 1047 * its primary pvo bucket for the matching PVO. 1048 */ 1049 vpvoh = &pmap_pvo_table[ptegidx ^ pmap_pteg_mask]; /* *2* */ 1050 TAILQ_FOREACH(pvo, vpvoh, pvo_olink) { 1051 PMAP_PVO_CHECK(pvo); /* sanity check */ 1052 1053 /* 1054 * We also need the pvo entry of the victim we are 1055 * replacing so save the R & C bits of the PTE. 1056 */ 1057 if (pmap_pte_compare(pt, &pvo->pvo_pte)) { 1058 victim_pvo = pvo; 1059 break; 1060 } 1061 } 1062 if (victim_pvo == NULL) 1063 panic("pmap_pte_spill: victim s-pte (%p) has " 1064 "no pvo entry!", pt); 1065 } 1066 1067 /* 1068 * The victim should be not be a kernel PVO/PTE entry. 1069 */ 1070 KASSERT(victim_pvo->pvo_pmap != pmap_kernel()); 1071 KASSERT(PVO_PTEGIDX_ISSET(victim_pvo)); 1072 KASSERT(PVO_PTEGIDX_GET(victim_pvo) == i); 1073 1074 /* 1075 * We are invalidating the TLB entry for the EA for the 1076 * we are replacing even though its valid; If we don't 1077 * we lose any ref/chg bit changes contained in the TLB 1078 * entry. 1079 */ 1080 source_pvo->pvo_pte.pte_hi &= ~PTE_HID; 1081 1082 /* 1083 * To enforce the PVO list ordering constraint that all 1084 * evicted entries should come before all valid entries, 1085 * move the source PVO to the tail of its list and the 1086 * victim PVO to the head of its list (which might not be 1087 * the same list, if the victim was using the secondary hash). 1088 */ 1089 TAILQ_REMOVE(pvoh, source_pvo, pvo_olink); 1090 TAILQ_INSERT_TAIL(pvoh, source_pvo, pvo_olink); 1091 TAILQ_REMOVE(vpvoh, victim_pvo, pvo_olink); 1092 TAILQ_INSERT_HEAD(vpvoh, victim_pvo, pvo_olink); 1093 pmap_pte_unset(pt, &victim_pvo->pvo_pte, victim_pvo->pvo_vaddr); 1094 pmap_pte_set(pt, &source_pvo->pvo_pte); 1095 victim_pvo->pvo_pmap->pm_evictions++; 1096 source_pvo->pvo_pmap->pm_evictions--; 1097 PVO_WHERE(victim_pvo, SPILL_UNSET); 1098 PVO_WHERE(source_pvo, SPILL_SET); 1099 1100 PVO_PTEGIDX_CLR(victim_pvo); 1101 PVO_PTEGIDX_SET(source_pvo, i); 1102 PMAPCOUNT2(pmap_evcnt_ptes_primary[i]); 1103 PMAPCOUNT(ptes_spilled); 1104 PMAPCOUNT(ptes_evicted); 1105 PMAPCOUNT(ptes_removed); 1106 1107 PMAP_PVO_CHECK(victim_pvo); 1108 PMAP_PVO_CHECK(source_pvo); 1109 return 1; 1110 } 1111 1112 /* 1113 * Restrict given range to physical memory 1114 */ 1115 void 1116 pmap_real_memory(paddr_t *start, psize_t *size) 1117 { 1118 struct mem_region *mp; 1119 1120 for (mp = mem; mp->size; mp++) { 1121 if (*start + *size > mp->start 1122 && *start < mp->start + mp->size) { 1123 if (*start < mp->start) { 1124 *size -= mp->start - *start; 1125 *start = mp->start; 1126 } 1127 if (*start + *size > mp->start + mp->size) 1128 *size = mp->start + mp->size - *start; 1129 return; 1130 } 1131 } 1132 *size = 0; 1133 } 1134 1135 /* 1136 * Initialize anything else for pmap handling. 1137 * Called during vm_init(). 1138 */ 1139 void 1140 pmap_init(void) 1141 { 1142 #ifdef __HAVE_PMAP_PHYSSEG 1143 struct pvo_tqhead *pvoh; 1144 int bank; 1145 long sz; 1146 char *attr; 1147 1148 pvoh = pmap_physseg.pvoh; 1149 attr = pmap_physseg.attrs; 1150 for (bank = 0; bank < vm_nphysseg; bank++) { 1151 sz = vm_physmem[bank].end - vm_physmem[bank].start; 1152 vm_physmem[bank].pmseg.pvoh = pvoh; 1153 vm_physmem[bank].pmseg.attrs = attr; 1154 for (; sz > 0; sz--, pvoh++, attr++) { 1155 TAILQ_INIT(pvoh); 1156 *attr = 0; 1157 } 1158 } 1159 #endif 1160 1161 pool_init(&pmap_mpvo_pool, sizeof(struct pvo_entry), 1162 sizeof(struct pvo_entry), 0, 0, "pmap_mpvopl", 1163 &pmap_pool_mallocator); 1164 1165 pool_setlowat(&pmap_mpvo_pool, 1008); 1166 1167 pmap_initialized = 1; 1168 1169 } 1170 1171 /* 1172 * How much virtual space does the kernel get? 1173 */ 1174 void 1175 pmap_virtual_space(vaddr_t *start, vaddr_t *end) 1176 { 1177 /* 1178 * For now, reserve one segment (minus some overhead) for kernel 1179 * virtual memory 1180 */ 1181 *start = VM_MIN_KERNEL_ADDRESS; 1182 *end = VM_MAX_KERNEL_ADDRESS; 1183 } 1184 1185 /* 1186 * Allocate, initialize, and return a new physical map. 1187 */ 1188 pmap_t 1189 pmap_create(void) 1190 { 1191 pmap_t pm; 1192 1193 pm = pool_get(&pmap_pool, PR_WAITOK); 1194 memset((caddr_t)pm, 0, sizeof *pm); 1195 pmap_pinit(pm); 1196 1197 DPRINTFN(CREATE,("pmap_create: pm %p:\n" 1198 "\t%06x %06x %06x %06x %06x %06x %06x %06x\n" 1199 "\t%06x %06x %06x %06x %06x %06x %06x %06x\n", pm, 1200 (unsigned int) pm->pm_sr[0], (unsigned int) pm->pm_sr[1], 1201 (unsigned int) pm->pm_sr[2], (unsigned int) pm->pm_sr[3], 1202 (unsigned int) pm->pm_sr[4], (unsigned int) pm->pm_sr[5], 1203 (unsigned int) pm->pm_sr[6], (unsigned int) pm->pm_sr[7], 1204 (unsigned int) pm->pm_sr[8], (unsigned int) pm->pm_sr[9], 1205 (unsigned int) pm->pm_sr[10], (unsigned int) pm->pm_sr[11], 1206 (unsigned int) pm->pm_sr[12], (unsigned int) pm->pm_sr[13], 1207 (unsigned int) pm->pm_sr[14], (unsigned int) pm->pm_sr[15])); 1208 return pm; 1209 } 1210 1211 /* 1212 * Initialize a preallocated and zeroed pmap structure. 1213 */ 1214 void 1215 pmap_pinit(pmap_t pm) 1216 { 1217 register_t entropy = MFTB(); 1218 register_t mask; 1219 int i; 1220 1221 /* 1222 * Allocate some segment registers for this pmap. 1223 */ 1224 pm->pm_refs = 1; 1225 for (i = 0; i < NPMAPS; i += VSID_NBPW) { 1226 static register_t pmap_vsidcontext; 1227 register_t hash; 1228 unsigned int n; 1229 1230 /* Create a new value by multiplying by a prime adding in 1231 * entropy from the timebase register. This is to make the 1232 * VSID more random so that the PT Hash function collides 1233 * less often. (note that the prime causes gcc to do shifts 1234 * instead of a multiply) 1235 */ 1236 pmap_vsidcontext = (pmap_vsidcontext * 0x1105) + entropy; 1237 hash = pmap_vsidcontext & (NPMAPS - 1); 1238 if (hash == 0) { /* 0 is special, avoid it */ 1239 entropy += 0xbadf00d; 1240 continue; 1241 } 1242 n = hash >> 5; 1243 mask = 1L << (hash & (VSID_NBPW-1)); 1244 hash = pmap_vsidcontext; 1245 if (pmap_vsid_bitmap[n] & mask) { /* collision? */ 1246 /* anything free in this bucket? */ 1247 if (~pmap_vsid_bitmap[n] == 0) { 1248 entropy = hash ^ (hash >> 16); 1249 continue; 1250 } 1251 i = ffs(~pmap_vsid_bitmap[n]) - 1; 1252 mask = 1L << i; 1253 hash &= ~(VSID_NBPW-1); 1254 hash |= i; 1255 } 1256 hash &= PTE_VSID >> PTE_VSID_SHFT; 1257 pmap_vsid_bitmap[n] |= mask; 1258 pm->pm_vsid = hash; 1259 #if defined (PPC_OEA) || defined (PPC_OEA64_BRIDGE) 1260 for (i = 0; i < 16; i++) 1261 pm->pm_sr[i] = VSID_MAKE(i, hash) | SR_PRKEY | 1262 SR_NOEXEC; 1263 #endif 1264 return; 1265 } 1266 panic("pmap_pinit: out of segments"); 1267 } 1268 1269 /* 1270 * Add a reference to the given pmap. 1271 */ 1272 void 1273 pmap_reference(pmap_t pm) 1274 { 1275 pm->pm_refs++; 1276 } 1277 1278 /* 1279 * Retire the given pmap from service. 1280 * Should only be called if the map contains no valid mappings. 1281 */ 1282 void 1283 pmap_destroy(pmap_t pm) 1284 { 1285 if (--pm->pm_refs == 0) { 1286 pmap_release(pm); 1287 pool_put(&pmap_pool, pm); 1288 } 1289 } 1290 1291 /* 1292 * Release any resources held by the given physical map. 1293 * Called when a pmap initialized by pmap_pinit is being released. 1294 */ 1295 void 1296 pmap_release(pmap_t pm) 1297 { 1298 int idx, mask; 1299 1300 if (pm->pm_sr[0] == 0) 1301 panic("pmap_release"); 1302 idx = pm->pm_vsid & (NPMAPS-1); 1303 mask = 1 << (idx % VSID_NBPW); 1304 idx /= VSID_NBPW; 1305 1306 KASSERT(pmap_vsid_bitmap[idx] & mask); 1307 pmap_vsid_bitmap[idx] &= ~mask; 1308 } 1309 1310 /* 1311 * Copy the range specified by src_addr/len 1312 * from the source map to the range dst_addr/len 1313 * in the destination map. 1314 * 1315 * This routine is only advisory and need not do anything. 1316 */ 1317 void 1318 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vaddr_t dst_addr, 1319 vsize_t len, vaddr_t src_addr) 1320 { 1321 PMAPCOUNT(copies); 1322 } 1323 1324 /* 1325 * Require that all active physical maps contain no 1326 * incorrect entries NOW. 1327 */ 1328 void 1329 pmap_update(struct pmap *pmap) 1330 { 1331 PMAPCOUNT(updates); 1332 TLBSYNC(); 1333 } 1334 1335 /* 1336 * Garbage collects the physical map system for 1337 * pages which are no longer used. 1338 * Success need not be guaranteed -- that is, there 1339 * may well be pages which are not referenced, but 1340 * others may be collected. 1341 * Called by the pageout daemon when pages are scarce. 1342 */ 1343 void 1344 pmap_collect(pmap_t pm) 1345 { 1346 PMAPCOUNT(collects); 1347 } 1348 1349 static inline int 1350 pmap_pvo_pte_index(const struct pvo_entry *pvo, int ptegidx) 1351 { 1352 int pteidx; 1353 /* 1354 * We can find the actual pte entry without searching by 1355 * grabbing the PTEG index from 3 unused bits in pte_lo[11:9] 1356 * and by noticing the HID bit. 1357 */ 1358 pteidx = ptegidx * 8 + PVO_PTEGIDX_GET(pvo); 1359 if (pvo->pvo_pte.pte_hi & PTE_HID) 1360 pteidx ^= pmap_pteg_mask * 8; 1361 return pteidx; 1362 } 1363 1364 volatile struct pte * 1365 pmap_pvo_to_pte(const struct pvo_entry *pvo, int pteidx) 1366 { 1367 volatile struct pte *pt; 1368 1369 #if !defined(DIAGNOSTIC) && !defined(DEBUG) && !defined(PMAPCHECK) 1370 if ((pvo->pvo_pte.pte_hi & PTE_VALID) == 0) 1371 return NULL; 1372 #endif 1373 1374 /* 1375 * If we haven't been supplied the ptegidx, calculate it. 1376 */ 1377 if (pteidx == -1) { 1378 int ptegidx; 1379 ptegidx = va_to_pteg(pvo->pvo_pmap, pvo->pvo_vaddr); 1380 pteidx = pmap_pvo_pte_index(pvo, ptegidx); 1381 } 1382 1383 pt = &pmap_pteg_table[pteidx >> 3].pt[pteidx & 7]; 1384 1385 #if !defined(DIAGNOSTIC) && !defined(DEBUG) && !defined(PMAPCHECK) 1386 return pt; 1387 #else 1388 if ((pvo->pvo_pte.pte_hi & PTE_VALID) && !PVO_PTEGIDX_ISSET(pvo)) { 1389 panic("pmap_pvo_to_pte: pvo %p: has valid pte in " 1390 "pvo but no valid pte index", pvo); 1391 } 1392 if ((pvo->pvo_pte.pte_hi & PTE_VALID) == 0 && PVO_PTEGIDX_ISSET(pvo)) { 1393 panic("pmap_pvo_to_pte: pvo %p: has valid pte index in " 1394 "pvo but no valid pte", pvo); 1395 } 1396 1397 if ((pt->pte_hi ^ (pvo->pvo_pte.pte_hi & ~PTE_VALID)) == PTE_VALID) { 1398 if ((pvo->pvo_pte.pte_hi & PTE_VALID) == 0) { 1399 #if defined(DEBUG) || defined(PMAPCHECK) 1400 pmap_pte_print(pt); 1401 #endif 1402 panic("pmap_pvo_to_pte: pvo %p: has valid pte in " 1403 "pmap_pteg_table %p but invalid in pvo", 1404 pvo, pt); 1405 } 1406 if (((pt->pte_lo ^ pvo->pvo_pte.pte_lo) & ~(PTE_CHG|PTE_REF)) != 0) { 1407 #if defined(DEBUG) || defined(PMAPCHECK) 1408 pmap_pte_print(pt); 1409 #endif 1410 panic("pmap_pvo_to_pte: pvo %p: pvo pte does " 1411 "not match pte %p in pmap_pteg_table", 1412 pvo, pt); 1413 } 1414 return pt; 1415 } 1416 1417 if (pvo->pvo_pte.pte_hi & PTE_VALID) { 1418 #if defined(DEBUG) || defined(PMAPCHECK) 1419 pmap_pte_print(pt); 1420 #endif 1421 panic("pmap_pvo_to_pte: pvo %p: has nomatching pte %p in " 1422 "pmap_pteg_table but valid in pvo", pvo, pt); 1423 } 1424 return NULL; 1425 #endif /* !(!DIAGNOSTIC && !DEBUG && !PMAPCHECK) */ 1426 } 1427 1428 struct pvo_entry * 1429 pmap_pvo_find_va(pmap_t pm, vaddr_t va, int *pteidx_p) 1430 { 1431 struct pvo_entry *pvo; 1432 int ptegidx; 1433 1434 va &= ~ADDR_POFF; 1435 ptegidx = va_to_pteg(pm, va); 1436 1437 TAILQ_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) { 1438 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 1439 if ((uintptr_t) pvo >= SEGMENT_LENGTH) 1440 panic("pmap_pvo_find_va: invalid pvo %p on " 1441 "list %#x (%p)", pvo, ptegidx, 1442 &pmap_pvo_table[ptegidx]); 1443 #endif 1444 if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) { 1445 if (pteidx_p) 1446 *pteidx_p = pmap_pvo_pte_index(pvo, ptegidx); 1447 return pvo; 1448 } 1449 } 1450 if ((pm == pmap_kernel()) && (va < SEGMENT_LENGTH)) 1451 panic("%s: returning NULL for %s pmap, va: 0x%08lx\n", __FUNCTION__, 1452 (pm == pmap_kernel() ? "kernel" : "user"), va); 1453 return NULL; 1454 } 1455 1456 #if defined(DEBUG) || defined(PMAPCHECK) 1457 void 1458 pmap_pvo_check(const struct pvo_entry *pvo) 1459 { 1460 struct pvo_head *pvo_head; 1461 struct pvo_entry *pvo0; 1462 volatile struct pte *pt; 1463 int failed = 0; 1464 1465 if ((uintptr_t)(pvo+1) >= SEGMENT_LENGTH) 1466 panic("pmap_pvo_check: pvo %p: invalid address", pvo); 1467 1468 if ((uintptr_t)(pvo->pvo_pmap+1) >= SEGMENT_LENGTH) { 1469 printf("pmap_pvo_check: pvo %p: invalid pmap address %p\n", 1470 pvo, pvo->pvo_pmap); 1471 failed = 1; 1472 } 1473 1474 if ((uintptr_t)TAILQ_NEXT(pvo, pvo_olink) >= SEGMENT_LENGTH || 1475 (((uintptr_t)TAILQ_NEXT(pvo, pvo_olink)) & 0x1f) != 0) { 1476 printf("pmap_pvo_check: pvo %p: invalid ovlink address %p\n", 1477 pvo, TAILQ_NEXT(pvo, pvo_olink)); 1478 failed = 1; 1479 } 1480 1481 if ((uintptr_t)LIST_NEXT(pvo, pvo_vlink) >= SEGMENT_LENGTH || 1482 (((uintptr_t)LIST_NEXT(pvo, pvo_vlink)) & 0x1f) != 0) { 1483 printf("pmap_pvo_check: pvo %p: invalid ovlink address %p\n", 1484 pvo, LIST_NEXT(pvo, pvo_vlink)); 1485 failed = 1; 1486 } 1487 1488 if (pvo->pvo_vaddr & PVO_MANAGED) { 1489 pvo_head = pa_to_pvoh(pvo->pvo_pte.pte_lo & PTE_RPGN, NULL); 1490 } else { 1491 if (pvo->pvo_vaddr < VM_MIN_KERNEL_ADDRESS) { 1492 printf("pmap_pvo_check: pvo %p: non kernel address " 1493 "on kernel unmanaged list\n", pvo); 1494 failed = 1; 1495 } 1496 pvo_head = &pmap_pvo_kunmanaged; 1497 } 1498 LIST_FOREACH(pvo0, pvo_head, pvo_vlink) { 1499 if (pvo0 == pvo) 1500 break; 1501 } 1502 if (pvo0 == NULL) { 1503 printf("pmap_pvo_check: pvo %p: not present " 1504 "on its vlist head %p\n", pvo, pvo_head); 1505 failed = 1; 1506 } 1507 if (pvo != pmap_pvo_find_va(pvo->pvo_pmap, pvo->pvo_vaddr, NULL)) { 1508 printf("pmap_pvo_check: pvo %p: not present " 1509 "on its olist head\n", pvo); 1510 failed = 1; 1511 } 1512 pt = pmap_pvo_to_pte(pvo, -1); 1513 if (pt == NULL) { 1514 if (pvo->pvo_pte.pte_hi & PTE_VALID) { 1515 printf("pmap_pvo_check: pvo %p: pte_hi VALID but " 1516 "no PTE\n", pvo); 1517 failed = 1; 1518 } 1519 } else { 1520 if ((uintptr_t) pt < (uintptr_t) &pmap_pteg_table[0] || 1521 (uintptr_t) pt >= 1522 (uintptr_t) &pmap_pteg_table[pmap_pteg_cnt]) { 1523 printf("pmap_pvo_check: pvo %p: pte %p not in " 1524 "pteg table\n", pvo, pt); 1525 failed = 1; 1526 } 1527 if (((((uintptr_t) pt) >> 3) & 7) != PVO_PTEGIDX_GET(pvo)) { 1528 printf("pmap_pvo_check: pvo %p: pte_hi VALID but " 1529 "no PTE\n", pvo); 1530 failed = 1; 1531 } 1532 if (pvo->pvo_pte.pte_hi != pt->pte_hi) { 1533 printf("pmap_pvo_check: pvo %p: pte_hi differ: " 1534 "%#x/%#x\n", pvo, (unsigned int) pvo->pvo_pte.pte_hi, (unsigned int) pt->pte_hi); 1535 failed = 1; 1536 } 1537 if (((pvo->pvo_pte.pte_lo ^ pt->pte_lo) & 1538 (PTE_PP|PTE_WIMG|PTE_RPGN)) != 0) { 1539 printf("pmap_pvo_check: pvo %p: pte_lo differ: " 1540 "%#x/%#x\n", pvo, 1541 (unsigned int) (pvo->pvo_pte.pte_lo & (PTE_PP|PTE_WIMG|PTE_RPGN)), 1542 (unsigned int) (pt->pte_lo & (PTE_PP|PTE_WIMG|PTE_RPGN))); 1543 failed = 1; 1544 } 1545 if ((pmap_pte_to_va(pt) ^ PVO_VADDR(pvo)) & 0x0fffffff) { 1546 printf("pmap_pvo_check: pvo %p: PTE %p derived VA %#lx" 1547 " doesn't not match PVO's VA %#lx\n", 1548 pvo, pt, pmap_pte_to_va(pt), PVO_VADDR(pvo)); 1549 failed = 1; 1550 } 1551 if (failed) 1552 pmap_pte_print(pt); 1553 } 1554 if (failed) 1555 panic("pmap_pvo_check: pvo %p, pm %p: bugcheck!", pvo, 1556 pvo->pvo_pmap); 1557 } 1558 #endif /* DEBUG || PMAPCHECK */ 1559 1560 /* 1561 * Search the PVO table looking for a non-wired entry. 1562 * If we find one, remove it and return it. 1563 */ 1564 1565 struct pvo_entry * 1566 pmap_pvo_reclaim(struct pmap *pm) 1567 { 1568 struct pvo_tqhead *pvoh; 1569 struct pvo_entry *pvo; 1570 uint32_t idx, endidx; 1571 1572 endidx = pmap_pvo_reclaim_nextidx; 1573 for (idx = (endidx + 1) & pmap_pteg_mask; idx != endidx; 1574 idx = (idx + 1) & pmap_pteg_mask) { 1575 pvoh = &pmap_pvo_table[idx]; 1576 TAILQ_FOREACH(pvo, pvoh, pvo_olink) { 1577 if ((pvo->pvo_vaddr & PVO_WIRED) == 0) { 1578 pmap_pvo_remove(pvo, -1, NULL); 1579 pmap_pvo_reclaim_nextidx = idx; 1580 PMAPCOUNT(pvos_reclaimed); 1581 return pvo; 1582 } 1583 } 1584 } 1585 return NULL; 1586 } 1587 1588 /* 1589 * This returns whether this is the first mapping of a page. 1590 */ 1591 int 1592 pmap_pvo_enter(pmap_t pm, struct pool *pl, struct pvo_head *pvo_head, 1593 vaddr_t va, paddr_t pa, register_t pte_lo, int flags) 1594 { 1595 struct pvo_entry *pvo; 1596 struct pvo_tqhead *pvoh; 1597 register_t msr; 1598 int ptegidx; 1599 int i; 1600 int poolflags = PR_NOWAIT; 1601 1602 /* 1603 * Compute the PTE Group index. 1604 */ 1605 va &= ~ADDR_POFF; 1606 ptegidx = va_to_pteg(pm, va); 1607 1608 msr = pmap_interrupts_off(); 1609 1610 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 1611 if (pmap_pvo_remove_depth > 0) 1612 panic("pmap_pvo_enter: called while pmap_pvo_remove active!"); 1613 if (++pmap_pvo_enter_depth > 1) 1614 panic("pmap_pvo_enter: called recursively!"); 1615 #endif 1616 1617 /* 1618 * Remove any existing mapping for this page. Reuse the 1619 * pvo entry if there a mapping. 1620 */ 1621 TAILQ_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) { 1622 if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) { 1623 #ifdef DEBUG 1624 if ((pmapdebug & PMAPDEBUG_PVOENTER) && 1625 ((pvo->pvo_pte.pte_lo ^ (pa|pte_lo)) & 1626 ~(PTE_REF|PTE_CHG)) == 0 && 1627 va < VM_MIN_KERNEL_ADDRESS) { 1628 printf("pmap_pvo_enter: pvo %p: dup %#x/%#lx\n", 1629 pvo, (unsigned int) pvo->pvo_pte.pte_lo, (unsigned int) pte_lo|pa); 1630 printf("pmap_pvo_enter: pte_hi=%#x sr=%#x\n", 1631 (unsigned int) pvo->pvo_pte.pte_hi, 1632 (unsigned int) pm->pm_sr[va >> ADDR_SR_SHFT]); 1633 pmap_pte_print(pmap_pvo_to_pte(pvo, -1)); 1634 #ifdef DDBX 1635 Debugger(); 1636 #endif 1637 } 1638 #endif 1639 PMAPCOUNT(mappings_replaced); 1640 pmap_pvo_remove(pvo, -1, NULL); 1641 break; 1642 } 1643 } 1644 1645 /* 1646 * If we aren't overwriting an mapping, try to allocate 1647 */ 1648 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 1649 --pmap_pvo_enter_depth; 1650 #endif 1651 pmap_interrupts_restore(msr); 1652 if (pvo) { 1653 pmap_pvo_free(pvo); 1654 } 1655 pvo = pool_get(pl, poolflags); 1656 1657 #ifdef DEBUG 1658 /* 1659 * Exercise pmap_pvo_reclaim() a little. 1660 */ 1661 if (pvo && (flags & PMAP_CANFAIL) != 0 && 1662 pmap_pvo_reclaim_debugctr++ > 0x1000 && 1663 (pmap_pvo_reclaim_debugctr & 0xff) == 0) { 1664 pool_put(pl, pvo); 1665 pvo = NULL; 1666 } 1667 #endif 1668 1669 msr = pmap_interrupts_off(); 1670 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 1671 ++pmap_pvo_enter_depth; 1672 #endif 1673 if (pvo == NULL) { 1674 pvo = pmap_pvo_reclaim(pm); 1675 if (pvo == NULL) { 1676 if ((flags & PMAP_CANFAIL) == 0) 1677 panic("pmap_pvo_enter: failed"); 1678 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 1679 pmap_pvo_enter_depth--; 1680 #endif 1681 PMAPCOUNT(pvos_failed); 1682 pmap_interrupts_restore(msr); 1683 return ENOMEM; 1684 } 1685 } 1686 1687 pvo->pvo_vaddr = va; 1688 pvo->pvo_pmap = pm; 1689 pvo->pvo_vaddr &= ~ADDR_POFF; 1690 if (flags & VM_PROT_EXECUTE) { 1691 PMAPCOUNT(exec_mappings); 1692 pvo_set_exec(pvo); 1693 } 1694 if (flags & PMAP_WIRED) 1695 pvo->pvo_vaddr |= PVO_WIRED; 1696 if (pvo_head != &pmap_pvo_kunmanaged) { 1697 pvo->pvo_vaddr |= PVO_MANAGED; 1698 PMAPCOUNT(mappings); 1699 } else { 1700 PMAPCOUNT(kernel_mappings); 1701 } 1702 pmap_pte_create(&pvo->pvo_pte, pm, va, pa | pte_lo); 1703 1704 LIST_INSERT_HEAD(pvo_head, pvo, pvo_vlink); 1705 if (pvo->pvo_pte.pte_lo & PVO_WIRED) 1706 pvo->pvo_pmap->pm_stats.wired_count++; 1707 pvo->pvo_pmap->pm_stats.resident_count++; 1708 #if defined(DEBUG) 1709 /* if (pm != pmap_kernel() && va < VM_MIN_KERNEL_ADDRESS) */ 1710 DPRINTFN(PVOENTER, 1711 ("pmap_pvo_enter: pvo %p: pm %p va %#lx pa %#lx\n", 1712 pvo, pm, va, pa)); 1713 #endif 1714 1715 /* 1716 * We hope this succeeds but it isn't required. 1717 */ 1718 pvoh = &pmap_pvo_table[ptegidx]; 1719 i = pmap_pte_insert(ptegidx, &pvo->pvo_pte); 1720 if (i >= 0) { 1721 PVO_PTEGIDX_SET(pvo, i); 1722 PVO_WHERE(pvo, ENTER_INSERT); 1723 PMAPCOUNT2(((pvo->pvo_pte.pte_hi & PTE_HID) 1724 ? pmap_evcnt_ptes_secondary : pmap_evcnt_ptes_primary)[i]); 1725 TAILQ_INSERT_TAIL(pvoh, pvo, pvo_olink); 1726 1727 } else { 1728 /* 1729 * Since we didn't have room for this entry (which makes it 1730 * and evicted entry), place it at the head of the list. 1731 */ 1732 TAILQ_INSERT_HEAD(pvoh, pvo, pvo_olink); 1733 PMAPCOUNT(ptes_evicted); 1734 pm->pm_evictions++; 1735 /* 1736 * If this is a kernel page, make sure it's active. 1737 */ 1738 if (pm == pmap_kernel()) { 1739 i = pmap_pte_spill(pm, va, FALSE); 1740 KASSERT(i); 1741 } 1742 } 1743 PMAP_PVO_CHECK(pvo); /* sanity check */ 1744 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 1745 pmap_pvo_enter_depth--; 1746 #endif 1747 pmap_interrupts_restore(msr); 1748 return 0; 1749 } 1750 1751 void 1752 pmap_pvo_remove(struct pvo_entry *pvo, int pteidx, struct pvo_head *pvol) 1753 { 1754 volatile struct pte *pt; 1755 int ptegidx; 1756 1757 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 1758 if (++pmap_pvo_remove_depth > 1) 1759 panic("pmap_pvo_remove: called recursively!"); 1760 #endif 1761 1762 /* 1763 * If we haven't been supplied the ptegidx, calculate it. 1764 */ 1765 if (pteidx == -1) { 1766 ptegidx = va_to_pteg(pvo->pvo_pmap, pvo->pvo_vaddr); 1767 pteidx = pmap_pvo_pte_index(pvo, ptegidx); 1768 } else { 1769 ptegidx = pteidx >> 3; 1770 if (pvo->pvo_pte.pte_hi & PTE_HID) 1771 ptegidx ^= pmap_pteg_mask; 1772 } 1773 PMAP_PVO_CHECK(pvo); /* sanity check */ 1774 1775 /* 1776 * If there is an active pte entry, we need to deactivate it 1777 * (and save the ref & chg bits). 1778 */ 1779 pt = pmap_pvo_to_pte(pvo, pteidx); 1780 if (pt != NULL) { 1781 pmap_pte_unset(pt, &pvo->pvo_pte, pvo->pvo_vaddr); 1782 PVO_WHERE(pvo, REMOVE); 1783 PVO_PTEGIDX_CLR(pvo); 1784 PMAPCOUNT(ptes_removed); 1785 } else { 1786 KASSERT(pvo->pvo_pmap->pm_evictions > 0); 1787 pvo->pvo_pmap->pm_evictions--; 1788 } 1789 1790 /* 1791 * Account for executable mappings. 1792 */ 1793 if (PVO_ISEXECUTABLE(pvo)) 1794 pvo_clear_exec(pvo); 1795 1796 /* 1797 * Update our statistics. 1798 */ 1799 pvo->pvo_pmap->pm_stats.resident_count--; 1800 if (pvo->pvo_pte.pte_lo & PVO_WIRED) 1801 pvo->pvo_pmap->pm_stats.wired_count--; 1802 1803 /* 1804 * Save the REF/CHG bits into their cache if the page is managed. 1805 */ 1806 if (pvo->pvo_vaddr & PVO_MANAGED) { 1807 register_t ptelo = pvo->pvo_pte.pte_lo; 1808 struct vm_page *pg = PHYS_TO_VM_PAGE(ptelo & PTE_RPGN); 1809 1810 if (pg != NULL) { 1811 /* 1812 * If this page was changed and it is mapped exec, 1813 * invalidate it. 1814 */ 1815 if ((ptelo & PTE_CHG) && 1816 (pmap_attr_fetch(pg) & PTE_EXEC)) { 1817 struct pvo_head *pvoh = vm_page_to_pvoh(pg); 1818 if (LIST_EMPTY(pvoh)) { 1819 DPRINTFN(EXEC, ("[pmap_pvo_remove: " 1820 "%#lx: clear-exec]\n", 1821 VM_PAGE_TO_PHYS(pg))); 1822 pmap_attr_clear(pg, PTE_EXEC); 1823 PMAPCOUNT(exec_uncached_pvo_remove); 1824 } else { 1825 DPRINTFN(EXEC, ("[pmap_pvo_remove: " 1826 "%#lx: syncicache]\n", 1827 VM_PAGE_TO_PHYS(pg))); 1828 pmap_syncicache(VM_PAGE_TO_PHYS(pg), 1829 PAGE_SIZE); 1830 PMAPCOUNT(exec_synced_pvo_remove); 1831 } 1832 } 1833 1834 pmap_attr_save(pg, ptelo & (PTE_REF|PTE_CHG)); 1835 } 1836 PMAPCOUNT(unmappings); 1837 } else { 1838 PMAPCOUNT(kernel_unmappings); 1839 } 1840 1841 /* 1842 * Remove the PVO from its lists and return it to the pool. 1843 */ 1844 LIST_REMOVE(pvo, pvo_vlink); 1845 TAILQ_REMOVE(&pmap_pvo_table[ptegidx], pvo, pvo_olink); 1846 if (pvol) { 1847 LIST_INSERT_HEAD(pvol, pvo, pvo_vlink); 1848 } 1849 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 1850 pmap_pvo_remove_depth--; 1851 #endif 1852 } 1853 1854 void 1855 pmap_pvo_free(struct pvo_entry *pvo) 1856 { 1857 1858 pool_put(pvo->pvo_vaddr & PVO_MANAGED ? &pmap_mpvo_pool : 1859 &pmap_upvo_pool, pvo); 1860 } 1861 1862 void 1863 pmap_pvo_free_list(struct pvo_head *pvol) 1864 { 1865 struct pvo_entry *pvo, *npvo; 1866 1867 for (pvo = LIST_FIRST(pvol); pvo != NULL; pvo = npvo) { 1868 npvo = LIST_NEXT(pvo, pvo_vlink); 1869 LIST_REMOVE(pvo, pvo_vlink); 1870 pmap_pvo_free(pvo); 1871 } 1872 } 1873 1874 /* 1875 * Mark a mapping as executable. 1876 * If this is the first executable mapping in the segment, 1877 * clear the noexec flag. 1878 */ 1879 STATIC void 1880 pvo_set_exec(struct pvo_entry *pvo) 1881 { 1882 struct pmap *pm = pvo->pvo_pmap; 1883 1884 if (pm == pmap_kernel() || PVO_ISEXECUTABLE(pvo)) { 1885 return; 1886 } 1887 pvo->pvo_vaddr |= PVO_EXECUTABLE; 1888 #if defined (PPC_OEA) || defined (PPC_OEA64_BRIDGE) 1889 { 1890 int sr = PVO_VADDR(pvo) >> ADDR_SR_SHFT; 1891 if (pm->pm_exec[sr]++ == 0) { 1892 pm->pm_sr[sr] &= ~SR_NOEXEC; 1893 } 1894 } 1895 #endif 1896 } 1897 1898 /* 1899 * Mark a mapping as non-executable. 1900 * If this was the last executable mapping in the segment, 1901 * set the noexec flag. 1902 */ 1903 STATIC void 1904 pvo_clear_exec(struct pvo_entry *pvo) 1905 { 1906 struct pmap *pm = pvo->pvo_pmap; 1907 1908 if (pm == pmap_kernel() || !PVO_ISEXECUTABLE(pvo)) { 1909 return; 1910 } 1911 pvo->pvo_vaddr &= ~PVO_EXECUTABLE; 1912 #if defined (PPC_OEA) || defined (PPC_OEA64_BRIDGE) 1913 { 1914 int sr = PVO_VADDR(pvo) >> ADDR_SR_SHFT; 1915 if (--pm->pm_exec[sr] == 0) { 1916 pm->pm_sr[sr] |= SR_NOEXEC; 1917 } 1918 } 1919 #endif 1920 } 1921 1922 /* 1923 * Insert physical page at pa into the given pmap at virtual address va. 1924 */ 1925 int 1926 pmap_enter(pmap_t pm, vaddr_t va, paddr_t pa, vm_prot_t prot, int flags) 1927 { 1928 struct mem_region *mp; 1929 struct pvo_head *pvo_head; 1930 struct vm_page *pg; 1931 struct pool *pl; 1932 register_t pte_lo; 1933 int error; 1934 u_int pvo_flags; 1935 u_int was_exec = 0; 1936 1937 if (__predict_false(!pmap_initialized)) { 1938 pvo_head = &pmap_pvo_kunmanaged; 1939 pl = &pmap_upvo_pool; 1940 pvo_flags = 0; 1941 pg = NULL; 1942 was_exec = PTE_EXEC; 1943 } else { 1944 pvo_head = pa_to_pvoh(pa, &pg); 1945 pl = &pmap_mpvo_pool; 1946 pvo_flags = PVO_MANAGED; 1947 } 1948 1949 DPRINTFN(ENTER, 1950 ("pmap_enter(%p, 0x%lx, 0x%lx, 0x%x, 0x%x):", 1951 pm, va, pa, prot, flags)); 1952 1953 /* 1954 * If this is a managed page, and it's the first reference to the 1955 * page clear the execness of the page. Otherwise fetch the execness. 1956 */ 1957 if (pg != NULL) 1958 was_exec = pmap_attr_fetch(pg) & PTE_EXEC; 1959 1960 DPRINTFN(ENTER, (" was_exec=%d", was_exec)); 1961 1962 /* 1963 * Assume the page is cache inhibited and access is guarded unless 1964 * it's in our available memory array. If it is in the memory array, 1965 * asssume it's in memory coherent memory. 1966 */ 1967 pte_lo = PTE_IG; 1968 if ((flags & PMAP_NC) == 0) { 1969 for (mp = mem; mp->size; mp++) { 1970 if (pa >= mp->start && pa < mp->start + mp->size) { 1971 pte_lo = PTE_M; 1972 break; 1973 } 1974 } 1975 } 1976 1977 if (prot & VM_PROT_WRITE) 1978 pte_lo |= PTE_BW; 1979 else 1980 pte_lo |= PTE_BR; 1981 1982 /* 1983 * If this was in response to a fault, "pre-fault" the PTE's 1984 * changed/referenced bit appropriately. 1985 */ 1986 if (flags & VM_PROT_WRITE) 1987 pte_lo |= PTE_CHG; 1988 if (flags & VM_PROT_ALL) 1989 pte_lo |= PTE_REF; 1990 1991 /* 1992 * We need to know if this page can be executable 1993 */ 1994 flags |= (prot & VM_PROT_EXECUTE); 1995 1996 /* 1997 * Record mapping for later back-translation and pte spilling. 1998 * This will overwrite any existing mapping. 1999 */ 2000 error = pmap_pvo_enter(pm, pl, pvo_head, va, pa, pte_lo, flags); 2001 2002 /* 2003 * Flush the real page from the instruction cache if this page is 2004 * mapped executable and cacheable and has not been flushed since 2005 * the last time it was modified. 2006 */ 2007 if (error == 0 && 2008 (flags & VM_PROT_EXECUTE) && 2009 (pte_lo & PTE_I) == 0 && 2010 was_exec == 0) { 2011 DPRINTFN(ENTER, (" syncicache")); 2012 PMAPCOUNT(exec_synced); 2013 pmap_syncicache(pa, PAGE_SIZE); 2014 if (pg != NULL) { 2015 pmap_attr_save(pg, PTE_EXEC); 2016 PMAPCOUNT(exec_cached); 2017 #if defined(DEBUG) || defined(PMAPDEBUG) 2018 if (pmapdebug & PMAPDEBUG_ENTER) 2019 printf(" marked-as-exec"); 2020 else if (pmapdebug & PMAPDEBUG_EXEC) 2021 printf("[pmap_enter: %#lx: marked-as-exec]\n", 2022 VM_PAGE_TO_PHYS(pg)); 2023 2024 #endif 2025 } 2026 } 2027 2028 DPRINTFN(ENTER, (": error=%d\n", error)); 2029 2030 return error; 2031 } 2032 2033 void 2034 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot) 2035 { 2036 struct mem_region *mp; 2037 register_t pte_lo; 2038 int error; 2039 2040 #if defined (PPC_OEA64_BRIDGE) 2041 if (va < VM_MIN_KERNEL_ADDRESS) 2042 panic("pmap_kenter_pa: attempt to enter " 2043 "non-kernel address %#lx!", va); 2044 #endif 2045 2046 DPRINTFN(KENTER, 2047 ("pmap_kenter_pa(%#lx,%#lx,%#x)\n", va, pa, prot)); 2048 2049 /* 2050 * Assume the page is cache inhibited and access is guarded unless 2051 * it's in our available memory array. If it is in the memory array, 2052 * asssume it's in memory coherent memory. 2053 */ 2054 pte_lo = PTE_IG; 2055 if ((prot & PMAP_NC) == 0) { 2056 for (mp = mem; mp->size; mp++) { 2057 if (pa >= mp->start && pa < mp->start + mp->size) { 2058 pte_lo = PTE_M; 2059 break; 2060 } 2061 } 2062 } 2063 2064 if (prot & VM_PROT_WRITE) 2065 pte_lo |= PTE_BW; 2066 else 2067 pte_lo |= PTE_BR; 2068 2069 /* 2070 * We don't care about REF/CHG on PVOs on the unmanaged list. 2071 */ 2072 error = pmap_pvo_enter(pmap_kernel(), &pmap_upvo_pool, 2073 &pmap_pvo_kunmanaged, va, pa, pte_lo, prot|PMAP_WIRED); 2074 2075 if (error != 0) 2076 panic("pmap_kenter_pa: failed to enter va %#lx pa %#lx: %d", 2077 va, pa, error); 2078 } 2079 2080 void 2081 pmap_kremove(vaddr_t va, vsize_t len) 2082 { 2083 if (va < VM_MIN_KERNEL_ADDRESS) 2084 panic("pmap_kremove: attempt to remove " 2085 "non-kernel address %#lx!", va); 2086 2087 DPRINTFN(KREMOVE,("pmap_kremove(%#lx,%#lx)\n", va, len)); 2088 pmap_remove(pmap_kernel(), va, va + len); 2089 } 2090 2091 /* 2092 * Remove the given range of mapping entries. 2093 */ 2094 void 2095 pmap_remove(pmap_t pm, vaddr_t va, vaddr_t endva) 2096 { 2097 struct pvo_head pvol; 2098 struct pvo_entry *pvo; 2099 register_t msr; 2100 int pteidx; 2101 2102 LIST_INIT(&pvol); 2103 msr = pmap_interrupts_off(); 2104 for (; va < endva; va += PAGE_SIZE) { 2105 pvo = pmap_pvo_find_va(pm, va, &pteidx); 2106 if (pvo != NULL) { 2107 pmap_pvo_remove(pvo, pteidx, &pvol); 2108 } 2109 } 2110 pmap_interrupts_restore(msr); 2111 pmap_pvo_free_list(&pvol); 2112 } 2113 2114 /* 2115 * Get the physical page address for the given pmap/virtual address. 2116 */ 2117 boolean_t 2118 pmap_extract(pmap_t pm, vaddr_t va, paddr_t *pap) 2119 { 2120 struct pvo_entry *pvo; 2121 register_t msr; 2122 2123 2124 /* 2125 * If this is a kernel pmap lookup, also check the battable 2126 * and if we get a hit, translate the VA to a PA using the 2127 * BAT entries. Don't check for VM_MAX_KERNEL_ADDRESS is 2128 * that will wrap back to 0. 2129 */ 2130 if (pm == pmap_kernel() && 2131 (va < VM_MIN_KERNEL_ADDRESS || 2132 (KERNEL2_SR < 15 && VM_MAX_KERNEL_ADDRESS <= va))) { 2133 KASSERT((va >> ADDR_SR_SHFT) != USER_SR); 2134 #if defined (PPC_OEA) 2135 if ((MFPVR() >> 16) != MPC601) { 2136 register_t batu = battable[va >> ADDR_SR_SHFT].batu; 2137 if (BAT_VALID_P(batu,0) && BAT_VA_MATCH_P(batu,va)) { 2138 register_t batl = 2139 battable[va >> ADDR_SR_SHFT].batl; 2140 register_t mask = 2141 (~(batu & BAT_BL) << 15) & ~0x1ffffL; 2142 if (pap) 2143 *pap = (batl & mask) | (va & ~mask); 2144 return TRUE; 2145 } 2146 } else { 2147 register_t batu = battable[va >> 23].batu; 2148 register_t batl = battable[va >> 23].batl; 2149 register_t sr = iosrtable[va >> ADDR_SR_SHFT]; 2150 if (BAT601_VALID_P(batl) && 2151 BAT601_VA_MATCH_P(batu, batl, va)) { 2152 register_t mask = 2153 (~(batl & BAT601_BSM) << 17) & ~0x1ffffL; 2154 if (pap) 2155 *pap = (batl & mask) | (va & ~mask); 2156 return TRUE; 2157 } else if (SR601_VALID_P(sr) && 2158 SR601_PA_MATCH_P(sr, va)) { 2159 if (pap) 2160 *pap = va; 2161 return TRUE; 2162 } 2163 } 2164 return FALSE; 2165 #elif defined (PPC_OEA64_BRIDGE) 2166 panic("%s: pm: %s, va: 0x%08lx\n", __FUNCTION__, 2167 (pm == pmap_kernel() ? "kernel" : "user"), va); 2168 #elif defined (PPC_OEA64) 2169 #error PPC_OEA64 not supported 2170 #endif /* PPC_OEA */ 2171 } 2172 2173 msr = pmap_interrupts_off(); 2174 pvo = pmap_pvo_find_va(pm, va & ~ADDR_POFF, NULL); 2175 if (pvo != NULL) { 2176 PMAP_PVO_CHECK(pvo); /* sanity check */ 2177 if (pap) 2178 *pap = (pvo->pvo_pte.pte_lo & PTE_RPGN) 2179 | (va & ADDR_POFF); 2180 } 2181 pmap_interrupts_restore(msr); 2182 return pvo != NULL; 2183 } 2184 2185 /* 2186 * Lower the protection on the specified range of this pmap. 2187 */ 2188 void 2189 pmap_protect(pmap_t pm, vaddr_t va, vaddr_t endva, vm_prot_t prot) 2190 { 2191 struct pvo_entry *pvo; 2192 volatile struct pte *pt; 2193 register_t msr; 2194 int pteidx; 2195 2196 /* 2197 * Since this routine only downgrades protection, we should 2198 * always be called with at least one bit not set. 2199 */ 2200 KASSERT(prot != VM_PROT_ALL); 2201 2202 /* 2203 * If there is no protection, this is equivalent to 2204 * remove the pmap from the pmap. 2205 */ 2206 if ((prot & VM_PROT_READ) == 0) { 2207 pmap_remove(pm, va, endva); 2208 return; 2209 } 2210 2211 msr = pmap_interrupts_off(); 2212 for (; va < endva; va += PAGE_SIZE) { 2213 pvo = pmap_pvo_find_va(pm, va, &pteidx); 2214 if (pvo == NULL) 2215 continue; 2216 PMAP_PVO_CHECK(pvo); /* sanity check */ 2217 2218 /* 2219 * Revoke executable if asked to do so. 2220 */ 2221 if ((prot & VM_PROT_EXECUTE) == 0) 2222 pvo_clear_exec(pvo); 2223 2224 #if 0 2225 /* 2226 * If the page is already read-only, no change 2227 * needs to be made. 2228 */ 2229 if ((pvo->pvo_pte.pte_lo & PTE_PP) == PTE_BR) 2230 continue; 2231 #endif 2232 /* 2233 * Grab the PTE pointer before we diddle with 2234 * the cached PTE copy. 2235 */ 2236 pt = pmap_pvo_to_pte(pvo, pteidx); 2237 /* 2238 * Change the protection of the page. 2239 */ 2240 pvo->pvo_pte.pte_lo &= ~PTE_PP; 2241 pvo->pvo_pte.pte_lo |= PTE_BR; 2242 2243 /* 2244 * If the PVO is in the page table, update 2245 * that pte at well. 2246 */ 2247 if (pt != NULL) { 2248 pmap_pte_change(pt, &pvo->pvo_pte, pvo->pvo_vaddr); 2249 PVO_WHERE(pvo, PMAP_PROTECT); 2250 PMAPCOUNT(ptes_changed); 2251 } 2252 2253 PMAP_PVO_CHECK(pvo); /* sanity check */ 2254 } 2255 pmap_interrupts_restore(msr); 2256 } 2257 2258 void 2259 pmap_unwire(pmap_t pm, vaddr_t va) 2260 { 2261 struct pvo_entry *pvo; 2262 register_t msr; 2263 2264 msr = pmap_interrupts_off(); 2265 pvo = pmap_pvo_find_va(pm, va, NULL); 2266 if (pvo != NULL) { 2267 if (pvo->pvo_vaddr & PVO_WIRED) { 2268 pvo->pvo_vaddr &= ~PVO_WIRED; 2269 pm->pm_stats.wired_count--; 2270 } 2271 PMAP_PVO_CHECK(pvo); /* sanity check */ 2272 } 2273 pmap_interrupts_restore(msr); 2274 } 2275 2276 /* 2277 * Lower the protection on the specified physical page. 2278 */ 2279 void 2280 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 2281 { 2282 struct pvo_head *pvo_head, pvol; 2283 struct pvo_entry *pvo, *next_pvo; 2284 volatile struct pte *pt; 2285 register_t msr; 2286 2287 KASSERT(prot != VM_PROT_ALL); 2288 LIST_INIT(&pvol); 2289 msr = pmap_interrupts_off(); 2290 2291 /* 2292 * When UVM reuses a page, it does a pmap_page_protect with 2293 * VM_PROT_NONE. At that point, we can clear the exec flag 2294 * since we know the page will have different contents. 2295 */ 2296 if ((prot & VM_PROT_READ) == 0) { 2297 DPRINTFN(EXEC, ("[pmap_page_protect: %#lx: clear-exec]\n", 2298 VM_PAGE_TO_PHYS(pg))); 2299 if (pmap_attr_fetch(pg) & PTE_EXEC) { 2300 PMAPCOUNT(exec_uncached_page_protect); 2301 pmap_attr_clear(pg, PTE_EXEC); 2302 } 2303 } 2304 2305 pvo_head = vm_page_to_pvoh(pg); 2306 for (pvo = LIST_FIRST(pvo_head); pvo != NULL; pvo = next_pvo) { 2307 next_pvo = LIST_NEXT(pvo, pvo_vlink); 2308 PMAP_PVO_CHECK(pvo); /* sanity check */ 2309 2310 /* 2311 * Downgrading to no mapping at all, we just remove the entry. 2312 */ 2313 if ((prot & VM_PROT_READ) == 0) { 2314 pmap_pvo_remove(pvo, -1, &pvol); 2315 continue; 2316 } 2317 2318 /* 2319 * If EXEC permission is being revoked, just clear the 2320 * flag in the PVO. 2321 */ 2322 if ((prot & VM_PROT_EXECUTE) == 0) 2323 pvo_clear_exec(pvo); 2324 2325 /* 2326 * If this entry is already RO, don't diddle with the 2327 * page table. 2328 */ 2329 if ((pvo->pvo_pte.pte_lo & PTE_PP) == PTE_BR) { 2330 PMAP_PVO_CHECK(pvo); 2331 continue; 2332 } 2333 2334 /* 2335 * Grab the PTE before the we diddle the bits so 2336 * pvo_to_pte can verify the pte contents are as 2337 * expected. 2338 */ 2339 pt = pmap_pvo_to_pte(pvo, -1); 2340 pvo->pvo_pte.pte_lo &= ~PTE_PP; 2341 pvo->pvo_pte.pte_lo |= PTE_BR; 2342 if (pt != NULL) { 2343 pmap_pte_change(pt, &pvo->pvo_pte, pvo->pvo_vaddr); 2344 PVO_WHERE(pvo, PMAP_PAGE_PROTECT); 2345 PMAPCOUNT(ptes_changed); 2346 } 2347 PMAP_PVO_CHECK(pvo); /* sanity check */ 2348 } 2349 pmap_interrupts_restore(msr); 2350 pmap_pvo_free_list(&pvol); 2351 } 2352 2353 /* 2354 * Activate the address space for the specified process. If the process 2355 * is the current process, load the new MMU context. 2356 */ 2357 void 2358 pmap_activate(struct lwp *l) 2359 { 2360 struct pcb *pcb = &l->l_addr->u_pcb; 2361 pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap; 2362 2363 DPRINTFN(ACTIVATE, 2364 ("pmap_activate: lwp %p (curlwp %p)\n", l, curlwp)); 2365 2366 /* 2367 * XXX Normally performed in cpu_fork(). 2368 */ 2369 pcb->pcb_pm = pmap; 2370 2371 /* 2372 * In theory, the SR registers need only be valid on return 2373 * to user space wait to do them there. 2374 */ 2375 if (l == curlwp) { 2376 /* Store pointer to new current pmap. */ 2377 curpm = pmap; 2378 } 2379 } 2380 2381 /* 2382 * Deactivate the specified process's address space. 2383 */ 2384 void 2385 pmap_deactivate(struct lwp *l) 2386 { 2387 } 2388 2389 boolean_t 2390 pmap_query_bit(struct vm_page *pg, int ptebit) 2391 { 2392 struct pvo_entry *pvo; 2393 volatile struct pte *pt; 2394 register_t msr; 2395 2396 if (pmap_attr_fetch(pg) & ptebit) 2397 return TRUE; 2398 2399 msr = pmap_interrupts_off(); 2400 LIST_FOREACH(pvo, vm_page_to_pvoh(pg), pvo_vlink) { 2401 PMAP_PVO_CHECK(pvo); /* sanity check */ 2402 /* 2403 * See if we saved the bit off. If so cache, it and return 2404 * success. 2405 */ 2406 if (pvo->pvo_pte.pte_lo & ptebit) { 2407 pmap_attr_save(pg, ptebit); 2408 PMAP_PVO_CHECK(pvo); /* sanity check */ 2409 pmap_interrupts_restore(msr); 2410 return TRUE; 2411 } 2412 } 2413 /* 2414 * No luck, now go thru the hard part of looking at the ptes 2415 * themselves. Sync so any pending REF/CHG bits are flushed 2416 * to the PTEs. 2417 */ 2418 SYNC(); 2419 LIST_FOREACH(pvo, vm_page_to_pvoh(pg), pvo_vlink) { 2420 PMAP_PVO_CHECK(pvo); /* sanity check */ 2421 /* 2422 * See if this pvo have a valid PTE. If so, fetch the 2423 * REF/CHG bits from the valid PTE. If the appropriate 2424 * ptebit is set, cache, it and return success. 2425 */ 2426 pt = pmap_pvo_to_pte(pvo, -1); 2427 if (pt != NULL) { 2428 pmap_pte_synch(pt, &pvo->pvo_pte); 2429 if (pvo->pvo_pte.pte_lo & ptebit) { 2430 pmap_attr_save(pg, ptebit); 2431 PMAP_PVO_CHECK(pvo); /* sanity check */ 2432 pmap_interrupts_restore(msr); 2433 return TRUE; 2434 } 2435 } 2436 } 2437 pmap_interrupts_restore(msr); 2438 return FALSE; 2439 } 2440 2441 boolean_t 2442 pmap_clear_bit(struct vm_page *pg, int ptebit) 2443 { 2444 struct pvo_head *pvoh = vm_page_to_pvoh(pg); 2445 struct pvo_entry *pvo; 2446 volatile struct pte *pt; 2447 register_t msr; 2448 int rv = 0; 2449 2450 msr = pmap_interrupts_off(); 2451 2452 /* 2453 * Fetch the cache value 2454 */ 2455 rv |= pmap_attr_fetch(pg); 2456 2457 /* 2458 * Clear the cached value. 2459 */ 2460 pmap_attr_clear(pg, ptebit); 2461 2462 /* 2463 * Sync so any pending REF/CHG bits are flushed to the PTEs (so we 2464 * can reset the right ones). Note that since the pvo entries and 2465 * list heads are accessed via BAT0 and are never placed in the 2466 * page table, we don't have to worry about further accesses setting 2467 * the REF/CHG bits. 2468 */ 2469 SYNC(); 2470 2471 /* 2472 * For each pvo entry, clear pvo's ptebit. If this pvo have a 2473 * valid PTE. If so, clear the ptebit from the valid PTE. 2474 */ 2475 LIST_FOREACH(pvo, pvoh, pvo_vlink) { 2476 PMAP_PVO_CHECK(pvo); /* sanity check */ 2477 pt = pmap_pvo_to_pte(pvo, -1); 2478 if (pt != NULL) { 2479 /* 2480 * Only sync the PTE if the bit we are looking 2481 * for is not already set. 2482 */ 2483 if ((pvo->pvo_pte.pte_lo & ptebit) == 0) 2484 pmap_pte_synch(pt, &pvo->pvo_pte); 2485 /* 2486 * If the bit we are looking for was already set, 2487 * clear that bit in the pte. 2488 */ 2489 if (pvo->pvo_pte.pte_lo & ptebit) 2490 pmap_pte_clear(pt, PVO_VADDR(pvo), ptebit); 2491 } 2492 rv |= pvo->pvo_pte.pte_lo & (PTE_CHG|PTE_REF); 2493 pvo->pvo_pte.pte_lo &= ~ptebit; 2494 PMAP_PVO_CHECK(pvo); /* sanity check */ 2495 } 2496 pmap_interrupts_restore(msr); 2497 2498 /* 2499 * If we are clearing the modify bit and this page was marked EXEC 2500 * and the user of the page thinks the page was modified, then we 2501 * need to clean it from the icache if it's mapped or clear the EXEC 2502 * bit if it's not mapped. The page itself might not have the CHG 2503 * bit set if the modification was done via DMA to the page. 2504 */ 2505 if ((ptebit & PTE_CHG) && (rv & PTE_EXEC)) { 2506 if (LIST_EMPTY(pvoh)) { 2507 DPRINTFN(EXEC, ("[pmap_clear_bit: %#lx: clear-exec]\n", 2508 VM_PAGE_TO_PHYS(pg))); 2509 pmap_attr_clear(pg, PTE_EXEC); 2510 PMAPCOUNT(exec_uncached_clear_modify); 2511 } else { 2512 DPRINTFN(EXEC, ("[pmap_clear_bit: %#lx: syncicache]\n", 2513 VM_PAGE_TO_PHYS(pg))); 2514 pmap_syncicache(VM_PAGE_TO_PHYS(pg), PAGE_SIZE); 2515 PMAPCOUNT(exec_synced_clear_modify); 2516 } 2517 } 2518 return (rv & ptebit) != 0; 2519 } 2520 2521 void 2522 pmap_procwr(struct proc *p, vaddr_t va, size_t len) 2523 { 2524 struct pvo_entry *pvo; 2525 size_t offset = va & ADDR_POFF; 2526 int s; 2527 2528 s = splvm(); 2529 while (len > 0) { 2530 size_t seglen = PAGE_SIZE - offset; 2531 if (seglen > len) 2532 seglen = len; 2533 pvo = pmap_pvo_find_va(p->p_vmspace->vm_map.pmap, va, NULL); 2534 if (pvo != NULL && PVO_ISEXECUTABLE(pvo)) { 2535 pmap_syncicache( 2536 (pvo->pvo_pte.pte_lo & PTE_RPGN) | offset, seglen); 2537 PMAP_PVO_CHECK(pvo); 2538 } 2539 va += seglen; 2540 len -= seglen; 2541 offset = 0; 2542 } 2543 splx(s); 2544 } 2545 2546 #if defined(DEBUG) || defined(PMAPCHECK) || defined(DDB) 2547 void 2548 pmap_pte_print(volatile struct pte *pt) 2549 { 2550 printf("PTE %p: ", pt); 2551 2552 #if defined(PPC_OEA) 2553 /* High word: */ 2554 printf("0x%08lx: [", pt->pte_hi); 2555 #elif defined (PPC_OEA64_BRIDGE) 2556 printf("0x%016llx: [", pt->pte_hi); 2557 #else /* PPC_OEA64 */ 2558 printf("0x%016lx: [", pt->pte_hi); 2559 #endif /* PPC_OEA */ 2560 2561 printf("%c ", (pt->pte_hi & PTE_VALID) ? 'v' : 'i'); 2562 printf("%c ", (pt->pte_hi & PTE_HID) ? 'h' : '-'); 2563 2564 #if defined (PPC_OEA) 2565 printf("0x%06lx 0x%02lx", 2566 (pt->pte_hi &~ PTE_VALID)>>PTE_VSID_SHFT, 2567 pt->pte_hi & PTE_API); 2568 printf(" (va 0x%08lx)] ", pmap_pte_to_va(pt)); 2569 #elif defined (PPC_OEA64) 2570 printf("0x%06lx 0x%02lx", 2571 (pt->pte_hi &~ PTE_VALID)>>PTE_VSID_SHFT, 2572 pt->pte_hi & PTE_API); 2573 printf(" (va 0x%016lx)] ", pmap_pte_to_va(pt)); 2574 #else 2575 /* PPC_OEA64_BRIDGE */ 2576 printf("0x%06llx 0x%02llx", 2577 (pt->pte_hi &~ PTE_VALID)>>PTE_VSID_SHFT, 2578 pt->pte_hi & PTE_API); 2579 printf(" (va 0x%08lx)] ", pmap_pte_to_va(pt)); 2580 #endif /* PPC_OEA */ 2581 2582 /* Low word: */ 2583 #if defined (PPC_OEA) 2584 printf(" 0x%08lx: [", pt->pte_lo); 2585 printf("0x%05lx... ", pt->pte_lo >> 12); 2586 #elif defined (PPC_OEA64) 2587 printf(" 0x%016lx: [", pt->pte_lo); 2588 printf("0x%012lx... ", pt->pte_lo >> 12); 2589 #else /* PPC_OEA64_BRIDGE */ 2590 printf(" 0x%016llx: [", pt->pte_lo); 2591 printf("0x%012llx... ", pt->pte_lo >> 12); 2592 #endif 2593 printf("%c ", (pt->pte_lo & PTE_REF) ? 'r' : 'u'); 2594 printf("%c ", (pt->pte_lo & PTE_CHG) ? 'c' : 'n'); 2595 printf("%c", (pt->pte_lo & PTE_W) ? 'w' : '.'); 2596 printf("%c", (pt->pte_lo & PTE_I) ? 'i' : '.'); 2597 printf("%c", (pt->pte_lo & PTE_M) ? 'm' : '.'); 2598 printf("%c ", (pt->pte_lo & PTE_G) ? 'g' : '.'); 2599 switch (pt->pte_lo & PTE_PP) { 2600 case PTE_BR: printf("br]\n"); break; 2601 case PTE_BW: printf("bw]\n"); break; 2602 case PTE_SO: printf("so]\n"); break; 2603 case PTE_SW: printf("sw]\n"); break; 2604 } 2605 } 2606 #endif 2607 2608 #if defined(DDB) 2609 void 2610 pmap_pteg_check(void) 2611 { 2612 volatile struct pte *pt; 2613 int i; 2614 int ptegidx; 2615 u_int p_valid = 0; 2616 u_int s_valid = 0; 2617 u_int invalid = 0; 2618 2619 for (ptegidx = 0; ptegidx < pmap_pteg_cnt; ptegidx++) { 2620 for (pt = pmap_pteg_table[ptegidx].pt, i = 8; --i >= 0; pt++) { 2621 if (pt->pte_hi & PTE_VALID) { 2622 if (pt->pte_hi & PTE_HID) 2623 s_valid++; 2624 else 2625 { 2626 p_valid++; 2627 } 2628 } else 2629 invalid++; 2630 } 2631 } 2632 printf("pteg_check: v(p) %#x (%d), v(s) %#x (%d), i %#x (%d)\n", 2633 p_valid, p_valid, s_valid, s_valid, 2634 invalid, invalid); 2635 } 2636 2637 void 2638 pmap_print_mmuregs(void) 2639 { 2640 int i; 2641 u_int cpuvers; 2642 #ifndef PPC_OEA64 2643 vaddr_t addr; 2644 register_t soft_sr[16]; 2645 #endif 2646 #if defined (PPC_OEA) && !defined (PPC_OEA64) && !defined (PPC_OEA64_BRIDGE) 2647 struct bat soft_ibat[4]; 2648 struct bat soft_dbat[4]; 2649 #endif 2650 register_t sdr1; 2651 2652 cpuvers = MFPVR() >> 16; 2653 __asm volatile ("mfsdr1 %0" : "=r"(sdr1)); 2654 #ifndef PPC_OEA64 2655 addr = 0; 2656 for (i = 0; i < 16; i++) { 2657 soft_sr[i] = MFSRIN(addr); 2658 addr += (1 << ADDR_SR_SHFT); 2659 } 2660 #endif 2661 2662 #if defined(PPC_OEA) && !defined (PPC_OEA64) && !defined (PPC_OEA64_BRIDGE) 2663 /* read iBAT (601: uBAT) registers */ 2664 __asm volatile ("mfibatu %0,0" : "=r"(soft_ibat[0].batu)); 2665 __asm volatile ("mfibatl %0,0" : "=r"(soft_ibat[0].batl)); 2666 __asm volatile ("mfibatu %0,1" : "=r"(soft_ibat[1].batu)); 2667 __asm volatile ("mfibatl %0,1" : "=r"(soft_ibat[1].batl)); 2668 __asm volatile ("mfibatu %0,2" : "=r"(soft_ibat[2].batu)); 2669 __asm volatile ("mfibatl %0,2" : "=r"(soft_ibat[2].batl)); 2670 __asm volatile ("mfibatu %0,3" : "=r"(soft_ibat[3].batu)); 2671 __asm volatile ("mfibatl %0,3" : "=r"(soft_ibat[3].batl)); 2672 2673 2674 if (cpuvers != MPC601) { 2675 /* read dBAT registers */ 2676 __asm volatile ("mfdbatu %0,0" : "=r"(soft_dbat[0].batu)); 2677 __asm volatile ("mfdbatl %0,0" : "=r"(soft_dbat[0].batl)); 2678 __asm volatile ("mfdbatu %0,1" : "=r"(soft_dbat[1].batu)); 2679 __asm volatile ("mfdbatl %0,1" : "=r"(soft_dbat[1].batl)); 2680 __asm volatile ("mfdbatu %0,2" : "=r"(soft_dbat[2].batu)); 2681 __asm volatile ("mfdbatl %0,2" : "=r"(soft_dbat[2].batl)); 2682 __asm volatile ("mfdbatu %0,3" : "=r"(soft_dbat[3].batu)); 2683 __asm volatile ("mfdbatl %0,3" : "=r"(soft_dbat[3].batl)); 2684 } 2685 #endif 2686 2687 printf("SDR1:\t0x%lx\n", (long) sdr1); 2688 #ifndef PPC_OEA64 2689 printf("SR[]:\t"); 2690 for (i = 0; i < 4; i++) 2691 printf("0x%08lx, ", (long) soft_sr[i]); 2692 printf("\n\t"); 2693 for ( ; i < 8; i++) 2694 printf("0x%08lx, ", (long) soft_sr[i]); 2695 printf("\n\t"); 2696 for ( ; i < 12; i++) 2697 printf("0x%08lx, ", (long) soft_sr[i]); 2698 printf("\n\t"); 2699 for ( ; i < 16; i++) 2700 printf("0x%08lx, ", (long) soft_sr[i]); 2701 printf("\n"); 2702 #endif 2703 2704 #if defined(PPC_OEA) && !defined (PPC_OEA64) && !defined (PPC_OEA64_BRIDGE) 2705 printf("%cBAT[]:\t", cpuvers == MPC601 ? 'u' : 'i'); 2706 for (i = 0; i < 4; i++) { 2707 printf("0x%08lx 0x%08lx, ", 2708 soft_ibat[i].batu, soft_ibat[i].batl); 2709 if (i == 1) 2710 printf("\n\t"); 2711 } 2712 if (cpuvers != MPC601) { 2713 printf("\ndBAT[]:\t"); 2714 for (i = 0; i < 4; i++) { 2715 printf("0x%08lx 0x%08lx, ", 2716 soft_dbat[i].batu, soft_dbat[i].batl); 2717 if (i == 1) 2718 printf("\n\t"); 2719 } 2720 } 2721 printf("\n"); 2722 #endif /* PPC_OEA... */ 2723 } 2724 2725 void 2726 pmap_print_pte(pmap_t pm, vaddr_t va) 2727 { 2728 struct pvo_entry *pvo; 2729 volatile struct pte *pt; 2730 int pteidx; 2731 2732 pvo = pmap_pvo_find_va(pm, va, &pteidx); 2733 if (pvo != NULL) { 2734 pt = pmap_pvo_to_pte(pvo, pteidx); 2735 if (pt != NULL) { 2736 #if defined (PPC_OEA) || defined (PPC_OEA64) 2737 printf("VA %#lx -> %p -> %s %#lx, %#lx\n", 2738 va, pt, 2739 pt->pte_hi & PTE_HID ? "(sec)" : "(pri)", 2740 pt->pte_hi, pt->pte_lo); 2741 #else /* PPC_OEA64_BRIDGE */ 2742 printf("VA %#lx -> %p -> %s %#llx, %#llx\n", 2743 va, pt, 2744 pt->pte_hi & PTE_HID ? "(sec)" : "(pri)", 2745 pt->pte_hi, pt->pte_lo); 2746 #endif 2747 } else { 2748 printf("No valid PTE found\n"); 2749 } 2750 } else { 2751 printf("Address not in pmap\n"); 2752 } 2753 } 2754 2755 void 2756 pmap_pteg_dist(void) 2757 { 2758 struct pvo_entry *pvo; 2759 int ptegidx; 2760 int depth; 2761 int max_depth = 0; 2762 unsigned int depths[64]; 2763 2764 memset(depths, 0, sizeof(depths)); 2765 for (ptegidx = 0; ptegidx < pmap_pteg_cnt; ptegidx++) { 2766 depth = 0; 2767 TAILQ_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) { 2768 depth++; 2769 } 2770 if (depth > max_depth) 2771 max_depth = depth; 2772 if (depth > 63) 2773 depth = 63; 2774 depths[depth]++; 2775 } 2776 2777 for (depth = 0; depth < 64; depth++) { 2778 printf(" [%2d]: %8u", depth, depths[depth]); 2779 if ((depth & 3) == 3) 2780 printf("\n"); 2781 if (depth == max_depth) 2782 break; 2783 } 2784 if ((depth & 3) != 3) 2785 printf("\n"); 2786 printf("Max depth found was %d\n", max_depth); 2787 } 2788 #endif /* DEBUG */ 2789 2790 #if defined(PMAPCHECK) || defined(DEBUG) 2791 void 2792 pmap_pvo_verify(void) 2793 { 2794 int ptegidx; 2795 int s; 2796 2797 s = splvm(); 2798 for (ptegidx = 0; ptegidx < pmap_pteg_cnt; ptegidx++) { 2799 struct pvo_entry *pvo; 2800 TAILQ_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) { 2801 if ((uintptr_t) pvo >= SEGMENT_LENGTH) 2802 panic("pmap_pvo_verify: invalid pvo %p " 2803 "on list %#x", pvo, ptegidx); 2804 pmap_pvo_check(pvo); 2805 } 2806 } 2807 splx(s); 2808 } 2809 #endif /* PMAPCHECK */ 2810 2811 2812 void * 2813 pmap_pool_ualloc(struct pool *pp, int flags) 2814 { 2815 struct pvo_page *pvop; 2816 2817 pvop = SIMPLEQ_FIRST(&pmap_upvop_head); 2818 if (pvop != NULL) { 2819 pmap_upvop_free--; 2820 SIMPLEQ_REMOVE_HEAD(&pmap_upvop_head, pvop_link); 2821 return pvop; 2822 } 2823 if (uvm.page_init_done != TRUE) { 2824 return (void *) uvm_pageboot_alloc(PAGE_SIZE); 2825 } 2826 return pmap_pool_malloc(pp, flags); 2827 } 2828 2829 void * 2830 pmap_pool_malloc(struct pool *pp, int flags) 2831 { 2832 struct pvo_page *pvop; 2833 struct vm_page *pg; 2834 2835 pvop = SIMPLEQ_FIRST(&pmap_mpvop_head); 2836 if (pvop != NULL) { 2837 pmap_mpvop_free--; 2838 SIMPLEQ_REMOVE_HEAD(&pmap_mpvop_head, pvop_link); 2839 return pvop; 2840 } 2841 again: 2842 pg = uvm_pagealloc_strat(NULL, 0, NULL, UVM_PGA_USERESERVE, 2843 UVM_PGA_STRAT_ONLY, VM_FREELIST_FIRST256); 2844 if (__predict_false(pg == NULL)) { 2845 if (flags & PR_WAITOK) { 2846 uvm_wait("plpg"); 2847 goto again; 2848 } else { 2849 return (0); 2850 } 2851 } 2852 return (void *) VM_PAGE_TO_PHYS(pg); 2853 } 2854 2855 void 2856 pmap_pool_ufree(struct pool *pp, void *va) 2857 { 2858 struct pvo_page *pvop; 2859 #if 0 2860 if (PHYS_TO_VM_PAGE((paddr_t) va) != NULL) { 2861 pmap_pool_mfree(va, size, tag); 2862 return; 2863 } 2864 #endif 2865 pvop = va; 2866 SIMPLEQ_INSERT_HEAD(&pmap_upvop_head, pvop, pvop_link); 2867 pmap_upvop_free++; 2868 if (pmap_upvop_free > pmap_upvop_maxfree) 2869 pmap_upvop_maxfree = pmap_upvop_free; 2870 } 2871 2872 void 2873 pmap_pool_mfree(struct pool *pp, void *va) 2874 { 2875 struct pvo_page *pvop; 2876 2877 pvop = va; 2878 SIMPLEQ_INSERT_HEAD(&pmap_mpvop_head, pvop, pvop_link); 2879 pmap_mpvop_free++; 2880 if (pmap_mpvop_free > pmap_mpvop_maxfree) 2881 pmap_mpvop_maxfree = pmap_mpvop_free; 2882 #if 0 2883 uvm_pagefree(PHYS_TO_VM_PAGE((paddr_t) va)); 2884 #endif 2885 } 2886 2887 /* 2888 * This routine in bootstraping to steal to-be-managed memory (which will 2889 * then be unmanaged). We use it to grab from the first 256MB for our 2890 * pmap needs and above 256MB for other stuff. 2891 */ 2892 vaddr_t 2893 pmap_steal_memory(vsize_t vsize, vaddr_t *vstartp, vaddr_t *vendp) 2894 { 2895 vsize_t size; 2896 vaddr_t va; 2897 paddr_t pa = 0; 2898 int npgs, bank; 2899 struct vm_physseg *ps; 2900 2901 if (uvm.page_init_done == TRUE) 2902 panic("pmap_steal_memory: called _after_ bootstrap"); 2903 2904 *vstartp = VM_MIN_KERNEL_ADDRESS; 2905 *vendp = VM_MAX_KERNEL_ADDRESS; 2906 2907 size = round_page(vsize); 2908 npgs = atop(size); 2909 2910 /* 2911 * PA 0 will never be among those given to UVM so we can use it 2912 * to indicate we couldn't steal any memory. 2913 */ 2914 for (ps = vm_physmem, bank = 0; bank < vm_nphysseg; bank++, ps++) { 2915 if (ps->free_list == VM_FREELIST_FIRST256 && 2916 ps->avail_end - ps->avail_start >= npgs) { 2917 pa = ptoa(ps->avail_start); 2918 break; 2919 } 2920 } 2921 2922 if (pa == 0) 2923 panic("pmap_steal_memory: no approriate memory to steal!"); 2924 2925 ps->avail_start += npgs; 2926 ps->start += npgs; 2927 2928 /* 2929 * If we've used up all the pages in the segment, remove it and 2930 * compact the list. 2931 */ 2932 if (ps->avail_start == ps->end) { 2933 /* 2934 * If this was the last one, then a very bad thing has occurred 2935 */ 2936 if (--vm_nphysseg == 0) 2937 panic("pmap_steal_memory: out of memory!"); 2938 2939 printf("pmap_steal_memory: consumed bank %d\n", bank); 2940 for (; bank < vm_nphysseg; bank++, ps++) { 2941 ps[0] = ps[1]; 2942 } 2943 } 2944 2945 va = (vaddr_t) pa; 2946 memset((caddr_t) va, 0, size); 2947 pmap_pages_stolen += npgs; 2948 #ifdef DEBUG 2949 if (pmapdebug && npgs > 1) { 2950 u_int cnt = 0; 2951 for (bank = 0, ps = vm_physmem; bank < vm_nphysseg; bank++, ps++) 2952 cnt += ps->avail_end - ps->avail_start; 2953 printf("pmap_steal_memory: stole %u (total %u) pages (%u left)\n", 2954 npgs, pmap_pages_stolen, cnt); 2955 } 2956 #endif 2957 2958 return va; 2959 } 2960 2961 /* 2962 * Find a chuck of memory with right size and alignment. 2963 */ 2964 void * 2965 pmap_boot_find_memory(psize_t size, psize_t alignment, int at_end) 2966 { 2967 struct mem_region *mp; 2968 paddr_t s, e; 2969 int i, j; 2970 2971 size = round_page(size); 2972 2973 DPRINTFN(BOOT, 2974 ("pmap_boot_find_memory: size=%lx, alignment=%lx, at_end=%d", 2975 size, alignment, at_end)); 2976 2977 if (alignment < PAGE_SIZE || (alignment & (alignment-1)) != 0) 2978 panic("pmap_boot_find_memory: invalid alignment %lx", 2979 alignment); 2980 2981 if (at_end) { 2982 if (alignment != PAGE_SIZE) 2983 panic("pmap_boot_find_memory: invalid ending " 2984 "alignment %lx", alignment); 2985 2986 for (mp = &avail[avail_cnt-1]; mp >= avail; mp--) { 2987 s = mp->start + mp->size - size; 2988 if (s >= mp->start && mp->size >= size) { 2989 DPRINTFN(BOOT,(": %lx\n", s)); 2990 DPRINTFN(BOOT, 2991 ("pmap_boot_find_memory: b-avail[%d] start " 2992 "0x%lx size 0x%lx\n", mp - avail, 2993 mp->start, mp->size)); 2994 mp->size -= size; 2995 DPRINTFN(BOOT, 2996 ("pmap_boot_find_memory: a-avail[%d] start " 2997 "0x%lx size 0x%lx\n", mp - avail, 2998 mp->start, mp->size)); 2999 return (void *) s; 3000 } 3001 } 3002 panic("pmap_boot_find_memory: no available memory"); 3003 } 3004 3005 for (mp = avail, i = 0; i < avail_cnt; i++, mp++) { 3006 s = (mp->start + alignment - 1) & ~(alignment-1); 3007 e = s + size; 3008 3009 /* 3010 * Is the calculated region entirely within the region? 3011 */ 3012 if (s < mp->start || e > mp->start + mp->size) 3013 continue; 3014 3015 DPRINTFN(BOOT,(": %lx\n", s)); 3016 if (s == mp->start) { 3017 /* 3018 * If the block starts at the beginning of region, 3019 * adjust the size & start. (the region may now be 3020 * zero in length) 3021 */ 3022 DPRINTFN(BOOT, 3023 ("pmap_boot_find_memory: b-avail[%d] start " 3024 "0x%lx size 0x%lx\n", i, mp->start, mp->size)); 3025 mp->start += size; 3026 mp->size -= size; 3027 DPRINTFN(BOOT, 3028 ("pmap_boot_find_memory: a-avail[%d] start " 3029 "0x%lx size 0x%lx\n", i, mp->start, mp->size)); 3030 } else if (e == mp->start + mp->size) { 3031 /* 3032 * If the block starts at the beginning of region, 3033 * adjust only the size. 3034 */ 3035 DPRINTFN(BOOT, 3036 ("pmap_boot_find_memory: b-avail[%d] start " 3037 "0x%lx size 0x%lx\n", i, mp->start, mp->size)); 3038 mp->size -= size; 3039 DPRINTFN(BOOT, 3040 ("pmap_boot_find_memory: a-avail[%d] start " 3041 "0x%lx size 0x%lx\n", i, mp->start, mp->size)); 3042 } else { 3043 /* 3044 * Block is in the middle of the region, so we 3045 * have to split it in two. 3046 */ 3047 for (j = avail_cnt; j > i + 1; j--) { 3048 avail[j] = avail[j-1]; 3049 } 3050 DPRINTFN(BOOT, 3051 ("pmap_boot_find_memory: b-avail[%d] start " 3052 "0x%lx size 0x%lx\n", i, mp->start, mp->size)); 3053 mp[1].start = e; 3054 mp[1].size = mp[0].start + mp[0].size - e; 3055 mp[0].size = s - mp[0].start; 3056 avail_cnt++; 3057 for (; i < avail_cnt; i++) { 3058 DPRINTFN(BOOT, 3059 ("pmap_boot_find_memory: a-avail[%d] " 3060 "start 0x%lx size 0x%lx\n", i, 3061 avail[i].start, avail[i].size)); 3062 } 3063 } 3064 return (void *) s; 3065 } 3066 panic("pmap_boot_find_memory: not enough memory for " 3067 "%lx/%lx allocation?", size, alignment); 3068 } 3069 3070 /* XXXSL: we dont have any BATs to do this, map in Segment 0 1:1 using page tables */ 3071 #if defined (PPC_OEA64_BRIDGE) 3072 int 3073 pmap_setup_segment0_map(int use_large_pages, ...) 3074 { 3075 vaddr_t va; 3076 3077 register_t pte_lo = 0x0; 3078 int ptegidx = 0, i = 0; 3079 struct pte pte; 3080 va_list ap; 3081 3082 /* Coherent + Supervisor RW, no user access */ 3083 pte_lo = PTE_M; 3084 3085 /* XXXSL 3086 * Map in 1st segment 1:1, we'll be careful not to spill kernel entries later, 3087 * these have to take priority. 3088 */ 3089 for (va = 0x0; va < SEGMENT_LENGTH; va += 0x1000) { 3090 ptegidx = va_to_pteg(pmap_kernel(), va); 3091 pmap_pte_create(&pte, pmap_kernel(), va, va | pte_lo); 3092 i = pmap_pte_insert(ptegidx, &pte); 3093 } 3094 3095 va_start(ap, use_large_pages); 3096 while (1) { 3097 paddr_t pa; 3098 size_t size; 3099 3100 va = va_arg(ap, vaddr_t); 3101 3102 if (va == 0) 3103 break; 3104 3105 pa = va_arg(ap, paddr_t); 3106 size = va_arg(ap, size_t); 3107 3108 for (; va < (va + size); va += 0x1000, pa += 0x1000) { 3109 #if 0 3110 printf("%s: Inserting: va: 0x%08lx, pa: 0x%08lx\n", __FUNCTION__, va, pa); 3111 #endif 3112 ptegidx = va_to_pteg(pmap_kernel(), va); 3113 pmap_pte_create(&pte, pmap_kernel(), va, pa | pte_lo); 3114 i = pmap_pte_insert(ptegidx, &pte); 3115 } 3116 } 3117 3118 TLBSYNC(); 3119 SYNC(); 3120 return (0); 3121 } 3122 #endif /* PPC_OEA64_BRIDGE */ 3123 3124 /* 3125 * This is not part of the defined PMAP interface and is specific to the 3126 * PowerPC architecture. This is called during initppc, before the system 3127 * is really initialized. 3128 */ 3129 void 3130 pmap_bootstrap(paddr_t kernelstart, paddr_t kernelend) 3131 { 3132 struct mem_region *mp, tmp; 3133 paddr_t s, e; 3134 psize_t size; 3135 int i, j; 3136 3137 /* 3138 * Get memory. 3139 */ 3140 mem_regions(&mem, &avail); 3141 #if defined(DEBUG) 3142 if (pmapdebug & PMAPDEBUG_BOOT) { 3143 printf("pmap_bootstrap: memory configuration:\n"); 3144 for (mp = mem; mp->size; mp++) { 3145 printf("pmap_bootstrap: mem start 0x%lx size 0x%lx\n", 3146 mp->start, mp->size); 3147 } 3148 for (mp = avail; mp->size; mp++) { 3149 printf("pmap_bootstrap: avail start 0x%lx size 0x%lx\n", 3150 mp->start, mp->size); 3151 } 3152 } 3153 #endif 3154 3155 /* 3156 * Find out how much physical memory we have and in how many chunks. 3157 */ 3158 for (mem_cnt = 0, mp = mem; mp->size; mp++) { 3159 if (mp->start >= pmap_memlimit) 3160 continue; 3161 if (mp->start + mp->size > pmap_memlimit) { 3162 size = pmap_memlimit - mp->start; 3163 physmem += btoc(size); 3164 } else { 3165 physmem += btoc(mp->size); 3166 } 3167 mem_cnt++; 3168 } 3169 3170 /* 3171 * Count the number of available entries. 3172 */ 3173 for (avail_cnt = 0, mp = avail; mp->size; mp++) 3174 avail_cnt++; 3175 3176 /* 3177 * Page align all regions. 3178 */ 3179 kernelstart = trunc_page(kernelstart); 3180 kernelend = round_page(kernelend); 3181 for (mp = avail, i = 0; i < avail_cnt; i++, mp++) { 3182 s = round_page(mp->start); 3183 mp->size -= (s - mp->start); 3184 mp->size = trunc_page(mp->size); 3185 mp->start = s; 3186 e = mp->start + mp->size; 3187 3188 DPRINTFN(BOOT, 3189 ("pmap_bootstrap: b-avail[%d] start 0x%lx size 0x%lx\n", 3190 i, mp->start, mp->size)); 3191 3192 /* 3193 * Don't allow the end to run beyond our artificial limit 3194 */ 3195 if (e > pmap_memlimit) 3196 e = pmap_memlimit; 3197 3198 /* 3199 * Is this region empty or strange? skip it. 3200 */ 3201 if (e <= s) { 3202 mp->start = 0; 3203 mp->size = 0; 3204 continue; 3205 } 3206 3207 /* 3208 * Does this overlap the beginning of kernel? 3209 * Does extend past the end of the kernel? 3210 */ 3211 else if (s < kernelstart && e > kernelstart) { 3212 if (e > kernelend) { 3213 avail[avail_cnt].start = kernelend; 3214 avail[avail_cnt].size = e - kernelend; 3215 avail_cnt++; 3216 } 3217 mp->size = kernelstart - s; 3218 } 3219 /* 3220 * Check whether this region overlaps the end of the kernel. 3221 */ 3222 else if (s < kernelend && e > kernelend) { 3223 mp->start = kernelend; 3224 mp->size = e - kernelend; 3225 } 3226 /* 3227 * Look whether this regions is completely inside the kernel. 3228 * Nuke it if it does. 3229 */ 3230 else if (s >= kernelstart && e <= kernelend) { 3231 mp->start = 0; 3232 mp->size = 0; 3233 } 3234 /* 3235 * If the user imposed a memory limit, enforce it. 3236 */ 3237 else if (s >= pmap_memlimit) { 3238 mp->start = -PAGE_SIZE; /* let's know why */ 3239 mp->size = 0; 3240 } 3241 else { 3242 mp->start = s; 3243 mp->size = e - s; 3244 } 3245 DPRINTFN(BOOT, 3246 ("pmap_bootstrap: a-avail[%d] start 0x%lx size 0x%lx\n", 3247 i, mp->start, mp->size)); 3248 } 3249 3250 /* 3251 * Move (and uncount) all the null return to the end. 3252 */ 3253 for (mp = avail, i = 0; i < avail_cnt; i++, mp++) { 3254 if (mp->size == 0) { 3255 tmp = avail[i]; 3256 avail[i] = avail[--avail_cnt]; 3257 avail[avail_cnt] = avail[i]; 3258 } 3259 } 3260 3261 /* 3262 * (Bubble)sort them into asecnding order. 3263 */ 3264 for (i = 0; i < avail_cnt; i++) { 3265 for (j = i + 1; j < avail_cnt; j++) { 3266 if (avail[i].start > avail[j].start) { 3267 tmp = avail[i]; 3268 avail[i] = avail[j]; 3269 avail[j] = tmp; 3270 } 3271 } 3272 } 3273 3274 /* 3275 * Make sure they don't overlap. 3276 */ 3277 for (mp = avail, i = 0; i < avail_cnt - 1; i++, mp++) { 3278 if (mp[0].start + mp[0].size > mp[1].start) { 3279 mp[0].size = mp[1].start - mp[0].start; 3280 } 3281 DPRINTFN(BOOT, 3282 ("pmap_bootstrap: avail[%d] start 0x%lx size 0x%lx\n", 3283 i, mp->start, mp->size)); 3284 } 3285 DPRINTFN(BOOT, 3286 ("pmap_bootstrap: avail[%d] start 0x%lx size 0x%lx\n", 3287 i, mp->start, mp->size)); 3288 3289 #ifdef PTEGCOUNT 3290 pmap_pteg_cnt = PTEGCOUNT; 3291 #else /* PTEGCOUNT */ 3292 3293 pmap_pteg_cnt = 0x1000; 3294 3295 while (pmap_pteg_cnt < physmem) 3296 pmap_pteg_cnt <<= 1; 3297 3298 pmap_pteg_cnt >>= 1; 3299 #endif /* PTEGCOUNT */ 3300 3301 #ifdef DEBUG 3302 DPRINTFN(BOOT, 3303 ("pmap_pteg_cnt: 0x%x\n", pmap_pteg_cnt)); 3304 #endif 3305 3306 /* 3307 * Find suitably aligned memory for PTEG hash table. 3308 */ 3309 size = pmap_pteg_cnt * sizeof(struct pteg); 3310 pmap_pteg_table = pmap_boot_find_memory(size, size, 0); 3311 3312 #ifdef DEBUG 3313 DPRINTFN(BOOT, 3314 ("PTEG cnt: 0x%x HTAB size: 0x%08x bytes, address: %p\n", pmap_pteg_cnt, (unsigned int)size, pmap_pteg_table)); 3315 #endif 3316 3317 3318 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 3319 if ( (uintptr_t) pmap_pteg_table + size > SEGMENT_LENGTH) 3320 panic("pmap_bootstrap: pmap_pteg_table end (%p + %lx) > 256MB", 3321 pmap_pteg_table, size); 3322 #endif 3323 3324 memset(__UNVOLATILE(pmap_pteg_table), 0, 3325 pmap_pteg_cnt * sizeof(struct pteg)); 3326 pmap_pteg_mask = pmap_pteg_cnt - 1; 3327 3328 /* 3329 * We cannot do pmap_steal_memory here since UVM hasn't been loaded 3330 * with pages. So we just steal them before giving them to UVM. 3331 */ 3332 size = sizeof(pmap_pvo_table[0]) * pmap_pteg_cnt; 3333 pmap_pvo_table = pmap_boot_find_memory(size, PAGE_SIZE, 0); 3334 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 3335 if ( (uintptr_t) pmap_pvo_table + size > SEGMENT_LENGTH) 3336 panic("pmap_bootstrap: pmap_pvo_table end (%p + %lx) > 256MB", 3337 pmap_pvo_table, size); 3338 #endif 3339 3340 for (i = 0; i < pmap_pteg_cnt; i++) 3341 TAILQ_INIT(&pmap_pvo_table[i]); 3342 3343 #ifndef MSGBUFADDR 3344 /* 3345 * Allocate msgbuf in high memory. 3346 */ 3347 msgbuf_paddr = 3348 (paddr_t) pmap_boot_find_memory(MSGBUFSIZE, PAGE_SIZE, 1); 3349 #endif 3350 3351 #ifdef __HAVE_PMAP_PHYSSEG 3352 { 3353 u_int npgs = 0; 3354 for (i = 0, mp = avail; i < avail_cnt; i++, mp++) 3355 npgs += btoc(mp->size); 3356 size = (sizeof(struct pvo_head) + 1) * npgs; 3357 pmap_physseg.pvoh = pmap_boot_find_memory(size, PAGE_SIZE, 0); 3358 pmap_physseg.attrs = (char *) &pmap_physseg.pvoh[npgs]; 3359 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 3360 if ((uintptr_t)pmap_physseg.pvoh + size > SEGMENT_LENGTH) 3361 panic("pmap_bootstrap: PVO list end (%p + %lx) > 256MB", 3362 pmap_physseg.pvoh, size); 3363 #endif 3364 } 3365 #endif 3366 3367 3368 for (mp = avail, i = 0; i < avail_cnt; mp++, i++) { 3369 paddr_t pfstart = atop(mp->start); 3370 paddr_t pfend = atop(mp->start + mp->size); 3371 if (mp->size == 0) 3372 continue; 3373 if (mp->start + mp->size <= SEGMENT_LENGTH) { 3374 uvm_page_physload(pfstart, pfend, pfstart, pfend, 3375 VM_FREELIST_FIRST256); 3376 } else if (mp->start >= SEGMENT_LENGTH) { 3377 uvm_page_physload(pfstart, pfend, pfstart, pfend, 3378 VM_FREELIST_DEFAULT); 3379 } else { 3380 pfend = atop(SEGMENT_LENGTH); 3381 uvm_page_physload(pfstart, pfend, pfstart, pfend, 3382 VM_FREELIST_FIRST256); 3383 pfstart = atop(SEGMENT_LENGTH); 3384 pfend = atop(mp->start + mp->size); 3385 uvm_page_physload(pfstart, pfend, pfstart, pfend, 3386 VM_FREELIST_DEFAULT); 3387 } 3388 } 3389 printf("Done calling uvm_page_physload()\n"); 3390 3391 /* 3392 * Make sure kernel vsid is allocated as well as VSID 0. 3393 */ 3394 pmap_vsid_bitmap[(KERNEL_VSIDBITS & (NPMAPS-1)) / VSID_NBPW] 3395 |= 1 << (KERNEL_VSIDBITS % VSID_NBPW); 3396 pmap_vsid_bitmap[0] |= 1; 3397 3398 /* 3399 * Initialize kernel pmap and hardware. 3400 */ 3401 3402 /* PPC_OEA64_BRIDGE does support these instructions */ 3403 #if defined (PPC_OEA) || defined (PPC_OEA64_BRIDGE) 3404 for (i = 0; i < 16; i++) { 3405 pmap_kernel()->pm_sr[i] = KERNELN_SEGMENT(i)|SR_PRKEY; 3406 __asm volatile ("mtsrin %0,%1" 3407 :: "r"(KERNELN_SEGMENT(i)|SR_PRKEY), "r"(i << ADDR_SR_SHFT)); 3408 } 3409 3410 pmap_kernel()->pm_sr[KERNEL_SR] = KERNEL_SEGMENT|SR_SUKEY|SR_PRKEY; 3411 __asm volatile ("mtsr %0,%1" 3412 :: "n"(KERNEL_SR), "r"(KERNEL_SEGMENT)); 3413 #ifdef KERNEL2_SR 3414 pmap_kernel()->pm_sr[KERNEL2_SR] = KERNEL2_SEGMENT|SR_SUKEY|SR_PRKEY; 3415 __asm volatile ("mtsr %0,%1" 3416 :: "n"(KERNEL2_SR), "r"(KERNEL2_SEGMENT)); 3417 #endif 3418 for (i = 0; i < 16; i++) { 3419 if (iosrtable[i] & SR601_T) { 3420 pmap_kernel()->pm_sr[i] = iosrtable[i]; 3421 __asm volatile ("mtsrin %0,%1" 3422 :: "r"(iosrtable[i]), "r"(i << ADDR_SR_SHFT)); 3423 } 3424 } 3425 #endif /* PPC_OEA || PPC_OEA64_BRIDGE */ 3426 #if defined (PPC_OEA) 3427 __asm volatile ("sync; mtsdr1 %0; isync" 3428 :: "r"((uintptr_t)pmap_pteg_table | (pmap_pteg_mask >> 10))); 3429 #elif defined (PPC_OEA64) || defined (PPC_OEA64_BRIDGE) 3430 __asm __volatile ("sync; mtsdr1 %0; isync" 3431 :: "r"((uintptr_t)pmap_pteg_table | (32 - cntlzw(pmap_pteg_mask >> 11)))); 3432 #endif 3433 tlbia(); 3434 3435 #ifdef ALTIVEC 3436 pmap_use_altivec = cpu_altivec; 3437 #endif 3438 3439 #ifdef DEBUG 3440 if (pmapdebug & PMAPDEBUG_BOOT) { 3441 u_int cnt; 3442 int bank; 3443 char pbuf[9]; 3444 for (cnt = 0, bank = 0; bank < vm_nphysseg; bank++) { 3445 cnt += vm_physmem[bank].avail_end - vm_physmem[bank].avail_start; 3446 printf("pmap_bootstrap: vm_physmem[%d]=%#lx-%#lx/%#lx\n", 3447 bank, 3448 ptoa(vm_physmem[bank].avail_start), 3449 ptoa(vm_physmem[bank].avail_end), 3450 ptoa(vm_physmem[bank].avail_end - vm_physmem[bank].avail_start)); 3451 } 3452 format_bytes(pbuf, sizeof(pbuf), ptoa((u_int64_t) cnt)); 3453 printf("pmap_bootstrap: UVM memory = %s (%u pages)\n", 3454 pbuf, cnt); 3455 } 3456 #endif 3457 3458 pool_init(&pmap_upvo_pool, sizeof(struct pvo_entry), 3459 sizeof(struct pvo_entry), 0, 0, "pmap_upvopl", 3460 &pmap_pool_uallocator); 3461 3462 pool_setlowat(&pmap_upvo_pool, 252); 3463 3464 pool_init(&pmap_pool, sizeof(struct pmap), 3465 sizeof(void *), 0, 0, "pmap_pl", &pmap_pool_uallocator); 3466 } 3467
Indexes created Mon Sep 29 21:09:56 GMT 2025