pmap.c revision 1.48
1 /* $NetBSD: pmap.c,v 1.48 2007/03/12 18:18:26 ad Exp $ */ 2 /*- 3 * Copyright (c) 2001 The NetBSD Foundation, Inc. 4 * All rights reserved. 5 * 6 * This code is derived from software contributed to The NetBSD Foundation 7 * by Matt Thomas <matt (at) 3am-software.com> of Allegro Networks, Inc. 8 * 9 * Support for PPC64 Bridge mode added by Sanjay Lal <sanjayl (at) kymasys.com> 10 * of Kyma Systems LLC. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. All advertising materials mentioning features or use of this software 21 * must display the following acknowledgement: 22 * This product includes software developed by the NetBSD 23 * Foundation, Inc. and its contributors. 24 * 4. Neither the name of The NetBSD Foundation nor the names of its 25 * contributors may be used to endorse or promote products derived 26 * from this software without specific prior written permission. 27 * 28 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 29 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 30 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 31 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 32 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 33 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 34 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 35 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 36 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 37 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 38 * POSSIBILITY OF SUCH DAMAGE. 39 */ 40 41 /* 42 * Copyright (C) 1995, 1996 Wolfgang Solfrank. 43 * Copyright (C) 1995, 1996 TooLs GmbH. 44 * All rights reserved. 45 * 46 * Redistribution and use in source and binary forms, with or without 47 * modification, are permitted provided that the following conditions 48 * are met: 49 * 1. Redistributions of source code must retain the above copyright 50 * notice, this list of conditions and the following disclaimer. 51 * 2. Redistributions in binary form must reproduce the above copyright 52 * notice, this list of conditions and the following disclaimer in the 53 * documentation and/or other materials provided with the distribution. 54 * 3. All advertising materials mentioning features or use of this software 55 * must display the following acknowledgement: 56 * This product includes software developed by TooLs GmbH. 57 * 4. The name of TooLs GmbH may not be used to endorse or promote products 58 * derived from this software without specific prior written permission. 59 * 60 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR 61 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 62 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 63 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 64 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 65 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 66 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 67 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 68 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 69 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 70 */ 71 72 #include <sys/cdefs.h> 73 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.48 2007/03/12 18:18:26 ad Exp $"); 74 75 #include "opt_ppcarch.h" 76 #include "opt_altivec.h" 77 #include "opt_pmap.h" 78 #include <sys/param.h> 79 #include <sys/malloc.h> 80 #include <sys/proc.h> 81 #include <sys/user.h> 82 #include <sys/pool.h> 83 #include <sys/queue.h> 84 #include <sys/device.h> /* for evcnt */ 85 #include <sys/systm.h> 86 87 #if __NetBSD_Version__ < 105010000 88 #include <vm/vm.h> 89 #include <vm/vm_kern.h> 90 #define splvm() splimp() 91 #endif 92 93 #include <uvm/uvm.h> 94 95 #include <machine/pcb.h> 96 #include <machine/powerpc.h> 97 #include <powerpc/spr.h> 98 #include <powerpc/oea/sr_601.h> 99 #include <powerpc/bat.h> 100 #include <powerpc/stdarg.h> 101 102 #if defined(DEBUG) || defined(PMAPCHECK) 103 #define STATIC 104 #else 105 #define STATIC static 106 #endif 107 108 #ifdef ALTIVEC 109 int pmap_use_altivec; 110 #endif 111 112 volatile struct pteg *pmap_pteg_table; 113 unsigned int pmap_pteg_cnt; 114 unsigned int pmap_pteg_mask; 115 #ifdef PMAP_MEMLIMIT 116 paddr_t pmap_memlimit = PMAP_MEMLIMIT; 117 #else 118 paddr_t pmap_memlimit = -PAGE_SIZE; /* there is no limit */ 119 #endif 120 121 struct pmap kernel_pmap_; 122 unsigned int pmap_pages_stolen; 123 u_long pmap_pte_valid; 124 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 125 u_long pmap_pvo_enter_depth; 126 u_long pmap_pvo_remove_depth; 127 #endif 128 129 int physmem; 130 #ifndef MSGBUFADDR 131 extern paddr_t msgbuf_paddr; 132 #endif 133 134 static struct mem_region *mem, *avail; 135 static u_int mem_cnt, avail_cnt; 136 137 #ifdef __HAVE_PMAP_PHYSSEG 138 /* 139 * This is a cache of referenced/modified bits. 140 * Bits herein are shifted by ATTRSHFT. 141 */ 142 #define ATTR_SHFT 4 143 struct pmap_physseg pmap_physseg; 144 #endif 145 146 /* 147 * The following structure is aligned to 32 bytes 148 */ 149 struct pvo_entry { 150 LIST_ENTRY(pvo_entry) pvo_vlink; /* Link to common virt page */ 151 TAILQ_ENTRY(pvo_entry) pvo_olink; /* Link to overflow entry */ 152 struct pte pvo_pte; /* Prebuilt PTE */ 153 pmap_t pvo_pmap; /* ptr to owning pmap */ 154 vaddr_t pvo_vaddr; /* VA of entry */ 155 #define PVO_PTEGIDX_MASK 0x0007 /* which PTEG slot */ 156 #define PVO_PTEGIDX_VALID 0x0008 /* slot is valid */ 157 #define PVO_WIRED 0x0010 /* PVO entry is wired */ 158 #define PVO_MANAGED 0x0020 /* PVO e. for managed page */ 159 #define PVO_EXECUTABLE 0x0040 /* PVO e. for executable page */ 160 #define PVO_WIRED_P(pvo) ((pvo)->pvo_vaddr & PVO_WIRED) 161 #define PVO_MANAGED_P(pvo) ((pvo)->pvo_vaddr & PVO_MANAGED) 162 #define PVO_EXECUTABLE_P(pvo) ((pvo)->pvo_vaddr & PVO_EXECUTABLE) 163 #define PVO_ENTER_INSERT 0 /* PVO has been removed */ 164 #define PVO_SPILL_UNSET 1 /* PVO has been evicted */ 165 #define PVO_SPILL_SET 2 /* PVO has been spilled */ 166 #define PVO_SPILL_INSERT 3 /* PVO has been inserted */ 167 #define PVO_PMAP_PAGE_PROTECT 4 /* PVO has changed */ 168 #define PVO_PMAP_PROTECT 5 /* PVO has changed */ 169 #define PVO_REMOVE 6 /* PVO has been removed */ 170 #define PVO_WHERE_MASK 15 171 #define PVO_WHERE_SHFT 8 172 } __attribute__ ((aligned (32))); 173 #define PVO_VADDR(pvo) ((pvo)->pvo_vaddr & ~ADDR_POFF) 174 #define PVO_PTEGIDX_GET(pvo) ((pvo)->pvo_vaddr & PVO_PTEGIDX_MASK) 175 #define PVO_PTEGIDX_ISSET(pvo) ((pvo)->pvo_vaddr & PVO_PTEGIDX_VALID) 176 #define PVO_PTEGIDX_CLR(pvo) \ 177 ((void)((pvo)->pvo_vaddr &= ~(PVO_PTEGIDX_VALID|PVO_PTEGIDX_MASK))) 178 #define PVO_PTEGIDX_SET(pvo,i) \ 179 ((void)((pvo)->pvo_vaddr |= (i)|PVO_PTEGIDX_VALID)) 180 #define PVO_WHERE(pvo,w) \ 181 ((pvo)->pvo_vaddr &= ~(PVO_WHERE_MASK << PVO_WHERE_SHFT), \ 182 (pvo)->pvo_vaddr |= ((PVO_ ## w) << PVO_WHERE_SHFT)) 183 184 TAILQ_HEAD(pvo_tqhead, pvo_entry); 185 struct pvo_tqhead *pmap_pvo_table; /* pvo entries by ptegroup index */ 186 struct pvo_head pmap_pvo_kunmanaged = LIST_HEAD_INITIALIZER(pmap_pvo_kunmanaged); /* list of unmanaged pages */ 187 struct pvo_head pmap_pvo_unmanaged = LIST_HEAD_INITIALIZER(pmap_pvo_unmanaged); /* list of unmanaged pages */ 188 189 struct pool pmap_pool; /* pool for pmap structures */ 190 struct pool pmap_upvo_pool; /* pool for pvo entries for unmanaged pages */ 191 struct pool pmap_mpvo_pool; /* pool for pvo entries for managed pages */ 192 193 /* 194 * We keep a cache of unmanaged pages to be used for pvo entries for 195 * unmanaged pages. 196 */ 197 struct pvo_page { 198 SIMPLEQ_ENTRY(pvo_page) pvop_link; 199 }; 200 SIMPLEQ_HEAD(pvop_head, pvo_page); 201 struct pvop_head pmap_upvop_head = SIMPLEQ_HEAD_INITIALIZER(pmap_upvop_head); 202 struct pvop_head pmap_mpvop_head = SIMPLEQ_HEAD_INITIALIZER(pmap_mpvop_head); 203 u_long pmap_upvop_free; 204 u_long pmap_upvop_maxfree; 205 u_long pmap_mpvop_free; 206 u_long pmap_mpvop_maxfree; 207 208 STATIC void *pmap_pool_ualloc(struct pool *, int); 209 STATIC void *pmap_pool_malloc(struct pool *, int); 210 211 STATIC void pmap_pool_ufree(struct pool *, void *); 212 STATIC void pmap_pool_mfree(struct pool *, void *); 213 214 static struct pool_allocator pmap_pool_mallocator = { 215 .pa_alloc = pmap_pool_malloc, 216 .pa_free = pmap_pool_mfree, 217 .pa_pagesz = 0, 218 }; 219 220 static struct pool_allocator pmap_pool_uallocator = { 221 .pa_alloc = pmap_pool_ualloc, 222 .pa_free = pmap_pool_ufree, 223 .pa_pagesz = 0, 224 }; 225 226 #if defined(DEBUG) || defined(PMAPCHECK) || defined(DDB) 227 void pmap_pte_print(volatile struct pte *); 228 void pmap_pteg_check(void); 229 void pmap_pteg_dist(void); 230 void pmap_print_pte(pmap_t, vaddr_t); 231 void pmap_print_mmuregs(void); 232 #endif 233 234 #if defined(DEBUG) || defined(PMAPCHECK) 235 #ifdef PMAPCHECK 236 int pmapcheck = 1; 237 #else 238 int pmapcheck = 0; 239 #endif 240 void pmap_pvo_verify(void); 241 STATIC void pmap_pvo_check(const struct pvo_entry *); 242 #define PMAP_PVO_CHECK(pvo) \ 243 do { \ 244 if (pmapcheck) \ 245 pmap_pvo_check(pvo); \ 246 } while (0) 247 #else 248 #define PMAP_PVO_CHECK(pvo) do { } while (/*CONSTCOND*/0) 249 #endif 250 STATIC int pmap_pte_insert(int, struct pte *); 251 STATIC int pmap_pvo_enter(pmap_t, struct pool *, struct pvo_head *, 252 vaddr_t, paddr_t, register_t, int); 253 STATIC void pmap_pvo_remove(struct pvo_entry *, int, struct pvo_head *); 254 STATIC void pmap_pvo_free(struct pvo_entry *); 255 STATIC void pmap_pvo_free_list(struct pvo_head *); 256 STATIC struct pvo_entry *pmap_pvo_find_va(pmap_t, vaddr_t, int *); 257 STATIC volatile struct pte *pmap_pvo_to_pte(const struct pvo_entry *, int); 258 STATIC struct pvo_entry *pmap_pvo_reclaim(struct pmap *); 259 STATIC void pvo_set_exec(struct pvo_entry *); 260 STATIC void pvo_clear_exec(struct pvo_entry *); 261 262 STATIC void tlbia(void); 263 264 STATIC void pmap_release(pmap_t); 265 STATIC void *pmap_boot_find_memory(psize_t, psize_t, int); 266 267 static uint32_t pmap_pvo_reclaim_nextidx; 268 #ifdef DEBUG 269 static int pmap_pvo_reclaim_debugctr; 270 #endif 271 272 #define VSID_NBPW (sizeof(uint32_t) * 8) 273 static uint32_t pmap_vsid_bitmap[NPMAPS / VSID_NBPW]; 274 275 static int pmap_initialized; 276 277 #if defined(DEBUG) || defined(PMAPDEBUG) 278 #define PMAPDEBUG_BOOT 0x0001 279 #define PMAPDEBUG_PTE 0x0002 280 #define PMAPDEBUG_EXEC 0x0008 281 #define PMAPDEBUG_PVOENTER 0x0010 282 #define PMAPDEBUG_PVOREMOVE 0x0020 283 #define PMAPDEBUG_ACTIVATE 0x0100 284 #define PMAPDEBUG_CREATE 0x0200 285 #define PMAPDEBUG_ENTER 0x1000 286 #define PMAPDEBUG_KENTER 0x2000 287 #define PMAPDEBUG_KREMOVE 0x4000 288 #define PMAPDEBUG_REMOVE 0x8000 289 290 unsigned int pmapdebug = 0; 291 292 # define DPRINTF(x) printf x 293 # define DPRINTFN(n, x) if (pmapdebug & PMAPDEBUG_ ## n) printf x 294 #else 295 # define DPRINTF(x) 296 # define DPRINTFN(n, x) 297 #endif 298 299 300 #ifdef PMAPCOUNTERS 301 #define PMAPCOUNT(ev) ((pmap_evcnt_ ## ev).ev_count++) 302 #define PMAPCOUNT2(ev) ((ev).ev_count++) 303 304 struct evcnt pmap_evcnt_mappings = 305 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 306 "pmap", "pages mapped"); 307 struct evcnt pmap_evcnt_unmappings = 308 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_mappings, 309 "pmap", "pages unmapped"); 310 311 struct evcnt pmap_evcnt_kernel_mappings = 312 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 313 "pmap", "kernel pages mapped"); 314 struct evcnt pmap_evcnt_kernel_unmappings = 315 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_kernel_mappings, 316 "pmap", "kernel pages unmapped"); 317 318 struct evcnt pmap_evcnt_mappings_replaced = 319 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 320 "pmap", "page mappings replaced"); 321 322 struct evcnt pmap_evcnt_exec_mappings = 323 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_mappings, 324 "pmap", "exec pages mapped"); 325 struct evcnt pmap_evcnt_exec_cached = 326 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_mappings, 327 "pmap", "exec pages cached"); 328 329 struct evcnt pmap_evcnt_exec_synced = 330 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_exec_mappings, 331 "pmap", "exec pages synced"); 332 struct evcnt pmap_evcnt_exec_synced_clear_modify = 333 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_exec_mappings, 334 "pmap", "exec pages synced (CM)"); 335 struct evcnt pmap_evcnt_exec_synced_pvo_remove = 336 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_exec_mappings, 337 "pmap", "exec pages synced (PR)"); 338 339 struct evcnt pmap_evcnt_exec_uncached_page_protect = 340 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_exec_mappings, 341 "pmap", "exec pages uncached (PP)"); 342 struct evcnt pmap_evcnt_exec_uncached_clear_modify = 343 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_exec_mappings, 344 "pmap", "exec pages uncached (CM)"); 345 struct evcnt pmap_evcnt_exec_uncached_zero_page = 346 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_exec_mappings, 347 "pmap", "exec pages uncached (ZP)"); 348 struct evcnt pmap_evcnt_exec_uncached_copy_page = 349 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_exec_mappings, 350 "pmap", "exec pages uncached (CP)"); 351 struct evcnt pmap_evcnt_exec_uncached_pvo_remove = 352 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, &pmap_evcnt_exec_mappings, 353 "pmap", "exec pages uncached (PR)"); 354 355 struct evcnt pmap_evcnt_updates = 356 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 357 "pmap", "updates"); 358 struct evcnt pmap_evcnt_collects = 359 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 360 "pmap", "collects"); 361 struct evcnt pmap_evcnt_copies = 362 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 363 "pmap", "copies"); 364 365 struct evcnt pmap_evcnt_ptes_spilled = 366 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 367 "pmap", "ptes spilled from overflow"); 368 struct evcnt pmap_evcnt_ptes_unspilled = 369 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 370 "pmap", "ptes not spilled"); 371 struct evcnt pmap_evcnt_ptes_evicted = 372 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 373 "pmap", "ptes evicted"); 374 375 struct evcnt pmap_evcnt_ptes_primary[8] = { 376 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 377 "pmap", "ptes added at primary[0]"), 378 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 379 "pmap", "ptes added at primary[1]"), 380 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 381 "pmap", "ptes added at primary[2]"), 382 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 383 "pmap", "ptes added at primary[3]"), 384 385 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 386 "pmap", "ptes added at primary[4]"), 387 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 388 "pmap", "ptes added at primary[5]"), 389 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 390 "pmap", "ptes added at primary[6]"), 391 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 392 "pmap", "ptes added at primary[7]"), 393 }; 394 struct evcnt pmap_evcnt_ptes_secondary[8] = { 395 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 396 "pmap", "ptes added at secondary[0]"), 397 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 398 "pmap", "ptes added at secondary[1]"), 399 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 400 "pmap", "ptes added at secondary[2]"), 401 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 402 "pmap", "ptes added at secondary[3]"), 403 404 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 405 "pmap", "ptes added at secondary[4]"), 406 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 407 "pmap", "ptes added at secondary[5]"), 408 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 409 "pmap", "ptes added at secondary[6]"), 410 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 411 "pmap", "ptes added at secondary[7]"), 412 }; 413 struct evcnt pmap_evcnt_ptes_removed = 414 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 415 "pmap", "ptes removed"); 416 struct evcnt pmap_evcnt_ptes_changed = 417 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 418 "pmap", "ptes changed"); 419 struct evcnt pmap_evcnt_pvos_reclaimed = 420 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 421 "pmap", "pvos reclaimed"); 422 struct evcnt pmap_evcnt_pvos_failed = 423 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, 424 "pmap", "pvo allocation failures"); 425 426 /* 427 * From pmap_subr.c 428 */ 429 extern struct evcnt pmap_evcnt_zeroed_pages; 430 extern struct evcnt pmap_evcnt_copied_pages; 431 extern struct evcnt pmap_evcnt_idlezeroed_pages; 432 433 EVCNT_ATTACH_STATIC(pmap_evcnt_mappings); 434 EVCNT_ATTACH_STATIC(pmap_evcnt_mappings_replaced); 435 EVCNT_ATTACH_STATIC(pmap_evcnt_unmappings); 436 437 EVCNT_ATTACH_STATIC(pmap_evcnt_kernel_mappings); 438 EVCNT_ATTACH_STATIC(pmap_evcnt_kernel_unmappings); 439 440 EVCNT_ATTACH_STATIC(pmap_evcnt_exec_mappings); 441 EVCNT_ATTACH_STATIC(pmap_evcnt_exec_cached); 442 EVCNT_ATTACH_STATIC(pmap_evcnt_exec_synced); 443 EVCNT_ATTACH_STATIC(pmap_evcnt_exec_synced_clear_modify); 444 EVCNT_ATTACH_STATIC(pmap_evcnt_exec_synced_pvo_remove); 445 446 EVCNT_ATTACH_STATIC(pmap_evcnt_exec_uncached_page_protect); 447 EVCNT_ATTACH_STATIC(pmap_evcnt_exec_uncached_clear_modify); 448 EVCNT_ATTACH_STATIC(pmap_evcnt_exec_uncached_zero_page); 449 EVCNT_ATTACH_STATIC(pmap_evcnt_exec_uncached_copy_page); 450 EVCNT_ATTACH_STATIC(pmap_evcnt_exec_uncached_pvo_remove); 451 452 EVCNT_ATTACH_STATIC(pmap_evcnt_zeroed_pages); 453 EVCNT_ATTACH_STATIC(pmap_evcnt_copied_pages); 454 EVCNT_ATTACH_STATIC(pmap_evcnt_idlezeroed_pages); 455 456 EVCNT_ATTACH_STATIC(pmap_evcnt_updates); 457 EVCNT_ATTACH_STATIC(pmap_evcnt_collects); 458 EVCNT_ATTACH_STATIC(pmap_evcnt_copies); 459 460 EVCNT_ATTACH_STATIC(pmap_evcnt_ptes_spilled); 461 EVCNT_ATTACH_STATIC(pmap_evcnt_ptes_unspilled); 462 EVCNT_ATTACH_STATIC(pmap_evcnt_ptes_evicted); 463 EVCNT_ATTACH_STATIC(pmap_evcnt_ptes_removed); 464 EVCNT_ATTACH_STATIC(pmap_evcnt_ptes_changed); 465 466 EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_primary, 0); 467 EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_primary, 1); 468 EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_primary, 2); 469 EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_primary, 3); 470 EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_primary, 4); 471 EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_primary, 5); 472 EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_primary, 6); 473 EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_primary, 7); 474 EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_secondary, 0); 475 EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_secondary, 1); 476 EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_secondary, 2); 477 EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_secondary, 3); 478 EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_secondary, 4); 479 EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_secondary, 5); 480 EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_secondary, 6); 481 EVCNT_ATTACH_STATIC2(pmap_evcnt_ptes_secondary, 7); 482 483 EVCNT_ATTACH_STATIC(pmap_evcnt_pvos_reclaimed); 484 EVCNT_ATTACH_STATIC(pmap_evcnt_pvos_failed); 485 #else 486 #define PMAPCOUNT(ev) ((void) 0) 487 #define PMAPCOUNT2(ev) ((void) 0) 488 #endif 489 490 #define TLBIE(va) __asm volatile("tlbie %0" :: "r"(va)) 491 492 /* XXXSL: this needs to be moved to assembler */ 493 #define TLBIEL(va) __asm __volatile("tlbie %0" :: "r"(va)) 494 495 #define TLBSYNC() __asm volatile("tlbsync") 496 #define SYNC() __asm volatile("sync") 497 #define EIEIO() __asm volatile("eieio") 498 #define MFMSR() mfmsr() 499 #define MTMSR(psl) mtmsr(psl) 500 #define MFPVR() mfpvr() 501 #define MFSRIN(va) mfsrin(va) 502 #define MFTB() mfrtcltbl() 503 504 #if defined (PPC_OEA) || defined (PPC_OEA64_BRIDGE) 505 static inline register_t 506 mfsrin(vaddr_t va) 507 { 508 register_t sr; 509 __asm volatile ("mfsrin %0,%1" : "=r"(sr) : "r"(va)); 510 return sr; 511 } 512 #endif /* PPC_OEA*/ 513 514 #if defined (PPC_OEA64_BRIDGE) 515 extern void mfmsr64 (register64_t *result); 516 #endif /* PPC_OEA64_BRIDGE */ 517 518 519 static inline register_t 520 pmap_interrupts_off(void) 521 { 522 register_t msr = MFMSR(); 523 if (msr & PSL_EE) 524 MTMSR(msr & ~PSL_EE); 525 return msr; 526 } 527 528 static void 529 pmap_interrupts_restore(register_t msr) 530 { 531 if (msr & PSL_EE) 532 MTMSR(msr); 533 } 534 535 static inline u_int32_t 536 mfrtcltbl(void) 537 { 538 539 if ((MFPVR() >> 16) == MPC601) 540 return (mfrtcl() >> 7); 541 else 542 return (mftbl()); 543 } 544 545 /* 546 * These small routines may have to be replaced, 547 * if/when we support processors other that the 604. 548 */ 549 550 void 551 tlbia(void) 552 { 553 char *i; 554 555 SYNC(); 556 #if defined(PPC_OEA) 557 /* 558 * Why not use "tlbia"? Because not all processors implement it. 559 * 560 * This needs to be a per-CPU callback to do the appropriate thing 561 * for the CPU. XXX 562 */ 563 for (i = 0; i < (char *)0x00040000; i += 0x00001000) { 564 TLBIE(i); 565 EIEIO(); 566 SYNC(); 567 } 568 #elif defined (PPC_OEA64) || defined (PPC_OEA64_BRIDGE) 569 printf("Invalidating ALL TLB entries......\n"); 570 /* This is specifically for the 970, 970UM v1.6 pp. 140. */ 571 for (i = 0; i <= (void *)0xFF000; i += 0x00001000) { 572 TLBIEL(i); 573 EIEIO(); 574 SYNC(); 575 } 576 #endif 577 TLBSYNC(); 578 SYNC(); 579 } 580 581 static inline register_t 582 va_to_vsid(const struct pmap *pm, vaddr_t addr) 583 { 584 #if defined (PPC_OEA) || defined (PPC_OEA64_BRIDGE) 585 return (pm->pm_sr[addr >> ADDR_SR_SHFT] & SR_VSID) >> SR_VSID_SHFT; 586 #else /* PPC_OEA64 */ 587 #if 0 588 const struct ste *ste; 589 register_t hash; 590 int i; 591 592 hash = (addr >> ADDR_ESID_SHFT) & ADDR_ESID_HASH; 593 594 /* 595 * Try the primary group first 596 */ 597 ste = pm->pm_stes[hash].stes; 598 for (i = 0; i < 8; i++, ste++) { 599 if (ste->ste_hi & STE_V) && 600 (addr & ~(ADDR_POFF|ADDR_PIDX)) == (ste->ste_hi & STE_ESID)) 601 return ste; 602 } 603 604 /* 605 * Then the secondary group. 606 */ 607 ste = pm->pm_stes[hash ^ ADDR_ESID_HASH].stes; 608 for (i = 0; i < 8; i++, ste++) { 609 if (ste->ste_hi & STE_V) && 610 (addr & ~(ADDR_POFF|ADDR_PIDX)) == (ste->ste_hi & STE_ESID)) 611 return addr; 612 } 613 614 return NULL; 615 #else 616 /* 617 * Rather than searching the STE groups for the VSID, we know 618 * how we generate that from the ESID and so do that. 619 */ 620 return VSID_MAKE(addr >> ADDR_SR_SHFT, pm->pm_vsid) >> SR_VSID_SHFT; 621 #endif 622 #endif /* PPC_OEA */ 623 } 624 625 static inline register_t 626 va_to_pteg(const struct pmap *pm, vaddr_t addr) 627 { 628 register_t hash; 629 630 hash = va_to_vsid(pm, addr) ^ ((addr & ADDR_PIDX) >> ADDR_PIDX_SHFT); 631 return hash & pmap_pteg_mask; 632 } 633 634 #if defined(DEBUG) || defined(PMAPCHECK) || defined(DDB) 635 /* 636 * Given a PTE in the page table, calculate the VADDR that hashes to it. 637 * The only bit of magic is that the top 4 bits of the address doesn't 638 * technically exist in the PTE. But we know we reserved 4 bits of the 639 * VSID for it so that's how we get it. 640 */ 641 static vaddr_t 642 pmap_pte_to_va(volatile const struct pte *pt) 643 { 644 vaddr_t va; 645 uintptr_t ptaddr = (uintptr_t) pt; 646 647 if (pt->pte_hi & PTE_HID) 648 ptaddr ^= (pmap_pteg_mask * sizeof(struct pteg)); 649 650 /* PPC Bits 10-19 PPC64 Bits 42-51 */ 651 #if defined(PPC_OEA) 652 va = ((pt->pte_hi >> PTE_VSID_SHFT) ^ (ptaddr / sizeof(struct pteg))) & 0x3ff; 653 #elif defined (PPC_OEA64) || defined (PPC_OEA64_BRIDGE) 654 va = ((pt->pte_hi >> PTE_VSID_SHFT) ^ (ptaddr / sizeof(struct pteg))) & 0x7ff; 655 #endif 656 va <<= ADDR_PIDX_SHFT; 657 658 /* PPC Bits 4-9 PPC64 Bits 36-41 */ 659 va |= (pt->pte_hi & PTE_API) << ADDR_API_SHFT; 660 661 #if defined(PPC_OEA64) 662 /* PPC63 Bits 0-35 */ 663 /* va |= VSID_TO_SR(pt->pte_hi >> PTE_VSID_SHFT) << ADDR_SR_SHFT; */ 664 #elif defined(PPC_OEA) || defined(PPC_OEA64_BRIDGE) 665 /* PPC Bits 0-3 */ 666 va |= VSID_TO_SR(pt->pte_hi >> PTE_VSID_SHFT) << ADDR_SR_SHFT; 667 #endif 668 669 return va; 670 } 671 #endif 672 673 static inline struct pvo_head * 674 pa_to_pvoh(paddr_t pa, struct vm_page **pg_p) 675 { 676 #ifdef __HAVE_VM_PAGE_MD 677 struct vm_page *pg; 678 679 pg = PHYS_TO_VM_PAGE(pa); 680 if (pg_p != NULL) 681 *pg_p = pg; 682 if (pg == NULL) 683 return &pmap_pvo_unmanaged; 684 return &pg->mdpage.mdpg_pvoh; 685 #endif 686 #ifdef __HAVE_PMAP_PHYSSEG 687 int bank, pg; 688 689 bank = vm_physseg_find(atop(pa), &pg); 690 if (pg_p != NULL) 691 *pg_p = pg; 692 if (bank == -1) 693 return &pmap_pvo_unmanaged; 694 return &vm_physmem[bank].pmseg.pvoh[pg]; 695 #endif 696 } 697 698 static inline struct pvo_head * 699 vm_page_to_pvoh(struct vm_page *pg) 700 { 701 #ifdef __HAVE_VM_PAGE_MD 702 return &pg->mdpage.mdpg_pvoh; 703 #endif 704 #ifdef __HAVE_PMAP_PHYSSEG 705 return pa_to_pvoh(VM_PAGE_TO_PHYS(pg), NULL); 706 #endif 707 } 708 709 710 #ifdef __HAVE_PMAP_PHYSSEG 711 static inline char * 712 pa_to_attr(paddr_t pa) 713 { 714 int bank, pg; 715 716 bank = vm_physseg_find(atop(pa), &pg); 717 if (bank == -1) 718 return NULL; 719 return &vm_physmem[bank].pmseg.attrs[pg]; 720 } 721 #endif 722 723 static inline void 724 pmap_attr_clear(struct vm_page *pg, int ptebit) 725 { 726 #ifdef __HAVE_PMAP_PHYSSEG 727 *pa_to_attr(VM_PAGE_TO_PHYS(pg)) &= ~(ptebit >> ATTR_SHFT); 728 #endif 729 #ifdef __HAVE_VM_PAGE_MD 730 pg->mdpage.mdpg_attrs &= ~ptebit; 731 #endif 732 } 733 734 static inline int 735 pmap_attr_fetch(struct vm_page *pg) 736 { 737 #ifdef __HAVE_PMAP_PHYSSEG 738 return *pa_to_attr(VM_PAGE_TO_PHYS(pg)) << ATTR_SHFT; 739 #endif 740 #ifdef __HAVE_VM_PAGE_MD 741 return pg->mdpage.mdpg_attrs; 742 #endif 743 } 744 745 static inline void 746 pmap_attr_save(struct vm_page *pg, int ptebit) 747 { 748 #ifdef __HAVE_PMAP_PHYSSEG 749 *pa_to_attr(VM_PAGE_TO_PHYS(pg)) |= (ptebit >> ATTR_SHFT); 750 #endif 751 #ifdef __HAVE_VM_PAGE_MD 752 pg->mdpage.mdpg_attrs |= ptebit; 753 #endif 754 } 755 756 static inline int 757 pmap_pte_compare(const volatile struct pte *pt, const struct pte *pvo_pt) 758 { 759 if (pt->pte_hi == pvo_pt->pte_hi 760 #if 0 761 && ((pt->pte_lo ^ pvo_pt->pte_lo) & 762 ~(PTE_REF|PTE_CHG)) == 0 763 #endif 764 ) 765 return 1; 766 return 0; 767 } 768 769 static inline void 770 pmap_pte_create(struct pte *pt, const struct pmap *pm, vaddr_t va, register_t pte_lo) 771 { 772 /* 773 * Construct the PTE. Default to IMB initially. Valid bit 774 * only gets set when the real pte is set in memory. 775 * 776 * Note: Don't set the valid bit for correct operation of tlb update. 777 */ 778 #if defined(PPC_OEA) 779 pt->pte_hi = (va_to_vsid(pm, va) << PTE_VSID_SHFT) 780 | (((va & ADDR_PIDX) >> (ADDR_API_SHFT - PTE_API_SHFT)) & PTE_API); 781 pt->pte_lo = pte_lo; 782 #elif defined (PPC_OEA64_BRIDGE) 783 pt->pte_hi = ((u_int64_t)va_to_vsid(pm, va) << PTE_VSID_SHFT) 784 | (((va & ADDR_PIDX) >> (ADDR_API_SHFT - PTE_API_SHFT)) & PTE_API); 785 pt->pte_lo = (u_int64_t) pte_lo; 786 #elif defined (PPC_OEA64) 787 #error PPC_OEA64 not supported 788 #endif /* PPC_OEA */ 789 } 790 791 static inline void 792 pmap_pte_synch(volatile struct pte *pt, struct pte *pvo_pt) 793 { 794 pvo_pt->pte_lo |= pt->pte_lo & (PTE_REF|PTE_CHG); 795 } 796 797 static inline void 798 pmap_pte_clear(volatile struct pte *pt, vaddr_t va, int ptebit) 799 { 800 /* 801 * As shown in Section 7.6.3.2.3 802 */ 803 pt->pte_lo &= ~ptebit; 804 TLBIE(va); 805 SYNC(); 806 EIEIO(); 807 TLBSYNC(); 808 SYNC(); 809 } 810 811 static inline void 812 pmap_pte_set(volatile struct pte *pt, struct pte *pvo_pt) 813 { 814 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 815 if (pvo_pt->pte_hi & PTE_VALID) 816 panic("pte_set: setting an already valid pte %p", pvo_pt); 817 #endif 818 pvo_pt->pte_hi |= PTE_VALID; 819 820 /* 821 * Update the PTE as defined in section 7.6.3.1 822 * Note that the REF/CHG bits are from pvo_pt and thus should 823 * have been saved so this routine can restore them (if desired). 824 */ 825 pt->pte_lo = pvo_pt->pte_lo; 826 EIEIO(); 827 pt->pte_hi = pvo_pt->pte_hi; 828 TLBSYNC(); 829 SYNC(); 830 pmap_pte_valid++; 831 } 832 833 static inline void 834 pmap_pte_unset(volatile struct pte *pt, struct pte *pvo_pt, vaddr_t va) 835 { 836 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 837 if ((pvo_pt->pte_hi & PTE_VALID) == 0) 838 panic("pte_unset: attempt to unset an inactive pte#1 %p/%p", pvo_pt, pt); 839 if ((pt->pte_hi & PTE_VALID) == 0) 840 panic("pte_unset: attempt to unset an inactive pte#2 %p/%p", pvo_pt, pt); 841 #endif 842 843 pvo_pt->pte_hi &= ~PTE_VALID; 844 /* 845 * Force the ref & chg bits back into the PTEs. 846 */ 847 SYNC(); 848 /* 849 * Invalidate the pte ... (Section 7.6.3.3) 850 */ 851 pt->pte_hi &= ~PTE_VALID; 852 SYNC(); 853 TLBIE(va); 854 SYNC(); 855 EIEIO(); 856 TLBSYNC(); 857 SYNC(); 858 /* 859 * Save the ref & chg bits ... 860 */ 861 pmap_pte_synch(pt, pvo_pt); 862 pmap_pte_valid--; 863 } 864 865 static inline void 866 pmap_pte_change(volatile struct pte *pt, struct pte *pvo_pt, vaddr_t va) 867 { 868 /* 869 * Invalidate the PTE 870 */ 871 pmap_pte_unset(pt, pvo_pt, va); 872 pmap_pte_set(pt, pvo_pt); 873 } 874 875 /* 876 * Try to insert the PTE @ *pvo_pt into the pmap_pteg_table at ptegidx 877 * (either primary or secondary location). 878 * 879 * Note: both the destination and source PTEs must not have PTE_VALID set. 880 */ 881 882 STATIC int 883 pmap_pte_insert(int ptegidx, struct pte *pvo_pt) 884 { 885 volatile struct pte *pt; 886 int i; 887 888 #if defined(DEBUG) 889 #if defined (PPC_OEA) 890 DPRINTFN(PTE, ("pmap_pte_insert: idx 0x%x, pte 0x%x 0x%x\n", 891 ptegidx, (unsigned int) pvo_pt->pte_hi, (unsigned int) pvo_pt->pte_lo)); 892 #elif defined (PPC_OEA64_BRIDGE) 893 DPRINTFN(PTE, ("pmap_pte_insert: idx 0x%x, pte 0x%016llx 0x%016llx\n", 894 ptegidx, (unsigned long long) pvo_pt->pte_hi, 895 (unsigned long long) pvo_pt->pte_lo)); 896 897 #endif 898 #endif 899 /* 900 * First try primary hash. 901 */ 902 for (pt = pmap_pteg_table[ptegidx].pt, i = 0; i < 8; i++, pt++) { 903 if ((pt->pte_hi & PTE_VALID) == 0) { 904 pvo_pt->pte_hi &= ~PTE_HID; 905 pmap_pte_set(pt, pvo_pt); 906 return i; 907 } 908 } 909 910 /* 911 * Now try secondary hash. 912 */ 913 ptegidx ^= pmap_pteg_mask; 914 for (pt = pmap_pteg_table[ptegidx].pt, i = 0; i < 8; i++, pt++) { 915 if ((pt->pte_hi & PTE_VALID) == 0) { 916 pvo_pt->pte_hi |= PTE_HID; 917 pmap_pte_set(pt, pvo_pt); 918 return i; 919 } 920 } 921 return -1; 922 } 923 924 /* 925 * Spill handler. 926 * 927 * Tries to spill a page table entry from the overflow area. 928 * This runs in either real mode (if dealing with a exception spill) 929 * or virtual mode when dealing with manually spilling one of the 930 * kernel's pte entries. In either case, interrupts are already 931 * disabled. 932 */ 933 934 int 935 pmap_pte_spill(struct pmap *pm, vaddr_t addr, bool exec) 936 { 937 struct pvo_entry *source_pvo, *victim_pvo, *next_pvo; 938 struct pvo_entry *pvo; 939 /* XXX: gcc -- vpvoh is always set at either *1* or *2* */ 940 struct pvo_tqhead *pvoh, *vpvoh = NULL; 941 int ptegidx, i, j; 942 volatile struct pteg *pteg; 943 volatile struct pte *pt; 944 945 ptegidx = va_to_pteg(pm, addr); 946 947 /* 948 * Have to substitute some entry. Use the primary hash for this. 949 * Use low bits of timebase as random generator. Make sure we are 950 * not picking a kernel pte for replacement. 951 */ 952 pteg = &pmap_pteg_table[ptegidx]; 953 i = MFTB() & 7; 954 for (j = 0; j < 8; j++) { 955 pt = &pteg->pt[i]; 956 if ((pt->pte_hi & PTE_VALID) == 0 || 957 VSID_TO_HASH((pt->pte_hi & PTE_VSID) >> PTE_VSID_SHFT) 958 != KERNEL_VSIDBITS) 959 break; 960 i = (i + 1) & 7; 961 } 962 KASSERT(j < 8); 963 964 source_pvo = NULL; 965 victim_pvo = NULL; 966 pvoh = &pmap_pvo_table[ptegidx]; 967 TAILQ_FOREACH(pvo, pvoh, pvo_olink) { 968 969 /* 970 * We need to find pvo entry for this address... 971 */ 972 PMAP_PVO_CHECK(pvo); /* sanity check */ 973 974 /* 975 * If we haven't found the source and we come to a PVO with 976 * a valid PTE, then we know we can't find it because all 977 * evicted PVOs always are first in the list. 978 */ 979 if (source_pvo == NULL && (pvo->pvo_pte.pte_hi & PTE_VALID)) 980 break; 981 if (source_pvo == NULL && pm == pvo->pvo_pmap && 982 addr == PVO_VADDR(pvo)) { 983 984 /* 985 * Now we have found the entry to be spilled into the 986 * pteg. Attempt to insert it into the page table. 987 */ 988 j = pmap_pte_insert(ptegidx, &pvo->pvo_pte); 989 if (j >= 0) { 990 PVO_PTEGIDX_SET(pvo, j); 991 PMAP_PVO_CHECK(pvo); /* sanity check */ 992 PVO_WHERE(pvo, SPILL_INSERT); 993 pvo->pvo_pmap->pm_evictions--; 994 PMAPCOUNT(ptes_spilled); 995 PMAPCOUNT2(((pvo->pvo_pte.pte_hi & PTE_HID) 996 ? pmap_evcnt_ptes_secondary 997 : pmap_evcnt_ptes_primary)[j]); 998 999 /* 1000 * Since we keep the evicted entries at the 1001 * from of the PVO list, we need move this 1002 * (now resident) PVO after the evicted 1003 * entries. 1004 */ 1005 next_pvo = TAILQ_NEXT(pvo, pvo_olink); 1006 1007 /* 1008 * If we don't have to move (either we were the 1009 * last entry or the next entry was valid), 1010 * don't change our position. Otherwise 1011 * move ourselves to the tail of the queue. 1012 */ 1013 if (next_pvo != NULL && 1014 !(next_pvo->pvo_pte.pte_hi & PTE_VALID)) { 1015 TAILQ_REMOVE(pvoh, pvo, pvo_olink); 1016 TAILQ_INSERT_TAIL(pvoh, pvo, pvo_olink); 1017 } 1018 return 1; 1019 } 1020 source_pvo = pvo; 1021 if (exec && !PVO_EXECUTABLE_P(source_pvo)) { 1022 return 0; 1023 } 1024 if (victim_pvo != NULL) 1025 break; 1026 } 1027 1028 /* 1029 * We also need the pvo entry of the victim we are replacing 1030 * so save the R & C bits of the PTE. 1031 */ 1032 if ((pt->pte_hi & PTE_HID) == 0 && victim_pvo == NULL && 1033 pmap_pte_compare(pt, &pvo->pvo_pte)) { 1034 vpvoh = pvoh; /* *1* */ 1035 victim_pvo = pvo; 1036 if (source_pvo != NULL) 1037 break; 1038 } 1039 } 1040 1041 if (source_pvo == NULL) { 1042 PMAPCOUNT(ptes_unspilled); 1043 return 0; 1044 } 1045 1046 if (victim_pvo == NULL) { 1047 if ((pt->pte_hi & PTE_HID) == 0) 1048 panic("pmap_pte_spill: victim p-pte (%p) has " 1049 "no pvo entry!", pt); 1050 1051 /* 1052 * If this is a secondary PTE, we need to search 1053 * its primary pvo bucket for the matching PVO. 1054 */ 1055 vpvoh = &pmap_pvo_table[ptegidx ^ pmap_pteg_mask]; /* *2* */ 1056 TAILQ_FOREACH(pvo, vpvoh, pvo_olink) { 1057 PMAP_PVO_CHECK(pvo); /* sanity check */ 1058 1059 /* 1060 * We also need the pvo entry of the victim we are 1061 * replacing so save the R & C bits of the PTE. 1062 */ 1063 if (pmap_pte_compare(pt, &pvo->pvo_pte)) { 1064 victim_pvo = pvo; 1065 break; 1066 } 1067 } 1068 if (victim_pvo == NULL) 1069 panic("pmap_pte_spill: victim s-pte (%p) has " 1070 "no pvo entry!", pt); 1071 } 1072 1073 /* 1074 * The victim should be not be a kernel PVO/PTE entry. 1075 */ 1076 KASSERT(victim_pvo->pvo_pmap != pmap_kernel()); 1077 KASSERT(PVO_PTEGIDX_ISSET(victim_pvo)); 1078 KASSERT(PVO_PTEGIDX_GET(victim_pvo) == i); 1079 1080 /* 1081 * We are invalidating the TLB entry for the EA for the 1082 * we are replacing even though its valid; If we don't 1083 * we lose any ref/chg bit changes contained in the TLB 1084 * entry. 1085 */ 1086 source_pvo->pvo_pte.pte_hi &= ~PTE_HID; 1087 1088 /* 1089 * To enforce the PVO list ordering constraint that all 1090 * evicted entries should come before all valid entries, 1091 * move the source PVO to the tail of its list and the 1092 * victim PVO to the head of its list (which might not be 1093 * the same list, if the victim was using the secondary hash). 1094 */ 1095 TAILQ_REMOVE(pvoh, source_pvo, pvo_olink); 1096 TAILQ_INSERT_TAIL(pvoh, source_pvo, pvo_olink); 1097 TAILQ_REMOVE(vpvoh, victim_pvo, pvo_olink); 1098 TAILQ_INSERT_HEAD(vpvoh, victim_pvo, pvo_olink); 1099 pmap_pte_unset(pt, &victim_pvo->pvo_pte, victim_pvo->pvo_vaddr); 1100 pmap_pte_set(pt, &source_pvo->pvo_pte); 1101 victim_pvo->pvo_pmap->pm_evictions++; 1102 source_pvo->pvo_pmap->pm_evictions--; 1103 PVO_WHERE(victim_pvo, SPILL_UNSET); 1104 PVO_WHERE(source_pvo, SPILL_SET); 1105 1106 PVO_PTEGIDX_CLR(victim_pvo); 1107 PVO_PTEGIDX_SET(source_pvo, i); 1108 PMAPCOUNT2(pmap_evcnt_ptes_primary[i]); 1109 PMAPCOUNT(ptes_spilled); 1110 PMAPCOUNT(ptes_evicted); 1111 PMAPCOUNT(ptes_removed); 1112 1113 PMAP_PVO_CHECK(victim_pvo); 1114 PMAP_PVO_CHECK(source_pvo); 1115 return 1; 1116 } 1117 1118 /* 1119 * Restrict given range to physical memory 1120 */ 1121 void 1122 pmap_real_memory(paddr_t *start, psize_t *size) 1123 { 1124 struct mem_region *mp; 1125 1126 for (mp = mem; mp->size; mp++) { 1127 if (*start + *size > mp->start 1128 && *start < mp->start + mp->size) { 1129 if (*start < mp->start) { 1130 *size -= mp->start - *start; 1131 *start = mp->start; 1132 } 1133 if (*start + *size > mp->start + mp->size) 1134 *size = mp->start + mp->size - *start; 1135 return; 1136 } 1137 } 1138 *size = 0; 1139 } 1140 1141 /* 1142 * Initialize anything else for pmap handling. 1143 * Called during vm_init(). 1144 */ 1145 void 1146 pmap_init(void) 1147 { 1148 #ifdef __HAVE_PMAP_PHYSSEG 1149 struct pvo_tqhead *pvoh; 1150 int bank; 1151 long sz; 1152 char *attr; 1153 1154 pvoh = pmap_physseg.pvoh; 1155 attr = pmap_physseg.attrs; 1156 for (bank = 0; bank < vm_nphysseg; bank++) { 1157 sz = vm_physmem[bank].end - vm_physmem[bank].start; 1158 vm_physmem[bank].pmseg.pvoh = pvoh; 1159 vm_physmem[bank].pmseg.attrs = attr; 1160 for (; sz > 0; sz--, pvoh++, attr++) { 1161 TAILQ_INIT(pvoh); 1162 *attr = 0; 1163 } 1164 } 1165 #endif 1166 1167 pool_init(&pmap_mpvo_pool, sizeof(struct pvo_entry), 1168 sizeof(struct pvo_entry), 0, 0, "pmap_mpvopl", 1169 &pmap_pool_mallocator, IPL_NONE); 1170 1171 pool_setlowat(&pmap_mpvo_pool, 1008); 1172 1173 pmap_initialized = 1; 1174 1175 } 1176 1177 /* 1178 * How much virtual space does the kernel get? 1179 */ 1180 void 1181 pmap_virtual_space(vaddr_t *start, vaddr_t *end) 1182 { 1183 /* 1184 * For now, reserve one segment (minus some overhead) for kernel 1185 * virtual memory 1186 */ 1187 *start = VM_MIN_KERNEL_ADDRESS; 1188 *end = VM_MAX_KERNEL_ADDRESS; 1189 } 1190 1191 /* 1192 * Allocate, initialize, and return a new physical map. 1193 */ 1194 pmap_t 1195 pmap_create(void) 1196 { 1197 pmap_t pm; 1198 1199 pm = pool_get(&pmap_pool, PR_WAITOK); 1200 memset((void *)pm, 0, sizeof *pm); 1201 pmap_pinit(pm); 1202 1203 DPRINTFN(CREATE,("pmap_create: pm %p:\n" 1204 "\t%06x %06x %06x %06x %06x %06x %06x %06x\n" 1205 "\t%06x %06x %06x %06x %06x %06x %06x %06x\n", pm, 1206 (unsigned int) pm->pm_sr[0], (unsigned int) pm->pm_sr[1], 1207 (unsigned int) pm->pm_sr[2], (unsigned int) pm->pm_sr[3], 1208 (unsigned int) pm->pm_sr[4], (unsigned int) pm->pm_sr[5], 1209 (unsigned int) pm->pm_sr[6], (unsigned int) pm->pm_sr[7], 1210 (unsigned int) pm->pm_sr[8], (unsigned int) pm->pm_sr[9], 1211 (unsigned int) pm->pm_sr[10], (unsigned int) pm->pm_sr[11], 1212 (unsigned int) pm->pm_sr[12], (unsigned int) pm->pm_sr[13], 1213 (unsigned int) pm->pm_sr[14], (unsigned int) pm->pm_sr[15])); 1214 return pm; 1215 } 1216 1217 /* 1218 * Initialize a preallocated and zeroed pmap structure. 1219 */ 1220 void 1221 pmap_pinit(pmap_t pm) 1222 { 1223 register_t entropy = MFTB(); 1224 register_t mask; 1225 int i; 1226 1227 /* 1228 * Allocate some segment registers for this pmap. 1229 */ 1230 pm->pm_refs = 1; 1231 for (i = 0; i < NPMAPS; i += VSID_NBPW) { 1232 static register_t pmap_vsidcontext; 1233 register_t hash; 1234 unsigned int n; 1235 1236 /* Create a new value by multiplying by a prime adding in 1237 * entropy from the timebase register. This is to make the 1238 * VSID more random so that the PT Hash function collides 1239 * less often. (note that the prime causes gcc to do shifts 1240 * instead of a multiply) 1241 */ 1242 pmap_vsidcontext = (pmap_vsidcontext * 0x1105) + entropy; 1243 hash = pmap_vsidcontext & (NPMAPS - 1); 1244 if (hash == 0) { /* 0 is special, avoid it */ 1245 entropy += 0xbadf00d; 1246 continue; 1247 } 1248 n = hash >> 5; 1249 mask = 1L << (hash & (VSID_NBPW-1)); 1250 hash = pmap_vsidcontext; 1251 if (pmap_vsid_bitmap[n] & mask) { /* collision? */ 1252 /* anything free in this bucket? */ 1253 if (~pmap_vsid_bitmap[n] == 0) { 1254 entropy = hash ^ (hash >> 16); 1255 continue; 1256 } 1257 i = ffs(~pmap_vsid_bitmap[n]) - 1; 1258 mask = 1L << i; 1259 hash &= ~(VSID_NBPW-1); 1260 hash |= i; 1261 } 1262 hash &= PTE_VSID >> PTE_VSID_SHFT; 1263 pmap_vsid_bitmap[n] |= mask; 1264 pm->pm_vsid = hash; 1265 #if defined (PPC_OEA) || defined (PPC_OEA64_BRIDGE) 1266 for (i = 0; i < 16; i++) 1267 pm->pm_sr[i] = VSID_MAKE(i, hash) | SR_PRKEY | 1268 SR_NOEXEC; 1269 #endif 1270 return; 1271 } 1272 panic("pmap_pinit: out of segments"); 1273 } 1274 1275 /* 1276 * Add a reference to the given pmap. 1277 */ 1278 void 1279 pmap_reference(pmap_t pm) 1280 { 1281 pm->pm_refs++; 1282 } 1283 1284 /* 1285 * Retire the given pmap from service. 1286 * Should only be called if the map contains no valid mappings. 1287 */ 1288 void 1289 pmap_destroy(pmap_t pm) 1290 { 1291 if (--pm->pm_refs == 0) { 1292 pmap_release(pm); 1293 pool_put(&pmap_pool, pm); 1294 } 1295 } 1296 1297 /* 1298 * Release any resources held by the given physical map. 1299 * Called when a pmap initialized by pmap_pinit is being released. 1300 */ 1301 void 1302 pmap_release(pmap_t pm) 1303 { 1304 int idx, mask; 1305 1306 KASSERT(pm->pm_stats.resident_count == 0); 1307 KASSERT(pm->pm_stats.wired_count == 0); 1308 1309 if (pm->pm_sr[0] == 0) 1310 panic("pmap_release"); 1311 idx = pm->pm_vsid & (NPMAPS-1); 1312 mask = 1 << (idx % VSID_NBPW); 1313 idx /= VSID_NBPW; 1314 1315 KASSERT(pmap_vsid_bitmap[idx] & mask); 1316 pmap_vsid_bitmap[idx] &= ~mask; 1317 } 1318 1319 /* 1320 * Copy the range specified by src_addr/len 1321 * from the source map to the range dst_addr/len 1322 * in the destination map. 1323 * 1324 * This routine is only advisory and need not do anything. 1325 */ 1326 void 1327 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vaddr_t dst_addr, 1328 vsize_t len, vaddr_t src_addr) 1329 { 1330 PMAPCOUNT(copies); 1331 } 1332 1333 /* 1334 * Require that all active physical maps contain no 1335 * incorrect entries NOW. 1336 */ 1337 void 1338 pmap_update(struct pmap *pmap) 1339 { 1340 PMAPCOUNT(updates); 1341 TLBSYNC(); 1342 } 1343 1344 /* 1345 * Garbage collects the physical map system for 1346 * pages which are no longer used. 1347 * Success need not be guaranteed -- that is, there 1348 * may well be pages which are not referenced, but 1349 * others may be collected. 1350 * Called by the pageout daemon when pages are scarce. 1351 */ 1352 void 1353 pmap_collect(pmap_t pm) 1354 { 1355 PMAPCOUNT(collects); 1356 } 1357 1358 static inline int 1359 pmap_pvo_pte_index(const struct pvo_entry *pvo, int ptegidx) 1360 { 1361 int pteidx; 1362 /* 1363 * We can find the actual pte entry without searching by 1364 * grabbing the PTEG index from 3 unused bits in pte_lo[11:9] 1365 * and by noticing the HID bit. 1366 */ 1367 pteidx = ptegidx * 8 + PVO_PTEGIDX_GET(pvo); 1368 if (pvo->pvo_pte.pte_hi & PTE_HID) 1369 pteidx ^= pmap_pteg_mask * 8; 1370 return pteidx; 1371 } 1372 1373 volatile struct pte * 1374 pmap_pvo_to_pte(const struct pvo_entry *pvo, int pteidx) 1375 { 1376 volatile struct pte *pt; 1377 1378 #if !defined(DIAGNOSTIC) && !defined(DEBUG) && !defined(PMAPCHECK) 1379 if ((pvo->pvo_pte.pte_hi & PTE_VALID) == 0) 1380 return NULL; 1381 #endif 1382 1383 /* 1384 * If we haven't been supplied the ptegidx, calculate it. 1385 */ 1386 if (pteidx == -1) { 1387 int ptegidx; 1388 ptegidx = va_to_pteg(pvo->pvo_pmap, pvo->pvo_vaddr); 1389 pteidx = pmap_pvo_pte_index(pvo, ptegidx); 1390 } 1391 1392 pt = &pmap_pteg_table[pteidx >> 3].pt[pteidx & 7]; 1393 1394 #if !defined(DIAGNOSTIC) && !defined(DEBUG) && !defined(PMAPCHECK) 1395 return pt; 1396 #else 1397 if ((pvo->pvo_pte.pte_hi & PTE_VALID) && !PVO_PTEGIDX_ISSET(pvo)) { 1398 panic("pmap_pvo_to_pte: pvo %p: has valid pte in " 1399 "pvo but no valid pte index", pvo); 1400 } 1401 if ((pvo->pvo_pte.pte_hi & PTE_VALID) == 0 && PVO_PTEGIDX_ISSET(pvo)) { 1402 panic("pmap_pvo_to_pte: pvo %p: has valid pte index in " 1403 "pvo but no valid pte", pvo); 1404 } 1405 1406 if ((pt->pte_hi ^ (pvo->pvo_pte.pte_hi & ~PTE_VALID)) == PTE_VALID) { 1407 if ((pvo->pvo_pte.pte_hi & PTE_VALID) == 0) { 1408 #if defined(DEBUG) || defined(PMAPCHECK) 1409 pmap_pte_print(pt); 1410 #endif 1411 panic("pmap_pvo_to_pte: pvo %p: has valid pte in " 1412 "pmap_pteg_table %p but invalid in pvo", 1413 pvo, pt); 1414 } 1415 if (((pt->pte_lo ^ pvo->pvo_pte.pte_lo) & ~(PTE_CHG|PTE_REF)) != 0) { 1416 #if defined(DEBUG) || defined(PMAPCHECK) 1417 pmap_pte_print(pt); 1418 #endif 1419 panic("pmap_pvo_to_pte: pvo %p: pvo pte does " 1420 "not match pte %p in pmap_pteg_table", 1421 pvo, pt); 1422 } 1423 return pt; 1424 } 1425 1426 if (pvo->pvo_pte.pte_hi & PTE_VALID) { 1427 #if defined(DEBUG) || defined(PMAPCHECK) 1428 pmap_pte_print(pt); 1429 #endif 1430 panic("pmap_pvo_to_pte: pvo %p: has nomatching pte %p in " 1431 "pmap_pteg_table but valid in pvo", pvo, pt); 1432 } 1433 return NULL; 1434 #endif /* !(!DIAGNOSTIC && !DEBUG && !PMAPCHECK) */ 1435 } 1436 1437 struct pvo_entry * 1438 pmap_pvo_find_va(pmap_t pm, vaddr_t va, int *pteidx_p) 1439 { 1440 struct pvo_entry *pvo; 1441 int ptegidx; 1442 1443 va &= ~ADDR_POFF; 1444 ptegidx = va_to_pteg(pm, va); 1445 1446 TAILQ_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) { 1447 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 1448 if ((uintptr_t) pvo >= SEGMENT_LENGTH) 1449 panic("pmap_pvo_find_va: invalid pvo %p on " 1450 "list %#x (%p)", pvo, ptegidx, 1451 &pmap_pvo_table[ptegidx]); 1452 #endif 1453 if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) { 1454 if (pteidx_p) 1455 *pteidx_p = pmap_pvo_pte_index(pvo, ptegidx); 1456 return pvo; 1457 } 1458 } 1459 if ((pm == pmap_kernel()) && (va < SEGMENT_LENGTH)) 1460 panic("%s: returning NULL for %s pmap, va: 0x%08lx\n", __FUNCTION__, 1461 (pm == pmap_kernel() ? "kernel" : "user"), va); 1462 return NULL; 1463 } 1464 1465 #if defined(DEBUG) || defined(PMAPCHECK) 1466 void 1467 pmap_pvo_check(const struct pvo_entry *pvo) 1468 { 1469 struct pvo_head *pvo_head; 1470 struct pvo_entry *pvo0; 1471 volatile struct pte *pt; 1472 int failed = 0; 1473 1474 if ((uintptr_t)(pvo+1) >= SEGMENT_LENGTH) 1475 panic("pmap_pvo_check: pvo %p: invalid address", pvo); 1476 1477 if ((uintptr_t)(pvo->pvo_pmap+1) >= SEGMENT_LENGTH) { 1478 printf("pmap_pvo_check: pvo %p: invalid pmap address %p\n", 1479 pvo, pvo->pvo_pmap); 1480 failed = 1; 1481 } 1482 1483 if ((uintptr_t)TAILQ_NEXT(pvo, pvo_olink) >= SEGMENT_LENGTH || 1484 (((uintptr_t)TAILQ_NEXT(pvo, pvo_olink)) & 0x1f) != 0) { 1485 printf("pmap_pvo_check: pvo %p: invalid ovlink address %p\n", 1486 pvo, TAILQ_NEXT(pvo, pvo_olink)); 1487 failed = 1; 1488 } 1489 1490 if ((uintptr_t)LIST_NEXT(pvo, pvo_vlink) >= SEGMENT_LENGTH || 1491 (((uintptr_t)LIST_NEXT(pvo, pvo_vlink)) & 0x1f) != 0) { 1492 printf("pmap_pvo_check: pvo %p: invalid ovlink address %p\n", 1493 pvo, LIST_NEXT(pvo, pvo_vlink)); 1494 failed = 1; 1495 } 1496 1497 if (PVO_MANAGED_P(pvo)) { 1498 pvo_head = pa_to_pvoh(pvo->pvo_pte.pte_lo & PTE_RPGN, NULL); 1499 } else { 1500 if (pvo->pvo_vaddr < VM_MIN_KERNEL_ADDRESS) { 1501 printf("pmap_pvo_check: pvo %p: non kernel address " 1502 "on kernel unmanaged list\n", pvo); 1503 failed = 1; 1504 } 1505 pvo_head = &pmap_pvo_kunmanaged; 1506 } 1507 LIST_FOREACH(pvo0, pvo_head, pvo_vlink) { 1508 if (pvo0 == pvo) 1509 break; 1510 } 1511 if (pvo0 == NULL) { 1512 printf("pmap_pvo_check: pvo %p: not present " 1513 "on its vlist head %p\n", pvo, pvo_head); 1514 failed = 1; 1515 } 1516 if (pvo != pmap_pvo_find_va(pvo->pvo_pmap, pvo->pvo_vaddr, NULL)) { 1517 printf("pmap_pvo_check: pvo %p: not present " 1518 "on its olist head\n", pvo); 1519 failed = 1; 1520 } 1521 pt = pmap_pvo_to_pte(pvo, -1); 1522 if (pt == NULL) { 1523 if (pvo->pvo_pte.pte_hi & PTE_VALID) { 1524 printf("pmap_pvo_check: pvo %p: pte_hi VALID but " 1525 "no PTE\n", pvo); 1526 failed = 1; 1527 } 1528 } else { 1529 if ((uintptr_t) pt < (uintptr_t) &pmap_pteg_table[0] || 1530 (uintptr_t) pt >= 1531 (uintptr_t) &pmap_pteg_table[pmap_pteg_cnt]) { 1532 printf("pmap_pvo_check: pvo %p: pte %p not in " 1533 "pteg table\n", pvo, pt); 1534 failed = 1; 1535 } 1536 if (((((uintptr_t) pt) >> 3) & 7) != PVO_PTEGIDX_GET(pvo)) { 1537 printf("pmap_pvo_check: pvo %p: pte_hi VALID but " 1538 "no PTE\n", pvo); 1539 failed = 1; 1540 } 1541 if (pvo->pvo_pte.pte_hi != pt->pte_hi) { 1542 printf("pmap_pvo_check: pvo %p: pte_hi differ: " 1543 "%#x/%#x\n", pvo, (unsigned int) pvo->pvo_pte.pte_hi, (unsigned int) pt->pte_hi); 1544 failed = 1; 1545 } 1546 if (((pvo->pvo_pte.pte_lo ^ pt->pte_lo) & 1547 (PTE_PP|PTE_WIMG|PTE_RPGN)) != 0) { 1548 printf("pmap_pvo_check: pvo %p: pte_lo differ: " 1549 "%#x/%#x\n", pvo, 1550 (unsigned int) (pvo->pvo_pte.pte_lo & (PTE_PP|PTE_WIMG|PTE_RPGN)), 1551 (unsigned int) (pt->pte_lo & (PTE_PP|PTE_WIMG|PTE_RPGN))); 1552 failed = 1; 1553 } 1554 if ((pmap_pte_to_va(pt) ^ PVO_VADDR(pvo)) & 0x0fffffff) { 1555 printf("pmap_pvo_check: pvo %p: PTE %p derived VA %#lx" 1556 " doesn't not match PVO's VA %#lx\n", 1557 pvo, pt, pmap_pte_to_va(pt), PVO_VADDR(pvo)); 1558 failed = 1; 1559 } 1560 if (failed) 1561 pmap_pte_print(pt); 1562 } 1563 if (failed) 1564 panic("pmap_pvo_check: pvo %p, pm %p: bugcheck!", pvo, 1565 pvo->pvo_pmap); 1566 } 1567 #endif /* DEBUG || PMAPCHECK */ 1568 1569 /* 1570 * Search the PVO table looking for a non-wired entry. 1571 * If we find one, remove it and return it. 1572 */ 1573 1574 struct pvo_entry * 1575 pmap_pvo_reclaim(struct pmap *pm) 1576 { 1577 struct pvo_tqhead *pvoh; 1578 struct pvo_entry *pvo; 1579 uint32_t idx, endidx; 1580 1581 endidx = pmap_pvo_reclaim_nextidx; 1582 for (idx = (endidx + 1) & pmap_pteg_mask; idx != endidx; 1583 idx = (idx + 1) & pmap_pteg_mask) { 1584 pvoh = &pmap_pvo_table[idx]; 1585 TAILQ_FOREACH(pvo, pvoh, pvo_olink) { 1586 if (!PVO_WIRED_P(pvo)) { 1587 pmap_pvo_remove(pvo, -1, NULL); 1588 pmap_pvo_reclaim_nextidx = idx; 1589 PMAPCOUNT(pvos_reclaimed); 1590 return pvo; 1591 } 1592 } 1593 } 1594 return NULL; 1595 } 1596 1597 /* 1598 * This returns whether this is the first mapping of a page. 1599 */ 1600 int 1601 pmap_pvo_enter(pmap_t pm, struct pool *pl, struct pvo_head *pvo_head, 1602 vaddr_t va, paddr_t pa, register_t pte_lo, int flags) 1603 { 1604 struct pvo_entry *pvo; 1605 struct pvo_tqhead *pvoh; 1606 register_t msr; 1607 int ptegidx; 1608 int i; 1609 int poolflags = PR_NOWAIT; 1610 1611 /* 1612 * Compute the PTE Group index. 1613 */ 1614 va &= ~ADDR_POFF; 1615 ptegidx = va_to_pteg(pm, va); 1616 1617 msr = pmap_interrupts_off(); 1618 1619 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 1620 if (pmap_pvo_remove_depth > 0) 1621 panic("pmap_pvo_enter: called while pmap_pvo_remove active!"); 1622 if (++pmap_pvo_enter_depth > 1) 1623 panic("pmap_pvo_enter: called recursively!"); 1624 #endif 1625 1626 /* 1627 * Remove any existing mapping for this page. Reuse the 1628 * pvo entry if there a mapping. 1629 */ 1630 TAILQ_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) { 1631 if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) { 1632 #ifdef DEBUG 1633 if ((pmapdebug & PMAPDEBUG_PVOENTER) && 1634 ((pvo->pvo_pte.pte_lo ^ (pa|pte_lo)) & 1635 ~(PTE_REF|PTE_CHG)) == 0 && 1636 va < VM_MIN_KERNEL_ADDRESS) { 1637 printf("pmap_pvo_enter: pvo %p: dup %#x/%#lx\n", 1638 pvo, (unsigned int) pvo->pvo_pte.pte_lo, (unsigned int) pte_lo|pa); 1639 printf("pmap_pvo_enter: pte_hi=%#x sr=%#x\n", 1640 (unsigned int) pvo->pvo_pte.pte_hi, 1641 (unsigned int) pm->pm_sr[va >> ADDR_SR_SHFT]); 1642 pmap_pte_print(pmap_pvo_to_pte(pvo, -1)); 1643 #ifdef DDBX 1644 Debugger(); 1645 #endif 1646 } 1647 #endif 1648 PMAPCOUNT(mappings_replaced); 1649 pmap_pvo_remove(pvo, -1, NULL); 1650 break; 1651 } 1652 } 1653 1654 /* 1655 * If we aren't overwriting an mapping, try to allocate 1656 */ 1657 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 1658 --pmap_pvo_enter_depth; 1659 #endif 1660 pmap_interrupts_restore(msr); 1661 if (pvo) { 1662 pmap_pvo_free(pvo); 1663 } 1664 pvo = pool_get(pl, poolflags); 1665 1666 #ifdef DEBUG 1667 /* 1668 * Exercise pmap_pvo_reclaim() a little. 1669 */ 1670 if (pvo && (flags & PMAP_CANFAIL) != 0 && 1671 pmap_pvo_reclaim_debugctr++ > 0x1000 && 1672 (pmap_pvo_reclaim_debugctr & 0xff) == 0) { 1673 pool_put(pl, pvo); 1674 pvo = NULL; 1675 } 1676 #endif 1677 1678 msr = pmap_interrupts_off(); 1679 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 1680 ++pmap_pvo_enter_depth; 1681 #endif 1682 if (pvo == NULL) { 1683 pvo = pmap_pvo_reclaim(pm); 1684 if (pvo == NULL) { 1685 if ((flags & PMAP_CANFAIL) == 0) 1686 panic("pmap_pvo_enter: failed"); 1687 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 1688 pmap_pvo_enter_depth--; 1689 #endif 1690 PMAPCOUNT(pvos_failed); 1691 pmap_interrupts_restore(msr); 1692 return ENOMEM; 1693 } 1694 } 1695 1696 pvo->pvo_vaddr = va; 1697 pvo->pvo_pmap = pm; 1698 pvo->pvo_vaddr &= ~ADDR_POFF; 1699 if (flags & VM_PROT_EXECUTE) { 1700 PMAPCOUNT(exec_mappings); 1701 pvo_set_exec(pvo); 1702 } 1703 if (flags & PMAP_WIRED) 1704 pvo->pvo_vaddr |= PVO_WIRED; 1705 if (pvo_head != &pmap_pvo_kunmanaged) { 1706 pvo->pvo_vaddr |= PVO_MANAGED; 1707 PMAPCOUNT(mappings); 1708 } else { 1709 PMAPCOUNT(kernel_mappings); 1710 } 1711 pmap_pte_create(&pvo->pvo_pte, pm, va, pa | pte_lo); 1712 1713 LIST_INSERT_HEAD(pvo_head, pvo, pvo_vlink); 1714 if (PVO_WIRED_P(pvo)) 1715 pvo->pvo_pmap->pm_stats.wired_count++; 1716 pvo->pvo_pmap->pm_stats.resident_count++; 1717 #if defined(DEBUG) 1718 /* if (pm != pmap_kernel() && va < VM_MIN_KERNEL_ADDRESS) */ 1719 DPRINTFN(PVOENTER, 1720 ("pmap_pvo_enter: pvo %p: pm %p va %#lx pa %#lx\n", 1721 pvo, pm, va, pa)); 1722 #endif 1723 1724 /* 1725 * We hope this succeeds but it isn't required. 1726 */ 1727 pvoh = &pmap_pvo_table[ptegidx]; 1728 i = pmap_pte_insert(ptegidx, &pvo->pvo_pte); 1729 if (i >= 0) { 1730 PVO_PTEGIDX_SET(pvo, i); 1731 PVO_WHERE(pvo, ENTER_INSERT); 1732 PMAPCOUNT2(((pvo->pvo_pte.pte_hi & PTE_HID) 1733 ? pmap_evcnt_ptes_secondary : pmap_evcnt_ptes_primary)[i]); 1734 TAILQ_INSERT_TAIL(pvoh, pvo, pvo_olink); 1735 1736 } else { 1737 /* 1738 * Since we didn't have room for this entry (which makes it 1739 * and evicted entry), place it at the head of the list. 1740 */ 1741 TAILQ_INSERT_HEAD(pvoh, pvo, pvo_olink); 1742 PMAPCOUNT(ptes_evicted); 1743 pm->pm_evictions++; 1744 /* 1745 * If this is a kernel page, make sure it's active. 1746 */ 1747 if (pm == pmap_kernel()) { 1748 i = pmap_pte_spill(pm, va, false); 1749 KASSERT(i); 1750 } 1751 } 1752 PMAP_PVO_CHECK(pvo); /* sanity check */ 1753 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 1754 pmap_pvo_enter_depth--; 1755 #endif 1756 pmap_interrupts_restore(msr); 1757 return 0; 1758 } 1759 1760 void 1761 pmap_pvo_remove(struct pvo_entry *pvo, int pteidx, struct pvo_head *pvol) 1762 { 1763 volatile struct pte *pt; 1764 int ptegidx; 1765 1766 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 1767 if (++pmap_pvo_remove_depth > 1) 1768 panic("pmap_pvo_remove: called recursively!"); 1769 #endif 1770 1771 /* 1772 * If we haven't been supplied the ptegidx, calculate it. 1773 */ 1774 if (pteidx == -1) { 1775 ptegidx = va_to_pteg(pvo->pvo_pmap, pvo->pvo_vaddr); 1776 pteidx = pmap_pvo_pte_index(pvo, ptegidx); 1777 } else { 1778 ptegidx = pteidx >> 3; 1779 if (pvo->pvo_pte.pte_hi & PTE_HID) 1780 ptegidx ^= pmap_pteg_mask; 1781 } 1782 PMAP_PVO_CHECK(pvo); /* sanity check */ 1783 1784 /* 1785 * If there is an active pte entry, we need to deactivate it 1786 * (and save the ref & chg bits). 1787 */ 1788 pt = pmap_pvo_to_pte(pvo, pteidx); 1789 if (pt != NULL) { 1790 pmap_pte_unset(pt, &pvo->pvo_pte, pvo->pvo_vaddr); 1791 PVO_WHERE(pvo, REMOVE); 1792 PVO_PTEGIDX_CLR(pvo); 1793 PMAPCOUNT(ptes_removed); 1794 } else { 1795 KASSERT(pvo->pvo_pmap->pm_evictions > 0); 1796 pvo->pvo_pmap->pm_evictions--; 1797 } 1798 1799 /* 1800 * Account for executable mappings. 1801 */ 1802 if (PVO_EXECUTABLE_P(pvo)) 1803 pvo_clear_exec(pvo); 1804 1805 /* 1806 * Update our statistics. 1807 */ 1808 pvo->pvo_pmap->pm_stats.resident_count--; 1809 if (PVO_WIRED_P(pvo)) 1810 pvo->pvo_pmap->pm_stats.wired_count--; 1811 1812 /* 1813 * Save the REF/CHG bits into their cache if the page is managed. 1814 */ 1815 if (PVO_MANAGED_P(pvo)) { 1816 register_t ptelo = pvo->pvo_pte.pte_lo; 1817 struct vm_page *pg = PHYS_TO_VM_PAGE(ptelo & PTE_RPGN); 1818 1819 if (pg != NULL) { 1820 /* 1821 * If this page was changed and it is mapped exec, 1822 * invalidate it. 1823 */ 1824 if ((ptelo & PTE_CHG) && 1825 (pmap_attr_fetch(pg) & PTE_EXEC)) { 1826 struct pvo_head *pvoh = vm_page_to_pvoh(pg); 1827 if (LIST_EMPTY(pvoh)) { 1828 DPRINTFN(EXEC, ("[pmap_pvo_remove: " 1829 "%#lx: clear-exec]\n", 1830 VM_PAGE_TO_PHYS(pg))); 1831 pmap_attr_clear(pg, PTE_EXEC); 1832 PMAPCOUNT(exec_uncached_pvo_remove); 1833 } else { 1834 DPRINTFN(EXEC, ("[pmap_pvo_remove: " 1835 "%#lx: syncicache]\n", 1836 VM_PAGE_TO_PHYS(pg))); 1837 pmap_syncicache(VM_PAGE_TO_PHYS(pg), 1838 PAGE_SIZE); 1839 PMAPCOUNT(exec_synced_pvo_remove); 1840 } 1841 } 1842 1843 pmap_attr_save(pg, ptelo & (PTE_REF|PTE_CHG)); 1844 } 1845 PMAPCOUNT(unmappings); 1846 } else { 1847 PMAPCOUNT(kernel_unmappings); 1848 } 1849 1850 /* 1851 * Remove the PVO from its lists and return it to the pool. 1852 */ 1853 LIST_REMOVE(pvo, pvo_vlink); 1854 TAILQ_REMOVE(&pmap_pvo_table[ptegidx], pvo, pvo_olink); 1855 if (pvol) { 1856 LIST_INSERT_HEAD(pvol, pvo, pvo_vlink); 1857 } 1858 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 1859 pmap_pvo_remove_depth--; 1860 #endif 1861 } 1862 1863 void 1864 pmap_pvo_free(struct pvo_entry *pvo) 1865 { 1866 1867 pool_put(PVO_MANAGED_P(pvo) ? &pmap_mpvo_pool : &pmap_upvo_pool, pvo); 1868 } 1869 1870 void 1871 pmap_pvo_free_list(struct pvo_head *pvol) 1872 { 1873 struct pvo_entry *pvo, *npvo; 1874 1875 for (pvo = LIST_FIRST(pvol); pvo != NULL; pvo = npvo) { 1876 npvo = LIST_NEXT(pvo, pvo_vlink); 1877 LIST_REMOVE(pvo, pvo_vlink); 1878 pmap_pvo_free(pvo); 1879 } 1880 } 1881 1882 /* 1883 * Mark a mapping as executable. 1884 * If this is the first executable mapping in the segment, 1885 * clear the noexec flag. 1886 */ 1887 STATIC void 1888 pvo_set_exec(struct pvo_entry *pvo) 1889 { 1890 struct pmap *pm = pvo->pvo_pmap; 1891 1892 if (pm == pmap_kernel() || PVO_EXECUTABLE_P(pvo)) { 1893 return; 1894 } 1895 pvo->pvo_vaddr |= PVO_EXECUTABLE; 1896 #if defined (PPC_OEA) || defined (PPC_OEA64_BRIDGE) 1897 { 1898 int sr = PVO_VADDR(pvo) >> ADDR_SR_SHFT; 1899 if (pm->pm_exec[sr]++ == 0) { 1900 pm->pm_sr[sr] &= ~SR_NOEXEC; 1901 } 1902 } 1903 #endif 1904 } 1905 1906 /* 1907 * Mark a mapping as non-executable. 1908 * If this was the last executable mapping in the segment, 1909 * set the noexec flag. 1910 */ 1911 STATIC void 1912 pvo_clear_exec(struct pvo_entry *pvo) 1913 { 1914 struct pmap *pm = pvo->pvo_pmap; 1915 1916 if (pm == pmap_kernel() || !PVO_EXECUTABLE_P(pvo)) { 1917 return; 1918 } 1919 pvo->pvo_vaddr &= ~PVO_EXECUTABLE; 1920 #if defined (PPC_OEA) || defined (PPC_OEA64_BRIDGE) 1921 { 1922 int sr = PVO_VADDR(pvo) >> ADDR_SR_SHFT; 1923 if (--pm->pm_exec[sr] == 0) { 1924 pm->pm_sr[sr] |= SR_NOEXEC; 1925 } 1926 } 1927 #endif 1928 } 1929 1930 /* 1931 * Insert physical page at pa into the given pmap at virtual address va. 1932 */ 1933 int 1934 pmap_enter(pmap_t pm, vaddr_t va, paddr_t pa, vm_prot_t prot, int flags) 1935 { 1936 struct mem_region *mp; 1937 struct pvo_head *pvo_head; 1938 struct vm_page *pg; 1939 struct pool *pl; 1940 register_t pte_lo; 1941 int error; 1942 u_int pvo_flags; 1943 u_int was_exec = 0; 1944 1945 if (__predict_false(!pmap_initialized)) { 1946 pvo_head = &pmap_pvo_kunmanaged; 1947 pl = &pmap_upvo_pool; 1948 pvo_flags = 0; 1949 pg = NULL; 1950 was_exec = PTE_EXEC; 1951 } else { 1952 pvo_head = pa_to_pvoh(pa, &pg); 1953 pl = &pmap_mpvo_pool; 1954 pvo_flags = PVO_MANAGED; 1955 } 1956 1957 DPRINTFN(ENTER, 1958 ("pmap_enter(%p, 0x%lx, 0x%lx, 0x%x, 0x%x):", 1959 pm, va, pa, prot, flags)); 1960 1961 /* 1962 * If this is a managed page, and it's the first reference to the 1963 * page clear the execness of the page. Otherwise fetch the execness. 1964 */ 1965 if (pg != NULL) 1966 was_exec = pmap_attr_fetch(pg) & PTE_EXEC; 1967 1968 DPRINTFN(ENTER, (" was_exec=%d", was_exec)); 1969 1970 /* 1971 * Assume the page is cache inhibited and access is guarded unless 1972 * it's in our available memory array. If it is in the memory array, 1973 * asssume it's in memory coherent memory. 1974 */ 1975 pte_lo = PTE_IG; 1976 if ((flags & PMAP_NC) == 0) { 1977 for (mp = mem; mp->size; mp++) { 1978 if (pa >= mp->start && pa < mp->start + mp->size) { 1979 pte_lo = PTE_M; 1980 break; 1981 } 1982 } 1983 } 1984 1985 if (prot & VM_PROT_WRITE) 1986 pte_lo |= PTE_BW; 1987 else 1988 pte_lo |= PTE_BR; 1989 1990 /* 1991 * If this was in response to a fault, "pre-fault" the PTE's 1992 * changed/referenced bit appropriately. 1993 */ 1994 if (flags & VM_PROT_WRITE) 1995 pte_lo |= PTE_CHG; 1996 if (flags & VM_PROT_ALL) 1997 pte_lo |= PTE_REF; 1998 1999 /* 2000 * We need to know if this page can be executable 2001 */ 2002 flags |= (prot & VM_PROT_EXECUTE); 2003 2004 /* 2005 * Record mapping for later back-translation and pte spilling. 2006 * This will overwrite any existing mapping. 2007 */ 2008 error = pmap_pvo_enter(pm, pl, pvo_head, va, pa, pte_lo, flags); 2009 2010 /* 2011 * Flush the real page from the instruction cache if this page is 2012 * mapped executable and cacheable and has not been flushed since 2013 * the last time it was modified. 2014 */ 2015 if (error == 0 && 2016 (flags & VM_PROT_EXECUTE) && 2017 (pte_lo & PTE_I) == 0 && 2018 was_exec == 0) { 2019 DPRINTFN(ENTER, (" syncicache")); 2020 PMAPCOUNT(exec_synced); 2021 pmap_syncicache(pa, PAGE_SIZE); 2022 if (pg != NULL) { 2023 pmap_attr_save(pg, PTE_EXEC); 2024 PMAPCOUNT(exec_cached); 2025 #if defined(DEBUG) || defined(PMAPDEBUG) 2026 if (pmapdebug & PMAPDEBUG_ENTER) 2027 printf(" marked-as-exec"); 2028 else if (pmapdebug & PMAPDEBUG_EXEC) 2029 printf("[pmap_enter: %#lx: marked-as-exec]\n", 2030 VM_PAGE_TO_PHYS(pg)); 2031 2032 #endif 2033 } 2034 } 2035 2036 DPRINTFN(ENTER, (": error=%d\n", error)); 2037 2038 return error; 2039 } 2040 2041 void 2042 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot) 2043 { 2044 struct mem_region *mp; 2045 register_t pte_lo; 2046 int error; 2047 2048 #if defined (PPC_OEA64_BRIDGE) 2049 if (va < VM_MIN_KERNEL_ADDRESS) 2050 panic("pmap_kenter_pa: attempt to enter " 2051 "non-kernel address %#lx!", va); 2052 #endif 2053 2054 DPRINTFN(KENTER, 2055 ("pmap_kenter_pa(%#lx,%#lx,%#x)\n", va, pa, prot)); 2056 2057 /* 2058 * Assume the page is cache inhibited and access is guarded unless 2059 * it's in our available memory array. If it is in the memory array, 2060 * asssume it's in memory coherent memory. 2061 */ 2062 pte_lo = PTE_IG; 2063 if ((prot & PMAP_NC) == 0) { 2064 for (mp = mem; mp->size; mp++) { 2065 if (pa >= mp->start && pa < mp->start + mp->size) { 2066 pte_lo = PTE_M; 2067 break; 2068 } 2069 } 2070 } 2071 2072 if (prot & VM_PROT_WRITE) 2073 pte_lo |= PTE_BW; 2074 else 2075 pte_lo |= PTE_BR; 2076 2077 /* 2078 * We don't care about REF/CHG on PVOs on the unmanaged list. 2079 */ 2080 error = pmap_pvo_enter(pmap_kernel(), &pmap_upvo_pool, 2081 &pmap_pvo_kunmanaged, va, pa, pte_lo, prot|PMAP_WIRED); 2082 2083 if (error != 0) 2084 panic("pmap_kenter_pa: failed to enter va %#lx pa %#lx: %d", 2085 va, pa, error); 2086 } 2087 2088 void 2089 pmap_kremove(vaddr_t va, vsize_t len) 2090 { 2091 if (va < VM_MIN_KERNEL_ADDRESS) 2092 panic("pmap_kremove: attempt to remove " 2093 "non-kernel address %#lx!", va); 2094 2095 DPRINTFN(KREMOVE,("pmap_kremove(%#lx,%#lx)\n", va, len)); 2096 pmap_remove(pmap_kernel(), va, va + len); 2097 } 2098 2099 /* 2100 * Remove the given range of mapping entries. 2101 */ 2102 void 2103 pmap_remove(pmap_t pm, vaddr_t va, vaddr_t endva) 2104 { 2105 struct pvo_head pvol; 2106 struct pvo_entry *pvo; 2107 register_t msr; 2108 int pteidx; 2109 2110 LIST_INIT(&pvol); 2111 msr = pmap_interrupts_off(); 2112 for (; va < endva; va += PAGE_SIZE) { 2113 pvo = pmap_pvo_find_va(pm, va, &pteidx); 2114 if (pvo != NULL) { 2115 pmap_pvo_remove(pvo, pteidx, &pvol); 2116 } 2117 } 2118 pmap_interrupts_restore(msr); 2119 pmap_pvo_free_list(&pvol); 2120 } 2121 2122 /* 2123 * Get the physical page address for the given pmap/virtual address. 2124 */ 2125 bool 2126 pmap_extract(pmap_t pm, vaddr_t va, paddr_t *pap) 2127 { 2128 struct pvo_entry *pvo; 2129 register_t msr; 2130 2131 2132 /* 2133 * If this is a kernel pmap lookup, also check the battable 2134 * and if we get a hit, translate the VA to a PA using the 2135 * BAT entries. Don't check for VM_MAX_KERNEL_ADDRESS is 2136 * that will wrap back to 0. 2137 */ 2138 if (pm == pmap_kernel() && 2139 (va < VM_MIN_KERNEL_ADDRESS || 2140 (KERNEL2_SR < 15 && VM_MAX_KERNEL_ADDRESS <= va))) { 2141 KASSERT((va >> ADDR_SR_SHFT) != USER_SR); 2142 #if defined (PPC_OEA) 2143 if ((MFPVR() >> 16) != MPC601) { 2144 register_t batu = battable[va >> ADDR_SR_SHFT].batu; 2145 if (BAT_VALID_P(batu,0) && BAT_VA_MATCH_P(batu,va)) { 2146 register_t batl = 2147 battable[va >> ADDR_SR_SHFT].batl; 2148 register_t mask = 2149 (~(batu & BAT_BL) << 15) & ~0x1ffffL; 2150 if (pap) 2151 *pap = (batl & mask) | (va & ~mask); 2152 return true; 2153 } 2154 } else { 2155 register_t batu = battable[va >> 23].batu; 2156 register_t batl = battable[va >> 23].batl; 2157 register_t sr = iosrtable[va >> ADDR_SR_SHFT]; 2158 if (BAT601_VALID_P(batl) && 2159 BAT601_VA_MATCH_P(batu, batl, va)) { 2160 register_t mask = 2161 (~(batl & BAT601_BSM) << 17) & ~0x1ffffL; 2162 if (pap) 2163 *pap = (batl & mask) | (va & ~mask); 2164 return true; 2165 } else if (SR601_VALID_P(sr) && 2166 SR601_PA_MATCH_P(sr, va)) { 2167 if (pap) 2168 *pap = va; 2169 return true; 2170 } 2171 } 2172 return false; 2173 #elif defined (PPC_OEA64_BRIDGE) 2174 panic("%s: pm: %s, va: 0x%08lx\n", __FUNCTION__, 2175 (pm == pmap_kernel() ? "kernel" : "user"), va); 2176 #elif defined (PPC_OEA64) 2177 #error PPC_OEA64 not supported 2178 #endif /* PPC_OEA */ 2179 } 2180 2181 msr = pmap_interrupts_off(); 2182 pvo = pmap_pvo_find_va(pm, va & ~ADDR_POFF, NULL); 2183 if (pvo != NULL) { 2184 PMAP_PVO_CHECK(pvo); /* sanity check */ 2185 if (pap) 2186 *pap = (pvo->pvo_pte.pte_lo & PTE_RPGN) 2187 | (va & ADDR_POFF); 2188 } 2189 pmap_interrupts_restore(msr); 2190 return pvo != NULL; 2191 } 2192 2193 /* 2194 * Lower the protection on the specified range of this pmap. 2195 */ 2196 void 2197 pmap_protect(pmap_t pm, vaddr_t va, vaddr_t endva, vm_prot_t prot) 2198 { 2199 struct pvo_entry *pvo; 2200 volatile struct pte *pt; 2201 register_t msr; 2202 int pteidx; 2203 2204 /* 2205 * Since this routine only downgrades protection, we should 2206 * always be called with at least one bit not set. 2207 */ 2208 KASSERT(prot != VM_PROT_ALL); 2209 2210 /* 2211 * If there is no protection, this is equivalent to 2212 * remove the pmap from the pmap. 2213 */ 2214 if ((prot & VM_PROT_READ) == 0) { 2215 pmap_remove(pm, va, endva); 2216 return; 2217 } 2218 2219 msr = pmap_interrupts_off(); 2220 for (; va < endva; va += PAGE_SIZE) { 2221 pvo = pmap_pvo_find_va(pm, va, &pteidx); 2222 if (pvo == NULL) 2223 continue; 2224 PMAP_PVO_CHECK(pvo); /* sanity check */ 2225 2226 /* 2227 * Revoke executable if asked to do so. 2228 */ 2229 if ((prot & VM_PROT_EXECUTE) == 0) 2230 pvo_clear_exec(pvo); 2231 2232 #if 0 2233 /* 2234 * If the page is already read-only, no change 2235 * needs to be made. 2236 */ 2237 if ((pvo->pvo_pte.pte_lo & PTE_PP) == PTE_BR) 2238 continue; 2239 #endif 2240 /* 2241 * Grab the PTE pointer before we diddle with 2242 * the cached PTE copy. 2243 */ 2244 pt = pmap_pvo_to_pte(pvo, pteidx); 2245 /* 2246 * Change the protection of the page. 2247 */ 2248 pvo->pvo_pte.pte_lo &= ~PTE_PP; 2249 pvo->pvo_pte.pte_lo |= PTE_BR; 2250 2251 /* 2252 * If the PVO is in the page table, update 2253 * that pte at well. 2254 */ 2255 if (pt != NULL) { 2256 pmap_pte_change(pt, &pvo->pvo_pte, pvo->pvo_vaddr); 2257 PVO_WHERE(pvo, PMAP_PROTECT); 2258 PMAPCOUNT(ptes_changed); 2259 } 2260 2261 PMAP_PVO_CHECK(pvo); /* sanity check */ 2262 } 2263 pmap_interrupts_restore(msr); 2264 } 2265 2266 void 2267 pmap_unwire(pmap_t pm, vaddr_t va) 2268 { 2269 struct pvo_entry *pvo; 2270 register_t msr; 2271 2272 msr = pmap_interrupts_off(); 2273 pvo = pmap_pvo_find_va(pm, va, NULL); 2274 if (pvo != NULL) { 2275 if (PVO_WIRED_P(pvo)) { 2276 pvo->pvo_vaddr &= ~PVO_WIRED; 2277 pm->pm_stats.wired_count--; 2278 } 2279 PMAP_PVO_CHECK(pvo); /* sanity check */ 2280 } 2281 pmap_interrupts_restore(msr); 2282 } 2283 2284 /* 2285 * Lower the protection on the specified physical page. 2286 */ 2287 void 2288 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 2289 { 2290 struct pvo_head *pvo_head, pvol; 2291 struct pvo_entry *pvo, *next_pvo; 2292 volatile struct pte *pt; 2293 register_t msr; 2294 2295 KASSERT(prot != VM_PROT_ALL); 2296 LIST_INIT(&pvol); 2297 msr = pmap_interrupts_off(); 2298 2299 /* 2300 * When UVM reuses a page, it does a pmap_page_protect with 2301 * VM_PROT_NONE. At that point, we can clear the exec flag 2302 * since we know the page will have different contents. 2303 */ 2304 if ((prot & VM_PROT_READ) == 0) { 2305 DPRINTFN(EXEC, ("[pmap_page_protect: %#lx: clear-exec]\n", 2306 VM_PAGE_TO_PHYS(pg))); 2307 if (pmap_attr_fetch(pg) & PTE_EXEC) { 2308 PMAPCOUNT(exec_uncached_page_protect); 2309 pmap_attr_clear(pg, PTE_EXEC); 2310 } 2311 } 2312 2313 pvo_head = vm_page_to_pvoh(pg); 2314 for (pvo = LIST_FIRST(pvo_head); pvo != NULL; pvo = next_pvo) { 2315 next_pvo = LIST_NEXT(pvo, pvo_vlink); 2316 PMAP_PVO_CHECK(pvo); /* sanity check */ 2317 2318 /* 2319 * Downgrading to no mapping at all, we just remove the entry. 2320 */ 2321 if ((prot & VM_PROT_READ) == 0) { 2322 pmap_pvo_remove(pvo, -1, &pvol); 2323 continue; 2324 } 2325 2326 /* 2327 * If EXEC permission is being revoked, just clear the 2328 * flag in the PVO. 2329 */ 2330 if ((prot & VM_PROT_EXECUTE) == 0) 2331 pvo_clear_exec(pvo); 2332 2333 /* 2334 * If this entry is already RO, don't diddle with the 2335 * page table. 2336 */ 2337 if ((pvo->pvo_pte.pte_lo & PTE_PP) == PTE_BR) { 2338 PMAP_PVO_CHECK(pvo); 2339 continue; 2340 } 2341 2342 /* 2343 * Grab the PTE before the we diddle the bits so 2344 * pvo_to_pte can verify the pte contents are as 2345 * expected. 2346 */ 2347 pt = pmap_pvo_to_pte(pvo, -1); 2348 pvo->pvo_pte.pte_lo &= ~PTE_PP; 2349 pvo->pvo_pte.pte_lo |= PTE_BR; 2350 if (pt != NULL) { 2351 pmap_pte_change(pt, &pvo->pvo_pte, pvo->pvo_vaddr); 2352 PVO_WHERE(pvo, PMAP_PAGE_PROTECT); 2353 PMAPCOUNT(ptes_changed); 2354 } 2355 PMAP_PVO_CHECK(pvo); /* sanity check */ 2356 } 2357 pmap_interrupts_restore(msr); 2358 pmap_pvo_free_list(&pvol); 2359 } 2360 2361 /* 2362 * Activate the address space for the specified process. If the process 2363 * is the current process, load the new MMU context. 2364 */ 2365 void 2366 pmap_activate(struct lwp *l) 2367 { 2368 struct pcb *pcb = &l->l_addr->u_pcb; 2369 pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap; 2370 2371 DPRINTFN(ACTIVATE, 2372 ("pmap_activate: lwp %p (curlwp %p)\n", l, curlwp)); 2373 2374 /* 2375 * XXX Normally performed in cpu_fork(). 2376 */ 2377 pcb->pcb_pm = pmap; 2378 2379 /* 2380 * In theory, the SR registers need only be valid on return 2381 * to user space wait to do them there. 2382 */ 2383 if (l == curlwp) { 2384 /* Store pointer to new current pmap. */ 2385 curpm = pmap; 2386 } 2387 } 2388 2389 /* 2390 * Deactivate the specified process's address space. 2391 */ 2392 void 2393 pmap_deactivate(struct lwp *l) 2394 { 2395 } 2396 2397 bool 2398 pmap_query_bit(struct vm_page *pg, int ptebit) 2399 { 2400 struct pvo_entry *pvo; 2401 volatile struct pte *pt; 2402 register_t msr; 2403 2404 if (pmap_attr_fetch(pg) & ptebit) 2405 return true; 2406 2407 msr = pmap_interrupts_off(); 2408 LIST_FOREACH(pvo, vm_page_to_pvoh(pg), pvo_vlink) { 2409 PMAP_PVO_CHECK(pvo); /* sanity check */ 2410 /* 2411 * See if we saved the bit off. If so cache, it and return 2412 * success. 2413 */ 2414 if (pvo->pvo_pte.pte_lo & ptebit) { 2415 pmap_attr_save(pg, ptebit); 2416 PMAP_PVO_CHECK(pvo); /* sanity check */ 2417 pmap_interrupts_restore(msr); 2418 return true; 2419 } 2420 } 2421 /* 2422 * No luck, now go thru the hard part of looking at the ptes 2423 * themselves. Sync so any pending REF/CHG bits are flushed 2424 * to the PTEs. 2425 */ 2426 SYNC(); 2427 LIST_FOREACH(pvo, vm_page_to_pvoh(pg), pvo_vlink) { 2428 PMAP_PVO_CHECK(pvo); /* sanity check */ 2429 /* 2430 * See if this pvo have a valid PTE. If so, fetch the 2431 * REF/CHG bits from the valid PTE. If the appropriate 2432 * ptebit is set, cache, it and return success. 2433 */ 2434 pt = pmap_pvo_to_pte(pvo, -1); 2435 if (pt != NULL) { 2436 pmap_pte_synch(pt, &pvo->pvo_pte); 2437 if (pvo->pvo_pte.pte_lo & ptebit) { 2438 pmap_attr_save(pg, ptebit); 2439 PMAP_PVO_CHECK(pvo); /* sanity check */ 2440 pmap_interrupts_restore(msr); 2441 return true; 2442 } 2443 } 2444 } 2445 pmap_interrupts_restore(msr); 2446 return false; 2447 } 2448 2449 bool 2450 pmap_clear_bit(struct vm_page *pg, int ptebit) 2451 { 2452 struct pvo_head *pvoh = vm_page_to_pvoh(pg); 2453 struct pvo_entry *pvo; 2454 volatile struct pte *pt; 2455 register_t msr; 2456 int rv = 0; 2457 2458 msr = pmap_interrupts_off(); 2459 2460 /* 2461 * Fetch the cache value 2462 */ 2463 rv |= pmap_attr_fetch(pg); 2464 2465 /* 2466 * Clear the cached value. 2467 */ 2468 pmap_attr_clear(pg, ptebit); 2469 2470 /* 2471 * Sync so any pending REF/CHG bits are flushed to the PTEs (so we 2472 * can reset the right ones). Note that since the pvo entries and 2473 * list heads are accessed via BAT0 and are never placed in the 2474 * page table, we don't have to worry about further accesses setting 2475 * the REF/CHG bits. 2476 */ 2477 SYNC(); 2478 2479 /* 2480 * For each pvo entry, clear pvo's ptebit. If this pvo have a 2481 * valid PTE. If so, clear the ptebit from the valid PTE. 2482 */ 2483 LIST_FOREACH(pvo, pvoh, pvo_vlink) { 2484 PMAP_PVO_CHECK(pvo); /* sanity check */ 2485 pt = pmap_pvo_to_pte(pvo, -1); 2486 if (pt != NULL) { 2487 /* 2488 * Only sync the PTE if the bit we are looking 2489 * for is not already set. 2490 */ 2491 if ((pvo->pvo_pte.pte_lo & ptebit) == 0) 2492 pmap_pte_synch(pt, &pvo->pvo_pte); 2493 /* 2494 * If the bit we are looking for was already set, 2495 * clear that bit in the pte. 2496 */ 2497 if (pvo->pvo_pte.pte_lo & ptebit) 2498 pmap_pte_clear(pt, PVO_VADDR(pvo), ptebit); 2499 } 2500 rv |= pvo->pvo_pte.pte_lo & (PTE_CHG|PTE_REF); 2501 pvo->pvo_pte.pte_lo &= ~ptebit; 2502 PMAP_PVO_CHECK(pvo); /* sanity check */ 2503 } 2504 pmap_interrupts_restore(msr); 2505 2506 /* 2507 * If we are clearing the modify bit and this page was marked EXEC 2508 * and the user of the page thinks the page was modified, then we 2509 * need to clean it from the icache if it's mapped or clear the EXEC 2510 * bit if it's not mapped. The page itself might not have the CHG 2511 * bit set if the modification was done via DMA to the page. 2512 */ 2513 if ((ptebit & PTE_CHG) && (rv & PTE_EXEC)) { 2514 if (LIST_EMPTY(pvoh)) { 2515 DPRINTFN(EXEC, ("[pmap_clear_bit: %#lx: clear-exec]\n", 2516 VM_PAGE_TO_PHYS(pg))); 2517 pmap_attr_clear(pg, PTE_EXEC); 2518 PMAPCOUNT(exec_uncached_clear_modify); 2519 } else { 2520 DPRINTFN(EXEC, ("[pmap_clear_bit: %#lx: syncicache]\n", 2521 VM_PAGE_TO_PHYS(pg))); 2522 pmap_syncicache(VM_PAGE_TO_PHYS(pg), PAGE_SIZE); 2523 PMAPCOUNT(exec_synced_clear_modify); 2524 } 2525 } 2526 return (rv & ptebit) != 0; 2527 } 2528 2529 void 2530 pmap_procwr(struct proc *p, vaddr_t va, size_t len) 2531 { 2532 struct pvo_entry *pvo; 2533 size_t offset = va & ADDR_POFF; 2534 int s; 2535 2536 s = splvm(); 2537 while (len > 0) { 2538 size_t seglen = PAGE_SIZE - offset; 2539 if (seglen > len) 2540 seglen = len; 2541 pvo = pmap_pvo_find_va(p->p_vmspace->vm_map.pmap, va, NULL); 2542 if (pvo != NULL && PVO_EXECUTABLE_P(pvo)) { 2543 pmap_syncicache( 2544 (pvo->pvo_pte.pte_lo & PTE_RPGN) | offset, seglen); 2545 PMAP_PVO_CHECK(pvo); 2546 } 2547 va += seglen; 2548 len -= seglen; 2549 offset = 0; 2550 } 2551 splx(s); 2552 } 2553 2554 #if defined(DEBUG) || defined(PMAPCHECK) || defined(DDB) 2555 void 2556 pmap_pte_print(volatile struct pte *pt) 2557 { 2558 printf("PTE %p: ", pt); 2559 2560 #if defined(PPC_OEA) 2561 /* High word: */ 2562 printf("0x%08lx: [", pt->pte_hi); 2563 #elif defined (PPC_OEA64_BRIDGE) 2564 printf("0x%016llx: [", pt->pte_hi); 2565 #else /* PPC_OEA64 */ 2566 printf("0x%016lx: [", pt->pte_hi); 2567 #endif /* PPC_OEA */ 2568 2569 printf("%c ", (pt->pte_hi & PTE_VALID) ? 'v' : 'i'); 2570 printf("%c ", (pt->pte_hi & PTE_HID) ? 'h' : '-'); 2571 2572 #if defined (PPC_OEA) 2573 printf("0x%06lx 0x%02lx", 2574 (pt->pte_hi &~ PTE_VALID)>>PTE_VSID_SHFT, 2575 pt->pte_hi & PTE_API); 2576 printf(" (va 0x%08lx)] ", pmap_pte_to_va(pt)); 2577 #elif defined (PPC_OEA64) 2578 printf("0x%06lx 0x%02lx", 2579 (pt->pte_hi &~ PTE_VALID)>>PTE_VSID_SHFT, 2580 pt->pte_hi & PTE_API); 2581 printf(" (va 0x%016lx)] ", pmap_pte_to_va(pt)); 2582 #else 2583 /* PPC_OEA64_BRIDGE */ 2584 printf("0x%06llx 0x%02llx", 2585 (pt->pte_hi &~ PTE_VALID)>>PTE_VSID_SHFT, 2586 pt->pte_hi & PTE_API); 2587 printf(" (va 0x%08lx)] ", pmap_pte_to_va(pt)); 2588 #endif /* PPC_OEA */ 2589 2590 /* Low word: */ 2591 #if defined (PPC_OEA) 2592 printf(" 0x%08lx: [", pt->pte_lo); 2593 printf("0x%05lx... ", pt->pte_lo >> 12); 2594 #elif defined (PPC_OEA64) 2595 printf(" 0x%016lx: [", pt->pte_lo); 2596 printf("0x%012lx... ", pt->pte_lo >> 12); 2597 #else /* PPC_OEA64_BRIDGE */ 2598 printf(" 0x%016llx: [", pt->pte_lo); 2599 printf("0x%012llx... ", pt->pte_lo >> 12); 2600 #endif 2601 printf("%c ", (pt->pte_lo & PTE_REF) ? 'r' : 'u'); 2602 printf("%c ", (pt->pte_lo & PTE_CHG) ? 'c' : 'n'); 2603 printf("%c", (pt->pte_lo & PTE_W) ? 'w' : '.'); 2604 printf("%c", (pt->pte_lo & PTE_I) ? 'i' : '.'); 2605 printf("%c", (pt->pte_lo & PTE_M) ? 'm' : '.'); 2606 printf("%c ", (pt->pte_lo & PTE_G) ? 'g' : '.'); 2607 switch (pt->pte_lo & PTE_PP) { 2608 case PTE_BR: printf("br]\n"); break; 2609 case PTE_BW: printf("bw]\n"); break; 2610 case PTE_SO: printf("so]\n"); break; 2611 case PTE_SW: printf("sw]\n"); break; 2612 } 2613 } 2614 #endif 2615 2616 #if defined(DDB) 2617 void 2618 pmap_pteg_check(void) 2619 { 2620 volatile struct pte *pt; 2621 int i; 2622 int ptegidx; 2623 u_int p_valid = 0; 2624 u_int s_valid = 0; 2625 u_int invalid = 0; 2626 2627 for (ptegidx = 0; ptegidx < pmap_pteg_cnt; ptegidx++) { 2628 for (pt = pmap_pteg_table[ptegidx].pt, i = 8; --i >= 0; pt++) { 2629 if (pt->pte_hi & PTE_VALID) { 2630 if (pt->pte_hi & PTE_HID) 2631 s_valid++; 2632 else 2633 { 2634 p_valid++; 2635 } 2636 } else 2637 invalid++; 2638 } 2639 } 2640 printf("pteg_check: v(p) %#x (%d), v(s) %#x (%d), i %#x (%d)\n", 2641 p_valid, p_valid, s_valid, s_valid, 2642 invalid, invalid); 2643 } 2644 2645 void 2646 pmap_print_mmuregs(void) 2647 { 2648 int i; 2649 u_int cpuvers; 2650 #ifndef PPC_OEA64 2651 vaddr_t addr; 2652 register_t soft_sr[16]; 2653 #endif 2654 #if defined (PPC_OEA) && !defined (PPC_OEA64) && !defined (PPC_OEA64_BRIDGE) 2655 struct bat soft_ibat[4]; 2656 struct bat soft_dbat[4]; 2657 #endif 2658 register_t sdr1; 2659 2660 cpuvers = MFPVR() >> 16; 2661 __asm volatile ("mfsdr1 %0" : "=r"(sdr1)); 2662 #ifndef PPC_OEA64 2663 addr = 0; 2664 for (i = 0; i < 16; i++) { 2665 soft_sr[i] = MFSRIN(addr); 2666 addr += (1 << ADDR_SR_SHFT); 2667 } 2668 #endif 2669 2670 #if defined(PPC_OEA) && !defined (PPC_OEA64) && !defined (PPC_OEA64_BRIDGE) 2671 /* read iBAT (601: uBAT) registers */ 2672 __asm volatile ("mfibatu %0,0" : "=r"(soft_ibat[0].batu)); 2673 __asm volatile ("mfibatl %0,0" : "=r"(soft_ibat[0].batl)); 2674 __asm volatile ("mfibatu %0,1" : "=r"(soft_ibat[1].batu)); 2675 __asm volatile ("mfibatl %0,1" : "=r"(soft_ibat[1].batl)); 2676 __asm volatile ("mfibatu %0,2" : "=r"(soft_ibat[2].batu)); 2677 __asm volatile ("mfibatl %0,2" : "=r"(soft_ibat[2].batl)); 2678 __asm volatile ("mfibatu %0,3" : "=r"(soft_ibat[3].batu)); 2679 __asm volatile ("mfibatl %0,3" : "=r"(soft_ibat[3].batl)); 2680 2681 2682 if (cpuvers != MPC601) { 2683 /* read dBAT registers */ 2684 __asm volatile ("mfdbatu %0,0" : "=r"(soft_dbat[0].batu)); 2685 __asm volatile ("mfdbatl %0,0" : "=r"(soft_dbat[0].batl)); 2686 __asm volatile ("mfdbatu %0,1" : "=r"(soft_dbat[1].batu)); 2687 __asm volatile ("mfdbatl %0,1" : "=r"(soft_dbat[1].batl)); 2688 __asm volatile ("mfdbatu %0,2" : "=r"(soft_dbat[2].batu)); 2689 __asm volatile ("mfdbatl %0,2" : "=r"(soft_dbat[2].batl)); 2690 __asm volatile ("mfdbatu %0,3" : "=r"(soft_dbat[3].batu)); 2691 __asm volatile ("mfdbatl %0,3" : "=r"(soft_dbat[3].batl)); 2692 } 2693 #endif 2694 2695 printf("SDR1:\t0x%lx\n", (long) sdr1); 2696 #ifndef PPC_OEA64 2697 printf("SR[]:\t"); 2698 for (i = 0; i < 4; i++) 2699 printf("0x%08lx, ", (long) soft_sr[i]); 2700 printf("\n\t"); 2701 for ( ; i < 8; i++) 2702 printf("0x%08lx, ", (long) soft_sr[i]); 2703 printf("\n\t"); 2704 for ( ; i < 12; i++) 2705 printf("0x%08lx, ", (long) soft_sr[i]); 2706 printf("\n\t"); 2707 for ( ; i < 16; i++) 2708 printf("0x%08lx, ", (long) soft_sr[i]); 2709 printf("\n"); 2710 #endif 2711 2712 #if defined(PPC_OEA) && !defined (PPC_OEA64) && !defined (PPC_OEA64_BRIDGE) 2713 printf("%cBAT[]:\t", cpuvers == MPC601 ? 'u' : 'i'); 2714 for (i = 0; i < 4; i++) { 2715 printf("0x%08lx 0x%08lx, ", 2716 soft_ibat[i].batu, soft_ibat[i].batl); 2717 if (i == 1) 2718 printf("\n\t"); 2719 } 2720 if (cpuvers != MPC601) { 2721 printf("\ndBAT[]:\t"); 2722 for (i = 0; i < 4; i++) { 2723 printf("0x%08lx 0x%08lx, ", 2724 soft_dbat[i].batu, soft_dbat[i].batl); 2725 if (i == 1) 2726 printf("\n\t"); 2727 } 2728 } 2729 printf("\n"); 2730 #endif /* PPC_OEA... */ 2731 } 2732 2733 void 2734 pmap_print_pte(pmap_t pm, vaddr_t va) 2735 { 2736 struct pvo_entry *pvo; 2737 volatile struct pte *pt; 2738 int pteidx; 2739 2740 pvo = pmap_pvo_find_va(pm, va, &pteidx); 2741 if (pvo != NULL) { 2742 pt = pmap_pvo_to_pte(pvo, pteidx); 2743 if (pt != NULL) { 2744 #if defined (PPC_OEA) || defined (PPC_OEA64) 2745 printf("VA %#lx -> %p -> %s %#lx, %#lx\n", 2746 va, pt, 2747 pt->pte_hi & PTE_HID ? "(sec)" : "(pri)", 2748 pt->pte_hi, pt->pte_lo); 2749 #else /* PPC_OEA64_BRIDGE */ 2750 printf("VA %#lx -> %p -> %s %#llx, %#llx\n", 2751 va, pt, 2752 pt->pte_hi & PTE_HID ? "(sec)" : "(pri)", 2753 pt->pte_hi, pt->pte_lo); 2754 #endif 2755 } else { 2756 printf("No valid PTE found\n"); 2757 } 2758 } else { 2759 printf("Address not in pmap\n"); 2760 } 2761 } 2762 2763 void 2764 pmap_pteg_dist(void) 2765 { 2766 struct pvo_entry *pvo; 2767 int ptegidx; 2768 int depth; 2769 int max_depth = 0; 2770 unsigned int depths[64]; 2771 2772 memset(depths, 0, sizeof(depths)); 2773 for (ptegidx = 0; ptegidx < pmap_pteg_cnt; ptegidx++) { 2774 depth = 0; 2775 TAILQ_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) { 2776 depth++; 2777 } 2778 if (depth > max_depth) 2779 max_depth = depth; 2780 if (depth > 63) 2781 depth = 63; 2782 depths[depth]++; 2783 } 2784 2785 for (depth = 0; depth < 64; depth++) { 2786 printf(" [%2d]: %8u", depth, depths[depth]); 2787 if ((depth & 3) == 3) 2788 printf("\n"); 2789 if (depth == max_depth) 2790 break; 2791 } 2792 if ((depth & 3) != 3) 2793 printf("\n"); 2794 printf("Max depth found was %d\n", max_depth); 2795 } 2796 #endif /* DEBUG */ 2797 2798 #if defined(PMAPCHECK) || defined(DEBUG) 2799 void 2800 pmap_pvo_verify(void) 2801 { 2802 int ptegidx; 2803 int s; 2804 2805 s = splvm(); 2806 for (ptegidx = 0; ptegidx < pmap_pteg_cnt; ptegidx++) { 2807 struct pvo_entry *pvo; 2808 TAILQ_FOREACH(pvo, &pmap_pvo_table[ptegidx], pvo_olink) { 2809 if ((uintptr_t) pvo >= SEGMENT_LENGTH) 2810 panic("pmap_pvo_verify: invalid pvo %p " 2811 "on list %#x", pvo, ptegidx); 2812 pmap_pvo_check(pvo); 2813 } 2814 } 2815 splx(s); 2816 } 2817 #endif /* PMAPCHECK */ 2818 2819 2820 void * 2821 pmap_pool_ualloc(struct pool *pp, int flags) 2822 { 2823 struct pvo_page *pvop; 2824 2825 pvop = SIMPLEQ_FIRST(&pmap_upvop_head); 2826 if (pvop != NULL) { 2827 pmap_upvop_free--; 2828 SIMPLEQ_REMOVE_HEAD(&pmap_upvop_head, pvop_link); 2829 return pvop; 2830 } 2831 if (uvm.page_init_done != true) { 2832 return (void *) uvm_pageboot_alloc(PAGE_SIZE); 2833 } 2834 return pmap_pool_malloc(pp, flags); 2835 } 2836 2837 void * 2838 pmap_pool_malloc(struct pool *pp, int flags) 2839 { 2840 struct pvo_page *pvop; 2841 struct vm_page *pg; 2842 2843 pvop = SIMPLEQ_FIRST(&pmap_mpvop_head); 2844 if (pvop != NULL) { 2845 pmap_mpvop_free--; 2846 SIMPLEQ_REMOVE_HEAD(&pmap_mpvop_head, pvop_link); 2847 return pvop; 2848 } 2849 again: 2850 pg = uvm_pagealloc_strat(NULL, 0, NULL, UVM_PGA_USERESERVE, 2851 UVM_PGA_STRAT_ONLY, VM_FREELIST_FIRST256); 2852 if (__predict_false(pg == NULL)) { 2853 if (flags & PR_WAITOK) { 2854 uvm_wait("plpg"); 2855 goto again; 2856 } else { 2857 return (0); 2858 } 2859 } 2860 return (void *) VM_PAGE_TO_PHYS(pg); 2861 } 2862 2863 void 2864 pmap_pool_ufree(struct pool *pp, void *va) 2865 { 2866 struct pvo_page *pvop; 2867 #if 0 2868 if (PHYS_TO_VM_PAGE((paddr_t) va) != NULL) { 2869 pmap_pool_mfree(va, size, tag); 2870 return; 2871 } 2872 #endif 2873 pvop = va; 2874 SIMPLEQ_INSERT_HEAD(&pmap_upvop_head, pvop, pvop_link); 2875 pmap_upvop_free++; 2876 if (pmap_upvop_free > pmap_upvop_maxfree) 2877 pmap_upvop_maxfree = pmap_upvop_free; 2878 } 2879 2880 void 2881 pmap_pool_mfree(struct pool *pp, void *va) 2882 { 2883 struct pvo_page *pvop; 2884 2885 pvop = va; 2886 SIMPLEQ_INSERT_HEAD(&pmap_mpvop_head, pvop, pvop_link); 2887 pmap_mpvop_free++; 2888 if (pmap_mpvop_free > pmap_mpvop_maxfree) 2889 pmap_mpvop_maxfree = pmap_mpvop_free; 2890 #if 0 2891 uvm_pagefree(PHYS_TO_VM_PAGE((paddr_t) va)); 2892 #endif 2893 } 2894 2895 /* 2896 * This routine in bootstraping to steal to-be-managed memory (which will 2897 * then be unmanaged). We use it to grab from the first 256MB for our 2898 * pmap needs and above 256MB for other stuff. 2899 */ 2900 vaddr_t 2901 pmap_steal_memory(vsize_t vsize, vaddr_t *vstartp, vaddr_t *vendp) 2902 { 2903 vsize_t size; 2904 vaddr_t va; 2905 paddr_t pa = 0; 2906 int npgs, bank; 2907 struct vm_physseg *ps; 2908 2909 if (uvm.page_init_done == true) 2910 panic("pmap_steal_memory: called _after_ bootstrap"); 2911 2912 *vstartp = VM_MIN_KERNEL_ADDRESS; 2913 *vendp = VM_MAX_KERNEL_ADDRESS; 2914 2915 size = round_page(vsize); 2916 npgs = atop(size); 2917 2918 /* 2919 * PA 0 will never be among those given to UVM so we can use it 2920 * to indicate we couldn't steal any memory. 2921 */ 2922 for (ps = vm_physmem, bank = 0; bank < vm_nphysseg; bank++, ps++) { 2923 if (ps->free_list == VM_FREELIST_FIRST256 && 2924 ps->avail_end - ps->avail_start >= npgs) { 2925 pa = ptoa(ps->avail_start); 2926 break; 2927 } 2928 } 2929 2930 if (pa == 0) 2931 panic("pmap_steal_memory: no approriate memory to steal!"); 2932 2933 ps->avail_start += npgs; 2934 ps->start += npgs; 2935 2936 /* 2937 * If we've used up all the pages in the segment, remove it and 2938 * compact the list. 2939 */ 2940 if (ps->avail_start == ps->end) { 2941 /* 2942 * If this was the last one, then a very bad thing has occurred 2943 */ 2944 if (--vm_nphysseg == 0) 2945 panic("pmap_steal_memory: out of memory!"); 2946 2947 printf("pmap_steal_memory: consumed bank %d\n", bank); 2948 for (; bank < vm_nphysseg; bank++, ps++) { 2949 ps[0] = ps[1]; 2950 } 2951 } 2952 2953 va = (vaddr_t) pa; 2954 memset((void *) va, 0, size); 2955 pmap_pages_stolen += npgs; 2956 #ifdef DEBUG 2957 if (pmapdebug && npgs > 1) { 2958 u_int cnt = 0; 2959 for (bank = 0, ps = vm_physmem; bank < vm_nphysseg; bank++, ps++) 2960 cnt += ps->avail_end - ps->avail_start; 2961 printf("pmap_steal_memory: stole %u (total %u) pages (%u left)\n", 2962 npgs, pmap_pages_stolen, cnt); 2963 } 2964 #endif 2965 2966 return va; 2967 } 2968 2969 /* 2970 * Find a chuck of memory with right size and alignment. 2971 */ 2972 void * 2973 pmap_boot_find_memory(psize_t size, psize_t alignment, int at_end) 2974 { 2975 struct mem_region *mp; 2976 paddr_t s, e; 2977 int i, j; 2978 2979 size = round_page(size); 2980 2981 DPRINTFN(BOOT, 2982 ("pmap_boot_find_memory: size=%lx, alignment=%lx, at_end=%d", 2983 size, alignment, at_end)); 2984 2985 if (alignment < PAGE_SIZE || (alignment & (alignment-1)) != 0) 2986 panic("pmap_boot_find_memory: invalid alignment %lx", 2987 alignment); 2988 2989 if (at_end) { 2990 if (alignment != PAGE_SIZE) 2991 panic("pmap_boot_find_memory: invalid ending " 2992 "alignment %lx", alignment); 2993 2994 for (mp = &avail[avail_cnt-1]; mp >= avail; mp--) { 2995 s = mp->start + mp->size - size; 2996 if (s >= mp->start && mp->size >= size) { 2997 DPRINTFN(BOOT,(": %lx\n", s)); 2998 DPRINTFN(BOOT, 2999 ("pmap_boot_find_memory: b-avail[%d] start " 3000 "0x%lx size 0x%lx\n", mp - avail, 3001 mp->start, mp->size)); 3002 mp->size -= size; 3003 DPRINTFN(BOOT, 3004 ("pmap_boot_find_memory: a-avail[%d] start " 3005 "0x%lx size 0x%lx\n", mp - avail, 3006 mp->start, mp->size)); 3007 return (void *) s; 3008 } 3009 } 3010 panic("pmap_boot_find_memory: no available memory"); 3011 } 3012 3013 for (mp = avail, i = 0; i < avail_cnt; i++, mp++) { 3014 s = (mp->start + alignment - 1) & ~(alignment-1); 3015 e = s + size; 3016 3017 /* 3018 * Is the calculated region entirely within the region? 3019 */ 3020 if (s < mp->start || e > mp->start + mp->size) 3021 continue; 3022 3023 DPRINTFN(BOOT,(": %lx\n", s)); 3024 if (s == mp->start) { 3025 /* 3026 * If the block starts at the beginning of region, 3027 * adjust the size & start. (the region may now be 3028 * zero in length) 3029 */ 3030 DPRINTFN(BOOT, 3031 ("pmap_boot_find_memory: b-avail[%d] start " 3032 "0x%lx size 0x%lx\n", i, mp->start, mp->size)); 3033 mp->start += size; 3034 mp->size -= size; 3035 DPRINTFN(BOOT, 3036 ("pmap_boot_find_memory: a-avail[%d] start " 3037 "0x%lx size 0x%lx\n", i, mp->start, mp->size)); 3038 } else if (e == mp->start + mp->size) { 3039 /* 3040 * If the block starts at the beginning of region, 3041 * adjust only the size. 3042 */ 3043 DPRINTFN(BOOT, 3044 ("pmap_boot_find_memory: b-avail[%d] start " 3045 "0x%lx size 0x%lx\n", i, mp->start, mp->size)); 3046 mp->size -= size; 3047 DPRINTFN(BOOT, 3048 ("pmap_boot_find_memory: a-avail[%d] start " 3049 "0x%lx size 0x%lx\n", i, mp->start, mp->size)); 3050 } else { 3051 /* 3052 * Block is in the middle of the region, so we 3053 * have to split it in two. 3054 */ 3055 for (j = avail_cnt; j > i + 1; j--) { 3056 avail[j] = avail[j-1]; 3057 } 3058 DPRINTFN(BOOT, 3059 ("pmap_boot_find_memory: b-avail[%d] start " 3060 "0x%lx size 0x%lx\n", i, mp->start, mp->size)); 3061 mp[1].start = e; 3062 mp[1].size = mp[0].start + mp[0].size - e; 3063 mp[0].size = s - mp[0].start; 3064 avail_cnt++; 3065 for (; i < avail_cnt; i++) { 3066 DPRINTFN(BOOT, 3067 ("pmap_boot_find_memory: a-avail[%d] " 3068 "start 0x%lx size 0x%lx\n", i, 3069 avail[i].start, avail[i].size)); 3070 } 3071 } 3072 return (void *) s; 3073 } 3074 panic("pmap_boot_find_memory: not enough memory for " 3075 "%lx/%lx allocation?", size, alignment); 3076 } 3077 3078 /* XXXSL: we dont have any BATs to do this, map in Segment 0 1:1 using page tables */ 3079 #if defined (PPC_OEA64_BRIDGE) 3080 int 3081 pmap_setup_segment0_map(int use_large_pages, ...) 3082 { 3083 vaddr_t va; 3084 3085 register_t pte_lo = 0x0; 3086 int ptegidx = 0, i = 0; 3087 struct pte pte; 3088 va_list ap; 3089 3090 /* Coherent + Supervisor RW, no user access */ 3091 pte_lo = PTE_M; 3092 3093 /* XXXSL 3094 * Map in 1st segment 1:1, we'll be careful not to spill kernel entries later, 3095 * these have to take priority. 3096 */ 3097 for (va = 0x0; va < SEGMENT_LENGTH; va += 0x1000) { 3098 ptegidx = va_to_pteg(pmap_kernel(), va); 3099 pmap_pte_create(&pte, pmap_kernel(), va, va | pte_lo); 3100 i = pmap_pte_insert(ptegidx, &pte); 3101 } 3102 3103 va_start(ap, use_large_pages); 3104 while (1) { 3105 paddr_t pa; 3106 size_t size; 3107 3108 va = va_arg(ap, vaddr_t); 3109 3110 if (va == 0) 3111 break; 3112 3113 pa = va_arg(ap, paddr_t); 3114 size = va_arg(ap, size_t); 3115 3116 for (; va < (va + size); va += 0x1000, pa += 0x1000) { 3117 #if 0 3118 printf("%s: Inserting: va: 0x%08lx, pa: 0x%08lx\n", __FUNCTION__, va, pa); 3119 #endif 3120 ptegidx = va_to_pteg(pmap_kernel(), va); 3121 pmap_pte_create(&pte, pmap_kernel(), va, pa | pte_lo); 3122 i = pmap_pte_insert(ptegidx, &pte); 3123 } 3124 } 3125 3126 TLBSYNC(); 3127 SYNC(); 3128 return (0); 3129 } 3130 #endif /* PPC_OEA64_BRIDGE */ 3131 3132 /* 3133 * This is not part of the defined PMAP interface and is specific to the 3134 * PowerPC architecture. This is called during initppc, before the system 3135 * is really initialized. 3136 */ 3137 void 3138 pmap_bootstrap(paddr_t kernelstart, paddr_t kernelend) 3139 { 3140 struct mem_region *mp, tmp; 3141 paddr_t s, e; 3142 psize_t size; 3143 int i, j; 3144 3145 /* 3146 * Get memory. 3147 */ 3148 mem_regions(&mem, &avail); 3149 #if defined(DEBUG) 3150 if (pmapdebug & PMAPDEBUG_BOOT) { 3151 printf("pmap_bootstrap: memory configuration:\n"); 3152 for (mp = mem; mp->size; mp++) { 3153 printf("pmap_bootstrap: mem start 0x%lx size 0x%lx\n", 3154 mp->start, mp->size); 3155 } 3156 for (mp = avail; mp->size; mp++) { 3157 printf("pmap_bootstrap: avail start 0x%lx size 0x%lx\n", 3158 mp->start, mp->size); 3159 } 3160 } 3161 #endif 3162 3163 /* 3164 * Find out how much physical memory we have and in how many chunks. 3165 */ 3166 for (mem_cnt = 0, mp = mem; mp->size; mp++) { 3167 if (mp->start >= pmap_memlimit) 3168 continue; 3169 if (mp->start + mp->size > pmap_memlimit) { 3170 size = pmap_memlimit - mp->start; 3171 physmem += btoc(size); 3172 } else { 3173 physmem += btoc(mp->size); 3174 } 3175 mem_cnt++; 3176 } 3177 3178 /* 3179 * Count the number of available entries. 3180 */ 3181 for (avail_cnt = 0, mp = avail; mp->size; mp++) 3182 avail_cnt++; 3183 3184 /* 3185 * Page align all regions. 3186 */ 3187 kernelstart = trunc_page(kernelstart); 3188 kernelend = round_page(kernelend); 3189 for (mp = avail, i = 0; i < avail_cnt; i++, mp++) { 3190 s = round_page(mp->start); 3191 mp->size -= (s - mp->start); 3192 mp->size = trunc_page(mp->size); 3193 mp->start = s; 3194 e = mp->start + mp->size; 3195 3196 DPRINTFN(BOOT, 3197 ("pmap_bootstrap: b-avail[%d] start 0x%lx size 0x%lx\n", 3198 i, mp->start, mp->size)); 3199 3200 /* 3201 * Don't allow the end to run beyond our artificial limit 3202 */ 3203 if (e > pmap_memlimit) 3204 e = pmap_memlimit; 3205 3206 /* 3207 * Is this region empty or strange? skip it. 3208 */ 3209 if (e <= s) { 3210 mp->start = 0; 3211 mp->size = 0; 3212 continue; 3213 } 3214 3215 /* 3216 * Does this overlap the beginning of kernel? 3217 * Does extend past the end of the kernel? 3218 */ 3219 else if (s < kernelstart && e > kernelstart) { 3220 if (e > kernelend) { 3221 avail[avail_cnt].start = kernelend; 3222 avail[avail_cnt].size = e - kernelend; 3223 avail_cnt++; 3224 } 3225 mp->size = kernelstart - s; 3226 } 3227 /* 3228 * Check whether this region overlaps the end of the kernel. 3229 */ 3230 else if (s < kernelend && e > kernelend) { 3231 mp->start = kernelend; 3232 mp->size = e - kernelend; 3233 } 3234 /* 3235 * Look whether this regions is completely inside the kernel. 3236 * Nuke it if it does. 3237 */ 3238 else if (s >= kernelstart && e <= kernelend) { 3239 mp->start = 0; 3240 mp->size = 0; 3241 } 3242 /* 3243 * If the user imposed a memory limit, enforce it. 3244 */ 3245 else if (s >= pmap_memlimit) { 3246 mp->start = -PAGE_SIZE; /* let's know why */ 3247 mp->size = 0; 3248 } 3249 else { 3250 mp->start = s; 3251 mp->size = e - s; 3252 } 3253 DPRINTFN(BOOT, 3254 ("pmap_bootstrap: a-avail[%d] start 0x%lx size 0x%lx\n", 3255 i, mp->start, mp->size)); 3256 } 3257 3258 /* 3259 * Move (and uncount) all the null return to the end. 3260 */ 3261 for (mp = avail, i = 0; i < avail_cnt; i++, mp++) { 3262 if (mp->size == 0) { 3263 tmp = avail[i]; 3264 avail[i] = avail[--avail_cnt]; 3265 avail[avail_cnt] = avail[i]; 3266 } 3267 } 3268 3269 /* 3270 * (Bubble)sort them into asecnding order. 3271 */ 3272 for (i = 0; i < avail_cnt; i++) { 3273 for (j = i + 1; j < avail_cnt; j++) { 3274 if (avail[i].start > avail[j].start) { 3275 tmp = avail[i]; 3276 avail[i] = avail[j]; 3277 avail[j] = tmp; 3278 } 3279 } 3280 } 3281 3282 /* 3283 * Make sure they don't overlap. 3284 */ 3285 for (mp = avail, i = 0; i < avail_cnt - 1; i++, mp++) { 3286 if (mp[0].start + mp[0].size > mp[1].start) { 3287 mp[0].size = mp[1].start - mp[0].start; 3288 } 3289 DPRINTFN(BOOT, 3290 ("pmap_bootstrap: avail[%d] start 0x%lx size 0x%lx\n", 3291 i, mp->start, mp->size)); 3292 } 3293 DPRINTFN(BOOT, 3294 ("pmap_bootstrap: avail[%d] start 0x%lx size 0x%lx\n", 3295 i, mp->start, mp->size)); 3296 3297 #ifdef PTEGCOUNT 3298 pmap_pteg_cnt = PTEGCOUNT; 3299 #else /* PTEGCOUNT */ 3300 3301 pmap_pteg_cnt = 0x1000; 3302 3303 while (pmap_pteg_cnt < physmem) 3304 pmap_pteg_cnt <<= 1; 3305 3306 pmap_pteg_cnt >>= 1; 3307 #endif /* PTEGCOUNT */ 3308 3309 #ifdef DEBUG 3310 DPRINTFN(BOOT, 3311 ("pmap_pteg_cnt: 0x%x\n", pmap_pteg_cnt)); 3312 #endif 3313 3314 /* 3315 * Find suitably aligned memory for PTEG hash table. 3316 */ 3317 size = pmap_pteg_cnt * sizeof(struct pteg); 3318 pmap_pteg_table = pmap_boot_find_memory(size, size, 0); 3319 3320 #ifdef DEBUG 3321 DPRINTFN(BOOT, 3322 ("PTEG cnt: 0x%x HTAB size: 0x%08x bytes, address: %p\n", pmap_pteg_cnt, (unsigned int)size, pmap_pteg_table)); 3323 #endif 3324 3325 3326 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 3327 if ( (uintptr_t) pmap_pteg_table + size > SEGMENT_LENGTH) 3328 panic("pmap_bootstrap: pmap_pteg_table end (%p + %lx) > 256MB", 3329 pmap_pteg_table, size); 3330 #endif 3331 3332 memset(__UNVOLATILE(pmap_pteg_table), 0, 3333 pmap_pteg_cnt * sizeof(struct pteg)); 3334 pmap_pteg_mask = pmap_pteg_cnt - 1; 3335 3336 /* 3337 * We cannot do pmap_steal_memory here since UVM hasn't been loaded 3338 * with pages. So we just steal them before giving them to UVM. 3339 */ 3340 size = sizeof(pmap_pvo_table[0]) * pmap_pteg_cnt; 3341 pmap_pvo_table = pmap_boot_find_memory(size, PAGE_SIZE, 0); 3342 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 3343 if ( (uintptr_t) pmap_pvo_table + size > SEGMENT_LENGTH) 3344 panic("pmap_bootstrap: pmap_pvo_table end (%p + %lx) > 256MB", 3345 pmap_pvo_table, size); 3346 #endif 3347 3348 for (i = 0; i < pmap_pteg_cnt; i++) 3349 TAILQ_INIT(&pmap_pvo_table[i]); 3350 3351 #ifndef MSGBUFADDR 3352 /* 3353 * Allocate msgbuf in high memory. 3354 */ 3355 msgbuf_paddr = 3356 (paddr_t) pmap_boot_find_memory(MSGBUFSIZE, PAGE_SIZE, 1); 3357 #endif 3358 3359 #ifdef __HAVE_PMAP_PHYSSEG 3360 { 3361 u_int npgs = 0; 3362 for (i = 0, mp = avail; i < avail_cnt; i++, mp++) 3363 npgs += btoc(mp->size); 3364 size = (sizeof(struct pvo_head) + 1) * npgs; 3365 pmap_physseg.pvoh = pmap_boot_find_memory(size, PAGE_SIZE, 0); 3366 pmap_physseg.attrs = (char *) &pmap_physseg.pvoh[npgs]; 3367 #if defined(DIAGNOSTIC) || defined(DEBUG) || defined(PMAPCHECK) 3368 if ((uintptr_t)pmap_physseg.pvoh + size > SEGMENT_LENGTH) 3369 panic("pmap_bootstrap: PVO list end (%p + %lx) > 256MB", 3370 pmap_physseg.pvoh, size); 3371 #endif 3372 } 3373 #endif 3374 3375 3376 for (mp = avail, i = 0; i < avail_cnt; mp++, i++) { 3377 paddr_t pfstart = atop(mp->start); 3378 paddr_t pfend = atop(mp->start + mp->size); 3379 if (mp->size == 0) 3380 continue; 3381 if (mp->start + mp->size <= SEGMENT_LENGTH) { 3382 uvm_page_physload(pfstart, pfend, pfstart, pfend, 3383 VM_FREELIST_FIRST256); 3384 } else if (mp->start >= SEGMENT_LENGTH) { 3385 uvm_page_physload(pfstart, pfend, pfstart, pfend, 3386 VM_FREELIST_DEFAULT); 3387 } else { 3388 pfend = atop(SEGMENT_LENGTH); 3389 uvm_page_physload(pfstart, pfend, pfstart, pfend, 3390 VM_FREELIST_FIRST256); 3391 pfstart = atop(SEGMENT_LENGTH); 3392 pfend = atop(mp->start + mp->size); 3393 uvm_page_physload(pfstart, pfend, pfstart, pfend, 3394 VM_FREELIST_DEFAULT); 3395 } 3396 } 3397 3398 /* 3399 * Make sure kernel vsid is allocated as well as VSID 0. 3400 */ 3401 pmap_vsid_bitmap[(KERNEL_VSIDBITS & (NPMAPS-1)) / VSID_NBPW] 3402 |= 1 << (KERNEL_VSIDBITS % VSID_NBPW); 3403 pmap_vsid_bitmap[0] |= 1; 3404 3405 /* 3406 * Initialize kernel pmap and hardware. 3407 */ 3408 3409 /* PPC_OEA64_BRIDGE does support these instructions */ 3410 #if defined (PPC_OEA) || defined (PPC_OEA64_BRIDGE) 3411 for (i = 0; i < 16; i++) { 3412 pmap_kernel()->pm_sr[i] = KERNELN_SEGMENT(i)|SR_PRKEY; 3413 __asm volatile ("mtsrin %0,%1" 3414 :: "r"(KERNELN_SEGMENT(i)|SR_PRKEY), "r"(i << ADDR_SR_SHFT)); 3415 } 3416 3417 pmap_kernel()->pm_sr[KERNEL_SR] = KERNEL_SEGMENT|SR_SUKEY|SR_PRKEY; 3418 __asm volatile ("mtsr %0,%1" 3419 :: "n"(KERNEL_SR), "r"(KERNEL_SEGMENT)); 3420 #ifdef KERNEL2_SR 3421 pmap_kernel()->pm_sr[KERNEL2_SR] = KERNEL2_SEGMENT|SR_SUKEY|SR_PRKEY; 3422 __asm volatile ("mtsr %0,%1" 3423 :: "n"(KERNEL2_SR), "r"(KERNEL2_SEGMENT)); 3424 #endif 3425 for (i = 0; i < 16; i++) { 3426 if (iosrtable[i] & SR601_T) { 3427 pmap_kernel()->pm_sr[i] = iosrtable[i]; 3428 __asm volatile ("mtsrin %0,%1" 3429 :: "r"(iosrtable[i]), "r"(i << ADDR_SR_SHFT)); 3430 } 3431 } 3432 #endif /* PPC_OEA || PPC_OEA64_BRIDGE */ 3433 #if defined (PPC_OEA) 3434 __asm volatile ("sync; mtsdr1 %0; isync" 3435 :: "r"((uintptr_t)pmap_pteg_table | (pmap_pteg_mask >> 10))); 3436 #elif defined (PPC_OEA64) || defined (PPC_OEA64_BRIDGE) 3437 __asm __volatile ("sync; mtsdr1 %0; isync" 3438 :: "r"((uintptr_t)pmap_pteg_table | (32 - cntlzw(pmap_pteg_mask >> 11)))); 3439 #endif 3440 tlbia(); 3441 3442 #ifdef ALTIVEC 3443 pmap_use_altivec = cpu_altivec; 3444 #endif 3445 3446 #ifdef DEBUG 3447 if (pmapdebug & PMAPDEBUG_BOOT) { 3448 u_int cnt; 3449 int bank; 3450 char pbuf[9]; 3451 for (cnt = 0, bank = 0; bank < vm_nphysseg; bank++) { 3452 cnt += vm_physmem[bank].avail_end - vm_physmem[bank].avail_start; 3453 printf("pmap_bootstrap: vm_physmem[%d]=%#lx-%#lx/%#lx\n", 3454 bank, 3455 ptoa(vm_physmem[bank].avail_start), 3456 ptoa(vm_physmem[bank].avail_end), 3457 ptoa(vm_physmem[bank].avail_end - vm_physmem[bank].avail_start)); 3458 } 3459 format_bytes(pbuf, sizeof(pbuf), ptoa((u_int64_t) cnt)); 3460 printf("pmap_bootstrap: UVM memory = %s (%u pages)\n", 3461 pbuf, cnt); 3462 } 3463 #endif 3464 3465 pool_init(&pmap_upvo_pool, sizeof(struct pvo_entry), 3466 sizeof(struct pvo_entry), 0, 0, "pmap_upvopl", 3467 &pmap_pool_uallocator, IPL_NONE); 3468 3469 pool_setlowat(&pmap_upvo_pool, 252); 3470 3471 pool_init(&pmap_pool, sizeof(struct pmap), 3472 sizeof(void *), 0, 0, "pmap_pl", &pmap_pool_uallocator, 3473 IPL_NONE); 3474 3475 #if defined(PMAP_NEED_MAPKERNEL) 3476 { 3477 extern int etext[], kernel_text[]; 3478 vaddr_t va, va_etext = (paddr_t) etext; 3479 paddr_t pa; 3480 register_t sr; 3481 3482 sr = KERNELN_SEGMENT(kernelstart >> ADDR_SR_SHFT) 3483 |SR_SUKEY|SR_PRKEY; 3484 3485 va = (vaddr_t) kernel_text; 3486 3487 for (pa = kernelstart; va < va_etext; 3488 pa += PAGE_SIZE, va += PAGE_SIZE) 3489 pmap_enter(pmap_kernel(), va, pa, 3490 VM_PROT_READ|VM_PROT_EXECUTE, 0); 3491 3492 for (; pa < kernelend; 3493 pa += PAGE_SIZE, va += PAGE_SIZE) 3494 pmap_enter(pmap_kernel(), va, pa, 3495 VM_PROT_READ|VM_PROT_WRITE, 0); 3496 3497 pmap_kernel()->pm_sr[kernelstart >> ADDR_SR_SHFT] = sr; 3498 __asm volatile ("mtsrin %0,%1" 3499 :: "r"(sr), "r"(kernelstart)); 3500 } 3501 #endif 3502 } 3503
Indexes created Sun Sep 28 12:09:53 GMT 2025