1 /* $NetBSD: pmap_68k.c,v 1.77 2026/08/06 13:15:54 thorpej Exp $ */ 2 3 /*- 4 * Copyright (c) 2025 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Pmap module for the Motorola 68851 / 68030 / 68040 / 68060 MMUs. 34 * (...and HP 68851-like MMU.) 35 * 36 * This implementation supports both 2-level and 3-level page table 37 * layouts. The 3-level is mandated by 68040 / 68060, and the 2-level 38 * is mandated by the HP MMU. The 68851 and 68030 can do either, and 39 * for now, the 2-level arrangement is retained for those MMUs, although 40 * eventually we will switch them to the 3-level configuration. 41 * 42 * To support both configurations, page tables are abstracted away from 43 * the page table pages that contain them. The interface pmap operations 44 * operate on "leaf" (page) tables, and only when one of those tables needs 45 * to be allocated or freed, do the differences between the two configurations 46 * need to be dealt with. All of the tables are kept in a red-black tree 47 * that's indexed by their "segment" number (where "segment" is defined as 48 * "the amount of space mapped by a single leaf table"). This avoids having 49 * to burn large amounts of kernel address space to access tables which are 50 * expected to be sparsely-populated. 51 * 52 * In order to reduce the number of tree lookups, the most recently used 53 * leaf table is cached, and the interface contract is such that bulk 54 * operations are allowed to access subsequent PTEs within a given table 55 * (segment) without having to perform another PTE lookup. 56 * 57 * This illustrates the initial table layout for a simple program 58 * (/usr/bin/yes) using the standard m68k address space layout (based 59 * on the historical 4.3BSD-on-hp300 layout, which was itself based on 60 * HP-UX in order to facilitate HP-UX binary compatibility back when 61 * that was considered to be important). This example uses a 4K page 62 * size. 63 * 64 * TEXTADDR is $0000.2000 (not always strictly true, but close enough) 65 * USRSTACK is $FFF0.0000 (grows down, first used page VA is $FFEF.F000) 66 * 67 * (TEXTADDR is $0000.2000 because the linker uses 8K page size for 68 * broader compatibility and keeps the 0-page unmapped so that NULL 69 * pointer dereferences blow up.) 70 * 71 * This is to say: the text / data / heap of this program are in the 72 * bottom 1MB of the address space, and the stack is in the second-from- 73 * the-top 1MB of the address space. 74 * 75 * In the 2-level layout, the level-1 table is 4KB in size, and has 1024 76 * entries. Those 1024 entries together represent the 4GB user address 77 * space, and each entry thus maps a 4MB "segment" by itself pointing to 78 * a level-2 table which themselves are 4KB in size and have 1024 entries 79 * (4MB / 1024 -> 4KB, which is the page size ... convenient!) So, when 80 * our very simple program is loaded, we have a table structure that looks 81 * like this: 82 * 83 * (4KB) 84 * +----------------------+ 85 * | Level-1 | 86 * |0 1023| 87 * +----------------------+ 88 * | | 89 * | | 90 * +---------+ +---------+ 91 * | | 92 * v v 93 * (4KB) (4KB) 94 * +----------------------+ +----------------------+ 95 * | Level-2 | | Level-2 | 96 * | 2 4 | | 767 | 97 * +----------------------+ +----------------------+ 98 * | | | 99 * | +-+ | 100 * v v v 101 * TEXT DATA/bss/heap stack 102 * 103 * As you can see, this requires 3 tables (1 level-1 and 2 level-2). Each 104 * table consumes a full 4KB page, so mapping this address space requires 105 * 3 total pages. 106 * 107 * In the 3-level layout, the level-1 and level-2 tables each contain 128 108 * entries, making them 512 bytes in size. When using 4KB pages, the level-3 109 * tables contain 64 entries, making them 256 bytes in size. 110 * 111 * So, assuming the same address space layout, the 3-level structure looks 112 * like this: 113 * 114 * (512B) 115 * +--------------+ 116 * | Level-1 | 117 * |0 127| 118 * +--------------+ 119 * | | 120 * +---+ +---+ 121 * v v 122 * (512B) (512B) 123 * +--------------+ +--------------+ 124 * | Level-2 | | Level-2 | 125 * |0 | | 123 | 126 * +--------------+ +--------------+ 127 * | | 128 * +---------+ +-----+ 129 * v v 130 * (256B) (256B) 131 * +------------+ +------------+ 132 * | Level-3 | | Level-3 | 133 * | 2 4 | | 63| 134 * +------------+ +------------+ 135 * | | | 136 * | +-+ | 137 * v v v 138 * TEXT DATA/bss/heap stack 139 * 140 * The table allocator has two pools of memory for tables in the 3-level 141 * configuration: one for "segment" tables (always 512 bytes) and one for 142 * "page" or "leaf" tables (256 bytes in size for 4K pages). Pages are 143 * allocated to the pools one at a time, and then the tables are allocated 144 * from the pages. Because of this, we only need two pages, 33% less (!), 145 * than the 2-level configuration to map the same address space. 146 * 147 * There is a cost, however: each access that misses the Address Translation 148 * Cache costs one extra memory cycle in the 3-level configuration. 149 * 150 * LOCKING IN THIS PMAP MODULE: 151 * 152 * MULTIPROCESSING IS NOT SUPPORTED IN THIS PMAP MODULE. Adding support 153 * for it would not be terribly difficult, but there is little value in 154 * doing that work until such time as a multiprocessor m68k machine exists 155 * that NetBSD runs on. 156 * 157 * As such, there is **no** locking performed of any data structures here. 158 * We do actually reap a benefit from this perceived laziness: we do not 159 * have to worry about lock ordering, which means we can take some shortcuts 160 * in some places (especially around pv_entry manipulation). 161 * 162 * THERE IS A CAVEAT, HOWEVER! Because there are no guard rails, we cannot, 163 * under any circumstances, yield the CPU during the critical section of a 164 * pmap operation, as doing so could cause the world to change beneath our 165 * feet, possibly rendering our work, for lack of a better term, "crashy". 166 * Specifically, this means: 167 * 168 * - Adaptive mutexes must not be acquired (e.g. when calling into 169 * other code, e.g. UVM to get a VA or a page). 170 * - Waiting for memory is not allowed. 171 * - The current thread may not be preempted. 172 * 173 * If any of those things are required, they must be performed outside of 174 * a critical section. If we discover that this is required while inside 175 * a critical section, then we must exit the critical section, perform the 176 * blocking work, re-enter the critical section and re-evaluate everything. 177 * Macros are provided to mark the boundaries of critical sections: 178 * 179 * - PMAP_CRIT_ENTER() 180 * - PMAP_CRIT_EXIT() 181 * 182 * XXX Alas, doesn't seem to be a way for us to hook into ASSERT_SLEEPABLE() 183 * XXX when inside a critical section. We should explore that for a future 184 * XXX enhancement. 185 */ 186 187 /* 188 * Current status: 189 * - Very stable multi-user on virt68k (qemu 68040; does not accurately 190 * model cache or ATC, but suitable for exercising large memory configs). 191 * 192 * - Very stable multi-user running memory pressure stress tests on 193 * small-memory configs in virt68k (qemu 68040 **with fixed MMU 194 * emulation**) and mac68k (10MB 68040, 68030). 195 * 196 * - Single-user mode on 68030 w/ no external cache (luna68k). 197 * 198 * - Single-user mode on 68040 (hp425t). 199 * 200 * - Multi-user mode on 68040 (NeXTstation, 20MB RAM) 201 * 202 * - Ports that have been adapted: hp300, luna68k, mac68k (default), 203 * mvme68k (default), news68k (see below), next68k (default), 204 * virt68k (default), x68k. 205 * 206 * XXX TODO XXX 207 * 208 * - Adapt amiga (hard), atari (hard), cesfic (easy). 209 * - Test on 68020. 210 * - Test on 68060. 211 * - Test on machines above listed as "not tested". 212 * - More rigorous testing in various emulators (Nono, UAE?). 213 * - Fix problems observed on news68k (external cache related?). 214 * - Finish HP MMU support and test on real HP MMU. 215 * - Convert '851 / '030 to 3-level. 216 * - Optimize ATC / cache manipulation. 217 * - Add some more instrumentation. 218 * - Eventually disable instrumentation by default. 219 * - ... 220 * - PROFIT! 221 */ 222 223 #include "opt_ddb.h" 224 #include "opt_kgdb.h" 225 #include "opt_m68k_arch.h" 226 227 #include <sys/cdefs.h> 228 __KERNEL_RCSID(0, "$NetBSD: pmap_68k.c,v 1.77 2026/08/06 13:15:54 thorpej Exp $"); 229 230 #include <sys/param.h> 231 #include <sys/systm.h> 232 #include <sys/evcnt.h> 233 #include <sys/proc.h> 234 #include <sys/pool.h> 235 #include <sys/cpu.h> 236 #include <sys/atomic.h> 237 #include <sys/kmem.h> 238 239 #include <machine/pcb.h> 240 241 #include <uvm/uvm.h> 242 #include <uvm/uvm_physseg.h> 243 244 #include <m68k/cacheops.h> 245 #include <m68k/mmu.h> 246 247 #if !defined(M68K_MMU_MOTOROLA) && !defined(M68K_MMU_HP) 248 #error Hit the road, Jack... 249 #endif 250 251 /****************************** SERIALIZATION ********************************/ 252 253 /* 254 * XXX Would like to make these do something lightweight-ish in 255 * XXX DIAGNOSTIC kernels (and also make ASSERT_SLEEPABLE() trip 256 * XXX if we're in a critical section). 257 */ 258 259 #define PMAP_CRIT_ENTER(code) code 260 #define PMAP_CRIT_EXIT(code) code 261 #define PMAP_CRIT_ASSERT() __nothing 262 263 static inline void 264 pmap_busy(pmap_t pmap) 265 { 266 pmap->pm_busy++; 267 KDASSERT(pmap->pm_busy != 0); 268 } 269 270 static inline void 271 pmap_unbusy(pmap_t pmap) 272 { 273 KDASSERT(pmap->pm_busy != 0); 274 pmap->pm_busy--; 275 } 276 277 /**************************** MMU CONFIGURATION ******************************/ 278 279 #include "opt_m68k_arch.h" 280 281 #if defined(M68K_MMU_68030) 282 #include <m68k/mmu_30.h> /* for cpu_kcore_hdr_t */ 283 #endif 284 285 /* 286 * We consider 3 different MMU classes: 287 * - 68851 (includes 68030) 288 * - 68040 (includes 68060) 289 * - HP MMU for 68020 (68851-like, 2-level 4K only, external VAC) 290 */ 291 292 #define MMU_CLASS_68851 0 293 #define MMU_CLASS_68040 1 294 #define MMU_CLASS_HP 3 295 296 static int pmap_mmuclass __read_mostly; 297 298 #if defined(M68K_MMU_68851) || defined(M68K_MMU_68030) 299 #define MMU_CONFIG_68851_CLASS 1 300 #else 301 #define MMU_CONFIG_68851_CLASS 0 302 #endif 303 304 #if defined(M68K_MMU_68040) || defined(M68K_MMU_68060) 305 #define MMU_CONFIG_68040_CLASS 1 306 #else 307 #define MMU_CONFIG_68040_CLASS 0 308 #endif 309 310 #if defined(M68K_MMU_HP) 311 #define MMU_CONFIG_HP_CLASS 1 312 #else 313 #define MMU_CONFIG_HP_CLASS 0 314 #endif 315 316 #define MMU_CONFIG_NCLASSES (MMU_CONFIG_68851_CLASS + \ 317 MMU_CONFIG_68040_CLASS + \ 318 MMU_CONFIG_HP_CLASS) 319 320 #if MMU_CONFIG_NCLASSES == 1 321 322 #if MMU_CONFIG_68851_CLASS 323 #define MMU_IS_68851_CLASS 1 324 #elif MMU_CONFIG_68040_CLASS 325 #define MMU_IS_68040_CLASS 1 326 #elif MMU_CONFIG_HP_CLASS 327 #define MMU_IS_HP_CLASS 1 328 #else 329 #error Single MMU config predicate error. 330 #endif 331 332 #else /* MMU_CONFIG_NCLASSES != 1 */ 333 334 #if MMU_CONFIG_68851_CLASS 335 #define MMU_IS_68851_CLASS (pmap_mmuclass == MMU_CLASS_68851) 336 #endif 337 338 #if MMU_CONFIG_68040_CLASS 339 #define MMU_IS_68040_CLASS (pmap_mmuclass == MMU_CLASS_68040) 340 #endif 341 342 #if MMU_CONFIG_HP_CLASS 343 #define MMU_IS_HP_CLASS (pmap_mmuclass == MMU_CLASS_HP) 344 #endif 345 346 #endif /* MMU_CONFIG_NCLASSES == 1 */ 347 348 #ifndef MMU_IS_68851_CLASS 349 #define MMU_IS_68851_CLASS 0 350 #endif 351 352 #ifndef MMU_IS_68040_CLASS 353 #define MMU_IS_68040_CLASS 0 354 #endif 355 356 #ifndef MMU_IS_HP_CLASS 357 #define MMU_IS_HP_CLASS 0 358 #endif 359 360 /* 361 * 68040 must always use 3-level. Eventually, we will switch the '851 362 * type over to 3-level as well, for for now, it gets 2-level. The 363 * HP MMU is stuck there for all eternity. 364 */ 365 #define MMU_USE_3L (MMU_IS_68040_CLASS) 366 #define MMU_USE_2L (!MMU_USE_3L) 367 368 /***************************** INSTRUMENTATION *******************************/ 369 370 #define PMAP_EVENT_COUNTERS 371 372 static struct evcnt pmap_nkptpages_initial_ev = 373 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap nkptpages", "initial"); 374 static struct evcnt pmap_nkptpages_current_ev = 375 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap nkptpages", "current"); 376 EVCNT_ATTACH_STATIC(pmap_nkptpages_initial_ev); 377 EVCNT_ATTACH_STATIC(pmap_nkptpages_current_ev); 378 379 static struct evcnt pmap_nkstpages_initial_ev = 380 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap nkstpages", "initial"); 381 static struct evcnt pmap_nkstpages_current_ev = 382 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap nkstpages", "current"); 383 EVCNT_ATTACH_STATIC(pmap_nkstpages_initial_ev); 384 EVCNT_ATTACH_STATIC(pmap_nkstpages_current_ev); 385 386 static struct evcnt pmap_nptpages_current_ev = 387 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap nptpages", "current"); 388 static struct evcnt pmap_nptpages_hiwat_ev = 389 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap nptpages", "hiwat"); 390 EVCNT_ATTACH_STATIC(pmap_nptpages_current_ev); 391 EVCNT_ATTACH_STATIC(pmap_nptpages_hiwat_ev); 392 393 static struct evcnt pmap_maxkva_ev = 394 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap", "maxkva"); 395 EVCNT_ATTACH_STATIC(pmap_maxkva_ev); 396 397 static struct evcnt pmap_kvalimit_ev = 398 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap", "kvalimit"); 399 EVCNT_ATTACH_STATIC(pmap_kvalimit_ev); 400 401 #ifdef PMAP_EVENT_COUNTERS 402 static struct evcnt pmap_pv_alloc_wait_ev = 403 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap pv_alloc", "wait"); 404 EVCNT_ATTACH_STATIC(pmap_pv_alloc_wait_ev); 405 406 static struct evcnt pmap_pv_alloc_nowait_ev = 407 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap pv_alloc", "nowait"); 408 EVCNT_ATTACH_STATIC(pmap_pv_alloc_nowait_ev); 409 410 static struct evcnt pmap_pv_enter_called_ev = 411 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap pv_enter", "called"); 412 EVCNT_ATTACH_STATIC(pmap_pv_enter_called_ev); 413 414 static struct evcnt pmap_pv_enter_usr_ci_ev = 415 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap pv_enter", "usr_ci"); 416 EVCNT_ATTACH_STATIC(pmap_pv_enter_usr_ci_ev); 417 418 #if MMU_CONFIG_HP_CLASS 419 static struct evcnt pmap_pv_enter_vac_ci_ev = 420 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap pv_enter", "vac_ci"); 421 EVCNT_ATTACH_STATIC(pmap_pv_enter_vac_ci_ev); 422 #endif 423 424 static struct evcnt pmap_pv_enter_ci_multi_ev = 425 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap pv_enter", "ci_multi"); 426 EVCNT_ATTACH_STATIC(pmap_pv_enter_ci_multi_ev); 427 428 static struct evcnt pmap_pv_remove_called_ev = 429 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap pv_remove", "called"); 430 EVCNT_ATTACH_STATIC(pmap_pv_remove_called_ev); 431 432 static struct evcnt pmap_pv_remove_ci_ev = 433 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap pv_remove", "ci"); 434 EVCNT_ATTACH_STATIC(pmap_pv_remove_ci_ev); 435 436 static struct evcnt pmap_pt_cache_hit_ev = 437 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap pt_cache", "hit"); 438 EVCNT_ATTACH_STATIC(pmap_pt_cache_hit_ev); 439 440 static struct evcnt pmap_pt_cache_miss_ev = 441 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap pt_cache", "miss"); 442 EVCNT_ATTACH_STATIC(pmap_pt_cache_miss_ev); 443 444 static struct evcnt pmap_enter_nowait_ev = 445 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap enter", "nowait"); 446 EVCNT_ATTACH_STATIC(pmap_enter_nowait_ev); 447 448 static struct evcnt pmap_enter_yeswait_ev = 449 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap enter", "yeswait"); 450 EVCNT_ATTACH_STATIC(pmap_enter_yeswait_ev); 451 452 static struct evcnt pmap_enter_pte_alloc_fail_ev = 453 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap enter", "pte alloc failed"); 454 EVCNT_ATTACH_STATIC(pmap_enter_pte_alloc_fail_ev); 455 456 static struct evcnt pmap_enter_pv_alloc_fail_ev = 457 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap enter", "pv alloc failed"); 458 EVCNT_ATTACH_STATIC(pmap_enter_pv_alloc_fail_ev); 459 460 static struct evcnt pmap_enter_valid_ev = 461 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap enter", "valid"); 462 EVCNT_ATTACH_STATIC(pmap_enter_valid_ev); 463 464 static struct evcnt pmap_enter_wire_change_ev = 465 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap enter", "wire change"); 466 EVCNT_ATTACH_STATIC(pmap_enter_wire_change_ev); 467 468 static struct evcnt pmap_enter_prot_change_ev = 469 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap enter", "prot change"); 470 EVCNT_ATTACH_STATIC(pmap_enter_prot_change_ev); 471 472 static struct evcnt pmap_enter_pa_change_ev = 473 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap enter", "pa change"); 474 EVCNT_ATTACH_STATIC(pmap_enter_pa_change_ev); 475 476 static struct evcnt pmap_enter_pv_recycle_ev = 477 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap enter", "pv recycle"); 478 EVCNT_ATTACH_STATIC(pmap_enter_pv_recycle_ev); 479 480 static struct evcnt pmap_prm_got_pg_ev = 481 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap prm", "got pg"); 482 static struct evcnt pmap_prm_lookup_pg_hit_ev = 483 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap prm", "lookup pg hit"); 484 static struct evcnt pmap_prm_lookup_pg_miss_ev = 485 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap prm", "lookup pg miss"); 486 EVCNT_ATTACH_STATIC(pmap_prm_got_pg_ev); 487 EVCNT_ATTACH_STATIC(pmap_prm_lookup_pg_hit_ev); 488 EVCNT_ATTACH_STATIC(pmap_prm_lookup_pg_miss_ev); 489 490 #define pmap_evcnt(e) pmap_ ## e ## _ev.ev_count++ 491 #else 492 #define pmap_evcnt(e) __nothing 493 #endif 494 495 static void (*pmap_load_urp_func)(paddr_t) __read_mostly; 496 497 static void 498 pmap_mmuclass_init(void) 499 { 500 switch (mmutype) { 501 #if MMU_CONFIG_68040_CLASS 502 case MMU_68040: 503 case MMU_68060: 504 pmap_mmuclass = MMU_CLASS_68040; 505 /* 506 * XXX This is messy because 68060 frequently gets 507 * XXX initialize to MMU_68040. Should be cleaned 508 * XXX up once the Hibler pmap is obsoleted. 509 */ 510 #if defined(M68040) 511 if (cputype == CPU_68040) { 512 pmap_load_urp_func = mmu_load_urp40; 513 } 514 #endif 515 #if defined(M68060) 516 if (cputype == CPU_68060) { 517 pmap_load_urp_func = mmu_load_urp60; 518 } 519 #endif 520 break; 521 #endif 522 #if MMU_CONFIG_68851_CLASS 523 case MMU_68851: 524 case MMU_68030: 525 pmap_mmuclass = MMU_CLASS_68851; 526 protorp[0] = MMU51_CRP_BITS; 527 pmap_load_urp_func = mmu_load_urp51; 528 break; 529 #endif 530 #if MMU_CONFIG_HP_CLASS 531 case MMU_HP: 532 pmap_mmuclass = MMU_CLASS_HP; 533 pmap_load_urp_func = mmu_load_urp20hp; 534 break; 535 #endif 536 default: 537 panic("%s: mmutype=%d not configured?", __func__, mmutype); 538 } 539 540 if (pmap_load_urp_func == NULL) { 541 panic("%s: No mmu_load_*() for cputype=%d mmutype=%d", 542 __func__, cputype, mmutype); 543 } 544 } 545 546 /* 547 * pmap_load_urp: 548 * 549 * Load the user root table into the MMU. 550 */ 551 static inline void 552 pmap_load_urp(paddr_t urp) 553 { 554 (*pmap_load_urp_func)(urp); 555 } 556 557 #if MMU_CONFIG_HP_CLASS 558 static vaddr_t pmap_aliasmask __read_mostly; 559 #endif 560 561 /* 562 * pmap_init_vac: 563 * 564 * Set up virtually-addressed cache information. Only relevant 565 * for the HP MMU. 566 */ 567 void 568 pmap_init_vac(size_t vacsize) 569 { 570 #if MMU_CONFIG_HP_CLASS 571 KASSERT(pmap_aliasmask == 0); 572 KASSERT(powerof2(vacsize)); 573 pmap_aliasmask = vacsize - 1; 574 #endif 575 } 576 577 /***************************** PHYS <-> VM PAGE ******************************/ 578 579 static bool pmap_initialized_p; 580 581 static inline struct vm_page * 582 pmap_pa_to_pg(paddr_t pa) 583 { 584 return pmap_initialized_p ? PHYS_TO_VM_PAGE(pa) : NULL; 585 } 586 587 static pt_entry_t pmap_changebit(struct vm_page *, pt_entry_t, pt_entry_t); 588 589 /*************************** RESOURCE MANAGEMENT *****************************/ 590 591 static struct pmap kernel_pmap_store; 592 struct pmap * const kernel_pmap_ptr = &kernel_pmap_store; 593 594 /* 595 * Physical address of kernel level 1 table. This name is compatible 596 * with the Hibler pmap's name. 597 */ 598 paddr_t Sysseg_pa; 599 600 /* 601 * Avoid a memory load when doing comparisons against pmap_kernel() 602 * within this compilation unit. 603 */ 604 #undef pmap_kernel 605 #define pmap_kernel() (&kernel_pmap_store) 606 607 static inline bool 608 active_pmap(pmap_t pmap) 609 { 610 return pmap == pmap_kernel() || 611 pmap == curproc->p_vmspace->vm_map.pmap; 612 } 613 614 static inline bool 615 active_user_pmap(pmap_t pmap) 616 { 617 return curproc != NULL && 618 pmap != pmap_kernel() && 619 pmap == curproc->p_vmspace->vm_map.pmap; 620 } 621 622 /* 623 * Number of tables per page table page: 624 * 0 - number of leaf page tables per page 625 * 1 - number of segment tables per page 626 */ 627 static unsigned int pmap_ptpage_table_counts[2]; 628 629 __CTASSERT(LA40_L1_COUNT == LA40_L2_COUNT); 630 631 static void 632 pmap_ptpage_init(void) 633 { 634 if (MMU_USE_3L) { 635 pmap_ptpage_table_counts[0] = PAGE_SIZE / TBL40_L3_SIZE; 636 pmap_ptpage_table_counts[1] = PAGE_SIZE / TBL40_L2_SIZE; 637 } else { 638 pmap_ptpage_table_counts[0] = 1; 639 pmap_ptpage_table_counts[1] = 1; 640 } 641 } 642 643 static struct vm_page * 644 pmap_page_alloc(bool nowait) 645 { 646 struct vm_page *pg; 647 const int flags = nowait ? UVM_PGA_USERESERVE : 0; 648 649 while ((pg = uvm_pagealloc(NULL, 0, NULL, flags)) == NULL) { 650 if (nowait) { 651 return NULL; 652 } 653 uvm_wait("pmappg"); 654 } 655 pg->flags &= ~PG_BUSY; /* never busy */ 656 657 return pg; 658 } 659 660 static struct pmap_ptpage * 661 pmap_ptpage_alloc(bool segtab, bool nowait) 662 { 663 const unsigned int tabcnt = pmap_ptpage_table_counts[segtab]; 664 const size_t size = sizeof(struct pmap_ptpage) + 665 (sizeof(struct pmap_table) * tabcnt); 666 const size_t tabsize = PAGE_SIZE / tabcnt; 667 struct pmap_ptpage *ptp; 668 struct pmap_table *pt; 669 struct vm_page *pg; 670 const int uvm_f_nowait = nowait ? UVM_KMF_NOWAIT : 0; 671 vaddr_t ptpva; 672 673 ptp = kmem_zalloc(size, nowait ? KM_NOSLEEP : KM_SLEEP); 674 if (__predict_false(ptp == NULL)) { 675 return NULL; 676 } 677 678 /* Allocate a VA for the PT page. */ 679 ptpva = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 680 UVM_KMF_VAONLY | uvm_f_nowait); 681 if (__predict_false(ptpva == 0)) { 682 kmem_free(ptp, size); 683 return NULL; 684 } 685 686 /* Get a page. */ 687 pg = pmap_page_alloc(nowait); 688 if (__predict_false(pg == NULL)) { 689 uvm_km_free(kernel_map, ptpva, PAGE_SIZE, UVM_KMF_VAONLY); 690 kmem_free(ptp, size); 691 return NULL; 692 } 693 694 /* Map the page cache-inhibited and zero it out. */ 695 pmap_kenter_pa(ptpva, VM_PAGE_TO_PHYS(pg), 696 UVM_PROT_READ | UVM_PROT_WRITE, PMAP_NOCACHE); 697 zeropage((void *)ptpva); 698 699 /* 700 * All resources for the PT page have been allocated. 701 * Now initialize it and the individual table descriptors. 702 */ 703 LIST_INIT(&ptp->ptp_freelist); 704 ptp->ptp_pg = pg; 705 ptp->ptp_vpagenum = m68k_btop(ptpva); 706 ptp->ptp_freecnt = tabcnt; 707 ptp->ptp_segtab = segtab; 708 709 for (unsigned int i = 0; i < tabcnt; ptpva += tabsize, i++) { 710 pt = &ptp->ptp_tables[i]; 711 pt->pt_ptpage = ptp; 712 pt->pt_entries = (pt_entry_t *)ptpva; 713 LIST_INSERT_HEAD(&ptp->ptp_freelist, pt, pt_freelist); 714 } 715 716 #ifdef PMAP_EVENT_COUNTERS 717 pmap_nptpages_current_ev.ev_count32++; 718 if (pmap_nptpages_current_ev.ev_count32 > 719 pmap_nptpages_hiwat_ev.ev_count32) { 720 pmap_nptpages_hiwat_ev.ev_count32 = 721 pmap_nptpages_current_ev.ev_count32; 722 } 723 #endif 724 725 return ptp; 726 } 727 728 static void 729 pmap_ptpage_free(struct pmap_ptpage *ptp) 730 { 731 const unsigned int tabcnt = pmap_ptpage_table_counts[ptp->ptp_segtab]; 732 const size_t size = sizeof(struct pmap_ptpage) + 733 (sizeof(struct pmap_table) * tabcnt); 734 735 uvm_km_free(kernel_map, m68k_ptob(ptp->ptp_vpagenum), PAGE_SIZE, 736 UVM_KMF_WIRED); 737 kmem_free(ptp, size); 738 739 #ifdef PMAP_EVENT_COUNTERS 740 pmap_nptpages_current_ev.ev_count32--; 741 #endif 742 } 743 744 static struct pool pmap_pool; 745 static struct pool pmap_pv_pool; 746 747 #define PMAP_PV_LOWAT 16 748 749 static void 750 pmap_alloc_init(void) 751 { 752 pool_init(&pmap_pv_pool, sizeof(struct pv_entry), 753 PVH_ATTR_MASK + 1, /* align */ 754 0, /* ioff */ 755 0, /* flags */ 756 "pmappv", /* wchan */ 757 &pool_allocator_meta, /* palloc */ 758 IPL_VM); /* ipl */ 759 760 /* 761 * Set a low water mark on the pv_entry pool, so that we are 762 * more likely to have these around even in extreme memory 763 * starvation. 764 */ 765 pool_setlowat(&pmap_pv_pool, PMAP_PV_LOWAT); 766 767 pool_init(&pmap_pool, sizeof(struct pmap), 768 0, /* align */ 769 0, /* ioff */ 770 0, /* flags */ 771 "pmappl", /* wchan */ 772 &pool_allocator_kmem, /* palloc */ 773 IPL_NONE); /* ipl */ 774 } 775 776 static inline pmap_t 777 pmap_alloc(void) 778 { 779 pmap_t pmap = pool_get(&pmap_pool, PR_WAITOK); 780 memset(pmap, 0, sizeof(*pmap)); 781 return pmap; 782 } 783 784 static inline void 785 pmap_free(pmap_t pmap) 786 { 787 pool_put(&pmap_pool, pmap); 788 } 789 790 static struct pv_entry * 791 pmap_pv_alloc(bool nowait) 792 { 793 struct pv_entry *pv; 794 795 #ifdef PMAP_EVENT_COUNTERS 796 if (nowait) { 797 pmap_evcnt(pv_alloc_nowait); 798 } else { 799 pmap_evcnt(pv_alloc_wait); 800 } 801 #endif 802 803 pv = pool_get(&pmap_pv_pool, nowait ? PR_NOWAIT : 0); 804 if (__predict_true(pv != NULL)) { 805 KASSERT((((uintptr_t)pv) & PVH_ATTR_MASK) == 0); 806 } 807 return pv; 808 } 809 810 static void 811 pmap_pv_free(struct pv_entry *pv) 812 { 813 pool_put(&pmap_pv_pool, pv); 814 } 815 816 /* 817 * Whenever we need to free resources back to the system, we want to 818 * do it in a batch with any locks released. So, we have this around 819 * to collect the garbage, as needed. 820 */ 821 struct pmap_completion { 822 struct pmap_ptpage_list pc_ptpages; 823 struct pmap_pv_list pc_pvlist; 824 }; 825 826 static inline void 827 pmap_completion_init(struct pmap_completion *pc) 828 { 829 TAILQ_INIT(&pc->pc_ptpages); 830 LIST_INIT(&pc->pc_pvlist); 831 } 832 833 static void 834 pmap_completion_fini(struct pmap_completion *pc) 835 { 836 struct pmap_ptpage *ptp; 837 struct pv_entry *pv; 838 839 while ((ptp = TAILQ_FIRST(&pc->pc_ptpages)) != NULL) { 840 TAILQ_REMOVE(&pc->pc_ptpages, ptp, ptp_list); 841 /* 842 * Can't assert ptp_freecnt here; it won't match up 843 * in the pmap_remove_all() case. 844 * 845 * KASSERT(ptp->ptp_freecnt == 846 * pmap_ptpage_table_counts[ptp->ptp_segtab]); 847 */ 848 pmap_ptpage_free(ptp); 849 } 850 851 while ((pv = LIST_FIRST(&pc->pc_pvlist)) != NULL) { 852 LIST_REMOVE(pv, pv_pmlist); 853 pmap_pv_free(pv); 854 } 855 } 856 857 /************************ PTE MANIPULATION HELPERS ***************************/ 858 859 /* Assert assumptions made in <machine/pmap.h>. */ 860 __CTASSERT(DT51_PAGE == PTE40_RESIDENT); 861 __CTASSERT(PTE51_WP == PTE40_W); 862 __CTASSERT(PTE51_U == PTE40_U); 863 __CTASSERT(PTE51_M == PTE40_M); 864 __CTASSERT(PTE51_CI == PTE40_CM_NC_SER); 865 866 static pt_entry_t pmap_pte_proto[UVM_PROT_ALL + 1]; 867 static pt_entry_t pmap_pte_proto_ci[UVM_PROT_ALL + 1]; 868 static pt_entry_t pmap_pte_proto_um[UVM_PROT_ALL + 1]; 869 static pt_entry_t pmap_ste_proto; 870 871 static inline paddr_t 872 pte_pa(pt_entry_t pte) 873 { 874 return pte & PTE40_PGA; 875 } 876 877 /* 878 * These predicate inlines compile down into BFEXTU, so are quite fast. 879 */ 880 881 static inline bool 882 pte_valid_p(pt_entry_t pte) 883 { 884 return !!(pte & PTE_VALID); 885 } 886 887 static inline bool 888 pte_wired_p(pt_entry_t pte) 889 { 890 return !!(pte & PTE_WIRED); 891 } 892 893 static inline bool 894 pte_managed_p(pt_entry_t pte) 895 { 896 return !!(pte & PTE_PVLIST); 897 } 898 899 static inline bool 900 pte_ci_p(pt_entry_t pte) 901 { 902 /* 903 * Happily, PTE51_CI is bit 6, which is set for both of the 904 * cache-inhibited modes on 68040, so we can just check for 905 * that. 906 */ 907 return !!(pte & PTE51_CI); 908 } 909 910 #define PTE_PROT_CHANGE_BITS (PTE_WP | PTE_CMASK) 911 912 static inline pt_entry_t 913 pte_change_prot(pt_entry_t opte, vm_prot_t prot) 914 { 915 pt_entry_t *pte_proto = pte_ci_p(opte) ? pmap_pte_proto_ci 916 : pmap_pte_proto; 917 918 return (opte & ~PTE_PROT_CHANGE_BITS) | pte_proto[prot]; 919 } 920 921 static inline pt_entry_t 922 pte_load(volatile pt_entry_t *ptep) 923 { 924 return *ptep; 925 } 926 927 static inline void 928 pte_store(volatile pt_entry_t *ptep, pt_entry_t npte) 929 { 930 *ptep = npte; 931 } 932 933 /* 934 * Don't inline the CAS.L instruction; some systems have non-working 935 * READ-MODIFY-WRITE cycle logic. This will ensure that we'll use 936 * restartable atomic sequence, if required. 937 * 938 * AND.L and OR.L don't use the RMC signal, so they aren't subject 939 * to the same constraints. 940 */ 941 static inline bool 942 pte_update(volatile pt_entry_t *ptep, pt_entry_t opte, pt_entry_t npte) 943 { 944 /* 945 * Use compare-and-swap to update the PTE. This ensures there's 946 * no possibility of losing any hardware-maintained bits when 947 * updating the PTE. 948 * 949 * XXX Should turn this into a single instruction when possible 950 * XXX to deduce at compile time. 951 */ 952 return atomic_cas_uint(ptep, opte, npte) == opte; 953 } 954 955 #if MMU_CONFIG_HP_CLASS 956 /* 957 * These are only used for HP MMU VAC shenanigans. There is no need 958 * for these to be truly atomic, and systems with an HP MMU can't do 959 * truly atomic operations anyway. 960 */ 961 static inline void 962 pte_set(volatile pt_entry_t *ptep, pt_entry_t bits) 963 { 964 *ptep |= bits; 965 } 966 967 static inline void 968 pte_mask(volatile pt_entry_t *ptep, pt_entry_t mask) 969 { 970 *ptep &= mask; 971 } 972 #endif /* MMU_CONFIG_HP_CLASS */ 973 974 static inline pt_entry_t 975 pte_set_ci(pt_entry_t pte) 976 { 977 return (pte & ~PTE_CMASK) | (MMU_IS_68040_CLASS ? PTE40_CM_NC_SER 978 : PTE51_CI); 979 } 980 981 static inline pt_entry_t 982 pte_clr_ci(pt_entry_t pte) 983 { 984 pte &= ~PTE_CMASK; 985 if (MMU_IS_68040_CLASS) { 986 pte |= (pte & PTE_WP) ? PTE40_CM_WT 987 : PTE40_CM_CB; 988 } 989 return pte; 990 } 991 992 static void 993 pmap_pte_proto_init(void) 994 { 995 pt_entry_t c_bits, ro_c_bits, rw_c_bits, ci_bits, prot_bits, um_bits; 996 int prot; 997 998 if (MMU_IS_68040_CLASS) { 999 ro_c_bits = PTE40_CM_WT; /* this is what the Hibler pmap did */ 1000 rw_c_bits = PTE40_CM_CB; 1001 ci_bits = PTE40_CM_NC_SER; 1002 } else { 1003 ro_c_bits = rw_c_bits = 0; 1004 ci_bits = PTE51_CI; 1005 } 1006 1007 for (prot = 1; prot <= UVM_PROT_ALL; prot++) { 1008 prot_bits = um_bits = 0; 1009 if (prot & UVM_PROT_WRITE) { 1010 um_bits = PTE_U | PTE_M; 1011 } else if (prot & (UVM_PROT_READ|UVM_PROT_EXEC)) { 1012 prot_bits = PTE_WP; 1013 um_bits = PTE_U; 1014 } 1015 c_bits = (prot & UVM_PROT_WRITE) ? rw_c_bits : ro_c_bits; 1016 pmap_pte_proto[prot] = PTE_VALID | prot_bits | c_bits; 1017 pmap_pte_proto_ci[prot] = PTE_VALID | prot_bits | ci_bits; 1018 pmap_pte_proto_um[prot] = um_bits; 1019 } 1020 1021 /* 1022 * from hp300/DOC/HPMMU.notes: 1023 * 1024 * Segment table entries: 1025 * 1026 * bits 31-12: Physical page frame number of PT page 1027 * bits 11-4: Reserved at zero (can software use them?) 1028 * bit 3: Reserved at one 1029 * bits 1-0: Valid bits (hardware uses bit 1) 1030 * 1031 * This is all roughly compatible with 68851 and 68040: 1032 * 1033 * bit 3: DTE51_U / UTE40_U (used) 1034 * bits 1-0: DT51_SHORT / UTE40_RESIDENT 1035 * 1036 * The Hibler pmap set "SG_U" in the 68040 case, but not in 1037 * any others (??), which seems at odds with HPMMU.notes, but 1038 * whatever. It does not seem to cause any harm to set the 1039 * "used" bit in all cases, so that's what we'll do. If it 1040 * does prove to be problematic, we can make adjustments. 1041 */ 1042 pmap_ste_proto = DTE51_U | DT51_SHORT; 1043 } 1044 1045 static inline pt_entry_t 1046 pmap_make_pte(paddr_t pa, vm_prot_t prot, u_int flags) 1047 { 1048 pt_entry_t *pte_proto = (flags & PMAP_NOCACHE) ? pmap_pte_proto_ci 1049 : pmap_pte_proto; 1050 1051 prot &= UVM_PROT_ALL; 1052 KASSERT(prot != 0); 1053 1054 /* 1055 * N.B. We seed the U and M bits in the PTE based on the access 1056 * that was used to produce the page fault because the MMU will 1057 * skip the write-back of a PTE upon access if the copy in the 1058 * ATC already has the bit set. 1059 */ 1060 pt_entry_t npte = pa | pte_proto[prot] | 1061 pmap_pte_proto_um[flags & UVM_PROT_ALL]; 1062 1063 if (flags & PMAP_WIRED) { 1064 npte |= PTE_WIRED; 1065 } 1066 1067 return npte; 1068 } 1069 1070 /************************** PAGE TABLE MANAGEMENT ****************************/ 1071 1072 /* 1073 * Kernel page table management works differently from user page table 1074 * management. An initial set of kernel PTs are allocated during early 1075 * bootstrap (enough to map the virtual addresses set up at that time, 1076 * plus a little extra to give the kernel some breathing room while 1077 * UVM gets initialized -- see pmap_bootstrap1()). If more PTs are 1078 * needed in order to expand the kernel address space, pmap_growkernel() 1079 * is called to allocate some more. We always allocate kernel PTs in 1080 * chunks of one page, allocating more inner segment tables as needed 1081 * to link them into the MMU tree (3-level), or just poking them in 1082 * directly to the level-1 table (2-level). 1083 * 1084 * The kernel PTs are mapped into a single linear array to make that 1085 * makes it possible to simply index by virtual page number to find 1086 * the PTE that maps that virtual address. 1087 */ 1088 #define PTPAGEVASZ ((PAGE_SIZE / sizeof(pt_entry_t)) * PAGE_SIZE) 1089 #define PTPAGEVAOFS (PTPAGEVASZ - 1) 1090 1091 #define pmap_round_ptpage(va) (((va) + PTPAGEVAOFS) & ~PTPAGEVAOFS) 1092 #define pmap_trunc_ptpage(va) ((va) & ~PTPAGEVAOFS) 1093 1094 /* 1095 * kernel_virtual_start marks the first kernel virtual address that 1096 * is handed off to UVM to manage. kernel_virtual_end marks the end 1097 * of the kernel address space that is currently mappable with the 1098 * number of pages allocated to kernel PTs. 1099 * 1100 * kernel_virtual_start is fixed once pmap_bootstrap1() completes. 1101 * kernel_virtual_end can be extended by calling pmap_growkernel(). 1102 * 1103 * kernel_virtual_max represents the absolute maximum. It starts at 1104 * KERNEL_MAX_ADDRESS, but may get clamped by fixed mappings that 1105 * start beyond the end of kernel virtual address space. 1106 * 1107 * kernel_virtual_max is exported to the rest of the kernel via 1108 * pmap_virtual_space() and VM_MAX_KERNEL_ADDRESS. 1109 */ 1110 #define KERNEL_MAX_ADDRESS ((vaddr_t)0 - PAGE_SIZE) 1111 static vaddr_t kernel_virtual_start, kernel_virtual_end; 1112 vaddr_t kernel_virtual_max = KERNEL_MAX_ADDRESS; 1113 1114 /* 1115 * kernel_stnext_pa and kernel_stnext_endpa together implement a 1116 * simple allocator for inner segment tables used in the 3-level 1117 * configuration. When the initial level-1 table is allocated 1118 * the remained of that page is set in kernel_stnext_pa, and 1119 * kernel_stnext_endpa is set to the next page boundary. When 1120 * a segment table is needed, kernel_stnext_pa is the address 1121 * of the next free table and is advanced by the L2 table size 1122 * (512 bytes). If that allocation attempt finds that kernel_stnext_pa 1123 * is equal to kernel_stnext_endpa, a new page is allocated and 1124 * kernel_stnext_pa and kernel_stnext_endpa updated to reflect 1125 * the newly-allocated page before the table is taken from it. 1126 */ 1127 static paddr_t kernel_stnext_pa, kernel_stnext_endpa; 1128 1129 /* 1130 * Null segment table that every pmap gets as its initial level 1 1131 * map. This is a single page allocated in pmap_bootstrap1(), and 1132 * we zero it out in pmap_init(). 1133 */ 1134 static paddr_t null_segtab_pa __read_mostly; 1135 1136 static inline void 1137 pmap_set_lev1map(pmap_t pmap, struct pmap_table *pt, paddr_t pa) 1138 { 1139 pmap->pm_lev1map = pt; 1140 pmap->pm_lev1pa = pa; 1141 if (active_user_pmap(pmap)) { 1142 #if MMU_CONFIG_HP_CLASS 1143 /* 1144 * N.B. re-loading the user segment table pointer also 1145 * invalidates the user side of the VAC, so no additional 1146 * work is necessary. 1147 */ 1148 #endif 1149 pmap_load_urp(pmap->pm_lev1pa); 1150 TBIAU(); /* XXX optimize? */ 1151 ICIA(); /* XXX optimize? */ 1152 } 1153 } 1154 1155 /* 1156 * Table accessors. 1157 */ 1158 static inline unsigned int 1159 pmap_pagenum(vaddr_t va) 1160 { 1161 return ((va) >> PAGE_SHIFT); 1162 } 1163 1164 static inline unsigned int 1165 pmap_segnum(vaddr_t va) 1166 { 1167 return MMU_USE_3L ? ((va) >> SEGSHIFT3L) : ((va) >> SEGSHIFT2L); 1168 } 1169 1170 static inline unsigned int 1171 pmap_st1_index(vaddr_t va) 1172 { 1173 return MMU_USE_3L ? LA40_RI(va) : LA2L_RI(va); 1174 } 1175 1176 static inline unsigned int 1177 pmap_st_index(vaddr_t va) 1178 { 1179 return MMU_USE_3L ? LA40_PI(va) : LA2L_RI(va); 1180 } 1181 1182 static inline unsigned int 1183 pmap_pt_index(vaddr_t va) 1184 { 1185 return MMU_USE_3L ? LA40_PGI(va) : LA2L_PGI(va); 1186 } 1187 1188 static inline vaddr_t 1189 pmap_trunc_seg(vaddr_t va) 1190 { 1191 return MMU_USE_3L ? pmap_trunc_seg_3L(va) : pmap_trunc_seg_2L(va); 1192 } 1193 1194 static inline vaddr_t 1195 pmap_trunc_seg1(vaddr_t va) 1196 { 1197 KASSERT(MMU_USE_3L); 1198 return pmap_trunc_seg1_3L(va); 1199 } 1200 1201 static inline vaddr_t 1202 pmap_round_seg(vaddr_t va) 1203 { 1204 return MMU_USE_3L ? pmap_round_seg_3L(va) : pmap_round_seg_2L(va); 1205 } 1206 1207 static inline vaddr_t 1208 pmap_next_seg(vaddr_t va) 1209 { 1210 return pmap_round_seg(va + PAGE_SIZE); 1211 } 1212 1213 static paddr_t 1214 pmap_table_pa(const struct pmap_table * const pt) 1215 { 1216 const struct pmap_ptpage * const ptp = pt->pt_ptpage; 1217 const vaddr_t ptpva = m68k_ptob(ptp->ptp_vpagenum); 1218 const vaddr_t ptva = (vaddr_t)pt->pt_entries; 1219 1220 return VM_PAGE_TO_PHYS(ptp->ptp_pg) + (ptva - ptpva); 1221 } 1222 1223 static inline unsigned int 1224 pmap_table_make_key(unsigned int segnum, bool segtab) 1225 { 1226 KASSERT((segnum & 0x80000000) == 0); 1227 return (segnum << 1) | (unsigned int)segtab; 1228 } 1229 1230 static int 1231 pmap_table_rb_compare_key(void *v __unused, const void *n, const void *k) 1232 { 1233 const struct pmap_table * const pt1 = n; 1234 const unsigned int k1 = pt1->pt_key; 1235 const unsigned int k2 = *(const unsigned int *)k; 1236 1237 return (int)(k1 - k2); 1238 } 1239 1240 static int 1241 pmap_table_rb_compare_nodes(void *v, const void *n1, const void *n2) 1242 { 1243 const struct pmap_table * const pt2 = n2; 1244 1245 return pmap_table_rb_compare_key(v, n1, &pt2->pt_key); 1246 } 1247 1248 static const rb_tree_ops_t pmap_table_rb_ops = { 1249 .rbto_compare_nodes = pmap_table_rb_compare_nodes, 1250 .rbto_compare_key = pmap_table_rb_compare_key, 1251 .rbto_node_offset = offsetof(struct pmap_table, pt_node), 1252 }; 1253 1254 static struct pmap_table * 1255 pmap_table_alloc(pmap_t pmap, bool segtab, bool nowait, 1256 struct pmap_completion *pc) 1257 { 1258 struct pmap_ptpage_list *pmlist = &pmap->pm_ptpages[segtab]; 1259 struct pmap_ptpage *ptp, *newptp = NULL; 1260 struct pmap_table *pt; 1261 1262 KASSERT(pc != NULL); 1263 1264 try_again: 1265 if ((ptp = TAILQ_FIRST(pmlist)) == NULL || ptp->ptp_freecnt == 0) { 1266 /* 1267 * No PT pages with free tables (empty PT pages are moved 1268 * to the tail of the list). Allocate a new PT page and 1269 * try again. If someone else successfully allocates one 1270 * while we're sleeping, then we'll use it and free what 1271 * we allocated back to the system. 1272 */ 1273 KASSERT(ptp == NULL || LIST_FIRST(&ptp->ptp_freelist) == NULL); 1274 if (newptp == NULL) { 1275 newptp = pmap_ptpage_alloc(segtab, nowait); 1276 if (newptp == NULL) { 1277 /* 1278 * If we didn't wait, then no one would 1279 * have allocted one behind our back. 1280 */ 1281 KASSERT(nowait); 1282 return NULL; 1283 } 1284 goto try_again; 1285 } 1286 ptp = newptp; 1287 TAILQ_INSERT_HEAD(pmlist, newptp, ptp_list); 1288 } 1289 if (__predict_false(newptp != NULL && ptp != newptp)) { 1290 /* Not using newly-allocated PT page; free it back. */ 1291 TAILQ_INSERT_TAIL(&pc->pc_ptpages, newptp, ptp_list); 1292 } 1293 pt = LIST_FIRST(&ptp->ptp_freelist); 1294 KASSERT(pt != NULL); 1295 LIST_REMOVE(pt, pt_freelist); 1296 ptp->ptp_freecnt--; 1297 if (ptp->ptp_freecnt == 0 && 1298 TAILQ_NEXT(ptp, ptp_list) != NULL) { 1299 TAILQ_REMOVE(pmlist, ptp, ptp_list); 1300 TAILQ_INSERT_TAIL(pmlist, ptp, ptp_list); 1301 } 1302 KASSERT(pt->pt_st == NULL); 1303 pt->pt_holdcnt = 1; 1304 1305 return pt; 1306 } 1307 1308 static void 1309 pmap_table_free(pmap_t pmap, struct pmap_table *pt, 1310 struct pmap_completion *pc) 1311 { 1312 struct pmap_ptpage *ptp = pt->pt_ptpage; 1313 struct pmap_ptpage_list *pmlist = &pmap->pm_ptpages[ptp->ptp_segtab]; 1314 1315 KASSERT(pt->pt_st == NULL); 1316 1317 LIST_INSERT_HEAD(&ptp->ptp_freelist, pt, pt_freelist); 1318 KASSERT(ptp->ptp_freecnt < pmap_ptpage_table_counts[ptp->ptp_segtab]); 1319 ptp->ptp_freecnt++; 1320 1321 /* 1322 * If the PT page no longer has any active tables, then 1323 * remove it from the pmap and queue it up to be given 1324 * back to the system. 1325 */ 1326 if (ptp->ptp_freecnt == pmap_ptpage_table_counts[ptp->ptp_segtab]) { 1327 TAILQ_REMOVE(pmlist, ptp, ptp_list); 1328 TAILQ_INSERT_TAIL(&pc->pc_ptpages, ptp, ptp_list); 1329 } 1330 /* 1331 * If the PT page now has exactly one free table, then 1332 * put it at the head of its list so that it is allocated 1333 * from first the next time a table is needed. 1334 */ 1335 else if (ptp->ptp_freecnt == 1) { 1336 TAILQ_REMOVE(pmlist, ptp, ptp_list); 1337 TAILQ_INSERT_HEAD(pmlist, ptp, ptp_list); 1338 } 1339 /* 1340 * Push this PT page down the list if it has more free tables 1341 * than the ones that come after. The goal is to keep PT pages 1342 * with the fewest free tables at the head of the list so that 1343 * they're allocated from first. This is an effort to keep 1344 * fragmentation at bay so as to increase the likelihood that 1345 * we can free PT pages back to the system. 1346 */ 1347 else { 1348 struct pmap_ptpage *next_ptp; 1349 for (next_ptp = TAILQ_NEXT(ptp, ptp_list); 1350 next_ptp != NULL; 1351 next_ptp = TAILQ_NEXT(next_ptp, ptp_list)) { 1352 if (next_ptp->ptp_freecnt < ptp->ptp_freecnt) { 1353 break; 1354 } 1355 } 1356 if (next_ptp != NULL && 1357 next_ptp != TAILQ_NEXT(ptp, ptp_list) && 1358 next_ptp->ptp_freecnt != 0) { 1359 TAILQ_REMOVE(pmlist, ptp, ptp_list); 1360 TAILQ_INSERT_AFTER(pmlist, next_ptp, ptp, ptp_list); 1361 } 1362 } 1363 } 1364 1365 /* 1366 * pmap_table_retain: 1367 * 1368 * Take a retain count on the specified table. Retain counts 1369 * are used to ensure the table remains stable while working 1370 * on it, and each mapping placed into the table also gets 1371 * a retain count. 1372 */ 1373 static inline void 1374 pmap_table_retain(struct pmap_table *pt) 1375 { 1376 if (__predict_true(pt != NULL)) { 1377 pt->pt_holdcnt++; 1378 KASSERT(pt->pt_holdcnt != 0); 1379 } 1380 } 1381 1382 /* 1383 * pmap_table_release: 1384 * 1385 * Release a previously-taken retain count on the specified 1386 * table. If the retain count drops to zero, the table is 1387 * unlinked from the lookup tree and the MMU tree and freed. 1388 */ 1389 static __noinline void 1390 pmap_table_release_slow(pmap_t pmap, struct pmap_table *pt, 1391 struct pmap_completion *pc) 1392 { 1393 KASSERT(pt != NULL); 1394 KASSERT(pt->pt_holdcnt != 0); 1395 pt->pt_holdcnt--; 1396 if (__predict_false(pt->pt_holdcnt != 0)) { 1397 return; 1398 } 1399 1400 /* 1401 * If the caller doesn't expect the count to go to zero, 1402 * they won't have bothered with a completion context. 1403 * Going to zero is unexpected in this case, so blow up 1404 * if it happens. 1405 */ 1406 KASSERT(pc != NULL); 1407 if (__predict_true(pt == pmap->pm_pt_cache)) { 1408 pmap->pm_pt_cache = NULL; 1409 } 1410 if (__predict_true(pt->pt_st != NULL)) { 1411 /* 1412 * This table needs to be unlinked from the lookup 1413 * tree and the MMU tree. 1414 */ 1415 pte_store(&pt->pt_st->pt_entries[pt->pt_stidx], 0); 1416 rb_tree_remove_node(&pmap->pm_tables, pt); 1417 pmap_table_release_slow(pmap, pt->pt_st, pc); 1418 pt->pt_st = NULL; 1419 } else if (pt == pmap->pm_lev1map) { 1420 pmap_set_lev1map(pmap, NULL, null_segtab_pa); 1421 } 1422 pmap_table_free(pmap, pt, pc); 1423 } 1424 1425 static inline void 1426 pmap_table_release(pmap_t pmap, struct pmap_table *pt, 1427 struct pmap_completion *pc) 1428 { 1429 if (__predict_true(pt != NULL)) { 1430 if (__predict_true(pt->pt_holdcnt > 1)) { 1431 pt->pt_holdcnt--; 1432 return; 1433 } 1434 pmap_table_release_slow(pmap, pt, pc); 1435 } 1436 } 1437 1438 /* 1439 * pmap_table_lookup: 1440 * 1441 * Lookup the table corresponding to the specified segment. 1442 */ 1443 static struct pmap_table * 1444 pmap_table_lookup(pmap_t pmap, unsigned int segnum, bool segtab) 1445 { 1446 const unsigned int key = pmap_table_make_key(segnum, segtab); 1447 struct pmap_table *pt; 1448 1449 if ((pt = pmap->pm_pt_cache) == NULL || pt->pt_key != key) { 1450 pmap_evcnt(pt_cache_miss); 1451 pt = rb_tree_find_node(&pmap->pm_tables, &key); 1452 if (__predict_true(!segtab)) { 1453 pmap->pm_pt_cache = pt; 1454 } 1455 } else { 1456 pmap_evcnt(pt_cache_hit); 1457 } 1458 if (pt != NULL) { 1459 pmap_table_retain(pt); 1460 } 1461 return pt; 1462 } 1463 1464 /* 1465 * pmap_table_insert: 1466 * 1467 * Allocate and insert a table into the tree at the specified 1468 * location. 1469 */ 1470 static struct pmap_table * 1471 pmap_table_insert(pmap_t pmap, struct pmap_table *t1, unsigned int stidx, 1472 unsigned int segnum, bool segtab, bool nowait, struct pmap_completion *pc) 1473 { 1474 struct pmap_table *t2, *ret_t; 1475 1476 t2 = pmap_table_lookup(pmap, segnum, segtab); 1477 if (t2 != NULL) { 1478 /* 1479 * Table at this level already exists, and looking 1480 * it up gave us a retain count, so we no longer need 1481 * the retain count on the upper level table (it is 1482 * retained-by-proxy by the table we just found). 1483 * We pass NULL for the completion context because 1484 * we don't expect the upper level table's retain count 1485 * to drop to zero, and we want things to blow up 1486 * loudly if it does! 1487 */ 1488 pmap_table_release(pmap, t1, NULL); 1489 return t2; 1490 } 1491 1492 /* Allocate the new table. */ 1493 PMAP_CRIT_EXIT(); 1494 t2 = pmap_table_alloc(pmap, segtab, nowait, pc); 1495 PMAP_CRIT_ENTER(); 1496 if (__predict_false(t2 == NULL)) { 1497 pmap_table_release(pmap, t1, pc); 1498 return NULL; 1499 } 1500 t2->pt_key = pmap_table_make_key(segnum, segtab); 1501 1502 /* 1503 * Now that we have the new table, we need to insert it into the 1504 * table lookup tree. If we blocked while allocating, it's possible 1505 * someone raced with us and inserted one behind our back, so we need 1506 * to check for that. 1507 */ 1508 ret_t = rb_tree_insert_node(&pmap->pm_tables, t2); 1509 if (__predict_false(ret_t != t2)) { 1510 /* 1511 * Someone beat us to the punch. If this happens, 1512 * then we also need to drop the retain count on 1513 * t1 because the table we just found already has 1514 * a retain count on it. 1515 */ 1516 pmap_table_retain(ret_t); 1517 pmap_table_release(pmap, t2, pc); 1518 pmap_table_release(pmap, t1, NULL); 1519 return ret_t; 1520 } 1521 1522 /* 1523 * Table has been successfully inserted into the lookup 1524 * tree, now link it into the MMU's tree. The new table 1525 * takes ownership of the retain count that was taken on 1526 * the upper level table while working. 1527 */ 1528 t2->pt_st = t1; 1529 t2->pt_stidx = (unsigned short)stidx; 1530 pte_store(&t1->pt_entries[stidx], pmap_ste_proto | pmap_table_pa(t2)); 1531 1532 return t2; 1533 } 1534 1535 /*************************** PTE LOOKUP FUNCTIONS ****************************/ 1536 1537 static pt_entry_t *kernel_ptes; 1538 1539 /* 1540 * pmap_kernel_pte: 1541 * 1542 * Get the PTE that maps the specified kernel virtual address. 1543 * 1544 * Take note: the caller *may assume* they they can linearly 1545 * access adjacent PTEs up until the address indicated by 1546 * virtual_end! That means, "pte++" is totally fine until you 1547 * get to the current limit of the kernel virtual address space! 1548 */ 1549 static pt_entry_t * 1550 pmap_kernel_pte(vaddr_t va) 1551 { 1552 /* 1553 * The kernel PTEs are mapped as a linear array, whose entries 1554 * represent the entire possible 4GB supervisor address space. 1555 * 1556 * Kernel PT pages are pre-allocated and mapped into this linear 1557 * space (via pmap_growkernel(), as needed) and never freed back. 1558 * So, as long as the VA is below virtual_end, we know that a PTE 1559 * exists to back it. 1560 * 1561 * We don't assert that the VA < virtual_end, however; there may 1562 * be special cases where we need to get a PTE that has been 1563 * statically-allocated out beyond where virtual space is allowed 1564 * to grow. We'll find out soon enough if a PT page doesn't back 1565 * it, because a fault will occur when the PTE is accessed. 1566 */ 1567 KASSERT(va >= VM_MIN_KERNEL_ADDRESS); 1568 return &kernel_ptes[m68k_btop(va - VM_MIN_KERNEL_ADDRESS)]; 1569 } 1570 1571 /* 1572 * pmap_pte_lookup: 1573 * 1574 * Lookup the PTE for the given address, returning a retained 1575 * reference to the table containing the PTE. 1576 * 1577 * Take note: the caller *may assume* they they can linearly 1578 * access adjacent PTEs that map addresses within the same 1579 * segment! That means, "pte++" is totally fine until you 1580 * get to the next segment boundary! 1581 */ 1582 static pt_entry_t * 1583 pmap_pte_lookup(pmap_t pmap, vaddr_t va, struct pmap_table **out_pt) 1584 { 1585 if (pmap == pmap_kernel()) { 1586 *out_pt = NULL; 1587 return pmap_kernel_pte(va); 1588 } 1589 1590 const unsigned int segnum = pmap_segnum(va); 1591 1592 struct pmap_table *pt = pmap_table_lookup(pmap, segnum, false); 1593 if (__predict_true(pt != NULL)) { 1594 *out_pt = pt; /* already retained */ 1595 return &pt->pt_entries[pmap_pt_index(va)]; 1596 } 1597 1598 *out_pt = NULL; 1599 return NULL; 1600 } 1601 1602 /* 1603 * pmap_pte_alloc: 1604 * 1605 * Like pmap_pte_lookup(), but allocates tables as necessary. 1606 * 1607 * We enter in a critical section, but may drop that along 1608 * the way and re-validate our own assumptions. Callers 1609 * (pmap_enter(), basically), should be aware of this. 1610 */ 1611 static pt_entry_t * 1612 pmap_pte_alloc(pmap_t pmap, vaddr_t va, struct pmap_table **out_pt, 1613 bool nowait, struct pmap_completion *pc) 1614 { 1615 struct pmap_table *st, *pt; 1616 pt_entry_t *ptep; 1617 1618 PMAP_CRIT_ASSERT(); 1619 1620 ptep = pmap_pte_lookup(pmap, va, out_pt); 1621 if (__predict_true(ptep != NULL)) { 1622 return ptep; 1623 } 1624 1625 /* 1626 * First get a reference on the top-level segment table and 1627 * retain it so that it's stable while we work. 1628 */ 1629 if (__predict_true((st = pmap->pm_lev1map) != NULL)) { 1630 pmap_table_retain(st); 1631 } else { 1632 /* 1633 * Oh look! Baby pmap's first mapping! Allocate 1634 * a segment table. 1635 */ 1636 PMAP_CRIT_EXIT(); 1637 st = pmap_table_alloc(pmap, true/*segtab*/, nowait, pc); 1638 PMAP_CRIT_ENTER(); 1639 if (__predict_false(st == NULL)) { 1640 return NULL; 1641 } 1642 1643 /* Re-validate that we still need the segment table. */ 1644 if (__predict_false(pmap->pm_lev1map != NULL)) { 1645 /* Raced and lost. */ 1646 pmap_table_release(pmap, st, pc); 1647 st = pmap->pm_lev1map; 1648 pmap_table_retain(st); 1649 } else { 1650 /* New table is returned to us retained. */ 1651 pmap_set_lev1map(pmap, st, pmap_table_pa(st)); 1652 } 1653 } 1654 1655 /* 1656 * Now we know that st points to a valid segment table with a 1657 * retain count that lets us safely reference it. 1658 */ 1659 1660 if (MMU_USE_3L) { 1661 /* Get the inner segment table for this virtual address. */ 1662 struct pmap_table * const st1 = st; 1663 st = pmap_table_insert(pmap, st1, pmap_st1_index(va), 1664 pmap_st1_index(va), true/*segtab*/, nowait, pc); 1665 if (__predict_false(st == NULL)) { 1666 pmap_table_release(pmap, st1, pc); 1667 return NULL; 1668 } 1669 } 1670 1671 /* We can now allocate and insert the leaf page table. */ 1672 pt = pmap_table_insert(pmap, st, pmap_st_index(va), pmap_segnum(va), 1673 false/*segtab*/, nowait, pc); 1674 if (__predict_false(pt == NULL)) { 1675 pmap_table_release(pmap, st, pc); 1676 return NULL; 1677 } 1678 1679 *out_pt = pt; 1680 return &pt->pt_entries[pmap_pt_index(va)]; 1681 } 1682 1683 /************************** P->V ENTRY MANAGEMENT ****************************/ 1684 1685 static inline pt_entry_t * 1686 pmap_pv_pte(struct pv_entry * const pv) 1687 { 1688 const vaddr_t va = PV_VA(pv); 1689 1690 if (__predict_true(pv->pv_pmap != pmap_kernel())) { 1691 KASSERT(pv->pv_pt != NULL); 1692 return &pv->pv_pt->pt_entries[pmap_pt_index(va)]; 1693 } 1694 return pmap_kernel_pte(va); 1695 } 1696 1697 #define MATCHING_PMAP(p1, p2) \ 1698 ((p1) == (p2) || \ 1699 (p1) == pmap_kernel() || (p2) == pmap_kernel()) 1700 1701 #define CONFLICTING_ALIAS(va1, va2) \ 1702 (((va1) & pmap_aliasmask) != ((va2) & pmap_aliasmask)) 1703 1704 /* 1705 * pmap_pv_enter: 1706 * 1707 * Add a physical->virtual entry to the pv table. Caller must provide 1708 * the storage for the new PV entry. 1709 * 1710 * We are responsible for storing the new PTE into the destination 1711 * table. We are also guaranteed that no mapping exists there, but 1712 * the MMU has a negative cache in the ATC (see 68030UM Figure 9-8. 1713 * Address Translation General Flowchart, ATC hit, B==1 case, as well 1714 * as 68040UM Figure 3-21. ATC Entry and Tag Fields, R bit and the 1715 * associated descriptive text), so we still have to handle ATC entry 1716 * invalidation. 1717 */ 1718 static void 1719 pmap_pv_enter(pmap_t pmap, struct vm_page *pg, vaddr_t va, vm_prot_t prot, 1720 struct pmap_table *pt, pt_entry_t npte, struct pv_entry *newpv) 1721 { 1722 const bool usr_ci = pte_ci_p(npte); 1723 struct pv_entry *pv; 1724 pt_entry_t opte; 1725 1726 pmap_evcnt(pv_enter_called); 1727 1728 PMAP_CRIT_ASSERT(); 1729 KASSERT(newpv != NULL); 1730 1731 npte |= PTE_PVLIST; 1732 1733 newpv->pv_pmap = pmap; 1734 newpv->pv_vf = va; 1735 newpv->pv_pt = pt; 1736 1737 pt_entry_t *ptep = pmap_pv_pte(newpv); 1738 1739 #ifdef DEBUG 1740 /* 1741 * Make sure the entry doesn't already exist. 1742 */ 1743 for (pv = VM_MDPAGE_PVS(pg); pv != NULL; pv = pv->pv_next) { 1744 if (pmap == pv->pv_pmap && va == PV_VA(pv)) { 1745 panic("%s: pmap=%p va=0x%08lx already in PV table", 1746 __func__, pmap, va); 1747 } 1748 } 1749 #endif 1750 1751 if (__predict_false(usr_ci)) { 1752 newpv->pv_vf |= PV_F_CI_USR; 1753 } 1754 1755 newpv->pv_next = VM_MDPAGE_PVS(pg); 1756 VM_MDPAGE_SETPVP(VM_MDPAGE_HEAD_PVP(pg), newpv); 1757 LIST_INSERT_HEAD(&pmap->pm_pvlist, newpv, pv_pmlist); 1758 VM_MDPAGE_ADD_UM(pg, npte); 1759 1760 /* 1761 * If this is an EXEC mapping, then we have to ensure that 1762 * the I$ doesn't load stale data. 1763 * 1764 * XXX Should have a soft-PTE bit for this. 1765 */ 1766 if (prot & UVM_PROT_EXEC) { 1767 #if MMU_CONFIG_68040_CLASS 1768 if (MMU_IS_68040_CLASS) { 1769 /* 1770 * XXX Potential future optimization: is only 1771 * XXX the DCFP() needed here to deal with 1772 * XXX write-back? Should we track EXEC-ness 1773 * XXX in the VM_MDPAGE? 1774 */ 1775 const paddr_t pa = VM_PAGE_TO_PHYS(pg); 1776 DCFP(pa); 1777 ICPP(pa); 1778 } 1779 #endif 1780 } 1781 1782 #if MMU_CONFIG_HP_CLASS 1783 if (MMU_IS_HP_CLASS) { 1784 /* Go handle the HP MMU's VAC. */ 1785 goto hp_mmu_vac_shenanigans; 1786 } 1787 #endif 1788 1789 /* 1790 * If the page is marked as being cache-inhibited, it means 1791 * there is at least one user-requested CI mapping already 1792 * (and that all of the extant mappings are thus CI). 1793 * 1794 * In this case, we need to make sure that the one we're 1795 * establishing now is CI as well. 1796 */ 1797 if (__predict_false(VM_MDPAGE_CI_P(pg))) { 1798 npte = pte_set_ci(npte); 1799 pte_store(ptep, npte); 1800 /* See below. */ 1801 if (active_pmap(pmap)) { 1802 TBIS(va); 1803 } 1804 return; 1805 } 1806 1807 /* Set the PTE for the new mapping. */ 1808 pte_store(ptep, npte); 1809 1810 /* 1811 * Invalidate the ATC entry **after** storing the PTE so that 1812 * there is no window where another MMU table walk finds the 1813 * stale invalid entry. 1814 */ 1815 if (active_pmap(pmap)) { 1816 TBIS(va); 1817 } 1818 1819 /* 1820 * If this is a user-requested CI mapping, we need to make 1821 * sure the page is purged from the cache and mark any other 1822 * mappings of this page CI as well. 1823 */ 1824 if (__predict_false(usr_ci)) { 1825 VM_MDPAGE_SET_CI(pg); 1826 1827 pmap_evcnt(pv_enter_usr_ci); 1828 1829 /* 1830 * There shouldn't be very many of these; CI mappings 1831 * of managed pages are typically only for coherent DMA 1832 * purposes, and multiple mappings of the same page are 1833 * extremely uncommon in that scenario. 1834 */ 1835 for (pv = newpv->pv_next; pv != NULL; pv = pv->pv_next) { 1836 pmap_evcnt(pv_enter_ci_multi); 1837 ptep = pmap_pv_pte(pv); 1838 for (;;) { 1839 opte = pte_load(ptep); 1840 npte = pte_set_ci(opte); 1841 if (pte_update(ptep, opte, npte)) { 1842 if (active_pmap(pv->pv_pmap)) { 1843 TBIS(PV_VA(pv)); 1844 } 1845 break; 1846 } 1847 } 1848 } 1849 #if MMU_CONFIG_68040_CLASS 1850 if (MMU_IS_68040_CLASS) { 1851 const paddr_t pa = VM_PAGE_TO_PHYS(pg); 1852 DCFP(pa); 1853 ICPP(pa); 1854 } 1855 #endif 1856 } 1857 return; 1858 1859 #if MMU_CONFIG_HP_CLASS 1860 hp_mmu_vac_shenanigans: 1861 /* 1862 * We have ourselves a VAC, so in addition to checking for 1863 * user-requested-CI mappings, we have to check for cache 1864 * aliases and cache-inhibit all mappings for a page that 1865 * have a cache alias conflict. 1866 * 1867 * - All mappings of a given page within the same pmap must 1868 * not collide. (The VAC is flushed when switching pmaps 1869 * by virtue of a new segment table pointer being loaded 1870 * into the user segment table register.) 1871 * 1872 * - The Hibler pmap check to see that the kernel doesn't have 1873 * conflicting mappings with any user pmap. We'll do the same, 1874 * which seems reasonable on the surface if you think about it 1875 * for a couple of minutes. 1876 * 1877 * - The Hibler pmap also just punts and cache-inhibits all 1878 * mappings once it becomes > 2, but we do NOT do that because 1879 * it will severely penalize shared libraries. 1880 * 1881 * N.B. The method used here will not universally render all 1882 * mappings for a given page uncached; only address spaces with 1883 * conflicts are penalized. 1884 * 1885 * XXX This probably only matters if one of the mappings is 1886 * XXX writable, as this is the only situation where data 1887 * XXX inconsistency could arise. There is probably room 1888 * XXX for further optimization if someone with one of these 1889 * XXX machines cares to take it up. 1890 */ 1891 bool flush_s_vac = false; 1892 bool flush_u_vac = false; 1893 1894 /* Set the PTE for the new mapping. */ 1895 pte_store(ptep, npte); 1896 1897 /* 1898 * Invalidate the ATC entry **after** storing the PTE so that 1899 * there is no window where another MMU table walk finds the 1900 * stale invalid entry. 1901 * 1902 * XXX I don't know that this is strictly necessary with the 1903 * XXX HP MMU, but there is basically zero documentation available 1904 * XXX for it, so we err on the side of caution. 1905 */ 1906 if (active_pmap(pmap)) { 1907 TBIS(va); 1908 } 1909 1910 vaddr_t pv_flags = newpv->pv_vf & PV_F_CI_USR; 1911 if (usr_ci) { 1912 pmap_evcnt(pv_enter_usr_ci); 1913 } 1914 1915 for (pv = newpv->pv_next; pv != NULL; pv = pv->pv_next) { 1916 if (MATCHING_PMAP(pmap, pv->pv_pmap) && 1917 CONFLICTING_ALIAS(va, PV_VA(pv))) { 1918 pmap_evcnt(pv_enter_vac_ci); 1919 pv_flags |= PV_F_CI_VAC; 1920 break; 1921 } 1922 } 1923 1924 if (__predict_true(pv_flags == 0)) { 1925 /* No new inhibitions! */ 1926 return; 1927 } 1928 1929 VM_MDPAGE_SET_CI(pg); 1930 for (pv = VM_MDPAGE_PVS(pg); pv != NULL; pv = pv->pv_next) { 1931 if (MATCHING_PMAP(pmap, pv->pv_pmap)) { 1932 pmap_evcnt(pv_enter_ci_multi); 1933 pv->pv_vf |= pv_flags; 1934 pte_set(pmap_pv_pte(pv), PTE51_CI); 1935 if (active_pmap(pv->pv_pmap)) { 1936 TBIS(PV_VA(pv)); 1937 if (pv->pv_pmap == pmap_kernel()) { 1938 flush_s_vac = true; 1939 } else { 1940 flush_u_vac = true; 1941 } 1942 } 1943 } 1944 } 1945 if (flush_u_vac && flush_s_vac) { 1946 DCIA(); 1947 } else if (flush_u_vac) { 1948 DCIU(); 1949 } else if (flush_s_vac) { 1950 DCIS(); 1951 } 1952 #endif /* MMU_CONFIG_HP_CLASS */ 1953 } 1954 1955 /* 1956 * pmap_pv_remove: 1957 * 1958 * Remove a physical->virtual entry from the pv table. 1959 */ 1960 static void 1961 pmap_pv_remove(pmap_t pmap, struct vm_page *pg, vaddr_t va, 1962 struct pmap_completion *pc) 1963 { 1964 struct pv_entry **pvp, *pv; 1965 pt_entry_t *ptep, opte, npte; 1966 1967 pmap_evcnt(pv_remove_called); 1968 1969 PMAP_CRIT_ASSERT(); 1970 1971 for (pvp = VM_MDPAGE_HEAD_PVP(pg), pv = VM_MDPAGE_PVS(pg); 1972 pv != NULL; 1973 pvp = &pv->pv_next, pv = *pvp) { 1974 if (pmap == pv->pv_pmap && va == PV_VA(pv)) { 1975 break; 1976 } 1977 } 1978 1979 KASSERT(pv != NULL); 1980 VM_MDPAGE_SETPVP(pvp, pv->pv_next); 1981 LIST_REMOVE(pv, pv_pmlist); 1982 1983 KASSERT(pc != NULL); 1984 LIST_INSERT_HEAD(&pc->pc_pvlist, pv, pv_pmlist); 1985 1986 #if MMU_CONFIG_HP_CLASS 1987 if (MMU_IS_HP_CLASS) { 1988 /* Go handle the HP MMU's VAC. */ 1989 goto hp_mmu_vac_shenanigans; 1990 } 1991 #endif 1992 1993 /* 1994 * If the page is marked as being cache-inhibited, then it 1995 * means there was at least one user-requested CI mapping 1996 * for the page. In that case, we need to scan the P->V 1997 * list to see if any remain, and if not, clear the CI 1998 * status for the page. 1999 * 2000 * N.B. This requires traversing the list twice: once to 2001 * check if any of the mappings are user-requested-CI, 2002 * and one again to fix them up. But, we're making a 2003 * classical space-vs-time trade-off here: Assuming that 2004 * this is a rare situation, it's better to pay the cpu 2005 * cost on the rare edge transitions rather than always pay 2006 * the memory cost of having a counter to track something 2007 * that almost never happens (and, when it does, the list 2008 * will be very short). 2009 */ 2010 if (__predict_false(VM_MDPAGE_CI_P(pg))) { 2011 pmap_evcnt(pv_remove_ci); 2012 for (pv = VM_MDPAGE_PVS(pg); pv != NULL; pv = pv->pv_next) { 2013 if (pv->pv_vf & PV_F_CI_USR) { 2014 /* 2015 * There is still at least one user-requested 2016 * CI mapping, so we can't change the page's CI 2017 * status. 2018 */ 2019 return; 2020 } 2021 } 2022 KASSERT(pv == NULL); 2023 VM_MDPAGE_CLR_CI(pg); 2024 for (pv = VM_MDPAGE_PVS(pg); pv != NULL; pv = pv->pv_next) { 2025 ptep = pmap_pv_pte(pv); 2026 for (;;) { 2027 opte = pte_load(ptep); 2028 npte = pte_clr_ci(opte); 2029 if (pte_update(ptep, opte, npte)) { 2030 if (active_pmap(pv->pv_pmap)) { 2031 TBIS(PV_VA(pv)); 2032 } 2033 break; 2034 } 2035 } 2036 } 2037 } 2038 return; 2039 2040 #if MMU_CONFIG_HP_CLASS 2041 hp_mmu_vac_shenanigans: 2042 /* 2043 * If we have a VAC and the page was cache-inhibited due to 2044 * a cache alias conflict, we can re-enable the cache if there 2045 * is just one such mapping left. 2046 */ 2047 if (__predict_false(VM_MDPAGE_CI_P(pg))) { 2048 vaddr_t all_ci_flags = PV_F_CI_USR; 2049 2050 pmap_evcnt(pv_remove_ci); 2051 2052 for (pv = VM_MDPAGE_PVS(pg); pv != NULL; pv = pv->pv_next) { 2053 if (! MATCHING_PMAP(pmap, pv->pv_pmap)) { 2054 continue; 2055 } 2056 if (pv->pv_vf & all_ci_flags) { 2057 /* 2058 * There is at least one CI_USR mapping 2059 * or more than one CI_VAC mapping, so 2060 * the CI status of the page remains 2061 * unchanged. 2062 */ 2063 return; 2064 } 2065 all_ci_flags |= pv->pv_vf & PV_F_CI_VAC; 2066 } 2067 KASSERT(pv == NULL); 2068 /* 2069 * We now know we can remove CI from the page mappings 2070 * in the matching address space. If no CI mappings 2071 * remain, then we can clear the CI indicator on the 2072 * page. 2073 */ 2074 all_ci_flags = 0; 2075 for (pv = VM_MDPAGE_PVS(pg); pv != NULL; pv = pv->pv_next) { 2076 if (! MATCHING_PMAP(pmap, pv->pv_pmap)) { 2077 all_ci_flags |= pv->pv_vf; 2078 continue; 2079 } 2080 pte_mask(pmap_pv_pte(pv), ~((uint32_t)PTE51_CI)); 2081 if (active_pmap(pv->pv_pmap)) { 2082 TBIS(PV_VA(pv)); 2083 } 2084 } 2085 all_ci_flags &= PV_F_CI_USR | PV_F_CI_VAC; 2086 if (__predict_true(all_ci_flags == 0)) { 2087 VM_MDPAGE_CLR_CI(pg); 2088 } 2089 } 2090 #endif /* MMU_CONFIG_HP_CLASS */ 2091 } 2092 2093 #undef CONFLICTING_ALIAS 2094 #undef MATCHING_PMAP 2095 2096 /***************** PMAP INTERFACE (AND ADJACENT) FUNCTIONS *******************/ 2097 2098 static inline void 2099 pmap_stat_update_impl(long *valp, int val) 2100 { 2101 *valp += val; 2102 } 2103 2104 #define pmap_stat_update(pm, stat, delta) \ 2105 pmap_stat_update_impl(&(pm)->pm_stats.stat, (delta)) 2106 2107 static inline void 2108 pmap_stat_set_impl(long *valp, int val) 2109 { 2110 atomic_store_relaxed(valp, val); 2111 } 2112 2113 #define pmap_stat_set(pm, stat, val) \ 2114 pmap_stat_set_impl(&(pm)->pm_stats.stat, (val)) 2115 2116 /* 2117 * pmap_pinit: 2118 * 2119 * Common bits of pmap structure initialization shared between 2120 * the kernel pmap and user pmaps. 2121 */ 2122 static void 2123 pmap_pinit(pmap_t pmap, paddr_t lev1pa) 2124 { 2125 pmap->pm_lev1pa = lev1pa; 2126 rb_tree_init(&pmap->pm_tables, &pmap_table_rb_ops); 2127 TAILQ_INIT(&pmap->pm_ptpages[0]); 2128 TAILQ_INIT(&pmap->pm_ptpages[1]); 2129 LIST_INIT(&pmap->pm_pvlist); 2130 2131 pmap->pm_refcnt = 1; 2132 } 2133 2134 /* 2135 * pmap_virtual_space: [ INTERFACE ] 2136 * 2137 * Define the initial bounds of the kernel virtual address space. 2138 * 2139 * In this implementation, the start address we return marks the 2140 * end of the statically allocated special kernel virtual addresses 2141 * set up in pmap_bootstrap1(). We return kernel_virtual_max as 2142 * the end because we can grow the kernel address space using 2143 * pmap_growkernel(). 2144 */ 2145 void 2146 pmap_virtual_space(vaddr_t *vstartp, vaddr_t *vendp) 2147 { 2148 *vstartp = kernel_virtual_start; 2149 *vendp = kernel_virtual_max; 2150 } 2151 2152 /* 2153 * pmap_init: [ INTERFACE ] 2154 * 2155 * Initialize the pmap module. Called by vm_init(), to initialize any 2156 * structures that the pmap system needs to map virtual memory. 2157 */ 2158 void 2159 pmap_init(void) 2160 { 2161 /* Initialize the pmap / pv_entry allocators. */ 2162 pmap_alloc_init(); 2163 2164 /* Initialize the PT page allocator. */ 2165 pmap_ptpage_init(); 2166 2167 /* Now it's safe to do P->V entry recording! */ 2168 pmap_initialized_p = true; 2169 } 2170 2171 /* 2172 * pmap_create: [ INTERFACE ] 2173 * 2174 * Create and return a physical map. 2175 */ 2176 pmap_t 2177 pmap_create(void) 2178 { 2179 pmap_t pmap; 2180 2181 /* 2182 * We reference the null segment table and and have a NULL 2183 * lev1map pointer until the first mapping is entered. 2184 */ 2185 pmap = pmap_alloc(); 2186 pmap_pinit(pmap, null_segtab_pa); 2187 2188 return pmap; 2189 } 2190 2191 /* 2192 * pmap_destroy: [ INTERFACE ] 2193 * 2194 * Drop the reference count on the specified pmap, releasing 2195 * all resources if the reference count drops to zero. 2196 */ 2197 void 2198 pmap_destroy(pmap_t pmap) 2199 { 2200 unsigned int newval; 2201 2202 PMAP_CRIT_ENTER(); 2203 KASSERT(pmap->pm_refcnt > 0); 2204 newval = --pmap->pm_refcnt; 2205 PMAP_CRIT_EXIT(); 2206 2207 if (newval) { 2208 return; 2209 } 2210 2211 /* We assume all mappings have been removed. */ 2212 KASSERT(pmap->pm_lev1map == NULL); 2213 KASSERT(pmap->pm_lev1pa == null_segtab_pa); 2214 2215 pmap_free(pmap); 2216 } 2217 2218 /* 2219 * pmap_reference: [ INTERFACE ] 2220 * 2221 * Add a reference to the specified pmap. 2222 */ 2223 void 2224 pmap_reference(pmap_t pmap) 2225 { 2226 PMAP_CRIT_ENTER(); 2227 pmap->pm_refcnt++; 2228 KASSERT(pmap->pm_refcnt > 0); 2229 PMAP_CRIT_EXIT(); 2230 } 2231 2232 /* 2233 * pmap_remove_mapping: 2234 * 2235 * Invalidate a single page denoted by pmap/va. 2236 * 2237 * If (ptep != NULL), it is the already computed PTE for the mapping. 2238 * 2239 * If (flags & PRM_TFLUSH), we must invalidate any TLB information. 2240 * 2241 * If (flags & PRM_CFLUSH), we must flush/invalidate any cache 2242 * information. 2243 * 2244 * If the caller wishes to prevent the page table from being freed, 2245 * they should perform an extra retain. 2246 */ 2247 #define PRM_TFLUSH __BIT(0) 2248 #define PRM_CFLUSH __BIT(1) 2249 static void 2250 pmap_remove_mapping(pmap_t pmap, vaddr_t va, pt_entry_t *ptep, 2251 struct pmap_table *pt, struct vm_page *pg, 2252 int flags, struct pmap_completion *pc) 2253 { 2254 KASSERT(pt != NULL || pmap == pmap_kernel()); 2255 KASSERT(ptep != NULL); 2256 const paddr_t opte = pte_load(ptep); 2257 KASSERT(pte_valid_p(opte)); 2258 const paddr_t pa = pte_pa(opte); 2259 KASSERT(pg == NULL || pa == VM_PAGE_TO_PHYS(pg)); 2260 2261 /* Update statistics. */ 2262 if (pte_wired_p(opte)) { 2263 pmap_stat_update(pmap, wired_count, -1); 2264 } 2265 pmap_stat_update(pmap, resident_count, -1); 2266 2267 if (flags & PRM_CFLUSH) { 2268 #if MMU_CONFIG_68040_CLASS 2269 if (MMU_IS_68040_CLASS) { 2270 DCFP(pa); 2271 ICPP(pa); 2272 } 2273 #endif 2274 #if MMU_CONFIG_HP_CLASS 2275 if (MMU_IS_HP_CLASS) { 2276 if (pmap == pmap_kernel()) { 2277 DCIS(); 2278 } else if (active_user_pmap(pmap)) { 2279 DCIU(); 2280 } 2281 } 2282 #endif 2283 } 2284 2285 if (__predict_false(pg == NULL)) { 2286 pg = pmap_pa_to_pg(pa); 2287 if (pg != NULL) { 2288 pmap_evcnt(prm_lookup_pg_hit); 2289 } else { 2290 pmap_evcnt(prm_lookup_pg_miss); 2291 } 2292 } else { 2293 pmap_evcnt(prm_got_pg); 2294 } 2295 if (__predict_true(pg != NULL)) { 2296 KASSERT(pte_managed_p(opte)); 2297 /* Update cached U/M bits from mapping that's going away. */ 2298 VM_MDPAGE_ADD_UM(pg, opte); 2299 pmap_pv_remove(pmap, pg, va, pc); 2300 } else { 2301 KASSERT(! pte_managed_p(opte)); 2302 } 2303 2304 /* 2305 * Zap the PTE and drop the retain count that the mapping 2306 * had on the table. 2307 */ 2308 pte_store(ptep, 0); 2309 pmap_table_release(pmap, pt, pc); 2310 2311 /* 2312 * Now that the ATC can't be reloaded from the PTE, invalidate 2313 * the ATC entry. 2314 */ 2315 if (__predict_true((flags & PRM_TFLUSH) != 0 && active_pmap(pmap))) { 2316 TBIS(va); 2317 } 2318 } 2319 2320 /* 2321 * pmap_remove: [ INTERFACE ] 2322 * 2323 * Remove the given range of addresses from the specified map. 2324 * 2325 * It is assumed that the start and end are properly rounded 2326 * to the page size. 2327 * 2328 * N.B. Callers of pmap_remove_internal() are expected to 2329 * provide an initialized completion context, which we 2330 * will finalize. 2331 */ 2332 static void 2333 pmap_remove_internal(pmap_t pmap, vaddr_t sva, vaddr_t eva, 2334 struct pmap_completion *pc) 2335 { 2336 pt_entry_t opte, *ptep; 2337 struct pmap_table *pt; 2338 vaddr_t nextseg; 2339 int prm_flags; 2340 #if MMU_CONFIG_HP_CLASS 2341 pt_entry_t all_ci = PTE51_CI; 2342 #endif 2343 2344 /* 2345 * If this is the kernel pmap, we can use a faster method 2346 * for accessing the PTEs (since the PT pages are always 2347 * resident). 2348 * 2349 * Note that this routine should NEVER be called from an 2350 * interrupt context; pmap_kremove() is used for that. 2351 */ 2352 prm_flags = active_pmap(pmap) ? PRM_TFLUSH : 0; 2353 if (pmap == pmap_kernel()) { 2354 PMAP_CRIT_ENTER(pmap_busy(pmap)); 2355 2356 for (ptep = pmap_kernel_pte(sva); sva < eva; 2357 ptep++, sva += PAGE_SIZE) { 2358 opte = pte_load(ptep); 2359 if (pte_valid_p(opte)) { 2360 #if MMU_CONFIG_HP_CLASS 2361 /* 2362 * If all of the PTEs we're zapping have the 2363 * cache-inhibit bit set, ci_pte will remain 2364 * non-zero and we'll be able to skip flushing 2365 * the VAC when we're done. 2366 */ 2367 all_ci &= opte; 2368 #endif 2369 pmap_remove_mapping(pmap, sva, ptep, NULL, 2370 NULL, prm_flags, pc); 2371 } 2372 } 2373 #if MMU_CONFIG_HP_CLASS 2374 if (MMU_IS_HP_CLASS && !all_ci) { 2375 /* 2376 * Cacheable mappings were removed, so invalidate 2377 * the cache. 2378 */ 2379 DCIS(); 2380 } 2381 #endif 2382 PMAP_CRIT_EXIT(pmap_unbusy(pmap)); 2383 2384 /* kernel PT pages are never freed. */ 2385 KASSERT(TAILQ_EMPTY(&pc->pc_ptpages)); 2386 2387 /* ...but we might have freed PV entries. */ 2388 pmap_completion_fini(pc); 2389 2390 return; 2391 } 2392 2393 PMAP_CRIT_ENTER(pmap_busy(pmap)); 2394 2395 while (sva < eva) { 2396 nextseg = pmap_next_seg(sva); 2397 if (nextseg == 0 || nextseg > eva) { 2398 nextseg = eva; 2399 } 2400 2401 ptep = pmap_pte_lookup(pmap, sva, &pt); 2402 if (ptep == NULL) { 2403 /* 2404 * No table for this address, meaning nothing 2405 * within this segment; advance to the next 2406 * one. 2407 */ 2408 sva = nextseg; 2409 continue; 2410 } 2411 2412 for (; sva < nextseg; ptep++, sva += PAGE_SIZE) { 2413 opte = pte_load(ptep); 2414 if (! pte_valid_p(opte)) { 2415 continue; 2416 } 2417 #if MMU_CONFIG_HP_CLASS 2418 /* 2419 * If all of the PTEs we're zapping have the 2420 * cache-inhibit bit set, ci_pte will remain 2421 * non-zero and we'll be able to skip flushing 2422 * the VAC when we're done. 2423 */ 2424 all_ci &= opte; 2425 #endif 2426 pmap_remove_mapping(pmap, sva, ptep, pt, NULL, 2427 prm_flags, pc); 2428 } 2429 pmap_table_release(pmap, pt, pc); 2430 } 2431 #if MMU_CONFIG_HP_CLASS 2432 if (MMU_IS_HP_CLASS && !all_ci) { 2433 /* 2434 * Cacheable mappings were removed, so invalidate 2435 * the cache. 2436 */ 2437 if (pmap == pmap_kernel()) { 2438 DCIS(); 2439 } else if (active_user_pmap(pmap)) { 2440 DCIU(); 2441 } 2442 } 2443 #endif 2444 PMAP_CRIT_EXIT(pmap_unbusy(pmap)); 2445 2446 pmap_completion_fini(pc); 2447 } 2448 2449 void 2450 pmap_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva) 2451 { 2452 struct pmap_completion pc; 2453 pmap_completion_init(&pc); 2454 pmap_remove_internal(pmap, sva, eva, &pc); 2455 /* pmap_remove_internal() calls pmap_completion_fini(). */ 2456 } 2457 2458 /* 2459 * pmap_remove_all: [ INTERFACE ] 2460 * 2461 * Remove all mappings from a pmap in bulk. This is only called 2462 * when it's known that the address space is no longer visible to 2463 * any user process (e.g. during exit or exec). 2464 */ 2465 bool 2466 pmap_remove_all(pmap_t pmap) 2467 { 2468 struct pmap_completion pc; 2469 struct pv_entry *pv; 2470 struct vm_page *pg; 2471 pt_entry_t opte; 2472 2473 KASSERT(pmap != pmap_kernel()); 2474 2475 /* 2476 * This process is pretty simple: 2477 * 2478 * ==> (1) Set the segment table pointer to the NULL segment table. 2479 * 2480 * ==> (2) Copy the PT page list to a tempory list and re-init. 2481 * 2482 * ==> (3) Walk the PV entry list and remove each entry. 2483 * 2484 * ==> (4) Zero the wired and resident count. 2485 * 2486 * Once we've done that, we just need to free everything 2487 * back to the system. 2488 */ 2489 2490 pmap_completion_init(&pc); 2491 2492 PMAP_CRIT_ENTER(pmap_busy(pmap)); 2493 2494 /* Step 1. */ 2495 pmap_set_lev1map(pmap, NULL, null_segtab_pa); 2496 2497 /* Step 2. */ 2498 pmap->pm_pt_cache = NULL; 2499 TAILQ_CONCAT(&pc.pc_ptpages, &pmap->pm_ptpages[0], ptp_list); 2500 TAILQ_CONCAT(&pc.pc_ptpages, &pmap->pm_ptpages[1], ptp_list); 2501 memset(&pmap->pm_tables, 0, sizeof(pmap->pm_tables)); 2502 rb_tree_init(&pmap->pm_tables, &pmap_table_rb_ops); 2503 KASSERT(RB_TREE_MIN(&pmap->pm_tables) == NULL); 2504 2505 /* Step 3. */ 2506 while ((pv = LIST_FIRST(&pmap->pm_pvlist)) != NULL) { 2507 KASSERT(pv->pv_pmap == pmap); 2508 opte = pte_load(pmap_pv_pte(pv)); 2509 pg = pmap_pa_to_pg(pte_pa(opte)); 2510 /* Update cached U/M bits from mapping that's going away. */ 2511 VM_MDPAGE_ADD_UM(pg, opte); 2512 pmap_pv_remove(pmap, pg, PV_VA(pv), &pc); 2513 } 2514 2515 /* Step 4. */ 2516 pmap_stat_set(pmap, wired_count, 0); 2517 pmap_stat_set(pmap, resident_count, 0); 2518 2519 PMAP_CRIT_EXIT(pmap_unbusy(pmap)); 2520 2521 pmap_completion_fini(&pc); 2522 2523 return true; 2524 } 2525 2526 /* 2527 * pmap_page_protect: [ INTERFACE ] 2528 * 2529 * Lower the permission for all mappings to a given page to 2530 * the permissions specified. 2531 */ 2532 void 2533 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 2534 { 2535 struct pmap_completion pc; 2536 struct pv_entry *pv; 2537 2538 if (prot & UVM_PROT_WRITE) { 2539 /* No protection to revoke. */ 2540 return; 2541 } 2542 2543 if (prot & UVM_PROT_READ) { 2544 /* Making page copy-on-write. */ 2545 pmap_changebit(pg, PTE_WP, ~0U); 2546 return; 2547 } 2548 2549 /* Removing all mappings for a page. */ 2550 pmap_completion_init(&pc); 2551 2552 PMAP_CRIT_ENTER(); 2553 2554 while ((pv = VM_MDPAGE_PVS(pg)) != NULL) { 2555 pmap_busy(pv->pv_pmap); 2556 pmap_remove_mapping(pv->pv_pmap, PV_VA(pv), pmap_pv_pte(pv), 2557 pv->pv_pt, pg, PRM_TFLUSH|PRM_CFLUSH, &pc); 2558 pmap_unbusy(pv->pv_pmap); 2559 } 2560 2561 PMAP_CRIT_EXIT(); 2562 2563 pmap_completion_fini(&pc); 2564 } 2565 2566 /* 2567 * pmap_protect: [ INTERFACE ] 2568 * 2569 * Set the physical protection on the specified range of this map 2570 * as requested. 2571 */ 2572 void 2573 pmap_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 2574 { 2575 pt_entry_t *ptep, opte, npte; 2576 struct pmap_table *pt; 2577 vaddr_t nextseg; 2578 #if MMU_CONFIG_68040_CLASS 2579 bool removing_write; 2580 #endif 2581 bool need_tflush; 2582 2583 if ((prot & UVM_PROT_READ) == 0) { 2584 struct pmap_completion pc; 2585 pmap_completion_init(&pc); 2586 pmap_remove_internal(pmap, sva, eva, &pc); 2587 /* pmap_remove_internal() calls pmap_completion_fini(). */ 2588 return; 2589 } 2590 2591 PMAP_CRIT_ENTER(pmap_busy(pmap)); 2592 2593 #if MMU_CONFIG_68040_CLASS 2594 removing_write = (prot & UVM_PROT_WRITE) == 0; 2595 #endif 2596 need_tflush = active_pmap(pmap); 2597 while (sva < eva) { 2598 nextseg = pmap_next_seg(sva); 2599 if (nextseg == 0 || nextseg > eva) { 2600 nextseg = eva; 2601 } 2602 2603 ptep = pmap_pte_lookup(pmap, sva, &pt); 2604 if (ptep == NULL) { 2605 /* 2606 * No table for this address, meaning nothing 2607 * within this segment; advance to the next 2608 * one. 2609 */ 2610 sva = nextseg; 2611 continue; 2612 } 2613 2614 /* 2615 * Change protection on mapping if it is valid and doesn't 2616 * already have the correct protection. 2617 */ 2618 for (; sva < nextseg; ptep++, sva += PAGE_SIZE) { 2619 try_again: 2620 opte = pte_load(ptep); 2621 if (! pte_valid_p(opte)) { 2622 continue; 2623 } 2624 npte = pte_change_prot(opte, prot); 2625 if (npte == opte) { 2626 continue; 2627 } 2628 #if MMU_CONFIG_68040_CLASS 2629 if (MMU_IS_68040_CLASS && removing_write) { 2630 /* 2631 * Clear caches if making RO (see section 2632 * "7.3 Cache Coherency" in the manual). 2633 */ 2634 paddr_t pa = pte_pa(opte); 2635 DCFP(pa); 2636 ICPP(pa); 2637 } 2638 #endif 2639 if (! pte_update(ptep, opte, npte)) { 2640 /* Lost race updating PTE; try again. */ 2641 goto try_again; 2642 } 2643 if (need_tflush) { 2644 TBIS(sva); 2645 } 2646 } 2647 pmap_table_release(pmap, pt, NULL); 2648 } 2649 2650 PMAP_CRIT_EXIT(pmap_unbusy(pmap)); 2651 } 2652 2653 /* 2654 * pmap_enter: [ INTERFACE ] 2655 * 2656 * Insert the given physical address (pa) at the specified 2657 * virtual address (va) in the target physical map with the 2658 * protection requested. 2659 * 2660 * If specified, the page will be wired down, meaning that 2661 * related pte can not be reclaimed. 2662 * 2663 * Note: This is the only routine which MAY NOT lazy-evaluate 2664 * or lose information. That is, this routine must actually 2665 * insert this page into the given map NOW. 2666 */ 2667 int 2668 pmap_enter(pmap_t pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 2669 { 2670 struct pmap_table *pt; 2671 pt_entry_t *ptep, npte, opte; 2672 struct pv_entry *newpv; 2673 struct pmap_completion pc; 2674 int error = 0; 2675 const bool nowait = !!(flags & PMAP_CANFAIL); 2676 2677 pmap_completion_init(&pc); 2678 2679 struct vm_page * const pg = pmap_pa_to_pg(pa); 2680 if (__predict_false(pg == NULL)) { 2681 /* 2682 * PA is not part of managed memory. Make the mapping 2683 * cache-inhibited on the assumption that it's a device. 2684 */ 2685 flags |= PMAP_NOCACHE; 2686 } 2687 2688 PMAP_CRIT_ENTER(pmap_busy(pmap)); 2689 2690 if (nowait) { 2691 pmap_evcnt(enter_nowait); 2692 } else { 2693 pmap_evcnt(enter_yeswait); 2694 } 2695 2696 /* Get the destination table. */ 2697 ptep = pmap_pte_alloc(pmap, va, &pt, nowait, &pc); 2698 if (__predict_false(ptep == NULL)) { 2699 pmap_evcnt(enter_pte_alloc_fail); 2700 error = ENOMEM; 2701 goto out; 2702 } 2703 2704 /* Compute the new PTE. */ 2705 npte = pmap_make_pte(pa, prot, flags); 2706 2707 /* Fetch old PTE. */ 2708 opte = pte_load(ptep); 2709 2710 /* 2711 * Check to see if there is a valid mapping at this address. 2712 * It might simply be a wiring or protection change. 2713 */ 2714 if (pte_valid_p(opte)) { 2715 pmap_evcnt(enter_valid); 2716 restart: 2717 if (pte_pa(opte) == pa) { 2718 /* 2719 * Just a protection or wiring change. 2720 * 2721 * Since the old PTE is handy, go ahead and update 2722 * the cached U/M attributes now. Normally we would 2723 * do this in pmap_remove_mapping(), but we're not 2724 * taking that path in this case. We also add in 2725 * any U/M attributes hinted by the access type 2726 * that brought us to pmap_enter() in the first 2727 * place (a write-fault on a writable page mapped 2728 * read-only during a page-out, for example). 2729 * 2730 * Also ensure that the PV list status of the mapping 2731 * is consistent. 2732 */ 2733 if (__predict_true(pg != NULL)) { 2734 VM_MDPAGE_ADD_UM(pg, opte | npte); 2735 KASSERT(pte_managed_p(opte)); 2736 npte |= PTE_PVLIST; 2737 } 2738 2739 /* Preserve cache-inhibited status. */ 2740 if (__predict_false(pte_ci_p(opte))) { 2741 npte = 2742 (npte & ~PTE_CMASK) | (opte & PTE_CMASK); 2743 } 2744 2745 /* Set the new PTE. */ 2746 pte_store(ptep, npte); 2747 2748 const pt_entry_t diff = opte ^ npte; 2749 2750 #ifdef PMAP_EVENT_COUNTERS 2751 if (diff & PTE_WIRED) { 2752 pmap_evcnt(enter_wire_change); 2753 } 2754 if (diff & PTE_WP) { 2755 pmap_evcnt(enter_prot_change); 2756 } 2757 #endif 2758 2759 if (pte_wired_p(diff)) { 2760 pmap_stat_update(pmap, wired_count, 2761 pte_wired_p(npte) ? 1 : -1); 2762 } 2763 if (diff) { 2764 #if MMU_CONFIG_68040_CLASS 2765 /* 2766 * Protection or caching status is changing; 2767 * flush the page from the cache. 2768 */ 2769 if (MMU_IS_68040_CLASS) { 2770 DCFP(pa); 2771 ICPP(pa); 2772 } 2773 #endif 2774 if (active_pmap(pmap)) { 2775 TBIS(va); 2776 #if MMU_CONFIG_HP_CLASS 2777 /* 2778 * If the new mapping is CI and the old 2779 * one is not, then flush the VAC. 2780 */ 2781 if (__predict_false(MMU_IS_HP_CLASS && 2782 pte_ci_p(diff) && 2783 pte_ci_p(npte))) { 2784 DCIA(); 2785 } 2786 #endif 2787 } 2788 } 2789 2790 /* All done! */ 2791 goto out_release; 2792 } 2793 2794 /* 2795 * The mapping has completely changed. Need to remove 2796 * the old one first. 2797 * 2798 * This drops the retain count on the PT owned by the 2799 * previous mapping, but the newly-entered mapping will 2800 * inherit the retain count taken when we looked up the 2801 * PTE. 2802 * 2803 * XXX Can we elide the ATC flush here? We're going to 2804 * XXX hit the ATC after setting the new PTE anyway. 2805 */ 2806 pmap_evcnt(enter_pa_change); 2807 pmap_remove_mapping(pmap, va, ptep, pt, NULL, 2808 PRM_TFLUSH|PRM_CFLUSH, &pc); 2809 } 2810 2811 /* 2812 * By the time we get here, we should be assured that the 2813 * PTE at ptep is invalid. 2814 */ 2815 KASSERT(! pte_valid_p(pte_load(ptep))); 2816 2817 /* Update pmap stats now. */ 2818 pmap_stat_update(pmap, resident_count, 1); 2819 if (__predict_false(pte_wired_p(npte))) { 2820 pmap_stat_update(pmap, wired_count, 1); 2821 } 2822 2823 if (__predict_true(pg != NULL)) { 2824 /* 2825 * Managed pages also go on the PV list, so we are 2826 * going to need a PV entry. 2827 */ 2828 newpv = LIST_FIRST(&pc.pc_pvlist); 2829 if (__predict_true(newpv == NULL)) { 2830 /* 2831 * No PV entry to recycle; allocate a new one. 2832 * Because this is an extremely common case, we 2833 * are first going to attempt allocation while 2834 * still in the critical section. If that fails 2835 * and waiting is allowed, we'll leave the critical 2836 * section and try a blocking allocation. 2837 */ 2838 newpv = pmap_pv_alloc(true/*nowait flag*/); 2839 if (__predict_false(newpv == NULL)) { 2840 if (nowait) { 2841 pmap_evcnt(enter_pv_alloc_fail); 2842 error = ENOMEM; 2843 goto out_release; 2844 } 2845 PMAP_CRIT_EXIT(); 2846 newpv = pmap_pv_alloc(false/*nowait flag*/); 2847 KASSERT(newpv != NULL); 2848 PMAP_CRIT_ENTER(); 2849 /* 2850 * Because we may have blocked while allocating 2851 * the PV entry, we have to re-validate our 2852 * environment, as another thread could have 2853 * inserted a mapping here behind our back. 2854 */ 2855 opte = pte_load(ptep); 2856 if (__predict_false(pte_valid_p(opte))) { 2857 pmap_stat_update(pmap, 2858 resident_count, -1); 2859 if (pte_wired_p(npte)) { 2860 pmap_stat_update(pmap, 2861 wired_count, -1); 2862 } 2863 LIST_INSERT_HEAD(&pc.pc_pvlist, 2864 newpv, pv_pmlist); 2865 goto restart; 2866 } 2867 } 2868 } else { 2869 pmap_evcnt(enter_pv_recycle); 2870 LIST_REMOVE(newpv, pv_pmlist); 2871 } 2872 2873 /* 2874 * Enter the mapping into the PV list. pmap_pv_enter() 2875 * will also set the PTE in the table. 2876 */ 2877 pmap_pv_enter(pmap, pg, va, prot, pt, npte, newpv); 2878 2879 /* 2880 * The new mapping takes ownership of the PT 2881 * retain count we took while looking up the PTE. 2882 */ 2883 goto out_crit_exit; 2884 } 2885 2886 /* 2887 * Not a managed mapping, so set the new PTE. As with managed 2888 * mappings, the new mapping takes ownership of the PT retain 2889 * count we took while looking up the PTE. 2890 */ 2891 pte_store(ptep, npte); 2892 2893 /* 2894 * See comments in pmap_pv_enter() for why we have to hit 2895 * the ATC here. 2896 */ 2897 if (active_pmap(pmap)) { 2898 TBIS(va); 2899 } 2900 goto out_crit_exit; 2901 2902 out_release: 2903 pmap_table_release(pmap, pt, &pc); 2904 out_crit_exit: 2905 PMAP_CRIT_EXIT(pmap_unbusy(pmap)); 2906 out: 2907 pmap_completion_fini(&pc); 2908 return error; 2909 } 2910 2911 /* 2912 * pmap_kenter_pa: [ INTERFACE ] 2913 * 2914 * Enter a va -> pa mapping into the kernel pmap without any 2915 * physical->virtual tracking. 2916 */ 2917 void 2918 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 2919 { 2920 pmap_t const pmap = pmap_kernel(); 2921 2922 KASSERT(va >= VM_MIN_KERNEL_ADDRESS); 2923 2924 pt_entry_t * const ptep = pmap_kernel_pte(va); 2925 2926 /* Build the new PTE. */ 2927 const pt_entry_t npte = pmap_make_pte(pa, prot, flags | PMAP_WIRED); 2928 2929 /* 2930 * If this is an EXEC mapping, then we have to ensure that 2931 * the I$ doesn't load stale data. 2932 */ 2933 if (__predict_false(prot & UVM_PROT_EXEC)) { 2934 #if MMU_CONFIG_68040_CLASS 2935 if (MMU_IS_68040_CLASS) { 2936 /* 2937 * XXX Potential future optimization: is only 2938 * XXX the DCFP() needed here to deal with 2939 * XXX write-back? 2940 */ 2941 DCFP(pa); 2942 ICPP(pa); 2943 } 2944 #endif 2945 } 2946 2947 /* There must not be a valid PTE here. */ 2948 KASSERT(! pte_valid_p(pte_load(ptep))); 2949 2950 /* Set the new PTE. */ 2951 pte_store(ptep, npte); 2952 2953 pmap_stat_update(pmap, resident_count, 1); 2954 pmap_stat_update(pmap, wired_count, 1); 2955 2956 /* 2957 * See comments in pmap_pv_enter() as for why we hit the ATC here. 2958 * This *should* be unnecessary because this is a wired kernel 2959 * mapping and no demand-page-ins should have happened at this 2960 * VA, but we're erring on the side of caution for now. 2961 */ 2962 TBIS(va); 2963 } 2964 2965 /* 2966 * pmap_kremove: [ INTERFACE ] 2967 * 2968 * Remove a mapping entered with pmap_kenter_pa() starting at va, 2969 * for size bytes (assumed to be page rounded). 2970 */ 2971 void 2972 pmap_kremove(vaddr_t va, vsize_t size) 2973 { 2974 pt_entry_t *ptep, opte; 2975 pmap_t const pmap = pmap_kernel(); 2976 int count = 0; 2977 #if MMU_CONFIG_HP_CLASS 2978 pt_entry_t all_ci = PTE51_CI; 2979 #endif 2980 2981 KASSERT(va >= VM_MIN_KERNEL_ADDRESS); 2982 2983 for (ptep = pmap_kernel_pte(va); size != 0; 2984 ptep++, size -= PAGE_SIZE, va += PAGE_SIZE) { 2985 opte = pte_load(ptep); 2986 if (pte_valid_p(opte)) { 2987 KASSERT(! pte_managed_p(opte)); 2988 KASSERT(pte_wired_p(opte)); 2989 #if MMU_CONFIG_HP_CLASS 2990 /* 2991 * If all of the PTEs we're zapping have the 2992 * cache-inhibit bit set, ci_pte will remain 2993 * non-zero and we'll be able to skip flushing 2994 * the VAC when we're done. 2995 */ 2996 all_ci &= opte; 2997 #endif 2998 /* Zap the mapping. */ 2999 pte_store(ptep, 0); 3000 TBIS(va); 3001 count++; 3002 } 3003 } 3004 #if MMU_CONFIG_HP_CLASS 3005 if (MMU_IS_HP_CLASS && !all_ci) { 3006 /* 3007 * Cacheable mappings were removed, so invalidate 3008 * the cache. 3009 */ 3010 DCIS(); 3011 } 3012 #endif 3013 /* Update stats. */ 3014 if (__predict_true(count != 0)) { 3015 pmap_stat_update(pmap, resident_count, -count); 3016 pmap_stat_update(pmap, wired_count, -count); 3017 } 3018 } 3019 3020 /* 3021 * pmap_unwire: [ INTERFACE ] 3022 * 3023 * Clear the wired attribute for a map/virtual-address pair. 3024 * 3025 * The mapping must already exist in the pmap. 3026 */ 3027 void 3028 pmap_unwire(pmap_t pmap, vaddr_t va) 3029 { 3030 struct pmap_table *pt; 3031 pt_entry_t opte, npte, *ptep; 3032 3033 PMAP_CRIT_ENTER(pmap_busy(pmap)); 3034 3035 ptep = pmap_pte_lookup(pmap, va, &pt); 3036 KASSERT(ptep != NULL); 3037 3038 for (;;) { 3039 opte = pte_load(ptep); 3040 KASSERT(pte_valid_p(opte)); 3041 3042 /* 3043 * If the wiring actually changed (always?), clear the wire 3044 * bit and update the wire count. Note that the wiring is 3045 * not a hardware characteristic so there is no need to 3046 * invalidate the ATC. 3047 */ 3048 if (! pte_wired_p(opte)) { 3049 break; 3050 } 3051 npte = opte & ~PTE_WIRED; 3052 if (pte_update(ptep, opte, npte)) { 3053 pmap_stat_update(pmap, wired_count, -1); 3054 break; 3055 } 3056 } 3057 3058 pmap_table_release(pmap, pt, NULL); 3059 3060 PMAP_CRIT_EXIT(pmap_unbusy(pmap)); 3061 } 3062 3063 /* 3064 * pmap_extract: [ INTERFACE ] 3065 * 3066 * Extract the physical address associated with the given 3067 * pmap/virtual address pair. 3068 * 3069 * pmap_extract_info: 3070 * 3071 * Like pmap_extract(), but also returns information 3072 * about the mapping (wired, cache-inhibited, etc.) 3073 */ 3074 bool 3075 pmap_extract_info(pmap_t pmap, vaddr_t va, paddr_t *pap, int *flagsp) 3076 { 3077 struct pmap_table *pt; 3078 pt_entry_t pte, *ptep; 3079 bool rv = false; 3080 3081 if (__predict_false(pmap == pmap_kernel() && 3082 va >= kernel_virtual_end)) { 3083 return false; 3084 } 3085 3086 PMAP_CRIT_ENTER(pmap_busy(pmap)); 3087 3088 ptep = pmap_pte_lookup(pmap, va, &pt); 3089 if (__predict_true(ptep != NULL)) { 3090 pte = pte_load(ptep); 3091 if (__predict_true(pte_valid_p(pte))) { 3092 if (__predict_true(pap != NULL)) { 3093 *pap = pte_pa(pte) | (va & PAGE_MASK); 3094 } 3095 if (__predict_false(flagsp != NULL)) { 3096 *flagsp = 3097 (pte_wired_p(pte) ? PMAP_WIRED : 0) | 3098 (pte_ci_p(pte) ? PMAP_NOCACHE : 0); 3099 } 3100 rv = true; 3101 } 3102 pmap_table_release(pmap, pt, NULL); 3103 } 3104 3105 PMAP_CRIT_EXIT(pmap_unbusy(pmap)); 3106 3107 return rv; 3108 } 3109 3110 bool 3111 pmap_extract(pmap_t pmap, vaddr_t va, paddr_t *pap) 3112 { 3113 return pmap_extract_info(pmap, va, pap, NULL); 3114 } 3115 3116 /* 3117 * vtophys: 3118 * 3119 * Dumber version of pmap_extract(pmap_kernel(), ...) 3120 */ 3121 paddr_t 3122 vtophys(vaddr_t va) 3123 { 3124 paddr_t pa; 3125 bool rv __diagused; 3126 3127 rv = pmap_extract_info(pmap_kernel(), va, &pa, NULL); 3128 KASSERT(rv); 3129 return rv ? pa : -1; 3130 } 3131 3132 /* 3133 * kvtop: 3134 * 3135 * Sigh. 3136 */ 3137 int 3138 kvtop(void *v) 3139 { 3140 return (int)vtophys((vaddr_t)v); 3141 } 3142 3143 /* 3144 * pmap_copy: [ INTERFACE ] 3145 * 3146 * Copy the mapping range specified by src_addr/len 3147 * from the source map to the range dst_addr/len 3148 * in the destination map. 3149 * 3150 * This routine is only advisory and need not do anything. 3151 */ 3152 /* call deleted in <machine/pmap.h> */ 3153 3154 /* 3155 * pmap_update: [ INTERFACE ] 3156 * 3157 * Require that all active physical maps contain no 3158 * incorrect entries NOW, by processing any deferred 3159 * pmap operations. 3160 */ 3161 /* call deleted in <machine/pmap.h> */ 3162 3163 /* 3164 * pmap_activate: [ INTERFACE ] 3165 * 3166 * Activate the pmap used by the specified process. This includes 3167 * reloading the MMU context of the current process, and marking 3168 * the pmap in use by the processor. 3169 */ 3170 void 3171 pmap_activate(struct lwp *l) 3172 { 3173 pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap; 3174 3175 KASSERT(l == curlwp); 3176 3177 /* 3178 * Because the kernel has a separate root pointer, we don't 3179 * need to activate the kernel pmap. 3180 */ 3181 if (pmap != pmap_kernel()) { 3182 PMAP_CRIT_ENTER(pmap_busy(pmap)); 3183 pmap_load_urp(pmap->pm_lev1pa); 3184 PMAP_CRIT_EXIT(); 3185 } 3186 } 3187 3188 /* 3189 * pmap_deactivate: [ INTERFACE ] 3190 * 3191 * Mark that the pmap used by the specified process is no longer 3192 * in use by the processor. 3193 */ 3194 void 3195 pmap_deactivate(struct lwp *l) 3196 { 3197 pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap; 3198 3199 if (pmap != pmap_kernel()) { 3200 PMAP_CRIT_ENTER(); 3201 PMAP_CRIT_EXIT(pmap_unbusy(pmap)); 3202 } 3203 } 3204 3205 static vaddr_t pmap_tmpmap_srcva; 3206 static vaddr_t pmap_tmpmap_dstva; 3207 3208 /* 3209 * pmap_zero_page: [ INTERFACE ] 3210 * 3211 * Zero the specified VM page by mapping the page into the kernel 3212 * and using memset() (or equivalent) to clear its contents. 3213 */ 3214 void 3215 pmap_zero_page(paddr_t pa) 3216 { 3217 const int flags = MMU_IS_HP_CLASS ? PMAP_NOCACHE|PMAP_WIRED 3218 : PMAP_WIRED; 3219 pt_entry_t * const dst_ptep = pmap_kernel_pte(pmap_tmpmap_dstva); 3220 3221 /* Build the new PTE. */ 3222 const pt_entry_t dst_pte = 3223 pmap_make_pte(pa, UVM_PROT_READ | UVM_PROT_WRITE, flags); 3224 3225 /* Set the new PTE. */ 3226 KASSERT(! pte_valid_p(pte_load(dst_ptep))); 3227 pte_store(dst_ptep, dst_pte); 3228 /* XXX Possibly being over-cautious here; see pmap_kenter_pa(). */ 3229 TBIS(pmap_tmpmap_dstva); 3230 3231 /* Zero the page. */ 3232 zeropage((void *)pmap_tmpmap_dstva); 3233 3234 /* Invalidate the PTEs. */ 3235 pte_store(dst_ptep, 0); 3236 TBIS(pmap_tmpmap_dstva); 3237 } 3238 3239 /* 3240 * pmap_copy_page: [ INTERFACE ] 3241 * 3242 * Copy the specified VM page by mapping the page(s) into the kernel 3243 * and using memcpy() (or equivalent). 3244 */ 3245 void 3246 pmap_copy_page(paddr_t src, paddr_t dst) 3247 { 3248 const int flags = MMU_IS_HP_CLASS ? PMAP_NOCACHE|PMAP_WIRED 3249 : PMAP_WIRED; 3250 pt_entry_t * const src_ptep = pmap_kernel_pte(pmap_tmpmap_srcva); 3251 pt_entry_t * const dst_ptep = pmap_kernel_pte(pmap_tmpmap_dstva); 3252 3253 /* Build the new PTEs. */ 3254 const pt_entry_t src_pte = 3255 pmap_make_pte(src, UVM_PROT_READ, flags); 3256 const pt_entry_t dst_pte = 3257 pmap_make_pte(dst, UVM_PROT_READ | UVM_PROT_WRITE, flags); 3258 3259 /* Set the new PTEs. */ 3260 KASSERT(! pte_valid_p(pte_load(src_ptep))); 3261 pte_store(src_ptep, src_pte); 3262 /* XXX Possibly being over-cautious here; see pmap_kenter_pa(). */ 3263 TBIS(pmap_tmpmap_srcva); 3264 3265 KASSERT(! pte_valid_p(pte_load(dst_ptep))); 3266 pte_store(dst_ptep, dst_pte); 3267 /* XXX Possibly being over-cautious here; see pmap_kenter_pa(). */ 3268 TBIS(pmap_tmpmap_dstva); 3269 3270 /* Copy the page. */ 3271 copypage((void *)pmap_tmpmap_srcva, (void *)pmap_tmpmap_dstva); 3272 3273 /* Invalidate the PTEs. */ 3274 pte_store(src_ptep, 0); 3275 TBIS(pmap_tmpmap_srcva); 3276 3277 pte_store(dst_ptep, 0); 3278 TBIS(pmap_tmpmap_dstva); 3279 } 3280 3281 /* 3282 * pmap_testbit: 3283 * 3284 * Test the modified / referenced bits of a physical page. 3285 */ 3286 static bool 3287 pmap_testbit(struct vm_page *pg, pt_entry_t bit) 3288 { 3289 struct pv_entry *pv; 3290 pt_entry_t combined_pte = 0; 3291 3292 PMAP_CRIT_ENTER(); 3293 3294 combined_pte = VM_MDPAGE_UM(pg); 3295 3296 for (pv = VM_MDPAGE_PVS(pg); 3297 (combined_pte & bit) == 0 && pv != NULL; 3298 pv = pv->pv_next) { 3299 pmap_busy(pv->pv_pmap); 3300 combined_pte |= pte_load(pmap_pv_pte(pv)); 3301 pmap_unbusy(pv->pv_pmap); 3302 } 3303 3304 VM_MDPAGE_ADD_UM(pg, combined_pte); 3305 3306 PMAP_CRIT_EXIT(); 3307 3308 return (combined_pte & bit) != 0; 3309 } 3310 3311 /* 3312 * pmap_is_referenced: [ INTERFACE ] 3313 * 3314 * Return whether or not the specified physical page has been referenced 3315 * by any physical maps. 3316 */ 3317 bool 3318 pmap_is_referenced(struct vm_page *pg) 3319 { 3320 return pmap_testbit(pg, PTE_U); 3321 } 3322 3323 /* 3324 * pmap_is_modified: [ INTERFACE ] 3325 * 3326 * Return whether or not the specified physical page has been modified 3327 * by any physical maps. 3328 */ 3329 bool 3330 pmap_is_modified(struct vm_page *pg) 3331 { 3332 return pmap_testbit(pg, PTE_M); 3333 } 3334 3335 /* 3336 * pmap_changebit: 3337 * 3338 * Test-and-change various bits (including mod/ref bits). 3339 * Returns the accumulated previously-set PTE bits. 3340 */ 3341 static pt_entry_t 3342 pmap_changebit(struct vm_page *pg, pt_entry_t set, pt_entry_t mask) 3343 { 3344 struct pv_entry *pv; 3345 pt_entry_t *ptep, combined_pte, opte, npte; 3346 3347 #if MMU_CONFIG_68040_CLASS 3348 /* 3349 * If we're making the page read-only or changing the caching 3350 * status of the page, we need to flush it the first time we 3351 * change a mapping. 3352 */ 3353 bool cflush_040; 3354 if (MMU_IS_68040_CLASS && 3355 ((set & PTE_CRIT_BITS) != 0 || 3356 (mask & PTE_CRIT_BITS) == 0)) { 3357 cflush_040 = true; 3358 } else { 3359 cflush_040 = false; 3360 } 3361 #endif 3362 3363 PMAP_CRIT_ENTER(); 3364 3365 /* 3366 * Since we need to report if the page was mod/ref'd before 3367 * we cleared the bit, we need to seed ourself with the current 3368 * state in the vm_page in the event there are no mappings 3369 * left to enumerate. 3370 */ 3371 combined_pte = VM_MDPAGE_UM(pg); 3372 3373 /* 3374 * Since we're running over every mapping for the page anyway, 3375 * we might as well synchronize any attribute bits that we're 3376 * not clearing. 3377 */ 3378 for (pv = VM_MDPAGE_PVS(pg); pv != NULL; pv = pv->pv_next) { 3379 pmap_busy(pv->pv_pmap); 3380 for (;;) { 3381 ptep = pmap_pv_pte(pv); 3382 opte = pte_load(ptep); 3383 npte = (opte | set) & mask; 3384 combined_pte |= opte; 3385 3386 if (opte == npte) { 3387 break; 3388 } 3389 #if MMU_CONFIG_68040_CLASS 3390 if (__predict_false(cflush_040)) { 3391 paddr_t pa = VM_PAGE_TO_PHYS(pg); 3392 DCFP(pa); 3393 ICPP(pa); 3394 cflush_040 = false; 3395 } 3396 #endif 3397 if (pte_update(ptep, opte, npte)) { 3398 break; 3399 } 3400 /* Lost race, try again. */ 3401 } 3402 /* 3403 * We must flush the ATC even if it's not a "critical" bit, 3404 * because the MMU will only write-back a PTE when the 3405 * U or M bits transition from 0 to 1. If we clear them 3406 * in our tracking structure and in the tables in memory 3407 * but not the ATC, then a sticky ATC entry might cause 3408 * a future U or M event to be missed. 3409 */ 3410 if (active_pmap(pv->pv_pmap)) { 3411 TBIS(PV_VA(pv)); 3412 } 3413 pmap_unbusy(pv->pv_pmap); 3414 } 3415 3416 /* 3417 * Update any attributes we looked at, clear the ones we're clearing. 3418 */ 3419 VM_MDPAGE_SET_UM(pg, (combined_pte | set) & mask); 3420 3421 PMAP_CRIT_EXIT(); 3422 3423 return combined_pte; 3424 } 3425 3426 /* 3427 * pmap_clear_modify: [ INTERFACE ] 3428 * 3429 * Clear the modify bits on the specified physical page. 3430 */ 3431 bool 3432 pmap_clear_modify(struct vm_page *pg) 3433 { 3434 return (pmap_changebit(pg, 0, (pt_entry_t)~PTE_M) & PTE_M) != 0; 3435 } 3436 3437 /* 3438 * pmap_clear_reference: [ INTERFACE ] 3439 * 3440 * Clear the reference bit on the specified physical page. 3441 */ 3442 bool 3443 pmap_clear_reference(struct vm_page *pg) 3444 { 3445 return (pmap_changebit(pg, 0, (pt_entry_t)~PTE_U) & PTE_U) != 0; 3446 } 3447 3448 /* 3449 * pmap_phys_address: [ INTERFACE ] 3450 * 3451 * Return the physical address corresponding to the specified 3452 * cookie. Used by the device pager to decode a device driver's 3453 * mmap entry point return value. 3454 */ 3455 paddr_t 3456 pmap_phys_address(paddr_t cookie) 3457 { 3458 return m68k_ptob(cookie); 3459 } 3460 3461 static pt_entry_t *kernel_lev1map; 3462 3463 /* 3464 * pmap_growkernel_alloc_page: 3465 * 3466 * Helper for pmap_growkernel(). 3467 */ 3468 static paddr_t 3469 pmap_growkernel_alloc_page(void) 3470 { 3471 /* 3472 * XXX Needs more work if we're going to do this during 3473 * XXX early bootstrap. 3474 */ 3475 if (! uvm.page_init_done) { 3476 panic("%s: called before UVM initialized", __func__); 3477 } 3478 3479 struct vm_page *pg = pmap_page_alloc(true/*nowait*/); 3480 if (pg == NULL) { 3481 panic("%s: out of memory", __func__); 3482 } 3483 3484 paddr_t pa = VM_PAGE_TO_PHYS(pg); 3485 pmap_zero_page(pa); 3486 #if MMU_CONFIG_68040_CLASS 3487 if (MMU_IS_68040_CLASS) { 3488 DCFP(pa); 3489 } 3490 #endif 3491 return pa; 3492 } 3493 3494 /* 3495 * pmap_growkernel_link_kptpage: 3496 * 3497 * Helper for pmap_growkernel(). 3498 */ 3499 static void 3500 pmap_growkernel_link_kptpage(vaddr_t va, paddr_t ptp_pa) 3501 { 3502 /* 3503 * This is trivial for the 2-level MMU configuration. 3504 */ 3505 if (MMU_USE_2L) { 3506 KASSERT((kernel_lev1map[LA2L_RI(va)] & DT51_SHORT) == 0); 3507 kernel_lev1map[LA2L_RI(va)] = pmap_ste_proto | ptp_pa; 3508 return; 3509 } 3510 3511 /* 3512 * N.B. pmap_zero_page() is used in this process, which 3513 * uses pmap_tmpmap_dstva. pmap_tmpmap_srcva is available 3514 * for our use, however, so that's what we used to temporarily 3515 * map inner segment table pages. 3516 */ 3517 const vaddr_t stpg_va = pmap_tmpmap_srcva; 3518 3519 paddr_t stpa, stpg_pa, stpgoff, last_stpg_pa = (paddr_t)-1; 3520 paddr_t pa = ptp_pa, end_pa = ptp_pa + PAGE_SIZE; 3521 pt_entry_t *stes; 3522 3523 for (; pa < end_pa; va += NBSEG3L, pa += TBL40_L3_SIZE) { 3524 if ((kernel_lev1map[LA40_RI(va)] & UTE40_RESIDENT) == 0) { 3525 /* Level-2 table for this segment needed. */ 3526 if (kernel_stnext_pa == kernel_stnext_endpa) { 3527 /* 3528 * No more slots left in the last page 3529 * we allocated for segment tables. Grab 3530 * another one. 3531 */ 3532 kernel_stnext_pa = pmap_growkernel_alloc_page(); 3533 kernel_stnext_endpa = 3534 kernel_stnext_pa + PAGE_SIZE; 3535 pmap_nkstpages_current_ev.ev_count++; 3536 } 3537 kernel_lev1map[LA40_RI(va)] = 3538 pmap_ste_proto | kernel_stnext_pa; 3539 kernel_stnext_pa += TBL40_L2_SIZE; 3540 } 3541 stpa = kernel_lev1map[LA40_RI(va)] & UTE40_PTA; 3542 stpg_pa = m68k_trunc_page(stpa); 3543 if (stpg_pa != last_stpg_pa) { 3544 if (last_stpg_pa != (paddr_t)-1) { 3545 pmap_kremove(stpg_va, PAGE_SIZE); 3546 } 3547 pmap_kenter_pa(stpg_va, stpg_pa, 3548 UVM_PROT_READ | UVM_PROT_WRITE, 3549 PMAP_WIRED | PMAP_NOCACHE); 3550 last_stpg_pa = stpg_pa; 3551 } 3552 stpgoff = stpa - stpg_pa; 3553 stes = (pt_entry_t *)(stpg_va + stpgoff); 3554 stes[LA40_PI(va)] = pmap_ste_proto | pa; 3555 } 3556 if (last_stpg_pa != (paddr_t)-1) { 3557 pmap_kremove(stpg_va, PAGE_SIZE); 3558 } 3559 } 3560 3561 /* 3562 * pmap_growkernel: [ INTERFACE ] 3563 * 3564 * Grow the kernel address space. This is a hint from the 3565 * upper layer to pre-allocate more kernel PT pages. 3566 */ 3567 vaddr_t 3568 pmap_growkernel(vaddr_t maxkvaddr) 3569 { 3570 PMAP_CRIT_ENTER(); 3571 3572 KASSERT((kernel_virtual_end & PTPAGEVAOFS) == 0); 3573 3574 vaddr_t new_maxkva = pmap_round_ptpage(maxkvaddr); 3575 if (new_maxkva < kernel_virtual_end) { 3576 /* 3577 * Great news! We already have what we need to map 3578 * the requested max address. This happens one during 3579 * early bootstrap before UVM's notion of "maxkvaddr" 3580 * has been initialized. 3581 */ 3582 new_maxkva = kernel_virtual_end; 3583 goto done; 3584 } 3585 3586 if (new_maxkva > kernel_virtual_max) { 3587 panic("%s: out of kernel VA space (req=0x%08lx limit=0x%08lx)", 3588 __func__, maxkvaddr, kernel_virtual_max); 3589 } 3590 3591 /* 3592 * Allocate PT pages and link them into the MMU tree as we 3593 * go. 3594 */ 3595 vaddr_t va, ptp_pa; 3596 for (va = kernel_virtual_end; va < new_maxkva; va += PTPAGEVASZ) { 3597 /* Allocate page and link it into the MMU tree. */ 3598 ptp_pa = pmap_growkernel_alloc_page(); 3599 pmap_growkernel_link_kptpage(va, ptp_pa); 3600 pmap_nkptpages_current_ev.ev_count++; 3601 3602 /* Map the PT page into the kernel PTE array. */ 3603 pmap_kenter_pa((vaddr_t)pmap_kernel_pte(va), 3604 ptp_pa, UVM_PROT_READ | UVM_PROT_WRITE, 3605 PMAP_WIRED | PMAP_NOCACHE); 3606 } 3607 kernel_virtual_end = new_maxkva; 3608 done: 3609 pmap_maxkva_ev.ev_count32 = new_maxkva; 3610 pmap_kvalimit_ev.ev_count32 = kernel_virtual_max; 3611 PMAP_CRIT_EXIT(); 3612 return new_maxkva; 3613 } 3614 3615 /* 3616 * pmap_prefer: [ INTERFACE ] 3617 * 3618 * Attempt to arrange for pages at a given VM object offset 3619 * to occupy the same virtually-addressed cache footprint 3620 * in order to avoid cache aliases. 3621 */ 3622 #if MMU_CONFIG_HP_CLASS 3623 static struct evcnt pmap_prefer_nochange_ev = 3624 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap prefer", "nochange"); 3625 static struct evcnt pmap_prefer_change_ev = 3626 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap prefer", "change"); 3627 3628 EVCNT_ATTACH_STATIC(pmap_prefer_change_ev); 3629 EVCNT_ATTACH_STATIC(pmap_prefer_nochange_ev); 3630 #endif 3631 void 3632 pmap_prefer(vaddr_t hint, vaddr_t *vap, int td) 3633 { 3634 #if MMU_CONFIG_HP_CLASS 3635 if (MMU_IS_HP_CLASS) { 3636 vaddr_t va = *vap; 3637 ptrdiff_t diff = (hint - va) & pmap_aliasmask; 3638 3639 if (diff == 0) { 3640 pmap_prefer_nochange_ev.ev_count++; 3641 } else { 3642 pmap_prefer_change_ev.ev_count++; 3643 if (__predict_false(td)) { 3644 va -= pmap_aliasmask + 1; 3645 } 3646 *vap = va + diff; 3647 } 3648 } 3649 #endif 3650 } 3651 3652 /* 3653 * pmap_procwr: [ INTERFACE ] 3654 * 3655 * Perform any cache synchronization required after writing 3656 * to a process's address space. 3657 */ 3658 void 3659 pmap_procwr(struct proc *p, vaddr_t va, size_t len) 3660 { 3661 /* 3662 * This is just a wrapper around the "cachectl" machdep 3663 * system call. 3664 * 3665 * XXX This is kind of gross, to be honest. 3666 */ 3667 (void)cachectl1(0x80000004, va, len, p); 3668 } 3669 3670 #if defined(DDB) || defined(KGDB) 3671 /* 3672 * pmap_db_write_text_enter: 3673 * 3674 * Temporarily map a page of kernel text read-write for the 3675 * kernel debugger. 3676 */ 3677 bool 3678 pmap_db_write_text_enter(vaddr_t pgva, struct pmap_db_write_text_context *ctx) 3679 { 3680 pt_entry_t *pte = pmap_kernel_pte(pgva); 3681 3682 #ifdef M68K_MMU_HP 3683 /* 3684 * Flush the supervisor side of the VAC to prevent 3685 * a cache hit on the old, read-only PTE. 3686 * 3687 * XXX Is this really necessary? 3688 */ 3689 if (MMU_IS_HP_CLASS) { 3690 DCIS(); 3691 } 3692 #endif 3693 3694 pt_entry_t opte = pte_load(pte); 3695 if (! pte_valid_p(opte)) { 3696 return false; 3697 } 3698 3699 /* 3700 * N.B. we use the 68851 PTE bit names here, but in 3701 * the case of the kernel text, it all works out vis 3702 * a vis the 68040 PTE bits. 3703 * 3704 * Note the mapping is cache-inhibited to save hair. 3705 */ 3706 pt_entry_t npte = (opte & ~PTE51_WP) | PTE51_CI; 3707 pte_store(pte, npte); 3708 TBIS(pgva); 3709 3710 ctx->pgva = pgva; 3711 ctx->ptep = pte; 3712 ctx->opte = opte; 3713 3714 return true; 3715 } 3716 3717 /* 3718 * pmap_db_write_text_exit: 3719 * 3720 * Undo the effects of pmap_db_write_text_enter(). 3721 */ 3722 void 3723 pmap_db_write_text_exit(struct pmap_db_write_text_context *ctx) 3724 { 3725 pte_store(ctx->ptep, ctx->opte); 3726 TBIS(ctx->pgva); 3727 } 3728 #endif /* DDB || KGDB */ 3729 3730 static paddr_t kernel_reloc_offset; 3731 static vaddr_t kernel_reloc_end; 3732 3733 /* 3734 * pmap_init_kcore_hdr: 3735 * 3736 * Initialize the m68k kernel crash dump header with information 3737 * necessary to perform KVA -> phys translations. 3738 * 3739 * Returns a pointer to the crash dump RAM segment entries for 3740 * machine-specific code to initialize. 3741 */ 3742 phys_ram_seg_t * 3743 pmap_init_kcore_hdr(cpu_kcore_hdr_t *h) 3744 { 3745 struct gen68k_kcore_hdr *m = &h->un._gen68k; 3746 3747 memset(h, 0, sizeof(*h)); 3748 3749 /* 3750 * Initialize the `dispatcher' portion of the header. 3751 */ 3752 strcpy(h->name, "gen68k"); 3753 h->page_size = PAGE_SIZE; 3754 h->kernbase = VM_MIN_KERNEL_ADDRESS; 3755 3756 /* Fixed relocation information. */ 3757 m->reloc = kernel_reloc_offset; 3758 m->relocend = kernel_reloc_end; 3759 3760 /* 3761 * Fill in information about our MMU configuration. 3762 * 3763 * We essentially pretend to be a 68851 as far as table- 3764 * walks are concerned. 3765 * 3766 * We provide the kernel's MMU_* constant so that the TT 3767 * registers can be interpreted correctly. 3768 */ 3769 m->mmutype = mmutype; 3770 m->tcr = MMU_USE_3L ? MMU51_3L_TCR_BITS : MMU51_TCR_BITS; 3771 m->srp[0] = MMU51_SRP_BITS; 3772 m->srp[1] = Sysseg_pa; 3773 3774 #if MMU_CONFIG_68040_CLASS 3775 if (MMU_IS_68040_CLASS) { 3776 m->itt0 = mmu_tt40[MMU_TTREG_ITT0]; 3777 m->itt1 = mmu_tt40[MMU_TTREG_ITT1]; 3778 m->tt0 = mmu_tt40[MMU_TTREG_DTT0]; 3779 m->tt1 = mmu_tt40[MMU_TTREG_DTT1]; 3780 } 3781 #endif 3782 #if defined(M68K_MMU_68030) 3783 if (mmutype == MMU_68030) { 3784 m->tt0 = mmu_tt30[MMU_TTREG_TT0]; 3785 m->tt1 = mmu_tt30[MMU_TTREG_TT1]; 3786 } 3787 #endif 3788 3789 return m->ram_segs; 3790 } 3791 3792 /***************************** PMAP BOOTSTRAP ********************************/ 3793 3794 /* 3795 * pmap_pa_has_static_mapping: 3796 * 3797 * Returns true if the specified PA (and length) has a static 3798 * mapping, and what that mapping is and its properties. 3799 */ 3800 bool 3801 pmap_pa_has_static_mapping(paddr_t pa, size_t len, vm_prot_t prot, 3802 vaddr_t *vap, int *flagsp) 3803 { 3804 #if defined(M68030) || defined(M68040) || defined(M68060) 3805 /* First, check for Transparent Translation. */ 3806 if (cputype >= CPU_68030 && 3807 mmu_range_is_tt(pa, len, prot, flagsp)) { 3808 *vap = pa; 3809 return true; 3810 } 3811 #endif 3812 3813 const struct pmap_bootmap *pmbm = machine_bootmap; 3814 paddr_t lastpg = m68k_btop(pa + (len - 1)); 3815 paddr_t firstpg = m68k_btop(pa); 3816 paddr_t tfirst, tlast; 3817 vaddr_t tblva; 3818 bool need_write = !!(prot & UVM_PROT_WRITE); 3819 3820 for (; pmbm->pmbm_vaddr != (vaddr_t)-1; pmbm++) { 3821 if (pmbm->pmbm_flags & (PMBM_F_VAONLY|PMBM_F_KEEPOUT)) { 3822 continue; 3823 } 3824 if (pmbm->pmbm_flags & PMBM_F_FIXEDVA) { 3825 tblva = pmbm->pmbm_vaddr; 3826 } else { 3827 tblva = *pmbm->pmbm_vaddr_ptr; 3828 } 3829 tfirst = m68k_btop(pmbm->pmbm_paddr); 3830 tlast = m68k_btop(pmbm->pmbm_paddr + (pmbm->pmbm_size - 1)); 3831 3832 if (firstpg >= tfirst && lastpg <= tlast) { 3833 if (need_write && 3834 (pmbm->pmbm_flags & PMBM_F_RO) != 0) { 3835 return false; 3836 } 3837 *vap = tblva + (pa - pmbm->pmbm_paddr); 3838 if (pmbm->pmbm_flags & PMBM_F_CI) { 3839 *flagsp = PMAP_NOCACHE; 3840 } else { 3841 *flagsp = 0; 3842 } 3843 return true; 3844 } 3845 } 3846 3847 return false; 3848 } 3849 3850 /* 3851 * pmap_va_is_static_mapping: 3852 * 3853 * Returns true if the specified VA (and length) is a static 3854 * mapping. 3855 */ 3856 bool 3857 pmap_va_is_static_mapping(vaddr_t va, size_t len) 3858 { 3859 #if defined(M68030) || defined(M68040) || defined(M68060) 3860 int map_flags; 3861 3862 /* First, check for Transparent Translation. */ 3863 if (cputype >= CPU_68030 && 3864 (mmu_range_is_tt(va, len, UVM_PROT_READ, &map_flags) || 3865 mmu_range_is_tt(va, len, UVM_PROT_WRITE, &map_flags) || 3866 mmu_range_is_tt(va, len, UVM_PROT_EXEC, &map_flags))) { 3867 return true; 3868 } 3869 #endif 3870 3871 const struct pmap_bootmap *pmbm = machine_bootmap; 3872 vaddr_t lastpg = m68k_btop(va + (len - 1)); 3873 vaddr_t firstpg = m68k_btop(va); 3874 vaddr_t tblva, tfirst, tlast; 3875 for (; pmbm->pmbm_vaddr != (vaddr_t)-1; pmbm++) { 3876 if (pmbm->pmbm_flags & (PMBM_F_VAONLY|PMBM_F_KEEPOUT)) { 3877 continue; 3878 } 3879 if (pmbm->pmbm_flags & PMBM_F_FIXEDVA) { 3880 tblva = pmbm->pmbm_vaddr; 3881 } else { 3882 tblva = *pmbm->pmbm_vaddr_ptr; 3883 } 3884 tfirst = m68k_btop(tblva); 3885 tlast = m68k_btop(tblva + (pmbm->pmbm_size - 1)); 3886 3887 if (firstpg >= tfirst && lastpg <= tlast) { 3888 return true; 3889 } 3890 } 3891 3892 return false; 3893 } 3894 3895 /* 3896 * The kernel virtual address space layout that this implementation is tuned 3897 * for assumes that KVA space begins at $0000.0000, that the static kernel 3898 * image (text/data/bss, etc.) resides at or near the bottom of this space, 3899 * and that all additional KVA that's mapped by PTEs grows upwards from there. 3900 * 3901 * Regions mapped by Transparent Translation registers (68030 and up) 3902 * are assumed to lie beyond where the KVA space is expected to grow. When 3903 * we encounter these regions in the machine_bootmap[] (represented by a 3904 * KEEPOUT entry), we clamp the maximum KVA to prevent its growth into that 3905 * region. The TT mechanism is not terribly precise, and only supports 3906 * VA==PA mappings, so it's only really suitable for device regions that 3907 * are in the upper reaches of the physical address space (at or beyond 1GB 3908 * or so). 3909 * 3910 * This implementation certainly could be adjusted to work with other address 3911 * space layouts, but the assumption asserted here is a bit baked-in. 3912 */ 3913 __CTASSERT(VM_MIN_KERNEL_ADDRESS == 0); 3914 3915 /* 3916 * The virtual kernel PTE array covers the entire 4GB kernel supervisor 3917 * address space, but is sparsely populated. The amount of VA space required 3918 * for this linear array is: 3919 * 3920 * (4GB / PAGE_SIZE) * sizeof(pt_entry_t) 3921 * -or- 3922 * 4KB: 4MB (1024 pages) 3923 * 8KB: 2MB (512 pages) 3924 * 3925 * To avoid doing 64-bit math, we calculate it like so: 3926 * 3927 * ((0xffffffff >> PAGE_SHIFT) + 1) * sizeof(pt_entry_t) 3928 * 3929 * The traditional name for this virtual array is "Sysmap". 3930 */ 3931 #define SYSMAP_VA_SIZE (((0xffffffffU >> PAGE_SHIFT) + 1) * sizeof(pt_entry_t)) 3932 3933 /* 3934 * In the Hibler/Utah pmap, the kernel PTE array was placed right near 3935 * the very top of the kernel virtual address space. This was because 3936 * of the hp300's unique physical memory arrangement: the last page of 3937 * memory is always located at PA $FFFF.F000 and the physical address 3938 * of the beginning of RAM varied based on the RAM size. This meant that 3939 * VA $FFFF.F000 is a convenient place to map the RAM VA==PA, making 3940 * transition between "MMU off" and "MMU on" (and vice versa) easier. 3941 * Since VA $FFFF.F000 was already going to be mapped, it made sense to 3942 * put something else along side of it in order to minimize waste in 3943 * PT pages. 3944 * 3945 * As noted above, this implementation is tuned for a growing-from-0 3946 * virtual space layout. However, we have a special case for this 3947 * particular requirement: if a platform defines SYSMAP_VA, then we 3948 * will assume it is as a high address, place the kernel PTE array at 3949 * that KVA, and ensure sufficient page tables to map from that VA until 3950 * the very end of the 4GB supervisor address space. These tables will 3951 * be allocated before the machine_bootmap[] is processed to map physical 3952 * addresses, thus allowing the machine_bootmap[] use it to map physical 3953 * addresses into one of these high virtual addresses if necessary. The 3954 * beginning of this region will also serve to clamp the maximum kernel 3955 * virtual address, in the same way as a KEEPOUT region in machine_bootmap[]. 3956 * 3957 * For reference, the traditional hp300 definition is: 3958 * 3959 * #define SYSMAP_VA ((vaddr_t)(0-PAGE_SIZE*NPTEPG*2)) 3960 * 3961 * ...and because the hp300 always used a 4KB page size (restriction 3962 * of HP MMU), this is: 0 - 4096*1024*2 3963 * -> 0 - 8388608 (8MB) 3964 * -> $FF80.0000 3965 * 3966 * Unfortunately (for the hp300), this means 2 PT pages for the top of 3967 * the address space (in the 2-level case), but that's unavoidable anyway 3968 * because of the last page being a separate mapping and the kernel PTE 3969 * array needs 4MB of space on its own. 3970 */ 3971 3972 static vaddr_t lwp0uarea; 3973 char * vmmap; 3974 3975 /* XXX Doesn't belong here. */ 3976 paddr_t avail_start; /* PA of first available physical page */ 3977 paddr_t avail_end; /* PA of last available physical page */ 3978 3979 extern char * kernel_text; 3980 extern char * etext; 3981 extern void * msgbufaddr; 3982 3983 /* 3984 * pmap_bootstrap1: 3985 * 3986 * Phase 1 of bootstrapping virtual memory. This is called before 3987 * the MMU is enabled to set up the initial kernel MMU tables and 3988 * allocate other important data structures. 3989 * 3990 * Because the MMU has not yet been turned on, and we don't know if 3991 * we're running VA==PA, we have to manually relocate all global 3992 * symbol references. 3993 * 3994 * Arguments: nextpa Physical address immediately 3995 * following the kernel / symbols / 3996 * etc. This will be page-rounded 3997 * before use. 3998 * 3999 * reloff VA<->PA relocation offset 4000 * 4001 * Returns: nextpa Updated value after all of the 4002 * allocations performed. 4003 */ 4004 paddr_t __attribute__((no_instrument_function)) 4005 pmap_bootstrap1(paddr_t nextpa, paddr_t reloff) 4006 { 4007 paddr_t lwp0upa, stnext_endpa, stnext_pa; 4008 paddr_t pa, kernimg_endpa, kern_lev1pa; 4009 vaddr_t va, nextva, kern_lev1va; 4010 pt_entry_t *pte, *epte; 4011 const struct pmap_bootmap *pmbm; 4012 int entry_count = 0; 4013 4014 #ifdef SYSMAP_VA 4015 #define VA_RANGE_DEFAULT 0 4016 #define VA_RANGE_KPTES 1 4017 #define INIT_NRANGES 2 4018 #else 4019 #define VA_RANGE_DEFAULT 0 4020 #define VA_RANGE_KPTES 0 4021 #define INIT_NRANGES 1 4022 #endif 4023 #define MAX_RANGES 8 4024 4025 struct va_range { 4026 vaddr_t start_va; 4027 vaddr_t end_va; 4028 paddr_t start_ptp; 4029 paddr_t end_ptp; 4030 } va_ranges[MAX_RANGES], *var; 4031 int nranges = INIT_NRANGES, r; 4032 4033 #define VA_IN_RANGE(va, var) \ 4034 ((va) >= (var)->start_va && \ 4035 ((va) < (var)->end_va || (var)->end_va == 0)) 4036 4037 #define VA_PTE_BASE(va, var) \ 4038 (&((pt_entry_t *) \ 4039 PMAP_BOOTSTRAP_RELOC_PA((var)->start_ptp))[ \ 4040 m68k_btop((va) - (var)->start_va)]) 4041 4042 #define RELOC(v, t) *((t *)PMAP_BOOTSTRAP_RELOC_GLOB(&(v))) 4043 4044 /* Record the relocation offset for kernel crash dumps. */ 4045 RELOC(kernel_reloc_offset, paddr_t) = reloff; 4046 4047 /* 4048 * First determination we have to make is our configuration: 4049 * Are we using a 2-level or 3-level table? For the purposes 4050 * of bootstrapping the kernel, it's "68040-class" and "other", 4051 * the former getting the 3-level table. 4052 */ 4053 const bool is_68040_class = RELOC(mmutype, int) == MMU_68040 || 4054 RELOC(mmutype, int) == MMU_68060; 4055 const bool use_3l = is_68040_class; 4056 4057 /* 4058 * Based on MMU class, figure out what the constant values of 4059 * segment / page table entries look like. 4060 * 4061 * See pmap_pte_proto_init(). 4062 */ 4063 pt_entry_t proto_ro_pte; /* read-only */ 4064 pt_entry_t proto_rw_pte; /* read-write */ 4065 pt_entry_t proto_rw_cwt_pte; /* read-write, cache-write-through */ 4066 pt_entry_t proto_rw_ci_pte; /* read-write, cache-inhibited */ 4067 pt_entry_t proto_ste; 4068 4069 if (is_68040_class) { 4070 proto_ro_pte = PTE_VALID|PTE_WIRED|PTE_WP|PTE40_CM_WT; 4071 proto_rw_pte = PTE_VALID|PTE_WIRED |PTE40_CM_CB; 4072 proto_rw_cwt_pte = PTE_VALID|PTE_WIRED |PTE40_CM_WT; 4073 proto_rw_ci_pte = PTE_VALID|PTE_WIRED |PTE40_CM_NC_SER; 4074 } else { 4075 proto_ro_pte = PTE_VALID|PTE_WIRED|PTE_WP; 4076 proto_rw_pte = PTE_VALID|PTE_WIRED; 4077 proto_rw_cwt_pte = PTE_VALID|PTE_WIRED; 4078 proto_rw_ci_pte = PTE_VALID|PTE_WIRED |PTE51_CI; 4079 } 4080 proto_ste = DTE51_U | DT51_SHORT; 4081 4082 /* 4083 * Allocate some important fixed virtual (and physical) addresses. 4084 * We use the sum total of this initial mapped kernel space to 4085 * determine how many initial kernel PT pages to allocate. The 4086 * things that consume physical space will come first, and the 4087 * virtual-space-{only,mostly} things come at the end. 4088 * 4089 * lwp0upa lwp0 u-area USPACE (p) 4090 * lwp0uarea (v) 4091 * 4092 * Sysseg_pa kernel lev1map PAGE_SIZE (p) 4093 * kernel_lev1map PAGE_SIZE (v, ci) 4094 * 4095 * ^^^^ end of simple relocation region ^^^^ 4096 * 4097 * null_segtab_pa null segtab PAGE_SIZE (p) 4098 * 4099 * tmpmap_srcva temp map, src PAGE_SIZE (v) 4100 * tmpmap_dstva temp map, dst PAGE_SIZE (v) 4101 * 4102 * vmmap ya tmp map PAGE_SIZE (v) 4103 * 4104 * msgbufaddr kernel msg buf round_page(MSGBUFSIZE) (v) 4105 * 4106 * kernel_ptes kernel PTEs SYSMAP_VA_SIZE (v, ci) 4107 * (see comments above) 4108 * 4109 * When we allocate the kernel lev1map, for the 2-level 4110 * configuration, there is no inner segment tables to allocate, 4111 * the leaf PT pages get poked directly into the level-1 table. 4112 * 4113 * In the 3-level configuration, to map all of the leaf tables, 4114 * inner segment table pages are allocated as necessary. We 4115 * first take those tables from the page containing the level-1 4116 * table, and allocate additional pages as necessary. 4117 */ 4118 4119 nextpa = m68k_round_page(nextpa); 4120 nextva = PMAP_BOOTSTRAP_PA_TO_VA(nextpa); 4121 4122 /* 4123 * nextpa now represents the end of the loaded kernel image. 4124 * This includes the .data + .bss segments, the debugger symbols, 4125 * and any other ancillary data loaded after the kernel. 4126 * 4127 * N.B. This represents the start of our dynamic memory allocation, 4128 * which will be referenced below when we zero the memory we've 4129 * allocated. 4130 */ 4131 kernimg_endpa = nextpa; 4132 4133 /* 4134 * lwp0 u-area. We allocate it here, and finish setting it 4135 * up in pmap_bootstrap2(). 4136 */ 4137 lwp0upa = nextpa; 4138 nextpa += USPACE; 4139 RELOC(lwp0uarea, vaddr_t) = nextva; 4140 nextva += USPACE; 4141 4142 size_t nstpages = 0; 4143 4144 /* kernel level-1 map */ 4145 RELOC(Sysseg_pa, paddr_t) = kern_lev1pa = nextpa; 4146 nextpa += PAGE_SIZE; 4147 RELOC(kernel_lev1map, vaddr_t) = kern_lev1va = nextva; 4148 nextva += PAGE_SIZE; 4149 nstpages++; 4150 4151 /* This is the end of the simple relocation region. */ 4152 RELOC(kernel_reloc_end, vaddr_t) = nextva; 4153 4154 /* 4155 * For 3-level configs, we now have space to allocate 4156 * inner segment tables. 4157 */ 4158 stnext_pa = kern_lev1pa + TBL40_L1_SIZE; 4159 stnext_endpa = m68k_round_page(stnext_pa); 4160 4161 /* null segment table */ 4162 #ifdef NULL_SEGTAB_PA 4163 RELOC(null_segtab_pa, paddr_t) = (paddr_t)NULL_SEGTAB_PA; 4164 #else 4165 RELOC(null_segtab_pa, paddr_t) = nextpa; 4166 nextpa += PAGE_SIZE; 4167 #endif 4168 4169 /* pmap temporary map addresses */ 4170 RELOC(pmap_tmpmap_srcva, vaddr_t) = nextva; 4171 nextva += PAGE_SIZE; 4172 RELOC(pmap_tmpmap_dstva, vaddr_t) = nextva; 4173 nextva += PAGE_SIZE; 4174 4175 /* vmmap temporary map address */ 4176 RELOC(vmmap, vaddr_t) = nextva; 4177 nextva += PAGE_SIZE; 4178 4179 /* kernel message buffer */ 4180 RELOC(msgbufaddr, vaddr_t) = nextva; 4181 nextva += m68k_round_page(MSGBUFSIZE); 4182 4183 /* Kernel PTE array. */ 4184 #ifdef SYSMAP_VA 4185 if ((vaddr_t)SYSMAP_VA < RELOC(kernel_virtual_max, vaddr_t)) { 4186 RELOC(kernel_virtual_max, vaddr_t) = (vaddr_t)SYSMAP_VA; 4187 } 4188 RELOC(kernel_ptes, vaddr_t) = (vaddr_t)SYSMAP_VA; 4189 va_ranges[VA_RANGE_KPTES].start_va = (vaddr_t)SYSMAP_VA; 4190 va_ranges[VA_RANGE_KPTES].end_va = 0; /* end of the address space */ 4191 #else 4192 RELOC(kernel_ptes, vaddr_t) = nextva; 4193 nextva += SYSMAP_VA_SIZE; 4194 #endif /* SYSMAP_VA */ 4195 4196 /* 4197 * Look through the machine_bootmap looking for any non-fixed 4198 * VAs we need to allocate. These will go into the default 4199 * VA range. 4200 */ 4201 pmbm = (const struct pmap_bootmap *) 4202 PMAP_BOOTSTRAP_RELOC_GLOB(machine_bootmap); 4203 for (; pmbm->pmbm_vaddr != (vaddr_t)-1; pmbm++) { 4204 if (pmbm->pmbm_size == 0) { 4205 continue; 4206 } 4207 if (pmbm->pmbm_flags & (PMBM_F_FIXEDVA | PMBM_F_KEEPOUT)) { 4208 va = m68k_trunc_page(pmbm->pmbm_vaddr); 4209 if (va < RELOC(kernel_virtual_max, vaddr_t)) { 4210 RELOC(kernel_virtual_max, vaddr_t) = va; 4211 } 4212 } else { 4213 *(vaddr_t *) 4214 PMAP_BOOTSTRAP_RELOC_GLOB(pmbm->pmbm_vaddr_ptr) = 4215 nextva; 4216 nextva += m68k_round_page(pmbm->pmbm_size); 4217 } 4218 } 4219 4220 /* UVM-managed kernel virtual starts here. */ 4221 RELOC(kernel_virtual_start, vaddr_t) = nextva; 4222 4223 /* 4224 * Allocate enough PT pages to map all of physical memory. 4225 * This should be sufficient to prevent pmap_growkernel() 4226 * from having to do any work before the VM system is set 4227 * up. 4228 * 4229 * XXX mac68k also relies on being able to map the last page 4230 * XXX of RAM VA==PA for the mmu-switchoff dance. Unlike hp300, 4231 * XXX this is not at a fixed location. However, RAM generally 4232 * XXX starts at $0000.0000 on Macs, so this calculation should 4233 * XXX be sufficient to ensure there is a PTE available for this 4234 * XXX purpose. 4235 * XXX TODO: Provide a way for cpu_startup() on mac68k to assert 4236 * XXX this (export kernel_virtual_end?). 4237 */ 4238 nextva += RELOC(physmem, psize_t) << PAGE_SHIFT; 4239 nextva = pmap_round_ptpage(nextva); 4240 if (nextva > RELOC(kernel_virtual_max, vaddr_t) || 4241 nextva < RELOC(kernel_virtual_start, vaddr_t)) { 4242 /* clamp it. */ 4243 nextva = RELOC(kernel_virtual_max, vaddr_t); 4244 } 4245 4246 /* 4247 * This marks the end of UVM-managed kernel virtual space, 4248 * until such time as pmap_growkernel() is called to expand 4249 * it. 4250 */ 4251 va_ranges[VA_RANGE_DEFAULT].start_va = 4252 pmap_trunc_ptpage(VM_MIN_KERNEL_ADDRESS); 4253 va_ranges[VA_RANGE_DEFAULT].end_va = 4254 pmap_round_ptpage(nextva); 4255 RELOC(kernel_virtual_end, vaddr_t) = nextva; 4256 4257 /* 4258 * Now that we know the end of the default range, look through 4259 * the machine_bootmap to see if there are any fixed VA ranges 4260 * we need to allocate static PTs for. 4261 */ 4262 pmbm = (const struct pmap_bootmap *) 4263 PMAP_BOOTSTRAP_RELOC_GLOB(machine_bootmap); 4264 for (; pmbm->pmbm_vaddr != (vaddr_t)-1; pmbm++) { 4265 if (pmbm->pmbm_size == 0) { 4266 continue; 4267 } 4268 if ((pmbm->pmbm_flags & (PMBM_F_FIXEDVA | PMBM_F_KEEPOUT)) != 4269 PMBM_F_FIXEDVA) { 4270 continue; 4271 } 4272 /* 4273 * Found one; see if there's a range already allocated 4274 * for it. 4275 */ 4276 for (r = 0; r < nranges; r++) { 4277 var = &va_ranges[r]; 4278 if (VA_IN_RANGE(pmbm->pmbm_vaddr, var)) { 4279 break; 4280 } 4281 } 4282 if (r < nranges) { 4283 /* Range of PTs already exists. */ 4284 continue; 4285 } 4286 /* 4287 * We need to allocate a range. If we're already at 4288 * the max, I guess we lose. 4289 */ 4290 if (r == MAX_RANGES) { 4291 break; 4292 } 4293 r = nranges++; 4294 va_ranges[r].start_va = pmap_trunc_ptpage(pmbm->pmbm_vaddr); 4295 va_ranges[r].end_va = 4296 pmap_round_ptpage(pmbm->pmbm_vaddr + pmbm->pmbm_size); 4297 } 4298 4299 /* 4300 * Now, compute the number of PT pages required to map the 4301 * required VA ranges and allocate them. 4302 */ 4303 size_t nptpages, total_ptpages = 0; 4304 for (r = 0; r < nranges; r++) { 4305 var = &va_ranges[r]; 4306 nptpages = (var->end_va - var->start_va) / PTPAGEVASZ; 4307 var->start_ptp = nextpa; 4308 nextpa += nptpages * PAGE_SIZE; 4309 var->end_ptp = nextpa; 4310 total_ptpages += nptpages; 4311 } 4312 4313 #ifdef PMAP_MACHINE_CHECK_BOOTSTRAP_ALLOCATIONS 4314 /* 4315 * Right here, the old mac68k Utah pmap_bootstrap1() has a 4316 * check to see if the kernel + bootstrap allocations fit 4317 * within one of the memory segments mapped by the loader. 4318 * This is a hook to accommodate that requirement. 4319 */ 4320 void (*alloc_checkfn)(paddr_t, paddr_t) = (void *) 4321 PMAP_BOOTSTRAP_RELOC_GLOB(pmap_machine_check_bootstrap_allocations); 4322 (*alloc_checkfn)(nextpa, reloff); 4323 #endif 4324 4325 /* 4326 * The bulk of the dynamic memory allocation is done (there 4327 * may be more below if we have to allocate more inner segment 4328 * table pages, but we'll burn that bridge when we come to it). 4329 * 4330 * Zero out all of these freshly-allocated pages. 4331 */ 4332 pte = (pt_entry_t *)PMAP_BOOTSTRAP_RELOC_PA(kernimg_endpa); 4333 epte = (pt_entry_t *)PMAP_BOOTSTRAP_RELOC_PA(nextpa); 4334 while (pte < epte) { 4335 *pte++ = 0; 4336 } 4337 4338 /* 4339 * Ok, let's get to mapping stuff! Almost everything is in 4340 * the default VA range. 4341 */ 4342 var = &va_ranges[VA_RANGE_DEFAULT]; 4343 4344 /* 4345 * Kernel text - read-only. 4346 * 4347 * ...that is, unless, a platform as some quirky requirement 4348 * (hello mac68k!). This hook lets a platform specify an 4349 * alternate proto PTE for the kernel text (in the mac68k case, 4350 * it will be read/write write-through-cacheable). Once the 4351 * kernel is up and running on its own mappings, machine-specific 4352 * code can perform any fixups as necessary. 4353 */ 4354 #ifndef PMAP_BOOTSTRAP_TEXT_PROTO_PTE 4355 #define PMAP_BOOTSTRAP_TEXT_PROTO_PTE proto_ro_pte 4356 #endif 4357 pa = PMAP_BOOTSTRAP_VA_TO_PA(m68k_trunc_page(&kernel_text)); 4358 pte = VA_PTE_BASE(&kernel_text, var); 4359 epte = VA_PTE_BASE(&etext, var); 4360 while (pte < epte) { 4361 *pte++ = PMAP_BOOTSTRAP_TEXT_PROTO_PTE | pa; 4362 pa += PAGE_SIZE; 4363 entry_count++; 4364 } 4365 4366 /* Remainder of kernel image - read-write. */ 4367 epte = VA_PTE_BASE(PMAP_BOOTSTRAP_PA_TO_VA(kernimg_endpa), var); 4368 while (pte < epte) { 4369 *pte++ = proto_rw_pte | pa; 4370 pa += PAGE_SIZE; 4371 entry_count++; 4372 } 4373 4374 /* lwp0 u-area - read-write. */ 4375 pa = lwp0upa; 4376 pte = VA_PTE_BASE(RELOC(lwp0uarea, vaddr_t), var); 4377 epte = VA_PTE_BASE(RELOC(lwp0uarea, vaddr_t) + USPACE, var); 4378 while (pte < epte) { 4379 *pte++ = proto_rw_pte | pa; 4380 pa += PAGE_SIZE; 4381 entry_count++; 4382 } 4383 4384 /* Kernel lev1map - read-write, cache-inhibited. */ 4385 pte = VA_PTE_BASE(kern_lev1va, var); 4386 *pte = proto_rw_ci_pte | kern_lev1pa; 4387 entry_count++; 4388 4389 /* 4390 * Kernel leaf PT pages - read-write, cache-inhibited. 4391 * 4392 * These will be in a different VA range if the machine 4393 * defines SYSMAP_VA. 4394 */ 4395 va = RELOC(kernel_ptes, vaddr_t); 4396 pt_entry_t *kptes = (pt_entry_t *)va; 4397 struct va_range *kpte_var = &va_ranges[VA_RANGE_KPTES]; 4398 4399 for (r = 0; r < nranges; r++) { 4400 var = &va_ranges[r]; 4401 va = (vaddr_t)(&kptes[m68k_btop(var->start_va)]); 4402 pte = VA_PTE_BASE(va, kpte_var); 4403 for (pa = var->start_ptp; pa < var->end_ptp; pa += PAGE_SIZE) { 4404 *pte++ = proto_rw_ci_pte | pa; 4405 entry_count++; 4406 } 4407 } 4408 4409 /* 4410 * Now perform any machine-specific mappings at VAs 4411 * allocated earlier. 4412 */ 4413 pmbm = (const struct pmap_bootmap *) 4414 PMAP_BOOTSTRAP_RELOC_GLOB(machine_bootmap); 4415 for (; pmbm->pmbm_vaddr != (vaddr_t)-1; pmbm++) { 4416 pt_entry_t proto; 4417 4418 if (pmbm->pmbm_size == 0 || 4419 (pmbm->pmbm_flags & (PMBM_F_VAONLY | PMBM_F_KEEPOUT))) { 4420 continue; 4421 } 4422 if (pmbm->pmbm_flags & PMBM_F_FIXEDVA) { 4423 va = pmbm->pmbm_vaddr; 4424 } else { 4425 va = *(vaddr_t *) 4426 PMAP_BOOTSTRAP_RELOC_GLOB(pmbm->pmbm_vaddr_ptr); 4427 } 4428 for (r = 0; r < nranges; r++) { 4429 var = &va_ranges[r]; 4430 if (VA_IN_RANGE(va, var)) { 4431 break; 4432 } 4433 } 4434 pa = pmbm->pmbm_paddr; 4435 pte = VA_PTE_BASE(va, var); 4436 switch (pmbm->pmbm_flags & (PMBM_F_CI|PMBM_F_CWT|PMBM_F_RO)) { 4437 case PMBM_F_CI|PMBM_F_RO: 4438 proto = proto_rw_ci_pte | PTE_WP; 4439 break; 4440 case PMBM_F_CWT: 4441 proto = proto_rw_cwt_pte; 4442 break; 4443 case PMBM_F_CI: 4444 proto = proto_rw_ci_pte; 4445 break; 4446 case PMBM_F_RO: 4447 proto = proto_ro_pte; 4448 break; 4449 default: 4450 proto = proto_rw_pte; 4451 break; 4452 } 4453 for (vsize_t size = m68k_round_page(pmbm->pmbm_size); 4454 size != 0; 4455 va += PAGE_SIZE, pa += PAGE_SIZE, size -= PAGE_SIZE) { 4456 *pte++ = proto | pa; 4457 entry_count++; 4458 } 4459 } 4460 4461 /* 4462 * Now that all of the individual VAs are mapped in the leaf 4463 * tables, it's time to link those tables into the segment 4464 * table. 4465 * 4466 * For the 2-level case, this is trivial. For the 3-level 4467 * case, we will have to allocate inner segment tables. 4468 */ 4469 for (r = 0; r < nranges; r++) { 4470 var = &va_ranges[r]; 4471 if (use_3l) { 4472 pt_entry_t *stes, *stes1 = (pt_entry_t *) 4473 PMAP_BOOTSTRAP_RELOC_PA(kern_lev1pa); 4474 for (va = var->start_va, pa = var->start_ptp; 4475 pa < var->end_ptp; 4476 va += NBSEG3L, pa += TBL40_L3_SIZE) { 4477 unsigned int ri = LA40_RI(va); 4478 if ((stes1[ri] & UTE40_RESIDENT) == 0) { 4479 /* 4480 * Level-2 table for this segment 4481 * needed. 4482 */ 4483 if (stnext_pa == stnext_endpa) { 4484 /* 4485 * No more slots left in the 4486 * last page we allocated for 4487 * segment tables. Grab 4488 * another one. 4489 */ 4490 stnext_pa = nextpa; 4491 nextpa += PAGE_SIZE; 4492 stnext_endpa = nextpa; 4493 nstpages++; 4494 #ifdef PMAP_MACHINE_CHECK_BOOTSTRAP_ALLOCATIONS 4495 (*alloc_checkfn)(nextpa, 4496 reloff); 4497 #endif 4498 /* 4499 * Zero out the new inner 4500 * segment table page. 4501 */ 4502 pte = (pt_entry_t *) 4503 PMAP_BOOTSTRAP_RELOC_PA( 4504 stnext_pa); 4505 epte = (pt_entry_t *) 4506 PMAP_BOOTSTRAP_RELOC_PA( 4507 stnext_endpa); 4508 while (pte < epte) { 4509 *pte++ = 0; 4510 } 4511 } 4512 stes1[ri] = proto_ste | stnext_pa; 4513 stnext_pa += TBL40_L2_SIZE; 4514 } 4515 stes = (pt_entry_t *) 4516 PMAP_BOOTSTRAP_RELOC_PA( 4517 stes1[ri] & UTE40_PTA); 4518 stes[LA40_PI(va)] = proto_ste | pa; 4519 } 4520 } else { 4521 pt_entry_t *stes = (pt_entry_t *) 4522 PMAP_BOOTSTRAP_RELOC_PA(kern_lev1pa); 4523 for (va = var->start_va, pa = var->start_ptp; 4524 pa < var->end_ptp; 4525 va += NBSEG2L, pa += PAGE_SIZE) { 4526 stes[LA2L_RI(va)] = proto_ste | pa; 4527 } 4528 } 4529 } 4530 4531 /* Instrumentation. */ 4532 RELOC(pmap_nkptpages_initial_ev.ev_count32, uint32_t) = 4533 RELOC(pmap_nkptpages_current_ev.ev_count32, uint32_t) = total_ptpages; 4534 RELOC(pmap_nkstpages_initial_ev.ev_count32, uint32_t) = 4535 RELOC(pmap_nkstpages_current_ev.ev_count32, uint32_t) = nstpages; 4536 4537 /* 4538 * Record the number of wired mappings we created above 4539 * in the kernel pmap stats. 4540 */ 4541 RELOC(kernel_pmap_store.pm_stats.resident_count, long) = entry_count; 4542 RELOC(kernel_pmap_store.pm_stats.wired_count, long) = entry_count; 4543 4544 /* 4545 * Stash any left-over segment table space for use by 4546 * pmap_growkernel() later. 4547 */ 4548 RELOC(kernel_stnext_pa, paddr_t) = stnext_pa; 4549 RELOC(kernel_stnext_endpa, paddr_t) = stnext_endpa; 4550 4551 return nextpa; 4552 } 4553 4554 /* 4555 * PAGE_SIZE should always evaluate to a compile-time constant in this 4556 * context. 4557 */ 4558 __CTASSERT(PAGE_SIZE != 0); 4559 4560 /* 4561 * pmap_bootstrap2: 4562 * 4563 * Phase 2 of bootstrapping virtual memory. This is called after 4564 * the MMU has been enabled to finish setting up run-time-computed 4565 * global pmap data, plus the lwp0 u-area, curlwp, and curpcb. 4566 * 4567 * Returns the new kernel %sp value for lwp0. 4568 */ 4569 void * 4570 pmap_bootstrap2(void) 4571 { 4572 /* Setup the MMU class; needed before anything else. */ 4573 pmap_mmuclass_init(); 4574 4575 /* Early low-level UVM initialization. */ 4576 uvmexp.pagesize = PAGE_SIZE; 4577 uvm_md_init(); 4578 4579 /* Initialize prototype PTEs; needed before anything else is mapped. */ 4580 pmap_pte_proto_init(); 4581 4582 /* Initialize the kernel pmap. */ 4583 pmap_pinit(pmap_kernel(), Sysseg_pa); 4584 4585 /* Initialize lwp0 u-area, curlwp, and curpcb. */ 4586 memset((void *)lwp0uarea, 0, USPACE); 4587 uvm_lwp_setuarea(&lwp0, lwp0uarea); 4588 curlwp = &lwp0; 4589 curpcb = lwp_getpcb(&lwp0); 4590 4591 /* Create a fake exception frame so that cpu_lwp_fork() can copy it. */ 4592 struct trapframe *tf = (struct trapframe *)(lwp0uarea + USPACE) - 1; 4593 tf->tf_sr = PSL_USER; 4594 lwp0.l_md.md_regs = (int *)tf; 4595 4596 /* 4597 * Initialize the source/destination control registers for 4598 * movs. 4599 */ 4600 setsfc(FC_USERD); 4601 setdfc(FC_USERD); 4602 4603 return tf; 4604 } 4605 4606 /***************************** PMAP DEBUGGING ********************************/ 4607 4608 #ifdef PMAP_DEBUG 4609 4610 void pmap_test_mod_ref(void); 4611 4612 void 4613 pmap_test_mod_ref(void) 4614 { 4615 struct vm_page *pg = pmap_page_alloc(false/*nowait*/); 4616 paddr_t pa = VM_PAGE_TO_PHYS(pg); 4617 vaddr_t va = (vaddr_t)vmmap; 4618 volatile int *loc = (volatile int *)va; 4619 int val; 4620 bool mod, ref; 4621 bool exp_mod, exp_ref; 4622 4623 /* Initialize page and mod/ref state to pristine. */ 4624 pmap_zero_page(pa); 4625 pmap_clear_modify(pg); 4626 pmap_clear_reference(pg); 4627 4628 mod = pmap_is_modified(pg); 4629 ref = pmap_is_referenced(pg); 4630 exp_mod = false; 4631 exp_ref = false; 4632 printf("%s: validating pristine page: mod=%d(%d) ref=%d(%d) (%s)\n", 4633 __func__, 4634 mod, exp_mod, ref, exp_ref, 4635 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 4636 4637 /* Enter non-seeded R/W mapping. */ 4638 pmap_enter(pmap_kernel(), va, pa, UVM_PROT_ALL, 0); 4639 4640 mod = pmap_is_modified(pg); 4641 ref = pmap_is_referenced(pg); 4642 exp_mod = false; 4643 exp_ref = false; 4644 printf("%s: enter(ALL, 0): mod=%d(%d) ref=%d(%d) (%s)\n", 4645 __func__, 4646 mod, exp_mod, ref, exp_ref, 4647 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 4648 4649 /* reference page */ 4650 val = *loc; 4651 4652 mod = pmap_is_modified(pg); 4653 ref = pmap_is_referenced(pg); 4654 exp_mod = false; 4655 exp_ref = true; 4656 printf("%s: ref val=%d: mod=%d(%d) ref=%d(%d) (%s)\n", 4657 __func__, 4658 val, 4659 mod, exp_mod, ref, exp_ref, 4660 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 4661 4662 /* validate clear behavior. */ 4663 exp_mod = mod; 4664 exp_ref = ref; 4665 mod = pmap_clear_modify(pg); 4666 ref = pmap_clear_reference(pg); 4667 printf("%s: checking clear 1: mod=%d(%d) ref=%d(%d) (%s)\n", 4668 __func__, 4669 mod, exp_mod, ref, exp_ref, 4670 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 4671 4672 mod = pmap_is_modified(pg); 4673 ref = pmap_is_referenced(pg); 4674 exp_mod = false; 4675 exp_ref = false; 4676 printf("%s: checking clear 2: mod=%d(%d) ref=%d(%d) (%s)\n", 4677 __func__, 4678 mod, exp_mod, ref, exp_ref, 4679 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 4680 4681 /* modify page */ 4682 *loc = 0xff; 4683 4684 mod = pmap_is_modified(pg); 4685 ref = pmap_is_referenced(pg); 4686 exp_mod = true; 4687 exp_ref = true; 4688 printf("%s: mod 1: mod=%d(%d) ref=%d(%d) (%s)\n", 4689 __func__, 4690 mod, exp_mod, ref, exp_ref, 4691 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 4692 4693 /* write-protect page */ 4694 pmap_page_protect(pg, UVM_PROT_READ); 4695 4696 mod = pmap_clear_modify(pg); 4697 ref = pmap_clear_reference(pg); 4698 exp_mod = true; 4699 exp_ref = true; 4700 printf("%s: page_protect(READ) mod 2: mod=%d(%d) ref=%d(%d) (%s)\n", 4701 __func__, 4702 mod, exp_mod, ref, exp_ref, 4703 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 4704 4705 mod = pmap_is_modified(pg); 4706 ref = pmap_is_referenced(pg); 4707 exp_mod = false; 4708 exp_ref = false; 4709 printf("%s: checking clear 3: mod=%d(%d) ref=%d(%d) (%s)\n", 4710 __func__, 4711 mod, exp_mod, ref, exp_ref, 4712 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 4713 4714 /* modify page again */ 4715 pmap_enter(pmap_kernel(), va, pa, UVM_PROT_ALL, 0); 4716 *loc = 0xaa; 4717 4718 mod = pmap_is_modified(pg); 4719 ref = pmap_is_referenced(pg); 4720 exp_mod = true; 4721 exp_ref = true; 4722 printf("%s: mod 3: mod=%d(%d) ref=%d(%d) (%s)\n", 4723 __func__, 4724 mod, exp_mod, ref, exp_ref, 4725 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 4726 4727 /* remove all mappings of page */ 4728 pmap_page_protect(pg, UVM_PROT_NONE); 4729 4730 mod = pmap_clear_modify(pg); 4731 ref = pmap_clear_reference(pg); 4732 exp_mod = true; 4733 exp_ref = true; 4734 printf("%s: page_protect(NONE) mod 4: mod=%d(%d) ref=%d(%d) (%s)\n", 4735 __func__, 4736 mod, exp_mod, ref, exp_ref, 4737 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 4738 4739 /* modify page again */ 4740 pmap_enter(pmap_kernel(), va, pa, UVM_PROT_ALL, 0); 4741 *loc = 0xaa; 4742 4743 mod = pmap_is_modified(pg); 4744 ref = pmap_is_referenced(pg); 4745 exp_mod = true; 4746 exp_ref = true; 4747 printf("%s: mod 5: mod=%d(%d) ref=%d(%d) (%s)\n", 4748 __func__, 4749 mod, exp_mod, ref, exp_ref, 4750 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 4751 4752 /* read-protect VA */ 4753 pmap_protect(pmap_kernel(), va, va+PAGE_SIZE, UVM_PROT_READ); 4754 4755 mod = pmap_is_modified(pg); 4756 ref = pmap_is_referenced(pg); 4757 exp_mod = true; 4758 exp_ref = true; 4759 printf("%s: va prot(READ) mod 6: mod=%d(%d) ref=%d(%d) (%s)\n", 4760 __func__, 4761 mod, exp_mod, ref, exp_ref, 4762 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 4763 4764 /* none-protect VA */ 4765 pmap_protect(pmap_kernel(), va, va+PAGE_SIZE, UVM_PROT_NONE); 4766 4767 mod = pmap_is_modified(pg); 4768 ref = pmap_is_referenced(pg); 4769 exp_mod = true; 4770 exp_ref = true; 4771 printf("%s: va prot(NONE) mod 7: mod=%d(%d) ref=%d(%d) (%s)\n", 4772 __func__, 4773 mod, exp_mod, ref, exp_ref, 4774 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 4775 4776 /* Reset the page. */ 4777 pmap_clear_modify(pg); 4778 pmap_clear_reference(pg); 4779 4780 mod = pmap_is_modified(pg); 4781 ref = pmap_is_referenced(pg); 4782 exp_mod = false; 4783 exp_ref = false; 4784 printf("%s: validating pristine page: mod=%d(%d) ref=%d(%d) (%s)\n", 4785 __func__, 4786 mod, exp_mod, ref, exp_ref, 4787 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 4788 4789 printf("%s: verifying mapping does not exist: (%s)\n", 4790 __func__, 4791 pmap_extract(pmap_kernel(), va, NULL) ? "FAIL" : "OK"); 4792 4793 /* Enter R-seeded R/W mapping. */ 4794 pmap_enter(pmap_kernel(), va, pa, UVM_PROT_READ|UVM_PROT_WRITE, 4795 UVM_PROT_READ); 4796 4797 mod = pmap_is_modified(pg); 4798 ref = pmap_is_referenced(pg); 4799 exp_mod = false; 4800 exp_ref = true; 4801 printf("%s: enter(R|W, R): mod=%d(%d) ref=%d(%d) (%s)\n", 4802 __func__, 4803 mod, exp_mod, ref, exp_ref, 4804 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 4805 4806 /* Enter W-seeded R/W mapping. */ 4807 pmap_enter(pmap_kernel(), va, pa, UVM_PROT_READ|UVM_PROT_WRITE, 4808 UVM_PROT_WRITE); 4809 4810 mod = pmap_is_modified(pg); 4811 ref = pmap_is_referenced(pg); 4812 exp_mod = true; 4813 exp_ref = true; 4814 printf("%s: enter(R|W, W): mod=%d(%d) ref=%d(%d) (%s)\n", 4815 __func__, 4816 mod, exp_mod, ref, exp_ref, 4817 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 4818 4819 pmap_clear_modify(pg); 4820 pmap_clear_reference(pg); 4821 4822 mod = pmap_is_modified(pg); 4823 ref = pmap_is_referenced(pg); 4824 exp_mod = false; 4825 exp_ref = false; 4826 printf("%s: validating pristine page: mod=%d(%d) ref=%d(%d) (%s)\n", 4827 __func__, 4828 mod, exp_mod, ref, exp_ref, 4829 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 4830 4831 /* 4832 * Now we're going to reference the page before modifying it 4833 * without any intervening clear. The MMU should update the 4834 * PTE once for the ref, and a second time for the mod, *even 4835 * if the ATC entry is still valid*. 4836 * 4837 * We are going to fetch the PTEs directly to validate the 4838 * MMU's behavior. 4839 */ 4840 pt_entry_t *ptep = pmap_kernel_pte(va); 4841 pt_entry_t ref_pte, mod_pte; 4842 int s = splhigh(); 4843 val = *loc; 4844 ref_pte = pte_load(ptep); 4845 *loc = 0; 4846 mod_pte = pte_load(ptep); 4847 splx(s); 4848 4849 mod = (mod_pte & PTE_M) != 0; 4850 ref = (ref_pte & PTE_U) != 0; 4851 exp_mod = true; 4852 exp_ref = true; 4853 printf("%s: validating MMU behavior: mod=%d(%d) ref=%d(%d) (%s)\n", 4854 __func__, 4855 mod, exp_mod, ref, exp_ref, 4856 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 4857 4858 /* all done. */ 4859 pmap_remove(pmap_kernel(), va, va+PAGE_SIZE); 4860 4861 printf("%s: done\n", __func__); 4862 } 4863 4864 #endif /* PMAP_DEBUG */ 4865
Indexes created Tue Aug 11 00:25:38 UTC 2026