1 /* $NetBSD: pmap_68k.c,v 1.80 2026/10/22 20:48:19 andvar Exp $ */ 2 3 /*- 4 * Copyright (c) 2025 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Pmap module for the Motorola 68851 / 68030 / 68040 / 68060 MMUs. 34 * (...and HP 68851-like MMU.) 35 * 36 * This implementation supports both 2-level and 3-level page table 37 * layouts. The 3-level is mandated by 68040 / 68060, and the 2-level 38 * is mandated by the HP MMU. The 68851 and 68030 can do either, and 39 * for now, the 2-level arrangement is retained for those MMUs, although 40 * eventually we will switch them to the 3-level configuration. 41 * 42 * To support both configurations, page tables are abstracted away from 43 * the page table pages that contain them. The interface pmap operations 44 * operate on "leaf" (page) tables, and only when one of those tables needs 45 * to be allocated or freed, do the differences between the two configurations 46 * need to be dealt with. All of the tables are kept in a red-black tree 47 * that's indexed by their "segment" number (where "segment" is defined as 48 * "the amount of space mapped by a single leaf table"). This avoids having 49 * to burn large amounts of kernel address space to access tables which are 50 * expected to be sparsely-populated. 51 * 52 * In order to reduce the number of tree lookups, the most recently used 53 * leaf table is cached, and the interface contract is such that bulk 54 * operations are allowed to access subsequent PTEs within a given table 55 * (segment) without having to perform another PTE lookup. 56 * 57 * This illustrates the initial table layout for a simple program 58 * (/usr/bin/yes) using the standard m68k address space layout (based 59 * on the historical 4.3BSD-on-hp300 layout, which was itself based on 60 * HP-UX in order to facilitate HP-UX binary compatibility back when 61 * that was considered to be important). This example uses a 4K page 62 * size. 63 * 64 * TEXTADDR is $0000.2000 (not always strictly true, but close enough) 65 * USRSTACK is $FFF0.0000 (grows down, first used page VA is $FFEF.F000) 66 * 67 * (TEXTADDR is $0000.2000 because the linker uses 8K page size for 68 * broader compatibility and keeps the 0-page unmapped so that NULL 69 * pointer dereferences blow up.) 70 * 71 * This is to say: the text / data / heap of this program are in the 72 * bottom 1MB of the address space, and the stack is in the second-from- 73 * the-top 1MB of the address space. 74 * 75 * In the 2-level layout, the level-1 table is 4KB in size, and has 1024 76 * entries. Those 1024 entries together represent the 4GB user address 77 * space, and each entry thus maps a 4MB "segment" by itself pointing to 78 * a level-2 table which themselves are 4KB in size and have 1024 entries 79 * (4MB / 1024 -> 4KB, which is the page size ... convenient!) So, when 80 * our very simple program is loaded, we have a table structure that looks 81 * like this: 82 * 83 * (4KB) 84 * +----------------------+ 85 * | Level-1 | 86 * |0 1023| 87 * +----------------------+ 88 * | | 89 * | | 90 * +---------+ +---------+ 91 * | | 92 * v v 93 * (4KB) (4KB) 94 * +----------------------+ +----------------------+ 95 * | Level-2 | | Level-2 | 96 * | 2 4 | | 767 | 97 * +----------------------+ +----------------------+ 98 * | | | 99 * | +-+ | 100 * v v v 101 * TEXT DATA/bss/heap stack 102 * 103 * As you can see, this requires 3 tables (1 level-1 and 2 level-2). Each 104 * table consumes a full 4KB page, so mapping this address space requires 105 * 3 total pages. 106 * 107 * In the 3-level layout, the level-1 and level-2 tables each contain 128 108 * entries, making them 512 bytes in size. When using 4KB pages, the level-3 109 * tables contain 64 entries, making them 256 bytes in size. 110 * 111 * So, assuming the same address space layout, the 3-level structure looks 112 * like this: 113 * 114 * (512B) 115 * +--------------+ 116 * | Level-1 | 117 * |0 127| 118 * +--------------+ 119 * | | 120 * +---+ +---+ 121 * v v 122 * (512B) (512B) 123 * +--------------+ +--------------+ 124 * | Level-2 | | Level-2 | 125 * |0 | | 123 | 126 * +--------------+ +--------------+ 127 * | | 128 * +---------+ +-----+ 129 * v v 130 * (256B) (256B) 131 * +------------+ +------------+ 132 * | Level-3 | | Level-3 | 133 * | 2 4 | | 63| 134 * +------------+ +------------+ 135 * | | | 136 * | +-+ | 137 * v v v 138 * TEXT DATA/bss/heap stack 139 * 140 * The table allocator has two pools of memory for tables in the 3-level 141 * configuration: one for "segment" tables (always 512 bytes) and one for 142 * "page" or "leaf" tables (256 bytes in size for 4K pages). Pages are 143 * allocated to the pools one at a time, and then the tables are allocated 144 * from the pages. Because of this, we only need two pages, 33% less (!), 145 * than the 2-level configuration to map the same address space. 146 * 147 * There is a cost, however: each access that misses the Address Translation 148 * Cache costs one extra memory cycle in the 3-level configuration. 149 * 150 * LOCKING IN THIS PMAP MODULE: 151 * 152 * MULTIPROCESSING IS NOT SUPPORTED IN THIS PMAP MODULE. Adding support 153 * for it would not be terribly difficult, but there is little value in 154 * doing that work until such time as a multiprocessor m68k machine exists 155 * that NetBSD runs on. 156 * 157 * As such, there is **no** locking performed of any data structures here. 158 * We do actually reap a benefit from this perceived laziness: we do not 159 * have to worry about lock ordering, which means we can take some shortcuts 160 * in some places (especially around pv_entry manipulation). 161 * 162 * THERE IS A CAVEAT, HOWEVER! Because there are no guard rails, we cannot, 163 * under any circumstances, yield the CPU during the critical section of a 164 * pmap operation, as doing so could cause the world to change beneath our 165 * feet, possibly rendering our work, for lack of a better term, "crashy". 166 * Specifically, this means: 167 * 168 * - Adaptive mutexes must not be acquired (e.g. when calling into 169 * other code, e.g. UVM to get a VA or a page). 170 * - Waiting for memory is not allowed. 171 * - The current thread may not be preempted. 172 * 173 * If any of those things are required, they must be performed outside of 174 * a critical section. If we discover that this is required while inside 175 * a critical section, then we must exit the critical section, perform the 176 * blocking work, re-enter the critical section and re-evaluate everything. 177 * Macros are provided to mark the boundaries of critical sections: 178 * 179 * - PMAP_CRIT_ENTER() 180 * - PMAP_CRIT_EXIT() 181 * 182 * XXX Alas, doesn't seem to be a way for us to hook into ASSERT_SLEEPABLE() 183 * XXX when inside a critical section. We should explore that for a future 184 * XXX enhancement. 185 */ 186 187 /* 188 * Current status: 189 * - Very stable multi-user on virt68k (qemu 68040; does not accurately 190 * model cache or ATC, but suitable for exercising large memory configs). 191 * 192 * - Very stable multi-user running memory pressure stress tests on 193 * small-memory configs in virt68k (qemu 68040 **with fixed MMU 194 * emulation**) and mac68k (10MB 68040, 68030). 195 * 196 * - Ports that have been adapted: hp300, luna68k (default), 197 * mac68k (default), mvme68k (default), news68k (default), 198 * next68k (default), virt68k (default), x68k. 199 * 200 * XXX TODO XXX 201 * 202 * - Adapt amiga (hard), atari (hard), cesfic (easy). 203 * - Test on 68020 + 68851. 204 * - Test on machines above listed as "not tested". 205 * - More rigorous testing in various emulators (Nono, UAE?). 206 * - Finish HP MMU support and test on real HP MMU. 207 * - Convert '851 / '030 to 3-level (maybe?). 208 * - Optimize ATC / cache manipulation. 209 * - Add some more instrumentation. 210 * - Eventually disable instrumentation by default. 211 * - ... 212 * - PROFIT! 213 */ 214 215 #include "opt_ddb.h" 216 #include "opt_kgdb.h" 217 #include "opt_m68k_arch.h" 218 219 #include <sys/cdefs.h> 220 __KERNEL_RCSID(0, "$NetBSD: pmap_68k.c,v 1.80 2026/10/22 20:48:19 andvar Exp $"); 221 222 #include <sys/param.h> 223 #include <sys/systm.h> 224 #include <sys/evcnt.h> 225 #include <sys/proc.h> 226 #include <sys/pool.h> 227 #include <sys/cpu.h> 228 #include <sys/atomic.h> 229 #include <sys/kmem.h> 230 231 #include <machine/pcb.h> 232 233 #include <uvm/uvm.h> 234 #include <uvm/uvm_physseg.h> 235 236 #include <m68k/cacheops.h> 237 #include <m68k/mmu.h> 238 239 #if !defined(M68K_MMU_MOTOROLA) && !defined(M68K_MMU_HP) 240 #error Hit the road, Jack... 241 #endif 242 243 /****************************** SERIALIZATION ********************************/ 244 245 /* 246 * XXX Would like to make these do something lightweight-ish in 247 * XXX DIAGNOSTIC kernels (and also make ASSERT_SLEEPABLE() trip 248 * XXX if we're in a critical section). 249 */ 250 251 #define PMAP_CRIT_ENTER(code) code 252 #define PMAP_CRIT_EXIT(code) code 253 #define PMAP_CRIT_ASSERT() __nothing 254 255 static inline void 256 pmap_busy(pmap_t pmap) 257 { 258 pmap->pm_busy++; 259 KDASSERT(pmap->pm_busy != 0); 260 } 261 262 static inline void 263 pmap_unbusy(pmap_t pmap) 264 { 265 KDASSERT(pmap->pm_busy != 0); 266 pmap->pm_busy--; 267 } 268 269 /**************************** MMU CONFIGURATION ******************************/ 270 271 #include "opt_m68k_arch.h" 272 273 #if defined(M68K_MMU_68030) 274 #include <m68k/mmu_30.h> /* for cpu_kcore_hdr_t */ 275 #endif 276 277 /* 278 * We consider 3 different MMU classes: 279 * - 68851 (includes 68030) 280 * - 68040 (includes 68060) 281 * - HP MMU for 68020 (68851-like, 2-level 4K only, external VAC) 282 */ 283 284 #define MMU_CLASS_68851 0 285 #define MMU_CLASS_68040 1 286 #define MMU_CLASS_HP 3 287 288 static int pmap_mmuclass __read_mostly; 289 290 #if defined(M68K_MMU_68851) || defined(M68K_MMU_68030) 291 #define MMU_CONFIG_68851_CLASS 1 292 #else 293 #define MMU_CONFIG_68851_CLASS 0 294 #endif 295 296 #if defined(M68K_MMU_68040) || defined(M68K_MMU_68060) 297 #define MMU_CONFIG_68040_CLASS 1 298 #else 299 #define MMU_CONFIG_68040_CLASS 0 300 #endif 301 302 #if defined(M68K_MMU_HP) 303 #define MMU_CONFIG_HP_CLASS 1 304 #else 305 #define MMU_CONFIG_HP_CLASS 0 306 #endif 307 308 #define MMU_CONFIG_NCLASSES (MMU_CONFIG_68851_CLASS + \ 309 MMU_CONFIG_68040_CLASS + \ 310 MMU_CONFIG_HP_CLASS) 311 312 #if MMU_CONFIG_NCLASSES == 1 313 314 #if MMU_CONFIG_68851_CLASS 315 #define MMU_IS_68851_CLASS 1 316 #elif MMU_CONFIG_68040_CLASS 317 #define MMU_IS_68040_CLASS 1 318 #elif MMU_CONFIG_HP_CLASS 319 #define MMU_IS_HP_CLASS 1 320 #else 321 #error Single MMU config predicate error. 322 #endif 323 324 #else /* MMU_CONFIG_NCLASSES != 1 */ 325 326 #if MMU_CONFIG_68851_CLASS 327 #define MMU_IS_68851_CLASS (pmap_mmuclass == MMU_CLASS_68851) 328 #endif 329 330 #if MMU_CONFIG_68040_CLASS 331 #define MMU_IS_68040_CLASS (pmap_mmuclass == MMU_CLASS_68040) 332 #endif 333 334 #if MMU_CONFIG_HP_CLASS 335 #define MMU_IS_HP_CLASS (pmap_mmuclass == MMU_CLASS_HP) 336 #endif 337 338 #endif /* MMU_CONFIG_NCLASSES == 1 */ 339 340 #ifndef MMU_IS_68851_CLASS 341 #define MMU_IS_68851_CLASS 0 342 #endif 343 344 #ifndef MMU_IS_68040_CLASS 345 #define MMU_IS_68040_CLASS 0 346 #endif 347 348 #ifndef MMU_IS_HP_CLASS 349 #define MMU_IS_HP_CLASS 0 350 #endif 351 352 /* 353 * 68040 must always use 3-level. Eventually, we will switch the '851 354 * type over to 3-level as well, for for now, it gets 2-level. The 355 * HP MMU is stuck there for all eternity. 356 */ 357 #define MMU_USE_3L (MMU_IS_68040_CLASS) 358 #define MMU_USE_2L (!MMU_USE_3L) 359 360 /***************************** INSTRUMENTATION *******************************/ 361 362 #define PMAP_EVENT_COUNTERS 363 364 static struct evcnt pmap_nkptpages_initial_ev = 365 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap nkptpages", "initial"); 366 static struct evcnt pmap_nkptpages_current_ev = 367 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap nkptpages", "current"); 368 EVCNT_ATTACH_STATIC(pmap_nkptpages_initial_ev); 369 EVCNT_ATTACH_STATIC(pmap_nkptpages_current_ev); 370 371 static struct evcnt pmap_nkstpages_initial_ev = 372 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap nkstpages", "initial"); 373 static struct evcnt pmap_nkstpages_current_ev = 374 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap nkstpages", "current"); 375 EVCNT_ATTACH_STATIC(pmap_nkstpages_initial_ev); 376 EVCNT_ATTACH_STATIC(pmap_nkstpages_current_ev); 377 378 static struct evcnt pmap_nptpages_current_ev = 379 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap nptpages", "current"); 380 static struct evcnt pmap_nptpages_hiwat_ev = 381 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap nptpages", "hiwat"); 382 EVCNT_ATTACH_STATIC(pmap_nptpages_current_ev); 383 EVCNT_ATTACH_STATIC(pmap_nptpages_hiwat_ev); 384 385 static struct evcnt pmap_maxkva_ev = 386 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap", "maxkva"); 387 EVCNT_ATTACH_STATIC(pmap_maxkva_ev); 388 389 static struct evcnt pmap_kvalimit_ev = 390 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap", "kvalimit"); 391 EVCNT_ATTACH_STATIC(pmap_kvalimit_ev); 392 393 #ifdef PMAP_EVENT_COUNTERS 394 static struct evcnt pmap_pv_alloc_wait_ev = 395 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap pv_alloc", "wait"); 396 EVCNT_ATTACH_STATIC(pmap_pv_alloc_wait_ev); 397 398 static struct evcnt pmap_pv_alloc_nowait_ev = 399 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap pv_alloc", "nowait"); 400 EVCNT_ATTACH_STATIC(pmap_pv_alloc_nowait_ev); 401 402 static struct evcnt pmap_pv_enter_called_ev = 403 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap pv_enter", "called"); 404 EVCNT_ATTACH_STATIC(pmap_pv_enter_called_ev); 405 406 static struct evcnt pmap_pv_enter_usr_ci_ev = 407 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap pv_enter", "usr_ci"); 408 EVCNT_ATTACH_STATIC(pmap_pv_enter_usr_ci_ev); 409 410 #if MMU_CONFIG_HP_CLASS 411 static struct evcnt pmap_pv_enter_vac_ci_ev = 412 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap pv_enter", "vac_ci"); 413 EVCNT_ATTACH_STATIC(pmap_pv_enter_vac_ci_ev); 414 #endif 415 416 static struct evcnt pmap_pv_enter_ci_multi_ev = 417 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap pv_enter", "ci_multi"); 418 EVCNT_ATTACH_STATIC(pmap_pv_enter_ci_multi_ev); 419 420 static struct evcnt pmap_pv_remove_called_ev = 421 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap pv_remove", "called"); 422 EVCNT_ATTACH_STATIC(pmap_pv_remove_called_ev); 423 424 static struct evcnt pmap_pv_remove_ci_ev = 425 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap pv_remove", "ci"); 426 EVCNT_ATTACH_STATIC(pmap_pv_remove_ci_ev); 427 428 static struct evcnt pmap_pt_cache_hit_ev = 429 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap pt_cache", "hit"); 430 EVCNT_ATTACH_STATIC(pmap_pt_cache_hit_ev); 431 432 static struct evcnt pmap_pt_cache_miss_ev = 433 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap pt_cache", "miss"); 434 EVCNT_ATTACH_STATIC(pmap_pt_cache_miss_ev); 435 436 static struct evcnt pmap_enter_nowait_ev = 437 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap enter", "nowait"); 438 EVCNT_ATTACH_STATIC(pmap_enter_nowait_ev); 439 440 static struct evcnt pmap_enter_yeswait_ev = 441 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap enter", "yeswait"); 442 EVCNT_ATTACH_STATIC(pmap_enter_yeswait_ev); 443 444 static struct evcnt pmap_enter_pte_alloc_fail_ev = 445 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap enter", "pte alloc failed"); 446 EVCNT_ATTACH_STATIC(pmap_enter_pte_alloc_fail_ev); 447 448 static struct evcnt pmap_enter_pv_alloc_fail_ev = 449 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap enter", "pv alloc failed"); 450 EVCNT_ATTACH_STATIC(pmap_enter_pv_alloc_fail_ev); 451 452 static struct evcnt pmap_enter_valid_ev = 453 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap enter", "valid"); 454 EVCNT_ATTACH_STATIC(pmap_enter_valid_ev); 455 456 static struct evcnt pmap_enter_wire_change_ev = 457 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap enter", "wire change"); 458 EVCNT_ATTACH_STATIC(pmap_enter_wire_change_ev); 459 460 static struct evcnt pmap_enter_prot_change_ev = 461 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap enter", "prot change"); 462 EVCNT_ATTACH_STATIC(pmap_enter_prot_change_ev); 463 464 static struct evcnt pmap_enter_pa_change_ev = 465 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap enter", "pa change"); 466 EVCNT_ATTACH_STATIC(pmap_enter_pa_change_ev); 467 468 static struct evcnt pmap_enter_pv_recycle_ev = 469 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap enter", "pv recycle"); 470 EVCNT_ATTACH_STATIC(pmap_enter_pv_recycle_ev); 471 472 static struct evcnt pmap_prm_got_pg_ev = 473 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap prm", "got pg"); 474 static struct evcnt pmap_prm_lookup_pg_hit_ev = 475 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap prm", "lookup pg hit"); 476 static struct evcnt pmap_prm_lookup_pg_miss_ev = 477 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap prm", "lookup pg miss"); 478 EVCNT_ATTACH_STATIC(pmap_prm_got_pg_ev); 479 EVCNT_ATTACH_STATIC(pmap_prm_lookup_pg_hit_ev); 480 EVCNT_ATTACH_STATIC(pmap_prm_lookup_pg_miss_ev); 481 482 #define pmap_evcnt(e) pmap_ ## e ## _ev.ev_count++ 483 #else 484 #define pmap_evcnt(e) __nothing 485 #endif 486 487 static void (*pmap_load_urp_func)(paddr_t) __read_mostly; 488 489 static void 490 pmap_mmuclass_init(void) 491 { 492 switch (mmutype) { 493 #if MMU_CONFIG_68040_CLASS 494 case MMU_68040: 495 case MMU_68060: 496 pmap_mmuclass = MMU_CLASS_68040; 497 /* 498 * XXX This is messy because 68060 frequently gets 499 * XXX initialize to MMU_68040. Should be cleaned 500 * XXX up once the Hibler pmap is obsoleted. 501 */ 502 #if defined(M68040) 503 if (cputype == CPU_68040) { 504 pmap_load_urp_func = mmu_load_urp40; 505 } 506 #endif 507 #if defined(M68060) 508 if (cputype == CPU_68060) { 509 pmap_load_urp_func = mmu_load_urp60; 510 } 511 #endif 512 break; 513 #endif 514 #if MMU_CONFIG_68851_CLASS 515 case MMU_68851: 516 case MMU_68030: 517 pmap_mmuclass = MMU_CLASS_68851; 518 protorp[0] = MMU51_CRP_BITS; 519 pmap_load_urp_func = mmu_load_urp51; 520 break; 521 #endif 522 #if MMU_CONFIG_HP_CLASS 523 case MMU_HP: 524 pmap_mmuclass = MMU_CLASS_HP; 525 pmap_load_urp_func = mmu_load_urp20hp; 526 break; 527 #endif 528 default: 529 panic("%s: mmutype=%d not configured?", __func__, mmutype); 530 } 531 532 if (pmap_load_urp_func == NULL) { 533 panic("%s: No mmu_load_*() for cputype=%d mmutype=%d", 534 __func__, cputype, mmutype); 535 } 536 } 537 538 /* 539 * pmap_load_urp: 540 * 541 * Load the user root table into the MMU. 542 */ 543 static inline void 544 pmap_load_urp(paddr_t urp) 545 { 546 (*pmap_load_urp_func)(urp); 547 } 548 549 #if MMU_CONFIG_HP_CLASS 550 static vaddr_t pmap_aliasmask __read_mostly; 551 #endif 552 553 /* 554 * pmap_init_vac: 555 * 556 * Set up virtually-addressed cache information. Only relevant 557 * for the HP MMU. 558 */ 559 void 560 pmap_init_vac(size_t vacsize) 561 { 562 #if MMU_CONFIG_HP_CLASS 563 KASSERT(pmap_aliasmask == 0); 564 KASSERT(powerof2(vacsize)); 565 pmap_aliasmask = vacsize - 1; 566 #endif 567 } 568 569 /***************************** PHYS <-> VM PAGE ******************************/ 570 571 static bool pmap_initialized_p; 572 573 static inline struct vm_page * 574 pmap_pa_to_pg(paddr_t pa) 575 { 576 return pmap_initialized_p ? PHYS_TO_VM_PAGE(pa) : NULL; 577 } 578 579 static pt_entry_t pmap_changebit(struct vm_page *, pt_entry_t, pt_entry_t); 580 581 /*************************** RESOURCE MANAGEMENT *****************************/ 582 583 static struct pmap kernel_pmap_store; 584 struct pmap * const kernel_pmap_ptr = &kernel_pmap_store; 585 586 /* 587 * Physical address of kernel level 1 table. This name is compatible 588 * with the Hibler pmap's name. 589 */ 590 paddr_t Sysseg_pa; 591 592 /* 593 * Avoid a memory load when doing comparisons against pmap_kernel() 594 * within this compilation unit. 595 */ 596 #undef pmap_kernel 597 #define pmap_kernel() (&kernel_pmap_store) 598 599 static inline bool 600 active_pmap(pmap_t pmap) 601 { 602 return pmap == pmap_kernel() || 603 pmap == curproc->p_vmspace->vm_map.pmap; 604 } 605 606 static inline bool 607 active_user_pmap(pmap_t pmap) 608 { 609 return curproc != NULL && 610 pmap != pmap_kernel() && 611 pmap == curproc->p_vmspace->vm_map.pmap; 612 } 613 614 /* 615 * Number of tables per page table page: 616 * 0 - number of leaf page tables per page 617 * 1 - number of segment tables per page 618 */ 619 static unsigned int pmap_ptpage_table_counts[2]; 620 621 __CTASSERT(LA40_L1_COUNT == LA40_L2_COUNT); 622 623 static void 624 pmap_ptpage_init(void) 625 { 626 if (MMU_USE_3L) { 627 pmap_ptpage_table_counts[0] = PAGE_SIZE / TBL40_L3_SIZE; 628 pmap_ptpage_table_counts[1] = PAGE_SIZE / TBL40_L2_SIZE; 629 } else { 630 pmap_ptpage_table_counts[0] = 1; 631 pmap_ptpage_table_counts[1] = 1; 632 } 633 } 634 635 static struct vm_page * 636 pmap_page_alloc(bool nowait) 637 { 638 uint64_t ticket; 639 struct vm_page *pg; 640 const int flags = nowait ? UVM_PGA_USERESERVE : 0; 641 642 while (ticket = uvm_wait_prepare(), 643 (pg = uvm_pagealloc(NULL, 0, NULL, flags)) == NULL) { 644 if (nowait) { 645 return NULL; 646 } 647 uvm_wait("pmappg", ticket); 648 } 649 pg->flags &= ~PG_BUSY; /* never busy */ 650 651 return pg; 652 } 653 654 static struct pmap_ptpage * 655 pmap_ptpage_alloc(bool segtab, bool nowait) 656 { 657 const unsigned int tabcnt = pmap_ptpage_table_counts[segtab]; 658 const size_t size = sizeof(struct pmap_ptpage) + 659 (sizeof(struct pmap_table) * tabcnt); 660 const size_t tabsize = PAGE_SIZE / tabcnt; 661 struct pmap_ptpage *ptp; 662 struct pmap_table *pt; 663 struct vm_page *pg; 664 const int uvm_f_nowait = nowait ? UVM_KMF_NOWAIT : 0; 665 vaddr_t ptpva; 666 667 ptp = kmem_zalloc(size, nowait ? KM_NOSLEEP : KM_SLEEP); 668 if (__predict_false(ptp == NULL)) { 669 return NULL; 670 } 671 672 /* Allocate a VA for the PT page. */ 673 ptpva = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 674 UVM_KMF_VAONLY | uvm_f_nowait); 675 if (__predict_false(ptpva == 0)) { 676 kmem_free(ptp, size); 677 return NULL; 678 } 679 680 /* Get a page. */ 681 pg = pmap_page_alloc(nowait); 682 if (__predict_false(pg == NULL)) { 683 uvm_km_free(kernel_map, ptpva, PAGE_SIZE, UVM_KMF_VAONLY); 684 kmem_free(ptp, size); 685 return NULL; 686 } 687 688 /* Map the page cache-inhibited and zero it out. */ 689 pmap_kenter_pa(ptpva, VM_PAGE_TO_PHYS(pg), 690 UVM_PROT_READ | UVM_PROT_WRITE, PMAP_NOCACHE); 691 zeropage((void *)ptpva); 692 693 /* 694 * All resources for the PT page have been allocated. 695 * Now initialize it and the individual table descriptors. 696 */ 697 LIST_INIT(&ptp->ptp_freelist); 698 ptp->ptp_pg = pg; 699 ptp->ptp_vpagenum = m68k_btop(ptpva); 700 ptp->ptp_freecnt = tabcnt; 701 ptp->ptp_segtab = segtab; 702 703 for (unsigned int i = 0; i < tabcnt; ptpva += tabsize, i++) { 704 pt = &ptp->ptp_tables[i]; 705 pt->pt_ptpage = ptp; 706 pt->pt_entries = (pt_entry_t *)ptpva; 707 LIST_INSERT_HEAD(&ptp->ptp_freelist, pt, pt_freelist); 708 } 709 710 #ifdef PMAP_EVENT_COUNTERS 711 pmap_nptpages_current_ev.ev_count32++; 712 if (pmap_nptpages_current_ev.ev_count32 > 713 pmap_nptpages_hiwat_ev.ev_count32) { 714 pmap_nptpages_hiwat_ev.ev_count32 = 715 pmap_nptpages_current_ev.ev_count32; 716 } 717 #endif 718 719 return ptp; 720 } 721 722 static void 723 pmap_ptpage_free(struct pmap_ptpage *ptp) 724 { 725 const unsigned int tabcnt = pmap_ptpage_table_counts[ptp->ptp_segtab]; 726 const size_t size = sizeof(struct pmap_ptpage) + 727 (sizeof(struct pmap_table) * tabcnt); 728 729 uvm_km_free(kernel_map, m68k_ptob(ptp->ptp_vpagenum), PAGE_SIZE, 730 UVM_KMF_WIRED); 731 kmem_free(ptp, size); 732 733 #ifdef PMAP_EVENT_COUNTERS 734 pmap_nptpages_current_ev.ev_count32--; 735 #endif 736 } 737 738 static struct pool pmap_pool; 739 static struct pool pmap_pv_pool; 740 741 #define PMAP_PV_LOWAT 16 742 743 static void 744 pmap_alloc_init(void) 745 { 746 pool_init(&pmap_pv_pool, sizeof(struct pv_entry), 747 PVH_ATTR_MASK + 1, /* align */ 748 0, /* ioff */ 749 0, /* flags */ 750 "pmappv", /* wchan */ 751 &pool_allocator_meta, /* palloc */ 752 IPL_VM); /* ipl */ 753 754 /* 755 * Set a low water mark on the pv_entry pool, so that we are 756 * more likely to have these around even in extreme memory 757 * starvation. 758 */ 759 pool_setlowat(&pmap_pv_pool, PMAP_PV_LOWAT); 760 761 pool_init(&pmap_pool, sizeof(struct pmap), 762 0, /* align */ 763 0, /* ioff */ 764 0, /* flags */ 765 "pmappl", /* wchan */ 766 &pool_allocator_kmem, /* palloc */ 767 IPL_NONE); /* ipl */ 768 } 769 770 static inline pmap_t 771 pmap_alloc(void) 772 { 773 pmap_t pmap = pool_get(&pmap_pool, PR_WAITOK); 774 memset(pmap, 0, sizeof(*pmap)); 775 return pmap; 776 } 777 778 static inline void 779 pmap_free(pmap_t pmap) 780 { 781 pool_put(&pmap_pool, pmap); 782 } 783 784 static struct pv_entry * 785 pmap_pv_alloc(bool nowait) 786 { 787 struct pv_entry *pv; 788 789 #ifdef PMAP_EVENT_COUNTERS 790 if (nowait) { 791 pmap_evcnt(pv_alloc_nowait); 792 } else { 793 pmap_evcnt(pv_alloc_wait); 794 } 795 #endif 796 797 pv = pool_get(&pmap_pv_pool, nowait ? PR_NOWAIT : 0); 798 if (__predict_true(pv != NULL)) { 799 KASSERT((((uintptr_t)pv) & PVH_ATTR_MASK) == 0); 800 } 801 return pv; 802 } 803 804 static void 805 pmap_pv_free(struct pv_entry *pv) 806 { 807 pool_put(&pmap_pv_pool, pv); 808 } 809 810 /* 811 * Whenever we need to free resources back to the system, we want to 812 * do it in a batch with any locks released. So, we have this around 813 * to collect the garbage, as needed. 814 */ 815 struct pmap_completion { 816 struct pmap_ptpage_list pc_ptpages; 817 struct pmap_pv_list pc_pvlist; 818 }; 819 820 static inline void 821 pmap_completion_init(struct pmap_completion *pc) 822 { 823 TAILQ_INIT(&pc->pc_ptpages); 824 LIST_INIT(&pc->pc_pvlist); 825 } 826 827 static void 828 pmap_completion_fini(struct pmap_completion *pc) 829 { 830 struct pmap_ptpage *ptp; 831 struct pv_entry *pv; 832 833 while ((ptp = TAILQ_FIRST(&pc->pc_ptpages)) != NULL) { 834 TAILQ_REMOVE(&pc->pc_ptpages, ptp, ptp_list); 835 /* 836 * Can't assert ptp_freecnt here; it won't match up 837 * in the pmap_remove_all() case. 838 * 839 * KASSERT(ptp->ptp_freecnt == 840 * pmap_ptpage_table_counts[ptp->ptp_segtab]); 841 */ 842 pmap_ptpage_free(ptp); 843 } 844 845 while ((pv = LIST_FIRST(&pc->pc_pvlist)) != NULL) { 846 LIST_REMOVE(pv, pv_pmlist); 847 pmap_pv_free(pv); 848 } 849 } 850 851 /************************ PTE MANIPULATION HELPERS ***************************/ 852 853 /* Assert assumptions made in <machine/pmap.h>. */ 854 __CTASSERT(DT51_PAGE == PTE40_RESIDENT); 855 __CTASSERT(PTE51_WP == PTE40_W); 856 __CTASSERT(PTE51_U == PTE40_U); 857 __CTASSERT(PTE51_M == PTE40_M); 858 __CTASSERT(PTE51_CI == PTE40_CM_NC_SER); 859 860 static pt_entry_t pmap_pte_proto[UVM_PROT_ALL + 1]; 861 static pt_entry_t pmap_pte_proto_ci[UVM_PROT_ALL + 1]; 862 static pt_entry_t pmap_pte_proto_um[UVM_PROT_ALL + 1]; 863 static pt_entry_t pmap_ste_proto; 864 865 static inline paddr_t 866 pte_pa(pt_entry_t pte) 867 { 868 return pte & PTE40_PGA; 869 } 870 871 /* 872 * These predicate inlines compile down into BFEXTU, so are quite fast. 873 */ 874 875 static inline bool 876 pte_valid_p(pt_entry_t pte) 877 { 878 return !!(pte & PTE_VALID); 879 } 880 881 static inline bool 882 pte_wired_p(pt_entry_t pte) 883 { 884 return !!(pte & PTE_WIRED); 885 } 886 887 static inline bool 888 pte_managed_p(pt_entry_t pte) 889 { 890 return !!(pte & PTE_PVLIST); 891 } 892 893 static inline bool 894 pte_ci_p(pt_entry_t pte) 895 { 896 /* 897 * Happily, PTE51_CI is bit 6, which is set for both of the 898 * cache-inhibited modes on 68040, so we can just check for 899 * that. 900 */ 901 return !!(pte & PTE51_CI); 902 } 903 904 #define PTE_PROT_CHANGE_BITS (PTE_WP | PTE_CMASK) 905 906 static inline pt_entry_t 907 pte_change_prot(pt_entry_t opte, vm_prot_t prot) 908 { 909 pt_entry_t *pte_proto = pte_ci_p(opte) ? pmap_pte_proto_ci 910 : pmap_pte_proto; 911 912 return (opte & ~PTE_PROT_CHANGE_BITS) | pte_proto[prot]; 913 } 914 915 static inline pt_entry_t 916 pte_load(volatile pt_entry_t *ptep) 917 { 918 return *ptep; 919 } 920 921 static inline void 922 pte_store(volatile pt_entry_t *ptep, pt_entry_t npte) 923 { 924 *ptep = npte; 925 } 926 927 /* 928 * Don't inline the CAS.L instruction; some systems have non-working 929 * READ-MODIFY-WRITE cycle logic. This will ensure that we'll use 930 * restartable atomic sequence, if required. 931 * 932 * AND.L and OR.L don't use the RMC signal, so they aren't subject 933 * to the same constraints. 934 */ 935 static inline bool 936 pte_update(volatile pt_entry_t *ptep, pt_entry_t opte, pt_entry_t npte) 937 { 938 /* 939 * Use compare-and-swap to update the PTE. This ensures there's 940 * no possibility of losing any hardware-maintained bits when 941 * updating the PTE. 942 * 943 * XXX Should turn this into a single instruction when possible 944 * XXX to deduce at compile time. 945 */ 946 return atomic_cas_uint(ptep, opte, npte) == opte; 947 } 948 949 #if MMU_CONFIG_HP_CLASS 950 /* 951 * These are only used for HP MMU VAC shenanigans. There is no need 952 * for these to be truly atomic, and systems with an HP MMU can't do 953 * truly atomic operations anyway. 954 */ 955 static inline void 956 pte_set(volatile pt_entry_t *ptep, pt_entry_t bits) 957 { 958 *ptep |= bits; 959 } 960 961 static inline void 962 pte_mask(volatile pt_entry_t *ptep, pt_entry_t mask) 963 { 964 *ptep &= mask; 965 } 966 #endif /* MMU_CONFIG_HP_CLASS */ 967 968 static inline pt_entry_t 969 pte_set_ci(pt_entry_t pte) 970 { 971 return (pte & ~PTE_CMASK) | (MMU_IS_68040_CLASS ? PTE40_CM_NC_SER 972 : PTE51_CI); 973 } 974 975 static inline pt_entry_t 976 pte_clr_ci(pt_entry_t pte) 977 { 978 pte &= ~PTE_CMASK; 979 if (MMU_IS_68040_CLASS) { 980 pte |= (pte & PTE_WP) ? PTE40_CM_WT 981 : PTE40_CM_CB; 982 } 983 return pte; 984 } 985 986 static void 987 pmap_pte_proto_init(void) 988 { 989 pt_entry_t c_bits, ro_c_bits, rw_c_bits, ci_bits, prot_bits, um_bits; 990 int prot; 991 992 if (MMU_IS_68040_CLASS) { 993 ro_c_bits = PTE40_CM_WT; /* this is what the Hibler pmap did */ 994 rw_c_bits = PTE40_CM_CB; 995 ci_bits = PTE40_CM_NC_SER; 996 } else { 997 ro_c_bits = rw_c_bits = 0; 998 ci_bits = PTE51_CI; 999 } 1000 1001 for (prot = 1; prot <= UVM_PROT_ALL; prot++) { 1002 prot_bits = um_bits = 0; 1003 if (prot & UVM_PROT_WRITE) { 1004 um_bits = PTE_U | PTE_M; 1005 } else if (prot & (UVM_PROT_READ|UVM_PROT_EXEC)) { 1006 prot_bits = PTE_WP; 1007 um_bits = PTE_U; 1008 } 1009 c_bits = (prot & UVM_PROT_WRITE) ? rw_c_bits : ro_c_bits; 1010 pmap_pte_proto[prot] = PTE_VALID | prot_bits | c_bits; 1011 pmap_pte_proto_ci[prot] = PTE_VALID | prot_bits | ci_bits; 1012 pmap_pte_proto_um[prot] = um_bits; 1013 } 1014 1015 /* 1016 * from hp300/DOC/HPMMU.notes: 1017 * 1018 * Segment table entries: 1019 * 1020 * bits 31-12: Physical page frame number of PT page 1021 * bits 11-4: Reserved at zero (can software use them?) 1022 * bit 3: Reserved at one 1023 * bits 1-0: Valid bits (hardware uses bit 1) 1024 * 1025 * This is all roughly compatible with 68851 and 68040: 1026 * 1027 * bit 3: DTE51_U / UTE40_U (used) 1028 * bits 1-0: DT51_SHORT / UTE40_RESIDENT 1029 * 1030 * The Hibler pmap set "SG_U" in the 68040 case, but not in 1031 * any others (??), which seems at odds with HPMMU.notes, but 1032 * whatever. It does not seem to cause any harm to set the 1033 * "used" bit in all cases, so that's what we'll do. If it 1034 * does prove to be problematic, we can make adjustments. 1035 */ 1036 pmap_ste_proto = DTE51_U | DT51_SHORT; 1037 } 1038 1039 static inline pt_entry_t 1040 pmap_make_pte(paddr_t pa, vm_prot_t prot, u_int flags) 1041 { 1042 pt_entry_t *pte_proto = (flags & PMAP_NOCACHE) ? pmap_pte_proto_ci 1043 : pmap_pte_proto; 1044 1045 prot &= UVM_PROT_ALL; 1046 KASSERT(prot != 0); 1047 1048 /* 1049 * N.B. We seed the U and M bits in the PTE based on the access 1050 * that was used to produce the page fault because the MMU will 1051 * skip the write-back of a PTE upon access if the copy in the 1052 * ATC already has the bit set. 1053 */ 1054 pt_entry_t npte = pa | pte_proto[prot] | 1055 pmap_pte_proto_um[flags & UVM_PROT_ALL]; 1056 1057 if (flags & PMAP_WIRED) { 1058 npte |= PTE_WIRED; 1059 } 1060 1061 return npte; 1062 } 1063 1064 /************************** PAGE TABLE MANAGEMENT ****************************/ 1065 1066 /* 1067 * Kernel page table management works differently from user page table 1068 * management. An initial set of kernel PTs are allocated during early 1069 * bootstrap (enough to map the virtual addresses set up at that time, 1070 * plus a little extra to give the kernel some breathing room while 1071 * UVM gets initialized -- see pmap_bootstrap1()). If more PTs are 1072 * needed in order to expand the kernel address space, pmap_growkernel() 1073 * is called to allocate some more. We always allocate kernel PTs in 1074 * chunks of one page, allocating more inner segment tables as needed 1075 * to link them into the MMU tree (3-level), or just poking them in 1076 * directly to the level-1 table (2-level). 1077 * 1078 * The kernel PTs are mapped into a single linear array to make that 1079 * makes it possible to simply index by virtual page number to find 1080 * the PTE that maps that virtual address. 1081 */ 1082 #define PTPAGEVASZ ((PAGE_SIZE / sizeof(pt_entry_t)) * PAGE_SIZE) 1083 #define PTPAGEVAOFS (PTPAGEVASZ - 1) 1084 1085 #define pmap_round_ptpage(va) (((va) + PTPAGEVAOFS) & ~PTPAGEVAOFS) 1086 #define pmap_trunc_ptpage(va) ((va) & ~PTPAGEVAOFS) 1087 1088 /* 1089 * kernel_virtual_start marks the first kernel virtual address that 1090 * is handed off to UVM to manage. kernel_virtual_end marks the end 1091 * of the kernel address space that is currently mappable with the 1092 * number of pages allocated to kernel PTs. 1093 * 1094 * kernel_virtual_start is fixed once pmap_bootstrap1() completes. 1095 * kernel_virtual_end can be extended by calling pmap_growkernel(). 1096 * 1097 * kernel_virtual_max represents the absolute maximum. It starts at 1098 * KERNEL_MAX_ADDRESS, but may get clamped by fixed mappings that 1099 * start beyond the end of kernel virtual address space. 1100 * 1101 * kernel_virtual_max is exported to the rest of the kernel via 1102 * pmap_virtual_space() and VM_MAX_KERNEL_ADDRESS. 1103 */ 1104 #define KERNEL_MAX_ADDRESS ((vaddr_t)0 - PAGE_SIZE) 1105 static vaddr_t kernel_virtual_start, kernel_virtual_end; 1106 vaddr_t kernel_virtual_max = KERNEL_MAX_ADDRESS; 1107 1108 /* 1109 * kernel_stnext_pa and kernel_stnext_endpa together implement a 1110 * simple allocator for inner segment tables used in the 3-level 1111 * configuration. When the initial level-1 table is allocated 1112 * the remained of that page is set in kernel_stnext_pa, and 1113 * kernel_stnext_endpa is set to the next page boundary. When 1114 * a segment table is needed, kernel_stnext_pa is the address 1115 * of the next free table and is advanced by the L2 table size 1116 * (512 bytes). If that allocation attempt finds that kernel_stnext_pa 1117 * is equal to kernel_stnext_endpa, a new page is allocated and 1118 * kernel_stnext_pa and kernel_stnext_endpa updated to reflect 1119 * the newly-allocated page before the table is taken from it. 1120 */ 1121 static paddr_t kernel_stnext_pa, kernel_stnext_endpa; 1122 1123 /* 1124 * Null segment table that every pmap gets as its initial level 1 1125 * map. This is a single page allocated in pmap_bootstrap1(), and 1126 * we zero it out in pmap_init(). 1127 */ 1128 static paddr_t null_segtab_pa __read_mostly; 1129 1130 static inline void 1131 pmap_set_lev1map(pmap_t pmap, struct pmap_table *pt, paddr_t pa) 1132 { 1133 pmap->pm_lev1map = pt; 1134 pmap->pm_lev1pa = pa; 1135 if (active_user_pmap(pmap)) { 1136 #if MMU_CONFIG_HP_CLASS 1137 /* 1138 * N.B. re-loading the user segment table pointer also 1139 * invalidates the user side of the VAC, so no additional 1140 * work is necessary. 1141 */ 1142 #endif 1143 pmap_load_urp(pmap->pm_lev1pa); 1144 TBIAU(); /* XXX optimize? */ 1145 ICIA(); /* XXX optimize? */ 1146 } 1147 } 1148 1149 /* 1150 * Table accessors. 1151 */ 1152 static inline unsigned int 1153 pmap_pagenum(vaddr_t va) 1154 { 1155 return ((va) >> PAGE_SHIFT); 1156 } 1157 1158 static inline unsigned int 1159 pmap_segnum(vaddr_t va) 1160 { 1161 return MMU_USE_3L ? ((va) >> SEGSHIFT3L) : ((va) >> SEGSHIFT2L); 1162 } 1163 1164 static inline unsigned int 1165 pmap_st1_index(vaddr_t va) 1166 { 1167 return MMU_USE_3L ? LA40_RI(va) : LA2L_RI(va); 1168 } 1169 1170 static inline unsigned int 1171 pmap_st_index(vaddr_t va) 1172 { 1173 return MMU_USE_3L ? LA40_PI(va) : LA2L_RI(va); 1174 } 1175 1176 static inline unsigned int 1177 pmap_pt_index(vaddr_t va) 1178 { 1179 return MMU_USE_3L ? LA40_PGI(va) : LA2L_PGI(va); 1180 } 1181 1182 static inline vaddr_t 1183 pmap_trunc_seg(vaddr_t va) 1184 { 1185 return MMU_USE_3L ? pmap_trunc_seg_3L(va) : pmap_trunc_seg_2L(va); 1186 } 1187 1188 static inline vaddr_t 1189 pmap_trunc_seg1(vaddr_t va) 1190 { 1191 KASSERT(MMU_USE_3L); 1192 return pmap_trunc_seg1_3L(va); 1193 } 1194 1195 static inline vaddr_t 1196 pmap_round_seg(vaddr_t va) 1197 { 1198 return MMU_USE_3L ? pmap_round_seg_3L(va) : pmap_round_seg_2L(va); 1199 } 1200 1201 static inline vaddr_t 1202 pmap_next_seg(vaddr_t va) 1203 { 1204 return pmap_round_seg(va + PAGE_SIZE); 1205 } 1206 1207 static paddr_t 1208 pmap_table_pa(const struct pmap_table * const pt) 1209 { 1210 const struct pmap_ptpage * const ptp = pt->pt_ptpage; 1211 const vaddr_t ptpva = m68k_ptob(ptp->ptp_vpagenum); 1212 const vaddr_t ptva = (vaddr_t)pt->pt_entries; 1213 1214 return VM_PAGE_TO_PHYS(ptp->ptp_pg) + (ptva - ptpva); 1215 } 1216 1217 static inline unsigned int 1218 pmap_table_make_key(unsigned int segnum, bool segtab) 1219 { 1220 KASSERT((segnum & 0x80000000) == 0); 1221 return (segnum << 1) | (unsigned int)segtab; 1222 } 1223 1224 static int 1225 pmap_table_rb_compare_key(void *v __unused, const void *n, const void *k) 1226 { 1227 const struct pmap_table * const pt1 = n; 1228 const unsigned int k1 = pt1->pt_key; 1229 const unsigned int k2 = *(const unsigned int *)k; 1230 1231 return (int)(k1 - k2); 1232 } 1233 1234 static int 1235 pmap_table_rb_compare_nodes(void *v, const void *n1, const void *n2) 1236 { 1237 const struct pmap_table * const pt2 = n2; 1238 1239 return pmap_table_rb_compare_key(v, n1, &pt2->pt_key); 1240 } 1241 1242 static const rb_tree_ops_t pmap_table_rb_ops = { 1243 .rbto_compare_nodes = pmap_table_rb_compare_nodes, 1244 .rbto_compare_key = pmap_table_rb_compare_key, 1245 .rbto_node_offset = offsetof(struct pmap_table, pt_node), 1246 }; 1247 1248 static struct pmap_table * 1249 pmap_table_alloc(pmap_t pmap, bool segtab, bool nowait, 1250 struct pmap_completion *pc) 1251 { 1252 struct pmap_ptpage_list *pmlist = &pmap->pm_ptpages[segtab]; 1253 struct pmap_ptpage *ptp, *newptp = NULL; 1254 struct pmap_table *pt; 1255 1256 KASSERT(pc != NULL); 1257 1258 try_again: 1259 if ((ptp = TAILQ_FIRST(pmlist)) == NULL || ptp->ptp_freecnt == 0) { 1260 /* 1261 * No PT pages with free tables (empty PT pages are moved 1262 * to the tail of the list). Allocate a new PT page and 1263 * try again. If someone else successfully allocates one 1264 * while we're sleeping, then we'll use it and free what 1265 * we allocated back to the system. 1266 */ 1267 KASSERT(ptp == NULL || LIST_FIRST(&ptp->ptp_freelist) == NULL); 1268 if (newptp == NULL) { 1269 newptp = pmap_ptpage_alloc(segtab, nowait); 1270 if (newptp == NULL) { 1271 /* 1272 * If we didn't wait, then no one would 1273 * have allocated one behind our back. 1274 */ 1275 KASSERT(nowait); 1276 return NULL; 1277 } 1278 goto try_again; 1279 } 1280 ptp = newptp; 1281 TAILQ_INSERT_HEAD(pmlist, newptp, ptp_list); 1282 } 1283 if (__predict_false(newptp != NULL && ptp != newptp)) { 1284 /* Not using newly-allocated PT page; free it back. */ 1285 TAILQ_INSERT_TAIL(&pc->pc_ptpages, newptp, ptp_list); 1286 } 1287 pt = LIST_FIRST(&ptp->ptp_freelist); 1288 KASSERT(pt != NULL); 1289 LIST_REMOVE(pt, pt_freelist); 1290 ptp->ptp_freecnt--; 1291 if (ptp->ptp_freecnt == 0 && 1292 TAILQ_NEXT(ptp, ptp_list) != NULL) { 1293 TAILQ_REMOVE(pmlist, ptp, ptp_list); 1294 TAILQ_INSERT_TAIL(pmlist, ptp, ptp_list); 1295 } 1296 KASSERT(pt->pt_st == NULL); 1297 pt->pt_holdcnt = 1; 1298 1299 return pt; 1300 } 1301 1302 static void 1303 pmap_table_free(pmap_t pmap, struct pmap_table *pt, 1304 struct pmap_completion *pc) 1305 { 1306 struct pmap_ptpage *ptp = pt->pt_ptpage; 1307 struct pmap_ptpage_list *pmlist = &pmap->pm_ptpages[ptp->ptp_segtab]; 1308 1309 KASSERT(pt->pt_st == NULL); 1310 1311 LIST_INSERT_HEAD(&ptp->ptp_freelist, pt, pt_freelist); 1312 KASSERT(ptp->ptp_freecnt < pmap_ptpage_table_counts[ptp->ptp_segtab]); 1313 ptp->ptp_freecnt++; 1314 1315 /* 1316 * If the PT page no longer has any active tables, then 1317 * remove it from the pmap and queue it up to be given 1318 * back to the system. 1319 */ 1320 if (ptp->ptp_freecnt == pmap_ptpage_table_counts[ptp->ptp_segtab]) { 1321 TAILQ_REMOVE(pmlist, ptp, ptp_list); 1322 TAILQ_INSERT_TAIL(&pc->pc_ptpages, ptp, ptp_list); 1323 } 1324 /* 1325 * If the PT page now has exactly one free table, then 1326 * put it at the head of its list so that it is allocated 1327 * from first the next time a table is needed. 1328 */ 1329 else if (ptp->ptp_freecnt == 1) { 1330 TAILQ_REMOVE(pmlist, ptp, ptp_list); 1331 TAILQ_INSERT_HEAD(pmlist, ptp, ptp_list); 1332 } 1333 /* 1334 * Push this PT page down the list if it has more free tables 1335 * than the ones that come after. The goal is to keep PT pages 1336 * with the fewest free tables at the head of the list so that 1337 * they're allocated from first. This is an effort to keep 1338 * fragmentation at bay so as to increase the likelihood that 1339 * we can free PT pages back to the system. 1340 */ 1341 else { 1342 struct pmap_ptpage *next_ptp; 1343 for (next_ptp = TAILQ_NEXT(ptp, ptp_list); 1344 next_ptp != NULL; 1345 next_ptp = TAILQ_NEXT(next_ptp, ptp_list)) { 1346 if (next_ptp->ptp_freecnt < ptp->ptp_freecnt) { 1347 break; 1348 } 1349 } 1350 if (next_ptp != NULL && 1351 next_ptp != TAILQ_NEXT(ptp, ptp_list) && 1352 next_ptp->ptp_freecnt != 0) { 1353 TAILQ_REMOVE(pmlist, ptp, ptp_list); 1354 TAILQ_INSERT_AFTER(pmlist, next_ptp, ptp, ptp_list); 1355 } 1356 } 1357 } 1358 1359 /* 1360 * pmap_table_retain: 1361 * 1362 * Take a retain count on the specified table. Retain counts 1363 * are used to ensure the table remains stable while working 1364 * on it, and each mapping placed into the table also gets 1365 * a retain count. 1366 */ 1367 static inline void 1368 pmap_table_retain(struct pmap_table *pt) 1369 { 1370 if (__predict_true(pt != NULL)) { 1371 pt->pt_holdcnt++; 1372 KASSERT(pt->pt_holdcnt != 0); 1373 } 1374 } 1375 1376 /* 1377 * pmap_table_release: 1378 * 1379 * Release a previously-taken retain count on the specified 1380 * table. If the retain count drops to zero, the table is 1381 * unlinked from the lookup tree and the MMU tree and freed. 1382 */ 1383 static __noinline void 1384 pmap_table_release_slow(pmap_t pmap, struct pmap_table *pt, 1385 struct pmap_completion *pc) 1386 { 1387 KASSERT(pt != NULL); 1388 KASSERT(pt->pt_holdcnt != 0); 1389 pt->pt_holdcnt--; 1390 if (__predict_false(pt->pt_holdcnt != 0)) { 1391 return; 1392 } 1393 1394 /* 1395 * If the caller doesn't expect the count to go to zero, 1396 * they won't have bothered with a completion context. 1397 * Going to zero is unexpected in this case, so blow up 1398 * if it happens. 1399 */ 1400 KASSERT(pc != NULL); 1401 if (__predict_true(pt == pmap->pm_pt_cache)) { 1402 pmap->pm_pt_cache = NULL; 1403 } 1404 if (__predict_true(pt->pt_st != NULL)) { 1405 /* 1406 * This table needs to be unlinked from the lookup 1407 * tree and the MMU tree. 1408 */ 1409 pte_store(&pt->pt_st->pt_entries[pt->pt_stidx], 0); 1410 rb_tree_remove_node(&pmap->pm_tables, pt); 1411 pmap_table_release_slow(pmap, pt->pt_st, pc); 1412 pt->pt_st = NULL; 1413 } else if (pt == pmap->pm_lev1map) { 1414 pmap_set_lev1map(pmap, NULL, null_segtab_pa); 1415 } 1416 pmap_table_free(pmap, pt, pc); 1417 } 1418 1419 static inline void 1420 pmap_table_release(pmap_t pmap, struct pmap_table *pt, 1421 struct pmap_completion *pc) 1422 { 1423 if (__predict_true(pt != NULL)) { 1424 if (__predict_true(pt->pt_holdcnt > 1)) { 1425 pt->pt_holdcnt--; 1426 return; 1427 } 1428 pmap_table_release_slow(pmap, pt, pc); 1429 } 1430 } 1431 1432 /* 1433 * pmap_table_lookup: 1434 * 1435 * Lookup the table corresponding to the specified segment. 1436 */ 1437 static struct pmap_table * 1438 pmap_table_lookup(pmap_t pmap, unsigned int segnum, bool segtab) 1439 { 1440 const unsigned int key = pmap_table_make_key(segnum, segtab); 1441 struct pmap_table *pt; 1442 1443 if ((pt = pmap->pm_pt_cache) == NULL || pt->pt_key != key) { 1444 pmap_evcnt(pt_cache_miss); 1445 pt = rb_tree_find_node(&pmap->pm_tables, &key); 1446 if (__predict_true(!segtab)) { 1447 pmap->pm_pt_cache = pt; 1448 } 1449 } else { 1450 pmap_evcnt(pt_cache_hit); 1451 } 1452 if (pt != NULL) { 1453 pmap_table_retain(pt); 1454 } 1455 return pt; 1456 } 1457 1458 /* 1459 * pmap_table_insert: 1460 * 1461 * Allocate and insert a table into the tree at the specified 1462 * location. 1463 */ 1464 static struct pmap_table * 1465 pmap_table_insert(pmap_t pmap, struct pmap_table *t1, unsigned int stidx, 1466 unsigned int segnum, bool segtab, bool nowait, struct pmap_completion *pc) 1467 { 1468 struct pmap_table *t2, *ret_t; 1469 1470 t2 = pmap_table_lookup(pmap, segnum, segtab); 1471 if (t2 != NULL) { 1472 /* 1473 * Table at this level already exists, and looking 1474 * it up gave us a retain count, so we no longer need 1475 * the retain count on the upper level table (it is 1476 * retained-by-proxy by the table we just found). 1477 * We pass NULL for the completion context because 1478 * we don't expect the upper level table's retain count 1479 * to drop to zero, and we want things to blow up 1480 * loudly if it does! 1481 */ 1482 pmap_table_release(pmap, t1, NULL); 1483 return t2; 1484 } 1485 1486 /* Allocate the new table. */ 1487 PMAP_CRIT_EXIT(); 1488 t2 = pmap_table_alloc(pmap, segtab, nowait, pc); 1489 PMAP_CRIT_ENTER(); 1490 if (__predict_false(t2 == NULL)) { 1491 pmap_table_release(pmap, t1, pc); 1492 return NULL; 1493 } 1494 t2->pt_key = pmap_table_make_key(segnum, segtab); 1495 1496 /* 1497 * Now that we have the new table, we need to insert it into the 1498 * table lookup tree. If we blocked while allocating, it's possible 1499 * someone raced with us and inserted one behind our back, so we need 1500 * to check for that. 1501 */ 1502 ret_t = rb_tree_insert_node(&pmap->pm_tables, t2); 1503 if (__predict_false(ret_t != t2)) { 1504 /* 1505 * Someone beat us to the punch. If this happens, 1506 * then we also need to drop the retain count on 1507 * t1 because the table we just found already has 1508 * a retain count on it. 1509 */ 1510 pmap_table_retain(ret_t); 1511 pmap_table_release(pmap, t2, pc); 1512 pmap_table_release(pmap, t1, NULL); 1513 return ret_t; 1514 } 1515 1516 /* 1517 * Table has been successfully inserted into the lookup 1518 * tree, now link it into the MMU's tree. The new table 1519 * takes ownership of the retain count that was taken on 1520 * the upper level table while working. 1521 */ 1522 t2->pt_st = t1; 1523 t2->pt_stidx = (unsigned short)stidx; 1524 pte_store(&t1->pt_entries[stidx], pmap_ste_proto | pmap_table_pa(t2)); 1525 1526 return t2; 1527 } 1528 1529 /*************************** PTE LOOKUP FUNCTIONS ****************************/ 1530 1531 static pt_entry_t *kernel_ptes; 1532 1533 /* 1534 * pmap_kernel_pte: 1535 * 1536 * Get the PTE that maps the specified kernel virtual address. 1537 * 1538 * Take note: the caller *may assume* they they can linearly 1539 * access adjacent PTEs up until the address indicated by 1540 * virtual_end! That means, "pte++" is totally fine until you 1541 * get to the current limit of the kernel virtual address space! 1542 */ 1543 static pt_entry_t * 1544 pmap_kernel_pte(vaddr_t va) 1545 { 1546 /* 1547 * The kernel PTEs are mapped as a linear array, whose entries 1548 * represent the entire possible 4GB supervisor address space. 1549 * 1550 * Kernel PT pages are pre-allocated and mapped into this linear 1551 * space (via pmap_growkernel(), as needed) and never freed back. 1552 * So, as long as the VA is below virtual_end, we know that a PTE 1553 * exists to back it. 1554 * 1555 * We don't assert that the VA < virtual_end, however; there may 1556 * be special cases where we need to get a PTE that has been 1557 * statically-allocated out beyond where virtual space is allowed 1558 * to grow. We'll find out soon enough if a PT page doesn't back 1559 * it, because a fault will occur when the PTE is accessed. 1560 */ 1561 KASSERT(va >= VM_MIN_KERNEL_ADDRESS); 1562 return &kernel_ptes[m68k_btop(va - VM_MIN_KERNEL_ADDRESS)]; 1563 } 1564 1565 /* 1566 * pmap_pte_lookup: 1567 * 1568 * Lookup the PTE for the given address, returning a retained 1569 * reference to the table containing the PTE. 1570 * 1571 * Take note: the caller *may assume* they they can linearly 1572 * access adjacent PTEs that map addresses within the same 1573 * segment! That means, "pte++" is totally fine until you 1574 * get to the next segment boundary! 1575 */ 1576 static pt_entry_t * 1577 pmap_pte_lookup(pmap_t pmap, vaddr_t va, struct pmap_table **out_pt) 1578 { 1579 if (pmap == pmap_kernel()) { 1580 *out_pt = NULL; 1581 return pmap_kernel_pte(va); 1582 } 1583 1584 const unsigned int segnum = pmap_segnum(va); 1585 1586 struct pmap_table *pt = pmap_table_lookup(pmap, segnum, false); 1587 if (__predict_true(pt != NULL)) { 1588 *out_pt = pt; /* already retained */ 1589 return &pt->pt_entries[pmap_pt_index(va)]; 1590 } 1591 1592 *out_pt = NULL; 1593 return NULL; 1594 } 1595 1596 /* 1597 * pmap_pte_alloc: 1598 * 1599 * Like pmap_pte_lookup(), but allocates tables as necessary. 1600 * 1601 * We enter in a critical section, but may drop that along 1602 * the way and re-validate our own assumptions. Callers 1603 * (pmap_enter(), basically), should be aware of this. 1604 */ 1605 static pt_entry_t * 1606 pmap_pte_alloc(pmap_t pmap, vaddr_t va, struct pmap_table **out_pt, 1607 bool nowait, struct pmap_completion *pc) 1608 { 1609 struct pmap_table *st, *pt; 1610 pt_entry_t *ptep; 1611 1612 PMAP_CRIT_ASSERT(); 1613 1614 ptep = pmap_pte_lookup(pmap, va, out_pt); 1615 if (__predict_true(ptep != NULL)) { 1616 return ptep; 1617 } 1618 1619 /* 1620 * First get a reference on the top-level segment table and 1621 * retain it so that it's stable while we work. 1622 */ 1623 if (__predict_true((st = pmap->pm_lev1map) != NULL)) { 1624 pmap_table_retain(st); 1625 } else { 1626 /* 1627 * Oh look! Baby pmap's first mapping! Allocate 1628 * a segment table. 1629 */ 1630 PMAP_CRIT_EXIT(); 1631 st = pmap_table_alloc(pmap, true/*segtab*/, nowait, pc); 1632 PMAP_CRIT_ENTER(); 1633 if (__predict_false(st == NULL)) { 1634 return NULL; 1635 } 1636 1637 /* Re-validate that we still need the segment table. */ 1638 if (__predict_false(pmap->pm_lev1map != NULL)) { 1639 /* Raced and lost. */ 1640 pmap_table_release(pmap, st, pc); 1641 st = pmap->pm_lev1map; 1642 pmap_table_retain(st); 1643 } else { 1644 /* New table is returned to us retained. */ 1645 pmap_set_lev1map(pmap, st, pmap_table_pa(st)); 1646 } 1647 } 1648 1649 /* 1650 * Now we know that st points to a valid segment table with a 1651 * retain count that lets us safely reference it. 1652 */ 1653 1654 if (MMU_USE_3L) { 1655 /* Get the inner segment table for this virtual address. */ 1656 struct pmap_table * const st1 = st; 1657 st = pmap_table_insert(pmap, st1, pmap_st1_index(va), 1658 pmap_st1_index(va), true/*segtab*/, nowait, pc); 1659 if (__predict_false(st == NULL)) { 1660 pmap_table_release(pmap, st1, pc); 1661 return NULL; 1662 } 1663 } 1664 1665 /* We can now allocate and insert the leaf page table. */ 1666 pt = pmap_table_insert(pmap, st, pmap_st_index(va), pmap_segnum(va), 1667 false/*segtab*/, nowait, pc); 1668 if (__predict_false(pt == NULL)) { 1669 pmap_table_release(pmap, st, pc); 1670 return NULL; 1671 } 1672 1673 *out_pt = pt; 1674 return &pt->pt_entries[pmap_pt_index(va)]; 1675 } 1676 1677 /************************** P->V ENTRY MANAGEMENT ****************************/ 1678 1679 static inline pt_entry_t * 1680 pmap_pv_pte(struct pv_entry * const pv) 1681 { 1682 const vaddr_t va = PV_VA(pv); 1683 1684 if (__predict_true(pv->pv_pmap != pmap_kernel())) { 1685 KASSERT(pv->pv_pt != NULL); 1686 return &pv->pv_pt->pt_entries[pmap_pt_index(va)]; 1687 } 1688 return pmap_kernel_pte(va); 1689 } 1690 1691 #define MATCHING_PMAP(p1, p2) \ 1692 ((p1) == (p2) || \ 1693 (p1) == pmap_kernel() || (p2) == pmap_kernel()) 1694 1695 #define CONFLICTING_ALIAS(va1, va2) \ 1696 (((va1) & pmap_aliasmask) != ((va2) & pmap_aliasmask)) 1697 1698 /* 1699 * pmap_pv_enter: 1700 * 1701 * Add a physical->virtual entry to the pv table. Caller must provide 1702 * the storage for the new PV entry. 1703 * 1704 * We are responsible for storing the new PTE into the destination 1705 * table. We are also guaranteed that no mapping exists there, but 1706 * the MMU has a negative cache in the ATC (see 68030UM Figure 9-8. 1707 * Address Translation General Flowchart, ATC hit, B==1 case, as well 1708 * as 68040UM Figure 3-21. ATC Entry and Tag Fields, R bit and the 1709 * associated descriptive text), so we still have to handle ATC entry 1710 * invalidation. 1711 */ 1712 static void 1713 pmap_pv_enter(pmap_t pmap, struct vm_page *pg, vaddr_t va, vm_prot_t prot, 1714 struct pmap_table *pt, pt_entry_t npte, struct pv_entry *newpv) 1715 { 1716 const bool usr_ci = pte_ci_p(npte); 1717 struct pv_entry *pv; 1718 pt_entry_t opte; 1719 1720 pmap_evcnt(pv_enter_called); 1721 1722 PMAP_CRIT_ASSERT(); 1723 KASSERT(newpv != NULL); 1724 1725 npte |= PTE_PVLIST; 1726 1727 newpv->pv_pmap = pmap; 1728 newpv->pv_vf = va; 1729 newpv->pv_pt = pt; 1730 1731 pt_entry_t *ptep = pmap_pv_pte(newpv); 1732 1733 #ifdef DEBUG 1734 /* 1735 * Make sure the entry doesn't already exist. 1736 */ 1737 for (pv = VM_MDPAGE_PVS(pg); pv != NULL; pv = pv->pv_next) { 1738 if (pmap == pv->pv_pmap && va == PV_VA(pv)) { 1739 panic("%s: pmap=%p va=0x%08lx already in PV table", 1740 __func__, pmap, va); 1741 } 1742 } 1743 #endif 1744 1745 if (__predict_false(usr_ci)) { 1746 newpv->pv_vf |= PV_F_CI_USR; 1747 } 1748 1749 newpv->pv_next = VM_MDPAGE_PVS(pg); 1750 VM_MDPAGE_SETPVP(VM_MDPAGE_HEAD_PVP(pg), newpv); 1751 LIST_INSERT_HEAD(&pmap->pm_pvlist, newpv, pv_pmlist); 1752 VM_MDPAGE_ADD_UM(pg, npte); 1753 1754 /* 1755 * If this is an EXEC mapping, then we have to ensure that 1756 * the I$ doesn't load stale data. 1757 * 1758 * XXX Should have a soft-PTE bit for this. 1759 */ 1760 if (prot & UVM_PROT_EXEC) { 1761 #if MMU_CONFIG_68040_CLASS 1762 if (MMU_IS_68040_CLASS) { 1763 /* 1764 * XXX Potential future optimization: is only 1765 * XXX the DCFP() needed here to deal with 1766 * XXX write-back? Should we track EXEC-ness 1767 * XXX in the VM_MDPAGE? 1768 */ 1769 const paddr_t pa = VM_PAGE_TO_PHYS(pg); 1770 DCFP(pa); 1771 ICPP(pa); 1772 } 1773 #endif 1774 } 1775 1776 #if MMU_CONFIG_HP_CLASS 1777 if (MMU_IS_HP_CLASS) { 1778 /* Go handle the HP MMU's VAC. */ 1779 goto hp_mmu_vac_shenanigans; 1780 } 1781 #endif 1782 1783 /* 1784 * If the page is marked as being cache-inhibited, it means 1785 * there is at least one user-requested CI mapping already 1786 * (and that all of the extant mappings are thus CI). 1787 * 1788 * In this case, we need to make sure that the one we're 1789 * establishing now is CI as well. 1790 */ 1791 if (__predict_false(VM_MDPAGE_CI_P(pg))) { 1792 npte = pte_set_ci(npte); 1793 pte_store(ptep, npte); 1794 /* See below. */ 1795 if (active_pmap(pmap)) { 1796 TBIS(va); 1797 } 1798 return; 1799 } 1800 1801 /* Set the PTE for the new mapping. */ 1802 pte_store(ptep, npte); 1803 1804 /* 1805 * Invalidate the ATC entry **after** storing the PTE so that 1806 * there is no window where another MMU table walk finds the 1807 * stale invalid entry. 1808 */ 1809 if (active_pmap(pmap)) { 1810 TBIS(va); 1811 } 1812 1813 /* 1814 * If this is a user-requested CI mapping, we need to make 1815 * sure the page is purged from the cache and mark any other 1816 * mappings of this page CI as well. 1817 */ 1818 if (__predict_false(usr_ci)) { 1819 VM_MDPAGE_SET_CI(pg); 1820 1821 pmap_evcnt(pv_enter_usr_ci); 1822 1823 /* 1824 * There shouldn't be very many of these; CI mappings 1825 * of managed pages are typically only for coherent DMA 1826 * purposes, and multiple mappings of the same page are 1827 * extremely uncommon in that scenario. 1828 */ 1829 for (pv = newpv->pv_next; pv != NULL; pv = pv->pv_next) { 1830 pmap_evcnt(pv_enter_ci_multi); 1831 ptep = pmap_pv_pte(pv); 1832 for (;;) { 1833 opte = pte_load(ptep); 1834 npte = pte_set_ci(opte); 1835 if (pte_update(ptep, opte, npte)) { 1836 if (active_pmap(pv->pv_pmap)) { 1837 TBIS(PV_VA(pv)); 1838 } 1839 break; 1840 } 1841 } 1842 } 1843 #if MMU_CONFIG_68040_CLASS 1844 if (MMU_IS_68040_CLASS) { 1845 const paddr_t pa = VM_PAGE_TO_PHYS(pg); 1846 DCFP(pa); 1847 ICPP(pa); 1848 } 1849 #endif 1850 } 1851 return; 1852 1853 #if MMU_CONFIG_HP_CLASS 1854 hp_mmu_vac_shenanigans: 1855 /* 1856 * We have ourselves a VAC, so in addition to checking for 1857 * user-requested-CI mappings, we have to check for cache 1858 * aliases and cache-inhibit all mappings for a page that 1859 * have a cache alias conflict. 1860 * 1861 * - All mappings of a given page within the same pmap must 1862 * not collide. (The VAC is flushed when switching pmaps 1863 * by virtue of a new segment table pointer being loaded 1864 * into the user segment table register.) 1865 * 1866 * - The Hibler pmap check to see that the kernel doesn't have 1867 * conflicting mappings with any user pmap. We'll do the same, 1868 * which seems reasonable on the surface if you think about it 1869 * for a couple of minutes. 1870 * 1871 * - The Hibler pmap also just punts and cache-inhibits all 1872 * mappings once it becomes > 2, but we do NOT do that because 1873 * it will severely penalize shared libraries. 1874 * 1875 * N.B. The method used here will not universally render all 1876 * mappings for a given page uncached; only address spaces with 1877 * conflicts are penalized. 1878 * 1879 * XXX This probably only matters if one of the mappings is 1880 * XXX writable, as this is the only situation where data 1881 * XXX inconsistency could arise. There is probably room 1882 * XXX for further optimization if someone with one of these 1883 * XXX machines cares to take it up. 1884 */ 1885 bool flush_s_vac = false; 1886 bool flush_u_vac = false; 1887 1888 /* Set the PTE for the new mapping. */ 1889 pte_store(ptep, npte); 1890 1891 /* 1892 * Invalidate the ATC entry **after** storing the PTE so that 1893 * there is no window where another MMU table walk finds the 1894 * stale invalid entry. 1895 * 1896 * XXX I don't know that this is strictly necessary with the 1897 * XXX HP MMU, but there is basically zero documentation available 1898 * XXX for it, so we err on the side of caution. 1899 */ 1900 if (active_pmap(pmap)) { 1901 TBIS(va); 1902 } 1903 1904 vaddr_t pv_flags = newpv->pv_vf & PV_F_CI_USR; 1905 if (usr_ci) { 1906 pmap_evcnt(pv_enter_usr_ci); 1907 } 1908 1909 for (pv = newpv->pv_next; pv != NULL; pv = pv->pv_next) { 1910 if (MATCHING_PMAP(pmap, pv->pv_pmap) && 1911 CONFLICTING_ALIAS(va, PV_VA(pv))) { 1912 pmap_evcnt(pv_enter_vac_ci); 1913 pv_flags |= PV_F_CI_VAC; 1914 break; 1915 } 1916 } 1917 1918 if (__predict_true(pv_flags == 0)) { 1919 /* No new inhibitions! */ 1920 return; 1921 } 1922 1923 VM_MDPAGE_SET_CI(pg); 1924 for (pv = VM_MDPAGE_PVS(pg); pv != NULL; pv = pv->pv_next) { 1925 if (MATCHING_PMAP(pmap, pv->pv_pmap)) { 1926 pmap_evcnt(pv_enter_ci_multi); 1927 pv->pv_vf |= pv_flags; 1928 pte_set(pmap_pv_pte(pv), PTE51_CI); 1929 if (active_pmap(pv->pv_pmap)) { 1930 TBIS(PV_VA(pv)); 1931 if (pv->pv_pmap == pmap_kernel()) { 1932 flush_s_vac = true; 1933 } else { 1934 flush_u_vac = true; 1935 } 1936 } 1937 } 1938 } 1939 if (flush_u_vac && flush_s_vac) { 1940 DCIA(); 1941 } else if (flush_u_vac) { 1942 DCIU(); 1943 } else if (flush_s_vac) { 1944 DCIS(); 1945 } 1946 #endif /* MMU_CONFIG_HP_CLASS */ 1947 } 1948 1949 /* 1950 * pmap_pv_remove: 1951 * 1952 * Remove a physical->virtual entry from the pv table. 1953 */ 1954 static void 1955 pmap_pv_remove(pmap_t pmap, struct vm_page *pg, vaddr_t va, 1956 struct pmap_completion *pc) 1957 { 1958 struct pv_entry **pvp, *pv; 1959 pt_entry_t *ptep, opte, npte; 1960 1961 pmap_evcnt(pv_remove_called); 1962 1963 PMAP_CRIT_ASSERT(); 1964 1965 for (pvp = VM_MDPAGE_HEAD_PVP(pg), pv = VM_MDPAGE_PVS(pg); 1966 pv != NULL; 1967 pvp = &pv->pv_next, pv = *pvp) { 1968 if (pmap == pv->pv_pmap && va == PV_VA(pv)) { 1969 break; 1970 } 1971 } 1972 1973 KASSERT(pv != NULL); 1974 VM_MDPAGE_SETPVP(pvp, pv->pv_next); 1975 LIST_REMOVE(pv, pv_pmlist); 1976 1977 KASSERT(pc != NULL); 1978 LIST_INSERT_HEAD(&pc->pc_pvlist, pv, pv_pmlist); 1979 1980 #if MMU_CONFIG_HP_CLASS 1981 if (MMU_IS_HP_CLASS) { 1982 /* Go handle the HP MMU's VAC. */ 1983 goto hp_mmu_vac_shenanigans; 1984 } 1985 #endif 1986 1987 /* 1988 * If the page is marked as being cache-inhibited, then it 1989 * means there was at least one user-requested CI mapping 1990 * for the page. In that case, we need to scan the P->V 1991 * list to see if any remain, and if not, clear the CI 1992 * status for the page. 1993 * 1994 * N.B. This requires traversing the list twice: once to 1995 * check if any of the mappings are user-requested-CI, 1996 * and one again to fix them up. But, we're making a 1997 * classical space-vs-time trade-off here: Assuming that 1998 * this is a rare situation, it's better to pay the cpu 1999 * cost on the rare edge transitions rather than always pay 2000 * the memory cost of having a counter to track something 2001 * that almost never happens (and, when it does, the list 2002 * will be very short). 2003 */ 2004 if (__predict_false(VM_MDPAGE_CI_P(pg))) { 2005 pmap_evcnt(pv_remove_ci); 2006 for (pv = VM_MDPAGE_PVS(pg); pv != NULL; pv = pv->pv_next) { 2007 if (pv->pv_vf & PV_F_CI_USR) { 2008 /* 2009 * There is still at least one user-requested 2010 * CI mapping, so we can't change the page's CI 2011 * status. 2012 */ 2013 return; 2014 } 2015 } 2016 KASSERT(pv == NULL); 2017 VM_MDPAGE_CLR_CI(pg); 2018 for (pv = VM_MDPAGE_PVS(pg); pv != NULL; pv = pv->pv_next) { 2019 ptep = pmap_pv_pte(pv); 2020 for (;;) { 2021 opte = pte_load(ptep); 2022 npte = pte_clr_ci(opte); 2023 if (pte_update(ptep, opte, npte)) { 2024 if (active_pmap(pv->pv_pmap)) { 2025 TBIS(PV_VA(pv)); 2026 } 2027 break; 2028 } 2029 } 2030 } 2031 } 2032 return; 2033 2034 #if MMU_CONFIG_HP_CLASS 2035 hp_mmu_vac_shenanigans: 2036 /* 2037 * If we have a VAC and the page was cache-inhibited due to 2038 * a cache alias conflict, we can re-enable the cache if there 2039 * is just one such mapping left. 2040 */ 2041 if (__predict_false(VM_MDPAGE_CI_P(pg))) { 2042 vaddr_t all_ci_flags = PV_F_CI_USR; 2043 2044 pmap_evcnt(pv_remove_ci); 2045 2046 for (pv = VM_MDPAGE_PVS(pg); pv != NULL; pv = pv->pv_next) { 2047 if (! MATCHING_PMAP(pmap, pv->pv_pmap)) { 2048 continue; 2049 } 2050 if (pv->pv_vf & all_ci_flags) { 2051 /* 2052 * There is at least one CI_USR mapping 2053 * or more than one CI_VAC mapping, so 2054 * the CI status of the page remains 2055 * unchanged. 2056 */ 2057 return; 2058 } 2059 all_ci_flags |= pv->pv_vf & PV_F_CI_VAC; 2060 } 2061 KASSERT(pv == NULL); 2062 /* 2063 * We now know we can remove CI from the page mappings 2064 * in the matching address space. If no CI mappings 2065 * remain, then we can clear the CI indicator on the 2066 * page. 2067 */ 2068 all_ci_flags = 0; 2069 for (pv = VM_MDPAGE_PVS(pg); pv != NULL; pv = pv->pv_next) { 2070 if (! MATCHING_PMAP(pmap, pv->pv_pmap)) { 2071 all_ci_flags |= pv->pv_vf; 2072 continue; 2073 } 2074 pte_mask(pmap_pv_pte(pv), ~((uint32_t)PTE51_CI)); 2075 if (active_pmap(pv->pv_pmap)) { 2076 TBIS(PV_VA(pv)); 2077 } 2078 } 2079 all_ci_flags &= PV_F_CI_USR | PV_F_CI_VAC; 2080 if (__predict_true(all_ci_flags == 0)) { 2081 VM_MDPAGE_CLR_CI(pg); 2082 } 2083 } 2084 #endif /* MMU_CONFIG_HP_CLASS */ 2085 } 2086 2087 #undef CONFLICTING_ALIAS 2088 #undef MATCHING_PMAP 2089 2090 /***************** PMAP INTERFACE (AND ADJACENT) FUNCTIONS *******************/ 2091 2092 static inline void 2093 pmap_stat_update_impl(long *valp, int val) 2094 { 2095 *valp += val; 2096 } 2097 2098 #define pmap_stat_update(pm, stat, delta) \ 2099 pmap_stat_update_impl(&(pm)->pm_stats.stat, (delta)) 2100 2101 static inline void 2102 pmap_stat_set_impl(long *valp, int val) 2103 { 2104 atomic_store_relaxed(valp, val); 2105 } 2106 2107 #define pmap_stat_set(pm, stat, val) \ 2108 pmap_stat_set_impl(&(pm)->pm_stats.stat, (val)) 2109 2110 /* 2111 * pmap_pinit: 2112 * 2113 * Common bits of pmap structure initialization shared between 2114 * the kernel pmap and user pmaps. 2115 */ 2116 static void 2117 pmap_pinit(pmap_t pmap, paddr_t lev1pa) 2118 { 2119 pmap->pm_lev1pa = lev1pa; 2120 rb_tree_init(&pmap->pm_tables, &pmap_table_rb_ops); 2121 TAILQ_INIT(&pmap->pm_ptpages[0]); 2122 TAILQ_INIT(&pmap->pm_ptpages[1]); 2123 LIST_INIT(&pmap->pm_pvlist); 2124 2125 pmap->pm_refcnt = 1; 2126 } 2127 2128 /* 2129 * pmap_virtual_space: [ INTERFACE ] 2130 * 2131 * Define the initial bounds of the kernel virtual address space. 2132 * 2133 * In this implementation, the start address we return marks the 2134 * end of the statically allocated special kernel virtual addresses 2135 * set up in pmap_bootstrap1(). We return kernel_virtual_max as 2136 * the end because we can grow the kernel address space using 2137 * pmap_growkernel(). 2138 */ 2139 void 2140 pmap_virtual_space(vaddr_t *vstartp, vaddr_t *vendp) 2141 { 2142 *vstartp = kernel_virtual_start; 2143 *vendp = kernel_virtual_max; 2144 } 2145 2146 /* 2147 * pmap_init: [ INTERFACE ] 2148 * 2149 * Initialize the pmap module. Called by vm_init(), to initialize any 2150 * structures that the pmap system needs to map virtual memory. 2151 */ 2152 void 2153 pmap_init(void) 2154 { 2155 /* Initialize the pmap / pv_entry allocators. */ 2156 pmap_alloc_init(); 2157 2158 /* Initialize the PT page allocator. */ 2159 pmap_ptpage_init(); 2160 2161 /* Now it's safe to do P->V entry recording! */ 2162 pmap_initialized_p = true; 2163 } 2164 2165 /* 2166 * pmap_create: [ INTERFACE ] 2167 * 2168 * Create and return a physical map. 2169 */ 2170 pmap_t 2171 pmap_create(void) 2172 { 2173 pmap_t pmap; 2174 2175 /* 2176 * We reference the null segment table and and have a NULL 2177 * lev1map pointer until the first mapping is entered. 2178 */ 2179 pmap = pmap_alloc(); 2180 pmap_pinit(pmap, null_segtab_pa); 2181 2182 return pmap; 2183 } 2184 2185 /* 2186 * pmap_destroy: [ INTERFACE ] 2187 * 2188 * Drop the reference count on the specified pmap, releasing 2189 * all resources if the reference count drops to zero. 2190 */ 2191 void 2192 pmap_destroy(pmap_t pmap) 2193 { 2194 unsigned int newval; 2195 2196 PMAP_CRIT_ENTER(); 2197 KASSERT(pmap->pm_refcnt > 0); 2198 newval = --pmap->pm_refcnt; 2199 PMAP_CRIT_EXIT(); 2200 2201 if (newval) { 2202 return; 2203 } 2204 2205 /* We assume all mappings have been removed. */ 2206 KASSERT(pmap->pm_lev1map == NULL); 2207 KASSERT(pmap->pm_lev1pa == null_segtab_pa); 2208 2209 pmap_free(pmap); 2210 } 2211 2212 /* 2213 * pmap_reference: [ INTERFACE ] 2214 * 2215 * Add a reference to the specified pmap. 2216 */ 2217 void 2218 pmap_reference(pmap_t pmap) 2219 { 2220 PMAP_CRIT_ENTER(); 2221 pmap->pm_refcnt++; 2222 KASSERT(pmap->pm_refcnt > 0); 2223 PMAP_CRIT_EXIT(); 2224 } 2225 2226 /* 2227 * pmap_remove_mapping: 2228 * 2229 * Invalidate a single page denoted by pmap/va. 2230 * 2231 * If (ptep != NULL), it is the already computed PTE for the mapping. 2232 * 2233 * If (flags & PRM_TFLUSH), we must invalidate any TLB information. 2234 * 2235 * If (flags & PRM_CFLUSH), we must flush/invalidate any cache 2236 * information. 2237 * 2238 * If the caller wishes to prevent the page table from being freed, 2239 * they should perform an extra retain. 2240 */ 2241 #define PRM_TFLUSH __BIT(0) 2242 #define PRM_CFLUSH __BIT(1) 2243 static void 2244 pmap_remove_mapping(pmap_t pmap, vaddr_t va, pt_entry_t *ptep, 2245 struct pmap_table *pt, struct vm_page *pg, 2246 int flags, struct pmap_completion *pc) 2247 { 2248 KASSERT(pt != NULL || pmap == pmap_kernel()); 2249 KASSERT(ptep != NULL); 2250 const paddr_t opte = pte_load(ptep); 2251 KASSERT(pte_valid_p(opte)); 2252 const paddr_t pa = pte_pa(opte); 2253 KASSERT(pg == NULL || pa == VM_PAGE_TO_PHYS(pg)); 2254 2255 /* Update statistics. */ 2256 if (pte_wired_p(opte)) { 2257 pmap_stat_update(pmap, wired_count, -1); 2258 } 2259 pmap_stat_update(pmap, resident_count, -1); 2260 2261 if (flags & PRM_CFLUSH) { 2262 #if MMU_CONFIG_68040_CLASS 2263 if (MMU_IS_68040_CLASS) { 2264 DCFP(pa); 2265 ICPP(pa); 2266 } 2267 #endif 2268 #if MMU_CONFIG_HP_CLASS 2269 if (MMU_IS_HP_CLASS) { 2270 if (pmap == pmap_kernel()) { 2271 DCIS(); 2272 } else if (active_user_pmap(pmap)) { 2273 DCIU(); 2274 } 2275 } 2276 #endif 2277 } 2278 2279 if (__predict_false(pg == NULL)) { 2280 pg = pmap_pa_to_pg(pa); 2281 if (pg != NULL) { 2282 pmap_evcnt(prm_lookup_pg_hit); 2283 } else { 2284 pmap_evcnt(prm_lookup_pg_miss); 2285 } 2286 } else { 2287 pmap_evcnt(prm_got_pg); 2288 } 2289 if (__predict_true(pg != NULL)) { 2290 KASSERT(pte_managed_p(opte)); 2291 /* Update cached U/M bits from mapping that's going away. */ 2292 VM_MDPAGE_ADD_UM(pg, opte); 2293 pmap_pv_remove(pmap, pg, va, pc); 2294 } else { 2295 KASSERT(! pte_managed_p(opte)); 2296 } 2297 2298 /* 2299 * Zap the PTE and drop the retain count that the mapping 2300 * had on the table. 2301 */ 2302 pte_store(ptep, 0); 2303 pmap_table_release(pmap, pt, pc); 2304 2305 /* 2306 * Now that the ATC can't be reloaded from the PTE, invalidate 2307 * the ATC entry. 2308 */ 2309 if (__predict_true((flags & PRM_TFLUSH) != 0 && active_pmap(pmap))) { 2310 TBIS(va); 2311 } 2312 } 2313 2314 /* 2315 * pmap_remove: [ INTERFACE ] 2316 * 2317 * Remove the given range of addresses from the specified map. 2318 * 2319 * It is assumed that the start and end are properly rounded 2320 * to the page size. 2321 * 2322 * N.B. Callers of pmap_remove_internal() are expected to 2323 * provide an initialized completion context, which we 2324 * will finalize. 2325 */ 2326 static void 2327 pmap_remove_internal(pmap_t pmap, vaddr_t sva, vaddr_t eva, 2328 struct pmap_completion *pc) 2329 { 2330 pt_entry_t opte, *ptep; 2331 struct pmap_table *pt; 2332 vaddr_t nextseg; 2333 int prm_flags; 2334 #if MMU_CONFIG_HP_CLASS 2335 pt_entry_t all_ci = PTE51_CI; 2336 #endif 2337 2338 /* 2339 * If this is the kernel pmap, we can use a faster method 2340 * for accessing the PTEs (since the PT pages are always 2341 * resident). 2342 * 2343 * Note that this routine should NEVER be called from an 2344 * interrupt context; pmap_kremove() is used for that. 2345 */ 2346 prm_flags = active_pmap(pmap) ? PRM_TFLUSH : 0; 2347 if (pmap == pmap_kernel()) { 2348 PMAP_CRIT_ENTER(pmap_busy(pmap)); 2349 2350 for (ptep = pmap_kernel_pte(sva); sva < eva; 2351 ptep++, sva += PAGE_SIZE) { 2352 opte = pte_load(ptep); 2353 if (pte_valid_p(opte)) { 2354 #if MMU_CONFIG_HP_CLASS 2355 /* 2356 * If all of the PTEs we're zapping have the 2357 * cache-inhibit bit set, ci_pte will remain 2358 * non-zero and we'll be able to skip flushing 2359 * the VAC when we're done. 2360 */ 2361 all_ci &= opte; 2362 #endif 2363 pmap_remove_mapping(pmap, sva, ptep, NULL, 2364 NULL, prm_flags, pc); 2365 } 2366 } 2367 #if MMU_CONFIG_HP_CLASS 2368 if (MMU_IS_HP_CLASS && !all_ci) { 2369 /* 2370 * Cacheable mappings were removed, so invalidate 2371 * the cache. 2372 */ 2373 DCIS(); 2374 } 2375 #endif 2376 PMAP_CRIT_EXIT(pmap_unbusy(pmap)); 2377 2378 /* kernel PT pages are never freed. */ 2379 KASSERT(TAILQ_EMPTY(&pc->pc_ptpages)); 2380 2381 /* ...but we might have freed PV entries. */ 2382 pmap_completion_fini(pc); 2383 2384 return; 2385 } 2386 2387 PMAP_CRIT_ENTER(pmap_busy(pmap)); 2388 2389 while (sva < eva) { 2390 nextseg = pmap_next_seg(sva); 2391 if (nextseg == 0 || nextseg > eva) { 2392 nextseg = eva; 2393 } 2394 2395 ptep = pmap_pte_lookup(pmap, sva, &pt); 2396 if (ptep == NULL) { 2397 /* 2398 * No table for this address, meaning nothing 2399 * within this segment; advance to the next 2400 * one. 2401 */ 2402 sva = nextseg; 2403 continue; 2404 } 2405 2406 for (; sva < nextseg; ptep++, sva += PAGE_SIZE) { 2407 opte = pte_load(ptep); 2408 if (! pte_valid_p(opte)) { 2409 continue; 2410 } 2411 #if MMU_CONFIG_HP_CLASS 2412 /* 2413 * If all of the PTEs we're zapping have the 2414 * cache-inhibit bit set, ci_pte will remain 2415 * non-zero and we'll be able to skip flushing 2416 * the VAC when we're done. 2417 */ 2418 all_ci &= opte; 2419 #endif 2420 pmap_remove_mapping(pmap, sva, ptep, pt, NULL, 2421 prm_flags, pc); 2422 } 2423 pmap_table_release(pmap, pt, pc); 2424 } 2425 #if MMU_CONFIG_HP_CLASS 2426 if (MMU_IS_HP_CLASS && !all_ci) { 2427 /* 2428 * Cacheable mappings were removed, so invalidate 2429 * the cache. 2430 */ 2431 if (pmap == pmap_kernel()) { 2432 DCIS(); 2433 } else if (active_user_pmap(pmap)) { 2434 DCIU(); 2435 } 2436 } 2437 #endif 2438 PMAP_CRIT_EXIT(pmap_unbusy(pmap)); 2439 2440 pmap_completion_fini(pc); 2441 } 2442 2443 void 2444 pmap_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva) 2445 { 2446 struct pmap_completion pc; 2447 pmap_completion_init(&pc); 2448 pmap_remove_internal(pmap, sva, eva, &pc); 2449 /* pmap_remove_internal() calls pmap_completion_fini(). */ 2450 } 2451 2452 /* 2453 * pmap_remove_all: [ INTERFACE ] 2454 * 2455 * Remove all mappings from a pmap in bulk. This is only called 2456 * when it's known that the address space is no longer visible to 2457 * any user process (e.g. during exit or exec). 2458 */ 2459 bool 2460 pmap_remove_all(pmap_t pmap) 2461 { 2462 struct pmap_completion pc; 2463 struct pv_entry *pv; 2464 struct vm_page *pg; 2465 pt_entry_t opte; 2466 2467 KASSERT(pmap != pmap_kernel()); 2468 2469 /* 2470 * This process is pretty simple: 2471 * 2472 * ==> (1) Set the segment table pointer to the NULL segment table. 2473 * 2474 * ==> (2) Copy the PT page list to a tempory list and re-init. 2475 * 2476 * ==> (3) Walk the PV entry list and remove each entry. 2477 * 2478 * ==> (4) Zero the wired and resident count. 2479 * 2480 * Once we've done that, we just need to free everything 2481 * back to the system. 2482 */ 2483 2484 pmap_completion_init(&pc); 2485 2486 PMAP_CRIT_ENTER(pmap_busy(pmap)); 2487 2488 /* Step 1. */ 2489 pmap_set_lev1map(pmap, NULL, null_segtab_pa); 2490 2491 /* Step 2. */ 2492 pmap->pm_pt_cache = NULL; 2493 TAILQ_CONCAT(&pc.pc_ptpages, &pmap->pm_ptpages[0], ptp_list); 2494 TAILQ_CONCAT(&pc.pc_ptpages, &pmap->pm_ptpages[1], ptp_list); 2495 memset(&pmap->pm_tables, 0, sizeof(pmap->pm_tables)); 2496 rb_tree_init(&pmap->pm_tables, &pmap_table_rb_ops); 2497 KASSERT(RB_TREE_MIN(&pmap->pm_tables) == NULL); 2498 2499 /* Step 3. */ 2500 while ((pv = LIST_FIRST(&pmap->pm_pvlist)) != NULL) { 2501 KASSERT(pv->pv_pmap == pmap); 2502 opte = pte_load(pmap_pv_pte(pv)); 2503 pg = pmap_pa_to_pg(pte_pa(opte)); 2504 /* Update cached U/M bits from mapping that's going away. */ 2505 VM_MDPAGE_ADD_UM(pg, opte); 2506 pmap_pv_remove(pmap, pg, PV_VA(pv), &pc); 2507 } 2508 2509 /* Step 4. */ 2510 pmap_stat_set(pmap, wired_count, 0); 2511 pmap_stat_set(pmap, resident_count, 0); 2512 2513 PMAP_CRIT_EXIT(pmap_unbusy(pmap)); 2514 2515 pmap_completion_fini(&pc); 2516 2517 return true; 2518 } 2519 2520 /* 2521 * pmap_page_protect: [ INTERFACE ] 2522 * 2523 * Lower the permission for all mappings to a given page to 2524 * the permissions specified. 2525 */ 2526 void 2527 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 2528 { 2529 struct pmap_completion pc; 2530 struct pv_entry *pv; 2531 2532 if (prot & UVM_PROT_WRITE) { 2533 /* No protection to revoke. */ 2534 return; 2535 } 2536 2537 if (prot & UVM_PROT_READ) { 2538 /* Making page copy-on-write. */ 2539 pmap_changebit(pg, PTE_WP, ~0U); 2540 return; 2541 } 2542 2543 /* Removing all mappings for a page. */ 2544 pmap_completion_init(&pc); 2545 2546 PMAP_CRIT_ENTER(); 2547 2548 while ((pv = VM_MDPAGE_PVS(pg)) != NULL) { 2549 pmap_busy(pv->pv_pmap); 2550 pmap_remove_mapping(pv->pv_pmap, PV_VA(pv), pmap_pv_pte(pv), 2551 pv->pv_pt, pg, PRM_TFLUSH|PRM_CFLUSH, &pc); 2552 pmap_unbusy(pv->pv_pmap); 2553 } 2554 2555 PMAP_CRIT_EXIT(); 2556 2557 pmap_completion_fini(&pc); 2558 } 2559 2560 /* 2561 * pmap_protect: [ INTERFACE ] 2562 * 2563 * Set the physical protection on the specified range of this map 2564 * as requested. 2565 */ 2566 void 2567 pmap_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 2568 { 2569 pt_entry_t *ptep, opte, npte; 2570 struct pmap_table *pt; 2571 vaddr_t nextseg; 2572 #if MMU_CONFIG_68040_CLASS 2573 bool removing_write; 2574 #endif 2575 bool need_tflush; 2576 2577 if ((prot & UVM_PROT_READ) == 0) { 2578 struct pmap_completion pc; 2579 pmap_completion_init(&pc); 2580 pmap_remove_internal(pmap, sva, eva, &pc); 2581 /* pmap_remove_internal() calls pmap_completion_fini(). */ 2582 return; 2583 } 2584 2585 PMAP_CRIT_ENTER(pmap_busy(pmap)); 2586 2587 #if MMU_CONFIG_68040_CLASS 2588 removing_write = (prot & UVM_PROT_WRITE) == 0; 2589 #endif 2590 need_tflush = active_pmap(pmap); 2591 while (sva < eva) { 2592 nextseg = pmap_next_seg(sva); 2593 if (nextseg == 0 || nextseg > eva) { 2594 nextseg = eva; 2595 } 2596 2597 ptep = pmap_pte_lookup(pmap, sva, &pt); 2598 if (ptep == NULL) { 2599 /* 2600 * No table for this address, meaning nothing 2601 * within this segment; advance to the next 2602 * one. 2603 */ 2604 sva = nextseg; 2605 continue; 2606 } 2607 2608 /* 2609 * Change protection on mapping if it is valid and doesn't 2610 * already have the correct protection. 2611 */ 2612 for (; sva < nextseg; ptep++, sva += PAGE_SIZE) { 2613 try_again: 2614 opte = pte_load(ptep); 2615 if (! pte_valid_p(opte)) { 2616 continue; 2617 } 2618 npte = pte_change_prot(opte, prot); 2619 if (npte == opte) { 2620 continue; 2621 } 2622 #if MMU_CONFIG_68040_CLASS 2623 if (MMU_IS_68040_CLASS && removing_write) { 2624 /* 2625 * Clear caches if making RO (see section 2626 * "7.3 Cache Coherency" in the manual). 2627 */ 2628 paddr_t pa = pte_pa(opte); 2629 DCFP(pa); 2630 ICPP(pa); 2631 } 2632 #endif 2633 if (! pte_update(ptep, opte, npte)) { 2634 /* Lost race updating PTE; try again. */ 2635 goto try_again; 2636 } 2637 if (need_tflush) { 2638 TBIS(sva); 2639 } 2640 } 2641 pmap_table_release(pmap, pt, NULL); 2642 } 2643 2644 PMAP_CRIT_EXIT(pmap_unbusy(pmap)); 2645 } 2646 2647 /* 2648 * pmap_enter: [ INTERFACE ] 2649 * 2650 * Insert the given physical address (pa) at the specified 2651 * virtual address (va) in the target physical map with the 2652 * protection requested. 2653 * 2654 * If specified, the page will be wired down, meaning that 2655 * related pte can not be reclaimed. 2656 * 2657 * Note: This is the only routine which MAY NOT lazy-evaluate 2658 * or lose information. That is, this routine must actually 2659 * insert this page into the given map NOW. 2660 */ 2661 int 2662 pmap_enter(pmap_t pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 2663 { 2664 struct pmap_table *pt; 2665 pt_entry_t *ptep, npte, opte; 2666 struct pv_entry *newpv; 2667 struct pmap_completion pc; 2668 int error = 0; 2669 const bool nowait = !!(flags & PMAP_CANFAIL); 2670 2671 pmap_completion_init(&pc); 2672 2673 struct vm_page * const pg = pmap_pa_to_pg(pa); 2674 if (__predict_false(pg == NULL)) { 2675 /* 2676 * PA is not part of managed memory. Make the mapping 2677 * cache-inhibited on the assumption that it's a device. 2678 */ 2679 flags |= PMAP_NOCACHE; 2680 } 2681 2682 PMAP_CRIT_ENTER(pmap_busy(pmap)); 2683 2684 if (nowait) { 2685 pmap_evcnt(enter_nowait); 2686 } else { 2687 pmap_evcnt(enter_yeswait); 2688 } 2689 2690 /* Get the destination table. */ 2691 ptep = pmap_pte_alloc(pmap, va, &pt, nowait, &pc); 2692 if (__predict_false(ptep == NULL)) { 2693 pmap_evcnt(enter_pte_alloc_fail); 2694 error = ENOMEM; 2695 goto out; 2696 } 2697 2698 /* Compute the new PTE. */ 2699 npte = pmap_make_pte(pa, prot, flags); 2700 2701 /* Fetch old PTE. */ 2702 opte = pte_load(ptep); 2703 2704 /* 2705 * Check to see if there is a valid mapping at this address. 2706 * It might simply be a wiring or protection change. 2707 */ 2708 if (pte_valid_p(opte)) { 2709 pmap_evcnt(enter_valid); 2710 restart: 2711 if (pte_pa(opte) == pa) { 2712 /* 2713 * Just a protection or wiring change. 2714 * 2715 * Since the old PTE is handy, go ahead and update 2716 * the cached U/M attributes now. Normally we would 2717 * do this in pmap_remove_mapping(), but we're not 2718 * taking that path in this case. We also add in 2719 * any U/M attributes hinted by the access type 2720 * that brought us to pmap_enter() in the first 2721 * place (a write-fault on a writable page mapped 2722 * read-only during a page-out, for example). 2723 * 2724 * Also ensure that the PV list status of the mapping 2725 * is consistent. 2726 */ 2727 if (__predict_true(pg != NULL)) { 2728 VM_MDPAGE_ADD_UM(pg, opte | npte); 2729 KASSERT(pte_managed_p(opte)); 2730 npte |= PTE_PVLIST; 2731 } 2732 2733 /* Preserve cache-inhibited status. */ 2734 if (__predict_false(pte_ci_p(opte))) { 2735 npte = 2736 (npte & ~PTE_CMASK) | (opte & PTE_CMASK); 2737 } 2738 2739 /* Set the new PTE. */ 2740 pte_store(ptep, npte); 2741 2742 const pt_entry_t diff = opte ^ npte; 2743 2744 #ifdef PMAP_EVENT_COUNTERS 2745 if (diff & PTE_WIRED) { 2746 pmap_evcnt(enter_wire_change); 2747 } 2748 if (diff & PTE_WP) { 2749 pmap_evcnt(enter_prot_change); 2750 } 2751 #endif 2752 2753 if (pte_wired_p(diff)) { 2754 pmap_stat_update(pmap, wired_count, 2755 pte_wired_p(npte) ? 1 : -1); 2756 } 2757 if (diff) { 2758 #if MMU_CONFIG_68040_CLASS 2759 /* 2760 * Protection or caching status is changing; 2761 * flush the page from the cache. 2762 */ 2763 if (MMU_IS_68040_CLASS) { 2764 DCFP(pa); 2765 ICPP(pa); 2766 } 2767 #endif 2768 if (active_pmap(pmap)) { 2769 TBIS(va); 2770 #if MMU_CONFIG_HP_CLASS 2771 /* 2772 * If the new mapping is CI and the old 2773 * one is not, then flush the VAC. 2774 */ 2775 if (__predict_false(MMU_IS_HP_CLASS && 2776 pte_ci_p(diff) && 2777 pte_ci_p(npte))) { 2778 DCIA(); 2779 } 2780 #endif 2781 } 2782 } 2783 2784 /* All done! */ 2785 goto out_release; 2786 } 2787 2788 /* 2789 * The mapping has completely changed. Need to remove 2790 * the old one first. 2791 * 2792 * This drops the retain count on the PT owned by the 2793 * previous mapping, but the newly-entered mapping will 2794 * inherit the retain count taken when we looked up the 2795 * PTE. 2796 * 2797 * XXX Can we elide the ATC flush here? We're going to 2798 * XXX hit the ATC after setting the new PTE anyway. 2799 */ 2800 pmap_evcnt(enter_pa_change); 2801 pmap_remove_mapping(pmap, va, ptep, pt, NULL, 2802 PRM_TFLUSH|PRM_CFLUSH, &pc); 2803 } 2804 2805 /* 2806 * By the time we get here, we should be assured that the 2807 * PTE at ptep is invalid. 2808 */ 2809 KASSERT(! pte_valid_p(pte_load(ptep))); 2810 2811 /* Update pmap stats now. */ 2812 pmap_stat_update(pmap, resident_count, 1); 2813 if (__predict_false(pte_wired_p(npte))) { 2814 pmap_stat_update(pmap, wired_count, 1); 2815 } 2816 2817 if (__predict_true(pg != NULL)) { 2818 /* 2819 * Managed pages also go on the PV list, so we are 2820 * going to need a PV entry. 2821 */ 2822 newpv = LIST_FIRST(&pc.pc_pvlist); 2823 if (__predict_true(newpv == NULL)) { 2824 /* 2825 * No PV entry to recycle; allocate a new one. 2826 * Because this is an extremely common case, we 2827 * are first going to attempt allocation while 2828 * still in the critical section. If that fails 2829 * and waiting is allowed, we'll leave the critical 2830 * section and try a blocking allocation. 2831 */ 2832 newpv = pmap_pv_alloc(true/*nowait flag*/); 2833 if (__predict_false(newpv == NULL)) { 2834 if (nowait) { 2835 pmap_evcnt(enter_pv_alloc_fail); 2836 error = ENOMEM; 2837 goto out_release; 2838 } 2839 PMAP_CRIT_EXIT(); 2840 newpv = pmap_pv_alloc(false/*nowait flag*/); 2841 KASSERT(newpv != NULL); 2842 PMAP_CRIT_ENTER(); 2843 /* 2844 * Because we may have blocked while allocating 2845 * the PV entry, we have to re-validate our 2846 * environment, as another thread could have 2847 * inserted a mapping here behind our back. 2848 */ 2849 opte = pte_load(ptep); 2850 if (__predict_false(pte_valid_p(opte))) { 2851 pmap_stat_update(pmap, 2852 resident_count, -1); 2853 if (pte_wired_p(npte)) { 2854 pmap_stat_update(pmap, 2855 wired_count, -1); 2856 } 2857 LIST_INSERT_HEAD(&pc.pc_pvlist, 2858 newpv, pv_pmlist); 2859 goto restart; 2860 } 2861 } 2862 } else { 2863 pmap_evcnt(enter_pv_recycle); 2864 LIST_REMOVE(newpv, pv_pmlist); 2865 } 2866 2867 /* 2868 * Enter the mapping into the PV list. pmap_pv_enter() 2869 * will also set the PTE in the table. 2870 */ 2871 pmap_pv_enter(pmap, pg, va, prot, pt, npte, newpv); 2872 2873 /* 2874 * The new mapping takes ownership of the PT 2875 * retain count we took while looking up the PTE. 2876 */ 2877 goto out_crit_exit; 2878 } 2879 2880 /* 2881 * Not a managed mapping, so set the new PTE. As with managed 2882 * mappings, the new mapping takes ownership of the PT retain 2883 * count we took while looking up the PTE. 2884 */ 2885 pte_store(ptep, npte); 2886 2887 /* 2888 * See comments in pmap_pv_enter() for why we have to hit 2889 * the ATC here. 2890 */ 2891 if (active_pmap(pmap)) { 2892 TBIS(va); 2893 } 2894 goto out_crit_exit; 2895 2896 out_release: 2897 pmap_table_release(pmap, pt, &pc); 2898 out_crit_exit: 2899 PMAP_CRIT_EXIT(pmap_unbusy(pmap)); 2900 out: 2901 pmap_completion_fini(&pc); 2902 return error; 2903 } 2904 2905 /* 2906 * pmap_kenter_pa: [ INTERFACE ] 2907 * 2908 * Enter a va -> pa mapping into the kernel pmap without any 2909 * physical->virtual tracking. 2910 */ 2911 void 2912 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 2913 { 2914 pmap_t const pmap = pmap_kernel(); 2915 2916 KASSERT(va >= VM_MIN_KERNEL_ADDRESS); 2917 2918 pt_entry_t * const ptep = pmap_kernel_pte(va); 2919 2920 /* Build the new PTE. */ 2921 const pt_entry_t npte = pmap_make_pte(pa, prot, flags | PMAP_WIRED); 2922 2923 /* 2924 * If this is an EXEC mapping, then we have to ensure that 2925 * the I$ doesn't load stale data. 2926 */ 2927 if (__predict_false(prot & UVM_PROT_EXEC)) { 2928 #if MMU_CONFIG_68040_CLASS 2929 if (MMU_IS_68040_CLASS) { 2930 /* 2931 * XXX Potential future optimization: is only 2932 * XXX the DCFP() needed here to deal with 2933 * XXX write-back? 2934 */ 2935 DCFP(pa); 2936 ICPP(pa); 2937 } 2938 #endif 2939 } 2940 2941 /* There must not be a valid PTE here. */ 2942 KASSERT(! pte_valid_p(pte_load(ptep))); 2943 2944 /* Set the new PTE. */ 2945 pte_store(ptep, npte); 2946 2947 pmap_stat_update(pmap, resident_count, 1); 2948 pmap_stat_update(pmap, wired_count, 1); 2949 2950 /* 2951 * See comments in pmap_pv_enter() as for why we hit the ATC here. 2952 * This *should* be unnecessary because this is a wired kernel 2953 * mapping and no demand-page-ins should have happened at this 2954 * VA, but we're erring on the side of caution for now. 2955 */ 2956 TBIS(va); 2957 } 2958 2959 /* 2960 * pmap_kremove: [ INTERFACE ] 2961 * 2962 * Remove a mapping entered with pmap_kenter_pa() starting at va, 2963 * for size bytes (assumed to be page rounded). 2964 */ 2965 void 2966 pmap_kremove(vaddr_t va, vsize_t size) 2967 { 2968 pt_entry_t *ptep, opte; 2969 pmap_t const pmap = pmap_kernel(); 2970 int count = 0; 2971 #if MMU_CONFIG_HP_CLASS 2972 pt_entry_t all_ci = PTE51_CI; 2973 #endif 2974 2975 KASSERT(va >= VM_MIN_KERNEL_ADDRESS); 2976 2977 for (ptep = pmap_kernel_pte(va); size != 0; 2978 ptep++, size -= PAGE_SIZE, va += PAGE_SIZE) { 2979 opte = pte_load(ptep); 2980 if (pte_valid_p(opte)) { 2981 KASSERT(! pte_managed_p(opte)); 2982 KASSERT(pte_wired_p(opte)); 2983 #if MMU_CONFIG_HP_CLASS 2984 /* 2985 * If all of the PTEs we're zapping have the 2986 * cache-inhibit bit set, ci_pte will remain 2987 * non-zero and we'll be able to skip flushing 2988 * the VAC when we're done. 2989 */ 2990 all_ci &= opte; 2991 #endif 2992 /* Zap the mapping. */ 2993 pte_store(ptep, 0); 2994 TBIS(va); 2995 count++; 2996 } 2997 } 2998 #if MMU_CONFIG_HP_CLASS 2999 if (MMU_IS_HP_CLASS && !all_ci) { 3000 /* 3001 * Cacheable mappings were removed, so invalidate 3002 * the cache. 3003 */ 3004 DCIS(); 3005 } 3006 #endif 3007 /* Update stats. */ 3008 if (__predict_true(count != 0)) { 3009 pmap_stat_update(pmap, resident_count, -count); 3010 pmap_stat_update(pmap, wired_count, -count); 3011 } 3012 } 3013 3014 /* 3015 * pmap_unwire: [ INTERFACE ] 3016 * 3017 * Clear the wired attribute for a map/virtual-address pair. 3018 * 3019 * The mapping must already exist in the pmap. 3020 */ 3021 void 3022 pmap_unwire(pmap_t pmap, vaddr_t va) 3023 { 3024 struct pmap_table *pt; 3025 pt_entry_t opte, npte, *ptep; 3026 3027 PMAP_CRIT_ENTER(pmap_busy(pmap)); 3028 3029 ptep = pmap_pte_lookup(pmap, va, &pt); 3030 KASSERT(ptep != NULL); 3031 3032 for (;;) { 3033 opte = pte_load(ptep); 3034 KASSERT(pte_valid_p(opte)); 3035 3036 /* 3037 * If the wiring actually changed (always?), clear the wire 3038 * bit and update the wire count. Note that the wiring is 3039 * not a hardware characteristic so there is no need to 3040 * invalidate the ATC. 3041 */ 3042 if (! pte_wired_p(opte)) { 3043 break; 3044 } 3045 npte = opte & ~PTE_WIRED; 3046 if (pte_update(ptep, opte, npte)) { 3047 pmap_stat_update(pmap, wired_count, -1); 3048 break; 3049 } 3050 } 3051 3052 pmap_table_release(pmap, pt, NULL); 3053 3054 PMAP_CRIT_EXIT(pmap_unbusy(pmap)); 3055 } 3056 3057 /* 3058 * pmap_extract: [ INTERFACE ] 3059 * 3060 * Extract the physical address associated with the given 3061 * pmap/virtual address pair. 3062 * 3063 * pmap_extract_info: 3064 * 3065 * Like pmap_extract(), but also returns information 3066 * about the mapping (wired, cache-inhibited, etc.) 3067 */ 3068 bool 3069 pmap_extract_info(pmap_t pmap, vaddr_t va, paddr_t *pap, int *flagsp) 3070 { 3071 struct pmap_table *pt; 3072 pt_entry_t pte, *ptep; 3073 bool rv = false; 3074 3075 if (__predict_false(pmap == pmap_kernel() && 3076 va >= kernel_virtual_end)) { 3077 return false; 3078 } 3079 3080 PMAP_CRIT_ENTER(pmap_busy(pmap)); 3081 3082 ptep = pmap_pte_lookup(pmap, va, &pt); 3083 if (__predict_true(ptep != NULL)) { 3084 pte = pte_load(ptep); 3085 if (__predict_true(pte_valid_p(pte))) { 3086 if (__predict_true(pap != NULL)) { 3087 *pap = pte_pa(pte) | (va & PAGE_MASK); 3088 } 3089 if (__predict_false(flagsp != NULL)) { 3090 *flagsp = 3091 (pte_wired_p(pte) ? PMAP_WIRED : 0) | 3092 (pte_ci_p(pte) ? PMAP_NOCACHE : 0); 3093 } 3094 rv = true; 3095 } 3096 pmap_table_release(pmap, pt, NULL); 3097 } 3098 3099 PMAP_CRIT_EXIT(pmap_unbusy(pmap)); 3100 3101 return rv; 3102 } 3103 3104 bool 3105 pmap_extract(pmap_t pmap, vaddr_t va, paddr_t *pap) 3106 { 3107 return pmap_extract_info(pmap, va, pap, NULL); 3108 } 3109 3110 /* 3111 * vtophys: 3112 * 3113 * Dumber version of pmap_extract(pmap_kernel(), ...) 3114 */ 3115 paddr_t 3116 vtophys(vaddr_t va) 3117 { 3118 paddr_t pa; 3119 bool rv __diagused; 3120 3121 rv = pmap_extract_info(pmap_kernel(), va, &pa, NULL); 3122 KASSERT(rv); 3123 return rv ? pa : -1; 3124 } 3125 3126 /* 3127 * kvtop: 3128 * 3129 * Sigh. 3130 */ 3131 int 3132 kvtop(void *v) 3133 { 3134 return (int)vtophys((vaddr_t)v); 3135 } 3136 3137 /* 3138 * pmap_copy: [ INTERFACE ] 3139 * 3140 * Copy the mapping range specified by src_addr/len 3141 * from the source map to the range dst_addr/len 3142 * in the destination map. 3143 * 3144 * This routine is only advisory and need not do anything. 3145 */ 3146 /* call deleted in <machine/pmap.h> */ 3147 3148 /* 3149 * pmap_update: [ INTERFACE ] 3150 * 3151 * Require that all active physical maps contain no 3152 * incorrect entries NOW, by processing any deferred 3153 * pmap operations. 3154 */ 3155 /* call deleted in <machine/pmap.h> */ 3156 3157 /* 3158 * pmap_activate: [ INTERFACE ] 3159 * 3160 * Activate the pmap used by the specified process. This includes 3161 * reloading the MMU context of the current process, and marking 3162 * the pmap in use by the processor. 3163 */ 3164 void 3165 pmap_activate(struct lwp *l) 3166 { 3167 pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap; 3168 3169 KASSERT(l == curlwp); 3170 3171 /* 3172 * Because the kernel has a separate root pointer, we don't 3173 * need to activate the kernel pmap. 3174 */ 3175 if (pmap != pmap_kernel()) { 3176 PMAP_CRIT_ENTER(pmap_busy(pmap)); 3177 pmap_load_urp(pmap->pm_lev1pa); 3178 PMAP_CRIT_EXIT(); 3179 } 3180 } 3181 3182 /* 3183 * pmap_deactivate: [ INTERFACE ] 3184 * 3185 * Mark that the pmap used by the specified process is no longer 3186 * in use by the processor. 3187 */ 3188 void 3189 pmap_deactivate(struct lwp *l) 3190 { 3191 pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap; 3192 3193 if (pmap != pmap_kernel()) { 3194 PMAP_CRIT_ENTER(); 3195 PMAP_CRIT_EXIT(pmap_unbusy(pmap)); 3196 } 3197 } 3198 3199 static vaddr_t pmap_tmpmap_srcva; 3200 static vaddr_t pmap_tmpmap_dstva; 3201 3202 /* 3203 * pmap_zero_page: [ INTERFACE ] 3204 * 3205 * Zero the specified VM page by mapping the page into the kernel 3206 * and using memset() (or equivalent) to clear its contents. 3207 */ 3208 void 3209 pmap_zero_page(paddr_t pa) 3210 { 3211 const int flags = MMU_IS_HP_CLASS ? PMAP_NOCACHE|PMAP_WIRED 3212 : PMAP_WIRED; 3213 pt_entry_t * const dst_ptep = pmap_kernel_pte(pmap_tmpmap_dstva); 3214 3215 /* Build the new PTE. */ 3216 const pt_entry_t dst_pte = 3217 pmap_make_pte(pa, UVM_PROT_READ | UVM_PROT_WRITE, flags); 3218 3219 /* Set the new PTE. */ 3220 KASSERT(! pte_valid_p(pte_load(dst_ptep))); 3221 pte_store(dst_ptep, dst_pte); 3222 /* XXX Possibly being over-cautious here; see pmap_kenter_pa(). */ 3223 TBIS(pmap_tmpmap_dstva); 3224 3225 /* Zero the page. */ 3226 zeropage((void *)pmap_tmpmap_dstva); 3227 3228 /* Invalidate the PTEs. */ 3229 pte_store(dst_ptep, 0); 3230 TBIS(pmap_tmpmap_dstva); 3231 } 3232 3233 /* 3234 * pmap_copy_page: [ INTERFACE ] 3235 * 3236 * Copy the specified VM page by mapping the page(s) into the kernel 3237 * and using memcpy() (or equivalent). 3238 */ 3239 void 3240 pmap_copy_page(paddr_t src, paddr_t dst) 3241 { 3242 const int flags = MMU_IS_HP_CLASS ? PMAP_NOCACHE|PMAP_WIRED 3243 : PMAP_WIRED; 3244 pt_entry_t * const src_ptep = pmap_kernel_pte(pmap_tmpmap_srcva); 3245 pt_entry_t * const dst_ptep = pmap_kernel_pte(pmap_tmpmap_dstva); 3246 3247 /* Build the new PTEs. */ 3248 const pt_entry_t src_pte = 3249 pmap_make_pte(src, UVM_PROT_READ, flags); 3250 const pt_entry_t dst_pte = 3251 pmap_make_pte(dst, UVM_PROT_READ | UVM_PROT_WRITE, flags); 3252 3253 /* Set the new PTEs. */ 3254 KASSERT(! pte_valid_p(pte_load(src_ptep))); 3255 pte_store(src_ptep, src_pte); 3256 /* XXX Possibly being over-cautious here; see pmap_kenter_pa(). */ 3257 TBIS(pmap_tmpmap_srcva); 3258 3259 KASSERT(! pte_valid_p(pte_load(dst_ptep))); 3260 pte_store(dst_ptep, dst_pte); 3261 /* XXX Possibly being over-cautious here; see pmap_kenter_pa(). */ 3262 TBIS(pmap_tmpmap_dstva); 3263 3264 /* Copy the page. */ 3265 copypage((void *)pmap_tmpmap_srcva, (void *)pmap_tmpmap_dstva); 3266 3267 /* Invalidate the PTEs. */ 3268 pte_store(src_ptep, 0); 3269 TBIS(pmap_tmpmap_srcva); 3270 3271 pte_store(dst_ptep, 0); 3272 TBIS(pmap_tmpmap_dstva); 3273 } 3274 3275 /* 3276 * pmap_testbit: 3277 * 3278 * Test the modified / referenced bits of a physical page. 3279 */ 3280 static bool 3281 pmap_testbit(struct vm_page *pg, pt_entry_t bit) 3282 { 3283 struct pv_entry *pv; 3284 pt_entry_t combined_pte = 0; 3285 3286 PMAP_CRIT_ENTER(); 3287 3288 combined_pte = VM_MDPAGE_UM(pg); 3289 3290 for (pv = VM_MDPAGE_PVS(pg); 3291 (combined_pte & bit) == 0 && pv != NULL; 3292 pv = pv->pv_next) { 3293 pmap_busy(pv->pv_pmap); 3294 combined_pte |= pte_load(pmap_pv_pte(pv)); 3295 pmap_unbusy(pv->pv_pmap); 3296 } 3297 3298 VM_MDPAGE_ADD_UM(pg, combined_pte); 3299 3300 PMAP_CRIT_EXIT(); 3301 3302 return (combined_pte & bit) != 0; 3303 } 3304 3305 /* 3306 * pmap_is_referenced: [ INTERFACE ] 3307 * 3308 * Return whether or not the specified physical page has been referenced 3309 * by any physical maps. 3310 */ 3311 bool 3312 pmap_is_referenced(struct vm_page *pg) 3313 { 3314 return pmap_testbit(pg, PTE_U); 3315 } 3316 3317 /* 3318 * pmap_is_modified: [ INTERFACE ] 3319 * 3320 * Return whether or not the specified physical page has been modified 3321 * by any physical maps. 3322 */ 3323 bool 3324 pmap_is_modified(struct vm_page *pg) 3325 { 3326 return pmap_testbit(pg, PTE_M); 3327 } 3328 3329 /* 3330 * pmap_changebit: 3331 * 3332 * Test-and-change various bits (including mod/ref bits). 3333 * Returns the accumulated previously-set PTE bits. 3334 */ 3335 static pt_entry_t 3336 pmap_changebit(struct vm_page *pg, pt_entry_t set, pt_entry_t mask) 3337 { 3338 struct pv_entry *pv; 3339 pt_entry_t *ptep, combined_pte, opte, npte; 3340 3341 #if MMU_CONFIG_68040_CLASS 3342 /* 3343 * If we're making the page read-only or changing the caching 3344 * status of the page, we need to flush it the first time we 3345 * change a mapping. 3346 */ 3347 bool cflush_040; 3348 if (MMU_IS_68040_CLASS && 3349 ((set & PTE_CRIT_BITS) != 0 || 3350 (mask & PTE_CRIT_BITS) == 0)) { 3351 cflush_040 = true; 3352 } else { 3353 cflush_040 = false; 3354 } 3355 #endif 3356 3357 PMAP_CRIT_ENTER(); 3358 3359 /* 3360 * Since we need to report if the page was mod/ref'd before 3361 * we cleared the bit, we need to seed ourself with the current 3362 * state in the vm_page in the event there are no mappings 3363 * left to enumerate. 3364 */ 3365 combined_pte = VM_MDPAGE_UM(pg); 3366 3367 /* 3368 * Since we're running over every mapping for the page anyway, 3369 * we might as well synchronize any attribute bits that we're 3370 * not clearing. 3371 */ 3372 for (pv = VM_MDPAGE_PVS(pg); pv != NULL; pv = pv->pv_next) { 3373 pmap_busy(pv->pv_pmap); 3374 for (;;) { 3375 ptep = pmap_pv_pte(pv); 3376 opte = pte_load(ptep); 3377 npte = (opte | set) & mask; 3378 combined_pte |= opte; 3379 3380 if (opte == npte) { 3381 break; 3382 } 3383 #if MMU_CONFIG_68040_CLASS 3384 if (__predict_false(cflush_040)) { 3385 paddr_t pa = VM_PAGE_TO_PHYS(pg); 3386 DCFP(pa); 3387 ICPP(pa); 3388 cflush_040 = false; 3389 } 3390 #endif 3391 if (pte_update(ptep, opte, npte)) { 3392 break; 3393 } 3394 /* Lost race, try again. */ 3395 } 3396 /* 3397 * We must flush the ATC even if it's not a "critical" bit, 3398 * because the MMU will only write-back a PTE when the 3399 * U or M bits transition from 0 to 1. If we clear them 3400 * in our tracking structure and in the tables in memory 3401 * but not the ATC, then a sticky ATC entry might cause 3402 * a future U or M event to be missed. 3403 */ 3404 if (active_pmap(pv->pv_pmap)) { 3405 TBIS(PV_VA(pv)); 3406 } 3407 pmap_unbusy(pv->pv_pmap); 3408 } 3409 3410 /* 3411 * Update any attributes we looked at, clear the ones we're clearing. 3412 */ 3413 VM_MDPAGE_SET_UM(pg, (combined_pte | set) & mask); 3414 3415 PMAP_CRIT_EXIT(); 3416 3417 return combined_pte; 3418 } 3419 3420 /* 3421 * pmap_clear_modify: [ INTERFACE ] 3422 * 3423 * Clear the modify bits on the specified physical page. 3424 */ 3425 bool 3426 pmap_clear_modify(struct vm_page *pg) 3427 { 3428 return (pmap_changebit(pg, 0, (pt_entry_t)~PTE_M) & PTE_M) != 0; 3429 } 3430 3431 /* 3432 * pmap_clear_reference: [ INTERFACE ] 3433 * 3434 * Clear the reference bit on the specified physical page. 3435 */ 3436 bool 3437 pmap_clear_reference(struct vm_page *pg) 3438 { 3439 return (pmap_changebit(pg, 0, (pt_entry_t)~PTE_U) & PTE_U) != 0; 3440 } 3441 3442 /* 3443 * pmap_phys_address: [ INTERFACE ] 3444 * 3445 * Return the physical address corresponding to the specified 3446 * cookie. Used by the device pager to decode a device driver's 3447 * mmap entry point return value. 3448 */ 3449 paddr_t 3450 pmap_phys_address(paddr_t cookie) 3451 { 3452 return m68k_ptob(cookie); 3453 } 3454 3455 static pt_entry_t *kernel_lev1map; 3456 3457 /* 3458 * pmap_growkernel_alloc_page: 3459 * 3460 * Helper for pmap_growkernel(). 3461 */ 3462 static paddr_t 3463 pmap_growkernel_alloc_page(void) 3464 { 3465 /* 3466 * XXX Needs more work if we're going to do this during 3467 * XXX early bootstrap. 3468 */ 3469 if (! uvm.page_init_done) { 3470 panic("%s: called before UVM initialized", __func__); 3471 } 3472 3473 struct vm_page *pg = pmap_page_alloc(true/*nowait*/); 3474 if (pg == NULL) { 3475 panic("%s: out of memory", __func__); 3476 } 3477 3478 paddr_t pa = VM_PAGE_TO_PHYS(pg); 3479 pmap_zero_page(pa); 3480 #if MMU_CONFIG_68040_CLASS 3481 if (MMU_IS_68040_CLASS) { 3482 DCFP(pa); 3483 } 3484 #endif 3485 return pa; 3486 } 3487 3488 /* 3489 * pmap_growkernel_link_kptpage: 3490 * 3491 * Helper for pmap_growkernel(). 3492 */ 3493 static void 3494 pmap_growkernel_link_kptpage(vaddr_t va, paddr_t ptp_pa) 3495 { 3496 /* 3497 * This is trivial for the 2-level MMU configuration. 3498 */ 3499 if (MMU_USE_2L) { 3500 KASSERT((kernel_lev1map[LA2L_RI(va)] & DT51_SHORT) == 0); 3501 kernel_lev1map[LA2L_RI(va)] = pmap_ste_proto | ptp_pa; 3502 return; 3503 } 3504 3505 /* 3506 * N.B. pmap_zero_page() is used in this process, which 3507 * uses pmap_tmpmap_dstva. pmap_tmpmap_srcva is available 3508 * for our use, however, so that's what we used to temporarily 3509 * map inner segment table pages. 3510 */ 3511 const vaddr_t stpg_va = pmap_tmpmap_srcva; 3512 3513 paddr_t stpa, stpg_pa, stpgoff, last_stpg_pa = (paddr_t)-1; 3514 paddr_t pa = ptp_pa, end_pa = ptp_pa + PAGE_SIZE; 3515 pt_entry_t *stes; 3516 3517 for (; pa < end_pa; va += NBSEG3L, pa += TBL40_L3_SIZE) { 3518 if ((kernel_lev1map[LA40_RI(va)] & UTE40_RESIDENT) == 0) { 3519 /* Level-2 table for this segment needed. */ 3520 if (kernel_stnext_pa == kernel_stnext_endpa) { 3521 /* 3522 * No more slots left in the last page 3523 * we allocated for segment tables. Grab 3524 * another one. 3525 */ 3526 kernel_stnext_pa = pmap_growkernel_alloc_page(); 3527 kernel_stnext_endpa = 3528 kernel_stnext_pa + PAGE_SIZE; 3529 pmap_nkstpages_current_ev.ev_count++; 3530 } 3531 kernel_lev1map[LA40_RI(va)] = 3532 pmap_ste_proto | kernel_stnext_pa; 3533 kernel_stnext_pa += TBL40_L2_SIZE; 3534 } 3535 stpa = kernel_lev1map[LA40_RI(va)] & UTE40_PTA; 3536 stpg_pa = m68k_trunc_page(stpa); 3537 if (stpg_pa != last_stpg_pa) { 3538 if (last_stpg_pa != (paddr_t)-1) { 3539 pmap_kremove(stpg_va, PAGE_SIZE); 3540 } 3541 pmap_kenter_pa(stpg_va, stpg_pa, 3542 UVM_PROT_READ | UVM_PROT_WRITE, 3543 PMAP_WIRED | PMAP_NOCACHE); 3544 last_stpg_pa = stpg_pa; 3545 } 3546 stpgoff = stpa - stpg_pa; 3547 stes = (pt_entry_t *)(stpg_va + stpgoff); 3548 stes[LA40_PI(va)] = pmap_ste_proto | pa; 3549 } 3550 if (last_stpg_pa != (paddr_t)-1) { 3551 pmap_kremove(stpg_va, PAGE_SIZE); 3552 } 3553 } 3554 3555 /* 3556 * pmap_growkernel: [ INTERFACE ] 3557 * 3558 * Grow the kernel address space. This is a hint from the 3559 * upper layer to pre-allocate more kernel PT pages. 3560 */ 3561 vaddr_t 3562 pmap_growkernel(vaddr_t maxkvaddr) 3563 { 3564 PMAP_CRIT_ENTER(); 3565 3566 KASSERT((kernel_virtual_end & PTPAGEVAOFS) == 0); 3567 3568 vaddr_t new_maxkva = pmap_round_ptpage(maxkvaddr); 3569 if (new_maxkva < kernel_virtual_end) { 3570 /* 3571 * Great news! We already have what we need to map 3572 * the requested max address. This happens one during 3573 * early bootstrap before UVM's notion of "maxkvaddr" 3574 * has been initialized. 3575 */ 3576 new_maxkva = kernel_virtual_end; 3577 goto done; 3578 } 3579 3580 if (new_maxkva > kernel_virtual_max) { 3581 panic("%s: out of kernel VA space (req=0x%08lx limit=0x%08lx)", 3582 __func__, maxkvaddr, kernel_virtual_max); 3583 } 3584 3585 /* 3586 * Allocate PT pages and link them into the MMU tree as we 3587 * go. 3588 */ 3589 vaddr_t va, ptp_pa; 3590 for (va = kernel_virtual_end; va < new_maxkva; va += PTPAGEVASZ) { 3591 /* Allocate page and link it into the MMU tree. */ 3592 ptp_pa = pmap_growkernel_alloc_page(); 3593 pmap_growkernel_link_kptpage(va, ptp_pa); 3594 pmap_nkptpages_current_ev.ev_count++; 3595 3596 /* Map the PT page into the kernel PTE array. */ 3597 pmap_kenter_pa((vaddr_t)pmap_kernel_pte(va), 3598 ptp_pa, UVM_PROT_READ | UVM_PROT_WRITE, 3599 PMAP_WIRED | PMAP_NOCACHE); 3600 } 3601 kernel_virtual_end = new_maxkva; 3602 done: 3603 pmap_maxkva_ev.ev_count32 = new_maxkva; 3604 pmap_kvalimit_ev.ev_count32 = kernel_virtual_max; 3605 PMAP_CRIT_EXIT(); 3606 return new_maxkva; 3607 } 3608 3609 /* 3610 * pmap_prefer: [ INTERFACE ] 3611 * 3612 * Attempt to arrange for pages at a given VM object offset 3613 * to occupy the same virtually-addressed cache footprint 3614 * in order to avoid cache aliases. 3615 */ 3616 #if MMU_CONFIG_HP_CLASS 3617 static struct evcnt pmap_prefer_nochange_ev = 3618 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap prefer", "nochange"); 3619 static struct evcnt pmap_prefer_change_ev = 3620 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap prefer", "change"); 3621 3622 EVCNT_ATTACH_STATIC(pmap_prefer_change_ev); 3623 EVCNT_ATTACH_STATIC(pmap_prefer_nochange_ev); 3624 #endif 3625 void 3626 pmap_prefer(vaddr_t hint, vaddr_t *vap, int td) 3627 { 3628 #if MMU_CONFIG_HP_CLASS 3629 if (MMU_IS_HP_CLASS) { 3630 vaddr_t va = *vap; 3631 ptrdiff_t diff = (hint - va) & pmap_aliasmask; 3632 3633 if (diff == 0) { 3634 pmap_prefer_nochange_ev.ev_count++; 3635 } else { 3636 pmap_prefer_change_ev.ev_count++; 3637 if (__predict_false(td)) { 3638 va -= pmap_aliasmask + 1; 3639 } 3640 *vap = va + diff; 3641 } 3642 } 3643 #endif 3644 } 3645 3646 /* 3647 * pmap_procwr: [ INTERFACE ] 3648 * 3649 * Perform any cache synchronization required after writing 3650 * to a process's address space. 3651 */ 3652 void 3653 pmap_procwr(struct proc *p, vaddr_t va, size_t len) 3654 { 3655 /* 3656 * This is just a wrapper around the "cachectl" machdep 3657 * system call. 3658 * 3659 * XXX This is kind of gross, to be honest. 3660 */ 3661 (void)cachectl1(0x80000004, va, len, p); 3662 } 3663 3664 #if defined(DDB) || defined(KGDB) 3665 /* 3666 * pmap_db_write_text_enter: 3667 * 3668 * Temporarily map a page of kernel text read-write for the 3669 * kernel debugger. 3670 */ 3671 bool 3672 pmap_db_write_text_enter(vaddr_t pgva, struct pmap_db_write_text_context *ctx) 3673 { 3674 pt_entry_t *pte = pmap_kernel_pte(pgva); 3675 3676 #ifdef M68K_MMU_HP 3677 /* 3678 * Flush the supervisor side of the VAC to prevent 3679 * a cache hit on the old, read-only PTE. 3680 * 3681 * XXX Is this really necessary? 3682 */ 3683 if (MMU_IS_HP_CLASS) { 3684 DCIS(); 3685 } 3686 #endif 3687 3688 pt_entry_t opte = pte_load(pte); 3689 if (! pte_valid_p(opte)) { 3690 return false; 3691 } 3692 3693 /* 3694 * N.B. we use the 68851 PTE bit names here, but in 3695 * the case of the kernel text, it all works out vis 3696 * a vis the 68040 PTE bits. 3697 * 3698 * Note the mapping is cache-inhibited to save hair. 3699 */ 3700 pt_entry_t npte = (opte & ~PTE51_WP) | PTE51_CI; 3701 pte_store(pte, npte); 3702 TBIS(pgva); 3703 3704 ctx->pgva = pgva; 3705 ctx->ptep = pte; 3706 ctx->opte = opte; 3707 3708 return true; 3709 } 3710 3711 /* 3712 * pmap_db_write_text_exit: 3713 * 3714 * Undo the effects of pmap_db_write_text_enter(). 3715 */ 3716 void 3717 pmap_db_write_text_exit(struct pmap_db_write_text_context *ctx) 3718 { 3719 pte_store(ctx->ptep, ctx->opte); 3720 TBIS(ctx->pgva); 3721 } 3722 #endif /* DDB || KGDB */ 3723 3724 static paddr_t kernel_reloc_offset; 3725 static vaddr_t kernel_reloc_end; 3726 3727 /* 3728 * pmap_init_kcore_hdr: 3729 * 3730 * Initialize the m68k kernel crash dump header with information 3731 * necessary to perform KVA -> phys translations. 3732 * 3733 * Returns a pointer to the crash dump RAM segment entries for 3734 * machine-specific code to initialize. 3735 */ 3736 phys_ram_seg_t * 3737 pmap_init_kcore_hdr(cpu_kcore_hdr_t *h) 3738 { 3739 struct gen68k_kcore_hdr *m = &h->un._gen68k; 3740 3741 memset(h, 0, sizeof(*h)); 3742 3743 /* 3744 * Initialize the `dispatcher' portion of the header. 3745 */ 3746 strcpy(h->name, "gen68k"); 3747 h->page_size = PAGE_SIZE; 3748 h->kernbase = VM_MIN_KERNEL_ADDRESS; 3749 3750 /* Fixed relocation information. */ 3751 m->reloc = kernel_reloc_offset; 3752 m->relocend = kernel_reloc_end; 3753 3754 /* 3755 * Fill in information about our MMU configuration. 3756 * 3757 * We essentially pretend to be a 68851 as far as table- 3758 * walks are concerned. 3759 * 3760 * We provide the kernel's MMU_* constant so that the TT 3761 * registers can be interpreted correctly. 3762 */ 3763 m->mmutype = mmutype; 3764 m->tcr = MMU_USE_3L ? MMU51_3L_TCR_BITS : MMU51_TCR_BITS; 3765 m->srp[0] = MMU51_SRP_BITS; 3766 m->srp[1] = Sysseg_pa; 3767 3768 #if MMU_CONFIG_68040_CLASS 3769 if (MMU_IS_68040_CLASS) { 3770 m->itt0 = mmu_tt40[MMU_TTREG_ITT0]; 3771 m->itt1 = mmu_tt40[MMU_TTREG_ITT1]; 3772 m->tt0 = mmu_tt40[MMU_TTREG_DTT0]; 3773 m->tt1 = mmu_tt40[MMU_TTREG_DTT1]; 3774 } 3775 #endif 3776 #if defined(M68K_MMU_68030) 3777 if (mmutype == MMU_68030) { 3778 m->tt0 = mmu_tt30[MMU_TTREG_TT0]; 3779 m->tt1 = mmu_tt30[MMU_TTREG_TT1]; 3780 } 3781 #endif 3782 3783 return m->ram_segs; 3784 } 3785 3786 /***************************** PMAP BOOTSTRAP ********************************/ 3787 3788 /* 3789 * pmap_pa_has_static_mapping: 3790 * 3791 * Returns true if the specified PA (and length) has a static 3792 * mapping, and what that mapping is and its properties. 3793 */ 3794 bool 3795 pmap_pa_has_static_mapping(paddr_t pa, size_t len, vm_prot_t prot, 3796 vaddr_t *vap, int *flagsp) 3797 { 3798 #if defined(M68030) || defined(M68040) || defined(M68060) 3799 /* First, check for Transparent Translation. */ 3800 if (cputype >= CPU_68030 && 3801 mmu_range_is_tt(pa, len, prot, flagsp)) { 3802 *vap = pa; 3803 return true; 3804 } 3805 #endif 3806 3807 const struct pmap_bootmap *pmbm = machine_bootmap; 3808 paddr_t lastpg = m68k_btop(pa + (len - 1)); 3809 paddr_t firstpg = m68k_btop(pa); 3810 paddr_t tfirst, tlast; 3811 vaddr_t tblva; 3812 bool need_write = !!(prot & UVM_PROT_WRITE); 3813 3814 for (; pmbm->pmbm_vaddr != (vaddr_t)-1; pmbm++) { 3815 if (pmbm->pmbm_flags & (PMBM_F_VAONLY|PMBM_F_KEEPOUT)) { 3816 continue; 3817 } 3818 if (pmbm->pmbm_flags & PMBM_F_FIXEDVA) { 3819 tblva = pmbm->pmbm_vaddr; 3820 } else { 3821 tblva = *pmbm->pmbm_vaddr_ptr; 3822 } 3823 tfirst = m68k_btop(pmbm->pmbm_paddr); 3824 tlast = m68k_btop(pmbm->pmbm_paddr + (pmbm->pmbm_size - 1)); 3825 3826 if (firstpg >= tfirst && lastpg <= tlast) { 3827 if (need_write && 3828 (pmbm->pmbm_flags & PMBM_F_RO) != 0) { 3829 return false; 3830 } 3831 *vap = tblva + (pa - pmbm->pmbm_paddr); 3832 if (pmbm->pmbm_flags & PMBM_F_CI) { 3833 *flagsp = PMAP_NOCACHE; 3834 } else { 3835 *flagsp = 0; 3836 } 3837 return true; 3838 } 3839 } 3840 3841 return false; 3842 } 3843 3844 /* 3845 * pmap_va_is_static_mapping: 3846 * 3847 * Returns true if the specified VA (and length) is a static 3848 * mapping. 3849 */ 3850 bool 3851 pmap_va_is_static_mapping(vaddr_t va, size_t len) 3852 { 3853 #if defined(M68030) || defined(M68040) || defined(M68060) 3854 int map_flags; 3855 3856 /* First, check for Transparent Translation. */ 3857 if (cputype >= CPU_68030 && 3858 (mmu_range_is_tt(va, len, UVM_PROT_READ, &map_flags) || 3859 mmu_range_is_tt(va, len, UVM_PROT_WRITE, &map_flags) || 3860 mmu_range_is_tt(va, len, UVM_PROT_EXEC, &map_flags))) { 3861 return true; 3862 } 3863 #endif 3864 3865 const struct pmap_bootmap *pmbm = machine_bootmap; 3866 vaddr_t lastpg = m68k_btop(va + (len - 1)); 3867 vaddr_t firstpg = m68k_btop(va); 3868 vaddr_t tblva, tfirst, tlast; 3869 for (; pmbm->pmbm_vaddr != (vaddr_t)-1; pmbm++) { 3870 if (pmbm->pmbm_flags & (PMBM_F_VAONLY|PMBM_F_KEEPOUT)) { 3871 continue; 3872 } 3873 if (pmbm->pmbm_flags & PMBM_F_FIXEDVA) { 3874 tblva = pmbm->pmbm_vaddr; 3875 } else { 3876 tblva = *pmbm->pmbm_vaddr_ptr; 3877 } 3878 tfirst = m68k_btop(tblva); 3879 tlast = m68k_btop(tblva + (pmbm->pmbm_size - 1)); 3880 3881 if (firstpg >= tfirst && lastpg <= tlast) { 3882 return true; 3883 } 3884 } 3885 3886 return false; 3887 } 3888 3889 /* 3890 * The kernel virtual address space layout that this implementation is tuned 3891 * for assumes that KVA space begins at $0000.0000, that the static kernel 3892 * image (text/data/bss, etc.) resides at or near the bottom of this space, 3893 * and that all additional KVA that's mapped by PTEs grows upwards from there. 3894 * 3895 * Regions mapped by Transparent Translation registers (68030 and up) 3896 * are assumed to lie beyond where the KVA space is expected to grow. When 3897 * we encounter these regions in the machine_bootmap[] (represented by a 3898 * KEEPOUT entry), we clamp the maximum KVA to prevent its growth into that 3899 * region. The TT mechanism is not terribly precise, and only supports 3900 * VA==PA mappings, so it's only really suitable for device regions that 3901 * are in the upper reaches of the physical address space (at or beyond 1GB 3902 * or so). 3903 * 3904 * This implementation certainly could be adjusted to work with other address 3905 * space layouts, but the assumption asserted here is a bit baked-in. 3906 */ 3907 __CTASSERT(VM_MIN_KERNEL_ADDRESS == 0); 3908 3909 /* 3910 * The virtual kernel PTE array covers the entire 4GB kernel supervisor 3911 * address space, but is sparsely populated. The amount of VA space required 3912 * for this linear array is: 3913 * 3914 * (4GB / PAGE_SIZE) * sizeof(pt_entry_t) 3915 * -or- 3916 * 4KB: 4MB (1024 pages) 3917 * 8KB: 2MB (512 pages) 3918 * 3919 * To avoid doing 64-bit math, we calculate it like so: 3920 * 3921 * ((0xffffffff >> PAGE_SHIFT) + 1) * sizeof(pt_entry_t) 3922 * 3923 * The traditional name for this virtual array is "Sysmap". 3924 */ 3925 #define SYSMAP_VA_SIZE (((0xffffffffU >> PAGE_SHIFT) + 1) * sizeof(pt_entry_t)) 3926 3927 /* 3928 * In the Hibler/Utah pmap, the kernel PTE array was placed right near 3929 * the very top of the kernel virtual address space. This was because 3930 * of the hp300's unique physical memory arrangement: the last page of 3931 * memory is always located at PA $FFFF.F000 and the physical address 3932 * of the beginning of RAM varied based on the RAM size. This meant that 3933 * VA $FFFF.F000 is a convenient place to map the RAM VA==PA, making 3934 * transition between "MMU off" and "MMU on" (and vice versa) easier. 3935 * Since VA $FFFF.F000 was already going to be mapped, it made sense to 3936 * put something else along side of it in order to minimize waste in 3937 * PT pages. 3938 * 3939 * As noted above, this implementation is tuned for a growing-from-0 3940 * virtual space layout. However, we have a special case for this 3941 * particular requirement: if a platform defines SYSMAP_VA, then we 3942 * will assume it is as a high address, place the kernel PTE array at 3943 * that KVA, and ensure sufficient page tables to map from that VA until 3944 * the very end of the 4GB supervisor address space. These tables will 3945 * be allocated before the machine_bootmap[] is processed to map physical 3946 * addresses, thus allowing the machine_bootmap[] use it to map physical 3947 * addresses into one of these high virtual addresses if necessary. The 3948 * beginning of this region will also serve to clamp the maximum kernel 3949 * virtual address, in the same way as a KEEPOUT region in machine_bootmap[]. 3950 * 3951 * For reference, the traditional hp300 definition is: 3952 * 3953 * #define SYSMAP_VA ((vaddr_t)(0-PAGE_SIZE*NPTEPG*2)) 3954 * 3955 * ...and because the hp300 always used a 4KB page size (restriction 3956 * of HP MMU), this is: 0 - 4096*1024*2 3957 * -> 0 - 8388608 (8MB) 3958 * -> $FF80.0000 3959 * 3960 * Unfortunately (for the hp300), this means 2 PT pages for the top of 3961 * the address space (in the 2-level case), but that's unavoidable anyway 3962 * because of the last page being a separate mapping and the kernel PTE 3963 * array needs 4MB of space on its own. 3964 */ 3965 3966 static vaddr_t lwp0uarea; 3967 char * vmmap; 3968 3969 /* XXX Doesn't belong here. */ 3970 paddr_t avail_start; /* PA of first available physical page */ 3971 paddr_t avail_end; /* PA of last available physical page */ 3972 3973 extern char * kernel_text; 3974 extern char * etext; 3975 extern void * msgbufaddr; 3976 3977 /* 3978 * pmap_bootstrap1: 3979 * 3980 * Phase 1 of bootstrapping virtual memory. This is called before 3981 * the MMU is enabled to set up the initial kernel MMU tables and 3982 * allocate other important data structures. 3983 * 3984 * Because the MMU has not yet been turned on, and we don't know if 3985 * we're running VA==PA, we have to manually relocate all global 3986 * symbol references. 3987 * 3988 * Arguments: nextpa Physical address immediately 3989 * following the kernel / symbols / 3990 * etc. This will be page-rounded 3991 * before use. 3992 * 3993 * reloff VA<->PA relocation offset 3994 * 3995 * Returns: nextpa Updated value after all of the 3996 * allocations performed. 3997 */ 3998 paddr_t __attribute__((no_instrument_function)) 3999 pmap_bootstrap1(paddr_t nextpa, paddr_t reloff) 4000 { 4001 paddr_t lwp0upa, stnext_endpa, stnext_pa; 4002 paddr_t pa, kernimg_endpa, kern_lev1pa; 4003 vaddr_t va, nextva, kern_lev1va; 4004 pt_entry_t *pte, *epte; 4005 const struct pmap_bootmap *pmbm; 4006 int entry_count = 0; 4007 4008 #ifdef SYSMAP_VA 4009 #define VA_RANGE_DEFAULT 0 4010 #define VA_RANGE_KPTES 1 4011 #define INIT_NRANGES 2 4012 #else 4013 #define VA_RANGE_DEFAULT 0 4014 #define VA_RANGE_KPTES 0 4015 #define INIT_NRANGES 1 4016 #endif 4017 #define MAX_RANGES 8 4018 4019 struct va_range { 4020 vaddr_t start_va; 4021 vaddr_t end_va; 4022 paddr_t start_ptp; 4023 paddr_t end_ptp; 4024 } va_ranges[MAX_RANGES], *var; 4025 int nranges = INIT_NRANGES, r; 4026 4027 #define VA_IN_RANGE(va, var) \ 4028 ((va) >= (var)->start_va && \ 4029 ((va) < (var)->end_va || (var)->end_va == 0)) 4030 4031 #define VA_PTE_BASE(va, var) \ 4032 (&((pt_entry_t *) \ 4033 PMAP_BOOTSTRAP_RELOC_PA((var)->start_ptp))[ \ 4034 m68k_btop((va) - (var)->start_va)]) 4035 4036 #define RELOC(v, t) *((t *)PMAP_BOOTSTRAP_RELOC_GLOB(&(v))) 4037 4038 /* Record the relocation offset for kernel crash dumps. */ 4039 RELOC(kernel_reloc_offset, paddr_t) = reloff; 4040 4041 /* 4042 * First determination we have to make is our configuration: 4043 * Are we using a 2-level or 3-level table? For the purposes 4044 * of bootstrapping the kernel, it's "68040-class" and "other", 4045 * the former getting the 3-level table. 4046 */ 4047 const bool is_68040_class = RELOC(mmutype, int) == MMU_68040 || 4048 RELOC(mmutype, int) == MMU_68060; 4049 const bool use_3l = is_68040_class; 4050 4051 /* 4052 * Based on MMU class, figure out what the constant values of 4053 * segment / page table entries look like. 4054 * 4055 * See pmap_pte_proto_init(). 4056 */ 4057 pt_entry_t proto_ro_pte; /* read-only */ 4058 pt_entry_t proto_rw_pte; /* read-write */ 4059 pt_entry_t proto_rw_cwt_pte; /* read-write, cache-write-through */ 4060 pt_entry_t proto_rw_ci_pte; /* read-write, cache-inhibited */ 4061 pt_entry_t proto_ste; 4062 4063 if (is_68040_class) { 4064 proto_ro_pte = PTE_VALID|PTE_WIRED|PTE_WP|PTE40_CM_WT; 4065 proto_rw_pte = PTE_VALID|PTE_WIRED |PTE40_CM_CB; 4066 proto_rw_cwt_pte = PTE_VALID|PTE_WIRED |PTE40_CM_WT; 4067 proto_rw_ci_pte = PTE_VALID|PTE_WIRED |PTE40_CM_NC_SER; 4068 } else { 4069 proto_ro_pte = PTE_VALID|PTE_WIRED|PTE_WP; 4070 proto_rw_pte = PTE_VALID|PTE_WIRED; 4071 proto_rw_cwt_pte = PTE_VALID|PTE_WIRED; 4072 proto_rw_ci_pte = PTE_VALID|PTE_WIRED |PTE51_CI; 4073 } 4074 proto_ste = DTE51_U | DT51_SHORT; 4075 4076 /* 4077 * Allocate some important fixed virtual (and physical) addresses. 4078 * We use the sum total of this initial mapped kernel space to 4079 * determine how many initial kernel PT pages to allocate. The 4080 * things that consume physical space will come first, and the 4081 * virtual-space-{only,mostly} things come at the end. 4082 * 4083 * lwp0upa lwp0 u-area USPACE (p) 4084 * lwp0uarea (v) 4085 * 4086 * Sysseg_pa kernel lev1map PAGE_SIZE (p) 4087 * kernel_lev1map PAGE_SIZE (v, ci) 4088 * 4089 * ^^^^ end of simple relocation region ^^^^ 4090 * 4091 * null_segtab_pa null segtab PAGE_SIZE (p) 4092 * 4093 * tmpmap_srcva temp map, src PAGE_SIZE (v) 4094 * tmpmap_dstva temp map, dst PAGE_SIZE (v) 4095 * 4096 * vmmap ya tmp map PAGE_SIZE (v) 4097 * 4098 * msgbufaddr kernel msg buf round_page(MSGBUFSIZE) (v) 4099 * 4100 * kernel_ptes kernel PTEs SYSMAP_VA_SIZE (v, ci) 4101 * (see comments above) 4102 * 4103 * When we allocate the kernel lev1map, for the 2-level 4104 * configuration, there is no inner segment tables to allocate, 4105 * the leaf PT pages get poked directly into the level-1 table. 4106 * 4107 * In the 3-level configuration, to map all of the leaf tables, 4108 * inner segment table pages are allocated as necessary. We 4109 * first take those tables from the page containing the level-1 4110 * table, and allocate additional pages as necessary. 4111 */ 4112 4113 nextpa = m68k_round_page(nextpa); 4114 nextva = PMAP_BOOTSTRAP_PA_TO_VA(nextpa); 4115 4116 /* 4117 * nextpa now represents the end of the loaded kernel image. 4118 * This includes the .data + .bss segments, the debugger symbols, 4119 * and any other ancillary data loaded after the kernel. 4120 * 4121 * N.B. This represents the start of our dynamic memory allocation, 4122 * which will be referenced below when we zero the memory we've 4123 * allocated. 4124 */ 4125 kernimg_endpa = nextpa; 4126 4127 /* 4128 * lwp0 u-area. We allocate it here, and finish setting it 4129 * up in pmap_bootstrap2(). 4130 */ 4131 lwp0upa = nextpa; 4132 nextpa += USPACE; 4133 RELOC(lwp0uarea, vaddr_t) = nextva; 4134 nextva += USPACE; 4135 4136 size_t nstpages = 0; 4137 4138 /* kernel level-1 map */ 4139 RELOC(Sysseg_pa, paddr_t) = kern_lev1pa = nextpa; 4140 nextpa += PAGE_SIZE; 4141 RELOC(kernel_lev1map, vaddr_t) = kern_lev1va = nextva; 4142 nextva += PAGE_SIZE; 4143 nstpages++; 4144 4145 /* This is the end of the simple relocation region. */ 4146 RELOC(kernel_reloc_end, vaddr_t) = nextva; 4147 4148 /* 4149 * For 3-level configs, we now have space to allocate 4150 * inner segment tables. 4151 */ 4152 stnext_pa = kern_lev1pa + TBL40_L1_SIZE; 4153 stnext_endpa = m68k_round_page(stnext_pa); 4154 4155 /* null segment table */ 4156 #ifdef NULL_SEGTAB_PA 4157 RELOC(null_segtab_pa, paddr_t) = (paddr_t)NULL_SEGTAB_PA; 4158 #else 4159 RELOC(null_segtab_pa, paddr_t) = nextpa; 4160 nextpa += PAGE_SIZE; 4161 #endif 4162 4163 /* pmap temporary map addresses */ 4164 RELOC(pmap_tmpmap_srcva, vaddr_t) = nextva; 4165 nextva += PAGE_SIZE; 4166 RELOC(pmap_tmpmap_dstva, vaddr_t) = nextva; 4167 nextva += PAGE_SIZE; 4168 4169 /* vmmap temporary map address */ 4170 RELOC(vmmap, vaddr_t) = nextva; 4171 nextva += PAGE_SIZE; 4172 4173 /* kernel message buffer */ 4174 RELOC(msgbufaddr, vaddr_t) = nextva; 4175 nextva += m68k_round_page(MSGBUFSIZE); 4176 4177 /* Kernel PTE array. */ 4178 #ifdef SYSMAP_VA 4179 if ((vaddr_t)SYSMAP_VA < RELOC(kernel_virtual_max, vaddr_t)) { 4180 RELOC(kernel_virtual_max, vaddr_t) = (vaddr_t)SYSMAP_VA; 4181 } 4182 RELOC(kernel_ptes, vaddr_t) = (vaddr_t)SYSMAP_VA; 4183 va_ranges[VA_RANGE_KPTES].start_va = (vaddr_t)SYSMAP_VA; 4184 va_ranges[VA_RANGE_KPTES].end_va = 0; /* end of the address space */ 4185 #else 4186 RELOC(kernel_ptes, vaddr_t) = nextva; 4187 nextva += SYSMAP_VA_SIZE; 4188 #endif /* SYSMAP_VA */ 4189 4190 /* 4191 * Look through the machine_bootmap looking for any non-fixed 4192 * VAs we need to allocate. These will go into the default 4193 * VA range. 4194 */ 4195 pmbm = (const struct pmap_bootmap *) 4196 PMAP_BOOTSTRAP_RELOC_GLOB(machine_bootmap); 4197 for (; pmbm->pmbm_vaddr != (vaddr_t)-1; pmbm++) { 4198 if (pmbm->pmbm_size == 0) { 4199 continue; 4200 } 4201 if (pmbm->pmbm_flags & (PMBM_F_FIXEDVA | PMBM_F_KEEPOUT)) { 4202 va = m68k_trunc_page(pmbm->pmbm_vaddr); 4203 if (va < RELOC(kernel_virtual_max, vaddr_t)) { 4204 RELOC(kernel_virtual_max, vaddr_t) = va; 4205 } 4206 } else { 4207 *(vaddr_t *) 4208 PMAP_BOOTSTRAP_RELOC_GLOB(pmbm->pmbm_vaddr_ptr) = 4209 nextva; 4210 nextva += m68k_round_page(pmbm->pmbm_size); 4211 } 4212 } 4213 4214 /* UVM-managed kernel virtual starts here. */ 4215 RELOC(kernel_virtual_start, vaddr_t) = nextva; 4216 4217 /* 4218 * Allocate enough PT pages to map all of physical memory. 4219 * This should be sufficient to prevent pmap_growkernel() 4220 * from having to do any work before the VM system is set 4221 * up. 4222 * 4223 * XXX mac68k also relies on being able to map the last page 4224 * XXX of RAM VA==PA for the mmu-switchoff dance. Unlike hp300, 4225 * XXX this is not at a fixed location. However, RAM generally 4226 * XXX starts at $0000.0000 on Macs, so this calculation should 4227 * XXX be sufficient to ensure there is a PTE available for this 4228 * XXX purpose. 4229 * XXX TODO: Provide a way for cpu_startup() on mac68k to assert 4230 * XXX this (export kernel_virtual_end?). 4231 */ 4232 nextva += RELOC(physmem, psize_t) << PAGE_SHIFT; 4233 nextva = pmap_round_ptpage(nextva); 4234 if (nextva > RELOC(kernel_virtual_max, vaddr_t) || 4235 nextva < RELOC(kernel_virtual_start, vaddr_t)) { 4236 /* clamp it. */ 4237 nextva = RELOC(kernel_virtual_max, vaddr_t); 4238 } 4239 4240 /* 4241 * This marks the end of UVM-managed kernel virtual space, 4242 * until such time as pmap_growkernel() is called to expand 4243 * it. 4244 */ 4245 va_ranges[VA_RANGE_DEFAULT].start_va = 4246 pmap_trunc_ptpage(VM_MIN_KERNEL_ADDRESS); 4247 va_ranges[VA_RANGE_DEFAULT].end_va = 4248 pmap_round_ptpage(nextva); 4249 RELOC(kernel_virtual_end, vaddr_t) = nextva; 4250 4251 /* 4252 * Now that we know the end of the default range, look through 4253 * the machine_bootmap to see if there are any fixed VA ranges 4254 * we need to allocate static PTs for. 4255 */ 4256 pmbm = (const struct pmap_bootmap *) 4257 PMAP_BOOTSTRAP_RELOC_GLOB(machine_bootmap); 4258 for (; pmbm->pmbm_vaddr != (vaddr_t)-1; pmbm++) { 4259 if (pmbm->pmbm_size == 0) { 4260 continue; 4261 } 4262 if ((pmbm->pmbm_flags & (PMBM_F_FIXEDVA | PMBM_F_KEEPOUT)) != 4263 PMBM_F_FIXEDVA) { 4264 continue; 4265 } 4266 /* 4267 * Found one; see if there's a range already allocated 4268 * for it. 4269 */ 4270 for (r = 0; r < nranges; r++) { 4271 var = &va_ranges[r]; 4272 if (VA_IN_RANGE(pmbm->pmbm_vaddr, var)) { 4273 break; 4274 } 4275 } 4276 if (r < nranges) { 4277 /* Range of PTs already exists. */ 4278 continue; 4279 } 4280 /* 4281 * We need to allocate a range. If we're already at 4282 * the max, I guess we lose. 4283 */ 4284 if (r == MAX_RANGES) { 4285 break; 4286 } 4287 r = nranges++; 4288 va_ranges[r].start_va = pmap_trunc_ptpage(pmbm->pmbm_vaddr); 4289 va_ranges[r].end_va = 4290 pmap_round_ptpage(pmbm->pmbm_vaddr + pmbm->pmbm_size); 4291 } 4292 4293 /* 4294 * Now, compute the number of PT pages required to map the 4295 * required VA ranges and allocate them. 4296 */ 4297 size_t nptpages, total_ptpages = 0; 4298 for (r = 0; r < nranges; r++) { 4299 var = &va_ranges[r]; 4300 nptpages = (var->end_va - var->start_va) / PTPAGEVASZ; 4301 var->start_ptp = nextpa; 4302 nextpa += nptpages * PAGE_SIZE; 4303 var->end_ptp = nextpa; 4304 total_ptpages += nptpages; 4305 } 4306 4307 #ifdef PMAP_MACHINE_CHECK_BOOTSTRAP_ALLOCATIONS 4308 /* 4309 * Right here, the old mac68k Utah pmap_bootstrap1() has a 4310 * check to see if the kernel + bootstrap allocations fit 4311 * within one of the memory segments mapped by the loader. 4312 * This is a hook to accommodate that requirement. 4313 */ 4314 void (*alloc_checkfn)(paddr_t, paddr_t) = (void *) 4315 PMAP_BOOTSTRAP_RELOC_GLOB(pmap_machine_check_bootstrap_allocations); 4316 (*alloc_checkfn)(nextpa, reloff); 4317 #endif 4318 4319 /* 4320 * The bulk of the dynamic memory allocation is done (there 4321 * may be more below if we have to allocate more inner segment 4322 * table pages, but we'll burn that bridge when we come to it). 4323 * 4324 * Zero out all of these freshly-allocated pages. 4325 */ 4326 pte = (pt_entry_t *)PMAP_BOOTSTRAP_RELOC_PA(kernimg_endpa); 4327 epte = (pt_entry_t *)PMAP_BOOTSTRAP_RELOC_PA(nextpa); 4328 while (pte < epte) { 4329 *pte++ = 0; 4330 } 4331 4332 /* 4333 * Ok, let's get to mapping stuff! Almost everything is in 4334 * the default VA range. 4335 */ 4336 var = &va_ranges[VA_RANGE_DEFAULT]; 4337 4338 /* 4339 * Kernel text - read-only. 4340 * 4341 * ...that is, unless, a platform as some quirky requirement 4342 * (hello mac68k!). This hook lets a platform specify an 4343 * alternate proto PTE for the kernel text (in the mac68k case, 4344 * it will be read/write write-through-cacheable). Once the 4345 * kernel is up and running on its own mappings, machine-specific 4346 * code can perform any fixups as necessary. 4347 */ 4348 #ifndef PMAP_BOOTSTRAP_TEXT_PROTO_PTE 4349 #define PMAP_BOOTSTRAP_TEXT_PROTO_PTE proto_ro_pte 4350 #endif 4351 pa = PMAP_BOOTSTRAP_VA_TO_PA(m68k_trunc_page(&kernel_text)); 4352 pte = VA_PTE_BASE(&kernel_text, var); 4353 epte = VA_PTE_BASE(&etext, var); 4354 while (pte < epte) { 4355 *pte++ = PMAP_BOOTSTRAP_TEXT_PROTO_PTE | pa; 4356 pa += PAGE_SIZE; 4357 entry_count++; 4358 } 4359 4360 /* Remainder of kernel image - read-write. */ 4361 epte = VA_PTE_BASE(PMAP_BOOTSTRAP_PA_TO_VA(kernimg_endpa), var); 4362 while (pte < epte) { 4363 *pte++ = proto_rw_pte | pa; 4364 pa += PAGE_SIZE; 4365 entry_count++; 4366 } 4367 4368 /* lwp0 u-area - read-write. */ 4369 pa = lwp0upa; 4370 pte = VA_PTE_BASE(RELOC(lwp0uarea, vaddr_t), var); 4371 epte = VA_PTE_BASE(RELOC(lwp0uarea, vaddr_t) + USPACE, var); 4372 while (pte < epte) { 4373 *pte++ = proto_rw_pte | pa; 4374 pa += PAGE_SIZE; 4375 entry_count++; 4376 } 4377 4378 /* Kernel lev1map - read-write, cache-inhibited. */ 4379 pte = VA_PTE_BASE(kern_lev1va, var); 4380 *pte = proto_rw_ci_pte | kern_lev1pa; 4381 entry_count++; 4382 4383 /* 4384 * Kernel leaf PT pages - read-write, cache-inhibited. 4385 * 4386 * These will be in a different VA range if the machine 4387 * defines SYSMAP_VA. 4388 */ 4389 va = RELOC(kernel_ptes, vaddr_t); 4390 pt_entry_t *kptes = (pt_entry_t *)va; 4391 struct va_range *kpte_var = &va_ranges[VA_RANGE_KPTES]; 4392 4393 for (r = 0; r < nranges; r++) { 4394 var = &va_ranges[r]; 4395 va = (vaddr_t)(&kptes[m68k_btop(var->start_va)]); 4396 pte = VA_PTE_BASE(va, kpte_var); 4397 for (pa = var->start_ptp; pa < var->end_ptp; pa += PAGE_SIZE) { 4398 *pte++ = proto_rw_ci_pte | pa; 4399 entry_count++; 4400 } 4401 } 4402 4403 /* 4404 * Now perform any machine-specific mappings at VAs 4405 * allocated earlier. 4406 */ 4407 pmbm = (const struct pmap_bootmap *) 4408 PMAP_BOOTSTRAP_RELOC_GLOB(machine_bootmap); 4409 for (; pmbm->pmbm_vaddr != (vaddr_t)-1; pmbm++) { 4410 pt_entry_t proto; 4411 4412 if (pmbm->pmbm_size == 0 || 4413 (pmbm->pmbm_flags & (PMBM_F_VAONLY | PMBM_F_KEEPOUT))) { 4414 continue; 4415 } 4416 if (pmbm->pmbm_flags & PMBM_F_FIXEDVA) { 4417 va = pmbm->pmbm_vaddr; 4418 } else { 4419 va = *(vaddr_t *) 4420 PMAP_BOOTSTRAP_RELOC_GLOB(pmbm->pmbm_vaddr_ptr); 4421 } 4422 for (r = 0; r < nranges; r++) { 4423 var = &va_ranges[r]; 4424 if (VA_IN_RANGE(va, var)) { 4425 break; 4426 } 4427 } 4428 pa = pmbm->pmbm_paddr; 4429 pte = VA_PTE_BASE(va, var); 4430 switch (pmbm->pmbm_flags & (PMBM_F_CI|PMBM_F_CWT|PMBM_F_RO)) { 4431 case PMBM_F_CI|PMBM_F_RO: 4432 proto = proto_rw_ci_pte | PTE_WP; 4433 break; 4434 case PMBM_F_CWT: 4435 proto = proto_rw_cwt_pte; 4436 break; 4437 case PMBM_F_CI: 4438 proto = proto_rw_ci_pte; 4439 break; 4440 case PMBM_F_RO: 4441 proto = proto_ro_pte; 4442 break; 4443 default: 4444 proto = proto_rw_pte; 4445 break; 4446 } 4447 for (vsize_t size = m68k_round_page(pmbm->pmbm_size); 4448 size != 0; 4449 va += PAGE_SIZE, pa += PAGE_SIZE, size -= PAGE_SIZE) { 4450 *pte++ = proto | pa; 4451 entry_count++; 4452 } 4453 } 4454 4455 /* 4456 * Now that all of the individual VAs are mapped in the leaf 4457 * tables, it's time to link those tables into the segment 4458 * table. 4459 * 4460 * For the 2-level case, this is trivial. For the 3-level 4461 * case, we will have to allocate inner segment tables. 4462 */ 4463 for (r = 0; r < nranges; r++) { 4464 var = &va_ranges[r]; 4465 if (use_3l) { 4466 pt_entry_t *stes, *stes1 = (pt_entry_t *) 4467 PMAP_BOOTSTRAP_RELOC_PA(kern_lev1pa); 4468 for (va = var->start_va, pa = var->start_ptp; 4469 pa < var->end_ptp; 4470 va += NBSEG3L, pa += TBL40_L3_SIZE) { 4471 unsigned int ri = LA40_RI(va); 4472 if ((stes1[ri] & UTE40_RESIDENT) == 0) { 4473 /* 4474 * Level-2 table for this segment 4475 * needed. 4476 */ 4477 if (stnext_pa == stnext_endpa) { 4478 /* 4479 * No more slots left in the 4480 * last page we allocated for 4481 * segment tables. Grab 4482 * another one. 4483 */ 4484 stnext_pa = nextpa; 4485 nextpa += PAGE_SIZE; 4486 stnext_endpa = nextpa; 4487 nstpages++; 4488 #ifdef PMAP_MACHINE_CHECK_BOOTSTRAP_ALLOCATIONS 4489 (*alloc_checkfn)(nextpa, 4490 reloff); 4491 #endif 4492 /* 4493 * Zero out the new inner 4494 * segment table page. 4495 */ 4496 pte = (pt_entry_t *) 4497 PMAP_BOOTSTRAP_RELOC_PA( 4498 stnext_pa); 4499 epte = (pt_entry_t *) 4500 PMAP_BOOTSTRAP_RELOC_PA( 4501 stnext_endpa); 4502 while (pte < epte) { 4503 *pte++ = 0; 4504 } 4505 } 4506 stes1[ri] = proto_ste | stnext_pa; 4507 stnext_pa += TBL40_L2_SIZE; 4508 } 4509 stes = (pt_entry_t *) 4510 PMAP_BOOTSTRAP_RELOC_PA( 4511 stes1[ri] & UTE40_PTA); 4512 stes[LA40_PI(va)] = proto_ste | pa; 4513 } 4514 } else { 4515 pt_entry_t *stes = (pt_entry_t *) 4516 PMAP_BOOTSTRAP_RELOC_PA(kern_lev1pa); 4517 for (va = var->start_va, pa = var->start_ptp; 4518 pa < var->end_ptp; 4519 va += NBSEG2L, pa += PAGE_SIZE) { 4520 stes[LA2L_RI(va)] = proto_ste | pa; 4521 } 4522 } 4523 } 4524 4525 /* Instrumentation. */ 4526 RELOC(pmap_nkptpages_initial_ev.ev_count32, uint32_t) = 4527 RELOC(pmap_nkptpages_current_ev.ev_count32, uint32_t) = total_ptpages; 4528 RELOC(pmap_nkstpages_initial_ev.ev_count32, uint32_t) = 4529 RELOC(pmap_nkstpages_current_ev.ev_count32, uint32_t) = nstpages; 4530 4531 /* 4532 * Record the number of wired mappings we created above 4533 * in the kernel pmap stats. 4534 */ 4535 RELOC(kernel_pmap_store.pm_stats.resident_count, long) = entry_count; 4536 RELOC(kernel_pmap_store.pm_stats.wired_count, long) = entry_count; 4537 4538 /* 4539 * Stash any left-over segment table space for use by 4540 * pmap_growkernel() later. 4541 */ 4542 RELOC(kernel_stnext_pa, paddr_t) = stnext_pa; 4543 RELOC(kernel_stnext_endpa, paddr_t) = stnext_endpa; 4544 4545 return nextpa; 4546 } 4547 4548 /* 4549 * PAGE_SIZE should always evaluate to a compile-time constant in this 4550 * context. 4551 */ 4552 __CTASSERT(PAGE_SIZE != 0); 4553 4554 /* 4555 * pmap_bootstrap2: 4556 * 4557 * Phase 2 of bootstrapping virtual memory. This is called after 4558 * the MMU has been enabled to finish setting up run-time-computed 4559 * global pmap data, plus the lwp0 u-area, curlwp, and curpcb. 4560 * 4561 * Returns the new kernel %sp value for lwp0. 4562 */ 4563 void * 4564 pmap_bootstrap2(void) 4565 { 4566 /* Setup the MMU class; needed before anything else. */ 4567 pmap_mmuclass_init(); 4568 4569 /* Early low-level UVM initialization. */ 4570 uvmexp.pagesize = PAGE_SIZE; 4571 uvm_md_init(); 4572 4573 /* Initialize prototype PTEs; needed before anything else is mapped. */ 4574 pmap_pte_proto_init(); 4575 4576 /* Initialize the kernel pmap. */ 4577 pmap_pinit(pmap_kernel(), Sysseg_pa); 4578 4579 /* Initialize lwp0 u-area, curlwp, and curpcb. */ 4580 memset((void *)lwp0uarea, 0, USPACE); 4581 uvm_lwp_setuarea(&lwp0, lwp0uarea); 4582 curlwp = &lwp0; 4583 curpcb = lwp_getpcb(&lwp0); 4584 4585 /* Create a fake exception frame so that cpu_lwp_fork() can copy it. */ 4586 struct trapframe *tf = (struct trapframe *)(lwp0uarea + USPACE) - 1; 4587 tf->tf_sr = PSL_USER; 4588 lwp0.l_md.md_regs = (int *)tf; 4589 4590 /* 4591 * Initialize the source/destination control registers for 4592 * movs. 4593 */ 4594 setsfc(FC_USERD); 4595 setdfc(FC_USERD); 4596 4597 return tf; 4598 } 4599 4600 /***************************** PMAP DEBUGGING ********************************/ 4601 4602 #ifdef PMAP_DEBUG 4603 4604 void pmap_test_mod_ref(void); 4605 4606 void 4607 pmap_test_mod_ref(void) 4608 { 4609 struct vm_page *pg = pmap_page_alloc(false/*nowait*/); 4610 paddr_t pa = VM_PAGE_TO_PHYS(pg); 4611 vaddr_t va = (vaddr_t)vmmap; 4612 volatile int *loc = (volatile int *)va; 4613 int val; 4614 bool mod, ref; 4615 bool exp_mod, exp_ref; 4616 4617 /* Initialize page and mod/ref state to pristine. */ 4618 pmap_zero_page(pa); 4619 pmap_clear_modify(pg); 4620 pmap_clear_reference(pg); 4621 4622 mod = pmap_is_modified(pg); 4623 ref = pmap_is_referenced(pg); 4624 exp_mod = false; 4625 exp_ref = false; 4626 printf("%s: validating pristine page: mod=%d(%d) ref=%d(%d) (%s)\n", 4627 __func__, 4628 mod, exp_mod, ref, exp_ref, 4629 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 4630 4631 /* Enter non-seeded R/W mapping. */ 4632 pmap_enter(pmap_kernel(), va, pa, UVM_PROT_ALL, 0); 4633 4634 mod = pmap_is_modified(pg); 4635 ref = pmap_is_referenced(pg); 4636 exp_mod = false; 4637 exp_ref = false; 4638 printf("%s: enter(ALL, 0): mod=%d(%d) ref=%d(%d) (%s)\n", 4639 __func__, 4640 mod, exp_mod, ref, exp_ref, 4641 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 4642 4643 /* reference page */ 4644 val = *loc; 4645 4646 mod = pmap_is_modified(pg); 4647 ref = pmap_is_referenced(pg); 4648 exp_mod = false; 4649 exp_ref = true; 4650 printf("%s: ref val=%d: mod=%d(%d) ref=%d(%d) (%s)\n", 4651 __func__, 4652 val, 4653 mod, exp_mod, ref, exp_ref, 4654 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 4655 4656 /* validate clear behavior. */ 4657 exp_mod = mod; 4658 exp_ref = ref; 4659 mod = pmap_clear_modify(pg); 4660 ref = pmap_clear_reference(pg); 4661 printf("%s: checking clear 1: mod=%d(%d) ref=%d(%d) (%s)\n", 4662 __func__, 4663 mod, exp_mod, ref, exp_ref, 4664 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 4665 4666 mod = pmap_is_modified(pg); 4667 ref = pmap_is_referenced(pg); 4668 exp_mod = false; 4669 exp_ref = false; 4670 printf("%s: checking clear 2: mod=%d(%d) ref=%d(%d) (%s)\n", 4671 __func__, 4672 mod, exp_mod, ref, exp_ref, 4673 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 4674 4675 /* modify page */ 4676 *loc = 0xff; 4677 4678 mod = pmap_is_modified(pg); 4679 ref = pmap_is_referenced(pg); 4680 exp_mod = true; 4681 exp_ref = true; 4682 printf("%s: mod 1: mod=%d(%d) ref=%d(%d) (%s)\n", 4683 __func__, 4684 mod, exp_mod, ref, exp_ref, 4685 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 4686 4687 /* write-protect page */ 4688 pmap_page_protect(pg, UVM_PROT_READ); 4689 4690 mod = pmap_clear_modify(pg); 4691 ref = pmap_clear_reference(pg); 4692 exp_mod = true; 4693 exp_ref = true; 4694 printf("%s: page_protect(READ) mod 2: mod=%d(%d) ref=%d(%d) (%s)\n", 4695 __func__, 4696 mod, exp_mod, ref, exp_ref, 4697 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 4698 4699 mod = pmap_is_modified(pg); 4700 ref = pmap_is_referenced(pg); 4701 exp_mod = false; 4702 exp_ref = false; 4703 printf("%s: checking clear 3: mod=%d(%d) ref=%d(%d) (%s)\n", 4704 __func__, 4705 mod, exp_mod, ref, exp_ref, 4706 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 4707 4708 /* modify page again */ 4709 pmap_enter(pmap_kernel(), va, pa, UVM_PROT_ALL, 0); 4710 *loc = 0xaa; 4711 4712 mod = pmap_is_modified(pg); 4713 ref = pmap_is_referenced(pg); 4714 exp_mod = true; 4715 exp_ref = true; 4716 printf("%s: mod 3: mod=%d(%d) ref=%d(%d) (%s)\n", 4717 __func__, 4718 mod, exp_mod, ref, exp_ref, 4719 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 4720 4721 /* remove all mappings of page */ 4722 pmap_page_protect(pg, UVM_PROT_NONE); 4723 4724 mod = pmap_clear_modify(pg); 4725 ref = pmap_clear_reference(pg); 4726 exp_mod = true; 4727 exp_ref = true; 4728 printf("%s: page_protect(NONE) mod 4: mod=%d(%d) ref=%d(%d) (%s)\n", 4729 __func__, 4730 mod, exp_mod, ref, exp_ref, 4731 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 4732 4733 /* modify page again */ 4734 pmap_enter(pmap_kernel(), va, pa, UVM_PROT_ALL, 0); 4735 *loc = 0xaa; 4736 4737 mod = pmap_is_modified(pg); 4738 ref = pmap_is_referenced(pg); 4739 exp_mod = true; 4740 exp_ref = true; 4741 printf("%s: mod 5: mod=%d(%d) ref=%d(%d) (%s)\n", 4742 __func__, 4743 mod, exp_mod, ref, exp_ref, 4744 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 4745 4746 /* read-protect VA */ 4747 pmap_protect(pmap_kernel(), va, va+PAGE_SIZE, UVM_PROT_READ); 4748 4749 mod = pmap_is_modified(pg); 4750 ref = pmap_is_referenced(pg); 4751 exp_mod = true; 4752 exp_ref = true; 4753 printf("%s: va prot(READ) mod 6: mod=%d(%d) ref=%d(%d) (%s)\n", 4754 __func__, 4755 mod, exp_mod, ref, exp_ref, 4756 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 4757 4758 /* none-protect VA */ 4759 pmap_protect(pmap_kernel(), va, va+PAGE_SIZE, UVM_PROT_NONE); 4760 4761 mod = pmap_is_modified(pg); 4762 ref = pmap_is_referenced(pg); 4763 exp_mod = true; 4764 exp_ref = true; 4765 printf("%s: va prot(NONE) mod 7: mod=%d(%d) ref=%d(%d) (%s)\n", 4766 __func__, 4767 mod, exp_mod, ref, exp_ref, 4768 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 4769 4770 /* Reset the page. */ 4771 pmap_clear_modify(pg); 4772 pmap_clear_reference(pg); 4773 4774 mod = pmap_is_modified(pg); 4775 ref = pmap_is_referenced(pg); 4776 exp_mod = false; 4777 exp_ref = false; 4778 printf("%s: validating pristine page: mod=%d(%d) ref=%d(%d) (%s)\n", 4779 __func__, 4780 mod, exp_mod, ref, exp_ref, 4781 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 4782 4783 printf("%s: verifying mapping does not exist: (%s)\n", 4784 __func__, 4785 pmap_extract(pmap_kernel(), va, NULL) ? "FAIL" : "OK"); 4786 4787 /* Enter R-seeded R/W mapping. */ 4788 pmap_enter(pmap_kernel(), va, pa, UVM_PROT_READ|UVM_PROT_WRITE, 4789 UVM_PROT_READ); 4790 4791 mod = pmap_is_modified(pg); 4792 ref = pmap_is_referenced(pg); 4793 exp_mod = false; 4794 exp_ref = true; 4795 printf("%s: enter(R|W, R): mod=%d(%d) ref=%d(%d) (%s)\n", 4796 __func__, 4797 mod, exp_mod, ref, exp_ref, 4798 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 4799 4800 /* Enter W-seeded R/W mapping. */ 4801 pmap_enter(pmap_kernel(), va, pa, UVM_PROT_READ|UVM_PROT_WRITE, 4802 UVM_PROT_WRITE); 4803 4804 mod = pmap_is_modified(pg); 4805 ref = pmap_is_referenced(pg); 4806 exp_mod = true; 4807 exp_ref = true; 4808 printf("%s: enter(R|W, W): mod=%d(%d) ref=%d(%d) (%s)\n", 4809 __func__, 4810 mod, exp_mod, ref, exp_ref, 4811 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 4812 4813 pmap_clear_modify(pg); 4814 pmap_clear_reference(pg); 4815 4816 mod = pmap_is_modified(pg); 4817 ref = pmap_is_referenced(pg); 4818 exp_mod = false; 4819 exp_ref = false; 4820 printf("%s: validating pristine page: mod=%d(%d) ref=%d(%d) (%s)\n", 4821 __func__, 4822 mod, exp_mod, ref, exp_ref, 4823 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 4824 4825 /* 4826 * Now we're going to reference the page before modifying it 4827 * without any intervening clear. The MMU should update the 4828 * PTE once for the ref, and a second time for the mod, *even 4829 * if the ATC entry is still valid*. 4830 * 4831 * We are going to fetch the PTEs directly to validate the 4832 * MMU's behavior. 4833 */ 4834 pt_entry_t *ptep = pmap_kernel_pte(va); 4835 pt_entry_t ref_pte, mod_pte; 4836 int s = splhigh(); 4837 val = *loc; 4838 ref_pte = pte_load(ptep); 4839 *loc = 0; 4840 mod_pte = pte_load(ptep); 4841 splx(s); 4842 4843 mod = (mod_pte & PTE_M) != 0; 4844 ref = (ref_pte & PTE_U) != 0; 4845 exp_mod = true; 4846 exp_ref = true; 4847 printf("%s: validating MMU behavior: mod=%d(%d) ref=%d(%d) (%s)\n", 4848 __func__, 4849 mod, exp_mod, ref, exp_ref, 4850 mod == exp_mod && ref == exp_ref ? "OK" : "FAIL"); 4851 4852 /* all done. */ 4853 pmap_remove(pmap_kernel(), va, va+PAGE_SIZE); 4854 4855 printf("%s: done\n", __func__); 4856 } 4857 4858 #endif /* PMAP_DEBUG */ 4859
Indexes created Sun Oct 25 00:26:45 UTC 2026