pmap.h revision 1.179
1 /* $NetBSD: pmap.h,v 1.179 2026/04/19 15:09:49 skrll Exp $ */ 2 3 /* 4 * Copyright (c) 2002, 2003 Wasabi Systems, Inc. 5 * All rights reserved. 6 * 7 * Written by Jason R. Thorpe & Steve C. Woodford for Wasabi Systems, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed for the NetBSD Project by 20 * Wasabi Systems, Inc. 21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 22 * or promote products derived from this software without specific prior 23 * written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 38 /* 39 * Copyright (c) 1994,1995 Mark Brinicombe. 40 * All rights reserved. 41 * 42 * Redistribution and use in source and binary forms, with or without 43 * modification, are permitted provided that the following conditions 44 * are met: 45 * 1. Redistributions of source code must retain the above copyright 46 * notice, this list of conditions and the following disclaimer. 47 * 2. Redistributions in binary form must reproduce the above copyright 48 * notice, this list of conditions and the following disclaimer in the 49 * documentation and/or other materials provided with the distribution. 50 * 3. All advertising materials mentioning features or use of this software 51 * must display the following acknowledgement: 52 * This product includes software developed by Mark Brinicombe 53 * 4. The name of the author may not be used to endorse or promote products 54 * derived from this software without specific prior written permission. 55 * 56 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 57 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 58 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 59 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 60 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 61 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 62 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 63 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 64 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 65 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 66 */ 67 68 #ifndef _ARM32_PMAP_H_ 69 #define _ARM32_PMAP_H_ 70 71 #ifdef _KERNEL 72 73 #include <arm/cpuconf.h> 74 #include <arm/arm32/pte.h> 75 #ifndef _LOCORE 76 #if defined(_KERNEL_OPT) 77 #include "opt_arm32_pmap.h" 78 #include "opt_multiprocessor.h" 79 #endif 80 #include <arm/cpufunc.h> 81 #include <arm/locore.h> 82 83 #include <sys/cpu.h> 84 85 #include <uvm/uvm_object.h> 86 87 #include <uvm/pmap/pmap_devmap.h> 88 #include <uvm/pmap/pmap_pvt.h> 89 #endif 90 91 #ifdef ARM_MMU_EXTENDED 92 #define PMAP_HWPAGEWALKER 1 93 #define PMAP_TLB_MAX 1 94 #if PMAP_TLB_MAX > 1 95 #define PMAP_TLB_NEED_SHOOTDOWN 1 96 #endif 97 #define PMAP_TLB_FLUSH_ASID_ON_RESET arm_has_tlbiasid_p 98 #define PMAP_TLB_NUM_PIDS 256 99 100 #define pmap_md_tlb_asid_max() (PMAP_TLB_NUM_PIDS - 1) 101 #include <uvm/pmap/tlb.h> 102 #include <uvm/pmap/pmap_tlb.h> 103 104 /* 105 * If we have an EXTENDED MMU and the address space is split evenly between 106 * user and kernel, we can use the TTBR0/TTBR1 to have separate L1 tables for 107 * user and kernel address spaces. 108 */ 109 #if (KERNEL_BASE & 0x80000000) == 0 110 #error ARMv6 or later systems must have a KERNEL_BASE >= 0x80000000 111 #endif 112 #endif /* ARM_MMU_EXTENDED */ 113 114 /* 115 * a pmap describes a processes' 4GB virtual address space. this 116 * virtual address space can be broken up into 4096 1MB regions which 117 * are described by L1 PTEs in the L1 table. 118 * 119 * There is a line drawn at KERNEL_BASE. Everything below that line 120 * changes when the VM context is switched. Everything above that line 121 * is the same no matter which VM context is running. This is achieved 122 * by making the L1 PTEs for those slots above KERNEL_BASE reference 123 * kernel L2 tables. 124 * 125 * The basic layout of the virtual address space thus looks like this: 126 * 127 * 0xffffffff 128 * . 129 * . 130 * . 131 * KERNEL_BASE 132 * -------------------- 133 * . 134 * . 135 * . 136 * 0x00000000 137 */ 138 139 /* 140 * The number of L2 descriptor tables which can be tracked by an l2_dtable. 141 * A bucket size of 16 provides for 16MB of contiguous virtual address 142 * space per l2_dtable. Most processes will, therefore, require only two or 143 * three of these to map their whole working set. 144 */ 145 #define L2_BUCKET_XLOG2 (L1_S_SHIFT) 146 #define L2_BUCKET_XSIZE (1 << L2_BUCKET_XLOG2) 147 #define L2_BUCKET_LOG2 4 148 #define L2_BUCKET_SIZE (1 << L2_BUCKET_LOG2) 149 150 /* 151 * Given the above "L2-descriptors-per-l2_dtable" constant, the number 152 * of l2_dtable structures required to track all possible page descriptors 153 * mappable by an L1 translation table is given by the following constants: 154 */ 155 #define L2_LOG2 (32 - (L2_BUCKET_XLOG2 + L2_BUCKET_LOG2)) 156 #define L2_SIZE (1 << L2_LOG2) 157 158 /* 159 * tell MI code that the cache is virtually-indexed. 160 * ARMv6 is physically-tagged but all others are virtually-tagged. 161 */ 162 #if (ARM_MMU_V6 + ARM_MMU_V7) > 0 163 #define PMAP_CACHE_VIPT 164 #else 165 #define PMAP_CACHE_VIVT 166 #endif 167 168 #ifndef _LOCORE 169 170 #ifndef ARM_MMU_EXTENDED 171 struct l1_ttable; 172 struct l2_dtable; 173 174 /* 175 * Track cache/tlb occupancy using the following structure 176 */ 177 union pmap_cache_state { 178 struct { 179 union { 180 uint8_t csu_cache_b[2]; 181 uint16_t csu_cache; 182 } cs_cache_u; 183 184 union { 185 uint8_t csu_tlb_b[2]; 186 uint16_t csu_tlb; 187 } cs_tlb_u; 188 } cs_s; 189 uint32_t cs_all; 190 }; 191 #define cs_cache_id cs_s.cs_cache_u.csu_cache_b[0] 192 #define cs_cache_d cs_s.cs_cache_u.csu_cache_b[1] 193 #define cs_cache cs_s.cs_cache_u.csu_cache 194 #define cs_tlb_id cs_s.cs_tlb_u.csu_tlb_b[0] 195 #define cs_tlb_d cs_s.cs_tlb_u.csu_tlb_b[1] 196 #define cs_tlb cs_s.cs_tlb_u.csu_tlb 197 198 /* 199 * Assigned to cs_all to force cacheops to work for a particular pmap 200 */ 201 #define PMAP_CACHE_STATE_ALL 0xffffffffu 202 #endif /* !ARM_MMU_EXTENDED */ 203 204 205 #define DEVMAP_ALIGN(a) ((a) & ~L1_S_OFFSET) 206 #define DEVMAP_SIZE(s) roundup2((s), L1_S_SIZE) 207 #define DEVMAP_FLAGS PMAP_DEV 208 209 /* 210 * The pmap structure itself 211 */ 212 struct pmap { 213 kmutex_t pm_lock; 214 u_int pm_refs; 215 #ifndef ARM_HAS_VBAR 216 pd_entry_t *pm_pl1vec; 217 pd_entry_t pm_l1vec; 218 #endif 219 struct l2_dtable *pm_l2[L2_SIZE]; 220 struct pmap_statistics pm_stats; 221 LIST_ENTRY(pmap) pm_list; 222 bool pm_remove_all; 223 #ifdef ARM_MMU_EXTENDED 224 pd_entry_t *pm_l1; 225 paddr_t pm_l1_pa; 226 #ifdef MULTIPROCESSOR 227 kcpuset_t *pm_onproc; 228 kcpuset_t *pm_active; 229 #if PMAP_TLB_MAX > 1 230 u_int pm_shootdown_pending; 231 #endif 232 #endif 233 struct pmap_asid_info pm_pai[PMAP_TLB_MAX]; 234 #else 235 struct l1_ttable *pm_l1; 236 union pmap_cache_state pm_cstate; 237 uint8_t pm_domain; 238 bool pm_activated; 239 #endif 240 }; 241 242 struct pmap_kernel { 243 struct pmap kernel_pmap; 244 }; 245 246 /* 247 * Physical / virtual address structure. In a number of places (particularly 248 * during bootstrapping) we need to keep track of the physical and virtual 249 * addresses of various pages 250 */ 251 typedef struct pv_addr { 252 SLIST_ENTRY(pv_addr) pv_list; 253 paddr_t pv_pa; 254 vaddr_t pv_va; 255 vsize_t pv_size; 256 uint8_t pv_cache; 257 uint8_t pv_prot; 258 } pv_addr_t; 259 typedef SLIST_HEAD(, pv_addr) pv_addrqh_t; 260 261 extern pv_addrqh_t pmap_freeq; 262 extern pv_addr_t kernelstack; 263 extern pv_addr_t abtstack; 264 extern pv_addr_t fiqstack; 265 extern pv_addr_t irqstack; 266 extern pv_addr_t undstack; 267 extern pv_addr_t idlestack; 268 extern pv_addr_t systempage; 269 extern pv_addr_t kernel_l1pt; 270 #if defined(EFI_RUNTIME) 271 extern pv_addr_t efirt_l1pt; 272 #endif 273 274 #ifdef ARM_MMU_EXTENDED 275 extern bool arm_has_tlbiasid_p; /* also in <arm/locore.h> */ 276 277 static inline void 278 pmap_md_asid_activate(tlb_asid_t asid, pmap_t pm, struct lwp *l) 279 { 280 KASSERT(kpreempt_disabled()); 281 282 struct cpu_info * const ci = curcpu(); 283 284 /* 285 * Assume that TTBR1 has only global mappings and TTBR0 only 286 * has non-global mappings. To prevent speculation from doing 287 * evil things translation table walks using TTBR0 are disabled 288 * whilst there is no active userland pmap. Assert this here. 289 * 290 * In doing so the new CONTEXTIDR (ASID) and new TTBR0 value are 291 * only enable once both are set. 292 */ 293 KASSERT((armreg_ttbcr_read() & TTBCR_S_PD0) != 0); 294 295 if (pm == pmap_kernel()) 296 return; 297 298 armreg_contextidr_write(asid); 299 armreg_ttbr_write(pm->pm_l1_pa | 300 (ci->ci_mpidr ? TTBR_MPATTR : TTBR_UPATTR)); 301 isb(); 302 303 /* 304 * Now we can reenable tablewalks since the CONTEXTIDR and TTRB0 305 * have been updated. 306 */ 307 const uint32_t old_ttbcr = armreg_ttbcr_read(); 308 armreg_ttbcr_write(old_ttbcr & ~TTBCR_S_PD0); 309 cpu_cpwait(); 310 311 KASSERTMSG(ci->ci_pmap_asid_cur == asid, "%u vs %u", 312 ci->ci_pmap_asid_cur, asid); 313 ci->ci_pmap_cur = pm; 314 } 315 316 void 317 static inline 318 pmap_md_asid_deactivate(pmap_t pm) 319 { 320 KASSERT(kpreempt_disabled()); 321 322 struct cpu_info * const ci = curcpu(); 323 /* 324 * Disable translation table walks from TTBR0 while no pmap has been 325 * activated. 326 */ 327 const uint32_t old_ttbcr = armreg_ttbcr_read(); 328 armreg_ttbcr_write(old_ttbcr | TTBCR_S_PD0); 329 isb(); 330 331 ci->ci_pmap_cur = pmap_kernel(); 332 } 333 #endif 334 335 /* 336 * Determine various modes for PTEs (user vs. kernel, cacheable 337 * vs. non-cacheable). 338 */ 339 #define PTE_KERNEL 0 340 #define PTE_USER 1 341 #define PTE_NOCACHE 0 342 #define PTE_CACHE 1 343 #define PTE_PAGETABLE 2 344 #define PTE_DEV 3 345 346 /* 347 * Flags that indicate attributes of pages or mappings of pages. 348 * 349 * The PVF_MOD and PVF_REF flags are stored in the mdpage for each 350 * page. PVF_WIRED, PVF_WRITE, and PVF_NC are kept in individual 351 * pv_entry's for each page. They live in the same "namespace" so 352 * that we can clear multiple attributes at a time. 353 * 354 * Note the "non-cacheable" flag generally means the page has 355 * multiple mappings in a given address space. 356 */ 357 #define PVF_MOD 0x01 /* page is modified */ 358 #define PVF_REF 0x02 /* page is referenced */ 359 #define PVF_WIRED 0x04 /* mapping is wired */ 360 #define PVF_WRITE 0x08 /* mapping is writable */ 361 #define PVF_EXEC 0x10 /* mapping is executable */ 362 #ifdef PMAP_CACHE_VIVT 363 #define PVF_UNC 0x20 /* mapping is 'user' non-cacheable */ 364 #define PVF_KNC 0x40 /* mapping is 'kernel' non-cacheable */ 365 #define PVF_NC (PVF_UNC|PVF_KNC) 366 #endif 367 #ifdef PMAP_CACHE_VIPT 368 #define PVF_NC 0x20 /* mapping is 'kernel' non-cacheable */ 369 #define PVF_MULTCLR 0x40 /* mapping is multi-colored */ 370 #endif 371 #define PVF_COLORED 0x80 /* page has or had a color */ 372 #define PVF_KENTRY 0x0100 /* page entered via pmap_kenter_pa */ 373 #define PVF_KMPAGE 0x0200 /* page is used for kmem */ 374 #define PVF_DIRTY 0x0400 /* page may have dirty cache lines */ 375 #define PVF_KMOD 0x0800 /* unmanaged page is modified */ 376 #define PVF_KWRITE (PVF_KENTRY|PVF_WRITE) 377 #define PVF_DMOD (PVF_MOD|PVF_KMOD|PVF_KMPAGE) 378 379 /* 380 * Commonly referenced structures 381 */ 382 extern int arm_poolpage_vmfreelist; 383 384 /* 385 * Macros that we need to export 386 */ 387 #define pmap_resident_count(pmap) ((pmap)->pm_stats.resident_count) 388 #define pmap_wired_count(pmap) ((pmap)->pm_stats.wired_count) 389 390 #define pmap_is_modified(pg) \ 391 (((pg)->mdpage.pvh_attrs & PVF_MOD) != 0) 392 #define pmap_is_referenced(pg) \ 393 (((pg)->mdpage.pvh_attrs & PVF_REF) != 0) 394 #define pmap_is_page_colored_p(md) \ 395 (((md)->pvh_attrs & PVF_COLORED) != 0) 396 397 #define pmap_copy(dp, sp, da, l, sa) /* nothing */ 398 399 #define pmap_phys_address(ppn) (arm_ptob((ppn))) 400 u_int arm32_mmap_flags(paddr_t); 401 #define ARM32_MMAP_WRITECOMBINE 0x40000000 402 #define ARM32_MMAP_CACHEABLE 0x20000000 403 #define ARM_MMAP_WRITECOMBINE ARM32_MMAP_WRITECOMBINE 404 #define ARM_MMAP_CACHEABLE ARM32_MMAP_CACHEABLE 405 #define pmap_mmap_flags(ppn) arm32_mmap_flags(ppn) 406 407 #define PMAP_PTE 0x10000000 /* kenter_pa */ 408 #define PMAP_DEV 0x20000000 /* kenter_pa */ 409 #define PMAP_DEV_SO 0x40000000 /* kenter_pa */ 410 #define PMAP_DEV_MASK (PMAP_DEV | PMAP_DEV_SO) 411 412 /* 413 * Functions that we need to export 414 */ 415 void pmap_procwr(struct proc *, vaddr_t, int); 416 bool pmap_remove_all(pmap_t); 417 bool pmap_extract(pmap_t, vaddr_t, paddr_t *); 418 419 #define PMAP_NEED_PROCWR 420 #define PMAP_GROWKERNEL /* turn on pmap_growkernel interface */ 421 #define PMAP_ENABLE_PMAP_KMPAGE /* enable the PMAP_KMPAGE flag */ 422 423 #if (ARM_MMU_V6 + ARM_MMU_V7) > 0 424 #define PMAP_PREFER(hint, vap, sz, td) pmap_prefer((hint), (vap), (td)) 425 void pmap_prefer(vaddr_t, vaddr_t *, int); 426 #endif 427 428 #ifdef ARM_MMU_EXTENDED 429 int pmap_maxproc_set(int); 430 struct pmap * 431 pmap_efirt(void); 432 #endif 433 434 void pmap_icache_sync_range(pmap_t, vaddr_t, vaddr_t); 435 436 /* Functions we use internally. */ 437 #ifdef PMAP_STEAL_MEMORY 438 void pmap_boot_pagealloc(psize_t, psize_t, psize_t, pv_addr_t *); 439 void pmap_boot_pageadd(pv_addr_t *); 440 vaddr_t pmap_steal_memory(vsize_t, vaddr_t *, vaddr_t *); 441 #endif 442 void pmap_bootstrap(vaddr_t, vaddr_t); 443 444 struct pmap * 445 pmap_efirt(void); 446 void pmap_activate_efirt(void); 447 void pmap_deactivate_efirt(void); 448 449 void pmap_do_remove(pmap_t, vaddr_t, vaddr_t, int); 450 int pmap_fault_fixup(pmap_t, vaddr_t, vm_prot_t, int); 451 int pmap_prefetchabt_fixup(void *); 452 bool pmap_get_pde_pte(pmap_t, vaddr_t, pd_entry_t **, pt_entry_t **); 453 bool pmap_get_pde(pmap_t, vaddr_t, pd_entry_t **); 454 bool pmap_extract_coherency(pmap_t, vaddr_t, paddr_t *, bool *); 455 456 void pmap_postinit(void); 457 458 void vector_page_setprot(int); 459 460 /* Bootstrapping routines. */ 461 void pmap_map_section(vaddr_t, vaddr_t, paddr_t, int, int); 462 void pmap_map_entry(vaddr_t, vaddr_t, paddr_t, int, int); 463 vsize_t pmap_map_chunk(vaddr_t, vaddr_t, paddr_t, vsize_t, int, int); 464 void pmap_unmap_chunk(vaddr_t, vaddr_t, vsize_t); 465 void pmap_link_l2pt(vaddr_t, vaddr_t, pv_addr_t *); 466 467 vsize_t pmap_kenter_range(vaddr_t, paddr_t, vsize_t, vm_prot_t, u_int); 468 469 /* 470 * Special page zero routine for use by the idle loop (no cache cleans). 471 */ 472 bool pmap_pageidlezero(paddr_t); 473 #define PMAP_PAGEIDLEZERO(pa) pmap_pageidlezero((pa)) 474 475 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS 476 /* 477 * For the pmap, this is a more useful way to map a direct mapped page. 478 * It returns either the direct-mapped VA or the VA supplied if it can't 479 * be direct mapped. 480 */ 481 vaddr_t pmap_direct_mapped_phys(paddr_t, bool *, vaddr_t); 482 #endif 483 484 /* 485 * used by dumpsys to record the PA of the L1 table 486 */ 487 uint32_t pmap_kernel_L1_addr(void); 488 /* 489 * The current top of kernel VM 490 */ 491 extern vaddr_t pmap_curmaxkvaddr; 492 493 #if defined(ARM_MMU_EXTENDED) && defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) 494 /* 495 * Ending VA of direct mapped memory (usually KERNEL_VM_BASE). 496 */ 497 extern vaddr_t pmap_directlimit; 498 #endif 499 500 /* 501 * Useful macros and constants 502 */ 503 504 /* Virtual address to page table entry */ 505 static inline pt_entry_t * 506 vtopte(vaddr_t va) 507 { 508 pd_entry_t *pdep; 509 pt_entry_t *ptep; 510 511 KASSERT(trunc_page(va) == va); 512 513 if (pmap_get_pde_pte(pmap_kernel(), va, &pdep, &ptep) == false) 514 return (NULL); 515 return (ptep); 516 } 517 518 /* 519 * Virtual address to physical address 520 */ 521 static inline paddr_t 522 vtophys(vaddr_t va) 523 { 524 paddr_t pa; 525 526 if (pmap_extract(pmap_kernel(), va, &pa) == false) 527 return (0); /* XXXSCW: Panic? */ 528 529 return (pa); 530 } 531 532 /* 533 * The new pmap ensures that page-tables are always mapping Write-Thru. 534 * Thus, on some platforms we can run fast and loose and avoid syncing PTEs 535 * on every change. 536 * 537 * Unfortunately, not all CPUs have a write-through cache mode. So we 538 * define PMAP_NEEDS_PTE_SYNC for C code to conditionally do PTE syncs, 539 * and if there is the chance for PTE syncs to be needed, we define 540 * PMAP_INCLUDE_PTE_SYNC so e.g. assembly code can include (and run) 541 * the code. 542 */ 543 extern int pmap_needs_pte_sync; 544 #if defined(_KERNEL_OPT) 545 /* 546 * Perform compile time evaluation of PMAP_NEEDS_PTE_SYNC when only a 547 * single MMU type is selected. 548 * 549 * StrongARM SA-1 caches do not have a write-through mode. So, on these, 550 * we need to do PTE syncs. Additionally, V6 MMUs also need PTE syncs. 551 * Finally, MEMC, GENERIC and XSCALE MMUs do not need PTE syncs. 552 * 553 * Use run time evaluation for all other cases. 554 * 555 */ 556 #if (ARM_NMMUS == 1) 557 #if (ARM_MMU_SA1 + ARM_MMU_V6 != 0) 558 #define PMAP_INCLUDE_PTE_SYNC 559 #define PMAP_NEEDS_PTE_SYNC 1 560 #elif (ARM_MMU_MEMC + ARM_MMU_GENERIC + ARM_MMU_XSCALE != 0) 561 #define PMAP_NEEDS_PTE_SYNC 0 562 #endif 563 #endif 564 #endif /* _KERNEL_OPT */ 565 566 /* 567 * Provide a fallback in case we were not able to determine it at 568 * compile-time. 569 */ 570 #ifndef PMAP_NEEDS_PTE_SYNC 571 #define PMAP_NEEDS_PTE_SYNC pmap_needs_pte_sync 572 #define PMAP_INCLUDE_PTE_SYNC 573 #endif 574 575 static inline void 576 pmap_ptesync(pt_entry_t *ptep, size_t cnt) 577 { 578 if (PMAP_NEEDS_PTE_SYNC) { 579 cpu_dcache_wb_range((vaddr_t)ptep, cnt * sizeof(pt_entry_t)); 580 #ifdef SHEEVA_L2_CACHE 581 cpu_sdcache_wb_range((vaddr_t)ptep, -1, 582 cnt * sizeof(pt_entry_t)); 583 #endif 584 } 585 dsb(sy); 586 } 587 588 #define PDE_SYNC(pdep) pmap_ptesync((pdep), 1) 589 #define PDE_SYNC_RANGE(pdep, cnt) pmap_ptesync((pdep), (cnt)) 590 #define PTE_SYNC(ptep) pmap_ptesync((ptep), PAGE_SIZE / L2_S_SIZE) 591 #define PTE_SYNC_RANGE(ptep, cnt) pmap_ptesync((ptep), (cnt)) 592 593 #define l1pte_valid_p(pde) ((pde) != 0) 594 #define l1pte_section_p(pde) (((pde) & L1_TYPE_MASK) == L1_TYPE_S) 595 #define l1pte_supersection_p(pde) (l1pte_section_p(pde) \ 596 && ((pde) & L1_S_V6_SUPER) != 0) 597 #define l1pte_page_p(pde) (((pde) & L1_TYPE_MASK) == L1_TYPE_C) 598 #define l1pte_fpage_p(pde) (((pde) & L1_TYPE_MASK) == L1_TYPE_F) 599 #define l1pte_pa(pde) ((pde) & L1_C_ADDR_MASK) 600 #define l1pte_index(v) ((vaddr_t)(v) >> L1_S_SHIFT) 601 602 static inline void 603 l1pte_setone(pt_entry_t *pdep, pt_entry_t pde) 604 { 605 *pdep = pde; 606 } 607 608 static inline void 609 l1pte_set(pt_entry_t *pdep, pt_entry_t pde) 610 { 611 *pdep = pde; 612 if (l1pte_page_p(pde)) { 613 KASSERTMSG((((uintptr_t)pdep / sizeof(pde)) & (PAGE_SIZE / L2_T_SIZE - 1)) == 0, "%p", pdep); 614 for (int k = 1; k < PAGE_SIZE / L2_T_SIZE; k++) { 615 pde += L2_T_SIZE; 616 pdep[k] = pde; 617 } 618 } else if (l1pte_supersection_p(pde)) { 619 KASSERTMSG((((uintptr_t)pdep / sizeof(pde)) & (L1_SS_SIZE / L1_S_SIZE - 1)) == 0, "%p", pdep); 620 for (int k = 1; k < L1_SS_SIZE / L1_S_SIZE; k++) { 621 pdep[k] = pde; 622 } 623 } 624 } 625 626 #define l2pte_index(v) ((((v) & L2_ADDR_BITS) >> PGSHIFT) << (PGSHIFT-L2_S_SHIFT)) 627 #define l2pte_valid_p(pte) (((pte) & L2_TYPE_MASK) != L2_TYPE_INV) 628 #define l2pte_pa(pte) ((pte) & L2_S_FRAME) 629 #define l1pte_lpage_p(pte) (((pte) & L2_TYPE_MASK) == L2_TYPE_L) 630 #define l2pte_minidata_p(pte) (((pte) & \ 631 (L2_B | L2_C | L2_XS_T_TEX(TEX_XSCALE_X)))\ 632 == (L2_C | L2_XS_T_TEX(TEX_XSCALE_X))) 633 634 static inline void 635 l2pte_set(pt_entry_t *ptep, pt_entry_t pte, pt_entry_t opte) 636 { 637 if (l1pte_lpage_p(pte)) { 638 KASSERTMSG((((uintptr_t)ptep / sizeof(pte)) & (L2_L_SIZE / L2_S_SIZE - 1)) == 0, "%p", ptep); 639 for (int k = 0; k < L2_L_SIZE / L2_S_SIZE; k++) { 640 *ptep++ = pte; 641 } 642 } else { 643 KASSERTMSG((((uintptr_t)ptep / sizeof(pte)) & (PAGE_SIZE / L2_S_SIZE - 1)) == 0, "%p", ptep); 644 for (int k = 0; k < PAGE_SIZE / L2_S_SIZE; k++) { 645 KASSERTMSG(*ptep == opte, "%#x [*%p] != %#x", *ptep, ptep, opte); 646 *ptep++ = pte; 647 pte += L2_S_SIZE; 648 if (opte) 649 opte += L2_S_SIZE; 650 } 651 } 652 } 653 654 static inline void 655 l2pte_reset(pt_entry_t *ptep) 656 { 657 KASSERTMSG((((uintptr_t)ptep / sizeof(*ptep)) & (PAGE_SIZE / L2_S_SIZE - 1)) == 0, "%p", ptep); 658 *ptep = 0; 659 for (int k = 1; k < PAGE_SIZE / L2_S_SIZE; k++) { 660 ptep[k] = 0; 661 } 662 } 663 664 /* L1 and L2 page table macros */ 665 #define pmap_pde_v(pde) l1pte_valid(*(pde)) 666 #define pmap_pde_section(pde) l1pte_section_p(*(pde)) 667 #define pmap_pde_supersection(pde) l1pte_supersection_p(*(pde)) 668 #define pmap_pde_page(pde) l1pte_page_p(*(pde)) 669 #define pmap_pde_fpage(pde) l1pte_fpage_p(*(pde)) 670 671 #define pmap_pte_v(pte) l2pte_valid_p(*(pte)) 672 #define pmap_pte_pa(pte) l2pte_pa(*(pte)) 673 674 static inline uint32_t 675 pte_value(pt_entry_t pte) 676 { 677 return pte; 678 } 679 680 static inline bool 681 pte_valid_p(pt_entry_t pte) 682 { 683 684 return l2pte_valid_p(pte); 685 } 686 687 688 /* Size of the kernel part of the L1 page table */ 689 #define KERNEL_PD_SIZE \ 690 (L1_TABLE_SIZE - (KERNEL_BASE >> L1_S_SHIFT) * sizeof(pd_entry_t)) 691 692 void bzero_page(vaddr_t); 693 void bcopy_page(vaddr_t, vaddr_t); 694 695 #ifdef FPU_VFP 696 void bzero_page_vfp(vaddr_t); 697 void bcopy_page_vfp(vaddr_t, vaddr_t); 698 #endif 699 700 /************************* ARM MMU configuration *****************************/ 701 702 #if (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6 + ARM_MMU_V7) != 0 703 void pmap_copy_page_generic(paddr_t, paddr_t); 704 void pmap_zero_page_generic(paddr_t); 705 706 void pmap_pte_init_generic(void); 707 #if defined(CPU_ARM8) 708 void pmap_pte_init_arm8(void); 709 #endif 710 #if defined(CPU_ARM9) 711 void pmap_pte_init_arm9(void); 712 #endif /* CPU_ARM9 */ 713 #if defined(CPU_ARM10) 714 void pmap_pte_init_arm10(void); 715 #endif /* CPU_ARM10 */ 716 #if defined(CPU_ARM11) /* ARM_MMU_V6 */ 717 void pmap_pte_init_arm11(void); 718 #endif /* CPU_ARM11 */ 719 #if defined(CPU_ARM11MPCORE) /* ARM_MMU_V6 */ 720 void pmap_pte_init_arm11mpcore(void); 721 #endif 722 #if ARM_MMU_V6 == 1 723 void pmap_pte_init_armv6(void); 724 #endif /* ARM_MMU_V6 */ 725 #if ARM_MMU_V7 == 1 726 void pmap_pte_init_armv7(void); 727 #endif /* ARM_MMU_V7 */ 728 #endif /* (ARM_MMU_GENERIC + ARM_MMU_SA1) != 0 */ 729 730 #if ARM_MMU_SA1 == 1 731 void pmap_pte_init_sa1(void); 732 #endif /* ARM_MMU_SA1 == 1 */ 733 734 #if ARM_MMU_XSCALE == 1 735 void pmap_copy_page_xscale(paddr_t, paddr_t); 736 void pmap_zero_page_xscale(paddr_t); 737 738 void pmap_pte_init_xscale(void); 739 740 void xscale_setup_minidata(vaddr_t, vaddr_t, paddr_t); 741 742 #define PMAP_UAREA(va) pmap_uarea(va) 743 void pmap_uarea(vaddr_t); 744 #endif /* ARM_MMU_XSCALE == 1 */ 745 746 extern pt_entry_t pte_l1_s_nocache_mode; 747 extern pt_entry_t pte_l2_l_nocache_mode; 748 extern pt_entry_t pte_l2_s_nocache_mode; 749 750 extern pt_entry_t pte_l1_s_cache_mode; 751 extern pt_entry_t pte_l2_l_cache_mode; 752 extern pt_entry_t pte_l2_s_cache_mode; 753 754 extern pt_entry_t pte_l1_s_cache_mode_pt; 755 extern pt_entry_t pte_l2_l_cache_mode_pt; 756 extern pt_entry_t pte_l2_s_cache_mode_pt; 757 758 extern pt_entry_t pte_l1_s_wc_mode; 759 extern pt_entry_t pte_l2_l_wc_mode; 760 extern pt_entry_t pte_l2_s_wc_mode; 761 762 extern pt_entry_t pte_l1_s_cache_mask; 763 extern pt_entry_t pte_l2_l_cache_mask; 764 extern pt_entry_t pte_l2_s_cache_mask; 765 766 extern pt_entry_t pte_l1_s_prot_u; 767 extern pt_entry_t pte_l1_s_prot_w; 768 extern pt_entry_t pte_l1_s_prot_ro; 769 extern pt_entry_t pte_l1_s_prot_mask; 770 771 extern pt_entry_t pte_l2_s_prot_u; 772 extern pt_entry_t pte_l2_s_prot_w; 773 extern pt_entry_t pte_l2_s_prot_ro; 774 extern pt_entry_t pte_l2_s_prot_mask; 775 776 extern pt_entry_t pte_l2_l_prot_u; 777 extern pt_entry_t pte_l2_l_prot_w; 778 extern pt_entry_t pte_l2_l_prot_ro; 779 extern pt_entry_t pte_l2_l_prot_mask; 780 781 extern pt_entry_t pte_l1_ss_proto; 782 extern pt_entry_t pte_l1_s_proto; 783 extern pt_entry_t pte_l1_c_proto; 784 extern pt_entry_t pte_l2_s_proto; 785 786 extern void (*pmap_copy_page_func)(paddr_t, paddr_t); 787 extern void (*pmap_zero_page_func)(paddr_t); 788 789 /* 790 * Global varaiables in cpufunc_asm_xscale.S supporting the Xscale 791 * cache clean/purge functions. 792 */ 793 extern vaddr_t xscale_minidata_clean_addr; 794 extern vsize_t xscale_minidata_clean_size; 795 extern vaddr_t xscale_cache_clean_addr; 796 extern vsize_t xscale_cache_clean_size; 797 798 #endif /* !_LOCORE */ 799 800 /*****************************************************************************/ 801 802 #define KERNEL_PID 0 /* The kernel uses ASID 0 */ 803 804 /* 805 * Definitions for MMU domains 806 */ 807 #define PMAP_DOMAINS 15 /* 15 'user' domains (1-15) */ 808 #define PMAP_DOMAIN_KERNEL 0 /* The kernel pmap uses domain #0 */ 809 810 #ifdef ARM_MMU_EXTENDED 811 #define PMAP_DOMAIN_USER 1 /* User pmaps use domain #1 */ 812 #define DOMAIN_DEFAULT ((DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL*2)) | (DOMAIN_CLIENT << (PMAP_DOMAIN_USER*2))) 813 #else 814 #define DOMAIN_DEFAULT ((DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL*2))) 815 #endif 816 817 /* 818 * These macros define the various bit masks in the PTE. 819 * 820 * We use these macros since we use different bits on different processor 821 * models. 822 */ 823 #define L1_S_PROT_U_generic (L1_S_AP(AP_U)) 824 #define L1_S_PROT_W_generic (L1_S_AP(AP_W)) 825 #define L1_S_PROT_RO_generic (0) 826 #define L1_S_PROT_MASK_generic (L1_S_PROT_U|L1_S_PROT_W|L1_S_PROT_RO) 827 828 #define L1_S_PROT_U_xscale (L1_S_AP(AP_U)) 829 #define L1_S_PROT_W_xscale (L1_S_AP(AP_W)) 830 #define L1_S_PROT_RO_xscale (0) 831 #define L1_S_PROT_MASK_xscale (L1_S_PROT_U|L1_S_PROT_W|L1_S_PROT_RO) 832 833 #define L1_S_PROT_U_armv6 (L1_S_AP(AP_R) | L1_S_AP(AP_U)) 834 #define L1_S_PROT_W_armv6 (L1_S_AP(AP_W)) 835 #define L1_S_PROT_RO_armv6 (L1_S_AP(AP_R) | L1_S_AP(AP_RO)) 836 #define L1_S_PROT_MASK_armv6 (L1_S_PROT_U|L1_S_PROT_W|L1_S_PROT_RO) 837 838 #define L1_S_PROT_U_armv7 (L1_S_AP(AP_R) | L1_S_AP(AP_U)) 839 #define L1_S_PROT_W_armv7 (L1_S_AP(AP_W)) 840 #define L1_S_PROT_RO_armv7 (L1_S_AP(AP_R) | L1_S_AP(AP_RO)) 841 #define L1_S_PROT_MASK_armv7 (L1_S_PROT_U|L1_S_PROT_W|L1_S_PROT_RO) 842 843 #define L1_S_NOCACHE_generic (0) 844 #define L1_S_NOCACHE_armv6n (L1_S_XS_TEX(1)) 845 846 #define L1_S_CACHE_MASK_generic (L1_S_B|L1_S_C) 847 #define L1_S_CACHE_MASK_xscale (L1_S_B|L1_S_C|L1_S_XS_TEX(TEX_XSCALE_X)) 848 #define L1_S_CACHE_MASK_armv6 (L1_S_B|L1_S_C|L1_S_XS_TEX(TEX_ARMV6_TEX)) 849 #define L1_S_CACHE_MASK_armv6n (L1_S_B|L1_S_C|L1_S_XS_TEX(TEX_ARMV6_TEX)|L1_S_V6_S) 850 #define L1_S_CACHE_MASK_armv7 (L1_S_B|L1_S_C|L1_S_XS_TEX(TEX_ARMV6_TEX)|L1_S_V6_S) 851 852 #define L2_L_PROT_U_generic (L2_AP(AP_U)) 853 #define L2_L_PROT_W_generic (L2_AP(AP_W)) 854 #define L2_L_PROT_RO_generic (0) 855 #define L2_L_PROT_MASK_generic (L2_L_PROT_U|L2_L_PROT_W|L2_L_PROT_RO) 856 857 #define L2_L_PROT_U_xscale (L2_AP(AP_U)) 858 #define L2_L_PROT_W_xscale (L2_AP(AP_W)) 859 #define L2_L_PROT_RO_xscale (0) 860 #define L2_L_PROT_MASK_xscale (L2_L_PROT_U|L2_L_PROT_W|L2_L_PROT_RO) 861 862 #define L2_L_PROT_U_armv6n (L2_AP0(AP_R) | L2_AP0(AP_U)) 863 #define L2_L_PROT_W_armv6n (L2_AP0(AP_W)) 864 #define L2_L_PROT_RO_armv6n (L2_AP0(AP_R) | L2_AP0(AP_RO)) 865 #define L2_L_PROT_MASK_armv6n (L2_L_PROT_U|L2_L_PROT_W|L2_L_PROT_RO) 866 867 #define L2_L_PROT_U_armv7 (L2_AP0(AP_R) | L2_AP0(AP_U)) 868 #define L2_L_PROT_W_armv7 (L2_AP0(AP_W)) 869 #define L2_L_PROT_RO_armv7 (L2_AP0(AP_R) | L2_AP0(AP_RO)) 870 #define L2_L_PROT_MASK_armv7 (L2_L_PROT_U|L2_L_PROT_W|L2_L_PROT_RO) 871 872 #define L2_L_CACHE_MASK_generic (L2_B|L2_C) 873 #define L2_L_CACHE_MASK_xscale (L2_B|L2_C|L2_XS_L_TEX(TEX_XSCALE_X)) 874 #define L2_L_CACHE_MASK_armv6 (L2_B|L2_C|L2_V6_L_TEX(TEX_ARMV6_TEX)) 875 #define L2_L_CACHE_MASK_armv6n (L2_B|L2_C|L2_V6_L_TEX(TEX_ARMV6_TEX)|L2_XS_S) 876 #define L2_L_CACHE_MASK_armv7 (L2_B|L2_C|L2_V6_L_TEX(TEX_ARMV6_TEX)|L2_XS_S) 877 878 #define L2_S_PROT_U_generic (L2_AP(AP_U)) 879 #define L2_S_PROT_W_generic (L2_AP(AP_W)) 880 #define L2_S_PROT_RO_generic (0) 881 #define L2_S_PROT_MASK_generic (L2_S_PROT_U|L2_S_PROT_W|L2_S_PROT_RO) 882 883 #define L2_S_PROT_U_xscale (L2_AP0(AP_U)) 884 #define L2_S_PROT_W_xscale (L2_AP0(AP_W)) 885 #define L2_S_PROT_RO_xscale (0) 886 #define L2_S_PROT_MASK_xscale (L2_S_PROT_U|L2_S_PROT_W|L2_S_PROT_RO) 887 888 #define L2_S_PROT_U_armv6n (L2_AP0(AP_R) | L2_AP0(AP_U)) 889 #define L2_S_PROT_W_armv6n (L2_AP0(AP_W)) 890 #define L2_S_PROT_RO_armv6n (L2_AP0(AP_R) | L2_AP0(AP_RO)) 891 #define L2_S_PROT_MASK_armv6n (L2_S_PROT_U|L2_S_PROT_W|L2_S_PROT_RO) 892 893 #define L2_S_PROT_U_armv7 (L2_AP0(AP_R) | L2_AP0(AP_U)) 894 #define L2_S_PROT_W_armv7 (L2_AP0(AP_W)) 895 #define L2_S_PROT_RO_armv7 (L2_AP0(AP_R) | L2_AP0(AP_RO)) 896 #define L2_S_PROT_MASK_armv7 (L2_S_PROT_U|L2_S_PROT_W|L2_S_PROT_RO) 897 898 #define L2_S_CACHE_MASK_generic (L2_B|L2_C) 899 #define L2_S_CACHE_MASK_xscale (L2_B|L2_C|L2_XS_T_TEX(TEX_XSCALE_X)) 900 #define L2_XS_CACHE_MASK_armv6 (L2_B|L2_C|L2_V6_XS_TEX(TEX_ARMV6_TEX)) 901 #ifdef ARMV6_EXTENDED_SMALL_PAGE 902 #define L2_S_CACHE_MASK_armv6c L2_XS_CACHE_MASK_armv6 903 #else 904 #define L2_S_CACHE_MASK_armv6c L2_S_CACHE_MASK_generic 905 #endif 906 #define L2_S_CACHE_MASK_armv6n (L2_B|L2_C|L2_V6_XS_TEX(TEX_ARMV6_TEX)|L2_XS_S) 907 #define L2_S_CACHE_MASK_armv7 (L2_B|L2_C|L2_V6_XS_TEX(TEX_ARMV6_TEX)|L2_XS_S) 908 909 910 #define L1_S_PROTO_generic (L1_TYPE_S | L1_S_IMP) 911 #define L1_S_PROTO_xscale (L1_TYPE_S) 912 #define L1_S_PROTO_armv6 (L1_TYPE_S) 913 #define L1_S_PROTO_armv7 (L1_TYPE_S) 914 915 #define L1_SS_PROTO_generic 0 916 #define L1_SS_PROTO_xscale 0 917 #define L1_SS_PROTO_armv6 (L1_TYPE_S | L1_S_V6_SS) 918 #define L1_SS_PROTO_armv7 (L1_TYPE_S | L1_S_V6_SS) 919 920 #define L1_C_PROTO_generic (L1_TYPE_C | L1_C_IMP2) 921 #define L1_C_PROTO_xscale (L1_TYPE_C) 922 #define L1_C_PROTO_armv6 (L1_TYPE_C) 923 #define L1_C_PROTO_armv7 (L1_TYPE_C) 924 925 #define L2_L_PROTO (L2_TYPE_L) 926 927 #define L2_S_PROTO_generic (L2_TYPE_S) 928 #define L2_S_PROTO_xscale (L2_TYPE_XS) 929 #ifdef ARMV6_EXTENDED_SMALL_PAGE 930 #define L2_S_PROTO_armv6c (L2_TYPE_XS) /* XP=0, extended small page */ 931 #else 932 #define L2_S_PROTO_armv6c (L2_TYPE_S) /* XP=0, subpage APs */ 933 #endif 934 #ifdef ARM_MMU_EXTENDED 935 #define L2_S_PROTO_armv6n (L2_TYPE_S|L2_XS_XN) 936 #else 937 #define L2_S_PROTO_armv6n (L2_TYPE_S) /* with XP=1 */ 938 #endif 939 #ifdef ARM_MMU_EXTENDED 940 #define L2_S_PROTO_armv7 (L2_TYPE_S|L2_XS_XN) 941 #else 942 #define L2_S_PROTO_armv7 (L2_TYPE_S) 943 #endif 944 945 /* 946 * User-visible names for the ones that vary with MMU class. 947 */ 948 949 #if ARM_NMMUS > 1 950 /* More than one MMU class configured; use variables. */ 951 #define L1_S_PROT_U pte_l1_s_prot_u 952 #define L1_S_PROT_W pte_l1_s_prot_w 953 #define L1_S_PROT_RO pte_l1_s_prot_ro 954 #define L1_S_PROT_MASK pte_l1_s_prot_mask 955 956 #define L2_S_PROT_U pte_l2_s_prot_u 957 #define L2_S_PROT_W pte_l2_s_prot_w 958 #define L2_S_PROT_RO pte_l2_s_prot_ro 959 #define L2_S_PROT_MASK pte_l2_s_prot_mask 960 961 #define L2_L_PROT_U pte_l2_l_prot_u 962 #define L2_L_PROT_W pte_l2_l_prot_w 963 #define L2_L_PROT_RO pte_l2_l_prot_ro 964 #define L2_L_PROT_MASK pte_l2_l_prot_mask 965 966 #define L1_S_CACHE_MASK pte_l1_s_cache_mask 967 #define L2_L_CACHE_MASK pte_l2_l_cache_mask 968 #define L2_S_CACHE_MASK pte_l2_s_cache_mask 969 970 #define L1_SS_PROTO pte_l1_ss_proto 971 #define L1_S_PROTO pte_l1_s_proto 972 #define L1_C_PROTO pte_l1_c_proto 973 #define L2_S_PROTO pte_l2_s_proto 974 975 #define pmap_copy_page(s, d) (*pmap_copy_page_func)((s), (d)) 976 #define pmap_zero_page(d) (*pmap_zero_page_func)((d)) 977 #elif (ARM_MMU_GENERIC + ARM_MMU_SA1) != 0 978 #define L1_S_PROT_U L1_S_PROT_U_generic 979 #define L1_S_PROT_W L1_S_PROT_W_generic 980 #define L1_S_PROT_RO L1_S_PROT_RO_generic 981 #define L1_S_PROT_MASK L1_S_PROT_MASK_generic 982 983 #define L2_S_PROT_U L2_S_PROT_U_generic 984 #define L2_S_PROT_W L2_S_PROT_W_generic 985 #define L2_S_PROT_RO L2_S_PROT_RO_generic 986 #define L2_S_PROT_MASK L2_S_PROT_MASK_generic 987 988 #define L2_L_PROT_U L2_L_PROT_U_generic 989 #define L2_L_PROT_W L2_L_PROT_W_generic 990 #define L2_L_PROT_RO L2_L_PROT_RO_generic 991 #define L2_L_PROT_MASK L2_L_PROT_MASK_generic 992 993 #define L1_S_CACHE_MASK L1_S_CACHE_MASK_generic 994 #define L2_L_CACHE_MASK L2_L_CACHE_MASK_generic 995 #define L2_S_CACHE_MASK L2_S_CACHE_MASK_generic 996 997 #define L1_SS_PROTO L1_SS_PROTO_generic 998 #define L1_S_PROTO L1_S_PROTO_generic 999 #define L1_C_PROTO L1_C_PROTO_generic 1000 #define L2_S_PROTO L2_S_PROTO_generic 1001 1002 #define pmap_copy_page(s, d) pmap_copy_page_generic((s), (d)) 1003 #define pmap_zero_page(d) pmap_zero_page_generic((d)) 1004 #elif ARM_MMU_V6N != 0 1005 #define L1_S_PROT_U L1_S_PROT_U_armv6 1006 #define L1_S_PROT_W L1_S_PROT_W_armv6 1007 #define L1_S_PROT_RO L1_S_PROT_RO_armv6 1008 #define L1_S_PROT_MASK L1_S_PROT_MASK_armv6 1009 1010 #define L2_S_PROT_U L2_S_PROT_U_armv6n 1011 #define L2_S_PROT_W L2_S_PROT_W_armv6n 1012 #define L2_S_PROT_RO L2_S_PROT_RO_armv6n 1013 #define L2_S_PROT_MASK L2_S_PROT_MASK_armv6n 1014 1015 #define L2_L_PROT_U L2_L_PROT_U_armv6n 1016 #define L2_L_PROT_W L2_L_PROT_W_armv6n 1017 #define L2_L_PROT_RO L2_L_PROT_RO_armv6n 1018 #define L2_L_PROT_MASK L2_L_PROT_MASK_armv6n 1019 1020 #define L1_S_CACHE_MASK L1_S_CACHE_MASK_armv6n 1021 #define L2_L_CACHE_MASK L2_L_CACHE_MASK_armv6n 1022 #define L2_S_CACHE_MASK L2_S_CACHE_MASK_armv6n 1023 1024 /* 1025 * These prototypes make writeable mappings, while the other MMU types 1026 * make read-only mappings. 1027 */ 1028 #define L1_SS_PROTO L1_SS_PROTO_armv6 1029 #define L1_S_PROTO L1_S_PROTO_armv6 1030 #define L1_C_PROTO L1_C_PROTO_armv6 1031 #define L2_S_PROTO L2_S_PROTO_armv6n 1032 1033 #define pmap_copy_page(s, d) pmap_copy_page_generic((s), (d)) 1034 #define pmap_zero_page(d) pmap_zero_page_generic((d)) 1035 #elif ARM_MMU_V6C != 0 1036 #define L1_S_PROT_U L1_S_PROT_U_generic 1037 #define L1_S_PROT_W L1_S_PROT_W_generic 1038 #define L1_S_PROT_RO L1_S_PROT_RO_generic 1039 #define L1_S_PROT_MASK L1_S_PROT_MASK_generic 1040 1041 #define L2_S_PROT_U L2_S_PROT_U_generic 1042 #define L2_S_PROT_W L2_S_PROT_W_generic 1043 #define L2_S_PROT_RO L2_S_PROT_RO_generic 1044 #define L2_S_PROT_MASK L2_S_PROT_MASK_generic 1045 1046 #define L2_L_PROT_U L2_L_PROT_U_generic 1047 #define L2_L_PROT_W L2_L_PROT_W_generic 1048 #define L2_L_PROT_RO L2_L_PROT_RO_generic 1049 #define L2_L_PROT_MASK L2_L_PROT_MASK_generic 1050 1051 #define L1_S_CACHE_MASK L1_S_CACHE_MASK_generic 1052 #define L2_L_CACHE_MASK L2_L_CACHE_MASK_generic 1053 #define L2_S_CACHE_MASK L2_S_CACHE_MASK_generic 1054 1055 #define L1_SS_PROTO L1_SS_PROTO_armv6 1056 #define L1_S_PROTO L1_S_PROTO_generic 1057 #define L1_C_PROTO L1_C_PROTO_generic 1058 #define L2_S_PROTO L2_S_PROTO_generic 1059 1060 #define pmap_copy_page(s, d) pmap_copy_page_generic((s), (d)) 1061 #define pmap_zero_page(d) pmap_zero_page_generic((d)) 1062 #elif ARM_MMU_XSCALE == 1 1063 #define L1_S_PROT_U L1_S_PROT_U_generic 1064 #define L1_S_PROT_W L1_S_PROT_W_generic 1065 #define L1_S_PROT_RO L1_S_PROT_RO_generic 1066 #define L1_S_PROT_MASK L1_S_PROT_MASK_generic 1067 1068 #define L2_S_PROT_U L2_S_PROT_U_xscale 1069 #define L2_S_PROT_W L2_S_PROT_W_xscale 1070 #define L2_S_PROT_RO L2_S_PROT_RO_xscale 1071 #define L2_S_PROT_MASK L2_S_PROT_MASK_xscale 1072 1073 #define L2_L_PROT_U L2_L_PROT_U_generic 1074 #define L2_L_PROT_W L2_L_PROT_W_generic 1075 #define L2_L_PROT_RO L2_L_PROT_RO_generic 1076 #define L2_L_PROT_MASK L2_L_PROT_MASK_generic 1077 1078 #define L1_S_CACHE_MASK L1_S_CACHE_MASK_xscale 1079 #define L2_L_CACHE_MASK L2_L_CACHE_MASK_xscale 1080 #define L2_S_CACHE_MASK L2_S_CACHE_MASK_xscale 1081 1082 #define L1_SS_PROTO L1_SS_PROTO_xscale 1083 #define L1_S_PROTO L1_S_PROTO_xscale 1084 #define L1_C_PROTO L1_C_PROTO_xscale 1085 #define L2_S_PROTO L2_S_PROTO_xscale 1086 1087 #define pmap_copy_page(s, d) pmap_copy_page_xscale((s), (d)) 1088 #define pmap_zero_page(d) pmap_zero_page_xscale((d)) 1089 #elif ARM_MMU_V7 == 1 1090 #define L1_S_PROT_U L1_S_PROT_U_armv7 1091 #define L1_S_PROT_W L1_S_PROT_W_armv7 1092 #define L1_S_PROT_RO L1_S_PROT_RO_armv7 1093 #define L1_S_PROT_MASK L1_S_PROT_MASK_armv7 1094 1095 #define L2_S_PROT_U L2_S_PROT_U_armv7 1096 #define L2_S_PROT_W L2_S_PROT_W_armv7 1097 #define L2_S_PROT_RO L2_S_PROT_RO_armv7 1098 #define L2_S_PROT_MASK L2_S_PROT_MASK_armv7 1099 1100 #define L2_L_PROT_U L2_L_PROT_U_armv7 1101 #define L2_L_PROT_W L2_L_PROT_W_armv7 1102 #define L2_L_PROT_RO L2_L_PROT_RO_armv7 1103 #define L2_L_PROT_MASK L2_L_PROT_MASK_armv7 1104 1105 #define L1_S_CACHE_MASK L1_S_CACHE_MASK_armv7 1106 #define L2_L_CACHE_MASK L2_L_CACHE_MASK_armv7 1107 #define L2_S_CACHE_MASK L2_S_CACHE_MASK_armv7 1108 1109 /* 1110 * These prototypes make writeable mappings, while the other MMU types 1111 * make read-only mappings. 1112 */ 1113 #define L1_SS_PROTO L1_SS_PROTO_armv7 1114 #define L1_S_PROTO L1_S_PROTO_armv7 1115 #define L1_C_PROTO L1_C_PROTO_armv7 1116 #define L2_S_PROTO L2_S_PROTO_armv7 1117 1118 #define pmap_copy_page(s, d) pmap_copy_page_generic((s), (d)) 1119 #define pmap_zero_page(d) pmap_zero_page_generic((d)) 1120 #endif /* ARM_NMMUS > 1 */ 1121 1122 /* 1123 * Macros to set and query the write permission on page descriptors. 1124 */ 1125 #define l1pte_set_writable(pte) (((pte) & ~L1_S_PROT_RO) | L1_S_PROT_W) 1126 #define l1pte_set_readonly(pte) (((pte) & ~L1_S_PROT_W) | L1_S_PROT_RO) 1127 1128 #define l2pte_set_writable(pte) (((pte) & ~L2_S_PROT_RO) | L2_S_PROT_W) 1129 #define l2pte_set_readonly(pte) (((pte) & ~L2_S_PROT_W) | L2_S_PROT_RO) 1130 1131 #define l2pte_writable_p(pte) (((pte) & L2_S_PROT_W) == L2_S_PROT_W && \ 1132 (L2_S_PROT_RO == 0 || \ 1133 ((pte) & L2_S_PROT_RO) != L2_S_PROT_RO)) 1134 1135 /* 1136 * These macros return various bits based on kernel/user and protection. 1137 * Note that the compiler will usually fold these at compile time. 1138 */ 1139 1140 #define L1_S_PROT(ku, pr) ( \ 1141 (((ku) == PTE_USER) ? \ 1142 L1_S_PROT_U | (((pr) & VM_PROT_WRITE) ? L1_S_PROT_W : 0) \ 1143 : \ 1144 (((L1_S_PROT_RO && \ 1145 ((pr) & (VM_PROT_READ | VM_PROT_WRITE)) == VM_PROT_READ) ? \ 1146 L1_S_PROT_RO : L1_S_PROT_W))) \ 1147 ) 1148 1149 #define L2_L_PROT(ku, pr) ( \ 1150 (((ku) == PTE_USER) ? \ 1151 L2_L_PROT_U | (((pr) & VM_PROT_WRITE) ? L2_L_PROT_W : 0) \ 1152 : \ 1153 (((L2_L_PROT_RO && \ 1154 ((pr) & (VM_PROT_READ | VM_PROT_WRITE)) == VM_PROT_READ) ? \ 1155 L2_L_PROT_RO : L2_L_PROT_W))) \ 1156 ) 1157 1158 #define L2_S_PROT(ku, pr) ( \ 1159 (((ku) == PTE_USER) ? \ 1160 L2_S_PROT_U | (((pr) & VM_PROT_WRITE) ? L2_S_PROT_W : 0) \ 1161 : \ 1162 (((L2_S_PROT_RO && \ 1163 ((pr) & (VM_PROT_READ | VM_PROT_WRITE)) == VM_PROT_READ) ? \ 1164 L2_S_PROT_RO : L2_S_PROT_W))) \ 1165 ) 1166 1167 /* 1168 * Macros to test if a mapping is mappable with an L1 SuperSection, 1169 * L1 Section, or an L2 Large Page mapping. 1170 */ 1171 #define L1_SS_MAPPABLE_P(va, pa, size) \ 1172 ((((va) | (pa)) & L1_SS_OFFSET) == 0 && (size) >= L1_SS_SIZE) 1173 1174 #define L1_S_MAPPABLE_P(va, pa, size) \ 1175 ((((va) | (pa)) & L1_S_OFFSET) == 0 && (size) >= L1_S_SIZE) 1176 1177 #define L2_L_MAPPABLE_P(va, pa, size) \ 1178 ((((va) | (pa)) & L2_L_OFFSET) == 0 && (size) >= L2_L_SIZE) 1179 1180 #define PMAP_MAPSIZE1 L2_L_SIZE 1181 #define PMAP_MAPSIZE2 L1_S_SIZE 1182 #if (ARM_MMU_V6 + ARM_MMU_V7) > 0 1183 #define PMAP_MAPSIZE3 L1_SS_SIZE 1184 #endif 1185 1186 #ifndef _LOCORE 1187 /* 1188 * Hooks for the pool allocator. 1189 */ 1190 #define POOL_VTOPHYS(va) vtophys((vaddr_t) (va)) 1191 extern paddr_t physical_start, physical_end; 1192 #ifdef PMAP_NEED_ALLOC_POOLPAGE 1193 struct vm_page *arm_pmap_alloc_poolpage(int); 1194 #define PMAP_ALLOC_POOLPAGE arm_pmap_alloc_poolpage 1195 #endif 1196 #if defined(PMAP_NEED_ALLOC_POOLPAGE) || defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) 1197 vaddr_t pmap_map_poolpage(paddr_t); 1198 paddr_t pmap_unmap_poolpage(vaddr_t); 1199 #define PMAP_MAP_POOLPAGE(pa) pmap_map_poolpage(pa) 1200 #define PMAP_UNMAP_POOLPAGE(va) pmap_unmap_poolpage(va) 1201 #endif 1202 1203 #define __HAVE_PMAP_PV_TRACK 1 1204 1205 void pmap_pv_protect(paddr_t, vm_prot_t); 1206 1207 struct pmap_page { 1208 SLIST_HEAD(,pv_entry) pvh_list; /* pv_entry list */ 1209 int pvh_attrs; /* page attributes */ 1210 u_int uro_mappings; 1211 u_int urw_mappings; 1212 union { 1213 u_short s_mappings[2]; /* Assume kernel count <= 65535 */ 1214 u_int i_mappings; 1215 } k_u; 1216 }; 1217 1218 /* 1219 * pmap-specific data store in the vm_page structure. 1220 */ 1221 #define __HAVE_VM_PAGE_MD 1222 struct vm_page_md { 1223 struct pmap_page pp; 1224 #define pvh_list pp.pvh_list 1225 #define pvh_attrs pp.pvh_attrs 1226 #define uro_mappings pp.uro_mappings 1227 #define urw_mappings pp.urw_mappings 1228 #define kro_mappings pp.k_u.s_mappings[0] 1229 #define krw_mappings pp.k_u.s_mappings[1] 1230 #define k_mappings pp.k_u.i_mappings 1231 }; 1232 1233 #define PMAP_PAGE_TO_MD(ppage) container_of((ppage), struct vm_page_md, pp) 1234 1235 /* 1236 * Set the default color of each page. 1237 */ 1238 #if ARM_MMU_V6 > 0 1239 #define VM_MDPAGE_PVH_ATTRS_INIT(pg) \ 1240 (pg)->mdpage.pvh_attrs = VM_PAGE_TO_PHYS(pg) & arm_cache_prefer_mask 1241 #else 1242 #define VM_MDPAGE_PVH_ATTRS_INIT(pg) \ 1243 (pg)->mdpage.pvh_attrs = 0 1244 #endif 1245 1246 #define VM_MDPAGE_INIT(pg) \ 1247 do { \ 1248 SLIST_INIT(&(pg)->mdpage.pvh_list); \ 1249 VM_MDPAGE_PVH_ATTRS_INIT(pg); \ 1250 (pg)->mdpage.uro_mappings = 0; \ 1251 (pg)->mdpage.urw_mappings = 0; \ 1252 (pg)->mdpage.k_mappings = 0; \ 1253 } while (/*CONSTCOND*/0) 1254 1255 #ifndef __BSD_PTENTRY_T__ 1256 #define __BSD_PTENTRY_T__ 1257 typedef uint32_t pt_entry_t; 1258 #define PRIxPTE PRIx32 1259 #endif 1260 1261 #endif /* !_LOCORE */ 1262 1263 #endif /* _KERNEL */ 1264 1265 #endif /* _ARM32_PMAP_H_ */ 1266
Indexes created Sun Nov 01 00:25:34 UTC 2026