1 /* $NetBSD: pmap_private.h,v 1.4 2022/08/21 09:12:43 riastradh Exp $ */ 2 3 /* 4 * Copyright (c) 1997 Charles D. Cranor and Washington University. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 /* 29 * Copyright (c) 2001 Wasabi Systems, Inc. 30 * All rights reserved. 31 * 32 * Written by Frank van der Linden for Wasabi Systems, Inc. 33 * 34 * Redistribution and use in source and binary forms, with or without 35 * modification, are permitted provided that the following conditions 36 * are met: 37 * 1. Redistributions of source code must retain the above copyright 38 * notice, this list of conditions and the following disclaimer. 39 * 2. Redistributions in binary form must reproduce the above copyright 40 * notice, this list of conditions and the following disclaimer in the 41 * documentation and/or other materials provided with the distribution. 42 * 3. All advertising materials mentioning features or use of this software 43 * must display the following acknowledgement: 44 * This product includes software developed for the NetBSD Project by 45 * Wasabi Systems, Inc. 46 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 47 * or promote products derived from this software without specific prior 48 * written permission. 49 * 50 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 51 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 52 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 53 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 54 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 55 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 56 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 57 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 58 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 59 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 60 * POSSIBILITY OF SUCH DAMAGE. 61 */ 62 63 #ifndef _AMD64_PMAP_PRIVATE_H_ 64 #define _AMD64_PMAP_PRIVATE_H_ 65 66 #ifdef __x86_64__ 67 68 #if defined(_KERNEL_OPT) 69 #include "opt_xen.h" 70 #include "opt_kasan.h" 71 #include "opt_kmsan.h" 72 #include "opt_kubsan.h" 73 #endif 74 75 #include <sys/atomic.h> 76 77 #include <machine/pte.h> 78 #include <machine/segments.h> 79 #ifdef _KERNEL 80 #include <machine/cpufunc.h> 81 #endif 82 83 #include <uvm/uvm_object.h> 84 #ifdef XENPV 85 #include <xen/xenfunc.h> 86 #include <xen/xenpmap.h> 87 #endif 88 89 #ifdef KASAN 90 #define L4_SLOT_KASAN 256 91 #define NL4_SLOT_KASAN 32 92 #endif 93 94 #ifdef KMSAN 95 #define L4_SLOT_KMSAN 256 96 #define NL4_SLOT_KMSAN 4 97 #endif 98 99 #define NL4_SLOT_DIRECT 32 100 101 #ifndef XENPV 102 #define L4_SLOT_PTE slotspace.area[SLAREA_PTE].sslot 103 #else 104 #define L4_SLOT_PTE 509 105 #endif 106 #define L4_SLOT_KERN slotspace.area[SLAREA_MAIN].sslot 107 #define L4_SLOT_KERNBASE 511 /* pl4_i(KERNBASE) */ 108 109 #define PDIR_SLOT_USERLIM 255 110 #define PDIR_SLOT_KERN L4_SLOT_KERN 111 #define PDIR_SLOT_PTE L4_SLOT_PTE 112 113 /* 114 * The following defines give the virtual addresses of various MMU 115 * data structures: 116 * PTE_BASE: the base VA of the linear PTE mappings 117 * PDP_BASE: the base VA of the recursive mapping of the PTD 118 */ 119 120 #ifndef XENPV 121 extern pt_entry_t *pte_base; 122 #define PTE_BASE pte_base 123 #else 124 #define PTE_BASE ((pt_entry_t *)VA_SIGN_NEG((L4_SLOT_PTE * NBPD_L4))) 125 #endif 126 127 #define L1_BASE PTE_BASE 128 #define L2_BASE ((pd_entry_t *)((char *)L1_BASE + L4_SLOT_PTE * NBPD_L3)) 129 #define L3_BASE ((pd_entry_t *)((char *)L2_BASE + L4_SLOT_PTE * NBPD_L2)) 130 #define L4_BASE ((pd_entry_t *)((char *)L3_BASE + L4_SLOT_PTE * NBPD_L1)) 131 132 #define PDP_BASE L4_BASE 133 134 #if defined(KMSAN) 135 #define NKL4_MAX_ENTRIES (unsigned long)1 /* 512GB only */ 136 #else 137 #define NKL4_MAX_ENTRIES (unsigned long)64 138 #endif 139 #define NKL3_MAX_ENTRIES (unsigned long)(NKL4_MAX_ENTRIES * 512) 140 #define NKL2_MAX_ENTRIES (unsigned long)(NKL3_MAX_ENTRIES * 512) 141 #define NKL1_MAX_ENTRIES (unsigned long)(NKL2_MAX_ENTRIES * 512) 142 143 #define NKL4_KIMG_ENTRIES 1 144 #define NKL3_KIMG_ENTRIES 1 145 #if defined(KUBSAN) || defined(KMSAN) 146 #define NKL2_KIMG_ENTRIES 64 /* really big kernel */ 147 #else 148 #define NKL2_KIMG_ENTRIES 48 149 #endif 150 151 /* 152 * Since kva space is below the kernel in its entirety, we start off 153 * with zero entries on each level. 154 */ 155 #define NKL4_START_ENTRIES 0 156 #define NKL3_START_ENTRIES 0 157 #define NKL2_START_ENTRIES 0 158 #define NKL1_START_ENTRIES 0 159 160 #define PTP_MASK_INITIALIZER { L1_MASK, L2_MASK, L3_MASK, L4_MASK } 161 #define PTP_FRAME_INITIALIZER { L1_FRAME, L2_FRAME, L3_FRAME, L4_FRAME } 162 #define PTP_SHIFT_INITIALIZER { L1_SHIFT, L2_SHIFT, L3_SHIFT, L4_SHIFT } 163 #define NKPTP_INITIALIZER { NKL1_START_ENTRIES, NKL2_START_ENTRIES, \ 164 NKL3_START_ENTRIES, NKL4_START_ENTRIES } 165 #define NKPTPMAX_INITIALIZER { NKL1_MAX_ENTRIES, NKL2_MAX_ENTRIES, \ 166 NKL3_MAX_ENTRIES, NKL4_MAX_ENTRIES } 167 #define NBPD_INITIALIZER { NBPD_L1, NBPD_L2, NBPD_L3, NBPD_L4 } 168 #define PDES_INITIALIZER { L2_BASE, L3_BASE, L4_BASE } 169 170 #define PTP_LEVELS 4 171 172 /* 173 * PTE_AVL usage: we make use of the ignored bits of the PTE 174 */ 175 #define PTE_WIRED PTE_AVL1 /* Wired Mapping */ 176 #define PTE_PVLIST PTE_AVL2 /* Mapping has entry on pvlist */ 177 #define PTE_X 0 /* Dummy */ 178 179 /* XXX To be deleted. */ 180 #define PG_W PTE_WIRED 181 #define PG_PVLIST PTE_PVLIST 182 #define PG_X PTE_X 183 184 void svs_pmap_sync(struct pmap *, int); 185 void svs_ldt_sync(struct pmap *); 186 void svs_lwp_switch(struct lwp *, struct lwp *); 187 void svs_pdir_switch(struct pmap *); 188 void svs_init(void); 189 extern bool svs_enabled; 190 extern bool svs_pcid; 191 192 #define _MACHINE_PMAP_PRIVATE_H_X86 193 #include <x86/pmap_private.h> 194 #undef _MACHINE_PMAP_PRIVATE_H_X86 195 196 #ifndef XENPV 197 198 #define pmap_pa2pte(a) (a) 199 #define pmap_pte2pa(a) ((a) & PTE_FRAME) 200 #define pmap_pte_set(p, n) do { *(p) = (n); } while (0) 201 #define pmap_pte_cas(p, o, n) atomic_cas_64((p), (o), (n)) 202 #define pmap_pte_testset(p, n) \ 203 atomic_swap_ulong((volatile unsigned long *)p, n) 204 #define pmap_pte_setbits(p, b) \ 205 atomic_or_ulong((volatile unsigned long *)p, b) 206 #define pmap_pte_clearbits(p, b) \ 207 atomic_and_ulong((volatile unsigned long *)p, ~(b)) 208 #define pmap_pte_flush() /* nothing */ 209 210 #else 211 212 extern kmutex_t pte_lock; 213 214 static __inline pt_entry_t 215 pmap_pa2pte(paddr_t pa) 216 { 217 return (pt_entry_t)xpmap_ptom_masked(pa); 218 } 219 220 static __inline paddr_t 221 pmap_pte2pa(pt_entry_t pte) 222 { 223 return xpmap_mtop_masked(pte & PTE_FRAME); 224 } 225 226 static __inline void 227 pmap_pte_set(pt_entry_t *pte, pt_entry_t npte) 228 { 229 int s = splvm(); 230 xpq_queue_pte_update(xpmap_ptetomach(pte), npte); 231 splx(s); 232 } 233 234 static __inline pt_entry_t 235 pmap_pte_cas(volatile pt_entry_t *ptep, pt_entry_t o, pt_entry_t n) 236 { 237 pt_entry_t opte; 238 239 mutex_enter(&pte_lock); 240 opte = *ptep; 241 if (opte == o) { 242 xpq_queue_pte_update(xpmap_ptetomach(__UNVOLATILE(ptep)), n); 243 xpq_flush_queue(); 244 } 245 mutex_exit(&pte_lock); 246 return opte; 247 } 248 249 static __inline pt_entry_t 250 pmap_pte_testset(volatile pt_entry_t *pte, pt_entry_t npte) 251 { 252 pt_entry_t opte; 253 254 mutex_enter(&pte_lock); 255 opte = *pte; 256 xpq_queue_pte_update(xpmap_ptetomach(__UNVOLATILE(pte)), npte); 257 xpq_flush_queue(); 258 mutex_exit(&pte_lock); 259 return opte; 260 } 261 262 static __inline void 263 pmap_pte_setbits(volatile pt_entry_t *pte, pt_entry_t bits) 264 { 265 mutex_enter(&pte_lock); 266 xpq_queue_pte_update(xpmap_ptetomach(__UNVOLATILE(pte)), (*pte) | bits); 267 xpq_flush_queue(); 268 mutex_exit(&pte_lock); 269 } 270 271 static __inline void 272 pmap_pte_clearbits(volatile pt_entry_t *pte, pt_entry_t bits) 273 { 274 mutex_enter(&pte_lock); 275 xpq_queue_pte_update(xpmap_ptetomach(__UNVOLATILE(pte)), 276 (*pte) & ~bits); 277 xpq_flush_queue(); 278 mutex_exit(&pte_lock); 279 } 280 281 static __inline void 282 pmap_pte_flush(void) 283 { 284 int s = splvm(); 285 xpq_flush_queue(); 286 splx(s); 287 } 288 289 #endif 290 291 #ifdef __HAVE_DIRECT_MAP 292 #define PMAP_DIRECT 293 294 static __inline int 295 pmap_direct_process(paddr_t pa, voff_t pgoff, size_t len, 296 int (*process)(void *, size_t, void *), void *arg) 297 { 298 vaddr_t va = PMAP_DIRECT_MAP(pa); 299 300 return process((void *)(va + pgoff), len, arg); 301 } 302 303 #endif /* __HAVE_DIRECT_MAP */ 304 305 void pmap_changeprot_local(vaddr_t, vm_prot_t); 306 307 #else /* !__x86_64__ */ 308 309 #include <i386/pmap_private.h> 310 311 #endif /* __x86_64__ */ 312 313 #endif /* _AMD64_PMAP_PRIVATE_H_ */ 314
Indexes created Mon Sep 22 13:09:51 GMT 2025