pmap.h revision 1.59
1/* $NetBSD: pmap.h,v 1.59 2019/02/11 14:59:32 cherry Exp $ */ 2 3/* 4 * Copyright (c) 1997 Charles D. Cranor and Washington University. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28/* 29 * Copyright (c) 2001 Wasabi Systems, Inc. 30 * All rights reserved. 31 * 32 * Written by Frank van der Linden for Wasabi Systems, Inc. 33 * 34 * Redistribution and use in source and binary forms, with or without 35 * modification, are permitted provided that the following conditions 36 * are met: 37 * 1. Redistributions of source code must retain the above copyright 38 * notice, this list of conditions and the following disclaimer. 39 * 2. Redistributions in binary form must reproduce the above copyright 40 * notice, this list of conditions and the following disclaimer in the 41 * documentation and/or other materials provided with the distribution. 42 * 3. All advertising materials mentioning features or use of this software 43 * must display the following acknowledgement: 44 * This product includes software developed for the NetBSD Project by 45 * Wasabi Systems, Inc. 46 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 47 * or promote products derived from this software without specific prior 48 * written permission. 49 * 50 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 51 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 52 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 53 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 54 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 55 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 56 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 57 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 58 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 59 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 60 * POSSIBILITY OF SUCH DAMAGE. 61 */ 62 63#ifndef _AMD64_PMAP_H_ 64#define _AMD64_PMAP_H_ 65 66#ifdef __x86_64__ 67 68#if defined(_KERNEL_OPT) 69#include "opt_xen.h" 70#include "opt_kasan.h" 71#endif 72 73#include <sys/atomic.h> 74 75#include <machine/pte.h> 76#include <machine/segments.h> 77#ifdef _KERNEL 78#include <machine/cpufunc.h> 79#endif 80 81#include <uvm/uvm_object.h> 82#ifdef XENPV 83#include <xen/xenfunc.h> 84#include <xen/xenpmap.h> 85#endif 86 87/* 88 * Mask to get rid of the sign-extended part of addresses. 89 */ 90#define VA_SIGN_MASK 0xffff000000000000 91#define VA_SIGN_NEG(va) ((va) | VA_SIGN_MASK) 92/* XXXfvdl this one's not right. */ 93#define VA_SIGN_POS(va) ((va) & ~VA_SIGN_MASK) 94 95#ifdef KASAN 96#define L4_SLOT_KASAN 256 97#define NL4_SLOT_KASAN 32 98#endif 99 100#define NL4_SLOT_DIRECT 32 101 102#ifndef XENPV 103#define L4_SLOT_PTE slotspace.area[SLAREA_PTE].sslot 104#else 105#define L4_SLOT_PTE 509 106#endif 107#define L4_SLOT_KERN slotspace.area[SLAREA_MAIN].sslot 108#define L4_SLOT_KERNBASE 511 /* pl4_i(KERNBASE) */ 109 110#define PDIR_SLOT_USERLIM 255 111#define PDIR_SLOT_KERN L4_SLOT_KERN 112#define PDIR_SLOT_PTE L4_SLOT_PTE 113 114/* 115 * The following defines give the virtual addresses of various MMU 116 * data structures: 117 * PTE_BASE: the base VA of the linear PTE mappings 118 * PDP_BASE: the base VA of the recursive mapping of the PTD 119 */ 120 121#ifndef XENPV 122extern pt_entry_t *pte_base; 123#define PTE_BASE pte_base 124#else 125#define PTE_BASE ((pt_entry_t *)VA_SIGN_NEG((L4_SLOT_PTE * NBPD_L4))) 126#endif 127 128#define L1_BASE PTE_BASE 129#define L2_BASE ((pd_entry_t *)((char *)L1_BASE + L4_SLOT_PTE * NBPD_L3)) 130#define L3_BASE ((pd_entry_t *)((char *)L2_BASE + L4_SLOT_PTE * NBPD_L2)) 131#define L4_BASE ((pd_entry_t *)((char *)L3_BASE + L4_SLOT_PTE * NBPD_L1)) 132 133#define PDP_BASE L4_BASE 134 135#define NKL4_MAX_ENTRIES (unsigned long)64 136#define NKL3_MAX_ENTRIES (unsigned long)(NKL4_MAX_ENTRIES * 512) 137#define NKL2_MAX_ENTRIES (unsigned long)(NKL3_MAX_ENTRIES * 512) 138#define NKL1_MAX_ENTRIES (unsigned long)(NKL2_MAX_ENTRIES * 512) 139 140#define NKL4_KIMG_ENTRIES 1 141#define NKL3_KIMG_ENTRIES 1 142#define NKL2_KIMG_ENTRIES 48 143 144/* 145 * Since kva space is below the kernel in its entirety, we start off 146 * with zero entries on each level. 147 */ 148#define NKL4_START_ENTRIES 0 149#define NKL3_START_ENTRIES 0 150#define NKL2_START_ENTRIES 0 151#define NKL1_START_ENTRIES 0 152 153#define PTP_MASK_INITIALIZER { L1_MASK, L2_MASK, L3_MASK, L4_MASK } 154#define PTP_FRAME_INITIALIZER { L1_FRAME, L2_FRAME, L3_FRAME, L4_FRAME } 155#define PTP_SHIFT_INITIALIZER { L1_SHIFT, L2_SHIFT, L3_SHIFT, L4_SHIFT } 156#define NKPTP_INITIALIZER { NKL1_START_ENTRIES, NKL2_START_ENTRIES, \ 157 NKL3_START_ENTRIES, NKL4_START_ENTRIES } 158#define NKPTPMAX_INITIALIZER { NKL1_MAX_ENTRIES, NKL2_MAX_ENTRIES, \ 159 NKL3_MAX_ENTRIES, NKL4_MAX_ENTRIES } 160#define NBPD_INITIALIZER { NBPD_L1, NBPD_L2, NBPD_L3, NBPD_L4 } 161#define PDES_INITIALIZER { L2_BASE, L3_BASE, L4_BASE } 162 163#define PTP_LEVELS 4 164 165/* 166 * PG_AVAIL usage: we make use of the ignored bits of the PTE 167 */ 168 169#define PG_W PG_AVAIL1 /* "wired" mapping */ 170#define PG_PVLIST PG_AVAIL2 /* mapping has entry on pvlist */ 171/* PG_AVAIL3 not used */ 172 173#define PG_X 0 /* dummy */ 174 175void svs_pmap_sync(struct pmap *, int); 176void svs_lwp_switch(struct lwp *, struct lwp *); 177void svs_pdir_switch(struct pmap *); 178void svs_init(void); 179extern bool svs_enabled; 180 181#include <x86/pmap.h> 182 183#ifndef XENPV 184#define pmap_pa2pte(a) (a) 185#define pmap_pte2pa(a) ((a) & PG_FRAME) 186#define pmap_pte_set(p, n) do { *(p) = (n); } while (0) 187#define pmap_pte_cas(p, o, n) atomic_cas_64((p), (o), (n)) 188#define pmap_pte_testset(p, n) \ 189 atomic_swap_ulong((volatile unsigned long *)p, n) 190#define pmap_pte_setbits(p, b) \ 191 atomic_or_ulong((volatile unsigned long *)p, b) 192#define pmap_pte_clearbits(p, b) \ 193 atomic_and_ulong((volatile unsigned long *)p, ~(b)) 194#define pmap_pte_flush() /* nothing */ 195#else 196extern kmutex_t pte_lock; 197 198static __inline pt_entry_t 199pmap_pa2pte(paddr_t pa) 200{ 201 return (pt_entry_t)xpmap_ptom_masked(pa); 202} 203 204static __inline paddr_t 205pmap_pte2pa(pt_entry_t pte) 206{ 207 return xpmap_mtop_masked(pte & PG_FRAME); 208} 209 210static __inline void 211pmap_pte_set(pt_entry_t *pte, pt_entry_t npte) 212{ 213 int s = splvm(); 214 xpq_queue_pte_update(xpmap_ptetomach(pte), npte); 215 splx(s); 216} 217 218static __inline pt_entry_t 219pmap_pte_cas(volatile pt_entry_t *ptep, pt_entry_t o, pt_entry_t n) 220{ 221 pt_entry_t opte; 222 223 mutex_enter(&pte_lock); 224 opte = *ptep; 225 if (opte == o) { 226 xpq_queue_pte_update(xpmap_ptetomach(__UNVOLATILE(ptep)), n); 227 xpq_flush_queue(); 228 } 229 230 mutex_exit(&pte_lock); 231 return opte; 232} 233 234static __inline pt_entry_t 235pmap_pte_testset(volatile pt_entry_t *pte, pt_entry_t npte) 236{ 237 pt_entry_t opte; 238 239 mutex_enter(&pte_lock); 240 opte = *pte; 241 xpq_queue_pte_update(xpmap_ptetomach(__UNVOLATILE(pte)), npte); 242 xpq_flush_queue(); 243 mutex_exit(&pte_lock); 244 return opte; 245} 246 247static __inline void 248pmap_pte_setbits(volatile pt_entry_t *pte, pt_entry_t bits) 249{ 250 mutex_enter(&pte_lock); 251 xpq_queue_pte_update(xpmap_ptetomach(__UNVOLATILE(pte)), (*pte) | bits); 252 xpq_flush_queue(); 253 mutex_exit(&pte_lock); 254} 255 256static __inline void 257pmap_pte_clearbits(volatile pt_entry_t *pte, pt_entry_t bits) 258{ 259 mutex_enter(&pte_lock); 260 xpq_queue_pte_update(xpmap_ptetomach(__UNVOLATILE(pte)), 261 (*pte) & ~bits); 262 xpq_flush_queue(); 263 mutex_exit(&pte_lock); 264} 265 266static __inline void 267pmap_pte_flush(void) 268{ 269 int s = splvm(); 270 xpq_flush_queue(); 271 splx(s); 272} 273#endif 274 275#ifdef __HAVE_DIRECT_MAP 276#define PMAP_DIRECT 277 278static __inline int 279pmap_direct_process(paddr_t pa, voff_t pgoff, size_t len, 280 int (*process)(void *, size_t, void *), void *arg) 281{ 282 vaddr_t va = PMAP_DIRECT_MAP(pa); 283 284 return process((void *)(va + pgoff), len, arg); 285} 286 287#endif /* __HAVE_DIRECT_MAP */ 288 289void pmap_changeprot_local(vaddr_t, vm_prot_t); 290 291#include <x86/pmap_pv.h> 292 293#define __HAVE_VM_PAGE_MD 294#define VM_MDPAGE_INIT(pg) \ 295 memset(&(pg)->mdpage, 0, sizeof((pg)->mdpage)); \ 296 PMAP_PAGE_INIT(&(pg)->mdpage.mp_pp) 297 298struct vm_page_md { 299 struct pmap_page mp_pp; 300}; 301 302#else /* !__x86_64__ */ 303 304#include <i386/pmap.h> 305 306#endif /* __x86_64__ */ 307 308#endif /* _AMD64_PMAP_H_ */ 309