Home | History | Annotate | Line # | Download | only in include
pmap.h revision 1.55 
      1 /* $NetBSD: pmap.h,v 1.55 2022/10/23 07:13:07 skrll Exp $ */
      2 
      3 /*-
      4  * Copyright (c) 2014 The NetBSD Foundation, Inc.
      5  * All rights reserved.
      6  *
      7  * This code is derived from software contributed to The NetBSD Foundation
      8  * by Matt Thomas of 3am Software Foundry.
      9  *
     10  * Redistribution and use in source and binary forms, with or without
     11  * modification, are permitted provided that the following conditions
     12  * are met:
     13  * 1. Redistributions of source code must retain the above copyright
     14  *    notice, this list of conditions and the following disclaimer.
     15  * 2. Redistributions in binary form must reproduce the above copyright
     16  *    notice, this list of conditions and the following disclaimer in the
     17  *    documentation and/or other materials provided with the distribution.
     18  *
     19  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     21  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     22  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     23  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     29  * POSSIBILITY OF SUCH DAMAGE.
     30  */
     31 
     32 #ifndef _AARCH64_PMAP_H_
     33 #define _AARCH64_PMAP_H_
     34 
     35 #ifdef __aarch64__
     36 
     37 #ifdef _KERNEL
     38 #ifdef _KERNEL_OPT
     39 #include "opt_kasan.h"
     40 #endif
     41 
     42 #include <sys/types.h>
     43 #include <sys/pool.h>
     44 #include <sys/queue.h>
     45 
     46 #include <uvm/uvm_pglist.h>
     47 
     48 #include <aarch64/armreg.h>
     49 #include <aarch64/pte.h>
     50 
     51 #define PMAP_NEED_PROCWR
     52 #define PMAP_GROWKERNEL
     53 #define PMAP_STEAL_MEMORY
     54 
     55 #define __HAVE_VM_PAGE_MD
     56 #define __HAVE_PMAP_PV_TRACK	1
     57 
     58 #define	PMAP_HWPAGEWALKER		1
     59 
     60 #define	PMAP_TLB_MAX			1
     61 #if PMAP_TLB_MAX > 1
     62 #define	PMAP_TLB_NEED_SHOOTDOWN		1
     63 #endif
     64 
     65 #define	PMAP_TLB_FLUSH_ASID_ON_RESET	true
     66 
     67 /* Maximum number of ASIDs. Some CPUs have less.*/
     68 #define	PMAP_TLB_NUM_PIDS		65536
     69 #define	PMAP_TLB_BITMAP_LENGTH		PMAP_TLB_NUM_PIDS
     70 #define	cpu_set_tlb_info(ci, ti)        ((void)((ci)->ci_tlb_info = (ti)))
     71 #if PMAP_TLB_MAX > 1
     72 #define	cpu_tlb_info(ci)		((ci)->ci_tlb_info)
     73 #else
     74 #define	cpu_tlb_info(ci)		(&pmap_tlb0_info)
     75 #endif
     76 
     77 static inline tlb_asid_t
     78 pmap_md_tlb_asid_max(void)
     79 {
     80 	switch (__SHIFTOUT(reg_id_aa64mmfr0_el1_read(), ID_AA64MMFR0_EL1_ASIDBITS)) {
     81 	case ID_AA64MMFR0_EL1_ASIDBITS_8BIT:
     82 		return (1U << 8) - 1;
     83 	case ID_AA64MMFR0_EL1_ASIDBITS_16BIT:
     84 		return (1U << 16) - 1;
     85 	default:
     86 		return 0;
     87 	}
     88 }
     89 
     90 #include <uvm/pmap/tlb.h>
     91 #include <uvm/pmap/pmap_tlb.h>
     92 
     93 #define KERNEL_PID		0	/* The kernel uses ASID 0 */
     94 
     95 #ifndef KASAN
     96 #define PMAP_MAP_POOLPAGE(pa)		AARCH64_PA_TO_KVA(pa)
     97 #define PMAP_UNMAP_POOLPAGE(va)		AARCH64_KVA_TO_PA(va)
     98 
     99 #define PMAP_DIRECT
    100 static __inline int
    101 pmap_direct_process(paddr_t pa, voff_t pgoff, size_t len,
    102     int (*process)(void *, size_t, void *), void *arg)
    103 {
    104 	vaddr_t va = AARCH64_PA_TO_KVA(pa);
    105 
    106 	return process((void *)(va + pgoff), len, arg);
    107 }
    108 #endif
    109 
    110 struct pmap {
    111 	kmutex_t pm_lock;
    112 	struct pool *pm_pvpool;
    113 	pd_entry_t *pm_l0table;			/* L0 table: 512G*512 */
    114 	paddr_t pm_l0table_pa;
    115 
    116 	LIST_HEAD(, vm_page) pm_vmlist;		/* for L[0123] tables */
    117 	LIST_HEAD(, pv_entry) pm_pvlist;	/* all pv of this process */
    118 
    119 	struct pmap_statistics pm_stats;
    120 	unsigned int pm_refcnt;
    121 	unsigned int pm_idlepdp;
    122 
    123 	kcpuset_t *pm_onproc;
    124 	kcpuset_t *pm_active;
    125 
    126 	struct pmap_asid_info pm_pai[PMAP_TLB_MAX];
    127 	bool pm_activated;
    128 };
    129 
    130 static inline paddr_t
    131 pmap_l0pa(struct pmap *pm)
    132 {
    133 	return pm->pm_l0table_pa;
    134 }
    135 
    136 /*
    137  * should be kept <=32 bytes sized to reduce memory consumption & cache misses,
    138  * but it doesn't...
    139  */
    140 struct pv_entry {
    141 	struct pv_entry *pv_next;
    142 	struct pmap *pv_pmap;
    143 	vaddr_t pv_va;	/* for embedded entry (pp_pv) also includes flags */
    144 	void *pv_ptep;	/* pointer for fast pte lookup */
    145 	LIST_ENTRY(pv_entry) pv_proc;	/* belonging to the process */
    146 };
    147 
    148 struct pmap_page {
    149 	kmutex_t pp_pvlock;
    150 	struct pv_entry pp_pv;
    151 };
    152 
    153 /* try to keep vm_page at or under 128 bytes to reduce cache misses */
    154 struct vm_page_md {
    155 	struct pmap_page mdpg_pp;
    156 };
    157 /* for page descriptor page only */
    158 #define	mdpg_ptep_parent	mdpg_pp.pp_pv.pv_ptep
    159 
    160 #define VM_MDPAGE_INIT(pg)					\
    161 	do {							\
    162 		PMAP_PAGE_INIT(&(pg)->mdpage.mdpg_pp);		\
    163 	} while (/*CONSTCOND*/ 0)
    164 
    165 #define PMAP_PAGE_INIT(pp)						\
    166 	do {								\
    167 		mutex_init(&(pp)->pp_pvlock, MUTEX_NODEBUG, IPL_NONE);	\
    168 		(pp)->pp_pv.pv_next = NULL;				\
    169 		(pp)->pp_pv.pv_pmap = NULL;				\
    170 		(pp)->pp_pv.pv_va = 0;					\
    171 		(pp)->pp_pv.pv_ptep = NULL;				\
    172 	} while (/*CONSTCOND*/ 0)
    173 
    174 /* saved permission bit for referenced/modified emulation */
    175 #define LX_BLKPAG_OS_READ		LX_BLKPAG_OS_0
    176 #define LX_BLKPAG_OS_WRITE		LX_BLKPAG_OS_1
    177 #define LX_BLKPAG_OS_WIRED		LX_BLKPAG_OS_2
    178 #define LX_BLKPAG_OS_BOOT		LX_BLKPAG_OS_3
    179 #define LX_BLKPAG_OS_RWMASK		(LX_BLKPAG_OS_WRITE | LX_BLKPAG_OS_READ)
    180 
    181 #define PMAP_PTE_OS0	"read"
    182 #define PMAP_PTE_OS1	"write"
    183 #define PMAP_PTE_OS2	"wired"
    184 #define PMAP_PTE_OS3	"boot"
    185 
    186 /* memory attributes are configured MAIR_EL1 in locore */
    187 #define LX_BLKPAG_ATTR_NORMAL_WB	__SHIFTIN(0, LX_BLKPAG_ATTR_INDX)
    188 #define LX_BLKPAG_ATTR_NORMAL_NC	__SHIFTIN(1, LX_BLKPAG_ATTR_INDX)
    189 #define LX_BLKPAG_ATTR_NORMAL_WT	__SHIFTIN(2, LX_BLKPAG_ATTR_INDX)
    190 #define LX_BLKPAG_ATTR_DEVICE_MEM	__SHIFTIN(3, LX_BLKPAG_ATTR_INDX)
    191 #define LX_BLKPAG_ATTR_DEVICE_MEM_NP	__SHIFTIN(4, LX_BLKPAG_ATTR_INDX)
    192 #define LX_BLKPAG_ATTR_MASK		LX_BLKPAG_ATTR_INDX
    193 
    194 #define lxpde_pa(pde)		((paddr_t)((pde) & LX_TBL_PA))
    195 #define lxpde_valid(pde)	(((pde) & LX_VALID) == LX_VALID)
    196 #define l0pde_pa(pde)		lxpde_pa(pde)
    197 #define l0pde_index(v)		(((vaddr_t)(v) & L0_ADDR_BITS) >> L0_SHIFT)
    198 #define l0pde_valid(pde)	lxpde_valid(pde)
    199 /* l0pte always contains table entries */
    200 
    201 #define l1pde_pa(pde)		lxpde_pa(pde)
    202 #define l1pde_index(v)		(((vaddr_t)(v) & L1_ADDR_BITS) >> L1_SHIFT)
    203 #define l1pde_valid(pde)	lxpde_valid(pde)
    204 #define l1pde_is_block(pde)	(((pde) & LX_TYPE) == LX_TYPE_BLK)
    205 #define l1pde_is_table(pde)	(((pde) & LX_TYPE) == LX_TYPE_TBL)
    206 
    207 #define l2pde_pa(pde)		lxpde_pa(pde)
    208 #define l2pde_index(v)		(((vaddr_t)(v) & L2_ADDR_BITS) >> L2_SHIFT)
    209 #define l2pde_valid(pde)	lxpde_valid(pde)
    210 #define l2pde_is_block(pde)	(((pde) & LX_TYPE) == LX_TYPE_BLK)
    211 #define l2pde_is_table(pde)	(((pde) & LX_TYPE) == LX_TYPE_TBL)
    212 
    213 #define l3pte_pa(pde)		lxpde_pa(pde)
    214 #define l3pte_executable(pde,user)	\
    215     (((pde) & ((user) ? LX_BLKPAG_UXN : LX_BLKPAG_PXN)) == 0)
    216 #define l3pte_readable(pde)	((pde) & LX_BLKPAG_AF)
    217 #define l3pte_writable(pde)	\
    218     (((pde) & (LX_BLKPAG_AF | LX_BLKPAG_AP)) == (LX_BLKPAG_AF | LX_BLKPAG_AP_RW))
    219 #define l3pte_index(v)		(((vaddr_t)(v) & L3_ADDR_BITS) >> L3_SHIFT)
    220 #define l3pte_valid(pde)	lxpde_valid(pde)
    221 #define l3pte_is_page(pde)	(((pde) & LX_TYPE) == L3_TYPE_PAG)
    222 /* l3pte contains always page entries */
    223 
    224 
    225 static inline uint64_t
    226 pte_value(pt_entry_t pte)
    227 {
    228 	return pte;
    229 }
    230 
    231 static inline bool
    232 pte_valid_p(pt_entry_t pte)
    233 {
    234 	return l3pte_valid(pte);
    235 }
    236 
    237 void pmap_bootstrap(vaddr_t, vaddr_t);
    238 bool pmap_fault_fixup(struct pmap *, vaddr_t, vm_prot_t, bool user);
    239 
    240 /* for ddb */
    241 pt_entry_t *kvtopte(vaddr_t);
    242 void pmap_db_pmap_print(struct pmap *, void (*)(const char *, ...) __printflike(1, 2));
    243 void pmap_db_mdpg_print(struct vm_page *, void (*)(const char *, ...) __printflike(1, 2));
    244 
    245 pd_entry_t *pmap_l0table(struct pmap *);
    246 
    247 /* change attribute of kernel segment */
    248 static inline pt_entry_t
    249 pmap_kvattr(pt_entry_t *ptep, vm_prot_t prot)
    250 {
    251 	pt_entry_t pte = *ptep;
    252 	const pt_entry_t opte = pte;
    253 
    254 	pte &= ~(LX_BLKPAG_AF | LX_BLKPAG_AP);
    255 	switch (prot & (VM_PROT_READ | VM_PROT_WRITE)) {
    256 	case 0:
    257 		break;
    258 	case VM_PROT_READ:
    259 		pte |= (LX_BLKPAG_AF | LX_BLKPAG_AP_RO);
    260 		break;
    261 	case VM_PROT_WRITE:
    262 	case VM_PROT_READ|VM_PROT_WRITE:
    263 		pte |= (LX_BLKPAG_AF | LX_BLKPAG_AP_RW);
    264 		break;
    265 	}
    266 
    267 	if ((prot & VM_PROT_EXECUTE) == 0) {
    268 		pte |= LX_BLKPAG_PXN;
    269 	} else {
    270 		pte |= LX_BLKPAG_AF;
    271 		pte &= ~LX_BLKPAG_PXN;
    272 	}
    273 
    274 	*ptep = pte;
    275 
    276 	return opte;
    277 }
    278 
    279 /* pmapboot.c */
    280 pd_entry_t *pmapboot_pagealloc(void);
    281 void pmapboot_enter(vaddr_t, paddr_t, psize_t, psize_t, pt_entry_t,
    282     void (*pr)(const char *, ...) __printflike(1, 2));
    283 void pmapboot_enter_range(vaddr_t, paddr_t, psize_t, pt_entry_t,
    284     void (*)(const char *, ...) __printflike(1, 2));
    285 int pmapboot_protect(vaddr_t, vaddr_t, vm_prot_t);
    286 
    287 /* Hooks for the pool allocator */
    288 paddr_t vtophys(vaddr_t);
    289 #define VTOPHYS_FAILED		((paddr_t)-1L)	/* POOL_PADDR_INVALID */
    290 #define POOL_VTOPHYS(va)	vtophys((vaddr_t) (va))
    291 
    292 /* devmap */
    293 struct pmap_devmap {
    294 	vaddr_t pd_va;		/* virtual address */
    295 	paddr_t pd_pa;		/* physical address */
    296 	psize_t pd_size;	/* size of region */
    297 	vm_prot_t pd_prot;	/* protection code */
    298 	u_int pd_flags;		/* flags for pmap_kenter_pa() */
    299 };
    300 
    301 void pmap_devmap_register(const struct pmap_devmap *);
    302 void pmap_devmap_bootstrap(vaddr_t, const struct pmap_devmap *);
    303 const struct pmap_devmap *pmap_devmap_find_pa(paddr_t, psize_t);
    304 const struct pmap_devmap *pmap_devmap_find_va(vaddr_t, vsize_t);
    305 vaddr_t pmap_devmap_phystov(paddr_t);
    306 paddr_t pmap_devmap_vtophys(paddr_t);
    307 
    308 #define L1_TRUNC_BLOCK(x)	((x) & L1_FRAME)
    309 #define L1_ROUND_BLOCK(x)	L1_TRUNC_BLOCK((x) + L1_SIZE - 1)
    310 #define L2_TRUNC_BLOCK(x)	((x) & L2_FRAME)
    311 #define L2_ROUND_BLOCK(x)	L2_TRUNC_BLOCK((x) + L2_SIZE - 1)
    312 #define L3_TRUNC_BLOCK(x)	((x) & L3_FRAME)
    313 #define L3_ROUND_BLOCK(x)	L3_TRUNC_BLOCK((x) + L3_SIZE - 1)
    314 
    315 #define DEVMAP_ALIGN(x)		L3_TRUNC_BLOCK((x))
    316 #define DEVMAP_SIZE(x)		L3_ROUND_BLOCK((x))
    317 
    318 #define	DEVMAP_ENTRY(va, pa, sz)				\
    319 	{							\
    320 		.pd_va = DEVMAP_ALIGN(va),			\
    321 		.pd_pa = DEVMAP_ALIGN(pa),			\
    322 		.pd_size = DEVMAP_SIZE(sz),			\
    323 		.pd_prot = VM_PROT_READ | VM_PROT_WRITE,	\
    324 		.pd_flags = PMAP_DEV				\
    325 	}
    326 #define	DEVMAP_ENTRY_END	{ 0 }
    327 
    328 /* mmap cookie and flags */
    329 #define AARCH64_MMAP_FLAG_SHIFT		(64 - PGSHIFT)
    330 #define AARCH64_MMAP_FLAG_MASK		0xf
    331 #define AARCH64_MMAP_WRITEBACK		0UL
    332 #define AARCH64_MMAP_NOCACHE		1UL
    333 #define AARCH64_MMAP_WRITECOMBINE	2UL
    334 #define AARCH64_MMAP_DEVICE		3UL
    335 
    336 #define ARM_MMAP_MASK			__BITS(63, AARCH64_MMAP_FLAG_SHIFT)
    337 #define ARM_MMAP_WRITECOMBINE		__SHIFTIN(AARCH64_MMAP_WRITECOMBINE, ARM_MMAP_MASK)
    338 #define ARM_MMAP_WRITEBACK		__SHIFTIN(AARCH64_MMAP_WRITEBACK, ARM_MMAP_MASK)
    339 #define ARM_MMAP_NOCACHE		__SHIFTIN(AARCH64_MMAP_NOCACHE, ARM_MMAP_MASK)
    340 #define ARM_MMAP_DEVICE			__SHIFTIN(AARCH64_MMAP_DEVICE, ARM_MMAP_MASK)
    341 
    342 #define	PMAP_PTE			0x10000000 /* kenter_pa */
    343 #define	PMAP_DEV			0x20000000 /* kenter_pa */
    344 #define	PMAP_DEV_NP			0x40000000 /* kenter_pa */
    345 #define	PMAP_DEV_MASK			(PMAP_DEV | PMAP_DEV_NP)
    346 
    347 static inline u_int
    348 aarch64_mmap_flags(paddr_t mdpgno)
    349 {
    350 	u_int nflag, pflag;
    351 
    352 	/*
    353 	 * aarch64 arch has 5 memory attributes defined:
    354 	 *
    355 	 *  WriteBack      - write back cache
    356 	 *  WriteThru      - write through cache
    357 	 *  NoCache        - no cache
    358 	 *  Device(nGnRE)  - no Gathering, no Reordering, Early write ack
    359 	 *  Device(nGnRnE) - no Gathering, no Reordering, no Early write ack
    360 	 *
    361 	 * but pmap has PMAP_{NOCACHE,WRITE_COMBINE,WRITE_BACK} flags.
    362 	 */
    363 
    364 	nflag = (mdpgno >> AARCH64_MMAP_FLAG_SHIFT) & AARCH64_MMAP_FLAG_MASK;
    365 	switch (nflag) {
    366 	case AARCH64_MMAP_DEVICE:
    367 		pflag = PMAP_DEV;
    368 		break;
    369 	case AARCH64_MMAP_WRITECOMBINE:
    370 		pflag = PMAP_WRITE_COMBINE;
    371 		break;
    372 	case AARCH64_MMAP_WRITEBACK:
    373 		pflag = PMAP_WRITE_BACK;
    374 		break;
    375 	case AARCH64_MMAP_NOCACHE:
    376 	default:
    377 		pflag = PMAP_NOCACHE;
    378 		break;
    379 	}
    380 	return pflag;
    381 }
    382 
    383 #define pmap_phys_address(pa)		aarch64_ptob((pa))
    384 #define pmap_mmap_flags(ppn)		aarch64_mmap_flags((ppn))
    385 
    386 #define pmap_update(pmap)		((void)0)
    387 #define pmap_copy(dp,sp,d,l,s)		((void)0)
    388 #define pmap_wired_count(pmap)		((pmap)->pm_stats.wired_count)
    389 #define pmap_resident_count(pmap)	((pmap)->pm_stats.resident_count)
    390 
    391 struct pmap *
    392 	pmap_efirt(void);
    393 void	pmap_activate_efirt(void);
    394 void	pmap_deactivate_efirt(void);
    395 
    396 void	pmap_procwr(struct proc *, vaddr_t, int);
    397 bool	pmap_extract_coherency(pmap_t, vaddr_t, paddr_t *, bool *);
    398 void	pmap_icache_sync_range(pmap_t, vaddr_t, vaddr_t);
    399 
    400 void	pmap_pv_init(void);
    401 void	pmap_pv_track(paddr_t, psize_t);
    402 void	pmap_pv_untrack(paddr_t, psize_t);
    403 void	pmap_pv_protect(paddr_t, vm_prot_t);
    404 
    405 #define	PMAP_MAPSIZE1	L2_SIZE
    406 
    407 #endif /* _KERNEL */
    408 
    409 #elif defined(__arm__)
    410 
    411 #include <arm/pmap.h>
    412 
    413 #endif /* __arm__/__aarch64__ */
    414 
    415 #endif /* !_AARCH64_PMAP_ */
    416