pmap.h revision 1.62 
1/*	$NetBSD: pmap.h,v 1.62 2019/08/07 06:23:48 maxv Exp $	*/
2
3/*
4 * Copyright (c) 1997 Charles D. Cranor and Washington University.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 *    notice, this list of conditions and the following disclaimer in the
14 *    documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28/*
29 * Copyright (c) 2001 Wasabi Systems, Inc.
30 * All rights reserved.
31 *
32 * Written by Frank van der Linden for Wasabi Systems, Inc.
33 *
34 * Redistribution and use in source and binary forms, with or without
35 * modification, are permitted provided that the following conditions
36 * are met:
37 * 1. Redistributions of source code must retain the above copyright
38 *    notice, this list of conditions and the following disclaimer.
39 * 2. Redistributions in binary form must reproduce the above copyright
40 *    notice, this list of conditions and the following disclaimer in the
41 *    documentation and/or other materials provided with the distribution.
42 * 3. All advertising materials mentioning features or use of this software
43 *    must display the following acknowledgement:
44 *      This product includes software developed for the NetBSD Project by
45 *      Wasabi Systems, Inc.
46 * 4. The name of Wasabi Systems, Inc. may not be used to endorse
47 *    or promote products derived from this software without specific prior
48 *    written permission.
49 *
50 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
51 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
52 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
53 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL WASABI SYSTEMS, INC
54 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
55 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
56 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
57 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
58 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
59 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
60 * POSSIBILITY OF SUCH DAMAGE.
61 */
62
63#ifndef	_AMD64_PMAP_H_
64#define	_AMD64_PMAP_H_
65
66#ifdef __x86_64__
67
68#if defined(_KERNEL_OPT)
69#include "opt_xen.h"
70#include "opt_kasan.h"
71#endif
72
73#include <sys/atomic.h>
74
75#include <machine/pte.h>
76#include <machine/segments.h>
77#ifdef _KERNEL
78#include <machine/cpufunc.h>
79#endif
80
81#include <uvm/uvm_object.h>
82#ifdef XENPV
83#include <xen/xenfunc.h>
84#include <xen/xenpmap.h>
85#endif
86
87/*
88 * Mask to get rid of the sign-extended part of addresses.
89 */
90#define VA_SIGN_MASK		0xffff000000000000
91#define VA_SIGN_NEG(va)		((va) | VA_SIGN_MASK)
92/* XXXfvdl this one's not right. */
93#define VA_SIGN_POS(va)		((va) & ~VA_SIGN_MASK)
94
95#ifdef KASAN
96#define L4_SLOT_KASAN		256
97#define NL4_SLOT_KASAN		32
98#endif
99
100#define NL4_SLOT_DIRECT		32
101
102#ifndef XENPV
103#define L4_SLOT_PTE		slotspace.area[SLAREA_PTE].sslot
104#else
105#define L4_SLOT_PTE		509
106#endif
107#define L4_SLOT_KERN		slotspace.area[SLAREA_MAIN].sslot
108#define L4_SLOT_KERNBASE	511 /* pl4_i(KERNBASE) */
109
110#define PDIR_SLOT_USERLIM	255
111#define PDIR_SLOT_KERN	L4_SLOT_KERN
112#define PDIR_SLOT_PTE	L4_SLOT_PTE
113
114/*
115 * The following defines give the virtual addresses of various MMU
116 * data structures:
117 * PTE_BASE: the base VA of the linear PTE mappings
118 * PDP_BASE: the base VA of the recursive mapping of the PTD
119 */
120
121#ifndef XENPV
122extern pt_entry_t *pte_base;
123#define PTE_BASE	pte_base
124#else
125#define PTE_BASE	((pt_entry_t *)VA_SIGN_NEG((L4_SLOT_PTE * NBPD_L4)))
126#endif
127
128#define L1_BASE	PTE_BASE
129#define L2_BASE	((pd_entry_t *)((char *)L1_BASE + L4_SLOT_PTE * NBPD_L3))
130#define L3_BASE	((pd_entry_t *)((char *)L2_BASE + L4_SLOT_PTE * NBPD_L2))
131#define L4_BASE	((pd_entry_t *)((char *)L3_BASE + L4_SLOT_PTE * NBPD_L1))
132
133#define PDP_BASE	L4_BASE
134
135#define NKL4_MAX_ENTRIES	(unsigned long)64
136#define NKL3_MAX_ENTRIES	(unsigned long)(NKL4_MAX_ENTRIES * 512)
137#define NKL2_MAX_ENTRIES	(unsigned long)(NKL3_MAX_ENTRIES * 512)
138#define NKL1_MAX_ENTRIES	(unsigned long)(NKL2_MAX_ENTRIES * 512)
139
140#define NKL4_KIMG_ENTRIES	1
141#define NKL3_KIMG_ENTRIES	1
142#define NKL2_KIMG_ENTRIES	48
143
144/*
145 * Since kva space is below the kernel in its entirety, we start off
146 * with zero entries on each level.
147 */
148#define NKL4_START_ENTRIES	0
149#define NKL3_START_ENTRIES	0
150#define NKL2_START_ENTRIES	0
151#define NKL1_START_ENTRIES	0
152
153#define PTP_MASK_INITIALIZER	{ L1_MASK, L2_MASK, L3_MASK, L4_MASK }
154#define PTP_FRAME_INITIALIZER	{ L1_FRAME, L2_FRAME, L3_FRAME, L4_FRAME }
155#define PTP_SHIFT_INITIALIZER	{ L1_SHIFT, L2_SHIFT, L3_SHIFT, L4_SHIFT }
156#define NKPTP_INITIALIZER	{ NKL1_START_ENTRIES, NKL2_START_ENTRIES, \
157				  NKL3_START_ENTRIES, NKL4_START_ENTRIES }
158#define NKPTPMAX_INITIALIZER	{ NKL1_MAX_ENTRIES, NKL2_MAX_ENTRIES, \
159				  NKL3_MAX_ENTRIES, NKL4_MAX_ENTRIES }
160#define NBPD_INITIALIZER	{ NBPD_L1, NBPD_L2, NBPD_L3, NBPD_L4 }
161#define PDES_INITIALIZER	{ L2_BASE, L3_BASE, L4_BASE }
162
163#define PTP_LEVELS	4
164
165/*
166 * PTE_AVL usage: we make use of the ignored bits of the PTE
167 */
168#define PTE_WIRED	PTE_AVL1	/* Wired Mapping */
169#define PTE_PVLIST	PTE_AVL2	/* Mapping has entry on pvlist */
170#define PTE_X		0		/* Dummy */
171
172/* XXX To be deleted. */
173#define PG_W		PTE_WIRED
174#define PG_PVLIST	PTE_PVLIST
175#define PG_X		PTE_X
176
177void svs_pmap_sync(struct pmap *, int);
178void svs_ldt_sync(struct pmap *);
179void svs_lwp_switch(struct lwp *, struct lwp *);
180void svs_pdir_switch(struct pmap *);
181void svs_init(void);
182extern bool svs_enabled;
183extern bool svs_pcid;
184
185#include <x86/pmap.h>
186
187#ifndef XENPV
188#define pmap_pa2pte(a)			(a)
189#define pmap_pte2pa(a)			((a) & PTE_FRAME)
190#define pmap_pte_set(p, n)		do { *(p) = (n); } while (0)
191#define pmap_pte_cas(p, o, n)		atomic_cas_64((p), (o), (n))
192#define pmap_pte_testset(p, n)		\
193    atomic_swap_ulong((volatile unsigned long *)p, n)
194#define pmap_pte_setbits(p, b)		\
195    atomic_or_ulong((volatile unsigned long *)p, b)
196#define pmap_pte_clearbits(p, b)	\
197    atomic_and_ulong((volatile unsigned long *)p, ~(b))
198#define pmap_pte_flush()		/* nothing */
199#else
200extern kmutex_t pte_lock;
201
202static __inline pt_entry_t
203pmap_pa2pte(paddr_t pa)
204{
205	return (pt_entry_t)xpmap_ptom_masked(pa);
206}
207
208static __inline paddr_t
209pmap_pte2pa(pt_entry_t pte)
210{
211	return xpmap_mtop_masked(pte & PTE_FRAME);
212}
213
214static __inline void
215pmap_pte_set(pt_entry_t *pte, pt_entry_t npte)
216{
217	int s = splvm();
218	xpq_queue_pte_update(xpmap_ptetomach(pte), npte);
219	splx(s);
220}
221
222static __inline pt_entry_t
223pmap_pte_cas(volatile pt_entry_t *ptep, pt_entry_t o, pt_entry_t n)
224{
225	pt_entry_t opte;
226
227	mutex_enter(&pte_lock);
228	opte = *ptep;
229	if (opte == o) {
230		xpq_queue_pte_update(xpmap_ptetomach(__UNVOLATILE(ptep)), n);
231		xpq_flush_queue();
232	}
233
234	mutex_exit(&pte_lock);
235	return opte;
236}
237
238static __inline pt_entry_t
239pmap_pte_testset(volatile pt_entry_t *pte, pt_entry_t npte)
240{
241	pt_entry_t opte;
242
243	mutex_enter(&pte_lock);
244	opte = *pte;
245	xpq_queue_pte_update(xpmap_ptetomach(__UNVOLATILE(pte)), npte);
246	xpq_flush_queue();
247	mutex_exit(&pte_lock);
248	return opte;
249}
250
251static __inline void
252pmap_pte_setbits(volatile pt_entry_t *pte, pt_entry_t bits)
253{
254	mutex_enter(&pte_lock);
255	xpq_queue_pte_update(xpmap_ptetomach(__UNVOLATILE(pte)), (*pte) | bits);
256	xpq_flush_queue();
257	mutex_exit(&pte_lock);
258}
259
260static __inline void
261pmap_pte_clearbits(volatile pt_entry_t *pte, pt_entry_t bits)
262{
263	mutex_enter(&pte_lock);
264	xpq_queue_pte_update(xpmap_ptetomach(__UNVOLATILE(pte)),
265	    (*pte) & ~bits);
266	xpq_flush_queue();
267	mutex_exit(&pte_lock);
268}
269
270static __inline void
271pmap_pte_flush(void)
272{
273	int s = splvm();
274	xpq_flush_queue();
275	splx(s);
276}
277#endif
278
279#ifdef __HAVE_DIRECT_MAP
280#define PMAP_DIRECT
281
282static __inline int
283pmap_direct_process(paddr_t pa, voff_t pgoff, size_t len,
284    int (*process)(void *, size_t, void *), void *arg)
285{
286	vaddr_t va = PMAP_DIRECT_MAP(pa);
287
288	return process((void *)(va + pgoff), len, arg);
289}
290
291#endif /* __HAVE_DIRECT_MAP */
292
293void pmap_changeprot_local(vaddr_t, vm_prot_t);
294
295#include <x86/pmap_pv.h>
296
297#define	__HAVE_VM_PAGE_MD
298#define	VM_MDPAGE_INIT(pg) \
299	memset(&(pg)->mdpage, 0, sizeof((pg)->mdpage)); \
300	PMAP_PAGE_INIT(&(pg)->mdpage.mp_pp)
301
302struct vm_page_md {
303	struct pmap_page mp_pp;
304};
305
306#else	/*	!__x86_64__	*/
307
308#include <i386/pmap.h>
309
310#endif	/*	__x86_64__	*/
311
312#endif	/* _AMD64_PMAP_H_ */
313