Home | History | Annotate | Line # | Download | only in booke
booke_pmap.c revision 1.18.6.2 
      1 /*	$NetBSD: booke_pmap.c,v 1.18.6.2 2015/12/27 12:09:40 skrll Exp $	*/
      2 /*-
      3  * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
      4  * All rights reserved.
      5  *
      6  * This code is derived from software contributed to The NetBSD Foundation
      7  * by Raytheon BBN Technologies Corp and Defense Advanced Research Projects
      8  * Agency and which was developed by Matt Thomas of 3am Software Foundry.
      9  *
     10  * This material is based upon work supported by the Defense Advanced Research
     11  * Projects Agency and Space and Naval Warfare Systems Center, Pacific, under
     12  * Contract No. N66001-09-C-2073.
     13  * Approved for Public Release, Distribution Unlimited
     14  *
     15  * Redistribution and use in source and binary forms, with or without
     16  * modification, are permitted provided that the following conditions
     17  * are met:
     18  * 1. Redistributions of source code must retain the above copyright
     19  *    notice, this list of conditions and the following disclaimer.
     20  * 2. Redistributions in binary form must reproduce the above copyright
     21  *    notice, this list of conditions and the following disclaimer in the
     22  *    documentation and/or other materials provided with the distribution.
     23  *
     24  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     25  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     26  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     27  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     28  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     29  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     30  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     31  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     32  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     33  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     34  * POSSIBILITY OF SUCH DAMAGE.
     35  */
     36 
     37 #define __PMAP_PRIVATE
     38 
     39 #include <sys/cdefs.h>
     40 
     41 __KERNEL_RCSID(0, "$NetBSD: booke_pmap.c,v 1.18.6.2 2015/12/27 12:09:40 skrll Exp $");
     42 
     43 #include <sys/param.h>
     44 #include <sys/kcore.h>
     45 #include <sys/buf.h>
     46 #include <sys/mutex.h>
     47 
     48 #include <uvm/uvm.h>
     49 
     50 #include <machine/pmap.h>
     51 
     52 #if defined(MULTIPROCESSOR)
     53 kmutex_t pmap_tlb_miss_lock;
     54 #endif
     55 
     56 /*
     57  * Initialize the kernel pmap.
     58  */
     59 #ifdef MULTIPROCESSOR
     60 #define	PMAP_SIZE	offsetof(struct pmap, pm_pai[PMAP_TLB_MAX])
     61 #else
     62 #define	PMAP_SIZE	sizeof(struct pmap)
     63 #endif
     64 
     65 CTASSERT(sizeof(pmap_segtab_t) == NBPG);
     66 
     67 pmap_segtab_t pmap_kernel_segtab;
     68 
     69 void
     70 pmap_procwr(struct proc *p, vaddr_t va, size_t len)
     71 {
     72 	struct pmap * const pmap = p->p_vmspace->vm_map.pmap;
     73 	vsize_t off = va & PAGE_SIZE;
     74 
     75 	kpreempt_disable();
     76 	for (const vaddr_t eva = va + len; va < eva; off = 0) {
     77 		const vaddr_t segeva = min(va + len, va - off + PAGE_SIZE);
     78 		pt_entry_t * const ptep = pmap_pte_lookup(pmap, va);
     79 		if (ptep == NULL) {
     80 			va = segeva;
     81 			continue;
     82 		}
     83 		pt_entry_t pt_entry = *ptep;
     84 		if (!pte_valid_p(pt_entry) || !pte_exec_p(pt_entry)) {
     85 			va = segeva;
     86 			continue;
     87 		}
     88 		kpreempt_enable();
     89 		dcache_wb(pte_to_paddr(pt_entry), segeva - va);
     90 		icache_inv(pte_to_paddr(pt_entry), segeva - va);
     91 		kpreempt_disable();
     92 		va = segeva;
     93 	}
     94 	kpreempt_enable();
     95 }
     96 
     97 void
     98 pmap_md_page_syncicache(struct vm_page *pg, const kcpuset_t *onproc)
     99 {
    100 	/*
    101 	 * If onproc is empty, we could do a
    102 	 * pmap_page_protect(pg, VM_PROT_NONE) and remove all
    103 	 * mappings of the page and clear its execness.  Then
    104 	 * the next time page is faulted, it will get icache
    105 	 * synched.  But this is easier. :)
    106 	 */
    107 	paddr_t pa = VM_PAGE_TO_PHYS(pg);
    108 	dcache_wb_page(pa);
    109 	icache_inv_page(pa);
    110 }
    111 
    112 vaddr_t
    113 pmap_md_direct_map_paddr(paddr_t pa)
    114 {
    115 	return (vaddr_t) pa;
    116 }
    117 
    118 bool
    119 pmap_md_direct_mapped_vaddr_p(vaddr_t va)
    120 {
    121 	return va < VM_MIN_KERNEL_ADDRESS || VM_MAX_KERNEL_ADDRESS <= va;
    122 }
    123 
    124 paddr_t
    125 pmap_md_direct_mapped_vaddr_to_paddr(vaddr_t va)
    126 {
    127 	return (paddr_t) va;
    128 }
    129 
    130 #ifdef PMAP_MINIMALTLB
    131 static pt_entry_t *
    132 kvtopte(const pmap_segtab_t *stp, vaddr_t va)
    133 {
    134 	pt_entry_t * const ptep = stp->seg_tab[va >> SEGSHIFT];
    135 	if (ptep == NULL)
    136 		return NULL;
    137 	return &ptep[(va & SEGOFSET) >> PAGE_SHIFT];
    138 }
    139 
    140 vaddr_t
    141 pmap_kvptefill(vaddr_t sva, vaddr_t eva, pt_entry_t pt_entry)
    142 {
    143 	const pmap_segtab_t * const stp = pmap_kernel()->pm_segtab;
    144 	KASSERT(sva == trunc_page(sva));
    145 	pt_entry_t *ptep = kvtopte(stp, sva);
    146 	for (; sva < eva; sva += NBPG) {
    147 		*ptep++ = pt_entry ? (sva | pt_entry) : 0;
    148 	}
    149 	return sva;
    150 }
    151 #endif
    152 
    153 /*
    154  *	Bootstrap the system enough to run with virtual memory.
    155  *	firstaddr is the first unused kseg0 address (not page aligned).
    156  */
    157 vaddr_t
    158 pmap_bootstrap(vaddr_t startkernel, vaddr_t endkernel,
    159 	phys_ram_seg_t *avail, size_t cnt)
    160 {
    161 	pmap_segtab_t * const stp = &pmap_kernel_segtab;
    162 
    163 	/*
    164 	 * Initialize the kernel segment table.
    165 	 */
    166 	pmap_kernel()->pm_segtab = stp;
    167 	curcpu()->ci_pmap_kern_segtab = stp;
    168 
    169 	KASSERT(endkernel == trunc_page(endkernel));
    170 
    171 	/* init the lock */
    172 	pmap_tlb_info_init(&pmap_tlb0_info);
    173 
    174 #if defined(MULTIPROCESSOR)
    175 	mutex_init(&pmap_tlb_miss_lock, MUTEX_SPIN, IPL_HIGH);
    176 #endif
    177 
    178 	/*
    179 	 * Compute the number of pages kmem_arena will have.
    180 	 */
    181 	kmeminit_nkmempages();
    182 
    183 	/*
    184 	 * Figure out how many PTE's are necessary to map the kernel.
    185 	 * We also reserve space for kmem_alloc_pageable() for vm_fork().
    186 	 */
    187 
    188 	/* Get size of buffer cache and set an upper limit */
    189 	buf_setvalimit((VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS) / 8);
    190 	vsize_t bufsz = buf_memcalc();
    191 	buf_setvalimit(bufsz);
    192 
    193 	vsize_t kv_nsegtabs = pmap_round_seg(VM_PHYS_SIZE
    194 	    + (ubc_nwins << ubc_winshift)
    195 	    + bufsz
    196 	    + 16 * NCARGS
    197 	    + pager_map_size
    198 	    + maxproc * USPACE
    199 	    + NBPG * nkmempages) >> SEGSHIFT;
    200 
    201 	/*
    202 	 * Initialize `FYI' variables.	Note we're relying on
    203 	 * the fact that BSEARCH sorts the vm_physmem[] array
    204 	 * for us.  Must do this before uvm_pageboot_alloc()
    205 	 * can be called.
    206 	 */
    207 	pmap_limits.avail_start = vm_physmem[0].start << PGSHIFT;
    208 	pmap_limits.avail_end = vm_physmem[vm_nphysseg - 1].end << PGSHIFT;
    209 	const size_t max_nsegtabs =
    210 	    (pmap_round_seg(VM_MAX_KERNEL_ADDRESS)
    211 		- pmap_trunc_seg(VM_MIN_KERNEL_ADDRESS)) / NBSEG;
    212 	if (kv_nsegtabs >= max_nsegtabs) {
    213 		pmap_limits.virtual_end = VM_MAX_KERNEL_ADDRESS;
    214 		kv_nsegtabs = max_nsegtabs;
    215 	} else {
    216 		pmap_limits.virtual_end = VM_MIN_KERNEL_ADDRESS
    217 		    + kv_nsegtabs * NBSEG;
    218 	}
    219 
    220 	/*
    221 	 * Now actually allocate the kernel PTE array (must be done
    222 	 * after virtual_end is initialized).
    223 	 */
    224 	const vaddr_t kv_segtabs = avail[0].start;
    225 	KASSERT(kv_segtabs == endkernel);
    226 	KASSERT(avail[0].size >= NBPG * kv_nsegtabs);
    227 	printf(" kv_nsegtabs=%#"PRIxVSIZE, kv_nsegtabs);
    228 	printf(" kv_segtabs=%#"PRIxVADDR, kv_segtabs);
    229 	avail[0].start += NBPG * kv_nsegtabs;
    230 	avail[0].size -= NBPG * kv_nsegtabs;
    231 	endkernel += NBPG * kv_nsegtabs;
    232 
    233 	/*
    234 	 * Initialize the kernel's two-level page level.  This only wastes
    235 	 * an extra page for the segment table and allows the user/kernel
    236 	 * access to be common.
    237 	 */
    238 	pt_entry_t **ptp = &stp->seg_tab[VM_MIN_KERNEL_ADDRESS >> SEGSHIFT];
    239 	pt_entry_t *ptep = (void *)kv_segtabs;
    240 	memset(ptep, 0, NBPG * kv_nsegtabs);
    241 	for (size_t i = 0; i < kv_nsegtabs; i++, ptep += NPTEPG) {
    242 		*ptp++ = ptep;
    243 	}
    244 
    245 #if PMAP_MINIMALTLB
    246 	const vsize_t dm_nsegtabs = (physmem + NPTEPG - 1) / NPTEPG;
    247 	const vaddr_t dm_segtabs = avail[0].start;
    248 	printf(" dm_nsegtabs=%#"PRIxVSIZE, dm_nsegtabs);
    249 	printf(" dm_segtabs=%#"PRIxVADDR, dm_segtabs);
    250 	KASSERT(dm_segtabs == endkernel);
    251 	KASSERT(avail[0].size >= NBPG * dm_nsegtabs);
    252 	avail[0].start += NBPG * dm_nsegtabs;
    253 	avail[0].size -= NBPG * dm_nsegtabs;
    254 	endkernel += NBPG * dm_nsegtabs;
    255 
    256 	ptp = stp->seg_tab;
    257 	ptep = (void *)dm_segtabs;
    258 	memset(ptep, 0, NBPG * dm_nsegtabs);
    259 	for (size_t i = 0; i < dm_nsegtabs; i++, ptp++, ptep += NPTEPG) {
    260 		*ptp = ptep;
    261 	}
    262 
    263 	/*
    264 	 */
    265 	extern uint32_t _fdata[], _etext[];
    266 	vaddr_t va;
    267 
    268 	/* Now make everything before the kernel inaccessible. */
    269 	va = pmap_kvptefill(NBPG, startkernel, 0);
    270 
    271 	/* Kernel text is readonly & executable */
    272 	va = pmap_kvptefill(va, round_page((vaddr_t)_etext),
    273 	    PTE_M | PTE_xR | PTE_xX);
    274 
    275 	/* Kernel .rdata is readonly */
    276 	va = pmap_kvptefill(va, trunc_page((vaddr_t)_fdata), PTE_M | PTE_xR);
    277 
    278 	/* Kernel .data/.bss + page tables are read-write */
    279 	va = pmap_kvptefill(va, round_page(endkernel), PTE_M | PTE_xR | PTE_xW);
    280 
    281 	/* message buffer page table pages are read-write */
    282 	(void) pmap_kvptefill(msgbuf_paddr, msgbuf_paddr+round_page(MSGBUFSIZE),
    283 	    PTE_M | PTE_xR | PTE_xW);
    284 #endif
    285 
    286 	for (size_t i = 0; i < cnt; i++) {
    287 		printf(" uvm_page_physload(%#lx,%#lx,%#lx,%#lx,%d)",
    288 		    atop(avail[i].start),
    289 		    atop(avail[i].start + avail[i].size) - 1,
    290 		    atop(avail[i].start),
    291 		    atop(avail[i].start + avail[i].size) - 1,
    292 		    VM_FREELIST_DEFAULT);
    293 		uvm_page_physload(
    294 		    atop(avail[i].start),
    295 		    atop(avail[i].start + avail[i].size) - 1,
    296 		    atop(avail[i].start),
    297 		    atop(avail[i].start + avail[i].size) - 1,
    298 		    VM_FREELIST_DEFAULT);
    299 	}
    300 
    301 	pmap_pvlist_lock_init(curcpu()->ci_ci.dcache_line_size);
    302 
    303 	/*
    304 	 * Initialize the pools.
    305 	 */
    306 	pool_init(&pmap_pmap_pool, PMAP_SIZE, 0, 0, 0, "pmappl",
    307 	    &pool_allocator_nointr, IPL_NONE);
    308 	pool_init(&pmap_pv_pool, sizeof(struct pv_entry), 0, 0, 0, "pvpl",
    309 	    &pmap_pv_page_allocator, IPL_NONE);
    310 
    311 	tlb_set_asid(0);
    312 
    313 	return endkernel;
    314 }
    315 
    316 struct vm_page *
    317 pmap_md_alloc_poolpage(int flags)
    318 {
    319 	/*
    320 	 * Any managed page works for us.
    321 	 */
    322 	return uvm_pagealloc(NULL, 0, NULL, flags);
    323 }
    324 
    325 vaddr_t
    326 pmap_md_map_poolpage(paddr_t pa, vsize_t size)
    327 {
    328 	const vaddr_t sva = (vaddr_t) pa;
    329 #ifdef PMAP_MINIMALTLB
    330 	const vaddr_t eva = sva + size;
    331 	pmap_kvptefill(sva, eva, PTE_M | PTE_xR | PTE_xW);
    332 #endif
    333 	return sva;
    334 }
    335 
    336 void
    337 pmap_md_unmap_poolpage(vaddr_t va, vsize_t size)
    338 {
    339 #ifdef PMAP_MINIMALTLB
    340 	struct pmap * const pm = pmap_kernel();
    341 	const vaddr_t eva = va + size;
    342 	pmap_kvptefill(va, eva, 0);
    343 	for (;va < eva; va += NBPG) {
    344 		pmap_tlb_invalidate_addr(pm, va);
    345 	}
    346 	pmap_update(pm);
    347 #endif
    348 }
    349 
    350 void
    351 pmap_zero_page(paddr_t pa)
    352 {
    353 	vaddr_t va = pmap_md_map_poolpage(pa, NBPG);
    354 	dcache_zero_page(va);
    355 
    356 	KASSERT(!VM_PAGEMD_EXECPAGE_P(VM_PAGE_TO_MD(PHYS_TO_VM_PAGE(va))));
    357 	pmap_md_unmap_poolpage(va, NBPG);
    358 }
    359 
    360 void
    361 pmap_copy_page(paddr_t src, paddr_t dst)
    362 {
    363 	const size_t line_size = curcpu()->ci_ci.dcache_line_size;
    364 	vaddr_t src_va = pmap_md_map_poolpage(src, NBPG);
    365 	vaddr_t dst_va = pmap_md_map_poolpage(dst, NBPG);
    366 	const vaddr_t end = src_va + PAGE_SIZE;
    367 
    368 	while (src_va < end) {
    369 		__asm __volatile(
    370 			"dcbt	%2,%0"	"\n\t"	/* touch next src cacheline */
    371 			"dcba	0,%1"	"\n\t" 	/* don't fetch dst cacheline */
    372 		    :: "b"(src_va), "b"(dst_va), "b"(line_size));
    373 		for (u_int i = 0;
    374 		     i < line_size;
    375 		     src_va += 32, dst_va += 32, i += 32) {
    376 			register_t tmp;
    377 			__asm __volatile(
    378 				"mr	%[tmp],31"	"\n\t"
    379 				"lmw	24,0(%[src])"	"\n\t"
    380 				"stmw	24,0(%[dst])"	"\n\t"
    381 				"mr	31,%[tmp]"	"\n\t"
    382 			    : [tmp] "=&r"(tmp)
    383 			    : [src] "b"(src_va), [dst] "b"(dst_va)
    384 			    : "r24", "r25", "r26", "r27",
    385 			      "r28", "r29", "r30", "memory");
    386 		}
    387 	}
    388 	pmap_md_unmap_poolpage(src_va, NBPG);
    389 	pmap_md_unmap_poolpage(dst_va, NBPG);
    390 
    391 	KASSERT(!VM_PAGEMD_EXECPAGE_P(VM_PAGE_TO_MD(PHYS_TO_VM_PAGE(dst))));
    392 }
    393 
    394 void
    395 pmap_md_init(void)
    396 {
    397 
    398 	/* nothing for now */
    399 }
    400 
    401 bool
    402 pmap_md_io_vaddr_p(vaddr_t va)
    403 {
    404 	return va >= pmap_limits.avail_end
    405 	    && !(VM_MIN_KERNEL_ADDRESS <= va && va < VM_MAX_KERNEL_ADDRESS);
    406 }
    407 
    408 bool
    409 pmap_md_tlb_check_entry(void *ctx, vaddr_t va, tlb_asid_t asid, pt_entry_t pte)
    410 {
    411 	pmap_t pm = ctx;
    412         struct pmap_asid_info * const pai = PMAP_PAI(pm, curcpu()->ci_tlb_info);
    413 
    414 	if (asid != pai->pai_asid)
    415 		return true;
    416 
    417 	const pt_entry_t * const ptep = pmap_pte_lookup(pm, va);
    418 	KASSERT(ptep != NULL);
    419 	pt_entry_t xpte = *ptep;
    420 	xpte &= ~((xpte & (PTE_UNSYNCED|PTE_UNMODIFIED)) << 1);
    421 	xpte ^= xpte & (PTE_UNSYNCED|PTE_UNMODIFIED|PTE_WIRED);
    422 
    423 	KASSERTMSG(pte == xpte,
    424 	    "pm=%p va=%#"PRIxVADDR" asid=%u: TLB pte (%#x) != real pte (%#x/%#x)",
    425 	    pm, va, asid, pte, xpte, *ptep);
    426 
    427 	return true;
    428 }
    429 
    430 #ifdef MULTIPROCESSOR
    431 void
    432 pmap_md_tlb_info_attach(struct pmap_tlb_info *ti, struct cpu_info *ci)
    433 {
    434 	/* nothing */
    435 }
    436 
    437 void
    438 pmap_md_tlb_miss_lock_enter(void)
    439 {
    440 
    441 	mutex_spin_enter(&pmap_tlb_miss_lock);
    442 }
    443 
    444 void
    445 pmap_md_tlb_miss_lock_exit(void)
    446 {
    447 
    448 	mutex_spin_exit(&pmap_tlb_miss_lock);
    449 }
    450 #endif /* MULTIPROCESSOR */
    451