Home | History | Annotate | Line # | Download | only in ibm4xx
pmap.c revision 1.93
      1 /*	$NetBSD: pmap.c,v 1.93 2020/09/10 03:32:46 rin Exp $	*/
      2 
      3 /*
      4  * Copyright 2001 Wasabi Systems, Inc.
      5  * All rights reserved.
      6  *
      7  * Written by Eduardo Horvath and Simon Burge for Wasabi Systems, Inc.
      8  *
      9  * Redistribution and use in source and binary forms, with or without
     10  * modification, are permitted provided that the following conditions
     11  * are met:
     12  * 1. Redistributions of source code must retain the above copyright
     13  *    notice, this list of conditions and the following disclaimer.
     14  * 2. Redistributions in binary form must reproduce the above copyright
     15  *    notice, this list of conditions and the following disclaimer in the
     16  *    documentation and/or other materials provided with the distribution.
     17  * 3. All advertising materials mentioning features or use of this software
     18  *    must display the following acknowledgement:
     19  *      This product includes software developed for the NetBSD Project by
     20  *      Wasabi Systems, Inc.
     21  * 4. The name of Wasabi Systems, Inc. may not be used to endorse
     22  *    or promote products derived from this software without specific prior
     23  *    written permission.
     24  *
     25  * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
     26  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     27  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     28  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL WASABI SYSTEMS, INC
     29  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     30  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     31  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     32  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     33  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     34  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     35  * POSSIBILITY OF SUCH DAMAGE.
     36  */
     37 
     38 /*
     39  * Copyright (C) 1995, 1996 Wolfgang Solfrank.
     40  * Copyright (C) 1995, 1996 TooLs GmbH.
     41  * All rights reserved.
     42  *
     43  * Redistribution and use in source and binary forms, with or without
     44  * modification, are permitted provided that the following conditions
     45  * are met:
     46  * 1. Redistributions of source code must retain the above copyright
     47  *    notice, this list of conditions and the following disclaimer.
     48  * 2. Redistributions in binary form must reproduce the above copyright
     49  *    notice, this list of conditions and the following disclaimer in the
     50  *    documentation and/or other materials provided with the distribution.
     51  * 3. All advertising materials mentioning features or use of this software
     52  *    must display the following acknowledgement:
     53  *	This product includes software developed by TooLs GmbH.
     54  * 4. The name of TooLs GmbH may not be used to endorse or promote products
     55  *    derived from this software without specific prior written permission.
     56  *
     57  * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
     58  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     59  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     60  * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
     61  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
     62  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
     63  * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
     64  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
     65  * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
     66  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     67  */
     68 
     69 #include <sys/cdefs.h>
     70 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.93 2020/09/10 03:32:46 rin Exp $");
     71 
     72 #ifdef _KERNEL_OPT
     73 #include "opt_ddb.h"
     74 #include "opt_pmap.h"
     75 #endif
     76 
     77 #include <sys/param.h>
     78 #include <sys/cpu.h>
     79 #include <sys/device.h>
     80 #include <sys/kmem.h>
     81 #include <sys/pool.h>
     82 #include <sys/proc.h>
     83 #include <sys/queue.h>
     84 #include <sys/systm.h>
     85 
     86 #include <uvm/uvm.h>
     87 
     88 #include <machine/powerpc.h>
     89 #include <machine/tlb.h>
     90 
     91 #include <powerpc/pcb.h>
     92 
     93 #include <powerpc/spr.h>
     94 #include <powerpc/ibm4xx/spr.h>
     95 
     96 #include <powerpc/ibm4xx/cpu.h>
     97 
     98 /*
     99  * kernmap is an array of PTEs large enough to map in
    100  * 4GB.  At 16KB/page it is 256K entries or 2MB.
    101  */
    102 #define KERNMAP_SIZE	((0xffffffffU/PAGE_SIZE)+1)
    103 void *kernmap;
    104 
    105 #define MINCTX		2
    106 #define NUMCTX		256
    107 
    108 volatile struct pmap *ctxbusy[NUMCTX];
    109 
    110 #define TLBF_USED	0x1
    111 #define	TLBF_REF	0x2
    112 #define	TLBF_LOCKED	0x4
    113 #define	TLB_LOCKED(i)	(tlb_info[(i)].ti_flags & TLBF_LOCKED)
    114 
    115 typedef struct tlb_info_s {
    116 	char	ti_flags;
    117 	char	ti_ctx;		/* TLB_PID assiciated with the entry */
    118 	u_int	ti_va;
    119 } tlb_info_t;
    120 
    121 volatile tlb_info_t tlb_info[NTLB];
    122 /* We'll use a modified FIFO replacement policy cause it's cheap */
    123 volatile int tlbnext;
    124 
    125 static int tlb_nreserved = 0;
    126 static int pmap_bootstrap_done = 0;
    127 
    128 /* Event counters */
    129 struct evcnt tlbmiss_ev = EVCNT_INITIALIZER(EVCNT_TYPE_TRAP,
    130 	NULL, "cpu", "tlbmiss");
    131 struct evcnt tlbflush_ev = EVCNT_INITIALIZER(EVCNT_TYPE_TRAP,
    132 	NULL, "cpu", "tlbflush");
    133 struct evcnt tlbenter_ev = EVCNT_INITIALIZER(EVCNT_TYPE_TRAP,
    134 	NULL, "cpu", "tlbenter");
    135 EVCNT_ATTACH_STATIC(tlbmiss_ev);
    136 EVCNT_ATTACH_STATIC(tlbflush_ev);
    137 EVCNT_ATTACH_STATIC(tlbenter_ev);
    138 
    139 struct pmap kernel_pmap_;
    140 struct pmap *const kernel_pmap_ptr = &kernel_pmap_;
    141 
    142 static int npgs;
    143 static u_int nextavail;
    144 #ifndef MSGBUFADDR
    145 extern paddr_t msgbuf_paddr;
    146 #endif
    147 
    148 static struct mem_region *mem, *avail;
    149 
    150 /*
    151  * This is a cache of referenced/modified bits.
    152  * Bits herein are shifted by ATTRSHFT.
    153  */
    154 static char *pmap_attrib;
    155 
    156 #define PV_WIRED	0x1
    157 #define PV_WIRE(pv)	((pv)->pv_va |= PV_WIRED)
    158 #define PV_UNWIRE(pv)	((pv)->pv_va &= ~PV_WIRED)
    159 #define PV_ISWIRED(pv)	((pv)->pv_va & PV_WIRED)
    160 #define PV_VA(pv)	((pv)->pv_va & ~PV_WIRED)
    161 #define PV_CMPVA(va,pv)	(!(PV_VA(pv) ^ (va)))
    162 
    163 struct pv_entry {
    164 	struct pv_entry *pv_next;	/* Linked list of mappings */
    165 	struct pmap *pv_pm;
    166 	vaddr_t pv_va;			/* virtual address of mapping */
    167 };
    168 
    169 /* Each index corresponds to TLB_SIZE_* value. */
    170 static size_t tlbsize[] = {
    171 	1024, 		/* TLB_SIZE_1K */
    172 	4096, 		/* TLB_SIZE_4K */
    173 	16384, 		/* TLB_SIZE_16K */
    174 	65536, 		/* TLB_SIZE_64K */
    175 	262144, 	/* TLB_SIZE_256K */
    176 	1048576, 	/* TLB_SIZE_1M */
    177 	4194304, 	/* TLB_SIZE_4M */
    178 	16777216, 	/* TLB_SIZE_16M */
    179 };
    180 
    181 struct pv_entry *pv_table;
    182 static struct pool pv_pool;
    183 
    184 static int pmap_initialized;
    185 
    186 static int ctx_flush(int);
    187 
    188 struct pv_entry *pa_to_pv(paddr_t);
    189 static inline char *pa_to_attr(paddr_t);
    190 
    191 static inline volatile u_int *pte_find(struct pmap *, vaddr_t);
    192 static inline int pte_enter(struct pmap *, vaddr_t, u_int);
    193 
    194 static inline int pmap_enter_pv(struct pmap *, vaddr_t, paddr_t, int);
    195 static void pmap_remove_pv(struct pmap *, vaddr_t, paddr_t);
    196 
    197 static inline void tlb_invalidate_entry(int);
    198 
    199 static int ppc4xx_tlb_size_mask(size_t, int *, int *);
    200 
    201 
    202 struct pv_entry *
    203 pa_to_pv(paddr_t pa)
    204 {
    205 	uvm_physseg_t bank;
    206 	psize_t pg;
    207 
    208 	bank = uvm_physseg_find(atop(pa), &pg);
    209 	if (bank == UVM_PHYSSEG_TYPE_INVALID)
    210 		return NULL;
    211 	return &uvm_physseg_get_pmseg(bank)->pvent[pg];
    212 }
    213 
    214 static inline char *
    215 pa_to_attr(paddr_t pa)
    216 {
    217 	uvm_physseg_t bank;
    218 	psize_t pg;
    219 
    220 	bank = uvm_physseg_find(atop(pa), &pg);
    221 	if (bank == UVM_PHYSSEG_TYPE_INVALID)
    222 		return NULL;
    223 	return &uvm_physseg_get_pmseg(bank)->attrs[pg];
    224 }
    225 
    226 /*
    227  * Insert PTE into page table.
    228  */
    229 int
    230 pte_enter(struct pmap *pm, vaddr_t va, u_int pte)
    231 {
    232 	int seg = STIDX(va);
    233 	int ptn = PTIDX(va);
    234 	u_int oldpte;
    235 
    236 	if (!pm->pm_ptbl[seg]) {
    237 		/* Don't allocate a page to clear a non-existent mapping. */
    238 		if (!pte)
    239 			return (0);
    240 		/* Allocate a page XXXX this will sleep! */
    241 		pm->pm_ptbl[seg] =
    242 		    (uint *)uvm_km_alloc(kernel_map, PAGE_SIZE, 0,
    243 		    UVM_KMF_WIRED | UVM_KMF_ZERO);
    244 	}
    245 	oldpte = pm->pm_ptbl[seg][ptn];
    246 	pm->pm_ptbl[seg][ptn] = pte;
    247 
    248 	/* Flush entry. */
    249 	ppc4xx_tlb_flush(va, pm->pm_ctx);
    250 	if (oldpte != pte) {
    251 		if (pte == 0)
    252 			pm->pm_stats.resident_count--;
    253 		else
    254 			pm->pm_stats.resident_count++;
    255 	}
    256 	return (1);
    257 }
    258 
    259 /*
    260  * Get a pointer to a PTE in a page table.
    261  */
    262 volatile u_int *
    263 pte_find(struct pmap *pm, vaddr_t va)
    264 {
    265 	int seg = STIDX(va);
    266 	int ptn = PTIDX(va);
    267 
    268 	if (pm->pm_ptbl[seg])
    269 		return (&pm->pm_ptbl[seg][ptn]);
    270 
    271 	return (NULL);
    272 }
    273 
    274 /*
    275  * This is called during initppc, before the system is really initialized.
    276  */
    277 void
    278 pmap_bootstrap(u_int kernelstart, u_int kernelend)
    279 {
    280 	struct mem_region *mp, *mp1;
    281 	int cnt, i;
    282 	u_int s, e, sz;
    283 
    284 	tlbnext = tlb_nreserved;
    285 
    286 	/*
    287 	 * Allocate the kernel page table at the end of
    288 	 * kernel space so it's in the locked TTE.
    289 	 */
    290 	kernmap = (void *)kernelend;
    291 
    292 	/*
    293 	 * Initialize kernel page table.
    294 	 */
    295 	for (i = 0; i < STSZ; i++) {
    296 		pmap_kernel()->pm_ptbl[i] = 0;
    297 	}
    298 	ctxbusy[0] = ctxbusy[1] = pmap_kernel();
    299 
    300 	/*
    301 	 * Announce page-size to the VM-system
    302 	 */
    303 	uvmexp.pagesize = NBPG;
    304 	uvm_md_init();
    305 
    306 	/*
    307 	 * Get memory.
    308 	 */
    309 	mem_regions(&mem, &avail);
    310 	for (mp = mem; mp->size; mp++) {
    311 		physmem += btoc(mp->size);
    312 		printf("+%lx,",mp->size);
    313 	}
    314 	printf("\n");
    315 	ppc4xx_tlb_init();
    316 	/*
    317 	 * Count the number of available entries.
    318 	 */
    319 	for (cnt = 0, mp = avail; mp->size; mp++)
    320 		cnt++;
    321 
    322 	/*
    323 	 * Page align all regions.
    324 	 * Non-page aligned memory isn't very interesting to us.
    325 	 * Also, sort the entries for ascending addresses.
    326 	 */
    327 	kernelstart &= ~PGOFSET;
    328 	kernelend = (kernelend + PGOFSET) & ~PGOFSET;
    329 	for (mp = avail; mp->size; mp++) {
    330 		s = mp->start;
    331 		e = mp->start + mp->size;
    332 		printf("%08x-%08x -> ",s,e);
    333 		/*
    334 		 * Check whether this region holds all of the kernel.
    335 		 */
    336 		if (s < kernelstart && e > kernelend) {
    337 			avail[cnt].start = kernelend;
    338 			avail[cnt++].size = e - kernelend;
    339 			e = kernelstart;
    340 		}
    341 		/*
    342 		 * Look whether this regions starts within the kernel.
    343 		 */
    344 		if (s >= kernelstart && s < kernelend) {
    345 			if (e <= kernelend)
    346 				goto empty;
    347 			s = kernelend;
    348 		}
    349 		/*
    350 		 * Now look whether this region ends within the kernel.
    351 		 */
    352 		if (e > kernelstart && e <= kernelend) {
    353 			if (s >= kernelstart)
    354 				goto empty;
    355 			e = kernelstart;
    356 		}
    357 		/*
    358 		 * Now page align the start and size of the region.
    359 		 */
    360 		s = round_page(s);
    361 		e = trunc_page(e);
    362 		if (e < s)
    363 			e = s;
    364 		sz = e - s;
    365 		printf("%08x-%08x = %x\n",s,e,sz);
    366 		/*
    367 		 * Check whether some memory is left here.
    368 		 */
    369 		if (sz == 0) {
    370 		empty:
    371 			memmove(mp, mp + 1,
    372 				(cnt - (mp - avail)) * sizeof *mp);
    373 			cnt--;
    374 			mp--;
    375 			continue;
    376 		}
    377 		/*
    378 		 * Do an insertion sort.
    379 		 */
    380 		npgs += btoc(sz);
    381 		for (mp1 = avail; mp1 < mp; mp1++)
    382 			if (s < mp1->start)
    383 				break;
    384 		if (mp1 < mp) {
    385 			memmove(mp1 + 1, mp1, (char *)mp - (char *)mp1);
    386 			mp1->start = s;
    387 			mp1->size = sz;
    388 		} else {
    389 			mp->start = s;
    390 			mp->size = sz;
    391 		}
    392 	}
    393 
    394 	/*
    395 	 * We cannot do pmap_steal_memory here,
    396 	 * since we don't run with translation enabled yet.
    397 	 */
    398 #ifndef MSGBUFADDR
    399 	/*
    400 	 * allow for msgbuf
    401 	 */
    402 	sz = round_page(MSGBUFSIZE);
    403 	mp = NULL;
    404 	for (mp1 = avail; mp1->size; mp1++)
    405 		if (mp1->size >= sz)
    406 			mp = mp1;
    407 	if (mp == NULL)
    408 		panic("not enough memory?");
    409 
    410 	npgs -= btoc(sz);
    411 	msgbuf_paddr = mp->start + mp->size - sz;
    412 	mp->size -= sz;
    413 	if (mp->size <= 0)
    414 		memmove(mp, mp + 1, (cnt - (mp - avail)) * sizeof *mp);
    415 #endif
    416 
    417 	for (mp = avail; mp->size; mp++)
    418 		uvm_page_physload(atop(mp->start), atop(mp->start + mp->size),
    419 			atop(mp->start), atop(mp->start + mp->size),
    420 			VM_FREELIST_DEFAULT);
    421 
    422 	/*
    423 	 * Initialize kernel pmap and hardware.
    424 	 */
    425 	/* Setup TLB pid allocator so it knows we alreadu using PID 1 */
    426 	pmap_kernel()->pm_ctx = KERNEL_PID;
    427 	nextavail = avail->start;
    428 
    429 	pmap_bootstrap_done = 1;
    430 }
    431 
    432 /*
    433  * Restrict given range to physical memory
    434  *
    435  * (Used by /dev/mem)
    436  */
    437 void
    438 pmap_real_memory(paddr_t *start, psize_t *size)
    439 {
    440 	struct mem_region *mp;
    441 
    442 	for (mp = mem; mp->size; mp++) {
    443 		if (*start + *size > mp->start &&
    444 		    *start < mp->start + mp->size) {
    445 			if (*start < mp->start) {
    446 				*size -= mp->start - *start;
    447 				*start = mp->start;
    448 			}
    449 			if (*start + *size > mp->start + mp->size)
    450 				*size = mp->start + mp->size - *start;
    451 			return;
    452 		}
    453 	}
    454 	*size = 0;
    455 }
    456 
    457 /*
    458  * Initialize anything else for pmap handling.
    459  * Called during vm_init().
    460  */
    461 void
    462 pmap_init(void)
    463 {
    464 	struct pv_entry *pv;
    465 	vsize_t sz;
    466 	vaddr_t addr;
    467 	int i, s;
    468 	int bank;
    469 	char *attr;
    470 
    471 	sz = (vsize_t)((sizeof(struct pv_entry) + 1) * npgs);
    472 	sz = round_page(sz);
    473 	addr = uvm_km_alloc(kernel_map, sz, 0, UVM_KMF_WIRED | UVM_KMF_ZERO);
    474 	s = splvm();
    475 	pv = pv_table = (struct pv_entry *)addr;
    476 	for (i = npgs; --i >= 0;)
    477 		pv++->pv_pm = NULL;
    478 	pmap_attrib = (char *)pv;
    479 	memset(pv, 0, npgs);
    480 
    481 	pv = pv_table;
    482 	attr = pmap_attrib;
    483 	for (bank = uvm_physseg_get_first();
    484 	     uvm_physseg_valid_p(bank);
    485 	     bank = uvm_physseg_get_next(bank)) {
    486 		sz = uvm_physseg_get_end(bank) - uvm_physseg_get_start(bank);
    487 		uvm_physseg_get_pmseg(bank)->pvent = pv;
    488 		uvm_physseg_get_pmseg(bank)->attrs = attr;
    489 		pv += sz;
    490 		attr += sz;
    491 	}
    492 
    493 	pmap_initialized = 1;
    494 	splx(s);
    495 
    496 	/* Setup a pool for additional pvlist structures */
    497 	pool_init(&pv_pool, sizeof(struct pv_entry), 0, 0, 0, "pv_entry", NULL,
    498 	    IPL_VM);
    499 }
    500 
    501 /*
    502  * How much virtual space is available to the kernel?
    503  */
    504 void
    505 pmap_virtual_space(vaddr_t *start, vaddr_t *end)
    506 {
    507 
    508 #if 0
    509 	/*
    510 	 * Reserve one segment for kernel virtual memory
    511 	 */
    512 	*start = (vaddr_t)(KERNEL_SR << ADDR_SR_SHFT);
    513 	*end = *start + SEGMENT_LENGTH;
    514 #else
    515 	*start = (vaddr_t) VM_MIN_KERNEL_ADDRESS;
    516 	*end = (vaddr_t) VM_MAX_KERNEL_ADDRESS;
    517 #endif
    518 }
    519 
    520 #ifdef PMAP_GROWKERNEL
    521 /*
    522  * Preallocate kernel page tables to a specified VA.
    523  * This simply loops through the first TTE for each
    524  * page table from the beginning of the kernel pmap,
    525  * reads the entry, and if the result is
    526  * zero (either invalid entry or no page table) it stores
    527  * a zero there, populating page tables in the process.
    528  * This is not the most efficient technique but i don't
    529  * expect it to be called that often.
    530  */
    531 extern struct vm_page *vm_page_alloc1(void);
    532 extern void vm_page_free1(struct vm_page *);
    533 
    534 vaddr_t kbreak = VM_MIN_KERNEL_ADDRESS;
    535 
    536 vaddr_t
    537 pmap_growkernel(vaddr_t maxkvaddr)
    538 {
    539 	int s;
    540 	int seg;
    541 	paddr_t pg;
    542 	struct pmap *pm = pmap_kernel();
    543 
    544 	s = splvm();
    545 
    546 	/* Align with the start of a page table */
    547 	for (kbreak &= ~(PTMAP-1); kbreak < maxkvaddr;
    548 	     kbreak += PTMAP) {
    549 		seg = STIDX(kbreak);
    550 
    551 		if (pte_find(pm, kbreak))
    552 			continue;
    553 
    554 		if (uvm.page_init_done) {
    555 			pg = (paddr_t)VM_PAGE_TO_PHYS(vm_page_alloc1());
    556 		} else {
    557 			if (!uvm_page_physget(&pg))
    558 				panic("pmap_growkernel: no memory");
    559 		}
    560 		if (!pg)
    561 			panic("pmap_growkernel: no pages");
    562 		pmap_zero_page((paddr_t)pg);
    563 
    564 		/* XXX This is based on all phymem being addressable */
    565 		pm->pm_ptbl[seg] = (u_int *)pg;
    566 	}
    567 	splx(s);
    568 	return (kbreak);
    569 }
    570 
    571 /*
    572  *	vm_page_alloc1:
    573  *
    574  *	Allocate and return a memory cell with no associated object.
    575  */
    576 struct vm_page *
    577 vm_page_alloc1(void)
    578 {
    579 	struct vm_page *pg;
    580 
    581 	pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE);
    582 	if (pg) {
    583 		pg->wire_count = 1;	/* no mappings yet */
    584 		pg->flags &= ~PG_BUSY;	/* never busy */
    585 	}
    586 	return pg;
    587 }
    588 
    589 /*
    590  *	vm_page_free1:
    591  *
    592  *	Returns the given page to the free list,
    593  *	disassociating it with any VM object.
    594  *
    595  *	Object and page must be locked prior to entry.
    596  */
    597 void
    598 vm_page_free1(struct vm_page *pg)
    599 {
    600 #ifdef DIAGNOSTIC
    601 	if (pg->flags != (PG_CLEAN|PG_FAKE)) {
    602 		printf("Freeing invalid page %p\n", pg);
    603 		printf("pa = %llx\n", (unsigned long long)VM_PAGE_TO_PHYS(pg));
    604 #ifdef DDB
    605 		Debugger();
    606 #endif
    607 		return;
    608 	}
    609 #endif
    610 	pg->flags |= PG_BUSY;
    611 	pg->wire_count = 0;
    612 	uvm_pagefree(pg);
    613 }
    614 #endif
    615 
    616 /*
    617  * Create and return a physical map.
    618  */
    619 struct pmap *
    620 pmap_create(void)
    621 {
    622 	struct pmap *pm;
    623 
    624 	pm = kmem_alloc(sizeof(*pm), KM_SLEEP);
    625 	memset(pm, 0, sizeof *pm);
    626 	pm->pm_refs = 1;
    627 	return pm;
    628 }
    629 
    630 /*
    631  * Add a reference to the given pmap.
    632  */
    633 void
    634 pmap_reference(struct pmap *pm)
    635 {
    636 
    637 	pm->pm_refs++;
    638 }
    639 
    640 /*
    641  * Retire the given pmap from service.
    642  * Should only be called if the map contains no valid mappings.
    643  */
    644 void
    645 pmap_destroy(struct pmap *pm)
    646 {
    647 	int i;
    648 
    649 	if (--pm->pm_refs > 0) {
    650 		return;
    651 	}
    652 	KASSERT(pm->pm_stats.resident_count == 0);
    653 	KASSERT(pm->pm_stats.wired_count == 0);
    654 	for (i = 0; i < STSZ; i++)
    655 		if (pm->pm_ptbl[i]) {
    656 			uvm_km_free(kernel_map, (vaddr_t)pm->pm_ptbl[i],
    657 			    PAGE_SIZE, UVM_KMF_WIRED);
    658 			pm->pm_ptbl[i] = NULL;
    659 		}
    660 	if (pm->pm_ctx)
    661 		ctx_free(pm);
    662 	kmem_free(pm, sizeof(*pm));
    663 }
    664 
    665 /*
    666  * Copy the range specified by src_addr/len
    667  * from the source map to the range dst_addr/len
    668  * in the destination map.
    669  *
    670  * This routine is only advisory and need not do anything.
    671  */
    672 void
    673 pmap_copy(struct pmap *dst_pmap, struct pmap *src_pmap, vaddr_t dst_addr,
    674 	  vsize_t len, vaddr_t src_addr)
    675 {
    676 }
    677 
    678 /*
    679  * Require that all active physical maps contain no
    680  * incorrect entries NOW.
    681  */
    682 void
    683 pmap_update(struct pmap *pmap)
    684 {
    685 }
    686 
    687 /*
    688  * Fill the given physical page with zeroes.
    689  */
    690 void
    691 pmap_zero_page(paddr_t pa)
    692 {
    693 
    694 #ifdef PPC_4XX_NOCACHE
    695 	memset((void *)pa, 0, PAGE_SIZE);
    696 #else
    697 	int i;
    698 
    699 	for (i = PAGE_SIZE/CACHELINESIZE; i > 0; i--) {
    700 		__asm volatile ("dcbz 0,%0" :: "r"(pa));
    701 		pa += CACHELINESIZE;
    702 	}
    703 #endif
    704 }
    705 
    706 /*
    707  * Copy the given physical source page to its destination.
    708  */
    709 void
    710 pmap_copy_page(paddr_t src, paddr_t dst)
    711 {
    712 
    713 	memcpy((void *)dst, (void *)src, PAGE_SIZE);
    714 	dcache_wbinv_page(dst);
    715 }
    716 
    717 /*
    718  * This returns != 0 on success.
    719  */
    720 static inline int
    721 pmap_enter_pv(struct pmap *pm, vaddr_t va, paddr_t pa, int flags)
    722 {
    723 	struct pv_entry *pv, *npv = NULL;
    724 	int s;
    725 
    726 	if (!pmap_initialized)
    727 		return 0;
    728 
    729 	s = splvm();
    730 	pv = pa_to_pv(pa);
    731 	if (!pv->pv_pm) {
    732 		/*
    733 		 * No entries yet, use header as the first entry.
    734 		 */
    735 		pv->pv_va = va;
    736 		pv->pv_pm = pm;
    737 		pv->pv_next = NULL;
    738 	} else {
    739 		/*
    740 		 * There is at least one other VA mapping this page.
    741 		 * Place this entry after the header.
    742 		 */
    743 		npv = pool_get(&pv_pool, PR_NOWAIT);
    744 		if (npv == NULL) {
    745 			if ((flags & PMAP_CANFAIL) == 0)
    746 				panic("pmap_enter_pv: failed");
    747 			splx(s);
    748 			return 0;
    749 		}
    750 		npv->pv_va = va;
    751 		npv->pv_pm = pm;
    752 		npv->pv_next = pv->pv_next;
    753 		pv->pv_next = npv;
    754 		pv = npv;
    755 	}
    756 	if (flags & PMAP_WIRED) {
    757 		PV_WIRE(pv);
    758 		pm->pm_stats.wired_count++;
    759 	}
    760 	splx(s);
    761 	return (1);
    762 }
    763 
    764 static void
    765 pmap_remove_pv(struct pmap *pm, vaddr_t va, paddr_t pa)
    766 {
    767 	struct pv_entry *pv, *npv;
    768 
    769 	/*
    770 	 * Remove from the PV table.
    771 	 */
    772 	pv = pa_to_pv(pa);
    773 	if (!pv)
    774 		return;
    775 
    776 	/*
    777 	 * If it is the first entry on the list, it is actually
    778 	 * in the header and we must copy the following entry up
    779 	 * to the header.  Otherwise we must search the list for
    780 	 * the entry.  In either case we free the now unused entry.
    781 	 */
    782 	if (pm == pv->pv_pm && PV_CMPVA(va, pv)) {
    783 		if (PV_ISWIRED(pv)) {
    784 			pm->pm_stats.wired_count--;
    785 		}
    786 		if ((npv = pv->pv_next)) {
    787 			*pv = *npv;
    788 			pool_put(&pv_pool, npv);
    789 		} else
    790 			pv->pv_pm = NULL;
    791 	} else {
    792 		for (; (npv = pv->pv_next) != NULL; pv = npv)
    793 			if (pm == npv->pv_pm && PV_CMPVA(va, npv))
    794 				break;
    795 		if (npv) {
    796 			pv->pv_next = npv->pv_next;
    797 			if (PV_ISWIRED(npv)) {
    798 				pm->pm_stats.wired_count--;
    799 			}
    800 			pool_put(&pv_pool, npv);
    801 		}
    802 	}
    803 }
    804 
    805 /*
    806  * Insert physical page at pa into the given pmap at virtual address va.
    807  */
    808 int
    809 pmap_enter(struct pmap *pm, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
    810 {
    811 	int s;
    812 	u_int tte;
    813 	bool managed;
    814 
    815 	/*
    816 	 * Have to remove any existing mapping first.
    817 	 */
    818 	pmap_remove(pm, va, va + PAGE_SIZE);
    819 
    820 	if (flags & PMAP_WIRED)
    821 		flags |= prot;
    822 
    823 	managed = uvm_pageismanaged(pa);
    824 
    825 	/*
    826 	 * Generate TTE.
    827 	 */
    828 	tte = TTE_PA(pa);
    829 	/* XXXX -- need to support multiple page sizes. */
    830 	tte |= TTE_SZ_16K;
    831 #ifdef	DIAGNOSTIC
    832 	if ((flags & (PMAP_NOCACHE | PME_WRITETHROUG)) ==
    833 		(PMAP_NOCACHE | PME_WRITETHROUG))
    834 		panic("pmap_enter: uncached & writethrough");
    835 #endif
    836 	if (flags & PMAP_NOCACHE)
    837 		/* Must be I/O mapping */
    838 		tte |= TTE_I | TTE_G;
    839 #ifdef PPC_4XX_NOCACHE
    840 	tte |= TTE_I;
    841 #else
    842 	else if (flags & PME_WRITETHROUG)
    843 		/* Uncached and writethrough are not compatible */
    844 		tte |= TTE_W;
    845 #endif
    846 	if (pm == pmap_kernel())
    847 		tte |= TTE_ZONE(ZONE_PRIV);
    848 	else
    849 		tte |= TTE_ZONE(ZONE_USER);
    850 
    851 	if (flags & VM_PROT_WRITE)
    852 		tte |= TTE_WR;
    853 
    854 	if (flags & VM_PROT_EXECUTE)
    855 		tte |= TTE_EX;
    856 
    857 	/*
    858 	 * Now record mapping for later back-translation.
    859 	 */
    860 	if (pmap_initialized && managed) {
    861 		char *attr;
    862 
    863 		if (!pmap_enter_pv(pm, va, pa, flags)) {
    864 			/* Could not enter pv on a managed page */
    865 			return 1;
    866 		}
    867 
    868 		/* Now set attributes. */
    869 		attr = pa_to_attr(pa);
    870 #ifdef DIAGNOSTIC
    871 		if (!attr)
    872 			panic("managed but no attr");
    873 #endif
    874 		if (flags & VM_PROT_ALL)
    875 			*attr |= PMAP_ATTR_REF;
    876 		if (flags & VM_PROT_WRITE)
    877 			*attr |= PMAP_ATTR_CHG;
    878 	}
    879 
    880 	s = splvm();
    881 
    882 	/* Insert page into page table. */
    883 	pte_enter(pm, va, tte);
    884 
    885 	/* If this is a real fault, enter it in the tlb */
    886 	if (tte && ((flags & PMAP_WIRED) == 0)) {
    887 		int s2 = splhigh();
    888 		ppc4xx_tlb_enter(pm->pm_ctx, va, tte);
    889 		splx(s2);
    890 	}
    891 	splx(s);
    892 
    893 	/* Flush the real memory from the instruction cache. */
    894 	if ((prot & VM_PROT_EXECUTE) && (tte & TTE_I) == 0)
    895 		__syncicache((void *)pa, PAGE_SIZE);
    896 
    897 	return 0;
    898 }
    899 
    900 void
    901 pmap_unwire(struct pmap *pm, vaddr_t va)
    902 {
    903 	struct pv_entry *pv;
    904 	paddr_t pa;
    905 	int s;
    906 
    907 	if (!pmap_extract(pm, va, &pa)) {
    908 		return;
    909 	}
    910 
    911 	pv = pa_to_pv(pa);
    912 	if (!pv)
    913 		return;
    914 
    915 	s = splvm();
    916 	while (pv != NULL) {
    917 		if (pm == pv->pv_pm && PV_CMPVA(va, pv)) {
    918 			if (PV_ISWIRED(pv)) {
    919 				PV_UNWIRE(pv);
    920 				pm->pm_stats.wired_count--;
    921 			}
    922 			break;
    923 		}
    924 		pv = pv->pv_next;
    925 	}
    926 	splx(s);
    927 }
    928 
    929 void
    930 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
    931 {
    932 	int s;
    933 	u_int tte;
    934 	struct pmap *pm = pmap_kernel();
    935 
    936 	/*
    937 	 * Generate TTE.
    938 	 *
    939 	 * XXXX
    940 	 *
    941 	 * Since the kernel does not handle execution privileges properly,
    942 	 * we will handle read and execute permissions together.
    943 	 */
    944 	tte = 0;
    945 	if (prot & VM_PROT_ALL) {
    946 
    947 		tte = TTE_PA(pa) | TTE_EX | TTE_ZONE(ZONE_PRIV);
    948 		/* XXXX -- need to support multiple page sizes. */
    949 		tte |= TTE_SZ_16K;
    950 #ifdef DIAGNOSTIC
    951 		if ((flags & (PMAP_NOCACHE | PME_WRITETHROUG)) ==
    952 			(PMAP_NOCACHE | PME_WRITETHROUG))
    953 			panic("pmap_kenter_pa: uncached & writethrough");
    954 #endif
    955 		if (flags & PMAP_NOCACHE)
    956 			/* Must be I/O mapping */
    957 			tte |= TTE_I | TTE_G;
    958 #ifdef PPC_4XX_NOCACHE
    959 		tte |= TTE_I;
    960 #else
    961 		else if (prot & PME_WRITETHROUG)
    962 			/* Uncached and writethrough are not compatible */
    963 			tte |= TTE_W;
    964 #endif
    965 		if (prot & VM_PROT_WRITE)
    966 			tte |= TTE_WR;
    967 	}
    968 
    969 	s = splvm();
    970 
    971 	/* Insert page into page table. */
    972 	pte_enter(pm, va, tte);
    973 	splx(s);
    974 }
    975 
    976 void
    977 pmap_kremove(vaddr_t va, vsize_t len)
    978 {
    979 
    980 	while (len > 0) {
    981 		pte_enter(pmap_kernel(), va, 0);
    982 		va += PAGE_SIZE;
    983 		len -= PAGE_SIZE;
    984 	}
    985 }
    986 
    987 /*
    988  * Remove the given range of mapping entries.
    989  */
    990 void
    991 pmap_remove(struct pmap *pm, vaddr_t va, vaddr_t endva)
    992 {
    993 	int s;
    994 	paddr_t pa;
    995 	volatile u_int *ptp;
    996 
    997 	s = splvm();
    998 	while (va < endva) {
    999 
   1000 		if ((ptp = pte_find(pm, va)) && (pa = *ptp)) {
   1001 			pa = TTE_PA(pa);
   1002 			pmap_remove_pv(pm, va, pa);
   1003 			*ptp = 0;
   1004 			ppc4xx_tlb_flush(va, pm->pm_ctx);
   1005 			pm->pm_stats.resident_count--;
   1006 		}
   1007 		va += PAGE_SIZE;
   1008 	}
   1009 
   1010 	splx(s);
   1011 }
   1012 
   1013 /*
   1014  * Get the physical page address for the given pmap/virtual address.
   1015  */
   1016 bool
   1017 pmap_extract(struct pmap *pm, vaddr_t va, paddr_t *pap)
   1018 {
   1019 	int seg = STIDX(va);
   1020 	int ptn = PTIDX(va);
   1021 	u_int pa = 0;
   1022 	int s;
   1023 
   1024 	s = splvm();
   1025 	if (pm->pm_ptbl[seg] && (pa = pm->pm_ptbl[seg][ptn]) && pap) {
   1026 		*pap = TTE_PA(pa) | (va & PGOFSET);
   1027 	}
   1028 	splx(s);
   1029 	return (pa != 0);
   1030 }
   1031 
   1032 /*
   1033  * Lower the protection on the specified range of this pmap.
   1034  *
   1035  * There are only two cases: either the protection is going to 0,
   1036  * or it is going to read-only.
   1037  */
   1038 void
   1039 pmap_protect(struct pmap *pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot)
   1040 {
   1041 	volatile u_int *ptp;
   1042 	int s, bic;
   1043 
   1044 	if ((prot & VM_PROT_READ) == 0) {
   1045 		pmap_remove(pm, sva, eva);
   1046 		return;
   1047 	}
   1048 	bic = 0;
   1049 	if ((prot & VM_PROT_WRITE) == 0) {
   1050 		bic |= TTE_WR;
   1051 	}
   1052 	if ((prot & VM_PROT_EXECUTE) == 0) {
   1053 		bic |= TTE_EX;
   1054 	}
   1055 	if (bic == 0) {
   1056 		return;
   1057 	}
   1058 	s = splvm();
   1059 	while (sva < eva) {
   1060 		if ((ptp = pte_find(pm, sva)) != NULL) {
   1061 			*ptp &= ~bic;
   1062 			ppc4xx_tlb_flush(sva, pm->pm_ctx);
   1063 		}
   1064 		sva += PAGE_SIZE;
   1065 	}
   1066 	splx(s);
   1067 }
   1068 
   1069 bool
   1070 pmap_check_attr(struct vm_page *pg, u_int mask, int clear)
   1071 {
   1072 	paddr_t pa;
   1073 	char *attr;
   1074 	int s, rv;
   1075 
   1076 	/*
   1077 	 * First modify bits in cache.
   1078 	 */
   1079 	pa = VM_PAGE_TO_PHYS(pg);
   1080 	attr = pa_to_attr(pa);
   1081 	if (attr == NULL)
   1082 		return false;
   1083 
   1084 	s = splvm();
   1085 	rv = ((*attr & mask) != 0);
   1086 	if (clear) {
   1087 		*attr &= ~mask;
   1088 		pmap_page_protect(pg, mask == PMAP_ATTR_CHG ? VM_PROT_READ : 0);
   1089 	}
   1090 	splx(s);
   1091 	return rv;
   1092 }
   1093 
   1094 
   1095 /*
   1096  * Lower the protection on the specified physical page.
   1097  *
   1098  * There are only two cases: either the protection is going to 0,
   1099  * or it is going to read-only.
   1100  */
   1101 void
   1102 pmap_page_protect(struct vm_page *pg, vm_prot_t prot)
   1103 {
   1104 	paddr_t pa = VM_PAGE_TO_PHYS(pg);
   1105 	vaddr_t va;
   1106 	struct pv_entry *pvh, *pv, *npv;
   1107 	struct pmap *pm;
   1108 
   1109 	pvh = pa_to_pv(pa);
   1110 	if (pvh == NULL)
   1111 		return;
   1112 
   1113 	/* Handle extra pvs which may be deleted in the operation */
   1114 	for (pv = pvh->pv_next; pv; pv = npv) {
   1115 		npv = pv->pv_next;
   1116 
   1117 		pm = pv->pv_pm;
   1118 		va = PV_VA(pv);
   1119 		pmap_protect(pm, va, va + PAGE_SIZE, prot);
   1120 	}
   1121 	/* Now check the head pv */
   1122 	if (pvh->pv_pm) {
   1123 		pv = pvh;
   1124 		pm = pv->pv_pm;
   1125 		va = PV_VA(pv);
   1126 		pmap_protect(pm, va, va + PAGE_SIZE, prot);
   1127 	}
   1128 }
   1129 
   1130 /*
   1131  * Activate the address space for the specified process.  If the process
   1132  * is the current process, load the new MMU context.
   1133  */
   1134 void
   1135 pmap_activate(struct lwp *l)
   1136 {
   1137 #if 0
   1138 	struct pcb *pcb = lwp_getpcb(l);
   1139 	pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap;
   1140 
   1141 	/*
   1142 	 * XXX Normally performed in cpu_lwp_fork().
   1143 	 */
   1144 	printf("pmap_activate(%p), pmap=%p\n",l,pmap);
   1145 	pcb->pcb_pm = pmap;
   1146 #endif
   1147 }
   1148 
   1149 /*
   1150  * Deactivate the specified process's address space.
   1151  */
   1152 void
   1153 pmap_deactivate(struct lwp *l)
   1154 {
   1155 }
   1156 
   1157 /*
   1158  * Synchronize caches corresponding to [addr, addr+len) in p.
   1159  */
   1160 void
   1161 pmap_procwr(struct proc *p, vaddr_t va, size_t len)
   1162 {
   1163 	struct pmap *pm = p->p_vmspace->vm_map.pmap;
   1164 	int msr, ctx, opid, step;
   1165 
   1166 	step = CACHELINESIZE;
   1167 
   1168 	/*
   1169 	 * Need to turn off IMMU and switch to user context.
   1170 	 * (icbi uses DMMU).
   1171 	 */
   1172 	if (!(ctx = pm->pm_ctx)) {
   1173 		/* No context -- assign it one */
   1174 		ctx_alloc(pm);
   1175 		ctx = pm->pm_ctx;
   1176 	}
   1177 	__asm volatile(
   1178 		"mfmsr %0;"
   1179 		"li %1, %7;"
   1180 		"andc %1,%0,%1;"
   1181 		"mtmsr %1;"
   1182 		"isync;"
   1183 		"mfpid %1;"
   1184 		"mtpid %2;"
   1185 		"isync;"
   1186 		"1:"
   1187 		"dcbst 0,%3;"
   1188 		"icbi 0,%3;"
   1189 		"add %3,%3,%5;"
   1190 		"addc. %4,%4,%6;"
   1191 		"bge 1b;"
   1192 		"sync;"
   1193 		"mtpid %1;"
   1194 		"mtmsr %0;"
   1195 		"isync;"
   1196 		: "=&r" (msr), "=&r" (opid)
   1197 		: "r" (ctx), "r" (va), "r" (len), "r" (step), "r" (-step),
   1198 		  "K" (PSL_IR | PSL_DR));
   1199 }
   1200 
   1201 static inline void
   1202 tlb_invalidate_entry(int i)
   1203 {
   1204 #ifdef PMAP_TLBDEBUG
   1205 	/*
   1206 	 * Clear only TLBHI[V] bit so that we can track invalidated entry.
   1207 	 */
   1208 	register_t msr, pid, hi;
   1209 
   1210 	KASSERT(mfspr(SPR_PID) == KERNEL_PID);
   1211 
   1212 	__asm volatile(
   1213 		"mfmsr	%0;"
   1214 		"li	%1,0;"
   1215 		"mtmsr	%1;"
   1216 		"mfpid	%1;"
   1217 		"tlbre	%2,%3,0;"
   1218 		"andc	%2,%2,%4;"
   1219 		"tlbwe	%2,%3,0;"
   1220 		"mtpid	%1;"
   1221 		"mtmsr	%0;"
   1222 		"isync;"
   1223 		: "=&r" (msr), "=&r" (pid), "=&r" (hi)
   1224 		: "r" (i), "r" (TLB_VALID));
   1225 #else
   1226 	/*
   1227 	 * Just clear entire TLBHI register.
   1228 	 */
   1229 	__asm volatile(
   1230 		"tlbwe %0,%1,0;"
   1231 		"isync;"
   1232 		: : "r" (0), "r" (i));
   1233 #endif
   1234 
   1235 	tlb_info[i].ti_ctx = 0;
   1236 	tlb_info[i].ti_flags = 0;
   1237 }
   1238 
   1239 /* This has to be done in real mode !!! */
   1240 void
   1241 ppc4xx_tlb_flush(vaddr_t va, int pid)
   1242 {
   1243 	u_long i, found;
   1244 	u_long msr;
   1245 
   1246 	/* If there's no context then it can't be mapped. */
   1247 	if (!pid)
   1248 		return;
   1249 
   1250 	__asm volatile(
   1251 		"mfpid %1;"		/* Save PID */
   1252 		"mfmsr %2;"		/* Save MSR */
   1253 		"li %0,0;"		/* Now clear MSR */
   1254 		"mtmsr %0;"
   1255 		"isync;"
   1256 		"mtpid %4;"		/* Set PID */
   1257 		"isync;"
   1258 		"tlbsx. %0,0,%3;"	/* Search TLB */
   1259 		"isync;"
   1260 		"mtpid %1;"		/* Restore PID */
   1261 		"mtmsr %2;"		/* Restore MSR */
   1262 		"isync;"
   1263 		"li %1,1;"
   1264 		"beq 1f;"
   1265 		"li %1,0;"
   1266 		"1:"
   1267 		: "=&r" (i), "=&r" (found), "=&r" (msr)
   1268 		: "r" (va), "r" (pid));
   1269 	if (found && !TLB_LOCKED(i)) {
   1270 		/* Now flush translation */
   1271 		tlb_invalidate_entry(i);
   1272 		tlbnext = i;
   1273 		/* Successful flushes */
   1274 		tlbflush_ev.ev_count++;
   1275 	}
   1276 }
   1277 
   1278 void
   1279 ppc4xx_tlb_flush_all(void)
   1280 {
   1281 	u_long i;
   1282 
   1283 	for (i = 0; i < NTLB; i++)
   1284 		if (!TLB_LOCKED(i))
   1285 			tlb_invalidate_entry(i);
   1286 
   1287 	__asm volatile("isync");
   1288 }
   1289 
   1290 /* Find a TLB entry to evict. */
   1291 static int
   1292 ppc4xx_tlb_find_victim(void)
   1293 {
   1294 	int flags;
   1295 
   1296 	for (;;) {
   1297 		if (++tlbnext >= NTLB)
   1298 			tlbnext = tlb_nreserved;
   1299 		flags = tlb_info[tlbnext].ti_flags;
   1300 		if (!(flags & TLBF_USED) ||
   1301 			(flags & (TLBF_LOCKED | TLBF_REF)) == 0) {
   1302 			u_long va, stack = (u_long)&va;
   1303 
   1304 			if (!((tlb_info[tlbnext].ti_va ^ stack) & (~PGOFSET)) &&
   1305 			    (tlb_info[tlbnext].ti_ctx == KERNEL_PID) &&
   1306 			     (flags & TLBF_USED)) {
   1307 				/* Kernel stack page */
   1308 				flags |= TLBF_REF;
   1309 				tlb_info[tlbnext].ti_flags = flags;
   1310 			} else {
   1311 				/* Found it! */
   1312 				return (tlbnext);
   1313 			}
   1314 		} else {
   1315 			tlb_info[tlbnext].ti_flags = (flags & ~TLBF_REF);
   1316 		}
   1317 	}
   1318 }
   1319 
   1320 void
   1321 ppc4xx_tlb_enter(int ctx, vaddr_t va, u_int pte)
   1322 {
   1323 	u_long th, tl, idx;
   1324 	int msr, pid;
   1325 	paddr_t pa;
   1326 	int sz;
   1327 
   1328 	tlbenter_ev.ev_count++;
   1329 
   1330 	sz = (pte & TTE_SZ_MASK) >> TTE_SZ_SHIFT;
   1331 	pa = (pte & TTE_RPN_MASK(sz));
   1332 	th = (va & TLB_EPN_MASK) | (sz << TLB_SIZE_SHFT) | TLB_VALID;
   1333 	tl = (pte & ~TLB_RPN_MASK) | pa;
   1334 	tl |= ppc4xx_tlbflags(va, pa);
   1335 
   1336 	idx = ppc4xx_tlb_find_victim();
   1337 
   1338 #ifdef DIAGNOSTIC
   1339 	if ((idx < tlb_nreserved) || (idx >= NTLB) || (idx & 63) == 0) {
   1340 		panic("ppc4xx_tlb_enter: replacing entry %ld", idx);
   1341 	}
   1342 #endif
   1343 
   1344 	tlb_info[idx].ti_va = (va & TLB_EPN_MASK);
   1345 	tlb_info[idx].ti_ctx = ctx;
   1346 	tlb_info[idx].ti_flags = TLBF_USED | TLBF_REF;
   1347 
   1348 	__asm volatile(
   1349 		"mfmsr %0;"			/* Save MSR */
   1350 		"li %1,0;"
   1351 		"mtmsr %1;"			/* Clear MSR */
   1352 		"isync;"
   1353 		"tlbwe %1,%3,0;"		/* Invalidate old entry. */
   1354 		"mfpid %1;"			/* Save old PID */
   1355 		"mtpid %2;"			/* Load translation ctx */
   1356 		"isync;"
   1357 		"tlbwe %4,%3,1; tlbwe %5,%3,0;"	/* Set TLB */
   1358 		"isync;"
   1359 		"mtpid %1; mtmsr %0;"		/* Restore PID and MSR */
   1360 		"isync;"
   1361 	: "=&r" (msr), "=&r" (pid)
   1362 	: "r" (ctx), "r" (idx), "r" (tl), "r" (th));
   1363 }
   1364 
   1365 void
   1366 ppc4xx_tlb_init(void)
   1367 {
   1368 	int i;
   1369 
   1370 	/* Mark reserved TLB entries */
   1371 	for (i = 0; i < tlb_nreserved; i++) {
   1372 		tlb_info[i].ti_flags = TLBF_LOCKED | TLBF_USED;
   1373 		tlb_info[i].ti_ctx = KERNEL_PID;
   1374 	}
   1375 
   1376 	/* Setup security zones */
   1377 	/* Z0 - accessible by kernel only if TLB entry permissions allow
   1378 	 * Z1,Z2 - access is controlled by TLB entry permissions
   1379 	 * Z3 - full access regardless of TLB entry permissions
   1380 	 */
   1381 
   1382 	__asm volatile(
   1383 		"mtspr %0,%1;"
   1384 		"isync;"
   1385 		::  "K"(SPR_ZPR), "r" (0x1b000000));
   1386 }
   1387 
   1388 /*
   1389  * ppc4xx_tlb_size_mask:
   1390  *
   1391  * 	Roundup size to supported page size, return TLBHI mask and real size.
   1392  */
   1393 static int
   1394 ppc4xx_tlb_size_mask(size_t size, int *mask, int *rsiz)
   1395 {
   1396 	int 			i;
   1397 
   1398 	for (i = 0; i < __arraycount(tlbsize); i++)
   1399 		if (size <= tlbsize[i]) {
   1400 			*mask = (i << TLB_SIZE_SHFT);
   1401 			*rsiz = tlbsize[i];
   1402 			return (0);
   1403 		}
   1404 	return (EINVAL);
   1405 }
   1406 
   1407 /*
   1408  * ppc4xx_tlb_mapiodev:
   1409  *
   1410  * 	Lookup virtual address of mapping previously entered via
   1411  * 	ppc4xx_tlb_reserve. Search TLB directly so that we don't
   1412  * 	need to waste extra storage for reserved mappings. Note
   1413  * 	that reading TLBHI also sets PID, but all reserved mappings
   1414  * 	use KERNEL_PID, so the side effect is nil.
   1415  */
   1416 void *
   1417 ppc4xx_tlb_mapiodev(paddr_t base, psize_t len)
   1418 {
   1419 	paddr_t 		pa;
   1420 	vaddr_t 		va;
   1421 	u_int 			lo, hi, sz;
   1422 	int 			i;
   1423 
   1424 	/* tlb_nreserved is only allowed to grow, so this is safe. */
   1425 	for (i = 0; i < tlb_nreserved; i++) {
   1426 		__asm volatile (
   1427 		    "	tlbre %0,%2,1 	\n" 	/* TLBLO */
   1428 		    "	tlbre %1,%2,0 	\n" 	/* TLBHI */
   1429 		    : "=&r" (lo), "=&r" (hi)
   1430 		    : "r" (i));
   1431 
   1432 		KASSERT(hi & TLB_VALID);
   1433 		KASSERT(mfspr(SPR_PID) == KERNEL_PID);
   1434 
   1435 		pa = (lo & TLB_RPN_MASK);
   1436 		if (base < pa)
   1437 			continue;
   1438 
   1439 		sz = tlbsize[(hi & TLB_SIZE_MASK) >> TLB_SIZE_SHFT];
   1440 		if ((base + len) > (pa + sz))
   1441 			continue;
   1442 
   1443 		va = (hi & TLB_EPN_MASK) + (base & (sz - 1)); 	/* sz = 2^n */
   1444 		return (void *)(va);
   1445 	}
   1446 
   1447 	return (NULL);
   1448 }
   1449 
   1450 /*
   1451  * ppc4xx_tlb_reserve:
   1452  *
   1453  * 	Map physical range to kernel virtual chunk via reserved TLB entry.
   1454  */
   1455 void
   1456 ppc4xx_tlb_reserve(paddr_t pa, vaddr_t va, size_t size, int flags)
   1457 {
   1458 	u_int 			lo, hi;
   1459 	int 			szmask, rsize;
   1460 
   1461 	/* Called before pmap_bootstrap(), va outside kernel space. */
   1462 	KASSERT(va < VM_MIN_KERNEL_ADDRESS || va >= VM_MAX_KERNEL_ADDRESS);
   1463 	KASSERT(! pmap_bootstrap_done);
   1464 	KASSERT(tlb_nreserved < NTLB);
   1465 
   1466 	/* Resolve size. */
   1467 	if (ppc4xx_tlb_size_mask(size, &szmask, &rsize) != 0)
   1468 		panic("ppc4xx_tlb_reserve: entry %d, %zuB too large",
   1469 		    size, tlb_nreserved);
   1470 
   1471 	/* Real size will be power of two >= 1024, so this is OK. */
   1472 	pa &= ~(rsize - 1); 	/* RPN */
   1473 	va &= ~(rsize - 1); 	/* EPN */
   1474 
   1475 	lo = pa | TLB_WR | flags;
   1476 	hi = va | TLB_VALID | szmask;
   1477 
   1478 #ifdef PPC_4XX_NOCACHE
   1479 	lo |= TLB_I;
   1480 #endif
   1481 
   1482 	__asm volatile(
   1483 	    "	tlbwe %1,%0,1 	\n" 	/* write TLBLO */
   1484 	    "	tlbwe %2,%0,0 	\n" 	/* write TLBHI */
   1485 	    "	isync 		\n"
   1486 	    : : "r" (tlb_nreserved), "r" (lo), "r" (hi));
   1487 
   1488 	tlb_nreserved++;
   1489 }
   1490 
   1491 /*
   1492  * We should pass the ctx in from trap code.
   1493  */
   1494 int
   1495 pmap_tlbmiss(vaddr_t va, int ctx)
   1496 {
   1497 	volatile u_int *pte;
   1498 	u_long tte;
   1499 
   1500 	tlbmiss_ev.ev_count++;
   1501 
   1502 	/*
   1503 	 * We will reserve 0 upto VM_MIN_KERNEL_ADDRESS for va == pa mappings.
   1504 	 * Physical RAM is expected to live in this range, care must be taken
   1505 	 * to not clobber 0 upto ${physmem} with device mappings in machdep
   1506 	 * code.
   1507 	 */
   1508 	if (ctx != KERNEL_PID ||
   1509 	    (va >= VM_MIN_KERNEL_ADDRESS && va < VM_MAX_KERNEL_ADDRESS)) {
   1510 		pte = pte_find((struct pmap *)__UNVOLATILE(ctxbusy[ctx]), va);
   1511 		if (pte == NULL) {
   1512 			/* Map unmanaged addresses directly for kernel access */
   1513 			return 1;
   1514 		}
   1515 		tte = *pte;
   1516 		if (tte == 0) {
   1517 			return 1;
   1518 		}
   1519 	} else {
   1520 		/* Create a 16MB writable mapping. */
   1521 #ifdef PPC_4XX_NOCACHE
   1522 		tte = TTE_PA(va) | TTE_ZONE(ZONE_PRIV) | TTE_SZ_16M | TTE_I |TTE_WR;
   1523 #else
   1524 		tte = TTE_PA(va) | TTE_ZONE(ZONE_PRIV) | TTE_SZ_16M | TTE_WR;
   1525 #endif
   1526 	}
   1527 	ppc4xx_tlb_enter(ctx, va, tte);
   1528 
   1529 	return 0;
   1530 }
   1531 
   1532 /*
   1533  * Flush all the entries matching a context from the TLB.
   1534  */
   1535 static int
   1536 ctx_flush(int cnum)
   1537 {
   1538 	int i;
   1539 
   1540 	/* We gotta steal this context */
   1541 	for (i = tlb_nreserved; i < NTLB; i++) {
   1542 		if (tlb_info[i].ti_ctx == cnum) {
   1543 			/* Can't steal ctx if it has a locked entry. */
   1544 			if (TLB_LOCKED(i)) {
   1545 #ifdef DIAGNOSTIC
   1546 				printf("ctx_flush: can't invalidate "
   1547 					"locked mapping %d "
   1548 					"for context %d\n", i, cnum);
   1549 #ifdef DDB
   1550 				Debugger();
   1551 #endif
   1552 #endif
   1553 				return (1);
   1554 			}
   1555 #ifdef DIAGNOSTIC
   1556 			if (i < tlb_nreserved)
   1557 				panic("TLB entry %d not locked", i);
   1558 #endif
   1559 			/*
   1560 			 * Invalidate particular TLB entry regardless of
   1561 			 * locked status
   1562 			 */
   1563 			tlb_invalidate_entry(i);
   1564 		}
   1565 	}
   1566 	return (0);
   1567 }
   1568 
   1569 /*
   1570  * Allocate a context.  If necessary, steal one from someone else.
   1571  *
   1572  * The new context is flushed from the TLB before returning.
   1573  */
   1574 int
   1575 ctx_alloc(struct pmap *pm)
   1576 {
   1577 	int s, cnum;
   1578 	static int next = MINCTX;
   1579 
   1580 	if (pm == pmap_kernel()) {
   1581 #ifdef DIAGNOSTIC
   1582 		printf("ctx_alloc: kernel pmap!\n");
   1583 #endif
   1584 		return (0);
   1585 	}
   1586 	s = splvm();
   1587 
   1588 	/* Find a likely context. */
   1589 	cnum = next;
   1590 	do {
   1591 		if ((++cnum) >= NUMCTX)
   1592 			cnum = MINCTX;
   1593 	} while (ctxbusy[cnum] != NULL && cnum != next);
   1594 
   1595 	/* Now clean it out */
   1596 oops:
   1597 	if (cnum < MINCTX)
   1598 		cnum = MINCTX; /* Never steal ctx 0 or 1 */
   1599 	if (ctx_flush(cnum)) {
   1600 		/* oops -- something's wired. */
   1601 		if ((++cnum) >= NUMCTX)
   1602 			cnum = MINCTX;
   1603 		goto oops;
   1604 	}
   1605 
   1606 	if (ctxbusy[cnum]) {
   1607 #ifdef DEBUG
   1608 		/* We should identify this pmap and clear it */
   1609 		printf("Warning: stealing context %d\n", cnum);
   1610 #endif
   1611 		ctxbusy[cnum]->pm_ctx = 0;
   1612 	}
   1613 	ctxbusy[cnum] = pm;
   1614 	next = cnum;
   1615 	splx(s);
   1616 	pm->pm_ctx = cnum;
   1617 
   1618 	return cnum;
   1619 }
   1620 
   1621 /*
   1622  * Give away a context.
   1623  */
   1624 void
   1625 ctx_free(struct pmap *pm)
   1626 {
   1627 	int oldctx;
   1628 
   1629 	oldctx = pm->pm_ctx;
   1630 
   1631 	if (oldctx == 0)
   1632 		panic("ctx_free: freeing kernel context");
   1633 #ifdef DIAGNOSTIC
   1634 	if (ctxbusy[oldctx] == 0)
   1635 		printf("ctx_free: freeing free context %d\n", oldctx);
   1636 	if (ctxbusy[oldctx] != pm) {
   1637 		printf("ctx_free: freeing someone esle's context\n "
   1638 		       "ctxbusy[%d] = %p, pm->pm_ctx = %p\n",
   1639 		       oldctx, (void *)(u_long)ctxbusy[oldctx], pm);
   1640 #ifdef DDB
   1641 		Debugger();
   1642 #endif
   1643 	}
   1644 #endif
   1645 	/* We should verify it has not been stolen and reallocated... */
   1646 	ctxbusy[oldctx] = NULL;
   1647 	ctx_flush(oldctx);
   1648 }
   1649 
   1650 
   1651 #ifdef DEBUG
   1652 /*
   1653  * Test ref/modify handling.
   1654  */
   1655 void pmap_testout(void);
   1656 void
   1657 pmap_testout(void)
   1658 {
   1659 	vaddr_t va;
   1660 	volatile int *loc;
   1661 	int val = 0;
   1662 	paddr_t pa;
   1663 	struct vm_page *pg;
   1664 	int ref, mod;
   1665 
   1666 	/* Allocate a page */
   1667 	va = (vaddr_t)uvm_km_alloc(kernel_map, PAGE_SIZE, 0,
   1668 	    UVM_KMF_WIRED | UVM_KMF_ZERO);
   1669 	loc = (int*)va;
   1670 
   1671 	pmap_extract(pmap_kernel(), va, &pa);
   1672 	pg = PHYS_TO_VM_PAGE(pa);
   1673 	pmap_unwire(pmap_kernel(), va);
   1674 
   1675 	pmap_kremove(va, PAGE_SIZE);
   1676 	pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0);
   1677 	pmap_update(pmap_kernel());
   1678 
   1679 	/* Now clear reference and modify */
   1680 	ref = pmap_clear_reference(pg);
   1681 	mod = pmap_clear_modify(pg);
   1682 	printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
   1683 	       (void *)(u_long)va, (long)pa,
   1684 	       ref, mod);
   1685 
   1686 	/* Check it's properly cleared */
   1687 	ref = pmap_is_referenced(pg);
   1688 	mod = pmap_is_modified(pg);
   1689 	printf("Checking cleared page: ref %d, mod %d\n",
   1690 	       ref, mod);
   1691 
   1692 	/* Reference page */
   1693 	val = *loc;
   1694 
   1695 	ref = pmap_is_referenced(pg);
   1696 	mod = pmap_is_modified(pg);
   1697 	printf("Referenced page: ref %d, mod %d val %x\n",
   1698 	       ref, mod, val);
   1699 
   1700 	/* Now clear reference and modify */
   1701 	ref = pmap_clear_reference(pg);
   1702 	mod = pmap_clear_modify(pg);
   1703 	printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
   1704 	       (void *)(u_long)va, (long)pa,
   1705 	       ref, mod);
   1706 
   1707 	/* Modify page */
   1708 	*loc = 1;
   1709 
   1710 	ref = pmap_is_referenced(pg);
   1711 	mod = pmap_is_modified(pg);
   1712 	printf("Modified page: ref %d, mod %d\n",
   1713 	       ref, mod);
   1714 
   1715 	/* Now clear reference and modify */
   1716 	ref = pmap_clear_reference(pg);
   1717 	mod = pmap_clear_modify(pg);
   1718 	printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
   1719 	       (void *)(u_long)va, (long)pa,
   1720 	       ref, mod);
   1721 
   1722 	/* Check it's properly cleared */
   1723 	ref = pmap_is_referenced(pg);
   1724 	mod = pmap_is_modified(pg);
   1725 	printf("Checking cleared page: ref %d, mod %d\n",
   1726 	       ref, mod);
   1727 
   1728 	/* Modify page */
   1729 	*loc = 1;
   1730 
   1731 	ref = pmap_is_referenced(pg);
   1732 	mod = pmap_is_modified(pg);
   1733 	printf("Modified page: ref %d, mod %d\n",
   1734 	       ref, mod);
   1735 
   1736 	/* Check pmap_protect() */
   1737 	pmap_protect(pmap_kernel(), va, va+1, VM_PROT_READ);
   1738 	pmap_update(pmap_kernel());
   1739 	ref = pmap_is_referenced(pg);
   1740 	mod = pmap_is_modified(pg);
   1741 	printf("pmap_protect(VM_PROT_READ): ref %d, mod %d\n",
   1742 	       ref, mod);
   1743 
   1744 	/* Now clear reference and modify */
   1745 	ref = pmap_clear_reference(pg);
   1746 	mod = pmap_clear_modify(pg);
   1747 	printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
   1748 	       (void *)(u_long)va, (long)pa,
   1749 	       ref, mod);
   1750 
   1751 	/* Reference page */
   1752 	val = *loc;
   1753 
   1754 	ref = pmap_is_referenced(pg);
   1755 	mod = pmap_is_modified(pg);
   1756 	printf("Referenced page: ref %d, mod %d val %x\n",
   1757 	       ref, mod, val);
   1758 
   1759 	/* Now clear reference and modify */
   1760 	ref = pmap_clear_reference(pg);
   1761 	mod = pmap_clear_modify(pg);
   1762 	printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
   1763 	       (void *)(u_long)va, (long)pa,
   1764 	       ref, mod);
   1765 
   1766 	/* Modify page */
   1767 #if 0
   1768 	pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0);
   1769 	pmap_update(pmap_kernel());
   1770 #endif
   1771 	*loc = 1;
   1772 
   1773 	ref = pmap_is_referenced(pg);
   1774 	mod = pmap_is_modified(pg);
   1775 	printf("Modified page: ref %d, mod %d\n",
   1776 	       ref, mod);
   1777 
   1778 	/* Check pmap_protect() */
   1779 	pmap_protect(pmap_kernel(), va, va+1, VM_PROT_NONE);
   1780 	pmap_update(pmap_kernel());
   1781 	ref = pmap_is_referenced(pg);
   1782 	mod = pmap_is_modified(pg);
   1783 	printf("pmap_protect(): ref %d, mod %d\n",
   1784 	       ref, mod);
   1785 
   1786 	/* Now clear reference and modify */
   1787 	ref = pmap_clear_reference(pg);
   1788 	mod = pmap_clear_modify(pg);
   1789 	printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
   1790 	       (void *)(u_long)va, (long)pa,
   1791 	       ref, mod);
   1792 
   1793 	/* Reference page */
   1794 	val = *loc;
   1795 
   1796 	ref = pmap_is_referenced(pg);
   1797 	mod = pmap_is_modified(pg);
   1798 	printf("Referenced page: ref %d, mod %d val %x\n",
   1799 	       ref, mod, val);
   1800 
   1801 	/* Now clear reference and modify */
   1802 	ref = pmap_clear_reference(pg);
   1803 	mod = pmap_clear_modify(pg);
   1804 	printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
   1805 	       (void *)(u_long)va, (long)pa,
   1806 	       ref, mod);
   1807 
   1808 	/* Modify page */
   1809 #if 0
   1810 	pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0);
   1811 	pmap_update(pmap_kernel());
   1812 #endif
   1813 	*loc = 1;
   1814 
   1815 	ref = pmap_is_referenced(pg);
   1816 	mod = pmap_is_modified(pg);
   1817 	printf("Modified page: ref %d, mod %d\n",
   1818 	       ref, mod);
   1819 
   1820 	/* Check pmap_pag_protect() */
   1821 	pmap_page_protect(pg, VM_PROT_READ);
   1822 	ref = pmap_is_referenced(pg);
   1823 	mod = pmap_is_modified(pg);
   1824 	printf("pmap_page_protect(VM_PROT_READ): ref %d, mod %d\n",
   1825 	       ref, mod);
   1826 
   1827 	/* Now clear reference and modify */
   1828 	ref = pmap_clear_reference(pg);
   1829 	mod = pmap_clear_modify(pg);
   1830 	printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
   1831 	       (void *)(u_long)va, (long)pa,
   1832 	       ref, mod);
   1833 
   1834 	/* Reference page */
   1835 	val = *loc;
   1836 
   1837 	ref = pmap_is_referenced(pg);
   1838 	mod = pmap_is_modified(pg);
   1839 	printf("Referenced page: ref %d, mod %d val %x\n",
   1840 	       ref, mod, val);
   1841 
   1842 	/* Now clear reference and modify */
   1843 	ref = pmap_clear_reference(pg);
   1844 	mod = pmap_clear_modify(pg);
   1845 	printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
   1846 	       (void *)(u_long)va, (long)pa,
   1847 	       ref, mod);
   1848 
   1849 	/* Modify page */
   1850 #if 0
   1851 	pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0);
   1852 	pmap_update(pmap_kernel());
   1853 #endif
   1854 	*loc = 1;
   1855 
   1856 	ref = pmap_is_referenced(pg);
   1857 	mod = pmap_is_modified(pg);
   1858 	printf("Modified page: ref %d, mod %d\n",
   1859 	       ref, mod);
   1860 
   1861 	/* Check pmap_pag_protect() */
   1862 	pmap_page_protect(pg, VM_PROT_NONE);
   1863 	ref = pmap_is_referenced(pg);
   1864 	mod = pmap_is_modified(pg);
   1865 	printf("pmap_page_protect(): ref %d, mod %d\n",
   1866 	       ref, mod);
   1867 
   1868 	/* Now clear reference and modify */
   1869 	ref = pmap_clear_reference(pg);
   1870 	mod = pmap_clear_modify(pg);
   1871 	printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
   1872 	       (void *)(u_long)va, (long)pa,
   1873 	       ref, mod);
   1874 
   1875 
   1876 	/* Reference page */
   1877 	val = *loc;
   1878 
   1879 	ref = pmap_is_referenced(pg);
   1880 	mod = pmap_is_modified(pg);
   1881 	printf("Referenced page: ref %d, mod %d val %x\n",
   1882 	       ref, mod, val);
   1883 
   1884 	/* Now clear reference and modify */
   1885 	ref = pmap_clear_reference(pg);
   1886 	mod = pmap_clear_modify(pg);
   1887 	printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
   1888 	       (void *)(u_long)va, (long)pa,
   1889 	       ref, mod);
   1890 
   1891 	/* Modify page */
   1892 #if 0
   1893 	pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0);
   1894 	pmap_update(pmap_kernel());
   1895 #endif
   1896 	*loc = 1;
   1897 
   1898 	ref = pmap_is_referenced(pg);
   1899 	mod = pmap_is_modified(pg);
   1900 	printf("Modified page: ref %d, mod %d\n",
   1901 	       ref, mod);
   1902 
   1903 	/* Unmap page */
   1904 	pmap_remove(pmap_kernel(), va, va+1);
   1905 	pmap_update(pmap_kernel());
   1906 	ref = pmap_is_referenced(pg);
   1907 	mod = pmap_is_modified(pg);
   1908 	printf("Unmapped page: ref %d, mod %d\n", ref, mod);
   1909 
   1910 	/* Now clear reference and modify */
   1911 	ref = pmap_clear_reference(pg);
   1912 	mod = pmap_clear_modify(pg);
   1913 	printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
   1914 	       (void *)(u_long)va, (long)pa, ref, mod);
   1915 
   1916 	/* Check it's properly cleared */
   1917 	ref = pmap_is_referenced(pg);
   1918 	mod = pmap_is_modified(pg);
   1919 	printf("Checking cleared page: ref %d, mod %d\n",
   1920 	       ref, mod);
   1921 
   1922 	pmap_remove(pmap_kernel(), va, va + PAGE_SIZE);
   1923 	pmap_kenter_pa(va, pa, VM_PROT_ALL, 0);
   1924 	uvm_km_free(kernel_map, (vaddr_t)va, PAGE_SIZE, UVM_KMF_WIRED);
   1925 }
   1926 #endif
   1927