Home | History | Annotate | Line # | Download | only in booke
e500_tlb.c revision 1.7.6.1
      1 /*	$NetBSD: e500_tlb.c,v 1.7.6.1 2012/04/05 21:33:17 mrg Exp $	*/
      2 /*-
      3  * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
      4  * All rights reserved.
      5  *
      6  * This code is derived from software contributed to The NetBSD Foundation
      7  * by Raytheon BBN Technologies Corp and Defense Advanced Research Projects
      8  * Agency and which was developed by Matt Thomas of 3am Software Foundry.
      9  *
     10  * This material is based upon work supported by the Defense Advanced Research
     11  * Projects Agency and Space and Naval Warfare Systems Center, Pacific, under
     12  * Contract No. N66001-09-C-2073.
     13  * Approved for Public Release, Distribution Unlimited
     14  *
     15  * Redistribution and use in source and binary forms, with or without
     16  * modification, are permitted provided that the following conditions
     17  * are met:
     18  * 1. Redistributions of source code must retain the above copyright
     19  *    notice, this list of conditions and the following disclaimer.
     20  * 2. Redistributions in binary form must reproduce the above copyright
     21  *    notice, this list of conditions and the following disclaimer in the
     22  *    documentation and/or other materials provided with the distribution.
     23  *
     24  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     25  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     26  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     27  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     28  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     29  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     30  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     31  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     32  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     33  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     34  * POSSIBILITY OF SUCH DAMAGE.
     35  */
     36 
     37 #define	__PMAP_PRIVATE
     38 
     39 #include <sys/cdefs.h>
     40 
     41 __KERNEL_RCSID(0, "$NetBSD: e500_tlb.c,v 1.7.6.1 2012/04/05 21:33:17 mrg Exp $");
     42 
     43 #include <sys/param.h>
     44 
     45 #include <uvm/uvm_extern.h>
     46 
     47 #include <powerpc/spr.h>
     48 #include <powerpc/booke/spr.h>
     49 #include <powerpc/booke/cpuvar.h>
     50 #include <powerpc/booke/e500reg.h>
     51 #include <powerpc/booke/e500var.h>
     52 #include <powerpc/booke/pmap.h>
     53 
     54 struct e500_tlb {
     55 	vaddr_t tlb_va;
     56 	uint32_t tlb_pte;
     57 	uint32_t tlb_asid;
     58 	vsize_t tlb_size;
     59 };
     60 
     61 struct e500_hwtlb {
     62 	uint32_t hwtlb_mas0;
     63 	uint32_t hwtlb_mas1;
     64 	uint32_t hwtlb_mas2;
     65 	uint32_t hwtlb_mas3;
     66 };
     67 
     68 struct e500_xtlb {
     69 	struct e500_tlb e_tlb;
     70 	struct e500_hwtlb e_hwtlb;
     71 	u_long e_refcnt;
     72 };
     73 
     74 static struct e500_tlb1 {
     75 	uint32_t tlb1_maxsize;
     76 	uint32_t tlb1_minsize;
     77 	u_int tlb1_numentries;
     78 	u_int tlb1_numfree;
     79 	u_int tlb1_freelist[32];
     80 	struct e500_xtlb tlb1_entries[32];
     81 } e500_tlb1;
     82 
     83 static inline register_t mftlb0cfg(void) __pure;
     84 static inline register_t mftlb1cfg(void) __pure;
     85 
     86 static inline register_t
     87 mftlb0cfg(void)
     88 {
     89 	register_t tlb0cfg;
     90 	__asm("mfspr %0, %1" : "=r"(tlb0cfg) : "n"(SPR_TLB0CFG));
     91 	return tlb0cfg;
     92 }
     93 
     94 static inline register_t
     95 mftlb1cfg(void)
     96 {
     97 	register_t tlb1cfg;
     98 	__asm("mfspr %0, %1" : "=r"(tlb1cfg) : "n"(SPR_TLB1CFG));
     99 	return tlb1cfg;
    100 }
    101 
    102 static struct e500_tlb
    103 hwtlb_to_tlb(const struct e500_hwtlb hwtlb)
    104 {
    105 	struct e500_tlb tlb;
    106 	register_t prot_mask;
    107 	u_int prot_shift;
    108 
    109 	tlb.tlb_va = MAS2_EPN & hwtlb.hwtlb_mas2;
    110 	tlb.tlb_size = 1024 << (2 * MASX_TSIZE_GET(hwtlb.hwtlb_mas1));
    111 	tlb.tlb_asid = MASX_TID_GET(hwtlb.hwtlb_mas1);
    112 	tlb.tlb_pte = (hwtlb.hwtlb_mas2 & MAS2_WIMGE)
    113 	    | (hwtlb.hwtlb_mas3 & MAS3_RPN);
    114 	if (hwtlb.hwtlb_mas1 & MAS1_TS) {
    115 		prot_mask = MAS3_UX|MAS3_UW|MAS3_UR;
    116 		prot_shift = PTE_RWX_SHIFT - 1;
    117 	} else {
    118 		prot_mask = MAS3_SX|MAS3_SW|MAS3_SR;
    119 		prot_shift = PTE_RWX_SHIFT;
    120 	}
    121 	tlb.tlb_pte |= (prot_mask & hwtlb.hwtlb_mas3) << prot_shift;
    122 	return tlb;
    123 }
    124 
    125 static inline struct e500_hwtlb
    126 hwtlb_read(uint32_t mas0, u_int slot)
    127 {
    128 	struct e500_hwtlb hwtlb;
    129 	register_t tlbcfg;
    130 
    131 	if (__predict_true(mas0 == MAS0_TLBSEL_TLB0)) {
    132 		tlbcfg = mftlb0cfg();
    133 	} else if (mas0 == MAS0_TLBSEL_TLB1) {
    134 		tlbcfg = mftlb1cfg();
    135 	} else {
    136 		panic("%s:%d: unexpected MAS0 %#" PRIx32,
    137 		    __func__, __LINE__, mas0);
    138 	}
    139 
    140 	/*
    141 	 * ESEL is the way we want to look up.
    142 	 * If tlbassoc is the same as tlbentries (like in TLB1) then the TLB is
    143 	 * fully associative, the entire slot is placed into ESEL.  If tlbassoc
    144 	 * is less then the number of tlb entries, the slot is split in two
    145 	 * fields.  Since the TLB is M rows by N ways, the lowers bits are for
    146 	 * row (MAS2[EPN]) and the upper for the way (MAS1[ESEL]).
    147 	 */
    148 	const u_int tlbassoc = TLBCFG_ASSOC(tlbcfg);
    149 	const u_int tlbentries = TLBCFG_NENTRY(tlbcfg);
    150 	const u_int esel_shift =
    151 	    __builtin_clz(tlbassoc) - __builtin_clz(tlbentries);
    152 
    153 	/*
    154 	 * Disable interrupts since we don't want anyone else mucking with
    155 	 * the MMU Assist registers
    156 	 */
    157 	const register_t msr = wrtee(0);
    158 	const register_t saved_mas0 = mfspr(SPR_MAS0);
    159 	mtspr(SPR_MAS0, mas0 | MAS0_ESEL_MAKE(slot >> esel_shift));
    160 
    161 	if (__predict_true(tlbassoc > tlbentries))
    162 		mtspr(SPR_MAS2, slot << PAGE_SHIFT);
    163 
    164 	/*
    165 	 * Now select the entry and grab its contents.
    166 	 */
    167 	__asm volatile("tlbre");
    168 
    169 	hwtlb.hwtlb_mas0 = mfspr(SPR_MAS0);
    170 	hwtlb.hwtlb_mas1 = mfspr(SPR_MAS1);
    171 	hwtlb.hwtlb_mas2 = mfspr(SPR_MAS2);
    172 	hwtlb.hwtlb_mas3 = mfspr(SPR_MAS3);
    173 
    174 	mtspr(SPR_MAS0, saved_mas0);
    175 	wrtee(msr);	/* restore interrupts */
    176 
    177 	return hwtlb;
    178 }
    179 
    180 static inline void
    181 hwtlb_write(const struct e500_hwtlb hwtlb, bool needs_sync)
    182 {
    183 	const register_t msr = wrtee(0);
    184 	const uint32_t saved_mas0 = mfspr(SPR_MAS0);
    185 
    186 	/*
    187 	 * Need to always write MAS0 and MAS1
    188 	 */
    189 	mtspr(SPR_MAS0, hwtlb.hwtlb_mas0);
    190 	mtspr(SPR_MAS1, hwtlb.hwtlb_mas1);
    191 
    192 	/*
    193 	 * Only write the VPN/WIMGE if this is in TLB0 or if a valid mapping.
    194 	 */
    195 	if ((hwtlb.hwtlb_mas0 & MAS0_TLBSEL) == MAS0_TLBSEL_TLB0
    196 	    || (hwtlb.hwtlb_mas1 & MAS1_V)) {
    197 		mtspr(SPR_MAS2, hwtlb.hwtlb_mas2);
    198 	}
    199 	/*
    200 	 * Only need to write the RPN/prot if we are dealing with a valid
    201 	 * mapping.
    202 	 */
    203 	if (hwtlb.hwtlb_mas1 & MAS1_V) {
    204 		mtspr(SPR_MAS3, hwtlb.hwtlb_mas3);
    205 	}
    206 
    207 #if 0
    208 	printf("%s->[%x,%x,%x,%x]\n",
    209 	    __func__,
    210 	    hwtlb.hwtlb_mas0, hwtlb.hwtlb_mas1,
    211 	    hwtlb.hwtlb_mas2, hwtlb.hwtlb_mas3);
    212 #endif
    213 	__asm volatile("tlbwe");
    214 	if (needs_sync) {
    215 		__asm volatile("tlbsync");
    216 		__asm volatile("isync");
    217 	}
    218 
    219 	mtspr(SPR_MAS0, saved_mas0);
    220 	wrtee(msr);
    221 }
    222 
    223 static struct e500_hwtlb
    224 tlb_to_hwtlb(const struct e500_tlb tlb)
    225 {
    226 	struct e500_hwtlb hwtlb;
    227 
    228 	KASSERT(trunc_page(tlb.tlb_va) == tlb.tlb_va);
    229 	KASSERT(tlb.tlb_size != 0);
    230 	KASSERT((tlb.tlb_size & (tlb.tlb_size - 1)) == 0);
    231 	const uint32_t prot_mask = tlb.tlb_pte & PTE_RWX_MASK;
    232 	if (__predict_true(tlb.tlb_size == PAGE_SIZE)) {
    233 		hwtlb.hwtlb_mas0 = 0;
    234 		hwtlb.hwtlb_mas1 = MAS1_V | MASX_TSIZE_MAKE(1);
    235 		/*
    236 		 * A non-zero ASID means this is a user page so mark it as
    237 		 * being in the user's address space.
    238 		 */
    239 		if (tlb.tlb_asid) {
    240 			hwtlb.hwtlb_mas1 |= MAS1_TS
    241 			    | MASX_TID_MAKE(tlb.tlb_asid);
    242 			hwtlb.hwtlb_mas3 = (prot_mask >> (PTE_RWX_SHIFT - 1))
    243 			    | ((prot_mask & ~PTE_xX) >> PTE_RWX_SHIFT);
    244 			KASSERT(prot_mask & PTE_xR);
    245 			KASSERT(hwtlb.hwtlb_mas3 & MAS3_UR);
    246 			CTASSERT(MAS3_UR == (PTE_xR >> (PTE_RWX_SHIFT - 1)));
    247 			CTASSERT(MAS3_SR == (PTE_xR >> PTE_RWX_SHIFT));
    248 		} else {
    249 			hwtlb.hwtlb_mas3 = prot_mask >> PTE_RWX_SHIFT;
    250 		}
    251 		if (tlb.tlb_pte & PTE_UNMODIFIED)
    252 			hwtlb.hwtlb_mas3 &= ~(MAS3_UW|MAS3_SW);
    253 		if (tlb.tlb_pte & PTE_UNSYNCED)
    254 			hwtlb.hwtlb_mas3 &= ~(MAS3_UX|MAS3_SX);
    255 	} else {
    256 		KASSERT(tlb.tlb_asid == 0);
    257 		KASSERT((tlb.tlb_size & 0xaaaaa7ff) == 0);
    258 		u_int cntlz = __builtin_clz(tlb.tlb_size);
    259 		KASSERT(cntlz & 1);
    260 		KASSERT(cntlz <= 19);
    261 		hwtlb.hwtlb_mas0 = MAS0_TLBSEL_TLB1;
    262 		/*
    263 		 * TSIZE is defined (4^TSIZE) Kbytes except a TSIZE of 0 is not
    264 		 * allowed.  So 1K would be 0x00000400 giving 21 leading zero
    265 		 * bits.  Subtracting the leading number of zero bits from 21
    266 		 * and dividing by 2 gives us the number that the MMU wants.
    267 		 */
    268 		hwtlb.hwtlb_mas1 = MASX_TSIZE_MAKE(((31 - 10) - cntlz) / 2)
    269 		    | MAS1_IPROT | MAS1_V;
    270 		hwtlb.hwtlb_mas3 = prot_mask >> PTE_RWX_SHIFT;
    271 	}
    272 	/* We are done with MAS1, on to MAS2 ... */
    273 	hwtlb.hwtlb_mas2 = tlb.tlb_va | (tlb.tlb_pte & PTE_WIMGE_MASK);
    274 	hwtlb.hwtlb_mas3 |= tlb.tlb_pte & PTE_RPN_MASK;
    275 
    276 	return hwtlb;
    277 }
    278 
    279 void *
    280 e500_tlb1_fetch(size_t slot)
    281 {
    282 	struct e500_tlb1 * const tlb1 = &e500_tlb1;
    283 
    284 	return &tlb1->tlb1_entries[slot].e_hwtlb;
    285 }
    286 
    287 void
    288 e500_tlb1_sync(void)
    289 {
    290 	struct e500_tlb1 * const tlb1 = &e500_tlb1;
    291 	for (u_int slot = 1; slot < tlb1->tlb1_numentries; slot++) {
    292 		const struct e500_hwtlb * const new_hwtlb =
    293 		    &tlb1->tlb1_entries[slot].e_hwtlb;
    294 		const struct e500_hwtlb old_hwtlb =
    295 		    hwtlb_read(MAS0_TLBSEL_TLB1, slot);
    296 #define CHANGED(n,o,f)	((n)->f != (o).f)
    297 		bool mas1_changed_p = CHANGED(new_hwtlb, old_hwtlb, hwtlb_mas1);
    298 		bool mas2_changed_p = CHANGED(new_hwtlb, old_hwtlb, hwtlb_mas2);
    299 		bool mas3_changed_p = CHANGED(new_hwtlb, old_hwtlb, hwtlb_mas3);
    300 #undef CHANGED
    301 		bool new_valid_p = (new_hwtlb->hwtlb_mas1 & MAS1_V) != 0;
    302 		bool old_valid_p = (old_hwtlb.hwtlb_mas1 & MAS1_V) != 0;
    303 		if ((new_valid_p || old_valid_p)
    304 		    && (mas1_changed_p
    305 			|| (new_valid_p
    306 			    && (mas2_changed_p || mas3_changed_p))))
    307 			hwtlb_write(*new_hwtlb, true);
    308 	}
    309 }
    310 
    311 static int
    312 e500_alloc_tlb1_entry(void)
    313 {
    314 	struct e500_tlb1 * const tlb1 = &e500_tlb1;
    315 
    316 	if (tlb1->tlb1_numfree == 0)
    317 		return -1;
    318 	const u_int slot = tlb1->tlb1_freelist[--tlb1->tlb1_numfree];
    319 	KASSERT((tlb1->tlb1_entries[slot].e_hwtlb.hwtlb_mas1 & MAS1_V) == 0);
    320 	tlb1->tlb1_entries[slot].e_hwtlb.hwtlb_mas0 =
    321 	    MAS0_TLBSEL_TLB1 | __SHIFTOUT(slot, MAS0_ESEL);
    322 	return (int)slot;
    323 }
    324 
    325 static void
    326 e500_free_tlb1_entry(struct e500_xtlb *xtlb, u_int slot, bool needs_sync)
    327 {
    328 	struct e500_tlb1 * const tlb1 = &e500_tlb1;
    329 	KASSERT(slot < tlb1->tlb1_numentries);
    330 	KASSERT(&tlb1->tlb1_entries[slot] == xtlb);
    331 
    332 	KASSERT(xtlb->e_hwtlb.hwtlb_mas0 == (MAS0_TLBSEL_TLB1|__SHIFTIN(slot, MAS0_ESEL)));
    333 	xtlb->e_hwtlb.hwtlb_mas1 &= ~(MAS1_V|MAS1_IPROT);
    334 	hwtlb_write(xtlb->e_hwtlb, needs_sync);
    335 
    336 	const register_t msr = wrtee(0);
    337 	tlb1->tlb1_freelist[tlb1->tlb1_numfree++] = slot;
    338 	wrtee(msr);
    339 }
    340 
    341 static tlb_asid_t
    342 e500_tlb_get_asid(void)
    343 {
    344 	return mfspr(SPR_PID0);
    345 }
    346 
    347 static void
    348 e500_tlb_set_asid(tlb_asid_t asid)
    349 {
    350 	mtspr(SPR_PID0, asid);
    351 }
    352 
    353 static void
    354 e500_tlb_invalidate_all(void)
    355 {
    356 	/*
    357 	 * This does a flash invalidate of all entries in TLB0.
    358 	 * We don't touch TLB1 since we don't expect those to be volatile.
    359 	 */
    360 #if 1
    361 	__asm volatile("tlbivax\t0, %0" :: "b"(4));	/* INV_ALL */
    362 #else
    363 	mtspr(SPR_MMUCSR0, MMUCSR0_TLB0_FL);
    364 	while (mfspr(SPR_MMUCSR0) != 0)
    365 		;
    366 #endif
    367 }
    368 
    369 static void
    370 e500_tlb_invalidate_globals(void)
    371 {
    372 	const size_t tlbassoc = TLBCFG_ASSOC(mftlb0cfg());
    373 	const size_t tlbentries = TLBCFG_NENTRY(mftlb0cfg());
    374 	const size_t max_epn = (tlbentries / tlbassoc) << PAGE_SHIFT;
    375 	const vaddr_t kstack_lo = (uintptr_t)curlwp->l_addr;
    376 	const vaddr_t kstack_hi = kstack_lo + USPACE - 1;
    377 	const vaddr_t epn_kstack_lo = kstack_lo & (max_epn - 1);
    378 	const vaddr_t epn_kstack_hi = kstack_hi & (max_epn - 1);
    379 
    380 	const register_t msr = wrtee(0);
    381 	for (size_t assoc = 0; assoc < tlbassoc; assoc++) {
    382 		mtspr(SPR_MAS0, MAS0_ESEL_MAKE(assoc) | MAS0_TLBSEL_TLB0);
    383 		for (size_t epn = 0; epn < max_epn; epn += PAGE_SIZE) {
    384 			mtspr(SPR_MAS2, epn);
    385 			__asm volatile("tlbre");
    386 			uint32_t mas1 = mfspr(SPR_MAS1);
    387 
    388 			/*
    389 			 * Make sure this is a valid kernel entry first.
    390 			 */
    391 			if ((mas1 & (MAS1_V|MAS1_TID|MAS1_TS)) != MAS1_V)
    392 				continue;
    393 
    394 			/*
    395 			 * We have a valid kernel TLB entry.  But if it matches
    396 			 * the stack we are currently running on, it would
    397 			 * unwise to invalidate it.  First see if the epn
    398 			 * overlaps the stack.  If it does then get the
    399 			 * VA and see if it really is part of the stack.
    400 			 */
    401 			if (epn_kstack_lo < epn_kstack_hi
    402 			    ? (epn_kstack_lo <= epn && epn <= epn_kstack_hi)
    403 			    : (epn <= epn_kstack_hi || epn_kstack_lo <= epn)) {
    404 				const uint32_t mas2_epn =
    405 				    mfspr(SPR_MAS2) & MAS2_EPN;
    406 				if (kstack_lo <= mas2_epn
    407 				    && mas2_epn <= kstack_hi)
    408 					continue;
    409 			}
    410 			mtspr(SPR_MAS1, mas1 ^ MAS1_V);
    411 			__asm volatile("tlbwe");
    412 		}
    413 	}
    414 	__asm volatile("isync");
    415 	wrtee(msr);
    416 }
    417 
    418 static void
    419 e500_tlb_invalidate_asids(tlb_asid_t asid_lo, tlb_asid_t asid_hi)
    420 {
    421 	const size_t tlbassoc = TLBCFG_ASSOC(mftlb0cfg());
    422 	const size_t tlbentries = TLBCFG_NENTRY(mftlb0cfg());
    423 	const size_t max_epn = (tlbentries / tlbassoc) << PAGE_SHIFT;
    424 
    425 	asid_lo = __SHIFTIN(asid_lo, MAS1_TID);
    426 	asid_hi = __SHIFTIN(asid_hi, MAS1_TID);
    427 
    428 	const register_t msr = wrtee(0);
    429 	for (size_t assoc = 0; assoc < tlbassoc; assoc++) {
    430 		mtspr(SPR_MAS0, MAS0_ESEL_MAKE(assoc) | MAS0_TLBSEL_TLB0);
    431 		for (size_t epn = 0; epn < max_epn; epn += PAGE_SIZE) {
    432 			mtspr(SPR_MAS2, epn);
    433 			__asm volatile("tlbre");
    434 			const uint32_t mas1 = mfspr(SPR_MAS1);
    435 			/*
    436 			 * If this is a valid entry for AS space 1 and
    437 			 * its asid matches the constraints of the caller,
    438 			 * clear its valid bit.
    439 			 */
    440 			if ((mas1 & (MAS1_V|MAS1_TS)) == (MAS1_V|MAS1_TS)
    441 			    && asid_lo <= (mas1 & MAS1_TID)
    442 			    && (mas1 & MAS1_TID) <= asid_hi) {
    443 				mtspr(SPR_MAS1, mas1 ^ MAS1_V);
    444 #if 0
    445 				printf("%s[%zu,%zu]->[%x]\n",
    446 				    __func__, assoc, epn, mas1);
    447 #endif
    448 				__asm volatile("tlbwe");
    449 			}
    450 		}
    451 	}
    452 	__asm volatile("isync");
    453 	wrtee(msr);
    454 }
    455 
    456 static u_int
    457 e500_tlb_record_asids(u_long *bitmap)
    458 {
    459 	const size_t tlbassoc = TLBCFG_ASSOC(mftlb0cfg());
    460 	const size_t tlbentries = TLBCFG_NENTRY(mftlb0cfg());
    461 	const size_t max_epn = (tlbentries / tlbassoc) << PAGE_SHIFT;
    462 	const size_t nbits = 8 * sizeof(bitmap[0]);
    463 	u_int found = 0;
    464 
    465 	const register_t msr = wrtee(0);
    466 	for (size_t assoc = 0; assoc < tlbassoc; assoc++) {
    467 		mtspr(SPR_MAS0, MAS0_ESEL_MAKE(assoc) | MAS0_TLBSEL_TLB0);
    468 		for (size_t epn = 0; epn < max_epn; epn += PAGE_SIZE) {
    469 			mtspr(SPR_MAS2, epn);
    470 			__asm volatile("tlbre");
    471 			const uint32_t mas1 = mfspr(SPR_MAS1);
    472 			/*
    473 			 * If this is a valid entry for AS space 1 and
    474 			 * its asid matches the constraints of the caller,
    475 			 * clear its valid bit.
    476 			 */
    477 			if ((mas1 & (MAS1_V|MAS1_TS)) == (MAS1_V|MAS1_TS)) {
    478 				const uint32_t asid = MASX_TID_GET(mas1);
    479 				const u_int i = asid / nbits;
    480 				const u_long mask = 1UL << (asid & (nbits - 1));
    481 				if ((bitmap[i] & mask) == 0) {
    482 					bitmap[i] |= mask;
    483 					found++;
    484 				}
    485 			}
    486 		}
    487 	}
    488 	wrtee(msr);
    489 
    490 	return found;
    491 }
    492 
    493 static void
    494 e500_tlb_invalidate_addr(vaddr_t va, tlb_asid_t asid)
    495 {
    496 	KASSERT((va & PAGE_MASK) == 0);
    497 	/*
    498 	 * Bits 60 & 61 have meaning
    499 	 */
    500 	__asm volatile("tlbivax\t0, %0" :: "b"(va));
    501 	__asm volatile("tlbsync");
    502 	__asm volatile("tlbsync");
    503 }
    504 
    505 static bool
    506 e500_tlb_update_addr(vaddr_t va, tlb_asid_t asid, pt_entry_t pte, bool insert)
    507 {
    508 	struct e500_hwtlb hwtlb = tlb_to_hwtlb(
    509 	    (struct e500_tlb){ .tlb_va = va, .tlb_asid = asid,
    510 		.tlb_size = PAGE_SIZE, .tlb_pte = pte,});
    511 
    512 	register_t msr = wrtee(0);
    513 	mtspr(SPR_MAS6, asid ? __SHIFTIN(asid, MAS6_SPID0) | MAS6_SAS : 0);
    514 	__asm volatile("tlbsx 0, %0" :: "b"(va));
    515 	register_t mas1 = mfspr(SPR_MAS1);
    516 	if ((mas1 & MAS1_V) == 0) {
    517 		if (!insert) {
    518 			wrtee(msr);
    519 #if 0
    520 			printf("%s(%#lx,%#x,%#x,%x)<no update>\n",
    521 			    __func__, va, asid, pte, insert);
    522 #endif
    523 			return false;
    524 		}
    525 		mtspr(SPR_MAS1, hwtlb.hwtlb_mas1);
    526 	}
    527 	mtspr(SPR_MAS2, hwtlb.hwtlb_mas2);
    528 	mtspr(SPR_MAS3, hwtlb.hwtlb_mas3);
    529 	__asm volatile("tlbwe");
    530 	if (asid == 0)
    531 		__asm volatile("isync");
    532 	wrtee(msr);
    533 #if 0
    534 	if (asid)
    535 	printf("%s(%#lx,%#x,%#x,%x)->[%x,%x,%x]\n",
    536 	    __func__, va, asid, pte, insert,
    537 	    hwtlb.hwtlb_mas1, hwtlb.hwtlb_mas2, hwtlb.hwtlb_mas3);
    538 #endif
    539 	return (mas1 & MAS1_V) != 0;
    540 }
    541 
    542 static void
    543 e500_tlb_write_entry(size_t index, const struct tlbmask *tlb)
    544 {
    545 }
    546 
    547 static void
    548 e500_tlb_read_entry(size_t index, struct tlbmask *tlb)
    549 {
    550 }
    551 
    552 static void
    553 e500_tlb_dump(void (*pr)(const char *, ...))
    554 {
    555 	const size_t tlbassoc = TLBCFG_ASSOC(mftlb0cfg());
    556 	const size_t tlbentries = TLBCFG_NENTRY(mftlb0cfg());
    557 	const size_t max_epn = (tlbentries / tlbassoc) << PAGE_SHIFT;
    558 	const uint32_t saved_mas0 = mfspr(SPR_MAS0);
    559 	size_t valid = 0;
    560 
    561 	if (pr == NULL)
    562 		pr = printf;
    563 
    564 	const register_t msr = wrtee(0);
    565 	for (size_t assoc = 0; assoc < tlbassoc; assoc++) {
    566 		struct e500_hwtlb hwtlb;
    567 		hwtlb.hwtlb_mas0 = MAS0_ESEL_MAKE(assoc) | MAS0_TLBSEL_TLB0;
    568 		mtspr(SPR_MAS0, hwtlb.hwtlb_mas0);
    569 		for (size_t epn = 0; epn < max_epn; epn += PAGE_SIZE) {
    570 			mtspr(SPR_MAS2, epn);
    571 			__asm volatile("tlbre");
    572 			hwtlb.hwtlb_mas1 = mfspr(SPR_MAS1);
    573 			/*
    574 			 * If this is a valid entry for AS space 1 and
    575 			 * its asid matches the constraints of the caller,
    576 			 * clear its valid bit.
    577 			 */
    578 			if (hwtlb.hwtlb_mas1 & MAS1_V) {
    579 				hwtlb.hwtlb_mas2 = mfspr(SPR_MAS2);
    580 				hwtlb.hwtlb_mas3 = mfspr(SPR_MAS3);
    581 				struct e500_tlb tlb = hwtlb_to_tlb(hwtlb);
    582 				(*pr)("[%zu,%zu]->[%x,%x,%x]",
    583 				    assoc, atop(epn),
    584 				    hwtlb.hwtlb_mas1,
    585 				    hwtlb.hwtlb_mas2,
    586 				    hwtlb.hwtlb_mas3);
    587 				(*pr)(": VA=%#lx size=4KB asid=%u pte=%x",
    588 				    tlb.tlb_va, tlb.tlb_asid, tlb.tlb_pte);
    589 				(*pr)(" (RPN=%#x,%s%s%s%s%s,%s%s%s%s%s)\n",
    590 				    tlb.tlb_pte & PTE_RPN_MASK,
    591 				    tlb.tlb_pte & PTE_xR ? "R" : "",
    592 				    tlb.tlb_pte & PTE_xW ? "W" : "",
    593 				    tlb.tlb_pte & PTE_UNMODIFIED ? "*" : "",
    594 				    tlb.tlb_pte & PTE_xX ? "X" : "",
    595 				    tlb.tlb_pte & PTE_UNSYNCED ? "*" : "",
    596 				    tlb.tlb_pte & PTE_W ? "W" : "",
    597 				    tlb.tlb_pte & PTE_I ? "I" : "",
    598 				    tlb.tlb_pte & PTE_M ? "M" : "",
    599 				    tlb.tlb_pte & PTE_G ? "G" : "",
    600 				    tlb.tlb_pte & PTE_E ? "E" : "");
    601 				valid++;
    602 			}
    603 		}
    604 	}
    605 	mtspr(SPR_MAS0, saved_mas0);
    606 	wrtee(msr);
    607 	(*pr)("%s: %zu valid entries\n", __func__, valid);
    608 }
    609 
    610 static void
    611 e500_tlb_walk(void *ctx, bool (*func)(void *, vaddr_t, uint32_t, uint32_t))
    612 {
    613 	const size_t tlbassoc = TLBCFG_ASSOC(mftlb0cfg());
    614 	const size_t tlbentries = TLBCFG_NENTRY(mftlb0cfg());
    615 	const size_t max_epn = (tlbentries / tlbassoc) << PAGE_SHIFT;
    616 	const uint32_t saved_mas0 = mfspr(SPR_MAS0);
    617 
    618 	const register_t msr = wrtee(0);
    619 	for (size_t assoc = 0; assoc < tlbassoc; assoc++) {
    620 		struct e500_hwtlb hwtlb;
    621 		hwtlb.hwtlb_mas0 = MAS0_ESEL_MAKE(assoc) | MAS0_TLBSEL_TLB0;
    622 		mtspr(SPR_MAS0, hwtlb.hwtlb_mas0);
    623 		for (size_t epn = 0; epn < max_epn; epn += PAGE_SIZE) {
    624 			mtspr(SPR_MAS2, epn);
    625 			__asm volatile("tlbre");
    626 			hwtlb.hwtlb_mas1 = mfspr(SPR_MAS1);
    627 			/*
    628 			 * If this is a valid entry for AS space 1 and
    629 			 * its asid matches the constraints of the caller,
    630 			 * clear its valid bit.
    631 			 */
    632 			if (hwtlb.hwtlb_mas1 & MAS1_V) {
    633 				hwtlb.hwtlb_mas2 = mfspr(SPR_MAS2);
    634 				hwtlb.hwtlb_mas3 = mfspr(SPR_MAS3);
    635 				struct e500_tlb tlb = hwtlb_to_tlb(hwtlb);
    636 				if (!(*func)(ctx, tlb.tlb_va, tlb.tlb_asid,
    637 				    tlb.tlb_pte))
    638 					break;
    639 			}
    640 		}
    641 	}
    642 	mtspr(SPR_MAS0, saved_mas0);
    643 	wrtee(msr);
    644 }
    645 
    646 static struct e500_xtlb *
    647 e500_tlb_lookup_xtlb(vaddr_t va, u_int *slotp)
    648 {
    649 	struct e500_tlb1 * const tlb1 = &e500_tlb1;
    650 	struct e500_xtlb *xtlb = tlb1->tlb1_entries;
    651 
    652 	/*
    653 	 * See if we have a TLB entry for the pa.
    654 	 */
    655 	for (u_int i = 0; i < tlb1->tlb1_numentries; i++, xtlb++) {
    656 		if ((xtlb->e_hwtlb.hwtlb_mas1 & MAS1_V)
    657 		    && xtlb->e_tlb.tlb_va <= va
    658 		    && va < xtlb->e_tlb.tlb_va + xtlb->e_tlb.tlb_size) {
    659 			if (slotp != NULL)
    660 				*slotp = i;
    661 			return xtlb;
    662 		}
    663 	}
    664 
    665 	return NULL;
    666 }
    667 
    668 static struct e500_xtlb *
    669 e500_tlb_lookup_xtlb2(vaddr_t va, vsize_t len)
    670 {
    671 	struct e500_tlb1 * const tlb1 = &e500_tlb1;
    672 	struct e500_xtlb *xtlb = tlb1->tlb1_entries;
    673 
    674 	/*
    675 	 * See if we have a TLB entry for the pa.
    676 	 */
    677 	for (u_int i = 0; i < tlb1->tlb1_numentries; i++, xtlb++) {
    678 		if ((xtlb->e_hwtlb.hwtlb_mas1 & MAS1_V)
    679 		    && xtlb->e_tlb.tlb_va < va + len
    680 		    && va < xtlb->e_tlb.tlb_va + xtlb->e_tlb.tlb_size) {
    681 			return xtlb;
    682 		}
    683 	}
    684 
    685 	return NULL;
    686 }
    687 
    688 static void *
    689 e500_tlb_mapiodev(paddr_t pa, psize_t len, bool prefetchable)
    690 {
    691 	struct e500_xtlb * const xtlb = e500_tlb_lookup_xtlb(pa, NULL);
    692 
    693 	/*
    694 	 * See if we have a TLB entry for the pa.  If completely falls within
    695 	 * mark the reference and return the pa.  But only if the tlb entry
    696 	 * is not cacheable.
    697 	 */
    698 	if (xtlb
    699 	    && pa + len <= xtlb->e_tlb.tlb_va + xtlb->e_tlb.tlb_size
    700 	    && (prefetchable
    701 		|| (xtlb->e_tlb.tlb_pte & PTE_WIG) == (PTE_I|PTE_G))) {
    702 		xtlb->e_refcnt++;
    703 		return (void *) pa;
    704 	}
    705 	return NULL;
    706 }
    707 
    708 static void
    709 e500_tlb_unmapiodev(vaddr_t va, vsize_t len)
    710 {
    711 	if (va < VM_MIN_KERNEL_ADDRESS || VM_MAX_KERNEL_ADDRESS <= va) {
    712 		struct e500_xtlb * const xtlb = e500_tlb_lookup_xtlb(va, NULL);
    713 		if (xtlb)
    714 			xtlb->e_refcnt--;
    715 	}
    716 }
    717 
    718 static int
    719 e500_tlb_ioreserve(vaddr_t va, vsize_t len, pt_entry_t pte)
    720 {
    721 	struct e500_tlb1 * const tlb1 = &e500_tlb1;
    722 	struct e500_xtlb *xtlb;
    723 
    724 	KASSERT(len & 0x55555000);
    725 	KASSERT((len & ~0x55555000) == 0);
    726 	KASSERT(len >= PAGE_SIZE);
    727 	KASSERT((len & (len - 1)) == 0);
    728 	KASSERT((va & (len - 1)) == 0);
    729 	KASSERT((pte & (len - 1)) == 0);
    730 
    731 	if ((xtlb = e500_tlb_lookup_xtlb2(va, len)) != NULL) {
    732 		if (va < xtlb->e_tlb.tlb_va
    733 		    || xtlb->e_tlb.tlb_va + xtlb->e_tlb.tlb_size < va + len
    734 		    || va - xtlb->e_tlb.tlb_va != pte - xtlb->e_tlb.tlb_pte)
    735 			return EBUSY;
    736 		xtlb->e_refcnt++;
    737 		return 0;
    738 	}
    739 
    740 	const int slot = e500_alloc_tlb1_entry();
    741 	if (slot < 0)
    742 		return ENOMEM;
    743 
    744 	xtlb = &tlb1->tlb1_entries[slot];
    745 	xtlb->e_tlb.tlb_va = va;
    746 	xtlb->e_tlb.tlb_size = len;
    747 	xtlb->e_tlb.tlb_pte = pte;
    748 	xtlb->e_tlb.tlb_asid = KERNEL_PID;
    749 
    750 	xtlb->e_hwtlb = tlb_to_hwtlb(xtlb->e_tlb);
    751 	xtlb->e_hwtlb.hwtlb_mas0 |= __SHIFTOUT(slot, MAS0_ESEL);
    752 	hwtlb_write(xtlb->e_hwtlb, true);
    753 	return 0;
    754 }
    755 
    756 static int
    757 e500_tlb_iorelease(vaddr_t va)
    758 {
    759 	u_int slot;
    760 	struct e500_xtlb * const xtlb = e500_tlb_lookup_xtlb(va, &slot);
    761 
    762 	if (xtlb == NULL)
    763 		return ENOENT;
    764 
    765 	if (xtlb->e_refcnt)
    766 		return EBUSY;
    767 
    768 	e500_free_tlb1_entry(xtlb, slot, true);
    769 
    770 	return 0;
    771 }
    772 
    773 static u_int
    774 e500_tlbmemmap(paddr_t memstart, psize_t memsize, struct e500_tlb1 *tlb1)
    775 {
    776 	u_int slotmask = 0;
    777 	u_int slots = 0, nextslot = 0;
    778 	KASSERT(tlb1->tlb1_numfree > 1);
    779 	KASSERT(((memstart + memsize - 1) & -memsize) == memstart);
    780 	for (paddr_t lastaddr = memstart; 0 < memsize; ) {
    781 		u_int cnt = __builtin_clz(memsize);
    782 		psize_t size = min(1UL << (31 - (cnt | 1)), tlb1->tlb1_maxsize);
    783 		slots += memsize / size;
    784 		if (slots > 4)
    785 			panic("%s: %d: can't map memory (%#lx) into TLB1: %s",
    786 			    __func__, __LINE__, memsize, "too fragmented");
    787 		if (slots > tlb1->tlb1_numfree - 1)
    788 			panic("%s: %d: can't map memory (%#lx) into TLB1: %s",
    789 			    __func__, __LINE__, memsize,
    790 			    "insufficent TLB entries");
    791 		for (; nextslot < slots; nextslot++) {
    792 			const u_int freeslot = e500_alloc_tlb1_entry();
    793 			struct e500_xtlb * const xtlb =
    794 			    &tlb1->tlb1_entries[freeslot];
    795 			xtlb->e_tlb.tlb_asid = KERNEL_PID;
    796 			xtlb->e_tlb.tlb_size = size;
    797 			xtlb->e_tlb.tlb_va = lastaddr;
    798 			xtlb->e_tlb.tlb_pte = lastaddr
    799 			    | PTE_M | PTE_xX | PTE_xW | PTE_xR;
    800 			lastaddr += size;
    801 			memsize -= size;
    802 			slotmask |= 1 << (31 - freeslot); /* clz friendly */
    803 		}
    804 	}
    805 
    806 	return nextslot;
    807 }
    808 static const struct tlb_md_ops e500_tlb_ops = {
    809 	.md_tlb_get_asid = e500_tlb_get_asid,
    810 	.md_tlb_set_asid = e500_tlb_set_asid,
    811 	.md_tlb_invalidate_all = e500_tlb_invalidate_all,
    812 	.md_tlb_invalidate_globals = e500_tlb_invalidate_globals,
    813 	.md_tlb_invalidate_asids = e500_tlb_invalidate_asids,
    814 	.md_tlb_invalidate_addr = e500_tlb_invalidate_addr,
    815 	.md_tlb_update_addr = e500_tlb_update_addr,
    816 	.md_tlb_record_asids = e500_tlb_record_asids,
    817 	.md_tlb_write_entry = e500_tlb_write_entry,
    818 	.md_tlb_read_entry = e500_tlb_read_entry,
    819 	.md_tlb_dump = e500_tlb_dump,
    820 	.md_tlb_walk = e500_tlb_walk,
    821 };
    822 
    823 static const struct tlb_md_io_ops e500_tlb_io_ops = {
    824 	.md_tlb_mapiodev = e500_tlb_mapiodev,
    825 	.md_tlb_unmapiodev = e500_tlb_unmapiodev,
    826 	.md_tlb_ioreserve = e500_tlb_ioreserve,
    827 	.md_tlb_iorelease = e500_tlb_iorelease,
    828 };
    829 
    830 void
    831 e500_tlb_init(vaddr_t endkernel, psize_t memsize)
    832 {
    833 	struct e500_tlb1 * const tlb1 = &e500_tlb1;
    834 
    835 #if 0
    836 	register_t mmucfg = mfspr(SPR_MMUCFG);
    837 	register_t mas4 = mfspr(SPR_MAS4);
    838 #endif
    839 
    840 	const uint32_t tlb1cfg = mftlb1cfg();
    841 	tlb1->tlb1_numentries = TLBCFG_NENTRY(tlb1cfg);
    842 	KASSERT(tlb1->tlb1_numentries <= __arraycount(tlb1->tlb1_entries));
    843 	/*
    844 	 * Limit maxsize to 1G since 4G isn't really useful to us.
    845 	 */
    846 	tlb1->tlb1_minsize = 1024 << (2 * TLBCFG_MINSIZE(tlb1cfg));
    847 	tlb1->tlb1_maxsize = 1024 << (2 * min(10, TLBCFG_MAXSIZE(tlb1cfg)));
    848 
    849 #ifdef VERBOSE_INITPPC
    850 	printf(" tlb1cfg=%#x numentries=%u minsize=%#xKB maxsize=%#xKB",
    851 	    tlb1cfg, tlb1->tlb1_numentries, tlb1->tlb1_minsize >> 10,
    852 	    tlb1->tlb1_maxsize >> 10);
    853 #endif
    854 
    855 	/*
    856 	 * Let's see what's in TLB1 and we need to invalidate any entry that
    857 	 * would fit within the kernel's mapped address space.
    858 	 */
    859 	psize_t memmapped = 0;
    860 	for (u_int i = 0; i < tlb1->tlb1_numentries; i++) {
    861 		struct e500_xtlb * const xtlb = &tlb1->tlb1_entries[i];
    862 
    863 		xtlb->e_hwtlb = hwtlb_read(MAS0_TLBSEL_TLB1, i);
    864 
    865 		if ((xtlb->e_hwtlb.hwtlb_mas1 & MAS1_V) == 0) {
    866 			tlb1->tlb1_freelist[tlb1->tlb1_numfree++] = i;
    867 #ifdef VERBOSE_INITPPC
    868 			printf(" TLB1[%u]=<unused>", i);
    869 #endif
    870 			continue;
    871 		}
    872 
    873 		xtlb->e_tlb = hwtlb_to_tlb(xtlb->e_hwtlb);
    874 #ifdef VERBOSE_INITPPC
    875 		printf(" TLB1[%u]=<%#lx,%#lx,%#x,%#x>",
    876 		    i, xtlb->e_tlb.tlb_va, xtlb->e_tlb.tlb_size,
    877 		    xtlb->e_tlb.tlb_asid, xtlb->e_tlb.tlb_pte);
    878 #endif
    879 		if ((VM_MIN_KERNEL_ADDRESS <= xtlb->e_tlb.tlb_va
    880 		    && xtlb->e_tlb.tlb_va < VM_MAX_KERNEL_ADDRESS)
    881 		    || (xtlb->e_tlb.tlb_va < VM_MIN_KERNEL_ADDRESS
    882 		        && VM_MIN_KERNEL_ADDRESS <
    883 			   xtlb->e_tlb.tlb_va + xtlb->e_tlb.tlb_size)) {
    884 #ifdef VERBOSE_INITPPC
    885 			printf("free");
    886 #endif
    887 			e500_free_tlb1_entry(xtlb, i, false);
    888 #ifdef VERBOSE_INITPPC
    889 			printf("d");
    890 #endif
    891 			continue;
    892 		}
    893 		if ((xtlb->e_hwtlb.hwtlb_mas1 & MAS1_IPROT) == 0) {
    894 			xtlb->e_hwtlb.hwtlb_mas1 |= MAS1_IPROT;
    895 			hwtlb_write(xtlb->e_hwtlb, false);
    896 #ifdef VERBOSE_INITPPC
    897 			printf("+iprot");
    898 #endif
    899 		}
    900 		if (xtlb->e_tlb.tlb_pte & PTE_I)
    901 			continue;
    902 
    903 		if (xtlb->e_tlb.tlb_va == 0
    904 		    || xtlb->e_tlb.tlb_va + xtlb->e_tlb.tlb_size <= memsize) {
    905 			memmapped += xtlb->e_tlb.tlb_size;
    906 		}
    907 	}
    908 
    909 	cpu_md_ops.md_tlb_ops = &e500_tlb_ops;
    910 	cpu_md_ops.md_tlb_io_ops = &e500_tlb_io_ops;
    911 
    912 	if (__predict_false(memmapped < memsize)) {
    913 		/*
    914 		 * Let's see how many TLB entries are needed to map memory.
    915 		 */
    916 		u_int slotmask = e500_tlbmemmap(0, memsize, tlb1);
    917 
    918 		/*
    919 		 * To map main memory into the TLB, we need to flush any
    920 		 * existing entries from the TLB that overlap the virtual
    921 		 * address space needed to map physical memory.  That may
    922 		 * include the entries for the pages currently used by the
    923 		 * stack or that we are executing.  So to avoid problems, we
    924 		 * are going to temporarily map the kernel and stack into AS 1,
    925 		 * switch to it, and clear out the TLB entries from AS 0,
    926 		 * install the new TLB entries to map memory, and then switch
    927 		 * back to AS 0 and free the temp entry used for AS1.
    928 		 */
    929 		u_int b = __builtin_clz(endkernel);
    930 
    931 		/*
    932 		 * If the kernel doesn't end on a clean power of 2, we need
    933 		 * to round the size up (by decrementing the number of leading
    934 		 * zero bits).  If the size isn't a power of 4KB, decrement
    935 		 * again to make it one.
    936 		 */
    937 		if (endkernel & (endkernel - 1))
    938 			b--;
    939 		if ((b & 1) == 0)
    940 			b--;
    941 
    942 		/*
    943 		 * Create a TLB1 mapping for the kernel in AS1.
    944 		 */
    945 		const u_int kslot = e500_alloc_tlb1_entry();
    946 		struct e500_xtlb * const kxtlb = &tlb1->tlb1_entries[kslot];
    947 		kxtlb->e_tlb.tlb_va = 0;
    948 		kxtlb->e_tlb.tlb_size = 1UL << (31 - b);
    949 		kxtlb->e_tlb.tlb_pte = PTE_M|PTE_xR|PTE_xW|PTE_xX;
    950 		kxtlb->e_tlb.tlb_asid = KERNEL_PID;
    951 
    952 		kxtlb->e_hwtlb = tlb_to_hwtlb(kxtlb->e_tlb);
    953 		kxtlb->e_hwtlb.hwtlb_mas0 |= __SHIFTOUT(kslot, MAS0_ESEL);
    954 		kxtlb->e_hwtlb.hwtlb_mas1 |= MAS1_TS;
    955 		hwtlb_write(kxtlb->e_hwtlb, true);
    956 
    957 		/*
    958 		 * Now that we have a TLB mapping in AS1 for the kernel and its
    959 		 * stack, we switch to AS1 to cleanup the TLB mappings for TLB0.
    960 		 */
    961 		const register_t saved_msr = mfmsr();
    962 		mtmsr(saved_msr | PSL_DS | PSL_IS);
    963 		__asm volatile("isync");
    964 
    965 		/*
    966 		 *** Invalidate all the TLB0 entries.
    967 		 */
    968 		e500_tlb_invalidate_all();
    969 
    970 		/*
    971 		 *** Now let's see if we have any entries in TLB1 that would
    972 		 *** overlap the ones we are about to install.  If so, nuke 'em.
    973 		 */
    974 		for (u_int i = 0; i < tlb1->tlb1_numentries; i++) {
    975 			struct e500_xtlb * const xtlb = &tlb1->tlb1_entries[i];
    976 			struct e500_hwtlb * const hwtlb = &xtlb->e_hwtlb;
    977 			if ((hwtlb->hwtlb_mas1 & (MAS1_V|MAS1_TS)) == MAS1_V
    978 			    && (hwtlb->hwtlb_mas2 & MAS2_EPN) < memsize) {
    979 				e500_free_tlb1_entry(xtlb, i, false);
    980 			}
    981 		}
    982 
    983 		/*
    984 		 *** Now we can add the TLB entries that will map physical
    985 		 *** memory.  If bit 0 [MSB] in slotmask is set, then tlb
    986 		 *** entry 0 contains a mapping for physical memory...
    987 		 */
    988 		struct e500_xtlb *entries = tlb1->tlb1_entries;
    989 		while (slotmask != 0) {
    990 			const u_int slot = __builtin_clz(slotmask);
    991 			hwtlb_write(entries[slot].e_hwtlb, false);
    992 			entries += slot + 1;
    993 			slotmask <<= slot + 1;
    994 		}
    995 
    996 		/*
    997 		 *** Synchronize the TLB and the instruction stream.
    998 		 */
    999 		__asm volatile("tlbsync");
   1000 		__asm volatile("isync");
   1001 
   1002 		/*
   1003 		 *** Switch back to AS 0.
   1004 		 */
   1005 		mtmsr(saved_msr);
   1006 		__asm volatile("isync");
   1007 
   1008 		/*
   1009 		 * Free the temporary TLB1 entry.
   1010 		 */
   1011 		e500_free_tlb1_entry(kxtlb, kslot, true);
   1012 	}
   1013 
   1014 	/*
   1015 	 * Finally set the MAS4 defaults.
   1016 	 */
   1017 	mtspr(SPR_MAS4, MAS4_TSIZED_4KB | MAS4_MD);
   1018 
   1019 	/*
   1020 	 * Invalidate all the TLB0 entries.
   1021 	 */
   1022 	e500_tlb_invalidate_all();
   1023 }
   1024 
   1025 void
   1026 e500_tlb_minimize(vaddr_t endkernel)
   1027 {
   1028 #ifdef PMAP_MINIMALTLB
   1029 	struct e500_tlb1 * const tlb1 = &e500_tlb1;
   1030 	extern uint32_t _fdata[];
   1031 
   1032 	u_int slot;
   1033 
   1034 	paddr_t boot_page = cpu_read_4(GUR_BPTR);
   1035 	if (boot_page & BPTR_EN) {
   1036 		/*
   1037 		 * shift it to an address
   1038 		 */
   1039 		boot_page = (boot_page & BPTR_BOOT_PAGE) << PAGE_SHIFT;
   1040 		pmap_kvptefill(boot_page, boot_page + NBPG,
   1041 		    PTE_M | PTE_xR | PTE_xW | PTE_xX);
   1042 	}
   1043 
   1044 
   1045 	KASSERT(endkernel - (uintptr_t)_fdata < 0x400000);
   1046 	KASSERT((uintptr_t)_fdata == 0x400000);
   1047 
   1048 	struct e500_xtlb *xtlb = e500_tlb_lookup_xtlb(endkernel, &slot);
   1049 
   1050 	KASSERT(xtlb == e500_tlb_lookup_xtlb2(0, endkernel));
   1051 	const u_int tmp_slot = e500_alloc_tlb1_entry();
   1052 	KASSERT(tmp_slot != (u_int) -1);
   1053 
   1054 	struct e500_xtlb * const tmp_xtlb = &tlb1->tlb1_entries[tmp_slot];
   1055 	tmp_xtlb->e_tlb = xtlb->e_tlb;
   1056 	tmp_xtlb->e_hwtlb = tlb_to_hwtlb(tmp_xtlb->e_tlb);
   1057 	tmp_xtlb->e_hwtlb.hwtlb_mas1 |= MAS1_TS;
   1058 	KASSERT((tmp_xtlb->e_hwtlb.hwtlb_mas0 & MAS0_TLBSEL) == MAS0_TLBSEL_TLB1);
   1059 	tmp_xtlb->e_hwtlb.hwtlb_mas0 |= __SHIFTIN(tmp_slot, MAS0_ESEL);
   1060 	hwtlb_write(tmp_xtlb->e_hwtlb, true);
   1061 
   1062 	const u_int text_slot = e500_alloc_tlb1_entry();
   1063 	KASSERT(text_slot != (u_int)-1);
   1064 	struct e500_xtlb * const text_xtlb = &tlb1->tlb1_entries[text_slot];
   1065 	text_xtlb->e_tlb.tlb_va = 0;
   1066 	text_xtlb->e_tlb.tlb_size = 0x400000;
   1067 	text_xtlb->e_tlb.tlb_pte = PTE_M | PTE_xR | PTE_xX | text_xtlb->e_tlb.tlb_va;
   1068 	text_xtlb->e_tlb.tlb_asid = 0;
   1069 	text_xtlb->e_hwtlb = tlb_to_hwtlb(text_xtlb->e_tlb);
   1070 	KASSERT((text_xtlb->e_hwtlb.hwtlb_mas0 & MAS0_TLBSEL) == MAS0_TLBSEL_TLB1);
   1071 	text_xtlb->e_hwtlb.hwtlb_mas0 |= __SHIFTIN(text_slot, MAS0_ESEL);
   1072 
   1073 	const u_int data_slot = e500_alloc_tlb1_entry();
   1074 	KASSERT(data_slot != (u_int)-1);
   1075 	struct e500_xtlb * const data_xtlb = &tlb1->tlb1_entries[data_slot];
   1076 	data_xtlb->e_tlb.tlb_va = 0x400000;
   1077 	data_xtlb->e_tlb.tlb_size = 0x400000;
   1078 	data_xtlb->e_tlb.tlb_pte = PTE_M | PTE_xR | PTE_xW | data_xtlb->e_tlb.tlb_va;
   1079 	data_xtlb->e_tlb.tlb_asid = 0;
   1080 	data_xtlb->e_hwtlb = tlb_to_hwtlb(data_xtlb->e_tlb);
   1081 	KASSERT((data_xtlb->e_hwtlb.hwtlb_mas0 & MAS0_TLBSEL) == MAS0_TLBSEL_TLB1);
   1082 	data_xtlb->e_hwtlb.hwtlb_mas0 |= __SHIFTIN(data_slot, MAS0_ESEL);
   1083 
   1084 	const register_t msr = mfmsr();
   1085 	const register_t ts_msr = (msr | PSL_DS | PSL_IS) & ~PSL_EE;
   1086 
   1087 	__asm __volatile(
   1088 		"mtmsr	%[ts_msr]"	"\n\t"
   1089 		"sync"			"\n\t"
   1090 		"isync"
   1091 	    ::	[ts_msr] "r" (ts_msr));
   1092 
   1093 #if 0
   1094 	hwtlb_write(text_xtlb->e_hwtlb, false);
   1095 	hwtlb_write(data_xtlb->e_hwtlb, false);
   1096 	e500_free_tlb1_entry(xtlb, slot, true);
   1097 #endif
   1098 
   1099 	__asm __volatile(
   1100 		"mtmsr	%[msr]"		"\n\t"
   1101 		"sync"			"\n\t"
   1102 		"isync"
   1103 	    ::	[msr] "r" (msr));
   1104 
   1105 	e500_free_tlb1_entry(tmp_xtlb, tmp_slot, true);
   1106 #endif	/* PMAP_MINIMALTLB */
   1107 }
   1108