e500_tlb.c revision 1.7.6.1 1 /* $NetBSD: e500_tlb.c,v 1.7.6.1 2012/04/05 21:33:17 mrg Exp $ */
2 /*-
3 * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
4 * All rights reserved.
5 *
6 * This code is derived from software contributed to The NetBSD Foundation
7 * by Raytheon BBN Technologies Corp and Defense Advanced Research Projects
8 * Agency and which was developed by Matt Thomas of 3am Software Foundry.
9 *
10 * This material is based upon work supported by the Defense Advanced Research
11 * Projects Agency and Space and Naval Warfare Systems Center, Pacific, under
12 * Contract No. N66001-09-C-2073.
13 * Approved for Public Release, Distribution Unlimited
14 *
15 * Redistribution and use in source and binary forms, with or without
16 * modification, are permitted provided that the following conditions
17 * are met:
18 * 1. Redistributions of source code must retain the above copyright
19 * notice, this list of conditions and the following disclaimer.
20 * 2. Redistributions in binary form must reproduce the above copyright
21 * notice, this list of conditions and the following disclaimer in the
22 * documentation and/or other materials provided with the distribution.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
26 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
31 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
32 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
33 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
34 * POSSIBILITY OF SUCH DAMAGE.
35 */
36
37 #define __PMAP_PRIVATE
38
39 #include <sys/cdefs.h>
40
41 __KERNEL_RCSID(0, "$NetBSD: e500_tlb.c,v 1.7.6.1 2012/04/05 21:33:17 mrg Exp $");
42
43 #include <sys/param.h>
44
45 #include <uvm/uvm_extern.h>
46
47 #include <powerpc/spr.h>
48 #include <powerpc/booke/spr.h>
49 #include <powerpc/booke/cpuvar.h>
50 #include <powerpc/booke/e500reg.h>
51 #include <powerpc/booke/e500var.h>
52 #include <powerpc/booke/pmap.h>
53
54 struct e500_tlb {
55 vaddr_t tlb_va;
56 uint32_t tlb_pte;
57 uint32_t tlb_asid;
58 vsize_t tlb_size;
59 };
60
61 struct e500_hwtlb {
62 uint32_t hwtlb_mas0;
63 uint32_t hwtlb_mas1;
64 uint32_t hwtlb_mas2;
65 uint32_t hwtlb_mas3;
66 };
67
68 struct e500_xtlb {
69 struct e500_tlb e_tlb;
70 struct e500_hwtlb e_hwtlb;
71 u_long e_refcnt;
72 };
73
74 static struct e500_tlb1 {
75 uint32_t tlb1_maxsize;
76 uint32_t tlb1_minsize;
77 u_int tlb1_numentries;
78 u_int tlb1_numfree;
79 u_int tlb1_freelist[32];
80 struct e500_xtlb tlb1_entries[32];
81 } e500_tlb1;
82
83 static inline register_t mftlb0cfg(void) __pure;
84 static inline register_t mftlb1cfg(void) __pure;
85
86 static inline register_t
87 mftlb0cfg(void)
88 {
89 register_t tlb0cfg;
90 __asm("mfspr %0, %1" : "=r"(tlb0cfg) : "n"(SPR_TLB0CFG));
91 return tlb0cfg;
92 }
93
94 static inline register_t
95 mftlb1cfg(void)
96 {
97 register_t tlb1cfg;
98 __asm("mfspr %0, %1" : "=r"(tlb1cfg) : "n"(SPR_TLB1CFG));
99 return tlb1cfg;
100 }
101
102 static struct e500_tlb
103 hwtlb_to_tlb(const struct e500_hwtlb hwtlb)
104 {
105 struct e500_tlb tlb;
106 register_t prot_mask;
107 u_int prot_shift;
108
109 tlb.tlb_va = MAS2_EPN & hwtlb.hwtlb_mas2;
110 tlb.tlb_size = 1024 << (2 * MASX_TSIZE_GET(hwtlb.hwtlb_mas1));
111 tlb.tlb_asid = MASX_TID_GET(hwtlb.hwtlb_mas1);
112 tlb.tlb_pte = (hwtlb.hwtlb_mas2 & MAS2_WIMGE)
113 | (hwtlb.hwtlb_mas3 & MAS3_RPN);
114 if (hwtlb.hwtlb_mas1 & MAS1_TS) {
115 prot_mask = MAS3_UX|MAS3_UW|MAS3_UR;
116 prot_shift = PTE_RWX_SHIFT - 1;
117 } else {
118 prot_mask = MAS3_SX|MAS3_SW|MAS3_SR;
119 prot_shift = PTE_RWX_SHIFT;
120 }
121 tlb.tlb_pte |= (prot_mask & hwtlb.hwtlb_mas3) << prot_shift;
122 return tlb;
123 }
124
125 static inline struct e500_hwtlb
126 hwtlb_read(uint32_t mas0, u_int slot)
127 {
128 struct e500_hwtlb hwtlb;
129 register_t tlbcfg;
130
131 if (__predict_true(mas0 == MAS0_TLBSEL_TLB0)) {
132 tlbcfg = mftlb0cfg();
133 } else if (mas0 == MAS0_TLBSEL_TLB1) {
134 tlbcfg = mftlb1cfg();
135 } else {
136 panic("%s:%d: unexpected MAS0 %#" PRIx32,
137 __func__, __LINE__, mas0);
138 }
139
140 /*
141 * ESEL is the way we want to look up.
142 * If tlbassoc is the same as tlbentries (like in TLB1) then the TLB is
143 * fully associative, the entire slot is placed into ESEL. If tlbassoc
144 * is less then the number of tlb entries, the slot is split in two
145 * fields. Since the TLB is M rows by N ways, the lowers bits are for
146 * row (MAS2[EPN]) and the upper for the way (MAS1[ESEL]).
147 */
148 const u_int tlbassoc = TLBCFG_ASSOC(tlbcfg);
149 const u_int tlbentries = TLBCFG_NENTRY(tlbcfg);
150 const u_int esel_shift =
151 __builtin_clz(tlbassoc) - __builtin_clz(tlbentries);
152
153 /*
154 * Disable interrupts since we don't want anyone else mucking with
155 * the MMU Assist registers
156 */
157 const register_t msr = wrtee(0);
158 const register_t saved_mas0 = mfspr(SPR_MAS0);
159 mtspr(SPR_MAS0, mas0 | MAS0_ESEL_MAKE(slot >> esel_shift));
160
161 if (__predict_true(tlbassoc > tlbentries))
162 mtspr(SPR_MAS2, slot << PAGE_SHIFT);
163
164 /*
165 * Now select the entry and grab its contents.
166 */
167 __asm volatile("tlbre");
168
169 hwtlb.hwtlb_mas0 = mfspr(SPR_MAS0);
170 hwtlb.hwtlb_mas1 = mfspr(SPR_MAS1);
171 hwtlb.hwtlb_mas2 = mfspr(SPR_MAS2);
172 hwtlb.hwtlb_mas3 = mfspr(SPR_MAS3);
173
174 mtspr(SPR_MAS0, saved_mas0);
175 wrtee(msr); /* restore interrupts */
176
177 return hwtlb;
178 }
179
180 static inline void
181 hwtlb_write(const struct e500_hwtlb hwtlb, bool needs_sync)
182 {
183 const register_t msr = wrtee(0);
184 const uint32_t saved_mas0 = mfspr(SPR_MAS0);
185
186 /*
187 * Need to always write MAS0 and MAS1
188 */
189 mtspr(SPR_MAS0, hwtlb.hwtlb_mas0);
190 mtspr(SPR_MAS1, hwtlb.hwtlb_mas1);
191
192 /*
193 * Only write the VPN/WIMGE if this is in TLB0 or if a valid mapping.
194 */
195 if ((hwtlb.hwtlb_mas0 & MAS0_TLBSEL) == MAS0_TLBSEL_TLB0
196 || (hwtlb.hwtlb_mas1 & MAS1_V)) {
197 mtspr(SPR_MAS2, hwtlb.hwtlb_mas2);
198 }
199 /*
200 * Only need to write the RPN/prot if we are dealing with a valid
201 * mapping.
202 */
203 if (hwtlb.hwtlb_mas1 & MAS1_V) {
204 mtspr(SPR_MAS3, hwtlb.hwtlb_mas3);
205 }
206
207 #if 0
208 printf("%s->[%x,%x,%x,%x]\n",
209 __func__,
210 hwtlb.hwtlb_mas0, hwtlb.hwtlb_mas1,
211 hwtlb.hwtlb_mas2, hwtlb.hwtlb_mas3);
212 #endif
213 __asm volatile("tlbwe");
214 if (needs_sync) {
215 __asm volatile("tlbsync");
216 __asm volatile("isync");
217 }
218
219 mtspr(SPR_MAS0, saved_mas0);
220 wrtee(msr);
221 }
222
223 static struct e500_hwtlb
224 tlb_to_hwtlb(const struct e500_tlb tlb)
225 {
226 struct e500_hwtlb hwtlb;
227
228 KASSERT(trunc_page(tlb.tlb_va) == tlb.tlb_va);
229 KASSERT(tlb.tlb_size != 0);
230 KASSERT((tlb.tlb_size & (tlb.tlb_size - 1)) == 0);
231 const uint32_t prot_mask = tlb.tlb_pte & PTE_RWX_MASK;
232 if (__predict_true(tlb.tlb_size == PAGE_SIZE)) {
233 hwtlb.hwtlb_mas0 = 0;
234 hwtlb.hwtlb_mas1 = MAS1_V | MASX_TSIZE_MAKE(1);
235 /*
236 * A non-zero ASID means this is a user page so mark it as
237 * being in the user's address space.
238 */
239 if (tlb.tlb_asid) {
240 hwtlb.hwtlb_mas1 |= MAS1_TS
241 | MASX_TID_MAKE(tlb.tlb_asid);
242 hwtlb.hwtlb_mas3 = (prot_mask >> (PTE_RWX_SHIFT - 1))
243 | ((prot_mask & ~PTE_xX) >> PTE_RWX_SHIFT);
244 KASSERT(prot_mask & PTE_xR);
245 KASSERT(hwtlb.hwtlb_mas3 & MAS3_UR);
246 CTASSERT(MAS3_UR == (PTE_xR >> (PTE_RWX_SHIFT - 1)));
247 CTASSERT(MAS3_SR == (PTE_xR >> PTE_RWX_SHIFT));
248 } else {
249 hwtlb.hwtlb_mas3 = prot_mask >> PTE_RWX_SHIFT;
250 }
251 if (tlb.tlb_pte & PTE_UNMODIFIED)
252 hwtlb.hwtlb_mas3 &= ~(MAS3_UW|MAS3_SW);
253 if (tlb.tlb_pte & PTE_UNSYNCED)
254 hwtlb.hwtlb_mas3 &= ~(MAS3_UX|MAS3_SX);
255 } else {
256 KASSERT(tlb.tlb_asid == 0);
257 KASSERT((tlb.tlb_size & 0xaaaaa7ff) == 0);
258 u_int cntlz = __builtin_clz(tlb.tlb_size);
259 KASSERT(cntlz & 1);
260 KASSERT(cntlz <= 19);
261 hwtlb.hwtlb_mas0 = MAS0_TLBSEL_TLB1;
262 /*
263 * TSIZE is defined (4^TSIZE) Kbytes except a TSIZE of 0 is not
264 * allowed. So 1K would be 0x00000400 giving 21 leading zero
265 * bits. Subtracting the leading number of zero bits from 21
266 * and dividing by 2 gives us the number that the MMU wants.
267 */
268 hwtlb.hwtlb_mas1 = MASX_TSIZE_MAKE(((31 - 10) - cntlz) / 2)
269 | MAS1_IPROT | MAS1_V;
270 hwtlb.hwtlb_mas3 = prot_mask >> PTE_RWX_SHIFT;
271 }
272 /* We are done with MAS1, on to MAS2 ... */
273 hwtlb.hwtlb_mas2 = tlb.tlb_va | (tlb.tlb_pte & PTE_WIMGE_MASK);
274 hwtlb.hwtlb_mas3 |= tlb.tlb_pte & PTE_RPN_MASK;
275
276 return hwtlb;
277 }
278
279 void *
280 e500_tlb1_fetch(size_t slot)
281 {
282 struct e500_tlb1 * const tlb1 = &e500_tlb1;
283
284 return &tlb1->tlb1_entries[slot].e_hwtlb;
285 }
286
287 void
288 e500_tlb1_sync(void)
289 {
290 struct e500_tlb1 * const tlb1 = &e500_tlb1;
291 for (u_int slot = 1; slot < tlb1->tlb1_numentries; slot++) {
292 const struct e500_hwtlb * const new_hwtlb =
293 &tlb1->tlb1_entries[slot].e_hwtlb;
294 const struct e500_hwtlb old_hwtlb =
295 hwtlb_read(MAS0_TLBSEL_TLB1, slot);
296 #define CHANGED(n,o,f) ((n)->f != (o).f)
297 bool mas1_changed_p = CHANGED(new_hwtlb, old_hwtlb, hwtlb_mas1);
298 bool mas2_changed_p = CHANGED(new_hwtlb, old_hwtlb, hwtlb_mas2);
299 bool mas3_changed_p = CHANGED(new_hwtlb, old_hwtlb, hwtlb_mas3);
300 #undef CHANGED
301 bool new_valid_p = (new_hwtlb->hwtlb_mas1 & MAS1_V) != 0;
302 bool old_valid_p = (old_hwtlb.hwtlb_mas1 & MAS1_V) != 0;
303 if ((new_valid_p || old_valid_p)
304 && (mas1_changed_p
305 || (new_valid_p
306 && (mas2_changed_p || mas3_changed_p))))
307 hwtlb_write(*new_hwtlb, true);
308 }
309 }
310
311 static int
312 e500_alloc_tlb1_entry(void)
313 {
314 struct e500_tlb1 * const tlb1 = &e500_tlb1;
315
316 if (tlb1->tlb1_numfree == 0)
317 return -1;
318 const u_int slot = tlb1->tlb1_freelist[--tlb1->tlb1_numfree];
319 KASSERT((tlb1->tlb1_entries[slot].e_hwtlb.hwtlb_mas1 & MAS1_V) == 0);
320 tlb1->tlb1_entries[slot].e_hwtlb.hwtlb_mas0 =
321 MAS0_TLBSEL_TLB1 | __SHIFTOUT(slot, MAS0_ESEL);
322 return (int)slot;
323 }
324
325 static void
326 e500_free_tlb1_entry(struct e500_xtlb *xtlb, u_int slot, bool needs_sync)
327 {
328 struct e500_tlb1 * const tlb1 = &e500_tlb1;
329 KASSERT(slot < tlb1->tlb1_numentries);
330 KASSERT(&tlb1->tlb1_entries[slot] == xtlb);
331
332 KASSERT(xtlb->e_hwtlb.hwtlb_mas0 == (MAS0_TLBSEL_TLB1|__SHIFTIN(slot, MAS0_ESEL)));
333 xtlb->e_hwtlb.hwtlb_mas1 &= ~(MAS1_V|MAS1_IPROT);
334 hwtlb_write(xtlb->e_hwtlb, needs_sync);
335
336 const register_t msr = wrtee(0);
337 tlb1->tlb1_freelist[tlb1->tlb1_numfree++] = slot;
338 wrtee(msr);
339 }
340
341 static tlb_asid_t
342 e500_tlb_get_asid(void)
343 {
344 return mfspr(SPR_PID0);
345 }
346
347 static void
348 e500_tlb_set_asid(tlb_asid_t asid)
349 {
350 mtspr(SPR_PID0, asid);
351 }
352
353 static void
354 e500_tlb_invalidate_all(void)
355 {
356 /*
357 * This does a flash invalidate of all entries in TLB0.
358 * We don't touch TLB1 since we don't expect those to be volatile.
359 */
360 #if 1
361 __asm volatile("tlbivax\t0, %0" :: "b"(4)); /* INV_ALL */
362 #else
363 mtspr(SPR_MMUCSR0, MMUCSR0_TLB0_FL);
364 while (mfspr(SPR_MMUCSR0) != 0)
365 ;
366 #endif
367 }
368
369 static void
370 e500_tlb_invalidate_globals(void)
371 {
372 const size_t tlbassoc = TLBCFG_ASSOC(mftlb0cfg());
373 const size_t tlbentries = TLBCFG_NENTRY(mftlb0cfg());
374 const size_t max_epn = (tlbentries / tlbassoc) << PAGE_SHIFT;
375 const vaddr_t kstack_lo = (uintptr_t)curlwp->l_addr;
376 const vaddr_t kstack_hi = kstack_lo + USPACE - 1;
377 const vaddr_t epn_kstack_lo = kstack_lo & (max_epn - 1);
378 const vaddr_t epn_kstack_hi = kstack_hi & (max_epn - 1);
379
380 const register_t msr = wrtee(0);
381 for (size_t assoc = 0; assoc < tlbassoc; assoc++) {
382 mtspr(SPR_MAS0, MAS0_ESEL_MAKE(assoc) | MAS0_TLBSEL_TLB0);
383 for (size_t epn = 0; epn < max_epn; epn += PAGE_SIZE) {
384 mtspr(SPR_MAS2, epn);
385 __asm volatile("tlbre");
386 uint32_t mas1 = mfspr(SPR_MAS1);
387
388 /*
389 * Make sure this is a valid kernel entry first.
390 */
391 if ((mas1 & (MAS1_V|MAS1_TID|MAS1_TS)) != MAS1_V)
392 continue;
393
394 /*
395 * We have a valid kernel TLB entry. But if it matches
396 * the stack we are currently running on, it would
397 * unwise to invalidate it. First see if the epn
398 * overlaps the stack. If it does then get the
399 * VA and see if it really is part of the stack.
400 */
401 if (epn_kstack_lo < epn_kstack_hi
402 ? (epn_kstack_lo <= epn && epn <= epn_kstack_hi)
403 : (epn <= epn_kstack_hi || epn_kstack_lo <= epn)) {
404 const uint32_t mas2_epn =
405 mfspr(SPR_MAS2) & MAS2_EPN;
406 if (kstack_lo <= mas2_epn
407 && mas2_epn <= kstack_hi)
408 continue;
409 }
410 mtspr(SPR_MAS1, mas1 ^ MAS1_V);
411 __asm volatile("tlbwe");
412 }
413 }
414 __asm volatile("isync");
415 wrtee(msr);
416 }
417
418 static void
419 e500_tlb_invalidate_asids(tlb_asid_t asid_lo, tlb_asid_t asid_hi)
420 {
421 const size_t tlbassoc = TLBCFG_ASSOC(mftlb0cfg());
422 const size_t tlbentries = TLBCFG_NENTRY(mftlb0cfg());
423 const size_t max_epn = (tlbentries / tlbassoc) << PAGE_SHIFT;
424
425 asid_lo = __SHIFTIN(asid_lo, MAS1_TID);
426 asid_hi = __SHIFTIN(asid_hi, MAS1_TID);
427
428 const register_t msr = wrtee(0);
429 for (size_t assoc = 0; assoc < tlbassoc; assoc++) {
430 mtspr(SPR_MAS0, MAS0_ESEL_MAKE(assoc) | MAS0_TLBSEL_TLB0);
431 for (size_t epn = 0; epn < max_epn; epn += PAGE_SIZE) {
432 mtspr(SPR_MAS2, epn);
433 __asm volatile("tlbre");
434 const uint32_t mas1 = mfspr(SPR_MAS1);
435 /*
436 * If this is a valid entry for AS space 1 and
437 * its asid matches the constraints of the caller,
438 * clear its valid bit.
439 */
440 if ((mas1 & (MAS1_V|MAS1_TS)) == (MAS1_V|MAS1_TS)
441 && asid_lo <= (mas1 & MAS1_TID)
442 && (mas1 & MAS1_TID) <= asid_hi) {
443 mtspr(SPR_MAS1, mas1 ^ MAS1_V);
444 #if 0
445 printf("%s[%zu,%zu]->[%x]\n",
446 __func__, assoc, epn, mas1);
447 #endif
448 __asm volatile("tlbwe");
449 }
450 }
451 }
452 __asm volatile("isync");
453 wrtee(msr);
454 }
455
456 static u_int
457 e500_tlb_record_asids(u_long *bitmap)
458 {
459 const size_t tlbassoc = TLBCFG_ASSOC(mftlb0cfg());
460 const size_t tlbentries = TLBCFG_NENTRY(mftlb0cfg());
461 const size_t max_epn = (tlbentries / tlbassoc) << PAGE_SHIFT;
462 const size_t nbits = 8 * sizeof(bitmap[0]);
463 u_int found = 0;
464
465 const register_t msr = wrtee(0);
466 for (size_t assoc = 0; assoc < tlbassoc; assoc++) {
467 mtspr(SPR_MAS0, MAS0_ESEL_MAKE(assoc) | MAS0_TLBSEL_TLB0);
468 for (size_t epn = 0; epn < max_epn; epn += PAGE_SIZE) {
469 mtspr(SPR_MAS2, epn);
470 __asm volatile("tlbre");
471 const uint32_t mas1 = mfspr(SPR_MAS1);
472 /*
473 * If this is a valid entry for AS space 1 and
474 * its asid matches the constraints of the caller,
475 * clear its valid bit.
476 */
477 if ((mas1 & (MAS1_V|MAS1_TS)) == (MAS1_V|MAS1_TS)) {
478 const uint32_t asid = MASX_TID_GET(mas1);
479 const u_int i = asid / nbits;
480 const u_long mask = 1UL << (asid & (nbits - 1));
481 if ((bitmap[i] & mask) == 0) {
482 bitmap[i] |= mask;
483 found++;
484 }
485 }
486 }
487 }
488 wrtee(msr);
489
490 return found;
491 }
492
493 static void
494 e500_tlb_invalidate_addr(vaddr_t va, tlb_asid_t asid)
495 {
496 KASSERT((va & PAGE_MASK) == 0);
497 /*
498 * Bits 60 & 61 have meaning
499 */
500 __asm volatile("tlbivax\t0, %0" :: "b"(va));
501 __asm volatile("tlbsync");
502 __asm volatile("tlbsync");
503 }
504
505 static bool
506 e500_tlb_update_addr(vaddr_t va, tlb_asid_t asid, pt_entry_t pte, bool insert)
507 {
508 struct e500_hwtlb hwtlb = tlb_to_hwtlb(
509 (struct e500_tlb){ .tlb_va = va, .tlb_asid = asid,
510 .tlb_size = PAGE_SIZE, .tlb_pte = pte,});
511
512 register_t msr = wrtee(0);
513 mtspr(SPR_MAS6, asid ? __SHIFTIN(asid, MAS6_SPID0) | MAS6_SAS : 0);
514 __asm volatile("tlbsx 0, %0" :: "b"(va));
515 register_t mas1 = mfspr(SPR_MAS1);
516 if ((mas1 & MAS1_V) == 0) {
517 if (!insert) {
518 wrtee(msr);
519 #if 0
520 printf("%s(%#lx,%#x,%#x,%x)<no update>\n",
521 __func__, va, asid, pte, insert);
522 #endif
523 return false;
524 }
525 mtspr(SPR_MAS1, hwtlb.hwtlb_mas1);
526 }
527 mtspr(SPR_MAS2, hwtlb.hwtlb_mas2);
528 mtspr(SPR_MAS3, hwtlb.hwtlb_mas3);
529 __asm volatile("tlbwe");
530 if (asid == 0)
531 __asm volatile("isync");
532 wrtee(msr);
533 #if 0
534 if (asid)
535 printf("%s(%#lx,%#x,%#x,%x)->[%x,%x,%x]\n",
536 __func__, va, asid, pte, insert,
537 hwtlb.hwtlb_mas1, hwtlb.hwtlb_mas2, hwtlb.hwtlb_mas3);
538 #endif
539 return (mas1 & MAS1_V) != 0;
540 }
541
542 static void
543 e500_tlb_write_entry(size_t index, const struct tlbmask *tlb)
544 {
545 }
546
547 static void
548 e500_tlb_read_entry(size_t index, struct tlbmask *tlb)
549 {
550 }
551
552 static void
553 e500_tlb_dump(void (*pr)(const char *, ...))
554 {
555 const size_t tlbassoc = TLBCFG_ASSOC(mftlb0cfg());
556 const size_t tlbentries = TLBCFG_NENTRY(mftlb0cfg());
557 const size_t max_epn = (tlbentries / tlbassoc) << PAGE_SHIFT;
558 const uint32_t saved_mas0 = mfspr(SPR_MAS0);
559 size_t valid = 0;
560
561 if (pr == NULL)
562 pr = printf;
563
564 const register_t msr = wrtee(0);
565 for (size_t assoc = 0; assoc < tlbassoc; assoc++) {
566 struct e500_hwtlb hwtlb;
567 hwtlb.hwtlb_mas0 = MAS0_ESEL_MAKE(assoc) | MAS0_TLBSEL_TLB0;
568 mtspr(SPR_MAS0, hwtlb.hwtlb_mas0);
569 for (size_t epn = 0; epn < max_epn; epn += PAGE_SIZE) {
570 mtspr(SPR_MAS2, epn);
571 __asm volatile("tlbre");
572 hwtlb.hwtlb_mas1 = mfspr(SPR_MAS1);
573 /*
574 * If this is a valid entry for AS space 1 and
575 * its asid matches the constraints of the caller,
576 * clear its valid bit.
577 */
578 if (hwtlb.hwtlb_mas1 & MAS1_V) {
579 hwtlb.hwtlb_mas2 = mfspr(SPR_MAS2);
580 hwtlb.hwtlb_mas3 = mfspr(SPR_MAS3);
581 struct e500_tlb tlb = hwtlb_to_tlb(hwtlb);
582 (*pr)("[%zu,%zu]->[%x,%x,%x]",
583 assoc, atop(epn),
584 hwtlb.hwtlb_mas1,
585 hwtlb.hwtlb_mas2,
586 hwtlb.hwtlb_mas3);
587 (*pr)(": VA=%#lx size=4KB asid=%u pte=%x",
588 tlb.tlb_va, tlb.tlb_asid, tlb.tlb_pte);
589 (*pr)(" (RPN=%#x,%s%s%s%s%s,%s%s%s%s%s)\n",
590 tlb.tlb_pte & PTE_RPN_MASK,
591 tlb.tlb_pte & PTE_xR ? "R" : "",
592 tlb.tlb_pte & PTE_xW ? "W" : "",
593 tlb.tlb_pte & PTE_UNMODIFIED ? "*" : "",
594 tlb.tlb_pte & PTE_xX ? "X" : "",
595 tlb.tlb_pte & PTE_UNSYNCED ? "*" : "",
596 tlb.tlb_pte & PTE_W ? "W" : "",
597 tlb.tlb_pte & PTE_I ? "I" : "",
598 tlb.tlb_pte & PTE_M ? "M" : "",
599 tlb.tlb_pte & PTE_G ? "G" : "",
600 tlb.tlb_pte & PTE_E ? "E" : "");
601 valid++;
602 }
603 }
604 }
605 mtspr(SPR_MAS0, saved_mas0);
606 wrtee(msr);
607 (*pr)("%s: %zu valid entries\n", __func__, valid);
608 }
609
610 static void
611 e500_tlb_walk(void *ctx, bool (*func)(void *, vaddr_t, uint32_t, uint32_t))
612 {
613 const size_t tlbassoc = TLBCFG_ASSOC(mftlb0cfg());
614 const size_t tlbentries = TLBCFG_NENTRY(mftlb0cfg());
615 const size_t max_epn = (tlbentries / tlbassoc) << PAGE_SHIFT;
616 const uint32_t saved_mas0 = mfspr(SPR_MAS0);
617
618 const register_t msr = wrtee(0);
619 for (size_t assoc = 0; assoc < tlbassoc; assoc++) {
620 struct e500_hwtlb hwtlb;
621 hwtlb.hwtlb_mas0 = MAS0_ESEL_MAKE(assoc) | MAS0_TLBSEL_TLB0;
622 mtspr(SPR_MAS0, hwtlb.hwtlb_mas0);
623 for (size_t epn = 0; epn < max_epn; epn += PAGE_SIZE) {
624 mtspr(SPR_MAS2, epn);
625 __asm volatile("tlbre");
626 hwtlb.hwtlb_mas1 = mfspr(SPR_MAS1);
627 /*
628 * If this is a valid entry for AS space 1 and
629 * its asid matches the constraints of the caller,
630 * clear its valid bit.
631 */
632 if (hwtlb.hwtlb_mas1 & MAS1_V) {
633 hwtlb.hwtlb_mas2 = mfspr(SPR_MAS2);
634 hwtlb.hwtlb_mas3 = mfspr(SPR_MAS3);
635 struct e500_tlb tlb = hwtlb_to_tlb(hwtlb);
636 if (!(*func)(ctx, tlb.tlb_va, tlb.tlb_asid,
637 tlb.tlb_pte))
638 break;
639 }
640 }
641 }
642 mtspr(SPR_MAS0, saved_mas0);
643 wrtee(msr);
644 }
645
646 static struct e500_xtlb *
647 e500_tlb_lookup_xtlb(vaddr_t va, u_int *slotp)
648 {
649 struct e500_tlb1 * const tlb1 = &e500_tlb1;
650 struct e500_xtlb *xtlb = tlb1->tlb1_entries;
651
652 /*
653 * See if we have a TLB entry for the pa.
654 */
655 for (u_int i = 0; i < tlb1->tlb1_numentries; i++, xtlb++) {
656 if ((xtlb->e_hwtlb.hwtlb_mas1 & MAS1_V)
657 && xtlb->e_tlb.tlb_va <= va
658 && va < xtlb->e_tlb.tlb_va + xtlb->e_tlb.tlb_size) {
659 if (slotp != NULL)
660 *slotp = i;
661 return xtlb;
662 }
663 }
664
665 return NULL;
666 }
667
668 static struct e500_xtlb *
669 e500_tlb_lookup_xtlb2(vaddr_t va, vsize_t len)
670 {
671 struct e500_tlb1 * const tlb1 = &e500_tlb1;
672 struct e500_xtlb *xtlb = tlb1->tlb1_entries;
673
674 /*
675 * See if we have a TLB entry for the pa.
676 */
677 for (u_int i = 0; i < tlb1->tlb1_numentries; i++, xtlb++) {
678 if ((xtlb->e_hwtlb.hwtlb_mas1 & MAS1_V)
679 && xtlb->e_tlb.tlb_va < va + len
680 && va < xtlb->e_tlb.tlb_va + xtlb->e_tlb.tlb_size) {
681 return xtlb;
682 }
683 }
684
685 return NULL;
686 }
687
688 static void *
689 e500_tlb_mapiodev(paddr_t pa, psize_t len, bool prefetchable)
690 {
691 struct e500_xtlb * const xtlb = e500_tlb_lookup_xtlb(pa, NULL);
692
693 /*
694 * See if we have a TLB entry for the pa. If completely falls within
695 * mark the reference and return the pa. But only if the tlb entry
696 * is not cacheable.
697 */
698 if (xtlb
699 && pa + len <= xtlb->e_tlb.tlb_va + xtlb->e_tlb.tlb_size
700 && (prefetchable
701 || (xtlb->e_tlb.tlb_pte & PTE_WIG) == (PTE_I|PTE_G))) {
702 xtlb->e_refcnt++;
703 return (void *) pa;
704 }
705 return NULL;
706 }
707
708 static void
709 e500_tlb_unmapiodev(vaddr_t va, vsize_t len)
710 {
711 if (va < VM_MIN_KERNEL_ADDRESS || VM_MAX_KERNEL_ADDRESS <= va) {
712 struct e500_xtlb * const xtlb = e500_tlb_lookup_xtlb(va, NULL);
713 if (xtlb)
714 xtlb->e_refcnt--;
715 }
716 }
717
718 static int
719 e500_tlb_ioreserve(vaddr_t va, vsize_t len, pt_entry_t pte)
720 {
721 struct e500_tlb1 * const tlb1 = &e500_tlb1;
722 struct e500_xtlb *xtlb;
723
724 KASSERT(len & 0x55555000);
725 KASSERT((len & ~0x55555000) == 0);
726 KASSERT(len >= PAGE_SIZE);
727 KASSERT((len & (len - 1)) == 0);
728 KASSERT((va & (len - 1)) == 0);
729 KASSERT((pte & (len - 1)) == 0);
730
731 if ((xtlb = e500_tlb_lookup_xtlb2(va, len)) != NULL) {
732 if (va < xtlb->e_tlb.tlb_va
733 || xtlb->e_tlb.tlb_va + xtlb->e_tlb.tlb_size < va + len
734 || va - xtlb->e_tlb.tlb_va != pte - xtlb->e_tlb.tlb_pte)
735 return EBUSY;
736 xtlb->e_refcnt++;
737 return 0;
738 }
739
740 const int slot = e500_alloc_tlb1_entry();
741 if (slot < 0)
742 return ENOMEM;
743
744 xtlb = &tlb1->tlb1_entries[slot];
745 xtlb->e_tlb.tlb_va = va;
746 xtlb->e_tlb.tlb_size = len;
747 xtlb->e_tlb.tlb_pte = pte;
748 xtlb->e_tlb.tlb_asid = KERNEL_PID;
749
750 xtlb->e_hwtlb = tlb_to_hwtlb(xtlb->e_tlb);
751 xtlb->e_hwtlb.hwtlb_mas0 |= __SHIFTOUT(slot, MAS0_ESEL);
752 hwtlb_write(xtlb->e_hwtlb, true);
753 return 0;
754 }
755
756 static int
757 e500_tlb_iorelease(vaddr_t va)
758 {
759 u_int slot;
760 struct e500_xtlb * const xtlb = e500_tlb_lookup_xtlb(va, &slot);
761
762 if (xtlb == NULL)
763 return ENOENT;
764
765 if (xtlb->e_refcnt)
766 return EBUSY;
767
768 e500_free_tlb1_entry(xtlb, slot, true);
769
770 return 0;
771 }
772
773 static u_int
774 e500_tlbmemmap(paddr_t memstart, psize_t memsize, struct e500_tlb1 *tlb1)
775 {
776 u_int slotmask = 0;
777 u_int slots = 0, nextslot = 0;
778 KASSERT(tlb1->tlb1_numfree > 1);
779 KASSERT(((memstart + memsize - 1) & -memsize) == memstart);
780 for (paddr_t lastaddr = memstart; 0 < memsize; ) {
781 u_int cnt = __builtin_clz(memsize);
782 psize_t size = min(1UL << (31 - (cnt | 1)), tlb1->tlb1_maxsize);
783 slots += memsize / size;
784 if (slots > 4)
785 panic("%s: %d: can't map memory (%#lx) into TLB1: %s",
786 __func__, __LINE__, memsize, "too fragmented");
787 if (slots > tlb1->tlb1_numfree - 1)
788 panic("%s: %d: can't map memory (%#lx) into TLB1: %s",
789 __func__, __LINE__, memsize,
790 "insufficent TLB entries");
791 for (; nextslot < slots; nextslot++) {
792 const u_int freeslot = e500_alloc_tlb1_entry();
793 struct e500_xtlb * const xtlb =
794 &tlb1->tlb1_entries[freeslot];
795 xtlb->e_tlb.tlb_asid = KERNEL_PID;
796 xtlb->e_tlb.tlb_size = size;
797 xtlb->e_tlb.tlb_va = lastaddr;
798 xtlb->e_tlb.tlb_pte = lastaddr
799 | PTE_M | PTE_xX | PTE_xW | PTE_xR;
800 lastaddr += size;
801 memsize -= size;
802 slotmask |= 1 << (31 - freeslot); /* clz friendly */
803 }
804 }
805
806 return nextslot;
807 }
808 static const struct tlb_md_ops e500_tlb_ops = {
809 .md_tlb_get_asid = e500_tlb_get_asid,
810 .md_tlb_set_asid = e500_tlb_set_asid,
811 .md_tlb_invalidate_all = e500_tlb_invalidate_all,
812 .md_tlb_invalidate_globals = e500_tlb_invalidate_globals,
813 .md_tlb_invalidate_asids = e500_tlb_invalidate_asids,
814 .md_tlb_invalidate_addr = e500_tlb_invalidate_addr,
815 .md_tlb_update_addr = e500_tlb_update_addr,
816 .md_tlb_record_asids = e500_tlb_record_asids,
817 .md_tlb_write_entry = e500_tlb_write_entry,
818 .md_tlb_read_entry = e500_tlb_read_entry,
819 .md_tlb_dump = e500_tlb_dump,
820 .md_tlb_walk = e500_tlb_walk,
821 };
822
823 static const struct tlb_md_io_ops e500_tlb_io_ops = {
824 .md_tlb_mapiodev = e500_tlb_mapiodev,
825 .md_tlb_unmapiodev = e500_tlb_unmapiodev,
826 .md_tlb_ioreserve = e500_tlb_ioreserve,
827 .md_tlb_iorelease = e500_tlb_iorelease,
828 };
829
830 void
831 e500_tlb_init(vaddr_t endkernel, psize_t memsize)
832 {
833 struct e500_tlb1 * const tlb1 = &e500_tlb1;
834
835 #if 0
836 register_t mmucfg = mfspr(SPR_MMUCFG);
837 register_t mas4 = mfspr(SPR_MAS4);
838 #endif
839
840 const uint32_t tlb1cfg = mftlb1cfg();
841 tlb1->tlb1_numentries = TLBCFG_NENTRY(tlb1cfg);
842 KASSERT(tlb1->tlb1_numentries <= __arraycount(tlb1->tlb1_entries));
843 /*
844 * Limit maxsize to 1G since 4G isn't really useful to us.
845 */
846 tlb1->tlb1_minsize = 1024 << (2 * TLBCFG_MINSIZE(tlb1cfg));
847 tlb1->tlb1_maxsize = 1024 << (2 * min(10, TLBCFG_MAXSIZE(tlb1cfg)));
848
849 #ifdef VERBOSE_INITPPC
850 printf(" tlb1cfg=%#x numentries=%u minsize=%#xKB maxsize=%#xKB",
851 tlb1cfg, tlb1->tlb1_numentries, tlb1->tlb1_minsize >> 10,
852 tlb1->tlb1_maxsize >> 10);
853 #endif
854
855 /*
856 * Let's see what's in TLB1 and we need to invalidate any entry that
857 * would fit within the kernel's mapped address space.
858 */
859 psize_t memmapped = 0;
860 for (u_int i = 0; i < tlb1->tlb1_numentries; i++) {
861 struct e500_xtlb * const xtlb = &tlb1->tlb1_entries[i];
862
863 xtlb->e_hwtlb = hwtlb_read(MAS0_TLBSEL_TLB1, i);
864
865 if ((xtlb->e_hwtlb.hwtlb_mas1 & MAS1_V) == 0) {
866 tlb1->tlb1_freelist[tlb1->tlb1_numfree++] = i;
867 #ifdef VERBOSE_INITPPC
868 printf(" TLB1[%u]=<unused>", i);
869 #endif
870 continue;
871 }
872
873 xtlb->e_tlb = hwtlb_to_tlb(xtlb->e_hwtlb);
874 #ifdef VERBOSE_INITPPC
875 printf(" TLB1[%u]=<%#lx,%#lx,%#x,%#x>",
876 i, xtlb->e_tlb.tlb_va, xtlb->e_tlb.tlb_size,
877 xtlb->e_tlb.tlb_asid, xtlb->e_tlb.tlb_pte);
878 #endif
879 if ((VM_MIN_KERNEL_ADDRESS <= xtlb->e_tlb.tlb_va
880 && xtlb->e_tlb.tlb_va < VM_MAX_KERNEL_ADDRESS)
881 || (xtlb->e_tlb.tlb_va < VM_MIN_KERNEL_ADDRESS
882 && VM_MIN_KERNEL_ADDRESS <
883 xtlb->e_tlb.tlb_va + xtlb->e_tlb.tlb_size)) {
884 #ifdef VERBOSE_INITPPC
885 printf("free");
886 #endif
887 e500_free_tlb1_entry(xtlb, i, false);
888 #ifdef VERBOSE_INITPPC
889 printf("d");
890 #endif
891 continue;
892 }
893 if ((xtlb->e_hwtlb.hwtlb_mas1 & MAS1_IPROT) == 0) {
894 xtlb->e_hwtlb.hwtlb_mas1 |= MAS1_IPROT;
895 hwtlb_write(xtlb->e_hwtlb, false);
896 #ifdef VERBOSE_INITPPC
897 printf("+iprot");
898 #endif
899 }
900 if (xtlb->e_tlb.tlb_pte & PTE_I)
901 continue;
902
903 if (xtlb->e_tlb.tlb_va == 0
904 || xtlb->e_tlb.tlb_va + xtlb->e_tlb.tlb_size <= memsize) {
905 memmapped += xtlb->e_tlb.tlb_size;
906 }
907 }
908
909 cpu_md_ops.md_tlb_ops = &e500_tlb_ops;
910 cpu_md_ops.md_tlb_io_ops = &e500_tlb_io_ops;
911
912 if (__predict_false(memmapped < memsize)) {
913 /*
914 * Let's see how many TLB entries are needed to map memory.
915 */
916 u_int slotmask = e500_tlbmemmap(0, memsize, tlb1);
917
918 /*
919 * To map main memory into the TLB, we need to flush any
920 * existing entries from the TLB that overlap the virtual
921 * address space needed to map physical memory. That may
922 * include the entries for the pages currently used by the
923 * stack or that we are executing. So to avoid problems, we
924 * are going to temporarily map the kernel and stack into AS 1,
925 * switch to it, and clear out the TLB entries from AS 0,
926 * install the new TLB entries to map memory, and then switch
927 * back to AS 0 and free the temp entry used for AS1.
928 */
929 u_int b = __builtin_clz(endkernel);
930
931 /*
932 * If the kernel doesn't end on a clean power of 2, we need
933 * to round the size up (by decrementing the number of leading
934 * zero bits). If the size isn't a power of 4KB, decrement
935 * again to make it one.
936 */
937 if (endkernel & (endkernel - 1))
938 b--;
939 if ((b & 1) == 0)
940 b--;
941
942 /*
943 * Create a TLB1 mapping for the kernel in AS1.
944 */
945 const u_int kslot = e500_alloc_tlb1_entry();
946 struct e500_xtlb * const kxtlb = &tlb1->tlb1_entries[kslot];
947 kxtlb->e_tlb.tlb_va = 0;
948 kxtlb->e_tlb.tlb_size = 1UL << (31 - b);
949 kxtlb->e_tlb.tlb_pte = PTE_M|PTE_xR|PTE_xW|PTE_xX;
950 kxtlb->e_tlb.tlb_asid = KERNEL_PID;
951
952 kxtlb->e_hwtlb = tlb_to_hwtlb(kxtlb->e_tlb);
953 kxtlb->e_hwtlb.hwtlb_mas0 |= __SHIFTOUT(kslot, MAS0_ESEL);
954 kxtlb->e_hwtlb.hwtlb_mas1 |= MAS1_TS;
955 hwtlb_write(kxtlb->e_hwtlb, true);
956
957 /*
958 * Now that we have a TLB mapping in AS1 for the kernel and its
959 * stack, we switch to AS1 to cleanup the TLB mappings for TLB0.
960 */
961 const register_t saved_msr = mfmsr();
962 mtmsr(saved_msr | PSL_DS | PSL_IS);
963 __asm volatile("isync");
964
965 /*
966 *** Invalidate all the TLB0 entries.
967 */
968 e500_tlb_invalidate_all();
969
970 /*
971 *** Now let's see if we have any entries in TLB1 that would
972 *** overlap the ones we are about to install. If so, nuke 'em.
973 */
974 for (u_int i = 0; i < tlb1->tlb1_numentries; i++) {
975 struct e500_xtlb * const xtlb = &tlb1->tlb1_entries[i];
976 struct e500_hwtlb * const hwtlb = &xtlb->e_hwtlb;
977 if ((hwtlb->hwtlb_mas1 & (MAS1_V|MAS1_TS)) == MAS1_V
978 && (hwtlb->hwtlb_mas2 & MAS2_EPN) < memsize) {
979 e500_free_tlb1_entry(xtlb, i, false);
980 }
981 }
982
983 /*
984 *** Now we can add the TLB entries that will map physical
985 *** memory. If bit 0 [MSB] in slotmask is set, then tlb
986 *** entry 0 contains a mapping for physical memory...
987 */
988 struct e500_xtlb *entries = tlb1->tlb1_entries;
989 while (slotmask != 0) {
990 const u_int slot = __builtin_clz(slotmask);
991 hwtlb_write(entries[slot].e_hwtlb, false);
992 entries += slot + 1;
993 slotmask <<= slot + 1;
994 }
995
996 /*
997 *** Synchronize the TLB and the instruction stream.
998 */
999 __asm volatile("tlbsync");
1000 __asm volatile("isync");
1001
1002 /*
1003 *** Switch back to AS 0.
1004 */
1005 mtmsr(saved_msr);
1006 __asm volatile("isync");
1007
1008 /*
1009 * Free the temporary TLB1 entry.
1010 */
1011 e500_free_tlb1_entry(kxtlb, kslot, true);
1012 }
1013
1014 /*
1015 * Finally set the MAS4 defaults.
1016 */
1017 mtspr(SPR_MAS4, MAS4_TSIZED_4KB | MAS4_MD);
1018
1019 /*
1020 * Invalidate all the TLB0 entries.
1021 */
1022 e500_tlb_invalidate_all();
1023 }
1024
1025 void
1026 e500_tlb_minimize(vaddr_t endkernel)
1027 {
1028 #ifdef PMAP_MINIMALTLB
1029 struct e500_tlb1 * const tlb1 = &e500_tlb1;
1030 extern uint32_t _fdata[];
1031
1032 u_int slot;
1033
1034 paddr_t boot_page = cpu_read_4(GUR_BPTR);
1035 if (boot_page & BPTR_EN) {
1036 /*
1037 * shift it to an address
1038 */
1039 boot_page = (boot_page & BPTR_BOOT_PAGE) << PAGE_SHIFT;
1040 pmap_kvptefill(boot_page, boot_page + NBPG,
1041 PTE_M | PTE_xR | PTE_xW | PTE_xX);
1042 }
1043
1044
1045 KASSERT(endkernel - (uintptr_t)_fdata < 0x400000);
1046 KASSERT((uintptr_t)_fdata == 0x400000);
1047
1048 struct e500_xtlb *xtlb = e500_tlb_lookup_xtlb(endkernel, &slot);
1049
1050 KASSERT(xtlb == e500_tlb_lookup_xtlb2(0, endkernel));
1051 const u_int tmp_slot = e500_alloc_tlb1_entry();
1052 KASSERT(tmp_slot != (u_int) -1);
1053
1054 struct e500_xtlb * const tmp_xtlb = &tlb1->tlb1_entries[tmp_slot];
1055 tmp_xtlb->e_tlb = xtlb->e_tlb;
1056 tmp_xtlb->e_hwtlb = tlb_to_hwtlb(tmp_xtlb->e_tlb);
1057 tmp_xtlb->e_hwtlb.hwtlb_mas1 |= MAS1_TS;
1058 KASSERT((tmp_xtlb->e_hwtlb.hwtlb_mas0 & MAS0_TLBSEL) == MAS0_TLBSEL_TLB1);
1059 tmp_xtlb->e_hwtlb.hwtlb_mas0 |= __SHIFTIN(tmp_slot, MAS0_ESEL);
1060 hwtlb_write(tmp_xtlb->e_hwtlb, true);
1061
1062 const u_int text_slot = e500_alloc_tlb1_entry();
1063 KASSERT(text_slot != (u_int)-1);
1064 struct e500_xtlb * const text_xtlb = &tlb1->tlb1_entries[text_slot];
1065 text_xtlb->e_tlb.tlb_va = 0;
1066 text_xtlb->e_tlb.tlb_size = 0x400000;
1067 text_xtlb->e_tlb.tlb_pte = PTE_M | PTE_xR | PTE_xX | text_xtlb->e_tlb.tlb_va;
1068 text_xtlb->e_tlb.tlb_asid = 0;
1069 text_xtlb->e_hwtlb = tlb_to_hwtlb(text_xtlb->e_tlb);
1070 KASSERT((text_xtlb->e_hwtlb.hwtlb_mas0 & MAS0_TLBSEL) == MAS0_TLBSEL_TLB1);
1071 text_xtlb->e_hwtlb.hwtlb_mas0 |= __SHIFTIN(text_slot, MAS0_ESEL);
1072
1073 const u_int data_slot = e500_alloc_tlb1_entry();
1074 KASSERT(data_slot != (u_int)-1);
1075 struct e500_xtlb * const data_xtlb = &tlb1->tlb1_entries[data_slot];
1076 data_xtlb->e_tlb.tlb_va = 0x400000;
1077 data_xtlb->e_tlb.tlb_size = 0x400000;
1078 data_xtlb->e_tlb.tlb_pte = PTE_M | PTE_xR | PTE_xW | data_xtlb->e_tlb.tlb_va;
1079 data_xtlb->e_tlb.tlb_asid = 0;
1080 data_xtlb->e_hwtlb = tlb_to_hwtlb(data_xtlb->e_tlb);
1081 KASSERT((data_xtlb->e_hwtlb.hwtlb_mas0 & MAS0_TLBSEL) == MAS0_TLBSEL_TLB1);
1082 data_xtlb->e_hwtlb.hwtlb_mas0 |= __SHIFTIN(data_slot, MAS0_ESEL);
1083
1084 const register_t msr = mfmsr();
1085 const register_t ts_msr = (msr | PSL_DS | PSL_IS) & ~PSL_EE;
1086
1087 __asm __volatile(
1088 "mtmsr %[ts_msr]" "\n\t"
1089 "sync" "\n\t"
1090 "isync"
1091 :: [ts_msr] "r" (ts_msr));
1092
1093 #if 0
1094 hwtlb_write(text_xtlb->e_hwtlb, false);
1095 hwtlb_write(data_xtlb->e_hwtlb, false);
1096 e500_free_tlb1_entry(xtlb, slot, true);
1097 #endif
1098
1099 __asm __volatile(
1100 "mtmsr %[msr]" "\n\t"
1101 "sync" "\n\t"
1102 "isync"
1103 :: [msr] "r" (msr));
1104
1105 e500_free_tlb1_entry(tmp_xtlb, tmp_slot, true);
1106 #endif /* PMAP_MINIMALTLB */
1107 }
1108