pmap.c revision 1.93 1 /* $NetBSD: pmap.c,v 1.93 2020/09/10 03:32:46 rin Exp $ */
2
3 /*
4 * Copyright 2001 Wasabi Systems, Inc.
5 * All rights reserved.
6 *
7 * Written by Eduardo Horvath and Simon Burge for Wasabi Systems, Inc.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 * 3. All advertising materials mentioning features or use of this software
18 * must display the following acknowledgement:
19 * This product includes software developed for the NetBSD Project by
20 * Wasabi Systems, Inc.
21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse
22 * or promote products derived from this software without specific prior
23 * written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC
29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
35 * POSSIBILITY OF SUCH DAMAGE.
36 */
37
38 /*
39 * Copyright (C) 1995, 1996 Wolfgang Solfrank.
40 * Copyright (C) 1995, 1996 TooLs GmbH.
41 * All rights reserved.
42 *
43 * Redistribution and use in source and binary forms, with or without
44 * modification, are permitted provided that the following conditions
45 * are met:
46 * 1. Redistributions of source code must retain the above copyright
47 * notice, this list of conditions and the following disclaimer.
48 * 2. Redistributions in binary form must reproduce the above copyright
49 * notice, this list of conditions and the following disclaimer in the
50 * documentation and/or other materials provided with the distribution.
51 * 3. All advertising materials mentioning features or use of this software
52 * must display the following acknowledgement:
53 * This product includes software developed by TooLs GmbH.
54 * 4. The name of TooLs GmbH may not be used to endorse or promote products
55 * derived from this software without specific prior written permission.
56 *
57 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
58 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
59 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
60 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
61 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
62 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
63 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
64 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
65 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
66 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
67 */
68
69 #include <sys/cdefs.h>
70 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.93 2020/09/10 03:32:46 rin Exp $");
71
72 #ifdef _KERNEL_OPT
73 #include "opt_ddb.h"
74 #include "opt_pmap.h"
75 #endif
76
77 #include <sys/param.h>
78 #include <sys/cpu.h>
79 #include <sys/device.h>
80 #include <sys/kmem.h>
81 #include <sys/pool.h>
82 #include <sys/proc.h>
83 #include <sys/queue.h>
84 #include <sys/systm.h>
85
86 #include <uvm/uvm.h>
87
88 #include <machine/powerpc.h>
89 #include <machine/tlb.h>
90
91 #include <powerpc/pcb.h>
92
93 #include <powerpc/spr.h>
94 #include <powerpc/ibm4xx/spr.h>
95
96 #include <powerpc/ibm4xx/cpu.h>
97
98 /*
99 * kernmap is an array of PTEs large enough to map in
100 * 4GB. At 16KB/page it is 256K entries or 2MB.
101 */
102 #define KERNMAP_SIZE ((0xffffffffU/PAGE_SIZE)+1)
103 void *kernmap;
104
105 #define MINCTX 2
106 #define NUMCTX 256
107
108 volatile struct pmap *ctxbusy[NUMCTX];
109
110 #define TLBF_USED 0x1
111 #define TLBF_REF 0x2
112 #define TLBF_LOCKED 0x4
113 #define TLB_LOCKED(i) (tlb_info[(i)].ti_flags & TLBF_LOCKED)
114
115 typedef struct tlb_info_s {
116 char ti_flags;
117 char ti_ctx; /* TLB_PID assiciated with the entry */
118 u_int ti_va;
119 } tlb_info_t;
120
121 volatile tlb_info_t tlb_info[NTLB];
122 /* We'll use a modified FIFO replacement policy cause it's cheap */
123 volatile int tlbnext;
124
125 static int tlb_nreserved = 0;
126 static int pmap_bootstrap_done = 0;
127
128 /* Event counters */
129 struct evcnt tlbmiss_ev = EVCNT_INITIALIZER(EVCNT_TYPE_TRAP,
130 NULL, "cpu", "tlbmiss");
131 struct evcnt tlbflush_ev = EVCNT_INITIALIZER(EVCNT_TYPE_TRAP,
132 NULL, "cpu", "tlbflush");
133 struct evcnt tlbenter_ev = EVCNT_INITIALIZER(EVCNT_TYPE_TRAP,
134 NULL, "cpu", "tlbenter");
135 EVCNT_ATTACH_STATIC(tlbmiss_ev);
136 EVCNT_ATTACH_STATIC(tlbflush_ev);
137 EVCNT_ATTACH_STATIC(tlbenter_ev);
138
139 struct pmap kernel_pmap_;
140 struct pmap *const kernel_pmap_ptr = &kernel_pmap_;
141
142 static int npgs;
143 static u_int nextavail;
144 #ifndef MSGBUFADDR
145 extern paddr_t msgbuf_paddr;
146 #endif
147
148 static struct mem_region *mem, *avail;
149
150 /*
151 * This is a cache of referenced/modified bits.
152 * Bits herein are shifted by ATTRSHFT.
153 */
154 static char *pmap_attrib;
155
156 #define PV_WIRED 0x1
157 #define PV_WIRE(pv) ((pv)->pv_va |= PV_WIRED)
158 #define PV_UNWIRE(pv) ((pv)->pv_va &= ~PV_WIRED)
159 #define PV_ISWIRED(pv) ((pv)->pv_va & PV_WIRED)
160 #define PV_VA(pv) ((pv)->pv_va & ~PV_WIRED)
161 #define PV_CMPVA(va,pv) (!(PV_VA(pv) ^ (va)))
162
163 struct pv_entry {
164 struct pv_entry *pv_next; /* Linked list of mappings */
165 struct pmap *pv_pm;
166 vaddr_t pv_va; /* virtual address of mapping */
167 };
168
169 /* Each index corresponds to TLB_SIZE_* value. */
170 static size_t tlbsize[] = {
171 1024, /* TLB_SIZE_1K */
172 4096, /* TLB_SIZE_4K */
173 16384, /* TLB_SIZE_16K */
174 65536, /* TLB_SIZE_64K */
175 262144, /* TLB_SIZE_256K */
176 1048576, /* TLB_SIZE_1M */
177 4194304, /* TLB_SIZE_4M */
178 16777216, /* TLB_SIZE_16M */
179 };
180
181 struct pv_entry *pv_table;
182 static struct pool pv_pool;
183
184 static int pmap_initialized;
185
186 static int ctx_flush(int);
187
188 struct pv_entry *pa_to_pv(paddr_t);
189 static inline char *pa_to_attr(paddr_t);
190
191 static inline volatile u_int *pte_find(struct pmap *, vaddr_t);
192 static inline int pte_enter(struct pmap *, vaddr_t, u_int);
193
194 static inline int pmap_enter_pv(struct pmap *, vaddr_t, paddr_t, int);
195 static void pmap_remove_pv(struct pmap *, vaddr_t, paddr_t);
196
197 static inline void tlb_invalidate_entry(int);
198
199 static int ppc4xx_tlb_size_mask(size_t, int *, int *);
200
201
202 struct pv_entry *
203 pa_to_pv(paddr_t pa)
204 {
205 uvm_physseg_t bank;
206 psize_t pg;
207
208 bank = uvm_physseg_find(atop(pa), &pg);
209 if (bank == UVM_PHYSSEG_TYPE_INVALID)
210 return NULL;
211 return &uvm_physseg_get_pmseg(bank)->pvent[pg];
212 }
213
214 static inline char *
215 pa_to_attr(paddr_t pa)
216 {
217 uvm_physseg_t bank;
218 psize_t pg;
219
220 bank = uvm_physseg_find(atop(pa), &pg);
221 if (bank == UVM_PHYSSEG_TYPE_INVALID)
222 return NULL;
223 return &uvm_physseg_get_pmseg(bank)->attrs[pg];
224 }
225
226 /*
227 * Insert PTE into page table.
228 */
229 int
230 pte_enter(struct pmap *pm, vaddr_t va, u_int pte)
231 {
232 int seg = STIDX(va);
233 int ptn = PTIDX(va);
234 u_int oldpte;
235
236 if (!pm->pm_ptbl[seg]) {
237 /* Don't allocate a page to clear a non-existent mapping. */
238 if (!pte)
239 return (0);
240 /* Allocate a page XXXX this will sleep! */
241 pm->pm_ptbl[seg] =
242 (uint *)uvm_km_alloc(kernel_map, PAGE_SIZE, 0,
243 UVM_KMF_WIRED | UVM_KMF_ZERO);
244 }
245 oldpte = pm->pm_ptbl[seg][ptn];
246 pm->pm_ptbl[seg][ptn] = pte;
247
248 /* Flush entry. */
249 ppc4xx_tlb_flush(va, pm->pm_ctx);
250 if (oldpte != pte) {
251 if (pte == 0)
252 pm->pm_stats.resident_count--;
253 else
254 pm->pm_stats.resident_count++;
255 }
256 return (1);
257 }
258
259 /*
260 * Get a pointer to a PTE in a page table.
261 */
262 volatile u_int *
263 pte_find(struct pmap *pm, vaddr_t va)
264 {
265 int seg = STIDX(va);
266 int ptn = PTIDX(va);
267
268 if (pm->pm_ptbl[seg])
269 return (&pm->pm_ptbl[seg][ptn]);
270
271 return (NULL);
272 }
273
274 /*
275 * This is called during initppc, before the system is really initialized.
276 */
277 void
278 pmap_bootstrap(u_int kernelstart, u_int kernelend)
279 {
280 struct mem_region *mp, *mp1;
281 int cnt, i;
282 u_int s, e, sz;
283
284 tlbnext = tlb_nreserved;
285
286 /*
287 * Allocate the kernel page table at the end of
288 * kernel space so it's in the locked TTE.
289 */
290 kernmap = (void *)kernelend;
291
292 /*
293 * Initialize kernel page table.
294 */
295 for (i = 0; i < STSZ; i++) {
296 pmap_kernel()->pm_ptbl[i] = 0;
297 }
298 ctxbusy[0] = ctxbusy[1] = pmap_kernel();
299
300 /*
301 * Announce page-size to the VM-system
302 */
303 uvmexp.pagesize = NBPG;
304 uvm_md_init();
305
306 /*
307 * Get memory.
308 */
309 mem_regions(&mem, &avail);
310 for (mp = mem; mp->size; mp++) {
311 physmem += btoc(mp->size);
312 printf("+%lx,",mp->size);
313 }
314 printf("\n");
315 ppc4xx_tlb_init();
316 /*
317 * Count the number of available entries.
318 */
319 for (cnt = 0, mp = avail; mp->size; mp++)
320 cnt++;
321
322 /*
323 * Page align all regions.
324 * Non-page aligned memory isn't very interesting to us.
325 * Also, sort the entries for ascending addresses.
326 */
327 kernelstart &= ~PGOFSET;
328 kernelend = (kernelend + PGOFSET) & ~PGOFSET;
329 for (mp = avail; mp->size; mp++) {
330 s = mp->start;
331 e = mp->start + mp->size;
332 printf("%08x-%08x -> ",s,e);
333 /*
334 * Check whether this region holds all of the kernel.
335 */
336 if (s < kernelstart && e > kernelend) {
337 avail[cnt].start = kernelend;
338 avail[cnt++].size = e - kernelend;
339 e = kernelstart;
340 }
341 /*
342 * Look whether this regions starts within the kernel.
343 */
344 if (s >= kernelstart && s < kernelend) {
345 if (e <= kernelend)
346 goto empty;
347 s = kernelend;
348 }
349 /*
350 * Now look whether this region ends within the kernel.
351 */
352 if (e > kernelstart && e <= kernelend) {
353 if (s >= kernelstart)
354 goto empty;
355 e = kernelstart;
356 }
357 /*
358 * Now page align the start and size of the region.
359 */
360 s = round_page(s);
361 e = trunc_page(e);
362 if (e < s)
363 e = s;
364 sz = e - s;
365 printf("%08x-%08x = %x\n",s,e,sz);
366 /*
367 * Check whether some memory is left here.
368 */
369 if (sz == 0) {
370 empty:
371 memmove(mp, mp + 1,
372 (cnt - (mp - avail)) * sizeof *mp);
373 cnt--;
374 mp--;
375 continue;
376 }
377 /*
378 * Do an insertion sort.
379 */
380 npgs += btoc(sz);
381 for (mp1 = avail; mp1 < mp; mp1++)
382 if (s < mp1->start)
383 break;
384 if (mp1 < mp) {
385 memmove(mp1 + 1, mp1, (char *)mp - (char *)mp1);
386 mp1->start = s;
387 mp1->size = sz;
388 } else {
389 mp->start = s;
390 mp->size = sz;
391 }
392 }
393
394 /*
395 * We cannot do pmap_steal_memory here,
396 * since we don't run with translation enabled yet.
397 */
398 #ifndef MSGBUFADDR
399 /*
400 * allow for msgbuf
401 */
402 sz = round_page(MSGBUFSIZE);
403 mp = NULL;
404 for (mp1 = avail; mp1->size; mp1++)
405 if (mp1->size >= sz)
406 mp = mp1;
407 if (mp == NULL)
408 panic("not enough memory?");
409
410 npgs -= btoc(sz);
411 msgbuf_paddr = mp->start + mp->size - sz;
412 mp->size -= sz;
413 if (mp->size <= 0)
414 memmove(mp, mp + 1, (cnt - (mp - avail)) * sizeof *mp);
415 #endif
416
417 for (mp = avail; mp->size; mp++)
418 uvm_page_physload(atop(mp->start), atop(mp->start + mp->size),
419 atop(mp->start), atop(mp->start + mp->size),
420 VM_FREELIST_DEFAULT);
421
422 /*
423 * Initialize kernel pmap and hardware.
424 */
425 /* Setup TLB pid allocator so it knows we alreadu using PID 1 */
426 pmap_kernel()->pm_ctx = KERNEL_PID;
427 nextavail = avail->start;
428
429 pmap_bootstrap_done = 1;
430 }
431
432 /*
433 * Restrict given range to physical memory
434 *
435 * (Used by /dev/mem)
436 */
437 void
438 pmap_real_memory(paddr_t *start, psize_t *size)
439 {
440 struct mem_region *mp;
441
442 for (mp = mem; mp->size; mp++) {
443 if (*start + *size > mp->start &&
444 *start < mp->start + mp->size) {
445 if (*start < mp->start) {
446 *size -= mp->start - *start;
447 *start = mp->start;
448 }
449 if (*start + *size > mp->start + mp->size)
450 *size = mp->start + mp->size - *start;
451 return;
452 }
453 }
454 *size = 0;
455 }
456
457 /*
458 * Initialize anything else for pmap handling.
459 * Called during vm_init().
460 */
461 void
462 pmap_init(void)
463 {
464 struct pv_entry *pv;
465 vsize_t sz;
466 vaddr_t addr;
467 int i, s;
468 int bank;
469 char *attr;
470
471 sz = (vsize_t)((sizeof(struct pv_entry) + 1) * npgs);
472 sz = round_page(sz);
473 addr = uvm_km_alloc(kernel_map, sz, 0, UVM_KMF_WIRED | UVM_KMF_ZERO);
474 s = splvm();
475 pv = pv_table = (struct pv_entry *)addr;
476 for (i = npgs; --i >= 0;)
477 pv++->pv_pm = NULL;
478 pmap_attrib = (char *)pv;
479 memset(pv, 0, npgs);
480
481 pv = pv_table;
482 attr = pmap_attrib;
483 for (bank = uvm_physseg_get_first();
484 uvm_physseg_valid_p(bank);
485 bank = uvm_physseg_get_next(bank)) {
486 sz = uvm_physseg_get_end(bank) - uvm_physseg_get_start(bank);
487 uvm_physseg_get_pmseg(bank)->pvent = pv;
488 uvm_physseg_get_pmseg(bank)->attrs = attr;
489 pv += sz;
490 attr += sz;
491 }
492
493 pmap_initialized = 1;
494 splx(s);
495
496 /* Setup a pool for additional pvlist structures */
497 pool_init(&pv_pool, sizeof(struct pv_entry), 0, 0, 0, "pv_entry", NULL,
498 IPL_VM);
499 }
500
501 /*
502 * How much virtual space is available to the kernel?
503 */
504 void
505 pmap_virtual_space(vaddr_t *start, vaddr_t *end)
506 {
507
508 #if 0
509 /*
510 * Reserve one segment for kernel virtual memory
511 */
512 *start = (vaddr_t)(KERNEL_SR << ADDR_SR_SHFT);
513 *end = *start + SEGMENT_LENGTH;
514 #else
515 *start = (vaddr_t) VM_MIN_KERNEL_ADDRESS;
516 *end = (vaddr_t) VM_MAX_KERNEL_ADDRESS;
517 #endif
518 }
519
520 #ifdef PMAP_GROWKERNEL
521 /*
522 * Preallocate kernel page tables to a specified VA.
523 * This simply loops through the first TTE for each
524 * page table from the beginning of the kernel pmap,
525 * reads the entry, and if the result is
526 * zero (either invalid entry or no page table) it stores
527 * a zero there, populating page tables in the process.
528 * This is not the most efficient technique but i don't
529 * expect it to be called that often.
530 */
531 extern struct vm_page *vm_page_alloc1(void);
532 extern void vm_page_free1(struct vm_page *);
533
534 vaddr_t kbreak = VM_MIN_KERNEL_ADDRESS;
535
536 vaddr_t
537 pmap_growkernel(vaddr_t maxkvaddr)
538 {
539 int s;
540 int seg;
541 paddr_t pg;
542 struct pmap *pm = pmap_kernel();
543
544 s = splvm();
545
546 /* Align with the start of a page table */
547 for (kbreak &= ~(PTMAP-1); kbreak < maxkvaddr;
548 kbreak += PTMAP) {
549 seg = STIDX(kbreak);
550
551 if (pte_find(pm, kbreak))
552 continue;
553
554 if (uvm.page_init_done) {
555 pg = (paddr_t)VM_PAGE_TO_PHYS(vm_page_alloc1());
556 } else {
557 if (!uvm_page_physget(&pg))
558 panic("pmap_growkernel: no memory");
559 }
560 if (!pg)
561 panic("pmap_growkernel: no pages");
562 pmap_zero_page((paddr_t)pg);
563
564 /* XXX This is based on all phymem being addressable */
565 pm->pm_ptbl[seg] = (u_int *)pg;
566 }
567 splx(s);
568 return (kbreak);
569 }
570
571 /*
572 * vm_page_alloc1:
573 *
574 * Allocate and return a memory cell with no associated object.
575 */
576 struct vm_page *
577 vm_page_alloc1(void)
578 {
579 struct vm_page *pg;
580
581 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE);
582 if (pg) {
583 pg->wire_count = 1; /* no mappings yet */
584 pg->flags &= ~PG_BUSY; /* never busy */
585 }
586 return pg;
587 }
588
589 /*
590 * vm_page_free1:
591 *
592 * Returns the given page to the free list,
593 * disassociating it with any VM object.
594 *
595 * Object and page must be locked prior to entry.
596 */
597 void
598 vm_page_free1(struct vm_page *pg)
599 {
600 #ifdef DIAGNOSTIC
601 if (pg->flags != (PG_CLEAN|PG_FAKE)) {
602 printf("Freeing invalid page %p\n", pg);
603 printf("pa = %llx\n", (unsigned long long)VM_PAGE_TO_PHYS(pg));
604 #ifdef DDB
605 Debugger();
606 #endif
607 return;
608 }
609 #endif
610 pg->flags |= PG_BUSY;
611 pg->wire_count = 0;
612 uvm_pagefree(pg);
613 }
614 #endif
615
616 /*
617 * Create and return a physical map.
618 */
619 struct pmap *
620 pmap_create(void)
621 {
622 struct pmap *pm;
623
624 pm = kmem_alloc(sizeof(*pm), KM_SLEEP);
625 memset(pm, 0, sizeof *pm);
626 pm->pm_refs = 1;
627 return pm;
628 }
629
630 /*
631 * Add a reference to the given pmap.
632 */
633 void
634 pmap_reference(struct pmap *pm)
635 {
636
637 pm->pm_refs++;
638 }
639
640 /*
641 * Retire the given pmap from service.
642 * Should only be called if the map contains no valid mappings.
643 */
644 void
645 pmap_destroy(struct pmap *pm)
646 {
647 int i;
648
649 if (--pm->pm_refs > 0) {
650 return;
651 }
652 KASSERT(pm->pm_stats.resident_count == 0);
653 KASSERT(pm->pm_stats.wired_count == 0);
654 for (i = 0; i < STSZ; i++)
655 if (pm->pm_ptbl[i]) {
656 uvm_km_free(kernel_map, (vaddr_t)pm->pm_ptbl[i],
657 PAGE_SIZE, UVM_KMF_WIRED);
658 pm->pm_ptbl[i] = NULL;
659 }
660 if (pm->pm_ctx)
661 ctx_free(pm);
662 kmem_free(pm, sizeof(*pm));
663 }
664
665 /*
666 * Copy the range specified by src_addr/len
667 * from the source map to the range dst_addr/len
668 * in the destination map.
669 *
670 * This routine is only advisory and need not do anything.
671 */
672 void
673 pmap_copy(struct pmap *dst_pmap, struct pmap *src_pmap, vaddr_t dst_addr,
674 vsize_t len, vaddr_t src_addr)
675 {
676 }
677
678 /*
679 * Require that all active physical maps contain no
680 * incorrect entries NOW.
681 */
682 void
683 pmap_update(struct pmap *pmap)
684 {
685 }
686
687 /*
688 * Fill the given physical page with zeroes.
689 */
690 void
691 pmap_zero_page(paddr_t pa)
692 {
693
694 #ifdef PPC_4XX_NOCACHE
695 memset((void *)pa, 0, PAGE_SIZE);
696 #else
697 int i;
698
699 for (i = PAGE_SIZE/CACHELINESIZE; i > 0; i--) {
700 __asm volatile ("dcbz 0,%0" :: "r"(pa));
701 pa += CACHELINESIZE;
702 }
703 #endif
704 }
705
706 /*
707 * Copy the given physical source page to its destination.
708 */
709 void
710 pmap_copy_page(paddr_t src, paddr_t dst)
711 {
712
713 memcpy((void *)dst, (void *)src, PAGE_SIZE);
714 dcache_wbinv_page(dst);
715 }
716
717 /*
718 * This returns != 0 on success.
719 */
720 static inline int
721 pmap_enter_pv(struct pmap *pm, vaddr_t va, paddr_t pa, int flags)
722 {
723 struct pv_entry *pv, *npv = NULL;
724 int s;
725
726 if (!pmap_initialized)
727 return 0;
728
729 s = splvm();
730 pv = pa_to_pv(pa);
731 if (!pv->pv_pm) {
732 /*
733 * No entries yet, use header as the first entry.
734 */
735 pv->pv_va = va;
736 pv->pv_pm = pm;
737 pv->pv_next = NULL;
738 } else {
739 /*
740 * There is at least one other VA mapping this page.
741 * Place this entry after the header.
742 */
743 npv = pool_get(&pv_pool, PR_NOWAIT);
744 if (npv == NULL) {
745 if ((flags & PMAP_CANFAIL) == 0)
746 panic("pmap_enter_pv: failed");
747 splx(s);
748 return 0;
749 }
750 npv->pv_va = va;
751 npv->pv_pm = pm;
752 npv->pv_next = pv->pv_next;
753 pv->pv_next = npv;
754 pv = npv;
755 }
756 if (flags & PMAP_WIRED) {
757 PV_WIRE(pv);
758 pm->pm_stats.wired_count++;
759 }
760 splx(s);
761 return (1);
762 }
763
764 static void
765 pmap_remove_pv(struct pmap *pm, vaddr_t va, paddr_t pa)
766 {
767 struct pv_entry *pv, *npv;
768
769 /*
770 * Remove from the PV table.
771 */
772 pv = pa_to_pv(pa);
773 if (!pv)
774 return;
775
776 /*
777 * If it is the first entry on the list, it is actually
778 * in the header and we must copy the following entry up
779 * to the header. Otherwise we must search the list for
780 * the entry. In either case we free the now unused entry.
781 */
782 if (pm == pv->pv_pm && PV_CMPVA(va, pv)) {
783 if (PV_ISWIRED(pv)) {
784 pm->pm_stats.wired_count--;
785 }
786 if ((npv = pv->pv_next)) {
787 *pv = *npv;
788 pool_put(&pv_pool, npv);
789 } else
790 pv->pv_pm = NULL;
791 } else {
792 for (; (npv = pv->pv_next) != NULL; pv = npv)
793 if (pm == npv->pv_pm && PV_CMPVA(va, npv))
794 break;
795 if (npv) {
796 pv->pv_next = npv->pv_next;
797 if (PV_ISWIRED(npv)) {
798 pm->pm_stats.wired_count--;
799 }
800 pool_put(&pv_pool, npv);
801 }
802 }
803 }
804
805 /*
806 * Insert physical page at pa into the given pmap at virtual address va.
807 */
808 int
809 pmap_enter(struct pmap *pm, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
810 {
811 int s;
812 u_int tte;
813 bool managed;
814
815 /*
816 * Have to remove any existing mapping first.
817 */
818 pmap_remove(pm, va, va + PAGE_SIZE);
819
820 if (flags & PMAP_WIRED)
821 flags |= prot;
822
823 managed = uvm_pageismanaged(pa);
824
825 /*
826 * Generate TTE.
827 */
828 tte = TTE_PA(pa);
829 /* XXXX -- need to support multiple page sizes. */
830 tte |= TTE_SZ_16K;
831 #ifdef DIAGNOSTIC
832 if ((flags & (PMAP_NOCACHE | PME_WRITETHROUG)) ==
833 (PMAP_NOCACHE | PME_WRITETHROUG))
834 panic("pmap_enter: uncached & writethrough");
835 #endif
836 if (flags & PMAP_NOCACHE)
837 /* Must be I/O mapping */
838 tte |= TTE_I | TTE_G;
839 #ifdef PPC_4XX_NOCACHE
840 tte |= TTE_I;
841 #else
842 else if (flags & PME_WRITETHROUG)
843 /* Uncached and writethrough are not compatible */
844 tte |= TTE_W;
845 #endif
846 if (pm == pmap_kernel())
847 tte |= TTE_ZONE(ZONE_PRIV);
848 else
849 tte |= TTE_ZONE(ZONE_USER);
850
851 if (flags & VM_PROT_WRITE)
852 tte |= TTE_WR;
853
854 if (flags & VM_PROT_EXECUTE)
855 tte |= TTE_EX;
856
857 /*
858 * Now record mapping for later back-translation.
859 */
860 if (pmap_initialized && managed) {
861 char *attr;
862
863 if (!pmap_enter_pv(pm, va, pa, flags)) {
864 /* Could not enter pv on a managed page */
865 return 1;
866 }
867
868 /* Now set attributes. */
869 attr = pa_to_attr(pa);
870 #ifdef DIAGNOSTIC
871 if (!attr)
872 panic("managed but no attr");
873 #endif
874 if (flags & VM_PROT_ALL)
875 *attr |= PMAP_ATTR_REF;
876 if (flags & VM_PROT_WRITE)
877 *attr |= PMAP_ATTR_CHG;
878 }
879
880 s = splvm();
881
882 /* Insert page into page table. */
883 pte_enter(pm, va, tte);
884
885 /* If this is a real fault, enter it in the tlb */
886 if (tte && ((flags & PMAP_WIRED) == 0)) {
887 int s2 = splhigh();
888 ppc4xx_tlb_enter(pm->pm_ctx, va, tte);
889 splx(s2);
890 }
891 splx(s);
892
893 /* Flush the real memory from the instruction cache. */
894 if ((prot & VM_PROT_EXECUTE) && (tte & TTE_I) == 0)
895 __syncicache((void *)pa, PAGE_SIZE);
896
897 return 0;
898 }
899
900 void
901 pmap_unwire(struct pmap *pm, vaddr_t va)
902 {
903 struct pv_entry *pv;
904 paddr_t pa;
905 int s;
906
907 if (!pmap_extract(pm, va, &pa)) {
908 return;
909 }
910
911 pv = pa_to_pv(pa);
912 if (!pv)
913 return;
914
915 s = splvm();
916 while (pv != NULL) {
917 if (pm == pv->pv_pm && PV_CMPVA(va, pv)) {
918 if (PV_ISWIRED(pv)) {
919 PV_UNWIRE(pv);
920 pm->pm_stats.wired_count--;
921 }
922 break;
923 }
924 pv = pv->pv_next;
925 }
926 splx(s);
927 }
928
929 void
930 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
931 {
932 int s;
933 u_int tte;
934 struct pmap *pm = pmap_kernel();
935
936 /*
937 * Generate TTE.
938 *
939 * XXXX
940 *
941 * Since the kernel does not handle execution privileges properly,
942 * we will handle read and execute permissions together.
943 */
944 tte = 0;
945 if (prot & VM_PROT_ALL) {
946
947 tte = TTE_PA(pa) | TTE_EX | TTE_ZONE(ZONE_PRIV);
948 /* XXXX -- need to support multiple page sizes. */
949 tte |= TTE_SZ_16K;
950 #ifdef DIAGNOSTIC
951 if ((flags & (PMAP_NOCACHE | PME_WRITETHROUG)) ==
952 (PMAP_NOCACHE | PME_WRITETHROUG))
953 panic("pmap_kenter_pa: uncached & writethrough");
954 #endif
955 if (flags & PMAP_NOCACHE)
956 /* Must be I/O mapping */
957 tte |= TTE_I | TTE_G;
958 #ifdef PPC_4XX_NOCACHE
959 tte |= TTE_I;
960 #else
961 else if (prot & PME_WRITETHROUG)
962 /* Uncached and writethrough are not compatible */
963 tte |= TTE_W;
964 #endif
965 if (prot & VM_PROT_WRITE)
966 tte |= TTE_WR;
967 }
968
969 s = splvm();
970
971 /* Insert page into page table. */
972 pte_enter(pm, va, tte);
973 splx(s);
974 }
975
976 void
977 pmap_kremove(vaddr_t va, vsize_t len)
978 {
979
980 while (len > 0) {
981 pte_enter(pmap_kernel(), va, 0);
982 va += PAGE_SIZE;
983 len -= PAGE_SIZE;
984 }
985 }
986
987 /*
988 * Remove the given range of mapping entries.
989 */
990 void
991 pmap_remove(struct pmap *pm, vaddr_t va, vaddr_t endva)
992 {
993 int s;
994 paddr_t pa;
995 volatile u_int *ptp;
996
997 s = splvm();
998 while (va < endva) {
999
1000 if ((ptp = pte_find(pm, va)) && (pa = *ptp)) {
1001 pa = TTE_PA(pa);
1002 pmap_remove_pv(pm, va, pa);
1003 *ptp = 0;
1004 ppc4xx_tlb_flush(va, pm->pm_ctx);
1005 pm->pm_stats.resident_count--;
1006 }
1007 va += PAGE_SIZE;
1008 }
1009
1010 splx(s);
1011 }
1012
1013 /*
1014 * Get the physical page address for the given pmap/virtual address.
1015 */
1016 bool
1017 pmap_extract(struct pmap *pm, vaddr_t va, paddr_t *pap)
1018 {
1019 int seg = STIDX(va);
1020 int ptn = PTIDX(va);
1021 u_int pa = 0;
1022 int s;
1023
1024 s = splvm();
1025 if (pm->pm_ptbl[seg] && (pa = pm->pm_ptbl[seg][ptn]) && pap) {
1026 *pap = TTE_PA(pa) | (va & PGOFSET);
1027 }
1028 splx(s);
1029 return (pa != 0);
1030 }
1031
1032 /*
1033 * Lower the protection on the specified range of this pmap.
1034 *
1035 * There are only two cases: either the protection is going to 0,
1036 * or it is going to read-only.
1037 */
1038 void
1039 pmap_protect(struct pmap *pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot)
1040 {
1041 volatile u_int *ptp;
1042 int s, bic;
1043
1044 if ((prot & VM_PROT_READ) == 0) {
1045 pmap_remove(pm, sva, eva);
1046 return;
1047 }
1048 bic = 0;
1049 if ((prot & VM_PROT_WRITE) == 0) {
1050 bic |= TTE_WR;
1051 }
1052 if ((prot & VM_PROT_EXECUTE) == 0) {
1053 bic |= TTE_EX;
1054 }
1055 if (bic == 0) {
1056 return;
1057 }
1058 s = splvm();
1059 while (sva < eva) {
1060 if ((ptp = pte_find(pm, sva)) != NULL) {
1061 *ptp &= ~bic;
1062 ppc4xx_tlb_flush(sva, pm->pm_ctx);
1063 }
1064 sva += PAGE_SIZE;
1065 }
1066 splx(s);
1067 }
1068
1069 bool
1070 pmap_check_attr(struct vm_page *pg, u_int mask, int clear)
1071 {
1072 paddr_t pa;
1073 char *attr;
1074 int s, rv;
1075
1076 /*
1077 * First modify bits in cache.
1078 */
1079 pa = VM_PAGE_TO_PHYS(pg);
1080 attr = pa_to_attr(pa);
1081 if (attr == NULL)
1082 return false;
1083
1084 s = splvm();
1085 rv = ((*attr & mask) != 0);
1086 if (clear) {
1087 *attr &= ~mask;
1088 pmap_page_protect(pg, mask == PMAP_ATTR_CHG ? VM_PROT_READ : 0);
1089 }
1090 splx(s);
1091 return rv;
1092 }
1093
1094
1095 /*
1096 * Lower the protection on the specified physical page.
1097 *
1098 * There are only two cases: either the protection is going to 0,
1099 * or it is going to read-only.
1100 */
1101 void
1102 pmap_page_protect(struct vm_page *pg, vm_prot_t prot)
1103 {
1104 paddr_t pa = VM_PAGE_TO_PHYS(pg);
1105 vaddr_t va;
1106 struct pv_entry *pvh, *pv, *npv;
1107 struct pmap *pm;
1108
1109 pvh = pa_to_pv(pa);
1110 if (pvh == NULL)
1111 return;
1112
1113 /* Handle extra pvs which may be deleted in the operation */
1114 for (pv = pvh->pv_next; pv; pv = npv) {
1115 npv = pv->pv_next;
1116
1117 pm = pv->pv_pm;
1118 va = PV_VA(pv);
1119 pmap_protect(pm, va, va + PAGE_SIZE, prot);
1120 }
1121 /* Now check the head pv */
1122 if (pvh->pv_pm) {
1123 pv = pvh;
1124 pm = pv->pv_pm;
1125 va = PV_VA(pv);
1126 pmap_protect(pm, va, va + PAGE_SIZE, prot);
1127 }
1128 }
1129
1130 /*
1131 * Activate the address space for the specified process. If the process
1132 * is the current process, load the new MMU context.
1133 */
1134 void
1135 pmap_activate(struct lwp *l)
1136 {
1137 #if 0
1138 struct pcb *pcb = lwp_getpcb(l);
1139 pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap;
1140
1141 /*
1142 * XXX Normally performed in cpu_lwp_fork().
1143 */
1144 printf("pmap_activate(%p), pmap=%p\n",l,pmap);
1145 pcb->pcb_pm = pmap;
1146 #endif
1147 }
1148
1149 /*
1150 * Deactivate the specified process's address space.
1151 */
1152 void
1153 pmap_deactivate(struct lwp *l)
1154 {
1155 }
1156
1157 /*
1158 * Synchronize caches corresponding to [addr, addr+len) in p.
1159 */
1160 void
1161 pmap_procwr(struct proc *p, vaddr_t va, size_t len)
1162 {
1163 struct pmap *pm = p->p_vmspace->vm_map.pmap;
1164 int msr, ctx, opid, step;
1165
1166 step = CACHELINESIZE;
1167
1168 /*
1169 * Need to turn off IMMU and switch to user context.
1170 * (icbi uses DMMU).
1171 */
1172 if (!(ctx = pm->pm_ctx)) {
1173 /* No context -- assign it one */
1174 ctx_alloc(pm);
1175 ctx = pm->pm_ctx;
1176 }
1177 __asm volatile(
1178 "mfmsr %0;"
1179 "li %1, %7;"
1180 "andc %1,%0,%1;"
1181 "mtmsr %1;"
1182 "isync;"
1183 "mfpid %1;"
1184 "mtpid %2;"
1185 "isync;"
1186 "1:"
1187 "dcbst 0,%3;"
1188 "icbi 0,%3;"
1189 "add %3,%3,%5;"
1190 "addc. %4,%4,%6;"
1191 "bge 1b;"
1192 "sync;"
1193 "mtpid %1;"
1194 "mtmsr %0;"
1195 "isync;"
1196 : "=&r" (msr), "=&r" (opid)
1197 : "r" (ctx), "r" (va), "r" (len), "r" (step), "r" (-step),
1198 "K" (PSL_IR | PSL_DR));
1199 }
1200
1201 static inline void
1202 tlb_invalidate_entry(int i)
1203 {
1204 #ifdef PMAP_TLBDEBUG
1205 /*
1206 * Clear only TLBHI[V] bit so that we can track invalidated entry.
1207 */
1208 register_t msr, pid, hi;
1209
1210 KASSERT(mfspr(SPR_PID) == KERNEL_PID);
1211
1212 __asm volatile(
1213 "mfmsr %0;"
1214 "li %1,0;"
1215 "mtmsr %1;"
1216 "mfpid %1;"
1217 "tlbre %2,%3,0;"
1218 "andc %2,%2,%4;"
1219 "tlbwe %2,%3,0;"
1220 "mtpid %1;"
1221 "mtmsr %0;"
1222 "isync;"
1223 : "=&r" (msr), "=&r" (pid), "=&r" (hi)
1224 : "r" (i), "r" (TLB_VALID));
1225 #else
1226 /*
1227 * Just clear entire TLBHI register.
1228 */
1229 __asm volatile(
1230 "tlbwe %0,%1,0;"
1231 "isync;"
1232 : : "r" (0), "r" (i));
1233 #endif
1234
1235 tlb_info[i].ti_ctx = 0;
1236 tlb_info[i].ti_flags = 0;
1237 }
1238
1239 /* This has to be done in real mode !!! */
1240 void
1241 ppc4xx_tlb_flush(vaddr_t va, int pid)
1242 {
1243 u_long i, found;
1244 u_long msr;
1245
1246 /* If there's no context then it can't be mapped. */
1247 if (!pid)
1248 return;
1249
1250 __asm volatile(
1251 "mfpid %1;" /* Save PID */
1252 "mfmsr %2;" /* Save MSR */
1253 "li %0,0;" /* Now clear MSR */
1254 "mtmsr %0;"
1255 "isync;"
1256 "mtpid %4;" /* Set PID */
1257 "isync;"
1258 "tlbsx. %0,0,%3;" /* Search TLB */
1259 "isync;"
1260 "mtpid %1;" /* Restore PID */
1261 "mtmsr %2;" /* Restore MSR */
1262 "isync;"
1263 "li %1,1;"
1264 "beq 1f;"
1265 "li %1,0;"
1266 "1:"
1267 : "=&r" (i), "=&r" (found), "=&r" (msr)
1268 : "r" (va), "r" (pid));
1269 if (found && !TLB_LOCKED(i)) {
1270 /* Now flush translation */
1271 tlb_invalidate_entry(i);
1272 tlbnext = i;
1273 /* Successful flushes */
1274 tlbflush_ev.ev_count++;
1275 }
1276 }
1277
1278 void
1279 ppc4xx_tlb_flush_all(void)
1280 {
1281 u_long i;
1282
1283 for (i = 0; i < NTLB; i++)
1284 if (!TLB_LOCKED(i))
1285 tlb_invalidate_entry(i);
1286
1287 __asm volatile("isync");
1288 }
1289
1290 /* Find a TLB entry to evict. */
1291 static int
1292 ppc4xx_tlb_find_victim(void)
1293 {
1294 int flags;
1295
1296 for (;;) {
1297 if (++tlbnext >= NTLB)
1298 tlbnext = tlb_nreserved;
1299 flags = tlb_info[tlbnext].ti_flags;
1300 if (!(flags & TLBF_USED) ||
1301 (flags & (TLBF_LOCKED | TLBF_REF)) == 0) {
1302 u_long va, stack = (u_long)&va;
1303
1304 if (!((tlb_info[tlbnext].ti_va ^ stack) & (~PGOFSET)) &&
1305 (tlb_info[tlbnext].ti_ctx == KERNEL_PID) &&
1306 (flags & TLBF_USED)) {
1307 /* Kernel stack page */
1308 flags |= TLBF_REF;
1309 tlb_info[tlbnext].ti_flags = flags;
1310 } else {
1311 /* Found it! */
1312 return (tlbnext);
1313 }
1314 } else {
1315 tlb_info[tlbnext].ti_flags = (flags & ~TLBF_REF);
1316 }
1317 }
1318 }
1319
1320 void
1321 ppc4xx_tlb_enter(int ctx, vaddr_t va, u_int pte)
1322 {
1323 u_long th, tl, idx;
1324 int msr, pid;
1325 paddr_t pa;
1326 int sz;
1327
1328 tlbenter_ev.ev_count++;
1329
1330 sz = (pte & TTE_SZ_MASK) >> TTE_SZ_SHIFT;
1331 pa = (pte & TTE_RPN_MASK(sz));
1332 th = (va & TLB_EPN_MASK) | (sz << TLB_SIZE_SHFT) | TLB_VALID;
1333 tl = (pte & ~TLB_RPN_MASK) | pa;
1334 tl |= ppc4xx_tlbflags(va, pa);
1335
1336 idx = ppc4xx_tlb_find_victim();
1337
1338 #ifdef DIAGNOSTIC
1339 if ((idx < tlb_nreserved) || (idx >= NTLB) || (idx & 63) == 0) {
1340 panic("ppc4xx_tlb_enter: replacing entry %ld", idx);
1341 }
1342 #endif
1343
1344 tlb_info[idx].ti_va = (va & TLB_EPN_MASK);
1345 tlb_info[idx].ti_ctx = ctx;
1346 tlb_info[idx].ti_flags = TLBF_USED | TLBF_REF;
1347
1348 __asm volatile(
1349 "mfmsr %0;" /* Save MSR */
1350 "li %1,0;"
1351 "mtmsr %1;" /* Clear MSR */
1352 "isync;"
1353 "tlbwe %1,%3,0;" /* Invalidate old entry. */
1354 "mfpid %1;" /* Save old PID */
1355 "mtpid %2;" /* Load translation ctx */
1356 "isync;"
1357 "tlbwe %4,%3,1; tlbwe %5,%3,0;" /* Set TLB */
1358 "isync;"
1359 "mtpid %1; mtmsr %0;" /* Restore PID and MSR */
1360 "isync;"
1361 : "=&r" (msr), "=&r" (pid)
1362 : "r" (ctx), "r" (idx), "r" (tl), "r" (th));
1363 }
1364
1365 void
1366 ppc4xx_tlb_init(void)
1367 {
1368 int i;
1369
1370 /* Mark reserved TLB entries */
1371 for (i = 0; i < tlb_nreserved; i++) {
1372 tlb_info[i].ti_flags = TLBF_LOCKED | TLBF_USED;
1373 tlb_info[i].ti_ctx = KERNEL_PID;
1374 }
1375
1376 /* Setup security zones */
1377 /* Z0 - accessible by kernel only if TLB entry permissions allow
1378 * Z1,Z2 - access is controlled by TLB entry permissions
1379 * Z3 - full access regardless of TLB entry permissions
1380 */
1381
1382 __asm volatile(
1383 "mtspr %0,%1;"
1384 "isync;"
1385 :: "K"(SPR_ZPR), "r" (0x1b000000));
1386 }
1387
1388 /*
1389 * ppc4xx_tlb_size_mask:
1390 *
1391 * Roundup size to supported page size, return TLBHI mask and real size.
1392 */
1393 static int
1394 ppc4xx_tlb_size_mask(size_t size, int *mask, int *rsiz)
1395 {
1396 int i;
1397
1398 for (i = 0; i < __arraycount(tlbsize); i++)
1399 if (size <= tlbsize[i]) {
1400 *mask = (i << TLB_SIZE_SHFT);
1401 *rsiz = tlbsize[i];
1402 return (0);
1403 }
1404 return (EINVAL);
1405 }
1406
1407 /*
1408 * ppc4xx_tlb_mapiodev:
1409 *
1410 * Lookup virtual address of mapping previously entered via
1411 * ppc4xx_tlb_reserve. Search TLB directly so that we don't
1412 * need to waste extra storage for reserved mappings. Note
1413 * that reading TLBHI also sets PID, but all reserved mappings
1414 * use KERNEL_PID, so the side effect is nil.
1415 */
1416 void *
1417 ppc4xx_tlb_mapiodev(paddr_t base, psize_t len)
1418 {
1419 paddr_t pa;
1420 vaddr_t va;
1421 u_int lo, hi, sz;
1422 int i;
1423
1424 /* tlb_nreserved is only allowed to grow, so this is safe. */
1425 for (i = 0; i < tlb_nreserved; i++) {
1426 __asm volatile (
1427 " tlbre %0,%2,1 \n" /* TLBLO */
1428 " tlbre %1,%2,0 \n" /* TLBHI */
1429 : "=&r" (lo), "=&r" (hi)
1430 : "r" (i));
1431
1432 KASSERT(hi & TLB_VALID);
1433 KASSERT(mfspr(SPR_PID) == KERNEL_PID);
1434
1435 pa = (lo & TLB_RPN_MASK);
1436 if (base < pa)
1437 continue;
1438
1439 sz = tlbsize[(hi & TLB_SIZE_MASK) >> TLB_SIZE_SHFT];
1440 if ((base + len) > (pa + sz))
1441 continue;
1442
1443 va = (hi & TLB_EPN_MASK) + (base & (sz - 1)); /* sz = 2^n */
1444 return (void *)(va);
1445 }
1446
1447 return (NULL);
1448 }
1449
1450 /*
1451 * ppc4xx_tlb_reserve:
1452 *
1453 * Map physical range to kernel virtual chunk via reserved TLB entry.
1454 */
1455 void
1456 ppc4xx_tlb_reserve(paddr_t pa, vaddr_t va, size_t size, int flags)
1457 {
1458 u_int lo, hi;
1459 int szmask, rsize;
1460
1461 /* Called before pmap_bootstrap(), va outside kernel space. */
1462 KASSERT(va < VM_MIN_KERNEL_ADDRESS || va >= VM_MAX_KERNEL_ADDRESS);
1463 KASSERT(! pmap_bootstrap_done);
1464 KASSERT(tlb_nreserved < NTLB);
1465
1466 /* Resolve size. */
1467 if (ppc4xx_tlb_size_mask(size, &szmask, &rsize) != 0)
1468 panic("ppc4xx_tlb_reserve: entry %d, %zuB too large",
1469 size, tlb_nreserved);
1470
1471 /* Real size will be power of two >= 1024, so this is OK. */
1472 pa &= ~(rsize - 1); /* RPN */
1473 va &= ~(rsize - 1); /* EPN */
1474
1475 lo = pa | TLB_WR | flags;
1476 hi = va | TLB_VALID | szmask;
1477
1478 #ifdef PPC_4XX_NOCACHE
1479 lo |= TLB_I;
1480 #endif
1481
1482 __asm volatile(
1483 " tlbwe %1,%0,1 \n" /* write TLBLO */
1484 " tlbwe %2,%0,0 \n" /* write TLBHI */
1485 " isync \n"
1486 : : "r" (tlb_nreserved), "r" (lo), "r" (hi));
1487
1488 tlb_nreserved++;
1489 }
1490
1491 /*
1492 * We should pass the ctx in from trap code.
1493 */
1494 int
1495 pmap_tlbmiss(vaddr_t va, int ctx)
1496 {
1497 volatile u_int *pte;
1498 u_long tte;
1499
1500 tlbmiss_ev.ev_count++;
1501
1502 /*
1503 * We will reserve 0 upto VM_MIN_KERNEL_ADDRESS for va == pa mappings.
1504 * Physical RAM is expected to live in this range, care must be taken
1505 * to not clobber 0 upto ${physmem} with device mappings in machdep
1506 * code.
1507 */
1508 if (ctx != KERNEL_PID ||
1509 (va >= VM_MIN_KERNEL_ADDRESS && va < VM_MAX_KERNEL_ADDRESS)) {
1510 pte = pte_find((struct pmap *)__UNVOLATILE(ctxbusy[ctx]), va);
1511 if (pte == NULL) {
1512 /* Map unmanaged addresses directly for kernel access */
1513 return 1;
1514 }
1515 tte = *pte;
1516 if (tte == 0) {
1517 return 1;
1518 }
1519 } else {
1520 /* Create a 16MB writable mapping. */
1521 #ifdef PPC_4XX_NOCACHE
1522 tte = TTE_PA(va) | TTE_ZONE(ZONE_PRIV) | TTE_SZ_16M | TTE_I |TTE_WR;
1523 #else
1524 tte = TTE_PA(va) | TTE_ZONE(ZONE_PRIV) | TTE_SZ_16M | TTE_WR;
1525 #endif
1526 }
1527 ppc4xx_tlb_enter(ctx, va, tte);
1528
1529 return 0;
1530 }
1531
1532 /*
1533 * Flush all the entries matching a context from the TLB.
1534 */
1535 static int
1536 ctx_flush(int cnum)
1537 {
1538 int i;
1539
1540 /* We gotta steal this context */
1541 for (i = tlb_nreserved; i < NTLB; i++) {
1542 if (tlb_info[i].ti_ctx == cnum) {
1543 /* Can't steal ctx if it has a locked entry. */
1544 if (TLB_LOCKED(i)) {
1545 #ifdef DIAGNOSTIC
1546 printf("ctx_flush: can't invalidate "
1547 "locked mapping %d "
1548 "for context %d\n", i, cnum);
1549 #ifdef DDB
1550 Debugger();
1551 #endif
1552 #endif
1553 return (1);
1554 }
1555 #ifdef DIAGNOSTIC
1556 if (i < tlb_nreserved)
1557 panic("TLB entry %d not locked", i);
1558 #endif
1559 /*
1560 * Invalidate particular TLB entry regardless of
1561 * locked status
1562 */
1563 tlb_invalidate_entry(i);
1564 }
1565 }
1566 return (0);
1567 }
1568
1569 /*
1570 * Allocate a context. If necessary, steal one from someone else.
1571 *
1572 * The new context is flushed from the TLB before returning.
1573 */
1574 int
1575 ctx_alloc(struct pmap *pm)
1576 {
1577 int s, cnum;
1578 static int next = MINCTX;
1579
1580 if (pm == pmap_kernel()) {
1581 #ifdef DIAGNOSTIC
1582 printf("ctx_alloc: kernel pmap!\n");
1583 #endif
1584 return (0);
1585 }
1586 s = splvm();
1587
1588 /* Find a likely context. */
1589 cnum = next;
1590 do {
1591 if ((++cnum) >= NUMCTX)
1592 cnum = MINCTX;
1593 } while (ctxbusy[cnum] != NULL && cnum != next);
1594
1595 /* Now clean it out */
1596 oops:
1597 if (cnum < MINCTX)
1598 cnum = MINCTX; /* Never steal ctx 0 or 1 */
1599 if (ctx_flush(cnum)) {
1600 /* oops -- something's wired. */
1601 if ((++cnum) >= NUMCTX)
1602 cnum = MINCTX;
1603 goto oops;
1604 }
1605
1606 if (ctxbusy[cnum]) {
1607 #ifdef DEBUG
1608 /* We should identify this pmap and clear it */
1609 printf("Warning: stealing context %d\n", cnum);
1610 #endif
1611 ctxbusy[cnum]->pm_ctx = 0;
1612 }
1613 ctxbusy[cnum] = pm;
1614 next = cnum;
1615 splx(s);
1616 pm->pm_ctx = cnum;
1617
1618 return cnum;
1619 }
1620
1621 /*
1622 * Give away a context.
1623 */
1624 void
1625 ctx_free(struct pmap *pm)
1626 {
1627 int oldctx;
1628
1629 oldctx = pm->pm_ctx;
1630
1631 if (oldctx == 0)
1632 panic("ctx_free: freeing kernel context");
1633 #ifdef DIAGNOSTIC
1634 if (ctxbusy[oldctx] == 0)
1635 printf("ctx_free: freeing free context %d\n", oldctx);
1636 if (ctxbusy[oldctx] != pm) {
1637 printf("ctx_free: freeing someone esle's context\n "
1638 "ctxbusy[%d] = %p, pm->pm_ctx = %p\n",
1639 oldctx, (void *)(u_long)ctxbusy[oldctx], pm);
1640 #ifdef DDB
1641 Debugger();
1642 #endif
1643 }
1644 #endif
1645 /* We should verify it has not been stolen and reallocated... */
1646 ctxbusy[oldctx] = NULL;
1647 ctx_flush(oldctx);
1648 }
1649
1650
1651 #ifdef DEBUG
1652 /*
1653 * Test ref/modify handling.
1654 */
1655 void pmap_testout(void);
1656 void
1657 pmap_testout(void)
1658 {
1659 vaddr_t va;
1660 volatile int *loc;
1661 int val = 0;
1662 paddr_t pa;
1663 struct vm_page *pg;
1664 int ref, mod;
1665
1666 /* Allocate a page */
1667 va = (vaddr_t)uvm_km_alloc(kernel_map, PAGE_SIZE, 0,
1668 UVM_KMF_WIRED | UVM_KMF_ZERO);
1669 loc = (int*)va;
1670
1671 pmap_extract(pmap_kernel(), va, &pa);
1672 pg = PHYS_TO_VM_PAGE(pa);
1673 pmap_unwire(pmap_kernel(), va);
1674
1675 pmap_kremove(va, PAGE_SIZE);
1676 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0);
1677 pmap_update(pmap_kernel());
1678
1679 /* Now clear reference and modify */
1680 ref = pmap_clear_reference(pg);
1681 mod = pmap_clear_modify(pg);
1682 printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
1683 (void *)(u_long)va, (long)pa,
1684 ref, mod);
1685
1686 /* Check it's properly cleared */
1687 ref = pmap_is_referenced(pg);
1688 mod = pmap_is_modified(pg);
1689 printf("Checking cleared page: ref %d, mod %d\n",
1690 ref, mod);
1691
1692 /* Reference page */
1693 val = *loc;
1694
1695 ref = pmap_is_referenced(pg);
1696 mod = pmap_is_modified(pg);
1697 printf("Referenced page: ref %d, mod %d val %x\n",
1698 ref, mod, val);
1699
1700 /* Now clear reference and modify */
1701 ref = pmap_clear_reference(pg);
1702 mod = pmap_clear_modify(pg);
1703 printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
1704 (void *)(u_long)va, (long)pa,
1705 ref, mod);
1706
1707 /* Modify page */
1708 *loc = 1;
1709
1710 ref = pmap_is_referenced(pg);
1711 mod = pmap_is_modified(pg);
1712 printf("Modified page: ref %d, mod %d\n",
1713 ref, mod);
1714
1715 /* Now clear reference and modify */
1716 ref = pmap_clear_reference(pg);
1717 mod = pmap_clear_modify(pg);
1718 printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
1719 (void *)(u_long)va, (long)pa,
1720 ref, mod);
1721
1722 /* Check it's properly cleared */
1723 ref = pmap_is_referenced(pg);
1724 mod = pmap_is_modified(pg);
1725 printf("Checking cleared page: ref %d, mod %d\n",
1726 ref, mod);
1727
1728 /* Modify page */
1729 *loc = 1;
1730
1731 ref = pmap_is_referenced(pg);
1732 mod = pmap_is_modified(pg);
1733 printf("Modified page: ref %d, mod %d\n",
1734 ref, mod);
1735
1736 /* Check pmap_protect() */
1737 pmap_protect(pmap_kernel(), va, va+1, VM_PROT_READ);
1738 pmap_update(pmap_kernel());
1739 ref = pmap_is_referenced(pg);
1740 mod = pmap_is_modified(pg);
1741 printf("pmap_protect(VM_PROT_READ): ref %d, mod %d\n",
1742 ref, mod);
1743
1744 /* Now clear reference and modify */
1745 ref = pmap_clear_reference(pg);
1746 mod = pmap_clear_modify(pg);
1747 printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
1748 (void *)(u_long)va, (long)pa,
1749 ref, mod);
1750
1751 /* Reference page */
1752 val = *loc;
1753
1754 ref = pmap_is_referenced(pg);
1755 mod = pmap_is_modified(pg);
1756 printf("Referenced page: ref %d, mod %d val %x\n",
1757 ref, mod, val);
1758
1759 /* Now clear reference and modify */
1760 ref = pmap_clear_reference(pg);
1761 mod = pmap_clear_modify(pg);
1762 printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
1763 (void *)(u_long)va, (long)pa,
1764 ref, mod);
1765
1766 /* Modify page */
1767 #if 0
1768 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0);
1769 pmap_update(pmap_kernel());
1770 #endif
1771 *loc = 1;
1772
1773 ref = pmap_is_referenced(pg);
1774 mod = pmap_is_modified(pg);
1775 printf("Modified page: ref %d, mod %d\n",
1776 ref, mod);
1777
1778 /* Check pmap_protect() */
1779 pmap_protect(pmap_kernel(), va, va+1, VM_PROT_NONE);
1780 pmap_update(pmap_kernel());
1781 ref = pmap_is_referenced(pg);
1782 mod = pmap_is_modified(pg);
1783 printf("pmap_protect(): ref %d, mod %d\n",
1784 ref, mod);
1785
1786 /* Now clear reference and modify */
1787 ref = pmap_clear_reference(pg);
1788 mod = pmap_clear_modify(pg);
1789 printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
1790 (void *)(u_long)va, (long)pa,
1791 ref, mod);
1792
1793 /* Reference page */
1794 val = *loc;
1795
1796 ref = pmap_is_referenced(pg);
1797 mod = pmap_is_modified(pg);
1798 printf("Referenced page: ref %d, mod %d val %x\n",
1799 ref, mod, val);
1800
1801 /* Now clear reference and modify */
1802 ref = pmap_clear_reference(pg);
1803 mod = pmap_clear_modify(pg);
1804 printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
1805 (void *)(u_long)va, (long)pa,
1806 ref, mod);
1807
1808 /* Modify page */
1809 #if 0
1810 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0);
1811 pmap_update(pmap_kernel());
1812 #endif
1813 *loc = 1;
1814
1815 ref = pmap_is_referenced(pg);
1816 mod = pmap_is_modified(pg);
1817 printf("Modified page: ref %d, mod %d\n",
1818 ref, mod);
1819
1820 /* Check pmap_pag_protect() */
1821 pmap_page_protect(pg, VM_PROT_READ);
1822 ref = pmap_is_referenced(pg);
1823 mod = pmap_is_modified(pg);
1824 printf("pmap_page_protect(VM_PROT_READ): ref %d, mod %d\n",
1825 ref, mod);
1826
1827 /* Now clear reference and modify */
1828 ref = pmap_clear_reference(pg);
1829 mod = pmap_clear_modify(pg);
1830 printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
1831 (void *)(u_long)va, (long)pa,
1832 ref, mod);
1833
1834 /* Reference page */
1835 val = *loc;
1836
1837 ref = pmap_is_referenced(pg);
1838 mod = pmap_is_modified(pg);
1839 printf("Referenced page: ref %d, mod %d val %x\n",
1840 ref, mod, val);
1841
1842 /* Now clear reference and modify */
1843 ref = pmap_clear_reference(pg);
1844 mod = pmap_clear_modify(pg);
1845 printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
1846 (void *)(u_long)va, (long)pa,
1847 ref, mod);
1848
1849 /* Modify page */
1850 #if 0
1851 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0);
1852 pmap_update(pmap_kernel());
1853 #endif
1854 *loc = 1;
1855
1856 ref = pmap_is_referenced(pg);
1857 mod = pmap_is_modified(pg);
1858 printf("Modified page: ref %d, mod %d\n",
1859 ref, mod);
1860
1861 /* Check pmap_pag_protect() */
1862 pmap_page_protect(pg, VM_PROT_NONE);
1863 ref = pmap_is_referenced(pg);
1864 mod = pmap_is_modified(pg);
1865 printf("pmap_page_protect(): ref %d, mod %d\n",
1866 ref, mod);
1867
1868 /* Now clear reference and modify */
1869 ref = pmap_clear_reference(pg);
1870 mod = pmap_clear_modify(pg);
1871 printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
1872 (void *)(u_long)va, (long)pa,
1873 ref, mod);
1874
1875
1876 /* Reference page */
1877 val = *loc;
1878
1879 ref = pmap_is_referenced(pg);
1880 mod = pmap_is_modified(pg);
1881 printf("Referenced page: ref %d, mod %d val %x\n",
1882 ref, mod, val);
1883
1884 /* Now clear reference and modify */
1885 ref = pmap_clear_reference(pg);
1886 mod = pmap_clear_modify(pg);
1887 printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
1888 (void *)(u_long)va, (long)pa,
1889 ref, mod);
1890
1891 /* Modify page */
1892 #if 0
1893 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0);
1894 pmap_update(pmap_kernel());
1895 #endif
1896 *loc = 1;
1897
1898 ref = pmap_is_referenced(pg);
1899 mod = pmap_is_modified(pg);
1900 printf("Modified page: ref %d, mod %d\n",
1901 ref, mod);
1902
1903 /* Unmap page */
1904 pmap_remove(pmap_kernel(), va, va+1);
1905 pmap_update(pmap_kernel());
1906 ref = pmap_is_referenced(pg);
1907 mod = pmap_is_modified(pg);
1908 printf("Unmapped page: ref %d, mod %d\n", ref, mod);
1909
1910 /* Now clear reference and modify */
1911 ref = pmap_clear_reference(pg);
1912 mod = pmap_clear_modify(pg);
1913 printf("Clearing page va %p pa %lx: ref %d, mod %d\n",
1914 (void *)(u_long)va, (long)pa, ref, mod);
1915
1916 /* Check it's properly cleared */
1917 ref = pmap_is_referenced(pg);
1918 mod = pmap_is_modified(pg);
1919 printf("Checking cleared page: ref %d, mod %d\n",
1920 ref, mod);
1921
1922 pmap_remove(pmap_kernel(), va, va + PAGE_SIZE);
1923 pmap_kenter_pa(va, pa, VM_PROT_ALL, 0);
1924 uvm_km_free(kernel_map, (vaddr_t)va, PAGE_SIZE, UVM_KMF_WIRED);
1925 }
1926 #endif
1927