Home | History | Annotate | Line # | Download | only in ofwboot
loadfile_machdep.c revision 1.10.18.1
      1  1.10.18.1     rmind /*	$NetBSD: loadfile_machdep.c,v 1.10.18.1 2014/05/18 17:45:26 rmind Exp $	*/
      2        1.1       cdi 
      3        1.1       cdi /*-
      4        1.1       cdi  * Copyright (c) 2005 The NetBSD Foundation, Inc.
      5        1.1       cdi  * All rights reserved.
      6        1.1       cdi  *
      7        1.1       cdi  * This work is based on the code contributed by Robert Drehmel to the
      8        1.1       cdi  * FreeBSD project.
      9        1.1       cdi  *
     10        1.1       cdi  * Redistribution and use in source and binary forms, with or without
     11        1.1       cdi  * modification, are permitted provided that the following conditions
     12        1.1       cdi  * are met:
     13        1.1       cdi  * 1. Redistributions of source code must retain the above copyright
     14        1.1       cdi  *    notice, this list of conditions and the following disclaimer.
     15        1.1       cdi  * 2. Redistributions in binary form must reproduce the above copyright
     16        1.1       cdi  *    notice, this list of conditions and the following disclaimer in the
     17        1.1       cdi  *    documentation and/or other materials provided with the distribution.
     18        1.1       cdi  *
     19        1.1       cdi  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     20        1.1       cdi  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     21        1.1       cdi  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     22        1.1       cdi  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     23        1.1       cdi  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     24        1.1       cdi  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     25        1.1       cdi  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     26        1.1       cdi  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     27        1.1       cdi  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     28        1.1       cdi  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     29        1.1       cdi  * POSSIBILITY OF SUCH DAMAGE.
     30        1.1       cdi  */
     31        1.1       cdi 
     32        1.1       cdi #include <lib/libsa/stand.h>
     33        1.8        he #include <lib/libkern/libkern.h>
     34        1.1       cdi 
     35        1.1       cdi #include <machine/pte.h>
     36        1.1       cdi #include <machine/cpu.h>
     37        1.1       cdi #include <machine/ctlreg.h>
     38        1.1       cdi #include <machine/vmparam.h>
     39        1.1       cdi #include <machine/promlib.h>
     40  1.10.18.1     rmind #include <machine/hypervisor.h>
     41        1.1       cdi 
     42        1.1       cdi #include "boot.h"
     43        1.1       cdi #include "openfirm.h"
     44        1.1       cdi 
     45        1.1       cdi 
     46        1.1       cdi #define MAXSEGNUM	50
     47        1.2       uwe #define hi(val)		((uint32_t)(((val) >> 32) & (uint32_t)-1))
     48        1.2       uwe #define lo(val)		((uint32_t)((val) & (uint32_t)-1))
     49        1.1       cdi 
     50        1.1       cdi #define roundup2(x, y)	(((x)+((y)-1))&(~((y)-1)))
     51        1.1       cdi 
     52        1.1       cdi 
     53        1.1       cdi typedef int phandle_t;
     54        1.1       cdi 
     55        1.2       uwe extern void	itlb_enter(vaddr_t, uint32_t, uint32_t);
     56        1.2       uwe extern void	dtlb_enter(vaddr_t, uint32_t, uint32_t);
     57        1.3    martin extern void	dtlb_replace(vaddr_t, uint32_t, uint32_t);
     58        1.1       cdi extern vaddr_t	itlb_va_to_pa(vaddr_t);
     59        1.1       cdi extern vaddr_t	dtlb_va_to_pa(vaddr_t);
     60        1.1       cdi 
     61        1.1       cdi static void	tlb_init(void);
     62  1.10.18.1     rmind static void	tlb_init_sun4u(void);
     63  1.10.18.1     rmind #ifdef SUN4V
     64  1.10.18.1     rmind static void	tlb_init_sun4v(void);
     65  1.10.18.1     rmind #endif
     66  1.10.18.1     rmind void	sparc64_finalize_tlb_sun4u(u_long);
     67  1.10.18.1     rmind #ifdef SUN4V
     68  1.10.18.1     rmind void	sparc64_finalize_tlb_sun4v(u_long);
     69  1.10.18.1     rmind #endif
     70        1.1       cdi static int	mmu_mapin(vaddr_t, vsize_t);
     71  1.10.18.1     rmind static int	mmu_mapin_sun4u(vaddr_t, vsize_t);
     72  1.10.18.1     rmind #ifdef SUN4V
     73  1.10.18.1     rmind static int	mmu_mapin_sun4v(vaddr_t, vsize_t);
     74  1.10.18.1     rmind #endif
     75        1.1       cdi static ssize_t	mmu_read(int, void *, size_t);
     76        1.1       cdi static void*	mmu_memcpy(void *, const void *, size_t);
     77        1.1       cdi static void*	mmu_memset(void *, int, size_t);
     78        1.1       cdi static void	mmu_freeall(void);
     79        1.1       cdi 
     80        1.1       cdi static int	ofw_mapin(vaddr_t, vsize_t);
     81        1.1       cdi static ssize_t	ofw_read(int, void *, size_t);
     82        1.1       cdi static void*	ofw_memcpy(void *, const void *, size_t);
     83        1.1       cdi static void*	ofw_memset(void *, int, size_t);
     84        1.1       cdi static void	ofw_freeall(void);
     85        1.1       cdi 
     86        1.9   tsutsui #if 0
     87        1.1       cdi static int	nop_mapin(vaddr_t, vsize_t);
     88        1.9   tsutsui #endif
     89        1.1       cdi static ssize_t	nop_read(int, void *, size_t);
     90        1.1       cdi static void*	nop_memcpy(void *, const void *, size_t);
     91        1.1       cdi static void*	nop_memset(void *, int, size_t);
     92        1.1       cdi static void	nop_freeall(void);
     93        1.1       cdi 
     94        1.1       cdi 
     95        1.1       cdi struct tlb_entry *dtlb_store = 0;
     96        1.1       cdi struct tlb_entry *itlb_store = 0;
     97        1.1       cdi 
     98        1.1       cdi int dtlb_slot;
     99        1.1       cdi int itlb_slot;
    100        1.1       cdi int dtlb_slot_max;
    101        1.1       cdi int itlb_slot_max;
    102        1.1       cdi 
    103        1.1       cdi static struct kvamap {
    104        1.1       cdi 	uint64_t start;
    105        1.1       cdi 	uint64_t end;
    106        1.1       cdi } kvamap[MAXSEGNUM];
    107        1.1       cdi 
    108        1.1       cdi static struct memsw {
    109        1.1       cdi 	ssize_t	(* read)(int f, void *addr, size_t size);
    110        1.1       cdi 	void*	(* memcpy)(void *dst, const void *src, size_t size);
    111        1.1       cdi 	void*	(* memset)(void *dst, int c, size_t size);
    112        1.1       cdi 	void	(* freeall)(void);
    113        1.1       cdi } memswa[] = {
    114        1.1       cdi 	{ nop_read, nop_memcpy, nop_memset, nop_freeall },
    115        1.1       cdi 	{ ofw_read, ofw_memcpy, ofw_memset, ofw_freeall },
    116        1.1       cdi 	{ mmu_read, mmu_memcpy, mmu_memset, mmu_freeall }
    117        1.1       cdi };
    118        1.1       cdi 
    119        1.1       cdi static struct memsw *memsw = &memswa[0];
    120        1.1       cdi 
    121  1.10.18.1     rmind #ifdef SUN4V
    122  1.10.18.1     rmind static int sun4v = 0;
    123  1.10.18.1     rmind #endif
    124        1.1       cdi 
    125        1.1       cdi /*
    126        1.1       cdi  * Check if a memory region is already mapped. Return length and virtual
    127        1.1       cdi  * address of unmapped sub-region, if any.
    128        1.1       cdi  */
    129        1.1       cdi static uint64_t
    130        1.1       cdi kvamap_extract(vaddr_t va, vsize_t len, vaddr_t *new_va)
    131        1.1       cdi {
    132        1.1       cdi 	int i;
    133        1.1       cdi 
    134        1.1       cdi 	*new_va  = va;
    135        1.1       cdi 	for (i = 0; (len > 0) && (i < MAXSEGNUM); i++) {
    136        1.1       cdi 		if (kvamap[i].start == NULL)
    137        1.1       cdi 			break;
    138        1.1       cdi 		if ((kvamap[i].start <= va) && (va < kvamap[i].end)) {
    139        1.1       cdi 			uint64_t va_len = kvamap[i].end - va + kvamap[i].start;
    140        1.1       cdi 			len = (va_len < len) ? len - va_len : 0;
    141        1.1       cdi 			*new_va = kvamap[i].end;
    142        1.1       cdi 		}
    143        1.1       cdi 	}
    144        1.1       cdi 
    145        1.1       cdi 	return (len);
    146        1.1       cdi }
    147        1.1       cdi 
    148        1.1       cdi /*
    149        1.1       cdi  * Record new kernel mapping.
    150        1.1       cdi  */
    151        1.1       cdi static void
    152        1.1       cdi kvamap_enter(uint64_t va, uint64_t len)
    153        1.1       cdi {
    154        1.1       cdi 	int i;
    155        1.1       cdi 
    156        1.1       cdi 	DPRINTF(("kvamap_enter: %d@%p\n", (int)len, (void*)(u_long)va));
    157        1.1       cdi 	for (i = 0; (len > 0) && (i < MAXSEGNUM); i++) {
    158        1.1       cdi 		if (kvamap[i].start == NULL) {
    159        1.1       cdi 			kvamap[i].start = va;
    160        1.1       cdi 			kvamap[i].end = va + len;
    161        1.1       cdi 			break;
    162        1.1       cdi 		}
    163        1.1       cdi 	}
    164        1.1       cdi 
    165        1.1       cdi 	if (i == MAXSEGNUM) {
    166        1.1       cdi 		panic("Too many allocations requested.");
    167        1.1       cdi 	}
    168        1.1       cdi }
    169        1.1       cdi 
    170        1.1       cdi /*
    171        1.1       cdi  * Initialize TLB as required by MMU mapping functions.
    172        1.1       cdi  */
    173        1.1       cdi static void
    174        1.1       cdi tlb_init(void)
    175        1.1       cdi {
    176        1.1       cdi 	phandle_t root;
    177  1.10.18.1     rmind #ifdef SUN4V
    178        1.1       cdi 	char buf[128];
    179  1.10.18.1     rmind #endif
    180        1.1       cdi 
    181        1.1       cdi 	if (dtlb_store != NULL) {
    182        1.1       cdi 		return;
    183        1.1       cdi 	}
    184        1.1       cdi 
    185  1.10.18.1     rmind 	if ( (root = prom_findroot()) == -1) {
    186  1.10.18.1     rmind 		panic("tlb_init: prom_findroot()");
    187  1.10.18.1     rmind 	}
    188  1.10.18.1     rmind #ifdef SUN4V
    189  1.10.18.1     rmind 	if (_prom_getprop(root, "compatible", buf, sizeof(buf)) > 0 &&
    190  1.10.18.1     rmind 		    strcmp(buf, "sun4v") == 0) {
    191  1.10.18.1     rmind 		tlb_init_sun4v();
    192  1.10.18.1     rmind 		sun4v = 1;
    193  1.10.18.1     rmind 	}
    194  1.10.18.1     rmind 	else {
    195  1.10.18.1     rmind #endif
    196  1.10.18.1     rmind 		tlb_init_sun4u();
    197  1.10.18.1     rmind #ifdef SUN4V
    198  1.10.18.1     rmind 	}
    199  1.10.18.1     rmind #endif
    200  1.10.18.1     rmind 
    201  1.10.18.1     rmind 	dtlb_store = alloc(dtlb_slot_max * sizeof(*dtlb_store));
    202  1.10.18.1     rmind 	itlb_store = alloc(itlb_slot_max * sizeof(*itlb_store));
    203  1.10.18.1     rmind 	if (dtlb_store == NULL || itlb_store == NULL) {
    204  1.10.18.1     rmind 		panic("tlb_init: malloc");
    205  1.10.18.1     rmind 	}
    206  1.10.18.1     rmind 
    207  1.10.18.1     rmind 	dtlb_slot = itlb_slot = 0;
    208  1.10.18.1     rmind }
    209  1.10.18.1     rmind 
    210  1.10.18.1     rmind /*
    211  1.10.18.1     rmind  * Initialize TLB as required by MMU mapping functions - sun4u.
    212  1.10.18.1     rmind  */
    213  1.10.18.1     rmind static void
    214  1.10.18.1     rmind tlb_init_sun4u(void)
    215  1.10.18.1     rmind {
    216  1.10.18.1     rmind 	phandle_t child;
    217  1.10.18.1     rmind 	phandle_t root;
    218  1.10.18.1     rmind 	char buf[128];
    219  1.10.18.1     rmind 	u_int bootcpu;
    220  1.10.18.1     rmind 	u_int cpu;
    221  1.10.18.1     rmind 
    222        1.1       cdi 	bootcpu = get_cpuid();
    223        1.1       cdi 
    224        1.1       cdi 	if ( (root = prom_findroot()) == -1) {
    225        1.1       cdi 		panic("tlb_init: prom_findroot()");
    226        1.1       cdi 	}
    227        1.1       cdi 
    228        1.1       cdi 	for (child = prom_firstchild(root); child != 0;
    229        1.1       cdi 			child = prom_nextsibling(child)) {
    230        1.1       cdi 		if (child == -1) {
    231        1.1       cdi 			panic("tlb_init: OF_child");
    232        1.1       cdi 		}
    233        1.1       cdi 		if (_prom_getprop(child, "device_type", buf, sizeof(buf)) > 0 &&
    234        1.1       cdi 		    strcmp(buf, "cpu") == 0) {
    235        1.1       cdi 			if (_prom_getprop(child, "upa-portid", &cpu,
    236        1.1       cdi 			    sizeof(cpu)) == -1 && _prom_getprop(child, "portid",
    237        1.1       cdi 			    &cpu, sizeof(cpu)) == -1)
    238        1.7  nakayama 				panic("tlb_init: prom_getprop");
    239        1.1       cdi 			if (cpu == bootcpu)
    240        1.1       cdi 				break;
    241        1.1       cdi 		}
    242        1.1       cdi 	}
    243        1.1       cdi 	if (cpu != bootcpu)
    244        1.7  nakayama 		panic("tlb_init: no node for bootcpu?!?!");
    245        1.1       cdi 	if (_prom_getprop(child, "#dtlb-entries", &dtlb_slot_max,
    246        1.1       cdi 	    sizeof(dtlb_slot_max)) == -1 ||
    247        1.1       cdi 	    _prom_getprop(child, "#itlb-entries", &itlb_slot_max,
    248        1.1       cdi 	    sizeof(itlb_slot_max)) == -1)
    249        1.7  nakayama 		panic("tlb_init: prom_getprop");
    250  1.10.18.1     rmind }
    251        1.1       cdi 
    252  1.10.18.1     rmind #ifdef SUN4V
    253  1.10.18.1     rmind /*
    254  1.10.18.1     rmind  * Initialize TLB as required by MMU mapping functions - sun4v.
    255  1.10.18.1     rmind  */
    256  1.10.18.1     rmind static void
    257  1.10.18.1     rmind tlb_init_sun4v(void)
    258  1.10.18.1     rmind {
    259  1.10.18.1     rmind 	psize_t len;
    260  1.10.18.1     rmind 	paddr_t pa;
    261  1.10.18.1     rmind 	int64_t hv_rc;
    262  1.10.18.1     rmind 
    263  1.10.18.1     rmind 	hv_mach_desc((paddr_t)NULL, &len); /* Trick to get actual length */
    264  1.10.18.1     rmind 	if ( !len ) {
    265  1.10.18.1     rmind 		panic("init_tlb: hv_mach_desc() failed");
    266  1.10.18.1     rmind 	}
    267  1.10.18.1     rmind 	pa = OF_alloc_phys(len, 16);
    268  1.10.18.1     rmind 	if ( pa == -1 ) {
    269  1.10.18.1     rmind 		panic("OF_alloc_phys() failed");
    270  1.10.18.1     rmind 	}
    271  1.10.18.1     rmind 	hv_rc = hv_mach_desc(pa, &len);
    272  1.10.18.1     rmind 	if (hv_rc != H_EOK) {
    273  1.10.18.1     rmind 		panic("hv_mach_desc() failed");
    274  1.10.18.1     rmind 	}
    275  1.10.18.1     rmind 	/* XXX dig out TLB node info - 64 is ok for loading the kernel */
    276  1.10.18.1     rmind 	dtlb_slot_max = itlb_slot_max = 64;
    277        1.1       cdi }
    278  1.10.18.1     rmind #endif
    279        1.1       cdi 
    280        1.1       cdi /*
    281        1.1       cdi  * Map requested memory region with permanent 4MB pages.
    282        1.1       cdi  */
    283        1.1       cdi static int
    284        1.1       cdi mmu_mapin(vaddr_t rva, vsize_t len)
    285        1.1       cdi {
    286        1.1       cdi 	len  = roundup2(len + (rva & PAGE_MASK_4M), PAGE_SIZE_4M);
    287        1.1       cdi 	rva &= ~PAGE_MASK_4M;
    288        1.1       cdi 
    289        1.1       cdi 	tlb_init();
    290  1.10.18.1     rmind 
    291  1.10.18.1     rmind #if SUN4V
    292  1.10.18.1     rmind 	if ( sun4v )
    293  1.10.18.1     rmind 		return mmu_mapin_sun4v(rva, len);
    294  1.10.18.1     rmind 	else
    295  1.10.18.1     rmind #endif
    296  1.10.18.1     rmind 		return mmu_mapin_sun4u(rva, len);
    297  1.10.18.1     rmind }
    298  1.10.18.1     rmind 
    299  1.10.18.1     rmind /*
    300  1.10.18.1     rmind  * Map requested memory region with permanent 4MB pages - sun4u.
    301  1.10.18.1     rmind  */
    302  1.10.18.1     rmind static int
    303  1.10.18.1     rmind mmu_mapin_sun4u(vaddr_t rva, vsize_t len)
    304  1.10.18.1     rmind {
    305  1.10.18.1     rmind 	uint64_t data;
    306  1.10.18.1     rmind 	paddr_t pa;
    307  1.10.18.1     rmind 	vaddr_t va, mva;
    308  1.10.18.1     rmind 
    309        1.7  nakayama 	for (pa = (paddr_t)-1; len > 0; rva = va) {
    310        1.1       cdi 		if ( (len = kvamap_extract(rva, len, &va)) == 0) {
    311        1.1       cdi 			/* The rest is already mapped */
    312        1.1       cdi 			break;
    313        1.1       cdi 		}
    314        1.1       cdi 
    315        1.1       cdi 		if (dtlb_va_to_pa(va) == (u_long)-1 ||
    316        1.1       cdi 		    itlb_va_to_pa(va) == (u_long)-1) {
    317        1.1       cdi 			/* Allocate a physical page, claim the virtual area */
    318        1.7  nakayama 			if (pa == (paddr_t)-1) {
    319        1.7  nakayama 				pa = OF_alloc_phys(PAGE_SIZE_4M, PAGE_SIZE_4M);
    320        1.7  nakayama 				if (pa == (paddr_t)-1)
    321        1.1       cdi 					panic("out of memory");
    322        1.7  nakayama 				mva = OF_claim_virt(va, PAGE_SIZE_4M);
    323        1.1       cdi 				if (mva != va) {
    324        1.1       cdi 					panic("can't claim virtual page "
    325        1.1       cdi 					    "(wanted %#lx, got %#lx)",
    326        1.1       cdi 					    va, mva);
    327        1.1       cdi 				}
    328        1.1       cdi 				/* The mappings may have changed, be paranoid. */
    329        1.1       cdi 				continue;
    330        1.1       cdi 			}
    331        1.1       cdi 
    332        1.1       cdi 			/*
    333        1.1       cdi 			 * Actually, we can only allocate two pages less at
    334        1.1       cdi 			 * most (depending on the kernel TSB size).
    335        1.1       cdi 			 */
    336        1.1       cdi 			if (dtlb_slot >= dtlb_slot_max)
    337        1.1       cdi 				panic("mmu_mapin: out of dtlb_slots");
    338        1.1       cdi 			if (itlb_slot >= itlb_slot_max)
    339        1.1       cdi 				panic("mmu_mapin: out of itlb_slots");
    340        1.1       cdi 
    341       1.10  nakayama 			DPRINTF(("mmu_mapin: 0x%lx:0x%x.0x%x\n", va,
    342       1.10  nakayama 			    hi(pa), lo(pa)));
    343        1.1       cdi 
    344  1.10.18.1     rmind 			data = SUN4U_TSB_DATA(0,	/* global */
    345        1.1       cdi 					PGSZ_4M,	/* 4mb page */
    346        1.1       cdi 					pa,		/* phys.address */
    347        1.1       cdi 					1,		/* privileged */
    348        1.1       cdi 					1,		/* write */
    349        1.1       cdi 					1,		/* cache */
    350        1.1       cdi 					1,		/* alias */
    351        1.1       cdi 					1,		/* valid */
    352        1.1       cdi 					0		/* endianness */
    353        1.1       cdi 					);
    354  1.10.18.1     rmind 			data |= SUN4U_TLB_L | SUN4U_TLB_CV; /* locked, virt.cache */
    355        1.1       cdi 
    356        1.1       cdi 			dtlb_store[dtlb_slot].te_pa = pa;
    357        1.1       cdi 			dtlb_store[dtlb_slot].te_va = va;
    358        1.1       cdi 			dtlb_slot++;
    359        1.1       cdi 			dtlb_enter(va, hi(data), lo(data));
    360        1.7  nakayama 			pa = (paddr_t)-1;
    361        1.1       cdi 		}
    362        1.1       cdi 
    363        1.1       cdi 		kvamap_enter(va, PAGE_SIZE_4M);
    364        1.1       cdi 
    365        1.1       cdi 		len -= len > PAGE_SIZE_4M ? PAGE_SIZE_4M : len;
    366        1.1       cdi 		va += PAGE_SIZE_4M;
    367        1.1       cdi 	}
    368        1.1       cdi 
    369        1.7  nakayama 	if (pa != (paddr_t)-1) {
    370        1.1       cdi 		OF_free_phys(pa, PAGE_SIZE_4M);
    371        1.1       cdi 	}
    372        1.1       cdi 
    373        1.1       cdi 	return (0);
    374        1.1       cdi }
    375        1.1       cdi 
    376  1.10.18.1     rmind #ifdef SUN4V
    377  1.10.18.1     rmind /*
    378  1.10.18.1     rmind  * Map requested memory region with permanent 4MB pages - sun4v.
    379  1.10.18.1     rmind  */
    380  1.10.18.1     rmind static int
    381  1.10.18.1     rmind mmu_mapin_sun4v(vaddr_t rva, vsize_t len)
    382  1.10.18.1     rmind {
    383  1.10.18.1     rmind 	uint64_t data;
    384  1.10.18.1     rmind 	paddr_t pa;
    385  1.10.18.1     rmind 	vaddr_t va, mva;
    386  1.10.18.1     rmind 	int64_t hv_rc;
    387  1.10.18.1     rmind 
    388  1.10.18.1     rmind 	for (pa = (paddr_t)-1; len > 0; rva = va) {
    389  1.10.18.1     rmind 		if ( (len = kvamap_extract(rva, len, &va)) == 0) {
    390  1.10.18.1     rmind 			/* The rest is already mapped */
    391  1.10.18.1     rmind 			break;
    392  1.10.18.1     rmind 		}
    393  1.10.18.1     rmind 
    394  1.10.18.1     rmind 		/* Allocate a physical page, claim the virtual area */
    395  1.10.18.1     rmind 		if (pa == (paddr_t)-1) {
    396  1.10.18.1     rmind 			pa = OF_alloc_phys(PAGE_SIZE_4M, PAGE_SIZE_4M);
    397  1.10.18.1     rmind 			if (pa == (paddr_t)-1)
    398  1.10.18.1     rmind 				panic("out of memory");
    399  1.10.18.1     rmind 			mva = OF_claim_virt(va, PAGE_SIZE_4M);
    400  1.10.18.1     rmind 			if (mva != va) {
    401  1.10.18.1     rmind 				panic("can't claim virtual page "
    402  1.10.18.1     rmind 				    "(wanted %#lx, got %#lx)",
    403  1.10.18.1     rmind 				    va, mva);
    404  1.10.18.1     rmind 			}
    405  1.10.18.1     rmind 		}
    406  1.10.18.1     rmind 
    407  1.10.18.1     rmind 		/*
    408  1.10.18.1     rmind 		 * Actually, we can only allocate two pages less at
    409  1.10.18.1     rmind 		 * most (depending on the kernel TSB size).
    410  1.10.18.1     rmind 		 */
    411  1.10.18.1     rmind 		if (dtlb_slot >= dtlb_slot_max)
    412  1.10.18.1     rmind 			panic("mmu_mapin: out of dtlb_slots");
    413  1.10.18.1     rmind 		if (itlb_slot >= itlb_slot_max)
    414  1.10.18.1     rmind 			panic("mmu_mapin: out of itlb_slots");
    415  1.10.18.1     rmind 
    416  1.10.18.1     rmind 		DPRINTF(("mmu_mapin: 0x%lx:0x%x.0x%x\n", va,
    417  1.10.18.1     rmind 		    hi(pa), lo(pa)));
    418  1.10.18.1     rmind 
    419  1.10.18.1     rmind 		data = SUN4V_TSB_DATA(
    420  1.10.18.1     rmind 			0,		/* global */
    421  1.10.18.1     rmind 			PGSZ_4M,	/* 4mb page */
    422  1.10.18.1     rmind 			pa,		/* phys.address */
    423  1.10.18.1     rmind 			1,		/* privileged */
    424  1.10.18.1     rmind 			1,		/* write */
    425  1.10.18.1     rmind 			1,		/* cache */
    426  1.10.18.1     rmind 			1,		/* alias */
    427  1.10.18.1     rmind 			1,		/* valid */
    428  1.10.18.1     rmind 			0		/* endianness */
    429  1.10.18.1     rmind 			);
    430  1.10.18.1     rmind 		data |= SUN4V_TLB_CV; /* virt.cache */
    431  1.10.18.1     rmind 
    432  1.10.18.1     rmind 		dtlb_store[dtlb_slot].te_pa = pa;
    433  1.10.18.1     rmind 		dtlb_store[dtlb_slot].te_va = va;
    434  1.10.18.1     rmind 		dtlb_slot++;
    435  1.10.18.1     rmind 		hv_rc = hv_mmu_map_perm_addr(va, data, MAP_DTLB);
    436  1.10.18.1     rmind 		if ( hv_rc != H_EOK ) {
    437  1.10.18.1     rmind 			panic("hv_mmu_map_perm_addr() failed - rc = %ld", hv_rc);
    438  1.10.18.1     rmind 		}
    439  1.10.18.1     rmind 
    440  1.10.18.1     rmind 		kvamap_enter(va, PAGE_SIZE_4M);
    441  1.10.18.1     rmind 
    442  1.10.18.1     rmind 		pa = (paddr_t)-1;
    443  1.10.18.1     rmind 
    444  1.10.18.1     rmind 		len -= len > PAGE_SIZE_4M ? PAGE_SIZE_4M : len;
    445  1.10.18.1     rmind 		va += PAGE_SIZE_4M;
    446  1.10.18.1     rmind 	}
    447  1.10.18.1     rmind 
    448  1.10.18.1     rmind 	if (pa != (paddr_t)-1) {
    449  1.10.18.1     rmind 		OF_free_phys(pa, PAGE_SIZE_4M);
    450  1.10.18.1     rmind 	}
    451  1.10.18.1     rmind 
    452  1.10.18.1     rmind 	return (0);
    453  1.10.18.1     rmind }
    454  1.10.18.1     rmind #endif
    455  1.10.18.1     rmind 
    456        1.1       cdi static ssize_t
    457        1.1       cdi mmu_read(int f, void *addr, size_t size)
    458        1.1       cdi {
    459        1.1       cdi 	mmu_mapin((vaddr_t)addr, size);
    460        1.1       cdi 	return read(f, addr, size);
    461        1.1       cdi }
    462        1.1       cdi 
    463        1.1       cdi static void*
    464        1.1       cdi mmu_memcpy(void *dst, const void *src, size_t size)
    465        1.1       cdi {
    466        1.1       cdi 	mmu_mapin((vaddr_t)dst, size);
    467        1.1       cdi 	return memcpy(dst, src, size);
    468        1.1       cdi }
    469        1.1       cdi 
    470        1.1       cdi static void*
    471        1.1       cdi mmu_memset(void *dst, int c, size_t size)
    472        1.1       cdi {
    473        1.1       cdi 	mmu_mapin((vaddr_t)dst, size);
    474        1.1       cdi 	return memset(dst, c, size);
    475        1.1       cdi }
    476        1.1       cdi 
    477        1.1       cdi static void
    478        1.1       cdi mmu_freeall(void)
    479        1.1       cdi {
    480        1.1       cdi 	int i;
    481        1.1       cdi 
    482        1.1       cdi 	dtlb_slot = itlb_slot = 0;
    483        1.1       cdi 	for (i = 0; i < MAXSEGNUM; i++) {
    484        1.1       cdi 		/* XXX return all mappings to PROM and unmap the pages! */
    485        1.1       cdi 		kvamap[i].start = kvamap[i].end = 0;
    486        1.1       cdi 	}
    487        1.1       cdi }
    488        1.1       cdi 
    489        1.1       cdi /*
    490        1.1       cdi  * Claim requested memory region in OpenFirmware allocation pool.
    491        1.1       cdi  */
    492        1.1       cdi static int
    493        1.1       cdi ofw_mapin(vaddr_t rva, vsize_t len)
    494        1.1       cdi {
    495        1.1       cdi 	vaddr_t va;
    496        1.1       cdi 
    497        1.1       cdi 	len  = roundup2(len + (rva & PAGE_MASK_4M), PAGE_SIZE_4M);
    498        1.1       cdi 	rva &= ~PAGE_MASK_4M;
    499        1.1       cdi 
    500        1.1       cdi 	if ( (len = kvamap_extract(rva, len, &va)) != 0) {
    501        1.1       cdi 		if (OF_claim((void *)(long)va, len, PAGE_SIZE_4M) == (void*)-1){
    502        1.1       cdi 			panic("ofw_mapin: Cannot claim memory.");
    503        1.1       cdi 		}
    504        1.1       cdi 		kvamap_enter(va, len);
    505        1.1       cdi 	}
    506        1.1       cdi 
    507        1.1       cdi 	return (0);
    508        1.1       cdi }
    509        1.1       cdi 
    510        1.1       cdi static ssize_t
    511        1.1       cdi ofw_read(int f, void *addr, size_t size)
    512        1.1       cdi {
    513        1.1       cdi 	ofw_mapin((vaddr_t)addr, size);
    514        1.1       cdi 	return read(f, addr, size);
    515        1.1       cdi }
    516        1.1       cdi 
    517        1.1       cdi static void*
    518        1.1       cdi ofw_memcpy(void *dst, const void *src, size_t size)
    519        1.1       cdi {
    520        1.1       cdi 	ofw_mapin((vaddr_t)dst, size);
    521        1.1       cdi 	return memcpy(dst, src, size);
    522        1.1       cdi }
    523        1.1       cdi 
    524        1.1       cdi static void*
    525        1.1       cdi ofw_memset(void *dst, int c, size_t size)
    526        1.1       cdi {
    527        1.1       cdi 	ofw_mapin((vaddr_t)dst, size);
    528        1.1       cdi 	return memset(dst, c, size);
    529        1.1       cdi }
    530        1.1       cdi 
    531        1.1       cdi static void
    532        1.1       cdi ofw_freeall(void)
    533        1.1       cdi {
    534        1.1       cdi 	int i;
    535        1.1       cdi 
    536        1.1       cdi 	dtlb_slot = itlb_slot = 0;
    537        1.1       cdi 	for (i = 0; i < MAXSEGNUM; i++) {
    538        1.1       cdi 		OF_release((void*)(u_long)kvamap[i].start,
    539        1.1       cdi 				(u_int)(kvamap[i].end - kvamap[i].start));
    540        1.1       cdi 		kvamap[i].start = kvamap[i].end = 0;
    541        1.1       cdi 	}
    542        1.1       cdi }
    543        1.1       cdi 
    544        1.1       cdi /*
    545        1.1       cdi  * NOP implementation exists solely for kernel header loading sake. Here
    546        1.1       cdi  * we use alloc() interface to allocate memory and avoid doing some dangerous
    547        1.1       cdi  * things.
    548        1.1       cdi  */
    549        1.1       cdi static ssize_t
    550        1.1       cdi nop_read(int f, void *addr, size_t size)
    551        1.1       cdi {
    552        1.1       cdi 	return read(f, addr, size);
    553        1.1       cdi }
    554        1.1       cdi 
    555        1.1       cdi static void*
    556        1.1       cdi nop_memcpy(void *dst, const void *src, size_t size)
    557        1.1       cdi {
    558        1.1       cdi 	/*
    559        1.1       cdi 	 * Real NOP to make LOAD_HDR work: loadfile_elfXX copies ELF headers
    560        1.1       cdi 	 * right after the highest kernel address which will not be mapped with
    561        1.1       cdi 	 * nop_XXX operations.
    562        1.1       cdi 	 */
    563        1.1       cdi 	return (dst);
    564        1.1       cdi }
    565        1.1       cdi 
    566        1.1       cdi static void*
    567        1.1       cdi nop_memset(void *dst, int c, size_t size)
    568        1.1       cdi {
    569        1.1       cdi 	return memset(dst, c, size);
    570        1.1       cdi }
    571        1.1       cdi 
    572        1.1       cdi static void
    573        1.1       cdi nop_freeall(void)
    574        1.1       cdi { }
    575        1.1       cdi 
    576        1.1       cdi /*
    577        1.1       cdi  * loadfile() hooks.
    578        1.1       cdi  */
    579        1.1       cdi ssize_t
    580        1.1       cdi sparc64_read(int f, void *addr, size_t size)
    581        1.1       cdi {
    582        1.1       cdi 	return (*memsw->read)(f, addr, size);
    583        1.1       cdi }
    584        1.1       cdi 
    585        1.1       cdi void*
    586        1.1       cdi sparc64_memcpy(void *dst, const void *src, size_t size)
    587        1.1       cdi {
    588        1.1       cdi 	return (*memsw->memcpy)(dst, src, size);
    589        1.1       cdi }
    590        1.1       cdi 
    591        1.1       cdi void*
    592        1.1       cdi sparc64_memset(void *dst, int c, size_t size)
    593        1.1       cdi {
    594        1.1       cdi 	return (*memsw->memset)(dst, c, size);
    595        1.1       cdi }
    596        1.1       cdi 
    597        1.1       cdi /*
    598        1.3    martin  * Remove write permissions from text mappings in the dTLB.
    599        1.3    martin  * Add entries in the iTLB.
    600        1.3    martin  */
    601        1.3    martin void
    602        1.3    martin sparc64_finalize_tlb(u_long data_va)
    603        1.3    martin {
    604  1.10.18.1     rmind #ifdef SUN4V
    605  1.10.18.1     rmind 	if ( sun4v )
    606  1.10.18.1     rmind 		sparc64_finalize_tlb_sun4v(data_va);
    607  1.10.18.1     rmind 	else
    608  1.10.18.1     rmind #endif
    609  1.10.18.1     rmind 		sparc64_finalize_tlb_sun4u(data_va);
    610  1.10.18.1     rmind }
    611  1.10.18.1     rmind 
    612  1.10.18.1     rmind /*
    613  1.10.18.1     rmind  * Remove write permissions from text mappings in the dTLB - sun4u.
    614  1.10.18.1     rmind  * Add entries in the iTLB.
    615  1.10.18.1     rmind  */
    616  1.10.18.1     rmind void
    617  1.10.18.1     rmind sparc64_finalize_tlb_sun4u(u_long data_va)
    618  1.10.18.1     rmind {
    619        1.3    martin 	int i;
    620        1.3    martin 	int64_t data;
    621        1.6    martin 	bool writable_text = false;
    622        1.3    martin 
    623        1.3    martin 	for (i = 0; i < dtlb_slot; i++) {
    624        1.6    martin 		if (dtlb_store[i].te_va >= data_va) {
    625        1.6    martin 			/*
    626        1.6    martin 			 * If (for whatever reason) the start of the
    627        1.6    martin 			 * writable section is right at the start of
    628        1.6    martin 			 * the kernel, we need to map it into the ITLB
    629        1.6    martin 			 * nevertheless (and don't make it readonly).
    630        1.6    martin 			 */
    631        1.6    martin 			if (i == 0 && dtlb_store[i].te_va == data_va)
    632        1.6    martin 				writable_text = true;
    633        1.6    martin 			else
    634        1.6    martin 				continue;
    635        1.6    martin 		}
    636        1.3    martin 
    637  1.10.18.1     rmind 		data = SUN4U_TSB_DATA(0,	/* global */
    638        1.3    martin 				PGSZ_4M,	/* 4mb page */
    639        1.3    martin 				dtlb_store[i].te_pa,	/* phys.address */
    640        1.3    martin 				1,		/* privileged */
    641        1.3    martin 				0,		/* write */
    642        1.3    martin 				1,		/* cache */
    643        1.3    martin 				1,		/* alias */
    644        1.3    martin 				1,		/* valid */
    645        1.3    martin 				0		/* endianness */
    646        1.3    martin 				);
    647  1.10.18.1     rmind 		data |= SUN4U_TLB_L | SUN4U_TLB_CV; /* locked, virt.cache */
    648        1.6    martin 		if (!writable_text)
    649        1.6    martin 			dtlb_replace(dtlb_store[i].te_va, hi(data), lo(data));
    650        1.3    martin 		itlb_store[itlb_slot] = dtlb_store[i];
    651        1.3    martin 		itlb_slot++;
    652        1.3    martin 		itlb_enter(dtlb_store[i].te_va, hi(data), lo(data));
    653        1.3    martin 	}
    654        1.6    martin 	if (writable_text)
    655        1.6    martin 		printf("WARNING: kernel text mapped writable!\n");
    656  1.10.18.1     rmind 
    657        1.3    martin }
    658        1.3    martin 
    659  1.10.18.1     rmind #ifdef SUN4V
    660  1.10.18.1     rmind /*
    661  1.10.18.1     rmind  * Remove write permissions from text mappings in the dTLB - sun4v.
    662  1.10.18.1     rmind  * Add entries in the iTLB.
    663  1.10.18.1     rmind  */
    664  1.10.18.1     rmind void
    665  1.10.18.1     rmind sparc64_finalize_tlb_sun4v(u_long data_va)
    666  1.10.18.1     rmind {
    667  1.10.18.1     rmind 	int i;
    668  1.10.18.1     rmind 	int64_t data;
    669  1.10.18.1     rmind 	bool writable_text = false;
    670  1.10.18.1     rmind 	int64_t hv_rc;
    671  1.10.18.1     rmind 
    672  1.10.18.1     rmind 	for (i = 0; i < dtlb_slot; i++) {
    673  1.10.18.1     rmind 		if (dtlb_store[i].te_va >= data_va) {
    674  1.10.18.1     rmind 			/*
    675  1.10.18.1     rmind 			 * If (for whatever reason) the start of the
    676  1.10.18.1     rmind 			 * writable section is right at the start of
    677  1.10.18.1     rmind 			 * the kernel, we need to map it into the ITLB
    678  1.10.18.1     rmind 			 * nevertheless (and don't make it readonly).
    679  1.10.18.1     rmind 			 */
    680  1.10.18.1     rmind 			if (i == 0 && dtlb_store[i].te_va == data_va)
    681  1.10.18.1     rmind 				writable_text = true;
    682  1.10.18.1     rmind 			else
    683  1.10.18.1     rmind 				continue;
    684  1.10.18.1     rmind 		}
    685  1.10.18.1     rmind 
    686  1.10.18.1     rmind 		data = SUN4V_TSB_DATA(
    687  1.10.18.1     rmind 			0,		/* global */
    688  1.10.18.1     rmind 			PGSZ_4M,	/* 4mb page */
    689  1.10.18.1     rmind 			dtlb_store[i].te_pa,	/* phys.address */
    690  1.10.18.1     rmind 			1,		/* privileged */
    691  1.10.18.1     rmind 			0,		/* write */
    692  1.10.18.1     rmind 			1,		/* cache */
    693  1.10.18.1     rmind 			1,		/* alias */
    694  1.10.18.1     rmind 			1,		/* valid */
    695  1.10.18.1     rmind 			0		/* endianness */
    696  1.10.18.1     rmind 			);
    697  1.10.18.1     rmind 		data |= SUN4V_TLB_CV|SUN4V_TLB_X; /* virt.cache, executable */
    698  1.10.18.1     rmind 		if (!writable_text) {
    699  1.10.18.1     rmind 			hv_rc = hv_mmu_unmap_perm_addr(dtlb_store[i].te_va,
    700  1.10.18.1     rmind 			                               MAP_DTLB);
    701  1.10.18.1     rmind 			if ( hv_rc != H_EOK ) {
    702  1.10.18.1     rmind 				panic("hv_mmu_unmap_perm_addr() failed - "
    703  1.10.18.1     rmind 				      "rc = %ld", hv_rc);
    704  1.10.18.1     rmind 			}
    705  1.10.18.1     rmind 			hv_rc = hv_mmu_map_perm_addr(dtlb_store[i].te_va, data,
    706  1.10.18.1     rmind 			                             MAP_DTLB);
    707  1.10.18.1     rmind 			if ( hv_rc != H_EOK ) {
    708  1.10.18.1     rmind 				panic("hv_mmu_map_perm_addr() failed - "
    709  1.10.18.1     rmind 				      "rc = %ld", hv_rc);
    710  1.10.18.1     rmind 			}
    711  1.10.18.1     rmind 		}
    712  1.10.18.1     rmind 
    713  1.10.18.1     rmind 		itlb_store[itlb_slot] = dtlb_store[i];
    714  1.10.18.1     rmind 		itlb_slot++;
    715  1.10.18.1     rmind 		hv_rc = hv_mmu_map_perm_addr(dtlb_store[i].te_va, data,
    716  1.10.18.1     rmind 		                             MAP_ITLB);
    717  1.10.18.1     rmind 		if ( hv_rc != H_EOK ) {
    718  1.10.18.1     rmind 			panic("hv_mmu_map_perm_addr() failed - rc = %ld", hv_rc);
    719  1.10.18.1     rmind 		}
    720  1.10.18.1     rmind 	}
    721  1.10.18.1     rmind 	if (writable_text)
    722  1.10.18.1     rmind 		printf("WARNING: kernel text mapped writable!\n");
    723  1.10.18.1     rmind }
    724  1.10.18.1     rmind #endif
    725  1.10.18.1     rmind 
    726        1.3    martin /*
    727        1.1       cdi  * Record kernel mappings in bootinfo structure.
    728        1.1       cdi  */
    729        1.1       cdi void
    730        1.1       cdi sparc64_bi_add(void)
    731        1.1       cdi {
    732        1.1       cdi 	int i;
    733        1.1       cdi 	int itlb_size, dtlb_size;
    734        1.1       cdi 	struct btinfo_count bi_count;
    735        1.1       cdi 	struct btinfo_tlb *bi_itlb, *bi_dtlb;
    736        1.1       cdi 
    737        1.1       cdi 	bi_count.count = itlb_slot;
    738        1.1       cdi 	bi_add(&bi_count, BTINFO_ITLB_SLOTS, sizeof(bi_count));
    739        1.1       cdi 	bi_count.count = dtlb_slot;
    740        1.1       cdi 	bi_add(&bi_count, BTINFO_DTLB_SLOTS, sizeof(bi_count));
    741        1.1       cdi 
    742        1.1       cdi 	itlb_size = sizeof(*bi_itlb) + sizeof(struct tlb_entry) * itlb_slot;
    743        1.1       cdi 	dtlb_size = sizeof(*bi_dtlb) + sizeof(struct tlb_entry) * dtlb_slot;
    744        1.1       cdi 
    745        1.1       cdi 	bi_itlb = alloc(itlb_size);
    746        1.1       cdi 	bi_dtlb = alloc(dtlb_size);
    747        1.1       cdi 
    748        1.1       cdi 	if ((bi_itlb == NULL) || (bi_dtlb == NULL)) {
    749        1.1       cdi 		panic("Out of memory in sparc64_bi_add.\n");
    750        1.1       cdi 	}
    751        1.1       cdi 
    752        1.1       cdi 	for (i = 0; i < itlb_slot; i++) {
    753        1.1       cdi 		bi_itlb->tlb[i].te_va = itlb_store[i].te_va;
    754        1.1       cdi 		bi_itlb->tlb[i].te_pa = itlb_store[i].te_pa;
    755        1.1       cdi 	}
    756        1.1       cdi 	bi_add(bi_itlb, BTINFO_ITLB, itlb_size);
    757        1.1       cdi 
    758        1.1       cdi 	for (i = 0; i < dtlb_slot; i++) {
    759        1.1       cdi 		bi_dtlb->tlb[i].te_va = dtlb_store[i].te_va;
    760        1.1       cdi 		bi_dtlb->tlb[i].te_pa = dtlb_store[i].te_pa;
    761        1.1       cdi 	}
    762        1.1       cdi 	bi_add(bi_dtlb, BTINFO_DTLB, dtlb_size);
    763        1.1       cdi }
    764        1.1       cdi 
    765        1.1       cdi /*
    766        1.1       cdi  * Choose kernel image mapping strategy:
    767        1.1       cdi  *
    768        1.1       cdi  * LOADFILE_NOP_ALLOCATOR	To load kernel image headers
    769        1.1       cdi  * LOADFILE_OFW_ALLOCATOR	To map the kernel by OpenFirmware means
    770        1.1       cdi  * LOADFILE_MMU_ALLOCATOR	To use permanent 4MB mappings
    771        1.1       cdi  */
    772        1.1       cdi void
    773        1.1       cdi loadfile_set_allocator(int type)
    774        1.1       cdi {
    775        1.1       cdi 	if (type >= (sizeof(memswa) / sizeof(struct memsw))) {
    776        1.1       cdi 		panic("Bad allocator request.\n");
    777        1.1       cdi 	}
    778        1.1       cdi 
    779        1.1       cdi 	/*
    780        1.1       cdi 	 * Release all memory claimed by previous allocator and schedule
    781        1.1       cdi 	 * another allocator for succeeding memory allocation calls.
    782        1.1       cdi 	 */
    783        1.1       cdi 	(*memsw->freeall)();
    784        1.1       cdi 	memsw = &memswa[type];
    785        1.1       cdi }
    786