arm32_kvminit.c revision 1.32
1 /* $NetBSD: arm32_kvminit.c,v 1.32 2014/10/29 14:14:14 skrll Exp $ */ 2 3 /* 4 * Copyright (c) 2002, 2003, 2005 Genetec Corporation. All rights reserved. 5 * Written by Hiroyuki Bessho for Genetec Corporation. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. The name of Genetec Corporation may not be used to endorse or 16 * promote products derived from this software without specific prior 17 * written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY GENETEC CORPORATION ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENETEC CORPORATION 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 * 31 * Copyright (c) 2001 Wasabi Systems, Inc. 32 * All rights reserved. 33 * 34 * Written by Jason R. Thorpe for Wasabi Systems, Inc. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice, this list of conditions and the following disclaimer. 41 * 2. Redistributions in binary form must reproduce the above copyright 42 * notice, this list of conditions and the following disclaimer in the 43 * documentation and/or other materials provided with the distribution. 44 * 3. All advertising materials mentioning features or use of this software 45 * must display the following acknowledgement: 46 * This product includes software developed for the NetBSD Project by 47 * Wasabi Systems, Inc. 48 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 49 * or promote products derived from this software without specific prior 50 * written permission. 51 * 52 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 53 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 54 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 55 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 56 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 57 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 58 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 59 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 60 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 61 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 62 * POSSIBILITY OF SUCH DAMAGE. 63 * 64 * Copyright (c) 1997,1998 Mark Brinicombe. 65 * Copyright (c) 1997,1998 Causality Limited. 66 * All rights reserved. 67 * 68 * Redistribution and use in source and binary forms, with or without 69 * modification, are permitted provided that the following conditions 70 * are met: 71 * 1. Redistributions of source code must retain the above copyright 72 * notice, this list of conditions and the following disclaimer. 73 * 2. Redistributions in binary form must reproduce the above copyright 74 * notice, this list of conditions and the following disclaimer in the 75 * documentation and/or other materials provided with the distribution. 76 * 3. All advertising materials mentioning features or use of this software 77 * must display the following acknowledgement: 78 * This product includes software developed by Mark Brinicombe 79 * for the NetBSD Project. 80 * 4. The name of the company nor the name of the author may be used to 81 * endorse or promote products derived from this software without specific 82 * prior written permission. 83 * 84 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 85 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 86 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 87 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 88 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 89 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 90 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 91 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 92 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 93 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 94 * SUCH DAMAGE. 95 * 96 * Copyright (c) 2007 Microsoft 97 * All rights reserved. 98 * 99 * Redistribution and use in source and binary forms, with or without 100 * modification, are permitted provided that the following conditions 101 * are met: 102 * 1. Redistributions of source code must retain the above copyright 103 * notice, this list of conditions and the following disclaimer. 104 * 2. Redistributions in binary form must reproduce the above copyright 105 * notice, this list of conditions and the following disclaimer in the 106 * documentation and/or other materials provided with the distribution. 107 * 3. All advertising materials mentioning features or use of this software 108 * must display the following acknowledgement: 109 * This product includes software developed by Microsoft 110 * 111 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 112 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 113 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 114 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTERS BE LIABLE FOR ANY DIRECT, 115 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 116 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 117 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 118 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 119 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 120 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 121 * SUCH DAMAGE. 122 */ 123 124 #include "opt_multiprocessor.h" 125 126 #include <sys/cdefs.h> 127 __KERNEL_RCSID(0, "$NetBSD: arm32_kvminit.c,v 1.32 2014/10/29 14:14:14 skrll Exp $"); 128 129 #include <sys/param.h> 130 #include <sys/device.h> 131 #include <sys/kernel.h> 132 #include <sys/reboot.h> 133 #include <sys/bus.h> 134 135 #include <dev/cons.h> 136 137 #include <uvm/uvm_extern.h> 138 139 #include <arm/locore.h> 140 #include <arm/db_machdep.h> 141 #include <arm/undefined.h> 142 #include <arm/bootconfig.h> 143 #include <arm/arm32/machdep.h> 144 145 struct bootmem_info bootmem_info; 146 147 extern void *msgbufaddr; 148 paddr_t msgbufphys; 149 paddr_t physical_start; 150 paddr_t physical_end; 151 152 extern char etext[]; 153 extern char __data_start[], _edata[]; 154 extern char __bss_start[], __bss_end__[]; 155 extern char _end[]; 156 157 /* Page tables for mapping kernel VM */ 158 #define KERNEL_L2PT_VMDATA_NUM 8 /* start with 32MB of KVM */ 159 160 /* 161 * Macros to translate between physical and virtual for a subset of the 162 * kernel address space. *Not* for general use. 163 */ 164 #if defined(KERNEL_BASE_VOFFSET) 165 #define KERN_VTOPHYS(bmi, va) \ 166 ((paddr_t)((vaddr_t)(va) - KERNEL_BASE_VOFFSET)) 167 #define KERN_PHYSTOV(bmi, pa) \ 168 ((vaddr_t)((paddr_t)(pa) + KERNEL_BASE_VOFFSET)) 169 #elif defined(ARM_MMU_EXTENDED) && defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) 170 #define KERN_VTOPHYS(bmi, va) \ 171 ((paddr_t)((vaddr_t)(va) - pmap_directbase + (bmi)->bmi_start)) 172 #define KERN_PHYSTOV(bmi, pa) \ 173 ((vaddr_t)((paddr_t)(pa) - (bmi)->bmi_start + pmap_directbase)) 174 #else 175 #define KERN_VTOPHYS(bmi, va) \ 176 ((paddr_t)((vaddr_t)(va) - KERNEL_BASE + (bmi)->bmi_start)) 177 #define KERN_PHYSTOV(bmi, pa) \ 178 ((vaddr_t)((paddr_t)(pa) - (bmi)->bmi_start + KERNEL_BASE)) 179 #endif 180 181 void 182 arm32_bootmem_init(paddr_t memstart, psize_t memsize, vsize_t kernelstart) 183 { 184 struct bootmem_info * const bmi = &bootmem_info; 185 pv_addr_t *pv = bmi->bmi_freeblocks; 186 187 #ifdef VERBOSE_INIT_ARM 188 printf("%s: memstart=%#lx, memsize=%#lx, kernelstart=%#lx\n", 189 __func__, memstart, memsize, kernelstart); 190 #endif 191 192 physical_start = bmi->bmi_start = memstart; 193 physical_end = bmi->bmi_end = memstart + memsize; 194 physmem = memsize / PAGE_SIZE; 195 196 /* 197 * Let's record where the kernel lives. 198 */ 199 bmi->bmi_kernelstart = kernelstart; 200 bmi->bmi_kernelend = KERN_VTOPHYS(bmi, round_page((vaddr_t)_end)); 201 202 #ifdef VERBOSE_INIT_ARM 203 printf("%s: kernelend=%#lx\n", __func__, bmi->bmi_kernelend); 204 #endif 205 206 /* 207 * Now the rest of the free memory must be after the kernel. 208 */ 209 pv->pv_pa = bmi->bmi_kernelend; 210 pv->pv_va = KERN_PHYSTOV(bmi, pv->pv_pa); 211 pv->pv_size = bmi->bmi_end - bmi->bmi_kernelend; 212 bmi->bmi_freepages += pv->pv_size / PAGE_SIZE; 213 #ifdef VERBOSE_INIT_ARM 214 printf("%s: adding %lu free pages: [%#lx..%#lx] (VA %#lx)\n", 215 __func__, pv->pv_size / PAGE_SIZE, pv->pv_pa, 216 pv->pv_pa + pv->pv_size - 1, pv->pv_va); 217 #endif 218 pv++; 219 220 /* 221 * Add a free block for any memory before the kernel. 222 */ 223 if (bmi->bmi_start < bmi->bmi_kernelstart) { 224 pv->pv_pa = bmi->bmi_start; 225 #if defined(ARM_MMU_EXTENDED) && defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) 226 pv->pv_va = pmap_directbase; 227 #else 228 pv->pv_va = KERNEL_BASE; 229 #endif 230 pv->pv_size = bmi->bmi_kernelstart - bmi->bmi_start; 231 bmi->bmi_freepages += pv->pv_size / PAGE_SIZE; 232 #ifdef VERBOSE_INIT_ARM 233 printf("%s: adding %lu free pages: [%#lx..%#lx] (VA %#lx)\n", 234 __func__, pv->pv_size / PAGE_SIZE, pv->pv_pa, 235 pv->pv_pa + pv->pv_size - 1, pv->pv_va); 236 #endif 237 pv++; 238 } 239 240 bmi->bmi_nfreeblocks = pv - bmi->bmi_freeblocks; 241 242 SLIST_INIT(&bmi->bmi_freechunks); 243 SLIST_INIT(&bmi->bmi_chunks); 244 } 245 246 static bool 247 concat_pvaddr(pv_addr_t *acc_pv, pv_addr_t *pv) 248 { 249 if (acc_pv->pv_pa + acc_pv->pv_size == pv->pv_pa 250 && acc_pv->pv_va + acc_pv->pv_size == pv->pv_va 251 && acc_pv->pv_prot == pv->pv_prot 252 && acc_pv->pv_cache == pv->pv_cache) { 253 #ifdef VERBOSE_INIT_ARMX 254 printf("%s: appending pv %p (%#lx..%#lx) to %#lx..%#lx\n", 255 __func__, pv, pv->pv_pa, pv->pv_pa + pv->pv_size + 1, 256 acc_pv->pv_pa, acc_pv->pv_pa + acc_pv->pv_size + 1); 257 #endif 258 acc_pv->pv_size += pv->pv_size; 259 return true; 260 } 261 262 return false; 263 } 264 265 static void 266 add_pages(struct bootmem_info *bmi, pv_addr_t *pv) 267 { 268 pv_addr_t **pvp = &SLIST_FIRST(&bmi->bmi_chunks); 269 while ((*pvp) != NULL && (*pvp)->pv_va <= pv->pv_va) { 270 pv_addr_t * const pv0 = (*pvp); 271 KASSERT(SLIST_NEXT(pv0, pv_list) == NULL || pv0->pv_pa < SLIST_NEXT(pv0, pv_list)->pv_pa); 272 if (concat_pvaddr(pv0, pv)) { 273 #ifdef VERBOSE_INIT_ARM 274 printf("%s: %s pv %p (%#lx..%#lx) to %#lx..%#lx\n", 275 __func__, "appending", pv, 276 pv->pv_pa, pv->pv_pa + pv->pv_size - 1, 277 pv0->pv_pa, pv0->pv_pa + pv0->pv_size - pv->pv_size - 1); 278 #endif 279 pv = SLIST_NEXT(pv0, pv_list); 280 if (pv != NULL && concat_pvaddr(pv0, pv)) { 281 #ifdef VERBOSE_INIT_ARM 282 printf("%s: %s pv %p (%#lx..%#lx) to %#lx..%#lx\n", 283 __func__, "merging", pv, 284 pv->pv_pa, pv->pv_pa + pv->pv_size - 1, 285 pv0->pv_pa, 286 pv0->pv_pa + pv0->pv_size - pv->pv_size - 1); 287 #endif 288 SLIST_REMOVE_AFTER(pv0, pv_list); 289 SLIST_INSERT_HEAD(&bmi->bmi_freechunks, pv, pv_list); 290 } 291 return; 292 } 293 KASSERT(pv->pv_va != (*pvp)->pv_va); 294 pvp = &SLIST_NEXT(*pvp, pv_list); 295 } 296 KASSERT((*pvp) == NULL || pv->pv_va < (*pvp)->pv_va); 297 pv_addr_t * const new_pv = SLIST_FIRST(&bmi->bmi_freechunks); 298 KASSERT(new_pv != NULL); 299 SLIST_REMOVE_HEAD(&bmi->bmi_freechunks, pv_list); 300 *new_pv = *pv; 301 SLIST_NEXT(new_pv, pv_list) = *pvp; 302 (*pvp) = new_pv; 303 #ifdef VERBOSE_INIT_ARM 304 printf("%s: adding pv %p (pa %#lx, va %#lx, %lu pages) ", 305 __func__, new_pv, new_pv->pv_pa, new_pv->pv_va, 306 new_pv->pv_size / PAGE_SIZE); 307 if (SLIST_NEXT(new_pv, pv_list)) 308 printf("before pa %#lx\n", SLIST_NEXT(new_pv, pv_list)->pv_pa); 309 else 310 printf("at tail\n"); 311 #endif 312 } 313 314 static void 315 valloc_pages(struct bootmem_info *bmi, pv_addr_t *pv, size_t npages, 316 int prot, int cache, bool zero_p) 317 { 318 size_t nbytes = npages * PAGE_SIZE; 319 pv_addr_t *free_pv = bmi->bmi_freeblocks; 320 size_t free_idx = 0; 321 static bool l1pt_found; 322 323 KASSERT(npages > 0); 324 325 /* 326 * If we haven't allocated the kernel L1 page table and we are aligned 327 * at a L1 table boundary, alloc the memory for it. 328 */ 329 if (!l1pt_found 330 && (free_pv->pv_pa & (L1_TABLE_SIZE - 1)) == 0 331 && free_pv->pv_size >= L1_TABLE_SIZE) { 332 l1pt_found = true; 333 valloc_pages(bmi, &kernel_l1pt, L1_TABLE_SIZE / PAGE_SIZE, 334 VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE, true); 335 add_pages(bmi, &kernel_l1pt); 336 } 337 338 while (nbytes > free_pv->pv_size) { 339 free_pv++; 340 free_idx++; 341 if (free_idx == bmi->bmi_nfreeblocks) { 342 panic("%s: could not allocate %zu bytes", 343 __func__, nbytes); 344 } 345 } 346 347 /* 348 * As we allocate the memory, make sure that we don't walk over 349 * our current first level translation table. 350 */ 351 KASSERT((armreg_ttbr_read() & ~(L1_TABLE_SIZE - 1)) != free_pv->pv_pa); 352 353 pv->pv_pa = free_pv->pv_pa; 354 pv->pv_va = free_pv->pv_va; 355 pv->pv_size = nbytes; 356 pv->pv_prot = prot; 357 pv->pv_cache = cache; 358 359 /* 360 * If PTE_PAGETABLE uses the same cache modes as PTE_CACHE 361 * just use PTE_CACHE. 362 */ 363 if (cache == PTE_PAGETABLE 364 && pte_l1_s_cache_mode == pte_l1_s_cache_mode_pt 365 && pte_l2_l_cache_mode == pte_l2_l_cache_mode_pt 366 && pte_l2_s_cache_mode == pte_l2_s_cache_mode_pt) 367 pv->pv_cache = PTE_CACHE; 368 369 free_pv->pv_pa += nbytes; 370 free_pv->pv_va += nbytes; 371 free_pv->pv_size -= nbytes; 372 if (free_pv->pv_size == 0) { 373 --bmi->bmi_nfreeblocks; 374 for (; free_idx < bmi->bmi_nfreeblocks; free_idx++) { 375 free_pv[0] = free_pv[1]; 376 } 377 } 378 379 bmi->bmi_freepages -= npages; 380 381 if (zero_p) 382 memset((void *)pv->pv_va, 0, nbytes); 383 } 384 385 void 386 arm32_kernel_vm_init(vaddr_t kernel_vm_base, vaddr_t vectors, vaddr_t iovbase, 387 const struct pmap_devmap *devmap, bool mapallmem_p) 388 { 389 struct bootmem_info * const bmi = &bootmem_info; 390 #ifdef MULTIPROCESSOR 391 const size_t cpu_num = arm_cpu_max; 392 #else 393 const size_t cpu_num = 1; 394 #endif 395 #ifdef ARM_HAS_VBAR 396 const bool map_vectors_p = false; 397 #elif defined(CPU_ARMV7) || defined(CPU_ARM11) 398 const bool map_vectors_p = vectors == ARM_VECTORS_HIGH 399 || (armreg_pfr1_read() & ARM_PFR1_SEC_MASK) == 0; 400 #else 401 const bool map_vectors_p = true; 402 #endif 403 404 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS 405 KASSERT(mapallmem_p); 406 #ifdef ARM_MMU_EXTENDED 407 /* 408 * We can only use address beneath kernel_vm_base to map physical 409 * memory. 410 */ 411 const psize_t physical_size = 412 roundup(physical_end - physical_start, L1_SS_SIZE); 413 KASSERT(kernel_vm_base >= physical_size); 414 /* 415 * If we don't have enough memory via TTBR1, we have use addresses 416 * from TTBR0 to map some of the physical memory. But try to use as 417 * much high memory space as possible. 418 */ 419 if (kernel_vm_base - KERNEL_BASE < physical_size) { 420 pmap_directbase = kernel_vm_base - physical_size; 421 printf("%s: changing pmap_directbase to %#lx\n", __func__, 422 pmap_directbase); 423 } 424 #else 425 KASSERT(kernel_vm_base - KERNEL_BASE >= physical_end - physical_start); 426 #endif /* ARM_MMU_EXTENDED */ 427 #endif /* __HAVE_MM_MD_DIRECT_MAPPED_PHYS */ 428 429 /* 430 * Calculate the number of L2 pages needed for mapping the 431 * kernel + data + stuff. Assume 2 L2 pages for kernel, 1 for vectors, 432 * and 1 for IO 433 */ 434 size_t kernel_size = bmi->bmi_kernelend; 435 kernel_size -= (bmi->bmi_kernelstart & -L2_S_SEGSIZE); 436 kernel_size += L1_TABLE_SIZE_REAL; 437 kernel_size += PAGE_SIZE * KERNEL_L2PT_VMDATA_NUM; 438 if (map_vectors_p) { 439 kernel_size += PAGE_SIZE; /* L2PT for VECTORS */ 440 } 441 if (iovbase) { 442 kernel_size += PAGE_SIZE; /* L2PT for IO */ 443 } 444 kernel_size += 445 cpu_num * (ABT_STACK_SIZE + FIQ_STACK_SIZE + IRQ_STACK_SIZE 446 + UND_STACK_SIZE + UPAGES) * PAGE_SIZE; 447 kernel_size += round_page(MSGBUFSIZE); 448 kernel_size += 0x10000; /* slop */ 449 if (!mapallmem_p) { 450 kernel_size += PAGE_SIZE 451 * ((kernel_size + L2_S_SEGSIZE - 1) / L2_S_SEGSIZE); 452 } 453 kernel_size = round_page(kernel_size); 454 455 /* 456 * Now we know how many L2 pages it will take. If we've mapped 457 * all of memory, then it won't take any. 458 */ 459 const size_t KERNEL_L2PT_KERNEL_NUM = mapallmem_p 460 ? 0 : round_page(kernel_size + L2_S_SEGSIZE - 1) / L2_S_SEGSIZE; 461 462 #ifdef VERBOSE_INIT_ARM 463 printf("%s: %zu L2 pages are needed to map %#zx kernel bytes\n", 464 __func__, KERNEL_L2PT_KERNEL_NUM, kernel_size); 465 #endif 466 467 KASSERT(KERNEL_L2PT_KERNEL_NUM + KERNEL_L2PT_VMDATA_NUM < __arraycount(bmi->bmi_l2pts)); 468 pv_addr_t * const kernel_l2pt = bmi->bmi_l2pts; 469 pv_addr_t * const vmdata_l2pt = kernel_l2pt + KERNEL_L2PT_KERNEL_NUM; 470 pv_addr_t msgbuf; 471 pv_addr_t text; 472 pv_addr_t data; 473 pv_addr_t chunks[KERNEL_L2PT_KERNEL_NUM+KERNEL_L2PT_VMDATA_NUM+11]; 474 #if ARM_MMU_XSCALE == 1 475 pv_addr_t minidataclean; 476 #endif 477 478 /* 479 * We need to allocate some fixed page tables to get the kernel going. 480 * 481 * We are going to allocate our bootstrap pages from the beginning of 482 * the free space that we just calculated. We allocate one page 483 * directory and a number of page tables and store the physical 484 * addresses in the bmi_l2pts array in bootmem_info. 485 * 486 * The kernel page directory must be on a 16K boundary. The page 487 * tables must be on 4K boundaries. What we do is allocate the 488 * page directory on the first 16K boundary that we encounter, and 489 * the page tables on 4K boundaries otherwise. Since we allocate 490 * at least 3 L2 page tables, we are guaranteed to encounter at 491 * least one 16K aligned region. 492 */ 493 494 #ifdef VERBOSE_INIT_ARM 495 printf("%s: allocating page tables for", __func__); 496 #endif 497 for (size_t i = 0; i < __arraycount(chunks); i++) { 498 SLIST_INSERT_HEAD(&bmi->bmi_freechunks, &chunks[i], pv_list); 499 } 500 501 kernel_l1pt.pv_pa = 0; 502 kernel_l1pt.pv_va = 0; 503 504 /* 505 * Allocate the L2 pages, but if we get to a page that is aligned for 506 * an L1 page table, we will allocate the pages for it first and then 507 * allocate the L2 page. 508 */ 509 510 if (map_vectors_p) { 511 /* 512 * First allocate L2 page for the vectors. 513 */ 514 #ifdef VERBOSE_INIT_ARM 515 printf(" vector"); 516 #endif 517 valloc_pages(bmi, &bmi->bmi_vector_l2pt, 1, 518 VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE, true); 519 add_pages(bmi, &bmi->bmi_vector_l2pt); 520 } 521 522 /* 523 * Now allocate L2 pages for the kernel 524 */ 525 #ifdef VERBOSE_INIT_ARM 526 printf(" kernel"); 527 #endif 528 KASSERT(mapallmem_p || KERNEL_L2PT_KERNEL_NUM > 0); 529 KASSERT(!mapallmem_p || KERNEL_L2PT_KERNEL_NUM == 0); 530 for (size_t idx = 0; idx < KERNEL_L2PT_KERNEL_NUM; ++idx) { 531 valloc_pages(bmi, &kernel_l2pt[idx], 1, 532 VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE, true); 533 add_pages(bmi, &kernel_l2pt[idx]); 534 } 535 536 /* 537 * Now allocate L2 pages for the initial kernel VA space. 538 */ 539 #ifdef VERBOSE_INIT_ARM 540 printf(" vm"); 541 #endif 542 for (size_t idx = 0; idx < KERNEL_L2PT_VMDATA_NUM; ++idx) { 543 valloc_pages(bmi, &vmdata_l2pt[idx], 1, 544 VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE, true); 545 add_pages(bmi, &vmdata_l2pt[idx]); 546 } 547 548 /* 549 * If someone wanted a L2 page for I/O, allocate it now. 550 */ 551 if (iovbase) { 552 #ifdef VERBOSE_INIT_ARM 553 printf(" io"); 554 #endif 555 valloc_pages(bmi, &bmi->bmi_io_l2pt, 1, 556 VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE, true); 557 add_pages(bmi, &bmi->bmi_io_l2pt); 558 } 559 560 #ifdef VERBOSE_INIT_ARM 561 printf("%s: allocating stacks\n", __func__); 562 #endif 563 564 /* Allocate stacks for all modes and CPUs */ 565 valloc_pages(bmi, &abtstack, ABT_STACK_SIZE * cpu_num, 566 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true); 567 add_pages(bmi, &abtstack); 568 valloc_pages(bmi, &fiqstack, FIQ_STACK_SIZE * cpu_num, 569 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true); 570 add_pages(bmi, &fiqstack); 571 valloc_pages(bmi, &irqstack, IRQ_STACK_SIZE * cpu_num, 572 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true); 573 add_pages(bmi, &irqstack); 574 valloc_pages(bmi, &undstack, UND_STACK_SIZE * cpu_num, 575 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true); 576 add_pages(bmi, &undstack); 577 valloc_pages(bmi, &idlestack, UPAGES * cpu_num, /* SVC32 */ 578 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true); 579 add_pages(bmi, &idlestack); 580 valloc_pages(bmi, &kernelstack, UPAGES, /* SVC32 */ 581 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true); 582 add_pages(bmi, &kernelstack); 583 584 /* Allocate the message buffer from the end of memory. */ 585 const size_t msgbuf_pgs = round_page(MSGBUFSIZE) / PAGE_SIZE; 586 valloc_pages(bmi, &msgbuf, msgbuf_pgs, 587 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, false); 588 add_pages(bmi, &msgbuf); 589 msgbufphys = msgbuf.pv_pa; 590 msgbufaddr = (void *)msgbuf.pv_va; 591 592 if (map_vectors_p) { 593 /* 594 * Allocate a page for the system vector page. 595 * This page will just contain the system vectors and can be 596 * shared by all processes. 597 */ 598 valloc_pages(bmi, &systempage, 1, VM_PROT_READ|VM_PROT_WRITE, 599 PTE_CACHE, true); 600 } 601 systempage.pv_va = vectors; 602 603 /* 604 * If the caller needed a few extra pages for some reason, allocate 605 * them now. 606 */ 607 #if ARM_MMU_XSCALE == 1 608 #if (ARM_NMMUS > 1) 609 if (xscale_use_minidata) 610 #endif 611 valloc_pages(bmi, &minidataclean, 1, 612 VM_PROT_READ|VM_PROT_WRITE, 0, true); 613 #endif 614 615 /* 616 * Ok we have allocated physical pages for the primary kernel 617 * page tables and stacks. Let's just confirm that. 618 */ 619 if (kernel_l1pt.pv_va == 0 620 && (!kernel_l1pt.pv_pa || (kernel_l1pt.pv_pa & (L1_TABLE_SIZE - 1)) != 0)) 621 panic("%s: Failed to allocate or align the kernel " 622 "page directory", __func__); 623 624 625 #ifdef VERBOSE_INIT_ARM 626 printf("Creating L1 page table at 0x%08lx\n", kernel_l1pt.pv_pa); 627 #endif 628 629 /* 630 * Now we start construction of the L1 page table 631 * We start by mapping the L2 page tables into the L1. 632 * This means that we can replace L1 mappings later on if necessary 633 */ 634 vaddr_t l1pt_va = kernel_l1pt.pv_va; 635 paddr_t l1pt_pa = kernel_l1pt.pv_pa; 636 637 if (map_vectors_p) { 638 /* Map the L2 pages tables in the L1 page table */ 639 pmap_link_l2pt(l1pt_va, systempage.pv_va & -L2_S_SEGSIZE, 640 &bmi->bmi_vector_l2pt); 641 #ifdef VERBOSE_INIT_ARM 642 printf("%s: adding L2 pt (VA %#lx, PA %#lx) " 643 "for VA %#lx\n (vectors)", 644 __func__, bmi->bmi_vector_l2pt.pv_va, 645 bmi->bmi_vector_l2pt.pv_pa, systempage.pv_va); 646 #endif 647 } 648 649 const vaddr_t kernel_base = 650 KERN_PHYSTOV(bmi, bmi->bmi_kernelstart & -L2_S_SEGSIZE); 651 for (size_t idx = 0; idx < KERNEL_L2PT_KERNEL_NUM; idx++) { 652 pmap_link_l2pt(l1pt_va, kernel_base + idx * L2_S_SEGSIZE, 653 &kernel_l2pt[idx]); 654 #ifdef VERBOSE_INIT_ARM 655 printf("%s: adding L2 pt (VA %#lx, PA %#lx) for VA %#lx (kernel)\n", 656 __func__, kernel_l2pt[idx].pv_va, 657 kernel_l2pt[idx].pv_pa, kernel_base + idx * L2_S_SEGSIZE); 658 #endif 659 } 660 661 for (size_t idx = 0; idx < KERNEL_L2PT_VMDATA_NUM; idx++) { 662 pmap_link_l2pt(l1pt_va, kernel_vm_base + idx * L2_S_SEGSIZE, 663 &vmdata_l2pt[idx]); 664 #ifdef VERBOSE_INIT_ARM 665 printf("%s: adding L2 pt (VA %#lx, PA %#lx) for VA %#lx (vm)\n", 666 __func__, vmdata_l2pt[idx].pv_va, vmdata_l2pt[idx].pv_pa, 667 kernel_vm_base + idx * L2_S_SEGSIZE); 668 #endif 669 } 670 if (iovbase) { 671 pmap_link_l2pt(l1pt_va, iovbase & -L2_S_SEGSIZE, &bmi->bmi_io_l2pt); 672 #ifdef VERBOSE_INIT_ARM 673 printf("%s: adding L2 pt (VA %#lx, PA %#lx) for VA %#lx (io)\n", 674 __func__, bmi->bmi_io_l2pt.pv_va, bmi->bmi_io_l2pt.pv_pa, 675 iovbase & -L2_S_SEGSIZE); 676 #endif 677 } 678 679 /* update the top of the kernel VM */ 680 pmap_curmaxkvaddr = 681 kernel_vm_base + (KERNEL_L2PT_VMDATA_NUM * L2_S_SEGSIZE); 682 683 #ifdef VERBOSE_INIT_ARM 684 printf("Mapping kernel\n"); 685 #endif 686 687 extern char etext[], _end[]; 688 size_t totalsize = bmi->bmi_kernelend - bmi->bmi_kernelstart; 689 size_t textsize = KERN_VTOPHYS(bmi, (uintptr_t)etext) - bmi->bmi_kernelstart; 690 691 textsize = (textsize + PGOFSET) & ~PGOFSET; 692 693 /* start at offset of kernel in RAM */ 694 695 text.pv_pa = bmi->bmi_kernelstart; 696 text.pv_va = KERN_PHYSTOV(bmi, bmi->bmi_kernelstart); 697 text.pv_size = textsize; 698 text.pv_prot = VM_PROT_READ|VM_PROT_WRITE; /* XXX VM_PROT_EXECUTE */ 699 text.pv_cache = PTE_CACHE; 700 701 #ifdef VERBOSE_INIT_ARM 702 printf("%s: adding chunk for kernel text %#lx..%#lx (VA %#lx)\n", 703 __func__, text.pv_pa, text.pv_pa + text.pv_size - 1, text.pv_va); 704 #endif 705 706 add_pages(bmi, &text); 707 708 data.pv_pa = text.pv_pa + textsize; 709 data.pv_va = text.pv_va + textsize; 710 data.pv_size = totalsize - textsize; 711 data.pv_prot = VM_PROT_READ|VM_PROT_WRITE; 712 data.pv_cache = PTE_CACHE; 713 714 #ifdef VERBOSE_INIT_ARM 715 printf("%s: adding chunk for kernel data/bss %#lx..%#lx (VA %#lx)\n", 716 __func__, data.pv_pa, data.pv_pa + data.pv_size - 1, data.pv_va); 717 #endif 718 719 add_pages(bmi, &data); 720 721 #ifdef VERBOSE_INIT_ARM 722 printf("Listing Chunks\n"); 723 724 pv_addr_t *lpv; 725 SLIST_FOREACH(lpv, &bmi->bmi_chunks, pv_list) { 726 printf("%s: pv %p: chunk VA %#lx..%#lx " 727 "(PA %#lx, prot %d, cache %d)\n", 728 __func__, lpv, lpv->pv_va, lpv->pv_va + lpv->pv_size - 1, 729 lpv->pv_pa, lpv->pv_prot, lpv->pv_cache); 730 } 731 printf("\nMapping Chunks\n"); 732 #endif 733 734 pv_addr_t cur_pv; 735 pv_addr_t *pv = SLIST_FIRST(&bmi->bmi_chunks); 736 if (!mapallmem_p || pv->pv_pa == bmi->bmi_start) { 737 cur_pv = *pv; 738 pv = SLIST_NEXT(pv, pv_list); 739 } else { 740 #if defined(ARM_MMU_EXTENDED) && defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) 741 cur_pv.pv_va = pmap_directbase; 742 #else 743 cur_pv.pv_va = KERNEL_BASE; 744 #endif 745 cur_pv.pv_pa = bmi->bmi_start; 746 cur_pv.pv_size = pv->pv_pa - bmi->bmi_start; 747 cur_pv.pv_prot = VM_PROT_READ | VM_PROT_WRITE; 748 cur_pv.pv_cache = PTE_CACHE; 749 } 750 while (pv != NULL) { 751 if (mapallmem_p) { 752 if (concat_pvaddr(&cur_pv, pv)) { 753 pv = SLIST_NEXT(pv, pv_list); 754 continue; 755 } 756 if (cur_pv.pv_pa + cur_pv.pv_size < pv->pv_pa) { 757 /* 758 * See if we can extend the current pv to emcompass the 759 * hole, and if so do it and retry the concatenation. 760 */ 761 if (cur_pv.pv_prot == (VM_PROT_READ|VM_PROT_WRITE) 762 && cur_pv.pv_cache == PTE_CACHE) { 763 cur_pv.pv_size = pv->pv_pa - cur_pv.pv_va; 764 continue; 765 } 766 767 /* 768 * We couldn't so emit the current chunk and then 769 */ 770 #ifdef VERBOSE_INIT_ARM 771 printf("%s: mapping chunk VA %#lx..%#lx " 772 "(PA %#lx, prot %d, cache %d)\n", 773 __func__, 774 cur_pv.pv_va, cur_pv.pv_va + cur_pv.pv_size - 1, 775 cur_pv.pv_pa, cur_pv.pv_prot, cur_pv.pv_cache); 776 #endif 777 pmap_map_chunk(l1pt_va, cur_pv.pv_va, cur_pv.pv_pa, 778 cur_pv.pv_size, cur_pv.pv_prot, cur_pv.pv_cache); 779 780 /* 781 * set the current chunk to the hole and try again. 782 */ 783 cur_pv.pv_pa += cur_pv.pv_size; 784 cur_pv.pv_va += cur_pv.pv_size; 785 cur_pv.pv_size = pv->pv_pa - cur_pv.pv_va; 786 cur_pv.pv_prot = VM_PROT_READ | VM_PROT_WRITE; 787 cur_pv.pv_cache = PTE_CACHE; 788 continue; 789 } 790 } 791 792 /* 793 * The new pv didn't concatenate so emit the current one 794 * and use the new pv as the current pv. 795 */ 796 #ifdef VERBOSE_INIT_ARM 797 printf("%s: mapping chunk VA %#lx..%#lx " 798 "(PA %#lx, prot %d, cache %d)\n", 799 __func__, cur_pv.pv_va, cur_pv.pv_va + cur_pv.pv_size - 1, 800 cur_pv.pv_pa, cur_pv.pv_prot, cur_pv.pv_cache); 801 #endif 802 pmap_map_chunk(l1pt_va, cur_pv.pv_va, cur_pv.pv_pa, 803 cur_pv.pv_size, cur_pv.pv_prot, cur_pv.pv_cache); 804 cur_pv = *pv; 805 pv = SLIST_NEXT(pv, pv_list); 806 } 807 808 /* 809 * If we are mapping all of memory, let's map the rest of memory. 810 */ 811 if (mapallmem_p && cur_pv.pv_pa + cur_pv.pv_size < bmi->bmi_end) { 812 if (cur_pv.pv_prot == (VM_PROT_READ | VM_PROT_WRITE) 813 && cur_pv.pv_cache == PTE_CACHE) { 814 cur_pv.pv_size = bmi->bmi_end - cur_pv.pv_pa; 815 } else { 816 #ifdef VERBOSE_INIT_ARM 817 printf("%s: mapping chunk VA %#lx..%#lx " 818 "(PA %#lx, prot %d, cache %d)\n", 819 __func__, cur_pv.pv_va, cur_pv.pv_va + cur_pv.pv_size - 1, 820 cur_pv.pv_pa, cur_pv.pv_prot, cur_pv.pv_cache); 821 #endif 822 pmap_map_chunk(l1pt_va, cur_pv.pv_va, cur_pv.pv_pa, 823 cur_pv.pv_size, cur_pv.pv_prot, cur_pv.pv_cache); 824 cur_pv.pv_pa += cur_pv.pv_size; 825 cur_pv.pv_va += cur_pv.pv_size; 826 cur_pv.pv_size = bmi->bmi_end - cur_pv.pv_pa; 827 cur_pv.pv_prot = VM_PROT_READ | VM_PROT_WRITE; 828 cur_pv.pv_cache = PTE_CACHE; 829 } 830 } 831 832 /* 833 * Now we map the final chunk. 834 */ 835 #ifdef VERBOSE_INIT_ARM 836 printf("%s: mapping last chunk VA %#lx..%#lx (PA %#lx, prot %d, cache %d)\n", 837 __func__, cur_pv.pv_va, cur_pv.pv_va + cur_pv.pv_size - 1, 838 cur_pv.pv_pa, cur_pv.pv_prot, cur_pv.pv_cache); 839 #endif 840 pmap_map_chunk(l1pt_va, cur_pv.pv_va, cur_pv.pv_pa, 841 cur_pv.pv_size, cur_pv.pv_prot, cur_pv.pv_cache); 842 843 /* 844 * Now we map the stuff that isn't directly after the kernel 845 */ 846 847 if (map_vectors_p) { 848 /* Map the vector page. */ 849 pmap_map_entry(l1pt_va, systempage.pv_va, systempage.pv_pa, 850 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); 851 } 852 853 /* Map the Mini-Data cache clean area. */ 854 #if ARM_MMU_XSCALE == 1 855 #if (ARM_NMMUS > 1) 856 if (xscale_use_minidata) 857 #endif 858 xscale_setup_minidata(l1pt_va, minidataclean.pv_va, 859 minidataclean.pv_pa); 860 #endif 861 862 /* 863 * Map integrated peripherals at same address in first level page 864 * table so that we can continue to use console. 865 */ 866 if (devmap) 867 pmap_devmap_bootstrap(l1pt_va, devmap); 868 869 #ifdef VERBOSE_INIT_ARM 870 /* Tell the user about where all the bits and pieces live. */ 871 printf("%22s Physical Virtual Num\n", " "); 872 printf("%22s Starting Ending Starting Ending Pages\n", " "); 873 874 static const char mem_fmt[] = 875 "%20s: 0x%08lx 0x%08lx 0x%08lx 0x%08lx %u\n"; 876 static const char mem_fmt_nov[] = 877 "%20s: 0x%08lx 0x%08lx %zu\n"; 878 879 printf(mem_fmt, "SDRAM", bmi->bmi_start, bmi->bmi_end - 1, 880 KERN_PHYSTOV(bmi, bmi->bmi_start), KERN_PHYSTOV(bmi, bmi->bmi_end - 1), 881 physmem); 882 printf(mem_fmt, "text section", 883 text.pv_pa, text.pv_pa + text.pv_size - 1, 884 text.pv_va, text.pv_va + text.pv_size - 1, 885 (int)(text.pv_size / PAGE_SIZE)); 886 printf(mem_fmt, "data section", 887 KERN_VTOPHYS(bmi, __data_start), KERN_VTOPHYS(bmi, _edata), 888 (vaddr_t)__data_start, (vaddr_t)_edata, 889 (int)((round_page((vaddr_t)_edata) 890 - trunc_page((vaddr_t)__data_start)) / PAGE_SIZE)); 891 printf(mem_fmt, "bss section", 892 KERN_VTOPHYS(bmi, __bss_start), KERN_VTOPHYS(bmi, __bss_end__), 893 (vaddr_t)__bss_start, (vaddr_t)__bss_end__, 894 (int)((round_page((vaddr_t)__bss_end__) 895 - trunc_page((vaddr_t)__bss_start)) / PAGE_SIZE)); 896 printf(mem_fmt, "L1 page directory", 897 kernel_l1pt.pv_pa, kernel_l1pt.pv_pa + L1_TABLE_SIZE - 1, 898 kernel_l1pt.pv_va, kernel_l1pt.pv_va + L1_TABLE_SIZE - 1, 899 L1_TABLE_SIZE / PAGE_SIZE); 900 printf(mem_fmt, "ABT stack (CPU 0)", 901 abtstack.pv_pa, abtstack.pv_pa + (ABT_STACK_SIZE * PAGE_SIZE) - 1, 902 abtstack.pv_va, abtstack.pv_va + (ABT_STACK_SIZE * PAGE_SIZE) - 1, 903 ABT_STACK_SIZE); 904 printf(mem_fmt, "FIQ stack (CPU 0)", 905 fiqstack.pv_pa, fiqstack.pv_pa + (FIQ_STACK_SIZE * PAGE_SIZE) - 1, 906 fiqstack.pv_va, fiqstack.pv_va + (FIQ_STACK_SIZE * PAGE_SIZE) - 1, 907 FIQ_STACK_SIZE); 908 printf(mem_fmt, "IRQ stack (CPU 0)", 909 irqstack.pv_pa, irqstack.pv_pa + (IRQ_STACK_SIZE * PAGE_SIZE) - 1, 910 irqstack.pv_va, irqstack.pv_va + (IRQ_STACK_SIZE * PAGE_SIZE) - 1, 911 IRQ_STACK_SIZE); 912 printf(mem_fmt, "UND stack (CPU 0)", 913 undstack.pv_pa, undstack.pv_pa + (UND_STACK_SIZE * PAGE_SIZE) - 1, 914 undstack.pv_va, undstack.pv_va + (UND_STACK_SIZE * PAGE_SIZE) - 1, 915 UND_STACK_SIZE); 916 printf(mem_fmt, "IDLE stack (CPU 0)", 917 idlestack.pv_pa, idlestack.pv_pa + (UPAGES * PAGE_SIZE) - 1, 918 idlestack.pv_va, idlestack.pv_va + (UPAGES * PAGE_SIZE) - 1, 919 UPAGES); 920 printf(mem_fmt, "SVC stack", 921 kernelstack.pv_pa, kernelstack.pv_pa + (UPAGES * PAGE_SIZE) - 1, 922 kernelstack.pv_va, kernelstack.pv_va + (UPAGES * PAGE_SIZE) - 1, 923 UPAGES); 924 printf(mem_fmt, "Message Buffer", 925 msgbuf.pv_pa, msgbuf.pv_pa + (msgbuf_pgs * PAGE_SIZE) - 1, 926 msgbuf.pv_va, msgbuf.pv_va + (msgbuf_pgs * PAGE_SIZE) - 1, 927 (int)msgbuf_pgs); 928 if (map_vectors_p) { 929 printf(mem_fmt, "Exception Vectors", 930 systempage.pv_pa, systempage.pv_pa + PAGE_SIZE - 1, 931 systempage.pv_va, systempage.pv_va + PAGE_SIZE - 1, 932 1); 933 } 934 for (size_t i = 0; i < bmi->bmi_nfreeblocks; i++) { 935 pv = &bmi->bmi_freeblocks[i]; 936 937 printf(mem_fmt_nov, "Free Memory", 938 pv->pv_pa, pv->pv_pa + pv->pv_size - 1, 939 pv->pv_size / PAGE_SIZE); 940 } 941 #endif 942 /* 943 * Now we have the real page tables in place so we can switch to them. 944 * Once this is done we will be running with the REAL kernel page 945 * tables. 946 */ 947 948 #if defined(VERBOSE_INIT_ARM) 949 printf("TTBR0=%#x", armreg_ttbr_read()); 950 #ifdef _ARM_ARCH_6 951 printf(" TTBR1=%#x TTBCR=%#x CONTEXTIDR=%#x", 952 armreg_ttbr1_read(), armreg_ttbcr_read(), 953 armreg_contextidr_read()); 954 #endif 955 printf("\n"); 956 #endif 957 958 /* Switch tables */ 959 #ifdef VERBOSE_INIT_ARM 960 printf("switching to new L1 page table @%#lx...", l1pt_pa); 961 #endif 962 963 #ifdef ARM_MMU_EXTENDED 964 cpu_domains((DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL*2)) 965 | (DOMAIN_CLIENT << (PMAP_DOMAIN_USER*2))); 966 #else 967 cpu_domains((DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL*2)) | DOMAIN_CLIENT); 968 #endif 969 cpu_idcache_wbinv_all(); 970 #ifdef VERBOSE_INIT_ARM 971 printf(" ttb"); 972 #endif 973 #ifdef ARM_MMU_EXTENDED 974 /* 975 * TTBCR should have been initialized by the MD start code. 976 */ 977 KASSERT((armreg_contextidr_read() & 0xff) == 0); 978 KASSERT(armreg_ttbcr_read() == __SHIFTIN(1, TTBCR_S_N)); 979 /* 980 * Disable lookups via TTBR0 until there is an activated pmap. 981 */ 982 armreg_ttbcr_write(armreg_ttbcr_read() | TTBCR_S_PD0); 983 cpu_setttb(l1pt_pa, KERNEL_PID); 984 arm_isb(); 985 #else 986 cpu_setttb(l1pt_pa, true); 987 cpu_domains(DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL*2)); 988 #endif 989 cpu_tlb_flushID(); 990 991 #ifdef VERBOSE_INIT_ARM 992 #ifdef ARM_MMU_EXTENDED 993 printf(" (TTBCR=%#x TTBR0=%#x TTBR1=%#x)", 994 armreg_ttbcr_read(), armreg_ttbr_read(), armreg_ttbr1_read()); 995 #else 996 printf(" (TTBR0=%#x)", armreg_ttbr_read()); 997 #endif 998 #endif 999 1000 #ifdef MULTIPROCESSOR 1001 /* 1002 * Kick the secondaries to load the TTB. After which they'll go 1003 * back to sleep to wait for the final kick so they will hatch. 1004 */ 1005 #ifdef VERBOSE_INIT_ARM 1006 printf(" hatchlings"); 1007 #endif 1008 cpu_boot_secondary_processors(); 1009 #endif 1010 1011 #ifdef VERBOSE_INIT_ARM 1012 printf(" OK\n"); 1013 #endif 1014 } 1015
Indexes created Mon Sep 29 21:09:56 GMT 2025