arm32_kvminit.c revision 1.68
1 /* $NetBSD: arm32_kvminit.c,v 1.68 2021/03/21 09:00:55 skrll Exp $ */ 2 3 /* 4 * Copyright (c) 2002, 2003, 2005 Genetec Corporation. All rights reserved. 5 * Written by Hiroyuki Bessho for Genetec Corporation. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. The name of Genetec Corporation may not be used to endorse or 16 * promote products derived from this software without specific prior 17 * written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY GENETEC CORPORATION ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENETEC CORPORATION 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 * 31 * Copyright (c) 2001 Wasabi Systems, Inc. 32 * All rights reserved. 33 * 34 * Written by Jason R. Thorpe for Wasabi Systems, Inc. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice, this list of conditions and the following disclaimer. 41 * 2. Redistributions in binary form must reproduce the above copyright 42 * notice, this list of conditions and the following disclaimer in the 43 * documentation and/or other materials provided with the distribution. 44 * 3. All advertising materials mentioning features or use of this software 45 * must display the following acknowledgement: 46 * This product includes software developed for the NetBSD Project by 47 * Wasabi Systems, Inc. 48 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 49 * or promote products derived from this software without specific prior 50 * written permission. 51 * 52 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 53 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 54 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 55 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 56 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 57 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 58 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 59 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 60 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 61 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 62 * POSSIBILITY OF SUCH DAMAGE. 63 * 64 * Copyright (c) 1997,1998 Mark Brinicombe. 65 * Copyright (c) 1997,1998 Causality Limited. 66 * All rights reserved. 67 * 68 * Redistribution and use in source and binary forms, with or without 69 * modification, are permitted provided that the following conditions 70 * are met: 71 * 1. Redistributions of source code must retain the above copyright 72 * notice, this list of conditions and the following disclaimer. 73 * 2. Redistributions in binary form must reproduce the above copyright 74 * notice, this list of conditions and the following disclaimer in the 75 * documentation and/or other materials provided with the distribution. 76 * 3. All advertising materials mentioning features or use of this software 77 * must display the following acknowledgement: 78 * This product includes software developed by Mark Brinicombe 79 * for the NetBSD Project. 80 * 4. The name of the company nor the name of the author may be used to 81 * endorse or promote products derived from this software without specific 82 * prior written permission. 83 * 84 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 85 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 86 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 87 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 88 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 89 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 90 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 91 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 92 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 93 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 94 * SUCH DAMAGE. 95 * 96 * Copyright (c) 2007 Microsoft 97 * All rights reserved. 98 * 99 * Redistribution and use in source and binary forms, with or without 100 * modification, are permitted provided that the following conditions 101 * are met: 102 * 1. Redistributions of source code must retain the above copyright 103 * notice, this list of conditions and the following disclaimer. 104 * 2. Redistributions in binary form must reproduce the above copyright 105 * notice, this list of conditions and the following disclaimer in the 106 * documentation and/or other materials provided with the distribution. 107 * 3. All advertising materials mentioning features or use of this software 108 * must display the following acknowledgement: 109 * This product includes software developed by Microsoft 110 * 111 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 112 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 113 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 114 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTERS BE LIABLE FOR ANY DIRECT, 115 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 116 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 117 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 118 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 119 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 120 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 121 * SUCH DAMAGE. 122 */ 123 124 #include "opt_arm_debug.h" 125 #include "opt_arm_start.h" 126 #include "opt_fdt.h" 127 #include "opt_multiprocessor.h" 128 129 #include <sys/cdefs.h> 130 __KERNEL_RCSID(0, "$NetBSD: arm32_kvminit.c,v 1.68 2021/03/21 09:00:55 skrll Exp $"); 131 132 #include <sys/param.h> 133 134 #include <sys/asan.h> 135 #include <sys/bus.h> 136 #include <sys/device.h> 137 #include <sys/kernel.h> 138 #include <sys/reboot.h> 139 140 #include <dev/cons.h> 141 142 #include <uvm/uvm_extern.h> 143 144 #include <arm/arm32/machdep.h> 145 #include <arm/bootconfig.h> 146 #include <arm/db_machdep.h> 147 #include <arm/locore.h> 148 #include <arm/undefined.h> 149 150 #if defined(FDT) 151 #include <arch/evbarm/fdt/platform.h> 152 #include <arm/fdt/arm_fdtvar.h> 153 #include <dev/fdt/fdt_memory.h> 154 #endif 155 156 #ifdef MULTIPROCESSOR 157 #ifndef __HAVE_CPU_UAREA_ALLOC_IDLELWP 158 #error __HAVE_CPU_UAREA_ALLOC_IDLELWP required to not waste pages for idlestack 159 #endif 160 #endif 161 162 #ifdef VERBOSE_INIT_ARM 163 #define VPRINTF(...) printf(__VA_ARGS__) 164 #else 165 #define VPRINTF(...) __nothing 166 #endif 167 168 #if defined(__HAVE_GENERIC_START) 169 #if defined(KERNEL_BASE_VOFFSET) 170 #error KERNEL_BASE_VOFFSET should not be defined with __HAVE_GENERIC_START 171 #endif 172 #endif 173 174 struct bootmem_info bootmem_info; 175 176 extern void *msgbufaddr; 177 paddr_t msgbufphys; 178 paddr_t physical_start; 179 paddr_t physical_end; 180 181 extern char etext[]; 182 extern char __data_start[], _edata[]; 183 extern char __bss_start[], __bss_end__[]; 184 extern char _end[]; 185 186 /* Page tables for mapping kernel VM */ 187 #define KERNEL_L2PT_VMDATA_NUM 8 /* start with 32MB of KVM */ 188 189 #ifdef KASAN 190 vaddr_t kasan_kernelstart; 191 vaddr_t kasan_kernelsize; 192 193 #define KERNEL_L2PT_KASAN_NUM howmany(VM_KERNEL_KASAN_SIZE, L2_S_SEGSIZE) 194 bool kasan_l2pts_created __attribute__((__section__(".data"))) = false; 195 pv_addr_t kasan_l2pt[KERNEL_L2PT_KASAN_NUM]; 196 #else 197 #define KERNEL_L2PT_KASAN_NUM 0 198 #endif 199 200 u_long kern_vtopdiff __attribute__((__section__(".data"))); 201 202 void 203 arm32_bootmem_init(paddr_t memstart, psize_t memsize, vsize_t kernelstart) 204 { 205 struct bootmem_info * const bmi = &bootmem_info; 206 pv_addr_t *pv = bmi->bmi_freeblocks; 207 208 /* 209 * FDT/generic start fills in kern_vtopdiff early 210 */ 211 #if defined(__HAVE_GENERIC_START) 212 extern char KERNEL_BASE_virt[]; 213 extern char const __stop__init_memory[]; 214 215 VPRINTF("%s: kern_vtopdiff=%#lx\n", __func__, kern_vtopdiff); 216 217 vaddr_t kstartva = trunc_page((vaddr_t)KERNEL_BASE_virt); 218 vaddr_t kendva = round_page((vaddr_t)__stop__init_memory); 219 220 kernelstart = KERN_VTOPHYS(kstartva); 221 222 VPRINTF("%s: kstartva=%#lx, kernelstart=%#lx\n", __func__, kstartva, kernelstart); 223 #else 224 vaddr_t kendva = round_page((vaddr_t)_end); 225 226 #if defined(KERNEL_BASE_VOFFSET) 227 kern_vtopdiff = KERNEL_BASE_VOFFSET; 228 #else 229 KASSERT(memstart == kernelstart); 230 kern_vtopdiff = KERNEL_BASE + memstart; 231 #endif 232 #endif 233 paddr_t kernelend = KERN_VTOPHYS(kendva); 234 235 VPRINTF("%s: memstart=%#lx, memsize=%#lx\n", __func__, 236 memstart, memsize); 237 VPRINTF("%s: kernelstart=%#lx, kernelend=%#lx\n", __func__, 238 kernelstart, kernelend); 239 240 physical_start = bmi->bmi_start = memstart; 241 physical_end = bmi->bmi_end = memstart + memsize; 242 #ifndef ARM_HAS_LPAE 243 if (physical_end == 0) { 244 physical_end = -PAGE_SIZE; 245 memsize -= PAGE_SIZE; 246 bmi->bmi_end -= PAGE_SIZE; 247 VPRINTF("%s: memsize shrunk by a page to avoid ending at 4GB\n", 248 __func__); 249 } 250 #endif 251 physmem = memsize / PAGE_SIZE; 252 253 /* 254 * Let's record where the kernel lives. 255 */ 256 257 bmi->bmi_kernelstart = kernelstart; 258 bmi->bmi_kernelend = kernelend; 259 260 #if defined(FDT) 261 fdt_memory_remove_range(bmi->bmi_kernelstart, 262 bmi->bmi_kernelend - bmi->bmi_kernelstart); 263 #endif 264 265 VPRINTF("%s: kernel phys start %#lx end %#lx\n", __func__, kernelstart, 266 kernelend); 267 268 #if 0 269 // XXX Makes RPI abort 270 KASSERT((kernelstart & (L2_S_SEGSIZE - 1)) == 0); 271 #endif 272 /* 273 * Now the rest of the free memory must be after the kernel. 274 */ 275 pv->pv_pa = bmi->bmi_kernelend; 276 pv->pv_va = KERN_PHYSTOV(pv->pv_pa); 277 pv->pv_size = bmi->bmi_end - bmi->bmi_kernelend; 278 bmi->bmi_freepages += pv->pv_size / PAGE_SIZE; 279 VPRINTF("%s: adding %lu free pages: [%#lx..%#lx] (VA %#lx)\n", 280 __func__, pv->pv_size / PAGE_SIZE, pv->pv_pa, 281 pv->pv_pa + pv->pv_size - 1, pv->pv_va); 282 pv++; 283 284 /* 285 * Add a free block for any memory before the kernel. 286 */ 287 if (bmi->bmi_start < bmi->bmi_kernelstart) { 288 pv->pv_pa = bmi->bmi_start; 289 pv->pv_va = KERN_PHYSTOV(pv->pv_pa); 290 pv->pv_size = bmi->bmi_kernelstart - pv->pv_pa; 291 bmi->bmi_freepages += pv->pv_size / PAGE_SIZE; 292 VPRINTF("%s: adding %lu free pages: [%#lx..%#lx] (VA %#lx)\n", 293 __func__, pv->pv_size / PAGE_SIZE, pv->pv_pa, 294 pv->pv_pa + pv->pv_size - 1, pv->pv_va); 295 pv++; 296 } 297 298 bmi->bmi_nfreeblocks = pv - bmi->bmi_freeblocks; 299 300 SLIST_INIT(&bmi->bmi_freechunks); 301 SLIST_INIT(&bmi->bmi_chunks); 302 } 303 304 static bool 305 concat_pvaddr(pv_addr_t *acc_pv, pv_addr_t *pv) 306 { 307 if (acc_pv->pv_pa + acc_pv->pv_size == pv->pv_pa 308 && acc_pv->pv_va + acc_pv->pv_size == pv->pv_va 309 && acc_pv->pv_prot == pv->pv_prot 310 && acc_pv->pv_cache == pv->pv_cache) { 311 #if 0 312 VPRINTF("%s: appending pv %p (%#lx..%#lx) to %#lx..%#lx\n", 313 __func__, pv, pv->pv_pa, pv->pv_pa + pv->pv_size, 314 acc_pv->pv_pa, acc_pv->pv_pa + acc_pv->pv_size); 315 #endif 316 acc_pv->pv_size += pv->pv_size; 317 return true; 318 } 319 320 return false; 321 } 322 323 static void 324 add_pages(struct bootmem_info *bmi, pv_addr_t *pv) 325 { 326 pv_addr_t **pvp = &SLIST_FIRST(&bmi->bmi_chunks); 327 while ((*pvp) != NULL && (*pvp)->pv_va <= pv->pv_va) { 328 pv_addr_t * const pv0 = (*pvp); 329 KASSERT(SLIST_NEXT(pv0, pv_list) == NULL || pv0->pv_pa < SLIST_NEXT(pv0, pv_list)->pv_pa); 330 if (concat_pvaddr(pv0, pv)) { 331 VPRINTF("%s: %s pv %p (%#lx..%#lx) to %#lx..%#lx\n", 332 __func__, "appending", pv, 333 pv->pv_pa, pv->pv_pa + pv->pv_size - 1, 334 pv0->pv_pa, pv0->pv_pa + pv0->pv_size - pv->pv_size - 1); 335 pv = SLIST_NEXT(pv0, pv_list); 336 if (pv != NULL && concat_pvaddr(pv0, pv)) { 337 VPRINTF("%s: %s pv %p (%#lx..%#lx) to %#lx..%#lx\n", 338 __func__, "merging", pv, 339 pv->pv_pa, pv->pv_pa + pv->pv_size - 1, 340 pv0->pv_pa, 341 pv0->pv_pa + pv0->pv_size - pv->pv_size - 1); 342 SLIST_REMOVE_AFTER(pv0, pv_list); 343 SLIST_INSERT_HEAD(&bmi->bmi_freechunks, pv, pv_list); 344 } 345 return; 346 } 347 KASSERT(pv->pv_va != (*pvp)->pv_va); 348 pvp = &SLIST_NEXT(*pvp, pv_list); 349 } 350 KASSERT((*pvp) == NULL || pv->pv_va < (*pvp)->pv_va); 351 pv_addr_t * const new_pv = SLIST_FIRST(&bmi->bmi_freechunks); 352 KASSERT(new_pv != NULL); 353 SLIST_REMOVE_HEAD(&bmi->bmi_freechunks, pv_list); 354 *new_pv = *pv; 355 SLIST_NEXT(new_pv, pv_list) = *pvp; 356 (*pvp) = new_pv; 357 358 VPRINTF("%s: adding pv %p (pa %#lx, va %#lx, %lu pages) ", 359 __func__, new_pv, new_pv->pv_pa, new_pv->pv_va, 360 new_pv->pv_size / PAGE_SIZE); 361 if (SLIST_NEXT(new_pv, pv_list)) { 362 VPRINTF("before pa %#lx\n", SLIST_NEXT(new_pv, pv_list)->pv_pa); 363 } else { 364 VPRINTF("at tail\n"); 365 } 366 } 367 368 static void 369 valloc_pages(struct bootmem_info *bmi, pv_addr_t *pv, size_t npages, 370 int prot, int cache, bool zero_p) 371 { 372 size_t nbytes = npages * PAGE_SIZE; 373 pv_addr_t *free_pv = bmi->bmi_freeblocks; 374 size_t free_idx = 0; 375 static bool l1pt_found; 376 377 KASSERT(npages > 0); 378 379 /* 380 * If we haven't allocated the kernel L1 page table and we are aligned 381 * at a L1 table boundary, alloc the memory for it. 382 */ 383 if (!l1pt_found 384 && (free_pv->pv_pa & (L1_TABLE_SIZE - 1)) == 0 385 && free_pv->pv_size >= L1_TABLE_SIZE) { 386 l1pt_found = true; 387 VPRINTF(" l1pt"); 388 389 valloc_pages(bmi, &kernel_l1pt, L1_TABLE_SIZE / PAGE_SIZE, 390 VM_PROT_READ | VM_PROT_WRITE, PTE_PAGETABLE, true); 391 add_pages(bmi, &kernel_l1pt); 392 } 393 394 while (nbytes > free_pv->pv_size) { 395 free_pv++; 396 free_idx++; 397 if (free_idx == bmi->bmi_nfreeblocks) { 398 panic("%s: could not allocate %zu bytes", 399 __func__, nbytes); 400 } 401 } 402 403 /* 404 * As we allocate the memory, make sure that we don't walk over 405 * our current first level translation table. 406 */ 407 KASSERT((armreg_ttbr_read() & ~(L1_TABLE_SIZE - 1)) != free_pv->pv_pa); 408 409 #if defined(FDT) 410 fdt_memory_remove_range(free_pv->pv_pa, nbytes); 411 #endif 412 pv->pv_pa = free_pv->pv_pa; 413 pv->pv_va = free_pv->pv_va; 414 pv->pv_size = nbytes; 415 pv->pv_prot = prot; 416 pv->pv_cache = cache; 417 418 /* 419 * If PTE_PAGETABLE uses the same cache modes as PTE_CACHE 420 * just use PTE_CACHE. 421 */ 422 if (cache == PTE_PAGETABLE 423 && pte_l1_s_cache_mode == pte_l1_s_cache_mode_pt 424 && pte_l2_l_cache_mode == pte_l2_l_cache_mode_pt 425 && pte_l2_s_cache_mode == pte_l2_s_cache_mode_pt) 426 pv->pv_cache = PTE_CACHE; 427 428 free_pv->pv_pa += nbytes; 429 free_pv->pv_va += nbytes; 430 free_pv->pv_size -= nbytes; 431 if (free_pv->pv_size == 0) { 432 --bmi->bmi_nfreeblocks; 433 for (; free_idx < bmi->bmi_nfreeblocks; free_idx++) { 434 free_pv[0] = free_pv[1]; 435 } 436 } 437 438 bmi->bmi_freepages -= npages; 439 440 if (zero_p) 441 memset((void *)pv->pv_va, 0, nbytes); 442 } 443 444 void 445 arm32_kernel_vm_init(vaddr_t kernel_vm_base, vaddr_t vectors, vaddr_t iovbase, 446 const struct pmap_devmap *devmap, bool mapallmem_p) 447 { 448 struct bootmem_info * const bmi = &bootmem_info; 449 #ifdef MULTIPROCESSOR 450 const size_t cpu_num = arm_cpu_max; 451 #else 452 const size_t cpu_num = 1; 453 #endif 454 455 #ifdef ARM_HAS_VBAR 456 const bool map_vectors_p = false; 457 #elif defined(CPU_ARMV7) || defined(CPU_ARM11) 458 const bool map_vectors_p = vectors == ARM_VECTORS_HIGH 459 || (armreg_pfr1_read() & ARM_PFR1_SEC_MASK) == 0; 460 #else 461 const bool map_vectors_p = true; 462 #endif 463 464 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS 465 KASSERT(mapallmem_p); 466 #ifdef ARM_MMU_EXTENDED 467 /* 468 * The direct map VA space ends at the start of the kernel VM space. 469 */ 470 pmap_directlimit = kernel_vm_base; 471 #else 472 KASSERT(kernel_vm_base - KERNEL_BASE >= physical_end - physical_start); 473 #endif /* ARM_MMU_EXTENDED */ 474 #endif /* __HAVE_MM_MD_DIRECT_MAPPED_PHYS */ 475 476 /* 477 * Calculate the number of L2 pages needed for mapping the 478 * kernel + data + stuff. Assume 2 L2 pages for kernel, 1 for vectors, 479 * and 1 for IO 480 */ 481 size_t kernel_size = bmi->bmi_kernelend; 482 kernel_size -= (bmi->bmi_kernelstart & -L2_S_SEGSIZE); 483 kernel_size += L1_TABLE_SIZE; 484 kernel_size += PAGE_SIZE * KERNEL_L2PT_VMDATA_NUM; 485 kernel_size += PAGE_SIZE * KERNEL_L2PT_KASAN_NUM; 486 if (map_vectors_p) { 487 kernel_size += PAGE_SIZE; /* L2PT for VECTORS */ 488 } 489 if (iovbase) { 490 kernel_size += PAGE_SIZE; /* L2PT for IO */ 491 } 492 kernel_size += 493 cpu_num * (ABT_STACK_SIZE + FIQ_STACK_SIZE + IRQ_STACK_SIZE 494 + UND_STACK_SIZE + UPAGES) * PAGE_SIZE; 495 kernel_size += round_page(MSGBUFSIZE); 496 kernel_size += 0x10000; /* slop */ 497 if (!mapallmem_p) { 498 kernel_size += PAGE_SIZE 499 * howmany(kernel_size, L2_S_SEGSIZE); 500 } 501 kernel_size = round_page(kernel_size); 502 503 /* 504 * Now we know how many L2 pages it will take. 505 */ 506 const size_t KERNEL_L2PT_KERNEL_NUM = 507 howmany(kernel_size, L2_S_SEGSIZE); 508 509 VPRINTF("%s: %zu L2 pages are needed to map %#zx kernel bytes\n", 510 __func__, KERNEL_L2PT_KERNEL_NUM, kernel_size); 511 512 KASSERT(KERNEL_L2PT_KERNEL_NUM + KERNEL_L2PT_VMDATA_NUM < __arraycount(bmi->bmi_l2pts)); 513 pv_addr_t * const kernel_l2pt = bmi->bmi_l2pts; 514 pv_addr_t * const vmdata_l2pt = kernel_l2pt + KERNEL_L2PT_KERNEL_NUM; 515 pv_addr_t msgbuf; 516 pv_addr_t text; 517 pv_addr_t data; 518 pv_addr_t chunks[__arraycount(bmi->bmi_l2pts) + 11]; 519 #if ARM_MMU_XSCALE == 1 520 pv_addr_t minidataclean; 521 #endif 522 523 /* 524 * We need to allocate some fixed page tables to get the kernel going. 525 * 526 * We are going to allocate our bootstrap pages from the beginning of 527 * the free space that we just calculated. We allocate one page 528 * directory and a number of page tables and store the physical 529 * addresses in the bmi_l2pts array in bootmem_info. 530 * 531 * The kernel page directory must be on a 16K boundary. The page 532 * tables must be on 4K boundaries. What we do is allocate the 533 * page directory on the first 16K boundary that we encounter, and 534 * the page tables on 4K boundaries otherwise. Since we allocate 535 * at least 3 L2 page tables, we are guaranteed to encounter at 536 * least one 16K aligned region. 537 */ 538 539 VPRINTF("%s: allocating page tables for", __func__); 540 for (size_t i = 0; i < __arraycount(chunks); i++) { 541 SLIST_INSERT_HEAD(&bmi->bmi_freechunks, &chunks[i], pv_list); 542 } 543 544 kernel_l1pt.pv_pa = 0; 545 kernel_l1pt.pv_va = 0; 546 547 /* 548 * Allocate the L2 pages, but if we get to a page that is aligned for 549 * an L1 page table, we will allocate the pages for it first and then 550 * allocate the L2 page. 551 */ 552 553 if (map_vectors_p) { 554 /* 555 * First allocate L2 page for the vectors. 556 */ 557 VPRINTF(" vector"); 558 valloc_pages(bmi, &bmi->bmi_vector_l2pt, 1, 559 VM_PROT_READ | VM_PROT_WRITE, PTE_PAGETABLE, true); 560 add_pages(bmi, &bmi->bmi_vector_l2pt); 561 } 562 563 /* 564 * Now allocate L2 pages for the kernel 565 */ 566 VPRINTF(" kernel"); 567 for (size_t idx = 0; idx < KERNEL_L2PT_KERNEL_NUM; ++idx) { 568 valloc_pages(bmi, &kernel_l2pt[idx], 1, 569 VM_PROT_READ | VM_PROT_WRITE, PTE_PAGETABLE, true); 570 add_pages(bmi, &kernel_l2pt[idx]); 571 } 572 573 /* 574 * Now allocate L2 pages for the initial kernel VA space. 575 */ 576 VPRINTF(" vm"); 577 for (size_t idx = 0; idx < KERNEL_L2PT_VMDATA_NUM; ++idx) { 578 valloc_pages(bmi, &vmdata_l2pt[idx], 1, 579 VM_PROT_READ | VM_PROT_WRITE, PTE_PAGETABLE, true); 580 add_pages(bmi, &vmdata_l2pt[idx]); 581 } 582 583 #ifdef KASAN 584 /* 585 * Now allocate L2 pages for the KASAN shadow map l2pt VA space. 586 */ 587 VPRINTF(" kasan"); 588 for (size_t idx = 0; idx < KERNEL_L2PT_KASAN_NUM; ++idx) { 589 valloc_pages(bmi, &kasan_l2pt[idx], 1, 590 VM_PROT_READ | VM_PROT_WRITE, PTE_PAGETABLE, true); 591 add_pages(bmi, &kasan_l2pt[idx]); 592 } 593 594 #endif 595 /* 596 * If someone wanted a L2 page for I/O, allocate it now. 597 */ 598 if (iovbase) { 599 VPRINTF(" io"); 600 valloc_pages(bmi, &bmi->bmi_io_l2pt, 1, 601 VM_PROT_READ | VM_PROT_WRITE, PTE_PAGETABLE, true); 602 add_pages(bmi, &bmi->bmi_io_l2pt); 603 } 604 605 VPRINTF("%s: allocating stacks\n", __func__); 606 607 /* Allocate stacks for all modes and CPUs */ 608 valloc_pages(bmi, &abtstack, ABT_STACK_SIZE * cpu_num, 609 VM_PROT_READ | VM_PROT_WRITE, PTE_CACHE, true); 610 add_pages(bmi, &abtstack); 611 valloc_pages(bmi, &fiqstack, FIQ_STACK_SIZE * cpu_num, 612 VM_PROT_READ | VM_PROT_WRITE, PTE_CACHE, true); 613 add_pages(bmi, &fiqstack); 614 valloc_pages(bmi, &irqstack, IRQ_STACK_SIZE * cpu_num, 615 VM_PROT_READ | VM_PROT_WRITE, PTE_CACHE, true); 616 add_pages(bmi, &irqstack); 617 valloc_pages(bmi, &undstack, UND_STACK_SIZE * cpu_num, 618 VM_PROT_READ | VM_PROT_WRITE, PTE_CACHE, true); 619 add_pages(bmi, &undstack); 620 valloc_pages(bmi, &idlestack, UPAGES * cpu_num, /* SVC32 */ 621 VM_PROT_READ | VM_PROT_WRITE, PTE_CACHE, true); 622 add_pages(bmi, &idlestack); 623 valloc_pages(bmi, &kernelstack, UPAGES, /* SVC32 */ 624 VM_PROT_READ | VM_PROT_WRITE, PTE_CACHE, true); 625 add_pages(bmi, &kernelstack); 626 627 /* Allocate the message buffer from the end of memory. */ 628 const size_t msgbuf_pgs = round_page(MSGBUFSIZE) / PAGE_SIZE; 629 valloc_pages(bmi, &msgbuf, msgbuf_pgs, 630 VM_PROT_READ | VM_PROT_WRITE, PTE_CACHE, false); 631 add_pages(bmi, &msgbuf); 632 msgbufphys = msgbuf.pv_pa; 633 msgbufaddr = (void *)msgbuf.pv_va; 634 635 #ifdef KASAN 636 kasan_kernelstart = KERNEL_BASE; 637 kasan_kernelsize = (msgbuf.pv_va + round_page(MSGBUFSIZE)) - KERNEL_BASE; 638 #endif 639 640 if (map_vectors_p) { 641 /* 642 * Allocate a page for the system vector page. 643 * This page will just contain the system vectors and can be 644 * shared by all processes. 645 */ 646 VPRINTF(" vector"); 647 648 valloc_pages(bmi, &systempage, 1, 649 VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE, 650 PTE_CACHE, true); 651 } 652 systempage.pv_va = vectors; 653 654 /* 655 * If the caller needed a few extra pages for some reason, allocate 656 * them now. 657 */ 658 #if ARM_MMU_XSCALE == 1 659 #if (ARM_NMMUS > 1) 660 if (xscale_use_minidata) 661 #endif 662 valloc_pages(bmi, &minidataclean, 1, 663 VM_PROT_READ | VM_PROT_WRITE, 0, true); 664 #endif 665 666 /* 667 * Ok we have allocated physical pages for the primary kernel 668 * page tables and stacks. Let's just confirm that. 669 */ 670 if (kernel_l1pt.pv_va == 0 671 && (!kernel_l1pt.pv_pa || (kernel_l1pt.pv_pa & (L1_TABLE_SIZE - 1)) != 0)) 672 panic("%s: Failed to allocate or align the kernel " 673 "page directory", __func__); 674 675 VPRINTF("Creating L1 page table at 0x%08lx/0x%08lx\n", 676 kernel_l1pt.pv_va, kernel_l1pt.pv_pa); 677 678 /* 679 * Now we start construction of the L1 page table 680 * We start by mapping the L2 page tables into the L1. 681 * This means that we can replace L1 mappings later on if necessary 682 */ 683 vaddr_t l1pt_va = kernel_l1pt.pv_va; 684 paddr_t l1pt_pa = kernel_l1pt.pv_pa; 685 686 if (map_vectors_p) { 687 /* Map the L2 pages tables in the L1 page table */ 688 const vaddr_t va = systempage.pv_va & -L2_S_SEGSIZE; 689 690 pmap_link_l2pt(l1pt_va, va, &bmi->bmi_vector_l2pt); 691 692 VPRINTF("%s: adding L2 pt (VA %#lx, PA %#lx) for VA %#lx %s\n", 693 __func__, bmi->bmi_vector_l2pt.pv_va, 694 bmi->bmi_vector_l2pt.pv_pa, systempage.pv_va, "(vectors)"); 695 } 696 697 /* 698 * This enforces an alignment requirement of L2_S_SEGSIZE for kernel 699 * start PA 700 */ 701 const vaddr_t kernel_base = 702 KERN_PHYSTOV(bmi->bmi_kernelstart & -L2_S_SEGSIZE); 703 704 VPRINTF("%s: kernel_base %lx KERNEL_L2PT_KERNEL_NUM %zu\n", __func__, 705 kernel_base, KERNEL_L2PT_KERNEL_NUM); 706 707 for (size_t idx = 0; idx < KERNEL_L2PT_KERNEL_NUM; idx++) { 708 const vaddr_t va = kernel_base + idx * L2_S_SEGSIZE; 709 710 pmap_link_l2pt(l1pt_va, va, &kernel_l2pt[idx]); 711 712 VPRINTF("%s: adding L2 pt (VA %#lx, PA %#lx) for VA %#lx %s\n", 713 __func__, kernel_l2pt[idx].pv_va, kernel_l2pt[idx].pv_pa, 714 va, "(kernel)"); 715 } 716 717 VPRINTF("%s: kernel_vm_base %lx KERNEL_L2PT_VMDATA_NUM %d\n", __func__, 718 kernel_vm_base, KERNEL_L2PT_VMDATA_NUM); 719 720 for (size_t idx = 0; idx < KERNEL_L2PT_VMDATA_NUM; idx++) { 721 const vaddr_t va = kernel_vm_base + idx * L2_S_SEGSIZE; 722 723 pmap_link_l2pt(l1pt_va, va, &vmdata_l2pt[idx]); 724 725 VPRINTF("%s: adding L2 pt (VA %#lx, PA %#lx) for VA %#lx %s\n", 726 __func__, vmdata_l2pt[idx].pv_va, vmdata_l2pt[idx].pv_pa, 727 va, "(vm)"); 728 } 729 if (iovbase) { 730 const vaddr_t va = iovbase & -L2_S_SEGSIZE; 731 732 pmap_link_l2pt(l1pt_va, va, &bmi->bmi_io_l2pt); 733 734 VPRINTF("%s: adding L2 pt (VA %#lx, PA %#lx) for VA %#lx %s\n", 735 __func__, bmi->bmi_io_l2pt.pv_va, bmi->bmi_io_l2pt.pv_pa, 736 va, "(io)"); 737 } 738 739 #ifdef KASAN 740 VPRINTF("%s: kasan_shadow_base %x KERNEL_L2PT_KASAN_NUM %d\n", __func__, 741 VM_KERNEL_KASAN_BASE, KERNEL_L2PT_KASAN_NUM); 742 743 for (size_t idx = 0; idx < KERNEL_L2PT_KASAN_NUM; idx++) { 744 const vaddr_t va = VM_KERNEL_KASAN_BASE + idx * L2_S_SEGSIZE; 745 746 pmap_link_l2pt(l1pt_va, va, &kasan_l2pt[idx]); 747 748 VPRINTF("%s: adding L2 pt (VA %#lx, PA %#lx) for VA %#lx %s\n", 749 __func__, kasan_l2pt[idx].pv_va, kasan_l2pt[idx].pv_pa, 750 va, "(kasan)"); 751 } 752 kasan_l2pts_created = true; 753 #endif 754 755 /* update the top of the kernel VM */ 756 pmap_curmaxkvaddr = 757 kernel_vm_base + (KERNEL_L2PT_VMDATA_NUM * L2_S_SEGSIZE); 758 759 // This could be done earlier and then the kernel data and pages 760 // allocated above would get merged (concatentated) 761 762 VPRINTF("Mapping kernel\n"); 763 764 extern char etext[]; 765 size_t totalsize = bmi->bmi_kernelend - bmi->bmi_kernelstart; 766 size_t textsize = KERN_VTOPHYS((uintptr_t)etext) - bmi->bmi_kernelstart; 767 768 textsize = (textsize + PGOFSET) & ~PGOFSET; 769 770 /* start at offset of kernel in RAM */ 771 772 text.pv_pa = bmi->bmi_kernelstart; 773 text.pv_va = KERN_PHYSTOV(bmi->bmi_kernelstart); 774 text.pv_size = textsize; 775 text.pv_prot = VM_PROT_READ | VM_PROT_EXECUTE; 776 text.pv_cache = PTE_CACHE; 777 778 VPRINTF("%s: adding chunk for kernel text %#lx..%#lx (VA %#lx)\n", 779 __func__, text.pv_pa, text.pv_pa + text.pv_size - 1, text.pv_va); 780 781 add_pages(bmi, &text); 782 783 data.pv_pa = text.pv_pa + textsize; 784 data.pv_va = text.pv_va + textsize; 785 data.pv_size = totalsize - textsize; 786 data.pv_prot = VM_PROT_READ | VM_PROT_WRITE; 787 data.pv_cache = PTE_CACHE; 788 789 VPRINTF("%s: adding chunk for kernel data/bss %#lx..%#lx (VA %#lx)\n", 790 __func__, data.pv_pa, data.pv_pa + data.pv_size - 1, data.pv_va); 791 792 add_pages(bmi, &data); 793 794 VPRINTF("Listing Chunks\n"); 795 796 pv_addr_t *lpv; 797 SLIST_FOREACH(lpv, &bmi->bmi_chunks, pv_list) { 798 VPRINTF("%s: pv %p: chunk VA %#lx..%#lx " 799 "(PA %#lx, prot %d, cache %d)\n", 800 __func__, lpv, lpv->pv_va, lpv->pv_va + lpv->pv_size - 1, 801 lpv->pv_pa, lpv->pv_prot, lpv->pv_cache); 802 } 803 VPRINTF("\nMapping Chunks\n"); 804 805 pv_addr_t cur_pv; 806 pv_addr_t *pv = SLIST_FIRST(&bmi->bmi_chunks); 807 if (!mapallmem_p || pv->pv_pa == bmi->bmi_start) { 808 cur_pv = *pv; 809 KASSERTMSG(cur_pv.pv_va >= KERNEL_BASE, "%#lx", cur_pv.pv_va); 810 pv = SLIST_NEXT(pv, pv_list); 811 } else { 812 cur_pv.pv_va = KERNEL_BASE; 813 cur_pv.pv_pa = KERN_VTOPHYS(cur_pv.pv_va); 814 cur_pv.pv_size = pv->pv_pa - cur_pv.pv_pa; 815 cur_pv.pv_prot = VM_PROT_READ | VM_PROT_WRITE; 816 cur_pv.pv_cache = PTE_CACHE; 817 } 818 while (pv != NULL) { 819 if (mapallmem_p) { 820 if (concat_pvaddr(&cur_pv, pv)) { 821 pv = SLIST_NEXT(pv, pv_list); 822 continue; 823 } 824 if (cur_pv.pv_pa + cur_pv.pv_size < pv->pv_pa) { 825 /* 826 * See if we can extend the current pv to emcompass the 827 * hole, and if so do it and retry the concatenation. 828 */ 829 if (cur_pv.pv_prot == (VM_PROT_READ | VM_PROT_WRITE) 830 && cur_pv.pv_cache == PTE_CACHE) { 831 cur_pv.pv_size = pv->pv_pa - cur_pv.pv_va; 832 continue; 833 } 834 835 /* 836 * We couldn't so emit the current chunk and then 837 */ 838 VPRINTF("%s: mapping chunk VA %#lx..%#lx " 839 "(PA %#lx, prot %d, cache %d)\n", 840 __func__, 841 cur_pv.pv_va, cur_pv.pv_va + cur_pv.pv_size - 1, 842 cur_pv.pv_pa, cur_pv.pv_prot, cur_pv.pv_cache); 843 pmap_map_chunk(l1pt_va, cur_pv.pv_va, cur_pv.pv_pa, 844 cur_pv.pv_size, cur_pv.pv_prot, cur_pv.pv_cache); 845 846 /* 847 * set the current chunk to the hole and try again. 848 */ 849 cur_pv.pv_pa += cur_pv.pv_size; 850 cur_pv.pv_va += cur_pv.pv_size; 851 cur_pv.pv_size = pv->pv_pa - cur_pv.pv_va; 852 cur_pv.pv_prot = VM_PROT_READ | VM_PROT_WRITE; 853 cur_pv.pv_cache = PTE_CACHE; 854 continue; 855 } 856 } 857 858 /* 859 * The new pv didn't concatenate so emit the current one 860 * and use the new pv as the current pv. 861 */ 862 VPRINTF("%s: mapping chunk VA %#lx..%#lx " 863 "(PA %#lx, prot %d, cache %d)\n", 864 __func__, cur_pv.pv_va, cur_pv.pv_va + cur_pv.pv_size - 1, 865 cur_pv.pv_pa, cur_pv.pv_prot, cur_pv.pv_cache); 866 pmap_map_chunk(l1pt_va, cur_pv.pv_va, cur_pv.pv_pa, 867 cur_pv.pv_size, cur_pv.pv_prot, cur_pv.pv_cache); 868 cur_pv = *pv; 869 pv = SLIST_NEXT(pv, pv_list); 870 } 871 872 /* 873 * If we are mapping all of memory, let's map the rest of memory. 874 */ 875 if (mapallmem_p && cur_pv.pv_pa + cur_pv.pv_size < bmi->bmi_end) { 876 if (cur_pv.pv_prot == (VM_PROT_READ | VM_PROT_WRITE) 877 && cur_pv.pv_cache == PTE_CACHE) { 878 cur_pv.pv_size = bmi->bmi_end - cur_pv.pv_pa; 879 } else { 880 KASSERTMSG(cur_pv.pv_va + cur_pv.pv_size <= kernel_vm_base, 881 "%#lx >= %#lx", cur_pv.pv_va + cur_pv.pv_size, 882 kernel_vm_base); 883 VPRINTF("%s: mapping chunk VA %#lx..%#lx " 884 "(PA %#lx, prot %d, cache %d)\n", 885 __func__, cur_pv.pv_va, cur_pv.pv_va + cur_pv.pv_size - 1, 886 cur_pv.pv_pa, cur_pv.pv_prot, cur_pv.pv_cache); 887 pmap_map_chunk(l1pt_va, cur_pv.pv_va, cur_pv.pv_pa, 888 cur_pv.pv_size, cur_pv.pv_prot, cur_pv.pv_cache); 889 cur_pv.pv_pa += cur_pv.pv_size; 890 cur_pv.pv_va += cur_pv.pv_size; 891 cur_pv.pv_size = bmi->bmi_end - cur_pv.pv_pa; 892 cur_pv.pv_prot = VM_PROT_READ | VM_PROT_WRITE; 893 cur_pv.pv_cache = PTE_CACHE; 894 } 895 } 896 897 /* 898 * The amount we can direct map is limited by the start of the 899 * virtual part of the kernel address space. Don't overrun 900 * into it. 901 */ 902 if (mapallmem_p && cur_pv.pv_va + cur_pv.pv_size > kernel_vm_base) { 903 cur_pv.pv_size = kernel_vm_base - cur_pv.pv_va; 904 } 905 906 /* 907 * Now we map the final chunk. 908 */ 909 VPRINTF("%s: mapping last chunk VA %#lx..%#lx (PA %#lx, prot %d, cache %d)\n", 910 __func__, cur_pv.pv_va, cur_pv.pv_va + cur_pv.pv_size - 1, 911 cur_pv.pv_pa, cur_pv.pv_prot, cur_pv.pv_cache); 912 pmap_map_chunk(l1pt_va, cur_pv.pv_va, cur_pv.pv_pa, 913 cur_pv.pv_size, cur_pv.pv_prot, cur_pv.pv_cache); 914 915 /* 916 * Now we map the stuff that isn't directly after the kernel 917 */ 918 if (map_vectors_p) { 919 /* Map the vector page. */ 920 pmap_map_entry(l1pt_va, systempage.pv_va, systempage.pv_pa, 921 VM_PROT_READ | VM_PROT_WRITE | VM_PROT_EXECUTE, PTE_CACHE); 922 } 923 924 /* Map the Mini-Data cache clean area. */ 925 #if ARM_MMU_XSCALE == 1 926 #if (ARM_NMMUS > 1) 927 if (xscale_use_minidata) 928 #endif 929 xscale_setup_minidata(l1pt_va, minidataclean.pv_va, 930 minidataclean.pv_pa); 931 #endif 932 933 /* 934 * Map integrated peripherals at same address in first level page 935 * table so that we can continue to use console. 936 */ 937 if (devmap) 938 pmap_devmap_bootstrap(l1pt_va, devmap); 939 940 /* Tell the user about where all the bits and pieces live. */ 941 VPRINTF("%22s Physical Virtual Num\n", " "); 942 VPRINTF("%22s Starting Ending Starting Ending Pages\n", " "); 943 944 #ifdef VERBOSE_INIT_ARM 945 static const char mem_fmt[] = 946 "%20s: 0x%08lx 0x%08lx 0x%08lx 0x%08lx %u\n"; 947 static const char mem_fmt_nov[] = 948 "%20s: 0x%08lx 0x%08lx %zu\n"; 949 #endif 950 951 #if 0 952 // XXX Doesn't make sense if kernel not at bottom of RAM 953 VPRINTF(mem_fmt, "SDRAM", bmi->bmi_start, bmi->bmi_end - 1, 954 KERN_PHYSTOV(bmi->bmi_start), KERN_PHYSTOV(bmi->bmi_end - 1), 955 (int)physmem); 956 #endif 957 VPRINTF(mem_fmt, "text section", 958 text.pv_pa, text.pv_pa + text.pv_size - 1, 959 text.pv_va, text.pv_va + text.pv_size - 1, 960 (int)(text.pv_size / PAGE_SIZE)); 961 VPRINTF(mem_fmt, "data section", 962 KERN_VTOPHYS((vaddr_t)__data_start), KERN_VTOPHYS((vaddr_t)_edata), 963 (vaddr_t)__data_start, (vaddr_t)_edata, 964 (int)((round_page((vaddr_t)_edata) 965 - trunc_page((vaddr_t)__data_start)) / PAGE_SIZE)); 966 VPRINTF(mem_fmt, "bss section", 967 KERN_VTOPHYS((vaddr_t)__bss_start), KERN_VTOPHYS((vaddr_t)__bss_end__), 968 (vaddr_t)__bss_start, (vaddr_t)__bss_end__, 969 (int)((round_page((vaddr_t)__bss_end__) 970 - trunc_page((vaddr_t)__bss_start)) / PAGE_SIZE)); 971 VPRINTF(mem_fmt, "L1 page directory", 972 kernel_l1pt.pv_pa, kernel_l1pt.pv_pa + L1_TABLE_SIZE - 1, 973 kernel_l1pt.pv_va, kernel_l1pt.pv_va + L1_TABLE_SIZE - 1, 974 L1_TABLE_SIZE / PAGE_SIZE); 975 VPRINTF(mem_fmt, "ABT stack (CPU 0)", 976 abtstack.pv_pa, abtstack.pv_pa + (ABT_STACK_SIZE * PAGE_SIZE) - 1, 977 abtstack.pv_va, abtstack.pv_va + (ABT_STACK_SIZE * PAGE_SIZE) - 1, 978 ABT_STACK_SIZE); 979 VPRINTF(mem_fmt, "FIQ stack (CPU 0)", 980 fiqstack.pv_pa, fiqstack.pv_pa + (FIQ_STACK_SIZE * PAGE_SIZE) - 1, 981 fiqstack.pv_va, fiqstack.pv_va + (FIQ_STACK_SIZE * PAGE_SIZE) - 1, 982 FIQ_STACK_SIZE); 983 VPRINTF(mem_fmt, "IRQ stack (CPU 0)", 984 irqstack.pv_pa, irqstack.pv_pa + (IRQ_STACK_SIZE * PAGE_SIZE) - 1, 985 irqstack.pv_va, irqstack.pv_va + (IRQ_STACK_SIZE * PAGE_SIZE) - 1, 986 IRQ_STACK_SIZE); 987 VPRINTF(mem_fmt, "UND stack (CPU 0)", 988 undstack.pv_pa, undstack.pv_pa + (UND_STACK_SIZE * PAGE_SIZE) - 1, 989 undstack.pv_va, undstack.pv_va + (UND_STACK_SIZE * PAGE_SIZE) - 1, 990 UND_STACK_SIZE); 991 VPRINTF(mem_fmt, "IDLE stack (CPU 0)", 992 idlestack.pv_pa, idlestack.pv_pa + (UPAGES * PAGE_SIZE) - 1, 993 idlestack.pv_va, idlestack.pv_va + (UPAGES * PAGE_SIZE) - 1, 994 UPAGES); 995 VPRINTF(mem_fmt, "SVC stack", 996 kernelstack.pv_pa, kernelstack.pv_pa + (UPAGES * PAGE_SIZE) - 1, 997 kernelstack.pv_va, kernelstack.pv_va + (UPAGES * PAGE_SIZE) - 1, 998 UPAGES); 999 VPRINTF(mem_fmt, "Message Buffer", 1000 msgbuf.pv_pa, msgbuf.pv_pa + (msgbuf_pgs * PAGE_SIZE) - 1, 1001 msgbuf.pv_va, msgbuf.pv_va + (msgbuf_pgs * PAGE_SIZE) - 1, 1002 (int)msgbuf_pgs); 1003 if (map_vectors_p) { 1004 VPRINTF(mem_fmt, "Exception Vectors", 1005 systempage.pv_pa, systempage.pv_pa + PAGE_SIZE - 1, 1006 systempage.pv_va, systempage.pv_va + PAGE_SIZE - 1, 1007 1); 1008 } 1009 for (size_t i = 0; i < bmi->bmi_nfreeblocks; i++) { 1010 pv = &bmi->bmi_freeblocks[i]; 1011 1012 VPRINTF(mem_fmt_nov, "Free Memory", 1013 pv->pv_pa, pv->pv_pa + pv->pv_size - 1, 1014 pv->pv_size / PAGE_SIZE); 1015 } 1016 /* 1017 * Now we have the real page tables in place so we can switch to them. 1018 * Once this is done we will be running with the REAL kernel page 1019 * tables. 1020 */ 1021 1022 VPRINTF("TTBR0=%#x", armreg_ttbr_read()); 1023 #ifdef _ARM_ARCH_6 1024 VPRINTF(" TTBR1=%#x TTBCR=%#x CONTEXTIDR=%#x", 1025 armreg_ttbr1_read(), armreg_ttbcr_read(), 1026 armreg_contextidr_read()); 1027 #endif 1028 VPRINTF("\n"); 1029 1030 /* Switch tables */ 1031 VPRINTF("switching to new L1 page table @%#lx...\n", l1pt_pa); 1032 1033 cpu_ttb = l1pt_pa; 1034 1035 cpu_domains(DOMAIN_DEFAULT); 1036 1037 cpu_idcache_wbinv_all(); 1038 1039 #ifdef __HAVE_GENERIC_START 1040 1041 /* 1042 * Turn on caches and set SCTLR/ACTLR 1043 */ 1044 cpu_setup(boot_args); 1045 #endif 1046 1047 VPRINTF(" ttb"); 1048 1049 #ifdef ARM_MMU_EXTENDED 1050 /* 1051 * TTBCR should have been initialized by the MD start code. 1052 */ 1053 KASSERT((armreg_contextidr_read() & 0xff) == 0); 1054 KASSERT(armreg_ttbcr_read() == __SHIFTIN(1, TTBCR_S_N)); 1055 /* 1056 * Disable lookups via TTBR0 until there is an activated pmap. 1057 */ 1058 armreg_ttbcr_write(armreg_ttbcr_read() | TTBCR_S_PD0); 1059 cpu_setttb(l1pt_pa, KERNEL_PID); 1060 isb(); 1061 #else 1062 cpu_setttb(l1pt_pa, true); 1063 #endif 1064 1065 cpu_tlb_flushID(); 1066 1067 #ifdef KASAN 1068 extern uint8_t start_stacks_bottom[]; 1069 kasan_early_init((void *)start_stacks_bottom); 1070 #endif 1071 1072 #ifdef ARM_MMU_EXTENDED 1073 VPRINTF("\nsctlr=%#x actlr=%#x\n", 1074 armreg_sctlr_read(), armreg_auxctl_read()); 1075 #else 1076 VPRINTF(" (TTBR0=%#x)", armreg_ttbr_read()); 1077 #endif 1078 1079 #ifdef MULTIPROCESSOR 1080 #ifndef __HAVE_GENERIC_START 1081 /* 1082 * Kick the secondaries to load the TTB. After which they'll go 1083 * back to sleep to wait for the final kick so they will hatch. 1084 */ 1085 VPRINTF(" hatchlings"); 1086 cpu_boot_secondary_processors(); 1087 #endif 1088 #endif 1089 1090 VPRINTF(" OK\n"); 1091 } 1092
Indexes created Mon Sep 29 21:09:56 GMT 2025