arm32_kvminit.c revision 1.46
1 /* $NetBSD: arm32_kvminit.c,v 1.46 2018/10/18 09:01:52 skrll Exp $ */ 2 3 /* 4 * Copyright (c) 2002, 2003, 2005 Genetec Corporation. All rights reserved. 5 * Written by Hiroyuki Bessho for Genetec Corporation. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. The name of Genetec Corporation may not be used to endorse or 16 * promote products derived from this software without specific prior 17 * written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY GENETEC CORPORATION ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENETEC CORPORATION 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 * 31 * Copyright (c) 2001 Wasabi Systems, Inc. 32 * All rights reserved. 33 * 34 * Written by Jason R. Thorpe for Wasabi Systems, Inc. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice, this list of conditions and the following disclaimer. 41 * 2. Redistributions in binary form must reproduce the above copyright 42 * notice, this list of conditions and the following disclaimer in the 43 * documentation and/or other materials provided with the distribution. 44 * 3. All advertising materials mentioning features or use of this software 45 * must display the following acknowledgement: 46 * This product includes software developed for the NetBSD Project by 47 * Wasabi Systems, Inc. 48 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 49 * or promote products derived from this software without specific prior 50 * written permission. 51 * 52 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 53 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 54 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 55 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 56 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 57 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 58 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 59 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 60 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 61 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 62 * POSSIBILITY OF SUCH DAMAGE. 63 * 64 * Copyright (c) 1997,1998 Mark Brinicombe. 65 * Copyright (c) 1997,1998 Causality Limited. 66 * All rights reserved. 67 * 68 * Redistribution and use in source and binary forms, with or without 69 * modification, are permitted provided that the following conditions 70 * are met: 71 * 1. Redistributions of source code must retain the above copyright 72 * notice, this list of conditions and the following disclaimer. 73 * 2. Redistributions in binary form must reproduce the above copyright 74 * notice, this list of conditions and the following disclaimer in the 75 * documentation and/or other materials provided with the distribution. 76 * 3. All advertising materials mentioning features or use of this software 77 * must display the following acknowledgement: 78 * This product includes software developed by Mark Brinicombe 79 * for the NetBSD Project. 80 * 4. The name of the company nor the name of the author may be used to 81 * endorse or promote products derived from this software without specific 82 * prior written permission. 83 * 84 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 85 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 86 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 87 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 88 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 89 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 90 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 91 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 92 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 93 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 94 * SUCH DAMAGE. 95 * 96 * Copyright (c) 2007 Microsoft 97 * All rights reserved. 98 * 99 * Redistribution and use in source and binary forms, with or without 100 * modification, are permitted provided that the following conditions 101 * are met: 102 * 1. Redistributions of source code must retain the above copyright 103 * notice, this list of conditions and the following disclaimer. 104 * 2. Redistributions in binary form must reproduce the above copyright 105 * notice, this list of conditions and the following disclaimer in the 106 * documentation and/or other materials provided with the distribution. 107 * 3. All advertising materials mentioning features or use of this software 108 * must display the following acknowledgement: 109 * This product includes software developed by Microsoft 110 * 111 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 112 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 113 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 114 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTERS BE LIABLE FOR ANY DIRECT, 115 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 116 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 117 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 118 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 119 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 120 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 121 * SUCH DAMAGE. 122 */ 123 124 #include "opt_arm_debug.h" 125 #include "opt_arm_start.h" 126 #include "opt_fdt.h" 127 #include "opt_multiprocessor.h" 128 129 #include <sys/cdefs.h> 130 __KERNEL_RCSID(0, "$NetBSD: arm32_kvminit.c,v 1.46 2018/10/18 09:01:52 skrll Exp $"); 131 132 #include <sys/param.h> 133 #include <sys/device.h> 134 #include <sys/kernel.h> 135 #include <sys/reboot.h> 136 #include <sys/bus.h> 137 138 #include <dev/cons.h> 139 140 #include <uvm/uvm_extern.h> 141 142 #include <arm/locore.h> 143 #include <arm/db_machdep.h> 144 #include <arm/undefined.h> 145 #include <arm/bootconfig.h> 146 #include <arm/arm32/machdep.h> 147 148 #if defined(FDT) 149 #include <arch/evbarm/fdt/platform.h> 150 #include <arm/fdt/arm_fdtvar.h> 151 #endif 152 153 #ifdef MULTIPROCESSOR 154 #ifndef __HAVE_CPU_UAREA_ALLOC_IDLELWP 155 #error __HAVE_CPU_UAREA_ALLOC_IDLELWP required to not waste pages for idlestack 156 #endif 157 #endif 158 159 #ifdef VERBOSE_INIT_ARM 160 #define VPRINTF(...) printf(__VA_ARGS__) 161 #else 162 #define VPRINTF(...) __nothing 163 #endif 164 165 struct bootmem_info bootmem_info; 166 167 extern void *msgbufaddr; 168 paddr_t msgbufphys; 169 paddr_t physical_start; 170 paddr_t physical_end; 171 172 extern char etext[]; 173 extern char __data_start[], _edata[]; 174 extern char __bss_start[], __bss_end__[]; 175 extern char _end[]; 176 177 /* Page tables for mapping kernel VM */ 178 #define KERNEL_L2PT_VMDATA_NUM 8 /* start with 32MB of KVM */ 179 180 u_long kern_vtopdiff __attribute__((__section__(".data"))); 181 182 void 183 arm32_bootmem_init(paddr_t memstart, psize_t memsize, vsize_t kernelstart) 184 { 185 struct bootmem_info * const bmi = &bootmem_info; 186 pv_addr_t *pv = bmi->bmi_freeblocks; 187 188 /* 189 * FDT/generic start fills in kern_vtopdiff early 190 */ 191 #if defined(__HAVE_GENERIC_START) 192 extern char KERNEL_BASE_virt[]; 193 extern char ARM_BOOTSTRAP_LxPT[]; 194 195 VPRINTF("%s: kern_vtopdiff=%#lx\n", __func__, kern_vtopdiff); 196 197 vaddr_t kstartva = trunc_page((vaddr_t)KERNEL_BASE_virt); 198 vaddr_t kendva = round_page((vaddr_t)ARM_BOOTSTRAP_LxPT + L1_TABLE_SIZE); 199 200 kernelstart = KERN_VTOPHYS(kstartva); 201 202 VPRINTF("%s: kstartva=%#lx, kernelstart=%#lx\n", __func__, kstartva, kernelstart); 203 #else 204 vaddr_t kendva = round_page((vaddr_t)_end); 205 206 #if defined(KERNEL_BASE_VOFFSET) 207 kern_vtopdiff = KERNEL_BASE_VOFFSET; 208 #else 209 KASSERT(memstart == kernelstart); 210 kern_vtopdiff = KERNEL_BASE + memstart; 211 #endif 212 #endif 213 paddr_t kernelend = KERN_VTOPHYS(kendva); 214 215 VPRINTF("%s: memstart=%#lx, memsize=%#lx\n", 216 __func__, memstart, memsize); 217 VPRINTF("%s: kernelstart=%#lx, kernelend=%#lx\n", __func__, 218 kernelstart, kernelend); 219 220 physical_start = bmi->bmi_start = memstart; 221 physical_end = bmi->bmi_end = memstart + memsize; 222 #ifndef ARM_HAS_LPAE 223 if (physical_end == 0) { 224 physical_end = -PAGE_SIZE; 225 memsize -= PAGE_SIZE; 226 bmi->bmi_end -= PAGE_SIZE; 227 VPRINTF("%s: memsize shrunk by a page to avoid ending at 4GB\n", 228 __func__); 229 } 230 #endif 231 physmem = memsize / PAGE_SIZE; 232 233 /* 234 * Let's record where the kernel lives. 235 */ 236 237 bmi->bmi_kernelstart = kernelstart; 238 bmi->bmi_kernelend = kernelend; 239 240 #if defined(FDT) 241 fdt_add_reserved_memory_range(bmi->bmi_kernelstart, 242 bmi->bmi_kernelend - bmi->bmi_kernelstart); 243 #endif 244 245 VPRINTF("%s: kernel phys start %#lx end %#lx\n", __func__, kernelstart, 246 kernelend); 247 248 #if 0 249 // XXX Makes RPI abort 250 KASSERT((kernelstart & (L2_S_SEGSIZE - 1)) == 0); 251 #endif 252 /* 253 * Now the rest of the free memory must be after the kernel. 254 */ 255 pv->pv_pa = bmi->bmi_kernelend; 256 pv->pv_va = KERN_PHYSTOV(pv->pv_pa); 257 pv->pv_size = bmi->bmi_end - bmi->bmi_kernelend; 258 bmi->bmi_freepages += pv->pv_size / PAGE_SIZE; 259 VPRINTF("%s: adding %lu free pages: [%#lx..%#lx] (VA %#lx)\n", 260 __func__, pv->pv_size / PAGE_SIZE, pv->pv_pa, 261 pv->pv_pa + pv->pv_size - 1, pv->pv_va); 262 pv++; 263 264 #if !defined(__HAVE_GENERIC_START) 265 /* 266 * Add a free block for any memory before the kernel. 267 */ 268 if (bmi->bmi_start < bmi->bmi_kernelstart) { 269 pv->pv_pa = bmi->bmi_start; 270 pv->pv_va = KERN_PHYSTOV(pv->pv_pa); 271 pv->pv_size = bmi->bmi_kernelstart - pv->pv_pa; 272 bmi->bmi_freepages += pv->pv_size / PAGE_SIZE; 273 VPRINTF("%s: adding %lu free pages: [%#lx..%#lx] (VA %#lx)\n", 274 __func__, pv->pv_size / PAGE_SIZE, pv->pv_pa, 275 pv->pv_pa + pv->pv_size - 1, pv->pv_va); 276 pv++; 277 } 278 #endif 279 280 bmi->bmi_nfreeblocks = pv - bmi->bmi_freeblocks; 281 282 SLIST_INIT(&bmi->bmi_freechunks); 283 SLIST_INIT(&bmi->bmi_chunks); 284 } 285 286 static bool 287 concat_pvaddr(pv_addr_t *acc_pv, pv_addr_t *pv) 288 { 289 if (acc_pv->pv_pa + acc_pv->pv_size == pv->pv_pa 290 && acc_pv->pv_va + acc_pv->pv_size == pv->pv_va 291 && acc_pv->pv_prot == pv->pv_prot 292 && acc_pv->pv_cache == pv->pv_cache) { 293 #if 0 294 VPRINTF("%s: appending pv %p (%#lx..%#lx) to %#lx..%#lx\n", 295 __func__, pv, pv->pv_pa, pv->pv_pa + pv->pv_size, 296 acc_pv->pv_pa, acc_pv->pv_pa + acc_pv->pv_size); 297 #endif 298 acc_pv->pv_size += pv->pv_size; 299 return true; 300 } 301 302 return false; 303 } 304 305 static void 306 add_pages(struct bootmem_info *bmi, pv_addr_t *pv) 307 { 308 pv_addr_t **pvp = &SLIST_FIRST(&bmi->bmi_chunks); 309 while ((*pvp) != NULL && (*pvp)->pv_va <= pv->pv_va) { 310 pv_addr_t * const pv0 = (*pvp); 311 KASSERT(SLIST_NEXT(pv0, pv_list) == NULL || pv0->pv_pa < SLIST_NEXT(pv0, pv_list)->pv_pa); 312 if (concat_pvaddr(pv0, pv)) { 313 VPRINTF("%s: %s pv %p (%#lx..%#lx) to %#lx..%#lx\n", 314 __func__, "appending", pv, 315 pv->pv_pa, pv->pv_pa + pv->pv_size - 1, 316 pv0->pv_pa, pv0->pv_pa + pv0->pv_size - pv->pv_size - 1); 317 pv = SLIST_NEXT(pv0, pv_list); 318 if (pv != NULL && concat_pvaddr(pv0, pv)) { 319 VPRINTF("%s: %s pv %p (%#lx..%#lx) to %#lx..%#lx\n", 320 __func__, "merging", pv, 321 pv->pv_pa, pv->pv_pa + pv->pv_size - 1, 322 pv0->pv_pa, 323 pv0->pv_pa + pv0->pv_size - pv->pv_size - 1); 324 SLIST_REMOVE_AFTER(pv0, pv_list); 325 SLIST_INSERT_HEAD(&bmi->bmi_freechunks, pv, pv_list); 326 } 327 return; 328 } 329 KASSERT(pv->pv_va != (*pvp)->pv_va); 330 pvp = &SLIST_NEXT(*pvp, pv_list); 331 } 332 KASSERT((*pvp) == NULL || pv->pv_va < (*pvp)->pv_va); 333 pv_addr_t * const new_pv = SLIST_FIRST(&bmi->bmi_freechunks); 334 KASSERT(new_pv != NULL); 335 SLIST_REMOVE_HEAD(&bmi->bmi_freechunks, pv_list); 336 *new_pv = *pv; 337 SLIST_NEXT(new_pv, pv_list) = *pvp; 338 (*pvp) = new_pv; 339 340 VPRINTF("%s: adding pv %p (pa %#lx, va %#lx, %lu pages) ", 341 __func__, new_pv, new_pv->pv_pa, new_pv->pv_va, 342 new_pv->pv_size / PAGE_SIZE); 343 if (SLIST_NEXT(new_pv, pv_list)) { 344 VPRINTF("before pa %#lx\n", SLIST_NEXT(new_pv, pv_list)->pv_pa); 345 } else { 346 VPRINTF("at tail\n"); 347 } 348 } 349 350 static void 351 valloc_pages(struct bootmem_info *bmi, pv_addr_t *pv, size_t npages, 352 int prot, int cache, bool zero_p) 353 { 354 size_t nbytes = npages * PAGE_SIZE; 355 pv_addr_t *free_pv = bmi->bmi_freeblocks; 356 size_t free_idx = 0; 357 static bool l1pt_found; 358 359 KASSERT(npages > 0); 360 361 /* 362 * If we haven't allocated the kernel L1 page table and we are aligned 363 * at a L1 table boundary, alloc the memory for it. 364 */ 365 if (!l1pt_found 366 && (free_pv->pv_pa & (L1_TABLE_SIZE - 1)) == 0 367 && free_pv->pv_size >= L1_TABLE_SIZE) { 368 l1pt_found = true; 369 VPRINTF(" l1pt"); 370 371 valloc_pages(bmi, &kernel_l1pt, L1_TABLE_SIZE / PAGE_SIZE, 372 VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE, true); 373 add_pages(bmi, &kernel_l1pt); 374 } 375 376 while (nbytes > free_pv->pv_size) { 377 free_pv++; 378 free_idx++; 379 if (free_idx == bmi->bmi_nfreeblocks) { 380 panic("%s: could not allocate %zu bytes", 381 __func__, nbytes); 382 } 383 } 384 385 /* 386 * As we allocate the memory, make sure that we don't walk over 387 * our current first level translation table. 388 */ 389 KASSERT((armreg_ttbr_read() & ~(L1_TABLE_SIZE - 1)) != free_pv->pv_pa); 390 391 #if defined(FDT) 392 fdt_add_reserved_memory_range(free_pv->pv_pa, nbytes); 393 #endif 394 pv->pv_pa = free_pv->pv_pa; 395 pv->pv_va = free_pv->pv_va; 396 pv->pv_size = nbytes; 397 pv->pv_prot = prot; 398 pv->pv_cache = cache; 399 400 /* 401 * If PTE_PAGETABLE uses the same cache modes as PTE_CACHE 402 * just use PTE_CACHE. 403 */ 404 if (cache == PTE_PAGETABLE 405 && pte_l1_s_cache_mode == pte_l1_s_cache_mode_pt 406 && pte_l2_l_cache_mode == pte_l2_l_cache_mode_pt 407 && pte_l2_s_cache_mode == pte_l2_s_cache_mode_pt) 408 pv->pv_cache = PTE_CACHE; 409 410 free_pv->pv_pa += nbytes; 411 free_pv->pv_va += nbytes; 412 free_pv->pv_size -= nbytes; 413 if (free_pv->pv_size == 0) { 414 --bmi->bmi_nfreeblocks; 415 for (; free_idx < bmi->bmi_nfreeblocks; free_idx++) { 416 free_pv[0] = free_pv[1]; 417 } 418 } 419 420 bmi->bmi_freepages -= npages; 421 422 if (zero_p) 423 memset((void *)pv->pv_va, 0, nbytes); 424 } 425 426 // Define a TTB value that can never be used. 427 uint32_t cpu_ttb = ~0; 428 429 void 430 arm32_kernel_vm_init(vaddr_t kernel_vm_base, vaddr_t vectors, vaddr_t iovbase, 431 const struct pmap_devmap *devmap, bool mapallmem_p) 432 { 433 struct bootmem_info * const bmi = &bootmem_info; 434 #ifdef MULTIPROCESSOR 435 const size_t cpu_num = arm_cpu_max; 436 #else 437 const size_t cpu_num = 1; 438 #endif 439 440 #ifdef ARM_HAS_VBAR 441 const bool map_vectors_p = false; 442 #elif defined(CPU_ARMV7) || defined(CPU_ARM11) 443 const bool map_vectors_p = vectors == ARM_VECTORS_HIGH 444 || (armreg_pfr1_read() & ARM_PFR1_SEC_MASK) == 0; 445 #else 446 const bool map_vectors_p = true; 447 #endif 448 449 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS 450 KASSERT(mapallmem_p); 451 #ifdef ARM_MMU_EXTENDED 452 /* 453 * The direct map VA space ends at the start of the kernel VM space. 454 */ 455 pmap_directlimit = kernel_vm_base; 456 #else 457 KASSERT(kernel_vm_base - KERNEL_BASE >= physical_end - physical_start); 458 #endif /* ARM_MMU_EXTENDED */ 459 #endif /* __HAVE_MM_MD_DIRECT_MAPPED_PHYS */ 460 461 /* 462 * Calculate the number of L2 pages needed for mapping the 463 * kernel + data + stuff. Assume 2 L2 pages for kernel, 1 for vectors, 464 * and 1 for IO 465 */ 466 size_t kernel_size = bmi->bmi_kernelend; 467 kernel_size -= (bmi->bmi_kernelstart & -L2_S_SEGSIZE); 468 kernel_size += L1_TABLE_SIZE_REAL; 469 kernel_size += PAGE_SIZE * KERNEL_L2PT_VMDATA_NUM; 470 if (map_vectors_p) { 471 kernel_size += PAGE_SIZE; /* L2PT for VECTORS */ 472 } 473 if (iovbase) { 474 kernel_size += PAGE_SIZE; /* L2PT for IO */ 475 } 476 kernel_size += 477 cpu_num * (ABT_STACK_SIZE + FIQ_STACK_SIZE + IRQ_STACK_SIZE 478 + UND_STACK_SIZE + UPAGES) * PAGE_SIZE; 479 kernel_size += round_page(MSGBUFSIZE); 480 kernel_size += 0x10000; /* slop */ 481 if (!mapallmem_p) { 482 kernel_size += PAGE_SIZE 483 * ((kernel_size + L2_S_SEGSIZE - 1) / L2_S_SEGSIZE); 484 } 485 kernel_size = round_page(kernel_size); 486 487 /* 488 * Now we know how many L2 pages it will take. 489 */ 490 const size_t KERNEL_L2PT_KERNEL_NUM = 491 round_page(kernel_size + L2_S_SEGSIZE - 1) / L2_S_SEGSIZE; 492 493 VPRINTF("%s: %zu L2 pages are needed to map %#zx kernel bytes\n", 494 __func__, KERNEL_L2PT_KERNEL_NUM, kernel_size); 495 496 KASSERT(KERNEL_L2PT_KERNEL_NUM + KERNEL_L2PT_VMDATA_NUM < __arraycount(bmi->bmi_l2pts)); 497 pv_addr_t * const kernel_l2pt = bmi->bmi_l2pts; 498 pv_addr_t * const vmdata_l2pt = kernel_l2pt + KERNEL_L2PT_KERNEL_NUM; 499 pv_addr_t msgbuf; 500 pv_addr_t text; 501 pv_addr_t data; 502 pv_addr_t chunks[KERNEL_L2PT_KERNEL_NUM+KERNEL_L2PT_VMDATA_NUM+11]; 503 #if ARM_MMU_XSCALE == 1 504 pv_addr_t minidataclean; 505 #endif 506 507 /* 508 * We need to allocate some fixed page tables to get the kernel going. 509 * 510 * We are going to allocate our bootstrap pages from the beginning of 511 * the free space that we just calculated. We allocate one page 512 * directory and a number of page tables and store the physical 513 * addresses in the bmi_l2pts array in bootmem_info. 514 * 515 * The kernel page directory must be on a 16K boundary. The page 516 * tables must be on 4K boundaries. What we do is allocate the 517 * page directory on the first 16K boundary that we encounter, and 518 * the page tables on 4K boundaries otherwise. Since we allocate 519 * at least 3 L2 page tables, we are guaranteed to encounter at 520 * least one 16K aligned region. 521 */ 522 523 VPRINTF("%s: allocating page tables for", __func__); 524 for (size_t i = 0; i < __arraycount(chunks); i++) { 525 SLIST_INSERT_HEAD(&bmi->bmi_freechunks, &chunks[i], pv_list); 526 } 527 528 kernel_l1pt.pv_pa = 0; 529 kernel_l1pt.pv_va = 0; 530 531 /* 532 * Allocate the L2 pages, but if we get to a page that is aligned for 533 * an L1 page table, we will allocate the pages for it first and then 534 * allocate the L2 page. 535 */ 536 537 if (map_vectors_p) { 538 /* 539 * First allocate L2 page for the vectors. 540 */ 541 VPRINTF(" vector"); 542 valloc_pages(bmi, &bmi->bmi_vector_l2pt, 1, 543 VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE, true); 544 add_pages(bmi, &bmi->bmi_vector_l2pt); 545 } 546 547 /* 548 * Now allocate L2 pages for the kernel 549 */ 550 VPRINTF(" kernel"); 551 for (size_t idx = 0; idx < KERNEL_L2PT_KERNEL_NUM; ++idx) { 552 valloc_pages(bmi, &kernel_l2pt[idx], 1, 553 VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE, true); 554 add_pages(bmi, &kernel_l2pt[idx]); 555 } 556 557 /* 558 * Now allocate L2 pages for the initial kernel VA space. 559 */ 560 VPRINTF(" vm"); 561 for (size_t idx = 0; idx < KERNEL_L2PT_VMDATA_NUM; ++idx) { 562 valloc_pages(bmi, &vmdata_l2pt[idx], 1, 563 VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE, true); 564 add_pages(bmi, &vmdata_l2pt[idx]); 565 } 566 567 /* 568 * If someone wanted a L2 page for I/O, allocate it now. 569 */ 570 if (iovbase) { 571 VPRINTF(" io"); 572 valloc_pages(bmi, &bmi->bmi_io_l2pt, 1, 573 VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE, true); 574 add_pages(bmi, &bmi->bmi_io_l2pt); 575 } 576 577 VPRINTF("%s: allocating stacks\n", __func__); 578 579 /* Allocate stacks for all modes and CPUs */ 580 valloc_pages(bmi, &abtstack, ABT_STACK_SIZE * cpu_num, 581 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true); 582 add_pages(bmi, &abtstack); 583 valloc_pages(bmi, &fiqstack, FIQ_STACK_SIZE * cpu_num, 584 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true); 585 add_pages(bmi, &fiqstack); 586 valloc_pages(bmi, &irqstack, IRQ_STACK_SIZE * cpu_num, 587 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true); 588 add_pages(bmi, &irqstack); 589 valloc_pages(bmi, &undstack, UND_STACK_SIZE * cpu_num, 590 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true); 591 add_pages(bmi, &undstack); 592 valloc_pages(bmi, &idlestack, UPAGES * cpu_num, /* SVC32 */ 593 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true); 594 add_pages(bmi, &idlestack); 595 valloc_pages(bmi, &kernelstack, UPAGES, /* SVC32 */ 596 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true); 597 add_pages(bmi, &kernelstack); 598 599 /* Allocate the message buffer from the end of memory. */ 600 const size_t msgbuf_pgs = round_page(MSGBUFSIZE) / PAGE_SIZE; 601 valloc_pages(bmi, &msgbuf, msgbuf_pgs, 602 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, false); 603 add_pages(bmi, &msgbuf); 604 msgbufphys = msgbuf.pv_pa; 605 msgbufaddr = (void *)msgbuf.pv_va; 606 607 if (map_vectors_p) { 608 /* 609 * Allocate a page for the system vector page. 610 * This page will just contain the system vectors and can be 611 * shared by all processes. 612 */ 613 VPRINTF(" vector"); 614 615 valloc_pages(bmi, &systempage, 1, VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE, 616 PTE_CACHE, true); 617 } 618 systempage.pv_va = vectors; 619 620 /* 621 * If the caller needed a few extra pages for some reason, allocate 622 * them now. 623 */ 624 #if ARM_MMU_XSCALE == 1 625 #if (ARM_NMMUS > 1) 626 if (xscale_use_minidata) 627 #endif 628 valloc_pages(bmi, &minidataclean, 1, 629 VM_PROT_READ|VM_PROT_WRITE, 0, true); 630 #endif 631 632 /* 633 * Ok we have allocated physical pages for the primary kernel 634 * page tables and stacks. Let's just confirm that. 635 */ 636 if (kernel_l1pt.pv_va == 0 637 && (!kernel_l1pt.pv_pa || (kernel_l1pt.pv_pa & (L1_TABLE_SIZE - 1)) != 0)) 638 panic("%s: Failed to allocate or align the kernel " 639 "page directory", __func__); 640 641 VPRINTF("Creating L1 page table at 0x%08lx/0x%08lx\n", 642 kernel_l1pt.pv_va, kernel_l1pt.pv_pa); 643 644 /* 645 * Now we start construction of the L1 page table 646 * We start by mapping the L2 page tables into the L1. 647 * This means that we can replace L1 mappings later on if necessary 648 */ 649 vaddr_t l1pt_va = kernel_l1pt.pv_va; 650 paddr_t l1pt_pa = kernel_l1pt.pv_pa; 651 652 if (map_vectors_p) { 653 /* Map the L2 pages tables in the L1 page table */ 654 pmap_link_l2pt(l1pt_va, systempage.pv_va & -L2_S_SEGSIZE, 655 &bmi->bmi_vector_l2pt); 656 VPRINTF("%s: adding L2 pt (VA %#lx, PA %#lx) " 657 "for VA %#lx\n (vectors)", 658 __func__, bmi->bmi_vector_l2pt.pv_va, 659 bmi->bmi_vector_l2pt.pv_pa, systempage.pv_va); 660 } 661 662 /* 663 * This enforces a alignment requirement of L2_S_SEGSIZE for kernel 664 * start PA 665 */ 666 const vaddr_t kernel_base = 667 KERN_PHYSTOV(bmi->bmi_kernelstart & -L2_S_SEGSIZE); 668 669 VPRINTF("%s: kernel_base %lx KERNEL_L2PT_KERNEL_NUM %zu\n", __func__, 670 kernel_base, KERNEL_L2PT_KERNEL_NUM); 671 672 for (size_t idx = 0; idx < KERNEL_L2PT_KERNEL_NUM; idx++) { 673 pmap_link_l2pt(l1pt_va, kernel_base + idx * L2_S_SEGSIZE, 674 &kernel_l2pt[idx]); 675 VPRINTF("%s: adding L2 pt (VA %#lx, PA %#lx) for VA %#lx (kernel)\n", 676 __func__, kernel_l2pt[idx].pv_va, 677 kernel_l2pt[idx].pv_pa, kernel_base + idx * L2_S_SEGSIZE); 678 } 679 680 VPRINTF("%s: kernel_vm_base %lx KERNEL_L2PT_KERNEL_NUM %zu\n", __func__, 681 kernel_vm_base, KERNEL_L2PT_KERNEL_NUM); 682 683 for (size_t idx = 0; idx < KERNEL_L2PT_VMDATA_NUM; idx++) { 684 pmap_link_l2pt(l1pt_va, kernel_vm_base + idx * L2_S_SEGSIZE, 685 &vmdata_l2pt[idx]); 686 VPRINTF("%s: adding L2 pt (VA %#lx, PA %#lx) for VA %#lx (vm)\n", 687 __func__, vmdata_l2pt[idx].pv_va, vmdata_l2pt[idx].pv_pa, 688 kernel_vm_base + idx * L2_S_SEGSIZE); 689 } 690 if (iovbase) { 691 pmap_link_l2pt(l1pt_va, iovbase & -L2_S_SEGSIZE, &bmi->bmi_io_l2pt); 692 VPRINTF("%s: adding L2 pt (VA %#lx, PA %#lx) for VA %#lx (io)\n", 693 __func__, bmi->bmi_io_l2pt.pv_va, bmi->bmi_io_l2pt.pv_pa, 694 iovbase & -L2_S_SEGSIZE); 695 } 696 697 /* update the top of the kernel VM */ 698 pmap_curmaxkvaddr = 699 kernel_vm_base + (KERNEL_L2PT_VMDATA_NUM * L2_S_SEGSIZE); 700 701 // This could be done earlier and then the kernel data and pages 702 // allocated above would get merged (concatentated) 703 704 VPRINTF("Mapping kernel\n"); 705 706 extern char etext[]; 707 size_t totalsize = bmi->bmi_kernelend - bmi->bmi_kernelstart; 708 size_t textsize = KERN_VTOPHYS((uintptr_t)etext) - bmi->bmi_kernelstart; 709 710 textsize = (textsize + PGOFSET) & ~PGOFSET; 711 712 /* start at offset of kernel in RAM */ 713 714 text.pv_pa = bmi->bmi_kernelstart; 715 text.pv_va = KERN_PHYSTOV(bmi->bmi_kernelstart); 716 text.pv_size = textsize; 717 text.pv_prot = VM_PROT_READ | VM_PROT_EXECUTE; 718 text.pv_cache = PTE_CACHE; 719 720 VPRINTF("%s: adding chunk for kernel text %#lx..%#lx (VA %#lx)\n", 721 __func__, text.pv_pa, text.pv_pa + text.pv_size - 1, text.pv_va); 722 723 add_pages(bmi, &text); 724 725 data.pv_pa = text.pv_pa + textsize; 726 data.pv_va = text.pv_va + textsize; 727 data.pv_size = totalsize - textsize; 728 data.pv_prot = VM_PROT_READ|VM_PROT_WRITE; 729 data.pv_cache = PTE_CACHE; 730 731 VPRINTF("%s: adding chunk for kernel data/bss %#lx..%#lx (VA %#lx)\n", 732 __func__, data.pv_pa, data.pv_pa + data.pv_size - 1, data.pv_va); 733 734 add_pages(bmi, &data); 735 736 VPRINTF("Listing Chunks\n"); 737 738 pv_addr_t *lpv; 739 SLIST_FOREACH(lpv, &bmi->bmi_chunks, pv_list) { 740 VPRINTF("%s: pv %p: chunk VA %#lx..%#lx " 741 "(PA %#lx, prot %d, cache %d)\n", 742 __func__, lpv, lpv->pv_va, lpv->pv_va + lpv->pv_size - 1, 743 lpv->pv_pa, lpv->pv_prot, lpv->pv_cache); 744 } 745 VPRINTF("\nMapping Chunks\n"); 746 747 pv_addr_t cur_pv; 748 pv_addr_t *pv = SLIST_FIRST(&bmi->bmi_chunks); 749 if (!mapallmem_p || pv->pv_pa == bmi->bmi_start) { 750 cur_pv = *pv; 751 KASSERTMSG(cur_pv.pv_va >= KERNEL_BASE, "%#lx", cur_pv.pv_va); 752 pv = SLIST_NEXT(pv, pv_list); 753 } else { 754 cur_pv.pv_va = KERNEL_BASE; 755 cur_pv.pv_pa = KERN_VTOPHYS(cur_pv.pv_va); 756 cur_pv.pv_size = pv->pv_pa - cur_pv.pv_pa; 757 cur_pv.pv_prot = VM_PROT_READ | VM_PROT_WRITE; 758 cur_pv.pv_cache = PTE_CACHE; 759 } 760 while (pv != NULL) { 761 if (mapallmem_p) { 762 if (concat_pvaddr(&cur_pv, pv)) { 763 pv = SLIST_NEXT(pv, pv_list); 764 continue; 765 } 766 if (cur_pv.pv_pa + cur_pv.pv_size < pv->pv_pa) { 767 /* 768 * See if we can extend the current pv to emcompass the 769 * hole, and if so do it and retry the concatenation. 770 */ 771 if (cur_pv.pv_prot == (VM_PROT_READ|VM_PROT_WRITE) 772 && cur_pv.pv_cache == PTE_CACHE) { 773 cur_pv.pv_size = pv->pv_pa - cur_pv.pv_va; 774 continue; 775 } 776 777 /* 778 * We couldn't so emit the current chunk and then 779 */ 780 VPRINTF("%s: mapping chunk VA %#lx..%#lx " 781 "(PA %#lx, prot %d, cache %d)\n", 782 __func__, 783 cur_pv.pv_va, cur_pv.pv_va + cur_pv.pv_size - 1, 784 cur_pv.pv_pa, cur_pv.pv_prot, cur_pv.pv_cache); 785 pmap_map_chunk(l1pt_va, cur_pv.pv_va, cur_pv.pv_pa, 786 cur_pv.pv_size, cur_pv.pv_prot, cur_pv.pv_cache); 787 788 /* 789 * set the current chunk to the hole and try again. 790 */ 791 cur_pv.pv_pa += cur_pv.pv_size; 792 cur_pv.pv_va += cur_pv.pv_size; 793 cur_pv.pv_size = pv->pv_pa - cur_pv.pv_va; 794 cur_pv.pv_prot = VM_PROT_READ | VM_PROT_WRITE; 795 cur_pv.pv_cache = PTE_CACHE; 796 continue; 797 } 798 } 799 800 /* 801 * The new pv didn't concatenate so emit the current one 802 * and use the new pv as the current pv. 803 */ 804 VPRINTF("%s: mapping chunk VA %#lx..%#lx " 805 "(PA %#lx, prot %d, cache %d)\n", 806 __func__, cur_pv.pv_va, cur_pv.pv_va + cur_pv.pv_size - 1, 807 cur_pv.pv_pa, cur_pv.pv_prot, cur_pv.pv_cache); 808 pmap_map_chunk(l1pt_va, cur_pv.pv_va, cur_pv.pv_pa, 809 cur_pv.pv_size, cur_pv.pv_prot, cur_pv.pv_cache); 810 cur_pv = *pv; 811 pv = SLIST_NEXT(pv, pv_list); 812 } 813 814 /* 815 * If we are mapping all of memory, let's map the rest of memory. 816 */ 817 if (mapallmem_p && cur_pv.pv_pa + cur_pv.pv_size < bmi->bmi_end) { 818 if (cur_pv.pv_prot == (VM_PROT_READ | VM_PROT_WRITE) 819 && cur_pv.pv_cache == PTE_CACHE) { 820 cur_pv.pv_size = bmi->bmi_end - cur_pv.pv_pa; 821 } else { 822 KASSERTMSG(cur_pv.pv_va + cur_pv.pv_size <= kernel_vm_base, 823 "%#lx >= %#lx", cur_pv.pv_va + cur_pv.pv_size, 824 kernel_vm_base); 825 VPRINTF("%s: mapping chunk VA %#lx..%#lx " 826 "(PA %#lx, prot %d, cache %d)\n", 827 __func__, cur_pv.pv_va, cur_pv.pv_va + cur_pv.pv_size - 1, 828 cur_pv.pv_pa, cur_pv.pv_prot, cur_pv.pv_cache); 829 pmap_map_chunk(l1pt_va, cur_pv.pv_va, cur_pv.pv_pa, 830 cur_pv.pv_size, cur_pv.pv_prot, cur_pv.pv_cache); 831 cur_pv.pv_pa += cur_pv.pv_size; 832 cur_pv.pv_va += cur_pv.pv_size; 833 cur_pv.pv_size = bmi->bmi_end - cur_pv.pv_pa; 834 cur_pv.pv_prot = VM_PROT_READ | VM_PROT_WRITE; 835 cur_pv.pv_cache = PTE_CACHE; 836 } 837 } 838 839 // The amount we can direct is limited by the start of the 840 // virtual part of the kernel address space. Don't overrun 841 // into it. 842 if (mapallmem_p && cur_pv.pv_va + cur_pv.pv_size > kernel_vm_base) { 843 cur_pv.pv_size = kernel_vm_base - cur_pv.pv_va; 844 } 845 846 /* 847 * Now we map the final chunk. 848 */ 849 VPRINTF("%s: mapping last chunk VA %#lx..%#lx (PA %#lx, prot %d, cache %d)\n", 850 __func__, cur_pv.pv_va, cur_pv.pv_va + cur_pv.pv_size - 1, 851 cur_pv.pv_pa, cur_pv.pv_prot, cur_pv.pv_cache); 852 pmap_map_chunk(l1pt_va, cur_pv.pv_va, cur_pv.pv_pa, 853 cur_pv.pv_size, cur_pv.pv_prot, cur_pv.pv_cache); 854 855 /* 856 * Now we map the stuff that isn't directly after the kernel 857 */ 858 if (map_vectors_p) { 859 /* Map the vector page. */ 860 pmap_map_entry(l1pt_va, systempage.pv_va, systempage.pv_pa, 861 VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE, PTE_CACHE); 862 } 863 864 /* Map the Mini-Data cache clean area. */ 865 #if ARM_MMU_XSCALE == 1 866 #if (ARM_NMMUS > 1) 867 if (xscale_use_minidata) 868 #endif 869 xscale_setup_minidata(l1pt_va, minidataclean.pv_va, 870 minidataclean.pv_pa); 871 #endif 872 873 /* 874 * Map integrated peripherals at same address in first level page 875 * table so that we can continue to use console. 876 */ 877 if (devmap) 878 pmap_devmap_bootstrap(l1pt_va, devmap); 879 880 /* Tell the user about where all the bits and pieces live. */ 881 VPRINTF("%22s Physical Virtual Num\n", " "); 882 VPRINTF("%22s Starting Ending Starting Ending Pages\n", " "); 883 884 #ifdef VERBOSE_INIT_ARM 885 static const char mem_fmt[] = 886 "%20s: 0x%08lx 0x%08lx 0x%08lx 0x%08lx %u\n"; 887 static const char mem_fmt_nov[] = 888 "%20s: 0x%08lx 0x%08lx %zu\n"; 889 #endif 890 891 #if 0 892 // XXX Doesn't make sense if kernel not at bottom of RAM 893 VPRINTF(mem_fmt, "SDRAM", bmi->bmi_start, bmi->bmi_end - 1, 894 KERN_PHYSTOV(bmi->bmi_start), KERN_PHYSTOV(bmi->bmi_end - 1), 895 (int)physmem); 896 #endif 897 VPRINTF(mem_fmt, "text section", 898 text.pv_pa, text.pv_pa + text.pv_size - 1, 899 text.pv_va, text.pv_va + text.pv_size - 1, 900 (int)(text.pv_size / PAGE_SIZE)); 901 VPRINTF(mem_fmt, "data section", 902 KERN_VTOPHYS((vaddr_t)__data_start), KERN_VTOPHYS((vaddr_t)_edata), 903 (vaddr_t)__data_start, (vaddr_t)_edata, 904 (int)((round_page((vaddr_t)_edata) 905 - trunc_page((vaddr_t)__data_start)) / PAGE_SIZE)); 906 VPRINTF(mem_fmt, "bss section", 907 KERN_VTOPHYS((vaddr_t)__bss_start), KERN_VTOPHYS((vaddr_t)__bss_end__), 908 (vaddr_t)__bss_start, (vaddr_t)__bss_end__, 909 (int)((round_page((vaddr_t)__bss_end__) 910 - trunc_page((vaddr_t)__bss_start)) / PAGE_SIZE)); 911 VPRINTF(mem_fmt, "L1 page directory", 912 kernel_l1pt.pv_pa, kernel_l1pt.pv_pa + L1_TABLE_SIZE - 1, 913 kernel_l1pt.pv_va, kernel_l1pt.pv_va + L1_TABLE_SIZE - 1, 914 L1_TABLE_SIZE / PAGE_SIZE); 915 VPRINTF(mem_fmt, "ABT stack (CPU 0)", 916 abtstack.pv_pa, abtstack.pv_pa + (ABT_STACK_SIZE * PAGE_SIZE) - 1, 917 abtstack.pv_va, abtstack.pv_va + (ABT_STACK_SIZE * PAGE_SIZE) - 1, 918 ABT_STACK_SIZE); 919 VPRINTF(mem_fmt, "FIQ stack (CPU 0)", 920 fiqstack.pv_pa, fiqstack.pv_pa + (FIQ_STACK_SIZE * PAGE_SIZE) - 1, 921 fiqstack.pv_va, fiqstack.pv_va + (FIQ_STACK_SIZE * PAGE_SIZE) - 1, 922 FIQ_STACK_SIZE); 923 VPRINTF(mem_fmt, "IRQ stack (CPU 0)", 924 irqstack.pv_pa, irqstack.pv_pa + (IRQ_STACK_SIZE * PAGE_SIZE) - 1, 925 irqstack.pv_va, irqstack.pv_va + (IRQ_STACK_SIZE * PAGE_SIZE) - 1, 926 IRQ_STACK_SIZE); 927 VPRINTF(mem_fmt, "UND stack (CPU 0)", 928 undstack.pv_pa, undstack.pv_pa + (UND_STACK_SIZE * PAGE_SIZE) - 1, 929 undstack.pv_va, undstack.pv_va + (UND_STACK_SIZE * PAGE_SIZE) - 1, 930 UND_STACK_SIZE); 931 VPRINTF(mem_fmt, "IDLE stack (CPU 0)", 932 idlestack.pv_pa, idlestack.pv_pa + (UPAGES * PAGE_SIZE) - 1, 933 idlestack.pv_va, idlestack.pv_va + (UPAGES * PAGE_SIZE) - 1, 934 UPAGES); 935 VPRINTF(mem_fmt, "SVC stack", 936 kernelstack.pv_pa, kernelstack.pv_pa + (UPAGES * PAGE_SIZE) - 1, 937 kernelstack.pv_va, kernelstack.pv_va + (UPAGES * PAGE_SIZE) - 1, 938 UPAGES); 939 VPRINTF(mem_fmt, "Message Buffer", 940 msgbuf.pv_pa, msgbuf.pv_pa + (msgbuf_pgs * PAGE_SIZE) - 1, 941 msgbuf.pv_va, msgbuf.pv_va + (msgbuf_pgs * PAGE_SIZE) - 1, 942 (int)msgbuf_pgs); 943 if (map_vectors_p) { 944 VPRINTF(mem_fmt, "Exception Vectors", 945 systempage.pv_pa, systempage.pv_pa + PAGE_SIZE - 1, 946 systempage.pv_va, systempage.pv_va + PAGE_SIZE - 1, 947 1); 948 } 949 for (size_t i = 0; i < bmi->bmi_nfreeblocks; i++) { 950 pv = &bmi->bmi_freeblocks[i]; 951 952 VPRINTF(mem_fmt_nov, "Free Memory", 953 pv->pv_pa, pv->pv_pa + pv->pv_size - 1, 954 pv->pv_size / PAGE_SIZE); 955 } 956 /* 957 * Now we have the real page tables in place so we can switch to them. 958 * Once this is done we will be running with the REAL kernel page 959 * tables. 960 */ 961 962 VPRINTF("TTBR0=%#x", armreg_ttbr_read()); 963 #ifdef _ARM_ARCH_6 964 VPRINTF(" TTBR1=%#x TTBCR=%#x CONTEXTIDR=%#x", 965 armreg_ttbr1_read(), armreg_ttbcr_read(), 966 armreg_contextidr_read()); 967 #endif 968 VPRINTF("\n"); 969 970 /* Switch tables */ 971 VPRINTF("switching to new L1 page table @%#lx...\n", l1pt_pa); 972 973 extern uint32_t cpu_ttb; 974 cpu_ttb = l1pt_pa; 975 976 cpu_domains(DOMAIN_DEFAULT); 977 978 cpu_idcache_wbinv_all(); 979 980 #ifdef __HAVE_GENERIC_START 981 982 /* 983 * Turn on caches and set SCTLR/ACTLR 984 */ 985 cpu_setup(boot_args); 986 #endif 987 988 VPRINTF(" ttb"); 989 990 #ifdef ARM_MMU_EXTENDED 991 /* 992 * TTBCR should have been initialized by the MD start code. 993 */ 994 KASSERT((armreg_contextidr_read() & 0xff) == 0); 995 KASSERT(armreg_ttbcr_read() == __SHIFTIN(1, TTBCR_S_N)); 996 /* 997 * Disable lookups via TTBR0 until there is an activated pmap. 998 */ 999 armreg_ttbcr_write(armreg_ttbcr_read() | TTBCR_S_PD0); 1000 cpu_setttb(l1pt_pa, KERNEL_PID); 1001 arm_isb(); 1002 #else 1003 cpu_setttb(l1pt_pa, true); 1004 #endif 1005 1006 cpu_tlb_flushID(); 1007 1008 #ifdef ARM_MMU_EXTENDED 1009 VPRINTF("\nsctlr=%#x actlr=%#x\n", 1010 armreg_sctlr_read(), armreg_auxctl_read()); 1011 #else 1012 VPRINTF(" (TTBR0=%#x)", armreg_ttbr_read()); 1013 #endif 1014 1015 #ifdef MULTIPROCESSOR 1016 #ifndef __HAVE_GENERIC_START 1017 /* 1018 * Kick the secondaries to load the TTB. After which they'll go 1019 * back to sleep to wait for the final kick so they will hatch. 1020 */ 1021 VPRINTF(" hatchlings"); 1022 cpu_boot_secondary_processors(); 1023 #endif 1024 #endif 1025 1026 VPRINTF(" OK\n"); 1027 } 1028
Indexes created Mon Sep 29 21:09:56 GMT 2025