arm32_kvminit.c revision 1.42
1 /* $NetBSD: arm32_kvminit.c,v 1.42 2018/07/31 07:00:48 skrll Exp $ */ 2 3 /* 4 * Copyright (c) 2002, 2003, 2005 Genetec Corporation. All rights reserved. 5 * Written by Hiroyuki Bessho for Genetec Corporation. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. The name of Genetec Corporation may not be used to endorse or 16 * promote products derived from this software without specific prior 17 * written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY GENETEC CORPORATION ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENETEC CORPORATION 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 * 31 * Copyright (c) 2001 Wasabi Systems, Inc. 32 * All rights reserved. 33 * 34 * Written by Jason R. Thorpe for Wasabi Systems, Inc. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice, this list of conditions and the following disclaimer. 41 * 2. Redistributions in binary form must reproduce the above copyright 42 * notice, this list of conditions and the following disclaimer in the 43 * documentation and/or other materials provided with the distribution. 44 * 3. All advertising materials mentioning features or use of this software 45 * must display the following acknowledgement: 46 * This product includes software developed for the NetBSD Project by 47 * Wasabi Systems, Inc. 48 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 49 * or promote products derived from this software without specific prior 50 * written permission. 51 * 52 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 53 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 54 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 55 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 56 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 57 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 58 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 59 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 60 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 61 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 62 * POSSIBILITY OF SUCH DAMAGE. 63 * 64 * Copyright (c) 1997,1998 Mark Brinicombe. 65 * Copyright (c) 1997,1998 Causality Limited. 66 * All rights reserved. 67 * 68 * Redistribution and use in source and binary forms, with or without 69 * modification, are permitted provided that the following conditions 70 * are met: 71 * 1. Redistributions of source code must retain the above copyright 72 * notice, this list of conditions and the following disclaimer. 73 * 2. Redistributions in binary form must reproduce the above copyright 74 * notice, this list of conditions and the following disclaimer in the 75 * documentation and/or other materials provided with the distribution. 76 * 3. All advertising materials mentioning features or use of this software 77 * must display the following acknowledgement: 78 * This product includes software developed by Mark Brinicombe 79 * for the NetBSD Project. 80 * 4. The name of the company nor the name of the author may be used to 81 * endorse or promote products derived from this software without specific 82 * prior written permission. 83 * 84 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 85 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 86 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 87 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 88 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 89 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 90 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 91 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 92 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 93 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 94 * SUCH DAMAGE. 95 * 96 * Copyright (c) 2007 Microsoft 97 * All rights reserved. 98 * 99 * Redistribution and use in source and binary forms, with or without 100 * modification, are permitted provided that the following conditions 101 * are met: 102 * 1. Redistributions of source code must retain the above copyright 103 * notice, this list of conditions and the following disclaimer. 104 * 2. Redistributions in binary form must reproduce the above copyright 105 * notice, this list of conditions and the following disclaimer in the 106 * documentation and/or other materials provided with the distribution. 107 * 3. All advertising materials mentioning features or use of this software 108 * must display the following acknowledgement: 109 * This product includes software developed by Microsoft 110 * 111 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 112 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 113 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 114 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTERS BE LIABLE FOR ANY DIRECT, 115 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 116 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 117 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 118 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 119 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 120 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 121 * SUCH DAMAGE. 122 */ 123 124 #include "opt_arm_debug.h" 125 #include "opt_fdt.h" 126 #include "opt_multiprocessor.h" 127 128 #include <sys/cdefs.h> 129 __KERNEL_RCSID(0, "$NetBSD: arm32_kvminit.c,v 1.42 2018/07/31 07:00:48 skrll Exp $"); 130 131 #include <sys/param.h> 132 #include <sys/device.h> 133 #include <sys/kernel.h> 134 #include <sys/reboot.h> 135 #include <sys/bus.h> 136 137 #include <dev/cons.h> 138 139 #include <uvm/uvm_extern.h> 140 141 #include <arm/locore.h> 142 #include <arm/db_machdep.h> 143 #include <arm/undefined.h> 144 #include <arm/bootconfig.h> 145 #include <arm/arm32/machdep.h> 146 147 #if defined(FDT) 148 #include <arch/evbarm/fdt/platform.h> 149 #endif 150 151 #ifdef MULTIPROCESSOR 152 #ifndef __HAVE_CPU_UAREA_ALLOC_IDLELWP 153 #error __HAVE_CPU_UAREA_ALLOC_IDLELWP required to not waste pages for idlestack 154 #endif 155 #endif 156 157 #ifdef VERBOSE_INIT_ARM 158 #define VPRINTF(...) printf(__VA_ARGS__) 159 #else 160 #define VPRINTF(...) do { } while (/* CONSTCOND */ 0) 161 #endif 162 163 struct bootmem_info bootmem_info; 164 165 extern void *msgbufaddr; 166 paddr_t msgbufphys; 167 paddr_t physical_start; 168 paddr_t physical_end; 169 170 extern char etext[]; 171 extern char __data_start[], _edata[]; 172 extern char __bss_start[], __bss_end__[]; 173 extern char _end[]; 174 175 /* Page tables for mapping kernel VM */ 176 #define KERNEL_L2PT_VMDATA_NUM 8 /* start with 32MB of KVM */ 177 178 /* 179 * Macros to translate between physical and virtual for a subset of the 180 * kernel address space. *Not* for general use. 181 */ 182 #if defined(KERNEL_BASE_VOFFSET) 183 #define KERN_VTOPHYS(bmi, va) \ 184 ((paddr_t)((vaddr_t)(va) - KERNEL_BASE_VOFFSET)) 185 #define KERN_PHYSTOV(bmi, pa) \ 186 ((vaddr_t)((paddr_t)(pa) + KERNEL_BASE_VOFFSET)) 187 #else 188 #define KERN_VTOPHYS(bmi, va) \ 189 ((paddr_t)((vaddr_t)(va) - KERNEL_BASE + (bmi)->bmi_start)) 190 #define KERN_PHYSTOV(bmi, pa) \ 191 ((vaddr_t)((paddr_t)(pa) - (bmi)->bmi_start + KERNEL_BASE)) 192 #endif 193 194 void 195 arm32_bootmem_init(paddr_t memstart, psize_t memsize, vsize_t kernelstart) 196 { 197 struct bootmem_info * const bmi = &bootmem_info; 198 pv_addr_t *pv = bmi->bmi_freeblocks; 199 200 VPRINTF("%s: memstart=%#lx, memsize=%#lx, kernelstart=%#lx\n", 201 __func__, memstart, memsize, kernelstart); 202 203 physical_start = bmi->bmi_start = memstart; 204 physical_end = bmi->bmi_end = memstart + memsize; 205 #ifndef ARM_HAS_LPAE 206 if (physical_end == 0) { 207 physical_end = -PAGE_SIZE; 208 memsize -= PAGE_SIZE; 209 bmi->bmi_end -= PAGE_SIZE; 210 VPRINTF("%s: memsize shrunk by a page to avoid ending at 4GB\n", 211 __func__); 212 } 213 #endif 214 physmem = memsize / PAGE_SIZE; 215 216 /* 217 * Let's record where the kernel lives. 218 */ 219 bmi->bmi_kernelstart = kernelstart; 220 bmi->bmi_kernelend = KERN_VTOPHYS(bmi, round_page((vaddr_t)_end)); 221 222 #if defined(FDT) 223 fdt_add_reserved_memory_range(bmi->bmi_kernelstart, 224 bmi->bmi_kernelend - bmi->bmi_kernelstart); 225 #endif 226 227 VPRINTF("%s: kernelend=%#lx\n", __func__, bmi->bmi_kernelend); 228 229 /* 230 * Now the rest of the free memory must be after the kernel. 231 */ 232 pv->pv_pa = bmi->bmi_kernelend; 233 pv->pv_va = KERN_PHYSTOV(bmi, pv->pv_pa); 234 pv->pv_size = bmi->bmi_end - bmi->bmi_kernelend; 235 bmi->bmi_freepages += pv->pv_size / PAGE_SIZE; 236 VPRINTF("%s: adding %lu free pages: [%#lx..%#lx] (VA %#lx)\n", 237 __func__, pv->pv_size / PAGE_SIZE, pv->pv_pa, 238 pv->pv_pa + pv->pv_size - 1, pv->pv_va); 239 pv++; 240 241 #if !defined(FDT) 242 /* 243 * Add a free block for any memory before the kernel. 244 */ 245 if (bmi->bmi_start < bmi->bmi_kernelstart) { 246 pv->pv_pa = bmi->bmi_start; 247 pv->pv_va = KERN_PHYSTOV(bmi, pv->pv_pa); 248 pv->pv_size = bmi->bmi_kernelstart - pv->pv_pa; 249 bmi->bmi_freepages += pv->pv_size / PAGE_SIZE; 250 VPRINTF("%s: adding %lu free pages: [%#lx..%#lx] (VA %#lx)\n", 251 __func__, pv->pv_size / PAGE_SIZE, pv->pv_pa, 252 pv->pv_pa + pv->pv_size - 1, pv->pv_va); 253 pv++; 254 } 255 #endif 256 257 bmi->bmi_nfreeblocks = pv - bmi->bmi_freeblocks; 258 259 SLIST_INIT(&bmi->bmi_freechunks); 260 SLIST_INIT(&bmi->bmi_chunks); 261 } 262 263 static bool 264 concat_pvaddr(pv_addr_t *acc_pv, pv_addr_t *pv) 265 { 266 if (acc_pv->pv_pa + acc_pv->pv_size == pv->pv_pa 267 && acc_pv->pv_va + acc_pv->pv_size == pv->pv_va 268 && acc_pv->pv_prot == pv->pv_prot 269 && acc_pv->pv_cache == pv->pv_cache) { 270 VPRINTF("%s: appending pv %p (%#lx..%#lx) to %#lx..%#lx\n", 271 __func__, pv, pv->pv_pa, pv->pv_pa + pv->pv_size + 1, 272 acc_pv->pv_pa, acc_pv->pv_pa + acc_pv->pv_size + 1); 273 acc_pv->pv_size += pv->pv_size; 274 return true; 275 } 276 277 return false; 278 } 279 280 static void 281 add_pages(struct bootmem_info *bmi, pv_addr_t *pv) 282 { 283 pv_addr_t **pvp = &SLIST_FIRST(&bmi->bmi_chunks); 284 while ((*pvp) != NULL && (*pvp)->pv_va <= pv->pv_va) { 285 pv_addr_t * const pv0 = (*pvp); 286 KASSERT(SLIST_NEXT(pv0, pv_list) == NULL || pv0->pv_pa < SLIST_NEXT(pv0, pv_list)->pv_pa); 287 if (concat_pvaddr(pv0, pv)) { 288 VPRINTF("%s: %s pv %p (%#lx..%#lx) to %#lx..%#lx\n", 289 __func__, "appending", pv, 290 pv->pv_pa, pv->pv_pa + pv->pv_size - 1, 291 pv0->pv_pa, pv0->pv_pa + pv0->pv_size - pv->pv_size - 1); 292 pv = SLIST_NEXT(pv0, pv_list); 293 if (pv != NULL && concat_pvaddr(pv0, pv)) { 294 VPRINTF("%s: %s pv %p (%#lx..%#lx) to %#lx..%#lx\n", 295 __func__, "merging", pv, 296 pv->pv_pa, pv->pv_pa + pv->pv_size - 1, 297 pv0->pv_pa, 298 pv0->pv_pa + pv0->pv_size - pv->pv_size - 1); 299 SLIST_REMOVE_AFTER(pv0, pv_list); 300 SLIST_INSERT_HEAD(&bmi->bmi_freechunks, pv, pv_list); 301 } 302 return; 303 } 304 KASSERT(pv->pv_va != (*pvp)->pv_va); 305 pvp = &SLIST_NEXT(*pvp, pv_list); 306 } 307 KASSERT((*pvp) == NULL || pv->pv_va < (*pvp)->pv_va); 308 pv_addr_t * const new_pv = SLIST_FIRST(&bmi->bmi_freechunks); 309 KASSERT(new_pv != NULL); 310 SLIST_REMOVE_HEAD(&bmi->bmi_freechunks, pv_list); 311 *new_pv = *pv; 312 SLIST_NEXT(new_pv, pv_list) = *pvp; 313 (*pvp) = new_pv; 314 315 VPRINTF("%s: adding pv %p (pa %#lx, va %#lx, %lu pages) ", 316 __func__, new_pv, new_pv->pv_pa, new_pv->pv_va, 317 new_pv->pv_size / PAGE_SIZE); 318 if (SLIST_NEXT(new_pv, pv_list)) { 319 VPRINTF("before pa %#lx\n", SLIST_NEXT(new_pv, pv_list)->pv_pa); 320 } else { 321 VPRINTF("at tail\n"); 322 } 323 } 324 325 static void 326 valloc_pages(struct bootmem_info *bmi, pv_addr_t *pv, size_t npages, 327 int prot, int cache, bool zero_p) 328 { 329 size_t nbytes = npages * PAGE_SIZE; 330 pv_addr_t *free_pv = bmi->bmi_freeblocks; 331 size_t free_idx = 0; 332 static bool l1pt_found; 333 334 KASSERT(npages > 0); 335 336 /* 337 * If we haven't allocated the kernel L1 page table and we are aligned 338 * at a L1 table boundary, alloc the memory for it. 339 */ 340 if (!l1pt_found 341 && (free_pv->pv_pa & (L1_TABLE_SIZE - 1)) == 0 342 && free_pv->pv_size >= L1_TABLE_SIZE) { 343 l1pt_found = true; 344 valloc_pages(bmi, &kernel_l1pt, L1_TABLE_SIZE / PAGE_SIZE, 345 VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE, true); 346 add_pages(bmi, &kernel_l1pt); 347 } 348 349 while (nbytes > free_pv->pv_size) { 350 free_pv++; 351 free_idx++; 352 if (free_idx == bmi->bmi_nfreeblocks) { 353 panic("%s: could not allocate %zu bytes", 354 __func__, nbytes); 355 } 356 } 357 358 /* 359 * As we allocate the memory, make sure that we don't walk over 360 * our current first level translation table. 361 */ 362 KASSERT((armreg_ttbr_read() & ~(L1_TABLE_SIZE - 1)) != free_pv->pv_pa); 363 364 #if defined(FDT) 365 fdt_add_reserved_memory_range(free_pv->pv_pa, nbytes); 366 #endif 367 pv->pv_pa = free_pv->pv_pa; 368 pv->pv_va = free_pv->pv_va; 369 pv->pv_size = nbytes; 370 pv->pv_prot = prot; 371 pv->pv_cache = cache; 372 373 /* 374 * If PTE_PAGETABLE uses the same cache modes as PTE_CACHE 375 * just use PTE_CACHE. 376 */ 377 if (cache == PTE_PAGETABLE 378 && pte_l1_s_cache_mode == pte_l1_s_cache_mode_pt 379 && pte_l2_l_cache_mode == pte_l2_l_cache_mode_pt 380 && pte_l2_s_cache_mode == pte_l2_s_cache_mode_pt) 381 pv->pv_cache = PTE_CACHE; 382 383 free_pv->pv_pa += nbytes; 384 free_pv->pv_va += nbytes; 385 free_pv->pv_size -= nbytes; 386 if (free_pv->pv_size == 0) { 387 --bmi->bmi_nfreeblocks; 388 for (; free_idx < bmi->bmi_nfreeblocks; free_idx++) { 389 free_pv[0] = free_pv[1]; 390 } 391 } 392 393 bmi->bmi_freepages -= npages; 394 395 if (zero_p) 396 memset((void *)pv->pv_va, 0, nbytes); 397 } 398 399 void 400 arm32_kernel_vm_init(vaddr_t kernel_vm_base, vaddr_t vectors, vaddr_t iovbase, 401 const struct pmap_devmap *devmap, bool mapallmem_p) 402 { 403 struct bootmem_info * const bmi = &bootmem_info; 404 #ifdef MULTIPROCESSOR 405 const size_t cpu_num = arm_cpu_max; 406 #else 407 const size_t cpu_num = 1; 408 #endif 409 #ifdef ARM_HAS_VBAR 410 const bool map_vectors_p = false; 411 #elif defined(CPU_ARMV7) || defined(CPU_ARM11) 412 const bool map_vectors_p = vectors == ARM_VECTORS_HIGH 413 || (armreg_pfr1_read() & ARM_PFR1_SEC_MASK) == 0; 414 #else 415 const bool map_vectors_p = true; 416 #endif 417 418 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS 419 KASSERT(mapallmem_p); 420 #ifdef ARM_MMU_EXTENDED 421 /* 422 * The direct map VA space ends at the start of the kernel VM space. 423 */ 424 pmap_directlimit = kernel_vm_base; 425 #else 426 KASSERT(kernel_vm_base - KERNEL_BASE >= physical_end - physical_start); 427 #endif /* ARM_MMU_EXTENDED */ 428 #endif /* __HAVE_MM_MD_DIRECT_MAPPED_PHYS */ 429 430 /* 431 * Calculate the number of L2 pages needed for mapping the 432 * kernel + data + stuff. Assume 2 L2 pages for kernel, 1 for vectors, 433 * and 1 for IO 434 */ 435 size_t kernel_size = bmi->bmi_kernelend; 436 kernel_size -= (bmi->bmi_kernelstart & -L2_S_SEGSIZE); 437 kernel_size += L1_TABLE_SIZE_REAL; 438 kernel_size += PAGE_SIZE * KERNEL_L2PT_VMDATA_NUM; 439 if (map_vectors_p) { 440 kernel_size += PAGE_SIZE; /* L2PT for VECTORS */ 441 } 442 if (iovbase) { 443 kernel_size += PAGE_SIZE; /* L2PT for IO */ 444 } 445 kernel_size += 446 cpu_num * (ABT_STACK_SIZE + FIQ_STACK_SIZE + IRQ_STACK_SIZE 447 + UND_STACK_SIZE + UPAGES) * PAGE_SIZE; 448 kernel_size += round_page(MSGBUFSIZE); 449 kernel_size += 0x10000; /* slop */ 450 if (!mapallmem_p) { 451 kernel_size += PAGE_SIZE 452 * ((kernel_size + L2_S_SEGSIZE - 1) / L2_S_SEGSIZE); 453 } 454 kernel_size = round_page(kernel_size); 455 456 /* 457 * Now we know how many L2 pages it will take. 458 */ 459 const size_t KERNEL_L2PT_KERNEL_NUM = 460 round_page(kernel_size + L2_S_SEGSIZE - 1) / L2_S_SEGSIZE; 461 462 VPRINTF("%s: %zu L2 pages are needed to map %#zx kernel bytes\n", 463 __func__, KERNEL_L2PT_KERNEL_NUM, kernel_size); 464 465 KASSERT(KERNEL_L2PT_KERNEL_NUM + KERNEL_L2PT_VMDATA_NUM < __arraycount(bmi->bmi_l2pts)); 466 pv_addr_t * const kernel_l2pt = bmi->bmi_l2pts; 467 pv_addr_t * const vmdata_l2pt = kernel_l2pt + KERNEL_L2PT_KERNEL_NUM; 468 pv_addr_t msgbuf; 469 pv_addr_t text; 470 pv_addr_t data; 471 pv_addr_t chunks[KERNEL_L2PT_KERNEL_NUM+KERNEL_L2PT_VMDATA_NUM+11]; 472 #if ARM_MMU_XSCALE == 1 473 pv_addr_t minidataclean; 474 #endif 475 476 /* 477 * We need to allocate some fixed page tables to get the kernel going. 478 * 479 * We are going to allocate our bootstrap pages from the beginning of 480 * the free space that we just calculated. We allocate one page 481 * directory and a number of page tables and store the physical 482 * addresses in the bmi_l2pts array in bootmem_info. 483 * 484 * The kernel page directory must be on a 16K boundary. The page 485 * tables must be on 4K boundaries. What we do is allocate the 486 * page directory on the first 16K boundary that we encounter, and 487 * the page tables on 4K boundaries otherwise. Since we allocate 488 * at least 3 L2 page tables, we are guaranteed to encounter at 489 * least one 16K aligned region. 490 */ 491 492 VPRINTF("%s: allocating page tables for", __func__); 493 for (size_t i = 0; i < __arraycount(chunks); i++) { 494 SLIST_INSERT_HEAD(&bmi->bmi_freechunks, &chunks[i], pv_list); 495 } 496 497 kernel_l1pt.pv_pa = 0; 498 kernel_l1pt.pv_va = 0; 499 500 /* 501 * Allocate the L2 pages, but if we get to a page that is aligned for 502 * an L1 page table, we will allocate the pages for it first and then 503 * allocate the L2 page. 504 */ 505 506 if (map_vectors_p) { 507 /* 508 * First allocate L2 page for the vectors. 509 */ 510 VPRINTF(" vector"); 511 valloc_pages(bmi, &bmi->bmi_vector_l2pt, 1, 512 VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE, true); 513 add_pages(bmi, &bmi->bmi_vector_l2pt); 514 } 515 516 /* 517 * Now allocate L2 pages for the kernel 518 */ 519 VPRINTF(" kernel"); 520 for (size_t idx = 0; idx < KERNEL_L2PT_KERNEL_NUM; ++idx) { 521 valloc_pages(bmi, &kernel_l2pt[idx], 1, 522 VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE, true); 523 add_pages(bmi, &kernel_l2pt[idx]); 524 } 525 526 /* 527 * Now allocate L2 pages for the initial kernel VA space. 528 */ 529 VPRINTF(" vm"); 530 for (size_t idx = 0; idx < KERNEL_L2PT_VMDATA_NUM; ++idx) { 531 valloc_pages(bmi, &vmdata_l2pt[idx], 1, 532 VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE, true); 533 add_pages(bmi, &vmdata_l2pt[idx]); 534 } 535 536 /* 537 * If someone wanted a L2 page for I/O, allocate it now. 538 */ 539 if (iovbase) { 540 VPRINTF(" io"); 541 valloc_pages(bmi, &bmi->bmi_io_l2pt, 1, 542 VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE, true); 543 add_pages(bmi, &bmi->bmi_io_l2pt); 544 } 545 546 VPRINTF("%s: allocating stacks\n", __func__); 547 548 /* Allocate stacks for all modes and CPUs */ 549 valloc_pages(bmi, &abtstack, ABT_STACK_SIZE * cpu_num, 550 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true); 551 add_pages(bmi, &abtstack); 552 valloc_pages(bmi, &fiqstack, FIQ_STACK_SIZE * cpu_num, 553 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true); 554 add_pages(bmi, &fiqstack); 555 valloc_pages(bmi, &irqstack, IRQ_STACK_SIZE * cpu_num, 556 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true); 557 add_pages(bmi, &irqstack); 558 valloc_pages(bmi, &undstack, UND_STACK_SIZE * cpu_num, 559 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true); 560 add_pages(bmi, &undstack); 561 valloc_pages(bmi, &idlestack, UPAGES * cpu_num, /* SVC32 */ 562 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true); 563 add_pages(bmi, &idlestack); 564 valloc_pages(bmi, &kernelstack, UPAGES, /* SVC32 */ 565 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true); 566 add_pages(bmi, &kernelstack); 567 568 /* Allocate the message buffer from the end of memory. */ 569 const size_t msgbuf_pgs = round_page(MSGBUFSIZE) / PAGE_SIZE; 570 valloc_pages(bmi, &msgbuf, msgbuf_pgs, 571 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, false); 572 add_pages(bmi, &msgbuf); 573 msgbufphys = msgbuf.pv_pa; 574 msgbufaddr = (void *)msgbuf.pv_va; 575 576 if (map_vectors_p) { 577 /* 578 * Allocate a page for the system vector page. 579 * This page will just contain the system vectors and can be 580 * shared by all processes. 581 */ 582 valloc_pages(bmi, &systempage, 1, VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE, 583 PTE_CACHE, true); 584 } 585 systempage.pv_va = vectors; 586 587 /* 588 * If the caller needed a few extra pages for some reason, allocate 589 * them now. 590 */ 591 #if ARM_MMU_XSCALE == 1 592 #if (ARM_NMMUS > 1) 593 if (xscale_use_minidata) 594 #endif 595 valloc_pages(bmi, &minidataclean, 1, 596 VM_PROT_READ|VM_PROT_WRITE, 0, true); 597 #endif 598 599 /* 600 * Ok we have allocated physical pages for the primary kernel 601 * page tables and stacks. Let's just confirm that. 602 */ 603 if (kernel_l1pt.pv_va == 0 604 && (!kernel_l1pt.pv_pa || (kernel_l1pt.pv_pa & (L1_TABLE_SIZE - 1)) != 0)) 605 panic("%s: Failed to allocate or align the kernel " 606 "page directory", __func__); 607 608 609 VPRINTF("Creating L1 page table at 0x%08lx\n", kernel_l1pt.pv_pa); 610 611 /* 612 * Now we start construction of the L1 page table 613 * We start by mapping the L2 page tables into the L1. 614 * This means that we can replace L1 mappings later on if necessary 615 */ 616 vaddr_t l1pt_va = kernel_l1pt.pv_va; 617 paddr_t l1pt_pa = kernel_l1pt.pv_pa; 618 619 if (map_vectors_p) { 620 /* Map the L2 pages tables in the L1 page table */ 621 pmap_link_l2pt(l1pt_va, systempage.pv_va & -L2_S_SEGSIZE, 622 &bmi->bmi_vector_l2pt); 623 VPRINTF("%s: adding L2 pt (VA %#lx, PA %#lx) " 624 "for VA %#lx\n (vectors)", 625 __func__, bmi->bmi_vector_l2pt.pv_va, 626 bmi->bmi_vector_l2pt.pv_pa, systempage.pv_va); 627 } 628 629 const vaddr_t kernel_base = 630 KERN_PHYSTOV(bmi, bmi->bmi_kernelstart & -L2_S_SEGSIZE); 631 for (size_t idx = 0; idx < KERNEL_L2PT_KERNEL_NUM; idx++) { 632 pmap_link_l2pt(l1pt_va, kernel_base + idx * L2_S_SEGSIZE, 633 &kernel_l2pt[idx]); 634 VPRINTF("%s: adding L2 pt (VA %#lx, PA %#lx) for VA %#lx (kernel)\n", 635 __func__, kernel_l2pt[idx].pv_va, 636 kernel_l2pt[idx].pv_pa, kernel_base + idx * L2_S_SEGSIZE); 637 } 638 639 for (size_t idx = 0; idx < KERNEL_L2PT_VMDATA_NUM; idx++) { 640 pmap_link_l2pt(l1pt_va, kernel_vm_base + idx * L2_S_SEGSIZE, 641 &vmdata_l2pt[idx]); 642 VPRINTF("%s: adding L2 pt (VA %#lx, PA %#lx) for VA %#lx (vm)\n", 643 __func__, vmdata_l2pt[idx].pv_va, vmdata_l2pt[idx].pv_pa, 644 kernel_vm_base + idx * L2_S_SEGSIZE); 645 } 646 if (iovbase) { 647 pmap_link_l2pt(l1pt_va, iovbase & -L2_S_SEGSIZE, &bmi->bmi_io_l2pt); 648 VPRINTF("%s: adding L2 pt (VA %#lx, PA %#lx) for VA %#lx (io)\n", 649 __func__, bmi->bmi_io_l2pt.pv_va, bmi->bmi_io_l2pt.pv_pa, 650 iovbase & -L2_S_SEGSIZE); 651 } 652 653 /* update the top of the kernel VM */ 654 pmap_curmaxkvaddr = 655 kernel_vm_base + (KERNEL_L2PT_VMDATA_NUM * L2_S_SEGSIZE); 656 657 VPRINTF("Mapping kernel\n"); 658 659 extern char etext[], _end[]; 660 size_t totalsize = bmi->bmi_kernelend - bmi->bmi_kernelstart; 661 size_t textsize = KERN_VTOPHYS(bmi, (uintptr_t)etext) - bmi->bmi_kernelstart; 662 663 textsize = (textsize + PGOFSET) & ~PGOFSET; 664 665 /* start at offset of kernel in RAM */ 666 667 text.pv_pa = bmi->bmi_kernelstart; 668 text.pv_va = KERN_PHYSTOV(bmi, bmi->bmi_kernelstart); 669 text.pv_size = textsize; 670 text.pv_prot = VM_PROT_READ | VM_PROT_EXECUTE; 671 text.pv_cache = PTE_CACHE; 672 673 VPRINTF("%s: adding chunk for kernel text %#lx..%#lx (VA %#lx)\n", 674 __func__, text.pv_pa, text.pv_pa + text.pv_size - 1, text.pv_va); 675 676 add_pages(bmi, &text); 677 678 data.pv_pa = text.pv_pa + textsize; 679 data.pv_va = text.pv_va + textsize; 680 data.pv_size = totalsize - textsize; 681 data.pv_prot = VM_PROT_READ|VM_PROT_WRITE; 682 data.pv_cache = PTE_CACHE; 683 684 VPRINTF("%s: adding chunk for kernel data/bss %#lx..%#lx (VA %#lx)\n", 685 __func__, data.pv_pa, data.pv_pa + data.pv_size - 1, data.pv_va); 686 687 add_pages(bmi, &data); 688 689 VPRINTF("Listing Chunks\n"); 690 691 pv_addr_t *lpv; 692 SLIST_FOREACH(lpv, &bmi->bmi_chunks, pv_list) { 693 VPRINTF("%s: pv %p: chunk VA %#lx..%#lx " 694 "(PA %#lx, prot %d, cache %d)\n", 695 __func__, lpv, lpv->pv_va, lpv->pv_va + lpv->pv_size - 1, 696 lpv->pv_pa, lpv->pv_prot, lpv->pv_cache); 697 } 698 VPRINTF("\nMapping Chunks\n"); 699 700 pv_addr_t cur_pv; 701 pv_addr_t *pv = SLIST_FIRST(&bmi->bmi_chunks); 702 if (!mapallmem_p || pv->pv_pa == bmi->bmi_start) { 703 cur_pv = *pv; 704 KASSERTMSG(cur_pv.pv_va >= KERNEL_BASE, "%#lx", cur_pv.pv_va); 705 pv = SLIST_NEXT(pv, pv_list); 706 } else { 707 cur_pv.pv_va = KERNEL_BASE; 708 cur_pv.pv_pa = KERN_VTOPHYS(bmi, cur_pv.pv_va); 709 cur_pv.pv_size = pv->pv_pa - cur_pv.pv_pa; 710 cur_pv.pv_prot = VM_PROT_READ | VM_PROT_WRITE; 711 cur_pv.pv_cache = PTE_CACHE; 712 } 713 while (pv != NULL) { 714 if (mapallmem_p) { 715 if (concat_pvaddr(&cur_pv, pv)) { 716 pv = SLIST_NEXT(pv, pv_list); 717 continue; 718 } 719 if (cur_pv.pv_pa + cur_pv.pv_size < pv->pv_pa) { 720 /* 721 * See if we can extend the current pv to emcompass the 722 * hole, and if so do it and retry the concatenation. 723 */ 724 if (cur_pv.pv_prot == (VM_PROT_READ|VM_PROT_WRITE) 725 && cur_pv.pv_cache == PTE_CACHE) { 726 cur_pv.pv_size = pv->pv_pa - cur_pv.pv_va; 727 continue; 728 } 729 730 /* 731 * We couldn't so emit the current chunk and then 732 */ 733 VPRINTF("%s: mapping chunk VA %#lx..%#lx " 734 "(PA %#lx, prot %d, cache %d)\n", 735 __func__, 736 cur_pv.pv_va, cur_pv.pv_va + cur_pv.pv_size - 1, 737 cur_pv.pv_pa, cur_pv.pv_prot, cur_pv.pv_cache); 738 pmap_map_chunk(l1pt_va, cur_pv.pv_va, cur_pv.pv_pa, 739 cur_pv.pv_size, cur_pv.pv_prot, cur_pv.pv_cache); 740 741 /* 742 * set the current chunk to the hole and try again. 743 */ 744 cur_pv.pv_pa += cur_pv.pv_size; 745 cur_pv.pv_va += cur_pv.pv_size; 746 cur_pv.pv_size = pv->pv_pa - cur_pv.pv_va; 747 cur_pv.pv_prot = VM_PROT_READ | VM_PROT_WRITE; 748 cur_pv.pv_cache = PTE_CACHE; 749 continue; 750 } 751 } 752 753 /* 754 * The new pv didn't concatenate so emit the current one 755 * and use the new pv as the current pv. 756 */ 757 VPRINTF("%s: mapping chunk VA %#lx..%#lx " 758 "(PA %#lx, prot %d, cache %d)\n", 759 __func__, cur_pv.pv_va, cur_pv.pv_va + cur_pv.pv_size - 1, 760 cur_pv.pv_pa, cur_pv.pv_prot, cur_pv.pv_cache); 761 pmap_map_chunk(l1pt_va, cur_pv.pv_va, cur_pv.pv_pa, 762 cur_pv.pv_size, cur_pv.pv_prot, cur_pv.pv_cache); 763 cur_pv = *pv; 764 pv = SLIST_NEXT(pv, pv_list); 765 } 766 767 /* 768 * If we are mapping all of memory, let's map the rest of memory. 769 */ 770 if (mapallmem_p && cur_pv.pv_pa + cur_pv.pv_size < bmi->bmi_end) { 771 if (cur_pv.pv_prot == (VM_PROT_READ | VM_PROT_WRITE) 772 && cur_pv.pv_cache == PTE_CACHE) { 773 cur_pv.pv_size = bmi->bmi_end - cur_pv.pv_pa; 774 } else { 775 KASSERTMSG(cur_pv.pv_va + cur_pv.pv_size <= kernel_vm_base, 776 "%#lx >= %#lx", cur_pv.pv_va + cur_pv.pv_size, 777 kernel_vm_base); 778 VPRINTF("%s: mapping chunk VA %#lx..%#lx " 779 "(PA %#lx, prot %d, cache %d)\n", 780 __func__, cur_pv.pv_va, cur_pv.pv_va + cur_pv.pv_size - 1, 781 cur_pv.pv_pa, cur_pv.pv_prot, cur_pv.pv_cache); 782 pmap_map_chunk(l1pt_va, cur_pv.pv_va, cur_pv.pv_pa, 783 cur_pv.pv_size, cur_pv.pv_prot, cur_pv.pv_cache); 784 cur_pv.pv_pa += cur_pv.pv_size; 785 cur_pv.pv_va += cur_pv.pv_size; 786 cur_pv.pv_size = bmi->bmi_end - cur_pv.pv_pa; 787 cur_pv.pv_prot = VM_PROT_READ | VM_PROT_WRITE; 788 cur_pv.pv_cache = PTE_CACHE; 789 } 790 } 791 792 // The amount we can direct is limited by the start of the 793 // virtual part of the kernel address space. Don't overrun 794 // into it. 795 if (mapallmem_p && cur_pv.pv_va + cur_pv.pv_size > kernel_vm_base) { 796 cur_pv.pv_size = kernel_vm_base - cur_pv.pv_va; 797 } 798 799 /* 800 * Now we map the final chunk. 801 */ 802 VPRINTF("%s: mapping last chunk VA %#lx..%#lx (PA %#lx, prot %d, cache %d)\n", 803 __func__, cur_pv.pv_va, cur_pv.pv_va + cur_pv.pv_size - 1, 804 cur_pv.pv_pa, cur_pv.pv_prot, cur_pv.pv_cache); 805 pmap_map_chunk(l1pt_va, cur_pv.pv_va, cur_pv.pv_pa, 806 cur_pv.pv_size, cur_pv.pv_prot, cur_pv.pv_cache); 807 808 /* 809 * Now we map the stuff that isn't directly after the kernel 810 */ 811 if (map_vectors_p) { 812 /* Map the vector page. */ 813 pmap_map_entry(l1pt_va, systempage.pv_va, systempage.pv_pa, 814 VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE, PTE_CACHE); 815 } 816 817 /* Map the Mini-Data cache clean area. */ 818 #if ARM_MMU_XSCALE == 1 819 #if (ARM_NMMUS > 1) 820 if (xscale_use_minidata) 821 #endif 822 xscale_setup_minidata(l1pt_va, minidataclean.pv_va, 823 minidataclean.pv_pa); 824 #endif 825 826 /* 827 * Map integrated peripherals at same address in first level page 828 * table so that we can continue to use console. 829 */ 830 if (devmap) 831 pmap_devmap_bootstrap(l1pt_va, devmap); 832 833 /* Tell the user about where all the bits and pieces live. */ 834 VPRINTF("%22s Physical Virtual Num\n", " "); 835 VPRINTF("%22s Starting Ending Starting Ending Pages\n", " "); 836 837 static const char mem_fmt[] = 838 "%20s: 0x%08lx 0x%08lx 0x%08lx 0x%08lx %u\n"; 839 static const char mem_fmt_nov[] = 840 "%20s: 0x%08lx 0x%08lx %zu\n"; 841 842 VPRINTF(mem_fmt, "SDRAM", bmi->bmi_start, bmi->bmi_end - 1, 843 KERN_PHYSTOV(bmi, bmi->bmi_start), KERN_PHYSTOV(bmi, bmi->bmi_end - 1), 844 (int)physmem); 845 VPRINTF(mem_fmt, "text section", 846 text.pv_pa, text.pv_pa + text.pv_size - 1, 847 text.pv_va, text.pv_va + text.pv_size - 1, 848 (int)(text.pv_size / PAGE_SIZE)); 849 VPRINTF(mem_fmt, "data section", 850 KERN_VTOPHYS(bmi, __data_start), KERN_VTOPHYS(bmi, _edata), 851 (vaddr_t)__data_start, (vaddr_t)_edata, 852 (int)((round_page((vaddr_t)_edata) 853 - trunc_page((vaddr_t)__data_start)) / PAGE_SIZE)); 854 VPRINTF(mem_fmt, "bss section", 855 KERN_VTOPHYS(bmi, __bss_start), KERN_VTOPHYS(bmi, __bss_end__), 856 (vaddr_t)__bss_start, (vaddr_t)__bss_end__, 857 (int)((round_page((vaddr_t)__bss_end__) 858 - trunc_page((vaddr_t)__bss_start)) / PAGE_SIZE)); 859 VPRINTF(mem_fmt, "L1 page directory", 860 kernel_l1pt.pv_pa, kernel_l1pt.pv_pa + L1_TABLE_SIZE - 1, 861 kernel_l1pt.pv_va, kernel_l1pt.pv_va + L1_TABLE_SIZE - 1, 862 L1_TABLE_SIZE / PAGE_SIZE); 863 VPRINTF(mem_fmt, "ABT stack (CPU 0)", 864 abtstack.pv_pa, abtstack.pv_pa + (ABT_STACK_SIZE * PAGE_SIZE) - 1, 865 abtstack.pv_va, abtstack.pv_va + (ABT_STACK_SIZE * PAGE_SIZE) - 1, 866 ABT_STACK_SIZE); 867 VPRINTF(mem_fmt, "FIQ stack (CPU 0)", 868 fiqstack.pv_pa, fiqstack.pv_pa + (FIQ_STACK_SIZE * PAGE_SIZE) - 1, 869 fiqstack.pv_va, fiqstack.pv_va + (FIQ_STACK_SIZE * PAGE_SIZE) - 1, 870 FIQ_STACK_SIZE); 871 VPRINTF(mem_fmt, "IRQ stack (CPU 0)", 872 irqstack.pv_pa, irqstack.pv_pa + (IRQ_STACK_SIZE * PAGE_SIZE) - 1, 873 irqstack.pv_va, irqstack.pv_va + (IRQ_STACK_SIZE * PAGE_SIZE) - 1, 874 IRQ_STACK_SIZE); 875 VPRINTF(mem_fmt, "UND stack (CPU 0)", 876 undstack.pv_pa, undstack.pv_pa + (UND_STACK_SIZE * PAGE_SIZE) - 1, 877 undstack.pv_va, undstack.pv_va + (UND_STACK_SIZE * PAGE_SIZE) - 1, 878 UND_STACK_SIZE); 879 VPRINTF(mem_fmt, "IDLE stack (CPU 0)", 880 idlestack.pv_pa, idlestack.pv_pa + (UPAGES * PAGE_SIZE) - 1, 881 idlestack.pv_va, idlestack.pv_va + (UPAGES * PAGE_SIZE) - 1, 882 UPAGES); 883 VPRINTF(mem_fmt, "SVC stack", 884 kernelstack.pv_pa, kernelstack.pv_pa + (UPAGES * PAGE_SIZE) - 1, 885 kernelstack.pv_va, kernelstack.pv_va + (UPAGES * PAGE_SIZE) - 1, 886 UPAGES); 887 VPRINTF(mem_fmt, "Message Buffer", 888 msgbuf.pv_pa, msgbuf.pv_pa + (msgbuf_pgs * PAGE_SIZE) - 1, 889 msgbuf.pv_va, msgbuf.pv_va + (msgbuf_pgs * PAGE_SIZE) - 1, 890 (int)msgbuf_pgs); 891 if (map_vectors_p) { 892 VPRINTF(mem_fmt, "Exception Vectors", 893 systempage.pv_pa, systempage.pv_pa + PAGE_SIZE - 1, 894 systempage.pv_va, systempage.pv_va + PAGE_SIZE - 1, 895 1); 896 } 897 for (size_t i = 0; i < bmi->bmi_nfreeblocks; i++) { 898 pv = &bmi->bmi_freeblocks[i]; 899 900 VPRINTF(mem_fmt_nov, "Free Memory", 901 pv->pv_pa, pv->pv_pa + pv->pv_size - 1, 902 pv->pv_size / PAGE_SIZE); 903 } 904 /* 905 * Now we have the real page tables in place so we can switch to them. 906 * Once this is done we will be running with the REAL kernel page 907 * tables. 908 */ 909 910 VPRINTF("TTBR0=%#x", armreg_ttbr_read()); 911 #ifdef _ARM_ARCH_6 912 VPRINTF(" TTBR1=%#x TTBCR=%#x CONTEXTIDR=%#x", 913 armreg_ttbr1_read(), armreg_ttbcr_read(), 914 armreg_contextidr_read()); 915 #endif 916 VPRINTF("\n"); 917 918 /* Switch tables */ 919 VPRINTF("switching to new L1 page table @%#lx...", l1pt_pa); 920 921 #ifdef ARM_MMU_EXTENDED 922 cpu_domains((DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL*2)) 923 | (DOMAIN_CLIENT << (PMAP_DOMAIN_USER*2))); 924 #else 925 cpu_domains((DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL*2)) | DOMAIN_CLIENT); 926 #endif 927 cpu_idcache_wbinv_all(); 928 VPRINTF(" ttb"); 929 #ifdef ARM_MMU_EXTENDED 930 /* 931 * TTBCR should have been initialized by the MD start code. 932 */ 933 KASSERT((armreg_contextidr_read() & 0xff) == 0); 934 KASSERT(armreg_ttbcr_read() == __SHIFTIN(1, TTBCR_S_N)); 935 /* 936 * Disable lookups via TTBR0 until there is an activated pmap. 937 */ 938 armreg_ttbcr_write(armreg_ttbcr_read() | TTBCR_S_PD0); 939 cpu_setttb(l1pt_pa, KERNEL_PID); 940 arm_isb(); 941 #else 942 cpu_setttb(l1pt_pa, true); 943 cpu_domains(DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL*2)); 944 #endif 945 cpu_tlb_flushID(); 946 947 #ifdef ARM_MMU_EXTENDED 948 VPRINTF(" (TTBCR=%#x TTBR0=%#x TTBR1=%#x)", 949 armreg_ttbcr_read(), armreg_ttbr_read(), armreg_ttbr1_read()); 950 #else 951 VPRINTF(" (TTBR0=%#x)", armreg_ttbr_read()); 952 #endif 953 954 #ifdef MULTIPROCESSOR 955 /* 956 * Kick the secondaries to load the TTB. After which they'll go 957 * back to sleep to wait for the final kick so they will hatch. 958 */ 959 VPRINTF(" hatchlings"); 960 cpu_boot_secondary_processors(); 961 #endif 962 963 VPRINTF(" OK\n"); 964 } 965
Indexes created Mon Sep 29 03:10:08 GMT 2025