arm32_kvminit.c revision 1.22
1 /* $NetBSD: arm32_kvminit.c,v 1.22 2013/08/29 15:46:17 riz Exp $ */ 2 3 /* 4 * Copyright (c) 2002, 2003, 2005 Genetec Corporation. All rights reserved. 5 * Written by Hiroyuki Bessho for Genetec Corporation. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. The name of Genetec Corporation may not be used to endorse or 16 * promote products derived from this software without specific prior 17 * written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY GENETEC CORPORATION ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENETEC CORPORATION 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 * 31 * Copyright (c) 2001 Wasabi Systems, Inc. 32 * All rights reserved. 33 * 34 * Written by Jason R. Thorpe for Wasabi Systems, Inc. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice, this list of conditions and the following disclaimer. 41 * 2. Redistributions in binary form must reproduce the above copyright 42 * notice, this list of conditions and the following disclaimer in the 43 * documentation and/or other materials provided with the distribution. 44 * 3. All advertising materials mentioning features or use of this software 45 * must display the following acknowledgement: 46 * This product includes software developed for the NetBSD Project by 47 * Wasabi Systems, Inc. 48 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 49 * or promote products derived from this software without specific prior 50 * written permission. 51 * 52 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 53 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 54 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 55 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 56 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 57 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 58 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 59 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 60 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 61 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 62 * POSSIBILITY OF SUCH DAMAGE. 63 * 64 * Copyright (c) 1997,1998 Mark Brinicombe. 65 * Copyright (c) 1997,1998 Causality Limited. 66 * All rights reserved. 67 * 68 * Redistribution and use in source and binary forms, with or without 69 * modification, are permitted provided that the following conditions 70 * are met: 71 * 1. Redistributions of source code must retain the above copyright 72 * notice, this list of conditions and the following disclaimer. 73 * 2. Redistributions in binary form must reproduce the above copyright 74 * notice, this list of conditions and the following disclaimer in the 75 * documentation and/or other materials provided with the distribution. 76 * 3. All advertising materials mentioning features or use of this software 77 * must display the following acknowledgement: 78 * This product includes software developed by Mark Brinicombe 79 * for the NetBSD Project. 80 * 4. The name of the company nor the name of the author may be used to 81 * endorse or promote products derived from this software without specific 82 * prior written permission. 83 * 84 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 85 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 86 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 87 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 88 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 89 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 90 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 91 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 92 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 93 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 94 * SUCH DAMAGE. 95 * 96 * Copyright (c) 2007 Microsoft 97 * All rights reserved. 98 * 99 * Redistribution and use in source and binary forms, with or without 100 * modification, are permitted provided that the following conditions 101 * are met: 102 * 1. Redistributions of source code must retain the above copyright 103 * notice, this list of conditions and the following disclaimer. 104 * 2. Redistributions in binary form must reproduce the above copyright 105 * notice, this list of conditions and the following disclaimer in the 106 * documentation and/or other materials provided with the distribution. 107 * 3. All advertising materials mentioning features or use of this software 108 * must display the following acknowledgement: 109 * This product includes software developed by Microsoft 110 * 111 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 112 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 113 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 114 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTERS BE LIABLE FOR ANY DIRECT, 115 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 116 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 117 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 118 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 119 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 120 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 121 * SUCH DAMAGE. 122 */ 123 124 #include <sys/cdefs.h> 125 __KERNEL_RCSID(0, "$NetBSD: arm32_kvminit.c,v 1.22 2013/08/29 15:46:17 riz Exp $"); 126 127 #include <sys/param.h> 128 #include <sys/device.h> 129 #include <sys/kernel.h> 130 #include <sys/reboot.h> 131 #include <sys/bus.h> 132 133 #include <dev/cons.h> 134 135 #include <uvm/uvm_extern.h> 136 137 #include <arm/db_machdep.h> 138 #include <arm/undefined.h> 139 #include <arm/bootconfig.h> 140 #include <arm/arm32/machdep.h> 141 142 #include "ksyms.h" 143 144 struct bootmem_info bootmem_info; 145 146 paddr_t msgbufphys; 147 paddr_t physical_start; 148 paddr_t physical_end; 149 150 extern char etext[]; 151 extern char __data_start[], _edata[]; 152 extern char __bss_start[], __bss_end__[]; 153 extern char _end[]; 154 155 /* Page tables for mapping kernel VM */ 156 #define KERNEL_L2PT_VMDATA_NUM 8 /* start with 32MB of KVM */ 157 158 /* 159 * Macros to translate between physical and virtual for a subset of the 160 * kernel address space. *Not* for general use. 161 */ 162 #define KERN_VTOPHYS(bmi, va) \ 163 ((paddr_t)((vaddr_t)(va) - KERNEL_BASE + (bmi)->bmi_start)) 164 #define KERN_PHYSTOV(bmi, pa) \ 165 ((vaddr_t)((paddr_t)(pa) - (bmi)->bmi_start + KERNEL_BASE)) 166 167 void 168 arm32_bootmem_init(paddr_t memstart, psize_t memsize, vsize_t kernelstart) 169 { 170 struct bootmem_info * const bmi = &bootmem_info; 171 pv_addr_t *pv = bmi->bmi_freeblocks; 172 173 #ifdef VERBOSE_INIT_ARM 174 printf("%s: memstart=%#lx, memsize=%#lx, kernelstart=%#lx\n", 175 __func__, memstart, memsize, kernelstart); 176 #endif 177 178 physical_start = bmi->bmi_start = memstart; 179 physical_end = bmi->bmi_end = memstart + memsize; 180 physmem = memsize / PAGE_SIZE; 181 182 /* 183 * Let's record where the kernel lives. 184 */ 185 bmi->bmi_kernelstart = kernelstart; 186 bmi->bmi_kernelend = KERN_VTOPHYS(bmi, round_page((vaddr_t)_end)); 187 188 #ifdef VERBOSE_INIT_ARM 189 printf("%s: kernelend=%#lx\n", __func__, bmi->bmi_kernelend); 190 #endif 191 192 /* 193 * Now the rest of the free memory must be after the kernel. 194 */ 195 pv->pv_pa = bmi->bmi_kernelend; 196 pv->pv_va = KERN_PHYSTOV(bmi, pv->pv_pa); 197 pv->pv_size = bmi->bmi_end - bmi->bmi_kernelend; 198 bmi->bmi_freepages += pv->pv_size / PAGE_SIZE; 199 #ifdef VERBOSE_INIT_ARM 200 printf("%s: adding %lu free pages: [%#lx..%#lx] (VA %#lx)\n", 201 __func__, pv->pv_size / PAGE_SIZE, pv->pv_pa, 202 pv->pv_pa + pv->pv_size - 1, pv->pv_va); 203 #endif 204 pv++; 205 206 /* 207 * Add a free block for any memory before the kernel. 208 */ 209 if (bmi->bmi_start < bmi->bmi_kernelstart) { 210 pv->pv_pa = bmi->bmi_start; 211 pv->pv_va = KERNEL_BASE; 212 pv->pv_size = bmi->bmi_kernelstart - bmi->bmi_start; 213 bmi->bmi_freepages += pv->pv_size / PAGE_SIZE; 214 #ifdef VERBOSE_INIT_ARM 215 printf("%s: adding %lu free pages: [%#lx..%#lx] (VA %#lx)\n", 216 __func__, pv->pv_size / PAGE_SIZE, pv->pv_pa, 217 pv->pv_pa + pv->pv_size - 1, pv->pv_va); 218 #endif 219 pv++; 220 } 221 222 bmi->bmi_nfreeblocks = pv - bmi->bmi_freeblocks; 223 224 SLIST_INIT(&bmi->bmi_freechunks); 225 SLIST_INIT(&bmi->bmi_chunks); 226 } 227 228 static bool 229 concat_pvaddr(pv_addr_t *acc_pv, pv_addr_t *pv) 230 { 231 if (acc_pv->pv_pa + acc_pv->pv_size == pv->pv_pa 232 && acc_pv->pv_va + acc_pv->pv_size == pv->pv_va 233 && acc_pv->pv_prot == pv->pv_prot 234 && acc_pv->pv_cache == pv->pv_cache) { 235 #ifdef VERBOSE_INIT_ARMX 236 printf("%s: appending pv %p (%#lx..%#lx) to %#lx..%#lx\n", 237 __func__, pv, pv->pv_pa, pv->pv_pa + pv->pv_size + 1, 238 acc_pv->pv_pa, acc_pv->pv_pa + acc_pv->pv_size + 1); 239 #endif 240 acc_pv->pv_size += pv->pv_size; 241 return true; 242 } 243 244 return false; 245 } 246 247 static void 248 add_pages(struct bootmem_info *bmi, pv_addr_t *pv) 249 { 250 pv_addr_t **pvp = &SLIST_FIRST(&bmi->bmi_chunks); 251 while ((*pvp) != NULL && (*pvp)->pv_va <= pv->pv_va) { 252 pv_addr_t * const pv0 = (*pvp); 253 KASSERT(SLIST_NEXT(pv0, pv_list) == NULL || pv0->pv_pa < SLIST_NEXT(pv0, pv_list)->pv_pa); 254 if (concat_pvaddr(pv0, pv)) { 255 #ifdef VERBOSE_INIT_ARM 256 printf("%s: %s pv %p (%#lx..%#lx) to %#lx..%#lx\n", 257 __func__, "appending", pv, 258 pv->pv_pa, pv->pv_pa + pv->pv_size - 1, 259 pv0->pv_pa, pv0->pv_pa + pv0->pv_size - pv->pv_size - 1); 260 #endif 261 pv = SLIST_NEXT(pv0, pv_list); 262 if (pv != NULL && concat_pvaddr(pv0, pv)) { 263 #ifdef VERBOSE_INIT_ARM 264 printf("%s: %s pv %p (%#lx..%#lx) to %#lx..%#lx\n", 265 __func__, "merging", pv, 266 pv->pv_pa, pv->pv_pa + pv->pv_size - 1, 267 pv0->pv_pa, 268 pv0->pv_pa + pv0->pv_size - pv->pv_size - 1); 269 #endif 270 SLIST_REMOVE_AFTER(pv0, pv_list); 271 SLIST_INSERT_HEAD(&bmi->bmi_freechunks, pv, pv_list); 272 } 273 return; 274 } 275 KASSERT(pv->pv_va != (*pvp)->pv_va); 276 pvp = &SLIST_NEXT(*pvp, pv_list); 277 } 278 KASSERT((*pvp) == NULL || pv->pv_va < (*pvp)->pv_va); 279 pv_addr_t * const new_pv = SLIST_FIRST(&bmi->bmi_freechunks); 280 KASSERT(new_pv != NULL); 281 SLIST_REMOVE_HEAD(&bmi->bmi_freechunks, pv_list); 282 *new_pv = *pv; 283 SLIST_NEXT(new_pv, pv_list) = *pvp; 284 (*pvp) = new_pv; 285 #ifdef VERBOSE_INIT_ARM 286 printf("%s: adding pv %p (pa %#lx, va %#lx, %lu pages) ", 287 __func__, new_pv, new_pv->pv_pa, new_pv->pv_va, 288 new_pv->pv_size / PAGE_SIZE); 289 if (SLIST_NEXT(new_pv, pv_list)) 290 printf("before pa %#lx\n", SLIST_NEXT(new_pv, pv_list)->pv_pa); 291 else 292 printf("at tail\n"); 293 #endif 294 } 295 296 static void 297 valloc_pages(struct bootmem_info *bmi, pv_addr_t *pv, size_t npages, 298 int prot, int cache, bool zero_p) 299 { 300 size_t nbytes = npages * PAGE_SIZE; 301 pv_addr_t *free_pv = bmi->bmi_freeblocks; 302 size_t free_idx = 0; 303 static bool l1pt_found; 304 305 /* 306 * If we haven't allocated the kernel L1 page table and we are aligned 307 * at a L1 table boundary, alloc the memory for it. 308 */ 309 if (!l1pt_found 310 && (free_pv->pv_pa & (L1_TABLE_SIZE - 1)) == 0 311 && free_pv->pv_size >= L1_TABLE_SIZE) { 312 l1pt_found = true; 313 valloc_pages(bmi, &kernel_l1pt, L1_TABLE_SIZE / PAGE_SIZE, 314 VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE, true); 315 add_pages(bmi, &kernel_l1pt); 316 } 317 318 while (nbytes > free_pv->pv_size) { 319 free_pv++; 320 free_idx++; 321 if (free_idx == bmi->bmi_nfreeblocks) { 322 panic("%s: could not allocate %zu bytes", 323 __func__, nbytes); 324 } 325 } 326 327 /* 328 * As we allocate the memory, make sure that we don't walk over 329 * our current first level translation table. 330 */ 331 KASSERT((armreg_ttbr_read() & ~(L1_TABLE_SIZE - 1)) != free_pv->pv_pa); 332 333 pv->pv_pa = free_pv->pv_pa; 334 pv->pv_va = free_pv->pv_va; 335 pv->pv_size = nbytes; 336 pv->pv_prot = prot; 337 pv->pv_cache = cache; 338 339 /* 340 * If PTE_PAGETABLE uses the same cache modes as PTE_CACHE 341 * just use PTE_CACHE. 342 */ 343 if (cache == PTE_PAGETABLE 344 && pte_l1_s_cache_mode == pte_l1_s_cache_mode_pt 345 && pte_l2_l_cache_mode == pte_l2_l_cache_mode_pt 346 && pte_l2_s_cache_mode == pte_l2_s_cache_mode_pt) 347 pv->pv_cache = PTE_CACHE; 348 349 free_pv->pv_pa += nbytes; 350 free_pv->pv_va += nbytes; 351 free_pv->pv_size -= nbytes; 352 if (free_pv->pv_size == 0) { 353 --bmi->bmi_nfreeblocks; 354 for (; free_idx < bmi->bmi_nfreeblocks; free_idx++) { 355 free_pv[0] = free_pv[1]; 356 } 357 } 358 359 bmi->bmi_freepages -= npages; 360 361 if (zero_p) 362 memset((void *)pv->pv_va, 0, nbytes); 363 } 364 365 void 366 arm32_kernel_vm_init(vaddr_t kernel_vm_base, vaddr_t vectors, vaddr_t iovbase, 367 const struct pmap_devmap *devmap, bool mapallmem_p) 368 { 369 struct bootmem_info * const bmi = &bootmem_info; 370 #ifdef MULTIPROCESSOR 371 const size_t cpu_num = arm_cpu_max + 1; 372 #else 373 const size_t cpu_num = 1; 374 #endif 375 #ifdef ARM_HAS_VBAR 376 const bool map_vectors_p = false; 377 #elif defined(CPU_ARMV7) || defined(CPU_ARM11) 378 const bool map_vectors_p = vectors == ARM_VECTORS_HIGH 379 || (armreg_pfr1_read() & ARM_PFR1_SEC_MASK) == 0; 380 #else 381 const bool map_vectors_p = true; 382 #endif 383 384 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS 385 KASSERT(mapallmem_p); 386 #endif 387 388 /* 389 * Calculate the number of L2 pages needed for mapping the 390 * kernel + data + stuff. Assume 2 L2 pages for kernel, 1 for vectors, 391 * and 1 for IO 392 */ 393 size_t kernel_size = bmi->bmi_kernelend; 394 kernel_size -= (bmi->bmi_kernelstart & -L2_S_SEGSIZE); 395 kernel_size += L1_TABLE_SIZE; 396 kernel_size += L2_TABLE_SIZE * (2 + 1 + KERNEL_L2PT_VMDATA_NUM + 1); 397 kernel_size += 398 cpu_num * (ABT_STACK_SIZE + FIQ_STACK_SIZE + IRQ_STACK_SIZE 399 + UND_STACK_SIZE + UPAGES) * PAGE_SIZE; 400 kernel_size += round_page(MSGBUFSIZE); 401 kernel_size += 0x10000; /* slop */ 402 kernel_size += PAGE_SIZE * (kernel_size + L2_S_SEGSIZE - 1) / L2_S_SEGSIZE; 403 kernel_size = round_page(kernel_size); 404 405 /* 406 * Now we know how many L2 pages it will take. 407 */ 408 const size_t KERNEL_L2PT_KERNEL_NUM = 409 (kernel_size + L2_S_SEGSIZE - 1) / L2_S_SEGSIZE; 410 411 #ifdef VERBOSE_INIT_ARM 412 printf("%s: %zu L2 pages are needed to map %#zx kernel bytes\n", 413 __func__, KERNEL_L2PT_KERNEL_NUM, kernel_size); 414 #endif 415 416 KASSERT(KERNEL_L2PT_KERNEL_NUM + KERNEL_L2PT_VMDATA_NUM < __arraycount(bmi->bmi_l2pts)); 417 pv_addr_t * const kernel_l2pt = bmi->bmi_l2pts; 418 pv_addr_t * const vmdata_l2pt = kernel_l2pt + KERNEL_L2PT_KERNEL_NUM; 419 pv_addr_t msgbuf; 420 pv_addr_t text; 421 pv_addr_t data; 422 pv_addr_t chunks[KERNEL_L2PT_KERNEL_NUM+KERNEL_L2PT_VMDATA_NUM+11]; 423 #if ARM_MMU_XSCALE == 1 424 pv_addr_t minidataclean; 425 #endif 426 427 /* 428 * We need to allocate some fixed page tables to get the kernel going. 429 * 430 * We are going to allocate our bootstrap pages from the beginning of 431 * the free space that we just calculated. We allocate one page 432 * directory and a number of page tables and store the physical 433 * addresses in the bmi_l2pts array in bootmem_info. 434 * 435 * The kernel page directory must be on a 16K boundary. The page 436 * tables must be on 4K boundaries. What we do is allocate the 437 * page directory on the first 16K boundary that we encounter, and 438 * the page tables on 4K boundaries otherwise. Since we allocate 439 * at least 3 L2 page tables, we are guaranteed to encounter at 440 * least one 16K aligned region. 441 */ 442 443 #ifdef VERBOSE_INIT_ARM 444 printf("%s: allocating page tables for", __func__); 445 #endif 446 for (size_t i = 0; i < __arraycount(chunks); i++) { 447 SLIST_INSERT_HEAD(&bmi->bmi_freechunks, &chunks[i], pv_list); 448 } 449 450 kernel_l1pt.pv_pa = 0; 451 kernel_l1pt.pv_va = 0; 452 453 /* 454 * Allocate the L2 pages, but if we get to a page that is aligned for 455 * an L1 page table, we will allocate the pages for it first and then 456 * allocate the L2 page. 457 */ 458 459 if (map_vectors_p) { 460 /* 461 * First allocate L2 page for the vectors. 462 */ 463 #ifdef VERBOSE_INIT_ARM 464 printf(" vector"); 465 #endif 466 valloc_pages(bmi, &bmi->bmi_vector_l2pt, 467 L2_TABLE_SIZE / PAGE_SIZE, VM_PROT_READ|VM_PROT_WRITE, 468 PTE_PAGETABLE, true); 469 add_pages(bmi, &bmi->bmi_vector_l2pt); 470 } 471 472 /* 473 * Now allocate L2 pages for the kernel 474 */ 475 #ifdef VERBOSE_INIT_ARM 476 printf(" kernel"); 477 #endif 478 for (size_t idx = 0; idx < KERNEL_L2PT_KERNEL_NUM; ++idx) { 479 valloc_pages(bmi, &kernel_l2pt[idx], L2_TABLE_SIZE / PAGE_SIZE, 480 VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE, true); 481 add_pages(bmi, &kernel_l2pt[idx]); 482 } 483 484 /* 485 * Now allocate L2 pages for the initial kernel VA space. 486 */ 487 #ifdef VERBOSE_INIT_ARM 488 printf(" vm"); 489 #endif 490 for (size_t idx = 0; idx < KERNEL_L2PT_VMDATA_NUM; ++idx) { 491 valloc_pages(bmi, &vmdata_l2pt[idx], L2_TABLE_SIZE / PAGE_SIZE, 492 VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE, true); 493 add_pages(bmi, &vmdata_l2pt[idx]); 494 } 495 496 /* 497 * If someone wanted a L2 page for I/O, allocate it now. 498 */ 499 if (iovbase != 0) { 500 #ifdef VERBOSE_INIT_ARM 501 printf(" io"); 502 #endif 503 valloc_pages(bmi, &bmi->bmi_io_l2pt, L2_TABLE_SIZE / PAGE_SIZE, 504 VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE, true); 505 add_pages(bmi, &bmi->bmi_io_l2pt); 506 } 507 508 #ifdef VERBOSE_INIT_ARM 509 printf("%s: allocating stacks\n", __func__); 510 #endif 511 512 /* Allocate stacks for all modes and CPUs */ 513 valloc_pages(bmi, &abtstack, ABT_STACK_SIZE * cpu_num, 514 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true); 515 add_pages(bmi, &abtstack); 516 valloc_pages(bmi, &fiqstack, FIQ_STACK_SIZE * cpu_num, 517 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true); 518 add_pages(bmi, &fiqstack); 519 valloc_pages(bmi, &irqstack, IRQ_STACK_SIZE * cpu_num, 520 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true); 521 add_pages(bmi, &irqstack); 522 valloc_pages(bmi, &undstack, UND_STACK_SIZE * cpu_num, 523 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true); 524 add_pages(bmi, &undstack); 525 valloc_pages(bmi, &idlestack, UPAGES * cpu_num, /* SVC32 */ 526 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true); 527 add_pages(bmi, &idlestack); 528 valloc_pages(bmi, &kernelstack, UPAGES, /* SVC32 */ 529 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true); 530 add_pages(bmi, &kernelstack); 531 532 /* Allocate the message buffer from the end of memory. */ 533 const size_t msgbuf_pgs = round_page(MSGBUFSIZE) / PAGE_SIZE; 534 valloc_pages(bmi, &msgbuf, msgbuf_pgs, 535 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, false); 536 add_pages(bmi, &msgbuf); 537 msgbufphys = msgbuf.pv_pa; 538 539 if (map_vectors_p) { 540 /* 541 * Allocate a page for the system vector page. 542 * This page will just contain the system vectors and can be 543 * shared by all processes. 544 */ 545 valloc_pages(bmi, &systempage, 1, VM_PROT_READ|VM_PROT_WRITE, 546 PTE_CACHE, true); 547 } 548 systempage.pv_va = vectors; 549 550 /* 551 * If the caller needed a few extra pages for some reason, allocate 552 * them now. 553 */ 554 #if ARM_MMU_XSCALE == 1 555 #if (ARM_NMMUS > 1) 556 if (xscale_use_minidata) 557 #endif 558 valloc_pages(bmi, extrapv, nextrapages, 559 VM_PROT_READ|VM_PROT_WRITE, 0, true); 560 #endif 561 562 /* 563 * Ok we have allocated physical pages for the primary kernel 564 * page tables and stacks. Let's just confirm that. 565 */ 566 if (kernel_l1pt.pv_va == 0 567 && (!kernel_l1pt.pv_pa || (kernel_l1pt.pv_pa & (L1_TABLE_SIZE - 1)) != 0)) 568 panic("%s: Failed to allocate or align the kernel " 569 "page directory", __func__); 570 571 572 #ifdef VERBOSE_INIT_ARM 573 printf("Creating L1 page table at 0x%08lx\n", kernel_l1pt.pv_pa); 574 #endif 575 576 /* 577 * Now we start construction of the L1 page table 578 * We start by mapping the L2 page tables into the L1. 579 * This means that we can replace L1 mappings later on if necessary 580 */ 581 vaddr_t l1pt_va = kernel_l1pt.pv_va; 582 paddr_t l1pt_pa = kernel_l1pt.pv_pa; 583 584 if (map_vectors_p) { 585 /* Map the L2 pages tables in the L1 page table */ 586 pmap_link_l2pt(l1pt_va, systempage.pv_va & -L2_S_SEGSIZE, 587 &bmi->bmi_vector_l2pt); 588 #ifdef VERBOSE_INIT_ARM 589 printf("%s: adding L2 pt (VA %#lx, PA %#lx) " 590 "for VA %#lx\n (vectors)", 591 __func__, bmi->bmi_vector_l2pt.pv_va, 592 bmi->bmi_vector_l2pt.pv_pa, systempage.pv_va); 593 #endif 594 } 595 596 const vaddr_t kernel_base = 597 KERN_PHYSTOV(bmi, bmi->bmi_kernelstart & -L2_S_SEGSIZE); 598 for (size_t idx = 0; idx < KERNEL_L2PT_KERNEL_NUM; idx++) { 599 pmap_link_l2pt(l1pt_va, kernel_base + idx * L2_S_SEGSIZE, 600 &kernel_l2pt[idx]); 601 #ifdef VERBOSE_INIT_ARM 602 printf("%s: adding L2 pt (VA %#lx, PA %#lx) for VA %#lx (kernel)\n", 603 __func__, kernel_l2pt[idx].pv_va, kernel_l2pt[idx].pv_pa, 604 kernel_base + idx * L2_S_SEGSIZE); 605 #endif 606 } 607 608 for (size_t idx = 0; idx < KERNEL_L2PT_VMDATA_NUM; idx++) { 609 pmap_link_l2pt(l1pt_va, kernel_vm_base + idx * L2_S_SEGSIZE, 610 &vmdata_l2pt[idx]); 611 #ifdef VERBOSE_INIT_ARM 612 printf("%s: adding L2 pt (VA %#lx, PA %#lx) for VA %#lx (vm)\n", 613 __func__, vmdata_l2pt[idx].pv_va, vmdata_l2pt[idx].pv_pa, 614 kernel_vm_base + idx * L2_S_SEGSIZE); 615 #endif 616 } 617 if (iovbase) { 618 pmap_link_l2pt(l1pt_va, iovbase & -L2_S_SEGSIZE, &bmi->bmi_io_l2pt); 619 #ifdef VERBOSE_INIT_ARM 620 printf("%s: adding L2 pt (VA %#lx, PA %#lx) for VA %#lx (io)\n", 621 __func__, bmi->bmi_io_l2pt.pv_va, bmi->bmi_io_l2pt.pv_pa, 622 iovbase & -L2_S_SEGSIZE); 623 #endif 624 } 625 626 /* update the top of the kernel VM */ 627 pmap_curmaxkvaddr = 628 kernel_vm_base + (KERNEL_L2PT_VMDATA_NUM * L2_S_SEGSIZE); 629 630 #ifdef VERBOSE_INIT_ARM 631 printf("Mapping kernel\n"); 632 #endif 633 634 extern char etext[], _end[]; 635 size_t totalsize = bmi->bmi_kernelend - bmi->bmi_kernelstart; 636 size_t textsize = KERN_VTOPHYS(bmi, (uintptr_t)etext) - bmi->bmi_kernelstart; 637 638 textsize = (textsize + PGOFSET) & ~PGOFSET; 639 640 /* start at offset of kernel in RAM */ 641 642 text.pv_pa = bmi->bmi_kernelstart; 643 text.pv_va = KERN_PHYSTOV(bmi, bmi->bmi_kernelstart); 644 text.pv_size = textsize; 645 text.pv_prot = VM_PROT_READ|VM_PROT_WRITE; /* XXX VM_PROT_EXECUTE */ 646 text.pv_cache = PTE_CACHE; 647 648 #ifdef VERBOSE_INIT_ARM 649 printf("%s: adding chunk for kernel text %#lx..%#lx (VA %#lx)\n", 650 __func__, text.pv_pa, text.pv_pa + text.pv_size - 1, text.pv_va); 651 #endif 652 653 add_pages(bmi, &text); 654 655 data.pv_pa = text.pv_pa + textsize; 656 data.pv_va = text.pv_va + textsize; 657 data.pv_size = totalsize - textsize; 658 data.pv_prot = VM_PROT_READ|VM_PROT_WRITE; 659 data.pv_cache = PTE_CACHE; 660 661 #ifdef VERBOSE_INIT_ARM 662 printf("%s: adding chunk for kernel data/bss %#lx..%#lx (VA %#lx)\n", 663 __func__, data.pv_pa, data.pv_pa + data.pv_size - 1, data.pv_va); 664 #endif 665 666 add_pages(bmi, &data); 667 668 #ifdef VERBOSE_INIT_ARM 669 printf("Listing Chunks\n"); 670 { 671 pv_addr_t *pv; 672 SLIST_FOREACH(pv, &bmi->bmi_chunks, pv_list) { 673 printf("%s: pv %p: chunk VA %#lx..%#lx " 674 "(PA %#lx, prot %d, cache %d)\n", 675 __func__, pv, pv->pv_va, pv->pv_va + pv->pv_size - 1, 676 pv->pv_pa, pv->pv_prot, pv->pv_cache); 677 } 678 } 679 printf("\nMapping Chunks\n"); 680 #endif 681 682 pv_addr_t cur_pv; 683 pv_addr_t *pv = SLIST_FIRST(&bmi->bmi_chunks); 684 if (!mapallmem_p || pv->pv_pa == bmi->bmi_start) { 685 cur_pv = *pv; 686 pv = SLIST_NEXT(pv, pv_list); 687 } else { 688 cur_pv.pv_va = KERNEL_BASE; 689 cur_pv.pv_pa = bmi->bmi_start; 690 cur_pv.pv_size = pv->pv_pa - bmi->bmi_start; 691 cur_pv.pv_prot = VM_PROT_READ | VM_PROT_WRITE; 692 cur_pv.pv_cache = PTE_CACHE; 693 } 694 while (pv != NULL) { 695 if (mapallmem_p) { 696 if (concat_pvaddr(&cur_pv, pv)) { 697 pv = SLIST_NEXT(pv, pv_list); 698 continue; 699 } 700 if (cur_pv.pv_pa + cur_pv.pv_size < pv->pv_pa) { 701 /* 702 * See if we can extend the current pv to emcompass the 703 * hole, and if so do it and retry the concatenation. 704 */ 705 if (cur_pv.pv_prot == (VM_PROT_READ|VM_PROT_WRITE) 706 && cur_pv.pv_cache == PTE_CACHE) { 707 cur_pv.pv_size = pv->pv_pa - cur_pv.pv_va; 708 continue; 709 } 710 711 /* 712 * We couldn't so emit the current chunk and then 713 */ 714 #ifdef VERBOSE_INIT_ARM 715 printf("%s: mapping chunk VA %#lx..%#lx " 716 "(PA %#lx, prot %d, cache %d)\n", 717 __func__, 718 cur_pv.pv_va, cur_pv.pv_va + cur_pv.pv_size - 1, 719 cur_pv.pv_pa, cur_pv.pv_prot, cur_pv.pv_cache); 720 #endif 721 pmap_map_chunk(l1pt_va, cur_pv.pv_va, cur_pv.pv_pa, 722 cur_pv.pv_size, cur_pv.pv_prot, cur_pv.pv_cache); 723 724 /* 725 * set the current chunk to the hole and try again. 726 */ 727 cur_pv.pv_pa += cur_pv.pv_size; 728 cur_pv.pv_va += cur_pv.pv_size; 729 cur_pv.pv_size = pv->pv_pa - cur_pv.pv_va; 730 cur_pv.pv_prot = VM_PROT_READ | VM_PROT_WRITE; 731 cur_pv.pv_cache = PTE_CACHE; 732 continue; 733 } 734 } 735 736 /* 737 * The new pv didn't concatenate so emit the current one 738 * and use the new pv as the current pv. 739 */ 740 #ifdef VERBOSE_INIT_ARM 741 printf("%s: mapping chunk VA %#lx..%#lx " 742 "(PA %#lx, prot %d, cache %d)\n", 743 __func__, cur_pv.pv_va, cur_pv.pv_va + cur_pv.pv_size - 1, 744 cur_pv.pv_pa, cur_pv.pv_prot, cur_pv.pv_cache); 745 #endif 746 pmap_map_chunk(l1pt_va, cur_pv.pv_va, cur_pv.pv_pa, 747 cur_pv.pv_size, cur_pv.pv_prot, cur_pv.pv_cache); 748 cur_pv = *pv; 749 pv = SLIST_NEXT(pv, pv_list); 750 } 751 752 /* 753 * If we are mapping all of memory, let's map the rest of memory. 754 */ 755 if (mapallmem_p && cur_pv.pv_pa + cur_pv.pv_size < bmi->bmi_end) { 756 if (cur_pv.pv_prot == (VM_PROT_READ | VM_PROT_WRITE) 757 && cur_pv.pv_cache == PTE_CACHE) { 758 cur_pv.pv_size = bmi->bmi_end - cur_pv.pv_pa; 759 } else { 760 #ifdef VERBOSE_INIT_ARM 761 printf("%s: mapping chunk VA %#lx..%#lx " 762 "(PA %#lx, prot %d, cache %d)\n", 763 __func__, cur_pv.pv_va, cur_pv.pv_va + cur_pv.pv_size - 1, 764 cur_pv.pv_pa, cur_pv.pv_prot, cur_pv.pv_cache); 765 #endif 766 pmap_map_chunk(l1pt_va, cur_pv.pv_va, cur_pv.pv_pa, 767 cur_pv.pv_size, cur_pv.pv_prot, cur_pv.pv_cache); 768 cur_pv.pv_pa += cur_pv.pv_size; 769 cur_pv.pv_va += cur_pv.pv_size; 770 cur_pv.pv_size = bmi->bmi_end - cur_pv.pv_pa; 771 cur_pv.pv_prot = VM_PROT_READ | VM_PROT_WRITE; 772 cur_pv.pv_cache = PTE_CACHE; 773 } 774 } 775 776 /* 777 * Now we map the final chunk. 778 */ 779 #ifdef VERBOSE_INIT_ARM 780 printf("%s: mapping last chunk VA %#lx..%#lx (PA %#lx, prot %d, cache %d)\n", 781 __func__, cur_pv.pv_va, cur_pv.pv_va + cur_pv.pv_size - 1, 782 cur_pv.pv_pa, cur_pv.pv_prot, cur_pv.pv_cache); 783 #endif 784 pmap_map_chunk(l1pt_va, cur_pv.pv_va, cur_pv.pv_pa, 785 cur_pv.pv_size, cur_pv.pv_prot, cur_pv.pv_cache); 786 787 /* 788 * Now we map the stuff that isn't directly after the kernel 789 */ 790 791 if (map_vectors_p) { 792 /* Map the vector page. */ 793 pmap_map_entry(l1pt_va, systempage.pv_va, systempage.pv_pa, 794 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); 795 } 796 797 /* Map the Mini-Data cache clean area. */ 798 #if ARM_MMU_XSCALE == 1 799 #if (ARM_NMMUS > 1) 800 if (xscale_use_minidata) 801 #endif 802 xscale_setup_minidata(l1_va, minidataclean.pv_va, 803 minidataclean.pv_pa); 804 #endif 805 806 /* 807 * Map integrated peripherals at same address in first level page 808 * table so that we can continue to use console. 809 */ 810 if (devmap) 811 pmap_devmap_bootstrap(l1pt_va, devmap); 812 813 #ifdef VERBOSE_INIT_ARM 814 /* Tell the user about where all the bits and pieces live. */ 815 printf("%22s Physical Virtual Num\n", " "); 816 printf("%22s Starting Ending Starting Ending Pages\n", " "); 817 818 static const char mem_fmt[] = 819 "%20s: 0x%08lx 0x%08lx 0x%08lx 0x%08lx %u\n"; 820 static const char mem_fmt_nov[] = 821 "%20s: 0x%08lx 0x%08lx %zu\n"; 822 823 printf(mem_fmt, "SDRAM", bmi->bmi_start, bmi->bmi_end - 1, 824 KERN_PHYSTOV(bmi, bmi->bmi_start), KERN_PHYSTOV(bmi, bmi->bmi_end - 1), 825 physmem); 826 printf(mem_fmt, "text section", 827 text.pv_pa, text.pv_pa + text.pv_size - 1, 828 text.pv_va, text.pv_va + text.pv_size - 1, 829 (int)(text.pv_size / PAGE_SIZE)); 830 printf(mem_fmt, "data section", 831 KERN_VTOPHYS(bmi, __data_start), KERN_VTOPHYS(bmi, _edata), 832 (vaddr_t)__data_start, (vaddr_t)_edata, 833 (int)((round_page((vaddr_t)_edata) 834 - trunc_page((vaddr_t)__data_start)) / PAGE_SIZE)); 835 printf(mem_fmt, "bss section", 836 KERN_VTOPHYS(bmi, __bss_start), KERN_VTOPHYS(bmi, __bss_end__), 837 (vaddr_t)__bss_start, (vaddr_t)__bss_end__, 838 (int)((round_page((vaddr_t)__bss_end__) 839 - trunc_page((vaddr_t)__bss_start)) / PAGE_SIZE)); 840 printf(mem_fmt, "L1 page directory", 841 kernel_l1pt.pv_pa, kernel_l1pt.pv_pa + L1_TABLE_SIZE - 1, 842 kernel_l1pt.pv_va, kernel_l1pt.pv_va + L1_TABLE_SIZE - 1, 843 L1_TABLE_SIZE / PAGE_SIZE); 844 printf(mem_fmt, "ABT stack (CPU 0)", 845 abtstack.pv_pa, abtstack.pv_pa + (ABT_STACK_SIZE * PAGE_SIZE) - 1, 846 abtstack.pv_va, abtstack.pv_va + (ABT_STACK_SIZE * PAGE_SIZE) - 1, 847 ABT_STACK_SIZE); 848 printf(mem_fmt, "FIQ stack (CPU 0)", 849 fiqstack.pv_pa, fiqstack.pv_pa + (FIQ_STACK_SIZE * PAGE_SIZE) - 1, 850 fiqstack.pv_va, fiqstack.pv_va + (FIQ_STACK_SIZE * PAGE_SIZE) - 1, 851 FIQ_STACK_SIZE); 852 printf(mem_fmt, "IRQ stack (CPU 0)", 853 irqstack.pv_pa, irqstack.pv_pa + (IRQ_STACK_SIZE * PAGE_SIZE) - 1, 854 irqstack.pv_va, irqstack.pv_va + (IRQ_STACK_SIZE * PAGE_SIZE) - 1, 855 IRQ_STACK_SIZE); 856 printf(mem_fmt, "UND stack (CPU 0)", 857 undstack.pv_pa, undstack.pv_pa + (UND_STACK_SIZE * PAGE_SIZE) - 1, 858 undstack.pv_va, undstack.pv_va + (UND_STACK_SIZE * PAGE_SIZE) - 1, 859 UND_STACK_SIZE); 860 printf(mem_fmt, "IDLE stack (CPU 0)", 861 idlestack.pv_pa, idlestack.pv_pa + (UPAGES * PAGE_SIZE) - 1, 862 idlestack.pv_va, idlestack.pv_va + (UPAGES * PAGE_SIZE) - 1, 863 UPAGES); 864 printf(mem_fmt, "SVC stack", 865 kernelstack.pv_pa, kernelstack.pv_pa + (UPAGES * PAGE_SIZE) - 1, 866 kernelstack.pv_va, kernelstack.pv_va + (UPAGES * PAGE_SIZE) - 1, 867 UPAGES); 868 printf(mem_fmt, "Message Buffer", 869 msgbuf.pv_pa, msgbuf.pv_pa + (msgbuf_pgs * PAGE_SIZE) - 1, 870 msgbuf.pv_va, msgbuf.pv_va + (msgbuf_pgs * PAGE_SIZE) - 1, 871 (int)msgbuf_pgs); 872 if (map_vectors_p) { 873 printf(mem_fmt, "Exception Vectors", 874 systempage.pv_pa, systempage.pv_pa + PAGE_SIZE - 1, 875 systempage.pv_va, systempage.pv_va + PAGE_SIZE - 1, 876 1); 877 } 878 for (size_t i = 0; i < bmi->bmi_nfreeblocks; i++) { 879 pv = &bmi->bmi_freeblocks[i]; 880 881 printf(mem_fmt_nov, "Free Memory", 882 pv->pv_pa, pv->pv_pa + pv->pv_size - 1, 883 pv->pv_size / PAGE_SIZE); 884 } 885 #endif 886 /* 887 * Now we have the real page tables in place so we can switch to them. 888 * Once this is done we will be running with the REAL kernel page 889 * tables. 890 */ 891 892 #if defined(VERBOSE_INIT_ARM) && 0 893 printf("TTBR0=%#x", armreg_ttbr_read()); 894 #ifdef _ARM_ARCH_6 895 printf(" TTBR1=%#x TTBCR=%#x", 896 armreg_ttbr1_read(), armreg_ttbcr_read()); 897 #endif 898 printf("\n"); 899 #endif 900 901 /* Switch tables */ 902 #ifdef VERBOSE_INIT_ARM 903 printf("switching to new L1 page table @%#lx...", l1pt_pa); 904 #endif 905 906 cpu_domains((DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL*2)) | DOMAIN_CLIENT); 907 cpu_idcache_wbinv_all(); 908 #ifdef ARM_MMU_EXTENDED 909 cpu_setttb(l1pt_pa, KERNEL_PID); 910 #else 911 cpu_setttb(l1pt_pa, true); 912 #endif 913 cpu_tlb_flushID(); 914 cpu_domains(DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL*2)); 915 916 #ifdef VERBOSE_INIT_ARM 917 printf("TTBR0=%#x OK\n", armreg_ttbr_read()); 918 #endif 919 } 920
Indexes created Sun Sep 28 07:09:56 GMT 2025