arm32_kvminit.c revision 1.47 1 /* $NetBSD: arm32_kvminit.c,v 1.47 2018/10/20 06:35:34 skrll Exp $ */
2
3 /*
4 * Copyright (c) 2002, 2003, 2005 Genetec Corporation. All rights reserved.
5 * Written by Hiroyuki Bessho for Genetec Corporation.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. The name of Genetec Corporation may not be used to endorse or
16 * promote products derived from this software without specific prior
17 * written permission.
18 *
19 * THIS SOFTWARE IS PROVIDED BY GENETEC CORPORATION ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENETEC CORPORATION
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 *
31 * Copyright (c) 2001 Wasabi Systems, Inc.
32 * All rights reserved.
33 *
34 * Written by Jason R. Thorpe for Wasabi Systems, Inc.
35 *
36 * Redistribution and use in source and binary forms, with or without
37 * modification, are permitted provided that the following conditions
38 * are met:
39 * 1. Redistributions of source code must retain the above copyright
40 * notice, this list of conditions and the following disclaimer.
41 * 2. Redistributions in binary form must reproduce the above copyright
42 * notice, this list of conditions and the following disclaimer in the
43 * documentation and/or other materials provided with the distribution.
44 * 3. All advertising materials mentioning features or use of this software
45 * must display the following acknowledgement:
46 * This product includes software developed for the NetBSD Project by
47 * Wasabi Systems, Inc.
48 * 4. The name of Wasabi Systems, Inc. may not be used to endorse
49 * or promote products derived from this software without specific prior
50 * written permission.
51 *
52 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
53 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
54 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
55 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC
56 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
57 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
58 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
59 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
60 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
61 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
62 * POSSIBILITY OF SUCH DAMAGE.
63 *
64 * Copyright (c) 1997,1998 Mark Brinicombe.
65 * Copyright (c) 1997,1998 Causality Limited.
66 * All rights reserved.
67 *
68 * Redistribution and use in source and binary forms, with or without
69 * modification, are permitted provided that the following conditions
70 * are met:
71 * 1. Redistributions of source code must retain the above copyright
72 * notice, this list of conditions and the following disclaimer.
73 * 2. Redistributions in binary form must reproduce the above copyright
74 * notice, this list of conditions and the following disclaimer in the
75 * documentation and/or other materials provided with the distribution.
76 * 3. All advertising materials mentioning features or use of this software
77 * must display the following acknowledgement:
78 * This product includes software developed by Mark Brinicombe
79 * for the NetBSD Project.
80 * 4. The name of the company nor the name of the author may be used to
81 * endorse or promote products derived from this software without specific
82 * prior written permission.
83 *
84 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
85 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
86 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
87 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
88 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
89 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
90 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
91 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
92 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
93 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
94 * SUCH DAMAGE.
95 *
96 * Copyright (c) 2007 Microsoft
97 * All rights reserved.
98 *
99 * Redistribution and use in source and binary forms, with or without
100 * modification, are permitted provided that the following conditions
101 * are met:
102 * 1. Redistributions of source code must retain the above copyright
103 * notice, this list of conditions and the following disclaimer.
104 * 2. Redistributions in binary form must reproduce the above copyright
105 * notice, this list of conditions and the following disclaimer in the
106 * documentation and/or other materials provided with the distribution.
107 * 3. All advertising materials mentioning features or use of this software
108 * must display the following acknowledgement:
109 * This product includes software developed by Microsoft
110 *
111 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
112 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
113 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
114 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTERS BE LIABLE FOR ANY DIRECT,
115 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
116 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
117 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
118 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
119 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
120 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
121 * SUCH DAMAGE.
122 */
123
124 #include "opt_arm_debug.h"
125 #include "opt_arm_start.h"
126 #include "opt_fdt.h"
127 #include "opt_multiprocessor.h"
128
129 #include <sys/cdefs.h>
130 __KERNEL_RCSID(0, "$NetBSD: arm32_kvminit.c,v 1.47 2018/10/20 06:35:34 skrll Exp $");
131
132 #include <sys/param.h>
133 #include <sys/device.h>
134 #include <sys/kernel.h>
135 #include <sys/reboot.h>
136 #include <sys/bus.h>
137
138 #include <dev/cons.h>
139
140 #include <uvm/uvm_extern.h>
141
142 #include <arm/locore.h>
143 #include <arm/db_machdep.h>
144 #include <arm/undefined.h>
145 #include <arm/bootconfig.h>
146 #include <arm/arm32/machdep.h>
147
148 #if defined(FDT)
149 #include <arch/evbarm/fdt/platform.h>
150 #include <arm/fdt/arm_fdtvar.h>
151 #endif
152
153 #ifdef MULTIPROCESSOR
154 #ifndef __HAVE_CPU_UAREA_ALLOC_IDLELWP
155 #error __HAVE_CPU_UAREA_ALLOC_IDLELWP required to not waste pages for idlestack
156 #endif
157 #endif
158
159 #ifdef VERBOSE_INIT_ARM
160 #define VPRINTF(...) printf(__VA_ARGS__)
161 #else
162 #define VPRINTF(...) __nothing
163 #endif
164
165 struct bootmem_info bootmem_info;
166
167 extern void *msgbufaddr;
168 paddr_t msgbufphys;
169 paddr_t physical_start;
170 paddr_t physical_end;
171
172 extern char etext[];
173 extern char __data_start[], _edata[];
174 extern char __bss_start[], __bss_end__[];
175 extern char _end[];
176
177 /* Page tables for mapping kernel VM */
178 #define KERNEL_L2PT_VMDATA_NUM 8 /* start with 32MB of KVM */
179
180 u_long kern_vtopdiff __attribute__((__section__(".data")));
181
182 void
183 arm32_bootmem_init(paddr_t memstart, psize_t memsize, vsize_t kernelstart)
184 {
185 struct bootmem_info * const bmi = &bootmem_info;
186 pv_addr_t *pv = bmi->bmi_freeblocks;
187
188 /*
189 * FDT/generic start fills in kern_vtopdiff early
190 */
191 #if defined(__HAVE_GENERIC_START)
192 extern char KERNEL_BASE_virt[];
193 extern char ARM_BOOTSTRAP_LxPT[];
194
195 VPRINTF("%s: kern_vtopdiff=%#lx\n", __func__, kern_vtopdiff);
196
197 vaddr_t kstartva = trunc_page((vaddr_t)KERNEL_BASE_virt);
198 vaddr_t kendva = round_page((vaddr_t)ARM_BOOTSTRAP_LxPT + L1_TABLE_SIZE);
199
200 kernelstart = KERN_VTOPHYS(kstartva);
201
202 VPRINTF("%s: kstartva=%#lx, kernelstart=%#lx\n", __func__, kstartva, kernelstart);
203 #else
204 vaddr_t kendva = round_page((vaddr_t)_end);
205
206 #if defined(KERNEL_BASE_VOFFSET)
207 kern_vtopdiff = KERNEL_BASE_VOFFSET;
208 #else
209 KASSERT(memstart == kernelstart);
210 kern_vtopdiff = KERNEL_BASE + memstart;
211 #endif
212 #endif
213 paddr_t kernelend = KERN_VTOPHYS(kendva);
214
215 VPRINTF("%s: memstart=%#lx, memsize=%#lx\n",
216 __func__, memstart, memsize);
217 VPRINTF("%s: kernelstart=%#lx, kernelend=%#lx\n", __func__,
218 kernelstart, kernelend);
219
220 physical_start = bmi->bmi_start = memstart;
221 physical_end = bmi->bmi_end = memstart + memsize;
222 #ifndef ARM_HAS_LPAE
223 if (physical_end == 0) {
224 physical_end = -PAGE_SIZE;
225 memsize -= PAGE_SIZE;
226 bmi->bmi_end -= PAGE_SIZE;
227 VPRINTF("%s: memsize shrunk by a page to avoid ending at 4GB\n",
228 __func__);
229 }
230 #endif
231 physmem = memsize / PAGE_SIZE;
232
233 /*
234 * Let's record where the kernel lives.
235 */
236
237 bmi->bmi_kernelstart = kernelstart;
238 bmi->bmi_kernelend = kernelend;
239
240 #if defined(FDT)
241 fdt_add_reserved_memory_range(bmi->bmi_kernelstart,
242 bmi->bmi_kernelend - bmi->bmi_kernelstart);
243 #endif
244
245 VPRINTF("%s: kernel phys start %#lx end %#lx\n", __func__, kernelstart,
246 kernelend);
247
248 #if 0
249 // XXX Makes RPI abort
250 KASSERT((kernelstart & (L2_S_SEGSIZE - 1)) == 0);
251 #endif
252 /*
253 * Now the rest of the free memory must be after the kernel.
254 */
255 pv->pv_pa = bmi->bmi_kernelend;
256 pv->pv_va = KERN_PHYSTOV(pv->pv_pa);
257 pv->pv_size = bmi->bmi_end - bmi->bmi_kernelend;
258 bmi->bmi_freepages += pv->pv_size / PAGE_SIZE;
259 VPRINTF("%s: adding %lu free pages: [%#lx..%#lx] (VA %#lx)\n",
260 __func__, pv->pv_size / PAGE_SIZE, pv->pv_pa,
261 pv->pv_pa + pv->pv_size - 1, pv->pv_va);
262 pv++;
263
264 #if !defined(__HAVE_GENERIC_START)
265 /*
266 * Add a free block for any memory before the kernel.
267 */
268 if (bmi->bmi_start < bmi->bmi_kernelstart) {
269 pv->pv_pa = bmi->bmi_start;
270 pv->pv_va = KERN_PHYSTOV(pv->pv_pa);
271 pv->pv_size = bmi->bmi_kernelstart - pv->pv_pa;
272 bmi->bmi_freepages += pv->pv_size / PAGE_SIZE;
273 VPRINTF("%s: adding %lu free pages: [%#lx..%#lx] (VA %#lx)\n",
274 __func__, pv->pv_size / PAGE_SIZE, pv->pv_pa,
275 pv->pv_pa + pv->pv_size - 1, pv->pv_va);
276 pv++;
277 }
278 #endif
279
280 bmi->bmi_nfreeblocks = pv - bmi->bmi_freeblocks;
281
282 SLIST_INIT(&bmi->bmi_freechunks);
283 SLIST_INIT(&bmi->bmi_chunks);
284 }
285
286 static bool
287 concat_pvaddr(pv_addr_t *acc_pv, pv_addr_t *pv)
288 {
289 if (acc_pv->pv_pa + acc_pv->pv_size == pv->pv_pa
290 && acc_pv->pv_va + acc_pv->pv_size == pv->pv_va
291 && acc_pv->pv_prot == pv->pv_prot
292 && acc_pv->pv_cache == pv->pv_cache) {
293 #if 0
294 VPRINTF("%s: appending pv %p (%#lx..%#lx) to %#lx..%#lx\n",
295 __func__, pv, pv->pv_pa, pv->pv_pa + pv->pv_size,
296 acc_pv->pv_pa, acc_pv->pv_pa + acc_pv->pv_size);
297 #endif
298 acc_pv->pv_size += pv->pv_size;
299 return true;
300 }
301
302 return false;
303 }
304
305 static void
306 add_pages(struct bootmem_info *bmi, pv_addr_t *pv)
307 {
308 pv_addr_t **pvp = &SLIST_FIRST(&bmi->bmi_chunks);
309 while ((*pvp) != NULL && (*pvp)->pv_va <= pv->pv_va) {
310 pv_addr_t * const pv0 = (*pvp);
311 KASSERT(SLIST_NEXT(pv0, pv_list) == NULL || pv0->pv_pa < SLIST_NEXT(pv0, pv_list)->pv_pa);
312 if (concat_pvaddr(pv0, pv)) {
313 VPRINTF("%s: %s pv %p (%#lx..%#lx) to %#lx..%#lx\n",
314 __func__, "appending", pv,
315 pv->pv_pa, pv->pv_pa + pv->pv_size - 1,
316 pv0->pv_pa, pv0->pv_pa + pv0->pv_size - pv->pv_size - 1);
317 pv = SLIST_NEXT(pv0, pv_list);
318 if (pv != NULL && concat_pvaddr(pv0, pv)) {
319 VPRINTF("%s: %s pv %p (%#lx..%#lx) to %#lx..%#lx\n",
320 __func__, "merging", pv,
321 pv->pv_pa, pv->pv_pa + pv->pv_size - 1,
322 pv0->pv_pa,
323 pv0->pv_pa + pv0->pv_size - pv->pv_size - 1);
324 SLIST_REMOVE_AFTER(pv0, pv_list);
325 SLIST_INSERT_HEAD(&bmi->bmi_freechunks, pv, pv_list);
326 }
327 return;
328 }
329 KASSERT(pv->pv_va != (*pvp)->pv_va);
330 pvp = &SLIST_NEXT(*pvp, pv_list);
331 }
332 KASSERT((*pvp) == NULL || pv->pv_va < (*pvp)->pv_va);
333 pv_addr_t * const new_pv = SLIST_FIRST(&bmi->bmi_freechunks);
334 KASSERT(new_pv != NULL);
335 SLIST_REMOVE_HEAD(&bmi->bmi_freechunks, pv_list);
336 *new_pv = *pv;
337 SLIST_NEXT(new_pv, pv_list) = *pvp;
338 (*pvp) = new_pv;
339
340 VPRINTF("%s: adding pv %p (pa %#lx, va %#lx, %lu pages) ",
341 __func__, new_pv, new_pv->pv_pa, new_pv->pv_va,
342 new_pv->pv_size / PAGE_SIZE);
343 if (SLIST_NEXT(new_pv, pv_list)) {
344 VPRINTF("before pa %#lx\n", SLIST_NEXT(new_pv, pv_list)->pv_pa);
345 } else {
346 VPRINTF("at tail\n");
347 }
348 }
349
350 static void
351 valloc_pages(struct bootmem_info *bmi, pv_addr_t *pv, size_t npages,
352 int prot, int cache, bool zero_p)
353 {
354 size_t nbytes = npages * PAGE_SIZE;
355 pv_addr_t *free_pv = bmi->bmi_freeblocks;
356 size_t free_idx = 0;
357 static bool l1pt_found;
358
359 KASSERT(npages > 0);
360
361 /*
362 * If we haven't allocated the kernel L1 page table and we are aligned
363 * at a L1 table boundary, alloc the memory for it.
364 */
365 if (!l1pt_found
366 && (free_pv->pv_pa & (L1_TABLE_SIZE - 1)) == 0
367 && free_pv->pv_size >= L1_TABLE_SIZE) {
368 l1pt_found = true;
369 VPRINTF(" l1pt");
370
371 valloc_pages(bmi, &kernel_l1pt, L1_TABLE_SIZE / PAGE_SIZE,
372 VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE, true);
373 add_pages(bmi, &kernel_l1pt);
374 }
375
376 while (nbytes > free_pv->pv_size) {
377 free_pv++;
378 free_idx++;
379 if (free_idx == bmi->bmi_nfreeblocks) {
380 panic("%s: could not allocate %zu bytes",
381 __func__, nbytes);
382 }
383 }
384
385 /*
386 * As we allocate the memory, make sure that we don't walk over
387 * our current first level translation table.
388 */
389 KASSERT((armreg_ttbr_read() & ~(L1_TABLE_SIZE - 1)) != free_pv->pv_pa);
390
391 #if defined(FDT)
392 fdt_add_reserved_memory_range(free_pv->pv_pa, nbytes);
393 #endif
394 pv->pv_pa = free_pv->pv_pa;
395 pv->pv_va = free_pv->pv_va;
396 pv->pv_size = nbytes;
397 pv->pv_prot = prot;
398 pv->pv_cache = cache;
399
400 /*
401 * If PTE_PAGETABLE uses the same cache modes as PTE_CACHE
402 * just use PTE_CACHE.
403 */
404 if (cache == PTE_PAGETABLE
405 && pte_l1_s_cache_mode == pte_l1_s_cache_mode_pt
406 && pte_l2_l_cache_mode == pte_l2_l_cache_mode_pt
407 && pte_l2_s_cache_mode == pte_l2_s_cache_mode_pt)
408 pv->pv_cache = PTE_CACHE;
409
410 free_pv->pv_pa += nbytes;
411 free_pv->pv_va += nbytes;
412 free_pv->pv_size -= nbytes;
413 if (free_pv->pv_size == 0) {
414 --bmi->bmi_nfreeblocks;
415 for (; free_idx < bmi->bmi_nfreeblocks; free_idx++) {
416 free_pv[0] = free_pv[1];
417 }
418 }
419
420 bmi->bmi_freepages -= npages;
421
422 if (zero_p)
423 memset((void *)pv->pv_va, 0, nbytes);
424 }
425
426 void
427 arm32_kernel_vm_init(vaddr_t kernel_vm_base, vaddr_t vectors, vaddr_t iovbase,
428 const struct pmap_devmap *devmap, bool mapallmem_p)
429 {
430 struct bootmem_info * const bmi = &bootmem_info;
431 #ifdef MULTIPROCESSOR
432 const size_t cpu_num = arm_cpu_max;
433 #else
434 const size_t cpu_num = 1;
435 #endif
436
437 #ifdef ARM_HAS_VBAR
438 const bool map_vectors_p = false;
439 #elif defined(CPU_ARMV7) || defined(CPU_ARM11)
440 const bool map_vectors_p = vectors == ARM_VECTORS_HIGH
441 || (armreg_pfr1_read() & ARM_PFR1_SEC_MASK) == 0;
442 #else
443 const bool map_vectors_p = true;
444 #endif
445
446 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS
447 KASSERT(mapallmem_p);
448 #ifdef ARM_MMU_EXTENDED
449 /*
450 * The direct map VA space ends at the start of the kernel VM space.
451 */
452 pmap_directlimit = kernel_vm_base;
453 #else
454 KASSERT(kernel_vm_base - KERNEL_BASE >= physical_end - physical_start);
455 #endif /* ARM_MMU_EXTENDED */
456 #endif /* __HAVE_MM_MD_DIRECT_MAPPED_PHYS */
457
458 /*
459 * Calculate the number of L2 pages needed for mapping the
460 * kernel + data + stuff. Assume 2 L2 pages for kernel, 1 for vectors,
461 * and 1 for IO
462 */
463 size_t kernel_size = bmi->bmi_kernelend;
464 kernel_size -= (bmi->bmi_kernelstart & -L2_S_SEGSIZE);
465 kernel_size += L1_TABLE_SIZE_REAL;
466 kernel_size += PAGE_SIZE * KERNEL_L2PT_VMDATA_NUM;
467 if (map_vectors_p) {
468 kernel_size += PAGE_SIZE; /* L2PT for VECTORS */
469 }
470 if (iovbase) {
471 kernel_size += PAGE_SIZE; /* L2PT for IO */
472 }
473 kernel_size +=
474 cpu_num * (ABT_STACK_SIZE + FIQ_STACK_SIZE + IRQ_STACK_SIZE
475 + UND_STACK_SIZE + UPAGES) * PAGE_SIZE;
476 kernel_size += round_page(MSGBUFSIZE);
477 kernel_size += 0x10000; /* slop */
478 if (!mapallmem_p) {
479 kernel_size += PAGE_SIZE
480 * ((kernel_size + L2_S_SEGSIZE - 1) / L2_S_SEGSIZE);
481 }
482 kernel_size = round_page(kernel_size);
483
484 /*
485 * Now we know how many L2 pages it will take.
486 */
487 const size_t KERNEL_L2PT_KERNEL_NUM =
488 round_page(kernel_size + L2_S_SEGSIZE - 1) / L2_S_SEGSIZE;
489
490 VPRINTF("%s: %zu L2 pages are needed to map %#zx kernel bytes\n",
491 __func__, KERNEL_L2PT_KERNEL_NUM, kernel_size);
492
493 KASSERT(KERNEL_L2PT_KERNEL_NUM + KERNEL_L2PT_VMDATA_NUM < __arraycount(bmi->bmi_l2pts));
494 pv_addr_t * const kernel_l2pt = bmi->bmi_l2pts;
495 pv_addr_t * const vmdata_l2pt = kernel_l2pt + KERNEL_L2PT_KERNEL_NUM;
496 pv_addr_t msgbuf;
497 pv_addr_t text;
498 pv_addr_t data;
499 pv_addr_t chunks[KERNEL_L2PT_KERNEL_NUM+KERNEL_L2PT_VMDATA_NUM+11];
500 #if ARM_MMU_XSCALE == 1
501 pv_addr_t minidataclean;
502 #endif
503
504 /*
505 * We need to allocate some fixed page tables to get the kernel going.
506 *
507 * We are going to allocate our bootstrap pages from the beginning of
508 * the free space that we just calculated. We allocate one page
509 * directory and a number of page tables and store the physical
510 * addresses in the bmi_l2pts array in bootmem_info.
511 *
512 * The kernel page directory must be on a 16K boundary. The page
513 * tables must be on 4K boundaries. What we do is allocate the
514 * page directory on the first 16K boundary that we encounter, and
515 * the page tables on 4K boundaries otherwise. Since we allocate
516 * at least 3 L2 page tables, we are guaranteed to encounter at
517 * least one 16K aligned region.
518 */
519
520 VPRINTF("%s: allocating page tables for", __func__);
521 for (size_t i = 0; i < __arraycount(chunks); i++) {
522 SLIST_INSERT_HEAD(&bmi->bmi_freechunks, &chunks[i], pv_list);
523 }
524
525 kernel_l1pt.pv_pa = 0;
526 kernel_l1pt.pv_va = 0;
527
528 /*
529 * Allocate the L2 pages, but if we get to a page that is aligned for
530 * an L1 page table, we will allocate the pages for it first and then
531 * allocate the L2 page.
532 */
533
534 if (map_vectors_p) {
535 /*
536 * First allocate L2 page for the vectors.
537 */
538 VPRINTF(" vector");
539 valloc_pages(bmi, &bmi->bmi_vector_l2pt, 1,
540 VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE, true);
541 add_pages(bmi, &bmi->bmi_vector_l2pt);
542 }
543
544 /*
545 * Now allocate L2 pages for the kernel
546 */
547 VPRINTF(" kernel");
548 for (size_t idx = 0; idx < KERNEL_L2PT_KERNEL_NUM; ++idx) {
549 valloc_pages(bmi, &kernel_l2pt[idx], 1,
550 VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE, true);
551 add_pages(bmi, &kernel_l2pt[idx]);
552 }
553
554 /*
555 * Now allocate L2 pages for the initial kernel VA space.
556 */
557 VPRINTF(" vm");
558 for (size_t idx = 0; idx < KERNEL_L2PT_VMDATA_NUM; ++idx) {
559 valloc_pages(bmi, &vmdata_l2pt[idx], 1,
560 VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE, true);
561 add_pages(bmi, &vmdata_l2pt[idx]);
562 }
563
564 /*
565 * If someone wanted a L2 page for I/O, allocate it now.
566 */
567 if (iovbase) {
568 VPRINTF(" io");
569 valloc_pages(bmi, &bmi->bmi_io_l2pt, 1,
570 VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE, true);
571 add_pages(bmi, &bmi->bmi_io_l2pt);
572 }
573
574 VPRINTF("%s: allocating stacks\n", __func__);
575
576 /* Allocate stacks for all modes and CPUs */
577 valloc_pages(bmi, &abtstack, ABT_STACK_SIZE * cpu_num,
578 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true);
579 add_pages(bmi, &abtstack);
580 valloc_pages(bmi, &fiqstack, FIQ_STACK_SIZE * cpu_num,
581 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true);
582 add_pages(bmi, &fiqstack);
583 valloc_pages(bmi, &irqstack, IRQ_STACK_SIZE * cpu_num,
584 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true);
585 add_pages(bmi, &irqstack);
586 valloc_pages(bmi, &undstack, UND_STACK_SIZE * cpu_num,
587 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true);
588 add_pages(bmi, &undstack);
589 valloc_pages(bmi, &idlestack, UPAGES * cpu_num, /* SVC32 */
590 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true);
591 add_pages(bmi, &idlestack);
592 valloc_pages(bmi, &kernelstack, UPAGES, /* SVC32 */
593 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true);
594 add_pages(bmi, &kernelstack);
595
596 /* Allocate the message buffer from the end of memory. */
597 const size_t msgbuf_pgs = round_page(MSGBUFSIZE) / PAGE_SIZE;
598 valloc_pages(bmi, &msgbuf, msgbuf_pgs,
599 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, false);
600 add_pages(bmi, &msgbuf);
601 msgbufphys = msgbuf.pv_pa;
602 msgbufaddr = (void *)msgbuf.pv_va;
603
604 if (map_vectors_p) {
605 /*
606 * Allocate a page for the system vector page.
607 * This page will just contain the system vectors and can be
608 * shared by all processes.
609 */
610 VPRINTF(" vector");
611
612 valloc_pages(bmi, &systempage, 1, VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE,
613 PTE_CACHE, true);
614 }
615 systempage.pv_va = vectors;
616
617 /*
618 * If the caller needed a few extra pages for some reason, allocate
619 * them now.
620 */
621 #if ARM_MMU_XSCALE == 1
622 #if (ARM_NMMUS > 1)
623 if (xscale_use_minidata)
624 #endif
625 valloc_pages(bmi, &minidataclean, 1,
626 VM_PROT_READ|VM_PROT_WRITE, 0, true);
627 #endif
628
629 /*
630 * Ok we have allocated physical pages for the primary kernel
631 * page tables and stacks. Let's just confirm that.
632 */
633 if (kernel_l1pt.pv_va == 0
634 && (!kernel_l1pt.pv_pa || (kernel_l1pt.pv_pa & (L1_TABLE_SIZE - 1)) != 0))
635 panic("%s: Failed to allocate or align the kernel "
636 "page directory", __func__);
637
638 VPRINTF("Creating L1 page table at 0x%08lx/0x%08lx\n",
639 kernel_l1pt.pv_va, kernel_l1pt.pv_pa);
640
641 /*
642 * Now we start construction of the L1 page table
643 * We start by mapping the L2 page tables into the L1.
644 * This means that we can replace L1 mappings later on if necessary
645 */
646 vaddr_t l1pt_va = kernel_l1pt.pv_va;
647 paddr_t l1pt_pa = kernel_l1pt.pv_pa;
648
649 if (map_vectors_p) {
650 /* Map the L2 pages tables in the L1 page table */
651 pmap_link_l2pt(l1pt_va, systempage.pv_va & -L2_S_SEGSIZE,
652 &bmi->bmi_vector_l2pt);
653 VPRINTF("%s: adding L2 pt (VA %#lx, PA %#lx) "
654 "for VA %#lx\n (vectors)",
655 __func__, bmi->bmi_vector_l2pt.pv_va,
656 bmi->bmi_vector_l2pt.pv_pa, systempage.pv_va);
657 }
658
659 /*
660 * This enforces a alignment requirement of L2_S_SEGSIZE for kernel
661 * start PA
662 */
663 const vaddr_t kernel_base =
664 KERN_PHYSTOV(bmi->bmi_kernelstart & -L2_S_SEGSIZE);
665
666 VPRINTF("%s: kernel_base %lx KERNEL_L2PT_KERNEL_NUM %zu\n", __func__,
667 kernel_base, KERNEL_L2PT_KERNEL_NUM);
668
669 for (size_t idx = 0; idx < KERNEL_L2PT_KERNEL_NUM; idx++) {
670 pmap_link_l2pt(l1pt_va, kernel_base + idx * L2_S_SEGSIZE,
671 &kernel_l2pt[idx]);
672 VPRINTF("%s: adding L2 pt (VA %#lx, PA %#lx) for VA %#lx (kernel)\n",
673 __func__, kernel_l2pt[idx].pv_va,
674 kernel_l2pt[idx].pv_pa, kernel_base + idx * L2_S_SEGSIZE);
675 }
676
677 VPRINTF("%s: kernel_vm_base %lx KERNEL_L2PT_KERNEL_NUM %zu\n", __func__,
678 kernel_vm_base, KERNEL_L2PT_KERNEL_NUM);
679
680 for (size_t idx = 0; idx < KERNEL_L2PT_VMDATA_NUM; idx++) {
681 pmap_link_l2pt(l1pt_va, kernel_vm_base + idx * L2_S_SEGSIZE,
682 &vmdata_l2pt[idx]);
683 VPRINTF("%s: adding L2 pt (VA %#lx, PA %#lx) for VA %#lx (vm)\n",
684 __func__, vmdata_l2pt[idx].pv_va, vmdata_l2pt[idx].pv_pa,
685 kernel_vm_base + idx * L2_S_SEGSIZE);
686 }
687 if (iovbase) {
688 pmap_link_l2pt(l1pt_va, iovbase & -L2_S_SEGSIZE, &bmi->bmi_io_l2pt);
689 VPRINTF("%s: adding L2 pt (VA %#lx, PA %#lx) for VA %#lx (io)\n",
690 __func__, bmi->bmi_io_l2pt.pv_va, bmi->bmi_io_l2pt.pv_pa,
691 iovbase & -L2_S_SEGSIZE);
692 }
693
694 /* update the top of the kernel VM */
695 pmap_curmaxkvaddr =
696 kernel_vm_base + (KERNEL_L2PT_VMDATA_NUM * L2_S_SEGSIZE);
697
698 // This could be done earlier and then the kernel data and pages
699 // allocated above would get merged (concatentated)
700
701 VPRINTF("Mapping kernel\n");
702
703 extern char etext[];
704 size_t totalsize = bmi->bmi_kernelend - bmi->bmi_kernelstart;
705 size_t textsize = KERN_VTOPHYS((uintptr_t)etext) - bmi->bmi_kernelstart;
706
707 textsize = (textsize + PGOFSET) & ~PGOFSET;
708
709 /* start at offset of kernel in RAM */
710
711 text.pv_pa = bmi->bmi_kernelstart;
712 text.pv_va = KERN_PHYSTOV(bmi->bmi_kernelstart);
713 text.pv_size = textsize;
714 text.pv_prot = VM_PROT_READ | VM_PROT_EXECUTE;
715 text.pv_cache = PTE_CACHE;
716
717 VPRINTF("%s: adding chunk for kernel text %#lx..%#lx (VA %#lx)\n",
718 __func__, text.pv_pa, text.pv_pa + text.pv_size - 1, text.pv_va);
719
720 add_pages(bmi, &text);
721
722 data.pv_pa = text.pv_pa + textsize;
723 data.pv_va = text.pv_va + textsize;
724 data.pv_size = totalsize - textsize;
725 data.pv_prot = VM_PROT_READ|VM_PROT_WRITE;
726 data.pv_cache = PTE_CACHE;
727
728 VPRINTF("%s: adding chunk for kernel data/bss %#lx..%#lx (VA %#lx)\n",
729 __func__, data.pv_pa, data.pv_pa + data.pv_size - 1, data.pv_va);
730
731 add_pages(bmi, &data);
732
733 VPRINTF("Listing Chunks\n");
734
735 pv_addr_t *lpv;
736 SLIST_FOREACH(lpv, &bmi->bmi_chunks, pv_list) {
737 VPRINTF("%s: pv %p: chunk VA %#lx..%#lx "
738 "(PA %#lx, prot %d, cache %d)\n",
739 __func__, lpv, lpv->pv_va, lpv->pv_va + lpv->pv_size - 1,
740 lpv->pv_pa, lpv->pv_prot, lpv->pv_cache);
741 }
742 VPRINTF("\nMapping Chunks\n");
743
744 pv_addr_t cur_pv;
745 pv_addr_t *pv = SLIST_FIRST(&bmi->bmi_chunks);
746 if (!mapallmem_p || pv->pv_pa == bmi->bmi_start) {
747 cur_pv = *pv;
748 KASSERTMSG(cur_pv.pv_va >= KERNEL_BASE, "%#lx", cur_pv.pv_va);
749 pv = SLIST_NEXT(pv, pv_list);
750 } else {
751 cur_pv.pv_va = KERNEL_BASE;
752 cur_pv.pv_pa = KERN_VTOPHYS(cur_pv.pv_va);
753 cur_pv.pv_size = pv->pv_pa - cur_pv.pv_pa;
754 cur_pv.pv_prot = VM_PROT_READ | VM_PROT_WRITE;
755 cur_pv.pv_cache = PTE_CACHE;
756 }
757 while (pv != NULL) {
758 if (mapallmem_p) {
759 if (concat_pvaddr(&cur_pv, pv)) {
760 pv = SLIST_NEXT(pv, pv_list);
761 continue;
762 }
763 if (cur_pv.pv_pa + cur_pv.pv_size < pv->pv_pa) {
764 /*
765 * See if we can extend the current pv to emcompass the
766 * hole, and if so do it and retry the concatenation.
767 */
768 if (cur_pv.pv_prot == (VM_PROT_READ|VM_PROT_WRITE)
769 && cur_pv.pv_cache == PTE_CACHE) {
770 cur_pv.pv_size = pv->pv_pa - cur_pv.pv_va;
771 continue;
772 }
773
774 /*
775 * We couldn't so emit the current chunk and then
776 */
777 VPRINTF("%s: mapping chunk VA %#lx..%#lx "
778 "(PA %#lx, prot %d, cache %d)\n",
779 __func__,
780 cur_pv.pv_va, cur_pv.pv_va + cur_pv.pv_size - 1,
781 cur_pv.pv_pa, cur_pv.pv_prot, cur_pv.pv_cache);
782 pmap_map_chunk(l1pt_va, cur_pv.pv_va, cur_pv.pv_pa,
783 cur_pv.pv_size, cur_pv.pv_prot, cur_pv.pv_cache);
784
785 /*
786 * set the current chunk to the hole and try again.
787 */
788 cur_pv.pv_pa += cur_pv.pv_size;
789 cur_pv.pv_va += cur_pv.pv_size;
790 cur_pv.pv_size = pv->pv_pa - cur_pv.pv_va;
791 cur_pv.pv_prot = VM_PROT_READ | VM_PROT_WRITE;
792 cur_pv.pv_cache = PTE_CACHE;
793 continue;
794 }
795 }
796
797 /*
798 * The new pv didn't concatenate so emit the current one
799 * and use the new pv as the current pv.
800 */
801 VPRINTF("%s: mapping chunk VA %#lx..%#lx "
802 "(PA %#lx, prot %d, cache %d)\n",
803 __func__, cur_pv.pv_va, cur_pv.pv_va + cur_pv.pv_size - 1,
804 cur_pv.pv_pa, cur_pv.pv_prot, cur_pv.pv_cache);
805 pmap_map_chunk(l1pt_va, cur_pv.pv_va, cur_pv.pv_pa,
806 cur_pv.pv_size, cur_pv.pv_prot, cur_pv.pv_cache);
807 cur_pv = *pv;
808 pv = SLIST_NEXT(pv, pv_list);
809 }
810
811 /*
812 * If we are mapping all of memory, let's map the rest of memory.
813 */
814 if (mapallmem_p && cur_pv.pv_pa + cur_pv.pv_size < bmi->bmi_end) {
815 if (cur_pv.pv_prot == (VM_PROT_READ | VM_PROT_WRITE)
816 && cur_pv.pv_cache == PTE_CACHE) {
817 cur_pv.pv_size = bmi->bmi_end - cur_pv.pv_pa;
818 } else {
819 KASSERTMSG(cur_pv.pv_va + cur_pv.pv_size <= kernel_vm_base,
820 "%#lx >= %#lx", cur_pv.pv_va + cur_pv.pv_size,
821 kernel_vm_base);
822 VPRINTF("%s: mapping chunk VA %#lx..%#lx "
823 "(PA %#lx, prot %d, cache %d)\n",
824 __func__, cur_pv.pv_va, cur_pv.pv_va + cur_pv.pv_size - 1,
825 cur_pv.pv_pa, cur_pv.pv_prot, cur_pv.pv_cache);
826 pmap_map_chunk(l1pt_va, cur_pv.pv_va, cur_pv.pv_pa,
827 cur_pv.pv_size, cur_pv.pv_prot, cur_pv.pv_cache);
828 cur_pv.pv_pa += cur_pv.pv_size;
829 cur_pv.pv_va += cur_pv.pv_size;
830 cur_pv.pv_size = bmi->bmi_end - cur_pv.pv_pa;
831 cur_pv.pv_prot = VM_PROT_READ | VM_PROT_WRITE;
832 cur_pv.pv_cache = PTE_CACHE;
833 }
834 }
835
836 // The amount we can direct is limited by the start of the
837 // virtual part of the kernel address space. Don't overrun
838 // into it.
839 if (mapallmem_p && cur_pv.pv_va + cur_pv.pv_size > kernel_vm_base) {
840 cur_pv.pv_size = kernel_vm_base - cur_pv.pv_va;
841 }
842
843 /*
844 * Now we map the final chunk.
845 */
846 VPRINTF("%s: mapping last chunk VA %#lx..%#lx (PA %#lx, prot %d, cache %d)\n",
847 __func__, cur_pv.pv_va, cur_pv.pv_va + cur_pv.pv_size - 1,
848 cur_pv.pv_pa, cur_pv.pv_prot, cur_pv.pv_cache);
849 pmap_map_chunk(l1pt_va, cur_pv.pv_va, cur_pv.pv_pa,
850 cur_pv.pv_size, cur_pv.pv_prot, cur_pv.pv_cache);
851
852 /*
853 * Now we map the stuff that isn't directly after the kernel
854 */
855 if (map_vectors_p) {
856 /* Map the vector page. */
857 pmap_map_entry(l1pt_va, systempage.pv_va, systempage.pv_pa,
858 VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE, PTE_CACHE);
859 }
860
861 /* Map the Mini-Data cache clean area. */
862 #if ARM_MMU_XSCALE == 1
863 #if (ARM_NMMUS > 1)
864 if (xscale_use_minidata)
865 #endif
866 xscale_setup_minidata(l1pt_va, minidataclean.pv_va,
867 minidataclean.pv_pa);
868 #endif
869
870 /*
871 * Map integrated peripherals at same address in first level page
872 * table so that we can continue to use console.
873 */
874 if (devmap)
875 pmap_devmap_bootstrap(l1pt_va, devmap);
876
877 /* Tell the user about where all the bits and pieces live. */
878 VPRINTF("%22s Physical Virtual Num\n", " ");
879 VPRINTF("%22s Starting Ending Starting Ending Pages\n", " ");
880
881 #ifdef VERBOSE_INIT_ARM
882 static const char mem_fmt[] =
883 "%20s: 0x%08lx 0x%08lx 0x%08lx 0x%08lx %u\n";
884 static const char mem_fmt_nov[] =
885 "%20s: 0x%08lx 0x%08lx %zu\n";
886 #endif
887
888 #if 0
889 // XXX Doesn't make sense if kernel not at bottom of RAM
890 VPRINTF(mem_fmt, "SDRAM", bmi->bmi_start, bmi->bmi_end - 1,
891 KERN_PHYSTOV(bmi->bmi_start), KERN_PHYSTOV(bmi->bmi_end - 1),
892 (int)physmem);
893 #endif
894 VPRINTF(mem_fmt, "text section",
895 text.pv_pa, text.pv_pa + text.pv_size - 1,
896 text.pv_va, text.pv_va + text.pv_size - 1,
897 (int)(text.pv_size / PAGE_SIZE));
898 VPRINTF(mem_fmt, "data section",
899 KERN_VTOPHYS((vaddr_t)__data_start), KERN_VTOPHYS((vaddr_t)_edata),
900 (vaddr_t)__data_start, (vaddr_t)_edata,
901 (int)((round_page((vaddr_t)_edata)
902 - trunc_page((vaddr_t)__data_start)) / PAGE_SIZE));
903 VPRINTF(mem_fmt, "bss section",
904 KERN_VTOPHYS((vaddr_t)__bss_start), KERN_VTOPHYS((vaddr_t)__bss_end__),
905 (vaddr_t)__bss_start, (vaddr_t)__bss_end__,
906 (int)((round_page((vaddr_t)__bss_end__)
907 - trunc_page((vaddr_t)__bss_start)) / PAGE_SIZE));
908 VPRINTF(mem_fmt, "L1 page directory",
909 kernel_l1pt.pv_pa, kernel_l1pt.pv_pa + L1_TABLE_SIZE - 1,
910 kernel_l1pt.pv_va, kernel_l1pt.pv_va + L1_TABLE_SIZE - 1,
911 L1_TABLE_SIZE / PAGE_SIZE);
912 VPRINTF(mem_fmt, "ABT stack (CPU 0)",
913 abtstack.pv_pa, abtstack.pv_pa + (ABT_STACK_SIZE * PAGE_SIZE) - 1,
914 abtstack.pv_va, abtstack.pv_va + (ABT_STACK_SIZE * PAGE_SIZE) - 1,
915 ABT_STACK_SIZE);
916 VPRINTF(mem_fmt, "FIQ stack (CPU 0)",
917 fiqstack.pv_pa, fiqstack.pv_pa + (FIQ_STACK_SIZE * PAGE_SIZE) - 1,
918 fiqstack.pv_va, fiqstack.pv_va + (FIQ_STACK_SIZE * PAGE_SIZE) - 1,
919 FIQ_STACK_SIZE);
920 VPRINTF(mem_fmt, "IRQ stack (CPU 0)",
921 irqstack.pv_pa, irqstack.pv_pa + (IRQ_STACK_SIZE * PAGE_SIZE) - 1,
922 irqstack.pv_va, irqstack.pv_va + (IRQ_STACK_SIZE * PAGE_SIZE) - 1,
923 IRQ_STACK_SIZE);
924 VPRINTF(mem_fmt, "UND stack (CPU 0)",
925 undstack.pv_pa, undstack.pv_pa + (UND_STACK_SIZE * PAGE_SIZE) - 1,
926 undstack.pv_va, undstack.pv_va + (UND_STACK_SIZE * PAGE_SIZE) - 1,
927 UND_STACK_SIZE);
928 VPRINTF(mem_fmt, "IDLE stack (CPU 0)",
929 idlestack.pv_pa, idlestack.pv_pa + (UPAGES * PAGE_SIZE) - 1,
930 idlestack.pv_va, idlestack.pv_va + (UPAGES * PAGE_SIZE) - 1,
931 UPAGES);
932 VPRINTF(mem_fmt, "SVC stack",
933 kernelstack.pv_pa, kernelstack.pv_pa + (UPAGES * PAGE_SIZE) - 1,
934 kernelstack.pv_va, kernelstack.pv_va + (UPAGES * PAGE_SIZE) - 1,
935 UPAGES);
936 VPRINTF(mem_fmt, "Message Buffer",
937 msgbuf.pv_pa, msgbuf.pv_pa + (msgbuf_pgs * PAGE_SIZE) - 1,
938 msgbuf.pv_va, msgbuf.pv_va + (msgbuf_pgs * PAGE_SIZE) - 1,
939 (int)msgbuf_pgs);
940 if (map_vectors_p) {
941 VPRINTF(mem_fmt, "Exception Vectors",
942 systempage.pv_pa, systempage.pv_pa + PAGE_SIZE - 1,
943 systempage.pv_va, systempage.pv_va + PAGE_SIZE - 1,
944 1);
945 }
946 for (size_t i = 0; i < bmi->bmi_nfreeblocks; i++) {
947 pv = &bmi->bmi_freeblocks[i];
948
949 VPRINTF(mem_fmt_nov, "Free Memory",
950 pv->pv_pa, pv->pv_pa + pv->pv_size - 1,
951 pv->pv_size / PAGE_SIZE);
952 }
953 /*
954 * Now we have the real page tables in place so we can switch to them.
955 * Once this is done we will be running with the REAL kernel page
956 * tables.
957 */
958
959 VPRINTF("TTBR0=%#x", armreg_ttbr_read());
960 #ifdef _ARM_ARCH_6
961 VPRINTF(" TTBR1=%#x TTBCR=%#x CONTEXTIDR=%#x",
962 armreg_ttbr1_read(), armreg_ttbcr_read(),
963 armreg_contextidr_read());
964 #endif
965 VPRINTF("\n");
966
967 /* Switch tables */
968 VPRINTF("switching to new L1 page table @%#lx...\n", l1pt_pa);
969
970 cpu_ttb = l1pt_pa;
971
972 cpu_domains(DOMAIN_DEFAULT);
973
974 cpu_idcache_wbinv_all();
975
976 #ifdef __HAVE_GENERIC_START
977
978 /*
979 * Turn on caches and set SCTLR/ACTLR
980 */
981 cpu_setup(boot_args);
982 #endif
983
984 VPRINTF(" ttb");
985
986 #ifdef ARM_MMU_EXTENDED
987 /*
988 * TTBCR should have been initialized by the MD start code.
989 */
990 KASSERT((armreg_contextidr_read() & 0xff) == 0);
991 KASSERT(armreg_ttbcr_read() == __SHIFTIN(1, TTBCR_S_N));
992 /*
993 * Disable lookups via TTBR0 until there is an activated pmap.
994 */
995 armreg_ttbcr_write(armreg_ttbcr_read() | TTBCR_S_PD0);
996 cpu_setttb(l1pt_pa, KERNEL_PID);
997 arm_isb();
998 #else
999 cpu_setttb(l1pt_pa, true);
1000 #endif
1001
1002 cpu_tlb_flushID();
1003
1004 #ifdef ARM_MMU_EXTENDED
1005 VPRINTF("\nsctlr=%#x actlr=%#x\n",
1006 armreg_sctlr_read(), armreg_auxctl_read());
1007 #else
1008 VPRINTF(" (TTBR0=%#x)", armreg_ttbr_read());
1009 #endif
1010
1011 #ifdef MULTIPROCESSOR
1012 #ifndef __HAVE_GENERIC_START
1013 /*
1014 * Kick the secondaries to load the TTB. After which they'll go
1015 * back to sleep to wait for the final kick so they will hatch.
1016 */
1017 VPRINTF(" hatchlings");
1018 cpu_boot_secondary_processors();
1019 #endif
1020 #endif
1021
1022 VPRINTF(" OK\n");
1023 }
1024