arm32_machdep.c revision 1.133 1 /* $NetBSD: arm32_machdep.c,v 1.133 2020/04/18 11:00:37 skrll Exp $ */
2
3 /*
4 * Copyright (c) 1994-1998 Mark Brinicombe.
5 * Copyright (c) 1994 Brini.
6 * All rights reserved.
7 *
8 * This code is derived from software written for Brini by Mark Brinicombe
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 * 3. All advertising materials mentioning features or use of this software
19 * must display the following acknowledgement:
20 * This product includes software developed by Mark Brinicombe
21 * for the NetBSD Project.
22 * 4. The name of the company nor the name of the author may be used to
23 * endorse or promote products derived from this software without specific
24 * prior written permission.
25 *
26 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
27 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
28 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
29 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
30 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
31 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
32 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
33 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
35 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 * SUCH DAMAGE.
37 *
38 * Machine dependent functions for kernel setup
39 *
40 * Created : 17/09/94
41 * Updated : 18/04/01 updated for new wscons
42 */
43
44 #include <sys/cdefs.h>
45 __KERNEL_RCSID(0, "$NetBSD: arm32_machdep.c,v 1.133 2020/04/18 11:00:37 skrll Exp $");
46
47 #include "opt_arm_debug.h"
48 #include "opt_arm_start.h"
49 #include "opt_fdt.h"
50 #include "opt_modular.h"
51 #include "opt_md.h"
52 #include "opt_multiprocessor.h"
53
54 #include <sys/param.h>
55
56 #include <sys/atomic.h>
57 #include <sys/buf.h>
58 #include <sys/cpu.h>
59 #include <sys/device.h>
60 #include <sys/intr.h>
61 #include <sys/ipi.h>
62 #include <sys/kauth.h>
63 #include <sys/kernel.h>
64 #include <sys/mbuf.h>
65 #include <sys/module.h>
66 #include <sys/mount.h>
67 #include <sys/msgbuf.h>
68 #include <sys/proc.h>
69 #include <sys/reboot.h>
70 #include <sys/sysctl.h>
71 #include <sys/systm.h>
72 #include <sys/xcall.h>
73
74 #include <uvm/uvm_extern.h>
75
76 #include <dev/cons.h>
77 #include <dev/mm.h>
78
79 #include <arm/locore.h>
80
81 #include <arm/arm32/machdep.h>
82
83 #include <machine/bootconfig.h>
84 #include <machine/pcb.h>
85
86 #if defined(FDT)
87 #include <arm/fdt/arm_fdtvar.h>
88 #include <arch/evbarm/fdt/platform.h>
89 #endif
90
91 #ifdef VERBOSE_INIT_ARM
92 #define VPRINTF(...) printf(__VA_ARGS__)
93 #ifdef __HAVE_GENERIC_START
94 void generic_prints(const char *);
95 void generic_printx(int);
96 #define VPRINTS(s) generic_prints(s)
97 #define VPRINTX(x) generic_printx(x)
98 #else
99 #define VPRINTS(s) __nothing
100 #define VPRINTX(x) __nothing
101 #endif
102 #else
103 #define VPRINTF(...) __nothing
104 #define VPRINTS(s) __nothing
105 #define VPRINTX(x) __nothing
106 #endif
107
108 void (*cpu_reset_address)(void); /* Used by locore */
109 paddr_t cpu_reset_address_paddr; /* Used by locore */
110
111 struct vm_map *phys_map = NULL;
112
113 #if defined(MEMORY_DISK_HOOKS) && !defined(MEMORY_DISK_ROOT_SIZE)
114 extern size_t md_root_size; /* Memory disc size */
115 #endif /* MEMORY_DISK_HOOKS && !MEMORY_DISK_ROOT_SIZE */
116
117 pv_addr_t kernelstack;
118 pv_addr_t abtstack;
119 pv_addr_t fiqstack;
120 pv_addr_t irqstack;
121 pv_addr_t undstack;
122 pv_addr_t idlestack;
123
124 void * msgbufaddr;
125 extern paddr_t msgbufphys;
126
127 int kernel_debug = 0;
128 int cpu_printfataltraps = 0;
129 int cpu_fpu_present;
130 int cpu_hwdiv_present;
131 int cpu_neon_present;
132 int cpu_simd_present;
133 int cpu_simdex_present;
134 int cpu_umull_present;
135 int cpu_synchprim_present;
136 int cpu_unaligned_sigbus;
137 const char *cpu_arch = "";
138
139 int cpu_instruction_set_attributes[6];
140 int cpu_memory_model_features[4];
141 int cpu_processor_features[2];
142 int cpu_media_and_vfp_features[2];
143
144 /* exported variable to be filled in by the bootloaders */
145 char *booted_kernel;
146
147 /* Prototypes */
148
149 void data_abort_handler(trapframe_t *frame);
150 void prefetch_abort_handler(trapframe_t *frame);
151 extern void configure(void);
152
153 /*
154 * arm32_vector_init:
155 *
156 * Initialize the vector page, and select whether or not to
157 * relocate the vectors.
158 *
159 * NOTE: We expect the vector page to be mapped at its expected
160 * destination.
161 */
162 void
163 arm32_vector_init(vaddr_t va, int which)
164 {
165 #if defined(CPU_ARMV7) || defined(CPU_ARM11) || defined(ARM_HAS_VBAR)
166 /*
167 * If this processor has the security extension, don't bother
168 * to move/map the vector page. Simply point VBAR to the copy
169 * that exists in the .text segment.
170 */
171 #ifndef ARM_HAS_VBAR
172 if (va == ARM_VECTORS_LOW
173 && (armreg_pfr1_read() & ARM_PFR1_SEC_MASK) != 0) {
174 #endif
175 extern const uint32_t page0rel[];
176 vector_page = (vaddr_t)page0rel;
177 KASSERT((vector_page & 0x1f) == 0);
178 armreg_vbar_write(vector_page);
179 VPRINTF(" vbar=%p", page0rel);
180 cpu_control(CPU_CONTROL_VECRELOC, 0);
181 return;
182 #ifndef ARM_HAS_VBAR
183 }
184 #endif
185 #endif
186 #ifndef ARM_HAS_VBAR
187 if (CPU_IS_PRIMARY(curcpu())) {
188 extern unsigned int page0[], page0_data[];
189 unsigned int *vectors = (int *) va;
190 unsigned int *vectors_data = vectors + (page0_data - page0);
191 int vec;
192
193 /*
194 * Loop through the vectors we're taking over, and copy the
195 * vector's insn and data word.
196 */
197 for (vec = 0; vec < ARM_NVEC; vec++) {
198 if ((which & (1 << vec)) == 0) {
199 /* Don't want to take over this vector. */
200 continue;
201 }
202 vectors[vec] = page0[vec];
203 vectors_data[vec] = page0_data[vec];
204 }
205
206 /* Now sync the vectors. */
207 cpu_icache_sync_range(va, (ARM_NVEC * 2) * sizeof(u_int));
208
209 vector_page = va;
210 }
211
212 if (va == ARM_VECTORS_HIGH) {
213 /*
214 * Assume the MD caller knows what it's doing here, and
215 * really does want the vector page relocated.
216 *
217 * Note: This has to be done here (and not just in
218 * cpu_setup()) because the vector page needs to be
219 * accessible *before* cpu_startup() is called.
220 * Think ddb(9) ...
221 *
222 * NOTE: If the CPU control register is not readable,
223 * this will totally fail! We'll just assume that
224 * any system that has high vector support has a
225 * readable CPU control register, for now. If we
226 * ever encounter one that does not, we'll have to
227 * rethink this.
228 */
229 cpu_control(CPU_CONTROL_VECRELOC, CPU_CONTROL_VECRELOC);
230 }
231 #endif
232 }
233
234 /*
235 * Debug function just to park the CPU
236 */
237
238 void
239 halt(void)
240 {
241 while (1)
242 cpu_sleep(0);
243 }
244
245
246 /* Sync the discs, unmount the filesystems, and adjust the todr */
247
248 void
249 bootsync(void)
250 {
251 static bool bootsyncdone = false;
252
253 if (bootsyncdone) return;
254
255 bootsyncdone = true;
256
257 /* Make sure we can still manage to do things */
258 if (GetCPSR() & I32_bit) {
259 /*
260 * If we get here then boot has been called without RB_NOSYNC
261 * and interrupts were disabled. This means the boot() call
262 * did not come from a user process e.g. shutdown, but must
263 * have come from somewhere in the kernel.
264 */
265 IRQenable;
266 printf("Warning IRQ's disabled during boot()\n");
267 }
268
269 vfs_shutdown();
270
271 resettodr();
272 }
273
274 /*
275 * void cpu_startup(void)
276 *
277 * Machine dependent startup code.
278 *
279 */
280 void
281 cpu_startup(void)
282 {
283 vaddr_t minaddr;
284 vaddr_t maxaddr;
285
286 #ifndef __HAVE_GENERIC_START
287 /* Set the CPU control register */
288 cpu_setup(boot_args);
289 #endif
290
291 #ifndef ARM_HAS_VBAR
292 /* Lock down zero page */
293 vector_page_setprot(VM_PROT_READ);
294 #endif
295
296 /*
297 * Give pmap a chance to set up a few more things now the vm
298 * is initialised
299 */
300 pmap_postinit();
301
302 #ifdef FDT
303 if (arm_fdt_platform()->ap_startup != NULL)
304 arm_fdt_platform()->ap_startup();
305 #endif
306
307 /*
308 * Initialize error message buffer (at end of core).
309 */
310
311 /* msgbufphys was setup during the secondary boot strap */
312 if (!pmap_extract(pmap_kernel(), (vaddr_t)msgbufaddr, NULL)) {
313 for (u_int loop = 0; loop < btoc(MSGBUFSIZE); ++loop) {
314 pmap_kenter_pa((vaddr_t)msgbufaddr + loop * PAGE_SIZE,
315 msgbufphys + loop * PAGE_SIZE,
316 VM_PROT_READ|VM_PROT_WRITE, 0);
317 }
318 }
319 pmap_update(pmap_kernel());
320 initmsgbuf(msgbufaddr, round_page(MSGBUFSIZE));
321
322 /*
323 * Allocate a submap for physio
324 */
325 minaddr = 0;
326 phys_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr,
327 VM_PHYS_SIZE, 0, false, NULL);
328
329 banner();
330
331 /*
332 * This is actually done by initarm_common, but not all ports use it
333 * yet so do it here to catch them as well
334 */
335 struct lwp * const l = &lwp0;
336 struct pcb * const pcb = lwp_getpcb(l);
337
338 /* Zero out the PCB. */
339 memset(pcb, 0, sizeof(*pcb));
340
341 pcb->pcb_ksp = uvm_lwp_getuarea(l) + USPACE_SVC_STACK_TOP;
342 pcb->pcb_ksp -= sizeof(struct trapframe);
343
344 struct trapframe * tf = (struct trapframe *)pcb->pcb_ksp;
345
346 /* Zero out the trapframe. */
347 memset(tf, 0, sizeof(*tf));
348 lwp_settrapframe(l, tf);
349
350 #if defined(__ARMEB__)
351 tf->tf_spsr = PSR_USR32_MODE | (CPU_IS_ARMV7_P() ? PSR_E_BIT : 0);
352 #else
353 tf->tf_spsr = PSR_USR32_MODE;
354 #endif
355
356 cpu_startup_hook();
357 }
358
359 __weak_alias(cpu_startup_hook,cpu_startup_default)
360 void
361 cpu_startup_default(void)
362 {
363 }
364
365 /*
366 * machine dependent system variables.
367 */
368 static int
369 sysctl_machdep_booted_device(SYSCTLFN_ARGS)
370 {
371 struct sysctlnode node;
372
373 if (booted_device == NULL)
374 return (EOPNOTSUPP);
375
376 node = *rnode;
377 node.sysctl_data = __UNCONST(device_xname(booted_device));
378 node.sysctl_size = strlen(device_xname(booted_device)) + 1;
379 return (sysctl_lookup(SYSCTLFN_CALL(&node)));
380 }
381
382 static int
383 sysctl_machdep_booted_kernel(SYSCTLFN_ARGS)
384 {
385 struct sysctlnode node;
386
387 if (booted_kernel == NULL || booted_kernel[0] == '\0')
388 return (EOPNOTSUPP);
389
390 node = *rnode;
391 node.sysctl_data = booted_kernel;
392 node.sysctl_size = strlen(booted_kernel) + 1;
393 return (sysctl_lookup(SYSCTLFN_CALL(&node)));
394 }
395
396 static int
397 sysctl_machdep_cpu_arch(SYSCTLFN_ARGS)
398 {
399 struct sysctlnode node = *rnode;
400 node.sysctl_data = __UNCONST(cpu_arch);
401 node.sysctl_size = strlen(cpu_arch) + 1;
402 return sysctl_lookup(SYSCTLFN_CALL(&node));
403 }
404
405 static int
406 sysctl_machdep_powersave(SYSCTLFN_ARGS)
407 {
408 struct sysctlnode node = *rnode;
409 int error, newval;
410
411 newval = cpu_do_powersave;
412 node.sysctl_data = &newval;
413 if (cpufuncs.cf_sleep == (void *) cpufunc_nullop)
414 node.sysctl_flags &= ~CTLFLAG_READWRITE;
415 error = sysctl_lookup(SYSCTLFN_CALL(&node));
416 if (error || newp == NULL || newval == cpu_do_powersave)
417 return (error);
418
419 if (newval < 0 || newval > 1)
420 return (EINVAL);
421 cpu_do_powersave = newval;
422
423 return (0);
424 }
425
426 SYSCTL_SETUP(sysctl_machdep_setup, "sysctl machdep subtree setup")
427 {
428
429 sysctl_createv(clog, 0, NULL, NULL,
430 CTLFLAG_PERMANENT,
431 CTLTYPE_NODE, "machdep", NULL,
432 NULL, 0, NULL, 0,
433 CTL_MACHDEP, CTL_EOL);
434
435 sysctl_createv(clog, 0, NULL, NULL,
436 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
437 CTLTYPE_INT, "debug", NULL,
438 NULL, 0, &kernel_debug, 0,
439 CTL_MACHDEP, CPU_DEBUG, CTL_EOL);
440 sysctl_createv(clog, 0, NULL, NULL,
441 CTLFLAG_PERMANENT,
442 CTLTYPE_STRING, "booted_device", NULL,
443 sysctl_machdep_booted_device, 0, NULL, 0,
444 CTL_MACHDEP, CPU_BOOTED_DEVICE, CTL_EOL);
445 sysctl_createv(clog, 0, NULL, NULL,
446 CTLFLAG_PERMANENT,
447 CTLTYPE_STRING, "booted_kernel", NULL,
448 sysctl_machdep_booted_kernel, 0, NULL, 0,
449 CTL_MACHDEP, CPU_BOOTED_KERNEL, CTL_EOL);
450 sysctl_createv(clog, 0, NULL, NULL,
451 CTLFLAG_PERMANENT,
452 CTLTYPE_STRUCT, "console_device", NULL,
453 sysctl_consdev, 0, NULL, sizeof(dev_t),
454 CTL_MACHDEP, CPU_CONSDEV, CTL_EOL);
455 sysctl_createv(clog, 0, NULL, NULL,
456 CTLFLAG_PERMANENT,
457 CTLTYPE_STRING, "cpu_arch", NULL,
458 sysctl_machdep_cpu_arch, 0, NULL, 0,
459 CTL_MACHDEP, CTL_CREATE, CTL_EOL);
460 sysctl_createv(clog, 0, NULL, NULL,
461 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
462 CTLTYPE_INT, "powersave", NULL,
463 sysctl_machdep_powersave, 0, &cpu_do_powersave, 0,
464 CTL_MACHDEP, CPU_POWERSAVE, CTL_EOL);
465 sysctl_createv(clog, 0, NULL, NULL,
466 CTLFLAG_PERMANENT|CTLFLAG_IMMEDIATE,
467 CTLTYPE_INT, "cpu_id", NULL,
468 NULL, curcpu()->ci_arm_cpuid, NULL, 0,
469 CTL_MACHDEP, CTL_CREATE, CTL_EOL);
470 #ifdef FPU_VFP
471 sysctl_createv(clog, 0, NULL, NULL,
472 CTLFLAG_PERMANENT|CTLFLAG_READONLY,
473 CTLTYPE_INT, "fpu_id", NULL,
474 NULL, 0, &cpu_info_store[0].ci_vfp_id, 0,
475 CTL_MACHDEP, CTL_CREATE, CTL_EOL);
476 #endif
477 sysctl_createv(clog, 0, NULL, NULL,
478 CTLFLAG_PERMANENT|CTLFLAG_READONLY,
479 CTLTYPE_INT, "fpu_present", NULL,
480 NULL, 0, &cpu_fpu_present, 0,
481 CTL_MACHDEP, CTL_CREATE, CTL_EOL);
482 sysctl_createv(clog, 0, NULL, NULL,
483 CTLFLAG_PERMANENT|CTLFLAG_READONLY,
484 CTLTYPE_INT, "hwdiv_present", NULL,
485 NULL, 0, &cpu_hwdiv_present, 0,
486 CTL_MACHDEP, CTL_CREATE, CTL_EOL);
487 sysctl_createv(clog, 0, NULL, NULL,
488 CTLFLAG_PERMANENT|CTLFLAG_READONLY,
489 CTLTYPE_INT, "neon_present", NULL,
490 NULL, 0, &cpu_neon_present, 0,
491 CTL_MACHDEP, CTL_CREATE, CTL_EOL);
492 sysctl_createv(clog, 0, NULL, NULL,
493 CTLFLAG_PERMANENT|CTLFLAG_READONLY,
494 CTLTYPE_STRUCT, "id_isar", NULL,
495 NULL, 0,
496 cpu_instruction_set_attributes,
497 sizeof(cpu_instruction_set_attributes),
498 CTL_MACHDEP, CTL_CREATE, CTL_EOL);
499 sysctl_createv(clog, 0, NULL, NULL,
500 CTLFLAG_PERMANENT|CTLFLAG_READONLY,
501 CTLTYPE_STRUCT, "id_mmfr", NULL,
502 NULL, 0,
503 cpu_memory_model_features,
504 sizeof(cpu_memory_model_features),
505 CTL_MACHDEP, CTL_CREATE, CTL_EOL);
506 sysctl_createv(clog, 0, NULL, NULL,
507 CTLFLAG_PERMANENT|CTLFLAG_READONLY,
508 CTLTYPE_STRUCT, "id_pfr", NULL,
509 NULL, 0,
510 cpu_processor_features,
511 sizeof(cpu_processor_features),
512 CTL_MACHDEP, CTL_CREATE, CTL_EOL);
513 sysctl_createv(clog, 0, NULL, NULL,
514 CTLFLAG_PERMANENT|CTLFLAG_READONLY,
515 CTLTYPE_STRUCT, "id_mvfr", NULL,
516 NULL, 0,
517 cpu_media_and_vfp_features,
518 sizeof(cpu_media_and_vfp_features),
519 CTL_MACHDEP, CTL_CREATE, CTL_EOL);
520 sysctl_createv(clog, 0, NULL, NULL,
521 CTLFLAG_PERMANENT|CTLFLAG_READONLY,
522 CTLTYPE_INT, "simd_present", NULL,
523 NULL, 0, &cpu_simd_present, 0,
524 CTL_MACHDEP, CTL_CREATE, CTL_EOL);
525 sysctl_createv(clog, 0, NULL, NULL,
526 CTLFLAG_PERMANENT|CTLFLAG_READONLY,
527 CTLTYPE_INT, "simdex_present", NULL,
528 NULL, 0, &cpu_simdex_present, 0,
529 CTL_MACHDEP, CTL_CREATE, CTL_EOL);
530 sysctl_createv(clog, 0, NULL, NULL,
531 CTLFLAG_PERMANENT|CTLFLAG_READONLY,
532 CTLTYPE_INT, "synchprim_present", NULL,
533 NULL, 0, &cpu_synchprim_present, 0,
534 CTL_MACHDEP, CTL_CREATE, CTL_EOL);
535 sysctl_createv(clog, 0, NULL, NULL,
536 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
537 CTLTYPE_INT, "printfataltraps", NULL,
538 NULL, 0, &cpu_printfataltraps, 0,
539 CTL_MACHDEP, CTL_CREATE, CTL_EOL);
540 cpu_unaligned_sigbus = !CPU_IS_ARMV6_P() && !CPU_IS_ARMV7_P();
541 sysctl_createv(clog, 0, NULL, NULL,
542 CTLFLAG_PERMANENT|CTLFLAG_READONLY,
543 CTLTYPE_INT, "unaligned_sigbus",
544 SYSCTL_DESCR("Do SIGBUS for fixed unaligned accesses"),
545 NULL, 0, &cpu_unaligned_sigbus, 0,
546 CTL_MACHDEP, CTL_CREATE, CTL_EOL);
547 }
548
549 void
550 parse_mi_bootargs(char *args)
551 {
552 int integer;
553
554 if (get_bootconf_option(args, "single", BOOTOPT_TYPE_BOOLEAN, &integer)
555 || get_bootconf_option(args, "-s", BOOTOPT_TYPE_BOOLEAN, &integer))
556 if (integer)
557 boothowto |= RB_SINGLE;
558 if (get_bootconf_option(args, "kdb", BOOTOPT_TYPE_BOOLEAN, &integer)
559 || get_bootconf_option(args, "-k", BOOTOPT_TYPE_BOOLEAN, &integer)
560 || get_bootconf_option(args, "-d", BOOTOPT_TYPE_BOOLEAN, &integer))
561 if (integer)
562 boothowto |= RB_KDB;
563 if (get_bootconf_option(args, "ask", BOOTOPT_TYPE_BOOLEAN, &integer)
564 || get_bootconf_option(args, "-a", BOOTOPT_TYPE_BOOLEAN, &integer))
565 if (integer)
566 boothowto |= RB_ASKNAME;
567
568 /* if (get_bootconf_option(args, "nbuf", BOOTOPT_TYPE_INT, &integer))
569 bufpages = integer;*/
570
571 #if defined(MEMORY_DISK_HOOKS) && !defined(MEMORY_DISK_ROOT_SIZE)
572 if (get_bootconf_option(args, "memorydisc", BOOTOPT_TYPE_INT, &integer)
573 || get_bootconf_option(args, "memorydisk", BOOTOPT_TYPE_INT, &integer)) {
574 md_root_size = integer;
575 md_root_size *= 1024;
576 if (md_root_size < 32*1024)
577 md_root_size = 32*1024;
578 if (md_root_size > 2048*1024)
579 md_root_size = 2048*1024;
580 }
581 #endif /* MEMORY_DISK_HOOKS && !MEMORY_DISK_ROOT_SIZE */
582
583 if (get_bootconf_option(args, "quiet", BOOTOPT_TYPE_BOOLEAN, &integer)
584 || get_bootconf_option(args, "-q", BOOTOPT_TYPE_BOOLEAN, &integer))
585 if (integer)
586 boothowto |= AB_QUIET;
587 if (get_bootconf_option(args, "verbose", BOOTOPT_TYPE_BOOLEAN, &integer)
588 || get_bootconf_option(args, "-v", BOOTOPT_TYPE_BOOLEAN, &integer))
589 if (integer)
590 boothowto |= AB_VERBOSE;
591 if (get_bootconf_option(args, "debug", BOOTOPT_TYPE_BOOLEAN, &integer)
592 || get_bootconf_option(args, "-x", BOOTOPT_TYPE_BOOLEAN, &integer))
593 if (integer)
594 boothowto |= AB_DEBUG;
595 }
596
597 #ifdef __HAVE_FAST_SOFTINTS
598 #if IPL_SOFTSERIAL != IPL_SOFTNET + 1
599 #error IPLs are screwed up
600 #elif IPL_SOFTNET != IPL_SOFTBIO + 1
601 #error IPLs are screwed up
602 #elif IPL_SOFTBIO != IPL_SOFTCLOCK + 1
603 #error IPLs are screwed up
604 #elif !(IPL_SOFTCLOCK > IPL_NONE)
605 #error IPLs are screwed up
606 #elif (IPL_NONE != 0)
607 #error IPLs are screwed up
608 #endif
609
610 #ifndef __HAVE_PIC_FAST_SOFTINTS
611 #define SOFTINT2IPLMAP \
612 (((IPL_SOFTSERIAL - IPL_SOFTCLOCK) << (SOFTINT_SERIAL * 4)) | \
613 ((IPL_SOFTNET - IPL_SOFTCLOCK) << (SOFTINT_NET * 4)) | \
614 ((IPL_SOFTBIO - IPL_SOFTCLOCK) << (SOFTINT_BIO * 4)) | \
615 ((IPL_SOFTCLOCK - IPL_SOFTCLOCK) << (SOFTINT_CLOCK * 4)))
616 #define SOFTINT2IPL(l) ((SOFTINT2IPLMAP >> ((l) * 4)) & 0x0f)
617
618 /*
619 * This returns a mask of softint IPLs that be dispatch at <ipl>
620 * SOFTIPLMASK(IPL_NONE) = 0x0000000f
621 * SOFTIPLMASK(IPL_SOFTCLOCK) = 0x0000000e
622 * SOFTIPLMASK(IPL_SOFTBIO) = 0x0000000c
623 * SOFTIPLMASK(IPL_SOFTNET) = 0x00000008
624 * SOFTIPLMASK(IPL_SOFTSERIAL) = 0x00000000
625 */
626 #define SOFTIPLMASK(ipl) ((0x0f << (ipl)) & 0x0f)
627
628 void softint_switch(lwp_t *, int);
629
630 void
631 softint_trigger(uintptr_t mask)
632 {
633 curcpu()->ci_softints |= mask;
634 }
635
636 void
637 softint_init_md(lwp_t *l, u_int level, uintptr_t *machdep)
638 {
639 lwp_t ** lp = &l->l_cpu->ci_softlwps[level];
640 KASSERT(*lp == NULL || *lp == l);
641 *lp = l;
642 *machdep = 1 << SOFTINT2IPL(level);
643 KASSERT(level != SOFTINT_CLOCK || *machdep == (1 << (IPL_SOFTCLOCK - IPL_SOFTCLOCK)));
644 KASSERT(level != SOFTINT_BIO || *machdep == (1 << (IPL_SOFTBIO - IPL_SOFTCLOCK)));
645 KASSERT(level != SOFTINT_NET || *machdep == (1 << (IPL_SOFTNET - IPL_SOFTCLOCK)));
646 KASSERT(level != SOFTINT_SERIAL || *machdep == (1 << (IPL_SOFTSERIAL - IPL_SOFTCLOCK)));
647 }
648
649 void
650 dosoftints(void)
651 {
652 struct cpu_info * const ci = curcpu();
653 const int opl = ci->ci_cpl;
654 const uint32_t softiplmask = SOFTIPLMASK(opl);
655
656 splhigh();
657 for (;;) {
658 u_int softints = ci->ci_softints & softiplmask;
659 KASSERT((softints != 0) == ((ci->ci_softints >> opl) != 0));
660 KASSERT(opl == IPL_NONE || (softints & (1 << (opl - IPL_SOFTCLOCK))) == 0);
661 if (softints == 0) {
662 splx(opl);
663 return;
664 }
665 #define DOSOFTINT(n) \
666 if (ci->ci_softints & (1 << (IPL_SOFT ## n - IPL_SOFTCLOCK))) { \
667 ci->ci_softints &= \
668 ~(1 << (IPL_SOFT ## n - IPL_SOFTCLOCK)); \
669 softint_switch(ci->ci_softlwps[SOFTINT_ ## n], \
670 IPL_SOFT ## n); \
671 continue; \
672 }
673 DOSOFTINT(SERIAL);
674 DOSOFTINT(NET);
675 DOSOFTINT(BIO);
676 DOSOFTINT(CLOCK);
677 panic("dosoftints wtf (softints=%u?, ipl=%d)", softints, opl);
678 }
679 }
680 #endif /* !__HAVE_PIC_FAST_SOFTINTS */
681 #endif /* __HAVE_FAST_SOFTINTS */
682
683 #ifdef MODULAR
684 /*
685 * Push any modules loaded by the boot loader.
686 */
687 void
688 module_init_md(void)
689 {
690 }
691 #endif /* MODULAR */
692
693 int
694 mm_md_physacc(paddr_t pa, vm_prot_t prot)
695 {
696 if (pa >= physical_start && pa < physical_end)
697 return 0;
698
699 return kauth_authorize_machdep(kauth_cred_get(),
700 KAUTH_MACHDEP_UNMANAGEDMEM, NULL, NULL, NULL, NULL);
701 }
702
703 #ifdef __HAVE_CPU_UAREA_ALLOC_IDLELWP
704 vaddr_t
705 cpu_uarea_alloc_idlelwp(struct cpu_info *ci)
706 {
707 const vaddr_t va = idlestack.pv_va + cpu_index(ci) * USPACE;
708 // printf("%s: %s: va=%lx\n", __func__, ci->ci_data.cpu_name, va);
709 return va;
710 }
711 #endif
712
713 #ifdef MULTIPROCESSOR
714 /*
715 * Initialise a secondary processor.
716 *
717 * printf isn't available to us for a number of reasons.
718 *
719 * - kprint_init has been called and printf will try to take locks which we
720 * can't do just yet because bootstrap translation tables do not allowing
721 * caching.
722 *
723 * - kmutex(9) relies on curcpu which isn't setup yet.
724 *
725 */
726 void
727 cpu_init_secondary_processor(int cpuindex)
728 {
729 // pmap_kernel has been successfully built and we can switch to it
730 cpu_domains(DOMAIN_DEFAULT);
731 cpu_idcache_wbinv_all();
732
733 VPRINTS("index: ");
734 VPRINTX(cpuindex);
735 VPRINTS(" ttb");
736
737 cpu_setup(boot_args);
738
739 #ifdef ARM_MMU_EXTENDED
740 /*
741 * TTBCR should have been initialized by the MD start code.
742 */
743 KASSERT((armreg_contextidr_read() & 0xff) == 0);
744 KASSERT(armreg_ttbcr_read() == __SHIFTIN(1, TTBCR_S_N));
745 /*
746 * Disable lookups via TTBR0 until there is an activated pmap.
747 */
748
749 armreg_ttbcr_write(armreg_ttbcr_read() | TTBCR_S_PD0);
750 cpu_setttb(pmap_kernel()->pm_l1_pa , KERNEL_PID);
751 arm_isb();
752 #else
753 cpu_setttb(pmap_kernel()->pm_l1->l1_physaddr, true);
754 #endif
755
756 cpu_tlb_flushID();
757
758 VPRINTS(" (TTBR0=");
759 VPRINTX(armreg_ttbr_read());
760 VPRINTS(")");
761
762 #ifdef ARM_MMU_EXTENDED
763 VPRINTS(" (TTBR1=");
764 VPRINTX(armreg_ttbr1_read());
765 VPRINTS(")");
766 VPRINTS(" (TTBCR=");
767 VPRINTX(armreg_ttbcr_read());
768 VPRINTS(")");
769 #endif
770
771 VPRINTS(" hatched|=");
772 VPRINTX(__BIT(cpuindex));
773 VPRINTS("\n\r");
774
775 cpu_set_hatched(cpuindex);
776
777 /* return to assembly to wait for cpu_boot_secondary_processors */
778 }
779
780 void
781 xc_send_ipi(struct cpu_info *ci)
782 {
783 KASSERT(kpreempt_disabled());
784 KASSERT(curcpu() != ci);
785
786 intr_ipi_send(ci != NULL ? ci->ci_kcpuset : NULL, IPI_XCALL);
787 }
788
789 void
790 cpu_ipi(struct cpu_info *ci)
791 {
792 KASSERT(kpreempt_disabled());
793 KASSERT(curcpu() != ci);
794
795 intr_ipi_send(ci != NULL ? ci->ci_kcpuset : NULL, IPI_GENERIC);
796 }
797
798 #endif /* MULTIPROCESSOR */
799
800 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS
801 bool
802 mm_md_direct_mapped_phys(paddr_t pa, vaddr_t *vap)
803 {
804 bool rv;
805 vaddr_t va = pmap_direct_mapped_phys(pa, &rv, 0);
806 if (rv) {
807 *vap = va;
808 }
809 return rv;
810 }
811 #endif
812
813 bool
814 mm_md_page_color(paddr_t pa, int *colorp)
815 {
816 #if (ARM_MMU_V6 + ARM_MMU_V7) != 0
817 *colorp = atop(pa & arm_cache_prefer_mask);
818
819 return arm_cache_prefer_mask ? false : true;
820 #else
821 *colorp = 0;
822
823 return true;
824 #endif
825 }
826
827 #if defined(FDT)
828 extern char KERNEL_BASE_phys[];
829 #define KERNEL_BASE_PHYS ((paddr_t)KERNEL_BASE_phys)
830
831 void
832 cpu_kernel_vm_init(paddr_t memory_start, psize_t memory_size)
833 {
834 const struct arm_platform *plat = arm_fdt_platform();
835
836 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS
837 const bool mapallmem_p = true;
838 #ifndef PMAP_NEED_ALLOC_POOLPAGE
839 if (memory_size > KERNEL_VM_BASE - KERNEL_BASE) {
840 VPRINTF("%s: dropping RAM size from %luMB to %uMB\n",
841 __func__, (unsigned long) (memory_size >> 20),
842 (KERNEL_VM_BASE - KERNEL_BASE) >> 20);
843 memory_size = KERNEL_VM_BASE - KERNEL_BASE;
844 }
845 #endif
846 #else
847 const bool mapallmem_p = false;
848 #endif
849
850 VPRINTF("%s: kernel phys start %" PRIxPADDR " end %" PRIxPADDR "\n",
851 __func__, memory_start, memory_start + memory_size);
852
853 arm32_bootmem_init(memory_start, memory_size, KERNEL_BASE_PHYS);
854 arm32_kernel_vm_init(KERNEL_VM_BASE, ARM_VECTORS_HIGH, 0,
855 plat->ap_devmap(), mapallmem_p);
856 }
857 #endif
858
859