1 /* $NetBSD: armv6_start.S,v 1.41 2025/10/09 06:15:16 skrll Exp $ */ 2 3 /*- 4 * Copyright (c) 2012, 2017, 2018 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Matt Thomas of 3am Software Foundry and Nick Hudson. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include "opt_arm_debug.h" 33 #include "opt_console.h" 34 #include "opt_cpuoptions.h" 35 #include "opt_cputypes.h" 36 #include "opt_fdt.h" 37 #include "opt_kasan.h" 38 #include "opt_multiprocessor.h" 39 40 #include <sys/cdefs.h> 41 42 #include <arm/asm.h> 43 #include <arm/armreg.h> 44 #include <arm/locore.h> 45 46 #include "assym.h" 47 48 #if defined(CONSADDR) && defined(CONADDR) 49 #error Only one of CONSADDR and CONADDR should be defined 50 #endif 51 52 #if defined(CONSADDR) 53 #define START_CONSADDR CONSADDR 54 #endif 55 #if defined(CONADDR) 56 #define START_CONSADDR CONADDR 57 #endif 58 59 #if defined(VERBOSE_INIT_ARM) 60 #define XPUTC(n) mov r0, n; bl uartputc 61 #define VPRINTF(string) bl generic_vprint; .asciz string; .align 2 62 #define VPRINTX(regno) mov r0, regno; bl generic_printx 63 #else 64 #define XPUTC(c) /* nothing */ 65 #define VPRINTF(string) /* nothing */ 66 #define VPRINTX(regno) /* nothing */ 67 #endif 68 69 #if defined(FDT) 70 #define MD_CPU_HATCH arm_fdt_cpu_hatch 71 #endif 72 73 #if defined(_ARM_ARCH_6) 74 #define L1_S_KRW_NOCACHE (L1_S_PROTO | L1_S_AP_KRW | L1_S_NOCACHE_armv6n) 75 #else 76 #define L1_S_KRW_NOCACHE (L1_S_PROTO | L1_S_AP_KRW | L1_S_NOCACHE_generic) 77 #endif 78 79 /* 80 * A generic kernel start routine. 81 * 82 * At this point, this code has been loaded into SDRAM and the MMU should be off 83 * with data caches disabled. 84 * 85 * linux image type should be used in uboot images to ensure this is the case. 86 */ 87 88 // Use callee saved registers 89 R_L1TABLE .req r4 90 R_VA .req r5 91 R_PA .req r6 92 R_NSEC .req r7 93 R_ATTR .req r8 94 R_DEVATTR .req r9 95 96 R_TMP1 .req r8 97 R_TMP2 .req r9 98 R_VTOPDIFF .req r10 99 R_FDTADDR .req r11 100 R_INDEX .req r11 101 102 .text 103 104 ENTRY_NP(generic_start) 105 106 #if defined(__ARMEB__) 107 /* Make sure sctlr.u = 1 when cpsr.e = 1. */ 108 mrc p15, 0, R_TMP1, c1, c0, 0 109 orr R_TMP1, R_TMP1, #CPU_CONTROL_UNAL_ENABLE 110 mcr p15, 0, R_TMP1, c1, c0, 0 111 112 setend be /* force big endian */ 113 #endif 114 115 /* disable IRQs/FIQs. */ 116 INTERRUPT_DISABLE(r0) 117 118 adr R_TMP1, generic_start 119 ldr R_VTOPDIFF, =generic_start 120 sub R_VTOPDIFF, R_VTOPDIFF, R_TMP1 121 122 ldr R_TMP1, =start_stacks_top 123 sub sp, R_TMP1, R_VTOPDIFF 124 125 mov r4, r0 126 mov r5, r1 127 mov r6, r2 128 mov r7, r3 129 130 // We can now call functions 131 132 VPRINTF("\n\rpc : ") 133 VPRINTX(pc) 134 135 VPRINTF("\n\roff : ") 136 VPRINTX(R_VTOPDIFF) 137 138 VPRINTF("\n\rsp : ") 139 VPRINTX(sp) 140 141 ldr R_TMP1, =(L1_S_SIZE - 1) 142 ands R_TMP2, R_VTOPDIFF, R_TMP1 143 bne arm_bad_vtopdiff 144 145 #if defined(FDTBASE) 146 /* 147 * ARM boot protocol has FDT address in r2 which is now in r6 148 */ 149 VPRINTF("\n\rfdt : ") 150 mov R_FDTADDR, r6 // Save fdt_addr_r for mapping later 151 152 VPRINTX(r6) 153 #endif 154 155 #if defined(VERBOSE_INIT_ARM) 156 VPRINTF("\n\rmidr : ") 157 mrc p15, 0, r0, c0, c0, 0 // MIDR 158 VPRINTX(r0) 159 VPRINTF("\n\rrevidr: ") 160 mrc p15, 0, r0, c0, c0, 6 // REVIDR 161 VPRINTX(r0) 162 VPRINTF("\n\rmpidr : ") 163 mrc p15, 0, r0, c0, c0, 5 // MPIDR 164 VPRINTX(r0) 165 VPRINTF("\n\rttb0 : ") 166 mrc p15, 0, r0, c2, c0, 0 // TTBR0 read 167 VPRINTX(r0) 168 #if defined(_ARM_ARCH_6) 169 VPRINTF("\n\rttb1 : ") 170 mrc p15, 0, r0, c2, c0, 1 // TTBR1 read 171 VPRINTX(r0) 172 VPRINTF("\n\rttcr : ") 173 mrc p15, 0, r0, c2, c0, 2 // TTBCR read 174 VPRINTX(r0) 175 #endif 176 VPRINTF("\n\r") 177 #endif 178 179 #if defined(_ARM_ARCH_7) 180 b generic_startv7 181 #elif defined(_ARM_ARCH_6) 182 b generic_startv6 183 #elif defined(_ARM_ARCH_5) 184 b generic_startv5 185 #else 186 #error Unsupported CPU 187 #endif 188 arm_bad_vtopdiff: 189 VPRINTF("\n\rpanic: vtop not L1_FRAME aligned (") 190 VPRINTX(R_VTOPDIFF) 191 VPRINTF(")\n\r") 192 1: b 1b 193 ASEND(generic_start) 194 195 /* 196 * Save the u-boot arguments (including FDT address) and the virtual to physical 197 * offset. 198 * 199 * Uses the following callee saved registers: 200 * 201 * r8 (R_TMP1), r9 (R_TMP2) 202 */ 203 generic_savevars: 204 mov R_TMP1, lr 205 /* 206 * Store virtual to physical address difference 207 */ 208 ldr R_TMP2, =kern_vtopdiff 209 sub R_TMP2, R_TMP2, R_VTOPDIFF 210 str R_VTOPDIFF, [R_TMP2] 211 212 /* 213 * store uboot arguments to uboot_args[4] 214 */ 215 ldr R_TMP2, =uboot_args 216 sub R_TMP2, R_VTOPDIFF 217 218 VPRINTF("\n\ruboot : ") 219 VPRINTX(R_TMP2) 220 str r4, [R_TMP2, #(4*0)] 221 str r5, [R_TMP2, #(4*1)] 222 str r6, [R_TMP2, #(4*2)] 223 str r7, [R_TMP2, #(4*3)] 224 225 #if defined(FDTBASE) 226 /* 227 * ARM boot protocol has FDT address in r2 which is now in r6 228 */ 229 VPRINTF("\n\rfdt : ") 230 ldr R_TMP2, =fdt_addr_r 231 sub R_TMP2, R_VTOPDIFF 232 str r6, [R_TMP2] 233 234 VPRINTX(r6) 235 #endif 236 237 RETr(R_TMP1) 238 239 .ltorg 240 241 /* 242 * Allocate some memory after the kernel image for stacks and 243 * bootstrap L1PT 244 */ 245 .section "_init_memory", "aw", %nobits 246 .p2align INIT_ARM_STACK_SHIFT 247 .global start_stacks_bottom 248 .global start_stacks_top 249 start_stacks_bottom: 250 .space INIT_ARM_TOTAL_STACK 251 start_stacks_top: 252 253 .section "_init_memory", "aw", %nobits 254 .p2align 14 /* 16KiB aligned */ 255 .global ARM_BOOTSTRAP_LxPT 256 ARM_BOOTSTRAP_LxPT: 257 TEMP_L1_TABLE: 258 .space L1_TABLE_SIZE 259 TEMP_L1_TABLE_END: 260 261 .text 262 .align 2 263 264 arm_build_translation_table: 265 push {r0, lr} 266 /* 267 * Initialise the l1pt for identity mapping of the kernel with caches 268 * off. This will be updated further with additional mappings to form 269 * the bootstrap table. 270 */ 271 ldr R_L1TABLE, =TEMP_L1_TABLE 272 sub R_L1TABLE, R_VTOPDIFF 273 274 // PA of kernel rounded down to nearest L1_S boundary 275 adr R_PA, generic_start 276 ldr r0, =(L1_S_SIZE - 1) 277 bic R_PA, R_PA, r0 278 279 // attribute to map kernel 280 ldr R_ATTR, =L1_S_KRW_NOCACHE 281 bl arm_boot_l1pt_init 282 283 /* 284 * Set up a preliminary mapping in the MMU to allow us to run 285 * at KERNEL_BASE_VIRT (determined from generic_start) with caches off. 286 */ 287 ldr R_L1TABLE, =TEMP_L1_TABLE 288 sub R_L1TABLE, R_VTOPDIFF 289 290 // Calculate kernel size plus 1M to grow into 291 ldr r0, =generic_start // kernel start VA 292 ldr r1, =TEMP_L1_TABLE_END // kernel end VA 293 add r1, #L1_S_SIZE // Make sure we have 1M to grow into 294 295 ldr r2, =(L1_S_SIZE - 1) 296 297 bic r0, r2 // trunc kernel start to 1M boundary 298 add r1, r2 299 bic r1, r2 // round kernel end to 1M boundary 300 301 sub r1, r0 // kernel size plus 1M to grow into 302 mov R_NSEC, r1, lsr #(L1_S_SHIFT) 303 304 ldr R_VA, =generic_start // VA of kernel 305 bic R_VA, r2 // ...rounded down to L1_S boundary 306 adr R_PA, generic_start // PA of kernel 307 bic R_PA, r2 // ...rounded down to L1_S boundary 308 309 // attribute to map kernel 310 ldr R_ATTR, =L1_S_KRW_NOCACHE 311 bl arm_boot_l1pt_entry 312 313 #if defined(FDTBASE) 314 VPRINTF("\n\rDTB") 315 316 /* 317 * Add DTB identity mapping (1MB) from originally r2 (but saved in 318 * R_FDTADDR) 319 */ 320 ldr r0, =(L1_S_SIZE - 1) /* align DTB PA to 1M */ 321 bic R_VA, R_FDTADDR, r0 322 mov R_PA, R_VA 323 mov R_NSEC, #1 /* 1MB mapping */ 324 325 ldr R_L1TABLE, =TEMP_L1_TABLE 326 sub R_L1TABLE, R_VTOPDIFF 327 bl arm_boot_l1pt_entry 328 #endif 329 330 #if defined(START_CONSADDR) 331 /* If START_CONSADDR exists add its identity mapping (1MB) */ 332 VPRINTF("\n\rCONSADDR") 333 ldr r0, =(L1_S_SIZE - 1) /* align DTB PA to 1M */ 334 ldr R_VA, =START_CONSADDR 335 bic R_VA, R_VA, r0 336 mov R_PA, R_VA 337 mov R_NSEC, #1 338 339 ldr R_ATTR, =(L1_S_PROTO | L1_S_AP_KRW) 340 orr R_ATTR, R_DEVATTR 341 342 ldr R_L1TABLE, =TEMP_L1_TABLE 343 sub R_L1TABLE, R_VTOPDIFF 344 bl arm_boot_l1pt_entry 345 #endif 346 347 XPUTC(#'M') 348 349 pop {r0, pc} 350 351 .ltorg 352 353 /* 354 * Set up a preliminary mapping in the MMU to allow us to run at 355 * KERNEL_BASE_VIRT (determined from generic_start) 356 * 357 * On Entry 358 * 359 * R_L1TABLE is the PA of the temporary L1PT 360 * R_ATTR is the attribute bits to set for each section mapping 361 * 362 * No R_VA/R_PA/R_NSEC needed here as we use 'generic_start' and 363 * 'TEMP_L1_TABLE_END' to calculate the initial direct VA:PA mapping 364 */ 365 366 /* We push r0 to maintain stack alignment only */ 367 arm_boot_l1pt_init: 368 push {r0, lr} 369 370 // Start address to clear memory. 371 mov r0, R_L1TABLE 372 373 // Zero the entire table so all virtual addresses are invalid. 374 add r1, r0, #L1_TABLE_SIZE // Ending address 375 mov r2, #0 376 mov r3, #0 377 1: stmia r0!, {r2-r3} 378 stmia r0!, {r2-r3} // 16 bytes per loop 379 cmp r0, r1 380 blt 1b 381 382 // Calculate the size of the kernel in L1_S_SIZE sections 383 ldr r0, =generic_start 384 ldr r1, =TEMP_L1_TABLE_END 385 386 ldr r2, =(L1_S_SIZE - 1) 387 388 bic r0, r2 // trunc kernel start to 1M boundary 389 add r1, r2 390 bic r1, r2 // round kernel end to 1M boundary 391 392 sub r1, r0 // kernel size plus 1M to grow into 393 mov R_NSEC, r1, lsr #(L1_S_SHIFT) 394 395 // identity mapping for size of kernel 396 adr R_VA, generic_start 397 bic R_VA, r2 398 mov R_PA, R_VA 399 400 pop {r0, lr} 401 /* Fallthrough */ 402 403 // We push r0 to maintain stack alignment only 404 arm_boot_l1pt_entry: 405 push {r0, lr} 406 407 VPRINTF("\n\r") 408 VPRINTX(R_L1TABLE) 409 410 VPRINTF(" va:") 411 VPRINTX(R_VA) 412 413 VPRINTF(" pa:") 414 VPRINTX(R_PA) 415 416 VPRINTF(" nsec:") 417 VPRINTX(R_NSEC) 418 419 VPRINTF(" attr:") 420 VPRINTX(R_ATTR) 421 422 VPRINTF("\n\r") 423 lsr R_VA, R_VA, #L1_S_SHIFT 424 orr R_PA, R_ATTR 425 426 2: 427 VPRINTX(R_L1TABLE) 428 XPUTC('[') 429 VPRINTX(R_VA) 430 XPUTC(']') 431 XPUTC('=') 432 VPRINTX(R_PA) 433 434 ldr r0, [R_L1TABLE, R_VA, lsl #2] 435 cmp r0, #0 436 cmpne r0, R_PA 437 bne arm_boot_overlap 438 439 str R_PA, [R_L1TABLE, R_VA, lsl #2] 440 add R_VA, R_VA, #1 441 add R_PA, R_PA, #(L1_S_SIZE) 442 443 VPRINTF("\n\r") 444 445 subs R_NSEC, R_NSEC, #1 446 bhi 2b 447 448 pop {r0, pc} 449 450 arm_boot_overlap: 451 VPRINTF("\n\rpanic: overlapping mappings\n\r") 452 3: 453 b 3b 454 455 #if defined(_ARM_ARCH_7) 456 generic_startv7: 457 458 .arch armv7a 459 .arch_extension sec 460 .arch_extension virt 461 462 VPRINTF("v7 : ") 463 464 bl armv7_init 465 bl generic_savevars 466 467 mov R_DEVATTR, #L1_S_V6_XN 468 bl arm_build_translation_table 469 470 /* 471 * Turn on the MMU. Return to virtual address space. 472 */ 473 movw r0, #:lower16:TEMP_L1_TABLE 474 movt r0, #:upper16:TEMP_L1_TABLE 475 sub r0, R_VTOPDIFF 476 477 // Return to virtual address after the call to armv7_mmuinit 478 movw lr, #:lower16:generic_vstartv7 479 movt lr, #:upper16:generic_vstartv7 480 b armv7_mmuinit 481 482 generic_vstartv7: 483 // Stack to KVA address 484 add sp, sp, R_VTOPDIFF 485 486 VPRINTF("virtual\n\r") 487 488 VPRINTF("prrr : ") 489 mrc p15, 0, r0, c10, c2, 0 // Primary Region Remap Register (PRRR) 490 VPRINTX(r0) 491 VPRINTF("\n\rnmrr : ") 492 mrc p15, 0, r0, c10, c2, 1 // Normal Memory Remap Register (NMRR) 493 VPRINTX(r0) 494 VPRINTF("\n\r") 495 496 #if defined(KASAN) 497 ldr r0, =start_stacks_bottom 498 bl _C_LABEL(kasan_early_init) 499 500 VPRINTF("kasan\n\r") 501 #endif 502 503 /* r0 = &cpu_info_store[0] */ 504 movw r0, #:lower16:cpu_info_store 505 movt r0, #:upper16:cpu_info_store 506 507 mrc p15, 0, r1, c0, c0, 0 // MIDR get 508 str r1, [r0, #CI_MIDR] 509 mrc p15, 0, r1, c0, c0, 5 // MPIDR get 510 str r1, [r0, #CI_MPIDR] 511 512 bl arm_cpu_topology_set 513 514 VPRINTF("go\n\r") 515 516 /* 517 * Jump to start in locore.S, which in turn will call initarm and main. 518 */ 519 b start 520 521 /* NOTREACHED */ 522 .ltorg 523 #elif defined(_ARM_ARCH_6) 524 generic_startv6: 525 VPRINTF("v6 : ") 526 527 bl armv6_init 528 bl generic_savevars 529 530 #if defined(ARM_MMU_EXTENDED) 531 mov R_DEVATTR, #L1_S_V6_XN 532 #else 533 mov R_DEVATTR, #0 534 #endif 535 bl arm_build_translation_table 536 537 XPUTC(#'E') 538 539 /* 540 * Turn on the MMU. Return to new enabled address space. 541 */ 542 ldr r0, =TEMP_L1_TABLE 543 sub r0, R_VTOPDIFF 544 545 ldr lr, =generic_vstartv6 546 b armv6_mmuinit 547 548 generic_vstartv6: 549 // Stack to KVA address 550 add sp, sp, R_VTOPDIFF 551 552 VPRINTF("virtual\n\r") 553 554 #if defined(KASAN) 555 ldr r0, =start_stacks_bottom 556 bl _C_LABEL(kasan_early_init) 557 558 VPRINTF("kasan\n\r") 559 #endif 560 561 VPRINTF("go\n\r") 562 563 /* 564 * Jump to start in locore.S, which in turn will call initarm and main. 565 */ 566 b start 567 568 /* NOTREACHED */ 569 .ltorg 570 571 #elif defined(_ARM_ARCH_5) 572 573 generic_startv5: 574 VPRINTF("v5 : ") 575 576 bl armv5_init 577 bl generic_savevars 578 579 mov R_DEVATTR, #0 580 bl arm_build_translation_table 581 582 XPUTC(#'E') 583 584 /* 585 * Turn on the MMU. Return to new enabled address space. 586 */ 587 ldr r0, =TEMP_L1_TABLE 588 sub r0, R_VTOPDIFF 589 590 ldr lr, =generic_vstartv5 591 b armv5_mmuinit 592 593 generic_vstartv5: 594 // convert stack back to virtual address 595 add sp, sp, R_VTOPDIFF 596 597 VPRINTF("virtual\n\r") 598 599 #if defined(KASAN) 600 ldr r0, =start_stacks_bottom 601 bl _C_LABEL(kasan_early_init) 602 603 VPRINTF("kasan\n\r") 604 #endif 605 606 VPRINTF("rgo\n\r") 607 608 //Jump to start in locore.S, which in turn will call initarm and main. 609 b start 610 611 .ltorg // not reached 612 #endif 613 614 #if defined(_ARM_ARCH_7) 615 616 // 617 // SCTLR register initialization values 618 // 619 #if defined(__ARMEL__) 620 #define CPU_CONTROL_EX_BEND_SET 0 621 #else 622 #define CPU_CONTROL_EX_BEND_SET CPU_CONTROL_EX_BEND 623 #endif 624 625 #if defined(ARM32_DISABLE_ALIGNMENT_FAULTS) 626 #define CPU_CONTROL_AFLT_ENABLE_CLR CPU_CONTROL_AFLT_ENABLE 627 #define CPU_CONTROL_AFLT_ENABLE_SET 0 628 #else 629 #define CPU_CONTROL_AFLT_ENABLE_CLR 0 630 #define CPU_CONTROL_AFLT_ENABLE_SET CPU_CONTROL_AFLT_ENABLE 631 #endif 632 633 #if defined(ARM_MMU_EXTENDED) 634 #define CPU_CONTROL_XP_ENABLE_CLR 0 635 #define CPU_CONTROL_XP_ENABLE_SET CPU_CONTROL_XP_ENABLE 636 #else 637 #define CPU_CONTROL_XP_ENABLE_CLR CPU_CONTROL_XP_ENABLE 638 #define CPU_CONTROL_XP_ENABLE_SET 0 639 #endif 640 641 /* SWP is only usable on uni-processor ARMv7 systems. */ 642 #ifdef MULTIPROCESSOR 643 #define CPU_CONTROL_XP_SWP_ENABLE 0 644 #else 645 #define CPU_CONTROL_XP_SWP_ENABLE CPU_CONTROL_SWP_ENABLE 646 #endif 647 648 // bits to set in the Control Register 649 // 650 #define CPU_CONTROL_SET ( \ 651 CPU_CONTROL_MMU_ENABLE | \ 652 CPU_CONTROL_UNAL_ENABLE | \ 653 CPU_CONTROL_EX_BEND_SET | \ 654 CPU_CONTROL_AFLT_ENABLE_SET | \ 655 CPU_CONTROL_XP_ENABLE_SET | \ 656 0) 657 658 // bits to clear in the Control Register 659 // 660 #define CPU_CONTROL_CLR ( \ 661 CPU_CONTROL_AFLT_ENABLE_CLR | \ 662 CPU_CONTROL_XP_ENABLE_CLR | \ 663 0) 664 665 /* 666 * Perform the initialization of the an ARMv7 core required by NetBSD. 667 * 668 * Uses the following callee saved registers: 669 * 670 * r8 (R_TMP1), r9 (R_TMP2) 671 */ 672 armv7_init: 673 674 .arch armv7a 675 .arch_extension sec 676 .arch_extension virt 677 678 mov R_TMP1, lr 679 mov R_TMP2, sp 680 681 /* 682 * Leave HYP mode and move into supervisor mode with IRQs/FIQs 683 * disabled. 684 */ 685 mrs r0, cpsr 686 and r0, r0, #(PSR_MODE) /* Mode is in the low 5 bits of CPSR */ 687 teq r0, #(PSR_HYP32_MODE) /* Hyp Mode? */ 688 bne 1f 689 690 XPUTC('h') 691 692 mov sp, #0 693 694 /* Set CNTVOFF to 0 */ 695 mov r1, #0 696 mcrr p15, 4, r1, r1, c14 697 698 /* Ensure that IRQ, and FIQ will be disabled after eret */ 699 mrs r0, cpsr 700 bic r0, r0, #(PSR_MODE) 701 orr r0, r0, #(PSR_SVC32_MODE) 702 orr r0, r0, #(I32_bit | F32_bit) 703 msr spsr_cxsf, r0 704 /* Exit hypervisor mode */ 705 adr lr, 2f 706 msr elr_hyp, lr 707 eret 708 709 1: 710 cpsid if, #PSR_SVC32_MODE // SVC32 with no interrupts 711 712 2: 713 mov r0, #0 714 msr spsr_sxc, r0 // set SPSR[23:8] to known value 715 716 mov sp, R_TMP2 717 718 XPUTC('A') 719 720 mrc p15, 0, r0, c1, c0, 0 721 tst r0, #CPU_CONTROL_DC_ENABLE 722 blne armv7_dcache_wbinv_all 723 724 // TeX remap 725 726 #define ARMV7_SCTLR_CLEAR ( \ 727 CPU_CONTROL_IC_ENABLE | \ 728 CPU_CONTROL_DC_ENABLE | \ 729 CPU_CONTROL_MMU_ENABLE | \ 730 CPU_CONTROL_BPRD_ENABLE | \ 731 CPU_CONTROL_TR_ENABLE | \ 732 0) 733 734 #define ARMV7_SCTLR_SET ( \ 735 CPU_CONTROL_UNAL_ENABLE | \ 736 CPU_CONTROL_XP_SWP_ENABLE | \ 737 0) 738 739 mrc p15, 0, r0, c1, c0, 0 740 movw r1, #:lower16:ARMV7_SCTLR_CLEAR 741 movt r1, #:upper16:ARMV7_SCTLR_CLEAR 742 movw r2, #:lower16:ARMV7_SCTLR_SET 743 movt r2, #:upper16:ARMV7_SCTLR_SET 744 745 mov R_TMP2, r0 // save for printing 746 bic r0, r0, r1 // disable icache/dcache/mmu 747 orr r0, r0, r2 // enable unaligned access 748 749 mcr p15, 0, r0, c1, c0, 0 // SCTLR write 750 dsb 751 isb 752 753 bl armv7_dcache_inv_all 754 mcr p15, 0, r0, c7, c5, 0 /* ICIALLU */ 755 dsb 756 isb 757 758 #if defined(VERBOSE_INIT_ARM) 759 XPUTC(#'B') 760 761 VPRINTF(" sctlr:") 762 VPRINTX(R_TMP2) 763 VPRINTF("/") 764 mrc p15, 0, r0, c1, c0, 0 765 VPRINTX(r0) 766 VPRINTF(" ") 767 768 XPUTC(#'C') 769 #endif 770 771 bx R_TMP1 // return 772 773 .ltorg 774 775 /* 776 * Transitions the CPU to using the TTB passed in r0. 777 * 778 * Uses the following callee saved registers: 779 * 780 * Callee saved: 781 * r4, r5 782 */ 783 784 armv7_mmuinit: 785 // Because the MMU may already be on do a typical sequence to set 786 // the Translation Table Base(s). 787 mov r4, lr 788 mov r5, r0 // save TTBR 789 790 XPUTC(#'F') 791 dsb // Drain the write buffers. 792 793 XPUTC(#'G') 794 mrc p15, 0, r1, c0, c0, 5 // MPIDR read 795 cmp r1, #0 796 orrlt r5, r5, #TTBR_MPATTR // MP, cachable (Normal WB) 797 orrge r5, r5, #TTBR_UPATTR // Non-MP, cacheable, normal WB 798 799 XPUTC(#'0') 800 mcr p15, 0, r5, c2, c0, 0 // TTBR0 write 801 802 #if defined(ARM_MMU_EXTENDED) 803 // When using split TTBRs, we need to set both since the physical 804 // addresses we were/are using might be in either. 805 XPUTC(#'1') 806 mcr p15, 0, r5, c2, c0, 1 // TTBR1 write 807 #endif 808 809 XPUTC(#'H') 810 #if defined(ARM_MMU_EXTENDED) 811 XPUTC(#'1') 812 mov r1, #TTBCR_S_N_1 // make sure TTBCR_S_N is 1 813 #else 814 XPUTC(#'0') 815 mov r1, #0 // make sure TTBCR is 0 816 #endif 817 mcr p15, 0, r1, c2, c0, 2 // TTBCR write 818 819 XPUTC(#'J') 820 mov r1, #0 // get KERNEL_PID 821 mcr p15, 0, r1, c13, c0, 1 // CONTEXTIDR write 822 823 isb 824 825 // Set the Domain Access register. Very important! 826 XPUTC(#'K') 827 mov r1, #DOMAIN_DEFAULT 828 mcr p15, 0, r1, c3, c0, 0 // DACR write 829 830 #if 0 831 832 /* 833 * Set TEX remap registers 834 * - All is set to uncacheable memory 835 */ 836 ldr r0, =0xAAAAA 837 mcr CP15_PRRR(r0) 838 mov r0, #0 839 mcr CP15_NMRR(r0) 840 #endif 841 842 XPUTC(#'I') 843 mov r1, #0 844 mcr p15, 0, r1, c8, c7, 0 // TLBIALL (just this core) 845 dsb 846 isb 847 848 // 849 // Enable the MMU, etc. 850 // 851 XPUTC(#'L') 852 XPUTC(#'\n') 853 XPUTC(#'\r') 854 mrc p15, 0, r1, c1, c0, 0 // SCTLR read 855 856 movw r3, #:lower16:CPU_CONTROL_SET 857 movt r3, #:upper16:CPU_CONTROL_SET 858 movw r2, #:lower16:CPU_CONTROL_CLR 859 movt r2, #:upper16:CPU_CONTROL_CLR 860 orr r0, r1, r3 861 bic r0, r0, r2 862 863 mcr p15, 0, r0, c1, c0, 0 /* SCTLR write */ 864 865 dsb 866 isb 867 868 mcr p15, 0, r0, c8, c7, 0 /* TLBIALL - Flush TLB */ 869 mcr p15, 0, r0, c7, c5, 6 /* BPIALL - Branch predictor invalidate all */ 870 dsb 871 isb 872 873 VPRINTF("MMU\n\r") 874 bx r4 // return 875 876 .p2align 2 877 878 .text 879 880 ENTRY_NP(cpu_mpstart) 881 #ifndef MULTIPROCESSOR 882 // 883 // If not MULTIPROCESSOR, drop CPU into power saving state. 884 // 885 3: wfi 886 b 3b 887 #else 888 #if defined(__ARMEB__) 889 setend be // switch to BE now 890 #endif 891 892 /* disable IRQs/FIQs. */ 893 cpsid if 894 895 adr R_TMP2, cpu_mpstart 896 ldr R_VTOPDIFF, =cpu_mpstart 897 sub R_VTOPDIFF, R_VTOPDIFF, R_TMP2 898 899 mrc p15, 0, r4, c0, c0, 5 // MPIDR get 900 and r4, #(MPIDR_AFF2|MPIDR_AFF1|MPIDR_AFF0) 901 902 mov r0, #0 903 ldr r1, =cpu_mpidr 904 sub r1, R_VTOPDIFF 905 1: 906 ldr r2, [r1, r0, lsl #2] // r2 = cpu_mpidr[r0] 907 cmp r2, r4 908 beq 2f // found our mpidr 909 910 add r0, #1 911 cmp r0, #MAXCPUS 912 bne 1b 913 914 // Not found our mpidr in the list - use Aff0 for cpuindex 915 and r0, r4, #7 916 2: 917 mov R_INDEX, r0 // save cpu_index for later 918 919 ldr R_TMP1, =start_stacks_top 920 sub sp, R_TMP1, R_VTOPDIFF 921 922 mov r5, R_INDEX 923 lsl r5, #INIT_ARM_STACK_SHIFT 924 sub sp, sp, r5 925 926 #if defined(VERBOSE_INIT_ARM) 927 VPRINTF("\n\rmidr : ") 928 mrc p15, 0, r0, c0, c0, 0 // MIDR 929 VPRINTX(r0) 930 VPRINTF("\n\rrevidr: ") 931 mrc p15, 0, r0, c0, c0, 6 // REVIDR 932 VPRINTX(r0) 933 VPRINTF("\n\rmpidr : ") 934 mrc p15, 0, r0, c0, c0, 5 // MPIDR 935 VPRINTX(r0) 936 #endif 937 VPRINTF("\n\rindex : ") 938 VPRINTX(R_INDEX) 939 VPRINTF("\n\rsp : ") 940 VPRINTX(sp) 941 XPUTC('\n') 942 XPUTC('\r') 943 944 // disables and clears caches 945 bl armv7_init 946 947 movw r0, #:lower16:TEMP_L1_TABLE 948 movt r0, #:upper16:TEMP_L1_TABLE 949 sub r0, R_VTOPDIFF 950 951 movw lr, #:lower16:armv7_mpcontinuation 952 movt lr, #:upper16:armv7_mpcontinuation 953 b armv7_mmuinit 954 ASEND(cpu_mpstart) 955 956 /* 957 * Now running with real kernel VA via bootstrap tables 958 */ 959 armv7_mpcontinuation: 960 // Stack to KVA address 961 add sp, sp, R_VTOPDIFF 962 963 VPRINTF("virtual\n\r") 964 965 // index into cpu_mpidr[] or cpu_number if not found 966 mov r0, R_INDEX 967 bl cpu_init_secondary_processor 968 969 /* 970 * Wait for cpu_boot_secondary_processors 971 */ 972 973 /* r6 = &arm_cpu_mbox[0] */ 974 movw r6, #:lower16:arm_cpu_mbox 975 movt r6, #:upper16:arm_cpu_mbox 976 977 mov r5, #1 // bitmask... 978 lsl r5, R_INDEX // ... for our cpu 979 980 /* wait for the mailbox start bit to become true */ 981 1: ldr r2, [r6] // load mbox 982 dmb // make it a load-acquire 983 tst r2, r5 // is our bit set? 984 wfeeq // no, back to waiting 985 beq 1b // no, and try again 986 987 movw r0, #:lower16:cpu_info 988 movt r0, #:upper16:cpu_info // get pointer to cpu_info 989 ldr r5, [r0, R_INDEX, lsl #2] // load our cpu_info 990 ldr r6, [r5, #CI_IDLELWP] // get the idlelwp 991 ldr r7, [r6, #L_PCB] // now get its pcb 992 ldr sp, [r7, #PCB_KSP] // finally, we can load our SP 993 #if defined(TPIDRPRW_IS_CURCPU) 994 mcr p15, 0, r5, c13, c0, 4 // squirrel away curcpu() 995 #elif defined(TPIDRPRW_IS_CURLWP) 996 mcr p15, 0, r6, c13, c0, 4 // squirrel away curlwp() 997 #else 998 #error either TPIDRPRW_IS_CURCPU or TPIDRPRW_IS_CURLWP must be defined 999 #endif 1000 /* 1001 * No membar needed because we're not switching from a 1002 * previous lwp, and the idle lwp we're switching to can't be 1003 * holding locks already; see cpu_switchto. 1004 */ 1005 str r6, [r5, #CI_CURLWP] // and note we are running on it 1006 1007 mov r0, r5 // pass cpu_info 1008 mov r1, R_INDEX // pass cpu_index 1009 movw r2, #:lower16:MD_CPU_HATCH // pass md_cpu_hatch 1010 movt r2, #:upper16:MD_CPU_HATCH // pass md_cpu_hatch 1011 bl _C_LABEL(cpu_hatch) 1012 b _C_LABEL(idle_loop) // never to return 1013 ASEND(armv7_mpcontinuation) 1014 #endif // MULTIPROCESSOR 1015 1016 #elif defined(_ARM_ARCH_6) 1017 1018 ENTRY_NP(armv6_init) 1019 /* 1020 * Workaround Erratum 411920 1021 * 1022 * - value of arg 'reg' Should Be Zero 1023 */ 1024 #define Invalidate_I_cache(reg) \ 1025 .p2align 5; \ 1026 mcr p15, 0, reg, c7, c5, 0; /* Invalidate Entire I cache */ \ 1027 mcr p15, 0, reg, c7, c5, 0; /* Invalidate Entire I cache */ \ 1028 mcr p15, 0, reg, c7, c5, 0; /* Invalidate Entire I cache */ \ 1029 mcr p15, 0, reg, c7, c5, 0; /* Invalidate Entire I cache */ \ 1030 nop; \ 1031 nop; \ 1032 nop; \ 1033 nop; \ 1034 nop; \ 1035 nop; \ 1036 nop; \ 1037 nop; \ 1038 nop; \ 1039 nop; \ 1040 nop; 1041 1042 mov R_TMP1, lr 1043 mov r0, #0 /* SBZ */ 1044 Invalidate_I_cache(r0) 1045 1046 mcr p15, 0, r0, c7, c14, 0 /* Clean and Invalidate Entire Data Cache */ 1047 1048 ldr r2, =(CPU_CONTROL_IC_ENABLE|CPU_CONTROL_DC_ENABLE) 1049 /* Disable I+D caches */ 1050 mrc p15, 0, r1, c1, c0, 0 /* " " " */ 1051 mov R_TMP2, r1 1052 bic r1, r1, r2 /* " " " */ 1053 mcr p15, 0, r1, c1, c0, 0 /* " " " */ 1054 1055 mcr p15, 0, r0, c7, c10, 4 /* Drain the write buffers. */ 1056 1057 #if defined(VERBOSE_INIT_ARM) 1058 XPUTC(#'B') 1059 1060 VPRINTF(" sctlr:") 1061 VPRINTX(R_TMP2) 1062 VPRINTF("/") 1063 mrc p15, 0, r0, c1, c0, 0 1064 VPRINTX(r0) 1065 VPRINTF(" ") 1066 1067 XPUTC(#'C') 1068 #endif 1069 1070 bx R_TMP1 1071 1072 .ltorg 1073 1074 armv6_mmuinit: 1075 mov r4, lr 1076 mov r5, r0 1077 1078 XPUTC(#'0') 1079 mcr p15, 0, r5, c2, c0, 0 // TTBR0 write 1080 1081 #if defined(ARM_MMU_EXTENDED) 1082 // When using split TTBRs, we need to set both since the physical 1083 // addresses we were/are using might be in either. 1084 XPUTC(#'1') 1085 mcr p15, 0, r5, c2, c0, 1 /* TTBR1 write */ 1086 #endif 1087 1088 XPUTC(#'H') 1089 #if defined(ARM_MMU_EXTENDED) 1090 XPUTC(#'1') 1091 mov r1, #TTBCR_S_N_1 /* make sure TTBCR_S_N is 1 */ 1092 #else 1093 XPUTC(#'0') 1094 mov r1, #0 /* make sure TTBCR is 0 */ 1095 #endif 1096 mcr p15, 0, r1, c2, c0, 2 /* TTBCR write */ 1097 1098 XPUTC(#'I') 1099 1100 mov r0, #0 1101 mcr p15, 0, r0, c8, c7, 0 /* Invalidate TLBs */ 1102 1103 XPUTC(#'K') 1104 /* Set the Domain Access register. Very important! */ 1105 mov r0, #DOMAIN_DEFAULT 1106 mcr p15, 0, r0, c3, c0, 0 1107 1108 /* 1109 * Enable the MMU, etc. 1110 */ 1111 1112 #if defined(VERBOSE_INIT_ARM) 1113 VPRINTF(" sctlr:") 1114 mrc p15, 0, r0, c1, c0, 0 1115 VPRINTX(r0) 1116 VPRINTF("/") 1117 #endif 1118 1119 mrc p15, 0, r0, c1, c0, 0 1120 1121 ldr r1, Lcontrol_wax 1122 and r0, r0, r1 1123 ldr r1, Lcontrol_clr 1124 bic r0, r0, r1 1125 ldr r1, Lcontrol_set 1126 orr r0, r0, r1 1127 1128 mov r6, r0 1129 VPRINTX(r6) 1130 VPRINTF(" ") 1131 1132 .align 5 1133 /* turn mmu on! */ 1134 mov r0, r6 1135 mcr p15, 0, r0, c1, c0, 0 1136 1137 /* 1138 * Ensure that the coprocessor has finished turning on the MMU. 1139 */ 1140 mrc p15, 0, r0, c0, c0, 0 /* Read an arbitrary value. */ 1141 mov r0, r0 /* Stall until read completes. */ 1142 1143 nop 1144 nop 1145 nop 1146 nop 1147 nop 1148 1149 mov pc, r4 1150 1151 .ltorg 1152 1153 /* bits to set in the Control Register */ 1154 Lcontrol_set: 1155 #if defined(ARM_MMU_EXTENDED) 1156 #define CPU_CONTROL_EXTRA CPU_CONTROL_XP_ENABLE 1157 #else 1158 #define CPU_CONTROL_EXTRA CPU_CONTROL_SYST_ENABLE 1159 #endif 1160 #if defined(__ARMEL__) 1161 #define CPU_CONTROL_EX_BEND_SET 0 1162 #else 1163 #define CPU_CONTROL_EX_BEND_SET CPU_CONTROL_EX_BEND 1164 #endif 1165 .word CPU_CONTROL_MMU_ENABLE | \ 1166 CPU_CONTROL_WBUF_ENABLE | /* not defined in 1176 (SBO) */ \ 1167 CPU_CONTROL_32BP_ENABLE | /* SBO */ \ 1168 CPU_CONTROL_32BD_ENABLE | /* SBO */ \ 1169 CPU_CONTROL_LABT_ENABLE | /* SBO */ \ 1170 (1 << 16) | /* SBO - Global enable for data tcm */ \ 1171 (1 << 18) | /* SBO - Global enable for insn tcm */ \ 1172 CPU_CONTROL_UNAL_ENABLE | \ 1173 CPU_CONTROL_EXTRA | \ 1174 CPU_CONTROL_EX_BEND_SET 1175 1176 /* bits to clear in the Control Register */ 1177 Lcontrol_clr: 1178 .word 0 1179 1180 /* bits to "write as existing" in the Control Register */ 1181 Lcontrol_wax: 1182 .word (3 << 30) | \ 1183 (1 << 29) | \ 1184 (1 << 28) | \ 1185 (3 << 26) | \ 1186 (3 << 19) | \ 1187 (1 << 17) | \ 1188 (1 << 10) 1189 #elif defined(_ARM_ARCH_5) 1190 armv5_init: 1191 bx lr 1192 1193 .ltorg 1194 1195 armv5_mmu_enable_bits: 1196 .word CPU_CONTROL_MMU_ENABLE | \ 1197 CPU_CONTROL_WBUF_ENABLE | /* should be one */ \ 1198 CPU_CONTROL_32BP_ENABLE | /* should be one */ \ 1199 CPU_CONTROL_32BD_ENABLE | /* should be one */ \ 1200 CPU_CONTROL_LABT_ENABLE | /* should be one */ \ 1201 (1 << 16) | /* should be one */ \ 1202 (1 << 18) /* should be one */ 1203 1204 1205 armv5_mmuinit: 1206 mov r4, lr 1207 mov r5, r0 // contains the vtop-corrected translation table address 1208 1209 /* write translation table base register */ 1210 mcr p15, 0, r5, c2, c0, 0 1211 VPRINTF("\n\rTTBR : ") 1212 VPRINTX(r5) 1213 1214 /* Invalidate TLBs */ 1215 mov r1, #0 1216 mcr p15, 0, r1, c8, c7, 0 1217 1218 /* Set the Domain Access register. Very important! */ 1219 mov r5, #DOMAIN_DEFAULT 1220 mcr p15, 0, r5, c3, c0, 0 1221 VPRINTF("\n\rDACR : ") 1222 VPRINTX(r5) 1223 1224 /* 1225 * Enable the MMU, etc. 1226 */ 1227 1228 VPRINTF("\n\rSCTLR: ") 1229 #if defined(VERBOSE_INIT_ARM) 1230 mrc p15, 0, r5, c1, c0, 0 // read status+control register 1231 VPRINTX(r5) 1232 VPRINTF("/") 1233 #endif 1234 1235 mrc p15, 0, r0, c1, c0, 0 // read control register 1236 mov r5, r0 1237 ldr r1, armv5_mmu_enable_bits 1238 orr r0, r0, r1 // set MMU bits 1239 1240 mov r6, r0 1241 VPRINTX(r6) 1242 1243 mcr p15, 0, r6, c1, c0, 0 // actually enable mmu 1244 nop // wait for mmu to have turned on 1245 nop 1246 nop 1247 1248 VPRINTF("\n\rMMU") 1249 1250 bx r4 // return 1251 #endif 1252 1253 ENTRY_NP(generic_vprint) 1254 push {r4, lr} 1255 1256 mov r4, lr 1257 b 2f 1258 1: 1259 bl uartputc 1260 1261 2: 1262 ldrb r0, [r4], #1 1263 cmp r0, #0 1264 bne 1b 1265 1266 add lr, r4, #3 1267 bic lr, #3 1268 1269 pop {r4} 1270 add sp, sp, #4 1271 mov pc, lr 1272 ASEND(generic_vprint) 1273 1274 ENTRY_NP(generic_prints) 1275 push {r4, lr} 1276 1277 mov r4, r0 1278 1: 1279 ldrb r0, [r4], #1 1280 cmp r0, #0 1281 popeq {r4, pc} 1282 1283 bl uartputc 1284 b 1b 1285 ASEND(generic_prints) 1286 1287 ENTRY_NP(generic_printx) 1288 push {r4, r5, r6, lr} 1289 mov r5, r0 1290 1291 mov r0, #'0' 1292 bl uartputc 1293 mov r0, #'x' 1294 bl uartputc 1295 1296 // Word size in bits 1297 mov r4, #32 1298 1: 1299 sub r4, r4, #4 // nibble shift 1300 lsr r3, r5, r4 // extract ... 1301 and r3, r3, #0xf // ... nibble 1302 1303 cmp r3, #9 1304 add r0, r3, #'0' 1305 addgt r0, r3, #'a' - 10 1306 bl uartputc 1307 1308 cmp r4, #0 1309 bne 1b 1310 pop {r4, r5, r6, pc} 1311 ASEND(generic_printx) 1312
Indexes created Sun Oct 12 02:09:55 GMT 2025