locore.S revision 1.5
1 /* $NetBSD: locore.S,v 1.5 2017/11/14 13:58:07 maxv Exp $ */ 2 3 /* 4 * Copyright (c) 1998, 2000, 2007, 2008, 2016, 2017 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Charles M. Hannum and by Maxime Villard. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #define _LOCORE 33 34 /* Override user-land alignment before including asm.h */ 35 #define ALIGN_DATA .align 8 36 #define ALIGN_TEXT .align 16,0x90 37 #define _ALIGN_TEXT ALIGN_TEXT 38 39 #include <machine/asm.h> 40 #include <machine/param.h> 41 #include <machine/pte.h> 42 #include <machine/psl.h> 43 #include <machine/segments.h> 44 #include <machine/specialreg.h> 45 #include <machine/trap.h> 46 47 #define _KERNEL 48 #include <machine/bootinfo.h> 49 #undef _KERNEL 50 51 #include "pdir.h" 52 #include "redef.h" 53 54 /* 32bit version of PG_NX */ 55 #define PG_NX32 0x80000000 56 57 #define TABLE_L2_ENTRIES (NKL2_KIMG_ENTRIES + 1) 58 #define TABLE_L3_ENTRIES NKL3_KIMG_ENTRIES 59 60 #define PROC0_PML4_OFF 0 61 #define PROC0_STK_OFF (PROC0_PML4_OFF + 1 * PAGE_SIZE) 62 #define PROC0_PTP3_OFF (PROC0_STK_OFF + UPAGES * PAGE_SIZE) 63 #define PROC0_PTP2_OFF (PROC0_PTP3_OFF + NKL4_KIMG_ENTRIES * PAGE_SIZE) 64 #define PROC0_PTP1_OFF (PROC0_PTP2_OFF + TABLE_L3_ENTRIES * PAGE_SIZE) 65 #define TABLESIZE \ 66 ((NKL4_KIMG_ENTRIES + TABLE_L3_ENTRIES + TABLE_L2_ENTRIES + 1 + UPAGES) \ 67 * PAGE_SIZE) 68 69 /* 70 * fillkpt - Fill in a kernel page table 71 * eax = pte (page frame | control | status) 72 * ebx = page table address 73 * ecx = number of pages to map 74 * 75 * Each entry is 8 (PDE_SIZE) bytes long: we must set the 4 upper bytes to 0. 76 */ 77 #define fillkpt \ 78 cmpl $0,%ecx ; /* zero-sized? */ \ 79 je 2f ; \ 80 1: movl $0,(PDE_SIZE-4)(%ebx) ; /* upper 32 bits: 0 */ \ 81 movl %eax,(%ebx) ; /* store phys addr */ \ 82 addl $PDE_SIZE,%ebx ; /* next PTE/PDE */ \ 83 addl $PAGE_SIZE,%eax ; /* next phys page */ \ 84 loop 1b ; \ 85 2: ; 86 87 /* 88 * fillkpt_nox - Same as fillkpt, but sets the NX/XD bit. 89 */ 90 #define fillkpt_nox \ 91 cmpl $0,%ecx ; /* zero-sized? */ \ 92 je 2f ; \ 93 pushl %ebp ; \ 94 movl _C_LABEL(nox_flag),%ebp ; \ 95 1: movl %ebp,(PDE_SIZE-4)(%ebx) ; /* upper 32 bits: NX */ \ 96 movl %eax,(%ebx) ; /* store phys addr */ \ 97 addl $PDE_SIZE,%ebx ; /* next PTE/PDE */ \ 98 addl $PAGE_SIZE,%eax ; /* next phys page */ \ 99 loop 1b ; \ 100 popl %ebp ; \ 101 2: ; 102 103 /* 104 * fillkpt_blank - Fill in a kernel page table with blank entries 105 * ebx = page table address 106 * ecx = number of pages to map 107 */ 108 #define fillkpt_blank \ 109 cmpl $0,%ecx ; /* zero-sized? */ \ 110 je 2f ; \ 111 1: movl $0,(PDE_SIZE-4)(%ebx) ; /* upper 32 bits: 0 */ \ 112 movl $0,(%ebx) ; /* lower 32 bits: 0 */ \ 113 addl $PDE_SIZE,%ebx ; /* next PTE/PDE */ \ 114 loop 1b ; \ 115 2: ; 116 117 /* 118 * Initialization 119 */ 120 .data 121 122 .globl _C_LABEL(tablesize) 123 .globl _C_LABEL(nox_flag) 124 .globl _C_LABEL(cpuid_level) 125 .globl _C_LABEL(esym) 126 .globl _C_LABEL(eblob) 127 .globl _C_LABEL(atdevbase) 128 .globl _C_LABEL(PDPpaddr) 129 .globl _C_LABEL(boothowto) 130 .globl _C_LABEL(bootinfo) 131 .globl _C_LABEL(biosbasemem) 132 .globl _C_LABEL(biosextmem) 133 .globl _C_LABEL(stkpa) 134 .globl _C_LABEL(stkva) 135 .globl _C_LABEL(kernpa_start) 136 .globl _C_LABEL(kernpa_end) 137 138 .type _C_LABEL(tablesize), @object 139 _C_LABEL(tablesize): .long TABLESIZE 140 END(tablesize) 141 .type _C_LABEL(nox_flag), @object 142 LABEL(nox_flag) .long 0 /* 32bit NOX flag, set if supported */ 143 END(nox_flag) 144 .type _C_LABEL(cpuid_level), @object 145 LABEL(cpuid_level) .long -1 /* max. level accepted by cpuid instr */ 146 END(cpuid_level) 147 .type _C_LABEL(esym), @object 148 LABEL(esym) .quad 0 /* ptr to end of syms */ 149 END(esym) 150 .type _C_LABEL(eblob), @object 151 LABEL(eblob) .quad 0 /* ptr to end of modules */ 152 END(eblob) 153 .type _C_LABEL(atdevbase), @object 154 LABEL(atdevbase) .quad 0 /* location of start of iomem in virt */ 155 END(atdevbase) 156 .type _C_LABEL(PDPpaddr), @object 157 LABEL(PDPpaddr) .quad 0 /* paddr of PTD, for libkvm */ 158 END(PDPpaddr) 159 .type _C_LABEL(biosbasemem), @object 160 LABEL(biosbasemem) .long 0 /* base memory reported by BIOS */ 161 END(biosbasemem) 162 .type _C_LABEL(biosextmem), @object 163 LABEL(biosextmem) .long 0 /* extended memory reported by BIOS */ 164 END(biosextmem) 165 .type _C_LABEL(stkpa), @object 166 LABEL(stkpa) .quad 0 167 END(stkpa) 168 .type _C_LABEL(stkva), @object 169 LABEL(stkva) .quad 0 170 END(stkva) 171 .type _C_LABEL(kernpa_start), @object 172 LABEL(kernpa_start) .quad 0 173 END(kernpa_start) 174 .type _C_LABEL(kernpa_end), @object 175 LABEL(kernpa_end) .quad 0 176 END(kernpa_end) 177 178 .globl gdt64_lo 179 .globl gdt64_start 180 181 #define GDT64_LIMIT gdt64_end-gdt64_start-1 182 /* Temporary gdt64, with base address in low memory */ 183 .type _C_LABEL(gdt64_lo), @object 184 LABEL(gdt64_lo) 185 .word GDT64_LIMIT 186 .quad gdt64_start 187 END(gdt64_lo) 188 .align 64 189 #undef GDT64_LIMIT 190 191 .type _C_LABEL(gdt64_start), @object 192 LABEL(gdt64_start) 193 .quad 0x0000000000000000 /* always empty */ 194 .quad 0x00af9a000000ffff /* kernel CS */ 195 .quad 0x00cf92000000ffff /* kernel DS */ 196 .quad 0x0000000000000000 /* kernel TSS [1/2] */ 197 .quad 0x0000000000000000 /* kernel TSS [2/2] */ 198 END(gdt64_start) 199 gdt64_end: 200 201 .type _C_LABEL(farjmp64), @object 202 _C_LABEL(farjmp64): 203 .long longmode 204 .word GSEL(GCODE_SEL, SEL_KPL) 205 END(farjmp64) 206 207 /* Space for the temporary stack */ 208 .size tmpstk, tmpstk - . 209 .space 512 210 tmpstk: 211 212 .text 213 214 ENTRY(start) 215 .code32 216 217 /* Warm boot */ 218 movw $0x1234,0x472 219 220 /* 221 * Load parameters from the stack (32 bits): 222 * boothowto, [bootdev], bootinfo, esym, biosextmem, biosbasemem 223 * We are not interested in 'bootdev'. 224 */ 225 226 /* Load 'boothowto' */ 227 movl 4(%esp),%eax 228 movl %eax,_C_LABEL(boothowto) 229 230 /* Load 'bootinfo' */ 231 movl 12(%esp),%eax 232 testl %eax,%eax /* bootinfo = NULL? */ 233 jz bootinfo_finished 234 235 movl (%eax),%ebx /* number of entries */ 236 movl $_C_LABEL(bootinfo),%ebp 237 movl %ebp,%edx 238 addl $BOOTINFO_MAXSIZE,%ebp 239 movl %ebx,(%edx) 240 addl $4,%edx 241 242 bootinfo_entryloop: 243 testl %ebx,%ebx /* no remaining entries? */ 244 jz bootinfo_finished 245 246 addl $4,%eax 247 movl (%eax),%ecx /* address of entry */ 248 pushl %edi 249 pushl %esi 250 pushl %eax 251 252 movl (%ecx),%eax /* btinfo_common::len (size of entry) */ 253 movl %edx,%edi 254 addl (%ecx),%edx /* update dest pointer */ 255 cmpl %ebp,%edx /* beyond bootinfo+BOOTINFO_MAXSIZE? */ 256 jg bootinfo_overflow 257 258 movl %ecx,%esi 259 movl %eax,%ecx 260 261 /* If any modules were loaded, record where they end. */ 262 cmpl $BTINFO_MODULELIST,4(%esi) /* btinfo_common::type */ 263 jne 0f 264 pushl 12(%esi) /* btinfo_modulelist::endpa */ 265 popl _C_LABEL(eblob) 266 0: 267 268 /* Record the information about the kernel. */ 269 cmpl $BTINFO_PREKERN,4(%esi) /* btinfo_common::type */ 270 jne 0f 271 pushl 8(%esi) /* btinfo_prekern::kernpa_start */ 272 popl _C_LABEL(kernpa_start) 273 pushl 12(%esi) /* btinfo_prekern::kernpa_end */ 274 popl _C_LABEL(kernpa_end) 275 0: 276 277 rep 278 movsb /* copy esi -> edi */ 279 popl %eax 280 popl %esi 281 popl %edi 282 subl $1,%ebx /* decrement the # of entries */ 283 jmp bootinfo_entryloop 284 285 bootinfo_overflow: 286 /* 287 * Cleanup for overflow case. Pop the registers, and correct the number 288 * of entries. 289 */ 290 popl %eax 291 popl %esi 292 popl %edi 293 movl $_C_LABEL(bootinfo),%ebp 294 movl %ebp,%edx 295 subl %ebx,(%edx) /* correct the number of entries */ 296 297 bootinfo_finished: 298 /* Load 'esym' */ 299 movl 16(%esp),%eax 300 movl $_C_LABEL(esym),%ebp 301 movl %eax,(%ebp) 302 303 /* Load 'biosextmem' */ 304 movl 20(%esp),%eax 305 movl $_C_LABEL(biosextmem),%ebp 306 movl %eax,(%ebp) 307 308 /* Load 'biosbasemem' */ 309 movl 24(%esp),%eax 310 movl $_C_LABEL(biosbasemem),%ebp 311 movl %eax,(%ebp) 312 313 /* 314 * Done with the parameters! 315 */ 316 317 /* First, reset the PSL. */ 318 pushl $PSL_MBO 319 popfl 320 321 /* Switch to new stack now. */ 322 movl $_C_LABEL(tmpstk),%esp 323 324 xorl %eax,%eax 325 cpuid 326 movl %eax,_C_LABEL(cpuid_level) 327 328 /* 329 * Retrieve the NX/XD flag. We use the 32bit version of PG_NX. 330 */ 331 movl $0x80000001,%eax 332 cpuid 333 andl $CPUID_NOX,%edx 334 jz no_NOX 335 movl $PG_NX32,_C_LABEL(nox_flag) 336 no_NOX: 337 338 /* 339 * There are four levels of pages in amd64: PML4 -> PDP -> PD -> PT. They will 340 * be referred to as: L4 -> L3 -> L2 -> L1. 341 * 342 * Physical address space: 343 * +---------------+----------+--------------+--------+---------------------+- 344 * | PREKERN IMAGE |**UNUSED**| KERNEL IMAGE | [SYMS] | [PRELOADED MODULES] | 345 * +---------------+----------+--------------+--------+---------------------+- 346 * (1) (2) (3) (4) 347 * ------------------+ 348 * BOOTSTRAP TABLES | 349 * ------------------+ 350 * (5) 351 * 352 * Virtual address space of the prekern: 353 * +---------------+----------+------------------+-------------+ 354 * | PREKERN IMAGE |**UNUSED**| BOOTSTRAP TABLES | ISA I/O MEM | 355 * +---------------+----------+------------------+-------------+ 356 * 357 * PROC0 STK is obviously not linked as a page level. It just happens to be 358 * caught between L4 and L3. 359 * 360 * (PROC0 STK + L4 + L3 + L2 + L1) is later referred to as BOOTSTRAP TABLES. 361 * 362 * Important note: the kernel segments are properly 4k-aligned 363 * (see kern.ldscript), so there's no need to enforce alignment. 364 */ 365 366 /* Find end of the prekern image; brings us on (1). */ 367 movl $_C_LABEL(__prekern_end),%edi 368 369 /* Find end of the kernel image; brings us on (2). */ 370 movl _C_LABEL(kernpa_end),%eax 371 testl %eax,%eax 372 jz 1f 373 movl %eax,%edi 374 1: 375 376 /* Find end of the kernel symbols; brings us on (3). */ 377 movl _C_LABEL(esym),%eax 378 testl %eax,%eax 379 jz 1f 380 movl %eax,%edi 381 1: 382 383 /* Find end of the kernel preloaded modules; brings us on (4). */ 384 movl _C_LABEL(eblob),%eax 385 testl %eax,%eax 386 jz 1f 387 movl %eax,%edi 388 1: 389 390 /* We are on (3). Align up for BOOTSTRAP TABLES. */ 391 movl %edi,%esi 392 addl $PGOFSET,%esi 393 andl $~PGOFSET,%esi 394 395 /* We are on the BOOTSTRAP TABLES. Save L4's physical address. */ 396 movl $_C_LABEL(PDPpaddr),%ebp 397 movl %esi,(%ebp) 398 movl $0,4(%ebp) 399 400 /* Now, zero out the BOOTSTRAP TABLES (before filling them in). */ 401 movl %esi,%edi 402 xorl %eax,%eax 403 cld 404 movl $TABLESIZE,%ecx 405 shrl $2,%ecx 406 rep 407 stosl /* copy eax -> edi */ 408 409 /* 410 * Build the page tables and levels. We go from L1 to L4, and link the levels 411 * together. 412 */ 413 /* 414 * Build L1. 415 */ 416 leal (PROC0_PTP1_OFF)(%esi),%ebx 417 418 /* Skip the area below the prekern text. */ 419 movl $(PREKERNTEXTOFF - PREKERNBASE),%ecx 420 shrl $PGSHIFT,%ecx 421 fillkpt_blank 422 423 /* Map the prekern text RX. */ 424 movl $(PREKERNTEXTOFF - PREKERNBASE),%eax /* start of TEXT */ 425 movl $_C_LABEL(__rodata_start),%ecx 426 subl %eax,%ecx 427 shrl $PGSHIFT,%ecx 428 orl $(PG_V|PG_KR),%eax 429 fillkpt 430 431 /* Map the prekern rodata R. */ 432 movl $_C_LABEL(__rodata_start),%eax 433 movl $_C_LABEL(__data_start),%ecx 434 subl %eax,%ecx 435 shrl $PGSHIFT,%ecx 436 orl $(PG_V|PG_KR),%eax 437 fillkpt_nox 438 439 /* Map the prekern data+bss RW. */ 440 movl $_C_LABEL(__data_start),%eax 441 movl $_C_LABEL(__prekern_end),%ecx 442 subl %eax,%ecx 443 shrl $PGSHIFT,%ecx 444 orl $(PG_V|PG_KW),%eax 445 fillkpt_nox 446 447 /* Map a RO view of the kernel. */ 448 movl $_C_LABEL(__prekern_end),%eax 449 movl %esi,%ecx /* start of BOOTSTRAP TABLES */ 450 subl %eax,%ecx 451 shrl $PGSHIFT,%ecx 452 orl $(PG_V|PG_KR),%eax 453 fillkpt_nox 454 455 /* Map the BOOTSTRAP TABLES RW. */ 456 movl %esi,%eax /* start of BOOTSTRAP TABLES */ 457 movl $TABLESIZE,%ecx /* length of BOOTSTRAP TABLES */ 458 shrl $PGSHIFT,%ecx 459 orl $(PG_V|PG_KW),%eax 460 fillkpt_nox 461 462 /* Map the ISA I/O MEM RW. */ 463 movl $IOM_BEGIN,%eax 464 movl $IOM_SIZE,%ecx /* size of ISA I/O MEM */ 465 shrl $PGSHIFT,%ecx 466 orl $(PG_V|PG_KW/*|PG_N*/),%eax 467 fillkpt_nox 468 469 /* 470 * Build L2. Linked to L1. 471 */ 472 leal (PROC0_PTP2_OFF)(%esi),%ebx 473 leal (PROC0_PTP1_OFF)(%esi),%eax 474 orl $(PG_V|PG_KW),%eax 475 movl $(NKL2_KIMG_ENTRIES+1),%ecx 476 fillkpt 477 478 /* 479 * Build L3. Linked to L2. 480 */ 481 leal (PROC0_PTP3_OFF)(%esi),%ebx 482 leal (PROC0_PTP2_OFF)(%esi),%eax 483 orl $(PG_V|PG_KW),%eax 484 movl $NKL3_KIMG_ENTRIES,%ecx 485 fillkpt 486 487 /* 488 * Build L4. Linked to L3. 489 */ 490 leal (PROC0_PML4_OFF)(%esi),%ebx 491 leal (PROC0_PTP3_OFF)(%esi),%eax 492 orl $(PG_V|PG_KW),%eax 493 movl $NKL4_KIMG_ENTRIES,%ecx 494 fillkpt 495 496 /* Install recursive top level PDE (one entry) */ 497 leal (PROC0_PML4_OFF + PDIR_SLOT_PTE * PDE_SIZE)(%esi),%ebx 498 leal (PROC0_PML4_OFF)(%esi),%eax 499 orl $(PG_V|PG_KW),%eax 500 movl $1,%ecx 501 fillkpt_nox 502 503 /* 504 * Startup checklist: 505 * 1. Enable PAE (and SSE while here). 506 */ 507 movl %cr4,%eax 508 orl $(CR4_PAE|CR4_OSFXSR|CR4_OSXMMEXCPT),%eax 509 movl %eax,%cr4 510 511 /* 512 * 2. Set Long Mode Enable in EFER. Also enable the syscall extensions, 513 * and NOX if available. 514 */ 515 movl $MSR_EFER,%ecx 516 rdmsr 517 xorl %eax,%eax 518 orl $(EFER_LME|EFER_SCE),%eax 519 movl _C_LABEL(nox_flag),%ebx 520 cmpl $0,%ebx 521 je skip_NOX 522 orl $(EFER_NXE),%eax 523 skip_NOX: 524 wrmsr 525 526 /* 527 * 3. Load %cr3 with pointer to PML4. 528 */ 529 movl %esi,%eax 530 movl %eax,%cr3 531 532 /* 533 * 4. Enable paging and the rest of it. 534 */ 535 movl %cr0,%eax 536 orl $(CR0_PE|CR0_PG|CR0_NE|CR0_TS|CR0_MP|CR0_WP|CR0_AM),%eax 537 movl %eax,%cr0 538 jmp compat 539 compat: 540 541 /* 542 * 5. Not quite done yet, we're now in a compatibility segment, in 543 * legacy mode. We must jump to a long mode segment. Need to set up 544 * a GDT with a long mode segment in it to do that. 545 */ 546 movl $_C_LABEL(gdt64_lo),%eax 547 lgdt (%eax) 548 movl $_C_LABEL(farjmp64),%eax 549 ljmp *(%eax) 550 551 .code64 552 longmode: 553 554 /* 555 * We have arrived. Everything is identity-mapped. 556 */ 557 558 /* Store atdevbase. */ 559 movq $TABLESIZE,%rdx 560 addq %rsi,%rdx 561 movq %rdx,_C_LABEL(atdevbase)(%rip) 562 563 /* Set up bootstrap stack. */ 564 leaq (PROC0_STK_OFF)(%rsi),%rax 565 movq %rax,_C_LABEL(stkpa)(%rip) 566 leaq (USPACE-FRAMESIZE)(%rax),%rsp 567 xorq %rbp,%rbp /* mark end of frames */ 568 569 xorw %ax,%ax 570 movw %ax,%gs 571 movw %ax,%fs 572 573 /* The first physical page available. */ 574 leaq (TABLESIZE)(%rsi),%rdi 575 576 /* 577 * Continue execution in C. 578 */ 579 call _C_LABEL(init_prekern) 580 581 ret 582 END(start) 583 584 /* -------------------------------------------------------------------------- */ 585 586 ENTRY(cpuid) 587 movq %rbx,%r8 588 movq %rdi,%rax 589 movq %rsi,%rcx 590 movq %rdx,%rsi 591 cpuid 592 movl %eax,0(%rsi) 593 movl %ebx,4(%rsi) 594 movl %ecx,8(%rsi) 595 movl %edx,12(%rsi) 596 movq %r8,%rbx 597 ret 598 END(cpuid) 599 600 ENTRY(lidt) 601 lidt (%rdi) 602 ret 603 END(lidt) 604 605 ENTRY(rdtsc) 606 xorq %rax,%rax 607 rdtsc 608 shlq $32,%rdx 609 orq %rdx,%rax 610 ret 611 END(rdtsc) 612 613 ENTRY(rdseed) 614 rdseed %rax 615 jc .Lsuccess 616 movq $(-1),%rax 617 ret 618 .Lsuccess: 619 movq %rax,(%rdi) 620 xorq %rax,%rax 621 ret 622 END(rdseed) 623 624 ENTRY(jump_kernel) 625 movq _C_LABEL(stkva),%rsp 626 xorq %rbp,%rbp 627 callq exec_kernel 628 END(jump_kernel) 629
Indexes created Mon Oct 20 15:10:11 GMT 2025