locore.S revision 1.1
1 /* $NetBSD: locore.S,v 1.1 2017/10/10 09:29:14 maxv Exp $ */ 2 3 /* 4 * Copyright (c) 1998, 2000, 2007, 2008, 2016, 2017 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Charles M. Hannum and by Maxime Villard. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #define _LOCORE 33 34 /* Override user-land alignment before including asm.h */ 35 #define ALIGN_DATA .align 8 36 #define ALIGN_TEXT .align 16,0x90 37 #define _ALIGN_TEXT ALIGN_TEXT 38 39 #include <machine/asm.h> 40 #include <machine/param.h> 41 #include <machine/pte.h> 42 #include <machine/psl.h> 43 #include <machine/segments.h> 44 #include <machine/specialreg.h> 45 #include <machine/trap.h> 46 47 #define _KERNEL 48 #include <machine/bootinfo.h> 49 #undef _KERNEL 50 51 #include "pdir.h" 52 #include "redef.h" 53 54 /* 32bit version of PG_NX */ 55 #define PG_NX32 0x80000000 56 57 #define TABLE_L2_ENTRIES (NKL2_KIMG_ENTRIES + 1) 58 #define TABLE_L3_ENTRIES NKL3_KIMG_ENTRIES 59 60 #define PROC0_PML4_OFF 0 61 #define PROC0_STK_OFF (PROC0_PML4_OFF + 1 * PAGE_SIZE) 62 #define PROC0_PTP3_OFF (PROC0_STK_OFF + UPAGES * PAGE_SIZE) 63 #define PROC0_PTP2_OFF (PROC0_PTP3_OFF + NKL4_KIMG_ENTRIES * PAGE_SIZE) 64 #define PROC0_PTP1_OFF (PROC0_PTP2_OFF + TABLE_L3_ENTRIES * PAGE_SIZE) 65 #define TABLESIZE \ 66 ((NKL4_KIMG_ENTRIES + TABLE_L3_ENTRIES + TABLE_L2_ENTRIES + 1 + UPAGES) \ 67 * PAGE_SIZE) 68 69 /* 70 * fillkpt - Fill in a kernel page table 71 * eax = pte (page frame | control | status) 72 * ebx = page table address 73 * ecx = number of pages to map 74 * 75 * Each entry is 8 (PDE_SIZE) bytes long: we must set the 4 upper bytes to 0. 76 */ 77 #define fillkpt \ 78 cmpl $0,%ecx ; /* zero-sized? */ \ 79 je 2f ; \ 80 1: movl $0,(PDE_SIZE-4)(%ebx) ; /* upper 32 bits: 0 */ \ 81 movl %eax,(%ebx) ; /* store phys addr */ \ 82 addl $PDE_SIZE,%ebx ; /* next PTE/PDE */ \ 83 addl $PAGE_SIZE,%eax ; /* next phys page */ \ 84 loop 1b ; \ 85 2: ; 86 87 /* 88 * fillkpt_nox - Same as fillkpt, but sets the NX/XD bit. 89 */ 90 #define fillkpt_nox \ 91 cmpl $0,%ecx ; /* zero-sized? */ \ 92 je 2f ; \ 93 pushl %ebp ; \ 94 movl _C_LABEL(nox_flag),%ebp ; \ 95 1: movl %ebp,(PDE_SIZE-4)(%ebx) ; /* upper 32 bits: NX */ \ 96 movl %eax,(%ebx) ; /* store phys addr */ \ 97 addl $PDE_SIZE,%ebx ; /* next PTE/PDE */ \ 98 addl $PAGE_SIZE,%eax ; /* next phys page */ \ 99 loop 1b ; \ 100 popl %ebp ; \ 101 2: ; 102 103 /* 104 * fillkpt_blank - Fill in a kernel page table with blank entries 105 * ebx = page table address 106 * ecx = number of pages to map 107 */ 108 #define fillkpt_blank \ 109 cmpl $0,%ecx ; /* zero-sized? */ \ 110 je 2f ; \ 111 1: movl $0,(PDE_SIZE-4)(%ebx) ; /* upper 32 bits: 0 */ \ 112 movl $0,(%ebx) ; /* lower 32 bits: 0 */ \ 113 addl $PDE_SIZE,%ebx ; /* next PTE/PDE */ \ 114 loop 1b ; \ 115 2: ; 116 117 /* 118 * Initialization 119 */ 120 .data 121 122 .globl _C_LABEL(tablesize) 123 .globl _C_LABEL(nox_flag) 124 .globl _C_LABEL(cpuid_level) 125 .globl _C_LABEL(esym) 126 .globl _C_LABEL(eblob) 127 .globl _C_LABEL(atdevbase) 128 .globl _C_LABEL(PDPpaddr) 129 .globl _C_LABEL(boothowto) 130 .globl _C_LABEL(bootinfo) 131 .globl _C_LABEL(biosbasemem) 132 .globl _C_LABEL(biosextmem) 133 .globl _C_LABEL(stkpa) 134 .globl _C_LABEL(stkva) 135 .globl _C_LABEL(kernpa_start) 136 .globl _C_LABEL(kernpa_end) 137 138 .type _C_LABEL(tablesize), @object 139 _C_LABEL(tablesize): .long TABLESIZE 140 END(tablesize) 141 .type _C_LABEL(nox_flag), @object 142 LABEL(nox_flag) .long 0 /* 32bit NOX flag, set if supported */ 143 END(nox_flag) 144 .type _C_LABEL(cpuid_level), @object 145 LABEL(cpuid_level) .long -1 /* max. level accepted by cpuid instr */ 146 END(cpuid_level) 147 .type _C_LABEL(esym), @object 148 LABEL(esym) .quad 0 /* ptr to end of syms */ 149 END(esym) 150 .type _C_LABEL(eblob), @object 151 LABEL(eblob) .quad 0 /* ptr to end of modules */ 152 END(eblob) 153 .type _C_LABEL(atdevbase), @object 154 LABEL(atdevbase) .quad 0 /* location of start of iomem in virt */ 155 END(atdevbase) 156 .type _C_LABEL(PDPpaddr), @object 157 LABEL(PDPpaddr) .quad 0 /* paddr of PTD, for libkvm */ 158 END(PDPpaddr) 159 .type _C_LABEL(biosbasemem), @object 160 LABEL(biosbasemem) .long 0 /* base memory reported by BIOS */ 161 END(biosbasemem) 162 .type _C_LABEL(biosextmem), @object 163 LABEL(biosextmem) .long 0 /* extended memory reported by BIOS */ 164 END(biosextmem) 165 .type _C_LABEL(stkpa), @object 166 LABEL(stkpa) .quad 0 167 END(stkpa) 168 .type _C_LABEL(stkva), @object 169 LABEL(stkva) .quad 0 170 END(stkva) 171 .type _C_LABEL(kernpa_start), @object 172 LABEL(kernpa_start) .quad 0 173 END(kernpa_start) 174 .type _C_LABEL(kernpa_end), @object 175 LABEL(kernpa_end) .quad 0 176 END(kernpa_end) 177 178 .globl gdt64_lo 179 .globl gdt64_start 180 181 #define GDT64_LIMIT gdt64_end-gdt64_start-1 182 /* Temporary gdt64, with base address in low memory */ 183 .type _C_LABEL(gdt64_lo), @object 184 LABEL(gdt64_lo) 185 .word GDT64_LIMIT 186 .quad gdt64_start 187 END(gdt64_lo) 188 .align 64 189 #undef GDT64_LIMIT 190 191 .type _C_LABEL(gdt64_start), @object 192 LABEL(gdt64_start) 193 .quad 0x0000000000000000 /* always empty */ 194 .quad 0x00af9a000000ffff /* kernel CS */ 195 .quad 0x00cf92000000ffff /* kernel DS */ 196 .quad 0x0000000000000000 /* kernel TSS [1/2] */ 197 .quad 0x0000000000000000 /* kernel TSS [2/2] */ 198 END(gdt64_start) 199 gdt64_end: 200 201 .type _C_LABEL(farjmp64), @object 202 _C_LABEL(farjmp64): 203 .long longmode 204 .word GSEL(GCODE_SEL, SEL_KPL) 205 END(farjmp64) 206 207 /* Space for the temporary stack */ 208 .size tmpstk, tmpstk - . 209 .space 512 210 tmpstk: 211 212 .text 213 214 ENTRY(start) 215 .code32 216 217 /* Warm boot */ 218 movw $0x1234,0x472 219 220 /* 221 * Load parameters from the stack (32 bits): 222 * boothowto, [bootdev], bootinfo, esym, biosextmem, biosbasemem 223 * We are not interested in 'bootdev'. 224 */ 225 226 /* Load 'boothowto' */ 227 movl 4(%esp),%eax 228 movl %eax,_C_LABEL(boothowto) 229 230 /* Load 'bootinfo' */ 231 movl 12(%esp),%eax 232 testl %eax,%eax /* bootinfo = NULL? */ 233 jz bootinfo_finished 234 235 movl (%eax),%ebx /* number of entries */ 236 movl $_C_LABEL(bootinfo),%ebp 237 movl %ebp,%edx 238 addl $BOOTINFO_MAXSIZE,%ebp 239 movl %ebx,(%edx) 240 addl $4,%edx 241 242 bootinfo_entryloop: 243 testl %ebx,%ebx /* no remaining entries? */ 244 jz bootinfo_finished 245 246 addl $4,%eax 247 movl (%eax),%ecx /* address of entry */ 248 pushl %edi 249 pushl %esi 250 pushl %eax 251 252 movl (%ecx),%eax /* btinfo_common::len (size of entry) */ 253 movl %edx,%edi 254 addl (%ecx),%edx /* update dest pointer */ 255 cmpl %ebp,%edx /* beyond bootinfo+BOOTINFO_MAXSIZE? */ 256 jg bootinfo_overflow 257 258 movl %ecx,%esi 259 movl %eax,%ecx 260 261 /* If any modules were loaded, record where they end. */ 262 cmpl $BTINFO_MODULELIST,4(%esi) /* btinfo_common::type */ 263 jne 0f 264 pushl 12(%esi) /* btinfo_modulelist::endpa */ 265 popl _C_LABEL(eblob) 266 0: 267 268 /* Record the information about the kernel. */ 269 cmpl $BTINFO_PREKERN,4(%esi) /* btinfo_common::type */ 270 jne 0f 271 pushl 8(%esi) /* btinfo_prekern::kernpa_start */ 272 popl _C_LABEL(kernpa_start) 273 pushl 12(%esi) /* btinfo_prekern::kernpa_end */ 274 popl _C_LABEL(kernpa_end) 275 0: 276 277 rep 278 movsb /* copy esi -> edi */ 279 popl %eax 280 popl %esi 281 popl %edi 282 subl $1,%ebx /* decrement the # of entries */ 283 jmp bootinfo_entryloop 284 285 bootinfo_overflow: 286 /* 287 * Cleanup for overflow case. Pop the registers, and correct the number 288 * of entries. 289 */ 290 popl %eax 291 popl %esi 292 popl %edi 293 movl $_C_LABEL(bootinfo),%ebp 294 movl %ebp,%edx 295 subl %ebx,(%edx) /* correct the number of entries */ 296 297 bootinfo_finished: 298 /* Load 'esym' */ 299 movl 16(%esp),%eax 300 movl $_C_LABEL(esym),%ebp 301 movl %eax,(%ebp) 302 303 /* Load 'biosextmem' */ 304 movl 20(%esp),%eax 305 movl $_C_LABEL(biosextmem),%ebp 306 movl %eax,(%ebp) 307 308 /* Load 'biosbasemem' */ 309 movl 24(%esp),%eax 310 movl $_C_LABEL(biosbasemem),%ebp 311 movl %eax,(%ebp) 312 313 /* 314 * Done with the parameters! 315 */ 316 317 /* First, reset the PSL. */ 318 pushl $PSL_MBO 319 popfl 320 321 /* Switch to new stack now. */ 322 movl $_C_LABEL(tmpstk),%esp 323 324 xorl %eax,%eax 325 cpuid 326 movl %eax,_C_LABEL(cpuid_level) 327 328 /* 329 * Retrieve the NX/XD flag. We use the 32bit version of PG_NX. 330 */ 331 movl $0x80000001,%eax 332 cpuid 333 andl $CPUID_NOX,%edx 334 jz no_NOX 335 movl $PG_NX32,_C_LABEL(nox_flag) 336 no_NOX: 337 338 /* 339 * There are four levels of pages in amd64: PML4 -> PDP -> PD -> PT. They will 340 * be referred to as: L4 -> L3 -> L2 -> L1. 341 * 342 * Physical address space: 343 * +---------------+----------+--------------+--------+---------------------+- 344 * | PREKERN IMAGE |**UNUSED**| KERNEL IMAGE | [SYMS] | [PRELOADED MODULES] | 345 * +---------------+----------+--------------+--------+---------------------+- 346 * (1) (2) (3) (4) 347 * ------------------+ 348 * BOOTSTRAP TABLES | 349 * ------------------+ 350 * (5) 351 * 352 * Virtual address space of the prekern: 353 * +---------------+----------+------------------+-------------+ 354 * | PREKERN IMAGE |**UNUSED**| BOOTSTRAP TABLES | ISA I/O MEM | 355 * +---------------+----------+------------------+-------------+ 356 * 357 * PROC0 STK is obviously not linked as a page level. It just happens to be 358 * caught between L4 and L3. 359 * 360 * (PROC0 STK + L4 + L3 + L2 + L1) is later referred to as BOOTSTRAP TABLES. 361 * 362 * Important note: the kernel segments are properly 4k-aligned 363 * (see kern.ldscript), so there's no need to enforce alignment. 364 */ 365 366 /* Find end of the prekern image; brings us on (1). */ 367 movl $_C_LABEL(__prekern_end),%edi 368 369 /* Find end of the kernel image; brind us on (2). */ 370 movl _C_LABEL(kernpa_end),%eax 371 testl %eax,%eax 372 jz 1f 373 movl %eax,%edi 374 1: 375 376 /* Find end of the kernel symbols; brinds us on (3). */ 377 #if (NKSYMS || defined(DDB) || defined(MODULAR)) && !defined(makeoptions_COPY_SYMTAB) /* XXX */ 378 movl _C_LABEL(esym),%eax 379 testl %eax,%eax 380 jz 1f 381 movl %eax,%edi 382 1: 383 #endif 384 385 /* Find end of the kernel preloaded modules; brings us on (4). */ 386 movl _C_LABEL(eblob),%eax 387 testl %eax,%eax 388 jz 1f 389 movl %eax,%edi 390 1: 391 392 /* We are on (3). Align up for BOOTSTRAP TABLES. */ 393 movl %edi,%esi 394 addl $PGOFSET,%esi 395 andl $~PGOFSET,%esi 396 397 /* We are on the BOOTSTRAP TABLES. Save L4's physical address. */ 398 movl $_C_LABEL(PDPpaddr),%ebp 399 movl %esi,(%ebp) 400 movl $0,4(%ebp) 401 402 /* Now, zero out the BOOTSTRAP TABLES (before filling them in). */ 403 movl %esi,%edi 404 xorl %eax,%eax 405 cld 406 movl $TABLESIZE,%ecx 407 shrl $2,%ecx 408 rep 409 stosl /* copy eax -> edi */ 410 411 /* 412 * Build the page tables and levels. We go from L1 to L4, and link the levels 413 * together. 414 */ 415 /* 416 * Build L1. 417 */ 418 leal (PROC0_PTP1_OFF)(%esi),%ebx 419 420 /* Skip the area below the prekern text. */ 421 movl $(PREKERNTEXTOFF - PREKERNBASE),%ecx 422 shrl $PGSHIFT,%ecx 423 fillkpt_blank 424 425 /* Map the prekern text RX. */ 426 movl $(PREKERNTEXTOFF - PREKERNBASE),%eax /* start of TEXT */ 427 movl $_C_LABEL(__rodata_start),%ecx 428 subl %eax,%ecx 429 shrl $PGSHIFT,%ecx 430 orl $(PG_V|PG_KR),%eax 431 fillkpt 432 433 /* Map the prekern rodata R. */ 434 movl $_C_LABEL(__rodata_start),%eax 435 movl $_C_LABEL(__data_start),%ecx 436 subl %eax,%ecx 437 shrl $PGSHIFT,%ecx 438 orl $(PG_V|PG_KR),%eax 439 fillkpt_nox 440 441 /* Map the prekern data+bss RW. */ 442 movl $_C_LABEL(__data_start),%eax 443 movl $_C_LABEL(__prekern_end),%ecx 444 subl %eax,%ecx 445 shrl $PGSHIFT,%ecx 446 orl $(PG_V|PG_KW),%eax 447 fillkpt_nox 448 449 /* Map some blank space, to keep pa = va. */ 450 movl $_C_LABEL(__prekern_end),%eax 451 movl %esi,%ecx /* start of BOOTSTRAP TABLES */ 452 subl %eax,%ecx 453 shrl $PGSHIFT,%ecx 454 fillkpt_blank 455 456 /* Map the BOOTSTRAP TABLES RW. */ 457 movl %esi,%eax /* start of BOOTSTRAP TABLES */ 458 movl $TABLESIZE,%ecx /* length of BOOTSTRAP TABLES */ 459 shrl $PGSHIFT,%ecx 460 orl $(PG_V|PG_KW),%eax 461 fillkpt_nox 462 463 /* Map the ISA I/O MEM RW. */ 464 movl $IOM_BEGIN,%eax 465 movl $IOM_SIZE,%ecx /* size of ISA I/O MEM */ 466 shrl $PGSHIFT,%ecx 467 orl $(PG_V|PG_KW/*|PG_N*/),%eax 468 fillkpt_nox 469 470 /* 471 * Build L2. Linked to L1. 472 */ 473 leal (PROC0_PTP2_OFF)(%esi),%ebx 474 leal (PROC0_PTP1_OFF)(%esi),%eax 475 orl $(PG_V|PG_KW),%eax 476 movl $(NKL2_KIMG_ENTRIES+1),%ecx 477 fillkpt 478 479 /* 480 * Build L3. Linked to L2. 481 */ 482 leal (PROC0_PTP3_OFF)(%esi),%ebx 483 leal (PROC0_PTP2_OFF)(%esi),%eax 484 orl $(PG_V|PG_KW),%eax 485 movl $NKL3_KIMG_ENTRIES,%ecx 486 fillkpt 487 488 /* 489 * Build L4. Linked to L3. 490 */ 491 leal (PROC0_PML4_OFF)(%esi),%ebx 492 leal (PROC0_PTP3_OFF)(%esi),%eax 493 orl $(PG_V|PG_KW),%eax 494 movl $NKL4_KIMG_ENTRIES,%ecx 495 fillkpt 496 497 /* Install recursive top level PDE (one entry) */ 498 leal (PROC0_PML4_OFF + PDIR_SLOT_PTE * PDE_SIZE)(%esi),%ebx 499 leal (PROC0_PML4_OFF)(%esi),%eax 500 orl $(PG_V|PG_KW),%eax 501 movl $1,%ecx 502 fillkpt_nox 503 504 /* 505 * Startup checklist: 506 * 1. Enable PAE (and SSE while here). 507 */ 508 movl %cr4,%eax 509 orl $(CR4_PAE|CR4_OSFXSR|CR4_OSXMMEXCPT),%eax 510 movl %eax,%cr4 511 512 /* 513 * 2. Set Long Mode Enable in EFER. Also enable the syscall extensions, 514 * and NOX if available. 515 */ 516 movl $MSR_EFER,%ecx 517 rdmsr 518 xorl %eax,%eax 519 orl $(EFER_LME|EFER_SCE),%eax 520 movl _C_LABEL(nox_flag),%ebx 521 cmpl $0,%ebx 522 je skip_NOX 523 orl $(EFER_NXE),%eax 524 skip_NOX: 525 wrmsr 526 527 /* 528 * 3. Load %cr3 with pointer to PML4. 529 */ 530 movl %esi,%eax 531 movl %eax,%cr3 532 533 /* 534 * 4. Enable paging and the rest of it. 535 */ 536 movl %cr0,%eax 537 orl $(CR0_PE|CR0_PG|CR0_NE|CR0_TS|CR0_MP|CR0_WP|CR0_AM),%eax 538 movl %eax,%cr0 539 jmp compat 540 compat: 541 542 /* 543 * 5. Not quite done yet, we're now in a compatibility segment, in 544 * legacy mode. We must jump to a long mode segment. Need to set up 545 * a GDT with a long mode segment in it to do that. 546 */ 547 movl $_C_LABEL(gdt64_lo),%eax 548 lgdt (%eax) 549 movl $_C_LABEL(farjmp64),%eax 550 ljmp *(%eax) 551 552 .code64 553 longmode: 554 555 /* 556 * We have arrived. Everything is identity-mapped. 557 */ 558 559 /* Store atdevbase. */ 560 movq $TABLESIZE,%rdx 561 addq %rsi,%rdx 562 movq %rdx,_C_LABEL(atdevbase)(%rip) 563 564 /* Set up bootstrap stack. */ 565 leaq (PROC0_STK_OFF)(%rsi),%rax 566 movq %rax,_C_LABEL(stkpa)(%rip) 567 leaq (USPACE-FRAMESIZE)(%rax),%rsp 568 xorq %rbp,%rbp /* mark end of frames */ 569 570 xorw %ax,%ax 571 movw %ax,%gs 572 movw %ax,%fs 573 574 /* The first physical page available. */ 575 leaq (TABLESIZE)(%rsi),%rdi 576 577 /* 578 * Continue execution in C. 579 */ 580 call _C_LABEL(init_prekern) 581 582 ret 583 END(start) 584 585 /* -------------------------------------------------------------------------- */ 586 587 ENTRY(lidt) 588 lidt (%rdi) 589 ret 590 591 ENTRY(rdtsc) 592 xorq %rax,%rax 593 rdtsc 594 shlq $32,%rdx 595 orq %rdx,%rax 596 ret 597 598 ENTRY(jump_kernel) 599 movq _C_LABEL(stkva),%rsp 600 movq $exec_kernel,%rax 601 jmpq *%rax 602 603
Indexes created Wed Oct 22 06:10:02 GMT 2025