locore.S revision 1.8.2.1 1 /* $NetBSD: locore.S,v 1.8.2.1 2019/06/10 22:05:47 christos Exp $ */
2
3 /*
4 * Copyright (c) 1998, 2000, 2007, 2008, 2016, 2017 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Charles M. Hannum and by Maxime Villard.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 #define _LOCORE
33
34 /* Override user-land alignment before including asm.h */
35 #define ALIGN_DATA .align 8
36 #define ALIGN_TEXT .align 16,0x90
37 #define _ALIGN_TEXT ALIGN_TEXT
38
39 #include <machine/asm.h>
40 #include <machine/param.h>
41 #include <machine/pte.h>
42 #include <machine/psl.h>
43 #include <machine/segments.h>
44 #include <machine/specialreg.h>
45 #include <machine/trap.h>
46
47 #define _KERNEL
48 #include <machine/bootinfo.h>
49 #undef _KERNEL
50
51 #include "pdir.h"
52 #include "redef.h"
53
54 /* 32bit version of PTE_NX */
55 #define PTE_NX32 0x80000000
56
57 #define TABLE_L2_ENTRIES (NKL2_KIMG_ENTRIES + 1)
58 #define TABLE_L3_ENTRIES NKL3_KIMG_ENTRIES
59
60 #define PROC0_PML4_OFF 0
61 #define PROC0_STK_OFF (PROC0_PML4_OFF + 1 * PAGE_SIZE)
62 #define PROC0_PTP3_OFF (PROC0_STK_OFF + UPAGES * PAGE_SIZE)
63 #define PROC0_PTP2_OFF (PROC0_PTP3_OFF + NKL4_KIMG_ENTRIES * PAGE_SIZE)
64 #define PROC0_PTP1_OFF (PROC0_PTP2_OFF + TABLE_L3_ENTRIES * PAGE_SIZE)
65 #define TABLESIZE \
66 ((NKL4_KIMG_ENTRIES + TABLE_L3_ENTRIES + TABLE_L2_ENTRIES + 1 + UPAGES) \
67 * PAGE_SIZE)
68
69 /*
70 * fillkpt - Fill in a kernel page table
71 * eax = pte (page frame | control | status)
72 * ebx = page table address
73 * ecx = number of pages to map
74 *
75 * Each entry is 8 (PDE_SIZE) bytes long: we must set the 4 upper bytes to 0.
76 */
77 #define fillkpt \
78 cmpl $0,%ecx ; /* zero-sized? */ \
79 je 2f ; \
80 1: movl $0,(PDE_SIZE-4)(%ebx) ; /* upper 32 bits: 0 */ \
81 movl %eax,(%ebx) ; /* store phys addr */ \
82 addl $PDE_SIZE,%ebx ; /* next PTE/PDE */ \
83 addl $PAGE_SIZE,%eax ; /* next phys page */ \
84 loop 1b ; \
85 2: ;
86
87 /*
88 * fillkpt_nox - Same as fillkpt, but sets the NX/XD bit.
89 */
90 #define fillkpt_nox \
91 cmpl $0,%ecx ; /* zero-sized? */ \
92 je 2f ; \
93 pushl %ebp ; \
94 movl _C_LABEL(nox_flag),%ebp ; \
95 1: movl %ebp,(PDE_SIZE-4)(%ebx) ; /* upper 32 bits: NX */ \
96 movl %eax,(%ebx) ; /* store phys addr */ \
97 addl $PDE_SIZE,%ebx ; /* next PTE/PDE */ \
98 addl $PAGE_SIZE,%eax ; /* next phys page */ \
99 loop 1b ; \
100 popl %ebp ; \
101 2: ;
102
103 /*
104 * fillkpt_blank - Fill in a kernel page table with blank entries
105 * ebx = page table address
106 * ecx = number of pages to map
107 */
108 #define fillkpt_blank \
109 cmpl $0,%ecx ; /* zero-sized? */ \
110 je 2f ; \
111 1: movl $0,(PDE_SIZE-4)(%ebx) ; /* upper 32 bits: 0 */ \
112 movl $0,(%ebx) ; /* lower 32 bits: 0 */ \
113 addl $PDE_SIZE,%ebx ; /* next PTE/PDE */ \
114 loop 1b ; \
115 2: ;
116
117 /*
118 * Initialization
119 */
120 .data
121
122 .globl _C_LABEL(tablesize)
123 .globl _C_LABEL(nox_flag)
124 .globl _C_LABEL(cpuid_level)
125 .globl _C_LABEL(esym)
126 .globl _C_LABEL(eblob)
127 .globl _C_LABEL(atdevbase)
128 .globl _C_LABEL(PDPpaddr)
129 .globl _C_LABEL(boothowto)
130 .globl _C_LABEL(bootinfo)
131 .globl _C_LABEL(biosbasemem)
132 .globl _C_LABEL(biosextmem)
133 .globl _C_LABEL(stkpa)
134 .globl _C_LABEL(stkva)
135 .globl _C_LABEL(kernpa_start)
136 .globl _C_LABEL(kernpa_end)
137
138 .type _C_LABEL(tablesize), @object
139 _C_LABEL(tablesize): .long TABLESIZE
140 END(tablesize)
141 .type _C_LABEL(nox_flag), @object
142 LABEL(nox_flag) .long 0 /* 32bit NOX flag, set if supported */
143 END(nox_flag)
144 .type _C_LABEL(cpuid_level), @object
145 LABEL(cpuid_level) .long -1 /* max. level accepted by cpuid instr */
146 END(cpuid_level)
147 .type _C_LABEL(esym), @object
148 LABEL(esym) .quad 0 /* ptr to end of syms */
149 END(esym)
150 .type _C_LABEL(eblob), @object
151 LABEL(eblob) .quad 0 /* ptr to end of modules */
152 END(eblob)
153 .type _C_LABEL(atdevbase), @object
154 LABEL(atdevbase) .quad 0 /* location of start of iomem in virt */
155 END(atdevbase)
156 .type _C_LABEL(PDPpaddr), @object
157 LABEL(PDPpaddr) .quad 0 /* paddr of PTD, for libkvm */
158 END(PDPpaddr)
159 .type _C_LABEL(biosbasemem), @object
160 LABEL(biosbasemem) .long 0 /* base memory reported by BIOS */
161 END(biosbasemem)
162 .type _C_LABEL(biosextmem), @object
163 LABEL(biosextmem) .long 0 /* extended memory reported by BIOS */
164 END(biosextmem)
165 .type _C_LABEL(stkpa), @object
166 LABEL(stkpa) .quad 0
167 END(stkpa)
168 .type _C_LABEL(stkva), @object
169 LABEL(stkva) .quad 0
170 END(stkva)
171 .type _C_LABEL(kernpa_start), @object
172 LABEL(kernpa_start) .quad 0
173 END(kernpa_start)
174 .type _C_LABEL(kernpa_end), @object
175 LABEL(kernpa_end) .quad 0
176 END(kernpa_end)
177
178 .globl gdt64_lo
179 .globl gdt64_start
180
181 #define GDT64_LIMIT gdt64_end-gdt64_start-1
182 /* Temporary gdt64, with base address in low memory */
183 .type _C_LABEL(gdt64_lo), @object
184 LABEL(gdt64_lo)
185 .word GDT64_LIMIT
186 .quad gdt64_start
187 END(gdt64_lo)
188 .align 64
189 #undef GDT64_LIMIT
190
191 .type _C_LABEL(gdt64_start), @object
192 LABEL(gdt64_start)
193 .quad 0x0000000000000000 /* always empty */
194 .quad 0x00af9a000000ffff /* kernel CS */
195 .quad 0x00cf92000000ffff /* kernel DS */
196 END(gdt64_start)
197 gdt64_end:
198
199 .type _C_LABEL(farjmp64), @object
200 _C_LABEL(farjmp64):
201 .long longmode
202 .word GSEL(GCODE_SEL, SEL_KPL)
203 END(farjmp64)
204
205 /* Space for the temporary stack */
206 .size tmpstk, tmpstk - .
207 .space 512
208 tmpstk:
209
210 .text
211
212 ENTRY(start)
213 .code32
214
215 /* Warm boot */
216 movw $0x1234,0x472
217
218 /*
219 * Load parameters from the stack (32 bits):
220 * boothowto, [bootdev], bootinfo, esym, biosextmem, biosbasemem
221 * We are not interested in 'bootdev'.
222 */
223
224 /* Load 'boothowto' */
225 movl 4(%esp),%eax
226 movl %eax,_C_LABEL(boothowto)
227
228 /* Load 'bootinfo' */
229 movl 12(%esp),%eax
230 testl %eax,%eax /* bootinfo = NULL? */
231 jz .Lbootinfo_finished
232
233 movl (%eax),%ebx /* number of entries */
234 movl $_C_LABEL(bootinfo),%ebp
235 movl %ebp,%edx
236 addl $BOOTINFO_MAXSIZE,%ebp
237 movl %ebx,(%edx)
238 addl $4,%edx
239
240 .Lbootinfo_entryloop:
241 testl %ebx,%ebx /* no remaining entries? */
242 jz .Lbootinfo_finished
243
244 addl $4,%eax
245 movl (%eax),%ecx /* address of entry */
246 pushl %edi
247 pushl %esi
248 pushl %eax
249
250 movl (%ecx),%eax /* btinfo_common::len (size of entry) */
251 movl %edx,%edi
252 addl (%ecx),%edx /* update dest pointer */
253 cmpl %ebp,%edx /* beyond bootinfo+BOOTINFO_MAXSIZE? */
254 jg .Lbootinfo_overflow
255
256 movl %ecx,%esi
257 movl %eax,%ecx
258
259 /* If any modules were loaded, record where they end. */
260 cmpl $BTINFO_MODULELIST,4(%esi) /* btinfo_common::type */
261 jne 0f
262 pushl 12(%esi) /* btinfo_modulelist::endpa */
263 popl _C_LABEL(eblob)
264 0:
265
266 /* Record the information about the kernel. */
267 cmpl $BTINFO_PREKERN,4(%esi) /* btinfo_common::type */
268 jne 0f
269 pushl 8(%esi) /* btinfo_prekern::kernpa_start */
270 popl _C_LABEL(kernpa_start)
271 pushl 12(%esi) /* btinfo_prekern::kernpa_end */
272 popl _C_LABEL(kernpa_end)
273 0:
274
275 rep
276 movsb /* copy esi -> edi */
277 popl %eax
278 popl %esi
279 popl %edi
280 subl $1,%ebx /* decrement the # of entries */
281 jmp .Lbootinfo_entryloop
282
283 .Lbootinfo_overflow:
284 /*
285 * Cleanup for overflow case. Pop the registers, and correct the number
286 * of entries.
287 */
288 popl %eax
289 popl %esi
290 popl %edi
291 movl $_C_LABEL(bootinfo),%ebp
292 movl %ebp,%edx
293 subl %ebx,(%edx) /* correct the number of entries */
294
295 .Lbootinfo_finished:
296 /* Load 'esym' */
297 movl 16(%esp),%eax
298 movl $_C_LABEL(esym),%ebp
299 movl %eax,(%ebp)
300
301 /* Load 'biosextmem' */
302 movl 20(%esp),%eax
303 movl $_C_LABEL(biosextmem),%ebp
304 movl %eax,(%ebp)
305
306 /* Load 'biosbasemem' */
307 movl 24(%esp),%eax
308 movl $_C_LABEL(biosbasemem),%ebp
309 movl %eax,(%ebp)
310
311 /*
312 * Done with the parameters!
313 */
314
315 /* First, reset the PSL. */
316 pushl $PSL_MBO
317 popfl
318
319 /* Switch to new stack now. */
320 movl $_C_LABEL(tmpstk),%esp
321
322 xorl %eax,%eax
323 cpuid
324 movl %eax,_C_LABEL(cpuid_level)
325
326 /*
327 * Retrieve the NX/XD flag. We use the 32bit version of PTE_NX.
328 */
329 movl $0x80000001,%eax
330 cpuid
331 andl $CPUID_NOX,%edx
332 jz .Lno_NOX
333 movl $PTE_NX32,_C_LABEL(nox_flag)
334 .Lno_NOX:
335
336 /*
337 * There are four levels of pages in amd64: PML4 -> PDP -> PD -> PT. They will
338 * be referred to as: L4 -> L3 -> L2 -> L1.
339 *
340 * Physical address space:
341 * +---------------+----------+--------------+--------+---------------------+-
342 * | PREKERN IMAGE |**UNUSED**| KERNEL IMAGE | [SYMS] | [PRELOADED MODULES] |
343 * +---------------+----------+--------------+--------+---------------------+-
344 * (1) (2) (3) (4)
345 * ------------------+
346 * BOOTSTRAP TABLES |
347 * ------------------+
348 * (5)
349 *
350 * The virtual address space is the same, since it is identity-mapped (va = pa).
351 * However, the KERNEL IMAGE is mapped as read-only: the prekern reads it, but
352 * won't write to it. (Needed when relocating the kernel.)
353 *
354 * PROC0 STK is obviously not linked as a page level. It just happens to be
355 * caught between L4 and L3.
356 *
357 * (PROC0 STK + L4 + L3 + L2 + L1) is later referred to as BOOTSTRAP TABLES.
358 *
359 * Important note: the prekern segments are properly 4k-aligned
360 * (see prekern.ldscript), so there's no need to enforce alignment.
361 */
362
363 /* Find end of the prekern image; brings us on (1). */
364 movl $_C_LABEL(__prekern_end),%edi
365
366 /* Find end of the kernel image; brings us on (2). */
367 movl _C_LABEL(kernpa_end),%eax
368 testl %eax,%eax
369 jz 1f
370 movl %eax,%edi
371 1:
372
373 /* Find end of the kernel symbols; brings us on (3). */
374 movl _C_LABEL(esym),%eax
375 testl %eax,%eax
376 jz 1f
377 movl %eax,%edi
378 1:
379
380 /* Find end of the kernel preloaded modules; brings us on (4). */
381 movl _C_LABEL(eblob),%eax
382 testl %eax,%eax
383 jz 1f
384 movl %eax,%edi
385 1:
386
387 /* We are on (3). Align up for BOOTSTRAP TABLES. */
388 movl %edi,%esi
389 addl $PGOFSET,%esi
390 andl $~PGOFSET,%esi
391
392 /* We are on the BOOTSTRAP TABLES. Save L4's physical address. */
393 movl $_C_LABEL(PDPpaddr),%ebp
394 movl %esi,(%ebp)
395 movl $0,4(%ebp)
396
397 /* Now, zero out the BOOTSTRAP TABLES (before filling them in). */
398 movl %esi,%edi
399 xorl %eax,%eax
400 cld
401 movl $TABLESIZE,%ecx
402 shrl $2,%ecx
403 rep
404 stosl /* copy eax -> edi */
405
406 /*
407 * Build the page tables and levels. We go from L1 to L4, and link the levels
408 * together.
409 */
410 /*
411 * Build L1.
412 */
413 leal (PROC0_PTP1_OFF)(%esi),%ebx
414
415 /* Skip the area below the prekern text. */
416 movl $(PREKERNTEXTOFF - PREKERNBASE),%ecx
417 shrl $PGSHIFT,%ecx
418 fillkpt_blank
419
420 /* Map the prekern text RX. */
421 movl $(PREKERNTEXTOFF - PREKERNBASE),%eax /* start of TEXT */
422 movl $_C_LABEL(__rodata_start),%ecx
423 subl %eax,%ecx
424 shrl $PGSHIFT,%ecx
425 orl $(PTE_P),%eax
426 fillkpt
427
428 /* Map the prekern rodata R. */
429 movl $_C_LABEL(__rodata_start),%eax
430 movl $_C_LABEL(__data_start),%ecx
431 subl %eax,%ecx
432 shrl $PGSHIFT,%ecx
433 orl $(PTE_P),%eax
434 fillkpt_nox
435
436 /* Map the prekern data+bss RW. */
437 movl $_C_LABEL(__data_start),%eax
438 movl $_C_LABEL(__prekern_end),%ecx
439 subl %eax,%ecx
440 shrl $PGSHIFT,%ecx
441 orl $(PTE_P|PTE_W),%eax
442 fillkpt_nox
443
444 /* Map a RO view of the kernel. */
445 movl $_C_LABEL(__prekern_end),%eax
446 movl %esi,%ecx /* start of BOOTSTRAP TABLES */
447 subl %eax,%ecx
448 shrl $PGSHIFT,%ecx
449 orl $(PTE_P),%eax
450 fillkpt_nox
451
452 /* Map the BOOTSTRAP TABLES RW. */
453 movl %esi,%eax /* start of BOOTSTRAP TABLES */
454 movl $TABLESIZE,%ecx /* length of BOOTSTRAP TABLES */
455 shrl $PGSHIFT,%ecx
456 orl $(PTE_P|PTE_W),%eax
457 fillkpt_nox
458
459 /* Map the ISA I/O MEM RW. */
460 movl $IOM_BEGIN,%eax
461 movl $IOM_SIZE,%ecx /* size of ISA I/O MEM */
462 shrl $PGSHIFT,%ecx
463 orl $(PTE_P|PTE_W/*|PTE_PCD*/),%eax
464 fillkpt_nox
465
466 /*
467 * Build L2. Linked to L1.
468 */
469 leal (PROC0_PTP2_OFF)(%esi),%ebx
470 leal (PROC0_PTP1_OFF)(%esi),%eax
471 orl $(PTE_P|PTE_W),%eax
472 movl $(NKL2_KIMG_ENTRIES+1),%ecx
473 fillkpt
474
475 /*
476 * Build L3. Linked to L2.
477 */
478 leal (PROC0_PTP3_OFF)(%esi),%ebx
479 leal (PROC0_PTP2_OFF)(%esi),%eax
480 orl $(PTE_P|PTE_W),%eax
481 movl $NKL3_KIMG_ENTRIES,%ecx
482 fillkpt
483
484 /*
485 * Build L4. Linked to L3.
486 */
487 leal (PROC0_PML4_OFF)(%esi),%ebx
488 leal (PROC0_PTP3_OFF)(%esi),%eax
489 orl $(PTE_P|PTE_W),%eax
490 movl $NKL4_KIMG_ENTRIES,%ecx
491 fillkpt
492
493 /* Install recursive top level PDE (one entry) */
494 leal (PROC0_PML4_OFF + PDIR_SLOT_PTE * PDE_SIZE)(%esi),%ebx
495 leal (PROC0_PML4_OFF)(%esi),%eax
496 orl $(PTE_P|PTE_W),%eax
497 movl $1,%ecx
498 fillkpt_nox
499
500 /*
501 * Startup checklist:
502 * 1. Enable PAE (and SSE while here).
503 */
504 movl %cr4,%eax
505 orl $(CR4_PAE|CR4_OSFXSR|CR4_OSXMMEXCPT),%eax
506 movl %eax,%cr4
507
508 /*
509 * 2. Set Long Mode Enable in EFER. Also enable the syscall extensions,
510 * and NOX if available.
511 */
512 movl $MSR_EFER,%ecx
513 rdmsr
514 xorl %eax,%eax
515 orl $(EFER_LME|EFER_SCE),%eax
516 movl _C_LABEL(nox_flag),%ebx
517 cmpl $0,%ebx
518 je .Lskip_NOX
519 orl $(EFER_NXE),%eax
520 .Lskip_NOX:
521 wrmsr
522
523 /*
524 * 3. Load %cr3 with pointer to PML4.
525 */
526 movl %esi,%eax
527 movl %eax,%cr3
528
529 /*
530 * 4. Enable paging and the rest of it.
531 */
532 movl %cr0,%eax
533 orl $(CR0_PE|CR0_PG|CR0_NE|CR0_TS|CR0_MP|CR0_WP|CR0_AM),%eax
534 movl %eax,%cr0
535 jmp compat
536 compat:
537
538 /*
539 * 5. Not quite done yet, we're now in a compatibility segment, in
540 * legacy mode. We must jump to a long mode segment. Need to set up
541 * a GDT with a long mode segment in it to do that.
542 */
543 movl $_C_LABEL(gdt64_lo),%eax
544 lgdt (%eax)
545 movl $_C_LABEL(farjmp64),%eax
546 ljmp *(%eax)
547
548 .code64
549 longmode:
550
551 /*
552 * We have arrived. Everything is identity-mapped.
553 */
554
555 /* Store atdevbase. */
556 movq $TABLESIZE,%rdx
557 addq %rsi,%rdx
558 movq %rdx,_C_LABEL(atdevbase)(%rip)
559
560 /* Set up bootstrap stack. */
561 leaq (PROC0_STK_OFF)(%rsi),%rax
562 movq %rax,_C_LABEL(stkpa)(%rip)
563 leaq (USPACE-FRAMESIZE)(%rax),%rsp
564 xorq %rbp,%rbp /* mark end of frames */
565
566 xorw %ax,%ax
567 movw %ax,%gs
568 movw %ax,%fs
569
570 movw $GSEL(GDATA_SEL, SEL_KPL),%ax
571 movw %ax,%ss
572
573 /* The first physical page available. */
574 leaq (TABLESIZE)(%rsi),%rdi
575
576 /*
577 * Continue execution in C.
578 */
579 call _C_LABEL(init_prekern)
580
581 ret
582 END(start)
583
584 /* -------------------------------------------------------------------------- */
585
586 ENTRY(cpuid)
587 movq %rbx,%r8
588 movq %rdi,%rax
589 movq %rsi,%rcx
590 movq %rdx,%rsi
591 cpuid
592 movl %eax,0(%rsi)
593 movl %ebx,4(%rsi)
594 movl %ecx,8(%rsi)
595 movl %edx,12(%rsi)
596 movq %r8,%rbx
597 ret
598 END(cpuid)
599
600 ENTRY(lidt)
601 lidt (%rdi)
602 ret
603 END(lidt)
604
605 ENTRY(rdtsc)
606 xorq %rax,%rax
607 rdtsc
608 shlq $32,%rdx
609 orq %rdx,%rax
610 ret
611 END(rdtsc)
612
613 ENTRY(rdseed)
614 rdseed %rax
615 jc .Lrdseed_success
616 movq $(-1),%rax
617 ret
618 .Lrdseed_success:
619 movq %rax,(%rdi)
620 xorq %rax,%rax
621 ret
622 END(rdseed)
623
624 ENTRY(rdrand)
625 rdrand %rax
626 jc .Lrdrand_success
627 movq $(-1),%rax
628 ret
629 .Lrdrand_success:
630 movq %rax,(%rdi)
631 xorq %rax,%rax
632 ret
633 END(rdrand)
634
635 ENTRY(jump_kernel)
636 movq _C_LABEL(stkva),%rsp
637 xorq %rbp,%rbp
638 callq exec_kernel
639 END(jump_kernel)
640