locore.S revision 1.7 1 /* $NetBSD: locore.S,v 1.7 2017/12/22 07:37:27 maxv Exp $ */
2
3 /*
4 * Copyright (c) 1998, 2000, 2007, 2008, 2016, 2017 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Charles M. Hannum and by Maxime Villard.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 #define _LOCORE
33
34 /* Override user-land alignment before including asm.h */
35 #define ALIGN_DATA .align 8
36 #define ALIGN_TEXT .align 16,0x90
37 #define _ALIGN_TEXT ALIGN_TEXT
38
39 #include <machine/asm.h>
40 #include <machine/param.h>
41 #include <machine/pte.h>
42 #include <machine/psl.h>
43 #include <machine/segments.h>
44 #include <machine/specialreg.h>
45 #include <machine/trap.h>
46
47 #define _KERNEL
48 #include <machine/bootinfo.h>
49 #undef _KERNEL
50
51 #include "pdir.h"
52 #include "redef.h"
53
54 /* 32bit version of PG_NX */
55 #define PG_NX32 0x80000000
56
57 #define TABLE_L2_ENTRIES (NKL2_KIMG_ENTRIES + 1)
58 #define TABLE_L3_ENTRIES NKL3_KIMG_ENTRIES
59
60 #define PROC0_PML4_OFF 0
61 #define PROC0_STK_OFF (PROC0_PML4_OFF + 1 * PAGE_SIZE)
62 #define PROC0_PTP3_OFF (PROC0_STK_OFF + UPAGES * PAGE_SIZE)
63 #define PROC0_PTP2_OFF (PROC0_PTP3_OFF + NKL4_KIMG_ENTRIES * PAGE_SIZE)
64 #define PROC0_PTP1_OFF (PROC0_PTP2_OFF + TABLE_L3_ENTRIES * PAGE_SIZE)
65 #define TABLESIZE \
66 ((NKL4_KIMG_ENTRIES + TABLE_L3_ENTRIES + TABLE_L2_ENTRIES + 1 + UPAGES) \
67 * PAGE_SIZE)
68
69 /*
70 * fillkpt - Fill in a kernel page table
71 * eax = pte (page frame | control | status)
72 * ebx = page table address
73 * ecx = number of pages to map
74 *
75 * Each entry is 8 (PDE_SIZE) bytes long: we must set the 4 upper bytes to 0.
76 */
77 #define fillkpt \
78 cmpl $0,%ecx ; /* zero-sized? */ \
79 je 2f ; \
80 1: movl $0,(PDE_SIZE-4)(%ebx) ; /* upper 32 bits: 0 */ \
81 movl %eax,(%ebx) ; /* store phys addr */ \
82 addl $PDE_SIZE,%ebx ; /* next PTE/PDE */ \
83 addl $PAGE_SIZE,%eax ; /* next phys page */ \
84 loop 1b ; \
85 2: ;
86
87 /*
88 * fillkpt_nox - Same as fillkpt, but sets the NX/XD bit.
89 */
90 #define fillkpt_nox \
91 cmpl $0,%ecx ; /* zero-sized? */ \
92 je 2f ; \
93 pushl %ebp ; \
94 movl _C_LABEL(nox_flag),%ebp ; \
95 1: movl %ebp,(PDE_SIZE-4)(%ebx) ; /* upper 32 bits: NX */ \
96 movl %eax,(%ebx) ; /* store phys addr */ \
97 addl $PDE_SIZE,%ebx ; /* next PTE/PDE */ \
98 addl $PAGE_SIZE,%eax ; /* next phys page */ \
99 loop 1b ; \
100 popl %ebp ; \
101 2: ;
102
103 /*
104 * fillkpt_blank - Fill in a kernel page table with blank entries
105 * ebx = page table address
106 * ecx = number of pages to map
107 */
108 #define fillkpt_blank \
109 cmpl $0,%ecx ; /* zero-sized? */ \
110 je 2f ; \
111 1: movl $0,(PDE_SIZE-4)(%ebx) ; /* upper 32 bits: 0 */ \
112 movl $0,(%ebx) ; /* lower 32 bits: 0 */ \
113 addl $PDE_SIZE,%ebx ; /* next PTE/PDE */ \
114 loop 1b ; \
115 2: ;
116
117 /*
118 * Initialization
119 */
120 .data
121
122 .globl _C_LABEL(tablesize)
123 .globl _C_LABEL(nox_flag)
124 .globl _C_LABEL(cpuid_level)
125 .globl _C_LABEL(esym)
126 .globl _C_LABEL(eblob)
127 .globl _C_LABEL(atdevbase)
128 .globl _C_LABEL(PDPpaddr)
129 .globl _C_LABEL(boothowto)
130 .globl _C_LABEL(bootinfo)
131 .globl _C_LABEL(biosbasemem)
132 .globl _C_LABEL(biosextmem)
133 .globl _C_LABEL(stkpa)
134 .globl _C_LABEL(stkva)
135 .globl _C_LABEL(kernpa_start)
136 .globl _C_LABEL(kernpa_end)
137
138 .type _C_LABEL(tablesize), @object
139 _C_LABEL(tablesize): .long TABLESIZE
140 END(tablesize)
141 .type _C_LABEL(nox_flag), @object
142 LABEL(nox_flag) .long 0 /* 32bit NOX flag, set if supported */
143 END(nox_flag)
144 .type _C_LABEL(cpuid_level), @object
145 LABEL(cpuid_level) .long -1 /* max. level accepted by cpuid instr */
146 END(cpuid_level)
147 .type _C_LABEL(esym), @object
148 LABEL(esym) .quad 0 /* ptr to end of syms */
149 END(esym)
150 .type _C_LABEL(eblob), @object
151 LABEL(eblob) .quad 0 /* ptr to end of modules */
152 END(eblob)
153 .type _C_LABEL(atdevbase), @object
154 LABEL(atdevbase) .quad 0 /* location of start of iomem in virt */
155 END(atdevbase)
156 .type _C_LABEL(PDPpaddr), @object
157 LABEL(PDPpaddr) .quad 0 /* paddr of PTD, for libkvm */
158 END(PDPpaddr)
159 .type _C_LABEL(biosbasemem), @object
160 LABEL(biosbasemem) .long 0 /* base memory reported by BIOS */
161 END(biosbasemem)
162 .type _C_LABEL(biosextmem), @object
163 LABEL(biosextmem) .long 0 /* extended memory reported by BIOS */
164 END(biosextmem)
165 .type _C_LABEL(stkpa), @object
166 LABEL(stkpa) .quad 0
167 END(stkpa)
168 .type _C_LABEL(stkva), @object
169 LABEL(stkva) .quad 0
170 END(stkva)
171 .type _C_LABEL(kernpa_start), @object
172 LABEL(kernpa_start) .quad 0
173 END(kernpa_start)
174 .type _C_LABEL(kernpa_end), @object
175 LABEL(kernpa_end) .quad 0
176 END(kernpa_end)
177
178 .globl gdt64_lo
179 .globl gdt64_start
180
181 #define GDT64_LIMIT gdt64_end-gdt64_start-1
182 /* Temporary gdt64, with base address in low memory */
183 .type _C_LABEL(gdt64_lo), @object
184 LABEL(gdt64_lo)
185 .word GDT64_LIMIT
186 .quad gdt64_start
187 END(gdt64_lo)
188 .align 64
189 #undef GDT64_LIMIT
190
191 .type _C_LABEL(gdt64_start), @object
192 LABEL(gdt64_start)
193 .quad 0x0000000000000000 /* always empty */
194 .quad 0x00af9a000000ffff /* kernel CS */
195 .quad 0x00cf92000000ffff /* kernel DS */
196 .quad 0x0000000000000000 /* kernel TSS [1/2] */
197 .quad 0x0000000000000000 /* kernel TSS [2/2] */
198 END(gdt64_start)
199 gdt64_end:
200
201 .type _C_LABEL(farjmp64), @object
202 _C_LABEL(farjmp64):
203 .long longmode
204 .word GSEL(GCODE_SEL, SEL_KPL)
205 END(farjmp64)
206
207 /* Space for the temporary stack */
208 .size tmpstk, tmpstk - .
209 .space 512
210 tmpstk:
211
212 .text
213
214 ENTRY(start)
215 .code32
216
217 /* Warm boot */
218 movw $0x1234,0x472
219
220 /*
221 * Load parameters from the stack (32 bits):
222 * boothowto, [bootdev], bootinfo, esym, biosextmem, biosbasemem
223 * We are not interested in 'bootdev'.
224 */
225
226 /* Load 'boothowto' */
227 movl 4(%esp),%eax
228 movl %eax,_C_LABEL(boothowto)
229
230 /* Load 'bootinfo' */
231 movl 12(%esp),%eax
232 testl %eax,%eax /* bootinfo = NULL? */
233 jz bootinfo_finished
234
235 movl (%eax),%ebx /* number of entries */
236 movl $_C_LABEL(bootinfo),%ebp
237 movl %ebp,%edx
238 addl $BOOTINFO_MAXSIZE,%ebp
239 movl %ebx,(%edx)
240 addl $4,%edx
241
242 bootinfo_entryloop:
243 testl %ebx,%ebx /* no remaining entries? */
244 jz bootinfo_finished
245
246 addl $4,%eax
247 movl (%eax),%ecx /* address of entry */
248 pushl %edi
249 pushl %esi
250 pushl %eax
251
252 movl (%ecx),%eax /* btinfo_common::len (size of entry) */
253 movl %edx,%edi
254 addl (%ecx),%edx /* update dest pointer */
255 cmpl %ebp,%edx /* beyond bootinfo+BOOTINFO_MAXSIZE? */
256 jg bootinfo_overflow
257
258 movl %ecx,%esi
259 movl %eax,%ecx
260
261 /* If any modules were loaded, record where they end. */
262 cmpl $BTINFO_MODULELIST,4(%esi) /* btinfo_common::type */
263 jne 0f
264 pushl 12(%esi) /* btinfo_modulelist::endpa */
265 popl _C_LABEL(eblob)
266 0:
267
268 /* Record the information about the kernel. */
269 cmpl $BTINFO_PREKERN,4(%esi) /* btinfo_common::type */
270 jne 0f
271 pushl 8(%esi) /* btinfo_prekern::kernpa_start */
272 popl _C_LABEL(kernpa_start)
273 pushl 12(%esi) /* btinfo_prekern::kernpa_end */
274 popl _C_LABEL(kernpa_end)
275 0:
276
277 rep
278 movsb /* copy esi -> edi */
279 popl %eax
280 popl %esi
281 popl %edi
282 subl $1,%ebx /* decrement the # of entries */
283 jmp bootinfo_entryloop
284
285 bootinfo_overflow:
286 /*
287 * Cleanup for overflow case. Pop the registers, and correct the number
288 * of entries.
289 */
290 popl %eax
291 popl %esi
292 popl %edi
293 movl $_C_LABEL(bootinfo),%ebp
294 movl %ebp,%edx
295 subl %ebx,(%edx) /* correct the number of entries */
296
297 bootinfo_finished:
298 /* Load 'esym' */
299 movl 16(%esp),%eax
300 movl $_C_LABEL(esym),%ebp
301 movl %eax,(%ebp)
302
303 /* Load 'biosextmem' */
304 movl 20(%esp),%eax
305 movl $_C_LABEL(biosextmem),%ebp
306 movl %eax,(%ebp)
307
308 /* Load 'biosbasemem' */
309 movl 24(%esp),%eax
310 movl $_C_LABEL(biosbasemem),%ebp
311 movl %eax,(%ebp)
312
313 /*
314 * Done with the parameters!
315 */
316
317 /* First, reset the PSL. */
318 pushl $PSL_MBO
319 popfl
320
321 /* Switch to new stack now. */
322 movl $_C_LABEL(tmpstk),%esp
323
324 xorl %eax,%eax
325 cpuid
326 movl %eax,_C_LABEL(cpuid_level)
327
328 /*
329 * Retrieve the NX/XD flag. We use the 32bit version of PG_NX.
330 */
331 movl $0x80000001,%eax
332 cpuid
333 andl $CPUID_NOX,%edx
334 jz no_NOX
335 movl $PG_NX32,_C_LABEL(nox_flag)
336 no_NOX:
337
338 /*
339 * There are four levels of pages in amd64: PML4 -> PDP -> PD -> PT. They will
340 * be referred to as: L4 -> L3 -> L2 -> L1.
341 *
342 * Physical address space:
343 * +---------------+----------+--------------+--------+---------------------+-
344 * | PREKERN IMAGE |**UNUSED**| KERNEL IMAGE | [SYMS] | [PRELOADED MODULES] |
345 * +---------------+----------+--------------+--------+---------------------+-
346 * (1) (2) (3) (4)
347 * ------------------+
348 * BOOTSTRAP TABLES |
349 * ------------------+
350 * (5)
351 *
352 * The virtual address space is the same, since it is identity-mapped (va = pa).
353 * However, the KERNEL IMAGE is mapped as read-only: the prekern reads it, but
354 * won't write to it. (Needed when relocating the kernel.)
355 *
356 * PROC0 STK is obviously not linked as a page level. It just happens to be
357 * caught between L4 and L3.
358 *
359 * (PROC0 STK + L4 + L3 + L2 + L1) is later referred to as BOOTSTRAP TABLES.
360 *
361 * Important note: the prekern segments are properly 4k-aligned
362 * (see prekern.ldscript), so there's no need to enforce alignment.
363 */
364
365 /* Find end of the prekern image; brings us on (1). */
366 movl $_C_LABEL(__prekern_end),%edi
367
368 /* Find end of the kernel image; brings us on (2). */
369 movl _C_LABEL(kernpa_end),%eax
370 testl %eax,%eax
371 jz 1f
372 movl %eax,%edi
373 1:
374
375 /* Find end of the kernel symbols; brings us on (3). */
376 movl _C_LABEL(esym),%eax
377 testl %eax,%eax
378 jz 1f
379 movl %eax,%edi
380 1:
381
382 /* Find end of the kernel preloaded modules; brings us on (4). */
383 movl _C_LABEL(eblob),%eax
384 testl %eax,%eax
385 jz 1f
386 movl %eax,%edi
387 1:
388
389 /* We are on (3). Align up for BOOTSTRAP TABLES. */
390 movl %edi,%esi
391 addl $PGOFSET,%esi
392 andl $~PGOFSET,%esi
393
394 /* We are on the BOOTSTRAP TABLES. Save L4's physical address. */
395 movl $_C_LABEL(PDPpaddr),%ebp
396 movl %esi,(%ebp)
397 movl $0,4(%ebp)
398
399 /* Now, zero out the BOOTSTRAP TABLES (before filling them in). */
400 movl %esi,%edi
401 xorl %eax,%eax
402 cld
403 movl $TABLESIZE,%ecx
404 shrl $2,%ecx
405 rep
406 stosl /* copy eax -> edi */
407
408 /*
409 * Build the page tables and levels. We go from L1 to L4, and link the levels
410 * together.
411 */
412 /*
413 * Build L1.
414 */
415 leal (PROC0_PTP1_OFF)(%esi),%ebx
416
417 /* Skip the area below the prekern text. */
418 movl $(PREKERNTEXTOFF - PREKERNBASE),%ecx
419 shrl $PGSHIFT,%ecx
420 fillkpt_blank
421
422 /* Map the prekern text RX. */
423 movl $(PREKERNTEXTOFF - PREKERNBASE),%eax /* start of TEXT */
424 movl $_C_LABEL(__rodata_start),%ecx
425 subl %eax,%ecx
426 shrl $PGSHIFT,%ecx
427 orl $(PG_V|PG_KR),%eax
428 fillkpt
429
430 /* Map the prekern rodata R. */
431 movl $_C_LABEL(__rodata_start),%eax
432 movl $_C_LABEL(__data_start),%ecx
433 subl %eax,%ecx
434 shrl $PGSHIFT,%ecx
435 orl $(PG_V|PG_KR),%eax
436 fillkpt_nox
437
438 /* Map the prekern data+bss RW. */
439 movl $_C_LABEL(__data_start),%eax
440 movl $_C_LABEL(__prekern_end),%ecx
441 subl %eax,%ecx
442 shrl $PGSHIFT,%ecx
443 orl $(PG_V|PG_KW),%eax
444 fillkpt_nox
445
446 /* Map a RO view of the kernel. */
447 movl $_C_LABEL(__prekern_end),%eax
448 movl %esi,%ecx /* start of BOOTSTRAP TABLES */
449 subl %eax,%ecx
450 shrl $PGSHIFT,%ecx
451 orl $(PG_V|PG_KR),%eax
452 fillkpt_nox
453
454 /* Map the BOOTSTRAP TABLES RW. */
455 movl %esi,%eax /* start of BOOTSTRAP TABLES */
456 movl $TABLESIZE,%ecx /* length of BOOTSTRAP TABLES */
457 shrl $PGSHIFT,%ecx
458 orl $(PG_V|PG_KW),%eax
459 fillkpt_nox
460
461 /* Map the ISA I/O MEM RW. */
462 movl $IOM_BEGIN,%eax
463 movl $IOM_SIZE,%ecx /* size of ISA I/O MEM */
464 shrl $PGSHIFT,%ecx
465 orl $(PG_V|PG_KW/*|PG_N*/),%eax
466 fillkpt_nox
467
468 /*
469 * Build L2. Linked to L1.
470 */
471 leal (PROC0_PTP2_OFF)(%esi),%ebx
472 leal (PROC0_PTP1_OFF)(%esi),%eax
473 orl $(PG_V|PG_KW),%eax
474 movl $(NKL2_KIMG_ENTRIES+1),%ecx
475 fillkpt
476
477 /*
478 * Build L3. Linked to L2.
479 */
480 leal (PROC0_PTP3_OFF)(%esi),%ebx
481 leal (PROC0_PTP2_OFF)(%esi),%eax
482 orl $(PG_V|PG_KW),%eax
483 movl $NKL3_KIMG_ENTRIES,%ecx
484 fillkpt
485
486 /*
487 * Build L4. Linked to L3.
488 */
489 leal (PROC0_PML4_OFF)(%esi),%ebx
490 leal (PROC0_PTP3_OFF)(%esi),%eax
491 orl $(PG_V|PG_KW),%eax
492 movl $NKL4_KIMG_ENTRIES,%ecx
493 fillkpt
494
495 /* Install recursive top level PDE (one entry) */
496 leal (PROC0_PML4_OFF + PDIR_SLOT_PTE * PDE_SIZE)(%esi),%ebx
497 leal (PROC0_PML4_OFF)(%esi),%eax
498 orl $(PG_V|PG_KW),%eax
499 movl $1,%ecx
500 fillkpt_nox
501
502 /*
503 * Startup checklist:
504 * 1. Enable PAE (and SSE while here).
505 */
506 movl %cr4,%eax
507 orl $(CR4_PAE|CR4_OSFXSR|CR4_OSXMMEXCPT),%eax
508 movl %eax,%cr4
509
510 /*
511 * 2. Set Long Mode Enable in EFER. Also enable the syscall extensions,
512 * and NOX if available.
513 */
514 movl $MSR_EFER,%ecx
515 rdmsr
516 xorl %eax,%eax
517 orl $(EFER_LME|EFER_SCE),%eax
518 movl _C_LABEL(nox_flag),%ebx
519 cmpl $0,%ebx
520 je skip_NOX
521 orl $(EFER_NXE),%eax
522 skip_NOX:
523 wrmsr
524
525 /*
526 * 3. Load %cr3 with pointer to PML4.
527 */
528 movl %esi,%eax
529 movl %eax,%cr3
530
531 /*
532 * 4. Enable paging and the rest of it.
533 */
534 movl %cr0,%eax
535 orl $(CR0_PE|CR0_PG|CR0_NE|CR0_TS|CR0_MP|CR0_WP|CR0_AM),%eax
536 movl %eax,%cr0
537 jmp compat
538 compat:
539
540 /*
541 * 5. Not quite done yet, we're now in a compatibility segment, in
542 * legacy mode. We must jump to a long mode segment. Need to set up
543 * a GDT with a long mode segment in it to do that.
544 */
545 movl $_C_LABEL(gdt64_lo),%eax
546 lgdt (%eax)
547 movl $_C_LABEL(farjmp64),%eax
548 ljmp *(%eax)
549
550 .code64
551 longmode:
552
553 /*
554 * We have arrived. Everything is identity-mapped.
555 */
556
557 /* Store atdevbase. */
558 movq $TABLESIZE,%rdx
559 addq %rsi,%rdx
560 movq %rdx,_C_LABEL(atdevbase)(%rip)
561
562 /* Set up bootstrap stack. */
563 leaq (PROC0_STK_OFF)(%rsi),%rax
564 movq %rax,_C_LABEL(stkpa)(%rip)
565 leaq (USPACE-FRAMESIZE)(%rax),%rsp
566 xorq %rbp,%rbp /* mark end of frames */
567
568 xorw %ax,%ax
569 movw %ax,%gs
570 movw %ax,%fs
571
572 /* The first physical page available. */
573 leaq (TABLESIZE)(%rsi),%rdi
574
575 /*
576 * Continue execution in C.
577 */
578 call _C_LABEL(init_prekern)
579
580 ret
581 END(start)
582
583 /* -------------------------------------------------------------------------- */
584
585 ENTRY(cpuid)
586 movq %rbx,%r8
587 movq %rdi,%rax
588 movq %rsi,%rcx
589 movq %rdx,%rsi
590 cpuid
591 movl %eax,0(%rsi)
592 movl %ebx,4(%rsi)
593 movl %ecx,8(%rsi)
594 movl %edx,12(%rsi)
595 movq %r8,%rbx
596 ret
597 END(cpuid)
598
599 ENTRY(lidt)
600 lidt (%rdi)
601 ret
602 END(lidt)
603
604 ENTRY(rdtsc)
605 xorq %rax,%rax
606 rdtsc
607 shlq $32,%rdx
608 orq %rdx,%rax
609 ret
610 END(rdtsc)
611
612 ENTRY(rdseed)
613 rdseed %rax
614 jc .Lrdseed_success
615 movq $(-1),%rax
616 ret
617 .Lrdseed_success:
618 movq %rax,(%rdi)
619 xorq %rax,%rax
620 ret
621 END(rdseed)
622
623 ENTRY(rdrand)
624 rdrand %rax
625 jc .Lrdrand_success
626 movq $(-1),%rax
627 ret
628 .Lrdrand_success:
629 movq %rax,(%rdi)
630 xorq %rax,%rax
631 ret
632 END(rdrand)
633
634 ENTRY(jump_kernel)
635 movq _C_LABEL(stkva),%rsp
636 xorq %rbp,%rbp
637 callq exec_kernel
638 END(jump_kernel)
639