locore.S revision 1.6.2.2 1 /* $NetBSD: locore.S,v 1.6.2.2 2017/12/03 11:35:48 jdolecek Exp $ */
2
3 /*
4 * Copyright (c) 1998, 2000, 2007, 2008, 2016, 2017 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Charles M. Hannum and by Maxime Villard.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 #define _LOCORE
33
34 /* Override user-land alignment before including asm.h */
35 #define ALIGN_DATA .align 8
36 #define ALIGN_TEXT .align 16,0x90
37 #define _ALIGN_TEXT ALIGN_TEXT
38
39 #include <machine/asm.h>
40 #include <machine/param.h>
41 #include <machine/pte.h>
42 #include <machine/psl.h>
43 #include <machine/segments.h>
44 #include <machine/specialreg.h>
45 #include <machine/trap.h>
46
47 #define _KERNEL
48 #include <machine/bootinfo.h>
49 #undef _KERNEL
50
51 #include "pdir.h"
52 #include "redef.h"
53
54 /* 32bit version of PG_NX */
55 #define PG_NX32 0x80000000
56
57 #define TABLE_L2_ENTRIES (NKL2_KIMG_ENTRIES + 1)
58 #define TABLE_L3_ENTRIES NKL3_KIMG_ENTRIES
59
60 #define PROC0_PML4_OFF 0
61 #define PROC0_STK_OFF (PROC0_PML4_OFF + 1 * PAGE_SIZE)
62 #define PROC0_PTP3_OFF (PROC0_STK_OFF + UPAGES * PAGE_SIZE)
63 #define PROC0_PTP2_OFF (PROC0_PTP3_OFF + NKL4_KIMG_ENTRIES * PAGE_SIZE)
64 #define PROC0_PTP1_OFF (PROC0_PTP2_OFF + TABLE_L3_ENTRIES * PAGE_SIZE)
65 #define TABLESIZE \
66 ((NKL4_KIMG_ENTRIES + TABLE_L3_ENTRIES + TABLE_L2_ENTRIES + 1 + UPAGES) \
67 * PAGE_SIZE)
68
69 /*
70 * fillkpt - Fill in a kernel page table
71 * eax = pte (page frame | control | status)
72 * ebx = page table address
73 * ecx = number of pages to map
74 *
75 * Each entry is 8 (PDE_SIZE) bytes long: we must set the 4 upper bytes to 0.
76 */
77 #define fillkpt \
78 cmpl $0,%ecx ; /* zero-sized? */ \
79 je 2f ; \
80 1: movl $0,(PDE_SIZE-4)(%ebx) ; /* upper 32 bits: 0 */ \
81 movl %eax,(%ebx) ; /* store phys addr */ \
82 addl $PDE_SIZE,%ebx ; /* next PTE/PDE */ \
83 addl $PAGE_SIZE,%eax ; /* next phys page */ \
84 loop 1b ; \
85 2: ;
86
87 /*
88 * fillkpt_nox - Same as fillkpt, but sets the NX/XD bit.
89 */
90 #define fillkpt_nox \
91 cmpl $0,%ecx ; /* zero-sized? */ \
92 je 2f ; \
93 pushl %ebp ; \
94 movl _C_LABEL(nox_flag),%ebp ; \
95 1: movl %ebp,(PDE_SIZE-4)(%ebx) ; /* upper 32 bits: NX */ \
96 movl %eax,(%ebx) ; /* store phys addr */ \
97 addl $PDE_SIZE,%ebx ; /* next PTE/PDE */ \
98 addl $PAGE_SIZE,%eax ; /* next phys page */ \
99 loop 1b ; \
100 popl %ebp ; \
101 2: ;
102
103 /*
104 * fillkpt_blank - Fill in a kernel page table with blank entries
105 * ebx = page table address
106 * ecx = number of pages to map
107 */
108 #define fillkpt_blank \
109 cmpl $0,%ecx ; /* zero-sized? */ \
110 je 2f ; \
111 1: movl $0,(PDE_SIZE-4)(%ebx) ; /* upper 32 bits: 0 */ \
112 movl $0,(%ebx) ; /* lower 32 bits: 0 */ \
113 addl $PDE_SIZE,%ebx ; /* next PTE/PDE */ \
114 loop 1b ; \
115 2: ;
116
117 /*
118 * Initialization
119 */
120 .data
121
122 .globl _C_LABEL(tablesize)
123 .globl _C_LABEL(nox_flag)
124 .globl _C_LABEL(cpuid_level)
125 .globl _C_LABEL(esym)
126 .globl _C_LABEL(eblob)
127 .globl _C_LABEL(atdevbase)
128 .globl _C_LABEL(PDPpaddr)
129 .globl _C_LABEL(boothowto)
130 .globl _C_LABEL(bootinfo)
131 .globl _C_LABEL(biosbasemem)
132 .globl _C_LABEL(biosextmem)
133 .globl _C_LABEL(stkpa)
134 .globl _C_LABEL(stkva)
135 .globl _C_LABEL(kernpa_start)
136 .globl _C_LABEL(kernpa_end)
137
138 .type _C_LABEL(tablesize), @object
139 _C_LABEL(tablesize): .long TABLESIZE
140 END(tablesize)
141 .type _C_LABEL(nox_flag), @object
142 LABEL(nox_flag) .long 0 /* 32bit NOX flag, set if supported */
143 END(nox_flag)
144 .type _C_LABEL(cpuid_level), @object
145 LABEL(cpuid_level) .long -1 /* max. level accepted by cpuid instr */
146 END(cpuid_level)
147 .type _C_LABEL(esym), @object
148 LABEL(esym) .quad 0 /* ptr to end of syms */
149 END(esym)
150 .type _C_LABEL(eblob), @object
151 LABEL(eblob) .quad 0 /* ptr to end of modules */
152 END(eblob)
153 .type _C_LABEL(atdevbase), @object
154 LABEL(atdevbase) .quad 0 /* location of start of iomem in virt */
155 END(atdevbase)
156 .type _C_LABEL(PDPpaddr), @object
157 LABEL(PDPpaddr) .quad 0 /* paddr of PTD, for libkvm */
158 END(PDPpaddr)
159 .type _C_LABEL(biosbasemem), @object
160 LABEL(biosbasemem) .long 0 /* base memory reported by BIOS */
161 END(biosbasemem)
162 .type _C_LABEL(biosextmem), @object
163 LABEL(biosextmem) .long 0 /* extended memory reported by BIOS */
164 END(biosextmem)
165 .type _C_LABEL(stkpa), @object
166 LABEL(stkpa) .quad 0
167 END(stkpa)
168 .type _C_LABEL(stkva), @object
169 LABEL(stkva) .quad 0
170 END(stkva)
171 .type _C_LABEL(kernpa_start), @object
172 LABEL(kernpa_start) .quad 0
173 END(kernpa_start)
174 .type _C_LABEL(kernpa_end), @object
175 LABEL(kernpa_end) .quad 0
176 END(kernpa_end)
177
178 .globl gdt64_lo
179 .globl gdt64_start
180
181 #define GDT64_LIMIT gdt64_end-gdt64_start-1
182 /* Temporary gdt64, with base address in low memory */
183 .type _C_LABEL(gdt64_lo), @object
184 LABEL(gdt64_lo)
185 .word GDT64_LIMIT
186 .quad gdt64_start
187 END(gdt64_lo)
188 .align 64
189 #undef GDT64_LIMIT
190
191 .type _C_LABEL(gdt64_start), @object
192 LABEL(gdt64_start)
193 .quad 0x0000000000000000 /* always empty */
194 .quad 0x00af9a000000ffff /* kernel CS */
195 .quad 0x00cf92000000ffff /* kernel DS */
196 .quad 0x0000000000000000 /* kernel TSS [1/2] */
197 .quad 0x0000000000000000 /* kernel TSS [2/2] */
198 END(gdt64_start)
199 gdt64_end:
200
201 .type _C_LABEL(farjmp64), @object
202 _C_LABEL(farjmp64):
203 .long longmode
204 .word GSEL(GCODE_SEL, SEL_KPL)
205 END(farjmp64)
206
207 /* Space for the temporary stack */
208 .size tmpstk, tmpstk - .
209 .space 512
210 tmpstk:
211
212 .text
213
214 ENTRY(start)
215 .code32
216
217 /* Warm boot */
218 movw $0x1234,0x472
219
220 /*
221 * Load parameters from the stack (32 bits):
222 * boothowto, [bootdev], bootinfo, esym, biosextmem, biosbasemem
223 * We are not interested in 'bootdev'.
224 */
225
226 /* Load 'boothowto' */
227 movl 4(%esp),%eax
228 movl %eax,_C_LABEL(boothowto)
229
230 /* Load 'bootinfo' */
231 movl 12(%esp),%eax
232 testl %eax,%eax /* bootinfo = NULL? */
233 jz bootinfo_finished
234
235 movl (%eax),%ebx /* number of entries */
236 movl $_C_LABEL(bootinfo),%ebp
237 movl %ebp,%edx
238 addl $BOOTINFO_MAXSIZE,%ebp
239 movl %ebx,(%edx)
240 addl $4,%edx
241
242 bootinfo_entryloop:
243 testl %ebx,%ebx /* no remaining entries? */
244 jz bootinfo_finished
245
246 addl $4,%eax
247 movl (%eax),%ecx /* address of entry */
248 pushl %edi
249 pushl %esi
250 pushl %eax
251
252 movl (%ecx),%eax /* btinfo_common::len (size of entry) */
253 movl %edx,%edi
254 addl (%ecx),%edx /* update dest pointer */
255 cmpl %ebp,%edx /* beyond bootinfo+BOOTINFO_MAXSIZE? */
256 jg bootinfo_overflow
257
258 movl %ecx,%esi
259 movl %eax,%ecx
260
261 /* If any modules were loaded, record where they end. */
262 cmpl $BTINFO_MODULELIST,4(%esi) /* btinfo_common::type */
263 jne 0f
264 pushl 12(%esi) /* btinfo_modulelist::endpa */
265 popl _C_LABEL(eblob)
266 0:
267
268 /* Record the information about the kernel. */
269 cmpl $BTINFO_PREKERN,4(%esi) /* btinfo_common::type */
270 jne 0f
271 pushl 8(%esi) /* btinfo_prekern::kernpa_start */
272 popl _C_LABEL(kernpa_start)
273 pushl 12(%esi) /* btinfo_prekern::kernpa_end */
274 popl _C_LABEL(kernpa_end)
275 0:
276
277 rep
278 movsb /* copy esi -> edi */
279 popl %eax
280 popl %esi
281 popl %edi
282 subl $1,%ebx /* decrement the # of entries */
283 jmp bootinfo_entryloop
284
285 bootinfo_overflow:
286 /*
287 * Cleanup for overflow case. Pop the registers, and correct the number
288 * of entries.
289 */
290 popl %eax
291 popl %esi
292 popl %edi
293 movl $_C_LABEL(bootinfo),%ebp
294 movl %ebp,%edx
295 subl %ebx,(%edx) /* correct the number of entries */
296
297 bootinfo_finished:
298 /* Load 'esym' */
299 movl 16(%esp),%eax
300 movl $_C_LABEL(esym),%ebp
301 movl %eax,(%ebp)
302
303 /* Load 'biosextmem' */
304 movl 20(%esp),%eax
305 movl $_C_LABEL(biosextmem),%ebp
306 movl %eax,(%ebp)
307
308 /* Load 'biosbasemem' */
309 movl 24(%esp),%eax
310 movl $_C_LABEL(biosbasemem),%ebp
311 movl %eax,(%ebp)
312
313 /*
314 * Done with the parameters!
315 */
316
317 /* First, reset the PSL. */
318 pushl $PSL_MBO
319 popfl
320
321 /* Switch to new stack now. */
322 movl $_C_LABEL(tmpstk),%esp
323
324 xorl %eax,%eax
325 cpuid
326 movl %eax,_C_LABEL(cpuid_level)
327
328 /*
329 * Retrieve the NX/XD flag. We use the 32bit version of PG_NX.
330 */
331 movl $0x80000001,%eax
332 cpuid
333 andl $CPUID_NOX,%edx
334 jz no_NOX
335 movl $PG_NX32,_C_LABEL(nox_flag)
336 no_NOX:
337
338 /*
339 * There are four levels of pages in amd64: PML4 -> PDP -> PD -> PT. They will
340 * be referred to as: L4 -> L3 -> L2 -> L1.
341 *
342 * Physical address space:
343 * +---------------+----------+--------------+--------+---------------------+-
344 * | PREKERN IMAGE |**UNUSED**| KERNEL IMAGE | [SYMS] | [PRELOADED MODULES] |
345 * +---------------+----------+--------------+--------+---------------------+-
346 * (1) (2) (3) (4)
347 * ------------------+
348 * BOOTSTRAP TABLES |
349 * ------------------+
350 * (5)
351 *
352 * Virtual address space of the prekern:
353 * +---------------+----------+------------------+-------------+
354 * | PREKERN IMAGE |**UNUSED**| BOOTSTRAP TABLES | ISA I/O MEM |
355 * +---------------+----------+------------------+-------------+
356 *
357 * PROC0 STK is obviously not linked as a page level. It just happens to be
358 * caught between L4 and L3.
359 *
360 * (PROC0 STK + L4 + L3 + L2 + L1) is later referred to as BOOTSTRAP TABLES.
361 *
362 * Important note: the kernel segments are properly 4k-aligned
363 * (see kern.ldscript), so there's no need to enforce alignment.
364 */
365
366 /* Find end of the prekern image; brings us on (1). */
367 movl $_C_LABEL(__prekern_end),%edi
368
369 /* Find end of the kernel image; brings us on (2). */
370 movl _C_LABEL(kernpa_end),%eax
371 testl %eax,%eax
372 jz 1f
373 movl %eax,%edi
374 1:
375
376 /* Find end of the kernel symbols; brings us on (3). */
377 movl _C_LABEL(esym),%eax
378 testl %eax,%eax
379 jz 1f
380 movl %eax,%edi
381 1:
382
383 /* Find end of the kernel preloaded modules; brings us on (4). */
384 movl _C_LABEL(eblob),%eax
385 testl %eax,%eax
386 jz 1f
387 movl %eax,%edi
388 1:
389
390 /* We are on (3). Align up for BOOTSTRAP TABLES. */
391 movl %edi,%esi
392 addl $PGOFSET,%esi
393 andl $~PGOFSET,%esi
394
395 /* We are on the BOOTSTRAP TABLES. Save L4's physical address. */
396 movl $_C_LABEL(PDPpaddr),%ebp
397 movl %esi,(%ebp)
398 movl $0,4(%ebp)
399
400 /* Now, zero out the BOOTSTRAP TABLES (before filling them in). */
401 movl %esi,%edi
402 xorl %eax,%eax
403 cld
404 movl $TABLESIZE,%ecx
405 shrl $2,%ecx
406 rep
407 stosl /* copy eax -> edi */
408
409 /*
410 * Build the page tables and levels. We go from L1 to L4, and link the levels
411 * together.
412 */
413 /*
414 * Build L1.
415 */
416 leal (PROC0_PTP1_OFF)(%esi),%ebx
417
418 /* Skip the area below the prekern text. */
419 movl $(PREKERNTEXTOFF - PREKERNBASE),%ecx
420 shrl $PGSHIFT,%ecx
421 fillkpt_blank
422
423 /* Map the prekern text RX. */
424 movl $(PREKERNTEXTOFF - PREKERNBASE),%eax /* start of TEXT */
425 movl $_C_LABEL(__rodata_start),%ecx
426 subl %eax,%ecx
427 shrl $PGSHIFT,%ecx
428 orl $(PG_V|PG_KR),%eax
429 fillkpt
430
431 /* Map the prekern rodata R. */
432 movl $_C_LABEL(__rodata_start),%eax
433 movl $_C_LABEL(__data_start),%ecx
434 subl %eax,%ecx
435 shrl $PGSHIFT,%ecx
436 orl $(PG_V|PG_KR),%eax
437 fillkpt_nox
438
439 /* Map the prekern data+bss RW. */
440 movl $_C_LABEL(__data_start),%eax
441 movl $_C_LABEL(__prekern_end),%ecx
442 subl %eax,%ecx
443 shrl $PGSHIFT,%ecx
444 orl $(PG_V|PG_KW),%eax
445 fillkpt_nox
446
447 /* Map a RO view of the kernel. */
448 movl $_C_LABEL(__prekern_end),%eax
449 movl %esi,%ecx /* start of BOOTSTRAP TABLES */
450 subl %eax,%ecx
451 shrl $PGSHIFT,%ecx
452 orl $(PG_V|PG_KR),%eax
453 fillkpt_nox
454
455 /* Map the BOOTSTRAP TABLES RW. */
456 movl %esi,%eax /* start of BOOTSTRAP TABLES */
457 movl $TABLESIZE,%ecx /* length of BOOTSTRAP TABLES */
458 shrl $PGSHIFT,%ecx
459 orl $(PG_V|PG_KW),%eax
460 fillkpt_nox
461
462 /* Map the ISA I/O MEM RW. */
463 movl $IOM_BEGIN,%eax
464 movl $IOM_SIZE,%ecx /* size of ISA I/O MEM */
465 shrl $PGSHIFT,%ecx
466 orl $(PG_V|PG_KW/*|PG_N*/),%eax
467 fillkpt_nox
468
469 /*
470 * Build L2. Linked to L1.
471 */
472 leal (PROC0_PTP2_OFF)(%esi),%ebx
473 leal (PROC0_PTP1_OFF)(%esi),%eax
474 orl $(PG_V|PG_KW),%eax
475 movl $(NKL2_KIMG_ENTRIES+1),%ecx
476 fillkpt
477
478 /*
479 * Build L3. Linked to L2.
480 */
481 leal (PROC0_PTP3_OFF)(%esi),%ebx
482 leal (PROC0_PTP2_OFF)(%esi),%eax
483 orl $(PG_V|PG_KW),%eax
484 movl $NKL3_KIMG_ENTRIES,%ecx
485 fillkpt
486
487 /*
488 * Build L4. Linked to L3.
489 */
490 leal (PROC0_PML4_OFF)(%esi),%ebx
491 leal (PROC0_PTP3_OFF)(%esi),%eax
492 orl $(PG_V|PG_KW),%eax
493 movl $NKL4_KIMG_ENTRIES,%ecx
494 fillkpt
495
496 /* Install recursive top level PDE (one entry) */
497 leal (PROC0_PML4_OFF + PDIR_SLOT_PTE * PDE_SIZE)(%esi),%ebx
498 leal (PROC0_PML4_OFF)(%esi),%eax
499 orl $(PG_V|PG_KW),%eax
500 movl $1,%ecx
501 fillkpt_nox
502
503 /*
504 * Startup checklist:
505 * 1. Enable PAE (and SSE while here).
506 */
507 movl %cr4,%eax
508 orl $(CR4_PAE|CR4_OSFXSR|CR4_OSXMMEXCPT),%eax
509 movl %eax,%cr4
510
511 /*
512 * 2. Set Long Mode Enable in EFER. Also enable the syscall extensions,
513 * and NOX if available.
514 */
515 movl $MSR_EFER,%ecx
516 rdmsr
517 xorl %eax,%eax
518 orl $(EFER_LME|EFER_SCE),%eax
519 movl _C_LABEL(nox_flag),%ebx
520 cmpl $0,%ebx
521 je skip_NOX
522 orl $(EFER_NXE),%eax
523 skip_NOX:
524 wrmsr
525
526 /*
527 * 3. Load %cr3 with pointer to PML4.
528 */
529 movl %esi,%eax
530 movl %eax,%cr3
531
532 /*
533 * 4. Enable paging and the rest of it.
534 */
535 movl %cr0,%eax
536 orl $(CR0_PE|CR0_PG|CR0_NE|CR0_TS|CR0_MP|CR0_WP|CR0_AM),%eax
537 movl %eax,%cr0
538 jmp compat
539 compat:
540
541 /*
542 * 5. Not quite done yet, we're now in a compatibility segment, in
543 * legacy mode. We must jump to a long mode segment. Need to set up
544 * a GDT with a long mode segment in it to do that.
545 */
546 movl $_C_LABEL(gdt64_lo),%eax
547 lgdt (%eax)
548 movl $_C_LABEL(farjmp64),%eax
549 ljmp *(%eax)
550
551 .code64
552 longmode:
553
554 /*
555 * We have arrived. Everything is identity-mapped.
556 */
557
558 /* Store atdevbase. */
559 movq $TABLESIZE,%rdx
560 addq %rsi,%rdx
561 movq %rdx,_C_LABEL(atdevbase)(%rip)
562
563 /* Set up bootstrap stack. */
564 leaq (PROC0_STK_OFF)(%rsi),%rax
565 movq %rax,_C_LABEL(stkpa)(%rip)
566 leaq (USPACE-FRAMESIZE)(%rax),%rsp
567 xorq %rbp,%rbp /* mark end of frames */
568
569 xorw %ax,%ax
570 movw %ax,%gs
571 movw %ax,%fs
572
573 /* The first physical page available. */
574 leaq (TABLESIZE)(%rsi),%rdi
575
576 /*
577 * Continue execution in C.
578 */
579 call _C_LABEL(init_prekern)
580
581 ret
582 END(start)
583
584 /* -------------------------------------------------------------------------- */
585
586 ENTRY(cpuid)
587 movq %rbx,%r8
588 movq %rdi,%rax
589 movq %rsi,%rcx
590 movq %rdx,%rsi
591 cpuid
592 movl %eax,0(%rsi)
593 movl %ebx,4(%rsi)
594 movl %ecx,8(%rsi)
595 movl %edx,12(%rsi)
596 movq %r8,%rbx
597 ret
598 END(cpuid)
599
600 ENTRY(lidt)
601 lidt (%rdi)
602 ret
603 END(lidt)
604
605 ENTRY(rdtsc)
606 xorq %rax,%rax
607 rdtsc
608 shlq $32,%rdx
609 orq %rdx,%rax
610 ret
611 END(rdtsc)
612
613 ENTRY(rdseed)
614 rdseed %rax
615 jc .Lrdseed_success
616 movq $(-1),%rax
617 ret
618 .Lrdseed_success:
619 movq %rax,(%rdi)
620 xorq %rax,%rax
621 ret
622 END(rdseed)
623
624 ENTRY(rdrand)
625 rdrand %rax
626 jc .Lrdrand_success
627 movq $(-1),%rax
628 ret
629 .Lrdrand_success:
630 movq %rax,(%rdi)
631 xorq %rax,%rax
632 ret
633 END(rdrand)
634
635 ENTRY(jump_kernel)
636 movq _C_LABEL(stkva),%rsp
637 xorq %rbp,%rbp
638 callq exec_kernel
639 END(jump_kernel)
640