Home | History | Annotate | Line # | Download | only in prekern
      1 /*	$NetBSD: locore.S,v 1.11 2019/03/19 19:15:57 maxv Exp $	*/
      2 
      3 /*
      4  * Copyright (c) 1998, 2000, 2007, 2008, 2016, 2017 The NetBSD Foundation, Inc.
      5  * All rights reserved.
      6  *
      7  * This code is derived from software contributed to The NetBSD Foundation
      8  * by Charles M. Hannum and by Maxime Villard.
      9  *
     10  * Redistribution and use in source and binary forms, with or without
     11  * modification, are permitted provided that the following conditions
     12  * are met:
     13  * 1. Redistributions of source code must retain the above copyright
     14  *    notice, this list of conditions and the following disclaimer.
     15  * 2. Redistributions in binary form must reproduce the above copyright
     16  *    notice, this list of conditions and the following disclaimer in the
     17  *    documentation and/or other materials provided with the distribution.
     18  *
     19  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     21  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     22  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     23  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     29  * POSSIBILITY OF SUCH DAMAGE.
     30  */
     31 
     32 #define _LOCORE
     33 
     34 /* Override user-land alignment before including asm.h */
     35 #define	ALIGN_DATA	.align	8
     36 #define ALIGN_TEXT	.align 16,0x90
     37 #define _ALIGN_TEXT	ALIGN_TEXT
     38 
     39 #include <machine/asm.h>
     40 #include <machine/param.h>
     41 #include <machine/pte.h>
     42 #include <machine/psl.h>
     43 #include <machine/segments.h>
     44 #include <machine/specialreg.h>
     45 #include <machine/trap.h>
     46 
     47 #define _KERNEL
     48 #include <machine/bootinfo.h>
     49 #undef _KERNEL
     50 
     51 #include "pdir.h"
     52 #include "redef.h"
     53 
     54 /* 32bit version of PTE_NX */
     55 #define PTE_NX32	0x80000000
     56 
     57 #define TABLE_L2_ENTRIES (NKL2_KIMG_ENTRIES + 1)
     58 #define TABLE_L3_ENTRIES NKL3_KIMG_ENTRIES
     59 
     60 #define PROC0_PML4_OFF	0
     61 #define PROC0_STK_OFF	(PROC0_PML4_OFF + 1 * PAGE_SIZE)
     62 #define PROC0_PTP3_OFF	(PROC0_STK_OFF + UPAGES * PAGE_SIZE)
     63 #define PROC0_PTP2_OFF	(PROC0_PTP3_OFF + NKL4_KIMG_ENTRIES * PAGE_SIZE)
     64 #define PROC0_PTP1_OFF	(PROC0_PTP2_OFF + TABLE_L3_ENTRIES * PAGE_SIZE)
     65 #define TABLESIZE \
     66   ((NKL4_KIMG_ENTRIES + TABLE_L3_ENTRIES + TABLE_L2_ENTRIES + 1 + UPAGES) \
     67     * PAGE_SIZE)
     68 
     69 /*
     70  * fillkpt - Fill in a kernel page table
     71  *	eax = pte (page frame | control | status)
     72  *	ebx = page table address
     73  *	ecx = number of pages to map
     74  *
     75  * Each entry is 8 (PDE_SIZE) bytes long: we must set the 4 upper bytes to 0.
     76  */
     77 #define fillkpt	\
     78 	cmpl	$0,%ecx			;	/* zero-sized? */	\
     79 	je 	2f			; \
     80 1:	movl	$0,(PDE_SIZE-4)(%ebx)	;	/* upper 32 bits: 0 */	\
     81 	movl	%eax,(%ebx)		;	/* store phys addr */	\
     82 	addl	$PDE_SIZE,%ebx		;	/* next PTE/PDE */	\
     83 	addl	$PAGE_SIZE,%eax		;	/* next phys page */	\
     84 	loop	1b			; \
     85 2:					;
     86 
     87 /*
     88  * fillkpt_nox - Same as fillkpt, but sets the NX/XD bit.
     89  */
     90 #define fillkpt_nox \
     91 	cmpl	$0,%ecx			;	/* zero-sized? */	\
     92 	je 	2f			; \
     93 	pushl	%ebp			; \
     94 	movl	_C_LABEL(nox_flag),%ebp	; \
     95 1:	movl	%ebp,(PDE_SIZE-4)(%ebx)	;	/* upper 32 bits: NX */ \
     96 	movl	%eax,(%ebx)		;	/* store phys addr */	\
     97 	addl	$PDE_SIZE,%ebx		;	/* next PTE/PDE */	\
     98 	addl	$PAGE_SIZE,%eax		;	/* next phys page */	\
     99 	loop	1b			; \
    100 	popl	%ebp			; \
    101 2:					;
    102 
    103 /*
    104  * fillkpt_blank - Fill in a kernel page table with blank entries
    105  *	ebx = page table address
    106  *	ecx = number of pages to map
    107  */
    108 #define fillkpt_blank	\
    109 	cmpl	$0,%ecx			;	/* zero-sized? */	\
    110 	je 	2f			; \
    111 1:	movl	$0,(PDE_SIZE-4)(%ebx)	;	/* upper 32 bits: 0 */	\
    112 	movl	$0,(%ebx)		;	/* lower 32 bits: 0 */	\
    113 	addl	$PDE_SIZE,%ebx		;	/* next PTE/PDE */	\
    114 	loop	1b			; \
    115 2:					;
    116 
    117 /*
    118  * Initialization
    119  */
    120 	.data
    121 
    122 	.globl	_C_LABEL(tablesize)
    123 	.globl	_C_LABEL(nox_flag)
    124 	.globl	_C_LABEL(cpuid_level)
    125 	.globl	_C_LABEL(esym)
    126 	.globl	_C_LABEL(eblob)
    127 	.globl	_C_LABEL(atdevbase)
    128 	.globl	_C_LABEL(PDPpaddr)
    129 	.globl	_C_LABEL(boothowto)
    130 	.globl	_C_LABEL(bootinfo)
    131 	.globl	_C_LABEL(biosbasemem)
    132 	.globl	_C_LABEL(biosextmem)
    133 	.globl	_C_LABEL(stkpa)
    134 	.globl	_C_LABEL(stkva)
    135 	.globl	_C_LABEL(kernpa_start)
    136 	.globl	_C_LABEL(kernpa_end)
    137 
    138 	.type	_C_LABEL(tablesize), @object
    139 _C_LABEL(tablesize):	.long	TABLESIZE
    140 END(tablesize)
    141 	.type	_C_LABEL(nox_flag), @object
    142 LABEL(nox_flag)		.long	0	/* 32bit NOX flag, set if supported */
    143 END(nox_flag)
    144 	.type	_C_LABEL(cpuid_level), @object
    145 LABEL(cpuid_level)	.long	-1	/* max. level accepted by cpuid instr */
    146 END(cpuid_level)
    147 	.type	_C_LABEL(esym), @object
    148 LABEL(esym)		.quad	0	/* ptr to end of syms */
    149 END(esym)
    150 	.type	_C_LABEL(eblob), @object
    151 LABEL(eblob)		.quad	0	/* ptr to end of modules */
    152 END(eblob)
    153 	.type	_C_LABEL(atdevbase), @object
    154 LABEL(atdevbase)	.quad	0	/* location of start of iomem in virt */
    155 END(atdevbase)
    156 	.type	_C_LABEL(PDPpaddr), @object
    157 LABEL(PDPpaddr)		.quad	0	/* paddr of PTD, for libkvm */
    158 END(PDPpaddr)
    159 	.type	_C_LABEL(biosbasemem), @object
    160 LABEL(biosbasemem)	.long	0	/* base memory reported by BIOS */
    161 END(biosbasemem)
    162 	.type	_C_LABEL(biosextmem), @object
    163 LABEL(biosextmem)	.long	0	/* extended memory reported by BIOS */
    164 END(biosextmem)
    165 	.type	_C_LABEL(stkpa), @object
    166 LABEL(stkpa)		.quad	0
    167 END(stkpa)
    168 	.type	_C_LABEL(stkva), @object
    169 LABEL(stkva)		.quad	0
    170 END(stkva)
    171 	.type	_C_LABEL(kernpa_start), @object
    172 LABEL(kernpa_start)	.quad	0
    173 END(kernpa_start)
    174 	.type	_C_LABEL(kernpa_end), @object
    175 LABEL(kernpa_end)	.quad	0
    176 END(kernpa_end)
    177 
    178 	.globl	gdt64_lo
    179 	.globl	gdt64_start
    180 
    181 #define GDT64_LIMIT gdt64_end-gdt64_start-1
    182 /* Temporary gdt64, with base address in low memory */
    183 	.type	_C_LABEL(gdt64_lo), @object
    184 LABEL(gdt64_lo)
    185 	.word	GDT64_LIMIT
    186 	.quad	gdt64_start
    187 END(gdt64_lo)
    188 .align 64
    189 #undef GDT64_LIMIT
    190 
    191 	.type	_C_LABEL(gdt64_start), @object
    192 LABEL(gdt64_start)
    193 	.quad 0x0000000000000000	/* always empty */
    194 	.quad 0x00af9a000000ffff	/* kernel CS */
    195 	.quad 0x00cf92000000ffff	/* kernel DS */
    196 END(gdt64_start)
    197 gdt64_end:
    198 
    199 	.type	_C_LABEL(farjmp64), @object
    200 _C_LABEL(farjmp64):
    201 	.long	longmode
    202 	.word	GSEL(GCODE_SEL, SEL_KPL)
    203 END(farjmp64)
    204 
    205 	/* Space for the temporary stack */
    206 	.size	tmpstk, tmpstk - .
    207 	.space	512
    208 tmpstk:
    209 
    210 	.text
    211 
    212 ENTRY(start)
    213 	.code32
    214 
    215 	/* Warm boot */
    216 	movw	$0x1234,0x472
    217 
    218 	/*
    219 	 * Load parameters from the stack (32 bits):
    220 	 *     boothowto, [bootdev], bootinfo, esym, biosextmem, biosbasemem
    221 	 * We are not interested in 'bootdev'.
    222 	 */
    223 
    224 	/* Load 'boothowto' */
    225 	movl	4(%esp),%eax
    226 	movl	%eax,_C_LABEL(boothowto)
    227 
    228 	/* Load 'bootinfo' */
    229 	movl	12(%esp),%eax
    230 	testl	%eax,%eax		/* bootinfo = NULL? */
    231 	jz	.Lbootinfo_finished
    232 
    233 	movl	(%eax),%ebx		/* number of entries */
    234 	movl	$_C_LABEL(bootinfo),%ebp
    235 	movl	%ebp,%edx
    236 	addl	$BOOTINFO_MAXSIZE,%ebp
    237 	movl	%ebx,(%edx)
    238 	addl	$4,%edx
    239 
    240 .Lbootinfo_entryloop:
    241 	testl	%ebx,%ebx		/* no remaining entries? */
    242 	jz	.Lbootinfo_finished
    243 
    244 	addl	$4,%eax
    245 	movl	(%eax),%ecx		/* address of entry */
    246 	pushl	%edi
    247 	pushl	%esi
    248 	pushl	%eax
    249 
    250 	movl	(%ecx),%eax		/* btinfo_common::len (size of entry) */
    251 	movl	%edx,%edi
    252 	addl	(%ecx),%edx		/* update dest pointer */
    253 	cmpl	%ebp,%edx		/* beyond bootinfo+BOOTINFO_MAXSIZE? */
    254 	jg	.Lbootinfo_overflow
    255 
    256 	movl	%ecx,%esi
    257 	movl	%eax,%ecx
    258 
    259 	/* If any modules were loaded, record where they end. */
    260 	cmpl	$BTINFO_MODULELIST,4(%esi) /* btinfo_common::type */
    261 	jne	0f
    262 	pushl	12(%esi)		/* btinfo_modulelist::endpa */
    263 	popl	_C_LABEL(eblob)
    264 0:
    265 
    266 	/* Record the information about the kernel. */
    267 	cmpl	$BTINFO_PREKERN,4(%esi) /* btinfo_common::type */
    268 	jne	0f
    269 	pushl	8(%esi)		/* btinfo_prekern::kernpa_start */
    270 	popl	_C_LABEL(kernpa_start)
    271 	pushl	12(%esi)	/* btinfo_prekern::kernpa_end */
    272 	popl	_C_LABEL(kernpa_end)
    273 0:
    274 
    275 	rep
    276 	movsb				/* copy esi -> edi */
    277 	popl	%eax
    278 	popl	%esi
    279 	popl	%edi
    280 	subl	$1,%ebx			/* decrement the # of entries */
    281 	jmp	.Lbootinfo_entryloop
    282 
    283 .Lbootinfo_overflow:
    284 	/*
    285 	 * Cleanup for overflow case. Pop the registers, and correct the number
    286 	 * of entries.
    287 	 */
    288 	popl	%eax
    289 	popl	%esi
    290 	popl	%edi
    291 	movl	$_C_LABEL(bootinfo),%ebp
    292 	movl	%ebp,%edx
    293 	subl	%ebx,(%edx)		/* correct the number of entries */
    294 
    295 .Lbootinfo_finished:
    296 	/* Load 'esym' */
    297 	movl	16(%esp),%eax
    298 	movl	$_C_LABEL(esym),%ebp
    299 	movl	%eax,(%ebp)
    300 
    301 	/* Load 'biosextmem' */
    302 	movl	20(%esp),%eax
    303 	movl	$_C_LABEL(biosextmem),%ebp
    304 	movl	%eax,(%ebp)
    305 
    306 	/* Load 'biosbasemem' */
    307 	movl	24(%esp),%eax
    308 	movl	$_C_LABEL(biosbasemem),%ebp
    309 	movl	%eax,(%ebp)
    310 
    311 	/*
    312 	 * Done with the parameters!
    313 	 */
    314 
    315 	/* First, reset the PSL. */
    316 	pushl	$PSL_MBO
    317 	popfl
    318 
    319 	/* Switch to new stack now. */
    320 	movl	$_C_LABEL(tmpstk),%esp
    321 
    322 	xorl	%eax,%eax
    323 	cpuid
    324 	movl	%eax,_C_LABEL(cpuid_level)
    325 
    326 	/*
    327 	 * Retrieve the NX/XD flag. We use the 32bit version of PTE_NX.
    328 	 */
    329 	movl	$0x80000001,%eax
    330 	cpuid
    331 	andl	$CPUID_NOX,%edx
    332 	jz	.Lno_NOX
    333 	movl	$PTE_NX32,_C_LABEL(nox_flag)
    334 .Lno_NOX:
    335 
    336 /*
    337  * There are four levels of pages in amd64: PML4 -> PDP -> PD -> PT. They will
    338  * be referred to as: L4 -> L3 -> L2 -> L1.
    339  *
    340  * Physical address space:
    341  * +---------------+----------+--------------+--------+---------------------+-
    342  * | PREKERN IMAGE |**UNUSED**| KERNEL IMAGE | [SYMS] | [PRELOADED MODULES] |
    343  * +---------------+----------+--------------+--------+---------------------+-
    344  *                (1)                       (2)      (3)                   (4)
    345  * ------------------+
    346  *  BOOTSTRAP TABLES |
    347  * ------------------+
    348  *                  (5)
    349  *
    350  * The virtual address space is the same, since it is identity-mapped (va = pa).
    351  * However, the KERNEL IMAGE is mapped as read-only: the prekern reads it, but
    352  * won't write to it. (Needed when relocating the kernel.)
    353  *
    354  * PROC0 STK is obviously not linked as a page level. It just happens to be
    355  * caught between L4 and L3.
    356  *
    357  * (PROC0 STK + L4 + L3 + L2 + L1) is later referred to as BOOTSTRAP TABLES.
    358  *
    359  * Important note: the prekern segments are properly 4k-aligned
    360  * (see prekern.ldscript), so there's no need to enforce alignment.
    361  */
    362 
    363 	/* Find end of the prekern image; brings us on (1). */
    364 	movl	$_C_LABEL(__prekern_end),%edi
    365 
    366 	/* Find end of the kernel image; brings us on (2). */
    367 	movl	_C_LABEL(kernpa_end),%eax
    368 	testl	%eax,%eax
    369 	jz	1f
    370 	movl	%eax,%edi
    371 1:
    372 
    373 	/* Find end of the kernel symbols; brings us on (3). */
    374 	movl	_C_LABEL(esym),%eax
    375 	testl	%eax,%eax
    376 	jz	1f
    377 	movl	%eax,%edi
    378 1:
    379 
    380 	/* Find end of the kernel preloaded modules; brings us on (4). */
    381 	movl	_C_LABEL(eblob),%eax
    382 	testl	%eax,%eax
    383 	jz	1f
    384 	movl	%eax,%edi
    385 1:
    386 
    387 	/* We are on (3). Align up for BOOTSTRAP TABLES. */
    388 	movl	%edi,%esi
    389 	addl	$PGOFSET,%esi
    390 	andl	$~PGOFSET,%esi
    391 
    392 	/* We are on the BOOTSTRAP TABLES. Save L4's physical address. */
    393 	movl	$_C_LABEL(PDPpaddr),%ebp
    394 	movl	%esi,(%ebp)
    395 	movl	$0,4(%ebp)
    396 
    397 	/* Now, zero out the BOOTSTRAP TABLES (before filling them in). */
    398 	movl	%esi,%edi
    399 	xorl	%eax,%eax
    400 	cld
    401 	movl	$TABLESIZE,%ecx
    402 	shrl	$2,%ecx
    403 	rep
    404 	stosl				/* copy eax -> edi */
    405 
    406 /*
    407  * Build the page tables and levels. We go from L1 to L4, and link the levels
    408  * together.
    409  */
    410 	/*
    411 	 * Build L1.
    412 	 */
    413 	leal	(PROC0_PTP1_OFF)(%esi),%ebx
    414 
    415 	/* Skip the area below the prekern text. */
    416 	movl	$(PREKERNTEXTOFF - PREKERNBASE),%ecx
    417 	shrl	$PGSHIFT,%ecx
    418 	fillkpt_blank
    419 
    420 	/* Map the prekern text RX. */
    421 	movl	$(PREKERNTEXTOFF - PREKERNBASE),%eax	/* start of TEXT */
    422 	movl	$_C_LABEL(__rodata_start),%ecx
    423 	subl	%eax,%ecx
    424 	shrl	$PGSHIFT,%ecx
    425 	orl	$(PTE_P),%eax
    426 	fillkpt
    427 
    428 	/* Map the prekern rodata R. */
    429 	movl	$_C_LABEL(__rodata_start),%eax
    430 	movl	$_C_LABEL(__data_start),%ecx
    431 	subl	%eax,%ecx
    432 	shrl	$PGSHIFT,%ecx
    433 	orl	$(PTE_P),%eax
    434 	fillkpt_nox
    435 
    436 	/* Map the prekern data+bss RW. */
    437 	movl	$_C_LABEL(__data_start),%eax
    438 	movl	$_C_LABEL(__prekern_end),%ecx
    439 	subl	%eax,%ecx
    440 	shrl	$PGSHIFT,%ecx
    441 	orl	$(PTE_P|PTE_W),%eax
    442 	fillkpt_nox
    443 
    444 	/* Map a RO view of the kernel. */
    445 	movl	$_C_LABEL(__prekern_end),%eax
    446 	movl	%esi,%ecx		/* start of BOOTSTRAP TABLES */
    447 	subl	%eax,%ecx
    448 	shrl	$PGSHIFT,%ecx
    449 	orl	$(PTE_P),%eax
    450 	fillkpt_nox
    451 
    452 	/* Map the BOOTSTRAP TABLES RW. */
    453 	movl	%esi,%eax		/* start of BOOTSTRAP TABLES */
    454 	movl	$TABLESIZE,%ecx		/* length of BOOTSTRAP TABLES */
    455 	shrl	$PGSHIFT,%ecx
    456 	orl	$(PTE_P|PTE_W),%eax
    457 	fillkpt_nox
    458 
    459 	/* Map the ISA I/O MEM RW. */
    460 	movl	$IOM_BEGIN,%eax
    461 	movl	$IOM_SIZE,%ecx	/* size of ISA I/O MEM */
    462 	shrl	$PGSHIFT,%ecx
    463 	orl	$(PTE_P|PTE_W/*|PTE_PCD*/),%eax
    464 	fillkpt_nox
    465 
    466 	/*
    467 	 * Build L2. Linked to L1.
    468 	 */
    469 	leal	(PROC0_PTP2_OFF)(%esi),%ebx
    470 	leal	(PROC0_PTP1_OFF)(%esi),%eax
    471 	orl	$(PTE_P|PTE_W),%eax
    472 	movl	$(NKL2_KIMG_ENTRIES+1),%ecx
    473 	fillkpt
    474 
    475 	/*
    476 	 * Build L3. Linked to L2.
    477 	 */
    478 	leal	(PROC0_PTP3_OFF)(%esi),%ebx
    479 	leal	(PROC0_PTP2_OFF)(%esi),%eax
    480 	orl	$(PTE_P|PTE_W),%eax
    481 	movl	$NKL3_KIMG_ENTRIES,%ecx
    482 	fillkpt
    483 
    484 	/*
    485 	 * Build L4. Linked to L3.
    486 	 */
    487 	leal	(PROC0_PML4_OFF)(%esi),%ebx
    488 	leal	(PROC0_PTP3_OFF)(%esi),%eax
    489 	orl	$(PTE_P|PTE_W),%eax
    490 	movl	$NKL4_KIMG_ENTRIES,%ecx
    491 	fillkpt
    492 
    493 	/* Install recursive top level PDE (one entry) */
    494 	leal	(PROC0_PML4_OFF + PDIR_SLOT_PTE * PDE_SIZE)(%esi),%ebx
    495 	leal	(PROC0_PML4_OFF)(%esi),%eax
    496 	orl	$(PTE_P|PTE_W),%eax
    497 	movl	$1,%ecx
    498 	fillkpt_nox
    499 
    500 	/*
    501 	 * Startup checklist:
    502 	 * 1. Enable PAE (and SSE while here).
    503 	 */
    504 	movl	%cr4,%eax
    505 	orl	$(CR4_PAE|CR4_OSFXSR|CR4_OSXMMEXCPT),%eax
    506 	movl	%eax,%cr4
    507 
    508 	/*
    509 	 * 2. Set Long Mode Enable in EFER. Also enable the syscall extensions,
    510 	 *    and NOX if available.
    511 	 */
    512 	movl	$MSR_EFER,%ecx
    513 	rdmsr
    514 	xorl	%eax,%eax
    515 	orl	$(EFER_LME|EFER_SCE),%eax
    516 	movl	_C_LABEL(nox_flag),%ebx
    517 	cmpl	$0,%ebx
    518 	je 	.Lskip_NOX
    519 	orl	$(EFER_NXE),%eax
    520 .Lskip_NOX:
    521 	wrmsr
    522 
    523 	/*
    524 	 * 3. Load %cr3 with pointer to PML4.
    525 	 */
    526 	movl	%esi,%eax
    527 	movl	%eax,%cr3
    528 
    529 	/*
    530 	 * 4. Enable paging and the rest of it.
    531 	 */
    532 	movl	%cr0,%eax
    533 	orl	$(CR0_PE|CR0_PG|CR0_NE|CR0_TS|CR0_MP|CR0_WP|CR0_AM),%eax
    534 	movl	%eax,%cr0
    535 	jmp	compat
    536 compat:
    537 
    538 	/*
    539 	 * 5. Not quite done yet, we're now in a compatibility segment, in
    540 	 *    legacy mode. We must jump to a long mode segment. Need to set up
    541 	 *    a GDT with a long mode segment in it to do that.
    542 	 */
    543 	movl	$_C_LABEL(gdt64_lo),%eax
    544 	lgdt	(%eax)
    545 	movl	$_C_LABEL(farjmp64),%eax
    546 	ljmp	*(%eax)
    547 
    548 	.code64
    549 longmode:
    550 
    551 	/*
    552 	 * We have arrived. Everything is identity-mapped.
    553 	 */
    554 
    555 	/* Store atdevbase. */
    556 	movq	$TABLESIZE,%rdx
    557 	addq	%rsi,%rdx
    558 	movq	%rdx,_C_LABEL(atdevbase)(%rip)
    559 
    560 	/* Set up bootstrap stack. */
    561 	leaq	(PROC0_STK_OFF)(%rsi),%rax
    562 	movq	%rax,_C_LABEL(stkpa)(%rip)
    563 	leaq	(USPACE-FRAMESIZE)(%rax),%rsp
    564 	xorq	%rbp,%rbp			/* mark end of frames */
    565 
    566 	xorw	%ax,%ax
    567 	movw	%ax,%gs
    568 	movw	%ax,%fs
    569 
    570 	movw	$GSEL(GDATA_SEL, SEL_KPL),%ax
    571 	movw	%ax,%ss
    572 
    573 	/* The first physical page available. */
    574 	leaq	(TABLESIZE)(%rsi),%rdi
    575 
    576 	/*
    577 	 * Continue execution in C.
    578 	 */
    579 	call	_C_LABEL(init_prekern)
    580 
    581 	ret
    582 END(start)
    583 
    584 /* -------------------------------------------------------------------------- */
    585 
    586 ENTRY(cpuid)
    587 	movq	%rbx,%r8
    588 	movq	%rdi,%rax
    589 	movq	%rsi,%rcx
    590 	movq	%rdx,%rsi
    591 	cpuid
    592 	movl	%eax,0(%rsi)
    593 	movl	%ebx,4(%rsi)
    594 	movl	%ecx,8(%rsi)
    595 	movl	%edx,12(%rsi)
    596 	movq	%r8,%rbx
    597 	ret
    598 END(cpuid)
    599 
    600 ENTRY(lidt)
    601 	lidt	(%rdi)
    602 	ret
    603 END(lidt)
    604 
    605 ENTRY(rdtsc)
    606 	xorq	%rax,%rax
    607 	rdtsc
    608 	shlq	$32,%rdx
    609 	orq	%rdx,%rax
    610 	ret
    611 END(rdtsc)
    612 
    613 ENTRY(rdseed)
    614 	rdseed	%rax
    615 	jc	.Lrdseed_success
    616 	movq	$(-1),%rax
    617 	ret
    618 .Lrdseed_success:
    619 	movq	%rax,(%rdi)
    620 	xorq	%rax,%rax
    621 	ret
    622 END(rdseed)
    623 
    624 ENTRY(rdrand)
    625 	rdrand	%rax
    626 	jc	.Lrdrand_success
    627 	movq	$(-1),%rax
    628 	ret
    629 .Lrdrand_success:
    630 	movq	%rax,(%rdi)
    631 	xorq	%rax,%rax
    632 	ret
    633 END(rdrand)
    634 
    635 ENTRY(jump_kernel)
    636 	movq	_C_LABEL(stkva),%rsp
    637 	xorq	%rbp,%rbp
    638 	callq	exec_kernel
    639 END(jump_kernel)
    640