OpenGrok

/*	$NetBSD: cpufunc.S,v 1.70 2025/09/06 02:53:22 riastradh Exp $	*/

/*
 * Copyright (c) 1998, 2007, 2008, 2020, 2023 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Charles M. Hannum, and by Andrew Doran.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/errno.h>

#include <machine/asm.h>
#include <machine/frameasm.h>
#include <machine/specialreg.h>
#include <machine/segments.h>

#include "opt_dtrace.h"
#include "opt_xen.h"
#include "opt_svs.h"

#include "assym.h"

/* Small and slow, so align less. */
#undef _ALIGN_TEXT
#define	_ALIGN_TEXT	.align 8

ENTRY(x86_lfence)
	lfence
	ret
END(x86_lfence)

ENTRY(x86_sfence)
	sfence
	ret
END(x86_sfence)

ENTRY(x86_mfence)
	mfence
	ret
END(x86_mfence)

ENTRY(paravirt_membar_sync)
	/*
	 * Store-before-load ordering with respect to matching logic
	 * on the hypervisor side.
	 *
	 * This is the same as membar_sync, but without hotpatching
	 * away the LOCK prefix on uniprocessor boots -- because under
	 * virtualization, we still have to coordinate with a `device'
	 * backed by a hypervisor that is potentially on another
	 * physical CPU even if we observe only one virtual CPU as the
	 * guest.
	 *
	 * See common/lib/libc/arch/x86_64/atomic/atomic.S for
	 * rationale and keep this in sync with the implementation
	 * of membar_sync there.
	 */
	lock
	addq	$0,-8(%rsp)
	ret
END(paravirt_membar_sync)

#ifdef XEN
STRONG_ALIAS(xen_mb,paravirt_membar_sync)
#endif	/* XEN */

#ifdef KDTRACE_HOOKS
ENTRY(dtrace_smap_enable)
#  ifndef XENPV
	SMAP_ENABLE
#  endif
	ret
END(dtrace_smap_enable)

ENTRY(dtrace_smap_disable)
#  ifndef XENPV
	SMAP_DISABLE
#  endif
	ret
END(dtrace_smap_disable)
#endif

#ifndef XENPV
ENTRY(invlpg)
#ifdef SVS
	movb	_C_LABEL(svs_pcid),%al
	testb	%al,%al
	jz	1f
	pushq	%rdi
	pushq	$PMAP_PCID_USER
	movq	$INVPCID_ADDRESS,%rax
	invpcid	(%rsp),%rax
	addq	$16,%rsp
1:	/* FALLTHROUGH */
#endif
	invlpg	(%rdi)
	ret
END(invlpg)

ENTRY(lgdt)
	/* Reload the descriptor table. */
	movq	%rdi,%rax
	lgdt	(%rax)
	/* Flush the prefetch queue. */
	jmp	1f
	nop
1:	jmp	_C_LABEL(lgdt_finish)
END(lgdt)

ENTRY(lidt)
	lidt	(%rdi)
	ret
END(lidt)

ENTRY(lldt)
	cmpl	%edi, CPUVAR(CURLDT)
	jne	1f
	ret
1:
	movl	%edi, CPUVAR(CURLDT)
	lldt	%di
	ret
END(lldt)

ENTRY(ltr)
	ltr	%di
	ret
END(ltr)

ENTRY(tlbflushg)
	movq	%cr4, %rax
	testq	$CR4_PGE, %rax
	jz	tlbflush
	movq	%rax, %rdx
	andq	$~CR4_PGE, %rdx
	movq	%rdx, %cr4
	movq	%rax, %cr4
	ret
END(tlbflushg)

ENTRY(tlbflush)
#ifdef SVS
	movb	_C_LABEL(svs_pcid),%al
	testb	%al,%al
	jz	1f
	xorq	%rax,%rax
	pushq	%rax
	pushq	%rax
	movq	$INVPCID_ALL_NONGLOBAL,%rax
	invpcid	(%rsp),%rax
	addq	$16,%rsp
	ret
#endif
1:	movq	%cr3, %rax
	movq	%rax, %cr3
	ret
END(tlbflush)

ENTRY(wbinvd)
	wbinvd
	ret
END(wbinvd)

ENTRY(setusergs)
	CLI(ax)
	swapgs
	movw	%di, %gs
	swapgs
	STI(ax)
	ret
END(setusergs)

ENTRY(x86_read_flags)
	pushfq
	popq	%rax
	KMSAN_INIT_RET(8)
	ret
END(x86_read_flags)

STRONG_ALIAS(x86_read_psl,x86_read_flags)

ENTRY(x86_write_flags)
	pushq	%rdi
	popfq
	ret
END(x86_write_flags)

STRONG_ALIAS(x86_write_psl,x86_write_flags)

ENTRY(smap_enable)
	SMAP_ENABLE
	ret
END(smap_enable)

ENTRY(smap_disable)
	SMAP_DISABLE
	ret
END(smap_disable)

/*
 * %rdi = name
 * %rsi = sel
 */
ENTRY(x86_hotpatch)
	/* save RFLAGS, and disable intrs */
	pushfq
	cli

	/* save CR0, and disable WP */
	movq	%cr0,%rcx
	pushq	%rcx
	andq	$~CR0_WP,%rcx
	movq	%rcx,%cr0

	callq	_C_LABEL(x86_hotpatch_apply)

	/* write back and invalidate cache */
	wbinvd

	/* restore CR0 */
	popq	%rcx
	movq	%rcx,%cr0

	/* flush instruction pipeline */
	pushq	%rax
	callq	x86_flush
	popq	%rax

	/* clean up */
	movq	%rax,%rdi
	callq	_C_LABEL(x86_hotpatch_cleanup)

	/* restore RFLAGS */
	popfq
	ret
END(x86_hotpatch)
#endif /* !XENPV */

/*
 * cpu_counter and cpu_counter32 could be exact same, but KMSAN needs to have
 * the correct size of the return value.
 */
#define SERIALIZE_lfence	lfence
#define SERIALIZE_mfence	mfence

#define ADD_counter32	addl	CPUVAR(CC_SKEW), %eax
#define ADD_counter	shlq	$32, %rdx		;\
			orq	%rdx, %rax		;\
			addq	CPUVAR(CC_SKEW), %rax

#define RSIZE_counter32	4
#define RSIZE_counter	8

#define CPU_COUNTER_FENCE(counter, fence)	\
ENTRY(cpu_ ## counter ## _ ## fence)		;\
	movq	CPUVAR(CURLWP), %rcx		;\
	leaq	L_RU+RU_NIVCSW(%rcx), %rcx	;\
1:						;\
	movq	(%rcx), %rdi			;\
	SERIALIZE_ ## fence			;\
	rdtsc					;\
	ADD_ ## counter				;\
	cmpq	%rdi, (%rcx)			;\
	jne	2f				;\
	KMSAN_INIT_RET(RSIZE_ ## counter)	;\
	ret					;\
2:						;\
	jmp	1b				;\
END(cpu_ ## counter ## _ ## fence)

CPU_COUNTER_FENCE(counter, lfence)
CPU_COUNTER_FENCE(counter, mfence)
CPU_COUNTER_FENCE(counter32, lfence)
CPU_COUNTER_FENCE(counter32, mfence)

#define CPU_COUNTER_CPUID(counter)		\
ENTRY(cpu_ ## counter ## _cpuid)		;\
	movq	%rbx, %r9			;\
	movq	CPUVAR(CURLWP), %r8		;\
	leaq	L_RU+RU_NIVCSW(%r8), %r8	;\
1:						;\
	movq	(%r8), %rdi			;\
	xor	%eax, %eax			;\
	cpuid					;\
	rdtsc					;\
	ADD_ ## counter				;\
	cmpq	%rdi, (%r8)			;\
	jne	2f				;\
	movq	%r9, %rbx			;\
	KMSAN_INIT_RET(RSIZE_ ## counter)	;\
	ret					;\
2:						;\
	jmp	1b				;\
END(cpu_ ## counter ## _cpuid)

CPU_COUNTER_CPUID(counter)
CPU_COUNTER_CPUID(counter32)

ENTRY(rdmsr_safe)
	movq	CPUVAR(CURLWP), %r8
	movq	L_PCB(%r8), %r8
	movq	$_C_LABEL(msr_onfault), PCB_ONFAULT(%r8)

	movl	%edi, %ecx
	rdmsr
	salq	$32, %rdx
	movl	%eax, %eax	/* zero-extend %eax -> %rax */
	orq	%rdx, %rax
	movq	%rax, (%rsi)

	xorq	%rax, %rax
	movq	%rax, PCB_ONFAULT(%r8)
#ifdef KMSAN
	movq	%rsi,%rdi
	movq	$8,%rsi
	xorq	%rdx,%rdx
	callq	_C_LABEL(kmsan_mark)
#endif
	KMSAN_INIT_RET(4)
	ret
END(rdmsr_safe)

ENTRY(msr_onfault)
	movq	CPUVAR(CURLWP), %r8
	movq	L_PCB(%r8), %r8
	movq	$0, PCB_ONFAULT(%r8)
	movl	$EFAULT, %eax
	ret
END(msr_onfault)

ENTRY(breakpoint)
	pushq	%rbp
	movq	%rsp, %rbp
	int	$0x03		/* paranoid, not 'int3' */
	leave
	ret
END(breakpoint)

ENTRY(x86_curcpu)
	movq	%gs:(CPU_INFO_SELF), %rax
	KMSAN_INIT_RET(8)
	ret
END(x86_curcpu)

ENTRY(x86_curlwp)
	movq	%gs:(CPU_INFO_CURLWP), %rax
	KMSAN_INIT_RET(8)
	ret
END(x86_curlwp)

ENTRY(__byte_swap_u32_variable)
	movl	%edi, %eax
	bswapl	%eax
	KMSAN_INIT_RET(4)
	ret
END(__byte_swap_u32_variable)

ENTRY(__byte_swap_u16_variable)
	movl	%edi, %eax
	xchgb	%al, %ah
	KMSAN_INIT_RET(2)
	ret
END(__byte_swap_u16_variable)

/*
 * Reload segments after a GDT change.
 */
ENTRY(lgdt_finish)
	movl	$GSEL(GDATA_SEL, SEL_KPL),%eax
	movl	%eax,%ds
	movl	%eax,%es
	movl	%eax,%ss
	jmp	_C_LABEL(x86_flush)
END(lgdt_finish)

/*
 * Flush instruction pipelines by doing an intersegment (far) return.
 */
ENTRY(x86_flush)
	popq	%rax
	pushq	$GSEL(GCODE_SEL, SEL_KPL)
	pushq	%rax
	lretq
END(x86_flush)

/* Waits - set up stack frame. */
ENTRY(x86_hlt)
	pushq	%rbp
	movq	%rsp, %rbp
	hlt
	leave
	ret
END(x86_hlt)

/* Waits - set up stack frame. */
ENTRY(x86_stihlt)
	pushq	%rbp
	movq	%rsp, %rbp
	sti
	hlt
	leave
	ret
END(x86_stihlt)

ENTRY(x86_monitor)
	movq	%rdi, %rax
	movq	%rsi, %rcx
	monitor	%rax, %rcx, %rdx
	ret
END(x86_monitor)

/* Waits - set up stack frame. */
ENTRY(x86_mwait)
	pushq	%rbp
	movq	%rsp, %rbp
	movq	%rdi, %rax
	movq	%rsi, %rcx
	mwait	%rax, %rcx
	leave
	ret
END(x86_mwait)

ENTRY(stts)
	movq	%cr0, %rax
	orq	$CR0_TS, %rax
	movq	%rax, %cr0
	ret
END(stts)

ENTRY(fldummy)
	ffree	%st(7)
	fldz
	ret
END(fldummy)

ENTRY(inb)
	movq	%rdi, %rdx
	xorq	%rax, %rax
	inb	%dx, %al
	KMSAN_INIT_RET(1)
	ret
END(inb)

ENTRY(inw)
	movq	%rdi, %rdx
	xorq	%rax, %rax
	inw	%dx, %ax
	KMSAN_INIT_RET(2)
	ret
END(inw)

ENTRY(inl)
	movq	%rdi, %rdx
	xorq	%rax, %rax
	inl	%dx, %eax
	KMSAN_INIT_RET(4)
	ret
END(inl)

ENTRY(outb)
	movq	%rdi, %rdx
	movq	%rsi, %rax
	outb	%al, %dx
	ret
END(outb)

ENTRY(outw)
	movq	%rdi, %rdx
	movq	%rsi, %rax
	outw	%ax, %dx
	ret
END(outw)

ENTRY(outl)
	movq	%rdi, %rdx
	movq	%rsi, %rax
	outl	%eax, %dx
	ret
END(outl)

/*
 * Used by SVS only, to make an atomic but fast copy. Doesn't have
 * sanitizer instrumentation, but sanitizers disable SVS, so no problem.
 */
ENTRY(svs_quad_copy)
	movq	%rdx,%rcx
	rep
	movsq
	ret
END(svs_quad_copy)