lock_stubs.S revision 1.29 
1/*	$NetBSD: lock_stubs.S,v 1.29 2018/07/14 14:29:40 maxv Exp $	*/
2
3/*-
4 * Copyright (c) 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Andrew Doran.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 *    notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 *    notice, this list of conditions and the following disclaimer in the
17 *    documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32/*
33 * AMD64 lock stubs.  Calling convention:
34 *
35 * %rdi		arg 1
36 * %rsi		arg 2
37 * %rdx		arg 3
38 * %rax		return value
39 */
40
41#include "opt_multiprocessor.h"
42#include "opt_lockdebug.h"
43
44#include <machine/asm.h>
45#include <machine/frameasm.h>
46
47#include "assym.h"
48
49#define	ENDLABEL(name,a) .align	a; LABEL(name)
50#define	LOCK(num)	\
51	HOTPATCH(HP_NAME_NOLOCK, 1)	; \
52	lock
53#define	RET(num)	\
54	HOTPATCH(HP_NAME_RETFENCE, 3)	; \
55	ret; nop; nop			; \
56	ret
57
58#ifndef LOCKDEBUG
59
60/*
61 * void mutex_enter(kmutex_t *mtx);
62 *
63 * Acquire a mutex and post a load fence.
64 */
65	.align	64
66
67ENTRY(mutex_enter)
68	movq	CPUVAR(CURLWP), %rcx
69	xorq	%rax, %rax
70	LOCK(1)
71	cmpxchgq %rcx, (%rdi)
72	jnz	1f
73	RET(1)
741:
75	jmp	_C_LABEL(mutex_vector_enter)
76END(mutex_enter)
77
78/*
79 * void mutex_exit(kmutex_t *mtx);
80 *
81 * Release a mutex and post a load fence.
82 *
83 * See comments in mutex_vector_enter() about doing this operation unlocked
84 * on multiprocessor systems, and comments in arch/x86/include/lock.h about
85 * memory ordering on Intel x86 systems.
86 */
87ENTRY(mutex_exit)
88	movq	CPUVAR(CURLWP), %rax
89	xorq	%rdx, %rdx
90	cmpxchgq %rdx, (%rdi)
91	jnz	1f
92	ret
931:
94	jmp	_C_LABEL(mutex_vector_exit)
95END(mutex_exit)
96
97/*
98 * void mutex_spin_enter(kmutex_t *mtx);
99 *
100 * Acquire a spin mutex and post a load fence.
101 */
102ENTRY(mutex_spin_enter)
103	movl	$1, %eax
104	movl	CPUVAR(ILEVEL), %esi
105	movzbl	MTX_IPL(%rdi), %ecx		/* new SPL */
106	cmpl	%ecx, %esi			/* higher? */
107	cmovgl	%esi, %ecx
108	movl	%ecx, CPUVAR(ILEVEL)		/* splraiseipl() */
109	subl	%eax, CPUVAR(MTX_COUNT)		/* decl doesnt set CF */
110	cmovncl	CPUVAR(MTX_OLDSPL), %esi
111	movl	%esi, CPUVAR(MTX_OLDSPL)
112	xchgb	%al, MTX_LOCK(%rdi)		/* lock */
113#ifdef MULTIPROCESSOR	/* XXX for xen */
114	testb	%al, %al
115	jnz	1f
116#endif
117	RET(2)
1181:
119	jmp	_C_LABEL(mutex_spin_retry)	/* failed; hard case */
120END(mutex_spin_enter)
121
122/*
123 * void mutex_spin_exit(kmutex_t *mtx);
124 *
125 * Release a spin mutex and post a load fence.
126 */
127ENTRY(mutex_spin_exit)
128#ifdef DIAGNOSTIC
129
130	movl	$0x0001, %eax			/* new + expected value */
131	movq	CPUVAR(SELF), %r8
132	cmpxchgb %ah, MTX_LOCK(%rdi)		/* unlock */
133	jnz	_C_LABEL(mutex_vector_exit)	/* hard case if problems */
134	movl	CPU_INFO_MTX_OLDSPL(%r8), %edi
135	incl	CPU_INFO_MTX_COUNT(%r8)
136	jnz	1f
137	cmpl	CPU_INFO_ILEVEL(%r8), %edi
138	jae	1f
139	movl	CPU_INFO_IUNMASK(%r8,%rdi,4), %esi
140	CLI(ax)
141	testl	CPU_INFO_IPENDING(%r8), %esi
142	jnz	_C_LABEL(Xspllower)
143	movl	%edi, CPU_INFO_ILEVEL(%r8)
144	STI(ax)
1451:	rep					/* double byte ret as branch */
146	ret					/* target: see AMD docs */
147
148#else	/* DIAGNOSTIC */
149
150	movq	CPUVAR(SELF), %rsi
151	movb	$0x00, MTX_LOCK(%rdi)
152	movl	CPU_INFO_MTX_OLDSPL(%rsi), %ecx
153	incl	CPU_INFO_MTX_COUNT(%rsi)
154	movl	CPU_INFO_ILEVEL(%rsi),%edx
155	cmovnzl	%edx,%ecx
156	pushq	%rbx
157	cmpl	%edx,%ecx			/* new level is lower? */
158	jae	2f
1591:
160	movl	CPU_INFO_IPENDING(%rsi),%eax
161	testl	%eax,CPU_INFO_IUNMASK(%rsi,%rcx,4)/* deferred interrupts? */
162	jnz	3f
163	movl	%eax,%ebx
164	cmpxchg8b CPU_INFO_ISTATE(%rsi)		/* swap in new ilevel */
165	jnz	4f
1662:
167	popq	%rbx
168	ret
1693:
170	popq	%rbx
171	movl	%ecx, %edi
172	jmp	_C_LABEL(Xspllower)
1734:
174	jmp	1b
175
176#endif	/* DIAGNOSTIC */
177
178END(mutex_spin_exit)
179
180/*
181 * void	rw_enter(krwlock_t *rwl, krw_t op);
182 *
183 * Acquire one hold on a RW lock.
184 */
185ENTRY(rw_enter)
186	cmpl	$RW_READER, %esi
187	jne	2f
188
189	/*
190	 * Reader: this is the most common case.
191	 */
192	movq	(%rdi), %rax
1930:
194	testb	$(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al
195	jnz	3f
196	leaq	RW_READ_INCR(%rax), %rdx
197	LOCK(2)
198	cmpxchgq %rdx, (%rdi)
199	jnz	1f
200	RET(3)
2011:
202	jmp	0b
203
204	/*
205	 * Writer: if the compare-and-set fails, don't bother retrying.
206	 */
2072:	movq	CPUVAR(CURLWP), %rcx
208	xorq	%rax, %rax
209	orq	$RW_WRITE_LOCKED, %rcx
210	LOCK(3)
211	cmpxchgq %rcx, (%rdi)
212	jnz	3f
213	RET(4)
2143:
215	jmp	_C_LABEL(rw_vector_enter)
216END(rw_enter)
217
218/*
219 * void	rw_exit(krwlock_t *rwl);
220 *
221 * Release one hold on a RW lock.
222 */
223ENTRY(rw_exit)
224	movq	(%rdi), %rax
225	testb	$RW_WRITE_LOCKED, %al
226	jnz	2f
227
228	/*
229	 * Reader
230	 */
2310:	testb	$RW_HAS_WAITERS, %al
232	jnz	3f
233	cmpq	$RW_READ_INCR, %rax
234	jb	3f
235	leaq	-RW_READ_INCR(%rax), %rdx
236	LOCK(4)
237	cmpxchgq %rdx, (%rdi)
238	jnz	1f
239	ret
2401:
241	jmp	0b
242
243	/*
244	 * Writer
245	 */
2462:	leaq	-RW_WRITE_LOCKED(%rax), %rdx
247	subq	CPUVAR(CURLWP), %rdx
248	jnz	3f
249	LOCK(5)
250	cmpxchgq %rdx, (%rdi)
251	jnz	3f
252	ret
253
2543:	jmp	_C_LABEL(rw_vector_exit)
255END(rw_exit)
256
257/*
258 * int	rw_tryenter(krwlock_t *rwl, krw_t op);
259 *
260 * Try to acquire one hold on a RW lock.
261 */
262ENTRY(rw_tryenter)
263	cmpl	$RW_READER, %esi
264	jne	2f
265
266	/*
267	 * Reader: this is the most common case.
268	 */
269	movq	(%rdi), %rax
2700:
271	testb	$(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al
272	jnz	4f
273	leaq	RW_READ_INCR(%rax), %rdx
274	LOCK(8)
275	cmpxchgq %rdx, (%rdi)
276	jnz	1f
277	movl	%edx, %eax			/* nonzero */
278	RET(5)
2791:
280	jmp	0b
281
282	/*
283	 * Writer: if the compare-and-set fails, don't bother retrying.
284	 */
2852:	movq	CPUVAR(CURLWP), %rcx
286	xorq	%rax, %rax
287	orq	$RW_WRITE_LOCKED, %rcx
288	LOCK(9)
289	cmpxchgq %rcx, (%rdi)
290	movl	$0, %eax
291	setz	%al
2923:
293	RET(6)
294	ret
2954:
296	xorl	%eax, %eax
297	jmp	3b
298END(rw_tryenter)
299
300#endif	/* LOCKDEBUG */
301
302/*
303 * Spinlocks.
304 */
305ENTRY(__cpu_simple_lock_init)
306	movb	$0, (%rdi)
307	ret
308END(__cpu_simple_lock_init)
309
310ENTRY(__cpu_simple_lock)
311	movl	$0x0100, %eax
3121:
313	LOCK(6)
314	cmpxchgb %ah, (%rdi)
315	jnz	2f
316	RET(7)
3172:
318	movl	$0x0100, %eax
319	pause
320	nop
321	nop
322	cmpb	$0, (%rdi)
323	je	1b
324	jmp	2b
325END(__cpu_simple_lock)
326
327ENTRY(__cpu_simple_unlock)
328	movb	$0, (%rdi)
329	ret
330END(__cpu_simple_unlock)
331
332ENTRY(__cpu_simple_lock_try)
333	movl	$0x0100, %eax
334	LOCK(7)
335	cmpxchgb %ah, (%rdi)
336	movl	$0, %eax
337	setz	%al
338	RET(8)
339END(__cpu_simple_lock_try)
340
341