lock_stubs.S revision 1.27 
1/*	$NetBSD: lock_stubs.S,v 1.27 2018/01/07 13:15:23 maxv Exp $	*/
2
3/*-
4 * Copyright (c) 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Andrew Doran.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 *    notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 *    notice, this list of conditions and the following disclaimer in the
17 *    documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32/*
33 * AMD64 lock stubs.  Calling convention:
34 *
35 * %rdi		arg 1
36 * %rsi		arg 2
37 * %rdx		arg 3
38 * %rax		return value
39 */
40
41#include "opt_multiprocessor.h"
42#include "opt_lockdebug.h"
43
44#include <machine/asm.h>
45#include <machine/frameasm.h>
46
47#include "assym.h"
48
49#define	ENDLABEL(name,a) .align	a; LABEL(name)
50#define	LOCK(num)	\
51	HOTPATCH(HP_NAME_NOLOCK, 1)	; \
52	lock
53#define	RET(num)	.Lret ## num: ret; nop; nop; ret
54
55#ifndef LOCKDEBUG
56
57/*
58 * void mutex_enter(kmutex_t *mtx);
59 *
60 * Acquire a mutex and post a load fence.
61 */
62	.align	64
63
64ENTRY(mutex_enter)
65	movq	CPUVAR(CURLWP), %rcx
66	xorq	%rax, %rax
67	LOCK(1)
68	cmpxchgq %rcx, (%rdi)
69	jnz	1f
70	RET(1)
711:
72	jmp	_C_LABEL(mutex_vector_enter)
73END(mutex_enter)
74
75/*
76 * void mutex_exit(kmutex_t *mtx);
77 *
78 * Release a mutex and post a load fence.
79 *
80 * See comments in mutex_vector_enter() about doing this operation unlocked
81 * on multiprocessor systems, and comments in arch/x86/include/lock.h about
82 * memory ordering on Intel x86 systems.
83 */
84ENTRY(mutex_exit)
85	movq	CPUVAR(CURLWP), %rax
86	xorq	%rdx, %rdx
87	cmpxchgq %rdx, (%rdi)
88	jnz	1f
89	ret
901:
91	jmp	_C_LABEL(mutex_vector_exit)
92END(mutex_exit)
93
94/*
95 * void mutex_spin_enter(kmutex_t *mtx);
96 *
97 * Acquire a spin mutex and post a load fence.
98 */
99ENTRY(mutex_spin_enter)
100	movl	$1, %eax
101	movl	CPUVAR(ILEVEL), %esi
102	movzbl	MTX_IPL(%rdi), %ecx		/* new SPL */
103	cmpl	%ecx, %esi			/* higher? */
104	cmovgl	%esi, %ecx
105	movl	%ecx, CPUVAR(ILEVEL)		/* splraiseipl() */
106	subl	%eax, CPUVAR(MTX_COUNT)		/* decl doesnt set CF */
107	cmovncl	CPUVAR(MTX_OLDSPL), %esi
108	movl	%esi, CPUVAR(MTX_OLDSPL)
109	xchgb	%al, MTX_LOCK(%rdi)		/* lock */
110#ifdef MULTIPROCESSOR	/* XXX for xen */
111	testb	%al, %al
112	jnz	1f
113#endif
114	RET(2)
1151:
116	jmp	_C_LABEL(mutex_spin_retry)	/* failed; hard case */
117END(mutex_spin_enter)
118
119/*
120 * void mutex_spin_exit(kmutex_t *mtx);
121 *
122 * Release a spin mutex and post a load fence.
123 */
124ENTRY(mutex_spin_exit)
125#ifdef DIAGNOSTIC
126
127	movl	$0x0001, %eax			/* new + expected value */
128	movq	CPUVAR(SELF), %r8
129	cmpxchgb %ah, MTX_LOCK(%rdi)		/* unlock */
130	jnz	_C_LABEL(mutex_vector_exit)	/* hard case if problems */
131	movl	CPU_INFO_MTX_OLDSPL(%r8), %edi
132	incl	CPU_INFO_MTX_COUNT(%r8)
133	jnz	1f
134	cmpl	CPU_INFO_ILEVEL(%r8), %edi
135	jae	1f
136	movl	CPU_INFO_IUNMASK(%r8,%rdi,4), %esi
137	CLI(ax)
138	testl	CPU_INFO_IPENDING(%r8), %esi
139	jnz	_C_LABEL(Xspllower)
140	movl	%edi, CPU_INFO_ILEVEL(%r8)
141	STI(ax)
1421:	rep					/* double byte ret as branch */
143	ret					/* target: see AMD docs */
144
145#else	/* DIAGNOSTIC */
146
147	movq	CPUVAR(SELF), %rsi
148	movb	$0x00, MTX_LOCK(%rdi)
149	movl	CPU_INFO_MTX_OLDSPL(%rsi), %ecx
150	incl	CPU_INFO_MTX_COUNT(%rsi)
151	movl	CPU_INFO_ILEVEL(%rsi),%edx
152	cmovnzl	%edx,%ecx
153	pushq	%rbx
154	cmpl	%edx,%ecx			/* new level is lower? */
155	jae	2f
1561:
157	movl	CPU_INFO_IPENDING(%rsi),%eax
158	testl	%eax,CPU_INFO_IUNMASK(%rsi,%rcx,4)/* deferred interrupts? */
159	jnz	3f
160	movl	%eax,%ebx
161	cmpxchg8b CPU_INFO_ISTATE(%rsi)		/* swap in new ilevel */
162	jnz	4f
1632:
164	popq	%rbx
165	ret
1663:
167	popq	%rbx
168	movl	%ecx, %edi
169	jmp	_C_LABEL(Xspllower)
1704:
171	jmp	1b
172
173#endif	/* DIAGNOSTIC */
174
175END(mutex_spin_exit)
176
177/*
178 * void	rw_enter(krwlock_t *rwl, krw_t op);
179 *
180 * Acquire one hold on a RW lock.
181 */
182ENTRY(rw_enter)
183	cmpl	$RW_READER, %esi
184	jne	2f
185
186	/*
187	 * Reader: this is the most common case.
188	 */
189	movq	(%rdi), %rax
1900:
191	testb	$(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al
192	jnz	3f
193	leaq	RW_READ_INCR(%rax), %rdx
194	LOCK(2)
195	cmpxchgq %rdx, (%rdi)
196	jnz	1f
197	RET(3)
1981:
199	jmp	0b
200
201	/*
202	 * Writer: if the compare-and-set fails, don't bother retrying.
203	 */
2042:	movq	CPUVAR(CURLWP), %rcx
205	xorq	%rax, %rax
206	orq	$RW_WRITE_LOCKED, %rcx
207	LOCK(3)
208	cmpxchgq %rcx, (%rdi)
209	jnz	3f
210	RET(4)
2113:
212	jmp	_C_LABEL(rw_vector_enter)
213END(rw_enter)
214
215/*
216 * void	rw_exit(krwlock_t *rwl);
217 *
218 * Release one hold on a RW lock.
219 */
220ENTRY(rw_exit)
221	movq	(%rdi), %rax
222	testb	$RW_WRITE_LOCKED, %al
223	jnz	2f
224
225	/*
226	 * Reader
227	 */
2280:	testb	$RW_HAS_WAITERS, %al
229	jnz	3f
230	cmpq	$RW_READ_INCR, %rax
231	jb	3f
232	leaq	-RW_READ_INCR(%rax), %rdx
233	LOCK(4)
234	cmpxchgq %rdx, (%rdi)
235	jnz	1f
236	ret
2371:
238	jmp	0b
239
240	/*
241	 * Writer
242	 */
2432:	leaq	-RW_WRITE_LOCKED(%rax), %rdx
244	subq	CPUVAR(CURLWP), %rdx
245	jnz	3f
246	LOCK(5)
247	cmpxchgq %rdx, (%rdi)
248	jnz	3f
249	ret
250
2513:	jmp	_C_LABEL(rw_vector_exit)
252END(rw_exit)
253
254/*
255 * int	rw_tryenter(krwlock_t *rwl, krw_t op);
256 *
257 * Try to acquire one hold on a RW lock.
258 */
259ENTRY(rw_tryenter)
260	cmpl	$RW_READER, %esi
261	jne	2f
262
263	/*
264	 * Reader: this is the most common case.
265	 */
266	movq	(%rdi), %rax
2670:
268	testb	$(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al
269	jnz	4f
270	leaq	RW_READ_INCR(%rax), %rdx
271	LOCK(8)
272	cmpxchgq %rdx, (%rdi)
273	jnz	1f
274	movl	%edx, %eax			/* nonzero */
275	RET(5)
2761:
277	jmp	0b
278
279	/*
280	 * Writer: if the compare-and-set fails, don't bother retrying.
281	 */
2822:	movq	CPUVAR(CURLWP), %rcx
283	xorq	%rax, %rax
284	orq	$RW_WRITE_LOCKED, %rcx
285	LOCK(9)
286	cmpxchgq %rcx, (%rdi)
287	movl	$0, %eax
288	setz	%al
2893:
290	RET(6)
291	ret
2924:
293	xorl	%eax, %eax
294	jmp	3b
295END(rw_tryenter)
296
297#endif	/* LOCKDEBUG */
298
299/*
300 * Spinlocks.
301 */
302ENTRY(__cpu_simple_lock_init)
303	movb	$0, (%rdi)
304	ret
305END(__cpu_simple_lock_init)
306
307NENTRY(__cpu_simple_lock)
308	movl	$0x0100, %eax
3091:
310	LOCK(6)
311	cmpxchgb %ah, (%rdi)
312	jnz	2f
313	RET(7)
3142:
315	movl	$0x0100, %eax
316	pause
317	nop
318	nop
319	cmpb	$0, (%rdi)
320	je	1b
321	jmp	2b
322END(__cpu_simple_lock)
323
324NENTRY(__cpu_simple_unlock)
325	movb	$0, (%rdi)
326	ret
327END(__cpu_simple_unlock)
328
329ENTRY(__cpu_simple_lock_try)
330	movl	$0x0100, %eax
331	LOCK(7)
332	cmpxchgb %ah, (%rdi)
333	movl	$0, %eax
334	setz	%al
335	RET(8)
336END(__cpu_simple_lock_try)
337
338	.type	_C_LABEL(x86_retpatch), @object
339LABEL(x86_retpatch)
340#ifndef LOCKDEBUG
341	.quad	.Lret1, .Lret2, .Lret3, .Lret4, .Lret5, .Lret6
342#endif
343	.quad	.Lret7, .Lret8
344	.quad	0
345END(x86_retpatch)
346