lock_stubs.S revision 1.31 
1/*	$NetBSD: lock_stubs.S,v 1.31 2019/02/11 14:59:32 cherry Exp $	*/
2
3/*-
4 * Copyright (c) 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Andrew Doran.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 *    notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 *    notice, this list of conditions and the following disclaimer in the
17 *    documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32/*
33 * AMD64 lock stubs.  Calling convention:
34 *
35 * %rdi		arg 1
36 * %rsi		arg 2
37 * %rdx		arg 3
38 * %rax		return value
39 */
40
41#include "opt_multiprocessor.h"
42#include "opt_lockdebug.h"
43
44#include <machine/asm.h>
45#include <machine/frameasm.h>
46
47#include "assym.h"
48
49#define	ENDLABEL(name,a) .align	a; LABEL(name)
50#define	LOCK(num)	\
51	HOTPATCH(HP_NAME_NOLOCK, 1)	; \
52	lock
53#define	RET(num)	\
54	HOTPATCH(HP_NAME_RETFENCE, 3)	; \
55	ret; nop; nop			; \
56	ret
57
58#ifndef LOCKDEBUG
59
60/*
61 * void mutex_enter(kmutex_t *mtx);
62 *
63 * Acquire a mutex and post a load fence.
64 */
65	.align	64
66
67ENTRY(mutex_enter)
68	movq	CPUVAR(CURLWP), %rcx
69	xorq	%rax, %rax
70	LOCK(1)
71	cmpxchgq %rcx, (%rdi)
72	jnz	1f
73	RET(1)
741:
75	jmp	_C_LABEL(mutex_vector_enter)
76END(mutex_enter)
77
78/*
79 * void mutex_exit(kmutex_t *mtx);
80 *
81 * Release a mutex and post a load fence.
82 *
83 * See comments in mutex_vector_enter() about doing this operation unlocked
84 * on multiprocessor systems, and comments in arch/x86/include/lock.h about
85 * memory ordering on Intel x86 systems.
86 */
87ENTRY(mutex_exit)
88	movq	CPUVAR(CURLWP), %rax
89	xorq	%rdx, %rdx
90	cmpxchgq %rdx, (%rdi)
91	jnz	1f
92	ret
931:
94	jmp	_C_LABEL(mutex_vector_exit)
95END(mutex_exit)
96
97/*
98 * void mutex_spin_enter(kmutex_t *mtx);
99 *
100 * Acquire a spin mutex and post a load fence.
101 */
102ENTRY(mutex_spin_enter)
103	movl	$1, %eax
104	movl	CPUVAR(ILEVEL), %esi
105	movzbl	MTX_IPL(%rdi), %ecx		/* new SPL */
106	cmpl	%ecx, %esi			/* higher? */
107	cmovgl	%esi, %ecx
108	movl	%ecx, CPUVAR(ILEVEL)		/* splraiseipl() */
109	subl	%eax, CPUVAR(MTX_COUNT)		/* decl doesnt set CF */
110	cmovncl	CPUVAR(MTX_OLDSPL), %esi
111	movl	%esi, CPUVAR(MTX_OLDSPL)
112	xchgb	%al, MTX_LOCK(%rdi)		/* lock */
113#ifdef MULTIPROCESSOR	/* XXX for xen */
114	testb	%al, %al
115	jnz	1f
116#endif
117	RET(2)
1181:
119	jmp	_C_LABEL(mutex_spin_retry)	/* failed; hard case */
120END(mutex_spin_enter)
121
122/*
123 * void mutex_spin_exit(kmutex_t *mtx);
124 *
125 * Release a spin mutex and post a load fence.
126 */
127ENTRY(mutex_spin_exit)
128#ifdef DIAGNOSTIC
129
130	movl	$0x0001, %eax			/* new + expected value */
131	movq	CPUVAR(SELF), %r8
132	cmpxchgb %ah, MTX_LOCK(%rdi)		/* unlock */
133	jnz	_C_LABEL(mutex_vector_exit)	/* hard case if problems */
134	movl	CPU_INFO_MTX_OLDSPL(%r8), %edi
135	incl	CPU_INFO_MTX_COUNT(%r8)
136	jnz	1f
137	cmpl	CPU_INFO_ILEVEL(%r8), %edi
138	jae	1f
139#if !defined(XENPV)
140	movl	CPU_INFO_IUNMASK(%r8,%rdi,4), %esi
141	CLI(ax)
142	testl	CPU_INFO_IPENDING(%r8), %esi
143	jnz	_C_LABEL(Xspllower)
144#endif
145#if defined(XEN)
146	movl	CPU_INFO_XUNMASK(%r8,%rdi,4), %esi
147	CLI(ax)
148	testl	CPU_INFO_XPENDING(%r8), %esi
149	jnz	_C_LABEL(Xspllower)
150#endif
151	movl	%edi, CPU_INFO_ILEVEL(%r8)
152	STI(ax)
1531:	rep					/* double byte ret as branch */
154	ret					/* target: see AMD docs */
155
156#else	/* DIAGNOSTIC */
157
158	movq	CPUVAR(SELF), %rsi
159	movb	$0x00, MTX_LOCK(%rdi)
160	movl	CPU_INFO_MTX_OLDSPL(%rsi), %ecx
161	incl	CPU_INFO_MTX_COUNT(%rsi)
162	movl	CPU_INFO_ILEVEL(%rsi),%edx
163	cmovnzl	%edx,%ecx
164	pushq	%rbx
165	cmpl	%edx,%ecx			/* new level is lower? */
166	jae	2f
1671:
168#if !defined(XENPV)
169	movl	CPU_INFO_IPENDING(%rsi),%eax
170	testl	%eax,CPU_INFO_IUNMASK(%rsi,%rcx,4)/* deferred interrupts? */
171	jnz	3f
172	movl	%eax,%ebx
173	cmpxchg8b CPU_INFO_ISTATE(%rsi)		/* swap in new ilevel */
174	jnz	4f
175#endif
176#if defined(XEN)
177	movl	CPU_INFO_XPENDING(%rsi),%eax
178	testl	%eax,CPU_INFO_XUNMASK(%rsi,%rcx,4)/* deferred interrupts? */
179	jnz	3f
180	movl	%edx, %eax
181	cmpxchgl %ecx, CPU_INFO_ILEVEL(%rsi)
182	jnz	4f
183#endif
1842:
185	popq	%rbx
186	ret
1873:
188	popq	%rbx
189	movl	%ecx, %edi
190	jmp	_C_LABEL(Xspllower)
1914:
192	jmp	1b
193
194#endif	/* DIAGNOSTIC */
195
196END(mutex_spin_exit)
197
198/*
199 * void	rw_enter(krwlock_t *rwl, krw_t op);
200 *
201 * Acquire one hold on a RW lock.
202 */
203ENTRY(rw_enter)
204	cmpl	$RW_READER, %esi
205	jne	2f
206
207	/*
208	 * Reader: this is the most common case.
209	 */
210	movq	(%rdi), %rax
2110:
212	testb	$(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al
213	jnz	3f
214	leaq	RW_READ_INCR(%rax), %rdx
215	LOCK(2)
216	cmpxchgq %rdx, (%rdi)
217	jnz	1f
218	RET(3)
2191:
220	jmp	0b
221
222	/*
223	 * Writer: if the compare-and-set fails, don't bother retrying.
224	 */
2252:	movq	CPUVAR(CURLWP), %rcx
226	xorq	%rax, %rax
227	orq	$RW_WRITE_LOCKED, %rcx
228	LOCK(3)
229	cmpxchgq %rcx, (%rdi)
230	jnz	3f
231	RET(4)
2323:
233	jmp	_C_LABEL(rw_vector_enter)
234END(rw_enter)
235
236/*
237 * void	rw_exit(krwlock_t *rwl);
238 *
239 * Release one hold on a RW lock.
240 */
241ENTRY(rw_exit)
242	movq	(%rdi), %rax
243	testb	$RW_WRITE_LOCKED, %al
244	jnz	2f
245
246	/*
247	 * Reader
248	 */
2490:	testb	$RW_HAS_WAITERS, %al
250	jnz	3f
251	cmpq	$RW_READ_INCR, %rax
252	jb	3f
253	leaq	-RW_READ_INCR(%rax), %rdx
254	LOCK(4)
255	cmpxchgq %rdx, (%rdi)
256	jnz	1f
257	ret
2581:
259	jmp	0b
260
261	/*
262	 * Writer
263	 */
2642:	leaq	-RW_WRITE_LOCKED(%rax), %rdx
265	subq	CPUVAR(CURLWP), %rdx
266	jnz	3f
267	LOCK(5)
268	cmpxchgq %rdx, (%rdi)
269	jnz	3f
270	ret
271
2723:	jmp	_C_LABEL(rw_vector_exit)
273END(rw_exit)
274
275/*
276 * int	rw_tryenter(krwlock_t *rwl, krw_t op);
277 *
278 * Try to acquire one hold on a RW lock.
279 */
280ENTRY(rw_tryenter)
281	cmpl	$RW_READER, %esi
282	jne	2f
283
284	/*
285	 * Reader: this is the most common case.
286	 */
287	movq	(%rdi), %rax
2880:
289	testb	$(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al
290	jnz	4f
291	leaq	RW_READ_INCR(%rax), %rdx
292	LOCK(8)
293	cmpxchgq %rdx, (%rdi)
294	jnz	1f
295	movl	%edx, %eax			/* nonzero */
296	RET(5)
2971:
298	jmp	0b
299
300	/*
301	 * Writer: if the compare-and-set fails, don't bother retrying.
302	 */
3032:	movq	CPUVAR(CURLWP), %rcx
304	xorq	%rax, %rax
305	orq	$RW_WRITE_LOCKED, %rcx
306	LOCK(9)
307	cmpxchgq %rcx, (%rdi)
308	movl	$0, %eax
309	setz	%al
3103:
311	RET(6)
312	ret
3134:
314	xorl	%eax, %eax
315	jmp	3b
316END(rw_tryenter)
317
318#endif	/* LOCKDEBUG */
319
320/*
321 * Spinlocks.
322 */
323ENTRY(__cpu_simple_lock_init)
324	movb	$0, (%rdi)
325	ret
326END(__cpu_simple_lock_init)
327
328ENTRY(__cpu_simple_lock)
329	movl	$0x0100, %eax
3301:
331	LOCK(6)
332	cmpxchgb %ah, (%rdi)
333	jnz	2f
334	RET(7)
3352:
336	movl	$0x0100, %eax
337	pause
338	nop
339	nop
340	cmpb	$0, (%rdi)
341	je	1b
342	jmp	2b
343END(__cpu_simple_lock)
344
345ENTRY(__cpu_simple_unlock)
346	movb	$0, (%rdi)
347	ret
348END(__cpu_simple_unlock)
349
350ENTRY(__cpu_simple_lock_try)
351	movl	$0x0100, %eax
352	LOCK(7)
353	cmpxchgb %ah, (%rdi)
354	movl	$0, %eax
355	setz	%al
356	RET(8)
357END(__cpu_simple_lock_try)
358
359