lock_stubs.S revision 1.19 
1/*	$NetBSD: lock_stubs.S,v 1.19 2008/05/25 15:56:12 chs Exp $	*/
2
3/*-
4 * Copyright (c) 2006, 2007, 2008 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Andrew Doran.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 *    notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 *    notice, this list of conditions and the following disclaimer in the
17 *    documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32/*
33 * AMD64 lock stubs.  Calling convention:
34 *
35 * %rdi		arg 1
36 * %rsi		arg 2
37 * %rdx		arg 3
38 * %rax		return value
39 */
40
41#include "opt_multiprocessor.h"
42#include "opt_lockdebug.h"
43
44#include <machine/asm.h>
45#include <machine/frameasm.h>
46
47#include "assym.h"
48
49#if defined(DIAGNOSTIC) || defined(MULTIPROCESSOR) || defined(LOCKDEBUG)
50#define	FULL
51#endif
52
53#define	ENDLABEL(name,a) .align	a; LABEL(name)
54#define	LOCK(num)	.Lpatch/**/num: lock
55
56#ifndef LOCKDEBUG
57
58/*
59 * void mutex_enter(kmutex_t *mtx);
60 *
61 * Acquire a mutex and post a load fence.
62 */
63	.align	64
64
65ENTRY(mutex_enter)
66	movq	CPUVAR(CURLWP), %rcx
67	xorq	%rax, %rax
68	LOCK(1)
69	cmpxchgq %rcx, MTX_OWNER(%rdi)
70	jnz	1f
71	ret
721:
73	jmp	_C_LABEL(mutex_vector_enter)
74
75/*
76 * void mutex_exit(kmutex_t *mtx);
77 *
78 * Release a mutex and post a load fence.
79 *
80 * See comments in mutex_vector_enter() about doing this operation unlocked
81 * on multiprocessor systems, and comments in arch/x86/include/lock.h about
82 * memory ordering on Intel x86 systems.
83 */
84ENTRY(mutex_exit)
85	movq	CPUVAR(CURLWP), %rax
86	xorq	%rdx, %rdx
87	cmpxchgq %rdx, MTX_OWNER(%rdi)
88	jnz	1f
89	ret
901:
91	jmp	_C_LABEL(mutex_vector_exit)
92
93/*
94 * void mutex_spin_enter(kmutex_t *mtx);
95 *
96 * Acquire a spin mutex and post a load fence.
97 */
98ENTRY(mutex_spin_enter)
99#if defined(FULL)
100	movl	$0x0100, %eax			/* new + expected value */
101#endif
102	movl	CPUVAR(ILEVEL), %esi
103	movzbl	MTX_IPL(%rdi), %ecx		/* new SPL */
104	cmpl	%ecx, %esi			/* higher? */
105	cmovgl	%esi, %ecx
106	movl	%ecx, CPUVAR(ILEVEL)		/* splraiseipl() */
107	subl	$1, CPUVAR(MTX_COUNT)		/* decl doesnt set CF */
108	cmovncl	CPUVAR(MTX_OLDSPL), %esi
109	movl	%esi, CPUVAR(MTX_OLDSPL)
110#if defined(FULL)
111	LOCK(11)
112	cmpxchgb %ah, MTX_LOCK(%rdi)		/* lock */
113	jnz	1f
114#endif
115	ret
1161:
117	jmp	_C_LABEL(mutex_spin_retry)	/* failed; hard case */
118
119/*
120 * void mutex_spin_exit(kmutex_t *mtx);
121 *
122 * Release a spin mutex and post a load fence.
123 */
124ENTRY(mutex_spin_exit)
125#ifdef DIAGNOSTIC
126
127	movl	$0x0001, %eax			/* new + expected value */
128	movq	CPUVAR(SELF), %r8
129	cmpxchgb %ah, MTX_LOCK(%rdi)		/* unlock */
130	jnz,pn	_C_LABEL(mutex_vector_exit)	/* hard case if problems */
131	movl	CPU_INFO_MTX_OLDSPL(%r8), %edi
132	incl	CPU_INFO_MTX_COUNT(%r8)
133	jnz	1f
134	cmpl	CPU_INFO_ILEVEL(%r8), %edi
135	jae	1f
136	movl	CPU_INFO_IUNMASK(%r8,%rdi,4), %esi
137	CLI(ax)
138	testl	CPU_INFO_IPENDING(%r8), %esi
139	jnz	_C_LABEL(Xspllower)
140	movl	%edi, CPU_INFO_ILEVEL(%r8)
141	STI(ax)
1421:	rep					/* double byte ret as branch */
143	ret					/* target: see AMD docs */
144
145#else	/* DIAGNOSTIC */
146
147	movq	CPUVAR(SELF), %rsi
148#ifdef MULTIPROCESSOR
149	movb	$0x00, MTX_LOCK(%rdi)
150#endif
151	movl	CPU_INFO_MTX_OLDSPL(%rsi), %ecx
152	incl	CPU_INFO_MTX_COUNT(%rsi)
153	movl	CPU_INFO_ILEVEL(%rsi),%edx
154	cmovnzl	%edx,%ecx
155	cmpl	%edx,%ecx			/* new level is lower? */
156	pushq	%rbx
157	jae,pn	2f
1581:
159	movl	CPU_INFO_IPENDING(%rsi),%eax
160	testl	%eax,CPU_INFO_IUNMASK(%rsi,%rcx,4)/* deferred interrupts? */
161	movl	%eax,%ebx
162	jnz,pn	3f
163	cmpxchg8b CPU_INFO_ISTATE(%rsi)		/* swap in new ilevel */
164	jnz,pn	1b
1652:
166	popq	%rbx
167	ret
1683:
169	popq	%rbx
170	movl	%ecx, %edi
171	jmp	_C_LABEL(Xspllower)
172
173#endif	/* DIAGNOSTIC */
174
175/*
176 * void	rw_enter(krwlock_t *rwl, krw_t op);
177 *
178 * Acquire one hold on a RW lock.
179 */
180ENTRY(rw_enter)
181	cmpl	$RW_READER, %esi
182	jne	2f
183
184	/*
185	 * Reader: this is the most common case.
186	 */
1871:	movq	RW_OWNER(%rdi), %rax
188	testb	$(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al
189	leaq	RW_READ_INCR(%rax), %rdx
190	jnz	3f
191	LOCK(2)
192	cmpxchgq %rdx, RW_OWNER(%rdi)
193	jnz,pn	1b
194	ret
195
196	/*
197	 * Writer: if the compare-and-set fails, don't bother retrying.
198	 */
1992:	movq	CPUVAR(CURLWP), %rcx
200	xorq	%rax, %rax
201	orq	$RW_WRITE_LOCKED, %rcx
202	LOCK(3)
203	cmpxchgq %rcx, RW_OWNER(%rdi)
204	jnz	3f
205	ret
2063:
207	jmp	_C_LABEL(rw_vector_enter)
208
209/*
210 * void	rw_exit(krwlock_t *rwl);
211 *
212 * Release one hold on a RW lock.
213 */
214ENTRY(rw_exit)
215	movq	RW_OWNER(%rdi), %rax
216	testb	$RW_WRITE_LOCKED, %al
217	jnz	2f
218
219	/*
220	 * Reader
221	 */
2221:	testb	$RW_HAS_WAITERS, %al
223	jnz	3f
224	cmpq	$RW_READ_INCR, %rax
225	leaq	-RW_READ_INCR(%rax), %rdx
226	jb	3f
227	LOCK(4)
228	cmpxchgq %rdx, RW_OWNER(%rdi)
229	jnz,pn	1b
230	ret
231
232	/*
233	 * Writer
234	 */
2352:	leaq	-RW_WRITE_LOCKED(%rax), %rdx
236	subq	CPUVAR(CURLWP), %rdx
237	jnz	3f
238	LOCK(5)
239	cmpxchgq %rdx, RW_OWNER(%rdi)
240	jnz	3f
241	ret
242
2433:	jmp	_C_LABEL(rw_vector_exit)
244
245/*
246 * int	rw_tryenter(krwlock_t *rwl, krw_t op);
247 *
248 * Try to acquire one hold on a RW lock.
249 */
250ENTRY(rw_tryenter)
251	cmpl	$RW_READER, %esi
252	jne	2f
253
254	/*
255	 * Reader: this is the most common case.
256	 */
257	movq	RW_OWNER(%rdi), %rax
2581:
259	testb	$(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al
260	leaq	RW_READ_INCR(%rax), %rdx
261	jnz	3f
262	LOCK(8)
263	cmpxchgq %rdx, RW_OWNER(%rdi)
264	jnz	1b
265	movl	%edi, %eax			/* nonzero */
266	ret
267
268	/*
269	 * Writer: if the compare-and-set fails, don't bother retrying.
270	 */
2712:	movq	CPUVAR(CURLWP), %rcx
272	xorq	%rax, %rax
273	orq	$RW_WRITE_LOCKED, %rcx
274	LOCK(9)
275	cmpxchgq %rcx, RW_OWNER(%rdi)
276	movl	$0, %eax
277	setz	%al
278	ret
279
2803:	xorl	%eax, %eax
281	ret
282
283#endif	/* LOCKDEBUG */
284
285/*
286 * Spinlocks.
287 */
288ENTRY(__cpu_simple_lock_init)
289	movb	$0, (%rdi)
290	ret
291
292NENTRY(__cpu_simple_lock)
293	movl	$0x0100, %eax
2941:
295	LOCK(6)
296	cmpxchgb %ah, (%rdi)
297	jnz	2f
298	ret
2992:
300	movl	$0x0100, %eax
301	pause
302	nop
303	nop
304	cmpb	$0, (%rdi)
305	je	1b
306	jmp	2b
307
308ENTRY(__cpu_simple_unlock)
309	movb	$0, (%rdi)
310	ret
311
312ENTRY(__cpu_simple_lock_try)
313	movl	$0x0100, %eax
314	LOCK(7)
315	cmpxchgb %ah, (%rdi)
316	movl	$0, %eax
317	setz	%al
318	ret
319
320/*
321 * Patchpoints to replace with NOP when ncpu == 1.
322 */
323#ifndef LOCKDEBUG
324LABEL(x86_lockpatch)
325	.quad	.Lpatch1, .Lpatch2, .Lpatch3, .Lpatch4
326	.quad	.Lpatch5, .Lpatch6, .Lpatch7, .Lpatch8
327	.quad	.Lpatch9
328#ifdef FULL
329	.quad	.Lpatch11
330#endif
331	.quad	0
332#endif
333