lock_stubs.S revision 1.7 
1/*	$NetBSD: lock_stubs.S,v 1.7 2007/11/10 20:06:23 ad Exp $	*/
2
3/*-
4 * Copyright (c) 2006, 2007 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Andrew Doran.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 *    notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 *    notice, this list of conditions and the following disclaimer in the
17 *    documentation and/or other materials provided with the distribution.
18 * 3. All advertising materials mentioning features or use of this software
19 *    must display the following acknowledgement:
20 *	This product includes software developed by the NetBSD
21 *	Foundation, Inc. and its contributors.
22 * 4. Neither the name of The NetBSD Foundation nor the names of its
23 *    contributors may be used to endorse or promote products derived
24 *    from this software without specific prior written permission.
25 *
26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
36 * POSSIBILITY OF SUCH DAMAGE.
37 */
38
39/*
40 * AMD64 lock stubs.  Calling convention:
41 *
42 * %rdi		arg 1
43 * %rsi		arg 2
44 * %rdx		arg 3
45 * %rax		return value
46 */
47
48#include "opt_multiprocessor.h"
49#include "opt_lockdebug.h"
50
51#include <machine/asm.h>
52
53#include "assym.h"
54
55#if defined(DIAGNOSTIC) || defined(MULTIPROCESSOR) || defined(LOCKDEBUG)
56#define	FULL
57#endif
58
59#define	END(name,a)	.align	a; LABEL(name)
60
61#define	LOCK(name)	LABEL(name) lock
62
63#ifndef LOCKDEBUG
64
65/*
66 * void mutex_enter(kmutex_t *mtx);
67 *
68 * Acquire a mutex and post a load fence.
69 */
70	.align	64
71
72NENTRY(mutex_enter)				/* 0x0000, 25 bytes */
73	movq	CPUVAR(CURLWP), %rcx
74	xorq	%rax, %rax
75	LOCK(lockpatch1)
76	cmpxchgq %rcx, MTX_OWNER(%rdi)
77	jnz,pn	_C_LABEL(mutex_vector_enter)
78	ret
79
80/*
81 * void mutex_exit(kmutex_t *mtx);
82 *
83 * Release a mutex and post a load fence.
84 *
85 * See comments in mutex_vector_enter() about doing this operation unlocked
86 * on multiprocessor systems, and comments in arch/x86/include/lock.h about
87 * memory ordering on Intel x86 systems.
88 */
89NENTRY(mutex_exit)				/* 0x0020, 24 bytes */
90	movq	CPUVAR(CURLWP), %rax
91	xorq	%rdx, %rdx
92	cmpxchgq %rdx, MTX_OWNER(%rdi)
93	jnz,pn	_C_LABEL(mutex_vector_exit)
94	ret
95
96/*
97 * void mutex_spin_enter(kmutex_t *mtx);
98 *
99 * Acquire a spin mutex and post a load fence.
100 */
101NENTRY(mutex_spin_enter)
102	movq	CPUVAR(SELF), %r8
103#if defined(FULL)
104	movl	$0x0100, %eax			/* new + expected value */
105#endif
106	movl	CPU_INFO_ILEVEL(%r8), %esi
107	subl	$1, CPU_INFO_MTX_COUNT(%r8)	/* decl doesnt set CF */
108	movzbl	MTX_IPL(%rdi), %ecx		/* new SPL */
109	cmovncl	CPU_INFO_MTX_OLDSPL(%r8), %esi
110	cmpl	%ecx, %esi			/* higher? */
111	movl	%esi, CPU_INFO_MTX_OLDSPL(%r8)
112	cmovgl	%esi, %ecx
113	movl	%ecx, CPU_INFO_ILEVEL(%r8)	/* splraiseipl() */
114#if defined(FULL)
115	LOCK(lockpatch11)
116	cmpxchgb %ah, MTX_LOCK(%rdi)		/* lock */
117	jnz,pn	_C_LABEL(mutex_spin_retry)	/* failed; hard case */
118#endif
119	ret
120
121/*
122 * void mutex_spin_exit(kmutex_t *mtx);
123 *
124 * Release a spin mutex and post a load fence.
125 */
126NENTRY(mutex_spin_exit)
127#ifdef DIAGNOSTIC
128
129	movl	$0x0001, %eax			/* new + expected value */
130	movq	CPUVAR(SELF), %r8
131	cmpxchgb %ah, MTX_LOCK(%rdi)		/* unlock */
132	jnz,pn	_C_LABEL(mutex_vector_exit)	/* hard case if problems */
133	movl	CPU_INFO_MTX_OLDSPL(%r8), %edi
134	incl	CPU_INFO_MTX_COUNT(%r8)
135	jnz	1f
136	cmpl	CPU_INFO_ILEVEL(%r8), %edi
137	jae	1f
138	movl	CPU_INFO_IUNMASK(%r8,%rdi,4), %esi
139	cli
140	testl	CPU_INFO_IPENDING(%r8), %esi
141	jnz	_C_LABEL(Xspllower)
142	movl	%edi, CPU_INFO_ILEVEL(%r8)
143	sti
1441:	rep					/* double byte ret as branch */
145	ret					/* target: see AMD docs */
146
147#else	/* DIAGNOSTIC */
148
149	movq	CPUVAR(SELF), %rsi
150#ifdef MULTIPROCESSOR
151	movb	$0x00, MTX_LOCK(%rdi)
152#endif
153	movl	CPU_INFO_MTX_OLDSPL(%rsi), %ecx
154	incl	CPU_INFO_MTX_COUNT(%rsi)
155	movl	CPU_INFO_ILEVEL(%rsi),%edx
156	cmovnzl	%edx,%ecx
157	cmpl	%edx,%ecx			/* new level is lower? */
158	pushq	%rbx
159	jae,pn	2f
1601:
161	movl	CPU_INFO_IPENDING(%rsi),%eax
162	testl	%eax,CPU_INFO_IUNMASK(%rsi,%rcx,4)/* deferred interrupts? */
163	movl	%eax,%ebx
164	jnz,pn	3f
165	cmpxchg8b CPU_INFO_ISTATE(%rsi)		/* swap in new ilevel */
166	jnz,pn	1b
1672:
168	popq	%rbx
169	ret
1703:
171	popq	%rbx
172	movl	%ecx, %edi
173	jmp	_C_LABEL(Xspllower)
174
175#endif	/* DIAGNOSTIC */
176
177/*
178 * void	rw_enter(krwlock_t *rwl, krw_t op);
179 *
180 * Acquire one hold on a RW lock.
181 */
182NENTRY(rw_enter)				/* 0x00c0, 62 bytes */
183	cmpl	$RW_READER, %esi
184	jne	2f
185
186	/*
187	 * Reader: this is the most common case.
188	 */
1891:	movq	RW_OWNER(%rdi), %rax
190	testb	$(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al
191	leaq	RW_READ_INCR(%rax), %rdx
192	jnz,pn	_C_LABEL(rw_vector_enter)
193	LOCK(lockpatch2)
194	cmpxchgq %rdx, RW_OWNER(%rdi)
195	jnz,pn	1b
196	ret
197
198	/*
199	 * Writer: if the compare-and-set fails, don't bother retrying.
200	 */
2012:	movq	CPUVAR(CURLWP), %rcx
202	xorq	%rax, %rax
203	orq	$RW_WRITE_LOCKED, %rcx
204	LOCK(lockpatch3)
205	cmpxchgq %rcx, RW_OWNER(%rdi)
206	jnz,pn	_C_LABEL(rw_vector_enter)
207	ret
208
209/*
210 * void	rw_exit(krwlock_t *rwl);
211 *
212 * Release one hold on a RW lock.
213 */
214NENTRY(rw_exit)					/* 0x0100, 64 bytes */
215	movq	RW_OWNER(%rdi), %rax
216	testb	$RW_WRITE_LOCKED, %al
217	jnz	2f
218
219	/*
220	 * Reader
221	 */
2221:	testb	$RW_HAS_WAITERS, %al
223	jnz,pn	3f
224	cmpq	$RW_READ_INCR, %rax
225	leaq	-RW_READ_INCR(%rax), %rdx
226	jb,pn	3f
227	LOCK(lockpatch4)
228	cmpxchgq %rdx, RW_OWNER(%rdi)
229	jnz,pn	1b
230	ret
231
232	/*
233	 * Writer
234	 */
2352:	leaq	-RW_WRITE_LOCKED(%rax), %rdx
236	subq	CPUVAR(CURLWP), %rdx
237	jnz,pn	3f
238	LOCK(lockpatch5)
239	cmpxchgq %rdx, RW_OWNER(%rdi)
240	jnz	3f
241	ret
242
2433:	jmp	_C_LABEL(rw_vector_exit)
244
245#endif	/* LOCKDEBUG */
246
247/*
248 * int _lock_cas(uintptr_t *val, uintptr_t old, uintptr_t new);
249 *
250 * Perform an atomic compare-and-set operation.
251 */
252NENTRY(_lock_cas)				/* 0x0140, 19 bytes */
253	movq	%rsi, %rax
254	LOCK(lockpatch6)
255	cmpxchgq %rdx, (%rdi)
256	movq	$0, %rax
257	setz	%al				/* = 1 if success */
258	ret
259
260/*
261 * Memory barrier operations, may be patched at runtime.
262 */
263	.align	8
264ENTRY(mb_read)
265	LOCK(lockpatch7)
266	addq	$0, 0(%rsp)
267	ret
268END(mb_read_end, 8)
269
270ENTRY(mb_write)
271	nop
272	ret
273END(mb_write_end, 8)
274
275ENTRY(mb_memory)
276	LOCK(lockpatch8)
277	addq	$0, 0(%rsp)
278	ret
279END(mb_memory_end, 8)
280
281ENTRY(sse2_mb_read)
282	lfence
283	ret
284END(sse2_mb_read_end, 8)
285
286ENTRY(sse2_mb_memory)
287	mfence
288	ret
289END(sse2_mb_memory_end, 8)
290
291/*
292 * Make sure code after the ret is properly encoded with nopness
293 * by gas, or could stall newer processors.
294 */
295
296ENTRY(x86_mb_nop)
297	nop
298	ret
299END(x86_mb_nop_end, 8)
300
301/*
302 * XXX Don't belong here.
303 */
304ENTRY(atomic_inc_uint)
305	LOCK(lockpatch9)
306	incl	(%rdi)
307	ret
308
309ENTRY(atomic_dec_uint_nv)
310	movl	$-1, %eax
311	LOCK(lockpatch10)
312	xaddl	%eax, (%rdi)
313	decl	%eax
314	ret
315
316/*
317 * Patchpoints to replace with NOP when ncpu == 1.
318 */
319#ifndef LOCKDEBUG
320LABEL(x86_lockpatch)
321	.quad	lockpatch1, lockpatch2, lockpatch3, lockpatch4
322	.quad	lockpatch5, lockpatch6, lockpatch7, lockpatch8
323	.quad	lockpatch9, lockpatch10
324#ifdef FULL
325	.quad	lockpatch11
326#endif
327	.quad	0
328#endif
329