sync.md revision 1.3 1 1.1 mrg ;; GCC machine description for i386 synchronization instructions.
2 1.3 mrg ;; Copyright (C) 2005-2013 Free Software Foundation, Inc.
3 1.1 mrg ;;
4 1.1 mrg ;; This file is part of GCC.
5 1.1 mrg ;;
6 1.1 mrg ;; GCC is free software; you can redistribute it and/or modify
7 1.1 mrg ;; it under the terms of the GNU General Public License as published by
8 1.1 mrg ;; the Free Software Foundation; either version 3, or (at your option)
9 1.1 mrg ;; any later version.
10 1.1 mrg ;;
11 1.1 mrg ;; GCC is distributed in the hope that it will be useful,
12 1.1 mrg ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
13 1.1 mrg ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 1.1 mrg ;; GNU General Public License for more details.
15 1.1 mrg ;;
16 1.1 mrg ;; You should have received a copy of the GNU General Public License
17 1.1 mrg ;; along with GCC; see the file COPYING3. If not see
18 1.1 mrg ;; <http://www.gnu.org/licenses/>.
19 1.1 mrg
20 1.3 mrg (define_c_enum "unspec" [
21 1.3 mrg UNSPEC_LFENCE
22 1.3 mrg UNSPEC_SFENCE
23 1.3 mrg UNSPEC_MFENCE
24 1.3 mrg UNSPEC_MOVA ; For __atomic support
25 1.3 mrg UNSPEC_LDA
26 1.3 mrg UNSPEC_STA
27 1.3 mrg ])
28 1.3 mrg
29 1.3 mrg (define_c_enum "unspecv" [
30 1.3 mrg UNSPECV_CMPXCHG
31 1.3 mrg UNSPECV_XCHG
32 1.3 mrg UNSPECV_LOCK
33 1.3 mrg ])
34 1.3 mrg
35 1.3 mrg (define_expand "sse2_lfence"
36 1.3 mrg [(set (match_dup 0)
37 1.3 mrg (unspec:BLK [(match_dup 0)] UNSPEC_LFENCE))]
38 1.3 mrg "TARGET_SSE2"
39 1.3 mrg {
40 1.3 mrg operands[0] = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (Pmode));
41 1.3 mrg MEM_VOLATILE_P (operands[0]) = 1;
42 1.3 mrg })
43 1.3 mrg
44 1.3 mrg (define_insn "*sse2_lfence"
45 1.3 mrg [(set (match_operand:BLK 0)
46 1.3 mrg (unspec:BLK [(match_dup 0)] UNSPEC_LFENCE))]
47 1.3 mrg "TARGET_SSE2"
48 1.3 mrg "lfence"
49 1.3 mrg [(set_attr "type" "sse")
50 1.3 mrg (set_attr "length_address" "0")
51 1.3 mrg (set_attr "atom_sse_attr" "lfence")
52 1.3 mrg (set_attr "memory" "unknown")])
53 1.3 mrg
54 1.3 mrg (define_expand "sse_sfence"
55 1.3 mrg [(set (match_dup 0)
56 1.3 mrg (unspec:BLK [(match_dup 0)] UNSPEC_SFENCE))]
57 1.3 mrg "TARGET_SSE || TARGET_3DNOW_A"
58 1.3 mrg {
59 1.3 mrg operands[0] = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (Pmode));
60 1.3 mrg MEM_VOLATILE_P (operands[0]) = 1;
61 1.3 mrg })
62 1.3 mrg
63 1.3 mrg (define_insn "*sse_sfence"
64 1.3 mrg [(set (match_operand:BLK 0)
65 1.3 mrg (unspec:BLK [(match_dup 0)] UNSPEC_SFENCE))]
66 1.3 mrg "TARGET_SSE || TARGET_3DNOW_A"
67 1.3 mrg "sfence"
68 1.3 mrg [(set_attr "type" "sse")
69 1.3 mrg (set_attr "length_address" "0")
70 1.3 mrg (set_attr "atom_sse_attr" "fence")
71 1.3 mrg (set_attr "memory" "unknown")])
72 1.1 mrg
73 1.3 mrg (define_expand "sse2_mfence"
74 1.1 mrg [(set (match_dup 0)
75 1.1 mrg (unspec:BLK [(match_dup 0)] UNSPEC_MFENCE))]
76 1.3 mrg "TARGET_SSE2"
77 1.1 mrg {
78 1.1 mrg operands[0] = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (Pmode));
79 1.1 mrg MEM_VOLATILE_P (operands[0]) = 1;
80 1.3 mrg })
81 1.1 mrg
82 1.3 mrg (define_insn "mfence_sse2"
83 1.3 mrg [(set (match_operand:BLK 0)
84 1.3 mrg (unspec:BLK [(match_dup 0)] UNSPEC_MFENCE))]
85 1.3 mrg "TARGET_64BIT || TARGET_SSE2"
86 1.3 mrg "mfence"
87 1.3 mrg [(set_attr "type" "sse")
88 1.3 mrg (set_attr "length_address" "0")
89 1.3 mrg (set_attr "atom_sse_attr" "fence")
90 1.3 mrg (set_attr "memory" "unknown")])
91 1.1 mrg
92 1.3 mrg (define_insn "mfence_nosse"
93 1.3 mrg [(set (match_operand:BLK 0)
94 1.1 mrg (unspec:BLK [(match_dup 0)] UNSPEC_MFENCE))
95 1.1 mrg (clobber (reg:CC FLAGS_REG))]
96 1.1 mrg "!(TARGET_64BIT || TARGET_SSE2)"
97 1.1 mrg "lock{%;} or{l}\t{$0, (%%esp)|DWORD PTR [esp], 0}"
98 1.1 mrg [(set_attr "memory" "unknown")])
99 1.1 mrg
100 1.3 mrg (define_expand "mem_thread_fence"
101 1.3 mrg [(match_operand:SI 0 "const_int_operand")] ;; model
102 1.3 mrg ""
103 1.3 mrg {
104 1.3 mrg enum memmodel model = (enum memmodel) (INTVAL (operands[0]) & MEMMODEL_MASK);
105 1.3 mrg
106 1.3 mrg /* Unless this is a SEQ_CST fence, the i386 memory model is strong
107 1.3 mrg enough not to require barriers of any kind. */
108 1.3 mrg if (model == MEMMODEL_SEQ_CST)
109 1.3 mrg {
110 1.3 mrg rtx (*mfence_insn)(rtx);
111 1.3 mrg rtx mem;
112 1.3 mrg
113 1.3 mrg if (TARGET_64BIT || TARGET_SSE2)
114 1.3 mrg mfence_insn = gen_mfence_sse2;
115 1.3 mrg else
116 1.3 mrg mfence_insn = gen_mfence_nosse;
117 1.3 mrg
118 1.3 mrg mem = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (Pmode));
119 1.3 mrg MEM_VOLATILE_P (mem) = 1;
120 1.3 mrg
121 1.3 mrg emit_insn (mfence_insn (mem));
122 1.3 mrg }
123 1.3 mrg DONE;
124 1.3 mrg })
125 1.3 mrg
126 1.3 mrg ;; ??? From volume 3 section 8.1.1 Guaranteed Atomic Operations,
127 1.3 mrg ;; Only beginning at Pentium family processors do we get any guarantee of
128 1.3 mrg ;; atomicity in aligned 64-bit quantities. Beginning at P6, we get a
129 1.3 mrg ;; guarantee for 64-bit accesses that do not cross a cacheline boundary.
130 1.3 mrg ;;
131 1.3 mrg ;; Note that the TARGET_CMPXCHG8B test below is a stand-in for "Pentium".
132 1.3 mrg ;;
133 1.3 mrg ;; Importantly, *no* processor makes atomicity guarantees for larger
134 1.3 mrg ;; accesses. In particular, there's no way to perform an atomic TImode
135 1.3 mrg ;; move, despite the apparent applicability of MOVDQA et al.
136 1.3 mrg
137 1.3 mrg (define_mode_iterator ATOMIC
138 1.3 mrg [QI HI SI
139 1.3 mrg (DI "TARGET_64BIT || (TARGET_CMPXCHG8B && (TARGET_80387 || TARGET_SSE))")
140 1.3 mrg ])
141 1.3 mrg
142 1.3 mrg (define_expand "atomic_load<mode>"
143 1.3 mrg [(set (match_operand:ATOMIC 0 "register_operand")
144 1.3 mrg (unspec:ATOMIC [(match_operand:ATOMIC 1 "memory_operand")
145 1.3 mrg (match_operand:SI 2 "const_int_operand")]
146 1.3 mrg UNSPEC_MOVA))]
147 1.3 mrg ""
148 1.3 mrg {
149 1.3 mrg /* For DImode on 32-bit, we can use the FPU to perform the load. */
150 1.3 mrg if (<MODE>mode == DImode && !TARGET_64BIT)
151 1.3 mrg emit_insn (gen_atomic_loaddi_fpu
152 1.3 mrg (operands[0], operands[1],
153 1.3 mrg assign_386_stack_local (DImode, SLOT_TEMP)));
154 1.3 mrg else
155 1.3 mrg emit_move_insn (operands[0], operands[1]);
156 1.3 mrg DONE;
157 1.3 mrg })
158 1.3 mrg
159 1.3 mrg (define_insn_and_split "atomic_loaddi_fpu"
160 1.3 mrg [(set (match_operand:DI 0 "nonimmediate_operand" "=x,m,?r")
161 1.3 mrg (unspec:DI [(match_operand:DI 1 "memory_operand" "m,m,m")]
162 1.3 mrg UNSPEC_MOVA))
163 1.3 mrg (clobber (match_operand:DI 2 "memory_operand" "=X,X,m"))
164 1.3 mrg (clobber (match_scratch:DF 3 "=X,xf,xf"))]
165 1.3 mrg "!TARGET_64BIT && (TARGET_80387 || TARGET_SSE)"
166 1.3 mrg "#"
167 1.3 mrg "&& reload_completed"
168 1.3 mrg [(const_int 0)]
169 1.3 mrg {
170 1.3 mrg rtx dst = operands[0], src = operands[1];
171 1.3 mrg rtx mem = operands[2], tmp = operands[3];
172 1.3 mrg
173 1.3 mrg if (SSE_REG_P (dst))
174 1.3 mrg emit_move_insn (dst, src);
175 1.3 mrg else
176 1.3 mrg {
177 1.3 mrg if (MEM_P (dst))
178 1.3 mrg mem = dst;
179 1.3 mrg
180 1.3 mrg if (STACK_REG_P (tmp))
181 1.3 mrg {
182 1.3 mrg emit_insn (gen_loaddi_via_fpu (tmp, src));
183 1.3 mrg emit_insn (gen_storedi_via_fpu (mem, tmp));
184 1.3 mrg }
185 1.3 mrg else
186 1.3 mrg {
187 1.3 mrg adjust_reg_mode (tmp, DImode);
188 1.3 mrg emit_move_insn (tmp, src);
189 1.3 mrg emit_move_insn (mem, tmp);
190 1.3 mrg }
191 1.3 mrg
192 1.3 mrg if (mem != dst)
193 1.3 mrg emit_move_insn (dst, mem);
194 1.3 mrg }
195 1.3 mrg DONE;
196 1.3 mrg })
197 1.3 mrg
198 1.3 mrg (define_expand "atomic_store<mode>"
199 1.3 mrg [(set (match_operand:ATOMIC 0 "memory_operand")
200 1.3 mrg (unspec:ATOMIC [(match_operand:ATOMIC 1 "register_operand")
201 1.3 mrg (match_operand:SI 2 "const_int_operand")]
202 1.3 mrg UNSPEC_MOVA))]
203 1.3 mrg ""
204 1.3 mrg {
205 1.3 mrg enum memmodel model = (enum memmodel) (INTVAL (operands[2]) & MEMMODEL_MASK);
206 1.3 mrg
207 1.3 mrg if (<MODE>mode == DImode && !TARGET_64BIT)
208 1.3 mrg {
209 1.3 mrg /* For DImode on 32-bit, we can use the FPU to perform the store. */
210 1.3 mrg /* Note that while we could perform a cmpxchg8b loop, that turns
211 1.3 mrg out to be significantly larger than this plus a barrier. */
212 1.3 mrg emit_insn (gen_atomic_storedi_fpu
213 1.3 mrg (operands[0], operands[1],
214 1.3 mrg assign_386_stack_local (DImode, SLOT_TEMP)));
215 1.3 mrg }
216 1.3 mrg else
217 1.3 mrg {
218 1.3 mrg /* For seq-cst stores, when we lack MFENCE, use XCHG. */
219 1.3 mrg if (model == MEMMODEL_SEQ_CST && !(TARGET_64BIT || TARGET_SSE2))
220 1.3 mrg {
221 1.3 mrg emit_insn (gen_atomic_exchange<mode> (gen_reg_rtx (<MODE>mode),
222 1.3 mrg operands[0], operands[1],
223 1.3 mrg operands[2]));
224 1.3 mrg DONE;
225 1.3 mrg }
226 1.3 mrg
227 1.3 mrg /* Otherwise use a store. */
228 1.3 mrg emit_insn (gen_atomic_store<mode>_1 (operands[0], operands[1],
229 1.3 mrg operands[2]));
230 1.3 mrg }
231 1.3 mrg /* ... followed by an MFENCE, if required. */
232 1.3 mrg if (model == MEMMODEL_SEQ_CST)
233 1.3 mrg emit_insn (gen_mem_thread_fence (operands[2]));
234 1.3 mrg DONE;
235 1.3 mrg })
236 1.3 mrg
237 1.3 mrg (define_insn "atomic_store<mode>_1"
238 1.3 mrg [(set (match_operand:SWI 0 "memory_operand" "=m")
239 1.3 mrg (unspec:SWI [(match_operand:SWI 1 "<nonmemory_operand>" "<r><i>")
240 1.3 mrg (match_operand:SI 2 "const_int_operand")]
241 1.3 mrg UNSPEC_MOVA))]
242 1.3 mrg ""
243 1.3 mrg "%K2mov{<imodesuffix>}\t{%1, %0|%0, %1}")
244 1.3 mrg
245 1.3 mrg (define_insn_and_split "atomic_storedi_fpu"
246 1.3 mrg [(set (match_operand:DI 0 "memory_operand" "=m,m,m")
247 1.3 mrg (unspec:DI [(match_operand:DI 1 "register_operand" "x,m,?r")]
248 1.3 mrg UNSPEC_MOVA))
249 1.3 mrg (clobber (match_operand:DI 2 "memory_operand" "=X,X,m"))
250 1.3 mrg (clobber (match_scratch:DF 3 "=X,xf,xf"))]
251 1.3 mrg "!TARGET_64BIT && (TARGET_80387 || TARGET_SSE)"
252 1.3 mrg "#"
253 1.3 mrg "&& reload_completed"
254 1.3 mrg [(const_int 0)]
255 1.3 mrg {
256 1.3 mrg rtx dst = operands[0], src = operands[1];
257 1.3 mrg rtx mem = operands[2], tmp = operands[3];
258 1.3 mrg
259 1.3 mrg if (!SSE_REG_P (src))
260 1.3 mrg {
261 1.3 mrg if (REG_P (src))
262 1.3 mrg {
263 1.3 mrg emit_move_insn (mem, src);
264 1.3 mrg src = mem;
265 1.3 mrg }
266 1.3 mrg
267 1.3 mrg if (STACK_REG_P (tmp))
268 1.3 mrg {
269 1.3 mrg emit_insn (gen_loaddi_via_fpu (tmp, src));
270 1.3 mrg emit_insn (gen_storedi_via_fpu (dst, tmp));
271 1.3 mrg DONE;
272 1.3 mrg }
273 1.3 mrg else
274 1.3 mrg {
275 1.3 mrg adjust_reg_mode (tmp, DImode);
276 1.3 mrg emit_move_insn (tmp, mem);
277 1.3 mrg src = tmp;
278 1.3 mrg }
279 1.3 mrg }
280 1.3 mrg emit_move_insn (dst, src);
281 1.3 mrg DONE;
282 1.3 mrg })
283 1.3 mrg
284 1.3 mrg ;; ??? You'd think that we'd be able to perform this via FLOAT + FIX_TRUNC
285 1.3 mrg ;; operations. But the fix_trunc patterns want way more setup than we want
286 1.3 mrg ;; to provide. Note that the scratch is DFmode instead of XFmode in order
287 1.3 mrg ;; to make it easy to allocate a scratch in either SSE or FP_REGs above.
288 1.3 mrg
289 1.3 mrg (define_insn "loaddi_via_fpu"
290 1.3 mrg [(set (match_operand:DF 0 "register_operand" "=f")
291 1.3 mrg (unspec:DF [(match_operand:DI 1 "memory_operand" "m")] UNSPEC_LDA))]
292 1.3 mrg "TARGET_80387"
293 1.3 mrg "fild%Z1\t%1"
294 1.3 mrg [(set_attr "type" "fmov")
295 1.3 mrg (set_attr "mode" "DF")
296 1.3 mrg (set_attr "fp_int_src" "true")])
297 1.3 mrg
298 1.3 mrg (define_insn "storedi_via_fpu"
299 1.3 mrg [(set (match_operand:DI 0 "memory_operand" "=m")
300 1.3 mrg (unspec:DI [(match_operand:DF 1 "register_operand" "f")] UNSPEC_STA))]
301 1.3 mrg "TARGET_80387"
302 1.3 mrg {
303 1.3 mrg gcc_assert (find_regno_note (insn, REG_DEAD, FIRST_STACK_REG) != NULL_RTX);
304 1.3 mrg
305 1.3 mrg return "fistp%Z0\t%0";
306 1.3 mrg }
307 1.3 mrg [(set_attr "type" "fmov")
308 1.3 mrg (set_attr "mode" "DI")])
309 1.3 mrg
310 1.3 mrg (define_expand "atomic_compare_and_swap<mode>"
311 1.3 mrg [(match_operand:QI 0 "register_operand") ;; bool success output
312 1.3 mrg (match_operand:SWI124 1 "register_operand") ;; oldval output
313 1.3 mrg (match_operand:SWI124 2 "memory_operand") ;; memory
314 1.3 mrg (match_operand:SWI124 3 "register_operand") ;; expected input
315 1.3 mrg (match_operand:SWI124 4 "register_operand") ;; newval input
316 1.3 mrg (match_operand:SI 5 "const_int_operand") ;; is_weak
317 1.3 mrg (match_operand:SI 6 "const_int_operand") ;; success model
318 1.3 mrg (match_operand:SI 7 "const_int_operand")] ;; failure model
319 1.3 mrg "TARGET_CMPXCHG"
320 1.3 mrg {
321 1.3 mrg emit_insn
322 1.3 mrg (gen_atomic_compare_and_swap<mode>_1
323 1.3 mrg (operands[1], operands[2], operands[3], operands[4], operands[6]));
324 1.3 mrg ix86_expand_setcc (operands[0], EQ, gen_rtx_REG (CCZmode, FLAGS_REG),
325 1.3 mrg const0_rtx);
326 1.3 mrg DONE;
327 1.3 mrg })
328 1.3 mrg
329 1.3 mrg (define_mode_iterator CASMODE
330 1.3 mrg [(DI "TARGET_64BIT || TARGET_CMPXCHG8B")
331 1.3 mrg (TI "TARGET_64BIT && TARGET_CMPXCHG16B")])
332 1.3 mrg (define_mode_attr CASHMODE [(DI "SI") (TI "DI")])
333 1.3 mrg
334 1.3 mrg (define_expand "atomic_compare_and_swap<mode>"
335 1.3 mrg [(match_operand:QI 0 "register_operand") ;; bool success output
336 1.3 mrg (match_operand:CASMODE 1 "register_operand") ;; oldval output
337 1.3 mrg (match_operand:CASMODE 2 "memory_operand") ;; memory
338 1.3 mrg (match_operand:CASMODE 3 "register_operand") ;; expected input
339 1.3 mrg (match_operand:CASMODE 4 "register_operand") ;; newval input
340 1.3 mrg (match_operand:SI 5 "const_int_operand") ;; is_weak
341 1.3 mrg (match_operand:SI 6 "const_int_operand") ;; success model
342 1.3 mrg (match_operand:SI 7 "const_int_operand")] ;; failure model
343 1.1 mrg "TARGET_CMPXCHG"
344 1.1 mrg {
345 1.3 mrg if (<MODE>mode == DImode && TARGET_64BIT)
346 1.3 mrg {
347 1.3 mrg emit_insn
348 1.3 mrg (gen_atomic_compare_and_swapdi_1
349 1.3 mrg (operands[1], operands[2], operands[3], operands[4], operands[6]));
350 1.3 mrg }
351 1.3 mrg else
352 1.1 mrg {
353 1.3 mrg enum machine_mode hmode = <CASHMODE>mode;
354 1.3 mrg rtx lo_o, lo_e, lo_n, hi_o, hi_e, hi_n, mem;
355 1.3 mrg
356 1.3 mrg lo_o = operands[1];
357 1.3 mrg mem = operands[2];
358 1.3 mrg lo_e = operands[3];
359 1.3 mrg lo_n = operands[4];
360 1.3 mrg hi_o = gen_highpart (hmode, lo_o);
361 1.3 mrg hi_e = gen_highpart (hmode, lo_e);
362 1.3 mrg hi_n = gen_highpart (hmode, lo_n);
363 1.3 mrg lo_o = gen_lowpart (hmode, lo_o);
364 1.3 mrg lo_e = gen_lowpart (hmode, lo_e);
365 1.3 mrg lo_n = gen_lowpart (hmode, lo_n);
366 1.3 mrg
367 1.3 mrg if (!cmpxchg8b_pic_memory_operand (mem, <MODE>mode))
368 1.3 mrg mem = replace_equiv_address (mem, force_reg (Pmode, XEXP (mem, 0)));
369 1.3 mrg
370 1.3 mrg emit_insn
371 1.3 mrg (gen_atomic_compare_and_swap<mode>_doubleword
372 1.3 mrg (lo_o, hi_o, mem, lo_e, hi_e, lo_n, hi_n, operands[6]));
373 1.1 mrg }
374 1.3 mrg
375 1.3 mrg ix86_expand_setcc (operands[0], EQ, gen_rtx_REG (CCZmode, FLAGS_REG),
376 1.3 mrg const0_rtx);
377 1.3 mrg DONE;
378 1.1 mrg })
379 1.1 mrg
380 1.3 mrg (define_insn "atomic_compare_and_swap<mode>_1"
381 1.1 mrg [(set (match_operand:SWI 0 "register_operand" "=a")
382 1.1 mrg (unspec_volatile:SWI
383 1.3 mrg [(match_operand:SWI 1 "memory_operand" "+m")
384 1.3 mrg (match_operand:SWI 2 "register_operand" "0")
385 1.3 mrg (match_operand:SWI 3 "register_operand" "<r>")
386 1.3 mrg (match_operand:SI 4 "const_int_operand")]
387 1.1 mrg UNSPECV_CMPXCHG))
388 1.3 mrg (set (match_dup 1)
389 1.3 mrg (unspec_volatile:SWI [(const_int 0)] UNSPECV_CMPXCHG))
390 1.1 mrg (set (reg:CCZ FLAGS_REG)
391 1.3 mrg (unspec_volatile:CCZ [(const_int 0)] UNSPECV_CMPXCHG))]
392 1.1 mrg "TARGET_CMPXCHG"
393 1.3 mrg "lock{%;} %K4cmpxchg{<imodesuffix>}\t{%3, %1|%1, %3}")
394 1.1 mrg
395 1.3 mrg ;; For double-word compare and swap, we are obliged to play tricks with
396 1.3 mrg ;; the input newval (op5:op6) because the Intel register numbering does
397 1.3 mrg ;; not match the gcc register numbering, so the pair must be CX:BX.
398 1.3 mrg ;; That said, in order to take advantage of possible lower-subreg opts,
399 1.3 mrg ;; treat all of the integral operands in the same way.
400 1.3 mrg
401 1.3 mrg ;; Operands 5 and 6 really need to be different registers, which in
402 1.3 mrg ;; this case means op5 must not be ecx. If op5 and op6 are the same
403 1.3 mrg ;; (like when the input is -1LL) GCC might chose to allocate op5 to ecx,
404 1.3 mrg ;; like op6. This breaks, as the xchg will move the PIC register
405 1.3 mrg ;; contents to %ecx then --> boom.
406 1.3 mrg
407 1.3 mrg (define_mode_attr doublemodesuffix [(SI "8") (DI "16")])
408 1.3 mrg (define_mode_attr regprefix [(SI "e") (DI "r")])
409 1.3 mrg
410 1.3 mrg (define_insn "atomic_compare_and_swap<dwi>_doubleword"
411 1.3 mrg [(set (match_operand:DWIH 0 "register_operand" "=a,a")
412 1.3 mrg (unspec_volatile:DWIH
413 1.3 mrg [(match_operand:<DWI> 2 "cmpxchg8b_pic_memory_operand" "+m,m")
414 1.3 mrg (match_operand:DWIH 3 "register_operand" "0,0")
415 1.3 mrg (match_operand:DWIH 4 "register_operand" "1,1")
416 1.3 mrg (match_operand:DWIH 5 "register_operand" "b,!*r")
417 1.3 mrg (match_operand:DWIH 6 "register_operand" "c,c")
418 1.3 mrg (match_operand:SI 7 "const_int_operand")]
419 1.1 mrg UNSPECV_CMPXCHG))
420 1.3 mrg (set (match_operand:DWIH 1 "register_operand" "=d,d")
421 1.3 mrg (unspec_volatile:DWIH [(const_int 0)] UNSPECV_CMPXCHG))
422 1.3 mrg (set (match_dup 2)
423 1.3 mrg (unspec_volatile:<DWI> [(const_int 0)] UNSPECV_CMPXCHG))
424 1.1 mrg (set (reg:CCZ FLAGS_REG)
425 1.3 mrg (unspec_volatile:CCZ [(const_int 0)] UNSPECV_CMPXCHG))
426 1.3 mrg (clobber (match_scratch:DWIH 8 "=X,&5"))]
427 1.3 mrg "TARGET_CMPXCHG<doublemodesuffix>B"
428 1.3 mrg {
429 1.3 mrg bool swap = REGNO (operands[5]) != BX_REG;
430 1.3 mrg const char *xchg = "xchg{<imodesuffix>}\t%%<regprefix>bx, %5";
431 1.3 mrg
432 1.3 mrg if (swap)
433 1.3 mrg output_asm_insn (xchg, operands);
434 1.3 mrg output_asm_insn ("lock{%;} %K7cmpxchg<doublemodesuffix>b\t%2", operands);
435 1.3 mrg if (swap)
436 1.3 mrg output_asm_insn (xchg, operands);
437 1.3 mrg
438 1.3 mrg return "";
439 1.3 mrg })
440 1.1 mrg
441 1.3 mrg ;; For operand 2 nonmemory_operand predicate is used instead of
442 1.3 mrg ;; register_operand to allow combiner to better optimize atomic
443 1.3 mrg ;; additions of constants.
444 1.3 mrg (define_insn "atomic_fetch_add<mode>"
445 1.1 mrg [(set (match_operand:SWI 0 "register_operand" "=<r>")
446 1.1 mrg (unspec_volatile:SWI
447 1.3 mrg [(match_operand:SWI 1 "memory_operand" "+m")
448 1.3 mrg (match_operand:SI 3 "const_int_operand")] ;; model
449 1.3 mrg UNSPECV_XCHG))
450 1.1 mrg (set (match_dup 1)
451 1.1 mrg (plus:SWI (match_dup 1)
452 1.3 mrg (match_operand:SWI 2 "nonmemory_operand" "0")))
453 1.1 mrg (clobber (reg:CC FLAGS_REG))]
454 1.1 mrg "TARGET_XADD"
455 1.3 mrg "lock{%;} %K3xadd{<imodesuffix>}\t{%0, %1|%1, %0}")
456 1.3 mrg
457 1.3 mrg ;; This peephole2 and following insn optimize
458 1.3 mrg ;; __sync_fetch_and_add (x, -N) == N into just lock {add,sub,inc,dec}
459 1.3 mrg ;; followed by testing of flags instead of lock xadd and comparisons.
460 1.3 mrg (define_peephole2
461 1.3 mrg [(set (match_operand:SWI 0 "register_operand")
462 1.3 mrg (match_operand:SWI 2 "const_int_operand"))
463 1.3 mrg (parallel [(set (match_dup 0)
464 1.3 mrg (unspec_volatile:SWI
465 1.3 mrg [(match_operand:SWI 1 "memory_operand")
466 1.3 mrg (match_operand:SI 4 "const_int_operand")]
467 1.3 mrg UNSPECV_XCHG))
468 1.3 mrg (set (match_dup 1)
469 1.3 mrg (plus:SWI (match_dup 1)
470 1.3 mrg (match_dup 0)))
471 1.3 mrg (clobber (reg:CC FLAGS_REG))])
472 1.3 mrg (set (reg:CCZ FLAGS_REG)
473 1.3 mrg (compare:CCZ (match_dup 0)
474 1.3 mrg (match_operand:SWI 3 "const_int_operand")))]
475 1.3 mrg "peep2_reg_dead_p (3, operands[0])
476 1.3 mrg && (unsigned HOST_WIDE_INT) INTVAL (operands[2])
477 1.3 mrg == -(unsigned HOST_WIDE_INT) INTVAL (operands[3])
478 1.3 mrg && !reg_overlap_mentioned_p (operands[0], operands[1])"
479 1.3 mrg [(parallel [(set (reg:CCZ FLAGS_REG)
480 1.3 mrg (compare:CCZ
481 1.3 mrg (unspec_volatile:SWI [(match_dup 1) (match_dup 4)]
482 1.3 mrg UNSPECV_XCHG)
483 1.3 mrg (match_dup 3)))
484 1.3 mrg (set (match_dup 1)
485 1.3 mrg (plus:SWI (match_dup 1)
486 1.3 mrg (match_dup 2)))])])
487 1.3 mrg
488 1.3 mrg (define_insn "*atomic_fetch_add_cmp<mode>"
489 1.3 mrg [(set (reg:CCZ FLAGS_REG)
490 1.3 mrg (compare:CCZ
491 1.3 mrg (unspec_volatile:SWI
492 1.3 mrg [(match_operand:SWI 0 "memory_operand" "+m")
493 1.3 mrg (match_operand:SI 3 "const_int_operand")] ;; model
494 1.3 mrg UNSPECV_XCHG)
495 1.3 mrg (match_operand:SWI 2 "const_int_operand" "i")))
496 1.3 mrg (set (match_dup 0)
497 1.3 mrg (plus:SWI (match_dup 0)
498 1.3 mrg (match_operand:SWI 1 "const_int_operand" "i")))]
499 1.3 mrg "(unsigned HOST_WIDE_INT) INTVAL (operands[1])
500 1.3 mrg == -(unsigned HOST_WIDE_INT) INTVAL (operands[2])"
501 1.3 mrg {
502 1.3 mrg if (incdec_operand (operands[1], <MODE>mode))
503 1.3 mrg {
504 1.3 mrg if (operands[1] == const1_rtx)
505 1.3 mrg return "lock{%;} %K3inc{<imodesuffix>}\t%0";
506 1.3 mrg else
507 1.3 mrg {
508 1.3 mrg gcc_assert (operands[1] == constm1_rtx);
509 1.3 mrg return "lock{%;} %K3dec{<imodesuffix>}\t%0";
510 1.3 mrg }
511 1.3 mrg }
512 1.3 mrg
513 1.3 mrg if (x86_maybe_negate_const_int (&operands[1], <MODE>mode))
514 1.3 mrg return "lock{%;} %K3sub{<imodesuffix>}\t{%1, %0|%0, %1}";
515 1.3 mrg
516 1.3 mrg return "lock{%;} %K3add{<imodesuffix>}\t{%1, %0|%0, %1}";
517 1.3 mrg })
518 1.1 mrg
519 1.1 mrg ;; Recall that xchg implicitly sets LOCK#, so adding it again wastes space.
520 1.3 mrg ;; In addition, it is always a full barrier, so we can ignore the memory model.
521 1.3 mrg (define_insn "atomic_exchange<mode>"
522 1.3 mrg [(set (match_operand:SWI 0 "register_operand" "=<r>") ;; output
523 1.1 mrg (unspec_volatile:SWI
524 1.3 mrg [(match_operand:SWI 1 "memory_operand" "+m") ;; memory
525 1.3 mrg (match_operand:SI 3 "const_int_operand")] ;; model
526 1.3 mrg UNSPECV_XCHG))
527 1.1 mrg (set (match_dup 1)
528 1.3 mrg (match_operand:SWI 2 "register_operand" "0"))] ;; input
529 1.1 mrg ""
530 1.3 mrg "%K3xchg{<imodesuffix>}\t{%1, %0|%0, %1}")
531 1.1 mrg
532 1.3 mrg (define_insn "atomic_add<mode>"
533 1.1 mrg [(set (match_operand:SWI 0 "memory_operand" "+m")
534 1.1 mrg (unspec_volatile:SWI
535 1.1 mrg [(plus:SWI (match_dup 0)
536 1.3 mrg (match_operand:SWI 1 "nonmemory_operand" "<r><i>"))
537 1.3 mrg (match_operand:SI 2 "const_int_operand")] ;; model
538 1.1 mrg UNSPECV_LOCK))
539 1.1 mrg (clobber (reg:CC FLAGS_REG))]
540 1.1 mrg ""
541 1.1 mrg {
542 1.3 mrg if (incdec_operand (operands[1], <MODE>mode))
543 1.1 mrg {
544 1.1 mrg if (operands[1] == const1_rtx)
545 1.3 mrg return "lock{%;} %K2inc{<imodesuffix>}\t%0";
546 1.3 mrg else
547 1.3 mrg {
548 1.3 mrg gcc_assert (operands[1] == constm1_rtx);
549 1.3 mrg return "lock{%;} %K2dec{<imodesuffix>}\t%0";
550 1.3 mrg }
551 1.1 mrg }
552 1.1 mrg
553 1.3 mrg if (x86_maybe_negate_const_int (&operands[1], <MODE>mode))
554 1.3 mrg return "lock{%;} %K2sub{<imodesuffix>}\t{%1, %0|%0, %1}";
555 1.3 mrg
556 1.3 mrg return "lock{%;} %K2add{<imodesuffix>}\t{%1, %0|%0, %1}";
557 1.1 mrg })
558 1.1 mrg
559 1.3 mrg (define_insn "atomic_sub<mode>"
560 1.1 mrg [(set (match_operand:SWI 0 "memory_operand" "+m")
561 1.1 mrg (unspec_volatile:SWI
562 1.1 mrg [(minus:SWI (match_dup 0)
563 1.3 mrg (match_operand:SWI 1 "nonmemory_operand" "<r><i>"))
564 1.3 mrg (match_operand:SI 2 "const_int_operand")] ;; model
565 1.1 mrg UNSPECV_LOCK))
566 1.1 mrg (clobber (reg:CC FLAGS_REG))]
567 1.1 mrg ""
568 1.1 mrg {
569 1.3 mrg if (incdec_operand (operands[1], <MODE>mode))
570 1.1 mrg {
571 1.1 mrg if (operands[1] == const1_rtx)
572 1.3 mrg return "lock{%;} %K2dec{<imodesuffix>}\t%0";
573 1.3 mrg else
574 1.3 mrg {
575 1.3 mrg gcc_assert (operands[1] == constm1_rtx);
576 1.3 mrg return "lock{%;} %K2inc{<imodesuffix>}\t%0";
577 1.3 mrg }
578 1.1 mrg }
579 1.1 mrg
580 1.3 mrg if (x86_maybe_negate_const_int (&operands[1], <MODE>mode))
581 1.3 mrg return "lock{%;} %K2add{<imodesuffix>}\t{%1, %0|%0, %1}";
582 1.3 mrg
583 1.3 mrg return "lock{%;} %K2sub{<imodesuffix>}\t{%1, %0|%0, %1}";
584 1.1 mrg })
585 1.1 mrg
586 1.3 mrg (define_insn "atomic_<logic><mode>"
587 1.1 mrg [(set (match_operand:SWI 0 "memory_operand" "+m")
588 1.1 mrg (unspec_volatile:SWI
589 1.1 mrg [(any_logic:SWI (match_dup 0)
590 1.3 mrg (match_operand:SWI 1 "nonmemory_operand" "<r><i>"))
591 1.3 mrg (match_operand:SI 2 "const_int_operand")] ;; model
592 1.1 mrg UNSPECV_LOCK))
593 1.1 mrg (clobber (reg:CC FLAGS_REG))]
594 1.1 mrg ""
595 1.3 mrg "lock{%;} %K2<logic>{<imodesuffix>}\t{%1, %0|%0, %1}")
596