Home | History | Annotate | Line # | Download | only in i386
sync.md revision 1.3
      1  1.1  mrg ;; GCC machine description for i386 synchronization instructions.
      2  1.3  mrg ;; Copyright (C) 2005-2013 Free Software Foundation, Inc.
      3  1.1  mrg ;;
      4  1.1  mrg ;; This file is part of GCC.
      5  1.1  mrg ;;
      6  1.1  mrg ;; GCC is free software; you can redistribute it and/or modify
      7  1.1  mrg ;; it under the terms of the GNU General Public License as published by
      8  1.1  mrg ;; the Free Software Foundation; either version 3, or (at your option)
      9  1.1  mrg ;; any later version.
     10  1.1  mrg ;;
     11  1.1  mrg ;; GCC is distributed in the hope that it will be useful,
     12  1.1  mrg ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
     13  1.1  mrg ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     14  1.1  mrg ;; GNU General Public License for more details.
     15  1.1  mrg ;;
     16  1.1  mrg ;; You should have received a copy of the GNU General Public License
     17  1.1  mrg ;; along with GCC; see the file COPYING3.  If not see
     18  1.1  mrg ;; <http://www.gnu.org/licenses/>.
     19  1.1  mrg 
     20  1.3  mrg (define_c_enum "unspec" [
     21  1.3  mrg   UNSPEC_LFENCE
     22  1.3  mrg   UNSPEC_SFENCE
     23  1.3  mrg   UNSPEC_MFENCE
     24  1.3  mrg   UNSPEC_MOVA	; For __atomic support
     25  1.3  mrg   UNSPEC_LDA
     26  1.3  mrg   UNSPEC_STA
     27  1.3  mrg ])
     28  1.3  mrg 
     29  1.3  mrg (define_c_enum "unspecv" [
     30  1.3  mrg   UNSPECV_CMPXCHG
     31  1.3  mrg   UNSPECV_XCHG
     32  1.3  mrg   UNSPECV_LOCK
     33  1.3  mrg ])
     34  1.3  mrg 
     35  1.3  mrg (define_expand "sse2_lfence"
     36  1.3  mrg   [(set (match_dup 0)
     37  1.3  mrg 	(unspec:BLK [(match_dup 0)] UNSPEC_LFENCE))]
     38  1.3  mrg   "TARGET_SSE2"
     39  1.3  mrg {
     40  1.3  mrg   operands[0] = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (Pmode));
     41  1.3  mrg   MEM_VOLATILE_P (operands[0]) = 1;
     42  1.3  mrg })
     43  1.3  mrg 
     44  1.3  mrg (define_insn "*sse2_lfence"
     45  1.3  mrg   [(set (match_operand:BLK 0)
     46  1.3  mrg 	(unspec:BLK [(match_dup 0)] UNSPEC_LFENCE))]
     47  1.3  mrg   "TARGET_SSE2"
     48  1.3  mrg   "lfence"
     49  1.3  mrg   [(set_attr "type" "sse")
     50  1.3  mrg    (set_attr "length_address" "0")
     51  1.3  mrg    (set_attr "atom_sse_attr" "lfence")
     52  1.3  mrg    (set_attr "memory" "unknown")])
     53  1.3  mrg 
     54  1.3  mrg (define_expand "sse_sfence"
     55  1.3  mrg   [(set (match_dup 0)
     56  1.3  mrg 	(unspec:BLK [(match_dup 0)] UNSPEC_SFENCE))]
     57  1.3  mrg   "TARGET_SSE || TARGET_3DNOW_A"
     58  1.3  mrg {
     59  1.3  mrg   operands[0] = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (Pmode));
     60  1.3  mrg   MEM_VOLATILE_P (operands[0]) = 1;
     61  1.3  mrg })
     62  1.3  mrg 
     63  1.3  mrg (define_insn "*sse_sfence"
     64  1.3  mrg   [(set (match_operand:BLK 0)
     65  1.3  mrg 	(unspec:BLK [(match_dup 0)] UNSPEC_SFENCE))]
     66  1.3  mrg   "TARGET_SSE || TARGET_3DNOW_A"
     67  1.3  mrg   "sfence"
     68  1.3  mrg   [(set_attr "type" "sse")
     69  1.3  mrg    (set_attr "length_address" "0")
     70  1.3  mrg    (set_attr "atom_sse_attr" "fence")
     71  1.3  mrg    (set_attr "memory" "unknown")])
     72  1.1  mrg 
     73  1.3  mrg (define_expand "sse2_mfence"
     74  1.1  mrg   [(set (match_dup 0)
     75  1.1  mrg 	(unspec:BLK [(match_dup 0)] UNSPEC_MFENCE))]
     76  1.3  mrg   "TARGET_SSE2"
     77  1.1  mrg {
     78  1.1  mrg   operands[0] = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (Pmode));
     79  1.1  mrg   MEM_VOLATILE_P (operands[0]) = 1;
     80  1.3  mrg })
     81  1.1  mrg 
     82  1.3  mrg (define_insn "mfence_sse2"
     83  1.3  mrg   [(set (match_operand:BLK 0)
     84  1.3  mrg 	(unspec:BLK [(match_dup 0)] UNSPEC_MFENCE))]
     85  1.3  mrg   "TARGET_64BIT || TARGET_SSE2"
     86  1.3  mrg   "mfence"
     87  1.3  mrg   [(set_attr "type" "sse")
     88  1.3  mrg    (set_attr "length_address" "0")
     89  1.3  mrg    (set_attr "atom_sse_attr" "fence")
     90  1.3  mrg    (set_attr "memory" "unknown")])
     91  1.1  mrg 
     92  1.3  mrg (define_insn "mfence_nosse"
     93  1.3  mrg   [(set (match_operand:BLK 0)
     94  1.1  mrg 	(unspec:BLK [(match_dup 0)] UNSPEC_MFENCE))
     95  1.1  mrg    (clobber (reg:CC FLAGS_REG))]
     96  1.1  mrg   "!(TARGET_64BIT || TARGET_SSE2)"
     97  1.1  mrg   "lock{%;} or{l}\t{$0, (%%esp)|DWORD PTR [esp], 0}"
     98  1.1  mrg   [(set_attr "memory" "unknown")])
     99  1.1  mrg 
    100  1.3  mrg (define_expand "mem_thread_fence"
    101  1.3  mrg   [(match_operand:SI 0 "const_int_operand")]		;; model
    102  1.3  mrg   ""
    103  1.3  mrg {
    104  1.3  mrg   enum memmodel model = (enum memmodel) (INTVAL (operands[0]) & MEMMODEL_MASK);
    105  1.3  mrg 
    106  1.3  mrg   /* Unless this is a SEQ_CST fence, the i386 memory model is strong
    107  1.3  mrg      enough not to require barriers of any kind.  */
    108  1.3  mrg   if (model == MEMMODEL_SEQ_CST)
    109  1.3  mrg     {
    110  1.3  mrg       rtx (*mfence_insn)(rtx);
    111  1.3  mrg       rtx mem;
    112  1.3  mrg 
    113  1.3  mrg       if (TARGET_64BIT || TARGET_SSE2)
    114  1.3  mrg 	mfence_insn = gen_mfence_sse2;
    115  1.3  mrg       else
    116  1.3  mrg 	mfence_insn = gen_mfence_nosse;
    117  1.3  mrg 
    118  1.3  mrg       mem = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (Pmode));
    119  1.3  mrg       MEM_VOLATILE_P (mem) = 1;
    120  1.3  mrg 
    121  1.3  mrg       emit_insn (mfence_insn (mem));
    122  1.3  mrg     }
    123  1.3  mrg   DONE;
    124  1.3  mrg })
    125  1.3  mrg 
    126  1.3  mrg ;; ??? From volume 3 section 8.1.1 Guaranteed Atomic Operations,
    127  1.3  mrg ;; Only beginning at Pentium family processors do we get any guarantee of
    128  1.3  mrg ;; atomicity in aligned 64-bit quantities.  Beginning at P6, we get a
    129  1.3  mrg ;; guarantee for 64-bit accesses that do not cross a cacheline boundary.
    130  1.3  mrg ;;
    131  1.3  mrg ;; Note that the TARGET_CMPXCHG8B test below is a stand-in for "Pentium".
    132  1.3  mrg ;;
    133  1.3  mrg ;; Importantly, *no* processor makes atomicity guarantees for larger
    134  1.3  mrg ;; accesses.  In particular, there's no way to perform an atomic TImode
    135  1.3  mrg ;; move, despite the apparent applicability of MOVDQA et al.
    136  1.3  mrg 
    137  1.3  mrg (define_mode_iterator ATOMIC
    138  1.3  mrg    [QI HI SI
    139  1.3  mrg     (DI "TARGET_64BIT || (TARGET_CMPXCHG8B && (TARGET_80387 || TARGET_SSE))")
    140  1.3  mrg    ])
    141  1.3  mrg 
    142  1.3  mrg (define_expand "atomic_load<mode>"
    143  1.3  mrg   [(set (match_operand:ATOMIC 0 "register_operand")
    144  1.3  mrg 	(unspec:ATOMIC [(match_operand:ATOMIC 1 "memory_operand")
    145  1.3  mrg 			(match_operand:SI 2 "const_int_operand")]
    146  1.3  mrg 		       UNSPEC_MOVA))]
    147  1.3  mrg   ""
    148  1.3  mrg {
    149  1.3  mrg   /* For DImode on 32-bit, we can use the FPU to perform the load.  */
    150  1.3  mrg   if (<MODE>mode == DImode && !TARGET_64BIT)
    151  1.3  mrg     emit_insn (gen_atomic_loaddi_fpu
    152  1.3  mrg 	       (operands[0], operands[1],
    153  1.3  mrg 	        assign_386_stack_local (DImode, SLOT_TEMP)));
    154  1.3  mrg   else
    155  1.3  mrg     emit_move_insn (operands[0], operands[1]);
    156  1.3  mrg   DONE;
    157  1.3  mrg })
    158  1.3  mrg 
    159  1.3  mrg (define_insn_and_split "atomic_loaddi_fpu"
    160  1.3  mrg   [(set (match_operand:DI 0 "nonimmediate_operand" "=x,m,?r")
    161  1.3  mrg 	(unspec:DI [(match_operand:DI 1 "memory_operand" "m,m,m")]
    162  1.3  mrg 		   UNSPEC_MOVA))
    163  1.3  mrg    (clobber (match_operand:DI 2 "memory_operand" "=X,X,m"))
    164  1.3  mrg    (clobber (match_scratch:DF 3 "=X,xf,xf"))]
    165  1.3  mrg   "!TARGET_64BIT && (TARGET_80387 || TARGET_SSE)"
    166  1.3  mrg   "#"
    167  1.3  mrg   "&& reload_completed"
    168  1.3  mrg   [(const_int 0)]
    169  1.3  mrg {
    170  1.3  mrg   rtx dst = operands[0], src = operands[1];
    171  1.3  mrg   rtx mem = operands[2], tmp = operands[3];
    172  1.3  mrg 
    173  1.3  mrg   if (SSE_REG_P (dst))
    174  1.3  mrg     emit_move_insn (dst, src);
    175  1.3  mrg   else
    176  1.3  mrg     {
    177  1.3  mrg       if (MEM_P (dst))
    178  1.3  mrg 	mem = dst;
    179  1.3  mrg 
    180  1.3  mrg       if (STACK_REG_P (tmp))
    181  1.3  mrg         {
    182  1.3  mrg 	  emit_insn (gen_loaddi_via_fpu (tmp, src));
    183  1.3  mrg 	  emit_insn (gen_storedi_via_fpu (mem, tmp));
    184  1.3  mrg 	}
    185  1.3  mrg       else
    186  1.3  mrg 	{
    187  1.3  mrg 	  adjust_reg_mode (tmp, DImode);
    188  1.3  mrg 	  emit_move_insn (tmp, src);
    189  1.3  mrg 	  emit_move_insn (mem, tmp);
    190  1.3  mrg 	}
    191  1.3  mrg 
    192  1.3  mrg       if (mem != dst)
    193  1.3  mrg 	emit_move_insn (dst, mem);
    194  1.3  mrg     }
    195  1.3  mrg   DONE;
    196  1.3  mrg })
    197  1.3  mrg 
    198  1.3  mrg (define_expand "atomic_store<mode>"
    199  1.3  mrg   [(set (match_operand:ATOMIC 0 "memory_operand")
    200  1.3  mrg 	(unspec:ATOMIC [(match_operand:ATOMIC 1 "register_operand")
    201  1.3  mrg 			(match_operand:SI 2 "const_int_operand")]
    202  1.3  mrg 		       UNSPEC_MOVA))]
    203  1.3  mrg   ""
    204  1.3  mrg {
    205  1.3  mrg   enum memmodel model = (enum memmodel) (INTVAL (operands[2]) & MEMMODEL_MASK);
    206  1.3  mrg 
    207  1.3  mrg   if (<MODE>mode == DImode && !TARGET_64BIT)
    208  1.3  mrg     {
    209  1.3  mrg       /* For DImode on 32-bit, we can use the FPU to perform the store.  */
    210  1.3  mrg       /* Note that while we could perform a cmpxchg8b loop, that turns
    211  1.3  mrg 	 out to be significantly larger than this plus a barrier.  */
    212  1.3  mrg       emit_insn (gen_atomic_storedi_fpu
    213  1.3  mrg 		 (operands[0], operands[1],
    214  1.3  mrg 	          assign_386_stack_local (DImode, SLOT_TEMP)));
    215  1.3  mrg     }
    216  1.3  mrg   else
    217  1.3  mrg     {
    218  1.3  mrg       /* For seq-cst stores, when we lack MFENCE, use XCHG.  */
    219  1.3  mrg       if (model == MEMMODEL_SEQ_CST && !(TARGET_64BIT || TARGET_SSE2))
    220  1.3  mrg 	{
    221  1.3  mrg 	  emit_insn (gen_atomic_exchange<mode> (gen_reg_rtx (<MODE>mode),
    222  1.3  mrg 						operands[0], operands[1],
    223  1.3  mrg 						operands[2]));
    224  1.3  mrg 	  DONE;
    225  1.3  mrg 	}
    226  1.3  mrg 
    227  1.3  mrg       /* Otherwise use a store.  */
    228  1.3  mrg       emit_insn (gen_atomic_store<mode>_1 (operands[0], operands[1],
    229  1.3  mrg 					   operands[2]));
    230  1.3  mrg     }
    231  1.3  mrg   /* ... followed by an MFENCE, if required.  */
    232  1.3  mrg   if (model == MEMMODEL_SEQ_CST)
    233  1.3  mrg     emit_insn (gen_mem_thread_fence (operands[2]));
    234  1.3  mrg   DONE;
    235  1.3  mrg })
    236  1.3  mrg 
    237  1.3  mrg (define_insn "atomic_store<mode>_1"
    238  1.3  mrg   [(set (match_operand:SWI 0 "memory_operand" "=m")
    239  1.3  mrg 	(unspec:SWI [(match_operand:SWI 1 "<nonmemory_operand>" "<r><i>")
    240  1.3  mrg 		     (match_operand:SI 2 "const_int_operand")]
    241  1.3  mrg 		    UNSPEC_MOVA))]
    242  1.3  mrg   ""
    243  1.3  mrg   "%K2mov{<imodesuffix>}\t{%1, %0|%0, %1}")
    244  1.3  mrg 
    245  1.3  mrg (define_insn_and_split "atomic_storedi_fpu"
    246  1.3  mrg   [(set (match_operand:DI 0 "memory_operand" "=m,m,m")
    247  1.3  mrg 	(unspec:DI [(match_operand:DI 1 "register_operand" "x,m,?r")]
    248  1.3  mrg 		   UNSPEC_MOVA))
    249  1.3  mrg    (clobber (match_operand:DI 2 "memory_operand" "=X,X,m"))
    250  1.3  mrg    (clobber (match_scratch:DF 3 "=X,xf,xf"))]
    251  1.3  mrg   "!TARGET_64BIT && (TARGET_80387 || TARGET_SSE)"
    252  1.3  mrg   "#"
    253  1.3  mrg   "&& reload_completed"
    254  1.3  mrg   [(const_int 0)]
    255  1.3  mrg {
    256  1.3  mrg   rtx dst = operands[0], src = operands[1];
    257  1.3  mrg   rtx mem = operands[2], tmp = operands[3];
    258  1.3  mrg 
    259  1.3  mrg   if (!SSE_REG_P (src))
    260  1.3  mrg     {
    261  1.3  mrg       if (REG_P (src))
    262  1.3  mrg 	{
    263  1.3  mrg 	  emit_move_insn (mem, src);
    264  1.3  mrg 	  src = mem;
    265  1.3  mrg 	}
    266  1.3  mrg 
    267  1.3  mrg       if (STACK_REG_P (tmp))
    268  1.3  mrg 	{
    269  1.3  mrg 	  emit_insn (gen_loaddi_via_fpu (tmp, src));
    270  1.3  mrg 	  emit_insn (gen_storedi_via_fpu (dst, tmp));
    271  1.3  mrg 	  DONE;
    272  1.3  mrg 	}
    273  1.3  mrg       else
    274  1.3  mrg 	{
    275  1.3  mrg 	  adjust_reg_mode (tmp, DImode);
    276  1.3  mrg 	  emit_move_insn (tmp, mem);
    277  1.3  mrg 	  src = tmp;
    278  1.3  mrg 	}
    279  1.3  mrg     }
    280  1.3  mrg   emit_move_insn (dst, src);
    281  1.3  mrg   DONE;
    282  1.3  mrg })
    283  1.3  mrg 
    284  1.3  mrg ;; ??? You'd think that we'd be able to perform this via FLOAT + FIX_TRUNC
    285  1.3  mrg ;; operations.  But the fix_trunc patterns want way more setup than we want
    286  1.3  mrg ;; to provide.  Note that the scratch is DFmode instead of XFmode in order
    287  1.3  mrg ;; to make it easy to allocate a scratch in either SSE or FP_REGs above.
    288  1.3  mrg 
    289  1.3  mrg (define_insn "loaddi_via_fpu"
    290  1.3  mrg   [(set (match_operand:DF 0 "register_operand" "=f")
    291  1.3  mrg 	(unspec:DF [(match_operand:DI 1 "memory_operand" "m")] UNSPEC_LDA))]
    292  1.3  mrg   "TARGET_80387"
    293  1.3  mrg   "fild%Z1\t%1"
    294  1.3  mrg   [(set_attr "type" "fmov")
    295  1.3  mrg    (set_attr "mode" "DF")
    296  1.3  mrg    (set_attr "fp_int_src" "true")])
    297  1.3  mrg 
    298  1.3  mrg (define_insn "storedi_via_fpu"
    299  1.3  mrg   [(set (match_operand:DI 0 "memory_operand" "=m")
    300  1.3  mrg 	(unspec:DI [(match_operand:DF 1 "register_operand" "f")] UNSPEC_STA))]
    301  1.3  mrg   "TARGET_80387"
    302  1.3  mrg {
    303  1.3  mrg   gcc_assert (find_regno_note (insn, REG_DEAD, FIRST_STACK_REG) != NULL_RTX);
    304  1.3  mrg 
    305  1.3  mrg   return "fistp%Z0\t%0";
    306  1.3  mrg }
    307  1.3  mrg   [(set_attr "type" "fmov")
    308  1.3  mrg    (set_attr "mode" "DI")])
    309  1.3  mrg 
    310  1.3  mrg (define_expand "atomic_compare_and_swap<mode>"
    311  1.3  mrg   [(match_operand:QI 0 "register_operand")	;; bool success output
    312  1.3  mrg    (match_operand:SWI124 1 "register_operand")	;; oldval output
    313  1.3  mrg    (match_operand:SWI124 2 "memory_operand")	;; memory
    314  1.3  mrg    (match_operand:SWI124 3 "register_operand")	;; expected input
    315  1.3  mrg    (match_operand:SWI124 4 "register_operand")	;; newval input
    316  1.3  mrg    (match_operand:SI 5 "const_int_operand")	;; is_weak
    317  1.3  mrg    (match_operand:SI 6 "const_int_operand")	;; success model
    318  1.3  mrg    (match_operand:SI 7 "const_int_operand")]	;; failure model
    319  1.3  mrg   "TARGET_CMPXCHG"
    320  1.3  mrg {
    321  1.3  mrg   emit_insn
    322  1.3  mrg    (gen_atomic_compare_and_swap<mode>_1
    323  1.3  mrg     (operands[1], operands[2], operands[3], operands[4], operands[6]));
    324  1.3  mrg   ix86_expand_setcc (operands[0], EQ, gen_rtx_REG (CCZmode, FLAGS_REG),
    325  1.3  mrg 		     const0_rtx);
    326  1.3  mrg   DONE;
    327  1.3  mrg })
    328  1.3  mrg 
    329  1.3  mrg (define_mode_iterator CASMODE
    330  1.3  mrg   [(DI "TARGET_64BIT || TARGET_CMPXCHG8B")
    331  1.3  mrg    (TI "TARGET_64BIT && TARGET_CMPXCHG16B")])
    332  1.3  mrg (define_mode_attr CASHMODE [(DI "SI") (TI "DI")])
    333  1.3  mrg 
    334  1.3  mrg (define_expand "atomic_compare_and_swap<mode>"
    335  1.3  mrg   [(match_operand:QI 0 "register_operand")	;; bool success output
    336  1.3  mrg    (match_operand:CASMODE 1 "register_operand")	;; oldval output
    337  1.3  mrg    (match_operand:CASMODE 2 "memory_operand")	;; memory
    338  1.3  mrg    (match_operand:CASMODE 3 "register_operand")	;; expected input
    339  1.3  mrg    (match_operand:CASMODE 4 "register_operand")	;; newval input
    340  1.3  mrg    (match_operand:SI 5 "const_int_operand")	;; is_weak
    341  1.3  mrg    (match_operand:SI 6 "const_int_operand")	;; success model
    342  1.3  mrg    (match_operand:SI 7 "const_int_operand")]	;; failure model
    343  1.1  mrg   "TARGET_CMPXCHG"
    344  1.1  mrg {
    345  1.3  mrg   if (<MODE>mode == DImode && TARGET_64BIT)
    346  1.3  mrg     {
    347  1.3  mrg       emit_insn
    348  1.3  mrg        (gen_atomic_compare_and_swapdi_1
    349  1.3  mrg 	(operands[1], operands[2], operands[3], operands[4], operands[6]));
    350  1.3  mrg     }
    351  1.3  mrg   else
    352  1.1  mrg     {
    353  1.3  mrg       enum machine_mode hmode = <CASHMODE>mode;
    354  1.3  mrg       rtx lo_o, lo_e, lo_n, hi_o, hi_e, hi_n, mem;
    355  1.3  mrg 
    356  1.3  mrg       lo_o = operands[1];
    357  1.3  mrg       mem  = operands[2];
    358  1.3  mrg       lo_e = operands[3];
    359  1.3  mrg       lo_n = operands[4];
    360  1.3  mrg       hi_o = gen_highpart (hmode, lo_o);
    361  1.3  mrg       hi_e = gen_highpart (hmode, lo_e);
    362  1.3  mrg       hi_n = gen_highpart (hmode, lo_n);
    363  1.3  mrg       lo_o = gen_lowpart (hmode, lo_o);
    364  1.3  mrg       lo_e = gen_lowpart (hmode, lo_e);
    365  1.3  mrg       lo_n = gen_lowpart (hmode, lo_n);
    366  1.3  mrg 
    367  1.3  mrg       if (!cmpxchg8b_pic_memory_operand (mem, <MODE>mode))
    368  1.3  mrg  	mem = replace_equiv_address (mem, force_reg (Pmode, XEXP (mem, 0)));
    369  1.3  mrg 
    370  1.3  mrg       emit_insn
    371  1.3  mrg        (gen_atomic_compare_and_swap<mode>_doubleword
    372  1.3  mrg         (lo_o, hi_o, mem, lo_e, hi_e, lo_n, hi_n, operands[6]));
    373  1.1  mrg     }
    374  1.3  mrg 
    375  1.3  mrg   ix86_expand_setcc (operands[0], EQ, gen_rtx_REG (CCZmode, FLAGS_REG),
    376  1.3  mrg 		     const0_rtx);
    377  1.3  mrg   DONE;
    378  1.1  mrg })
    379  1.1  mrg 
    380  1.3  mrg (define_insn "atomic_compare_and_swap<mode>_1"
    381  1.1  mrg   [(set (match_operand:SWI 0 "register_operand" "=a")
    382  1.1  mrg 	(unspec_volatile:SWI
    383  1.3  mrg 	  [(match_operand:SWI 1 "memory_operand" "+m")
    384  1.3  mrg 	   (match_operand:SWI 2 "register_operand" "0")
    385  1.3  mrg 	   (match_operand:SWI 3 "register_operand" "<r>")
    386  1.3  mrg 	   (match_operand:SI 4 "const_int_operand")]
    387  1.1  mrg 	  UNSPECV_CMPXCHG))
    388  1.3  mrg    (set (match_dup 1)
    389  1.3  mrg 	(unspec_volatile:SWI [(const_int 0)] UNSPECV_CMPXCHG))
    390  1.1  mrg    (set (reg:CCZ FLAGS_REG)
    391  1.3  mrg         (unspec_volatile:CCZ [(const_int 0)] UNSPECV_CMPXCHG))]
    392  1.1  mrg   "TARGET_CMPXCHG"
    393  1.3  mrg   "lock{%;} %K4cmpxchg{<imodesuffix>}\t{%3, %1|%1, %3}")
    394  1.1  mrg 
    395  1.3  mrg ;; For double-word compare and swap, we are obliged to play tricks with
    396  1.3  mrg ;; the input newval (op5:op6) because the Intel register numbering does
    397  1.3  mrg ;; not match the gcc register numbering, so the pair must be CX:BX.
    398  1.3  mrg ;; That said, in order to take advantage of possible lower-subreg opts,
    399  1.3  mrg ;; treat all of the integral operands in the same way.
    400  1.3  mrg 
    401  1.3  mrg ;; Operands 5 and 6 really need to be different registers, which in
    402  1.3  mrg ;; this case means op5 must not be ecx.  If op5 and op6 are the same
    403  1.3  mrg ;; (like when the input is -1LL) GCC might chose to allocate op5 to ecx,
    404  1.3  mrg ;; like op6.  This breaks, as the xchg will move the PIC register
    405  1.3  mrg ;; contents to %ecx then --> boom.
    406  1.3  mrg 
    407  1.3  mrg (define_mode_attr doublemodesuffix [(SI "8") (DI "16")])
    408  1.3  mrg (define_mode_attr regprefix [(SI "e") (DI "r")])
    409  1.3  mrg 
    410  1.3  mrg (define_insn "atomic_compare_and_swap<dwi>_doubleword"
    411  1.3  mrg   [(set (match_operand:DWIH 0 "register_operand" "=a,a")
    412  1.3  mrg 	(unspec_volatile:DWIH
    413  1.3  mrg 	  [(match_operand:<DWI> 2 "cmpxchg8b_pic_memory_operand" "+m,m")
    414  1.3  mrg 	   (match_operand:DWIH 3 "register_operand" "0,0")
    415  1.3  mrg 	   (match_operand:DWIH 4 "register_operand" "1,1")
    416  1.3  mrg 	   (match_operand:DWIH 5 "register_operand" "b,!*r")
    417  1.3  mrg 	   (match_operand:DWIH 6 "register_operand" "c,c")
    418  1.3  mrg 	   (match_operand:SI 7 "const_int_operand")]
    419  1.1  mrg 	  UNSPECV_CMPXCHG))
    420  1.3  mrg    (set (match_operand:DWIH 1 "register_operand" "=d,d")
    421  1.3  mrg 	(unspec_volatile:DWIH [(const_int 0)] UNSPECV_CMPXCHG))
    422  1.3  mrg    (set (match_dup 2)
    423  1.3  mrg 	(unspec_volatile:<DWI> [(const_int 0)] UNSPECV_CMPXCHG))
    424  1.1  mrg    (set (reg:CCZ FLAGS_REG)
    425  1.3  mrg         (unspec_volatile:CCZ [(const_int 0)] UNSPECV_CMPXCHG))
    426  1.3  mrg    (clobber (match_scratch:DWIH 8 "=X,&5"))]
    427  1.3  mrg   "TARGET_CMPXCHG<doublemodesuffix>B"
    428  1.3  mrg {
    429  1.3  mrg   bool swap = REGNO (operands[5]) != BX_REG;
    430  1.3  mrg   const char *xchg = "xchg{<imodesuffix>}\t%%<regprefix>bx, %5";
    431  1.3  mrg 
    432  1.3  mrg   if (swap)
    433  1.3  mrg     output_asm_insn (xchg, operands);
    434  1.3  mrg   output_asm_insn ("lock{%;} %K7cmpxchg<doublemodesuffix>b\t%2", operands);
    435  1.3  mrg   if (swap)
    436  1.3  mrg     output_asm_insn (xchg, operands);
    437  1.3  mrg 
    438  1.3  mrg   return "";
    439  1.3  mrg })
    440  1.1  mrg 
    441  1.3  mrg ;; For operand 2 nonmemory_operand predicate is used instead of
    442  1.3  mrg ;; register_operand to allow combiner to better optimize atomic
    443  1.3  mrg ;; additions of constants.
    444  1.3  mrg (define_insn "atomic_fetch_add<mode>"
    445  1.1  mrg   [(set (match_operand:SWI 0 "register_operand" "=<r>")
    446  1.1  mrg 	(unspec_volatile:SWI
    447  1.3  mrg 	  [(match_operand:SWI 1 "memory_operand" "+m")
    448  1.3  mrg 	   (match_operand:SI 3 "const_int_operand")]		;; model
    449  1.3  mrg 	  UNSPECV_XCHG))
    450  1.1  mrg    (set (match_dup 1)
    451  1.1  mrg 	(plus:SWI (match_dup 1)
    452  1.3  mrg 		  (match_operand:SWI 2 "nonmemory_operand" "0")))
    453  1.1  mrg    (clobber (reg:CC FLAGS_REG))]
    454  1.1  mrg   "TARGET_XADD"
    455  1.3  mrg   "lock{%;} %K3xadd{<imodesuffix>}\t{%0, %1|%1, %0}")
    456  1.3  mrg 
    457  1.3  mrg ;; This peephole2 and following insn optimize
    458  1.3  mrg ;; __sync_fetch_and_add (x, -N) == N into just lock {add,sub,inc,dec}
    459  1.3  mrg ;; followed by testing of flags instead of lock xadd and comparisons.
    460  1.3  mrg (define_peephole2
    461  1.3  mrg   [(set (match_operand:SWI 0 "register_operand")
    462  1.3  mrg 	(match_operand:SWI 2 "const_int_operand"))
    463  1.3  mrg    (parallel [(set (match_dup 0)
    464  1.3  mrg 		   (unspec_volatile:SWI
    465  1.3  mrg 		     [(match_operand:SWI 1 "memory_operand")
    466  1.3  mrg 		      (match_operand:SI 4 "const_int_operand")]
    467  1.3  mrg 		     UNSPECV_XCHG))
    468  1.3  mrg 	      (set (match_dup 1)
    469  1.3  mrg 		   (plus:SWI (match_dup 1)
    470  1.3  mrg 			     (match_dup 0)))
    471  1.3  mrg 	      (clobber (reg:CC FLAGS_REG))])
    472  1.3  mrg    (set (reg:CCZ FLAGS_REG)
    473  1.3  mrg 	(compare:CCZ (match_dup 0)
    474  1.3  mrg 		     (match_operand:SWI 3 "const_int_operand")))]
    475  1.3  mrg   "peep2_reg_dead_p (3, operands[0])
    476  1.3  mrg    && (unsigned HOST_WIDE_INT) INTVAL (operands[2])
    477  1.3  mrg       == -(unsigned HOST_WIDE_INT) INTVAL (operands[3])
    478  1.3  mrg    && !reg_overlap_mentioned_p (operands[0], operands[1])"
    479  1.3  mrg   [(parallel [(set (reg:CCZ FLAGS_REG)
    480  1.3  mrg 		   (compare:CCZ
    481  1.3  mrg 		     (unspec_volatile:SWI [(match_dup 1) (match_dup 4)]
    482  1.3  mrg 					  UNSPECV_XCHG)
    483  1.3  mrg 		     (match_dup 3)))
    484  1.3  mrg 	      (set (match_dup 1)
    485  1.3  mrg 		   (plus:SWI (match_dup 1)
    486  1.3  mrg 			     (match_dup 2)))])])
    487  1.3  mrg 
    488  1.3  mrg (define_insn "*atomic_fetch_add_cmp<mode>"
    489  1.3  mrg   [(set (reg:CCZ FLAGS_REG)
    490  1.3  mrg 	(compare:CCZ
    491  1.3  mrg 	  (unspec_volatile:SWI
    492  1.3  mrg 	    [(match_operand:SWI 0 "memory_operand" "+m")
    493  1.3  mrg 	     (match_operand:SI 3 "const_int_operand")]		;; model
    494  1.3  mrg 	    UNSPECV_XCHG)
    495  1.3  mrg 	  (match_operand:SWI 2 "const_int_operand" "i")))
    496  1.3  mrg    (set (match_dup 0)
    497  1.3  mrg 	(plus:SWI (match_dup 0)
    498  1.3  mrg 		  (match_operand:SWI 1 "const_int_operand" "i")))]
    499  1.3  mrg   "(unsigned HOST_WIDE_INT) INTVAL (operands[1])
    500  1.3  mrg    == -(unsigned HOST_WIDE_INT) INTVAL (operands[2])"
    501  1.3  mrg {
    502  1.3  mrg   if (incdec_operand (operands[1], <MODE>mode))
    503  1.3  mrg     {
    504  1.3  mrg       if (operands[1] == const1_rtx)
    505  1.3  mrg 	return "lock{%;} %K3inc{<imodesuffix>}\t%0";
    506  1.3  mrg       else
    507  1.3  mrg 	{
    508  1.3  mrg 	  gcc_assert (operands[1] == constm1_rtx);
    509  1.3  mrg 	  return "lock{%;} %K3dec{<imodesuffix>}\t%0";
    510  1.3  mrg 	}
    511  1.3  mrg     }
    512  1.3  mrg 
    513  1.3  mrg   if (x86_maybe_negate_const_int (&operands[1], <MODE>mode))
    514  1.3  mrg     return "lock{%;} %K3sub{<imodesuffix>}\t{%1, %0|%0, %1}";
    515  1.3  mrg 
    516  1.3  mrg   return "lock{%;} %K3add{<imodesuffix>}\t{%1, %0|%0, %1}";
    517  1.3  mrg })
    518  1.1  mrg 
    519  1.1  mrg ;; Recall that xchg implicitly sets LOCK#, so adding it again wastes space.
    520  1.3  mrg ;; In addition, it is always a full barrier, so we can ignore the memory model.
    521  1.3  mrg (define_insn "atomic_exchange<mode>"
    522  1.3  mrg   [(set (match_operand:SWI 0 "register_operand" "=<r>")		;; output
    523  1.1  mrg 	(unspec_volatile:SWI
    524  1.3  mrg 	  [(match_operand:SWI 1 "memory_operand" "+m")		;; memory
    525  1.3  mrg 	   (match_operand:SI 3 "const_int_operand")]		;; model
    526  1.3  mrg 	  UNSPECV_XCHG))
    527  1.1  mrg    (set (match_dup 1)
    528  1.3  mrg 	(match_operand:SWI 2 "register_operand" "0"))]		;; input
    529  1.1  mrg   ""
    530  1.3  mrg   "%K3xchg{<imodesuffix>}\t{%1, %0|%0, %1}")
    531  1.1  mrg 
    532  1.3  mrg (define_insn "atomic_add<mode>"
    533  1.1  mrg   [(set (match_operand:SWI 0 "memory_operand" "+m")
    534  1.1  mrg 	(unspec_volatile:SWI
    535  1.1  mrg 	  [(plus:SWI (match_dup 0)
    536  1.3  mrg 		     (match_operand:SWI 1 "nonmemory_operand" "<r><i>"))
    537  1.3  mrg 	   (match_operand:SI 2 "const_int_operand")]		;; model
    538  1.1  mrg 	  UNSPECV_LOCK))
    539  1.1  mrg    (clobber (reg:CC FLAGS_REG))]
    540  1.1  mrg   ""
    541  1.1  mrg {
    542  1.3  mrg   if (incdec_operand (operands[1], <MODE>mode))
    543  1.1  mrg     {
    544  1.1  mrg       if (operands[1] == const1_rtx)
    545  1.3  mrg 	return "lock{%;} %K2inc{<imodesuffix>}\t%0";
    546  1.3  mrg       else
    547  1.3  mrg 	{
    548  1.3  mrg 	  gcc_assert (operands[1] == constm1_rtx);
    549  1.3  mrg 	  return "lock{%;} %K2dec{<imodesuffix>}\t%0";
    550  1.3  mrg 	}
    551  1.1  mrg     }
    552  1.1  mrg 
    553  1.3  mrg   if (x86_maybe_negate_const_int (&operands[1], <MODE>mode))
    554  1.3  mrg     return "lock{%;} %K2sub{<imodesuffix>}\t{%1, %0|%0, %1}";
    555  1.3  mrg 
    556  1.3  mrg   return "lock{%;} %K2add{<imodesuffix>}\t{%1, %0|%0, %1}";
    557  1.1  mrg })
    558  1.1  mrg 
    559  1.3  mrg (define_insn "atomic_sub<mode>"
    560  1.1  mrg   [(set (match_operand:SWI 0 "memory_operand" "+m")
    561  1.1  mrg 	(unspec_volatile:SWI
    562  1.1  mrg 	  [(minus:SWI (match_dup 0)
    563  1.3  mrg 		      (match_operand:SWI 1 "nonmemory_operand" "<r><i>"))
    564  1.3  mrg 	   (match_operand:SI 2 "const_int_operand")]		;; model
    565  1.1  mrg 	  UNSPECV_LOCK))
    566  1.1  mrg    (clobber (reg:CC FLAGS_REG))]
    567  1.1  mrg   ""
    568  1.1  mrg {
    569  1.3  mrg   if (incdec_operand (operands[1], <MODE>mode))
    570  1.1  mrg     {
    571  1.1  mrg       if (operands[1] == const1_rtx)
    572  1.3  mrg 	return "lock{%;} %K2dec{<imodesuffix>}\t%0";
    573  1.3  mrg       else
    574  1.3  mrg 	{
    575  1.3  mrg 	  gcc_assert (operands[1] == constm1_rtx);
    576  1.3  mrg 	  return "lock{%;} %K2inc{<imodesuffix>}\t%0";
    577  1.3  mrg 	}
    578  1.1  mrg     }
    579  1.1  mrg 
    580  1.3  mrg   if (x86_maybe_negate_const_int (&operands[1], <MODE>mode))
    581  1.3  mrg     return "lock{%;} %K2add{<imodesuffix>}\t{%1, %0|%0, %1}";
    582  1.3  mrg 
    583  1.3  mrg   return "lock{%;} %K2sub{<imodesuffix>}\t{%1, %0|%0, %1}";
    584  1.1  mrg })
    585  1.1  mrg 
    586  1.3  mrg (define_insn "atomic_<logic><mode>"
    587  1.1  mrg   [(set (match_operand:SWI 0 "memory_operand" "+m")
    588  1.1  mrg 	(unspec_volatile:SWI
    589  1.1  mrg 	  [(any_logic:SWI (match_dup 0)
    590  1.3  mrg 			  (match_operand:SWI 1 "nonmemory_operand" "<r><i>"))
    591  1.3  mrg 	   (match_operand:SI 2 "const_int_operand")]		;; model
    592  1.1  mrg 	  UNSPECV_LOCK))
    593  1.1  mrg    (clobber (reg:CC FLAGS_REG))]
    594  1.1  mrg   ""
    595  1.3  mrg   "lock{%;} %K2<logic>{<imodesuffix>}\t{%1, %0|%0, %1}")
    596