aarch64-sve.md revision 1.1.1.2
1 ;; Machine description for AArch64 SVE. 2 ;; Copyright (C) 2009-2019 Free Software Foundation, Inc. 3 ;; Contributed by ARM Ltd. 4 ;; 5 ;; This file is part of GCC. 6 ;; 7 ;; GCC is free software; you can redistribute it and/or modify it 8 ;; under the terms of the GNU General Public License as published by 9 ;; the Free Software Foundation; either version 3, or (at your option) 10 ;; any later version. 11 ;; 12 ;; GCC is distributed in the hope that it will be useful, but 13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of 14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 ;; General Public License for more details. 16 ;; 17 ;; You should have received a copy of the GNU General Public License 18 ;; along with GCC; see the file COPYING3. If not see 19 ;; <http://www.gnu.org/licenses/>. 20 21 ;; Note on the handling of big-endian SVE 22 ;; -------------------------------------- 23 ;; 24 ;; On big-endian systems, Advanced SIMD mov<mode> patterns act in the 25 ;; same way as movdi or movti would: the first byte of memory goes 26 ;; into the most significant byte of the register and the last byte 27 ;; of memory goes into the least significant byte of the register. 28 ;; This is the most natural ordering for Advanced SIMD and matches 29 ;; the ABI layout for 64-bit and 128-bit vector types. 30 ;; 31 ;; As a result, the order of bytes within the register is what GCC 32 ;; expects for a big-endian target, and subreg offsets therefore work 33 ;; as expected, with the first element in memory having subreg offset 0 34 ;; and the last element in memory having the subreg offset associated 35 ;; with a big-endian lowpart. However, this ordering also means that 36 ;; GCC's lane numbering does not match the architecture's numbering: 37 ;; GCC always treats the element at the lowest address in memory 38 ;; (subreg offset 0) as element 0, while the architecture treats 39 ;; the least significant end of the register as element 0. 40 ;; 41 ;; The situation for SVE is different. We want the layout of the 42 ;; SVE register to be same for mov<mode> as it is for maskload<mode>: 43 ;; logically, a mov<mode> load must be indistinguishable from a 44 ;; maskload<mode> whose mask is all true. We therefore need the 45 ;; register layout to match LD1 rather than LDR. The ABI layout of 46 ;; SVE types also matches LD1 byte ordering rather than LDR byte ordering. 47 ;; 48 ;; As a result, the architecture lane numbering matches GCC's lane 49 ;; numbering, with element 0 always being the first in memory. 50 ;; However: 51 ;; 52 ;; - Applying a subreg offset to a register does not give the element 53 ;; that GCC expects: the first element in memory has the subreg offset 54 ;; associated with a big-endian lowpart while the last element in memory 55 ;; has subreg offset 0. We handle this via TARGET_CAN_CHANGE_MODE_CLASS. 56 ;; 57 ;; - We cannot use LDR and STR for spill slots that might be accessed 58 ;; via subregs, since although the elements have the order GCC expects, 59 ;; the order of the bytes within the elements is different. We instead 60 ;; access spill slots via LD1 and ST1, using secondary reloads to 61 ;; reserve a predicate register. 62 63 64 ;; SVE data moves. 65 (define_expand "mov<mode>" 66 [(set (match_operand:SVE_ALL 0 "nonimmediate_operand") 67 (match_operand:SVE_ALL 1 "general_operand"))] 68 "TARGET_SVE" 69 { 70 /* Use the predicated load and store patterns where possible. 71 This is required for big-endian targets (see the comment at the 72 head of the file) and increases the addressing choices for 73 little-endian. */ 74 if ((MEM_P (operands[0]) || MEM_P (operands[1])) 75 && can_create_pseudo_p ()) 76 { 77 aarch64_expand_sve_mem_move (operands[0], operands[1], <VPRED>mode); 78 DONE; 79 } 80 81 if (CONSTANT_P (operands[1])) 82 { 83 aarch64_expand_mov_immediate (operands[0], operands[1], 84 gen_vec_duplicate<mode>); 85 DONE; 86 } 87 88 /* Optimize subregs on big-endian targets: we can use REV[BHW] 89 instead of going through memory. */ 90 if (BYTES_BIG_ENDIAN 91 && aarch64_maybe_expand_sve_subreg_move (operands[0], operands[1])) 92 DONE; 93 } 94 ) 95 96 ;; A pattern for optimizing SUBREGs that have a reinterpreting effect 97 ;; on big-endian targets; see aarch64_maybe_expand_sve_subreg_move 98 ;; for details. We use a special predicate for operand 2 to reduce 99 ;; the number of patterns. 100 (define_insn_and_split "*aarch64_sve_mov<mode>_subreg_be" 101 [(set (match_operand:SVE_ALL 0 "aarch64_sve_nonimmediate_operand" "=w") 102 (unspec:SVE_ALL 103 [(match_operand:VNx16BI 1 "register_operand" "Upl") 104 (match_operand 2 "aarch64_any_register_operand" "w")] 105 UNSPEC_REV_SUBREG))] 106 "TARGET_SVE && BYTES_BIG_ENDIAN" 107 "#" 108 "&& reload_completed" 109 [(const_int 0)] 110 { 111 aarch64_split_sve_subreg_move (operands[0], operands[1], operands[2]); 112 DONE; 113 } 114 ) 115 116 ;; Unpredicated moves (little-endian). Only allow memory operations 117 ;; during and after RA; before RA we want the predicated load and 118 ;; store patterns to be used instead. 119 (define_insn "*aarch64_sve_mov<mode>_le" 120 [(set (match_operand:SVE_ALL 0 "aarch64_sve_nonimmediate_operand" "=w, Utr, w, w") 121 (match_operand:SVE_ALL 1 "aarch64_sve_general_operand" "Utr, w, w, Dn"))] 122 "TARGET_SVE 123 && !BYTES_BIG_ENDIAN 124 && ((lra_in_progress || reload_completed) 125 || (register_operand (operands[0], <MODE>mode) 126 && nonmemory_operand (operands[1], <MODE>mode)))" 127 "@ 128 ldr\t%0, %1 129 str\t%1, %0 130 mov\t%0.d, %1.d 131 * return aarch64_output_sve_mov_immediate (operands[1]);" 132 ) 133 134 ;; Unpredicated moves (big-endian). Memory accesses require secondary 135 ;; reloads. 136 (define_insn "*aarch64_sve_mov<mode>_be" 137 [(set (match_operand:SVE_ALL 0 "register_operand" "=w, w") 138 (match_operand:SVE_ALL 1 "aarch64_nonmemory_operand" "w, Dn"))] 139 "TARGET_SVE && BYTES_BIG_ENDIAN" 140 "@ 141 mov\t%0.d, %1.d 142 * return aarch64_output_sve_mov_immediate (operands[1]);" 143 ) 144 145 ;; Handle big-endian memory reloads. We use byte PTRUE for all modes 146 ;; to try to encourage reuse. 147 (define_expand "aarch64_sve_reload_be" 148 [(parallel 149 [(set (match_operand 0) 150 (match_operand 1)) 151 (clobber (match_operand:VNx16BI 2 "register_operand" "=Upl"))])] 152 "TARGET_SVE && BYTES_BIG_ENDIAN" 153 { 154 /* Create a PTRUE. */ 155 emit_move_insn (operands[2], CONSTM1_RTX (VNx16BImode)); 156 157 /* Refer to the PTRUE in the appropriate mode for this move. */ 158 machine_mode mode = GET_MODE (operands[0]); 159 machine_mode pred_mode 160 = aarch64_sve_pred_mode (GET_MODE_UNIT_SIZE (mode)).require (); 161 rtx pred = gen_lowpart (pred_mode, operands[2]); 162 163 /* Emit a predicated load or store. */ 164 aarch64_emit_sve_pred_move (operands[0], pred, operands[1]); 165 DONE; 166 } 167 ) 168 169 ;; A predicated load or store for which the predicate is known to be 170 ;; all-true. Note that this pattern is generated directly by 171 ;; aarch64_emit_sve_pred_move, so changes to this pattern will 172 ;; need changes there as well. 173 (define_insn_and_split "@aarch64_pred_mov<mode>" 174 [(set (match_operand:SVE_ALL 0 "nonimmediate_operand" "=w, w, m") 175 (unspec:SVE_ALL 176 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl") 177 (match_operand:SVE_ALL 2 "nonimmediate_operand" "w, m, w")] 178 UNSPEC_MERGE_PTRUE))] 179 "TARGET_SVE 180 && (register_operand (operands[0], <MODE>mode) 181 || register_operand (operands[2], <MODE>mode))" 182 "@ 183 # 184 ld1<Vesize>\t%0.<Vetype>, %1/z, %2 185 st1<Vesize>\t%2.<Vetype>, %1, %0" 186 "&& register_operand (operands[0], <MODE>mode) 187 && register_operand (operands[2], <MODE>mode)" 188 [(set (match_dup 0) (match_dup 2))] 189 ) 190 191 (define_expand "movmisalign<mode>" 192 [(set (match_operand:SVE_ALL 0 "nonimmediate_operand") 193 (match_operand:SVE_ALL 1 "general_operand"))] 194 "TARGET_SVE" 195 { 196 /* Equivalent to a normal move for our purpooses. */ 197 emit_move_insn (operands[0], operands[1]); 198 DONE; 199 } 200 ) 201 202 (define_insn "maskload<mode><vpred>" 203 [(set (match_operand:SVE_ALL 0 "register_operand" "=w") 204 (unspec:SVE_ALL 205 [(match_operand:<VPRED> 2 "register_operand" "Upl") 206 (match_operand:SVE_ALL 1 "memory_operand" "m")] 207 UNSPEC_LD1_SVE))] 208 "TARGET_SVE" 209 "ld1<Vesize>\t%0.<Vetype>, %2/z, %1" 210 ) 211 212 (define_insn "maskstore<mode><vpred>" 213 [(set (match_operand:SVE_ALL 0 "memory_operand" "+m") 214 (unspec:SVE_ALL [(match_operand:<VPRED> 2 "register_operand" "Upl") 215 (match_operand:SVE_ALL 1 "register_operand" "w") 216 (match_dup 0)] 217 UNSPEC_ST1_SVE))] 218 "TARGET_SVE" 219 "st1<Vesize>\t%1.<Vetype>, %2, %0" 220 ) 221 222 ;; Unpredicated gather loads. 223 (define_expand "gather_load<mode>" 224 [(set (match_operand:SVE_SD 0 "register_operand") 225 (unspec:SVE_SD 226 [(match_dup 5) 227 (match_operand:DI 1 "aarch64_reg_or_zero") 228 (match_operand:<V_INT_EQUIV> 2 "register_operand") 229 (match_operand:DI 3 "const_int_operand") 230 (match_operand:DI 4 "aarch64_gather_scale_operand_<Vesize>") 231 (mem:BLK (scratch))] 232 UNSPEC_LD1_GATHER))] 233 "TARGET_SVE" 234 { 235 operands[5] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode)); 236 } 237 ) 238 239 ;; Predicated gather loads for 32-bit elements. Operand 3 is true for 240 ;; unsigned extension and false for signed extension. 241 (define_insn "mask_gather_load<mode>" 242 [(set (match_operand:SVE_S 0 "register_operand" "=w, w, w, w, w") 243 (unspec:SVE_S 244 [(match_operand:<VPRED> 5 "register_operand" "Upl, Upl, Upl, Upl, Upl") 245 (match_operand:DI 1 "aarch64_reg_or_zero" "Z, rk, rk, rk, rk") 246 (match_operand:<V_INT_EQUIV> 2 "register_operand" "w, w, w, w, w") 247 (match_operand:DI 3 "const_int_operand" "i, Z, Ui1, Z, Ui1") 248 (match_operand:DI 4 "aarch64_gather_scale_operand_w" "Ui1, Ui1, Ui1, i, i") 249 (mem:BLK (scratch))] 250 UNSPEC_LD1_GATHER))] 251 "TARGET_SVE" 252 "@ 253 ld1w\t%0.s, %5/z, [%2.s] 254 ld1w\t%0.s, %5/z, [%1, %2.s, sxtw] 255 ld1w\t%0.s, %5/z, [%1, %2.s, uxtw] 256 ld1w\t%0.s, %5/z, [%1, %2.s, sxtw %p4] 257 ld1w\t%0.s, %5/z, [%1, %2.s, uxtw %p4]" 258 ) 259 260 ;; Predicated gather loads for 64-bit elements. The value of operand 3 261 ;; doesn't matter in this case. 262 (define_insn "mask_gather_load<mode>" 263 [(set (match_operand:SVE_D 0 "register_operand" "=w, w, w") 264 (unspec:SVE_D 265 [(match_operand:<VPRED> 5 "register_operand" "Upl, Upl, Upl") 266 (match_operand:DI 1 "aarch64_reg_or_zero" "Z, rk, rk") 267 (match_operand:<V_INT_EQUIV> 2 "register_operand" "w, w, w") 268 (match_operand:DI 3 "const_int_operand") 269 (match_operand:DI 4 "aarch64_gather_scale_operand_d" "Ui1, Ui1, i") 270 (mem:BLK (scratch))] 271 UNSPEC_LD1_GATHER))] 272 "TARGET_SVE" 273 "@ 274 ld1d\t%0.d, %5/z, [%2.d] 275 ld1d\t%0.d, %5/z, [%1, %2.d] 276 ld1d\t%0.d, %5/z, [%1, %2.d, lsl %p4]" 277 ) 278 279 ;; Unpredicated scatter store. 280 (define_expand "scatter_store<mode>" 281 [(set (mem:BLK (scratch)) 282 (unspec:BLK 283 [(match_dup 5) 284 (match_operand:DI 0 "aarch64_reg_or_zero") 285 (match_operand:<V_INT_EQUIV> 1 "register_operand") 286 (match_operand:DI 2 "const_int_operand") 287 (match_operand:DI 3 "aarch64_gather_scale_operand_<Vesize>") 288 (match_operand:SVE_SD 4 "register_operand")] 289 UNSPEC_ST1_SCATTER))] 290 "TARGET_SVE" 291 { 292 operands[5] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode)); 293 } 294 ) 295 296 ;; Predicated scatter stores for 32-bit elements. Operand 2 is true for 297 ;; unsigned extension and false for signed extension. 298 (define_insn "mask_scatter_store<mode>" 299 [(set (mem:BLK (scratch)) 300 (unspec:BLK 301 [(match_operand:<VPRED> 5 "register_operand" "Upl, Upl, Upl, Upl, Upl") 302 (match_operand:DI 0 "aarch64_reg_or_zero" "Z, rk, rk, rk, rk") 303 (match_operand:<V_INT_EQUIV> 1 "register_operand" "w, w, w, w, w") 304 (match_operand:DI 2 "const_int_operand" "i, Z, Ui1, Z, Ui1") 305 (match_operand:DI 3 "aarch64_gather_scale_operand_w" "Ui1, Ui1, Ui1, i, i") 306 (match_operand:SVE_S 4 "register_operand" "w, w, w, w, w")] 307 UNSPEC_ST1_SCATTER))] 308 "TARGET_SVE" 309 "@ 310 st1w\t%4.s, %5, [%1.s] 311 st1w\t%4.s, %5, [%0, %1.s, sxtw] 312 st1w\t%4.s, %5, [%0, %1.s, uxtw] 313 st1w\t%4.s, %5, [%0, %1.s, sxtw %p3] 314 st1w\t%4.s, %5, [%0, %1.s, uxtw %p3]" 315 ) 316 317 ;; Predicated scatter stores for 64-bit elements. The value of operand 2 318 ;; doesn't matter in this case. 319 (define_insn "mask_scatter_store<mode>" 320 [(set (mem:BLK (scratch)) 321 (unspec:BLK 322 [(match_operand:<VPRED> 5 "register_operand" "Upl, Upl, Upl") 323 (match_operand:DI 0 "aarch64_reg_or_zero" "Z, rk, rk") 324 (match_operand:<V_INT_EQUIV> 1 "register_operand" "w, w, w") 325 (match_operand:DI 2 "const_int_operand") 326 (match_operand:DI 3 "aarch64_gather_scale_operand_d" "Ui1, Ui1, i") 327 (match_operand:SVE_D 4 "register_operand" "w, w, w")] 328 UNSPEC_ST1_SCATTER))] 329 "TARGET_SVE" 330 "@ 331 st1d\t%4.d, %5, [%1.d] 332 st1d\t%4.d, %5, [%0, %1.d] 333 st1d\t%4.d, %5, [%0, %1.d, lsl %p3]" 334 ) 335 336 ;; SVE structure moves. 337 (define_expand "mov<mode>" 338 [(set (match_operand:SVE_STRUCT 0 "nonimmediate_operand") 339 (match_operand:SVE_STRUCT 1 "general_operand"))] 340 "TARGET_SVE" 341 { 342 /* Big-endian loads and stores need to be done via LD1 and ST1; 343 see the comment at the head of the file for details. */ 344 if ((MEM_P (operands[0]) || MEM_P (operands[1])) 345 && BYTES_BIG_ENDIAN) 346 { 347 gcc_assert (can_create_pseudo_p ()); 348 aarch64_expand_sve_mem_move (operands[0], operands[1], <VPRED>mode); 349 DONE; 350 } 351 352 if (CONSTANT_P (operands[1])) 353 { 354 aarch64_expand_mov_immediate (operands[0], operands[1]); 355 DONE; 356 } 357 } 358 ) 359 360 ;; Unpredicated structure moves (little-endian). 361 (define_insn "*aarch64_sve_mov<mode>_le" 362 [(set (match_operand:SVE_STRUCT 0 "aarch64_sve_nonimmediate_operand" "=w, Utr, w, w") 363 (match_operand:SVE_STRUCT 1 "aarch64_sve_general_operand" "Utr, w, w, Dn"))] 364 "TARGET_SVE && !BYTES_BIG_ENDIAN" 365 "#" 366 [(set_attr "length" "<insn_length>")] 367 ) 368 369 ;; Unpredicated structure moves (big-endian). Memory accesses require 370 ;; secondary reloads. 371 (define_insn "*aarch64_sve_mov<mode>_le" 372 [(set (match_operand:SVE_STRUCT 0 "register_operand" "=w, w") 373 (match_operand:SVE_STRUCT 1 "aarch64_nonmemory_operand" "w, Dn"))] 374 "TARGET_SVE && BYTES_BIG_ENDIAN" 375 "#" 376 [(set_attr "length" "<insn_length>")] 377 ) 378 379 ;; Split unpredicated structure moves into pieces. This is the same 380 ;; for both big-endian and little-endian code, although it only needs 381 ;; to handle memory operands for little-endian code. 382 (define_split 383 [(set (match_operand:SVE_STRUCT 0 "aarch64_sve_nonimmediate_operand") 384 (match_operand:SVE_STRUCT 1 "aarch64_sve_general_operand"))] 385 "TARGET_SVE && reload_completed" 386 [(const_int 0)] 387 { 388 rtx dest = operands[0]; 389 rtx src = operands[1]; 390 if (REG_P (dest) && REG_P (src)) 391 aarch64_simd_emit_reg_reg_move (operands, <VSINGLE>mode, <vector_count>); 392 else 393 for (unsigned int i = 0; i < <vector_count>; ++i) 394 { 395 rtx subdest = simplify_gen_subreg (<VSINGLE>mode, dest, <MODE>mode, 396 i * BYTES_PER_SVE_VECTOR); 397 rtx subsrc = simplify_gen_subreg (<VSINGLE>mode, src, <MODE>mode, 398 i * BYTES_PER_SVE_VECTOR); 399 emit_insn (gen_rtx_SET (subdest, subsrc)); 400 } 401 DONE; 402 } 403 ) 404 405 ;; Predicated structure moves. This works for both endiannesses but in 406 ;; practice is only useful for big-endian. 407 (define_insn_and_split "@aarch64_pred_mov<mode>" 408 [(set (match_operand:SVE_STRUCT 0 "aarch64_sve_struct_nonimmediate_operand" "=w, w, Utx") 409 (unspec:SVE_STRUCT 410 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl") 411 (match_operand:SVE_STRUCT 2 "aarch64_sve_struct_nonimmediate_operand" "w, Utx, w")] 412 UNSPEC_MERGE_PTRUE))] 413 "TARGET_SVE 414 && (register_operand (operands[0], <MODE>mode) 415 || register_operand (operands[2], <MODE>mode))" 416 "#" 417 "&& reload_completed" 418 [(const_int 0)] 419 { 420 for (unsigned int i = 0; i < <vector_count>; ++i) 421 { 422 rtx subdest = simplify_gen_subreg (<VSINGLE>mode, operands[0], 423 <MODE>mode, 424 i * BYTES_PER_SVE_VECTOR); 425 rtx subsrc = simplify_gen_subreg (<VSINGLE>mode, operands[2], 426 <MODE>mode, 427 i * BYTES_PER_SVE_VECTOR); 428 aarch64_emit_sve_pred_move (subdest, operands[1], subsrc); 429 } 430 DONE; 431 } 432 [(set_attr "length" "<insn_length>")] 433 ) 434 435 (define_expand "mov<mode>" 436 [(set (match_operand:PRED_ALL 0 "nonimmediate_operand") 437 (match_operand:PRED_ALL 1 "general_operand"))] 438 "TARGET_SVE" 439 { 440 if (GET_CODE (operands[0]) == MEM) 441 operands[1] = force_reg (<MODE>mode, operands[1]); 442 } 443 ) 444 445 (define_insn "*aarch64_sve_mov<mode>" 446 [(set (match_operand:PRED_ALL 0 "nonimmediate_operand" "=Upa, m, Upa, Upa, Upa") 447 (match_operand:PRED_ALL 1 "general_operand" "Upa, Upa, m, Dz, Dm"))] 448 "TARGET_SVE 449 && (register_operand (operands[0], <MODE>mode) 450 || register_operand (operands[1], <MODE>mode))" 451 "@ 452 mov\t%0.b, %1.b 453 str\t%1, %0 454 ldr\t%0, %1 455 pfalse\t%0.b 456 * return aarch64_output_ptrue (<MODE>mode, '<Vetype>');" 457 ) 458 459 ;; Handle extractions from a predicate by converting to an integer vector 460 ;; and extracting from there. 461 (define_expand "vec_extract<vpred><Vel>" 462 [(match_operand:<VEL> 0 "register_operand") 463 (match_operand:<VPRED> 1 "register_operand") 464 (match_operand:SI 2 "nonmemory_operand") 465 ;; Dummy operand to which we can attach the iterator. 466 (reg:SVE_I V0_REGNUM)] 467 "TARGET_SVE" 468 { 469 rtx tmp = gen_reg_rtx (<MODE>mode); 470 emit_insn (gen_aarch64_sve_dup<mode>_const (tmp, operands[1], 471 CONST1_RTX (<MODE>mode), 472 CONST0_RTX (<MODE>mode))); 473 emit_insn (gen_vec_extract<mode><Vel> (operands[0], tmp, operands[2])); 474 DONE; 475 } 476 ) 477 478 (define_expand "vec_extract<mode><Vel>" 479 [(set (match_operand:<VEL> 0 "register_operand") 480 (vec_select:<VEL> 481 (match_operand:SVE_ALL 1 "register_operand") 482 (parallel [(match_operand:SI 2 "nonmemory_operand")])))] 483 "TARGET_SVE" 484 { 485 poly_int64 val; 486 if (poly_int_rtx_p (operands[2], &val) 487 && known_eq (val, GET_MODE_NUNITS (<MODE>mode) - 1)) 488 { 489 /* The last element can be extracted with a LASTB and a false 490 predicate. */ 491 rtx sel = force_reg (<VPRED>mode, CONST0_RTX (<VPRED>mode)); 492 emit_insn (gen_extract_last_<mode> (operands[0], sel, operands[1])); 493 DONE; 494 } 495 if (!CONST_INT_P (operands[2])) 496 { 497 /* Create an index with operand[2] as the base and -1 as the step. 498 It will then be zero for the element we care about. */ 499 rtx index = gen_lowpart (<VEL_INT>mode, operands[2]); 500 index = force_reg (<VEL_INT>mode, index); 501 rtx series = gen_reg_rtx (<V_INT_EQUIV>mode); 502 emit_insn (gen_vec_series<v_int_equiv> (series, index, constm1_rtx)); 503 504 /* Get a predicate that is true for only that element. */ 505 rtx zero = CONST0_RTX (<V_INT_EQUIV>mode); 506 rtx cmp = gen_rtx_EQ (<V_INT_EQUIV>mode, series, zero); 507 rtx sel = gen_reg_rtx (<VPRED>mode); 508 emit_insn (gen_vec_cmp<v_int_equiv><vpred> (sel, cmp, series, zero)); 509 510 /* Select the element using LASTB. */ 511 emit_insn (gen_extract_last_<mode> (operands[0], sel, operands[1])); 512 DONE; 513 } 514 } 515 ) 516 517 ;; Extract element zero. This is a special case because we want to force 518 ;; the registers to be the same for the second alternative, and then 519 ;; split the instruction into nothing after RA. 520 (define_insn_and_split "*vec_extract<mode><Vel>_0" 521 [(set (match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand" "=r, w, Utv") 522 (vec_select:<VEL> 523 (match_operand:SVE_ALL 1 "register_operand" "w, 0, w") 524 (parallel [(const_int 0)])))] 525 "TARGET_SVE" 526 { 527 operands[1] = gen_rtx_REG (<V128>mode, REGNO (operands[1])); 528 switch (which_alternative) 529 { 530 case 0: 531 return "umov\\t%<vwcore>0, %1.<Vetype>[0]"; 532 case 1: 533 return "#"; 534 case 2: 535 return "st1\\t{%1.<Vetype>}[0], %0"; 536 default: 537 gcc_unreachable (); 538 } 539 } 540 "&& reload_completed 541 && REG_P (operands[0]) 542 && REGNO (operands[0]) == REGNO (operands[1])" 543 [(const_int 0)] 544 { 545 emit_note (NOTE_INSN_DELETED); 546 DONE; 547 } 548 [(set_attr "type" "neon_to_gp_q, untyped, neon_store1_one_lane_q")] 549 ) 550 551 ;; Extract an element from the Advanced SIMD portion of the register. 552 ;; We don't just reuse the aarch64-simd.md pattern because we don't 553 ;; want any change in lane number on big-endian targets. 554 (define_insn "*vec_extract<mode><Vel>_v128" 555 [(set (match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand" "=r, w, Utv") 556 (vec_select:<VEL> 557 (match_operand:SVE_ALL 1 "register_operand" "w, w, w") 558 (parallel [(match_operand:SI 2 "const_int_operand")])))] 559 "TARGET_SVE 560 && IN_RANGE (INTVAL (operands[2]) * GET_MODE_SIZE (<VEL>mode), 1, 15)" 561 { 562 operands[1] = gen_rtx_REG (<V128>mode, REGNO (operands[1])); 563 switch (which_alternative) 564 { 565 case 0: 566 return "umov\\t%<vwcore>0, %1.<Vetype>[%2]"; 567 case 1: 568 return "dup\\t%<Vetype>0, %1.<Vetype>[%2]"; 569 case 2: 570 return "st1\\t{%1.<Vetype>}[%2], %0"; 571 default: 572 gcc_unreachable (); 573 } 574 } 575 [(set_attr "type" "neon_to_gp_q, neon_dup_q, neon_store1_one_lane_q")] 576 ) 577 578 ;; Extract an element in the range of DUP. This pattern allows the 579 ;; source and destination to be different. 580 (define_insn "*vec_extract<mode><Vel>_dup" 581 [(set (match_operand:<VEL> 0 "register_operand" "=w") 582 (vec_select:<VEL> 583 (match_operand:SVE_ALL 1 "register_operand" "w") 584 (parallel [(match_operand:SI 2 "const_int_operand")])))] 585 "TARGET_SVE 586 && IN_RANGE (INTVAL (operands[2]) * GET_MODE_SIZE (<VEL>mode), 16, 63)" 587 { 588 operands[0] = gen_rtx_REG (<MODE>mode, REGNO (operands[0])); 589 return "dup\t%0.<Vetype>, %1.<Vetype>[%2]"; 590 } 591 ) 592 593 ;; Extract an element outside the range of DUP. This pattern requires the 594 ;; source and destination to be the same. 595 (define_insn "*vec_extract<mode><Vel>_ext" 596 [(set (match_operand:<VEL> 0 "register_operand" "=w") 597 (vec_select:<VEL> 598 (match_operand:SVE_ALL 1 "register_operand" "0") 599 (parallel [(match_operand:SI 2 "const_int_operand")])))] 600 "TARGET_SVE && INTVAL (operands[2]) * GET_MODE_SIZE (<VEL>mode) >= 64" 601 { 602 operands[0] = gen_rtx_REG (<MODE>mode, REGNO (operands[0])); 603 operands[2] = GEN_INT (INTVAL (operands[2]) * GET_MODE_SIZE (<VEL>mode)); 604 return "ext\t%0.b, %0.b, %0.b, #%2"; 605 } 606 ) 607 608 ;; Extract the last active element of operand 1 into operand 0. 609 ;; If no elements are active, extract the last inactive element instead. 610 (define_insn "extract_last_<mode>" 611 [(set (match_operand:<VEL> 0 "register_operand" "=r, w") 612 (unspec:<VEL> 613 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl") 614 (match_operand:SVE_ALL 2 "register_operand" "w, w")] 615 UNSPEC_LASTB))] 616 "TARGET_SVE" 617 "@ 618 lastb\t%<vwcore>0, %1, %2.<Vetype> 619 lastb\t%<Vetype>0, %1, %2.<Vetype>" 620 ) 621 622 (define_expand "vec_duplicate<mode>" 623 [(parallel 624 [(set (match_operand:SVE_ALL 0 "register_operand") 625 (vec_duplicate:SVE_ALL 626 (match_operand:<VEL> 1 "aarch64_sve_dup_operand"))) 627 (clobber (scratch:<VPRED>))])] 628 "TARGET_SVE" 629 { 630 if (MEM_P (operands[1])) 631 { 632 rtx ptrue = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode)); 633 emit_insn (gen_sve_ld1r<mode> (operands[0], ptrue, operands[1], 634 CONST0_RTX (<MODE>mode))); 635 DONE; 636 } 637 } 638 ) 639 640 ;; Accept memory operands for the benefit of combine, and also in case 641 ;; the scalar input gets spilled to memory during RA. We want to split 642 ;; the load at the first opportunity in order to allow the PTRUE to be 643 ;; optimized with surrounding code. 644 (define_insn_and_split "*vec_duplicate<mode>_reg" 645 [(set (match_operand:SVE_ALL 0 "register_operand" "=w, w, w") 646 (vec_duplicate:SVE_ALL 647 (match_operand:<VEL> 1 "aarch64_sve_dup_operand" "r, w, Uty"))) 648 (clobber (match_scratch:<VPRED> 2 "=X, X, Upl"))] 649 "TARGET_SVE" 650 "@ 651 mov\t%0.<Vetype>, %<vwcore>1 652 mov\t%0.<Vetype>, %<Vetype>1 653 #" 654 "&& MEM_P (operands[1])" 655 [(const_int 0)] 656 { 657 if (GET_CODE (operands[2]) == SCRATCH) 658 operands[2] = gen_reg_rtx (<VPRED>mode); 659 emit_move_insn (operands[2], CONSTM1_RTX (<VPRED>mode)); 660 emit_insn (gen_sve_ld1r<mode> (operands[0], operands[2], operands[1], 661 CONST0_RTX (<MODE>mode))); 662 DONE; 663 } 664 [(set_attr "length" "4,4,8")] 665 ) 666 667 ;; This is used for vec_duplicate<mode>s from memory, but can also 668 ;; be used by combine to optimize selects of a a vec_duplicate<mode> 669 ;; with zero. 670 (define_insn "sve_ld1r<mode>" 671 [(set (match_operand:SVE_ALL 0 "register_operand" "=w") 672 (unspec:SVE_ALL 673 [(match_operand:<VPRED> 1 "register_operand" "Upl") 674 (vec_duplicate:SVE_ALL 675 (match_operand:<VEL> 2 "aarch64_sve_ld1r_operand" "Uty")) 676 (match_operand:SVE_ALL 3 "aarch64_simd_imm_zero")] 677 UNSPEC_SEL))] 678 "TARGET_SVE" 679 "ld1r<Vesize>\t%0.<Vetype>, %1/z, %2" 680 ) 681 682 ;; Load 128 bits from memory and duplicate to fill a vector. Since there 683 ;; are so few operations on 128-bit "elements", we don't define a VNx1TI 684 ;; and simply use vectors of bytes instead. 685 (define_insn "*sve_ld1rq<Vesize>" 686 [(set (match_operand:SVE_ALL 0 "register_operand" "=w") 687 (unspec:SVE_ALL 688 [(match_operand:<VPRED> 1 "register_operand" "Upl") 689 (match_operand:TI 2 "aarch64_sve_ld1r_operand" "Uty")] 690 UNSPEC_LD1RQ))] 691 "TARGET_SVE" 692 "ld1rq<Vesize>\t%0.<Vetype>, %1/z, %2" 693 ) 694 695 ;; Implement a predicate broadcast by shifting the low bit of the scalar 696 ;; input into the top bit and using a WHILELO. An alternative would be to 697 ;; duplicate the input and do a compare with zero. 698 (define_expand "vec_duplicate<mode>" 699 [(set (match_operand:PRED_ALL 0 "register_operand") 700 (vec_duplicate:PRED_ALL (match_operand 1 "register_operand")))] 701 "TARGET_SVE" 702 { 703 rtx tmp = gen_reg_rtx (DImode); 704 rtx op1 = gen_lowpart (DImode, operands[1]); 705 emit_insn (gen_ashldi3 (tmp, op1, gen_int_mode (63, DImode))); 706 emit_insn (gen_while_ultdi<mode> (operands[0], const0_rtx, tmp)); 707 DONE; 708 } 709 ) 710 711 (define_insn "vec_series<mode>" 712 [(set (match_operand:SVE_I 0 "register_operand" "=w, w, w") 713 (vec_series:SVE_I 714 (match_operand:<VEL> 1 "aarch64_sve_index_operand" "Usi, r, r") 715 (match_operand:<VEL> 2 "aarch64_sve_index_operand" "r, Usi, r")))] 716 "TARGET_SVE" 717 "@ 718 index\t%0.<Vetype>, #%1, %<vw>2 719 index\t%0.<Vetype>, %<vw>1, #%2 720 index\t%0.<Vetype>, %<vw>1, %<vw>2" 721 ) 722 723 ;; Optimize {x, x, x, x, ...} + {0, n, 2*n, 3*n, ...} if n is in range 724 ;; of an INDEX instruction. 725 (define_insn "*vec_series<mode>_plus" 726 [(set (match_operand:SVE_I 0 "register_operand" "=w") 727 (plus:SVE_I 728 (vec_duplicate:SVE_I 729 (match_operand:<VEL> 1 "register_operand" "r")) 730 (match_operand:SVE_I 2 "immediate_operand")))] 731 "TARGET_SVE && aarch64_check_zero_based_sve_index_immediate (operands[2])" 732 { 733 operands[2] = aarch64_check_zero_based_sve_index_immediate (operands[2]); 734 return "index\t%0.<Vetype>, %<vw>1, #%2"; 735 } 736 ) 737 738 ;; Unpredicated LD[234]. 739 (define_expand "vec_load_lanes<mode><vsingle>" 740 [(set (match_operand:SVE_STRUCT 0 "register_operand") 741 (unspec:SVE_STRUCT 742 [(match_dup 2) 743 (match_operand:SVE_STRUCT 1 "memory_operand")] 744 UNSPEC_LDN))] 745 "TARGET_SVE" 746 { 747 operands[2] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode)); 748 } 749 ) 750 751 ;; Predicated LD[234]. 752 (define_insn "vec_mask_load_lanes<mode><vsingle>" 753 [(set (match_operand:SVE_STRUCT 0 "register_operand" "=w") 754 (unspec:SVE_STRUCT 755 [(match_operand:<VPRED> 2 "register_operand" "Upl") 756 (match_operand:SVE_STRUCT 1 "memory_operand" "m")] 757 UNSPEC_LDN))] 758 "TARGET_SVE" 759 "ld<vector_count><Vesize>\t%0, %2/z, %1" 760 ) 761 762 ;; Unpredicated ST[234]. This is always a full update, so the dependence 763 ;; on the old value of the memory location (via (match_dup 0)) is redundant. 764 ;; There doesn't seem to be any obvious benefit to treating the all-true 765 ;; case differently though. In particular, it's very unlikely that we'll 766 ;; only find out during RTL that a store_lanes is dead. 767 (define_expand "vec_store_lanes<mode><vsingle>" 768 [(set (match_operand:SVE_STRUCT 0 "memory_operand") 769 (unspec:SVE_STRUCT 770 [(match_dup 2) 771 (match_operand:SVE_STRUCT 1 "register_operand") 772 (match_dup 0)] 773 UNSPEC_STN))] 774 "TARGET_SVE" 775 { 776 operands[2] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode)); 777 } 778 ) 779 780 ;; Predicated ST[234]. 781 (define_insn "vec_mask_store_lanes<mode><vsingle>" 782 [(set (match_operand:SVE_STRUCT 0 "memory_operand" "+m") 783 (unspec:SVE_STRUCT 784 [(match_operand:<VPRED> 2 "register_operand" "Upl") 785 (match_operand:SVE_STRUCT 1 "register_operand" "w") 786 (match_dup 0)] 787 UNSPEC_STN))] 788 "TARGET_SVE" 789 "st<vector_count><Vesize>\t%1, %2, %0" 790 ) 791 792 (define_expand "vec_perm<mode>" 793 [(match_operand:SVE_ALL 0 "register_operand") 794 (match_operand:SVE_ALL 1 "register_operand") 795 (match_operand:SVE_ALL 2 "register_operand") 796 (match_operand:<V_INT_EQUIV> 3 "aarch64_sve_vec_perm_operand")] 797 "TARGET_SVE && GET_MODE_NUNITS (<MODE>mode).is_constant ()" 798 { 799 aarch64_expand_sve_vec_perm (operands[0], operands[1], 800 operands[2], operands[3]); 801 DONE; 802 } 803 ) 804 805 (define_insn "*aarch64_sve_tbl<mode>" 806 [(set (match_operand:SVE_ALL 0 "register_operand" "=w") 807 (unspec:SVE_ALL 808 [(match_operand:SVE_ALL 1 "register_operand" "w") 809 (match_operand:<V_INT_EQUIV> 2 "register_operand" "w")] 810 UNSPEC_TBL))] 811 "TARGET_SVE" 812 "tbl\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>" 813 ) 814 815 (define_insn "*aarch64_sve_<perm_insn><perm_hilo><mode>" 816 [(set (match_operand:PRED_ALL 0 "register_operand" "=Upa") 817 (unspec:PRED_ALL [(match_operand:PRED_ALL 1 "register_operand" "Upa") 818 (match_operand:PRED_ALL 2 "register_operand" "Upa")] 819 PERMUTE))] 820 "TARGET_SVE" 821 "<perm_insn><perm_hilo>\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>" 822 ) 823 824 (define_insn "aarch64_sve_<perm_insn><perm_hilo><mode>" 825 [(set (match_operand:SVE_ALL 0 "register_operand" "=w") 826 (unspec:SVE_ALL [(match_operand:SVE_ALL 1 "register_operand" "w") 827 (match_operand:SVE_ALL 2 "register_operand" "w")] 828 PERMUTE))] 829 "TARGET_SVE" 830 "<perm_insn><perm_hilo>\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>" 831 ) 832 833 (define_insn "*aarch64_sve_rev64<mode>" 834 [(set (match_operand:SVE_BHS 0 "register_operand" "=w") 835 (unspec:SVE_BHS 836 [(match_operand:VNx2BI 1 "register_operand" "Upl") 837 (unspec:SVE_BHS [(match_operand:SVE_BHS 2 "register_operand" "w")] 838 UNSPEC_REV64)] 839 UNSPEC_MERGE_PTRUE))] 840 "TARGET_SVE" 841 "rev<Vesize>\t%0.d, %1/m, %2.d" 842 ) 843 844 (define_insn "*aarch64_sve_rev32<mode>" 845 [(set (match_operand:SVE_BH 0 "register_operand" "=w") 846 (unspec:SVE_BH 847 [(match_operand:VNx4BI 1 "register_operand" "Upl") 848 (unspec:SVE_BH [(match_operand:SVE_BH 2 "register_operand" "w")] 849 UNSPEC_REV32)] 850 UNSPEC_MERGE_PTRUE))] 851 "TARGET_SVE" 852 "rev<Vesize>\t%0.s, %1/m, %2.s" 853 ) 854 855 (define_insn "*aarch64_sve_rev16vnx16qi" 856 [(set (match_operand:VNx16QI 0 "register_operand" "=w") 857 (unspec:VNx16QI 858 [(match_operand:VNx8BI 1 "register_operand" "Upl") 859 (unspec:VNx16QI [(match_operand:VNx16QI 2 "register_operand" "w")] 860 UNSPEC_REV16)] 861 UNSPEC_MERGE_PTRUE))] 862 "TARGET_SVE" 863 "revb\t%0.h, %1/m, %2.h" 864 ) 865 866 (define_insn "*aarch64_sve_rev<mode>" 867 [(set (match_operand:SVE_ALL 0 "register_operand" "=w") 868 (unspec:SVE_ALL [(match_operand:SVE_ALL 1 "register_operand" "w")] 869 UNSPEC_REV))] 870 "TARGET_SVE" 871 "rev\t%0.<Vetype>, %1.<Vetype>") 872 873 (define_insn "*aarch64_sve_dup_lane<mode>" 874 [(set (match_operand:SVE_ALL 0 "register_operand" "=w") 875 (vec_duplicate:SVE_ALL 876 (vec_select:<VEL> 877 (match_operand:SVE_ALL 1 "register_operand" "w") 878 (parallel [(match_operand:SI 2 "const_int_operand")]))))] 879 "TARGET_SVE 880 && IN_RANGE (INTVAL (operands[2]) * GET_MODE_SIZE (<VEL>mode), 0, 63)" 881 "dup\t%0.<Vetype>, %1.<Vetype>[%2]" 882 ) 883 884 ;; Note that the immediate (third) operand is the lane index not 885 ;; the byte index. 886 (define_insn "*aarch64_sve_ext<mode>" 887 [(set (match_operand:SVE_ALL 0 "register_operand" "=w") 888 (unspec:SVE_ALL [(match_operand:SVE_ALL 1 "register_operand" "0") 889 (match_operand:SVE_ALL 2 "register_operand" "w") 890 (match_operand:SI 3 "const_int_operand")] 891 UNSPEC_EXT))] 892 "TARGET_SVE 893 && IN_RANGE (INTVAL (operands[3]) * GET_MODE_SIZE (<VEL>mode), 0, 255)" 894 { 895 operands[3] = GEN_INT (INTVAL (operands[3]) * GET_MODE_SIZE (<VEL>mode)); 896 return "ext\\t%0.b, %0.b, %2.b, #%3"; 897 } 898 ) 899 900 (define_insn "add<mode>3" 901 [(set (match_operand:SVE_I 0 "register_operand" "=w, w, w, w") 902 (plus:SVE_I 903 (match_operand:SVE_I 1 "register_operand" "%0, 0, 0, w") 904 (match_operand:SVE_I 2 "aarch64_sve_add_operand" "vsa, vsn, vsi, w")))] 905 "TARGET_SVE" 906 "@ 907 add\t%0.<Vetype>, %0.<Vetype>, #%D2 908 sub\t%0.<Vetype>, %0.<Vetype>, #%N2 909 * return aarch64_output_sve_inc_dec_immediate (\"%0.<Vetype>\", operands[2]); 910 add\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>" 911 ) 912 913 (define_insn "sub<mode>3" 914 [(set (match_operand:SVE_I 0 "register_operand" "=w, w") 915 (minus:SVE_I 916 (match_operand:SVE_I 1 "aarch64_sve_arith_operand" "w, vsa") 917 (match_operand:SVE_I 2 "register_operand" "w, 0")))] 918 "TARGET_SVE" 919 "@ 920 sub\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype> 921 subr\t%0.<Vetype>, %0.<Vetype>, #%D1" 922 ) 923 924 ;; Unpredicated multiplication. 925 (define_expand "mul<mode>3" 926 [(set (match_operand:SVE_I 0 "register_operand") 927 (unspec:SVE_I 928 [(match_dup 3) 929 (mult:SVE_I 930 (match_operand:SVE_I 1 "register_operand") 931 (match_operand:SVE_I 2 "aarch64_sve_mul_operand"))] 932 UNSPEC_MERGE_PTRUE))] 933 "TARGET_SVE" 934 { 935 operands[3] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode)); 936 } 937 ) 938 939 ;; Multiplication predicated with a PTRUE. We don't actually need the 940 ;; predicate for the first alternative, but using Upa or X isn't likely 941 ;; to gain much and would make the instruction seem less uniform to the 942 ;; register allocator. 943 (define_insn_and_split "*mul<mode>3" 944 [(set (match_operand:SVE_I 0 "register_operand" "=w, w, ?&w") 945 (unspec:SVE_I 946 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl") 947 (mult:SVE_I 948 (match_operand:SVE_I 2 "register_operand" "%0, 0, w") 949 (match_operand:SVE_I 3 "aarch64_sve_mul_operand" "vsm, w, w"))] 950 UNSPEC_MERGE_PTRUE))] 951 "TARGET_SVE" 952 "@ 953 # 954 mul\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype> 955 movprfx\t%0, %2\;mul\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>" 956 ; Split the unpredicated form after reload, so that we don't have 957 ; the unnecessary PTRUE. 958 "&& reload_completed 959 && !register_operand (operands[3], <MODE>mode)" 960 [(set (match_dup 0) (mult:SVE_I (match_dup 2) (match_dup 3)))] 961 "" 962 [(set_attr "movprfx" "*,*,yes")] 963 ) 964 965 ;; Unpredicated multiplications by a constant (post-RA only). 966 ;; These are generated by splitting a predicated instruction whose 967 ;; predicate is unused. 968 (define_insn "*post_ra_mul<mode>3" 969 [(set (match_operand:SVE_I 0 "register_operand" "=w") 970 (mult:SVE_I 971 (match_operand:SVE_I 1 "register_operand" "0") 972 (match_operand:SVE_I 2 "aarch64_sve_mul_immediate")))] 973 "TARGET_SVE && reload_completed" 974 "mul\t%0.<Vetype>, %0.<Vetype>, #%2" 975 ) 976 977 (define_insn "*madd<mode>" 978 [(set (match_operand:SVE_I 0 "register_operand" "=w, w, ?&w") 979 (plus:SVE_I 980 (unspec:SVE_I 981 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl") 982 (mult:SVE_I (match_operand:SVE_I 2 "register_operand" "%0, w, w") 983 (match_operand:SVE_I 3 "register_operand" "w, w, w"))] 984 UNSPEC_MERGE_PTRUE) 985 (match_operand:SVE_I 4 "register_operand" "w, 0, w")))] 986 "TARGET_SVE" 987 "@ 988 mad\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype> 989 mla\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype> 990 movprfx\t%0, %4\;mla\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>" 991 [(set_attr "movprfx" "*,*,yes")] 992 ) 993 994 (define_insn "*msub<mode>3" 995 [(set (match_operand:SVE_I 0 "register_operand" "=w, w, ?&w") 996 (minus:SVE_I 997 (match_operand:SVE_I 4 "register_operand" "w, 0, w") 998 (unspec:SVE_I 999 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl") 1000 (mult:SVE_I (match_operand:SVE_I 2 "register_operand" "%0, w, w") 1001 (match_operand:SVE_I 3 "register_operand" "w, w, w"))] 1002 UNSPEC_MERGE_PTRUE)))] 1003 "TARGET_SVE" 1004 "@ 1005 msb\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype> 1006 mls\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype> 1007 movprfx\t%0, %4\;mls\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>" 1008 [(set_attr "movprfx" "*,*,yes")] 1009 ) 1010 1011 ;; Unpredicated highpart multiplication. 1012 (define_expand "<su>mul<mode>3_highpart" 1013 [(set (match_operand:SVE_I 0 "register_operand") 1014 (unspec:SVE_I 1015 [(match_dup 3) 1016 (unspec:SVE_I [(match_operand:SVE_I 1 "register_operand") 1017 (match_operand:SVE_I 2 "register_operand")] 1018 MUL_HIGHPART)] 1019 UNSPEC_MERGE_PTRUE))] 1020 "TARGET_SVE" 1021 { 1022 operands[3] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode)); 1023 } 1024 ) 1025 1026 ;; Predicated highpart multiplication. 1027 (define_insn "*<su>mul<mode>3_highpart" 1028 [(set (match_operand:SVE_I 0 "register_operand" "=w, ?&w") 1029 (unspec:SVE_I 1030 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl") 1031 (unspec:SVE_I [(match_operand:SVE_I 2 "register_operand" "%0, w") 1032 (match_operand:SVE_I 3 "register_operand" "w, w")] 1033 MUL_HIGHPART)] 1034 UNSPEC_MERGE_PTRUE))] 1035 "TARGET_SVE" 1036 "@ 1037 <su>mulh\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype> 1038 movprfx\t%0, %2\;<su>mulh\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>" 1039 [(set_attr "movprfx" "*,yes")] 1040 ) 1041 1042 ;; Unpredicated division. 1043 (define_expand "<optab><mode>3" 1044 [(set (match_operand:SVE_SDI 0 "register_operand") 1045 (unspec:SVE_SDI 1046 [(match_dup 3) 1047 (SVE_INT_BINARY_SD:SVE_SDI 1048 (match_operand:SVE_SDI 1 "register_operand") 1049 (match_operand:SVE_SDI 2 "register_operand"))] 1050 UNSPEC_MERGE_PTRUE))] 1051 "TARGET_SVE" 1052 { 1053 operands[3] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode)); 1054 } 1055 ) 1056 1057 ;; Division predicated with a PTRUE. 1058 (define_insn "*<optab><mode>3" 1059 [(set (match_operand:SVE_SDI 0 "register_operand" "=w, w, ?&w") 1060 (unspec:SVE_SDI 1061 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl") 1062 (SVE_INT_BINARY_SD:SVE_SDI 1063 (match_operand:SVE_SDI 2 "register_operand" "0, w, w") 1064 (match_operand:SVE_SDI 3 "aarch64_sve_mul_operand" "w, 0, w"))] 1065 UNSPEC_MERGE_PTRUE))] 1066 "TARGET_SVE" 1067 "@ 1068 <sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype> 1069 <sve_int_op>r\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype> 1070 movprfx\t%0, %2\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>" 1071 [(set_attr "movprfx" "*,*,yes")] 1072 ) 1073 1074 ;; Unpredicated NEG, NOT and POPCOUNT. 1075 (define_expand "<optab><mode>2" 1076 [(set (match_operand:SVE_I 0 "register_operand") 1077 (unspec:SVE_I 1078 [(match_dup 2) 1079 (SVE_INT_UNARY:SVE_I (match_operand:SVE_I 1 "register_operand"))] 1080 UNSPEC_MERGE_PTRUE))] 1081 "TARGET_SVE" 1082 { 1083 operands[2] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode)); 1084 } 1085 ) 1086 1087 ;; NEG, NOT and POPCOUNT predicated with a PTRUE. 1088 (define_insn "*<optab><mode>2" 1089 [(set (match_operand:SVE_I 0 "register_operand" "=w") 1090 (unspec:SVE_I 1091 [(match_operand:<VPRED> 1 "register_operand" "Upl") 1092 (SVE_INT_UNARY:SVE_I 1093 (match_operand:SVE_I 2 "register_operand" "w"))] 1094 UNSPEC_MERGE_PTRUE))] 1095 "TARGET_SVE" 1096 "<sve_int_op>\t%0.<Vetype>, %1/m, %2.<Vetype>" 1097 ) 1098 1099 ;; Vector AND, ORR and XOR. 1100 (define_insn "<optab><mode>3" 1101 [(set (match_operand:SVE_I 0 "register_operand" "=w, w") 1102 (LOGICAL:SVE_I 1103 (match_operand:SVE_I 1 "register_operand" "%0, w") 1104 (match_operand:SVE_I 2 "aarch64_sve_logical_operand" "vsl, w")))] 1105 "TARGET_SVE" 1106 "@ 1107 <logical>\t%0.<Vetype>, %0.<Vetype>, #%C2 1108 <logical>\t%0.d, %1.d, %2.d" 1109 ) 1110 1111 ;; Vector AND, ORR and XOR on floating-point modes. We avoid subregs 1112 ;; by providing this, but we need to use UNSPECs since rtx logical ops 1113 ;; aren't defined for floating-point modes. 1114 (define_insn "*<optab><mode>3" 1115 [(set (match_operand:SVE_F 0 "register_operand" "=w") 1116 (unspec:SVE_F [(match_operand:SVE_F 1 "register_operand" "w") 1117 (match_operand:SVE_F 2 "register_operand" "w")] 1118 LOGICALF))] 1119 "TARGET_SVE" 1120 "<logicalf_op>\t%0.d, %1.d, %2.d" 1121 ) 1122 1123 ;; REG_EQUAL notes on "not<mode>3" should ensure that we can generate 1124 ;; this pattern even though the NOT instruction itself is predicated. 1125 (define_insn "bic<mode>3" 1126 [(set (match_operand:SVE_I 0 "register_operand" "=w") 1127 (and:SVE_I 1128 (not:SVE_I (match_operand:SVE_I 1 "register_operand" "w")) 1129 (match_operand:SVE_I 2 "register_operand" "w")))] 1130 "TARGET_SVE" 1131 "bic\t%0.d, %2.d, %1.d" 1132 ) 1133 1134 ;; Predicate AND. We can reuse one of the inputs as the GP. 1135 (define_insn "and<mode>3" 1136 [(set (match_operand:PRED_ALL 0 "register_operand" "=Upa") 1137 (and:PRED_ALL (match_operand:PRED_ALL 1 "register_operand" "Upa") 1138 (match_operand:PRED_ALL 2 "register_operand" "Upa")))] 1139 "TARGET_SVE" 1140 "and\t%0.b, %1/z, %1.b, %2.b" 1141 ) 1142 1143 ;; Unpredicated predicate ORR and XOR. 1144 (define_expand "<optab><mode>3" 1145 [(set (match_operand:PRED_ALL 0 "register_operand") 1146 (and:PRED_ALL 1147 (LOGICAL_OR:PRED_ALL 1148 (match_operand:PRED_ALL 1 "register_operand") 1149 (match_operand:PRED_ALL 2 "register_operand")) 1150 (match_dup 3)))] 1151 "TARGET_SVE" 1152 { 1153 operands[3] = force_reg (<MODE>mode, CONSTM1_RTX (<MODE>mode)); 1154 } 1155 ) 1156 1157 ;; Predicated predicate ORR and XOR. 1158 (define_insn "pred_<optab><mode>3" 1159 [(set (match_operand:PRED_ALL 0 "register_operand" "=Upa") 1160 (and:PRED_ALL 1161 (LOGICAL:PRED_ALL 1162 (match_operand:PRED_ALL 2 "register_operand" "Upa") 1163 (match_operand:PRED_ALL 3 "register_operand" "Upa")) 1164 (match_operand:PRED_ALL 1 "register_operand" "Upa")))] 1165 "TARGET_SVE" 1166 "<logical>\t%0.b, %1/z, %2.b, %3.b" 1167 ) 1168 1169 ;; Perform a logical operation on operands 2 and 3, using operand 1 as 1170 ;; the GP (which is known to be a PTRUE). Store the result in operand 0 1171 ;; and set the flags in the same way as for PTEST. The (and ...) in the 1172 ;; UNSPEC_PTEST_PTRUE is logically redundant, but means that the tested 1173 ;; value is structurally equivalent to rhs of the second set. 1174 (define_insn "*<optab><mode>3_cc" 1175 [(set (reg:CC CC_REGNUM) 1176 (compare:CC 1177 (unspec:SI [(match_operand:PRED_ALL 1 "register_operand" "Upa") 1178 (and:PRED_ALL 1179 (LOGICAL:PRED_ALL 1180 (match_operand:PRED_ALL 2 "register_operand" "Upa") 1181 (match_operand:PRED_ALL 3 "register_operand" "Upa")) 1182 (match_dup 1))] 1183 UNSPEC_PTEST_PTRUE) 1184 (const_int 0))) 1185 (set (match_operand:PRED_ALL 0 "register_operand" "=Upa") 1186 (and:PRED_ALL (LOGICAL:PRED_ALL (match_dup 2) (match_dup 3)) 1187 (match_dup 1)))] 1188 "TARGET_SVE" 1189 "<logical>s\t%0.b, %1/z, %2.b, %3.b" 1190 ) 1191 1192 ;; Unpredicated predicate inverse. 1193 (define_expand "one_cmpl<mode>2" 1194 [(set (match_operand:PRED_ALL 0 "register_operand") 1195 (and:PRED_ALL 1196 (not:PRED_ALL (match_operand:PRED_ALL 1 "register_operand")) 1197 (match_dup 2)))] 1198 "TARGET_SVE" 1199 { 1200 operands[2] = force_reg (<MODE>mode, CONSTM1_RTX (<MODE>mode)); 1201 } 1202 ) 1203 1204 ;; Predicated predicate inverse. 1205 (define_insn "*one_cmpl<mode>3" 1206 [(set (match_operand:PRED_ALL 0 "register_operand" "=Upa") 1207 (and:PRED_ALL 1208 (not:PRED_ALL (match_operand:PRED_ALL 2 "register_operand" "Upa")) 1209 (match_operand:PRED_ALL 1 "register_operand" "Upa")))] 1210 "TARGET_SVE" 1211 "not\t%0.b, %1/z, %2.b" 1212 ) 1213 1214 ;; Predicated predicate BIC and ORN. 1215 (define_insn "*<nlogical><mode>3" 1216 [(set (match_operand:PRED_ALL 0 "register_operand" "=Upa") 1217 (and:PRED_ALL 1218 (NLOGICAL:PRED_ALL 1219 (not:PRED_ALL (match_operand:PRED_ALL 2 "register_operand" "Upa")) 1220 (match_operand:PRED_ALL 3 "register_operand" "Upa")) 1221 (match_operand:PRED_ALL 1 "register_operand" "Upa")))] 1222 "TARGET_SVE" 1223 "<nlogical>\t%0.b, %1/z, %3.b, %2.b" 1224 ) 1225 1226 ;; Predicated predicate NAND and NOR. 1227 (define_insn "*<logical_nn><mode>3" 1228 [(set (match_operand:PRED_ALL 0 "register_operand" "=Upa") 1229 (and:PRED_ALL 1230 (NLOGICAL:PRED_ALL 1231 (not:PRED_ALL (match_operand:PRED_ALL 2 "register_operand" "Upa")) 1232 (not:PRED_ALL (match_operand:PRED_ALL 3 "register_operand" "Upa"))) 1233 (match_operand:PRED_ALL 1 "register_operand" "Upa")))] 1234 "TARGET_SVE" 1235 "<logical_nn>\t%0.b, %1/z, %2.b, %3.b" 1236 ) 1237 1238 ;; Unpredicated LSL, LSR and ASR by a vector. 1239 (define_expand "v<optab><mode>3" 1240 [(set (match_operand:SVE_I 0 "register_operand") 1241 (unspec:SVE_I 1242 [(match_dup 3) 1243 (ASHIFT:SVE_I 1244 (match_operand:SVE_I 1 "register_operand") 1245 (match_operand:SVE_I 2 "aarch64_sve_<lr>shift_operand"))] 1246 UNSPEC_MERGE_PTRUE))] 1247 "TARGET_SVE" 1248 { 1249 operands[3] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode)); 1250 } 1251 ) 1252 1253 ;; LSL, LSR and ASR by a vector, predicated with a PTRUE. We don't 1254 ;; actually need the predicate for the first alternative, but using Upa 1255 ;; or X isn't likely to gain much and would make the instruction seem 1256 ;; less uniform to the register allocator. 1257 (define_insn_and_split "*v<optab><mode>3" 1258 [(set (match_operand:SVE_I 0 "register_operand" "=w, w, ?&w") 1259 (unspec:SVE_I 1260 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl") 1261 (ASHIFT:SVE_I 1262 (match_operand:SVE_I 2 "register_operand" "w, 0, w") 1263 (match_operand:SVE_I 3 "aarch64_sve_<lr>shift_operand" "D<lr>, w, w"))] 1264 UNSPEC_MERGE_PTRUE))] 1265 "TARGET_SVE" 1266 "@ 1267 # 1268 <shift>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype> 1269 movprfx\t%0, %2\;<shift>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>" 1270 "&& reload_completed 1271 && !register_operand (operands[3], <MODE>mode)" 1272 [(set (match_dup 0) (ASHIFT:SVE_I (match_dup 2) (match_dup 3)))] 1273 "" 1274 [(set_attr "movprfx" "*,*,yes")] 1275 ) 1276 1277 ;; Unpredicated shift operations by a constant (post-RA only). 1278 ;; These are generated by splitting a predicated instruction whose 1279 ;; predicate is unused. 1280 (define_insn "*post_ra_v<optab><mode>3" 1281 [(set (match_operand:SVE_I 0 "register_operand" "=w") 1282 (ASHIFT:SVE_I 1283 (match_operand:SVE_I 1 "register_operand" "w") 1284 (match_operand:SVE_I 2 "aarch64_simd_<lr>shift_imm")))] 1285 "TARGET_SVE && reload_completed" 1286 "<shift>\t%0.<Vetype>, %1.<Vetype>, #%2" 1287 ) 1288 1289 ;; LSL, LSR and ASR by a scalar, which expands into one of the vector 1290 ;; shifts above. 1291 (define_expand "<ASHIFT:optab><mode>3" 1292 [(set (match_operand:SVE_I 0 "register_operand") 1293 (ASHIFT:SVE_I (match_operand:SVE_I 1 "register_operand") 1294 (match_operand:<VEL> 2 "general_operand")))] 1295 "TARGET_SVE" 1296 { 1297 rtx amount; 1298 if (CONST_INT_P (operands[2])) 1299 { 1300 amount = gen_const_vec_duplicate (<MODE>mode, operands[2]); 1301 if (!aarch64_sve_<lr>shift_operand (operands[2], <MODE>mode)) 1302 amount = force_reg (<MODE>mode, amount); 1303 } 1304 else 1305 { 1306 amount = gen_reg_rtx (<MODE>mode); 1307 emit_insn (gen_vec_duplicate<mode> (amount, 1308 convert_to_mode (<VEL>mode, 1309 operands[2], 0))); 1310 } 1311 emit_insn (gen_v<optab><mode>3 (operands[0], operands[1], amount)); 1312 DONE; 1313 } 1314 ) 1315 1316 ;; Test all bits of operand 1. Operand 0 is a GP that is known to hold PTRUE. 1317 ;; 1318 ;; Using UNSPEC_PTEST_PTRUE allows combine patterns to assume that the GP 1319 ;; is a PTRUE even if the optimizers haven't yet been able to propagate 1320 ;; the constant. We would use a separate unspec code for PTESTs involving 1321 ;; GPs that might not be PTRUEs. 1322 (define_insn "ptest_ptrue<mode>" 1323 [(set (reg:CC CC_REGNUM) 1324 (compare:CC 1325 (unspec:SI [(match_operand:PRED_ALL 0 "register_operand" "Upa") 1326 (match_operand:PRED_ALL 1 "register_operand" "Upa")] 1327 UNSPEC_PTEST_PTRUE) 1328 (const_int 0)))] 1329 "TARGET_SVE" 1330 "ptest\t%0, %1.b" 1331 ) 1332 1333 ;; Set element I of the result if operand1 + J < operand2 for all J in [0, I]. 1334 ;; with the comparison being unsigned. 1335 (define_insn "while_ult<GPI:mode><PRED_ALL:mode>" 1336 [(set (match_operand:PRED_ALL 0 "register_operand" "=Upa") 1337 (unspec:PRED_ALL [(match_operand:GPI 1 "aarch64_reg_or_zero" "rZ") 1338 (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")] 1339 UNSPEC_WHILE_LO)) 1340 (clobber (reg:CC CC_REGNUM))] 1341 "TARGET_SVE" 1342 "whilelo\t%0.<PRED_ALL:Vetype>, %<w>1, %<w>2" 1343 ) 1344 1345 ;; WHILELO sets the flags in the same way as a PTEST with a PTRUE GP. 1346 ;; Handle the case in which both results are useful. The GP operand 1347 ;; to the PTEST isn't needed, so we allow it to be anything. 1348 (define_insn_and_split "while_ult<GPI:mode><PRED_ALL:mode>_cc" 1349 [(set (reg:CC CC_REGNUM) 1350 (compare:CC 1351 (unspec:SI [(match_operand:PRED_ALL 1) 1352 (unspec:PRED_ALL 1353 [(match_operand:GPI 2 "aarch64_reg_or_zero" "rZ") 1354 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")] 1355 UNSPEC_WHILE_LO)] 1356 UNSPEC_PTEST_PTRUE) 1357 (const_int 0))) 1358 (set (match_operand:PRED_ALL 0 "register_operand" "=Upa") 1359 (unspec:PRED_ALL [(match_dup 2) 1360 (match_dup 3)] 1361 UNSPEC_WHILE_LO))] 1362 "TARGET_SVE" 1363 "whilelo\t%0.<PRED_ALL:Vetype>, %<w>2, %<w>3" 1364 ;; Force the compiler to drop the unused predicate operand, so that we 1365 ;; don't have an unnecessary PTRUE. 1366 "&& !CONSTANT_P (operands[1])" 1367 [(const_int 0)] 1368 { 1369 emit_insn (gen_while_ult<GPI:mode><PRED_ALL:mode>_cc 1370 (operands[0], CONSTM1_RTX (<MODE>mode), 1371 operands[2], operands[3])); 1372 DONE; 1373 } 1374 ) 1375 1376 ;; Integer comparisons predicated with a PTRUE. 1377 (define_insn "*cmp<cmp_op><mode>" 1378 [(set (match_operand:<VPRED> 0 "register_operand" "=Upa, Upa") 1379 (unspec:<VPRED> 1380 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl") 1381 (SVE_INT_CMP:<VPRED> 1382 (match_operand:SVE_I 2 "register_operand" "w, w") 1383 (match_operand:SVE_I 3 "aarch64_sve_cmp_<sve_imm_con>_operand" "<sve_imm_con>, w"))] 1384 UNSPEC_MERGE_PTRUE)) 1385 (clobber (reg:CC CC_REGNUM))] 1386 "TARGET_SVE" 1387 "@ 1388 cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, #%3 1389 cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>" 1390 ) 1391 1392 ;; Integer comparisons predicated with a PTRUE in which only the flags result 1393 ;; is interesting. 1394 (define_insn "*cmp<cmp_op><mode>_ptest" 1395 [(set (reg:CC CC_REGNUM) 1396 (compare:CC 1397 (unspec:SI 1398 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl") 1399 (unspec:<VPRED> 1400 [(match_dup 1) 1401 (SVE_INT_CMP:<VPRED> 1402 (match_operand:SVE_I 2 "register_operand" "w, w") 1403 (match_operand:SVE_I 3 "aarch64_sve_cmp_<sve_imm_con>_operand" "<sve_imm_con>, w"))] 1404 UNSPEC_MERGE_PTRUE)] 1405 UNSPEC_PTEST_PTRUE) 1406 (const_int 0))) 1407 (clobber (match_scratch:<VPRED> 0 "=Upa, Upa"))] 1408 "TARGET_SVE" 1409 "@ 1410 cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, #%3 1411 cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>" 1412 ) 1413 1414 ;; Integer comparisons predicated with a PTRUE in which both the flag and 1415 ;; predicate results are interesting. 1416 (define_insn "*cmp<cmp_op><mode>_cc" 1417 [(set (reg:CC CC_REGNUM) 1418 (compare:CC 1419 (unspec:SI 1420 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl") 1421 (unspec:<VPRED> 1422 [(match_dup 1) 1423 (SVE_INT_CMP:<VPRED> 1424 (match_operand:SVE_I 2 "register_operand" "w, w") 1425 (match_operand:SVE_I 3 "aarch64_sve_cmp_<sve_imm_con>_operand" "<sve_imm_con>, w"))] 1426 UNSPEC_MERGE_PTRUE)] 1427 UNSPEC_PTEST_PTRUE) 1428 (const_int 0))) 1429 (set (match_operand:<VPRED> 0 "register_operand" "=Upa, Upa") 1430 (unspec:<VPRED> 1431 [(match_dup 1) 1432 (SVE_INT_CMP:<VPRED> 1433 (match_dup 2) 1434 (match_dup 3))] 1435 UNSPEC_MERGE_PTRUE))] 1436 "TARGET_SVE" 1437 "@ 1438 cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, #%3 1439 cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>" 1440 ) 1441 1442 ;; Predicated integer comparisons, formed by combining a PTRUE-predicated 1443 ;; comparison with an AND. Split the instruction into its preferred form 1444 ;; (below) at the earliest opportunity, in order to get rid of the 1445 ;; redundant operand 1. 1446 (define_insn_and_split "*pred_cmp<cmp_op><mode>_combine" 1447 [(set (match_operand:<VPRED> 0 "register_operand" "=Upa, Upa") 1448 (and:<VPRED> 1449 (unspec:<VPRED> 1450 [(match_operand:<VPRED> 1) 1451 (SVE_INT_CMP:<VPRED> 1452 (match_operand:SVE_I 2 "register_operand" "w, w") 1453 (match_operand:SVE_I 3 "aarch64_sve_cmp_<sve_imm_con>_operand" "<sve_imm_con>, w"))] 1454 UNSPEC_MERGE_PTRUE) 1455 (match_operand:<VPRED> 4 "register_operand" "Upl, Upl"))) 1456 (clobber (reg:CC CC_REGNUM))] 1457 "TARGET_SVE" 1458 "#" 1459 "&& 1" 1460 [(parallel 1461 [(set (match_dup 0) 1462 (and:<VPRED> 1463 (SVE_INT_CMP:<VPRED> 1464 (match_dup 2) 1465 (match_dup 3)) 1466 (match_dup 4))) 1467 (clobber (reg:CC CC_REGNUM))])] 1468 ) 1469 1470 ;; Predicated integer comparisons. 1471 (define_insn "*pred_cmp<cmp_op><mode>" 1472 [(set (match_operand:<VPRED> 0 "register_operand" "=Upa, Upa") 1473 (and:<VPRED> 1474 (SVE_INT_CMP:<VPRED> 1475 (match_operand:SVE_I 2 "register_operand" "w, w") 1476 (match_operand:SVE_I 3 "aarch64_sve_cmp_<sve_imm_con>_operand" "<sve_imm_con>, w")) 1477 (match_operand:<VPRED> 1 "register_operand" "Upl, Upl"))) 1478 (clobber (reg:CC CC_REGNUM))] 1479 "TARGET_SVE" 1480 "@ 1481 cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, #%3 1482 cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>" 1483 ) 1484 1485 ;; Floating-point comparisons predicated with a PTRUE. 1486 (define_insn "*fcm<cmp_op><mode>" 1487 [(set (match_operand:<VPRED> 0 "register_operand" "=Upa, Upa") 1488 (unspec:<VPRED> 1489 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl") 1490 (SVE_FP_CMP:<VPRED> 1491 (match_operand:SVE_F 2 "register_operand" "w, w") 1492 (match_operand:SVE_F 3 "aarch64_simd_reg_or_zero" "Dz, w"))] 1493 UNSPEC_MERGE_PTRUE))] 1494 "TARGET_SVE" 1495 "@ 1496 fcm<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, #0.0 1497 fcm<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>" 1498 ) 1499 1500 (define_insn "*fcmuo<mode>" 1501 [(set (match_operand:<VPRED> 0 "register_operand" "=Upa") 1502 (unspec:<VPRED> 1503 [(match_operand:<VPRED> 1 "register_operand" "Upl") 1504 (unordered:<VPRED> 1505 (match_operand:SVE_F 2 "register_operand" "w") 1506 (match_operand:SVE_F 3 "register_operand" "w"))] 1507 UNSPEC_MERGE_PTRUE))] 1508 "TARGET_SVE" 1509 "fcmuo\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>" 1510 ) 1511 1512 ;; Floating-point comparisons predicated on a PTRUE, with the results ANDed 1513 ;; with another predicate P. This does not have the same trapping behavior 1514 ;; as predicating the comparison itself on P, but it's a legitimate fold, 1515 ;; since we can drop any potentially-trapping operations whose results 1516 ;; are not needed. 1517 ;; 1518 ;; Split the instruction into its preferred form (below) at the earliest 1519 ;; opportunity, in order to get rid of the redundant operand 1. 1520 (define_insn_and_split "*fcm<cmp_op><mode>_and_combine" 1521 [(set (match_operand:<VPRED> 0 "register_operand" "=Upa, Upa") 1522 (and:<VPRED> 1523 (unspec:<VPRED> 1524 [(match_operand:<VPRED> 1) 1525 (SVE_FP_CMP 1526 (match_operand:SVE_F 2 "register_operand" "w, w") 1527 (match_operand:SVE_F 3 "aarch64_simd_reg_or_zero" "Dz, w"))] 1528 UNSPEC_MERGE_PTRUE) 1529 (match_operand:<VPRED> 4 "register_operand" "Upl, Upl")))] 1530 "TARGET_SVE" 1531 "#" 1532 "&& 1" 1533 [(set (match_dup 0) 1534 (and:<VPRED> 1535 (SVE_FP_CMP:<VPRED> 1536 (match_dup 2) 1537 (match_dup 3)) 1538 (match_dup 4)))] 1539 ) 1540 1541 (define_insn_and_split "*fcmuo<mode>_and_combine" 1542 [(set (match_operand:<VPRED> 0 "register_operand" "=Upa") 1543 (and:<VPRED> 1544 (unspec:<VPRED> 1545 [(match_operand:<VPRED> 1) 1546 (unordered 1547 (match_operand:SVE_F 2 "register_operand" "w") 1548 (match_operand:SVE_F 3 "register_operand" "w"))] 1549 UNSPEC_MERGE_PTRUE) 1550 (match_operand:<VPRED> 4 "register_operand" "Upl")))] 1551 "TARGET_SVE" 1552 "#" 1553 "&& 1" 1554 [(set (match_dup 0) 1555 (and:<VPRED> 1556 (unordered:<VPRED> 1557 (match_dup 2) 1558 (match_dup 3)) 1559 (match_dup 4)))] 1560 ) 1561 1562 ;; Unpredicated floating-point comparisons, with the results ANDed 1563 ;; with another predicate. This is a valid fold for the same reasons 1564 ;; as above. 1565 (define_insn "*fcm<cmp_op><mode>_and" 1566 [(set (match_operand:<VPRED> 0 "register_operand" "=Upa, Upa") 1567 (and:<VPRED> 1568 (SVE_FP_CMP:<VPRED> 1569 (match_operand:SVE_F 2 "register_operand" "w, w") 1570 (match_operand:SVE_F 3 "aarch64_simd_reg_or_zero" "Dz, w")) 1571 (match_operand:<VPRED> 1 "register_operand" "Upl, Upl")))] 1572 "TARGET_SVE" 1573 "@ 1574 fcm<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, #0.0 1575 fcm<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>" 1576 ) 1577 1578 (define_insn "*fcmuo<mode>_and" 1579 [(set (match_operand:<VPRED> 0 "register_operand" "=Upa") 1580 (and:<VPRED> 1581 (unordered:<VPRED> 1582 (match_operand:SVE_F 2 "register_operand" "w") 1583 (match_operand:SVE_F 3 "register_operand" "w")) 1584 (match_operand:<VPRED> 1 "register_operand" "Upl")))] 1585 "TARGET_SVE" 1586 "fcmuo\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>" 1587 ) 1588 1589 ;; Predicated floating-point comparisons. We don't need a version 1590 ;; of this for unordered comparisons. 1591 (define_insn "*pred_fcm<cmp_op><mode>" 1592 [(set (match_operand:<VPRED> 0 "register_operand" "=Upa, Upa") 1593 (unspec:<VPRED> 1594 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl") 1595 (match_operand:SVE_F 2 "register_operand" "w, w") 1596 (match_operand:SVE_F 3 "aarch64_simd_reg_or_zero" "Dz, w")] 1597 SVE_COND_FP_CMP))] 1598 "TARGET_SVE" 1599 "@ 1600 fcm<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, #0.0 1601 fcm<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>" 1602 ) 1603 1604 ;; vcond_mask operand order: true, false, mask 1605 ;; UNSPEC_SEL operand order: mask, true, false (as for VEC_COND_EXPR) 1606 ;; SEL operand order: mask, true, false 1607 (define_insn "vcond_mask_<mode><vpred>" 1608 [(set (match_operand:SVE_ALL 0 "register_operand" "=w") 1609 (unspec:SVE_ALL 1610 [(match_operand:<VPRED> 3 "register_operand" "Upa") 1611 (match_operand:SVE_ALL 1 "register_operand" "w") 1612 (match_operand:SVE_ALL 2 "register_operand" "w")] 1613 UNSPEC_SEL))] 1614 "TARGET_SVE" 1615 "sel\t%0.<Vetype>, %3, %1.<Vetype>, %2.<Vetype>" 1616 ) 1617 1618 ;; Selects between a duplicated immediate and zero. 1619 (define_insn "aarch64_sve_dup<mode>_const" 1620 [(set (match_operand:SVE_I 0 "register_operand" "=w") 1621 (unspec:SVE_I 1622 [(match_operand:<VPRED> 1 "register_operand" "Upl") 1623 (match_operand:SVE_I 2 "aarch64_sve_dup_immediate") 1624 (match_operand:SVE_I 3 "aarch64_simd_imm_zero")] 1625 UNSPEC_SEL))] 1626 "TARGET_SVE" 1627 "mov\t%0.<Vetype>, %1/z, #%2" 1628 ) 1629 1630 ;; Integer (signed) vcond. Don't enforce an immediate range here, since it 1631 ;; depends on the comparison; leave it to aarch64_expand_sve_vcond instead. 1632 (define_expand "vcond<mode><v_int_equiv>" 1633 [(set (match_operand:SVE_ALL 0 "register_operand") 1634 (if_then_else:SVE_ALL 1635 (match_operator 3 "comparison_operator" 1636 [(match_operand:<V_INT_EQUIV> 4 "register_operand") 1637 (match_operand:<V_INT_EQUIV> 5 "nonmemory_operand")]) 1638 (match_operand:SVE_ALL 1 "register_operand") 1639 (match_operand:SVE_ALL 2 "register_operand")))] 1640 "TARGET_SVE" 1641 { 1642 aarch64_expand_sve_vcond (<MODE>mode, <V_INT_EQUIV>mode, operands); 1643 DONE; 1644 } 1645 ) 1646 1647 ;; Integer vcondu. Don't enforce an immediate range here, since it 1648 ;; depends on the comparison; leave it to aarch64_expand_sve_vcond instead. 1649 (define_expand "vcondu<mode><v_int_equiv>" 1650 [(set (match_operand:SVE_ALL 0 "register_operand") 1651 (if_then_else:SVE_ALL 1652 (match_operator 3 "comparison_operator" 1653 [(match_operand:<V_INT_EQUIV> 4 "register_operand") 1654 (match_operand:<V_INT_EQUIV> 5 "nonmemory_operand")]) 1655 (match_operand:SVE_ALL 1 "register_operand") 1656 (match_operand:SVE_ALL 2 "register_operand")))] 1657 "TARGET_SVE" 1658 { 1659 aarch64_expand_sve_vcond (<MODE>mode, <V_INT_EQUIV>mode, operands); 1660 DONE; 1661 } 1662 ) 1663 1664 ;; Floating-point vcond. All comparisons except FCMUO allow a zero 1665 ;; operand; aarch64_expand_sve_vcond handles the case of an FCMUO 1666 ;; with zero. 1667 (define_expand "vcond<mode><v_fp_equiv>" 1668 [(set (match_operand:SVE_SD 0 "register_operand") 1669 (if_then_else:SVE_SD 1670 (match_operator 3 "comparison_operator" 1671 [(match_operand:<V_FP_EQUIV> 4 "register_operand") 1672 (match_operand:<V_FP_EQUIV> 5 "aarch64_simd_reg_or_zero")]) 1673 (match_operand:SVE_SD 1 "register_operand") 1674 (match_operand:SVE_SD 2 "register_operand")))] 1675 "TARGET_SVE" 1676 { 1677 aarch64_expand_sve_vcond (<MODE>mode, <V_FP_EQUIV>mode, operands); 1678 DONE; 1679 } 1680 ) 1681 1682 ;; Signed integer comparisons. Don't enforce an immediate range here, since 1683 ;; it depends on the comparison; leave it to aarch64_expand_sve_vec_cmp_int 1684 ;; instead. 1685 (define_expand "vec_cmp<mode><vpred>" 1686 [(parallel 1687 [(set (match_operand:<VPRED> 0 "register_operand") 1688 (match_operator:<VPRED> 1 "comparison_operator" 1689 [(match_operand:SVE_I 2 "register_operand") 1690 (match_operand:SVE_I 3 "nonmemory_operand")])) 1691 (clobber (reg:CC CC_REGNUM))])] 1692 "TARGET_SVE" 1693 { 1694 aarch64_expand_sve_vec_cmp_int (operands[0], GET_CODE (operands[1]), 1695 operands[2], operands[3]); 1696 DONE; 1697 } 1698 ) 1699 1700 ;; Unsigned integer comparisons. Don't enforce an immediate range here, since 1701 ;; it depends on the comparison; leave it to aarch64_expand_sve_vec_cmp_int 1702 ;; instead. 1703 (define_expand "vec_cmpu<mode><vpred>" 1704 [(parallel 1705 [(set (match_operand:<VPRED> 0 "register_operand") 1706 (match_operator:<VPRED> 1 "comparison_operator" 1707 [(match_operand:SVE_I 2 "register_operand") 1708 (match_operand:SVE_I 3 "nonmemory_operand")])) 1709 (clobber (reg:CC CC_REGNUM))])] 1710 "TARGET_SVE" 1711 { 1712 aarch64_expand_sve_vec_cmp_int (operands[0], GET_CODE (operands[1]), 1713 operands[2], operands[3]); 1714 DONE; 1715 } 1716 ) 1717 1718 ;; Floating-point comparisons. All comparisons except FCMUO allow a zero 1719 ;; operand; aarch64_expand_sve_vec_cmp_float handles the case of an FCMUO 1720 ;; with zero. 1721 (define_expand "vec_cmp<mode><vpred>" 1722 [(set (match_operand:<VPRED> 0 "register_operand") 1723 (match_operator:<VPRED> 1 "comparison_operator" 1724 [(match_operand:SVE_F 2 "register_operand") 1725 (match_operand:SVE_F 3 "aarch64_simd_reg_or_zero")]))] 1726 "TARGET_SVE" 1727 { 1728 aarch64_expand_sve_vec_cmp_float (operands[0], GET_CODE (operands[1]), 1729 operands[2], operands[3], false); 1730 DONE; 1731 } 1732 ) 1733 1734 ;; Branch based on predicate equality or inequality. 1735 (define_expand "cbranch<mode>4" 1736 [(set (pc) 1737 (if_then_else 1738 (match_operator 0 "aarch64_equality_operator" 1739 [(match_operand:PRED_ALL 1 "register_operand") 1740 (match_operand:PRED_ALL 2 "aarch64_simd_reg_or_zero")]) 1741 (label_ref (match_operand 3 "")) 1742 (pc)))] 1743 "" 1744 { 1745 rtx ptrue = force_reg (<MODE>mode, CONSTM1_RTX (<MODE>mode)); 1746 rtx pred; 1747 if (operands[2] == CONST0_RTX (<MODE>mode)) 1748 pred = operands[1]; 1749 else 1750 { 1751 pred = gen_reg_rtx (<MODE>mode); 1752 emit_insn (gen_pred_xor<mode>3 (pred, ptrue, operands[1], 1753 operands[2])); 1754 } 1755 emit_insn (gen_ptest_ptrue<mode> (ptrue, pred)); 1756 operands[1] = gen_rtx_REG (CCmode, CC_REGNUM); 1757 operands[2] = const0_rtx; 1758 } 1759 ) 1760 1761 ;; Unpredicated integer MIN/MAX. 1762 (define_expand "<su><maxmin><mode>3" 1763 [(set (match_operand:SVE_I 0 "register_operand") 1764 (unspec:SVE_I 1765 [(match_dup 3) 1766 (MAXMIN:SVE_I (match_operand:SVE_I 1 "register_operand") 1767 (match_operand:SVE_I 2 "register_operand"))] 1768 UNSPEC_MERGE_PTRUE))] 1769 "TARGET_SVE" 1770 { 1771 operands[3] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode)); 1772 } 1773 ) 1774 1775 ;; Integer MIN/MAX predicated with a PTRUE. 1776 (define_insn "*<su><maxmin><mode>3" 1777 [(set (match_operand:SVE_I 0 "register_operand" "=w, ?&w") 1778 (unspec:SVE_I 1779 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl") 1780 (MAXMIN:SVE_I (match_operand:SVE_I 2 "register_operand" "%0, w") 1781 (match_operand:SVE_I 3 "register_operand" "w, w"))] 1782 UNSPEC_MERGE_PTRUE))] 1783 "TARGET_SVE" 1784 "@ 1785 <su><maxmin>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype> 1786 movprfx\t%0, %2\;<su><maxmin>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>" 1787 [(set_attr "movprfx" "*,yes")] 1788 ) 1789 1790 ;; Unpredicated floating-point MIN/MAX. 1791 (define_expand "<su><maxmin><mode>3" 1792 [(set (match_operand:SVE_F 0 "register_operand") 1793 (unspec:SVE_F 1794 [(match_dup 3) 1795 (FMAXMIN:SVE_F (match_operand:SVE_F 1 "register_operand") 1796 (match_operand:SVE_F 2 "register_operand"))] 1797 UNSPEC_MERGE_PTRUE))] 1798 "TARGET_SVE" 1799 { 1800 operands[3] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode)); 1801 } 1802 ) 1803 1804 ;; Floating-point MIN/MAX predicated with a PTRUE. 1805 (define_insn "*<su><maxmin><mode>3" 1806 [(set (match_operand:SVE_F 0 "register_operand" "=w, ?&w") 1807 (unspec:SVE_F 1808 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl") 1809 (FMAXMIN:SVE_F (match_operand:SVE_F 2 "register_operand" "%0, w") 1810 (match_operand:SVE_F 3 "register_operand" "w, w"))] 1811 UNSPEC_MERGE_PTRUE))] 1812 "TARGET_SVE" 1813 "@ 1814 f<maxmin>nm\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype> 1815 movprfx\t%0, %2\;f<maxmin>nm\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>" 1816 [(set_attr "movprfx" "*,yes")] 1817 ) 1818 1819 ;; Unpredicated fmin/fmax. 1820 (define_expand "<maxmin_uns><mode>3" 1821 [(set (match_operand:SVE_F 0 "register_operand") 1822 (unspec:SVE_F 1823 [(match_dup 3) 1824 (unspec:SVE_F [(match_operand:SVE_F 1 "register_operand") 1825 (match_operand:SVE_F 2 "register_operand")] 1826 FMAXMIN_UNS)] 1827 UNSPEC_MERGE_PTRUE))] 1828 "TARGET_SVE" 1829 { 1830 operands[3] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode)); 1831 } 1832 ) 1833 1834 ;; fmin/fmax predicated with a PTRUE. 1835 (define_insn "*<maxmin_uns><mode>3" 1836 [(set (match_operand:SVE_F 0 "register_operand" "=w, ?&w") 1837 (unspec:SVE_F 1838 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl") 1839 (unspec:SVE_F [(match_operand:SVE_F 2 "register_operand" "%0, w") 1840 (match_operand:SVE_F 3 "register_operand" "w, w")] 1841 FMAXMIN_UNS)] 1842 UNSPEC_MERGE_PTRUE))] 1843 "TARGET_SVE" 1844 "@ 1845 <maxmin_uns_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype> 1846 movprfx\t%0, %2\;<maxmin_uns_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>" 1847 [(set_attr "movprfx" "*,yes")] 1848 ) 1849 1850 ;; Predicated integer operations with select. 1851 (define_expand "cond_<optab><mode>" 1852 [(set (match_operand:SVE_I 0 "register_operand") 1853 (unspec:SVE_I 1854 [(match_operand:<VPRED> 1 "register_operand") 1855 (SVE_INT_BINARY:SVE_I 1856 (match_operand:SVE_I 2 "register_operand") 1857 (match_operand:SVE_I 3 "register_operand")) 1858 (match_operand:SVE_I 4 "aarch64_simd_reg_or_zero")] 1859 UNSPEC_SEL))] 1860 "TARGET_SVE" 1861 ) 1862 1863 (define_expand "cond_<optab><mode>" 1864 [(set (match_operand:SVE_SDI 0 "register_operand") 1865 (unspec:SVE_SDI 1866 [(match_operand:<VPRED> 1 "register_operand") 1867 (SVE_INT_BINARY_SD:SVE_SDI 1868 (match_operand:SVE_SDI 2 "register_operand") 1869 (match_operand:SVE_SDI 3 "register_operand")) 1870 (match_operand:SVE_SDI 4 "aarch64_simd_reg_or_zero")] 1871 UNSPEC_SEL))] 1872 "TARGET_SVE" 1873 ) 1874 1875 ;; Predicated integer operations with select matching the output operand. 1876 (define_insn "*cond_<optab><mode>_0" 1877 [(set (match_operand:SVE_I 0 "register_operand" "+w, w, ?&w") 1878 (unspec:SVE_I 1879 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl") 1880 (SVE_INT_BINARY:SVE_I 1881 (match_operand:SVE_I 2 "register_operand" "0, w, w") 1882 (match_operand:SVE_I 3 "register_operand" "w, 0, w")) 1883 (match_dup 0)] 1884 UNSPEC_SEL))] 1885 "TARGET_SVE" 1886 "@ 1887 <sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype> 1888 <sve_int_op_rev>\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype> 1889 movprfx\t%0, %1/m, %2\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>" 1890 [(set_attr "movprfx" "*,*,yes")] 1891 ) 1892 1893 (define_insn "*cond_<optab><mode>_0" 1894 [(set (match_operand:SVE_SDI 0 "register_operand" "+w, w, ?&w") 1895 (unspec:SVE_SDI 1896 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl") 1897 (SVE_INT_BINARY_SD:SVE_SDI 1898 (match_operand:SVE_SDI 2 "register_operand" "0, w, w") 1899 (match_operand:SVE_SDI 3 "register_operand" "w, 0, w")) 1900 (match_dup 0)] 1901 UNSPEC_SEL))] 1902 "TARGET_SVE" 1903 "@ 1904 <sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype> 1905 <sve_int_op_rev>\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype> 1906 movprfx\t%0, %1/m, %2\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>" 1907 [(set_attr "movprfx" "*,*,yes")] 1908 ) 1909 1910 ;; Predicated integer operations with select matching the first operand. 1911 (define_insn "*cond_<optab><mode>_2" 1912 [(set (match_operand:SVE_I 0 "register_operand" "=w, ?&w") 1913 (unspec:SVE_I 1914 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl") 1915 (SVE_INT_BINARY:SVE_I 1916 (match_operand:SVE_I 2 "register_operand" "0, w") 1917 (match_operand:SVE_I 3 "register_operand" "w, w")) 1918 (match_dup 2)] 1919 UNSPEC_SEL))] 1920 "TARGET_SVE" 1921 "@ 1922 <sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype> 1923 movprfx\t%0, %2\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>" 1924 [(set_attr "movprfx" "*,yes")] 1925 ) 1926 1927 (define_insn "*cond_<optab><mode>_2" 1928 [(set (match_operand:SVE_SDI 0 "register_operand" "=w, ?&w") 1929 (unspec:SVE_SDI 1930 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl") 1931 (SVE_INT_BINARY_SD:SVE_SDI 1932 (match_operand:SVE_SDI 2 "register_operand" "0, w") 1933 (match_operand:SVE_SDI 3 "register_operand" "w, w")) 1934 (match_dup 2)] 1935 UNSPEC_SEL))] 1936 "TARGET_SVE" 1937 "@ 1938 <sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype> 1939 movprfx\t%0, %2\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>" 1940 [(set_attr "movprfx" "*,yes")] 1941 ) 1942 1943 ;; Predicated integer operations with select matching the second operand. 1944 (define_insn "*cond_<optab><mode>_3" 1945 [(set (match_operand:SVE_I 0 "register_operand" "=w, ?&w") 1946 (unspec:SVE_I 1947 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl") 1948 (SVE_INT_BINARY:SVE_I 1949 (match_operand:SVE_I 2 "register_operand" "w, w") 1950 (match_operand:SVE_I 3 "register_operand" "0, w")) 1951 (match_dup 3)] 1952 UNSPEC_SEL))] 1953 "TARGET_SVE" 1954 "@ 1955 <sve_int_op_rev>\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype> 1956 movprfx\t%0, %3\;<sve_int_op_rev>\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>" 1957 [(set_attr "movprfx" "*,yes")] 1958 ) 1959 1960 (define_insn "*cond_<optab><mode>_3" 1961 [(set (match_operand:SVE_SDI 0 "register_operand" "=w, ?&w") 1962 (unspec:SVE_SDI 1963 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl") 1964 (SVE_INT_BINARY_SD:SVE_SDI 1965 (match_operand:SVE_SDI 2 "register_operand" "w, w") 1966 (match_operand:SVE_SDI 3 "register_operand" "0, w")) 1967 (match_dup 3)] 1968 UNSPEC_SEL))] 1969 "TARGET_SVE" 1970 "@ 1971 <sve_int_op_rev>\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype> 1972 movprfx\t%0, %3\;<sve_int_op_rev>\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>" 1973 [(set_attr "movprfx" "*,yes")] 1974 ) 1975 1976 ;; Predicated integer operations with select matching zero. 1977 (define_insn "*cond_<optab><mode>_z" 1978 [(set (match_operand:SVE_I 0 "register_operand" "=&w") 1979 (unspec:SVE_I 1980 [(match_operand:<VPRED> 1 "register_operand" "Upl") 1981 (SVE_INT_BINARY:SVE_I 1982 (match_operand:SVE_I 2 "register_operand" "w") 1983 (match_operand:SVE_I 3 "register_operand" "w")) 1984 (match_operand:SVE_I 4 "aarch64_simd_imm_zero")] 1985 UNSPEC_SEL))] 1986 "TARGET_SVE" 1987 "movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>" 1988 [(set_attr "movprfx" "yes")] 1989 ) 1990 1991 (define_insn "*cond_<optab><mode>_z" 1992 [(set (match_operand:SVE_SDI 0 "register_operand" "=&w") 1993 (unspec:SVE_SDI 1994 [(match_operand:<VPRED> 1 "register_operand" "Upl") 1995 (SVE_INT_BINARY_SD:SVE_SDI 1996 (match_operand:SVE_SDI 2 "register_operand" "w") 1997 (match_operand:SVE_SDI 3 "register_operand" "w")) 1998 (match_operand:SVE_SDI 4 "aarch64_simd_imm_zero")] 1999 UNSPEC_SEL))] 2000 "TARGET_SVE" 2001 "movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;<sve_int_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>" 2002 [(set_attr "movprfx" "yes")] 2003 ) 2004 2005 ;; Synthetic predications with select unmatched. 2006 (define_insn "*cond_<optab><mode>_any" 2007 [(set (match_operand:SVE_I 0 "register_operand" "=&w") 2008 (unspec:SVE_I 2009 [(match_operand:<VPRED> 1 "register_operand" "Upl") 2010 (SVE_INT_BINARY:SVE_I 2011 (match_operand:SVE_I 2 "register_operand" "w") 2012 (match_operand:SVE_I 3 "register_operand" "w")) 2013 (match_operand:SVE_I 4 "register_operand" "w")] 2014 UNSPEC_SEL))] 2015 "TARGET_SVE" 2016 "#" 2017 ) 2018 2019 (define_insn "*cond_<optab><mode>_any" 2020 [(set (match_operand:SVE_SDI 0 "register_operand" "=&w") 2021 (unspec:SVE_SDI 2022 [(match_operand:<VPRED> 1 "register_operand" "Upl") 2023 (SVE_INT_BINARY_SD:SVE_I 2024 (match_operand:SVE_SDI 2 "register_operand" "w") 2025 (match_operand:SVE_SDI 3 "register_operand" "w")) 2026 (match_operand:SVE_SDI 4 "register_operand" "w")] 2027 UNSPEC_SEL))] 2028 "TARGET_SVE" 2029 "#" 2030 ) 2031 2032 (define_split 2033 [(set (match_operand:SVE_I 0 "register_operand") 2034 (unspec:SVE_I 2035 [(match_operand:<VPRED> 1 "register_operand") 2036 (match_operator:SVE_I 5 "aarch64_sve_any_binary_operator" 2037 [(match_operand:SVE_I 2 "register_operand") 2038 (match_operand:SVE_I 3 "register_operand")]) 2039 (match_operand:SVE_I 4 "register_operand")] 2040 UNSPEC_SEL))] 2041 "TARGET_SVE && reload_completed 2042 && !(rtx_equal_p (operands[0], operands[4]) 2043 || rtx_equal_p (operands[2], operands[4]) 2044 || rtx_equal_p (operands[3], operands[4]))" 2045 ; Not matchable by any one insn or movprfx insn. We need a separate select. 2046 [(set (match_dup 0) 2047 (unspec:SVE_I [(match_dup 1) (match_dup 2) (match_dup 4)] 2048 UNSPEC_SEL)) 2049 (set (match_dup 0) 2050 (unspec:SVE_I 2051 [(match_dup 1) 2052 (match_op_dup 5 [(match_dup 0) (match_dup 3)]) 2053 (match_dup 0)] 2054 UNSPEC_SEL))] 2055 ) 2056 2057 ;; Set operand 0 to the last active element in operand 3, or to tied 2058 ;; operand 1 if no elements are active. 2059 (define_insn "fold_extract_last_<mode>" 2060 [(set (match_operand:<VEL> 0 "register_operand" "=r, w") 2061 (unspec:<VEL> 2062 [(match_operand:<VEL> 1 "register_operand" "0, 0") 2063 (match_operand:<VPRED> 2 "register_operand" "Upl, Upl") 2064 (match_operand:SVE_ALL 3 "register_operand" "w, w")] 2065 UNSPEC_CLASTB))] 2066 "TARGET_SVE" 2067 "@ 2068 clastb\t%<vwcore>0, %2, %<vwcore>0, %3.<Vetype> 2069 clastb\t%<vw>0, %2, %<vw>0, %3.<Vetype>" 2070 ) 2071 2072 ;; Unpredicated integer add reduction. 2073 (define_expand "reduc_plus_scal_<mode>" 2074 [(set (match_operand:<VEL> 0 "register_operand") 2075 (unspec:<VEL> [(match_dup 2) 2076 (match_operand:SVE_I 1 "register_operand")] 2077 UNSPEC_ADDV))] 2078 "TARGET_SVE" 2079 { 2080 operands[2] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode)); 2081 } 2082 ) 2083 2084 ;; Predicated integer add reduction. The result is always 64-bits. 2085 (define_insn "*reduc_plus_scal_<mode>" 2086 [(set (match_operand:<VEL> 0 "register_operand" "=w") 2087 (unspec:<VEL> [(match_operand:<VPRED> 1 "register_operand" "Upl") 2088 (match_operand:SVE_I 2 "register_operand" "w")] 2089 UNSPEC_ADDV))] 2090 "TARGET_SVE" 2091 "uaddv\t%d0, %1, %2.<Vetype>" 2092 ) 2093 2094 ;; Unpredicated floating-point add reduction. 2095 (define_expand "reduc_plus_scal_<mode>" 2096 [(set (match_operand:<VEL> 0 "register_operand") 2097 (unspec:<VEL> [(match_dup 2) 2098 (match_operand:SVE_F 1 "register_operand")] 2099 UNSPEC_FADDV))] 2100 "TARGET_SVE" 2101 { 2102 operands[2] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode)); 2103 } 2104 ) 2105 2106 ;; Predicated floating-point add reduction. 2107 (define_insn "*reduc_plus_scal_<mode>" 2108 [(set (match_operand:<VEL> 0 "register_operand" "=w") 2109 (unspec:<VEL> [(match_operand:<VPRED> 1 "register_operand" "Upl") 2110 (match_operand:SVE_F 2 "register_operand" "w")] 2111 UNSPEC_FADDV))] 2112 "TARGET_SVE" 2113 "faddv\t%<Vetype>0, %1, %2.<Vetype>" 2114 ) 2115 2116 ;; Unpredicated integer MIN/MAX reduction. 2117 (define_expand "reduc_<maxmin_uns>_scal_<mode>" 2118 [(set (match_operand:<VEL> 0 "register_operand") 2119 (unspec:<VEL> [(match_dup 2) 2120 (match_operand:SVE_I 1 "register_operand")] 2121 MAXMINV))] 2122 "TARGET_SVE" 2123 { 2124 operands[2] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode)); 2125 } 2126 ) 2127 2128 ;; Predicated integer MIN/MAX reduction. 2129 (define_insn "*reduc_<maxmin_uns>_scal_<mode>" 2130 [(set (match_operand:<VEL> 0 "register_operand" "=w") 2131 (unspec:<VEL> [(match_operand:<VPRED> 1 "register_operand" "Upl") 2132 (match_operand:SVE_I 2 "register_operand" "w")] 2133 MAXMINV))] 2134 "TARGET_SVE" 2135 "<maxmin_uns_op>v\t%<Vetype>0, %1, %2.<Vetype>" 2136 ) 2137 2138 ;; Unpredicated floating-point MIN/MAX reduction. 2139 (define_expand "reduc_<maxmin_uns>_scal_<mode>" 2140 [(set (match_operand:<VEL> 0 "register_operand") 2141 (unspec:<VEL> [(match_dup 2) 2142 (match_operand:SVE_F 1 "register_operand")] 2143 FMAXMINV))] 2144 "TARGET_SVE" 2145 { 2146 operands[2] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode)); 2147 } 2148 ) 2149 2150 ;; Predicated floating-point MIN/MAX reduction. 2151 (define_insn "*reduc_<maxmin_uns>_scal_<mode>" 2152 [(set (match_operand:<VEL> 0 "register_operand" "=w") 2153 (unspec:<VEL> [(match_operand:<VPRED> 1 "register_operand" "Upl") 2154 (match_operand:SVE_F 2 "register_operand" "w")] 2155 FMAXMINV))] 2156 "TARGET_SVE" 2157 "<maxmin_uns_op>v\t%<Vetype>0, %1, %2.<Vetype>" 2158 ) 2159 2160 (define_expand "reduc_<optab>_scal_<mode>" 2161 [(set (match_operand:<VEL> 0 "register_operand") 2162 (unspec:<VEL> [(match_dup 2) 2163 (match_operand:SVE_I 1 "register_operand")] 2164 BITWISEV))] 2165 "TARGET_SVE" 2166 { 2167 operands[2] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode)); 2168 } 2169 ) 2170 2171 (define_insn "*reduc_<optab>_scal_<mode>" 2172 [(set (match_operand:<VEL> 0 "register_operand" "=w") 2173 (unspec:<VEL> [(match_operand:<VPRED> 1 "register_operand" "Upl") 2174 (match_operand:SVE_I 2 "register_operand" "w")] 2175 BITWISEV))] 2176 "TARGET_SVE" 2177 "<bit_reduc_op>\t%<Vetype>0, %1, %2.<Vetype>" 2178 ) 2179 2180 ;; Unpredicated in-order FP reductions. 2181 (define_expand "fold_left_plus_<mode>" 2182 [(set (match_operand:<VEL> 0 "register_operand") 2183 (unspec:<VEL> [(match_dup 3) 2184 (match_operand:<VEL> 1 "register_operand") 2185 (match_operand:SVE_F 2 "register_operand")] 2186 UNSPEC_FADDA))] 2187 "TARGET_SVE" 2188 { 2189 operands[3] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode)); 2190 } 2191 ) 2192 2193 ;; In-order FP reductions predicated with PTRUE. 2194 (define_insn "*fold_left_plus_<mode>" 2195 [(set (match_operand:<VEL> 0 "register_operand" "=w") 2196 (unspec:<VEL> [(match_operand:<VPRED> 1 "register_operand" "Upl") 2197 (match_operand:<VEL> 2 "register_operand" "0") 2198 (match_operand:SVE_F 3 "register_operand" "w")] 2199 UNSPEC_FADDA))] 2200 "TARGET_SVE" 2201 "fadda\t%<Vetype>0, %1, %<Vetype>0, %3.<Vetype>" 2202 ) 2203 2204 ;; Predicated form of the above in-order reduction. 2205 (define_insn "*pred_fold_left_plus_<mode>" 2206 [(set (match_operand:<VEL> 0 "register_operand" "=w") 2207 (unspec:<VEL> 2208 [(match_operand:<VEL> 1 "register_operand" "0") 2209 (unspec:SVE_F 2210 [(match_operand:<VPRED> 2 "register_operand" "Upl") 2211 (match_operand:SVE_F 3 "register_operand" "w") 2212 (match_operand:SVE_F 4 "aarch64_simd_imm_zero")] 2213 UNSPEC_SEL)] 2214 UNSPEC_FADDA))] 2215 "TARGET_SVE" 2216 "fadda\t%<Vetype>0, %2, %<Vetype>0, %3.<Vetype>" 2217 ) 2218 2219 ;; Unpredicated floating-point addition. 2220 (define_expand "add<mode>3" 2221 [(set (match_operand:SVE_F 0 "register_operand") 2222 (unspec:SVE_F 2223 [(match_dup 3) 2224 (plus:SVE_F 2225 (match_operand:SVE_F 1 "register_operand") 2226 (match_operand:SVE_F 2 "aarch64_sve_float_arith_with_sub_operand"))] 2227 UNSPEC_MERGE_PTRUE))] 2228 "TARGET_SVE" 2229 { 2230 operands[3] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode)); 2231 } 2232 ) 2233 2234 ;; Floating-point addition predicated with a PTRUE. 2235 (define_insn_and_split "*add<mode>3" 2236 [(set (match_operand:SVE_F 0 "register_operand" "=w, w, w") 2237 (unspec:SVE_F 2238 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl") 2239 (plus:SVE_F 2240 (match_operand:SVE_F 2 "register_operand" "%0, 0, w") 2241 (match_operand:SVE_F 3 "aarch64_sve_float_arith_with_sub_operand" "vsA, vsN, w"))] 2242 UNSPEC_MERGE_PTRUE))] 2243 "TARGET_SVE" 2244 "@ 2245 fadd\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3 2246 fsub\t%0.<Vetype>, %1/m, %0.<Vetype>, #%N3 2247 #" 2248 ; Split the unpredicated form after reload, so that we don't have 2249 ; the unnecessary PTRUE. 2250 "&& reload_completed 2251 && register_operand (operands[3], <MODE>mode)" 2252 [(set (match_dup 0) (plus:SVE_F (match_dup 2) (match_dup 3)))] 2253 ) 2254 2255 ;; Unpredicated floating-point subtraction. 2256 (define_expand "sub<mode>3" 2257 [(set (match_operand:SVE_F 0 "register_operand") 2258 (unspec:SVE_F 2259 [(match_dup 3) 2260 (minus:SVE_F 2261 (match_operand:SVE_F 1 "aarch64_sve_float_arith_operand") 2262 (match_operand:SVE_F 2 "register_operand"))] 2263 UNSPEC_MERGE_PTRUE))] 2264 "TARGET_SVE" 2265 { 2266 operands[3] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode)); 2267 } 2268 ) 2269 2270 ;; Floating-point subtraction predicated with a PTRUE. 2271 (define_insn_and_split "*sub<mode>3" 2272 [(set (match_operand:SVE_F 0 "register_operand" "=w, w, w, w") 2273 (unspec:SVE_F 2274 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl, Upl") 2275 (minus:SVE_F 2276 (match_operand:SVE_F 2 "aarch64_sve_float_arith_operand" "0, 0, vsA, w") 2277 (match_operand:SVE_F 3 "aarch64_sve_float_arith_with_sub_operand" "vsA, vsN, 0, w"))] 2278 UNSPEC_MERGE_PTRUE))] 2279 "TARGET_SVE 2280 && (register_operand (operands[2], <MODE>mode) 2281 || register_operand (operands[3], <MODE>mode))" 2282 "@ 2283 fsub\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3 2284 fadd\t%0.<Vetype>, %1/m, %0.<Vetype>, #%N3 2285 fsubr\t%0.<Vetype>, %1/m, %0.<Vetype>, #%2 2286 #" 2287 ; Split the unpredicated form after reload, so that we don't have 2288 ; the unnecessary PTRUE. 2289 "&& reload_completed 2290 && register_operand (operands[2], <MODE>mode) 2291 && register_operand (operands[3], <MODE>mode)" 2292 [(set (match_dup 0) (minus:SVE_F (match_dup 2) (match_dup 3)))] 2293 ) 2294 2295 ;; Unpredicated floating-point multiplication. 2296 (define_expand "mul<mode>3" 2297 [(set (match_operand:SVE_F 0 "register_operand") 2298 (unspec:SVE_F 2299 [(match_dup 3) 2300 (mult:SVE_F 2301 (match_operand:SVE_F 1 "register_operand") 2302 (match_operand:SVE_F 2 "aarch64_sve_float_mul_operand"))] 2303 UNSPEC_MERGE_PTRUE))] 2304 "TARGET_SVE" 2305 { 2306 operands[3] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode)); 2307 } 2308 ) 2309 2310 ;; Floating-point multiplication predicated with a PTRUE. 2311 (define_insn_and_split "*mul<mode>3" 2312 [(set (match_operand:SVE_F 0 "register_operand" "=w, w") 2313 (unspec:SVE_F 2314 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl") 2315 (mult:SVE_F 2316 (match_operand:SVE_F 2 "register_operand" "%0, w") 2317 (match_operand:SVE_F 3 "aarch64_sve_float_mul_operand" "vsM, w"))] 2318 UNSPEC_MERGE_PTRUE))] 2319 "TARGET_SVE" 2320 "@ 2321 fmul\t%0.<Vetype>, %1/m, %0.<Vetype>, #%3 2322 #" 2323 ; Split the unpredicated form after reload, so that we don't have 2324 ; the unnecessary PTRUE. 2325 "&& reload_completed 2326 && register_operand (operands[3], <MODE>mode)" 2327 [(set (match_dup 0) (mult:SVE_F (match_dup 2) (match_dup 3)))] 2328 ) 2329 2330 ;; Unpredicated floating-point binary operations (post-RA only). 2331 ;; These are generated by splitting a predicated instruction whose 2332 ;; predicate is unused. 2333 (define_insn "*post_ra_<sve_fp_op><mode>3" 2334 [(set (match_operand:SVE_F 0 "register_operand" "=w") 2335 (SVE_UNPRED_FP_BINARY:SVE_F 2336 (match_operand:SVE_F 1 "register_operand" "w") 2337 (match_operand:SVE_F 2 "register_operand" "w")))] 2338 "TARGET_SVE && reload_completed" 2339 "<sve_fp_op>\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>") 2340 2341 ;; Unpredicated fma (%0 = (%1 * %2) + %3). 2342 (define_expand "fma<mode>4" 2343 [(set (match_operand:SVE_F 0 "register_operand") 2344 (unspec:SVE_F 2345 [(match_dup 4) 2346 (fma:SVE_F (match_operand:SVE_F 1 "register_operand") 2347 (match_operand:SVE_F 2 "register_operand") 2348 (match_operand:SVE_F 3 "register_operand"))] 2349 UNSPEC_MERGE_PTRUE))] 2350 "TARGET_SVE" 2351 { 2352 operands[4] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode)); 2353 } 2354 ) 2355 2356 ;; fma predicated with a PTRUE. 2357 (define_insn "*fma<mode>4" 2358 [(set (match_operand:SVE_F 0 "register_operand" "=w, w, ?&w") 2359 (unspec:SVE_F 2360 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl") 2361 (fma:SVE_F (match_operand:SVE_F 3 "register_operand" "%0, w, w") 2362 (match_operand:SVE_F 4 "register_operand" "w, w, w") 2363 (match_operand:SVE_F 2 "register_operand" "w, 0, w"))] 2364 UNSPEC_MERGE_PTRUE))] 2365 "TARGET_SVE" 2366 "@ 2367 fmad\t%0.<Vetype>, %1/m, %4.<Vetype>, %2.<Vetype> 2368 fmla\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype> 2369 movprfx\t%0, %2\;fmla\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype>" 2370 [(set_attr "movprfx" "*,*,yes")] 2371 ) 2372 2373 ;; Unpredicated fnma (%0 = (-%1 * %2) + %3). 2374 (define_expand "fnma<mode>4" 2375 [(set (match_operand:SVE_F 0 "register_operand") 2376 (unspec:SVE_F 2377 [(match_dup 4) 2378 (fma:SVE_F (neg:SVE_F 2379 (match_operand:SVE_F 1 "register_operand")) 2380 (match_operand:SVE_F 2 "register_operand") 2381 (match_operand:SVE_F 3 "register_operand"))] 2382 UNSPEC_MERGE_PTRUE))] 2383 "TARGET_SVE" 2384 { 2385 operands[4] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode)); 2386 } 2387 ) 2388 2389 ;; fnma predicated with a PTRUE. 2390 (define_insn "*fnma<mode>4" 2391 [(set (match_operand:SVE_F 0 "register_operand" "=w, w, ?&w") 2392 (unspec:SVE_F 2393 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl") 2394 (fma:SVE_F (neg:SVE_F 2395 (match_operand:SVE_F 3 "register_operand" "%0, w, w")) 2396 (match_operand:SVE_F 4 "register_operand" "w, w, w") 2397 (match_operand:SVE_F 2 "register_operand" "w, 0, w"))] 2398 UNSPEC_MERGE_PTRUE))] 2399 "TARGET_SVE" 2400 "@ 2401 fmsb\t%0.<Vetype>, %1/m, %4.<Vetype>, %2.<Vetype> 2402 fmls\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype> 2403 movprfx\t%0, %2\;fmls\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype>" 2404 [(set_attr "movprfx" "*,*,yes")] 2405 ) 2406 2407 ;; Unpredicated fms (%0 = (%1 * %2) - %3). 2408 (define_expand "fms<mode>4" 2409 [(set (match_operand:SVE_F 0 "register_operand") 2410 (unspec:SVE_F 2411 [(match_dup 4) 2412 (fma:SVE_F (match_operand:SVE_F 1 "register_operand") 2413 (match_operand:SVE_F 2 "register_operand") 2414 (neg:SVE_F 2415 (match_operand:SVE_F 3 "register_operand")))] 2416 UNSPEC_MERGE_PTRUE))] 2417 "TARGET_SVE" 2418 { 2419 operands[4] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode)); 2420 } 2421 ) 2422 2423 ;; fms predicated with a PTRUE. 2424 (define_insn "*fms<mode>4" 2425 [(set (match_operand:SVE_F 0 "register_operand" "=w, w, ?&w") 2426 (unspec:SVE_F 2427 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl") 2428 (fma:SVE_F (match_operand:SVE_F 3 "register_operand" "%0, w, w") 2429 (match_operand:SVE_F 4 "register_operand" "w, w, w") 2430 (neg:SVE_F 2431 (match_operand:SVE_F 2 "register_operand" "w, 0, w")))] 2432 UNSPEC_MERGE_PTRUE))] 2433 "TARGET_SVE" 2434 "@ 2435 fnmsb\t%0.<Vetype>, %1/m, %4.<Vetype>, %2.<Vetype> 2436 fnmls\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype> 2437 movprfx\t%0, %2\;fnmls\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype>" 2438 [(set_attr "movprfx" "*,*,yes")] 2439 ) 2440 2441 ;; Unpredicated fnms (%0 = (-%1 * %2) - %3). 2442 (define_expand "fnms<mode>4" 2443 [(set (match_operand:SVE_F 0 "register_operand") 2444 (unspec:SVE_F 2445 [(match_dup 4) 2446 (fma:SVE_F (neg:SVE_F 2447 (match_operand:SVE_F 1 "register_operand")) 2448 (match_operand:SVE_F 2 "register_operand") 2449 (neg:SVE_F 2450 (match_operand:SVE_F 3 "register_operand")))] 2451 UNSPEC_MERGE_PTRUE))] 2452 "TARGET_SVE" 2453 { 2454 operands[4] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode)); 2455 } 2456 ) 2457 2458 ;; fnms predicated with a PTRUE. 2459 (define_insn "*fnms<mode>4" 2460 [(set (match_operand:SVE_F 0 "register_operand" "=w, w, ?&w") 2461 (unspec:SVE_F 2462 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl") 2463 (fma:SVE_F (neg:SVE_F 2464 (match_operand:SVE_F 3 "register_operand" "%0, w, w")) 2465 (match_operand:SVE_F 4 "register_operand" "w, w, w") 2466 (neg:SVE_F 2467 (match_operand:SVE_F 2 "register_operand" "w, 0, w")))] 2468 UNSPEC_MERGE_PTRUE))] 2469 "TARGET_SVE" 2470 "@ 2471 fnmad\t%0.<Vetype>, %1/m, %4.<Vetype>, %2.<Vetype> 2472 fnmla\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype> 2473 movprfx\t%0, %2\;fnmla\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype>" 2474 [(set_attr "movprfx" "*,*,yes")] 2475 ) 2476 2477 ;; Unpredicated floating-point division. 2478 (define_expand "div<mode>3" 2479 [(set (match_operand:SVE_F 0 "register_operand") 2480 (unspec:SVE_F 2481 [(match_dup 3) 2482 (div:SVE_F (match_operand:SVE_F 1 "register_operand") 2483 (match_operand:SVE_F 2 "register_operand"))] 2484 UNSPEC_MERGE_PTRUE))] 2485 "TARGET_SVE" 2486 { 2487 operands[3] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode)); 2488 } 2489 ) 2490 2491 ;; Floating-point division predicated with a PTRUE. 2492 (define_insn "*div<mode>3" 2493 [(set (match_operand:SVE_F 0 "register_operand" "=w, w, ?&w") 2494 (unspec:SVE_F 2495 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl") 2496 (div:SVE_F (match_operand:SVE_F 2 "register_operand" "0, w, w") 2497 (match_operand:SVE_F 3 "register_operand" "w, 0, w"))] 2498 UNSPEC_MERGE_PTRUE))] 2499 "TARGET_SVE" 2500 "@ 2501 fdiv\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype> 2502 fdivr\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype> 2503 movprfx\t%0, %2\;fdiv\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>" 2504 [(set_attr "movprfx" "*,*,yes")] 2505 ) 2506 2507 ;; Unpredicated FNEG, FABS and FSQRT. 2508 (define_expand "<optab><mode>2" 2509 [(set (match_operand:SVE_F 0 "register_operand") 2510 (unspec:SVE_F 2511 [(match_dup 2) 2512 (SVE_FP_UNARY:SVE_F (match_operand:SVE_F 1 "register_operand"))] 2513 UNSPEC_MERGE_PTRUE))] 2514 "TARGET_SVE" 2515 { 2516 operands[2] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode)); 2517 } 2518 ) 2519 2520 ;; FNEG, FABS and FSQRT predicated with a PTRUE. 2521 (define_insn "*<optab><mode>2" 2522 [(set (match_operand:SVE_F 0 "register_operand" "=w") 2523 (unspec:SVE_F 2524 [(match_operand:<VPRED> 1 "register_operand" "Upl") 2525 (SVE_FP_UNARY:SVE_F (match_operand:SVE_F 2 "register_operand" "w"))] 2526 UNSPEC_MERGE_PTRUE))] 2527 "TARGET_SVE" 2528 "<sve_fp_op>\t%0.<Vetype>, %1/m, %2.<Vetype>" 2529 ) 2530 2531 ;; Unpredicated FRINTy. 2532 (define_expand "<frint_pattern><mode>2" 2533 [(set (match_operand:SVE_F 0 "register_operand") 2534 (unspec:SVE_F 2535 [(match_dup 2) 2536 (unspec:SVE_F [(match_operand:SVE_F 1 "register_operand")] 2537 FRINT)] 2538 UNSPEC_MERGE_PTRUE))] 2539 "TARGET_SVE" 2540 { 2541 operands[2] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode)); 2542 } 2543 ) 2544 2545 ;; FRINTy predicated with a PTRUE. 2546 (define_insn "*<frint_pattern><mode>2" 2547 [(set (match_operand:SVE_F 0 "register_operand" "=w") 2548 (unspec:SVE_F 2549 [(match_operand:<VPRED> 1 "register_operand" "Upl") 2550 (unspec:SVE_F [(match_operand:SVE_F 2 "register_operand" "w")] 2551 FRINT)] 2552 UNSPEC_MERGE_PTRUE))] 2553 "TARGET_SVE" 2554 "frint<frint_suffix>\t%0.<Vetype>, %1/m, %2.<Vetype>" 2555 ) 2556 2557 ;; Unpredicated conversion of floats to integers of the same size (HF to HI, 2558 ;; SF to SI or DF to DI). 2559 (define_expand "<fix_trunc_optab><mode><v_int_equiv>2" 2560 [(set (match_operand:<V_INT_EQUIV> 0 "register_operand") 2561 (unspec:<V_INT_EQUIV> 2562 [(match_dup 2) 2563 (FIXUORS:<V_INT_EQUIV> 2564 (match_operand:SVE_F 1 "register_operand"))] 2565 UNSPEC_MERGE_PTRUE))] 2566 "TARGET_SVE" 2567 { 2568 operands[2] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode)); 2569 } 2570 ) 2571 2572 ;; Conversion of SF to DI, SI or HI, predicated with a PTRUE. 2573 (define_insn "*<fix_trunc_optab>v16hsf<mode>2" 2574 [(set (match_operand:SVE_HSDI 0 "register_operand" "=w") 2575 (unspec:SVE_HSDI 2576 [(match_operand:<VPRED> 1 "register_operand" "Upl") 2577 (FIXUORS:SVE_HSDI 2578 (match_operand:VNx8HF 2 "register_operand" "w"))] 2579 UNSPEC_MERGE_PTRUE))] 2580 "TARGET_SVE" 2581 "fcvtz<su>\t%0.<Vetype>, %1/m, %2.h" 2582 ) 2583 2584 ;; Conversion of SF to DI or SI, predicated with a PTRUE. 2585 (define_insn "*<fix_trunc_optab>vnx4sf<mode>2" 2586 [(set (match_operand:SVE_SDI 0 "register_operand" "=w") 2587 (unspec:SVE_SDI 2588 [(match_operand:<VPRED> 1 "register_operand" "Upl") 2589 (FIXUORS:SVE_SDI 2590 (match_operand:VNx4SF 2 "register_operand" "w"))] 2591 UNSPEC_MERGE_PTRUE))] 2592 "TARGET_SVE" 2593 "fcvtz<su>\t%0.<Vetype>, %1/m, %2.s" 2594 ) 2595 2596 ;; Conversion of DF to DI or SI, predicated with a PTRUE. 2597 (define_insn "*<fix_trunc_optab>vnx2df<mode>2" 2598 [(set (match_operand:SVE_SDI 0 "register_operand" "=w") 2599 (unspec:SVE_SDI 2600 [(match_operand:VNx2BI 1 "register_operand" "Upl") 2601 (FIXUORS:SVE_SDI 2602 (match_operand:VNx2DF 2 "register_operand" "w"))] 2603 UNSPEC_MERGE_PTRUE))] 2604 "TARGET_SVE" 2605 "fcvtz<su>\t%0.<Vetype>, %1/m, %2.d" 2606 ) 2607 2608 ;; Unpredicated conversion of integers to floats of the same size 2609 ;; (HI to HF, SI to SF or DI to DF). 2610 (define_expand "<optab><v_int_equiv><mode>2" 2611 [(set (match_operand:SVE_F 0 "register_operand") 2612 (unspec:SVE_F 2613 [(match_dup 2) 2614 (FLOATUORS:SVE_F 2615 (match_operand:<V_INT_EQUIV> 1 "register_operand"))] 2616 UNSPEC_MERGE_PTRUE))] 2617 "TARGET_SVE" 2618 { 2619 operands[2] = force_reg (<VPRED>mode, CONSTM1_RTX (<VPRED>mode)); 2620 } 2621 ) 2622 2623 ;; Conversion of DI, SI or HI to the same number of HFs, predicated 2624 ;; with a PTRUE. 2625 (define_insn "*<optab><mode>vnx8hf2" 2626 [(set (match_operand:VNx8HF 0 "register_operand" "=w") 2627 (unspec:VNx8HF 2628 [(match_operand:<VPRED> 1 "register_operand" "Upl") 2629 (FLOATUORS:VNx8HF 2630 (match_operand:SVE_HSDI 2 "register_operand" "w"))] 2631 UNSPEC_MERGE_PTRUE))] 2632 "TARGET_SVE" 2633 "<su_optab>cvtf\t%0.h, %1/m, %2.<Vetype>" 2634 ) 2635 2636 ;; Conversion of DI or SI to the same number of SFs, predicated with a PTRUE. 2637 (define_insn "*<optab><mode>vnx4sf2" 2638 [(set (match_operand:VNx4SF 0 "register_operand" "=w") 2639 (unspec:VNx4SF 2640 [(match_operand:<VPRED> 1 "register_operand" "Upl") 2641 (FLOATUORS:VNx4SF 2642 (match_operand:SVE_SDI 2 "register_operand" "w"))] 2643 UNSPEC_MERGE_PTRUE))] 2644 "TARGET_SVE" 2645 "<su_optab>cvtf\t%0.s, %1/m, %2.<Vetype>" 2646 ) 2647 2648 ;; Conversion of DI or SI to DF, predicated with a PTRUE. 2649 (define_insn "aarch64_sve_<optab><mode>vnx2df2" 2650 [(set (match_operand:VNx2DF 0 "register_operand" "=w") 2651 (unspec:VNx2DF 2652 [(match_operand:VNx2BI 1 "register_operand" "Upl") 2653 (FLOATUORS:VNx2DF 2654 (match_operand:SVE_SDI 2 "register_operand" "w"))] 2655 UNSPEC_MERGE_PTRUE))] 2656 "TARGET_SVE" 2657 "<su_optab>cvtf\t%0.d, %1/m, %2.<Vetype>" 2658 ) 2659 2660 ;; Conversion of DFs to the same number of SFs, or SFs to the same number 2661 ;; of HFs. 2662 (define_insn "*trunc<Vwide><mode>2" 2663 [(set (match_operand:SVE_HSF 0 "register_operand" "=w") 2664 (unspec:SVE_HSF 2665 [(match_operand:<VWIDE_PRED> 1 "register_operand" "Upl") 2666 (unspec:SVE_HSF 2667 [(match_operand:<VWIDE> 2 "register_operand" "w")] 2668 UNSPEC_FLOAT_CONVERT)] 2669 UNSPEC_MERGE_PTRUE))] 2670 "TARGET_SVE" 2671 "fcvt\t%0.<Vetype>, %1/m, %2.<Vewtype>" 2672 ) 2673 2674 ;; Conversion of SFs to the same number of DFs, or HFs to the same number 2675 ;; of SFs. 2676 (define_insn "aarch64_sve_extend<mode><Vwide>2" 2677 [(set (match_operand:<VWIDE> 0 "register_operand" "=w") 2678 (unspec:<VWIDE> 2679 [(match_operand:<VWIDE_PRED> 1 "register_operand" "Upl") 2680 (unspec:<VWIDE> 2681 [(match_operand:SVE_HSF 2 "register_operand" "w")] 2682 UNSPEC_FLOAT_CONVERT)] 2683 UNSPEC_MERGE_PTRUE))] 2684 "TARGET_SVE" 2685 "fcvt\t%0.<Vewtype>, %1/m, %2.<Vetype>" 2686 ) 2687 2688 ;; Unpack the low or high half of a predicate, where "high" refers to 2689 ;; the low-numbered lanes for big-endian and the high-numbered lanes 2690 ;; for little-endian. 2691 (define_expand "vec_unpack<su>_<perm_hilo>_<mode>" 2692 [(match_operand:<VWIDE> 0 "register_operand") 2693 (unspec:<VWIDE> [(match_operand:PRED_BHS 1 "register_operand")] 2694 UNPACK)] 2695 "TARGET_SVE" 2696 { 2697 emit_insn ((<hi_lanes_optab> 2698 ? gen_aarch64_sve_punpkhi_<PRED_BHS:mode> 2699 : gen_aarch64_sve_punpklo_<PRED_BHS:mode>) 2700 (operands[0], operands[1])); 2701 DONE; 2702 } 2703 ) 2704 2705 ;; PUNPKHI and PUNPKLO. 2706 (define_insn "aarch64_sve_punpk<perm_hilo>_<mode>" 2707 [(set (match_operand:<VWIDE> 0 "register_operand" "=Upa") 2708 (unspec:<VWIDE> [(match_operand:PRED_BHS 1 "register_operand" "Upa")] 2709 UNPACK_UNSIGNED))] 2710 "TARGET_SVE" 2711 "punpk<perm_hilo>\t%0.h, %1.b" 2712 ) 2713 2714 ;; Unpack the low or high half of a vector, where "high" refers to 2715 ;; the low-numbered lanes for big-endian and the high-numbered lanes 2716 ;; for little-endian. 2717 (define_expand "vec_unpack<su>_<perm_hilo>_<SVE_BHSI:mode>" 2718 [(match_operand:<VWIDE> 0 "register_operand") 2719 (unspec:<VWIDE> [(match_operand:SVE_BHSI 1 "register_operand")] UNPACK)] 2720 "TARGET_SVE" 2721 { 2722 emit_insn ((<hi_lanes_optab> 2723 ? gen_aarch64_sve_<su>unpkhi_<SVE_BHSI:mode> 2724 : gen_aarch64_sve_<su>unpklo_<SVE_BHSI:mode>) 2725 (operands[0], operands[1])); 2726 DONE; 2727 } 2728 ) 2729 2730 ;; SUNPKHI, UUNPKHI, SUNPKLO and UUNPKLO. 2731 (define_insn "aarch64_sve_<su>unpk<perm_hilo>_<SVE_BHSI:mode>" 2732 [(set (match_operand:<VWIDE> 0 "register_operand" "=w") 2733 (unspec:<VWIDE> [(match_operand:SVE_BHSI 1 "register_operand" "w")] 2734 UNPACK))] 2735 "TARGET_SVE" 2736 "<su>unpk<perm_hilo>\t%0.<Vewtype>, %1.<Vetype>" 2737 ) 2738 2739 ;; Unpack one half of a VNx4SF to VNx2DF, or one half of a VNx8HF to VNx4SF. 2740 ;; First unpack the source without conversion, then float-convert the 2741 ;; unpacked source. 2742 (define_expand "vec_unpacks_<perm_hilo>_<mode>" 2743 [(match_operand:<VWIDE> 0 "register_operand") 2744 (unspec:SVE_HSF [(match_operand:SVE_HSF 1 "register_operand")] 2745 UNPACK_UNSIGNED)] 2746 "TARGET_SVE" 2747 { 2748 /* Use ZIP to do the unpack, since we don't care about the upper halves 2749 and since it has the nice property of not needing any subregs. 2750 If using UUNPK* turns out to be preferable, we could model it as 2751 a ZIP whose first operand is zero. */ 2752 rtx temp = gen_reg_rtx (<MODE>mode); 2753 emit_insn ((<hi_lanes_optab> 2754 ? gen_aarch64_sve_zip2<mode> 2755 : gen_aarch64_sve_zip1<mode>) 2756 (temp, operands[1], operands[1])); 2757 rtx ptrue = force_reg (<VWIDE_PRED>mode, CONSTM1_RTX (<VWIDE_PRED>mode)); 2758 emit_insn (gen_aarch64_sve_extend<mode><Vwide>2 (operands[0], 2759 ptrue, temp)); 2760 DONE; 2761 } 2762 ) 2763 2764 ;; Unpack one half of a VNx4SI to VNx2DF. First unpack from VNx4SI 2765 ;; to VNx2DI, reinterpret the VNx2DI as a VNx4SI, then convert the 2766 ;; unpacked VNx4SI to VNx2DF. 2767 (define_expand "vec_unpack<su_optab>_float_<perm_hilo>_vnx4si" 2768 [(match_operand:VNx2DF 0 "register_operand") 2769 (FLOATUORS:VNx2DF 2770 (unspec:VNx2DI [(match_operand:VNx4SI 1 "register_operand")] 2771 UNPACK_UNSIGNED))] 2772 "TARGET_SVE" 2773 { 2774 /* Use ZIP to do the unpack, since we don't care about the upper halves 2775 and since it has the nice property of not needing any subregs. 2776 If using UUNPK* turns out to be preferable, we could model it as 2777 a ZIP whose first operand is zero. */ 2778 rtx temp = gen_reg_rtx (VNx4SImode); 2779 emit_insn ((<hi_lanes_optab> 2780 ? gen_aarch64_sve_zip2vnx4si 2781 : gen_aarch64_sve_zip1vnx4si) 2782 (temp, operands[1], operands[1])); 2783 rtx ptrue = force_reg (VNx2BImode, CONSTM1_RTX (VNx2BImode)); 2784 emit_insn (gen_aarch64_sve_<FLOATUORS:optab>vnx4sivnx2df2 (operands[0], 2785 ptrue, temp)); 2786 DONE; 2787 } 2788 ) 2789 2790 ;; Predicate pack. Use UZP1 on the narrower type, which discards 2791 ;; the high part of each wide element. 2792 (define_insn "vec_pack_trunc_<Vwide>" 2793 [(set (match_operand:PRED_BHS 0 "register_operand" "=Upa") 2794 (unspec:PRED_BHS 2795 [(match_operand:<VWIDE> 1 "register_operand" "Upa") 2796 (match_operand:<VWIDE> 2 "register_operand" "Upa")] 2797 UNSPEC_PACK))] 2798 "TARGET_SVE" 2799 "uzp1\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>" 2800 ) 2801 2802 ;; Integer pack. Use UZP1 on the narrower type, which discards 2803 ;; the high part of each wide element. 2804 (define_insn "vec_pack_trunc_<Vwide>" 2805 [(set (match_operand:SVE_BHSI 0 "register_operand" "=w") 2806 (unspec:SVE_BHSI 2807 [(match_operand:<VWIDE> 1 "register_operand" "w") 2808 (match_operand:<VWIDE> 2 "register_operand" "w")] 2809 UNSPEC_PACK))] 2810 "TARGET_SVE" 2811 "uzp1\t%0.<Vetype>, %1.<Vetype>, %2.<Vetype>" 2812 ) 2813 2814 ;; Convert two vectors of DF to SF, or two vectors of SF to HF, and pack 2815 ;; the results into a single vector. 2816 (define_expand "vec_pack_trunc_<Vwide>" 2817 [(set (match_dup 4) 2818 (unspec:SVE_HSF 2819 [(match_dup 3) 2820 (unspec:SVE_HSF [(match_operand:<VWIDE> 1 "register_operand")] 2821 UNSPEC_FLOAT_CONVERT)] 2822 UNSPEC_MERGE_PTRUE)) 2823 (set (match_dup 5) 2824 (unspec:SVE_HSF 2825 [(match_dup 3) 2826 (unspec:SVE_HSF [(match_operand:<VWIDE> 2 "register_operand")] 2827 UNSPEC_FLOAT_CONVERT)] 2828 UNSPEC_MERGE_PTRUE)) 2829 (set (match_operand:SVE_HSF 0 "register_operand") 2830 (unspec:SVE_HSF [(match_dup 4) (match_dup 5)] UNSPEC_UZP1))] 2831 "TARGET_SVE" 2832 { 2833 operands[3] = force_reg (<VWIDE_PRED>mode, CONSTM1_RTX (<VWIDE_PRED>mode)); 2834 operands[4] = gen_reg_rtx (<MODE>mode); 2835 operands[5] = gen_reg_rtx (<MODE>mode); 2836 } 2837 ) 2838 2839 ;; Convert two vectors of DF to SI and pack the results into a single vector. 2840 (define_expand "vec_pack_<su>fix_trunc_vnx2df" 2841 [(set (match_dup 4) 2842 (unspec:VNx4SI 2843 [(match_dup 3) 2844 (FIXUORS:VNx4SI (match_operand:VNx2DF 1 "register_operand"))] 2845 UNSPEC_MERGE_PTRUE)) 2846 (set (match_dup 5) 2847 (unspec:VNx4SI 2848 [(match_dup 3) 2849 (FIXUORS:VNx4SI (match_operand:VNx2DF 2 "register_operand"))] 2850 UNSPEC_MERGE_PTRUE)) 2851 (set (match_operand:VNx4SI 0 "register_operand") 2852 (unspec:VNx4SI [(match_dup 4) (match_dup 5)] UNSPEC_UZP1))] 2853 "TARGET_SVE" 2854 { 2855 operands[3] = force_reg (VNx2BImode, CONSTM1_RTX (VNx2BImode)); 2856 operands[4] = gen_reg_rtx (VNx4SImode); 2857 operands[5] = gen_reg_rtx (VNx4SImode); 2858 } 2859 ) 2860 2861 ;; Predicated floating-point operations with select. 2862 (define_expand "cond_<optab><mode>" 2863 [(set (match_operand:SVE_F 0 "register_operand") 2864 (unspec:SVE_F 2865 [(match_operand:<VPRED> 1 "register_operand") 2866 (unspec:SVE_F 2867 [(match_operand:SVE_F 2 "register_operand") 2868 (match_operand:SVE_F 3 "register_operand")] 2869 SVE_COND_FP_BINARY) 2870 (match_operand:SVE_F 4 "aarch64_simd_reg_or_zero")] 2871 UNSPEC_SEL))] 2872 "TARGET_SVE" 2873 ) 2874 2875 ;; Predicated floating-point operations with select matching output. 2876 (define_insn "*cond_<optab><mode>_0" 2877 [(set (match_operand:SVE_F 0 "register_operand" "+w, w, ?&w") 2878 (unspec:SVE_F 2879 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl") 2880 (unspec:SVE_F 2881 [(match_operand:SVE_F 2 "register_operand" "0, w, w") 2882 (match_operand:SVE_F 3 "register_operand" "w, 0, w")] 2883 SVE_COND_FP_BINARY) 2884 (match_dup 0)] 2885 UNSPEC_SEL))] 2886 "TARGET_SVE" 2887 "@ 2888 <sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype> 2889 <sve_fp_op_rev>\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype> 2890 movprfx\t%0, %1/m, %2\;<sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>" 2891 [(set_attr "movprfx" "*,*,yes")] 2892 ) 2893 2894 ;; Predicated floating-point operations with select matching first operand. 2895 (define_insn "*cond_<optab><mode>_2" 2896 [(set (match_operand:SVE_F 0 "register_operand" "=w, ?&w") 2897 (unspec:SVE_F 2898 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl") 2899 (unspec:SVE_F 2900 [(match_operand:SVE_F 2 "register_operand" "0, w") 2901 (match_operand:SVE_F 3 "register_operand" "w, w")] 2902 SVE_COND_FP_BINARY) 2903 (match_dup 2)] 2904 UNSPEC_SEL))] 2905 "TARGET_SVE" 2906 "@ 2907 <sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype> 2908 movprfx\t%0, %2\;<sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>" 2909 [(set_attr "movprfx" "*,yes")] 2910 ) 2911 2912 ;; Predicated floating-point operations with select matching second operand. 2913 (define_insn "*cond_<optab><mode>_3" 2914 [(set (match_operand:SVE_F 0 "register_operand" "=w, ?&w") 2915 (unspec:SVE_F 2916 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl") 2917 (unspec:SVE_F 2918 [(match_operand:SVE_F 2 "register_operand" "w, w") 2919 (match_operand:SVE_F 3 "register_operand" "0, w")] 2920 SVE_COND_FP_BINARY) 2921 (match_dup 3)] 2922 UNSPEC_SEL))] 2923 "TARGET_SVE" 2924 "@ 2925 <sve_fp_op_rev>\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype> 2926 movprfx\t%0, %3\;<sve_fp_op_rev>\t%0.<Vetype>, %1/m, %0.<Vetype>, %2.<Vetype>" 2927 [(set_attr "movprfx" "*,yes")] 2928 ) 2929 2930 ;; Predicated floating-point operations with select matching zero. 2931 (define_insn "*cond_<optab><mode>_z" 2932 [(set (match_operand:SVE_F 0 "register_operand" "=&w") 2933 (unspec:SVE_F 2934 [(match_operand:<VPRED> 1 "register_operand" "Upl") 2935 (unspec:SVE_F 2936 [(match_operand:SVE_F 2 "register_operand" "w") 2937 (match_operand:SVE_F 3 "register_operand" "w")] 2938 SVE_COND_FP_BINARY) 2939 (match_operand:SVE_F 4 "aarch64_simd_imm_zero")] 2940 UNSPEC_SEL))] 2941 "TARGET_SVE" 2942 "movprfx\t%0.<Vetype>, %1/z, %2.<Vetype>\;<sve_fp_op>\t%0.<Vetype>, %1/m, %0.<Vetype>, %3.<Vetype>" 2943 [(set_attr "movprfx" "yes")] 2944 ) 2945 2946 ;; Synthetic predication of floating-point operations with select unmatched. 2947 (define_insn_and_split "*cond_<optab><mode>_any" 2948 [(set (match_operand:SVE_F 0 "register_operand" "=&w") 2949 (unspec:SVE_F 2950 [(match_operand:<VPRED> 1 "register_operand" "Upl") 2951 (unspec:SVE_F 2952 [(match_operand:SVE_F 2 "register_operand" "w") 2953 (match_operand:SVE_F 3 "register_operand" "w")] 2954 SVE_COND_FP_BINARY) 2955 (match_operand:SVE_F 4 "register_operand" "w")] 2956 UNSPEC_SEL))] 2957 "TARGET_SVE" 2958 "#" 2959 "&& reload_completed 2960 && !(rtx_equal_p (operands[0], operands[4]) 2961 || rtx_equal_p (operands[2], operands[4]) 2962 || rtx_equal_p (operands[3], operands[4]))" 2963 ; Not matchable by any one insn or movprfx insn. We need a separate select. 2964 [(set (match_dup 0) 2965 (unspec:SVE_F [(match_dup 1) (match_dup 2) (match_dup 4)] UNSPEC_SEL)) 2966 (set (match_dup 0) 2967 (unspec:SVE_F 2968 [(match_dup 1) 2969 (unspec:SVE_F [(match_dup 0) (match_dup 3)] SVE_COND_FP_BINARY) 2970 (match_dup 0)] 2971 UNSPEC_SEL))] 2972 ) 2973 2974 ;; Predicated floating-point ternary operations with select. 2975 (define_expand "cond_<optab><mode>" 2976 [(set (match_operand:SVE_F 0 "register_operand") 2977 (unspec:SVE_F 2978 [(match_operand:<VPRED> 1 "register_operand") 2979 (unspec:SVE_F 2980 [(match_operand:SVE_F 2 "register_operand") 2981 (match_operand:SVE_F 3 "register_operand") 2982 (match_operand:SVE_F 4 "register_operand")] 2983 SVE_COND_FP_TERNARY) 2984 (match_operand:SVE_F 5 "aarch64_simd_reg_or_zero")] 2985 UNSPEC_SEL))] 2986 "TARGET_SVE" 2987 { 2988 /* Swap the multiplication operands if the fallback value is the 2989 second of the two. */ 2990 if (rtx_equal_p (operands[3], operands[5])) 2991 std::swap (operands[2], operands[3]); 2992 }) 2993 2994 ;; Predicated floating-point ternary operations using the FMAD-like form. 2995 (define_insn "*cond_<optab><mode>_2" 2996 [(set (match_operand:SVE_F 0 "register_operand" "=w, ?&w") 2997 (unspec:SVE_F 2998 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl") 2999 (unspec:SVE_F 3000 [(match_operand:SVE_F 2 "register_operand" "0, w") 3001 (match_operand:SVE_F 3 "register_operand" "w, w") 3002 (match_operand:SVE_F 4 "register_operand" "w, w")] 3003 SVE_COND_FP_TERNARY) 3004 (match_dup 2)] 3005 UNSPEC_SEL))] 3006 "TARGET_SVE" 3007 "@ 3008 <sve_fmad_op>\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype> 3009 movprfx\t%0, %2\;<sve_fmad_op>\t%0.<Vetype>, %1/m, %3.<Vetype>, %4.<Vetype>" 3010 [(set_attr "movprfx" "*,yes")] 3011 ) 3012 3013 ;; Predicated floating-point ternary operations using the FMLA-like form. 3014 (define_insn "*cond_<optab><mode>_4" 3015 [(set (match_operand:SVE_F 0 "register_operand" "=w, ?&w") 3016 (unspec:SVE_F 3017 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl") 3018 (unspec:SVE_F 3019 [(match_operand:SVE_F 2 "register_operand" "w, w") 3020 (match_operand:SVE_F 3 "register_operand" "w, w") 3021 (match_operand:SVE_F 4 "register_operand" "0, w")] 3022 SVE_COND_FP_TERNARY) 3023 (match_dup 4)] 3024 UNSPEC_SEL))] 3025 "TARGET_SVE" 3026 "@ 3027 <sve_fmla_op>\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype> 3028 movprfx\t%0, %4\;<sve_fmla_op>\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype>" 3029 [(set_attr "movprfx" "*,yes")] 3030 ) 3031 3032 ;; Predicated floating-point ternary operations in which the value for 3033 ;; inactive lanes is distinct from the other inputs. 3034 (define_insn_and_split "*cond_<optab><mode>_any" 3035 [(set (match_operand:SVE_F 0 "register_operand" "=&w, &w, ?&w") 3036 (unspec:SVE_F 3037 [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl, Upl") 3038 (unspec:SVE_F 3039 [(match_operand:SVE_F 2 "register_operand" "w, w, w") 3040 (match_operand:SVE_F 3 "register_operand" "w, w, w") 3041 (match_operand:SVE_F 4 "register_operand" "w, w, w")] 3042 SVE_COND_FP_TERNARY) 3043 (match_operand:SVE_F 5 "aarch64_simd_reg_or_zero" "Dz, 0, w")] 3044 UNSPEC_SEL))] 3045 "TARGET_SVE 3046 && !rtx_equal_p (operands[2], operands[5]) 3047 && !rtx_equal_p (operands[3], operands[5]) 3048 && !rtx_equal_p (operands[4], operands[5])" 3049 "@ 3050 movprfx\t%0.<Vetype>, %1/z, %4.<Vetype>\;<sve_fmla_op>\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype> 3051 movprfx\t%0.<Vetype>, %1/m, %4.<Vetype>\;<sve_fmla_op>\t%0.<Vetype>, %1/m, %2.<Vetype>, %3.<Vetype> 3052 #" 3053 "&& reload_completed 3054 && !CONSTANT_P (operands[5]) 3055 && !rtx_equal_p (operands[0], operands[5])" 3056 [(set (match_dup 0) 3057 (unspec:SVE_F [(match_dup 1) (match_dup 4) (match_dup 5)] UNSPEC_SEL)) 3058 (set (match_dup 0) 3059 (unspec:SVE_F 3060 [(match_dup 1) 3061 (unspec:SVE_F [(match_dup 2) (match_dup 3) (match_dup 0)] 3062 SVE_COND_FP_TERNARY) 3063 (match_dup 0)] 3064 UNSPEC_SEL))] 3065 "" 3066 [(set_attr "movprfx" "yes")] 3067 ) 3068 3069 ;; Shift an SVE vector left and insert a scalar into element 0. 3070 (define_insn "vec_shl_insert_<mode>" 3071 [(set (match_operand:SVE_ALL 0 "register_operand" "=w, w") 3072 (unspec:SVE_ALL 3073 [(match_operand:SVE_ALL 1 "register_operand" "0, 0") 3074 (match_operand:<VEL> 2 "register_operand" "rZ, w")] 3075 UNSPEC_INSR))] 3076 "TARGET_SVE" 3077 "@ 3078 insr\t%0.<Vetype>, %<vwcore>2 3079 insr\t%0.<Vetype>, %<Vetype>2" 3080 ) 3081 3082 (define_expand "copysign<mode>3" 3083 [(match_operand:SVE_F 0 "register_operand") 3084 (match_operand:SVE_F 1 "register_operand") 3085 (match_operand:SVE_F 2 "register_operand")] 3086 "TARGET_SVE" 3087 { 3088 rtx sign = gen_reg_rtx (<V_INT_EQUIV>mode); 3089 rtx mant = gen_reg_rtx (<V_INT_EQUIV>mode); 3090 rtx int_res = gen_reg_rtx (<V_INT_EQUIV>mode); 3091 int bits = GET_MODE_UNIT_BITSIZE (<MODE>mode) - 1; 3092 3093 rtx arg1 = lowpart_subreg (<V_INT_EQUIV>mode, operands[1], <MODE>mode); 3094 rtx arg2 = lowpart_subreg (<V_INT_EQUIV>mode, operands[2], <MODE>mode); 3095 3096 emit_insn (gen_and<v_int_equiv>3 3097 (sign, arg2, 3098 aarch64_simd_gen_const_vector_dup (<V_INT_EQUIV>mode, 3099 HOST_WIDE_INT_M1U 3100 << bits))); 3101 emit_insn (gen_and<v_int_equiv>3 3102 (mant, arg1, 3103 aarch64_simd_gen_const_vector_dup (<V_INT_EQUIV>mode, 3104 ~(HOST_WIDE_INT_M1U 3105 << bits)))); 3106 emit_insn (gen_ior<v_int_equiv>3 (int_res, sign, mant)); 3107 emit_move_insn (operands[0], gen_lowpart (<MODE>mode, int_res)); 3108 DONE; 3109 } 3110 ) 3111 3112 (define_expand "xorsign<mode>3" 3113 [(match_operand:SVE_F 0 "register_operand") 3114 (match_operand:SVE_F 1 "register_operand") 3115 (match_operand:SVE_F 2 "register_operand")] 3116 "TARGET_SVE" 3117 { 3118 rtx sign = gen_reg_rtx (<V_INT_EQUIV>mode); 3119 rtx int_res = gen_reg_rtx (<V_INT_EQUIV>mode); 3120 int bits = GET_MODE_UNIT_BITSIZE (<MODE>mode) - 1; 3121 3122 rtx arg1 = lowpart_subreg (<V_INT_EQUIV>mode, operands[1], <MODE>mode); 3123 rtx arg2 = lowpart_subreg (<V_INT_EQUIV>mode, operands[2], <MODE>mode); 3124 3125 emit_insn (gen_and<v_int_equiv>3 3126 (sign, arg2, 3127 aarch64_simd_gen_const_vector_dup (<V_INT_EQUIV>mode, 3128 HOST_WIDE_INT_M1U 3129 << bits))); 3130 emit_insn (gen_xor<v_int_equiv>3 (int_res, arg1, sign)); 3131 emit_move_insn (operands[0], gen_lowpart (<MODE>mode, int_res)); 3132 DONE; 3133 } 3134 ) 3135
Indexes created Thu Aug 13 00:25:44 UTC 2026