1 /* Builtins' description for AArch64 SIMD architecture. 2 Copyright (C) 2011-2024 Free Software Foundation, Inc. 3 Contributed by ARM Ltd. 4 5 This file is part of GCC. 6 7 GCC is free software; you can redistribute it and/or modify it 8 under the terms of the GNU General Public License as published by 9 the Free Software Foundation; either version 3, or (at your option) 10 any later version. 11 12 GCC is distributed in the hope that it will be useful, but 13 WITHOUT ANY WARRANTY; without even the implied warranty of 14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 General Public License for more details. 16 17 You should have received a copy of the GNU General Public License 18 along with GCC; see the file COPYING3. If not see 19 <http://www.gnu.org/licenses/>. */ 20 21 #define IN_TARGET_CODE 1 22 23 #include "config.h" 24 #include "system.h" 25 #include "coretypes.h" 26 #include "tm.h" 27 #include "function.h" 28 #include "basic-block.h" 29 #include "rtl.h" 30 #include "tree.h" 31 #include "gimple.h" 32 #include "ssa.h" 33 #include "memmodel.h" 34 #include "tm_p.h" 35 #include "expmed.h" 36 #include "optabs.h" 37 #include "recog.h" 38 #include "diagnostic-core.h" 39 #include "fold-const.h" 40 #include "stor-layout.h" 41 #include "explow.h" 42 #include "expr.h" 43 #include "langhooks.h" 44 #include "gimple-iterator.h" 45 #include "case-cfn-macros.h" 46 #include "emit-rtl.h" 47 #include "stringpool.h" 48 #include "attribs.h" 49 #include "gimple-fold.h" 50 #include "builtins.h" 51 #include "aarch64-builtins.h" 52 53 #define v8qi_UP E_V8QImode 54 #define v8di_UP E_V8DImode 55 #define v4hi_UP E_V4HImode 56 #define v4hf_UP E_V4HFmode 57 #define v2si_UP E_V2SImode 58 #define v2sf_UP E_V2SFmode 59 #define v1df_UP E_V1DFmode 60 #define v1di_UP E_V1DImode 61 #define di_UP E_DImode 62 #define df_UP E_DFmode 63 #define v16qi_UP E_V16QImode 64 #define v8hi_UP E_V8HImode 65 #define v8hf_UP E_V8HFmode 66 #define v4si_UP E_V4SImode 67 #define v4sf_UP E_V4SFmode 68 #define v2di_UP E_V2DImode 69 #define v2df_UP E_V2DFmode 70 #define ti_UP E_TImode 71 #define oi_UP E_OImode 72 #define ci_UP E_CImode 73 #define xi_UP E_XImode 74 #define si_UP E_SImode 75 #define sf_UP E_SFmode 76 #define hi_UP E_HImode 77 #define hf_UP E_HFmode 78 #define qi_UP E_QImode 79 #define bf_UP E_BFmode 80 #define v4bf_UP E_V4BFmode 81 #define v8bf_UP E_V8BFmode 82 #define v2x8qi_UP E_V2x8QImode 83 #define v2x4hi_UP E_V2x4HImode 84 #define v2x4hf_UP E_V2x4HFmode 85 #define v2x4bf_UP E_V2x4BFmode 86 #define v2x2si_UP E_V2x2SImode 87 #define v2x2sf_UP E_V2x2SFmode 88 #define v2x1di_UP E_V2x1DImode 89 #define v2x1df_UP E_V2x1DFmode 90 #define v2x16qi_UP E_V2x16QImode 91 #define v2x8hi_UP E_V2x8HImode 92 #define v2x8hf_UP E_V2x8HFmode 93 #define v2x8bf_UP E_V2x8BFmode 94 #define v2x4si_UP E_V2x4SImode 95 #define v2x4sf_UP E_V2x4SFmode 96 #define v2x2di_UP E_V2x2DImode 97 #define v2x2df_UP E_V2x2DFmode 98 #define v3x8qi_UP E_V3x8QImode 99 #define v3x4hi_UP E_V3x4HImode 100 #define v3x4hf_UP E_V3x4HFmode 101 #define v3x4bf_UP E_V3x4BFmode 102 #define v3x2si_UP E_V3x2SImode 103 #define v3x2sf_UP E_V3x2SFmode 104 #define v3x1di_UP E_V3x1DImode 105 #define v3x1df_UP E_V3x1DFmode 106 #define v3x16qi_UP E_V3x16QImode 107 #define v3x8hi_UP E_V3x8HImode 108 #define v3x8hf_UP E_V3x8HFmode 109 #define v3x8bf_UP E_V3x8BFmode 110 #define v3x4si_UP E_V3x4SImode 111 #define v3x4sf_UP E_V3x4SFmode 112 #define v3x2di_UP E_V3x2DImode 113 #define v3x2df_UP E_V3x2DFmode 114 #define v4x8qi_UP E_V4x8QImode 115 #define v4x4hi_UP E_V4x4HImode 116 #define v4x4hf_UP E_V4x4HFmode 117 #define v4x4bf_UP E_V4x4BFmode 118 #define v4x2si_UP E_V4x2SImode 119 #define v4x2sf_UP E_V4x2SFmode 120 #define v4x1di_UP E_V4x1DImode 121 #define v4x1df_UP E_V4x1DFmode 122 #define v4x16qi_UP E_V4x16QImode 123 #define v4x8hi_UP E_V4x8HImode 124 #define v4x8hf_UP E_V4x8HFmode 125 #define v4x8bf_UP E_V4x8BFmode 126 #define v4x4si_UP E_V4x4SImode 127 #define v4x4sf_UP E_V4x4SFmode 128 #define v4x2di_UP E_V4x2DImode 129 #define v4x2df_UP E_V4x2DFmode 130 #define UP(X) X##_UP 131 132 #define MODE_d_bf16 E_V4BFmode 133 #define MODE_d_f16 E_V4HFmode 134 #define MODE_d_f32 E_V2SFmode 135 #define MODE_d_f64 E_V1DFmode 136 #define MODE_d_s8 E_V8QImode 137 #define MODE_d_s16 E_V4HImode 138 #define MODE_d_s32 E_V2SImode 139 #define MODE_d_s64 E_V1DImode 140 #define MODE_d_u8 E_V8QImode 141 #define MODE_d_u16 E_V4HImode 142 #define MODE_d_u32 E_V2SImode 143 #define MODE_d_u64 E_V1DImode 144 #define MODE_d_p8 E_V8QImode 145 #define MODE_d_p16 E_V4HImode 146 #define MODE_d_p64 E_V1DImode 147 #define MODE_q_bf16 E_V8BFmode 148 #define MODE_q_f16 E_V8HFmode 149 #define MODE_q_f32 E_V4SFmode 150 #define MODE_q_f64 E_V2DFmode 151 #define MODE_q_s8 E_V16QImode 152 #define MODE_q_s16 E_V8HImode 153 #define MODE_q_s32 E_V4SImode 154 #define MODE_q_s64 E_V2DImode 155 #define MODE_q_u8 E_V16QImode 156 #define MODE_q_u16 E_V8HImode 157 #define MODE_q_u32 E_V4SImode 158 #define MODE_q_u64 E_V2DImode 159 #define MODE_q_p8 E_V16QImode 160 #define MODE_q_p16 E_V8HImode 161 #define MODE_q_p64 E_V2DImode 162 #define MODE_q_p128 E_TImode 163 164 #define QUAL_bf16 qualifier_none 165 #define QUAL_f16 qualifier_none 166 #define QUAL_f32 qualifier_none 167 #define QUAL_f64 qualifier_none 168 #define QUAL_s8 qualifier_none 169 #define QUAL_s16 qualifier_none 170 #define QUAL_s32 qualifier_none 171 #define QUAL_s64 qualifier_none 172 #define QUAL_u8 qualifier_unsigned 173 #define QUAL_u16 qualifier_unsigned 174 #define QUAL_u32 qualifier_unsigned 175 #define QUAL_u64 qualifier_unsigned 176 #define QUAL_p8 qualifier_poly 177 #define QUAL_p16 qualifier_poly 178 #define QUAL_p64 qualifier_poly 179 #define QUAL_p128 qualifier_poly 180 181 #define LENGTH_d "" 182 #define LENGTH_q "q" 183 184 #define SIMD_INTR_MODE(suffix, length) MODE_##length##_##suffix 185 #define SIMD_INTR_QUAL(suffix) QUAL_##suffix 186 #define SIMD_INTR_LENGTH_CHAR(length) LENGTH_##length 187 188 #define SIMD_MAX_BUILTIN_ARGS 5 189 190 /* Flags that describe what a function might do. */ 191 const unsigned int FLAG_NONE = 0U; 192 const unsigned int FLAG_READ_FPCR = 1U << 0; 193 const unsigned int FLAG_RAISE_FP_EXCEPTIONS = 1U << 1; 194 const unsigned int FLAG_READ_MEMORY = 1U << 2; 195 const unsigned int FLAG_PREFETCH_MEMORY = 1U << 3; 196 const unsigned int FLAG_WRITE_MEMORY = 1U << 4; 197 198 /* Not all FP intrinsics raise FP exceptions or read FPCR register, 199 use this flag to suppress it. */ 200 const unsigned int FLAG_AUTO_FP = 1U << 5; 201 202 const unsigned int FLAG_FP = FLAG_READ_FPCR | FLAG_RAISE_FP_EXCEPTIONS; 203 const unsigned int FLAG_ALL = FLAG_READ_FPCR | FLAG_RAISE_FP_EXCEPTIONS 204 | FLAG_READ_MEMORY | FLAG_PREFETCH_MEMORY | FLAG_WRITE_MEMORY; 205 const unsigned int FLAG_STORE = FLAG_WRITE_MEMORY | FLAG_AUTO_FP; 206 const unsigned int FLAG_LOAD = FLAG_READ_MEMORY | FLAG_AUTO_FP; 207 208 typedef struct 209 { 210 const char *name; 211 machine_mode mode; 212 const enum insn_code code; 213 unsigned int fcode; 214 enum aarch64_type_qualifiers *qualifiers; 215 unsigned int flags; 216 } aarch64_simd_builtin_datum; 217 218 static enum aarch64_type_qualifiers 219 aarch64_types_unop_qualifiers[SIMD_MAX_BUILTIN_ARGS] 220 = { qualifier_none, qualifier_none }; 221 #define TYPES_UNOP (aarch64_types_unop_qualifiers) 222 static enum aarch64_type_qualifiers 223 aarch64_types_unopu_qualifiers[SIMD_MAX_BUILTIN_ARGS] 224 = { qualifier_unsigned, qualifier_unsigned }; 225 #define TYPES_UNOPU (aarch64_types_unopu_qualifiers) 226 static enum aarch64_type_qualifiers 227 aarch64_types_unopus_qualifiers[SIMD_MAX_BUILTIN_ARGS] 228 = { qualifier_unsigned, qualifier_none }; 229 #define TYPES_UNOPUS (aarch64_types_unopus_qualifiers) 230 static enum aarch64_type_qualifiers 231 aarch64_types_binop_qualifiers[SIMD_MAX_BUILTIN_ARGS] 232 = { qualifier_none, qualifier_none, qualifier_maybe_immediate }; 233 #define TYPES_BINOP (aarch64_types_binop_qualifiers) 234 static enum aarch64_type_qualifiers 235 aarch64_types_binopu_qualifiers[SIMD_MAX_BUILTIN_ARGS] 236 = { qualifier_unsigned, qualifier_unsigned, qualifier_unsigned }; 237 #define TYPES_BINOPU (aarch64_types_binopu_qualifiers) 238 static enum aarch64_type_qualifiers 239 aarch64_types_binop_uus_qualifiers[SIMD_MAX_BUILTIN_ARGS] 240 = { qualifier_unsigned, qualifier_unsigned, qualifier_none }; 241 #define TYPES_BINOP_UUS (aarch64_types_binop_uus_qualifiers) 242 static enum aarch64_type_qualifiers 243 aarch64_types_binop_ssu_qualifiers[SIMD_MAX_BUILTIN_ARGS] 244 = { qualifier_none, qualifier_none, qualifier_unsigned }; 245 #define TYPES_BINOP_SSU (aarch64_types_binop_ssu_qualifiers) 246 static enum aarch64_type_qualifiers 247 aarch64_types_binop_uss_qualifiers[SIMD_MAX_BUILTIN_ARGS] 248 = { qualifier_unsigned, qualifier_none, qualifier_none }; 249 #define TYPES_BINOP_USS (aarch64_types_binop_uss_qualifiers) 250 static enum aarch64_type_qualifiers 251 aarch64_types_binopp_qualifiers[SIMD_MAX_BUILTIN_ARGS] 252 = { qualifier_poly, qualifier_poly, qualifier_poly }; 253 #define TYPES_BINOPP (aarch64_types_binopp_qualifiers) 254 static enum aarch64_type_qualifiers 255 aarch64_types_binop_ppu_qualifiers[SIMD_MAX_BUILTIN_ARGS] 256 = { qualifier_poly, qualifier_poly, qualifier_unsigned }; 257 #define TYPES_BINOP_PPU (aarch64_types_binop_ppu_qualifiers) 258 259 static enum aarch64_type_qualifiers 260 aarch64_types_ternop_qualifiers[SIMD_MAX_BUILTIN_ARGS] 261 = { qualifier_none, qualifier_none, qualifier_none, qualifier_none }; 262 #define TYPES_TERNOP (aarch64_types_ternop_qualifiers) 263 static enum aarch64_type_qualifiers 264 aarch64_types_ternop_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS] 265 = { qualifier_none, qualifier_none, qualifier_none, qualifier_lane_index }; 266 #define TYPES_TERNOP_LANE (aarch64_types_ternop_lane_qualifiers) 267 static enum aarch64_type_qualifiers 268 aarch64_types_ternopu_qualifiers[SIMD_MAX_BUILTIN_ARGS] 269 = { qualifier_unsigned, qualifier_unsigned, 270 qualifier_unsigned, qualifier_unsigned }; 271 #define TYPES_TERNOPU (aarch64_types_ternopu_qualifiers) 272 static enum aarch64_type_qualifiers 273 aarch64_types_ternopu_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS] 274 = { qualifier_unsigned, qualifier_unsigned, 275 qualifier_unsigned, qualifier_lane_index }; 276 #define TYPES_TERNOPU_LANE (aarch64_types_ternopu_lane_qualifiers) 277 static enum aarch64_type_qualifiers 278 aarch64_types_ternopu_imm_qualifiers[SIMD_MAX_BUILTIN_ARGS] 279 = { qualifier_unsigned, qualifier_unsigned, 280 qualifier_unsigned, qualifier_immediate }; 281 #define TYPES_TERNOPUI (aarch64_types_ternopu_imm_qualifiers) 282 static enum aarch64_type_qualifiers 283 aarch64_types_ternop_sssu_qualifiers[SIMD_MAX_BUILTIN_ARGS] 284 = { qualifier_none, qualifier_none, qualifier_none, qualifier_unsigned }; 285 #define TYPES_TERNOP_SSSU (aarch64_types_ternop_sssu_qualifiers) 286 static enum aarch64_type_qualifiers 287 aarch64_types_ternop_ssus_qualifiers[SIMD_MAX_BUILTIN_ARGS] 288 = { qualifier_none, qualifier_none, qualifier_unsigned, qualifier_none }; 289 #define TYPES_TERNOP_SSUS (aarch64_types_ternop_ssus_qualifiers) 290 static enum aarch64_type_qualifiers 291 aarch64_types_ternop_suss_qualifiers[SIMD_MAX_BUILTIN_ARGS] 292 = { qualifier_none, qualifier_unsigned, qualifier_none, qualifier_none }; 293 #define TYPES_TERNOP_SUSS (aarch64_types_ternop_suss_qualifiers) 294 static enum aarch64_type_qualifiers 295 aarch64_types_binop_pppu_qualifiers[SIMD_MAX_BUILTIN_ARGS] 296 = { qualifier_poly, qualifier_poly, qualifier_poly, qualifier_unsigned }; 297 #define TYPES_TERNOP_PPPU (aarch64_types_binop_pppu_qualifiers) 298 299 static enum aarch64_type_qualifiers 300 aarch64_types_quadop_lane_pair_qualifiers[SIMD_MAX_BUILTIN_ARGS] 301 = { qualifier_none, qualifier_none, qualifier_none, 302 qualifier_none, qualifier_lane_pair_index }; 303 #define TYPES_QUADOP_LANE_PAIR (aarch64_types_quadop_lane_pair_qualifiers) 304 static enum aarch64_type_qualifiers 305 aarch64_types_quadop_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS] 306 = { qualifier_none, qualifier_none, qualifier_none, 307 qualifier_none, qualifier_lane_index }; 308 #define TYPES_QUADOP_LANE (aarch64_types_quadop_lane_qualifiers) 309 static enum aarch64_type_qualifiers 310 aarch64_types_quadopu_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS] 311 = { qualifier_unsigned, qualifier_unsigned, qualifier_unsigned, 312 qualifier_unsigned, qualifier_lane_index }; 313 #define TYPES_QUADOPU_LANE (aarch64_types_quadopu_lane_qualifiers) 314 315 static enum aarch64_type_qualifiers 316 aarch64_types_quadopssus_lane_quadtup_qualifiers[SIMD_MAX_BUILTIN_ARGS] 317 = { qualifier_none, qualifier_none, qualifier_unsigned, 318 qualifier_none, qualifier_lane_quadtup_index }; 319 #define TYPES_QUADOPSSUS_LANE_QUADTUP \ 320 (aarch64_types_quadopssus_lane_quadtup_qualifiers) 321 static enum aarch64_type_qualifiers 322 aarch64_types_quadopsssu_lane_quadtup_qualifiers[SIMD_MAX_BUILTIN_ARGS] 323 = { qualifier_none, qualifier_none, qualifier_none, 324 qualifier_unsigned, qualifier_lane_quadtup_index }; 325 #define TYPES_QUADOPSSSU_LANE_QUADTUP \ 326 (aarch64_types_quadopsssu_lane_quadtup_qualifiers) 327 328 static enum aarch64_type_qualifiers 329 aarch64_types_quadopu_imm_qualifiers[SIMD_MAX_BUILTIN_ARGS] 330 = { qualifier_unsigned, qualifier_unsigned, qualifier_unsigned, 331 qualifier_unsigned, qualifier_immediate }; 332 #define TYPES_QUADOPUI (aarch64_types_quadopu_imm_qualifiers) 333 334 static enum aarch64_type_qualifiers 335 aarch64_types_binop_imm_qualifiers[SIMD_MAX_BUILTIN_ARGS] 336 = { qualifier_none, qualifier_none, qualifier_immediate }; 337 #define TYPES_GETREG (aarch64_types_binop_imm_qualifiers) 338 #define TYPES_SHIFTIMM (aarch64_types_binop_imm_qualifiers) 339 static enum aarch64_type_qualifiers 340 aarch64_types_shift_to_unsigned_qualifiers[SIMD_MAX_BUILTIN_ARGS] 341 = { qualifier_unsigned, qualifier_none, qualifier_immediate }; 342 #define TYPES_SHIFTIMM_USS (aarch64_types_shift_to_unsigned_qualifiers) 343 static enum aarch64_type_qualifiers 344 aarch64_types_fcvt_from_unsigned_qualifiers[SIMD_MAX_BUILTIN_ARGS] 345 = { qualifier_none, qualifier_unsigned, qualifier_immediate }; 346 #define TYPES_FCVTIMM_SUS (aarch64_types_fcvt_from_unsigned_qualifiers) 347 static enum aarch64_type_qualifiers 348 aarch64_types_unsigned_shift_qualifiers[SIMD_MAX_BUILTIN_ARGS] 349 = { qualifier_unsigned, qualifier_unsigned, qualifier_immediate }; 350 #define TYPES_USHIFTIMM (aarch64_types_unsigned_shift_qualifiers) 351 #define TYPES_USHIFT2IMM (aarch64_types_ternopu_imm_qualifiers) 352 static enum aarch64_type_qualifiers 353 aarch64_types_shift2_to_unsigned_qualifiers[SIMD_MAX_BUILTIN_ARGS] 354 = { qualifier_unsigned, qualifier_unsigned, qualifier_none, qualifier_immediate }; 355 #define TYPES_SHIFT2IMM_UUSS (aarch64_types_shift2_to_unsigned_qualifiers) 356 357 static enum aarch64_type_qualifiers 358 aarch64_types_ternop_s_imm_qualifiers[SIMD_MAX_BUILTIN_ARGS] 359 = { qualifier_none, qualifier_none, qualifier_none, qualifier_immediate}; 360 #define TYPES_SETREG (aarch64_types_ternop_s_imm_qualifiers) 361 #define TYPES_SHIFTINSERT (aarch64_types_ternop_s_imm_qualifiers) 362 #define TYPES_SHIFTACC (aarch64_types_ternop_s_imm_qualifiers) 363 #define TYPES_SHIFT2IMM (aarch64_types_ternop_s_imm_qualifiers) 364 365 static enum aarch64_type_qualifiers 366 aarch64_types_ternop_p_imm_qualifiers[SIMD_MAX_BUILTIN_ARGS] 367 = { qualifier_poly, qualifier_poly, qualifier_poly, qualifier_immediate}; 368 #define TYPES_SHIFTINSERTP (aarch64_types_ternop_p_imm_qualifiers) 369 370 static enum aarch64_type_qualifiers 371 aarch64_types_unsigned_shiftacc_qualifiers[SIMD_MAX_BUILTIN_ARGS] 372 = { qualifier_unsigned, qualifier_unsigned, qualifier_unsigned, 373 qualifier_immediate }; 374 #define TYPES_USHIFTACC (aarch64_types_unsigned_shiftacc_qualifiers) 375 376 static enum aarch64_type_qualifiers 377 aarch64_types_load1_qualifiers[SIMD_MAX_BUILTIN_ARGS] 378 = { qualifier_none, qualifier_const_pointer_map_mode }; 379 #define TYPES_LOAD1 (aarch64_types_load1_qualifiers) 380 #define TYPES_LOADSTRUCT (aarch64_types_load1_qualifiers) 381 static enum aarch64_type_qualifiers 382 aarch64_types_load1_u_qualifiers[SIMD_MAX_BUILTIN_ARGS] 383 = { qualifier_unsigned, qualifier_const_pointer_map_mode }; 384 #define TYPES_LOAD1_U (aarch64_types_load1_u_qualifiers) 385 #define TYPES_LOADSTRUCT_U (aarch64_types_load1_u_qualifiers) 386 static enum aarch64_type_qualifiers 387 aarch64_types_load1_p_qualifiers[SIMD_MAX_BUILTIN_ARGS] 388 = { qualifier_poly, qualifier_const_pointer_map_mode }; 389 #define TYPES_LOAD1_P (aarch64_types_load1_p_qualifiers) 390 #define TYPES_LOADSTRUCT_P (aarch64_types_load1_p_qualifiers) 391 392 static enum aarch64_type_qualifiers 393 aarch64_types_loadstruct_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS] 394 = { qualifier_none, qualifier_const_pointer_map_mode, 395 qualifier_none, qualifier_struct_load_store_lane_index }; 396 #define TYPES_LOADSTRUCT_LANE (aarch64_types_loadstruct_lane_qualifiers) 397 static enum aarch64_type_qualifiers 398 aarch64_types_loadstruct_lane_u_qualifiers[SIMD_MAX_BUILTIN_ARGS] 399 = { qualifier_unsigned, qualifier_const_pointer_map_mode, 400 qualifier_unsigned, qualifier_struct_load_store_lane_index }; 401 #define TYPES_LOADSTRUCT_LANE_U (aarch64_types_loadstruct_lane_u_qualifiers) 402 static enum aarch64_type_qualifiers 403 aarch64_types_loadstruct_lane_p_qualifiers[SIMD_MAX_BUILTIN_ARGS] 404 = { qualifier_poly, qualifier_const_pointer_map_mode, 405 qualifier_poly, qualifier_struct_load_store_lane_index }; 406 #define TYPES_LOADSTRUCT_LANE_P (aarch64_types_loadstruct_lane_p_qualifiers) 407 408 static enum aarch64_type_qualifiers 409 aarch64_types_bsl_p_qualifiers[SIMD_MAX_BUILTIN_ARGS] 410 = { qualifier_poly, qualifier_unsigned, 411 qualifier_poly, qualifier_poly }; 412 #define TYPES_BSL_P (aarch64_types_bsl_p_qualifiers) 413 static enum aarch64_type_qualifiers 414 aarch64_types_bsl_s_qualifiers[SIMD_MAX_BUILTIN_ARGS] 415 = { qualifier_none, qualifier_unsigned, 416 qualifier_none, qualifier_none }; 417 #define TYPES_BSL_S (aarch64_types_bsl_s_qualifiers) 418 static enum aarch64_type_qualifiers 419 aarch64_types_bsl_u_qualifiers[SIMD_MAX_BUILTIN_ARGS] 420 = { qualifier_unsigned, qualifier_unsigned, 421 qualifier_unsigned, qualifier_unsigned }; 422 #define TYPES_BSL_U (aarch64_types_bsl_u_qualifiers) 423 424 /* The first argument (return type) of a store should be void type, 425 which we represent with qualifier_void. Their first operand will be 426 a DImode pointer to the location to store to, so we must use 427 qualifier_map_mode | qualifier_pointer to build a pointer to the 428 element type of the vector. */ 429 static enum aarch64_type_qualifiers 430 aarch64_types_store1_qualifiers[SIMD_MAX_BUILTIN_ARGS] 431 = { qualifier_void, qualifier_pointer_map_mode, qualifier_none }; 432 #define TYPES_STORE1 (aarch64_types_store1_qualifiers) 433 #define TYPES_STORESTRUCT (aarch64_types_store1_qualifiers) 434 static enum aarch64_type_qualifiers 435 aarch64_types_store1_u_qualifiers[SIMD_MAX_BUILTIN_ARGS] 436 = { qualifier_void, qualifier_pointer_map_mode, qualifier_unsigned }; 437 #define TYPES_STORE1_U (aarch64_types_store1_u_qualifiers) 438 #define TYPES_STORESTRUCT_U (aarch64_types_store1_u_qualifiers) 439 static enum aarch64_type_qualifiers 440 aarch64_types_store1_p_qualifiers[SIMD_MAX_BUILTIN_ARGS] 441 = { qualifier_void, qualifier_pointer_map_mode, qualifier_poly }; 442 #define TYPES_STORE1_P (aarch64_types_store1_p_qualifiers) 443 #define TYPES_STORESTRUCT_P (aarch64_types_store1_p_qualifiers) 444 445 static enum aarch64_type_qualifiers 446 aarch64_types_storestruct_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS] 447 = { qualifier_void, qualifier_pointer_map_mode, 448 qualifier_none, qualifier_struct_load_store_lane_index }; 449 #define TYPES_STORESTRUCT_LANE (aarch64_types_storestruct_lane_qualifiers) 450 static enum aarch64_type_qualifiers 451 aarch64_types_storestruct_lane_u_qualifiers[SIMD_MAX_BUILTIN_ARGS] 452 = { qualifier_void, qualifier_pointer_map_mode, 453 qualifier_unsigned, qualifier_struct_load_store_lane_index }; 454 #define TYPES_STORESTRUCT_LANE_U (aarch64_types_storestruct_lane_u_qualifiers) 455 static enum aarch64_type_qualifiers 456 aarch64_types_storestruct_lane_p_qualifiers[SIMD_MAX_BUILTIN_ARGS] 457 = { qualifier_void, qualifier_pointer_map_mode, 458 qualifier_poly, qualifier_struct_load_store_lane_index }; 459 #define TYPES_STORESTRUCT_LANE_P (aarch64_types_storestruct_lane_p_qualifiers) 460 461 #define CF0(N, X) CODE_FOR_aarch64_##N##X 462 #define CF1(N, X) CODE_FOR_##N##X##1 463 #define CF2(N, X) CODE_FOR_##N##X##2 464 #define CF3(N, X) CODE_FOR_##N##X##3 465 #define CF4(N, X) CODE_FOR_##N##X##4 466 #define CF10(N, X) CODE_FOR_##N##X 467 468 /* Define cascading VAR<N> macros that are used from 469 aarch64-builtin-iterators.h to iterate over modes. These definitions 470 will end up generating a number of VAR1 expansions and code later on in the 471 file should redefine VAR1 to whatever it needs to process on a per-mode 472 basis. */ 473 #define VAR2(T, N, MAP, FLAG, A, B) \ 474 VAR1 (T, N, MAP, FLAG, A) \ 475 VAR1 (T, N, MAP, FLAG, B) 476 #define VAR3(T, N, MAP, FLAG, A, B, C) \ 477 VAR2 (T, N, MAP, FLAG, A, B) \ 478 VAR1 (T, N, MAP, FLAG, C) 479 #define VAR4(T, N, MAP, FLAG, A, B, C, D) \ 480 VAR3 (T, N, MAP, FLAG, A, B, C) \ 481 VAR1 (T, N, MAP, FLAG, D) 482 #define VAR5(T, N, MAP, FLAG, A, B, C, D, E) \ 483 VAR4 (T, N, MAP, FLAG, A, B, C, D) \ 484 VAR1 (T, N, MAP, FLAG, E) 485 #define VAR6(T, N, MAP, FLAG, A, B, C, D, E, F) \ 486 VAR5 (T, N, MAP, FLAG, A, B, C, D, E) \ 487 VAR1 (T, N, MAP, FLAG, F) 488 #define VAR7(T, N, MAP, FLAG, A, B, C, D, E, F, G) \ 489 VAR6 (T, N, MAP, FLAG, A, B, C, D, E, F) \ 490 VAR1 (T, N, MAP, FLAG, G) 491 #define VAR8(T, N, MAP, FLAG, A, B, C, D, E, F, G, H) \ 492 VAR7 (T, N, MAP, FLAG, A, B, C, D, E, F, G) \ 493 VAR1 (T, N, MAP, FLAG, H) 494 #define VAR9(T, N, MAP, FLAG, A, B, C, D, E, F, G, H, I) \ 495 VAR8 (T, N, MAP, FLAG, A, B, C, D, E, F, G, H) \ 496 VAR1 (T, N, MAP, FLAG, I) 497 #define VAR10(T, N, MAP, FLAG, A, B, C, D, E, F, G, H, I, J) \ 498 VAR9 (T, N, MAP, FLAG, A, B, C, D, E, F, G, H, I) \ 499 VAR1 (T, N, MAP, FLAG, J) 500 #define VAR11(T, N, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K) \ 501 VAR10 (T, N, MAP, FLAG, A, B, C, D, E, F, G, H, I, J) \ 502 VAR1 (T, N, MAP, FLAG, K) 503 #define VAR12(T, N, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K, L) \ 504 VAR11 (T, N, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K) \ 505 VAR1 (T, N, MAP, FLAG, L) 506 #define VAR13(T, N, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K, L, M) \ 507 VAR12 (T, N, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K, L) \ 508 VAR1 (T, N, MAP, FLAG, M) 509 #define VAR14(T, X, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K, L, M, N) \ 510 VAR13 (T, X, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K, L, M) \ 511 VAR1 (T, X, MAP, FLAG, N) 512 #define VAR15(T, X, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K, L, M, N, O) \ 513 VAR14 (T, X, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K, L, M, N) \ 514 VAR1 (T, X, MAP, FLAG, O) 515 #define VAR16(T, X, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P) \ 516 VAR15 (T, X, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K, L, M, N, O) \ 517 VAR1 (T, X, MAP, FLAG, P) 518 519 #include "aarch64-builtin-iterators.h" 520 521 /* The builtins below should be expanded through the standard optabs 522 CODE_FOR_[u]avg<mode>3_[floor,ceil]. However the mapping scheme in 523 aarch64-simd-builtins.def does not easily allow us to have a pre-mode 524 ("uavg") and post-mode string ("_ceil") in the CODE_FOR_* construction. 525 So the builtins use a name that is natural for AArch64 instructions 526 e.g. "aarch64_srhadd<mode>" and we re-map these to the optab-related 527 CODE_FOR_ here. */ 528 #undef VAR1 529 #define VAR1(F,T1,T2,I,M) \ 530 constexpr insn_code CODE_FOR_aarch64_##F##M = CODE_FOR_##T1##M##3##T2; 531 532 BUILTIN_VDQ_BHSI (srhadd, avg, _ceil, 0) 533 BUILTIN_VDQ_BHSI (urhadd, uavg, _ceil, 0) 534 BUILTIN_VDQ_BHSI (shadd, avg, _floor, 0) 535 BUILTIN_VDQ_BHSI (uhadd, uavg, _floor, 0) 536 537 #undef VAR1 538 #define VAR1(T, N, MAP, FLAG, A) \ 539 {#N #A, UP (A), CF##MAP (N, A), 0, TYPES_##T, FLAG_##FLAG}, 540 541 static aarch64_simd_builtin_datum aarch64_simd_builtin_data[] = { 542 #include "aarch64-simd-builtins.def" 543 }; 544 545 /* There's only 8 CRC32 builtins. Probably not worth their own .def file. */ 546 #define AARCH64_CRC32_BUILTINS \ 547 CRC32_BUILTIN (crc32b, QI) \ 548 CRC32_BUILTIN (crc32h, HI) \ 549 CRC32_BUILTIN (crc32w, SI) \ 550 CRC32_BUILTIN (crc32x, DI) \ 551 CRC32_BUILTIN (crc32cb, QI) \ 552 CRC32_BUILTIN (crc32ch, HI) \ 553 CRC32_BUILTIN (crc32cw, SI) \ 554 CRC32_BUILTIN (crc32cx, DI) 555 556 /* The next 8 FCMLA instrinsics require some special handling compared the 557 normal simd intrinsics. */ 558 #define AARCH64_SIMD_FCMLA_LANEQ_BUILTINS \ 559 FCMLA_LANEQ_BUILTIN (0, v2sf, fcmla, V2SF, false) \ 560 FCMLA_LANEQ_BUILTIN (90, v2sf, fcmla, V2SF, false) \ 561 FCMLA_LANEQ_BUILTIN (180, v2sf, fcmla, V2SF, false) \ 562 FCMLA_LANEQ_BUILTIN (270, v2sf, fcmla, V2SF, false) \ 563 FCMLA_LANEQ_BUILTIN (0, v4hf, fcmla_laneq, V4HF, true) \ 564 FCMLA_LANEQ_BUILTIN (90, v4hf, fcmla_laneq, V4HF, true) \ 565 FCMLA_LANEQ_BUILTIN (180, v4hf, fcmla_laneq, V4HF, true) \ 566 FCMLA_LANEQ_BUILTIN (270, v4hf, fcmla_laneq, V4HF, true) \ 567 568 569 /* vreinterpret intrinsics are defined for any pair of element types. 570 { _bf16 } { _bf16 } 571 { _f16 _f32 _f64 } { _f16 _f32 _f64 } 572 { _s8 _s16 _s32 _s64 } x { _s8 _s16 _s32 _s64 } 573 { _u8 _u16 _u32 _u64 } { _u8 _u16 _u32 _u64 } 574 { _p8 _p16 _p64 } { _p8 _p16 _p64 }. */ 575 #define VREINTERPRET_BUILTIN2(A, B) \ 576 VREINTERPRET_BUILTIN (A, B, d) 577 578 #define VREINTERPRET_BUILTINS1(A) \ 579 VREINTERPRET_BUILTIN2 (A, bf16) \ 580 VREINTERPRET_BUILTIN2 (A, f16) \ 581 VREINTERPRET_BUILTIN2 (A, f32) \ 582 VREINTERPRET_BUILTIN2 (A, f64) \ 583 VREINTERPRET_BUILTIN2 (A, s8) \ 584 VREINTERPRET_BUILTIN2 (A, s16) \ 585 VREINTERPRET_BUILTIN2 (A, s32) \ 586 VREINTERPRET_BUILTIN2 (A, s64) \ 587 VREINTERPRET_BUILTIN2 (A, u8) \ 588 VREINTERPRET_BUILTIN2 (A, u16) \ 589 VREINTERPRET_BUILTIN2 (A, u32) \ 590 VREINTERPRET_BUILTIN2 (A, u64) \ 591 VREINTERPRET_BUILTIN2 (A, p8) \ 592 VREINTERPRET_BUILTIN2 (A, p16) \ 593 VREINTERPRET_BUILTIN2 (A, p64) 594 595 #define VREINTERPRET_BUILTINS \ 596 VREINTERPRET_BUILTINS1 (bf16) \ 597 VREINTERPRET_BUILTINS1 (f16) \ 598 VREINTERPRET_BUILTINS1 (f32) \ 599 VREINTERPRET_BUILTINS1 (f64) \ 600 VREINTERPRET_BUILTINS1 (s8) \ 601 VREINTERPRET_BUILTINS1 (s16) \ 602 VREINTERPRET_BUILTINS1 (s32) \ 603 VREINTERPRET_BUILTINS1 (s64) \ 604 VREINTERPRET_BUILTINS1 (u8) \ 605 VREINTERPRET_BUILTINS1 (u16) \ 606 VREINTERPRET_BUILTINS1 (u32) \ 607 VREINTERPRET_BUILTINS1 (u64) \ 608 VREINTERPRET_BUILTINS1 (p8) \ 609 VREINTERPRET_BUILTINS1 (p16) \ 610 VREINTERPRET_BUILTINS1 (p64) 611 612 /* vreinterpretq intrinsics are additionally defined for p128. 613 { _bf16 } { _bf16 } 614 { _f16 _f32 _f64 } { _f16 _f32 _f64 } 615 { _s8 _s16 _s32 _s64 } x { _s8 _s16 _s32 _s64 } 616 { _u8 _u16 _u32 _u64 } { _u8 _u16 _u32 _u64 } 617 { _p8 _p16 _p64 _p128 } { _p8 _p16 _p64 _p128 }. */ 618 #define VREINTERPRETQ_BUILTIN2(A, B) \ 619 VREINTERPRET_BUILTIN (A, B, q) 620 621 #define VREINTERPRETQ_BUILTINS1(A) \ 622 VREINTERPRETQ_BUILTIN2 (A, bf16) \ 623 VREINTERPRETQ_BUILTIN2 (A, f16) \ 624 VREINTERPRETQ_BUILTIN2 (A, f32) \ 625 VREINTERPRETQ_BUILTIN2 (A, f64) \ 626 VREINTERPRETQ_BUILTIN2 (A, s8) \ 627 VREINTERPRETQ_BUILTIN2 (A, s16) \ 628 VREINTERPRETQ_BUILTIN2 (A, s32) \ 629 VREINTERPRETQ_BUILTIN2 (A, s64) \ 630 VREINTERPRETQ_BUILTIN2 (A, u8) \ 631 VREINTERPRETQ_BUILTIN2 (A, u16) \ 632 VREINTERPRETQ_BUILTIN2 (A, u32) \ 633 VREINTERPRETQ_BUILTIN2 (A, u64) \ 634 VREINTERPRETQ_BUILTIN2 (A, p8) \ 635 VREINTERPRETQ_BUILTIN2 (A, p16) \ 636 VREINTERPRETQ_BUILTIN2 (A, p64) \ 637 VREINTERPRETQ_BUILTIN2 (A, p128) 638 639 #define VREINTERPRETQ_BUILTINS \ 640 VREINTERPRETQ_BUILTINS1 (bf16) \ 641 VREINTERPRETQ_BUILTINS1 (f16) \ 642 VREINTERPRETQ_BUILTINS1 (f32) \ 643 VREINTERPRETQ_BUILTINS1 (f64) \ 644 VREINTERPRETQ_BUILTINS1 (s8) \ 645 VREINTERPRETQ_BUILTINS1 (s16) \ 646 VREINTERPRETQ_BUILTINS1 (s32) \ 647 VREINTERPRETQ_BUILTINS1 (s64) \ 648 VREINTERPRETQ_BUILTINS1 (u8) \ 649 VREINTERPRETQ_BUILTINS1 (u16) \ 650 VREINTERPRETQ_BUILTINS1 (u32) \ 651 VREINTERPRETQ_BUILTINS1 (u64) \ 652 VREINTERPRETQ_BUILTINS1 (p8) \ 653 VREINTERPRETQ_BUILTINS1 (p16) \ 654 VREINTERPRETQ_BUILTINS1 (p64) \ 655 VREINTERPRETQ_BUILTINS1 (p128) 656 657 #define AARCH64_SIMD_VREINTERPRET_BUILTINS \ 658 VREINTERPRET_BUILTINS \ 659 VREINTERPRETQ_BUILTINS 660 661 typedef struct 662 { 663 const char *name; 664 machine_mode mode; 665 const enum insn_code icode; 666 unsigned int fcode; 667 } aarch64_crc_builtin_datum; 668 669 /* Hold information about how to expand the FCMLA_LANEQ builtins. */ 670 typedef struct 671 { 672 const char *name; 673 machine_mode mode; 674 const enum insn_code icode; 675 unsigned int fcode; 676 bool lane; 677 } aarch64_fcmla_laneq_builtin_datum; 678 679 /* Hold information about how to declare SIMD intrinsics. */ 680 typedef struct 681 { 682 const char *name; 683 unsigned int fcode; 684 unsigned int op_count; 685 machine_mode op_modes[SIMD_MAX_BUILTIN_ARGS]; 686 enum aarch64_type_qualifiers qualifiers[SIMD_MAX_BUILTIN_ARGS]; 687 unsigned int flags; 688 bool skip; 689 } aarch64_simd_intrinsic_datum; 690 691 #define CRC32_BUILTIN(N, M) \ 692 AARCH64_BUILTIN_##N, 693 694 #define FCMLA_LANEQ_BUILTIN(I, N, X, M, T) \ 695 AARCH64_SIMD_BUILTIN_FCMLA_LANEQ##I##_##M, 696 697 #define VREINTERPRET_BUILTIN(A, B, L) \ 698 AARCH64_SIMD_BUILTIN_VREINTERPRET##L##_##A##_##B, 699 700 #undef VAR1 701 #define VAR1(T, N, MAP, FLAG, A) \ 702 AARCH64_SIMD_BUILTIN_##T##_##N##A, 703 704 enum aarch64_builtins 705 { 706 AARCH64_BUILTIN_MIN, 707 708 AARCH64_BUILTIN_GET_FPCR, 709 AARCH64_BUILTIN_SET_FPCR, 710 AARCH64_BUILTIN_GET_FPSR, 711 AARCH64_BUILTIN_SET_FPSR, 712 713 AARCH64_BUILTIN_GET_FPCR64, 714 AARCH64_BUILTIN_SET_FPCR64, 715 AARCH64_BUILTIN_GET_FPSR64, 716 AARCH64_BUILTIN_SET_FPSR64, 717 718 AARCH64_BUILTIN_RSQRT_DF, 719 AARCH64_BUILTIN_RSQRT_SF, 720 AARCH64_BUILTIN_RSQRT_V2DF, 721 AARCH64_BUILTIN_RSQRT_V2SF, 722 AARCH64_BUILTIN_RSQRT_V4SF, 723 AARCH64_SIMD_BUILTIN_BASE, 724 AARCH64_SIMD_BUILTIN_LANE_CHECK, 725 #include "aarch64-simd-builtins.def" 726 /* The first enum element which is based on an insn_data pattern. */ 727 AARCH64_SIMD_PATTERN_START = AARCH64_SIMD_BUILTIN_LANE_CHECK + 1, 728 AARCH64_SIMD_BUILTIN_MAX = AARCH64_SIMD_PATTERN_START 729 + ARRAY_SIZE (aarch64_simd_builtin_data) - 1, 730 AARCH64_CRC32_BUILTIN_BASE, 731 AARCH64_CRC32_BUILTINS 732 AARCH64_CRC32_BUILTIN_MAX, 733 /* SIMD intrinsic builtins. */ 734 AARCH64_SIMD_VREINTERPRET_BUILTINS 735 /* ARMv8.3-A Pointer Authentication Builtins. */ 736 AARCH64_PAUTH_BUILTIN_AUTIA1716, 737 AARCH64_PAUTH_BUILTIN_PACIA1716, 738 AARCH64_PAUTH_BUILTIN_AUTIB1716, 739 AARCH64_PAUTH_BUILTIN_PACIB1716, 740 AARCH64_PAUTH_BUILTIN_XPACLRI, 741 /* Special cased Armv8.3-A Complex FMA by Lane quad Builtins. */ 742 AARCH64_SIMD_FCMLA_LANEQ_BUILTIN_BASE, 743 AARCH64_SIMD_FCMLA_LANEQ_BUILTINS 744 /* Builtin for Arm8.3-a Javascript conversion instruction. */ 745 AARCH64_JSCVT, 746 /* TME builtins. */ 747 AARCH64_TME_BUILTIN_TSTART, 748 AARCH64_TME_BUILTIN_TCOMMIT, 749 AARCH64_TME_BUILTIN_TTEST, 750 AARCH64_TME_BUILTIN_TCANCEL, 751 /* Armv8.5-a RNG instruction builtins. */ 752 AARCH64_BUILTIN_RNG_RNDR, 753 AARCH64_BUILTIN_RNG_RNDRRS, 754 /* MEMTAG builtins. */ 755 AARCH64_MEMTAG_BUILTIN_START, 756 AARCH64_MEMTAG_BUILTIN_IRG, 757 AARCH64_MEMTAG_BUILTIN_GMI, 758 AARCH64_MEMTAG_BUILTIN_SUBP, 759 AARCH64_MEMTAG_BUILTIN_INC_TAG, 760 AARCH64_MEMTAG_BUILTIN_SET_TAG, 761 AARCH64_MEMTAG_BUILTIN_GET_TAG, 762 AARCH64_MEMTAG_BUILTIN_END, 763 /* LS64 builtins. */ 764 AARCH64_LS64_BUILTIN_LD64B, 765 AARCH64_LS64_BUILTIN_ST64B, 766 AARCH64_LS64_BUILTIN_ST64BV, 767 AARCH64_LS64_BUILTIN_ST64BV0, 768 AARCH64_REV16, 769 AARCH64_REV16L, 770 AARCH64_REV16LL, 771 AARCH64_RBIT, 772 AARCH64_RBITL, 773 AARCH64_RBITLL, 774 /* System register builtins. */ 775 AARCH64_RSR, 776 AARCH64_RSRP, 777 AARCH64_RSR64, 778 AARCH64_RSRF, 779 AARCH64_RSRF64, 780 AARCH64_RSR128, 781 AARCH64_WSR, 782 AARCH64_WSRP, 783 AARCH64_WSR64, 784 AARCH64_WSRF, 785 AARCH64_WSRF64, 786 AARCH64_WSR128, 787 AARCH64_PLD, 788 AARCH64_PLDX, 789 AARCH64_PLI, 790 AARCH64_PLIX, 791 AARCH64_BUILTIN_MAX 792 }; 793 794 #undef CRC32_BUILTIN 795 #define CRC32_BUILTIN(N, M) \ 796 {"__builtin_aarch64_"#N, E_##M##mode, CODE_FOR_aarch64_##N, AARCH64_BUILTIN_##N}, 797 798 static aarch64_crc_builtin_datum aarch64_crc_builtin_data[] = { 799 AARCH64_CRC32_BUILTINS 800 }; 801 802 803 #undef FCMLA_LANEQ_BUILTIN 804 #define FCMLA_LANEQ_BUILTIN(I, N, X, M, T) \ 805 {"__builtin_aarch64_fcmla_laneq"#I#N, E_##M##mode, CODE_FOR_aarch64_##X##I##N, \ 806 AARCH64_SIMD_BUILTIN_FCMLA_LANEQ##I##_##M, T}, 807 808 /* This structure contains how to manage the mapping form the builtin to the 809 instruction to generate in the backend and how to invoke the instruction. */ 810 static aarch64_fcmla_laneq_builtin_datum aarch64_fcmla_lane_builtin_data[] = { 811 AARCH64_SIMD_FCMLA_LANEQ_BUILTINS 812 }; 813 814 #undef VREINTERPRET_BUILTIN 815 #define VREINTERPRET_BUILTIN(A, B, L) \ 816 {"vreinterpret" SIMD_INTR_LENGTH_CHAR(L) "_" #A "_" #B, \ 817 AARCH64_SIMD_BUILTIN_VREINTERPRET##L##_##A##_##B, \ 818 2, \ 819 { SIMD_INTR_MODE(A, L), SIMD_INTR_MODE(B, L) }, \ 820 { SIMD_INTR_QUAL(A), SIMD_INTR_QUAL(B) }, \ 821 FLAG_AUTO_FP, \ 822 SIMD_INTR_MODE(A, L) == SIMD_INTR_MODE(B, L) \ 823 && SIMD_INTR_QUAL(A) == SIMD_INTR_QUAL(B) \ 824 }, 825 826 static const aarch64_simd_intrinsic_datum aarch64_simd_intrinsic_data[] = { 827 AARCH64_SIMD_VREINTERPRET_BUILTINS 828 }; 829 830 831 #undef CRC32_BUILTIN 832 833 static GTY(()) tree aarch64_builtin_decls[AARCH64_BUILTIN_MAX]; 834 835 #define NUM_DREG_TYPES 6 836 #define NUM_QREG_TYPES 6 837 838 /* Internal scalar builtin types. These types are used to support 839 neon intrinsic builtins. They are _not_ user-visible types. Therefore 840 the mangling for these types are implementation defined. */ 841 const char *aarch64_scalar_builtin_types[] = { 842 "__builtin_aarch64_simd_qi", 843 "__builtin_aarch64_simd_hi", 844 "__builtin_aarch64_simd_si", 845 "__builtin_aarch64_simd_hf", 846 "__builtin_aarch64_simd_sf", 847 "__builtin_aarch64_simd_di", 848 "__builtin_aarch64_simd_df", 849 "__builtin_aarch64_simd_poly8", 850 "__builtin_aarch64_simd_poly16", 851 "__builtin_aarch64_simd_poly64", 852 "__builtin_aarch64_simd_poly128", 853 "__builtin_aarch64_simd_ti", 854 "__builtin_aarch64_simd_uqi", 855 "__builtin_aarch64_simd_uhi", 856 "__builtin_aarch64_simd_usi", 857 "__builtin_aarch64_simd_udi", 858 "__builtin_aarch64_simd_ei", 859 "__builtin_aarch64_simd_oi", 860 "__builtin_aarch64_simd_ci", 861 "__builtin_aarch64_simd_xi", 862 "__builtin_aarch64_simd_bf", 863 NULL 864 }; 865 866 extern GTY(()) aarch64_simd_type_info aarch64_simd_types[]; 867 868 #define ENTRY(E, M, Q, G) \ 869 {E, "__" #E, #G "__" #E, NULL_TREE, NULL_TREE, E_##M##mode, qualifier_##Q}, 870 struct aarch64_simd_type_info aarch64_simd_types [] = { 871 #include "aarch64-simd-builtin-types.def" 872 }; 873 #undef ENTRY 874 875 static machine_mode aarch64_simd_tuple_modes[ARM_NEON_H_TYPES_LAST][3]; 876 static GTY(()) tree aarch64_simd_tuple_types[ARM_NEON_H_TYPES_LAST][3]; 877 878 static GTY(()) tree aarch64_simd_intOI_type_node = NULL_TREE; 879 static GTY(()) tree aarch64_simd_intCI_type_node = NULL_TREE; 880 static GTY(()) tree aarch64_simd_intXI_type_node = NULL_TREE; 881 882 /* The user-visible __fp16 type, and a pointer to that type. Used 883 across the back-end. */ 884 tree aarch64_fp16_type_node = NULL_TREE; 885 tree aarch64_fp16_ptr_type_node = NULL_TREE; 886 887 /* Back-end node type for brain float (bfloat) types. */ 888 tree aarch64_bf16_ptr_type_node = NULL_TREE; 889 890 /* Wrapper around add_builtin_function. NAME is the name of the built-in 891 function, TYPE is the function type, CODE is the function subcode 892 (relative to AARCH64_BUILTIN_GENERAL), and ATTRS is the function 893 attributes. */ 894 static tree 895 aarch64_general_add_builtin (const char *name, tree type, unsigned int code, 896 tree attrs = NULL_TREE) 897 { 898 code = (code << AARCH64_BUILTIN_SHIFT) | AARCH64_BUILTIN_GENERAL; 899 return add_builtin_function (name, type, code, BUILT_IN_MD, 900 NULL, attrs); 901 } 902 903 static tree 904 aarch64_general_simulate_builtin (const char *name, tree fntype, 905 unsigned int code, 906 tree attrs = NULL_TREE) 907 { 908 code = (code << AARCH64_BUILTIN_SHIFT) | AARCH64_BUILTIN_GENERAL; 909 return simulate_builtin_function_decl (input_location, name, fntype, 910 code, NULL, attrs); 911 } 912 913 static const char * 914 aarch64_mangle_builtin_scalar_type (const_tree type) 915 { 916 int i = 0; 917 918 while (aarch64_scalar_builtin_types[i] != NULL) 919 { 920 const char *name = aarch64_scalar_builtin_types[i]; 921 922 if (TREE_CODE (TYPE_NAME (type)) == TYPE_DECL 923 && DECL_NAME (TYPE_NAME (type)) 924 && !strcmp (IDENTIFIER_POINTER (DECL_NAME (TYPE_NAME (type))), name)) 925 return aarch64_scalar_builtin_types[i]; 926 i++; 927 } 928 return NULL; 929 } 930 931 static const char * 932 aarch64_mangle_builtin_vector_type (const_tree type) 933 { 934 tree attrs = TYPE_ATTRIBUTES (type); 935 if (tree attr = lookup_attribute ("Advanced SIMD type", attrs)) 936 { 937 tree mangled_name = TREE_VALUE (TREE_VALUE (attr)); 938 return IDENTIFIER_POINTER (mangled_name); 939 } 940 941 return NULL; 942 } 943 944 const char * 945 aarch64_general_mangle_builtin_type (const_tree type) 946 { 947 const char *mangle; 948 /* Walk through all the AArch64 builtins types tables to filter out the 949 incoming type. */ 950 if ((mangle = aarch64_mangle_builtin_vector_type (type)) 951 || (mangle = aarch64_mangle_builtin_scalar_type (type))) 952 return mangle; 953 954 return NULL; 955 } 956 957 /* Helper function for aarch64_simd_builtin_type. */ 958 static tree 959 aarch64_int_or_fp_type (machine_mode mode, 960 enum aarch64_type_qualifiers qualifiers) 961 { 962 #define QUAL_TYPE(M) ((qualifiers & qualifier_unsigned) \ 963 ? unsigned_int##M##_type_node : int##M##_type_node); 964 switch (mode) 965 { 966 case E_QImode: 967 return QUAL_TYPE (QI); 968 case E_HImode: 969 return QUAL_TYPE (HI); 970 case E_SImode: 971 return QUAL_TYPE (SI); 972 case E_DImode: 973 return QUAL_TYPE (DI); 974 case E_TImode: 975 return QUAL_TYPE (TI); 976 case E_OImode: 977 return aarch64_simd_intOI_type_node; 978 case E_CImode: 979 return aarch64_simd_intCI_type_node; 980 case E_XImode: 981 return aarch64_simd_intXI_type_node; 982 case E_HFmode: 983 return aarch64_fp16_type_node; 984 case E_SFmode: 985 return float_type_node; 986 case E_DFmode: 987 return double_type_node; 988 case E_BFmode: 989 return bfloat16_type_node; 990 default: 991 gcc_unreachable (); 992 } 993 #undef QUAL_TYPE 994 } 995 996 /* Helper function for aarch64_simd_builtin_type. */ 997 static tree 998 aarch64_lookup_simd_type_in_table (machine_mode mode, 999 enum aarch64_type_qualifiers qualifiers) 1000 { 1001 int i; 1002 int nelts = ARRAY_SIZE (aarch64_simd_types); 1003 int q = qualifiers & (qualifier_poly | qualifier_unsigned); 1004 1005 for (i = 0; i < nelts; i++) 1006 { 1007 if (aarch64_simd_types[i].mode == mode 1008 && aarch64_simd_types[i].q == q) 1009 return aarch64_simd_types[i].itype; 1010 if (aarch64_simd_tuple_types[i][0] != NULL_TREE) 1011 for (int j = 0; j < 3; j++) 1012 if (aarch64_simd_tuple_modes[i][j] == mode 1013 && aarch64_simd_types[i].q == q) 1014 return aarch64_simd_tuple_types[i][j]; 1015 } 1016 1017 return NULL_TREE; 1018 } 1019 1020 /* Return a type for an operand with specified mode and qualifiers. */ 1021 static tree 1022 aarch64_simd_builtin_type (machine_mode mode, 1023 enum aarch64_type_qualifiers qualifiers) 1024 { 1025 tree type = NULL_TREE; 1026 1027 /* For pointers, we want a pointer to the basic type of the vector. */ 1028 if ((qualifiers & qualifier_pointer) && VECTOR_MODE_P (mode)) 1029 mode = GET_MODE_INNER (mode); 1030 1031 /* Non-poly scalar modes map to standard types not in the table. */ 1032 if ((qualifiers & qualifier_poly) || VECTOR_MODE_P (mode)) 1033 type = aarch64_lookup_simd_type_in_table (mode, qualifiers); 1034 else 1035 type = aarch64_int_or_fp_type (mode, qualifiers); 1036 1037 gcc_assert (type != NULL_TREE); 1038 1039 /* Add qualifiers. */ 1040 if (qualifiers & qualifier_const) 1041 type = build_qualified_type (type, TYPE_QUAL_CONST); 1042 if (qualifiers & qualifier_pointer) 1043 type = build_pointer_type (type); 1044 1045 return type; 1046 } 1047 1048 static void 1049 aarch64_init_simd_builtin_types (void) 1050 { 1051 int i; 1052 int nelts = ARRAY_SIZE (aarch64_simd_types); 1053 tree tdecl; 1054 1055 /* Init all the element types built by the front-end. */ 1056 aarch64_simd_types[Int8x8_t].eltype = intQI_type_node; 1057 aarch64_simd_types[Int8x16_t].eltype = intQI_type_node; 1058 aarch64_simd_types[Int16x4_t].eltype = intHI_type_node; 1059 aarch64_simd_types[Int16x8_t].eltype = intHI_type_node; 1060 aarch64_simd_types[Int32x2_t].eltype = intSI_type_node; 1061 aarch64_simd_types[Int32x4_t].eltype = intSI_type_node; 1062 aarch64_simd_types[Int64x1_t].eltype = intDI_type_node; 1063 aarch64_simd_types[Int64x2_t].eltype = intDI_type_node; 1064 aarch64_simd_types[Uint8x8_t].eltype = unsigned_intQI_type_node; 1065 aarch64_simd_types[Uint8x16_t].eltype = unsigned_intQI_type_node; 1066 aarch64_simd_types[Uint16x4_t].eltype = unsigned_intHI_type_node; 1067 aarch64_simd_types[Uint16x8_t].eltype = unsigned_intHI_type_node; 1068 aarch64_simd_types[Uint32x2_t].eltype = unsigned_intSI_type_node; 1069 aarch64_simd_types[Uint32x4_t].eltype = unsigned_intSI_type_node; 1070 aarch64_simd_types[Uint64x1_t].eltype = unsigned_intDI_type_node; 1071 aarch64_simd_types[Uint64x2_t].eltype = unsigned_intDI_type_node; 1072 1073 /* Poly types are a world of their own. */ 1074 aarch64_simd_types[Poly8_t].eltype = aarch64_simd_types[Poly8_t].itype = 1075 build_distinct_type_copy (unsigned_intQI_type_node); 1076 /* Prevent front-ends from transforming Poly8_t arrays into string 1077 literals. */ 1078 TYPE_STRING_FLAG (aarch64_simd_types[Poly8_t].eltype) = false; 1079 1080 aarch64_simd_types[Poly16_t].eltype = aarch64_simd_types[Poly16_t].itype = 1081 build_distinct_type_copy (unsigned_intHI_type_node); 1082 aarch64_simd_types[Poly64_t].eltype = aarch64_simd_types[Poly64_t].itype = 1083 build_distinct_type_copy (unsigned_intDI_type_node); 1084 aarch64_simd_types[Poly128_t].eltype = aarch64_simd_types[Poly128_t].itype = 1085 build_distinct_type_copy (unsigned_intTI_type_node); 1086 /* Init poly vector element types with scalar poly types. */ 1087 aarch64_simd_types[Poly8x8_t].eltype = aarch64_simd_types[Poly8_t].itype; 1088 aarch64_simd_types[Poly8x16_t].eltype = aarch64_simd_types[Poly8_t].itype; 1089 aarch64_simd_types[Poly16x4_t].eltype = aarch64_simd_types[Poly16_t].itype; 1090 aarch64_simd_types[Poly16x8_t].eltype = aarch64_simd_types[Poly16_t].itype; 1091 aarch64_simd_types[Poly64x1_t].eltype = aarch64_simd_types[Poly64_t].itype; 1092 aarch64_simd_types[Poly64x2_t].eltype = aarch64_simd_types[Poly64_t].itype; 1093 1094 /* Continue with standard types. */ 1095 aarch64_simd_types[Float16x4_t].eltype = aarch64_fp16_type_node; 1096 aarch64_simd_types[Float16x8_t].eltype = aarch64_fp16_type_node; 1097 aarch64_simd_types[Float32x2_t].eltype = float_type_node; 1098 aarch64_simd_types[Float32x4_t].eltype = float_type_node; 1099 aarch64_simd_types[Float64x1_t].eltype = double_type_node; 1100 aarch64_simd_types[Float64x2_t].eltype = double_type_node; 1101 1102 /* Init Bfloat vector types with underlying __bf16 type. */ 1103 aarch64_simd_types[Bfloat16x4_t].eltype = bfloat16_type_node; 1104 aarch64_simd_types[Bfloat16x8_t].eltype = bfloat16_type_node; 1105 1106 for (i = 0; i < nelts; i++) 1107 { 1108 tree eltype = aarch64_simd_types[i].eltype; 1109 machine_mode mode = aarch64_simd_types[i].mode; 1110 1111 if (aarch64_simd_types[i].itype == NULL) 1112 { 1113 tree type = build_vector_type (eltype, GET_MODE_NUNITS (mode)); 1114 type = build_distinct_type_copy (type); 1115 SET_TYPE_STRUCTURAL_EQUALITY (type); 1116 1117 tree mangled_name = get_identifier (aarch64_simd_types[i].mangle); 1118 tree value = tree_cons (NULL_TREE, mangled_name, NULL_TREE); 1119 TYPE_ATTRIBUTES (type) 1120 = tree_cons (get_identifier ("Advanced SIMD type"), value, 1121 TYPE_ATTRIBUTES (type)); 1122 aarch64_simd_types[i].itype = type; 1123 } 1124 1125 tdecl = add_builtin_type (aarch64_simd_types[i].name, 1126 aarch64_simd_types[i].itype); 1127 TYPE_NAME (aarch64_simd_types[i].itype) = tdecl; 1128 } 1129 1130 #define AARCH64_BUILD_SIGNED_TYPE(mode) \ 1131 make_signed_type (GET_MODE_PRECISION (mode)); 1132 aarch64_simd_intOI_type_node = AARCH64_BUILD_SIGNED_TYPE (OImode); 1133 aarch64_simd_intCI_type_node = AARCH64_BUILD_SIGNED_TYPE (CImode); 1134 aarch64_simd_intXI_type_node = AARCH64_BUILD_SIGNED_TYPE (XImode); 1135 #undef AARCH64_BUILD_SIGNED_TYPE 1136 1137 tdecl = add_builtin_type 1138 ("__builtin_aarch64_simd_oi" , aarch64_simd_intOI_type_node); 1139 TYPE_NAME (aarch64_simd_intOI_type_node) = tdecl; 1140 tdecl = add_builtin_type 1141 ("__builtin_aarch64_simd_ci" , aarch64_simd_intCI_type_node); 1142 TYPE_NAME (aarch64_simd_intCI_type_node) = tdecl; 1143 tdecl = add_builtin_type 1144 ("__builtin_aarch64_simd_xi" , aarch64_simd_intXI_type_node); 1145 TYPE_NAME (aarch64_simd_intXI_type_node) = tdecl; 1146 } 1147 1148 static void 1149 aarch64_init_simd_builtin_scalar_types (void) 1150 { 1151 /* Define typedefs for all the standard scalar types. */ 1152 (*lang_hooks.types.register_builtin_type) (intQI_type_node, 1153 "__builtin_aarch64_simd_qi"); 1154 (*lang_hooks.types.register_builtin_type) (intHI_type_node, 1155 "__builtin_aarch64_simd_hi"); 1156 (*lang_hooks.types.register_builtin_type) (aarch64_fp16_type_node, 1157 "__builtin_aarch64_simd_hf"); 1158 (*lang_hooks.types.register_builtin_type) (intSI_type_node, 1159 "__builtin_aarch64_simd_si"); 1160 (*lang_hooks.types.register_builtin_type) (float_type_node, 1161 "__builtin_aarch64_simd_sf"); 1162 (*lang_hooks.types.register_builtin_type) (intDI_type_node, 1163 "__builtin_aarch64_simd_di"); 1164 (*lang_hooks.types.register_builtin_type) (double_type_node, 1165 "__builtin_aarch64_simd_df"); 1166 (*lang_hooks.types.register_builtin_type) (unsigned_intQI_type_node, 1167 "__builtin_aarch64_simd_poly8"); 1168 (*lang_hooks.types.register_builtin_type) (unsigned_intHI_type_node, 1169 "__builtin_aarch64_simd_poly16"); 1170 (*lang_hooks.types.register_builtin_type) (unsigned_intDI_type_node, 1171 "__builtin_aarch64_simd_poly64"); 1172 (*lang_hooks.types.register_builtin_type) (unsigned_intTI_type_node, 1173 "__builtin_aarch64_simd_poly128"); 1174 (*lang_hooks.types.register_builtin_type) (intTI_type_node, 1175 "__builtin_aarch64_simd_ti"); 1176 (*lang_hooks.types.register_builtin_type) (bfloat16_type_node, 1177 "__builtin_aarch64_simd_bf"); 1178 /* Unsigned integer types for various mode sizes. */ 1179 (*lang_hooks.types.register_builtin_type) (unsigned_intQI_type_node, 1180 "__builtin_aarch64_simd_uqi"); 1181 (*lang_hooks.types.register_builtin_type) (unsigned_intHI_type_node, 1182 "__builtin_aarch64_simd_uhi"); 1183 (*lang_hooks.types.register_builtin_type) (unsigned_intSI_type_node, 1184 "__builtin_aarch64_simd_usi"); 1185 (*lang_hooks.types.register_builtin_type) (unsigned_intDI_type_node, 1186 "__builtin_aarch64_simd_udi"); 1187 } 1188 1189 /* Return a set of FLAG_* flags derived from FLAGS 1190 that describe what a function with result MODE could do, 1191 taking the command-line flags into account. */ 1192 static unsigned int 1193 aarch64_call_properties (unsigned int flags, machine_mode mode) 1194 { 1195 if (!(flags & FLAG_AUTO_FP) && FLOAT_MODE_P (mode)) 1196 flags |= FLAG_FP; 1197 1198 /* -fno-trapping-math means that we can assume any FP exceptions 1199 are not user-visible. */ 1200 if (!flag_trapping_math) 1201 flags &= ~FLAG_RAISE_FP_EXCEPTIONS; 1202 1203 return flags; 1204 } 1205 1206 /* Return true if calls to a function with flags F and mode MODE 1207 could modify some form of global state. */ 1208 static bool 1209 aarch64_modifies_global_state_p (unsigned int f, machine_mode mode) 1210 { 1211 unsigned int flags = aarch64_call_properties (f, mode); 1212 1213 if (flags & FLAG_RAISE_FP_EXCEPTIONS) 1214 return true; 1215 1216 if (flags & FLAG_PREFETCH_MEMORY) 1217 return true; 1218 1219 return flags & FLAG_WRITE_MEMORY; 1220 } 1221 1222 /* Return true if calls to a function with flags F and mode MODE 1223 could read some form of global state. */ 1224 static bool 1225 aarch64_reads_global_state_p (unsigned int f, machine_mode mode) 1226 { 1227 unsigned int flags = aarch64_call_properties (f, mode); 1228 1229 if (flags & FLAG_READ_FPCR) 1230 return true; 1231 1232 return flags & FLAG_READ_MEMORY; 1233 } 1234 1235 /* Return true if calls to a function with flags F and mode MODE 1236 could raise a signal. */ 1237 static bool 1238 aarch64_could_trap_p (unsigned int f, machine_mode mode) 1239 { 1240 unsigned int flags = aarch64_call_properties (f, mode); 1241 1242 if (flags & FLAG_RAISE_FP_EXCEPTIONS) 1243 return true; 1244 1245 if (flags & (FLAG_READ_MEMORY | FLAG_WRITE_MEMORY)) 1246 return true; 1247 1248 return false; 1249 } 1250 1251 /* Add attribute NAME to ATTRS. */ 1252 static tree 1253 aarch64_add_attribute (const char *name, tree attrs) 1254 { 1255 return tree_cons (get_identifier (name), NULL_TREE, attrs); 1256 } 1257 1258 /* Return the appropriate attributes for a function that has 1259 flags F and mode MODE. */ 1260 static tree 1261 aarch64_get_attributes (unsigned int f, machine_mode mode) 1262 { 1263 tree attrs = NULL_TREE; 1264 1265 if (!aarch64_modifies_global_state_p (f, mode)) 1266 { 1267 if (aarch64_reads_global_state_p (f, mode)) 1268 attrs = aarch64_add_attribute ("pure", attrs); 1269 else 1270 attrs = aarch64_add_attribute ("const", attrs); 1271 } 1272 1273 if (!flag_non_call_exceptions || !aarch64_could_trap_p (f, mode)) 1274 attrs = aarch64_add_attribute ("nothrow", attrs); 1275 1276 return aarch64_add_attribute ("leaf", attrs); 1277 } 1278 1279 /* Due to the architecture not providing lane variant of the lane instructions 1280 for fcmla we can't use the standard simd builtin expansion code, but we 1281 still want the majority of the validation that would normally be done. */ 1282 1283 void 1284 aarch64_init_fcmla_laneq_builtins (void) 1285 { 1286 unsigned int i = 0; 1287 1288 for (i = 0; i < ARRAY_SIZE (aarch64_fcmla_lane_builtin_data); ++i) 1289 { 1290 aarch64_fcmla_laneq_builtin_datum* d 1291 = &aarch64_fcmla_lane_builtin_data[i]; 1292 tree argtype = aarch64_simd_builtin_type (d->mode, qualifier_none); 1293 machine_mode quadmode = GET_MODE_2XWIDER_MODE (d->mode).require (); 1294 tree quadtype = aarch64_simd_builtin_type (quadmode, qualifier_none); 1295 tree lanetype 1296 = aarch64_simd_builtin_type (SImode, qualifier_lane_pair_index); 1297 tree ftype = build_function_type_list (argtype, argtype, argtype, 1298 quadtype, lanetype, NULL_TREE); 1299 tree attrs = aarch64_get_attributes (FLAG_FP, d->mode); 1300 tree fndecl 1301 = aarch64_general_add_builtin (d->name, ftype, d->fcode, attrs); 1302 1303 aarch64_builtin_decls[d->fcode] = fndecl; 1304 } 1305 } 1306 1307 void 1308 aarch64_init_simd_intrinsics (void) 1309 { 1310 unsigned int i = 0; 1311 1312 for (i = 0; i < ARRAY_SIZE (aarch64_simd_intrinsic_data); ++i) 1313 { 1314 auto d = &aarch64_simd_intrinsic_data[i]; 1315 1316 if (d->skip) 1317 continue; 1318 1319 tree return_type = void_type_node; 1320 tree args = void_list_node; 1321 1322 for (int op_num = d->op_count - 1; op_num >= 0; op_num--) 1323 { 1324 machine_mode op_mode = d->op_modes[op_num]; 1325 enum aarch64_type_qualifiers qualifiers = d->qualifiers[op_num]; 1326 1327 tree eltype = aarch64_simd_builtin_type (op_mode, qualifiers); 1328 1329 if (op_num == 0) 1330 return_type = eltype; 1331 else 1332 args = tree_cons (NULL_TREE, eltype, args); 1333 } 1334 1335 tree ftype = build_function_type (return_type, args); 1336 tree attrs = aarch64_get_attributes (d->flags, d->op_modes[0]); 1337 unsigned int code 1338 = (d->fcode << AARCH64_BUILTIN_SHIFT | AARCH64_BUILTIN_GENERAL); 1339 tree fndecl = simulate_builtin_function_decl (input_location, d->name, 1340 ftype, code, NULL, attrs); 1341 aarch64_builtin_decls[d->fcode] = fndecl; 1342 } 1343 } 1344 1345 void 1346 aarch64_init_simd_builtin_functions (bool called_from_pragma) 1347 { 1348 unsigned int i, fcode = AARCH64_SIMD_PATTERN_START; 1349 1350 if (!called_from_pragma) 1351 { 1352 tree lane_check_fpr = build_function_type_list (void_type_node, 1353 size_type_node, 1354 size_type_node, 1355 intSI_type_node, 1356 NULL); 1357 aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_LANE_CHECK] 1358 = aarch64_general_add_builtin ("__builtin_aarch64_im_lane_boundsi", 1359 lane_check_fpr, 1360 AARCH64_SIMD_BUILTIN_LANE_CHECK); 1361 } 1362 1363 for (i = 0; i < ARRAY_SIZE (aarch64_simd_builtin_data); i++, fcode++) 1364 { 1365 bool print_type_signature_p = false; 1366 char type_signature[SIMD_MAX_BUILTIN_ARGS + 1] = { 0 }; 1367 aarch64_simd_builtin_datum *d = &aarch64_simd_builtin_data[i]; 1368 char namebuf[60]; 1369 tree ftype = NULL; 1370 tree fndecl = NULL; 1371 1372 d->fcode = fcode; 1373 1374 /* We must track two variables here. op_num is 1375 the operand number as in the RTL pattern. This is 1376 required to access the mode (e.g. V4SF mode) of the 1377 argument, from which the base type can be derived. 1378 arg_num is an index in to the qualifiers data, which 1379 gives qualifiers to the type (e.g. const unsigned). 1380 The reason these two variables may differ by one is the 1381 void return type. While all return types take the 0th entry 1382 in the qualifiers array, there is no operand for them in the 1383 RTL pattern. */ 1384 int op_num = insn_data[d->code].n_operands - 1; 1385 int arg_num = d->qualifiers[0] & qualifier_void 1386 ? op_num + 1 1387 : op_num; 1388 tree return_type = void_type_node, args = void_list_node; 1389 tree eltype; 1390 1391 int struct_mode_args = 0; 1392 for (int j = op_num; j >= 0; j--) 1393 { 1394 machine_mode op_mode = insn_data[d->code].operand[j].mode; 1395 if (aarch64_advsimd_struct_mode_p (op_mode)) 1396 struct_mode_args++; 1397 } 1398 1399 if ((called_from_pragma && struct_mode_args == 0) 1400 || (!called_from_pragma && struct_mode_args > 0)) 1401 continue; 1402 1403 /* Build a function type directly from the insn_data for this 1404 builtin. The build_function_type () function takes care of 1405 removing duplicates for us. */ 1406 for (; op_num >= 0; arg_num--, op_num--) 1407 { 1408 machine_mode op_mode = insn_data[d->code].operand[op_num].mode; 1409 enum aarch64_type_qualifiers qualifiers = d->qualifiers[arg_num]; 1410 1411 if (qualifiers & qualifier_unsigned) 1412 { 1413 type_signature[op_num] = 'u'; 1414 print_type_signature_p = true; 1415 } 1416 else if (qualifiers & qualifier_poly) 1417 { 1418 type_signature[op_num] = 'p'; 1419 print_type_signature_p = true; 1420 } 1421 else 1422 type_signature[op_num] = 's'; 1423 1424 /* Some builtins have different user-facing types 1425 for certain arguments, encoded in d->mode. */ 1426 if (qualifiers & qualifier_map_mode) 1427 op_mode = d->mode; 1428 1429 eltype = aarch64_simd_builtin_type (op_mode, qualifiers); 1430 1431 /* If we have reached arg_num == 0, we are at a non-void 1432 return type. Otherwise, we are still processing 1433 arguments. */ 1434 if (arg_num == 0) 1435 return_type = eltype; 1436 else 1437 args = tree_cons (NULL_TREE, eltype, args); 1438 } 1439 1440 ftype = build_function_type (return_type, args); 1441 1442 gcc_assert (ftype != NULL); 1443 1444 if (print_type_signature_p) 1445 snprintf (namebuf, sizeof (namebuf), "__builtin_aarch64_%s_%s", 1446 d->name, type_signature); 1447 else 1448 snprintf (namebuf, sizeof (namebuf), "__builtin_aarch64_%s", 1449 d->name); 1450 1451 tree attrs = aarch64_get_attributes (d->flags, d->mode); 1452 1453 if (called_from_pragma) 1454 { 1455 unsigned int raw_code 1456 = (fcode << AARCH64_BUILTIN_SHIFT) | AARCH64_BUILTIN_GENERAL; 1457 fndecl = simulate_builtin_function_decl (input_location, namebuf, 1458 ftype, raw_code, NULL, 1459 attrs); 1460 } 1461 else 1462 fndecl = aarch64_general_add_builtin (namebuf, ftype, fcode, attrs); 1463 1464 aarch64_builtin_decls[fcode] = fndecl; 1465 } 1466 } 1467 1468 /* Register the tuple type that contains NUM_VECTORS of the AdvSIMD type 1469 indexed by TYPE_INDEX. */ 1470 static void 1471 register_tuple_type (unsigned int num_vectors, unsigned int type_index) 1472 { 1473 aarch64_simd_type_info *type = &aarch64_simd_types[type_index]; 1474 1475 /* Synthesize the name of the user-visible vector tuple type. */ 1476 const char *vector_type_name = type->name; 1477 char tuple_type_name[sizeof ("bfloat16x4x2_t")]; 1478 snprintf (tuple_type_name, sizeof (tuple_type_name), "%.*sx%d_t", 1479 (int) strlen (vector_type_name) - 4, vector_type_name + 2, 1480 num_vectors); 1481 tuple_type_name[0] = TOLOWER (tuple_type_name[0]); 1482 1483 tree vector_type = type->itype; 1484 tree array_type = build_array_type_nelts (vector_type, num_vectors); 1485 if (type->mode == DImode) 1486 { 1487 if (num_vectors == 2) 1488 SET_TYPE_MODE (array_type, V2x1DImode); 1489 else if (num_vectors == 3) 1490 SET_TYPE_MODE (array_type, V3x1DImode); 1491 else if (num_vectors == 4) 1492 SET_TYPE_MODE (array_type, V4x1DImode); 1493 } 1494 1495 unsigned int alignment 1496 = known_eq (GET_MODE_SIZE (type->mode), 16) ? 128 : 64; 1497 machine_mode tuple_mode = TYPE_MODE_RAW (array_type); 1498 gcc_assert (VECTOR_MODE_P (tuple_mode) 1499 && TYPE_MODE (array_type) == tuple_mode 1500 && TYPE_ALIGN (array_type) == alignment); 1501 1502 tree field = build_decl (input_location, FIELD_DECL, 1503 get_identifier ("val"), array_type); 1504 1505 tree t = lang_hooks.types.simulate_record_decl (input_location, 1506 tuple_type_name, 1507 make_array_slice (&field, 1508 1)); 1509 gcc_assert (TYPE_MODE_RAW (t) == TYPE_MODE (t) 1510 && (flag_pack_struct 1511 || maximum_field_alignment 1512 || (TYPE_MODE_RAW (t) == tuple_mode 1513 && TYPE_ALIGN (t) == alignment))); 1514 1515 aarch64_simd_tuple_modes[type_index][num_vectors - 2] = tuple_mode; 1516 aarch64_simd_tuple_types[type_index][num_vectors - 2] = t; 1517 } 1518 1519 static bool 1520 aarch64_scalar_builtin_type_p (aarch64_simd_type t) 1521 { 1522 return (t == Poly8_t || t == Poly16_t || t == Poly64_t || t == Poly128_t); 1523 } 1524 1525 /* Enable AARCH64_FL_* flags EXTRA_FLAGS on top of the base Advanced SIMD 1526 set. */ 1527 aarch64_simd_switcher::aarch64_simd_switcher (aarch64_feature_flags extra_flags) 1528 : m_old_asm_isa_flags (aarch64_asm_isa_flags), 1529 m_old_general_regs_only (TARGET_GENERAL_REGS_ONLY) 1530 { 1531 /* Changing the ISA flags should be enough here. We shouldn't need to 1532 pay the compile-time cost of a full target switch. */ 1533 global_options.x_target_flags &= ~MASK_GENERAL_REGS_ONLY; 1534 aarch64_set_asm_isa_flags (AARCH64_FL_FP | AARCH64_FL_SIMD | extra_flags); 1535 } 1536 1537 aarch64_simd_switcher::~aarch64_simd_switcher () 1538 { 1539 if (m_old_general_regs_only) 1540 global_options.x_target_flags |= MASK_GENERAL_REGS_ONLY; 1541 aarch64_set_asm_isa_flags (m_old_asm_isa_flags); 1542 } 1543 1544 /* Implement #pragma GCC aarch64 "arm_neon.h". 1545 1546 The types and functions defined here need to be available internally 1547 during LTO as well. */ 1548 void 1549 handle_arm_neon_h (void) 1550 { 1551 aarch64_simd_switcher simd; 1552 1553 /* Register the AdvSIMD vector tuple types. */ 1554 for (unsigned int i = 0; i < ARM_NEON_H_TYPES_LAST; i++) 1555 for (unsigned int count = 2; count <= 4; ++count) 1556 if (!aarch64_scalar_builtin_type_p (aarch64_simd_types[i].type)) 1557 register_tuple_type (count, i); 1558 1559 aarch64_init_simd_builtin_functions (true); 1560 aarch64_init_simd_intrinsics (); 1561 } 1562 1563 static void 1564 aarch64_init_simd_builtins (void) 1565 { 1566 aarch64_init_simd_builtin_types (); 1567 1568 /* Strong-typing hasn't been implemented for all AdvSIMD builtin intrinsics. 1569 Therefore we need to preserve the old __builtin scalar types. It can be 1570 removed once all the intrinsics become strongly typed using the qualifier 1571 system. */ 1572 aarch64_init_simd_builtin_scalar_types (); 1573 1574 aarch64_init_simd_builtin_functions (false); 1575 if (in_lto_p) 1576 handle_arm_neon_h (); 1577 1578 /* Initialize the remaining fcmla_laneq intrinsics. */ 1579 aarch64_init_fcmla_laneq_builtins (); 1580 } 1581 1582 static void 1583 aarch64_init_crc32_builtins () 1584 { 1585 tree usi_type = aarch64_simd_builtin_type (SImode, qualifier_unsigned); 1586 unsigned int i = 0; 1587 1588 for (i = 0; i < ARRAY_SIZE (aarch64_crc_builtin_data); ++i) 1589 { 1590 aarch64_crc_builtin_datum* d = &aarch64_crc_builtin_data[i]; 1591 tree argtype = aarch64_simd_builtin_type (d->mode, qualifier_unsigned); 1592 tree ftype = build_function_type_list (usi_type, usi_type, argtype, NULL_TREE); 1593 tree attrs = aarch64_get_attributes (FLAG_NONE, d->mode); 1594 tree fndecl 1595 = aarch64_general_add_builtin (d->name, ftype, d->fcode, attrs); 1596 1597 aarch64_builtin_decls[d->fcode] = fndecl; 1598 } 1599 } 1600 1601 /* Add builtins for reciprocal square root. */ 1602 1603 void 1604 aarch64_init_builtin_rsqrt (void) 1605 { 1606 tree fndecl = NULL; 1607 tree ftype = NULL; 1608 1609 tree V2SF_type_node = build_vector_type (float_type_node, 2); 1610 tree V2DF_type_node = build_vector_type (double_type_node, 2); 1611 tree V4SF_type_node = build_vector_type (float_type_node, 4); 1612 1613 struct builtin_decls_data 1614 { 1615 tree type_node; 1616 const char *builtin_name; 1617 int function_code; 1618 }; 1619 1620 builtin_decls_data bdda[] = 1621 { 1622 { double_type_node, "__builtin_aarch64_rsqrt_df", AARCH64_BUILTIN_RSQRT_DF }, 1623 { float_type_node, "__builtin_aarch64_rsqrt_sf", AARCH64_BUILTIN_RSQRT_SF }, 1624 { V2DF_type_node, "__builtin_aarch64_rsqrt_v2df", AARCH64_BUILTIN_RSQRT_V2DF }, 1625 { V2SF_type_node, "__builtin_aarch64_rsqrt_v2sf", AARCH64_BUILTIN_RSQRT_V2SF }, 1626 { V4SF_type_node, "__builtin_aarch64_rsqrt_v4sf", AARCH64_BUILTIN_RSQRT_V4SF } 1627 }; 1628 1629 builtin_decls_data *bdd = bdda; 1630 builtin_decls_data *bdd_end = bdd + (ARRAY_SIZE (bdda)); 1631 1632 for (; bdd < bdd_end; bdd++) 1633 { 1634 ftype = build_function_type_list (bdd->type_node, bdd->type_node, NULL_TREE); 1635 tree attrs = aarch64_get_attributes (FLAG_FP, TYPE_MODE (bdd->type_node)); 1636 fndecl = aarch64_general_add_builtin (bdd->builtin_name, 1637 ftype, bdd->function_code, attrs); 1638 aarch64_builtin_decls[bdd->function_code] = fndecl; 1639 } 1640 } 1641 1642 /* Initialize the backend types that support the user-visible __fp16 1643 type, also initialize a pointer to that type, to be used when 1644 forming HFAs. */ 1645 1646 static void 1647 aarch64_init_fp16_types (void) 1648 { 1649 aarch64_fp16_type_node = make_node (REAL_TYPE); 1650 TYPE_PRECISION (aarch64_fp16_type_node) = 16; 1651 layout_type (aarch64_fp16_type_node); 1652 1653 (*lang_hooks.types.register_builtin_type) (aarch64_fp16_type_node, "__fp16"); 1654 aarch64_fp16_ptr_type_node = build_pointer_type (aarch64_fp16_type_node); 1655 } 1656 1657 /* Initialize the backend REAL_TYPE type supporting bfloat types. */ 1658 static void 1659 aarch64_init_bf16_types (void) 1660 { 1661 lang_hooks.types.register_builtin_type (bfloat16_type_node, "__bf16"); 1662 aarch64_bf16_ptr_type_node = build_pointer_type (bfloat16_type_node); 1663 } 1664 1665 /* Pointer authentication builtins that will become NOP on legacy platform. 1666 Currently, these builtins are for internal use only (libgcc EH unwinder). */ 1667 1668 void 1669 aarch64_init_pauth_hint_builtins (void) 1670 { 1671 /* Pointer Authentication builtins. */ 1672 tree ftype_pointer_auth 1673 = build_function_type_list (ptr_type_node, ptr_type_node, 1674 unsigned_intDI_type_node, NULL_TREE); 1675 tree ftype_pointer_strip 1676 = build_function_type_list (ptr_type_node, ptr_type_node, NULL_TREE); 1677 1678 aarch64_builtin_decls[AARCH64_PAUTH_BUILTIN_AUTIA1716] 1679 = aarch64_general_add_builtin ("__builtin_aarch64_autia1716", 1680 ftype_pointer_auth, 1681 AARCH64_PAUTH_BUILTIN_AUTIA1716); 1682 aarch64_builtin_decls[AARCH64_PAUTH_BUILTIN_PACIA1716] 1683 = aarch64_general_add_builtin ("__builtin_aarch64_pacia1716", 1684 ftype_pointer_auth, 1685 AARCH64_PAUTH_BUILTIN_PACIA1716); 1686 aarch64_builtin_decls[AARCH64_PAUTH_BUILTIN_AUTIB1716] 1687 = aarch64_general_add_builtin ("__builtin_aarch64_autib1716", 1688 ftype_pointer_auth, 1689 AARCH64_PAUTH_BUILTIN_AUTIB1716); 1690 aarch64_builtin_decls[AARCH64_PAUTH_BUILTIN_PACIB1716] 1691 = aarch64_general_add_builtin ("__builtin_aarch64_pacib1716", 1692 ftype_pointer_auth, 1693 AARCH64_PAUTH_BUILTIN_PACIB1716); 1694 aarch64_builtin_decls[AARCH64_PAUTH_BUILTIN_XPACLRI] 1695 = aarch64_general_add_builtin ("__builtin_aarch64_xpaclri", 1696 ftype_pointer_strip, 1697 AARCH64_PAUTH_BUILTIN_XPACLRI); 1698 } 1699 1700 /* Initialize the transactional memory extension (TME) builtins. */ 1701 static void 1702 aarch64_init_tme_builtins (void) 1703 { 1704 tree ftype_uint64_void 1705 = build_function_type_list (uint64_type_node, NULL); 1706 tree ftype_void_void 1707 = build_function_type_list (void_type_node, NULL); 1708 tree ftype_void_uint64 1709 = build_function_type_list (void_type_node, uint64_type_node, NULL); 1710 1711 aarch64_builtin_decls[AARCH64_TME_BUILTIN_TSTART] 1712 = aarch64_general_simulate_builtin ("__tstart", ftype_uint64_void, 1713 AARCH64_TME_BUILTIN_TSTART); 1714 aarch64_builtin_decls[AARCH64_TME_BUILTIN_TTEST] 1715 = aarch64_general_simulate_builtin ("__ttest", ftype_uint64_void, 1716 AARCH64_TME_BUILTIN_TTEST); 1717 aarch64_builtin_decls[AARCH64_TME_BUILTIN_TCOMMIT] 1718 = aarch64_general_simulate_builtin ("__tcommit", ftype_void_void, 1719 AARCH64_TME_BUILTIN_TCOMMIT); 1720 aarch64_builtin_decls[AARCH64_TME_BUILTIN_TCANCEL] 1721 = aarch64_general_simulate_builtin ("__tcancel", ftype_void_uint64, 1722 AARCH64_TME_BUILTIN_TCANCEL); 1723 } 1724 1725 /* Add builtins for Random Number instructions. */ 1726 1727 static void 1728 aarch64_init_rng_builtins (void) 1729 { 1730 tree unsigned_ptr_type 1731 = build_pointer_type (get_typenode_from_name (UINT64_TYPE)); 1732 tree ftype 1733 = build_function_type_list (integer_type_node, unsigned_ptr_type, NULL); 1734 aarch64_builtin_decls[AARCH64_BUILTIN_RNG_RNDR] 1735 = aarch64_general_add_builtin ("__builtin_aarch64_rndr", ftype, 1736 AARCH64_BUILTIN_RNG_RNDR); 1737 aarch64_builtin_decls[AARCH64_BUILTIN_RNG_RNDRRS] 1738 = aarch64_general_add_builtin ("__builtin_aarch64_rndrrs", ftype, 1739 AARCH64_BUILTIN_RNG_RNDRRS); 1740 } 1741 1742 /* Add builtins for reading system register. */ 1743 static void 1744 aarch64_init_rwsr_builtins (void) 1745 { 1746 tree fntype = NULL; 1747 tree const_char_ptr_type 1748 = build_pointer_type (build_type_variant (char_type_node, true, false)); 1749 1750 #define AARCH64_INIT_RWSR_BUILTINS_DECL(F, N, T) \ 1751 aarch64_builtin_decls[AARCH64_##F] \ 1752 = aarch64_general_add_builtin ("__builtin_aarch64_"#N, T, AARCH64_##F); 1753 1754 fntype 1755 = build_function_type_list (uint32_type_node, const_char_ptr_type, NULL); 1756 AARCH64_INIT_RWSR_BUILTINS_DECL (RSR, rsr, fntype); 1757 1758 fntype 1759 = build_function_type_list (ptr_type_node, const_char_ptr_type, NULL); 1760 AARCH64_INIT_RWSR_BUILTINS_DECL (RSRP, rsrp, fntype); 1761 1762 fntype 1763 = build_function_type_list (uint64_type_node, const_char_ptr_type, NULL); 1764 AARCH64_INIT_RWSR_BUILTINS_DECL (RSR64, rsr64, fntype); 1765 1766 fntype 1767 = build_function_type_list (float_type_node, const_char_ptr_type, NULL); 1768 AARCH64_INIT_RWSR_BUILTINS_DECL (RSRF, rsrf, fntype); 1769 1770 fntype 1771 = build_function_type_list (double_type_node, const_char_ptr_type, NULL); 1772 AARCH64_INIT_RWSR_BUILTINS_DECL (RSRF64, rsrf64, fntype); 1773 1774 fntype 1775 = build_function_type_list (uint128_type_node, const_char_ptr_type, NULL); 1776 AARCH64_INIT_RWSR_BUILTINS_DECL (RSR128, rsr128, fntype); 1777 1778 fntype 1779 = build_function_type_list (void_type_node, const_char_ptr_type, 1780 uint32_type_node, NULL); 1781 1782 AARCH64_INIT_RWSR_BUILTINS_DECL (WSR, wsr, fntype); 1783 1784 fntype 1785 = build_function_type_list (void_type_node, const_char_ptr_type, 1786 const_ptr_type_node, NULL); 1787 AARCH64_INIT_RWSR_BUILTINS_DECL (WSRP, wsrp, fntype); 1788 1789 fntype 1790 = build_function_type_list (void_type_node, const_char_ptr_type, 1791 uint64_type_node, NULL); 1792 AARCH64_INIT_RWSR_BUILTINS_DECL (WSR64, wsr64, fntype); 1793 1794 fntype 1795 = build_function_type_list (void_type_node, const_char_ptr_type, 1796 float_type_node, NULL); 1797 AARCH64_INIT_RWSR_BUILTINS_DECL (WSRF, wsrf, fntype); 1798 1799 fntype 1800 = build_function_type_list (void_type_node, const_char_ptr_type, 1801 double_type_node, NULL); 1802 AARCH64_INIT_RWSR_BUILTINS_DECL (WSRF64, wsrf64, fntype); 1803 1804 fntype 1805 = build_function_type_list (void_type_node, const_char_ptr_type, 1806 uint128_type_node, NULL); 1807 AARCH64_INIT_RWSR_BUILTINS_DECL (WSR128, wsr128, fntype); 1808 } 1809 1810 /* Add builtins for data and instrution prefetch. */ 1811 static void 1812 aarch64_init_prefetch_builtin (void) 1813 { 1814 #define AARCH64_INIT_PREFETCH_BUILTIN(INDEX, N) \ 1815 aarch64_builtin_decls[INDEX] = \ 1816 aarch64_general_add_builtin ("__builtin_aarch64_" N, ftype, INDEX) 1817 1818 tree ftype; 1819 tree cv_argtype; 1820 cv_argtype = build_qualified_type (void_type_node, TYPE_QUAL_CONST 1821 | TYPE_QUAL_VOLATILE); 1822 cv_argtype = build_pointer_type (cv_argtype); 1823 1824 ftype = build_function_type_list (void_type_node, cv_argtype, NULL); 1825 AARCH64_INIT_PREFETCH_BUILTIN (AARCH64_PLD, "pld"); 1826 AARCH64_INIT_PREFETCH_BUILTIN (AARCH64_PLI, "pli"); 1827 1828 ftype = build_function_type_list (void_type_node, unsigned_type_node, 1829 unsigned_type_node, unsigned_type_node, 1830 cv_argtype, NULL); 1831 AARCH64_INIT_PREFETCH_BUILTIN (AARCH64_PLDX, "pldx"); 1832 1833 ftype = build_function_type_list (void_type_node, unsigned_type_node, 1834 unsigned_type_node, cv_argtype, NULL); 1835 AARCH64_INIT_PREFETCH_BUILTIN (AARCH64_PLIX, "plix"); 1836 } 1837 1838 /* Initialize the memory tagging extension (MTE) builtins. */ 1839 static GTY(()) struct aarch64_mte 1840 { 1841 tree ftype; 1842 enum insn_code icode; 1843 } aarch64_memtag_builtin_data[AARCH64_MEMTAG_BUILTIN_END - 1844 AARCH64_MEMTAG_BUILTIN_START - 1]; 1845 1846 static void 1847 aarch64_init_memtag_builtins (void) 1848 { 1849 tree fntype = NULL; 1850 1851 #define AARCH64_INIT_MEMTAG_BUILTINS_DECL(F, N, I, T) \ 1852 aarch64_builtin_decls[AARCH64_MEMTAG_BUILTIN_##F] \ 1853 = aarch64_general_simulate_builtin ("__arm_mte_"#N, T, \ 1854 AARCH64_MEMTAG_BUILTIN_##F); \ 1855 aarch64_memtag_builtin_data[AARCH64_MEMTAG_BUILTIN_##F - \ 1856 AARCH64_MEMTAG_BUILTIN_START - 1] = \ 1857 {T, CODE_FOR_##I}; 1858 1859 fntype = build_function_type_list (ptr_type_node, ptr_type_node, 1860 uint64_type_node, NULL); 1861 AARCH64_INIT_MEMTAG_BUILTINS_DECL (IRG, create_random_tag, irg, fntype); 1862 1863 fntype = build_function_type_list (uint64_type_node, ptr_type_node, 1864 uint64_type_node, NULL); 1865 AARCH64_INIT_MEMTAG_BUILTINS_DECL (GMI, exclude_tag, gmi, fntype); 1866 1867 fntype = build_function_type_list (ptrdiff_type_node, ptr_type_node, 1868 ptr_type_node, NULL); 1869 AARCH64_INIT_MEMTAG_BUILTINS_DECL (SUBP, ptrdiff, subp, fntype); 1870 1871 fntype = build_function_type_list (ptr_type_node, ptr_type_node, 1872 unsigned_type_node, NULL); 1873 AARCH64_INIT_MEMTAG_BUILTINS_DECL (INC_TAG, increment_tag, addg, fntype); 1874 1875 fntype = build_function_type_list (void_type_node, ptr_type_node, NULL); 1876 AARCH64_INIT_MEMTAG_BUILTINS_DECL (SET_TAG, set_tag, stg, fntype); 1877 1878 fntype = build_function_type_list (ptr_type_node, ptr_type_node, NULL); 1879 AARCH64_INIT_MEMTAG_BUILTINS_DECL (GET_TAG, get_tag, ldg, fntype); 1880 1881 #undef AARCH64_INIT_MEMTAG_BUILTINS_DECL 1882 } 1883 1884 /* Add builtins for Load/store 64 Byte instructions. */ 1885 1886 typedef struct 1887 { 1888 const char *name; 1889 unsigned int code; 1890 tree type; 1891 } ls64_builtins_data; 1892 1893 static GTY(()) tree ls64_arm_data_t = NULL_TREE; 1894 1895 static void 1896 aarch64_init_ls64_builtins_types (void) 1897 { 1898 /* Synthesize: 1899 1900 typedef struct { 1901 uint64_t val[8]; 1902 } __arm_data512_t; */ 1903 const char *tuple_type_name = "__arm_data512_t"; 1904 tree node_type = get_typenode_from_name (UINT64_TYPE); 1905 tree array_type = build_array_type_nelts (node_type, 8); 1906 SET_TYPE_MODE (array_type, V8DImode); 1907 1908 gcc_assert (TYPE_MODE_RAW (array_type) == TYPE_MODE (array_type)); 1909 gcc_assert (TYPE_ALIGN (array_type) == 64); 1910 1911 tree field = build_decl (input_location, FIELD_DECL, 1912 get_identifier ("val"), array_type); 1913 1914 ls64_arm_data_t = lang_hooks.types.simulate_record_decl (input_location, 1915 tuple_type_name, 1916 make_array_slice (&field, 1)); 1917 1918 gcc_assert (TYPE_MODE (ls64_arm_data_t) == V8DImode); 1919 gcc_assert (TYPE_MODE_RAW (ls64_arm_data_t) == TYPE_MODE (ls64_arm_data_t)); 1920 gcc_assert (TYPE_ALIGN (ls64_arm_data_t) == 64); 1921 } 1922 1923 static void 1924 aarch64_init_ls64_builtins (void) 1925 { 1926 aarch64_init_ls64_builtins_types (); 1927 1928 ls64_builtins_data data[4] = { 1929 {"__arm_ld64b", AARCH64_LS64_BUILTIN_LD64B, 1930 build_function_type_list (ls64_arm_data_t, 1931 const_ptr_type_node, NULL_TREE)}, 1932 {"__arm_st64b", AARCH64_LS64_BUILTIN_ST64B, 1933 build_function_type_list (void_type_node, ptr_type_node, 1934 ls64_arm_data_t, NULL_TREE)}, 1935 {"__arm_st64bv", AARCH64_LS64_BUILTIN_ST64BV, 1936 build_function_type_list (uint64_type_node, ptr_type_node, 1937 ls64_arm_data_t, NULL_TREE)}, 1938 {"__arm_st64bv0", AARCH64_LS64_BUILTIN_ST64BV0, 1939 build_function_type_list (uint64_type_node, ptr_type_node, 1940 ls64_arm_data_t, NULL_TREE)}, 1941 }; 1942 1943 for (size_t i = 0; i < ARRAY_SIZE (data); ++i) 1944 aarch64_builtin_decls[data[i].code] 1945 = aarch64_general_simulate_builtin (data[i].name, data[i].type, 1946 data[i].code); 1947 } 1948 1949 static void 1950 aarch64_init_data_intrinsics (void) 1951 { 1952 tree uint32_fntype = build_function_type_list (uint32_type_node, 1953 uint32_type_node, NULL_TREE); 1954 tree ulong_fntype = build_function_type_list (long_unsigned_type_node, 1955 long_unsigned_type_node, 1956 NULL_TREE); 1957 tree uint64_fntype = build_function_type_list (uint64_type_node, 1958 uint64_type_node, NULL_TREE); 1959 aarch64_builtin_decls[AARCH64_REV16] 1960 = aarch64_general_add_builtin ("__builtin_aarch64_rev16", uint32_fntype, 1961 AARCH64_REV16); 1962 aarch64_builtin_decls[AARCH64_REV16L] 1963 = aarch64_general_add_builtin ("__builtin_aarch64_rev16l", ulong_fntype, 1964 AARCH64_REV16L); 1965 aarch64_builtin_decls[AARCH64_REV16LL] 1966 = aarch64_general_add_builtin ("__builtin_aarch64_rev16ll", uint64_fntype, 1967 AARCH64_REV16LL); 1968 aarch64_builtin_decls[AARCH64_RBIT] 1969 = aarch64_general_add_builtin ("__builtin_aarch64_rbit", uint32_fntype, 1970 AARCH64_RBIT); 1971 aarch64_builtin_decls[AARCH64_RBITL] 1972 = aarch64_general_add_builtin ("__builtin_aarch64_rbitl", ulong_fntype, 1973 AARCH64_RBITL); 1974 aarch64_builtin_decls[AARCH64_RBITLL] 1975 = aarch64_general_add_builtin ("__builtin_aarch64_rbitll", uint64_fntype, 1976 AARCH64_RBITLL); 1977 } 1978 1979 /* Implement #pragma GCC aarch64 "arm_acle.h". */ 1980 void 1981 handle_arm_acle_h (void) 1982 { 1983 aarch64_init_ls64_builtins (); 1984 aarch64_init_tme_builtins (); 1985 aarch64_init_memtag_builtins (); 1986 } 1987 1988 /* Initialize fpsr fpcr getters and setters. */ 1989 1990 static void 1991 aarch64_init_fpsr_fpcr_builtins (void) 1992 { 1993 tree ftype_set 1994 = build_function_type_list (void_type_node, unsigned_type_node, NULL); 1995 tree ftype_get 1996 = build_function_type_list (unsigned_type_node, NULL); 1997 1998 aarch64_builtin_decls[AARCH64_BUILTIN_GET_FPCR] 1999 = aarch64_general_add_builtin ("__builtin_aarch64_get_fpcr", 2000 ftype_get, 2001 AARCH64_BUILTIN_GET_FPCR); 2002 aarch64_builtin_decls[AARCH64_BUILTIN_SET_FPCR] 2003 = aarch64_general_add_builtin ("__builtin_aarch64_set_fpcr", 2004 ftype_set, 2005 AARCH64_BUILTIN_SET_FPCR); 2006 aarch64_builtin_decls[AARCH64_BUILTIN_GET_FPSR] 2007 = aarch64_general_add_builtin ("__builtin_aarch64_get_fpsr", 2008 ftype_get, 2009 AARCH64_BUILTIN_GET_FPSR); 2010 aarch64_builtin_decls[AARCH64_BUILTIN_SET_FPSR] 2011 = aarch64_general_add_builtin ("__builtin_aarch64_set_fpsr", 2012 ftype_set, 2013 AARCH64_BUILTIN_SET_FPSR); 2014 2015 ftype_set 2016 = build_function_type_list (void_type_node, long_long_unsigned_type_node, 2017 NULL); 2018 ftype_get 2019 = build_function_type_list (long_long_unsigned_type_node, NULL); 2020 2021 aarch64_builtin_decls[AARCH64_BUILTIN_GET_FPCR64] 2022 = aarch64_general_add_builtin ("__builtin_aarch64_get_fpcr64", 2023 ftype_get, 2024 AARCH64_BUILTIN_GET_FPCR64); 2025 aarch64_builtin_decls[AARCH64_BUILTIN_SET_FPCR64] 2026 = aarch64_general_add_builtin ("__builtin_aarch64_set_fpcr64", 2027 ftype_set, 2028 AARCH64_BUILTIN_SET_FPCR64); 2029 aarch64_builtin_decls[AARCH64_BUILTIN_GET_FPSR64] 2030 = aarch64_general_add_builtin ("__builtin_aarch64_get_fpsr64", 2031 ftype_get, 2032 AARCH64_BUILTIN_GET_FPSR64); 2033 aarch64_builtin_decls[AARCH64_BUILTIN_SET_FPSR64] 2034 = aarch64_general_add_builtin ("__builtin_aarch64_set_fpsr64", 2035 ftype_set, 2036 AARCH64_BUILTIN_SET_FPSR64); 2037 } 2038 2039 /* Initialize all builtins in the AARCH64_BUILTIN_GENERAL group. */ 2040 2041 void 2042 aarch64_general_init_builtins (void) 2043 { 2044 aarch64_init_fpsr_fpcr_builtins (); 2045 2046 aarch64_init_fp16_types (); 2047 2048 aarch64_init_bf16_types (); 2049 2050 { 2051 aarch64_simd_switcher simd; 2052 aarch64_init_simd_builtins (); 2053 } 2054 2055 aarch64_init_crc32_builtins (); 2056 aarch64_init_builtin_rsqrt (); 2057 aarch64_init_rng_builtins (); 2058 aarch64_init_data_intrinsics (); 2059 2060 aarch64_init_rwsr_builtins (); 2061 aarch64_init_prefetch_builtin (); 2062 2063 tree ftype_jcvt 2064 = build_function_type_list (intSI_type_node, double_type_node, NULL); 2065 aarch64_builtin_decls[AARCH64_JSCVT] 2066 = aarch64_general_add_builtin ("__builtin_aarch64_jcvtzs", ftype_jcvt, 2067 AARCH64_JSCVT); 2068 2069 /* Initialize pointer authentication builtins which are backed by instructions 2070 in NOP encoding space. 2071 2072 NOTE: these builtins are supposed to be used by libgcc unwinder only, as 2073 there is no support on return address signing under ILP32, we don't 2074 register them. */ 2075 if (!TARGET_ILP32) 2076 aarch64_init_pauth_hint_builtins (); 2077 2078 if (in_lto_p) 2079 handle_arm_acle_h (); 2080 } 2081 2082 /* Implement TARGET_BUILTIN_DECL for the AARCH64_BUILTIN_GENERAL group. */ 2083 tree 2084 aarch64_general_builtin_decl (unsigned code, bool) 2085 { 2086 if (code >= AARCH64_BUILTIN_MAX) 2087 return error_mark_node; 2088 2089 return aarch64_builtin_decls[code]; 2090 } 2091 2092 /* True if we've already complained about attempts to use functions 2093 when the required extension is disabled. */ 2094 static bool reported_missing_extension_p; 2095 2096 /* True if we've already complained about attempts to use functions 2097 which require registers that are missing. */ 2098 static bool reported_missing_registers_p; 2099 2100 /* Report an error against LOCATION that the user has tried to use 2101 function FNDECL when extension EXTENSION is disabled. */ 2102 static void 2103 aarch64_report_missing_extension (location_t location, tree fndecl, 2104 const char *extension) 2105 { 2106 /* Avoid reporting a slew of messages for a single oversight. */ 2107 if (reported_missing_extension_p) 2108 return; 2109 2110 error_at (location, "ACLE function %qD requires ISA extension %qs", 2111 fndecl, extension); 2112 inform (location, "you can enable %qs using the command-line" 2113 " option %<-march%>, or by using the %<target%>" 2114 " attribute or pragma", extension); 2115 reported_missing_extension_p = true; 2116 } 2117 2118 /* Report an error against LOCATION that the user has tried to use 2119 function FNDECL when non-general registers are disabled. */ 2120 static void 2121 aarch64_report_missing_registers (location_t location, tree fndecl) 2122 { 2123 /* Avoid reporting a slew of messages for a single oversight. */ 2124 if (reported_missing_registers_p) 2125 return; 2126 2127 error_at (location, 2128 "ACLE function %qD is incompatible with the use of %qs", 2129 fndecl, "-mgeneral-regs-only"); 2130 reported_missing_registers_p = true; 2131 } 2132 2133 /* Check whether all the AARCH64_FL_* values in REQUIRED_EXTENSIONS are 2134 enabled, given that those extensions are required for function FNDECL. 2135 Report an error against LOCATION if not. */ 2136 bool 2137 aarch64_check_required_extensions (location_t location, tree fndecl, 2138 aarch64_feature_flags required_extensions) 2139 { 2140 if ((required_extensions & ~aarch64_isa_flags) == 0) 2141 return true; 2142 2143 auto missing_extensions = required_extensions & ~aarch64_asm_isa_flags; 2144 2145 if (missing_extensions == 0) 2146 { 2147 /* All required extensions are enabled in aarch64_asm_isa_flags, so the 2148 error must be the use of general-regs-only. */ 2149 aarch64_report_missing_registers (location, fndecl); 2150 return false; 2151 } 2152 2153 if (missing_extensions & AARCH64_FL_SM_OFF) 2154 { 2155 error_at (location, "ACLE function %qD cannot be called when" 2156 " SME streaming mode is enabled", fndecl); 2157 return false; 2158 } 2159 2160 if (missing_extensions & AARCH64_FL_SM_ON) 2161 { 2162 error_at (location, "ACLE function %qD can only be called when" 2163 " SME streaming mode is enabled", fndecl); 2164 return false; 2165 } 2166 2167 if (missing_extensions & AARCH64_FL_ZA_ON) 2168 { 2169 error_at (location, "ACLE function %qD can only be called from" 2170 " a function that has %qs state", fndecl, "za"); 2171 return false; 2172 } 2173 2174 static const struct { 2175 aarch64_feature_flags flag; 2176 const char *name; 2177 } extensions[] = { 2178 #define AARCH64_OPT_EXTENSION(EXT_NAME, IDENT, C, D, E, F) \ 2179 { AARCH64_FL_##IDENT, EXT_NAME }, 2180 #include "aarch64-option-extensions.def" 2181 }; 2182 2183 for (unsigned int i = 0; i < ARRAY_SIZE (extensions); ++i) 2184 if (missing_extensions & extensions[i].flag) 2185 { 2186 aarch64_report_missing_extension (location, fndecl, extensions[i].name); 2187 return false; 2188 } 2189 gcc_unreachable (); 2190 } 2191 2192 bool 2193 aarch64_general_check_builtin_call (location_t location, vec<location_t>, 2194 unsigned int code, tree fndecl, 2195 unsigned int nargs ATTRIBUTE_UNUSED, tree *args) 2196 { 2197 tree decl = aarch64_builtin_decls[code]; 2198 switch (code) 2199 { 2200 case AARCH64_RSR: 2201 case AARCH64_RSRP: 2202 case AARCH64_RSR64: 2203 case AARCH64_RSRF: 2204 case AARCH64_RSRF64: 2205 case AARCH64_WSR: 2206 case AARCH64_WSRP: 2207 case AARCH64_WSR64: 2208 case AARCH64_WSRF: 2209 case AARCH64_WSRF64: 2210 { 2211 tree addr = STRIP_NOPS (args[0]); 2212 if (TREE_CODE (TREE_TYPE (addr)) != POINTER_TYPE 2213 || TREE_CODE (addr) != ADDR_EXPR 2214 || TREE_CODE (TREE_OPERAND (addr, 0)) != STRING_CST) 2215 { 2216 error_at (location, 2217 "first argument to %qD must be a string literal", 2218 fndecl); 2219 return false; 2220 } 2221 break; 2222 } 2223 2224 case AARCH64_TME_BUILTIN_TSTART: 2225 case AARCH64_TME_BUILTIN_TCOMMIT: 2226 case AARCH64_TME_BUILTIN_TTEST: 2227 case AARCH64_TME_BUILTIN_TCANCEL: 2228 return aarch64_check_required_extensions (location, decl, 2229 AARCH64_FL_TME); 2230 2231 case AARCH64_LS64_BUILTIN_LD64B: 2232 case AARCH64_LS64_BUILTIN_ST64B: 2233 case AARCH64_LS64_BUILTIN_ST64BV: 2234 case AARCH64_LS64_BUILTIN_ST64BV0: 2235 return aarch64_check_required_extensions (location, decl, 2236 AARCH64_FL_LS64); 2237 2238 default: 2239 break; 2240 } 2241 2242 if (code >= AARCH64_MEMTAG_BUILTIN_START 2243 && code <= AARCH64_MEMTAG_BUILTIN_END) 2244 return aarch64_check_required_extensions (location, decl, 2245 AARCH64_FL_MEMTAG); 2246 2247 return true; 2248 } 2249 2250 typedef enum 2251 { 2252 SIMD_ARG_COPY_TO_REG, 2253 SIMD_ARG_CONSTANT, 2254 SIMD_ARG_LANE_INDEX, 2255 SIMD_ARG_STRUCT_LOAD_STORE_LANE_INDEX, 2256 SIMD_ARG_LANE_PAIR_INDEX, 2257 SIMD_ARG_LANE_QUADTUP_INDEX, 2258 SIMD_ARG_STOP 2259 } builtin_simd_arg; 2260 2261 2262 static rtx 2263 aarch64_simd_expand_args (rtx target, int icode, int have_retval, 2264 tree exp, builtin_simd_arg *args, 2265 machine_mode builtin_mode) 2266 { 2267 rtx pat; 2268 rtx op[SIMD_MAX_BUILTIN_ARGS + 1]; /* First element for result operand. */ 2269 int opc = 0; 2270 2271 if (have_retval) 2272 { 2273 machine_mode tmode = insn_data[icode].operand[0].mode; 2274 if (!target 2275 || GET_MODE (target) != tmode 2276 || !(*insn_data[icode].operand[0].predicate) (target, tmode)) 2277 target = gen_reg_rtx (tmode); 2278 op[opc++] = target; 2279 } 2280 2281 for (;;) 2282 { 2283 builtin_simd_arg thisarg = args[opc - have_retval]; 2284 2285 if (thisarg == SIMD_ARG_STOP) 2286 break; 2287 else 2288 { 2289 tree arg = CALL_EXPR_ARG (exp, opc - have_retval); 2290 machine_mode mode = insn_data[icode].operand[opc].mode; 2291 op[opc] = expand_normal (arg); 2292 2293 switch (thisarg) 2294 { 2295 case SIMD_ARG_COPY_TO_REG: 2296 if (POINTER_TYPE_P (TREE_TYPE (arg))) 2297 op[opc] = convert_memory_address (Pmode, op[opc]); 2298 /*gcc_assert (GET_MODE (op[opc]) == mode); */ 2299 if (!(*insn_data[icode].operand[opc].predicate) 2300 (op[opc], mode)) 2301 op[opc] = copy_to_mode_reg (mode, op[opc]); 2302 break; 2303 2304 case SIMD_ARG_STRUCT_LOAD_STORE_LANE_INDEX: 2305 gcc_assert (opc > 1); 2306 if (CONST_INT_P (op[opc])) 2307 { 2308 unsigned int nunits 2309 = GET_MODE_NUNITS (builtin_mode).to_constant (); 2310 aarch64_simd_lane_bounds (op[opc], 0, nunits, exp); 2311 /* Keep to GCC-vector-extension lane indices in the RTL. */ 2312 op[opc] = aarch64_endian_lane_rtx (builtin_mode, 2313 INTVAL (op[opc])); 2314 } 2315 goto constant_arg; 2316 2317 case SIMD_ARG_LANE_INDEX: 2318 /* Must be a previous operand into which this is an index. */ 2319 gcc_assert (opc > 0); 2320 if (CONST_INT_P (op[opc])) 2321 { 2322 machine_mode vmode = insn_data[icode].operand[opc - 1].mode; 2323 unsigned int nunits 2324 = GET_MODE_NUNITS (vmode).to_constant (); 2325 aarch64_simd_lane_bounds (op[opc], 0, nunits, exp); 2326 /* Keep to GCC-vector-extension lane indices in the RTL. */ 2327 op[opc] = aarch64_endian_lane_rtx (vmode, INTVAL (op[opc])); 2328 } 2329 /* If the lane index isn't a constant then error out. */ 2330 goto constant_arg; 2331 2332 case SIMD_ARG_LANE_PAIR_INDEX: 2333 /* Must be a previous operand into which this is an index and 2334 index is restricted to nunits / 2. */ 2335 gcc_assert (opc > 0); 2336 if (CONST_INT_P (op[opc])) 2337 { 2338 machine_mode vmode = insn_data[icode].operand[opc - 1].mode; 2339 unsigned int nunits 2340 = GET_MODE_NUNITS (vmode).to_constant (); 2341 aarch64_simd_lane_bounds (op[opc], 0, nunits / 2, exp); 2342 /* Keep to GCC-vector-extension lane indices in the RTL. */ 2343 int lane = INTVAL (op[opc]); 2344 op[opc] = gen_int_mode (ENDIAN_LANE_N (nunits / 2, lane), 2345 SImode); 2346 } 2347 /* If the lane index isn't a constant then error out. */ 2348 goto constant_arg; 2349 case SIMD_ARG_LANE_QUADTUP_INDEX: 2350 /* Must be a previous operand into which this is an index and 2351 index is restricted to nunits / 4. */ 2352 gcc_assert (opc > 0); 2353 if (CONST_INT_P (op[opc])) 2354 { 2355 machine_mode vmode = insn_data[icode].operand[opc - 1].mode; 2356 unsigned int nunits 2357 = GET_MODE_NUNITS (vmode).to_constant (); 2358 aarch64_simd_lane_bounds (op[opc], 0, nunits / 4, exp); 2359 /* Keep to GCC-vector-extension lane indices in the RTL. */ 2360 int lane = INTVAL (op[opc]); 2361 op[opc] = gen_int_mode (ENDIAN_LANE_N (nunits / 4, lane), 2362 SImode); 2363 } 2364 /* If the lane index isn't a constant then error out. */ 2365 goto constant_arg; 2366 case SIMD_ARG_CONSTANT: 2367 constant_arg: 2368 if (!(*insn_data[icode].operand[opc].predicate) 2369 (op[opc], mode)) 2370 { 2371 error_at (EXPR_LOCATION (exp), 2372 "argument %d must be a constant immediate", 2373 opc + 1 - have_retval); 2374 return const0_rtx; 2375 } 2376 break; 2377 2378 case SIMD_ARG_STOP: 2379 gcc_unreachable (); 2380 } 2381 2382 opc++; 2383 } 2384 } 2385 2386 switch (opc) 2387 { 2388 case 1: 2389 pat = GEN_FCN (icode) (op[0]); 2390 break; 2391 2392 case 2: 2393 pat = GEN_FCN (icode) (op[0], op[1]); 2394 break; 2395 2396 case 3: 2397 pat = GEN_FCN (icode) (op[0], op[1], op[2]); 2398 break; 2399 2400 case 4: 2401 pat = GEN_FCN (icode) (op[0], op[1], op[2], op[3]); 2402 break; 2403 2404 case 5: 2405 pat = GEN_FCN (icode) (op[0], op[1], op[2], op[3], op[4]); 2406 break; 2407 2408 case 6: 2409 pat = GEN_FCN (icode) (op[0], op[1], op[2], op[3], op[4], op[5]); 2410 break; 2411 2412 default: 2413 gcc_unreachable (); 2414 } 2415 2416 if (!pat) 2417 return NULL_RTX; 2418 2419 emit_insn (pat); 2420 2421 return target; 2422 } 2423 2424 /* Expand an AArch64 AdvSIMD builtin(intrinsic). */ 2425 rtx 2426 aarch64_simd_expand_builtin (int fcode, tree exp, rtx target) 2427 { 2428 if (fcode == AARCH64_SIMD_BUILTIN_LANE_CHECK) 2429 { 2430 rtx totalsize = expand_normal (CALL_EXPR_ARG (exp, 0)); 2431 rtx elementsize = expand_normal (CALL_EXPR_ARG (exp, 1)); 2432 if (CONST_INT_P (totalsize) && CONST_INT_P (elementsize) 2433 && UINTVAL (elementsize) != 0 2434 && UINTVAL (totalsize) != 0) 2435 { 2436 rtx lane_idx = expand_normal (CALL_EXPR_ARG (exp, 2)); 2437 if (CONST_INT_P (lane_idx)) 2438 aarch64_simd_lane_bounds (lane_idx, 0, 2439 UINTVAL (totalsize) 2440 / UINTVAL (elementsize), 2441 exp); 2442 else 2443 error_at (EXPR_LOCATION (exp), 2444 "lane index must be a constant immediate"); 2445 } 2446 else 2447 error_at (EXPR_LOCATION (exp), 2448 "total size and element size must be a nonzero " 2449 "constant immediate"); 2450 /* Don't generate any RTL. */ 2451 return const0_rtx; 2452 } 2453 aarch64_simd_builtin_datum *d = 2454 &aarch64_simd_builtin_data[fcode - AARCH64_SIMD_PATTERN_START]; 2455 enum insn_code icode = d->code; 2456 builtin_simd_arg args[SIMD_MAX_BUILTIN_ARGS + 1]; 2457 int num_args = insn_data[d->code].n_operands; 2458 int is_void = 0; 2459 int k; 2460 2461 is_void = !!(d->qualifiers[0] & qualifier_void); 2462 2463 num_args += is_void; 2464 2465 for (k = 1; k < num_args; k++) 2466 { 2467 /* We have four arrays of data, each indexed in a different fashion. 2468 qualifiers - element 0 always describes the function return type. 2469 operands - element 0 is either the operand for return value (if 2470 the function has a non-void return type) or the operand for the 2471 first argument. 2472 expr_args - element 0 always holds the first argument. 2473 args - element 0 is always used for the return type. */ 2474 int qualifiers_k = k; 2475 int operands_k = k - is_void; 2476 int expr_args_k = k - 1; 2477 2478 if (d->qualifiers[qualifiers_k] & qualifier_lane_index) 2479 args[k] = SIMD_ARG_LANE_INDEX; 2480 else if (d->qualifiers[qualifiers_k] & qualifier_lane_pair_index) 2481 args[k] = SIMD_ARG_LANE_PAIR_INDEX; 2482 else if (d->qualifiers[qualifiers_k] & qualifier_lane_quadtup_index) 2483 args[k] = SIMD_ARG_LANE_QUADTUP_INDEX; 2484 else if (d->qualifiers[qualifiers_k] & qualifier_struct_load_store_lane_index) 2485 args[k] = SIMD_ARG_STRUCT_LOAD_STORE_LANE_INDEX; 2486 else if (d->qualifiers[qualifiers_k] & qualifier_immediate) 2487 args[k] = SIMD_ARG_CONSTANT; 2488 else if (d->qualifiers[qualifiers_k] & qualifier_maybe_immediate) 2489 { 2490 rtx arg 2491 = expand_normal (CALL_EXPR_ARG (exp, 2492 (expr_args_k))); 2493 /* Handle constants only if the predicate allows it. */ 2494 bool op_const_int_p = 2495 (CONST_INT_P (arg) 2496 && (*insn_data[icode].operand[operands_k].predicate) 2497 (arg, insn_data[icode].operand[operands_k].mode)); 2498 args[k] = op_const_int_p ? SIMD_ARG_CONSTANT : SIMD_ARG_COPY_TO_REG; 2499 } 2500 else 2501 args[k] = SIMD_ARG_COPY_TO_REG; 2502 2503 } 2504 args[k] = SIMD_ARG_STOP; 2505 2506 /* The interface to aarch64_simd_expand_args expects a 0 if 2507 the function is void, and a 1 if it is not. */ 2508 return aarch64_simd_expand_args 2509 (target, icode, !is_void, exp, &args[1], d->mode); 2510 } 2511 2512 rtx 2513 aarch64_crc32_expand_builtin (int fcode, tree exp, rtx target) 2514 { 2515 rtx pat; 2516 aarch64_crc_builtin_datum *d 2517 = &aarch64_crc_builtin_data[fcode - (AARCH64_CRC32_BUILTIN_BASE + 1)]; 2518 enum insn_code icode = d->icode; 2519 tree arg0 = CALL_EXPR_ARG (exp, 0); 2520 tree arg1 = CALL_EXPR_ARG (exp, 1); 2521 rtx op0 = expand_normal (arg0); 2522 rtx op1 = expand_normal (arg1); 2523 machine_mode tmode = insn_data[icode].operand[0].mode; 2524 machine_mode mode0 = insn_data[icode].operand[1].mode; 2525 machine_mode mode1 = insn_data[icode].operand[2].mode; 2526 2527 if (! target 2528 || GET_MODE (target) != tmode 2529 || ! (*insn_data[icode].operand[0].predicate) (target, tmode)) 2530 target = gen_reg_rtx (tmode); 2531 2532 gcc_assert ((GET_MODE (op0) == mode0 || GET_MODE (op0) == VOIDmode) 2533 && (GET_MODE (op1) == mode1 || GET_MODE (op1) == VOIDmode)); 2534 2535 if (! (*insn_data[icode].operand[1].predicate) (op0, mode0)) 2536 op0 = copy_to_mode_reg (mode0, op0); 2537 if (! (*insn_data[icode].operand[2].predicate) (op1, mode1)) 2538 op1 = copy_to_mode_reg (mode1, op1); 2539 2540 pat = GEN_FCN (icode) (target, op0, op1); 2541 if (!pat) 2542 return NULL_RTX; 2543 2544 emit_insn (pat); 2545 return target; 2546 } 2547 2548 /* Function to expand reciprocal square root builtins. */ 2549 2550 static rtx 2551 aarch64_expand_builtin_rsqrt (int fcode, tree exp, rtx target) 2552 { 2553 tree arg0 = CALL_EXPR_ARG (exp, 0); 2554 rtx op0 = expand_normal (arg0); 2555 2556 rtx (*gen) (rtx, rtx); 2557 2558 switch (fcode) 2559 { 2560 case AARCH64_BUILTIN_RSQRT_DF: 2561 gen = gen_rsqrtdf2; 2562 break; 2563 case AARCH64_BUILTIN_RSQRT_SF: 2564 gen = gen_rsqrtsf2; 2565 break; 2566 case AARCH64_BUILTIN_RSQRT_V2DF: 2567 gen = gen_rsqrtv2df2; 2568 break; 2569 case AARCH64_BUILTIN_RSQRT_V2SF: 2570 gen = gen_rsqrtv2sf2; 2571 break; 2572 case AARCH64_BUILTIN_RSQRT_V4SF: 2573 gen = gen_rsqrtv4sf2; 2574 break; 2575 default: gcc_unreachable (); 2576 } 2577 2578 if (!target) 2579 target = gen_reg_rtx (GET_MODE (op0)); 2580 2581 emit_insn (gen (target, op0)); 2582 2583 return target; 2584 } 2585 2586 /* Expand a FCMLA lane expression EXP with code FCODE and 2587 result going to TARGET if that is convenient. */ 2588 2589 rtx 2590 aarch64_expand_fcmla_builtin (tree exp, rtx target, int fcode) 2591 { 2592 int bcode = fcode - AARCH64_SIMD_FCMLA_LANEQ_BUILTIN_BASE - 1; 2593 aarch64_fcmla_laneq_builtin_datum* d 2594 = &aarch64_fcmla_lane_builtin_data[bcode]; 2595 machine_mode quadmode = GET_MODE_2XWIDER_MODE (d->mode).require (); 2596 rtx op0 = force_reg (d->mode, expand_normal (CALL_EXPR_ARG (exp, 0))); 2597 rtx op1 = force_reg (d->mode, expand_normal (CALL_EXPR_ARG (exp, 1))); 2598 rtx op2 = force_reg (quadmode, expand_normal (CALL_EXPR_ARG (exp, 2))); 2599 tree tmp = CALL_EXPR_ARG (exp, 3); 2600 rtx lane_idx = expand_expr (tmp, NULL_RTX, VOIDmode, EXPAND_INITIALIZER); 2601 2602 /* Validate that the lane index is a constant. */ 2603 if (!CONST_INT_P (lane_idx)) 2604 { 2605 error_at (EXPR_LOCATION (exp), 2606 "argument %d must be a constant immediate", 4); 2607 return const0_rtx; 2608 } 2609 2610 /* Validate that the index is within the expected range. */ 2611 int nunits = GET_MODE_NUNITS (quadmode).to_constant (); 2612 aarch64_simd_lane_bounds (lane_idx, 0, nunits / 2, exp); 2613 2614 /* Generate the correct register and mode. */ 2615 int lane = INTVAL (lane_idx); 2616 2617 if (lane < nunits / 4) 2618 op2 = simplify_gen_subreg (d->mode, op2, quadmode, 2619 subreg_lowpart_offset (d->mode, quadmode)); 2620 else 2621 { 2622 /* Select the upper 64 bits, either a V2SF or V4HF, this however 2623 is quite messy, as the operation required even though simple 2624 doesn't have a simple RTL pattern, and seems it's quite hard to 2625 define using a single RTL pattern. The target generic version 2626 gen_highpart_mode generates code that isn't optimal. */ 2627 rtx temp1 = gen_reg_rtx (d->mode); 2628 rtx temp2 = gen_reg_rtx (DImode); 2629 temp1 = simplify_gen_subreg (d->mode, op2, quadmode, 2630 subreg_lowpart_offset (d->mode, quadmode)); 2631 temp1 = simplify_gen_subreg (V2DImode, temp1, d->mode, 0); 2632 if (BYTES_BIG_ENDIAN) 2633 emit_insn (gen_aarch64_get_lanev2di (temp2, temp1, const0_rtx)); 2634 else 2635 emit_insn (gen_aarch64_get_lanev2di (temp2, temp1, const1_rtx)); 2636 op2 = simplify_gen_subreg (d->mode, temp2, GET_MODE (temp2), 0); 2637 2638 /* And recalculate the index. */ 2639 lane -= nunits / 4; 2640 } 2641 2642 /* Keep to GCC-vector-extension lane indices in the RTL, only nunits / 4 2643 (max nunits in range check) are valid. Which means only 0-1, so we 2644 only need to know the order in a V2mode. */ 2645 lane_idx = aarch64_endian_lane_rtx (V2DImode, lane); 2646 2647 if (!target 2648 || !REG_P (target) 2649 || GET_MODE (target) != d->mode) 2650 target = gen_reg_rtx (d->mode); 2651 2652 rtx pat = NULL_RTX; 2653 2654 if (d->lane) 2655 pat = GEN_FCN (d->icode) (target, op0, op1, op2, lane_idx); 2656 else 2657 pat = GEN_FCN (d->icode) (target, op0, op1, op2); 2658 2659 if (!pat) 2660 return NULL_RTX; 2661 2662 emit_insn (pat); 2663 return target; 2664 } 2665 2666 /* Function to expand an expression EXP which calls one of the Transactional 2667 Memory Extension (TME) builtins FCODE with the result going to TARGET. */ 2668 static rtx 2669 aarch64_expand_builtin_tme (int fcode, tree exp, rtx target) 2670 { 2671 switch (fcode) 2672 { 2673 case AARCH64_TME_BUILTIN_TSTART: 2674 target = gen_reg_rtx (DImode); 2675 emit_insn (GEN_FCN (CODE_FOR_tstart) (target)); 2676 break; 2677 2678 case AARCH64_TME_BUILTIN_TTEST: 2679 target = gen_reg_rtx (DImode); 2680 emit_insn (GEN_FCN (CODE_FOR_ttest) (target)); 2681 break; 2682 2683 case AARCH64_TME_BUILTIN_TCOMMIT: 2684 emit_insn (GEN_FCN (CODE_FOR_tcommit) ()); 2685 break; 2686 2687 case AARCH64_TME_BUILTIN_TCANCEL: 2688 { 2689 tree arg0 = CALL_EXPR_ARG (exp, 0); 2690 rtx op0 = expand_normal (arg0); 2691 if (CONST_INT_P (op0) && UINTVAL (op0) <= 65536) 2692 emit_insn (GEN_FCN (CODE_FOR_tcancel) (op0)); 2693 else 2694 { 2695 error_at (EXPR_LOCATION (exp), 2696 "argument must be a 16-bit constant immediate"); 2697 return const0_rtx; 2698 } 2699 } 2700 break; 2701 2702 default : 2703 gcc_unreachable (); 2704 } 2705 return target; 2706 } 2707 2708 /* Function to expand an expression EXP which calls one of the Load/Store 2709 64 Byte extension (LS64) builtins FCODE with the result going to TARGET. */ 2710 static rtx 2711 aarch64_expand_builtin_ls64 (int fcode, tree exp, rtx target) 2712 { 2713 expand_operand ops[3]; 2714 2715 switch (fcode) 2716 { 2717 case AARCH64_LS64_BUILTIN_LD64B: 2718 { 2719 rtx op0 = expand_normal (CALL_EXPR_ARG (exp, 0)); 2720 create_output_operand (&ops[0], target, V8DImode); 2721 create_input_operand (&ops[1], op0, DImode); 2722 expand_insn (CODE_FOR_ld64b, 2, ops); 2723 return ops[0].value; 2724 } 2725 case AARCH64_LS64_BUILTIN_ST64B: 2726 { 2727 rtx op0 = expand_normal (CALL_EXPR_ARG (exp, 0)); 2728 rtx op1 = expand_normal (CALL_EXPR_ARG (exp, 1)); 2729 create_input_operand (&ops[0], op0, DImode); 2730 create_input_operand (&ops[1], op1, V8DImode); 2731 expand_insn (CODE_FOR_st64b, 2, ops); 2732 return const0_rtx; 2733 } 2734 case AARCH64_LS64_BUILTIN_ST64BV: 2735 { 2736 rtx op0 = expand_normal (CALL_EXPR_ARG (exp, 0)); 2737 rtx op1 = expand_normal (CALL_EXPR_ARG (exp, 1)); 2738 create_output_operand (&ops[0], target, DImode); 2739 create_input_operand (&ops[1], op0, DImode); 2740 create_input_operand (&ops[2], op1, V8DImode); 2741 expand_insn (CODE_FOR_st64bv, 3, ops); 2742 return ops[0].value; 2743 } 2744 case AARCH64_LS64_BUILTIN_ST64BV0: 2745 { 2746 rtx op0 = expand_normal (CALL_EXPR_ARG (exp, 0)); 2747 rtx op1 = expand_normal (CALL_EXPR_ARG (exp, 1)); 2748 create_output_operand (&ops[0], target, DImode); 2749 create_input_operand (&ops[1], op0, DImode); 2750 create_input_operand (&ops[2], op1, V8DImode); 2751 expand_insn (CODE_FOR_st64bv0, 3, ops); 2752 return ops[0].value; 2753 } 2754 } 2755 2756 gcc_unreachable (); 2757 } 2758 2759 /* Expand a random number builtin EXP with code FCODE, putting the result 2760 int TARGET. If IGNORE is true the return value is ignored. */ 2761 2762 rtx 2763 aarch64_expand_rng_builtin (tree exp, rtx target, int fcode, int ignore) 2764 { 2765 rtx pat; 2766 enum insn_code icode; 2767 if (fcode == AARCH64_BUILTIN_RNG_RNDR) 2768 icode = CODE_FOR_aarch64_rndr; 2769 else if (fcode == AARCH64_BUILTIN_RNG_RNDRRS) 2770 icode = CODE_FOR_aarch64_rndrrs; 2771 else 2772 gcc_unreachable (); 2773 2774 rtx rand = gen_reg_rtx (DImode); 2775 pat = GEN_FCN (icode) (rand); 2776 if (!pat) 2777 return NULL_RTX; 2778 2779 tree arg0 = CALL_EXPR_ARG (exp, 0); 2780 rtx res_addr = expand_normal (arg0); 2781 res_addr = convert_memory_address (Pmode, res_addr); 2782 rtx res_mem = gen_rtx_MEM (DImode, res_addr); 2783 emit_insn (pat); 2784 emit_move_insn (res_mem, rand); 2785 /* If the status result is unused don't generate the CSET code. */ 2786 if (ignore) 2787 return target; 2788 2789 rtx cc_reg = gen_rtx_REG (CC_Zmode, CC_REGNUM); 2790 rtx cmp_rtx = gen_rtx_fmt_ee (EQ, SImode, cc_reg, const0_rtx); 2791 emit_insn (gen_aarch64_cstoresi (target, cmp_rtx, cc_reg)); 2792 return target; 2793 } 2794 2795 /* Expand the read/write system register builtin EXPs. */ 2796 rtx 2797 aarch64_expand_rwsr_builtin (tree exp, rtx target, int fcode) 2798 { 2799 tree arg0, arg1; 2800 rtx const_str, input_val, subreg; 2801 enum machine_mode mode; 2802 enum insn_code icode; 2803 class expand_operand ops[2]; 2804 2805 arg0 = CALL_EXPR_ARG (exp, 0); 2806 2807 bool write_op = (fcode == AARCH64_WSR 2808 || fcode == AARCH64_WSRP 2809 || fcode == AARCH64_WSR64 2810 || fcode == AARCH64_WSRF 2811 || fcode == AARCH64_WSRF64 2812 || fcode == AARCH64_WSR128); 2813 2814 bool op128 = (fcode == AARCH64_RSR128 || fcode == AARCH64_WSR128); 2815 enum machine_mode sysreg_mode = op128 ? TImode : DImode; 2816 2817 if (op128 && !TARGET_D128) 2818 { 2819 error_at (EXPR_LOCATION (exp), "128-bit system register support requires" 2820 " the %<d128%> extension"); 2821 return const0_rtx; 2822 } 2823 2824 /* Argument 0 (system register name) must be a string literal. */ 2825 gcc_assert (TREE_CODE (arg0) == ADDR_EXPR 2826 && TREE_CODE (TREE_TYPE (arg0)) == POINTER_TYPE 2827 && TREE_CODE (TREE_OPERAND (arg0, 0)) == STRING_CST); 2828 2829 const char *name_input = TREE_STRING_POINTER (TREE_OPERAND (arg0, 0)); 2830 2831 tree len_tree = c_strlen (arg0, 1); 2832 if (len_tree == NULL_TREE) 2833 { 2834 error_at (EXPR_LOCATION (exp), "invalid system register name provided"); 2835 return const0_rtx; 2836 } 2837 2838 size_t len = TREE_INT_CST_LOW (len_tree); 2839 char *sysreg_name = xstrdup (name_input); 2840 2841 for (unsigned pos = 0; pos <= len; pos++) 2842 sysreg_name[pos] = TOLOWER (sysreg_name[pos]); 2843 2844 const char* name_output = aarch64_retrieve_sysreg ((const char *) sysreg_name, 2845 write_op, op128); 2846 if (name_output == NULL) 2847 { 2848 error_at (EXPR_LOCATION (exp), "invalid system register name %qs", 2849 sysreg_name); 2850 return const0_rtx; 2851 } 2852 2853 /* Assign the string corresponding to the system register name to an RTX. */ 2854 const_str = rtx_alloc (CONST_STRING); 2855 PUT_CODE (const_str, CONST_STRING); 2856 XSTR (const_str, 0) = ggc_strdup (name_output); 2857 2858 /* Set up expander operands and call instruction expansion. */ 2859 if (write_op) 2860 { 2861 arg1 = CALL_EXPR_ARG (exp, 1); 2862 mode = TYPE_MODE (TREE_TYPE (arg1)); 2863 input_val = copy_to_mode_reg (mode, expand_normal (arg1)); 2864 2865 icode = (op128 ? CODE_FOR_aarch64_write_sysregti 2866 : CODE_FOR_aarch64_write_sysregdi); 2867 2868 switch (fcode) 2869 { 2870 case AARCH64_WSR: 2871 case AARCH64_WSRP: 2872 case AARCH64_WSR64: 2873 case AARCH64_WSRF64: 2874 case AARCH64_WSR128: 2875 subreg = lowpart_subreg (sysreg_mode, input_val, mode); 2876 break; 2877 case AARCH64_WSRF: 2878 subreg = gen_lowpart_SUBREG (SImode, input_val); 2879 subreg = gen_lowpart_SUBREG (DImode, subreg); 2880 break; 2881 } 2882 2883 create_fixed_operand (&ops[0], const_str); 2884 create_input_operand (&ops[1], subreg, sysreg_mode); 2885 expand_insn (icode, 2, ops); 2886 2887 return target; 2888 } 2889 2890 /* Read operations are implied by !write_op. */ 2891 gcc_assert (call_expr_nargs (exp) == 1); 2892 2893 icode = (op128 ? CODE_FOR_aarch64_read_sysregti 2894 : CODE_FOR_aarch64_read_sysregdi); 2895 2896 /* Emit the initial read_sysregdi rtx. */ 2897 create_output_operand (&ops[0], target, sysreg_mode); 2898 create_fixed_operand (&ops[1], const_str); 2899 expand_insn (icode, 2, ops); 2900 target = ops[0].value; 2901 2902 /* Do any necessary post-processing on the result. */ 2903 switch (fcode) 2904 { 2905 case AARCH64_RSR: 2906 case AARCH64_RSRP: 2907 case AARCH64_RSR64: 2908 case AARCH64_RSRF64: 2909 case AARCH64_RSR128: 2910 return lowpart_subreg (TYPE_MODE (TREE_TYPE (exp)), target, sysreg_mode); 2911 case AARCH64_RSRF: 2912 subreg = gen_lowpart_SUBREG (SImode, target); 2913 return gen_lowpart_SUBREG (SFmode, subreg); 2914 default: 2915 gcc_unreachable (); 2916 } 2917 } 2918 2919 /* Ensure argument ARGNO in EXP represents a const-type argument in the range 2920 [MINVAL, MAXVAL). */ 2921 static HOST_WIDE_INT 2922 require_const_argument (tree exp, unsigned int argno, HOST_WIDE_INT minval, 2923 HOST_WIDE_INT maxval) 2924 { 2925 maxval--; 2926 tree arg = CALL_EXPR_ARG (exp, argno); 2927 if (TREE_CODE (arg) != INTEGER_CST) 2928 error_at (EXPR_LOCATION (exp), "Constant-type argument expected"); 2929 2930 auto argval = wi::to_widest (arg); 2931 2932 if (argval < minval || argval > maxval) 2933 error_at (EXPR_LOCATION (exp), 2934 "argument %d must be a constant immediate " 2935 "in range [%wd,%wd]", argno + 1, minval, maxval); 2936 2937 HOST_WIDE_INT retval = argval.to_shwi (); 2938 return retval; 2939 } 2940 2941 2942 /* Expand a prefetch builtin EXP. */ 2943 void 2944 aarch64_expand_prefetch_builtin (tree exp, int fcode) 2945 { 2946 int kind_id = -1; 2947 int level_id = -1; 2948 int rettn_id = -1; 2949 char prfop[11]; 2950 class expand_operand ops[2]; 2951 2952 static const char *kind_s[] = {"PLD", "PST", "PLI"}; 2953 static const char *level_s[] = {"L1", "L2", "L3", "SLC"}; 2954 static const char *rettn_s[] = {"KEEP", "STRM"}; 2955 2956 /* Each of the four prefetch builtins takes a different number of arguments, 2957 but proceeds to call the PRFM insn which requires 4 pieces of information 2958 to be fully defined. Where one of these takes less than 4 arguments, set 2959 sensible defaults. */ 2960 switch (fcode) 2961 { 2962 case AARCH64_PLDX: 2963 break; 2964 case AARCH64_PLIX: 2965 kind_id = 2; 2966 break; 2967 case AARCH64_PLI: 2968 case AARCH64_PLD: 2969 kind_id = (fcode == AARCH64_PLD) ? 0 : 2; 2970 level_id = 0; 2971 rettn_id = 0; 2972 break; 2973 default: 2974 gcc_unreachable (); 2975 } 2976 2977 /* Any -1 id variable is to be user-supplied. Here we fill these in and run 2978 bounds checks on them. "PLI" is used only implicitly by AARCH64_PLI & 2979 AARCH64_PLIX, never explicitly. */ 2980 int argno = 0; 2981 if (kind_id < 0) 2982 kind_id = require_const_argument (exp, argno++, 0, ARRAY_SIZE (kind_s) - 1); 2983 if (level_id < 0) 2984 level_id = require_const_argument (exp, argno++, 0, ARRAY_SIZE (level_s)); 2985 if (rettn_id < 0) 2986 rettn_id = require_const_argument (exp, argno++, 0, ARRAY_SIZE (rettn_s)); 2987 rtx address = expand_expr (CALL_EXPR_ARG (exp, argno), NULL_RTX, Pmode, 2988 EXPAND_NORMAL); 2989 2990 if (seen_error ()) 2991 return; 2992 2993 sprintf (prfop, "%s%s%s", kind_s[kind_id], 2994 level_s[level_id], 2995 rettn_s[rettn_id]); 2996 2997 rtx const_str = rtx_alloc (CONST_STRING); 2998 PUT_CODE (const_str, CONST_STRING); 2999 XSTR (const_str, 0) = ggc_strdup (prfop); 3000 3001 create_fixed_operand (&ops[0], const_str); 3002 create_address_operand (&ops[1], address); 3003 maybe_expand_insn (CODE_FOR_aarch64_pldx, 2, ops); 3004 } 3005 3006 /* Expand an expression EXP that calls a MEMTAG built-in FCODE 3007 with result going to TARGET. */ 3008 static rtx 3009 aarch64_expand_builtin_memtag (int fcode, tree exp, rtx target) 3010 { 3011 if (TARGET_ILP32) 3012 { 3013 error ("Memory Tagging Extension does not support %<-mabi=ilp32%>"); 3014 return const0_rtx; 3015 } 3016 3017 rtx pat = NULL; 3018 enum insn_code icode = aarch64_memtag_builtin_data[fcode - 3019 AARCH64_MEMTAG_BUILTIN_START - 1].icode; 3020 3021 rtx op0 = expand_normal (CALL_EXPR_ARG (exp, 0)); 3022 machine_mode mode0 = GET_MODE (op0); 3023 op0 = force_reg (mode0 == VOIDmode ? DImode : mode0, op0); 3024 op0 = convert_to_mode (DImode, op0, true); 3025 3026 switch (fcode) 3027 { 3028 case AARCH64_MEMTAG_BUILTIN_IRG: 3029 case AARCH64_MEMTAG_BUILTIN_GMI: 3030 case AARCH64_MEMTAG_BUILTIN_SUBP: 3031 case AARCH64_MEMTAG_BUILTIN_INC_TAG: 3032 { 3033 if (! target 3034 || GET_MODE (target) != DImode 3035 || ! (*insn_data[icode].operand[0].predicate) (target, DImode)) 3036 target = gen_reg_rtx (DImode); 3037 3038 if (fcode == AARCH64_MEMTAG_BUILTIN_INC_TAG) 3039 { 3040 rtx op1 = expand_normal (CALL_EXPR_ARG (exp, 1)); 3041 3042 if ((*insn_data[icode].operand[3].predicate) (op1, QImode)) 3043 { 3044 pat = GEN_FCN (icode) (target, op0, const0_rtx, op1); 3045 break; 3046 } 3047 error_at (EXPR_LOCATION (exp), 3048 "argument %d must be a constant immediate " 3049 "in range [0,15]", 2); 3050 return const0_rtx; 3051 } 3052 else 3053 { 3054 rtx op1 = expand_normal (CALL_EXPR_ARG (exp, 1)); 3055 machine_mode mode1 = GET_MODE (op1); 3056 op1 = force_reg (mode1 == VOIDmode ? DImode : mode1, op1); 3057 op1 = convert_to_mode (DImode, op1, true); 3058 pat = GEN_FCN (icode) (target, op0, op1); 3059 } 3060 break; 3061 } 3062 case AARCH64_MEMTAG_BUILTIN_GET_TAG: 3063 target = op0; 3064 pat = GEN_FCN (icode) (target, op0, const0_rtx); 3065 break; 3066 case AARCH64_MEMTAG_BUILTIN_SET_TAG: 3067 pat = GEN_FCN (icode) (op0, op0, const0_rtx); 3068 break; 3069 default: 3070 gcc_unreachable(); 3071 } 3072 3073 if (!pat) 3074 return NULL_RTX; 3075 3076 emit_insn (pat); 3077 return target; 3078 } 3079 3080 /* Function to expand an expression EXP which calls one of the ACLE Data 3081 Intrinsic builtins FCODE with the result going to TARGET. */ 3082 static rtx 3083 aarch64_expand_builtin_data_intrinsic (unsigned int fcode, tree exp, rtx target) 3084 { 3085 expand_operand ops[2]; 3086 machine_mode mode = TYPE_MODE (TREE_TYPE (exp)); 3087 create_output_operand (&ops[0], target, mode); 3088 create_input_operand (&ops[1], expand_normal (CALL_EXPR_ARG (exp, 0)), mode); 3089 enum insn_code icode; 3090 3091 switch (fcode) 3092 { 3093 case AARCH64_REV16: 3094 case AARCH64_REV16L: 3095 case AARCH64_REV16LL: 3096 icode = code_for_aarch64_rev16 (mode); 3097 break; 3098 case AARCH64_RBIT: 3099 case AARCH64_RBITL: 3100 case AARCH64_RBITLL: 3101 icode = code_for_aarch64_rbit (mode); 3102 break; 3103 default: 3104 gcc_unreachable (); 3105 } 3106 3107 expand_insn (icode, 2, ops); 3108 return ops[0].value; 3109 } 3110 3111 /* Expand an expression EXP as fpsr or fpcr setter (depending on 3112 UNSPEC) using MODE. */ 3113 static void 3114 aarch64_expand_fpsr_fpcr_setter (int unspec, machine_mode mode, tree exp) 3115 { 3116 tree arg = CALL_EXPR_ARG (exp, 0); 3117 rtx op = force_reg (mode, expand_normal (arg)); 3118 emit_insn (gen_aarch64_set (unspec, mode, op)); 3119 } 3120 3121 /* Expand a fpsr or fpcr getter (depending on UNSPEC) using MODE. 3122 Return the target. */ 3123 static rtx 3124 aarch64_expand_fpsr_fpcr_getter (enum insn_code icode, machine_mode mode, 3125 rtx target) 3126 { 3127 expand_operand op; 3128 create_output_operand (&op, target, mode); 3129 expand_insn (icode, 1, &op); 3130 return op.value; 3131 } 3132 3133 /* Expand an expression EXP that calls built-in function FCODE, 3134 with result going to TARGET if that's convenient. IGNORE is true 3135 if the result of the builtin is ignored. */ 3136 rtx 3137 aarch64_general_expand_builtin (unsigned int fcode, tree exp, rtx target, 3138 int ignore) 3139 { 3140 int icode; 3141 rtx op0; 3142 tree arg0; 3143 3144 switch (fcode) 3145 { 3146 case AARCH64_BUILTIN_GET_FPCR: 3147 return aarch64_expand_fpsr_fpcr_getter (CODE_FOR_aarch64_get_fpcrsi, 3148 SImode, target); 3149 case AARCH64_BUILTIN_SET_FPCR: 3150 aarch64_expand_fpsr_fpcr_setter (UNSPECV_SET_FPCR, SImode, exp); 3151 return target; 3152 case AARCH64_BUILTIN_GET_FPSR: 3153 return aarch64_expand_fpsr_fpcr_getter (CODE_FOR_aarch64_get_fpsrsi, 3154 SImode, target); 3155 case AARCH64_BUILTIN_SET_FPSR: 3156 aarch64_expand_fpsr_fpcr_setter (UNSPECV_SET_FPSR, SImode, exp); 3157 return target; 3158 case AARCH64_BUILTIN_GET_FPCR64: 3159 return aarch64_expand_fpsr_fpcr_getter (CODE_FOR_aarch64_get_fpcrdi, 3160 DImode, target); 3161 case AARCH64_BUILTIN_SET_FPCR64: 3162 aarch64_expand_fpsr_fpcr_setter (UNSPECV_SET_FPCR, DImode, exp); 3163 return target; 3164 case AARCH64_BUILTIN_GET_FPSR64: 3165 return aarch64_expand_fpsr_fpcr_getter (CODE_FOR_aarch64_get_fpsrdi, 3166 DImode, target); 3167 case AARCH64_BUILTIN_SET_FPSR64: 3168 aarch64_expand_fpsr_fpcr_setter (UNSPECV_SET_FPSR, DImode, exp); 3169 return target; 3170 case AARCH64_PAUTH_BUILTIN_AUTIA1716: 3171 case AARCH64_PAUTH_BUILTIN_PACIA1716: 3172 case AARCH64_PAUTH_BUILTIN_AUTIB1716: 3173 case AARCH64_PAUTH_BUILTIN_PACIB1716: 3174 case AARCH64_PAUTH_BUILTIN_XPACLRI: 3175 arg0 = CALL_EXPR_ARG (exp, 0); 3176 op0 = force_reg (Pmode, expand_normal (arg0)); 3177 3178 if (fcode == AARCH64_PAUTH_BUILTIN_XPACLRI) 3179 { 3180 rtx lr = gen_rtx_REG (Pmode, R30_REGNUM); 3181 icode = CODE_FOR_xpaclri; 3182 emit_move_insn (lr, op0); 3183 emit_insn (GEN_FCN (icode) ()); 3184 return lr; 3185 } 3186 else 3187 { 3188 tree arg1 = CALL_EXPR_ARG (exp, 1); 3189 rtx op1 = force_reg (Pmode, expand_normal (arg1)); 3190 switch (fcode) 3191 { 3192 case AARCH64_PAUTH_BUILTIN_AUTIA1716: 3193 icode = CODE_FOR_autia1716; 3194 break; 3195 case AARCH64_PAUTH_BUILTIN_AUTIB1716: 3196 icode = CODE_FOR_autib1716; 3197 break; 3198 case AARCH64_PAUTH_BUILTIN_PACIA1716: 3199 icode = CODE_FOR_pacia1716; 3200 break; 3201 case AARCH64_PAUTH_BUILTIN_PACIB1716: 3202 icode = CODE_FOR_pacib1716; 3203 break; 3204 default: 3205 icode = 0; 3206 gcc_unreachable (); 3207 } 3208 3209 rtx x16_reg = gen_rtx_REG (Pmode, R16_REGNUM); 3210 rtx x17_reg = gen_rtx_REG (Pmode, R17_REGNUM); 3211 emit_move_insn (x17_reg, op0); 3212 emit_move_insn (x16_reg, op1); 3213 emit_insn (GEN_FCN (icode) ()); 3214 return x17_reg; 3215 } 3216 3217 case AARCH64_JSCVT: 3218 { 3219 expand_operand ops[2]; 3220 create_output_operand (&ops[0], target, SImode); 3221 op0 = expand_normal (CALL_EXPR_ARG (exp, 0)); 3222 create_input_operand (&ops[1], op0, DFmode); 3223 expand_insn (CODE_FOR_aarch64_fjcvtzs, 2, ops); 3224 return ops[0].value; 3225 } 3226 3227 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ0_V2SF: 3228 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ90_V2SF: 3229 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ180_V2SF: 3230 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ270_V2SF: 3231 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ0_V4HF: 3232 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ90_V4HF: 3233 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ180_V4HF: 3234 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ270_V4HF: 3235 return aarch64_expand_fcmla_builtin (exp, target, fcode); 3236 case AARCH64_BUILTIN_RNG_RNDR: 3237 case AARCH64_BUILTIN_RNG_RNDRRS: 3238 return aarch64_expand_rng_builtin (exp, target, fcode, ignore); 3239 case AARCH64_RSR: 3240 case AARCH64_RSRP: 3241 case AARCH64_RSR64: 3242 case AARCH64_RSRF: 3243 case AARCH64_RSRF64: 3244 case AARCH64_RSR128: 3245 case AARCH64_WSR: 3246 case AARCH64_WSRP: 3247 case AARCH64_WSR64: 3248 case AARCH64_WSRF: 3249 case AARCH64_WSRF64: 3250 case AARCH64_WSR128: 3251 return aarch64_expand_rwsr_builtin (exp, target, fcode); 3252 case AARCH64_PLD: 3253 case AARCH64_PLDX: 3254 case AARCH64_PLI: 3255 case AARCH64_PLIX: 3256 aarch64_expand_prefetch_builtin (exp, fcode); 3257 return target; 3258 } 3259 3260 if (fcode >= AARCH64_SIMD_BUILTIN_BASE && fcode <= AARCH64_SIMD_BUILTIN_MAX) 3261 return aarch64_simd_expand_builtin (fcode, exp, target); 3262 else if (fcode >= AARCH64_CRC32_BUILTIN_BASE && fcode <= AARCH64_CRC32_BUILTIN_MAX) 3263 return aarch64_crc32_expand_builtin (fcode, exp, target); 3264 3265 if (fcode == AARCH64_BUILTIN_RSQRT_DF 3266 || fcode == AARCH64_BUILTIN_RSQRT_SF 3267 || fcode == AARCH64_BUILTIN_RSQRT_V2DF 3268 || fcode == AARCH64_BUILTIN_RSQRT_V2SF 3269 || fcode == AARCH64_BUILTIN_RSQRT_V4SF) 3270 return aarch64_expand_builtin_rsqrt (fcode, exp, target); 3271 3272 if (fcode == AARCH64_TME_BUILTIN_TSTART 3273 || fcode == AARCH64_TME_BUILTIN_TCOMMIT 3274 || fcode == AARCH64_TME_BUILTIN_TTEST 3275 || fcode == AARCH64_TME_BUILTIN_TCANCEL) 3276 return aarch64_expand_builtin_tme (fcode, exp, target); 3277 3278 if (fcode == AARCH64_LS64_BUILTIN_LD64B 3279 || fcode == AARCH64_LS64_BUILTIN_ST64B 3280 || fcode == AARCH64_LS64_BUILTIN_ST64BV 3281 || fcode == AARCH64_LS64_BUILTIN_ST64BV0) 3282 return aarch64_expand_builtin_ls64 (fcode, exp, target); 3283 3284 if (fcode >= AARCH64_MEMTAG_BUILTIN_START 3285 && fcode <= AARCH64_MEMTAG_BUILTIN_END) 3286 return aarch64_expand_builtin_memtag (fcode, exp, target); 3287 if (fcode >= AARCH64_REV16 3288 && fcode <= AARCH64_RBITLL) 3289 return aarch64_expand_builtin_data_intrinsic (fcode, exp, target); 3290 3291 gcc_unreachable (); 3292 } 3293 3294 /* Return builtin for reciprocal square root. */ 3295 3296 tree 3297 aarch64_general_builtin_rsqrt (unsigned int fn) 3298 { 3299 if (fn == AARCH64_SIMD_BUILTIN_UNOP_sqrtv2df) 3300 return aarch64_builtin_decls[AARCH64_BUILTIN_RSQRT_V2DF]; 3301 if (fn == AARCH64_SIMD_BUILTIN_UNOP_sqrtv2sf) 3302 return aarch64_builtin_decls[AARCH64_BUILTIN_RSQRT_V2SF]; 3303 if (fn == AARCH64_SIMD_BUILTIN_UNOP_sqrtv4sf) 3304 return aarch64_builtin_decls[AARCH64_BUILTIN_RSQRT_V4SF]; 3305 return NULL_TREE; 3306 } 3307 3308 /* Return true if the lane check can be removed as there is no 3309 error going to be emitted. */ 3310 static bool 3311 aarch64_fold_builtin_lane_check (tree arg0, tree arg1, tree arg2) 3312 { 3313 if (TREE_CODE (arg0) != INTEGER_CST) 3314 return false; 3315 if (TREE_CODE (arg1) != INTEGER_CST) 3316 return false; 3317 if (TREE_CODE (arg2) != INTEGER_CST) 3318 return false; 3319 3320 auto totalsize = wi::to_widest (arg0); 3321 auto elementsize = wi::to_widest (arg1); 3322 if (totalsize == 0 || elementsize == 0) 3323 return false; 3324 auto lane = wi::to_widest (arg2); 3325 auto high = wi::udiv_trunc (totalsize, elementsize); 3326 return wi::ltu_p (lane, high); 3327 } 3328 3329 #undef VAR1 3330 #define VAR1(T, N, MAP, FLAG, A) \ 3331 case AARCH64_SIMD_BUILTIN_##T##_##N##A: 3332 3333 #undef VREINTERPRET_BUILTIN 3334 #define VREINTERPRET_BUILTIN(A, B, L) \ 3335 case AARCH64_SIMD_BUILTIN_VREINTERPRET##L##_##A##_##B: 3336 3337 3338 /* Try to fold a call to the built-in function with subcode FCODE. The 3339 function is passed the N_ARGS arguments in ARGS and it returns a value 3340 of type TYPE. Return the new expression on success and NULL_TREE on 3341 failure. */ 3342 tree 3343 aarch64_general_fold_builtin (unsigned int fcode, tree type, 3344 unsigned int n_args ATTRIBUTE_UNUSED, tree *args) 3345 { 3346 switch (fcode) 3347 { 3348 BUILTIN_VDQF (UNOP, abs, 2, ALL) 3349 return fold_build1 (ABS_EXPR, type, args[0]); 3350 VAR1 (UNOP, floatv2si, 2, ALL, v2sf) 3351 VAR1 (UNOP, floatv4si, 2, ALL, v4sf) 3352 VAR1 (UNOP, floatv2di, 2, ALL, v2df) 3353 return fold_build1 (FLOAT_EXPR, type, args[0]); 3354 AARCH64_SIMD_VREINTERPRET_BUILTINS 3355 return fold_build1 (VIEW_CONVERT_EXPR, type, args[0]); 3356 case AARCH64_SIMD_BUILTIN_LANE_CHECK: 3357 gcc_assert (n_args == 3); 3358 if (aarch64_fold_builtin_lane_check (args[0], args[1], args[2])) 3359 return void_node; 3360 break; 3361 default: 3362 break; 3363 } 3364 3365 return NULL_TREE; 3366 } 3367 3368 enum aarch64_simd_type 3369 get_mem_type_for_load_store (unsigned int fcode) 3370 { 3371 switch (fcode) 3372 { 3373 VAR1 (LOAD1, ld1, 0, LOAD, v8qi) 3374 VAR1 (STORE1, st1, 0, STORE, v8qi) 3375 return Int8x8_t; 3376 VAR1 (LOAD1, ld1, 0, LOAD, v16qi) 3377 VAR1 (STORE1, st1, 0, STORE, v16qi) 3378 return Int8x16_t; 3379 VAR1 (LOAD1, ld1, 0, LOAD, v4hi) 3380 VAR1 (STORE1, st1, 0, STORE, v4hi) 3381 return Int16x4_t; 3382 VAR1 (LOAD1, ld1, 0, LOAD, v8hi) 3383 VAR1 (STORE1, st1, 0, STORE, v8hi) 3384 return Int16x8_t; 3385 VAR1 (LOAD1, ld1, 0, LOAD, v2si) 3386 VAR1 (STORE1, st1, 0, STORE, v2si) 3387 return Int32x2_t; 3388 VAR1 (LOAD1, ld1, 0, LOAD, v4si) 3389 VAR1 (STORE1, st1, 0, STORE, v4si) 3390 return Int32x4_t; 3391 VAR1 (LOAD1, ld1, 0, LOAD, v2di) 3392 VAR1 (STORE1, st1, 0, STORE, v2di) 3393 return Int64x2_t; 3394 VAR1 (LOAD1_U, ld1, 0, LOAD, v8qi) 3395 VAR1 (STORE1_U, st1, 0, STORE, v8qi) 3396 return Uint8x8_t; 3397 VAR1 (LOAD1_U, ld1, 0, LOAD, v16qi) 3398 VAR1 (STORE1_U, st1, 0, STORE, v16qi) 3399 return Uint8x16_t; 3400 VAR1 (LOAD1_U, ld1, 0, LOAD, v4hi) 3401 VAR1 (STORE1_U, st1, 0, STORE, v4hi) 3402 return Uint16x4_t; 3403 VAR1 (LOAD1_U, ld1, 0, LOAD, v8hi) 3404 VAR1 (STORE1_U, st1, 0, STORE, v8hi) 3405 return Uint16x8_t; 3406 VAR1 (LOAD1_U, ld1, 0, LOAD, v2si) 3407 VAR1 (STORE1_U, st1, 0, STORE, v2si) 3408 return Uint32x2_t; 3409 VAR1 (LOAD1_U, ld1, 0, LOAD, v4si) 3410 VAR1 (STORE1_U, st1, 0, STORE, v4si) 3411 return Uint32x4_t; 3412 VAR1 (LOAD1_U, ld1, 0, LOAD, v2di) 3413 VAR1 (STORE1_U, st1, 0, STORE, v2di) 3414 return Uint64x2_t; 3415 VAR1 (LOAD1_P, ld1, 0, LOAD, v8qi) 3416 VAR1 (STORE1_P, st1, 0, STORE, v8qi) 3417 return Poly8x8_t; 3418 VAR1 (LOAD1_P, ld1, 0, LOAD, v16qi) 3419 VAR1 (STORE1_P, st1, 0, STORE, v16qi) 3420 return Poly8x16_t; 3421 VAR1 (LOAD1_P, ld1, 0, LOAD, v4hi) 3422 VAR1 (STORE1_P, st1, 0, STORE, v4hi) 3423 return Poly16x4_t; 3424 VAR1 (LOAD1_P, ld1, 0, LOAD, v8hi) 3425 VAR1 (STORE1_P, st1, 0, STORE, v8hi) 3426 return Poly16x8_t; 3427 VAR1 (LOAD1_P, ld1, 0, LOAD, v2di) 3428 VAR1 (STORE1_P, st1, 0, STORE, v2di) 3429 return Poly64x2_t; 3430 VAR1 (LOAD1, ld1, 0, LOAD, v4hf) 3431 VAR1 (STORE1, st1, 0, STORE, v4hf) 3432 return Float16x4_t; 3433 VAR1 (LOAD1, ld1, 0, LOAD, v8hf) 3434 VAR1 (STORE1, st1, 0, STORE, v8hf) 3435 return Float16x8_t; 3436 VAR1 (LOAD1, ld1, 0, LOAD, v4bf) 3437 VAR1 (STORE1, st1, 0, STORE, v4bf) 3438 return Bfloat16x4_t; 3439 VAR1 (LOAD1, ld1, 0, LOAD, v8bf) 3440 VAR1 (STORE1, st1, 0, STORE, v8bf) 3441 return Bfloat16x8_t; 3442 VAR1 (LOAD1, ld1, 0, LOAD, v2sf) 3443 VAR1 (STORE1, st1, 0, STORE, v2sf) 3444 return Float32x2_t; 3445 VAR1 (LOAD1, ld1, 0, LOAD, v4sf) 3446 VAR1 (STORE1, st1, 0, STORE, v4sf) 3447 return Float32x4_t; 3448 VAR1 (LOAD1, ld1, 0, LOAD, v2df) 3449 VAR1 (STORE1, st1, 0, STORE, v2df) 3450 return Float64x2_t; 3451 default: 3452 gcc_unreachable (); 3453 break; 3454 } 3455 } 3456 3457 /* We've seen a vector load from address ADDR. Record it in 3458 vector_load_decls, if appropriate. */ 3459 static void 3460 aarch64_record_vector_load_arg (tree addr) 3461 { 3462 tree decl = aarch64_vector_load_decl (addr); 3463 if (!decl) 3464 return; 3465 if (!cfun->machine->vector_load_decls) 3466 cfun->machine->vector_load_decls = hash_set<tree>::create_ggc (31); 3467 cfun->machine->vector_load_decls->add (decl); 3468 } 3469 3470 /* Try to fold STMT, given that it's a call to the built-in function with 3471 subcode FCODE. Return the new statement on success and null on 3472 failure. */ 3473 gimple * 3474 aarch64_general_gimple_fold_builtin (unsigned int fcode, gcall *stmt, 3475 gimple_stmt_iterator *gsi ATTRIBUTE_UNUSED) 3476 { 3477 gimple *new_stmt = NULL; 3478 unsigned nargs = gimple_call_num_args (stmt); 3479 tree *args = (nargs > 0 3480 ? gimple_call_arg_ptr (stmt, 0) 3481 : &error_mark_node); 3482 3483 /* We use gimple's IFN_REDUC_(PLUS|MIN|MAX)s for float, signed int 3484 and unsigned int; it will distinguish according to the types of 3485 the arguments to the __builtin. */ 3486 switch (fcode) 3487 { 3488 BUILTIN_VALL (UNOP, reduc_plus_scal_, 10, ALL) 3489 BUILTIN_VDQ_I (UNOPU, reduc_plus_scal_, 10, NONE) 3490 new_stmt = gimple_build_call_internal (IFN_REDUC_PLUS, 3491 1, args[0]); 3492 gimple_call_set_lhs (new_stmt, gimple_call_lhs (stmt)); 3493 break; 3494 3495 /* Lower sqrt builtins to gimple/internal function sqrt. */ 3496 BUILTIN_VHSDF_DF (UNOP, sqrt, 2, FP) 3497 new_stmt = gimple_build_call_internal (IFN_SQRT, 3498 1, args[0]); 3499 gimple_call_set_lhs (new_stmt, gimple_call_lhs (stmt)); 3500 break; 3501 3502 BUILTIN_VDC (BINOP, combine, 0, AUTO_FP) 3503 BUILTIN_VD_I (BINOPU, combine, 0, NONE) 3504 BUILTIN_VDC_P (BINOPP, combine, 0, NONE) 3505 { 3506 tree first_part, second_part; 3507 if (BYTES_BIG_ENDIAN) 3508 { 3509 second_part = args[0]; 3510 first_part = args[1]; 3511 } 3512 else 3513 { 3514 first_part = args[0]; 3515 second_part = args[1]; 3516 } 3517 tree ret_type = gimple_call_return_type (stmt); 3518 tree ctor = build_constructor_va (ret_type, 2, NULL_TREE, first_part, 3519 NULL_TREE, second_part); 3520 new_stmt = gimple_build_assign (gimple_call_lhs (stmt), ctor); 3521 } 3522 break; 3523 3524 /*lower store and load neon builtins to gimple. */ 3525 BUILTIN_VALL_F16 (LOAD1, ld1, 0, LOAD) 3526 BUILTIN_VDQ_I (LOAD1_U, ld1, 0, LOAD) 3527 BUILTIN_VALLP_NO_DI (LOAD1_P, ld1, 0, LOAD) 3528 /* Punt until after inlining, so that we stand more chance of 3529 recording something meaningful in vector_load_decls. */ 3530 if (!cfun->after_inlining) 3531 break; 3532 aarch64_record_vector_load_arg (args[0]); 3533 if (!BYTES_BIG_ENDIAN) 3534 { 3535 enum aarch64_simd_type mem_type 3536 = get_mem_type_for_load_store(fcode); 3537 aarch64_simd_type_info simd_type 3538 = aarch64_simd_types[mem_type]; 3539 tree elt_ptr_type = build_pointer_type_for_mode (simd_type.eltype, 3540 VOIDmode, true); 3541 tree zero = build_zero_cst (elt_ptr_type); 3542 /* Use element type alignment. */ 3543 tree access_type 3544 = build_aligned_type (simd_type.itype, 3545 TYPE_ALIGN (simd_type.eltype)); 3546 new_stmt 3547 = gimple_build_assign (gimple_get_lhs (stmt), 3548 fold_build2 (MEM_REF, 3549 access_type, 3550 args[0], zero)); 3551 gimple_set_vuse (new_stmt, gimple_vuse (stmt)); 3552 gimple_set_vdef (new_stmt, gimple_vdef (stmt)); 3553 } 3554 break; 3555 3556 BUILTIN_VALL_F16 (STORE1, st1, 0, STORE) 3557 BUILTIN_VDQ_I (STORE1_U, st1, 0, STORE) 3558 BUILTIN_VALLP_NO_DI (STORE1_P, st1, 0, STORE) 3559 if (!BYTES_BIG_ENDIAN) 3560 { 3561 enum aarch64_simd_type mem_type 3562 = get_mem_type_for_load_store(fcode); 3563 aarch64_simd_type_info simd_type 3564 = aarch64_simd_types[mem_type]; 3565 tree elt_ptr_type = build_pointer_type_for_mode (simd_type.eltype, 3566 VOIDmode, true); 3567 tree zero = build_zero_cst (elt_ptr_type); 3568 /* Use element type alignment. */ 3569 tree access_type 3570 = build_aligned_type (simd_type.itype, 3571 TYPE_ALIGN (simd_type.eltype)); 3572 new_stmt 3573 = gimple_build_assign (fold_build2 (MEM_REF, access_type, 3574 args[0], zero), 3575 args[1]); 3576 gimple_set_vuse (new_stmt, gimple_vuse (stmt)); 3577 gimple_set_vdef (new_stmt, gimple_vdef (stmt)); 3578 } 3579 break; 3580 3581 BUILTIN_VDQIF (UNOP, reduc_smax_scal_, 10, ALL) 3582 BUILTIN_VDQ_BHSI (UNOPU, reduc_umax_scal_, 10, ALL) 3583 new_stmt = gimple_build_call_internal (IFN_REDUC_MAX, 3584 1, args[0]); 3585 gimple_call_set_lhs (new_stmt, gimple_call_lhs (stmt)); 3586 break; 3587 BUILTIN_VDQIF (UNOP, reduc_smin_scal_, 10, ALL) 3588 BUILTIN_VDQ_BHSI (UNOPU, reduc_umin_scal_, 10, ALL) 3589 new_stmt = gimple_build_call_internal (IFN_REDUC_MIN, 3590 1, args[0]); 3591 gimple_call_set_lhs (new_stmt, gimple_call_lhs (stmt)); 3592 break; 3593 BUILTIN_VSDQ_I_DI (BINOP, ashl, 3, NONE) 3594 if (TREE_CODE (args[1]) == INTEGER_CST 3595 && wi::ltu_p (wi::to_wide (args[1]), element_precision (args[0]))) 3596 new_stmt = gimple_build_assign (gimple_call_lhs (stmt), 3597 LSHIFT_EXPR, args[0], args[1]); 3598 break; 3599 BUILTIN_VSDQ_I_DI (BINOP, sshl, 0, NONE) 3600 BUILTIN_VSDQ_I_DI (BINOP_UUS, ushl, 0, NONE) 3601 { 3602 tree cst = args[1]; 3603 tree ctype = TREE_TYPE (cst); 3604 /* Left shifts can be both scalar or vector, e.g. uint64x1_t is 3605 treated as a scalar type not a vector one. */ 3606 if ((cst = uniform_integer_cst_p (cst)) != NULL_TREE) 3607 { 3608 wide_int wcst = wi::to_wide (cst); 3609 tree unit_ty = TREE_TYPE (cst); 3610 3611 wide_int abs_cst = wi::abs (wcst); 3612 if (wi::geu_p (abs_cst, element_precision (args[0]))) 3613 break; 3614 3615 if (wi::neg_p (wcst, TYPE_SIGN (ctype))) 3616 { 3617 tree final_cst; 3618 final_cst = wide_int_to_tree (unit_ty, abs_cst); 3619 if (TREE_CODE (cst) != INTEGER_CST) 3620 final_cst = build_uniform_cst (ctype, final_cst); 3621 3622 new_stmt = gimple_build_assign (gimple_call_lhs (stmt), 3623 RSHIFT_EXPR, args[0], 3624 final_cst); 3625 } 3626 else 3627 new_stmt = gimple_build_assign (gimple_call_lhs (stmt), 3628 LSHIFT_EXPR, args[0], args[1]); 3629 } 3630 } 3631 break; 3632 BUILTIN_VDQ_I (SHIFTIMM, ashr, 3, NONE) 3633 VAR1 (SHIFTIMM, ashr_simd, 0, NONE, di) 3634 BUILTIN_VDQ_I (USHIFTIMM, lshr, 3, NONE) 3635 VAR1 (USHIFTIMM, lshr_simd, 0, NONE, di) 3636 if (TREE_CODE (args[1]) == INTEGER_CST 3637 && wi::ltu_p (wi::to_wide (args[1]), element_precision (args[0]))) 3638 new_stmt = gimple_build_assign (gimple_call_lhs (stmt), 3639 RSHIFT_EXPR, args[0], args[1]); 3640 break; 3641 BUILTIN_GPF (BINOP, fmulx, 0, ALL) 3642 { 3643 gcc_assert (nargs == 2); 3644 bool a0_cst_p = TREE_CODE (args[0]) == REAL_CST; 3645 bool a1_cst_p = TREE_CODE (args[1]) == REAL_CST; 3646 if (a0_cst_p || a1_cst_p) 3647 { 3648 if (a0_cst_p && a1_cst_p) 3649 { 3650 tree t0 = TREE_TYPE (args[0]); 3651 real_value a0 = (TREE_REAL_CST (args[0])); 3652 real_value a1 = (TREE_REAL_CST (args[1])); 3653 if (real_equal (&a1, &dconst0)) 3654 std::swap (a0, a1); 3655 /* According to real_equal (), +0 equals -0. */ 3656 if (real_equal (&a0, &dconst0) && real_isinf (&a1)) 3657 { 3658 real_value res = dconst2; 3659 res.sign = a0.sign ^ a1.sign; 3660 new_stmt = gimple_build_assign (gimple_call_lhs (stmt), 3661 REAL_CST, 3662 build_real (t0, res)); 3663 } 3664 else 3665 new_stmt = gimple_build_assign (gimple_call_lhs (stmt), 3666 MULT_EXPR, 3667 args[0], args[1]); 3668 } 3669 else /* a0_cst_p ^ a1_cst_p. */ 3670 { 3671 real_value const_part = a0_cst_p 3672 ? TREE_REAL_CST (args[0]) : TREE_REAL_CST (args[1]); 3673 if (!real_equal (&const_part, &dconst0) 3674 && !real_isinf (&const_part)) 3675 new_stmt = gimple_build_assign (gimple_call_lhs (stmt), 3676 MULT_EXPR, args[0], 3677 args[1]); 3678 } 3679 } 3680 if (new_stmt) 3681 { 3682 gimple_set_vuse (new_stmt, gimple_vuse (stmt)); 3683 gimple_set_vdef (new_stmt, gimple_vdef (stmt)); 3684 } 3685 break; 3686 } 3687 case AARCH64_SIMD_BUILTIN_LANE_CHECK: 3688 if (aarch64_fold_builtin_lane_check (args[0], args[1], args[2])) 3689 { 3690 unlink_stmt_vdef (stmt); 3691 release_defs (stmt); 3692 new_stmt = gimple_build_nop (); 3693 } 3694 break; 3695 default: 3696 break; 3697 } 3698 3699 /* GIMPLE assign statements (unlike calls) require a non-null lhs. If we 3700 created an assign statement with a null lhs, then fix this by assigning 3701 to a new (and subsequently unused) variable. */ 3702 if (new_stmt && is_gimple_assign (new_stmt) && !gimple_assign_lhs (new_stmt)) 3703 { 3704 tree new_lhs = make_ssa_name (gimple_call_return_type (stmt)); 3705 gimple_assign_set_lhs (new_stmt, new_lhs); 3706 } 3707 3708 return new_stmt; 3709 } 3710 3711 void 3712 aarch64_atomic_assign_expand_fenv (tree *hold, tree *clear, tree *update) 3713 { 3714 const unsigned AARCH64_FE_INVALID = 1; 3715 const unsigned AARCH64_FE_DIVBYZERO = 2; 3716 const unsigned AARCH64_FE_OVERFLOW = 4; 3717 const unsigned AARCH64_FE_UNDERFLOW = 8; 3718 const unsigned AARCH64_FE_INEXACT = 16; 3719 const unsigned HOST_WIDE_INT AARCH64_FE_ALL_EXCEPT = (AARCH64_FE_INVALID 3720 | AARCH64_FE_DIVBYZERO 3721 | AARCH64_FE_OVERFLOW 3722 | AARCH64_FE_UNDERFLOW 3723 | AARCH64_FE_INEXACT); 3724 const unsigned HOST_WIDE_INT AARCH64_FE_EXCEPT_SHIFT = 8; 3725 tree fenv_cr, fenv_sr, get_fpcr, set_fpcr, mask_cr, mask_sr; 3726 tree ld_fenv_cr, ld_fenv_sr, masked_fenv_cr, masked_fenv_sr, hold_fnclex_cr; 3727 tree hold_fnclex_sr, new_fenv_var, reload_fenv, restore_fnenv, get_fpsr, set_fpsr; 3728 tree update_call, atomic_feraiseexcept, hold_fnclex, masked_fenv, ld_fenv; 3729 3730 /* Generate the equivalence of : 3731 unsigned int fenv_cr; 3732 fenv_cr = __builtin_aarch64_get_fpcr (); 3733 3734 unsigned int fenv_sr; 3735 fenv_sr = __builtin_aarch64_get_fpsr (); 3736 3737 Now set all exceptions to non-stop 3738 unsigned int mask_cr 3739 = ~(AARCH64_FE_ALL_EXCEPT << AARCH64_FE_EXCEPT_SHIFT); 3740 unsigned int masked_cr; 3741 masked_cr = fenv_cr & mask_cr; 3742 3743 And clear all exception flags 3744 unsigned int maske_sr = ~AARCH64_FE_ALL_EXCEPT; 3745 unsigned int masked_cr; 3746 masked_sr = fenv_sr & mask_sr; 3747 3748 __builtin_aarch64_set_cr (masked_cr); 3749 __builtin_aarch64_set_sr (masked_sr); */ 3750 3751 fenv_cr = create_tmp_var_raw (unsigned_type_node); 3752 fenv_sr = create_tmp_var_raw (unsigned_type_node); 3753 3754 get_fpcr = aarch64_builtin_decls[AARCH64_BUILTIN_GET_FPCR]; 3755 set_fpcr = aarch64_builtin_decls[AARCH64_BUILTIN_SET_FPCR]; 3756 get_fpsr = aarch64_builtin_decls[AARCH64_BUILTIN_GET_FPSR]; 3757 set_fpsr = aarch64_builtin_decls[AARCH64_BUILTIN_SET_FPSR]; 3758 3759 mask_cr = build_int_cst (unsigned_type_node, 3760 ~(AARCH64_FE_ALL_EXCEPT << AARCH64_FE_EXCEPT_SHIFT)); 3761 mask_sr = build_int_cst (unsigned_type_node, 3762 ~(AARCH64_FE_ALL_EXCEPT)); 3763 3764 ld_fenv_cr = build4 (TARGET_EXPR, unsigned_type_node, 3765 fenv_cr, build_call_expr (get_fpcr, 0), 3766 NULL_TREE, NULL_TREE); 3767 ld_fenv_sr = build4 (TARGET_EXPR, unsigned_type_node, 3768 fenv_sr, build_call_expr (get_fpsr, 0), 3769 NULL_TREE, NULL_TREE); 3770 3771 masked_fenv_cr = build2 (BIT_AND_EXPR, unsigned_type_node, fenv_cr, mask_cr); 3772 masked_fenv_sr = build2 (BIT_AND_EXPR, unsigned_type_node, fenv_sr, mask_sr); 3773 3774 hold_fnclex_cr = build_call_expr (set_fpcr, 1, masked_fenv_cr); 3775 hold_fnclex_sr = build_call_expr (set_fpsr, 1, masked_fenv_sr); 3776 3777 hold_fnclex = build2 (COMPOUND_EXPR, void_type_node, hold_fnclex_cr, 3778 hold_fnclex_sr); 3779 masked_fenv = build2 (COMPOUND_EXPR, void_type_node, masked_fenv_cr, 3780 masked_fenv_sr); 3781 ld_fenv = build2 (COMPOUND_EXPR, void_type_node, ld_fenv_cr, ld_fenv_sr); 3782 3783 *hold = build2 (COMPOUND_EXPR, void_type_node, 3784 build2 (COMPOUND_EXPR, void_type_node, masked_fenv, ld_fenv), 3785 hold_fnclex); 3786 3787 /* Store the value of masked_fenv to clear the exceptions: 3788 __builtin_aarch64_set_fpsr (masked_fenv_sr); */ 3789 3790 *clear = build_call_expr (set_fpsr, 1, masked_fenv_sr); 3791 3792 /* Generate the equivalent of : 3793 unsigned int new_fenv_var; 3794 new_fenv_var = __builtin_aarch64_get_fpsr (); 3795 3796 __builtin_aarch64_set_fpsr (fenv_sr); 3797 3798 __atomic_feraiseexcept (new_fenv_var); */ 3799 3800 new_fenv_var = create_tmp_var_raw (unsigned_type_node); 3801 reload_fenv = build4 (TARGET_EXPR, unsigned_type_node, 3802 new_fenv_var, build_call_expr (get_fpsr, 0), 3803 NULL_TREE, NULL_TREE); 3804 restore_fnenv = build_call_expr (set_fpsr, 1, fenv_sr); 3805 atomic_feraiseexcept = builtin_decl_implicit (BUILT_IN_ATOMIC_FERAISEEXCEPT); 3806 update_call = build_call_expr (atomic_feraiseexcept, 1, 3807 fold_convert (integer_type_node, new_fenv_var)); 3808 *update = build2 (COMPOUND_EXPR, void_type_node, 3809 build2 (COMPOUND_EXPR, void_type_node, 3810 reload_fenv, restore_fnenv), update_call); 3811 } 3812 3813 /* Resolve overloaded MEMTAG build-in functions. */ 3814 #define AARCH64_BUILTIN_SUBCODE(F) \ 3815 (DECL_MD_FUNCTION_CODE (F) >> AARCH64_BUILTIN_SHIFT) 3816 3817 static tree 3818 aarch64_resolve_overloaded_memtag (location_t loc, 3819 tree fndecl, void *pass_params) 3820 { 3821 vec<tree, va_gc> *params = static_cast<vec<tree, va_gc> *> (pass_params); 3822 unsigned param_num = params ? params->length() : 0; 3823 unsigned int fcode = AARCH64_BUILTIN_SUBCODE (fndecl); 3824 tree inittype = aarch64_memtag_builtin_data[ 3825 fcode - AARCH64_MEMTAG_BUILTIN_START - 1].ftype; 3826 unsigned arg_num = list_length (TYPE_ARG_TYPES (inittype)) - 1; 3827 3828 if (param_num != arg_num) 3829 { 3830 TREE_TYPE (fndecl) = inittype; 3831 return NULL_TREE; 3832 } 3833 tree retype = NULL; 3834 3835 if (fcode == AARCH64_MEMTAG_BUILTIN_SUBP) 3836 { 3837 tree t0 = TREE_TYPE ((*params)[0]); 3838 tree t1 = TREE_TYPE ((*params)[1]); 3839 3840 if (t0 == error_mark_node || TREE_CODE (t0) != POINTER_TYPE) 3841 t0 = ptr_type_node; 3842 if (t1 == error_mark_node || TREE_CODE (t1) != POINTER_TYPE) 3843 t1 = ptr_type_node; 3844 3845 if (TYPE_MODE (t0) != DImode) 3846 warning_at (loc, 1, "expected 64-bit address but argument 1 is %d-bit", 3847 (int)tree_to_shwi (DECL_SIZE ((*params)[0]))); 3848 3849 if (TYPE_MODE (t1) != DImode) 3850 warning_at (loc, 1, "expected 64-bit address but argument 2 is %d-bit", 3851 (int)tree_to_shwi (DECL_SIZE ((*params)[1]))); 3852 3853 retype = build_function_type_list (ptrdiff_type_node, t0, t1, NULL); 3854 } 3855 else 3856 { 3857 tree t0 = TREE_TYPE ((*params)[0]); 3858 3859 if (t0 == error_mark_node || TREE_CODE (t0) != POINTER_TYPE) 3860 { 3861 TREE_TYPE (fndecl) = inittype; 3862 return NULL_TREE; 3863 } 3864 3865 if (TYPE_MODE (t0) != DImode) 3866 warning_at (loc, 1, "expected 64-bit address but argument 1 is %d-bit", 3867 (int)tree_to_shwi (DECL_SIZE ((*params)[0]))); 3868 3869 switch (fcode) 3870 { 3871 case AARCH64_MEMTAG_BUILTIN_IRG: 3872 retype = build_function_type_list (t0, t0, uint64_type_node, NULL); 3873 break; 3874 case AARCH64_MEMTAG_BUILTIN_GMI: 3875 retype = build_function_type_list (uint64_type_node, t0, 3876 uint64_type_node, NULL); 3877 break; 3878 case AARCH64_MEMTAG_BUILTIN_INC_TAG: 3879 retype = build_function_type_list (t0, t0, unsigned_type_node, NULL); 3880 break; 3881 case AARCH64_MEMTAG_BUILTIN_SET_TAG: 3882 retype = build_function_type_list (void_type_node, t0, NULL); 3883 break; 3884 case AARCH64_MEMTAG_BUILTIN_GET_TAG: 3885 retype = build_function_type_list (t0, t0, NULL); 3886 break; 3887 default: 3888 return NULL_TREE; 3889 } 3890 } 3891 3892 if (!retype || retype == error_mark_node) 3893 TREE_TYPE (fndecl) = inittype; 3894 else 3895 TREE_TYPE (fndecl) = retype; 3896 3897 return NULL_TREE; 3898 } 3899 3900 /* Called at aarch64_resolve_overloaded_builtin in aarch64-c.cc. */ 3901 tree 3902 aarch64_resolve_overloaded_builtin_general (location_t loc, tree function, 3903 void *pass_params) 3904 { 3905 unsigned int fcode = AARCH64_BUILTIN_SUBCODE (function); 3906 3907 if (fcode >= AARCH64_MEMTAG_BUILTIN_START 3908 && fcode <= AARCH64_MEMTAG_BUILTIN_END) 3909 return aarch64_resolve_overloaded_memtag(loc, function, pass_params); 3910 3911 return NULL_TREE; 3912 } 3913 3914 #undef AARCH64_CHECK_BUILTIN_MODE 3915 #undef AARCH64_FIND_FRINT_VARIANT 3916 #undef CF0 3917 #undef CF1 3918 #undef CF2 3919 #undef CF3 3920 #undef CF4 3921 #undef CF10 3922 #undef VAR1 3923 #undef VAR2 3924 #undef VAR3 3925 #undef VAR4 3926 #undef VAR5 3927 #undef VAR6 3928 #undef VAR7 3929 #undef VAR8 3930 #undef VAR9 3931 #undef VAR10 3932 #undef VAR11 3933 3934 #include "gt-aarch64-builtins.h" 3935
Indexes created Thu Jul 16 00:26:27 UTC 2026