1 /* Cost model implementation for RISC-V 'V' Extension for GNU compiler. 2 Copyright (C) 2023-2024 Free Software Foundation, Inc. 3 Contributed by Juzhe Zhong (juzhe.zhong (at) rivai.ai), RiVAI Technologies Ltd. 4 5 This file is part of GCC. 6 7 GCC is free software; you can redistribute it and/or modify 8 it under the terms of the GNU General Public License as published by 9 the Free Software Foundation; either version 3, or (at your option) 10 any later version. 11 12 GCC is distributed in the hope that it will be useful, 13 but WITHOUT ANY WARRANTY; without even the implied warranty of 14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 GNU General Public License for more details. 16 17 You should have received a copy of the GNU General Public License 18 along with GCC; see the file COPYING3. If not see 19 <http://www.gnu.org/licenses/>. */ 20 21 #define IN_TARGET_CODE 1 22 23 #define INCLUDE_STRING 24 #include "config.h" 25 #include "system.h" 26 #include "coretypes.h" 27 #include "tm.h" 28 #include "target.h" 29 #include "function.h" 30 #include "tree.h" 31 #include "basic-block.h" 32 #include "rtl.h" 33 #include "gimple.h" 34 #include "targhooks.h" 35 #include "cfgloop.h" 36 #include "fold-const.h" 37 #include "tm_p.h" 38 #include "tree-vectorizer.h" 39 #include "gimple-iterator.h" 40 #include "bitmap.h" 41 #include "ssa.h" 42 #include "backend.h" 43 #include "tree-data-ref.h" 44 #include "tree-ssa-loop-niter.h" 45 #include "tree-hash-traits.h" 46 47 /* This file should be included last. */ 48 #include "riscv-vector-costs.h" 49 50 namespace riscv_vector { 51 52 /* Dynamic LMUL philosophy - Local linear-scan SSA live range based analysis 53 determine LMUL 54 55 - Collect all vectorize STMTs locally for each loop block. 56 - Build program point based graph, ignore non-vectorize STMTs: 57 58 vectorize STMT 0 - point 0 59 scalar STMT 0 - ignore. 60 vectorize STMT 1 - point 1 61 ... 62 - Compute the number of live V_REGs live at each program point 63 - Determine LMUL in VECTOR COST model according to the program point 64 which has maximum live V_REGs. 65 66 Note: 67 68 - BIGGEST_MODE is the biggest LMUL auto-vectorization element mode. 69 It's important for mixed size auto-vectorization (Conversions, ... etc). 70 E.g. For a loop that is vectorizing conversion of INT32 -> INT64. 71 The biggest mode is DImode and LMUL = 8, LMUL = 4 for SImode. 72 We compute the number live V_REGs at each program point according to 73 this information. 74 - We only compute program points and live ranges locally (within a block) 75 since we just need to compute the number of live V_REGs at each program 76 point and we are not really allocating the registers for each SSA. 77 We can make the variable has another local live range in another block 78 if it live out/live in to another block. Such approach doesn't affect 79 out accurate live range analysis. 80 - Current analysis didn't consider any instruction scheduling which 81 may improve the register pressure. So we are conservatively doing the 82 analysis which may end up with smaller LMUL. 83 TODO: Maybe we could support a reasonable live range shrink algorithm 84 which take advantage of instruction scheduling. 85 - We may have these following possible autovec modes analysis: 86 87 1. M8 -> M4 -> M2 -> M1 (stop analysis here) -> MF2 -> MF4 -> MF8 88 2. M8 -> M1(M4) -> MF2(M2) -> MF4(M1) (stop analysis here) -> MF8(MF2) 89 3. M1(M8) -> MF2(M4) -> MF4(M2) -> MF8(M1) 90 */ 91 92 static bool 93 is_gimple_assign_or_call (gimple *stmt) 94 { 95 return is_gimple_assign (stmt) || is_gimple_call (stmt); 96 } 97 98 /* Return the program point of 1st vectorized lanes statement. */ 99 static unsigned int 100 get_first_lane_point (const vec<stmt_point> program_points, 101 stmt_vec_info stmt_info) 102 { 103 for (const auto program_point : program_points) 104 if (program_point.stmt_info == DR_GROUP_FIRST_ELEMENT (stmt_info)) 105 return program_point.point; 106 return 0; 107 } 108 109 /* Return the program point of last vectorized lanes statement. */ 110 static unsigned int 111 get_last_lane_point (const vec<stmt_point> program_points, 112 stmt_vec_info stmt_info) 113 { 114 unsigned int max_point = 0; 115 for (auto s = DR_GROUP_FIRST_ELEMENT (stmt_info); s != NULL; 116 s = DR_GROUP_NEXT_ELEMENT (s)) 117 { 118 for (const auto program_point : program_points) 119 if (program_point.stmt_info == s && program_point.point > max_point) 120 max_point = program_point.point; 121 } 122 return max_point; 123 } 124 125 /* Return the last variable that is in the live range list. */ 126 static pair * 127 get_live_range (hash_map<tree, pair> *live_ranges, tree arg) 128 { 129 auto *r = live_ranges->get (arg); 130 if (r) 131 return r; 132 else 133 { 134 tree t = arg; 135 gimple *def_stmt = NULL; 136 while (t && TREE_CODE (t) == SSA_NAME && !r 137 && (def_stmt = SSA_NAME_DEF_STMT (t))) 138 { 139 if (gimple_assign_cast_p (def_stmt)) 140 { 141 t = gimple_assign_rhs1 (def_stmt); 142 r = live_ranges->get (t); 143 def_stmt = NULL; 144 } 145 else 146 /* FIXME: Currently we don't see any fold for 147 non-conversion statements. */ 148 t = NULL_TREE; 149 } 150 if (r) 151 return r; 152 else 153 { 154 bool insert_p = live_ranges->put (arg, pair (0, 0)); 155 gcc_assert (!insert_p); 156 return live_ranges->get (arg); 157 } 158 } 159 } 160 161 /* Collect all STMTs that are vectorized and compute their program points. 162 Note that we don't care about the STMTs that are not vectorized and 163 we only build the local graph (within a block) of program points. 164 165 Loop: 166 bb 2: 167 STMT 1 (be vectorized) -- point 0 168 STMT 2 (not be vectorized) -- ignored 169 STMT 3 (be vectorized) -- point 1 170 STMT 4 (be vectorized) -- point 2 171 STMT 5 (be vectorized) -- point 3 172 ... 173 bb 3: 174 STMT 1 (be vectorized) -- point 0 175 STMT 2 (be vectorized) -- point 1 176 STMT 3 (not be vectorized) -- ignored 177 STMT 4 (not be vectorized) -- ignored 178 STMT 5 (be vectorized) -- point 2 179 ... 180 */ 181 static void 182 compute_local_program_points ( 183 vec_info *vinfo, 184 hash_map<basic_block, vec<stmt_point>> &program_points_per_bb) 185 { 186 if (loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo)) 187 { 188 class loop *loop = LOOP_VINFO_LOOP (loop_vinfo); 189 basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo); 190 unsigned int nbbs = loop->num_nodes; 191 gimple_stmt_iterator si; 192 unsigned int i; 193 /* Collect the stmts that is vectorized and mark their program point. */ 194 for (i = 0; i < nbbs; i++) 195 { 196 int point = 1; 197 basic_block bb = bbs[i]; 198 vec<stmt_point> program_points = vNULL; 199 if (dump_enabled_p ()) 200 dump_printf_loc (MSG_NOTE, vect_location, 201 "Compute local program points for bb %d:\n", 202 bb->index); 203 for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si)) 204 { 205 if (!is_gimple_assign_or_call (gsi_stmt (si))) 206 continue; 207 stmt_vec_info stmt_info = vinfo->lookup_stmt (gsi_stmt (si)); 208 enum stmt_vec_info_type type 209 = STMT_VINFO_TYPE (vect_stmt_to_vectorize (stmt_info)); 210 if (type != undef_vec_info_type) 211 { 212 stmt_point info = {point, gsi_stmt (si), stmt_info}; 213 program_points.safe_push (info); 214 point++; 215 if (dump_enabled_p ()) 216 dump_printf_loc (MSG_NOTE, vect_location, 217 "program point %d: %G", info.point, 218 gsi_stmt (si)); 219 } 220 } 221 program_points_per_bb.put (bb, program_points); 222 } 223 } 224 } 225 226 static machine_mode 227 get_biggest_mode (machine_mode mode1, machine_mode mode2) 228 { 229 unsigned int mode1_size = GET_MODE_BITSIZE (mode1).to_constant (); 230 unsigned int mode2_size = GET_MODE_BITSIZE (mode2).to_constant (); 231 return mode1_size >= mode2_size ? mode1 : mode2; 232 } 233 234 /* Return true if OP is invariant. */ 235 236 static bool 237 loop_invariant_op_p (class loop *loop, 238 tree op) 239 { 240 if (is_gimple_constant (op)) 241 return true; 242 if (SSA_NAME_IS_DEFAULT_DEF (op) 243 || !flow_bb_inside_loop_p (loop, gimple_bb (SSA_NAME_DEF_STMT (op)))) 244 return true; 245 return false; 246 } 247 248 /* Return true if the variable should be counted into liveness. */ 249 static bool 250 variable_vectorized_p (class loop *loop, stmt_vec_info stmt_info, tree var, 251 bool lhs_p) 252 { 253 if (!var) 254 return false; 255 gimple *stmt = STMT_VINFO_STMT (stmt_info); 256 enum stmt_vec_info_type type 257 = STMT_VINFO_TYPE (vect_stmt_to_vectorize (stmt_info)); 258 if (is_gimple_call (stmt) && gimple_call_internal_p (stmt)) 259 { 260 if (gimple_call_internal_fn (stmt) == IFN_MASK_STORE 261 || gimple_call_internal_fn (stmt) == IFN_MASK_LOAD) 262 { 263 /* .MASK_LOAD (_5, 32B, _33) 264 ^ ^ ^ 265 Only the 3rd argument will be vectorized and consume 266 a vector register. */ 267 if (TREE_CODE (TREE_TYPE (var)) == BOOLEAN_TYPE 268 || (is_gimple_reg (var) && !POINTER_TYPE_P (TREE_TYPE (var)))) 269 return true; 270 else 271 return false; 272 } 273 } 274 else if (is_gimple_assign (stmt)) 275 { 276 tree_code tcode = gimple_assign_rhs_code (stmt); 277 /* vi variant doesn't need to allocate such statement. 278 E.g. tmp_15 = _4 + 1; will be transformed into vadd.vi 279 so the INTEGER_CST '1' doesn't need a vector register. */ 280 switch (tcode) 281 { 282 case PLUS_EXPR: 283 case BIT_IOR_EXPR: 284 case BIT_XOR_EXPR: 285 case BIT_AND_EXPR: 286 return TREE_CODE (var) != INTEGER_CST 287 || !tree_fits_shwi_p (var) 288 || !IN_RANGE (tree_to_shwi (var), -16, 15); 289 case MINUS_EXPR: 290 return TREE_CODE (var) != INTEGER_CST 291 || !tree_fits_shwi_p (var) 292 || !IN_RANGE (tree_to_shwi (var), -16, 15) 293 || gimple_assign_rhs1 (stmt) != var; 294 case LSHIFT_EXPR: 295 case RSHIFT_EXPR: 296 return gimple_assign_rhs2 (stmt) != var 297 || !loop_invariant_op_p (loop, var); 298 default: 299 break; 300 } 301 } 302 303 if (lhs_p) 304 return is_gimple_reg (var) 305 && (!POINTER_TYPE_P (TREE_TYPE (var)) 306 || type != store_vec_info_type); 307 else 308 return poly_int_tree_p (var) 309 || (is_gimple_val (var) 310 && (!POINTER_TYPE_P (TREE_TYPE (var)) 311 || type != load_vec_info_type)); 312 } 313 314 /* Compute local live ranges of each vectorized variable. 315 Note that we only compute local live ranges (within a block) since 316 local live ranges information is accurate enough for us to determine 317 the LMUL/vectorization factor of the loop. 318 319 Loop: 320 bb 2: 321 STMT 1 -- point 0 322 STMT 2 (def SSA 1) -- point 1 323 STMT 3 (use SSA 1) -- point 2 324 STMT 4 -- point 3 325 bb 3: 326 STMT 1 -- point 0 327 STMT 2 -- point 1 328 STMT 3 -- point 2 329 STMT 4 (use SSA 2) -- point 3 330 331 The live range of SSA 1 is [1, 3] in bb 2. 332 The live range of SSA 2 is [0, 4] in bb 3. */ 333 static machine_mode 334 compute_local_live_ranges ( 335 loop_vec_info loop_vinfo, 336 const hash_map<basic_block, vec<stmt_point>> &program_points_per_bb, 337 hash_map<basic_block, hash_map<tree, pair>> &live_ranges_per_bb) 338 { 339 machine_mode biggest_mode = QImode; 340 class loop *loop = LOOP_VINFO_LOOP (loop_vinfo); 341 if (!program_points_per_bb.is_empty ()) 342 { 343 auto_vec<tree> visited_vars; 344 unsigned int i; 345 for (hash_map<basic_block, vec<stmt_point>>::iterator iter 346 = program_points_per_bb.begin (); 347 iter != program_points_per_bb.end (); ++iter) 348 { 349 basic_block bb = (*iter).first; 350 vec<stmt_point> program_points = (*iter).second; 351 bool existed_p = false; 352 hash_map<tree, pair> *live_ranges 353 = &live_ranges_per_bb.get_or_insert (bb, &existed_p); 354 gcc_assert (!existed_p); 355 if (dump_enabled_p ()) 356 dump_printf_loc (MSG_NOTE, vect_location, 357 "Compute local live ranges for bb %d:\n", 358 bb->index); 359 for (const auto program_point : program_points) 360 { 361 unsigned int point = program_point.point; 362 gimple *stmt = program_point.stmt; 363 tree lhs = gimple_get_lhs (stmt); 364 if (variable_vectorized_p (loop, program_point.stmt_info, lhs, 365 true)) 366 { 367 biggest_mode = get_biggest_mode (biggest_mode, 368 TYPE_MODE (TREE_TYPE (lhs))); 369 bool existed_p = false; 370 pair &live_range 371 = live_ranges->get_or_insert (lhs, &existed_p); 372 gcc_assert (!existed_p); 373 if (STMT_VINFO_MEMORY_ACCESS_TYPE (program_point.stmt_info) 374 == VMAT_LOAD_STORE_LANES) 375 point = get_first_lane_point (program_points, 376 program_point.stmt_info); 377 live_range = pair (point, point); 378 } 379 for (i = 0; i < gimple_num_args (stmt); i++) 380 { 381 tree var = gimple_arg (stmt, i); 382 if (variable_vectorized_p (loop, program_point.stmt_info, var, 383 false)) 384 { 385 biggest_mode 386 = get_biggest_mode (biggest_mode, 387 TYPE_MODE (TREE_TYPE (var))); 388 bool existed_p = false; 389 pair &live_range 390 = live_ranges->get_or_insert (var, &existed_p); 391 if (STMT_VINFO_MEMORY_ACCESS_TYPE ( 392 program_point.stmt_info) 393 == VMAT_LOAD_STORE_LANES) 394 point = get_last_lane_point (program_points, 395 program_point.stmt_info); 396 else if (existed_p) 397 point = MAX (live_range.second, point); 398 if (existed_p) 399 /* We will grow the live range for each use. */ 400 live_range = pair (live_range.first, point); 401 else 402 { 403 gimple *def_stmt; 404 if (TREE_CODE (var) == SSA_NAME 405 && (def_stmt = SSA_NAME_DEF_STMT (var)) 406 && gimple_bb (def_stmt) == bb 407 && is_gimple_assign_or_call (def_stmt)) 408 { 409 live_ranges->remove (var); 410 for (unsigned int j = 0; 411 j < gimple_num_args (def_stmt); j++) 412 { 413 tree arg = gimple_arg (def_stmt, j); 414 auto *r = get_live_range (live_ranges, arg); 415 gcc_assert (r); 416 (*r).second = MAX (point, (*r).second); 417 biggest_mode = get_biggest_mode ( 418 biggest_mode, TYPE_MODE (TREE_TYPE (arg))); 419 } 420 } 421 else 422 /* The splat vector lives the whole block. */ 423 live_range = pair (0, program_points.length ()); 424 } 425 } 426 } 427 } 428 if (dump_enabled_p ()) 429 for (hash_map<tree, pair>::iterator iter = live_ranges->begin (); 430 iter != live_ranges->end (); ++iter) 431 dump_printf_loc (MSG_NOTE, vect_location, 432 "%T: type = %T, start = %d, end = %d\n", 433 (*iter).first, TREE_TYPE ((*iter).first), 434 (*iter).second.first, (*iter).second.second); 435 } 436 } 437 if (dump_enabled_p ()) 438 dump_printf_loc (MSG_NOTE, vect_location, "Biggest mode = %s\n", 439 GET_MODE_NAME (biggest_mode)); 440 return biggest_mode; 441 } 442 443 /* Compute the mode for MODE, BIGGEST_MODE and LMUL. 444 445 E.g. If mode = SImode, biggest_mode = DImode, LMUL = M4. 446 Then return RVVM4SImode (LMUL = 4, element mode = SImode). */ 447 static unsigned int 448 compute_nregs_for_mode (loop_vec_info loop_vinfo, machine_mode mode, 449 machine_mode biggest_mode, int lmul) 450 { 451 unsigned int rgroup_size = LOOP_VINFO_LENS (loop_vinfo).is_empty () 452 ? 1 453 : LOOP_VINFO_LENS (loop_vinfo).length (); 454 unsigned int mode_size = GET_MODE_SIZE (mode).to_constant (); 455 unsigned int biggest_size = GET_MODE_SIZE (biggest_mode).to_constant (); 456 gcc_assert (biggest_size >= mode_size); 457 unsigned int ratio = biggest_size / mode_size; 458 /* RVV mask bool modes always consume 1 vector register regardless LMUL. */ 459 unsigned int nregs = mode == BImode ? 1 : lmul / ratio; 460 return MAX (nregs, 1) * rgroup_size; 461 } 462 463 /* This function helps to determine whether current LMUL will cause 464 potential vector register (V_REG) spillings according to live range 465 information. 466 467 - First, compute how many variable are alive of each program point 468 in each bb of the loop. 469 - Second, compute how many V_REGs are alive of each program point 470 in each bb of the loop according the BIGGEST_MODE and the variable 471 mode. 472 - Third, Return the maximum V_REGs are alive of the loop. */ 473 static unsigned int 474 max_number_of_live_regs (loop_vec_info loop_vinfo, const basic_block bb, 475 const hash_map<tree, pair> &live_ranges, 476 unsigned int max_point, machine_mode biggest_mode, 477 int lmul) 478 { 479 unsigned int max_nregs = 0; 480 unsigned int i; 481 unsigned int live_point = 0; 482 auto_vec<unsigned int> live_vars_vec; 483 live_vars_vec.safe_grow_cleared (max_point, true); 484 for (hash_map<tree, pair>::iterator iter = live_ranges.begin (); 485 iter != live_ranges.end (); ++iter) 486 { 487 tree var = (*iter).first; 488 pair live_range = (*iter).second; 489 for (i = live_range.first + 1; i <= live_range.second; i++) 490 { 491 machine_mode mode = TREE_CODE (TREE_TYPE (var)) == BOOLEAN_TYPE 492 ? BImode 493 : TYPE_MODE (TREE_TYPE (var)); 494 unsigned int nregs 495 = compute_nregs_for_mode (loop_vinfo, mode, biggest_mode, lmul); 496 live_vars_vec[i] += nregs; 497 if (live_vars_vec[i] > max_nregs) 498 { 499 max_nregs = live_vars_vec[i]; 500 live_point = i; 501 } 502 } 503 } 504 505 /* Collect user explicit RVV type. */ 506 auto_vec<basic_block> all_preds 507 = get_all_dominated_blocks (CDI_POST_DOMINATORS, bb); 508 tree t; 509 FOR_EACH_SSA_NAME (i, t, cfun) 510 { 511 machine_mode mode = TYPE_MODE (TREE_TYPE (t)); 512 if (!lookup_vector_type_attribute (TREE_TYPE (t)) 513 && !riscv_v_ext_vls_mode_p (mode)) 514 continue; 515 516 gimple *def = SSA_NAME_DEF_STMT (t); 517 if (gimple_bb (def) && !all_preds.contains (gimple_bb (def))) 518 continue; 519 use_operand_p use_p; 520 imm_use_iterator iterator; 521 522 FOR_EACH_IMM_USE_FAST (use_p, iterator, t) 523 { 524 if (!USE_STMT (use_p) || is_gimple_debug (USE_STMT (use_p)) 525 || !dominated_by_p (CDI_POST_DOMINATORS, bb, 526 gimple_bb (USE_STMT (use_p)))) 527 continue; 528 529 int regno_alignment = riscv_get_v_regno_alignment (mode); 530 max_nregs += regno_alignment; 531 if (dump_enabled_p ()) 532 dump_printf_loc ( 533 MSG_NOTE, vect_location, 534 "Explicit used SSA %T, vectype = %T, mode = %s, cause %d " 535 "V_REG live in bb %d at program point %d\n", 536 t, TREE_TYPE (t), GET_MODE_NAME (mode), regno_alignment, 537 bb->index, live_point); 538 break; 539 } 540 } 541 542 if (dump_enabled_p ()) 543 dump_printf_loc ( 544 MSG_NOTE, vect_location, 545 "Maximum lmul = %d, At most %d number of live V_REG at program " 546 "point %d for bb %d\n", 547 lmul, max_nregs, live_point, bb->index); 548 return max_nregs; 549 } 550 551 /* Get STORE value. */ 552 static tree 553 get_store_value (gimple *stmt) 554 { 555 if (is_gimple_call (stmt) && gimple_call_internal_p (stmt)) 556 { 557 if (gimple_call_internal_fn (stmt) == IFN_MASK_STORE) 558 return gimple_call_arg (stmt, 3); 559 else 560 gcc_unreachable (); 561 } 562 else 563 return gimple_assign_rhs1 (stmt); 564 } 565 566 /* Return true if it is non-contiguous load/store. */ 567 static bool 568 non_contiguous_memory_access_p (stmt_vec_info stmt_info) 569 { 570 enum stmt_vec_info_type type 571 = STMT_VINFO_TYPE (vect_stmt_to_vectorize (stmt_info)); 572 return ((type == load_vec_info_type || type == store_vec_info_type) 573 && !adjacent_dr_p (STMT_VINFO_DATA_REF (stmt_info))); 574 } 575 576 /* Return the LMUL of the current analysis. */ 577 static int 578 compute_estimated_lmul (loop_vec_info loop_vinfo, machine_mode mode) 579 { 580 gcc_assert (GET_MODE_BITSIZE (mode).is_constant ()); 581 int regno_alignment = riscv_get_v_regno_alignment (loop_vinfo->vector_mode); 582 if (riscv_v_ext_vls_mode_p (loop_vinfo->vector_mode)) 583 return regno_alignment; 584 else if (known_eq (LOOP_VINFO_SLP_UNROLLING_FACTOR (loop_vinfo), 1U)) 585 { 586 int estimated_vf = vect_vf_for_cost (loop_vinfo); 587 int estimated_lmul = estimated_vf * GET_MODE_BITSIZE (mode).to_constant () 588 / TARGET_MIN_VLEN; 589 if (estimated_lmul > RVV_M8) 590 return regno_alignment; 591 else 592 return estimated_lmul; 593 } 594 else 595 { 596 /* Estimate the VLA SLP LMUL. */ 597 if (regno_alignment > RVV_M1) 598 return regno_alignment; 599 else if (mode != QImode 600 || LOOP_VINFO_SLP_UNROLLING_FACTOR (loop_vinfo).is_constant ()) 601 { 602 int ratio; 603 if (can_div_trunc_p (BYTES_PER_RISCV_VECTOR, 604 GET_MODE_SIZE (loop_vinfo->vector_mode), &ratio)) 605 { 606 if (ratio == 1) 607 return RVV_M4; 608 else if (ratio == 2) 609 return RVV_M2; 610 } 611 } 612 } 613 return 0; 614 } 615 616 /* Update the live ranges according PHI. 617 618 Loop: 619 bb 2: 620 STMT 1 -- point 0 621 STMT 2 (def SSA 1) -- point 1 622 STMT 3 (use SSA 1) -- point 2 623 STMT 4 -- point 3 624 bb 3: 625 SSA 2 = PHI<SSA 1> 626 STMT 1 -- point 0 627 STMT 2 -- point 1 628 STMT 3 (use SSA 2) -- point 2 629 STMT 4 -- point 3 630 631 Before this function, the SSA 1 live range is [2, 3] in bb 2 632 and SSA 2 is [0, 3] in bb 3. 633 634 Then, after this function, we update SSA 1 live range in bb 2 635 into [2, 4] since SSA 1 is live out into bb 3. */ 636 static void 637 update_local_live_ranges ( 638 vec_info *vinfo, 639 hash_map<basic_block, vec<stmt_point>> &program_points_per_bb, 640 hash_map<basic_block, hash_map<tree, pair>> &live_ranges_per_bb, 641 machine_mode *biggest_mode) 642 { 643 loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo); 644 if (!loop_vinfo) 645 return; 646 647 class loop *loop = LOOP_VINFO_LOOP (loop_vinfo); 648 basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo); 649 unsigned int nbbs = loop->num_nodes; 650 unsigned int i, j; 651 gphi_iterator psi; 652 gimple_stmt_iterator si; 653 for (i = 0; i < nbbs; i++) 654 { 655 basic_block bb = bbs[i]; 656 if (dump_enabled_p ()) 657 dump_printf_loc (MSG_NOTE, vect_location, 658 "Update local program points for bb %d:\n", 659 bbs[i]->index); 660 for (psi = gsi_start_phis (bb); !gsi_end_p (psi); gsi_next (&psi)) 661 { 662 gphi *phi = psi.phi (); 663 stmt_vec_info stmt_info = vinfo->lookup_stmt (phi); 664 if (STMT_VINFO_TYPE (vect_stmt_to_vectorize (stmt_info)) 665 == undef_vec_info_type) 666 continue; 667 668 for (j = 0; j < gimple_phi_num_args (phi); j++) 669 { 670 edge e = gimple_phi_arg_edge (phi, j); 671 tree def = gimple_phi_arg_def (phi, j); 672 auto *live_ranges = live_ranges_per_bb.get (bb); 673 auto *live_range = live_ranges->get (def); 674 if (poly_int_tree_p (def)) 675 { 676 /* Insert live range of INTEGER_CST or POLY_CST since we will 677 need to allocate a vector register for it. 678 679 E.g. # j_17 = PHI <j_11(9), 0(5)> will be transformed 680 into # vect_vec_iv_.8_24 = PHI <_25(9), { 0, ... }(5)> 681 682 The live range for such value is short which only lives 683 from program point 0 to 1. */ 684 if (live_range) 685 { 686 unsigned int start = (*live_range).first; 687 (*live_range).first = 0; 688 if (dump_enabled_p ()) 689 dump_printf_loc ( 690 MSG_NOTE, vect_location, 691 "Update %T start point from %d to 0:\n", def, start); 692 } 693 else 694 { 695 live_ranges->put (def, pair (0, 1)); 696 auto &program_points = (*program_points_per_bb.get (bb)); 697 if (program_points.is_empty ()) 698 { 699 stmt_point info = {1, phi, stmt_info}; 700 program_points.safe_push (info); 701 } 702 if (dump_enabled_p ()) 703 dump_printf_loc (MSG_NOTE, vect_location, 704 "Add %T start point from 0 to 1:\n", 705 def); 706 } 707 continue; 708 } 709 if (live_range && flow_bb_inside_loop_p (loop, e->src)) 710 { 711 unsigned int start = (*live_range).first; 712 (*live_range).first = 0; 713 if (dump_enabled_p ()) 714 dump_printf_loc (MSG_NOTE, vect_location, 715 "Update %T start point from %d to %d:\n", 716 def, start, (*live_range).first); 717 } 718 live_ranges = live_ranges_per_bb.get (e->src); 719 if (!program_points_per_bb.get (e->src)) 720 continue; 721 unsigned int max_point 722 = (*program_points_per_bb.get (e->src)).length (); 723 live_range = live_ranges->get (def); 724 if (!live_range) 725 continue; 726 727 unsigned int end = (*live_range).second; 728 (*live_range).second = max_point; 729 if (dump_enabled_p ()) 730 dump_printf_loc (MSG_NOTE, vect_location, 731 "Update %T end point from %d to %d:\n", def, 732 end, (*live_range).second); 733 } 734 } 735 for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si)) 736 { 737 if (!is_gimple_assign_or_call (gsi_stmt (si))) 738 continue; 739 stmt_vec_info stmt_info = vinfo->lookup_stmt (gsi_stmt (si)); 740 enum stmt_vec_info_type type 741 = STMT_VINFO_TYPE (vect_stmt_to_vectorize (stmt_info)); 742 if (non_contiguous_memory_access_p (stmt_info) 743 /* LOAD_LANES/STORE_LANES doesn't need a perm indice. */ 744 && STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info) 745 != VMAT_LOAD_STORE_LANES) 746 { 747 /* For non-adjacent load/store STMT, we will potentially 748 convert it into: 749 750 1. MASK_LEN_GATHER_LOAD (..., perm indice). 751 2. Continguous load/store + VEC_PERM (..., perm indice) 752 753 We will be likely using one more vector variable. */ 754 unsigned int max_point 755 = (*program_points_per_bb.get (bb)).length (); 756 auto *live_ranges = live_ranges_per_bb.get (bb); 757 bool existed_p = false; 758 tree var = type == load_vec_info_type 759 ? gimple_get_lhs (gsi_stmt (si)) 760 : get_store_value (gsi_stmt (si)); 761 tree sel_type = build_nonstandard_integer_type ( 762 TYPE_PRECISION (TREE_TYPE (var)), 1); 763 *biggest_mode 764 = get_biggest_mode (*biggest_mode, TYPE_MODE (sel_type)); 765 tree sel = build_decl (UNKNOWN_LOCATION, VAR_DECL, 766 get_identifier ("vect_perm"), sel_type); 767 pair &live_range = live_ranges->get_or_insert (sel, &existed_p); 768 gcc_assert (!existed_p); 769 live_range = pair (0, max_point); 770 if (dump_enabled_p ()) 771 dump_printf_loc (MSG_NOTE, vect_location, 772 "Add perm indice %T, start = 0, end = %d\n", 773 sel, max_point); 774 if (!LOOP_VINFO_LENS (loop_vinfo).is_empty () 775 && LOOP_VINFO_LENS (loop_vinfo).length () > 1) 776 { 777 /* If we are vectorizing a permutation when the rgroup number 778 > 1, we will need additional mask to shuffle the second 779 vector. */ 780 tree mask = build_decl (UNKNOWN_LOCATION, VAR_DECL, 781 get_identifier ("vect_perm_mask"), 782 boolean_type_node); 783 pair &live_range 784 = live_ranges->get_or_insert (mask, &existed_p); 785 gcc_assert (!existed_p); 786 live_range = pair (0, max_point); 787 if (dump_enabled_p ()) 788 dump_printf_loc (MSG_NOTE, vect_location, 789 "Add perm mask %T, start = 0, end = %d\n", 790 mask, max_point); 791 } 792 } 793 } 794 } 795 } 796 797 /* Compute the maximum live V_REGS. */ 798 static bool 799 has_unexpected_spills_p (loop_vec_info loop_vinfo) 800 { 801 /* Compute local program points. 802 It's a fast and effective computation. */ 803 hash_map<basic_block, vec<stmt_point>> program_points_per_bb; 804 compute_local_program_points (loop_vinfo, program_points_per_bb); 805 806 /* Compute local live ranges. */ 807 hash_map<basic_block, hash_map<tree, pair>> live_ranges_per_bb; 808 machine_mode biggest_mode 809 = compute_local_live_ranges (loop_vinfo, program_points_per_bb, 810 live_ranges_per_bb); 811 812 /* Update live ranges according to PHI. */ 813 update_local_live_ranges (loop_vinfo, program_points_per_bb, 814 live_ranges_per_bb, &biggest_mode); 815 816 int lmul = compute_estimated_lmul (loop_vinfo, biggest_mode); 817 gcc_assert (lmul <= RVV_M8); 818 /* TODO: We calculate the maximum live vars base on current STMTS 819 sequence. We can support live range shrink if it can give us 820 big improvement in the future. */ 821 if (lmul > RVV_M1) 822 { 823 if (!live_ranges_per_bb.is_empty ()) 824 { 825 unsigned int max_nregs = 0; 826 for (hash_map<basic_block, hash_map<tree, pair>>::iterator iter 827 = live_ranges_per_bb.begin (); 828 iter != live_ranges_per_bb.end (); ++iter) 829 { 830 basic_block bb = (*iter).first; 831 unsigned int max_point 832 = (*program_points_per_bb.get (bb)).length () + 1; 833 if ((*iter).second.is_empty ()) 834 continue; 835 /* We prefer larger LMUL unless it causes register spillings. */ 836 unsigned int nregs 837 = max_number_of_live_regs (loop_vinfo, bb, (*iter).second, 838 max_point, biggest_mode, lmul); 839 if (nregs > max_nregs) 840 max_nregs = nregs; 841 } 842 live_ranges_per_bb.empty (); 843 if (max_nregs > V_REG_NUM) 844 return true; 845 } 846 } 847 if (!program_points_per_bb.is_empty ()) 848 { 849 for (hash_map<basic_block, vec<stmt_point>>::iterator iter 850 = program_points_per_bb.begin (); 851 iter != program_points_per_bb.end (); ++iter) 852 { 853 vec<stmt_point> program_points = (*iter).second; 854 if (!program_points.is_empty ()) 855 program_points.release (); 856 } 857 program_points_per_bb.empty (); 858 } 859 return false; 860 } 861 862 costs::costs (vec_info *vinfo, bool costing_for_scalar) 863 : vector_costs (vinfo, costing_for_scalar) 864 { 865 if (costing_for_scalar) 866 m_cost_type = SCALAR_COST; 867 else if (riscv_v_ext_vector_mode_p (vinfo->vector_mode)) 868 m_cost_type = VLA_VECTOR_COST; 869 else 870 m_cost_type = VLS_VECTOR_COST; 871 } 872 873 /* Do one-time initialization of the costs given that we're 874 costing the loop vectorization described by LOOP_VINFO. */ 875 void 876 costs::analyze_loop_vinfo (loop_vec_info loop_vinfo) 877 { 878 /* Detect whether we're vectorizing for VLA and should apply the unrolling 879 heuristic described above m_unrolled_vls_niters. */ 880 record_potential_vls_unrolling (loop_vinfo); 881 882 /* Detect whether the LOOP has unexpected spills. */ 883 record_potential_unexpected_spills (loop_vinfo); 884 } 885 886 /* Analyze the vectorized program stataments and use dynamic LMUL 887 heuristic to detect whether the loop has unexpected spills. */ 888 void 889 costs::record_potential_unexpected_spills (loop_vec_info loop_vinfo) 890 { 891 /* We only want to apply the heuristic if LOOP_VINFO is being 892 vectorized for VLA and known NITERS VLS loop. */ 893 if (rvv_max_lmul == RVV_DYNAMIC 894 && (m_cost_type == VLA_VECTOR_COST 895 || (m_cost_type == VLS_VECTOR_COST 896 && LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)))) 897 { 898 bool post_dom_available_p = dom_info_available_p (CDI_POST_DOMINATORS); 899 if (!post_dom_available_p) 900 calculate_dominance_info (CDI_POST_DOMINATORS); 901 m_has_unexpected_spills_p = has_unexpected_spills_p (loop_vinfo); 902 if (!post_dom_available_p) 903 free_dominance_info (CDI_POST_DOMINATORS); 904 } 905 } 906 907 /* Decide whether to use the unrolling heuristic described above 908 m_unrolled_vls_niters, updating that field if so. LOOP_VINFO 909 describes the loop that we're vectorizing. */ 910 void 911 costs::record_potential_vls_unrolling (loop_vec_info loop_vinfo) 912 { 913 /* We only want to apply the heuristic if LOOP_VINFO is being 914 vectorized for VLA. */ 915 if (m_cost_type != VLA_VECTOR_COST) 916 return; 917 918 /* We don't want to apply the heuristic to outer loops, since it's 919 harder to track two levels of unrolling. */ 920 if (LOOP_VINFO_LOOP (loop_vinfo)->inner) 921 return; 922 923 /* Only handle cases in which the number of VLS iterations 924 would be known at compile time but the number of SVE iterations 925 would not. */ 926 if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo) 927 || BYTES_PER_RISCV_VECTOR.is_constant ()) 928 return; 929 930 /* Guess how many times the VLS loop would iterate and make 931 sure that it is within the complete unrolling limit. Even if the 932 number of iterations is small enough, the number of statements might 933 not be, which is why we need to estimate the number of statements too. */ 934 unsigned int vls_vf = vect_vf_for_cost (loop_vinfo); 935 unsigned HOST_WIDE_INT unrolled_vls_niters 936 = LOOP_VINFO_INT_NITERS (loop_vinfo) / vls_vf; 937 if (unrolled_vls_niters > (unsigned int) param_max_completely_peel_times) 938 return; 939 940 /* Record that we're applying the heuristic and should try to estimate 941 the number of statements in the VLS loop. */ 942 m_unrolled_vls_niters = unrolled_vls_niters; 943 } 944 945 /* Return true if (a) we're applying the VLS vs. VLA unrolling 946 heuristic described above m_unrolled_vls_niters and (b) the heuristic 947 says that we should prefer the VLS loop. */ 948 bool 949 costs::prefer_unrolled_loop () const 950 { 951 if (!m_unrolled_vls_stmts) 952 return false; 953 954 if (dump_enabled_p ()) 955 dump_printf_loc (MSG_NOTE, vect_location, 956 "Number of insns in" 957 " unrolled VLS loop = " HOST_WIDE_INT_PRINT_UNSIGNED "\n", 958 m_unrolled_vls_stmts); 959 960 /* The balance here is tricky. On the one hand, we can't be sure whether 961 the code is vectorizable with VLS or not. However, even if 962 it isn't vectorizable with VLS, there's a possibility that 963 the scalar code could also be unrolled. Some of the code might then 964 benefit from SLP, or from using LDP and STP. We therefore apply 965 the heuristic regardless of can_use_vls_p. */ 966 return (m_unrolled_vls_stmts 967 && (m_unrolled_vls_stmts 968 <= (unsigned int) param_max_completely_peeled_insns)); 969 } 970 971 bool 972 costs::better_main_loop_than_p (const vector_costs *uncast_other) const 973 { 974 auto other = static_cast<const costs *> (uncast_other); 975 auto this_loop_vinfo = as_a<loop_vec_info> (this->m_vinfo); 976 auto other_loop_vinfo = as_a<loop_vec_info> (other->m_vinfo); 977 978 if (dump_enabled_p ()) 979 dump_printf_loc (MSG_NOTE, vect_location, 980 "Comparing two main loops (%s at VF %d vs %s at VF %d)\n", 981 GET_MODE_NAME (this_loop_vinfo->vector_mode), 982 vect_vf_for_cost (this_loop_vinfo), 983 GET_MODE_NAME (other_loop_vinfo->vector_mode), 984 vect_vf_for_cost (other_loop_vinfo)); 985 986 /* Apply the unrolling heuristic described above m_unrolled_vls_niters. */ 987 if (bool (m_unrolled_vls_stmts) != bool (other->m_unrolled_vls_stmts) 988 && m_cost_type != other->m_cost_type) 989 { 990 bool this_prefer_unrolled = this->prefer_unrolled_loop (); 991 bool other_prefer_unrolled = other->prefer_unrolled_loop (); 992 if (this_prefer_unrolled != other_prefer_unrolled) 993 { 994 if (dump_enabled_p ()) 995 dump_printf_loc (MSG_NOTE, vect_location, 996 "Preferring VLS loop because" 997 " it can be unrolled\n"); 998 return other_prefer_unrolled; 999 } 1000 } 1001 else if (rvv_max_lmul == RVV_DYNAMIC) 1002 { 1003 if (other->m_has_unexpected_spills_p) 1004 { 1005 if (dump_enabled_p ()) 1006 dump_printf_loc (MSG_NOTE, vect_location, 1007 "Preferring smaller LMUL loop because" 1008 " it has unexpected spills\n"); 1009 return true; 1010 } 1011 else if (riscv_v_ext_vector_mode_p (other_loop_vinfo->vector_mode)) 1012 { 1013 if (LOOP_VINFO_NITERS_KNOWN_P (other_loop_vinfo)) 1014 { 1015 if (maybe_gt (LOOP_VINFO_INT_NITERS (this_loop_vinfo), 1016 LOOP_VINFO_VECT_FACTOR (this_loop_vinfo))) 1017 { 1018 if (dump_enabled_p ()) 1019 dump_printf_loc (MSG_NOTE, vect_location, 1020 "Keep current LMUL loop because" 1021 " known NITERS exceed the new VF\n"); 1022 return false; 1023 } 1024 } 1025 else 1026 { 1027 if (dump_enabled_p ()) 1028 dump_printf_loc (MSG_NOTE, vect_location, 1029 "Keep current LMUL loop because" 1030 " it is unknown NITERS\n"); 1031 return false; 1032 } 1033 } 1034 } 1035 /* If NITERS is unknown, we should not use VLS modes to vectorize 1036 the loop since we don't support partial vectors for VLS modes, 1037 that is, we will have full vectors (VLSmodes) on loop body 1038 and partial vectors (VLAmodes) on loop epilogue which is very 1039 inefficient. Instead, we should apply partial vectors (VLAmodes) 1040 on loop body without an epilogue on unknown NITERS loop. */ 1041 else if (!LOOP_VINFO_NITERS_KNOWN_P (this_loop_vinfo) 1042 && m_cost_type == VLS_VECTOR_COST) 1043 return false; 1044 1045 return vector_costs::better_main_loop_than_p (other); 1046 } 1047 1048 /* Adjust vectorization cost after calling riscv_builtin_vectorization_cost. 1049 For some statement, we would like to further fine-grain tweak the cost on 1050 top of riscv_builtin_vectorization_cost handling which doesn't have any 1051 information on statement operation codes etc. */ 1052 1053 unsigned 1054 costs::adjust_stmt_cost (enum vect_cost_for_stmt kind, loop_vec_info loop, 1055 stmt_vec_info stmt_info, 1056 slp_tree, tree vectype, int stmt_cost) 1057 { 1058 const cpu_vector_cost *costs = get_vector_costs (); 1059 switch (kind) 1060 { 1061 case scalar_to_vec: 1062 stmt_cost += (FLOAT_TYPE_P (vectype) ? costs->regmove->FR2VR 1063 : costs->regmove->GR2VR); 1064 break; 1065 case vec_to_scalar: 1066 stmt_cost += (FLOAT_TYPE_P (vectype) ? costs->regmove->VR2FR 1067 : costs->regmove->VR2GR); 1068 break; 1069 case vector_load: 1070 case vector_store: 1071 { 1072 /* Unit-stride vector loads and stores do not have offset addressing 1073 as opposed to scalar loads and stores. 1074 If the address depends on a variable we need an additional 1075 add/sub for each load/store in the worst case. */ 1076 if (stmt_info && stmt_info->stmt) 1077 { 1078 data_reference *dr = STMT_VINFO_DATA_REF (stmt_info); 1079 class loop *father = stmt_info->stmt->bb->loop_father; 1080 if (!loop && father && !father->inner && father->superloops) 1081 { 1082 tree ref; 1083 if (TREE_CODE (dr->ref) != MEM_REF 1084 || !(ref = TREE_OPERAND (dr->ref, 0)) 1085 || TREE_CODE (ref) != SSA_NAME) 1086 break; 1087 1088 if (SSA_NAME_IS_DEFAULT_DEF (ref)) 1089 break; 1090 1091 if (memrefs.contains ({ref, cst0})) 1092 break; 1093 1094 memrefs.add ({ref, cst0}); 1095 1096 /* In case we have not seen REF before and the base address 1097 is a pointer operation try a bit harder. */ 1098 tree base = DR_BASE_ADDRESS (dr); 1099 if (TREE_CODE (base) == POINTER_PLUS_EXPR 1100 || TREE_CODE (base) == POINTER_DIFF_EXPR) 1101 { 1102 /* Deconstruct BASE's first operand. If it is a binary 1103 operation, i.e. a base and an "offset" store this 1104 pair. Only increase the stmt_cost if we haven't seen 1105 it before. */ 1106 tree argp = TREE_OPERAND (base, 1); 1107 typedef std::pair<tree, tree> addr_pair; 1108 addr_pair pair; 1109 if (TREE_CODE_CLASS (TREE_CODE (argp)) == tcc_binary) 1110 { 1111 tree argp0 = tree_strip_nop_conversions 1112 (TREE_OPERAND (argp, 0)); 1113 tree argp1 = TREE_OPERAND (argp, 1); 1114 pair = addr_pair (argp0, argp1); 1115 if (memrefs.contains (pair)) 1116 break; 1117 1118 memrefs.add (pair); 1119 stmt_cost += builtin_vectorization_cost (scalar_stmt, 1120 NULL_TREE, 0); 1121 } 1122 } 1123 } 1124 } 1125 break; 1126 } 1127 1128 default: 1129 break; 1130 } 1131 return stmt_cost; 1132 } 1133 1134 unsigned 1135 costs::add_stmt_cost (int count, vect_cost_for_stmt kind, 1136 stmt_vec_info stmt_info, slp_tree node, tree vectype, 1137 int misalign, vect_cost_model_location where) 1138 { 1139 int stmt_cost 1140 = targetm.vectorize.builtin_vectorization_cost (kind, vectype, misalign); 1141 1142 /* Do one-time initialization based on the vinfo. */ 1143 loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (m_vinfo); 1144 if (!m_analyzed_vinfo) 1145 { 1146 if (loop_vinfo) 1147 analyze_loop_vinfo (loop_vinfo); 1148 1149 memrefs.empty (); 1150 m_analyzed_vinfo = true; 1151 } 1152 1153 if (stmt_info) 1154 { 1155 /* If we're applying the VLA vs. VLS unrolling heuristic, 1156 estimate the number of statements in the unrolled VLS 1157 loop. For simplicitly, we assume that one iteration of the 1158 VLS loop would need the same number of statements 1159 as one iteration of the VLA loop. */ 1160 if (where == vect_body && m_unrolled_vls_niters) 1161 m_unrolled_vls_stmts += count * m_unrolled_vls_niters; 1162 } 1163 1164 if (vectype) 1165 stmt_cost = adjust_stmt_cost (kind, loop_vinfo, stmt_info, node, vectype, 1166 stmt_cost); 1167 1168 return record_stmt_cost (stmt_info, where, count * stmt_cost); 1169 } 1170 1171 /* For some target specific vectorization cost which can't be handled per stmt, 1172 we check the requisite conditions and adjust the vectorization cost 1173 accordingly if satisfied. One typical example is to model and adjust 1174 loop_len cost for known_lt (NITERS, VF). */ 1175 1176 void 1177 costs::adjust_vect_cost_per_loop (loop_vec_info loop_vinfo) 1178 { 1179 if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo) 1180 && !LOOP_VINFO_USING_DECREMENTING_IV_P (loop_vinfo)) 1181 { 1182 /* In middle-end loop vectorizer, we don't count the loop_len cost in 1183 vect_estimate_min_profitable_iters when NITERS < VF, that is, we only 1184 count cost of len that we need to iterate loop more than once with VF. 1185 It's correct for most of the cases: 1186 1187 E.g. VF = [4, 4] 1188 for (int i = 0; i < 3; i ++) 1189 a[i] += b[i]; 1190 1191 We don't need to cost MIN_EXPR or SELECT_VL for the case above. 1192 1193 However, for some inefficient vectorized cases, it does use MIN_EXPR 1194 to generate len. 1195 1196 E.g. VF = [256, 256] 1197 1198 Loop body: 1199 # loop_len_110 = PHI <18(2), _119(11)> 1200 ... 1201 _117 = MIN_EXPR <ivtmp_114, 18>; 1202 _118 = 18 - _117; 1203 _119 = MIN_EXPR <_118, POLY_INT_CST [256, 256]>; 1204 ... 1205 1206 Epilogue: 1207 ... 1208 _112 = .VEC_EXTRACT (vect_patt_27.14_109, _111); 1209 1210 We cost 1 unconditionally for this situation like other targets which 1211 apply mask as the loop control. */ 1212 rgroup_controls *rgc; 1213 unsigned int num_vectors_m1; 1214 unsigned int body_stmts = 0; 1215 FOR_EACH_VEC_ELT (LOOP_VINFO_LENS (loop_vinfo), num_vectors_m1, rgc) 1216 if (rgc->type) 1217 body_stmts += num_vectors_m1 + 1; 1218 1219 add_stmt_cost (body_stmts, scalar_stmt, NULL, NULL, NULL_TREE, 0, 1220 vect_body); 1221 } 1222 } 1223 1224 void 1225 costs::finish_cost (const vector_costs *scalar_costs) 1226 { 1227 if (loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (m_vinfo)) 1228 { 1229 adjust_vect_cost_per_loop (loop_vinfo); 1230 } 1231 vector_costs::finish_cost (scalar_costs); 1232 } 1233 1234 } // namespace riscv_vector 1235
Indexes created Thu Jul 16 00:26:27 UTC 2026