Home | History | Annotate | Line # | Download | only in AArch64 
      1 //===-- AArch64SelectionDAGInfo.cpp - AArch64 SelectionDAG Info -----------===//
      2 //
      3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
      4 // See https://llvm.org/LICENSE.txt for license information.
      5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
      6 //
      7 //===----------------------------------------------------------------------===//
      8 //
      9 // This file implements the AArch64SelectionDAGInfo class.
     10 //
     11 //===----------------------------------------------------------------------===//
     12 
     13 #include "AArch64TargetMachine.h"
     14 using namespace llvm;
     15 
     16 #define DEBUG_TYPE "aarch64-selectiondag-info"
     17 
     18 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemset(
     19     SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
     20     SDValue Size, Align Alignment, bool isVolatile,
     21     MachinePointerInfo DstPtrInfo) const {
     22   // Check to see if there is a specialized entry-point for memory zeroing.
     23   ConstantSDNode *V = dyn_cast<ConstantSDNode>(Src);
     24   ConstantSDNode *SizeValue = dyn_cast<ConstantSDNode>(Size);
     25   const AArch64Subtarget &STI =
     26       DAG.getMachineFunction().getSubtarget<AArch64Subtarget>();
     27   const char *bzeroName = (V && V->isNullValue())
     28       ? DAG.getTargetLoweringInfo().getLibcallName(RTLIB::BZERO) : nullptr;
     29   // For small size (< 256), it is not beneficial to use bzero
     30   // instead of memset.
     31   if (bzeroName && (!SizeValue || SizeValue->getZExtValue() > 256)) {
     32     const AArch64TargetLowering &TLI = *STI.getTargetLowering();
     33 
     34     EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout());
     35     Type *IntPtrTy = Type::getInt8PtrTy(*DAG.getContext());
     36     TargetLowering::ArgListTy Args;
     37     TargetLowering::ArgListEntry Entry;
     38     Entry.Node = Dst;
     39     Entry.Ty = IntPtrTy;
     40     Args.push_back(Entry);
     41     Entry.Node = Size;
     42     Args.push_back(Entry);
     43     TargetLowering::CallLoweringInfo CLI(DAG);
     44     CLI.setDebugLoc(dl)
     45         .setChain(Chain)
     46         .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
     47                       DAG.getExternalSymbol(bzeroName, IntPtr),
     48                       std::move(Args))
     49         .setDiscardResult();
     50     std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
     51     return CallResult.second;
     52   }
     53   return SDValue();
     54 }
     55 
     56 static const int kSetTagLoopThreshold = 176;
     57 
     58 static SDValue EmitUnrolledSetTag(SelectionDAG &DAG, const SDLoc &dl,
     59                                   SDValue Chain, SDValue Ptr, uint64_t ObjSize,
     60                                   const MachineMemOperand *BaseMemOperand,
     61                                   bool ZeroData) {
     62   MachineFunction &MF = DAG.getMachineFunction();
     63   unsigned ObjSizeScaled = ObjSize / 16;
     64 
     65   SDValue TagSrc = Ptr;
     66   if (Ptr.getOpcode() == ISD::FrameIndex) {
     67     int FI = cast<FrameIndexSDNode>(Ptr)->getIndex();
     68     Ptr = DAG.getTargetFrameIndex(FI, MVT::i64);
     69     // A frame index operand may end up as [SP + offset] => it is fine to use SP
     70     // register as the tag source.
     71     TagSrc = DAG.getRegister(AArch64::SP, MVT::i64);
     72   }
     73 
     74   const unsigned OpCode1 = ZeroData ? AArch64ISD::STZG : AArch64ISD::STG;
     75   const unsigned OpCode2 = ZeroData ? AArch64ISD::STZ2G : AArch64ISD::ST2G;
     76 
     77   SmallVector<SDValue, 8> OutChains;
     78   unsigned OffsetScaled = 0;
     79   while (OffsetScaled < ObjSizeScaled) {
     80     if (ObjSizeScaled - OffsetScaled >= 2) {
     81       SDValue AddrNode =
     82           DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(OffsetScaled * 16), dl);
     83       SDValue St = DAG.getMemIntrinsicNode(
     84           OpCode2, dl, DAG.getVTList(MVT::Other),
     85           {Chain, TagSrc, AddrNode},
     86           MVT::v4i64,
     87           MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16 * 2));
     88       OffsetScaled += 2;
     89       OutChains.push_back(St);
     90       continue;
     91     }
     92 
     93     if (ObjSizeScaled - OffsetScaled > 0) {
     94       SDValue AddrNode =
     95           DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(OffsetScaled * 16), dl);
     96       SDValue St = DAG.getMemIntrinsicNode(
     97           OpCode1, dl, DAG.getVTList(MVT::Other),
     98           {Chain, TagSrc, AddrNode},
     99           MVT::v2i64,
    100           MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16));
    101       OffsetScaled += 1;
    102       OutChains.push_back(St);
    103     }
    104   }
    105 
    106   SDValue Res = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
    107   return Res;
    108 }
    109 
    110 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForSetTag(
    111     SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Addr,
    112     SDValue Size, MachinePointerInfo DstPtrInfo, bool ZeroData) const {
    113   uint64_t ObjSize = cast<ConstantSDNode>(Size)->getZExtValue();
    114   assert(ObjSize % 16 == 0);
    115 
    116   MachineFunction &MF = DAG.getMachineFunction();
    117   MachineMemOperand *BaseMemOperand = MF.getMachineMemOperand(
    118       DstPtrInfo, MachineMemOperand::MOStore, ObjSize, Align(16));
    119 
    120   bool UseSetTagRangeLoop =
    121       kSetTagLoopThreshold >= 0 && (int)ObjSize >= kSetTagLoopThreshold;
    122   if (!UseSetTagRangeLoop)
    123     return EmitUnrolledSetTag(DAG, dl, Chain, Addr, ObjSize, BaseMemOperand,
    124                               ZeroData);
    125 
    126   const EVT ResTys[] = {MVT::i64, MVT::i64, MVT::Other};
    127 
    128   unsigned Opcode;
    129   if (Addr.getOpcode() == ISD::FrameIndex) {
    130     int FI = cast<FrameIndexSDNode>(Addr)->getIndex();
    131     Addr = DAG.getTargetFrameIndex(FI, MVT::i64);
    132     Opcode = ZeroData ? AArch64::STZGloop : AArch64::STGloop;
    133   } else {
    134     Opcode = ZeroData ? AArch64::STZGloop_wback : AArch64::STGloop_wback;
    135   }
    136   SDValue Ops[] = {DAG.getTargetConstant(ObjSize, dl, MVT::i64), Addr, Chain};
    137   SDNode *St = DAG.getMachineNode(Opcode, dl, ResTys, Ops);
    138 
    139   DAG.setNodeMemRefs(cast<MachineSDNode>(St), {BaseMemOperand});
    140   return SDValue(St, 2);
    141 }
    142