Author: Cullen Rhodes Date: 2026-06-08T08:18:46+01:00 New Revision: b7afecbf9bd698d2d5ce1546f54e98b0bdb68c49
URL: https://github.com/llvm/llvm-project/commit/b7afecbf9bd698d2d5ce1546f54e98b0bdb68c49 DIFF: https://github.com/llvm/llvm-project/commit/b7afecbf9bd698d2d5ce1546f54e98b0bdb68c49.diff LOG: Revert "[GlobalISel] Add a shared matcher for memcpy-family instructions (NFC…" This reverts commit 57d94c90ec4d69d8d5a6a755ad40155b01531ed1. Added: Modified: llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h llvm/include/llvm/CodeGen/GlobalISel/Utils.h llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp llvm/lib/CodeGen/GlobalISel/Utils.cpp Removed: ################################################################################ diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h index c9cd1099d3e9e..47d7da7a0eb52 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h @@ -305,16 +305,13 @@ class LegalizerHelper { // Memcpy family legalization helpers. LegalizeResult lowerMemset(MachineInstr &MI, Register Dst, Register Val, uint64_t KnownLen, Align Alignment, - bool IsVolatile, bool DstAlignCanChange, - ArrayRef<LLT> MemOps); + bool IsVolatile); LegalizeResult lowerMemcpy(MachineInstr &MI, Register Dst, Register Src, - uint64_t KnownLen, Align DstAlign, Align SrcAlign, - bool IsVolatile, bool DstAlignCanChange, - ArrayRef<LLT> MemOps); + uint64_t KnownLen, uint64_t Limit, Align DstAlign, + Align SrcAlign, bool IsVolatile); LegalizeResult lowerMemmove(MachineInstr &MI, Register Dst, Register Src, uint64_t KnownLen, Align DstAlign, Align SrcAlign, - bool IsVolatile, bool DstAlignCanChange, - ArrayRef<LLT> MemOps); + bool IsVolatile); // Implements floating-point environment read/write via library function call. LegalizeResult createGetStateLibcall(MachineInstr &MI, @@ -573,12 +570,6 @@ class LegalizerHelper { LLVM_ABI LegalizeResult lowerAbsDiffToMinMax(MachineInstr &MI); LLVM_ABI LegalizeResult lowerFAbs(MachineInstr &MI); LLVM_ABI LegalizeResult lowerVectorReduction(MachineInstr &MI); - LLVM_ABI LegalizeResult lowerMemCpyFamily(MachineInstr &MI, Register Dst, - Register Src, uint64_t KnownLen, - Align DstAlign, Align SrcAlign, - bool IsVolatile, - bool DstAlignCanChange, - ArrayRef<LLT> MemOps); LLVM_ABI LegalizeResult lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen = 0); LLVM_ABI LegalizeResult lowerVAArg(MachineInstr &MI); diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h index cfb7f376ff8e8..4689cc4df78b3 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h @@ -25,7 +25,6 @@ #include "llvm/Support/Compiler.h" #include <cstdint> -#include <vector> namespace llvm { @@ -204,20 +203,6 @@ LLVM_ABI std::optional<ValueAndVReg> getAnyConstantVRegValWithLookThrough( Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs = true, bool LookThroughAnyExt = false); -using MemCpyFamilyLoweringInfo = - std::tuple<Register, Register, uint64_t, Align, Align, bool, bool, - std::vector<LLT>>; - -/// Matcher for memcpy-like instructions. For non-zero lengths, \p MemOps -/// contains the load/store types to emit. -LLVM_ABI bool canLowerMemCpyFamily(const MachineInstr &MI, - const MachineRegisterInfo &MRI, - unsigned MaxLen, Register &Dst, - Register &Src, uint64_t &KnownLen, - Align &DstAlign, Align &SrcAlign, - bool &IsVolatile, bool &DstAlignCanChange, - std::vector<LLT> &MemOps); - struct FPValueAndVReg { APFloat Value; Register VReg; diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp index 960b93c9a1369..afe93f617a119 100644 --- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -10700,6 +10700,82 @@ LegalizerHelper::LegalizeResult LegalizerHelper::lowerMulfix(MachineInstr &MI) { return Legalized; } +static bool shouldLowerMemFuncForSize(const MachineFunction &MF) { + // On Darwin, -Os means optimize for size without hurting performance, so + // only really optimize for size when -Oz (MinSize) is used. + if (MF.getTarget().getTargetTriple().isOSDarwin()) + return MF.getFunction().hasMinSize(); + return MF.getFunction().hasOptSize(); +} + +// Returns a list of types to use for memory op lowering in MemOps. A partial +// port of findOptimalMemOpLowering in TargetLowering. +static bool findGISelOptimalMemOpLowering(std::vector<LLT> &MemOps, + unsigned Limit, const MemOp &Op, + unsigned DstAS, unsigned SrcAS, + const AttributeList &FuncAttributes, + const TargetLowering &TLI) { + if (Op.isMemcpyWithFixedDstAlign() && Op.getSrcAlign() < Op.getDstAlign()) + return false; + + LLT Ty = TLI.getOptimalMemOpLLT(Op, FuncAttributes); + + if (Ty == LLT()) { + // Use the largest scalar type whose alignment constraints are satisfied. + // We only need to check DstAlign here as SrcAlign is always greater or + // equal to DstAlign (or zero). + Ty = LLT::integer(64); + if (Op.isFixedDstAlign()) + while (Op.getDstAlign() < Ty.getSizeInBytes() && + !TLI.allowsMisalignedMemoryAccesses(Ty, DstAS, Op.getDstAlign())) + Ty = LLT::integer(Ty.getSizeInBytes()); + assert(Ty.getSizeInBits() > 0 && "Could not find valid type"); + // FIXME: check for the largest legal type we can load/store to. + } + + unsigned NumMemOps = 0; + uint64_t Size = Op.size(); + while (Size) { + unsigned TySize = Ty.getSizeInBytes(); + while (TySize > Size) { + // For now, only use non-vector load / store's for the left-over pieces. + LLT NewTy = Ty; + // FIXME: check for mem op safety and legality of the types. Not all of + // SDAGisms map cleanly to GISel concepts. + if (NewTy.isVector()) + NewTy = + NewTy.getSizeInBits() > 64 ? LLT::integer(64) : LLT::integer(32); + NewTy = LLT::integer(llvm::bit_floor(NewTy.getSizeInBits() - 1)); + unsigned NewTySize = NewTy.getSizeInBytes(); + assert(NewTySize > 0 && "Could not find appropriate type"); + + // If the new LLT cannot cover all of the remaining bits, then consider + // issuing a (or a pair of) unaligned and overlapping load / store. + unsigned Fast; + // Need to get a VT equivalent for allowMisalignedMemoryAccesses(). + MVT VT = getMVTForLLT(Ty); + if (NumMemOps && Op.allowOverlap() && NewTySize < Size && + TLI.allowsMisalignedMemoryAccesses( + VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1), + MachineMemOperand::MONone, &Fast) && + Fast) + TySize = Size; + else { + Ty = NewTy; + TySize = NewTySize; + } + } + + if (++NumMemOps > Limit) + return false; + + MemOps.push_back(Ty); + Size -= TySize; + } + + return true; +} + // Get a vectorized representation of the memset value operand, GISel edition. static Register getMemsetValue(Register Val, LLT Ty, MachineIRBuilder &MIB) { MachineRegisterInfo &MRI = *MIB.getMRI(); @@ -10736,19 +10812,39 @@ static Register getMemsetValue(Register Val, LLT Ty, MachineIRBuilder &MIB) { LegalizerHelper::LegalizeResult LegalizerHelper::lowerMemset(MachineInstr &MI, Register Dst, Register Val, uint64_t KnownLen, Align Alignment, - bool IsVolatile, bool DstAlignCanChange, - ArrayRef<LLT> MemOps) { + bool IsVolatile) { auto &MF = *MI.getParent()->getParent(); const auto &TLI = *MF.getSubtarget().getTargetLowering(); auto &DL = MF.getDataLayout(); LLVMContext &C = MF.getFunction().getContext(); assert(KnownLen != 0 && "Have a zero length memset length!"); - assert(!MemOps.empty() && "Expected at least one memory op"); + bool DstAlignCanChange = false; MachineFrameInfo &MFI = MF.getFrameInfo(); + bool OptSize = shouldLowerMemFuncForSize(MF); + MachineInstr *FIDef = getOpcodeDef(TargetOpcode::G_FRAME_INDEX, Dst, MRI); + if (FIDef && !MFI.isFixedObjectIndex(FIDef->getOperand(1).getIndex())) + DstAlignCanChange = true; + + unsigned Limit = TLI.getMaxStoresPerMemset(OptSize); + std::vector<LLT> MemOps; + const auto &DstMMO = **MI.memoperands_begin(); + MachinePointerInfo DstPtrInfo = DstMMO.getPointerInfo(); + + auto ValVRegAndVal = getIConstantVRegValWithLookThrough(Val, MRI); + bool IsZeroVal = ValVRegAndVal && ValVRegAndVal->Value == 0; + + if (!findGISelOptimalMemOpLowering(MemOps, Limit, + MemOp::Set(KnownLen, DstAlignCanChange, + Alignment, + /*IsZeroMemset=*/IsZeroVal, + /*IsVolatile=*/IsVolatile), + DstPtrInfo.getAddrSpace(), ~0u, + MF.getFunction().getAttributes(), TLI)) + return UnableToLegalize; if (DstAlignCanChange) { // Get an estimate of the type from the LLT. @@ -10828,26 +10924,41 @@ LegalizerHelper::lowerMemset(MachineInstr &MI, Register Dst, Register Val, LegalizerHelper::LegalizeResult LegalizerHelper::lowerMemcpy(MachineInstr &MI, Register Dst, Register Src, - uint64_t KnownLen, Align DstAlign, Align SrcAlign, - bool IsVolatile, bool DstAlignCanChange, - ArrayRef<LLT> MemOps) { + uint64_t KnownLen, uint64_t Limit, Align DstAlign, + Align SrcAlign, bool IsVolatile) { auto &MF = *MI.getParent()->getParent(); + const auto &TLI = *MF.getSubtarget().getTargetLowering(); auto &DL = MF.getDataLayout(); LLVMContext &C = MF.getFunction().getContext(); assert(KnownLen != 0 && "Have a zero length memcpy length!"); - assert(!MemOps.empty() && "Expected at least one memory op"); + bool DstAlignCanChange = false; MachineFrameInfo &MFI = MF.getFrameInfo(); Align Alignment = std::min(DstAlign, SrcAlign); + MachineInstr *FIDef = getOpcodeDef(TargetOpcode::G_FRAME_INDEX, Dst, MRI); + if (FIDef && !MFI.isFixedObjectIndex(FIDef->getOperand(1).getIndex())) + DstAlignCanChange = true; // FIXME: infer better src pointer alignment like SelectionDAG does here. // FIXME: also use the equivalent of isMemSrcFromConstant and alwaysinlining // if the memcpy is in a tail call position. + std::vector<LLT> MemOps; + const auto &DstMMO = **MI.memoperands_begin(); const auto &SrcMMO = **std::next(MI.memoperands_begin()); + MachinePointerInfo DstPtrInfo = DstMMO.getPointerInfo(); + MachinePointerInfo SrcPtrInfo = SrcMMO.getPointerInfo(); + + if (!findGISelOptimalMemOpLowering( + MemOps, Limit, + MemOp::Copy(KnownLen, DstAlignCanChange, Alignment, SrcAlign, + IsVolatile), + DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(), + MF.getFunction().getAttributes(), TLI)) + return UnableToLegalize; if (DstAlignCanChange) { // Get an estimate of the type from the LLT. @@ -10922,20 +11033,41 @@ LegalizerHelper::lowerMemcpy(MachineInstr &MI, Register Dst, Register Src, LegalizerHelper::LegalizeResult LegalizerHelper::lowerMemmove(MachineInstr &MI, Register Dst, Register Src, uint64_t KnownLen, Align DstAlign, Align SrcAlign, - bool IsVolatile, bool DstAlignCanChange, - ArrayRef<LLT> MemOps) { + bool IsVolatile) { auto &MF = *MI.getParent()->getParent(); + const auto &TLI = *MF.getSubtarget().getTargetLowering(); auto &DL = MF.getDataLayout(); LLVMContext &C = MF.getFunction().getContext(); assert(KnownLen != 0 && "Have a zero length memmove length!"); - assert(!MemOps.empty() && "Expected at least one memory op"); + bool DstAlignCanChange = false; MachineFrameInfo &MFI = MF.getFrameInfo(); + bool OptSize = shouldLowerMemFuncForSize(MF); Align Alignment = std::min(DstAlign, SrcAlign); + MachineInstr *FIDef = getOpcodeDef(TargetOpcode::G_FRAME_INDEX, Dst, MRI); + if (FIDef && !MFI.isFixedObjectIndex(FIDef->getOperand(1).getIndex())) + DstAlignCanChange = true; + + unsigned Limit = TLI.getMaxStoresPerMemmove(OptSize); + std::vector<LLT> MemOps; + const auto &DstMMO = **MI.memoperands_begin(); const auto &SrcMMO = **std::next(MI.memoperands_begin()); + MachinePointerInfo DstPtrInfo = DstMMO.getPointerInfo(); + MachinePointerInfo SrcPtrInfo = SrcMMO.getPointerInfo(); + + // FIXME: SelectionDAG always passes false for 'AllowOverlap', apparently due + // to a bug in it's findOptimalMemOpLowering implementation. For now do the + // same thing here. + if (!findGISelOptimalMemOpLowering( + MemOps, Limit, + MemOp::Copy(KnownLen, DstAlignCanChange, Alignment, SrcAlign, + /*IsVolatile*/ true), + DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(), + MF.getFunction().getAttributes(), TLI)) + return UnableToLegalize; if (DstAlignCanChange) { // Get an estimate of the type from the LLT. @@ -11004,44 +11136,61 @@ LegalizerHelper::lowerMemmove(MachineInstr &MI, Register Dst, Register Src, return Legalized; } -LegalizerHelper::LegalizeResult LegalizerHelper::lowerMemCpyFamily( - MachineInstr &MI, Register Dst, Register Src, uint64_t KnownLen, - Align DstAlign, Align SrcAlign, bool IsVolatile, bool DstAlignCanChange, - ArrayRef<LLT> MemOps) { +LegalizerHelper::LegalizeResult +LegalizerHelper::lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen) { const unsigned Opc = MI.getOpcode(); + // This combine is fairly complex so it's not written with a separate + // matcher function. assert((Opc == TargetOpcode::G_MEMCPY || Opc == TargetOpcode::G_MEMCPY_INLINE || Opc == TargetOpcode::G_MEMMOVE || Opc == TargetOpcode::G_MEMSET) && "Expected memcpy like instruction"); + auto MMOIt = MI.memoperands_begin(); + const MachineMemOperand *MemOp = *MMOIt; + + Align DstAlign = MemOp->getBaseAlign(); + Align SrcAlign; + auto [Dst, Src, Len] = MI.getFirst3Regs(); + + if (Opc != TargetOpcode::G_MEMSET) { + assert(MMOIt != MI.memoperands_end() && "Expected a second MMO on MI"); + MemOp = *(++MMOIt); + SrcAlign = MemOp->getBaseAlign(); + } + + // See if this is a constant length copy + auto LenVRegAndVal = getIConstantVRegValWithLookThrough(Len, MRI); + if (!LenVRegAndVal) { + // FIXME: support dynamically sized G_MEMCPY_INLINE + assert(Opc != TargetOpcode::G_MEMCPY_INLINE && + "inline memcpy with dynamic size is not yet supported"); + return UnableToLegalize; + } + uint64_t KnownLen = LenVRegAndVal->Value.getZExtValue(); + if (KnownLen == 0) { MI.eraseFromParent(); return Legalized; } + if (Opc != TargetOpcode::G_MEMCPY_INLINE && MaxLen && KnownLen > MaxLen) + return UnableToLegalize; + + bool IsVolatile = MemOp->isVolatile(); if (Opc == TargetOpcode::G_MEMCPY || Opc == TargetOpcode::G_MEMCPY_INLINE) { - return lowerMemcpy(MI, Dst, Src, KnownLen, DstAlign, SrcAlign, IsVolatile, - DstAlignCanChange, MemOps); + auto &MF = *MI.getParent()->getParent(); + const auto &TLI = *MF.getSubtarget().getTargetLowering(); + bool OptSize = shouldLowerMemFuncForSize(MF); + uint64_t Limit = Opc == TargetOpcode::G_MEMCPY_INLINE + ? std::numeric_limits<uint64_t>::max() + : TLI.getMaxStoresPerMemcpy(OptSize); + return lowerMemcpy(MI, Dst, Src, KnownLen, Limit, DstAlign, SrcAlign, + IsVolatile); } if (Opc == TargetOpcode::G_MEMMOVE) - return lowerMemmove(MI, Dst, Src, KnownLen, DstAlign, SrcAlign, IsVolatile, - DstAlignCanChange, MemOps); + return lowerMemmove(MI, Dst, Src, KnownLen, DstAlign, SrcAlign, IsVolatile); if (Opc == TargetOpcode::G_MEMSET) - return lowerMemset(MI, Dst, Src, KnownLen, DstAlign, IsVolatile, - DstAlignCanChange, MemOps); + return lowerMemset(MI, Dst, Src, KnownLen, DstAlign, IsVolatile); return UnableToLegalize; } - -LegalizerHelper::LegalizeResult -LegalizerHelper::lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen) { - Register Dst, Src; - uint64_t KnownLen; - Align DstAlign, SrcAlign; - bool IsVolatile, DstAlignCanChange; - std::vector<LLT> MemOps; - if (!canLowerMemCpyFamily(MI, MRI, MaxLen, Dst, Src, KnownLen, DstAlign, - SrcAlign, IsVolatile, DstAlignCanChange, MemOps)) - return UnableToLegalize; - return lowerMemCpyFamily(MI, Dst, Src, KnownLen, DstAlign, SrcAlign, - IsVolatile, DstAlignCanChange, MemOps); -} diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp index 673ebd33f56aa..7ae093282c49d 100644 --- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp +++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp @@ -20,7 +20,6 @@ #include "llvm/CodeGen/GlobalISel/LostDebugLocObserver.h" #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h" #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" -#include "llvm/CodeGen/LowLevelTypeUtils.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h" @@ -37,10 +36,8 @@ #include "llvm/Support/UndefPoison.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Transforms/Utils/SizeOpts.h" -#include <limits> #include <numeric> #include <optional> -#include <tuple> #define DEBUG_TYPE "globalisel-utils" @@ -2081,177 +2078,3 @@ llvm::GFConstant::getConstant(Register Const, const MachineRegisterInfo &MRI) { return GFConstant(MayBeConstant->Value, GFConstantKind::Scalar); } - -// Returns a list of types to use for memory op lowering in MemOps. A partial -// port of findOptimalMemOpLowering in TargetLowering. -static bool findGISelOptimalMemOpLowering(std::vector<LLT> &MemOps, - unsigned Limit, const MemOp &Op, - unsigned DstAS, unsigned SrcAS, - const AttributeList &FuncAttributes, - const TargetLowering &TLI) { - if (Op.isMemcpyWithFixedDstAlign() && Op.getSrcAlign() < Op.getDstAlign()) - return false; - - LLT Ty = TLI.getOptimalMemOpLLT(Op, FuncAttributes); - - if (Ty == LLT()) { - // Use the largest scalar type whose alignment constraints are satisfied. - // We only need to check DstAlign here as SrcAlign is always greater or - // equal to DstAlign (or zero). - Ty = LLT::integer(64); - if (Op.isFixedDstAlign()) - while (Op.getDstAlign() < Ty.getSizeInBytes() && - !TLI.allowsMisalignedMemoryAccesses(Ty, DstAS, Op.getDstAlign())) - Ty = LLT::integer(Ty.getSizeInBytes()); - assert(Ty.getSizeInBits() > 0 && "Could not find valid type"); - // FIXME: check for the largest legal type we can load/store to. - } - - unsigned NumMemOps = 0; - uint64_t Size = Op.size(); - while (Size) { - unsigned TySize = Ty.getSizeInBytes(); - while (TySize > Size) { - // For now, only use non-vector load / store's for the left-over pieces. - LLT NewTy = Ty; - // FIXME: check for mem op safety and legality of the types. Not all of - // SDAGisms map cleanly to GISel concepts. - if (NewTy.isVector()) - NewTy = - NewTy.getSizeInBits() > 64 ? LLT::integer(64) : LLT::integer(32); - NewTy = LLT::integer(llvm::bit_floor(NewTy.getSizeInBits() - 1)); - unsigned NewTySize = NewTy.getSizeInBytes(); - assert(NewTySize > 0 && "Could not find appropriate type"); - - // If the new LLT cannot cover all of the remaining bits, then consider - // issuing a (or a pair of) unaligned and overlapping load / store. - unsigned Fast; - // Need to get a VT equivalent for allowMisalignedMemoryAccesses(). - MVT VT = getMVTForLLT(Ty); - if (NumMemOps && Op.allowOverlap() && NewTySize < Size && - TLI.allowsMisalignedMemoryAccesses( - VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1), - MachineMemOperand::MONone, &Fast) && - Fast) - TySize = Size; - else { - Ty = NewTy; - TySize = NewTySize; - } - } - - if (++NumMemOps > Limit) - return false; - - MemOps.push_back(Ty); - Size -= TySize; - } - - return true; -} - -bool llvm::canLowerMemCpyFamily(const MachineInstr &MI, - const MachineRegisterInfo &MRI, unsigned MaxLen, - Register &Dst, Register &Src, - uint64_t &KnownLen, Align &DstAlign, - Align &SrcAlign, bool &IsVolatile, - bool &DstAlignCanChange, - std::vector<LLT> &MemOps) { - const unsigned Opc = MI.getOpcode(); - assert((Opc == TargetOpcode::G_MEMCPY || - Opc == TargetOpcode::G_MEMCPY_INLINE || - Opc == TargetOpcode::G_MEMMOVE || Opc == TargetOpcode::G_MEMSET) && - "Expected memcpy like instruction"); - - auto MMOIt = MI.memoperands_begin(); - const MachineMemOperand *MemOp = *MMOIt; - - DstAlign = MemOp->getBaseAlign(); - Register Len; - std::tie(Dst, Src, Len) = MI.getFirst3Regs(); - - if (Opc != TargetOpcode::G_MEMSET) { - assert(MMOIt != MI.memoperands_end() && "Expected a second MMO on MI"); - MemOp = *(++MMOIt); - SrcAlign = MemOp->getBaseAlign(); - } - - // See if this is a constant length copy. - auto LenVRegAndVal = getIConstantVRegValWithLookThrough(Len, MRI); - if (!LenVRegAndVal) { - // FIXME: support dynamically sized G_MEMCPY_INLINE - assert(Opc != TargetOpcode::G_MEMCPY_INLINE && - "inline memcpy with dynamic size is not yet supported"); - return false; - } - - KnownLen = LenVRegAndVal->Value.getZExtValue(); - DstAlignCanChange = false; - - if (KnownLen == 0) - return true; - - if (Opc != TargetOpcode::G_MEMCPY_INLINE && MaxLen && KnownLen > MaxLen) - return false; - - IsVolatile = MemOp->isVolatile(); - const MachineFunction &MF = *MI.getParent()->getParent(); - const auto &TLI = *MF.getSubtarget().getTargetLowering(); - // On Darwin, -Os means optimize for size without hurting performance, so - // only really optimize for size when -Oz (MinSize) is used. - bool OptSize = MF.getTarget().getTargetTriple().isOSDarwin() - ? MF.getFunction().hasMinSize() - : MF.getFunction().hasOptSize(); - - const MachineFrameInfo &MFI = MF.getFrameInfo(); - MachineInstr *FIDef = getOpcodeDef(TargetOpcode::G_FRAME_INDEX, Dst, MRI); - if (FIDef && !MFI.isFixedObjectIndex(FIDef->getOperand(1).getIndex())) - DstAlignCanChange = true; - - const auto &DstMMO = **MI.memoperands_begin(); - MachinePointerInfo DstPtrInfo = DstMMO.getPointerInfo(); - - switch (Opc) { - case TargetOpcode::G_MEMCPY_INLINE: - case TargetOpcode::G_MEMCPY: { - const auto &SrcMMO = **std::next(MI.memoperands_begin()); - MachinePointerInfo SrcPtrInfo = SrcMMO.getPointerInfo(); - uint64_t Limit = Opc == TargetOpcode::G_MEMCPY_INLINE - ? std::numeric_limits<uint64_t>::max() - : TLI.getMaxStoresPerMemcpy(OptSize); - return findGISelOptimalMemOpLowering( - MemOps, Limit, - MemOp::Copy(KnownLen, DstAlignCanChange, std::min(DstAlign, SrcAlign), - SrcAlign, IsVolatile), - DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(), - MF.getFunction().getAttributes(), TLI); - } - case TargetOpcode::G_MEMMOVE: { - const auto &SrcMMO = **std::next(MI.memoperands_begin()); - MachinePointerInfo SrcPtrInfo = SrcMMO.getPointerInfo(); - unsigned Limit = TLI.getMaxStoresPerMemmove(OptSize); - // FIXME: SelectionDAG always passes false for 'AllowOverlap', apparently - // due to a bug in it's findOptimalMemOpLowering implementation. For now do - // the same thing here. - return findGISelOptimalMemOpLowering( - MemOps, Limit, - MemOp::Copy(KnownLen, DstAlignCanChange, std::min(DstAlign, SrcAlign), - SrcAlign, /*IsVolatile=*/true), - DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(), - MF.getFunction().getAttributes(), TLI); - } - case TargetOpcode::G_MEMSET: { - unsigned Limit = TLI.getMaxStoresPerMemset(OptSize); - auto ValVRegAndVal = getIConstantVRegValWithLookThrough(Src, MRI); - bool IsZeroVal = ValVRegAndVal && ValVRegAndVal->Value == 0; - return findGISelOptimalMemOpLowering( - MemOps, Limit, - MemOp::Set(KnownLen, DstAlignCanChange, DstAlign, - /*IsZeroMemset=*/IsZeroVal, - /*IsVolatile=*/IsVolatile), - DstPtrInfo.getAddrSpace(), ~0u, MF.getFunction().getAttributes(), TLI); - } - default: - llvm_unreachable("Unexpected memcpy-family opcode"); - } -} _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
