https://github.com/jjppp created https://github.com/llvm/llvm-project/pull/201623
Fixes #201448 This PR fixes downcast failures in lowering `__builtin_prefetch` for immediate arguments. The code currently assumes that the immediate operands are `ConstantInt`s. This is not always true as poison values may come from UB expressions (e.g., `__builtin_prefetch(0, 2 >> 32)`). This would cause the downcast `cast<ConstantInt>` to fail. Since the source program already contains UB, this PR avoids downcast failure by replacing poisonous values with a legal default value `0` in CodeGen. A regression test is also added to cover the poison immediate argument case. >From 5e77e5fdad4ead4d908a3640117d0fdbaa73cd77 Mon Sep 17 00:00:00 2001 From: jpwang <[email protected]> Date: Fri, 5 Jun 2026 00:16:33 +0800 Subject: [PATCH] [Clang][CodeGen] Avoid failing downcast when __builtin_prefech's immarg is poison --- clang/test/CodeGen/prefetch-poison-rw.c | 6 + .../SelectionDAG/SelectionDAGBuilder.cpp | 965 ++++++++++-------- 2 files changed, 517 insertions(+), 454 deletions(-) create mode 100644 clang/test/CodeGen/prefetch-poison-rw.c diff --git a/clang/test/CodeGen/prefetch-poison-rw.c b/clang/test/CodeGen/prefetch-poison-rw.c new file mode 100644 index 0000000000000..c13d4bb246d54 --- /dev/null +++ b/clang/test/CodeGen/prefetch-poison-rw.c @@ -0,0 +1,6 @@ +// RUN: %clang_cc1 -triple x86_64-pc-linux -emit-llvm %s -o - | FileCheck %s + +void test_poison_rw(void) { + __builtin_prefetch(0, 2 >> 32); + // CHECK: call void @llvm.prefetch.p0(ptr null, i32 0, i32 3, i32 1) +} diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index eca5bb1598ae0..ff55e3862610a 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -190,11 +190,12 @@ getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, // Assemble the power of 2 part. unsigned RoundParts = llvm::bit_floor(NumParts); unsigned RoundBits = PartBits * RoundParts; - EVT RoundVT = RoundBits == ValueBits ? - ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits); + EVT RoundVT = RoundBits == ValueBits + ? ValueVT + : EVT::getIntegerVT(*DAG.getContext(), RoundBits); SDValue Lo, Hi; - EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2); + EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits / 2); if (RoundParts > 2) { Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2, PartVT, HalfVT, V, @@ -262,7 +263,7 @@ getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, ValueVT.bitsLT(PartEVT)) { // For an FP value in an integer part, we need to truncate to the right // width first. - PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits()); + PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits()); Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val); } @@ -277,8 +278,8 @@ getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, // indicate whether the truncated bits will always be // zero or sign-extension. if (AssertOp) - Val = DAG.getNode(*AssertOp, DL, PartEVT, Val, - DAG.getValueType(ValueVT)); + Val = + DAG.getNode(*AssertOp, DL, PartEVT, Val, DAG.getValueType(ValueVT)); return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val); } return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val); @@ -368,7 +369,7 @@ static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL, NumParts = NumRegs; // Silence a compiler warning. assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!"); assert(RegisterVT.getSizeInBits() == - Parts[0].getSimpleValueType().getSizeInBits() && + Parts[0].getSimpleValueType().getSizeInBits() && "Part type sizes don't match!"); // Assemble the parts into intermediate operands. @@ -451,21 +452,21 @@ static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL, return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); if (ValueVT.getVectorNumElements() != 1) { - // Certain ABIs require that vectors are passed as integers. For vectors - // are the same size, this is an obvious bitcast. - if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) { - return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); - } else if (ValueVT.bitsLT(PartEVT)) { - const uint64_t ValueSize = ValueVT.getFixedSizeInBits(); - EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize); - // Drop the extra bits. - Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val); - return DAG.getBitcast(ValueVT, Val); - } - - diagnosePossiblyInvalidConstraint( - *DAG.getContext(), V, "non-trivial scalar-to-vector conversion"); - return DAG.getUNDEF(ValueVT); + // Certain ABIs require that vectors are passed as integers. For vectors + // are the same size, this is an obvious bitcast. + if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) { + return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); + } else if (ValueVT.bitsLT(PartEVT)) { + const uint64_t ValueSize = ValueVT.getFixedSizeInBits(); + EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize); + // Drop the extra bits. + Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val); + return DAG.getBitcast(ValueVT, Val); + } + + diagnosePossiblyInvalidConstraint( + *DAG.getContext(), V, "non-trivial scalar-to-vector conversion"); + return DAG.getUNDEF(ValueVT); } // Handle cases such as i8 -> <1 x i1> @@ -542,12 +543,11 @@ getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, if (ValueVT.isFloatingPoint()) { // FP values need to be bitcast, then extended if they are being put // into a larger container. - ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits()); + ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits()); Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); } assert((PartVT.isInteger() || PartVT == MVT::x86mmx) && - ValueVT.isInteger() && - "Unknown mismatch!"); + ValueVT.isInteger() && "Unknown mismatch!"); ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); Val = DAG.getNode(ExtendKind, DL, ValueVT, Val); if (PartVT == MVT::x86mmx) @@ -560,8 +560,7 @@ getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, } else if (NumParts * PartBits < ValueVT.getSizeInBits()) { // If the parts cover less bits than value has, truncate the value. assert((PartVT.isInteger() || PartVT == MVT::x86mmx) && - ValueVT.isInteger() && - "Unknown mismatch!"); + ValueVT.isInteger() && "Unknown mismatch!"); ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val); if (PartVT == MVT::x86mmx) @@ -592,8 +591,9 @@ getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned RoundParts = llvm::bit_floor(NumParts); unsigned RoundBits = RoundParts * PartBits; unsigned OddParts = NumParts - RoundParts; - SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val, - DAG.getShiftAmountConstant(RoundBits, ValueVT, DL)); + SDValue OddVal = + DAG.getNode(ISD::SRL, DL, ValueVT, Val, + DAG.getShiftAmountConstant(RoundBits, ValueVT, DL)); getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V, CallConv); @@ -609,22 +609,21 @@ getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, // The number of parts is a power of 2. Repeatedly bisect the value using // EXTRACT_ELEMENT. - Parts[0] = DAG.getNode(ISD::BITCAST, DL, - EVT::getIntegerVT(*DAG.getContext(), - ValueVT.getSizeInBits()), - Val); + Parts[0] = DAG.getNode( + ISD::BITCAST, DL, + EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits()), Val); for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) { for (unsigned i = 0; i < NumParts; i += StepSize) { unsigned ThisBits = StepSize * PartBits / 2; EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits); SDValue &Part0 = Parts[i]; - SDValue &Part1 = Parts[i+StepSize/2]; + SDValue &Part1 = Parts[i + StepSize / 2]; - Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, - ThisVT, Part0, DAG.getIntPtrConstant(1, DL)); - Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, - ThisVT, Part0, DAG.getIntPtrConstant(0, DL)); + Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, ThisVT, Part0, + DAG.getIntPtrConstant(1, DL)); + Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, ThisVT, Part0, + DAG.getIntPtrConstant(0, DL)); if (ThisBits == PartBits && ThisVT != PartVT) { Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0); @@ -915,9 +914,9 @@ SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG, for (unsigned i = 0; i != NumRegs; ++i) { SDValue P; if (!Glue) { - P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT); + P = DAG.getCopyFromReg(Chain, dl, Regs[Part + i], RegisterVT); } else { - P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Glue); + P = DAG.getCopyFromReg(Chain, dl, Regs[Part + i], RegisterVT, *Glue); *Glue = P.getValue(2); } @@ -930,7 +929,7 @@ SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG, continue; const FunctionLoweringInfo::LiveOutInfo *LOI = - FuncInfo.GetLiveOutRegInfo(Regs[Part+i]); + FuncInfo.GetLiveOutRegInfo(Regs[Part + i]); if (!LOI) continue; @@ -1027,7 +1026,7 @@ void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, // c3 = TokenFactor c1, c2 // ... // = op c3, ..., f2 - Chain = Chains[NumRegs-1]; + Chain = Chains[NumRegs - 1]; else Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); } @@ -1075,8 +1074,8 @@ void RegsForValue::AddInlineAsmOperands(InlineAsm::Kind Code, bool HasMatching, for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) { MVT RegisterVT = RegVTs[Value]; - unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value], - RegisterVT); + unsigned NumRegs = + TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value], RegisterVT); for (unsigned i = 0; i != NumRegs; ++i) { assert(Reg < Regs.size() && "Mismatch in # registers expected"); Register TheReg = Regs[Reg++]; @@ -1145,7 +1144,7 @@ SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) { for (; i != e; ++i) { assert(Pending[i].getNode()->getNumOperands() > 1); if (Pending[i].getNode()->getOperand(0) == Root) - break; // Don't add the root if we already indirectly depend on it. + break; // Don't add the root if we already indirectly depend on it. } if (i == e) @@ -1207,11 +1206,9 @@ SDValue SelectionDAGBuilder::getRoot() { // Chain up all pending constrained intrinsics together with all // pending loads, by simply appending them to PendingLoads and // then calling getMemoryRoot(). - PendingLoads.reserve(PendingLoads.size() + - PendingConstrainedFP.size() + + PendingLoads.reserve(PendingLoads.size() + PendingConstrainedFP.size() + PendingConstrainedFPStrict.size()); - PendingLoads.append(PendingConstrainedFP.begin(), - PendingConstrainedFP.end()); + PendingLoads.append(PendingConstrainedFP.begin(), PendingConstrainedFP.end()); PendingLoads.append(PendingConstrainedFPStrict.begin(), PendingConstrainedFPStrict.end()); PendingConstrainedFP.clear(); @@ -1425,10 +1422,13 @@ void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) { // Note: this doesn't use InstVisitor, because it has to work with // ConstantExpr's in addition to instructions. switch (Opcode) { - default: llvm_unreachable("Unknown instruction type encountered!"); + default: + llvm_unreachable("Unknown instruction type encountered!"); // Build the switch statement using the Instruction.def file. -#define HANDLE_INST(NUM, OPCODE, CLASS) \ - case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break; +#define HANDLE_INST(NUM, OPCODE, CLASS) \ + case Instruction::OPCODE: \ + visit##OPCODE((const CLASS &)I); \ + break; #include "llvm/IR/Instruction.def" } } @@ -1786,8 +1786,8 @@ SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) { DAG.getDataLayout(), InReg, Ty, std::nullopt); // This is not an ABI copy. SDValue Chain = DAG.getEntryNode(); - Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, - V); + Result = + RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V); resolveDanglingDebugInfo(V, Result); } @@ -1800,7 +1800,8 @@ SDValue SelectionDAGBuilder::getValue(const Value *V) { // to do this first, so that we don't create a CopyFromReg if we already // have a regular SDValue. SDValue &N = NodeMap[V]; - if (N.getNode()) return N; + if (N.getNode()) + return N; // If there's a virtual register allocated and initialized for this // value, use it. @@ -1903,7 +1904,8 @@ SDValue SelectionDAGBuilder::getValueImpl(const Value *V) { for (const Use &U : C->operands()) { SDNode *Val = getValue(U).getNode(); // If the operand is an empty aggregate, there are no values. - if (!Val) continue; + if (!Val) + continue; // Add each leaf value from the operand to the Constants list // to form a flattened list of all the values. for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i) @@ -1914,7 +1916,7 @@ SDValue SelectionDAGBuilder::getValueImpl(const Value *V) { } if (const ConstantDataSequential *CDS = - dyn_cast<ConstantDataSequential>(C)) { + dyn_cast<ConstantDataSequential>(C)) { SmallVector<SDValue, 4> Ops; for (uint64_t i = 0, e = CDS->getNumElements(); i != e; ++i) { SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode(); @@ -2118,7 +2120,7 @@ static void findUnwindDestinations( SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>> &UnwindDests) { EHPersonality Personality = - classifyEHPersonality(FuncInfo.Fn->getPersonalityFn()); + classifyEHPersonality(FuncInfo.Fn->getPersonalityFn()); bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX; bool IsCoreCLR = Personality == EHPersonality::CoreCLR; bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX; @@ -2245,8 +2247,7 @@ void SelectionDAGBuilder::visitRet(const ReturnInst &I) { commonAlignment(BaseAlign, Offsets[i])); } - Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), - MVT::Other, Chains); + Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, Chains); } else if (I.getNumOperands() != 0) { SmallVector<Type *, 4> Types; ComputeValueTypes(DL, I.getOperand(0)->getType(), Types); @@ -2341,7 +2342,7 @@ void SelectionDAGBuilder::visitRet(const ReturnInst &I) { bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg(); CallingConv::ID CallConv = - DAG.getMachineFunction().getFunction().getCallingConv(); + DAG.getMachineFunction().getFunction().getCallingConv(); Chain = DAG.getTargetLoweringInfo().LowerReturn( Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG); @@ -2374,17 +2375,19 @@ void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) { /// CopyTo/FromReg. void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) { // No need to export constants. - if (!isa<Instruction>(V) && !isa<Argument>(V)) return; + if (!isa<Instruction>(V) && !isa<Argument>(V)) + return; // Already exported? - if (FuncInfo.isExportedInst(V)) return; + if (FuncInfo.isExportedInst(V)) + return; Register Reg = FuncInfo.InitializeRegForValue(V); CopyValueToVirtualRegister(V, Reg); } -bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V, - const BasicBlock *FromBB) { +bool SelectionDAGBuilder::isExportableFromCurrentBlock( + const Value *V, const BasicBlock *FromBB) { // The operands of the setcc have to be in this block. We don't know // how to export them from some other block. if (const Instruction *VI = dyn_cast<Instruction>(V)) { @@ -2447,15 +2450,10 @@ static bool InBlock(const Value *V, const BasicBlock *BB) { /// EmitBranchForMergedCondition - Helper method for FindMergedConditions. /// This function emits a branch and is used at the leaves of an OR or an /// AND operator tree. -void -SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond, - MachineBasicBlock *TBB, - MachineBasicBlock *FBB, - MachineBasicBlock *CurBB, - MachineBasicBlock *SwitchBB, - BranchProbability TProb, - BranchProbability FProb, - bool InvertCond) { +void SelectionDAGBuilder::EmitBranchForMergedCondition( + const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, + MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB, + BranchProbability TProb, BranchProbability FProb, bool InvertCond) { const BasicBlock *BB = CurBB->getBasicBlock(); // If the leaf of the tree is a comparison, merge the condition into @@ -2494,8 +2492,8 @@ SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond, // Create a CaseBlock record representing this branch. ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ; - CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()), - nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb); + CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()), nullptr, TBB, + FBB, CurBB, getCurSDLoc(), TProb, FProb); SL->SwitchCases.push_back(CB); } @@ -2635,15 +2633,11 @@ bool SelectionDAGBuilder::shouldKeepJumpConditionsTogether( return true; } -void SelectionDAGBuilder::FindMergedConditions(const Value *Cond, - MachineBasicBlock *TBB, - MachineBasicBlock *FBB, - MachineBasicBlock *CurBB, - MachineBasicBlock *SwitchBB, - Instruction::BinaryOps Opc, - BranchProbability TProb, - BranchProbability FProb, - bool InvertCond) { +void SelectionDAGBuilder::FindMergedConditions( + const Value *Cond, MachineBasicBlock *TBB, MachineBasicBlock *FBB, + MachineBasicBlock *CurBB, MachineBasicBlock *SwitchBB, + Instruction::BinaryOps Opc, BranchProbability TProb, + BranchProbability FProb, bool InvertCond) { // Skip over not part of the tree and remember to invert op and operands at // next level. Value *NotCond; @@ -2682,8 +2676,8 @@ void SelectionDAGBuilder::FindMergedConditions(const Value *Cond, if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() || !InBlock(BOpOp0, CurBB->getBasicBlock()) || !InBlock(BOpOp1, CurBB->getBasicBlock())) { - EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB, - TProb, FProb, InvertCond); + EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB, TProb, FProb, + InvertCond); return; } @@ -2765,9 +2759,10 @@ void SelectionDAGBuilder::FindMergedConditions(const Value *Cond, /// If the set of cases should be emitted as a series of branches, return true. /// If we should emit this as a bunch of and/or'd together conditions, return /// false. -bool -SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) { - if (Cases.size() != 2) return true; +bool SelectionDAGBuilder::ShouldEmitAsBranches( + const std::vector<CaseBlock> &Cases) { + if (Cases.size() != 2) + return true; // If this is two comparisons of the same values or'd or and'd together, they // will get folded into a single comparison, so don't emit two blocks. @@ -2780,8 +2775,7 @@ SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) { // Handle: (X != null) | (Y != null) --> (X|Y) != 0 // Handle: (X == null) & (Y == null) --> (X|Y) == 0 - if (Cases[0].CmpRHS == Cases[1].CmpRHS && - Cases[0].CC == Cases[1].CC && + if (Cases[0].CmpRHS == Cases[1].CmpRHS && Cases[0].CC == Cases[1].CC && isa<Constant>(Cases[0].CmpRHS) && cast<Constant>(Cases[0].CmpRHS)->isNullValue()) { if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB) @@ -2948,8 +2942,8 @@ void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB, } else { assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now"); - const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue(); - const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue(); + const APInt &Low = cast<ConstantInt>(CB.CmpLHS)->getValue(); + const APInt &High = cast<ConstantInt>(CB.CmpRHS)->getValue(); SDValue CmpOp = getValue(CB.CmpMHS); EVT VT = CmpOp.getValueType(); @@ -2958,10 +2952,10 @@ void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB, Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT), ISD::SETLE); } else { - SDValue SUB = DAG.getNode(ISD::SUB, dl, - VT, CmpOp, DAG.getConstant(Low, dl, VT)); - Cond = DAG.getSetCC(dl, MVT::i1, SUB, - DAG.getConstant(High-Low, dl, VT), ISD::SETULE); + SDValue SUB = + DAG.getNode(ISD::SUB, dl, VT, CmpOp, DAG.getConstant(Low, dl, VT)); + Cond = DAG.getSetCC(dl, MVT::i1, SUB, DAG.getConstant(High - Low, dl, VT), + ISD::SETULE); } } @@ -3044,12 +3038,12 @@ void SelectionDAGBuilder::visitJumpTableHeader(SwitchCG::JumpTable &JT, // for the switch statement if the value being switched on exceeds the // largest case in the switch. SDValue CMP = DAG.getSetCC( - dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), - Sub.getValueType()), + dl, + TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), + Sub.getValueType()), Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT); - SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, - MVT::Other, CopyTo, CMP, + SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, MVT::Other, CopyTo, CMP, DAG.getBasicBlock(JT.Default)); // Avoid emitting unnecessary branches to the next block. @@ -3186,8 +3180,7 @@ void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD, SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, MVT::Other, getControlRoot(), Cmp, DAG.getBasicBlock(SPD.getFailureMBB())); // Otherwise branch to success MBB. - SDValue Br = DAG.getNode(ISD::BR, dl, - MVT::Other, BrCond, + SDValue Br = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond, DAG.getBasicBlock(SPD.getSuccessMBB())); DAG.setRoot(Br); @@ -3307,7 +3300,7 @@ void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B, B.Reg = FuncInfo.CreateReg(B.RegVT); SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub); - MachineBasicBlock* MBB = B.Cases[0].ThisBB; + MachineBasicBlock *MBB = B.Cases[0].ThisBB; if (!B.FallthroughUnreachable) addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb); @@ -3317,7 +3310,8 @@ void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B, SDValue Root = CopyTo; if (!B.FallthroughUnreachable) { // Conditional branch to the default block. - SDValue RangeCmp = DAG.getSetCC(dl, + SDValue RangeCmp = DAG.getSetCC( + dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), RangeSub.getValueType()), RangeSub, DAG.getConstant(B.Range, dl, RangeSub.getValueType()), @@ -3360,12 +3354,12 @@ void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB, ShiftOp, DAG.getConstant(llvm::countr_one(B.Mask), dl, VT), ISD::SETNE); } else { // Make desired shift - SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT, - DAG.getConstant(1, dl, VT), ShiftOp); + SDValue SwitchVal = + DAG.getNode(ISD::SHL, dl, VT, DAG.getConstant(1, dl, VT), ShiftOp); // Emit bit tests and jumps - SDValue AndOp = DAG.getNode(ISD::AND, dl, - VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT)); + SDValue AndOp = DAG.getNode(ISD::AND, dl, VT, SwitchVal, + DAG.getConstant(B.Mask, dl, VT)); Cmp = DAG.getSetCC( dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT), AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE); @@ -3380,14 +3374,13 @@ void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB, // and hence we need to normalize them to let the sum of them become one. SwitchBB->normalizeSuccProbs(); - SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl, - MVT::Other, getControlRoot(), + SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl, MVT::Other, getControlRoot(), Cmp, DAG.getBasicBlock(B.TargetBB)); // Avoid emitting unnecessary branches to the next block. if (NextMBB != NextBlock(SwitchBB)) - BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd, - DAG.getBasicBlock(NextMBB)); + BrAnd = + DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd, DAG.getBasicBlock(NextMBB)); DAG.setRoot(BrAnd); } @@ -3425,9 +3418,9 @@ void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) { case Intrinsic::seh_try_end: case Intrinsic::seh_scope_end: if (EHPadMBB) - // a block referenced by EH table - // so dtor-funclet not removed by opts - EHPadMBB->setMachineBlockAddressTaken(); + // a block referenced by EH table + // so dtor-funclet not removed by opts + EHPadMBB->setMachineBlockAddressTaken(); break; case Intrinsic::experimental_patchpoint_void: case Intrinsic::experimental_patchpoint: @@ -3577,8 +3570,7 @@ void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) { CallBrMBB->normalizeSuccProbs(); // Drop into default successor. - DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), - MVT::Other, getControlRoot(), + DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(), DAG.getBasicBlock(Return))); } @@ -3587,8 +3579,7 @@ void SelectionDAGBuilder::visitResume(const ResumeInst &RI) { } void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) { - assert(FuncInfo.MBB->isEHPad() && - "Call to landingpad not in landing pad!"); + assert(FuncInfo.MBB->isEHPad() && "Call to landingpad not in landing pad!"); // If there aren't registers to copy the values into (e.g., during SjLj // exceptions), then don't bother to create these DAG nodes. @@ -3629,8 +3620,8 @@ void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) { dl, ValueVTs[1]); // Merge into one. - SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl, - DAG.getVTList(ValueVTs), Ops); + SDValue Res = + DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Ops); setValue(&LP, Res); } @@ -3656,16 +3647,15 @@ void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) { BasicBlock *BB = I.getSuccessor(i); bool Inserted = Done.insert(BB).second; if (!Inserted) - continue; + continue; MachineBasicBlock *Succ = FuncInfo.getMBB(BB); addSuccessorWithProb(IndirectBrMBB, Succ); } IndirectBrMBB->normalizeSuccProbs(); - DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(), - MVT::Other, getControlRoot(), - getValue(I.getAddress()))); + DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(), MVT::Other, + getControlRoot(), getValue(I.getAddress()))); } void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) { @@ -3682,8 +3672,8 @@ void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) { Flags.copyFMF(*FPOp); SDValue Op = getValue(I.getOperand(0)); - SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(), - Op, Flags); + SDValue UnNodeValue = + DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(), Op, Flags); setValue(&I, UnNodeValue); } @@ -3702,8 +3692,8 @@ void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) { SDValue Op1 = getValue(I.getOperand(0)); SDValue Op2 = getValue(I.getOperand(1)); - SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), - Op1, Op2, Flags); + SDValue BinNodeValue = + DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2, Flags); setValue(&I, BinNodeValue); } @@ -3741,8 +3731,8 @@ void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) { Flags.setExact(exact); Flags.setNoSignedWrap(nsw); Flags.setNoUnsignedWrap(nuw); - SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2, - Flags); + SDValue Res = + DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2, Flags); setValue(&I, Res); } @@ -3807,9 +3797,8 @@ void SelectionDAGBuilder::visitFCmp(const FCmpInst &I) { // Check if the condition of the select has one use or two users that are both // selects with the same condition. static bool hasOnlySelectUsers(const Value *Cond) { - return llvm::all_of(Cond->users(), [](const Value *V) { - return isa<SelectInst>(V); - }); + return llvm::all_of(Cond->users(), + [](const Value *V) { return isa<SelectInst>(V); }); } void SelectionDAGBuilder::visitSelect(const User &I) { @@ -3817,12 +3806,13 @@ void SelectionDAGBuilder::visitSelect(const User &I) { ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(), ValueVTs); unsigned NumValues = ValueVTs.size(); - if (NumValues == 0) return; + if (NumValues == 0) + return; SmallVector<SDValue, 4> Values(NumValues); - SDValue Cond = getValue(I.getOperand(0)); - SDValue LHSVal = getValue(I.getOperand(1)); - SDValue RHSVal = getValue(I.getOperand(2)); + SDValue Cond = getValue(I.getOperand(0)); + SDValue LHSVal = getValue(I.getOperand(1)); + SDValue RHSVal = getValue(I.getOperand(2)); SmallVector<SDValue, 1> BaseOps(1, Cond); ISD::NodeType OpCode = Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT; @@ -3851,8 +3841,8 @@ void SelectionDAGBuilder::visitSelect(const User &I) { // If the vselect is legal, assume we want to leave this as a vector setcc + // vselect. Otherwise, if this is going to be scalarized, we want to see if // min/max is legal on the scalar type. - bool UseScalarMinMax = VT.isVector() && - !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT); + bool UseScalarMinMax = + VT.isVector() && !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT); // ValueTracking's select pattern matching does not account for -0.0, // so we can't lower to FMINIMUM/FMAXIMUM because those nodes specify that @@ -3861,17 +3851,27 @@ void SelectionDAGBuilder::visitSelect(const User &I) { auto SPR = matchSelectPattern(&I, LHS, RHS); ISD::NodeType Opc = ISD::DELETED_NODE; switch (SPR.Flavor) { - case SPF_UMAX: Opc = ISD::UMAX; break; - case SPF_UMIN: Opc = ISD::UMIN; break; - case SPF_SMAX: Opc = ISD::SMAX; break; - case SPF_SMIN: Opc = ISD::SMIN; break; + case SPF_UMAX: + Opc = ISD::UMAX; + break; + case SPF_UMIN: + Opc = ISD::UMIN; + break; + case SPF_SMAX: + Opc = ISD::SMAX; + break; + case SPF_SMIN: + Opc = ISD::SMIN; + break; case SPF_FMINNUM: if (!TLI.isProfitableToCombineMinNumMaxNum(VT)) break; switch (SPR.NaNBehavior) { - case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?"); - case SPNB_RETURNS_NAN: break; + case SPNB_NA: + llvm_unreachable("No NaN behavior for FP op?"); + case SPNB_RETURNS_NAN: + break; case SPNB_RETURNS_OTHER: Opc = ISD::FMINIMUMNUM; Flags.setNoSignedZeros(true); @@ -3889,8 +3889,10 @@ void SelectionDAGBuilder::visitSelect(const User &I) { break; switch (SPR.NaNBehavior) { - case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?"); - case SPNB_RETURNS_NAN: break; + case SPNB_NA: + llvm_unreachable("No NaN behavior for FP op?"); + case SPNB_RETURNS_NAN: + break; case SPNB_RETURNS_OTHER: Opc = ISD::FMAXIMUMNUM; Flags.setNoSignedZeros(true); @@ -3910,7 +3912,8 @@ void SelectionDAGBuilder::visitSelect(const User &I) { IsUnaryAbs = true; Opc = ISD::ABS; break; - default: break; + default: + break; } if (!IsUnaryAbs && Opc != ISD::DELETED_NODE && @@ -4112,17 +4115,16 @@ void SelectionDAGBuilder::visitBitCast(const User &I) { // BitCast assures us that source and destination are the same size so this is // either a BITCAST or a no-op. if (DestVT != N.getValueType()) - setValue(&I, DAG.getNode(ISD::BITCAST, dl, - DestVT, N)); // convert types. + setValue(&I, DAG.getNode(ISD::BITCAST, dl, DestVT, N)); // convert types. // Check if the original LLVM IR Operand was a ConstantInt, because getValue() // might fold any kind of constant expression to an integer constant and that // is not what we are looking for. Only recognize a bitcast of a genuine // constant integer as an opaque constant. - else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0))) + else if (ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0))) setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false, - /*isOpaque*/true)); + /*isOpaque*/ true)); else - setValue(&I, N); // noop cast. + setValue(&I, N); // noop cast. } void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) { @@ -4339,7 +4341,7 @@ void SelectionDAGBuilder::visitShuffleVector(const User &I) { // replacing the shuffle with extract and build vector. // to insert and build vector. EVT EltVT = VT.getVectorElementType(); - SmallVector<SDValue,8> Ops; + SmallVector<SDValue, 8> Ops; for (int Idx : Mask) { SDValue Res; @@ -4347,7 +4349,8 @@ void SelectionDAGBuilder::visitShuffleVector(const User &I) { Res = DAG.getUNDEF(EltVT); } else { SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2; - if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts; + if (Idx >= (int)SrcNumElts) + Idx -= SrcNumElts; Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src, DAG.getVectorIdxConstant(Idx, DL)); @@ -4390,19 +4393,20 @@ void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) { unsigned i = 0; // Copy the beginning value(s) from the original aggregate. for (; i != LinearIndex; ++i) - Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) : - SDValue(Agg.getNode(), Agg.getResNo() + i); + Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) + : SDValue(Agg.getNode(), Agg.getResNo() + i); // Copy values from the inserted value(s). if (NumValValues) { SDValue Val = getValue(Op1); for (; i != LinearIndex + NumValValues; ++i) - Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) : - SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex); + Values[i] = + FromUndef ? DAG.getUNDEF(AggValueVTs[i]) + : SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex); } // Copy remaining value(s) from the original aggregate. for (; i != NumAggValues; ++i) - Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) : - SDValue(Agg.getNode(), Agg.getResNo() + i); + Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) + : SDValue(Agg.getNode(), Agg.getResNo() + i); setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), DAG.getVTList(AggValueVTs), Values)); @@ -4435,9 +4439,9 @@ void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) { // Copy out the selected value(s). for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i) Values[i - LinearIndex] = - OutOfUndef ? - DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) : - SDValue(Agg.getNode(), Agg.getResNo() + i); + OutOfUndef + ? DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) + : SDValue(Agg.getNode(), Agg.getResNo() + i); setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), DAG.getVTList(ValValueVTs), Values)); @@ -4618,7 +4622,7 @@ void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) { // If this is a fixed sized alloca in the entry block of the function, // allocate it statically on the stack. if (FuncInfo.StaticAllocaMap.count(&I)) - return; // getValue will auto-populate this. + return; // getValue will auto-populate this. SDLoc dl = getCurSDLoc(); Type *Ty = I.getAllocatedType(); @@ -4806,8 +4810,8 @@ void SelectionDAGBuilder::visitLoad(const LoadInst &I) { PendingLoads.push_back(Chain); } - setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl, - DAG.getVTList(ValueVTs), Values)); + setValue(&I, + DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values)); } void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) { @@ -4837,8 +4841,7 @@ void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) { assert(DAG.getTargetLoweringInfo().supportSwiftError() && "call visitLoadFromSwiftError when backend supports swifterror"); - assert(!I.isVolatile() && - !I.hasMetadata(LLVMContext::MD_nontemporal) && + assert(!I.isVolatile() && !I.hasMetadata(LLVMContext::MD_nontemporal) && !I.hasMetadata(LLVMContext::MD_invariant_load) && "Support volatile, non temporal, invariant for load_from_swift_error"); @@ -4998,7 +5001,7 @@ void SelectionDAGBuilder::visitMaskedStore(const CallInst &I, static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index, SDValue &Scale, SelectionDAGBuilder *SDB, const BasicBlock *CurBB, uint64_t ElemSize) { - SelectionDAG& DAG = SDB->DAG; + SelectionDAG &DAG = SDB->DAG; const TargetLowering &TLI = DAG.getTargetLoweringInfo(); const DataLayout &DL = DAG.getDataLayout(); @@ -5085,7 +5088,7 @@ void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) { Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index); } - SDValue Ops[] = { getMemoryRoot(), Src0, Mask, Base, Index, Scale }; + SDValue Ops[] = {getMemoryRoot(), Src0, Mask, Base, Index, Scale}; SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl, Ops, MMO, ISD::SIGNED_SCALED, false); DAG.setRoot(Scatter); @@ -5184,7 +5187,7 @@ void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) { Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index); } - SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale }; + SDValue Ops[] = {Root, Src0, Mask, Base, Index, Scale}; SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl, Ops, MMO, ISD::SIGNED_SCALED, ISD::NON_EXTLOAD); @@ -5213,11 +5216,10 @@ void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) { MemVT.getStoreSize(), I.getAlign(), AAMDNodes(), nullptr, SSID, SuccessOrdering, FailureOrdering); - SDValue L = DAG.getAtomicCmpSwap(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, - dl, MemVT, VTs, InChain, - getValue(I.getPointerOperand()), - getValue(I.getCompareOperand()), - getValue(I.getNewValOperand()), MMO); + SDValue L = DAG.getAtomicCmpSwap( + ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, MemVT, VTs, InChain, + getValue(I.getPointerOperand()), getValue(I.getCompareOperand()), + getValue(I.getNewValOperand()), MMO); SDValue OutChain = L.getValue(2); @@ -5229,22 +5231,53 @@ void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) { SDLoc dl = getCurSDLoc(); ISD::NodeType NT; switch (I.getOperation()) { - default: llvm_unreachable("Unknown atomicrmw operation"); - case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break; - case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break; - case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break; - case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break; - case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break; - case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break; - case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break; - case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break; - case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break; - case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break; - case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break; - case AtomicRMWInst::FAdd: NT = ISD::ATOMIC_LOAD_FADD; break; - case AtomicRMWInst::FSub: NT = ISD::ATOMIC_LOAD_FSUB; break; - case AtomicRMWInst::FMax: NT = ISD::ATOMIC_LOAD_FMAX; break; - case AtomicRMWInst::FMin: NT = ISD::ATOMIC_LOAD_FMIN; break; + default: + llvm_unreachable("Unknown atomicrmw operation"); + case AtomicRMWInst::Xchg: + NT = ISD::ATOMIC_SWAP; + break; + case AtomicRMWInst::Add: + NT = ISD::ATOMIC_LOAD_ADD; + break; + case AtomicRMWInst::Sub: + NT = ISD::ATOMIC_LOAD_SUB; + break; + case AtomicRMWInst::And: + NT = ISD::ATOMIC_LOAD_AND; + break; + case AtomicRMWInst::Nand: + NT = ISD::ATOMIC_LOAD_NAND; + break; + case AtomicRMWInst::Or: + NT = ISD::ATOMIC_LOAD_OR; + break; + case AtomicRMWInst::Xor: + NT = ISD::ATOMIC_LOAD_XOR; + break; + case AtomicRMWInst::Max: + NT = ISD::ATOMIC_LOAD_MAX; + break; + case AtomicRMWInst::Min: + NT = ISD::ATOMIC_LOAD_MIN; + break; + case AtomicRMWInst::UMax: + NT = ISD::ATOMIC_LOAD_UMAX; + break; + case AtomicRMWInst::UMin: + NT = ISD::ATOMIC_LOAD_UMIN; + break; + case AtomicRMWInst::FAdd: + NT = ISD::ATOMIC_LOAD_FADD; + break; + case AtomicRMWInst::FSub: + NT = ISD::ATOMIC_LOAD_FSUB; + break; + case AtomicRMWInst::FMax: + NT = ISD::ATOMIC_LOAD_FMAX; + break; + case AtomicRMWInst::FMin: + NT = ISD::ATOMIC_LOAD_FMIN; + break; case AtomicRMWInst::FMaximum: NT = ISD::ATOMIC_LOAD_FMAXIMUM; break; @@ -5285,9 +5318,8 @@ void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) { I.getAlign(), AAMDNodes(), nullptr, SSID, Ordering); SDValue L = - DAG.getAtomic(NT, dl, MemVT, InChain, - getValue(I.getPointerOperand()), getValue(I.getValOperand()), - MMO); + DAG.getAtomic(NT, dl, MemVT, InChain, getValue(I.getPointerOperand()), + getValue(I.getValOperand()), MMO); SDValue OutChain = L.getValue(1); @@ -5402,7 +5434,7 @@ SmallVector<SDValue, 8> SelectionDAGBuilder::getTargetIntrinsicOperands( // Build the operand list. SmallVector<SDValue, 8> Ops; - if (HasChain) { // If this intrinsic has side-effects, chainify it. + if (HasChain) { // If this intrinsic has side-effects, chainify it. if (OnlyLoad) { // We don't need to serialize loads against other loads. Ops.push_back(DAG.getRoot()); @@ -5701,8 +5733,8 @@ static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl, /// limited-precision mode. static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags) { - if (Op.getValueType() == MVT::f32 && - LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { + if (Op.getValueType() == MVT::f32 && LimitFloatPrecision > 0 && + LimitFloatPrecision <= 18) { // Put the exponent in the right bit position for later addition to the // final result: @@ -5725,8 +5757,8 @@ static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags) { // TODO: What fast-math-flags should be set on the floating-point nodes? - if (Op.getValueType() == MVT::f32 && - LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { + if (Op.getValueType() == MVT::f32 && LimitFloatPrecision > 0 && + LimitFloatPrecision <= 18) { SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op); // Scale the exponent by log(2). @@ -5824,8 +5856,8 @@ static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags) { // TODO: What fast-math-flags should be set on the floating-point nodes? - if (Op.getValueType() == MVT::f32 && - LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { + if (Op.getValueType() == MVT::f32 && LimitFloatPrecision > 0 && + LimitFloatPrecision <= 18) { SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op); // Get the exponent. @@ -5921,8 +5953,8 @@ static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags) { // TODO: What fast-math-flags should be set on the floating-point nodes? - if (Op.getValueType() == MVT::f32 && - LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { + if (Op.getValueType() == MVT::f32 && LimitFloatPrecision > 0 && + LimitFloatPrecision <= 18) { SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op); // Scale the exponent by log10(2) [0.30102999f]. @@ -6009,8 +6041,8 @@ static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, /// limited-precision mode. static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, const TargetLowering &TLI, SDNodeFlags Flags) { - if (Op.getValueType() == MVT::f32 && - LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) + if (Op.getValueType() == MVT::f32 && LimitFloatPrecision > 0 && + LimitFloatPrecision <= 18) return getLimitedPrecisionExp2(Op, dl, DAG); // No special expansion. @@ -6099,9 +6131,9 @@ static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS, return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS); } -static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL, - SDValue LHS, SDValue RHS, SDValue Scale, - SelectionDAG &DAG, const TargetLowering &TLI) { +static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL, SDValue LHS, + SDValue RHS, SDValue Scale, SelectionDAG &DAG, + const TargetLowering &TLI) { EVT VT = LHS.getValueType(); bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT; bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT; @@ -6128,8 +6160,8 @@ static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL, if ((ScaleInt > 0 || (Saturating && Signed)) && (TLI.isTypeLegal(VT) || (VT.isVector() && TLI.isTypeLegal(VT.getVectorElementType())))) { - TargetLowering::LegalizeAction Action = TLI.getFixedPointOperationAction( - Opcode, VT, ScaleInt); + TargetLowering::LegalizeAction Action = + TLI.getFixedPointOperationAction(Opcode, VT, ScaleInt); if (Action != TargetLowering::Legal && Action != TargetLowering::Custom) { EVT PromVT; if (VT.isScalarInteger()) @@ -6253,8 +6285,8 @@ bool SelectionDAGBuilder::EmitFuncArgumentDbgValue( // we should only emit as ArgDbgValue if the Variable is an argument to the // current function, and the dbg.value intrinsic is found in the entry // block. - bool VariableIsFunctionInputArg = Variable->isParameter() && - !DL->getInlinedAt(); + bool VariableIsFunctionInputArg = + Variable->isParameter() && !DL->getInlinedAt(); bool IsInPrologue = SDNodeOrder == LowestSDNodeOrder; if (!IsInPrologue && !VariableIsFunctionInputArg) return false; @@ -6331,7 +6363,7 @@ bool SelectionDAGBuilder::EmitFuncArgumentDbgValue( SDValue LCandidate = peekThroughBitcasts(N); if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(LCandidate.getNode())) if (FrameIndexSDNode *FINode = - dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode())) + dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode())) Op = MachineOperand::CreateFI(FINode->getIndex()); } @@ -6382,8 +6414,8 @@ bool SelectionDAGBuilder::EmitFuncArgumentDbgValue( }; // Check if ValueMap has reg number. - DenseMap<const Value *, Register>::const_iterator - VMI = FuncInfo.ValueMap.find(V); + DenseMap<const Value *, Register>::const_iterator VMI = + FuncInfo.ValueMap.find(V); if (VMI != FuncInfo.ValueMap.end()) { const auto &TLI = DAG.getTargetLoweringInfo(); RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second, @@ -6646,9 +6678,15 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, setValue(&I, DAG.getVScale(sdl, VT, APInt(VT.getSizeInBits(), 1))); return; } - case Intrinsic::vastart: visitVAStart(I); return; - case Intrinsic::vaend: visitVAEnd(I); return; - case Intrinsic::vacopy: visitVACopy(I); return; + case Intrinsic::vastart: + visitVAStart(I); + return; + case Intrinsic::vaend: + visitVAEnd(I); + return; + case Intrinsic::vacopy: + visitVACopy(I); + return; case Intrinsic::returnaddress: setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl, TLI.getValueType(DAG.getDataLayout(), I.getType()), @@ -6676,8 +6714,8 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, SDValue RegName = DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata())); EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); - Res = DAG.getNode(ISD::READ_REGISTER, sdl, - DAG.getVTList(VT, MVT::Other), Chain, RegName); + Res = DAG.getNode(ISD::READ_REGISTER, sdl, DAG.getVTList(VT, MVT::Other), + Chain, RegName); setValue(&I, Res); DAG.setRoot(Res.getValue(1)); return; @@ -6737,11 +6775,10 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, // FIXME: Support passing different dest/src alignments to the memcpy DAG // node. SDValue Root = isVol ? getRoot() : getMemoryRoot(); - SDValue MC = DAG.getMemcpy(Root, sdl, Dst, Src, Size, Alignment, isVol, - MCI.isForceInlined(), &I, std::nullopt, - MachinePointerInfo(I.getArgOperand(0)), - MachinePointerInfo(I.getArgOperand(1)), - I.getAAMetadata(), BatchAA); + SDValue MC = DAG.getMemcpy( + Root, sdl, Dst, Src, Size, Alignment, isVol, MCI.isForceInlined(), &I, + std::nullopt, MachinePointerInfo(I.getArgOperand(0)), + MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata(), BatchAA); updateDAGForMaybeTailCall(MC); return; } @@ -6758,8 +6795,8 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, bool isVol = MSII.isVolatile(); SDValue Root = isVol ? getRoot() : getMemoryRoot(); SDValue MC = DAG.getMemset( - Root, sdl, Dst, Value, Size, DstAlign, isVol, MSII.isForceInlined(), - &I, MachinePointerInfo(I.getArgOperand(0)), I.getAAMetadata()); + Root, sdl, Dst, Value, Size, DstAlign, isVol, MSII.isForceInlined(), &I, + MachinePointerInfo(I.getArgOperand(0)), I.getAAMetadata()); updateDAGForMaybeTailCall(MC); return; } @@ -6868,9 +6905,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, case Intrinsic::eh_return_i32: case Intrinsic::eh_return_i64: DAG.getMachineFunction().setCallsEHReturn(true); - DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl, - MVT::Other, - getControlRoot(), + DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl, MVT::Other, getControlRoot(), getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)))); return; @@ -6893,7 +6928,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, // Get and store the index of the function context. MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); AllocaInst *FnCtx = - cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts()); + cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts()); int FI = FuncInfo.StaticAllocaMap[FnCtx]; MFI.setFunctionContextIndex(FI); return; @@ -6909,12 +6944,12 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, return; } case Intrinsic::eh_sjlj_longjmp: - DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other, - getRoot(), getValue(I.getArgOperand(0)))); + DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other, getRoot(), + getValue(I.getArgOperand(0)))); return; case Intrinsic::eh_sjlj_setup_dispatch: - DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_SETUP_DISPATCH, sdl, MVT::Other, - getRoot())); + DAG.setRoot( + DAG.getNode(ISD::EH_SJLJ_SETUP_DISPATCH, sdl, MVT::Other, getRoot())); return; case Intrinsic::masked_gather: visitMaskedGather(I); @@ -7034,8 +7069,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, // clang-format on EVT RetVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); - setValue(&I, DAG.getNode(Opcode, sdl, RetVT, - getValue(I.getArgOperand(0)))); + setValue(&I, DAG.getNode(Opcode, sdl, RetVT, getValue(I.getArgOperand(0)))); return; } case Intrinsic::minnum: @@ -7150,9 +7184,9 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, SelectionDAG::FlagInserter FlagsInserter(DAG, Flags); SDValue Result; - Result = DAG.getNode( - ISD::FPTRUNC_ROUND, sdl, VT, getValue(I.getArgOperand(0)), - DAG.getTargetConstant((int)*RoundMode, sdl, MVT::i32)); + Result = + DAG.getNode(ISD::FPTRUNC_ROUND, sdl, VT, getValue(I.getArgOperand(0)), + DAG.getTargetConstant((int)*RoundMode, sdl, MVT::i32)); setValue(&I, Result); return; @@ -7474,8 +7508,8 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, SDValue Op1 = getValue(I.getArgOperand(0)); SDValue Op2 = getValue(I.getArgOperand(1)); SDValue Op3 = getValue(I.getArgOperand(2)); - setValue(&I, expandDivFix(FixedPointIntrinsicToOpcode(Intrinsic), sdl, - Op1, Op2, Op3, DAG, TLI)); + setValue(&I, expandDivFix(FixedPointIntrinsicToOpcode(Intrinsic), sdl, Op1, + Op2, Op3, DAG, TLI)); return; } case Intrinsic::smax: { @@ -7536,7 +7570,8 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, } case Intrinsic::stackrestore: Res = getValue(I.getArgOperand(0)); - DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res)); + DAG.setRoot( + DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res)); return; case Intrinsic::get_dynamic_area_offset: { SDValue Op = getRoot(); @@ -7585,7 +7620,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, if (TLI.useLoadStackGuardNode(M)) Src = getLoadStackGuard(DAG, sdl, Chain); else - Src = getValue(I.getArgOperand(0)); // The guard's value. + Src = getValue(I.getArgOperand(0)); // The guard's value. AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1)); @@ -7679,7 +7714,8 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, case Intrinsic::gcwrite: llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!"); case Intrinsic::get_rounding: - Res = DAG.getNode(ISD::GET_ROUNDING, sdl, {MVT::i32, MVT::Other}, getRoot()); + Res = + DAG.getNode(ISD::GET_ROUNDING, sdl, {MVT::i32, MVT::Other}, getRoot()); setValue(&I, Res); DAG.setRoot(Res.getValue(1)); return; @@ -7711,7 +7747,8 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, cast<ConstantInt>(I.getArgOperand(0))->getZExtValue(), sdl, MVT::i32))); break; - default: llvm_unreachable("unknown trap intrinsic"); + default: + llvm_unreachable("unknown trap intrinsic"); } DAG.addNoMergeSiteInfo(DAG.getRoot().getNode(), I.hasFnAttr(Attribute::NoMerge)); @@ -7748,13 +7785,26 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, case Intrinsic::smul_with_overflow: { ISD::NodeType Op; switch (Intrinsic) { - default: llvm_unreachable("Impossible intrinsic"); // Can't reach here. - case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break; - case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break; - case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break; - case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break; - case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break; - case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break; + default: + llvm_unreachable("Impossible intrinsic"); // Can't reach here. + case Intrinsic::uadd_with_overflow: + Op = ISD::UADDO; + break; + case Intrinsic::sadd_with_overflow: + Op = ISD::SADDO; + break; + case Intrinsic::usub_with_overflow: + Op = ISD::USUBO; + break; + case Intrinsic::ssub_with_overflow: + Op = ISD::SSUBO; + break; + case Intrinsic::umul_with_overflow: + Op = ISD::UMULO; + break; + case Intrinsic::smul_with_overflow: + Op = ISD::SMULO; + break; } SDValue Op1 = getValue(I.getArgOperand(0)); SDValue Op2 = getValue(I.getArgOperand(1)); @@ -7767,17 +7817,28 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, return; } case Intrinsic::prefetch: { + auto *DefaultConstantInt = + llvm::ConstantInt::get(llvm::Type::getInt32Ty(*DAG.getContext()), 0); + auto GetConstantIntOrDefault = [&](Value *Val) { + if (auto *Constant = dyn_cast<ConstantInt>(Val)) { + return Constant; + } + // Cases where Val is non-constant should be rejected by Sema. + return DefaultConstantInt; + }; + SDValue Ops[5]; - unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue(); - auto Flags = rw == 0 ? MachineMemOperand::MOLoad :MachineMemOperand::MOStore; + unsigned rw = GetConstantIntOrDefault(I.getArgOperand(1))->getZExtValue(); + auto Flags = + rw == 0 ? MachineMemOperand::MOLoad : MachineMemOperand::MOStore; Ops[0] = DAG.getRoot(); Ops[1] = getValue(I.getArgOperand(0)); - Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl, - MVT::i32); - Ops[3] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(2)), sdl, - MVT::i32); - Ops[4] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(3)), sdl, - MVT::i32); + Ops[2] = DAG.getTargetConstant(*GetConstantIntOrDefault(I.getArgOperand(1)), + sdl, MVT::i32); + Ops[3] = DAG.getTargetConstant(*GetConstantIntOrDefault(I.getArgOperand(2)), + sdl, MVT::i32); + Ops[4] = DAG.getTargetConstant(*GetConstantIntOrDefault(I.getArgOperand(3)), + sdl, MVT::i32); SDValue Result = DAG.getMemIntrinsicNode( ISD::PREFETCH, sdl, DAG.getVTList(MVT::Other), Ops, EVT::getIntegerVT(*Context, 8), MachinePointerInfo(I.getArgOperand(0)), @@ -7917,8 +7978,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, // Create a MCSymbol for the label to avoid any target lowering // that would make this PC relative. SDValue OffsetSym = DAG.getMCSymbol(FrameAllocSym, PtrVT); - SDValue OffsetVal = - DAG.getNode(ISD::LOCAL_RECOVER, sdl, PtrVT, OffsetSym); + SDValue OffsetVal = DAG.getNode(ISD::LOCAL_RECOVER, sdl, PtrVT, OffsetSym); // Add the offset to the FP. SDValue Add = DAG.getMemBasePlusOffset(FPVal, OffsetVal, sdl); @@ -8298,10 +8358,10 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, SDValue VectorIndex = DAG.getSplat(VecTy, sdl, Index); SDValue VectorTripCount = DAG.getSplat(VecTy, sdl, TripCount); SDValue VectorStep = DAG.getStepVector(sdl, VecTy); - SDValue VectorInduction = DAG.getNode( - ISD::UADDSAT, sdl, VecTy, VectorIndex, VectorStep); - SDValue SetCC = DAG.getSetCC(sdl, CCVT, VectorInduction, - VectorTripCount, ISD::CondCode::SETULT); + SDValue VectorInduction = + DAG.getNode(ISD::UADDSAT, sdl, VecTy, VectorIndex, VectorStep); + SDValue SetCC = DAG.getSetCC(sdl, CCVT, VectorInduction, VectorTripCount, + ISD::CondCode::SETULT); setValue(&I, SetCC); return; } @@ -8329,8 +8389,8 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, CountVT = VT; } - SDValue MaxEVL = DAG.getElementCount(sdl, CountVT, - ElementCount::get(VF, IsScalable)); + SDValue MaxEVL = + DAG.getElementCount(sdl, CountVT, ElementCount::get(VF, IsScalable)); SDValue UMin = DAG.getNode(ISD::UMIN, sdl, CountVT, Count, MaxEVL); // Clip to the result type if needed. @@ -8598,7 +8658,8 @@ void SelectionDAGBuilder::visitConstrainedFPIntrinsic( unsigned Opcode; switch (FPI.getIntrinsicID()) { - default: llvm_unreachable("Impossible intrinsic"); // Can't reach here. + default: + llvm_unreachable("Impossible intrinsic"); // Can't reach here. #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ case Intrinsic::INTRINSIC: \ Opcode = ISD::STRICT_##DAGN; \ @@ -8625,7 +8686,8 @@ void SelectionDAGBuilder::visitConstrainedFPIntrinsic( // A few strict DAG nodes carry additional operands that are not // set up by the default code above. switch (Opcode) { - default: break; + default: + break; case ISD::STRICT_FP_ROUND: Opers.push_back( DAG.getTargetConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()))); @@ -8757,7 +8819,7 @@ void SelectionDAGBuilder::visitVPGather( if (!Alignment) Alignment = DAG.getEVTAlign(VT.getScalarType()); unsigned AS = - PtrOperand->getType()->getScalarType()->getPointerAddressSpace(); + PtrOperand->getType()->getScalarType()->getPointerAddressSpace(); MachineMemOperand::Flags MMOFlags = TLI.getVPIntrinsicMemOperandFlags(VPIntrin); MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( @@ -9461,10 +9523,9 @@ bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) { const Value *Length = I.getArgOperand(2); const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); - std::pair<SDValue, SDValue> Res = - TSI.EmitTargetCodeForMemchr(DAG, getCurSDLoc(), DAG.getRoot(), - getValue(Src), getValue(Char), getValue(Length), - MachinePointerInfo(Src)); + std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemchr( + DAG, getCurSDLoc(), DAG.getRoot(), getValue(Src), getValue(Char), + getValue(Length), MachinePointerInfo(Src)); if (Res.first.getNode()) { setValue(&I, Res.first); PendingLoads.push_back(Res.second); @@ -9603,10 +9664,9 @@ bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) { const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1); const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); - std::pair<SDValue, SDValue> Res = - TSI.EmitTargetCodeForStrnlen(DAG, getCurSDLoc(), DAG.getRoot(), - getValue(Arg0), getValue(Arg1), - MachinePointerInfo(Arg0)); + std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForStrnlen( + DAG, getCurSDLoc(), DAG.getRoot(), getValue(Arg0), getValue(Arg1), + MachinePointerInfo(Arg0)); if (Res.first.getNode()) { processIntegerCallValue(I, Res.first, false); PendingLoads.push_back(Res.second); @@ -9706,7 +9766,8 @@ void SelectionDAGBuilder::visitCall(const CallInst &I) { if (!I.isNoBuiltin() && !F->hasLocalLinkage() && F->hasName() && LibInfo->getLibFunc(*F, Func) && LibInfo->hasOptimizedCodeGen(Func)) { switch (Func) { - default: break; + default: + break; case LibFunc_bcmp: if (visitMemCmpBCmpCall(I)) return; @@ -9936,8 +9997,7 @@ class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo { RegsForValue AssignedRegs; explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info) - : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr, 0) { - } + : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr, 0) {} /// Whether or not this operand accesses memory bool hasMemory(const TargetLowering &TLI) const { @@ -9953,7 +10013,6 @@ class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo { } }; - } // end anonymous namespace /// Make sure that the output operand \p OpInfo and its corresponding input @@ -10497,15 +10556,14 @@ static bool prepareDAGLevelOperands(ConstraintDecisionInfo &Info, } // Treat indirect 'X' constraint as memory. - if (OpInfo.ConstraintType == TargetLowering::C_Other && - OpInfo.isIndirect) + if (OpInfo.ConstraintType == TargetLowering::C_Other && OpInfo.isIndirect) OpInfo.ConstraintType = TargetLowering::C_Memory; if (OpInfo.ConstraintType == TargetLowering::C_Immediate || OpInfo.ConstraintType == TargetLowering::C_Other) { std::vector<SDValue> Ops; TLI.LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode, - Ops, DAG); + Ops, DAG); if (Ops.empty()) { if (OpInfo.ConstraintType == TargetLowering::C_Immediate) if (isa<ConstantSDNode>(InOperandVal)) { @@ -10839,8 +10897,7 @@ void SelectionDAGBuilder::emitInlineAsmError(const CallBase &Call, } void SelectionDAGBuilder::visitVAStart(const CallInst &I) { - DAG.setRoot(DAG.getNode(ISD::VASTART, getCurSDLoc(), - MVT::Other, getRoot(), + DAG.setRoot(DAG.getNode(ISD::VASTART, getCurSDLoc(), MVT::Other, getRoot(), getValue(I.getArgOperand(0)), DAG.getSrcValue(I.getArgOperand(0)))); } @@ -10861,15 +10918,13 @@ void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) { } void SelectionDAGBuilder::visitVAEnd(const CallInst &I) { - DAG.setRoot(DAG.getNode(ISD::VAEND, getCurSDLoc(), - MVT::Other, getRoot(), + DAG.setRoot(DAG.getNode(ISD::VAEND, getCurSDLoc(), MVT::Other, getRoot(), getValue(I.getArgOperand(0)), DAG.getSrcValue(I.getArgOperand(0)))); } void SelectionDAGBuilder::visitVACopy(const CallInst &I) { - DAG.setRoot(DAG.getNode(ISD::VACOPY, getCurSDLoc(), - MVT::Other, getRoot(), + DAG.setRoot(DAG.getNode(ISD::VACOPY, getCurSDLoc(), MVT::Other, getRoot(), getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)), DAG.getSrcValue(I.getArgOperand(0)), @@ -10946,8 +11001,7 @@ void SelectionDAGBuilder::populateCallLoweringInfo( // Populate the argument list. // Attributes for args start at offset 1, after the return attribute. - for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs; - ArgI != ArgE; ++ArgI) { + for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs; ArgI != ArgE; ++ArgI) { const Value *V = Call->getOperand(ArgI); assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic."); @@ -11084,13 +11138,13 @@ void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB, SDValue Callee = getValue(CB.getArgOperand(PatchPointOpers::TargetPos)); // Handle immediate and symbolic callees. - if (auto* ConstCallee = dyn_cast<ConstantSDNode>(Callee)) + if (auto *ConstCallee = dyn_cast<ConstantSDNode>(Callee)) Callee = DAG.getIntPtrConstant(ConstCallee->getZExtValue(), dl, /*isTarget=*/true); - else if (auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Callee)) - Callee = DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(), - SDLoc(SymbolicCallee), - SymbolicCallee->getValueType(0)); + else if (auto *SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Callee)) + Callee = DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(), + SDLoc(SymbolicCallee), + SymbolicCallee->getValueType(0)); // Get the real number of arguments participating in the call <numArgs> SDValue NArgVal = getValue(CB.getArgOperand(PatchPointOpers::NArgPos)); @@ -11167,7 +11221,7 @@ void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB, Ops.push_back(getValue(CB.getArgOperand(i))); // Push the arguments from the call instruction. - SDNode::op_iterator e = HasGlue ? Call->op_end()-2 : Call->op_end()-1; + SDNode::op_iterator e = HasGlue ? Call->op_end() - 2 : Call->op_end() - 1; Ops.append(Call->op_begin() + 2, e); // Push live variables for the stack map. @@ -11373,7 +11427,8 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const { MF.getFrameInfo().CreateStackObject(TySize, Alignment, false); Type *StackSlotPtrType = PointerType::get(Context, DL.getAllocaAddrSpace()); - DemoteStackSlot = CLI.DAG.getFrameIndex(DemoteStackIdx, getFrameIndexTy(DL)); + DemoteStackSlot = + CLI.DAG.getFrameIndex(DemoteStackIdx, getFrameIndexTy(DL)); ArgListEntry Entry(DemoteStackSlot, StackSlotPtrType); Entry.IsSRet = true; Entry.Alignment = Alignment; @@ -11455,8 +11510,8 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const { } EVT VT = getValueType(DL, ArgTy); - SDValue Op = SDValue(Args[i].Node.getNode(), - Args[i].Node.getResNo() + Value); + SDValue Op = + SDValue(Args[i].Node.getNode(), Args[i].Node.getResNo() + Value); ISD::ArgFlagsTy Flags; // Certain targets (such as MIPS), may have a different ABI alignment @@ -11554,8 +11609,8 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const { else if (Args[i].IsZExt) ExtendKind = ISD::ZERO_EXTEND; - // Conservatively only handle 'returned' on non-vectors that can be lowered, - // for now. + // Conservatively only handle 'returned' on non-vectors that can be + // lowered, for now. if (Args[i].IsReturned && !Op.getValueType().isVector() && CanLowerReturn) { assert((CLI.RetTy == Args[i].Ty || @@ -11719,7 +11774,7 @@ void TargetLowering::LowerOperationWrapper(SDNode *N, // If the original node has multiple results, then the return node should // have the same number of results. assert((N->getNumValues() == Res->getNumValues()) && - "Lowering returned the wrong number of results!"); + "Lowering returned the wrong number of results!"); // Places new result values base on N result number. for (unsigned I = 0, E = N->getNumValues(); I != E; ++I) @@ -11770,7 +11825,7 @@ static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) { const BasicBlock &Entry = A->getParent()->front(); for (const User *U : A->users()) if (cast<Instruction>(U)->getParent() != &Entry || isa<SwitchInst>(U)) - return false; // Use not in entry block. + return false; // Use not in entry block. return true; } @@ -11782,10 +11837,9 @@ using ArgCopyElisionMapTy = /// Scan the entry block of the function in FuncInfo for arguments that look /// like copies into a local alloca. Record any copied arguments in /// ArgCopyElisionCandidates. -static void -findArgumentCopyElisionCandidates(const DataLayout &DL, - FunctionLoweringInfo *FuncInfo, - ArgCopyElisionMapTy &ArgCopyElisionCandidates) { +static void findArgumentCopyElisionCandidates( + const DataLayout &DL, FunctionLoweringInfo *FuncInfo, + ArgCopyElisionMapTy &ArgCopyElisionCandidates) { // Record the state of every static alloca used in the entry block. Argument // allocas are all used in the entry block, so we need approximately as many // entries as we have arguments. @@ -12098,9 +12152,9 @@ void SelectionDAGISel::LowerArguments(const Function &F) { // For scalable vectors, use the minimum size; individual targets // are responsible for handling scalable vector arguments and // return values. - ISD::InputArg MyFlags( - Flags, RegisterVT, VT, ArgTy, isArgValueUsed, ArgNo, - i * RegisterVT.getStoreSize().getKnownMinValue()); + ISD::InputArg MyFlags(Flags, RegisterVT, VT, ArgTy, isArgValueUsed, + ArgNo, + i * RegisterVT.getStoreSize().getKnownMinValue()); if (NumRegs > 1 && i == 0) MyFlags.Flags.setSplit(); // if it isn't first piece, alignment must be 1 @@ -12144,8 +12198,8 @@ void SelectionDAGISel::LowerArguments(const Function &F) { getCopyFromParts(DAG, dl, &InVals[0], 1, RegVT, VT, nullptr, NewRoot, F.getCallingConv(), AssertOp); - MachineFunction& MF = SDB->DAG.getMachineFunction(); - MachineRegisterInfo& RegInfo = MF.getRegInfo(); + MachineFunction &MF = SDB->DAG.getMachineFunction(); + MachineRegisterInfo &RegInfo = MF.getRegInfo(); Register SRetReg = RegInfo.createVirtualRegister(TLI->getRegClassFor(RegVT)); FuncInfo->DemoteRegister = SRetReg; @@ -12182,17 +12236,16 @@ void SelectionDAGISel::LowerArguments(const Function &F) { ArrayRef(&InVals[i], NumParts), ArgHasUses); } - // If this argument is unused then remember its value. It is used to generate - // debugging information. + // If this argument is unused then remember its value. It is used to + // generate debugging information. bool isSwiftErrorArg = - TLI->supportSwiftError() && - Arg.hasAttribute(Attribute::SwiftError); + TLI->supportSwiftError() && Arg.hasAttribute(Attribute::SwiftError); if (!ArgHasUses && !isSwiftErrorArg) { SDB->setUnusedArgValue(&Arg, InVals[i]); // Also remember any frame index for use in FastISel. if (FrameIndexSDNode *FI = - dyn_cast<FrameIndexSDNode>(InVals[i].getNode())) + dyn_cast<FrameIndexSDNode>(InVals[i].getNode())) FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex()); } @@ -12236,7 +12289,7 @@ void SelectionDAGISel::LowerArguments(const Function &F) { // Note down frame index. if (FrameIndexSDNode *FI = - dyn_cast<FrameIndexSDNode>(ArgValues[0].getNode())) + dyn_cast<FrameIndexSDNode>(ArgValues[0].getNode())) FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex()); SDValue Res = DAG.getMergeValues(ArrayRef(ArgValues.data(), NumValues), @@ -12253,9 +12306,9 @@ void SelectionDAGISel::LowerArguments(const Function &F) { // significant bits will be in the second operand. unsigned LowAddressOp = DAG.getDataLayout().isBigEndian() ? 1 : 0; if (LoadSDNode *LNode = - dyn_cast<LoadSDNode>(Res.getOperand(LowAddressOp).getNode())) + dyn_cast<LoadSDNode>(Res.getOperand(LowAddressOp).getNode())) if (FrameIndexSDNode *FI = - dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode())) + dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode())) FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex()); } @@ -12319,8 +12372,8 @@ void SelectionDAGISel::LowerArguments(const Function &F) { /// directly add them, because expansion might result in multiple MBB's for one /// BB. As such, the start of the BB might correspond to a different MBB than /// the end. -void -SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) { +void SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks( + const BasicBlock *LLVMBB) { const TargetLowering &TLI = DAG.getTargetLoweringInfo(); SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled; @@ -12328,7 +12381,8 @@ SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) { // Check PHI nodes in successors that expect a value to be available from this // block. for (const BasicBlock *SuccBB : successors(LLVMBB->getTerminator())) { - if (!isa<PHINode>(SuccBB->begin())) continue; + if (!isa<PHINode>(SuccBB->begin())) + continue; MachineBasicBlock *SuccMBB = FuncInfo.getMBB(SuccBB); // If this terminator has multiple identical successors (common for @@ -12384,7 +12438,8 @@ SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) { SmallVector<EVT, 4> ValueVTs; ComputeValueVTs(TLI, DAG.getDataLayout(), PN.getType(), ValueVTs); for (EVT VT : ValueVTs) { - const unsigned NumRegisters = TLI.getNumRegisters(*DAG.getContext(), VT); + const unsigned NumRegisters = + TLI.getNumRegisters(*DAG.getContext(), VT); for (unsigned i = 0; i != NumRegisters; ++i) FuncInfo.PHINodesToUpdate.emplace_back(&*MBBI++, Reg + i); Reg += NumRegisters; @@ -12490,14 +12545,14 @@ void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond, // as a tie-breaker as clusters are guaranteed to never overlap. llvm::sort(W.FirstCluster, W.LastCluster + 1, [](const CaseCluster &a, const CaseCluster &b) { - return a.Prob != b.Prob ? - a.Prob > b.Prob : - a.Low->getValue().slt(b.Low->getValue()); - }); + return a.Prob != b.Prob + ? a.Prob > b.Prob + : a.Low->getValue().slt(b.Low->getValue()); + }); // Rearrange the case blocks so that the last one falls through if possible // without changing the order of probabilities. - for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) { + for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster;) { --I; if (I->Prob > W.LastCluster->Prob) break; @@ -12526,136 +12581,137 @@ void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond, } else { Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock()); CurMF->insert(BBI, Fallthrough); - // Put Cond in a virtual register to make it available from the new blocks. + // Put Cond in a virtual register to make it available from the new + // blocks. ExportFromCurrentBlock(Cond); } UnhandledProbs -= I->Prob; switch (I->Kind) { - case CC_JumpTable: { - // FIXME: Optimize away range check based on pivot comparisons. - JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first; - SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second; - - // The jump block hasn't been inserted yet; insert it here. - MachineBasicBlock *JumpMBB = JT->MBB; - CurMF->insert(BBI, JumpMBB); - - auto JumpProb = I->Prob; - auto FallthroughProb = UnhandledProbs; - - // If the default statement is a target of the jump table, we evenly - // distribute the default probability to successors of CurMBB. Also - // update the probability on the edge from JumpMBB to Fallthrough. - for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(), - SE = JumpMBB->succ_end(); - SI != SE; ++SI) { - if (*SI == DefaultMBB) { - JumpProb += DefaultProb / 2; - FallthroughProb -= DefaultProb / 2; - JumpMBB->setSuccProbability(SI, DefaultProb / 2); - JumpMBB->normalizeSuccProbs(); - break; - } + case CC_JumpTable: { + // FIXME: Optimize away range check based on pivot comparisons. + JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first; + SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second; + + // The jump block hasn't been inserted yet; insert it here. + MachineBasicBlock *JumpMBB = JT->MBB; + CurMF->insert(BBI, JumpMBB); + + auto JumpProb = I->Prob; + auto FallthroughProb = UnhandledProbs; + + // If the default statement is a target of the jump table, we evenly + // distribute the default probability to successors of CurMBB. Also + // update the probability on the edge from JumpMBB to Fallthrough. + for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(), + SE = JumpMBB->succ_end(); + SI != SE; ++SI) { + if (*SI == DefaultMBB) { + JumpProb += DefaultProb / 2; + FallthroughProb -= DefaultProb / 2; + JumpMBB->setSuccProbability(SI, DefaultProb / 2); + JumpMBB->normalizeSuccProbs(); + break; } + } - // If the default clause is unreachable, propagate that knowledge into - // JTH->FallthroughUnreachable which will use it to suppress the range - // check. - // - // However, don't do this if we're doing branch target enforcement, - // because a table branch _without_ a range check can be a tempting JOP - // gadget - out-of-bounds inputs that are impossible in correct - // execution become possible again if an attacker can influence the - // control flow. So if an attacker doesn't already have a BTI bypass - // available, we don't want them to be able to get one out of this - // table branch. - if (FallthroughUnreachable) { - Function &CurFunc = CurMF->getFunction(); - if (!CurFunc.hasFnAttribute("branch-target-enforcement")) - JTH->FallthroughUnreachable = true; - } + // If the default clause is unreachable, propagate that knowledge into + // JTH->FallthroughUnreachable which will use it to suppress the range + // check. + // + // However, don't do this if we're doing branch target enforcement, + // because a table branch _without_ a range check can be a tempting JOP + // gadget - out-of-bounds inputs that are impossible in correct + // execution become possible again if an attacker can influence the + // control flow. So if an attacker doesn't already have a BTI bypass + // available, we don't want them to be able to get one out of this + // table branch. + if (FallthroughUnreachable) { + Function &CurFunc = CurMF->getFunction(); + if (!CurFunc.hasFnAttribute("branch-target-enforcement")) + JTH->FallthroughUnreachable = true; + } - if (!JTH->FallthroughUnreachable) - addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb); - addSuccessorWithProb(CurMBB, JumpMBB, JumpProb); - CurMBB->normalizeSuccProbs(); + if (!JTH->FallthroughUnreachable) + addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb); + addSuccessorWithProb(CurMBB, JumpMBB, JumpProb); + CurMBB->normalizeSuccProbs(); - // The jump table header will be inserted in our current block, do the - // range check, and fall through to our fallthrough block. - JTH->HeaderBB = CurMBB; - JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader. + // The jump table header will be inserted in our current block, do the + // range check, and fall through to our fallthrough block. + JTH->HeaderBB = CurMBB; + JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader. - // If we're in the right place, emit the jump table header right now. - if (CurMBB == SwitchMBB) { - visitJumpTableHeader(*JT, *JTH, SwitchMBB); - JTH->Emitted = true; - } - break; + // If we're in the right place, emit the jump table header right now. + if (CurMBB == SwitchMBB) { + visitJumpTableHeader(*JT, *JTH, SwitchMBB); + JTH->Emitted = true; + } + break; + } + case CC_BitTests: { + // FIXME: Optimize away range check based on pivot comparisons. + BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex]; + + // The bit test blocks haven't been inserted yet; insert them here. + for (BitTestCase &BTC : BTB->Cases) + CurMF->insert(BBI, BTC.ThisBB); + + // Fill in fields of the BitTestBlock. + BTB->Parent = CurMBB; + BTB->Default = Fallthrough; + + BTB->DefaultProb = UnhandledProbs; + // If the cases in bit test don't form a contiguous range, we evenly + // distribute the probability on the edge to Fallthrough to two + // successors of CurMBB. + if (!BTB->ContiguousRange) { + BTB->Prob += DefaultProb / 2; + BTB->DefaultProb -= DefaultProb / 2; } - case CC_BitTests: { - // FIXME: Optimize away range check based on pivot comparisons. - BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex]; - - // The bit test blocks haven't been inserted yet; insert them here. - for (BitTestCase &BTC : BTB->Cases) - CurMF->insert(BBI, BTC.ThisBB); - - // Fill in fields of the BitTestBlock. - BTB->Parent = CurMBB; - BTB->Default = Fallthrough; - - BTB->DefaultProb = UnhandledProbs; - // If the cases in bit test don't form a contiguous range, we evenly - // distribute the probability on the edge to Fallthrough to two - // successors of CurMBB. - if (!BTB->ContiguousRange) { - BTB->Prob += DefaultProb / 2; - BTB->DefaultProb -= DefaultProb / 2; - } - if (FallthroughUnreachable) - BTB->FallthroughUnreachable = true; + if (FallthroughUnreachable) + BTB->FallthroughUnreachable = true; - // If we're in the right place, emit the bit test header right now. - if (CurMBB == SwitchMBB) { - visitBitTestHeader(*BTB, SwitchMBB); - BTB->Emitted = true; - } - break; + // If we're in the right place, emit the bit test header right now. + if (CurMBB == SwitchMBB) { + visitBitTestHeader(*BTB, SwitchMBB); + BTB->Emitted = true; + } + break; + } + case CC_Range: { + const Value *RHS, *LHS, *MHS; + ISD::CondCode CC; + if (I->Low == I->High) { + // Check Cond == I->Low. + CC = ISD::SETEQ; + LHS = Cond; + RHS = I->Low; + MHS = nullptr; + } else { + // Check I->Low <= Cond <= I->High. + CC = ISD::SETLE; + LHS = I->Low; + MHS = Cond; + RHS = I->High; } - case CC_Range: { - const Value *RHS, *LHS, *MHS; - ISD::CondCode CC; - if (I->Low == I->High) { - // Check Cond == I->Low. - CC = ISD::SETEQ; - LHS = Cond; - RHS=I->Low; - MHS = nullptr; - } else { - // Check I->Low <= Cond <= I->High. - CC = ISD::SETLE; - LHS = I->Low; - MHS = Cond; - RHS = I->High; - } - // If Fallthrough is unreachable, fold away the comparison. - if (FallthroughUnreachable) - CC = ISD::SETTRUE; + // If Fallthrough is unreachable, fold away the comparison. + if (FallthroughUnreachable) + CC = ISD::SETTRUE; - // The false probability is the sum of all unhandled cases. - CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB, - getCurSDLoc(), I->Prob, UnhandledProbs); + // The false probability is the sum of all unhandled cases. + CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB, + getCurSDLoc(), I->Prob, UnhandledProbs); - if (CurMBB == SwitchMBB) - visitSwitchCase(CB, SwitchMBB); - else - SL->SwitchCases.push_back(CB); + if (CurMBB == SwitchMBB) + visitSwitchCase(CB, SwitchMBB); + else + SL->SwitchCases.push_back(CB); - break; - } + break; + } } CurMBB = Fallthrough; } @@ -12708,8 +12764,8 @@ void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList, // single cluster, RHS.Low == Pivot, and we can branch to its destination // directly if RHS.High equals the current upper bound. MachineBasicBlock *RightMBB; - if (FirstRight == LastRight && FirstRight->Kind == CC_Range && - W.LT && (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) { + if (FirstRight == LastRight && FirstRight->Kind == CC_Range && W.LT && + (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) { RightMBB = FirstRight->MBB; } else { RightMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock()); @@ -12996,7 +13052,8 @@ void SelectionDAGBuilder::visitFreeze(const FreezeInst &I) { ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(), ValueVTs); unsigned NumValues = ValueVTs.size(); - if (NumValues == 0) return; + if (NumValues == 0) + return; SmallVector<SDValue, 4> Values(NumValues); SDValue Op = getValue(I.getOperand(0)); _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
