llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-backend-msp430 @llvm/pr-subscribers-clang-codegen @llvm/pr-subscribers-backend-aarch64 @llvm/pr-subscribers-backend-amdgpu Author: Pavel Iliin (ilinpv) <details> <summary>Changes</summary> …instructions The patch adds new `__builtin_consistent` which when used with control flow conditions provides a hint to compiler that branch or switch is very unlikely to change direction. The hint is used to generate efficient conditional branch instructions on targets which support them. Currently it is `BC.cond` on AArch64 when FEAT_HBC enabled. --- Patch is 161.45 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/72175.diff 88 Files Affected: - (modified) bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp (+4-1) - (modified) clang/docs/LanguageExtensions.rst (+29) - (modified) clang/include/clang/Basic/Builtins.def (+3-2) - (modified) clang/lib/Analysis/CalledOnceCheck.cpp (+1) - (modified) clang/lib/CodeGen/CGBuiltin.cpp (+6-4) - (modified) clang/lib/CodeGen/CGStmt.cpp (+11-6) - (modified) clang/lib/CodeGen/CodeGenFunction.cpp (+12-6) - (modified) clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp (+2-1) - (added) clang/test/CodeGen/builtin-consistent.c (+37) - (modified) llvm/include/llvm/CodeGen/MachineInstr.h (+1) - (modified) llvm/include/llvm/CodeGen/SelectionDAGNodes.h (+7-1) - (modified) llvm/include/llvm/CodeGen/TargetInstrInfo.h (+4-3) - (modified) llvm/include/llvm/IR/FixedMetadataKinds.def (+1) - (modified) llvm/include/llvm/IR/IRBuilder.h (+16-10) - (modified) llvm/include/llvm/IR/MDBuilder.h (+3) - (modified) llvm/lib/CodeGen/BranchFolding.cpp (+29-16) - (modified) llvm/lib/CodeGen/MIRPrinter.cpp (+2) - (modified) llvm/lib/CodeGen/MachineBasicBlock.cpp (+15-10) - (modified) llvm/lib/CodeGen/MachineInstr.cpp (+5) - (modified) llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp (+20-14) - (modified) llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp (+3) - (modified) llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp (+1-1) - (modified) llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp (+35-22) - (modified) llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h (+7-5) - (modified) llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp (+3) - (modified) llvm/lib/IR/IRBuilder.cpp (+2-1) - (modified) llvm/lib/IR/MDBuilder.cpp (+5) - (modified) llvm/lib/Target/AArch64/AArch64CondBrTuning.cpp (+5-1) - (modified) llvm/lib/Target/AArch64/AArch64ConditionOptimizer.cpp (+1-1) - (modified) llvm/lib/Target/AArch64/AArch64FastISel.cpp (+31-10) - (modified) llvm/lib/Target/AArch64/AArch64ISelLowering.cpp (+30-20) - (modified) llvm/lib/Target/AArch64/AArch64ISelLowering.h (+1) - (modified) llvm/lib/Target/AArch64/AArch64InstrFormats.td (+2-2) - (modified) llvm/lib/Target/AArch64/AArch64InstrInfo.cpp (+34-11) - (modified) llvm/lib/Target/AArch64/AArch64InstrInfo.h (+9-5) - (modified) llvm/lib/Target/AArch64/AArch64InstrInfo.td (+4-2) - (modified) llvm/lib/Target/AArch64/AArch64MacroFusion.cpp (+2-1) - (modified) llvm/lib/Target/AArch64/AArch64RedundantCopyElimination.cpp (+1-1) - (modified) llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp (+20-7) - (modified) llvm/lib/Target/AMDGPU/R600InstrInfo.cpp (+4-4) - (modified) llvm/lib/Target/AMDGPU/R600InstrInfo.h (+4-4) - (modified) llvm/lib/Target/AMDGPU/SIInstrInfo.cpp (+4-4) - (modified) llvm/lib/Target/AMDGPU/SIInstrInfo.h (+4-4) - (modified) llvm/lib/Target/ARC/ARCInstrInfo.cpp (+4-3) - (modified) llvm/lib/Target/ARC/ARCInstrInfo.h (+4-4) - (modified) llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp (+4-3) - (modified) llvm/lib/Target/ARM/ARMBaseInstrInfo.h (+4-4) - (modified) llvm/lib/Target/AVR/AVRInstrInfo.cpp (+4-3) - (modified) llvm/lib/Target/AVR/AVRInstrInfo.h (+4-4) - (modified) llvm/lib/Target/BPF/BPFInstrInfo.cpp (+4-4) - (modified) llvm/lib/Target/BPF/BPFInstrInfo.h (+5-4) - (modified) llvm/lib/Target/CSKY/CSKYInstrInfo.cpp (+8-5) - (modified) llvm/lib/Target/CSKY/CSKYInstrInfo.h (+4-4) - (modified) llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp (+5-4) - (modified) llvm/lib/Target/Hexagon/HexagonInstrInfo.h (+4-4) - (modified) llvm/lib/Target/Lanai/LanaiInstrInfo.cpp (+4-4) - (modified) llvm/lib/Target/Lanai/LanaiInstrInfo.h (+5-5) - (modified) llvm/lib/Target/LoongArch/LoongArchInstrInfo.cpp (+8-4) - (modified) llvm/lib/Target/LoongArch/LoongArchInstrInfo.h (+4-4) - (modified) llvm/lib/Target/M68k/M68kInstrInfo.cpp (+8-5) - (modified) llvm/lib/Target/M68k/M68kInstrInfo.h (+4-4) - (modified) llvm/lib/Target/MSP430/MSP430InstrInfo.cpp (+4-3) - (modified) llvm/lib/Target/MSP430/MSP430InstrInfo.h (+4-4) - (modified) llvm/lib/Target/Mips/MipsInstrInfo.cpp (+4-4) - (modified) llvm/lib/Target/Mips/MipsInstrInfo.h (+4-4) - (modified) llvm/lib/Target/NVPTX/NVPTXInstrInfo.cpp (+4-4) - (modified) llvm/lib/Target/NVPTX/NVPTXInstrInfo.h (+4-4) - (modified) llvm/lib/Target/PowerPC/PPCInstrInfo.cpp (+4-4) - (modified) llvm/lib/Target/PowerPC/PPCInstrInfo.h (+4-4) - (modified) llvm/lib/Target/RISCV/RISCVInstrInfo.cpp (+8-5) - (modified) llvm/lib/Target/RISCV/RISCVInstrInfo.h (+4-4) - (modified) llvm/lib/Target/SPIRV/SPIRVInstrInfo.cpp (+8-5) - (modified) llvm/lib/Target/SPIRV/SPIRVInstrInfo.h (+4-4) - (modified) llvm/lib/Target/Sparc/SparcInstrInfo.cpp (+4-4) - (modified) llvm/lib/Target/Sparc/SparcInstrInfo.h (+4-4) - (modified) llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp (+4-3) - (modified) llvm/lib/Target/SystemZ/SystemZInstrInfo.h (+4-4) - (modified) llvm/lib/Target/VE/VEInstrInfo.cpp (+4-3) - (modified) llvm/lib/Target/VE/VEInstrInfo.h (+4-4) - (modified) llvm/lib/Target/WebAssembly/WebAssemblyInstrInfo.cpp (+8-4) - (modified) llvm/lib/Target/WebAssembly/WebAssemblyInstrInfo.h (+4-4) - (modified) llvm/lib/Target/X86/X86InstrInfo.cpp (+4-4) - (modified) llvm/lib/Target/X86/X86InstrInfo.h (+4-4) - (modified) llvm/lib/Target/XCore/XCoreInstrInfo.cpp (+4-4) - (modified) llvm/lib/Target/XCore/XCoreInstrInfo.h (+4-4) - (modified) llvm/lib/Transforms/Utils/SimplifyCFG.cpp (+6-4) - (modified) llvm/test/CodeGen/AArch64/cond-br-tuning.ll (+107-10) - (modified) llvm/test/CodeGen/AArch64/tbl-loops.ll (+372-23) ``````````diff diff --git a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp index b852b9fbc9c52f4..725e0e2adb821cf 100644 --- a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp +++ b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp @@ -962,7 +962,8 @@ class AArch64MCPlusBuilder : public MCPlusBuilder { if (isTB(Inst) || isCB(Inst)) { Inst.setOpcode(getInvertedBranchOpcode(Inst.getOpcode())); assert(Inst.getOpcode() != 0 && "Invalid branch instruction"); - } else if (Inst.getOpcode() == AArch64::Bcc) { + } else if (Inst.getOpcode() == AArch64::Bcc || + Inst.getOpcode() == AArch64::BCcc) { Inst.getOperand(0).setImm(AArch64CC::getInvertedCondCode( static_cast<AArch64CC::CondCode>(Inst.getOperand(0).getImm()))); assert(Inst.getOperand(0).getImm() != AArch64CC::AL && @@ -991,6 +992,8 @@ class AArch64MCPlusBuilder : public MCPlusBuilder { case AArch64::B: return 28; case AArch64::BL: return 28; case AArch64::Bcc: return 21; + case AArch64::BCcc: + return 21; } } diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst index 294210c6ac140a9..8f68f144691cc55 100644 --- a/clang/docs/LanguageExtensions.rst +++ b/clang/docs/LanguageExtensions.rst @@ -3102,6 +3102,35 @@ flow conditions such as in ``if`` and ``switch`` statements. Query for this feature with ``__has_builtin(__builtin_unpredictable)``. +``__builtin_consistent`` +------------------------ + +``__builtin_consistent`` is used to indicate that the value of an expression is +very likely to be consistent, e.g. branch having expression as condition will +behave very consistently and is very unlikely to change direction. + +**Syntax**: + +.. code-block:: c++ + + __builtin_consistent(long long) + +**Example of use**: + +.. code-block:: c++ + + if (__builtin_consistent(x > 0)) { + foo(); + } + +**Description**: + +The ``__builtin_consistent()`` builtin used with control flow conditions will +provide information about branch consistently behavior which can be used to +generate more efficient conditional branch instructions if target supports them +(like AArch64 FEAT_HBC ``BC.cond``). + +Query for this feature with ``__has_builtin(__builtin_consistent)``. ``__builtin_expect`` -------------------- diff --git a/clang/include/clang/Basic/Builtins.def b/clang/include/clang/Basic/Builtins.def index ec39e926889b936..948964c90dc76ae 100644 --- a/clang/include/clang/Basic/Builtins.def +++ b/clang/include/clang/Basic/Builtins.def @@ -667,8 +667,9 @@ BUILTIN(__builtin___printf_chk, "iicC*R.", "Fp:1:") BUILTIN(__builtin___vfprintf_chk, "iP*RicC*Ra", "FP:2:") BUILTIN(__builtin___vprintf_chk, "iicC*Ra", "FP:1:") -BUILTIN(__builtin_unpredictable, "LiLi" , "nc") -BUILTIN(__builtin_expect, "LiLiLi" , "ncE") +BUILTIN(__builtin_unpredictable, "LiLi", "nc") +BUILTIN(__builtin_consistent, "LiLi", "nc") +BUILTIN(__builtin_expect, "LiLiLi", "ncE") BUILTIN(__builtin_expect_with_probability, "LiLiLid", "ncE") BUILTIN(__builtin_prefetch, "vvC*.", "nc") BUILTIN(__builtin_readcyclecounter, "ULLi", "n") diff --git a/clang/lib/Analysis/CalledOnceCheck.cpp b/clang/lib/Analysis/CalledOnceCheck.cpp index 5b4fc24b6f0e2a5..d1725f6151dbabc 100644 --- a/clang/lib/Analysis/CalledOnceCheck.cpp +++ b/clang/lib/Analysis/CalledOnceCheck.cpp @@ -365,6 +365,7 @@ class DeclRefFinder } case Builtin::BI__builtin_unpredictable: + case Builtin::BI__builtin_consistent: return Visit(CE->getArg(0)); default: diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp index 09309a3937fb613..63ccd42f9e31329 100644 --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -3137,10 +3137,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, "cast"); return RValue::get(Result); } - case Builtin::BI__builtin_unpredictable: { - // Always return the argument of __builtin_unpredictable. LLVM does not - // handle this builtin. Metadata for this builtin should be added directly - // to instructions such as branches or switches that use it. + case Builtin::BI__builtin_unpredictable: + case Builtin::BI__builtin_consistent: { + // Always return the argument of __builtin_unpredictable and + // __builtin_consistent. LLVM does not handle these builtins. Metadata for + // these builtins should be added directly to instructions such as branches + // or switches that use it. return RValue::get(EmitScalarExpr(E->getArg(0))); } case Builtin::BI__builtin_expect: { diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp index c719df1bfa05036..2aa4b8253c7aff1 100644 --- a/clang/lib/CodeGen/CGStmt.cpp +++ b/clang/lib/CodeGen/CGStmt.cpp @@ -2059,16 +2059,21 @@ void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) { EmitBlock(SwitchExit.getBlock(), true); incrementProfileCounter(&S); - // If the switch has a condition wrapped by __builtin_unpredictable, - // create metadata that specifies that the switch is unpredictable. - // Don't bother if not optimizing because that metadata would not be used. + // If the switch has a condition wrapped by __builtin_unpredictable or + // __builtin_consistent, create metadata that specifies that the switch is + // unpredictable or consistent correspondingly. Don't bother if not optimizing + // because that metadata would not be used. auto *Call = dyn_cast<CallExpr>(S.getCond()); if (Call && CGM.getCodeGenOpts().OptimizationLevel != 0) { auto *FD = dyn_cast_or_null<FunctionDecl>(Call->getCalleeDecl()); - if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) { + if (FD) { llvm::MDBuilder MDHelper(getLLVMContext()); - SwitchInsn->setMetadata(llvm::LLVMContext::MD_unpredictable, - MDHelper.createUnpredictable()); + if (FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) + SwitchInsn->setMetadata(llvm::LLVMContext::MD_unpredictable, + MDHelper.createUnpredictable()); + if (FD->getBuiltinID() == Builtin::BI__builtin_consistent) + SwitchInsn->setMetadata(llvm::LLVMContext::MD_consistent, + MDHelper.createConsistent()); } } diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp index 0f2b9055b88eb04..a372913f8732c76 100644 --- a/clang/lib/CodeGen/CodeGenFunction.cpp +++ b/clang/lib/CodeGen/CodeGenFunction.cpp @@ -1908,16 +1908,21 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, llvm::MDNode *Weights = nullptr; llvm::MDNode *Unpredictable = nullptr; + llvm::MDNode *Consistent = nullptr; - // If the branch has a condition wrapped by __builtin_unpredictable, - // create metadata that specifies that the branch is unpredictable. - // Don't bother if not optimizing because that metadata would not be used. + // If the branch has a condition wrapped by __builtin_unpredictable or + // __builtin_consistent, create metadata that specifies that the branch is + // unpredictable or consistent correspondingly. Don't bother if not optimizing + // because that metadata would not be used. auto *Call = dyn_cast<CallExpr>(Cond->IgnoreImpCasts()); if (Call && CGM.getCodeGenOpts().OptimizationLevel != 0) { auto *FD = dyn_cast_or_null<FunctionDecl>(Call->getCalleeDecl()); - if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) { + if (FD) { llvm::MDBuilder MDHelper(getLLVMContext()); - Unpredictable = MDHelper.createUnpredictable(); + if (FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) + Unpredictable = MDHelper.createUnpredictable(); + if (FD->getBuiltinID() == Builtin::BI__builtin_consistent) + Consistent = MDHelper.createConsistent(); } } @@ -1932,7 +1937,8 @@ void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond, Weights = createProfileWeights(TrueCount, CurrentCount - TrueCount); } - Builder.CreateCondBr(CondV, TrueBlock, FalseBlock, Weights, Unpredictable); + Builder.CreateCondBr(CondV, TrueBlock, FalseBlock, Weights, Unpredictable, + Consistent); } /// ErrorUnsupported - Print out an error that codegen doesn't support the diff --git a/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp index 4a56156de4b27fe..3de5c6676a732fd 100644 --- a/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp @@ -63,12 +63,13 @@ bool BuiltinFunctionChecker::evalCall(const CallEvent &Call, } case Builtin::BI__builtin_unpredictable: + case Builtin::BI__builtin_consistent: case Builtin::BI__builtin_expect: case Builtin::BI__builtin_expect_with_probability: case Builtin::BI__builtin_assume_aligned: case Builtin::BI__builtin_addressof: case Builtin::BI__builtin_function_start: { - // For __builtin_unpredictable, __builtin_expect, + // For __builtin_unpredictable, __builtin_consistent, __builtin_expect, // __builtin_expect_with_probability and __builtin_assume_aligned, // just return the value of the subexpression. // __builtin_addressof is going from a reference to a pointer, but those diff --git a/clang/test/CodeGen/builtin-consistent.c b/clang/test/CodeGen/builtin-consistent.c new file mode 100644 index 000000000000000..3f1be494acf1412 --- /dev/null +++ b/clang/test/CodeGen/builtin-consistent.c @@ -0,0 +1,37 @@ +// RUN: %clang_cc1 -triple aarch64-unknown-unknown -emit-llvm -disable-llvm-passes -o - %s -O1 | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-unknown-unknown -emit-llvm -o - %s -O0 | FileCheck %s --check-prefix=CHECK_O0 + +void f(void); +void g(void); +void consistent_branch(int x) { +// CHECK-LABEL: define{{.*}} void @consistent_branch( +// CHECK-NOT: builtin_consistent +// CHECK: !consistent [[METADATA:.+]] +// CHECK_O0-NOT: builtin_consistent +// CHECK_O0-NOT: !consistent + if (__builtin_consistent(x > 0)) + f(); + + if (x || __builtin_consistent(x != 0)) + g(); +} + +int consistent_switch(int x) { +// CHECK-LABEL: @consistent_switch( +// CHECK-NOT: builtin_consistent +// CHECK: !consistent [[METADATA:.+]] +// CHECK_O0-NOT: builtin_consistent +// CHECK_O0-NOT: !consistent + switch(__builtin_consistent(x)) { + default: + return x; + case 0: + case 1: + case 2: + return 1; + case 3: + return x-1; + }; +} +// CHECK: [[METADATA]] = !{i1 true} + diff --git a/llvm/include/llvm/CodeGen/MachineInstr.h b/llvm/include/llvm/CodeGen/MachineInstr.h index bd72ac23fc9c08e..1fa27026d80f81c 100644 --- a/llvm/include/llvm/CodeGen/MachineInstr.h +++ b/llvm/include/llvm/CodeGen/MachineInstr.h @@ -114,6 +114,7 @@ class MachineInstr // this instruction. Unpredictable = 1 << 16, // Instruction with unpredictable condition. NoConvergent = 1 << 17, // Call does not require convergence guarantees. + Consistent = 1 << 18, // Instruction condition behaves consistently. }; private: diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h index 4df56aac4aa17ba..58680867953f847 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h @@ -398,6 +398,8 @@ struct SDNodeFlags { bool NoFPExcept : 1; // Instructions with attached 'unpredictable' metadata on IR level. bool Unpredictable : 1; + // Instructions with attached 'consistent' metadata on IR level. + bool Consistent : 1; public: /// Default constructor turns off all optimization flags. @@ -405,7 +407,8 @@ struct SDNodeFlags { : NoUnsignedWrap(false), NoSignedWrap(false), Exact(false), NonNeg(false), NoNaNs(false), NoInfs(false), NoSignedZeros(false), AllowReciprocal(false), AllowContract(false), ApproximateFuncs(false), - AllowReassociation(false), NoFPExcept(false), Unpredictable(false) {} + AllowReassociation(false), NoFPExcept(false), Unpredictable(false), + Consistent(false) {} /// Propagate the fast-math-flags from an IR FPMathOperator. void copyFMF(const FPMathOperator &FPMO) { @@ -432,6 +435,7 @@ struct SDNodeFlags { void setAllowReassociation(bool b) { AllowReassociation = b; } void setNoFPExcept(bool b) { NoFPExcept = b; } void setUnpredictable(bool b) { Unpredictable = b; } + void setConsistent(bool b) { Consistent = b; } // These are accessors for each flag. bool hasNoUnsignedWrap() const { return NoUnsignedWrap; } @@ -447,6 +451,7 @@ struct SDNodeFlags { bool hasAllowReassociation() const { return AllowReassociation; } bool hasNoFPExcept() const { return NoFPExcept; } bool hasUnpredictable() const { return Unpredictable; } + bool hasConsistent() const { return Consistent; } /// Clear any flags in this flag set that aren't also set in Flags. All /// flags will be cleared if Flags are undefined. @@ -464,6 +469,7 @@ struct SDNodeFlags { AllowReassociation &= Flags.AllowReassociation; NoFPExcept &= Flags.NoFPExcept; Unpredictable &= Flags.Unpredictable; + Consistent &= Flags.Consistent; } }; diff --git a/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/llvm/include/llvm/CodeGen/TargetInstrInfo.h index 8e7499ac626a747..e893baa90c14dbb 100644 --- a/llvm/include/llvm/CodeGen/TargetInstrInfo.h +++ b/llvm/include/llvm/CodeGen/TargetInstrInfo.h @@ -698,7 +698,8 @@ class TargetInstrInfo : public MCInstrInfo { /// If \p BytesRemoved is non-null, report the change in code size from the /// removed instructions. virtual unsigned removeBranch(MachineBasicBlock &MBB, - int *BytesRemoved = nullptr) const { + int *BytesRemoved = nullptr, + bool *IsConsistent = nullptr) const { llvm_unreachable("Target didn't implement TargetInstrInfo::removeBranch!"); } @@ -718,8 +719,8 @@ class TargetInstrInfo : public MCInstrInfo { virtual unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond, - const DebugLoc &DL, - int *BytesAdded = nullptr) const { + const DebugLoc &DL, int *BytesAdded = nullptr, + bool IsConsistent = false) const { llvm_unreachable("Target didn't implement TargetInstrInfo::insertBranch!"); } diff --git a/llvm/include/llvm/IR/FixedMetadataKinds.def b/llvm/include/llvm/IR/FixedMetadataKinds.def index b375d0f0912060f..6dd0627c028bd84 100644 --- a/llvm/include/llvm/IR/FixedMetadataKinds.def +++ b/llvm/include/llvm/IR/FixedMetadataKinds.def @@ -51,3 +51,4 @@ LLVM_FIXED_MD_KIND(MD_kcfi_type, "kcfi_type", 36) LLVM_FIXED_MD_KIND(MD_pcsections, "pcsections", 37) LLVM_FIXED_MD_KIND(MD_DIAssignID, "DIAssignID", 38) LLVM_FIXED_MD_KIND(MD_coro_outside_frame, "coro.outside.frame", 39) +LLVM_FIXED_MD_KIND(MD_consistent, "consistent", 40) diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h index e3c4e76f90a4cfc..0cd6a7340845733 100644 --- a/llvm/include/llvm/IR/IRBuilder.h +++ b/llvm/include/llvm/IR/IRBuilder.h @@ -1064,15 +1064,18 @@ class IRBuilderBase { //===--------------------------------------------------------------------===// private: - /// Helper to add branch weight and unpredictable metadata onto an - /// instruction. + /// Helper to add branch weight, unpredictable and consistent metadata onto + /// an instruction. /// \returns The annotated instruction. template <typename InstTy> - InstTy *addBranchMetadata(InstTy *I, MDNode *Weights, MDNode *Unpredictable) { + InstTy *addBranchMetadata(InstTy *I, MDNode *Weights, MDNode *Unpred, + MDNode *Consist) { if (Weights) I->setMetadata(LLVMContext::MD_prof, Weights); - if (Unpredictable) - I->setMetadata(LLVMContext::MD_unpredictable, Unpredictable); + if (Unpred) + I->setMetadata(LLVMContext::MD_unpredictable, Unpred); + if (Consist) + I->setMetadata(LLVMContext::MD_consistent, Consist); return I; } @@ -1110,9 +1113,10 @@ class IRBuilderBase { /// instruction. BranchInst *CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, MDNode *BranchWeights = nullptr, - MDNode *Unpredictable = nullptr) { + MDNode *Unpredictable = nullptr, + MDNode *Consistent = nullptr) { return Insert(addBranchMetadata(BranchInst::Create(True, False, Cond), - BranchWeights, Unpredictable)); + BranchWeights, Unpredictable, Consistent)); } /// Create a conditional 'br Cond, TrueDest, FalseDest' @@ -1121,7 +1125,8 @@ class IRBuilderBase { Instruction *MDSrc) { BranchInst *Br = BranchInst::Create(True, False, Cond); if (MDSrc) { - unsigned WL[4] = {LLVMContext::MD_prof, LLVMContext::MD_unpredictable, + unsigned WL[5] = {LLVMContext::MD_prof, LLVMContext::MD_unpredictable, + LLVMContext::MD_consistent, LLVMContext::MD_make_implicit, LLVMContext::MD_dbg}; Br->copyMetadata(*MDSrc, WL); } @@ -1133,9 +1138,10 @@ class IRBuilderBase { /// allocation). SwitchInst *CreateSwitch(Value *V, BasicBlock *Dest, unsigned NumCases = 10, MDNode *BranchWeights = nullptr, - MDNode *Unpredictable = nullptr) { + MDNode *Unpredictable = nullptr, + MDNode *Consistent = nullptr) { return Insert(addBranchMetadata(SwitchInst::Create(V, Dest, NumCases), - BranchWeights, Unpredictable)); + BranchWeights, Unpredictable, Consistent)); } /// Create an indirect branch instruction with the specified address diff --git a/llvm/include/llvm/IR/MDBuilder.h b/llvm/include/llvm/IR/MDBuilder.h index 39165453de16b09..f3ec99568cfa4c6 100644 --- a/llvm/include/llvm/IR/MDBuilder.h +++ b/llvm/include/llvm/IR/MDBuilder.h @@ -67,6 +67,9 @@ class MDBuilder { /// Return metadata specifying that a branch or switch is unpredictable. MDNode *createUnpredictable(); + /// Return metadata specifying that a branch or switch behaves consistently. + MDNode *createConsistent(); + /// Return metadata containing the entry \p Count for a function, a boolean /// \Synthetic indicating whether the counts were synthetized, and the /// GUIDs stored in \p Imports that need to be imported for sample PGO, to diff --git a/llvm/lib/CodeGen/BranchFolding.cpp b/llvm/lib/CodeGen/BranchFolding.cpp index 0801296cab49f8f..e8fca44a816d94a 100644 --- a/llvm/lib/CodeGen/BranchFolding.cpp +++ b/llvm/lib/CodeGen/BranchFolding.cpp @@ -465,8 +465,10 @@ static void FixTail(MachineBasicBlock *CurMBB, MachineBasicBlock *SuccBB, MachineBasicBlock *NextBB = &*I; if (TBB == NextBB && !Cond.empty() && !FBB) { if (!TII->reverseBranchCondition(Cond)) { - TII->removeBranch(*CurMBB); - TII->insertBranch(*CurMBB, SuccBB, nullptr, Cond, dl); + bool IsConsistent = false; + TII->removeBranch(*CurMBB, nullptr, &IsConsistent); + TII->insertBranch(*CurMBB, SuccBB, nullptr, Cond, dl, nullptr, + IsConsistent); return; } } @@ -1116,12 +1118,13 @@ bool BranchFolder::TailMergeBlocks(MachineFunction &MF) { // Remove the unconditional branch at the end, if any. if (TBB && (Cond.empty() || FBB)) { + bool IsConsistent = false; DebugLoc dl = PBB->findBranchDebugLoc(); - TII->removeBranch(*PBB); + TII->removeBranch(*PBB, nullptr, &IsConsistent); if (!Cond.empty()) // reinsert conditional branch only, for now - TII->insertBranch(*PBB, (TBB == IBB) ? FBB : TBB, nullptr, - NewCond, dl); + ... [truncated] `````````` </details> https://github.com/llvm/llvm-project/pull/72175 _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
