HsiangKai created this revision.
HsiangKai added reviewers: craig.topper, rogfer01, frasercrmck.
Herald added subscribers: achieveartificialintelligence, StephenFan, vkmr,
dexonsmith, evandro, luismarques, apazos, sameer.abuasal, s.egerton, Jim,
benna, psnobl, jocewei, PkmX, the_o, brucehoult, MartinMosbeck, edward-jones,
zzheng, jrtc27, shiva0217, kito-cheng, niosHD, sabuasal, simoncook, johnrusso,
rbar, asb.
HsiangKai requested review of this revision.
Herald added subscribers: cfe-commits, MaskRay.
Herald added a project: clang.
In the original design, we levarage _mt intrinsics to define macros for
_m intrinsics. Such as,
__builtin_rvv_vadd_vv_i8m1_mt((vbool8_t)(op0), (vint8m1_t)(op1),
(vint8m1_t)(op2), (vint8m1_t)(op3), (size_t)(op4), (size_t)VE_TAIL_AGNOSTIC)
However, we could not define generic interface for mask intrinsics any
more due to clang_builtin_alias only accepts clang builtins as its
argument.
In the example,
__rvv_overloaded
__attribute__((clang_builtin_alias(__builtin_rvv_vadd_vv_i8m1_mt)))
vint8m1_t vadd(vbool8_t op0, vint8m1_t op1, vint8m1_t op2, vint8m1_t
op3, size_t op4, size_t op5);
op5 is the tail policy argument. When users want to use vadd generic
interface for masked vector add, they need to specify tail policy in the
previous design. In this patch, we define _m intrinsics as clang
builtins to solve the problem.
Repository:
rG LLVM Github Monorepo
https://reviews.llvm.org/D110684
Files:
clang/include/clang/Basic/IdentifierTable.h
clang/include/clang/Basic/riscv_vector.td
clang/test/CodeGen/RISCV/rvv-intrinsics/vadd-policy.c
clang/utils/TableGen/RISCVVEmitter.cpp
Index: clang/utils/TableGen/RISCVVEmitter.cpp
===================================================================
--- clang/utils/TableGen/RISCVVEmitter.cpp
+++ clang/utils/TableGen/RISCVVEmitter.cpp
@@ -158,6 +158,7 @@
bool HasMaskedOffOperand;
bool HasVL;
bool HasPolicy;
+ bool IsPolicyIntrinsic;
bool HasNoMaskedOverloaded;
bool HasAutoDef; // There is automiatic definition in header
std::string ManualCodegen;
@@ -173,7 +174,8 @@
RVVIntrinsic(StringRef Name, StringRef Suffix, StringRef MangledName,
StringRef MangledSuffix, StringRef IRName, bool HasSideEffects,
bool IsMask, bool HasMaskedOffOperand, bool HasVL,
- bool HasPolicy, bool HasNoMaskedOverloaded, bool HasAutoDef,
+ bool HasPolicy, bool IsPolicyIntrinsic,
+ bool HasNoMaskedOverloaded, bool HasAutoDef,
StringRef ManualCodegen, const RVVTypes &Types,
const std::vector<int64_t> &IntrinsicTypes,
StringRef RequiredExtension, unsigned NF);
@@ -185,6 +187,7 @@
bool hasMaskedOffOperand() const { return HasMaskedOffOperand; }
bool hasVL() const { return HasVL; }
bool hasPolicy() const { return HasPolicy; }
+ bool isPolicyIntrinsic() const { return IsPolicyIntrinsic; }
bool hasNoMaskedOverloaded() const { return HasNoMaskedOverloaded; }
bool hasManualCodegen() const { return !ManualCodegen.empty(); }
bool hasAutoDef() const { return HasAutoDef; }
@@ -201,9 +204,6 @@
// init the RVVIntrinsic ID and IntrinsicTypes.
void emitCodeGenSwitchBody(raw_ostream &o) const;
- // Emit the define macors for mask intrinsics using _mt intrinsics.
- void emitIntrinsicMaskMacro(raw_ostream &o) const;
-
// Emit the macros for mapping C/C++ intrinsic function to builtin functions.
void emitIntrinsicMacro(raw_ostream &o) const;
@@ -764,14 +764,16 @@
StringRef NewMangledName, StringRef MangledSuffix,
StringRef IRName, bool HasSideEffects, bool IsMask,
bool HasMaskedOffOperand, bool HasVL, bool HasPolicy,
- bool HasNoMaskedOverloaded, bool HasAutoDef,
- StringRef ManualCodegen, const RVVTypes &OutInTypes,
+ bool IsPolicyIntrinsic, bool HasNoMaskedOverloaded,
+ bool HasAutoDef, StringRef ManualCodegen,
+ const RVVTypes &OutInTypes,
const std::vector<int64_t> &NewIntrinsicTypes,
StringRef RequiredExtension, unsigned NF)
: IRName(IRName), HasSideEffects(HasSideEffects), IsMask(IsMask),
HasMaskedOffOperand(HasMaskedOffOperand), HasVL(HasVL),
- HasPolicy(HasPolicy), HasNoMaskedOverloaded(HasNoMaskedOverloaded),
- HasAutoDef(HasAutoDef), ManualCodegen(ManualCodegen.str()), NF(NF) {
+ HasPolicy(HasPolicy), IsPolicyIntrinsic(IsPolicyIntrinsic),
+ HasNoMaskedOverloaded(HasNoMaskedOverloaded), HasAutoDef(HasAutoDef),
+ ManualCodegen(ManualCodegen.str()), NF(NF) {
// Init Name and MangledName
Name = NewName.str();
@@ -785,7 +787,7 @@
MangledName += "_" + MangledSuffix.str();
if (IsMask) {
Name += "_m";
- if (HasPolicy)
+ if (IsPolicyIntrinsic)
Name += "t";
}
// Init RISC-V extensions
@@ -839,9 +841,15 @@
if (isMask()) {
if (hasVL()) {
- if (hasPolicy())
- OS << " std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2);\n";
- else
+ if (hasPolicy()) {
+ if (isPolicyIntrinsic())
+ OS << " std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2);\n";
+ else {
+ OS << " std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);\n";
+ OS << " Ops.push_back(ConstantInt::get(Ops.back()->getType(), "
+ "1));\n";
+ }
+ } else
OS << " std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);\n";
} else {
OS << " std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());\n";
@@ -882,24 +890,6 @@
OS << ")\n";
}
-void RVVIntrinsic::emitIntrinsicMaskMacro(raw_ostream &OS) const {
- OS << "#define " << getName().drop_back() << "(";
- if (!InputTypes.empty()) {
- ListSeparator LS;
- for (unsigned i = 0, e = InputTypes.size() - 1; i != e; ++i)
- OS << LS << "op" << i;
- }
- OS << ") \\\n";
- OS << "__builtin_rvv_" << getName() << "(";
- ListSeparator LS;
- if (!InputTypes.empty()) {
- for (unsigned i = 0, e = InputTypes.size() - 1; i != e; ++i)
- OS << LS << "(" << InputTypes[i]->getTypeStr() << ")(op" << i << ")";
- }
- OS << LS << "(size_t)VE_TAIL_AGNOSTIC";
- OS << ")\n";
-}
-
void RVVIntrinsic::emitMangledFuncDef(raw_ostream &OS) const {
OS << "__attribute__((clang_builtin_alias(";
OS << "__builtin_rvv_" << getName() << ")))\n";
@@ -1014,12 +1004,6 @@
Inst.emitIntrinsicMacro(OS);
});
- // Use _mt to implement _m intrinsics.
- emitArchMacroAndBody(Defs, OS, [](raw_ostream &OS, const RVVIntrinsic &Inst) {
- if (Inst.isMask() && Inst.hasPolicy())
- Inst.emitIntrinsicMaskMacro(OS);
- });
-
OS << "#define __riscv_v_intrinsic_overloading 1\n";
// Print Overloaded APIs
@@ -1065,7 +1049,9 @@
std::stable_sort(Defs.begin(), Defs.end(),
[](const std::unique_ptr<RVVIntrinsic> &A,
const std::unique_ptr<RVVIntrinsic> &B) {
- return A->getIRName() < B->getIRName();
+ return A->getIRName() < B->getIRName() ||
+ (A->getIRName() == B->getIRName() &&
+ A->isPolicyIntrinsic() == false);
});
// Print switch body when the ir name or ManualCodegen changes from previous
// iteration.
@@ -1073,7 +1059,8 @@
for (auto &Def : Defs) {
StringRef CurIRName = Def->getIRName();
if (CurIRName != PrevDef->getIRName() ||
- (Def->getManualCodegen() != PrevDef->getManualCodegen())) {
+ (Def->getManualCodegen() != PrevDef->getManualCodegen()) ||
+ (Def->isPolicyIntrinsic() != PrevDef->isPolicyIntrinsic())) {
PrevDef->emitCodeGenSwitchBody(OS);
}
PrevDef = Def.get();
@@ -1183,8 +1170,9 @@
ProtoMaskSeq.push_back("z");
}
+ SmallVector<std::string> ProtoMaskPolicySeq = ProtoMaskSeq;
if (HasPolicy) {
- ProtoMaskSeq.push_back("z");
+ ProtoMaskPolicySeq.push_back("z");
}
// Create Intrinsics for each type and LMUL.
@@ -1201,8 +1189,9 @@
Out.push_back(std::make_unique<RVVIntrinsic>(
Name, SuffixStr, MangledName, MangledSuffixStr, IRName,
HasSideEffects, /*IsMask=*/false, /*HasMaskedOffOperand=*/false,
- HasVL, HasPolicy, HasNoMaskedOverloaded, HasAutoDef, ManualCodegen,
- Types.getValue(), IntrinsicTypes, RequiredExtension, NF));
+ HasVL, HasPolicy, /*IsPolicyIntrinsic*/ false,
+ HasNoMaskedOverloaded, HasAutoDef, ManualCodegen, Types.getValue(),
+ IntrinsicTypes, RequiredExtension, NF));
if (HasMask) {
// Create a mask intrinsic
Optional<RVVTypes> MaskTypes =
@@ -1210,8 +1199,18 @@
Out.push_back(std::make_unique<RVVIntrinsic>(
Name, SuffixStr, MangledName, MangledSuffixStr, IRNameMask,
HasSideEffects, /*IsMask=*/true, HasMaskedOffOperand, HasVL,
- HasPolicy, HasNoMaskedOverloaded, HasAutoDef, ManualCodegenMask,
- MaskTypes.getValue(), IntrinsicTypes, RequiredExtension, NF));
+ HasPolicy, /*IsPolicyIntrinsic=*/false, HasNoMaskedOverloaded,
+ HasAutoDef, ManualCodegenMask, MaskTypes.getValue(),
+ IntrinsicTypes, RequiredExtension, NF));
+ if (HasPolicy) {
+ MaskTypes = computeTypes(I, Log2LMUL, NF, ProtoMaskPolicySeq);
+ Out.push_back(std::make_unique<RVVIntrinsic>(
+ Name, SuffixStr, MangledName, MangledSuffixStr, IRNameMask,
+ HasSideEffects, /*IsMask=*/true, HasMaskedOffOperand, HasVL,
+ HasPolicy, /*IsPolicyIntrinsic=*/true, HasNoMaskedOverloaded,
+ HasAutoDef, ManualCodegenMask, MaskTypes.getValue(),
+ IntrinsicTypes, RequiredExtension, NF));
+ }
}
} // end for Log2LMULList
} // end for TypeRange
Index: clang/test/CodeGen/RISCV/rvv-intrinsics/vadd-policy.c
===================================================================
--- clang/test/CodeGen/RISCV/rvv-intrinsics/vadd-policy.c
+++ clang/test/CodeGen/RISCV/rvv-intrinsics/vadd-policy.c
@@ -42,3 +42,30 @@
vint8m1_t test_vadd_ta(vbool8_t mask, vint8m1_t maskedoff, vint8m1_t op1, vint8m1_t op2, size_t vl) {
return vadd_vv_i8m1_mt(mask, maskedoff, op1, op2, vl, VE_TAIL_AGNOSTIC);
}
+
+// CHECK-RV64-LABEL: @test_generic_vadd(
+// CHECK-RV64-NEXT: entry:
+// CHECK-RV64-NEXT: [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vadd.nxv8i8.nxv8i8.i64(<vscale x 8 x i8> [[OP1:%.*]], <vscale x 8 x i8> [[OP2:%.*]], i64 [[VL:%.*]])
+// CHECK-RV64-NEXT: ret <vscale x 8 x i8> [[TMP0]]
+//
+vint8m1_t test_generic_vadd(vint8m1_t op1, vint8m1_t op2, size_t vl) {
+ return vadd(op1, op2, vl);
+}
+
+// CHECK-RV64-LABEL: @test_generic_vadd_m(
+// CHECK-RV64-NEXT: entry:
+// CHECK-RV64-NEXT: [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vadd.mask.nxv8i8.nxv8i8.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8> [[OP1:%.*]], <vscale x 8 x i8> [[OP2:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 1)
+// CHECK-RV64-NEXT: ret <vscale x 8 x i8> [[TMP0]]
+//
+vint8m1_t test_generic_vadd_m(vbool8_t mask, vint8m1_t maskedoff, vint8m1_t op1, vint8m1_t op2, size_t vl) {
+ return vadd(mask, maskedoff, op1, op2, vl);
+}
+
+// CHECK-RV64-LABEL: @test_generic_vadd_mt(
+// CHECK-RV64-NEXT: entry:
+// CHECK-RV64-NEXT: [[TMP0:%.*]] = call <vscale x 8 x i8> @llvm.riscv.vadd.mask.nxv8i8.nxv8i8.i64(<vscale x 8 x i8> [[MASKEDOFF:%.*]], <vscale x 8 x i8> [[OP1:%.*]], <vscale x 8 x i8> [[OP2:%.*]], <vscale x 8 x i1> [[MASK:%.*]], i64 [[VL:%.*]], i64 0)
+// CHECK-RV64-NEXT: ret <vscale x 8 x i8> [[TMP0]]
+//
+vint8m1_t test_generic_vadd_mt(vbool8_t mask, vint8m1_t maskedoff, vint8m1_t op1, vint8m1_t op2, size_t vl) {
+ return vadd(mask, maskedoff, op1, op2, vl, VE_TAIL_UNDISTURBED);
+}
Index: clang/include/clang/Basic/riscv_vector.td
===================================================================
--- clang/include/clang/Basic/riscv_vector.td
+++ clang/include/clang/Basic/riscv_vector.td
@@ -599,9 +599,13 @@
IntrinsicTypes = {ResultType, Ops[1]->getType()};
Ops[0] = Builder.CreateBitCast(Ops[0], ResultType->getPointerTo());
}],
- ManualCodegenMask= [{
+ ManualCodegenMask = [{
// Move mask to right before vl.
- std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2);
+ if (Ops.size() == 4) {
+ std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
+ Ops.push_back(ConstantInt::get(Ops.back()->getType(), 1));
+ } else if (Ops.size() == 5)
+ std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2);
IntrinsicTypes = {ResultType, Ops[3]->getType()};
Ops[1] = Builder.CreateBitCast(Ops[1], ResultType->getPointerTo());
}] in {
@@ -644,7 +648,11 @@
ManualCodegenMask = [{
{
// Move mask to right before vl.
- std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2);
+ if (Ops.size() == 5) {
+ std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
+ Ops.push_back(ConstantInt::get(Ops.back()->getType(), 1));
+ } else if (Ops.size() == 6)
+ std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2);
IntrinsicTypes = {ResultType, Ops[4]->getType()};
Ops[1] = Builder.CreateBitCast(Ops[1], ResultType->getPointerTo());
Value *NewVL = Ops[2];
@@ -681,7 +689,11 @@
}],
ManualCodegenMask= [{
// Move mask to right before vl.
- std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2);
+ if (Ops.size() == 5) {
+ std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
+ Ops.push_back(ConstantInt::get(Ops.back()->getType(), 1));
+ } else if (Ops.size() == 6)
+ std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2);
IntrinsicTypes = {ResultType, Ops[4]->getType()};
Ops[1] = Builder.CreateBitCast(Ops[1], ResultType->getPointerTo());
}] in {
@@ -701,7 +713,11 @@
}],
ManualCodegenMask = [{
// Move mask to right before vl.
- std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2);
+ if (Ops.size() == 5) {
+ std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
+ Ops.push_back(ConstantInt::get(Ops.back()->getType(), 1));
+ } else if (Ops.size() == 6)
+ std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2);
IntrinsicTypes = {ResultType, Ops[2]->getType(), Ops[4]->getType()};
Ops[1] = Builder.CreateBitCast(Ops[1], ResultType->getPointerTo());
}] in {
@@ -885,7 +901,10 @@
Operands.push_back(Ops[2 * NF + 1]);
Operands.push_back(Ops[NF]);
Operands.push_back(Ops[2 * NF + 2]);
- Operands.push_back(Ops[2 * NF + 3]);
+ if (Ops.size() == 2 * NF + 4)
+ Operands.push_back(Ops[2 * NF + 3]);
+ else
+ Operands.push_back(ConstantInt::get(Ops.back()->getType(), 1));
assert(Operands.size() == NF + 4);
llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
llvm::Value *LoadValue = Builder.CreateCall(F, Operands, "");
@@ -958,7 +977,10 @@
Operands.push_back(Ops[2 * NF + 1]);
Operands.push_back(Ops[NF]);
Operands.push_back(Ops[2 * NF + 3]);
- Operands.push_back(Ops[2 * NF + 4]);
+ if (Ops.size() == 2 * NF + 5)
+ Operands.push_back(Ops[2 * NF + 4]);
+ else
+ Operands.push_back(ConstantInt::get(Ops.back()->getType(), 1));
Value *NewVL = Ops[2 * NF + 2];
assert(Operands.size() == NF + 4);
llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
@@ -1032,7 +1054,10 @@
Operands.push_back(Ops[2 * NF + 2]);
Operands.push_back(Ops[NF]);
Operands.push_back(Ops[2 * NF + 3]);
- Operands.push_back(Ops[2 * NF + 4]);
+ if (Ops.size() == 2 * NF + 5)
+ Operands.push_back(Ops[2 * NF + 4]);
+ else
+ Operands.push_back(ConstantInt::get(Ops.back()->getType(), 1));
assert(Operands.size() == NF + 5);
llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
llvm::Value *LoadValue = Builder.CreateCall(F, Operands, "");
@@ -1099,7 +1124,10 @@
Operands.push_back(Ops[2 * NF + 2]);
Operands.push_back(Ops[NF]);
Operands.push_back(Ops[2 * NF + 3]);
- Operands.push_back(Ops[2 * NF + 4]);
+ if (Ops.size() == 2 * NF + 5)
+ Operands.push_back(Ops[2 * NF + 4]);
+ else
+ Operands.push_back(ConstantInt::get(Ops.back()->getType(), 1));
assert(Operands.size() == NF + 5);
llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
llvm::Value *LoadValue = Builder.CreateCall(F, Operands, "");
@@ -1318,7 +1346,11 @@
}],
ManualCodegenMask = [{
{
- std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2);
+ if (Ops.size() == 4) {
+ std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
+ Ops.push_back(ConstantInt::get(Ops.back()->getType(), 1));
+ } else if (Ops.size() == 5)
+ std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2);
// maskedoff, op1, mask, vl
IntrinsicTypes = {ResultType,
cast<llvm::VectorType>(ResultType)->getElementType(),
@@ -1348,7 +1380,11 @@
}],
ManualCodegenMask = [{
{
- std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2);
+ if (Ops.size() == 4) {
+ std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
+ Ops.push_back(ConstantInt::get(Ops.back()->getType(), 1));
+ } else if (Ops.size() == 5)
+ std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2);
// maskedoff, op1, mask, vl
IntrinsicTypes = {ResultType,
cast<llvm::VectorType>(ResultType)->getElementType(),
@@ -1395,7 +1431,11 @@
}],
ManualCodegenMask = [{
{
- std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2);
+ if (Ops.size() == 4) {
+ std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
+ Ops.push_back(ConstantInt::get(Ops.back()->getType(), 1));
+ } else if (Ops.size() == 5)
+ std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2);
// maskedoff, op1, mask, vl
IntrinsicTypes = {ResultType,
Ops[1]->getType(),
@@ -1427,7 +1467,11 @@
}],
ManualCodegenMask = [{
{
- std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2);
+ if (Ops.size() == 4) {
+ std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
+ Ops.push_back(ConstantInt::get(Ops.back()->getType(), 1));
+ } else if (Ops.size() == 5)
+ std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2);
// maskedoff, op1, mask, vl
IntrinsicTypes = {ResultType,
Ops[1]->getType(),
@@ -1462,7 +1506,11 @@
}],
ManualCodegenMask = [{
{
- std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2);
+ if (Ops.size() == 4) {
+ std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
+ Ops.push_back(ConstantInt::get(Ops.back()->getType(), 1));
+ } else if (Ops.size() == 5)
+ std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2);
// maskedoff, op1, mask, vl
IntrinsicTypes = {ResultType,
Ops[1]->getType(),
Index: clang/include/clang/Basic/IdentifierTable.h
===================================================================
--- clang/include/clang/Basic/IdentifierTable.h
+++ clang/include/clang/Basic/IdentifierTable.h
@@ -56,7 +56,7 @@
/// of a pointer to one of these classes.
enum { IdentifierInfoAlignment = 8 };
-static constexpr int ObjCOrBuiltinIDBits = 16;
+static constexpr int ObjCOrBuiltinIDBits = 17;
/// One of these records is kept for each identifier that
/// is lexed. This contains information about whether the token was \#define'd,
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
