Author: Piyou Chen Date: 2022-12-21T20:28:12-08:00 New Revision: 5370255ff18ac101a73685b77615148142ee7552
URL: https://github.com/llvm/llvm-project/commit/5370255ff18ac101a73685b77615148142ee7552 DIFF: https://github.com/llvm/llvm-project/commit/5370255ff18ac101a73685b77615148142ee7552.diff LOG: [RISCV] Merge Masked and unMasked RVV manual codegen RVV intrinsic function will generate riscv_vector_builtin_cg.inc for CGBuiltin.cpp to produce the corresponding RVV intrinsic LLVM IR. In this stage, riscv_vector.td will describe the bunch of manual codegen C++ code to tell CGBuiltin how to handle these instructions. In this patch, we merge the masked RVV manual codegen and unmasked RVV manual codegen to reduce the number of manual codegen, and make more policy addition easier in the future. This is a clean-up job that will not affect the RVV intrinsic functionality. Reviewed By: kito-cheng Differential Revision: https://reviews.llvm.org/D140361 Added: Modified: clang/include/clang/Basic/riscv_vector.td clang/lib/CodeGen/CGBuiltin.cpp clang/utils/TableGen/RISCVVEmitter.cpp Removed: ################################################################################ diff --git a/clang/include/clang/Basic/riscv_vector.td b/clang/include/clang/Basic/riscv_vector.td index 90e18df0403e0..7dbd331b2c5ec 100644 --- a/clang/include/clang/Basic/riscv_vector.td +++ b/clang/include/clang/Basic/riscv_vector.td @@ -217,7 +217,6 @@ class RVVBuiltin<string suffix, string prototype, string type_range, // Manual code in clang codegen riscv_vector_builtin_cg.inc code ManualCodegen = [{}]; - code MaskedManualCodegen = [{}]; // When emit the automatic clang codegen, it describes what types we have to use // to obtain the specific LLVM intrinsic. -1 means the return type, otherwise, @@ -627,31 +626,18 @@ multiclass RVVVLEFFBuiltin<list<string> types> { UnMaskedPolicyScheme = HasPassthruOperand, ManualCodegen = [{ { - if (DefaultPolicy == TAIL_AGNOSTIC) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); - IntrinsicTypes = {ResultType, Ops[3]->getType()}; - Ops[1] = Builder.CreateBitCast(Ops[1], ResultType->getPointerTo()); - Value *NewVL = Ops[2]; - Ops.erase(Ops.begin() + 2); - llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); - llvm::Value *LoadValue = Builder.CreateCall(F, Ops, ""); - llvm::Value *V = Builder.CreateExtractValue(LoadValue, {0}); - // Store new_vl. - clang::CharUnits Align = - CGM.getNaturalPointeeTypeAlignment(E->getArg(1)->getType()); - llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {1}); - Builder.CreateStore(Val, Address(NewVL, Val->getType(), Align)); - return V; - } - }], - MaskedManualCodegen = [{ - { - // Move mask to right before vl. - std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); - if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); - Ops.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy)); - IntrinsicTypes = {ResultType, Ops[4]->getType()}; + if (IsMasked) { + // Move mask to right before vl. + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + Ops.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy)); + IntrinsicTypes = {ResultType, Ops[4]->getType()}; + } else { + if (DefaultPolicy == TAIL_AGNOSTIC) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + IntrinsicTypes = {ResultType, Ops[3]->getType()}; + } Ops[1] = Builder.CreateBitCast(Ops[1], ResultType->getPointerTo()); Value *NewVL = Ops[2]; Ops.erase(Ops.begin() + 2); @@ -659,8 +645,11 @@ multiclass RVVVLEFFBuiltin<list<string> types> { llvm::Value *LoadValue = Builder.CreateCall(F, Ops, ""); llvm::Value *V = Builder.CreateExtractValue(LoadValue, {0}); // Store new_vl. - clang::CharUnits Align = CGM.getNaturalPointeeTypeAlignment( - E->getArg(E->getNumArgs()-2)->getType()); + clang::CharUnits Align; + if (IsMasked) + Align = CGM.getNaturalPointeeTypeAlignment(E->getArg(E->getNumArgs()-2)->getType()); + else + Align = CGM.getNaturalPointeeTypeAlignment(E->getArg(1)->getType()); llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {1}); Builder.CreateStore(Val, Address(NewVL, Val->getType(), Align)); return V; @@ -720,16 +709,18 @@ multiclass RVVIndexedLoad<string op> { let HasMaskedOffOperand = false, MaskedPolicyScheme = NonePolicy, ManualCodegen = [{ - // Builtin: (ptr, value, vl). Intrinsic: (value, ptr, vl) - std::swap(Ops[0], Ops[1]); - Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo()); - IntrinsicTypes = {Ops[0]->getType(), Ops[2]->getType()}; - }], - MaskedManualCodegen= [{ - // Builtin: (mask, ptr, value, vl). Intrinsic: (value, ptr, mask, vl) - std::swap(Ops[0], Ops[2]); + if (IsMasked) { + // Builtin: (mask, ptr, value, vl). Intrinsic: (value, ptr, mask, vl) + std::swap(Ops[0], Ops[2]); + } else { + // Builtin: (ptr, value, vl). Intrinsic: (value, ptr, vl) + std::swap(Ops[0], Ops[1]); + } Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo()); - IntrinsicTypes = {Ops[0]->getType(), Ops[3]->getType()}; + if (IsMasked) + IntrinsicTypes = {Ops[0]->getType(), Ops[3]->getType()}; + else + IntrinsicTypes = {Ops[0]->getType(), Ops[2]->getType()}; }] in { class RVVVSEMaskBuiltin : RVVBuiltin<"m", "0PUem", "c"> { let Name = "vsm_v"; @@ -757,16 +748,18 @@ multiclass RVVVSSEBuiltin<list<string> types> { HasMaskedOffOperand = false, MaskedPolicyScheme = NonePolicy, ManualCodegen = [{ - // Builtin: (ptr, stride, value, vl). Intrinsic: (value, ptr, stride, vl) - std::rotate(Ops.begin(), Ops.begin() + 2, Ops.begin() + 3); - Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo()); - IntrinsicTypes = {Ops[0]->getType(), Ops[3]->getType()}; - }], - MaskedManualCodegen= [{ - // Builtin: (mask, ptr, stride, value, vl). Intrinsic: (value, ptr, stride, mask, vl) - std::swap(Ops[0], Ops[3]); + if (IsMasked) { + // Builtin: (mask, ptr, stride, value, vl). Intrinsic: (value, ptr, stride, mask, vl) + std::swap(Ops[0], Ops[3]); + } else { + // Builtin: (ptr, stride, value, vl). Intrinsic: (value, ptr, stride, vl) + std::rotate(Ops.begin(), Ops.begin() + 2, Ops.begin() + 3); + } Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo()); - IntrinsicTypes = {Ops[0]->getType(), Ops[4]->getType()}; + if (IsMasked) + IntrinsicTypes = {Ops[0]->getType(), Ops[4]->getType()}; + else + IntrinsicTypes = {Ops[0]->getType(), Ops[3]->getType()}; }] in { foreach type = types in { def : RVVBuiltin<"v", "0Petv", type>; @@ -781,16 +774,18 @@ multiclass RVVIndexedStore<string op> { let HasMaskedOffOperand = false, MaskedPolicyScheme = NonePolicy, ManualCodegen = [{ - // Builtin: (ptr, index, value, vl). Intrinsic: (value, ptr, index, vl) - std::rotate(Ops.begin(), Ops.begin() + 2, Ops.begin() + 3); - Ops[1] = Builder.CreateBitCast(Ops[1],Ops[0]->getType()->getPointerTo()); - IntrinsicTypes = {Ops[0]->getType(), Ops[2]->getType(), Ops[3]->getType()}; - }], - MaskedManualCodegen= [{ - // Builtin: (mask, ptr, index, value, vl). Intrinsic: (value, ptr, index, mask, vl) - std::swap(Ops[0], Ops[3]); + if (IsMasked) { + // Builtin: (mask, ptr, index, value, vl). Intrinsic: (value, ptr, index, mask, vl) + std::swap(Ops[0], Ops[3]); + } else { + // Builtin: (ptr, index, value, vl). Intrinsic: (value, ptr, index, vl) + std::rotate(Ops.begin(), Ops.begin() + 2, Ops.begin() + 3); + } Ops[1] = Builder.CreateBitCast(Ops[1], Ops[0]->getType()->getPointerTo()); - IntrinsicTypes = {Ops[0]->getType(), Ops[2]->getType(), Ops[4]->getType()}; + if (IsMasked) + IntrinsicTypes = {Ops[0]->getType(), Ops[2]->getType(), Ops[4]->getType()}; + else + IntrinsicTypes = {Ops[0]->getType(), Ops[2]->getType(), Ops[3]->getType()}; }] in { foreach type = TypeList in { foreach eew_list = EEWList[0-2] in { @@ -846,57 +841,43 @@ multiclass RVVUnitStridedSegLoad<string op> { ManualCodegen = [{ { ResultType = ConvertType(E->getArg(0)->getType()->getPointeeType()); - // TA builtin: (val0 address, val1 address, ..., ptr, vl) - // TU builtin: (val0 address, ..., passthru0, ..., ptr, vl) IntrinsicTypes = {ResultType, Ops.back()->getType()}; - // intrinsic: (passthru0, passthru1, ..., ptr, vl) SmallVector<llvm::Value*, 10> Operands; - if (DefaultPolicy == TAIL_AGNOSTIC) { - for (unsigned I = 0; I < NF; ++I) - Operands.push_back(llvm::PoisonValue::get(ResultType)); - Operands.push_back(Ops[NF]); - Operands.push_back(Ops[NF + 1]); - } else { - for (unsigned I = 0; I < NF; ++I) - Operands.push_back(Ops[NF + I]); - Operands.push_back(Ops[2 * NF]); - Operands.push_back(Ops[2 * NF + 1]); - } - llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); - llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); - clang::CharUnits Align = - CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType()); - llvm::Value *V; - for (unsigned I = 0; I < NF; ++I) { - llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {I}); - V = Builder.CreateStore(Val, Address(Ops[I], Val->getType(), Align)); - } - return V; - } - }], - MaskedManualCodegen = [{ - { - // TAMA builtin: (val0 address, ..., mask, ptr, vl) - // builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, vl) - // intrinsic: (maskedoff0, ..., ptr, mask, vl) - ResultType = ConvertType(E->getArg(0)->getType()->getPointeeType()); - IntrinsicTypes = {ResultType, Ops.back()->getType()}; - SmallVector<llvm::Value*, 12> Operands; - if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC) { - for (unsigned I = 0; I < NF; ++I) - Operands.push_back(llvm::PoisonValue::get(ResultType)); - Operands.push_back(Ops[NF + 1]); - Operands.push_back(Ops[NF]); - Operands.push_back(Ops[NF + 2]); + if (IsMasked) { + // TAMA builtin: (val0 address, ..., mask, ptr, vl) + // builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, vl) + // intrinsic: (maskedoff0, ..., ptr, mask, vl) + if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC) { + for (unsigned I = 0; I < NF; ++I) + Operands.push_back(llvm::PoisonValue::get(ResultType)); + Operands.push_back(Ops[NF + 1]); + Operands.push_back(Ops[NF]); + Operands.push_back(Ops[NF + 2]); + } else { + for (unsigned I = 0; I < NF; ++I) + Operands.push_back(Ops[NF + I + 1]); + Operands.push_back(Ops[2 * NF + 1]); + Operands.push_back(Ops[NF]); + Operands.push_back(Ops[2 * NF + 2]); + } + Operands.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy)); + assert(Operands.size() == NF + 4); } else { - for (unsigned I = 0; I < NF; ++I) - Operands.push_back(Ops[NF + I + 1]); - Operands.push_back(Ops[2 * NF + 1]); - Operands.push_back(Ops[NF]); - Operands.push_back(Ops[2 * NF + 2]); + // TA builtin: (val0 address, val1 address, ..., ptr, vl) + // TU builtin: (val0 address, ..., passthru0, ..., ptr, vl) + // intrinsic: (passthru0, passthru1, ..., ptr, vl) + if (DefaultPolicy == TAIL_AGNOSTIC) { + for (unsigned I = 0; I < NF; ++I) + Operands.push_back(llvm::PoisonValue::get(ResultType)); + Operands.push_back(Ops[NF]); + Operands.push_back(Ops[NF + 1]); + } else { + for (unsigned I = 0; I < NF; ++I) + Operands.push_back(Ops[NF + I]); + Operands.push_back(Ops[2 * NF]); + Operands.push_back(Ops[2 * NF + 1]); + } } - Operands.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy)); - assert(Operands.size() == NF + 4); llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); clang::CharUnits Align = @@ -908,7 +889,7 @@ multiclass RVVUnitStridedSegLoad<string op> { } return V; } - }] in { + }] in { defvar PV = PVString<nf, /*signed=*/true>.S; defvar PUV = PVString<nf, /*signed=*/false>.S; def : RVVBuiltin<"v", "0" # PV # "PCe", type>; @@ -936,65 +917,50 @@ multiclass RVVUnitStridedSegLoadFF<string op> { NF = nf, ManualCodegen = [{ { - // TA builtin: (val0 address, val1 address, ..., ptr, new_vl, vl) - // TU builtin: (val0 address, ..., passthru0, ..., ptr, new_vl, vl) ResultType = ConvertType(E->getArg(0)->getType()->getPointeeType()); IntrinsicTypes = {ResultType, Ops.back()->getType()}; - // intrinsic: (passthru0, passthru1, ..., ptr, vl) SmallVector<llvm::Value*, 12> Operands; Value *NewVL; - if (DefaultPolicy == TAIL_AGNOSTIC) { - for (unsigned I = 0; I < NF; ++I) - Operands.push_back(llvm::PoisonValue::get(ResultType)); - Operands.push_back(Ops[NF]); - Operands.push_back(Ops[NF + 2]); - NewVL = Ops[NF + 1]; - } else { - for (unsigned I = 0; I < NF; ++I) - Operands.push_back(Ops[NF + I]); - Operands.push_back(Ops[2 * NF]); - Operands.push_back(Ops[2 * NF + 2]); - NewVL = Ops[2 * NF + 1]; - } - llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); - llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); - clang::CharUnits Align = - CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType()); - for (unsigned I = 0; I < NF; ++I) { - llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {I}); - Builder.CreateStore(Val, Address(Ops[I], Val->getType(), Align)); - } - // Store new_vl. - llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {NF}); - return Builder.CreateStore(Val, Address(NewVL, Val->getType(), Align)); - } - }], - MaskedManualCodegen = [{ - { - // TAMA builtin: (val0 address, ..., mask, ptr, new_vl, vl) - // builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, new_vl, vl) - // intrinsic: (maskedoff0, ..., ptr, mask, vl) - ResultType = ConvertType(E->getArg(0)->getType()->getPointeeType()); - IntrinsicTypes = {ResultType, Ops.back()->getType()}; - SmallVector<llvm::Value*, 12> Operands; - Value *NewVL; - if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC) { - for (unsigned I = 0; I < NF; ++I) - Operands.push_back(llvm::PoisonValue::get(ResultType)); - Operands.push_back(Ops[NF + 1]); - Operands.push_back(Ops[NF]); - Operands.push_back(Ops[NF + 3]); - NewVL = Ops[NF + 2]; + + if (IsMasked) { + // TAMA builtin: (val0 address, ..., mask, ptr, new_vl, vl) + // builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, new_vl, vl) + // intrinsic: (maskedoff0, ..., ptr, mask, vl) + if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC) { + for (unsigned I = 0; I < NF; ++I) + Operands.push_back(llvm::PoisonValue::get(ResultType)); + Operands.push_back(Ops[NF + 1]); + Operands.push_back(Ops[NF]); + Operands.push_back(Ops[NF + 3]); + NewVL = Ops[NF + 2]; + } else { + for (unsigned I = 0; I < NF; ++I) + Operands.push_back(Ops[NF + I + 1]); + Operands.push_back(Ops[2 * NF + 1]); + Operands.push_back(Ops[NF]); + Operands.push_back(Ops[2 * NF + 3]); + NewVL = Ops[2 * NF + 2]; + } + Operands.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy)); + assert(Operands.size() == NF + 4); } else { - for (unsigned I = 0; I < NF; ++I) - Operands.push_back(Ops[NF + I + 1]); - Operands.push_back(Ops[2 * NF + 1]); - Operands.push_back(Ops[NF]); - Operands.push_back(Ops[2 * NF + 3]); - NewVL = Ops[2 * NF + 2]; + // TA builtin: (val0 address, val1 address, ..., ptr, new_vl, vl) + // TU builtin: (val0 address, ..., passthru0, ..., ptr, new_vl, vl) + // intrinsic: (passthru0, passthru1, ..., ptr, vl) + if (DefaultPolicy == TAIL_AGNOSTIC) { + for (unsigned I = 0; I < NF; ++I) + Operands.push_back(llvm::PoisonValue::get(ResultType)); + Operands.push_back(Ops[NF]); + Operands.push_back(Ops[NF + 2]); + NewVL = Ops[NF + 1]; + } else { + for (unsigned I = 0; I < NF; ++I) + Operands.push_back(Ops[NF + I]); + Operands.push_back(Ops[2 * NF]); + Operands.push_back(Ops[2 * NF + 2]); + NewVL = Ops[2 * NF + 1]; + } } - Operands.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy)); - assert(Operands.size() == NF + 4); llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); clang::CharUnits Align = @@ -1007,7 +973,7 @@ multiclass RVVUnitStridedSegLoadFF<string op> { llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {NF}); return Builder.CreateStore(Val, Address(NewVL, Val->getType(), Align)); } - }] in { + }] in { defvar PV = PVString<nf, /*signed=*/true>.S; defvar PUV = PVString<nf, /*signed=*/false>.S; def : RVVBuiltin<"v", "0" # PV # "PCe" # "Pz", type>; @@ -1035,62 +1001,49 @@ multiclass RVVStridedSegLoad<string op> { NF = nf, ManualCodegen = [{ { - // TA builtin: (val0 address, val1 address, ..., ptr, stride, vl) - // TU builtin: (val0 address, ..., passthru0, ..., ptr, stride, vl) ResultType = ConvertType(E->getArg(0)->getType()->getPointeeType()); IntrinsicTypes = {ResultType, Ops.back()->getType()}; - // intrinsic: (passthru0, passthru1, ..., ptr, stride, vl) SmallVector<llvm::Value*, 12> Operands; - if (DefaultPolicy == TAIL_AGNOSTIC) { - for (unsigned I = 0; I < NF; ++I) - Operands.push_back(llvm::PoisonValue::get(ResultType)); - Operands.push_back(Ops[NF]); - Operands.push_back(Ops[NF + 1]); - Operands.push_back(Ops[NF + 2]); - } else { - for (unsigned I = 0; I < NF; ++I) - Operands.push_back(Ops[NF + I]); - Operands.push_back(Ops[2 * NF]); - Operands.push_back(Ops[2 * NF + 1]); - Operands.push_back(Ops[2 * NF + 2]); - } - llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); - llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); - clang::CharUnits Align = - CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType()); - llvm::Value *V; - for (unsigned I = 0; I < NF; ++I) { - llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {I}); - V = Builder.CreateStore(Val, Address(Ops[I], Val->getType(), Align)); - } - return V; - } - }], - MaskedManualCodegen = [{ - { - //TAMA builtin: (val0 address, ..., mask, ptr, stride, vl) - // builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, stride, vl) - // intrinsic: (maskedoff0, ..., ptr, stride, mask, vl) - ResultType = ConvertType(E->getArg(0)->getType()->getPointeeType()); - IntrinsicTypes = {ResultType, Ops.back()->getType()}; - SmallVector<llvm::Value*, 12> Operands; - if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC) { - for (unsigned I = 0; I < NF; ++I) - Operands.push_back(llvm::PoisonValue::get(ResultType)); - Operands.push_back(Ops[NF + 1]); - Operands.push_back(Ops[NF + 2]); - Operands.push_back(Ops[NF]); - Operands.push_back(Ops[NF + 3]); + + if (IsMasked) { + // TAMA builtin: (val0 address, ..., mask, ptr, stride, vl) + // builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, stride, vl) + // intrinsic: (maskedoff0, ..., ptr, stride, mask, vl) + if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC) { + for (unsigned I = 0; I < NF; ++I) + Operands.push_back(llvm::PoisonValue::get(ResultType)); + Operands.push_back(Ops[NF + 1]); + Operands.push_back(Ops[NF + 2]); + Operands.push_back(Ops[NF]); + Operands.push_back(Ops[NF + 3]); + } else { + for (unsigned I = 0; I < NF; ++I) + Operands.push_back(Ops[NF + I + 1]); + Operands.push_back(Ops[2 * NF + 1]); + Operands.push_back(Ops[2 * NF + 2]); + Operands.push_back(Ops[NF]); + Operands.push_back(Ops[2 * NF + 3]); + } + Operands.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy)); + assert(Operands.size() == NF + 5); } else { - for (unsigned I = 0; I < NF; ++I) - Operands.push_back(Ops[NF + I + 1]); - Operands.push_back(Ops[2 * NF + 1]); - Operands.push_back(Ops[2 * NF + 2]); - Operands.push_back(Ops[NF]); - Operands.push_back(Ops[2 * NF + 3]); + // TA builtin: (val0 address, val1 address, ..., ptr, stride, vl) + // TU builtin: (val0 address, ..., passthru0, ..., ptr, stride, vl) + // intrinsic: (passthru0, passthru1, ..., ptr, stride, vl) + if (DefaultPolicy == TAIL_AGNOSTIC) { + for (unsigned I = 0; I < NF; ++I) + Operands.push_back(llvm::PoisonValue::get(ResultType)); + Operands.push_back(Ops[NF]); + Operands.push_back(Ops[NF + 1]); + Operands.push_back(Ops[NF + 2]); + } else { + for (unsigned I = 0; I < NF; ++I) + Operands.push_back(Ops[NF + I]); + Operands.push_back(Ops[2 * NF]); + Operands.push_back(Ops[2 * NF + 1]); + Operands.push_back(Ops[2 * NF + 2]); + } } - Operands.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy)); - assert(Operands.size() == NF + 5); llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); clang::CharUnits Align = @@ -1102,7 +1055,7 @@ multiclass RVVStridedSegLoad<string op> { } return V; } - }] in { + }] in { defvar PV = PVString<nf, /*signed=*/true>.S; defvar PUV = PVString<nf, /*signed=*/false>.S; def : RVVBuiltin<"v", "0" # PV # "PCe" # "t", type>; @@ -1126,64 +1079,51 @@ multiclass RVVIndexedSegLoad<string op> { NF = nf, ManualCodegen = [{ { - // TA builtin: (val0 address, val1 address, ..., ptr, index, vl) - // TU builtin: (val0 address, ..., passthru0, ..., ptr, index, vl) ResultType = ConvertType(E->getArg(0)->getType()->getPointeeType()); - // intrinsic: (passthru0, passthru1, ..., ptr, index, vl) SmallVector<llvm::Value*, 12> Operands; - if (DefaultPolicy == TAIL_AGNOSTIC) { - for (unsigned I = 0; I < NF; ++I) - Operands.push_back(llvm::PoisonValue::get(ResultType)); - Operands.push_back(Ops[NF]); - Operands.push_back(Ops[NF + 1]); - Operands.push_back(Ops[NF + 2]); - IntrinsicTypes = {ResultType, Ops[NF + 1]->getType(), Ops.back()->getType()}; - } else { - for (unsigned I = 0; I < NF; ++I) - Operands.push_back(Ops[NF + I]); - Operands.push_back(Ops[2 * NF]); - Operands.push_back(Ops[2 * NF + 1]); - Operands.push_back(Ops[2 * NF + 2]); - IntrinsicTypes = {ResultType, Ops[2 * NF + 1]->getType(), Ops.back()->getType()}; - } - llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); - llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); - clang::CharUnits Align = - CGM.getNaturalPointeeTypeAlignment(E->getArg(0)->getType()); - llvm::Value *V; - for (unsigned I = 0; I < NF; ++I) { - llvm::Value *Val = Builder.CreateExtractValue(LoadValue, {I}); - V = Builder.CreateStore(Val, Address(Ops[I], Val->getType(), Align)); - } - return V; - } - }], - MaskedManualCodegen = [{ - { - // TAMA builtin: (val0 address, ..., mask, ptr, index, vl) - // builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, index, vl) - ResultType = ConvertType(E->getArg(0)->getType()->getPointeeType()); - // intrinsic: (maskedoff0, ..., ptr, index, mask, vl) - SmallVector<llvm::Value*, 12> Operands; - if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC) { - for (unsigned I = 0; I < NF; ++I) - Operands.push_back(llvm::PoisonValue::get(ResultType)); - Operands.push_back(Ops[NF + 1]); - Operands.push_back(Ops[NF + 2]); - Operands.push_back(Ops[NF]); - Operands.push_back(Ops[NF + 3]); - IntrinsicTypes = {ResultType, Ops[NF + 2]->getType(), Ops.back()->getType()}; + if (IsMasked) { + // TAMA builtin: (val0 address, ..., mask, ptr, index, vl) + // builtin: (val0 address, ..., mask, maskedoff0, ..., ptr, index, vl) + // intrinsic: (maskedoff0, ..., ptr, index, mask, vl) + if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC) { + for (unsigned I = 0; I < NF; ++I) + Operands.push_back(llvm::PoisonValue::get(ResultType)); + Operands.push_back(Ops[NF + 1]); + Operands.push_back(Ops[NF + 2]); + Operands.push_back(Ops[NF]); + Operands.push_back(Ops[NF + 3]); + IntrinsicTypes = {ResultType, Ops[NF + 2]->getType(), Ops.back()->getType()}; + } else { + for (unsigned I = 0; I < NF; ++I) + Operands.push_back(Ops[NF + I + 1]); + Operands.push_back(Ops[2 * NF + 1]); + Operands.push_back(Ops[2 * NF + 2]); + Operands.push_back(Ops[NF]); + Operands.push_back(Ops[2 * NF + 3]); + IntrinsicTypes = {ResultType, Ops[2 * NF + 2]->getType(), Ops.back()->getType()}; + } + Operands.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy)); + assert(Operands.size() == NF + 5); } else { - for (unsigned I = 0; I < NF; ++I) - Operands.push_back(Ops[NF + I + 1]); - Operands.push_back(Ops[2 * NF + 1]); - Operands.push_back(Ops[2 * NF + 2]); - Operands.push_back(Ops[NF]); - Operands.push_back(Ops[2 * NF + 3]); - IntrinsicTypes = {ResultType, Ops[2 * NF + 2]->getType(), Ops.back()->getType()}; + // TA builtin: (val0 address, val1 address, ..., ptr, index, vl) + // TU builtin: (val0 address, ..., passthru0, ..., ptr, index, vl) + // intrinsic: (passthru0, passthru1, ..., ptr, index, vl) + if (DefaultPolicy == TAIL_AGNOSTIC) { + for (unsigned I = 0; I < NF; ++I) + Operands.push_back(llvm::PoisonValue::get(ResultType)); + Operands.push_back(Ops[NF]); + Operands.push_back(Ops[NF + 1]); + Operands.push_back(Ops[NF + 2]); + IntrinsicTypes = {ResultType, Ops[NF + 1]->getType(), Ops.back()->getType()}; + } else { + for (unsigned I = 0; I < NF; ++I) + Operands.push_back(Ops[NF + I]); + Operands.push_back(Ops[2 * NF]); + Operands.push_back(Ops[2 * NF + 1]); + Operands.push_back(Ops[2 * NF + 2]); + IntrinsicTypes = {ResultType, Ops[2 * NF + 1]->getType(), Ops.back()->getType()}; + } } - Operands.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy)); - assert(Operands.size() == NF + 5); llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); clang::CharUnits Align = @@ -1195,7 +1135,7 @@ multiclass RVVIndexedSegLoad<string op> { } return V; } - }] in { + }] in { defvar PV = PVString<nf, /*signed=*/true>.S; defvar PUV = PVString<nf, /*signed=*/false>.S; def : RVVBuiltin<"v", "0" # PV # "PCe" # eew_type # "Uv", type>; @@ -1236,21 +1176,20 @@ multiclass RVVUnitStridedSegStore<string op> { MaskedPolicyScheme = NonePolicy, ManualCodegen = [{ { - // Builtin: (ptr, val0, val1, ..., vl) - // Intrinsic: (val0, val1, ..., ptr, vl) - std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); - IntrinsicTypes = {Ops[0]->getType(), Ops[NF + 1]->getType()}; - assert(Ops.size() == NF + 2); - } - }], - MaskedManualCodegen = [{ - { - // Builtin: (mask, ptr, val0, val1, ..., vl) - // Intrinsic: (val0, val1, ..., ptr, mask, vl) - std::rotate(Ops.begin(), Ops.begin() + 2, Ops.end() - 1); - std::swap(Ops[NF], Ops[NF + 1]); - IntrinsicTypes = {Ops[0]->getType(), Ops[NF + 2]->getType()}; - assert(Ops.size() == NF + 3); + if (IsMasked) { + // Builtin: (mask, ptr, val0, val1, ..., vl) + // Intrinsic: (val0, val1, ..., ptr, mask, vl) + std::rotate(Ops.begin(), Ops.begin() + 2, Ops.end() - 1); + std::swap(Ops[NF], Ops[NF + 1]); + IntrinsicTypes = {Ops[0]->getType(), Ops[NF + 2]->getType()}; + assert(Ops.size() == NF + 3); + } else { + // Builtin: (ptr, val0, val1, ..., vl) + // Intrinsic: (val0, val1, ..., ptr, vl) + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + IntrinsicTypes = {Ops[0]->getType(), Ops[NF + 1]->getType()}; + assert(Ops.size() == NF + 2); + } } }] in { defvar V = VString<nf, /*signed=*/true>.S; @@ -1282,21 +1221,19 @@ multiclass RVVStridedSegStore<string op> { MaskedPolicyScheme = NonePolicy, ManualCodegen = [{ { - // Builtin: (ptr, stride, val0, val1, ..., vl). - // Intrinsic: (val0, val1, ..., ptr, stride, vl) - std::rotate(Ops.begin(), Ops.begin() + 2, Ops.end() - 1); - IntrinsicTypes = {Ops[0]->getType(), Ops[NF + 1]->getType()}; - assert(Ops.size() == NF + 3); - } - }], - MaskedManualCodegen = [{ - { - // Builtin: (mask, ptr, stride, val0, val1, ..., vl). - // Intrinsic: (val0, val1, ..., ptr, stride, mask, vl) - std::rotate(Ops.begin(), Ops.begin() + 3, Ops.end() - 1); - std::rotate(Ops.begin() + NF, Ops.begin() + NF + 1, Ops.begin() + NF + 3); + if (IsMasked) { + // Builtin: (mask, ptr, stride, val0, val1, ..., vl). + // Intrinsic: (val0, val1, ..., ptr, stride, mask, vl) + std::rotate(Ops.begin(), Ops.begin() + 3, Ops.end() - 1); + std::rotate(Ops.begin() + NF, Ops.begin() + NF + 1, Ops.begin() + NF + 3); + assert(Ops.size() == NF + 4); + } else { + // Builtin: (ptr, stride, val0, val1, ..., vl). + // Intrinsic: (val0, val1, ..., ptr, stride, vl) + std::rotate(Ops.begin(), Ops.begin() + 2, Ops.end() - 1); + assert(Ops.size() == NF + 3); + } IntrinsicTypes = {Ops[0]->getType(), Ops[NF + 1]->getType()}; - assert(Ops.size() == NF + 4); } }] in { defvar V = VString<nf, /*signed=*/true>.S; @@ -1324,23 +1261,22 @@ multiclass RVVIndexedSegStore<string op> { MaskedPolicyScheme = NonePolicy, ManualCodegen = [{ { - // Builtin: (ptr, index, val0, val1, ..., vl) - // Intrinsic: (val0, val1, ..., ptr, index, vl) - std::rotate(Ops.begin(), Ops.begin() + 2, Ops.end() - 1); - IntrinsicTypes = {Ops[0]->getType(), - Ops[NF + 1]->getType(), Ops[NF + 2]->getType()}; - assert(Ops.size() == NF + 3); - } - }], - MaskedManualCodegen = [{ - { - // Builtin: (mask, ptr, index, val0, val1, ..., vl) - // Intrinsic: (val0, val1, ..., ptr, index, mask, vl) - std::rotate(Ops.begin(), Ops.begin() + 3, Ops.end() - 1); - std::rotate(Ops.begin() + NF, Ops.begin() + NF + 1, Ops.begin() + NF + 3); - IntrinsicTypes = {Ops[0]->getType(), - Ops[NF + 1]->getType(), Ops[NF + 3]->getType()}; - assert(Ops.size() == NF + 4); + if (IsMasked) { + // Builtin: (mask, ptr, index, val0, val1, ..., vl) + // Intrinsic: (val0, val1, ..., ptr, index, mask, vl) + std::rotate(Ops.begin(), Ops.begin() + 3, Ops.end() - 1); + std::rotate(Ops.begin() + NF, Ops.begin() + NF + 1, Ops.begin() + NF + 3); + IntrinsicTypes = {Ops[0]->getType(), + Ops[NF + 1]->getType(), Ops[NF + 3]->getType()}; + assert(Ops.size() == NF + 4); + } else { + // Builtin: (ptr, index, val0, val1, ..., vl) + // Intrinsic: (val0, val1, ..., ptr, index, vl) + std::rotate(Ops.begin(), Ops.begin() + 2, Ops.end() - 1); + IntrinsicTypes = {Ops[0]->getType(), + Ops[NF + 1]->getType(), Ops[NF + 2]->getType()}; + assert(Ops.size() == NF + 3); + } } }] in { defvar V = VString<nf, /*signed=*/true>.S; @@ -1362,25 +1298,25 @@ multiclass RVVPseudoUnaryBuiltin<string IR, string type_range> { UnMaskedPolicyScheme = HasPassthruOperand, ManualCodegen = [{ { - if (DefaultPolicy == TAIL_AGNOSTIC) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); - auto ElemTy = cast<llvm::VectorType>(ResultType)->getElementType(); - Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(ElemTy)); - // passthru, op1, op2, vl - IntrinsicTypes = {ResultType, ElemTy, Ops[3]->getType()}; - break; - } - }], - MaskedManualCodegen = [{ - { - std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); - if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + if (IsMasked) { + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + } else { + if (DefaultPolicy == TAIL_AGNOSTIC) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + } auto ElemTy = cast<llvm::VectorType>(ResultType)->getElementType(); Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(ElemTy)); - Ops.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy)); - // maskedoff, op1, op2, mask, vl, policy - IntrinsicTypes = {ResultType, ElemTy, Ops[4]->getType()}; + + if (IsMasked) { + Ops.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy)); + // maskedoff, op1, op2, mask, vl, policy + IntrinsicTypes = {ResultType, ElemTy, Ops[4]->getType()}; + } else { + // passthru, op1, op2, vl + IntrinsicTypes = {ResultType, ElemTy, Ops[3]->getType()}; + } break; } }] in { @@ -1395,31 +1331,29 @@ multiclass RVVPseudoVNotBuiltin<string IR, string type_range> { UnMaskedPolicyScheme = HasPassthruOperand, ManualCodegen = [{ { - if (DefaultPolicy == TAIL_AGNOSTIC) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); - auto ElemTy = cast<llvm::VectorType>(ResultType)->getElementType(); - Ops.insert(Ops.begin() + 2, - llvm::Constant::getAllOnesValue(ElemTy)); - // passthru, op1, op2, vl - IntrinsicTypes = {ResultType, - ElemTy, - Ops[3]->getType()}; - break; - } - }], - MaskedManualCodegen = [{ - { - std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); - if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + if (IsMasked) { + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + } else { + if (DefaultPolicy == TAIL_AGNOSTIC) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + } auto ElemTy = cast<llvm::VectorType>(ResultType)->getElementType(); Ops.insert(Ops.begin() + 2, llvm::Constant::getAllOnesValue(ElemTy)); - Ops.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy)); - // maskedoff, op1, po2, mask, vl, policy - IntrinsicTypes = {ResultType, - ElemTy, - Ops[4]->getType()}; + if (IsMasked) { + Ops.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy)); + // maskedoff, op1, po2, mask, vl, policy + IntrinsicTypes = {ResultType, + ElemTy, + Ops[4]->getType()}; + } else { + // passthru, op1, op2, vl + IntrinsicTypes = {ResultType, + ElemTy, + Ops[3]->getType()}; + } break; } }] in { @@ -1452,26 +1386,25 @@ multiclass RVVPseudoVFUnaryBuiltin<string IR, string type_range> { UnMaskedPolicyScheme = HasPassthruOperand, ManualCodegen = [{ { - if (DefaultPolicy == TAIL_AGNOSTIC) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); - // op1, po2, vl - IntrinsicTypes = {ResultType, - Ops[1]->getType(), Ops[2]->getType()}; - Ops.insert(Ops.begin() + 2, Ops[1]); - break; - } - }], - MaskedManualCodegen = [{ - { - std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); - if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); - Ops.insert(Ops.begin() + 2, Ops[1]); - Ops.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy)); - // maskedoff, op1, op2, mask, vl - IntrinsicTypes = {ResultType, - Ops[2]->getType(), - Ops.back()->getType()}; + if (IsMasked) { + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + Ops.insert(Ops.begin() + 2, Ops[1]); + Ops.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy)); + // maskedoff, op1, op2, mask, vl + IntrinsicTypes = {ResultType, + Ops[2]->getType(), + Ops.back()->getType()}; + } else { + if (DefaultPolicy == TAIL_AGNOSTIC) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + // op1, po2, vl + IntrinsicTypes = {ResultType, + Ops[1]->getType(), Ops[2]->getType()}; + Ops.insert(Ops.begin() + 2, Ops[1]); + break; + } break; } }] in { @@ -1488,31 +1421,30 @@ multiclass RVVPseudoVWCVTBuiltin<string IR, string MName, string type_range, UnMaskedPolicyScheme = HasPassthruOperand, ManualCodegen = [{ { - if (DefaultPolicy == TAIL_AGNOSTIC) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); - auto ElemTy = cast<llvm::VectorType>(ResultType)->getElementType(); - Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(ElemTy)); - // passtru, op1, op2, vl - IntrinsicTypes = {ResultType, - Ops[1]->getType(), - ElemTy, - Ops[3]->getType()}; - break; - } - }], - MaskedManualCodegen = [{ - { - std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); - if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + if (IsMasked) { + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + } else { + if (DefaultPolicy == TAIL_AGNOSTIC) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + } auto ElemTy = cast<llvm::VectorType>(ResultType)->getElementType(); Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(ElemTy)); - Ops.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy)); - // maskedoff, op1, op2, mask, vl, policy - IntrinsicTypes = {ResultType, - Ops[1]->getType(), - ElemTy, - Ops[4]->getType()}; + if (IsMasked) { + Ops.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy)); + // maskedoff, op1, op2, mask, vl, policy + IntrinsicTypes = {ResultType, + Ops[1]->getType(), + ElemTy, + Ops[4]->getType()}; + } else { + // passtru, op1, op2, vl + IntrinsicTypes = {ResultType, + Ops[1]->getType(), + ElemTy, + Ops[3]->getType()}; + } break; } }] in { @@ -1531,29 +1463,29 @@ multiclass RVVPseudoVNCVTBuiltin<string IR, string MName, string type_range, UnMaskedPolicyScheme = HasPassthruOperand, ManualCodegen = [{ { - if (DefaultPolicy == TAIL_AGNOSTIC) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); - Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(Ops.back()->getType())); - // passthru, op1, xlen, vl - IntrinsicTypes = {ResultType, - Ops[1]->getType(), - Ops[3]->getType(), - Ops[3]->getType()}; - break; - } - }], - MaskedManualCodegen = [{ - { - std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); - if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC) - Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); - Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(Ops.back()->getType())); - Ops.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy)); - // maskedoff, op1, xlen, mask, vl - IntrinsicTypes = {ResultType, - Ops[1]->getType(), - Ops[4]->getType(), - Ops[4]->getType()}; + if (IsMasked) { + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + if (DefaultPolicy == TAIL_AGNOSTIC_MASK_AGNOSTIC) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + } else { + if (DefaultPolicy == TAIL_AGNOSTIC) + Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType)); + } + Ops.insert(Ops.begin() + 2, llvm::Constant::getNullValue(Ops.back()->getType())); + if (IsMasked) { + Ops.push_back(ConstantInt::get(Ops.back()->getType(), DefaultPolicy)); + // maskedoff, op1, xlen, mask, vl + IntrinsicTypes = {ResultType, + Ops[1]->getType(), + Ops[4]->getType(), + Ops[4]->getType()}; + } else { + // passthru, op1, xlen, vl + IntrinsicTypes = {ResultType, + Ops[1]->getType(), + Ops[3]->getType(), + Ops[3]->getType()}; + } break; } }] in { diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp index ef0802e4f6e3e..fc7b73d9ee6e5 100644 --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -19442,6 +19442,7 @@ Value *CodeGenFunction::EmitRISCVBuiltinExpr(unsigned BuiltinID, constexpr unsigned TAIL_AGNOSTIC = 1; constexpr unsigned TAIL_AGNOSTIC_MASK_AGNOSTIC = 3; int DefaultPolicy = TAIL_UNDISTURBED; + bool IsMasked = false; // Required for overloaded intrinsics. llvm::SmallVector<llvm::Type *, 2> IntrinsicTypes; diff --git a/clang/utils/TableGen/RISCVVEmitter.cpp b/clang/utils/TableGen/RISCVVEmitter.cpp index ecf3b35e4a47d..ddf2c5c52393e 100644 --- a/clang/utils/TableGen/RISCVVEmitter.cpp +++ b/clang/utils/TableGen/RISCVVEmitter.cpp @@ -171,6 +171,10 @@ void emitCodeGenSwitchBody(const RVVIntrinsic *RVVI, raw_ostream &OS) { if (RVVI->hasManualCodegen()) { OS << " DefaultPolicy = " << RVVI->getDefaultPolicyBits() << ";\n"; + if (RVVI->isMasked()) + OS << "IsMasked = true;\n"; + else + OS << "IsMasked = false;\n"; OS << RVVI->getManualCodegen(); OS << "break;\n"; return; @@ -517,7 +521,6 @@ void RVVEmitter::createRVVIntrinsics( bool SupportOverloading = R->getValueAsBit("SupportOverloading"); bool HasBuiltinAlias = R->getValueAsBit("HasBuiltinAlias"); StringRef ManualCodegen = R->getValueAsString("ManualCodegen"); - StringRef MaskedManualCodegen = R->getValueAsString("MaskedManualCodegen"); std::vector<int64_t> IntrinsicTypes = R->getValueAsListOfInts("IntrinsicTypes"); std::vector<StringRef> RequiredFeatures = @@ -598,7 +601,7 @@ void RVVEmitter::createRVVIntrinsics( Out.push_back(std::make_unique<RVVIntrinsic>( Name, SuffixStr, OverloadedName, OverloadedSuffixStr, MaskedIRName, /*IsMasked=*/true, HasMaskedOffOperand, HasVL, MaskedPolicyScheme, - SupportOverloading, HasBuiltinAlias, MaskedManualCodegen, + SupportOverloading, HasBuiltinAlias, ManualCodegen, *MaskTypes, IntrinsicTypes, RequiredFeatures, NF, Policy(), IsPrototypeDefaultTU)); if (MaskedPolicyScheme == PolicyScheme::SchemeNone) @@ -614,7 +617,7 @@ void RVVEmitter::createRVVIntrinsics( Name, SuffixStr, OverloadedName, OverloadedSuffixStr, MaskedIRName, /*IsMasked=*/true, HasMaskedOffOperand, HasVL, MaskedPolicyScheme, SupportOverloading, HasBuiltinAlias, - MaskedManualCodegen, *PolicyTypes, IntrinsicTypes, + ManualCodegen, *PolicyTypes, IntrinsicTypes, RequiredFeatures, NF, P, IsPrototypeDefaultTU)); } } // End for Log2LMULList _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
