https://github.com/gysit created https://github.com/llvm/llvm-project/pull/221879
llvm.intr.masked.load, masked.store, masked.gather and masked.scatter implemented neither AccessGroupOpInterface nor AliasAnalysisOpInterface, so they could not carry access_groups, alias_scopes, noalias_scopes or tbaa. Importing LLVM IR that attached such metadata to one of them silently dropped it, while llvm.memcpy in the same module preserved it. Also widen alignment from I32Attr to I64Attr, matching the rest of the LLVM dialect and LLVM proper, and make it optional. LangRef gives the default alignment of these four intrinsics as the ABI alignment of the type, exactly as for llvm.load and llvm.store, so an absent align parameter attribute now imports as an absent alignment attribute instead of being materialised as align 1, and an absent alignment attribute exports without an align parameter attribute. This changes the textual IR, as alignment = 4 : i32 becomes alignment = 4 : i64. Downstream updates should rewrite only lines that mention a masked intrinsic rather than blanket-substituting i32 with i64; other i32 spellings must stay, including llvm.intr.matrix.*'s rows and columns attributes and i32 vector and element types on the intrinsics themselves. >From 8e46c942d4b12130a38b64133403dc9540dd3513 Mon Sep 17 00:00:00 2001 From: Tobias Gysi <[email protected]> Date: Mon, 7 Sep 2026 20:57:52 +0200 Subject: [PATCH] [mlir][llvm] Fix metadata and alignment on masked memory intrinsics llvm.intr.masked.load, masked.store, masked.gather and masked.scatter implemented neither AccessGroupOpInterface nor AliasAnalysisOpInterface, so they could not carry access_groups, alias_scopes, noalias_scopes or tbaa. Importing LLVM IR that attached such metadata to one of them silently dropped it, while llvm.memcpy in the same module preserved it. Also widen alignment from I32Attr to I64Attr, matching the rest of the LLVM dialect and LLVM proper, and make it optional. LangRef gives the default alignment of these four intrinsics as the ABI alignment of the type, exactly as for llvm.load and llvm.store, so an absent align parameter attribute now imports as an absent alignment attribute instead of being materialised as align 1, and an absent alignment attribute exports without an align parameter attribute. This changes the textual IR, as alignment = 4 : i32 becomes alignment = 4 : i64. Downstream updates should rewrite only lines that mention a masked intrinsic rather than blanket-substituting i32 with i64; other i32 spellings must stay, including llvm.intr.matrix.*'s rows and columns attributes and i32 vector and element types on the intrinsics themselves. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> --- .../CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp | 4 +- .../mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td | 214 ++++++++++++++---- .../include/mlir/Dialect/LLVMIR/LLVMOpBase.td | 4 +- .../VectorToLLVM/ConvertVectorToLLVM.cpp | 5 +- mlir/lib/Dialect/LLVMIR/IR/LLVMInterfaces.cpp | 16 ++ .../VectorToLLVM/use-vector-alignment.mlir | 32 +-- .../vector-to-llvm-interface.mlir | 52 ++--- .../VectorToLLVM/vector-to-llvm.mlir | 8 +- .../VectorToLLVM/vector-xfer-to-llvm.mlir | 16 +- mlir/test/Dialect/LLVMIR/opaque-ptr.mlir | 4 +- mlir/test/Dialect/LLVMIR/roundtrip.mlir | 23 ++ mlir/test/Target/LLVMIR/Import/intrinsic.ll | 54 +++-- .../Import/masked-intrinsic-metadata.ll | 64 ++++++ .../test/Target/LLVMIR/llvmir-intrinsics.mlir | 28 ++- mlir/test/Target/LLVMIR/llvmir-invalid.mlir | 12 +- .../LLVMIR/masked-intrinsic-metadata.mlir | 73 ++++++ mlir/test/Target/LLVMIR/opaque-ptr.mlir | 4 +- 17 files changed, 466 insertions(+), 147 deletions(-) create mode 100644 mlir/test/Target/LLVMIR/Import/masked-intrinsic-metadata.ll create mode 100644 mlir/test/Target/LLVMIR/masked-intrinsic-metadata.mlir diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp index 31e01c770c55b..69b73b6fa516e 100644 --- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp +++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp @@ -2373,11 +2373,9 @@ cir::direct::CIRToLLVMVecMaskedLoadOpLowering::matchAndRewrite( unsigned alignment = (unsigned)opAlign.value_or(dataLayout.getTypeABIAlignment(llvmResTy)); - mlir::IntegerAttr alignAttr = rewriter.getI32IntegerAttr(alignment); - auto newLoad = mlir::LLVM::MaskedLoadOp::create( rewriter, op.getLoc(), llvmResTy, adaptor.getAddr(), adaptor.getMask(), - adaptor.getPassThru(), alignAttr); + adaptor.getPassThru(), alignment); rewriter.replaceOp(op, newLoad.getResult()); return mlir::success(); diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td index f40f3f5ebda93..8afbc0336a2d2 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td @@ -17,6 +17,7 @@ class LLVM_UnaryIntrOpBase<string func, Type element, list<Trait> traits = [], bit requiresFastmath = 0> : LLVM_OneResultIntrOp<func, [], [0], !listconcat([Pure, SameOperandsAndResultType], traits), + /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0, requiresFastmath> { dag commonArgs = (ins LLVM_ScalarOrVectorOf<element>:$in); let assemblyFormat = "`(` operands `)` attr-dict `:` " @@ -39,6 +40,7 @@ class LLVM_BinarySameArgsIntrOpBase<string func, Type element, list<Trait> traits = [], bit requiresFastmath = 0> : LLVM_OneResultIntrOp<func, [], [0], !listconcat([Pure, SameOperandsAndResultType], traits), + /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0, requiresFastmath> { dag commonArgs = (ins LLVM_ScalarOrVectorOf<element>:$a, LLVM_ScalarOrVectorOf<element>:$b); @@ -63,6 +65,7 @@ class LLVM_TernarySameArgsIntrOpBase<string func, Type element, list<Trait> traits = [], bit requiresFastmath = 0> : LLVM_OneResultIntrOp<func, [], [0], !listconcat([Pure, SameOperandsAndResultType], traits), + /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0, requiresFastmath> { dag commonArgs = (ins LLVM_ScalarOrVectorOf<element>:$a, LLVM_ScalarOrVectorOf<element>:$b, @@ -87,6 +90,7 @@ class LLVM_TernarySameArgsIntrOpF<string func, list<Trait> traits = []> : class LLVM_CountZerosIntrOp<string func, list<Trait> traits = []> : LLVM_OneResultIntrOp<func, [], [0], !listconcat([Pure, SameOperandsAndResultType], traits), + /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0, /*requiresFastmath=*/0, /*requiresArgAndResultAttrs=*/0, /*immArgPositions=*/[1], /*immArgAttrNames=*/["is_zero_poison"]> { let arguments = (ins LLVM_ScalarOrVectorOf<AnySignlessInteger>:$in, @@ -94,6 +98,7 @@ class LLVM_CountZerosIntrOp<string func, list<Trait> traits = []> : } def LLVM_AbsOp : LLVM_OneResultIntrOp<"abs", [], [0], [Pure], + /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0, /*requiresFastmath=*/0, /*requiresArgAndResultAttrs=*/0, /*immArgPositions=*/[1], /*immArgAttrNames=*/["is_int_min_poison"]> { let arguments = (ins LLVM_ScalarOrVectorOf<AnySignlessInteger>:$in, @@ -108,6 +113,7 @@ def LLVM_ArithmeticFenceOp } def LLVM_IsFPClass : LLVM_OneResultIntrOp<"is.fpclass", [], [0], [Pure], + /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0, /*requiresFastmath=*/0, /*requiresArgAndResultAttrs=*/0, /*immArgPositions=*/[1], /*immArgAttrNames=*/["bit"]> { let arguments = (ins LLVM_ScalarOrVectorOf<LLVM_AnyFloat>:$in, I32Attr:$bit); @@ -115,7 +121,8 @@ def LLVM_IsFPClass : LLVM_OneResultIntrOp<"is.fpclass", [], [0], [Pure], class LLVM_PowFI<string func> : LLVM_OneResultIntrOp<func, [], [0,1], - [Pure], /*requiresFastmath=*/1> { + [Pure], /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0, + /*requiresFastmath=*/1> { let arguments = (ins LLVM_ScalarOrVectorOf<LLVM_AnyFloat>:$val, AnySignlessInteger:$power, @@ -388,6 +395,7 @@ def LLVM_LifetimeEndOp : LLVM_LifetimeBaseOp<"lifetime.end">; def LLVM_InvariantStartOp : LLVM_OneResultIntrOp<"invariant.start", [], [1], [DeclareOpInterfaceMethods<PromotableOpInterface>], + /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0, /*requiresFastmath=*/0, /*requiresArgAndResultAttrs=*/0, /*immArgPositions=*/[0], /*immArgAttrNames=*/["size"]> { let arguments = (ins I64Attr:$size, LLVM_AnyPointer:$ptr); @@ -441,6 +449,8 @@ class LLVM_ConstrainedIntr<string mnem, int numArgs, !gt(hasRoundingMode, 0) : [DeclareOpInterfaceMethods<RoundingModeOpInterface>], true : []) # extraTraits, + /*requiresAccessGroup=*/0, + /*requiresAliasAnalysis=*/0, /*requiresFastmath=*/1, /*requiresArgAndResultAttrs=*/0, /*immArgPositions=*/[], @@ -589,6 +599,7 @@ class LLVM_ConstrainedFCmpIntrBase<string mnem> "operands", "arg_0", "res", "::getI1SameShape($_self)">, DeclareOpInterfaceMethods<FPExceptionBehaviorOpInterface>], + /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0, /*requiresFastmath=*/1> { let arguments = (ins FCmpPredicate:$predicate, LLVM_Type:$arg_0, @@ -880,6 +891,7 @@ def LLVM_ExpectOp def LLVM_ExpectWithProbabilityOp : LLVM_OneResultIntrOp<"expect.with.probability", [], [0], [Pure, AllTypesMatch<["val", "expected", "res"]>], + /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0, /*requiresFastmath=*/0, /*requiresArgAndResultAttrs=*/0, /*immArgPositions=*/[2], /*immArgAttrNames=*/["prob"]> { let arguments = (ins AnySignlessInteger:$val, @@ -1184,6 +1196,7 @@ def LLVM_StackRestoreOp : LLVM_ZeroResultIntrOp<"stackrestore", [0]> { class LLVM_VecReductionBase<string mnem, Type element, bit requiresFastmath=0> : LLVM_OneResultIntrOp<"vector.reduce." # mnem, [], [0], [Pure, SameOperandsAndResultElementType], + /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0, requiresFastmath> { dag commonArgs = (ins LLVM_VectorOf<element>:$in); } @@ -1210,6 +1223,8 @@ class LLVM_VecReductionAccBase<string mnem, Type element> /*overloadedResults=*/[], /*overloadedOperands=*/[1], /*traits=*/[Pure, SameOperandsAndResultElementType], + /*requiresAccessGroup=*/0, + /*requiresAliasAnalysis=*/0, /*requiresFastmath=*/1>, Arguments<(ins element:$start_value, LLVM_VectorOf<element>:$input, @@ -1363,104 +1378,206 @@ def LLVM_GetActiveLaneMaskOp } /// Create a call to Masked Load intrinsic. -def LLVM_MaskedLoadOp : LLVM_OneResultIntrOp<"masked.load"> { - let arguments = (ins LLVM_AnyPointer:$data, LLVM_VectorOf<I1>:$mask, - Optional<LLVM_AnyVector>:$pass_thru, I32Attr:$alignment, - UnitAttr:$nontemporal); +def LLVM_MaskedLoadOp + : LLVM_OneResultIntrOp<"masked.load", /*overloadedResults=*/[], + /*overloadedOperands=*/[], /*traits=*/[], + /*requiresAccessGroup=*/1, /*requiresAliasAnalysis=*/1> { + dag args = (ins LLVM_AnyPointer:$data, LLVM_VectorOf<I1>:$mask, + Optional<LLVM_AnyVector>:$pass_thru, + OptionalAttr<I64Attr>:$alignment, UnitAttr:$nontemporal); + let arguments = !con(args, baseArgs); let results = (outs LLVM_AnyVector:$res); + let builders = [ + OpBuilder<(ins "Type":$res, "Value":$data, "Value":$mask, + "Value":$pass_thru, "uint64_t":$alignment, + CArg<"bool", "false">:$nontemporal), [{ + build($_builder, $_state, res, data, mask, pass_thru, + $_builder.getI64IntegerAttr(alignment), + nontemporal ? $_builder.getUnitAttr() : nullptr, + /*access_groups=*/nullptr, /*alias_scopes=*/nullptr, + /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr); + }]> + ]; let assemblyFormat = "operands attr-dict `:` functional-type(operands, results)"; string llvmBuilder = [{ + // An absent alignment attribute means the ABI alignment of the type, which + // LLVM spells as an absent align parameter attribute. The IRBuilder helper + // always attaches one, so remove it again in that case. + llvm::Align alignment = + $alignment ? llvm::Align(*$alignment) : llvm::Align(1); auto *inst = $pass_thru ? builder.CreateMaskedLoad( - $_resultType, $data, llvm::Align($alignment), $mask, $pass_thru) : + $_resultType, $data, alignment, $mask, $pass_thru) : builder.CreateMaskedLoad( - $_resultType, $data, llvm::Align($alignment), $mask); + $_resultType, $data, alignment, $mask); + if (!$alignment) + inst->removeParamAttr(0, llvm::Attribute::Alignment); $res = inst; - }] #setNonTemporalMetadataCode; + }] # setNonTemporalMetadataCode + # setAccessGroupsMetadataCode + # setAliasAnalysisMetadataCode; string mlirBuilder = [{ auto *intrinInst = dyn_cast<llvm::IntrinsicInst>(inst); bool nontemporal = intrinInst->hasMetadata(llvm::LLVMContext::MD_nontemporal); - llvm::Align alignment = intrinInst->getParamAlign(0).valueOrOne(); + llvm::MaybeAlign alignment = intrinInst->getParamAlign(0); $res = LLVM::MaskedLoadOp::create($_builder, $_location, $_resultType, $data, $mask, $pass_thru, - $_builder.getI32IntegerAttr(alignment.value()), - nontemporal ? $_builder.getUnitAttr() : nullptr); + alignment ? $_builder.getI64IntegerAttr(alignment->value()) + : ::mlir::IntegerAttr(), + nontemporal ? $_builder.getUnitAttr() : nullptr, + /*access_groups=*/nullptr, /*alias_scopes=*/nullptr, + /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr); }]; - list<int> llvmArgIndices = [0, 1, 2, -1, -1]; + list<int> llvmArgIndices = [0, 1, 2, -1, -1, -1, -1, -1, -1]; } /// Create a call to Masked Store intrinsic. -def LLVM_MaskedStoreOp : LLVM_ZeroResultIntrOp<"masked.store"> { - let arguments = (ins LLVM_AnyVector:$value, LLVM_AnyPointer:$data, - LLVM_VectorOf<I1>:$mask, I32Attr:$alignment, - UnitAttr:$nontemporal); - let builders = [LLVM_VoidResultTypeOpBuilder, LLVM_ZeroResultOpBuilder]; +def LLVM_MaskedStoreOp + : LLVM_ZeroResultIntrOp<"masked.store", /*overloadedOperands=*/[], + /*traits=*/[], /*requiresAccessGroup=*/1, /*requiresAliasAnalysis=*/1> { + dag args = (ins LLVM_AnyVector:$value, LLVM_AnyPointer:$data, + LLVM_VectorOf<I1>:$mask, OptionalAttr<I64Attr>:$alignment, + UnitAttr:$nontemporal); + let arguments = !con(args, baseArgs); + let builders = [ + LLVM_VoidResultTypeOpBuilder, LLVM_ZeroResultOpBuilder, + OpBuilder<(ins "Value":$value, "Value":$data, "Value":$mask, + "uint64_t":$alignment, + CArg<"bool", "false">:$nontemporal), [{ + build($_builder, $_state, value, data, mask, + $_builder.getI64IntegerAttr(alignment), + nontemporal ? $_builder.getUnitAttr() : nullptr, + /*access_groups=*/nullptr, /*alias_scopes=*/nullptr, + /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr); + }]> + ]; let assemblyFormat = "$value `,` $data `,` $mask attr-dict `:` " "type($value) `,` type($mask) `into` qualified(type($data))"; string llvmBuilder = [{ - auto *inst = builder.CreateMaskedStore( - $value, $data, llvm::Align($alignment), $mask); - }] #setNonTemporalMetadataCode; + // An absent alignment attribute means the ABI alignment of the type, which + // LLVM spells as an absent align parameter attribute. The IRBuilder helper + // always attaches one, so remove it again in that case. + llvm::Align alignment = + $alignment ? llvm::Align(*$alignment) : llvm::Align(1); + auto *inst = builder.CreateMaskedStore($value, $data, alignment, $mask); + if (!$alignment) + inst->removeParamAttr(1, llvm::Attribute::Alignment); + }] # setNonTemporalMetadataCode + # setAccessGroupsMetadataCode + # setAliasAnalysisMetadataCode; string mlirBuilder = [{ auto *intrinInst = dyn_cast<llvm::IntrinsicInst>(inst); - llvm::Align alignment = intrinInst->getParamAlign(1).valueOrOne(); + llvm::MaybeAlign alignment = intrinInst->getParamAlign(1); bool nontemporal = intrinInst->hasMetadata(llvm::LLVMContext::MD_nontemporal); $_op = LLVM::MaskedStoreOp::create($_builder, $_location, - $value, $data, $mask, $_builder.getI32IntegerAttr(alignment.value()), - nontemporal ? $_builder.getUnitAttr() : nullptr); + $value, $data, $mask, + alignment ? $_builder.getI64IntegerAttr(alignment->value()) + : ::mlir::IntegerAttr(), + nontemporal ? $_builder.getUnitAttr() : nullptr, + /*access_groups=*/nullptr, /*alias_scopes=*/nullptr, + /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr); }]; - list<int> llvmArgIndices = [0, 1, 2, -1, -1]; + list<int> llvmArgIndices = [0, 1, 2, -1, -1, -1, -1, -1, -1]; } /// Create a call to Masked Gather intrinsic. -def LLVM_masked_gather : LLVM_OneResultIntrOp<"masked.gather"> { - let arguments = (ins LLVM_VectorOf<LLVM_AnyPointer>:$ptrs, - LLVM_VectorOf<I1>:$mask, Variadic<LLVM_AnyVector>:$pass_thru, - I32Attr:$alignment); +def LLVM_masked_gather + : LLVM_OneResultIntrOp<"masked.gather", /*overloadedResults=*/[], + /*overloadedOperands=*/[], /*traits=*/[], + /*requiresAccessGroup=*/1, /*requiresAliasAnalysis=*/1> { + dag args = (ins LLVM_VectorOf<LLVM_AnyPointer>:$ptrs, + LLVM_VectorOf<I1>:$mask, Variadic<LLVM_AnyVector>:$pass_thru, + OptionalAttr<I64Attr>:$alignment); + let arguments = !con(args, baseArgs); let results = (outs LLVM_AnyVector:$res); - let builders = [LLVM_OneResultOpBuilder]; + let builders = [ + LLVM_OneResultOpBuilder, + OpBuilder<(ins "Type":$res, "Value":$ptrs, "Value":$mask, + "ValueRange":$pass_thru, "uint64_t":$alignment), [{ + build($_builder, $_state, res, ptrs, mask, pass_thru, + $_builder.getI64IntegerAttr(alignment), + /*access_groups=*/nullptr, /*alias_scopes=*/nullptr, + /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr); + }]> + ]; let assemblyFormat = "operands attr-dict `:` functional-type(operands, results)"; string llvmBuilder = [{ - $res = $pass_thru.empty() ? builder.CreateMaskedGather( - $_resultType, $ptrs, llvm::Align($alignment), $mask) : + // An absent alignment attribute means the ABI alignment of the type, which + // LLVM spells as an absent align parameter attribute. The IRBuilder helper + // always attaches one, so remove it again in that case. + llvm::Align alignment = + $alignment ? llvm::Align(*$alignment) : llvm::Align(1); + auto *inst = $pass_thru.empty() ? builder.CreateMaskedGather( + $_resultType, $ptrs, alignment, $mask) : builder.CreateMaskedGather( - $_resultType, $ptrs, llvm::Align($alignment), $mask, $pass_thru[0]); - }]; + $_resultType, $ptrs, alignment, $mask, $pass_thru[0]); + if (!$alignment) + inst->removeParamAttr(0, llvm::Attribute::Alignment); + $res = inst; + }] # setAccessGroupsMetadataCode + # setAliasAnalysisMetadataCode; string mlirBuilder = [{ auto *intrinInst = dyn_cast<llvm::IntrinsicInst>(inst); - llvm::Align alignment = intrinInst->getParamAlign(0).valueOrOne(); + llvm::MaybeAlign alignment = intrinInst->getParamAlign(0); $res = LLVM::masked_gather::create($_builder, $_location, $_resultType, $ptrs, $mask, $pass_thru, - $_builder.getI32IntegerAttr(alignment.value())); + alignment ? $_builder.getI64IntegerAttr(alignment->value()) + : ::mlir::IntegerAttr(), + /*access_groups=*/nullptr, /*alias_scopes=*/nullptr, + /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr); }]; - list<int> llvmArgIndices = [0, 1, 2, -1]; + list<int> llvmArgIndices = [0, 1, 2, -1, -1, -1, -1, -1]; let hasVerifier = 1; } /// Create a call to Masked Scatter intrinsic. -def LLVM_masked_scatter : LLVM_ZeroResultIntrOp<"masked.scatter"> { - let arguments = (ins LLVM_AnyVector:$value, LLVM_VectorOf<LLVM_AnyPointer>:$ptrs, - LLVM_VectorOf<I1>:$mask, I32Attr:$alignment); - let builders = [LLVM_VoidResultTypeOpBuilder, LLVM_ZeroResultOpBuilder]; +def LLVM_masked_scatter + : LLVM_ZeroResultIntrOp<"masked.scatter", /*overloadedOperands=*/[], + /*traits=*/[], /*requiresAccessGroup=*/1, /*requiresAliasAnalysis=*/1> { + dag args = (ins LLVM_AnyVector:$value, + LLVM_VectorOf<LLVM_AnyPointer>:$ptrs, + LLVM_VectorOf<I1>:$mask, OptionalAttr<I64Attr>:$alignment); + let arguments = !con(args, baseArgs); + let builders = [ + LLVM_VoidResultTypeOpBuilder, LLVM_ZeroResultOpBuilder, + OpBuilder<(ins "Value":$value, "Value":$ptrs, "Value":$mask, + "uint64_t":$alignment), [{ + build($_builder, $_state, value, ptrs, mask, + $_builder.getI64IntegerAttr(alignment), + /*access_groups=*/nullptr, /*alias_scopes=*/nullptr, + /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr); + }]> + ]; let assemblyFormat = "$value `,` $ptrs `,` $mask attr-dict `:` " "type($value) `,` type($mask) `into` type($ptrs)"; string llvmBuilder = [{ - builder.CreateMaskedScatter( - $value, $ptrs, llvm::Align($alignment), $mask); - }]; + // An absent alignment attribute means the ABI alignment of the type, which + // LLVM spells as an absent align parameter attribute. The IRBuilder helper + // always attaches one, so remove it again in that case. + llvm::Align alignment = + $alignment ? llvm::Align(*$alignment) : llvm::Align(1); + auto *inst = builder.CreateMaskedScatter($value, $ptrs, alignment, $mask); + if (!$alignment) + inst->removeParamAttr(1, llvm::Attribute::Alignment); + }] # setAccessGroupsMetadataCode + # setAliasAnalysisMetadataCode; string mlirBuilder = [{ auto *intrinInst = dyn_cast<llvm::IntrinsicInst>(inst); - llvm::Align alignment = intrinInst->getParamAlign(1).valueOrOne(); + llvm::MaybeAlign alignment = intrinInst->getParamAlign(1); $_op = LLVM::masked_scatter::create($_builder, $_location, - $value, $ptrs, $mask, $_builder.getI32IntegerAttr(alignment.value())); + $value, $ptrs, $mask, + alignment ? $_builder.getI64IntegerAttr(alignment->value()) + : ::mlir::IntegerAttr(), + /*access_groups=*/nullptr, /*alias_scopes=*/nullptr, + /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr); }]; - list<int> llvmArgIndices = [0, 1, 2, -1]; + list<int> llvmArgIndices = [0, 1, 2, -1, -1, -1, -1, -1]; let hasVerifier = 1; } @@ -1468,7 +1585,8 @@ def LLVM_masked_scatter : LLVM_ZeroResultIntrOp<"masked.scatter"> { /// Create a call to Masked Expand Load intrinsic. def LLVM_masked_expandload : LLVM_OneResultIntrOp<"masked.expandload", [0], [0], - /*traits=*/[], /*requiresFastMath=*/0, /*requiresArgAndResultAttrs=*/1, + /*traits=*/[], /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0, + /*requiresFastmath=*/0, /*requiresArgAndResultAttrs=*/1, /*immArgPositions=*/[], /*immArgAttrNames=*/[]> { dag args = (ins LLVM_AnyPointer:$ptr, LLVM_VectorOf<I1>:$mask, @@ -1580,6 +1698,7 @@ def LLVM_vector_insert PredOpTrait<"it is not inserting scalable into fixed-length vectors.", CPred<"!isScalableVectorType($srcvec.getType()) || " "isScalableVectorType($dstvec.getType())">>], + /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0, /*requiresFastmath=*/0, /*requiresArgAndResultAttrs=*/0, /*immArgPositions=*/[2], /*immArgAttrNames=*/["pos"]> { let arguments = (ins LLVM_AnyVector:$dstvec, LLVM_AnyVector:$srcvec, @@ -1614,6 +1733,7 @@ def LLVM_vector_extract PredOpTrait<"it is not extracting scalable from fixed-length vectors.", CPred<"!isScalableVectorType($res.getType()) || " "isScalableVectorType($srcvec.getType())">>], + /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0, /*requiresFastmath=*/0, /*requiresArgAndResultAttrs=*/0, /*immArgPositions=*/[1], /*immArgAttrNames=*/["pos"]> { let arguments = (ins LLVM_AnyVector:$srcvec, I64Attr:$pos); diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td index a787840663171..c71f9d0a3f54f 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td @@ -473,12 +473,14 @@ class LLVM_ZeroResultIntrOp<string mnem, list<int> overloadedOperands = [], class LLVM_OneResultIntrOp<string mnem, list<int> overloadedResults = [], list<int> overloadedOperands = [], list<Trait> traits = [], + bit requiresAccessGroup = 0, + bit requiresAliasAnalysis = 0, bit requiresFastmath = 0, bit requiresArgAndResultAttrs = 0, list<int> immArgPositions = [], list<string> immArgAttrNames = []> : LLVM_IntrOp<mnem, overloadedResults, overloadedOperands, traits, 1, - /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0, + requiresAccessGroup, requiresAliasAnalysis, requiresFastmath, requiresArgAndResultAttrs, /*requiresOpBundles=*/0, immArgPositions, immArgAttrNames>; diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp index 152c45b695cee..c1252b86cbcbb 100644 --- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp +++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp @@ -345,7 +345,7 @@ class VectorGatherOpConversion // Replace with the gather intrinsic. rewriter.replaceOpWithNewOp<LLVM::masked_gather>( gather, typeConverter->convertType(vType), ptrs, adaptor.getMask(), - adaptor.getPassThru(), rewriter.getI32IntegerAttr(align)); + adaptor.getPassThru(), align); return success(); } @@ -403,8 +403,7 @@ class VectorScatterOpConversion // Replace with the scatter intrinsic. rewriter.replaceOpWithNewOp<LLVM::masked_scatter>( - scatter, adaptor.getValueToStore(), ptrs, adaptor.getMask(), - rewriter.getI32IntegerAttr(align)); + scatter, adaptor.getValueToStore(), ptrs, adaptor.getMask(), align); return success(); } diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMInterfaces.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMInterfaces.cpp index 406ccd6849972..44a9284f5a5ab 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMInterfaces.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMInterfaces.cpp @@ -116,6 +116,22 @@ SmallVector<Value> mlir::LLVM::MemsetInlineOp::getAccessedOperands() { return {getDst()}; } +SmallVector<Value> mlir::LLVM::MaskedLoadOp::getAccessedOperands() { + return {getData()}; +} + +SmallVector<Value> mlir::LLVM::MaskedStoreOp::getAccessedOperands() { + return {getData()}; +} + +SmallVector<Value> mlir::LLVM::masked_gather::getAccessedOperands() { + return {getPtrs()}; +} + +SmallVector<Value> mlir::LLVM::masked_scatter::getAccessedOperands() { + return {getPtrs()}; +} + SmallVector<Value> mlir::LLVM::CallOp::getAccessedOperands() { // Note: This must not use `getArgOperands`, which excludes the variadic // arguments of a call to a variadic callee. Those are passed to the callee diff --git a/mlir/test/Conversion/VectorToLLVM/use-vector-alignment.mlir b/mlir/test/Conversion/VectorToLLVM/use-vector-alignment.mlir index bc0bb9d69ec80..fd4a9eb765e1b 100644 --- a/mlir/test/Conversion/VectorToLLVM/use-vector-alignment.mlir +++ b/mlir/test/Conversion/VectorToLLVM/use-vector-alignment.mlir @@ -72,8 +72,8 @@ func.func @masked_load(%base: memref<?xf32>, %mask: vector<16xi1>, %passthru: ve // ALL-LABEL: func @masked_load -// VEC-ALIGN: %[[L:.*]] = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 64 : i32} : (!llvm.ptr, vector<16xi1>, vector<16xf32>) -> vector<16xf32> -// MEMREF-ALIGN: %[[L:.*]] = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 4 : i32} : (!llvm.ptr, vector<16xi1>, vector<16xf32>) -> vector<16xf32> +// VEC-ALIGN: %[[L:.*]] = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 64 : i64} : (!llvm.ptr, vector<16xi1>, vector<16xf32>) -> vector<16xf32> +// MEMREF-ALIGN: %[[L:.*]] = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 4 : i64} : (!llvm.ptr, vector<16xi1>, vector<16xf32>) -> vector<16xf32> // ----- @@ -85,8 +85,8 @@ func.func @masked_load_with_alignment_attribute(%base: memref<?xf32>, %mask: vec // ALL-LABEL: func @masked_load_with_alignment_attribute -// VEC-ALIGN: %[[L:.*]] = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : (!llvm.ptr, vector<16xi1>, vector<16xf32>) -> vector<16xf32> -// MEMREF-ALIGN: %[[L:.*]] = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : (!llvm.ptr, vector<16xi1>, vector<16xf32>) -> vector<16xf32> +// VEC-ALIGN: %[[L:.*]] = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i64} : (!llvm.ptr, vector<16xi1>, vector<16xf32>) -> vector<16xf32> +// MEMREF-ALIGN: %[[L:.*]] = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i64} : (!llvm.ptr, vector<16xi1>, vector<16xf32>) -> vector<16xf32> // ----- @@ -102,8 +102,8 @@ func.func @masked_store(%base: memref<?xf32>, %mask: vector<16xi1>, %passthru: v // ALL-LABEL: func @masked_store -// VEC-ALIGN: llvm.intr.masked.store %{{.*}}, %{{.*}}, %{{.*}} {alignment = 64 : i32} : vector<16xf32>, vector<16xi1> into !llvm.ptr -// MEMREF-ALIGN: llvm.intr.masked.store %{{.*}}, %{{.*}}, %{{.*}} {alignment = 4 : i32} : vector<16xf32>, vector<16xi1> into !llvm.ptr +// VEC-ALIGN: llvm.intr.masked.store %{{.*}}, %{{.*}}, %{{.*}} {alignment = 64 : i64} : vector<16xf32>, vector<16xi1> into !llvm.ptr +// MEMREF-ALIGN: llvm.intr.masked.store %{{.*}}, %{{.*}}, %{{.*}} {alignment = 4 : i64} : vector<16xf32>, vector<16xi1> into !llvm.ptr // ----- @@ -115,8 +115,8 @@ func.func @masked_store_with_alignment_attribute(%base: memref<?xf32>, %mask: ve // ALL-LABEL: func @masked_store_with_alignment_attribute -// VEC-ALIGN: llvm.intr.masked.store %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : vector<16xf32>, vector<16xi1> into !llvm.ptr -// MEMREF-ALIGN: llvm.intr.masked.store %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : vector<16xf32>, vector<16xi1> into !llvm.ptr +// VEC-ALIGN: llvm.intr.masked.store %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i64} : vector<16xf32>, vector<16xi1> into !llvm.ptr +// MEMREF-ALIGN: llvm.intr.masked.store %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i64} : vector<16xf32>, vector<16xi1> into !llvm.ptr // ----- @@ -132,8 +132,8 @@ func.func @scatter(%base: memref<?xf32>, %index: vector<3xi32>, %mask: vector<3x // ALL-LABEL: func @scatter -// VEC-ALIGN: llvm.intr.masked.scatter %{{.*}}, %{{.*}}, %{{.*}} {alignment = 16 : i32} : vector<3xf32>, vector<3xi1> into vector<3x!llvm.ptr> -// MEMREF-ALIGN: llvm.intr.masked.scatter %{{.*}}, %{{.*}}, %{{.*}} {alignment = 4 : i32} : vector<3xf32>, vector<3xi1> into vector<3x!llvm.ptr> +// VEC-ALIGN: llvm.intr.masked.scatter %{{.*}}, %{{.*}}, %{{.*}} {alignment = 16 : i64} : vector<3xf32>, vector<3xi1> into vector<3x!llvm.ptr> +// MEMREF-ALIGN: llvm.intr.masked.scatter %{{.*}}, %{{.*}}, %{{.*}} {alignment = 4 : i64} : vector<3xf32>, vector<3xi1> into vector<3x!llvm.ptr> // ----- @@ -145,8 +145,8 @@ func.func @scatter_with_alignment_attribute(%base: memref<?xf32>, %index: vector // ALL-LABEL: func @scatter_with_alignment_attribute -// VEC-ALIGN: llvm.intr.masked.scatter %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : vector<3xf32>, vector<3xi1> into vector<3x!llvm.ptr> -// MEMREF-ALIGN: llvm.intr.masked.scatter %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : vector<3xf32>, vector<3xi1> into vector<3x!llvm.ptr> +// VEC-ALIGN: llvm.intr.masked.scatter %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i64} : vector<3xf32>, vector<3xi1> into vector<3x!llvm.ptr> +// MEMREF-ALIGN: llvm.intr.masked.scatter %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i64} : vector<3xf32>, vector<3xi1> into vector<3x!llvm.ptr> // ----- @@ -162,8 +162,8 @@ func.func @gather(%base: memref<?xf32>, %index: vector<3xi32>, %mask: vector<3xi // ALL-LABEL: func @gather -// VEC-ALIGN: %[[G:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 16 : i32} : (vector<3x!llvm.ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32> -// MEMREF-ALIGN: %[[G:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 4 : i32} : (vector<3x!llvm.ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32> +// VEC-ALIGN: %[[G:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 16 : i64} : (vector<3x!llvm.ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32> +// MEMREF-ALIGN: %[[G:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 4 : i64} : (vector<3x!llvm.ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32> // ----- @@ -175,5 +175,5 @@ func.func @gather_with_alignment_attribute(%base: memref<?xf32>, %index: vector< // ALL-LABEL: func @gather_with_alignment_attribute -// VEC-ALIGN: %[[G:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : (vector<3x!llvm.ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32> -// MEMREF-ALIGN: %[[G:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : (vector<3x!llvm.ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32> +// VEC-ALIGN: %[[G:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i64} : (vector<3x!llvm.ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32> +// MEMREF-ALIGN: %[[G:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i64} : (vector<3x!llvm.ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32> diff --git a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir index 4b89bc60a3d76..f7c7929ef9605 100644 --- a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir +++ b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir @@ -1591,7 +1591,7 @@ func.func @masked_load(%arg0: memref<?xf32>, %arg1: vector<16xi1>, %arg2: vector // CHECK: %[[CO:.*]] = arith.constant 0 : index // CHECK: %[[C:.*]] = builtin.unrealized_conversion_cast %[[CO]] : index to i64 // CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%[[C]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 -// CHECK: %[[L:.*]] = llvm.intr.masked.load %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i32} : (!llvm.ptr, vector<16xi1>, vector<16xf32>) -> vector<16xf32> +// CHECK: %[[L:.*]] = llvm.intr.masked.load %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i64} : (!llvm.ptr, vector<16xi1>, vector<16xf32>) -> vector<16xf32> // CHECK: return %[[L]] : vector<16xf32> // ----- @@ -1606,7 +1606,7 @@ func.func @masked_load_scalable(%arg0: memref<?xf32>, %arg1: vector<[16]xi1>, %a // CHECK: %[[CO:.*]] = arith.constant 0 : index // CHECK: %[[C:.*]] = builtin.unrealized_conversion_cast %[[CO]] : index to i64 // CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%[[C]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 -// CHECK: %[[L:.*]] = llvm.intr.masked.load %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i32} : (!llvm.ptr, vector<[16]xi1>, vector<[16]xf32>) -> vector<[16]xf32> +// CHECK: %[[L:.*]] = llvm.intr.masked.load %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i64} : (!llvm.ptr, vector<[16]xi1>, vector<[16]xf32>) -> vector<[16]xf32> // CHECK: return %[[L]] : vector<[16]xf32> // ----- @@ -1617,7 +1617,7 @@ func.func @masked_load_index(%arg0: memref<?xindex>, %arg1: vector<16xi1>, %arg2 return %0 : vector<16xindex> } // CHECK-LABEL: func @masked_load_index -// CHECK: %{{.*}} = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : (!llvm.ptr, vector<16xi1>, vector<16xi64>) -> vector<16xi64> +// CHECK: %{{.*}} = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i64} : (!llvm.ptr, vector<16xi1>, vector<16xi64>) -> vector<16xi64> // ----- @@ -1627,7 +1627,7 @@ func.func @masked_load_index_scalable(%arg0: memref<?xindex>, %arg1: vector<[16] return %0 : vector<[16]xindex> } // CHECK-LABEL: func @masked_load_index_scalable -// CHECK: %{{.*}} = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : (!llvm.ptr, vector<[16]xi1>, vector<[16]xi64>) -> vector<[16]xi64> +// CHECK: %{{.*}} = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i64} : (!llvm.ptr, vector<[16]xi1>, vector<[16]xi64>) -> vector<[16]xi64> // ----- @@ -1637,7 +1637,7 @@ func.func @masked_load_with_alignment(%arg0: memref<?xf32>, %arg1: vector<16xi1> } // CHECK-LABEL: func @masked_load_with_alignment -// CHECK: llvm.intr.masked.load %{{.*}} {alignment = 2 : i32} : (!llvm.ptr, vector<16xi1>, vector<16xf32>) -> vector<16xf32> +// CHECK: llvm.intr.masked.load %{{.*}} {alignment = 2 : i64} : (!llvm.ptr, vector<16xi1>, vector<16xf32>) -> vector<16xf32> // ----- @@ -1655,7 +1655,7 @@ func.func @masked_store(%arg0: memref<?xf32>, %arg1: vector<16xi1>, %arg2: vecto // CHECK: %[[CO:.*]] = arith.constant 0 : index // CHECK: %[[C:.*]] = builtin.unrealized_conversion_cast %[[CO]] : index to i64 // CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%[[C]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 -// CHECK: llvm.intr.masked.store %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i32} : vector<16xf32>, vector<16xi1> into !llvm.ptr +// CHECK: llvm.intr.masked.store %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i64} : vector<16xf32>, vector<16xi1> into !llvm.ptr // ----- @@ -1669,7 +1669,7 @@ func.func @masked_store_scalable(%arg0: memref<?xf32>, %arg1: vector<[16]xi1>, % // CHECK: %[[CO:.*]] = arith.constant 0 : index // CHECK: %[[C:.*]] = builtin.unrealized_conversion_cast %[[CO]] : index to i64 // CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%[[C]]] : (!llvm.ptr, i64) -> !llvm.ptr, f32 -// CHECK: llvm.intr.masked.store %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i32} : vector<[16]xf32>, vector<[16]xi1> into !llvm.ptr +// CHECK: llvm.intr.masked.store %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i64} : vector<[16]xf32>, vector<[16]xi1> into !llvm.ptr // ----- @@ -1679,7 +1679,7 @@ func.func @masked_store_index(%arg0: memref<?xindex>, %arg1: vector<16xi1>, %arg return } // CHECK-LABEL: func @masked_store_index -// CHECK: llvm.intr.masked.store %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : vector<16xi64>, vector<16xi1> into !llvm.ptr +// CHECK: llvm.intr.masked.store %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i64} : vector<16xi64>, vector<16xi1> into !llvm.ptr // ----- @@ -1689,7 +1689,7 @@ func.func @masked_store_index_scalable(%arg0: memref<?xindex>, %arg1: vector<[16 return } // CHECK-LABEL: func @masked_store_index_scalable -// CHECK: llvm.intr.masked.store %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : vector<[16]xi64>, vector<[16]xi1> into !llvm.ptr +// CHECK: llvm.intr.masked.store %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i64} : vector<[16]xi64>, vector<[16]xi1> into !llvm.ptr // ----- @@ -1699,7 +1699,7 @@ func.func @masked_store_with_alignment(%arg0: memref<?xf32>, %arg1: vector<16xi1 } // CHECK-LABEL: func @masked_store_with_alignment -// CHECK: llvm.intr.masked.store %{{.*}} {alignment = 2 : i32} : vector<16xf32>, vector<16xi1> into !llvm.ptr +// CHECK: llvm.intr.masked.store %{{.*}} {alignment = 2 : i64} : vector<16xf32>, vector<16xi1> into !llvm.ptr // ----- @@ -1715,7 +1715,7 @@ func.func @gather(%arg0: memref<?xf32>, %arg1: vector<3xi32>, %arg2: vector<3xi1 // CHECK-LABEL: func @gather // CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, vector<3xi32>) -> vector<3x!llvm.ptr>, f32 -// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i32} : (vector<3x!llvm.ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32> +// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i64} : (vector<3x!llvm.ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32> // CHECK: return %[[G]] : vector<3xf32> // ----- @@ -1728,7 +1728,7 @@ func.func @gather_scalable(%arg0: memref<?xf32>, %arg1: vector<[3]xi32>, %arg2: // CHECK-LABEL: func @gather_scalable // CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, vector<[3]xi32>) -> vector<[3]x!llvm.ptr>, f32 -// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i32} : (vector<[3]x!llvm.ptr>, vector<[3]xi1>, vector<[3]xf32>) -> vector<[3]xf32> +// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i64} : (vector<[3]x!llvm.ptr>, vector<[3]xi1>, vector<[3]xf32>) -> vector<[3]xf32> // CHECK: return %[[G]] : vector<[3]xf32> // ----- @@ -1741,7 +1741,7 @@ func.func @gather_global_memory(%arg0: memref<?xf32, 1>, %arg1: vector<3xi32>, % // CHECK-LABEL: func @gather_global_memory // CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr<1>, vector<3xi32>) -> vector<3x!llvm.ptr<1>>, f32 -// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i32} : (vector<3x!llvm.ptr<1>>, vector<3xi1>, vector<3xf32>) -> vector<3xf32> +// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i64} : (vector<3x!llvm.ptr<1>>, vector<3xi1>, vector<3xf32>) -> vector<3xf32> // CHECK: return %[[G]] : vector<3xf32> // ----- @@ -1754,7 +1754,7 @@ func.func @gather_global_memory_scalable(%arg0: memref<?xf32, 1>, %arg1: vector< // CHECK-LABEL: func @gather_global_memory_scalable // CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr<1>, vector<[3]xi32>) -> vector<[3]x!llvm.ptr<1>>, f32 -// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i32} : (vector<[3]x!llvm.ptr<1>>, vector<[3]xi1>, vector<[3]xf32>) -> vector<[3]xf32> +// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i64} : (vector<[3]x!llvm.ptr<1>>, vector<[3]xi1>, vector<[3]xf32>) -> vector<[3]xf32> // CHECK: return %[[G]] : vector<[3]xf32> // ----- @@ -1768,7 +1768,7 @@ func.func @gather_index(%arg0: memref<?xindex>, %arg1: vector<3xindex>, %arg2: v // CHECK-LABEL: func @gather_index // CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, vector<3xi64>) -> vector<3x!llvm.ptr>, i64 -// CHECK: %[[G:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : (vector<3x!llvm.ptr>, vector<3xi1>, vector<3xi64>) -> vector<3xi64> +// CHECK: %[[G:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i64} : (vector<3x!llvm.ptr>, vector<3xi1>, vector<3xi64>) -> vector<3xi64> // CHECK: %{{.*}} = builtin.unrealized_conversion_cast %[[G]] : vector<3xi64> to vector<3xindex> // ----- @@ -1781,7 +1781,7 @@ func.func @gather_index_scalable(%arg0: memref<?xindex>, %arg1: vector<[3]xindex // CHECK-LABEL: func @gather_index_scalable // CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, vector<[3]xi64>) -> vector<[3]x!llvm.ptr>, i64 -// CHECK: %[[G:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : (vector<[3]x!llvm.ptr>, vector<[3]xi1>, vector<[3]xi64>) -> vector<[3]xi64> +// CHECK: %[[G:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i64} : (vector<[3]x!llvm.ptr>, vector<[3]xi1>, vector<[3]xi64>) -> vector<[3]xi64> // CHECK: %{{.*}} = builtin.unrealized_conversion_cast %[[G]] : vector<[3]xi64> to vector<[3]xindex> // ----- @@ -1795,7 +1795,7 @@ func.func @gather_1d_from_2d(%arg0: memref<4x4xf32>, %arg1: vector<4xi32>, %arg2 // CHECK-LABEL: func @gather_1d_from_2d // CHECK: %[[B:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, i64) -> !llvm.ptr, f32 // CHECK: %[[P:.*]] = llvm.getelementptr %[[B]][%{{.*}}] : (!llvm.ptr, vector<4xi32>) -> vector<4x!llvm.ptr>, f32 -// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i32} : (vector<4x!llvm.ptr>, vector<4xi1>, vector<4xf32>) -> vector<4xf32> +// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i64} : (vector<4x!llvm.ptr>, vector<4xi1>, vector<4xf32>) -> vector<4xf32> // CHECK: return %[[G]] : vector<4xf32> // ----- @@ -1809,7 +1809,7 @@ func.func @gather_1d_from_2d_scalable(%arg0: memref<4x?xf32>, %arg1: vector<[4]x // CHECK-LABEL: func @gather_1d_from_2d_scalable // CHECK: %[[B:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, i64) -> !llvm.ptr, f32 // CHECK: %[[P:.*]] = llvm.getelementptr %[[B]][%{{.*}}] : (!llvm.ptr, vector<[4]xi32>) -> vector<[4]x!llvm.ptr>, f32 -// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i32} : (vector<[4]x!llvm.ptr>, vector<[4]xi1>, vector<[4]xf32>) -> vector<[4]xf32> +// CHECK: %[[G:.*]] = llvm.intr.masked.gather %[[P]], %{{.*}}, %{{.*}} {alignment = 4 : i64} : (vector<[4]x!llvm.ptr>, vector<[4]xi1>, vector<[4]xf32>) -> vector<[4]xf32> // CHECK: return %[[G]] : vector<[4]xf32> // ----- @@ -1820,7 +1820,7 @@ func.func @gather_with_alignment(%arg0: memref<?xf32>, %arg1: vector<3xi32>, %ar } // CHECK-LABEL: func @gather_with_alignment -// CHECK: llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : (vector<3x!llvm.ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32> +// CHECK: llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i64} : (vector<3x!llvm.ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32> // ----- @@ -1850,7 +1850,7 @@ func.func @scatter(%arg0: memref<?xf32>, %arg1: vector<3xi32>, %arg2: vector<3xi // CHECK-LABEL: func @scatter // CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, vector<3xi32>) -> vector<3x!llvm.ptr>, f32 -// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i32} : vector<3xf32>, vector<3xi1> into vector<3x!llvm.ptr> +// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i64} : vector<3xf32>, vector<3xi1> into vector<3x!llvm.ptr> // ----- @@ -1862,7 +1862,7 @@ func.func @scatter_scalable(%arg0: memref<?xf32>, %arg1: vector<[3]xi32>, %arg2: // CHECK-LABEL: func @scatter_scalable // CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, vector<[3]xi32>) -> vector<[3]x!llvm.ptr>, f32 -// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i32} : vector<[3]xf32>, vector<[3]xi1> into vector<[3]x!llvm.ptr> +// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i64} : vector<[3]xf32>, vector<[3]xi1> into vector<[3]x!llvm.ptr> // ----- @@ -1874,7 +1874,7 @@ func.func @scatter_index(%arg0: memref<?xindex>, %arg1: vector<3xindex>, %arg2: // CHECK-LABEL: func @scatter_index // CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, vector<3xi64>) -> vector<3x!llvm.ptr>, i64 -// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 8 : i32} : vector<3xi64>, vector<3xi1> into vector<3x!llvm.ptr> +// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 8 : i64} : vector<3xi64>, vector<3xi1> into vector<3x!llvm.ptr> // ----- @@ -1886,7 +1886,7 @@ func.func @scatter_index_scalable(%arg0: memref<?xindex>, %arg1: vector<[3]xinde // CHECK-LABEL: func @scatter_index_scalable // CHECK: %[[P:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, vector<[3]xi64>) -> vector<[3]x!llvm.ptr>, i64 -// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 8 : i32} : vector<[3]xi64>, vector<[3]xi1> into vector<[3]x!llvm.ptr> +// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 8 : i64} : vector<[3]xi64>, vector<[3]xi1> into vector<[3]x!llvm.ptr> // ----- @@ -1899,7 +1899,7 @@ func.func @scatter_1d_into_2d(%arg0: memref<4x4xf32>, %arg1: vector<4xi32>, %arg // CHECK-LABEL: func @scatter_1d_into_2d // CHECK: %[[B:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, i64) -> !llvm.ptr, f32 // CHECK: %[[P:.*]] = llvm.getelementptr %[[B]][%{{.*}}] : (!llvm.ptr, vector<4xi32>) -> vector<4x!llvm.ptr>, f32 -// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i32} : vector<4xf32>, vector<4xi1> into vector<4x!llvm.ptr> +// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i64} : vector<4xf32>, vector<4xi1> into vector<4x!llvm.ptr> // ----- @@ -1912,7 +1912,7 @@ func.func @scatter_1d_into_2d_scalable(%arg0: memref<4x?xf32>, %arg1: vector<[4] // CHECK-LABEL: func @scatter_1d_into_2d_scalable // CHECK: %[[B:.*]] = llvm.getelementptr %{{.*}}[%{{.*}}] : (!llvm.ptr, i64) -> !llvm.ptr, f32 // CHECK: %[[P:.*]] = llvm.getelementptr %[[B]][%{{.*}}] : (!llvm.ptr, vector<[4]xi32>) -> vector<[4]x!llvm.ptr>, f32 -// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i32} : vector<[4]xf32>, vector<[4]xi1> into vector<[4]x!llvm.ptr> +// CHECK: llvm.intr.masked.scatter %{{.*}}, %[[P]], %{{.*}} {alignment = 4 : i64} : vector<[4]xf32>, vector<[4]xi1> into vector<[4]x!llvm.ptr> // ----- @@ -1922,7 +1922,7 @@ func.func @scatter_with_alignment(%arg0: memref<?xf32>, %arg1: vector<3xi32>, %a } // CHECK-LABEL: func @scatter_with_alignment -// CHECK: llvm.intr.masked.scatter %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : vector<3xf32>, vector<3xi1> into vector<3x!llvm.ptr> +// CHECK: llvm.intr.masked.scatter %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i64} : vector<3xf32>, vector<3xi1> into vector<3x!llvm.ptr> // ----- diff --git a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir index e6dbb7b55d48d..6eba18fafff05 100644 --- a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir +++ b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir @@ -1606,8 +1606,8 @@ func.func @gather_with_mask(%arg0: memref<?xf32>, %arg1: vector<2x3xi32>, %arg2: } // CHECK-LABEL: func @gather_with_mask -// CHECK: %[[G0:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 4 : i32} : (vector<3x!llvm.ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32> -// CHECK: %[[G1:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 4 : i32} : (vector<3x!llvm.ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32> +// CHECK: %[[G0:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 4 : i64} : (vector<3x!llvm.ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32> +// CHECK: %[[G1:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 4 : i64} : (vector<3x!llvm.ptr>, vector<3xi1>, vector<3xf32>) -> vector<3xf32> // ----- @@ -1622,8 +1622,8 @@ func.func @gather_with_mask_scalable(%arg0: memref<?xf32>, %arg1: vector<2x[3]xi } // CHECK-LABEL: func @gather_with_mask_scalable -// CHECK: %[[G0:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 4 : i32} : (vector<[3]x!llvm.ptr>, vector<[3]xi1>, vector<[3]xf32>) -> vector<[3]xf32> -// CHECK: %[[G1:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 4 : i32} : (vector<[3]x!llvm.ptr>, vector<[3]xi1>, vector<[3]xf32>) -> vector<[3]xf32> +// CHECK: %[[G0:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 4 : i64} : (vector<[3]x!llvm.ptr>, vector<[3]xi1>, vector<[3]xf32>) -> vector<[3]xf32> +// CHECK: %[[G1:.*]] = llvm.intr.masked.gather %{{.*}}, %{{.*}}, %{{.*}} {alignment = 4 : i64} : (vector<[3]x!llvm.ptr>, vector<[3]xi1>, vector<[3]xf32>) -> vector<[3]xf32> // ----- diff --git a/mlir/test/Conversion/VectorToLLVM/vector-xfer-to-llvm.mlir b/mlir/test/Conversion/VectorToLLVM/vector-xfer-to-llvm.mlir index 18deadd0d7a79..3b314830d58c1 100644 --- a/mlir/test/Conversion/VectorToLLVM/vector-xfer-to-llvm.mlir +++ b/mlir/test/Conversion/VectorToLLVM/vector-xfer-to-llvm.mlir @@ -41,7 +41,7 @@ func.func @transfer_read_write_1d(%A : memref<?xf32>, %base: index) -> vector<17 // // 6. Rewrite as a masked read. // CHECK: %[[loaded:.*]] = llvm.intr.masked.load %[[gep]], %[[mask]], -// CHECK-SAME: %[[PASS_THROUGH]] {alignment = 4 : i32} : +// CHECK-SAME: %[[PASS_THROUGH]] {alignment = 4 : i64} : // CHECK-SAME: -> vector<17xf32> // // 1. Let dim be the memref dimension, compute the in-bound index (dim - offset) @@ -62,7 +62,7 @@ func.func @transfer_read_write_1d(%A : memref<?xf32>, %base: index) -> vector<17 // // 4. Rewrite as a masked write. // CHECK: llvm.intr.masked.store %[[loaded]], %[[gep_b]], %[[mask_b]] -// CHECK-SAME: {alignment = 4 : i32} : +// CHECK-SAME: {alignment = 4 : i64} : // CHECK-SAME: vector<17xf32>, vector<17xi1> into !llvm.ptr func.func @transfer_read_write_1d_scalable(%A : memref<?xf32>, %base: index) -> vector<[17]xf32> { @@ -105,7 +105,7 @@ func.func @transfer_read_write_1d_scalable(%A : memref<?xf32>, %base: index) -> // // 6. Rewrite as a masked read. // CHECK: %[[loaded:.*]] = llvm.intr.masked.load %[[gep]], %[[mask]], -// CHECK-SAME: %[[PASS_THROUGH]] {alignment = 4 : i32} : +// CHECK-SAME: %[[PASS_THROUGH]] {alignment = 4 : i64} : // CHECK-SAME: -> vector<[17]xf32> // // 1. Let dim be the memref dimension, compute the in-bound index (dim - offset) @@ -129,7 +129,7 @@ func.func @transfer_read_write_1d_scalable(%A : memref<?xf32>, %base: index) -> // // 5. Rewrite as a masked write. // CHECK: llvm.intr.masked.store %[[loaded]], %[[gep_b]], %[[mask_b]] -// CHECK-SAME: {alignment = 4 : i32} : +// CHECK-SAME: {alignment = 4 : i64} : // CHECK-SAME: vector<[17]xf32>, vector<[17]xi1> into !llvm.ptr // ----- @@ -149,10 +149,10 @@ func.func @transfer_read_write_index_1d(%A : memref<?xindex>, %base: index) -> v // CHECK: %[[SPLAT:.*]] = arith.constant dense<7> : vector<17xindex> // CHECK: %{{.*}} = builtin.unrealized_conversion_cast %[[SPLAT]] : vector<17xindex> to vector<17xi64> -// CHECK: %[[loaded:.*]] = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : +// CHECK: %[[loaded:.*]] = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i64} : // CHECK-SAME: (!llvm.ptr, vector<17xi1>, vector<17xi64>) -> vector<17xi64> -// CHECK: llvm.intr.masked.store %[[loaded]], %{{.*}}, %{{.*}} {alignment = 8 : i32} : +// CHECK: llvm.intr.masked.store %[[loaded]], %{{.*}}, %{{.*}} {alignment = 8 : i64} : // CHECK-SAME: vector<17xi64>, vector<17xi1> into !llvm.ptr func.func @transfer_read_write_index_1d_scalable(%A : memref<?xindex>, %base: index) -> vector<[17]xindex> { @@ -170,10 +170,10 @@ func.func @transfer_read_write_index_1d_scalable(%A : memref<?xindex>, %base: in // CHECK: %[[SPLAT:.*]] = arith.constant dense<7> : vector<[17]xindex> // CHECK: %{{.*}} = builtin.unrealized_conversion_cast %[[SPLAT]] : vector<[17]xindex> to vector<[17]xi64> -// CHECK: %[[loaded:.*]] = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i32} : +// CHECK: %[[loaded:.*]] = llvm.intr.masked.load %{{.*}}, %{{.*}}, %{{.*}} {alignment = 8 : i64} : // CHECK-SAME: (!llvm.ptr, vector<[17]xi1>, vector<[17]xi64>) -> vector<[17]xi64> -// CHECK: llvm.intr.masked.store %[[loaded]], %{{.*}}, %{{.*}} {alignment = 8 : i32} : +// CHECK: llvm.intr.masked.store %[[loaded]], %{{.*}}, %{{.*}} {alignment = 8 : i64} : // CHECK-SAME: vector<[17]xi64>, vector<[17]xi1> into !llvm.ptr // ----- diff --git a/mlir/test/Dialect/LLVMIR/opaque-ptr.mlir b/mlir/test/Dialect/LLVMIR/opaque-ptr.mlir index 82c3e5bf2e2db..8a4357e62a5cc 100644 --- a/mlir/test/Dialect/LLVMIR/opaque-ptr.mlir +++ b/mlir/test/Dialect/LLVMIR/opaque-ptr.mlir @@ -62,7 +62,7 @@ llvm.func @opaque_ptr_matrix_load_store(%ptr: !llvm.ptr, %stride: i64) -> vector llvm.func @opaque_ptr_masked_load(%arg0: !llvm.ptr, %arg1: vector<7xi1>) -> vector<7xf32> { // CHECK: = llvm.intr.masked.load // CHECK: (!llvm.ptr, vector<7xi1>) -> vector<7xf32> - %0 = llvm.intr.masked.load %arg0, %arg1 { alignment = 1: i32} : + %0 = llvm.intr.masked.load %arg0, %arg1 { alignment = 1: i64} : (!llvm.ptr, vector<7xi1>) -> vector<7xf32> llvm.return %0 : vector<7 x f32> } @@ -71,7 +71,7 @@ llvm.func @opaque_ptr_masked_load(%arg0: !llvm.ptr, %arg1: vector<7xi1>) -> vect llvm.func @opaque_ptr_gather(%M: vector<7x!llvm.ptr>, %mask: vector<7xi1>) -> vector<7xf32> { // CHECK: = llvm.intr.masked.gather // CHECK: (vector<7x!llvm.ptr>, vector<7xi1>) -> vector<7xf32> - %a = llvm.intr.masked.gather %M, %mask { alignment = 1: i32} : + %a = llvm.intr.masked.gather %M, %mask { alignment = 1: i64} : (vector<7x!llvm.ptr>, vector<7xi1>) -> vector<7xf32> llvm.return %a : vector<7xf32> } diff --git a/mlir/test/Dialect/LLVMIR/roundtrip.mlir b/mlir/test/Dialect/LLVMIR/roundtrip.mlir index 42581ca2e3d92..9c594680e0496 100644 --- a/mlir/test/Dialect/LLVMIR/roundtrip.mlir +++ b/mlir/test/Dialect/LLVMIR/roundtrip.mlir @@ -1296,3 +1296,26 @@ llvm.func @repeated_function_metadata() attributes { #llvm.func_metadata<"type", #llvm.md_node<#llvm.md_const<0 : i64>, #llvm.md_string<"typeid1">>> ] } + +#rt_alias_scope_domain = #llvm.alias_scope_domain<id = distinct[4]<>, description = "rt domain"> +#rt_alias_scope = #llvm.alias_scope<id = distinct[5]<>, domain = #rt_alias_scope_domain> +#rt_access_group = #llvm.access_group<id = distinct[6]<>> + +// CHECK-LABEL: @masked_intrinsic_metadata_roundtrip +llvm.func @masked_intrinsic_metadata_roundtrip(%ptr: !llvm.ptr, %mask: vector<7xi1>) { + // CHECK: llvm.intr.masked.load + // CHECK-SAME: access_groups = [#{{[^]]*}}] + // CHECK-SAME: alias_scopes = [#{{[^]]*}}] + // CHECK-SAME: noalias_scopes = [#{{[^]]*}}] + %0 = llvm.intr.masked.load %ptr, %mask { + alignment = 4 : i64, + access_groups = [#rt_access_group], + alias_scopes = [#rt_alias_scope], + noalias_scopes = [#rt_alias_scope]} : (!llvm.ptr, vector<7xi1>) -> vector<7xf32> + // CHECK: llvm.intr.masked.store + // CHECK-SAME: access_groups = [#{{[^]]*}}] + llvm.intr.masked.store %0, %ptr, %mask { + alignment = 4 : i64, + access_groups = [#rt_access_group]} : vector<7xf32>, vector<7xi1> into !llvm.ptr + llvm.return +} diff --git a/mlir/test/Target/LLVMIR/Import/intrinsic.ll b/mlir/test/Target/LLVMIR/Import/intrinsic.ll index aaebaa2f46d1c..206809c905054 100644 --- a/mlir/test/Target/LLVMIR/Import/intrinsic.ll +++ b/mlir/test/Target/LLVMIR/Import/intrinsic.ll @@ -515,19 +515,25 @@ define <7 x i1> @get_active_lane_mask(i64 %0, i64 %1) { ; CHECK-SAME: %[[MASK:[a-zA-Z0-9]+]] define void @masked_load_store_intrinsics(ptr %vec, <7 x i1> %mask) { ; CHECK: %[[UNDEF:.+]] = llvm.mlir.undef - ; CHECK: %[[VAL1:.+]] = llvm.intr.masked.load %[[VEC]], %[[MASK]], %[[UNDEF]] {alignment = 1 : i32} + ; CHECK: %[[VAL1:.+]] = llvm.intr.masked.load %[[VEC]], %[[MASK]], %[[UNDEF]] {alignment = 1 : i64} ; CHECK-SAME: (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> vector<7xf32> - %1 = call <7 x float> @llvm.masked.load.v7f32.p0(ptr %vec, i32 1, <7 x i1> %mask, <7 x float> undef) - ; CHECK: %[[VAL2:.+]] = llvm.intr.masked.load %[[VEC]], %[[MASK]], %[[VAL1]] {alignment = 4 : i32} - %2 = call <7 x float> @llvm.masked.load.v7f32.p0(ptr %vec, i32 4, <7 x i1> %mask, <7 x float> %1) - ; CHECK: %[[VAL3:.+]] = llvm.intr.masked.load %[[VEC]], %[[MASK]], %[[VAL1]] {alignment = 4 : i32, nontemporal} - %3 = call <7 x float> @llvm.masked.load.v7f32.p0(ptr %vec, i32 4, <7 x i1> %mask, <7 x float> %1), !nontemporal !{i32 1} - ; CHECK: llvm.intr.masked.store %[[VAL2]], %[[VEC]], %[[MASK]] {alignment = 8 : i32} + %1 = call <7 x float> @llvm.masked.load.v7f32.p0(ptr align 1 %vec, <7 x i1> %mask, <7 x float> undef) + ; CHECK: %[[VAL2:.+]] = llvm.intr.masked.load %[[VEC]], %[[MASK]], %[[VAL1]] {alignment = 4 : i64} + %2 = call <7 x float> @llvm.masked.load.v7f32.p0(ptr align 4 %vec, <7 x i1> %mask, <7 x float> %1) + ; CHECK: %[[VAL3:.+]] = llvm.intr.masked.load %[[VEC]], %[[MASK]], %[[VAL1]] {alignment = 4 : i64, nontemporal} + %3 = call <7 x float> @llvm.masked.load.v7f32.p0(ptr align 4 %vec, <7 x i1> %mask, <7 x float> %1), !nontemporal !{i32 1} + ; CHECK: %[[VAL4:.+]] = llvm.intr.masked.load %[[VEC]], %[[MASK]], %[[UNDEF]] : + ; CHECK-SAME: (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> vector<7xf32>{{$}} + %4 = call <7 x float> @llvm.masked.load.v7f32.p0(ptr %vec, <7 x i1> %mask, <7 x float> undef) + ; CHECK: llvm.intr.masked.store %[[VAL2]], %[[VEC]], %[[MASK]] {alignment = 8 : i64} ; CHECK-SAME: vector<7xf32>, vector<7xi1> into !llvm.ptr - call void @llvm.masked.store.v7f32.p0(<7 x float> %2, ptr %vec, i32 8, <7 x i1> %mask) - ; CHECK: llvm.intr.masked.store %[[VAL2]], %[[VEC]], %[[MASK]] {alignment = 8 : i32, nontemporal} - ; CHECK-SAME: vector<7xf32>, vector<7xi1> into !llvm.ptr - call void @llvm.masked.store.v7f32.p0(<7 x float> %2, ptr %vec, i32 8, <7 x i1> %mask), !nontemporal !{i32 1} + call void @llvm.masked.store.v7f32.p0(<7 x float> %2, ptr align 8 %vec, <7 x i1> %mask) + ; CHECK: llvm.intr.masked.store %[[VAL2]], %[[VEC]], %[[MASK]] {alignment = 8 : i64, nontemporal} + ; CHECK-SAME: vector<7xf32>, vector<7xi1> into !llvm.ptr + call void @llvm.masked.store.v7f32.p0(<7 x float> %2, ptr align 8 %vec, <7 x i1> %mask), !nontemporal !{i32 1} + ; CHECK: llvm.intr.masked.store %[[VAL4]], %[[VEC]], %[[MASK]] : + ; CHECK-SAME: vector<7xf32>, vector<7xi1> into !llvm.ptr{{$}} + call void @llvm.masked.store.v7f32.p0(<7 x float> %4, ptr %vec, <7 x i1> %mask) ret void } @@ -536,14 +542,20 @@ define void @masked_load_store_intrinsics(ptr %vec, <7 x i1> %mask) { ; CHECK-SAME: %[[MASK:[a-zA-Z0-9]+]] define void @masked_gather_scatter_intrinsics(<7 x ptr> %vec, <7 x i1> %mask) { ; CHECK: %[[UNDEF:.+]] = llvm.mlir.undef - ; CHECK: %[[VAL1:.+]] = llvm.intr.masked.gather %[[VEC]], %[[MASK]], %[[UNDEF]] {alignment = 1 : i32} + ; CHECK: %[[VAL1:.+]] = llvm.intr.masked.gather %[[VEC]], %[[MASK]], %[[UNDEF]] {alignment = 1 : i64} ; CHECK-SAME: (vector<7x!llvm.ptr>, vector<7xi1>, vector<7xf32>) -> vector<7xf32> - %1 = call <7 x float> @llvm.masked.gather.v7f32.v7p0(<7 x ptr> %vec, i32 1, <7 x i1> %mask, <7 x float> undef) - ; CHECK: %[[VAL2:.+]] = llvm.intr.masked.gather %[[VEC]], %[[MASK]], %[[VAL1]] {alignment = 4 : i32} - %2 = call <7 x float> @llvm.masked.gather.v7f32.v7p0(<7 x ptr> %vec, i32 4, <7 x i1> %mask, <7 x float> %1) - ; CHECK: llvm.intr.masked.scatter %[[VAL2]], %[[VEC]], %[[MASK]] {alignment = 8 : i32} + %1 = call <7 x float> @llvm.masked.gather.v7f32.v7p0(<7 x ptr> align 1 %vec, <7 x i1> %mask, <7 x float> undef) + ; CHECK: %[[VAL2:.+]] = llvm.intr.masked.gather %[[VEC]], %[[MASK]], %[[VAL1]] {alignment = 4 : i64} + %2 = call <7 x float> @llvm.masked.gather.v7f32.v7p0(<7 x ptr> align 4 %vec, <7 x i1> %mask, <7 x float> %1) + ; CHECK: %[[VAL3:.+]] = llvm.intr.masked.gather %[[VEC]], %[[MASK]], %[[UNDEF]] : + ; CHECK-SAME: (vector<7x!llvm.ptr>, vector<7xi1>, vector<7xf32>) -> vector<7xf32>{{$}} + %3 = call <7 x float> @llvm.masked.gather.v7f32.v7p0(<7 x ptr> %vec, <7 x i1> %mask, <7 x float> undef) + ; CHECK: llvm.intr.masked.scatter %[[VAL2]], %[[VEC]], %[[MASK]] {alignment = 8 : i64} ; CHECK-SAME: vector<7xf32>, vector<7xi1> into vector<7x!llvm.ptr> - call void @llvm.masked.scatter.v7f32.v7p0(<7 x float> %2, <7 x ptr> %vec, i32 8, <7 x i1> %mask) + call void @llvm.masked.scatter.v7f32.v7p0(<7 x float> %2, <7 x ptr> align 8 %vec, <7 x i1> %mask) + ; CHECK: llvm.intr.masked.scatter %[[VAL3]], %[[VEC]], %[[MASK]] : + ; CHECK-SAME: vector<7xf32>, vector<7xi1> into vector<7x!llvm.ptr>{{$}} + call void @llvm.masked.scatter.v7f32.v7p0(<7 x float> %3, <7 x ptr> %vec, <7 x i1> %mask) ret void } @@ -1812,10 +1824,10 @@ declare <48 x float> @llvm.matrix.transpose.v48f32(<48 x float>, i32 immarg, i32 declare <48 x float> @llvm.matrix.column.major.load.v48f32.i64(ptr nocapture, i64, i1 immarg, i32 immarg, i32 immarg) declare void @llvm.matrix.column.major.store.v48f32.i64(<48 x float>, ptr nocapture writeonly, i64, i1 immarg, i32 immarg, i32 immarg) declare <7 x i1> @llvm.get.active.lane.mask.v7i1.i64(i64, i64) -declare <7 x float> @llvm.masked.load.v7f32.p0(ptr, i32 immarg, <7 x i1>, <7 x float>) -declare void @llvm.masked.store.v7f32.p0(<7 x float>, ptr, i32 immarg, <7 x i1>) -declare <7 x float> @llvm.masked.gather.v7f32.v7p0(<7 x ptr>, i32 immarg, <7 x i1>, <7 x float>) -declare void @llvm.masked.scatter.v7f32.v7p0(<7 x float>, <7 x ptr>, i32 immarg, <7 x i1>) +declare <7 x float> @llvm.masked.load.v7f32.p0(ptr, <7 x i1>, <7 x float>) +declare void @llvm.masked.store.v7f32.p0(<7 x float>, ptr, <7 x i1>) +declare <7 x float> @llvm.masked.gather.v7f32.v7p0(<7 x ptr>, <7 x i1>, <7 x float>) +declare void @llvm.masked.scatter.v7f32.v7p0(<7 x float>, <7 x ptr>, <7 x i1>) declare <7 x float> @llvm.masked.expandload.v7f32(ptr, <7 x i1>, <7 x float>) declare void @llvm.masked.compressstore.v7f32(<7 x float>, ptr, <7 x i1>) declare void @llvm.var.annotation.p0.p0(ptr, ptr, ptr, i32, ptr) diff --git a/mlir/test/Target/LLVMIR/Import/masked-intrinsic-metadata.ll b/mlir/test/Target/LLVMIR/Import/masked-intrinsic-metadata.ll new file mode 100644 index 0000000000000..a94ca48c671df --- /dev/null +++ b/mlir/test/Target/LLVMIR/Import/masked-intrinsic-metadata.ll @@ -0,0 +1,64 @@ +; RUN: mlir-translate -import-llvm %s | FileCheck %s + +; CHECK-DAG: #[[$AG:.+]] = #llvm.access_group<id = {{.*}}> +; CHECK-DAG: #[[DOMAIN:.+]] = #llvm.alias_scope_domain<id = {{.*}}, description = "domain"> +; CHECK-DAG: #[[TBAA_ROOT:.+]] = #llvm.tbaa_root<id = "Simple C/C++ TBAA"> +; CHECK-DAG: #[[$SCOPE:.+]] = #llvm.alias_scope<id = {{.*}}, domain = #[[DOMAIN]], description = "scope"> +; CHECK-DAG: #[[$NOALIAS:.+]] = #llvm.alias_scope<id = {{.*}}, domain = #[[DOMAIN]], description = "noalias"> +; CHECK-DAG: #[[TBAA_CHAR:.+]] = #llvm.tbaa_type_desc<id = "omnipotent char", members = {<#[[TBAA_ROOT]], 0>}> +; CHECK-DAG: #[[TBAA_INT:.+]] = #llvm.tbaa_type_desc<id = "int", members = {<#[[TBAA_CHAR]], 0>}> +; CHECK-DAG: #[[$TBAA_TAG:.+]] = #llvm.tbaa_tag<base_type = #[[TBAA_INT]], access_type = #[[TBAA_INT]], offset = 0> + +; CHECK-LABEL: @masked_load_store_metadata +define void @masked_load_store_metadata(ptr %ptr, <7 x i1> %mask, <7 x float> %val) { + ; CHECK: llvm.intr.masked.load + ; CHECK-SAME: access_groups = [#[[$AG]]] + ; CHECK-SAME: alias_scopes = [#[[$SCOPE]]] + ; CHECK-SAME: alignment = 4 : i64 + ; CHECK-SAME: noalias_scopes = [#[[$NOALIAS]]] + ; CHECK-SAME: tbaa = [#[[$TBAA_TAG]]] + %1 = call <7 x float> @llvm.masked.load.v7f32.p0(ptr align 4 %ptr, <7 x i1> %mask, <7 x float> poison), !tbaa !0, !llvm.access.group !4, !alias.scope !5, !noalias !7 + ; CHECK: llvm.intr.masked.store + ; CHECK-SAME: access_groups = [#[[$AG]]] + ; CHECK-SAME: alias_scopes = [#[[$SCOPE]]] + ; CHECK-SAME: alignment = 4 : i64 + ; CHECK-SAME: noalias_scopes = [#[[$NOALIAS]]] + ; CHECK-SAME: tbaa = [#[[$TBAA_TAG]]] + call void @llvm.masked.store.v7f32.p0(<7 x float> %val, ptr align 4 %ptr, <7 x i1> %mask), !tbaa !0, !llvm.access.group !4, !alias.scope !5, !noalias !7 + ret void +} + +; CHECK-LABEL: @masked_gather_scatter_metadata +define void @masked_gather_scatter_metadata(<7 x ptr> %ptrs, <7 x i1> %mask, <7 x float> %val) { + ; CHECK: llvm.intr.masked.gather + ; CHECK-SAME: access_groups = [#[[$AG]]] + ; CHECK-SAME: alias_scopes = [#[[$SCOPE]]] + ; CHECK-SAME: alignment = 4 : i64 + ; CHECK-SAME: noalias_scopes = [#[[$NOALIAS]]] + ; CHECK-SAME: tbaa = [#[[$TBAA_TAG]]] + %1 = call <7 x float> @llvm.masked.gather.v7f32.v7p0(<7 x ptr> align 4 %ptrs, <7 x i1> %mask, <7 x float> poison), !tbaa !0, !llvm.access.group !4, !alias.scope !5, !noalias !7 + ; CHECK: llvm.intr.masked.scatter + ; CHECK-SAME: access_groups = [#[[$AG]]] + ; CHECK-SAME: alias_scopes = [#[[$SCOPE]]] + ; CHECK-SAME: alignment = 4 : i64 + ; CHECK-SAME: noalias_scopes = [#[[$NOALIAS]]] + ; CHECK-SAME: tbaa = [#[[$TBAA_TAG]]] + call void @llvm.masked.scatter.v7f32.v7p0(<7 x float> %val, <7 x ptr> align 4 %ptrs, <7 x i1> %mask), !tbaa !0, !llvm.access.group !4, !alias.scope !5, !noalias !7 + ret void +} + +declare <7 x float> @llvm.masked.load.v7f32.p0(ptr, <7 x i1>, <7 x float>) +declare void @llvm.masked.store.v7f32.p0(<7 x float>, ptr, <7 x i1>) +declare <7 x float> @llvm.masked.gather.v7f32.v7p0(<7 x ptr>, <7 x i1>, <7 x float>) +declare void @llvm.masked.scatter.v7f32.v7p0(<7 x float>, <7 x ptr>, <7 x i1>) + +!0 = !{!1, !1, i64 0} +!1 = !{!"int", !2, i64 0} +!2 = !{!"omnipotent char", !3, i64 0} +!3 = !{!"Simple C/C++ TBAA"} +!4 = distinct !{} +!5 = !{!6} +!6 = distinct !{!6, !9, !"scope"} +!7 = !{!8} +!8 = distinct !{!8, !9, !"noalias"} +!9 = distinct !{!9, !"domain"} diff --git a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir index b3776c9a99f97..b451dcc5d4e26 100644 --- a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir +++ b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir @@ -564,19 +564,25 @@ llvm.func @get_active_lane_mask(%base: i64, %n: i64) -> (vector<7xi1>) { // CHECK-LABEL: @masked_load_store_intrinsics llvm.func @masked_load_store_intrinsics(%A: !llvm.ptr, %mask: vector<7xi1>) { // CHECK: call <7 x float> @llvm.masked.load.v7f32.p0(ptr align 1 %{{.*}}, <7 x i1> %{{.*}}, <7 x float> poison) - %a = llvm.intr.masked.load %A, %mask { alignment = 1: i32} : + %a = llvm.intr.masked.load %A, %mask { alignment = 1: i64} : (!llvm.ptr, vector<7xi1>) -> vector<7xf32> // CHECK: call <7 x float> @llvm.masked.load.v7f32.p0(ptr align 1 %{{.*}}, <7 x i1> %{{.*}}, <7 x float> poison), !nontemporal !1 - %b = llvm.intr.masked.load %A, %mask { alignment = 1: i32, nontemporal} : + %b = llvm.intr.masked.load %A, %mask { alignment = 1: i64, nontemporal} : (!llvm.ptr, vector<7xi1>) -> vector<7xf32> // CHECK: call <7 x float> @llvm.masked.load.v7f32.p0(ptr align 1 %{{.*}}, <7 x i1> %{{.*}}, <7 x float> %{{.*}}) - %c = llvm.intr.masked.load %A, %mask, %a { alignment = 1: i32} : + %c = llvm.intr.masked.load %A, %mask, %a { alignment = 1: i64} : (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> vector<7xf32> + // CHECK: call <7 x float> @llvm.masked.load.v7f32.p0(ptr %{{.*}}, <7 x i1> %{{.*}}, <7 x float> poison){{$}} + %d = llvm.intr.masked.load %A, %mask : + (!llvm.ptr, vector<7xi1>) -> vector<7xf32> // CHECK: call void @llvm.masked.store.v7f32.p0(<7 x float> %{{.*}}, ptr align 1 %0, <7 x i1> %{{.*}}) - llvm.intr.masked.store %b, %A, %mask { alignment = 1: i32} : + llvm.intr.masked.store %b, %A, %mask { alignment = 1: i64} : vector<7xf32>, vector<7xi1> into !llvm.ptr // CHECK: call void @llvm.masked.store.v7f32.p0(<7 x float> %{{.*}}, ptr align 1 %0, <7 x i1> %{{.*}}), !nontemporal !{{.*}} - llvm.intr.masked.store %b, %A, %mask { alignment = 1: i32, nontemporal} : + llvm.intr.masked.store %b, %A, %mask { alignment = 1: i64, nontemporal} : + vector<7xf32>, vector<7xi1> into !llvm.ptr + // CHECK: call void @llvm.masked.store.v7f32.p0(<7 x float> %{{.*}}, ptr %0, <7 x i1> %{{.*}}){{$}} + llvm.intr.masked.store %d, %A, %mask : vector<7xf32>, vector<7xi1> into !llvm.ptr llvm.return } @@ -584,13 +590,19 @@ llvm.func @masked_load_store_intrinsics(%A: !llvm.ptr, %mask: vector<7xi1>) { // CHECK-LABEL: @masked_gather_scatter_intrinsics llvm.func @masked_gather_scatter_intrinsics(%M: vector<7 x !llvm.ptr>, %mask: vector<7xi1>) { // CHECK: call <7 x float> @llvm.masked.gather.v7f32.v7p0(<7 x ptr> align 1 %{{.*}}, <7 x i1> %{{.*}}, <7 x float> poison) - %a = llvm.intr.masked.gather %M, %mask { alignment = 1: i32} : + %a = llvm.intr.masked.gather %M, %mask { alignment = 1: i64} : (vector<7 x !llvm.ptr>, vector<7xi1>) -> vector<7xf32> // CHECK: call <7 x float> @llvm.masked.gather.v7f32.v7p0(<7 x ptr> align 1 %{{.*}}, <7 x i1> %{{.*}}, <7 x float> %{{.*}}) - %b = llvm.intr.masked.gather %M, %mask, %a { alignment = 1: i32} : + %b = llvm.intr.masked.gather %M, %mask, %a { alignment = 1: i64} : (vector<7 x !llvm.ptr>, vector<7xi1>, vector<7xf32>) -> vector<7xf32> + // CHECK: call <7 x float> @llvm.masked.gather.v7f32.v7p0(<7 x ptr> %{{.*}}, <7 x i1> %{{.*}}, <7 x float> poison){{$}} + %c = llvm.intr.masked.gather %M, %mask : + (vector<7 x !llvm.ptr>, vector<7xi1>) -> vector<7xf32> // CHECK: call void @llvm.masked.scatter.v7f32.v7p0(<7 x float> %{{.*}}, <7 x ptr> align 1 %{{.*}}, <7 x i1> %{{.*}}) - llvm.intr.masked.scatter %b, %M, %mask { alignment = 1: i32} : + llvm.intr.masked.scatter %b, %M, %mask { alignment = 1: i64} : + vector<7xf32>, vector<7xi1> into vector<7 x !llvm.ptr> + // CHECK: call void @llvm.masked.scatter.v7f32.v7p0(<7 x float> %{{.*}}, <7 x ptr> %{{.*}}, <7 x i1> %{{.*}}){{$}} + llvm.intr.masked.scatter %c, %M, %mask : vector<7xf32>, vector<7xi1> into vector<7 x !llvm.ptr> llvm.return } diff --git a/mlir/test/Target/LLVMIR/llvmir-invalid.mlir b/mlir/test/Target/LLVMIR/llvmir-invalid.mlir index ef6681ca3821f..6dd74420e21aa 100644 --- a/mlir/test/Target/LLVMIR/llvmir-invalid.mlir +++ b/mlir/test/Target/LLVMIR/llvmir-invalid.mlir @@ -198,7 +198,7 @@ llvm.func @active_lane_intr_wrong_type(%base : i64, %n : vector<7xi64>) -> vecto llvm.func @masked_load_intr_wrong_type(%ptr : i64, %mask : vector<7xi1>) -> vector<7xf32> { // expected-error @below{{op operand #0 must be LLVM pointer type, but got 'i64'}} - %0 = llvm.intr.masked.load %ptr, %mask { alignment = 1: i32} : (i64, vector<7xi1>) -> vector<7xf32> + %0 = llvm.intr.masked.load %ptr, %mask { alignment = 1: i64} : (i64, vector<7xi1>) -> vector<7xf32> llvm.return %0 : vector<7xf32> } @@ -206,7 +206,7 @@ llvm.func @masked_load_intr_wrong_type(%ptr : i64, %mask : vector<7xi1>) -> vect llvm.func @masked_store_intr_wrong_type(%vec : vector<7xf32>, %ptr : !llvm.ptr, %mask : vector<7xi32>) { // expected-error @below{{op operand #2 must be LLVM dialect-compatible vector of 1-bit signless integer, but got 'vector<7xi32>}} - llvm.intr.masked.store %vec, %ptr, %mask { alignment = 1: i32} : vector<7xf32>, vector<7xi32> into !llvm.ptr + llvm.intr.masked.store %vec, %ptr, %mask { alignment = 1: i64} : vector<7xf32>, vector<7xi32> into !llvm.ptr llvm.return } @@ -214,7 +214,7 @@ llvm.func @masked_store_intr_wrong_type(%vec : vector<7xf32>, %ptr : !llvm.ptr, llvm.func @masked_gather_intr_wrong_type(%ptrs : vector<7xf32>, %mask : vector<7xi1>) -> vector<7xf32> { // expected-error @below{{op operand #0 must be LLVM dialect-compatible vector of LLVM pointer type, but got 'vector<7xf32>'}} - %0 = llvm.intr.masked.gather %ptrs, %mask { alignment = 1: i32} : (vector<7xf32>, vector<7xi1>) -> vector<7xf32> + %0 = llvm.intr.masked.gather %ptrs, %mask { alignment = 1: i64} : (vector<7xf32>, vector<7xi1>) -> vector<7xf32> llvm.return %0 : vector<7xf32> } @@ -222,7 +222,7 @@ llvm.func @masked_gather_intr_wrong_type(%ptrs : vector<7xf32>, %mask : vector<7 llvm.func @masked_gather_intr_wrong_type_scalable(%ptrs : vector<7x!llvm.ptr>, %mask : vector<[7]xi1>) -> vector<[7]xf32> { // expected-error @below{{expected operand #1 type to be 'vector<[7]x!llvm.ptr>'}} - %0 = llvm.intr.masked.gather %ptrs, %mask { alignment = 1: i32} : (vector<7x!llvm.ptr>, vector<[7]xi1>) -> vector<[7]xf32> + %0 = llvm.intr.masked.gather %ptrs, %mask { alignment = 1: i64} : (vector<7x!llvm.ptr>, vector<[7]xi1>) -> vector<[7]xf32> llvm.return %0 : vector<[7]xf32> } @@ -230,7 +230,7 @@ llvm.func @masked_gather_intr_wrong_type_scalable(%ptrs : vector<7x!llvm.ptr>, % llvm.func @masked_scatter_intr_wrong_type(%vec : f32, %ptrs : vector<7x!llvm.ptr>, %mask : vector<7xi1>) { // expected-error @below{{invalid kind of type specified: expected builtin.vector, but found 'f32'}} - llvm.intr.masked.scatter %vec, %ptrs, %mask { alignment = 1: i32} : f32, vector<7xi1> into vector<7x!llvm.ptr> + llvm.intr.masked.scatter %vec, %ptrs, %mask { alignment = 1: i64} : f32, vector<7xi1> into vector<7x!llvm.ptr> llvm.return } @@ -238,7 +238,7 @@ llvm.func @masked_scatter_intr_wrong_type(%vec : f32, %ptrs : vector<7x!llvm.ptr llvm.func @masked_scatter_intr_wrong_type_scalable(%vec : vector<[7]xf32>, %ptrs : vector<7x!llvm.ptr>, %mask : vector<[7]xi1>) { // expected-error @below{{expected operand #2 type to be 'vector<[7]x!llvm.ptr>'}} - llvm.intr.masked.scatter %vec, %ptrs, %mask { alignment = 1: i32} : vector<[7]xf32>, vector<[7]xi1> into vector<7x!llvm.ptr> + llvm.intr.masked.scatter %vec, %ptrs, %mask { alignment = 1: i64} : vector<[7]xf32>, vector<[7]xi1> into vector<7x!llvm.ptr> llvm.return } diff --git a/mlir/test/Target/LLVMIR/masked-intrinsic-metadata.mlir b/mlir/test/Target/LLVMIR/masked-intrinsic-metadata.mlir new file mode 100644 index 0000000000000..354030afc5fdc --- /dev/null +++ b/mlir/test/Target/LLVMIR/masked-intrinsic-metadata.mlir @@ -0,0 +1,73 @@ +// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s + +#alias_scope_domain = #llvm.alias_scope_domain<id = distinct[0]<>, description = "The domain"> +#alias_scope1 = #llvm.alias_scope<id = distinct[1]<>, domain = #alias_scope_domain, description = "The first scope"> +#alias_scope2 = #llvm.alias_scope<id = distinct[2]<>, domain = #alias_scope_domain, description = "The second scope"> +#access_group = #llvm.access_group<id = distinct[3]<>> +#tbaa_root = #llvm.tbaa_root<id = "Simple C/C++ TBAA"> +#tbaa_type_desc = #llvm.tbaa_type_desc<id = "int", members = {<#tbaa_root, 0>}> +#tbaa_tag = #llvm.tbaa_tag<base_type = #tbaa_type_desc, access_type = #tbaa_type_desc, offset = 0> + +// CHECK-LABEL: @masked_load_store_metadata +llvm.func @masked_load_store_metadata(%ptr: !llvm.ptr, %mask: vector<7xi1>) { + // CHECK: call <7 x float> @llvm.masked.load.v7f32.p0 + // CHECK-SAME: !tbaa ![[$TBAA:[0-9]+]] + // CHECK-SAME: !alias.scope ![[$SCOPE1:[0-9]+]] + // CHECK-SAME: !noalias ![[$SCOPE2:[0-9]+]] + // CHECK-SAME: !llvm.access.group ![[$AG:[0-9]+]] + %0 = llvm.intr.masked.load %ptr, %mask { + alignment = 4 : i64, + access_groups = [#access_group], + alias_scopes = [#alias_scope1], + noalias_scopes = [#alias_scope2], + tbaa = [#tbaa_tag]} : (!llvm.ptr, vector<7xi1>) -> vector<7xf32> + // CHECK: call void @llvm.masked.store.v7f32.p0 + // CHECK-SAME: !tbaa ![[$TBAA]] + // CHECK-SAME: !alias.scope ![[$SCOPE1]] + // CHECK-SAME: !noalias ![[$SCOPE2]] + // CHECK-SAME: !llvm.access.group ![[$AG]] + llvm.intr.masked.store %0, %ptr, %mask { + alignment = 4 : i64, + access_groups = [#access_group], + alias_scopes = [#alias_scope1], + noalias_scopes = [#alias_scope2], + tbaa = [#tbaa_tag]} : vector<7xf32>, vector<7xi1> into !llvm.ptr + llvm.return +} + +// CHECK-LABEL: @masked_gather_scatter_metadata +llvm.func @masked_gather_scatter_metadata(%ptrs: vector<7 x !llvm.ptr>, %mask: vector<7xi1>) { + // CHECK: call <7 x float> @llvm.masked.gather.v7f32.v7p0 + // CHECK-SAME: !tbaa ![[$TBAA]] + // CHECK-SAME: !alias.scope ![[$SCOPE1]] + // CHECK-SAME: !noalias ![[$SCOPE2]] + // CHECK-SAME: !llvm.access.group ![[$AG]] + %0 = llvm.intr.masked.gather %ptrs, %mask { + alignment = 4 : i64, + access_groups = [#access_group], + alias_scopes = [#alias_scope1], + noalias_scopes = [#alias_scope2], + tbaa = [#tbaa_tag]} : (vector<7 x !llvm.ptr>, vector<7xi1>) -> vector<7xf32> + // CHECK: call void @llvm.masked.scatter.v7f32.v7p0 + // CHECK-SAME: !tbaa ![[$TBAA]] + // CHECK-SAME: !alias.scope ![[$SCOPE1]] + // CHECK-SAME: !noalias ![[$SCOPE2]] + // CHECK-SAME: !llvm.access.group ![[$AG]] + llvm.intr.masked.scatter %0, %ptrs, %mask { + alignment = 4 : i64, + access_groups = [#access_group], + alias_scopes = [#alias_scope1], + noalias_scopes = [#alias_scope2], + tbaa = [#tbaa_tag]} : vector<7xf32>, vector<7xi1> into vector<7 x !llvm.ptr> + llvm.return +} + +// CHECK-DAG: ![[$TBAA]] = !{![[TBAA_TYPE:[0-9]+]], ![[TBAA_TYPE]], i64 0} +// CHECK-DAG: ![[TBAA_TYPE]] = !{!"int", ![[TBAA_ROOT:[0-9]+]], i64 0} +// CHECK-DAG: ![[TBAA_ROOT]] = !{!"Simple C/C++ TBAA"} +// CHECK-DAG: ![[$SCOPE1]] = !{![[SCOPE1_DECL:[0-9]+]]} +// CHECK-DAG: ![[SCOPE1_DECL]] = distinct !{![[SCOPE1_DECL]], ![[DOMAIN:[0-9]+]], !"The first scope"} +// CHECK-DAG: ![[DOMAIN]] = distinct !{![[DOMAIN]], !"The domain"} +// CHECK-DAG: ![[$SCOPE2]] = !{![[SCOPE2_DECL:[0-9]+]]} +// CHECK-DAG: ![[SCOPE2_DECL]] = distinct !{![[SCOPE2_DECL]], ![[DOMAIN]], !"The second scope"} +// CHECK-DAG: ![[$AG]] = distinct !{} diff --git a/mlir/test/Target/LLVMIR/opaque-ptr.mlir b/mlir/test/Target/LLVMIR/opaque-ptr.mlir index cd7c31a0945a8..dee6f00f61c05 100644 --- a/mlir/test/Target/LLVMIR/opaque-ptr.mlir +++ b/mlir/test/Target/LLVMIR/opaque-ptr.mlir @@ -60,7 +60,7 @@ llvm.func @opaque_ptr_matrix_load_store(%ptr: !llvm.ptr, %stride: i64) -> vector // CHECK-LABEL: @opaque_ptr_masked_load llvm.func @opaque_ptr_masked_load(%arg0: !llvm.ptr, %arg1: vector<7xi1>) -> vector<7xf32> { // CHECK: call <7 x float> @llvm.masked.load.v7f32.p0(ptr - %0 = llvm.intr.masked.load %arg0, %arg1 { alignment = 1: i32} : + %0 = llvm.intr.masked.load %arg0, %arg1 { alignment = 1: i64} : (!llvm.ptr, vector<7xi1>) -> vector<7xf32> llvm.return %0 : vector<7 x f32> } @@ -68,7 +68,7 @@ llvm.func @opaque_ptr_masked_load(%arg0: !llvm.ptr, %arg1: vector<7xi1>) -> vect // CHECK-LABEL: @opaque_ptr_gather llvm.func @opaque_ptr_gather(%M: vector<7 x !llvm.ptr>, %mask: vector<7xi1>) -> vector<7xf32> { // CHECK: call <7 x float> @llvm.masked.gather.v7f32.v7p0(<7 x ptr> align 1 {{.*}}, - %a = llvm.intr.masked.gather %M, %mask { alignment = 1: i32} : + %a = llvm.intr.masked.gather %M, %mask { alignment = 1: i64} : (vector<7 x !llvm.ptr>, vector<7xi1>) -> vector<7xf32> llvm.return %a : vector<7xf32> } _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
