https://github.com/farzonl created https://github.com/llvm/llvm-project/pull/173201
fixes #172805 If we don't use vector type and instead continue to pass on the matrix type when we enter `EmitExtVectorElementExpr` Then we don't need to store the row and column length on the LValue. Using the Matrix type means we can reuse the isMatrixRow() cases in EmitLoadOfLValue and EmitStoreThroughLValue and not have to support a new lValue that is a hybrid between the ExtVectorElt and MatrixRow cases. All we need to do to support this is pass the list of colum indices as a ConstantDataVector and check the size of this Vector to know how many column iterations we need to do. Further just index into the vector to fetch the right encoded element index value. >From 8a8cff008c1a7d399f2982c4438c5da52d7e95c5 Mon Sep 17 00:00:00 2001 From: Farzon Lotfi <[email protected]> Date: Sun, 21 Dec 2025 15:51:46 -0500 Subject: [PATCH] [HLSL][Matrix] Implement dynamic single subscript swizzle fixes #172805 If we don't use vector type and instead continue to pass on the matrix type when we enter `EmitExtVectorElementExpr` Then we don't need to store the row and column length on the LValue. Using the Matrix type means we can reuse the isMatrixRow() cases in EmitLoadOfLValue and EmitStoreThroughLValue and not have to support a new lValue that is a hybrid between the ExtVectorElt and MatrixRow cases. All we need to do to support this is pas the list of colum indices as a ConstantDataVector and check the size of this Vector to know how many column iterations we need to do. Further just index into the vector to fetch the right encoded element index value. --- clang/lib/CodeGen/CGExpr.cpp | 55 ++++++-- clang/lib/CodeGen/CGValue.h | 34 +++-- .../MatrixSingleSubscriptDynamicSwizzle.hlsl | 122 +++++++++++++++++- 3 files changed, 185 insertions(+), 26 deletions(-) diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp index d784e1d6c68fe..a8c7bf471c269 100644 --- a/clang/lib/CodeGen/CGExpr.cpp +++ b/clang/lib/CodeGen/CGExpr.cpp @@ -2471,17 +2471,27 @@ RValue CodeGenFunction::EmitLoadOfLValue(LValue LV, SourceLocation Loc) { unsigned NumRows = MT->getNumRows(); unsigned NumCols = MT->getNumColumns(); - + unsigned NumLanes = NumCols; llvm::Value *MatrixVec = EmitLoadOfScalar(LV, Loc); llvm::Value *Row = LV.getMatrixRowIdx(); llvm::Type *ElemTy = ConvertType(MT->getElementType()); llvm::Type *RowTy = llvm::FixedVectorType::get(ElemTy, MT->getNumColumns()); llvm::Value *Result = llvm::PoisonValue::get(RowTy); // <NumCols x T> - + llvm::Constant *ColConstsIndices = nullptr; llvm::MatrixBuilder MB(Builder); - for (unsigned Col = 0; Col < NumCols; ++Col) { - llvm::Value *ColIdx = llvm::ConstantInt::get(Row->getType(), Col); + if (LV.isMatrixRowSwizzle()) { + ColConstsIndices = LV.getMatrixRowElts(); + NumLanes = llvm::cast<llvm::FixedVectorType>(ColConstsIndices->getType()) + ->getNumElements(); + } + + for (unsigned Col = 0; Col < NumLanes; ++Col) { + llvm::Value *ColIdx; + if (LV.isMatrixRowSwizzle()) + ColIdx = ColConstsIndices->getAggregateElement(Col); + else + ColIdx = llvm::ConstantInt::get(Row->getType(), Col); bool IsMatrixRowMajor = getLangOpts().getDefaultMatrixMemoryLayout() == LangOptions::MatrixMemoryLayout::MatrixRowMajor; llvm::Value *EltIndex = @@ -2726,6 +2736,7 @@ void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst, unsigned NumRows = MT->getNumRows(); unsigned NumCols = MT->getNumColumns(); + unsigned NumLanes = NumCols; llvm::Value *MatrixVec = Builder.CreateLoad(Dst.getAddress(), "matrix.load"); @@ -2734,8 +2745,20 @@ void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst, llvm::Value *RowVal = Src.getScalarVal(); // <NumCols x T> llvm::MatrixBuilder MB(Builder); - for (unsigned Col = 0; Col < NumCols; ++Col) { - llvm::Value *ColIdx = llvm::ConstantInt::get(Row->getType(), Col); + llvm::Constant *ColConstsIndices = nullptr; + if (Dst.isMatrixRowSwizzle()) { + ColConstsIndices = Dst.getMatrixRowElts(); + NumLanes = + llvm::cast<llvm::FixedVectorType>(ColConstsIndices->getType()) + ->getNumElements(); + } + + for (unsigned Col = 0; Col < NumLanes; ++Col) { + llvm::Value *ColIdx; + if (Dst.isMatrixRowSwizzle()) + ColIdx = ColConstsIndices->getAggregateElement(Col); + else + ColIdx = llvm::ConstantInt::get(Row->getType(), Col); bool IsMatrixRowMajor = getLangOpts().getDefaultMatrixMemoryLayout() == LangOptions::MatrixMemoryLayout::MatrixRowMajor; llvm::Value *EltIndex = @@ -4967,11 +4990,9 @@ LValue CodeGenFunction::EmitMatrixSingleSubscriptExpr( const MatrixSingleSubscriptExpr *E) { LValue Base = EmitLValue(E->getBase()); llvm::Value *RowIdx = EmitMatrixIndexExpr(E->getRowIdx()); - const auto *MatTy = E->getBase()->getType()->castAs<ConstantMatrixType>(); return LValue::MakeMatrixRow( MaybeConvertMatrixAddress(Base.getAddress(), *this), RowIdx, - MatTy->getNumColumns(), MatTy->getNumRows(), E->getBase()->getType(), - Base.getBaseInfo(), TBAAAccessInfo()); + E->getBase()->getType(), Base.getBaseInfo(), TBAAAccessInfo()); } LValue CodeGenFunction::EmitMatrixSubscriptExpr(const MatrixSubscriptExpr *E) { @@ -5253,8 +5274,10 @@ EmitExtVectorElementExpr(const ExtVectorElementExpr *E) { if (auto *RowIdx = llvm::dyn_cast<llvm::ConstantInt>(Base.getMatrixRowIdx())) { llvm::SmallVector<llvm::Constant *> MatIndices; - unsigned NumCols = Base.getMatrixNumCols(); - unsigned NumRows = Base.getMatrixNumRows(); + QualType MatTy = Base.getType(); + const ConstantMatrixType *MT = MatTy->castAs<ConstantMatrixType>(); + unsigned NumCols = MT->getNumColumns(); + unsigned NumRows = MT->getNumRows(); MatIndices.reserve(NumCols); unsigned Row = RowIdx->getZExtValue(); @@ -5269,7 +5292,15 @@ EmitExtVectorElementExpr(const ExtVectorElementExpr *E) { E->getBase()->getType(), Base.getBaseInfo(), TBAAAccessInfo()); } - return EmitUnsupportedLValue(E, "Matrix single index swizzle"); + llvm::Constant *Cols = + llvm::ConstantDataVector::get(getLLVMContext(), Indices); + // Note: inntentionally not using E.getType() so we can reuse isMatrixRow() + // implementations in EmitLoadOfLValue & EmitStoreThroughLValue and don't + // need the LValue to have its own number of rows and columns when the + // type is a vector. + return LValue::MakeMatrixRowSwizzle( + Base.getMatrixAddress(), Base.getMatrixRowIdx(), Cols, Base.getType(), + Base.getBaseInfo(), TBAAAccessInfo()); } assert(Base.isExtVectorElt() && "Can only subscript lvalue vec elts here!"); diff --git a/clang/lib/CodeGen/CGValue.h b/clang/lib/CodeGen/CGValue.h index c3ae130014192..f92c8eed04972 100644 --- a/clang/lib/CodeGen/CGValue.h +++ b/clang/lib/CodeGen/CGValue.h @@ -211,7 +211,7 @@ class LValue { }; // Note: Only meaningful when isMatrixRow() and the row is swizzled. - unsigned NumCols, NumRows; + llvm::Constant *MatrixRowElts = nullptr; QualType Type; @@ -290,6 +290,9 @@ class LValue { bool isGlobalReg() const { return LVType == GlobalReg; } bool isMatrixElt() const { return LVType == MatrixElt; } bool isMatrixRow() const { return LVType == MatrixRow; } + bool isMatrixRowSwizzle() const { + return isMatrixRow() && MatrixRowElts != nullptr; + } bool isVolatileQualified() const { return Quals.hasVolatile(); } bool isRestrictQualified() const { return Quals.hasRestrict(); } @@ -411,14 +414,9 @@ class LValue { return MatrixRowIdx; } - unsigned getMatrixNumRows() const { - assert(isMatrixRow()); - return NumRows; - } - - unsigned getMatrixNumCols() const { - assert(isMatrixRow()); - return NumCols; + llvm::Constant *getMatrixRowElts() const { + assert(isMatrixRowSwizzle() && "not a matrix row swizzle lvalue"); + return MatrixRowElts; } // extended vector elements. @@ -510,18 +508,30 @@ class LValue { } static LValue MakeMatrixRow(Address Addr, llvm::Value *RowIdx, - unsigned NumCols, unsigned NumRows, QualType MatrixTy, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo) { LValue LV; LV.LVType = MatrixRow; LV.MatrixRowIdx = RowIdx; // store the row index here - LV.NumCols = NumCols; - LV.NumRows = NumRows; + LV.MatrixRowElts = nullptr; LV.Initialize(MatrixTy, MatrixTy.getQualifiers(), Addr, BaseInfo, TBAAInfo); return LV; } + static LValue MakeMatrixRowSwizzle(Address MatAddr, llvm::Value *RowIdx, + llvm::Constant *Cols, QualType MatrixTy, + LValueBaseInfo BaseInfo, + TBAAAccessInfo TBAAInfo) { + LValue LV; + LV.LVType = MatrixRow; + LV.Addr = MatAddr; + LV.MatrixRowIdx = RowIdx; + LV.MatrixRowElts = Cols; // swizzle columns + LV.Initialize(MatrixTy, MatrixTy.getQualifiers(), MatAddr, BaseInfo, + TBAAInfo); + return LV; + } + static LValue MakeMatrixElt(Address matAddress, llvm::Value *Idx, QualType type, LValueBaseInfo BaseInfo, TBAAAccessInfo TBAAInfo) { diff --git a/clang/test/CodeGenHLSL/BasicFeatures/MatrixSingleSubscriptDynamicSwizzle.hlsl b/clang/test/CodeGenHLSL/BasicFeatures/MatrixSingleSubscriptDynamicSwizzle.hlsl index 82b7007fcdb3e..cdee71627b907 100644 --- a/clang/test/CodeGenHLSL/BasicFeatures/MatrixSingleSubscriptDynamicSwizzle.hlsl +++ b/clang/test/CodeGenHLSL/BasicFeatures/MatrixSingleSubscriptDynamicSwizzle.hlsl @@ -1,11 +1,129 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 6 // RUN: %clang_cc1 -triple dxil-pc-shadermodel6.7-library -disable-llvm-passes -emit-llvm -finclude-default-header -o - %s | FileCheck %s -// BUG: https://github.com/llvm/llvm-project/issues/170777 -// XFAIL: * +// CHECK-LABEL: define hidden void @_Z9setMatrixRu11matrix_typeILm4ELm4EfEiDv4_f( +// CHECK-SAME: ptr noalias noundef nonnull align 4 dereferenceable(64) [[M:%.*]], i32 noundef [[INDEX:%.*]], <4 x float> noundef nofpclass(nan inf) [[V:%.*]]) #[[ATTR0:[0-9]+]] { +// CHECK-NEXT: [[ENTRY:.*:]] +// CHECK-NEXT: [[M_ADDR:%.*]] = alloca ptr, align 4 +// CHECK-NEXT: [[INDEX_ADDR:%.*]] = alloca i32, align 4 +// CHECK-NEXT: [[V_ADDR:%.*]] = alloca <4 x float>, align 16 +// CHECK-NEXT: store ptr [[M]], ptr [[M_ADDR]], align 4 +// CHECK-NEXT: store i32 [[INDEX]], ptr [[INDEX_ADDR]], align 4 +// CHECK-NEXT: store <4 x float> [[V]], ptr [[V_ADDR]], align 16 +// CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr [[V_ADDR]], align 16 +// CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[M_ADDR]], align 4, !nonnull [[META3:![0-9]+]], !align [[META4:![0-9]+]] +// CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[INDEX_ADDR]], align 4 +// CHECK-NEXT: [[MATRIX_LOAD:%.*]] = load <16 x float>, ptr [[TMP1]], align 4 +// CHECK-NEXT: [[TMP3:%.*]] = add i32 12, [[TMP2]] +// CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP0]], i32 0 +// CHECK-NEXT: [[TMP5:%.*]] = insertelement <16 x float> [[MATRIX_LOAD]], float [[TMP4]], i32 [[TMP3]] +// CHECK-NEXT: [[TMP6:%.*]] = add i32 8, [[TMP2]] +// CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x float> [[TMP0]], i32 1 +// CHECK-NEXT: [[TMP8:%.*]] = insertelement <16 x float> [[TMP5]], float [[TMP7]], i32 [[TMP6]] +// CHECK-NEXT: [[TMP9:%.*]] = add i32 4, [[TMP2]] +// CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x float> [[TMP0]], i32 2 +// CHECK-NEXT: [[TMP11:%.*]] = insertelement <16 x float> [[TMP8]], float [[TMP10]], i32 [[TMP9]] +// CHECK-NEXT: [[TMP12:%.*]] = add i32 0, [[TMP2]] +// CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x float> [[TMP0]], i32 3 +// CHECK-NEXT: [[TMP14:%.*]] = insertelement <16 x float> [[TMP11]], float [[TMP13]], i32 [[TMP12]] +// CHECK-NEXT: store <16 x float> [[TMP14]], ptr [[TMP1]], align 4 +// CHECK-NEXT: ret void +// void setMatrix(out float4x4 M, int index, float4 V) { M[index].abgr = V; } +// CHECK-LABEL: define hidden noundef nofpclass(nan inf) <3 x float> @_Z9getMatrixu11matrix_typeILm4ELm4EfEi( +// CHECK-SAME: <16 x float> noundef nofpclass(nan inf) [[M:%.*]], i32 noundef [[INDEX:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: [[ENTRY:.*:]] +// CHECK-NEXT: [[RETVAL:%.*]] = alloca <3 x float>, align 16 +// CHECK-NEXT: [[M_ADDR:%.*]] = alloca [16 x float], align 4 +// CHECK-NEXT: [[INDEX_ADDR:%.*]] = alloca i32, align 4 +// CHECK-NEXT: store <16 x float> [[M]], ptr [[M_ADDR]], align 4 +// CHECK-NEXT: store i32 [[INDEX]], ptr [[INDEX_ADDR]], align 4 +// CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[INDEX_ADDR]], align 4 +// CHECK-NEXT: [[TMP1:%.*]] = load <16 x float>, ptr [[M_ADDR]], align 4 +// CHECK-NEXT: [[TMP2:%.*]] = add i32 0, [[TMP0]] +// CHECK-NEXT: [[TMP3:%.*]] = extractelement <16 x float> [[TMP1]], i32 [[TMP2]] +// CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x float> poison, float [[TMP3]], i32 0 +// CHECK-NEXT: [[TMP5:%.*]] = add i32 4, [[TMP0]] +// CHECK-NEXT: [[TMP6:%.*]] = extractelement <16 x float> [[TMP1]], i32 [[TMP5]] +// CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x float> [[TMP4]], float [[TMP6]], i32 1 +// CHECK-NEXT: [[TMP8:%.*]] = add i32 8, [[TMP0]] +// CHECK-NEXT: [[TMP9:%.*]] = extractelement <16 x float> [[TMP1]], i32 [[TMP8]] +// CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x float> [[TMP7]], float [[TMP9]], i32 2 +// CHECK-NEXT: store <4 x float> [[TMP10]], ptr [[RETVAL]], align 16 +// CHECK-NEXT: [[TMP11:%.*]] = load <3 x float>, ptr [[RETVAL]], align 16 +// CHECK-NEXT: ret <3 x float> [[TMP11]] +// float3 getMatrix(float4x4 M, int index) { return M[index].rgb; } + +// CHECK-LABEL: define hidden noundef <3 x i32> @_Z19getMatrixSwizzle2x3Ru11matrix_typeILm2ELm3EiEi( +// CHECK-SAME: ptr noalias noundef nonnull align 4 dereferenceable(24) [[M:%.*]], i32 noundef [[INDEX:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: [[ENTRY:.*:]] +// CHECK-NEXT: [[M_ADDR:%.*]] = alloca ptr, align 4 +// CHECK-NEXT: [[INDEX_ADDR:%.*]] = alloca i32, align 4 +// CHECK-NEXT: store ptr [[M]], ptr [[M_ADDR]], align 4 +// CHECK-NEXT: store i32 [[INDEX]], ptr [[INDEX_ADDR]], align 4 +// CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[M_ADDR]], align 4, !nonnull [[META3]], !align [[META4]] +// CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[INDEX_ADDR]], align 4 +// CHECK-NEXT: [[TMP2:%.*]] = load <6 x i32>, ptr [[TMP0]], align 4 +// CHECK-NEXT: [[TMP3:%.*]] = add i32 4, [[TMP1]] +// CHECK-NEXT: [[TMP4:%.*]] = extractelement <6 x i32> [[TMP2]], i32 [[TMP3]] +// CHECK-NEXT: [[TMP5:%.*]] = insertelement <3 x i32> poison, i32 [[TMP4]], i32 0 +// CHECK-NEXT: [[TMP6:%.*]] = add i32 0, [[TMP1]] +// CHECK-NEXT: [[TMP7:%.*]] = extractelement <6 x i32> [[TMP2]], i32 [[TMP6]] +// CHECK-NEXT: [[TMP8:%.*]] = insertelement <3 x i32> [[TMP5]], i32 [[TMP7]], i32 1 +// CHECK-NEXT: [[TMP9:%.*]] = add i32 2, [[TMP1]] +// CHECK-NEXT: [[TMP10:%.*]] = extractelement <6 x i32> [[TMP2]], i32 [[TMP9]] +// CHECK-NEXT: [[TMP11:%.*]] = insertelement <3 x i32> [[TMP8]], i32 [[TMP10]], i32 2 +// CHECK-NEXT: ret <3 x i32> [[TMP11]] +// +int3 getMatrixSwizzle2x3(out int2x3 M, int index) { + return M[index].brg; +} + +// CHECK-LABEL: define hidden void @_Z26setMatrixSwizzleFromMatrixRu11matrix_typeILm2ELm3EiES_i( +// CHECK-SAME: ptr noalias noundef nonnull align 4 dereferenceable(24) [[M:%.*]], <6 x i32> noundef [[N:%.*]], i32 noundef [[INDEX:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: [[ENTRY:.*:]] +// CHECK-NEXT: [[M_ADDR:%.*]] = alloca ptr, align 4 +// CHECK-NEXT: [[N_ADDR:%.*]] = alloca [6 x i32], align 4 +// CHECK-NEXT: [[INDEX_ADDR:%.*]] = alloca i32, align 4 +// CHECK-NEXT: store ptr [[M]], ptr [[M_ADDR]], align 4 +// CHECK-NEXT: store <6 x i32> [[N]], ptr [[N_ADDR]], align 4 +// CHECK-NEXT: store i32 [[INDEX]], ptr [[INDEX_ADDR]], align 4 +// CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[INDEX_ADDR]], align 4 +// CHECK-NEXT: [[TMP1:%.*]] = load <6 x i32>, ptr [[N_ADDR]], align 4 +// CHECK-NEXT: [[TMP2:%.*]] = add i32 0, [[TMP0]] +// CHECK-NEXT: [[MATRIX_ELEM:%.*]] = extractelement <6 x i32> [[TMP1]], i32 [[TMP2]] +// CHECK-NEXT: [[MATRIX_ROW_INS:%.*]] = insertelement <3 x i32> poison, i32 [[MATRIX_ELEM]], i32 0 +// CHECK-NEXT: [[TMP3:%.*]] = add i32 2, [[TMP0]] +// CHECK-NEXT: [[MATRIX_ELEM1:%.*]] = extractelement <6 x i32> [[TMP1]], i32 [[TMP3]] +// CHECK-NEXT: [[MATRIX_ROW_INS2:%.*]] = insertelement <3 x i32> [[MATRIX_ROW_INS]], i32 [[MATRIX_ELEM1]], i32 1 +// CHECK-NEXT: [[TMP4:%.*]] = add i32 4, [[TMP0]] +// CHECK-NEXT: [[MATRIX_ELEM3:%.*]] = extractelement <6 x i32> [[TMP1]], i32 [[TMP4]] +// CHECK-NEXT: [[MATRIX_ROW_INS4:%.*]] = insertelement <3 x i32> [[MATRIX_ROW_INS2]], i32 [[MATRIX_ELEM3]], i32 2 +// CHECK-NEXT: [[TMP5:%.*]] = load ptr, ptr [[M_ADDR]], align 4, !nonnull [[META3]], !align [[META4]] +// CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[INDEX_ADDR]], align 4 +// CHECK-NEXT: [[MATRIX_LOAD:%.*]] = load <6 x i32>, ptr [[TMP5]], align 4 +// CHECK-NEXT: [[TMP7:%.*]] = add i32 4, [[TMP6]] +// CHECK-NEXT: [[TMP8:%.*]] = extractelement <3 x i32> [[MATRIX_ROW_INS4]], i32 0 +// CHECK-NEXT: [[TMP9:%.*]] = insertelement <6 x i32> [[MATRIX_LOAD]], i32 [[TMP8]], i32 [[TMP7]] +// CHECK-NEXT: [[TMP10:%.*]] = add i32 0, [[TMP6]] +// CHECK-NEXT: [[TMP11:%.*]] = extractelement <3 x i32> [[MATRIX_ROW_INS4]], i32 1 +// CHECK-NEXT: [[TMP12:%.*]] = insertelement <6 x i32> [[TMP9]], i32 [[TMP11]], i32 [[TMP10]] +// CHECK-NEXT: [[TMP13:%.*]] = add i32 2, [[TMP6]] +// CHECK-NEXT: [[TMP14:%.*]] = extractelement <3 x i32> [[MATRIX_ROW_INS4]], i32 2 +// CHECK-NEXT: [[TMP15:%.*]] = insertelement <6 x i32> [[TMP12]], i32 [[TMP14]], i32 [[TMP13]] +// CHECK-NEXT: store <6 x i32> [[TMP15]], ptr [[TMP5]], align 4 +// CHECK-NEXT: ret void +// +void setMatrixSwizzleFromMatrix(out int2x3 M, int2x3 N, int index) { + M[index].brg = N[index]; +} +//. +// CHECK: [[META3]] = !{} +// CHECK: [[META4]] = !{i64 4} +//. _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
