[llvm-branch-commits] [llvm] AMDGPU: Handle folding vector splats of inline split f64 inline immediates (PR #140878)
https://github.com/arsenm updated
https://github.com/llvm/llvm-project/pull/140878
>From da0e7f3709e013348cde68701e8739c2912a4e4f Mon Sep 17 00:00:00 2001
From: Matt Arsenault
Date: Mon, 19 May 2025 21:51:06 +0200
Subject: [PATCH] AMDGPU: Handle folding vector splats of inline split f64
inline immediates
Recognize a reg_sequence with 32-bit elements that produce a 64-bit
splat value. This enables folding f64 constants into mfma operands
---
llvm/lib/Target/AMDGPU/SIFoldOperands.cpp | 103 --
.../CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll | 41 +--
2 files changed, 76 insertions(+), 68 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index 8d7b68dbd5682..0ed06c37507af 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -227,12 +227,12 @@ class SIFoldOperandsImpl {
getRegSeqInit(SmallVectorImpl> &Defs,
Register UseReg) const;
- std::pair
+ std::pair
isRegSeqSplat(MachineInstr &RegSeg) const;
- MachineOperand *tryFoldRegSeqSplat(MachineInstr *UseMI, unsigned UseOpIdx,
- MachineOperand *SplatVal,
- const TargetRegisterClass *SplatRC) const;
+ bool tryFoldRegSeqSplat(MachineInstr *UseMI, unsigned UseOpIdx,
+ int64_t SplatVal,
+ const TargetRegisterClass *SplatRC) const;
bool tryToFoldACImm(const FoldableDef &OpToFold, MachineInstr *UseMI,
unsigned UseOpIdx,
@@ -966,15 +966,15 @@ const TargetRegisterClass
*SIFoldOperandsImpl::getRegSeqInit(
return getRegSeqInit(*Def, Defs);
}
-std::pair
+std::pair
SIFoldOperandsImpl::isRegSeqSplat(MachineInstr &RegSeq) const {
SmallVector, 32> Defs;
const TargetRegisterClass *SrcRC = getRegSeqInit(RegSeq, Defs);
if (!SrcRC)
return {};
- // TODO: Recognize 64-bit splats broken into 32-bit pieces (i.e. recognize
- // every other other element is 0 for 64-bit immediates)
+ bool TryToMatchSplat64 = false;
+
int64_t Imm;
for (unsigned I = 0, E = Defs.size(); I != E; ++I) {
const MachineOperand *Op = Defs[I].first;
@@ -986,38 +986,75 @@ SIFoldOperandsImpl::isRegSeqSplat(MachineInstr &RegSeq)
const {
Imm = SubImm;
continue;
}
-if (Imm != SubImm)
+
+if (Imm != SubImm) {
+ if (I == 1 && (E & 1) == 0) {
+// If we have an even number of inputs, there's a chance this is a
+// 64-bit element splat broken into 32-bit pieces.
+TryToMatchSplat64 = true;
+break;
+ }
+
return {}; // Can only fold splat constants
+}
+ }
+
+ if (!TryToMatchSplat64)
+return {Defs[0].first->getImm(), SrcRC};
+
+ // Fallback to recognizing 64-bit splats broken into 32-bit pieces
+ // (i.e. recognize every other other element is 0 for 64-bit immediates)
+ int64_t SplatVal64;
+ for (unsigned I = 0, E = Defs.size(); I != E; I += 2) {
+const MachineOperand *Op0 = Defs[I].first;
+const MachineOperand *Op1 = Defs[I + 1].first;
+
+if (!Op0->isImm() || !Op1->isImm())
+ return {};
+
+unsigned SubReg0 = Defs[I].second;
+unsigned SubReg1 = Defs[I + 1].second;
+
+// Assume we're going to generally encounter reg_sequences with sorted
+// subreg indexes, so reject any that aren't consecutive.
+if (TRI->getChannelFromSubReg(SubReg0) + 1 !=
+TRI->getChannelFromSubReg(SubReg1))
+ return {};
+
+int64_t MergedVal = Make_64(Op1->getImm(), Op0->getImm());
+if (I == 0)
+ SplatVal64 = MergedVal;
+else if (SplatVal64 != MergedVal)
+ return {};
}
- return {Defs[0].first, SrcRC};
+ const TargetRegisterClass *RC64 = TRI->getSubRegisterClass(
+ MRI->getRegClass(RegSeq.getOperand(0).getReg()), AMDGPU::sub0_sub1);
+
+ return {SplatVal64, RC64};
}
-MachineOperand *SIFoldOperandsImpl::tryFoldRegSeqSplat(
-MachineInstr *UseMI, unsigned UseOpIdx, MachineOperand *SplatVal,
+bool SIFoldOperandsImpl::tryFoldRegSeqSplat(
+MachineInstr *UseMI, unsigned UseOpIdx, int64_t SplatVal,
const TargetRegisterClass *SplatRC) const {
const MCInstrDesc &Desc = UseMI->getDesc();
if (UseOpIdx >= Desc.getNumOperands())
-return nullptr;
+return false;
// Filter out unhandled pseudos.
if (!AMDGPU::isSISrcOperand(Desc, UseOpIdx))
-return nullptr;
+return false;
int16_t RCID = Desc.operands()[UseOpIdx].RegClass;
if (RCID == -1)
-return nullptr;
+return false;
+
+ const TargetRegisterClass *OpRC = TRI->getRegClass(RCID);
// Special case 0/-1, since when interpreted as a 64-bit element both halves
- // have the same bits. Effectively this code does not handle 64-bit element
- // operands correctly, as the incoming 64-bit constants are already split
into
- // 32-bit sequence elements.
- //
- // TODO: We should try to figure out how to interpret the reg_sequence as a
- /
[llvm-branch-commits] [llvm] AMDGPU: Handle folding vector splats of inline split f64 inline immediates (PR #140878)
https://github.com/arsenm updated
https://github.com/llvm/llvm-project/pull/140878
>From da0e7f3709e013348cde68701e8739c2912a4e4f Mon Sep 17 00:00:00 2001
From: Matt Arsenault
Date: Mon, 19 May 2025 21:51:06 +0200
Subject: [PATCH] AMDGPU: Handle folding vector splats of inline split f64
inline immediates
Recognize a reg_sequence with 32-bit elements that produce a 64-bit
splat value. This enables folding f64 constants into mfma operands
---
llvm/lib/Target/AMDGPU/SIFoldOperands.cpp | 103 --
.../CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll | 41 +--
2 files changed, 76 insertions(+), 68 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index 8d7b68dbd5682..0ed06c37507af 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -227,12 +227,12 @@ class SIFoldOperandsImpl {
getRegSeqInit(SmallVectorImpl> &Defs,
Register UseReg) const;
- std::pair
+ std::pair
isRegSeqSplat(MachineInstr &RegSeg) const;
- MachineOperand *tryFoldRegSeqSplat(MachineInstr *UseMI, unsigned UseOpIdx,
- MachineOperand *SplatVal,
- const TargetRegisterClass *SplatRC) const;
+ bool tryFoldRegSeqSplat(MachineInstr *UseMI, unsigned UseOpIdx,
+ int64_t SplatVal,
+ const TargetRegisterClass *SplatRC) const;
bool tryToFoldACImm(const FoldableDef &OpToFold, MachineInstr *UseMI,
unsigned UseOpIdx,
@@ -966,15 +966,15 @@ const TargetRegisterClass
*SIFoldOperandsImpl::getRegSeqInit(
return getRegSeqInit(*Def, Defs);
}
-std::pair
+std::pair
SIFoldOperandsImpl::isRegSeqSplat(MachineInstr &RegSeq) const {
SmallVector, 32> Defs;
const TargetRegisterClass *SrcRC = getRegSeqInit(RegSeq, Defs);
if (!SrcRC)
return {};
- // TODO: Recognize 64-bit splats broken into 32-bit pieces (i.e. recognize
- // every other other element is 0 for 64-bit immediates)
+ bool TryToMatchSplat64 = false;
+
int64_t Imm;
for (unsigned I = 0, E = Defs.size(); I != E; ++I) {
const MachineOperand *Op = Defs[I].first;
@@ -986,38 +986,75 @@ SIFoldOperandsImpl::isRegSeqSplat(MachineInstr &RegSeq)
const {
Imm = SubImm;
continue;
}
-if (Imm != SubImm)
+
+if (Imm != SubImm) {
+ if (I == 1 && (E & 1) == 0) {
+// If we have an even number of inputs, there's a chance this is a
+// 64-bit element splat broken into 32-bit pieces.
+TryToMatchSplat64 = true;
+break;
+ }
+
return {}; // Can only fold splat constants
+}
+ }
+
+ if (!TryToMatchSplat64)
+return {Defs[0].first->getImm(), SrcRC};
+
+ // Fallback to recognizing 64-bit splats broken into 32-bit pieces
+ // (i.e. recognize every other other element is 0 for 64-bit immediates)
+ int64_t SplatVal64;
+ for (unsigned I = 0, E = Defs.size(); I != E; I += 2) {
+const MachineOperand *Op0 = Defs[I].first;
+const MachineOperand *Op1 = Defs[I + 1].first;
+
+if (!Op0->isImm() || !Op1->isImm())
+ return {};
+
+unsigned SubReg0 = Defs[I].second;
+unsigned SubReg1 = Defs[I + 1].second;
+
+// Assume we're going to generally encounter reg_sequences with sorted
+// subreg indexes, so reject any that aren't consecutive.
+if (TRI->getChannelFromSubReg(SubReg0) + 1 !=
+TRI->getChannelFromSubReg(SubReg1))
+ return {};
+
+int64_t MergedVal = Make_64(Op1->getImm(), Op0->getImm());
+if (I == 0)
+ SplatVal64 = MergedVal;
+else if (SplatVal64 != MergedVal)
+ return {};
}
- return {Defs[0].first, SrcRC};
+ const TargetRegisterClass *RC64 = TRI->getSubRegisterClass(
+ MRI->getRegClass(RegSeq.getOperand(0).getReg()), AMDGPU::sub0_sub1);
+
+ return {SplatVal64, RC64};
}
-MachineOperand *SIFoldOperandsImpl::tryFoldRegSeqSplat(
-MachineInstr *UseMI, unsigned UseOpIdx, MachineOperand *SplatVal,
+bool SIFoldOperandsImpl::tryFoldRegSeqSplat(
+MachineInstr *UseMI, unsigned UseOpIdx, int64_t SplatVal,
const TargetRegisterClass *SplatRC) const {
const MCInstrDesc &Desc = UseMI->getDesc();
if (UseOpIdx >= Desc.getNumOperands())
-return nullptr;
+return false;
// Filter out unhandled pseudos.
if (!AMDGPU::isSISrcOperand(Desc, UseOpIdx))
-return nullptr;
+return false;
int16_t RCID = Desc.operands()[UseOpIdx].RegClass;
if (RCID == -1)
-return nullptr;
+return false;
+
+ const TargetRegisterClass *OpRC = TRI->getRegClass(RCID);
// Special case 0/-1, since when interpreted as a 64-bit element both halves
- // have the same bits. Effectively this code does not handle 64-bit element
- // operands correctly, as the incoming 64-bit constants are already split
into
- // 32-bit sequence elements.
- //
- // TODO: We should try to figure out how to interpret the reg_sequence as a
- /
[llvm-branch-commits] [llvm] AMDGPU: Handle folding vector splats of inline split f64 inline immediates (PR #140878)
https://github.com/kosarev approved this pull request. https://github.com/llvm/llvm-project/pull/140878 ___ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
[llvm-branch-commits] [llvm] AMDGPU: Handle folding vector splats of inline split f64 inline immediates (PR #140878)
https://github.com/arsenm updated
https://github.com/llvm/llvm-project/pull/140878
>From 5d5a54cff58c1096973d3a9c28f728ca0afe3889 Mon Sep 17 00:00:00 2001
From: Matt Arsenault
Date: Mon, 19 May 2025 21:51:06 +0200
Subject: [PATCH] AMDGPU: Handle folding vector splats of inline split f64
inline immediates
Recognize a reg_sequence with 32-bit elements that produce a 64-bit
splat value. This enables folding f64 constants into mfma operands
---
llvm/lib/Target/AMDGPU/SIFoldOperands.cpp | 103 --
.../CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll | 41 +--
2 files changed, 76 insertions(+), 68 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index 565abad9b0366..8121098a97bd1 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -227,12 +227,12 @@ class SIFoldOperandsImpl {
getRegSeqInit(SmallVectorImpl> &Defs,
Register UseReg) const;
- std::pair
+ std::pair
isRegSeqSplat(MachineInstr &RegSeg) const;
- MachineOperand *tryFoldRegSeqSplat(MachineInstr *UseMI, unsigned UseOpIdx,
- MachineOperand *SplatVal,
- const TargetRegisterClass *SplatRC) const;
+ bool tryFoldRegSeqSplat(MachineInstr *UseMI, unsigned UseOpIdx,
+ int64_t SplatVal,
+ const TargetRegisterClass *SplatRC) const;
bool tryToFoldACImm(const FoldableDef &OpToFold, MachineInstr *UseMI,
unsigned UseOpIdx,
@@ -966,15 +966,15 @@ const TargetRegisterClass
*SIFoldOperandsImpl::getRegSeqInit(
return getRegSeqInit(*Def, Defs);
}
-std::pair
+std::pair
SIFoldOperandsImpl::isRegSeqSplat(MachineInstr &RegSeq) const {
SmallVector, 32> Defs;
const TargetRegisterClass *SrcRC = getRegSeqInit(RegSeq, Defs);
if (!SrcRC)
return {};
- // TODO: Recognize 64-bit splats broken into 32-bit pieces (i.e. recognize
- // every other other element is 0 for 64-bit immediates)
+ bool TryToMatchSplat64 = false;
+
int64_t Imm;
for (unsigned I = 0, E = Defs.size(); I != E; ++I) {
const MachineOperand *Op = Defs[I].first;
@@ -986,38 +986,75 @@ SIFoldOperandsImpl::isRegSeqSplat(MachineInstr &RegSeq)
const {
Imm = SubImm;
continue;
}
-if (Imm != SubImm)
+
+if (Imm != SubImm) {
+ if (I == 1 && (E & 1) == 0) {
+// If we have an even number of inputs, there's a chance this is a
+// 64-bit element splat broken into 32-bit pieces.
+TryToMatchSplat64 = true;
+break;
+ }
+
return {}; // Can only fold splat constants
+}
+ }
+
+ if (!TryToMatchSplat64)
+return {Defs[0].first->getImm(), SrcRC};
+
+ // Fallback to recognizing 64-bit splats broken into 32-bit pieces
+ // (i.e. recognize every other other element is 0 for 64-bit immediates)
+ int64_t SplatVal64;
+ for (unsigned I = 0, E = Defs.size(); I != E; I += 2) {
+const MachineOperand *Op0 = Defs[I].first;
+const MachineOperand *Op1 = Defs[I + 1].first;
+
+if (!Op0->isImm() || !Op1->isImm())
+ return {};
+
+unsigned SubReg0 = Defs[I].second;
+unsigned SubReg1 = Defs[I + 1].second;
+
+// Assume we're going to generally encounter reg_sequences with sorted
+// subreg indexes, so reject any that aren't consecutive.
+if (TRI->getChannelFromSubReg(SubReg0) + 1 !=
+TRI->getChannelFromSubReg(SubReg1))
+ return {};
+
+int64_t MergedVal = Make_64(Op1->getImm(), Op0->getImm());
+if (I == 0)
+ SplatVal64 = MergedVal;
+else if (SplatVal64 != MergedVal)
+ return {};
}
- return {Defs[0].first, SrcRC};
+ const TargetRegisterClass *RC64 = TRI->getSubRegisterClass(
+ MRI->getRegClass(RegSeq.getOperand(0).getReg()), AMDGPU::sub0_sub1);
+
+ return {SplatVal64, RC64};
}
-MachineOperand *SIFoldOperandsImpl::tryFoldRegSeqSplat(
-MachineInstr *UseMI, unsigned UseOpIdx, MachineOperand *SplatVal,
+bool SIFoldOperandsImpl::tryFoldRegSeqSplat(
+MachineInstr *UseMI, unsigned UseOpIdx, int64_t SplatVal,
const TargetRegisterClass *SplatRC) const {
const MCInstrDesc &Desc = UseMI->getDesc();
if (UseOpIdx >= Desc.getNumOperands())
-return nullptr;
+return false;
// Filter out unhandled pseudos.
if (!AMDGPU::isSISrcOperand(Desc, UseOpIdx))
-return nullptr;
+return false;
int16_t RCID = Desc.operands()[UseOpIdx].RegClass;
if (RCID == -1)
-return nullptr;
+return false;
+
+ const TargetRegisterClass *OpRC = TRI->getRegClass(RCID);
// Special case 0/-1, since when interpreted as a 64-bit element both halves
- // have the same bits. Effectively this code does not handle 64-bit element
- // operands correctly, as the incoming 64-bit constants are already split
into
- // 32-bit sequence elements.
- //
- // TODO: We should try to figure out how to interpret the reg_sequence as a
- /
[llvm-branch-commits] [llvm] AMDGPU: Handle folding vector splats of inline split f64 inline immediates (PR #140878)
https://github.com/arsenm updated
https://github.com/llvm/llvm-project/pull/140878
>From 5d5a54cff58c1096973d3a9c28f728ca0afe3889 Mon Sep 17 00:00:00 2001
From: Matt Arsenault
Date: Mon, 19 May 2025 21:51:06 +0200
Subject: [PATCH] AMDGPU: Handle folding vector splats of inline split f64
inline immediates
Recognize a reg_sequence with 32-bit elements that produce a 64-bit
splat value. This enables folding f64 constants into mfma operands
---
llvm/lib/Target/AMDGPU/SIFoldOperands.cpp | 103 --
.../CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll | 41 +--
2 files changed, 76 insertions(+), 68 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index 565abad9b0366..8121098a97bd1 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -227,12 +227,12 @@ class SIFoldOperandsImpl {
getRegSeqInit(SmallVectorImpl> &Defs,
Register UseReg) const;
- std::pair
+ std::pair
isRegSeqSplat(MachineInstr &RegSeg) const;
- MachineOperand *tryFoldRegSeqSplat(MachineInstr *UseMI, unsigned UseOpIdx,
- MachineOperand *SplatVal,
- const TargetRegisterClass *SplatRC) const;
+ bool tryFoldRegSeqSplat(MachineInstr *UseMI, unsigned UseOpIdx,
+ int64_t SplatVal,
+ const TargetRegisterClass *SplatRC) const;
bool tryToFoldACImm(const FoldableDef &OpToFold, MachineInstr *UseMI,
unsigned UseOpIdx,
@@ -966,15 +966,15 @@ const TargetRegisterClass
*SIFoldOperandsImpl::getRegSeqInit(
return getRegSeqInit(*Def, Defs);
}
-std::pair
+std::pair
SIFoldOperandsImpl::isRegSeqSplat(MachineInstr &RegSeq) const {
SmallVector, 32> Defs;
const TargetRegisterClass *SrcRC = getRegSeqInit(RegSeq, Defs);
if (!SrcRC)
return {};
- // TODO: Recognize 64-bit splats broken into 32-bit pieces (i.e. recognize
- // every other other element is 0 for 64-bit immediates)
+ bool TryToMatchSplat64 = false;
+
int64_t Imm;
for (unsigned I = 0, E = Defs.size(); I != E; ++I) {
const MachineOperand *Op = Defs[I].first;
@@ -986,38 +986,75 @@ SIFoldOperandsImpl::isRegSeqSplat(MachineInstr &RegSeq)
const {
Imm = SubImm;
continue;
}
-if (Imm != SubImm)
+
+if (Imm != SubImm) {
+ if (I == 1 && (E & 1) == 0) {
+// If we have an even number of inputs, there's a chance this is a
+// 64-bit element splat broken into 32-bit pieces.
+TryToMatchSplat64 = true;
+break;
+ }
+
return {}; // Can only fold splat constants
+}
+ }
+
+ if (!TryToMatchSplat64)
+return {Defs[0].first->getImm(), SrcRC};
+
+ // Fallback to recognizing 64-bit splats broken into 32-bit pieces
+ // (i.e. recognize every other other element is 0 for 64-bit immediates)
+ int64_t SplatVal64;
+ for (unsigned I = 0, E = Defs.size(); I != E; I += 2) {
+const MachineOperand *Op0 = Defs[I].first;
+const MachineOperand *Op1 = Defs[I + 1].first;
+
+if (!Op0->isImm() || !Op1->isImm())
+ return {};
+
+unsigned SubReg0 = Defs[I].second;
+unsigned SubReg1 = Defs[I + 1].second;
+
+// Assume we're going to generally encounter reg_sequences with sorted
+// subreg indexes, so reject any that aren't consecutive.
+if (TRI->getChannelFromSubReg(SubReg0) + 1 !=
+TRI->getChannelFromSubReg(SubReg1))
+ return {};
+
+int64_t MergedVal = Make_64(Op1->getImm(), Op0->getImm());
+if (I == 0)
+ SplatVal64 = MergedVal;
+else if (SplatVal64 != MergedVal)
+ return {};
}
- return {Defs[0].first, SrcRC};
+ const TargetRegisterClass *RC64 = TRI->getSubRegisterClass(
+ MRI->getRegClass(RegSeq.getOperand(0).getReg()), AMDGPU::sub0_sub1);
+
+ return {SplatVal64, RC64};
}
-MachineOperand *SIFoldOperandsImpl::tryFoldRegSeqSplat(
-MachineInstr *UseMI, unsigned UseOpIdx, MachineOperand *SplatVal,
+bool SIFoldOperandsImpl::tryFoldRegSeqSplat(
+MachineInstr *UseMI, unsigned UseOpIdx, int64_t SplatVal,
const TargetRegisterClass *SplatRC) const {
const MCInstrDesc &Desc = UseMI->getDesc();
if (UseOpIdx >= Desc.getNumOperands())
-return nullptr;
+return false;
// Filter out unhandled pseudos.
if (!AMDGPU::isSISrcOperand(Desc, UseOpIdx))
-return nullptr;
+return false;
int16_t RCID = Desc.operands()[UseOpIdx].RegClass;
if (RCID == -1)
-return nullptr;
+return false;
+
+ const TargetRegisterClass *OpRC = TRI->getRegClass(RCID);
// Special case 0/-1, since when interpreted as a 64-bit element both halves
- // have the same bits. Effectively this code does not handle 64-bit element
- // operands correctly, as the incoming 64-bit constants are already split
into
- // 32-bit sequence elements.
- //
- // TODO: We should try to figure out how to interpret the reg_sequence as a
- /
[llvm-branch-commits] [llvm] AMDGPU: Handle folding vector splats of inline split f64 inline immediates (PR #140878)
https://github.com/arsenm updated
https://github.com/llvm/llvm-project/pull/140878
>From 1f79a2b6309fc6cb67d746652486845bc9795ff3 Mon Sep 17 00:00:00 2001
From: Matt Arsenault
Date: Mon, 19 May 2025 21:51:06 +0200
Subject: [PATCH] AMDGPU: Handle folding vector splats of inline split f64
inline immediates
Recognize a reg_sequence with 32-bit elements that produce a 64-bit
splat value. This enables folding f64 constants into mfma operands
---
llvm/lib/Target/AMDGPU/SIFoldOperands.cpp | 103 --
.../CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll | 41 +--
2 files changed, 76 insertions(+), 68 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index 507ae46179866..6eaa801fa568d 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -226,12 +226,12 @@ class SIFoldOperandsImpl {
getRegSeqInit(SmallVectorImpl> &Defs,
Register UseReg) const;
- std::pair
+ std::pair
isRegSeqSplat(MachineInstr &RegSeg) const;
- MachineOperand *tryFoldRegSeqSplat(MachineInstr *UseMI, unsigned UseOpIdx,
- MachineOperand *SplatVal,
- const TargetRegisterClass *SplatRC) const;
+ bool tryFoldRegSeqSplat(MachineInstr *UseMI, unsigned UseOpIdx,
+ int64_t SplatVal,
+ const TargetRegisterClass *SplatRC) const;
bool tryToFoldACImm(const FoldableDef &OpToFold, MachineInstr *UseMI,
unsigned UseOpIdx,
@@ -964,15 +964,15 @@ const TargetRegisterClass
*SIFoldOperandsImpl::getRegSeqInit(
return getRegSeqInit(*Def, Defs);
}
-std::pair
+std::pair
SIFoldOperandsImpl::isRegSeqSplat(MachineInstr &RegSeq) const {
SmallVector, 32> Defs;
const TargetRegisterClass *SrcRC = getRegSeqInit(RegSeq, Defs);
if (!SrcRC)
return {};
- // TODO: Recognize 64-bit splats broken into 32-bit pieces (i.e. recognize
- // every other other element is 0 for 64-bit immediates)
+ bool TryToMatchSplat64 = false;
+
int64_t Imm;
for (unsigned I = 0, E = Defs.size(); I != E; ++I) {
const MachineOperand *Op = Defs[I].first;
@@ -984,38 +984,75 @@ SIFoldOperandsImpl::isRegSeqSplat(MachineInstr &RegSeq)
const {
Imm = SubImm;
continue;
}
-if (Imm != SubImm)
+
+if (Imm != SubImm) {
+ if (I == 1 && (E & 1) == 0) {
+// If we have an even number of inputs, there's a chance this is a
+// 64-bit element splat broken into 32-bit pieces.
+TryToMatchSplat64 = true;
+break;
+ }
+
return {}; // Can only fold splat constants
+}
+ }
+
+ if (!TryToMatchSplat64)
+return {Defs[0].first->getImm(), SrcRC};
+
+ // Fallback to recognizing 64-bit splats broken into 32-bit pieces
+ // (i.e. recognize every other other element is 0 for 64-bit immediates)
+ int64_t SplatVal64;
+ for (unsigned I = 0, E = Defs.size(); I != E; I += 2) {
+const MachineOperand *Op0 = Defs[I].first;
+const MachineOperand *Op1 = Defs[I + 1].first;
+
+if (!Op0->isImm() || !Op1->isImm())
+ return {};
+
+unsigned SubReg0 = Defs[I].second;
+unsigned SubReg1 = Defs[I + 1].second;
+
+// Assume we're going to generally encounter reg_sequences with sorted
+// subreg indexes, so reject any that aren't consecutive.
+if (TRI->getChannelFromSubReg(SubReg0) + 1 !=
+TRI->getChannelFromSubReg(SubReg1))
+ return {};
+
+int64_t MergedVal = Make_64(Op1->getImm(), Op0->getImm());
+if (I == 0)
+ SplatVal64 = MergedVal;
+else if (SplatVal64 != MergedVal)
+ return {};
}
- return {Defs[0].first, SrcRC};
+ const TargetRegisterClass *RC64 = TRI->getSubRegisterClass(
+ MRI->getRegClass(RegSeq.getOperand(0).getReg()), AMDGPU::sub0_sub1);
+
+ return {SplatVal64, RC64};
}
-MachineOperand *SIFoldOperandsImpl::tryFoldRegSeqSplat(
-MachineInstr *UseMI, unsigned UseOpIdx, MachineOperand *SplatVal,
+bool SIFoldOperandsImpl::tryFoldRegSeqSplat(
+MachineInstr *UseMI, unsigned UseOpIdx, int64_t SplatVal,
const TargetRegisterClass *SplatRC) const {
const MCInstrDesc &Desc = UseMI->getDesc();
if (UseOpIdx >= Desc.getNumOperands())
-return nullptr;
+return false;
// Filter out unhandled pseudos.
if (!AMDGPU::isSISrcOperand(Desc, UseOpIdx))
-return nullptr;
+return false;
int16_t RCID = Desc.operands()[UseOpIdx].RegClass;
if (RCID == -1)
-return nullptr;
+return false;
+
+ const TargetRegisterClass *OpRC = TRI->getRegClass(RCID);
// Special case 0/-1, since when interpreted as a 64-bit element both halves
- // have the same bits. Effectively this code does not handle 64-bit element
- // operands correctly, as the incoming 64-bit constants are already split
into
- // 32-bit sequence elements.
- //
- // TODO: We should try to figure out how to interpret the reg_sequence as a
- /
[llvm-branch-commits] [llvm] AMDGPU: Handle folding vector splats of inline split f64 inline immediates (PR #140878)
https://github.com/arsenm updated
https://github.com/llvm/llvm-project/pull/140878
>From 1f79a2b6309fc6cb67d746652486845bc9795ff3 Mon Sep 17 00:00:00 2001
From: Matt Arsenault
Date: Mon, 19 May 2025 21:51:06 +0200
Subject: [PATCH] AMDGPU: Handle folding vector splats of inline split f64
inline immediates
Recognize a reg_sequence with 32-bit elements that produce a 64-bit
splat value. This enables folding f64 constants into mfma operands
---
llvm/lib/Target/AMDGPU/SIFoldOperands.cpp | 103 --
.../CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll | 41 +--
2 files changed, 76 insertions(+), 68 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index 507ae46179866..6eaa801fa568d 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -226,12 +226,12 @@ class SIFoldOperandsImpl {
getRegSeqInit(SmallVectorImpl> &Defs,
Register UseReg) const;
- std::pair
+ std::pair
isRegSeqSplat(MachineInstr &RegSeg) const;
- MachineOperand *tryFoldRegSeqSplat(MachineInstr *UseMI, unsigned UseOpIdx,
- MachineOperand *SplatVal,
- const TargetRegisterClass *SplatRC) const;
+ bool tryFoldRegSeqSplat(MachineInstr *UseMI, unsigned UseOpIdx,
+ int64_t SplatVal,
+ const TargetRegisterClass *SplatRC) const;
bool tryToFoldACImm(const FoldableDef &OpToFold, MachineInstr *UseMI,
unsigned UseOpIdx,
@@ -964,15 +964,15 @@ const TargetRegisterClass
*SIFoldOperandsImpl::getRegSeqInit(
return getRegSeqInit(*Def, Defs);
}
-std::pair
+std::pair
SIFoldOperandsImpl::isRegSeqSplat(MachineInstr &RegSeq) const {
SmallVector, 32> Defs;
const TargetRegisterClass *SrcRC = getRegSeqInit(RegSeq, Defs);
if (!SrcRC)
return {};
- // TODO: Recognize 64-bit splats broken into 32-bit pieces (i.e. recognize
- // every other other element is 0 for 64-bit immediates)
+ bool TryToMatchSplat64 = false;
+
int64_t Imm;
for (unsigned I = 0, E = Defs.size(); I != E; ++I) {
const MachineOperand *Op = Defs[I].first;
@@ -984,38 +984,75 @@ SIFoldOperandsImpl::isRegSeqSplat(MachineInstr &RegSeq)
const {
Imm = SubImm;
continue;
}
-if (Imm != SubImm)
+
+if (Imm != SubImm) {
+ if (I == 1 && (E & 1) == 0) {
+// If we have an even number of inputs, there's a chance this is a
+// 64-bit element splat broken into 32-bit pieces.
+TryToMatchSplat64 = true;
+break;
+ }
+
return {}; // Can only fold splat constants
+}
+ }
+
+ if (!TryToMatchSplat64)
+return {Defs[0].first->getImm(), SrcRC};
+
+ // Fallback to recognizing 64-bit splats broken into 32-bit pieces
+ // (i.e. recognize every other other element is 0 for 64-bit immediates)
+ int64_t SplatVal64;
+ for (unsigned I = 0, E = Defs.size(); I != E; I += 2) {
+const MachineOperand *Op0 = Defs[I].first;
+const MachineOperand *Op1 = Defs[I + 1].first;
+
+if (!Op0->isImm() || !Op1->isImm())
+ return {};
+
+unsigned SubReg0 = Defs[I].second;
+unsigned SubReg1 = Defs[I + 1].second;
+
+// Assume we're going to generally encounter reg_sequences with sorted
+// subreg indexes, so reject any that aren't consecutive.
+if (TRI->getChannelFromSubReg(SubReg0) + 1 !=
+TRI->getChannelFromSubReg(SubReg1))
+ return {};
+
+int64_t MergedVal = Make_64(Op1->getImm(), Op0->getImm());
+if (I == 0)
+ SplatVal64 = MergedVal;
+else if (SplatVal64 != MergedVal)
+ return {};
}
- return {Defs[0].first, SrcRC};
+ const TargetRegisterClass *RC64 = TRI->getSubRegisterClass(
+ MRI->getRegClass(RegSeq.getOperand(0).getReg()), AMDGPU::sub0_sub1);
+
+ return {SplatVal64, RC64};
}
-MachineOperand *SIFoldOperandsImpl::tryFoldRegSeqSplat(
-MachineInstr *UseMI, unsigned UseOpIdx, MachineOperand *SplatVal,
+bool SIFoldOperandsImpl::tryFoldRegSeqSplat(
+MachineInstr *UseMI, unsigned UseOpIdx, int64_t SplatVal,
const TargetRegisterClass *SplatRC) const {
const MCInstrDesc &Desc = UseMI->getDesc();
if (UseOpIdx >= Desc.getNumOperands())
-return nullptr;
+return false;
// Filter out unhandled pseudos.
if (!AMDGPU::isSISrcOperand(Desc, UseOpIdx))
-return nullptr;
+return false;
int16_t RCID = Desc.operands()[UseOpIdx].RegClass;
if (RCID == -1)
-return nullptr;
+return false;
+
+ const TargetRegisterClass *OpRC = TRI->getRegClass(RCID);
// Special case 0/-1, since when interpreted as a 64-bit element both halves
- // have the same bits. Effectively this code does not handle 64-bit element
- // operands correctly, as the incoming 64-bit constants are already split
into
- // 32-bit sequence elements.
- //
- // TODO: We should try to figure out how to interpret the reg_sequence as a
- /
[llvm-branch-commits] [llvm] AMDGPU: Handle folding vector splats of inline split f64 inline immediates (PR #140878)
https://github.com/arsenm updated
https://github.com/llvm/llvm-project/pull/140878
>From 97f782d35ea55aa17a7a5ed21b60bf5010a44573 Mon Sep 17 00:00:00 2001
From: Matt Arsenault
Date: Mon, 19 May 2025 21:51:06 +0200
Subject: [PATCH] AMDGPU: Handle folding vector splats of inline split f64
inline immediates
Recognize a reg_sequence with 32-bit elements that produce a 64-bit
splat value. This enables folding f64 constants into mfma operands
---
llvm/lib/Target/AMDGPU/SIFoldOperands.cpp | 103 --
.../CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll | 41 +--
2 files changed, 76 insertions(+), 68 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index 78d4fdf84a287..3ee7c25ccb1ae 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -227,12 +227,12 @@ class SIFoldOperandsImpl {
getRegSeqInit(SmallVectorImpl> &Defs,
Register UseReg) const;
- std::pair
+ std::pair
isRegSeqSplat(MachineInstr &RegSeg) const;
- MachineOperand *tryFoldRegSeqSplat(MachineInstr *UseMI, unsigned UseOpIdx,
- MachineOperand *SplatVal,
- const TargetRegisterClass *SplatRC) const;
+ bool tryFoldRegSeqSplat(MachineInstr *UseMI, unsigned UseOpIdx,
+ int64_t SplatVal,
+ const TargetRegisterClass *SplatRC) const;
bool tryToFoldACImm(const FoldableDef &OpToFold, MachineInstr *UseMI,
unsigned UseOpIdx,
@@ -966,15 +966,15 @@ const TargetRegisterClass
*SIFoldOperandsImpl::getRegSeqInit(
return getRegSeqInit(*Def, Defs);
}
-std::pair
+std::pair
SIFoldOperandsImpl::isRegSeqSplat(MachineInstr &RegSeq) const {
SmallVector, 32> Defs;
const TargetRegisterClass *SrcRC = getRegSeqInit(RegSeq, Defs);
if (!SrcRC)
return {};
- // TODO: Recognize 64-bit splats broken into 32-bit pieces (i.e. recognize
- // every other other element is 0 for 64-bit immediates)
+ bool TryToMatchSplat64 = false;
+
int64_t Imm;
for (unsigned I = 0, E = Defs.size(); I != E; ++I) {
const MachineOperand *Op = Defs[I].first;
@@ -986,38 +986,75 @@ SIFoldOperandsImpl::isRegSeqSplat(MachineInstr &RegSeq)
const {
Imm = SubImm;
continue;
}
-if (Imm != SubImm)
+
+if (Imm != SubImm) {
+ if (I == 1 && (E & 1) == 0) {
+// If we have an even number of inputs, there's a chance this is a
+// 64-bit element splat broken into 32-bit pieces.
+TryToMatchSplat64 = true;
+break;
+ }
+
return {}; // Can only fold splat constants
+}
+ }
+
+ if (!TryToMatchSplat64)
+return {Defs[0].first->getImm(), SrcRC};
+
+ // Fallback to recognizing 64-bit splats broken into 32-bit pieces
+ // (i.e. recognize every other other element is 0 for 64-bit immediates)
+ int64_t SplatVal64;
+ for (unsigned I = 0, E = Defs.size(); I != E; I += 2) {
+const MachineOperand *Op0 = Defs[I].first;
+const MachineOperand *Op1 = Defs[I + 1].first;
+
+if (!Op0->isImm() || !Op1->isImm())
+ return {};
+
+unsigned SubReg0 = Defs[I].second;
+unsigned SubReg1 = Defs[I + 1].second;
+
+// Assume we're going to generally encounter reg_sequences with sorted
+// subreg indexes, so reject any that aren't consecutive.
+if (TRI->getChannelFromSubReg(SubReg0) + 1 !=
+TRI->getChannelFromSubReg(SubReg1))
+ return {};
+
+int64_t MergedVal = Make_64(Op1->getImm(), Op0->getImm());
+if (I == 0)
+ SplatVal64 = MergedVal;
+else if (SplatVal64 != MergedVal)
+ return {};
}
- return {Defs[0].first, SrcRC};
+ const TargetRegisterClass *RC64 = TRI->getSubRegisterClass(
+ MRI->getRegClass(RegSeq.getOperand(0).getReg()), AMDGPU::sub0_sub1);
+
+ return {SplatVal64, RC64};
}
-MachineOperand *SIFoldOperandsImpl::tryFoldRegSeqSplat(
-MachineInstr *UseMI, unsigned UseOpIdx, MachineOperand *SplatVal,
+bool SIFoldOperandsImpl::tryFoldRegSeqSplat(
+MachineInstr *UseMI, unsigned UseOpIdx, int64_t SplatVal,
const TargetRegisterClass *SplatRC) const {
const MCInstrDesc &Desc = UseMI->getDesc();
if (UseOpIdx >= Desc.getNumOperands())
-return nullptr;
+return false;
// Filter out unhandled pseudos.
if (!AMDGPU::isSISrcOperand(Desc, UseOpIdx))
-return nullptr;
+return false;
int16_t RCID = Desc.operands()[UseOpIdx].RegClass;
if (RCID == -1)
-return nullptr;
+return false;
+
+ const TargetRegisterClass *OpRC = TRI->getRegClass(RCID);
// Special case 0/-1, since when interpreted as a 64-bit element both halves
- // have the same bits. Effectively this code does not handle 64-bit element
- // operands correctly, as the incoming 64-bit constants are already split
into
- // 32-bit sequence elements.
- //
- // TODO: We should try to figure out how to interpret the reg_sequence as a
- /
[llvm-branch-commits] [llvm] AMDGPU: Handle folding vector splats of inline split f64 inline immediates (PR #140878)
https://github.com/arsenm updated
https://github.com/llvm/llvm-project/pull/140878
>From 97f782d35ea55aa17a7a5ed21b60bf5010a44573 Mon Sep 17 00:00:00 2001
From: Matt Arsenault
Date: Mon, 19 May 2025 21:51:06 +0200
Subject: [PATCH] AMDGPU: Handle folding vector splats of inline split f64
inline immediates
Recognize a reg_sequence with 32-bit elements that produce a 64-bit
splat value. This enables folding f64 constants into mfma operands
---
llvm/lib/Target/AMDGPU/SIFoldOperands.cpp | 103 --
.../CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll | 41 +--
2 files changed, 76 insertions(+), 68 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index 78d4fdf84a287..3ee7c25ccb1ae 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -227,12 +227,12 @@ class SIFoldOperandsImpl {
getRegSeqInit(SmallVectorImpl> &Defs,
Register UseReg) const;
- std::pair
+ std::pair
isRegSeqSplat(MachineInstr &RegSeg) const;
- MachineOperand *tryFoldRegSeqSplat(MachineInstr *UseMI, unsigned UseOpIdx,
- MachineOperand *SplatVal,
- const TargetRegisterClass *SplatRC) const;
+ bool tryFoldRegSeqSplat(MachineInstr *UseMI, unsigned UseOpIdx,
+ int64_t SplatVal,
+ const TargetRegisterClass *SplatRC) const;
bool tryToFoldACImm(const FoldableDef &OpToFold, MachineInstr *UseMI,
unsigned UseOpIdx,
@@ -966,15 +966,15 @@ const TargetRegisterClass
*SIFoldOperandsImpl::getRegSeqInit(
return getRegSeqInit(*Def, Defs);
}
-std::pair
+std::pair
SIFoldOperandsImpl::isRegSeqSplat(MachineInstr &RegSeq) const {
SmallVector, 32> Defs;
const TargetRegisterClass *SrcRC = getRegSeqInit(RegSeq, Defs);
if (!SrcRC)
return {};
- // TODO: Recognize 64-bit splats broken into 32-bit pieces (i.e. recognize
- // every other other element is 0 for 64-bit immediates)
+ bool TryToMatchSplat64 = false;
+
int64_t Imm;
for (unsigned I = 0, E = Defs.size(); I != E; ++I) {
const MachineOperand *Op = Defs[I].first;
@@ -986,38 +986,75 @@ SIFoldOperandsImpl::isRegSeqSplat(MachineInstr &RegSeq)
const {
Imm = SubImm;
continue;
}
-if (Imm != SubImm)
+
+if (Imm != SubImm) {
+ if (I == 1 && (E & 1) == 0) {
+// If we have an even number of inputs, there's a chance this is a
+// 64-bit element splat broken into 32-bit pieces.
+TryToMatchSplat64 = true;
+break;
+ }
+
return {}; // Can only fold splat constants
+}
+ }
+
+ if (!TryToMatchSplat64)
+return {Defs[0].first->getImm(), SrcRC};
+
+ // Fallback to recognizing 64-bit splats broken into 32-bit pieces
+ // (i.e. recognize every other other element is 0 for 64-bit immediates)
+ int64_t SplatVal64;
+ for (unsigned I = 0, E = Defs.size(); I != E; I += 2) {
+const MachineOperand *Op0 = Defs[I].first;
+const MachineOperand *Op1 = Defs[I + 1].first;
+
+if (!Op0->isImm() || !Op1->isImm())
+ return {};
+
+unsigned SubReg0 = Defs[I].second;
+unsigned SubReg1 = Defs[I + 1].second;
+
+// Assume we're going to generally encounter reg_sequences with sorted
+// subreg indexes, so reject any that aren't consecutive.
+if (TRI->getChannelFromSubReg(SubReg0) + 1 !=
+TRI->getChannelFromSubReg(SubReg1))
+ return {};
+
+int64_t MergedVal = Make_64(Op1->getImm(), Op0->getImm());
+if (I == 0)
+ SplatVal64 = MergedVal;
+else if (SplatVal64 != MergedVal)
+ return {};
}
- return {Defs[0].first, SrcRC};
+ const TargetRegisterClass *RC64 = TRI->getSubRegisterClass(
+ MRI->getRegClass(RegSeq.getOperand(0).getReg()), AMDGPU::sub0_sub1);
+
+ return {SplatVal64, RC64};
}
-MachineOperand *SIFoldOperandsImpl::tryFoldRegSeqSplat(
-MachineInstr *UseMI, unsigned UseOpIdx, MachineOperand *SplatVal,
+bool SIFoldOperandsImpl::tryFoldRegSeqSplat(
+MachineInstr *UseMI, unsigned UseOpIdx, int64_t SplatVal,
const TargetRegisterClass *SplatRC) const {
const MCInstrDesc &Desc = UseMI->getDesc();
if (UseOpIdx >= Desc.getNumOperands())
-return nullptr;
+return false;
// Filter out unhandled pseudos.
if (!AMDGPU::isSISrcOperand(Desc, UseOpIdx))
-return nullptr;
+return false;
int16_t RCID = Desc.operands()[UseOpIdx].RegClass;
if (RCID == -1)
-return nullptr;
+return false;
+
+ const TargetRegisterClass *OpRC = TRI->getRegClass(RCID);
// Special case 0/-1, since when interpreted as a 64-bit element both halves
- // have the same bits. Effectively this code does not handle 64-bit element
- // operands correctly, as the incoming 64-bit constants are already split
into
- // 32-bit sequence elements.
- //
- // TODO: We should try to figure out how to interpret the reg_sequence as a
- /
[llvm-branch-commits] [llvm] AMDGPU: Handle folding vector splats of inline split f64 inline immediates (PR #140878)
https://github.com/arsenm updated
https://github.com/llvm/llvm-project/pull/140878
>From 609dc72abf36343e62c4bb0bc149f9ba453f4236 Mon Sep 17 00:00:00 2001
From: Matt Arsenault
Date: Mon, 19 May 2025 21:51:06 +0200
Subject: [PATCH] AMDGPU: Handle folding vector splats of inline split f64
inline immediates
Recognize a reg_sequence with 32-bit elements that produce a 64-bit
splat value. This enables folding f64 constants into mfma operands
---
llvm/lib/Target/AMDGPU/SIFoldOperands.cpp | 103 --
.../CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll | 41 +--
2 files changed, 76 insertions(+), 68 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index d7a4fa85e4034..7cf2549804bee 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -227,12 +227,12 @@ class SIFoldOperandsImpl {
getRegSeqInit(SmallVectorImpl> &Defs,
Register UseReg) const;
- std::pair
+ std::pair
isRegSeqSplat(MachineInstr &RegSeg) const;
- MachineOperand *tryFoldRegSeqSplat(MachineInstr *UseMI, unsigned UseOpIdx,
- MachineOperand *SplatVal,
- const TargetRegisterClass *SplatRC) const;
+ bool tryFoldRegSeqSplat(MachineInstr *UseMI, unsigned UseOpIdx,
+ int64_t SplatVal,
+ const TargetRegisterClass *SplatRC) const;
bool tryToFoldACImm(const FoldableDef &OpToFold, MachineInstr *UseMI,
unsigned UseOpIdx,
@@ -966,15 +966,15 @@ const TargetRegisterClass
*SIFoldOperandsImpl::getRegSeqInit(
return getRegSeqInit(*Def, Defs);
}
-std::pair
+std::pair
SIFoldOperandsImpl::isRegSeqSplat(MachineInstr &RegSeq) const {
SmallVector, 32> Defs;
const TargetRegisterClass *SrcRC = getRegSeqInit(RegSeq, Defs);
if (!SrcRC)
return {};
- // TODO: Recognize 64-bit splats broken into 32-bit pieces (i.e. recognize
- // every other other element is 0 for 64-bit immediates)
+ bool TryToMatchSplat64 = false;
+
int64_t Imm;
for (unsigned I = 0, E = Defs.size(); I != E; ++I) {
const MachineOperand *Op = Defs[I].first;
@@ -986,38 +986,75 @@ SIFoldOperandsImpl::isRegSeqSplat(MachineInstr &RegSeq)
const {
Imm = SubImm;
continue;
}
-if (Imm != SubImm)
+
+if (Imm != SubImm) {
+ if (I == 1 && (E & 1) == 0) {
+// If we have an even number of inputs, there's a chance this is a
+// 64-bit element splat broken into 32-bit pieces.
+TryToMatchSplat64 = true;
+break;
+ }
+
return {}; // Can only fold splat constants
+}
+ }
+
+ if (!TryToMatchSplat64)
+return {Defs[0].first->getImm(), SrcRC};
+
+ // Fallback to recognizing 64-bit splats broken into 32-bit pieces
+ // (i.e. recognize every other other element is 0 for 64-bit immediates)
+ int64_t SplatVal64;
+ for (unsigned I = 0, E = Defs.size(); I != E; I += 2) {
+const MachineOperand *Op0 = Defs[I].first;
+const MachineOperand *Op1 = Defs[I + 1].first;
+
+if (!Op0->isImm() || !Op1->isImm())
+ return {};
+
+unsigned SubReg0 = Defs[I].second;
+unsigned SubReg1 = Defs[I + 1].second;
+
+// Assume we're going to generally encounter reg_sequences with sorted
+// subreg indexes, so reject any that aren't consecutive.
+if (TRI->getChannelFromSubReg(SubReg0) + 1 !=
+TRI->getChannelFromSubReg(SubReg1))
+ return {};
+
+int64_t MergedVal = Make_64(Op1->getImm(), Op0->getImm());
+if (I == 0)
+ SplatVal64 = MergedVal;
+else if (SplatVal64 != MergedVal)
+ return {};
}
- return {Defs[0].first, SrcRC};
+ const TargetRegisterClass *RC64 = TRI->getSubRegisterClass(
+ MRI->getRegClass(RegSeq.getOperand(0).getReg()), AMDGPU::sub0_sub1);
+
+ return {SplatVal64, RC64};
}
-MachineOperand *SIFoldOperandsImpl::tryFoldRegSeqSplat(
-MachineInstr *UseMI, unsigned UseOpIdx, MachineOperand *SplatVal,
+bool SIFoldOperandsImpl::tryFoldRegSeqSplat(
+MachineInstr *UseMI, unsigned UseOpIdx, int64_t SplatVal,
const TargetRegisterClass *SplatRC) const {
const MCInstrDesc &Desc = UseMI->getDesc();
if (UseOpIdx >= Desc.getNumOperands())
-return nullptr;
+return false;
// Filter out unhandled pseudos.
if (!AMDGPU::isSISrcOperand(Desc, UseOpIdx))
-return nullptr;
+return false;
int16_t RCID = Desc.operands()[UseOpIdx].RegClass;
if (RCID == -1)
-return nullptr;
+return false;
+
+ const TargetRegisterClass *OpRC = TRI->getRegClass(RCID);
// Special case 0/-1, since when interpreted as a 64-bit element both halves
- // have the same bits. Effectively this code does not handle 64-bit element
- // operands correctly, as the incoming 64-bit constants are already split
into
- // 32-bit sequence elements.
- //
- // TODO: We should try to figure out how to interpret the reg_sequence as a
- /
[llvm-branch-commits] [llvm] AMDGPU: Handle folding vector splats of inline split f64 inline immediates (PR #140878)
https://github.com/arsenm updated
https://github.com/llvm/llvm-project/pull/140878
>From 609dc72abf36343e62c4bb0bc149f9ba453f4236 Mon Sep 17 00:00:00 2001
From: Matt Arsenault
Date: Mon, 19 May 2025 21:51:06 +0200
Subject: [PATCH] AMDGPU: Handle folding vector splats of inline split f64
inline immediates
Recognize a reg_sequence with 32-bit elements that produce a 64-bit
splat value. This enables folding f64 constants into mfma operands
---
llvm/lib/Target/AMDGPU/SIFoldOperands.cpp | 103 --
.../CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll | 41 +--
2 files changed, 76 insertions(+), 68 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index d7a4fa85e4034..7cf2549804bee 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -227,12 +227,12 @@ class SIFoldOperandsImpl {
getRegSeqInit(SmallVectorImpl> &Defs,
Register UseReg) const;
- std::pair
+ std::pair
isRegSeqSplat(MachineInstr &RegSeg) const;
- MachineOperand *tryFoldRegSeqSplat(MachineInstr *UseMI, unsigned UseOpIdx,
- MachineOperand *SplatVal,
- const TargetRegisterClass *SplatRC) const;
+ bool tryFoldRegSeqSplat(MachineInstr *UseMI, unsigned UseOpIdx,
+ int64_t SplatVal,
+ const TargetRegisterClass *SplatRC) const;
bool tryToFoldACImm(const FoldableDef &OpToFold, MachineInstr *UseMI,
unsigned UseOpIdx,
@@ -966,15 +966,15 @@ const TargetRegisterClass
*SIFoldOperandsImpl::getRegSeqInit(
return getRegSeqInit(*Def, Defs);
}
-std::pair
+std::pair
SIFoldOperandsImpl::isRegSeqSplat(MachineInstr &RegSeq) const {
SmallVector, 32> Defs;
const TargetRegisterClass *SrcRC = getRegSeqInit(RegSeq, Defs);
if (!SrcRC)
return {};
- // TODO: Recognize 64-bit splats broken into 32-bit pieces (i.e. recognize
- // every other other element is 0 for 64-bit immediates)
+ bool TryToMatchSplat64 = false;
+
int64_t Imm;
for (unsigned I = 0, E = Defs.size(); I != E; ++I) {
const MachineOperand *Op = Defs[I].first;
@@ -986,38 +986,75 @@ SIFoldOperandsImpl::isRegSeqSplat(MachineInstr &RegSeq)
const {
Imm = SubImm;
continue;
}
-if (Imm != SubImm)
+
+if (Imm != SubImm) {
+ if (I == 1 && (E & 1) == 0) {
+// If we have an even number of inputs, there's a chance this is a
+// 64-bit element splat broken into 32-bit pieces.
+TryToMatchSplat64 = true;
+break;
+ }
+
return {}; // Can only fold splat constants
+}
+ }
+
+ if (!TryToMatchSplat64)
+return {Defs[0].first->getImm(), SrcRC};
+
+ // Fallback to recognizing 64-bit splats broken into 32-bit pieces
+ // (i.e. recognize every other other element is 0 for 64-bit immediates)
+ int64_t SplatVal64;
+ for (unsigned I = 0, E = Defs.size(); I != E; I += 2) {
+const MachineOperand *Op0 = Defs[I].first;
+const MachineOperand *Op1 = Defs[I + 1].first;
+
+if (!Op0->isImm() || !Op1->isImm())
+ return {};
+
+unsigned SubReg0 = Defs[I].second;
+unsigned SubReg1 = Defs[I + 1].second;
+
+// Assume we're going to generally encounter reg_sequences with sorted
+// subreg indexes, so reject any that aren't consecutive.
+if (TRI->getChannelFromSubReg(SubReg0) + 1 !=
+TRI->getChannelFromSubReg(SubReg1))
+ return {};
+
+int64_t MergedVal = Make_64(Op1->getImm(), Op0->getImm());
+if (I == 0)
+ SplatVal64 = MergedVal;
+else if (SplatVal64 != MergedVal)
+ return {};
}
- return {Defs[0].first, SrcRC};
+ const TargetRegisterClass *RC64 = TRI->getSubRegisterClass(
+ MRI->getRegClass(RegSeq.getOperand(0).getReg()), AMDGPU::sub0_sub1);
+
+ return {SplatVal64, RC64};
}
-MachineOperand *SIFoldOperandsImpl::tryFoldRegSeqSplat(
-MachineInstr *UseMI, unsigned UseOpIdx, MachineOperand *SplatVal,
+bool SIFoldOperandsImpl::tryFoldRegSeqSplat(
+MachineInstr *UseMI, unsigned UseOpIdx, int64_t SplatVal,
const TargetRegisterClass *SplatRC) const {
const MCInstrDesc &Desc = UseMI->getDesc();
if (UseOpIdx >= Desc.getNumOperands())
-return nullptr;
+return false;
// Filter out unhandled pseudos.
if (!AMDGPU::isSISrcOperand(Desc, UseOpIdx))
-return nullptr;
+return false;
int16_t RCID = Desc.operands()[UseOpIdx].RegClass;
if (RCID == -1)
-return nullptr;
+return false;
+
+ const TargetRegisterClass *OpRC = TRI->getRegClass(RCID);
// Special case 0/-1, since when interpreted as a 64-bit element both halves
- // have the same bits. Effectively this code does not handle 64-bit element
- // operands correctly, as the incoming 64-bit constants are already split
into
- // 32-bit sequence elements.
- //
- // TODO: We should try to figure out how to interpret the reg_sequence as a
- /
[llvm-branch-commits] [llvm] AMDGPU: Handle folding vector splats of inline split f64 inline immediates (PR #140878)
https://github.com/arsenm ready_for_review https://github.com/llvm/llvm-project/pull/140878 ___ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
[llvm-branch-commits] [llvm] AMDGPU: Handle folding vector splats of inline split f64 inline immediates (PR #140878)
arsenm wrote: > [!WARNING] > This pull request is not mergeable via GitHub because a downstack PR is > open. Once all requirements are satisfied, merge this PR as a stack href="https://app.graphite.dev/github/pr/llvm/llvm-project/140878?utm_source=stack-comment-downstack-mergeability-warning"; > >on Graphite. > https://graphite.dev/docs/merge-pull-requests";>Learn more * **#140878** https://app.graphite.dev/github/pr/llvm/llvm-project/140878?utm_source=stack-comment-icon"; target="_blank">https://static.graphite.dev/graphite-32x32-black.png"; alt="Graphite" width="10px" height="10px"/> 👈 https://app.graphite.dev/github/pr/llvm/llvm-project/140878?utm_source=stack-comment-view-in-graphite"; target="_blank">(View in Graphite) * **#140608** https://app.graphite.dev/github/pr/llvm/llvm-project/140608?utm_source=stack-comment-icon"; target="_blank">https://static.graphite.dev/graphite-32x32-black.png"; alt="Graphite" width="10px" height="10px"/> * **#140607** https://app.graphite.dev/github/pr/llvm/llvm-project/140607?utm_source=stack-comment-icon"; target="_blank">https://static.graphite.dev/graphite-32x32-black.png"; alt="Graphite" width="10px" height="10px"/> * **#140587** https://app.graphite.dev/github/pr/llvm/llvm-project/140587?utm_source=stack-comment-icon"; target="_blank">https://static.graphite.dev/graphite-32x32-black.png"; alt="Graphite" width="10px" height="10px"/> * **#140580** https://app.graphite.dev/github/pr/llvm/llvm-project/140580?utm_source=stack-comment-icon"; target="_blank">https://static.graphite.dev/graphite-32x32-black.png"; alt="Graphite" width="10px" height="10px"/> * `main` This stack of pull requests is managed by https://graphite.dev?utm-source=stack-comment";>Graphite. Learn more about https://stacking.dev/?utm_source=stack-comment";>stacking. https://github.com/llvm/llvm-project/pull/140878 ___ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
[llvm-branch-commits] [llvm] AMDGPU: Handle folding vector splats of inline split f64 inline immediates (PR #140878)
llvmbot wrote:
@llvm/pr-subscribers-backend-amdgpu
Author: Matt Arsenault (arsenm)
Changes
Recognize a reg_sequence with 32-bit elements that produce a 64-bit
splat value. This enables folding f64 constants into mfma operands
---
Full diff: https://github.com/llvm/llvm-project/pull/140878.diff
2 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/SIFoldOperands.cpp (+70-33)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll (+6-35)
``diff
diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index eb7fb94e25f5c..70e3974bb22b4 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -227,12 +227,12 @@ class SIFoldOperandsImpl {
getRegSeqInit(SmallVectorImpl> &Defs,
Register UseReg) const;
- std::pair
+ std::pair
isRegSeqSplat(MachineInstr &RegSeg) const;
- MachineOperand *tryFoldRegSeqSplat(MachineInstr *UseMI, unsigned UseOpIdx,
- MachineOperand *SplatVal,
- const TargetRegisterClass *SplatRC) const;
+ bool tryFoldRegSeqSplat(MachineInstr *UseMI, unsigned UseOpIdx,
+ int64_t SplatVal,
+ const TargetRegisterClass *SplatRC) const;
bool tryToFoldACImm(const FoldableDef &OpToFold, MachineInstr *UseMI,
unsigned UseOpIdx,
@@ -967,15 +967,15 @@ const TargetRegisterClass
*SIFoldOperandsImpl::getRegSeqInit(
return getRegSeqInit(*Def, Defs);
}
-std::pair
+std::pair
SIFoldOperandsImpl::isRegSeqSplat(MachineInstr &RegSeq) const {
SmallVector, 32> Defs;
const TargetRegisterClass *SrcRC = getRegSeqInit(RegSeq, Defs);
if (!SrcRC)
return {};
- // TODO: Recognize 64-bit splats broken into 32-bit pieces (i.e. recognize
- // every other other element is 0 for 64-bit immediates)
+ bool TryToMatchSplat64 = false;
+
int64_t Imm;
for (unsigned I = 0, E = Defs.size(); I != E; ++I) {
const MachineOperand *Op = Defs[I].first;
@@ -987,38 +987,75 @@ SIFoldOperandsImpl::isRegSeqSplat(MachineInstr &RegSeq)
const {
Imm = SubImm;
continue;
}
-if (Imm != SubImm)
+
+if (Imm != SubImm) {
+ if (I == 1 && (E & 1) == 0) {
+// If we have an even number of inputs, there's a chance this is a
+// 64-bit element splat broken into 32-bit pieces.
+TryToMatchSplat64 = true;
+break;
+ }
+
return {}; // Can only fold splat constants
+}
+ }
+
+ if (!TryToMatchSplat64)
+return {Defs[0].first->getImm(), SrcRC};
+
+ // Fallback to recognizing 64-bit splats broken into 32-bit pieces
+ // (i.e. recognize every other other element is 0 for 64-bit immediates)
+ int64_t SplatVal64;
+ for (unsigned I = 0, E = Defs.size(); I != E; I += 2) {
+const MachineOperand *Op0 = Defs[I].first;
+const MachineOperand *Op1 = Defs[I + 1].first;
+
+if (!Op0->isImm() || !Op1->isImm())
+ return {};
+
+unsigned SubReg0 = Defs[I].second;
+unsigned SubReg1 = Defs[I + 1].second;
+
+// Assume we're going to generally encounter reg_sequences with sorted
+// subreg indexes, so reject any that aren't consecutive.
+if (TRI->getChannelFromSubReg(SubReg0) + 1 !=
+TRI->getChannelFromSubReg(SubReg1))
+ return {};
+
+int64_t MergedVal = Make_64(Op1->getImm(), Op0->getImm());
+if (I == 0)
+ SplatVal64 = MergedVal;
+else if (SplatVal64 != MergedVal)
+ return {};
}
- return {Defs[0].first, SrcRC};
+ const TargetRegisterClass *RC64 = TRI->getSubRegisterClass(
+ MRI->getRegClass(RegSeq.getOperand(0).getReg()), AMDGPU::sub0_sub1);
+
+ return {SplatVal64, RC64};
}
-MachineOperand *SIFoldOperandsImpl::tryFoldRegSeqSplat(
-MachineInstr *UseMI, unsigned UseOpIdx, MachineOperand *SplatVal,
+bool SIFoldOperandsImpl::tryFoldRegSeqSplat(
+MachineInstr *UseMI, unsigned UseOpIdx, int64_t SplatVal,
const TargetRegisterClass *SplatRC) const {
const MCInstrDesc &Desc = UseMI->getDesc();
if (UseOpIdx >= Desc.getNumOperands())
-return nullptr;
+return false;
// Filter out unhandled pseudos.
if (!AMDGPU::isSISrcOperand(Desc, UseOpIdx))
-return nullptr;
+return false;
int16_t RCID = Desc.operands()[UseOpIdx].RegClass;
if (RCID == -1)
-return nullptr;
+return false;
+
+ const TargetRegisterClass *OpRC = TRI->getRegClass(RCID);
// Special case 0/-1, since when interpreted as a 64-bit element both halves
- // have the same bits. Effectively this code does not handle 64-bit element
- // operands correctly, as the incoming 64-bit constants are already split
into
- // 32-bit sequence elements.
- //
- // TODO: We should try to figure out how to interpret the reg_sequence as a
- // split 64-bit splat constant, or use 64-bit pseudos for materializing f64
- // constants.
- if (SplatVal->getImm() != 0 && SplatVal->getImm() != -1) {
-
[llvm-branch-commits] [llvm] AMDGPU: Handle folding vector splats of inline split f64 inline immediates (PR #140878)
https://github.com/arsenm created
https://github.com/llvm/llvm-project/pull/140878
Recognize a reg_sequence with 32-bit elements that produce a 64-bit
splat value. This enables folding f64 constants into mfma operands
>From b68d880b7872cd90d3aa79419800cfc505305b76 Mon Sep 17 00:00:00 2001
From: Matt Arsenault
Date: Mon, 19 May 2025 21:51:06 +0200
Subject: [PATCH] AMDGPU: Handle folding vector splats of inline split f64
inline immediates
Recognize a reg_sequence with 32-bit elements that produce a 64-bit
splat value. This enables folding f64 constants into mfma operands
---
llvm/lib/Target/AMDGPU/SIFoldOperands.cpp | 103 --
.../CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx90a.ll | 41 +--
2 files changed, 76 insertions(+), 68 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index eb7fb94e25f5c..70e3974bb22b4 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -227,12 +227,12 @@ class SIFoldOperandsImpl {
getRegSeqInit(SmallVectorImpl> &Defs,
Register UseReg) const;
- std::pair
+ std::pair
isRegSeqSplat(MachineInstr &RegSeg) const;
- MachineOperand *tryFoldRegSeqSplat(MachineInstr *UseMI, unsigned UseOpIdx,
- MachineOperand *SplatVal,
- const TargetRegisterClass *SplatRC) const;
+ bool tryFoldRegSeqSplat(MachineInstr *UseMI, unsigned UseOpIdx,
+ int64_t SplatVal,
+ const TargetRegisterClass *SplatRC) const;
bool tryToFoldACImm(const FoldableDef &OpToFold, MachineInstr *UseMI,
unsigned UseOpIdx,
@@ -967,15 +967,15 @@ const TargetRegisterClass
*SIFoldOperandsImpl::getRegSeqInit(
return getRegSeqInit(*Def, Defs);
}
-std::pair
+std::pair
SIFoldOperandsImpl::isRegSeqSplat(MachineInstr &RegSeq) const {
SmallVector, 32> Defs;
const TargetRegisterClass *SrcRC = getRegSeqInit(RegSeq, Defs);
if (!SrcRC)
return {};
- // TODO: Recognize 64-bit splats broken into 32-bit pieces (i.e. recognize
- // every other other element is 0 for 64-bit immediates)
+ bool TryToMatchSplat64 = false;
+
int64_t Imm;
for (unsigned I = 0, E = Defs.size(); I != E; ++I) {
const MachineOperand *Op = Defs[I].first;
@@ -987,38 +987,75 @@ SIFoldOperandsImpl::isRegSeqSplat(MachineInstr &RegSeq)
const {
Imm = SubImm;
continue;
}
-if (Imm != SubImm)
+
+if (Imm != SubImm) {
+ if (I == 1 && (E & 1) == 0) {
+// If we have an even number of inputs, there's a chance this is a
+// 64-bit element splat broken into 32-bit pieces.
+TryToMatchSplat64 = true;
+break;
+ }
+
return {}; // Can only fold splat constants
+}
+ }
+
+ if (!TryToMatchSplat64)
+return {Defs[0].first->getImm(), SrcRC};
+
+ // Fallback to recognizing 64-bit splats broken into 32-bit pieces
+ // (i.e. recognize every other other element is 0 for 64-bit immediates)
+ int64_t SplatVal64;
+ for (unsigned I = 0, E = Defs.size(); I != E; I += 2) {
+const MachineOperand *Op0 = Defs[I].first;
+const MachineOperand *Op1 = Defs[I + 1].first;
+
+if (!Op0->isImm() || !Op1->isImm())
+ return {};
+
+unsigned SubReg0 = Defs[I].second;
+unsigned SubReg1 = Defs[I + 1].second;
+
+// Assume we're going to generally encounter reg_sequences with sorted
+// subreg indexes, so reject any that aren't consecutive.
+if (TRI->getChannelFromSubReg(SubReg0) + 1 !=
+TRI->getChannelFromSubReg(SubReg1))
+ return {};
+
+int64_t MergedVal = Make_64(Op1->getImm(), Op0->getImm());
+if (I == 0)
+ SplatVal64 = MergedVal;
+else if (SplatVal64 != MergedVal)
+ return {};
}
- return {Defs[0].first, SrcRC};
+ const TargetRegisterClass *RC64 = TRI->getSubRegisterClass(
+ MRI->getRegClass(RegSeq.getOperand(0).getReg()), AMDGPU::sub0_sub1);
+
+ return {SplatVal64, RC64};
}
-MachineOperand *SIFoldOperandsImpl::tryFoldRegSeqSplat(
-MachineInstr *UseMI, unsigned UseOpIdx, MachineOperand *SplatVal,
+bool SIFoldOperandsImpl::tryFoldRegSeqSplat(
+MachineInstr *UseMI, unsigned UseOpIdx, int64_t SplatVal,
const TargetRegisterClass *SplatRC) const {
const MCInstrDesc &Desc = UseMI->getDesc();
if (UseOpIdx >= Desc.getNumOperands())
-return nullptr;
+return false;
// Filter out unhandled pseudos.
if (!AMDGPU::isSISrcOperand(Desc, UseOpIdx))
-return nullptr;
+return false;
int16_t RCID = Desc.operands()[UseOpIdx].RegClass;
if (RCID == -1)
-return nullptr;
+return false;
+
+ const TargetRegisterClass *OpRC = TRI->getRegClass(RCID);
// Special case 0/-1, since when interpreted as a 64-bit element both halves
- // have the same bits. Effectively this code does not handle 64-bit element
- // operands correctly, as the incoming 64-bit constants are alrea
