================
@@ -7484,6 +7495,215 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan
&Plan, VFRange &Range,
});
}
+void VPlanTransforms::multiversionForUnitStridedMemOps(
+ VPlan &Plan, VPCostContext &CostCtx, VFSelectionContext &Config,
+ VPRecipeBuilder &RecipeBuilder, VFRange &Range,
+ SmallVectorImpl<VPInstruction *> &MemOps) {
+ ScalarEvolution *SE = CostCtx.PSE.getSE();
+ PredicatedScalarEvolution StrideMVPSE(*SE, const_cast<Loop &>(*CostCtx.L));
+ SCEVUnionPredicate StridePredicates({}, *SE);
+
+ erase_if(MemOps, [&](VPInstruction *VPI) -> bool {
+ if (RecipeBuilder.isConsecutiveWithoutVPlanBasedStrideSpeculation(VPI))
+ return false;
+ auto *PtrOp = VPI->getOpcode() == Instruction::Load ? VPI->getOperand(0)
+ : VPI->getOperand(1);
+
+ const SCEV *PtrSCEV =
+ vputils::getSCEVExprForVPValue(PtrOp, CostCtx.PSE, CostCtx.L);
+ const SCEV *Start, *Stride;
+
+ if (!match(PtrSCEV, m_scev_AffineAddRec(m_SCEV(Start), m_SCEV(Stride),
+ m_SpecificLoop(CostCtx.L))))
+ return false;
+
+ Type *ScalarTy = VPI->getOpcode() == Instruction::Load
+ ? VPI->getScalarType()
+ : VPI->getOperand(0)->getScalarType();
+
+ if (VPI->getMask()) {
+ Instruction *I = VPI->getUnderlyingInstr();
+ if (!LoopVectorizationPlanner::getDecisionAndClampRange(
+ [&](ElementCount VF) -> bool {
+ return Config.isLegalMaskedLoadOrStore(I, VF);
+ },
+ Range))
+ return false;
+ }
+
+ const auto *TypeSize = cast<SCEVConstant>(SE->getSizeOfExpr(
+ Stride->getType(), SE->getDataLayout().getTypeAllocSize(ScalarTy)));
+
+ auto ReplaceWithUnitStrided = [&]() {
+ VPBuilder Builder(VPI);
+ VPValue *StrideInBytes = Plan.getOrAddLiveIn(TypeSize->getValue());
+ auto *VecPtr =
+ new VPVectorPointerRecipe(PtrOp, ScalarTy, StrideInBytes,
+ GEPNoWrapFlags::none(),
VPI->getDebugLoc());
+ Builder.insert(VecPtr);
+ if (VPI->getOpcode() == Instruction::Load) {
+ auto *WideLoad = new VPWidenLoadRecipe(
+ cast<LoadInst>(*VPI->getUnderlyingInstr()), VecPtr, VPI->getMask(),
+ true, *VPI, VPI->getDebugLoc());
+ Builder.insert(WideLoad);
+ VPI->replaceAllUsesWith(WideLoad);
+ } else {
+ auto *WideStore = new VPWidenStoreRecipe(
+ cast<StoreInst>(*VPI->getUnderlyingInstr()), VecPtr,
+ VPI->getOperand(0), VPI->getMask(), true, *VPI,
VPI->getDebugLoc());
+ Builder.insert(WideStore);
+ }
+ VPI->eraseFromParent();
+ };
+
+ if (isa<SCEVConstant>(Stride)) {
+ if (Stride != TypeSize)
+ return false;
+
+ // Earlier MV helped with this memory operation too.
+ ReplaceWithUnitStrided();
+ return true;
+ }
----------------
eas wrote:
```c++
for (int i = 0; i < N; ++i) {
%ld0 = A[i * %stride];
%ld1 = B[i * %stride];
}
```
After processing `%ld0`
```c++
for (int i = 0; i < N; ++i) {
Widen %ld0 = A[i * 1];
%ld1 = B[i * 1]; // Processing this now.
}
```
`@shared_stride` test case goes through this code path.
https://github.com/llvm/llvm-project/pull/182595
_______________________________________________
llvm-branch-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
