================
@@ -7309,6 +7331,231 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan
&Plan, VFRange &Range,
});
}
+void VPlanTransforms::multiversionForUnitStridedMemOps(
+ VPlan &Plan, VPCostContext &CostCtx, VPRecipeBuilder &RecipeBuilder,
+ VFRange &Range, SmallVectorImpl<VPInstruction *> &MemOps) {
+ if (CostCtx.L->getHeader()->getParent()->hasOptSize())
+ return;
+ SmallVector<VPInstruction *> RemainingOps;
+
+ ScalarEvolution *SE = CostCtx.PSE.getSE();
+
+ PredicatedScalarEvolution StrideMVPSE(*SE, const_cast<Loop &>(*CostCtx.L));
+
+ SCEVUnionPredicate StridePredicates({}, *SE);
+
+ // Use `for_each` so that we could do `return Skip();`.
+ for_each(MemOps, [&](VPInstruction *VPI) {
+ auto Skip = [&]() { RemainingOps.push_back(VPI); };
+ if (RecipeBuilder.isConsecutiveWithoutVPlanBasedStrideSpeculation(VPI))
+ return Skip();
+ auto *PtrOp = VPI->getOpcode() == Instruction::Load ? VPI->getOperand(0)
+ : VPI->getOperand(1);
+
+ const SCEV *PtrSCEV =
+ vputils::getSCEVExprForVPValue(PtrOp, CostCtx.PSE, CostCtx.L);
+ const SCEV *Start = nullptr;
+ const SCEV *Stride = nullptr;
+
+ if (!match(PtrSCEV, m_scev_AffineAddRec(m_SCEV(Start), m_SCEV(Stride),
+ m_SpecificLoop(CostCtx.L)))) {
+ return Skip();
+ }
+
+ Type *ScalarTy = VPI->getOpcode() == Instruction::Load
+ ? VPI->getScalarType()
+ : VPI->getOperand(0)->getScalarType();
+
+ if (VPI->getMask()) {
+ auto &TTI = CostCtx.TTI;
+ Instruction *I = VPI->getUnderlyingInstr();
+ unsigned AS = getLoadStoreAddressSpace(I);
+ const Align Alignment = getLoadStoreAlignment(I);
+ if (!LoopVectorizationPlanner::getDecisionAndClampRange(
+ [&](ElementCount VF) -> bool {
+ Type *VTy = VectorType::get(ScalarTy, VF);
+ return VPI->getOpcode() == Instruction::Load
+ ? (TTI.isLegalMaskedLoad(VTy, Alignment, AS) ||
+ TTI.isLegalMaskedGather(VTy, Alignment))
+ : (TTI.isLegalMaskedStore(VTy, Alignment, AS) ||
+ TTI.isLegalMaskedScatter(VTy, Alignment));
----------------
eas wrote:
Done, although I'm not 100% sure about it.
1) That's the first usage of `VFSelectionContext` in `VPlanTransforms.cpp` and
the comment at
https://github.com/llvm/llvm-project/blob/727ce6e036e7def74f14244107db2db1acc03429/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h#L614-L616
isn't very clear about its intent to me.
2) We're speculating unit-strideness, so I switched to
`isLegalMaskedLoadOrStore` only, without gather/scatter. However, an argument
can be made that if some accesses are unmasked, and the rest remain as
gather/scatter, that's still a legal transformation. I think that part can be
handled later in future PRs though.
https://github.com/llvm/llvm-project/pull/182595
_______________________________________________
llvm-branch-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
