Skip to content

Commit 619e9bf

Browse files
committed
Revert "Change the way to convert reverse operation When EVL lowering."
This reverts commit 293a127.
1 parent de622a4 commit 619e9bf

File tree

1 file changed

+27
-53
lines changed

1 file changed

+27
-53
lines changed

llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

Lines changed: 27 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -2664,22 +2664,6 @@ static inline RemoveMask_match<Op0_t, Op1_t> m_RemoveMask(const Op0_t &In,
26642664
return RemoveMask_match<Op0_t, Op1_t>(In, Out);
26652665
}
26662666

2667-
/// If \p R is a VPInstruction::Reverse, return a VPWidenIntrinsicRecipe
2668-
/// for the vp.reverse intrinsic using \p EVL. Returns nullptr otherwise.
2669-
static VPWidenIntrinsicRecipe *
2670-
getEVLReverse(VPRecipeBase &R, VPTypeAnalysis &TypeInfo, VPValue &EVL) {
2671-
VPValue *ReversedVal;
2672-
if (!match(&R,
2673-
m_VPInstruction<VPInstruction::Reverse>(m_VPValue(ReversedVal))))
2674-
return nullptr;
2675-
2676-
auto *Reverse = cast<VPInstruction>(&R);
2677-
VPlan *Plan = Reverse->getParent()->getPlan();
2678-
return new VPWidenIntrinsicRecipe(
2679-
Intrinsic::experimental_vp_reverse, {ReversedVal, Plan->getTrue(), &EVL},
2680-
TypeInfo.inferScalarType(Reverse), {}, {}, Reverse->getDebugLoc());
2681-
}
2682-
26832667
/// Try to optimize a \p CurRecipe masked by \p HeaderMask to a corresponding
26842668
/// EVL-based recipe without the header mask. Returns nullptr if no EVL-based
26852669
/// recipe could be created.
@@ -2754,6 +2738,32 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask,
27542738
return nullptr;
27552739
}
27562740

2741+
static void convertToEVLReverse(VPlan &Plan, VPTypeAnalysis &TypeInfo,
2742+
VPValue &EVL) {
2743+
SmallVector<VPRecipeBase *> ToRemove;
2744+
2745+
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
2746+
vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry()))) {
2747+
for (VPRecipeBase &R : make_early_inc_range(reverse(*VPBB))) {
2748+
auto *VPI = dyn_cast<VPInstruction>(&R);
2749+
if (!VPI || VPI->getOpcode() != VPInstruction::Reverse)
2750+
continue;
2751+
2752+
SmallVector<VPValue *> Ops(VPI->operands());
2753+
Ops.append({Plan.getTrue(), &EVL});
2754+
auto *NewReverse = new VPWidenIntrinsicRecipe(
2755+
Intrinsic::experimental_vp_reverse, Ops,
2756+
TypeInfo.inferScalarType(VPI), {}, {}, VPI->getDebugLoc());
2757+
NewReverse->insertBefore(VPI);
2758+
VPI->replaceAllUsesWith(NewReverse);
2759+
ToRemove.push_back(VPI);
2760+
}
2761+
}
2762+
2763+
for (VPRecipeBase *R : ToRemove)
2764+
R->eraseFromParent();
2765+
}
2766+
27572767
/// Replace recipes with their EVL variants.
27582768
static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
27592769
VPTypeAnalysis TypeInfo(Plan);
@@ -2868,44 +2878,8 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
28682878
}
28692879
}
28702880
ToErase.push_back(CurRecipe);
2871-
2872-
// Convert general reverse operations on loaded values and stored values
2873-
// into vp.reverse, when the VPVectorEndPointerRecipe adjusting the access
2874-
// address uses EVL instead of VF.
2875-
// TODO: Extend conversion along the def-use/use-def chain, as reverse
2876-
// operations may be eliminated or moved in the future.
2877-
if (auto *MemR = dyn_cast<VPWidenMemoryRecipe>(EVLRecipe)) {
2878-
if (!match(MemR->getAddr(), m_VecEndPtr(m_VPValue(), m_Specific(&EVL))))
2879-
continue;
2880-
assert(MemR->isReverse() &&
2881-
"Only reverse access uses VPVectorEndPointerRecipe as address");
2882-
2883-
VPRecipeBase *Candidate = nullptr;
2884-
if (auto *LoadR = dyn_cast<VPWidenLoadEVLRecipe>(MemR)) {
2885-
assert(LoadR->getNumUsers() == 1 &&
2886-
"Unexpected user number of reverse load");
2887-
Candidate = cast<VPRecipeBase>(*LoadR->user_begin());
2888-
} else if (auto *StoreR = dyn_cast<VPWidenStoreEVLRecipe>(MemR)) {
2889-
VPValue *StoredVal = StoreR->getStoredValue();
2890-
// Skip if the stored value is not defined in the loop region.
2891-
if (StoredVal->isDefinedOutsideLoopRegions())
2892-
continue;
2893-
Candidate = StoredVal->getDefiningRecipe();
2894-
}
2895-
assert(Candidate && "Must have one reverse operation for reverse access");
2896-
2897-
if (match(Candidate, m_Intrinsic<Intrinsic::experimental_vp_reverse>()))
2898-
continue;
2899-
2900-
VPWidenIntrinsicRecipe *NewReverse =
2901-
getEVLReverse(*Candidate, TypeInfo, EVL);
2902-
assert(NewReverse &&
2903-
"Unable to get an EVL reverse when tail folding by EVL");
2904-
NewReverse->insertBefore(Candidate);
2905-
cast<VPInstruction>(Candidate)->replaceAllUsesWith(NewReverse);
2906-
ToErase.push_back(Candidate);
2907-
}
29082881
}
2882+
convertToEVLReverse(Plan, TypeInfo, EVL);
29092883
// Remove dead EVL mask.
29102884
if (EVLMask->getNumUsers() == 0)
29112885
ToErase.push_back(EVLMask->getDefiningRecipe());

0 commit comments

Comments
 (0)