@@ -2664,22 +2664,6 @@ static inline RemoveMask_match<Op0_t, Op1_t> m_RemoveMask(const Op0_t &In,
26642664 return RemoveMask_match<Op0_t, Op1_t>(In, Out);
26652665}
26662666
2667- // / If \p R is a VPInstruction::Reverse, return a VPWidenIntrinsicRecipe
2668- // / for the vp.reverse intrinsic using \p EVL. Returns nullptr otherwise.
2669- static VPWidenIntrinsicRecipe *
2670- getEVLReverse (VPRecipeBase &R, VPTypeAnalysis &TypeInfo, VPValue &EVL) {
2671- VPValue *ReversedVal;
2672- if (!match (&R,
2673- m_VPInstruction<VPInstruction::Reverse>(m_VPValue (ReversedVal))))
2674- return nullptr ;
2675-
2676- auto *Reverse = cast<VPInstruction>(&R);
2677- VPlan *Plan = Reverse->getParent ()->getPlan ();
2678- return new VPWidenIntrinsicRecipe (
2679- Intrinsic::experimental_vp_reverse, {ReversedVal, Plan->getTrue (), &EVL},
2680- TypeInfo.inferScalarType (Reverse), {}, {}, Reverse->getDebugLoc ());
2681- }
2682-
26832667// / Try to optimize a \p CurRecipe masked by \p HeaderMask to a corresponding
26842668// / EVL-based recipe without the header mask. Returns nullptr if no EVL-based
26852669// / recipe could be created.
@@ -2754,6 +2738,32 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask,
27542738 return nullptr ;
27552739}
27562740
2741+ static void convertToEVLReverse (VPlan &Plan, VPTypeAnalysis &TypeInfo,
2742+ VPValue &EVL) {
2743+ SmallVector<VPRecipeBase *> ToRemove;
2744+
2745+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
2746+ vp_depth_first_shallow (Plan.getVectorLoopRegion ()->getEntry ()))) {
2747+ for (VPRecipeBase &R : make_early_inc_range (reverse (*VPBB))) {
2748+ auto *VPI = dyn_cast<VPInstruction>(&R);
2749+ if (!VPI || VPI->getOpcode () != VPInstruction::Reverse)
2750+ continue ;
2751+
2752+ SmallVector<VPValue *> Ops (VPI->operands ());
2753+ Ops.append ({Plan.getTrue (), &EVL});
2754+ auto *NewReverse = new VPWidenIntrinsicRecipe (
2755+ Intrinsic::experimental_vp_reverse, Ops,
2756+ TypeInfo.inferScalarType (VPI), {}, {}, VPI->getDebugLoc ());
2757+ NewReverse->insertBefore (VPI);
2758+ VPI->replaceAllUsesWith (NewReverse);
2759+ ToRemove.push_back (VPI);
2760+ }
2761+ }
2762+
2763+ for (VPRecipeBase *R : ToRemove)
2764+ R->eraseFromParent ();
2765+ }
2766+
27572767// / Replace recipes with their EVL variants.
27582768static void transformRecipestoEVLRecipes (VPlan &Plan, VPValue &EVL) {
27592769 VPTypeAnalysis TypeInfo (Plan);
@@ -2868,44 +2878,8 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
28682878 }
28692879 }
28702880 ToErase.push_back (CurRecipe);
2871-
2872- // Convert general reverse operations on loaded values and stored values
2873- // into vp.reverse, when the VPVectorEndPointerRecipe adjusting the access
2874- // address uses EVL instead of VF.
2875- // TODO: Extend conversion along the def-use/use-def chain, as reverse
2876- // operations may be eliminated or moved in the future.
2877- if (auto *MemR = dyn_cast<VPWidenMemoryRecipe>(EVLRecipe)) {
2878- if (!match (MemR->getAddr (), m_VecEndPtr (m_VPValue (), m_Specific (&EVL))))
2879- continue ;
2880- assert (MemR->isReverse () &&
2881- " Only reverse access uses VPVectorEndPointerRecipe as address" );
2882-
2883- VPRecipeBase *Candidate = nullptr ;
2884- if (auto *LoadR = dyn_cast<VPWidenLoadEVLRecipe>(MemR)) {
2885- assert (LoadR->getNumUsers () == 1 &&
2886- " Unexpected user number of reverse load" );
2887- Candidate = cast<VPRecipeBase>(*LoadR->user_begin ());
2888- } else if (auto *StoreR = dyn_cast<VPWidenStoreEVLRecipe>(MemR)) {
2889- VPValue *StoredVal = StoreR->getStoredValue ();
2890- // Skip if the stored value is not defined in the loop region.
2891- if (StoredVal->isDefinedOutsideLoopRegions ())
2892- continue ;
2893- Candidate = StoredVal->getDefiningRecipe ();
2894- }
2895- assert (Candidate && " Must have one reverse operation for reverse access" );
2896-
2897- if (match (Candidate, m_Intrinsic<Intrinsic::experimental_vp_reverse>()))
2898- continue ;
2899-
2900- VPWidenIntrinsicRecipe *NewReverse =
2901- getEVLReverse (*Candidate, TypeInfo, EVL);
2902- assert (NewReverse &&
2903- " Unable to get an EVL reverse when tail folding by EVL" );
2904- NewReverse->insertBefore (Candidate);
2905- cast<VPInstruction>(Candidate)->replaceAllUsesWith (NewReverse);
2906- ToErase.push_back (Candidate);
2907- }
29082881 }
2882+ convertToEVLReverse (Plan, TypeInfo, EVL);
29092883 // Remove dead EVL mask.
29102884 if (EVLMask->getNumUsers () == 0 )
29112885 ToErase.push_back (EVLMask->getDefiningRecipe ());
0 commit comments