Change the way to convert reverse operation When EVL lowering.

Author: Mel-Chen

llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

@@ -2630,6 +2630,22 @@ static inline RemoveMask_match<Op0_t, Op1_t> m_RemoveMask(const Op0_t &In,
   return RemoveMask_match<Op0_t, Op1_t>(In, Out);
 }
 
+/// If \p R is a VPInstruction::Reverse, return a VPWidenIntrinsicRecipe
+/// for the vp.reverse intrinsic using \p EVL. Returns nullptr otherwise.
+static VPWidenIntrinsicRecipe *
+getEVLReverse(VPRecipeBase &R, VPTypeAnalysis &TypeInfo, VPValue &EVL) {
+  VPValue *ReversedVal;
+  if (!match(&R,
+             m_VPInstruction<VPInstruction::Reverse>(m_VPValue(ReversedVal))))
+    return nullptr;
+
+  auto *Reverse = cast<VPInstruction>(&R);
+  VPlan *Plan = Reverse->getParent()->getPlan();
+  return new VPWidenIntrinsicRecipe(
+      Intrinsic::experimental_vp_reverse, {ReversedVal, Plan->getTrue(), &EVL},
+      TypeInfo.inferScalarType(Reverse), {}, {}, Reverse->getDebugLoc());
+}
+
 /// Try to optimize a \p CurRecipe masked by \p HeaderMask to a corresponding
 /// EVL-based recipe without the header mask. Returns nullptr if no EVL-based
 /// recipe could be created.
@@ -2704,32 +2720,6 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask,
   return nullptr;
 }
 
-static void convertToEVLReverse(VPlan &Plan, VPTypeAnalysis &TypeInfo,
-                                VPValue &EVL) {
-  SmallVector<VPRecipeBase *> ToRemove;
-
-  for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
-           vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry()))) {
-    for (VPRecipeBase &R : make_early_inc_range(reverse(*VPBB))) {
-      auto *VPI = dyn_cast<VPInstruction>(&R);
-      if (!VPI || VPI->getOpcode() != VPInstruction::Reverse)
-        continue;
-
-      SmallVector<VPValue *> Ops(VPI->operands());
-      Ops.append({Plan.getTrue(), &EVL});
-      auto *NewReverse = new VPWidenIntrinsicRecipe(
-          Intrinsic::experimental_vp_reverse, Ops,
-          TypeInfo.inferScalarType(VPI), {}, {}, VPI->getDebugLoc());
-      NewReverse->insertBefore(VPI);
-      VPI->replaceAllUsesWith(NewReverse);
-      ToRemove.push_back(VPI);
-    }
-  }
-
-  for (VPRecipeBase *R : ToRemove)
-    R->eraseFromParent();
-}
-
 /// Replace recipes with their EVL variants.
 static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
   VPTypeAnalysis TypeInfo(Plan);
@@ -2844,8 +2834,44 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
       }
     }
     ToErase.push_back(CurRecipe);
+
+    // Convert general reverse operations on loaded values and stored values
+    // into vp.reverse, when the VPVectorEndPointerRecipe adjusting the access
+    // address uses EVL instead of VF.
+    // TODO: Extend conversion along the def-use/use-def chain, as reverse
+    // operations may be eliminated or moved in the future.
+    if (auto *MemR = dyn_cast<VPWidenMemoryRecipe>(EVLRecipe)) {
+      if (!match(MemR->getAddr(), m_VecEndPtr(m_VPValue(), m_Specific(&EVL))))
+        continue;
+      assert(MemR->isReverse() &&
+             "Only reverse access uses VPVectorEndPointerRecipe as address");
+
+      VPRecipeBase *Candidate = nullptr;
+      if (auto *LoadR = dyn_cast<VPWidenLoadEVLRecipe>(MemR)) {
+        assert(LoadR->getNumUsers() == 1 &&
+               "Unexpected user number of reverse load");
+        Candidate = cast<VPRecipeBase>(*LoadR->user_begin());
+      } else if (auto *StoreR = dyn_cast<VPWidenStoreEVLRecipe>(MemR)) {
+        VPValue *StoredVal = StoreR->getStoredValue();
+        // Skip if the stored value is not defined in the loop region.
+        if (StoredVal->isDefinedOutsideLoopRegions())
+          continue;
+        Candidate = StoredVal->getDefiningRecipe();
+      }
+      assert(Candidate && "Must have one reverse operation for reverse access");
+
+      if (match(Candidate, m_Intrinsic<Intrinsic::experimental_vp_reverse>()))
+        continue;
+
+      VPWidenIntrinsicRecipe *NewReverse =
+          getEVLReverse(*Candidate, TypeInfo, EVL);
+      assert(NewReverse &&
+             "Unable to get an EVL reverse when tail folding by EVL");
+      NewReverse->insertBefore(Candidate);
+      cast<VPInstruction>(Candidate)->replaceAllUsesWith(NewReverse);
+      ToErase.push_back(Candidate);
+    }
   }
-  convertToEVLReverse(Plan, TypeInfo, EVL);
   // Remove dead EVL mask.
   if (EVLMask->getNumUsers() == 0)
     ToErase.push_back(EVLMask->getDefiningRecipe());