Split the appoarch that transforms the reverse to vp.reverse for loaded result and stored value

Author: Mel-Chen

llvm/lib/Transforms/Vectorize/VPlan.h

@@ -3492,6 +3492,15 @@ struct VPWidenStoreEVLRecipe final : public VPWidenMemoryRecipe {
     setMask(Mask);
   }
 
+  VPWidenStoreEVLRecipe(VPWidenStoreRecipe &S, VPValue *Addr,
+                        VPValue *StoredVal, VPValue &EVL, VPValue *Mask)
+      : VPWidenMemoryRecipe(VPDef::VPWidenStoreEVLSC, S.getIngredient(),
+                            {Addr, StoredVal, &EVL}, S.isConsecutive(),
+                            S.isReverse(), S, S.getDebugLoc()) {
+    assert(isReverse() && "Only reverse access need to set new stored value");
+    setMask(Mask);
+  }
+
   VP_CLASSOF_IMPL(VPDef::VPWidenStoreEVLSC)
 
   /// Return the address accessed by this recipe.

llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

@@ -2630,22 +2630,6 @@ static inline RemoveMask_match<Op0_t, Op1_t> m_RemoveMask(const Op0_t &In,
   return RemoveMask_match<Op0_t, Op1_t>(In, Out);
 }
 
-/// If \p R is a VPInstruction::Reverse, return a VPWidenIntrinsicRecipe
-/// for the vp.reverse intrinsic using \p EVL. Returns nullptr otherwise.
-static VPWidenIntrinsicRecipe *
-getEVLReverse(VPRecipeBase &R, VPTypeAnalysis &TypeInfo, VPValue &EVL) {
-  VPValue *ReversedVal;
-  if (!match(&R,
-             m_VPInstruction<VPInstruction::Reverse>(m_VPValue(ReversedVal))))
-    return nullptr;
-
-  auto *Reverse = cast<VPInstruction>(&R);
-  VPlan *Plan = Reverse->getParent()->getPlan();
-  return new VPWidenIntrinsicRecipe(
-      Intrinsic::experimental_vp_reverse, {ReversedVal, Plan->getTrue(), &EVL},
-      TypeInfo.inferScalarType(Reverse), {}, {}, Reverse->getDebugLoc());
-}
-
 /// Try to optimize a \p CurRecipe masked by \p HeaderMask to a corresponding
 /// EVL-based recipe without the header mask. Returns nullptr if no EVL-based
 /// recipe could be created.
@@ -2687,12 +2671,31 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask,
     return new VPWidenStoreEVLRecipe(cast<VPWidenStoreRecipe>(CurRecipe), Addr,
                                      EVL, Mask);
 
-  if (match(&CurRecipe, m_MaskedStore(m_VPValue(EndPtr), m_VPValue(),
+  VPValue *StoredVal;
+  if (match(&CurRecipe, m_MaskedStore(m_VPValue(EndPtr), m_VPValue(StoredVal),
                                       m_RemoveMask(HeaderMask, Mask))) &&
       match(EndPtr, m_VecEndPtr(m_VPValue(Addr), m_Specific(&Plan->getVF()))) &&
-      cast<VPWidenStoreRecipe>(CurRecipe).isReverse())
-    return new VPWidenStoreEVLRecipe(cast<VPWidenStoreRecipe>(CurRecipe),
-                                     AdjustEndPtr(EndPtr), EVL, Mask);
+      cast<VPWidenStoreRecipe>(CurRecipe).isReverse()) {
+    auto *StoreR = cast<VPWidenStoreRecipe>(&CurRecipe);
+    // Convert general reverse operations on stored value into vp.reverse.
+    // Skip if the stored value is not defined in the loop region.
+    if (!StoredVal->isDefinedOutsideLoopRegions()) {
+      VPValue *ReversedVal;
+      bool IsReverse = match(StoredVal, m_VPInstruction<VPInstruction::Reverse>(
+                                            m_VPValue(ReversedVal)));
+      assert(IsReverse && "The stored value of reverse store must be defined "
+                          "by a reverse operation");
+      auto *Reverse = cast<VPInstruction>(StoredVal);
+      auto *NewReverse = new VPWidenIntrinsicRecipe(
+          Intrinsic::experimental_vp_reverse,
+          {ReversedVal, Plan->getTrue(), &EVL},
+          TypeInfo.inferScalarType(Reverse), {}, {}, Reverse->getDebugLoc());
+      NewReverse->insertBefore(Reverse);
+      return new VPWidenStoreEVLRecipe(*StoreR, AdjustEndPtr(EndPtr),
+                                       NewReverse, EVL, Mask);
+    }
+    return new VPWidenStoreEVLRecipe(*StoreR, AdjustEndPtr(EndPtr), EVL, Mask);
+  }
 
   if (auto *Rdx = dyn_cast<VPReductionRecipe>(&CurRecipe))
     if (Rdx->isConditional() &&
@@ -2835,41 +2838,32 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
     }
     ToErase.push_back(CurRecipe);
 
-    // Convert general reverse operations on loaded values and stored values
-    // into vp.reverse, when the VPVectorEndPointerRecipe adjusting the access
-    // address uses EVL instead of VF.
-    // TODO: Extend conversion along the def-use/use-def chain, as reverse
-    // operations may be eliminated or moved in the future.
-    if (auto *MemR = dyn_cast<VPWidenMemoryRecipe>(EVLRecipe)) {
-      if (!match(MemR->getAddr(), m_VecEndPtr(m_VPValue(), m_Specific(&EVL))))
+    // Convert general reverse operations on loaded results into vp.reverse,
+    // when the VPVectorEndPointerRecipe adjusting the access address uses EVL
+    // instead of VF.
+    if (auto *LoadR = dyn_cast<VPWidenLoadEVLRecipe>(EVLRecipe)) {
+      if (!match(LoadR->getAddr(), m_VecEndPtr(m_VPValue(), m_Specific(&EVL))))
         continue;
-      assert(MemR->isReverse() &&
+      assert(LoadR->isReverse() &&
              "Only reverse access uses VPVectorEndPointerRecipe as address");
-
-      VPRecipeBase *Candidate = nullptr;
-      if (auto *LoadR = dyn_cast<VPWidenLoadEVLRecipe>(MemR)) {
-        assert(LoadR->getNumUsers() == 1 &&
-               "Unexpected user number of reverse load");
-        Candidate = cast<VPRecipeBase>(*LoadR->user_begin());
-      } else if (auto *StoreR = dyn_cast<VPWidenStoreEVLRecipe>(MemR)) {
-        VPValue *StoredVal = StoreR->getStoredValue();
-        // Skip if the stored value is not defined in the loop region.
-        if (StoredVal->isDefinedOutsideLoopRegions())
-          continue;
-        Candidate = StoredVal->getDefiningRecipe();
-      }
-      assert(Candidate && "Must have one reverse operation for reverse access");
-
-      if (match(Candidate, m_Intrinsic<Intrinsic::experimental_vp_reverse>()))
-        continue;
-
-      VPWidenIntrinsicRecipe *NewReverse =
-          getEVLReverse(*Candidate, TypeInfo, EVL);
-      assert(NewReverse &&
-             "Unable to get an EVL reverse when tail folding by EVL");
-      NewReverse->insertBefore(Candidate);
-      cast<VPInstruction>(Candidate)->replaceAllUsesWith(NewReverse);
-      ToErase.push_back(Candidate);
+      // TODO: Extend conversion along the use-def chain, as reverse operations
+      // may be eliminated or sunk in the future.
+      assert(LoadR->getNumUsers() == 1 &&
+             "Unexpected user number of reverse load");
+      auto *UserR = cast<VPRecipeBase>(*LoadR->user_begin());
+      VPValue *ReversedVal;
+      bool IsReverse = match(UserR, m_VPInstruction<VPInstruction::Reverse>(
+                                        m_VPValue(ReversedVal)));
+      assert(IsReverse && "The defined value of reverse load must be used by a "
+                          "reverse operation");
+      auto *Reverse = cast<VPInstruction>(UserR);
+      auto *NewReverse = new VPWidenIntrinsicRecipe(
+          Intrinsic::experimental_vp_reverse,
+          {ReversedVal, Plan.getTrue(), &EVL},
+          TypeInfo.inferScalarType(Reverse), {}, {}, Reverse->getDebugLoc());
+      NewReverse->insertBefore(Reverse);
+      Reverse->replaceAllUsesWith(NewReverse);
+      ToErase.push_back(Reverse);
     }
   }
   // Remove dead EVL mask.