adjust recipe in tryToWidenMemory

Author: Mel-Chen

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

@@ -7734,9 +7734,9 @@ void EpilogueVectorizerEpilogueLoop::printDebugTracesAtEnd() {
   });
 }
 
-VPWidenMemoryRecipe *
-VPRecipeBuilder::tryToWidenMemory(Instruction *I, ArrayRef<VPValue *> Operands,
-                                  VFRange &Range) {
+VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(Instruction *I,
+                                                ArrayRef<VPValue *> Operands,
+                                                VFRange &Range) {
   assert((isa<LoadInst>(I) || isa<StoreInst>(I)) &&
          "Must be called with either a load or store");
 
@@ -7793,14 +7793,30 @@ VPRecipeBuilder::tryToWidenMemory(Instruction *I, ArrayRef<VPValue *> Operands,
     Builder.insert(VectorPtr);
     Ptr = VectorPtr;
   }
-  if (LoadInst *Load = dyn_cast<LoadInst>(I))
-    return new VPWidenLoadRecipe(*Load, Ptr, Mask, Consecutive, Reverse,
-                                 VPIRMetadata(*Load, LVer), I->getDebugLoc());
 
-  StoreInst *Store = cast<StoreInst>(I);
-  return new VPWidenStoreRecipe(*Store, Ptr, Operands[0], Mask, Consecutive,
+  if (auto *Load = dyn_cast<LoadInst>(I)) {
+    auto *LoadR =
+        new VPWidenLoadRecipe(*Load, Ptr, Mask, Consecutive, Reverse,
+                              VPIRMetadata(*Load, LVer), Load->getDebugLoc());
+    if (Reverse) {
+      Builder.insert(LoadR);
+      return new VPInstruction(VPInstruction::Reverse, {LoadR},
+                               LoadR->getDebugLoc());
+    }
+    return LoadR;
+  }
+
+  auto *Store = cast<StoreInst>(I);
+  VPValue *StoredVal = Operands[0];
+  if (Reverse) {
+    auto *ReverseR = new VPInstruction(VPInstruction::Reverse, {StoredVal},
+                                       Store->getDebugLoc());
+    Builder.insert(ReverseR);
+    StoredVal = ReverseR;
+  }
+  return new VPWidenStoreRecipe(*Store, Ptr, StoredVal, Mask, Consecutive,
                                 Reverse, VPIRMetadata(*Store, LVer),
-                                I->getDebugLoc());
+                                Store->getDebugLoc());
 }
 
 /// Creates a VPWidenIntOrFpInductionRecpipe for \p Phi. If needed, it will also
@@ -8878,10 +8894,6 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
   // bring the VPlan to its final state.
   // ---------------------------------------------------------------------------
 
-  // Adjust the result of reverse memory accesses.
-  VPlanTransforms::runPass(VPlanTransforms::adjustRecipesForReverseAccesses,
-                           *Plan);
-
   // Adjust the recipes for any inloop reductions.
   adjustRecipesForReductions(Plan, RecipeBuilder, Range.Start);
 

llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h

@@ -96,9 +96,8 @@ class VPRecipeBuilder {
   /// Check if the load or store instruction \p I should widened for \p
   /// Range.Start and potentially masked. Such instructions are handled by a
   /// recipe that takes an additional VPInstruction for the mask.
-  VPWidenMemoryRecipe *tryToWidenMemory(Instruction *I,
-                                        ArrayRef<VPValue *> Operands,
-                                        VFRange &Range);
+  VPRecipeBase *tryToWidenMemory(Instruction *I, ArrayRef<VPValue *> Operands,
+                                 VFRange &Range);
 
   /// Check if an induction recipe should be constructed for \p Phi. If so build
   /// and return it. If not, return null.

llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

@@ -3557,34 +3557,3 @@ void VPlanTransforms::addBranchWeightToMiddleTerminator(
       MDB.createBranchWeights({1, VectorStep - 1}, /*IsExpected=*/false);
   MiddleTerm->addMetadata(LLVMContext::MD_prof, BranchWeights);
 }
-
-void VPlanTransforms::adjustRecipesForReverseAccesses(VPlan &Plan) {
-  if (Plan.hasScalarVFOnly())
-    return;
-
-  for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
-           vp_depth_first_deep(Plan.getVectorLoopRegion()))) {
-    for (VPRecipeBase &R : *VPBB) {
-      auto *MemR = dyn_cast<VPWidenMemoryRecipe>(&R);
-      if (!MemR || !MemR->isReverse())
-        continue;
-
-      if (auto *L = dyn_cast<VPWidenLoadRecipe>(MemR)) {
-        auto *Reverse =
-            new VPInstruction(VPInstruction::Reverse, {L}, L->getDebugLoc());
-        Reverse->insertAfter(L);
-        L->replaceAllUsesWith(Reverse);
-        Reverse->setOperand(0, L);
-        continue;
-      }
-
-      if (auto *S = dyn_cast<VPWidenStoreRecipe>(MemR)) {
-        VPValue *StoredVal = S->getStoredValue();
-        auto *Reverse = new VPInstruction(VPInstruction::Reverse, {StoredVal},
-                                          S->getDebugLoc());
-        Reverse->insertBefore(S);
-        S->setOperand(1, Reverse);
-      }
-    }
-  }
-}

llvm/lib/Transforms/Vectorize/VPlanTransforms.h

@@ -284,20 +284,6 @@ struct VPlanTransforms {
   static void
   addBranchWeightToMiddleTerminator(VPlan &Plan, ElementCount VF,
                                     std::optional<unsigned> VScaleForTuning);
-
-  /// Add reverse recipes for reverse memory accesses.
-  /// For reverse loads, transform
-  ///   WIDEN ir<%L> = load vp<%addr>
-  /// into
-  ///   WIDEN ir<%L> = load vp<%addr>
-  ///   EMIT   vp<%RevL> = reverse ir<%L>
-  ///
-  /// For reverse stores, transform
-  ///   WIDEN store vp<%addr>, ir<%SVal>
-  /// into
-  ///   EMIT   vp<%RevS> = reverse ir<%SVal>
-  ///   WIDEN  store vp<%addr>, vp<%RevS>
-  static void adjustRecipesForReverseAccesses(VPlan &Plan);
 };
 
 } // namespace llvm