R600/structurizer: improve loop handling

Generate more than one loop if it seems to make sense.

This is a candidate for the stable branch.

Signed-off-by: Christian König <christian.koenig@amd.com>
Reviewed-by: Tom Stellard <thomas.stellard@amd.com>

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@175348 91177308-0d34-0410-b5e6-96231b3b80d8

1 files changed, 148 insertions, 196 deletions

diff --git a/lib/Target/R600/AMDGPUStructurizeCFG.cpp b/lib/Target/R600/AMDGPUStructurizeCFG.cpp
index e97e049be5..c2b084afc7 100644
--- a/lib/Target/R600/AMDGPUStructurizeCFG.cpp
+++ b/lib/Target/R600/AMDGPUStructurizeCFG.cpp
@@ -43,6 +43,7 @@ typedef DenseMap<DomTreeNode *, unsigned> DTN2UnsignedMap;
 typedef DenseMap<BasicBlock *, PhiMap> BBPhiMap;
 typedef DenseMap<BasicBlock *, Value *> BBPredicates;
 typedef DenseMap<BasicBlock *, BBPredicates> PredMap;
+typedef DenseMap<BasicBlock *, BasicBlock*> BB2BBMap;
 typedef DenseMap<BasicBlock *, BBVector> BB2BBVecMap;
 
 // The name for newly created blocks.
@@ -175,29 +176,30 @@ class AMDGPUStructurizeCFG : public RegionPass {
 
   RNVector Order;
   BBSet Visited;
-  PredMap Predicates;
+
   BBPhiMap DeletedPhis;
   BB2BBVecMap AddedPhis;
+
+  PredMap Predicates;
   BranchVector Conditions;
 
-  BasicBlock *LoopStart;
-  BasicBlock *LoopEnd;
-  BBSet LoopTargets;
-  BBPredicates LoopPred;
+  BB2BBMap Loops;
+  PredMap LoopPreds;
+  BranchVector LoopConds;
 
-  void orderNodes();
+  RegionNode *PrevNode;
 
-  Value *buildCondition(BranchInst *Term, unsigned Idx, bool Invert);
+  void orderNodes();
 
-  bool analyzeLoopStart(BasicBlock *From, BasicBlock *To, Value *Condition);
+  void analyzeLoops(RegionNode *N);
 
-  void analyzeNode(RegionNode *N);
+  Value *buildCondition(BranchInst *Term, unsigned Idx, bool Invert);
 
-  void analyzeLoopEnd(RegionNode *N);
+  void gatherPredicates(RegionNode *N);
 
   void collectInfos();
 
-  void insertConditions();
+  void insertConditions(bool Loops);
 
   void delPhiValues(BasicBlock *From, BasicBlock *To);
 
@@ -212,17 +214,19 @@ class AMDGPUStructurizeCFG : public RegionPass {
 
   BasicBlock *getNextFlow(BasicBlock *Dominator);
 
-  BasicBlock *needPrefix(RegionNode *&Prev, RegionNode *Node);
+  BasicBlock *needPrefix(bool NeedEmpty);
 
   BasicBlock *needPostfix(BasicBlock *Flow, bool ExitUseAllowed);
 
-  RegionNode *getNextPrev(BasicBlock *Next);
+  void setPrevNode(BasicBlock *BB);
 
   bool dominatesPredicates(BasicBlock *BB, RegionNode *Node);
 
-  bool isPredictableTrue(RegionNode *Who, RegionNode *Where);
+  bool isPredictableTrue(RegionNode *Node);
+
+  void wireFlow(bool ExitUseAllowed, BasicBlock *LoopEnd);
 
-  RegionNode *wireFlow(RegionNode *&Prev, bool ExitUseAllowed);
+  void handleLoops(bool ExitUseAllowed, BasicBlock *LoopEnd);
 
   void createFlow();
 
@@ -278,6 +282,29 @@ void AMDGPUStructurizeCFG::orderNodes() {
   }
 }
 
+/// \brief Determine the end of the loops
+void AMDGPUStructurizeCFG::analyzeLoops(RegionNode *N) {
+
+  if (N->isSubRegion()) {
+    // Test for exit as back edge
+    BasicBlock *Exit = N->getNodeAs<Region>()->getExit();
+    if (Visited.count(Exit))
+      Loops[Exit] = N->getEntry();
+
+  } else {
+    // Test for sucessors as back edge
+    BasicBlock *BB = N->getNodeAs<BasicBlock>();
+    BranchInst *Term = cast<BranchInst>(BB->getTerminator());
+
+    for (unsigned i = 0, e = Term->getNumSuccessors(); i != e; ++i) {
+      BasicBlock *Succ = Term->getSuccessor(i);
+
+      if (Visited.count(Succ))
+        Loops[Succ] = BB;
+    }
+  }
+}
+
 /// \brief Build the condition for one edge
 Value *AMDGPUStructurizeCFG::buildCondition(BranchInst *Term, unsigned Idx,
                                             bool Invert) {
@@ -291,54 +318,20 @@ Value *AMDGPUStructurizeCFG::buildCondition(BranchInst *Term, unsigned Idx,
   return Cond;
 }
 
-/// \brief Analyze the start of a loop and insert predicates as necessary
-bool AMDGPUStructurizeCFG::analyzeLoopStart(BasicBlock *From, BasicBlock *To,
-                                            Value *Condition) {
-  LoopPred[From] = Condition;
-  LoopTargets.insert(To);
-  if (!LoopStart) {
-    LoopStart = To;
-    return true;
-
-  } else if (LoopStart == To)
-    return true;
-
-  // We need to handle the case of intersecting loops, e. g.
-  //
-  //    /----<-----
-  //    |         |
-  // -> A -> B -> C -> D
-  //         |         |
-  //         -----<----/
-
-  RNVector::reverse_iterator OI = Order.rbegin(), OE = Order.rend();
-
-  for (;OI != OE; ++OI)
-    if ((*OI)->getEntry() == LoopStart)
-      break;
-
-  for (;OI != OE && (*OI)->getEntry() != To; ++OI) {
-    BBPredicates &Pred = Predicates[(*OI)->getEntry()];
-    if (!Pred.count(From))
-      Pred[From] = Condition;
-  }
-  return false;
-}
-
 /// \brief Analyze the predecessors of each block and build up predicates
-void AMDGPUStructurizeCFG::analyzeNode(RegionNode *N) {
+void AMDGPUStructurizeCFG::gatherPredicates(RegionNode *N) {
+
   RegionInfo *RI = ParentRegion->getRegionInfo();
   BasicBlock *BB = N->getEntry();
   BBPredicates &Pred = Predicates[BB];
+  BBPredicates &LPred = LoopPreds[BB];
 
   for (pred_iterator PI = pred_begin(BB), PE = pred_end(BB);
        PI != PE; ++PI) {
 
-    if (!ParentRegion->contains(*PI)) {
-      // It's a branch from outside into our region entry
-      Pred[*PI] = BoolTrue;
+    // Ignore it if it's a branch from outside into our region entry
+    if (!ParentRegion->contains(*PI))
       continue;
-    }
 
     Region *R = RI->getRegionFor(*PI);
     if (R == ParentRegion) {
@@ -355,7 +348,7 @@ void AMDGPUStructurizeCFG::analyzeNode(RegionNode *N) {
           if (Term->isConditional()) {
             // Try to treat it like an ELSE block
             BasicBlock *Other = Term->getSuccessor(!i);
-            if (Visited.count(Other) && !LoopTargets.count(Other) &&
+            if (Visited.count(Other) && !Loops.count(Other) &&
                 !Pred.count(Other) && !Pred.count(*PI)) {
 
               Pred[Other] = BoolFalse;
@@ -363,13 +356,12 @@ void AMDGPUStructurizeCFG::analyzeNode(RegionNode *N) {
               continue;
             }
           }
-
+          Pred[*PI] = buildCondition(Term, i, false);
+ 
         } else {
           // Back edge
-          if (analyzeLoopStart(*PI, BB, buildCondition(Term, i, true)))
-            continue;
+          LPred[*PI] = buildCondition(Term, i, true);
         }
-        Pred[*PI] = buildCondition(Term, i, false);
       }
 
     } else {
@@ -383,34 +375,10 @@ void AMDGPUStructurizeCFG::analyzeNode(RegionNode *N) {
         continue;
 
       BasicBlock *Entry = R->getEntry();
-      if (!Visited.count(Entry))
-        if (analyzeLoopStart(Entry, BB, BoolFalse))
-          continue;
-
-      Pred[Entry] = BoolTrue;
-    }
-  }
-}
-
-/// \brief Determine the end of the loop
-void AMDGPUStructurizeCFG::analyzeLoopEnd(RegionNode *N) {
-
-  if (N->isSubRegion()) {
-    // Test for exit as back edge
-    BasicBlock *Exit = N->getNodeAs<Region>()->getExit();
-    if (Visited.count(Exit))
-      LoopEnd = N->getEntry();
-
-  } else {
-    // Test for sucessors as back edge
-    BasicBlock *BB = N->getNodeAs<BasicBlock>();
-    BranchInst *Term = cast<BranchInst>(BB->getTerminator());
-
-    for (unsigned i = 0, e = Term->getNumSuccessors(); i != e; ++i) {
-      BasicBlock *Succ = Term->getSuccessor(i);
-
-      if (Visited.count(Succ))
-        LoopEnd = BB;
+      if (Visited.count(Entry))
+        Pred[Entry] = BoolTrue;
+      else
+        LPred[Entry] = BoolFalse;
     }
   }
 }
@@ -422,9 +390,8 @@ void AMDGPUStructurizeCFG::collectInfos() {
   Predicates.clear();
 
   // and loop infos
-  LoopStart = LoopEnd = 0;
-  LoopTargets.clear();
-  LoopPred.clear();
+  Loops.clear();
+  LoopPreds.clear();
 
   // Reset the visited nodes
   Visited.clear();
@@ -433,42 +400,37 @@ void AMDGPUStructurizeCFG::collectInfos() {
        OI != OE; ++OI) {
 
     // Analyze all the conditions leading to a node
-    analyzeNode(*OI);
+    gatherPredicates(*OI);
 
     // Remember that we've seen this node
     Visited.insert((*OI)->getEntry());
 
-    // Find the last back edge
-    analyzeLoopEnd(*OI);
+    // Find the last back edges
+    analyzeLoops(*OI);
   }
-
-  // Both or neither must be set
-  assert(!LoopStart == !LoopEnd);
 }
 
 /// \brief Insert the missing branch conditions
-void AMDGPUStructurizeCFG::insertConditions() {
+void AMDGPUStructurizeCFG::insertConditions(bool Loops) {
+  BranchVector &Conds = Loops ? LoopConds : Conditions;
+  Value *Default = Loops ? BoolTrue : BoolFalse;
   SSAUpdater PhiInserter;
 
-  for (BranchVector::iterator I = Conditions.begin(),
-       E = Conditions.end(); I != E; ++I) {
+  for (BranchVector::iterator I = Conds.begin(),
+       E = Conds.end(); I != E; ++I) {
 
     BranchInst *Term = *I;
-    BasicBlock *Parent = Term->getParent();
-
     assert(Term->isConditional());
 
-    Value *Default = (Parent == LoopEnd) ? BoolTrue : BoolFalse;
+    BasicBlock *Parent = Term->getParent();
+    BasicBlock *SuccTrue = Term->getSuccessor(0);
+    BasicBlock *SuccFalse = Term->getSuccessor(1);
 
     PhiInserter.Initialize(Boolean, "");
     PhiInserter.AddAvailableValue(&Func->getEntryBlock(), Default);
-    if (Parent == LoopEnd)
-      PhiInserter.AddAvailableValue(LoopStart, BoolTrue);
-    else
-      PhiInserter.AddAvailableValue(Parent, BoolFalse);
+    PhiInserter.AddAvailableValue(Loops ? SuccFalse : Parent, Default);
 
-    BasicBlock *Succ = Term->getSuccessor(0);
-    BBPredicates &Preds = (Parent == LoopEnd) ? LoopPred : Predicates[Succ];
+    BBPredicates &Preds = Loops ? LoopPreds[SuccFalse] : Predicates[SuccTrue];
 
     NearestCommonDominator Dominator(DT);
     Dominator.addBlock(Parent, false);
@@ -648,54 +610,24 @@ BasicBlock *AMDGPUStructurizeCFG::getNextFlow(BasicBlock *Dominator) {
 }
 
 /// \brief Create a new or reuse the previous node as flow node
-BasicBlock *AMDGPUStructurizeCFG::needPrefix(RegionNode *&Prev,
-                                             RegionNode *Node) {
-
-  if (!Prev || Prev->isSubRegion() ||
-      (Node && Node->getEntry() == LoopStart)) {
-
-    // We need to insert a flow node, first figure out the dominator
-    DomTreeNode *Dominator = Prev ? DT->getNode(Prev->getEntry()) : 0;
-    if (!Dominator)
-      Dominator = DT->getNode(Node->getEntry())->getIDom();
-    assert(Dominator && "Illegal loop to function entry");
-
-    // then create the flow node
-    BasicBlock *Flow = getNextFlow(Dominator->getBlock());
-
-    // wire up the new flow
-    if (Prev) {
-      changeExit(Prev, Flow, true);
-    } else {
-      // Parent regions entry needs predicates, create a new region entry
-      BasicBlock *Entry = Node->getEntry();
-      for (pred_iterator I = pred_begin(Entry), E = pred_end(Entry);
-           I != E;) {
-
-        BasicBlock *BB = *(I++);
-        if (ParentRegion->contains(BB))
-          continue;
+BasicBlock *AMDGPUStructurizeCFG::needPrefix(bool NeedEmpty) {
 
-        // Remove PHY values from outside to our entry node
-        delPhiValues(BB, Entry);
+  BasicBlock *Entry = PrevNode->getEntry();
 
-        // Update the branch instructions
-        BB->getTerminator()->replaceUsesOfWith(Entry, Flow);
-      }
+  if (!PrevNode->isSubRegion()) {
+    killTerminator(Entry);
+    if (!NeedEmpty || Entry->getFirstInsertionPt() == Entry->end())
+      return Entry;
 
-      // Populate the region tree with the new entry
-      for (Region *R = ParentRegion; R && R->getEntry() == Entry;
-           R = R->getParent()) {
-        R->replaceEntry(Flow);
-      }
-    }
-    Prev = ParentRegion->getBBNode(Flow);
+  } 
 
-  } else {
-    killTerminator(Prev->getEntry());
-  }
+  // create a new flow node
+  BasicBlock *Flow = getNextFlow(Entry);
 
-  return Prev->getEntry();
+  // and wire it up
+  changeExit(PrevNode, Flow, true);
+  PrevNode = ParentRegion->getBBNode(Flow);
+  return Flow;
 }
 
 /// \brief Returns the region exit if possible, otherwise just a new flow node
@@ -711,9 +643,9 @@ BasicBlock *AMDGPUStructurizeCFG::needPostfix(BasicBlock *Flow,
   return getNextFlow(Flow);
 }
 
-/// \brief Returns the region node for Netx, or null if Next is the exit
-RegionNode *AMDGPUStructurizeCFG::getNextPrev(BasicBlock *Next) {
-  return ParentRegion->contains(Next) ? ParentRegion->getBBNode(Next) : 0;
+/// \brief Set the previous node
+void AMDGPUStructurizeCFG::setPrevNode(BasicBlock *BB) {
+  PrevNode =  ParentRegion->contains(BB) ? ParentRegion->getBBNode(BB) : 0;
 }
 
 /// \brief Does BB dominate all the predicates of Node ?
@@ -729,11 +661,14 @@ bool AMDGPUStructurizeCFG::dominatesPredicates(BasicBlock *BB, RegionNode *Node)
 }
 
 /// \brief Can we predict that this node will always be called?
-bool AMDGPUStructurizeCFG::isPredictableTrue(RegionNode *Who,
-                                             RegionNode *Where) {
+bool AMDGPUStructurizeCFG::isPredictableTrue(RegionNode *Node) {
 
-  BBPredicates &Preds = Predicates[Who->getEntry()];
-  bool Dominated = Where == 0;
+  BBPredicates &Preds = Predicates[Node->getEntry()];
+  bool Dominated = false;
+
+  // Regionentry is always true
+  if (PrevNode == 0)
+    return true;
 
   for (BBPredicates::iterator I = Preds.begin(), E = Preds.end();
        I != E; ++I) {
@@ -741,7 +676,7 @@ bool AMDGPUStructurizeCFG::isPredictableTrue(RegionNode *Who,
     if (I->second != BoolTrue)
       return false;
 
-    if (!Dominated && DT->dominates(I->first, Where->getEntry()))
+    if (!Dominated && DT->dominates(I->first, PrevNode->getEntry()))
       Dominated = true;
   }
 
@@ -750,45 +685,69 @@ bool AMDGPUStructurizeCFG::isPredictableTrue(RegionNode *Who,
 }
 
 /// Take one node from the order vector and wire it up
-RegionNode *AMDGPUStructurizeCFG::wireFlow(RegionNode *&Prev,
-                                           bool ExitUseAllowed) {
+void AMDGPUStructurizeCFG::wireFlow(bool ExitUseAllowed,
+                                    BasicBlock *LoopEnd) {
 
   RegionNode *Node = Order.pop_back_val();
+  Visited.insert(Node->getEntry());
 
-  if (isPredictableTrue(Node, Prev)) {
+  if (isPredictableTrue(Node)) {
     // Just a linear flow
-    if (Prev) {
-      changeExit(Prev, Node->getEntry(), true);
+    if (PrevNode) {
+      changeExit(PrevNode, Node->getEntry(), true);
     }
-    Prev = Node;
+    PrevNode = Node;
 
   } else {
     // Insert extra prefix node (or reuse last one)
-    BasicBlock *Flow = needPrefix(Prev, Node);
-    if (Node->getEntry() == LoopStart)
-      LoopStart = Flow;
+    BasicBlock *Flow = needPrefix(false);
 
     // Insert extra postfix node (or use exit instead)
     BasicBlock *Entry = Node->getEntry();
-    BasicBlock *Next = needPostfix(Flow, ExitUseAllowed && Entry != LoopEnd);
+    BasicBlock *Next = needPostfix(Flow, ExitUseAllowed);
 
     // let it point to entry and next block
     Conditions.push_back(BranchInst::Create(Entry, Next, BoolUndef, Flow));
     addPhiValues(Flow, Entry);
     DT->changeImmediateDominator(Entry, Flow);
 
-    Prev = Node;
-    while (!Order.empty() && Node->getEntry() != LoopEnd &&
-           !LoopTargets.count(Order.back()->getEntry()) &&
+    PrevNode = Node;
+    while (!Order.empty() && !Visited.count(LoopEnd) &&
            dominatesPredicates(Entry, Order.back())) {
-      Node = wireFlow(Prev, false);
+      handleLoops(false, LoopEnd);
     }
 
-    changeExit(Prev, Next, false);
-    Prev = getNextPrev(Next);
+    changeExit(PrevNode, Next, false);
+    setPrevNode(Next);
   }
+}
 
-  return Node;
+void AMDGPUStructurizeCFG::handleLoops(bool ExitUseAllowed,
+                                       BasicBlock *LoopEnd) {
+  RegionNode *Node = Order.back();
+  BasicBlock *LoopStart = Node->getEntry();
+
+  if (!Loops.count(LoopStart)) {
+    wireFlow(ExitUseAllowed, LoopEnd);
+    return;
+  }
+
+  if (!isPredictableTrue(Node))
+    LoopStart = needPrefix(true);
+
+  LoopEnd = Loops[Node->getEntry()];
+  wireFlow(false, LoopEnd);
+  while (!Visited.count(LoopEnd)) {
+    handleLoops(false, LoopEnd);
+  }
+
+  // Create an extra loop end node
+  LoopEnd = needPrefix(false);
+  BasicBlock *Next = needPostfix(LoopEnd, ExitUseAllowed);
+  LoopConds.push_back(BranchInst::Create(Next, LoopStart,
+                                         BoolUndef, LoopEnd));
+  addPhiValues(LoopEnd, LoopStart);
+  setPrevNode(Next);
 }
 
 /// After this function control flow looks like it should be, but
@@ -801,26 +760,17 @@ void AMDGPUStructurizeCFG::createFlow() {
   DeletedPhis.clear();
   AddedPhis.clear();
   Conditions.clear();
+  LoopConds.clear();
 
-  RegionNode *Prev = 0;
-  while (!Order.empty()) {
-
-    RegionNode *Node = wireFlow(Prev, EntryDominatesExit);
-
-    // Create an extra loop end node
-    if (Node->getEntry() == LoopEnd) {
-      LoopEnd = needPrefix(Prev, 0);
-      BasicBlock *Next = needPostfix(LoopEnd, EntryDominatesExit);
+  PrevNode = 0;
+  Visited.clear();
 
-      Conditions.push_back(BranchInst::Create(Next, LoopStart,
-                                              BoolUndef, LoopEnd));
-      addPhiValues(LoopEnd, LoopStart);
-      Prev = getNextPrev(Next);
-    }
+  while (!Order.empty()) {
+    handleLoops(EntryDominatesExit, 0);
   }
 
-  if (Prev)
-    changeExit(Prev, Exit, EntryDominatesExit);
+  if (PrevNode)
+    changeExit(PrevNode, Exit, EntryDominatesExit);
   else
     assert(EntryDominatesExit);
 }
@@ -880,19 +830,21 @@ bool AMDGPUStructurizeCFG::runOnRegion(Region *R, RGPassManager &RGM) {
   orderNodes();
   collectInfos();
   createFlow();
-  insertConditions();
+  insertConditions(false);
+  insertConditions(true);
   setPhiValues();
   rebuildSSA();
 
   // Cleanup
   Order.clear();
   Visited.clear();
-  Predicates.clear();
   DeletedPhis.clear();
   AddedPhis.clear();
+  Predicates.clear();
   Conditions.clear();
-  LoopTargets.clear();
-  LoopPred.clear();
+  Loops.clear();
+  LoopPreds.clear();
+  LoopConds.clear();
 
   return true;
 }