Generalize r153635 to deal with TokenFactor chains; also clean up the logic and fix the tests. rdar://11069732, rdar://11236106

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@154604 91177308-0d34-0410-b5e6-96231b3b80d8

1 files changed, 51 insertions, 41 deletions

diff --git a/lib/Target/X86/X86ISelDAGToDAG.cpp b/lib/Target/X86/X86ISelDAGToDAG.cpp
index e4246d3f51..8e2b1d6b5d 100644
--- a/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -1852,58 +1852,69 @@ static bool HasNoSignedComparisonUses(SDNode *N) {
 /// is suitable for doing the {load; increment or decrement; store} to modify
 /// transformation.
 static bool isLoadIncOrDecStore(StoreSDNode *StoreNode, unsigned Opc, 
-                                SDValue &StoredVal) {
+                                SDValue StoredVal, SelectionDAG *CurDAG,
+                                LoadSDNode* &LoadNode, SDValue &InputChain) {
 
   // is the value stored the result of a DEC or INC?
   if (!(Opc == X86ISD::DEC || Opc == X86ISD::INC)) return false;
 
-  // is the chain predecessor to the store a load?
-  SDValue Chain = StoreNode->getChain();
-  if (Chain->getOpcode() != ISD::LOAD) return false;
-  
-  // is the size of the value one that we can handle? (i.e. 64, 32, 16, or 8)
-  LoadSDNode *LoadNode = cast<LoadSDNode>(Chain.getNode());
-  EVT LdVT = LoadNode->getMemoryVT();    
-  if (LdVT != MVT::i64 && LdVT != MVT::i32 && LdVT != MVT::i16 && 
-      LdVT != MVT::i8)
-    return false;
-
-  // quick check of whether the store is simple
-  SDValue Undef = StoreNode->getOffset();
-  if (Undef->getOpcode() != ISD::UNDEF) return false;
-
   // is the stored value result 0 of the load?
   if (StoredVal.getResNo() != 0) return false;
 
   // are there other uses of the loaded value than the inc or dec?
   if (!StoredVal.getNode()->hasNUsesOfValue(1, 0)) return false;
 
-  // is there exactly one use of the load?
-  if (!LoadNode->hasNUsesOfValue(1, 0)) return false;
-  
-  // are the load and store connected by the chain?
-  if (StoredVal->getOperand(0).getNode() != LoadNode) return false;
-
-  //OPC_CheckPredicate, 1, // Predicate_nontemporalstore
-  if (StoreNode->isNonTemporal())
+  // is the store non-extending and non-indexed?
+  if (!ISD::isNormalStore(StoreNode) || StoreNode->isNonTemporal())
     return false;
 
-  // is the address of the store the same as the load?
-  SDValue Address = StoreNode->getBasePtr();
-  if (LoadNode->getBasePtr() != Address ||
-      LoadNode->getOffset() != Undef)
-    return false;
+  SDValue Load = StoredVal->getOperand(0);
+  // Is the stored value a non-extending and non-indexed load?
+  if (!ISD::isNormalLoad(Load.getNode())) return false;
 
-  // is the load non-extending and non-indexed?
-  if (!ISD::isNormalLoad(LoadNode))
+  // Return LoadNode by reference.
+  LoadNode = cast<LoadSDNode>(Load);
+  // is the size of the value one that we can handle? (i.e. 64, 32, 16, or 8)
+  EVT LdVT = LoadNode->getMemoryVT();    
+  if (LdVT != MVT::i64 && LdVT != MVT::i32 && LdVT != MVT::i16 && 
+      LdVT != MVT::i8)
     return false;
 
-  // is the store non-extending and non-indexed?
-  if (!ISD::isNormalStore(StoreNode))
+  // Is store the only read of the loaded value?
+  if (!Load.hasOneUse())
     return false;
+  
+  // Is the address of the store the same as the load?
+  if (LoadNode->getBasePtr() != StoreNode->getBasePtr() ||
+      LoadNode->getOffset() != StoreNode->getOffset())
+    return false;
+
+  // Check if the chain is produced by the load or is a TokenFactor with
+  // the load output chain as an operand. Return InputChain by reference.
+  SDValue Chain = StoreNode->getChain();
 
-  // check load chain has only one use (from the store)
-  if (!Chain.hasOneUse())
+  bool ChainCheck = false;
+  if (Chain == Load.getValue(1)) {
+    ChainCheck = true;
+    InputChain = LoadNode->getChain();
+  } else if (Chain.getOpcode() == ISD::TokenFactor) {
+    SmallVector<SDValue, 4> ChainOps;
+    for (unsigned i = 0, e = Chain.getNumOperands(); i != e; ++i) {
+      SDValue Op = Chain.getOperand(i);
+      if (Op == Load.getValue(1)) {
+        ChainCheck = true;
+        continue;
+      }
+      ChainOps.push_back(Op);
+    }
+
+    if (ChainCheck)
+      // Make a new TokenFactor with all the other input chains except
+      // for the load.
+      InputChain = CurDAG->getNode(ISD::TokenFactor, Chain.getDebugLoc(),
+                                   MVT::Other, &ChainOps[0], ChainOps.size());
+  }
+  if (!ChainCheck)
     return false;
 
   return true;
@@ -2455,12 +2466,11 @@ SDNode *X86DAGToDAGISel::Select(SDNode *Node) {
     SDValue StoredVal = StoreNode->getOperand(1);
     unsigned Opc = StoredVal->getOpcode();
 
-    if (!isLoadIncOrDecStore(StoreNode, Opc, StoredVal)) break;
-
-    // Merge the input chains if they are not intra-pattern references.
-    SDValue Chain = StoreNode->getOperand(0);
-    LoadSDNode *LoadNode = cast<LoadSDNode>(Chain.getNode());
-    SDValue InputChain = LoadNode->getOperand(0);
+    LoadSDNode *LoadNode = 0;
+    SDValue InputChain;
+    if (!isLoadIncOrDecStore(StoreNode, Opc, StoredVal, CurDAG,
+                             LoadNode, InputChain))
+      break;
 
     SDValue Base, Scale, Index, Disp, Segment;
     if (!SelectAddr(LoadNode, LoadNode->getBasePtr(),