Merge commit 'c723eb1aef817d47feec620933ee1ec6005cdd14'

This merges r159618 from upstream into master. It goes with clang rev
af50aab0c317462129d73ae8000c6394c718598d

Conflicts:
	include/llvm/CodeGen/LexicalScopes.h
	include/llvm/Target/TargetOptions.h
	lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
	lib/Target/ARM/ARMBaseInstrInfo.cpp
	lib/Target/ARM/ARMTargetMachine.cpp
	lib/Target/ARM/ARMTargetObjectFile.cpp
	lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp
	lib/Target/Mips/MipsISelDAGToDAG.cpp
	lib/Target/Mips/MipsInstrFPU.td
	lib/Target/Mips/MipsMCInstLower.cpp
	lib/Target/Mips/MipsTargetMachine.cpp
	lib/Target/TargetMachine.cpp
	lib/Target/X86/X86ISelLowering.cpp
	lib/Target/X86/X86RegisterInfo.cpp
	lib/Target/X86/X86TargetObjectFile.cpp
	lib/Target/X86/X86TargetObjectFile.h
	tools/llc/llc.cpp

(tools/llc/llc.cpp is from a merged version of r160532 because it was a bit
hairy and I didn't want to redo it.)

1 files changed, 56 insertions, 17 deletions

diff --git a/lib/CodeGen/MachineScheduler.cpp b/lib/CodeGen/MachineScheduler.cpp
index 1783cbe21e..847bf1e76e 100644
--- a/lib/CodeGen/MachineScheduler.cpp
+++ b/lib/CodeGen/MachineScheduler.cpp
@@ -351,15 +351,21 @@ class ScheduleDAGMI : public ScheduleDAGInstrs {
   IntervalPressure BotPressure;
   RegPressureTracker BotRPTracker;
 
+#ifndef NDEBUG
   /// The number of instructions scheduled so far. Used to cut off the
   /// scheduler at the point determined by misched-cutoff.
   unsigned NumInstrsScheduled;
+#endif
 public:
   ScheduleDAGMI(MachineSchedContext *C, MachineSchedStrategy *S):
     ScheduleDAGInstrs(*C->MF, *C->MLI, *C->MDT, /*IsPostRA=*/false, C->LIS),
     AA(C->AA), RegClassInfo(C->RegClassInfo), SchedImpl(S),
     RPTracker(RegPressure), CurrentTop(), TopRPTracker(TopPressure),
-    CurrentBottom(), BotRPTracker(BotPressure), NumInstrsScheduled(0) {}
+    CurrentBottom(), BotRPTracker(BotPressure) {
+#ifndef NDEBUG
+    NumInstrsScheduled = 0;
+#endif
+  }
 
   ~ScheduleDAGMI() {
     delete SchedImpl;
@@ -396,11 +402,17 @@ public:
   }
 
   /// getIssueWidth - Return the max instructions per scheduling group.
-  ///
   unsigned getIssueWidth() const {
     return InstrItins ? InstrItins->Props.IssueWidth : 1;
   }
 
+  /// getNumMicroOps - Return the number of issue slots required for this MI.
+  unsigned getNumMicroOps(MachineInstr *MI) const {
+    if (!InstrItins) return 1;
+    int UOps = InstrItins->getNumMicroOps(MI->getDesc().getSchedClass());
+    return (UOps >= 0) ? UOps : TII->getNumMicroOps(InstrItins, MI);
+  }
+
 protected:
   void initRegPressure();
   void updateScheduledPressure(std::vector<unsigned> NewMaxPressure);
@@ -782,6 +794,8 @@ class ConvergingScheduler : public MachineSchedStrategy {
   /// current cycle in whichever direction at has moved, and maintains the state
   /// of "hazards" and other interlocks at the current cycle.
   struct SchedBoundary {
+    ScheduleDAGMI *DAG;
+
     ReadyQueue Available;
     ReadyQueue Pending;
     bool CheckPending;
@@ -800,7 +814,7 @@ class ConvergingScheduler : public MachineSchedStrategy {
     /// Pending queues extend the ready queues with the same ID and the
     /// PendingFlag set.
     SchedBoundary(unsigned ID, const Twine &Name):
-      Available(ID, Name+".A"),
+      DAG(0), Available(ID, Name+".A"),
       Pending(ID << ConvergingScheduler::LogMaxQID, Name+".P"),
       CheckPending(false), HazardRec(0), CurrCycle(0), IssueCount(0),
       MinReadyCycle(UINT_MAX), MaxMinLatency(0) {}
@@ -811,11 +825,13 @@ class ConvergingScheduler : public MachineSchedStrategy {
       return Available.getID() == ConvergingScheduler::TopQID;
     }
 
+    bool checkHazard(SUnit *SU);
+
     void releaseNode(SUnit *SU, unsigned ReadyCycle);
 
     void bumpCycle();
 
-    void bumpNode(SUnit *SU, unsigned IssueWidth);
+    void bumpNode(SUnit *SU);
 
     void releasePending();
 
@@ -868,6 +884,8 @@ protected:
 void ConvergingScheduler::initialize(ScheduleDAGMI *dag) {
   DAG = dag;
   TRI = DAG->TRI;
+  Top.DAG = dag;
+  Bot.DAG = dag;
 
   // Initialize the HazardRecognizers.
   const TargetMachine &TM = DAG->MF.getTarget();
@@ -917,6 +935,29 @@ void ConvergingScheduler::releaseBottomNode(SUnit *SU) {
   Bot.releaseNode(SU, SU->BotReadyCycle);
 }
 
+/// Does this SU have a hazard within the current instruction group.
+///
+/// The scheduler supports two modes of hazard recognition. The first is the
+/// ScheduleHazardRecognizer API. It is a fully general hazard recognizer that
+/// supports highly complicated in-order reservation tables
+/// (ScoreboardHazardRecognizer) and arbitraty target-specific logic.
+///
+/// The second is a streamlined mechanism that checks for hazards based on
+/// simple counters that the scheduler itself maintains. It explicitly checks
+/// for instruction dispatch limitations, including the number of micro-ops that
+/// can dispatch per cycle.
+///
+/// TODO: Also check whether the SU must start a new group.
+bool ConvergingScheduler::SchedBoundary::checkHazard(SUnit *SU) {
+  if (HazardRec->isEnabled())
+    return HazardRec->getHazardType(SU) != ScheduleHazardRecognizer::NoHazard;
+
+  if (IssueCount + DAG->getNumMicroOps(SU->getInstr()) > DAG->getIssueWidth())
+    return true;
+
+  return false;
+}
+
 void ConvergingScheduler::SchedBoundary::releaseNode(SUnit *SU,
                                                      unsigned ReadyCycle) {
   if (ReadyCycle < MinReadyCycle)
@@ -924,9 +965,7 @@ void ConvergingScheduler::SchedBoundary::releaseNode(SUnit *SU,
 
   // Check for interlocks first. For the purpose of other heuristics, an
   // instruction that cannot issue appears as if it's not in the ReadyQueue.
-  if (ReadyCycle > CurrCycle
-      || (HazardRec->isEnabled() && (HazardRec->getHazardType(SU)
-                                     != ScheduleHazardRecognizer::NoHazard)))
+  if (ReadyCycle > CurrCycle || checkHazard(SU))
     Pending.push(SU);
   else
     Available.push(SU);
@@ -934,7 +973,8 @@ void ConvergingScheduler::SchedBoundary::releaseNode(SUnit *SU,
 
 /// Move the boundary of scheduled code by one cycle.
 void ConvergingScheduler::SchedBoundary::bumpCycle() {
-  IssueCount = 0;
+  unsigned Width = DAG->getIssueWidth();
+  IssueCount = (IssueCount <= Width) ? 0 : IssueCount - Width;
 
   assert(MinReadyCycle < UINT_MAX && "MinReadyCycle uninitialized");
   unsigned NextCycle = std::max(CurrCycle + 1, MinReadyCycle);
@@ -959,8 +999,7 @@ void ConvergingScheduler::SchedBoundary::bumpCycle() {
 }
 
 /// Move the boundary of scheduled code by one SUnit.
-void ConvergingScheduler::SchedBoundary::bumpNode(SUnit *SU,
-                                                  unsigned IssueWidth) {
+void ConvergingScheduler::SchedBoundary::bumpNode(SUnit *SU) {
   // Update the reservation table.
   if (HazardRec->isEnabled()) {
     if (!isTop() && SU->isCall) {
@@ -970,9 +1009,10 @@ void ConvergingScheduler::SchedBoundary::bumpNode(SUnit *SU,
     }
     HazardRec->EmitInstruction(SU);
   }
-  // Check the instruction group size limit.
-  ++IssueCount;
-  if (IssueCount == IssueWidth) {
+  // Check the instruction group dispatch limit.
+  // TODO: Check if this SU must end a dispatch group.
+  IssueCount += DAG->getNumMicroOps(SU->getInstr());
+  if (IssueCount >= DAG->getIssueWidth()) {
     DEBUG(dbgs() << "*** Max instrs at cycle " << CurrCycle << '\n');
     bumpCycle();
   }
@@ -997,8 +1037,7 @@ void ConvergingScheduler::SchedBoundary::releasePending() {
     if (ReadyCycle > CurrCycle)
       continue;
 
-    if (HazardRec->isEnabled()
-        && HazardRec->getHazardType(SU) != ScheduleHazardRecognizer::NoHazard)
+    if (checkHazard(SU))
       continue;
 
     Available.push(SU);
@@ -1271,11 +1310,11 @@ SUnit *ConvergingScheduler::pickNode(bool &IsTopNode) {
 void ConvergingScheduler::schedNode(SUnit *SU, bool IsTopNode) {
   if (IsTopNode) {
     SU->TopReadyCycle = Top.CurrCycle;
-    Top.bumpNode(SU, DAG->getIssueWidth());
+    Top.bumpNode(SU);
   }
   else {
     SU->BotReadyCycle = Bot.CurrCycle;
-    Bot.bumpNode(SU, DAG->getIssueWidth());
+    Bot.bumpNode(SU);
   }
 }