1 files changed, 189 insertions, 56 deletions

diff --git a/lib/CodeGen/RegAllocLinearScan.cpp b/lib/CodeGen/RegAllocLinearScan.cpp
index 26af53ef83..ba2d3aad9e 100644
--- a/lib/CodeGen/RegAllocLinearScan.cpp
+++ b/lib/CodeGen/RegAllocLinearScan.cpp
@@ -45,6 +45,7 @@ using namespace llvm;
 STATISTIC(NumIters     , "Number of iterations performed");
 STATISTIC(NumBacktracks, "Number of times we had to backtrack");
 STATISTIC(NumCoalesce,   "Number of copies coalesced");
+STATISTIC(NumDowngrade,  "Number of registers downgraded");
 
 static cl::opt<bool>
 NewHeuristic("new-spilling-heuristic",
@@ -74,6 +75,19 @@ namespace {
     EquivalenceClasses<const TargetRegisterClass*> RelatedRegClasses;
     DenseMap<unsigned, const TargetRegisterClass*> OneClassForEachPhysReg;
 
+    // NextReloadMap - For each register in the map, it maps to the another
+    // register which is defined by a reload from the same stack slot and
+    // both reloads are in the same basic block.
+    DenseMap<unsigned, unsigned> NextReloadMap;
+
+    // DowngradedRegs - A set of registers which are being "downgraded", i.e.
+    // un-favored for allocation.
+    SmallSet<unsigned, 8> DowngradedRegs;
+
+    // DowngradeMap - A map from virtual registers to physical registers being
+    // downgraded for the virtual registers.
+    DenseMap<unsigned, unsigned> DowngradeMap;
+
     MachineFunction* mf_;
     MachineRegisterInfo* mri_;
     const TargetMachine* tm_;
@@ -151,6 +165,16 @@ namespace {
     /// ones to the active list.
     void processInactiveIntervals(unsigned CurPoint);
 
+    /// hasNextReloadInterval - Return the next liveinterval that's being
+    /// defined by a reload from the same SS as the specified one.
+    LiveInterval *hasNextReloadInterval(LiveInterval *cur);
+
+    /// DowngradeRegister - Downgrade a register for allocation.
+    void DowngradeRegister(LiveInterval *li, unsigned Reg);
+
+    /// UpgradeRegister - Upgrade a register for allocation.
+    void UpgradeRegister(unsigned Reg);
+
     /// assignRegOrStackSlotAtInterval - assign a register if one
     /// is available, or spill.
     void assignRegOrStackSlotAtInterval(LiveInterval* cur);
@@ -184,6 +208,10 @@ namespace {
     /// getFreePhysReg - return a free physical register for this virtual
     /// register interval if we have one, otherwise return 0.
     unsigned getFreePhysReg(LiveInterval* cur);
+    unsigned getFreePhysReg(const TargetRegisterClass *RC,
+                            unsigned MaxInactiveCount,
+                            SmallVector<unsigned, 256> &inactiveCounts,
+                            bool SkipDGRegs);
 
     /// assignVirt2StackSlot - assigns this virtual register to a
     /// stack slot. returns the stack slot
@@ -211,17 +239,15 @@ static RegisterPass<RALinScan>
 X("linearscan-regalloc", "Linear Scan Register Allocator");
 
 void RALinScan::ComputeRelatedRegClasses() {
-  const TargetRegisterInfo &TRI = *tri_;
-  
   // First pass, add all reg classes to the union, and determine at least one
   // reg class that each register is in.
   bool HasAliases = false;
-  for (TargetRegisterInfo::regclass_iterator RCI = TRI.regclass_begin(),
-       E = TRI.regclass_end(); RCI != E; ++RCI) {
+  for (TargetRegisterInfo::regclass_iterator RCI = tri_->regclass_begin(),
+       E = tri_->regclass_end(); RCI != E; ++RCI) {
     RelatedRegClasses.insert(*RCI);
     for (TargetRegisterClass::iterator I = (*RCI)->begin(), E = (*RCI)->end();
          I != E; ++I) {
-      HasAliases = HasAliases || *TRI.getAliasSet(*I) != 0;
+      HasAliases = HasAliases || *tri_->getAliasSet(*I) != 0;
       
       const TargetRegisterClass *&PRC = OneClassForEachPhysReg[*I];
       if (PRC) {
@@ -241,7 +267,7 @@ void RALinScan::ComputeRelatedRegClasses() {
     for (DenseMap<unsigned, const TargetRegisterClass*>::iterator
          I = OneClassForEachPhysReg.begin(), E = OneClassForEachPhysReg.end();
          I != E; ++I)
-      for (const unsigned *AS = TRI.getAliasSet(I->first); *AS; ++AS)
+      for (const unsigned *AS = tri_->getAliasSet(I->first); *AS; ++AS)
         RelatedRegClasses.unionSets(I->second, OneClassForEachPhysReg[*AS]);
 }
 
@@ -326,6 +352,9 @@ bool RALinScan::runOnMachineFunction(MachineFunction &fn) {
   active_.clear();
   inactive_.clear();
   handled_.clear();
+  NextReloadMap.clear();
+  DowngradedRegs.clear();
+  DowngradeMap.clear();
 
   return true;
 }
@@ -525,6 +554,9 @@ void RALinScan::updateSpillWeights(std::vector<float> &Weights,
   SmallSet<unsigned, 4> Processed;
   SmallSet<unsigned, 4> SuperAdded;
   SmallVector<unsigned, 4> Supers;
+  // Unfavor downgraded registers for spilling.
+  if (DowngradedRegs.count(reg))
+    weight *= 2.0f;
   Weights[reg] += weight;
   Processed.insert(reg);
   for (const unsigned* as = tri_->getAliasSet(reg); *as; ++as) {
@@ -692,6 +724,43 @@ static bool weightsAreClose(float w1, float w2) {
   return (diff / w2) <= 0.05f;  // Within 5%.
 }
 
+LiveInterval *RALinScan::hasNextReloadInterval(LiveInterval *cur) {
+  DenseMap<unsigned, unsigned>::iterator I = NextReloadMap.find(cur->reg);
+  if (I == NextReloadMap.end())
+    return 0;
+  return &li_->getInterval(I->second);
+}
+
+void RALinScan::DowngradeRegister(LiveInterval *li, unsigned Reg) {
+  bool isNew = DowngradedRegs.insert(Reg);
+  isNew = isNew; // Silence compiler warning.
+  assert(isNew && "Multiple reloads holding the same register?");
+  DowngradeMap.insert(std::make_pair(li->reg, Reg));
+  for (const unsigned *AS = tri_->getAliasSet(Reg); *AS; ++AS) {
+    isNew = DowngradedRegs.insert(*AS);
+    isNew = isNew; // Silence compiler warning.
+    assert(isNew && "Multiple reloads holding the same register?");
+    DowngradeMap.insert(std::make_pair(li->reg, *AS));
+  }
+  ++NumDowngrade;
+}
+
+void RALinScan::UpgradeRegister(unsigned Reg) {
+  if (Reg) {
+    DowngradedRegs.erase(Reg);
+    for (const unsigned *AS = tri_->getAliasSet(Reg); *AS; ++AS)
+      DowngradedRegs.erase(*AS);
+  }
+}
+
+namespace {
+  struct LISorter {
+    bool operator()(LiveInterval* A, LiveInterval* B) {
+      return A->beginNumber() < B->beginNumber();
+    }
+  };
+}
+
 /// assignRegOrStackSlotAtInterval - assign a register if one is available, or
 /// spill.
 void RALinScan::assignRegOrStackSlotAtInterval(LiveInterval* cur)
@@ -836,6 +905,15 @@ void RALinScan::assignRegOrStackSlotAtInterval(LiveInterval* cur)
     prt_->addRegUse(physReg);
     active_.push_back(std::make_pair(cur, cur->begin()));
     handled_.push_back(cur);
+
+    // "Upgrade" the physical register since it has been allocated.
+    UpgradeRegister(physReg);
+    if (LiveInterval *NextReloadLI = hasNextReloadInterval(cur)) {
+      // "Downgrade" physReg to try to keep physReg from being allocated until
+      // the next reload from the same SS is allocated. 
+      NextReloadLI->preference = physReg;
+      DowngradeRegister(cur, physReg);
+    }
     return;
   }
   DOUT << "no free registers\n";
@@ -896,9 +974,10 @@ void RALinScan::assignRegOrStackSlotAtInterval(LiveInterval* cur)
     if (cur->weight == HUGE_VALF ||
         li_->getApproximateInstructionCount(*cur) == 0) {
       // Spill a physical register around defs and uses.
-      if (li_->spillPhysRegAroundRegDefsUses(*cur, minReg, *vrm_))
+      if (li_->spillPhysRegAroundRegDefsUses(*cur, minReg, *vrm_)) {
+        DowngradedRegs.clear();
         assignRegOrStackSlotAtInterval(cur);
-      else {
+      } else {
         cerr << "Ran out of registers during register allocation!\n";
         exit(1);
       }
@@ -919,7 +998,7 @@ void RALinScan::assignRegOrStackSlotAtInterval(LiveInterval* cur)
           DOUT << tri_->getName(RegsWeights[i].first)
                << " (" << RegsWeights[i].second << ")\n");
 
-  // if the current has the minimum weight, we need to spill it and
+  // If the current has the minimum weight, we need to spill it and
   // add any added intervals back to unhandled, and restart
   // linearscan.
   if (cur->weight != HUGE_VALF && cur->weight <= minWeight) {
@@ -928,12 +1007,13 @@ void RALinScan::assignRegOrStackSlotAtInterval(LiveInterval* cur)
     SmallVector<LiveInterval*, 8> spillIs;
     std::vector<LiveInterval*> added =
       li_->addIntervalsForSpills(*cur, spillIs, loopInfo, *vrm_, SSWeight);
+    std::sort(added.begin(), added.end(), LISorter());
     addStackInterval(cur, ls_, li_, SSWeight, *vrm_);
     if (added.empty())
       return;  // Early exit if all spills were folded.
 
-    // Merge added with unhandled.  Note that we know that
-    // addIntervalsForSpills returns intervals sorted by their starting
+    // Merge added with unhandled.  Note that we have already sorted
+    // intervals returned by addIntervalsForSpills by their starting
     // point.
     for (unsigned i = 0, e = added.size(); i != e; ++i)
       unhandled_.push(added[i]);
@@ -942,7 +1022,7 @@ void RALinScan::assignRegOrStackSlotAtInterval(LiveInterval* cur)
 
   ++NumBacktracks;
 
-  // push the current interval back to unhandled since we are going
+  // Push the current interval back to unhandled since we are going
   // to re-run at least this iteration. Since we didn't modify it it
   // should go back right in the front of the list
   unhandled_.push(cur);
@@ -1020,10 +1100,15 @@ void RALinScan::assignRegOrStackSlotAtInterval(LiveInterval* cur)
       unhandled_.push(i);
     }
 
-    // It interval has a preference, it must be defined by a copy. Clear the
-    // preference now since the source interval allocation may have been undone
-    // as well.
-    i->preference = 0;
+    DenseMap<unsigned, unsigned>::iterator ii = DowngradeMap.find(i->reg);
+    if (ii == DowngradeMap.end())
+      // It interval has a preference, it must be defined by a copy. Clear the
+      // preference now since the source interval allocation may have been
+      // undone as well.
+      i->preference = 0;
+    else {
+      UpgradeRegister(ii->second);
+    }
   }
 
   // Rewind the iterators in the active, inactive, and fixed lists back to the
@@ -1032,7 +1117,7 @@ void RALinScan::assignRegOrStackSlotAtInterval(LiveInterval* cur)
   RevertVectorIteratorsTo(inactive_, earliestStart);
   RevertVectorIteratorsTo(fixed_, earliestStart);
 
-  // scan the rest and undo each interval that expired after t and
+  // Scan the rest and undo each interval that expired after t and
   // insert it in active (the next iteration of the algorithm will
   // put it in inactive if required)
   for (unsigned i = 0, e = handled_.size(); i != e; ++i) {
@@ -1046,9 +1131,87 @@ void RALinScan::assignRegOrStackSlotAtInterval(LiveInterval* cur)
     }
   }
 
-  // merge added with unhandled
-  for (unsigned i = 0, e = added.size(); i != e; ++i)
-    unhandled_.push(added[i]);
+  // Merge added with unhandled.
+  // This also update the NextReloadMap. That is, it adds mapping from a
+  // register defined by a reload from SS to the next reload from SS in the
+  // same basic block.
+  MachineBasicBlock *LastReloadMBB = 0;
+  LiveInterval *LastReload = 0;
+  int LastReloadSS = VirtRegMap::NO_STACK_SLOT;
+  std::sort(added.begin(), added.end(), LISorter());
+  for (unsigned i = 0, e = added.size(); i != e; ++i) {
+    LiveInterval *ReloadLi = added[i];
+    if (ReloadLi->weight == HUGE_VALF &&
+        li_->getApproximateInstructionCount(*ReloadLi) == 0) {
+      unsigned ReloadIdx = ReloadLi->beginNumber();
+      MachineBasicBlock *ReloadMBB = li_->getMBBFromIndex(ReloadIdx);
+      int ReloadSS = vrm_->getStackSlot(ReloadLi->reg);
+      if (LastReloadMBB == ReloadMBB && LastReloadSS == ReloadSS) {
+        // Last reload of same SS is in the same MBB. We want to try to
+        // allocate both reloads the same register and make sure the reg
+        // isn't clobbered in between if at all possible.
+        assert(LastReload->beginNumber() < ReloadIdx);
+        NextReloadMap.insert(std::make_pair(LastReload->reg, ReloadLi->reg));
+      }
+      LastReloadMBB = ReloadMBB;
+      LastReload = ReloadLi;
+      LastReloadSS = ReloadSS;
+    }
+    unhandled_.push(ReloadLi);
+  }
+}
+
+unsigned RALinScan::getFreePhysReg(const TargetRegisterClass *RC,
+                                   unsigned MaxInactiveCount,
+                                   SmallVector<unsigned, 256> &inactiveCounts,
+                                   bool SkipDGRegs) {
+  unsigned FreeReg = 0;
+  unsigned FreeRegInactiveCount = 0;
+
+  TargetRegisterClass::iterator I = RC->allocation_order_begin(*mf_);
+  TargetRegisterClass::iterator E = RC->allocation_order_end(*mf_);
+  assert(I != E && "No allocatable register in this register class!");
+
+  // Scan for the first available register.
+  for (; I != E; ++I) {
+    unsigned Reg = *I;
+    // Ignore "downgraded" registers.
+    if (SkipDGRegs && DowngradedRegs.count(Reg))
+      continue;
+    if (prt_->isRegAvail(Reg)) {
+      FreeReg = Reg;
+      if (FreeReg < inactiveCounts.size())
+        FreeRegInactiveCount = inactiveCounts[FreeReg];
+      else
+        FreeRegInactiveCount = 0;
+      break;
+    }
+  }
+
+  // If there are no free regs, or if this reg has the max inactive count,
+  // return this register.
+  if (FreeReg == 0 || FreeRegInactiveCount == MaxInactiveCount)
+    return FreeReg;
+  
+  // Continue scanning the registers, looking for the one with the highest
+  // inactive count.  Alkis found that this reduced register pressure very
+  // slightly on X86 (in rev 1.94 of this file), though this should probably be
+  // reevaluated now.
+  for (; I != E; ++I) {
+    unsigned Reg = *I;
+    // Ignore "downgraded" registers.
+    if (SkipDGRegs && DowngradedRegs.count(Reg))
+      continue;
+    if (prt_->isRegAvail(Reg) && Reg < inactiveCounts.size() &&
+        FreeRegInactiveCount < inactiveCounts[Reg]) {
+      FreeReg = Reg;
+      FreeRegInactiveCount = inactiveCounts[Reg];
+      if (FreeRegInactiveCount == MaxInactiveCount)
+        break;    // We found the one with the max inactive count.
+    }
+  }
+
+  return FreeReg;
 }
 
 /// getFreePhysReg - return a free physical register for this virtual register
@@ -1078,9 +1241,6 @@ unsigned RALinScan::getFreePhysReg(LiveInterval *cur) {
     }
   }
 
-  unsigned FreeReg = 0;
-  unsigned FreeRegInactiveCount = 0;
-
   // If copy coalescer has assigned a "preferred" register, check if it's
   // available first.
   if (cur->preference) {
@@ -1094,40 +1254,13 @@ unsigned RALinScan::getFreePhysReg(LiveInterval *cur) {
            << tri_->getName(cur->preference) << "\n";
   }
 
-  // Scan for the first available register.
-  TargetRegisterClass::iterator I = RC->allocation_order_begin(*mf_);
-  TargetRegisterClass::iterator E = RC->allocation_order_end(*mf_);
-  assert(I != E && "No allocatable register in this register class!");
-  for (; I != E; ++I)
-    if (prt_->isRegAvail(*I)) {
-      FreeReg = *I;
-      if (FreeReg < inactiveCounts.size())
-        FreeRegInactiveCount = inactiveCounts[FreeReg];
-      else
-        FreeRegInactiveCount = 0;
-      break;
-    }
-
-  // If there are no free regs, or if this reg has the max inactive count,
-  // return this register.
-  if (FreeReg == 0 || FreeRegInactiveCount == MaxInactiveCount) return FreeReg;
-  
-  // Continue scanning the registers, looking for the one with the highest
-  // inactive count.  Alkis found that this reduced register pressure very
-  // slightly on X86 (in rev 1.94 of this file), though this should probably be
-  // reevaluated now.
-  for (; I != E; ++I) {
-    unsigned Reg = *I;
-    if (prt_->isRegAvail(Reg) && Reg < inactiveCounts.size() &&
-        FreeRegInactiveCount < inactiveCounts[Reg]) {
-      FreeReg = Reg;
-      FreeRegInactiveCount = inactiveCounts[Reg];
-      if (FreeRegInactiveCount == MaxInactiveCount)
-        break;    // We found the one with the max inactive count.
-    }
+  if (!DowngradedRegs.empty()) {
+    unsigned FreeReg = getFreePhysReg(RC, MaxInactiveCount, inactiveCounts,
+                                      true);
+    if (FreeReg)
+      return FreeReg;
   }
-  
-  return FreeReg;
+  return getFreePhysReg(RC, MaxInactiveCount, inactiveCounts, false);
 }
 
 FunctionPass* llvm::createLinearScanRegisterAllocator() {