Reimplement SourceLocation. Instead of having a

fileid/offset pair, it now contains a bit discriminating between
mapped locations and file locations.  This separates the tables for
macros and files in SourceManager, and allows better separation of
concepts in the rest of the compiler.  This allows us to have *many*
macro instantiations before running out of 'addressing space'.

This is also more efficient, because testing whether something is a
macro expansion is now a bit test instead of a table lookup (which
also used to require having a srcmgr around, now it doesn't).

This is fully functional, but there are several refinements and
optimizations left.


git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@40103 91177308-0d34-0410-b5e6-96231b3b80d8

13 files changed, 288 insertions, 270 deletions

diff --git a/Basic/SourceManager.cpp b/Basic/SourceManager.cpp
index 2177c982d4..3e2623c8e4 100644
--- a/Basic/SourceManager.cpp
+++ b/Basic/SourceManager.cpp
@@ -18,7 +18,6 @@
 #include <algorithm>
 #include <iostream>
 #include <fcntl.h>
-
 using namespace clang;
 using namespace SrcMgr;
 using llvm::MemoryBuffer;
@@ -141,7 +140,7 @@ unsigned SourceManager::createFileID(const InfoRec *File,
   // FilePos field.
   unsigned FileSize = File->second.Buffer->getBufferSize();
   if (FileSize+1 < (1 << SourceLocation::FilePosBits)) {
-    FileIDs.push_back(FileIDInfo::getNormalBuffer(IncludePos, 0, File));
+    FileIDs.push_back(FileIDInfo::get(IncludePos, 0, File));
     assert(FileIDs.size() < (1 << SourceLocation::FileIDBits) &&
            "Ran out of file ID's!");
     return FileIDs.size();
@@ -152,7 +151,7 @@ unsigned SourceManager::createFileID(const InfoRec *File,
 
   unsigned ChunkNo = 0;
   while (1) {
-    FileIDs.push_back(FileIDInfo::getNormalBuffer(IncludePos, ChunkNo++, File));
+    FileIDs.push_back(FileIDInfo::get(IncludePos, ChunkNo++, File));
 
     if (FileSize+1 < (1 << SourceLocation::FilePosBits)) break;
     FileSize -= (1 << SourceLocation::FilePosBits);
@@ -176,7 +175,14 @@ SourceLocation SourceManager::getInstantiationLoc(SourceLocation VirtLoc,
   // Resolve InstantLoc down to a real logical location.
   InstantLoc = getLogicalLoc(InstantLoc);
   
+  // FIXME: intelligently cache macroid's.
+  MacroIDs.push_back(MacroIDInfo::get(InstantLoc, PhysLoc));
+  
+  return SourceLocation::getMacroLoc(MacroIDs.size()-1, 0, 0);
+  
+#if 0
   unsigned InstantiationFileID;
+  
   // If this is the same instantiation as was requested last time, return this
   // immediately.
   if (PhysLoc.getFileID() == LastInstantiationLoc_MacroFID &&
@@ -193,7 +199,9 @@ SourceLocation SourceManager::getInstantiationLoc(SourceLocation VirtLoc,
     LastInstantiationLoc_InstantLoc = InstantLoc;
     LastInstantiationLoc_Result     = InstantiationFileID;
   }
-  return SourceLocation(InstantiationFileID, PhysLoc.getRawFilePos());
+  return SourceLocation::getMacroLoc(InstantiationFileID, 
+                                     PhysLoc.getRawFilePos());
+#endif
 }
 
 
@@ -203,34 +211,21 @@ SourceLocation SourceManager::getInstantiationLoc(SourceLocation VirtLoc,
 const char *SourceManager::getCharacterData(SourceLocation SL) const {
   // Note that this is a hot function in the getSpelling() path, which is
   // heavily used by -E mode.
-  unsigned FileID = SL.getFileID();
-  assert(FileID && "Invalid source location!");
+  SL = getPhysicalLoc(SL);
   
-  return getFileInfo(FileID)->Buffer->getBufferStart() + getFilePos(SL);
+  return getFileInfo(SL.getFileID())->Buffer->getBufferStart() + 
+         getFullFilePos(SL);
 }
 
-/// getIncludeLoc - Return the location of the #include for the specified
-/// FileID.
-SourceLocation SourceManager::getIncludeLoc(unsigned FileID) const {
-  const SrcMgr::FileIDInfo *FIDInfo = getFIDInfo(FileID);
-
-  // For Macros, the physical loc is specified by the MacroTokenFileID.
-  if (FIDInfo->IDType == SrcMgr::FileIDInfo::MacroExpansion)
-    FIDInfo = &FileIDs[FIDInfo->u.MacroTokenFileID-1];
-  
-  return FIDInfo->IncludeLoc;
-}  
-
 
-/// getColumnNumber - Return the column # for the specified include position.
+/// getColumnNumber - Return the column # for the specified file position.
 /// this is significantly cheaper to compute than the line number.  This returns
 /// zero if the column number isn't known.
 unsigned SourceManager::getColumnNumber(SourceLocation Loc) const {
-  Loc = getLogicalLoc(Loc);
   unsigned FileID = Loc.getFileID();
   if (FileID == 0) return 0;
   
-  unsigned FilePos = getFilePos(Loc);
+  unsigned FilePos = getFullFilePos(Loc);
   const MemoryBuffer *Buffer = getBuffer(FileID);
   const char *Buf = Buffer->getBufferStart();
 
@@ -244,7 +239,6 @@ unsigned SourceManager::getColumnNumber(SourceLocation Loc) const {
 /// the SourceLocation specifies.  This can be modified with #line directives,
 /// etc.
 std::string SourceManager::getSourceName(SourceLocation Loc) {
-  Loc = getLogicalLoc(Loc);
   unsigned FileID = Loc.getFileID();
   if (FileID == 0) return "";
   return getFileInfo(FileID)->Buffer->getBufferIdentifier();
@@ -256,7 +250,6 @@ std::string SourceManager::getSourceName(SourceLocation Loc) {
 /// line offsets for the MemoryBuffer, so this is not cheap: use only when
 /// about to emit a diagnostic.
 unsigned SourceManager::getLineNumber(SourceLocation Loc) {
-  Loc = getLogicalLoc(Loc);
   unsigned FileID = Loc.getFileID();
   if (FileID == 0) return 0;
   FileInfo *FileInfo = getFileInfo(FileID);
@@ -317,28 +310,10 @@ unsigned SourceManager::getLineNumber(SourceLocation Loc) {
   // type approaches to make good (tight?) initial guesses based on the
   // assumption that all lines are the same average size.
   unsigned *Pos = std::lower_bound(SourceLineCache, SourceLineCache+NumLines,
-                                   getFilePos(Loc)+1);
+                                   getFullFilePos(Loc)+1);
   return Pos-SourceLineCache;
 }
 
-/// getSourceFilePos - This method returns the *logical* offset from the start
-/// of the file that the specified SourceLocation represents.  This returns
-/// the location of the *logical* character data, not the physical file
-/// position.  In the case of macros, for example, this returns where the
-/// macro was instantiated, not where the characters for the macro can be
-/// found.
-unsigned SourceManager::getSourceFilePos(SourceLocation Loc) const {
- 
-  // If this is a macro, we need to get the instantiation location.
-  const SrcMgr::FileIDInfo *FIDInfo = getFIDInfo(Loc.getFileID());
-  while (FIDInfo->IDType == SrcMgr::FileIDInfo::MacroExpansion) {
-    Loc = FIDInfo->IncludeLoc;
-    FIDInfo = getFIDInfo(Loc.getFileID());
-  }
-  
-  return getFilePos(Loc);
-}
-
 /// PrintStats - Print statistics to stderr.
 ///
 void SourceManager::PrintStats() const {
@@ -346,17 +321,8 @@ void SourceManager::PrintStats() const {
   std::cerr << FileInfos.size() << " files mapped, " << MemBufferInfos.size()
             << " mem buffers mapped, " << FileIDs.size() 
             << " file ID's allocated.\n";
-  unsigned NumBuffers = 0, NumMacros = 0;
-  for (unsigned i = 0, e = FileIDs.size(); i != e; ++i) {
-    if (FileIDs[i].IDType == FileIDInfo::NormalBuffer)
-      ++NumBuffers;
-    else if (FileIDs[i].IDType == FileIDInfo::MacroExpansion)
-      ++NumMacros;
-    else
-      assert(0 && "Unknown FileID!");
-  }
-  std::cerr << "  " << NumBuffers << " normal buffer FileID's, "
-            << NumMacros << " macro expansion FileID's.\n";
+  std::cerr << "  " << FileIDs.size() << " normal buffer FileID's, "
+            << MacroIDs.size() << " macro expansion FileID's.\n";
     
   
   
diff --git a/Driver/DiagChecker.cpp b/Driver/DiagChecker.cpp
index 76b0526d4e..9830c0d69e 100644
--- a/Driver/DiagChecker.cpp
+++ b/Driver/DiagChecker.cpp
@@ -60,7 +60,7 @@ static void FindDiagnostics(const std::string &Comment,
     if (OpenDiag == std::string::npos) {
       fprintf(stderr,
               "oops:%d: Cannot find beginning of expected error string\n",
-              SourceMgr.getLineNumber(Pos));
+              SourceMgr.getLogicalLineNumber(Pos));
       break;
     }
 
@@ -70,7 +70,7 @@ static void FindDiagnostics(const std::string &Comment,
     if (CloseDiag == std::string::npos) {
       fprintf(stderr,
               "oops:%d: Cannot find end of expected error string\n",
-              SourceMgr.getLineNumber(Pos));
+              SourceMgr.getLogicalLineNumber(Pos));
       break;
     }
 
@@ -126,7 +126,7 @@ static bool PrintProblem(SourceManager &SourceMgr,
 
   for (const_diag_iterator I = diag_begin, E = diag_end; I != E; ++I)
     fprintf(stderr, "  Line %d: %s\n",
-            SourceMgr.getLineNumber(I->first),
+            SourceMgr.getLogicalLineNumber(I->first),
             I->second.c_str());
 
   return true;
@@ -144,12 +144,12 @@ static bool CompareDiagLists(SourceManager &SourceMgr,
   DiagList DiffList;
 
   for (const_diag_iterator I = d1_begin, E = d1_end; I != E; ++I) {
-    unsigned LineNo1 = SourceMgr.getLineNumber(I->first);
+    unsigned LineNo1 = SourceMgr.getLogicalLineNumber(I->first);
     const std::string &Diag1 = I->second;
     bool Found = false;
 
     for (const_diag_iterator II = d2_begin, IE = d2_end; II != IE; ++II) {
-      unsigned LineNo2 = SourceMgr.getLineNumber(II->first);
+      unsigned LineNo2 = SourceMgr.getLogicalLineNumber(II->first);
       if (LineNo1 != LineNo2) continue;
 
       const std::string &Diag2 = II->second;
diff --git a/Driver/PrintPreprocessedOutput.cpp b/Driver/PrintPreprocessedOutput.cpp
index 4a4f6783da..c22167315b 100644
--- a/Driver/PrintPreprocessedOutput.cpp
+++ b/Driver/PrintPreprocessedOutput.cpp
@@ -146,7 +146,7 @@ void PrintPPOutputPPCallbacks::MoveToLine(SourceLocation Loc) {
     return;
   }
   
-  unsigned LineNo = PP.getSourceManager().getLineNumber(Loc);
+  unsigned LineNo = PP.getSourceManager().getLogicalLineNumber(Loc);
   
   // If this line is "close enough" to the original line, just print newlines,
   // otherwise print a #line directive.
@@ -191,7 +191,7 @@ void PrintPPOutputPPCallbacks::FileChanged(SourceLocation Loc,
   // #include directive was at.
   SourceManager &SourceMgr = PP.getSourceManager();
   if (Reason == PPCallbacks::EnterFile) {
-    MoveToLine(SourceMgr.getIncludeLoc(Loc.getFileID()));
+    MoveToLine(SourceMgr.getIncludeLoc(Loc));
   } else if (Reason == PPCallbacks::SystemHeaderPragma) {
     MoveToLine(Loc);
     
@@ -200,6 +200,7 @@ void PrintPPOutputPPCallbacks::FileChanged(SourceLocation Loc,
     // strange behavior.
   }
   
+  Loc = SourceMgr.getLogicalLoc(Loc);
   CurLine = SourceMgr.getLineNumber(Loc);
   CurFilename = '"' + Lexer::Stringify(SourceMgr.getSourceName(Loc)) + '"';
   FileType = FileType;
@@ -256,8 +257,8 @@ void PrintPPOutputPPCallbacks::HandleFirstTokOnLine(LexerToken &Tok) {
   
   // Print out space characters so that the first token on a line is
   // indented for easy reading.
-  unsigned ColNo = 
-    PP.getSourceManager().getColumnNumber(Tok.getLocation());
+  const SourceManager &SourceMgr = PP.getSourceManager();
+  unsigned ColNo = SourceMgr.getLogicalColumnNumber(Tok.getLocation());
   
   // This hack prevents stuff like:
   // #define HASH #
diff --git a/Driver/TextDiagnosticPrinter.cpp b/Driver/TextDiagnosticPrinter.cpp
index f1a36ba503..f37732c241 100644
--- a/Driver/TextDiagnosticPrinter.cpp
+++ b/Driver/TextDiagnosticPrinter.cpp
@@ -32,16 +32,15 @@ NoCaretDiagnostics("fno-caret-diagnostics",
 
 void TextDiagnosticPrinter::
 PrintIncludeStack(SourceLocation Pos) {
-  unsigned FileID = Pos.getFileID();
-  if (FileID == 0) return;
-  
+  if (Pos.isInvalid()) return;
+
+  Pos = SourceMgr.getLogicalLoc(Pos);
+
   // Print out the other include frames first.
-  PrintIncludeStack(SourceMgr.getIncludeLoc(FileID));
-  
+  PrintIncludeStack(SourceMgr.getIncludeLoc(Pos));
   unsigned LineNo = SourceMgr.getLineNumber(Pos);
   
-  const llvm::MemoryBuffer *Buffer = SourceMgr.getBuffer(FileID);
-  std::cerr << "In file included from " << Buffer->getBufferIdentifier()
+  std::cerr << "In file included from " << SourceMgr.getSourceName(Pos)
             << ":" << LineNo << ":\n";
 }
 
@@ -55,16 +54,16 @@ void TextDiagnosticPrinter::HighlightRange(const SourceRange &R,
          "Expect a correspondence between source and carat line!");
   if (!R.isValid()) return;
 
-  unsigned StartLineNo = SourceMgr.getLineNumber(R.Begin());
+  unsigned StartLineNo = SourceMgr.getLogicalLineNumber(R.Begin());
   if (StartLineNo > LineNo) return;  // No intersection.
   
-  unsigned EndLineNo = SourceMgr.getLineNumber(R.End());
+  unsigned EndLineNo = SourceMgr.getLogicalLineNumber(R.End());
   if (EndLineNo < LineNo) return;  // No intersection.
   
   // Compute the column number of the start.
   unsigned StartColNo = 0;
   if (StartLineNo == LineNo) {
-    StartColNo = SourceMgr.getColumnNumber(R.Begin());
+    StartColNo = SourceMgr.getLogicalColumnNumber(R.Begin());
     if (StartColNo) --StartColNo;  // Zero base the col #.
   }
 
@@ -76,7 +75,7 @@ void TextDiagnosticPrinter::HighlightRange(const SourceRange &R,
   // Compute the column number of the end.
   unsigned EndColNo = CaratLine.size();
   if (EndLineNo == LineNo) {
-    EndColNo = SourceMgr.getColumnNumber(R.End());
+    EndColNo = SourceMgr.getLogicalColumnNumber(R.End());
     if (EndColNo) {
       --EndColNo;  // Zero base the col #.
       
@@ -112,8 +111,8 @@ unsigned TextDiagnosticPrinter::GetTokenLength(SourceLocation Loc) {
   unsigned FileID = Loc.getFileID();
   
   // Create a lexer starting at the beginning of this token.
-  Lexer TheLexer(SourceMgr.getBuffer(FileID), FileID,
-                 *ThePreprocessor,  StrData);
+  Lexer TheLexer(SourceMgr.getBuffer(FileID), Loc,
+                 *ThePreprocessor, StrData);
   
   LexerToken TheTok;
   TheLexer.LexRawToken(TheTok);
@@ -128,35 +127,33 @@ void TextDiagnosticPrinter::HandleDiagnostic(Diagnostic::Level Level,
                                              unsigned NumStrs,
                                              const SourceRange *Ranges,
                                              unsigned NumRanges) {
-  unsigned LineNo = 0, FilePos = 0, FileID = 0, ColNo = 0;
-  unsigned LineStart = 0, LineEnd = 0;
-  const llvm::MemoryBuffer *Buffer = 0;
+  unsigned LineNo = 0, ColNo = 0;
+  const char *LineStart = 0, *LineEnd = 0;
   
   if (Pos.isValid()) {
-    LineNo = SourceMgr.getLineNumber(Pos);
-    FileID  = SourceMgr.getLogicalLoc(Pos).getFileID();
+    SourceLocation LPos = SourceMgr.getLogicalLoc(Pos);
+    LineNo = SourceMgr.getLineNumber(LPos);
     
     // First, if this diagnostic is not in the main file, print out the
     // "included from" lines.
-    if (LastWarningLoc != SourceMgr.getIncludeLoc(FileID)) {
-      LastWarningLoc = SourceMgr.getIncludeLoc(FileID);
+    if (LastWarningLoc != SourceMgr.getIncludeLoc(LPos)) {
+      LastWarningLoc = SourceMgr.getIncludeLoc(LPos);
       PrintIncludeStack(LastWarningLoc);
     }
   
     // Compute the column number.  Rewind from the current position to the start
     // of the line.
-    ColNo = SourceMgr.getColumnNumber(Pos);
-    FilePos = SourceMgr.getSourceFilePos(Pos);
-    LineStart = FilePos-ColNo+1;  // Column # is 1-based
+    ColNo = SourceMgr.getColumnNumber(LPos);
+    const char *TokLogicalPtr = SourceMgr.getCharacterData(LPos);
+    LineStart = TokLogicalPtr-ColNo+1;  // Column # is 1-based
   
     // Compute the line end.  Scan forward from the error position to the end of
     // the line.
-    Buffer = SourceMgr.getBuffer(FileID);
-    const char *Buf = Buffer->getBufferStart();
+    const llvm::MemoryBuffer *Buffer = SourceMgr.getBuffer(LPos.getFileID());
     const char *BufEnd = Buffer->getBufferEnd();
-    LineEnd = FilePos;
-    while (Buf+LineEnd != BufEnd && 
-           Buf[LineEnd] != '\n' && Buf[LineEnd] != '\r')
+    LineEnd = TokLogicalPtr;
+    while (LineEnd != BufEnd && 
+           *LineEnd != '\n' && *LineEnd != '\r')
       ++LineEnd;
   
     std::cerr << Buffer->getBufferIdentifier() 
@@ -180,8 +177,7 @@ void TextDiagnosticPrinter::HandleDiagnostic(Diagnostic::Level Level,
   
   if (!NoCaretDiagnostics && Pos.isValid()) {
     // Get the line of the source file.
-    const char *Buf = Buffer->getBufferStart();
-    std::string SourceLine(Buf+LineStart, Buf+LineEnd);
+    std::string SourceLine(LineStart, LineEnd);
     
     // Create a line for the carat that is filled with spaces that is the same
     // length as the line of source code.
diff --git a/Driver/TextDiagnostics.cpp b/Driver/TextDiagnostics.cpp
index 4fc7e0c921..3c29cca696 100644
--- a/Driver/TextDiagnostics.cpp
+++ b/Driver/TextDiagnostics.cpp
@@ -45,9 +45,7 @@ bool TextDiagnostics::IgnoreDiagnostic(Diagnostic::Level Level,
     // diagnostic.
     if (Level == Diagnostic::Warning ||
         Level == Diagnostic::Note) {
-      SourceLocation PhysLoc = SourceMgr.getPhysicalLoc(Pos);
-      const FileEntry *F = SourceMgr.getFileEntryForFileID(PhysLoc.getFileID());
-      if (F) {
+      if (const FileEntry *F = SourceMgr.getFileEntryForLoc(Pos)) {
         DirectoryLookup::DirType DirInfo = TheHeaderSearch->getFileDirFlavor(F);
         if (DirInfo == DirectoryLookup::SystemHeaderDir ||
             DirInfo == DirectoryLookup::ExternCSystemHeaderDir)
diff --git a/Lex/Lexer.cpp b/Lex/Lexer.cpp
index 1775b2f7bf..a1db060c3e 100644
--- a/Lex/Lexer.cpp
+++ b/Lex/Lexer.cpp
@@ -27,17 +27,17 @@
 #include "clang/Lex/Lexer.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Basic/Diagnostic.h"
-#include "clang/Basic/SourceLocation.h"
+#include "clang/Basic/SourceManager.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include <cctype>
 using namespace clang;
 
 static void InitCharacterInfo();
 
-Lexer::Lexer(const llvm::MemoryBuffer *File, unsigned fileid, Preprocessor &pp,
-             const char *BufStart, const char *BufEnd)
+Lexer::Lexer(const llvm::MemoryBuffer *File, SourceLocation fileloc,
+             Preprocessor &pp, const char *BufStart, const char *BufEnd)
   : BufferEnd(BufEnd ? BufEnd : File->getBufferEnd()),
-    InputFile(File), CurFileID(fileid), PP(pp), Features(PP.getLangOptions()) {
+    InputFile(File), FileLoc(fileloc), PP(pp), Features(PP.getLangOptions()) {
   Is_PragmaLexer = false;
   IsMainFile = false;
   InitCharacterInfo();
@@ -151,7 +151,24 @@ static inline bool isNumberBody(unsigned char c) {
 SourceLocation Lexer::getSourceLocation(const char *Loc) const {
   assert(Loc >= InputFile->getBufferStart() && Loc <= BufferEnd &&
          "Location out of range for this buffer!");
-  return SourceLocation(CurFileID, Loc-InputFile->getBufferStart());
+
+  // In the normal case, we're just lexing from a simple file buffer, return
+  // the file id from FileLoc with the offset specified.
+  unsigned CharNo = Loc-InputFile->getBufferStart();
+  if (FileLoc.isFileID())
+    return SourceLocation::getFileLoc(FileLoc.getFileID(), CharNo);
+  
+  // Otherwise, we're lexing "mapped tokens".  This is used for things like
+  // _Pragma handling.  Combine the instantiation location of FileLoc with the
+  // physical location.
+  SourceManager &SourceMgr = PP.getSourceManager();
+
+  // Create a new SLoc which is expanded from logical(FileLoc) but whose
+  // characters come from phys(FileLoc)+Offset.
+  SourceLocation VirtLoc = SourceMgr.getLogicalLoc(FileLoc);
+  SourceLocation PhysLoc = SourceMgr.getPhysicalLoc(FileLoc);
+  PhysLoc = SourceLocation::getFileLoc(PhysLoc.getFileID(), CharNo);
+  return SourceMgr.getInstantiationLoc(PhysLoc, VirtLoc);
 }
 
 
diff --git a/Lex/MacroExpander.cpp b/Lex/MacroExpander.cpp
index e474906050..9a80ac3c3f 100644
--- a/Lex/MacroExpander.cpp
+++ b/Lex/MacroExpander.cpp
@@ -582,7 +582,8 @@ void MacroExpander::PasteTokens(LexerToken &Tok) {
       assert(FileID && "Could not get FileID for paste?");
       
       // Make a lexer object so that we lex and expand the paste result.
-      Lexer *TL = new Lexer(SourceMgr.getBuffer(FileID), FileID, PP,
+      Lexer *TL = new Lexer(SourceMgr.getBuffer(FileID),
+                            SourceLocation::getFileLoc(FileID, 0), PP,
                             ResultStrData, 
                             ResultStrData+LHSLen+RHSLen /*don't include null*/);
       
diff --git a/Lex/Pragma.cpp b/Lex/Pragma.cpp
index de59934b4d..596b7e76e4 100644
--- a/Lex/Pragma.cpp
+++ b/Lex/Pragma.cpp
@@ -140,12 +140,12 @@ void Preprocessor::Handle_Pragma(LexerToken &Tok) {
   SourceLocation TokLoc = CreateString(&StrVal[0], StrVal.size(), StrLoc);
   const char *StrData = SourceMgr.getCharacterData(TokLoc);
 
-  unsigned FileID = TokLoc.getFileID();
+  unsigned FileID = SourceMgr.getPhysicalLoc(TokLoc).getFileID();
   assert(FileID && "Could not get FileID for _Pragma?");
 
   // Make and enter a lexer object so that we lex and expand the tokens just
   // like any others.
-  Lexer *TL = new Lexer(SourceMgr.getBuffer(FileID), FileID, *this,
+  Lexer *TL = new Lexer(SourceMgr.getBuffer(FileID), TokLoc, *this,
                         StrData, StrData+StrVal.size()-1 /* no null */);
   
   // Ensure that the lexer thinks it is inside a directive, so that end \n will
@@ -175,10 +175,10 @@ void Preprocessor::HandlePragmaOnce(LexerToken &OnceTok) {
   }
   
   // Get the current file lexer we're looking at.  Ignore _Pragma 'files' etc.
-  unsigned FileID = getCurrentFileLexer()->getCurFileID();
+  SourceLocation FileLoc = getCurrentFileLexer()->getFileLoc();
   
   // Mark the file as a once-only file now.
-  HeaderInfo.MarkFileIncludeOnce(SourceMgr.getFileEntryForFileID(FileID));
+  HeaderInfo.MarkFileIncludeOnce(SourceMgr.getFileEntryForLoc(FileLoc));
 }
 
 /// HandlePragmaPoison - Handle #pragma GCC poison.  PoisonTok is the 'poison'.
@@ -233,8 +233,7 @@ void Preprocessor::HandlePragmaSystemHeader(LexerToken &SysHeaderTok) {
   Lexer *TheLexer = getCurrentFileLexer();
   
   // Mark the file as a system header.
-  const FileEntry *File = 
-    SourceMgr.getFileEntryForFileID(TheLexer->getCurFileID());
+  const FileEntry *File = SourceMgr.getFileEntryForLoc(TheLexer->getFileLoc());
   HeaderInfo.MarkFileSystemHeader(File);
   
   // Notify the client, if desired, that we are in a new source file.
@@ -274,8 +273,8 @@ void Preprocessor::HandlePragmaDependency(LexerToken &DependencyTok) {
     return Diag(FilenameTok, diag::err_pp_file_not_found,
                 std::string(FilenameStart, FilenameEnd));
   
-  unsigned FileID = getCurrentFileLexer()->getCurFileID();
-  const FileEntry *CurFile = SourceMgr.getFileEntryForFileID(FileID);
+  SourceLocation FileLoc = getCurrentFileLexer()->getFileLoc();
+  const FileEntry *CurFile = SourceMgr.getFileEntryForLoc(FileLoc);
 
   // If this file is older than the file it depends on, emit a diagnostic.
   if (CurFile && CurFile->getModificationTime() < File->getModificationTime()) {
diff --git a/Lex/Preprocessor.cpp b/Lex/Preprocessor.cpp
index 7b6c57186c..c3fd554264 100644
--- a/Lex/Preprocessor.cpp
+++ b/Lex/Preprocessor.cpp
@@ -258,34 +258,41 @@ CreateString(const char *Buf, unsigned Len, SourceLocation SLoc) {
 /// token, return a new location that specifies a character within the token.
 SourceLocation Preprocessor::AdvanceToTokenCharacter(SourceLocation TokStart, 
                                                      unsigned CharNo) {
-  // If they request the first char of the token, we're trivially done.
-  if (CharNo == 0) return TokStart;
+  // If they request the first char of the token, we're trivially done.  If this
+  // is a macro expansion, it doesn't make sense to point to a character within
+  // the instantiation point (the name).  We could point to the source
+  // character, but without also pointing to instantiation info, this is
+  // confusing.
+  if (CharNo == 0 || TokStart.isMacroID()) return TokStart;
   
   // Figure out how many physical characters away the specified logical
   // character is.  This needs to take into consideration newlines and
   // trigraphs.
-  const char *TokStartPtr = SourceMgr.getCharacterData(TokStart);
-  const char *TokPtr = TokStartPtr;
+  const char *TokPtr = SourceMgr.getCharacterData(TokStart);
+  unsigned PhysOffset = 0;
   
   // The usual case is that tokens don't contain anything interesting.  Skip
   // over the uninteresting characters.  If a token only consists of simple
   // chars, this method is extremely fast.
   while (CharNo && Lexer::isObviouslySimpleCharacter(*TokPtr))
-    ++TokPtr, --CharNo;
+    ++TokPtr, --CharNo, ++PhysOffset;
   
   // If we have a character that may be a trigraph or escaped newline, create a
   // lexer to parse it correctly.
-  unsigned FileID = TokStart.getFileID();
-  const llvm::MemoryBuffer *SrcBuf = SourceMgr.getBuffer(FileID);
   if (CharNo != 0) {
     // Create a lexer starting at this token position.
-    Lexer TheLexer(SrcBuf, FileID, *this, TokPtr);
+    const llvm::MemoryBuffer *SrcBuf =SourceMgr.getBuffer(TokStart.getFileID());
+    Lexer TheLexer(SrcBuf, TokStart, *this, TokPtr);
     LexerToken Tok;
     // Skip over characters the remaining characters.
+    const char *TokStartPtr = TokPtr;
     for (; CharNo; --CharNo)
       TheLexer.getAndAdvanceChar(TokPtr, Tok);
+    
+    PhysOffset += TokPtr-TokStartPtr;
   }
-  return SourceLocation(FileID, TokPtr-SrcBuf->getBufferStart());
+  
+  return TokStart.getFileLocWithOffset(PhysOffset);
 }
 
 
@@ -306,8 +313,8 @@ const FileEntry *Preprocessor::LookupFile(const char *FilenameStart,
   // info about where the current file is.
   const FileEntry *CurFileEnt = 0;
   if (!FromDir) {
-    unsigned TheFileID = getCurrentFileLexer()->getCurFileID();
-    CurFileEnt = SourceMgr.getFileEntryForFileID(TheFileID);
+    SourceLocation FileLoc = getCurrentFileLexer()->getFileLoc();
+    CurFileEnt = SourceMgr.getFileEntryForLoc(FileLoc);
   }
   
   // Do a standard file entry lookup.
@@ -321,7 +328,7 @@ const FileEntry *Preprocessor::LookupFile(const char *FilenameStart,
   // to one of the headers on the #include stack.  Walk the list of the current
   // headers on the #include stack and pass them to HeaderInfo.
   if (CurLexer && !CurLexer->Is_PragmaLexer) {
-    CurFileEnt = SourceMgr.getFileEntryForFileID(CurLexer->getCurFileID());
+    CurFileEnt = SourceMgr.getFileEntryForLoc(CurLexer->getFileLoc());
     if ((FE = HeaderInfo.LookupSubframeworkHeader(FilenameStart, FilenameEnd,
                                                   CurFileEnt)))
       return FE;
@@ -330,8 +337,7 @@ const FileEntry *Preprocessor::LookupFile(const char *FilenameStart,
   for (unsigned i = 0, e = IncludeMacroStack.size(); i != e; ++i) {
     IncludeStackInfo &ISEntry = IncludeMacroStack[e-i-1];
     if (ISEntry.TheLexer && !ISEntry.TheLexer->Is_PragmaLexer) {
-      CurFileEnt =
-        SourceMgr.getFileEntryForFileID(ISEntry.TheLexer->getCurFileID());
+      CurFileEnt = SourceMgr.getFileEntryForLoc(ISEntry.TheLexer->getFileLoc());
       if ((FE = HeaderInfo.LookupSubframeworkHeader(FilenameStart, FilenameEnd,
                                                     CurFileEnt)))
         return FE;
@@ -385,7 +391,8 @@ void Preprocessor::EnterSourceFile(unsigned FileID,
     MaxIncludeStackDepth = IncludeMacroStack.size();
 
   const llvm::MemoryBuffer *Buffer = SourceMgr.getBuffer(FileID);
-  Lexer *TheLexer = new Lexer(Buffer, FileID, *this);
+  Lexer *TheLexer = new Lexer(Buffer, SourceLocation::getFileLoc(FileID, 0),
+                              *this);
   if (isMainFile) TheLexer->setIsMainFile();
   EnterSourceFileWithLexer(TheLexer, CurDir);
 }  
@@ -410,10 +417,10 @@ void Preprocessor::EnterSourceFileWithLexer(Lexer *TheLexer,
     
     // Get the file entry for the current file.
     if (const FileEntry *FE = 
-          SourceMgr.getFileEntryForFileID(CurLexer->getCurFileID()))
+           SourceMgr.getFileEntryForLoc(CurLexer->getFileLoc()))
       FileType = HeaderInfo.getFileDirFlavor(FE);
     
-    Callbacks->FileChanged(SourceLocation(CurLexer->getCurFileID(), 0),
+    Callbacks->FileChanged(CurLexer->getFileLoc(),
                            PPCallbacks::EnterFile, FileType);
   }
 }
@@ -878,7 +885,7 @@ void Preprocessor::ExpandBuiltinMacro(LexerToken &Tok) {
   
   if (II == Ident__LINE__) {
     // __LINE__ expands to a simple numeric value.
-    sprintf(TmpBuffer, "%u", SourceMgr.getLineNumber(Tok.getLocation()));
+    sprintf(TmpBuffer, "%u", SourceMgr.getLogicalLineNumber(Tok.getLocation()));
     unsigned Length = strlen(TmpBuffer);
     Tok.setKind(tok::numeric_constant);
     Tok.setLength(Length);
@@ -887,15 +894,15 @@ void Preprocessor::ExpandBuiltinMacro(LexerToken &Tok) {
     SourceLocation Loc = Tok.getLocation();
     if (II == Ident__BASE_FILE__) {
       Diag(Tok, diag::ext_pp_base_file);
-      SourceLocation NextLoc = SourceMgr.getIncludeLoc(Loc.getFileID());
-      while (NextLoc.getFileID() != 0) {
+      SourceLocation NextLoc = SourceMgr.getIncludeLoc(Loc);
+      while (NextLoc.isValid()) {
         Loc = NextLoc;
-        NextLoc = SourceMgr.getIncludeLoc(Loc.getFileID());
+        NextLoc = SourceMgr.getIncludeLoc(Loc);
       }
     }
     
     // Escape this filename.  Turn '\' -> '\\' '"' -> '\"'
-    std::string FN = SourceMgr.getSourceName(Loc);
+    std::string FN = SourceMgr.getSourceName(SourceMgr.getLogicalLoc(Loc));
     FN = '"' + Lexer::Stringify(FN) + '"';
     Tok.setKind(tok::string_literal);
     Tok.setLength(FN.size());
@@ -917,9 +924,9 @@ void Preprocessor::ExpandBuiltinMacro(LexerToken &Tok) {
 
     // Compute the include depth of this token.
     unsigned Depth = 0;
-    SourceLocation Loc = SourceMgr.getIncludeLoc(Tok.getLocation().getFileID());
-    for (; Loc.getFileID() != 0; ++Depth)
-      Loc = SourceMgr.getIncludeLoc(Loc.getFileID());
+    SourceLocation Loc = SourceMgr.getIncludeLoc(Tok.getLocation());
+    for (; Loc.isValid(); ++Depth)
+      Loc = SourceMgr.getIncludeLoc(Loc);
     
     // __INCLUDE_LEVEL__ expands to a simple numeric value.
     sprintf(TmpBuffer, "%u", Depth);
@@ -938,7 +945,7 @@ void Preprocessor::ExpandBuiltinMacro(LexerToken &Tok) {
     Lexer *TheLexer = getCurrentFileLexer();
     
     if (TheLexer)
-      CurFile = SourceMgr.getFileEntryForFileID(TheLexer->getCurFileID());
+      CurFile = SourceMgr.getFileEntryForLoc(TheLexer->getFileLoc());
     
     // If this file is older than the file it depends on, emit a diagnostic.
     const char *Result;
@@ -1061,7 +1068,7 @@ bool Preprocessor::HandleEndOfFile(LexerToken &Result, bool isEndOfMacro) {
           CurLexer->MIOpt.GetControllingMacroAtEndOfFile()) {
       // Okay, this has a controlling macro, remember in PerFileInfo.
       if (const FileEntry *FE = 
-            SourceMgr.getFileEntryForFileID(CurLexer->getCurFileID()))
+            SourceMgr.getFileEntryForLoc(CurLexer->getFileLoc()))
         HeaderInfo.SetFileControllingMacro(FE, ControllingMacro);
     }
   }
@@ -1078,7 +1085,7 @@ bool Preprocessor::HandleEndOfFile(LexerToken &Result, bool isEndOfMacro) {
       
       // Get the file entry for the current file.
       if (const FileEntry *FE = 
-            SourceMgr.getFileEntryForFileID(CurLexer->getCurFileID()))
+            SourceMgr.getFileEntryForLoc(CurLexer->getFileLoc()))
         FileType = HeaderInfo.getFileDirFlavor(FE);
 
       Callbacks->FileChanged(CurLexer->getSourceLocation(CurLexer->BufferPtr),
diff --git a/Lex/ScratchBuffer.cpp b/Lex/ScratchBuffer.cpp
index 12cb0965ce..6678ceff59 100644
--- a/Lex/ScratchBuffer.cpp
+++ b/Lex/ScratchBuffer.cpp
@@ -43,7 +43,7 @@ SourceLocation ScratchBuffer::getToken(const char *Buf, unsigned Len) {
   assert(BytesUsed-Len < (1 << SourceLocation::FilePosBits) &&
          "Out of range file position!");
   
-  return SourceLocation(FileID, BytesUsed-Len);
+  return SourceLocation::getFileLoc(FileID, BytesUsed-Len);
 }
 
 
diff --git a/include/clang/Basic/SourceLocation.h b/include/clang/Basic/SourceLocation.h
index 1a90da9b09..d107b0bd70 100644
--- a/include/clang/Basic/SourceLocation.h
+++ b/include/clang/Basic/SourceLocation.h
@@ -14,6 +14,8 @@
 #ifndef LLVM_CLANG_SOURCELOCATION_H
 #define LLVM_CLANG_SOURCELOCATION_H
 
+#include <cassert>
+
 namespace clang {
     
 /// SourceLocation - This is a carefully crafted 32-bit identifier that encodes
@@ -24,14 +26,20 @@ class SourceLocation {
 public:
   enum {
     FileIDBits  = 14,
-    FilePosBits = 32-FileIDBits
+    FilePosBits = 32-1-FileIDBits,
+    
+    MacroIDBits       = 23,
+    MacroPhysOffsBits = 5,
+    MacroLogOffBits   = 3
   };
 
   SourceLocation() : ID(0) {}  // 0 is an invalid FileID.
   
-  /// SourceLocation constructor - Create a new SourceLocation object with the 
-  /// specified FileID and FilePos.
-  SourceLocation(unsigned FileID, unsigned FilePos) {
+  bool isFileID() const { return (ID >> 31) == 0; }
+  bool isMacroID() const { return (ID >> 31) != 0; }
+  
+  static SourceLocation getFileLoc(unsigned FileID, unsigned FilePos) {
+    SourceLocation L;
     // If a FilePos is larger than (1<<FilePosBits), the SourceManager makes
     // enough consequtive FileIDs that we have one for each chunk.
     if (FilePos >= (1 << FilePosBits)) {
@@ -44,26 +52,68 @@ public:
     if (FileID >= (1 << FileIDBits))
       FileID = (1 << FileIDBits)-1;
     
-    ID = (FileID << FilePosBits) | FilePos;
+    L.ID = (FileID << FilePosBits) | FilePos;
+    return L;
   }
   
+  static SourceLocation getMacroLoc(unsigned MacroID, unsigned PhysOffs,
+                                    unsigned LogOffs) {
+    SourceLocation L;
+    
+    assert(MacroID < (1 << MacroIDBits) && "Too many macros!");
+    assert(PhysOffs < (1 << MacroPhysOffsBits) && "Physoffs too large!");
+    assert(LogOffs  < (1 << MacroLogOffBits) && "Logical offs too large!");
+    
+    L.ID = (1 << 31) | (MacroID << (MacroPhysOffsBits+MacroLogOffBits)) |
+           (PhysOffs << MacroLogOffBits) | LogOffs;
+    return L;
+  }
+  
+  
   /// isValid - Return true if this is a valid SourceLocation object.  Invalid
   /// SourceLocations are often used when events have no corresponding location
   /// in the source (e.g. a diagnostic is required for a command line option).
   ///
   bool isValid() const { return ID != 0; }
+  bool isInvalid() const { return ID == 0; }
   
   /// getFileID - Return the file identifier for this SourceLocation.  This
   /// FileID