this massive patch introduces a simple new abstraction: it makes

"FileID" a concept that is now enforced by the compiler's type checker
instead of yet-another-random-unsigned floating around.

This is an important distinction from the "FileID" currently tracked by
SourceLocation.  *That* FileID may refer to the start of a file or to a
chunk within it.  The new FileID *only* refers to the file (and its 
#include stack and eventually #line data), it cannot refer to a chunk.

FileID is a completely opaque datatype to all clients, only SourceManager
is allowed to poke and prod it.



git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@62407 91177308-0d34-0410-b5e6-96231b3b80d8

9 files changed, 80 insertions, 88 deletions

diff --git a/lib/Lex/Lexer.cpp b/lib/Lex/Lexer.cpp
index 5a14c1356a..d63c8cc37b 100644
--- a/lib/Lex/Lexer.cpp
+++ b/lib/Lex/Lexer.cpp
@@ -62,14 +62,18 @@ tok::ObjCKeywordKind Token::getObjCKeywordID() const {
 /// with the specified preprocessor managing the lexing process.  This lexer
 /// assumes that the associated file buffer and Preprocessor objects will
 /// outlive it, so it doesn't take ownership of either of them.
-Lexer::Lexer(SourceLocation fileloc, Preprocessor &pp,
+Lexer::Lexer(SourceLocation fileloc, Preprocessor &PP,
              const char *BufStart, const char *BufEnd)
-  : PreprocessorLexer(&pp, fileloc), FileLoc(fileloc),
-    Features(pp.getLangOptions()) {
+// FIXME: This is really horrible and only needed for _Pragma lexers, split this
+// out of the main lexer path!
+  : PreprocessorLexer(&PP, 
+                      PP.getSourceManager().getCanonicalFileID(
+                      PP.getSourceManager().getSpellingLoc(fileloc))),
+                      FileLoc(fileloc),
+    Features(PP.getLangOptions()) {
       
-  SourceManager &SourceMgr = PP->getSourceManager();
-  unsigned InputFileID = SourceMgr.getSpellingLoc(FileLoc).getFileID();
-  const llvm::MemoryBuffer *InputFile = SourceMgr.getBuffer(InputFileID);
+  SourceManager &SourceMgr = PP.getSourceManager();
+  const llvm::MemoryBuffer *InputFile = SourceMgr.getBuffer(getFileID());
       
   Is_PragmaLexer = false;
   InitCharacterInfo();
@@ -103,7 +107,7 @@ Lexer::Lexer(SourceLocation fileloc, Preprocessor &pp,
   
   // Default to keeping comments if the preprocessor wants them.
   ExtendedTokenMode = 0;
-  SetCommentRetentionState(PP->getCommentRetentionState());
+  SetCommentRetentionState(PP.getCommentRetentionState());
 }
 
 /// Lexer constructor - Create a new raw lexer object.  This object is only
@@ -187,9 +191,7 @@ unsigned Lexer::MeasureTokenLength(SourceLocation Loc,
   // all obviously single-char tokens.  This could use 
   // Lexer::isObviouslySimpleCharacter for example to handle identifiers or
   // something.
-  
-  
-  const char *BufEnd = SM.getBufferData(Loc.getFileID()).second;
+  const char *BufEnd = SM.getBufferData(Loc).second;
   
   // Create a langops struct and enable trigraphs.  This is sufficient for
   // measuring tokens.
@@ -303,6 +305,8 @@ SourceLocation Lexer::getSourceLocation(const char *Loc) const {
   if (FileLoc.isFileID())
     return SourceLocation::getFileLoc(FileLoc.getFileID(), CharNo);
   
+  // Otherwise, this is the _Pragma lexer case, which pretends that all of the
+  // tokens are lexed from where the _Pragma was defined.
   assert(PP && "This doesn't work on raw lexers");
   return GetMappedTokenLoc(*PP, FileLoc, CharNo);
 }
diff --git a/lib/Lex/PPDirectives.cpp b/lib/Lex/PPDirectives.cpp
index 0566ec54b0..f7551a2532 100644
--- a/lib/Lex/PPDirectives.cpp
+++ b/lib/Lex/PPDirectives.cpp
@@ -394,8 +394,8 @@ const FileEntry *Preprocessor::LookupFile(const char *FilenameStart,
   // info about where the current file is.
   const FileEntry *CurFileEnt = 0;
   if (!FromDir) {
-    unsigned FileID = getCurrentFileLexer()->getFileID();
-    CurFileEnt = SourceMgr.getFileEntryForID(FileID);
+    FileID FID = getCurrentFileLexer()->getFileID();
+    CurFileEnt = SourceMgr.getFileEntryForID(FID);
   }
   
   // Do a standard file entry lookup.
@@ -786,16 +786,16 @@ void Preprocessor::HandleIncludeDirective(Token &IncludeTok,
           SourceMgr.getFileCharacteristic(getCurrentFileLexer()->getFileID()));
   
   // Look up the file, create a File ID for it.
-  unsigned FileID = SourceMgr.createFileID(File, FilenameTok.getLocation(),
-                                           FileCharacter);
-  if (FileID == 0) {
+  FileID FID = SourceMgr.createFileID(File, FilenameTok.getLocation(),
+                                      FileCharacter);
+  if (FID.isInvalid()) {
     Diag(FilenameTok, diag::err_pp_file_not_found)
       << std::string(FilenameStart, FilenameEnd);
     return;
   }
 
   // Finally, if all is good, enter the new file!
-  EnterSourceFile(FileID, CurDir);
+  EnterSourceFile(FID, CurDir);
 }
 
 /// HandleIncludeNextDirective - Implements #include_next.
diff --git a/lib/Lex/PPLexerChange.cpp b/lib/Lex/PPLexerChange.cpp
index 0ecca3a8cc..cc8ccc4c10 100644
--- a/lib/Lex/PPLexerChange.cpp
+++ b/lib/Lex/PPLexerChange.cpp
@@ -66,8 +66,7 @@ PreprocessorLexer *Preprocessor::getCurrentFileLexer() const {
 /// EnterSourceFile - Add a source file to the top of the include stack and
 /// start lexing tokens from it instead of the current buffer.  Return true
 /// on failure.
-void Preprocessor::EnterSourceFile(unsigned FileID,
-                                   const DirectoryLookup *CurDir) {
+void Preprocessor::EnterSourceFile(FileID FID, const DirectoryLookup *CurDir) {
   assert(CurTokenLexer == 0 && "Cannot #include a file inside a macro!");
   ++NumEnteredSourceFiles;
   
@@ -75,8 +74,7 @@ void Preprocessor::EnterSourceFile(unsigned FileID,
     MaxIncludeStackDepth = IncludeMacroStack.size();
 
   if (PTH) {
-    PTHLexer* PL =
-      PTH->CreateLexer(FileID, getSourceManager().getFileEntryForID(FileID));
+    PTHLexer *PL = PTH->CreateLexer(FID, SourceMgr.getFileEntryForID(FID));
 
     if (PL) {
       EnterSourceFileWithPTH(PL, CurDir);
@@ -84,7 +82,7 @@ void Preprocessor::EnterSourceFile(unsigned FileID,
     }
   }
   
-  Lexer *TheLexer = new Lexer(SourceLocation::getFileLoc(FileID, 0), *this);
+  Lexer *TheLexer = new Lexer(SourceMgr.getLocForStartOfFile(FID), *this);
   EnterSourceFileWithLexer(TheLexer, CurDir);
 }  
 
@@ -125,10 +123,9 @@ void Preprocessor::EnterSourceFileWithPTH(PTHLexer *PL,
 
   // Notify the client, if desired, that we are in a new source file.
   if (Callbacks) {
-    unsigned FileID = CurPPLexer->getFileID();
-    SrcMgr::CharacteristicKind FileType =
-      SourceMgr.getFileCharacteristic(CurPPLexer->getFileID());    
-    Callbacks->FileChanged(SourceLocation::getFileLoc(FileID, 0),
+    FileID FID = CurPPLexer->getFileID();
+    SrcMgr::CharacteristicKind FileType = SourceMgr.getFileCharacteristic(FID);
+    Callbacks->FileChanged(SourceMgr.getLocForStartOfFile(FID),
                            PPCallbacks::EnterFile, FileType);
   }
 }
diff --git a/lib/Lex/PTHLexer.cpp b/lib/Lex/PTHLexer.cpp
index 6401b9ac8f..b2870f46fd 100644
--- a/lib/Lex/PTHLexer.cpp
+++ b/lib/Lex/PTHLexer.cpp
@@ -23,7 +23,6 @@
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/OwningPtr.h"
-
 using namespace clang;
 
 #define DISK_TOKEN_SIZE (1+1+3+4+2)
@@ -48,15 +47,14 @@ static inline uint32_t Read32(const char*& data) {
 // PTHLexer methods.
 //===----------------------------------------------------------------------===//
 
-PTHLexer::PTHLexer(Preprocessor& pp, SourceLocation fileloc, const char* D,
-                   const char* ppcond,
-                   PTHSpellingSearch& mySpellingSrch,
-                   PTHManager& PM)
-  : PreprocessorLexer(&pp, fileloc), TokBuf(D), CurPtr(D), LastHashTokPtr(0),
+PTHLexer::PTHLexer(Preprocessor &PP, FileID FID, const char *D,
+                   const char *ppcond,
+                   PTHSpellingSearch &mySpellingSrch, PTHManager &PM)
+  : PreprocessorLexer(&PP, FID), TokBuf(D), CurPtr(D), LastHashTokPtr(0),
     PPCond(ppcond), CurPPCondPtr(ppcond), MySpellingSrch(mySpellingSrch),
-    PTHMgr(PM)
-{      
-  FileID = fileloc.getFileID();
+    PTHMgr(PM) {
+      
+  FileStartLoc = PP.getSourceManager().getLocForStartOfFile(FID);
 }
 
 void PTHLexer::Lex(Token& Tok) {
@@ -96,7 +94,7 @@ LexNextToken:
   Tok.setFlag(flags);
   assert(!LexingRawMode);
   Tok.setIdentifierInfo(perID ? PTHMgr.GetIdentifierInfo(perID-1) : 0);
-  Tok.setLocation(SourceLocation::getFileLoc(FileID, FileOffset));
+  Tok.setLocation(FileStartLoc.getFileLocWithOffset(FileOffset));
   Tok.setLength(Len);
 
   //===--------------------------------------==//
@@ -295,23 +293,27 @@ SourceLocation PTHLexer::getSourceLocation() {
     | (((uint32_t) ((uint8_t) p[1])) << 8)
     | (((uint32_t) ((uint8_t) p[2])) << 16)
     | (((uint32_t) ((uint8_t) p[3])) << 24);
-  return SourceLocation::getFileLoc(FileID, offset);
+  return FileStartLoc.getFileLocWithOffset(offset);
 }
 
 //===----------------------------------------------------------------------===//
 // getSpelling() - Use cached data in PTH files for getSpelling().
 //===----------------------------------------------------------------------===//
 
-unsigned PTHManager::getSpelling(unsigned FileID, unsigned fpos,
-                                 const char *& Buffer) {
-  
-  llvm::DenseMap<unsigned,PTHSpellingSearch*>::iterator I =
-    SpellingMap.find(FileID);
+unsigned PTHManager::getSpelling(FileID FID, unsigned FPos,
+                                 const char *&Buffer) {
+  llvm::DenseMap<FileID, PTHSpellingSearch*>::iterator I =SpellingMap.find(FID);
 
   if (I == SpellingMap.end())
       return 0;
 
-  return I->second->getSpellingBinarySearch(fpos, Buffer);
+  return I->second->getSpellingBinarySearch(FPos, Buffer);
+}
+
+unsigned PTHManager::getSpelling(SourceLocation Loc, const char *&Buffer) {
+  std::pair<FileID, unsigned> LocInfo =
+    PP->getSourceManager().getDecomposedFileLoc(Loc);
+  return getSpelling(LocInfo.first, LocInfo.second, Buffer);
 }
 
 unsigned PTHManager::getSpellingAtPTHOffset(unsigned PTHOffset,
@@ -420,14 +422,17 @@ unsigned PTHSpellingSearch::getSpellingBinarySearch(unsigned fpos,
   return 0;
 }
 
-unsigned PTHLexer::getSpelling(SourceLocation sloc, const char *&Buffer) {
-  SourceManager& SM = PP->getSourceManager();
-  sloc = SM.getSpellingLoc(sloc);
-  unsigned fid = SM.getCanonicalFileID(sloc);
-  unsigned fpos = SM.getFullFilePos(sloc);
+unsigned PTHLexer::getSpelling(SourceLocation Loc, const char *&Buffer) {
+  SourceManager &SM = PP->getSourceManager();
+  Loc = SM.getSpellingLoc(Loc);
+  std::pair<FileID, unsigned> LocInfo = SM.getDecomposedFileLoc(Loc);
+
+  FileID FID = LocInfo.first;
+  unsigned FPos = LocInfo.second;
   
-  return (fid == FileID ) ? MySpellingSrch.getSpellingLinearSearch(fpos, Buffer)
-                          : PTHMgr.getSpelling(fid, fpos, Buffer);  
+  if (FID == getFileID())
+    return MySpellingSrch.getSpellingLinearSearch(FPos, Buffer);
+  return PTHMgr.getSpelling(FID, FPos, Buffer);  
 }
 
 //===----------------------------------------------------------------------===//
@@ -662,15 +667,14 @@ IdentifierInfo* PTHManager::get(const char *NameStart, const char *NameEnd) {
 }
 
 
-PTHLexer* PTHManager::CreateLexer(unsigned FileID, const FileEntry* FE) {
-  
+PTHLexer* PTHManager::CreateLexer(FileID FID, const FileEntry* FE) {
   if (!FE)
     return 0;
   
   // Lookup the FileEntry object in our file lookup data structure.  It will
   // return a variant that indicates whether or not there is an offset within
   // the PTH file that contains cached tokens.
-  PTHFileLookup::Val FileData = ((PTHFileLookup*) FileLookup)->Lookup(FE);
+  PTHFileLookup::Val FileData = ((PTHFileLookup*)FileLookup)->Lookup(FE);
   
   if (!FileData.isValid()) // No tokens available.
     return 0;
@@ -694,9 +698,8 @@ PTHLexer* PTHManager::CreateLexer(unsigned FileID, const FileEntry* FE) {
   
   // Create the SpellingSearch object for this FileID.
   PTHSpellingSearch* ss = new PTHSpellingSearch(*this, len, spellingTable);
-  SpellingMap[FileID] = ss;
+  SpellingMap[FID] = ss;
   
   assert(PP && "No preprocessor set yet!");
-  return new PTHLexer(*PP, SourceLocation::getFileLoc(FileID, 0), data, ppcond,
-                      *ss, *this); 
+  return new PTHLexer(*PP, FID, data, ppcond, *ss, *this); 
 }
diff --git a/lib/Lex/Pragma.cpp b/lib/Lex/Pragma.cpp
index 667e4361a8..860301650b 100644
--- a/lib/Lex/Pragma.cpp
+++ b/lib/Lex/Pragma.cpp
@@ -191,10 +191,8 @@ void Preprocessor::HandlePragmaOnce(Token &OnceTok) {
   }
   
   // Get the current file lexer we're looking at.  Ignore _Pragma 'files' etc.
-  unsigned FileID = getCurrentFileLexer()->getFileID();
-  
   // Mark the file as a once-only file now.
-  HeaderInfo.MarkFileIncludeOnce(SourceMgr.getFileEntryForID(FileID));
+  HeaderInfo.MarkFileIncludeOnce(getCurrentFileLexer()->getFileEntry());
 }
 
 void Preprocessor::HandlePragmaMark() {
@@ -256,8 +254,7 @@ void Preprocessor::HandlePragmaSystemHeader(Token &SysHeaderTok) {
   PreprocessorLexer *TheLexer = getCurrentFileLexer();
   
   // Mark the file as a system header.
-  const FileEntry *File = SourceMgr.getFileEntryForID(TheLexer->getFileID());
-  HeaderInfo.MarkFileSystemHeader(File);
+  HeaderInfo.MarkFileSystemHeader(TheLexer->getFileEntry());
   
   // Notify the client, if desired, that we are in a new source file.
   if (Callbacks)
@@ -299,8 +296,7 @@ void Preprocessor::HandlePragmaDependency(Token &DependencyTok) {
     return;
   }
   
-  unsigned FileID = getCurrentFileLexer()->getFileID();
-  const FileEntry *CurFile = SourceMgr.getFileEntryForID(FileID);
+  const FileEntry *CurFile = getCurrentFileLexer()->getFileEntry();
 
   // If this file is older than the file it depends on, emit a diagnostic.
   if (CurFile && CurFile->getModificationTime() < File->getModificationTime()) {
diff --git a/lib/Lex/Preprocessor.cpp b/lib/Lex/Preprocessor.cpp
index bbe0f5c0f9..976e1a2f7a 100644
--- a/lib/Lex/Preprocessor.cpp
+++ b/lib/Lex/Preprocessor.cpp
@@ -200,9 +200,7 @@ std::string Preprocessor::getSpelling(const Token &Tok) const {
   
   if (PTH) {
     SourceLocation SLoc = SourceMgr.getSpellingLoc(Tok.getLocation());
-    unsigned fid = SourceMgr.getCanonicalFileID(SLoc);
-    unsigned fpos = SourceMgr.getFullFilePos(SLoc);
-    if (unsigned Len = PTH->getSpelling(fid, fpos, TokStart)) {
+    if (unsigned Len = PTH->getSpelling(SLoc, TokStart)) {
       assert(!Tok.needsCleaning());
       return std::string(TokStart, TokStart+Len);
     }
@@ -256,10 +254,8 @@ unsigned Preprocessor::getSpelling(const Token &Tok,
     if (CurPTHLexer) {
       Len = CurPTHLexer.get()->getSpelling(Tok.getLocation(), Buffer);      
     } else {
-      SourceLocation SLoc = SourceMgr.getSpellingLoc(Tok.getLocation());
-      unsigned FID = SourceMgr.getCanonicalFileID(SLoc);
-      unsigned FPos = SourceMgr.getFullFilePos(SLoc);      
-      Len = PTH->getSpelling(FID, FPos, Buffer);      
+      Len = PTH->getSpelling(SourceMgr.getSpellingLoc(Tok.getLocation()),
+                             Buffer);      
     }
 
     // Did we find a spelling?  If so return its length.  Otherwise fall
@@ -656,15 +652,14 @@ static void InitializePredefinedMacros(Preprocessor &PP,
 /// which implicitly adds the builtin defines etc.
 void Preprocessor::EnterMainSourceFile() {
   
-  unsigned MainFileID = SourceMgr.getMainFileID();
+  FileID MainFileID = SourceMgr.getMainFileID();
   
   // Enter the main file source buffer.
   EnterSourceFile(MainFileID, 0);
   
   // Tell the header info that the main file was entered.  If the file is later
   // #imported, it won't be re-entered.
-  if (const FileEntry *FE = 
-        SourceMgr.getFileEntryForLoc(SourceLocation::getFileLoc(MainFileID, 0)))
+  if (const FileEntry *FE = SourceMgr.getFileEntryForID(MainFileID))
     HeaderInfo.IncrementIncludeCount(FE);
     
   std::vector<char> PrologFile;
@@ -685,11 +680,11 @@ void Preprocessor::EnterMainSourceFile() {
     llvm::MemoryBuffer::getMemBufferCopy(&PrologFile.front(),&PrologFile.back(),
                                          "<predefines>");
   assert(SB && "Cannot fail to create predefined source buffer");
-  unsigned FileID = SourceMgr.createFileIDForMemBuffer(SB);
-  assert(FileID && "Could not create FileID for predefines?");
+  FileID FID = SourceMgr.createFileIDForMemBuffer(SB);
+  assert(!FID.isInvalid() && "Could not create FileID for predefines?");
   
   // Start parsing the predefines.
-  EnterSourceFile(FileID, 0);
+  EnterSourceFile(FID, 0);
 }
 
 
diff --git a/lib/Lex/PreprocessorLexer.cpp b/lib/Lex/PreprocessorLexer.cpp
index 07329e0cc7..e98afccbba 100644
--- a/lib/Lex/PreprocessorLexer.cpp
+++ b/lib/Lex/PreprocessorLexer.cpp
@@ -17,14 +17,6 @@
 #include "clang/Basic/SourceManager.h"
 using namespace clang;
 
-PreprocessorLexer::PreprocessorLexer(Preprocessor* pp, SourceLocation L)
-  :  PP(pp), FileID(pp->getSourceManager().getSpellingLoc(L).getFileID()),
-     ParsingPreprocessorDirective(false),
-     ParsingFilename(false),
-     LexingRawMode(false) {}
-
-PreprocessorLexer::~PreprocessorLexer() {}
-
 /// LexIncludeFilename - After the preprocessor has parsed a #include, lex and
 /// (potentially) macro expand the filename.
 void PreprocessorLexer::LexIncludeFilename(Token &FilenameTok) {
@@ -45,3 +37,9 @@ void PreprocessorLexer::LexIncludeFilename(Token &FilenameTok) {
   if (FilenameTok.is(tok::eom))
     PP->Diag(FilenameTok.getLocation(), diag::err_pp_expects_filename);
 }
+
+/// getFileEntry - Return the FileEntry corresponding to this FileID.  Like
+/// getFileID(), this only works for lexers with attached preprocessors.
+const FileEntry *PreprocessorLexer::getFileEntry() const {
+  return PP->getSourceManager().getFileEntryForID(getFileID());
+}
diff --git a/lib/Lex/ScratchBuffer.cpp b/lib/Lex/ScratchBuffer.cpp
index 99fbdf7565..ec07a71baf 100644
--- a/lib/Lex/ScratchBuffer.cpp
+++ b/lib/Lex/ScratchBuffer.cpp
@@ -24,7 +24,6 @@ static const unsigned ScratchBufSize = 4060;
 ScratchBuffer::ScratchBuffer(SourceManager &SM) : SourceMgr(SM), CurBuffer(0) {
   // Set BytesUsed so that the first call to getToken will require an alloc.
   BytesUsed = ScratchBufSize;
-  FileID = 0;
 }
 
 /// getToken - Splat the specified text into a temporary MemoryBuffer and
@@ -44,7 +43,7 @@ SourceLocation ScratchBuffer::getToken(const char *Buf, unsigned Len) {
   assert(BytesUsed-Len < (1 << SourceLocation::FilePosBits) &&
          "Out of range file position!");
   
-  return SourceLocation::getFileLoc(FileID, BytesUsed-Len);
+  return BufferStartLoc.getFileLocWithOffset(BytesUsed-Len);
 }
 
 
@@ -66,7 +65,8 @@ void ScratchBuffer::AllocScratchBuffer(unsigned RequestLen) {
   
   llvm::MemoryBuffer *Buf = 
     llvm::MemoryBuffer::getNewMemBuffer(RequestLen, "<scratch space>");
-  FileID = SourceMgr.createFileIDForMemBuffer(Buf);
+  FileID FID = SourceMgr.createFileIDForMemBuffer(Buf);
+  BufferStartLoc = SourceMgr.getLocForStartOfFile(FID);
   CurBuffer = const_cast<char*>(Buf->getBufferStart());
   BytesUsed = 0;
 }
diff --git a/lib/Lex/TokenLexer.cpp b/lib/Lex/TokenLexer.cpp
index 6399e867b3..fde29359fd 100644
--- a/lib/Lex/TokenLexer.cpp
+++ b/lib/Lex/TokenLexer.cpp
@@ -392,8 +392,7 @@ bool TokenLexer::PasteTokens(Token &Tok) {
       SourceManager &SourceMgr = PP.getSourceManager();
       const char *ResultStrData = SourceMgr.getCharacterData(ResultTokLoc);
       
-      const llvm::MemoryBuffer *Buffer = 
-        SourceMgr.getBuffer(ResultTokLoc.getFileID());
+      const llvm::MemoryBuffer *Buffer = SourceMgr.getBuffer(ResultTokLoc);
       
       // Make a lexer object so that we lex and expand the paste result.
       Lexer TL(ResultTokLoc, PP.getLangOptions(), ResultStrData,