5 files changed, 26 insertions, 27 deletions

diff --git a/Driver/RewriteMacros.cpp b/Driver/RewriteMacros.cpp
index 5106367839..8931014081 100644
--- a/Driver/RewriteMacros.cpp
+++ b/Driver/RewriteMacros.cpp
@@ -73,7 +73,7 @@ static void LexRawTokensFromMainFile(Preprocessor &PP,
   
   Token RawTok;
   do {
-    RawLex.LexRawToken(RawTok);
+    RawLex.LexFromRawLexer(RawTok);
     
     // If we have an identifier with no identifier info for our raw token, look
     // up the indentifier info.  This is important for equality comparison of
diff --git a/include/clang/Lex/Lexer.h b/include/clang/Lex/Lexer.h
index 0ba51a6037..9b17377d7c 100644
--- a/include/clang/Lex/Lexer.h
+++ b/include/clang/Lex/Lexer.h
@@ -105,11 +105,11 @@ public:
         const char *BufStart = 0, const char *BufEnd = 0);
   
   /// Lexer constructor - Create a new raw lexer object.  This object is only
-  /// suitable for calls to 'LexRawToken'.  This lexer assumes that the
-  /// associated file buffer will outlive it, so it doesn't take ownership of
-  /// either of them.
+  /// suitable for calls to 'LexRawToken'.  This lexer assumes that the text
+  /// range will outlive it, so it doesn't take ownership of it.
   Lexer(SourceLocation FileLoc, const LangOptions &Features,
-        const char *BufStart, const char *BufEnd);
+        const char *BufStart, const char *BufEnd,
+        const llvm::MemoryBuffer *FromFile = 0);
   
   /// getFeatures - Return the language features currently enabled.  NOTE: this
   /// lexer modifies features as a file is parsed!
@@ -141,14 +141,12 @@ public:
     LexTokenInternal(Result);
   }
   
-  /// LexRawToken - Switch the lexer to raw mode, lex a token into Result and
-  /// switch it back.  Return true if the 'next character to read' pointer
-  /// points and the end of the lexer buffer, false otherwise.
-  bool LexRawToken(Token &Result) {
-    assert(!(PP && LexingRawMode) && "Already in raw mode!");
-    LexingRawMode = true;
+  /// LexFromRawLexer - Lex a token from a designated raw lexer (one with no
+  /// associated preprocessor object.  Return true if the 'next character to
+  /// read' pointer points and the end of the lexer buffer, false otherwise.
+  bool LexFromRawLexer(Token &Result) {
+    assert(LexingRawMode && "Not already in raw mode!");
     Lex(Result);
-    LexingRawMode = PP == 0;
     // Note that lexing to the end of the buffer doesn't implicitly delete the
     // lexer when in raw mode.
     return BufferPtr == BufferEnd; 
diff --git a/lib/Lex/Lexer.cpp b/lib/Lex/Lexer.cpp
index c99dc1d4b8..e930f3def2 100644
--- a/lib/Lex/Lexer.cpp
+++ b/lib/Lex/Lexer.cpp
@@ -104,15 +104,16 @@ Lexer::Lexer(SourceLocation fileloc, Preprocessor &pp,
 }
 
 /// Lexer constructor - Create a new raw lexer object.  This object is only
-/// suitable for calls to 'LexRawToken'.  This lexer assumes that the
-/// associated file buffer will outlive it, so it doesn't take ownership of it.
+/// suitable for calls to 'LexRawToken'.  This lexer assumes that the text
+/// range will outlive it, so it doesn't take ownership of it.
 Lexer::Lexer(SourceLocation fileloc, const LangOptions &features,
-             const char *BufStart, const char *BufEnd)
+             const char *BufStart, const char *BufEnd,
+             const llvm::MemoryBuffer *FromFile)
   : FileLoc(fileloc), PP(0), Features(features) {
   Is_PragmaLexer = false;
   InitCharacterInfo();
   
-  BufferStart = BufStart;
+  BufferStart = FromFile ? FromFile->getBufferStart() : BufStart;
   BufferPtr = BufStart;
   BufferEnd = BufEnd;
   
@@ -192,7 +193,7 @@ unsigned Lexer::MeasureTokenLength(SourceLocation Loc,
   // Create a lexer starting at the beginning of this token.
   Lexer TheLexer(Loc, LangOpts, StrData, BufEnd);
   Token TheTok;
-  TheLexer.LexRawToken(TheTok);
+  TheLexer.LexFromRawLexer(TheTok);
   return TheTok.getLength();
 }
 
diff --git a/lib/Lex/TokenLexer.cpp b/lib/Lex/TokenLexer.cpp
index a3fe7350f2..d3acaeaab5 100644
--- a/lib/Lex/TokenLexer.cpp
+++ b/lib/Lex/TokenLexer.cpp
@@ -396,23 +396,23 @@ bool TokenLexer::PasteTokens(Token &Tok) {
       SourceManager &SourceMgr = PP.getSourceManager();
       const char *ResultStrData = SourceMgr.getCharacterData(ResultTokLoc);
       
+      const llvm::MemoryBuffer *Buffer = 
+        SourceMgr.getBuffer(ResultTokLoc.getFileID());
+      
       // Make a lexer object so that we lex and expand the paste result.
-      Lexer *TL = new Lexer(ResultTokLoc, PP, ResultStrData, 
-                            ResultStrData+LHSLen+RHSLen /*don't include null*/);
+      Lexer TL(ResultTokLoc, PP.getLangOptions(), ResultStrData, 
+               ResultStrData+LHSLen+RHSLen /*don't include null*/, Buffer);
       
       // Lex a token in raw mode.  This way it won't look up identifiers
       // automatically, lexing off the end will return an eof token, and
       // warnings are disabled.  This returns true if the result token is the
       // entire buffer.
-      bool IsComplete = TL->LexRawToken(Result);
+      bool IsComplete = TL.LexFromRawLexer(Result);
       
       // If we got an EOF token, we didn't form even ONE token.  For example, we
       // did "/ ## /" to get "//".
       IsComplete &= Result.isNot(tok::eof);
       isInvalid = !IsComplete;
-      
-      // We're now done with the temporary lexer.
-      delete TL;
     }
     
     // If pasting the two tokens didn't form a full new token, this is an error.
diff --git a/lib/Rewrite/HTMLRewrite.cpp b/lib/Rewrite/HTMLRewrite.cpp
index b411e568f6..67afc08b9f 100644
--- a/lib/Rewrite/HTMLRewrite.cpp
+++ b/lib/Rewrite/HTMLRewrite.cpp
@@ -357,7 +357,7 @@ void html::SyntaxHighlight(Rewriter &R, unsigned FileID, Preprocessor &PP) {
   // Lex all the tokens in raw mode, to avoid entering #includes or expanding
   // macros.
   Token Tok;
-  L.LexRawToken(Tok);
+  L.LexFromRawLexer(Tok);
   
   while (Tok.isNot(tok::eof)) {
     // Since we are lexing unexpanded tokens, all tokens are from the main
@@ -398,10 +398,10 @@ void html::SyntaxHighlight(Rewriter &R, unsigned FileID, Preprocessor &PP) {
       // Eat all of the tokens until we get to the next one at the start of
       // line.
       unsigned TokEnd = TokOffs+TokLen;
-      L.LexRawToken(Tok);
+      L.LexFromRawLexer(Tok);
       while (!Tok.isAtStartOfLine() && Tok.isNot(tok::eof)) {
         TokEnd = SourceMgr.getFullFilePos(Tok.getLocation())+Tok.getLength();
-        L.LexRawToken(Tok);
+        L.LexFromRawLexer(Tok);
       }
       
       // Find end of line.  This is a hack.
@@ -413,7 +413,7 @@ void html::SyntaxHighlight(Rewriter &R, unsigned FileID, Preprocessor &PP) {
     }
     }
     
-    L.LexRawToken(Tok);
+    L.LexFromRawLexer(Tok);
   }
 }