At one point there were going to be lexer and parser tokens.

Since that point is now long gone, we should rename LexerToken to
Token, as it is the only kind of token we have.


git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@40105 91177308-0d34-0410-b5e6-96231b3b80d8

26 files changed, 280 insertions, 280 deletions

diff --git a/Driver/DiagChecker.cpp b/Driver/DiagChecker.cpp
index 9830c0d69e..279b55b773 100644
--- a/Driver/DiagChecker.cpp
+++ b/Driver/DiagChecker.cpp
@@ -91,7 +91,7 @@ static void FindExpectedDiags(Preprocessor &PP, unsigned MainFileID,
   // Enter the cave.
   PP.EnterSourceFile(MainFileID, 0, true);
 
-  LexerToken Tok;
+  Token Tok;
   do {
     PP.Lex(Tok);
 
diff --git a/Driver/PrintPreprocessedOutput.cpp b/Driver/PrintPreprocessedOutput.cpp
index c22167315b..46278f1a07 100644
--- a/Driver/PrintPreprocessedOutput.cpp
+++ b/Driver/PrintPreprocessedOutput.cpp
@@ -128,9 +128,9 @@ public:
   virtual void Ident(SourceLocation Loc, const std::string &str);
   
 
-  void HandleFirstTokOnLine(LexerToken &Tok);
+  void HandleFirstTokOnLine(Token &Tok);
   void MoveToLine(SourceLocation Loc);
-  bool AvoidConcat(const LexerToken &PrevTok, const LexerToken &Tok);
+  bool AvoidConcat(const Token &PrevTok, const Token &Tok);
 };
 }
 
@@ -250,7 +250,7 @@ void PrintPPOutputPPCallbacks::Ident(SourceLocation Loc, const std::string &S) {
 
 /// HandleFirstTokOnLine - When emitting a preprocessed file in -E mode, this
 /// is called for the first token on each new line.
-void PrintPPOutputPPCallbacks::HandleFirstTokOnLine(LexerToken &Tok) {
+void PrintPPOutputPPCallbacks::HandleFirstTokOnLine(Token &Tok) {
   // Figure out what line we went to and insert the appropriate number of
   // newline characters.
   MoveToLine(Tok.getLocation());
@@ -281,7 +281,7 @@ struct UnknownPragmaHandler : public PragmaHandler {
   
   UnknownPragmaHandler(const char *prefix, PrintPPOutputPPCallbacks *callbacks)
     : PragmaHandler(0), Prefix(prefix), Callbacks(callbacks) {}
-  virtual void HandlePragma(Preprocessor &PP, LexerToken &PragmaTok) {
+  virtual void HandlePragma(Preprocessor &PP, Token &PragmaTok) {
     // Figure out what line we went to and insert the appropriate number of
     // newline characters.
     Callbacks->MoveToLine(PragmaTok.getLocation());
@@ -311,8 +311,8 @@ struct UnknownPragmaHandler : public PragmaHandler {
 /// the resulting output won't have incorrect concatenations going on.  Examples
 /// include "..", which we print with a space between, because we don't want to
 /// track enough to tell "x.." from "...".
-bool PrintPPOutputPPCallbacks::AvoidConcat(const LexerToken &PrevTok,
-                                           const LexerToken &Tok) {
+bool PrintPPOutputPPCallbacks::AvoidConcat(const Token &PrevTok,
+                                           const Token &Tok) {
   char Buffer[256];
   
   // If we haven't emitted a token on this line yet, PrevTok isn't useful to
@@ -394,7 +394,7 @@ void clang::DoPrintPreprocessedInput(unsigned MainFileID, Preprocessor &PP,
   
   InitOutputBuffer();
   
-  LexerToken Tok, PrevTok;
+  Token Tok, PrevTok;
   char Buffer[256];
   PrintPPOutputPPCallbacks *Callbacks = new PrintPPOutputPPCallbacks(PP);
   PP.setPPCallbacks(Callbacks);
diff --git a/Driver/TextDiagnosticPrinter.cpp b/Driver/TextDiagnosticPrinter.cpp
index 87f33741b0..ac48e254bc 100644
--- a/Driver/TextDiagnosticPrinter.cpp
+++ b/Driver/TextDiagnosticPrinter.cpp
@@ -111,7 +111,7 @@ unsigned TextDiagnosticPrinter::GetTokenLength(SourceLocation Loc) {
   
   // Create a lexer starting at the beginning of this token.
   Lexer TheLexer(Loc, *ThePreprocessor, StrData);
-  LexerToken TheTok;
+  Token TheTok;
   TheLexer.LexRawToken(TheTok);
   return TheTok.getLength();
 }
diff --git a/Driver/clang.cpp b/Driver/clang.cpp
index 404baadf48..ac12ef7a72 100644
--- a/Driver/clang.cpp
+++ b/Driver/clang.cpp
@@ -754,7 +754,7 @@ static unsigned InitializePreprocessor(Preprocessor &PP,
   PP.EnterSourceFile(FileID, 0);
 
   // Lex the file, which will read all the macros.
-  LexerToken Tok;
+  Token Tok;
   PP.Lex(Tok);
   assert(Tok.getKind() == tok::eof && "Didn't read entire file!");
 
@@ -775,7 +775,7 @@ static void ProcessInputFile(Preprocessor &PP, unsigned MainFileID,
     fprintf(stderr, "Unexpected program action!\n");
     return;
   case DumpTokens: {                 // Token dump mode.
-    LexerToken Tok;
+    Token Tok;
     // Start parsing the specified input file.
     PP.EnterSourceFile(MainFileID, 0, true);
     do {
@@ -786,7 +786,7 @@ static void ProcessInputFile(Preprocessor &PP, unsigned MainFileID,
     break;
   }
   case RunPreprocessorOnly: {        // Just lex as fast as we can, no output.
-    LexerToken Tok;
+    Token Tok;
     // Start parsing the specified input file.
     PP.EnterSourceFile(MainFileID, 0, true);
     do {
diff --git a/Lex/Lexer.cpp b/Lex/Lexer.cpp
index 4efd62113d..1518d8895c 100644
--- a/Lex/Lexer.cpp
+++ b/Lex/Lexer.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 //
-//  This file implements the Lexer and LexerToken interfaces.
+//  This file implements the Lexer and Token interfaces.
 //
 //===----------------------------------------------------------------------===//
 //
@@ -248,7 +248,7 @@ static char DecodeTrigraphChar(const char *CP, Lexer *L) {
 /// be updated to match.
 ///
 char Lexer::getCharAndSizeSlow(const char *Ptr, unsigned &Size,
-                               LexerToken *Tok) {
+                               Token *Tok) {
   // If we have a slash, look for an escaped newline.
   if (Ptr[0] == '\\') {
     ++Size;
@@ -264,7 +264,7 @@ Slash:
         ++SizeTmp;
         if (Ptr[SizeTmp-1] == '\n' || Ptr[SizeTmp-1] == '\r') {
           // Remember that this token needs to be cleaned.
-          if (Tok) Tok->setFlag(LexerToken::NeedsCleaning);
+          if (Tok) Tok->setFlag(Token::NeedsCleaning);
 
           // Warn if there was whitespace between the backslash and newline.
           if (SizeTmp != 1 && Tok)
@@ -294,7 +294,7 @@ Slash:
     // a trigraph warning.  If so, and if trigraphs are enabled, return it.
     if (char C = DecodeTrigraphChar(Ptr+2, Tok ? this : 0)) {
       // Remember that this token needs to be cleaned.
-      if (Tok) Tok->setFlag(LexerToken::NeedsCleaning);
+      if (Tok) Tok->setFlag(Token::NeedsCleaning);
 
       Ptr += 3;
       Size += 3;
@@ -372,7 +372,7 @@ Slash:
 // Helper methods for lexing.
 //===----------------------------------------------------------------------===//
 
-void Lexer::LexIdentifier(LexerToken &Result, const char *CurPtr) {
+void Lexer::LexIdentifier(Token &Result, const char *CurPtr) {
   // Match [_A-Za-z0-9]*, we have already matched [_A-Za-z$]
   unsigned Size;
   unsigned char C = *CurPtr++;
@@ -436,7 +436,7 @@ FinishIdentifier:
 /// LexNumericConstant - Lex the remainer of a integer or floating point
 /// constant. From[-1] is the first character lexed.  Return the end of the
 /// constant.
-void Lexer::LexNumericConstant(LexerToken &Result, const char *CurPtr) {
+void Lexer::LexNumericConstant(Token &Result, const char *CurPtr) {
   unsigned Size;
   char C = getCharAndSize(CurPtr, Size);
   char PrevCh = 0;
@@ -463,7 +463,7 @@ void Lexer::LexNumericConstant(LexerToken &Result, const char *CurPtr) {
 
 /// LexStringLiteral - Lex the remainder of a string literal, after having lexed
 /// either " or L".
-void Lexer::LexStringLiteral(LexerToken &Result, const char *CurPtr, bool Wide){
+void Lexer::LexStringLiteral(Token &Result, const char *CurPtr, bool Wide){
   const char *NulCharacter = 0; // Does this string contain the \0 character?
   
   char C = getAndAdvanceChar(CurPtr, Result);
@@ -495,7 +495,7 @@ void Lexer::LexStringLiteral(LexerToken &Result, const char *CurPtr, bool Wide){
 
 /// LexAngledStringLiteral - Lex the remainder of an angled string literal,
 /// after having lexed the '<' character.  This is used for #include filenames.
-void Lexer::LexAngledStringLiteral(LexerToken &Result, const char *CurPtr) {
+void Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) {
   const char *NulCharacter = 0; // Does this string contain the \0 character?
   
   char C = getAndAdvanceChar(CurPtr, Result);
@@ -528,7 +528,7 @@ void Lexer::LexAngledStringLiteral(LexerToken &Result, const char *CurPtr) {
 
 /// LexCharConstant - Lex the remainder of a character constant, after having
 /// lexed either ' or L'.
-void Lexer::LexCharConstant(LexerToken &Result, const char *CurPtr) {
+void Lexer::LexCharConstant(Token &Result, const char *CurPtr) {
   const char *NulCharacter = 0; // Does this character contain the \0 character?
 
   // Handle the common case of 'x' and '\y' efficiently.
@@ -576,7 +576,7 @@ void Lexer::LexCharConstant(LexerToken &Result, const char *CurPtr) {
 
 /// SkipWhitespace - Efficiently skip over a series of whitespace characters.
 /// Update BufferPtr to point to the next non-whitespace character and return.
-void Lexer::SkipWhitespace(LexerToken &Result, const char *CurPtr) {
+void Lexer::SkipWhitespace(Token &Result, const char *CurPtr) {
   // Whitespace - Skip it, then return the token after the whitespace.
   unsigned char Char = *CurPtr;  // Skip consequtive spaces efficiently.
   while (1) {
@@ -596,16 +596,16 @@ void Lexer::SkipWhitespace(LexerToken &Result, const char *CurPtr) {
     
     // ok, but handle newline.
     // The returned token is at the start of the line.
-    Result.setFlag(LexerToken::StartOfLine);
+    Result.setFlag(Token::StartOfLine);
     // No leading whitespace seen so far.
-    Result.clearFlag(LexerToken::LeadingSpace);
+    Result.clearFlag(Token::LeadingSpace);
     Char = *++CurPtr;
   }
 
   // If this isn't immediately after a newline, there is leading space.
   char PrevChar = CurPtr[-1];
   if (PrevChar != '\n' && PrevChar != '\r')
-    Result.setFlag(LexerToken::LeadingSpace);
+    Result.setFlag(Token::LeadingSpace);
 
   // If the next token is obviously a // or /* */ comment, skip it efficiently
   // too (without going through the big switch stmt).
@@ -625,7 +625,7 @@ void Lexer::SkipWhitespace(LexerToken &Result, const char *CurPtr) {
 // SkipBCPLComment - We have just read the // characters from input.  Skip until
 // we find the newline character thats terminate the comment.  Then update
 /// BufferPtr and return.
-bool Lexer::SkipBCPLComment(LexerToken &Result, const char *CurPtr) {
+bool Lexer::SkipBCPLComment(Token &Result, const char *CurPtr) {
   // If BCPL comments aren't explicitly enabled for this language, emit an
   // extension warning.
   if (!Features.BCPLComment) {
@@ -704,15 +704,15 @@ bool Lexer::SkipBCPLComment(LexerToken &Result, const char *CurPtr) {
   ++CurPtr;
     
   // The next returned token is at the start of the line.
-  Result.setFlag(LexerToken::StartOfLine);
+  Result.setFlag(Token::StartOfLine);
   // No leading whitespace seen so far.
-  Result.clearFlag(LexerToken::LeadingSpace);
+  Result.clearFlag(Token::LeadingSpace);
     
   // It is common for the tokens immediately after a // comment to be
   // whitespace (indentation for the next line).  Instead of going through the
   // big switch, handle it efficiently now.
   if (isWhitespace(*CurPtr)) {
-    Result.setFlag(LexerToken::LeadingSpace);
+    Result.setFlag(Token::LeadingSpace);
     SkipWhitespace(Result, CurPtr+1);
     return true;
   }
@@ -723,7 +723,7 @@ bool Lexer::SkipBCPLComment(LexerToken &Result, const char *CurPtr) {
 
 /// SaveBCPLComment - If in save-comment mode, package up this BCPL comment in
 /// an appropriate way and return it.
-bool Lexer::SaveBCPLComment(LexerToken &Result, const char *CurPtr) {
+bool Lexer::SaveBCPLComment(Token &Result, const char *CurPtr) {
   Result.setKind(tok::comment);
   FormTokenWithChars(Result, CurPtr);
   
@@ -812,7 +812,7 @@ static bool isEndOfBlockCommentWithEscapedNewLine(const char *CurPtr,
 /// because they cannot cause the comment to end.  The only thing that can
 /// happen is the comment could end with an escaped newline between the */ end
 /// of comment.
-bool Lexer::SkipBlockComment(LexerToken &Result, const char *CurPtr) {
+bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr) {
   // Scan one character past where we should, looking for a '/' character.  Once
   // we find it, check to see if it was preceeded by a *.  This common
   // optimization helps people who like to put a lot of * characters in their
@@ -907,14 +907,14 @@ bool Lexer::SkipBlockComment(LexerToken &Result, const char *CurPtr) {
   // whitespace.  Instead of going through the big switch, handle it
   // efficiently now.
   if (isHorizontalWhitespace(*CurPtr)) {
-    Result.setFlag(LexerToken::LeadingSpace);
+    Result.setFlag(Token::LeadingSpace);
     SkipWhitespace(Result, CurPtr+1);
     return true;
   }
 
   // Otherwise, just return so that the next character will be lexed as a token.
   BufferPtr = CurPtr;
-  Result.setFlag(LexerToken::LeadingSpace);
+  Result.setFlag(Token::LeadingSpace);
   return true;
 }
 
@@ -924,7 +924,7 @@ bool Lexer::SkipBlockComment(LexerToken &Result, const char *CurPtr) {
 
 /// LexIncludeFilename - After the preprocessor has parsed a #include, lex and
 /// (potentially) macro expand the filename.
-void Lexer::LexIncludeFilename(LexerToken &FilenameTok) {
+void Lexer::LexIncludeFilename(Token &FilenameTok) {
   assert(ParsingPreprocessorDirective &&
          ParsingFilename == false &&
          "Must be in a preprocessing directive!");
@@ -949,7 +949,7 @@ std::string Lexer::ReadToEndOfLine() {
   assert(ParsingPreprocessorDirective && ParsingFilename == false &&
          "Must be in a preprocessing directive!");
   std::string Result;
-  LexerToken Tmp;
+  Token Tmp;
 
   // CurPtr - Cache BufferPtr in an automatic variable.
   const char *CurPtr = BufferPtr;
@@ -987,7 +987,7 @@ std::string Lexer::ReadToEndOfLine() {
 /// condition, reporting diagnostics and handling other edge cases as required.
 /// This returns true if Result contains a token, false if PP.Lex should be
 /// called again.
-bool Lexer::LexEndOfFile(LexerToken &Result, const char *CurPtr) {
+bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) {
   // If we hit the end of the file while parsing a preprocessor directive,
   // end the preprocessor directive first.  The next token returned will
   // then be the end of file.
@@ -1046,7 +1046,7 @@ unsigned Lexer::isNextPPTokenLParen() {
   // Save state that can be changed while lexing so that we can restore it.
   const char *TmpBufferPtr = BufferPtr;
   
-  LexerToken Tok;
+  Token Tok;
   Tok.startToken();
   LexTokenInternal(Tok);
   
@@ -1069,10 +1069,10 @@ unsigned Lexer::isNextPPTokenLParen() {
 /// preprocessing token, not a normal token, as such, it is an internal
 /// interface.  It assumes that the Flags of result have been cleared before
 /// calling this.
-void Lexer::LexTokenInternal(LexerToken &Result) {
+void Lexer::LexTokenInternal(Token &Result) {
 LexNextToken:
   // New token, can't need cleaning yet.
-  Result.clearFlag(LexerToken::NeedsCleaning);
+  Result.clearFlag(Token::NeedsCleaning);
   Result.setIdentifierInfo(0);
   
   // CurPtr - Cache BufferPtr in an automatic variable.
@@ -1084,7 +1084,7 @@ LexNextToken:
     while ((*CurPtr == ' ') || (*CurPtr == '\t'))
       ++CurPtr;
     BufferPtr = CurPtr;
-    Result.setFlag(LexerToken::LeadingSpace);
+    Result.setFlag(Token::LeadingSpace);
   }
   
   unsigned SizeTmp, SizeTmp2;   // Temporaries for use in cases below.
@@ -1104,7 +1104,7 @@ LexNextToken:
     }
     
     Diag(CurPtr-1, diag::null_in_file);
-    Result.setFlag(LexerToken::LeadingSpace);
+    Result.setFlag(Token::LeadingSpace);
     SkipWhitespace(Result, CurPtr);
     goto LexNextToken;   // GCC isn't tail call eliminating.
   case '\n':
@@ -1125,16 +1125,16 @@ LexNextToken:
       break;
     }
     // The returned token is at the start of the line.
-    Result.setFlag(LexerToken::StartOfLine);
+    Result.setFlag(Token::StartOfLine);
     // No leading whitespace seen so far.
-    Result.clearFlag(LexerToken::LeadingSpace);
+    Result.clearFlag(Token::LeadingSpace);
     SkipWhitespace(Result, CurPtr);
     goto LexNextToken;   // GCC isn't tail call eliminating.
   case ' ':
   case '\t':
   case '\f':
   case '\v':
-    Result.setFlag(LexerToken::LeadingSpace);
+    Result.setFlag(Token::LeadingSpace);
     SkipWhitespace(Result, CurPtr);
     goto LexNextToken;   // GCC isn't tail call eliminating.
 
@@ -1346,7 +1346,7 @@ LexNextToken:
             // want us starting at the beginning of the line again.  If so, set
             // the StartOfLine flag.
             if (IsAtStartOfLine) {
-              Result.setFlag(LexerToken::StartOfLine);
+              Result.setFlag(Token::StartOfLine);
               IsAtStartOfLine = false;
             }
             goto LexNextToken;   // GCC isn't tail call eliminating.
@@ -1475,7 +1475,7 @@ LexNextToken:
           // want us starting at the beginning of the line again.  If so, set
           // the StartOfLine flag.
           if (IsAtStartOfLine) {
-            Result.setFlag(LexerToken::StartOfLine);
+            Result.setFlag(Token::StartOfLine);
             IsAtStartOfLine = false;
           }
           goto LexNextToken;   // GCC isn't tail call eliminating.
diff --git a/Lex/LiteralSupport.cpp b/Lex/LiteralSupport.cpp
index cd2082a3cb..76a70ab7aa 100644
--- a/Lex/LiteralSupport.cpp
+++ b/Lex/LiteralSupport.cpp
@@ -535,7 +535,7 @@ CharLiteralParser::CharLiteralParser(const char *begin, const char *end,
 ///         hex-digit hex-digit hex-digit hex-digit
 ///
 StringLiteralParser::
-StringLiteralParser(const LexerToken *StringToks, unsigned NumStringToks,
+StringLiteralParser(const Token *StringToks, unsigned NumStringToks,
                     Preprocessor &pp, TargetInfo &t)
   : PP(pp), Target(t) {
   // Scan all of the string portions, remember the max individual token length,
diff --git a/Lex/MacroExpander.cpp b/Lex/MacroExpander.cpp
index 53ff3f14ca..fade854657 100644
--- a/Lex/MacroExpander.cpp
+++ b/Lex/MacroExpander.cpp
@@ -25,21 +25,21 @@ using namespace clang;
 
 /// MacroArgs ctor function - This destroys the vector passed in.
 MacroArgs *MacroArgs::create(const MacroInfo *MI,
-                             const LexerToken *UnexpArgTokens,
+                             const Token *UnexpArgTokens,
                              unsigned NumToks, bool VarargsElided) {
   assert(MI->isFunctionLike() &&
          "Can't have args for an object-like macro!");
 
   // Allocate memory for the MacroArgs object with the lexer tokens at the end.
   MacroArgs *Result = (MacroArgs*)malloc(sizeof(MacroArgs) +
-                                         NumToks*sizeof(LexerToken));
+                                         NumToks*sizeof(Token));
   // Construct the macroargs object.
   new (Result) MacroArgs(NumToks, VarargsElided);
   
   // Copy the actual unexpanded tokens to immediately after the result ptr.
   if (NumToks)
-    memcpy(const_cast<LexerToken*>(Result->getUnexpArgument(0)),
-           UnexpArgTokens, NumToks*sizeof(LexerToken));
+    memcpy(const_cast<Token*>(Result->getUnexpArgument(0)),
+           UnexpArgTokens, NumToks*sizeof(Token));
   
   return Result;
 }
@@ -57,7 +57,7 @@ void MacroArgs::destroy() {
 /// getArgLength - Given a pointer to an expanded or unexpanded argument,
 /// return the number of tokens, not counting the EOF, that make up the
 /// argument.
-unsigned MacroArgs::getArgLength(const LexerToken *ArgPtr) {
+unsigned MacroArgs::getArgLength(const Token *ArgPtr) {
   unsigned NumArgTokens = 0;
   for (; ArgPtr->getKind() != tok::eof; ++ArgPtr)
     ++NumArgTokens;
@@ -67,11 +67,11 @@ unsigned MacroArgs::getArgLength(const LexerToken *ArgPtr) {
 
 /// getUnexpArgument - Return the unexpanded tokens for the specified formal.
 ///
-const LexerToken *MacroArgs::getUnexpArgument(unsigned Arg) const {
+const Token *MacroArgs::getUnexpArgument(unsigned Arg) const {
   // The unexpanded argument tokens start immediately after the MacroArgs object
   // in memory.
-  const LexerToken *Start = (const LexerToken *)(this+1);
-  const LexerToken *Result = Start;
+  const Token *Start = (const Token *)(this+1);
+  const Token *Result = Start;
   // Scan to find Arg.
   for (; Arg; ++Result) {
     assert(Result < Start+NumUnexpArgTokens && "Invalid arg #");
@@ -84,7 +84,7 @@ const LexerToken *MacroArgs::getUnexpArgument(unsigned Arg) const {
 
 /// ArgNeedsPreexpansion - If we can prove that the argument won't be affected
 /// by pre-expansion, return false.  Otherwise, conservatively return true.
-bool MacroArgs::ArgNeedsPreexpansion(const LexerToken *ArgTok) const {
+bool MacroArgs::ArgNeedsPreexpansion(const Token *ArgTok) const {
   // If there are no identifiers in the argument list, or if the identifiers are
   // known to not be macros, pre-expansion won't modify it.
   for (; ArgTok->getKind() != tok::eof; ++ArgTok)
@@ -99,7 +99,7 @@ bool MacroArgs::ArgNeedsPreexpansion(const LexerToken *ArgTok) const {
 
 /// getPreExpArgument - Return the pre-expanded form of the specified
 /// argument.
-const std::vector<LexerToken> &
+const std::vector<Token> &
 MacroArgs::getPreExpArgument(unsigned Arg, Preprocessor &PP) {
   assert(Arg < NumUnexpArgTokens && "Invalid argument number!");
   
@@ -107,10 +107,10 @@ MacroArgs::getPreExpArgument(unsigned Arg, Preprocessor &PP) {
   if (PreExpArgTokens.empty())
     PreExpArgTokens.resize(NumUnexpArgTokens);
 
-  std::vector<LexerToken> &Result = PreExpArgTokens[Arg];
+  std::vector<Token> &Result = PreExpArgTokens[Arg];
   if (!Result.empty()) return Result;
 
-  const LexerToken *AT = getUnexpArgument(Arg);
+  const Token *AT = getUnexpArgument(Arg);
   unsigned NumToks = getArgLength(AT)+1;  // Include the EOF.
   
   // Otherwise, we have to pre-expand this argument, populating Result.  To do
@@ -121,7 +121,7 @@ MacroArgs::getPreExpArgument(unsigned Arg, Preprocessor &PP) {
 
   // Lex all of the macro-expanded tokens into Result.
   do {
-    Result.push_back(LexerToken());
+    Result.push_back(Token());
     PP.Lex(Result.back());
   } while (Result.back().getKind() != tok::eof);
   
@@ -139,20 +139,20 @@ MacroArgs::getPreExpArgument(unsigned Arg, Preprocessor &PP) {
 /// tokens into the literal string token that should be produced by the C #
 /// preprocessor operator.
 ///
-static LexerToken StringifyArgument(const LexerToken *ArgToks,
+static Token StringifyArgument(const Token *ArgToks,
                                     Preprocessor &PP, bool Charify = false) {
-  LexerToken Tok;
+  Token Tok;
   Tok.startToken();
   Tok.setKind(tok::string_literal);
 
-  const LexerToken *ArgTokStart = ArgToks;
+  const Token *ArgTokStart = ArgToks;
   
   // Stringify all the tokens.
   std::string Result = "\"";
   // FIXME: Optimize this loop to not use std::strings.
   bool isFirst = true;
   for (; ArgToks->getKind() != tok::eof; ++ArgToks) {
-    const LexerToken &Tok = *ArgToks;
+    const Token &Tok = *ArgToks;
     if (!isFirst && (Tok.hasLeadingSpace() || Tok.isAtStartOfLine()))
       Result += ' ';
     isFirst = false;
@@ -214,7 +214,7 @@ static LexerToken StringifyArgument(const LexerToken *ArgToks,
 
 /// getStringifiedArgument - Compute, cache, and return the specified argument
 /// that has been 'stringified' as required by the # operator.
-const LexerToken &MacroArgs::getStringifiedArgument(unsigned ArgNo,
+const Token &MacroArgs::getStringifiedArgument(unsigned ArgNo,
                                                     Preprocessor &PP) {
   assert(ArgNo < NumUnexpArgTokens && "Invalid argument number!");
   if (StringifiedArgs.empty()) {
@@ -233,7 +233,7 @@ const LexerToken &MacroArgs::getStringifiedArgument(unsigned ArgNo,
 
 /// Create a macro expander for the specified macro with the specified actual
 /// arguments.  Note that this ctor takes ownership of the ActualArgs pointer.
-void MacroExpander::Init(LexerToken &Tok, MacroArgs *Actuals) {
+void MacroExpander::Init(Token &Tok, MacroArgs *Actuals) {
   // If the client is reusing a macro expander, make sure to free any memory
   // associated with it.
   destroy();
@@ -262,7 +262,7 @@ void MacroExpander::Init(LexerToken &Tok, MacroArgs *Actuals) {
 
 /// Create a macro expander for the specified token stream.  This does not
 /// take ownership of the specified token vector.
-void MacroExpander::Init(const LexerToken *TokArray, unsigned NumToks) {
+void MacroExpander::Init(const Token *TokArray, unsigned NumToks) {
   // If the client is reusing a macro expander, make sure to free any memory
   // associated with it.
   destroy();
@@ -298,7 +298,7 @@ void MacroExpander::destroy() {
 /// Expand the arguments of a function-like macro so that we can quickly
 /// return preexpanded tokens from MacroTokens.
 void MacroExpander::ExpandFunctionArguments() {
-  llvm::SmallVector<LexerToken, 128> ResultToks;
+  llvm::SmallVector<Token, 128> ResultToks;
   
   // Loop through the MacroTokens tokens, expanding them into ResultToks.  Keep
   // track of whether we change anything.  If not, no need to keep them.  If so,
@@ -314,12 +314,12 @@ void MacroExpander::ExpandFunctionArguments() {
     // If we found the stringify operator, get the argument stringified.  The
     // preprocessor already verified that the following token is a macro name
     // when the #define was parsed.
-    const LexerToken &CurTok = MacroTokens[i];
+    const Token &CurTok = MacroTokens[i];
     if (CurTok.getKind() == tok::hash || CurTok.getKind() == tok::hashat) {
       int ArgNo = Macro->getArgumentNum(MacroTokens[i+1].getIdentifierInfo());
       assert(ArgNo != -1 && "Token following # is not an argument?");
     
-      LexerToken Res;
+      Token Res;
       if (CurTok.getKind() == tok::hash)  // Stringify
         Res = ActualArgs->getStringifiedArgument(ArgNo, PP);
       else {
@@ -330,7 +330,7 @@ void MacroExpander::ExpandFunctionArguments() {
       // The stringified/charified string leading space flag gets set to match
       // the #/#@ operator.
       if (CurTok.hasLeadingSpace() || NextTokGetsSpace)
-        Res.setFlag(LexerToken::LeadingSpace);
+        Res.setFlag(Token::LeadingSpace);
       
       ResultToks.push_back(Res);
       MadeChange = true;
@@ -348,7 +348,7 @@ void MacroExpander::ExpandFunctionArguments() {
       ResultToks.push_back(CurTok);
 
       if (NextTokGetsSpace) {
-        ResultToks.back().setFlag(LexerToken::LeadingSpace);
+        ResultToks.back().setFlag(Token::LeadingSpace);
         NextTokGetsSpace = false;
       }
       continue;
@@ -368,11 +368,11 @@ void MacroExpander::ExpandFunctionArguments() {
     // argument and substitute the expanded tokens into the result.  This is
     // C99 6.10.3.1p1.
     if (!PasteBefore && !PasteAfter) {
-      const LexerToken *ResultArgToks;
+      const Token *ResultArgToks;
 
       // Only preexpand the argument if it could possibly need it.  This
       // avoids some work in common cases.
-      const LexerToken *ArgTok = ActualArgs->getUnexpArgument(ArgNo);
+      const Token *ArgTok = ActualArgs->getUnexpArgument(ArgNo);
       if (ActualArgs->ArgNeedsPreexpansion(ArgTok))
         ResultArgToks = &ActualArgs->getPreExpArgument(ArgNo, PP)[0];
       else
@@ -387,7 +387,7 @@ void MacroExpander::ExpandFunctionArguments() {
         // If any tokens were substituted from the argument, the whitespace
         // before the first token should match the whitespace of the arg
         // identifier.
-        ResultToks[FirstResult].setFlagValue(LexerToken::LeadingSpace,
+        ResultToks[FirstResult].setFlagValue(Token::LeadingSpace,
                                              CurTok.hasLeadingSpace() ||
                                              NextTokGetsSpace);
         NextTokGetsSpace = false;
@@ -401,7 +401,7 @@ void MacroExpander::ExpandFunctionArguments() {
     
     // Okay, we have a token that is either the LHS or RHS of a paste (##)
     // argument.  It gets substituted as its non-pre-expanded tokens.
-    const LexerToken *ArgToks = ActualArgs->getUnexpArgument(ArgNo);
+    const Token *ArgToks = ActualArgs->getUnexpArgument(ArgNo);
     unsigned NumToks = MacroArgs::getArgLength(ArgToks);
     if (NumToks) {  // Not an empty argument?
       ResultToks.append(ArgToks, ArgToks+NumToks);
@@ -409,7 +409,7 @@ void MacroExpander::ExpandFunctionArguments() {
       // If the next token was supposed to get leading whitespace, ensure it has
       // it now.
       if (NextTokGetsSpace) {
-        ResultToks[ResultToks.size()-NumToks].setFlag(LexerToken::LeadingSpace);
+        ResultToks[ResultToks.size()-NumToks].setFlag(Token::LeadingSpace);
         NextTokGetsSpace = false;
       }
       continue;
@@ -451,16 +451,16 @@ void MacroExpander::ExpandFunctionArguments() {
   if (MadeChange) {
     // This is deleted in the dtor.
     NumMacroTokens = ResultToks.size();
-    LexerToken *Res = new LexerToken[ResultToks.size()];
+    Token *Res = new Token[ResultToks.size()];
     if (NumMacroTokens)
-      memcpy(Res, &ResultToks[0], NumMacroTokens*sizeof(LexerToken));
+      memcpy(Res, &ResultToks[0], NumMacroTokens*sizeof(Token));
     MacroTokens = Res;
   }
 }
 
 /// Lex - Lex and return a token from this macro stream.
 ///
-void MacroExpander::Lex(LexerToken &Tok) {
+void MacroExpander::Lex(Token &Tok) {
   // Lexing off the end of the macro, pop this macro off the expansion stack.
   if (isAtEnd()) {
     // If this is a macro (not a token stream), mark the macro enabled now
@@ -503,8 +503,8 @@ void MacroExpander::Lex(LexerToken &Tok) {
   // If this is the first token, set the lexical properties of the token to
   // match the lexical properties of the macro identifier.
   if (isFirstToken) {
-    Tok.setFlagValue(LexerToken::StartOfLine , AtStartOfLine);
-    Tok.setFlagValue(LexerToken::LeadingSpace, HasLeadingSpace);
+    Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);
+    Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace);
   }
   
   // Handle recursive expansion!
@@ -517,7 +517,7 @@ void MacroExpander::Lex(LexerToken &Tok) {
 /// PasteTokens - Tok is the LHS of a ## operator, and CurToken is the ##
 /// operator.  Read the ## and RHS, and paste the LHS/RHS together.  If there
 /// are is another ## after it, chomp it iteratively.  Return the result as Tok.
-void MacroExpander::PasteTokens(LexerToken &Tok) {
+void MacroExpander::PasteTokens(Token &Tok) {
   llvm::SmallVector<char, 128> Buffer;
   do {
     // Consume the ## operator.
@@ -526,7 +526,7 @@ void MacroExpander::PasteTokens(LexerToken &Tok) {
     assert(!isAtEnd() && "No token on the RHS of a paste operator!");
   
     // Get the RHS token.
-    const LexerToken &RHS = MacroTokens[CurToken];
+    const Token &RHS = MacroTokens[CurToken];
   
     bool isInvalid = false;
 
@@ -556,7 +556,7 @@ void MacroExpander::PasteTokens(LexerToken &Tok) {
     SourceLocation ResultTokLoc = PP.CreateString(&Buffer[0], Buffer.size());
     
     // Lex the resultant pasted token into Result.
-    LexerToken Result;
+    Token Result;
     
     // Avoid testing /*, as the lexer would think it is the start of a comment
     // and emit an error that it is unter