aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Driver/DiagChecker.cpp2
-rw-r--r--Driver/PrintPreprocessedOutput.cpp14
-rw-r--r--Driver/TextDiagnosticPrinter.cpp2
-rw-r--r--Driver/clang.cpp6
-rw-r--r--Lex/Lexer.cpp68
-rw-r--r--Lex/LiteralSupport.cpp2
-rw-r--r--Lex/MacroExpander.cpp80
-rw-r--r--Lex/MacroInfo.cpp4
-rw-r--r--Lex/PPExpressions.cpp10
-rw-r--r--Lex/Pragma.cpp26
-rw-r--r--Lex/Preprocessor.cpp108
-rw-r--r--Parse/ParseDecl.cpp2
-rw-r--r--Parse/ParseExpr.cpp12
-rw-r--r--Parse/ParseInit.cpp2
-rw-r--r--Parse/ParseStmt.cpp2
-rw-r--r--Sema/Sema.h8
-rw-r--r--Sema/SemaExpr.cpp6
-rw-r--r--include/clang/Lex/Lexer.h42
-rw-r--r--include/clang/Lex/LiteralSupport.h4
-rw-r--r--include/clang/Lex/MacroExpander.h32
-rw-r--r--include/clang/Lex/MacroInfo.h10
-rw-r--r--include/clang/Lex/Pragma.h6
-rw-r--r--include/clang/Lex/Preprocessor.h82
-rw-r--r--include/clang/Lex/Token.h (renamed from include/clang/Lex/LexerToken.h)12
-rw-r--r--include/clang/Parse/Action.h8
-rw-r--r--include/clang/Parse/Parser.h10
26 files changed, 280 insertions, 280 deletions
diff --git a/Driver/DiagChecker.cpp b/Driver/DiagChecker.cpp
index 9830c0d69e..279b55b773 100644
--- a/Driver/DiagChecker.cpp
+++ b/Driver/DiagChecker.cpp
@@ -91,7 +91,7 @@ static void FindExpectedDiags(Preprocessor &PP, unsigned MainFileID,
// Enter the cave.
PP.EnterSourceFile(MainFileID, 0, true);
- LexerToken Tok;
+ Token Tok;
do {
PP.Lex(Tok);
diff --git a/Driver/PrintPreprocessedOutput.cpp b/Driver/PrintPreprocessedOutput.cpp
index c22167315b..46278f1a07 100644
--- a/Driver/PrintPreprocessedOutput.cpp
+++ b/Driver/PrintPreprocessedOutput.cpp
@@ -128,9 +128,9 @@ public:
virtual void Ident(SourceLocation Loc, const std::string &str);
- void HandleFirstTokOnLine(LexerToken &Tok);
+ void HandleFirstTokOnLine(Token &Tok);
void MoveToLine(SourceLocation Loc);
- bool AvoidConcat(const LexerToken &PrevTok, const LexerToken &Tok);
+ bool AvoidConcat(const Token &PrevTok, const Token &Tok);
};
}
@@ -250,7 +250,7 @@ void PrintPPOutputPPCallbacks::Ident(SourceLocation Loc, const std::string &S) {
/// HandleFirstTokOnLine - When emitting a preprocessed file in -E mode, this
/// is called for the first token on each new line.
-void PrintPPOutputPPCallbacks::HandleFirstTokOnLine(LexerToken &Tok) {
+void PrintPPOutputPPCallbacks::HandleFirstTokOnLine(Token &Tok) {
// Figure out what line we went to and insert the appropriate number of
// newline characters.
MoveToLine(Tok.getLocation());
@@ -281,7 +281,7 @@ struct UnknownPragmaHandler : public PragmaHandler {
UnknownPragmaHandler(const char *prefix, PrintPPOutputPPCallbacks *callbacks)
: PragmaHandler(0), Prefix(prefix), Callbacks(callbacks) {}
- virtual void HandlePragma(Preprocessor &PP, LexerToken &PragmaTok) {
+ virtual void HandlePragma(Preprocessor &PP, Token &PragmaTok) {
// Figure out what line we went to and insert the appropriate number of
// newline characters.
Callbacks->MoveToLine(PragmaTok.getLocation());
@@ -311,8 +311,8 @@ struct UnknownPragmaHandler : public PragmaHandler {
/// the resulting output won't have incorrect concatenations going on. Examples
/// include "..", which we print with a space between, because we don't want to
/// track enough to tell "x.." from "...".
-bool PrintPPOutputPPCallbacks::AvoidConcat(const LexerToken &PrevTok,
- const LexerToken &Tok) {
+bool PrintPPOutputPPCallbacks::AvoidConcat(const Token &PrevTok,
+ const Token &Tok) {
char Buffer[256];
// If we haven't emitted a token on this line yet, PrevTok isn't useful to
@@ -394,7 +394,7 @@ void clang::DoPrintPreprocessedInput(unsigned MainFileID, Preprocessor &PP,
InitOutputBuffer();
- LexerToken Tok, PrevTok;
+ Token Tok, PrevTok;
char Buffer[256];
PrintPPOutputPPCallbacks *Callbacks = new PrintPPOutputPPCallbacks(PP);
PP.setPPCallbacks(Callbacks);
diff --git a/Driver/TextDiagnosticPrinter.cpp b/Driver/TextDiagnosticPrinter.cpp
index 87f33741b0..ac48e254bc 100644
--- a/Driver/TextDiagnosticPrinter.cpp
+++ b/Driver/TextDiagnosticPrinter.cpp
@@ -111,7 +111,7 @@ unsigned TextDiagnosticPrinter::GetTokenLength(SourceLocation Loc) {
// Create a lexer starting at the beginning of this token.
Lexer TheLexer(Loc, *ThePreprocessor, StrData);
- LexerToken TheTok;
+ Token TheTok;
TheLexer.LexRawToken(TheTok);
return TheTok.getLength();
}
diff --git a/Driver/clang.cpp b/Driver/clang.cpp
index 404baadf48..ac12ef7a72 100644
--- a/Driver/clang.cpp
+++ b/Driver/clang.cpp
@@ -754,7 +754,7 @@ static unsigned InitializePreprocessor(Preprocessor &PP,
PP.EnterSourceFile(FileID, 0);
// Lex the file, which will read all the macros.
- LexerToken Tok;
+ Token Tok;
PP.Lex(Tok);
assert(Tok.getKind() == tok::eof && "Didn't read entire file!");
@@ -775,7 +775,7 @@ static void ProcessInputFile(Preprocessor &PP, unsigned MainFileID,
fprintf(stderr, "Unexpected program action!\n");
return;
case DumpTokens: { // Token dump mode.
- LexerToken Tok;
+ Token Tok;
// Start parsing the specified input file.
PP.EnterSourceFile(MainFileID, 0, true);
do {
@@ -786,7 +786,7 @@ static void ProcessInputFile(Preprocessor &PP, unsigned MainFileID,
break;
}
case RunPreprocessorOnly: { // Just lex as fast as we can, no output.
- LexerToken Tok;
+ Token Tok;
// Start parsing the specified input file.
PP.EnterSourceFile(MainFileID, 0, true);
do {
diff --git a/Lex/Lexer.cpp b/Lex/Lexer.cpp
index 4efd62113d..1518d8895c 100644
--- a/Lex/Lexer.cpp
+++ b/Lex/Lexer.cpp
@@ -7,7 +7,7 @@
//
//===----------------------------------------------------------------------===//
//
-// This file implements the Lexer and LexerToken interfaces.
+// This file implements the Lexer and Token interfaces.
//
//===----------------------------------------------------------------------===//
//
@@ -248,7 +248,7 @@ static char DecodeTrigraphChar(const char *CP, Lexer *L) {
/// be updated to match.
///
char Lexer::getCharAndSizeSlow(const char *Ptr, unsigned &Size,
- LexerToken *Tok) {
+ Token *Tok) {
// If we have a slash, look for an escaped newline.
if (Ptr[0] == '\\') {
++Size;
@@ -264,7 +264,7 @@ Slash:
++SizeTmp;
if (Ptr[SizeTmp-1] == '\n' || Ptr[SizeTmp-1] == '\r') {
// Remember that this token needs to be cleaned.
- if (Tok) Tok->setFlag(LexerToken::NeedsCleaning);
+ if (Tok) Tok->setFlag(Token::NeedsCleaning);
// Warn if there was whitespace between the backslash and newline.
if (SizeTmp != 1 && Tok)
@@ -294,7 +294,7 @@ Slash:
// a trigraph warning. If so, and if trigraphs are enabled, return it.
if (char C = DecodeTrigraphChar(Ptr+2, Tok ? this : 0)) {
// Remember that this token needs to be cleaned.
- if (Tok) Tok->setFlag(LexerToken::NeedsCleaning);
+ if (Tok) Tok->setFlag(Token::NeedsCleaning);
Ptr += 3;
Size += 3;
@@ -372,7 +372,7 @@ Slash:
// Helper methods for lexing.
//===----------------------------------------------------------------------===//
-void Lexer::LexIdentifier(LexerToken &Result, const char *CurPtr) {
+void Lexer::LexIdentifier(Token &Result, const char *CurPtr) {
// Match [_A-Za-z0-9]*, we have already matched [_A-Za-z$]
unsigned Size;
unsigned char C = *CurPtr++;
@@ -436,7 +436,7 @@ FinishIdentifier:
/// LexNumericConstant - Lex the remainer of a integer or floating point
/// constant. From[-1] is the first character lexed. Return the end of the
/// constant.
-void Lexer::LexNumericConstant(LexerToken &Result, const char *CurPtr) {
+void Lexer::LexNumericConstant(Token &Result, const char *CurPtr) {
unsigned Size;
char C = getCharAndSize(CurPtr, Size);
char PrevCh = 0;
@@ -463,7 +463,7 @@ void Lexer::LexNumericConstant(LexerToken &Result, const char *CurPtr) {
/// LexStringLiteral - Lex the remainder of a string literal, after having lexed
/// either " or L".
-void Lexer::LexStringLiteral(LexerToken &Result, const char *CurPtr, bool Wide){
+void Lexer::LexStringLiteral(Token &Result, const char *CurPtr, bool Wide){
const char *NulCharacter = 0; // Does this string contain the \0 character?
char C = getAndAdvanceChar(CurPtr, Result);
@@ -495,7 +495,7 @@ void Lexer::LexStringLiteral(LexerToken &Result, const char *CurPtr, bool Wide){
/// LexAngledStringLiteral - Lex the remainder of an angled string literal,
/// after having lexed the '<' character. This is used for #include filenames.
-void Lexer::LexAngledStringLiteral(LexerToken &Result, const char *CurPtr) {
+void Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) {
const char *NulCharacter = 0; // Does this string contain the \0 character?
char C = getAndAdvanceChar(CurPtr, Result);
@@ -528,7 +528,7 @@ void Lexer::LexAngledStringLiteral(LexerToken &Result, const char *CurPtr) {
/// LexCharConstant - Lex the remainder of a character constant, after having
/// lexed either ' or L'.
-void Lexer::LexCharConstant(LexerToken &Result, const char *CurPtr) {
+void Lexer::LexCharConstant(Token &Result, const char *CurPtr) {
const char *NulCharacter = 0; // Does this character contain the \0 character?
// Handle the common case of 'x' and '\y' efficiently.
@@ -576,7 +576,7 @@ void Lexer::LexCharConstant(LexerToken &Result, const char *CurPtr) {
/// SkipWhitespace - Efficiently skip over a series of whitespace characters.
/// Update BufferPtr to point to the next non-whitespace character and return.
-void Lexer::SkipWhitespace(LexerToken &Result, const char *CurPtr) {
+void Lexer::SkipWhitespace(Token &Result, const char *CurPtr) {
// Whitespace - Skip it, then return the token after the whitespace.
unsigned char Char = *CurPtr; // Skip consequtive spaces efficiently.
while (1) {
@@ -596,16 +596,16 @@ void Lexer::SkipWhitespace(LexerToken &Result, const char *CurPtr) {
// ok, but handle newline.
// The returned token is at the start of the line.
- Result.setFlag(LexerToken::StartOfLine);
+ Result.setFlag(Token::StartOfLine);
// No leading whitespace seen so far.
- Result.clearFlag(LexerToken::LeadingSpace);
+ Result.clearFlag(Token::LeadingSpace);
Char = *++CurPtr;
}
// If this isn't immediately after a newline, there is leading space.
char PrevChar = CurPtr[-1];
if (PrevChar != '\n' && PrevChar != '\r')
- Result.setFlag(LexerToken::LeadingSpace);
+ Result.setFlag(Token::LeadingSpace);
// If the next token is obviously a // or /* */ comment, skip it efficiently
// too (without going through the big switch stmt).
@@ -625,7 +625,7 @@ void Lexer::SkipWhitespace(LexerToken &Result, const char *CurPtr) {
// SkipBCPLComment - We have just read the // characters from input. Skip until
// we find the newline character thats terminate the comment. Then update
/// BufferPtr and return.
-bool Lexer::SkipBCPLComment(LexerToken &Result, const char *CurPtr) {
+bool Lexer::SkipBCPLComment(Token &Result, const char *CurPtr) {
// If BCPL comments aren't explicitly enabled for this language, emit an
// extension warning.
if (!Features.BCPLComment) {
@@ -704,15 +704,15 @@ bool Lexer::SkipBCPLComment(LexerToken &Result, const char *CurPtr) {
++CurPtr;
// The next returned token is at the start of the line.
- Result.setFlag(LexerToken::StartOfLine);
+ Result.setFlag(Token::StartOfLine);
// No leading whitespace seen so far.
- Result.clearFlag(LexerToken::LeadingSpace);
+ Result.clearFlag(Token::LeadingSpace);
// It is common for the tokens immediately after a // comment to be
// whitespace (indentation for the next line). Instead of going through the
// big switch, handle it efficiently now.
if (isWhitespace(*CurPtr)) {
- Result.setFlag(LexerToken::LeadingSpace);
+ Result.setFlag(Token::LeadingSpace);
SkipWhitespace(Result, CurPtr+1);
return true;
}
@@ -723,7 +723,7 @@ bool Lexer::SkipBCPLComment(LexerToken &Result, const char *CurPtr) {
/// SaveBCPLComment - If in save-comment mode, package up this BCPL comment in
/// an appropriate way and return it.
-bool Lexer::SaveBCPLComment(LexerToken &Result, const char *CurPtr) {
+bool Lexer::SaveBCPLComment(Token &Result, const char *CurPtr) {
Result.setKind(tok::comment);
FormTokenWithChars(Result, CurPtr);
@@ -812,7 +812,7 @@ static bool isEndOfBlockCommentWithEscapedNewLine(const char *CurPtr,
/// because they cannot cause the comment to end. The only thing that can
/// happen is the comment could end with an escaped newline between the */ end
/// of comment.
-bool Lexer::SkipBlockComment(LexerToken &Result, const char *CurPtr) {
+bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr) {
// Scan one character past where we should, looking for a '/' character. Once
// we find it, check to see if it was preceeded by a *. This common
// optimization helps people who like to put a lot of * characters in their
@@ -907,14 +907,14 @@ bool Lexer::SkipBlockComment(LexerToken &Result, const char *CurPtr) {
// whitespace. Instead of going through the big switch, handle it
// efficiently now.
if (isHorizontalWhitespace(*CurPtr)) {
- Result.setFlag(LexerToken::LeadingSpace);
+ Result.setFlag(Token::LeadingSpace);
SkipWhitespace(Result, CurPtr+1);
return true;
}
// Otherwise, just return so that the next character will be lexed as a token.
BufferPtr = CurPtr;
- Result.setFlag(LexerToken::LeadingSpace);
+ Result.setFlag(Token::LeadingSpace);
return true;
}
@@ -924,7 +924,7 @@ bool Lexer::SkipBlockComment(LexerToken &Result, const char *CurPtr) {
/// LexIncludeFilename - After the preprocessor has parsed a #include, lex and
/// (potentially) macro expand the filename.
-void Lexer::LexIncludeFilename(LexerToken &FilenameTok) {
+void Lexer::LexIncludeFilename(Token &FilenameTok) {
assert(ParsingPreprocessorDirective &&
ParsingFilename == false &&
"Must be in a preprocessing directive!");
@@ -949,7 +949,7 @@ std::string Lexer::ReadToEndOfLine() {
assert(ParsingPreprocessorDirective && ParsingFilename == false &&
"Must be in a preprocessing directive!");
std::string Result;
- LexerToken Tmp;
+ Token Tmp;
// CurPtr - Cache BufferPtr in an automatic variable.
const char *CurPtr = BufferPtr;
@@ -987,7 +987,7 @@ std::string Lexer::ReadToEndOfLine() {
/// condition, reporting diagnostics and handling other edge cases as required.
/// This returns true if Result contains a token, false if PP.Lex should be
/// called again.
-bool Lexer::LexEndOfFile(LexerToken &Result, const char *CurPtr) {
+bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) {
// If we hit the end of the file while parsing a preprocessor directive,
// end the preprocessor directive first. The next token returned will
// then be the end of file.
@@ -1046,7 +1046,7 @@ unsigned Lexer::isNextPPTokenLParen() {
// Save state that can be changed while lexing so that we can restore it.
const char *TmpBufferPtr = BufferPtr;
- LexerToken Tok;
+ Token Tok;
Tok.startToken();
LexTokenInternal(Tok);
@@ -1069,10 +1069,10 @@ unsigned Lexer::isNextPPTokenLParen() {
/// preprocessing token, not a normal token, as such, it is an internal
/// interface. It assumes that the Flags of result have been cleared before
/// calling this.
-void Lexer::LexTokenInternal(LexerToken &Result) {
+void Lexer::LexTokenInternal(Token &Result) {
LexNextToken:
// New token, can't need cleaning yet.
- Result.clearFlag(LexerToken::NeedsCleaning);
+ Result.clearFlag(Token::NeedsCleaning);
Result.setIdentifierInfo(0);
// CurPtr - Cache BufferPtr in an automatic variable.
@@ -1084,7 +1084,7 @@ LexNextToken:
while ((*CurPtr == ' ') || (*CurPtr == '\t'))
++CurPtr;
BufferPtr = CurPtr;
- Result.setFlag(LexerToken::LeadingSpace);
+ Result.setFlag(Token::LeadingSpace);
}
unsigned SizeTmp, SizeTmp2; // Temporaries for use in cases below.
@@ -1104,7 +1104,7 @@ LexNextToken:
}
Diag(CurPtr-1, diag::null_in_file);
- Result.setFlag(LexerToken::LeadingSpace);
+ Result.setFlag(Token::LeadingSpace);
SkipWhitespace(Result, CurPtr);
goto LexNextToken; // GCC isn't tail call eliminating.
case '\n':
@@ -1125,16 +1125,16 @@ LexNextToken:
break;
}
// The returned token is at the start of the line.
- Result.setFlag(LexerToken::StartOfLine);
+ Result.setFlag(Token::StartOfLine);
// No leading whitespace seen so far.
- Result.clearFlag(LexerToken::LeadingSpace);
+ Result.clearFlag(Token::LeadingSpace);
SkipWhitespace(Result, CurPtr);
goto LexNextToken; // GCC isn't tail call eliminating.
case ' ':
case '\t':
case '\f':
case '\v':
- Result.setFlag(LexerToken::LeadingSpace);
+ Result.setFlag(Token::LeadingSpace);
SkipWhitespace(Result, CurPtr);
goto LexNextToken; // GCC isn't tail call eliminating.
@@ -1346,7 +1346,7 @@ LexNextToken:
// want us starting at the beginning of the line again. If so, set
// the StartOfLine flag.
if (IsAtStartOfLine) {
- Result.setFlag(LexerToken::StartOfLine);
+ Result.setFlag(Token::StartOfLine);
IsAtStartOfLine = false;
}
goto LexNextToken; // GCC isn't tail call eliminating.
@@ -1475,7 +1475,7 @@ LexNextToken:
// want us starting at the beginning of the line again. If so, set
// the StartOfLine flag.
if (IsAtStartOfLine) {
- Result.setFlag(LexerToken::StartOfLine);
+ Result.setFlag(Token::StartOfLine);
IsAtStartOfLine = false;
}
goto LexNextToken; // GCC isn't tail call eliminating.
diff --git a/Lex/LiteralSupport.cpp b/Lex/LiteralSupport.cpp
index cd2082a3cb..76a70ab7aa 100644
--- a/Lex/LiteralSupport.cpp
+++ b/Lex/LiteralSupport.cpp
@@ -535,7 +535,7 @@ CharLiteralParser::CharLiteralParser(const char *begin, const char *end,
/// hex-digit hex-digit hex-digit hex-digit
///
StringLiteralParser::
-StringLiteralParser(const LexerToken *StringToks, unsigned NumStringToks,
+StringLiteralParser(const Token *StringToks, unsigned NumStringToks,
Preprocessor &pp, TargetInfo &t)
: PP(pp), Target(t) {
// Scan all of the string portions, remember the max individual token length,
diff --git a/Lex/MacroExpander.cpp b/Lex/MacroExpander.cpp
index 53ff3f14ca..fade854657 100644
--- a/Lex/MacroExpander.cpp
+++ b/Lex/MacroExpander.cpp
@@ -25,21 +25,21 @@ using namespace clang;
/// MacroArgs ctor function - This destroys the vector passed in.
MacroArgs *MacroArgs::create(const MacroInfo *MI,
- const LexerToken *UnexpArgTokens,
+ const Token *UnexpArgTokens,
unsigned NumToks, bool VarargsElided) {
assert(MI->isFunctionLike() &&
"Can't have args for an object-like macro!");
// Allocate memory for the MacroArgs object with the lexer tokens at the end.
MacroArgs *Result = (MacroArgs*)malloc(sizeof(MacroArgs) +
- NumToks*sizeof(LexerToken));
+ NumToks*sizeof(Token));
// Construct the macroargs object.
new (Result) MacroArgs(NumToks, VarargsElided);
// Copy the actual unexpanded tokens to immediately after the result ptr.
if (NumToks)
- memcpy(const_cast<LexerToken*>(Result->getUnexpArgument(0)),
- UnexpArgTokens, NumToks*sizeof(LexerToken));
+ memcpy(const_cast<Token*>(Result->getUnexpArgument(0)),
+ UnexpArgTokens, NumToks*sizeof(Token));
return Result;
}
@@ -57,7 +57,7 @@ void MacroArgs::destroy() {
/// getArgLength - Given a pointer to an expanded or unexpanded argument,
/// return the number of tokens, not counting the EOF, that make up the
/// argument.
-unsigned MacroArgs::getArgLength(const LexerToken *ArgPtr) {
+unsigned MacroArgs::getArgLength(const Token *ArgPtr) {
unsigned NumArgTokens = 0;
for (; ArgPtr->getKind() != tok::eof; ++ArgPtr)
++NumArgTokens;
@@ -67,11 +67,11 @@ unsigned MacroArgs::getArgLength(const LexerToken *ArgPtr) {
/// getUnexpArgument - Return the unexpanded tokens for the specified formal.
///
-const LexerToken *MacroArgs::getUnexpArgument(unsigned Arg) const {
+const Token *MacroArgs::getUnexpArgument(unsigned Arg) const {
// The unexpanded argument tokens start immediately after the MacroArgs object
// in memory.
- const LexerToken *Start = (const LexerToken *)(this+1);
- const LexerToken *Result = Start;
+ const Token *Start = (const Token *)(this+1);
+ const Token *Result = Start;
// Scan to find Arg.
for (; Arg; ++Result) {
assert(Result < Start+NumUnexpArgTokens && "Invalid arg #");
@@ -84,7 +84,7 @@ const LexerToken *MacroArgs::getUnexpArgument(unsigned Arg) const {
/// ArgNeedsPreexpansion - If we can prove that the argument won't be affected
/// by pre-expansion, return false. Otherwise, conservatively return true.
-bool MacroArgs::ArgNeedsPreexpansion(const LexerToken *ArgTok) const {
+bool MacroArgs::ArgNeedsPreexpansion(const Token *ArgTok) const {
// If there are no identifiers in the argument list, or if the identifiers are
// known to not be macros, pre-expansion won't modify it.
for (; ArgTok->getKind() != tok::eof; ++ArgTok)
@@ -99,7 +99,7 @@ bool MacroArgs::ArgNeedsPreexpansion(const LexerToken *ArgTok) const {
/// getPreExpArgument - Return the pre-expanded form of the specified
/// argument.
-const std::vector<LexerToken> &
+const std::vector<Token> &
MacroArgs::getPreExpArgument(unsigned Arg, Preprocessor &PP) {
assert(Arg < NumUnexpArgTokens && "Invalid argument number!");
@@ -107,10 +107,10 @@ MacroArgs::getPreExpArgument(unsigned Arg, Preprocessor &PP) {
if (PreExpArgTokens.empty())
PreExpArgTokens.resize(NumUnexpArgTokens);
- std::vector<LexerToken> &Result = PreExpArgTokens[Arg];
+ std::vector<Token> &Result = PreExpArgTokens[Arg];
if (!Result.empty()) return Result;
- const LexerToken *AT = getUnexpArgument(Arg);
+ const Token *AT = getUnexpArgument(Arg);
unsigned NumToks = getArgLength(AT)+1; // Include the EOF.
// Otherwise, we have to pre-expand this argument, populating Result. To do
@@ -121,7 +121,7 @@ MacroArgs::getPreExpArgument(unsigned Arg, Preprocessor &PP) {
// Lex all of the macro-expanded tokens into Result.
do {
- Result.push_back(LexerToken());
+ Result.push_back(Token());
PP.Lex(Result.back());
} while (Result.back().getKind() != tok::eof);
@@ -139,20 +139,20 @@ MacroArgs::getPreExpArgument(unsigned Arg, Preprocessor &PP) {
/// tokens into the literal string token that should be produced by the C #
/// preprocessor operator.
///
-static LexerToken StringifyArgument(const LexerToken *ArgToks,
+static Token StringifyArgument(const Token *ArgToks,
Preprocessor &PP, bool Charify = false) {
- LexerToken Tok;
+ Token Tok;
Tok.startToken();
Tok.setKind(tok::string_literal);
- const LexerToken *ArgTokStart = ArgToks;
+ const Token *ArgTokStart = ArgToks;
// Stringify all the tokens.
std::string Result = "\"";
// FIXME: Optimize this loop to not use std::strings.
bool isFirst = true;
for (; ArgToks->getKind() != tok::eof; ++ArgToks) {
- const LexerToken &Tok = *ArgToks;
+ const Token &Tok = *ArgToks;
if (!isFirst && (Tok.hasLeadingSpace() || Tok.isAtStartOfLine()))
Result += ' ';
isFirst = false;
@@ -214,7 +214,7 @@ static LexerToken StringifyArgument(const LexerToken *ArgToks,
/// getStringifiedArgument - Compute, cache, and return the specified argument
/// that has been 'stringified' as required by the # operator.
-const LexerToken &MacroArgs::getStringifiedArgument(unsigned ArgNo,
+const Token &MacroArgs::getStringifiedArgument(unsigned ArgNo,
Preprocessor &PP) {
assert(ArgNo < NumUnexpArgTokens && "Invalid argument number!");
if (StringifiedArgs.empty()) {
@@ -233,7 +233,7 @@ const LexerToken &MacroArgs::getStringifiedArgument(unsigned ArgNo,
/// Create a macro expander for the specified macro with the specified actual
/// arguments. Note that this ctor takes ownership of the ActualArgs pointer.
-void MacroExpander::Init(LexerToken &Tok, MacroArgs *Actuals) {
+void MacroExpander::Init(Token &Tok, MacroArgs *Actuals) {
// If the client is reusing a macro expander, make sure to free any memory
// associated with it.
destroy();
@@ -262,7 +262,7 @@ void MacroExpander::Init(LexerToken &Tok, MacroArgs *Actuals) {
/// Create a macro expander for the specified token stream. This does not
/// take ownership of the specified token vector.
-void MacroExpander::Init(const LexerToken *TokArray, unsigned NumToks) {
+void MacroExpander::Init(const Token *TokArray, unsigned NumToks) {
// If the client is reusing a macro expander, make sure to free any memory
// associated with it.
destroy();
@@ -298,7 +298,7 @@ void MacroExpander::destroy() {
/// Expand the arguments of a function-like macro so that we can quickly
/// return preexpanded tokens from MacroTokens.
void MacroExpander::ExpandFunctionArguments() {
- llvm::SmallVector<LexerToken, 128> ResultToks;
+ llvm::SmallVector<Token, 128> ResultToks;
// Loop through the MacroTokens tokens, expanding them into ResultToks. Keep
// track of whether we change anything. If not, no need to keep them. If so,
@@ -314,12 +314,12 @@ void MacroExpander::ExpandFunctionArguments() {
// If we found the stringify operator, get the argument stringified. The
// preprocessor already verified that the following token is a macro name
// when the #define was parsed.
- const LexerToken &CurTok = MacroTokens[i];
+ const Token &CurTok = MacroTokens[i];
if (CurTok.getKind() == tok::hash || CurTok.getKind() == tok::hashat) {
int ArgNo = Macro->getArgumentNum(MacroTokens[i+1].getIdentifierInfo());
assert(ArgNo != -1 && "Token following # is not an argument?");
- LexerToken Res;
+ Token Res;
if (CurTok.getKind() == tok::hash) // Stringify
Res = ActualArgs->getStringifiedArgument(ArgNo, PP);
else {
@@ -330,7 +330,7 @@ void MacroExpander::ExpandFunctionArguments() {
// The stringified/charified string leading space flag gets set to match
// the #/#@ operator.
if (CurTok.hasLeadingSpace() || NextTokGetsSpace)
- Res.setFlag(LexerToken::LeadingSpace);
+ Res.setFlag(Token::LeadingSpace);
ResultToks.push_back(Res);
MadeChange = true;
@@ -348,7 +348,7 @@ void MacroExpander::ExpandFunctionArguments() {
ResultToks.push_back(CurTok);
if (NextTokGetsSpace) {
- ResultToks.back().setFlag(LexerToken::LeadingSpace);
+ ResultToks.back().setFlag(Token::LeadingSpace);
NextTokGetsSpace = false;
}
continue;
@@ -368,11 +368,11 @@ void MacroExpander::ExpandFunctionArguments() {
// argument and substitute the expanded tokens into the result. This is
// C99 6.10.3.1p1.
if (!PasteBefore && !PasteAfter) {
- const LexerToken *ResultArgToks;
+ const Token *ResultArgToks;
// Only preexpand the argument if it could possibly need it. This
// avoids some work in common cases.
- const LexerToken *ArgTok = ActualArgs->getUnexpArgument(ArgNo);
+ const Token *ArgTok = ActualArgs->getUnexpArgument(ArgNo);
if (ActualArgs->ArgNeedsPreexpansion(ArgTok))
ResultArgToks = &ActualArgs->getPreExpArgument(ArgNo, PP)[0];
else
@@ -387,7 +387,7 @@ void MacroExpander::ExpandFunctionArguments() {
// If any tokens were substituted from the argument, the whitespace
// before the first token should match the whitespace of the arg
// identifier.
- ResultToks[FirstResult].setFlagValue(LexerToken::LeadingSpace,
+ ResultToks[FirstResult].setFlagValue(Token::LeadingSpace,
CurTok.hasLeadingSpace() ||
NextTokGetsSpace);
NextTokGetsSpace = false;
@@ -401,7 +401,7 @@ void MacroExpander::ExpandFunctionArguments() {
// Okay, we have a token that is either the LHS or RHS of a paste (##)
// argument. It gets substituted as its non-pre-expanded tokens.
- const LexerToken *ArgToks = ActualArgs->getUnexpArgument(ArgNo);
+ const Token *ArgToks = ActualArgs->getUnexpArgument(ArgNo);
unsigned NumToks = MacroArgs::getArgLength(ArgToks);
if (NumToks) { // Not an empty argument?
ResultToks.append(ArgToks, ArgToks+NumToks);
@@ -409,7 +409,7 @@ void MacroExpander::ExpandFunctionArguments() {
// If the next token was supposed to get leading whitespace, ensure it has
// it now.
if (NextTokGetsSpace) {
- ResultToks[ResultToks.size()-NumToks].setFlag(LexerToken::LeadingSpace);
+ ResultToks[ResultToks.size()-NumToks].setFlag(Token::LeadingSpace);
NextTokGetsSpace = false;
}
continue;
@@ -451,16 +451,16 @@ void MacroExpander::ExpandFunctionArguments() {
if (MadeChange) {
// This is deleted in the dtor.
NumMacroTokens = ResultToks.size();
- LexerToken *Res = new LexerToken[ResultToks.size()];
+ Token *Res = new Token[ResultToks.size()];
if (NumMacroTokens)
- memcpy(Res, &ResultToks[0], NumMacroTokens*sizeof(LexerToken));
+ memcpy(Res, &ResultToks[0], NumMacroTokens*sizeof(Token));
MacroTokens = Res;
}
}
/// Lex - Lex and return a token from this macro stream.
///
-void MacroExpander::Lex(LexerToken &Tok) {
+void MacroExpander::Lex(Token &Tok) {
// Lexing off the end of the macro, pop this macro off the expansion stack.
if (isAtEnd()) {
// If this is a macro (not a token stream), mark the macro enabled now
@@ -503,8 +503,8 @@ void MacroExpander::Lex(LexerToken &Tok) {
// If this is the first token, set the lexical properties of the token to
// match the lexical properties of the macro identifier.
if (isFirstToken) {
- Tok.setFlagValue(LexerToken::StartOfLine , AtStartOfLine);
- Tok.setFlagValue(LexerToken::LeadingSpace, HasLeadingSpace);
+ Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);
+ Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace);
}
// Handle recursive expansion!
@@ -517,7 +517,7 @@ void MacroExpander::Lex(LexerToken &Tok) {
/// PasteTokens - Tok is the LHS of a ## operator, and CurToken is the ##
/// operator. Read the ## and RHS, and paste the LHS/RHS together. If there
/// are is another ## after it, chomp it iteratively. Return the result as Tok.
-void MacroExpander::PasteTokens(LexerToken &Tok) {
+void MacroExpander::PasteTokens(Token &Tok) {
llvm::SmallVector<char, 128> Buffer;
do {
// Consume the ## operator.
@@ -526,7 +526,7 @@ void MacroExpander::PasteTokens(LexerToken &Tok) {
assert(!isAtEnd() && "No token on the RHS of a paste operator!");
// Get the RHS token.
- const LexerToken &RHS = MacroTokens[CurToken];
+ const Token &RHS = MacroTokens[CurToken];
bool isInvalid = false;
@@ -556,7 +556,7 @@ void MacroExpander::PasteTokens(LexerToken &Tok) {
SourceLocation ResultTokLoc = PP.CreateString(&Buffer[0], Buffer.size());
// Lex the resultant pasted token into Result.
- LexerToken Result;
+ Token Result;
// Avoid testing /*, as the lexer would think it is the start of a comment
// and emit an error that it is unterminated.
@@ -613,8 +613,8 @@ void MacroExpander::PasteTokens(LexerToken &Tok) {
// FIXME: Turn __VARRGS__ into "not a token"?
// Transfer properties of the LHS over the the Result.
- Result.setFlagValue(LexerToken::StartOfLine , Tok.isAtStartOfLine());
- Result.setFlagValue(LexerToken::LeadingSpace, Tok.hasLeadingSpace());
+ Result.setFlagValue(Token::StartOfLine , Tok.isAtStartOfLine());
+ Result.setFlagValue(Token::LeadingSpace, Tok.hasLeadingSpace());
// Finally, replace LHS with the result, consume the RHS, and iterate.
++CurToken;
diff --git a/Lex/MacroInfo.cpp b/Lex/MacroInfo.cpp
index a9a8ad7fd6..70e5140796 100644
--- a/Lex/MacroInfo.cpp
+++ b/Lex/MacroInfo.cpp
@@ -50,8 +50,8 @@ bool MacroInfo::isIdenticalTo(const MacroInfo &Other, Preprocessor &PP) const {
// Check all the tokens.
for (unsigned i = 0, e = Repla