aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--include/clang/Lex/Preprocessor.h19
-rw-r--r--include/clang/Lex/TokenLexer.h9
-rw-r--r--lib/Lex/PPLexerChange.cpp4
-rw-r--r--lib/Lex/PPMacroExpansion.cpp41
-rw-r--r--lib/Lex/Preprocessor.cpp4
-rw-r--r--lib/Lex/TokenLexer.cpp14
6 files changed, 78 insertions, 13 deletions
diff --git a/include/clang/Lex/Preprocessor.h b/include/clang/Lex/Preprocessor.h
index abba959dee..56fc9d1727 100644
--- a/include/clang/Lex/Preprocessor.h
+++ b/include/clang/Lex/Preprocessor.h
@@ -29,6 +29,7 @@
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/OwningPtr.h"
#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/ArrayRef.h"
#include "llvm/Support/Allocator.h"
#include <vector>
@@ -240,6 +241,14 @@ class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
unsigned NumCachedTokenLexers;
TokenLexer *TokenLexerCache[TokenLexerCacheSize];
+ /// \brief Keeps macro expanded tokens for TokenLexers.
+ //
+ /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
+ /// going to lex in the cache and when it finishes the tokens are removed
+ /// from the end of the cache.
+ llvm::SmallVector<Token, 16> MacroExpandedTokens;
+ std::vector<std::pair<TokenLexer *, size_t> > MacroExpandingLexersStack;
+
/// \brief A record of the macro definitions and instantiations that
/// occurred during preprocessing.
///
@@ -979,6 +988,16 @@ private:
/// the macro should not be expanded return true, otherwise return false.
bool HandleMacroExpandedIdentifier(Token &Tok, MacroInfo *MI);
+ /// \brief Cache macro expanded tokens for TokenLexers.
+ //
+ /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
+ /// going to lex in the cache and when it finishes the tokens are removed
+ /// from the end of the cache.
+ Token *cacheMacroExpandedTokens(TokenLexer *tokLexer,
+ llvm::ArrayRef<Token> tokens);
+ void removeCachedMacroExpandedTokensOfLastLexer();
+ friend void TokenLexer::ExpandFunctionArguments();
+
/// isNextPPTokenLParen - Determine whether the next preprocessor token to be
/// lexed is a '('. If so, consume the token and return true, if not, this
/// method should have no observable side-effect on the lexed tokens.
diff --git a/include/clang/Lex/TokenLexer.h b/include/clang/Lex/TokenLexer.h
index 6ae00cd586..8f530053ca 100644
--- a/include/clang/Lex/TokenLexer.h
+++ b/include/clang/Lex/TokenLexer.h
@@ -43,10 +43,13 @@ class TokenLexer {
/// Tokens - This is the pointer to an array of tokens that the macro is
/// defined to, with arguments expanded for function-like macros. If this is
/// a token stream, these are the tokens we are returning. This points into
- /// the macro definition we are lexing from, a scratch buffer allocated from
- /// the preprocessor's bump pointer allocator, or some other buffer that we
- /// may or may not own (depending on OwnsTokens).
+ /// the macro definition we are lexing from, a cache buffer that is owned by
+ /// the preprocessor, or some other buffer that we may or may not own
+ /// (depending on OwnsTokens).
+ /// Note that if it points into Preprocessor's cache buffer, the Preprocessor
+ /// may update the pointer as needed.
const Token *Tokens;
+ friend class Preprocessor;
/// NumTokens - This is the length of the Tokens array.
///
diff --git a/lib/Lex/PPLexerChange.cpp b/lib/Lex/PPLexerChange.cpp
index bf0a7fbfef..bf28199b88 100644
--- a/lib/Lex/PPLexerChange.cpp
+++ b/lib/Lex/PPLexerChange.cpp
@@ -265,6 +265,10 @@ bool Preprocessor::HandleEndOfTokenLexer(Token &Result) {
assert(CurTokenLexer && !CurPPLexer &&
"Ending a macro when currently in a #include file!");
+ if (!MacroExpandingLexersStack.empty() &&
+ MacroExpandingLexersStack.back().first == CurTokenLexer.get())
+ removeCachedMacroExpandedTokensOfLastLexer();
+
// Delete or cache the now-dead macro expander.
if (NumCachedTokenLexers == TokenLexerCacheSize)
CurTokenLexer.reset();
diff --git a/lib/Lex/PPMacroExpansion.cpp b/lib/Lex/PPMacroExpansion.cpp
index 9e8533a448..d7c4defb36 100644
--- a/lib/Lex/PPMacroExpansion.cpp
+++ b/lib/Lex/PPMacroExpansion.cpp
@@ -22,6 +22,7 @@
#include "clang/Lex/CodeCompletionHandler.h"
#include "clang/Lex/ExternalPreprocessorSource.h"
#include "llvm/ADT/StringSwitch.h"
+#include "llvm/ADT/STLExtras.h"
#include "llvm/Config/config.h"
#include "llvm/Support/raw_ostream.h"
#include <cstdio>
@@ -490,6 +491,46 @@ MacroArgs *Preprocessor::ReadFunctionLikeMacroArgs(Token &MacroName,
isVarargsElided, *this);
}
+/// \brief Keeps macro expanded tokens for TokenLexers.
+//
+/// Works like a stack; a TokenLexer adds the macro expanded tokens that is
+/// going to lex in the cache and when it finishes the tokens are removed
+/// from the end of the cache.
+Token *Preprocessor::cacheMacroExpandedTokens(TokenLexer *tokLexer,
+ llvm::ArrayRef<Token> tokens) {
+ assert(tokLexer);
+ if (tokens.empty())
+ return 0;
+
+ size_t newIndex = MacroExpandedTokens.size();
+ bool cacheNeedsToGrow = tokens.size() >
+ MacroExpandedTokens.capacity()-MacroExpandedTokens.size();
+ MacroExpandedTokens.append(tokens.begin(), tokens.end());
+
+ if (cacheNeedsToGrow) {
+ // Go through all the TokenLexers whose 'Tokens' pointer points in the
+ // buffer and update the pointers to the (potential) new buffer array.
+ for (unsigned i = 0, e = MacroExpandingLexersStack.size(); i != e; ++i) {
+ TokenLexer *prevLexer;
+ size_t tokIndex;
+ llvm::tie(prevLexer, tokIndex) = MacroExpandingLexersStack[i];
+ prevLexer->Tokens = MacroExpandedTokens.data() + tokIndex;
+ }
+ }
+
+ MacroExpandingLexersStack.push_back(std::make_pair(tokLexer, newIndex));
+ return MacroExpandedTokens.data() + newIndex;
+}
+
+void Preprocessor::removeCachedMacroExpandedTokensOfLastLexer() {
+ assert(!MacroExpandingLexersStack.empty());
+ size_t tokIndex = MacroExpandingLexersStack.back().second;
+ assert(tokIndex < MacroExpandedTokens.size());
+ // Pop the cached macro expanded tokens from the end.
+ MacroExpandedTokens.resize(tokIndex);
+ MacroExpandingLexersStack.pop_back();
+}
+
/// ComputeDATE_TIME - Compute the current time, enter it into the specified
/// scratch buffer, then return DATELoc/TIMELoc locations with the position of
/// the identifier tokens inserted.
diff --git a/lib/Lex/Preprocessor.cpp b/lib/Lex/Preprocessor.cpp
index 48a23880f3..2f43c8e30b 100644
--- a/lib/Lex/Preprocessor.cpp
+++ b/lib/Lex/Preprocessor.cpp
@@ -118,6 +118,8 @@ Preprocessor::Preprocessor(Diagnostic &diags, const LangOptions &opts,
Preprocessor::~Preprocessor() {
assert(BacktrackPositions.empty() && "EnableBacktrack/Backtrack imbalance!");
+ assert(MacroExpandingLexersStack.empty() && MacroExpandedTokens.empty() &&
+ "Preprocessor::HandleEndOfTokenLexer should have cleared those");
while (!IncludeMacroStack.empty()) {
delete IncludeMacroStack.back().TheLexer;
@@ -226,7 +228,7 @@ Preprocessor::macro_begin(bool IncludeExternalMacros) const {
}
size_t Preprocessor::getTotalMemory() const {
- return BP.getTotalMemory();
+ return BP.getTotalMemory() + MacroExpandedTokens.capacity()*sizeof(Token);
}
Preprocessor::macro_iterator
diff --git a/lib/Lex/TokenLexer.cpp b/lib/Lex/TokenLexer.cpp
index e7cff8bdc3..f30c44e0e6 100644
--- a/lib/Lex/TokenLexer.cpp
+++ b/lib/Lex/TokenLexer.cpp
@@ -284,15 +284,11 @@ void TokenLexer::ExpandFunctionArguments() {
assert(!OwnsTokens && "This would leak if we already own the token list");
// This is deleted in the dtor.
NumTokens = ResultToks.size();
- llvm::BumpPtrAllocator &Alloc = PP.getPreprocessorAllocator();
- Token *Res =
- static_cast<Token *>(Alloc.Allocate(sizeof(Token)*ResultToks.size(),
- llvm::alignOf<Token>()));
- if (NumTokens)
- memcpy(Res, &ResultToks[0], NumTokens*sizeof(Token));
- Tokens = Res;
-
- // The preprocessor bump pointer owns these tokens, not us.
+ // The tokens will be added to Preprocessor's cache and will be removed
+ // when this TokenLexer finishes lexing them.
+ Tokens = PP.cacheMacroExpandedTokens(this, ResultToks);
+
+ // The preprocessor cache of macro expanded tokens owns these tokens,not us.
OwnsTokens = false;
}
}
generated by cgit v1.2.3-18-g5258 (git 2.32.0) at 2025-08-04 00:51:21 +0000