aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Driver/RewriteTest.cpp31
-rw-r--r--include/clang/Rewrite/TokenRewriter.h23
-rw-r--r--lib/Rewrite/TokenRewriter.cpp53
3 files changed, 82 insertions, 25 deletions
diff --git a/Driver/RewriteTest.cpp b/Driver/RewriteTest.cpp
index 6709860272..5af4d0e395 100644
--- a/Driver/RewriteTest.cpp
+++ b/Driver/RewriteTest.cpp
@@ -11,9 +11,9 @@
//
//===----------------------------------------------------------------------===//
-#include "clang/Rewrite/TokenRewriter.h"
#include "clang.h"
#include "clang/Lex/Preprocessor.h"
+#include "clang/Rewrite/TokenRewriter.h"
#include <iostream>
void clang::DoRewriteTest(Preprocessor &PP, const std::string &InFileName,
@@ -22,27 +22,18 @@ void clang::DoRewriteTest(Preprocessor &PP, const std::string &InFileName,
const LangOptions &LangOpts = PP.getLangOptions();
TokenRewriter Rewriter(SM.getMainFileID(), SM, LangOpts);
-
-
-
-
-
- std::pair<const char*,const char*> File =SM.getBufferData(SM.getMainFileID());
-
- // Create a lexer to lex all the tokens of the main file in raw mode. Even
- // though it is in raw mode, it will not return comments.
- Lexer RawLex(SourceLocation::getFileLoc(SM.getMainFileID(), 0),
- LangOpts, File.first, File.second);
-
- RawLex.SetKeepWhitespaceMode(true);
-
- Token RawTok;
- RawLex.LexFromRawLexer(RawTok);
- while (RawTok.isNot(tok::eof)) {
- std::cout << PP.getSpelling(RawTok);
- RawLex.LexFromRawLexer(RawTok);
+
+ // Throw <i> </i> tags around comments.
+ for (TokenRewriter::token_iterator I = Rewriter.token_begin(),
+ E = Rewriter.token_end(); I != E; ++I) {
+ if (I->isNot(tok::comment)) continue;
+
+ Rewriter.AddTokenBefore(I, "<i>");
+ Rewriter.AddTokenAfter(I, "</i>");
}
+
+ // Print out the output.
for (TokenRewriter::token_iterator I = Rewriter.token_begin(),
E = Rewriter.token_end(); I != E; ++I)
std::cout << PP.getSpelling(*I);
diff --git a/include/clang/Rewrite/TokenRewriter.h b/include/clang/Rewrite/TokenRewriter.h
index da0141ff34..1703ea7885 100644
--- a/include/clang/Rewrite/TokenRewriter.h
+++ b/include/clang/Rewrite/TokenRewriter.h
@@ -16,12 +16,14 @@
#define LLVM_CLANG_TOKENREWRITER_H
#include "clang/Basic/SourceLocation.h"
+#include "llvm/ADT/OwningPtr.h"
#include <list>
#include <map>
namespace clang {
class Token;
class LangOptions;
+ class ScratchBuffer;
class TokenRewriter {
/// TokenList - This is the list of raw tokens that make up this file. Each
@@ -37,20 +39,37 @@ namespace clang {
/// backwards.
std::map<SourceLocation, TokenRefTy> TokenAtLoc;
+ /// ScratchBuf - This is the buffer that we create scratch tokens from.
+ ///
+ llvm::OwningPtr<ScratchBuffer> ScratchBuf;
+
+ TokenRewriter(const TokenRewriter&); // DO NOT IMPLEMENT
+ void operator=(const TokenRewriter&); // DO NOT IMPLEMENT.
public:
/// TokenRewriter - This creates a TokenRewriter for the file with the
/// specified FileID.
TokenRewriter(unsigned FileID, SourceManager &SM, const LangOptions &LO);
-
+ ~TokenRewriter();
typedef std::list<Token>::const_iterator token_iterator;
token_iterator token_begin() const { return TokenList.begin(); }
token_iterator token_end() const { return TokenList.end(); }
+
+ token_iterator AddTokenBefore(token_iterator I, const char *Val);
+ token_iterator AddTokenAfter(token_iterator I, const char *Val) {
+ assert(I != token_end() && "Cannot insert after token_end()!");
+ return AddTokenBefore(++I, Val);
+ }
+
private:
+ /// RemapIterator - Convert from token_iterator (a const iterator) to
+ /// TokenRefTy (a non-const iterator).
+ TokenRefTy RemapIterator(token_iterator I);
+
/// AddToken - Add the specified token into the Rewriter before the other
/// position.
- void AddToken(const Token &T, TokenRefTy Where);
+ TokenRefTy AddToken(const Token &T, TokenRefTy Where);
};
diff --git a/lib/Rewrite/TokenRewriter.cpp b/lib/Rewrite/TokenRewriter.cpp
index 0362bacdfc..f288ac2402 100644
--- a/lib/Rewrite/TokenRewriter.cpp
+++ b/lib/Rewrite/TokenRewriter.cpp
@@ -14,11 +14,13 @@
#include "clang/Rewrite/TokenRewriter.h"
#include "clang/Lex/Lexer.h"
+#include "clang/Lex/ScratchBuffer.h"
#include "clang/Basic/SourceManager.h"
using namespace clang;
TokenRewriter::TokenRewriter(unsigned FileID, SourceManager &SM,
const LangOptions &LangOpts) {
+ ScratchBuf.reset(new ScratchBuffer(SM));
std::pair<const char*,const char*> File = SM.getBufferData(FileID);
@@ -33,21 +35,66 @@ TokenRewriter::TokenRewriter(unsigned FileID, SourceManager &SM,
Token RawTok;
RawLex.LexFromRawLexer(RawTok);
while (RawTok.isNot(tok::eof)) {
+#if 0
+ if (Tok.is(tok::identifier)) {
+ // Look up the identifier info for the token. This should use
+ // IdentifierTable directly instead of PP.
+ Tok.setIdentifierInfo(PP.LookUpIdentifierInfo(Tok));
+ }
+#endif
+
AddToken(RawTok, TokenList.end());
RawLex.LexFromRawLexer(RawTok);
}
+}
+
+TokenRewriter::~TokenRewriter() {
+}
+
+
+/// RemapIterator - Convert from token_iterator (a const iterator) to
+/// TokenRefTy (a non-const iterator).
+TokenRewriter::TokenRefTy TokenRewriter::RemapIterator(token_iterator I) {
+ if (I == token_end()) return TokenList.end();
-
+ // FIXME: This is horrible, we should use our own list or something to avoid
+ // this.
+ std::map<SourceLocation, TokenRefTy>::iterator MapIt =
+ TokenAtLoc.find(I->getLocation());
+ assert(MapIt != TokenAtLoc.end() && "iterator not in rewriter?");
+ return MapIt->second;
}
+
/// AddToken - Add the specified token into the Rewriter before the other
/// position.
-void TokenRewriter::AddToken(const Token &T, TokenRefTy Where) {
+TokenRewriter::TokenRefTy
+TokenRewriter::AddToken(const Token &T, TokenRefTy Where) {
Where = TokenList.insert(Where, T);
bool InsertSuccess = TokenAtLoc.insert(std::make_pair(T.getLocation(),
Where)).second;
assert(InsertSuccess && "Token location already in rewriter!");
InsertSuccess = InsertSuccess;
+ return Where;
}
-
+
+
+TokenRewriter::token_iterator
+TokenRewriter::AddTokenBefore(token_iterator I, const char *Val){
+ unsigned Len = strlen(Val);
+
+ // Plop the string into the scratch buffer, then create a token for this
+ // string.
+ Token Tok;
+ Tok.startToken();
+ Tok.setLocation(ScratchBuf->getToken(Val, Len));
+ Tok.setLength(Len);
+
+ // TODO: Form a whole lexer around this and relex the token! For now, just
+ // set kind to tok::unknown.
+ Tok.setKind(tok::unknown);
+
+ return AddToken(Tok, RemapIterator(I));
+}
+
generated by cgit v1.2.3-18-g5258 (git 2.32.0) at 2025-08-02 22:48:21 +0000