aboutsummaryrefslogtreecommitdiff
path: root/tools/clang-cc/CacheTokens.cpp
diff options
context:
space:
mode:
authorEli Friedman <eli.friedman@gmail.com>2009-05-19 04:14:29 +0000
committerEli Friedman <eli.friedman@gmail.com>2009-05-19 04:14:29 +0000
commitb09f6e15c59b89d5820db8ef40598eb1d1323c1f (patch)
treed9a29654b63ecb2d24bbaa2ed0eec28c36c6216b /tools/clang-cc/CacheTokens.cpp
parentb4685713856cab28aeeb4afd640a8bbfb662a8b8 (diff)
Move clang-cc.h to lib/Frontend/Utils.h, and move the associated .cpp
files to lib/Frontend. git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@72099 91177308-0d34-0410-b5e6-96231b3b80d8
Diffstat (limited to 'tools/clang-cc/CacheTokens.cpp')
-rw-r--r--tools/clang-cc/CacheTokens.cpp658
1 files changed, 0 insertions, 658 deletions
diff --git a/tools/clang-cc/CacheTokens.cpp b/tools/clang-cc/CacheTokens.cpp
deleted file mode 100644
index 53352360fb..0000000000
--- a/tools/clang-cc/CacheTokens.cpp
+++ /dev/null
@@ -1,658 +0,0 @@
-//===--- CacheTokens.cpp - Caching of lexer tokens for PTH support --------===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This provides a possible implementation of PTH support for Clang that is
-// based on caching lexed tokens and identifiers.
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang-cc.h"
-#include "clang/Basic/FileManager.h"
-#include "clang/Basic/SourceManager.h"
-#include "clang/Basic/IdentifierTable.h"
-#include "clang/Basic/Diagnostic.h"
-#include "clang/Basic/OnDiskHashTable.h"
-#include "clang/Lex/Lexer.h"
-#include "clang/Lex/Preprocessor.h"
-#include "llvm/ADT/StringMap.h"
-#include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/raw_ostream.h"
-#include "llvm/System/Path.h"
-#include "llvm/Support/Compiler.h"
-#include "llvm/Support/Streams.h"
-
-// FIXME: put this somewhere else?
-#ifndef S_ISDIR
-#define S_ISDIR(x) (((x)&_S_IFDIR)!=0)
-#endif
-
-using namespace clang;
-using namespace clang::io;
-
-//===----------------------------------------------------------------------===//
-// PTH-specific stuff.
-//===----------------------------------------------------------------------===//
-
-namespace {
-class VISIBILITY_HIDDEN PTHEntry {
- Offset TokenData, PPCondData;
-
-public:
- PTHEntry() {}
-
- PTHEntry(Offset td, Offset ppcd)
- : TokenData(td), PPCondData(ppcd) {}
-
- Offset getTokenOffset() const { return TokenData; }
- Offset getPPCondTableOffset() const { return PPCondData; }
-};
-
-
-class VISIBILITY_HIDDEN PTHEntryKeyVariant {
- union { const FileEntry* FE; const char* Path; };
- enum { IsFE = 0x1, IsDE = 0x2, IsNoExist = 0x0 } Kind;
- struct stat *StatBuf;
-public:
- PTHEntryKeyVariant(const FileEntry *fe)
- : FE(fe), Kind(IsFE), StatBuf(0) {}
-
- PTHEntryKeyVariant(struct stat* statbuf, const char* path)
- : Path(path), Kind(IsDE), StatBuf(new struct stat(*statbuf)) {}
-
- PTHEntryKeyVariant(const char* path)
- : Path(path), Kind(IsNoExist), StatBuf(0) {}
-
- bool isFile() const { return Kind == IsFE; }
-
- const char* getCString() const {
- return Kind == IsFE ? FE->getName() : Path;
- }
-
- unsigned getKind() const { return (unsigned) Kind; }
-
- void EmitData(llvm::raw_ostream& Out) {
- switch (Kind) {
- case IsFE:
- // Emit stat information.
- ::Emit32(Out, FE->getInode());
- ::Emit32(Out, FE->getDevice());
- ::Emit16(Out, FE->getFileMode());
- ::Emit64(Out, FE->getModificationTime());
- ::Emit64(Out, FE->getSize());
- break;
- case IsDE:
- // Emit stat information.
- ::Emit32(Out, (uint32_t) StatBuf->st_ino);
- ::Emit32(Out, (uint32_t) StatBuf->st_dev);
- ::Emit16(Out, (uint16_t) StatBuf->st_mode);
- ::Emit64(Out, (uint64_t) StatBuf->st_mtime);
- ::Emit64(Out, (uint64_t) StatBuf->st_size);
- delete StatBuf;
- break;
- default:
- break;
- }
- }
-
- unsigned getRepresentationLength() const {
- return Kind == IsNoExist ? 0 : 4 + 4 + 2 + 8 + 8;
- }
-};
-
-class VISIBILITY_HIDDEN FileEntryPTHEntryInfo {
-public:
- typedef PTHEntryKeyVariant key_type;
- typedef key_type key_type_ref;
-
- typedef PTHEntry data_type;
- typedef const PTHEntry& data_type_ref;
-
- static unsigned ComputeHash(PTHEntryKeyVariant V) {
- return BernsteinHash(V.getCString());
- }
-
- static std::pair<unsigned,unsigned>
- EmitKeyDataLength(llvm::raw_ostream& Out, PTHEntryKeyVariant V,
- const PTHEntry& E) {
-
- unsigned n = strlen(V.getCString()) + 1 + 1;
- ::Emit16(Out, n);
-
- unsigned m = V.getRepresentationLength() + (V.isFile() ? 4 + 4 : 0);
- ::Emit8(Out, m);
-
- return std::make_pair(n, m);
- }
-
- static void EmitKey(llvm::raw_ostream& Out, PTHEntryKeyVariant V, unsigned n){
- // Emit the entry kind.
- ::Emit8(Out, (unsigned) V.getKind());
- // Emit the string.
- Out.write(V.getCString(), n - 1);
- }
-
- static void EmitData(llvm::raw_ostream& Out, PTHEntryKeyVariant V,
- const PTHEntry& E, unsigned) {
-
-
- // For file entries emit the offsets into the PTH file for token data
- // and the preprocessor blocks table.
- if (V.isFile()) {
- ::Emit32(Out, E.getTokenOffset());
- ::Emit32(Out, E.getPPCondTableOffset());
- }
-
- // Emit any other data associated with the key (i.e., stat information).
- V.EmitData(Out);
- }
-};
-
-class OffsetOpt {
- bool valid;
- Offset off;
-public:
- OffsetOpt() : valid(false) {}
- bool hasOffset() const { return valid; }
- Offset getOffset() const { assert(valid); return off; }
- void setOffset(Offset o) { off = o; valid = true; }
-};
-} // end anonymous namespace
-
-typedef OnDiskChainedHashTableGenerator<FileEntryPTHEntryInfo> PTHMap;
-typedef llvm::DenseMap<const IdentifierInfo*,uint32_t> IDMap;
-typedef llvm::StringMap<OffsetOpt, llvm::BumpPtrAllocator> CachedStrsTy;
-
-namespace {
-class VISIBILITY_HIDDEN PTHWriter {
- IDMap IM;
- llvm::raw_fd_ostream& Out;
- Preprocessor& PP;
- uint32_t idcount;
- PTHMap PM;
- CachedStrsTy CachedStrs;
- Offset CurStrOffset;
- std::vector<llvm::StringMapEntry<OffsetOpt>*> StrEntries;
-
- //// Get the persistent id for the given IdentifierInfo*.
- uint32_t ResolveID(const IdentifierInfo* II);
-
- /// Emit a token to the PTH file.
- void EmitToken(const Token& T);
-
- void Emit8(uint32_t V) {
- Out << (unsigned char)(V);
- }
-
- void Emit16(uint32_t V) { ::Emit16(Out, V); }
-
- void Emit24(uint32_t V) {
- Out << (unsigned char)(V);
- Out << (unsigned char)(V >> 8);
- Out << (unsigned char)(V >> 16);
- assert((V >> 24) == 0);
- }
-
- void Emit32(uint32_t V) { ::Emit32(Out, V); }
-
- void EmitBuf(const char *Ptr, unsigned NumBytes) {
- Out.write(Ptr, NumBytes);
- }
-
- /// EmitIdentifierTable - Emits two tables to the PTH file. The first is
- /// a hashtable mapping from identifier strings to persistent IDs.
- /// The second is a straight table mapping from persistent IDs to string data
- /// (the keys of the first table).
- std::pair<Offset, Offset> EmitIdentifierTable();
-
- /// EmitFileTable - Emit a table mapping from file name strings to PTH
- /// token data.
- Offset EmitFileTable() { return PM.Emit(Out); }
-
- PTHEntry LexTokens(Lexer& L);
- Offset EmitCachedSpellings();
-
-public:
- PTHWriter(llvm::raw_fd_ostream& out, Preprocessor& pp)
- : Out(out), PP(pp), idcount(0), CurStrOffset(0) {}
-
- PTHMap &getPM() { return PM; }
- void GeneratePTH(const std::string *MainFile = 0);
-};
-} // end anonymous namespace
-
-uint32_t PTHWriter::ResolveID(const IdentifierInfo* II) {
- // Null IdentifierInfo's map to the persistent ID 0.
- if (!II)
- return 0;
-
- IDMap::iterator I = IM.find(II);
- if (I != IM.end())
- return I->second; // We've already added 1.
-
- IM[II] = ++idcount; // Pre-increment since '0' is reserved for NULL.
- return idcount;
-}
-
-void PTHWriter::EmitToken(const Token& T) {
- // Emit the token kind, flags, and length.
- Emit32(((uint32_t) T.getKind()) | ((((uint32_t) T.getFlags())) << 8)|
- (((uint32_t) T.getLength()) << 16));
-
- if (!T.isLiteral()) {
- Emit32(ResolveID(T.getIdentifierInfo()));
- } else {
- // We cache *un-cleaned* spellings. This gives us 100% fidelity with the
- // source code.
- const char* s = T.getLiteralData();
- unsigned len = T.getLength();
-
- // Get the string entry.
- llvm::StringMapEntry<OffsetOpt> *E = &CachedStrs.GetOrCreateValue(s, s+len);
-
- // If this is a new string entry, bump the PTH offset.
- if (!E->getValue().hasOffset()) {
- E->getValue().setOffset(CurStrOffset);
- StrEntries.push_back(E);
- CurStrOffset += len + 1;
- }
-
- // Emit the relative offset into the PTH file for the spelling string.
- Emit32(E->getValue().getOffset());
- }
-
- // Emit the offset into the original source file of this token so that we
- // can reconstruct its SourceLocation.
- Emit32(PP.getSourceManager().getFileOffset(T.getLocation()));
-}
-
-PTHEntry PTHWriter::LexTokens(Lexer& L) {
- // Pad 0's so that we emit tokens to a 4-byte alignment.
- // This speed up reading them back in.
- Pad(Out, 4);
- Offset off = (Offset) Out.tell();
-
- // Keep track of matching '#if' ... '#endif'.
- typedef std::vector<std::pair<Offset, unsigned> > PPCondTable;
- PPCondTable PPCond;
- std::vector<unsigned> PPStartCond;
- bool ParsingPreprocessorDirective = false;
- Token Tok;
-
- do {
- L.LexFromRawLexer(Tok);
- NextToken:
-
- if ((Tok.isAtStartOfLine() || Tok.is(tok::eof)) &&
- ParsingPreprocessorDirective) {
- // Insert an eom token into the token cache. It has the same
- // position as the next token that is not on the same line as the
- // preprocessor directive. Observe that we continue processing
- // 'Tok' when we exit this branch.
- Token Tmp = Tok;
- Tmp.setKind(tok::eom);
- Tmp.clearFlag(Token::StartOfLine);
- Tmp.setIdentifierInfo(0);
- EmitToken(Tmp);
- ParsingPreprocessorDirective = false;
- }
-
- if (Tok.is(tok::identifier)) {
- Tok.setIdentifierInfo(PP.LookUpIdentifierInfo(Tok));
- EmitToken(Tok);
- continue;
- }
-
- if (Tok.is(tok::hash) && Tok.isAtStartOfLine()) {
- // Special processing for #include. Store the '#' token and lex
- // the next token.
- assert(!ParsingPreprocessorDirective);
- Offset HashOff = (Offset) Out.tell();
- EmitToken(Tok);
-
- // Get the next token.
- L.LexFromRawLexer(Tok);
-
- // If we see the start of line, then we had a null directive "#".
- if (Tok.isAtStartOfLine())
- goto NextToken;
-
- // Did we see 'include'/'import'/'include_next'?
- if (Tok.isNot(tok::identifier)) {
- EmitToken(Tok);
- continue;
- }
-
- IdentifierInfo* II = PP.LookUpIdentifierInfo(Tok);
- Tok.setIdentifierInfo(II);
- tok::PPKeywordKind K = II->getPPKeywordID();
-
- ParsingPreprocessorDirective = true;
-
- switch (K) {
- case tok::pp_not_keyword:
- // Invalid directives "#foo" can occur in #if 0 blocks etc, just pass
- // them through.
- default:
- break;
-
- case tok::pp_include:
- case tok::pp_import:
- case tok::pp_include_next: {
- // Save the 'include' token.
- EmitToken(Tok);
- // Lex the next token as an include string.
- L.setParsingPreprocessorDirective(true);
- L.LexIncludeFilename(Tok);
- L.setParsingPreprocessorDirective(false);
- assert(!Tok.isAtStartOfLine());
- if (Tok.is(tok::identifier))
- Tok.setIdentifierInfo(PP.LookUpIdentifierInfo(Tok));
-
- break;
- }
- case tok::pp_if:
- case tok::pp_ifdef:
- case tok::pp_ifndef: {
- // Add an entry for '#if' and friends. We initially set the target
- // index to 0. This will get backpatched when we hit #endif.
- PPStartCond.push_back(PPCond.size());
- PPCond.push_back(std::make_pair(HashOff, 0U));
- break;
- }
- case tok::pp_endif: {
- // Add an entry for '#endif'. We set the target table index to itself.
- // This will later be set to zero when emitting to the PTH file. We
- // use 0 for uninitialized indices because that is easier to debug.
- unsigned index = PPCond.size();
- // Backpatch the opening '#if' entry.
- assert(!PPStartCond.empty());
- assert(PPCond.size() > PPStartCond.back());
- assert(PPCond[PPStartCond.back()].second == 0);
- PPCond[PPStartCond.back()].second = index;
- PPStartCond.pop_back();
- // Add the new entry to PPCond.
- PPCond.push_back(std::make_pair(HashOff, index));
- EmitToken(Tok);
-
- // Some files have gibberish on the same line as '#endif'.
- // Discard these tokens.
- do
- L.LexFromRawLexer(Tok);
- while (Tok.isNot(tok::eof) && !Tok.isAtStartOfLine());
- // We have the next token in hand.
- // Don't immediately lex the next one.
- goto NextToken;
- }
- case tok::pp_elif:
- case tok::pp_else: {
- // Add an entry for #elif or #else.
- // This serves as both a closing and opening of a conditional block.
- // This means that its entry will get backpatched later.
- unsigned index = PPCond.size();
- // Backpatch the previous '#if' entry.
- assert(!PPStartCond.empty());
- assert(PPCond.size() > PPStartCond.back());
- assert(PPCond[PPStartCond.back()].second == 0);
- PPCond[PPStartCond.back()].second = index;
- PPStartCond.pop_back();
- // Now add '#elif' as a new block opening.
- PPCond.push_back(std::make_pair(HashOff, 0U));
- PPStartCond.push_back(index);
- break;
- }
- }
- }
-
- EmitToken(Tok);
- }
- while (Tok.isNot(tok::eof));
-
- assert(PPStartCond.empty() && "Error: imblanced preprocessor conditionals.");
-
- // Next write out PPCond.
- Offset PPCondOff = (Offset) Out.tell();
-
- // Write out the size of PPCond so that clients can identifer empty tables.
- Emit32(PPCond.size());
-
- for (unsigned i = 0, e = PPCond.size(); i!=e; ++i) {
- Emit32(PPCond[i].first - off);
- uint32_t x = PPCond[i].second;
- assert(x != 0 && "PPCond entry not backpatched.");
- // Emit zero for #endifs. This allows us to do checking when
- // we read the PTH file back in.
- Emit32(x == i ? 0 : x);
- }
-
- return PTHEntry(off, PPCondOff);
-}
-
-Offset PTHWriter::EmitCachedSpellings() {
- // Write each cached strings to the PTH file.
- Offset SpellingsOff = Out.tell();
-
- for (std::vector<llvm::StringMapEntry<OffsetOpt>*>::iterator
- I = StrEntries.begin(), E = StrEntries.end(); I!=E; ++I)
- EmitBuf((*I)->getKeyData(), (*I)->getKeyLength()+1 /*nul included*/);
-
- return SpellingsOff;
-}
-
-void PTHWriter::GeneratePTH(const std::string *MainFile) {
- // Generate the prologue.
- Out << "cfe-pth";
- Emit32(PTHManager::Version);
-
- // Leave 4 words for the prologue.
- Offset PrologueOffset = Out.tell();
- for (unsigned i = 0; i < 4; ++i)
- Emit32(0);
-
- // Write the name of the MainFile.
- if (MainFile && !MainFile->empty()) {
- Emit16(MainFile->length());
- EmitBuf(MainFile->data(), MainFile->length());
- } else {
- // String with 0 bytes.
- Emit16(0);
- }
- Emit8(0);
-
- // Iterate over all the files in SourceManager. Create a lexer
- // for each file and cache the tokens.
- SourceManager &SM = PP.getSourceManager();
- const LangOptions &LOpts = PP.getLangOptions();
-
- for (SourceManager::fileinfo_iterator I = SM.fileinfo_begin(),
- E = SM.fileinfo_end(); I != E; ++I) {
- const SrcMgr::ContentCache &C = *I->second;
- const FileEntry *FE = C.Entry;
-
- // FIXME: Handle files with non-absolute paths.
- llvm::sys::Path P(FE->getName());
- if (!P.isAbsolute())
- continue;
-
- const llvm::MemoryBuffer *B = C.getBuffer();
- if (!B) continue;
-
- FileID FID = SM.createFileID(FE, SourceLocation(), SrcMgr::C_User);
- Lexer L(FID, SM, LOpts);
- PM.insert(FE, LexTokens(L));
- }
-
- // Write out the identifier table.
- const std::pair<Offset,Offset> &IdTableOff = EmitIdentifierTable();
-
- // Write out the cached strings table.
- Offset SpellingOff = EmitCachedSpellings();
-
- // Write out the file table.
- Offset FileTableOff = EmitFileTable();
-
- // Finally, write the prologue.
- Out.seek(PrologueOffset);
- Emit32(IdTableOff.first);
- Emit32(IdTableOff.second);
- Emit32(FileTableOff);
- Emit32(SpellingOff);
-}
-
-namespace {
-/// StatListener - A simple "interpose" object used to monitor stat calls
-/// invoked by FileManager while processing the original sources used
-/// as input to PTH generation. StatListener populates the PTHWriter's
-/// file map with stat information for directories as well as negative stats.
-/// Stat information for files are populated elsewhere.
-class StatListener : public StatSysCallCache {
- PTHMap &PM;
-public:
- StatListener(PTHMap &pm) : PM(pm) {}
- ~StatListener() {}
-
- int stat(const char *path, struct stat *buf) {
- int result = ::stat(path, buf);
-
- if (result != 0) // Failed 'stat'.
- PM.insert(path, PTHEntry());
- else if (S_ISDIR(buf->st_mode)) {
- // Only cache directories with absolute paths.
- if (!llvm::sys::Path(path).isAbsolute())
- return result;
-
- PM.insert(PTHEntryKeyVariant(buf, path), PTHEntry());
- }
-
- return result;
- }
-};
-} // end anonymous namespace
-
-
-void clang::CacheTokens(Preprocessor &PP, llvm::raw_fd_ostream* OS) {
- // Get the name of the main file.
- const SourceManager &SrcMgr = PP.getSourceManager();
- const FileEntry *MainFile = SrcMgr.getFileEntryForID(SrcMgr.getMainFileID());
- llvm::sys::Path MainFilePath(MainFile->getName());
- std::string MainFileName;
-
- if (!MainFilePath.isAbsolute()) {
- llvm::sys::Path P = llvm::sys::Path::GetCurrentDirectory();
- P.appendComponent(MainFilePath.toString());
- MainFileName = P.toString();
- } else {
- MainFileName = MainFilePath.toString();
- }
-
- // Create the PTHWriter.
- PTHWriter PW(*OS, PP);
-
- // Install the 'stat' system call listener in the FileManager.
- PP.getFileManager().setStatCache(new StatListener(PW.getPM()));
-
- // Lex through the entire file. This will populate SourceManager with
- // all of the header information.
- Token Tok;
- PP.EnterMainSourceFile();
- do { PP.Lex(Tok); } while (Tok.isNot(tok::eof));
-
- // Generate the PTH file.
- PP.getFileManager().setStatCache(0);
- PW.GeneratePTH(&MainFileName);
-}
-
-//===----------------------------------------------------------------------===//
-
-class PTHIdKey {
-public:
- const IdentifierInfo* II;
- uint32_t FileOffset;
-};
-
-namespace {
-class VISIBILITY_HIDDEN PTHIdentifierTableTrait {
-public:
- typedef PTHIdKey* key_type;
- typedef key_type key_type_ref;
-
- typedef uint32_t data_type;
- typedef data_type data_type_ref;
-
- static unsigned ComputeHash(PTHIdKey* key) {
- return BernsteinHash(key->II->getName());
- }
-
- static std::pair<unsigned,unsigned>
- EmitKeyDataLength(llvm::raw_ostream& Out, const PTHIdKey* key, uint32_t) {
- unsigned n = strlen(key->II->getName()) + 1;
- ::Emit16(Out, n);
- return std::make_pair(n, sizeof(uint32_t));
- }
-
- static void EmitKey(llvm::raw_ostream& Out, PTHIdKey* key, unsigned n) {
- // Record the location of the key data. This is used when generating
- // the mapping from persistent IDs to strings.
- key->FileOffset = Out.tell();
- Out.write(key->II->getName(), n);
- }
-
- static void EmitData(llvm::raw_ostream& Out, PTHIdKey*, uint32_t pID,
- unsigned) {
- ::Emit32(Out, pID);
- }
-};
-} // end anonymous namespace
-
-/// EmitIdentifierTable - Emits two tables to the PTH file. The first is
-/// a hashtable mapping from identifier strings to persistent IDs. The second
-/// is a straight table mapping from persistent IDs to string data (the
-/// keys of the first table).
-///
-std::pair<Offset,Offset> PTHWriter::EmitIdentifierTable() {
- // Build two maps:
- // (1) an inverse map from persistent IDs -> (IdentifierInfo*,Offset)
- // (2) a map from (IdentifierInfo*, Offset)* -> persistent IDs
-
- // Note that we use 'calloc', so all the bytes are 0.
- PTHIdKey *IIDMap = (PTHIdKey*)calloc(idcount, sizeof(PTHIdKey));
-
- // Create the hashtable.
- OnDiskChainedHashTableGenerator<PTHIdentifierTableTrait> IIOffMap;
-
- // Generate mapping from persistent IDs -> IdentifierInfo*.
- for (IDMap::iterator I = IM.begin(), E = IM.end(); I != E; ++I) {
- // Decrement by 1 because we are using a vector for the lookup and
- // 0 is reserved for NULL.
- assert(I->second > 0);
- assert(I->second-1 < idcount);
- unsigned idx = I->second-1;
-
- // Store the mapping from persistent ID to IdentifierInfo*
- IIDMap[idx].II = I->first;
-
- // Store the reverse mapping in a hashtable.
- IIOffMap.insert(&IIDMap[idx], I->second);
- }
-
- // Write out the inverse map first. This causes the PCIDKey entries to
- // record PTH file offsets for the string data. This is used to write
- // the second table.
- Offset StringTableOffset = IIOffMap.Emit(Out);
-
- // Now emit the table mapping from persistent IDs to PTH file offsets.
- Offset IDOff = Out.tell();
- Emit32(idcount); // Emit the number of identifiers.
- for (unsigned i = 0 ; i < idcount; ++i)
- Emit32(IIDMap[i].FileOffset);
-
- // Finally, release the inverse map.
- free(IIDMap);
-
- return std::make_pair(IDOff, StringTableOffset);
-}