Stage two of getting CFE top correct.

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@39734 91177308-0d34-0410-b5e6-96231b3b80d8

13 files changed, 2164 insertions, 0 deletions

diff --git a/include/clang/Lex/DirectoryLookup.h b/include/clang/Lex/DirectoryLookup.h
new file mode 100644
index 0000000000..a1cfb0a340
--- /dev/null
+++ b/include/clang/Lex/DirectoryLookup.h
@@ -0,0 +1,70 @@
+//===--- DirectoryLookup.h - Info for searching for headers -----*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file was developed by Chris Lattner and is distributed under
+// the University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the DirectoryLookup interface.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LEX_DIRECTORYLOOKUP_H
+#define LLVM_CLANG_LEX_DIRECTORYLOOKUP_H
+
+namespace clang {
+class DirectoryEntry;
+
+/// DirectoryLookup - This class is used to specify the search order for
+/// directories in #include directives.
+class DirectoryLookup {
+public:
+  enum DirType {
+    NormalHeaderDir,
+    SystemHeaderDir,
+    ExternCSystemHeaderDir
+  };
+private:  
+  /// Dir - This is the actual directory that we're referring to.
+  ///
+  const DirectoryEntry *Dir;
+  
+  /// DirCharacteristic - The type of directory this is, one of the DirType enum
+  /// values.
+  DirType DirCharacteristic : 2;
+  
+  /// UserSupplied - True if this is a user-supplied directory.
+  ///
+  bool UserSupplied : 1;
+  
+  /// Framework - True if this is a framework directory search-path.
+  ///
+  bool Framework : 1;
+public:
+  DirectoryLookup(const DirectoryEntry *dir, DirType DT, bool isUser,
+                  bool isFramework)
+    : Dir(dir), DirCharacteristic(DT), UserSupplied(isUser),
+      Framework(isFramework) {}
+  
+  /// getDir - Return the directory that this entry refers to.
+  ///
+  const DirectoryEntry *getDir() const { return Dir; }
+  
+  /// DirCharacteristic - The type of directory this is, one of the DirType enum
+  /// values.
+  DirType getDirCharacteristic() const { return DirCharacteristic; }
+  
+  /// isUserSupplied - True if this is a user-supplied directory.
+  ///
+  bool isUserSupplied() const { return UserSupplied; }
+  
+  /// isFramework - True if this is a framework directory.
+  ///
+  bool isFramework() const { return Framework; }
+};
+
+}  // end namespace clang
+
+#endif
diff --git a/include/clang/Lex/HeaderSearch.h b/include/clang/Lex/HeaderSearch.h
new file mode 100644
index 0000000000..79b3366ecd
--- /dev/null
+++ b/include/clang/Lex/HeaderSearch.h
@@ -0,0 +1,166 @@
+//===--- HeaderSearch.h - Resolve Header File Locations ---------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file was developed by Chris Lattner and is distributed under
+// the University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the HeaderSearch interface.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LEX_HEADERSEARCH_H
+#define LLVM_CLANG_LEX_HEADERSEARCH_H
+
+#include "clang/Lex/DirectoryLookup.h"
+#include "llvm/ADT/StringMap.h"
+#include <vector>
+
+namespace clang {
+class FileEntry;
+class FileManager;
+class IdentifierInfo;
+
+  
+/// HeaderSearch - This class encapsulates the information needed to find the
+/// file referenced by a #include or #include_next, (sub-)framework lookup, etc.
+class HeaderSearch {
+  FileManager &FileMgr;
+  
+  /// #include search path information.  Requests for #include "x" search the
+  /// directory of the #including file first, then each directory in SearchDirs
+  /// consequtively. Requests for <x> search the current dir first, then each
+  /// directory in SearchDirs, starting at SystemDirIdx, consequtively.  If
+  /// NoCurDirSearch is true, then the check for the file in the current
+  /// directory is supressed.
+  std::vector<DirectoryLookup> SearchDirs;
+  unsigned SystemDirIdx;
+  bool NoCurDirSearch;
+  
+  /// PreFileInfo - The preprocessor keeps track of this information for each
+  /// file that is #included.
+  struct PerFileInfo {
+    /// isImport - True if this is a #import'd or #pragma once file.
+    bool isImport : 1;
+    
+    /// DirInfo - Keep track of whether this is a system header, and if so,
+    /// whether it is C++ clean or not.  This can be set by the include paths or
+    /// by #pragma gcc system_header.
+    DirectoryLookup::DirType DirInfo : 2;
+    
+    /// NumIncludes - This is the number of times the file has been included
+    /// already.
+    unsigned short NumIncludes;
+    
+    /// ControllingMacro - If this file has a #ifndef XXX (or equivalent) guard
+    /// that protects the entire contents of the file, this is the identifier
+    /// for the macro that controls whether or not it has any effect.
+    const IdentifierInfo *ControllingMacro;
+    
+    PerFileInfo() : isImport(false), DirInfo(DirectoryLookup::NormalHeaderDir),
+      NumIncludes(0), ControllingMacro(0) {}
+  };
+  
+  /// FileInfo - This contains all of the preprocessor-specific data about files
+  /// that are included.  The vector is indexed by the FileEntry's UID.
+  ///
+  std::vector<PerFileInfo> FileInfo;
+
+  /// FrameworkMap - This is a collection mapping a framework or subframework
+  /// name like "Carbon" to the Carbon.framework directory.
+  llvm::StringMap<const DirectoryEntry *> FrameworkMap;
+
+  // Various statistics we track for performance analysis.
+  unsigned NumIncluded;
+  unsigned NumMultiIncludeFileOptzn;
+  unsigned NumFrameworkLookups, NumSubFrameworkLookups;
+public:
+  HeaderSearch(FileManager &FM);
+
+  FileManager &getFileMgr() const { return FileMgr; }
+
+  /// SetSearchPaths - Interface for setting the file search paths.
+  ///
+  void SetSearchPaths(const std::vector<DirectoryLookup> &dirs,
+                      unsigned systemDirIdx, bool noCurDirSearch) {
+    SearchDirs = dirs;
+    SystemDirIdx = systemDirIdx;
+    NoCurDirSearch = noCurDirSearch;
+  }
+  
+  /// ClearFileInfo - Forget everything we know about headers so far.
+  void ClearFileInfo() {
+    FileInfo.clear();
+  }
+  
+  /// LookupFile - Given a "foo" or <foo> reference, look up the indicated file,
+  /// return null on failure.  isAngled indicates whether the file reference is
+  /// a <> reference.  If successful, this returns 'UsedDir', the
+  /// DirectoryLookup member the file was found in, or null if not applicable.
+  /// If CurDir is non-null, the file was found in the specified directory
+  /// search location.  This is used to implement #include_next.  CurFileEnt, if
+  /// non-null, indicates where the #including file is, in case a relative
+  /// search is needed.
+  const FileEntry *LookupFile(const char *FilenameStart,
+                              const char *FilenameEnd, bool isAngled,
+                              const DirectoryLookup *FromDir,
+                              const DirectoryLookup *&CurDir,
+                              const FileEntry *CurFileEnt);
+  
+  /// LookupSubframeworkHeader - Look up a subframework for the specified
+  /// #include file.  For example, if #include'ing <HIToolbox/HIToolbox.h> from
+  /// within ".../Carbon.framework/Headers/Carbon.h", check to see if HIToolbox
+  /// is a subframework within Carbon.framework.  If so, return the FileEntry
+  /// for the designated file, otherwise return null.
+  const FileEntry *LookupSubframeworkHeader(const char *FilenameStart,
+                                            const char *FilenameEnd,
+                                            const FileEntry *RelativeFileEnt);
+  
+  /// ShouldEnterIncludeFile - Mark the specified file as a target of of a
+  /// #include, #include_next, or #import directive.  Return false if #including
+  /// the file will have no effect or true if we should include it.
+  bool ShouldEnterIncludeFile(const FileEntry *File, bool isImport);
+  
+  
+  /// getFileDirFlavor - Return whether the specified file is a normal header,
+  /// a system header, or a C++ friendly system header.
+  DirectoryLookup::DirType getFileDirFlavor(const FileEntry *File) {
+    return getFileInfo(File).DirInfo;
+  }
+    
+  /// MarkFileIncludeOnce - Mark the specified file as a "once only" file, e.g.
+  /// due to #pragma once.
+  void MarkFileIncludeOnce(const FileEntry *File) {
+    getFileInfo(File).isImport = true;
+  }
+
+  /// MarkFileSystemHeader - Mark the specified fiel as a system header, e.g.
+  /// due to #pragma GCC system_header.
+  void MarkFileSystemHeader(const FileEntry *File) {
+    getFileInfo(File).DirInfo = DirectoryLookup::SystemHeaderDir;
+  }
+  
+  /// SetFileControllingMacro - Mark the specified file as having a controlling
+  /// macro.  This is used by the multiple-include optimization to eliminate
+  /// no-op #includes.
+  void SetFileControllingMacro(const FileEntry *File,
+                               const IdentifierInfo *ControllingMacro) {
+    getFileInfo(File).ControllingMacro = ControllingMacro;
+  }
+  
+  void PrintStats();
+private:
+  const FileEntry *DoFrameworkLookup(const DirectoryEntry *Dir,
+                                     const char *FilenameStart,
+                                     const char *FilenameEnd);
+      
+  /// getFileInfo - Return the PerFileInfo structure for the specified
+  /// FileEntry.
+  PerFileInfo &getFileInfo(const FileEntry *FE);
+};
+
+}  // end namespace clang
+
+#endif
diff --git a/include/clang/Lex/IdentifierTable.h b/include/clang/Lex/IdentifierTable.h
new file mode 100644
index 0000000000..4edba60094
--- /dev/null
+++ b/include/clang/Lex/IdentifierTable.h
@@ -0,0 +1,171 @@
+//===--- IdentifierTable.h - Hash table for identifier lookup ---*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file was developed by Chris Lattner and is distributed under
+// the University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the IdentifierInfo and IdentifierTable interfaces.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LEX_IDENTIFIERTABLE_H
+#define LLVM_CLANG_LEX_IDENTIFIERTABLE_H
+
+#include "clang/Basic/TokenKinds.h"
+#include "llvm/ADT/StringMap.h"
+#include <string> 
+
+namespace clang {
+  class MacroInfo;
+  class LangOptions;
+  
+/// IdentifierInfo - One of these records is kept for each identifier that
+/// is lexed.  This contains information about whether the token was #define'd,
+/// is a language keyword, or if it is a front-end token of some sort (e.g. a
+/// variable or function name).  The preprocessor keeps this information in a
+/// set, and all tok::identifier tokens have a pointer to one of these.  
+class IdentifierInfo {
+  MacroInfo *Macro;                // Set if this identifier is #define'd.
+  tok::TokenKind TokenID      : 8; // Front-end token ID or tok::identifier.
+  tok::PPKeywordKind PPID     : 5; // ID for preprocessor command like #'ifdef'.
+  tok::ObjCKeywordKind ObjCID : 5; // ID for objc @ keyword like @'protocol'.
+  unsigned BuiltinID          :12; // ID if this is a builtin (__builtin_inf).
+  bool IsExtension            : 1; // True if identifier is a lang extension.
+  bool IsPoisoned             : 1; // True if identifier is poisoned.
+  bool IsOtherTargetMacro     : 1; // True if ident is macro on another target.
+  bool IsCPPOperatorKeyword   : 1; // True if ident is a C++ operator keyword.
+  bool IsNonPortableBuiltin   : 1; // True if builtin varies across targets.
+  void *FETokenInfo;               // Managed by the language front-end.
+  IdentifierInfo(const IdentifierInfo&);  // NONCOPYABLE.
+public:
+  IdentifierInfo();
+  ~IdentifierInfo();
+
+  /// getName - Return the actual string for this identifier.  The length of
+  /// this string is stored in NameLen, and the returned string is properly null
+  /// terminated.
+  ///
+  const char *getName() const {
+    // String data is stored immediately after the IdentifierInfo object.
+    return (const char*)(this+1);
+  }
+  
+  /// getMacroInfo - Return macro information about this identifier, or null if
+  /// it is not a macro.
+  MacroInfo *getMacroInfo() const { return Macro; }
+  void setMacroInfo(MacroInfo *I) { Macro = I; }
+  
+  /// get/setTokenID - If this is a source-language token (e.g. 'for'), this API
+  /// can be used to cause the lexer to map identifiers to source-language
+  /// tokens.
+  tok::TokenKind getTokenID() const { return TokenID; }
+  void setTokenID(tok::TokenKind ID) { TokenID = ID; }
+  
+  /// getPPKeywordID - Return the preprocessor keyword ID for this identifier.
+  /// For example, define will return tok::pp_define.
+  tok::PPKeywordKind getPPKeywordID() const { return PPID; }
+  void setPPKeywordID(tok::PPKeywordKind ID) { PPID = ID; }
+  
+  /// getObjCKeywordID - Return the Objective-C keyword ID for the this
+  /// identifier.  For example, 'class' will return tok::objc_class if ObjC is
+  /// enabled.
+  tok::ObjCKeywordKind getObjCKeywordID() const { return ObjCID; }
+  void setObjCKeywordID(tok::ObjCKeywordKind ID) { ObjCID = ID; }
+  
+  /// getBuiltinID - Return a value indicating whether this is a builtin
+  /// function.  0 is not-built-in.  1 is builtin-for-some-nonprimary-target.
+  /// 2+ are specific builtin functions.
+  unsigned getBuiltinID() const { return BuiltinID; }
+  void setBuiltinID(unsigned ID) {
+    assert(ID < (1 << 12) && "ID too large for field!");
+    BuiltinID = ID;
+  }
+  
+  /// isNonPortableBuiltin - Return true if this identifier corresponds to a
+  /// builtin on some other target, but isn't one on this target, or if it is on
+  /// the target but not on another, or if it is on both but it differs somehow
+  /// in behavior.
+  bool isNonPortableBuiltin() const { return IsNonPortableBuiltin; }
+  void setNonPortableBuiltin(bool Val) { IsNonPortableBuiltin = Val; }
+  
+  /// get/setExtension - Initialize information about whether or not this
+  /// language token is an extension.  This controls extension warnings, and is
+  /// only valid if a custom token ID is set.
+  bool isExtensionToken() const { return IsExtension; }
+  void setIsExtensionToken(bool Val) { IsExtension = Val; }
+  
+  /// setIsPoisoned - Mark this identifier as poisoned.  After poisoning, the
+  /// Preprocessor will emit an error every time this token is used.
+  void setIsPoisoned(bool Value = true) { IsPoisoned = Value; }
+  
+  /// isPoisoned - Return true if this token has been poisoned.
+  bool isPoisoned() const { return IsPoisoned; }
+  
+  /// setIsOtherTargetMacro/isOtherTargetMacro control whether this identifier
+  /// is seen as being a macro on some other target.
+  void setIsOtherTargetMacro(bool Val = true) { IsOtherTargetMacro = Val; }
+  bool isOtherTargetMacro() const { return IsOtherTargetMacro; }
+
+  /// isCPlusPlusOperatorKeyword/setIsCPlusPlusOperatorKeyword controls whether
+  /// this identifier is a C++ alternate representation of an operator.
+  void setIsCPlusplusOperatorKeyword(bool Val = true)
+    { IsCPPOperatorKeyword = Val; }
+  bool isCPlusPlusOperatorKeyword() const { return IsCPPOperatorKeyword; }
+
+  /// getFETokenInfo/setFETokenInfo - The language front-end is allowed to
+  /// associate arbitrary metadata with this token.
+  template<typename T>
+  T *getFETokenInfo() const { return static_cast<T*>(FETokenInfo); }
+  void setFETokenInfo(void *T) { FETokenInfo = T; }
+};
+
+
+
+/// IdentifierTable - This table implements an efficient mapping from strings to
+/// IdentifierInfo nodes.  It has no other purpose, but this is an
+/// extremely performance-critical piece of the code, as each occurrance of
+/// every identifier goes through here when lexed.
+class IdentifierTable {
+  // Shark shows that using MallocAllocator is *much* slower than using this
+  // BumpPtrAllocator!
+  typedef llvm::StringMap<IdentifierInfo, llvm::BumpPtrAllocator> HashTableTy;
+  HashTableTy HashTable;
+public:
+  /// IdentifierTable ctor - Create the identifier table, populating it with
+  /// info about the language keywords for the language specified by LangOpts.
+  IdentifierTable(const LangOptions &LangOpts);
+  
+  /// get - Return the identifier token info for the specified named identifier.
+  ///
+  IdentifierInfo &get(const char *NameStart, const char *NameEnd) {
+    return HashTable.GetOrCreateValue(NameStart, NameEnd).getValue();
+  }
+  
+  IdentifierInfo &get(const char *Name) {
+    return get(Name, Name+strlen(Name));
+  }
+  IdentifierInfo &get(const std::string &Name) {
+    // Don't use c_str() here: no need to be null terminated.
+    const char *NameBytes = &Name[0];
+    return get(NameBytes, NameBytes+Name.size());
+  }
+  
+  typedef HashTableTy::const_iterator iterator;
+  typedef HashTableTy::const_iterator const_iterator;
+  
+  iterator begin() const { return HashTable.begin(); }
+  iterator end() const   { return HashTable.end(); }
+  
+  /// PrintStats - Print some statistics to stderr that indicate how well the
+  /// hashing is doing.
+  void PrintStats() const;
+private:
+  void AddKeywords(const LangOptions &LangOpts);
+};
+
+}  // end namespace clang
+
+#endif
diff --git a/include/clang/Lex/Lexer.h b/include/clang/Lex/Lexer.h
new file mode 100644
index 0000000000..db0c7183d2
--- /dev/null
+++ b/include/clang/Lex/Lexer.h
@@ -0,0 +1,353 @@
+//===--- Lexer.h - C Language Family Lexer ----------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file was developed by Chris Lattner and is distributed under
+// the University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines the Lexer interface.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LEXER_H
+#define LLVM_CLANG_LEXER_H
+
+#include "clang/Lex/LexerToken.h"
+#include "clang/Lex/MultipleIncludeOpt.h"
+#include "clang/Basic/LangOptions.h"
+#include <string>
+#include <vector>
+
+namespace llvm {
+  class MemoryBuffer;
+}
+  
+namespace clang {
+class Diagnostic;
+class Preprocessor;
+
+/// Lexer - This provides a simple interface that turns a text buffer into a
+/// stream of tokens.  This provides no support for file reading or buffering,
+/// or buffering/seeking of tokens, only forward lexing is supported.  It relies
+/// on the specified Preprocessor object to handle preprocessor directives, etc.
+class Lexer {
+  //===--------------------------------------------------------------------===//
+  // Constant configuration values for this lexer.
+  const char * const BufferEnd;  // End of the buffer.
+  const llvm::MemoryBuffer *InputFile; // The file we are reading from.
+  unsigned CurFileID;            // FileID for the current input file.
+  Preprocessor &PP;              // Preprocessor object controlling lexing.
+  LangOptions Features;          // Features enabled by this language (cache).
+  bool Is_PragmaLexer;           // True if lexer for _Pragma handling.
+  bool IsMainFile;               // True if top-level file.
+  
+  //===--------------------------------------------------------------------===//
+  // Context-specific lexing flags set by the preprocessor.
+  //
+  
+  /// ParsingPreprocessorDirective - This is true when parsing #XXX.  This turns
+  /// '\n' into a tok::eom token.
+  bool ParsingPreprocessorDirective;
+  
+  /// ParsingFilename - True after #include: this turns <xx> into a
+  /// tok::angle_string_literal token.
+  bool ParsingFilename;
+  
+  /// LexingRawMode - True if in raw mode:  This flag disables interpretation of
+  /// tokens and is a far faster mode to lex in than non-raw-mode.  This flag:
+  ///  1. If EOF of the current lexer is found, the include stack isn't popped.
+  ///  2. Identifier information is not looked up for identifier tokens.  As an
+  ///     effect of this, implicit macro expansion is naturally disabled.
+  ///  3. "#" tokens at the start of a line are treated as normal tokens, not
+  ///     implicitly transformed by the lexer.
+  ///  4. All diagnostic messages are disabled, except for unterminated /*.
+  ///  5. The only callback made into the preprocessor is to report a hard error
+  ///     on an unterminated '/*' comment.
+  bool LexingRawMode;
+  
+  /// KeepCommentMode - The lexer can optionally keep C & BCPL-style comments,
+  /// and return them as tokens.  This is used for -C and -CC modes.
+  bool KeepCommentMode;
+  
+  //===--------------------------------------------------------------------===//
+  // Context that changes as the file is lexed.
+  // NOTE: any state that mutates when in raw mode must have save/restore code
+  // in Lexer::isNextPPTokenLParen.
+
+  // BufferPtr - Current pointer into the buffer.  This is the next character
+  // to be lexed.
+  const char *BufferPtr;
+
+  // IsAtStartOfLine - True if the next lexed token should get the "start of
+  // line" flag set on it.
+  bool IsAtStartOfLine;
+
+  /// MIOpt - This is a state machine that detects the #ifndef-wrapping a file 
+  /// idiom for the multiple-include optimization.
+  MultipleIncludeOpt MIOpt;
+  
+  /// ConditionalStack - Information about the set of #if/#ifdef/#ifndef blocks
+  /// we are currently in.
+  std::vector<PPConditionalInfo> ConditionalStack;
+  
+  friend class Preprocessor;
+public:
+    
+  /// Lexer constructor - Create a new lexer object for the specified buffer
+  /// with the specified preprocessor managing the lexing process.  This lexer
+  /// assumes that the specified MemoryBuffer and Preprocessor objects will
+  /// outlive it, but doesn't take ownership of either pointer.
+    Lexer(const llvm::MemoryBuffer *InBuffer, unsigned CurFileID, 
+          Preprocessor &PP, const char *BufStart = 0, const char *BufEnd = 0);
+  
+  /// getFeatures - Return the language features currently enabled.  NOTE: this
+  /// lexer modifies features as a file is parsed!
+  const LangOptions &getFeatures() const { return Features; }
+
+  /// getCurFileID - Return the FileID for the file we are lexing out of.  This
+  /// implicitly encodes the include path to get to the file.
+  unsigned getCurFileID() const { return CurFileID; }
+  
+  /// setIsMainFile - Mark this lexer as being the lexer for the top-level
+  /// source file.
+  void setIsMainFile() {
+    IsMainFile = true;
+  }
+  
+  /// isMainFile - Return true if this is the top-level file.
+  ///
+  bool isMainFile() const { return IsMainFile; }
+  
+  /// Lex - Return the next token in the file.  If this is the end of file, it
+  /// return the tok::eof token.  Return true if an error occurred and
+  /// compilation should terminate, false if normal.  This implicitly involves
+  /// the preprocessor.
+  void Lex(LexerToken &Result) {
+    // Start a new token.
+    Result.startToken();
+    
+    // NOTE, any changes here should also change code after calls to 
+    // Preprocessor::HandleDirective
+    if (IsAtStartOfLine) {
+      Result.setFlag(LexerToken::StartOfLine);
+      IsAtStartOfLine = false;
+    }
+   
+    // Get a token.  Note that this may delete the current lexer if the end of
+    // file is reached.
+    LexTokenInternal(Result);
+  }
+  
+  /// LexRawToken - Switch the lexer to raw mode, lex a token into Result and
+  /// switch it back.  Return true if the 'next character to read' pointer
+  /// points and the end of the lexer buffer, false otherwise.
+  bool LexRawToken(LexerToken &Result) {
+    assert(!LexingRawMode && "Already in raw mode!");
+    LexingRawMode = true;
+    Lex(Result);
+    LexingRawMode = false;
+    return BufferPtr == BufferEnd; 
+  }
+  
+  /// ReadToEndOfLine - Read the rest of the current preprocessor line as an
+  /// uninterpreted string.  This switches the lexer out of directive mode.
+  std::string ReadToEndOfLine();
+  
+ 
+  /// Diag - Forwarding function for diagnostics.  This translate a source
+  /// position in the current buffer into a SourceLocation object for rendering.
+  void Diag(const char *Loc, unsigned DiagID,
+            const std::string &Msg = std::string()) const;
+  void Diag(SourceLocation Loc, unsigned DiagID,
+            const std::string &Msg = std::string()) const;
+
+  /// getSourceLocation - Return a source location identifier for the specified
+  /// offset in the current file.
+  SourceLocation getSourceLocation(const char *Loc) const;
+  
+  /// Stringify - Convert the specified string into a C string by escaping '\'
+  /// and " characters.  This does not add surrounding ""'s to the string.
+  /// If Charify is true, this escapes the ' character instead of ".
+  static std::string Stringify(const std::string &Str, bool Charify = false);
+  
+  //===--------------------------------------------------------------------===//
+  // Internal implementation interfaces.
+private:
+
+  /// LexTokenInternal - Internal interface to lex a preprocessing token. Called
+  /// by Lex.
+  ///
+  void LexTokenInternal(LexerToken &Result);
+
+  /// FormTokenWithChars - When we lex a token, we have identified a span
+  /// starting at BufferPtr, going to TokEnd that forms the token.  This method
+  /// takes that range and assigns it to the token as its location and size.  In
+  /// addition, since tokens cannot overlap, this also updates BufferPtr to be
+  /// TokEnd.
+  void FormTokenWithChars(LexerToken &Result, const char *TokEnd) {
+    Result.setLocation(getSourceLocation(BufferPtr));
+    Result.setLength(TokEnd-BufferPtr);
+    BufferPtr = TokEnd;
+  }
+  
+  /// isNextPPTokenLParen - Return 1 if the next unexpanded token will return a
+  /// tok::l_paren token, 0 if it is something else and 2 if there are no more
+  /// tokens in the buffer controlled by this lexer.
+  unsigned isNextPPTokenLParen();
+
+  //===--------------------------------------------------------------------===//
+  // Lexer character reading interfaces.
+  
+  // This lexer is built on two interfaces for reading characters, both of which
+  // automatically provide phase 1/2 translation.  getAndAdvanceChar is used
+  // when we know that we will be reading a character from the input buffer and
+  // that this character will be part of the result token. This occurs in (f.e.)
+  // string processing, because we know we need to read until we find the
+  // closing '"' character.
+  //
+  // The second interface is the combination of PeekCharAndSize with
+  // ConsumeChar.  PeekCharAndSize reads a phase 1/2 translated character,
+  // returning it and its size.  If the lexer decides that this character is
+  // part of the current token, it calls ConsumeChar on it.  This two stage
+  // approach allows us to emit diagnostics for characters (e.g. warnings about
+  // trigraphs), knowing that they only are emitted if the character is
+  // consumed.
+  
+  
+  /// getAndAdvanceChar - Read a single 'character' from the specified buffer,
+  /// advance over it, and return it.  This is tricky in several cases.  Here we
+  /// just handle the trivial case and fall-back to the non-inlined
+  /// getCharAndSizeSlow method to handle the hard case.
+  inline char getAndAdvanceChar(const char *&Ptr, LexerToken &Tok) {
+    // If this is not a trigraph and not a UCN or escaped newline, return
+    // quickly.
+    if (Ptr[0] != '?' && Ptr[0] != '\\') return *Ptr++;
+    
+    unsigned Size = 0;
+    char C = getCharAndSizeSlow(Ptr, Size, &Tok);
+    Ptr += Size;
+    return C;
+  }
+  
+  /// ConsumeChar - When a character (identified by PeekCharAndSize) is consumed
+  /// and added to a given token, check to see if there are diagnostics that
+  /// need to be emitted or flags that need to be set on the token.  If so, do
+  /// it.
+  const char *ConsumeChar(const char *Ptr, unsigned Size, LexerToken &Tok) {
+    // Normal case, we consumed exactly one token.  Just return it.
+    if (Size == 1)
+      return Ptr+Size;
+
+    // Otherwise, re-lex the character with a current token, allowing
+    // diagnostics to be emitted and flags to be set.
+    Size = 0;
+    getCharAndSizeSlow(Ptr, Size, &Tok);
+    return Ptr+Size;
+  }
+  
+  /// getCharAndSize - Peek a single 'character' from the specified buffer,
+  /// get its size, and return it.  This is tricky in several cases.  Here we
+  /// just handle the trivial case and fall-back to the non-inlined
+  /// getCharAndSizeSlow method to handle the hard case.
+  inline char getCharAndSize(const char *Ptr, unsigned &Size) {
+    // If this is not a trigraph and not a UCN or escaped newline, return
+    // quickly.
+    if (Ptr[0] != '?' && Ptr[0] != '\\') {
+      Size = 1;
+      return *Ptr;
+    }
+    
+    Size = 0;
+    return getCharAndSizeSlow(Ptr, Size);
+  }
+  
+  /// getCharAndSizeSlow - Handle the slow/uncommon case of the getCharAndSize
+  /// method.
+  char getCharAndSizeSlow(const char *Ptr, unsigned &Size, LexerToken *Tok = 0);
+  
+  /// getCharAndSizeNoWarn - Like the getCharAndSize method, but does not ever
+  /// emit a warning.
+  static inline char getCharAndSizeNoWarn(const char *Ptr, unsigned &Size,
+                                          const LangOptions &Features) {
+    // If this is not a trigraph and not a UCN or escaped newline, return
+    // quickly.
+    if (Ptr[0] != '?' && Ptr[0] != '\\') {
+      Size = 1;
+      return *Ptr;
+    }
+    
+    Size = 0;
+    return getCharAndSizeSlowNoWarn(Ptr, Size, Features);
+  }
+  
+  /// getCharAndSizeSlowNoWarn - Same as getCharAndSizeSlow, but never emits a
+  /// diagnostic.
+  static char getCharAndSizeSlowNoWarn(const char *Ptr, unsigned &Size,
+                                       const LangOptions &Features);
+  
+  //===--------------------------------------------------------------------===//
+  // #if directive handling.
+  
+  /// pushConditionalLevel - When we enter a #if directive, this keeps track of
+  /// what we are currently in for diagnostic emission (e.g. #if with missing
+  /// #endif).
+  void pushConditionalLevel(SourceLocation DirectiveStart, bool WasSkipping,
+                            bool FoundNonSkip, bool FoundElse) {
+    PPConditionalInfo CI;
+    CI.IfLoc = DirectiveStart;
+    CI.WasSkipping = WasSkipping;
+    CI.FoundNonSkip = FoundNonSkip;
+    CI.FoundElse = FoundElse;
+    ConditionalStack.push_back(CI);
+  }
+  void pushConditionalLevel(const PPConditionalInfo &CI) {
+    ConditionalStack.push_back(CI);
+  }    
+  
+  /// popConditionalLevel - Remove an entry off the top of the conditional
+  /// stack, returning information about it.  If the conditional stack is empty,
+  /// this returns true and does not fill in the arguments.
+  bool popConditionalLevel(PPConditionalInfo &CI) {
+    if (ConditionalStack.empty()) return true;
+    CI = ConditionalStack.back();
+    ConditionalStack.pop_back();
+    return false;
+  }
+  
+  /// peekConditionalLevel - Return the top of the conditional stack.  This
+  /// requires that there be a conditional active.
+  PPConditionalInfo &peekConditionalLevel() {
+    assert(!ConditionalStack.empty() && "No conditionals active!");
+    return ConditionalStack.back();
+  }
+  
+  unsigned getConditionalStackDepth() const { return ConditionalStack.size(); }
+  
+  //===--------------------------------------------------------------------===//
+  // Other lexer functions.
+  
+  // Helper functions to lex the remainder of a token of the specific type.
+  void LexIdentifier         (LexerToken &Result, const char *CurPtr);
+  void LexNumericConstant    (LexerToken &Result, const char *CurPtr);
+  void LexStringLiteral      (LexerToken &Result, const char *CurPtr,bool Wide);
+  void LexAngledStringLiteral(LexerToken &Result, const char *CurPtr);
+  void LexCharConstant       (LexerToken &Result, const char *CurPtr);
+  bool LexEndOfFile          (LexerToken &Result, const char *CurPtr);
+  
+  void SkipWhitespace        (LexerToken &Result, const char *CurPtr);
+  bool SkipBCPLComment       (LexerToken &Result, const char *CurPtr);
+  bool SkipBlockComment      (LexerToken &Result, const char *CurPtr);
+  bool SaveBCPLComment       (LexerToken &Result, const char *CurPtr);
+  
+  /// LexIncludeFilename - After the preprocessor has parsed a #include, lex and
+  /// (potentially) macro expand the filename.  If the sequence parsed is not
+  /// lexically legal, emit a diagnostic and return a result EOM token.
+  void LexIncludeFilename(LexerToken &Result);
+};
+
+
+}  // end namespace clang
+
+#endif
diff --git a/include/clang/Lex/LexerToken.h b/include/clang/Lex/LexerToken.h
new file mode 100644
index 0000000000..119a1d92ad
--- /dev/null
+++ b/include/clang/Lex/LexerToken.h
@@ -0,0 +1,137 @@
+//===--- LexerToken.h - Token interface -------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file was developed by Chris Lattner and is distributed under
+// the University of Illinois Open Source License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines the LexerToken interface.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LEXERTOKEN_H
+#define LLVM_CLANG_LEXERTOKEN_H
+
+#include "clang/Basic/TokenKinds.h"
+#include "clang/Basic/SourceLocation.h"
+
+namespace clang {
+
+class IdentifierInfo;
+
+/// LexerToken - This structure provides full information about a lexed token.
+/// It is not intended to be space efficient, it is intended to return as much
+/// information as possible about each returned token.  This is expected to be
+/// compressed into a smaller form if memory footprint is important.
+class LexerToken {
+  /// The location and length of the token text itself.
+  SourceLocation Loc;
+  unsigned Length;
+  
+  /// IdentifierInfo - If this was an identifier, this points to the uniqued
+  /// information about this identifie