Make a major restructuring of the clang tree: introduce a top-level

lib dir and move all the libraries into it.  This follows the main
llvm tree, and allows the libraries to be built in parallel.  The
top level now enforces that all the libs are built before Driver,
but we don't care what order the libs are built in.  This speeds
up parallel builds, particularly incremental ones.


git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@48402 91177308-0d34-0410-b5e6-96231b3b80d8

10 files changed, 2787 insertions, 0 deletions

diff --git a/lib/Basic/Diagnostic.cpp b/lib/Basic/Diagnostic.cpp
new file mode 100644
index 0000000000..f62b8f126c
--- /dev/null
+++ b/lib/Basic/Diagnostic.cpp
@@ -0,0 +1,232 @@
+//===--- Diagnostic.cpp - C Language Family Diagnostic Handling -----------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file implements the Diagnostic-related interfaces.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Basic/Diagnostic.h"
+#include "clang/Basic/SourceLocation.h"
+#include <cassert>
+#include <vector>
+#include <map>
+#include <cstring>
+using namespace clang;
+
+//===----------------------------------------------------------------------===//
+// Builtin Diagnostic information
+//===----------------------------------------------------------------------===//
+
+/// Flag values for diagnostics.
+enum {
+  // Diagnostic classes.
+  NOTE       = 0x01,
+  WARNING    = 0x02,
+  EXTENSION  = 0x03,
+  ERROR      = 0x04,
+  class_mask = 0x07
+};
+
+/// DiagnosticFlags - A set of flags, or'd together, that describe the
+/// diagnostic.
+static unsigned char DiagnosticFlags[] = {
+#define DIAG(ENUM,FLAGS,DESC) FLAGS,
+#include "clang/Basic/DiagnosticKinds.def"
+  0
+};
+
+/// getDiagClass - Return the class field of the diagnostic.
+///
+static unsigned getBuiltinDiagClass(unsigned DiagID) {
+  assert(DiagID < diag::NUM_BUILTIN_DIAGNOSTICS &&
+         "Diagnostic ID out of range!");
+  return DiagnosticFlags[DiagID] & class_mask;
+}
+
+/// DiagnosticText - An english message to print for the diagnostic.  These
+/// should be localized.
+static const char * const DiagnosticText[] = {
+#define DIAG(ENUM,FLAGS,DESC) DESC,
+#include "clang/Basic/DiagnosticKinds.def"
+  0
+};
+
+//===----------------------------------------------------------------------===//
+// Custom Diagnostic information
+//===----------------------------------------------------------------------===//
+
+namespace clang {
+  namespace diag {
+    class CustomDiagInfo {
+      typedef std::pair<Diagnostic::Level, std::string> DiagDesc;
+      std::vector<DiagDesc> DiagInfo;
+      std::map<DiagDesc, unsigned> DiagIDs;
+    public:
+      
+      /// getDescription - Return the description of the specified custom
+      /// diagnostic.
+      const char *getDescription(unsigned DiagID) const {
+        assert(this && DiagID-diag::NUM_BUILTIN_DIAGNOSTICS < DiagInfo.size() &&
+               "Invalid diagnosic ID");
+        return DiagInfo[DiagID-diag::NUM_BUILTIN_DIAGNOSTICS].second.c_str();
+      }
+      
+      /// getLevel - Return the level of the specified custom diagnostic.
+      Diagnostic::Level getLevel(unsigned DiagID) const {
+        assert(this && DiagID-diag::NUM_BUILTIN_DIAGNOSTICS < DiagInfo.size() &&
+               "Invalid diagnosic ID");
+        return DiagInfo[DiagID-diag::NUM_BUILTIN_DIAGNOSTICS].first;
+      }
+      
+      unsigned getOrCreateDiagID(Diagnostic::Level L, const char *Message) {
+        DiagDesc D(L, Message);
+        // Check to see if it already exists.
+        std::map<DiagDesc, unsigned>::iterator I = DiagIDs.lower_bound(D);
+        if (I != DiagIDs.end() && I->first == D)
+          return I->second;
+        
+        // If not, assign a new ID.
+        unsigned ID = DiagInfo.size()+diag::NUM_BUILTIN_DIAGNOSTICS;
+        DiagIDs.insert(std::make_pair(D, ID));
+        DiagInfo.push_back(D);
+        return ID;
+      }
+    };
+    
+  } // end diag namespace 
+} // end clang namespace 
+
+
+//===----------------------------------------------------------------------===//
+// Common Diagnostic implementation
+//===----------------------------------------------------------------------===//
+
+Diagnostic::Diagnostic(DiagnosticClient &client) : Client(client) {
+  WarningsAsErrors = false;
+  WarnOnExtensions = false;
+  ErrorOnExtensions = false;
+  // Clear all mappings, setting them to MAP_DEFAULT.
+  memset(DiagMappings, 0, sizeof(DiagMappings));
+  
+  ErrorOccurred = false;
+  NumDiagnostics = 0;
+  NumErrors = 0;
+  CustomDiagInfo = 0;
+}
+
+Diagnostic::~Diagnostic() {
+  delete CustomDiagInfo;
+}
+
+/// getCustomDiagID - Return an ID for a diagnostic with the specified message
+/// and level.  If this is the first request for this diagnosic, it is
+/// registered and created, otherwise the existing ID is returned.
+unsigned Diagnostic::getCustomDiagID(Level L, const char *Message) {
+  if (CustomDiagInfo == 0) 
+    CustomDiagInfo = new diag::CustomDiagInfo();
+  return CustomDiagInfo->getOrCreateDiagID(L, Message);
+}
+
+
+/// isBuiltinNoteWarningOrExtension - Return true if the unmapped diagnostic
+/// level of the specified diagnostic ID is a Note, Warning, or Extension.
+/// Note that this only works on builtin diagnostics, not custom ones.
+bool Diagnostic::isBuiltinNoteWarningOrExtension(unsigned DiagID) {
+  return DiagID < diag::NUM_BUILTIN_DIAGNOSTICS && 
+         getBuiltinDiagClass(DiagID) < ERROR;
+}
+
+
+/// getDescription - Given a diagnostic ID, return a description of the
+/// issue.
+const char *Diagnostic::getDescription(unsigned DiagID) {
+  if (DiagID < diag::NUM_BUILTIN_DIAGNOSTICS)
+    return DiagnosticText[DiagID];
+  else 
+    return CustomDiagInfo->getDescription(DiagID);
+}
+
+/// getDiagnosticLevel - Based on the way the client configured the Diagnostic
+/// object, classify the specified diagnostic ID into a Level, consumable by
+/// the DiagnosticClient.
+Diagnostic::Level Diagnostic::getDiagnosticLevel(unsigned DiagID) const {
+  // Handle custom diagnostics, which cannot be mapped.
+  if (DiagID >= diag::NUM_BUILTIN_DIAGNOSTICS)
+    return CustomDiagInfo->getLevel(DiagID);
+  
+  unsigned DiagClass = getBuiltinDiagClass(DiagID);
+  
+  // Specific non-error diagnostics may be mapped to various levels from ignored
+  // to error.
+  if (DiagClass < ERROR) {
+    switch (getDiagnosticMapping((diag::kind)DiagID)) {
+    case diag::MAP_DEFAULT: break;
+    case diag::MAP_IGNORE:  return Ignored;
+    case diag::MAP_WARNING: DiagClass = WARNING; break;
+    case diag::MAP_ERROR:   DiagClass = ERROR; break;
+    }
+  }
+  
+  // Map diagnostic classes based on command line argument settings.
+  if (DiagClass == EXTENSION) {
+    if (ErrorOnExtensions)
+      DiagClass = ERROR;
+    else if (WarnOnExtensions)
+      DiagClass = WARNING;
+    else
+      return Ignored;
+  }
+  
+  // If warnings are to be treated as errors, indicate this as such.
+  if (DiagClass == WARNING && WarningsAsErrors)
+    DiagClass = ERROR;
+  
+  switch (DiagClass) {
+  default: assert(0 && "Unknown diagnostic class!");
+  case NOTE:        return Diagnostic::Note;
+  case WARNING:     return Diagnostic::Warning;
+  case ERROR:       return Diagnostic::Error;
+  }
+}
+
+/// Report - Issue the message to the client. If the client wants us to stop
+/// compilation, return true, otherwise return false.  DiagID is a member of
+/// the diag::kind enum.  
+void Diagnostic::Report(FullSourceLoc Pos, unsigned DiagID,
+                        const std::string *Strs, unsigned NumStrs,
+                        const SourceRange *Ranges, unsigned NumRanges) {
+  
+  // Figure out the diagnostic level of this message.
+  Diagnostic::Level DiagLevel = getDiagnosticLevel(DiagID);
+  
+  // If the client doesn't care about this message, don't issue it.
+  if (DiagLevel == Diagnostic::Ignored)
+    return;
+
+  // If this is not an error and we are in a system header, ignore it.  We have
+  // to check on the original class here, because we also want to ignore
+  // extensions and warnings in -Werror and -pedantic-errors modes, which *map*
+  // warnings/extensions to errors.
+  if (DiagID < diag::NUM_BUILTIN_DIAGNOSTICS &&
+      getBuiltinDiagClass(DiagID) != ERROR &&
+      Client.isInSystemHeader(Pos))
+    return;
+  
+  if (DiagLevel >= Diagnostic::Error) {
+    ErrorOccurred = true;
+    ++NumErrors;
+  }
+
+  // Finally, report it.
+  Client.HandleDiagnostic(*this, DiagLevel, Pos, (diag::kind)DiagID,
+                          Strs, NumStrs, Ranges, NumRanges);
+  ++NumDiagnostics;
+}
+
+DiagnosticClient::~DiagnosticClient() {}
diff --git a/lib/Basic/FileManager.cpp b/lib/Basic/FileManager.cpp
new file mode 100644
index 0000000000..cfc08ed084
--- /dev/null
+++ b/lib/Basic/FileManager.cpp
@@ -0,0 +1,275 @@
+///===--- FileManager.cpp - File System Probing and Caching ----------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file implements the FileManager interface.
+//
+//===----------------------------------------------------------------------===//
+//
+// TODO: This should index all interesting directories with dirent calls.
+//  getdirentries ?
+//  opendir/readdir_r/closedir ?
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Basic/FileManager.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/Bitcode/Serialize.h"
+#include "llvm/Bitcode/Deserialize.h"
+#include "llvm/Support/Streams.h"
+#include "llvm/Config/config.h"
+using namespace clang;
+
+// FIXME: Enhance libsystem to support inode and other fields.
+#include <sys/stat.h>
+
+#if defined(_MSC_VER)
+#define S_ISDIR(s) (_S_IFDIR & s)
+#endif
+
+/// NON_EXISTENT_DIR - A special value distinct from null that is used to
+/// represent a dir name that doesn't exist on the disk.
+#define NON_EXISTENT_DIR reinterpret_cast<DirectoryEntry*>((intptr_t)-1)
+
+#ifdef LLVM_ON_WIN32
+
+#define IS_DIR_SEPARATOR_CHAR(x) ((x) == '/' || (x) == '\\')
+
+namespace {
+  static std::string GetFullPath(const char *relPath)
+  {
+    char *absPathStrPtr = _fullpath(NULL, relPath, 0);
+    assert(absPathStrPtr && "_fullpath() returned NULL!");
+
+    std::string absPath(absPathStrPtr);
+
+    free(absPathStrPtr);
+    return absPath;
+  }
+}
+
+class FileManager::UniqueDirContainer {
+  /// UniqueDirs - Cache from full path to existing directories/files.
+  ///
+  llvm::StringMap<DirectoryEntry> UniqueDirs;  
+
+public:
+  DirectoryEntry &getDirectory(const char *Name, struct stat &StatBuf) {
+    std::string FullPath(GetFullPath(Name));
+    return UniqueDirs.GetOrCreateValue(
+                              FullPath.c_str(),
+                              FullPath.c_str() + FullPath.size()
+                                                                ).getValue();
+  }
+  
+  size_t size() { return UniqueDirs.size(); }
+};
+
+class FileManager::UniqueFileContainer {
+  /// UniqueFiles - Cache from full path to existing directories/files.
+  ///
+  llvm::StringMap<FileEntry> UniqueFiles;
+
+public:
+  FileEntry &getFile(const char *Name, struct stat &StatBuf) {
+    std::string FullPath(GetFullPath(Name));
+    return UniqueFiles.GetOrCreateValue(
+                               FullPath.c_str(),
+                               FullPath.c_str() + FullPath.size()
+                                                                 ).getValue();
+  }
+
+  size_t size() { return UniqueFiles.size(); }
+};
+
+#else
+
+#define IS_DIR_SEPARATOR_CHAR(x) ((x) == '/')
+
+class FileManager::UniqueDirContainer {
+  /// UniqueDirs - Cache from ID's to existing directories/files.
+  ///
+  std::map<std::pair<dev_t, ino_t>, DirectoryEntry> UniqueDirs;  
+
+public:
+  DirectoryEntry &getDirectory(const char *Name, struct stat &StatBuf) {
+    return UniqueDirs[std::make_pair(StatBuf.st_dev, StatBuf.st_ino)];
+  }
+
+  size_t size() { return UniqueDirs.size(); }
+};
+
+class FileManager::UniqueFileContainer {
+  /// UniqueFiles - Cache from ID's to existing directories/files.
+  ///
+  std::set<FileEntry> UniqueFiles;
+
+public:
+  FileEntry &getFile(const char *Name, struct stat &StatBuf) {
+    return
+      const_cast<FileEntry&>(
+                    *UniqueFiles.insert(FileEntry(StatBuf.st_dev,
+                                                  StatBuf.st_ino)).first);
+  }
+
+  size_t size() { return UniqueFiles.size(); }
+};
+
+#endif
+
+
+FileManager::FileManager() : UniqueDirs(*new UniqueDirContainer),
+                             UniqueFiles(*new UniqueFileContainer),
+                             DirEntries(64), FileEntries(64), NextFileUID(0)
+{
+  NumDirLookups = NumFileLookups = 0;
+  NumDirCacheMisses = NumFileCacheMisses = 0;
+}
+
+FileManager::~FileManager() {
+  delete &UniqueDirs;
+  delete &UniqueFiles;
+}
+
+
+/// getDirectory - Lookup, cache, and verify the specified directory.  This
+/// returns null if the directory doesn't exist.
+/// 
+const DirectoryEntry *FileManager::getDirectory(const char *NameStart,
+                                                const char *NameEnd) {
+  ++NumDirLookups;
+  llvm::StringMapEntry<DirectoryEntry *> &NamedDirEnt =
+    DirEntries.GetOrCreateValue(NameStart, NameEnd);
+  
+  // See if there is already an entry in the map.
+  if (NamedDirEnt.getValue())
+    return NamedDirEnt.getValue() == NON_EXISTENT_DIR
+              ? 0 : NamedDirEnt.getValue();
+  
+  ++NumDirCacheMisses;
+  
+  // By default, initialize it to invalid.
+  NamedDirEnt.setValue(NON_EXISTENT_DIR);
+  
+  // Get the null-terminated directory name as stored as the key of the
+  // DirEntries map.
+  const char *InterndDirName = NamedDirEnt.getKeyData();
+  
+  // Check to see if the directory exists.
+  struct stat StatBuf;
+  if (stat(InterndDirName, &StatBuf) ||   // Error stat'ing.
+      !S_ISDIR(StatBuf.st_mode))          // Not a directory?
+    return 0;
+
+  // It exists.  See if we have already opened a directory with the same inode.
+  // This occurs when one dir is symlinked to another, for example.    
+  DirectoryEntry &UDE = UniqueDirs.getDirectory(InterndDirName, StatBuf);
+  
+  NamedDirEnt.setValue(&UDE);
+  if (UDE.getName()) // Already have an entry with this inode, return it.
+    return &UDE;
+  
+  // Otherwise, we don't have this directory yet, add it.  We use the string
+  // key from the DirEntries map as the string.
+  UDE.Name  = InterndDirName;
+  return &UDE;
+}
+
+/// NON_EXISTENT_FILE - A special value distinct from null that is used to
+/// represent a filename that doesn't exist on the disk.
+#define NON_EXISTENT_FILE reinterpret_cast<FileEntry*>((intptr_t)-1)
+
+/// getFile - Lookup, cache, and verify the specified file.  This returns null
+/// if the file doesn't exist.
+/// 
+const FileEntry *FileManager::getFile(const char *NameStart,
+                                      const char *NameEnd) {
+  ++NumFileLookups;
+  
+  // See if there is already an entry in the map.
+  llvm::StringMapEntry<FileEntry *> &NamedFileEnt =
+    FileEntries.GetOrCreateValue(NameStart, NameEnd);
+
+  // See if there is already an entry in the map.
+  if (NamedFileEnt.getValue())
+    return NamedFileEnt.getValue() == NON_EXISTENT_FILE
+                 ? 0 : NamedFileEnt.getValue();
+  
+  ++NumFileCacheMisses;
+
+  // By default, initialize it to invalid.
+  NamedFileEnt.setValue(NON_EXISTENT_FILE);
+
+  // Figure out what directory it is in.   If the string contains a / in it,
+  // strip off everything after it.
+  // FIXME: this logic should be in sys::Path.
+  const char *SlashPos = NameEnd-1;
+  while (SlashPos >= NameStart && !IS_DIR_SEPARATOR_CHAR(SlashPos[0]))
+    --SlashPos;
+  
+  const DirectoryEntry *DirInfo;
+  if (SlashPos < NameStart) {
+    // Use the current directory if file has no path component.
+    const char *Name = ".";
+    DirInfo = getDirectory(Name, Name+1);
+  } else if (SlashPos == NameEnd-1)
+    return 0;       // If filename ends with a /, it's a directory.
+  else
+    DirInfo = getDirectory(NameStart, SlashPos);
+  
+  if (DirInfo == 0)  // Directory doesn't exist, file can't exist.
+    return 0;
+  
+  // Get the null-terminated file name as stored as the key of the
+  // FileEntries map.
+  const char *InterndFileName = NamedFileEnt.getKeyData();
+  
+  // FIXME: Use the directory info to prune this, before doing the stat syscall.
+  // FIXME: This will reduce the # syscalls.
+  
+  // Nope, there isn't.  Check to see if the file exists.
+  struct stat StatBuf;
+  //llvm::cerr << "STATING: " << Filename;
+  if (stat(InterndFileName, &StatBuf) ||   // Error stat'ing.
+      S_ISDIR(StatBuf.st_mode)) {           // A directory?
+    // If this file doesn't exist, we leave a null in FileEntries for this path.
+    //llvm::cerr << ": Not existing\n";
+    return 0;
+  }
+  //llvm::cerr << ": exists\n";
+  
+  // It exists.  See if we have already opened a file with the same inode.
+  // This occurs when one dir is symlinked to another, for example.
+  FileEntry &UFE = UniqueFiles.getFile(InterndFileName, StatBuf);
+  
+  NamedFileEnt.setValue(&UFE);
+  if (UFE.getName())  // Already have an entry with this inode, return it.
+    return &UFE;
+
+  // Otherwise, we don't have this directory yet, add it.
+  // FIXME: Change the name to be a char* that points back to the 'FileEntries'
+  // key.
+  UFE.Name    = InterndFileName;
+  UFE.Size    = StatBuf.st_size;
+  UFE.ModTime = StatBuf.st_mtime;
+  UFE.Dir     = DirInfo;
+  UFE.UID     = NextFileUID++;
+  return &UFE;
+}
+
+void FileManager::PrintStats() const {
+  llvm::cerr << "\n*** File Manager Stats:\n";
+  llvm::cerr << UniqueFiles.size() << " files found, "
+             << UniqueDirs.size() << " dirs found.\n";
+  llvm::cerr << NumDirLookups << " dir lookups, "
+             << NumDirCacheMisses << " dir cache misses.\n";
+  llvm::cerr << NumFileLookups << " file lookups, "
+             << NumFileCacheMisses << " file cache misses.\n";
+  
+  //llvm::cerr << PagesMapped << BytesOfPagesMapped << FSLookups;
+}
diff --git a/lib/Basic/IdentifierTable.cpp b/lib/Basic/IdentifierTable.cpp
new file mode 100644
index 0000000000..65e984a0f7
--- /dev/null
+++ b/lib/Basic/IdentifierTable.cpp
@@ -0,0 +1,551 @@
+//===--- IdentifierTable.cpp - Hash table for identifier lookup -----------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the IdentifierInfo, IdentifierVisitor, and
+// IdentifierTable interfaces.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Basic/IdentifierTable.h"
+#include "clang/Basic/LangOptions.h"
+#include "llvm/ADT/FoldingSet.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/Bitcode/Serialize.h"
+#include "llvm/Bitcode/Deserialize.h"
+
+using namespace clang;
+
+//===----------------------------------------------------------------------===//
+// IdentifierInfo Implementation
+//===----------------------------------------------------------------------===//
+
+IdentifierInfo::IdentifierInfo() {
+  TokenID = tok::identifier;
+  ObjCID = tok::objc_not_keyword;
+  BuiltinID = 0;
+  HasMacro = false;
+  IsExtension = false;
+  IsPoisoned = false;
+  IsCPPOperatorKeyword = false;
+  FETokenInfo = 0;
+}
+
+//===----------------------------------------------------------------------===//
+// IdentifierTable Implementation
+//===----------------------------------------------------------------------===//
+
+IdentifierTable::IdentifierTable(const LangOptions &LangOpts)
+  // Start with space for 8K identifiers.
+  : HashTable(8192) {
+
+  // Populate the identifier table with info about keywords for the current
+  // language.
+  AddKeywords(LangOpts);
+}
+
+// This cstor is intended to be used only for serialization.
+IdentifierTable::IdentifierTable() : HashTable(8192) {}
+
+//===----------------------------------------------------------------------===//
+// Language Keyword Implementation
+//===----------------------------------------------------------------------===//
+
+/// AddKeyword - This method is used to associate a token ID with specific
+/// identifiers because they are language keywords.  This causes the lexer to
+/// automatically map matching identifiers to specialized token codes.
+///
+/// The C90/C99/CPP/CPP0x flags are set to 0 if the token should be
+/// enabled in the specified langauge, set to 1 if it is an extension
+/// in the specified language, and set to 2 if disabled in the
+/// specified language.
+static void AddKeyword(const char *Keyword, unsigned KWLen,
+                       tok::TokenKind TokenCode,
+                       int C90, int C99, int CXX, int CXX0x, int BoolSupport,
+                       const LangOptions &LangOpts, IdentifierTable &Table) {
+  int Flags = 0;
+  if (BoolSupport != 0) {
+    Flags = LangOpts.Boolean ? BoolSupport : 2;
+  } else if (LangOpts.CPlusPlus) {
+    Flags = LangOpts.CPlusPlus0x ? CXX0x : CXX;
+  } else if (LangOpts.C99) {
+    Flags = C99;
+  } else {
+    Flags = C90;
+  }
+  
+  // Don't add this keyword if disabled in this language or if an extension
+  // and extensions are disabled.
+  if (Flags + LangOpts.NoExtensions >= 2) return;
+  
+  IdentifierInfo &Info = Table.get(Keyword, Keyword+KWLen);
+  Info.setTokenID(TokenCode);
+  Info.setIsExtensionToken(Flags == 1);
+}
+
+static void AddAlias(const char *Keyword, unsigned KWLen,
+                     tok::TokenKind AliaseeID,
+                     const char *AliaseeKeyword, unsigned AliaseeKWLen,
+                     const LangOptions &LangOpts, IdentifierTable &Table) {
+  IdentifierInfo &AliasInfo = Table.get(Keyword, Keyword+KWLen);
+  IdentifierInfo &AliaseeInfo = Table.get(AliaseeKeyword,
+                                          AliaseeKeyword+AliaseeKWLen);
+  AliasInfo.setTokenID(AliaseeID);
+  AliasInfo.setIsExtensionToken(AliaseeInfo.isExtensionToken());
+}  
+
+/// AddCXXOperatorKeyword - Register a C++ operator keyword alternative
+/// representations.
+static void AddCXXOperatorKeyword(const char *Keyword, unsigned KWLen,
+                                  tok::TokenKind TokenCode,
+                                  IdentifierTable &Table) {
+  IdentifierInfo &Info = Table.get(Keyword, Keyword + KWLen);
+  Info.setTokenID(TokenCode);
+  Info.setIsCPlusPlusOperatorKeyword();
+}
+
+/// AddObjCKeyword - Register an Objective-C @keyword like "class" "selector" or 
+/// "property".
+static void AddObjCKeyword(tok::ObjCKeywordKind ObjCID, 
+                           const char *Name, unsigned NameLen,
+                           IdentifierTable &Table) {
+  Table.get(Name, Name+NameLen).setObjCKeywordID(ObjCID);
+}
+
+/// AddKeywords - Add all keywords to the symbol table.
+///
+void IdentifierTable::AddKeywords(const LangOptions &LangOpts) {
+  enum {
+    C90Shift = 0,
+    EXTC90   = 1 << C90Shift,
+    NOTC90   = 2 << C90Shift,
+    C99Shift = 2,
+    EXTC99   = 1 << C99Shift,
+    NOTC99   = 2 << C99Shift,
+    CPPShift = 4,
+    EXTCPP   = 1 << CPPShift,
+    NOTCPP   = 2 << CPPShift,
+    CPP0xShift = 6,
+    EXTCPP0x   = 1 << CPP0xShift,
+    NOTCPP0x   = 2 << CPP0xShift,
+    BoolShift = 8,
+    BOOLSUPPORT = 1 << BoolShift,
+    Mask     = 3
+  };
+  
+  // Add keywords and tokens for the current language.
+#define KEYWORD(NAME, FLAGS) \
+  AddKeyword(#NAME, strlen(#NAME), tok::kw_ ## NAME,  \
+             ((FLAGS) >> C90Shift) & Mask, \
+             ((FLAGS) >> C99Shift) & Mask, \
+             ((FLAGS) >> CPPShift) & Mask, \
+             ((FLAGS) >> CPP0xShift) & Mask, \
+             ((FLAGS) >> BoolShift) & Mask, LangOpts, *this);
+#define ALIAS(NAME, TOK) \
+  AddAlias(NAME, strlen(NAME), tok::kw_ ## TOK, #TOK, strlen(#TOK),  \
+           LangOpts, *this);
+#define CXX_KEYWORD_OPERATOR(NAME, ALIAS) \
+  if (LangOpts.CXXOperatorNames)          \
+    AddCXXOperatorKeyword(#NAME, strlen(#NAME), tok::ALIAS, *this);
+#define OBJC1_AT_KEYWORD(NAME) \
+  if (LangOpts.ObjC1)          \
+    AddObjCKeyword(tok::objc_##NAME, #NAME, strlen(#NAME), *this);
+#define OBJC2_AT_KEYWORD(NAME) \
+  if (LangOpts.ObjC2)          \
+    AddObjCKeyword(tok::objc_##NAME, #NAME, strlen(#NAME), *this);
+#include "clang/Basic/TokenKinds.def"
+}
+
+tok::PPKeywordKind IdentifierInfo::getPPKeywordID() const {
+  // We use a perfect hash function here involving the length of the keyword,
+  // the first and third character.  For preprocessor ID's there are no
+  // collisions (if there were, the switch below would complain about duplicate
+  // case values).  Note that this depends on 'if' being null terminated.
+  
+#define HASH(LEN, FIRST, THIRD) \
+  (LEN << 5) + (((FIRST-'a') + (THIRD-'a')) & 31)
+#define CASE(LEN, FIRST, THIRD, NAME) \
+  case HASH(LEN, FIRST, THIRD): \
+    return memcmp(Name, #NAME, LEN) ? tok::pp_not_keyword : tok::pp_ ## NAME
+    
+  unsigned Len = getLength();
+  if (Len < 2) return tok::pp_not_keyword;
+  const char *Name = getName();
+  switch (HASH(Len, Name[0], Name[2])) {
+  default: return tok::pp_not_keyword;
+  CASE( 2, 'i', '\0', if);
+  CASE( 4, 'e', 'i', elif);
+  CASE( 4, 'e', 's', else);
+  CASE( 4, 'l', 'n', line);
+  CASE( 4, 's', 'c', sccs);
+  CASE( 5, 'e', 'd', endif);
+  CASE( 5, 'e', 'r', error);
+  CASE( 5, 'i', 'e', ident);
+  CASE( 5, 'i', 'd', ifdef);
+  CASE( 5, 'u', 'd', undef);
+
+  CASE( 6, 'a', 's', assert);
+  CASE( 6, 'd', 'f', define);
+  CASE( 6, 'i', 'n', ifndef);
+  CASE( 6, 'i', 'p', import);
+  CASE( 6, 'p', 'a', pragma);
+
+  CASE( 7, 'd', 'f', defined);
+  CASE( 7, 'i', 'c', include);
+  CASE( 7, 'w', 'r', warning);
+
+  CASE( 8, 'u', 'a', unassert);
+  CASE(12, 'i', 'c', include_next);
+#undef CASE
+#undef HASH
+  }
+}
+
+//===----------------------------------------------------------------------===//
+// Stats Implementation
+//===----------------------------------------------------------------------===//
+
+/// PrintStats - Print statistics about how well the identifier table is doing
+/// at hashing identifiers.
+void IdentifierTable::PrintStats() const {
+  unsigned NumBuckets = HashTable.getNumBuckets();
+  unsigned NumIdentifiers = HashTable.getNumItems();
+  unsigned NumEmptyBuckets = NumBuckets-NumIdentifiers;
+  unsigned AverageIdentifierSize = 0;
+  unsigned MaxIdentifierLength = 0;
+  
+  // TODO: Figure out maximum times an identifier had to probe for -stats.
+  for (llvm::StringMap<IdentifierInfo, llvm::BumpPtrAllocator>::const_iterator
+       I = HashTable.begin(), E = HashTable.end(); I != E; ++I) {
+    unsigned IdLen = I->getKeyLength();
+    AverageIdentifierSize += IdLen;
+    if (MaxIdentifierLength < IdLen)
+      MaxIdentifierLength = IdLen;
+  }
+  
+  fprintf(stderr, "\n*** Identifier Table Stats:\n");
+  fprintf(stderr, "# Identifiers:   %d\n", NumIdentifiers);
+  fprintf(stderr, "# Empty Buckets: %d\n", NumEmptyBuckets);
+  fprintf(stderr, "Hash density (#identifiers per bucket): %f\n",
+          NumIdentifiers/(double)NumBuckets);
+  fprintf(stderr, "Ave identifier length: %f\n",
+          (AverageIdentifierSize/(double)NumIdentifiers));
+  fprintf(stderr, "Max identifier length: %d\n", MaxIdentifierLength);
+  
+  // Compute statistics about the memory allocated for identifiers.
+  HashTable.getAllocator().PrintStats();
+}
+
+//===----------------------------------------------------------------------===//
+// SelectorTable Implementation
+//===----------------------------------------------------------------------===//
+
+unsigned llvm::DenseMapInfo<clang::Selector>::getHashValue(clang::Selector S) {
+  return DenseMapInfo<void*>::getHashValue(S.getAsOpaquePtr());
+}
+
+
+/// MultiKeywordSelector - One of these variable length records is kept for each
+/// selector containing more than one keyword. We use a folding set
+/// to unique aggregate names (keyword selectors in ObjC parlance). Access to 
+/// this class is provided strictly through Selector.
+namespace clang {
+class MultiKeywordSelector : public llvm::FoldingSetNode {
+  friend SelectorTable* SelectorTable::CreateAndRegister(llvm::Deserializer&);
+  MultiKeywordSelector(unsigned nKeys) : NumArgs(nKeys) {}
+public:  
+  unsigned NumArgs;
+
+  // Constructor for keyword selectors.
+  MultiKeywordSelector(unsigned nKeys, IdentifierInfo **IIV) {
+    assert((nKeys > 1) && "not a multi-keyword selector");
+    NumArgs = nKeys;
+    
+    // Fill in the trailing keyword array.
+    IdentifierInfo **KeyInfo = reinterpret_cast<IdentifierInfo **>(this+1);
+    for (unsigned i = 0; i != nKeys; ++i)
+      KeyInfo[i] = IIV[i];
+  }  
+  
+  // getName - Derive the full selector name and return it.
+  std::string getName() const;
+    
+  unsigned getNumArgs() const { return NumArgs; }
+  
+  typedef IdentifierInfo *const *keyword_iterator;
+  keyword_iterator keyword_begin() const {
+    return reinterpret_cast<keyword_iterator>(this+1);
+  }
+  keyword_iterator keyword_end() const { 
+    return keyword_begin()+NumArgs; 
+  }
+  IdentifierInfo *getIdentifierInfoForSlot(unsigned i) const {
+    assert(i < NumArgs && "getIdentifierInfoForSlot(): illegal index");
+    return keyword_begin()[i];
+  }
+  static void Profile(llvm::FoldingSetNodeID &ID, 
+                      keyword_iterator ArgTys, unsigned NumArgs) {
+    ID.AddInteger(NumArgs);
+    for (unsigned i = 0; i != NumArgs; ++i)
+      ID.AddPointer(ArgTys[i]);
+  }
+  void Profile(llvm::FoldingSetNodeID &ID) {
+    Profile(ID, keyword_begin(), NumArgs);
+  }
+};
+} // end namespace clang.
+
+unsigned Selector::getNumArgs() const {
+  unsigned IIF = getIdentifierInfoFlag();
+  if (IIF == ZeroArg)
+    return 0;
+  if (IIF == OneArg)
+    return 1;
+  // We point to a MultiKeywordSelector (pointer doesn't contain any flags).
+  MultiKeywordSelector *SI = reinterpret_cast<MultiKeywordSelector *>(InfoPtr);
+  return SI->getNumArgs(); 
+}
+
+IdentifierInfo *Selector::getIdentifierInfoForSlot(unsigned argIndex) const {
+  if (IdentifierInfo *II = getAsIdentifierInfo()) {
+    assert(argIndex == 0 && "illegal keyword index");
+    return II;
+  }
+  // We point to a MultiKeywordSelector (pointer doesn't contain any flags).
+  MultiKeywordSelector *SI = reinterpret_cast<MultiKeywordSelector *>(InfoPtr);
+  return SI->getIdentifierInfoForSlot(argIndex);
+}
+
+std::string MultiKeywordSelector::getName() const {
+  std::string Result;
+  unsigned Length = 0;
+  for (keyword_iterator I = keyword_begin(), E = keyword_end(); I != E; ++I) {
+    if (*I)
+      Length += (*I)->getLength();
+    ++Length;  // :
+  }
+  
+  Result.reserve(Length);
+  
+  for (keyword_iterator I = keyword_begin(), E = keyword_end(); I != E; ++I) {
+    if (*I)
+      Result.insert(Result.end(), (*I)->getName(),
+                    (*I)->getName()+(*I)->getLength());
+    Result.push_back(':');
+  }
+  
+  return Result;
+}
+
+std::string Selector::getName() const {
+  if (IdentifierInfo *II = getAsIdentifierInfo()) {
+    if (getNumArgs() == 0)
+      return II->getName();
+    
+    std::string Res = II->getName();
+    Res += ":";
+    return Res;
+  }
+  
+  // We have a multiple keyword selector (no embedded flags).
+  return reinterpret_cast<MultiKeywordSelector *>(InfoPtr)->getName();
+}
+
+
+Selector SelectorTable::getSelector(unsigned nKeys, IdentifierInfo **IIV) {
+  if (nKeys < 2)
+    return Selector(IIV[0], nKeys);
+  
+  llvm::FoldingSet<MultiKeywordSelector> *SelTab;
+  
+  SelTab = static_cast<llvm::FoldingSet<MultiKeywordSelector> *>(Impl);
+    
+  // Unique selector, to guarantee there is one per name.
+  llvm::FoldingSetNodeID ID;
+  MultiKeywordSelector::Profile(ID, IIV, nKeys);
+
+  void *InsertPos = 0;
+  if (MultiKeywordSelector *SI = SelTab->FindNodeOrInsertPos(ID, InsertPos))
+    return Selector(SI);
+  
+  // MultiKeywordSelector objects are not allocated with new because they have a
+  // variable size array (for parameter types) at the end of them.
+  MultiKeywordSelector *SI = 
+    (MultiKeywordSelector*)malloc(sizeof(MultiKeywordSelector) + 
+                                  nKeys*sizeof(IdentifierInfo *));
+  new (SI) MultiKeywordSelector(nKeys, IIV);
+  SelTab->InsertNode(SI, InsertPos);
+  return Selector(SI);
+}
+
+SelectorTable::SelectorTable() {
+  Impl = new llvm::FoldingSet<MultiKeywordSelector>;
+}
+
+SelectorTable::~SelectorTable() {
+  delete static_cast<llvm::FoldingSet<MultiKeywordSelector> *>(Impl);
+}
+
+//===----------------------------------------------------------------------===//
+// Serialization for IdentifierInfo and IdentifierTable.
+//===----------------------------------------------------------------------===//
+
+void IdentifierInfo::Emit(llvm::Serializer& S) const {
+  S.EmitInt(getTokenID());
+  S.EmitInt(getBuiltinID());
+  S.EmitInt(getObjCKeywordID());  
+  S.EmitBool(hasMacroDefinition());
+  S.EmitBool(isExtensionToken());
+  S.EmitBool(isPoisoned());
+  S.EmitBool(isCPlusPlusOperatorK