Make a major restructuring of the clang tree: introduce a top-level

lib dir and move all the libraries into it.  This follows the main
llvm tree, and allows the libraries to be built in parallel.  The
top level now enforces that all the libs are built before Driver,
but we don't care what order the libs are built in.  This speeds
up parallel builds, particularly incremental ones.


git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@48402 91177308-0d34-0410-b5e6-96231b3b80d8

16 files changed, 7673 insertions, 0 deletions

diff --git a/lib/Lex/HeaderMap.cpp b/lib/Lex/HeaderMap.cpp
new file mode 100644
index 0000000000..282e742b4c
--- /dev/null
+++ b/lib/Lex/HeaderMap.cpp
@@ -0,0 +1,242 @@
+//===--- HeaderMap.cpp - A file that acts like dir of symlinks ------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the HeaderMap interface.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Lex/HeaderMap.h"
+#include "clang/Basic/FileManager.h"
+#include "llvm/ADT/OwningPtr.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/Support/DataTypes.h"
+#include "llvm/Support/MathExtras.h"
+#include "llvm/Support/MemoryBuffer.h"
+using namespace clang;
+
+//===----------------------------------------------------------------------===//
+// Data Structures and Manifest Constants
+//===----------------------------------------------------------------------===//
+
+enum {
+  HMAP_HeaderMagicNumber = ('h' << 24) | ('m' << 16) | ('a' << 8) | 'p',
+  HMAP_HeaderVersion = 1,
+  
+  HMAP_EmptyBucketKey = 0 
+};
+
+namespace clang {
+struct HMapBucket {
+  uint32_t Key;          // Offset (into strings) of key.
+
+  uint32_t Prefix;     // Offset (into strings) of value prefix.
+  uint32_t Suffix;     // Offset (into strings) of value suffix.
+};
+
+struct HMapHeader {
+  uint32_t Magic;           // Magic word, also indicates byte order.
+  uint16_t Version;         // Version number -- currently 1.
+  uint16_t Reserved;        // Reserved for future use - zero for now.
+  uint32_t StringsOffset;   // Offset to start of string pool.
+  uint32_t NumEntries;      // Number of entries in the string table.
+  uint32_t NumBuckets;      // Number of buckets (always a power of 2).
+  uint32_t MaxValueLength;  // Length of longest result path (excluding nul).
+  // An array of 'NumBuckets' HMapBucket objects follows this header.
+  // Strings follow the buckets, at StringsOffset.
+};
+} // end namespace clang.
+
+/// HashHMapKey - This is the 'well known' hash function required by the file
+/// format, used to look up keys in the hash table.  The hash table uses simple
+/// linear probing based on this function.
+static inline unsigned HashHMapKey(const char *S, const char *End) {
+  unsigned Result = 0;
+  
+  for (; S != End; S++)
+    Result += tolower(*S) * 13;
+  return Result;
+}
+
+
+
+//===----------------------------------------------------------------------===//
+// Verification and Construction
+//===----------------------------------------------------------------------===//
+
+/// HeaderMap::Create - This attempts to load the specified file as a header
+/// map.  If it doesn't look like a HeaderMap, it gives up and returns null.
+/// If it looks like a HeaderMap but is obviously corrupted, it puts a reason
+/// into the string error argument and returns null.
+const HeaderMap *HeaderMap::Create(const FileEntry *FE) {
+  // If the file is too small to be a header map, ignore it.
+  unsigned FileSize = FE->getSize();
+  if (FileSize <= sizeof(HMapHeader)) return 0;
+  
+  llvm::OwningPtr<const llvm::MemoryBuffer> FileBuffer( 
+    llvm::MemoryBuffer::getFile(FE->getName(), strlen(FE->getName()), 0,
+                                FE->getSize()));
+  if (FileBuffer == 0) return 0;  // Unreadable file?
+  const char *FileStart = FileBuffer->getBufferStart();
+
+  // We know the file is at least as big as the header, check it now.
+  const HMapHeader *Header = reinterpret_cast<const HMapHeader*>(FileStart);
+  
+  // Sniff it to see if it's a headermap by checking the magic number and
+  // version.
+  bool NeedsByteSwap;
+  if (Header->Magic == HMAP_HeaderMagicNumber && 
+      Header->Version == HMAP_HeaderVersion)
+    NeedsByteSwap = false;
+  else if (Header->Magic == llvm::ByteSwap_32(HMAP_HeaderMagicNumber) &&
+           Header->Version == llvm::ByteSwap_16(HMAP_HeaderVersion))
+    NeedsByteSwap = true;  // Mixed endianness headermap.
+  else 
+    return 0;  // Not a header map.
+  
+  if (Header->Reserved != 0) return 0;
+
+  // Okay, everything looks good, create the header map.
+  return new HeaderMap(FileBuffer.take(), NeedsByteSwap);
+}
+
+HeaderMap::~HeaderMap() {
+  delete FileBuffer;
+}
+
+//===----------------------------------------------------------------------===//
+//  Utility Methods
+//===----------------------------------------------------------------------===//
+
+
+/// getFileName - Return the filename of the headermap.
+const char *HeaderMap::getFileName() const {
+  return FileBuffer->getBufferIdentifier();
+}
+
+unsigned HeaderMap::getEndianAdjustedWord(unsigned X) const {
+  if (!NeedsBSwap) return X;
+  return llvm::ByteSwap_32(X);
+}
+
+/// getHeader - Return a reference to the file header, in unbyte-swapped form.
+/// This method cannot fail.
+const HMapHeader &HeaderMap::getHeader() const {
+  // We know the file is at least as big as the header.  Return it.
+  return *reinterpret_cast<const HMapHeader*>(FileBuffer->getBufferStart());
+}
+
+/// getBucket - Return the specified hash table bucket from the header map,
+/// bswap'ing its fields as appropriate.  If the bucket number is not valid,
+/// this return a bucket with an empty key (0).
+HMapBucket HeaderMap::getBucket(unsigned BucketNo) const {
+  HMapBucket Result;
+  Result.Key = HMAP_EmptyBucketKey;
+  
+  const HMapBucket *BucketArray = 
+    reinterpret_cast<const HMapBucket*>(FileBuffer->getBufferStart() +
+                                        sizeof(HMapHeader));
+  
+  const HMapBucket *BucketPtr = BucketArray+BucketNo;
+  if ((char*)(BucketPtr+1) > FileBuffer->getBufferEnd())
+    return Result;  // Invalid buffer, corrupt hmap.
+
+  // Otherwise, the bucket is valid.  Load the values, bswapping as needed.
+  Result.Key    = getEndianAdjustedWord(BucketPtr->Key);
+  Result.Prefix = getEndianAdjustedWord(BucketPtr->Prefix);
+  Result.Suffix = getEndianAdjustedWord(BucketPtr->Suffix);
+  return Result;
+}
+
+/// getString - Look up the specified string in the string table.  If the string
+/// index is not valid, it returns an empty string.
+const char *HeaderMap::getString(unsigned StrTabIdx) const {
+  // Add the start of the string table to the idx.
+  StrTabIdx += getEndianAdjustedWord(getHeader().StringsOffset);
+  
+  // Check for invalid index.
+  if (StrTabIdx >= FileBuffer->getBufferSize())
+    return 0;
+  
+  // Otherwise, we have a valid pointer into the file.  Just return it.  We know
+  // that the "string" can not overrun the end of the file, because the buffer
+  // is nul terminated by virtue of being a MemoryBuffer.
+  return FileBuffer->getBufferStart()+StrTabIdx;
+}
+
+/// StringsEqualWithoutCase - Compare the specified two strings for case-
+/// insensitive equality, returning true if they are equal.  Both strings are
+/// known to have the same length.
+static bool StringsEqualWithoutCase(const char *S1, const char *S2,
+                                    unsigned Len) {
+  for (; Len; ++S1, ++S2, --Len)
+    if (tolower(*S1) != tolower(*S2))
+      return false;
+  return true;
+}
+
+//===----------------------------------------------------------------------===//
+// The Main Drivers
+//===----------------------------------------------------------------------===//
+
+/// dump - Print the contents of this headermap to stderr.
+void HeaderMap::dump() const {
+  const HMapHeader &Hdr = getHeader();
+  unsigned NumBuckets = getEndianAdjustedWord(Hdr.NumBuckets);
+  
+  fprintf(stderr, "Header Map %s:\n  %d buckets, %d entries\n", 
+          getFileName(), NumBuckets,
+          getEndianAdjustedWord(Hdr.NumEntries));
+  
+  for (unsigned i = 0; i != NumBuckets; ++i) {
+    HMapBucket B = getBucket(i);
+    if (B.Key == HMAP_EmptyBucketKey) continue;
+    
+    const char *Key    = getString(B.Key);
+    const char *Prefix = getString(B.Prefix);
+    const char *Suffix = getString(B.Suffix);
+    fprintf(stderr, "  %d. %s -> '%s' '%s'\n", i, Key, Prefix, Suffix);
+  }
+}
+
+/// LookupFile - Check to see if the specified relative filename is located in
+/// this HeaderMap.  If so, open it and return its FileEntry.
+const FileEntry *HeaderMap::LookupFile(const char *FilenameStart,
+                                       const char *FilenameEnd,
+                                       FileManager &FM) const {
+  const HMapHeader &Hdr = getHeader();
+  unsigned NumBuckets = getEndianAdjustedWord(Hdr.NumBuckets);
+
+  // If the number of buckets is not a power of two, the headermap is corrupt.
+  // Don't probe infinitely.
+  if (NumBuckets & (NumBuckets-1))
+    return 0;
+  
+  // Linearly probe the hash table.
+  for (unsigned Bucket = HashHMapKey(FilenameStart, FilenameEnd);; ++Bucket) {
+    HMapBucket B = getBucket(Bucket & (NumBuckets-1));
+    if (B.Key == HMAP_EmptyBucketKey) return 0; // Hash miss.
+    
+    // See if the key matches.  If not, probe on.
+    const char *Key = getString(B.Key);
+    unsigned BucketKeyLen = strlen(Key);
+    if (BucketKeyLen != unsigned(FilenameEnd-FilenameStart))
+      continue;
+    
+    // See if the actual strings equal.
+    if (!StringsEqualWithoutCase(FilenameStart, Key, BucketKeyLen))
+      continue;
+    
+    // If so, we have a match in the hash table.  Construct the destination
+    // path.
+    llvm::SmallString<1024> DestPath;
+    DestPath += getString(B.Prefix);
+    DestPath += getString(B.Suffix);
+    return FM.getFile(DestPath.begin(), DestPath.end());
+  }
+}
diff --git a/lib/Lex/HeaderSearch.cpp b/lib/Lex/HeaderSearch.cpp
new file mode 100644
index 0000000000..44ae35c8b7
--- /dev/null
+++ b/lib/Lex/HeaderSearch.cpp
@@ -0,0 +1,425 @@
+//===--- HeaderSearch.cpp - Resolve Header File Locations ---===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file implements the DirectoryLookup and HeaderSearch interfaces.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Lex/HeaderSearch.h"
+#include "clang/Lex/HeaderMap.h"
+#include "clang/Basic/FileManager.h"
+#include "clang/Basic/IdentifierTable.h"
+#include "llvm/System/Path.h"
+#include "llvm/ADT/SmallString.h"
+using namespace clang;
+
+HeaderSearch::HeaderSearch(FileManager &FM) : FileMgr(FM), FrameworkMap(64) {
+  SystemDirIdx = 0;
+  NoCurDirSearch = false;
+  
+  NumIncluded = 0;
+  NumMultiIncludeFileOptzn = 0;
+  NumFrameworkLookups = NumSubFrameworkLookups = 0;
+}
+
+HeaderSearch::~HeaderSearch() {
+  // Delete headermaps.
+  for (unsigned i = 0, e = HeaderMaps.size(); i != e; ++i)
+    delete HeaderMaps[i].second;
+}
+                           
+void HeaderSearch::PrintStats() {
+  fprintf(stderr, "\n*** HeaderSearch Stats:\n");
+  fprintf(stderr, "%d files tracked.\n", (int)FileInfo.size());
+  unsigned NumOnceOnlyFiles = 0, MaxNumIncludes = 0, NumSingleIncludedFiles = 0;
+  for (unsigned i = 0, e = FileInfo.size(); i != e; ++i) {
+    NumOnceOnlyFiles += FileInfo[i].isImport;
+    if (MaxNumIncludes < FileInfo[i].NumIncludes)
+      MaxNumIncludes = FileInfo[i].NumIncludes;
+    NumSingleIncludedFiles += FileInfo[i].NumIncludes == 1;
+  }
+  fprintf(stderr, "  %d #import/#pragma once files.\n", NumOnceOnlyFiles);
+  fprintf(stderr, "  %d included exactly once.\n", NumSingleIncludedFiles);
+  fprintf(stderr, "  %d max times a file is included.\n", MaxNumIncludes);
+  
+  fprintf(stderr, "  %d #include/#include_next/#import.\n", NumIncluded);
+  fprintf(stderr, "    %d #includes skipped due to"
+          " the multi-include optimization.\n", NumMultiIncludeFileOptzn);
+  
+  fprintf(stderr, "%d framework lookups.\n", NumFrameworkLookups);
+  fprintf(stderr, "%d subframework lookups.\n", NumSubFrameworkLookups);
+}
+
+/// CreateHeaderMap - This method returns a HeaderMap for the specified
+/// FileEntry, uniquing them through the the 'HeaderMaps' datastructure.
+const HeaderMap *HeaderSearch::CreateHeaderMap(const FileEntry *FE) {
+  // We expect the number of headermaps to be small, and almost always empty.
+  // If it ever grows, use of a linear search should be re-evaluated.
+  if (!HeaderMaps.empty()) {
+    for (unsigned i = 0, e = HeaderMaps.size(); i != e; ++i)
+      // Pointer equality comparison of FileEntries works because they are
+      // already uniqued by inode.
+      if (HeaderMaps[i].first == FE) 
+        return HeaderMaps[i].second;
+  }
+  
+  if (const HeaderMap *HM = HeaderMap::Create(FE)) {
+    HeaderMaps.push_back(std::make_pair(FE, HM));
+    return HM;
+  }
+    
+  return 0;
+}
+
+//===----------------------------------------------------------------------===//
+// File lookup within a DirectoryLookup scope
+//===----------------------------------------------------------------------===//
+
+/// getName - Return the directory or filename corresponding to this lookup
+/// object.
+const char *DirectoryLookup::getName() const {
+  if (isNormalDir())
+    return getDir()->getName();
+  if (isFramework())
+    return getFrameworkDir()->getName();
+  assert(isHeaderMap() && "Unknown DirectoryLookup");
+  return getHeaderMap()->getFileName();
+}
+
+
+/// LookupFile - Lookup the specified file in this search path, returning it
+/// if it exists or returning null if not.
+const FileEntry *DirectoryLookup::LookupFile(const char *FilenameStart,
+                                             const char *FilenameEnd,
+                                             HeaderSearch &HS) const {
+  llvm::SmallString<1024> TmpDir;
+  if (isNormalDir()) {
+    // Concatenate the requested file onto the directory.
+    // FIXME: Portability.  Filename concatenation should be in sys::Path.
+    TmpDir += getDir()->getName();
+    TmpDir.push_back('/');
+    TmpDir.append(FilenameStart, FilenameEnd);
+    return HS.getFileMgr().getFile(TmpDir.begin(), TmpDir.end());
+  }
+  
+  if (isFramework())
+    return DoFrameworkLookup(FilenameStart, FilenameEnd, HS);
+  
+  assert(isHeaderMap() && "Unknown directory lookup");
+  return getHeaderMap()->LookupFile(FilenameStart, FilenameEnd,HS.getFileMgr());
+}
+
+
+/// DoFrameworkLookup - Do a lookup of the specified file in the current
+/// DirectoryLookup, which is a framework directory.
+const FileEntry *DirectoryLookup::DoFrameworkLookup(const char *FilenameStart,
+                                                    const char *FilenameEnd,
+                                                    HeaderSearch &HS) const {
+  FileManager &FileMgr = HS.getFileMgr();
+  
+  // Framework names must have a '/' in the filename.
+  const char *SlashPos = std::find(FilenameStart, FilenameEnd, '/');
+  if (SlashPos == FilenameEnd) return 0;
+  
+  // Find out if this is the home for the specified framework, by checking
+  // HeaderSearch.  Possible answer are yes/no and unknown.
+  const DirectoryEntry *&FrameworkDirCache = 
+    HS.LookupFrameworkCache(FilenameStart, SlashPos);
+  
+  // If it is known and in some other directory, fail.
+  if (FrameworkDirCache && FrameworkDirCache != getFrameworkDir())
+    return 0;
+  
+  // Otherwise, construct the path to this framework dir.
+  
+  // FrameworkName = "/System/Library/Frameworks/"
+  llvm::SmallString<1024> FrameworkName;
+  FrameworkName += getFrameworkDir()->getName();
+  if (FrameworkName.empty() || FrameworkName.back() != '/')
+    FrameworkName.push_back('/');
+  
+  // FrameworkName = "/System/Library/Frameworks/Cocoa"
+  FrameworkName.append(FilenameStart, SlashPos);
+  
+  // FrameworkName = "/System/Library/Frameworks/Cocoa.framework/"
+  FrameworkName += ".framework/";
+  
+  // If the cache entry is still unresolved, query to see if the cache entry is
+  // still unresolved.  If so, check its existence now.
+  if (FrameworkDirCache == 0) {
+    HS.IncrementFrameworkLookupCount();
+    
+    // If the framework dir doesn't exist, we fail.
+    // FIXME: It's probably more efficient to query this with FileMgr.getDir.
+    if (!llvm::sys::Path(std::string(FrameworkName.begin(), 
+                                     FrameworkName.end())).exists())
+      return 0;
+    
+    // Otherwise, if it does, remember that this is the right direntry for this
+    // framework.
+    FrameworkDirCache = getFrameworkDir();
+  }
+  
+  // Check "/System/Library/Frameworks/Cocoa.framework/Headers/file.h"
+  unsigned OrigSize = FrameworkName.size();
+  
+  FrameworkName += "Headers/";
+  FrameworkName.append(SlashPos+1, FilenameEnd);
+  if (const FileEntry *FE = FileMgr.getFile(FrameworkName.begin(),
+                                            FrameworkName.end())) {
+    return FE;
+  }
+  
+  // Check "/System/Library/Frameworks/Cocoa.framework/PrivateHeaders/file.h"
+  const char *Private = "Private";
+  FrameworkName.insert(FrameworkName.begin()+OrigSize, Private, 
+                       Private+strlen(Private));
+  return FileMgr.getFile(FrameworkName.begin(), FrameworkName.end());
+}
+
+
+//===----------------------------------------------------------------------===//
+// Header File Location.
+//===----------------------------------------------------------------------===//
+
+
+/// LookupFile - Given a "foo" or <foo> reference, look up the indicated file,
+/// return null on failure.  isAngled indicates whether the file reference is
+/// for system #include's or not (i.e. using <> instead of "").  CurFileEnt, if
+/// non-null, indicates where the #including file is, in case a relative search
+/// is needed.
+const FileEntry *HeaderSearch::LookupFile(const char *FilenameStart,
+                                          const char *FilenameEnd, 
+                                          bool isAngled,
+                                          const DirectoryLookup *FromDir,
+                                          const DirectoryLookup *&CurDir,
+                                          const FileEntry *CurFileEnt) {
+  // If 'Filename' is absolute, check to see if it exists and no searching.
+  // FIXME: Portability.  This should be a sys::Path interface, this doesn't
+  // handle things like C:\foo.txt right, nor win32 \\network\device\blah.
+  if (FilenameStart[0] == '/') {
+    CurDir = 0;
+
+    // If this was an #include_next "/absolute/file", fail.
+    if (FromDir) return 0;
+    
+    // Otherwise, just return the file.
+    return FileMgr.getFile(FilenameStart, FilenameEnd);
+  }
+  
+  // Step #0, unless disabled, check to see if the file is in the #includer's
+  // directory.  This has to be based on CurFileEnt, not CurDir, because
+  // CurFileEnt could be a #include of a subdirectory (#include "foo/bar.h") and
+  // a subsequent include of "baz.h" should resolve to "whatever/foo/baz.h".
+  // This search is not done for <> headers.
+  if (CurFileEnt && !isAngled && !NoCurDirSearch) {
+    llvm::SmallString<1024> TmpDir;
+    // Concatenate the requested file onto the directory.
+    // FIXME: Portability.  Filename concatenation should be in sys::Path.
+    TmpDir += CurFileEnt->getDir()->getName();
+    TmpDir.push_back('/');
+    TmpDir.append(FilenameStart, FilenameEnd);
+    if (const FileEntry *FE = FileMgr.getFile(TmpDir.begin(), TmpDir.end())) {
+      // Leave CurDir unset.
+      // This file is a system header or C++ unfriendly if the old file is.
+      //
+      // Note that the temporary 'DirInfo' is required here, as either call to
+      // getFileInfo could resize the vector and we don't want to rely on order
+      // of evaluation.
+      unsigned DirInfo = getFileInfo(CurFileEnt).DirInfo;
+      getFileInfo(FE).DirInfo = DirInfo;
+      return FE;
+    }
+  }
+  
+  CurDir = 0;
+
+  // If this is a system #include, ignore the user #include locs.
+  unsigned i = isAngled ? SystemDirIdx : 0;
+  
+  // If this is a #include_next request, start searching after the directory the
+  // file was found in.
+  if (FromDir)
+    i = FromDir-&SearchDirs[0];
+  
+  // Cache all of the lookups performed by this method.  Many headers are
+  // multiply included, and the "pragma once" optimization prevents them from
+  // being relex/pp'd, but they would still have to search through a
+  // (potentially huge) series of SearchDirs to find it.
+  std::pair<unsigned, unsigned> &CacheLookup =
+    LookupFileCache.GetOrCreateValue(FilenameStart, FilenameEnd).getValue();
+
+  // If the entry has been previously looked up, the first value will be
+  // non-zero.  If the value is equal to i (the start point of our search), then
+  // this is a matching hit.
+  if (CacheLookup.first == i+1) {
+    // Skip querying potentially lots of directories for this lookup.
+    i = CacheLookup.second;
+  } else {
+    // Otherwise, this is the first query, or the previous query didn't match
+    // our search start.  We will fill in our found location below, so prime the
+    // start point value.
+    CacheLookup.first = i+1;
+  }
+  
+  // Check each directory in sequence to see if it contains this file.
+  for (; i != SearchDirs.size(); ++i) {
+    const FileEntry *FE = 
+      SearchDirs[i].LookupFile(FilenameStart, FilenameEnd, *this);
+    if (!FE) continue;
+    
+    CurDir = &SearchDirs[i];
+    
+    // This file is a system header or C++ unfriendly if the dir is.
+    getFileInfo(FE).DirInfo = CurDir->getDirCharacteristic();
+    
+    // Remember this location for the next lookup we do.
+    CacheLookup.second = i;
+    return FE;
+  }
+  
+  // Otherwise, didn't find it. Remember we didn't find this.
+  CacheLookup.second = SearchDirs.size();
+  return 0;
+}
+
+/// LookupSubframeworkHeader - Look up a subframework for the specified
+/// #include file.  For example, if #include'ing <HIToolbox/HIToolbox.h> from
+/// within ".../Carbon.framework/Headers/Carbon.h", check to see if HIToolbox
+/// is a subframework within Carbon.framework.  If so, return the FileEntry
+/// for the designated file, otherwise return null.
+const FileEntry *HeaderSearch::
+LookupSubframeworkHeader(const char *FilenameStart,
+                         const char *FilenameEnd,
+                         const FileEntry *ContextFileEnt) {
+  assert(ContextFileEnt && "No context file?");
+  
+  // Framework names must have a '/' in the filename.  Find it.
+  const char *SlashPos = std::find(FilenameStart, FilenameEnd, '/');
+  if (SlashPos == FilenameEnd) return 0;
+  
+  // Look up the base framework name of the ContextFileEnt.
+  const char *ContextName = ContextFileEnt->getName();
+    
+  // If the context info wasn't a framework, couldn't be a subframework.
+  const char *FrameworkPos = strstr(ContextName, ".framework/");
+  if (FrameworkPos == 0)
+    return 0;
+  
+  llvm::SmallString<1024> FrameworkName(ContextName, 
+                                        FrameworkPos+strlen(".framework/"));
+
+  // Append Frameworks/HIToolbox.framework/
+  FrameworkName += "Frameworks/";
+  FrameworkName.append(FilenameStart, SlashPos);
+  FrameworkName += ".framework/";
+
+  llvm::StringMapEntry<const DirectoryEntry *> &CacheLookup =
+    FrameworkMap.GetOrCreateValue(FilenameStart, SlashPos);
+  
+  // Some other location?
+  if (CacheLookup.getValue() &&
+      CacheLookup.getKeyLength() == FrameworkName.size() &&
+      memcmp(CacheLookup.getKeyData(), &FrameworkName[0],
+             CacheLookup.getKeyLength()) != 0)
+    return 0;
+  
+  // Cache subframework.
+  if (CacheLookup.getValue() == 0) {
+    ++NumSubFrameworkLookups;
+    
+    // If the framework dir doesn't exist, we fail.
+    const DirectoryEntry *Dir = FileMgr.getDirectory(FrameworkName.begin(),
+                                                     FrameworkName.end());
+    if (Dir == 0) return 0;
+    
+    // Otherwise, if it does, remember that this is the right direntry for this
+    // framework.
+    CacheLookup.setValue(Dir);
+  }
+  
+  const FileEntry *FE = 0;
+
+  // Check ".../Frameworks/HIToolbox.framework/Headers/HIToolbox.h"
+  llvm::SmallString<1024> HeadersFilename(FrameworkName);
+  HeadersFilename += "Headers/";
+  HeadersFilename.append(SlashPos+1, FilenameEnd);
+  if (!(FE = FileMgr.getFile(HeadersFilename.begin(),
+                             HeadersFilename.end()))) {
+    
+    // Check ".../Frameworks/HIToolbox.framework/PrivateHeaders/HIToolbox.h"
+    HeadersFilename = FrameworkName;
+    HeadersFilename += "PrivateHeaders/";
+    HeadersFilename.append(SlashPos+1, FilenameEnd);
+    if (!(FE = FileMgr.getFile(HeadersFilename.begin(), HeadersFilename.end())))
+      return 0;
+  }
+  
+  // This file is a system header or C++ unfriendly if the old file is.
+  //
+  // Note that the temporary 'DirInfo' is required here, as either call to
+  // getFileInfo could resize the vector and we don't want to rely on order
+  // of evaluation.
+  unsigned DirInfo = getFileInfo(ContextFileEnt).DirInfo;
+  getFileInfo(FE).DirInfo = DirInfo;
+  return FE;
+}
+
+//===----------------------------------------------------------------------===//
+// File Info Management.
+//===----------------------------------------------------------------------===//
+
+
+/// getFileInfo - Return the PerFileInfo structure for the specified
+/// FileEntry.
+HeaderSearch::PerFileInfo &HeaderSearch::getFileInfo(const FileEntry *FE) {
+  if (FE->getUID() >= FileInfo.size())
+    FileInfo.resize(FE->getUID()+1);
+  return FileInfo[FE->getUID()];
+}  
+
+/// ShouldEnterIncludeFile - Mark the specified file as a target of of a
+/// #include, #include_next, or #import directive.  Return false if #including
+/// the file will have no effect or true if we should include it.
+bool HeaderSearch::ShouldEnterIncludeFile(const FileEntry *File, bool isImport){
+  ++NumIncluded; // Count # of attempted #includes.
+
+  // Get information about this file.
+  PerFileInfo &FileInfo = getFileInfo(File);
+  
+  // If this is a #import directive, check that we have not already imported
+  // this header.
+  if (isImport) {
+    // If this has already been imported, don't import it again.
+    FileInfo.isImport = true;
+    
+    // Has this already been #import'ed or #include'd?
+    if (FileInfo.NumIncludes) return false;
+  } else {
+    // Otherwise, if this is a #include of a file that was previously #import'd
+    // or if this is the second #include of a #pragma once file, ignore it.
+    if (FileInfo.isImport)
+      return false;
+  }
+  
+  // Next, check to see if the file is wrapped with #ifndef guards.  If so, and
+  // if the macro that guards it is defined, we know the #include has no effect.
+  if (FileInfo.ControllingMacro &&
+      FileInfo.ControllingMacro->hasMacroDefinition()) {
+    ++NumMultiIncludeFileOptzn;
+    return false;
+  }
+  
+  // Increment the number of times this file has been included.
+  ++FileInfo.NumIncludes;
+  
+  return true;
+}
+
+
diff --git a/lib/Lex/Lexer.cpp b/lib/Lex/Lexer.cpp
new file mode 100644
index 0000000000..98bbb38630
--- /dev/null
+++ b/lib/Lex/Lexer.cpp
@@ -0,0 +1,1661 @@
+//===--- Lexer.cpp - C Language Family Lexer ------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file implements the Lexer and Token interfaces.
+//
+//===----------------------------------------------------------------------===//
+//
+// TODO: GCC Diagnostics emitted by the lexer:
+// PEDWARN: (form feed|vertical tab) in preprocessing directive
+//
+// Universal characters, unicode, char mapping:
+// WARNING: `%.*s' is not in NFKC
+// WARNING: `%.*s' is not in NFC
+//
+// Other:
+// TODO: Options to support:
+//    -fexec-charset,-fwide-exec-charset
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Lex/Lexer.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Basic/Diagnostic.h"
+#include "clang/Basic/SourceManager.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include <cctype>
+using namespace clang;
+
+static void InitCharacterInfo();
+
+//===----------------------------------------------------------------------===//
+// Token Class Implementation
+//===----------------------------------------------------------------------===//
+
+/// isObjCAtKeyword - Return true if we have an ObjC keyword identifier. 
+bool Token::isObjCAtKeyword(tok::ObjCKeywordKind objcKey) const {
+  return is(tok::identifier) && 
+         getIdentifierInfo()->getObjCKeywordID() == objcKey;
+}
+
+/// getObjCKeywordID - Return the ObjC keyword kind.
+tok::ObjCKeywordKind Token::getObjCKeywordID() const {
+  IdentifierInfo *specId = getIdentifierInfo();
+  return specId ? specId->getObjCKeywordID() : tok::objc_not_keyword;
+}
+
+/// isNamedIdentifier - Return true if this token is a ppidentifier with the
+/// specified name.  For example, tok.isNamedIdentifier("this").
+bool Token::isNamedIdentifier(const char *Name) const {
+  return IdentInfo && !strcmp(IdentInfo->getName(), Name);
+}
+
+
+//===----------------------------------------------------------------------===//
+// Lexer Class Implementation
+//===----------------------------------------------------------------------===//
+
+
+/// Lexer constructor - Create a new lexer object for the specified buffer
+/// with the specified preprocessor managing the lexing process.  This lexer
+/// assumes that the associated file buffer and Preprocessor objects will
+/// outlive it, so it doesn't take ownership of either of them.
+Lexer::Lexer(SourceLocation fileloc, Preprocessor &pp,
+             const char *BufStart, const char *BufEnd)
+  : FileLoc(fileloc), PP(&pp), Features(pp.getLangOptions()) {
+      
+  SourceManager &SourceMgr = PP->getSourceManager();
+  unsigned InputFileID = SourceMgr.getPhysicalLoc(FileLoc).getFileID();
+  const llvm::MemoryBuffer *InputFile = SourceMgr.getBuffer(InputFileID);
+      
+  Is_PragmaLexer = false;
+  InitCharacterInfo();
+  
+  // BufferStart must always be InputFile->getBufferStart().
+  BufferStart = InputFile->getBufferStart();
+  
+  // BufferPtr and BufferEnd can start out somewhere inside the current buffer.
+  // If unspecified, they starts at the start/end of the buffer.
+  BufferPtr = BufStart ? BufStart : BufferStart;
+  BufferEnd = BufEnd ? BufEnd : InputFile->getBufferEnd();
+
+  assert(BufferEnd[0] == 0 &&
+         "We assume that the input buffer has a null character at the end"
+         " to simplify lexing!");
+  
+  // Start of the file is a start of line.
+  IsAtStartOfLine = true;
+
+  // We are not after parsing a #.
+  ParsingPreprocessorDirective = false;
+
+  // We are not after parsing #include.
+  ParsingFilename = false;
+
+  // We are not in raw mode.  Raw mode disables diagnostics and interpretation
+  // of tokens (e.g. identifiers, thus disabling macro expansion).  It is used
+  // to quickly lex the tokens of the buffer, e.g. when handling a "#if 0" block
+  // or otherwise skipping over tokens.
+  LexingRawMode = false;
+  
+  // Default to keeping comments if requested.
+  KeepCommentMode = PP->getCommentRetentionState();
+}
+
+/// Lexer constructor - Create a new raw lexer object.  This object is only
+/// suitable for calls to 'LexRawToken'.  This lexer assumes that the
+/// associated file buffer will outlive it, so it doesn't take ownership of
+/// either of them.
+Lexer::Lexer(SourceLocation fileloc, const LangOptions &features,
+             const char *BufStart, const char *BufEnd)
+  : FileLoc(fileloc), PP(0), Features(features) {
+  Is_PragmaLexer = false;
+  InitCharacterInfo();
+  
+  BufferStart = BufStart;
+  BufferPtr = BufStart;
+  BufferEnd = BufEnd;
+  
+  assert(BufferEnd[0] == 0 &&
+         "We assume that the input buffer has a null character at the end"
+         " to simplify lexing!");
+  
+  // Start of the file is a start of line.
+  IsAtStartOfLine = true;
+  
+  // We are not after parsing a #.
+  ParsingPreprocessorDirective = false;
+  
+  // We are not after parsing #include.
+  ParsingFilename = false;
+  
+  // We *are* in raw mode.
+  LexingRawMode = true;
+  
+  // Never keep comments in raw mode.
+  KeepCommentMode = false;
+}
+
+
+/// Stringify - Convert the specified string into a C string, with surrounding
+/// ""'s, and with escaped \ and " characters.
+std::string Lexer::Stringify(const std::string &Str, bool Charify) {
+  std::string Result = Str;
+  char Quote = Charify ? '\'' : '"';
+  for (unsigned i = 0, e = Result.size(); i != e; ++i) {