Introduce the notion of a "preprocessing record", which keeps track of

the macro definitions and macro instantiations that are found
during preprocessing. Preprocessing records are *not* generated by
default; rather, we provide a PPCallbacks subclass that hooks into the
existing callback mechanism to record this activity.

The only client of preprocessing records is CIndex, which keeps track
of macro definitions and instantations so that they can be exposed via
cursors. At present, only token annotation uses these facilities, and
only for macro instantiations; both will change in the near
future. However, with this change, token annotation properly annotates
macro instantiations that do not produce any tokens and instantiations
of macros that are later undef'd, improving our consistency.

Preprocessing directives that are not macro definitions are still
handled by clang_annotateTokens() via re-lexing, so that we don't have
to track every preprocessing directive in the preprocessing record.

Performance impact of preprocessing records is still TBD, although it
is limited to CIndex and therefore out of the path of the main compiler.


git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@98836 91177308-0d34-0410-b5e6-96231b3b80d8

9 files changed, 415 insertions, 128 deletions

diff --git a/include/clang/Frontend/ASTUnit.h b/include/clang/Frontend/ASTUnit.h
index 27ec12e4e4..0616d03226 100644
--- a/include/clang/Frontend/ASTUnit.h
+++ b/include/clang/Frontend/ASTUnit.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_CLANG_FRONTEND_ASTUNIT_H
 #define LLVM_CLANG_FRONTEND_ASTUNIT_H
 
+#include "clang/Lex/PreprocessingRecord.h"
 #include "clang/Basic/SourceManager.h"
 #include "llvm/ADT/OwningPtr.h"
 #include "clang/Basic/FileManager.h"
@@ -53,7 +54,8 @@ class ASTUnit {
   llvm::OwningPtr<TargetInfo>       Target;
   llvm::OwningPtr<Preprocessor>     PP;
   llvm::OwningPtr<ASTContext>       Ctx;
-
+  llvm::OwningPtr<PreprocessingRecord> Preprocessing;
+  
   /// Optional owned invocation, just used to make the invocation used in
   /// LoadFromCommandLine available.
   llvm::OwningPtr<CompilerInvocation> Invocation;
@@ -135,6 +137,12 @@ public:
   const ASTContext &getASTContext() const { return *Ctx.get(); }
         ASTContext &getASTContext()       { return *Ctx.get(); }
 
+  const PreprocessingRecord &getPreprocessingRecord() const { 
+    return *Preprocessing.get(); 
+  }
+  PreprocessingRecord &getPreprocessingRecord() { return *Preprocessing.get(); }
+  bool hasPreprocessingRecord() { return Preprocessing.get() != 0; }
+    
   const FileManager &getFileManager() const { return FileMgr; }
         FileManager &getFileManager()       { return FileMgr; }
 
@@ -204,7 +212,8 @@ public:
   static ASTUnit *LoadFromCompilerInvocation(CompilerInvocation *CI,
                                              Diagnostic &Diags,
                                              bool OnlyLocalDecls = false,
-                                             bool CaptureDiagnostics = false);
+                                             bool CaptureDiagnostics = false,
+                                         bool WantPreprocessingRecord = false);
 
   /// LoadFromCommandLine - Create an ASTUnit from a vector of command line
   /// arguments, which must specify exactly one source file.
@@ -227,7 +236,8 @@ public:
                                       bool OnlyLocalDecls = false,
                                       RemappedFile *RemappedFiles = 0,
                                       unsigned NumRemappedFiles = 0,
-                                      bool CaptureDiagnostics = false);
+                                      bool CaptureDiagnostics = false,
+                                      bool WantPreprocessingRecord = false);
 };
 
 } // namespace clang
diff --git a/include/clang/Lex/PreprocessingRecord.h b/include/clang/Lex/PreprocessingRecord.h
new file mode 100644
index 0000000000..4ebd4e6039
--- /dev/null
+++ b/include/clang/Lex/PreprocessingRecord.h
@@ -0,0 +1,235 @@
+//===--- PreprocessingRecord.h - Record of Preprocessing --------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines the PreprocessingRecord class, which maintains a record
+//  of what occurred during preprocessing.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_CLANG_LEX_PREPROCESSINGRECORD_H
+#define LLVM_CLANG_LEX_PREPROCESSINGRECORD_H
+
+#include "clang/Lex/PPCallbacks.h"
+#include "clang/Basic/SourceLocation.h"
+#include "llvm/ADT/PointerUnion.h"
+#include "llvm/Support/Allocator.h"
+#include <vector>
+
+namespace clang {
+  class IdentifierInfo;
+  class PreprocessingRecord;
+}
+
+/// \brief Allocates memory within a Clang preprocessing record.
+void* operator new(size_t bytes, clang::PreprocessingRecord& PR,
+                   unsigned alignment = 8) throw();
+
+/// \brief Frees memory allocated in a Clang preprocessing record.
+void operator delete(void* ptr, clang::PreprocessingRecord& PR,
+                     unsigned) throw();
+
+namespace clang {
+  /// \brief Base class that describes a preprocessed entity, which may be a
+  /// preprocessor directive or macro instantiation.
+  class PreprocessedEntity {
+  public:
+    /// \brief The kind of preprocessed entity an object describes.
+    enum EntityKind {
+      /// \brief A macro instantiation.
+      MacroInstantiationKind,
+      
+      /// \brief A preprocessing directive whose kind is not specified.
+      ///
+      /// This kind will be used for any preprocessing directive that does not
+      /// have a more specific kind within the \c DirectiveKind enumeration.
+      PreprocessingDirectiveKind,
+      
+      /// \brief A macro definition.
+      MacroDefinitionKind,
+      
+      FirstPreprocessingDirective = PreprocessingDirectiveKind,
+      LastPreprocessingDirective = MacroDefinitionKind
+    };
+
+  private:
+    /// \brief The kind of preprocessed entity that this object describes.
+    EntityKind Kind;
+    
+    /// \brief The source range that covers this preprocessed entity.
+    SourceRange Range;
+    
+  protected:
+    PreprocessedEntity(EntityKind Kind, SourceRange Range)
+      : Kind(Kind), Range(Range) { }
+    
+  public:
+    /// \brief Retrieve the kind of preprocessed entity stored in this object.
+    EntityKind getKind() const { return Kind; }
+    
+    /// \brief Retrieve the source range that covers this entire preprocessed 
+    /// entity.
+    SourceRange getSourceRange() const { return Range; }
+    
+    // Implement isa/cast/dyncast/etc.
+    static bool classof(const PreprocessedEntity *) { return true; }
+
+    // Only allow allocation of preprocessed entities using the allocator 
+    // in PreprocessingRecord or by doing a placement new.
+    void* operator new(size_t bytes, PreprocessingRecord& PR,
+                       unsigned alignment = 8) throw() {
+      return ::operator new(bytes, PR, alignment);
+    }
+    
+    void* operator new(size_t bytes, void* mem) throw() {
+      return mem;
+    }
+    
+    void operator delete(void* ptr, PreprocessingRecord& PR, 
+                         unsigned alignment) throw() {
+      return ::operator delete(ptr, PR, alignment);
+    }
+    
+    void operator delete(void*, std::size_t) throw() { }
+    void operator delete(void*, void*) throw() { }
+    
+  private:
+    // Make vanilla 'new' and 'delete' illegal for preprocessed entities.
+    void* operator new(size_t bytes) throw();
+    void operator delete(void* data) throw();
+  };
+  
+  /// \brief Records the location of a macro instantiation.
+  class MacroInstantiation : public PreprocessedEntity {
+    /// \brief The name of the macro being instantiation.
+    IdentifierInfo *Name;
+    
+    /// \brief The location of the definition of the macro being instantiated.
+    SourceLocation DefinitionLocation;
+    
+  public:
+    MacroInstantiation(IdentifierInfo *Name, SourceRange Range,
+                       SourceLocation DefinitionLocation)
+      : PreprocessedEntity(MacroInstantiationKind, Range), Name(Name), 
+        DefinitionLocation(DefinitionLocation) { }
+    
+    /// \brief The name of the macro being instantiated.
+    IdentifierInfo *getName() const { return Name; }
+    
+    /// \brief The location of the definition of the macro being instantiated.
+    /// FIXME: Could we just provide MacroInfo pointers instead, by teaching
+    /// the preprocessor to hold on to them when we care to keep them around?
+    SourceLocation getDefinitionLocation() const { return DefinitionLocation; }
+
+    // Implement isa/cast/dyncast/etc.
+    static bool classof(const PreprocessedEntity *PE) {
+      return PE->getKind() == MacroInstantiationKind;
+    }
+    static bool classof(const MacroInstantiation *) { return true; }
+
+  };
+  
+  /// \brief Records the presence of a preprocessor directive.
+  class PreprocessingDirective : public PreprocessedEntity {
+  public:
+    PreprocessingDirective(EntityKind Kind, SourceRange Range) 
+      : PreprocessedEntity(Kind, Range) { }
+    
+    // Implement isa/cast/dyncast/etc.
+    static bool classof(const PreprocessedEntity *PD) { 
+      return PD->getKind() >= FirstPreprocessingDirective &&
+             PD->getKind() <= LastPreprocessingDirective;
+    }
+    static bool classof(const PreprocessingDirective *) { return true; }    
+  };
+  
+  /// \brief Record the location of a macro definition.
+  class MacroDefinition : public PreprocessingDirective {
+    /// \brief The name of the macro being defined.
+    const IdentifierInfo *Name;
+    
+    /// \brief The location of the macro name in the macro definition.
+    SourceLocation Location;
+
+  public:
+    explicit MacroDefinition(const IdentifierInfo *Name, SourceLocation Location,
+                             SourceRange Range)
+      : PreprocessingDirective(MacroDefinitionKind, Range), Name(Name), 
+        Location(Location) { }
+    
+    /// \brief Retrieve the name of the macro being defined.
+    const IdentifierInfo *getMacroName() const { return Name; }
+    
+    /// \brief Retrieve the location of the macro name in the definition.
+    SourceLocation getLocation() const { return Location; }
+    
+    // Implement isa/cast/dyncast/etc.
+    static bool classof(const PreprocessingDirective *PD) {
+      return PD->getKind() == MacroDefinitionKind;
+    }
+    static bool classof(const MacroDefinition *) { return true; }
+  };
+    
+  /// \brief A record of the steps taken while preprocessing a source file,
+  /// including the various preprocessing directives processed, macros 
+  /// instantiated, etc.
+  class PreprocessingRecord {
+    /// \brief Allocator used to store preprocessing objects.
+    llvm::BumpPtrAllocator BumpAlloc;
+
+    /// \brief The set of preprocessed entities in this record, in order they
+    /// were seen.
+    std::vector<PreprocessedEntity *> PreprocessedEntities;
+    
+  public:
+    /// \brief Allocate memory in the preprocessing record.
+    void *Allocate(unsigned Size, unsigned Align = 8) {
+      return BumpAlloc.Allocate(Size, Align);
+    }
+    
+    /// \brief Deallocate memory in the preprocessing record.
+    void Deallocate(void *Ptr) { }
+    
+    // Iteration over the preprocessed entities.
+    typedef std::vector<PreprocessedEntity *>::iterator iterator;
+    typedef std::vector<PreprocessedEntity *>::const_iterator const_iterator;
+    iterator begin() { return PreprocessedEntities.begin(); }
+    iterator end() { return PreprocessedEntities.end(); }
+    const_iterator begin() const { return PreprocessedEntities.begin(); }
+    const_iterator end() const { return PreprocessedEntities.end(); }
+    
+    /// \brief Add a new preprocessed entity to this record.
+    void addPreprocessedEntity(PreprocessedEntity *Entity);
+  };
+  
+  /// \brief Preprocessor callback action used to populate a preprocessing
+  /// record.
+  class PopulatePreprocessingRecord : public PPCallbacks {
+    /// \brief The preprocessing record this action will populate.
+    PreprocessingRecord &Record;
+    
+  public:
+    explicit PopulatePreprocessingRecord(PreprocessingRecord &Record)
+      : Record(Record) { }
+    
+    virtual void MacroExpands(const Token &Id, const MacroInfo* MI);
+    virtual void MacroDefined(const IdentifierInfo *II, const MacroInfo *MI);
+  };
+  
+} // end namespace clang
+
+inline void* operator new(size_t bytes, clang::PreprocessingRecord& PR,
+                          unsigned alignment) throw() {
+  return PR.Allocate(bytes, alignment);
+}
+
+inline void operator delete(void* ptr, clang::PreprocessingRecord& PR,
+                            unsigned) throw() {
+  PR.Deallocate(ptr);
+}
+
+#endif // LLVM_CLANG_LEX_PREPROCESSINGRECORD_H
diff --git a/lib/Frontend/ASTUnit.cpp b/lib/Frontend/ASTUnit.cpp
index 935c415243..1aaa536cb3 100644
--- a/lib/Frontend/ASTUnit.cpp
+++ b/lib/Frontend/ASTUnit.cpp
@@ -278,7 +278,8 @@ public:
 ASTUnit *ASTUnit::LoadFromCompilerInvocation(CompilerInvocation *CI,
                                              Diagnostic &Diags,
                                              bool OnlyLocalDecls,
-                                             bool CaptureDiagnostics) {
+                                             bool CaptureDiagnostics,
+                                             bool WantPreprocessingRecord) {
   // Create the compiler instance to use for building the AST.
   CompilerInstance Clang;
   llvm::OwningPtr<ASTUnit> AST;
@@ -328,6 +329,15 @@ ASTUnit *ASTUnit::LoadFromCompilerInvocation(CompilerInvocation *CI,
   // Create the preprocessor.
   Clang.createPreprocessor();
 
+  // If the ASTUnit was requested to store information about preprocessing,
+  // create storage for that information and attach an appropriate callback to 
+  // populate that storage.
+  if (WantPreprocessingRecord) {
+    AST->Preprocessing.reset(new PreprocessingRecord);
+    Clang.getPreprocessor().addPPCallbacks(
+                          new PopulatePreprocessingRecord(*AST->Preprocessing));
+  }
+      
   Act.reset(new TopLevelDeclTrackerAction(*AST));
   if (!Act->BeginSourceFile(Clang, Clang.getFrontendOpts().Inputs[0].second,
                            /*IsAST=*/false))
@@ -367,7 +377,8 @@ ASTUnit *ASTUnit::LoadFromCommandLine(const char **ArgBegin,
                                       bool OnlyLocalDecls,
                                       RemappedFile *RemappedFiles,
                                       unsigned NumRemappedFiles,
-                                      bool CaptureDiagnostics) {
+                                      bool CaptureDiagnostics,
+                                      bool WantPreprocessingRecord) {
   llvm::SmallVector<const char *, 16> Args;
   Args.push_back("<clang>"); // FIXME: Remove dummy argument.
   Args.insert(Args.end(), ArgBegin, ArgEnd);
@@ -419,5 +430,6 @@ ASTUnit *ASTUnit::LoadFromCommandLine(const char **ArgBegin,
 
   CI->getFrontendOpts().DisableFree = true;
   return LoadFromCompilerInvocation(CI.take(), Diags, OnlyLocalDecls,
-                                    CaptureDiagnostics);
+                                    CaptureDiagnostics, 
+                                    WantPreprocessingRecord);
 }
diff --git a/lib/Lex/CMakeLists.txt b/lib/Lex/CMakeLists.txt
index 81a1e01f96..632fbc6340 100644
--- a/lib/Lex/CMakeLists.txt
+++ b/lib/Lex/CMakeLists.txt
@@ -16,6 +16,7 @@ add_clang_library(clangLex
   PPMacroExpansion.cpp
   PTHLexer.cpp
   Pragma.cpp
+  PreprocessingRecord.cpp
   Preprocessor.cpp
   PreprocessorLexer.cpp
   ScratchBuffer.cpp
diff --git a/lib/Lex/PreprocessingRecord.cpp b/lib/Lex/PreprocessingRecord.cpp
new file mode 100644
index 0000000000..fe081b69bb
--- /dev/null
+++ b/lib/Lex/PreprocessingRecord.cpp
@@ -0,0 +1,37 @@
+//===--- PreprocessingRecord.cpp - Record of Preprocessing ------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file implements the PreprocessingRecord class, which maintains a record
+//  of what occurred during preprocessing, and its helpers.
+//
+//===----------------------------------------------------------------------===//
+#include "clang/Lex/PreprocessingRecord.h"
+#include "clang/Lex/MacroInfo.h"
+#include "clang/Lex/Token.h"
+
+using namespace clang;
+
+void PreprocessingRecord::addPreprocessedEntity(PreprocessedEntity *Entity) {
+  PreprocessedEntities.push_back(Entity);
+}
+
+void PopulatePreprocessingRecord::MacroExpands(const Token &Id, 
+                                               const MacroInfo* MI) {
+  Record.addPreprocessedEntity(
+                        new (Record) MacroInstantiation(Id.getIdentifierInfo(),
+                                                       Id.getLocation(),
+                                                      MI->getDefinitionLoc()));
+}
+
+void PopulatePreprocessingRecord::MacroDefined(const IdentifierInfo *II, 
+                                               const MacroInfo *MI) {
+  SourceRange R(MI->getDefinitionLoc(), MI->getDefinitionEndLoc());
+  Record.addPreprocessedEntity(
+                  new (Record) MacroDefinition(II, MI->getDefinitionLoc(), R));
+}
diff --git a/test/Index/annotate-tokens-pp.c b/test/Index/annotate-tokens-pp.c
index e646948c89..db808d0943 100644
--- a/test/Index/annotate-tokens-pp.c
+++ b/test/Index/annotate-tokens-pp.c
@@ -1,32 +1,58 @@
+#define NOTHING(X,Y)
+#define STILL_NOTHING NOTHING(honk,warble)
 #define BAR baz
 #define WIBBLE(X, Y) X##Y
-float WIBBLE(int, float);
-int BAR;
+NOTHING(more,junk) float WIBBLE(int, float);
+int BAR STILL_NOTHING;
 #include "foo.h"
+#undef BAR
 
-// RUN: c-index-test -test-annotate-tokens=%s:1:1:6:1 -I%S/Inputs %s | FileCheck %s
-// CHECK: Punctuation: "#" [1:1 - 1:2] preprocessing directive=
-// CHECK: Identifier: "define" [1:2 - 1:8] preprocessing directive=
-// CHECK: Identifier: "BAR" [1:9 - 1:12] preprocessing directive=
-// CHECK: Identifier: "baz" [1:13 - 1:16] preprocessing directive=
+// RUN: c-index-test -test-annotate-tokens=%s:2:1:9:1 -I%S/Inputs %s | FileCheck %s
 // CHECK: Punctuation: "#" [2:1 - 2:2] preprocessing directive=
 // CHECK: Identifier: "define" [2:2 - 2:8] preprocessing directive=
-// CHECK: Identifier: "WIBBLE" [2:9 - 2:15] preprocessing directive=
-// CHECK: Punctuation: "(" [2:15 - 2:16] preprocessing directive=
-// CHECK: Identifier: "X" [2:16 - 2:17] preprocessing directive=
-// CHECK: Punctuation: "," [2:17 - 2:18] preprocessing directive=
-// CHECK: Identifier: "Y" [2:19 - 2:20] preprocessing directive=
-// CHECK: Punctuation: ")" [2:20 - 2:21] preprocessing directive=
-// CHECK: Identifier: "WIBBLE" [3:7 - 3:13] macro instantiation=
-// CHECK: Punctuation: "(" [3:13 - 3:14]
-// CHECK: Keyword: "int" [3:14 - 3:17]
-// CHECK: Punctuation: "," [3:17 - 3:18]
-// CHECK: Keyword: "float" [3:19 - 3:24]
-// CHECK: Punctuation: ")" [3:24 - 3:25]
-// CHECK: Punctuation: ";" [3:25 - 3:26]
-// CHECK: Keyword: "int" [4:1 - 4:4]
-// CHECK: Identifier: "BAR" [4:5 - 4:8] macro instantiation=
-// CHECK: Punctuation: ";" [4:8 - 4:9]
-// CHECK: Punctuation: "#" [5:1 - 5:2] preprocessing directive=
-// CHECK: Identifier: "include" [5:2 - 5:9] preprocessing directive=
-// CHECK: Literal: ""foo.h"" [5:10 - 5:17] preprocessing directive=
+// CHECK: Identifier: "STILL_NOTHING" [2:9 - 2:22] preprocessing directive=
+// CHECK: Identifier: "NOTHING" [2:23 - 2:30] preprocessing directive=
+// CHECK: Punctuation: "(" [2:30 - 2:31] preprocessing directive=
+// CHECK: Identifier: "honk" [2:31 - 2:35] preprocessing directive=
+// CHECK: Punctuation: "," [2:35 - 2:36] preprocessing directive=
+// CHECK: Identifier: "warble" [2:36 - 2:42] preprocessing directive=
+// CHECK: Punctuation: ")" [2:42 - 2:43] preprocessing directive=
+// CHECK: Punctuation: "#" [3:1 - 3:2] preprocessing directive=
+// CHECK: Identifier: "define" [3:2 - 3:8] preprocessing directive=
+// CHECK: Identifier: "BAR" [3:9 - 3:12] preprocessing directive=
+// CHECK: Identifier: "baz" [3:13 - 3:16] preprocessing directive=
+// CHECK: Punctuation: "#" [4:1 - 4:2] preprocessing directive=
+// CHECK: Identifier: "define" [4:2 - 4:8] preprocessing directive=
+// CHECK: Identifier: "WIBBLE" [4:9 - 4:15] preprocessing directive=
+// CHECK: Punctuation: "(" [4:15 - 4:16] preprocessing directive=
+// CHECK: Identifier: "X" [4:16 - 4:17] preprocessing directive=
+// CHECK: Punctuation: "," [4:17 - 4:18] preprocessing directive=
+// CHECK: Identifier: "Y" [4:19 - 4:20] preprocessing directive=
+// CHECK: Punctuation: ")" [4:20 - 4:21] preprocessing directive=
+// CHECK: Identifier: "X" [4:22 - 4:23] preprocessing directive=
+// CHECK: Punctuation: "##" [4:23 - 4:25] preprocessing directive=
+// CHECK: Identifier: "Y" [4:25 - 4:26] preprocessing directive=
+// CHECK: Identifier: "NOTHING" [5:1 - 5:8] macro instantiation=NOTHING
+// CHECK: Punctuation: "(" [5:8 - 5:9]
+// CHECK: Identifier: "more" [5:9 - 5:13]
+// CHECK: Punctuation: "," [5:13 - 5:14]
+// CHECK: Identifier: "junk" [5:14 - 5:18]
+// CHECK: Punctuation: ")" [5:18 - 5:19]
+// CHECK: Keyword: "float" [5:20 - 5:25]
+// CHECK: Identifier: "WIBBLE" [5:26 - 5:32] macro instantiation=WIBBLE
+// CHECK: Punctuation: "(" [5:32 - 5:33]
+// CHECK: Keyword: "int" [5:33 - 5:36]
+// CHECK: Punctuation: "," [5:36 - 5:37]
+// CHECK: Keyword: "float" [5:38 - 5:43]
+// CHECK: Punctuation: ")" [5:43 - 5:44]
+// CHECK: Punctuation: ";" [5:44 - 5:45]
+// CHECK: Keyword: "int" [6:1 - 6:4]
+// CHECK: Identifier: "BAR" [6:5 - 6:8] macro instantiation=BAR
+// CHECK: Identifier: "STILL_NOTHING" [6:9 - 6:22] macro instantiation=STILL_NOTHING
+// CHECK: Punctuation: ";" [6:22 - 6:23]
+// CHECK: Punctuation: "#" [7:1 - 7:2] preprocessing directive=
+// CHECK: Identifier: "include" [7:2 - 7:9] preprocessing directive=
+// CHECK: Literal: ""foo.h"" [7:10 - 7:17] preprocessing directive=
+// CHECK: Punctuation: "#" [8:1 - 8:2] preprocessing directive=
+// CHECK: Identifier: "undef" [8:2 - 8:7] preprocessing directive=
+// CHECK: Identifier: "BAR" [8:8 - 8:11] preprocessing directive=
diff --git a/tools/CIndex/CIndex.cpp b/tools/CIndex/CIndex.cpp
index ed0c562eb0..deab84698c 100644
--- a/tools/CIndex/CIndex.cpp
+++ b/tools/CIndex/CIndex.cpp
@@ -1008,7 +1008,8 @@ clang_createTranslationUnitFromSourceFile(CXIndex CIdx,
                                    CXXIdx->getOnlyLocalDecls(),
                                    RemappedFiles.data(),
                                    RemappedFiles.size(),
-                                   /*CaptureDiagnostics=*/true));
+                                   /*CaptureDiagnostics=*/true,
+                                   /*WantPreprocessingRecord=*/true));
 
     // FIXME: Until we have broader testing, just drop the entire AST if we
     // encountered an error.
@@ -1438,6 +1439,10 @@ CXString clang_getCursorSpelling(CXCursor C) {
     return createCXString("");
   }
 
+  if (C.kind == CXCursor_MacroInstantiation)
+    return createCXString(getCursorMacroInstantiation(C)->getName()
+                                                           ->getNameStart());
+
   if (clang_isDeclaration(C.kind))
     return getDeclSpelling(getCursorDecl(C));
 
@@ -1656,7 +1661,8 @@ CXSourceLocation clang_getCursorLocation(CXCursor C) {
   }
 
   if (C.kind == CXCursor_MacroInstantiation) {
-    SourceLocation L = cxcursor::getCursorMacroInstantiation(C).getBegin();
+    SourceLocation L
+      = cxcursor::getCursorMacroInstantiation(C)->getSourceRange().getBegin();
     return cxloc::translateSourceLocation(getCursorContext(C), L);
   }
   
@@ -1717,7 +1723,7 @@ CXSourceRange clang_getCursorExtent(CXCursor C) {
   }
 
   if (C.kind == CXCursor_MacroInstantiation) {
-    SourceRange R = cxcursor::getCursorMacroInstantiation(C);
+    SourceRange R = cxcursor::getCursorMacroInstantiation(C)->getSourceRange();
     return cxloc::translateSourceRange(getCursorContext(C), R);
   }
   
@@ -2030,16 +2036,36 @@ void clang_enableStackTraces(void) {
 //===----------------------------------------------------------------------===//
 
 namespace {
-/// IgnoringDiagClient - This is a diagnostic client that just ignores all
-/// diags.
-class IgnoringDiagClient : public DiagnosticClient {
-  void HandleDiagnostic(Diagnostic::Level DiagLevel,
-                        const DiagnosticInfo &Info) {
-    // Just ignore it.
-  }
-};
+  class ComparePreprocessedEntityLocation {
+    SourceManager &SM;
+
+  public:
+    explicit ComparePreprocessedEntityLocation(SourceManager &SM) : SM(SM) { }
+
+    bool operator()(const PreprocessedEntity *Entity, SourceLocation Loc) {
+      return SM.isBeforeInTranslationUnit(Entity->getSourceRange().getEnd(), 
+                                          Loc);
+    }
+
+    bool operator()(SourceLocation Loc, const PreprocessedEntity *Entity) {
+      return SM.isBeforeInTranslationUnit(Loc, 
+                                          Entity->getSourceRange().getBegin());
+    }
+
+    bool operator()(const PreprocessedEntity *Entity, SourceRange R) {
+      return SM.isBeforeInTranslationUnit(Entity->getSourceRange().getEnd(), 
+                                          R.getBegin());
+    }
+
+    bool operator()(SourceRange R, const PreprocessedEntity *Entity) {
+      return SM.isBeforeInTranslationUnit(R.getEnd(), 
+                                          Entity->getSourceRange().getBegin());
+    }
+  };
 }
 
+
+
 /* CXToken layout:
  *   int_data[0]: a CXTokenKind
  *   int_data[1]: starting token location
@@ -2298,13 +2324,13 @@ void clang_annotateTokens(CXTranslationUnit TU,
   if (RelexOkay) {
     Lexer Lex(SourceMgr.getLocForStartOfFile(BeginLocInfo.first),
               CXXUnit->getASTContext().getLangOptions(),
-              Buffer.begin(), Buffer.data() + BeginLocInfo.second, Buffer.end());
+              Buffer.begin(), Buffer.data() + BeginLocInfo.second, 
+              Buffer.end());
     Lex.SetCommentRetentionState(true);
     
     // Lex tokens in raw mode until we hit the end of the range, to avoid 
     // entering #includes or expanding macros.
     std::vector<Token> TokenStream;
-    Preprocessor &PP = CXXUnit->getPreprocessor();
     while (true) {
       Token Tok;
       Lex.LexFromRawLexer(Tok);
@@ -2339,84 +2365,30 @@ void clang_annotateTokens(CXTranslationUnit TU,
         continue;
       }
       
-      // If this is a ## token, change its kind to unknown so that
-      // repreprocessing it will not produce an error.
-      if (Tok.is(tok::hashhash))
-        Tok.setKind(tok::unknown);
-      
-      // If this raw token is an identifier, the raw lexer won't have
-      // looked up the corresponding identifier info for it.  Do this
-      // now so that it will be macro expanded when we re-preprocess
-      // it.
-      if (Tok.is(tok::identifier)) {
-        // Change the kind of this identifier to the appropriate token kind, e.g.
-        // turning "for" into a keyword.
-        Tok.setKind(PP.LookUpIdentifierInfo(Tok)->getTokenID());
-      }
-      
-      TokenStream.push_back(Tok);
-      
       if (Tok.is(tok::eof))
         break;
     }
+  }
+   
+  if (CXXUnit->hasPreprocessingRecord()) {
+    PreprocessingRecord &PPRec = CXXUnit->getPreprocessingRecord();
+    std::pair<PreprocessingRecord::iterator, PreprocessingRecord::iterator>
+      Entities = std::equal_range(PPRec.begin(), PPRec.end(), RegionOfInterest,
+                                  ComparePreprocessedEntityLocation(SourceMgr));
+    for (; Entities.first != Entities.second; ++Entities.first) {
+      PreprocessedEntity *Entity = *Entities.first;
+      if (MacroInstantiation *MI = dyn_cast<MacroInstantiation>(Entity)) {
+        SourceLocation Loc = MI->getSourceRange().getBegin();
+        if (Loc.isFileID()) {
+          Annotated[Loc.getRawEncoding()]
+            = MakeMacroInstantiationCursor(MI, CXXUnit);
+        }
 
-    // Temporarily change the diagnostics object so that we ignore any
-    // generated diagnostics from this pass.
-    IgnoringDiagClient TmpDC;
-    Diagnostic TmpDiags(&TmpDC);
-    Diagnostic *OldDiags = &PP.getDiagnostics();
-    PP.setDiagnostics(TmpDiags);
-
-    // Inform the preprocessor that we don't want comments.
-    PP.SetCommentRetentionState(false, false);
-    
-    // Enter the tokens we just lexed.  This will cause them to be macro expanded
-    // but won't enter sub-files (because we removed #'s).
-    PP.EnterTokenStream(&TokenStream[0], TokenStream.size(), false, false);
-
-    // Lex all the tokens.
-    Token Tok;
-    PP.Lex(Tok);
-    while (Tok.isNot(tok::eof)) {
-      // Ignore non-macro tokens.
-      if (!Tok.getLocation().isMacroID()) {
-        PP.Lex(Tok);
-        continue;
-      }
-
-      // Okay, we have the first token of a macro expansion. Keep
-      // track of the range of the macro expansion.
-      std::pair<SourceLocation, SourceLocation> LLoc =
-        SourceMgr.getInstantiationRange(Tok.getLocation());
-
-      // Ignore tokens whose instantiation location was not the main file.
-      if (SourceMgr.getFileID(LLoc.first) != BeginLocInfo.first) {
-        PP.Lex(Tok);
         continue;
       }
 
-      assert(SourceMgr.getFileID(LLoc.second) == BeginLocInfo.first &&
-             "Start and end of expansion must be in the same ultimate file!");
-
-      // Okay, eat this token, getting the next one.
-      PP.Lex(Tok);
-
-      // Skip all the rest of the tokens that are part of this macro
-      // instantiation.  It would be really nice to pop up a window with all the
-      // spelling of the tokens or something.
-      while (!Tok.is(tok::eof) &&
-             SourceMgr.getInstantiationLoc(Tok.getLocation()) == LLoc.first)
-        PP.Lex(Tok);
-
-      CXCursor Cursor
-        = cxcursor::MakeMacroInstantiationCursor(SourceRange(LLoc.first, 
-                                                             LLoc.second), 
-                                                 CXXUnit);
-      Annotated[LLoc.first.getRawEncoding()] = Cursor;
+      // FIXME: expose other preprocessed entities.
     }
-
-    // Restore diagnostics object back to its own thing.
-    PP.setDiagnostics(*OldDiags);
   }
   
   for (unsigned I = 0; I != NumTokens; ++I) {
diff --git a/tools/CIndex/CXCursor.cpp b/tools/CIndex/CXCursor.cpp
index aa81d60f61..1e5265ee9a 100644
--- a/tools/CIndex/CXCursor.cpp
+++ b/tools/CIndex/CXCursor.cpp
@@ -314,22 +314,15 @@ SourceRange cxcursor::getCursorPreprocessingDirective(CXCursor C) {
                                       reinterpret_cast<uintptr_t> (C.data[1])));
 }
 
-CXCursor cxcursor::MakeMacroInstantiationCursor(SourceRange Range, 
+CXCursor cxcursor::MakeMacroInstantiationCursor(MacroInstantiation *MI, 
                                                 ASTUnit *TU) {
-  CXCursor C = { CXCursor_MacroInstantiation, 
-                 { reinterpret_cast<void *>(Range.getBegin().getRawEncoding()),
-                   reinterpret_cast<void *>(Range.getEnd().getRawEncoding()),
-                   TU }
-               };
+  CXCursor C = { CXCursor_MacroInstantiation, { MI, 0, TU } };
   return C;
 }
 
-SourceRange cxcursor::getCursorMacroInstantiation(CXCursor C) {
+MacroInstantiation *cxcursor::getCursorMacroInstantiation(CXCursor C) {
   assert(C.kind == CXCursor_MacroInstantiation);
-  return SourceRange(SourceLocation::getFromRawEncoding(
-                                      reinterpret_cast<uintptr_t> (C.data[0])),
-                     SourceLocation::getFromRawEncoding(
-                                      reinterpret_cast<uintptr_t> (C.data[1])));
+  return static_cast<MacroInstantiation *>(C.data[0]);
 }
 
 Decl *cxcursor::getCursorDecl(CXCursor Cursor) {
diff --git a/tools/CIndex/CXCursor.h b/tools/CIndex/CXCursor.h
index 12103b6d7b..24ac1e414e 100644
--- a/tools/CIndex/CXCursor.h
+++ b/tools/CIndex/CXCursor.h
@@ -25,6 +25,7 @@ class ASTUnit;
 class Attr;
 class Decl;
 class Expr;
+class MacroInstantiation;
 class NamedDecl;
 class ObjCInterfaceDecl;
 class ObjCProtocolDecl;
@@ -80,11 +81,11 @@ CXCursor MakePreprocessingDirectiveCursor(SourceRange Range, ASTUnit *TU);
 SourceRange getCursorPreprocessingDirective(CXCursor C);
 
 /// \brief Create a macro instantiation cursor.
-CXCursor MakeMacroInstantiationCursor(SourceRange Range, ASTUnit *TU);
+CXCursor MakeMacroInstantiationCursor(MacroInstantiation *, ASTUnit *TU);
 
 /// \brief Unpack a given macro instantiation cursor to retrieve its
 /// source range.
-SourceRange getCursorMacroInstantiation(CXCursor C);
+MacroInstantiation *getCursorMacroInstantiation(CXCursor C);
 
 Decl *getCursorDecl(CXCursor Cursor);
 Expr *getCursorExpr(CXCursor Cursor);