Add support for retrieving the Doxygen comment associated with a given

declaration in the AST. 

The new ASTContext::getCommentForDecl function searches for a comment
that is attached to the given declaration, and returns that comment, 
which may be composed of several comment blocks.

Comments are always available in an AST. However, to avoid harming
performance, we don't actually parse the comments. Rather, we keep the
source ranges of all of the comments within a large, sorted vector,
then lazily extract comments via a binary search in that vector only
when needed (which never occurs in a "normal" compile).

Comments are written to a precompiled header/AST file as a blob of
source ranges. That blob is only lazily loaded when one requests a
comment for a declaration (this never occurs in a "normal" compile). 

The indexer testbed now supports comment extraction. When the
-point-at location points to a declaration with a Doxygen-style
comment, the indexer testbed prints the associated comment
block(s). See test/Index/comments.c for an example.

Some notes:
  - We don't actually attempt to parse the comment blocks themselves,
  beyond identifying them as Doxygen comment blocks to associate them
  with a declaration.
  - We won't find comment blocks that aren't adjacent to the
  declaration, because we start our search based on the location of
  the declaration.
  - We don't go through the necessary hops to find, for example,
  whether some redeclaration of a declaration has comments when our
  current declaration does not. Similarly, we don't attempt to
  associate a \param Foo marker in a function body comment with the
  parameter named Foo (although that is certainly possible).
  - Verification of my "no performance impact" claims is still "to be
  done".



git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@74704 91177308-0d34-0410-b5e6-96231b3b80d8

8 files changed, 74 insertions, 3 deletions

diff --git a/include/clang/AST/ASTContext.h b/include/clang/AST/ASTContext.h
index 450b28aefb..041a0f33ad 100644
--- a/include/clang/AST/ASTContext.h
+++ b/include/clang/AST/ASTContext.h
@@ -141,10 +141,19 @@ class ASTContext {
   ///  this ASTContext object.
   LangOptions LangOpts;
 
+  /// \brief Whether we have already loaded comment source ranges from an
+  /// external source.
+  bool LoadedExternalComments;
+  
   /// MallocAlloc/BumpAlloc - The allocator objects used to create AST objects.
   bool FreeMemory;
   llvm::MallocAllocator MallocAlloc;
   llvm::BumpPtrAllocator BumpAlloc;
+  
+  /// \brief Mapping from declarations to their comments, once we have
+  /// already looked up the comment associated with a given declaration.
+  llvm::DenseMap<const Decl *, std::string> DeclComments;
+  
 public:  
   TargetInfo &Target;
   IdentifierTable &Idents;
@@ -154,6 +163,10 @@ public:
   llvm::OwningPtr<ExternalASTSource> ExternalSource;
   clang::PrintingPolicy PrintingPolicy;
 
+  /// \brief Source ranges for all of the comments in the source file,
+  /// sorted in order of appearance in the translation unit.
+  std::vector<SourceRange> Comments;
+  
   SourceManager& getSourceManager() { return SourceMgr; }
   const SourceManager& getSourceManager() const { return SourceMgr; }
   void *Allocate(unsigned Size, unsigned Align = 8) {
@@ -178,7 +191,8 @@ public:
   
   TranslationUnitDecl *getTranslationUnitDecl() const { return TUDecl; }
 
-
+  const char *getCommentForDecl(const Decl *D);
+  
   // Builtin Types.
   QualType VoidTy;
   QualType BoolTy;
diff --git a/include/clang/AST/ExternalASTSource.h b/include/clang/AST/ExternalASTSource.h
index 4219bd507a..6f862a55d4 100644
--- a/include/clang/AST/ExternalASTSource.h
+++ b/include/clang/AST/ExternalASTSource.h
@@ -18,6 +18,7 @@
 #include "clang/AST/Type.h"
 #include "llvm/ADT/SmallVector.h"
 #include <cassert>
+#include <vector>
 namespace clang {
 
 class ASTConsumer;
@@ -57,6 +58,14 @@ public:
 
   virtual ~ExternalASTSource();
 
+  /// \brief Reads the source ranges that correspond to comments from
+  /// an external AST source.
+  ///
+  /// \param Comments the contents of this vector will be
+  /// replaced with the sorted set of source ranges corresponding to
+  /// comments in the source code.
+  virtual void ReadComments(std::vector<SourceRange> &Comments) = 0;
+  
   /// \brief Resolve a type ID into a type, potentially building a new
   /// type.
   virtual QualType GetType(uint32_t ID) = 0;
diff --git a/include/clang/Frontend/PCHBitCodes.h b/include/clang/Frontend/PCHBitCodes.h
index 63222725d5..80aa248a78 100644
--- a/include/clang/Frontend/PCHBitCodes.h
+++ b/include/clang/Frontend/PCHBitCodes.h
@@ -218,7 +218,11 @@ namespace clang {
 
       /// \brief Record code for the original file that was used to
       /// generate the precompiled header.
-      ORIGINAL_FILE_NAME = 20
+      ORIGINAL_FILE_NAME = 20,
+      
+      /// \brief Record code for the sorted array of source ranges where
+      /// comments were encountered in the source code.
+      COMMENT_RANGES = 21
     };
 
     /// \brief Record types used within a source manager block.
diff --git a/include/clang/Frontend/PCHReader.h b/include/clang/Frontend/PCHReader.h
index b3ed364343..8291f4697a 100644
--- a/include/clang/Frontend/PCHReader.h
+++ b/include/clang/Frontend/PCHReader.h
@@ -279,6 +279,12 @@ private:
   /// been loaded.
   llvm::SmallVector<Selector, 16> SelectorsLoaded;
 
+  /// \brief A sorted array of source ranges containing comments.
+  SourceRange *Comments;
+      
+  /// \brief The number of source ranges in the Comments array.
+  unsigned NumComments;
+      
   /// \brief The set of external definitions stored in the the PCH
   /// file.
   llvm::SmallVector<uint64_t, 16> ExternalDefinitions;
@@ -452,6 +458,14 @@ public:
   /// build prior to including the precompiled header.
   const std::string &getSuggestedPredefines() { return SuggestedPredefines; }
 
+  /// \brief Reads the source ranges that correspond to comments from
+  /// an external AST source.
+  ///
+  /// \param Comments the contents of this vector will be
+  /// replaced with the sorted set of source ranges corresponding to
+  /// comments in the source code.
+  virtual void ReadComments(std::vector<SourceRange> &Comments);
+      
   /// \brief Resolve a type ID into a type, potentially building a new
   /// type.
   virtual QualType GetType(pch::TypeID ID);
diff --git a/include/clang/Frontend/PCHWriter.h b/include/clang/Frontend/PCHWriter.h
index 5cb939547f..c663442e64 100644
--- a/include/clang/Frontend/PCHWriter.h
+++ b/include/clang/Frontend/PCHWriter.h
@@ -166,6 +166,7 @@ private:
   void WriteSourceManagerBlock(SourceManager &SourceMgr, 
                                const Preprocessor &PP);
   void WritePreprocessor(const Preprocessor &PP);
+  void WriteComments(ASTContext &Context);
   void WriteType(const Type *T);
   void WriteTypesBlock(ASTContext &Context);
   uint64_t WriteDeclContextLexicalBlock(ASTContext &Context, DeclContext *DC);
diff --git a/include/clang/Lex/Preprocessor.h b/include/clang/Lex/Preprocessor.h
index f229881bfc..6d5ed72455 100644
--- a/include/clang/Lex/Preprocessor.h
+++ b/include/clang/Lex/Preprocessor.h
@@ -26,6 +26,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/OwningPtr.h"
 #include "llvm/Support/Allocator.h"
+#include <vector>
 
 namespace clang {
   
@@ -35,6 +36,7 @@ class FileEntry;
 class HeaderSearch;
 class PragmaNamespace;
 class PragmaHandler;
+class CommentHandler;
 class ScratchBuffer;
 class TargetInfo;
 class PPCallbacks;
@@ -109,6 +111,10 @@ class Preprocessor {
   /// with this preprocessor.
   PragmaNamespace *PragmaHandlers;
   
+  /// \brief Tracks all of the comment handlers that the client registered 
+  /// with this preprocessor.
+  std::vector<CommentHandler *> CommentHandlers;
+  
   /// CurLexer - This is the current top of the stack that we're lexing from if
   /// not expanding a macro and we are lexing directly from source code.
   ///  Only one of CurLexer, CurPTHLexer, or CurTokenLexer will be non-null.
@@ -301,6 +307,14 @@ public:
   /// to remove a handler that has not been registered.
   void RemovePragmaHandler(const char *Namespace, PragmaHandler *Handler);
 
+  /// \brief Add the specified comment handler to the preprocessor.
+  void AddCommentHandler(CommentHandler *Handler);
+  
+  /// \brief Remove the specified comment handler.
+  ///
+  /// It is an error to remove a handler that has not been registered.
+  void RemoveCommentHandler(CommentHandler *Handler);
+  
   /// EnterMainSourceFile - Enter the specified FileID as the main source file,
   /// which implicitly adds the builtin defines etc.
   void EnterMainSourceFile();  
@@ -791,6 +805,7 @@ public:
   void HandlePragmaSystemHeader(Token &SysHeaderTok);
   void HandlePragmaDependency(Token &DependencyTok);
   void HandlePragmaComment(Token &CommentTok);
+  void HandleComment(SourceRange Comment);
 };
 
 /// PreprocessorFactory - A generic factory interface for lazily creating
@@ -801,6 +816,15 @@ public:
   virtual Preprocessor* CreatePreprocessor() = 0;  
 };
   
+/// \brief Abstract base class that describes a handler that will receive 
+/// source ranges for each of the comments encountered in the source file.
+class CommentHandler {
+public:
+  virtual ~CommentHandler();
+  
+  virtual void HandleComment(Preprocessor &PP, SourceRange Comment) = 0;
+};
+  
 }  // end namespace clang
 
 #endif
diff --git a/include/clang/Parse/Action.h b/include/clang/Parse/Action.h
index 24262a1126..d969562977 100644
--- a/include/clang/Parse/Action.h
+++ b/include/clang/Parse/Action.h
@@ -152,6 +152,10 @@ public:
   /// an empty string if not.  This is used for pretty crash reporting. 
   virtual std::string getDeclName(DeclPtrTy D) { return ""; }
   
+  /// \brief Invoked for each comment in the source code, providing the source
+  /// range that contains the comment.
+  virtual void ActOnComment(SourceRange Comment) { }
+  
   //===--------------------------------------------------------------------===//
   // Declaration Tracking Callbacks.
   //===--------------------------------------------------------------------===//
diff --git a/include/clang/Parse/Parser.h b/include/clang/Parse/Parser.h
index e5f62ec683..e2380542ae 100644
--- a/include/clang/Parse/Parser.h
+++ b/include/clang/Parse/Parser.h
@@ -82,7 +82,8 @@ class Parser {
   llvm::OwningPtr<PragmaHandler> PackHandler;
   llvm::OwningPtr<PragmaHandler> UnusedHandler;
   llvm::OwningPtr<PragmaHandler> WeakHandler;
-
+  llvm::OwningPtr<clang::CommentHandler> CommentHandler;
+  
   /// Whether the '>' token acts as an operator or not. This will be
   /// true except when we are parsing an expression within a C++
   /// template argument list, where the '>' closes the template