Add libclang APIs to walk comments ASTs and an API to convert a comment to an

HTML fragment.

For testing, c-index-test now has even more output:
* HTML rendering of a comment
* comment AST tree dump in S-expressions like Comment::dump(), but implemented
* with libclang APIs.


git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@160577 91177308-0d34-0410-b5e6-96231b3b80d8

10 files changed, 1686 insertions, 23 deletions

diff --git a/include/clang-c/Index.h b/include/clang-c/Index.h
index 33a92e4653..d95f010a6f 100644
--- a/include/clang-c/Index.h
+++ b/include/clang-c/Index.h
@@ -2037,6 +2037,13 @@ typedef struct {
 } CXCursor;
 
 /**
+ * \brief A comment AST node.
+ */
+typedef struct {
+  const void *Data;
+} CXComment;
+
+/**
  * \defgroup CINDEX_CURSOR_MANIP Cursor manipulations
  *
  * @{
@@ -3170,12 +3177,402 @@ CINDEX_LINKAGE CXSourceRange clang_Cursor_getCommentRange(CXCursor C);
 CINDEX_LINKAGE CXString clang_Cursor_getRawCommentText(CXCursor C);
 
 /**
- * \brief Given a cursor that represents a declaration, return the associated
- * \\brief paragraph; otherwise return the first paragraph.
+ * \brief Given a cursor that represents a documentable entity (e.g.,
+ * declaration), return the associated \\brief paragraph; otherwise return the
+ * first paragraph.
  */
 CINDEX_LINKAGE CXString clang_Cursor_getBriefCommentText(CXCursor C);
 
 /**
+ * \brief Given a cursor that represents a documentable entity (e.g.,
+ * declaration), return the associated parsed comment as a
+ * \c CXComment_FullComment AST node.
+ */
+CINDEX_LINKAGE CXComment clang_Cursor_getParsedComment(CXCursor C);
+
+/**
+ * @}
+ */
+
+/**
+ * \defgroup CINDEX_COMMENT Comment AST introspection
+ *
+ * The routines in this group provide access to information in the
+ * documentation comment ASTs.
+ *
+ * @{
+ */
+
+/**
+ * \brief Describes the type of the comment AST node (\c CXComment).  A comment
+ * node can be considered block content (e. g., paragraph), inline content
+ * (plain text) or neither (the root AST node).
+ */
+enum CXCommentKind {
+  /**
+   * \brief Null comment.  No AST node is constructed at the requested location
+   * because there is no text or a syntax error.
+   */
+  CXComment_Null = 0,
+
+  /**
+   * \brief Plain text.  Inline content.
+   */
+  CXComment_Text = 1,
+
+  /**
+   * \brief A command with word-like arguments that is considered inline content.
+   *
+   * For example: \\c command.
+   */
+  CXComment_InlineCommand = 2,
+
+  /**
+   * \brief HTML start tag with attributes (name-value pairs).  Considered
+   * inline content.
+   *
+   * For example:
+   * \verbatim
+   * <br> <br /> <a href="http://example.org/">
+   * \endverbatim
+   */
+  CXComment_HTMLStartTag = 3,
+
+  /**
+   * \brief HTML end tag.  Considered inline content.
+   *
+   * For example:
+   * \verbatim
+   * </a>
+   * \endverbatim
+   */
+  CXComment_HTMLEndTag = 4,
+
+  /**
+   * \brief A paragraph, contains inline comment.  The paragraph itself is
+   * block content.
+   */
+  CXComment_Paragraph = 5,
+
+  /**
+   * \brief A command that has zero or more word-like arguments (number of
+   * word-like arguments depends on command name) and a paragraph as an
+   * argument.  Block command is block content.
+   *
+   * Paragraph argument is also a child of the block command.
+   *
+   * For example: \\brief has 0 word-like arguments and a paragraph argument.
+   *
+   * AST nodes of special kinds that parser knows about (e. g., \\param
+   * command) have their own node kinds.
+   */
+  CXComment_BlockCommand = 6,
+
+  /**
+   * \brief A \\param or \\arg command that describes the function parameter
+   * (name, passing direction, description).
+   *
+   * \brief For example: \\param [in] ParamName description.
+   */
+  CXComment_ParamCommand = 7,
+
+  /**
+   * \brief A verbatim block command (e. g., preformatted code).  Verbatim
+   * block has an opening and a closing command and contains multiple lines of
+   * text (\c CXComment_VerbatimBlockLine child nodes).
+   *
+   * For example:
+   * \\verbatim
+   * aaa
+   * \\endverbatim
+   */
+  CXComment_VerbatimBlockCommand = 8,
+
+  /**
+   * \brief A line of text that is contained within a
+   * CXComment_VerbatimBlockCommand node.
+   */
+  CXComment_VerbatimBlockLine = 9,
+
+  /**
+   * \brief A verbatim line command.  Verbatim line has an opening command,
+   * a single line of text (up to the newline after the opening command) and
+   * has no closing command.
+   */
+  CXComment_VerbatimLine = 10,
+
+  /**
+   * \brief A full comment attached to a declaration, contains block content.
+   */
+  CXComment_FullComment = 11
+};
+
+/**
+ * \brief Describes parameter passing direction for \\param or \\arg command.
+ */
+enum CXCommentParamPassDirection {
+  /**
+   * \brief The parameter is an input parameter.
+   */
+  CXCommentParamPassDirection_In,
+
+  /**
+   * \brief The parameter is an output parameter.
+   */
+  CXCommentParamPassDirection_Out,
+
+  /**
+   * \brief The parameter is an input and output parameter.
+   */
+  CXCommentParamPassDirection_InOut
+};
+
+/**
+ * \param Comment AST node of any kind.
+ *
+ * \returns the type of the AST node.
+ */
+CINDEX_LINKAGE enum CXCommentKind clang_Comment_getKind(CXComment Comment);
+
+/**
+ * \param Comment AST node of any kind.
+ *
+ * \returns number of children of the AST node.
+ */
+CINDEX_LINKAGE unsigned clang_Comment_getNumChildren(CXComment Comment);
+
+/**
+ * \param Comment AST node of any kind.
+ *
+ * \param ArgIdx argument index (zero-based).
+ *
+ * \returns the specified child of the AST node.
+ */
+CINDEX_LINKAGE
+CXComment clang_Comment_getChild(CXComment Comment, unsigned ChildIdx);
+
+/**
+ * \brief A \c CXComment_Paragraph node is considered whitespace if it contains
+ * only \c CXComment_Text nodes that are empty or whitespace.
+ *
+ * Other AST nodes (except \c CXComment_Paragraph and \c CXComment_Text) are
+ * never considered whitespace.
+ *
+ * \returns non-zero if \c Comment is whitespace.
+ */
+CINDEX_LINKAGE unsigned clang_Comment_isWhitespace(CXComment Comment);
+
+/**
+ * \returns non-zero if \c Comment is inline content and has a newline
+ * immediately following it in the comment text.  Newlines between paragraphs
+ * do not count.
+ */
+CINDEX_LINKAGE
+unsigned clang_InlineContentComment_hasTrailingNewline(CXComment Comment);
+
+/**
+ * \param Comment a \c CXComment_Text AST node.
+ *
+ * \returns text contained in the AST node.
+ */
+CINDEX_LINKAGE CXString clang_TextComment_getText(CXComment Comment);
+
+/**
+ * \param Comment a \c CXComment_InlineCommand AST node.
+ *
+ * \returns name of the inline command.
+ */
+CINDEX_LINKAGE
+CXString clang_InlineCommandComment_getCommandName(CXComment Comment);
+
+/**
+ * \param Comment a \c CXComment_InlineCommand AST node.
+ *
+ * \returns number of command arguments.
+ */
+CINDEX_LINKAGE
+unsigned clang_InlineCommandComment_getNumArgs(CXComment Comment);
+
+/**
+ * \param Comment a \c CXComment_InlineCommand AST node.
+ *
+ * \param ArgIdx argument index (zero-based).
+ *
+ * \returns text of the specified argument.
+ */
+CINDEX_LINKAGE
+CXString clang_InlineCommandComment_getArgText(CXComment Comment,
+                                               unsigned ArgIdx);
+
+/**
+ * \param Comment a \c CXComment_HTMLStartTag or \c CXComment_HTMLEndTag AST
+ * node.
+ *
+ * \returns HTML tag name.
+ */
+CINDEX_LINKAGE CXString clang_HTMLTagComment_getTagName(CXComment Comment);
+
+/**
+ * \param Comment a \c CXComment_HTMLStartTag AST node.
+ *
+ * \returns non-zero if tag is self-closing (for example, &lt;br /&gt;).
+ */
+CINDEX_LINKAGE
+unsigned clang_HTMLStartTagComment_isSelfClosing(CXComment Comment);
+
+/**
+ * \param Comment a \c CXComment_HTMLStartTag AST node.
+ *
+ * \returns number of attributes (name-value pairs) attached to the start tag.
+ */
+CINDEX_LINKAGE unsigned clang_HTMLStartTag_getNumAttrs(CXComment Comment);
+
+/**
+ * \param Comment a \c CXComment_HTMLStartTag AST node.
+ *
+ * \param AttrIdx attribute index (zero-based).
+ *
+ * \returns name of the specified attribute.
+ */
+CINDEX_LINKAGE
+CXString clang_HTMLStartTag_getAttrName(CXComment Comment, unsigned AttrIdx);
+
+/**
+ * \param Comment a \c CXComment_HTMLStartTag AST node.
+ *
+ * \param AttrIdx attribute index (zero-based).
+ *
+ * \returns value of the specified attribute.
+ */
+CINDEX_LINKAGE
+CXString clang_HTMLStartTag_getAttrValue(CXComment Comment, unsigned AttrIdx);
+
+/**
+ * \param Comment a \c CXComment_BlockCommand AST node.
+ *
+ * \returns name of the block command.
+ */
+CINDEX_LINKAGE
+CXString clang_BlockCommandComment_getCommandName(CXComment Comment);
+
+/**
+ * \param Comment a \c CXComment_BlockCommand AST node.
+ *
+ * \returns number of word-like arguments.
+ */
+CINDEX_LINKAGE
+unsigned clang_BlockCommandComment_getNumArgs(CXComment Comment);
+
+/**
+ * \param Comment a \c CXComment_BlockCommand AST node.
+ *
+ * \param ArgIdx argument index (zero-based).
+ *
+ * \returns text of the specified word-like argument.
+ */
+CINDEX_LINKAGE
+CXString clang_BlockCommandComment_getArgText(CXComment Comment,
+                                              unsigned ArgIdx);
+
+/**
+ * \param Comment a \c CXComment_BlockCommand or
+ * \c CXComment_VerbatimBlockCommand AST node.
+ *
+ * \returns paragraph argument of the block command.
+ */
+CINDEX_LINKAGE
+CXComment clang_BlockCommandComment_getParagraph(CXComment Comment);
+
+/**
+ * \param Comment a \c CXComment_ParamCommand AST node.
+ *
+ * \returns parameter name.
+ */
+CINDEX_LINKAGE
+CXString clang_ParamCommandComment_getParamName(CXComment Comment);
+
+/**
+ * \param Comment a \c CXComment_ParamCommand AST node.
+ *
+ * \returns non-zero if the parameter that this AST node represents was found
+ * in the function prototype and \c clang_ParamCommandComment_getParamIndex
+ * function will return a meaningful value.
+ */
+CINDEX_LINKAGE
+unsigned clang_ParamCommandComment_isParamIndexValid(CXComment Comment);
+
+/**
+ * \param Comment a \c CXComment_ParamCommand AST node.
+ *
+ * \returns zero-based parameter index in function prototype.
+ */
+CINDEX_LINKAGE
+unsigned clang_ParamCommandComment_getParamIndex(CXComment Comment);
+
+/**
+ * \param Comment a \c CXComment_ParamCommand AST node.
+ *
+ * \returns non-zero if parameter passing direction was specified explicitly in
+ * the comment.
+ */
+CINDEX_LINKAGE
+unsigned clang_ParamCommandComment_isDirectionExplicit(CXComment Comment);
+
+/**
+ * \param Comment a \c CXComment_ParamCommand AST node.
+ *
+ * \returns parameter passing direction.
+ */
+CINDEX_LINKAGE
+enum CXCommentParamPassDirection clang_ParamCommandComment_getDirection(
+                                                            CXComment Comment);
+
+/**
+ * \param Comment a \c CXComment_VerbatimBlockLine AST node.
+ *
+ * \returns text contained in the AST node.
+ */
+CINDEX_LINKAGE
+CXString clang_VerbatimBlockLineComment_getText(CXComment Comment);
+
+/**
+ * \param Comment a \c CXComment_VerbatimLine AST node.
+ *
+ * \returns text contained in the AST node.
+ */
+CINDEX_LINKAGE CXString clang_VerbatimLineComment_getText(CXComment Comment);
+
+/**
+ * \brief Convert an HTML tag AST node to string.
+ *
+ * \param Comment a \c CXComment_HTMLStartTag or \c CXComment_HTMLEndTag AST
+ * node.
+ *
+ * \returns string containing an HTML tag.
+ */
+CINDEX_LINKAGE CXString clang_HTMLTagComment_getAsString(CXComment Comment);
+
+/**
+ * \brief Convert a given full parsed comment to an HTML fragment.
+ *
+ * Specific details of HTML layout are subject to change.  Don't try to parse
+ * this HTML back into an AST, use other APIs instead.
+ *
+ * Currently the following CSS classes are used:
+ * \li "para-brief" for \\brief paragraph and equivalent commands;
+ * \li "para-returns" for \\returns paragraph and equivalent commands;
+ * \li "word-returns" for the "Returns" word in \\returns paragraph.
+ *
+ * Argument list is rendered as \<dl\> list with arguments sorted in function
+ * prototype order.
+ *
+ * \param Comment a \c CXComment_FullComment AST node.
+ *
+ * \returns string containing an HTML fragment.
+ */
+CINDEX_LINKAGE CXString clang_FullComment_getAsHTML(CXComment Comment);
+
+/**
  * @}
  */
 
diff --git a/include/clang/AST/Comment.h b/include/clang/AST/Comment.h
index f59b5e504a..3eee704be0 100644
--- a/include/clang/AST/Comment.h
+++ b/include/clang/AST/Comment.h
@@ -613,6 +613,10 @@ public:
     return Paragraph;
   }
 
+  bool hasNonWhitespaceParagraph() const {
+    return Paragraph && !Paragraph->isWhitespace();
+  }
+
   void setParagraph(ParagraphComment *PC) {
     Paragraph = PC;
     SourceLocation NewLocEnd = PC->getLocEnd();
diff --git a/include/clang/AST/CommentVisitor.h b/include/clang/AST/CommentVisitor.h
index 159725af26..47867a634f 100644
--- a/include/clang/AST/CommentVisitor.h
+++ b/include/clang/AST/CommentVisitor.h
@@ -24,6 +24,9 @@ public:
  return static_cast<ImplClass*>(this)->visit ## NAME(static_cast<PTR(CLASS)>(C))
 
   RetTy visit(PTR(Comment) C) {
+    if (!C)
+      return RetTy();
+
     switch (C->getCommentKind()) {
     default: llvm_unreachable("Unknown comment kind!");
 #define ABSTRACT_COMMENT(COMMENT)
diff --git a/test/Index/annotate-comments.cpp b/test/Index/annotate-comments.cpp
index afce06a1da..9ed1c30c49 100644
--- a/test/Index/annotate-comments.cpp
+++ b/test/Index/annotate-comments.cpp
@@ -221,6 +221,100 @@ void isdoxy49(void);
 /// \returns ddd IS_DOXYGEN_END
 void isdoxy50(int);
 
+/// Aaa.
+void comment_to_html_conversion_1();
+
+/// \brief Aaa.
+void comment_to_html_conversion_2();
+
+/// \short Aaa.
+void comment_to_html_conversion_3();
+
+/// Aaa.
+///
+/// \brief Bbb.
+void comment_to_html_conversion_4();
+
+/// Aaa.
+///
+/// \brief Bbb.
+///
+/// Ccc.
+void comment_to_html_conversion_5();
+
+/// \brief Aaa.
+/// \brief Bbb.
+void comment_to_html_conversion_6();
+
+/// Aaa.
+///
+/// \return Bbb.
+void comment_to_html_conversion_7();
+
+/// Aaa.
+///
+/// \returns Bbb.
+void comment_to_html_conversion_8();
+
+/// \returns Aaa.
+/// \returns Bbb.
+void comment_to_html_conversion_9();
+
+/// Aaa.
+///
+/// Bbb.
+///
+/// \returns Ccc.
+void comment_to_html_conversion_10();
+
+/// \param x1 Aaa.
+void comment_to_html_conversion_11(int x1);
+
+/// \param zzz Aaa.
+void comment_to_html_conversion_12(int x1);
+
+/// \param x2 Bbb.
+/// \param x1 Aaa.
+void comment_to_html_conversion_13(int x1, int x2);
+
+/// \param x2 Bbb.
+/// \param zzz Aaa.
+/// \param x1 Aaa.
+void comment_to_html_conversion_14(int x1, int x2);
+
+/// \brief Aaa.
+///
+/// Bbb.
+///
+/// \param x2 Ddd.
+/// \param x1 Ccc.
+/// \returns Eee.
+void comment_to_html_conversion_15(int x1, int x2);
+
+/// <br><a href="http://example.com/">Aaa</a>
+void comment_to_html_conversion_16();
+
+/// \verbatim
+/// <a href="http://example.com/">Aaa</a>
+/// <a href='http://example.com/'>Aaa</a>
+/// \endverbatim
+void comment_to_html_conversion_17();
+
+/// \b Aaa
+void comment_to_html_conversion_18();
+
+/// \c Aaa \p Bbb
+void comment_to_html_conversion_19();
+
+/// \a Aaa \e Bbb \em Ccc
+void comment_to_html_conversion_20();
+
+/// \\ \@ \& \$ \# \< \> \% \" \. \::
+void comment_to_html_conversion_21();
+
+/// &amp; &lt; &gt; &quot;
+void comment_to_html_conversion_22();
+
 #endif
 
 // RUN: rm -rf %t
@@ -290,3 +384,242 @@ void isdoxy50(int);
 // CHECK: annotate-comments.cpp:218:6: FunctionDecl=isdoxy49:{{.*}} BriefComment=[IS_DOXYGEN_START Aaa]
 // CHECK: annotate-comments.cpp:222:6: FunctionDecl=isdoxy50:{{.*}} BriefComment=[Returns ddd IS_DOXYGEN_END]
 
+// CHECK: annotate-comments.cpp:225:6: FunctionDecl=comment_to_html_conversion_1:{{.*}} FullCommentAsHTML=[<p class="para-brief"> Aaa.</p>]
+// CHECK-NEXT:  CommentAST=[
+// CHECK-NEXT:    (CXComment_FullComment
+// CHECK-NEXT:       (CXComment_Paragraph
+// CHECK-NEXT:         (CXComment_Text Text=[ Aaa.])))]
+// CHECK: annotate-comments.cpp:228:6: FunctionDecl=comment_to_html_conversion_2:{{.*}} FullCommentAsHTML=[<p class="para-brief"> Aaa.</p>]
+// CHECK-NEXT:  CommentAST=[
+// CHECK-NEXT:    (CXComment_FullComment
+// CHECK-NEXT:       (CXComment_Paragraph IsWhitespace
+// CHECK-NEXT:         (CXComment_Text Text=[ ] IsWhitespace))
+// CHECK-NEXT:       (CXComment_BlockCommand CommandName=[brief]
+// CHECK-NEXT:         (CXComment_Paragraph
+// CHECK-NEXT:           (CXComment_Text Text=[ Aaa.]))))]
+// CHECK: annotate-comments.cpp:231:6: FunctionDecl=comment_to_html_conversion_3:{{.*}} FullCommentAsHTML=[<p class="para-brief"> Aaa.</p>]
+// CHECK-NEXT:  CommentAST=[
+// CHECK-NEXT:    (CXComment_FullComment
+// CHECK-NEXT:       (CXComment_Paragraph IsWhitespace
+// CHECK-NEXT:         (CXComment_Text Text=[ ] IsWhitespace))
+// CHECK-NEXT:       (CXComment_BlockCommand CommandName=[short]
+// CHECK-NEXT:         (CXComment_Paragraph
+// CHECK-NEXT:           (CXComment_Text Text=[ Aaa.]))))]
+// CHECK: annotate-comments.cpp:236:6: FunctionDecl=comment_to_html_conversion_4:{{.*}} FullCommentAsHTML=[<p class="para-brief"> Bbb.</p><p> Aaa.</p>]
+// CHECK-NEXT:  CommentAST=[
+// CHECK-NEXT:    (CXComment_FullComment
+// CHECK-NEXT:       (CXComment_Paragraph
+// CHECK-NEXT:         (CXComment_Text Text=[ Aaa.]))
+// CHECK-NEXT:       (CXComment_Paragraph IsWhitespace
+// CHECK-NEXT:         (CXComment_Text Text=[ ] IsWhitespace))
+// CHECK-NEXT:       (CXComment_BlockCommand CommandName=[brief]
+// CHECK-NEXT:         (CXComment_Paragraph
+// CHECK-NEXT:           (CXComment_Text Text=[ Bbb.]))))]
+// CHECK: annotate-comments.cpp:243:6: FunctionDecl=comment_to_html_conversion_5:{{.*}} FullCommentAsHTML=[<p class="para-brief"> Bbb.</p><p> Aaa.</p><p> Ccc.</p>]
+// CHECK-NEXT:  CommentAST=[
+// CHECK-NEXT:    (CXComment_FullComment
+// CHECK-NEXT:       (CXComment_Paragraph
+// CHECK-NEXT:         (CXComment_Text Text=[ Aaa.]))
+// CHECK-NEXT:       (CXComment_Paragraph IsWhitespace
+// CHECK-NEXT:         (CXComment_Text Text=[ ] IsWhitespace))
+// CHECK-NEXT:       (CXComment_BlockCommand CommandName=[brief]
+// CHECK-NEXT:         (CXComment_Paragraph
+// CHECK-NEXT:           (CXComment_Text Text=[ Bbb.])))
+// CHECK-NEXT:       (CXComment_Paragraph
+// CHECK-NEXT:         (CXComment_Text Text=[ Ccc.])))]
+// CHECK: annotate-comments.cpp:247:6: FunctionDecl=comment_to_html_conversion_6:{{.*}} FullCommentAsHTML=[<p class="para-brief"> Aaa. </p><p class="para-brief"> Bbb.</p>]
+// CHECK-NEXT:  CommentAST=[
+// CHECK-NEXT:    (CXComment_FullComment
+// CHECK-NEXT:       (CXComment_Paragraph IsWhitespace
+// CHECK-NEXT:         (CXComment_Text Text=[ ] IsWhitespace))
+// CHECK-NEXT:       (CXComment_BlockCommand CommandName=[brief]
+// CHECK-NEXT:         (CXComment_Paragraph
+// CHECK-NEXT:           (CXComment_Text Text=[ Aaa.] HasTrailingNewline)
+// CHECK-NEXT:           (CXComment_Text Text=[ ] IsWhitespace)))
+// CHECK-NEXT:       (CXComment_BlockCommand CommandName=[brief]
+// CHECK-NEXT:         (CXComment_Paragraph
+// CHECK-NEXT:           (CXComment_Text Text=[ Bbb.]))))]
+// CHECK: annotate-comments.cpp:252:6: FunctionDecl=comment_to_html_conversion_7:{{.*}} FullCommentAsHTML=[<p class="para-brief"> Aaa.</p><p class="para-returns"><span class="word-returns">Returns</span>  Bbb.</p>]
+// CHECK-NEXT:  CommentAST=[
+// CHECK-NEXT:    (CXComment_FullComment
+// CHECK-NEXT:       (CXComment_Paragraph
+// CHECK-NEXT:         (CXComment_Text Text=[ Aaa.]))
+// CHECK-NEXT:       (CXComment_Paragraph IsWhitespace
+// CHECK-NEXT:         (CXComment_Text Text=[ ] IsWhitespace))
+// CHECK-NEXT:       (CXComment_BlockCommand CommandName=[return]
+// CHECK-NEXT:         (CXComment_Paragraph
+// CHECK-NEXT:           (CXComment_Text Text=[ Bbb.]))))]
+// CHECK: annotate-comments.cpp:257:6: FunctionDecl=comment_to_html_conversion_8:{{.*}} FullCommentAsHTML=[<p class="para-brief"> Aaa.</p><p class="para-returns"><span class="word-returns">Returns</span>  Bbb.</p>]
+// CHECK-NEXT:  CommentAST=[
+// CHECK-NEXT:    (CXComment_FullComment
+// CHECK-NEXT:       (CXComment_Paragraph
+// CHECK-NEXT:         (CXComment_Text Text=[ Aaa.]))
+// CHECK-NEXT:       (CXComment_Paragraph IsWhitespace
+// CHECK-NEXT:         (CXComment_Text Text=[ ] IsWhitespace))
+// CHECK-NEXT:       (CXComment_BlockCommand CommandName=[returns]
+// CHECK-NEXT:         (CXComment_Paragraph
+// CHECK-NEXT:           (CXComment_Text Text=[ Bbb.]))))]
+// CHECK: annotate-comments.cpp:261:6: FunctionDecl=comment_to_html_conversion_9:{{.*}} FullCommentAsHTML=[<p class="para-returns"><span class="word-returns">Returns</span>  Bbb.</p><p class="para-returns"><span class="word-returns">Returns</span>  Aaa. </p>]
+// CHECK-NEXT:  CommentAST=[
+// CHECK-NEXT:    (CXComment_FullComment
+// CHECK-NEXT:       (CXComment_Paragraph IsWhitespace
+// CHECK-NEXT:         (CXComment_Text Text=[ ] IsWhitespace))
+// CHECK-NEXT:       (CXComment_BlockCommand CommandName=[returns]
+// CHECK-NEXT:         (CXComment_Paragraph
+// CHECK-NEXT:           (CXComment_Text Text=[ Aaa.] HasTrailingNewline)
+// CHECK-NEXT:           (CXComment_Text Text=[ ] IsWhitespace)))
+// CHECK-NEXT:       (CXComment_BlockCommand CommandName=[returns]
+// CHECK-NEXT:         (CXComment_Paragraph
+// CHECK-NEXT:           (CXComment_Text Text=[ Bbb.]))))]
+// CHECK: annotate-comments.cpp:268:6: FunctionDecl=comment_to_html_conversion_10:{{.*}} FullCommentAsHTML=[<p class="para-brief"> Aaa.</p><p> Bbb.</p><p class="para-returns"><span class="word-returns">Returns</span>  Ccc.</p>]
+// CHECK-NEXT:  CommentAST=[
+// CHECK-NEXT:    (CXComment_FullComment
+// CHECK-NEXT:       (CXComment_Paragraph
+// CHECK-NEXT:         (CXComment_Text Text=[ Aaa.]))
+// CHECK-NEXT:       (CXComment_Paragraph
+// CHECK-NEXT:         (CXComment_Text Text=[ Bbb.]))
+// CHECK-NEXT:       (CXComment_Paragraph IsWhitespace
+// CHECK-NEXT:         (CXComment_Text Text=[ ] IsWhitespace))
+// CHECK-NEXT:       (CXComment_BlockCommand CommandName=[returns]
+// CHECK-NEXT:         (CXComment_Paragraph
+// CHECK-NEXT:           (CXComment_Text Text=[ Ccc.]))))]
+// CHECK: annotate-comments.cpp:271:6: FunctionDecl=comment_to_html_conversion_11:{{.*}} FullCommentAsHTML=[<dl><dt>x1</dt><dd> Aaa.</dd></dl>]
+// CHECK-NEXT:  CommentAST=[
+// CHECK-NEXT:    (CXComment_FullComment
+// CHECK-NEXT:       (CXComment_Paragraph IsWhitespace
+// CHECK-NEXT:         (CXComment_Text Text=[ ] IsWhitespace))
+// CHECK-NEXT:       (CXComment_ParamCommand in implicitly ParamName=[x1] ParamIndex=0
+// CHECK-NEXT:         (CXComment_Paragraph
+// CHECK-NEXT:           (CXComment_Text Text=[ Aaa.]))))]
+// CHECK: annotate-comments.cpp:274:6: FunctionDecl=comment_to_html_conversion_12:{{.*}} FullCommentAsHTML=[<dl><dt>zzz</dt><dd> Aaa.</dd></dl>]
+// CHECK-NEXT:  CommentAST=[
+// CHECK-NEXT:    (CXComment_FullComment
+// CHECK-NEXT:       (CXComment_Paragraph IsWhitespace
+// CHECK-NEXT:         (CXComment_Text Text=[ ] IsWhitespace))
+// CHECK-NEXT:       (CXComment_ParamCommand in implicitly ParamName=[zzz] ParamIndex=Invalid
+// CHECK-NEXT:         (CXComment_Paragraph
+// CHECK-NEXT:           (CXComment_Text Text=[ Aaa.]))))]
+// CHECK: annotate-comments.cpp:278:6: FunctionDecl=comment_to_html_conversion_13:{{.*}} FullCommentAsHTML=[<dl><dt>x1</dt><dd> Aaa.</dd><dt>x2</dt><dd> Bbb. </dd></dl>]
+// CHECK-NEXT:  CommentAST=[
+// CHECK-NEXT:    (CXComment_FullComment
+// CHECK-NEXT:       (CXComment_Paragraph IsWhitespace
+// CHECK-NEXT:         (CXComment_Text Text=[ ] IsWhitespace))
+// CHECK-NEXT:       (CXComment_ParamCommand in implicitly ParamName=[x2] ParamIndex=1
+// CHECK-NEXT:         (CXComment_Paragraph
+// CHECK-NEXT:           (CXComment_Text Text=[ Bbb.] HasTrailingNewline)
+// CHECK-NEXT:           (CXComment_Text Text=[ ] IsWhitespace)))
+// CHECK-NEXT:       (CXComment_ParamCommand in implicitly ParamName=[x1] ParamIndex=0
+// CHECK-NEXT:         (CXComment_Paragraph
+// CHECK-NEXT:           (CXComment_Text Text=[ Aaa.]))))]
+// CHECK: annotate-comments.cpp:283:6: FunctionDecl=comment_to_html_conversion_14:{{.*}} FullCommentAsHTML=[<dl><dt>x1</dt><dd> Aaa.</dd><dt>x2</dt><dd> Bbb. </dd><dt>zzz</dt><dd> Aaa. </dd></dl>]
+// CHECK-NEXT:  CommentAST=[
+// CHECK-NEXT:    (CXComment_FullComment
+// CHECK-NEXT:       (CXComment_Paragraph IsWhitespace
+// CHECK-NEXT:         (CXComment_Text Text=[ ] IsWhitespace))
+// CHECK-NEXT:       (CXComment_ParamCommand in implicitly ParamName=[x2] ParamIndex=1
+// CHECK-NEXT:         (CXComment_Paragraph
+// CHECK-NEXT:           (CXComment_Text Text=[ Bbb.] HasTrailingNewline)
+// CHECK-NEXT:           (CXComment_Text Text=[ ] IsWhitespace)))
+// CHECK-NEXT:       (CXComment_ParamCommand in implicitly ParamName=[zzz] ParamIndex=Invalid
+// CHECK-NEXT:         (CXComment_Paragraph
+// CHECK-NEXT:           (CXComment_Text Text=[ Aaa.] HasTrailingNewline)
+// CHECK-NEXT:           (CXComment_Text Text=[ ] IsWhitespace)))
+// CHECK-NEXT:       (CXComment_ParamCommand in implicitly ParamName=[x1] ParamIndex=0
+// CHECK-NEXT:         (CXComment_Paragraph
+// CHECK-NEXT:           (CXComment_Text Text=[ Aaa.]))))]
+// CHECK: annotate-comments.cpp:292:6: FunctionDecl=comment_to_html_conversion_15:{{.*}} FullCommentAsHTML=[<p class="para-brief"> Aaa.</p><p> Bbb.</p><dl><dt>x1</dt><dd> Ccc. </dd><dt>x2</dt><dd> Ddd. </dd></dl><p class="para-returns"><span class="word-returns">Returns</span>  Eee.</p>]
+// CHECK-NEXT:  CommentAST=[
+// CHECK-NEXT:    (CXComment_FullComment
+// CHECK-NEXT:       (CXComment_Paragraph IsWhitespace
+// CHECK-NEXT:         (CXComment_Text Text=[ ] IsWhitespace))
+// CHECK-NEXT:       (CXComment_BlockCommand CommandName=[brief]
+// CHECK-NEXT:         (CXComment_Paragraph
+// CHECK-NEXT:           (CXComment_Text Text=[ Aaa.])))
+// CHECK-NEXT:       (CXComment_Paragraph
+// CHECK-NEXT:         (CXComment_Text Text=[ Bbb.]))
+// CHECK-NEXT:       (CXComment_Paragraph IsWhitespace
+// CHECK-NEXT:         (CXComment_Text Text=[ ] IsWhitespace))
+// CHECK-NEXT:       (CXComment_ParamCommand in implicitly ParamName=[x2] ParamIndex=1
+// CHECK-NEXT:         (CXComment_Paragraph
+// CHECK-NEXT:           (CXComment_Text Text=[ Ddd.] HasTrailingNewline)
+// CHECK-NEXT:           (CXComment_Text Text=[ ] IsWhitespace)))
+// CHECK-NEXT:       (CXComment_ParamCommand in implicitly ParamName=[x1] ParamIndex=0
+// CHECK-NEXT:         (CXComment_Paragraph
+// CHECK-NEXT:           (CXComment_Text Text=[ Ccc.] HasTrailingNewline)
+// CHECK-NEXT:           (CXComment_Text Text=[ ] IsWhitespace)))
+// CHECK-NEXT:       (CXComment_BlockCommand CommandName=[returns]
+// CHECK-NEXT:         (CXComment_Paragraph
+// CHECK-NEXT:           (CXComment_Text Text=[ Eee.]))))]
+// CHECK: annotate-comments.cpp:295:6: FunctionDecl=comment_to_html_conversion_16:{{.*}} FullCommentAsHTML=[<p class="para-brief"> <br><a href="http://example.com/ ">Aaa</a></p>]
+// CHECK-NEXT:  CommentAST=[
+// CHECK-NEXT:    (CXComment_FullComment
+// CHECK-NEXT:       (CXComment_Paragraph
+// CHECK-NEXT:         (CXComment_Text Text=[ ] IsWhitespace)
+// CHECK-NEXT:         (CXComment_HTMLStartTag Name=[br])
+// CHECK-NEXT:         (CXComment_HTMLStartTag Name=[a] Attrs: href=http://example.com/)
+// CHECK-NEXT:         (CXComment_Text Text=[Aaa])
+// CHECK-NEXT:         (CXComment_HTMLEndTag Name=[a])))]
+// CHECK: annotate-comments.cpp:301:6: FunctionDecl=comment_to_html_conversion_17:{{.*}} FullCommentAsHTML=[<pre> &lt;a href=&quot;http:&#47;&#47;example.com&#47;&quot;&gt;Aaa&lt;&#47;a&gt;\n &lt;a href=&#39;http:&#47;&#47;example.com&#47;&#39;&gt;Aaa&lt;&#47;a&gt;</pre>]
+// CHECK-NEXT:  CommentAST=[
+// CHECK-NEXT:    (CXComment_FullComment
+// CHECK-NEXT:       (CXComment_Paragraph IsWhitespace
+// CHECK-NEXT:         (CXComment_Text Text=[ ] IsWhitespace))
+// CHECK-NEXT:       (CXComment_VerbatimBlockCommand CommandName=[verbatim]
+// CHECK-NEXT:         (CXComment_VerbatimBlockLine Text=[ <a href="http://example.com/">Aaa</a>])
+// CHECK-NEXT:         (CXComment_VerbatimBlockLine Text=[ <a href='http://example.com/'>Aaa</a>])))]
+// CHECK: annotate-comments.cpp:304:6: FunctionDecl=comment_to_html_conversion_18:{{.*}} FullCommentAsHTML=[<p class="para-brief"> <b>Aaa</b></p>]
+// CHECK-NEXT:  CommentAST=[
+// CHECK-NEXT:    (CXComment_FullComment
+// CHECK-NEXT:       (CXComment_Paragraph
+// CHECK-NEXT:         (CXComment_Text Text=[ ] IsWhitespace)
+// CHECK-NEXT:         (CXComment_InlineCommand CommandName=[b] Arg[0]=Aaa)))]
+// CHECK: annotate-comments.cpp:307:6: FunctionDecl=comment_to_html_conversion_19:{{.*}} FullCommentAsHTML=[<p class="para-brief"> <tt>Aaa</tt> <tt>Bbb</tt></p>]
+// CHECK-NEXT:  CommentAST=[
+// CHECK-NEXT:    (CXComment_FullComment
+// CHECK-NEXT:       (CXComment_Paragraph
+// CHECK-NEXT:         (CXComment_Text Text=[ ] IsWhitespace)
+// CHECK-NEXT:         (CXComment_InlineCommand CommandName=[c] Arg[0]=Aaa)
+// CHECK-NEXT:         (CXComment_Text Text=[ ] IsWhitespace)
+// CHECK-NEXT:         (CXComment_InlineCommand CommandName=[p] Arg[0]=Bbb)))]
+// CHECK: annotate-comments.cpp:310:6: FunctionDecl=comment_to_html_conversion_20:{{.*}} FullCommentAsHTML=[<p class="para-brief"> <em>Aaa</em> <em>Bbb</em> <em>Ccc</em></p>]
+// CHECK-NEXT:  CommentAST=[
+// CHECK-NEXT:    (CXComment_FullComment
+// CHECK-NEXT:       (CXComment_Paragraph
+// CHECK-NEXT:         (CXComment_Text Text=[ ] IsWhitespace)
+// CHECK-NEXT:         (CXComment_InlineCommand CommandName=[a] Arg[0]=Aaa)
+// CHECK-NEXT:         (CXComment_Text Text=[ ] IsWhitespace)
+// CHECK-NEXT:         (CXComment_InlineCommand CommandName=[e] Arg[0]=Bbb)
+// CHECK-NEXT:         (CXComment_Text Text=[ ] IsWhitespace)
+// CHECK-NEXT:         (CXComment_InlineCommand CommandName=[em] Arg[0]=Ccc)))]
+// CHECK: annotate-comments.cpp:313:6: FunctionDecl=comment_to_html_conversion_21:{{.*}} FullCommentAsHTML=[<p class="para-brief"> \ @ &amp; $ # &lt; &gt; % &quot; . ::</p>]
+// CHECK-NEXT:  CommentAST=[
+// CHECK-NEXT:    (CXComment_FullComment
+// CHECK-NEXT:       (CXComment_Paragraph
+// CHECK-NEXT:         (CXComment_Text Text=[ ] IsWhitespace)
+// CHECK-NEXT:         (CXComment_Text Text=[\])
+// CHECK-NEXT:         (CXComment_Text Text=[ ] IsWhitespace)
+// CHECK-NEXT:         (CXComment_Text Text=[@])
+// CHECK-NEXT:         (CXComment_Text Text=[ ] IsWhitespace)
+// CHECK-NEXT:         (CXComment_Text Text=[&])
+// CHECK-NEXT:         (CXComment_Text Text=[ ] IsWhitespace)
+// CHECK-NEXT:         (CXComment_Text Text=[$])
+// CHECK-NEXT:         (CXComment_Text Text=[ ] IsWhitespace)
+// CHECK-NEXT:         (CXComment_Text Text=[#])
+// CHECK-NEXT:         (CXComment_Text Text=[ ] IsWhitespace)
+// CHECK-NEXT:         (CXComment_Text Text=[<])
+// CHECK-NEXT:         (CXComment_Text Text=[ ] IsWhitespace)
+// CHECK-NEXT:         (CXComment_Text Text=[>])
+// CHECK-NEXT:         (CXComment_Text Text=[ ] IsWhitespace)
+// CHECK-NEXT:         (CXComment_Text Text=[%])
+// CHECK-NEXT:         (CXComment_Text Text=[ ] IsWhitespace)
+// CHECK-NEXT:         (CXComment_Text Text=["])
+// CHECK-NEXT:         (CXComment_Text Text=[ ] IsWhitespace)
+// CHECK-NEXT:         (CXComment_Text Text=[.])
+// CHECK-NEXT:         (CXComment_Text Text=[ ] IsWhitespace)
+// CHECK-NEXT:         (CXComment_Text Text=[::])))]
+// CHECK: annotate-comments.cpp:316:6: FunctionDecl=comment_to_html_conversion_22:{{.*}} FullCommentAsHTML=[<p class="para-brief"> &amp;amp; &amp;lt; &amp;gt; &amp;quot;</p>]
+// CHECK:  CommentAST=[
+// CHECK:    (CXComment_FullComment
+// CHECK:       (CXComment_Paragraph
+// CHECK:         (CXComment_Text Text=[ &amp; &lt; &gt; &quot;])))]
+
diff --git a/tools/c-index-test/c-index-test.c b/tools/c-index-test/c-index-test.c
index b3b5b8447f..bcfbece3ff 100644
--- a/tools/c-index-test/c-index-test.c
+++ b/tools/c-index-test/c-index-test.c
@@ -183,8 +183,9 @@ int parse_remapped_files(int argc, const char **argv, int start_arg,
 /* Pretty-printing.                                                           */
 /******************************************************************************/
 
-static void PrintCString(const char *Prefix, const char *CStr) {
-  printf(" %s=[", Prefix);
+static const char *FileCheckPrefix = "CHECK";
+
+static void PrintCString(const char *CStr) {
   if (CStr != NULL && CStr[0] != '\0') {
     for ( ; *CStr; ++CStr) {
       const char C = *CStr;
@@ -198,9 +199,25 @@ static void PrintCString(const char *Prefix, const char *CStr) {
       }
     }
   }
+}
+
+static void PrintCStringWithPrefix(const char *Prefix, const char *CStr) {
+  printf(" %s=[", Prefix);
+  PrintCString(CStr);
   printf("]");
 }
 
+static void PrintCXStringAndDispose(CXString Str) {
+  PrintCString(clang_getCString(Str));
+  clang_disposeString(Str);
+}
+
+static void PrintCXStringWithPrefixAndDispose(const char *Prefix,
+                                              CXString Str) {
+  PrintCStringWithPrefix(Prefix, clang_getCString(Str));
+  clang_disposeString(Str);
+}
+
 static void PrintRange(CXSourceRange R, const char *str) {
   CXFile begin_file, end_file;
   unsigned begin_line, begin_column, end_line, end_column;
@@ -233,6 +250,188 @@ static void printVersion(const char *Prefix, CXVersion Version) {
   printf(".%d", Version.Subminor);
 }
 
+struct CommentASTDumpingContext {
+  int IndentLevel;
+};
+
+static void DumpCXCommentInternal(struct CommentASTDumpingContext *Ctx,
+                                  CXComment Comment) {
+  Ctx->IndentLevel++;
+  for (unsigned i = 0, e = Ctx->IndentLevel; i != e; ++i)
+    printf("  ");
+
+  printf("(");
+  enum CXCommentKind Kind = clang_Comment_getKind(Comment);
+  switch (Kind) {
+  case CXComment_Null:
+    printf("CXComment_Null");
+    break;
+  case CXComment_Text:
+    printf("CXComment_Text");
+    PrintCXStringWithPrefixAndDispose("Text",
+                                      clang_TextComment_getText(Comment));
+    if (clang_Comment_isWhitespace(Comment))
+      printf(" IsWhitespace");
+    if (clang_InlineContentComment_hasTrailingNewline(Comment))
+      printf(" HasTrailingNewline");
+    break;
+  case CXComment_InlineCommand:
+    printf("CXComment_InlineCommand");