Comment parsing: improve the fidelity of XML output for many block commands

This change introduces a 'kind' attribute for the <Para> tag, that captures the
kind of the parent block command.

For example:

\todo Meow.

used to be just <Para>Meow.</Para>, but now it is
<Para kind="todo">Meow.</Para>


git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@174216 91177308-0d34-0410-b5e6-96231b3b80d8

15 files changed, 213 insertions, 4 deletions

diff --git a/bindings/xml/comment-xml-schema.rng b/bindings/xml/comment-xml-schema.rng
index ab73707b08..39ba07af5a 100644
--- a/bindings/xml/comment-xml-schema.rng
+++ b/bindings/xml/comment-xml-schema.rng
@@ -499,6 +499,29 @@
   <define name="TextBlockContent">
     <choice>
       <element name="Para">
+        <optional>
+          <attribute name="kind">
+            <choice>
+              <value>author</value>
+              <value>authors</value>
+              <value>bug</value>
+              <value>copyright</value>
+              <value>date</value>
+              <value>invariant</value>
+              <value>note</value>
+              <value>post</value>
+              <value>pre</value>
+              <value>remark</value>
+              <value>remarks</value>
+              <value>sa</value>
+              <value>see</value>
+              <value>since</value>
+              <value>todo</value>
+              <value>version</value>
+              <value>warning</value>
+            </choice>
+          </attribute>
+        </optional>
         <zeroOrMore>
           <ref name="TextInlineContent" />
         </zeroOrMore>
diff --git a/include/clang/AST/CMakeLists.txt b/include/clang/AST/CMakeLists.txt
index 61ba275ce6..ba54fa2aa9 100644
--- a/include/clang/AST/CMakeLists.txt
+++ b/include/clang/AST/CMakeLists.txt
@@ -41,3 +41,7 @@ clang_tablegen(CommentCommandInfo.inc -gen-clang-comment-command-info
   SOURCE CommentCommands.td
   TARGET ClangCommentCommandInfo)
 
+clang_tablegen(CommentCommandList.inc -gen-clang-comment-command-list
+  SOURCE CommentCommands.td
+  TARGET ClangCommentCommandList)
+
diff --git a/include/clang/AST/CommentCommandTraits.h b/include/clang/AST/CommentCommandTraits.h
index 1ec7929af5..8f9fca372b 100644
--- a/include/clang/AST/CommentCommandTraits.h
+++ b/include/clang/AST/CommentCommandTraits.h
@@ -109,6 +109,13 @@ struct CommandInfo {
 /// in comments.
 class CommandTraits {
 public:
+  enum KnownCommandIDs {
+#define COMMENT_COMMAND(NAME) KCI_##NAME,
+#include "clang/AST/CommentCommandList.inc"
+#undef COMMENT_COMMAND
+    KCI_Last
+  };
+
   CommandTraits(llvm::BumpPtrAllocator &Allocator);
 
   /// \returns a CommandInfo object for a given command name or
diff --git a/include/clang/AST/CommentCommands.td b/include/clang/AST/CommentCommands.td
index b5e1c292af..658f396abe 100644
--- a/include/clang/AST/CommentCommands.td
+++ b/include/clang/AST/CommentCommands.td
@@ -77,6 +77,9 @@ def Em : InlineCommand<"em">;
 def Brief : BlockCommand<"brief"> { let IsBriefCommand = 1; }
 def Short : BlockCommand<"short"> { let IsBriefCommand = 1; }
 
+// Opposite of \brief, it is the default in our implementation.
+def Details : BlockCommand<"details">;
+
 def Returns : BlockCommand<"returns"> { let IsReturnsCommand = 1; }
 def Return  : BlockCommand<"return"> { let IsReturnsCommand = 1; }
 def Result  : BlockCommand<"result"> { let IsReturnsCommand = 1; }
@@ -104,7 +107,6 @@ def Authors    : BlockCommand<"authors">;
 def Bug        : BlockCommand<"bug">;
 def Copyright  : BlockCommand<"copyright">;
 def Date       : BlockCommand<"date">;
-def Details    : BlockCommand<"details">;
 def Invariant  : BlockCommand<"invariant">;
 def Note       : BlockCommand<"note">;
 def Post       : BlockCommand<"post">;
diff --git a/include/clang/AST/Makefile b/include/clang/AST/Makefile
index 143339d185..70b87f6464 100644
--- a/include/clang/AST/Makefile
+++ b/include/clang/AST/Makefile
@@ -65,3 +65,8 @@ $(ObjDir)/CommentCommandInfo.inc.tmp : $(PROJ_SRC_DIR)/CommentCommands.td \
 	$(Echo) "Building Clang comment command info with tblgen"
 	$(Verb) $(ClangTableGen) -gen-clang-comment-command-info -o $(call SYSPATH, $@) $<
 
+$(ObjDir)/CommentCommandList.inc.tmp : $(PROJ_SRC_DIR)/CommentCommands.td \
+                                              $(CLANG_TBLGEN) $(ObjDir)/.dir
+	$(Echo) "Building Clang list of comment commands with tblgen"
+	$(Verb) $(ClangTableGen) -gen-clang-comment-command-list -o $(call SYSPATH, $@) $<
+
diff --git a/lib/AST/CMakeLists.txt b/lib/AST/CMakeLists.txt
index 328f385153..14bbc2f724 100644
--- a/lib/AST/CMakeLists.txt
+++ b/lib/AST/CMakeLists.txt
@@ -65,6 +65,7 @@ add_dependencies(clangAST
   ClangAttrImpl
   ClangAttrDump
   ClangCommentCommandInfo
+  ClangCommentCommandList
   ClangCommentNodes
   ClangCommentHTMLTags
   ClangCommentHTMLTagsProperties
diff --git a/test/Index/Inputs/CommentXML/invalid-para-kind-01.xml b/test/Index/Inputs/CommentXML/invalid-para-kind-01.xml
new file mode 100644
index 0000000000..9b8204211e
--- /dev/null
+++ b/test/Index/Inputs/CommentXML/invalid-para-kind-01.xml
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Function>
+<Name>aaa</Name>
+<Abstract><Para>Aaa.</Para></Abstract>
+<Discussion>
+  <Para kind="">Bbb</Para>
+</Discussion>
+</Function>
+
diff --git a/test/Index/Inputs/CommentXML/invalid-para-kind-02.xml b/test/Index/Inputs/CommentXML/invalid-para-kind-02.xml
new file mode 100644
index 0000000000..a1a2900835
--- /dev/null
+++ b/test/Index/Inputs/CommentXML/invalid-para-kind-02.xml
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Function>
+<Name>aaa</Name>
+<Abstract><Para>Aaa.</Para></Abstract>
+<Discussion>
+  <Para kind="zzz">Bbb</Para>
+</Discussion>
+</Function>
+
diff --git a/test/Index/Inputs/CommentXML/valid-para-kind-01.xml b/test/Index/Inputs/CommentXML/valid-para-kind-01.xml
new file mode 100644
index 0000000000..f796dcfe2a
--- /dev/null
+++ b/test/Index/Inputs/CommentXML/valid-para-kind-01.xml
@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Function>
+<Name>aaa</Name>
+<Abstract><Para>Aaa.</Para></Abstract>
+<Discussion>
+  <Para>Bbb</Para>
+  <Para kind="author">Bbb</Para>
+  <Para kind="authors">Bbb</Para>
+  <Para kind="bug">Bbb</Para>
+  <Para kind="copyright">Bbb</Para>
+  <Para kind="date">Bbb</Para>
+  <Para kind="invariant">Bbb</Para>
+  <Para kind="note">Bbb</Para>
+  <Para kind="post">Bbb</Para>
+  <Para kind="pre">Bbb</Para>
+  <Para kind="remark">Bbb</Para>
+  <Para kind="remarks">Bbb</Para>
+  <Para kind="sa">Bbb</Para>
+  <Para kind="see">Bbb</Para>
+  <Para kind="since">Bbb</Para>
+  <Para kind="todo">Bbb</Para>
+  <Para kind="version">Bbb</Para>
+  <Para kind="warning">Bbb</Para>
+</Discussion>
+</Function>
+
diff --git a/test/Index/comment-to-html-xml-conversion.cpp b/test/Index/comment-to-html-xml-conversion.cpp
index da225ba54c..b33fa4ace4 100644
--- a/test/Index/comment-to-html-xml-conversion.cpp
+++ b/test/Index/comment-to-html-xml-conversion.cpp
@@ -766,5 +766,31 @@ enum class comment_to_xml_conversion_17 {
 // CHECK: comment-to-html-xml-conversion.cpp:[[@LINE-1]]:3: EnumConstantDecl=comment_to_xml_conversion_18:{{.*}} FullCommentAsXML=[<Variable file="{{[^"]+}}comment-to-html-xml-conversion.cpp" line="[[@LINE-1]]" column="3"><Name>comment_to_xml_conversion_18</Name><USR>c:@E@comment_to_xml_conversion_17@comment_to_xml_conversion_18</USR><Declaration>comment_to_xml_conversion_18</Declaration><Abstract><Para> Aaa.</Para></Abstract></Variable>]
 };
 
+/// Aaa.
+/// \todo Bbb.
+void comment_to_xml_conversion_todo_1();
+// CHECK: comment-to-html-xml-conversion.cpp:[[@LINE-1]]:6: FunctionDecl=comment_to_xml_conversion_todo_1:{{.*}} FullCommentAsXML=[<Function file="{{[^"]+}}comment-to-html-xml-conversion.cpp" line="[[@LINE-1]]" column="6"><Name>comment_to_xml_conversion_todo_1</Name><USR>c:@F@comment_to_xml_conversion_todo_1#</USR><Declaration>void comment_to_xml_conversion_todo_1()</Declaration><Abstract><Para> Aaa. </Para></Abstract><Discussion><Para kind="todo"> Bbb.</Para></Discussion></Function>]
+
+/// Aaa.
+/// \todo Bbb.
+///
+/// Ccc.
+void comment_to_xml_conversion_todo_2();
+// CHECK: comment-to-html-xml-conversion.cpp:[[@LINE-1]]:6: FunctionDecl=comment_to_xml_conversion_todo_2:{{.*}} FullCommentAsXML=[<Function file="{{[^"]+}}comment-to-html-xml-conversion.cpp" line="[[@LINE-1]]" column="6"><Name>comment_to_xml_conversion_todo_2</Name><USR>c:@F@comment_to_xml_conversion_todo_2#</USR><Declaration>void comment_to_xml_conversion_todo_2()</Declaration><Abstract><Para> Aaa. </Para></Abstract><Discussion><Para kind="todo"> Bbb.</Para><Para> Ccc.</Para></Discussion></Function>]
+
+/// Aaa.
+/// \todo Bbb.
+///
+/// Ccc.
+/// \todo Ddd.
+void comment_to_xml_conversion_todo_3();
+// CHECK: comment-to-html-xml-conversion.cpp:[[@LINE-1]]:6: FunctionDecl=comment_to_xml_conversion_todo_3:{{.*}} FullCommentAsXML=[<Function file="{{[^"]+}}comment-to-html-xml-conversion.cpp" line="[[@LINE-1]]" column="6"><Name>comment_to_xml_conversion_todo_3</Name><USR>c:@F@comment_to_xml_conversion_todo_3#</USR><Declaration>void comment_to_xml_conversion_todo_3()</Declaration><Abstract><Para> Aaa. </Para></Abstract><Discussion><Para kind="todo"> Bbb.</Para><Para> Ccc. </Para><Para kind="todo"> Ddd.</Para></Discussion></Function>]
+
+/// Aaa.
+/// \todo Bbb.
+/// \todo Ccc.
+void comment_to_xml_conversion_todo_4();
+// CHECK: comment-to-html-xml-conversion.cpp:[[@LINE-1]]:6: FunctionDecl=comment_to_xml_conversion_todo_4:{{.*}} FullCommentAsXML=[<Function file="{{[^"]+}}comment-to-html-xml-conversion.cpp" line="[[@LINE-1]]" column="6"><Name>comment_to_xml_conversion_todo_4</Name><USR>c:@F@comment_to_xml_conversion_todo_4#</USR><Declaration>void comment_to_xml_conversion_todo_4()</Declaration><Abstract><Para> Aaa. </Para></Abstract><Discussion><Para kind="todo"> Bbb. </Para><Para kind="todo"> Ccc.</Para></Discussion></Function>]
+
 #endif
 
diff --git a/test/Index/comment-xml-schema.c b/test/Index/comment-xml-schema.c
index 91ea7b2283..b8560f7e27 100644
--- a/test/Index/comment-xml-schema.c
+++ b/test/Index/comment-xml-schema.c
@@ -30,6 +30,8 @@
 // RUN: xmllint --noout --relaxng %S/../../bindings/xml/comment-xml-schema.rng %S/Inputs/CommentXML/valid-typedef-02.xml
 //
 // RUN: xmllint --noout --relaxng %S/../../bindings/xml/comment-xml-schema.rng %S/Inputs/CommentXML/valid-enum-01.xml
+//
+// RUN: xmllint --noout --relaxng %S/../../bindings/xml/comment-xml-schema.rng %S/Inputs/CommentXML/valid-para-kind-01.xml
 
 // RUN: xmllint --noout --relaxng %S/../../bindings/xml/comment-xml-schema.rng %S/Inputs/CommentXML/invalid-function-01.xml 2>&1 | FileCheck %s -check-prefix=INVALID
 // RUN: xmllint --noout --relaxng %S/../../bindings/xml/comment-xml-schema.rng %S/Inputs/CommentXML/invalid-function-02.xml 2>&1 | FileCheck %s -check-prefix=INVALID
@@ -43,6 +45,9 @@
 // RUN: xmllint --noout --relaxng %S/../../bindings/xml/comment-xml-schema.rng %S/Inputs/CommentXML/invalid-function-10.xml 2>&1 | FileCheck %s -check-prefix=INVALID
 // RUN: xmllint --noout --relaxng %S/../../bindings/xml/comment-xml-schema.rng %S/Inputs/CommentXML/invalid-function-11.xml 2>&1 | FileCheck %s -check-prefix=INVALID
 // RUN: xmllint --noout --relaxng %S/../../bindings/xml/comment-xml-schema.rng %S/Inputs/CommentXML/invalid-function-12.xml 2>&1 | FileCheck %s -check-prefix=INVALID
+//
+// RUN: xmllint --noout --relaxng %S/../../bindings/xml/comment-xml-schema.rng %S/Inputs/CommentXML/invalid-para-kind-01.xml 2>&1 | FileCheck %s -check-prefix=INVALID
+// RUN: xmllint --noout --relaxng %S/../../bindings/xml/comment-xml-schema.rng %S/Inputs/CommentXML/invalid-para-kind-02.xml 2>&1 | FileCheck %s -check-prefix=INVALID
 
 // CHECK-INVALID: fails to validate
 
diff --git a/tools/libclang/CXComment.cpp b/tools/libclang/CXComment.cpp
index 0a90c658f9..69002b4d23 100644
--- a/tools/libclang/CXComment.cpp
+++ b/tools/libclang/CXComment.cpp
@@ -882,6 +882,10 @@ public:
 
   // Block content.
   void visitParagraphComment(const ParagraphComment *C);
+
+  void appendParagraphCommentWithKind(const ParagraphComment *C,
+                                      StringRef Kind);
+
   void visitBlockCommandComment(const BlockCommandComment *C);
   void visitParamCommandComment(const ParamCommandComment *C);
   void visitTParamCommandComment(const TParamCommandComment *C);
@@ -893,7 +897,7 @@ public:
 
   // Helpers.
   void appendToResultWithXMLEscaping(StringRef S);
-      
+
   void formatTextOfDeclaration(const DeclInfo *DI,
                                SmallString<128> &Declaration);
 
@@ -1006,10 +1010,20 @@ void CommentASTToXMLConverter::visitHTMLEndTagComment(const HTMLEndTagComment *C
 }
 
 void CommentASTToXMLConverter::visitParagraphComment(const ParagraphComment *C) {
+  appendParagraphCommentWithKind(C, StringRef());
+}
+
+void CommentASTToXMLConverter::appendParagraphCommentWithKind(
+                                  const ParagraphComment *C,
+                                  StringRef ParagraphKind) {
   if (C->isWhitespace())
     return;
 
-  Result << "<Para>";
+  if (ParagraphKind.empty())
+    Result << "<Para>";
+  else
+    Result << "<Para kind=\"" << ParagraphKind << "\">";
+
   for (Comment::child_iterator I = C->child_begin(), E = C->child_end();
        I != E; ++I) {
     visit(*I);
@@ -1018,7 +1032,32 @@ void CommentASTToXMLConverter::visitParagraphComment(const ParagraphComment *C)
 }
 
 void CommentASTToXMLConverter::visitBlockCommandComment(const BlockCommandComment *C) {
-  visit(C->getParagraph());
+  StringRef ParagraphKind;
+
+  switch (C->getCommandID()) {
+  case CommandTraits::KCI_author:
+  case CommandTraits::KCI_authors:
+  case CommandTraits::KCI_bug:
+  case CommandTraits::KCI_copyright:
+  case CommandTraits::KCI_date:
+  case CommandTraits::KCI_invariant:
+  case CommandTraits::KCI_note:
+  case CommandTraits::KCI_post:
+  case CommandTraits::KCI_pre:
+  case CommandTraits::KCI_remark:
+  case CommandTraits::KCI_remarks:
+  case CommandTraits::KCI_sa:
+  case CommandTraits::KCI_see:
+  case CommandTraits::KCI_since:
+  case CommandTraits::KCI_todo:
+  case CommandTraits::KCI_version:
+  case CommandTraits::KCI_warning:
+    ParagraphKind = C->getCommandName(Traits);
+  default:
+    break;
+  }
+
+  appendParagraphCommentWithKind(C->getParagraph(), ParagraphKind);
 }
 
 void CommentASTToXMLConverter::visitParamCommandComment(const ParamCommandComment *C) {
diff --git a/utils/TableGen/ClangCommentCommandInfoEmitter.cpp b/utils/TableGen/ClangCommentCommandInfoEmitter.cpp
index e6253942e0..4dafc2eec9 100644
--- a/utils/TableGen/ClangCommentCommandInfoEmitter.cpp
+++ b/utils/TableGen/ClangCommentCommandInfoEmitter.cpp
@@ -71,5 +71,49 @@ void EmitClangCommentCommandInfo(RecordKeeper &Records, raw_ostream &OS) {
   OS << "  return NULL;\n"
      << "}\n\n";
 }
+
+static std::string MangleName(StringRef Str) {
+  std::string Mangled;
+  for (unsigned i = 0, e = Str.size(); i != e; ++i) {
+    switch (Str[i]) {
+    default:
+      Mangled += Str[i];
+      break;
+    case '[':
+      Mangled += "lsquare";
+      break;
+    case ']':
+      Mangled += "rsquare";
+      break;
+    case '{':
+      Mangled += "lbrace";
+      break;
+    case '}':
+      Mangled += "rbrace";
+      break;
+    case '$':
+      Mangled += "dollar";
+      break;
+    }
+  }
+  return Mangled;
+}
+
+void EmitClangCommentCommandList(RecordKeeper &Records, raw_ostream &OS) {
+  emitSourceFileHeader("A list of commands useable in documentation "
+                       "comments", OS);
+
+  OS << "#ifndef COMMENT_COMMAND\n"
+     << "#  define COMMENT_COMMAND(NAME)\n"
+     << "#endif\n";
+
+  std::vector<Record *> Tags = Records.getAllDerivedDefinitions("Command");
+  for (size_t i = 0, e = Tags.size(); i != e; ++i) {
+    Record &Tag = *Tags[i];
+    std::string MangledName = MangleName(Tag.getValueAsString("Name"));
+
+    OS << "COMMENT_COMMAND(" << MangledName << ")\n";
+  }
+}
 } // end namespace clang
 
diff --git a/utils/TableGen/TableGen.cpp b/utils/TableGen/TableGen.cpp
index 4097339b9a..3df8940b05 100644
--- a/utils/TableGen/TableGen.cpp
+++ b/utils/TableGen/TableGen.cpp
@@ -46,6 +46,7 @@ enum ActionType {
   GenClangCommentHTMLTagsProperties,
   GenClangCommentHTMLNamedCharacterReferences,
   GenClangCommentCommandInfo,
+  GenClangCommentCommandList,
   GenOptParserDefs, GenOptParserImpl,
   GenArmNeon,
   GenArmNeonSema,
@@ -118,6 +119,10 @@ namespace {
                                "references to UTF-8 sequences"),
                     clEnumValN(GenClangCommentCommandInfo,
                                "gen-clang-comment-command-info",
+                               "Generate command properties for commands that "
+                               "are used in documentation comments"),
+                    clEnumValN(GenClangCommentCommandList,
+                               "gen-clang-comment-command-list",
                                "Generate list of commands that are used in "
                                "documentation comments"),
                     clEnumValN(GenArmNeon, "gen-arm-neon",
@@ -205,6 +210,9 @@ bool ClangTableGenMain(raw_ostream &OS, RecordKeeper &Records) {
   case GenClangCommentCommandInfo:
     EmitClangCommentCommandInfo(Records, OS);
     break;
+  case GenClangCommentCommandList:
+    EmitClangCommentCommandList(Records, OS);
+    break;
   case GenOptParserDefs:
     EmitOptParser(Records, OS, true);
     break;
diff --git a/utils/TableGen/TableGenBackends.h b/utils/TableGen/TableGenBackends.h
index 3bc4c906c0..03708b6a76 100644
--- a/utils/TableGen/TableGenBackends.h
+++ b/utils/TableGen/TableGenBackends.h
@@ -54,6 +54,7 @@ void EmitClangCommentHTMLTagsProperties(RecordKeeper &Records, raw_ostream &OS);
 void EmitClangCommentHTMLNamedCharacterReferences(RecordKeeper &Records, raw_ostream &OS);
 
 void EmitClangCommentCommandInfo(RecordKeeper &Records, raw_ostream &OS);
+void EmitClangCommentCommandList(RecordKeeper &Records, raw_ostream &OS);
 
 void EmitNeon(RecordKeeper &Records, raw_ostream &OS);
 void EmitNeonSema(RecordKeeper &Records, raw_ostream &OS);