Comment AST: TableGen'ize all command lists in CommentCommandTraits.cpp.

Now we have a list of all commands.  This is a good thing in itself, but it
also enables us to easily implement typo correction for command names.

With this change we have objects that contain information about each command,
so it makes sense to resolve command name just once during lexing (currently we
store command names as strings and do a linear search every time some property
value is needed).  Thus comment token and AST nodes were changed to contain a
command ID -- index into a tables of builtin and registered commands.  Unknown
commands are registered during parsing and thus are also uniformly assigned an
ID.  Using an ID instead of a StringRef is also a nice memory optimization
since ID is a small integer that fits into a common bitfield in Comment class.

This change implies that to get any information about a command (even a command
name) we need a CommandTraits object to resolve the command ID to CommandInfo*.
Currently a fresh temporary CommandTraits object is created whenever it is
needed since it does not have any state.  But with this change it has state --
new commands can be registered, so a CommandTraits object was added to
ASTContext.

Also, in libclang CXComment has to be expanded to include a CXTranslationUnit
so that all functions working on comment AST nodes can get a CommandTraits
object.  This breaks binary compatibility of CXComment APIs.

Now clang_FullComment_getAsXML(CXTranslationUnit TU, CXComment CXC) doesn't
need TU parameter anymore, so it was removed.  This is a source-incompatible
change for this C API.



git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@163540 91177308-0d34-0410-b5e6-96231b3b80d8

4 files changed, 66 insertions, 42 deletions

diff --git a/tools/c-index-test/c-index-test.c b/tools/c-index-test/c-index-test.c
index 511f1a90bb..85dcddd91c 100644
--- a/tools/c-index-test/c-index-test.c
+++ b/tools/c-index-test/c-index-test.c
@@ -512,8 +512,7 @@ static void ValidateCommentXML(const char *Str,
 #endif
 }
 
-static void PrintCursorComments(CXTranslationUnit TU,
-                                CXCursor Cursor,
+static void PrintCursorComments(CXCursor Cursor,
                                 CommentXMLValidationData *ValidationData) {
   {
     CXString RawComment;
@@ -543,7 +542,7 @@ static void PrintCursorComments(CXTranslationUnit TU,
                                         clang_FullComment_getAsHTML(Comment));
       {
         CXString XML;
-        XML = clang_FullComment_getAsXML(TU, Comment);
+        XML = clang_FullComment_getAsXML(Comment);
         PrintCXStringWithPrefix("FullCommentAsXML", XML);
         ValidateCommentXML(clang_getCString(XML), ValidationData);
         clang_disposeString(XML);
@@ -781,7 +780,7 @@ static void PrintCursor(CXCursor Cursor,
         PrintRange(RefNameRange, "RefName");
     }
 
-    PrintCursorComments(TU, Cursor, ValidationData);
+    PrintCursorComments(Cursor, ValidationData);
   }
 }
 
diff --git a/tools/libclang/CIndex.cpp b/tools/libclang/CIndex.cpp
index ae488abf39..6cf40fd7f7 100644
--- a/tools/libclang/CIndex.cpp
+++ b/tools/libclang/CIndex.cpp
@@ -5800,13 +5800,13 @@ CXString clang_Cursor_getBriefCommentText(CXCursor C) {
 
 CXComment clang_Cursor_getParsedComment(CXCursor C) {
   if (!clang_isDeclaration(C.kind))
-    return cxcomment::createCXComment(NULL);
+    return cxcomment::createCXComment(NULL, NULL);
 
   const Decl *D = getCursorDecl(C);
   const ASTContext &Context = getCursorContext(C);
   const comments::FullComment *FC = Context.getCommentForDecl(D);
 
-  return cxcomment::createCXComment(FC);
+  return cxcomment::createCXComment(FC, getCursorTU(C));
 }
 
 } // end: extern "C"
diff --git a/tools/libclang/CXComment.cpp b/tools/libclang/CXComment.cpp
index c5c9ca8cf3..4e26a9e984 100644
--- a/tools/libclang/CXComment.cpp
+++ b/tools/libclang/CXComment.cpp
@@ -15,12 +15,10 @@
 #include "CXString.h"
 #include "CXComment.h"
 #include "CXCursor.h"
-#include "CXTranslationUnit.h"
 
 #include "clang/AST/CommentVisitor.h"
 #include "clang/AST/CommentCommandTraits.h"
 #include "clang/AST/Decl.h"
-#include "clang/Frontend/ASTUnit.h"
 
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -94,9 +92,9 @@ unsigned clang_Comment_getNumChildren(CXComment CXC) {
 CXComment clang_Comment_getChild(CXComment CXC, unsigned ChildIdx) {
   const Comment *C = getASTNode(CXC);
   if (!C || ChildIdx >= C->child_count())
-    return createCXComment(NULL);
+    return createCXComment(NULL, NULL);
 
-  return createCXComment(*(C->child_begin() + ChildIdx));
+  return createCXComment(*(C->child_begin() + ChildIdx), CXC.TranslationUnit);
 }
 
 unsigned clang_Comment_isWhitespace(CXComment CXC) {
@@ -134,7 +132,8 @@ CXString clang_InlineCommandComment_getCommandName(CXComment CXC) {
   if (!ICC)
     return createCXString((const char *) 0);
 
-  return createCXString(ICC->getCommandName(), /*DupString=*/ false);
+  const CommandTraits &Traits = getCommandTraits(CXC);
+  return createCXString(ICC->getCommandName(Traits), /*DupString=*/ false);
 }
 
 enum CXCommentInlineCommandRenderKind
@@ -221,7 +220,8 @@ CXString clang_BlockCommandComment_getCommandName(CXComment CXC) {
   if (!BCC)
     return createCXString((const char *) 0);
 
-  return createCXString(BCC->getCommandName(), /*DupString=*/ false);
+  const CommandTraits &Traits = getCommandTraits(CXC);
+  return createCXString(BCC->getCommandName(Traits), /*DupString=*/ false);
 }
 
 unsigned clang_BlockCommandComment_getNumArgs(CXComment CXC) {
@@ -244,9 +244,9 @@ CXString clang_BlockCommandComment_getArgText(CXComment CXC,
 CXComment clang_BlockCommandComment_getParagraph(CXComment CXC) {
   const BlockCommandComment *BCC = getASTNodeAs<BlockCommandComment>(CXC);
   if (!BCC)
-    return createCXComment(NULL);
+    return createCXComment(NULL, NULL);
 
-  return createCXComment(BCC->getParagraph());
+  return createCXComment(BCC->getParagraph(), CXC.TranslationUnit);
 }
 
 CXString clang_ParamCommandComment_getParamName(CXComment CXC) {
@@ -405,7 +405,8 @@ public:
 /// Separate parts of a FullComment.
 struct FullCommentParts {
   /// Take a full comment apart and initialize members accordingly.
-  FullCommentParts(const FullComment *C);
+  FullCommentParts(const FullComment *C,
+                   const CommandTraits &Traits);
 
   const BlockContentComment *Brief;
   const ParagraphComment *FirstParagraph;
@@ -415,9 +416,9 @@ struct FullCommentParts {
   SmallVector<const BlockContentComment *, 8> MiscBlocks;
 };
 
-FullCommentParts::FullCommentParts(const FullComment *C) :
+FullCommentParts::FullCommentParts(const FullComment *C,
+                                   const CommandTraits &Traits) :
     Brief(NULL), FirstParagraph(NULL), Returns(NULL) {
-  const CommandTraits Traits;
   for (Comment::child_iterator I = C->child_begin(), E = C->child_end();
        I != E; ++I) {
     const Comment *Child = *I;
@@ -440,12 +441,12 @@ FullCommentParts::FullCommentParts(const FullComment *C) :
 
     case Comment::BlockCommandCommentKind: {
       const BlockCommandComment *BCC = cast<BlockCommandComment>(Child);
-      StringRef CommandName = BCC->getCommandName();
-      if (!Brief && Traits.isBriefCommand(CommandName)) {
+      const CommandInfo *Info = Traits.getCommandInfo(BCC->getCommandID());
+      if (!Brief && Info->IsBriefCommand) {
         Brief = BCC;
         break;
       }
-      if (!Returns && Traits.isReturnsCommand(CommandName)) {
+      if (!Returns && Info->IsReturnsCommand) {
         Returns = BCC;
         break;
       }
@@ -483,7 +484,8 @@ FullCommentParts::FullCommentParts(const FullComment *C) :
 
     case Comment::VerbatimLineCommentKind: {
       const VerbatimLineComment *VLC = cast<VerbatimLineComment>(Child);
-      if (!Traits.isDeclarationCommand(VLC->getCommandName()))
+      const CommandInfo *Info = Traits.getCommandInfo(VLC->getCommandID());
+      if (!Info->IsDeclarationCommand)
         MiscBlocks.push_back(VLC);
       break;
     }
@@ -533,7 +535,10 @@ class CommentASTToHTMLConverter :
     public ConstCommentVisitor<CommentASTToHTMLConverter> {
 public:
   /// \param Str accumulator for HTML.
-  CommentASTToHTMLConverter(SmallVectorImpl<char> &Str) : Result(Str) { }
+  CommentASTToHTMLConverter(SmallVectorImpl<char> &Str,
+                            const CommandTraits &Traits) :
+      Result(Str), Traits(Traits)
+  { }
 
   // Inline content.
   void visitTextComment(const TextComment *C);
@@ -561,10 +566,10 @@ public:
   void appendToResultWithHTMLEscaping(StringRef S);
 
 private:
-  const CommandTraits Traits;
-
   /// Output stream for HTML.
   llvm::raw_svector_ostream Result;
+
+  const CommandTraits &Traits;
 };
 } // end unnamed namespace
 
@@ -637,14 +642,14 @@ void CommentASTToHTMLConverter::visitParagraphComment(
 
 void CommentASTToHTMLConverter::visitBlockCommandComment(
                                   const BlockCommandComment *C) {
-  StringRef CommandName = C->getCommandName();
-  if (Traits.isBriefCommand(CommandName)) {
+  const CommandInfo *Info = Traits.getCommandInfo(C->getCommandID());
+  if (Info->IsBriefCommand) {
     Result << "<p class=\"para-brief\">";
     visitNonStandaloneParagraphComment(C->getParagraph());
     Result << "</p>";
     return;
   }
-  if (Traits.isReturnsCommand(CommandName)) {
+  if (Info->IsReturnsCommand) {
     Result << "<p class=\"para-returns\">"
               "<span class=\"word-returns\">Returns</span> ";
     visitNonStandaloneParagraphComment(C->getParagraph());
@@ -735,7 +740,7 @@ void CommentASTToHTMLConverter::visitVerbatimLineComment(
 }
 
 void CommentASTToHTMLConverter::visitFullComment(const FullComment *C) {
-  FullCommentParts Parts(C);
+  FullCommentParts Parts(C, Traits);
 
   bool FirstParagraphIsBrief = false;
   if (Parts.Brief)
@@ -822,7 +827,7 @@ CXString clang_HTMLTagComment_getAsString(CXComment CXC) {
     return createCXString((const char *) 0);
 
   SmallString<128> HTML;
-  CommentASTToHTMLConverter Converter(HTML);
+  CommentASTToHTMLConverter Converter(HTML, getCommandTraits(CXC));
   Converter.visit(HTC);
   return createCXString(HTML.str(), /* DupString = */ true);
 }
@@ -833,7 +838,7 @@ CXString clang_FullComment_getAsHTML(CXComment CXC) {
     return createCXString((const char *) 0);
 
   SmallString<1024> HTML;
-  CommentASTToHTMLConverter Converter(HTML);
+  CommentASTToHTMLConverter Converter(HTML, getCommandTraits(CXC));
   Converter.visit(FC);
   return createCXString(HTML.str(), /* DupString = */ true);
 }
@@ -845,9 +850,10 @@ class CommentASTToXMLConverter :
     public ConstCommentVisitor<CommentASTToXMLConverter> {
 public:
   /// \param Str accumulator for XML.
-  CommentASTToXMLConverter(const SourceManager &SM,
-                           SmallVectorImpl<char> &Str) :
-    SM(SM), Result(Str) { }
+  CommentASTToXMLConverter(SmallVectorImpl<char> &Str,
+                           const CommandTraits &Traits,
+                           const SourceManager &SM) :
+      Result(Str), Traits(Traits), SM(SM) { }
 
   // Inline content.
   void visitTextComment(const TextComment *C);
@@ -870,10 +876,11 @@ public:
   void appendToResultWithXMLEscaping(StringRef S);
 
 private:
-  const SourceManager &SM;
-
   /// Output stream for XML.
   llvm::raw_svector_ostream Result;
+
+  const CommandTraits &Traits;
+  const SourceManager &SM;
 };
 } // end unnamed namespace
 
@@ -991,7 +998,7 @@ void CommentASTToXMLConverter::visitVerbatimBlockComment(
   if (NumLines == 0)
     return;
 
-  Result << llvm::StringSwitch<const char *>(C->getCommandName())
+  Result << llvm::StringSwitch<const char *>(C->getCommandName(Traits))
       .Case("code", "<Verbatim xml:space=\"preserve\" kind=\"code\">")
       .Default("<Verbatim xml:space=\"preserve\" kind=\"verbatim\">");
   for (unsigned i = 0; i != NumLines; ++i) {
@@ -1015,7 +1022,7 @@ void CommentASTToXMLConverter::visitVerbatimLineComment(
 }
 
 void CommentASTToXMLConverter::visitFullComment(const FullComment *C) {
-  FullCommentParts Parts(C);
+  FullCommentParts Parts(C, Traits);
 
   const DeclInfo *DI = C->getDeclInfo();
   StringRef RootEndTag;
@@ -1213,15 +1220,16 @@ void CommentASTToXMLConverter::appendToResultWithXMLEscaping(StringRef S) {
 
 extern "C" {
 
-CXString clang_FullComment_getAsXML(CXTranslationUnit TU, CXComment CXC) {
+CXString clang_FullComment_getAsXML(CXComment CXC) {
   const FullComment *FC = getASTNodeAs<FullComment>(CXC);
   if (!FC)
     return createCXString((const char *) 0);
 
+  CXTranslationUnit TU = CXC.TranslationUnit;
   SourceManager &SM = static_cast<ASTUnit *>(TU->TUData)->getSourceManager();
 
   SmallString<1024> XML;
-  CommentASTToXMLConverter Converter(SM, XML);
+  CommentASTToXMLConverter Converter(XML, getCommandTraits(CXC), SM);
   Converter.visit(FC);
   return createCXString(XML.str(), /* DupString = */ true);
 }
diff --git a/tools/libclang/CXComment.h b/tools/libclang/CXComment.h
index 753877e6c7..513431709f 100644
--- a/tools/libclang/CXComment.h
+++ b/tools/libclang/CXComment.h
@@ -15,20 +15,29 @@
 #define LLVM_CLANG_CXCOMMENT_H
 
 #include "clang-c/Index.h"
+#include "CXTranslationUnit.h"
 
 #include "clang/AST/Comment.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/Frontend/ASTUnit.h"
 
 namespace clang {
+namespace comments {
+  class CommandTraits;
+}
+
 namespace cxcomment {
 
-inline CXComment createCXComment(const comments::Comment *C) {
+inline CXComment createCXComment(const comments::Comment *C,
+                                 CXTranslationUnit TU) {
   CXComment Result;
-  Result.Data = C;
+  Result.ASTNode = C;
+  Result.TranslationUnit = TU;
   return Result;
 }
 
 inline const comments::Comment *getASTNode(CXComment CXC) {
-  return static_cast<const comments::Comment *>(CXC.Data);
+  return static_cast<const comments::Comment *>(CXC.ASTNode);
 }
 
 template<typename T>
@@ -40,6 +49,14 @@ inline const T *getASTNodeAs(CXComment CXC) {
   return dyn_cast<T>(C);
 }
 
+inline ASTContext &getASTContext(CXComment CXC) {
+  return static_cast<ASTUnit *>(CXC.TranslationUnit->TUData)->getASTContext();
+}
+
+inline comments::CommandTraits &getCommandTraits(CXComment CXC) {
+  return getASTContext(CXC).getCommentCommandTraits();
+}
+
 } // end namespace cxcomment
 } // end namespace clang