Comment AST: TableGen'ize all command lists in CommentCommandTraits.cpp.

Now we have a list of all commands.  This is a good thing in itself, but it
also enables us to easily implement typo correction for command names.

With this change we have objects that contain information about each command,
so it makes sense to resolve command name just once during lexing (currently we
store command names as strings and do a linear search every time some property
value is needed).  Thus comment token and AST nodes were changed to contain a
command ID -- index into a tables of builtin and registered commands.  Unknown
commands are registered during parsing and thus are also uniformly assigned an
ID.  Using an ID instead of a StringRef is also a nice memory optimization
since ID is a small integer that fits into a common bitfield in Comment class.

This change implies that to get any information about a command (even a command
name) we need a CommandTraits object to resolve the command ID to CommandInfo*.
Currently a fresh temporary CommandTraits object is created whenever it is
needed since it does not have any state.  But with this change it has state --
new commands can be registered, so a CommandTraits object was added to
ASTContext.

Also, in libclang CXComment has to be expanded to include a CXTranslationUnit
so that all functions working on comment AST nodes can get a CommandTraits
object.  This breaks binary compatibility of CXComment APIs.

Now clang_FullComment_getAsXML(CXTranslationUnit TU, CXComment CXC) doesn't
need TU parameter anymore, so it was removed.  This is a source-incompatible
change for this C API.



git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@163540 91177308-0d34-0410-b5e6-96231b3b80d8

30 files changed, 879 insertions, 555 deletions

diff --git a/include/clang-c/Index.h b/include/clang-c/Index.h
index cf21d23668..912e657480 100644
--- a/include/clang-c/Index.h
+++ b/include/clang-c/Index.h
@@ -2040,7 +2040,8 @@ typedef struct {
  * \brief A comment AST node.
  */
 typedef struct {
-  const void *Data;
+  const void *ASTNode;
+  CXTranslationUnit TranslationUnit;
 } CXComment;
 
 /**
@@ -3692,14 +3693,11 @@ CINDEX_LINKAGE CXString clang_FullComment_getAsHTML(CXComment Comment);
  * A Relax NG schema for the XML can be found in comment-xml-schema.rng file
  * inside clang source tree.
  *
- * \param TU the translation unit \c Comment belongs to.
- *
  * \param Comment a \c CXComment_FullComment AST node.
  *
  * \returns string containing an XML document.
  */
-CINDEX_LINKAGE CXString clang_FullComment_getAsXML(CXTranslationUnit TU,
-                                                   CXComment Comment);
+CINDEX_LINKAGE CXString clang_FullComment_getAsXML(CXComment Comment);
 
 /**
  * @}
diff --git a/include/clang/AST/ASTContext.h b/include/clang/AST/ASTContext.h
index 93ed1c4f5d..0897582205 100644
--- a/include/clang/AST/ASTContext.h
+++ b/include/clang/AST/ASTContext.h
@@ -29,6 +29,7 @@
 #include "clang/AST/Type.h"
 #include "clang/AST/CanonicalType.h"
 #include "clang/AST/RawCommentList.h"
+#include "clang/AST/CommentCommandTraits.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/FoldingSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
@@ -529,6 +530,14 @@ public:
   /// Returns NULL if no comment is attached.
   comments::FullComment *getCommentForDecl(const Decl *D) const;
 
+private:
+  mutable comments::CommandTraits CommentCommandTraits;
+
+public:
+  comments::CommandTraits &getCommentCommandTraits() const {
+    return CommentCommandTraits;
+  }
+
   /// \brief Retrieve the attributes for the given declaration.
   AttrVec& getDeclAttrs(const Decl *D);
 
diff --git a/include/clang/AST/CMakeLists.txt b/include/clang/AST/CMakeLists.txt
index 339250b674..4c4c0fb0a0 100644
--- a/include/clang/AST/CMakeLists.txt
+++ b/include/clang/AST/CMakeLists.txt
@@ -28,3 +28,7 @@ clang_tablegen(CommentHTMLTagsProperties.inc -gen-clang-comment-html-tags-proper
   SOURCE CommentHTMLTags.td
   TARGET ClangCommentHTMLTagsProperties)
 
+clang_tablegen(CommentCommandInfo.inc -gen-clang-comment-command-info
+  SOURCE CommentCommands.td
+  TARGET ClangCommentCommandInfo)
+
diff --git a/include/clang/AST/Comment.h b/include/clang/AST/Comment.h
index 09d0a26731..cf43fc3e20 100644
--- a/include/clang/AST/Comment.h
+++ b/include/clang/AST/Comment.h
@@ -16,6 +16,7 @@
 
 #include "clang/Basic/SourceLocation.h"
 #include "clang/AST/Type.h"
+#include "clang/AST/CommentCommandTraits.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
 
@@ -74,8 +75,9 @@ protected:
     unsigned : NumInlineContentCommentBits;
 
     unsigned RenderKind : 2;
+    unsigned CommandID : 8;
   };
-  enum { NumInlineCommandCommentBits = NumInlineContentCommentBits + 1 };
+  enum { NumInlineCommandCommentBits = NumInlineContentCommentBits + 10 };
 
   class HTMLStartTagCommentBitfields {
     friend class HTMLStartTagComment;
@@ -101,10 +103,19 @@ protected:
   };
   enum { NumParagraphCommentBits = NumCommentBits + 2 };
 
+  class BlockCommandCommentBitfields {
+    friend class BlockCommandComment;
+
+    unsigned : NumCommentBits;
+
+    unsigned CommandID : 8;
+  };
+  enum { NumBlockCommandCommentBits = NumCommentBits + 8 };
+
   class ParamCommandCommentBitfields {
     friend class ParamCommandComment;
 
-    unsigned : NumCommentBits;
+    unsigned : NumBlockCommandCommentBits;
 
     /// Parameter passing direction, see ParamCommandComment::PassDirection.
     unsigned Direction : 2;
@@ -112,7 +123,7 @@ protected:
     /// True if direction was specified explicitly in the comment.
     unsigned IsDirectionExplicit : 1;
   };
-  enum { NumParamCommandCommentBits = 11 };
+  enum { NumParamCommandCommentBits = NumBlockCommandCommentBits + 3 };
 
   union {
     CommentBitfields CommentBits;
@@ -121,6 +132,7 @@ protected:
     InlineCommandCommentBitfields InlineCommandCommentBits;
     HTMLStartTagCommentBitfields HTMLStartTagCommentBits;
     ParagraphCommentBitfields ParagraphCommentBits;
+    BlockCommandCommentBitfields BlockCommandCommentBits;
     ParamCommandCommentBitfields ParamCommandCommentBits;
   };
 
@@ -158,8 +170,9 @@ public:
   const char *getCommentKindName() const;
 
   LLVM_ATTRIBUTE_USED void dump() const;
-  LLVM_ATTRIBUTE_USED void dump(SourceManager &SM) const;
-  void dump(llvm::raw_ostream &OS, SourceManager *SM) const;
+  LLVM_ATTRIBUTE_USED void dump(const ASTContext &Context) const;
+  void dump(llvm::raw_ostream &OS, const CommandTraits *Traits,
+            const SourceManager *SM) const;
 
   static bool classof(const Comment *) { return true; }
 
@@ -273,21 +286,19 @@ public:
   };
 
 protected:
-  /// Command name.
-  StringRef Name;
-
   /// Command arguments.
   llvm::ArrayRef<Argument> Args;
 
 public:
   InlineCommandComment(SourceLocation LocBegin,
                        SourceLocation LocEnd,
-                       StringRef Name,
+                       unsigned CommandID,
                        RenderKind RK,
                        llvm::ArrayRef<Argument> Args) :
       InlineContentComment(InlineCommandCommentKind, LocBegin, LocEnd),
-      Name(Name), Args(Args) {
+      Args(Args) {
     InlineCommandCommentBits.RenderKind = RK;
+    InlineCommandCommentBits.CommandID = CommandID;
   }
 
   static bool classof(const Comment *C) {
@@ -300,8 +311,12 @@ public:
 
   child_iterator child_end() const { return NULL; }
 
-  StringRef getCommandName() const {
-    return Name;
+  unsigned getCommandID() const {
+    return InlineCommandCommentBits.CommandID;
+  }
+
+  StringRef getCommandName(const CommandTraits &Traits) const {
+    return Traits.getCommandInfo(getCommandID())->Name;
   }
 
   SourceRange getCommandNameRange() const {
@@ -566,9 +581,6 @@ public:
   };
 
 protected:
-  /// Command name.
-  StringRef Name;
-
   /// Word-like arguments.
   llvm::ArrayRef<Argument> Args;
 
@@ -578,21 +590,21 @@ protected:
   BlockCommandComment(CommentKind K,
                       SourceLocation LocBegin,
                       SourceLocation LocEnd,
-                      StringRef Name) :
+                      unsigned CommandID) :
       BlockContentComment(K, LocBegin, LocEnd),
-      Name(Name),
       Paragraph(NULL) {
-    setLocation(getCommandNameRange().getBegin());
+    setLocation(getCommandNameBeginLoc());
+    BlockCommandCommentBits.CommandID = CommandID;
   }
 
 public:
   BlockCommandComment(SourceLocation LocBegin,
                       SourceLocation LocEnd,
-                      StringRef Name) :
+                      unsigned CommandID) :
       BlockContentComment(BlockCommandCommentKind, LocBegin, LocEnd),
-      Name(Name),
       Paragraph(NULL) {
-    setLocation(getCommandNameRange().getBegin());
+    setLocation(getCommandNameBeginLoc());
+    BlockCommandCommentBits.CommandID = CommandID;
   }
 
   static bool classof(const Comment *C) {
@@ -610,12 +622,21 @@ public:
     return reinterpret_cast<child_iterator>(&Paragraph + 1);
   }
 
-  StringRef getCommandName() const {
-    return Name;
+  unsigned getCommandID() const {
+    return BlockCommandCommentBits.CommandID;
   }
 
-  SourceRange getCommandNameRange() const {
-    return SourceRange(getLocStart().getLocWithOffset(1),
+  StringRef getCommandName(const CommandTraits &Traits) const {
+    return Traits.getCommandInfo(getCommandID())->Name;
+  }
+
+  SourceLocation getCommandNameBeginLoc() const {
+    return getLocStart().getLocWithOffset(1);
+  }
+
+  SourceRange getCommandNameRange(const CommandTraits &Traits) const {
+    StringRef Name = getCommandName(Traits);
+    return SourceRange(getCommandNameBeginLoc(),
                        getLocStart().getLocWithOffset(1 + Name.size()));
   }
 
@@ -667,8 +688,9 @@ public:
 
   ParamCommandComment(SourceLocation LocBegin,
                       SourceLocation LocEnd,
-                      StringRef Name) :
-      BlockCommandComment(ParamCommandCommentKind, LocBegin, LocEnd, Name),
+                      unsigned CommandID) :
+      BlockCommandComment(ParamCommandCommentKind, LocBegin, LocEnd,
+                          CommandID),
       ParamIndex(InvalidParamIndex) {
     ParamCommandCommentBits.Direction = In;
     ParamCommandCommentBits.IsDirectionExplicit = false;
@@ -748,8 +770,8 @@ private:
 public:
   TParamCommandComment(SourceLocation LocBegin,
                        SourceLocation LocEnd,
-                       StringRef Name) :
-      BlockCommandComment(TParamCommandCommentKind, LocBegin, LocEnd, Name)
+                       unsigned CommandID) :
+      BlockCommandComment(TParamCommandCommentKind, LocBegin, LocEnd, CommandID)
   { }
 
   static bool classof(const Comment *C) {
@@ -830,9 +852,9 @@ protected:
 public:
   VerbatimBlockComment(SourceLocation LocBegin,
                        SourceLocation LocEnd,
-                       StringRef Name) :
+                       unsigned CommandID) :
       BlockCommandComment(VerbatimBlockCommentKind,
-                          LocBegin, LocEnd, Name)
+                          LocBegin, LocEnd, CommandID)
   { }
 
   static bool classof(const Comment *C) {
@@ -882,12 +904,12 @@ protected:
 public:
   VerbatimLineComment(SourceLocation LocBegin,
                       SourceLocation LocEnd,
-                      StringRef Name,
+                      unsigned CommandID,
                       SourceLocation TextBegin,
                       StringRef Text) :
       BlockCommandComment(VerbatimLineCommentKind,
                           LocBegin, LocEnd,
-                          Name),
+                          CommandID),
       Text(Text),
       TextBegin(TextBegin)
   { }
diff --git a/include/clang/AST/CommentCommandTraits.h b/include/clang/AST/CommentCommandTraits.h
index 5f0269a465..07d9a489e3 100644
--- a/include/clang/AST/CommentCommandTraits.h
+++ b/include/clang/AST/CommentCommandTraits.h
@@ -19,136 +19,125 @@
 #include "clang/Basic/LLVM.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/StringSwitch.h"
+#include "llvm/Support/Allocator.h"
+#include "llvm/Support/ErrorHandling.h"
 
 namespace clang {
 namespace comments {
 
-/// This class provides informaiton about commands that can be used
-/// in comments.
-class CommandTraits {
-public:
-  CommandTraits() { }
+/// \brief Information about a single command.
+///
+/// When reordering, adding or removing members please update the corresponding
+/// TableGen backend.
+struct CommandInfo {
+  unsigned getID() const {
+    return ID;
+  }
+
+  const char *Name;
+
+  /// Name of the command that ends the verbatim block.
+  const char *EndCommandName;
+
+  unsigned ID : 8;
+
+  /// Number of word-like arguments for a given block command, except for
+  /// \\param and \\tparam commands -- these have special argument parsers.
+  unsigned NumArgs : 4;
+
+  /// True if this command is a inline command (of any kind).
+  unsigned IsInlineCommand : 1;
+
+  /// True if this command is a block command (of any kind).
+  unsigned IsBlockCommand : 1;
+
+  /// True if this command is introducing a brief documentation
+  /// paragraph (\\brief or an alias).
+  unsigned IsBriefCommand : 1;
+
+  /// True if this command is \\returns or an alias.
+  unsigned IsReturnsCommand : 1;
+
+  /// True if this command is introducing documentation for a function
+  /// parameter (\\param or an alias).
+  unsigned IsParamCommand : 1;
 
-  /// \brief Check if a given command is a verbatim-like block command.
+  /// True if this command is introducing documentation for
+  /// a template parameter (\\tparam or an alias).
+  unsigned IsTParamCommand : 1;
+
+  /// \brief True if this command is a verbatim-like block command.
   ///
   /// A verbatim-like block command eats every character (except line starting
   /// decorations) until matching end command is seen or comment end is hit.
-  ///
-  /// \param StartName name of the command that starts the verbatim block.
-  /// \param [out] EndName name of the command that ends the verbatim block.
-  ///
-  /// \returns true if a given command is a verbatim block command.
-  bool isVerbatimBlockCommand(StringRef StartName, StringRef &EndName) const;
+  unsigned IsVerbatimBlockCommand : 1;
 
-  /// \brief Register a new verbatim block command.
-  void addVerbatimBlockCommand(StringRef StartName, StringRef EndName);
+  /// \brief True if this command is an end command for a verbatim-like block.
+  unsigned IsVerbatimBlockEndCommand : 1;
 
-  /// \brief Check if a given command is a verbatim line command.
+  /// \brief True if this command is a verbatim line command.
   ///
   /// A verbatim-like line command eats everything until a newline is seen or
   /// comment end is hit.
-  bool isVerbatimLineCommand(StringRef Name) const;
+  unsigned IsVerbatimLineCommand : 1;
 
-  /// \brief Check if a given command is a command that contains a declaration
-  /// for the entity being documented.
+  /// \brief True if this command contains a declaration for the entity being
+  /// documented.
   ///
   /// For example:
   /// \code
   ///   \fn void f(int a);
   /// \endcode
-  bool isDeclarationCommand(StringRef Name) const;
+  unsigned IsDeclarationCommand : 1;
 
-  /// \brief Register a new verbatim line command.
-  void addVerbatimLineCommand(StringRef Name);
+  /// \brief True if this command is unknown.  This \c CommandInfo object was
+  /// created during parsing.
+  unsigned IsUnknownCommand : 1;
+};
 
-  /// \brief Check if a given command is a block command (of any kind).
-  bool isBlockCommand(StringRef Name) const;
+/// This class provides information about commands that can be used
+/// in comments.
+class CommandTraits {
+public:
+  CommandTraits(llvm::BumpPtrAllocator &Allocator);
 
-  /// \brief Check if a given command is introducing documentation for
-  /// a function parameter (\\param or an alias).
-  bool isParamCommand(StringRef Name) const;
+  /// \returns a CommandInfo object for a given command name or
+  /// NULL if no CommandInfo object exists for this command.
+  const CommandInfo *getCommandInfoOrNULL(StringRef Name) const;
 
-  /// \brief Check if a given command is introducing documentation for
-  /// a template parameter (\\tparam or an alias).
-  bool isTParamCommand(StringRef Name) const;
+  const CommandInfo *getCommandInfo(StringRef Name) const {
+    if (const CommandInfo *Info = getCommandInfoOrNULL(Name))
+      return Info;
+    llvm_unreachable("the command should be known");
+  }
 
-  /// \brief Check if a given command is introducing a brief documentation
-  /// paragraph (\\brief or an alias).
-  bool isBriefCommand(StringRef Name) const;
+  const CommandInfo *getCommandInfo(unsigned CommandID) const;
 
-  /// \brief Check if a given command is \\brief or an alias.
-  bool isReturnsCommand(StringRef Name) const;
+  const CommandInfo *registerUnknownCommand(StringRef CommandName);
 
-  /// \returns the number of word-like arguments for a given block command,
-  /// except for \\param and \\tparam commands -- these have special argument
-  /// parsers.
-  unsigned getBlockCommandNumArgs(StringRef Name) const;
+  /// \returns a CommandInfo object for a given command name or
+  /// NULL if \c Name is not a builtin command.
+  static const CommandInfo *getBuiltinCommandInfo(StringRef Name);
 
-  /// \brief Check if a given command is a inline command (of any kind).
-  bool isInlineCommand(StringRef Name) const;
+  /// \returns a CommandInfo object for a given command ID or
+  /// NULL if \c CommandID is not a builtin command.
+  static const CommandInfo *getBuiltinCommandInfo(unsigned CommandID);
 
 private:
-  struct VerbatimBlockCommand {
-    StringRef StartName;
-    StringRef EndName;
-  };
-
-  typedef SmallVector<VerbatimBlockCommand, 4> VerbatimBlockCommandVector;
+  CommandTraits(const CommandTraits &) LLVM_DELETED_FUNCTION;
+  void operator=(const CommandTraits &) LLVM_DELETED_FUNCTION;
 
-  /// Registered additional verbatim-like block commands.
-  VerbatimBlockCommandVector VerbatimBlockCommands;
+  const CommandInfo *getRegisteredCommandInfo(StringRef Name) const;
+  const CommandInfo *getRegisteredCommandInfo(unsigned CommandID) const;
 
-  struct VerbatimLineCommand {
-    StringRef Name;
-  };
+  unsigned NextID;
 
-  typedef SmallVector<VerbatimLineCommand, 4> VerbatimLineCommandVector;
+  /// Allocator for CommandInfo objects.
+  llvm::BumpPtrAllocator &Allocator;
 
-  /// Registered verbatim-like line commands.
-  VerbatimLineCommandVector VerbatimLineCommands;
+  SmallVector<CommandInfo *, 4> RegisteredCommands;
 };
 
-inline bool CommandTraits::isBlockCommand(StringRef Name) const {
-  return isBriefCommand(Name) || isReturnsCommand(Name) ||
-      isParamCommand(Name) || isTParamCommand(Name) ||
-      llvm::StringSwitch<bool>(Name)
-      .Case("author", true)
-      .Case("authors", true)
-      .Case("pre", true)
-      .Case("post", true)
-      .Default(false);
-}
-
-inline bool CommandTraits::isParamCommand(StringRef Name) const {
-  return Name == "param";
-}
-
-inline bool CommandTraits::isTParamCommand(StringRef Name) const {
-  return Name == "tparam" || // Doxygen
-         Name == "templatefield"; // HeaderDoc
-}
-
-inline bool CommandTraits::isBriefCommand(StringRef Name) const {
-  return Name == "brief" || Name == "short";
-}
-
-inline bool CommandTraits::isReturnsCommand(StringRef Name) const {
-  return Name == "returns" || Name == "return" || Name == "result";
-}
-
-inline unsigned CommandTraits::getBlockCommandNumArgs(StringRef Name) const {
-  return 0;
-}
-
-inline bool CommandTraits::isInlineCommand(StringRef Name) const {
-  return llvm::StringSwitch<bool>(Name)
-      .Case("b", true)
-      .Cases("c", "p", true)
-      .Cases("a", "e", "em", true)
-      .Default(false);
-}
-
 } // end namespace comments
 } // end namespace clang
 
diff --git a/include/clang/AST/CommentCommands.td b/include/clang/AST/CommentCommands.td
new file mode 100644
index 0000000000..16ce4fca1b
--- /dev/null
+++ b/include/clang/AST/CommentCommands.td
@@ -0,0 +1,133 @@
+class Command<string name> {
+  string Name = name;
+  string EndCommandName = "";
+
+  int NumArgs = 0;
+
+  bit IsInlineCommand = 0;
+
+  bit IsBlockCommand = 0;
+  bit IsBriefCommand = 0;
+  bit IsReturnsCommand = 0;
+  bit IsParamCommand = 0;
+  bit IsTParamCommand = 0;
+
+  bit IsVerbatimBlockCommand = 0;
+  bit IsVerbatimBlockEndCommand = 0;
+  bit IsVerbatimLineCommand = 0;
+  bit IsDeclarationCommand = 0;
+}
+
+class InlineCommand<string name> : Command<name> {
+  let IsInlineCommand = 1;
+}
+
+class BlockCommand<string name> : Command<name> {
+  let IsBlockCommand = 1;
+}
+
+class VerbatimBlockCommand<string name> : Command<name> {
+  let EndCommandName = name;
+  let IsVerbatimBlockCommand = 1;
+}
+
+multiclass VerbatimBlockCommand<string name, string endCommandName> {
+  def Begin : Command<name> {
+    let EndCommandName = endCommandName;
+    let IsVerbatimBlockCommand = 1;
+  }
+
+  def End : Command<endCommandName> {
+    let IsVerbatimBlockEndCommand = 1;
+  }
+}
+
+class VerbatimLineCommand<string name> : Command<name> {
+  let IsVerbatimLineCommand = 1;
+}
+
+class DeclarationVerbatimLineCommand<string name> :
+      VerbatimLineCommand<name> {
+  let IsDeclarationCommand = 1;
+}
+
+def B  : InlineCommand<"b">;
+def C  : InlineCommand<"c">;
+def P  : InlineCommand<"p">;
+def A  : InlineCommand<"a">;
+def E  : InlineCommand<"e">;
+def Em : InlineCommand<"em">;
+
+def Brief : BlockCommand<"brief"> { let IsBriefCommand = 1; }
+def Short : BlockCommand<"short"> { let IsBriefCommand = 1; }
+
+def Returns : BlockCommand<"returns"> { let IsReturnsCommand = 1; }
+def Return  : BlockCommand<"return"> { let IsReturnsCommand = 1; }
+def Result  : BlockCommand<"result"> { let IsReturnsCommand = 1; }
+
+def Param : BlockCommand<"param"> { let IsParamCommand = 1; }
+
+// Doxygen
+def Tparam : BlockCommand<"tparam"> { let IsTParamCommand = 1; }
+
+// HeaderDoc
+def Templatefield : BlockCommand<"templatefield"> { let IsTParamCommand = 1; }
+
+def Author  : BlockCommand<"author">;
+def Authors : BlockCommand<"authors">;
+def Pre     : BlockCommand<"pre">;
+def Post    : BlockCommand<"post">;
+
+defm Code      : VerbatimBlockCommand<"code", "endcode">;
+defm Verbatim  : VerbatimBlockCommand<"verbatim", "endverbatim">;
+defm Htmlonly  : VerbatimBlockCommand<"htmlonly", "endhtmlonly">;
+defm Latexonly : VerbatimBlockCommand<"latexonly", "endlatexonly">;
+defm Xmlonly   : VerbatimBlockCommand<"xmlonly", "endxmlonly">;
+defm Manonly   : VerbatimBlockCommand<"manonly", "endmanonly">;
+defm Rtfonly   : VerbatimBlockCommand<"rtfonly", "endrtfonly">;
+
+defm Dot : VerbatimBlockCommand<"dot", "enddot">;
+defm Msc : VerbatimBlockCommand<"msc", "endmsc">;
+
+// These commands have special support in lexer.
+def  FDollar  : VerbatimBlockCommand<"f$">; // Inline LaTeX formula
+defm FBracket : VerbatimBlockCommand<"f[", "f]">; // Displayed LaTeX formula
+defm FBrace   : VerbatimBlockCommand<"f{", "f}">; // LaTeX environment
+
+def Defgroup   : VerbatimLineCommand<"defgroup">;
+def Ingroup    : VerbatimLineCommand<"ingroup">;
+def Addtogroup : VerbatimLineCommand<"addtogroup">;
+def Weakgroup  : VerbatimLineCommand<"weakgroup">;
+def Name       : VerbatimLineCommand<"name">;
+
+def Section       : VerbatimLineCommand<"section">;
+def Subsection    : VerbatimLineCommand<"subsection">;
+def Subsubsection : VerbatimLineCommand<"subsubsection">;
+def Paragraph     : VerbatimLineCommand<"paragraph">;
+
+def Mainpage : VerbatimLineCommand<"mainpage">;
+def Subpage  : VerbatimLineCommand<"subpage">;
+def Ref      : VerbatimLineCommand<"ref">;
+
+// Doxygen commands.
+def Fn       : DeclarationVerbatimLineCommand<"fn">;
+def Var      : DeclarationVerbatimLineCommand<"var">;
+def Property : DeclarationVerbatimLineCommand<"property">;
+def Typedef  : DeclarationVerbatimLineCommand<"typedef">;
+def Overload : DeclarationVerbatimLineCommand<"overload">;
+
+// HeaderDoc commands.
+def Class     : DeclarationVerbatimLineCommand<"class">;
+def Interface : DeclarationVerbatimLineCommand<"interface">;
+def Protocol  : DeclarationVerbatimLineCommand<"protocol">;
+def Category  : DeclarationVerbatimLineCommand<"category">;
+def Template  : DeclarationVerbatimLineCommand<"template">;
+def Function  : DeclarationVerbatimLineCommand<"function">;
+def Method    : DeclarationVerbatimLineCommand<"method">;
+def Callback  : DeclarationVerbatimLineCommand<"callback">;
+def Const     : DeclarationVerbatimLineCommand<"const">;
+def Constant  : DeclarationVerbatimLineCommand<"constant">;
+def Struct    : DeclarationVerbatimLineCommand<"struct">;
+def Union     : DeclarationVerbatimLineCommand<"union">;
+def Enum      : DeclarationVerbatimLineCommand<"enum">;
+
diff --git a/include/clang/AST/CommentLexer.h b/include/clang/AST/CommentLexer.h
index e72052191d..99b95d3404 100644
--- a/include/clang/AST/CommentLexer.h
+++ b/include/clang/AST/CommentLexer.h
@@ -26,6 +26,7 @@ namespace comments {
 
 class Lexer;
 class TextTokenRetokenizer;
+struct CommandInfo;
 class CommandTraits;
 
 namespace tok {
@@ -33,6 +34,7 @@ enum TokenKind {
   eof,
   newline,
   text,
+  unknown_command,
   command,
   verbatim_block_begin,
   verbatim_block_line,
@@ -65,8 +67,14 @@ class Token {
   unsigned Length;
 
   /// Contains text value associated with a token.
-  const char *TextPtr1;
-  unsigned TextLen1;
+  const char *TextPtr;
+
+  /// Integer value associated with a token.
+  ///
+  /// If the token is a konwn command, contains command ID and TextPtr is
+  /// unused (command spelling can be found with CommandTraits).  Otherwise,
+  /// contains the length of the string that starts at TextPtr.
+  unsigned IntVal;
 
 public:
   SourceLocation getLocation() const LLVM_READONLY { return Loc; }
@@ -89,113 +97,120 @@ public:
 
   StringRef getText() const LLVM_READONLY {
     assert(is(tok::text));
-    return StringRef(TextPtr1, TextLen1);
+    return StringRef(TextPtr, IntVal);
   }
 
   void setText(StringRef Text) {
     assert(is(tok::text));
-    TextPtr1 = Text.data();
-    TextLen1 = Text.size();
+    TextPtr = Text.data();
+    IntVal = Text.size();
+  }
+
+  StringRef getUnknownCommandName() const LLVM_READONLY {
+    assert(is(tok::unknown_command));
+    return StringRef(TextPtr, IntVal);
+  }
+
+  void setUnknownCommandName(StringRef Name) {
+    assert(is(tok::unknown_command));
+    TextPtr = Name.data();
+    IntVal = Name.size();
   }
 
-  StringRef getCommandName() const LLVM_READONLY {
+  unsigned getCommandID() const LLVM_READONLY {
     assert(is(tok::command));
-    return StringRef(TextPtr1, TextLen1);
+    return IntVal;
   }
 
-  void setCommandName(StringRef Name) {
+  void setCommandID(unsigned ID) {
     assert(is(tok::command));
-    TextPtr1 = Name.data();
-    TextLen1 = Name.size();
+    IntVal = ID;
   }
 
-  StringRef getVerbatimBlockName() const LLVM_READONLY {
+  unsigned getVerbatimBlockID() const LLVM_READONLY {
     assert(is(tok::verbatim_block_begin) || is(tok::verbatim_block_end));
-    return StringRef(TextPtr1, TextLen1);
+    return IntVal;
   }
 
-  void setVerbatimBlockName(StringRef Name) {
+  void setVerbatimBlockID(unsigned ID) {
     assert(is(tok::verbatim_block_begin) || is(tok::verbatim_block_end));
-    TextPtr1 = Name.data();
-    TextLen1 = Name.size();
+    IntVal = ID;
   }
 
   StringRef getVerbatimBlockText() const LLVM_READONLY {
     assert(is(tok::verbatim_block_line));
-    return StringRef(TextPtr1, TextLen1);
+    return StringRef(TextPtr, IntVal);
   }
 
   void setVerbatimBlockText(StringRef Text) {
     assert(is(tok::verbatim_block_line));
-    TextPtr1 = Text.data();
-    TextLen1 = Text.size();
+    TextPtr = Text.data();
+    IntVal = Text.size();
   }
 
-  /// Returns the name of verbatim line command.
-  StringRef getVerbatimLineName() const LLVM_READONLY {
+  unsigned getVerbatimLineID() const LLVM_READONLY {
     assert(is(tok::verbatim_line_name));
-    return StringRef(TextPtr1, TextLen1);
+    return IntVal;
   }
 
-  void setVerbatimLineName(StringRef Name) {
+  void setVerbatimLineID(unsigned ID) {
     assert(is(tok::verbatim_line_name));
-    TextPtr1 = Name.data();
-    Tex