Comment AST: TableGen'ize all command lists in CommentCommandTraits.cpp.

Now we have a list of all commands.  This is a good thing in itself, but it
also enables us to easily implement typo correction for command names.

With this change we have objects that contain information about each command,
so it makes sense to resolve command name just once during lexing (currently we
store command names as strings and do a linear search every time some property
value is needed).  Thus comment token and AST nodes were changed to contain a
command ID -- index into a tables of builtin and registered commands.  Unknown
commands are registered during parsing and thus are also uniformly assigned an
ID.  Using an ID instead of a StringRef is also a nice memory optimization
since ID is a small integer that fits into a common bitfield in Comment class.

This change implies that to get any information about a command (even a command
name) we need a CommandTraits object to resolve the command ID to CommandInfo*.
Currently a fresh temporary CommandTraits object is created whenever it is
needed since it does not have any state.  But with this change it has state --
new commands can be registered, so a CommandTraits object was added to
ASTContext.

Also, in libclang CXComment has to be expanded to include a CXTranslationUnit
so that all functions working on comment AST nodes can get a CommandTraits
object.  This breaks binary compatibility of CXComment APIs.

Now clang_FullComment_getAsXML(CXTranslationUnit TU, CXComment CXC) doesn't
need TU parameter anymore, so it was removed.  This is a source-incompatible
change for this C API.



git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@163540 91177308-0d34-0410-b5e6-96231b3b80d8

10 files changed, 172 insertions, 200 deletions

diff --git a/lib/AST/ASTContext.cpp b/lib/AST/ASTContext.cpp
index d64ed11d7c..9fd40b206d 100644
--- a/lib/AST/ASTContext.cpp
+++ b/lib/AST/ASTContext.cpp
@@ -573,6 +573,7 @@ ASTContext::ASTContext(LangOptions& LOpts, SourceManager &SM,
     DeclarationNames(*this),
     ExternalSource(0), Listener(0),
     Comments(SM), CommentsLoaded(false),
+    CommentCommandTraits(BumpAlloc),
     LastSDM(0, 0),
     UniqueBlockByRefTypeID(0) 
 {
diff --git a/lib/AST/CMakeLists.txt b/lib/AST/CMakeLists.txt
index 14c0aa5eed..d20d77ef7e 100644
--- a/lib/AST/CMakeLists.txt
+++ b/lib/AST/CMakeLists.txt
@@ -64,6 +64,7 @@ add_dependencies(clangAST
   ClangAttrClasses
   ClangAttrList
   ClangAttrImpl
+  ClangCommentCommandInfo
   ClangCommentNodes
   ClangCommentHTMLTags
   ClangCommentHTMLTagsProperties
diff --git a/lib/AST/Comment.cpp b/lib/AST/Comment.cpp
index 2af3896156..a6a21f3923 100644
--- a/lib/AST/Comment.cpp
+++ b/lib/AST/Comment.cpp
@@ -7,6 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "clang/AST/ASTContext.h"
 #include "clang/AST/Comment.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclObjC.h"
@@ -37,11 +38,12 @@ void Comment::dump() const {
   // in CommentDumper.cpp, that object file would be removed by linker because
   // none of its functions are referenced by other object files, despite the
   // LLVM_ATTRIBUTE_USED.
-  dump(llvm::errs(), NULL);
+  dump(llvm::errs(), NULL, NULL);
 }
 
-void Comment::dump(SourceManager &SM) const {
-  dump(llvm::errs(), &SM);
+void Comment::dump(const ASTContext &Context) const {
+  dump(llvm::errs(), &Context.getCommentCommandTraits(),
+       &Context.getSourceManager());
 }
 
 namespace {
diff --git a/lib/AST/CommentBriefParser.cpp b/lib/AST/CommentBriefParser.cpp
index 5a9b10d2db..95daa7e3f8 100644
--- a/lib/AST/CommentBriefParser.cpp
+++ b/lib/AST/CommentBriefParser.cpp
@@ -79,14 +79,14 @@ std::string BriefParser::Parse() {
     }
 
     if (Tok.is(tok::command)) {
-      StringRef Name = Tok.getCommandName();
-      if (Traits.isBriefCommand(Name)) {
+      const CommandInfo *Info = Traits.getCommandInfo(Tok.getCommandID());
+      if (Info->IsBriefCommand) {
         FirstParagraphOrBrief.clear();
         InBrief = true;
         ConsumeToken();
         continue;
       }
-      if (Traits.isReturnsCommand(Name)) {
+      if (Info->IsReturnsCommand) {
         InReturns = true;
         InBrief = false;
         InFirstParagraph = false;
@@ -95,7 +95,7 @@ std::string BriefParser::Parse() {
         continue;
       }
       // Block commands implicitly start a new paragraph.
-      if (Traits.isBlockCommand(Name)) {
+      if (Info->IsBlockCommand) {
         // We found an implicit paragraph end.
         InFirstParagraph = false;
         if (InBrief)
diff --git a/lib/AST/CommentCommandTraits.cpp b/lib/AST/CommentCommandTraits.cpp
index dc7a0bd175..26e2dee3a2 100644
--- a/lib/AST/CommentCommandTraits.cpp
+++ b/lib/AST/CommentCommandTraits.cpp
@@ -8,125 +8,63 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/AST/CommentCommandTraits.h"
-#include "llvm/ADT/StringSwitch.h"
+#include "llvm/ADT/STLExtras.h"
 
 namespace clang {
 namespace comments {
 
-// TODO: tablegen
+#include "clang/AST/CommentCommandInfo.inc"
 
-bool CommandTraits::isVerbatimBlockCommand(StringRef StartName,
-                                           StringRef &EndName) const {
-  const char *Result = llvm::StringSwitch<const char *>(StartName)
-    .Case("code", "endcode")
-    .Case("verbatim", "endverbatim")
-    .Case("htmlonly", "endhtmlonly")
-    .Case("latexonly", "endlatexonly")
-    .Case("xmlonly", "endxmlonly")
-    .Case("manonly", "endmanonly")
-    .Case("rtfonly", "endrtfonly")
+CommandTraits::CommandTraits(llvm::BumpPtrAllocator &Allocator) :
+    NextID(llvm::array_lengthof(Commands)), Allocator(Allocator)
+{ }
 
-    .Case("dot", "enddot")
-    .Case("msc", "endmsc")
-
-    .Case("f$", "f$") // Inline LaTeX formula
-    .Case("f[", "f]") // Displayed LaTeX formula
-    .Case("f{", "f}") // LaTeX environment
-
-    .Default(NULL);
-
-  if (Result) {
-    EndName = Result;
-    return true;
-  }
-
-  for (VerbatimBlockCommandVector::const_iterator
-           I = VerbatimBlockCommands.begin(),
-           E = VerbatimBlockCommands.end();
-       I != E; ++I)
-    if (I->StartName == StartName) {
-      EndName = I->EndName;
-      return true;
-    }
-
-  return false;
+const CommandInfo *CommandTraits::getCommandInfoOrNULL(StringRef Name) const {
+  if (const CommandInfo *Info = getBuiltinCommandInfo(Name))
+    return Info;
+  return getRegisteredCommandInfo(Name);
 }
 
-bool CommandTraits::isVerbatimLineCommand(StringRef Name) const {
-  bool Result = isDeclarationCommand(Name) || llvm::StringSwitch<bool>(Name)
-  .Case("defgroup", true)
-  .Case("ingroup", true)
-  .Case("addtogroup", true)
-  .Case("weakgroup", true)
-  .Case("name", true)
-
-  .Case("section", true)
-  .Case("subsection", true)
-  .Case("subsubsection", true)
-  .Case("paragraph", true)
-
-  .Case("mainpage", true)
-  .Case("subpage", true)
-  .Case("ref", true)
+const CommandInfo *CommandTraits::getCommandInfo(unsigned CommandID) const {
+  if (const CommandInfo *Info = getBuiltinCommandInfo(CommandID))
+    return Info;
+  return getRegisteredCommandInfo(CommandID);
+}
 
-  .Default(false);
+const CommandInfo *CommandTraits::registerUnknownCommand(StringRef CommandName) {
+  char *Name = Allocator.Allocate<char>(CommandName.size());
+  memcpy(Name, CommandName.data(), CommandName.size());
+  Name[CommandName.size() + 1] = '\0';
 
-  if (Result)
-    return true;
+  // Value-initialize (=zero-initialize in this case) a new CommandInfo.
+  CommandInfo *Info = new (Allocator) CommandInfo();
+  Info->Name = Name;
+  Info->ID = NextID++;
 
-  for (VerbatimLineCommandVector::const_iterator
-           I = VerbatimLineCommands.begin(),
-           E = VerbatimLineCommands.end();
-       I != E; ++I)
-    if (I->Name == Name)
-      return true;
+  RegisteredCommands.push_back(Info);
 
-  return false;
+  return Info;
 }
 
-bool CommandTraits::isDeclarationCommand(StringRef Name) const {
-  return llvm::StringSwitch<bool>(Name)
-      // Doxygen commands.
-      .Case("fn", true)
-      .Case("var", true)
-      .Case("property", true)
-      .Case("typedef", true)
-
-      .Case("overload", true)
-
-      // HeaderDoc commands.
-      .Case("class", true)
-      .Case("interface", true)
-      .Case("protocol", true)
-      .Case("category", true)
-      .Case("template", true)
-      .Case("function", true)
-      .Case("method", true)
-      .Case("callback", true)
-      .Case("var", true)
-      .Case("const", true)
-      .Case("constant", true)
-      .Case("property", true)
-      .Case("struct", true)
-      .Case("union", true)
-      .Case("typedef", true)
-      .Case("enum", true)
-
-      .Default(false);
+const CommandInfo *CommandTraits::getBuiltinCommandInfo(
+                                                  unsigned CommandID) {
+  if (CommandID < llvm::array_lengthof(Commands))
+    return &Commands[CommandID];
+  return NULL;
 }
 
-void CommandTraits::addVerbatimBlockCommand(StringRef StartName,
-                                            StringRef EndName) {
-  VerbatimBlockCommand VBC;
-  VBC.StartName = StartName;
-  VBC.EndName = EndName;
-  VerbatimBlockCommands.push_back(VBC);
+const CommandInfo *CommandTraits::getRegisteredCommandInfo(
+                                                  StringRef Name) const {
+  for (unsigned i = 0, e = RegisteredCommands.size(); i != e; ++i) {
+    if (RegisteredCommands[i]->Name == Name)
+      return RegisteredCommands[i];
+  }
+  return NULL;
 }
 
-void CommandTraits::addVerbatimLineCommand(StringRef Name) {
-  VerbatimLineCommand VLC;
-  VLC.Name = Name;
-  VerbatimLineCommands.push_back(VLC);
+const CommandInfo *CommandTraits::getRegisteredCommandInfo(
+                                                  unsigned CommandID) const {
+  return RegisteredCommands[CommandID - llvm::array_lengthof(Commands)];
 }
 
 } // end namespace comments
diff --git a/lib/AST/CommentDumper.cpp b/lib/AST/CommentDumper.cpp
index dffc8233a0..f6fb3b1baa 100644
--- a/lib/AST/CommentDumper.cpp
+++ b/lib/AST/CommentDumper.cpp
@@ -16,12 +16,15 @@ namespace comments {
 namespace {
 class CommentDumper: public comments::ConstCommentVisitor<CommentDumper> {
   raw_ostream &OS;
-  SourceManager *SM;
+  const CommandTraits *Traits;
+  const SourceManager *SM;
   unsigned IndentLevel;
 
 public:
-  CommentDumper(raw_ostream &OS, SourceManager *SM) :
-      OS(OS), SM(SM), IndentLevel(0)
+  CommentDumper(raw_ostream &OS,
+                const CommandTraits *Traits,
+                const SourceManager *SM) :
+      OS(OS), Traits(Traits), SM(SM), IndentLevel(0)
   { }
 
   void dumpIndent() const {
@@ -56,6 +59,15 @@ public:
   void visitVerbatimLineComment(const VerbatimLineComment *C);
 
   void visitFullComment(const FullComment *C);
+
+  const char *getCommandName(unsigned CommandID) {
+    if (Traits)
+      return Traits->getCommandInfo(CommandID)->Name;
+    const CommandInfo *Info = CommandTraits::getBuiltinCommandInfo(CommandID);
+    if (Info)
+      return Info->Name;
+    return "<not a builtin command>";
+  }
 };
 
 void CommentDumper::dumpSourceRange(const Comment *C) {
@@ -107,7 +119,7 @@ void CommentDumper::visitTextComment(const TextComment *C) {
 void CommentDumper::visitInlineCommandComment(const InlineCommandComment *C) {
   dumpComment(C);
 
-  OS << " Name=\"" << C->getCommandName() << "\"";
+  OS << " Name=\"" << getCommandName(C->getCommandID()) << "\"";
   switch (C->getRenderKind()) {
   case InlineCommandComment::RenderNormal:
     OS << " RenderNormal";
@@ -155,7 +167,7 @@ void CommentDumper::visitParagraphComment(const ParagraphComment *C) {
 void CommentDumper::visitBlockCommandComment(const BlockCommandComment *C) {
   dumpComment(C);
 
-  OS << " Name=\"" << C->getCommandName() << "\"";
+  OS << " Name=\"" << getCommandName(C->getCommandID()) << "\"";
   for (unsigned i = 0, e = C->getNumArgs(); i != e; ++i)
     OS << " Arg[" << i << "]=\"" << C->getArgText(i) << "\"";
 }
@@ -198,7 +210,7 @@ void CommentDumper::visitTParamCommandComment(const TParamCommandComment *C) {
 void CommentDumper::visitVerbatimBlockComment(const VerbatimBlockComment *C) {
   dumpComment(C);
 
-  OS << " Name=\"" << C->getCommandName() << "\""
+  OS << " Name=\"" << getCommandName(C->getCommandID()) << "\""
         " CloseName=\"" << C->getCloseName() << "\"";
 }
 
@@ -220,8 +232,9 @@ void CommentDumper::visitFullComment(const FullComment *C) {
 
 } // unnamed namespace
 
-void Comment::dump(llvm::raw_ostream &OS, SourceManager *SM) const {
-  CommentDumper D(llvm::errs(), SM);
+void Comment::dump(llvm::raw_ostream &OS, const CommandTraits *Traits,
+                   const SourceManager *SM) const {
+  CommentDumper D(llvm::errs(), Traits, SM);
   D.dumpSubtree(this);
   llvm::errs() << '\n';
 }
diff --git a/lib/AST/CommentLexer.cpp b/lib/AST/CommentLexer.cpp
index 53440c169f..fde2c40fa5 100644
--- a/lib/AST/CommentLexer.cpp
+++ b/lib/AST/CommentLexer.cpp
@@ -359,18 +359,23 @@ void Lexer::lexCommentText(Token &T) {
         }
 
         const StringRef CommandName(BufferPtr + 1, Length);
-        StringRef EndName;
 
-        if (Traits.isVerbatimBlockCommand(CommandName, EndName)) {
-          setupAndLexVerbatimBlock(T, TokenPtr, *BufferPtr, EndName);
+        const CommandInfo *Info = Traits.getCommandInfoOrNULL(CommandName);
+        if (!Info) {
+          formTokenWithChars(T, TokenPtr, tok::unknown_command);
+          T.setUnknownCommandName(CommandName);
           return;
         }
-        if (Traits.isVerbatimLineCommand(CommandName)) {
-          setupAndLexVerbatimLine(T, TokenPtr);
+        if (Info->IsVerbatimBlockCommand) {
+          setupAndLexVerbatimBlock(T, TokenPtr, *BufferPtr, Info);
+          return;
+        }
+        if (Info->IsVerbatimLineCommand) {
+          setupAndLexVerbatimLine(T, TokenPtr, Info);
           return;
         }
         formTokenWithChars(T, TokenPtr, tok::command);
-        T.setCommandName(CommandName);
+        T.setCommandID(Info->getID());
         return;
       }
 
@@ -423,14 +428,15 @@ void Lexer::lexCommentText(Token &T) {
 
 void Lexer::setupAndLexVerbatimBlock(Token &T,
                                      const char *TextBegin,
-                                     char Marker, StringRef EndName) {
+                                     char Marker, const CommandInfo *Info) {
+  assert(Info->IsVerbatimBlockCommand);
+
   VerbatimBlockEndCommandName.clear();
   VerbatimBlockEndCommandName.append(Marker == '\\' ? "\\" : "@");
-  VerbatimBlockEndCommandName.append(EndName);
+  VerbatimBlockEndCommandName.append(Info->EndCommandName);
 
-  StringRef Name(BufferPtr + 1, TextBegin - (BufferPtr + 1));
   formTokenWithChars(T, TextBegin, tok::verbatim_block_begin);
-  T.setVerbatimBlockName(Name);
+  T.setVerbatimBlockID(Info->getID());
 
   // If there is a newline following the verbatim opening command, skip the
   // newline so that we don't create an tok::verbatim_block_line with empty
@@ -471,7 +477,7 @@ again:
     const char *End = BufferPtr + VerbatimBlockEndCommandName.size();
     StringRef Name(BufferPtr + 1, End - (BufferPtr + 1));
     formTokenWithChars(T, End, tok::verbatim_block_end);
-    T.setVerbatimBlockName(Name);
+    T.setVerbatimBlockID(Traits.getCommandInfo(Name)->getID());
     State = LS_Normal;
     return;
   } else {
@@ -501,10 +507,11 @@ void Lexer::lexVerbatimBlockBody(Token &T) {
   lexVerbatimBlockFirstLine(T);
 }
 
-void Lexer::setupAndLexVerbatimLine(Token &T, const char *TextBegin) {
-  const StringRef Name(BufferPtr + 1, TextBegin - BufferPtr - 1);
+void Lexer::setupAndLexVerbatimLine(Token &T, const char *TextBegin,
+                                    const CommandInfo *Info) {
+  assert(Info->IsVerbatimLineCommand);
   formTokenWithChars(T, TextBegin, tok::verbatim_line_name);
-  T.setVerbatimLineName(Name);
+  T.setVerbatimLineID(Info->getID());
 
   State = LS_VerbatimLineText;
 }
diff --git a/lib/AST/CommentParser.cpp b/lib/AST/CommentParser.cpp
index 43abf6a476..f6acd9645f 100644
--- a/lib/AST/CommentParser.cpp
+++ b/lib/AST/CommentParser.cpp
@@ -132,8 +132,8 @@ class TextTokenRetokenizer {
     Result.setKind(tok::text);
     Result.setLength(TokLength);
 #ifndef NDEBUG
-    Result.TextPtr1 = "<UNSET>";
-    Result.TextLen1 = 7;
+    Result.TextPtr = "<UNSET>";
+    Result.IntVal = 7;
 #endif
     Result.setText(Text);
   }
@@ -312,26 +312,26 @@ BlockCommandComment *Parser::parseBlockCommand() {
   BlockCommandComment *BC;
   bool IsParam = false;
   bool IsTParam = false;
-  unsigned NumArgs = 0;
-  if (Traits.isParamCommand(Tok.getCommandName())) {
+  const CommandInfo *Info = Traits.getCommandInfo(Tok.getCommandID());
+  if (Info->IsParamCommand) {
     IsParam = true;
     PC = S.actOnParamCommandStart(Tok.getLocation(),
                                   Tok.getEndLocation(),
-                                  Tok.getCommandName());
-  } if (Traits.isTParamCommand(Tok.getCommandName())) {
+                                  Tok.getCommandID());
+  } if (Info->IsTParamCommand) {
     IsTParam = true;
     TPC = S.actOnTParamCommandStart(Tok.getLocation(),
                                     Tok.getEndLocation(),
-                                    Tok.getCommandName());
+                                    Tok.getCommandID());
   } else {
-    NumArgs = Traits.getBlockCommandNumArgs(Tok.getCommandName());
     BC = S.actOnBlockCommandStart(Tok.getLocation(),
                                   Tok.getEndLocation(),
-                                  Tok.getCommandName());
+                                  Tok.getCommandID());
   }
   consumeToken();
 
-  if (Tok.is(tok::command) && Traits.isBlockCommand(Tok.getCommandName())) {
+  if (Tok.is(tok::command) &&
+      Traits.getCommandInfo(Tok.getCommandID())->IsBlockCommand) {
     // Block command ahead.  We can't nest block commands, so pretend that this
     // command has an empty argument.
     ParagraphComment *Paragraph = S.actOnParagraphComment(
@@ -348,7 +348,7 @@ BlockCommandComment *Parser::parseBlockCommand() {
     }
   }
 
-  if (IsParam || IsTParam || NumArgs > 0) {
+  if (IsParam || IsTParam || Info->NumArgs > 0) {
     // In order to parse command arguments we need to retokenize a few
     // following text tokens.
     TextTokenRetokenizer Retokenizer(Allocator, *this);
@@ -358,7 +358,7 @@ BlockCommandComment *Parser::parseBlockCommand() {
     else if (IsTParam)
       parseTParamCommandArgs(TPC, Retokenizer);
     else
-      parseBlockCommandArgs(BC, Retokenizer, NumArgs);
+      parseBlockCommandArgs(BC, Retokenizer, Info->NumArgs);
 
     Retokenizer.putBackLeftoverTokens();
   }
@@ -394,14 +394,14 @@ InlineCommandComment *Parser::parseInlineCommand() {
   if (ArgTokValid) {
     IC = S.actOnInlineCommand(CommandTok.getLocation(),
                               CommandTok.getEndLocation(),
-                              CommandTok.getCommandName(),
+                              CommandTok.getCommandID(),
                               ArgTok.getLocation(),
                               ArgTok.getEndLocation(),
                               ArgTok.getText());
   } else {
     IC = S.actOnInlineCommand(CommandTok.getLocation(),
                               CommandTok.getEndLocation(),
-                              CommandTok.getCommandName());
+                              CommandTok.getCommandID());
   }
 
   Retokenizer.putBackLeftoverTokens();
@@ -540,23 +540,24 @@ BlockContentComment *Parser::parseParagraphOrBlockCommand() {
       assert(Content.size() != 0);
       break; // Block content or EOF ahead, finish this parapgaph.
 
-    case tok::command:
-      if (Traits.isBlockCommand(Tok.getCommandName())) {
+    case tok::unknown_command:
+      Content.push_back(S.actOnUnknownCommand(Tok.getLocation(),
+                                              Tok.getEndLocation(),
+                                              Tok.getUnknownCommandName()));
+      consumeToken();
+      continue;
+
+    case tok::command: {
+      const CommandInfo *Info = Traits.getCommandInfo(Tok.getCommandID());
+      if (Info->IsBlockCommand) {
         if (Content.size() == 0)
           return parseBlockCommand();
         break; // Block command ahead, finish this parapgaph.
       }
-      if (Traits.isInlineCommand(Tok.getCommandName())) {
-        Content.push_back(parseInlineCommand());
-        continue;
-      }
-
-      // Not a block command, not an inline command ==> an unknown command.
-      Content.push_back(S.actOnUnknownCommand(Tok.getLocation(),
-                                              Tok.getEndLocation(),
-                                              Tok.getCommandName()));
-      consumeToken();
+      assert(Info->IsInlineCommand);
+      Content.push_back(parseInlineCommand());
       continue;
+    }
 
     case tok::newline: {
       consumeToken();
@@ -606,7 +607,7 @@ VerbatimBlockComment *Parser::parseVerbatimBlock() {
 
   VerbatimBlockComment *VB =
       S.actOnVerbatimBlockStart(Tok.getLocation(),
-                                Tok.getVerbatimBlockName());
+                                Tok.getVerbatimBlockID());
   consumeToken();
 
   // Don't create an empty line if verbatim opening command is followed
@@ -634,8 +635,9 @@ VerbatimBlockComment *Parser::parseVerbatimBlock() {
   }
 
   if (Tok.is(tok::verbatim_block_end)) {
+    const CommandInfo *Info = Traits.getCommandInfo(Tok.getVerbatimBlockID());
     S.actOnVerbatimBlockFinish(VB, Tok.getLocation(),
-                               Tok.getVerbatimBlockName(),
+                               Info->Name,
                                S.copyArray(llvm::makeArrayRef(Lines)));
     consumeToken();
   } else {
@@ -666,7 +668,7 @@ VerbatimLineComment *Parser::parseVerbatimLine() {
   }
 
   VerbatimLineComment *VL = S.actOnVerbatimLine(NameTok.getLocation(),
-                                                NameTok.getVerbatimLineName(),
+                                                NameTok.getVerbatimLineID(),
                                                 TextBegin,
                                                 Text);
   consumeToken();
@@ -676,6 +678,7 @@ VerbatimLineComment *Parser::parseVerbatimLine() {
 BlockContentComment *Parser::parseBlockContent() {
   switch (Tok.getKind()) {
   case tok::text:
+  case tok::unknown_command:
   case tok::command:
   case tok::html_start_tag:
   case tok::html_end_tag:
diff --git a/lib/AST/CommentSema.cpp b/lib/AST/CommentSema.cpp
index ffb6e86788..953afe147e 100644
--- a/lib/AST/CommentSema.cpp
+++ b/lib/AST/CommentSema.cpp
@@ -23,7 +23,7 @@ namespace {
 } // unnamed namespace
 
 Sema::Sema(llvm::BumpPtrAllocator &Allocator, const SourceManager &SourceMgr,
-           DiagnosticsEngine &Diags, const CommandTraits &Traits) :
+           DiagnosticsEngine &Diags, CommandTraits &Traits) :
     Allocator(Allocator), SourceMgr(SourceMgr), Diags(Diags), Traits(Traits),
     ThisDeclInfo(NULL), BriefCommand(NULL), ReturnsCommand(NULL) {
 }
@@ -44,8 +44,8 @@ ParagraphComment *Sema::actOnParagraphComment(
 
 BlockCommandComment *Sema::actOnBlockCommandStart(SourceLocation LocBegin,
                                                   SourceLocation LocEnd,
-                                                  StringRef Name) {
-  return new (Allocator) BlockCommandComment(LocBegin, LocEnd, Name);
+                                                  unsigned CommandID) {
+  return new (Allocator) BlockCommandComment(LocBegin, LocEnd, CommandID);
 }
 
 void Sema::actOnBlockCommandArgs(BlockCommandComment *Command,
@@ -63,14 +63,14 @@ void Sema::actOnBlockCommandFinish(BlockCommandComment *Command,
 
 ParamCommandComment *Sema::actOnParamCommandStart(SourceLocation LocBegin,
                                                   SourceLocation LocEnd,
-                                                  StringRef Name) {
+                                                  unsigned CommandID) {
   ParamCommandComment *Command =
-      new (Allocator) ParamCommandComment(LocBegin, LocEnd, Name);
+      new (Allocator) ParamCommandComment(LocBegin, LocEnd, CommandID);
 
   if (!isFunctionDecl())
     Diag(Command->getLocation(),
          diag::warn_doc_param_not_attached_to_a_function_decl)
-      << Command->getCommandNameRange();
+      << Command->getCommandNameRange(Traits);
 
   return Command;
 }
@@ -156,14 +156,14 @@ void Sema::actOnParamCommandFinish(ParamCommandComment *Command,
 
 TParamCommandComment *Sema::actOnTParamCommandStart(SourceLocation LocBegin,
                                                     SourceLocation LocEnd,
-                                                    StringRef Name) {
+                                                    unsigned CommandID) {
   TParamCommandComment *Command =
-      new (Allocator) TParamCommandComment(LocBegin, LocEnd, Name);
+      new (Allocator) TParamCommandComment(LocBegin, LocEnd, CommandID);
 
   if (!isTemplateOrSpecialization())
     Diag(Command->getLocation(),
          diag::warn_doc_tparam_not_attached_to_a_template_decl)
-      << Command->getCommandNameRange();
+      << Command->getCommandNameRange(Traits);
 
   return Command;
 }
@@ -239,19 +239,20 @@ void Sema::actOnTParamCommandFinish(TParamCommandComment *Command,
 
 InlineCommandComment *Sema::actOnInlineCommand(SourceLocation CommandLocBegin,
                                                SourceLocation CommandLocEnd,
-                                               StringRef CommandName) {
+                                               unsigned CommandID) {
   ArrayRef<InlineCommandComment::Argument> Args;
+  StringRef CommandName = Traits.getCommandInfo(CommandID)->Name;
   return new (Allocator) InlineCommandComment(
                                   CommandLocBegin,
                                   CommandLocEnd,
-                                  CommandName,
+                                  CommandID,
                                   getInlineCommandRenderKind(CommandName),
                                   Args);
 }
 
 InlineCommandComment *Sema::actOnInlineCommand(SourceLocation CommandLocBegin,
                                                SourceLocation CommandLocEnd,
-                                               StringRef CommandName,
+                                               unsigned CommandID,
                                                SourceLocation ArgLocBegin,
                                                SourceLocation ArgLocEnd,
                                                StringRef Arg) {
@@ -259,11 +260,12 @@ InlineCommandComment *Sema::actOnInlineCommand(SourceLocation CommandLocBegin,
   Argument *A = new (Allocator) Argument(SourceRange(ArgLocBegin,
                                                      ArgLocEnd),
                                          Arg);
+  StringRef CommandName = Traits.getCommandInfo(CommandID)->Name;
 
   return new (Allocator) InlineCommandComment(
                                   CommandLocBegin,
                                   CommandLocEnd,
-                                  CommandName,
+                                  CommandID,
                                   getInlineCommandRenderKind(CommandName),
                                   llvm::makeArrayRef(A, 1));
 }
@@ -272,8 +274,9 @@ InlineContentComment *Sema::actOnUnknownCommand(SourceLocation LocBegin,
                                                 SourceLocation LocEnd,
                                                 StringRef Name) {
   ArrayRef<InlineCommandComment::Argument> Args;
+  unsigned CommandID = Traits.registerUnknownCommand(Name)->getID();
   return new (Allocator) InlineCommandComment(
-                                  LocBegin, LocEnd, Name,
+                                  LocBegin, LocEnd, CommandID,
                                   InlineCommandComment::RenderNormal,
                                   Args);
 }
@@ -285,11 +288,12 @@ TextComment *Sema::actOnText(SourceLocation LocBegin,
 }
 
 VerbatimBlockComment *Sema::actOnVerbatimBlockStart(SourceLocation Loc,
-                                                    StringRef Name) {
+                                                    unsigned CommandID) {
+  StringRef CommandName = Traits.getCommandInfo(CommandID)->Name;
   return new (Allocator) VerbatimBlockComment(
                                   Loc,
-                                  Loc.getLocWithOffset(1 + Name.size()),
-                                  Name);
+                                  Loc.getLocWithOffset(1 + CommandName.size()),
+                                  CommandID);
 }
 
 VerbatimBlockLineComment *Sema::actOnVerbatimBlockLine(SourceLocation Loc,
@@ -307,13 +311,13 @@ void Sema::actOnVerbatimBlockFinish(
 }
 
 VerbatimLineComment *Sema::actOnVerbatimLine(SourceLocation LocBegin,
-                                             StringRef Name,
+                                             unsigned CommandID,
                                              SourceLocation TextBegin,
                                              StringRef Text) {
   return new (Allocator) VerbatimLineComment(
                               LocBegin,
                               TextBegin.getLocWithOffset(Text.size()),
-                              Name,
+                              CommandID,
                               TextBegin,
                               Text);
 }
@@ -411,15 +415,15 @@ void Sema::checkBlockCommandEmptyParagraph(BlockCommandComment *Command) {
     if (Command->getNumArgs() > 0)
       DiagLoc = Command->getArgRange(Command->getNumArgs() - 1).getEnd();
     if (!DiagLoc.isValid())
-      DiagLoc = Command->getCommandNameRange().getEnd();
+      DiagLoc = Command->getCommandNameRange(Traits).getEnd();
     Diag(DiagLoc, diag::warn_doc_block_command_empty_paragraph)
-      << Command->getCommandName()
+      << Command->getCommandName(Traits)
       << Command->getSourceRange();
   }
 }
 
 void Sema::checkReturnsCommand(const BlockCommandComment *Command) {
-  if (!Traits.isReturnsCommand(Command->getCommandName()))
+  if (!Traits.getCommandInfo(Command->getCommandID())->IsReturnsCommand)
     return;
   if (isFunctionDecl()) {
     if (ThisDeclInfo->ResultType->isVoidType()) {
@@ -440,7 +444,7 @@ void Sema::checkReturnsCommand(const BlockCommandComment *Command) {
       }
       Diag(Command->getLocation(),
            diag::warn_doc_returns_attached_to_a_void_function)
-        << Command->getCommandName()
+        << Command->getCommandName(Traits)
         << DiagKind
         << Command->getSourceRange();
     }
@@ -448,20 +452,20 @@ void Sema::checkReturnsCommand(const BlockCommandComment *Command) {
   }
   Diag(Command->getLocation(),
        diag::warn_doc_returns_not_attached_to_a_function_decl)
-    << Command->getCommandName()
+    << Command->getCommandName(Traits)
     << Command->getSourceRange();
 }
 
 void Sema::checkBlockCommandDuplicate(const BlockCommandComment *Command) {
-  StringRef Name = Command->getCommandName();
+  const CommandInfo *Info = Traits.getCommandInfo(Command->getCommandID());
   const BlockCommandComment *PrevCommand = NULL;
-  if (Traits.isBriefCommand(Name)) {
+  if (Info->IsBriefCommand) {
     if (!BriefCommand) {
       BriefCommand = Command;
       return;
     }
     PrevCommand = BriefCommand;
-  } else if (Traits.isReturnsCommand(Name)) {
+  } else if (Info->IsReturnsCommand) {
     if (!ReturnsCommand) {
       ReturnsCommand = Command;
       return;
@@ -471,18 +475,20 @@ void Sema::checkBlockCommandDuplicate(const BlockCommandComment *Command) {
     // We don't want to check this command for duplicates.
     return;
   }
+  StringRef CommandName = Command->getCommandName(Traits);
+  StringRef PrevCommandName = PrevCommand->getCommandName(Traits);
   Diag(Command->getLocation(), diag::warn_doc_block_command_duplicate)
-      << Name
+      << CommandName
       << Command->getSourceRange();
-  if (Name == PrevCommand->getCommandName())
+  if (CommandName == PrevCommandName)
     Diag(PrevCommand->getLocation(), diag::note_doc_block_command_previous)
-        << PrevCommand->getCommandName()
-        << Command->getSourceRange();
+        << PrevCommandName
+        << PrevCommand->getSourceRange();
   else
     Diag(PrevCommand->getLocation(),
          diag::note_doc_block_command_previous_alias)
-        << PrevCommand->getCommandName()
-        << Name;
+        << PrevCommandName
+        << CommandName;
 }
 
 void Sema::resolveParamCommandIndexes(const FullComment *FC) {
@@ -740,7 +746,7 @@ StringRef Sema::correctTypoInTParamReference(
 
 InlineCommandComment::RenderKind
 Sema::getInlineCommandRenderKind(StringRef Name) const {
-  assert(Traits.isInlineCommand(Name));
+  assert(Traits.getCommandInfo(Name)->IsInlineCommand);
 
   return llvm::StringSwitch<InlineCommandComment::RenderKind>(Name)
       .Case("b", InlineCommandComment::RenderBold)
diff --git a/lib/AST/RawCommentList.cpp b/lib/AST/RawCommentList.cpp
index ae5740e5b9..28ef6112b8 100644
--- a/lib/AST/RawCommentList.cpp
+++ b/lib/AST/RawCommentList.cpp
@@ -143,10 +143,10 @@ const char *RawComment::extractBriefText(const ASTContext &Context) const {
   // a separate allocator for all temporary stuff.
   llvm::BumpPtrAllocator Allocator;
 
-  comments::CommandTraits Traits;
-  comments::Lexer L(Allocator, Traits, Range.getBegin(),
+  comments::Lexer L(Allocator, Context.getCommentCommandTraits(),
+                    Range.getBegin(),
                     RawText.begin(), RawText.end());
-  comments::BriefParser P(L, Traits);
+  comments::BriefParser P(L, Context.getCommentCommandTraits());
 
   const std::string Result = P.Parse();
   const unsigned BriefTextLength = Result.size();
@@ -163,15 +163,16 @@ comments::FullComment *RawComment::parse(const ASTContext &Context,
   // Make sure that RawText is valid.
   getRawText(Context.getSourceManager());
 
-  comments::CommandTraits Traits;
-  comments::Lexer L(Context.getAllocator(), Traits,
+  comments::Lexer L(Context.getAllocator(), Context.getCommentCommandTraits(),
                     getSourceRang