Remove unsigned and a pointer from a comment token (so that each token can have only one semantic string value attached to it), at a cost of adding an additional token.

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@159270 91177308-0d34-0410-b5e6-96231b3b80d8

3 files changed, 67 insertions, 52 deletions

diff --git a/include/clang/AST/CommentLexer.h b/include/clang/AST/CommentLexer.h
index 7f7ae62758..1a43341b88 100644
--- a/include/clang/AST/CommentLexer.h
+++ b/include/clang/AST/CommentLexer.h
@@ -34,7 +34,8 @@ enum TokenKind {
   verbatim_block_begin,
   verbatim_block_line,
   verbatim_block_end,
-  verbatim_line,
+  verbatim_line_name,
+  verbatim_line_text,
   html_tag_open,      // <tag
   html_ident,         // attr
   html_equals,        // =
@@ -75,10 +76,6 @@ class Token {
   const char *TextPtr1;
   unsigned TextLen1;
 
-  /// Contains text value associated with a token.
-  const char *TextPtr2;
-  unsigned TextLen2;
-
 public:
   SourceLocation getLocation() const LLVM_READONLY { return Loc; }
   void setLocation(SourceLocation SL) { Loc = SL; }
@@ -138,25 +135,25 @@ public:
 
   /// Returns the name of verbatim line command.
   StringRef getVerbatimLineName() const LLVM_READONLY {
-    assert(is(tok::verbatim_line));
+    assert(is(tok::verbatim_line_name));
     return StringRef(TextPtr1, TextLen1);
   }
 
   void setVerbatimLineName(StringRef Name) {
-    assert(is(tok::verbatim_line));
+    assert(is(tok::verbatim_line_name));
     TextPtr1 = Name.data();
     TextLen1 = Name.size();
   }
 
   StringRef getVerbatimLineText() const LLVM_READONLY {
-    assert(is(tok::verbatim_line));
-    return StringRef(TextPtr2, TextLen2);
+    assert(is(tok::verbatim_line_text));
+    return StringRef(TextPtr1, TextLen1);
   }
 
   void setVerbatimLineText(StringRef Text) {
-    assert(is(tok::verbatim_line));
-    TextPtr2 = Text.data();
-    TextLen2 = Text.size();
+    assert(is(tok::verbatim_line_text));
+    TextPtr1 = Text.data();
+    TextLen1 = Text.size();
   }
 
   StringRef getHTMLTagOpenName() const LLVM_READONLY {
@@ -245,6 +242,10 @@ private:
     /// decorations.
     LS_VerbatimBlockBody,
 
+    /// Finished lexing verbatim line beginning command, will lex text (one
+    /// line).
+    LS_VerbatimLineText,
+
     /// Finished lexing \verbatim <TAG \endverbatim part, lexing tag attributes.
     LS_HTMLOpenTag
   };
@@ -292,8 +293,6 @@ private:
 #ifndef NDEBUG
     Result.TextPtr1 = "<UNSET>";
     Result.TextLen1 = 7;
-    Result.TextPtr2 = "<UNSET>";
-    Result.TextLen2 = 7;
 #endif
     BufferPtr = TokEnd;
   }
@@ -320,7 +319,9 @@ private:
 
   void lexVerbatimBlockBody(Token &T);
 
-  void lexVerbatimLine(Token &T, const char *TextBegin);
+  void setupAndLexVerbatimLine(Token &T, const char *TextBegin);
+
+  void lexVerbatimLineText(Token &T);
 
   void setupAndLexHTMLOpenTag(Token &T);
 
diff --git a/lib/AST/CommentLexer.cpp b/lib/AST/CommentLexer.cpp
index f9acd2ac91..0b76050ff0 100644
--- a/lib/AST/CommentLexer.cpp
+++ b/lib/AST/CommentLexer.cpp
@@ -264,6 +264,9 @@ void Lexer::lexCommentText(Token &T) {
   case LS_VerbatimBlockBody:
     lexVerbatimBlockBody(T);
     return;
+  case LS_VerbatimLineText:
+    lexVerbatimLineText(T);
+    return;
   case LS_HTMLOpenTag:
     lexHTMLOpenTag(T);
     return;
@@ -333,7 +336,7 @@ void Lexer::lexCommentText(Token &T) {
           return;
         }
         if (isVerbatimLineCommand(CommandName)) {
-          lexVerbatimLine(T, TokenPtr);
+          setupAndLexVerbatimLine(T, TokenPtr);
           return;
         }
         formTokenWithChars(T, TokenPtr, tok::command);
@@ -444,16 +447,24 @@ void Lexer::lexVerbatimBlockBody(Token &T) {
   lexVerbatimBlockFirstLine(T);
 }
 
-void Lexer::lexVerbatimLine(Token &T, const char *TextBegin) {
-  // Extract current line.
-  const char *Newline = findNewline(BufferPtr, CommentEnd);
-
+void Lexer::setupAndLexVerbatimLine(Token &T, const char *TextBegin) {
   const StringRef Name(BufferPtr + 1, TextBegin - BufferPtr - 1);
-  const StringRef Text(TextBegin, Newline - TextBegin);
-
-  formTokenWithChars(T, Newline, tok::verbatim_line);
+  formTokenWithChars(T, TextBegin, tok::verbatim_line_name);
   T.setVerbatimLineName(Name);
+
+  State = LS_VerbatimLineText;
+}
+
+void Lexer::lexVerbatimLineText(Token &T) {
+  assert(State == LS_VerbatimLineText);
+
+  // Extract current line.
+  const char *Newline = findNewline(BufferPtr, CommentEnd);
+  const StringRef Text(BufferPtr, Newline - BufferPtr);
+  formTokenWithChars(T, Newline, tok::verbatim_line_text);
   T.setVerbatimLineText(Text);
+
+  State = LS_Normal;
 }
 
 void Lexer::setupAndLexHTMLOpenTag(Token &T) {
diff --git a/unittests/AST/CommentLexer.cpp b/unittests/AST/CommentLexer.cpp
index b73d16ca00..a1e2427880 100644
--- a/unittests/AST/CommentLexer.cpp
+++ b/unittests/AST/CommentLexer.cpp
@@ -588,15 +588,14 @@ TEST_F(CommentLexerTest, VerbatimLine1) {
 
     ASSERT_EQ(4U, Toks.size());
 
-    ASSERT_EQ(tok::text,          Toks[0].getKind());
-    ASSERT_EQ(StringRef(" "),     Toks[0].getText());
+    ASSERT_EQ(tok::text,               Toks[0].getKind());
+    ASSERT_EQ(StringRef(" "),          Toks[0].getText());
 
-    ASSERT_EQ(tok::verbatim_line, Toks[1].getKind());
-    ASSERT_EQ(StringRef("fn"),    Toks[1].getVerbatimLineName());
-    ASSERT_EQ(StringRef(""),      Toks[1].getVerbatimLineText());
+    ASSERT_EQ(tok::verbatim_line_name, Toks[1].getKind());
+    ASSERT_EQ(StringRef("fn"),         Toks[1].getVerbatimLineName());
 
-    ASSERT_EQ(tok::newline,       Toks[2].getKind());
-    ASSERT_EQ(tok::newline,       Toks[3].getKind());
+    ASSERT_EQ(tok::newline,            Toks[2].getKind());
+    ASSERT_EQ(tok::newline,            Toks[3].getKind());
   }
 }
 
@@ -612,18 +611,20 @@ TEST_F(CommentLexerTest, VerbatimLine2) {
 
     lexString(Sources[i], Toks);
 
-    ASSERT_EQ(4U, Toks.size());
+    ASSERT_EQ(5U, Toks.size());
 
-    ASSERT_EQ(tok::text,          Toks[0].getKind());
-    ASSERT_EQ(StringRef(" "),     Toks[0].getText());
+    ASSERT_EQ(tok::text,               Toks[0].getKind());
+    ASSERT_EQ(StringRef(" "),          Toks[0].getText());
+
+    ASSERT_EQ(tok::verbatim_line_name, Toks[1].getKind());
+    ASSERT_EQ(StringRef("fn"),         Toks[1].getVerbatimLineName());
 
-    ASSERT_EQ(tok::verbatim_line, Toks[1].getKind());
-    ASSERT_EQ(StringRef("fn"),    Toks[1].getVerbatimLineName());
+    ASSERT_EQ(tok::verbatim_line_text, Toks[2].getKind());
     ASSERT_EQ(StringRef(" void *foo(const char *zzz = \"\\$\");"),
-                                  Toks[1].getVerbatimLineText());
+                                       Toks[2].getVerbatimLineText());
 
-    ASSERT_EQ(tok::newline,       Toks[2].getKind());
-    ASSERT_EQ(tok::newline,       Toks[3].getKind());
+    ASSERT_EQ(tok::newline,            Toks[3].getKind());
+    ASSERT_EQ(tok::newline,            Toks[4].getKind());
   }
 }
 
@@ -638,26 +639,28 @@ TEST_F(CommentLexerTest, VerbatimLine3) {
 
   lexString(Source, Toks);
 
-  ASSERT_EQ(8U, Toks.size());
+  ASSERT_EQ(9U, Toks.size());
 
-  ASSERT_EQ(tok::text,          Toks[0].getKind());
-  ASSERT_EQ(StringRef(" "),     Toks[0].getText());
+  ASSERT_EQ(tok::text,               Toks[0].getKind());
+  ASSERT_EQ(StringRef(" "),          Toks[0].getText());
+
+  ASSERT_EQ(tok::verbatim_line_name, Toks[1].getKind());
+  ASSERT_EQ(StringRef("fn"),         Toks[1].getVerbatimLineName());
 
-  ASSERT_EQ(tok::verbatim_line, Toks[1].getKind());
-  ASSERT_EQ(StringRef("fn"),    Toks[1].getVerbatimLineName());
+  ASSERT_EQ(tok::verbatim_line_text, Toks[2].getKind());
   ASSERT_EQ(StringRef(" void *foo(const char *zzz = \"\\$\");"),
-                                Toks[1].getVerbatimLineText());
-  ASSERT_EQ(tok::newline,       Toks[2].getKind());
+                                     Toks[2].getVerbatimLineText());
+  ASSERT_EQ(tok::newline,            Toks[3].getKind());
 
-  ASSERT_EQ(tok::text,          Toks[3].getKind());
-  ASSERT_EQ(StringRef(" Meow"), Toks[3].getText());
-  ASSERT_EQ(tok::newline,       Toks[4].getKind());
+  ASSERT_EQ(tok::text,               Toks[4].getKind());
+  ASSERT_EQ(StringRef(" Meow"),      Toks[4].getText());
+  ASSERT_EQ(tok::newline,            Toks[5].getKind());
 
-  ASSERT_EQ(tok::text,          Toks[5].getKind());
-  ASSERT_EQ(StringRef(" "),     Toks[5].getText());
+  ASSERT_EQ(tok::text,               Toks[6].getKind());
+  ASSERT_EQ(StringRef(" "),          Toks[6].getText());
 
-  ASSERT_EQ(tok::newline,       Toks[6].getKind());
-  ASSERT_EQ(tok::newline,       Toks[7].getKind());
+  ASSERT_EQ(tok::newline,            Toks[7].getKind());
+  ASSERT_EQ(tok::newline,            Toks[8].getKind());
 }
 
 TEST_F(CommentLexerTest, HTML1) {