Unified token breaking logic: support for line comments.

Summary:
Added BreakableLineComment, moved common code from
BreakableBlockComment to newly added BreakableComment. As a side-effect of the
rewrite, found another problem with escaped newlines and had to change
code which removes trailing whitespace from line comments not to break after
this patch.

Reviewers: klimek, djasper

Reviewed By: klimek

CC: cfe-commits

Differential Revision: http://llvm-reviews.chandlerc.com/D682

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@179693 91177308-0d34-0410-b5e6-96231b3b80d8

1 files changed, 15 insertions, 22 deletions

diff --git a/lib/Format/Format.cpp b/lib/Format/Format.cpp
index a847d4ef9b..5c064a6df9 100644
--- a/lib/Format/Format.cpp
+++ b/lib/Format/Format.cpp
@@ -617,54 +617,47 @@ private:
     unsigned StartColumn = State.Column - Current.FormatTok.TokenLength;
     if (Current.is(tok::string_literal)) {
       // Only break up default narrow strings.
-      const char *LiteralData = Current.FormatTok.Tok.getLiteralData();
+      const char *LiteralData = SourceMgr.getCharacterData(
+          Current.FormatTok.getStartOfNonWhitespace());
       if (!LiteralData || *LiteralData != '"')
         return 0;
 
-      Token.reset(new BreakableStringLiteral(Current.FormatTok, StartColumn));
+      Token.reset(new BreakableStringLiteral(SourceMgr, Current.FormatTok,
+                                             StartColumn));
     } else if (Current.Type == TT_BlockComment) {
       BreakableBlockComment *BBC =
           new BreakableBlockComment(SourceMgr, Current, StartColumn);
       if (!DryRun)
         BBC->alignLines(Whitespaces);
       Token.reset(BBC);
+    } else if (Current.Type == TT_LineComment) {
+      Token.reset(new BreakableLineComment(SourceMgr, Current, StartColumn));
     } else {
       return 0;
     }
 
-    if (Token->getPrefixLength() + Token->getSuffixLength(0) >
-        getColumnLimit()) {
-      return 0;
-    }
-
     bool BreakInserted = false;
     unsigned Penalty = 0;
     for (unsigned LineIndex = 0; LineIndex < Token->getLineCount();
          ++LineIndex) {
-      unsigned TokenLineSize = Token->getLineSize(LineIndex);
       unsigned TailOffset = 0;
       unsigned RemainingLength =
           Token->getLineLengthAfterSplit(LineIndex, TailOffset);
       while (RemainingLength > getColumnLimit()) {
-        unsigned DecorationLength =
-            RemainingLength - (TokenLineSize - TailOffset);
-        if (DecorationLength + 1 > getColumnLimit()) {
-          // Can't reduce line length by splitting here.
-          break;
-        }
         BreakableToken::Split Split =
             Token->getSplit(LineIndex, TailOffset, getColumnLimit());
         if (Split.first == StringRef::npos)
           break;
         assert(Split.first != 0);
+        unsigned NewRemainingLength = Token->getLineLengthAfterSplit(
+            LineIndex, TailOffset + Split.first + Split.second);
+        if (NewRemainingLength >= RemainingLength)
+          break;
         if (!DryRun) {
           Token->insertBreak(LineIndex, TailOffset, Split, Line.InPPDirective,
                              Whitespaces);
         }
         TailOffset += Split.first + Split.second;
-        unsigned NewRemainingLength =
-            Token->getLineLengthAfterSplit(LineIndex, TailOffset);
-        assert(NewRemainingLength < RemainingLength);
         RemainingLength = NewRemainingLength;
         Penalty += Style.PenaltyExcessCharacter;
         BreakInserted = true;
@@ -925,6 +918,11 @@ public:
     // Now FormatTok is the next non-whitespace token.
     FormatTok.TokenLength = Text.size();
 
+    if (FormatTok.Tok.is(tok::comment)) {
+      FormatTok.TrailingWhiteSpaceLength = Text.size() - Text.rtrim().size();
+      FormatTok.TokenLength -= FormatTok.TrailingWhiteSpaceLength;
+    }
+
     // In case the token starts with escaped newlines, we want to
     // take them into account as whitespace - this pattern is quite frequent
     // in macro definitions.
@@ -951,11 +949,6 @@ public:
       GreaterStashed = true;
     }
 
-    // If we reformat comments, we remove trailing whitespace. Update the length
-    // accordingly.
-    if (FormatTok.Tok.is(tok::comment))
-      FormatTok.TokenLength = Text.rtrim().size();
-
     return FormatTok;
   }