Move the token annotator into separate files.

No functional changes. Also removed experimental-warning from all of
clang-format's files, as it is no longer accurate.

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@173830 91177308-0d34-0410-b5e6-96231b3b80d8

1 files changed, 1 insertions, 1030 deletions

diff --git a/lib/Format/Format.cpp b/lib/Format/Format.cpp
index ecd6f5d385..9dfde34e8b 100644
--- a/lib/Format/Format.cpp
+++ b/lib/Format/Format.cpp
@@ -11,13 +11,11 @@
 /// \brief This file implements functions declared in Format.h. This will be
 /// split into separate files as we go.
 ///
-/// This is EXPERIMENTAL code under heavy development. It is not in a state yet,
-/// where it can be used to format real code.
-///
 //===----------------------------------------------------------------------===//
 
 #define DEBUG_TYPE "format-formatter"
 
+#include "TokenAnnotator.h"
 #include "UnwrappedLineParser.h"
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/OperatorPrecedence.h"
@@ -34,140 +32,6 @@
 namespace clang {
 namespace format {
 
-enum TokenType {
-  TT_BinaryOperator,
-  TT_BlockComment,
-  TT_CastRParen,
-  TT_ConditionalExpr,
-  TT_CtorInitializerColon,
-  TT_ImplicitStringLiteral,
-  TT_LineComment,
-  TT_ObjCBlockLParen,
-  TT_ObjCDecl,
-  TT_ObjCMethodSpecifier,
-  TT_ObjCMethodExpr,
-  TT_ObjCProperty,
-  TT_OverloadedOperator,
-  TT_PointerOrReference,
-  TT_PureVirtualSpecifier,
-  TT_RangeBasedForLoopColon,
-  TT_StartOfName,
-  TT_TemplateCloser,
-  TT_TemplateOpener,
-  TT_TrailingUnaryOperator,
-  TT_UnaryOperator,
-  TT_Unknown
-};
-
-enum LineType {
-  LT_Invalid,
-  LT_Other,
-  LT_BuilderTypeCall,
-  LT_PreprocessorDirective,
-  LT_VirtualFunctionDecl,
-  LT_ObjCDecl, // An @interface, @implementation, or @protocol line.
-  LT_ObjCMethodDecl,
-  LT_ObjCProperty // An @property line.
-};
-
-class AnnotatedToken {
-public:
-  explicit AnnotatedToken(const FormatToken &FormatTok)
-      : FormatTok(FormatTok), Type(TT_Unknown), SpaceRequiredBefore(false),
-        CanBreakBefore(false), MustBreakBefore(false),
-        ClosesTemplateDeclaration(false), MatchingParen(NULL),
-        ParameterCount(1), Parent(NULL) {
-  }
-
-  bool is(tok::TokenKind Kind) const { return FormatTok.Tok.is(Kind); }
-  bool isNot(tok::TokenKind Kind) const { return FormatTok.Tok.isNot(Kind); }
-
-  bool isObjCAtKeyword(tok::ObjCKeywordKind Kind) const {
-    return FormatTok.Tok.isObjCAtKeyword(Kind);
-  }
-
-  FormatToken FormatTok;
-
-  TokenType Type;
-
-  bool SpaceRequiredBefore;
-  bool CanBreakBefore;
-  bool MustBreakBefore;
-
-  bool ClosesTemplateDeclaration;
-
-  AnnotatedToken *MatchingParen;
-
-  /// \brief Number of parameters, if this is "(", "[" or "<".
-  ///
-  /// This is initialized to 1 as we don't need to distinguish functions with
-  /// 0 parameters from functions with 1 parameter. Thus, we can simply count
-  /// the number of commas.
-  unsigned ParameterCount;
-
-  /// \brief The total length of the line up to and including this token.
-  unsigned TotalLength;
-
-  /// \brief Penalty for inserting a line break before this token.
-  unsigned SplitPenalty;
-
-  std::vector<AnnotatedToken> Children;
-  AnnotatedToken *Parent;
-
-  const AnnotatedToken *getPreviousNoneComment() const {
-    AnnotatedToken *Tok = Parent;
-    while (Tok != NULL && Tok->is(tok::comment))
-      Tok = Tok->Parent;
-    return Tok;
-  }
-};
-
-class AnnotatedLine {
-public:
-  AnnotatedLine(const UnwrappedLine &Line)
-      : First(Line.Tokens.front()), Level(Line.Level),
-        InPPDirective(Line.InPPDirective),
-        MustBeDeclaration(Line.MustBeDeclaration) {
-    assert(!Line.Tokens.empty());
-    AnnotatedToken *Current = &First;
-    for (std::list<FormatToken>::const_iterator I = ++Line.Tokens.begin(),
-                                                E = Line.Tokens.end();
-         I != E; ++I) {
-      Current->Children.push_back(AnnotatedToken(*I));
-      Current->Children[0].Parent = Current;
-      Current = &Current->Children[0];
-    }
-    Last = Current;
-  }
-  AnnotatedLine(const AnnotatedLine &Other)
-      : First(Other.First), Type(Other.Type), Level(Other.Level),
-        InPPDirective(Other.InPPDirective),
-        MustBeDeclaration(Other.MustBeDeclaration) {
-    Last = &First;
-    while (!Last->Children.empty()) {
-      Last->Children[0].Parent = Last;
-      Last = &Last->Children[0];
-    }
-  }
-
-  AnnotatedToken First;
-  AnnotatedToken *Last;
-
-  LineType Type;
-  unsigned Level;
-  bool InPPDirective;
-  bool MustBeDeclaration;
-};
-
-static prec::Level getPrecedence(const AnnotatedToken &Tok) {
-  return getBinOpPrecedence(Tok.FormatTok.Tok.getKind(), true, true);
-}
-
-bool isBinaryOperator(const AnnotatedToken &Tok) {
-  // Comma is a binary operator, but does not behave as such wrt. formatting.
-  return getPrecedence(Tok) > prec::Comma;
-}
-
 FormatStyle getLLVMStyle() {
   FormatStyle LLVMStyle;
   LLVMStyle.ColumnLimit = 80;
@@ -338,15 +202,6 @@ private:
   tooling::Replacements Replaces;
 };
 
-/// \brief Returns if a token is an Objective-C selector name.
-///
-/// For example, "bar" is a selector name in [foo bar:(4 + 5)].
-static bool isObjCSelectorName(const AnnotatedToken &Tok) {
-  return Tok.is(tok::identifier) && !Tok.Children.empty() &&
-         Tok.Children[0].is(tok::colon) &&
-         Tok.Children[0].Type == TT_ObjCMethodExpr;
-}
-
 class UnwrappedLineFormatter {
 public:
   UnwrappedLineFormatter(const FormatStyle &Style, SourceManager &SourceMgr,
@@ -813,890 +668,6 @@ private:
   OptimizationParameters Parameters;
 };
 
-/// \brief Determines extra information about the tokens comprising an
-/// \c UnwrappedLine.
-class TokenAnnotator {
-public:
-  TokenAnnotator(const FormatStyle &Style, SourceManager &SourceMgr, Lexer &Lex,
-                 AnnotatedLine &Line)
-      : Style(Style), SourceMgr(SourceMgr), Lex(Lex), Line(Line) {
-  }
-
-  /// \brief A parser that gathers additional information about tokens.
-  ///
-  /// The \c TokenAnnotator tries to matches parenthesis and square brakets and
-  /// store a parenthesis levels. It also tries to resolve matching "<" and ">"
-  /// into template parameter lists.
-  class AnnotatingParser {
-  public:
-    AnnotatingParser(AnnotatedToken &RootToken)
-        : CurrentToken(&RootToken), KeywordVirtualFound(false),
-          ColonIsObjCMethodExpr(false), ColonIsForRangeExpr(false) {
-    }
-
-    /// \brief A helper class to manage AnnotatingParser::ColonIsObjCMethodExpr.
-    struct ObjCSelectorRAII {
-      AnnotatingParser &P;
-      bool ColonWasObjCMethodExpr;
-
-      ObjCSelectorRAII(AnnotatingParser &P)
-          : P(P), ColonWasObjCMethodExpr(P.ColonIsObjCMethodExpr) {
-      }
-
-      ~ObjCSelectorRAII() { P.ColonIsObjCMethodExpr = ColonWasObjCMethodExpr; }
-
-      void markStart(AnnotatedToken &Left) {
-        P.ColonIsObjCMethodExpr = true;
-        Left.Type = TT_ObjCMethodExpr;
-      }
-
-      void markEnd(AnnotatedToken &Right) { Right.Type = TT_ObjCMethodExpr; }
-    };
-
-    bool parseAngle() {
-      if (CurrentToken == NULL)
-        return false;
-      AnnotatedToken *Left = CurrentToken->Parent;
-      while (CurrentToken != NULL) {
-        if (CurrentToken->is(tok::greater)) {
-          Left->MatchingParen = CurrentToken;
-          CurrentToken->MatchingParen = Left;
-          CurrentToken->Type = TT_TemplateCloser;
-          next();
-          return true;
-        }
-        if (CurrentToken->is(tok::r_paren) || CurrentToken->is(tok::r_square) ||
-            CurrentToken->is(tok::r_brace))
-          return false;
-        if (CurrentToken->is(tok::pipepipe) || CurrentToken->is(tok::ampamp) ||
-            CurrentToken->is(tok::question) || CurrentToken->is(tok::colon))
-          return false;
-        if (CurrentToken->is(tok::comma))
-          ++Left->ParameterCount;
-        if (!consumeToken())
-          return false;
-      }
-      return false;
-    }
-
-    bool parseParens(bool LookForDecls = false) {
-      if (CurrentToken == NULL)
-        return false;
-      bool StartsObjCMethodExpr = false;
-      AnnotatedToken *Left = CurrentToken->Parent;
-      if (CurrentToken->is(tok::caret)) {
-        // ^( starts a block.
-        Left->Type = TT_ObjCBlockLParen;
-      } else if (AnnotatedToken *MaybeSel = Left->Parent) {
-        // @selector( starts a selector.
-        if (MaybeSel->isObjCAtKeyword(tok::objc_selector) && MaybeSel->Parent &&
-            MaybeSel->Parent->is(tok::at)) {
-          StartsObjCMethodExpr = true;
-        }
-      }
-
-      ObjCSelectorRAII objCSelector(*this);
-      if (StartsObjCMethodExpr)
-        objCSelector.markStart(*Left);
-
-      while (CurrentToken != NULL) {
-        // LookForDecls is set when "if (" has been seen. Check for
-        // 'identifier' '*' 'identifier' followed by not '=' -- this
-        // '*' has to be a binary operator but determineStarAmpUsage() will
-        // categorize it as an unary operator, so set the right type here.
-        if (LookForDecls && !CurrentToken->Children.empty()) {
-          AnnotatedToken &Prev = *CurrentToken->Parent;
-          AnnotatedToken &Next = CurrentToken->Children[0];
-          if (Prev.Parent->is(tok::identifier) &&
-              (Prev.is(tok::star) || Prev.is(tok::amp)) &&
-              CurrentToken->is(tok::identifier) && Next.isNot(tok::equal)) {
-            Prev.Type = TT_BinaryOperator;
-            LookForDecls = false;
-          }
-        }
-
-        if (CurrentToken->is(tok::r_paren)) {
-          Left->MatchingParen = CurrentToken;
-          CurrentToken->MatchingParen = Left;
-
-          if (StartsObjCMethodExpr)
-            objCSelector.markEnd(*CurrentToken);
-
-          next();
-          return true;
-        }
-        if (CurrentToken->is(tok::r_square) || CurrentToken->is(tok::r_brace))
-          return false;
-        if (CurrentToken->is(tok::comma))
-          ++Left->ParameterCount;
-        if (!consumeToken())
-          return false;
-      }
-      return false;
-    }
-
-    bool parseSquare() {
-      if (!CurrentToken)
-        return false;
-
-      // A '[' could be an index subscript (after an indentifier or after
-      // ')' or ']'), or it could be the start of an Objective-C method
-      // expression.
-      AnnotatedToken *Left = CurrentToken->Parent;
-      bool StartsObjCMethodExpr =
-          !Left->Parent || Left->Parent->is(tok::colon) ||
-          Left->Parent->is(tok::l_square) || Left->Parent->is(tok::l_paren) ||
-          Left->Parent->is(tok::kw_return) || Left->Parent->is(tok::kw_throw) ||
-          getBinOpPrecedence(Left->Parent->FormatTok.Tok.getKind(), true,
-                             true) > prec::Unknown;
-
-      ObjCSelectorRAII objCSelector(*this);
-      if (StartsObjCMethodExpr)
-        objCSelector.markStart(*Left);
-
-      while (CurrentToken != NULL) {
-        if (CurrentToken->is(tok::r_square)) {
-          if (!CurrentToken->Children.empty() &&
-              CurrentToken->Children[0].is(tok::l_paren)) {
-            // An ObjC method call can't be followed by an open parenthesis.
-            // FIXME: Do we incorrectly label ":" with this?
-            StartsObjCMethodExpr = false;
-            Left->Type = TT_Unknown;
-          }
-          if (StartsObjCMethodExpr)
-            objCSelector.markEnd(*CurrentToken);
-          Left->MatchingParen = CurrentToken;
-          CurrentToken->MatchingParen = Left;
-          next();
-          return true;
-        }
-        if (CurrentToken->is(tok::r_paren) || CurrentToken->is(tok::r_brace))
-          return false;
-        if (CurrentToken->is(tok::comma))
-          ++Left->ParameterCount;
-        if (!consumeToken())
-          return false;
-      }
-      return false;
-    }
-
-    bool parseBrace() {
-      // Lines are fine to end with '{'.
-      if (CurrentToken == NULL)
-        return true;
-      AnnotatedToken *Left = CurrentToken->Parent;
-      while (CurrentToken != NULL) {
-        if (CurrentToken->is(tok::r_brace)) {
-          Left->MatchingParen = CurrentToken;
-          CurrentToken->MatchingParen = Left;
-          next();
-          return true;
-        }
-        if (CurrentToken->is(tok::r_paren) || CurrentToken->is(tok::r_square))
-          return false;
-        if (!consumeToken())
-          return false;
-      }
-      return true;
-    }
-
-    bool parseConditional() {
-      while (CurrentToken != NULL) {
-        if (CurrentToken->is(tok::colon)) {
-          CurrentToken->Type = TT_ConditionalExpr;
-          next();
-          return true;
-        }
-        if (!consumeToken())
-          return false;
-      }
-      return false;
-    }
-
-    bool parseTemplateDeclaration() {
-      if (CurrentToken != NULL && CurrentToken->is(tok::less)) {
-        CurrentToken->Type = TT_TemplateOpener;
-        next();
-        if (!parseAngle())
-          return false;
-        CurrentToken->Parent->ClosesTemplateDeclaration = true;
-        return true;
-      }
-      return false;
-    }
-
-    bool consumeToken() {
-      AnnotatedToken *Tok = CurrentToken;
-      next();
-      switch (Tok->FormatTok.Tok.getKind()) {
-      case tok::plus:
-      case tok::minus:
-        // At the start of the line, +/- specific ObjectiveC method
-        // declarations.
-        if (Tok->Parent == NULL)
-          Tok->Type = TT_ObjCMethodSpecifier;
-        break;
-      case tok::colon:
-        // Colons from ?: are handled in parseConditional().
-        if (Tok->Parent->is(tok::r_paren))
-          Tok->Type = TT_CtorInitializerColon;
-        else if (ColonIsObjCMethodExpr)
-          Tok->Type = TT_ObjCMethodExpr;
-        else if (ColonIsForRangeExpr)
-          Tok->Type = TT_RangeBasedForLoopColon;
-        break;
-      case tok::kw_if:
-      case tok::kw_while:
-        if (CurrentToken != NULL && CurrentToken->is(tok::l_paren)) {
-          next();
-          if (!parseParens(/*LookForDecls=*/ true))
-            return false;
-        }
-        break;
-      case tok::kw_for:
-        ColonIsForRangeExpr = true;
-        next();
-        if (!parseParens())
-          return false;
-        break;
-      case tok::l_paren:
-        if (!parseParens())
-          return false;
-        break;
-      case tok::l_square:
-        if (!parseSquare())
-          return false;
-        break;
-      case tok::l_brace:
-        if (!parseBrace())
-          return false;
-        break;
-      case tok::less:
-        if (parseAngle())
-          Tok->Type = TT_TemplateOpener;
-        else {
-          Tok->Type = TT_BinaryOperator;
-          CurrentToken = Tok;
-          next();
-        }
-        break;
-      case tok::r_paren:
-      case tok::r_square:
-        return false;
-      case tok::r_brace:
-        // Lines can start with '}'.
-        if (Tok->Parent != NULL)
-          return false;
-        break;
-      case tok::greater:
-        Tok->Type = TT_BinaryOperator;
-        break;
-      case tok::kw_operator:
-        if (CurrentToken != NULL && CurrentToken->is(tok::l_paren)) {
-          CurrentToken->Type = TT_OverloadedOperator;
-          next();
-          if (CurrentToken != NULL && CurrentToken->is(tok::r_paren)) {
-            CurrentToken->Type = TT_OverloadedOperator;
-            next();
-          }
-        } else {
-          while (CurrentToken != NULL && CurrentToken->isNot(tok::l_paren)) {
-            CurrentToken->Type = TT_OverloadedOperator;
-            next();
-          }
-        }
-        break;
-      case tok::question:
-        parseConditional();
-        break;
-      case tok::kw_template:
-        parseTemplateDeclaration();
-        break;
-      default:
-        break;
-      }
-      return true;
-    }
-
-    void parseIncludeDirective() {
-      next();
-      if (CurrentToken != NULL && CurrentToken->is(tok::less)) {
-        next();
-        while (CurrentToken != NULL) {
-          if (CurrentToken->isNot(tok::comment) ||
-              !CurrentToken->Children.empty())
-            CurrentToken->Type = TT_ImplicitStringLiteral;
-          next();
-        }
-      } else {
-        while (CurrentToken != NULL) {
-          next();
-        }
-      }
-    }
-
-    void parseWarningOrError() {
-      next();
-      // We still want to format the whitespace left of the first token of the
-      // warning or error.
-      next();
-      while (CurrentToken != NULL) {
-        CurrentToken->Type = TT_ImplicitStringLiteral;
-        next();
-      }
-    }
-
-    void parsePreprocessorDirective() {
-      next();
-      if (CurrentToken == NULL)
-        return;
-      // Hashes in the middle of a line can lead to any strange token
-      // sequence.
-      if (CurrentToken->FormatTok.Tok.getIdentifierInfo() == NULL)
-        return;
-      switch (
-          CurrentToken->FormatTok.Tok.getIdentifierInfo()->getPPKeywordID()) {
-      case tok::pp_include:
-      case tok::pp_import:
-        parseIncludeDirective();
-        break;
-      case tok::pp_error:
-      case tok::pp_warning:
-        parseWarningOrError();
-        break;
-      default:
-        break;
-      }
-    }
-
-    LineType parseLine() {
-      int PeriodsAndArrows = 0;
-      bool CanBeBuilderTypeStmt = true;
-      if (CurrentToken->is(tok::hash)) {
-        parsePreprocessorDirective();
-        return LT_PreprocessorDirective;
-      }
-      while (CurrentToken != NULL) {
-        if (CurrentToken->is(tok::kw_virtual))
-          KeywordVirtualFound = true;
-        if (CurrentToken->is(tok::period) || CurrentToken->is(tok::arrow))
-          ++PeriodsAndArrows;
-        if (getPrecedence(*CurrentToken) > prec::Assignment &&
-            CurrentToken->isNot(tok::less) && CurrentToken->isNot(tok::greater))
-          CanBeBuilderTypeStmt = false;
-        if (!consumeToken())
-          return LT_Invalid;
-      }
-      if (KeywordVirtualFound)
-        return LT_VirtualFunctionDecl;
-
-      // Assume a builder-type call if there are 2 or more "." and "->".
-      if (PeriodsAndArrows >= 2 && CanBeBuilderTypeStmt)
-        return LT_BuilderTypeCall;
-
-      return LT_Other;
-    }
-
-    void next() {
-      if (CurrentToken != NULL && !CurrentToken->Children.empty())
-        CurrentToken = &CurrentToken->Children[0];
-      else
-        CurrentToken = NULL;
-    }
-
-  private:
-    AnnotatedToken *CurrentToken;
-    bool KeywordVirtualFound;
-    bool ColonIsObjCMethodExpr;
-    bool ColonIsForRangeExpr;
-  };
-
-  void calculateExtraInformation(AnnotatedToken &Current) {
-    Current.SpaceRequiredBefore = spaceRequiredBefore(Current);
-
-    if (Current.FormatTok.MustBreakBefore) {
-      Current.MustBreakBefore = true;
-    } else {
-      if (Current.Type == TT_LineComment) {
-        Current.MustBreakBefore = Current.FormatTok.NewlinesBefore > 0;
-      } else if ((Current.Parent->is(tok::comment) &&
-                  Current.FormatTok.NewlinesBefore > 0) ||
-                 (Current.is(tok::string_literal) &&
-                  Current.Parent->is(tok::string_literal))) {
-        Current.MustBreakBefore = true;
-      } else {
-        Current.MustBreakBefore = false;
-      }
-    }
-    Current.CanBreakBefore = Current.MustBreakBefore || canBreakBefore(Current);
-    if (Current.MustBreakBefore)
-      Current.TotalLength = Current.Parent->TotalLength + Style.ColumnLimit;
-    else
-      Current.TotalLength =
-          Current.Parent->TotalLength + Current.FormatTok.TokenLength +
-          (Current.SpaceRequiredBefore ? 1 : 0);
-    // FIXME: Only calculate this if CanBreakBefore is true once static
-    // initializers etc. are sorted out.
-    Current.SplitPenalty = splitPenalty(Current);
-    if (!Current.Children.empty())
-      calculateExtraInformation(Current.Children[0]);
-  }
-
-  void annotate() {
-    AnnotatingParser Parser(Line.First);
-    Line.Type = Parser.parseLine();
-    if (Line.Type == LT_Invalid)
-      return;
-
-    bool LookForFunctionName = Line.MustBeDeclaration;
-    determineTokenTypes(Line.First, /*IsExpression=*/ false,
-                        LookForFunctionName);
-
-    if (Line.First.Type == TT_ObjCMethodSpecifier)
-      Line.Type = LT_ObjCMethodDecl;
-    else if (Line.First.Type == TT_ObjCDecl)
-      Line.Type = LT_ObjCDecl;
-    else if (Line.First.Type == TT_ObjCProperty)
-      Line.Type = LT_ObjCProperty;
-
-    Line.First.SpaceRequiredBefore = true;
-    Line.First.MustBreakBefore = Line.First.FormatTok.MustBreakBefore;
-    Line.First.CanBreakBefore = Line.First.MustBreakBefore;
-
-    Line.First.TotalLength = Line.First.FormatTok.TokenLength;
-    if (!Line.First.Children.empty())
-      calculateExtraInformation(Line.First.Children[0]);
-  }
-
-private:
-  /// \brief Calculate the penalty for splitting before \c Tok.
-  unsigned splitPenalty(const AnnotatedToken &Tok) {
-    const AnnotatedToken &Left = *Tok.Parent;
-    const AnnotatedToken &Right = Tok;
-
-    if (Left.is(tok::l_brace) && Right.isNot(tok::l_brace))
-      return 50;
-    if (Left.is(tok::equal) && Right.is(tok::l_brace))
-      return 150;
-    if (Left.is(tok::coloncolon))
-      return 500;
-
-    if (Left.Type == TT_RangeBasedForLoopColon)
-      return 5;
-
-    if (Right.is(tok::arrow) || Right.is(tok::period)) {
-      if (Left.is(tok::r_paren) && Line.Type == LT_BuilderTypeCall)
-        return 5; // Should be smaller than breaking at a nested comma.
-      return 150;
-    }
-
-    // In for-loops, prefer breaking at ',' and ';'.
-    if (Line.First.is(tok::kw_for) &&
-        (Left.isNot(tok::comma) && Left.isNot(tok::semi)))
-      return 20;
-
-    if (Left.is(tok::semi) || Left.is(tok::comma))
-      return 0;
-
-    // In Objective-C method expressions, prefer breaking before "param:" over
-    // breaking after it.
-    if (isObjCSelectorName(Right))
-      return 0;
-    if (Right.is(tok::colon) && Right.Type == TT_ObjCMethodExpr)
-      return 20;
-
-    if (Left.is(tok::l_paren))
-      return 20;
-    // FIXME: The penalty for a trailing "<" or "[" being higher than the
-    // penalty for a trainling "(" is a temporary workaround until we can
-    // properly avoid breaking in array subscripts or template parameters.
-    if (Left.is(tok::l_square) || Left.Type == TT_TemplateOpener)
-      return 50;
-
-    if (Left.Type == TT_ConditionalExpr)
-      return prec::Assignment;
-    prec::Level Level = getPrecedence(Left);
-
-    if (Level != prec::Unknown)
-      return Level;
-
-    return 3;
-  }
-
-  void determineTokenTypes(AnnotatedToken &Current, bool IsExpression,
-                           bool LookForFunctionName) {
-    if (getPrecedence(Current) == prec::Assignment) {
-      IsExpression = true;
-      AnnotatedToken *Previous = Current.Parent;
-      while (Previous != NULL) {
-        if (Previous->Type == TT_BinaryOperator &&
-            (Previous->is(tok::star) || Previous->is(tok::amp))) {
-          Previous->Type = TT_PointerOrReference;
-        }
-        Previous = Previous->Parent;
-      }
-    }
-    if (Current.is(tok::kw_return) || Current.is(tok::kw_throw) ||
-        (Current.is(tok::l_paren) && !Line.MustBeDeclaration &&
-         (Current.Parent == NULL || Current.Parent->isNot(tok::kw_for))))
-      IsExpression = true;
-
-    if (Current.Type == TT_Unknown) {
-      if (LookForFunctionName && Current.is(tok::l_paren)) {
-        findFunctionName(&Current);
-        LookForFunctionName = false;
-      } else if (Current.is(tok::star) || Current.is(tok::amp)) {
-        Current.Type = determineStarAmpUsage(Current, IsExpression);
-      } else if (Current.is(tok::minus) || Current.is(tok::plus) ||
-                 Current.is(tok::caret)) {
-        Current.Type = determinePlusMinusCaretUsage(Current);
-      } else if (Current.is(tok::minusminus) || Current.is(tok::plusplus)) {
-        Current.Type = determineIncrementUsage(Current);
-      } else if (Current.is(tok::exclaim)) {
-        Current.Type = TT_UnaryOperator;
-      } else if (isBinaryOperator(Current)) {
-        Current.Type = TT_BinaryOperator;
-      } else if (Current.is(tok::comment)) {
-        std::string Data(Lexer::getSpelling(Current.FormatTok.Tok, SourceMgr,
-                                            Lex.getLangOpts()));
-        if (StringRef(Data).startswith("//"))
-          Current.Type = TT_LineComment;
-        else
-          Current.Type = TT_BlockComment;
-      } else if (Current.is(tok::r_paren) &&
-                 (Current.Parent->Type == TT_PointerOrReference ||
-                  Current.Parent->Type == TT_TemplateCloser) &&
-                 (Current.Children.empty() ||
-                  (Current.Children[0].isNot(tok::equal) &&
-                   Current.Children[0].isNot(tok::semi) &&
-                   Current.Children[0].isNot(tok::l_brace)))) {
-        // FIXME: We need to get smarter and understand more cases of casts.
-        Current.Type = TT_CastRParen;
-      } else if (Current.is(tok::at) && Current.Children.size()) {
-        switch (Current.Children[0].FormatTok.Tok.getObjCKeywordID()) {
-        case tok::objc_interface:
-        case tok::objc_implementation:
-        case tok::objc_protocol:
-          Current.Type = TT_ObjCDecl;
-          break;
-        case tok::objc_property:
-          Current.Type = TT_ObjCProperty;
-          break;
-        default:
-          break;
-        }
-      }
-    }
-
-    if (!Current.Children.empty())
-      determineTokenTypes(Current.Children[0], IsExpression,
-                          LookForFunctionName);
-  }
-
-  /// \brief Starting from \p Current, this searches backwards for an
-  /// identifier which could be the start of a function name and marks it.
-  void findFunctionName(AnnotatedToken *Current) {
-    AnnotatedToken *Parent = Current->Parent;
-    while (Parent != NULL && Parent->Parent != NULL) {
-      if (Parent->is(tok::identifier) &&
-          (Parent->Parent->is(tok::identifier) ||
-           Parent->Parent->Type == TT_PointerOrReference ||
-           Parent->Parent->Type == TT_TemplateCloser)) {
-        Parent->Type = TT_StartOfName;
-        break;
-      }
-      Parent = Parent->Parent;
-    }
-  }
-
-  /// \brief Returns the previous token ignoring comments.
-  const AnnotatedToken *getPreviousToken(const AnnotatedToken &Tok) {
-    const AnnotatedToken *PrevToken = Tok.Parent;
-    while (PrevToken != NULL && PrevToken->is(tok::comment))
-      PrevToken = PrevToken->Parent;
-    return PrevToken;
-  }
-
-  /// \brief Returns the next token ignoring comments.
-  const AnnotatedToken *getNextToken(const AnnotatedToken &Tok) {
-    if (Tok.Children.empty())
-      return NULL;
-    const AnnotatedToken *NextToken = &Tok.Children[0];
-    while (NextToken->is(tok::comment)) {
-      if (NextToken->Children.empty())
-        return NULL;
-      NextToken = &NextToken->Children[0];
-    }
-    return NextToken;
-  }
-
-  /// \brief Return the type of the given token assuming it is * or &.
-  TokenType
-  determineStarAmpUsage(const AnnotatedToken &Tok, bool IsExpression) {
-    const AnnotatedToken *PrevToken = getPreviousToken(Tok);
-    if (PrevToken == NULL)
-      return TT_UnaryOperator;
-
-    const AnnotatedToken *NextToken = getNextToken(Tok);
-    if (NextToken == NULL)
-      return TT_Unknown;
-
-    if (NextToken->is(tok::l_square) && NextToken->Type != TT_ObjCMethodExpr)
-      return TT_PointerOrReference;
-
-    if (PrevToken->is(tok::l_paren) || PrevToken->is(tok::l_square) ||
-        PrevToken->is(tok::l_brace) || PrevToken->is(tok::comma) ||
-        PrevToken->is(tok::kw_return) || PrevToken->is(tok::colon) ||
-        PrevToken->Type == TT_BinaryOperator ||
-        PrevToken->Type == TT_UnaryOperator || PrevToken->Type == TT_CastRParen)
-      return TT_UnaryOperator;
-
-    if (PrevToken->FormatTok.Tok.isLiteral() || PrevToken->is(tok::r_paren) ||
-        PrevToken->is(tok::r_square) || NextToken->FormatTok.Tok.isLiteral() ||
-        NextToken->is(tok::plus) || NextToken->is(tok::minus) ||
-        NextToken->is(tok::plusplus) || NextToken->is(tok::minusminus) ||
-        NextToken->is(tok::tilde) || NextToken->is(tok::exclaim) ||
-        NextToken->is(tok::l_paren) || NextToken->is(tok::l_square) ||
-        NextToken->is(tok::kw_alignof) || NextToken->is(tok::kw_sizeof))
-      return TT_BinaryOperator;
-
-    if (NextToken->is(tok::comma) || NextToken->is(tok::r_paren) ||
-        NextToken->is(tok::greater))
-      return TT_PointerOrReference;
-
-    // It is very unlikely that we are going to find a pointer or reference type
-    // definition on the RHS of an assignment.
-    if (IsExpression)
-      return TT_BinaryOperator;
-
-    return TT_PointerOrReference;
-  }
-
-  TokenType determinePlusMinusCaretUsage(const AnnotatedToken &Tok) {
-    const AnnotatedToken *PrevToken = getPreviousToken(Tok);
-    if (PrevToken == NULL)
-      return TT_UnaryOperator;
-
-    // Use heuristics to recognize unary operators.
-    if (PrevToken->is(tok::equal) || PrevToken->is(tok::l_paren) ||
-        PrevToken->is(tok::comma) || PrevToken->is(tok::l_square) ||
-        PrevToken->is(tok::question) || PrevToken->is(tok::colon) ||
-        PrevToken->is(tok::kw_return) || PrevToken->is(tok::kw_case) ||
-        PrevToken->is(tok::at) || PrevToken->is(tok::l_brace))
-      return TT_UnaryOperator;
-
-    // There can't be to consecutive binary operators.
-    if (PrevToken->Type == TT_BinaryOperator)
-      return TT_UnaryOperator;
-
-    // Fall back to marking the token as binary operator.
-    return TT_BinaryOperator;
-  }
-
-  /// \brief Determine whether ++/-- are pre- or post-increments/-decrements.
-  TokenType determineIncrementUsage(const AnnotatedToken &Tok) {
-    const AnnotatedToken *PrevToken = getPreviousToken(Tok);
-    if (PrevToken == NULL)
-      return TT_UnaryOperator;
-    if (PrevToken->is(tok::r_paren) || PrevToken->is(tok::r_square) ||
-        PrevToken->is(tok::identifier))
-      return TT_TrailingUnaryOperator;
-
-    return TT_UnaryOperator;
-  }
-
-  bool spaceRequiredBetween(const AnnotatedToken &Left,
-                            const AnnotatedToken &Right) {
-    if (Right.is(tok::hashhash))
-      return Left.is(tok::hash);
-    if (Left.is(tok::hashhash) || Left.is(tok::hash))
-      return Right.is(tok::hash);
-    if (Right.is(tok::r_paren) || Right.is(tok::semi) || Right.is(tok::comma))
-      return false;
-    if (Right.is(tok::less) &&
-        (Left.is(tok::kw_template) ||
-         (Line.Type == LT_ObjCDecl && Style.ObjCSpaceBeforeProtocolList)))
-      return true;
-    if (Left.is(tok::arrow) || Right.is(tok::arrow))
-      return false;
-    if (Left.is(tok::exclaim) || Left.is(tok::tilde))
-      return false;
-    if (Left.is(tok::at) &&
-        (Right.is(tok::identifier) || Right.is(tok::string_literal) ||
-         Right.is(tok::char_constant) || Right.is(tok::numeric_constant) ||
-         Right.is(tok::l_paren) || Right.is(tok::l_brace) ||
-         Right.is(tok::kw_true) || Right.is(tok::kw_false)))
-      return false;
-    if (Left.is(tok::coloncolon))
-      return false;
-    if (Right.is(tok::coloncolon))
-      return Left.isNot(tok::identifier) && Left.isNot(tok::greater);
-    if (Left.is(tok::less) || Right.is(tok::greater) || Right.is(tok::less))
-      return false;
-    if (Right.is(tok::amp) || Right.is(tok::star))
-      return Left.FormatTok.Tok.isLiteral() ||
-             (Left.isNot(tok::star) && Left.isNot(tok::amp) &&
-              !Style.PointerAndReferenceBindToType);
-    if (Left.is(tok::amp) || Left.is(tok::star))
-      return Right.FormatTok.Tok.isLiteral() ||
-             Style.PointerAndReferenceBindToType;
-    if (Right.is(tok::star) && Left.is(tok::l_paren))
-      return false;
-    if (Left.is(tok::l_square) || Right.is(tok::r_square))
-      return false;
-    if (Right.is(tok::l_square) && Right.Type != TT_ObjCMethodExpr)
-      return false;
-    if (Left.is(tok::period) || Right.is(tok::period))
-      return false;
-    if (Left.is(tok::colon))
-      return Left.Type != TT_ObjCMethodExpr;
-    if (Right.is(tok::colon))
-      return Right.Type != TT_ObjCMethodExpr;
-    if (Left.is(tok::l_paren))
-      return false;
-    if (Right.is(tok::l_paren)) {
-      return Line.Type == LT_ObjCDecl || Left.is(tok::kw_if) ||
-             Left.is(tok::kw_for) || Left.is(tok::kw_while) ||
-             Left.is(tok::kw_switch) || Left.is(tok::kw_return) ||
-             Left.is(tok::kw_catch) || Left.is(tok::kw_new) ||
-             Left.is(tok::kw_delete);
-    }
-    if (Left.is(tok::at) &&
-        Right.FormatTok.Tok.getObjCKeywordID() != tok::objc_not_keyword)
-      return false;
-    if (Left.is(tok::l_brace) && Right.is(tok::r_brace))
-      return false;
-    return true;
-  }
-
-  bool spaceRequiredBefore(const AnnotatedToken &Tok) {
-    if (Line.Type == LT_ObjCMethodDecl) {
-      if (Tok.is(tok::identifier) && !Tok.Children.empty() &&
-          Tok.Children[0].is(tok::colon) && Tok.Parent->is(tok::identifier))
-        return true;
-      if (Tok.is(tok::colon))
-        return false;
-      if (Tok.Parent->Type == TT_ObjCMethodSpecifier)
-        return true;
-      if (Tok.Parent->is(tok::r_paren) && Tok.is(tok::identifier))
-        // Don't space between ')' and <id>
-        return false;
-      if (Tok.Parent->is(tok::colon) && Tok.is(tok::l_paren))
-        // Don't space between ':' and '('
-        return false;
-    }
-    if (Line.Type == LT_ObjCProperty &&
-        (Tok.is(tok::equal) || Tok.Parent->is(tok::equal)))
-      return false;
-
-    if (Tok.Parent->is(tok::comma))
-      return true;
-    if (Tok.Type == TT_CtorInitializerColon || Tok.Type == TT_ObjCBlockLParen)
-      return true;
-    if (Tok.Type == TT_OverloadedOperator)
-      return Tok.is(tok::identifier) || Tok.is(tok::kw_new) ||
-             Tok.is(tok::kw_delete) || Tok.is(tok::kw_bool);
-    if (Tok.Parent->Type == TT_OverloadedOperator)
-      return false;
-    if (Tok.is(tok::colon))
-      return Line.First.isNot(tok::kw_case) && !Tok.Children.empty() &&
-             Tok.Type != TT_ObjCMethodExpr;
-    if (Tok.Parent->Type == TT_UnaryOperator ||
-        Tok.Parent->Type == TT_CastRParen)
-      return false;
-    if (Tok.Type == TT_UnaryOperator)
-      return Tok.Parent->isNot(tok::l_paren) &&
-             Tok.Parent->isNot(tok::l_square) && Tok.Parent->isNot(tok::at) &&
-             (Tok.Parent->isNot(tok::colon) ||
-              Tok.Parent->Type != TT_ObjCMethodExpr);
-    if (Tok.Parent->is(tok::greater) && Tok.is(tok::greater)) {
-      return Tok.Type == TT_TemplateCloser && Tok.Parent->Type ==
-             TT_TemplateCloser && Style.SplitTemplateClosingGreater;
-    }
-    if (Tok.Type == TT_BinaryOperator || Tok.Parent->Type == TT_BinaryOperator)
-      return true;
-    if (Tok.Parent->Type == TT_TemplateCloser && Tok.is(tok::l_paren))
-      return false;
-    if (Tok.is(tok::less) && Line.First.is(tok::hash))
-      return true;
-    if (Tok.Type == TT_TrailingUnaryOperator)
-      return false;
-    return spaceRequiredBetween(*Tok.Parent, Tok);
-  }
-
-  bool canBreakBefore(const AnnotatedToken &Right) {
-    const AnnotatedToken &Left = *Right.Parent;
-    if (Line.Type == LT_ObjCMethodDecl) {
-      if (Right.is(tok::identifier) && !Right.Children.empty() &&
-          Right.Children[0].is(tok::colon) && Left.is(tok::identifier))
-        return t