Don't warn about Unicode characters in -E mode.

People use the C preprocessor for things other than C files. Some of them
have Unicode characters. We shouldn't warn about Unicode characters
appearing outside of identifiers in this case.

There's not currently a way for the preprocessor to tell if it's in -E mode,
so I added a new flag, derived from the PreprocessorOutputOptions. This is
only used by the Unicode warnings for now, but could conceivably be used by
other warnings or even behavioral differences later.

<rdar://problem/13107323>

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@173881 91177308-0d34-0410-b5e6-96231b3b80d8

3 files changed, 65 insertions, 21 deletions

diff --git a/lib/Frontend/CompilerInstance.cpp b/lib/Frontend/CompilerInstance.cpp
index a7f0770ec7..27f96b0974 100644
--- a/lib/Frontend/CompilerInstance.cpp
+++ b/lib/Frontend/CompilerInstance.cpp
@@ -243,6 +243,8 @@ void CompilerInstance::createPreprocessor() {
 
   InitializePreprocessor(*PP, PPOpts, getHeaderSearchOpts(), getFrontendOpts());
 
+  PP->setPreprocessedOutput(getPreprocessorOutputOpts().ShowCPP);
+
   // Set up the module path, including the hash for the
   // module-creation options.
   SmallString<256> SpecificModuleCache(
diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp
index f49f30d878..b4b0ddb1c4 100644
--- a/lib/Frontend/CompilerInvocation.cpp
+++ b/lib/Frontend/CompilerInvocation.cpp
@@ -1395,9 +1395,48 @@ static void ParsePreprocessorArgs(PreprocessorOptions &Opts, ArgList &Args,
 }
 
 static void ParsePreprocessorOutputArgs(PreprocessorOutputOptions &Opts,
-                                        ArgList &Args) {
+                                        ArgList &Args,
+                                        frontend::ActionKind Action) {
   using namespace options;
-  Opts.ShowCPP = !Args.hasArg(OPT_dM);
+
+  switch (Action) {
+  case frontend::ASTDeclList:
+  case frontend::ASTDump:
+  case frontend::ASTDumpXML:
+  case frontend::ASTPrint:
+  case frontend::ASTView:
+  case frontend::EmitAssembly:
+  case frontend::EmitBC:
+  case frontend::EmitHTML:
+  case frontend::EmitLLVM:
+  case frontend::EmitLLVMOnly:
+  case frontend::EmitCodeGenOnly:
+  case frontend::EmitObj:
+  case frontend::FixIt:
+  case frontend::GenerateModule:
+  case frontend::GeneratePCH:
+  case frontend::GeneratePTH:
+  case frontend::ParseSyntaxOnly:
+  case frontend::PluginAction:
+  case frontend::PrintDeclContext:
+  case frontend::RewriteObjC:
+  case frontend::RewriteTest:
+  case frontend::RunAnalysis:
+  case frontend::MigrateSource:
+    Opts.ShowCPP = 0;
+    break;
+
+  case frontend::DumpRawTokens:
+  case frontend::DumpTokens:
+  case frontend::InitOnly:
+  case frontend::PrintPreamble:
+  case frontend::PrintPreprocessedInput:
+  case frontend::RewriteMacros:
+  case frontend::RunPreprocessorOnly:
+    Opts.ShowCPP = !Args.hasArg(OPT_dM);
+    break;
+  }
+
   Opts.ShowComments = Args.hasArg(OPT_C);
   Opts.ShowLineMarkers = !Args.hasArg(OPT_P);
   Opts.ShowMacroComments = Args.hasArg(OPT_CC);
@@ -1478,7 +1517,8 @@ bool CompilerInvocation::CreateFromArgs(CompilerInvocation &Res,
   // parameters from the function and the "FileManager.h" #include.
   FileManager FileMgr(Res.getFileSystemOpts());
   ParsePreprocessorArgs(Res.getPreprocessorOpts(), *Args, FileMgr, Diags);
-  ParsePreprocessorOutputArgs(Res.getPreprocessorOutputOpts(), *Args);
+  ParsePreprocessorOutputArgs(Res.getPreprocessorOutputOpts(), *Args,
+                              Res.getFrontendOpts().ProgramAction);
   ParseTargetArgs(Res.getTargetOpts(), *Args);
 
   return Success;
diff --git a/lib/Lex/Lexer.cpp b/lib/Lex/Lexer.cpp
index 3e3aaae5f5..08f406b069 100644
--- a/lib/Lex/Lexer.cpp
+++ b/lib/Lex/Lexer.cpp
@@ -2811,14 +2811,13 @@ static bool isUnicodeWhitespace(uint32_t C) {
 }
 
 void Lexer::LexUnicode(Token &Result, uint32_t C, const char *CurPtr) {
-  if (isUnicodeWhitespace(C)) {
-    if (!isLexingRawMode()) {
-      CharSourceRange CharRange =
-        CharSourceRange::getCharRange(getSourceLocation(),
-                                      getSourceLocation(CurPtr));
-      Diag(BufferPtr, diag::ext_unicode_whitespace)
-        << CharRange;
-    }
+  if (!isLexingRawMode() && !PP->isPreprocessedOutput() &&
+      isUnicodeWhitespace(C)) {
+    CharSourceRange CharRange =
+      CharSourceRange::getCharRange(getSourceLocation(),
+                                    getSourceLocation(CurPtr));
+    Diag(BufferPtr, diag::ext_unicode_whitespace)
+      << CharRange;
 
     Result.setFlag(Token::LeadingSpace);
     if (SkipWhitespace(Result, CurPtr))
@@ -2832,7 +2831,8 @@ void Lexer::LexUnicode(Token &Result, uint32_t C, const char *CurPtr) {
     return LexIdentifier(Result, CurPtr);
   }
 
-  if (!isASCII(*BufferPtr) && !isAllowedIDChar(C)) {
+  if (!isLexingRawMode() && !PP->isPreprocessedOutput() &&
+      !isASCII(*BufferPtr) && !isAllowedIDChar(C)) {
     // Non-ASCII characters tend to creep into source code unintentionally.
     // Instead of letting the parser complain about the unknown token,
     // just drop the character.
@@ -2842,13 +2842,11 @@ void Lexer::LexUnicode(Token &Result, uint32_t C, const char *CurPtr) {
     // loophole in the mapping of Unicode characters to basic character set
     // characters that allows us to map these particular characters to, say,
     // whitespace.
-    if (!isLexingRawMode()) {
-      CharSourceRange CharRange =
-        CharSourceRange::getCharRange(getSourceLocation(),
-                                      getSourceLocation(CurPtr));
-      Diag(BufferPtr, diag::err_non_ascii)
-        << FixItHint::CreateRemoval(CharRange);
-    }
+    CharSourceRange CharRange =
+      CharSourceRange::getCharRange(getSourceLocation(),
+                                    getSourceLocation(CurPtr));
+    Diag(BufferPtr, diag::err_non_ascii)
+      << FixItHint::CreateRemoval(CharRange);
 
     BufferPtr = CurPtr;
     return LexTokenInternal(Result);
@@ -3537,11 +3535,15 @@ LexNextToken:
     if (Status == conversionOK)
       return LexUnicode(Result, CodePoint, CurPtr);
     
+    if (isLexingRawMode() || PP->isPreprocessedOutput()) {
+      Kind = tok::unknown;
+      break;
+    }
+
     // Non-ASCII characters tend to creep into source code unintentionally.
     // Instead of letting the parser complain about the unknown token,
     // just diagnose the invalid UTF-8, then drop the character.
-    if (!isLexingRawMode())
-      Diag(CurPtr, diag::err_invalid_utf8);
+    Diag(CurPtr, diag::err_invalid_utf8);
 
     BufferPtr = CurPtr+1;
     goto LexNextToken;