Introduce support for a simple module import declaration, which

loads the named module. The syntax itself is intentionally hideous and
will be replaced at some later point with something more
palatable. For now, we're focusing on the semantics:
  - Module imports are handled first by the preprocessor (to get macro
  definitions) and then the same tokens are also handled by the parser
  (to get declarations). If both happen (as in normal compilation),
  the second one is redundant, because we currently have no way to
  hide macros or declarations when loading a module. Chris gets credit
  for this mad-but-workable scheme.
  - The Preprocessor now holds on to a reference to a module loader,
  which is responsible for loading named modules. CompilerInstance is
  the only important module loader: it now knows how to create and
  wire up an AST reader on demand to actually perform the module load.
  - We search for modules in the include path, using the module name
  with the suffix ".pcm" (precompiled module) for the file name. This
  is a temporary hack; we hope to improve the situation in the
  future.



git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@138679 91177308-0d34-0410-b5e6-96231b3b80d8

6 files changed, 154 insertions, 37 deletions

diff --git a/lib/Frontend/ASTUnit.cpp b/lib/Frontend/ASTUnit.cpp
index 4a5a29a9c9..7f25a0045e 100644
--- a/lib/Frontend/ASTUnit.cpp
+++ b/lib/Frontend/ASTUnit.cpp
@@ -616,7 +616,7 @@ ASTUnit *ASTUnit::LoadFromASTFile(const std::string &Filename,
   AST->Target = TargetInfo::CreateTargetInfo(AST->getDiagnostics(),
                                              TargetOpts);
   AST->PP = new Preprocessor(AST->getDiagnostics(), LangInfo, *AST->Target,
-                             AST->getSourceManager(), HeaderInfo);
+                             AST->getSourceManager(), HeaderInfo, *AST);
   Preprocessor &PP = *AST->PP;
 
   PP.setPredefines(Reader->getSuggestedPredefines());
diff --git a/lib/Frontend/CompilerInstance.cpp b/lib/Frontend/CompilerInstance.cpp
index ec8b6dc910..f53d0b064c 100644
--- a/lib/Frontend/CompilerInstance.cpp
+++ b/lib/Frontend/CompilerInstance.cpp
@@ -191,52 +191,38 @@ void CompilerInstance::createSourceManager(FileManager &FileMgr) {
 // Preprocessor
 
 void CompilerInstance::createPreprocessor() {
-  PP = createPreprocessor(getDiagnostics(), getLangOpts(),
-                          getPreprocessorOpts(), getHeaderSearchOpts(),
-                          getDependencyOutputOpts(), getTarget(),
-                          getFrontendOpts(), getSourceManager(),
-                          getFileManager());
-}
-
-Preprocessor *
-CompilerInstance::createPreprocessor(Diagnostic &Diags,
-                                     const LangOptions &LangInfo,
-                                     const PreprocessorOptions &PPOpts,
-                                     const HeaderSearchOptions &HSOpts,
-                                     const DependencyOutputOptions &DepOpts,
-                                     const TargetInfo &Target,
-                                     const FrontendOptions &FEOpts,
-                                     SourceManager &SourceMgr,
-                                     FileManager &FileMgr) {
+  const PreprocessorOptions &PPOpts = getPreprocessorOpts();
+  
   // Create a PTH manager if we are using some form of a token cache.
   PTHManager *PTHMgr = 0;
   if (!PPOpts.TokenCache.empty())
-    PTHMgr = PTHManager::Create(PPOpts.TokenCache, Diags);
-
+    PTHMgr = PTHManager::Create(PPOpts.TokenCache, getDiagnostics());
+  
   // Create the Preprocessor.
-  HeaderSearch *HeaderInfo = new HeaderSearch(FileMgr);
-  Preprocessor *PP = new Preprocessor(Diags, LangInfo, Target,
-                                      SourceMgr, *HeaderInfo, PTHMgr,
-                                      /*OwnsHeaderSearch=*/true);
-
+  HeaderSearch *HeaderInfo = new HeaderSearch(getFileManager());
+  PP = new Preprocessor(getDiagnostics(), getLangOpts(), getTarget(),
+                        getSourceManager(), *HeaderInfo, *this, PTHMgr,
+                        /*OwnsHeaderSearch=*/true);
+  
   // Note that this is different then passing PTHMgr to Preprocessor's ctor.
   // That argument is used as the IdentifierInfoLookup argument to
   // IdentifierTable's ctor.
   if (PTHMgr) {
-    PTHMgr->setPreprocessor(PP);
+    PTHMgr->setPreprocessor(&*PP);
     PP->setPTHManager(PTHMgr);
   }
-
+  
   if (PPOpts.DetailedRecord)
     PP->createPreprocessingRecord(
-                       PPOpts.DetailedRecordIncludesNestedMacroExpansions);
+                                  PPOpts.DetailedRecordIncludesNestedMacroExpansions);
+  
+  InitializePreprocessor(*PP, PPOpts, getHeaderSearchOpts(), getFrontendOpts());
   
-  InitializePreprocessor(*PP, PPOpts, HSOpts, FEOpts);
-
   // Handle generating dependencies, if requested.
+  const DependencyOutputOptions &DepOpts = getDependencyOutputOpts();
   if (!DepOpts.OutputFile.empty())
     AttachDependencyFileGen(*PP, DepOpts);
-
+  
   // Handle generating header include information, if requested.
   if (DepOpts.ShowHeaderIncludes)
     AttachHeaderIncludeGen(*PP);
@@ -247,8 +233,6 @@ CompilerInstance::createPreprocessor(Diagnostic &Diags,
     AttachHeaderIncludeGen(*PP, /*ShowAllHeaders=*/true, OutputPath,
                            /*ShowDepth=*/false);
   }
-
-  return PP;
 }
 
 // ASTContext
@@ -640,4 +624,68 @@ bool CompilerInstance::ExecuteAction(FrontendAction &Act) {
   return !getDiagnostics().getClient()->getNumErrors();
 }
 
+ModuleKey CompilerInstance::loadModule(SourceLocation ImportLoc, 
+                                       IdentifierInfo &ModuleName,
+                                       SourceLocation ModuleNameLoc) {  
+  // Determine what file we're searching from.
+  SourceManager &SourceMgr = getSourceManager();
+  SourceLocation ExpandedImportLoc = SourceMgr.getExpansionLoc(ImportLoc);
+  const FileEntry *CurFile
+    = SourceMgr.getFileEntryForID(SourceMgr.getFileID(ExpandedImportLoc));
+  if (!CurFile)
+    CurFile = SourceMgr.getFileEntryForID(SourceMgr.getMainFileID());
+
+  // Search for a module with the given name.
+  std::string Filename = ModuleName.getName().str();
+  Filename += ".pcm";
+  const DirectoryLookup *CurDir = 0;  
+  const FileEntry *ModuleFile
+    = PP->getHeaderSearchInfo().LookupFile(Filename, /*isAngled=*/false,
+                                           /*FromDir=*/0, CurDir, CurFile, 
+                                           /*SearchPath=*/0, 
+                                           /*RelativePath=*/0);
+  if (!ModuleFile) {
+    getDiagnostics().Report(ModuleNameLoc, diag::warn_module_not_found)
+      << ModuleName.getName()
+      << SourceRange(ImportLoc, ModuleNameLoc);
+    return 0;
+  }
+  
+  // If we don't already have an ASTReader, create one now.
+  if (!ModuleManager) {
+    std::string Sysroot = getHeaderSearchOpts().Sysroot;
+    const PreprocessorOptions &PPOpts = getPreprocessorOpts();
+    ModuleManager = new ASTReader(getPreprocessor(), &*Context,
+                                  Sysroot.empty() ? "" : Sysroot.c_str(),
+                                  PPOpts.DisablePCHValidation, 
+                                  PPOpts.DisableStatCache);
+    ModuleManager->setDeserializationListener(
+      getASTConsumer().GetASTDeserializationListener());
+    getASTContext().setASTMutationListener(
+      getASTConsumer().GetASTMutationListener());
+    llvm::OwningPtr<ExternalASTSource> Source;
+    Source.reset(ModuleManager);
+    getASTContext().setExternalSource(Source);
+    ModuleManager->InitializeSema(getSema());
+  }
+  
+  // Try to load the module we found.
+  switch (ModuleManager->ReadAST(ModuleFile->getName(),
+                                 serialization::MK_Module)) {
+  case ASTReader::Success:
+    break;
+
+  case ASTReader::IgnorePCH:
+    // FIXME: The ASTReader will already have complained, but can we showhorn
+    // that diagnostic information into a more useful form?
+    return 0;
+      
+  case ASTReader::Failure:
+    // Already complained.
+    return 0;
+  }
+  
+  // FIXME: The module file's FileEntry makes a poor key indeed!
+  return (ModuleKey)ModuleFile;
+}
 
diff --git a/lib/Lex/Lexer.cpp b/lib/Lex/Lexer.cpp
index 1ec50cd2c5..64b8744922 100644
--- a/lib/Lex/Lexer.cpp
+++ b/lib/Lex/Lexer.cpp
@@ -1277,6 +1277,7 @@ FinishIdentifier:
     // preprocessor, which may macro expand it or something.
     if (II->isHandleIdentifierCase())
       PP->HandleIdentifier(Result);
+    
     return;
   }
 
diff --git a/lib/Lex/Preprocessor.cpp b/lib/Lex/Preprocessor.cpp
index 24d36cdb25..1d1687d84b 100644
--- a/lib/Lex/Preprocessor.cpp
+++ b/lib/Lex/Preprocessor.cpp
@@ -35,6 +35,7 @@
 #include "clang/Lex/ScratchBuffer.h"
 #include "clang/Lex/LexDiagnostic.h"
 #include "clang/Lex/CodeCompletionHandler.h"
+#include "clang/Lex/ModuleLoader.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/TargetInfo.h"
@@ -50,12 +51,12 @@ ExternalPreprocessorSource::~ExternalPreprocessorSource() { }
 
 Preprocessor::Preprocessor(Diagnostic &diags, const LangOptions &opts,
                            const TargetInfo &target, SourceManager &SM,
-                           HeaderSearch &Headers,
+                           HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
                            IdentifierInfoLookup* IILookup,
                            bool OwnsHeaders)
   : Diags(&diags), Features(opts), Target(target),FileMgr(Headers.getFileMgr()),
-    SourceMgr(SM),
-    HeaderInfo(Headers), ExternalSource(0),
+    SourceMgr(SM), HeaderInfo(Headers), TheModuleLoader(TheModuleLoader),
+    ExternalSource(0), 
     Identifiers(opts, IILookup), BuiltinInfo(Target), CodeComplete(0),
     CodeCompletionFile(0), SkipMainFilePreamble(0, true), CurPPLexer(0), 
     CurDirLookup(0), Callbacks(0), MacroArgCache(0), Record(0), MIChainHead(0),
@@ -483,7 +484,7 @@ void Preprocessor::HandleIdentifier(Token &Identifier) {
     if (!DisableMacroExpansion && !Identifier.isExpandDisabled()) {
       if (MI->isEnabled()) {
         if (!HandleMacroExpandedIdentifier(Identifier, MI))
-          return;
+          goto finish;
       } else {
         // C99 6.10.3.4p2 says that a disabled macro may never again be
         // expanded, even if it's in a context where it could be expanded in the
@@ -505,6 +506,33 @@ void Preprocessor::HandleIdentifier(Token &Identifier) {
   // like "#define TY typeof", "TY(1) x".
   if (II.isExtensionToken() && !DisableMacroExpansion)
     Diag(Identifier, diag::ext_token_used);
+  
+finish:
+  // If we have the start of a module import, handle it now.
+  if (Identifier.is(tok::kw___import__) &&
+      !InMacroArgs && !DisableMacroExpansion)
+    HandleModuleImport(Identifier);
+}
+
+void Preprocessor::HandleModuleImport(Token &Import) {
+  // The token sequence 
+  //
+  //   __import__ identifier
+  //
+  // indicates a module import directive. We load the module and then 
+  // leave the token sequence for the parser.
+  Token ModuleNameTok = LookAhead(0);
+  if (ModuleNameTok.getKind() != tok::identifier)
+    return;
+  
+  (void)TheModuleLoader.loadModule(Import.getLocation(),
+                                   *ModuleNameTok.getIdentifierInfo(), 
+                                   ModuleNameTok.getLocation());
+  
+  // FIXME: Transmogrify __import__ into some kind of AST-only __import__ that
+  // is not recognized by the preprocessor but is recognized by the parser.
+  // It would also be useful to stash the ModuleKey somewhere, so we don't try
+  // to load the module twice.
 }
 
 void Preprocessor::AddCommentHandler(CommentHandler *Handler) {
@@ -535,6 +563,8 @@ bool Preprocessor::HandleComment(Token &result, SourceRange Comment) {
   return true;
 }
 
+ModuleLoader::~ModuleLoader() { }
+
 CommentHandler::~CommentHandler() { }
 
 CodeCompletionHandler::~CodeCompletionHandler() { }
diff --git a/lib/Parse/Parser.cpp b/lib/Parse/Parser.cpp
index 0da2b4302c..39edab1911 100644
--- a/lib/Parse/Parser.cpp
+++ b/lib/Parse/Parser.cpp
@@ -679,6 +679,9 @@ Parser::ParseExternalDeclaration(ParsedAttributesWithRange &attrs,
     ParseMicrosoftIfExistsExternalDeclaration();
     return DeclGroupPtrTy();
 
+  case tok::kw___import__:
+    return ParseModuleImport();
+      
   default:
   dont_know:
     // We can't tell whether this is a function-definition or declaration yet.
@@ -1541,3 +1544,24 @@ void Parser::ParseMicrosoftIfExistsExternalDeclaration() {
   }
   ConsumeBrace();
 }
+
+Parser::DeclGroupPtrTy Parser::ParseModuleImport() {
+  assert(Tok.is(tok::kw___import__) && "Improper start to module import");
+  SourceLocation ImportLoc = ConsumeToken();
+  
+  // Parse the module name.
+  if (!Tok.is(tok::identifier)) {
+    Diag(Tok, diag::err_module_expected_ident);
+    SkipUntil(tok::semi);
+    return DeclGroupPtrTy();
+  }
+  
+  IdentifierInfo &ModuleName = *Tok.getIdentifierInfo();
+  SourceLocation ModuleNameLoc = ConsumeToken();
+  DeclResult Import = Actions.ActOnModuleImport(ImportLoc, ModuleName, ModuleNameLoc);
+  ExpectAndConsumeSemi(diag::err_module_expected_semi);
+  if (Import.isInvalid())
+    return DeclGroupPtrTy();
+  
+  return Actions.ConvertDeclToDeclGroup(Import.get());
+}
diff --git a/lib/Sema/SemaDecl.cpp b/lib/Sema/SemaDecl.cpp
index 9f43aa2f05..8f08af4778 100644
--- a/lib/Sema/SemaDecl.cpp
+++ b/lib/Sema/SemaDecl.cpp
@@ -38,6 +38,7 @@
 // FIXME: layering (ideally, Sema shouldn't be dependent on Lex API's)
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Lex/HeaderSearch.h"
+#include "clang/Lex/ModuleLoader.h"
 #include "llvm/ADT/Triple.h"
 #include <algorithm>
 #include <cstring>
@@ -9294,6 +9295,19 @@ Decl *Sema::ActOnFileScopeAsmDecl(Expr *expr,
   return New;
 }
 
+DeclResult Sema::ActOnModuleImport(SourceLocation ImportLoc,
+                                   IdentifierInfo &ModuleName,
+                                   SourceLocation ModuleNameLoc) {
+  ModuleKey Module = PP.getModuleLoader().loadModule(ImportLoc, 
+                                                     ModuleName, ModuleNameLoc);
+  if (!Module)
+    return true;
+  
+  // FIXME: Actually create a declaration to describe the module import.
+  (void)Module;
+  return DeclResult((Decl *)0);
+}
+
 void Sema::ActOnPragmaWeakID(IdentifierInfo* Name,
                              SourceLocation PragmaLoc,
                              SourceLocation NameLoc) {