Introduce basic support for parsing module map files.

Module map files provide a way to map between headers and modules, so
that we can layer a module system on top of existing headers without
changing those headers at all.

This commit introduces the module map file parser and the module map
that it generates, and wires up the module map file parser so that
we'll automatically find module map files as part of header
search. Note that we don't yet use the information stored in the
module map.



git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@144402 91177308-0d34-0410-b5e6-96231b3b80d8

3 files changed, 598 insertions, 1 deletions

diff --git a/lib/Lex/CMakeLists.txt b/lib/Lex/CMakeLists.txt
index 80e2820101..39c4fd590d 100644
--- a/lib/Lex/CMakeLists.txt
+++ b/lib/Lex/CMakeLists.txt
@@ -11,6 +11,7 @@ add_clang_library(clangLex
   LiteralSupport.cpp
   MacroArgs.cpp
   MacroInfo.cpp
+  ModuleMap.cpp
   PPCaching.cpp
   PPDirectives.cpp
   PPExpressions.cpp
diff --git a/lib/Lex/HeaderSearch.cpp b/lib/Lex/HeaderSearch.cpp
index 837b913d47..6403cfb868 100644
--- a/lib/Lex/HeaderSearch.cpp
+++ b/lib/Lex/HeaderSearch.cpp
@@ -13,6 +13,7 @@
 
 #include "clang/Lex/HeaderSearch.h"
 #include "clang/Lex/HeaderMap.h"
+#include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/IdentifierTable.h"
 #include "llvm/Support/FileSystem.h"
@@ -37,7 +38,8 @@ HeaderFileInfo::getControllingMacro(ExternalIdentifierLookup *External) {
 ExternalHeaderFileInfoSource::~ExternalHeaderFileInfoSource() {}
 
 HeaderSearch::HeaderSearch(FileManager &FM, DiagnosticsEngine &Diags)
-  : FileMgr(FM), Diags(Diags), FrameworkMap(64) 
+  : FileMgr(FM), Diags(Diags), FrameworkMap(64), 
+    ModMap(FileMgr, *Diags.getClient()) 
 {
   AngledDirIdx = 0;
   SystemDirIdx = 0;
@@ -192,6 +194,24 @@ const FileEntry *DirectoryLookup::LookupFile(
       RelativePath->clear();
       RelativePath->append(Filename.begin(), Filename.end());
     }
+    
+    // If we have a module map that might map this header, load it and
+    // check whether we'll have a suggestion for a module.
+    if (SuggestedModule && HS.hasModuleMap(TmpDir, getDir())) {
+      const FileEntry *File = HS.getFileMgr().getFile(TmpDir.str(), 
+                                                      /*openFile=*/false);
+      if (!File)
+        return File;
+      
+      // If there is a module that corresponds to this header, 
+      // suggest it.
+      StringRef Module = HS.getModuleForHeader(File);
+      if (!Module.empty() && Module != BuildingModule)
+        *SuggestedModule = Module;
+      
+      return File;
+    }
+    
     return HS.getFileMgr().getFile(TmpDir.str(), /*openFile=*/true);
   }
 
@@ -688,3 +708,73 @@ size_t HeaderSearch::getTotalMemory() const {
 StringRef HeaderSearch::getUniqueFrameworkName(StringRef Framework) {
   return FrameworkNames.GetOrCreateValue(Framework).getKey();
 }
+
+bool HeaderSearch::hasModuleMap(StringRef FileName, 
+                                const DirectoryEntry *Root) {
+  llvm::SmallVector<const DirectoryEntry *, 2> FixUpDirectories;
+  
+  StringRef DirName = FileName;
+  do {
+    // Get the parent directory name.
+    DirName = llvm::sys::path::parent_path(DirName);
+    if (DirName.empty())
+      return false;
+    
+    // Determine whether this directory exists.
+    const DirectoryEntry *Dir = FileMgr.getDirectory(DirName);
+    if (!Dir)
+      return false;
+    
+    llvm::DenseMap<const DirectoryEntry *, bool>::iterator
+      KnownDir = DirectoryHasModuleMap.find(Dir);
+    if (KnownDir != DirectoryHasModuleMap.end()) {
+      // We have seen this directory before. If it has no module map file,
+      // we're done.
+      if (!KnownDir->second)
+        return false;
+      
+      // All of the directories we stepped through inherit this module map
+      // file.
+      for (unsigned I = 0, N = FixUpDirectories.size(); I != N; ++I)
+        DirectoryHasModuleMap[FixUpDirectories[I]] = true;
+      
+      return true;
+    }
+    
+    // We have not checked for a module map file in this directory yet;
+    // do so now.
+    llvm::SmallString<128> ModuleMapFileName;
+    ModuleMapFileName += Dir->getName();
+    llvm::sys::path::append(ModuleMapFileName, "module.map");
+    if (const FileEntry *ModuleMapFile = FileMgr.getFile(ModuleMapFileName)) {
+      // We have found a module map file. Try to parse it.
+      if (!ModMap.parseModuleMapFile(ModuleMapFile)) {
+        // This directory has a module map.
+        DirectoryHasModuleMap[Dir] = true;
+        
+        // All of the directories we stepped through inherit this module map
+        // file.
+        for (unsigned I = 0, N = FixUpDirectories.size(); I != N; ++I)
+          DirectoryHasModuleMap[FixUpDirectories[I]] = true;
+        
+        return true;
+      }
+    }
+
+    // This directory did not have a module map file.
+    DirectoryHasModuleMap[Dir] = false;
+    
+    // Keep track of all of the directories we checked, so we can mark them as
+    // having module maps if we eventually do find a module map.
+    FixUpDirectories.push_back(Dir);
+  } while (true);
+  
+  return false;
+}
+
+StringRef HeaderSearch::getModuleForHeader(const FileEntry *File) {
+  // FIXME: Actually look for the corresponding module for this header.
+  return StringRef();
+}
+
+
diff --git a/lib/Lex/ModuleMap.cpp b/lib/Lex/ModuleMap.cpp
new file mode 100644
index 0000000000..69bc52636c
--- /dev/null
+++ b/lib/Lex/ModuleMap.cpp
@@ -0,0 +1,506 @@
+//===--- ModuleMap.cpp - Describe the layout of modules ---------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the ModuleMap implementation, which describes the layout
+// of a module as it relates to headers.
+//
+//===----------------------------------------------------------------------===//
+#include "clang/Lex/ModuleMap.h"
+#include "clang/Lex/Lexer.h"
+#include "clang/Lex/LiteralSupport.h"
+#include "clang/Lex/LexDiagnostic.h"
+#include "clang/Basic/Diagnostic.h"
+#include "clang/Basic/FileManager.h"
+#include "clang/Basic/TargetInfo.h"
+#include "clang/Basic/TargetOptions.h"
+#include "llvm/Support/Allocator.h"
+#include "llvm/Support/Host.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringSwitch.h"
+using namespace clang;
+
+//----------------------------------------------------------------------------//
+// Module
+//----------------------------------------------------------------------------//
+
+std::string ModuleMap::Module::getFullModuleName() const {
+  llvm::SmallVector<StringRef, 2> Names;
+  
+  // Build up the set of module names (from innermost to outermost).
+  for (const Module *M = this; M; M = M->Parent)
+    Names.push_back(M->Name);
+  
+  std::string Result;
+  for (llvm::SmallVector<StringRef, 2>::reverse_iterator I = Names.rbegin(),
+                                                      IEnd = Names.rend(); 
+       I != IEnd; ++I) {
+    if (!Result.empty())
+      Result += '.';
+    
+    Result += *I;
+  }
+  
+  return Result;
+}
+
+//----------------------------------------------------------------------------//
+// Module map
+//----------------------------------------------------------------------------//
+
+ModuleMap::ModuleMap(FileManager &FileMgr, const DiagnosticConsumer &DC) {
+  llvm::IntrusiveRefCntPtr<DiagnosticIDs> DiagIDs(new DiagnosticIDs);
+  Diags = llvm::IntrusiveRefCntPtr<DiagnosticsEngine>(
+            new DiagnosticsEngine(DiagIDs));
+  Diags->setClient(DC.clone(*Diags), /*ShouldOwnClient=*/true);
+  SourceMgr = new SourceManager(*Diags, FileMgr);
+}
+
+ModuleMap::~ModuleMap() {
+  delete SourceMgr;
+}
+
+static void indent(llvm::raw_ostream &OS, unsigned Spaces) {
+  OS << std::string(' ', Spaces);
+}
+
+static void dumpModule(llvm::raw_ostream &OS, ModuleMap::Module *M, 
+                       unsigned Indent) {
+  indent(OS, Indent);
+  if (M->IsExplicit)
+    OS << "explicit ";
+  OS << M->Name << " {\n";
+  
+  if (M->UmbrellaHeader) {
+    indent(OS, Indent + 2);
+    OS << "umbrella \"" << M->UmbrellaHeader->getName() << "\"\n";
+  }
+  
+  for (unsigned I = 0, N = M->Headers.size(); I != N; ++I) {
+    indent(OS, Indent + 2);
+    OS << "header \"" << M->Headers[I]->getName() << "\"\n";
+  }
+  
+  for (llvm::StringMap<ModuleMap::Module *>::iterator 
+            MI = M->SubModules.begin(), 
+         MIEnd = M->SubModules.end();
+       MI != MIEnd; ++MI)
+    dumpModule(llvm::errs(), MI->getValue(), Indent + 2);
+  
+  indent(OS, Indent);
+  OS << "}\n";
+}
+
+void ModuleMap::dump() {
+  llvm::errs() << "Modules:";
+  for (llvm::StringMap<Module *>::iterator M = Modules.begin(), 
+                                        MEnd = Modules.end(); 
+       M != MEnd; ++M)
+    dumpModule(llvm::errs(), M->getValue(), 2);
+  
+  llvm::errs() << "Headers:";
+  for (llvm::DenseMap<const FileEntry *, Module *>::iterator 
+            H = Headers.begin(),
+         HEnd = Headers.end();
+       H != HEnd; ++H) {
+    llvm::errs() << "  \"" << H->first->getName() << "\" -> " 
+                 << H->second->getFullModuleName() << "\n";
+  }
+}
+
+//----------------------------------------------------------------------------//
+// Module map file parser
+//----------------------------------------------------------------------------//
+
+namespace clang {
+  /// \brief A token in a module map file.
+  struct MMToken {
+    enum TokenKind {
+      EndOfFile,
+      HeaderKeyword,
+      Identifier,
+      ExplicitKeyword,
+      ModuleKeyword,
+      UmbrellaKeyword,
+      StringLiteral,
+      LBrace,
+      RBrace
+    } Kind;
+    
+    unsigned Location;
+    unsigned StringLength;
+    const char *StringData;
+    
+    void clear() {
+      Kind = EndOfFile;
+      Location = 0;
+      StringLength = 0;
+      StringData = 0;
+    }
+    
+    bool is(TokenKind K) const { return Kind == K; }
+    
+    SourceLocation getLocation() const {
+      return SourceLocation::getFromRawEncoding(Location);
+    }
+    
+    StringRef getString() const {
+      return StringRef(StringData, StringLength);
+    }
+  };
+  
+  class ModuleMapParser {
+    Lexer &L;
+    SourceManager &SourceMgr;
+    DiagnosticsEngine &Diags;
+    ModuleMap &Map;
+    
+    /// \brief Whether an error occurred.
+    bool HadError;
+    
+    /// \brief Default target information, used only for string literal
+    /// parsing.
+    TargetInfo *Target;
+    
+    /// \brief Stores string data for the various string literals referenced
+    /// during parsing.
+    llvm::BumpPtrAllocator StringData;
+    
+    /// \brief The current token.
+    MMToken Tok;
+    
+    /// \brief The active module.
+    ModuleMap::Module *ActiveModule;
+    
+    /// \brief Consume the current token and return its location.
+    SourceLocation consumeToken();
+    
+    /// \brief Skip tokens until we reach the a token with the given kind
+    /// (or the end of the file).
+    void skipUntil(MMToken::TokenKind K);
+    
+    void parseModuleDecl();
+    void parseUmbrellaDecl();
+    void parseHeaderDecl();
+    
+  public:
+    typedef ModuleMap::Module Module;
+    
+    explicit ModuleMapParser(Lexer &L, SourceManager &SourceMgr, 
+                             DiagnosticsEngine &Diags,
+                             ModuleMap &Map)
+      : L(L), SourceMgr(SourceMgr), Diags(Diags), Map(Map), HadError(false), 
+        ActiveModule(0)
+    {
+      TargetOptions TargetOpts;
+      TargetOpts.Triple = llvm::sys::getDefaultTargetTriple();
+      Target = TargetInfo::CreateTargetInfo(Diags, TargetOpts);
+      
+      Tok.clear();
+      consumeToken();
+    }
+    
+    bool parseModuleMapFile();
+  };
+}
+
+SourceLocation ModuleMapParser::consumeToken() {
+retry:
+  SourceLocation Result = Tok.getLocation();
+  Tok.clear();
+  
+  Token LToken;
+  L.LexFromRawLexer(LToken);
+  Tok.Location = LToken.getLocation().getRawEncoding();
+  switch (LToken.getKind()) {
+  case tok::raw_identifier:
+    Tok.StringData = LToken.getRawIdentifierData();
+    Tok.StringLength = LToken.getLength();
+    Tok.Kind = llvm::StringSwitch<MMToken::TokenKind>(Tok.getString())
+                 .Case("header", MMToken::HeaderKeyword)
+                 .Case("explicit", MMToken::ExplicitKeyword)
+                 .Case("module", MMToken::ModuleKeyword)
+                 .Case("umbrella", MMToken::UmbrellaKeyword)
+                 .Default(MMToken::Identifier);
+    break;
+      
+  case tok::eof:
+    Tok.Kind = MMToken::EndOfFile;
+    break;
+      
+  case tok::l_brace:
+    Tok.Kind = MMToken::LBrace;
+    break;
+
+  case tok::r_brace:
+    Tok.Kind = MMToken::RBrace;
+    break;
+      
+  case tok::string_literal: {
+    // Parse the string literal.
+    LangOptions LangOpts;
+    StringLiteralParser StringLiteral(&LToken, 1, SourceMgr, LangOpts, *Target);
+    if (StringLiteral.hadError)
+      goto retry;
+    
+    // Copy the string literal into our string data allocator.
+    unsigned Length = StringLiteral.GetStringLength();
+    char *Saved = StringData.Allocate<char>(Length + 1);
+    memcpy(Saved, StringLiteral.GetString().data(), Length);
+    Saved[Length] = 0;
+    
+    // Form the token.
+    Tok.Kind = MMToken::StringLiteral;
+    Tok.StringData = Saved;
+    Tok.StringLength = Length;
+    break;
+  }
+      
+  case tok::comment:
+    goto retry;
+      
+  default:
+    Diags.Report(LToken.getLocation(), diag::err_mmap_unknown_token);
+    HadError = true;
+    goto retry;
+  }
+  
+  return Result;
+}
+
+void ModuleMapParser::skipUntil(MMToken::TokenKind K) {
+  unsigned braceDepth = 0;
+  do {
+    switch (Tok.Kind) {
+    case MMToken::EndOfFile:
+      return;
+
+    case MMToken::LBrace:
+      if (Tok.is(K) && braceDepth == 0)
+        return;
+        
+      ++braceDepth;
+      break;
+    
+    case MMToken::RBrace:
+      if (braceDepth > 0)
+        --braceDepth;
+      else if (Tok.is(K))
+        return;
+      break;
+        
+    default:
+      if (braceDepth == 0 && Tok.is(K))
+        return;
+      break;
+    }
+    
+   consumeToken();
+  } while (true);
+}
+
+/// \brief Parse a module declaration.
+///
+///   module-declaration:
+///     'module' identifier { module-member* }
+///
+///   module-member:
+///     umbrella-declaration
+///     header-declaration
+///     'explicit'[opt] module-declaration
+void ModuleMapParser::parseModuleDecl() {
+  assert(Tok.is(MMToken::ExplicitKeyword) || Tok.is(MMToken::ModuleKeyword));
+  
+  // Parse 'explicit' keyword, if present.
+  bool Explicit = false;
+  if (Tok.is(MMToken::ExplicitKeyword)) {
+    consumeToken();
+    Explicit = true;
+  }
+  
+  // Parse 'module' keyword.
+  if (!Tok.is(MMToken::ModuleKeyword)) {
+    Diags.Report(Tok.getLocation(), 
+                 diag::err_mmap_expected_module_after_explicit);
+    consumeToken();
+    HadError = true;
+    return;
+  }
+  consumeToken(); // 'module' keyword
+  
+  // Parse the module name.
+  if (!Tok.is(MMToken::Identifier)) {
+    Diags.Report(Tok.getLocation(), diag::err_mmap_expected_module_name);
+    HadError = true;
+    return;    
+  }
+  StringRef ModuleName = Tok.getString();
+  SourceLocation ModuleNameLoc = consumeToken();
+  
+  // Parse the opening brace.
+  if (!Tok.is(MMToken::LBrace)) {
+    Diags.Report(Tok.getLocation(), diag::err_mmap_expected_lbrace)
+      << ModuleName;
+    HadError = true;
+    return;
+  }  
+  SourceLocation LBraceLoc = consumeToken();
+  
+  // Determine whether this (sub)module has already been defined.
+  llvm::StringMap<Module *> &ModuleSpace
+    = ActiveModule? ActiveModule->SubModules : Map.Modules;
+  llvm::StringMap<Module *>::iterator ExistingModule
+    = ModuleSpace.find(ModuleName);
+  if (ExistingModule != ModuleSpace.end()) {
+    Diags.Report(ModuleNameLoc, diag::err_mmap_module_redefinition)
+      << ModuleName;
+    Diags.Report(ExistingModule->getValue()->DefinitionLoc,
+                 diag::note_mmap_prev_definition);
+    
+    // Skip the module definition.
+    skipUntil(MMToken::RBrace);
+    if (Tok.is(MMToken::RBrace))
+      consumeToken();
+    
+    HadError = true;
+    return;
+  }
+
+  // Start defining this module.
+  ActiveModule = new Module(ModuleName, ModuleNameLoc, ActiveModule, Explicit);
+  ModuleSpace[ModuleName] = ActiveModule;
+  
+  bool Done = false;
+  do {
+    switch (Tok.Kind) {
+    case MMToken::EndOfFile:
+    case MMToken::RBrace:
+      Done = true;
+      break;
+        
+    case MMToken::ExplicitKeyword:
+    case MMToken::ModuleKeyword:
+      parseModuleDecl();
+      break;
+        
+    case MMToken::HeaderKeyword:
+      parseHeaderDecl();
+      break;
+        
+    case MMToken::UmbrellaKeyword:
+      parseUmbrellaDecl();
+      break;
+        
+    default:
+      Diags.Report(Tok.getLocation(), diag::err_mmap_expected_member);
+      consumeToken();
+      break;        
+    }
+  } while (!Done);
+
+  if (Tok.is(MMToken::RBrace))
+    consumeToken();
+  else {
+    Diags.Report(Tok.getLocation(), diag::err_mmap_expected_rbrace);
+    Diags.Report(LBraceLoc, diag::note_mmap_lbrace_match);
+    HadError = true;
+  }
+
+  // We're done parsing this module. Pop back to our parent scope.
+  ActiveModule = ActiveModule->Parent;
+}
+ 
+/// \brief Parse an umbrella header declaration.
+///
+///   umbrella-declaration:
+///     'umbrella' string-literal
+void ModuleMapParser::parseUmbrellaDecl() {
+  assert(Tok.is(MMToken::UmbrellaKeyword));
+  SourceLocation UmbrellaLoc = consumeToken();
+  
+  // Parse the header name.
+  if (!Tok.is(MMToken::StringLiteral)) {
+    Diags.Report(Tok.getLocation(), diag::err_mmap_expected_header) 
+      << "umbrella";
+    HadError = true;
+    return;
+  }
+  StringRef FileName = Tok.getString();
+  SourceLocation FileNameLoc = consumeToken();
+
+  // FIXME: Record the umbrella header.
+}
+
+/// \brief Parse a header declaration.
+///
+///   header-declaration:
+///     'header' string-literal
+void ModuleMapParser::parseHeaderDecl() {
+  assert(Tok.is(MMToken::HeaderKeyword));
+  SourceLocation HeaderLoc = consumeToken();
+  
+  // Parse the header name.
+  if (!Tok.is(MMToken::StringLiteral)) {
+    Diags.Report(Tok.getLocation(), diag::err_mmap_expected_header) 
+      << "header";
+    HadError = true;
+    return;
+  }
+  StringRef FileName = Tok.getString();
+  SourceLocation FileNameLoc = consumeToken();
+  
+  // FIXME: Record the header.  
+}
+
+/// \brief Parse a module map file.
+///
+///   module-map-file:
+///     module-declaration*
+bool ModuleMapParser::parseModuleMapFile() {
+  do {
+    switch (Tok.Kind) {
+    case MMToken::EndOfFile:
+      return HadError;
+      
+    case MMToken::ModuleKeyword:
+      parseModuleDecl();
+      break;
+      
+    case MMToken::ExplicitKeyword:
+    case MMToken::HeaderKeyword:
+    case MMToken::Identifier:
+    case MMToken::LBrace:
+    case MMToken::RBrace:
+    case MMToken::StringLiteral:
+    case MMToken::UmbrellaKeyword:
+      Diags.Report(Tok.getLocation(), diag::err_mmap_expected_module);
+      HadError = true;
+      consumeToken();
+      break;
+    }
+  } while (true);
+  
+  return HadError;
+}
+
+bool ModuleMap::parseModuleMapFile(const FileEntry *File) {
+  FileID ID = SourceMgr->createFileID(File, SourceLocation(), SrcMgr::C_User);
+  const llvm::MemoryBuffer *Buffer = SourceMgr->getBuffer(ID);
+  if (!Buffer)
+    return true;
+  
+  // Parse this module map file.
+  Lexer L(ID, SourceMgr->getBuffer(ID), *SourceMgr, LangOpts);
+  Diags->getClient()->BeginSourceFile(LangOpts);
+  ModuleMapParser Parser(L, *SourceMgr, *Diags, *this);
+  bool Result = Parser.parseModuleMapFile();
+  Diags->getClient()->EndSourceFile();
+  
+  return Result;
+}