Add an option to allocate JITed global data separately from code. By

default, this option is not enabled to support clients who rely on
this behavior.

Fixes http://llvm.org/PR4483

A patch to allocate additional memory for globals after we run out is
forthcoming.

Patch by Reid Kleckner!


git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@75059 91177308-0d34-0410-b5e6-96231b3b80d8

5 files changed, 126 insertions, 61 deletions

diff --git a/lib/ExecutionEngine/JIT/JIT.cpp b/lib/ExecutionEngine/JIT/JIT.cpp
index 785b5b9284..55ce16c354 100644
--- a/lib/ExecutionEngine/JIT/JIT.cpp
+++ b/lib/ExecutionEngine/JIT/JIT.cpp
@@ -197,8 +197,10 @@ void DarwinRegisterFrame(void* FrameBegin) {
 ExecutionEngine *ExecutionEngine::createJIT(ModuleProvider *MP,
                                             std::string *ErrorStr,
                                             JITMemoryManager *JMM,
-                                            CodeGenOpt::Level OptLevel) {
-  ExecutionEngine *EE = JIT::createJIT(MP, ErrorStr, JMM, OptLevel);
+                                            CodeGenOpt::Level OptLevel,
+                                            bool GVsWithCode) {
+  ExecutionEngine *EE = JIT::createJIT(MP, ErrorStr, JMM, OptLevel,
+                                       GVsWithCode);
   if (!EE) return 0;
   
   // Make sure we can resolve symbols in the program as well. The zero arg
@@ -208,8 +210,8 @@ ExecutionEngine *ExecutionEngine::createJIT(ModuleProvider *MP,
 }
 
 JIT::JIT(ModuleProvider *MP, TargetMachine &tm, TargetJITInfo &tji,
-         JITMemoryManager *JMM, CodeGenOpt::Level OptLevel)
-  : ExecutionEngine(MP), TM(tm), TJI(tji) {
+         JITMemoryManager *JMM, CodeGenOpt::Level OptLevel, bool GVsWithCode)
+  : ExecutionEngine(MP), TM(tm), TJI(tji), AllocateGVsWithCode(GVsWithCode) {
   setTargetData(TM.getTargetData());
 
   jitstate = new JITState(MP);
@@ -677,37 +679,11 @@ void *JIT::getOrEmitGlobalVariable(const GlobalVariable *GV) {
     }
     addGlobalMapping(GV, Ptr);
   } else {
-    // GlobalVariable's which are not "constant" will cause trouble in a server
-    // situation. It's returned in the same block of memory as code which may
-    // not be writable.
-    if (isGVCompilationDisabled() && !GV->isConstant()) {
-      cerr << "Compilation of non-internal GlobalValue is disabled!\n";
-      abort();
-    }
     // If the global hasn't been emitted to memory yet, allocate space and
-    // emit it into memory.  It goes in the same array as the generated
-    // code, jump tables, etc.
-    const Type *GlobalType = GV->getType()->getElementType();
-    size_t S = getTargetData()->getTypeAllocSize(GlobalType);
-    size_t A = getTargetData()->getPreferredAlignment(GV);
-    if (GV->isThreadLocal()) {
-      MutexGuard locked(lock);
-      Ptr = TJI.allocateThreadLocalMemory(S);
-    } else if (TJI.allocateSeparateGVMemory()) {
-      if (A <= 8) {
-        Ptr = malloc(S);
-      } else {
-        // Allocate S+A bytes of memory, then use an aligned pointer within that
-        // space.
-        Ptr = malloc(S+A);
-        unsigned MisAligned = ((intptr_t)Ptr & (A-1));
-        Ptr = (char*)Ptr + (MisAligned ? (A-MisAligned) : 0);
-      }
-    } else {
-      Ptr = JCE->allocateSpace(S, A);
-    }
+    // emit it into memory.
+    Ptr = getMemoryForGV(GV);
     addGlobalMapping(GV, Ptr);
-    EmitGlobalVariable(GV);
+    EmitGlobalVariable(GV);  // Initialize the variable.
   }
   return Ptr;
 }
@@ -742,14 +718,42 @@ void *JIT::recompileAndRelinkFunction(Function *F) {
 /// on the target.
 ///
 char* JIT::getMemoryForGV(const GlobalVariable* GV) {
-  const Type *ElTy = GV->getType()->getElementType();
-  size_t GVSize = (size_t)getTargetData()->getTypeAllocSize(ElTy);
+  char *Ptr;
+
+  // GlobalVariable's which are not "constant" will cause trouble in a server
+  // situation. It's returned in the same block of memory as code which may
+  // not be writable.
+  if (isGVCompilationDisabled() && !GV->isConstant()) {
+    cerr << "Compilation of non-internal GlobalValue is disabled!\n";
+    abort();
+  }
+
+  // Some applications require globals and code to live together, so they may
+  // be allocated into the same buffer, but in general globals are allocated
+  // through the memory manager which puts them near the code but not in the
+  // same buffer.
+  const Type *GlobalType = GV->getType()->getElementType();
+  size_t S = getTargetData()->getTypeAllocSize(GlobalType);
+  size_t A = getTargetData()->getPreferredAlignment(GV);
   if (GV->isThreadLocal()) {
     MutexGuard locked(lock);
-    return TJI.allocateThreadLocalMemory(GVSize);
+    Ptr = TJI.allocateThreadLocalMemory(S);
+  } else if (TJI.allocateSeparateGVMemory()) {
+    if (A <= 8) {
+      Ptr = (char*)malloc(S);
+    } else {
+      // Allocate S+A bytes of memory, then use an aligned pointer within that
+      // space.
+      Ptr = (char*)malloc(S+A);
+      unsigned MisAligned = ((intptr_t)Ptr & (A-1));
+      Ptr = Ptr + (MisAligned ? (A-MisAligned) : 0);
+    }
+  } else if (AllocateGVsWithCode) {
+    Ptr = (char*)JCE->allocateSpace(S, A);
   } else {
-    return new char[GVSize];
+    Ptr = (char*)JCE->allocateGlobal(S, A);
   }
+  return Ptr;
 }
 
 void JIT::addPendingFunction(Function *F) {
diff --git a/lib/ExecutionEngine/JIT/JIT.h b/lib/ExecutionEngine/JIT/JIT.h
index 66417a71b2..c992509b08 100644
--- a/lib/ExecutionEngine/JIT/JIT.h
+++ b/lib/ExecutionEngine/JIT/JIT.h
@@ -55,10 +55,16 @@ class JIT : public ExecutionEngine {
   JITCodeEmitter *JCE;     // JCE object
   std::vector<JITEventListener*> EventListeners;
 
+  /// AllocateGVsWithCode - Some applications require that global variables and
+  /// code be allocated into the same region of memory, in which case this flag
+  /// should be set to true.  Doing so breaks freeMachineCodeForFunction.
+  bool AllocateGVsWithCode;
+
   JITState *jitstate;
 
-  JIT(ModuleProvider *MP, TargetMachine &tm, TargetJITInfo &tji, 
-      JITMemoryManager *JMM, CodeGenOpt::Level OptLevel);
+  JIT(ModuleProvider *MP, TargetMachine &tm, TargetJITInfo &tji,
+      JITMemoryManager *JMM, CodeGenOpt::Level OptLevel,
+      bool AllocateGVsWithCode);
 public:
   ~JIT();
 
@@ -75,8 +81,9 @@ public:
   ///
   static ExecutionEngine *create(ModuleProvider *MP, std::string *Err,
                                  CodeGenOpt::Level OptLevel =
-                                   CodeGenOpt::Default) {
-    return createJIT(MP, Err, 0, OptLevel);
+                                   CodeGenOpt::Default,
+                                 bool AllocateGVsWithCode = true) {
+    return createJIT(MP, Err, 0, OptLevel, AllocateGVsWithCode);
   }
 
   virtual void addModuleProvider(ModuleProvider *MP);
@@ -151,9 +158,11 @@ public:
   /// getCodeEmitter - Return the code emitter this JIT is emitting into.
   JITCodeEmitter *getCodeEmitter() const { return JCE; }
   
-  static ExecutionEngine *createJIT(ModuleProvider *MP, std::string *Err,
+  static ExecutionEngine *createJIT(ModuleProvider *MP,
+                                    std::string *Err,
                                     JITMemoryManager *JMM,
-                                    CodeGenOpt::Level OptLevel);
+                                    CodeGenOpt::Level OptLevel,
+                                    bool AllocateGVsWithCode);
 
 
   // Run the JIT on F and return information about the generated code
diff --git a/lib/ExecutionEngine/JIT/JITEmitter.cpp b/lib/ExecutionEngine/JIT/JITEmitter.cpp
index 8fe7ab848b..cfc3680e8a 100644
--- a/lib/ExecutionEngine/JIT/JITEmitter.cpp
+++ b/lib/ExecutionEngine/JIT/JITEmitter.cpp
@@ -527,6 +527,11 @@ namespace {
     /// allocate a new one of the given size.
     virtual void *allocateSpace(uintptr_t Size, unsigned Alignment);
 
+    /// allocateGlobal - Allocate memory for a global.  Unlike allocateSpace,
+    /// this method does not allocate memory in the current output buffer,
+    /// because a global may live longer than the current function.
+    virtual void *allocateGlobal(uintptr_t Size, unsigned Alignment);
+
     virtual void addRelocation(const MachineRelocation &MR) {
       Relocations.push_back(MR);
     }
@@ -1161,6 +1166,11 @@ void* JITEmitter::allocateSpace(uintptr_t Size, unsigned Alignment) {
   return CurBufferPtr;
 }
 
+void* JITEmitter::allocateGlobal(uintptr_t Size, unsigned Alignment) {
+  // Delegate this call through the memory manager.
+  return MemMgr->allocateGlobal(Size, Alignment);
+}
+
 void JITEmitter::emitConstantPool(MachineConstantPool *MCP) {
   if (TheJIT->getJITInfo().hasCustomConstantPool())
     return;
diff --git a/lib/ExecutionEngine/JIT/JITMemoryManager.cpp b/lib/ExecutionEngine/JIT/JITMemoryManager.cpp
index 70ccdccb80..9820493828 100644
--- a/lib/ExecutionEngine/JIT/JITMemoryManager.cpp
+++ b/lib/ExecutionEngine/JIT/JITMemoryManager.cpp
@@ -251,6 +251,8 @@ namespace {
   /// middle of emitting a function, and we don't know how large the function we
   /// are emitting is.
   class VISIBILITY_HIDDEN DefaultJITMemoryManager : public JITMemoryManager {
+    bool PoisonMemory;  // Whether to poison freed memory.
+
     std::vector<sys::MemoryBlock> Blocks; // Memory blocks allocated by the JIT
     FreeRangeHeader *FreeMemoryList;      // Circular list of free blocks.
     
@@ -258,6 +260,7 @@ namespace {
     MemoryRangeHeader *CurBlock;
     
     uint8_t *CurStubPtr, *StubBase;
+    uint8_t *CurGlobalPtr, *GlobalEnd;
     uint8_t *GOTBase;     // Target Specific reserved memory
     void *DlsymTable;     // Stub external symbol information
 
@@ -324,7 +327,7 @@ namespace {
       CurBlock = FreeMemoryList;
       FreeMemoryList = FreeMemoryList->AllocateBlock();
 
-      uint8_t *result = (uint8_t *)CurBlock+1;
+      uint8_t *result = (uint8_t *)(CurBlock + 1);
 
       if (Alignment == 0) Alignment = 1;
       result = (uint8_t*)(((intptr_t)result+Alignment-1) &
@@ -336,6 +339,30 @@ namespace {
       return result;
     }
 
+    /// allocateGlobal - Allocate memory for a global.  Unlike allocateSpace,
+    /// this method does not touch the current block and can be called at any
+    /// time.
+    uint8_t *allocateGlobal(uintptr_t Size, unsigned Alignment) {
+      uint8_t *Result = CurGlobalPtr;
+
+      // Align the pointer.
+      if (Alignment == 0) Alignment = 1;
+      Result = (uint8_t*)(((uintptr_t)Result + Alignment-1) &
+                          ~(uintptr_t)(Alignment-1));
+
+      // Move the current global pointer forward.
+      CurGlobalPtr += Result - CurGlobalPtr + Size;
+
+      // Check for overflow.
+      if (CurGlobalPtr > GlobalEnd) {
+        // FIXME: Allocate more memory.
+        fprintf(stderr, "JIT ran out of memory for globals!\n");
+        abort();
+      }
+
+      return Result;
+    }
+
     /// startExceptionTable - Use startFunctionBody to allocate memory for the 
     /// function's exception table.
     uint8_t* startExceptionTable(const Function* F, uintptr_t &ActualSize) {
@@ -375,12 +402,12 @@ namespace {
       // Find the block that is allocated for this function.
       MemoryRangeHeader *MemRange = I->second;
       assert(MemRange->ThisAllocated && "Block isn't allocated!");
-      
+
       // Fill the buffer with garbage!
-#ifndef NDEBUG
-      memset(MemRange+1, 0xCD, MemRange->BlockSize-sizeof(*MemRange));
-#endif
-      
+      if (PoisonMemory) {
+        memset(MemRange+1, 0xCD, MemRange->BlockSize-sizeof(*MemRange));
+      }
+
       // Free the memory.
       FreeMemoryList = MemRange->FreeBlock(FreeMemoryList);
       
@@ -393,12 +420,12 @@ namespace {
       // Find the block that is allocated for this function.
       MemRange = I->second;
       assert(MemRange->ThisAllocated && "Block isn't allocated!");
-      
+
       // Fill the buffer with garbage!
-#ifndef NDEBUG
-      memset(MemRange+1, 0xCD, MemRange->BlockSize-sizeof(*MemRange));
-#endif
-      
+      if (PoisonMemory) {
+        memset(MemRange+1, 0xCD, MemRange->BlockSize-sizeof(*MemRange));
+      }
+
       // Free the memory.
       FreeMemoryList = MemRange->FreeBlock(FreeMemoryList);
       
@@ -420,10 +447,22 @@ namespace {
       for (unsigned i = 0, e = Blocks.size(); i != e; ++i)
         sys::Memory::setExecutable(Blocks[i]);
     }
+
+    /// setPoisonMemory - Controls whether we write garbage over freed memory.
+    ///
+    void setPoisonMemory(bool poison) {
+      PoisonMemory = poison;
+    }
   };
 }
 
 DefaultJITMemoryManager::DefaultJITMemoryManager() {
+#ifdef NDEBUG
+  PoisonMemory = true;
+#else
+  PoisonMemory = false;
+#endif
+
   // Allocate a 16M block of memory for functions.
 #if defined(__APPLE__) && defined(__arm__)
   sys::MemoryBlock MemBlock = getNewMemoryBlock(4 << 20);
@@ -433,11 +472,13 @@ DefaultJITMemoryManager::DefaultJITMemoryManager() {
 
   uint8_t *MemBase = static_cast<uint8_t*>(MemBlock.base());
 
-  // Allocate stubs backwards from the base, allocate functions forward
-  // from the base.
+  // Allocate stubs backwards to the base, globals forward from the stubs, and
+  // functions forward after globals.
   StubBase   = MemBase;
   CurStubPtr = MemBase + 512*1024; // Use 512k for stubs, working backwards.
-  
+  CurGlobalPtr = CurStubPtr;       // Use 2M for globals, working forwards.
+  GlobalEnd = CurGlobalPtr + 2*1024*1024;
+
   // We set up the memory chunk with 4 mem regions, like this:
   //  [ START
   //    [ Free      #0 ] -> Large space to allocate functions from.
@@ -474,7 +515,7 @@ DefaultJITMemoryManager::DefaultJITMemoryManager() {
   // Add a FreeRangeHeader to the start of the function body region, indicating
   // that the space is free.  Mark the previous block allocated so we never look
   // at it.
-  FreeRangeHeader *Mem0 = (FreeRangeHeader*)CurStubPtr;
+  FreeRangeHeader *Mem0 = (FreeRangeHeader*)GlobalEnd;
   Mem0->ThisAllocated = 0;
   Mem0->PrevAllocated = 1;
   Mem0->BlockSize = (char*)Mem1-(char*)Mem0;
@@ -522,7 +563,7 @@ uint8_t *DefaultJITMemoryManager::allocateStub(const GlobalValue* F,
 
 sys::MemoryBlock DefaultJITMemoryManager::getNewMemoryBlock(unsigned size) {
   // Allocate a new block close to the last one.
-  const sys::MemoryBlock *BOld = Blocks.empty() ? 0 : &Blocks.front();
+  const sys::MemoryBlock *BOld = Blocks.empty() ? 0 : &Blocks.back();
   std::string ErrMsg;
   sys::MemoryBlock B = sys::Memory::AllocateRWX(size, BOld, &ErrMsg);
   if (B.base() == 0) {
diff --git a/lib/ExecutionEngine/JIT/TargetSelect.cpp b/lib/ExecutionEngine/JIT/TargetSelect.cpp
index 0f20819307..24dd013639 100644
--- a/lib/ExecutionEngine/JIT/TargetSelect.cpp
+++ b/lib/ExecutionEngine/JIT/TargetSelect.cpp
@@ -43,7 +43,8 @@ MAttrs("mattr",
 ///
 ExecutionEngine *JIT::createJIT(ModuleProvider *MP, std::string *ErrorStr,
                                 JITMemoryManager *JMM,
-                                CodeGenOpt::Level OptLevel) {
+                                CodeGenOpt::Level OptLevel,
+                                bool AllocateGVsWithCode) {
   const TargetMachineRegistry::entry *TheArch = MArch;
   if (TheArch == 0) {
     std::string Error;
@@ -75,7 +76,7 @@ ExecutionEngine *JIT::createJIT(ModuleProvider *MP, std::string *ErrorStr,
 
   // If the target supports JIT code generation, return a new JIT now.
   if (TargetJITInfo *TJ = Target->getJITInfo())
-    return new JIT(MP, *Target, *TJ, JMM, OptLevel);
+    return new JIT(MP, *Target, *TJ, JMM, OptLevel, AllocateGVsWithCode);
 
   if (ErrorStr)
     *ErrorStr = "target does not support JIT code generation";