Support for mem* library functions in stable bitcode via intrinsics.

* Don't preserve external linking for mem{cpy,move,cmp} during LTO.
* In the RewritePNaClLibraryCalls pass - add rewriting of mem* calls to
appropriate intrinsics, similarly to the way it was done for longjmp.

BUG=https://code.google.com/p/nativeclient/issues/detail?id=3493
R=mseaborn@chromium.org

Review URL: https://codereview.chromium.org/17622003

4 files changed, 442 insertions, 108 deletions

diff --git a/lib/CodeGen/IntrinsicLowering.cpp b/lib/CodeGen/IntrinsicLowering.cpp
index 0c035718a4..f16e5978d9 100644
--- a/lib/CodeGen/IntrinsicLowering.cpp
+++ b/lib/CodeGen/IntrinsicLowering.cpp
@@ -108,7 +108,6 @@ static CallInst *ReplaceCallWith(const char *NewFn, CallInst *CI,
 // Bitcode can then internalize symbols.
 static const char *IntrinsicNames[] = {
   "abort",
-  "memcpy", "memset", "memmove",
   "powf", "pow", "powl",
   NULL
 };
diff --git a/lib/Transforms/NaCl/RewritePNaClLibraryCalls.cpp b/lib/Transforms/NaCl/RewritePNaClLibraryCalls.cpp
index ea50457070..726c3157c1 100644
--- a/lib/Transforms/NaCl/RewritePNaClLibraryCalls.cpp
+++ b/lib/Transforms/NaCl/RewritePNaClLibraryCalls.cpp
@@ -19,6 +19,7 @@
 #include "llvm/IR/Module.h"
 #include "llvm/Pass.h"
 #include "llvm/Transforms/NaCl.h"
+#include <cstdarg>
 
 using namespace llvm;
 
@@ -27,7 +28,9 @@ namespace {
   public:
     static char ID;
     RewritePNaClLibraryCalls() :
-        ModulePass(ID), TheModule(NULL) {
+        ModulePass(ID), TheModule(NULL), Context(NULL),
+        LongjmpIntrinsic(NULL), MemcpyIntrinsic(NULL),
+        MemmoveIntrinsic(NULL), MemsetIntrinsic(NULL) {
       // This is a module pass because it may have to introduce
       // intrinsic declarations into the module and modify a global function.
       initializeRewritePNaClLibraryCallsPass(*PassRegistry::getPassRegistry());
@@ -35,23 +38,79 @@ namespace {
 
     virtual bool runOnModule(Module &M);
   private:
-    /// Rewrites the given \p Call of setjmp to a direct intrinsic call.
-    void rewriteSetjmpCall(CallInst *Call);
-
-    /// Rewrites the given \p Call of longjmp to a direct intrinsic call.
-    void rewriteLongjmpCall(CallInst *Call);
+    typedef void (RewritePNaClLibraryCalls::*SanityCheckFunc)(Function *);
+    typedef void (RewritePNaClLibraryCalls::*RewriteCallFunc)(CallInst *);
+    typedef void (RewritePNaClLibraryCalls::*PopulateWrapperFunc)(Function *);
+
+    /// Handles a certain pattern of library function -> intrinsic rewrites.
+    /// Currently all library functions this pass knows how to rewrite fall into
+    /// this pattern.
+    /// RewriteLibraryCall performs the rewrite for a single library function
+    /// and is customized by a number of method pointers that collectively
+    /// handle one of the supported library functions.
+    ///
+    /// \p LibraryFunctionName Name of the library function to look for.
+    /// \p SanityChecker Method that makes sure the library function has the
+    ///    signature we expect it to have.
+    /// \p CallRewriter Method that rewrites the library function call into an
+    ///    intrinsic call.
+    /// \p OnlyCallsAllowed True iff only calls are allowed to this library
+    ///    function.
+    /// \p WrapperPopulator If not only calls are allowed, this method is
+    ///    called to populate the body of the library function with a wrapped
+    ///    intrinsic call. If only calls are allowed, this parameter may be set
+    ///    to NULL.
+    bool RewriteLibraryCall(
+        const char *LibraryFunctionName,
+        SanityCheckFunc SanityChecker,
+        RewriteCallFunc CallRewriter,
+        bool OnlyCallsAllowed,
+        PopulateWrapperFunc WrapperPopulator);
 
-    /// Populates the body of longjmp as a wrapper of the intrinsic call.
-    /// Should only be called once. Modifies the given \p LongjmpFunc.
-    void populateLongjmpWrapper(Function *LongjmpFunc);
-
-    /// Sanity check that the types of these functions are what we expect them
-    /// to be.
     void sanityCheckSetjmpFunc(Function *SetjmpFunc);
     void sanityCheckLongjmpFunc(Function *LongjmpFunc);
+    void sanityCheckMemcpyFunc(Function *MemcpyFunc);
+    void sanityCheckMemmoveFunc(Function *MemmoveFunc);
+    void sanityCheckMemsetFunc(Function *MemsetFunc);
+
+    void rewriteSetjmpCall(CallInst *Call);
+    void rewriteLongjmpCall(CallInst *Call);
+    void rewriteMemcpyCall(CallInst *Call);
+    void rewriteMemmoveCall(CallInst *Call);
+    void rewriteMemsetCall(CallInst *Call);
 
-    /// The module this pass runs on.
+    void populateLongjmpWrapper(Function *LongjmpFunc);
+    void populateMemcpyWrapper(Function *MemcpyFunc);
+    void populateMemmoveWrapper(Function *MemmoveFunc);
+    void populateMemsetWrapper(Function *MemsetFunc);
+
+    /// Generic implementation of populating a wrapper function.
+    /// Initially, the function exists in the module as a declaration with
+    /// unnamed arguments. This method is called with a NULL-terminated list
+    /// of argument names that get assigned in the generated IR for
+    /// readability.
+    void populateWrapperCommon(
+        Function *Func,
+        StringRef FuncName,
+        RewriteCallFunc CallRewriter,
+        bool CallCannotReturn,
+        ...);
+
+    /// Find and cache known intrinsics.
+    Function *findLongjmpIntrinsic();
+    Function *findMemcpyIntrinsic();
+    Function *findMemmoveIntrinsic();
+    Function *findMemsetIntrinsic();
+
+    /// Cached data that remains the same throughout a module run.
     Module *TheModule;
+    LLVMContext *Context;
+
+    /// These are cached but computed lazily.
+    Function *LongjmpIntrinsic;
+    Function *MemcpyIntrinsic;
+    Function *MemmoveIntrinsic;
+    Function *MemsetIntrinsic;
   };
 }
 
@@ -60,60 +119,46 @@ INITIALIZE_PASS(RewritePNaClLibraryCalls, "rewrite-pnacl-library-calls",
                 "Rewrite PNaCl library calls to stable intrinsics",
                 false, false)
 
-bool RewritePNaClLibraryCalls::runOnModule(Module &M) {
-  TheModule = &M;
+bool RewritePNaClLibraryCalls::RewriteLibraryCall(
+    const char *LibraryFunctionName,
+    SanityCheckFunc SanityChecker,
+    RewriteCallFunc CallRewriter,
+    bool OnlyCallsAllowed,
+    PopulateWrapperFunc WrapperPopulator) {
   bool Changed = false;
 
-  // Iterate over all uses of the setjmp, if it exists in the module with
-  // external linkage. If it exists but the linkage is not external, this may
-  // come from code that defines its own function named setjmp and doesn't
-  // include <setjmp.h>. In such a case we leave the code as it is.
-  //
-  // The calls are replaced with intrinsics. All other uses of setjmp are
-  // disallowed (taking the address of setjmp is disallowed in C and C++).
-  Function *SetjmpFunc = TheModule->getFunction("setjmp");
-
-  if (SetjmpFunc && SetjmpFunc->hasExternalLinkage()) {
-    sanityCheckSetjmpFunc(SetjmpFunc);
-
-    for (Value::use_iterator UI = SetjmpFunc->use_begin(),
-                             UE = SetjmpFunc->use_end(); UI != UE;) {
-      Value *Use = *UI++;
-      if (CallInst *Call = dyn_cast<CallInst>(Use)) {
-        rewriteSetjmpCall(Call);
-        Changed = true;
-      } else {
-        report_fatal_error("Taking the address of setjmp is invalid");
-      }
-    }
-    SetjmpFunc->eraseFromParent();
-  }
-
-  // For longjmp things are a little more complicated, since longjmp's address
-  // can be taken. Therefore, longjmp can appear in a variety of Uses. The
-  // common case is still a direct call and we want that to be as efficient as
-  // possible, so we rewrite it into a direct intrinsic call. If there are other
-  // uses, the actual body of longjmp is populated with a wrapper that calls
-  // the intrinsic.
-  Function *LongjmpFunc = TheModule->getFunction("longjmp");
-
-  if (LongjmpFunc && LongjmpFunc->hasExternalLinkage()) {
-    sanityCheckLongjmpFunc(LongjmpFunc);
+  Function *LibFunc = TheModule->getFunction(LibraryFunctionName);
 
-    for (Value::use_iterator UI = LongjmpFunc->use_begin(),
-                             UE = LongjmpFunc->use_end(); UI != UE;) {
+  // Iterate over all uses of this function, if it exists in the module with
+  // external linkage. If it exists but the linkage is not external, this may
+  // come from code that defines its own private function with the same name
+  // and doesn't actually include the standard libc header declaring it.
+  // In such a case we leave the code as it is.
+  if (LibFunc && LibFunc->hasExternalLinkage()) {
+    (this->*(SanityChecker))(LibFunc);
+
+    // Handle all uses that are calls. These are simply replaced with
+    // equivalent intrinsic calls.
+    for (Value::use_iterator UI = LibFunc->use_begin(),
+                             UE = LibFunc->use_end(); UI != UE;) {
       Value *Use = *UI++;
       if (CallInst *Call = dyn_cast<CallInst>(Use)) {
-        rewriteLongjmpCall(Call);
+        (this->*(CallRewriter))(Call);
         Changed = true;
       }
     }
 
-    // If additional uses remain, these aren't calls; populate the wrapper.
-    if (LongjmpFunc->use_empty()) {
-      LongjmpFunc->eraseFromParent();
+    if (LibFunc->use_empty()) {
+      LibFunc->eraseFromParent();
+    } else if (OnlyCallsAllowed) {
+      // If additional uses remain, these aren't calls.
+      report_fatal_error(Twine("Taking the address of ") +
+                         LibraryFunctionName + " is invalid");
     } else {
-      populateLongjmpWrapper(LongjmpFunc);
+      // If non-call uses remain and allowed for this function, populate it
+      // with a wrapper.
+      (this->*(WrapperPopulator))(LibFunc);
+      LibFunc->setLinkage(Function::InternalLinkage);
       Changed = true;
     }
   }
@@ -121,27 +166,43 @@ bool RewritePNaClLibraryCalls::runOnModule(Module &M) {
   return Changed;
 }
 
-void RewritePNaClLibraryCalls::rewriteSetjmpCall(CallInst *Call) {
-  // Find the intrinsic function.
-  Function *NaClSetjmpFunc = Intrinsic::getDeclaration(TheModule,
-                                                       Intrinsic::nacl_setjmp);
-  // Cast the jmp_buf argument to the type NaClSetjmpCall expects.
-  Type *PtrTy = NaClSetjmpFunc->getFunctionType()->getParamType(0);
-  BitCastInst *JmpBufCast = new BitCastInst(Call->getArgOperand(0), PtrTy,
-                                            "jmp_buf_i8", Call);
-  const DebugLoc &DLoc = Call->getDebugLoc();
-  JmpBufCast->setDebugLoc(DLoc);
+bool RewritePNaClLibraryCalls::runOnModule(Module &M) {
+  TheModule = &M;
+  Context = &TheModule->getContext();
+  bool Changed = false;
 
-  // Emit the updated call.
-  SmallVector<Value *, 1> Args;
-  Args.push_back(JmpBufCast);
-  CallInst *NaClSetjmpCall = CallInst::Create(NaClSetjmpFunc, Args, "", Call);
-  NaClSetjmpCall->setDebugLoc(DLoc);
-  NaClSetjmpCall->takeName(Call);
+  Changed |= RewriteLibraryCall(
+      "setjmp",
+      &RewritePNaClLibraryCalls::sanityCheckSetjmpFunc,
+      &RewritePNaClLibraryCalls::rewriteSetjmpCall,
+      true,
+      NULL);
+  Changed |= RewriteLibraryCall(
+      "longjmp",
+      &RewritePNaClLibraryCalls::sanityCheckLongjmpFunc,
+      &RewritePNaClLibraryCalls::rewriteLongjmpCall,
+      false,
+      &RewritePNaClLibraryCalls::populateLongjmpWrapper);
+  Changed |= RewriteLibraryCall(
+      "memset",
+      &RewritePNaClLibraryCalls::sanityCheckMemsetFunc,
+      &RewritePNaClLibraryCalls::rewriteMemsetCall,
+      false,
+      &RewritePNaClLibraryCalls::populateMemsetWrapper);
+  Changed |= RewriteLibraryCall(
+      "memcpy",
+      &RewritePNaClLibraryCalls::sanityCheckMemcpyFunc,
+      &RewritePNaClLibraryCalls::rewriteMemcpyCall,
+      false,
+      &RewritePNaClLibraryCalls::populateMemcpyWrapper);
+  Changed |= RewriteLibraryCall(
+      "memmove",
+      &RewritePNaClLibraryCalls::sanityCheckMemmoveFunc,
+      &RewritePNaClLibraryCalls::rewriteMemmoveCall,
+      false,
+      &RewritePNaClLibraryCalls::populateMemmoveWrapper);
 
-  // Replace the original call.
-  Call->replaceAllUsesWith(NaClSetjmpCall);
-  Call->eraseFromParent();
+  return Changed;
 }
 
 void RewritePNaClLibraryCalls::sanityCheckLongjmpFunc(Function *LongjmpFunc) {
@@ -163,10 +224,64 @@ void RewritePNaClLibraryCalls::sanityCheckSetjmpFunc(Function *SetjmpFunc) {
   }
 }
 
+void RewritePNaClLibraryCalls::sanityCheckMemsetFunc(Function *MemsetFunc) {
+  FunctionType *FTy = MemsetFunc->getFunctionType();
+  if (!(FTy->getNumParams() == 3 &&
+        FTy->getReturnType()->isPointerTy() &&
+        FTy->getParamType(0)->isPointerTy() &&
+        FTy->getParamType(1)->isIntegerTy() &&
+        FTy->getParamType(2)->isIntegerTy())) {
+    report_fatal_error("Wrong signature of memset");
+  }
+}
+
+void RewritePNaClLibraryCalls::sanityCheckMemcpyFunc(Function *MemcpyFunc) {
+  FunctionType *FTy = MemcpyFunc->getFunctionType();
+  if (!(FTy->getNumParams() == 3 &&
+        FTy->getReturnType()->isPointerTy() &&
+        FTy->getParamType(0)->isPointerTy() &&
+        FTy->getParamType(1)->isPointerTy() &&
+        FTy->getParamType(2)->isIntegerTy())) {
+    report_fatal_error("Wrong signature of memcpy");
+  }
+}
+
+void RewritePNaClLibraryCalls::sanityCheckMemmoveFunc(Function *MemmoveFunc) {
+  FunctionType *FTy = MemmoveFunc->getFunctionType();
+  if (!(FTy->getNumParams() == 3 &&
+        FTy->getReturnType()->isPointerTy() &&
+        FTy->getParamType(0)->isPointerTy() &&
+        FTy->getParamType(1)->isPointerTy() &&
+        FTy->getParamType(2)->isIntegerTy())) {
+    report_fatal_error("Wrong signature of memmove");
+  }
+}
+
+void RewritePNaClLibraryCalls::rewriteSetjmpCall(CallInst *Call) {
+  // Find the intrinsic function.
+  Function *NaClSetjmpFunc = Intrinsic::getDeclaration(TheModule,
+                                                       Intrinsic::nacl_setjmp);
+  // Cast the jmp_buf argument to the type NaClSetjmpCall expects.
+  Type *PtrTy = NaClSetjmpFunc->getFunctionType()->getParamType(0);
+  BitCastInst *JmpBufCast = new BitCastInst(Call->getArgOperand(0), PtrTy,
+                                            "jmp_buf_i8", Call);
+  const DebugLoc &DLoc = Call->getDebugLoc();
+  JmpBufCast->setDebugLoc(DLoc);
+
+  // Emit the updated call.
+  Value *Args[] = { JmpBufCast };
+  CallInst *NaClSetjmpCall = CallInst::Create(NaClSetjmpFunc, Args, "", Call);
+  NaClSetjmpCall->setDebugLoc(DLoc);
+  NaClSetjmpCall->takeName(Call);
+
+  // Replace the original call.
+  Call->replaceAllUsesWith(NaClSetjmpCall);
+  Call->eraseFromParent();
+}
+
 void RewritePNaClLibraryCalls::rewriteLongjmpCall(CallInst *Call) {
   // Find the intrinsic function.
-  Function *NaClLongjmpFunc = Intrinsic::getDeclaration(
-      TheModule, Intrinsic::nacl_longjmp);
+  Function *NaClLongjmpFunc = findLongjmpIntrinsic();
   // Cast the jmp_buf argument to the type NaClLongjmpCall expects.
   Type *PtrTy = NaClLongjmpFunc->getFunctionType()->getParamType(0);
   BitCastInst *JmpBufCast = new BitCastInst(Call->getArgOperand(0), PtrTy,
@@ -175,9 +290,7 @@ void RewritePNaClLibraryCalls::rewriteLongjmpCall(CallInst *Call) {
   JmpBufCast->setDebugLoc(DLoc);
 
   // Emit the call.
-  SmallVector<Value *, 2> Args;
-  Args.push_back(JmpBufCast);
-  Args.push_back(Call->getArgOperand(1));
+  Value *Args[] = { JmpBufCast, Call->getArgOperand(1) };
   CallInst *NaClLongjmpCall = CallInst::Create(NaClLongjmpFunc, Args, "", Call);
   NaClLongjmpCall->setDebugLoc(DLoc);
   // No takeName here since longjmp is a void call that does not get assigned to
@@ -188,39 +301,191 @@ void RewritePNaClLibraryCalls::rewriteLongjmpCall(CallInst *Call) {
   Call->eraseFromParent();
 }
 
+void RewritePNaClLibraryCalls::rewriteMemcpyCall(CallInst *Call) {
+  Function *MemcpyIntrinsic = findMemcpyIntrinsic();
+  // dest, src, len, align, isvolatile
+  Value *Args[] = { Call->getArgOperand(0),
+                    Call->getArgOperand(1),
+                    Call->getArgOperand(2),
+                    ConstantInt::get(Type::getInt32Ty(*Context), 1),
+                    ConstantInt::get(Type::getInt1Ty(*Context), 0) };
+  CallInst *MemcpyIntrinsicCall = CallInst::Create(MemcpyIntrinsic,
+                                                   Args, "", Call);
+  MemcpyIntrinsicCall->setDebugLoc(Call->getDebugLoc());
+
+  // libc memcpy returns the source pointer, but the LLVM intrinsic doesn't; if
+  // the return value has actual uses, just replace them with the dest
+  // argument itself.
+  Call->replaceAllUsesWith(Call->getArgOperand(0));
+  Call->eraseFromParent();
+}
+
+void RewritePNaClLibraryCalls::rewriteMemmoveCall(CallInst *Call) {
+  Function *MemmoveIntrinsic = findMemmoveIntrinsic();
+  // dest, src, len, align, isvolatile
+  Value *Args[] = { Call->getArgOperand(0),
+                    Call->getArgOperand(1),
+                    Call->getArgOperand(2),
+                    ConstantInt::get(Type::getInt32Ty(*Context), 1),
+                    ConstantInt::get(Type::getInt1Ty(*Context), 0) };
+  CallInst *MemmoveIntrinsicCall = CallInst::Create(MemmoveIntrinsic,
+                                                    Args, "", Call);
+  MemmoveIntrinsicCall->setDebugLoc(Call->getDebugLoc());
+
+  // libc memmove returns the source pointer, but the LLVM intrinsic doesn't; if
+  // the return value has actual uses, just replace them with the dest
+  // argument itself.
+  Call->replaceAllUsesWith(Call->getArgOperand(0));
+  Call->eraseFromParent();
+}
+
+void RewritePNaClLibraryCalls::rewriteMemsetCall(CallInst *Call) {
+  Function *MemsetIntrinsic = findMemsetIntrinsic();
+  // libc memset has 'int c' for the filler byte, but the LLVM intrinsic uses
+  // a i8; truncation is required.
+  TruncInst *ByteTrunc = new TruncInst(Call->getArgOperand(1),
+                                       Type::getInt8Ty(*Context),
+                                       "trunc_byte", Call);
+
+  const DebugLoc &DLoc = Call->getDebugLoc();
+  ByteTrunc->setDebugLoc(DLoc);
+
+  // dest, val, len, align, isvolatile
+  Value *Args[] = { Call->getArgOperand(0),
+                    ByteTrunc,
+                    Call->getArgOperand(2),
+                    ConstantInt::get(Type::getInt32Ty(*Context), 1),
+                    ConstantInt::get(Type::getInt1Ty(*Context), 0) };
+  CallInst *MemsetIntrinsicCall = CallInst::Create(MemsetIntrinsic,
+                                                   Args, "", Call);
+  MemsetIntrinsicCall->setDebugLoc(DLoc);
+
+  // libc memset returns the source pointer, but the LLVM intrinsic doesn't; if
+  // the return value has actual uses, just replace them with the dest
+  // argument itself.
+  Call->replaceAllUsesWith(Call->getArgOperand(0));
+  Call->eraseFromParent();
+}
+
+void RewritePNaClLibraryCalls::populateWrapperCommon(
+      Function *Func,
+      StringRef FuncName,
+      RewriteCallFunc CallRewriter,
+      bool CallCannotReturn,
+      ...) {
+  if (!Func->isDeclaration()) {
+    report_fatal_error(Twine("Expected ") + FuncName +
+                       " to be declared, not defined");
+  }
+
+  // Populate the function body with code.
+  BasicBlock *BB = BasicBlock::Create(*Context, "entry", Func);
+
+  // Collect and name the function arguments.
+  Function::arg_iterator FuncArgs = Func->arg_begin();
+  SmallVector<Value *, 4> Args;
+  va_list ap;
+  va_start(ap, CallCannotReturn);
+  while (true) {
+    // Iterate over the varargs until a terminated NULL is encountered.
+    const char *ArgName = va_arg(ap, const char *);
+    if (!ArgName)
+      break;
+    Value *Arg = FuncArgs++;
+    Arg->setName(ArgName);
+    Args.push_back(Arg);
+  }
+  va_end(ap);
+
+  // Emit a call to self, and then call CallRewriter to rewrite it to the
+  // intrinsic. This is done in order to keep the call rewriting logic in a
+  // single place.
+  CallInst *SelfCall = CallInst::Create(Func, Args, "", BB);
+
+  if (CallCannotReturn) {
+    new UnreachableInst(*Context, BB);
+  } else if (Func->getReturnType()->isVoidTy()) {
+    ReturnInst::Create(*Context, BB);
+  } else {
+    ReturnInst::Create(*Context, SelfCall, BB);
+  }
+
+  (this->*(CallRewriter))(SelfCall);
+}
+
 void RewritePNaClLibraryCalls::populateLongjmpWrapper(Function *LongjmpFunc) {
-  assert(LongjmpFunc->size() == 0 &&
-      "Expected to be called when longjmp has an empty body");
+  populateWrapperCommon(
+      /* Func             */ LongjmpFunc,
+      /* FuncName         */ "longjmp",
+      /* CallRewriter     */ &RewritePNaClLibraryCalls::rewriteLongjmpCall,
+      /* CallCannotReturn */ true,
+      /* ...              */ "env", "val", NULL);
+}
 
-  // Populate longjmp with code.
-  LLVMContext &Context = TheModule->getContext();
-  BasicBlock *BB = BasicBlock::Create(Context, "entry", LongjmpFunc);
+void RewritePNaClLibraryCalls::populateMemcpyWrapper(Function *MemcpyFunc) {
+  populateWrapperCommon(
+      /* Func             */ MemcpyFunc,
+      /* FuncName         */ "memcpy",
+      /* CallRewriter     */ &RewritePNaClLibraryCalls::rewriteMemcpyCall,
+      /* CallCannotReturn */ false,
+      /* ...              */ "dest", "src", "len", NULL);
+}
 
-  Function::arg_iterator LongjmpArgs = LongjmpFunc->arg_begin();
-  Value *EnvArg = LongjmpArgs++;
-  EnvArg->setName("env");
-  Value *ValArg = LongjmpArgs++;
-  ValArg->setName("val");
+void RewritePNaClLibraryCalls::populateMemmoveWrapper(Function *MemmoveFunc) {
+  populateWrapperCommon(
+      /* Func             */ MemmoveFunc,
+      /* FuncName         */ "memmove",
+      /* CallRewriter     */ &RewritePNaClLibraryCalls::rewriteMemmoveCall,
+      /* CallCannotReturn */ false,
+      /* ...              */ "dest", "src", "len", NULL);
+}
 
-  // Find the intrinsic function.
-  Function *NaClLongjmpFunc = Intrinsic::getDeclaration(
-      TheModule, Intrinsic::nacl_longjmp);
-  // Cast the jmp_buf argument to the type NaClLongjmpCall expects.
-  Type *PtrTy = NaClLongjmpFunc->getFunctionType()->getParamType(0);
-  BitCastInst *JmpBufCast = new BitCastInst(EnvArg,  PtrTy, "jmp_buf_i8", BB);
+void RewritePNaClLibraryCalls::populateMemsetWrapper(Function *MemsetFunc) {
+  populateWrapperCommon(
+      /* Func             */ MemsetFunc,
+      /* FuncName         */ "memset",
+      /* CallRewriter     */ &RewritePNaClLibraryCalls::rewriteMemsetCall,
+      /* CallCannotReturn */ false,
+      /* ...              */ "dest", "val", "len", NULL);
+}
 
-  // Emit the call.
-  SmallVector<Value *, 2> Args;
-  Args.push_back(JmpBufCast);
-  Args.push_back(ValArg);
-  CallInst::Create(NaClLongjmpFunc, Args, "", BB);
+Function *RewritePNaClLibraryCalls::findLongjmpIntrinsic() {
+  if (!LongjmpIntrinsic) {
+    LongjmpIntrinsic = Intrinsic::getDeclaration(
+        TheModule, Intrinsic::nacl_longjmp);
+  }
+  return LongjmpIntrinsic;
+}
 
-  // Insert an unreachable instruction to terminate this function since longjmp
-  // does not return.
-  new UnreachableInst(Context, BB);
+Function *RewritePNaClLibraryCalls::findMemcpyIntrinsic() {
+  if (!MemcpyIntrinsic) {
+    Type *Tys[] = { Type::getInt8PtrTy(*Context),
+                    Type::getInt8PtrTy(*Context),
+                    Type::getInt32Ty(*Context) };
+    MemcpyIntrinsic = Intrinsic::getDeclaration(
+        TheModule, Intrinsic::memcpy, Tys);
+  }
+  return MemcpyIntrinsic;
+}
+
+Function *RewritePNaClLibraryCalls::findMemmoveIntrinsic() {
+  if (!MemmoveIntrinsic) {
+    Type *Tys[] = { Type::getInt8PtrTy(*Context),
+                    Type::getInt8PtrTy(*Context),
+                    Type::getInt32Ty(*Context) };
+    MemmoveIntrinsic = Intrinsic::getDeclaration(
+        TheModule, Intrinsic::memmove, Tys);
+  }
+  return MemmoveIntrinsic;
+}
 
-  // Finally, set the linkage to internal
-  LongjmpFunc->setLinkage(Function::InternalLinkage);
+Function *RewritePNaClLibraryCalls::findMemsetIntrinsic() {
+  if (!MemsetIntrinsic) {
+    Type *Tys[] = { Type::getInt8PtrTy(*Context), Type::getInt32Ty(*Context) };
+    MemsetIntrinsic = Intrinsic::getDeclaration(
+        TheModule, Intrinsic::memset, Tys);
+  }
+  return MemsetIntrinsic;
 }
 
 ModulePass *llvm::createRewritePNaClLibraryCallsPass() {
diff --git a/test/Transforms/NaCl/rewrite-memfuncs-no-store.ll b/test/Transforms/NaCl/rewrite-memfuncs-no-store.ll
new file mode 100644
index 0000000000..4d91774c34
--- /dev/null
+++ b/test/Transforms/NaCl/rewrite-memfuncs-no-store.ll
@@ -0,0 +1,40 @@
+; RUN: opt < %s -rewrite-pnacl-library-calls -S | FileCheck %s
+; RUN: opt < %s -rewrite-pnacl-library-calls -S | FileCheck %s -check-prefix=CLEANED
+
+declare i8* @memcpy(i8*, i8*, i32)
+declare i8* @memmove(i8*, i8*, i32)
+declare i8* @memset(i8*, i32, i32)
+
+; No declaration or definition of the library functions should remain, since
+; the only uses of mem* functions are calls.
+; CLEANED-NOT: @memcpy
+; CLEANED-NOT: @memmove
+; CLEANED-NOT: @memset
+
+define i8* @call_memcpy(i8* %dest, i8* %src, i32 %len) {
+  %result = call i8* @memcpy(i8* %dest, i8* %src, i32 %len)
+  ret i8* %result
+; CHECK: call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dest, i8* %src, i32 %len, i32 1, i1 false)
+; CHECK: ret i8* %dest
+}
+
+define i8* @call_memmove(i8* %dest, i8* %src, i32 %len) {
+  %result = call i8* @memmove(i8* %dest, i8* %src, i32 %len)
+  ret i8* %result
+; CHECK: call void @llvm.memmove.p0i8.p0i8.i32(i8* %dest, i8* %src, i32 %len, i32 1, i1 false)
+; CHECK: ret i8* %dest
+}
+
+define i8* @call_memset(i8* %dest, i32 %c, i32 %len) {
+  %result = call i8* @memset(i8* %dest, i32 %c, i32 %len)
+  ret i8* %result
+; CHECK: %trunc_byte = trunc i32 %c to i8
+; CHECK: call void @llvm.memset.p0i8.i32(i8* %dest, i8 %trunc_byte, i32 %len, i32 1, i1 false)
+; CHECK: ret i8* %dest
+}
+
+; CHECK: declare void @llvm.memset.p0i8.i32(i8* nocapture, i8, i32, i32, i1)
+
+; CHECK: declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i32, i1)
+
+; CHECK: declare void @llvm.memmove.p0i8.p0i8.i32(i8* nocapture, i8* nocapture, i32, i32, i1)
diff --git a/test/Transforms/NaCl/rewrite-memfuncs-noncall-uses.ll b/test/Transforms/NaCl/rewrite-memfuncs-noncall-uses.ll
new file mode 100644
index 0000000000..5c6bdfdcb5
--- /dev/null
+++ b/test/Transforms/NaCl/rewrite-memfuncs-noncall-uses.ll
@@ -0,0 +1,30 @@
+; RUN: opt < %s -rewrite-pnacl-library-calls -S | FileCheck %s
+; Check that the rewrite pass behaves correctly in the presence 
+; of various uses of mem* that are not calls.
+
+@fpcpy = global i8* (i8*, i8*, i32)* @memcpy
+; CHECK: @fpcpy = global i8* (i8*, i8*, i32)* @memcpy
+@fpmove = global i8* (i8*, i8*, i32)* @memmove
+; CHECK: @fpmove = global i8* (i8*, i8*, i32)* @memmove
+@fpset = global i8* (i8*, i32, i32)* @memset
+; CHECK: @fpset = global i8* (i8*, i32, i32)* @memset
+
+; CHECK: define internal i8* @memcpy(i8* %dest, i8* %src, i32 %len) {
+; CHECK:   call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dest, i8* %src, i32 %len, i32 1, i1 false)
+; CHECK:   ret i8* %dest
+; CHECK: }
+
+; CHECK: define internal i8* @memmove(i8* %dest, i8* %src, i32 %len) {
+; CHECK:   call void @llvm.memmove.p0i8.p0i8.i32(i8* %dest, i8* %src, i32 %len, i32 1, i1 false)
+; CHECK:   ret i8* %dest
+; CHECK: }
+
+; CHECK: define internal i8* @memset(i8* %dest, i32 %val, i32 %len) {
+; CHECK:   %trunc_byte = trunc i32 %val to i8
+; CHECK:   call void @llvm.memset.p0i8.i32(i8* %dest, i8 %trunc_byte, i32 %len, i32 1, i1 false)
+; CHECK:   ret i8* %dest
+; CHECK: }
+
+declare i8* @memcpy(i8*, i8*, i32)
+declare i8* @memmove(i8*, i8*, i32)
+declare i8* @memset(i8*, i32, i32)