Add the new @llvm.nacl.atomic.fence.all intrinsic

This is a follow-up to:
  https://codereview.chromium.org/22240002/
And requires the Clang changes from:
  https://codereview.chromium.org/22294002/

This new intrinsic represents ``asm("":::"~{memory}")`` as well as ``__sync_synchronize()``, and in IR it corresponds to a sequentially-consistent fence surrounded by ``call void asm sideeffect "", "~{memory}"()``.

R=jvoung@chromium.org
TEST= ninja check-all
BUG= https://code.google.com/p/nativeclient/issues/detail?id=3475

Review URL: https://codereview.chromium.org/22474008

19 files changed, 252 insertions, 150 deletions

diff --git a/include/llvm/IR/InlineAsm.h b/include/llvm/IR/InlineAsm.h
index 33e4ab8522..2c4a558200 100644
--- a/include/llvm/IR/InlineAsm.h
+++ b/include/llvm/IR/InlineAsm.h
@@ -92,6 +92,13 @@ public:
   ///
   static bool Verify(FunctionType *Ty, StringRef Constraints);
 
+  // @LOCALMOD-START
+  /// isAsmMemory - Returns true if the Instruction corresponds to
+  /// ``asm("":::"memory")``, which is often used as a compiler barrier.
+  ///
+  bool isAsmMemory() const;
+  // @LOCALMOD-END
+
   // Constraint String Parsing 
   enum ConstraintPrefix {
     isInput,            // 'x'
diff --git a/include/llvm/IR/NaClAtomicIntrinsics.h b/include/llvm/IR/NaClAtomicIntrinsics.h
index 680c644024..b87c1ad77f 100644
--- a/include/llvm/IR/NaClAtomicIntrinsics.h
+++ b/include/llvm/IR/NaClAtomicIntrinsics.h
@@ -22,7 +22,7 @@ namespace llvm {
 
 namespace NaCl {
 
-static const size_t NumAtomicIntrinsics = 5;
+static const size_t NumAtomicIntrinsics = 6;
 static const size_t NumAtomicIntrinsicOverloadTypes = 4;
 static const size_t MaxAtomicIntrinsicsParameters = 5;
 
diff --git a/include/llvm/InitializePasses.h b/include/llvm/InitializePasses.h
index 8352868d89..fd59d14b54 100644
--- a/include/llvm/InitializePasses.h
+++ b/include/llvm/InitializePasses.h
@@ -295,10 +295,10 @@ void initializePNaClABIVerifyFunctionsPass(PassRegistry&);
 void initializePNaClABIVerifyModulePass(PassRegistry&);
 void initializePromoteI1OpsPass(PassRegistry&);
 void initializePromoteIntegersPass(PassRegistry&);
+void initializeRemoveAsmMemoryPass(PassRegistry&);
 void initializeReplacePtrsWithIntsPass(PassRegistry&);
 void initializeResolveAliasesPass(PassRegistry&);
 void initializeResolvePNaClIntrinsicsPass(PassRegistry&);
-void initializeRewriteAsmDirectivesPass(PassRegistry&);
 void initializeRewriteAtomicsPass(PassRegistry&);
 void initializeRewriteLLVMIntrinsicsPass(PassRegistry&);
 void initializeRewritePNaClLibraryCallsPass(PassRegistry&);
diff --git a/include/llvm/Transforms/NaCl.h b/include/llvm/Transforms/NaCl.h
index 3cf306499e..e1be3cf85b 100644
--- a/include/llvm/Transforms/NaCl.h
+++ b/include/llvm/Transforms/NaCl.h
@@ -28,8 +28,8 @@ FunctionPass *createExpandConstantExprPass();
 FunctionPass *createExpandStructRegsPass();
 FunctionPass *createInsertDivideCheckPass();
 FunctionPass *createPromoteIntegersPass();
+FunctionPass *createRemoveAsmMemoryPass();
 FunctionPass *createResolvePNaClIntrinsicsPass();
-FunctionPass *createRewriteAsmDirectivesPass();
 ModulePass *createAddPNaClExternalDeclsPass();
 ModulePass *createCanonicalizeMemIntrinsicsPass();
 ModulePass *createExpandArithWithOverflowPass();
diff --git a/lib/Analysis/NaCl/PNaClABIVerifyFunctions.cpp b/lib/Analysis/NaCl/PNaClABIVerifyFunctions.cpp
index e55a9f0df4..94bb691243 100644
--- a/lib/Analysis/NaCl/PNaClABIVerifyFunctions.cpp
+++ b/lib/Analysis/NaCl/PNaClABIVerifyFunctions.cpp
@@ -419,7 +419,8 @@ const char *PNaClABIVerifyFunctions::checkInstruction(const Instruction *Inst) {
           case Intrinsic::nacl_atomic_store:
           case Intrinsic::nacl_atomic_rmw:
           case Intrinsic::nacl_atomic_cmpxchg:
-          case Intrinsic::nacl_atomic_fence: {
+          case Intrinsic::nacl_atomic_fence:
+          case Intrinsic::nacl_atomic_fence_all: {
             // All overloads have memory order and RMW operation in the
             // same parameter, arbitrarily use the I32 overload.
             Type *T = Type::getInt32Ty(
diff --git a/lib/Analysis/NaCl/PNaClABIVerifyModule.cpp b/lib/Analysis/NaCl/PNaClABIVerifyModule.cpp
index e781680f3f..66cc66b75f 100644
--- a/lib/Analysis/NaCl/PNaClABIVerifyModule.cpp
+++ b/lib/Analysis/NaCl/PNaClABIVerifyModule.cpp
@@ -214,6 +214,7 @@ AllowedIntrinsics::AllowedIntrinsics(LLVMContext *Context) : Context(Context) {
     addIntrinsic(Intrinsic::nacl_atomic_cmpxchg, AtomicTypes[T]);
   }
   addIntrinsic(Intrinsic::nacl_atomic_fence);
+  addIntrinsic(Intrinsic::nacl_atomic_fence_all);
 
   addIntrinsic(Intrinsic::nacl_atomic_is_lock_free);
 
diff --git a/lib/IR/InlineAsm.cpp b/lib/IR/InlineAsm.cpp
index 9f2a9fea4b..121fe1528e 100644
--- a/lib/IR/InlineAsm.cpp
+++ b/lib/IR/InlineAsm.cpp
@@ -293,3 +293,18 @@ bool InlineAsm::Verify(FunctionType *Ty, StringRef ConstStr) {
   return true;
 }
 
+// @LOCALMOD-START
+bool InlineAsm::isAsmMemory() const {
+  bool retVoid = getFunctionType()->getReturnType()->isVoidTy();
+  bool noArgs = getFunctionType()->getNumParams() == 0 &&
+      !getFunctionType()->isVarArg();
+  bool isEmptyAsm = AsmString.empty();
+  // Different triples will encode "touch everything" differently, e.g.:
+  //  - le32-unknown-nacl has "~{memory}".
+  //  - x86 "~{memory},~{dirflag},~{fpsr},~{flags}".
+  // The following code therefore only searches for memory.
+  bool touchesMemory = Constraints.find("~{memory}") != std::string::npos;
+
+  return retVoid && noArgs && hasSideEffects() && isEmptyAsm && touchesMemory;
+}
+// @LOCALMOD-END
diff --git a/lib/IR/NaClAtomicIntrinsics.cpp b/lib/IR/NaClAtomicIntrinsics.cpp
index 5f463380c4..02fe295fad 100644
--- a/lib/IR/NaClAtomicIntrinsics.cpp
+++ b/lib/IR/NaClAtomicIntrinsics.cpp
@@ -55,6 +55,7 @@ AtomicIntrinsics::AtomicIntrinsics(LLVMContext &C) {
   INIT(RMW, Ptr, Int, Mem, NoP, rmw);
   INIT(Ptr, Int, Int, Mem, Mem, cmpxchg);
   INIT(Mem, NoP, NoP, NoP, NoP, fence);
+  INIT(NoP, NoP, NoP, NoP, NoP, fence_all);
 }
 
 AtomicIntrinsics::View AtomicIntrinsics::allIntrinsicsAndOverloads() const {
diff --git a/lib/Transforms/NaCl/CMakeLists.txt b/lib/Transforms/NaCl/CMakeLists.txt
index 909cd8c68d..014d322d55 100644
--- a/lib/Transforms/NaCl/CMakeLists.txt
+++ b/lib/Transforms/NaCl/CMakeLists.txt
@@ -18,9 +18,9 @@ add_llvm_library(LLVMNaClTransforms
   PNaClABISimplify.cpp
   PromoteI1Ops.cpp
   PromoteIntegers.cpp
+  RemoveAsmMemory.cpp
   ReplacePtrsWithInts.cpp
   ResolvePNaClIntrinsics.cpp
-  RewriteAsmDirectives.cpp
   RewriteAtomics.cpp
   RewriteLLVMIntrinsics.cpp
   RewritePNaClLibraryCalls.cpp
diff --git a/lib/Transforms/NaCl/PNaClABISimplify.cpp b/lib/Transforms/NaCl/PNaClABISimplify.cpp
index 4f8e0e4f87..f662ada99a 100644
--- a/lib/Transforms/NaCl/PNaClABISimplify.cpp
+++ b/lib/Transforms/NaCl/PNaClABISimplify.cpp
@@ -30,10 +30,8 @@ void llvm::PNaClABISimplifyAddPreOptPasses(PassManager &PM) {
   // LowerExpect converts Intrinsic::expect into branch weights,
   // which can then be removed after BlockPlacement.
   PM.add(createLowerExpectIntrinsicPass());
-  // Rewrite unsupported intrinsics and inline assembly directives to
-  // simpler and portable constructs.
+  // Rewrite unsupported intrinsics to simpler and portable constructs.
   PM.add(createRewriteLLVMIntrinsicsPass());
-  PM.add(createRewriteAsmDirectivesPass());
   // LowerInvoke prevents use of C++ exception handling, which is not
   // yet supported in the PNaCl ABI.
   PM.add(createLowerInvokePass());
@@ -98,6 +96,10 @@ void llvm::PNaClABISimplifyAddPostOptPasses(PassManager &PM) {
   PM.add(createExpandGetElementPtrPass());
   // Rewrite atomic and volatile instructions with intrinsic calls.
   PM.add(createRewriteAtomicsPass());
+  // Remove ``asm("":::"memory")``. This must occur after rewriting
+  // atomics: a ``fence seq_cst`` surrounded by ``asm("":::"memory")``
+  // has special meaning and is translated differently.
+  PM.add(createRemoveAsmMemoryPass());
   // ReplacePtrsWithInts assumes that getelementptr instructions and
   // ConstantExprs have already been expanded out.
   PM.add(createReplacePtrsWithIntsPass());
diff --git a/lib/Transforms/NaCl/RemoveAsmMemory.cpp b/lib/Transforms/NaCl/RemoveAsmMemory.cpp
new file mode 100644
index 0000000000..5295fe5ff8
--- /dev/null
+++ b/lib/Transforms/NaCl/RemoveAsmMemory.cpp
@@ -0,0 +1,84 @@
+//===- RemoveAsmMemory.cpp - Remove ``asm("":::"memory")`` ----------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass removes all instances of ``asm("":::"memory")``.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/Twine.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/InlineAsm.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Module.h"
+#include "llvm/InstVisitor.h"
+#include "llvm/Pass.h"
+#include <string>
+
+using namespace llvm;
+
+namespace {
+class RemoveAsmMemory : public FunctionPass {
+public:
+  static char ID; // Pass identification, replacement for typeid
+  RemoveAsmMemory() : FunctionPass(ID) {
+    initializeRemoveAsmMemoryPass(*PassRegistry::getPassRegistry());
+  }
+
+  virtual bool runOnFunction(Function &F);
+};
+
+class AsmDirectivesVisitor : public InstVisitor<AsmDirectivesVisitor> {
+public:
+  AsmDirectivesVisitor(Function &F)
+      : F(F), C(F.getParent()->getContext()), ModifiedFunction(false) {}
+  ~AsmDirectivesVisitor() {}
+  bool modifiedFunction() const { return ModifiedFunction; }
+
+  /// Only Call Instructions are ever inline assembly directives.
+  void visitCallInst(CallInst &CI);
+
+private:
+  Function &F;
+  LLVMContext &C;
+  bool ModifiedFunction;
+
+  AsmDirectivesVisitor() LLVM_DELETED_FUNCTION;
+  AsmDirectivesVisitor(const AsmDirectivesVisitor &) LLVM_DELETED_FUNCTION;
+  AsmDirectivesVisitor &operator=(
+      const AsmDirectivesVisitor &) LLVM_DELETED_FUNCTION;
+};
+}
+
+char RemoveAsmMemory::ID = 0;
+INITIALIZE_PASS(RemoveAsmMemory, "remove-asm-memory",
+                "remove all instances of ``asm(\"\":::\"memory\")``", false,
+                false)
+
+bool RemoveAsmMemory::runOnFunction(Function &F) {
+  AsmDirectivesVisitor AV(F);
+  AV.visit(F);
+  return AV.modifiedFunction();
+}
+
+void AsmDirectivesVisitor::visitCallInst(CallInst &CI) {
+  if (!CI.isInlineAsm() ||
+      !cast<InlineAsm>(CI.getCalledValue())->isAsmMemory())
+    return;
+
+  // In NaCl ``asm("":::"memory")`` always comes in pairs, straddling a
+  // sequentially consistent fence. Other passes rewrite this fence to
+  // an equivalent stable NaCl intrinsic, meaning that this assembly can
+  // be removed.
+  CI.eraseFromParent();
+  ModifiedFunction = true;
+}
+
+namespace llvm {
+FunctionPass *createRemoveAsmMemoryPass() { return new RemoveAsmMemory(); }
+}
diff --git a/lib/Transforms/NaCl/ResolvePNaClIntrinsics.cpp b/lib/Transforms/NaCl/ResolvePNaClIntrinsics.cpp
index 25e1a8620b..fc5138574d 100644
--- a/lib/Transforms/NaCl/ResolvePNaClIntrinsics.cpp
+++ b/lib/Transforms/NaCl/ResolvePNaClIntrinsics.cpp
@@ -18,6 +18,8 @@
 
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/IR/Constant.h"
+#include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/InlineAsm.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Intrinsics.h"
@@ -242,6 +244,20 @@ private:
       I = new FenceInst(M->getContext(),
                         thawMemoryOrder(Call->getArgOperand(0)), SS, Call);
       break;
+    case Intrinsic::nacl_atomic_fence_all: {
+      FunctionType *FTy =
+          FunctionType::get(Type::getVoidTy(M->getContext()), false);
+      std::string AsmString; // Empty.
+      std::string Constraints("~{memory}");
+      bool HasSideEffect = true;
+      CallInst *Asm = CallInst::Create(
+          InlineAsm::get(FTy, AsmString, Constraints, HasSideEffect), "", Call);
+      Asm->setDebugLoc(Call->getDebugLoc());
+      I = new FenceInst(M->getContext(), SequentiallyConsistent, SS, Asm);
+      Asm = CallInst::Create(
+          InlineAsm::get(FTy, AsmString, Constraints, HasSideEffect), "", I);
+      Asm->setDebugLoc(Call->getDebugLoc());
+    } break;
     }
     I->setName(Call->getName());
     I->setDebugLoc(Call->getDebugLoc());
diff --git a/lib/Transforms/NaCl/RewriteAsmDirectives.cpp b/lib/Transforms/NaCl/RewriteAsmDirectives.cpp
deleted file mode 100644
index cb726d2365..0000000000
--- a/lib/Transforms/NaCl/RewriteAsmDirectives.cpp
+++ /dev/null
@@ -1,111 +0,0 @@
-//===- RewriteAsmDirectives.cpp - Handle Architecture-Independent Assembly-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This pass rewrites any inline assembly directive which is portable
-// into LLVM bitcode.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/ADT/Twine.h"
-#include "llvm/IR/Function.h"
-#include "llvm/IR/InlineAsm.h"
-#include "llvm/IR/Instructions.h"
-#include "llvm/IR/Module.h"
-#include "llvm/InstVisitor.h"
-#include "llvm/Pass.h"
-#include <string>
-
-using namespace llvm;
-
-namespace {
-class RewriteAsmDirectives : public FunctionPass {
-public:
-  static char ID; // Pass identification, replacement for typeid
-  RewriteAsmDirectives() : FunctionPass(ID) {
-    initializeRewriteAsmDirectivesPass(*PassRegistry::getPassRegistry());
-  }
-
-  virtual bool runOnFunction(Function &F);
-};
-
-class AsmDirectivesVisitor : public InstVisitor<AsmDirectivesVisitor> {
-public:
-  AsmDirectivesVisitor(Function &F)
-      : F(F), C(F.getParent()->getContext()), ModifiedFunction(false) {}
-  ~AsmDirectivesVisitor() {}
-  bool modifiedFunction() const { return ModifiedFunction; }
-
-  /// Only Call Instructions are ever inline assembly directives.
-  void visitCallInst(CallInst &CI);
-
-private:
-  Function &F;
-  LLVMContext &C;
-  bool ModifiedFunction;
-
-  AsmDirectivesVisitor() LLVM_DELETED_FUNCTION;
-  AsmDirectivesVisitor(const AsmDirectivesVisitor &) LLVM_DELETED_FUNCTION;
-  AsmDirectivesVisitor &operator=(const AsmDirectivesVisitor &) LLVM_DELETED_FUNCTION;
-};
-}
-
-char RewriteAsmDirectives::ID = 0;
-INITIALIZE_PASS(
-    RewriteAsmDirectives, "rewrite-asm-directives",
-    "rewrite portable inline assembly directives into non-asm LLVM IR",
-    false, false)
-
-bool RewriteAsmDirectives::runOnFunction(Function &F) {
-  AsmDirectivesVisitor AV(F);
-  AV.visit(F);
-  return AV.modifiedFunction();
-}
-
-void AsmDirectivesVisitor::visitCallInst(CallInst &CI) {
-  if (!CI.isInlineAsm())
-    return;
-
-  Instruction *Replacement = NULL;
-
-  InlineAsm *IA = cast<InlineAsm>(CI.getCalledValue());
-  std::string AsmStr(IA->getAsmString());
-  std::string ConstraintStr(IA->getConstraintString());
-  Type *T = CI.getType();
-
-  bool isEmptyAsm = AsmStr.empty();
-  // Different triples will encode "touch everything" differently, e.g.:
-  //  - le32-unknown-nacl has "~{memory}".
-  //  - x86 "~{memory},~{dirflag},~{fpsr},~{flags}".
-  // The following code therefore only searches for memory: this pass
-  // deals with portable assembly, touching anything else than memory in
-  // an empty assembly statement is meaningless.
-  bool touchesMemory = ConstraintStr.find("~{memory}") != std::string::npos;
-
-  if (T->isVoidTy() && IA->hasSideEffects() && isEmptyAsm && touchesMemory) {
-    // asm("":::"memory") => fence seq_cst
-    // This transformation is safe and strictly stronger: the former is
-    // purely a compiler fence, whereas the latter is a compiler fence
-    // as well as a hardware fence which orders all loads and stores on
-    // the current thread of execution.
-    Replacement = new FenceInst(C, SequentiallyConsistent, CrossThread, &CI);
-  }
-
-  if (Replacement) {
-    Replacement->setDebugLoc(CI.getDebugLoc());
-    CI.replaceAllUsesWith(Replacement);
-    CI.eraseFromParent();
-    ModifiedFunction = true;
-  }
-}
-
-namespace llvm {
-FunctionPass *createRewriteAsmDirectivesPass() {
-  return new RewriteAsmDirectives();
-}
-}
diff --git a/lib/Transforms/NaCl/RewriteAtomics.cpp b/lib/Transforms/NaCl/RewriteAtomics.cpp
index 26c834a874..e22e9fdac9 100644
--- a/lib/Transforms/NaCl/RewriteAtomics.cpp
+++ b/lib/Transforms/NaCl/RewriteAtomics.cpp
@@ -18,6 +18,7 @@
 #include "llvm/ADT/Twine.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/Function.h"
+#include "llvm/IR/InlineAsm.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/Module.h"
@@ -241,9 +242,9 @@ void AtomicVisitor::replaceInstructionWithIntrinsicCall(
   ModifiedModule = true;
 }
 
-/// %res = load {atomic|volatile} T* %ptr memory_order, align sizeof(T)
+///   %res = load {atomic|volatile} T* %ptr memory_order, align sizeof(T)
 /// becomes:
-/// %res = call T @llvm.nacl.atomic.load.i<size>(%ptr, memory_order)
+///   %res = call T @llvm.nacl.atomic.load.i<size>(%ptr, memory_order)
 void AtomicVisitor::visitLoadInst(LoadInst &I) {
   if (I.isSimple())
     return;
@@ -254,9 +255,9 @@ void AtomicVisitor::visitLoadInst(LoadInst &I) {
                                       PH.OriginalPET, PH.PET, Args);
 }
 
-/// store {atomic|volatile} T %val, T* %ptr memory_order, align sizeof(T)
+///   store {atomic|volatile} T %val, T* %ptr memory_order, align sizeof(T)
 /// becomes:
-/// call void @llvm.nacl.atomic.store.i<size>(%val, %ptr, memory_order)
+///   call void @llvm.nacl.atomic.store.i<size>(%val, %ptr, memory_order)
 void AtomicVisitor::visitStoreInst(StoreInst &I) {
   if (I.isSimple())
     return;
@@ -278,9 +279,9 @@ void AtomicVisitor::visitStoreInst(StoreInst &I) {
                                       PH.OriginalPET, PH.PET, Args);
 }
 
-/// %res = atomicrmw OP T* %ptr, T %val memory_order
+///   %res = atomicrmw OP T* %ptr, T %val memory_order
 /// becomes:
-/// %res = call T @llvm.nacl.atomic.rmw.i<size>(OP, %ptr, %val, memory_order)
+///   %res = call T @llvm.nacl.atomic.rmw.i<size>(OP, %ptr, %val, memory_order)
 void AtomicVisitor::visitAtomicRMWInst(AtomicRMWInst &I) {
   NaCl::AtomicRMWOperation Op;
   switch (I.getOperation()) {
@@ -300,11 +301,11 @@ void AtomicVisitor::visitAtomicRMWInst(AtomicRMWInst &I) {
                                       PH.OriginalPET, PH.PET, Args);
 }
 
-/// %res = cmpxchg T* %ptr, T %old, T %new memory_order
+///   %res = cmpxchg T* %ptr, T %old, T %new memory_order
 /// becomes:
-/// %res = call T @llvm.nacl.atomic.cmpxchg.i<size>(
-///     %object, %expected, %desired, memory_order_success,
-///     memory_order_failure)
+///   %res = call T @llvm.nacl.atomic.cmpxchg.i<size>(
+///       %object, %expected, %desired, memory_order_success,
+///       memory_order_failure)
 void AtomicVisitor::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
   PointerHelper<AtomicCmpXchgInst> PH(*this, I);
   checkSizeMatchesType(I, PH.BitSize, I.getCompareOperand()->getType());
@@ -319,14 +320,38 @@ void AtomicVisitor::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
                                       PH.OriginalPET, PH.PET, Args);
 }
 
-/// fence memory_order
+///   fence memory_order
 /// becomes:
-/// call void @llvm.nacl.atomic.fence(memory_order)
+///   call void @llvm.nacl.atomic.fence(memory_order)
+/// and
+///   call void asm sideeffect "", "~{memory}"()
+///   fence seq_cst
+///   call void asm sideeffect "", "~{memory}"()
+/// becomes:
+///   call void asm sideeffect "", "~{memory}"()
+///   call void @llvm.nacl.atomic.fence.all()
+///   call void asm sideeffect "", "~{memory}"()
+/// Note that the assembly gets eliminated by the -remove-asm-memory pass.
 void AtomicVisitor::visitFenceInst(FenceInst &I) {
   Type *T = Type::getInt32Ty(C); // Fences aren't overloaded on type.
-  Value *Args[] = { freezeMemoryOrder(I) };
-  replaceInstructionWithIntrinsicCall(I, Intrinsic::nacl_atomic_fence, T, T,
-                                      Args);
+  BasicBlock::InstListType &IL(I.getParent()->getInstList());
+  bool isFirst = IL.empty() || &*I.getParent()->getInstList().begin() == &I;
+  bool isLast = IL.empty() || &*I.getParent()->getInstList().rbegin() == &I;
+  CallInst *PrevC = isFirst ? 0 : dyn_cast<CallInst>(I.getPrevNode());
+  CallInst *NextC = isLast ? 0 : dyn_cast<CallInst>(I.getNextNode());
+
+  if ((PrevC && PrevC->isInlineAsm() &&
+       cast<InlineAsm>(PrevC->getCalledValue())->isAsmMemory()) &&
+      (NextC && NextC->isInlineAsm() &&
+       cast<InlineAsm>(NextC->getCalledValue())->isAsmMemory()) &&
+      I.getOrdering() == SequentiallyConsistent) {
+    replaceInstructionWithIntrinsicCall(I, Intrinsic::nacl_atomic_fence_all, T,
+                                        T, ArrayRef<Value *>());
+  } else {
+    Value *Args[] = { freezeMemoryOrder(I) };
+    replaceInstructionWithIntrinsicCall(I, Intrinsic::nacl_atomic_fence, T, T,
+                                        Args);
+  }
 }
 
 ModulePass *llvm::createRewriteAtomicsPass() { return new RewriteAtomics(); }
diff --git a/test/NaCl/PNaClABI/intrinsics.ll b/test/NaCl/PNaClABI/intrinsics.ll
index a210491823..736de63c59 100644
--- a/test/NaCl/PNaClABI/intrinsics.ll
+++ b/test/NaCl/PNaClABI/intrinsics.ll
@@ -54,6 +54,7 @@ declare i16 @llvm.nacl.atomic.cmpxchg.i16(i16*, i16, i16, i32, i32)
 declare i32 @llvm.nacl.atomic.cmpxchg.i32(i32*, i32, i32, i32, i32)
 declare i64 @llvm.nacl.atomic.cmpxchg.i64(i64*, i64, i64, i32, i32)
 declare void @llvm.nacl.atomic.fence(i32)
+declare void @llvm.nacl.atomic.fence.all()
 declare i1 @llvm.nacl.atomic.is.lock.free(i32, i8*)
 
 declare i16 @llvm.bswap.i16(i16)
diff --git a/test/Transforms/NaCl/atomics.ll b/test/Transforms/NaCl/atomics.ll
index b124241062..96b2b7d6e1 100644
--- a/test/Transforms/NaCl/atomics.ll
+++ b/test/Transforms/NaCl/atomics.ll
@@ -1,4 +1,4 @@
-; RUN: opt -nacl-rewrite-atomics -S < %s | FileCheck %s
+; RUN: opt -nacl-rewrite-atomics -remove-asm-memory -S < %s | FileCheck %s
 
 ; Each of these tests validates that the corresponding legacy GCC-style
 ; builtins are properly rewritten to NaCl atomic builtins. Only the
@@ -210,11 +210,44 @@ define i64 @test_val_compare_and_swap_i64(i64* %ptr, i64 %oldval, i64 %newval) {
   ret i64 %res
 }
 
+; This patterns gets emitted by C11/C++11 atomic thread fences.
+;
+; CHECK: @test_c11_fence
+define void @test_c11_fence() {
+  ; CHECK-NEXT: call void @llvm.nacl.atomic.fence(i32 6)
+  ; CHECK-NEXT: ret void
+  fence seq_cst
+  ret void
+}
+
+; This pattern gets emitted for ``__sync_synchronize`` and
+; ``asm("":::"memory")`` when Clang is configured for NaCl.
+;
 ; CHECK: @test_synchronize
 define void @test_synchronize() {
-  ; CHECK-NEXT: call void @llvm.nacl.atomic.fence(i32 6)
+  ; CHECK-NEXT: call void @llvm.nacl.atomic.fence.all()
   ; CHECK-NEXT: ret void
+  call void asm sideeffect "", "~{memory}"()
+  fence seq_cst
+  call void asm sideeffect "", "~{memory}"()
+  ret void
+}
+
+; Make sure the above pattern is respected and not partially-matched.
+;
+; CHECK: @test_synchronize_bad1
+define void @test_synchronize_bad1() {
+  ; CHECK-NOT: call void @llvm.nacl.atomic.fence.all()
+  call void asm sideeffect "", "~{memory}"()
+  fence seq_cst
+  ret void
+}
+
+; CHECK: @test_synchronize_bad2
+define void @test_synchronize_bad2() {
+  ; CHECK-NOT: call void @llvm.nacl.atomic.fence.all()
   fence seq_cst
+  call void asm sideeffect "", "~{memory}"()
   ret void
 }
 
diff --git a/test/Transforms/NaCl/rewrite-asm-memory.ll b/test/Transforms/NaCl/remove-asm-memory.ll
index 8481c831ee..ae799a7459 100644
--- a/test/Transforms/NaCl/rewrite-asm-memory.ll
+++ b/test/Transforms/NaCl/remove-asm-memory.ll
@@ -1,24 +1,37 @@
-; RUN: opt < %s -rewrite-asm-directives -S | FileCheck %s
-; RUN: opt < %s -O3 -rewrite-asm-directives -S | FileCheck %s
-; RUN: opt < %s -O3 -rewrite-asm-directives -S | FileCheck %s -check-prefix=ELIM
-; RUN: opt < %s -rewrite-asm-directives -S | FileCheck %s -check-prefix=CLEANED
+; RUN: opt < %s -nacl-rewrite-atomics -remove-asm-memory -S | \
+; RUN:       FileCheck %s
+; RUN: opt < %s -O3 -nacl-rewrite-atomics -remove-asm-memory -S | \
+; RUN:       FileCheck %s
+; RUN: opt < %s -O3 -nacl-rewrite-atomics -remove-asm-memory -S | \
+; RUN:       FileCheck %s -check-prefix=ELIM
+; RUN: opt < %s -nacl-rewrite-atomics -remove-asm-memory -S | \
+; RUN:       FileCheck %s -check-prefix=CLEANED
 
-; Test that asm("":::"memory"), a compiler barrier, gets rewritten to a
-; sequentially-consistent fence. The test is also run at O3 to make sure
-; that loads and stores don't get unexpectedly eliminated.
+; ``asm("":::"memory")`` is used as a compiler barrier and the GCC-style
+; builtin ``__sync_synchronize`` is intended as a barrier for all memory
+; that could be observed by external threads. They both get rewritten
+; for NaCl by Clang to a sequentially-consistent fence surrounded by
+; ``call void asm sideeffect "", "~{memory}"``.
+;
+; The test is also run at O3 to make sure that non-volatile and
+; non-atomic loads and stores to escaping objects (i.e. loads and stores
+; which could be observed by other threads) don't get unexpectedly
+; eliminated.
 
 ; CLEANED-NOT: asm
 
+target datalayout = "p:32:32:32"
+
 @a = external global i32
 @b = external global i32
 
-; Different triples encode "touch everything" constraints differently.
+; Different triples encode ``asm("":::"memory")``'s "touch everything"
+; constraints differently.  They should get detected and removed.
 define void @memory_assembly_encoding_test() {
 ; CHECK: @memory_assembly_encoding_test()
   call void asm sideeffect "", "~{memory}"()
   call void asm sideeffect "", "~{memory},~{dirflag},~{fpsr},~{flags}"()
-  ; CHECK-NEXT: fence seq_cst
-  ; CHECK-NEXT: fence seq_cst
+  call void asm sideeffect "", "~{foo},~{memory},~{bar}"()
 
   ret void
   ; CHECK-NEXT: ret void
@@ -29,9 +42,11 @@ define void @memory_assembly_ordering_test() {
   %1 = load i32* @a, align 4
   store i32 %1, i32* @b, align 4
   call void asm sideeffect "", "~{memory}"()
+  fence seq_cst
+  call void asm sideeffect "", "~{memory}"()
   ; CHECK-NEXT: %1 = load i32* @a, align 4
   ; CHECK-NEXT: store i32 %1, i32* @b, align 4
-  ; CHECK-NEXT: fence seq_cst
+  ; CHECK-NEXT: call void @llvm.nacl.atomic.fence.all()
 
   ; Redundant load from the previous location, and store to the same
   ; location (making the previous one dead). Shouldn't get eliminated
@@ -39,9 +54,11 @@ define void @memory_assembly_ordering_test() {
   %2 = load i32* @a, align 4
   store i32 %2, i32* @b, align 4
   call void asm sideeffect "", "~{memory}"()
+  fence seq_cst
+  call void asm sideeffect "", "~{memory}"()
   ; CHECK-NEXT: %2 = load i32* @a, align 4
   ; CHECK-NEXT: store i32 %2, i32* @b, align 4
-  ; CHECK-NEXT: fence seq_cst
+  ; CHECK-NEXT: call void @llvm.nacl.atomic.fence.all()
 
   ; Same here.
   %3 = load i32* @a, align 4
diff --git a/test/Transforms/NaCl/resolve-pnacl-intrinsics.ll b/test/Transforms/NaCl/resolve-pnacl-intrinsics.ll
index 737561ee1d..0a297057fe 100644
--- a/test/Transforms/NaCl/resolve-pnacl-intrinsics.ll
+++ b/test/Transforms/NaCl/resolve-pnacl-intrinsics.ll
@@ -29,6 +29,7 @@ declare i16 @llvm.nacl.atomic.cmpxchg.i16(i16*, i16, i16, i32, i32)
 declare i32 @llvm.nacl.atomic.cmpxchg.i32(i32*, i32, i32, i32, i32)
 declare i64 @llvm.nacl.atomic.cmpxchg.i64(i64*, i64, i64, i32, i32)
 declare void @llvm.nacl.atomic.fence(i32)
+declare void @llvm.nacl.atomic.fence.all()
 declare i1 @llvm.nacl.atomic.is.lock.free(i32, i8*)
 
 ; These declarations must be here because the function pass expects
@@ -93,10 +94,19 @@ define i32 @test_val_compare_and_swap_i32(i32* %ptr, i32 %oldval, i32 %newval) {
   ret i32 %1
 }
 
+; CHECK: @test_c11_fence
+define void @test_c11_fence() {
+  ; CHECK: fence seq_cst
+  call void @llvm.nacl.atomic.fence(i32 6)
+  ret void
+}
+
 ; CHECK: @test_synchronize
 define void @test_synchronize() {
+  ; CHECK: call void asm sideeffect "", "~{memory}"()
   ; CHECK: fence seq_cst
-  call void @llvm.nacl.atomic.fence(i32 6)
+  ; CHECK: call void asm sideeffect "", "~{memory}"()
+  call void @llvm.nacl.atomic.fence.all()
   ret void
 }
 
diff --git a/tools/opt/opt.cpp b/tools/opt/opt.cpp
index 5682a55909..6386b6b9e3 100644
--- a/tools/opt/opt.cpp
+++ b/tools/opt/opt.cpp
@@ -623,10 +623,10 @@ int main(int argc, char **argv) {
   initializePNaClABIVerifyModulePass(Registry);
   initializePromoteI1OpsPass(Registry);
   initializePromoteIntegersPass(Registry);
+  initializeRemoveAsmMemoryPass(Registry);
   initializeReplacePtrsWithIntsPass(Registry);
   initializeResolveAliasesPass(Registry);
   initializeResolvePNaClIntrinsicsPass(Registry);
-  initializeRewriteAsmDirectivesPass(Registry);
   initializeRewriteAtomicsPass(Registry);
   initializeRewriteLLVMIntrinsicsPass(Registry);
   initializeRewritePNaClLibraryCallsPass(Registry);