PNaCl ABI checker: Disallow va_arg, varargs functions and varargs calls

This doesn't disallow the va_start/va_end/va_copy intrinsics yet;
these will get disallowed later when we add whitelisting of intrinsics
to the PNaCl ABI checker.

BUG=https://code.google.com/p/nativeclient/issues/detail?id=3338
TEST=test/NaCl/PNaClABI/*.ll

Review URL: https://codereview.chromium.org/13884013

4 files changed, 37 insertions, 2 deletions

diff --git a/lib/Analysis/NaCl/PNaClABIVerifyFunctions.cpp b/lib/Analysis/NaCl/PNaClABIVerifyFunctions.cpp
index 4c4c935ec5..54e2e19340 100644
--- a/lib/Analysis/NaCl/PNaClABIVerifyFunctions.cpp
+++ b/lib/Analysis/NaCl/PNaClABIVerifyFunctions.cpp
@@ -67,6 +67,8 @@ bool PNaClABIVerifyFunctions::runOnFunction(Function &F) {
         default:
         // We expand GetElementPtr out into arithmetic.
         case Instruction::GetElementPtr:
+        // VAArg is expanded out by ExpandVarArgs.
+        case Instruction::VAArg:
         // Zero-cost C++ exception handling is not supported yet.
         case Instruction::Invoke:
         case Instruction::LandingPad:
@@ -134,8 +136,16 @@ bool PNaClABIVerifyFunctions::runOnFunction(Function &F) {
         case Instruction::FCmp:
         case Instruction::PHI:
         case Instruction::Select:
+          break;
         case Instruction::Call:
-        case Instruction::VAArg:
+          // Pointers to varargs function types are not yet
+          // disallowed, but we do disallow defining or calling
+          // functions of varargs types.
+          if (cast<CallInst>(BBI)->getCalledValue()->getType()
+              ->getPointerElementType()->isFunctionVarArg()) {
+            Reporter->addError() << "Function " << F.getName() <<
+                " contains a disallowed varargs function call\n";
+          }
           break;
       }
       // Check the types. First check the type of the instruction.
diff --git a/lib/Analysis/NaCl/PNaClABIVerifyModule.cpp b/lib/Analysis/NaCl/PNaClABIVerifyModule.cpp
index d98868f53c..857f6a24af 100644
--- a/lib/Analysis/NaCl/PNaClABIVerifyModule.cpp
+++ b/lib/Analysis/NaCl/PNaClABIVerifyModule.cpp
@@ -145,6 +145,12 @@ bool PNaClABIVerifyModule::runOnModule(Module &M) {
             PNaClABITypeChecker::getTypeName(PT) << "\n";
       }
     }
+    // Pointers to varargs function types are not yet disallowed, but
+    // we do disallow defining or calling functions of varargs types.
+    if (MI->isVarArg()) {
+      Reporter->addError() << "Function " << MI->getName() <<
+          " is a variable-argument function (disallowed)\n";
+    }
 
     if (MI->hasSection()) {
       Reporter->addError() << "Function " << MI->getName() <<
diff --git a/test/NaCl/PNaClABI/abi-varargs.ll b/test/NaCl/PNaClABI/abi-varargs.ll
new file mode 100644
index 0000000000..2a299f5493
--- /dev/null
+++ b/test/NaCl/PNaClABI/abi-varargs.ll
@@ -0,0 +1,13 @@
+; RUN: pnacl-abicheck < %s | FileCheck %s
+
+define void @varargs_func(i32 %arg, ...) {
+  ret void
+}
+; CHECK: Function varargs_func is a variable-argument function (disallowed)
+
+define void @call_varargs_func(i32 %ptr) {
+  %ptr2 = inttoptr i32 %ptr to void (i32, ...)*
+  call void (i32, ...)* %ptr2(i32 123)
+  ret void
+}
+; CHECK: Function call_varargs_func contains a disallowed varargs function call
diff --git a/test/NaCl/PNaClABI/instructions.ll b/test/NaCl/PNaClABI/instructions.ll
index e00e7f89f6..968b47f6fc 100644
--- a/test/NaCl/PNaClABI/instructions.ll
+++ b/test/NaCl/PNaClABI/instructions.ll
@@ -103,7 +103,6 @@ foo:
 ; phi predecessor labels have to match to appease module verifier
   %a3 = phi i32 [0, %entry], [0, %foo]
   %a4 = select i1 true, i8 undef, i8 undef
-  %a5 = va_arg i8** undef, i32
   call void @conversion()
   br i1 undef, label %foo, label %bar
 bar:
@@ -127,5 +126,12 @@ onerror:
   ret void
 }
 
+define i32 @va_arg(i8* %va_list) {
+  %val = va_arg i8* %va_list, i32
+  ret i32 %val
+}
+; CHECK-NOT: disallowed
+; CHECK: Function va_arg has disallowed instruction: va_arg
+
 ; CHECK-NOT: disallowed
 ; If another check is added, there should be a check-not in between each check