PNaCl ABI checker: Disallow va_arg, varargs functions and varargs calls

This doesn't disallow the va_start/va_end/va_copy intrinsics yet;
these will get disallowed later when we add whitelisting of intrinsics
to the PNaCl ABI checker.

BUG=https://code.google.com/p/nativeclient/issues/detail?id=3338
TEST=test/NaCl/PNaClABI/*.ll

Review URL: https://codereview.chromium.org/13884013

2 files changed, 17 insertions, 1 deletions

diff --git a/lib/Analysis/NaCl/PNaClABIVerifyFunctions.cpp b/lib/Analysis/NaCl/PNaClABIVerifyFunctions.cpp
index 4c4c935ec5..54e2e19340 100644
--- a/lib/Analysis/NaCl/PNaClABIVerifyFunctions.cpp
+++ b/lib/Analysis/NaCl/PNaClABIVerifyFunctions.cpp
@@ -67,6 +67,8 @@ bool PNaClABIVerifyFunctions::runOnFunction(Function &F) {
         default:
         // We expand GetElementPtr out into arithmetic.
         case Instruction::GetElementPtr:
+        // VAArg is expanded out by ExpandVarArgs.
+        case Instruction::VAArg:
         // Zero-cost C++ exception handling is not supported yet.
         case Instruction::Invoke:
         case Instruction::LandingPad:
@@ -134,8 +136,16 @@ bool PNaClABIVerifyFunctions::runOnFunction(Function &F) {
         case Instruction::FCmp:
         case Instruction::PHI:
         case Instruction::Select:
+          break;
         case Instruction::Call:
-        case Instruction::VAArg:
+          // Pointers to varargs function types are not yet
+          // disallowed, but we do disallow defining or calling
+          // functions of varargs types.
+          if (cast<CallInst>(BBI)->getCalledValue()->getType()
+              ->getPointerElementType()->isFunctionVarArg()) {
+            Reporter->addError() << "Function " << F.getName() <<
+                " contains a disallowed varargs function call\n";
+          }
           break;
       }
       // Check the types. First check the type of the instruction.
diff --git a/lib/Analysis/NaCl/PNaClABIVerifyModule.cpp b/lib/Analysis/NaCl/PNaClABIVerifyModule.cpp
index d98868f53c..857f6a24af 100644
--- a/lib/Analysis/NaCl/PNaClABIVerifyModule.cpp
+++ b/lib/Analysis/NaCl/PNaClABIVerifyModule.cpp
@@ -145,6 +145,12 @@ bool PNaClABIVerifyModule::runOnModule(Module &M) {
             PNaClABITypeChecker::getTypeName(PT) << "\n";
       }
     }
+    // Pointers to varargs function types are not yet disallowed, but
+    // we do disallow defining or calling functions of varargs types.
+    if (MI->isVarArg()) {
+      Reporter->addError() << "Function " << MI->getName() <<
+          " is a variable-argument function (disallowed)\n";
+    }
 
     if (MI->hasSection()) {
       Reporter->addError() << "Function " << MI->getName() <<