Fix an x86-64 fast-isel address calculation overflow.

Disallow large displacements in NaCl addressing modes, by
rejecting the fast-isel selection and using the logic in the
DAG-based isel.

BUG= https://code.google.com/p/nativeclient/issues/detail?id=3502
R=dschuff@chromium.org

Review URL: https://codereview.chromium.org/16959013

1 files changed, 12 insertions, 0 deletions

diff --git a/lib/Target/X86/X86FastISel.cpp b/lib/Target/X86/X86FastISel.cpp
index df3b87942a..a4d6df8fe1 100644
--- a/lib/Target/X86/X86FastISel.cpp
+++ b/lib/Target/X86/X86FastISel.cpp
@@ -405,6 +405,18 @@ static bool isLegalAddressingModeForNaCl(const X86Subtarget *Subtarget,
       ++NumFastIselNaClFailures;
       return false;
     }
+
+    // See X86DAGToDAGISel::FoldOffsetIntoAddress().
+    // Check for the equivalent of
+    // ((AM.BaseType == X86ISelAddressMode::RegBase ||
+    //   AM.BaseType == X86ISelAddressMode::FrameIndexBase) &&
+    //  (Val > 65535 || Val < -65536))
+    if ((AM.BaseType == X86AddressMode::RegBase ||
+         AM.BaseType == X86AddressMode::FrameIndexBase) &&
+        (AM.Disp > 65535 || AM.Disp < -65536)) {
+      ++NumFastIselNaClFailures;
+      return false;
+    }
   }
 
   return true;