From 0ef0e2e6d0a45cdbc792eee9d76f0a4b7cda5c8f Mon Sep 17 00:00:00 2001 From: Benjamin Kramer Date: Tue, 18 Dec 2012 18:40:20 +0000 Subject: LoopVectorize: Emit reductions as log2(vectorsize) shuffles + vector ops instead of scalar operations. For example on x86 with SSE4.2 a <8 x i8> add reduction becomes movdqa %xmm0, %xmm1 movhlps %xmm1, %xmm1 ## xmm1 = xmm1[1,1] paddw %xmm0, %xmm1 pshufd $1, %xmm1, %xmm0 ## xmm0 = xmm1[1,0,0,0] paddw %xmm1, %xmm0 phaddw %xmm0, %xmm0 pextrb $0, %xmm0, %edx instead of pextrb $2, %xmm0, %esi pextrb $0, %xmm0, %edx addb %sil, %dl pextrb $4, %xmm0, %esi addb %dl, %sil pextrb $6, %xmm0, %edx addb %sil, %dl pextrb $8, %xmm0, %esi addb %dl, %sil pextrb $10, %xmm0, %edi pextrb $14, %xmm0, %edx addb %sil, %dil pextrb $12, %xmm0, %esi addb %dil, %sil addb %sil, %dl git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@170439 91177308-0d34-0410-b5e6-96231b3b80d8 --- test/Transforms/LoopVectorize/reduction.ll | 40 ++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) (limited to 'test/Transforms/LoopVectorize') diff --git a/test/Transforms/LoopVectorize/reduction.ll b/test/Transforms/LoopVectorize/reduction.ll index c1848b35fc..bc1c0290de 100644 --- a/test/Transforms/LoopVectorize/reduction.ll +++ b/test/Transforms/LoopVectorize/reduction.ll @@ -7,6 +7,11 @@ target triple = "x86_64-apple-macosx10.8.0" ;CHECK: phi <4 x i32> ;CHECK: load <4 x i32> ;CHECK: add <4 x i32> +;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> +;CHECK: add <4 x i32> +;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> +;CHECK: add <4 x i32> +;CHECK: extractelement <4 x i32> %{{.*}}, i32 0 ;CHECK: ret i32 define i32 @reduction_sum(i32 %n, i32* noalias nocapture %A, i32* noalias nocapture %B) nounwind uwtable readonly noinline ssp { %1 = icmp sgt i32 %n, 0 @@ -37,6 +42,11 @@ define i32 @reduction_sum(i32 %n, i32* noalias nocapture %A, i32* noalias nocapt ;CHECK: phi <4 x i32> ;CHECK: load <4 x i32> ;CHECK: mul <4 x i32> +;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> +;CHECK: mul <4 x i32> +;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> +;CHECK: mul <4 x i32> +;CHECK: extractelement <4 x i32> %{{.*}}, i32 0 ;CHECK: ret i32 define i32 @reduction_prod(i32 %n, i32* noalias nocapture %A, i32* noalias nocapture %B) nounwind uwtable readonly noinline ssp { %1 = icmp sgt i32 %n, 0 @@ -67,6 +77,11 @@ define i32 @reduction_prod(i32 %n, i32* noalias nocapture %A, i32* noalias nocap ;CHECK: phi <4 x i32> ;CHECK: load <4 x i32> ;CHECK: mul nsw <4 x i32> +;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> +;CHECK: add <4 x i32> +;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> +;CHECK: add <4 x i32> +;CHECK: extractelement <4 x i32> %{{.*}}, i32 0 ;CHECK: ret i32 define i32 @reduction_mix(i32 %n, i32* noalias nocapture %A, i32* noalias nocapture %B) nounwind uwtable readonly noinline ssp { %1 = icmp sgt i32 %n, 0 @@ -95,6 +110,11 @@ define i32 @reduction_mix(i32 %n, i32* noalias nocapture %A, i32* noalias nocapt ;CHECK: @reduction_mul ;CHECK: mul <4 x i32> +;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> +;CHECK: mul <4 x i32> +;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> +;CHECK: mul <4 x i32> +;CHECK: extractelement <4 x i32> %{{.*}}, i32 0 ;CHECK: ret i32 define i32 @reduction_mul(i32 %n, i32* noalias nocapture %A, i32* noalias nocapture %B) nounwind uwtable readonly noinline ssp { %1 = icmp sgt i32 %n, 0 @@ -124,6 +144,11 @@ define i32 @reduction_mul(i32 %n, i32* noalias nocapture %A, i32* noalias nocapt ;CHECK: @start_at_non_zero ;CHECK: phi <4 x i32> ;CHECK: +;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> +;CHECK: add <4 x i32> +;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> +;CHECK: add <4 x i32> +;CHECK: extractelement <4 x i32> %{{.*}}, i32 0 ;CHECK: ret i32 define i32 @start_at_non_zero(i32* nocapture %in, i32* nocapture %coeff, i32* nocapture %out, i32 %n) nounwind uwtable readonly ssp { entry: @@ -152,6 +177,11 @@ for.end: ; preds = %for.body, %entry ;CHECK: @reduction_and ;CHECK: and <4 x i32> ;CHECK: +;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> +;CHECK: and <4 x i32> +;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> +;CHECK: and <4 x i32> +;CHECK: extractelement <4 x i32> %{{.*}}, i32 0 ;CHECK: ret i32 define i32 @reduction_and(i32 %n, i32* nocapture %A, i32* nocapture %B) nounwind uwtable readonly { entry: @@ -179,6 +209,11 @@ for.end: ; preds = %for.body, %entry ;CHECK: @reduction_or ;CHECK: or <4 x i32> +;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> +;CHECK: or <4 x i32> +;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> +;CHECK: or <4 x i32> +;CHECK: extractelement <4 x i32> %{{.*}}, i32 0 ;CHECK: ret i32 define i32 @reduction_or(i32 %n, i32* nocapture %A, i32* nocapture %B) nounwind uwtable readonly { entry: @@ -206,6 +241,11 @@ for.end: ; preds = %for.body, %entry ;CHECK: @reduction_xor ;CHECK: xor <4 x i32> +;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> +;CHECK: xor <4 x i32> +;CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> undef, <4 x i32> +;CHECK: xor <4 x i32> +;CHECK: extractelement <4 x i32> %{{.*}}, i32 0 ;CHECK: ret i32 define i32 @reduction_xor(i32 %n, i32* nocapture %A, i32* nocapture %B) nounwind uwtable readonly { entry: -- cgit v1.2.3-70-g09d2