Author: Simon Pilgrim Date: 2021-01-11T12:51:18Z New Revision: 8112a2598ce180ab4cd106f154a71e813fc28d91
URL: https://github.com/llvm/llvm-project/commit/8112a2598ce180ab4cd106f154a71e813fc28d91 DIFF: https://github.com/llvm/llvm-project/commit/8112a2598ce180ab4cd106f154a71e813fc28d91.diff LOG: [X86][SSE] Add 'vectorized sum' test patterns These are often generated when building a vector from the reduction sums of independent vectors. I've implemented some typical patterns from various v4f32/v4i32 based off current codegen emitted from the vectorizers, although these tests are more about tweaking some hadd style backend folds to handle whatever the vectorizers/vectorcombine throws at us... Added: llvm/test/CodeGen/X86/horizontal-sum.ll Modified: Removed: ################################################################################ diff --git a/llvm/test/CodeGen/X86/horizontal-sum.ll b/llvm/test/CodeGen/X86/horizontal-sum.ll new file mode 100644 index 000000000000..47d44171d99a --- /dev/null +++ b/llvm/test/CodeGen/X86/horizontal-sum.ll @@ -0,0 +1,1189 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+ssse3 | FileCheck %s --check-prefixes=SSSE3,SSSE3-SLOW +; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+ssse3,fast-hops | FileCheck %s --check-prefixes=SSSE3,SSSE3-FAST +; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX-SLOW,AVX1-SLOW +; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx,fast-hops | FileCheck %s --check-prefixes=AVX,AVX-FAST,AVX1-FAST +; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX-SLOW,AVX2-SLOW +; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2,fast-hops | FileCheck %s --check-prefixes=AVX,AVX-FAST,AVX2-FAST + +; Vectorized Pairwise Sum Reductions +; e.g. +; inline STYPE sum(VTYPE x) { +; return (x[0] + x[1]) + (x[2] + x[3]); +; } +; +; VTYPE sum4(VTYPE A0, VTYPE A1, VTYPE A2, VTYPE A3) { +; return (VTYPE) { sum( A0 ), sum( A1 ), sum( A2 ), sum( A3 ) }; +; } + +define <4 x float> @pair_sum_v4f32_v4f32(<4 x float> %0, <4 x float> %1, <4 x float> %2, <4 x float> %3) { +; SSSE3-SLOW-LABEL: pair_sum_v4f32_v4f32: +; SSSE3-SLOW: # %bb.0: +; SSSE3-SLOW-NEXT: haddps %xmm0, %xmm0 +; SSSE3-SLOW-NEXT: movshdup {{.*#+}} xmm4 = xmm0[1,1,3,3] +; SSSE3-SLOW-NEXT: addps %xmm4, %xmm0 +; SSSE3-SLOW-NEXT: haddps %xmm1, %xmm1 +; SSSE3-SLOW-NEXT: movshdup {{.*#+}} xmm4 = xmm1[1,1,3,3] +; SSSE3-SLOW-NEXT: addps %xmm1, %xmm4 +; SSSE3-SLOW-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1] +; SSSE3-SLOW-NEXT: haddps %xmm2, %xmm2 +; SSSE3-SLOW-NEXT: movshdup {{.*#+}} xmm1 = xmm2[1,1,3,3] +; SSSE3-SLOW-NEXT: addps %xmm2, %xmm1 +; SSSE3-SLOW-NEXT: haddps %xmm3, %xmm3 +; SSSE3-SLOW-NEXT: movshdup {{.*#+}} xmm2 = xmm3[1,1,3,3] +; SSSE3-SLOW-NEXT: addps %xmm3, %xmm2 +; SSSE3-SLOW-NEXT: movlhps {{.*#+}} xmm2 = xmm2[0],xmm1[0] +; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,0] +; SSSE3-SLOW-NEXT: retq +; +; SSSE3-FAST-LABEL: pair_sum_v4f32_v4f32: +; SSSE3-FAST: # %bb.0: +; SSSE3-FAST-NEXT: haddps %xmm0, %xmm0 +; SSSE3-FAST-NEXT: haddps %xmm1, %xmm1 +; SSSE3-FAST-NEXT: haddps %xmm1, %xmm0 +; SSSE3-FAST-NEXT: haddps %xmm2, %xmm2 +; SSSE3-FAST-NEXT: haddps %xmm3, %xmm3 +; SSSE3-FAST-NEXT: haddps %xmm2, %xmm3 +; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,2],xmm3[2,0] +; SSSE3-FAST-NEXT: retq +; +; AVX1-SLOW-LABEL: pair_sum_v4f32_v4f32: +; AVX1-SLOW: # %bb.0: +; AVX1-SLOW-NEXT: vhaddps %xmm0, %xmm0, %xmm0 +; AVX1-SLOW-NEXT: vmovshdup {{.*#+}} xmm4 = xmm0[1,1,3,3] +; AVX1-SLOW-NEXT: vaddps %xmm4, %xmm0, %xmm0 +; AVX1-SLOW-NEXT: vhaddps %xmm1, %xmm1, %xmm1 +; AVX1-SLOW-NEXT: vmovshdup {{.*#+}} xmm4 = xmm1[1,1,3,3] +; AVX1-SLOW-NEXT: vaddps %xmm4, %xmm1, %xmm1 +; AVX1-SLOW-NEXT: vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; AVX1-SLOW-NEXT: vhaddps %xmm2, %xmm2, %xmm1 +; AVX1-SLOW-NEXT: vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3] +; AVX1-SLOW-NEXT: vaddps %xmm2, %xmm1, %xmm1 +; AVX1-SLOW-NEXT: vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; AVX1-SLOW-NEXT: vhaddps %xmm3, %xmm3, %xmm1 +; AVX1-SLOW-NEXT: vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3] +; AVX1-SLOW-NEXT: vaddps %xmm2, %xmm1, %xmm1 +; AVX1-SLOW-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0] +; AVX1-SLOW-NEXT: retq +; +; AVX1-FAST-LABEL: pair_sum_v4f32_v4f32: +; AVX1-FAST: # %bb.0: +; AVX1-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0 +; AVX1-FAST-NEXT: vhaddps %xmm1, %xmm1, %xmm1 +; AVX1-FAST-NEXT: vhaddps %xmm1, %xmm0, %xmm0 +; AVX1-FAST-NEXT: vhaddps %xmm2, %xmm2, %xmm1 +; AVX1-FAST-NEXT: vhaddps %xmm1, %xmm1, %xmm1 +; AVX1-FAST-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,1] +; AVX1-FAST-NEXT: vhaddps %xmm3, %xmm3, %xmm1 +; AVX1-FAST-NEXT: vhaddps %xmm1, %xmm1, %xmm1 +; AVX1-FAST-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0] +; AVX1-FAST-NEXT: retq +; +; AVX2-SLOW-LABEL: pair_sum_v4f32_v4f32: +; AVX2-SLOW: # %bb.0: +; AVX2-SLOW-NEXT: vhaddps %xmm0, %xmm0, %xmm0 +; AVX2-SLOW-NEXT: vmovshdup {{.*#+}} xmm4 = xmm0[1,1,3,3] +; AVX2-SLOW-NEXT: vaddps %xmm4, %xmm0, %xmm0 +; AVX2-SLOW-NEXT: vhaddps %xmm1, %xmm1, %xmm1 +; AVX2-SLOW-NEXT: vmovshdup {{.*#+}} xmm4 = xmm1[1,1,3,3] +; AVX2-SLOW-NEXT: vaddps %xmm4, %xmm1, %xmm1 +; AVX2-SLOW-NEXT: vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; AVX2-SLOW-NEXT: vhaddps %xmm2, %xmm2, %xmm1 +; AVX2-SLOW-NEXT: vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3] +; AVX2-SLOW-NEXT: vaddps %xmm2, %xmm1, %xmm1 +; AVX2-SLOW-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,3] +; AVX2-SLOW-NEXT: vhaddps %xmm3, %xmm3, %xmm1 +; AVX2-SLOW-NEXT: vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3] +; AVX2-SLOW-NEXT: vaddps %xmm2, %xmm1, %xmm1 +; AVX2-SLOW-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0] +; AVX2-SLOW-NEXT: retq +; +; AVX2-FAST-LABEL: pair_sum_v4f32_v4f32: +; AVX2-FAST: # %bb.0: +; AVX2-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0 +; AVX2-FAST-NEXT: vhaddps %xmm1, %xmm1, %xmm1 +; AVX2-FAST-NEXT: vhaddps %xmm1, %xmm0, %xmm0 +; AVX2-FAST-NEXT: vhaddps %xmm2, %xmm2, %xmm1 +; AVX2-FAST-NEXT: vhaddps %xmm1, %xmm1, %xmm1 +; AVX2-FAST-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,3] +; AVX2-FAST-NEXT: vhaddps %xmm3, %xmm3, %xmm1 +; AVX2-FAST-NEXT: vhaddps %xmm1, %xmm1, %xmm1 +; AVX2-FAST-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0] +; AVX2-FAST-NEXT: retq + %5 = shufflevector <4 x float> %0, <4 x float> poison, <2 x i32> <i32 0, i32 2> + %6 = shufflevector <4 x float> %0, <4 x float> poison, <2 x i32> <i32 1, i32 3> + %7 = fadd <2 x float> %5, %6 + %8 = shufflevector <2 x float> %7, <2 x float> poison, <2 x i32> <i32 1, i32 undef> + %9 = fadd <2 x float> %7, %8 + %10 = shufflevector <4 x float> %1, <4 x float> poison, <2 x i32> <i32 0, i32 2> + %11 = shufflevector <4 x float> %1, <4 x float> poison, <2 x i32> <i32 1, i32 3> + %12 = fadd <2 x float> %10, %11 + %13 = shufflevector <2 x float> %12, <2 x float> poison, <2 x i32> <i32 1, i32 undef> + %14 = fadd <2 x float> %12, %13 + %15 = shufflevector <2 x float> %9, <2 x float> %14, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef> + %16 = shufflevector <4 x float> %2, <4 x float> poison, <2 x i32> <i32 0, i32 2> + %17 = shufflevector <4 x float> %2, <4 x float> poison, <2 x i32> <i32 1, i32 3> + %18 = fadd <2 x float> %16, %17 + %19 = shufflevector <2 x float> %18, <2 x float> poison, <2 x i32> <i32 1, i32 undef> + %20 = fadd <2 x float> %18, %19 + %21 = shufflevector <2 x float> %20, <2 x float> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef> + %22 = shufflevector <4 x float> %15, <4 x float> %21, <4 x i32> <i32 0, i32 1, i32 4, i32 undef> + %23 = shufflevector <4 x float> %3, <4 x float> poison, <2 x i32> <i32 0, i32 2> + %24 = shufflevector <4 x float> %3, <4 x float> poison, <2 x i32> <i32 1, i32 3> + %25 = fadd <2 x float> %23, %24 + %26 = shufflevector <2 x float> %25, <2 x float> poison, <2 x i32> <i32 1, i32 undef> + %27 = fadd <2 x float> %25, %26 + %28 = shufflevector <2 x float> %27, <2 x float> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef> + %29 = shufflevector <4 x float> %22, <4 x float> %28, <4 x i32> <i32 0, i32 1, i32 2, i32 4> + ret <4 x float> %29 +} + +define <4 x i32> @pair_sum_v4i32_v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2, <4 x i32> %3) { +; SSSE3-SLOW-LABEL: pair_sum_v4i32_v4i32: +; SSSE3-SLOW: # %bb.0: +; SSSE3-SLOW-NEXT: phaddd %xmm0, %xmm0 +; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,1,1] +; SSSE3-SLOW-NEXT: paddd %xmm4, %xmm0 +; SSSE3-SLOW-NEXT: phaddd %xmm1, %xmm1 +; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm4 = xmm1[1,1,1,1] +; SSSE3-SLOW-NEXT: paddd %xmm1, %xmm4 +; SSSE3-SLOW-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1] +; SSSE3-SLOW-NEXT: phaddd %xmm2, %xmm2 +; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm1 = xmm2[1,1,1,1] +; SSSE3-SLOW-NEXT: paddd %xmm2, %xmm1 +; SSSE3-SLOW-NEXT: phaddd %xmm3, %xmm3 +; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm2 = xmm3[1,1,1,1] +; SSSE3-SLOW-NEXT: paddd %xmm3, %xmm2 +; SSSE3-SLOW-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0] +; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,0] +; SSSE3-SLOW-NEXT: retq +; +; SSSE3-FAST-LABEL: pair_sum_v4i32_v4i32: +; SSSE3-FAST: # %bb.0: +; SSSE3-FAST-NEXT: phaddd %xmm0, %xmm0 +; SSSE3-FAST-NEXT: phaddd %xmm1, %xmm1 +; SSSE3-FAST-NEXT: phaddd %xmm1, %xmm0 +; SSSE3-FAST-NEXT: phaddd %xmm2, %xmm2 +; SSSE3-FAST-NEXT: phaddd %xmm3, %xmm3 +; SSSE3-FAST-NEXT: phaddd %xmm2, %xmm3 +; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,2],xmm3[2,0] +; SSSE3-FAST-NEXT: retq +; +; AVX1-SLOW-LABEL: pair_sum_v4i32_v4i32: +; AVX1-SLOW: # %bb.0: +; AVX1-SLOW-NEXT: vphaddd %xmm0, %xmm0, %xmm0 +; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1] +; AVX1-SLOW-NEXT: vpaddd %xmm4, %xmm0, %xmm0 +; AVX1-SLOW-NEXT: vphaddd %xmm1, %xmm1, %xmm1 +; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[1,1,1,1] +; AVX1-SLOW-NEXT: vpaddd %xmm4, %xmm1, %xmm1 +; AVX1-SLOW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; AVX1-SLOW-NEXT: vphaddd %xmm2, %xmm2, %xmm1 +; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1] +; AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm1, %xmm1 +; AVX1-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; AVX1-SLOW-NEXT: vphaddd %xmm3, %xmm3, %xmm1 +; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1] +; AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm1, %xmm1 +; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0] +; AVX1-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5],xmm1[6,7] +; AVX1-SLOW-NEXT: retq +; +; AVX1-FAST-LABEL: pair_sum_v4i32_v4i32: +; AVX1-FAST: # %bb.0: +; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0 +; AVX1-FAST-NEXT: vphaddd %xmm1, %xmm1, %xmm1 +; AVX1-FAST-NEXT: vphaddd %xmm1, %xmm0, %xmm0 +; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3] +; AVX1-FAST-NEXT: vphaddd %xmm2, %xmm2, %xmm1 +; AVX1-FAST-NEXT: vphaddd %xmm1, %xmm1, %xmm1 +; AVX1-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; AVX1-FAST-NEXT: vphaddd %xmm3, %xmm3, %xmm1 +; AVX1-FAST-NEXT: vphaddd %xmm1, %xmm1, %xmm1 +; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0] +; AVX1-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5],xmm1[6,7] +; AVX1-FAST-NEXT: retq +; +; AVX2-SLOW-LABEL: pair_sum_v4i32_v4i32: +; AVX2-SLOW: # %bb.0: +; AVX2-SLOW-NEXT: vphaddd %xmm0, %xmm0, %xmm0 +; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1] +; AVX2-SLOW-NEXT: vpaddd %xmm4, %xmm0, %xmm0 +; AVX2-SLOW-NEXT: vphaddd %xmm1, %xmm1, %xmm1 +; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[1,1,1,1] +; AVX2-SLOW-NEXT: vpaddd %xmm4, %xmm1, %xmm1 +; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; AVX2-SLOW-NEXT: vphaddd %xmm2, %xmm2, %xmm1 +; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1] +; AVX2-SLOW-NEXT: vpaddd %xmm2, %xmm1, %xmm1 +; AVX2-SLOW-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,3] +; AVX2-SLOW-NEXT: vphaddd %xmm3, %xmm3, %xmm1 +; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1] +; AVX2-SLOW-NEXT: vpaddd %xmm2, %xmm1, %xmm1 +; AVX2-SLOW-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0] +; AVX2-SLOW-NEXT: retq +; +; AVX2-FAST-LABEL: pair_sum_v4i32_v4i32: +; AVX2-FAST: # %bb.0: +; AVX2-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0 +; AVX2-FAST-NEXT: vphaddd %xmm1, %xmm1, %xmm1 +; AVX2-FAST-NEXT: vphaddd %xmm1, %xmm0, %xmm0 +; AVX2-FAST-NEXT: vphaddd %xmm2, %xmm2, %xmm1 +; AVX2-FAST-NEXT: vphaddd %xmm1, %xmm1, %xmm1 +; AVX2-FAST-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,3] +; AVX2-FAST-NEXT: vphaddd %xmm3, %xmm3, %xmm1 +; AVX2-FAST-NEXT: vphaddd %xmm1, %xmm1, %xmm1 +; AVX2-FAST-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0] +; AVX2-FAST-NEXT: retq + %5 = shufflevector <4 x i32> %0, <4 x i32> poison, <2 x i32> <i32 0, i32 2> + %6 = shufflevector <4 x i32> %0, <4 x i32> poison, <2 x i32> <i32 1, i32 3> + %7 = add <2 x i32> %5, %6 + %8 = shufflevector <2 x i32> %7, <2 x i32> poison, <2 x i32> <i32 1, i32 undef> + %9 = add <2 x i32> %7, %8 + %10 = shufflevector <4 x i32> %1, <4 x i32> poison, <2 x i32> <i32 0, i32 2> + %11 = shufflevector <4 x i32> %1, <4 x i32> poison, <2 x i32> <i32 1, i32 3> + %12 = add <2 x i32> %10, %11 + %13 = shufflevector <2 x i32> %12, <2 x i32> poison, <2 x i32> <i32 1, i32 undef> + %14 = add <2 x i32> %12, %13 + %15 = shufflevector <2 x i32> %9, <2 x i32> %14, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef> + %16 = shufflevector <4 x i32> %2, <4 x i32> poison, <2 x i32> <i32 0, i32 2> + %17 = shufflevector <4 x i32> %2, <4 x i32> poison, <2 x i32> <i32 1, i32 3> + %18 = add <2 x i32> %16, %17 + %19 = shufflevector <2 x i32> %18, <2 x i32> poison, <2 x i32> <i32 1, i32 undef> + %20 = add <2 x i32> %18, %19 + %21 = shufflevector <2 x i32> %20, <2 x i32> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef> + %22 = shufflevector <4 x i32> %15, <4 x i32> %21, <4 x i32> <i32 0, i32 1, i32 4, i32 undef> + %23 = shufflevector <4 x i32> %3, <4 x i32> poison, <2 x i32> <i32 0, i32 2> + %24 = shufflevector <4 x i32> %3, <4 x i32> poison, <2 x i32> <i32 1, i32 3> + %25 = add <2 x i32> %23, %24 + %26 = shufflevector <2 x i32> %25, <2 x i32> poison, <2 x i32> <i32 1, i32 undef> + %27 = add <2 x i32> %25, %26 + %28 = shufflevector <2 x i32> %27, <2 x i32> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef> + %29 = shufflevector <4 x i32> %22, <4 x i32> %28, <4 x i32> <i32 0, i32 1, i32 2, i32 4> + ret <4 x i32> %29 +} + +define <8 x float> @pair_sum_v8f32_v4f32(<4 x float> %0, <4 x float> %1, <4 x float> %2, <4 x float> %3, <4 x float> %4, <4 x float> %5, <4 x float> %6, <4 x float> %7) { +; SSSE3-SLOW-LABEL: pair_sum_v8f32_v4f32: +; SSSE3-SLOW: # %bb.0: +; SSSE3-SLOW-NEXT: haddps %xmm0, %xmm0 +; SSSE3-SLOW-NEXT: movshdup {{.*#+}} xmm8 = xmm0[1,1,3,3] +; SSSE3-SLOW-NEXT: addps %xmm8, %xmm0 +; SSSE3-SLOW-NEXT: haddps %xmm1, %xmm1 +; SSSE3-SLOW-NEXT: movshdup {{.*#+}} xmm8 = xmm1[1,1,3,3] +; SSSE3-SLOW-NEXT: addps %xmm1, %xmm8 +; SSSE3-SLOW-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm8[0],xmm0[1],xmm8[1] +; SSSE3-SLOW-NEXT: movaps %xmm2, %xmm1 +; SSSE3-SLOW-NEXT: haddps %xmm3, %xmm1 +; SSSE3-SLOW-NEXT: haddps %xmm2, %xmm2 +; SSSE3-SLOW-NEXT: haddps %xmm3, %xmm3 +; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,0],xmm2[1,0] +; SSSE3-SLOW-NEXT: movaps %xmm5, %xmm2 +; SSSE3-SLOW-NEXT: haddps %xmm4, %xmm5 +; SSSE3-SLOW-NEXT: haddps %xmm4, %xmm4 +; SSSE3-SLOW-NEXT: haddps %xmm2, %xmm2 +; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,1],xmm4[1,1] +; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm3 = xmm3[2,0],xmm2[2,0] +; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,2],xmm5[2,0] +; SSSE3-SLOW-NEXT: addps %xmm3, %xmm1 +; SSSE3-SLOW-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; SSSE3-SLOW-NEXT: haddps %xmm6, %xmm6 +; SSSE3-SLOW-NEXT: haddps %xmm7, %xmm7 +; SSSE3-SLOW-NEXT: haddps %xmm7, %xmm6 +; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,3],xmm6[0,2] +; SSSE3-SLOW-NEXT: retq +; +; SSSE3-FAST-LABEL: pair_sum_v8f32_v4f32: +; SSSE3-FAST: # %bb.0: +; SSSE3-FAST-NEXT: haddps %xmm0, %xmm0 +; SSSE3-FAST-NEXT: haddps %xmm1, %xmm1 +; SSSE3-FAST-NEXT: haddps %xmm1, %xmm0 +; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,2,1,3] +; SSSE3-FAST-NEXT: movaps %xmm2, %xmm1 +; SSSE3-FAST-NEXT: haddps %xmm3, %xmm1 +; SSSE3-FAST-NEXT: haddps %xmm2, %xmm2 +; SSSE3-FAST-NEXT: haddps %xmm3, %xmm3 +; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,0],xmm2[1,0] +; SSSE3-FAST-NEXT: movaps %xmm5, %xmm2 +; SSSE3-FAST-NEXT: haddps %xmm4, %xmm5 +; SSSE3-FAST-NEXT: haddps %xmm4, %xmm4 +; SSSE3-FAST-NEXT: haddps %xmm2, %xmm2 +; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,1],xmm4[1,1] +; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm3 = xmm3[2,0],xmm2[2,0] +; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,2],xmm5[2,0] +; SSSE3-FAST-NEXT: addps %xmm3, %xmm1 +; SSSE3-FAST-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; SSSE3-FAST-NEXT: haddps %xmm6, %xmm6 +; SSSE3-FAST-NEXT: haddps %xmm7, %xmm7 +; SSSE3-FAST-NEXT: haddps %xmm7, %xmm6 +; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,3],xmm6[0,2] +; SSSE3-FAST-NEXT: retq +; +; AVX1-SLOW-LABEL: pair_sum_v8f32_v4f32: +; AVX1-SLOW: # %bb.0: +; AVX1-SLOW-NEXT: vhaddps %xmm0, %xmm0, %xmm8 +; AVX1-SLOW-NEXT: vmovshdup {{.*#+}} xmm0 = xmm8[1,1,3,3] +; AVX1-SLOW-NEXT: vaddps %xmm0, %xmm8, %xmm8 +; AVX1-SLOW-NEXT: vhaddps %xmm1, %xmm1, %xmm1 +; AVX1-SLOW-NEXT: vmovshdup {{.*#+}} xmm0 = xmm1[1,1,3,3] +; AVX1-SLOW-NEXT: vaddps %xmm0, %xmm1, %xmm0 +; AVX1-SLOW-NEXT: vunpcklps {{.*#+}} xmm0 = xmm8[0],xmm0[0],xmm8[1],xmm0[1] +; AVX1-SLOW-NEXT: vhaddps %xmm3, %xmm2, %xmm1 +; AVX1-SLOW-NEXT: vhaddps %xmm2, %xmm2, %xmm2 +; AVX1-SLOW-NEXT: vhaddps %xmm3, %xmm3, %xmm3 +; AVX1-SLOW-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[1],xmm3[1],zero,zero +; AVX1-SLOW-NEXT: vhaddps %xmm4, %xmm4, %xmm3 +; AVX1-SLOW-NEXT: vhaddps %xmm5, %xmm5, %xmm4 +; AVX1-SLOW-NEXT: vshufps {{.*#+}} xmm1 = xmm1[0,2],xmm3[0,1] +; AVX1-SLOW-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[0] +; AVX1-SLOW-NEXT: vshufps {{.*#+}} xmm2 = xmm2[0,1],xmm3[1,3] +; AVX1-SLOW-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm4[1] +; AVX1-SLOW-NEXT: vaddps %xmm2, %xmm1, %xmm1 +; AVX1-SLOW-NEXT: vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; AVX1-SLOW-NEXT: vpermilpd {{.*#+}} xmm1 = xmm1[1,0] +; AVX1-SLOW-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1 +; AVX1-SLOW-NEXT: vhaddps %xmm6, %xmm6, %xmm2 +; AVX1-SLOW-NEXT: vhaddps %xmm7, %xmm7, %xmm3 +; AVX1-SLOW-NEXT: vhaddps %xmm3, %xmm2, %xmm2 +; AVX1-SLOW-NEXT: vpermilps {{.*#+}} xmm2 = xmm2[0,2,2,3] +; AVX1-SLOW-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-SLOW-NEXT: vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[2] +; AVX1-SLOW-NEXT: retq +; +; AVX1-FAST-LABEL: pair_sum_v8f32_v4f32: +; AVX1-FAST: # %bb.0: +; AVX1-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0 +; AVX1-FAST-NEXT: vhaddps %xmm1, %xmm1, %xmm1 +; AVX1-FAST-NEXT: vhaddps %xmm1, %xmm0, %xmm0 +; AVX1-FAST-NEXT: vhaddps %xmm3, %xmm2, %xmm1 +; AVX1-FAST-NEXT: vhaddps %xmm2, %xmm2, %xmm2 +; AVX1-FAST-NEXT: vhaddps %xmm3, %xmm3, %xmm3 +; AVX1-FAST-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[1],xmm3[1],zero,zero +; AVX1-FAST-NEXT: vhaddps %xmm4, %xmm4, %xmm3 +; AVX1-FAST-NEXT: vhaddps %xmm5, %xmm5, %xmm4 +; AVX1-FAST-NEXT: vshufps {{.*#+}} xmm1 = xmm1[0,2],xmm3[0,1] +; AVX1-FAST-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[0] +; AVX1-FAST-NEXT: vshufps {{.*#+}} xmm2 = xmm2[0,1],xmm3[1,3] +; AVX1-FAST-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm4[1] +; AVX1-FAST-NEXT: vaddps %xmm2, %xmm1, %xmm1 +; AVX1-FAST-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,1] +; AVX1-FAST-NEXT: vpermilpd {{.*#+}} xmm1 = xmm1[1,0] +; AVX1-FAST-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1 +; AVX1-FAST-NEXT: vhaddps %xmm6, %xmm6, %xmm2 +; AVX1-FAST-NEXT: vhaddps %xmm7, %xmm7, %xmm3 +; AVX1-FAST-NEXT: vhaddps %xmm3, %xmm2, %xmm2 +; AVX1-FAST-NEXT: vpermilps {{.*#+}} xmm2 = xmm2[0,2,2,3] +; AVX1-FAST-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX1-FAST-NEXT: vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[2] +; AVX1-FAST-NEXT: retq +; +; AVX2-SLOW-LABEL: pair_sum_v8f32_v4f32: +; AVX2-SLOW: # %bb.0: +; AVX2-SLOW-NEXT: vhaddps %xmm0, %xmm0, %xmm8 +; AVX2-SLOW-NEXT: vmovshdup {{.*#+}} xmm0 = xmm8[1,1,3,3] +; AVX2-SLOW-NEXT: vaddps %xmm0, %xmm8, %xmm8 +; AVX2-SLOW-NEXT: vhaddps %xmm1, %xmm1, %xmm1 +; AVX2-SLOW-NEXT: vmovshdup {{.*#+}} xmm0 = xmm1[1,1,3,3] +; AVX2-SLOW-NEXT: vaddps %xmm0, %xmm1, %xmm0 +; AVX2-SLOW-NEXT: vunpcklps {{.*#+}} xmm0 = xmm8[0],xmm0[0],xmm8[1],xmm0[1] +; AVX2-SLOW-NEXT: vhaddps %xmm4, %xmm4, %xmm1 +; AVX2-SLOW-NEXT: vhaddps %xmm5, %xmm5, %xmm4 +; AVX2-SLOW-NEXT: vhaddps %xmm3, %xmm2, %xmm2 +; AVX2-SLOW-NEXT: vshufps {{.*#+}} xmm3 = xmm2[0,2],xmm1[0,3] +; AVX2-SLOW-NEXT: vinsertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[0] +; AVX2-SLOW-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3] +; AVX2-SLOW-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[1] +; AVX2-SLOW-NEXT: vaddps %xmm1, %xmm3, %xmm1 +; AVX2-SLOW-NEXT: vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; AVX2-SLOW-NEXT: vpermilpd {{.*#+}} xmm1 = xmm1[1,0] +; AVX2-SLOW-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1 +; AVX2-SLOW-NEXT: vhaddps %xmm6, %xmm6, %xmm2 +; AVX2-SLOW-NEXT: vhaddps %xmm7, %xmm7, %xmm3 +; AVX2-SLOW-NEXT: vhaddps %xmm3, %xmm2, %xmm2 +; AVX2-SLOW-NEXT: vpermilps {{.*#+}} xmm2 = xmm2[0,2,2,3] +; AVX2-SLOW-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX2-SLOW-NEXT: vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[2] +; AVX2-SLOW-NEXT: retq +; +; AVX2-FAST-LABEL: pair_sum_v8f32_v4f32: +; AVX2-FAST: # %bb.0: +; AVX2-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0 +; AVX2-FAST-NEXT: vhaddps %xmm1, %xmm1, %xmm1 +; AVX2-FAST-NEXT: vhaddps %xmm1, %xmm0, %xmm0 +; AVX2-FAST-NEXT: vhaddps %xmm4, %xmm4, %xmm1 +; AVX2-FAST-NEXT: vhaddps %xmm5, %xmm5, %xmm4 +; AVX2-FAST-NEXT: vhaddps %xmm3, %xmm2, %xmm2 +; AVX2-FAST-NEXT: vshufps {{.*#+}} xmm3 = xmm2[0,2],xmm1[0,3] +; AVX2-FAST-NEXT: vinsertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[0] +; AVX2-FAST-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3] +; AVX2-FAST-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[1] +; AVX2-FAST-NEXT: vaddps %xmm1, %xmm3, %xmm1 +; AVX2-FAST-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,1] +; AVX2-FAST-NEXT: vpermilpd {{.*#+}} xmm1 = xmm1[1,0] +; AVX2-FAST-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1 +; AVX2-FAST-NEXT: vhaddps %xmm6, %xmm6, %xmm2 +; AVX2-FAST-NEXT: vhaddps %xmm7, %xmm7, %xmm3 +; AVX2-FAST-NEXT: vhaddps %xmm3, %xmm2, %xmm2 +; AVX2-FAST-NEXT: vpermilps {{.*#+}} xmm2 = xmm2[0,2,2,3] +; AVX2-FAST-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0 +; AVX2-FAST-NEXT: vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[2] +; AVX2-FAST-NEXT: retq + %9 = shufflevector <4 x float> %0, <4 x float> poison, <2 x i32> <i32 0, i32 2> + %10 = shufflevector <4 x float> %0, <4 x float> poison, <2 x i32> <i32 1, i32 3> + %11 = fadd <2 x float> %9, %10 + %12 = shufflevector <2 x float> %11, <2 x float> poison, <2 x i32> <i32 1, i32 undef> + %13 = fadd <2 x float> %11, %12 + %14 = shufflevector <4 x float> %1, <4 x float> poison, <2 x i32> <i32 0, i32 2> + %15 = shufflevector <4 x float> %1, <4 x float> poison, <2 x i32> <i32 1, i32 3> + %16 = fadd <2 x float> %14, %15 + %17 = shufflevector <2 x float> %16, <2 x float> poison, <2 x i32> <i32 1, i32 undef> + %18 = fadd <2 x float> %16, %17 + %19 = shufflevector <2 x float> %13, <2 x float> %18, <8 x i32> <i32 0, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> + %20 = shufflevector <4 x float> %2, <4 x float> poison, <2 x i32> <i32 0, i32 2> + %21 = shufflevector <4 x float> %2, <4 x float> poison, <2 x i32> <i32 1, i32 3> + %22 = fadd <2 x float> %20, %21 + %23 = shufflevector <4 x float> %3, <4 x float> poison, <2 x i32> <i32 0, i32 2> + %24 = shufflevector <4 x float> %3, <4 x float> poison, <2 x i32> <i32 1, i32 3> + %25 = fadd <2 x float> %23, %24 + %26 = shufflevector <4 x float> %4, <4 x float> poison, <2 x i32> <i32 0, i32 2> + %27 = shufflevector <4 x float> %4, <4 x float> poison, <2 x i32> <i32 1, i32 3> + %28 = fadd <2 x float> %26, %27 + %29 = shufflevector <2 x float> %28, <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef> + %30 = shufflevector <4 x float> %5, <4 x float> poison, <2 x i32> <i32 0, i32 2> + %31 = shufflevector <4 x float> %5, <4 x float> poison, <2 x i32> <i32 1, i32 3> + %32 = fadd <2 x float> %30, %31 + %33 = shufflevector <2 x float> %32, <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef> + %34 = shufflevector <2 x float> %22, <2 x float> %25, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef> + %35 = shufflevector <4 x float> %34, <4 x float> %29, <4 x i32> <i32 0, i32 1, i32 4, i32 undef> + %36 = shufflevector <4 x float> %35, <4 x float> %33, <4 x i32> <i32 0, i32 1, i32 2, i32 4> + %37 = shufflevector <2 x float> %22, <2 x float> %25, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef> + %38 = shufflevector <4 x float> %37, <4 x float> %29, <4 x i32> <i32 0, i32 1, i32 5, i32 undef> + %39 = shufflevector <4 x float> %38, <4 x float> %33, <4 x i32> <i32 0, i32 1, i32 2, i32 5> + %40 = fadd <4 x float> %36, %39 + %41 = shufflevector <4 x float> %40, <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef> + %42 = shufflevector <8 x float> %19, <8 x float> %41, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 10, i32 11, i32 undef, i32 undef> + %43 = shufflevector <4 x float> %6, <4 x float> poison, <2 x i32> <i32 0, i32 2> + %44 = shufflevector <4 x float> %6, <4 x float> poison, <2 x i32> <i32 1, i32 3> + %45 = fadd <2 x float> %43, %44 + %46 = shufflevector <4 x float> %7, <4 x float> poison, <2 x i32> <i32 0, i32 2> + %47 = shufflevector <4 x float> %7, <4 x float> poison, <2 x i32> <i32 1, i32 3> + %48 = fadd <2 x float> %46, %47 + %49 = shufflevector <2 x float> %45, <2 x float> %48, <2 x i32> <i32 0, i32 2> + %50 = shufflevector <2 x float> %45, <2 x float> %48, <2 x i32> <i32 1, i32 3> + %51 = fadd <2 x float> %49, %50 + %52 = shufflevector <2 x float> %51, <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> + %53 = shufflevector <8 x float> %42, <8 x float> %52, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9> + ret <8 x float> %53 +} + +define <8 x i32> @pair_sum_v8i32_v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2, <4 x i32> %3, <4 x i32> %4, <4 x i32> %5, <4 x i32> %6, <4 x i32> %7) { +; SSSE3-SLOW-LABEL: pair_sum_v8i32_v4i32: +; SSSE3-SLOW: # %bb.0: +; SSSE3-SLOW-NEXT: movdqa %xmm2, %xmm8 +; SSSE3-SLOW-NEXT: phaddd %xmm0, %xmm0 +; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,1,1] +; SSSE3-SLOW-NEXT: paddd %xmm2, %xmm0 +; SSSE3-SLOW-NEXT: phaddd %xmm1, %xmm1 +; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,1,1] +; SSSE3-SLOW-NEXT: paddd %xmm1, %xmm2 +; SSSE3-SLOW-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1] +; SSSE3-SLOW-NEXT: phaddd %xmm3, %xmm8 +; SSSE3-SLOW-NEXT: movdqa %xmm5, %xmm1 +; SSSE3-SLOW-NEXT: phaddd %xmm4, %xmm5 +; SSSE3-SLOW-NEXT: phaddd %xmm4, %xmm4 +; SSSE3-SLOW-NEXT: phaddd %xmm1, %xmm1 +; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,1],xmm4[1,1] +; SSSE3-SLOW-NEXT: movdqa %xmm8, %xmm2 +; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,2],xmm5[2,0] +; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm8 = xmm8[1,3],xmm1[2,0] +; SSSE3-SLOW-NEXT: paddd %xmm2, %xmm8 +; SSSE3-SLOW-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm8[0] +; SSSE3-SLOW-NEXT: phaddd %xmm6, %xmm6 +; SSSE3-SLOW-NEXT: phaddd %xmm7, %xmm7 +; SSSE3-SLOW-NEXT: phaddd %xmm7, %xmm6 +; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm8 = xmm8[2,3],xmm6[0,2] +; SSSE3-SLOW-NEXT: movaps %xmm8, %xmm1 +; SSSE3-SLOW-NEXT: retq +; +; SSSE3-FAST-LABEL: pair_sum_v8i32_v4i32: +; SSSE3-FAST: # %bb.0: +; SSSE3-FAST-NEXT: phaddd %xmm0, %xmm0 +; SSSE3-FAST-NEXT: phaddd %xmm1, %xmm1 +; SSSE3-FAST-NEXT: phaddd %xmm1, %xmm0 +; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,1,3] +; SSSE3-FAST-NEXT: phaddd %xmm3, %xmm2 +; SSSE3-FAST-NEXT: movdqa %xmm5, %xmm1 +; SSSE3-FAST-NEXT: phaddd %xmm4, %xmm5 +; SSSE3-FAST-NEXT: phaddd %xmm4, %xmm4 +; SSSE3-FAST-NEXT: phaddd %xmm1, %xmm1 +; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,1],xmm4[1,1] +; SSSE3-FAST-NEXT: movdqa %xmm2, %xmm3 +; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,2],xmm5[2,0] +; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,3],xmm1[2,0] +; SSSE3-FAST-NEXT: paddd %xmm3, %xmm2 +; SSSE3-FAST-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0] +; SSSE3-FAST-NEXT: phaddd %xmm6, %xmm6 +; SSSE3-FAST-NEXT: phaddd %xmm7, %xmm7 +; SSSE3-FAST-NEXT: phaddd %xmm7, %xmm6 +; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,3],xmm6[0,2] +; SSSE3-FAST-NEXT: movaps %xmm2, %xmm1 +; SSSE3-FAST-NEXT: retq +; +; AVX1-SLOW-LABEL: pair_sum_v8i32_v4i32: +; AVX1-SLOW: # %bb.0: +; AVX1-SLOW-NEXT: vphaddd %xmm0, %xmm0, %xmm8 +; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm8[1,1,1,1] +; AVX1-SLOW-NEXT: vpaddd %xmm0, %xmm8, %xmm8 +; AVX1-SLOW-NEXT: vphaddd %xmm1, %xmm1, %xmm1 +; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm1[1,1,1,1] +; AVX1-SLOW-NEXT: vpaddd %xmm0, %xmm1, %xmm0 +; AVX1-SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm0[0],xmm8[1],xmm0[1] +; AVX1-SLOW-NEXT: vphaddd %xmm3, %xmm2, %xmm1 +; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,2,1,3] +; AVX1-SLOW-NEXT: vphaddd %xmm2, %xmm2, %xmm2 +; AVX1-SLOW-NEXT: vphaddd %xmm3, %xmm3, %xmm3 +; AVX1-SLOW-NEXT: vphaddd %xmm4, %xmm4, %xmm4 +; AVX1-SLOW-NEXT: vphaddd %xmm5, %xmm5, %xmm5 +; AVX1-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm4[0] +; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm5[0,0,0,0] +; AVX1-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3,4,5],xmm0[6,7] +; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[1,1,1,1] +; AVX1-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5,6,7] +; AVX1-SLOW-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm4[1],zero +; AVX1-SLOW-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm5[1] +; AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0 +; AVX1-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm8[0],xmm0[0] +; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3] +; AVX1-SLOW-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; AVX1-SLOW-NEXT: vphaddd %xmm6, %xmm6, %xmm2 +; AVX1-SLOW-NEXT: vphaddd %xmm7, %xmm7, %xmm3 +; AVX1-SLOW-NEXT: vphaddd %xmm3, %xmm2, %xmm2 +; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] +; AVX1-SLOW-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-SLOW-NEXT: vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[2] +; AVX1-SLOW-NEXT: retq +; +; AVX1-FAST-LABEL: pair_sum_v8i32_v4i32: +; AVX1-FAST: # %bb.0: +; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0 +; AVX1-FAST-NEXT: vphaddd %xmm1, %xmm1, %xmm1 +; AVX1-FAST-NEXT: vphaddd %xmm1, %xmm0, %xmm0 +; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm8 = xmm0[0,2,1,3] +; AVX1-FAST-NEXT: vphaddd %xmm3, %xmm2, %xmm1 +; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,2,1,3] +; AVX1-FAST-NEXT: vphaddd %xmm2, %xmm2, %xmm2 +; AVX1-FAST-NEXT: vphaddd %xmm3, %xmm3, %xmm3 +; AVX1-FAST-NEXT: vphaddd %xmm4, %xmm4, %xmm4 +; AVX1-FAST-NEXT: vphaddd %xmm5, %xmm5, %xmm5 +; AVX1-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm4[0] +; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm5[0,0,0,0] +; AVX1-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3,4,5],xmm0[6,7] +; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[1,1,1,1] +; AVX1-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5,6,7] +; AVX1-FAST-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm4[1],zero +; AVX1-FAST-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1,2],xmm5[3] +; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0 +; AVX1-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm8[0],xmm0[0] +; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3] +; AVX1-FAST-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; AVX1-FAST-NEXT: vphaddd %xmm6, %xmm6, %xmm2 +; AVX1-FAST-NEXT: vphaddd %xmm7, %xmm7, %xmm3 +; AVX1-FAST-NEXT: vphaddd %xmm3, %xmm2, %xmm2 +; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] +; AVX1-FAST-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-FAST-NEXT: vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[2] +; AVX1-FAST-NEXT: retq +; +; AVX2-SLOW-LABEL: pair_sum_v8i32_v4i32: +; AVX2-SLOW: # %bb.0: +; AVX2-SLOW-NEXT: vphaddd %xmm0, %xmm0, %xmm8 +; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm8[1,1,1,1] +; AVX2-SLOW-NEXT: vpaddd %xmm0, %xmm8, %xmm8 +; AVX2-SLOW-NEXT: vphaddd %xmm1, %xmm1, %xmm1 +; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm1[1,1,1,1] +; AVX2-SLOW-NEXT: vpaddd %xmm0, %xmm1, %xmm0 +; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm8[0],xmm0[0],xmm8[1],xmm0[1] +; AVX2-SLOW-NEXT: vphaddd %xmm4, %xmm4, %xmm1 +; AVX2-SLOW-NEXT: vphaddd %xmm5, %xmm5, %xmm4 +; AVX2-SLOW-NEXT: vphaddd %xmm3, %xmm2, %xmm2 +; AVX2-SLOW-NEXT: vshufps {{.*#+}} xmm3 = xmm2[0,2],xmm1[0,3] +; AVX2-SLOW-NEXT: vinsertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[0] +; AVX2-SLOW-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3] +; AVX2-SLOW-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[1] +; AVX2-SLOW-NEXT: vpaddd %xmm1, %xmm3, %xmm1 +; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3] +; AVX2-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 +; AVX2-SLOW-NEXT: vphaddd %xmm6, %xmm6, %xmm1 +; AVX2-SLOW-NEXT: vphaddd %xmm7, %xmm7, %xmm2 +; AVX2-SLOW-NEXT: vphaddd %xmm2, %xmm1, %xmm1 +; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] +; AVX2-SLOW-NEXT: vpbroadcastq %xmm1, %ymm1 +; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7] +; AVX2-SLOW-NEXT: retq +; +; AVX2-FAST-LABEL: pair_sum_v8i32_v4i32: +; AVX2-FAST: # %bb.0: +; AVX2-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0 +; AVX2-FAST-NEXT: vphaddd %xmm1, %xmm1, %xmm1 +; AVX2-FAST-NEXT: vphaddd %xmm1, %xmm0, %xmm0 +; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3] +; AVX2-FAST-NEXT: vphaddd %xmm4, %xmm4, %xmm1 +; AVX2-FAST-NEXT: vphaddd %xmm5, %xmm5, %xmm4 +; AVX2-FAST-NEXT: vphaddd %xmm3, %xmm2, %xmm2 +; AVX2-FAST-NEXT: vshufps {{.*#+}} xmm3 = xmm2[0,2],xmm1[0,3] +; AVX2-FAST-NEXT: vinsertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[0] +; AVX2-FAST-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3] +; AVX2-FAST-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[3] +; AVX2-FAST-NEXT: vpaddd %xmm1, %xmm3, %xmm1 +; AVX2-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3] +; AVX2-FAST-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0 +; AVX2-FAST-NEXT: vphaddd %xmm6, %xmm6, %xmm1 +; AVX2-FAST-NEXT: vphaddd %xmm7, %xmm7, %xmm2 +; AVX2-FAST-NEXT: vphaddd %xmm2, %xmm1, %xmm1 +; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] +; AVX2-FAST-NEXT: vpbroadcastq %xmm1, %ymm1 +; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7] +; AVX2-FAST-NEXT: retq + %9 = shufflevector <4 x i32> %0, <4 x i32> poison, <2 x i32> <i32 0, i32 2> + %10 = shufflevector <4 x i32> %0, <4 x i32> poison, <2 x i32> <i32 1, i32 3> + %11 = add <2 x i32> %9, %10 + %12 = shufflevector <2 x i32> %11, <2 x i32> poison, <2 x i32> <i32 1, i32 undef> + %13 = add <2 x i32> %11, %12 + %14 = shufflevector <4 x i32> %1, <4 x i32> poison, <2 x i32> <i32 0, i32 2> + %15 = shufflevector <4 x i32> %1, <4 x i32> poison, <2 x i32> <i32 1, i32 3> + %16 = add <2 x i32> %14, %15 + %17 = shufflevector <2 x i32> %16, <2 x i32> poison, <2 x i32> <i32 1, i32 undef> + %18 = add <2 x i32> %16, %17 + %19 = shufflevector <2 x i32> %13, <2 x i32> %18, <8 x i32> <i32 0, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> + %20 = shufflevector <4 x i32> %2, <4 x i32> poison, <2 x i32> <i32 0, i32 2> + %21 = shufflevector <4 x i32> %2, <4 x i32> poison, <2 x i32> <i32 1, i32 3> + %22 = add <2 x i32> %20, %21 + %23 = shufflevector <4 x i32> %3, <4 x i32> poison, <2 x i32> <i32 0, i32 2> + %24 = shufflevector <4 x i32> %3, <4 x i32> poison, <2 x i32> <i32 1, i32 3> + %25 = add <2 x i32> %23, %24 + %26 = shufflevector <4 x i32> %4, <4 x i32> poison, <2 x i32> <i32 0, i32 2> + %27 = shufflevector <4 x i32> %4, <4 x i32> poison, <2 x i32> <i32 1, i32 3> + %28 = add <2 x i32> %26, %27 + %29 = shufflevector <2 x i32> %28, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef> + %30 = shufflevector <4 x i32> %5, <4 x i32> poison, <2 x i32> <i32 0, i32 2> + %31 = shufflevector <4 x i32> %5, <4 x i32> poison, <2 x i32> <i32 1, i32 3> + %32 = add <2 x i32> %30, %31 + %33 = shufflevector <2 x i32> %32, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef> + %34 = shufflevector <2 x i32> %22, <2 x i32> %25, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef> + %35 = shufflevector <4 x i32> %34, <4 x i32> %29, <4 x i32> <i32 0, i32 1, i32 4, i32 undef> + %36 = shufflevector <4 x i32> %35, <4 x i32> %33, <4 x i32> <i32 0, i32 1, i32 2, i32 4> + %37 = shufflevector <2 x i32> %22, <2 x i32> %25, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef> + %38 = shufflevector <4 x i32> %37, <4 x i32> %29, <4 x i32> <i32 0, i32 1, i32 5, i32 undef> + %39 = shufflevector <4 x i32> %38, <4 x i32> %33, <4 x i32> <i32 0, i32 1, i32 2, i32 5> + %40 = add <4 x i32> %36, %39 + %41 = shufflevector <4 x i32> %40, <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef> + %42 = shufflevector <8 x i32> %19, <8 x i32> %41, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 10, i32 11, i32 undef, i32 undef> + %43 = shufflevector <4 x i32> %6, <4 x i32> poison, <2 x i32> <i32 0, i32 2> + %44 = shufflevector <4 x i32> %6, <4 x i32> poison, <2 x i32> <i32 1, i32 3> + %45 = add <2 x i32> %43, %44 + %46 = shufflevector <4 x i32> %7, <4 x i32> poison, <2 x i32> <i32 0, i32 2> + %47 = shufflevector <4 x i32> %7, <4 x i32> poison, <2 x i32> <i32 1, i32 3> + %48 = add <2 x i32> %46, %47 + %49 = shufflevector <2 x i32> %45, <2 x i32> %48, <2 x i32> <i32 0, i32 2> + %50 = shufflevector <2 x i32> %45, <2 x i32> %48, <2 x i32> <i32 1, i32 3> + %51 = add <2 x i32> %49, %50 + %52 = shufflevector <2 x i32> %51, <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef> + %53 = shufflevector <8 x i32> %42, <8 x i32> %52, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9> + ret <8 x i32> %53 +} + +; Vectorized Sequential Sum Reductions +; e.g. +; inline STYPE sum(VTYPE x) { +; return ((x[0] + x[1]) + x[2]) + x[3]; +; } +; +; VTYPE sum4(VTYPE A0, VTYPE A1, VTYPE A2, VTYPE A3) { +; return (VTYPE) { sum( A0 ), sum( A1 ), sum( A2 ), sum( A3 ) }; +; } + +define <4 x float> @sequential_sum_v4f32_v4f32(<4 x float> %0, <4 x float> %1, <4 x float> %2, <4 x float> %3) { +; SSSE3-SLOW-LABEL: sequential_sum_v4f32_v4f32: +; SSSE3-SLOW: # %bb.0: +; SSSE3-SLOW-NEXT: movaps %xmm0, %xmm4 +; SSSE3-SLOW-NEXT: movaps %xmm0, %xmm5 +; SSSE3-SLOW-NEXT: haddps %xmm1, %xmm5 +; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm5 = xmm5[0,2,2,3] +; SSSE3-SLOW-NEXT: unpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSSE3-SLOW-NEXT: addps %xmm5, %xmm0 +; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,3],xmm4[3,3] +; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0,2,3] +; SSSE3-SLOW-NEXT: addps %xmm1, %xmm0 +; SSSE3-SLOW-NEXT: movshdup {{.*#+}} xmm1 = xmm2[1,1,3,3] +; SSSE3-SLOW-NEXT: addps %xmm2, %xmm1 +; SSSE3-SLOW-NEXT: movaps %xmm2, %xmm4 +; SSSE3-SLOW-NEXT: unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm2[1] +; SSSE3-SLOW-NEXT: addps %xmm1, %xmm4 +; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm2 = xmm2[3,3,3,3] +; SSSE3-SLOW-NEXT: addps %xmm4, %xmm2 +; SSSE3-SLOW-NEXT: movshdup {{.*#+}} xmm1 = xmm3[1,1,3,3] +; SSSE3-SLOW-NEXT: addps %xmm3, %xmm1 +; SSSE3-SLOW-NEXT: movaps %xmm3, %xmm4 +; SSSE3-SLOW-NEXT: unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm3[1] +; SSSE3-SLOW-NEXT: addps %xmm1, %xmm4 +; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,3,3,3] +; SSSE3-SLOW-NEXT: addps %xmm4, %xmm3 +; SSSE3-SLOW-NEXT: movlhps {{.*#+}} xmm3 = xmm3[0],xmm2[0] +; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,0] +; SSSE3-SLOW-NEXT: retq +; +; SSSE3-FAST-LABEL: sequential_sum_v4f32_v4f32: +; SSSE3-FAST: # %bb.0: +; SSSE3-FAST-NEXT: movaps %xmm0, %xmm4 +; SSSE3-FAST-NEXT: movaps %xmm0, %xmm5 +; SSSE3-FAST-NEXT: haddps %xmm1, %xmm5 +; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm5 = xmm5[0,2,2,3] +; SSSE3-FAST-NEXT: unpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSSE3-FAST-NEXT: addps %xmm5, %xmm0 +; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,3],xmm4[3,3] +; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0,2,3] +; SSSE3-FAST-NEXT: addps %xmm1, %xmm0 +; SSSE3-FAST-NEXT: movaps %xmm2, %xmm1 +; SSSE3-FAST-NEXT: haddps %xmm2, %xmm1 +; SSSE3-FAST-NEXT: movaps %xmm2, %xmm4 +; SSSE3-FAST-NEXT: unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm2[1] +; SSSE3-FAST-NEXT: addps %xmm1, %xmm4 +; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm2 = xmm2[3,3,3,3] +; SSSE3-FAST-NEXT: addps %xmm4, %xmm2 +; SSSE3-FAST-NEXT: movaps %xmm3, %xmm1 +; SSSE3-FAST-NEXT: haddps %xmm3, %xmm1 +; SSSE3-FAST-NEXT: movaps %xmm3, %xmm4 +; SSSE3-FAST-NEXT: unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm3[1] +; SSSE3-FAST-NEXT: addps %xmm1, %xmm4 +; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,3,3,3] +; SSSE3-FAST-NEXT: addps %xmm4, %xmm3 +; SSSE3-FAST-NEXT: movlhps {{.*#+}} xmm3 = xmm3[0],xmm2[0] +; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,0] +; SSSE3-FAST-NEXT: retq +; +; AVX-SLOW-LABEL: sequential_sum_v4f32_v4f32: +; AVX-SLOW: # %bb.0: +; AVX-SLOW-NEXT: vhaddps %xmm1, %xmm0, %xmm4 +; AVX-SLOW-NEXT: vpermilps {{.*#+}} xmm4 = xmm4[0,2,2,3] +; AVX-SLOW-NEXT: vunpckhps {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; AVX-SLOW-NEXT: vaddps %xmm4, %xmm5, %xmm4 +; AVX-SLOW-NEXT: vpermilpd {{.*#+}} xmm1 = xmm1[1,0] +; AVX-SLOW-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[3],xmm1[1],zero,zero +; AVX-SLOW-NEXT: vaddps %xmm4, %xmm0, %xmm0 +; AVX-SLOW-NEXT: vmovshdup {{.*#+}} xmm1 = xmm2[1,1,3,3] +; AVX-SLOW-NEXT: vaddps %xmm2, %xmm1, %xmm1 +; AVX-SLOW-NEXT: vpermilpd {{.*#+}} xmm4 = xmm2[1,0] +; AVX-SLOW-NEXT: vaddps %xmm1, %xmm4, %xmm1 +; AVX-SLOW-NEXT: vpermilps {{.*#+}} xmm2 = xmm2[3,3,3,3] +; AVX-SLOW-NEXT: vaddps %xmm1, %xmm2, %xmm1 +; AVX-SLOW-NEXT: vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; AVX-SLOW-NEXT: vmovshdup {{.*#+}} xmm1 = xmm3[1,1,3,3] +; AVX-SLOW-NEXT: vaddps %xmm3, %xmm1, %xmm1 +; AVX-SLOW-NEXT: vpermilpd {{.*#+}} xmm2 = xmm3[1,0] +; AVX-SLOW-NEXT: vaddps %xmm1, %xmm2, %xmm1 +; AVX-SLOW-NEXT: vpermilps {{.*#+}} xmm2 = xmm3[3,3,3,3] +; AVX-SLOW-NEXT: vaddps %xmm1, %xmm2, %xmm1 +; AVX-SLOW-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0] +; AVX-SLOW-NEXT: retq +; +; AVX-FAST-LABEL: sequential_sum_v4f32_v4f32: +; AVX-FAST: # %bb.0: +; AVX-FAST-NEXT: vhaddps %xmm1, %xmm0, %xmm4 +; AVX-FAST-NEXT: vpermilps {{.*#+}} xmm4 = xmm4[0,2,2,3] +; AVX-FAST-NEXT: vunpckhps {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; AVX-FAST-NEXT: vaddps %xmm4, %xmm5, %xmm4 +; AVX-FAST-NEXT: vpermilpd {{.*#+}} xmm1 = xmm1[1,0] +; AVX-FAST-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[3],xmm1[1],zero,zero +; AVX-FAST-NEXT: vaddps %xmm4, %xmm0, %xmm0 +; AVX-FAST-NEXT: vhaddps %xmm2, %xmm2, %xmm1 +; AVX-FAST-NEXT: vpermilpd {{.*#+}} xmm4 = xmm2[1,0] +; AVX-FAST-NEXT: vaddps %xmm1, %xmm4, %xmm1 +; AVX-FAST-NEXT: vpermilps {{.*#+}} xmm2 = xmm2[3,3,3,3] +; AVX-FAST-NEXT: vaddps %xmm1, %xmm2, %xmm1 +; AVX-FAST-NEXT: vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; AVX-FAST-NEXT: vhaddps %xmm3, %xmm3, %xmm1 +; AVX-FAST-NEXT: vpermilpd {{.*#+}} xmm2 = xmm3[1,0] +; AVX-FAST-NEXT: vaddps %xmm1, %xmm2, %xmm1 +; AVX-FAST-NEXT: vpermilps {{.*#+}} xmm2 = xmm3[3,3,3,3] +; AVX-FAST-NEXT: vaddps %xmm1, %xmm2, %xmm1 +; AVX-FAST-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0] +; AVX-FAST-NEXT: retq + %5 = shufflevector <4 x float> %0, <4 x float> %1, <2 x i32> <i32 0, i32 4> + %6 = shufflevector <4 x float> %0, <4 x float> %1, <2 x i32> <i32 1, i32 5> + %7 = fadd <2 x float> %5, %6 + %8 = shufflevector <4 x float> %0, <4 x float> %1, <2 x i32> <i32 2, i32 6> + %9 = fadd <2 x float> %8, %7 + %10 = shufflevector <4 x float> %0, <4 x float> %1, <2 x i32> <i32 3, i32 7> + %11 = fadd <2 x float> %10, %9 + %12 = shufflevector <2 x float> %11, <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef> + %13 = shufflevector <4 x float> %2, <4 x float> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef> + %14 = fadd <4 x float> %13, %2 + %15 = shufflevector <4 x float> %2, <4 x float> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef> + %16 = fadd <4 x float> %15, %14 + %17 = shufflevector <4 x float> %2, <4 x float> poison, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef> + %18 = fadd <4 x float> %17, %16 + %19 = shufflevector <4 x float> %12, <4 x float> %18, <4 x i32> <i32 0, i32 1, i32 4, i32 undef> + %20 = shufflevector <4 x float> %3, <4 x float> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef> + %21 = fadd <4 x float> %20, %3 + %22 = shufflevector <4 x float> %3, <4 x float> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef> + %23 = fadd <4 x float> %22, %21 + %24 = shufflevector <4 x float> %3, <4 x float> poison, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef> + %25 = fadd <4 x float> %24, %23 + %26 = shufflevector <4 x float> %19, <4 x float> %25, <4 x i32> <i32 0, i32 1, i32 2, i32 4> + ret <4 x float> %26 +} + +define <4 x i32> @sequential_sum_v4i32_v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2, <4 x i32> %3) { +; SSSE3-SLOW-LABEL: sequential_sum_v4i32_v4i32: +; SSSE3-SLOW: # %bb.0: +; SSSE3-SLOW-NEXT: movdqa %xmm0, %xmm4 +; SSSE3-SLOW-NEXT: phaddd %xmm1, %xmm4 +; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3] +; SSSE3-SLOW-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3] +; SSSE3-SLOW-NEXT: paddd %xmm4, %xmm1 +; SSSE3-SLOW-NEXT: paddd %xmm1, %xmm0 +; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm1 = xmm2[1,1,1,1] +; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm4 = xmm2[2,3,2,3] +; SSSE3-SLOW-NEXT: paddd %xmm2, %xmm4 +; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm2 = xmm2[3,3,3,3] +; SSSE3-SLOW-NEXT: paddd %xmm4, %xmm2 +; SSSE3-SLOW-NEXT: paddd %xmm1, %xmm2 +; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1] +; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm4 = xmm3[2,3,2,3] +; SSSE3-SLOW-NEXT: paddd %xmm3, %xmm4 +; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm3 = xmm3[3,3,3,3] +; SSSE3-SLOW-NEXT: paddd %xmm4, %xmm3 +; SSSE3-SLOW-NEXT: paddd %xmm1, %xmm3 +; SSSE3-SLOW-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm2[0] +; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,0] +; SSSE3-SLOW-NEXT: retq +; +; SSSE3-FAST-LABEL: sequential_sum_v4i32_v4i32: +; SSSE3-FAST: # %bb.0: +; SSSE3-FAST-NEXT: movdqa %xmm0, %xmm4 +; SSSE3-FAST-NEXT: phaddd %xmm1, %xmm4 +; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3] +; SSSE3-FAST-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3] +; SSSE3-FAST-NEXT: paddd %xmm4, %xmm1 +; SSSE3-FAST-NEXT: paddd %xmm1, %xmm0 +; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3] +; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm4 = xmm2[3,3,3,3] +; SSSE3-FAST-NEXT: phaddd %xmm2, %xmm2 +; SSSE3-FAST-NEXT: paddd %xmm2, %xmm4 +; SSSE3-FAST-NEXT: paddd %xmm1, %xmm4 +; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3] +; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm2 = xmm3[3,3,3,3] +; SSSE3-FAST-NEXT: phaddd %xmm3, %xmm3 +; SSSE3-FAST-NEXT: paddd %xmm3, %xmm2 +; SSSE3-FAST-NEXT: paddd %xmm1, %xmm2 +; SSSE3-FAST-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0] +; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,0] +; SSSE3-FAST-NEXT: retq +; +; AVX1-SLOW-LABEL: sequential_sum_v4i32_v4i32: +; AVX1-SLOW: # %bb.0: +; AVX1-SLOW-NEXT: vphaddd %xmm1, %xmm0, %xmm4 +; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,2,2,3] +; AVX1-SLOW-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3] +; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3] +; AVX1-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7] +; AVX1-SLOW-NEXT: vpaddd %xmm0, %xmm4, %xmm0 +; AVX1-SLOW-NEXT: vpaddd %xmm0, %xmm5, %xmm0 +; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[1,1,1,1] +; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[2,3,2,3] +; AVX1-SLOW-NEXT: vpaddd %xmm4, %xmm2, %xmm4 +; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3] +; AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm4, %xmm2 +; AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm1, %xmm1 +; AVX1-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm3[1,1,1,1] +; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[2,3,2,3] +; AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm3, %xmm2 +; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3] +; AVX1-SLOW-NEXT: vpaddd %xmm3, %xmm2, %xmm2 +; AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm1, %xmm1 +; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0] +; AVX1-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5],xmm1[6,7] +; AVX1-SLOW-NEXT: retq +; +; AVX1-FAST-LABEL: sequential_sum_v4i32_v4i32: +; AVX1-FAST: # %bb.0: +; AVX1-FAST-NEXT: vphaddd %xmm1, %xmm0, %xmm4 +; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,2,2,3] +; AVX1-FAST-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3] +; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3] +; AVX1-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7] +; AVX1-FAST-NEXT: vpaddd %xmm0, %xmm4, %xmm0 +; AVX1-FAST-NEXT: vpaddd %xmm0, %xmm5, %xmm0 +; AVX1-FAST-NEXT: vphaddd %xmm2, %xmm2, %xmm1 +; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[2,3,2,3] +; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3] +; AVX1-FAST-NEXT: vpaddd %xmm2, %xmm1, %xmm1 +; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm4, %xmm1 +; AVX1-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; AVX1-FAST-NEXT: vphaddd %xmm3, %xmm3, %xmm1 +; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[2,3,2,3] +; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3] +; AVX1-FAST-NEXT: vpaddd %xmm3, %xmm1, %xmm1 +; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm2, %xmm1 +; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0] +; AVX1-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5],xmm1[6,7] +; AVX1-FAST-NEXT: retq +; +; AVX2-SLOW-LABEL: sequential_sum_v4i32_v4i32: +; AVX2-SLOW: # %bb.0: +; AVX2-SLOW-NEXT: vphaddd %xmm1, %xmm0, %xmm4 +; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,2,2,3] +; AVX2-SLOW-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3] +; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3] +; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3] +; AVX2-SLOW-NEXT: vpaddd %xmm0, %xmm4, %xmm0 +; AVX2-SLOW-NEXT: vpaddd %xmm0, %xmm5, %xmm0 +; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[1,1,1,1] +; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[2,3,2,3] +; AVX2-SLOW-NEXT: vpaddd %xmm4, %xmm2, %xmm4 +; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3] +; AVX2-SLOW-NEXT: vpaddd %xmm2, %xmm4, %xmm2 +; AVX2-SLOW-NEXT: vpaddd %xmm2, %xmm1, %xmm1 +; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm3[1,1,1,1] +; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[2,3,2,3] +; AVX2-SLOW-NEXT: vpaddd %xmm2, %xmm3, %xmm2 +; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3] +; AVX2-SLOW-NEXT: vpaddd %xmm3, %xmm2, %xmm2 +; AVX2-SLOW-NEXT: vpaddd %xmm2, %xmm1, %xmm1 +; AVX2-SLOW-NEXT: vpbroadcastd %xmm1, %xmm1 +; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3] +; AVX2-SLOW-NEXT: retq +; +; AVX2-FAST-LABEL: sequential_sum_v4i32_v4i32: +; AVX2-FAST: # %bb.0: +; AVX2-FAST-NEXT: vphaddd %xmm1, %xmm0, %xmm4 +; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,2,2,3] +; AVX2-FAST-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3] +; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3] +; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3] +; AVX2-FAST-NEXT: vpaddd %xmm0, %xmm4, %xmm0 +; AVX2-FAST-NEXT: vpaddd %xmm0, %xmm5, %xmm0 +; AVX2-FAST-NEXT: vphaddd %xmm2, %xmm2, %xmm1 +; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[2,3,2,3] +; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3] +; AVX2-FAST-NEXT: vpaddd %xmm2, %xmm1, %xmm1 +; AVX2-FAST-NEXT: vpaddd %xmm1, %xmm4, %xmm1 +; AVX2-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; AVX2-FAST-NEXT: vphaddd %xmm3, %xmm3, %xmm1 +; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[2,3,2,3] +; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3] +; AVX2-FAST-NEXT: vpaddd %xmm3, %xmm1, %xmm1 +; AVX2-FAST-NEXT: vpaddd %xmm1, %xmm2, %xmm1 +; AVX2-FAST-NEXT: vpbroadcastd %xmm1, %xmm1 +; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3] +; AVX2-FAST-NEXT: retq + %5 = shufflevector <4 x i32> %0, <4 x i32> %1, <2 x i32> <i32 0, i32 4> + %6 = shufflevector <4 x i32> %0, <4 x i32> %1, <2 x i32> <i32 1, i32 5> + %7 = add <2 x i32> %5, %6 + %8 = shufflevector <4 x i32> %0, <4 x i32> %1, <2 x i32> <i32 2, i32 6> + %9 = add <2 x i32> %8, %7 + %10 = shufflevector <4 x i32> %0, <4 x i32> %1, <2 x i32> <i32 3, i32 7> + %11 = add <2 x i32> %10, %9 + %12 = shufflevector <2 x i32> %11, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef> + %13 = shufflevector <4 x i32> %2, <4 x i32> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef> + %14 = add <4 x i32> %13, %2 + %15 = shufflevector <4 x i32> %2, <4 x i32> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef> + %16 = add <4 x i32> %15, %14 + %17 = shufflevector <4 x i32> %2, <4 x i32> poison, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef> + %18 = add <4 x i32> %17, %16 + %19 = shufflevector <4 x i32> %12, <4 x i32> %18, <4 x i32> <i32 0, i32 1, i32 4, i32 undef> + %20 = shufflevector <4 x i32> %3, <4 x i32> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef> + %21 = add <4 x i32> %20, %3 + %22 = shufflevector <4 x i32> %3, <4 x i32> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef> + %23 = add <4 x i32> %22, %21 + %24 = shufflevector <4 x i32> %3, <4 x i32> poison, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef> + %25 = add <4 x i32> %24, %23 + %26 = shufflevector <4 x i32> %19, <4 x i32> %25, <4 x i32> <i32 0, i32 1, i32 2, i32 4> + ret <4 x i32> %26 +} + +; Vectorized Reductions +; e.g. +; VTYPE sum4(VTYPE A0, VTYPE A1, VTYPE A2, VTYPE A3) { +; return (VTYPE) { reduce( A0 ), reduce( A1 ), reduce( A2 ), reduce( A3 ) }; +; } + +define <4 x float> @reduction_sum_v4f32_v4f32(<4 x float> %0, <4 x float> %1, <4 x float> %2, <4 x float> %3) { +; SSSE3-LABEL: reduction_sum_v4f32_v4f32: +; SSSE3: # %bb.0: +; SSSE3-NEXT: movshdup {{.*#+}} xmm5 = xmm0[1,1,3,3] +; SSSE3-NEXT: movss {{.*#+}} xmm4 = mem[0],zero,zero,zero +; SSSE3-NEXT: addss %xmm4, %xmm5 +; SSSE3-NEXT: movaps %xmm0, %xmm6 +; SSSE3-NEXT: unpckhpd {{.*#+}} xmm6 = xmm6[1],xmm0[1] +; SSSE3-NEXT: addss %xmm5, %xmm6 +; SSSE3-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,3,3,3] +; SSSE3-NEXT: addss %xmm6, %xmm0 +; SSSE3-NEXT: movshdup {{.*#+}} xmm5 = xmm1[1,1,3,3] +; SSSE3-NEXT: addss %xmm4, %xmm5 +; SSSE3-NEXT: movaps %xmm1, %xmm6 +; SSSE3-NEXT: unpckhpd {{.*#+}} xmm6 = xmm6[1],xmm1[1] +; SSSE3-NEXT: addss %xmm5, %xmm6 +; SSSE3-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,3,3,3] +; SSSE3-NEXT: addss %xmm6, %xmm1 +; SSSE3-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSSE3-NEXT: movshdup {{.*#+}} xmm1 = xmm2[1,1,3,3] +; SSSE3-NEXT: addss %xmm4, %xmm1 +; SSSE3-NEXT: movaps %xmm2, %xmm5 +; SSSE3-NEXT: unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm2[1] +; SSSE3-NEXT: addss %xmm1, %xmm5 +; SSSE3-NEXT: shufps {{.*#+}} xmm2 = xmm2[3,3,3,3] +; SSSE3-NEXT: addss %xmm5, %xmm2 +; SSSE3-NEXT: movshdup {{.*#+}} xmm1 = xmm3[1,1,3,3] +; SSSE3-NEXT: addss %xmm4, %xmm1 +; SSSE3-NEXT: movaps %xmm3, %xmm4 +; SSSE3-NEXT: unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm3[1] +; SSSE3-NEXT: addss %xmm1, %xmm4 +; SSSE3-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,3,3,3] +; SSSE3-NEXT: addss %xmm4, %xmm3 +; SSSE3-NEXT: unpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1] +; SSSE3-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm2[0] +; SSSE3-NEXT: retq +; +; AVX-LABEL: reduction_sum_v4f32_v4f32: +; AVX: # %bb.0: +; AVX-NEXT: vmovshdup {{.*#+}} xmm4 = xmm0[1,1,3,3] +; AVX-NEXT: vmovss {{.*#+}} xmm5 = mem[0],zero,zero,zero +; AVX-NEXT: vaddss %xmm5, %xmm4, %xmm4 +; AVX-NEXT: vpermilpd {{.*#+}} xmm6 = xmm0[1,0] +; AVX-NEXT: vaddss %xmm6, %xmm4, %xmm4 +; AVX-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3] +; AVX-NEXT: vaddss %xmm0, %xmm4, %xmm0 +; AVX-NEXT: vmovshdup {{.*#+}} xmm4 = xmm1[1,1,3,3] +; AVX-NEXT: vaddss %xmm5, %xmm4, %xmm4 +; AVX-NEXT: vpermilpd {{.*#+}} xmm6 = xmm1[1,0] +; AVX-NEXT: vaddss %xmm6, %xmm4, %xmm4 +; AVX-NEXT: vpermilps {{.*#+}} xmm1 = xmm1[3,3,3,3] +; AVX-NEXT: vaddss %xmm1, %xmm4, %xmm1 +; AVX-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3] +; AVX-NEXT: vmovshdup {{.*#+}} xmm1 = xmm2[1,1,3,3] +; AVX-NEXT: vaddss %xmm5, %xmm1, %xmm1 +; AVX-NEXT: vpermilpd {{.*#+}} xmm4 = xmm2[1,0] +; AVX-NEXT: vaddss %xmm4, %xmm1, %xmm1 +; AVX-NEXT: vpermilps {{.*#+}} xmm2 = xmm2[3,3,3,3] +; AVX-NEXT: vaddss %xmm2, %xmm1, %xmm1 +; AVX-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0],xmm0[3] +; AVX-NEXT: vmovshdup {{.*#+}} xmm1 = xmm3[1,1,3,3] +; AVX-NEXT: vaddss %xmm5, %xmm1, %xmm1 +; AVX-NEXT: vpermilpd {{.*#+}} xmm2 = xmm3[1,0] +; AVX-NEXT: vaddss %xmm2, %xmm1, %xmm1 +; AVX-NEXT: vpermilps {{.*#+}} xmm2 = xmm3[3,3,3,3] +; AVX-NEXT: vaddss %xmm2, %xmm1, %xmm1 +; AVX-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0] +; AVX-NEXT: retq + %5 = call float @llvm.vector.reduce.fadd.f32.v4f32(float undef, <4 x float> %0) + %6 = call float @llvm.vector.reduce.fadd.f32.v4f32(float undef, <4 x float> %1) + %7 = call float @llvm.vector.reduce.fadd.f32.v4f32(float undef, <4 x float> %2) + %8 = call float @llvm.vector.reduce.fadd.f32.v4f32(float undef, <4 x float> %3) + %9 = insertelement <4 x float> undef, float %5, i32 0 + %10 = insertelement <4 x float> %9, float %6, i32 1 + %11 = insertelement <4 x float> %10, float %7, i32 2 + %12 = insertelement <4 x float> %11, float %8, i32 3 + ret <4 x float> %12 +} +declare float @llvm.vector.reduce.fadd.f32.v4f32(float, <4 x float>) + +define <4 x i32> @reduction_sum_v4i32_v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2, <4 x i32> %3) { +; SSSE3-SLOW-LABEL: reduction_sum_v4i32_v4i32: +; SSSE3-SLOW: # %bb.0: +; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3] +; SSSE3-SLOW-NEXT: paddd %xmm0, %xmm4 +; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,1,1] +; SSSE3-SLOW-NEXT: paddd %xmm4, %xmm0 +; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3] +; SSSE3-SLOW-NEXT: paddd %xmm1, %xmm4 +; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm1 = xmm4[1,1,1,1] +; SSSE3-SLOW-NEXT: paddd %xmm4, %xmm1 +; SSSE3-SLOW-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3] +; SSSE3-SLOW-NEXT: paddd %xmm2, %xmm1 +; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,1,1] +; SSSE3-SLOW-NEXT: paddd %xmm1, %xmm2 +; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3] +; SSSE3-SLOW-NEXT: paddd %xmm3, %xmm1 +; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,1,1,1] +; SSSE3-SLOW-NEXT: paddd %xmm1, %xmm3 +; SSSE3-SLOW-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1] +; SSSE3-SLOW-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0] +; SSSE3-SLOW-NEXT: retq +; +; SSSE3-FAST-LABEL: reduction_sum_v4i32_v4i32: +; SSSE3-FAST: # %bb.0: +; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3] +; SSSE3-FAST-NEXT: paddd %xmm4, %xmm0 +; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3] +; SSSE3-FAST-NEXT: paddd %xmm1, %xmm4 +; SSSE3-FAST-NEXT: phaddd %xmm4, %xmm0 +; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3] +; SSSE3-FAST-NEXT: paddd %xmm2, %xmm1 +; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3] +; SSSE3-FAST-NEXT: paddd %xmm3, %xmm2 +; SSSE3-FAST-NEXT: phaddd %xmm2, %xmm1 +; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2] +; SSSE3-FAST-NEXT: retq +; +; AVX-SLOW-LABEL: reduction_sum_v4i32_v4i32: +; AVX-SLOW: # %bb.0: +; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3] +; AVX-SLOW-NEXT: vpaddd %xmm4, %xmm0, %xmm0 +; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1] +; AVX-SLOW-NEXT: vpaddd %xmm4, %xmm0, %xmm0 +; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[2,3,2,3] +; AVX-SLOW-NEXT: vpaddd %xmm4, %xmm1, %xmm1 +; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[1,1,1,1] +; AVX-SLOW-NEXT: vpaddd %xmm4, %xmm1, %xmm1 +; AVX-SLOW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[2,3,2,3] +; AVX-SLOW-NEXT: vpaddd %xmm1, %xmm2, %xmm1 +; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1] +; AVX-SLOW-NEXT: vpaddd %xmm2, %xmm1, %xmm1 +; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[2,3,2,3] +; AVX-SLOW-NEXT: vpaddd %xmm2, %xmm3, %xmm2 +; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[1,1,1,1] +; AVX-SLOW-NEXT: vpaddd %xmm3, %xmm2, %xmm2 +; AVX-SLOW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] +; AVX-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; AVX-SLOW-NEXT: retq +; +; AVX-FAST-LABEL: reduction_sum_v4i32_v4i32: +; AVX-FAST: # %bb.0: +; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3] +; AVX-FAST-NEXT: vpaddd %xmm4, %xmm0, %xmm0 +; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[2,3,2,3] +; AVX-FAST-NEXT: vpaddd %xmm4, %xmm1, %xmm1 +; AVX-FAST-NEXT: vphaddd %xmm1, %xmm0, %xmm0 +; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3] +; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[2,3,2,3] +; AVX-FAST-NEXT: vpaddd %xmm1, %xmm2, %xmm1 +; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[2,3,2,3] +; AVX-FAST-NEXT: vpaddd %xmm2, %xmm3, %xmm2 +; AVX-FAST-NEXT: vphaddd %xmm2, %xmm1, %xmm1 +; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,2,1,3] +; AVX-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; AVX-FAST-NEXT: retq + %5 = call i32 @llvm.vector.reduce.add.i32.v4i32(<4 x i32> %0) + %6 = call i32 @llvm.vector.reduce.add.i32.v4i32(<4 x i32> %1) + %7 = call i32 @llvm.vector.reduce.add.i32.v4i32(<4 x i32> %2) + %8 = call i32 @llvm.vector.reduce.add.i32.v4i32(<4 x i32> %3) + %9 = insertelement <4 x i32> undef, i32 %5, i32 0 + %10 = insertelement <4 x i32> %9, i32 %6, i32 1 + %11 = insertelement <4 x i32> %10, i32 %7, i32 2 + %12 = insertelement <4 x i32> %11, i32 %8, i32 3 + ret <4 x i32> %12 +} +declare i32 @llvm.vector.reduce.add.i32.v4i32(<4 x i32>) _______________________________________________ llvm-branch-commits mailing list llvm-branch-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits