Author: Oscar Priego Date: 2026-08-26T11:01:03+02:00 New Revision: f1cdfbeb84e95d83d9952a22ccd7069e4ecab23a
URL: https://github.com/llvm/llvm-project/commit/f1cdfbeb84e95d83d9952a22ccd7069e4ecab23a DIFF: https://github.com/llvm/llvm-project/commit/f1cdfbeb84e95d83d9952a22ccd7069e4ecab23a.diff LOG: [X86] Fix NaN handling in minimumnum/maximumnum zero fixup (#217420) Fix the X86 lowering of `minimumnum`/`maximumnum` when the first operand is NaN and signed-zero handling is required. `PSEUDO_FMIN`/`PSEUDO_FMAX` already select the numeric second operand when the first operand is NaN. However, the subsequent signed-zero fixup can modify that numeric result using the sign bit of the first operand. For example, `minimumnum(-qNaN, +1.0)` can therefore turn the correctly selected `+1.0` into `-1.0`. Restore the second operand when the first operand is NaN after the signed-zero fixup. The additional check is restricted to numeric min/max operations where NaNs cannot be ignored and the first operand is not already known to be non-NaN. The existing handling for a NaN second operand remains unchanged. This also fixes the symmetric `maximumnum(+qNaN, -1.0)` case and applies to signaling NaNs as well. Fast paths for `nnan`, `nsz`, known non-NaN operands, and native AVX10.2 `VMINMAX*` lowering remain unchanged. Fixes #217376 AI-assisted development tools were used during this contribution. All generated output was reviewed and modified by the author. The final patch, tests, and validation steps were determined and verified by the author. (cherry picked from commit 4a1c14678024d9b424cbca33774afdccdb14019e) Added: Modified: llvm/lib/Target/X86/X86ISelLowering.cpp llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll Removed: ################################################################################ diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 9452d18018db6..1183ad64f1554 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -30194,6 +30194,13 @@ static SDValue LowerFMINIMUM_FMAXIMUM(SDValue Op, const X86Subtarget &Subtarget, DAG.getVectorIdxConstant(0, DL)); else MinMax = Result; + + // The signed-zero fixup may corrupt the numeric NewY result with the sign + // bit of a NaN NewX. Restore NewY in that case. + if (IsNum && !IgnoreNaN && !IsXNeverNaN) { + SDValue IsXNaN = DAG.getSetCC(DL, SetCCType, NewX, NewX, ISD::SETUO); + MinMax = DAG.getSelect(DL, VT, IsXNaN, NewY, MinMax); + } } if (IgnoreNaN || DAG.isKnownNeverNaN(IsNum ? NewY : NewX)) diff --git a/llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll b/llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll index 8abd68701676c..19d7832ea6991 100644 --- a/llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll +++ b/llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll @@ -27,16 +27,21 @@ declare <4 x bfloat> @llvm.maximumnum.v4bf16(<4 x bfloat>, <4 x bfloat>) define float @test_fmaximumnum(float %x, float %y) nounwind { ; SSE2-LABEL: test_fmaximumnum: ; SSE2: # %bb.0: -; SSE2-NEXT: movaps %xmm0, %xmm3 -; SSE2-NEXT: maxss %xmm1, %xmm3 +; SSE2-NEXT: movaps %xmm0, %xmm2 +; SSE2-NEXT: cmpunordss %xmm0, %xmm2 +; SSE2-NEXT: movaps %xmm2, %xmm3 +; SSE2-NEXT: andps %xmm1, %xmm3 +; SSE2-NEXT: movaps %xmm0, %xmm4 +; SSE2-NEXT: maxss %xmm1, %xmm4 +; SSE2-NEXT: movaps {{.*#+}} xmm5 = [NaN,NaN,NaN,NaN] +; SSE2-NEXT: orps %xmm0, %xmm5 +; SSE2-NEXT: andps %xmm4, %xmm5 +; SSE2-NEXT: andnps %xmm5, %xmm2 +; SSE2-NEXT: orps %xmm3, %xmm2 ; SSE2-NEXT: cmpunordss %xmm1, %xmm1 -; SSE2-NEXT: movaps %xmm1, %xmm2 -; SSE2-NEXT: andps %xmm0, %xmm2 -; SSE2-NEXT: orps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; SSE2-NEXT: andps %xmm3, %xmm0 -; SSE2-NEXT: andnps %xmm0, %xmm1 -; SSE2-NEXT: orps %xmm1, %xmm2 -; SSE2-NEXT: movaps %xmm2, %xmm0 +; SSE2-NEXT: andps %xmm1, %xmm0 +; SSE2-NEXT: andnps %xmm2, %xmm1 +; SSE2-NEXT: orps %xmm1, %xmm0 ; SSE2-NEXT: retq ; ; AVX1-LABEL: test_fmaximumnum: @@ -44,6 +49,8 @@ define float @test_fmaximumnum(float %x, float %y) nounwind { ; AVX1-NEXT: vmaxss %xmm1, %xmm0, %xmm2 ; AVX1-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm3 ; AVX1-NEXT: vandps %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm3 +; AVX1-NEXT: vblendvps %xmm3, %xmm1, %xmm2, %xmm2 ; AVX1-NEXT: vcmpunordss %xmm1, %xmm1, %xmm1 ; AVX1-NEXT: vblendvps %xmm1, %xmm0, %xmm2, %xmm0 ; AVX1-NEXT: retq @@ -54,6 +61,8 @@ define float @test_fmaximumnum(float %x, float %y) nounwind { ; AVX512F-NEXT: vorps %xmm2, %xmm0, %xmm2 ; AVX512F-NEXT: vmaxss %xmm1, %xmm0, %xmm3 ; AVX512F-NEXT: vandps %xmm3, %xmm2, %xmm2 +; AVX512F-NEXT: vcmpunordss %xmm0, %xmm0, %k1 +; AVX512F-NEXT: vmovss %xmm1, %xmm2, %xmm2 {%k1} ; AVX512F-NEXT: vcmpunordss %xmm1, %xmm1, %k1 ; AVX512F-NEXT: vmovss %xmm0, %xmm2, %xmm2 {%k1} ; AVX512F-NEXT: vmovaps %xmm2, %xmm0 @@ -65,6 +74,8 @@ define float @test_fmaximumnum(float %x, float %y) nounwind { ; AVX512DQ-NEXT: vorps %xmm2, %xmm0, %xmm2 ; AVX512DQ-NEXT: vmaxss %xmm1, %xmm0, %xmm3 ; AVX512DQ-NEXT: vandps %xmm3, %xmm2, %xmm2 +; AVX512DQ-NEXT: vcmpunordss %xmm0, %xmm0, %k1 +; AVX512DQ-NEXT: vmovss %xmm1, %xmm2, %xmm2 {%k1} ; AVX512DQ-NEXT: vcmpunordss %xmm1, %xmm1, %k1 ; AVX512DQ-NEXT: vmovss %xmm0, %xmm2, %xmm2 {%k1} ; AVX512DQ-NEXT: vmovaps %xmm2, %xmm0 @@ -74,6 +85,8 @@ define float @test_fmaximumnum(float %x, float %y) nounwind { ; AVX512BF16: # %bb.0: ; AVX512BF16-NEXT: vmaxss %xmm1, %xmm0, %xmm2 ; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm2 = xmm2 & (xmm0 | m32bcst) +; AVX512BF16-NEXT: vcmpunordss %xmm0, %xmm0, %k1 +; AVX512BF16-NEXT: vmovss %xmm1, %xmm2, %xmm2 {%k1} ; AVX512BF16-NEXT: vcmpunordss %xmm1, %xmm1, %k1 ; AVX512BF16-NEXT: vmovss %xmm0, %xmm2, %xmm2 {%k1} ; AVX512BF16-NEXT: vmovaps %xmm2, %xmm0 @@ -92,6 +105,8 @@ define float @test_fmaximumnum(float %x, float %y) nounwind { ; X86-NEXT: vmaxss %xmm0, %xmm1, %xmm2 ; X86-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1, %xmm3 ; X86-NEXT: vandps %xmm2, %xmm3, %xmm2 +; X86-NEXT: vcmpunordss %xmm1, %xmm1, %xmm3 +; X86-NEXT: vblendvps %xmm3, %xmm0, %xmm2, %xmm2 ; X86-NEXT: vcmpunordss %xmm0, %xmm0, %xmm0 ; X86-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0 ; X86-NEXT: vmovss %xmm0, (%esp) @@ -102,6 +117,103 @@ define float @test_fmaximumnum(float %x, float %y) nounwind { ret float %1 } +; Keep both operands unknown so lowering must handle a NaN in the first operand +; after the signed-zero fixup. +define double @test_fmaximumnum_f64(double %x, double %y) nounwind { +; SSE2-LABEL: test_fmaximumnum_f64: +; SSE2: # %bb.0: +; SSE2-NEXT: movapd %xmm0, %xmm2 +; SSE2-NEXT: cmpunordsd %xmm0, %xmm2 +; SSE2-NEXT: movapd %xmm2, %xmm3 +; SSE2-NEXT: andpd %xmm1, %xmm3 +; SSE2-NEXT: movapd %xmm0, %xmm4 +; SSE2-NEXT: maxsd %xmm1, %xmm4 +; SSE2-NEXT: movapd {{.*#+}} xmm5 = [NaN,NaN] +; SSE2-NEXT: orpd %xmm0, %xmm5 +; SSE2-NEXT: andpd %xmm4, %xmm5 +; SSE2-NEXT: andnpd %xmm5, %xmm2 +; SSE2-NEXT: orpd %xmm3, %xmm2 +; SSE2-NEXT: cmpunordsd %xmm1, %xmm1 +; SSE2-NEXT: andpd %xmm1, %xmm0 +; SSE2-NEXT: andnpd %xmm2, %xmm1 +; SSE2-NEXT: orpd %xmm1, %xmm0 +; SSE2-NEXT: retq +; +; AVX1-LABEL: test_fmaximumnum_f64: +; AVX1: # %bb.0: +; AVX1-NEXT: vmaxsd %xmm1, %xmm0, %xmm2 +; AVX1-NEXT: vorpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm3 +; AVX1-NEXT: vandpd %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vcmpunordsd %xmm0, %xmm0, %xmm3 +; AVX1-NEXT: vblendvpd %xmm3, %xmm1, %xmm2, %xmm2 +; AVX1-NEXT: vcmpunordsd %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vblendvpd %xmm1, %xmm0, %xmm2, %xmm0 +; AVX1-NEXT: retq +; +; AVX512F-LABEL: test_fmaximumnum_f64: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vmaxsd %xmm1, %xmm0, %xmm2 +; AVX512F-NEXT: vorpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm3 +; AVX512F-NEXT: vandpd %xmm2, %xmm3, %xmm2 +; AVX512F-NEXT: vcmpunordsd %xmm0, %xmm0, %k1 +; AVX512F-NEXT: vmovsd %xmm1, %xmm2, %xmm2 {%k1} +; AVX512F-NEXT: vcmpunordsd %xmm1, %xmm1, %k1 +; AVX512F-NEXT: vmovsd %xmm0, %xmm2, %xmm2 {%k1} +; AVX512F-NEXT: vmovapd %xmm2, %xmm0 +; AVX512F-NEXT: retq +; +; AVX512DQ-LABEL: test_fmaximumnum_f64: +; AVX512DQ: # %bb.0: +; AVX512DQ-NEXT: vmaxsd %xmm1, %xmm0, %xmm2 +; AVX512DQ-NEXT: vorpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm3 +; AVX512DQ-NEXT: vandpd %xmm2, %xmm3, %xmm2 +; AVX512DQ-NEXT: vcmpunordsd %xmm0, %xmm0, %k1 +; AVX512DQ-NEXT: vmovsd %xmm1, %xmm2, %xmm2 {%k1} +; AVX512DQ-NEXT: vcmpunordsd %xmm1, %xmm1, %k1 +; AVX512DQ-NEXT: vmovsd %xmm0, %xmm2, %xmm2 {%k1} +; AVX512DQ-NEXT: vmovapd %xmm2, %xmm0 +; AVX512DQ-NEXT: retq +; +; AVX512BF16-LABEL: test_fmaximumnum_f64: +; AVX512BF16: # %bb.0: +; AVX512BF16-NEXT: vmaxsd %xmm1, %xmm0, %xmm2 +; AVX512BF16-NEXT: vpternlogq {{.*#+}} xmm2 = xmm2 & (xmm0 | m64bcst) +; AVX512BF16-NEXT: vcmpunordsd %xmm0, %xmm0, %k1 +; AVX512BF16-NEXT: vmovsd %xmm1, %xmm2, %xmm2 {%k1} +; AVX512BF16-NEXT: vcmpunordsd %xmm1, %xmm1, %k1 +; AVX512BF16-NEXT: vmovsd %xmm0, %xmm2, %xmm2 {%k1} +; AVX512BF16-NEXT: vmovapd %xmm2, %xmm0 +; AVX512BF16-NEXT: retq +; +; AVX10_2-LABEL: test_fmaximumnum_f64: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vminmaxsd $21, %xmm1, %xmm0 +; AVX10_2-NEXT: retq +; +; X86-LABEL: test_fmaximumnum_f64: +; X86: # %bb.0: +; X86-NEXT: pushl %ebp +; X86-NEXT: movl %esp, %ebp +; X86-NEXT: andl $-8, %esp +; X86-NEXT: subl $8, %esp +; X86-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero +; X86-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero +; X86-NEXT: vmaxsd %xmm0, %xmm1, %xmm2 +; X86-NEXT: vorpd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1, %xmm3 +; X86-NEXT: vandpd %xmm2, %xmm3, %xmm2 +; X86-NEXT: vcmpunordsd %xmm1, %xmm1, %xmm3 +; X86-NEXT: vblendvpd %xmm3, %xmm0, %xmm2, %xmm2 +; X86-NEXT: vcmpunordsd %xmm0, %xmm0, %xmm0 +; X86-NEXT: vblendvpd %xmm0, %xmm1, %xmm2, %xmm0 +; X86-NEXT: vmovlpd %xmm0, (%esp) +; X86-NEXT: fldl (%esp) +; X86-NEXT: movl %ebp, %esp +; X86-NEXT: popl %ebp +; X86-NEXT: retl + %r = call double @llvm.maximumnum.f64(double %x, double %y) + ret double %r +} + define <4 x float> @test_fmaximumnum_scalarize(<4 x float> %x, <4 x float> %y) { ; SSE2-LABEL: test_fmaximumnum_scalarize: ; SSE2: # %bb.0: @@ -403,26 +515,38 @@ define float @test_fmaximumnum_combine_cmps(float %x, float %y) nounwind { ; SSE2: # %bb.0: ; SSE2-NEXT: divss %xmm0, %xmm1 ; SSE2-NEXT: movaps %xmm0, %xmm2 -; SSE2-NEXT: maxss %xmm1, %xmm2 +; SSE2-NEXT: cmpunordss %xmm0, %xmm2 +; SSE2-NEXT: movaps %xmm2, %xmm3 +; SSE2-NEXT: andps %xmm1, %xmm3 +; SSE2-NEXT: movaps %xmm0, %xmm4 +; SSE2-NEXT: maxss %xmm1, %xmm4 ; SSE2-NEXT: orps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; SSE2-NEXT: andps %xmm2, %xmm0 +; SSE2-NEXT: andps %xmm4, %xmm0 +; SSE2-NEXT: andnps %xmm0, %xmm2 +; SSE2-NEXT: orps %xmm3, %xmm2 +; SSE2-NEXT: movaps %xmm2, %xmm0 ; SSE2-NEXT: retq ; ; AVX1-LABEL: test_fmaximumnum_combine_cmps: ; AVX1: # %bb.0: ; AVX1-NEXT: vdivss %xmm0, %xmm1, %xmm1 -; AVX1-NEXT: vmaxss %xmm1, %xmm0, %xmm1 -; AVX1-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 -; AVX1-NEXT: vandps %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vmaxss %xmm1, %xmm0, %xmm2 +; AVX1-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm3 +; AVX1-NEXT: vandps %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm0 +; AVX1-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0 ; AVX1-NEXT: retq ; ; AVX512F-LABEL: test_fmaximumnum_combine_cmps: ; AVX512F: # %bb.0: -; AVX512F-NEXT: vdivss %xmm0, %xmm1, %xmm1 -; AVX512F-NEXT: vbroadcastss {{.*#+}} xmm2 = [NaN,NaN,NaN,NaN] -; AVX512F-NEXT: vorps %xmm2, %xmm0, %xmm2 -; AVX512F-NEXT: vmaxss %xmm1, %xmm0, %xmm0 -; AVX512F-NEXT: vandps %xmm0, %xmm2, %xmm0 +; AVX512F-NEXT: vdivss %xmm0, %xmm1, %xmm2 +; AVX512F-NEXT: vbroadcastss {{.*#+}} xmm1 = [NaN,NaN,NaN,NaN] +; AVX512F-NEXT: vorps %xmm1, %xmm0, %xmm1 +; AVX512F-NEXT: vmaxss %xmm2, %xmm0, %xmm3 +; AVX512F-NEXT: vandps %xmm3, %xmm1, %xmm1 +; AVX512F-NEXT: vcmpunordss %xmm0, %xmm0, %k1 +; AVX512F-NEXT: vmovss %xmm2, %xmm1, %xmm1 {%k1} +; AVX512F-NEXT: vmovaps %xmm1, %xmm0 ; AVX512F-NEXT: retq ; ; AVX512DQ-LABEL: test_fmaximumnum_combine_cmps: @@ -438,9 +562,12 @@ define float @test_fmaximumnum_combine_cmps(float %x, float %y) nounwind { ; ; AVX512BF16-LABEL: test_fmaximumnum_combine_cmps: ; AVX512BF16: # %bb.0: -; AVX512BF16-NEXT: vdivss %xmm0, %xmm1, %xmm1 -; AVX512BF16-NEXT: vmaxss %xmm1, %xmm0, %xmm1 -; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm0 = xmm1 & (xmm0 | m32bcst) +; AVX512BF16-NEXT: vdivss %xmm0, %xmm1, %xmm2 +; AVX512BF16-NEXT: vmaxss %xmm2, %xmm0, %xmm1 +; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm1 = xmm1 & (xmm0 | m32bcst) +; AVX512BF16-NEXT: vcmpunordss %xmm0, %xmm0, %k1 +; AVX512BF16-NEXT: vmovss %xmm2, %xmm1, %xmm1 {%k1} +; AVX512BF16-NEXT: vmovaps %xmm1, %xmm0 ; AVX512BF16-NEXT: retq ; ; AVX10_2-LABEL: test_fmaximumnum_combine_cmps: @@ -455,9 +582,11 @@ define float @test_fmaximumnum_combine_cmps(float %x, float %y) nounwind { ; X86-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero ; X86-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero ; X86-NEXT: vdivss %xmm0, %xmm1, %xmm1 -; X86-NEXT: vmaxss %xmm1, %xmm0, %xmm1 -; X86-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0 -; X86-NEXT: vandps %xmm1, %xmm0, %xmm0 +; X86-NEXT: vmaxss %xmm1, %xmm0, %xmm2 +; X86-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm3 +; X86-NEXT: vandps %xmm2, %xmm3, %xmm2 +; X86-NEXT: vcmpunordss %xmm0, %xmm0, %xmm0 +; X86-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0 ; X86-NEXT: vmovss %xmm0, (%esp) ; X86-NEXT: flds (%esp) ; X86-NEXT: popl %eax @@ -474,16 +603,21 @@ define float @test_fmaximumnum_combine_cmps(float %x, float %y) nounwind { define float @test_fminimumnum(float %x, float %y) nounwind { ; SSE2-LABEL: test_fminimumnum: ; SSE2: # %bb.0: -; SSE2-NEXT: movaps %xmm0, %xmm3 -; SSE2-NEXT: minss %xmm1, %xmm3 +; SSE2-NEXT: movaps %xmm0, %xmm2 +; SSE2-NEXT: cmpunordss %xmm0, %xmm2 +; SSE2-NEXT: movaps %xmm2, %xmm3 +; SSE2-NEXT: andps %xmm1, %xmm3 +; SSE2-NEXT: movaps %xmm0, %xmm4 +; SSE2-NEXT: minss %xmm1, %xmm4 +; SSE2-NEXT: movaps {{.*#+}} xmm5 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0] +; SSE2-NEXT: andps %xmm0, %xmm5 +; SSE2-NEXT: orps %xmm4, %xmm5 +; SSE2-NEXT: andnps %xmm5, %xmm2 +; SSE2-NEXT: orps %xmm3, %xmm2 ; SSE2-NEXT: cmpunordss %xmm1, %xmm1 -; SSE2-NEXT: movaps %xmm1, %xmm2 -; SSE2-NEXT: andps %xmm0, %xmm2 -; SSE2-NEXT: andps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; SSE2-NEXT: orps %xmm3, %xmm0 -; SSE2-NEXT: andnps %xmm0, %xmm1 -; SSE2-NEXT: orps %xmm1, %xmm2 -; SSE2-NEXT: movaps %xmm2, %xmm0 +; SSE2-NEXT: andps %xmm1, %xmm0 +; SSE2-NEXT: andnps %xmm2, %xmm1 +; SSE2-NEXT: orps %xmm1, %xmm0 ; SSE2-NEXT: retq ; ; AVX1-LABEL: test_fminimumnum: @@ -491,6 +625,8 @@ define float @test_fminimumnum(float %x, float %y) nounwind { ; AVX1-NEXT: vminss %xmm1, %xmm0, %xmm2 ; AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm3 ; AVX1-NEXT: vorps %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm3 +; AVX1-NEXT: vblendvps %xmm3, %xmm1, %xmm2, %xmm2 ; AVX1-NEXT: vcmpunordss %xmm1, %xmm1, %xmm1 ; AVX1-NEXT: vblendvps %xmm1, %xmm0, %xmm2, %xmm0 ; AVX1-NEXT: retq @@ -501,6 +637,8 @@ define float @test_fminimumnum(float %x, float %y) nounwind { ; AVX512F-NEXT: vandps %xmm2, %xmm0, %xmm2 ; AVX512F-NEXT: vminss %xmm1, %xmm0, %xmm3 ; AVX512F-NEXT: vorps %xmm3, %xmm2, %xmm2 +; AVX512F-NEXT: vcmpunordss %xmm0, %xmm0, %k1 +; AVX512F-NEXT: vmovss %xmm1, %xmm2, %xmm2 {%k1} ; AVX512F-NEXT: vcmpunordss %xmm1, %xmm1, %k1 ; AVX512F-NEXT: vmovss %xmm0, %xmm2, %xmm2 {%k1} ; AVX512F-NEXT: vmovaps %xmm2, %xmm0 @@ -512,6 +650,8 @@ define float @test_fminimumnum(float %x, float %y) nounwind { ; AVX512DQ-NEXT: vandps %xmm2, %xmm0, %xmm2 ; AVX512DQ-NEXT: vminss %xmm1, %xmm0, %xmm3 ; AVX512DQ-NEXT: vorps %xmm3, %xmm2, %xmm2 +; AVX512DQ-NEXT: vcmpunordss %xmm0, %xmm0, %k1 +; AVX512DQ-NEXT: vmovss %xmm1, %xmm2, %xmm2 {%k1} ; AVX512DQ-NEXT: vcmpunordss %xmm1, %xmm1, %k1 ; AVX512DQ-NEXT: vmovss %xmm0, %xmm2, %xmm2 {%k1} ; AVX512DQ-NEXT: vmovaps %xmm2, %xmm0 @@ -521,6 +661,8 @@ define float @test_fminimumnum(float %x, float %y) nounwind { ; AVX512BF16: # %bb.0: ; AVX512BF16-NEXT: vminss %xmm1, %xmm0, %xmm2 ; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm2 = xmm2 | (xmm0 & m32bcst) +; AVX512BF16-NEXT: vcmpunordss %xmm0, %xmm0, %k1 +; AVX512BF16-NEXT: vmovss %xmm1, %xmm2, %xmm2 {%k1} ; AVX512BF16-NEXT: vcmpunordss %xmm1, %xmm1, %k1 ; AVX512BF16-NEXT: vmovss %xmm0, %xmm2, %xmm2 {%k1} ; AVX512BF16-NEXT: vmovaps %xmm2, %xmm0 @@ -539,6 +681,8 @@ define float @test_fminimumnum(float %x, float %y) nounwind { ; X86-NEXT: vminss %xmm0, %xmm1, %xmm2 ; X86-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1, %xmm3 ; X86-NEXT: vorps %xmm2, %xmm3, %xmm2 +; X86-NEXT: vcmpunordss %xmm1, %xmm1, %xmm3 +; X86-NEXT: vblendvps %xmm3, %xmm0, %xmm2, %xmm2 ; X86-NEXT: vcmpunordss %xmm0, %xmm0, %xmm0 ; X86-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0 ; X86-NEXT: vmovss %xmm0, (%esp) @@ -549,6 +693,103 @@ define float @test_fminimumnum(float %x, float %y) nounwind { ret float %1 } +; Keep both operands unknown so lowering must handle a NaN in the first operand +; after the signed-zero fixup. +define double @test_fminimumnum_f64(double %x, double %y) nounwind { +; SSE2-LABEL: test_fminimumnum_f64: +; SSE2: # %bb.0: +; SSE2-NEXT: movapd %xmm0, %xmm2 +; SSE2-NEXT: cmpunordsd %xmm0, %xmm2 +; SSE2-NEXT: movapd %xmm2, %xmm3 +; SSE2-NEXT: andpd %xmm1, %xmm3 +; SSE2-NEXT: movapd %xmm0, %xmm4 +; SSE2-NEXT: minsd %xmm1, %xmm4 +; SSE2-NEXT: movapd {{.*#+}} xmm5 = [-0.0E+0,-0.0E+0] +; SSE2-NEXT: andpd %xmm0, %xmm5 +; SSE2-NEXT: orpd %xmm4, %xmm5 +; SSE2-NEXT: andnpd %xmm5, %xmm2 +; SSE2-NEXT: orpd %xmm3, %xmm2 +; SSE2-NEXT: cmpunordsd %xmm1, %xmm1 +; SSE2-NEXT: andpd %xmm1, %xmm0 +; SSE2-NEXT: andnpd %xmm2, %xmm1 +; SSE2-NEXT: orpd %xmm1, %xmm0 +; SSE2-NEXT: retq +; +; AVX1-LABEL: test_fminimumnum_f64: +; AVX1: # %bb.0: +; AVX1-NEXT: vminsd %xmm1, %xmm0, %xmm2 +; AVX1-NEXT: vandpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm3 +; AVX1-NEXT: vorpd %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vcmpunordsd %xmm0, %xmm0, %xmm3 +; AVX1-NEXT: vblendvpd %xmm3, %xmm1, %xmm2, %xmm2 +; AVX1-NEXT: vcmpunordsd %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vblendvpd %xmm1, %xmm0, %xmm2, %xmm0 +; AVX1-NEXT: retq +; +; AVX512F-LABEL: test_fminimumnum_f64: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vminsd %xmm1, %xmm0, %xmm2 +; AVX512F-NEXT: vandpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm3 +; AVX512F-NEXT: vorpd %xmm2, %xmm3, %xmm2 +; AVX512F-NEXT: vcmpunordsd %xmm0, %xmm0, %k1 +; AVX512F-NEXT: vmovsd %xmm1, %xmm2, %xmm2 {%k1} +; AVX512F-NEXT: vcmpunordsd %xmm1, %xmm1, %k1 +; AVX512F-NEXT: vmovsd %xmm0, %xmm2, %xmm2 {%k1} +; AVX512F-NEXT: vmovapd %xmm2, %xmm0 +; AVX512F-NEXT: retq +; +; AVX512DQ-LABEL: test_fminimumnum_f64: +; AVX512DQ: # %bb.0: +; AVX512DQ-NEXT: vminsd %xmm1, %xmm0, %xmm2 +; AVX512DQ-NEXT: vandpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm3 +; AVX512DQ-NEXT: vorpd %xmm2, %xmm3, %xmm2 +; AVX512DQ-NEXT: vcmpunordsd %xmm0, %xmm0, %k1 +; AVX512DQ-NEXT: vmovsd %xmm1, %xmm2, %xmm2 {%k1} +; AVX512DQ-NEXT: vcmpunordsd %xmm1, %xmm1, %k1 +; AVX512DQ-NEXT: vmovsd %xmm0, %xmm2, %xmm2 {%k1} +; AVX512DQ-NEXT: vmovapd %xmm2, %xmm0 +; AVX512DQ-NEXT: retq +; +; AVX512BF16-LABEL: test_fminimumnum_f64: +; AVX512BF16: # %bb.0: +; AVX512BF16-NEXT: vminsd %xmm1, %xmm0, %xmm2 +; AVX512BF16-NEXT: vpternlogq {{.*#+}} xmm2 = xmm2 | (xmm0 & m64bcst) +; AVX512BF16-NEXT: vcmpunordsd %xmm0, %xmm0, %k1 +; AVX512BF16-NEXT: vmovsd %xmm1, %xmm2, %xmm2 {%k1} +; AVX512BF16-NEXT: vcmpunordsd %xmm1, %xmm1, %k1 +; AVX512BF16-NEXT: vmovsd %xmm0, %xmm2, %xmm2 {%k1} +; AVX512BF16-NEXT: vmovapd %xmm2, %xmm0 +; AVX512BF16-NEXT: retq +; +; AVX10_2-LABEL: test_fminimumnum_f64: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vminmaxsd $20, %xmm1, %xmm0 +; AVX10_2-NEXT: retq +; +; X86-LABEL: test_fminimumnum_f64: +; X86: # %bb.0: +; X86-NEXT: pushl %ebp +; X86-NEXT: movl %esp, %ebp +; X86-NEXT: andl $-8, %esp +; X86-NEXT: subl $8, %esp +; X86-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero +; X86-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero +; X86-NEXT: vminsd %xmm0, %xmm1, %xmm2 +; X86-NEXT: vandpd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1, %xmm3 +; X86-NEXT: vorpd %xmm2, %xmm3, %xmm2 +; X86-NEXT: vcmpunordsd %xmm1, %xmm1, %xmm3 +; X86-NEXT: vblendvpd %xmm3, %xmm0, %xmm2, %xmm2 +; X86-NEXT: vcmpunordsd %xmm0, %xmm0, %xmm0 +; X86-NEXT: vblendvpd %xmm0, %xmm1, %xmm2, %xmm0 +; X86-NEXT: vmovlpd %xmm0, (%esp) +; X86-NEXT: fldl (%esp) +; X86-NEXT: movl %ebp, %esp +; X86-NEXT: popl %ebp +; X86-NEXT: retl + %r = call double @llvm.minimumnum.f64(double %x, double %y) + ret double %r +} + define <2 x double> @test_fminimumnum_scalarize(<2 x double> %x, <2 x double> %y) { ; SSE2-LABEL: test_fminimumnum_scalarize: ; SSE2: # %bb.0: @@ -827,26 +1068,38 @@ define float @test_fminimumnum_combine_cmps(float %x, float %y) nounwind { ; SSE2: # %bb.0: ; SSE2-NEXT: divss %xmm0, %xmm1 ; SSE2-NEXT: movaps %xmm0, %xmm2 -; SSE2-NEXT: minss %xmm1, %xmm2 +; SSE2-NEXT: cmpunordss %xmm0, %xmm2 +; SSE2-NEXT: movaps %xmm2, %xmm3 +; SSE2-NEXT: andps %xmm1, %xmm3 +; SSE2-NEXT: movaps %xmm0, %xmm4 +; SSE2-NEXT: minss %xmm1, %xmm4 ; SSE2-NEXT: andps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; SSE2-NEXT: orps %xmm2, %xmm0 +; SSE2-NEXT: orps %xmm4, %xmm0 +; SSE2-NEXT: andnps %xmm0, %xmm2 +; SSE2-NEXT: orps %xmm3, %xmm2 +; SSE2-NEXT: movaps %xmm2, %xmm0 ; SSE2-NEXT: retq ; ; AVX1-LABEL: test_fminimumnum_combine_cmps: ; AVX1: # %bb.0: ; AVX1-NEXT: vdivss %xmm0, %xmm1, %xmm1 -; AVX1-NEXT: vminss %xmm1, %xmm0, %xmm1 -; AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 -; AVX1-NEXT: vorps %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vminss %xmm1, %xmm0, %xmm2 +; AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm3 +; AVX1-NEXT: vorps %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm0 +; AVX1-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0 ; AVX1-NEXT: retq ; ; AVX512F-LABEL: test_fminimumnum_combine_cmps: ; AVX512F: # %bb.0: -; AVX512F-NEXT: vdivss %xmm0, %xmm1, %xmm1 -; AVX512F-NEXT: vbroadcastss {{.*#+}} xmm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0] -; AVX512F-NEXT: vandps %xmm2, %xmm0, %xmm2 -; AVX512F-NEXT: vminss %xmm1, %xmm0, %xmm0 -; AVX512F-NEXT: vorps %xmm0, %xmm2, %xmm0 +; AVX512F-NEXT: vdivss %xmm0, %xmm1, %xmm2 +; AVX512F-NEXT: vbroadcastss {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0] +; AVX512F-NEXT: vandps %xmm1, %xmm0, %xmm1 +; AVX512F-NEXT: vminss %xmm2, %xmm0, %xmm3 +; AVX512F-NEXT: vorps %xmm3, %xmm1, %xmm1 +; AVX512F-NEXT: vcmpunordss %xmm0, %xmm0, %k1 +; AVX512F-NEXT: vmovss %xmm2, %xmm1, %xmm1 {%k1} +; AVX512F-NEXT: vmovaps %xmm1, %xmm0 ; AVX512F-NEXT: retq ; ; AVX512DQ-LABEL: test_fminimumnum_combine_cmps: @@ -862,9 +1115,12 @@ define float @test_fminimumnum_combine_cmps(float %x, float %y) nounwind { ; ; AVX512BF16-LABEL: test_fminimumnum_combine_cmps: ; AVX512BF16: # %bb.0: -; AVX512BF16-NEXT: vdivss %xmm0, %xmm1, %xmm1 -; AVX512BF16-NEXT: vminss %xmm1, %xmm0, %xmm1 -; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm0 = (xmm0 & m32bcst) | xmm1 +; AVX512BF16-NEXT: vdivss %xmm0, %xmm1, %xmm2 +; AVX512BF16-NEXT: vminss %xmm2, %xmm0, %xmm1 +; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm1 = xmm1 | (xmm0 & m32bcst) +; AVX512BF16-NEXT: vcmpunordss %xmm0, %xmm0, %k1 +; AVX512BF16-NEXT: vmovss %xmm2, %xmm1, %xmm1 {%k1} +; AVX512BF16-NEXT: vmovaps %xmm1, %xmm0 ; AVX512BF16-NEXT: retq ; ; AVX10_2-LABEL: test_fminimumnum_combine_cmps: @@ -879,9 +1135,11 @@ define float @test_fminimumnum_combine_cmps(float %x, float %y) nounwind { ; X86-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero ; X86-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero ; X86-NEXT: vdivss %xmm0, %xmm1, %xmm1 -; X86-NEXT: vminss %xmm1, %xmm0, %xmm1 -; X86-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0 -; X86-NEXT: vorps %xmm1, %xmm0, %xmm0 +; X86-NEXT: vminss %xmm1, %xmm0, %xmm2 +; X86-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm3 +; X86-NEXT: vorps %xmm2, %xmm3, %xmm2 +; X86-NEXT: vcmpunordss %xmm0, %xmm0, %xmm0 +; X86-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0 ; X86-NEXT: vmovss %xmm0, (%esp) ; X86-NEXT: flds (%esp) ; X86-NEXT: popl %eax @@ -894,16 +1152,21 @@ define float @test_fminimumnum_combine_cmps(float %x, float %y) nounwind { define <2 x double> @test_fminimumnum_vector(<2 x double> %x, <2 x double> %y) { ; SSE2-LABEL: test_fminimumnum_vector: ; SSE2: # %bb.0: -; SSE2-NEXT: movapd %xmm0, %xmm3 -; SSE2-NEXT: minpd %xmm1, %xmm3 -; SSE2-NEXT: cmpunordpd %xmm1, %xmm1 ; SSE2-NEXT: movapd %xmm0, %xmm2 -; SSE2-NEXT: andpd %xmm1, %xmm2 -; SSE2-NEXT: andpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; SSE2-NEXT: orpd %xmm3, %xmm0 -; SSE2-NEXT: andnpd %xmm0, %xmm1 -; SSE2-NEXT: orpd %xmm1, %xmm2 -; SSE2-NEXT: movapd %xmm2, %xmm0 +; SSE2-NEXT: cmpunordpd %xmm0, %xmm2 +; SSE2-NEXT: movapd %xmm1, %xmm3 +; SSE2-NEXT: andpd %xmm2, %xmm3 +; SSE2-NEXT: movapd %xmm0, %xmm4 +; SSE2-NEXT: minpd %xmm1, %xmm4 +; SSE2-NEXT: movapd {{.*#+}} xmm5 = [-0.0E+0,-0.0E+0] +; SSE2-NEXT: andpd %xmm0, %xmm5 +; SSE2-NEXT: orpd %xmm4, %xmm5 +; SSE2-NEXT: andnpd %xmm5, %xmm2 +; SSE2-NEXT: orpd %xmm3, %xmm2 +; SSE2-NEXT: cmpunordpd %xmm1, %xmm1 +; SSE2-NEXT: andpd %xmm1, %xmm0 +; SSE2-NEXT: andnpd %xmm2, %xmm1 +; SSE2-NEXT: orpd %xmm1, %xmm0 ; SSE2-NEXT: retq ; ; AVX1-LABEL: test_fminimumnum_vector: @@ -911,6 +1174,8 @@ define <2 x double> @test_fminimumnum_vector(<2 x double> %x, <2 x double> %y) { ; AVX1-NEXT: vminpd %xmm1, %xmm0, %xmm2 ; AVX1-NEXT: vandpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm3 ; AVX1-NEXT: vorpd %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vcmpunordpd %xmm0, %xmm0, %xmm3 +; AVX1-NEXT: vblendvpd %xmm3, %xmm1, %xmm2, %xmm2 ; AVX1-NEXT: vcmpunordpd %xmm1, %xmm1, %xmm1 ; AVX1-NEXT: vblendvpd %xmm1, %xmm0, %xmm2, %xmm0 ; AVX1-NEXT: retq @@ -920,6 +1185,8 @@ define <2 x double> @test_fminimumnum_vector(<2 x double> %x, <2 x double> %y) { ; AVX512F-NEXT: vminpd %xmm1, %xmm0, %xmm2 ; AVX512F-NEXT: vandpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm3 ; AVX512F-NEXT: vorpd %xmm2, %xmm3, %xmm2 +; AVX512F-NEXT: vcmpunordpd %xmm0, %xmm0, %xmm3 +; AVX512F-NEXT: vblendvpd %xmm3, %xmm1, %xmm2, %xmm2 ; AVX512F-NEXT: vcmpunordpd %xmm1, %xmm1, %xmm1 ; AVX512F-NEXT: vblendvpd %xmm1, %xmm0, %xmm2, %xmm0 ; AVX512F-NEXT: retq @@ -929,6 +1196,8 @@ define <2 x double> @test_fminimumnum_vector(<2 x double> %x, <2 x double> %y) { ; AVX512DQ-NEXT: vminpd %xmm1, %xmm0, %xmm2 ; AVX512DQ-NEXT: vandpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm3 ; AVX512DQ-NEXT: vorpd %xmm2, %xmm3, %xmm2 +; AVX512DQ-NEXT: vcmpunordpd %xmm0, %xmm0, %xmm3 +; AVX512DQ-NEXT: vblendvpd %xmm3, %xmm1, %xmm2, %xmm2 ; AVX512DQ-NEXT: vcmpunordpd %xmm1, %xmm1, %xmm1 ; AVX512DQ-NEXT: vblendvpd %xmm1, %xmm0, %xmm2, %xmm0 ; AVX512DQ-NEXT: retq @@ -937,6 +1206,8 @@ define <2 x double> @test_fminimumnum_vector(<2 x double> %x, <2 x double> %y) { ; AVX512BF16: # %bb.0: ; AVX512BF16-NEXT: vminpd %xmm1, %xmm0, %xmm2 ; AVX512BF16-NEXT: vpternlogq {{.*#+}} xmm2 = xmm2 | (xmm0 & m64bcst) +; AVX512BF16-NEXT: vcmpunordpd %xmm0, %xmm0, %k1 +; AVX512BF16-NEXT: vmovapd %xmm1, %xmm2 {%k1} ; AVX512BF16-NEXT: vcmpunordpd %xmm1, %xmm1, %k1 ; AVX512BF16-NEXT: vmovapd %xmm0, %xmm2 {%k1} ; AVX512BF16-NEXT: vmovapd %xmm2, %xmm0 @@ -952,6 +1223,8 @@ define <2 x double> @test_fminimumnum_vector(<2 x double> %x, <2 x double> %y) { ; X86-NEXT: vminpd %xmm1, %xmm0, %xmm2 ; X86-NEXT: vandpd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm3 ; X86-NEXT: vorpd %xmm2, %xmm3, %xmm2 +; X86-NEXT: vcmpunordpd %xmm0, %xmm0, %xmm3 +; X86-NEXT: vblendvpd %xmm3, %xmm1, %xmm2, %xmm2 ; X86-NEXT: vcmpunordpd %xmm1, %xmm1, %xmm1 ; X86-NEXT: vblendvpd %xmm1, %xmm0, %xmm2, %xmm0 ; X86-NEXT: retl @@ -983,6 +1256,92 @@ define <4 x float> @test_fmaximumnum_vector(<4 x float> %x, <4 x float> %y) { ret <4 x float> %r } +; Exercise the same first-operand NaN correction lane-wise. +define <4 x float> @test_fmaximumnum_vector_nan(<4 x float> %x, <4 x float> %y) { +; SSE2-LABEL: test_fmaximumnum_vector_nan: +; SSE2: # %bb.0: +; SSE2-NEXT: movaps %xmm0, %xmm2 +; SSE2-NEXT: cmpunordps %xmm0, %xmm2 +; SSE2-NEXT: movaps %xmm1, %xmm3 +; SSE2-NEXT: andps %xmm2, %xmm3 +; SSE2-NEXT: movaps %xmm0, %xmm4 +; SSE2-NEXT: maxps %xmm1, %xmm4 +; SSE2-NEXT: movaps {{.*#+}} xmm5 = [NaN,NaN,NaN,NaN] +; SSE2-NEXT: orps %xmm0, %xmm5 +; SSE2-NEXT: andps %xmm4, %xmm5 +; SSE2-NEXT: andnps %xmm5, %xmm2 +; SSE2-NEXT: orps %xmm3, %xmm2 +; SSE2-NEXT: cmpunordps %xmm1, %xmm1 +; SSE2-NEXT: andps %xmm1, %xmm0 +; SSE2-NEXT: andnps %xmm2, %xmm1 +; SSE2-NEXT: orps %xmm1, %xmm0 +; SSE2-NEXT: retq +; +; AVX1-LABEL: test_fmaximumnum_vector_nan: +; AVX1: # %bb.0: +; AVX1-NEXT: vmaxps %xmm1, %xmm0, %xmm2 +; AVX1-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm3 +; AVX1-NEXT: vandps %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vcmpunordps %xmm0, %xmm0, %xmm3 +; AVX1-NEXT: vblendvps %xmm3, %xmm1, %xmm2, %xmm2 +; AVX1-NEXT: vcmpunordps %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vblendvps %xmm1, %xmm0, %xmm2, %xmm0 +; AVX1-NEXT: retq +; +; AVX512F-LABEL: test_fmaximumnum_vector_nan: +; AVX512F: # %bb.0: +; AVX512F-NEXT: vmaxps %xmm1, %xmm0, %xmm2 +; AVX512F-NEXT: vbroadcastss {{.*#+}} xmm3 = [NaN,NaN,NaN,NaN] +; AVX512F-NEXT: vorps %xmm3, %xmm0, %xmm3 +; AVX512F-NEXT: vandps %xmm2, %xmm3, %xmm2 +; AVX512F-NEXT: vcmpunordps %xmm0, %xmm0, %xmm3 +; AVX512F-NEXT: vblendvps %xmm3, %xmm1, %xmm2, %xmm2 +; AVX512F-NEXT: vcmpunordps %xmm1, %xmm1, %xmm1 +; AVX512F-NEXT: vblendvps %xmm1, %xmm0, %xmm2, %xmm0 +; AVX512F-NEXT: retq +; +; AVX512DQ-LABEL: test_fmaximumnum_vector_nan: +; AVX512DQ: # %bb.0: +; AVX512DQ-NEXT: vmaxps %xmm1, %xmm0, %xmm2 +; AVX512DQ-NEXT: vbroadcastss {{.*#+}} xmm3 = [NaN,NaN,NaN,NaN] +; AVX512DQ-NEXT: vorps %xmm3, %xmm0, %xmm3 +; AVX512DQ-NEXT: vandps %xmm2, %xmm3, %xmm2 +; AVX512DQ-NEXT: vcmpunordps %xmm0, %xmm0, %xmm3 +; AVX512DQ-NEXT: vblendvps %xmm3, %xmm1, %xmm2, %xmm2 +; AVX512DQ-NEXT: vcmpunordps %xmm1, %xmm1, %xmm1 +; AVX512DQ-NEXT: vblendvps %xmm1, %xmm0, %xmm2, %xmm0 +; AVX512DQ-NEXT: retq +; +; AVX512BF16-LABEL: test_fmaximumnum_vector_nan: +; AVX512BF16: # %bb.0: +; AVX512BF16-NEXT: vmaxps %xmm1, %xmm0, %xmm2 +; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm2 = xmm2 & (xmm0 | m32bcst) +; AVX512BF16-NEXT: vcmpunordps %xmm0, %xmm0, %k1 +; AVX512BF16-NEXT: vmovaps %xmm1, %xmm2 {%k1} +; AVX512BF16-NEXT: vcmpunordps %xmm1, %xmm1, %k1 +; AVX512BF16-NEXT: vmovaps %xmm0, %xmm2 {%k1} +; AVX512BF16-NEXT: vmovaps %xmm2, %xmm0 +; AVX512BF16-NEXT: retq +; +; AVX10_2-LABEL: test_fmaximumnum_vector_nan: +; AVX10_2: # %bb.0: +; AVX10_2-NEXT: vminmaxps $21, %xmm1, %xmm0, %xmm0 +; AVX10_2-NEXT: retq +; +; X86-LABEL: test_fmaximumnum_vector_nan: +; X86: # %bb.0: +; X86-NEXT: vmaxps %xmm1, %xmm0, %xmm2 +; X86-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm3 +; X86-NEXT: vandps %xmm2, %xmm3, %xmm2 +; X86-NEXT: vcmpunordps %xmm0, %xmm0, %xmm3 +; X86-NEXT: vblendvps %xmm3, %xmm1, %xmm2, %xmm2 +; X86-NEXT: vcmpunordps %xmm1, %xmm1, %xmm1 +; X86-NEXT: vblendvps %xmm1, %xmm0, %xmm2, %xmm0 +; X86-NEXT: retl + %r = call <4 x float> @llvm.maximumnum.v4f32(<4 x float> %x, <4 x float> %y) + ret <4 x float> %r +} + define <2 x double> @test_fminimumnum_vector_zero(<2 x double> %x) { ; SSE2-LABEL: test_fminimumnum_vector_zero: ; SSE2: # %bb.0: @@ -1174,47 +1533,62 @@ define <2 x double> @test_fminimumnum_vector_partially_zero(<2 x double> %x) { define <2 x double> @test_fminimumnum_vector_ diff erent_zeros(<2 x double> %x) { ; SSE2-LABEL: test_fminimumnum_vector_ diff erent_zeros: ; SSE2: # %bb.0: -; SSE2-NEXT: xorpd %xmm1, %xmm1 -; SSE2-NEXT: movhpd {{.*#+}} xmm1 = xmm1[0],mem[0] -; SSE2-NEXT: movapd %xmm0, %xmm2 -; SSE2-NEXT: minpd %xmm1, %xmm2 +; SSE2-NEXT: movapd %xmm0, %xmm1 +; SSE2-NEXT: cmpunordpd %xmm0, %xmm1 +; SSE2-NEXT: xorpd %xmm2, %xmm2 +; SSE2-NEXT: movhpd {{.*#+}} xmm2 = xmm2[0],mem[0] +; SSE2-NEXT: movapd %xmm0, %xmm3 +; SSE2-NEXT: minpd %xmm2, %xmm3 +; SSE2-NEXT: andpd %xmm1, %xmm2 ; SSE2-NEXT: andpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; SSE2-NEXT: orpd %xmm2, %xmm0 +; SSE2-NEXT: orpd %xmm3, %xmm0 +; SSE2-NEXT: andnpd %xmm0, %xmm1 +; SSE2-NEXT: orpd %xmm2, %xmm1 +; SSE2-NEXT: movapd %xmm1, %xmm0 ; SSE2-NEXT: retq ; ; AVX1-LABEL: test_fminimumnum_vector_ diff erent_zeros: ; AVX1: # %bb.0: ; AVX1-NEXT: vxorpd %xmm1, %xmm1, %xmm1 ; AVX1-NEXT: vmovhpd {{.*#+}} xmm1 = xmm1[0],mem[0] -; AVX1-NEXT: vminpd %xmm1, %xmm0, %xmm1 -; AVX1-NEXT: vandpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 -; AVX1-NEXT: vorpd %xmm1, %xmm0, %xmm0 +; AVX1-NEXT: vminpd %xmm1, %xmm0, %xmm2 +; AVX1-NEXT: vandpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm3 +; AVX1-NEXT: vorpd %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vcmpunordpd %xmm0, %xmm0, %xmm0 +; AVX1-NEXT: vblendvpd %xmm0, %xmm1, %xmm2, %xmm0 ; AVX1-NEXT: retq ; ; AVX512F-LABEL: test_fminimumnum_vector_ diff erent_zeros: ; AVX512F: # %bb.0: ; AVX512F-NEXT: vxorpd %xmm1, %xmm1, %xmm1 ; AVX512F-NEXT: vmovhpd {{.*#+}} xmm1 = xmm1[0],mem[0] -; AVX512F-NEXT: vminpd %xmm1, %xmm0, %xmm1 -; AVX512F-NEXT: vandpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 -; AVX512F-NEXT: vorpd %xmm1, %xmm0, %xmm0 +; AVX512F-NEXT: vminpd %xmm1, %xmm0, %xmm2 +; AVX512F-NEXT: vandpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm3 +; AVX512F-NEXT: vorpd %xmm2, %xmm3, %xmm2 +; AVX512F-NEXT: vcmpunordpd %xmm0, %xmm0, %xmm0 +; AVX512F-NEXT: vblendvpd %xmm0, %xmm1, %xmm2, %xmm0 ; AVX512F-NEXT: retq ; ; AVX512DQ-LABEL: test_fminimumnum_vector_ diff erent_zeros: ; AVX512DQ: # %bb.0: ; AVX512DQ-NEXT: vxorpd %xmm1, %xmm1, %xmm1 ; AVX512DQ-NEXT: vmovhpd {{.*#+}} xmm1 = xmm1[0],mem[0] -; AVX512DQ-NEXT: vminpd %xmm1, %xmm0, %xmm1 -; AVX512DQ-NEXT: vandpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 -; AVX512DQ-NEXT: vorpd %xmm1, %xmm0, %xmm0 +; AVX512DQ-NEXT: vminpd %xmm1, %xmm0, %xmm2 +; AVX512DQ-NEXT: vandpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm3 +; AVX512DQ-NEXT: vorpd %xmm2, %xmm3, %xmm2 +; AVX512DQ-NEXT: vcmpunordpd %xmm0, %xmm0, %xmm0 +; AVX512DQ-NEXT: vblendvpd %xmm0, %xmm1, %xmm2, %xmm0 ; AVX512DQ-NEXT: retq ; ; AVX512BF16-LABEL: test_fminimumnum_vector_ diff erent_zeros: ; AVX512BF16: # %bb.0: ; AVX512BF16-NEXT: vxorpd %xmm1, %xmm1, %xmm1 -; AVX512BF16-NEXT: vmovhpd {{.*#+}} xmm1 = xmm1[0],mem[0] -; AVX512BF16-NEXT: vminpd %xmm1, %xmm0, %xmm1 -; AVX512BF16-NEXT: vpternlogq {{.*#+}} xmm0 = (xmm0 & m64bcst) | xmm1 +; AVX512BF16-NEXT: vmovhpd {{.*#+}} xmm2 = xmm1[0],mem[0] +; AVX512BF16-NEXT: vminpd %xmm2, %xmm0, %xmm1 +; AVX512BF16-NEXT: vpternlogq {{.*#+}} xmm1 = xmm1 | (xmm0 & m64bcst) +; AVX512BF16-NEXT: vcmpunordpd %xmm0, %xmm0, %k1 +; AVX512BF16-NEXT: vmovapd %xmm2, %xmm1 {%k1} +; AVX512BF16-NEXT: vmovapd %xmm1, %xmm0 ; AVX512BF16-NEXT: retq ; ; AVX10_2-LABEL: test_fminimumnum_vector_ diff erent_zeros: @@ -1228,9 +1602,11 @@ define <2 x double> @test_fminimumnum_vector_ diff erent_zeros(<2 x double> %x) { ; X86: # %bb.0: ; X86-NEXT: vxorpd %xmm1, %xmm1, %xmm1 ; X86-NEXT: vmovhpd {{.*#+}} xmm1 = xmm1[0],mem[0] -; X86-NEXT: vminpd %xmm1, %xmm0, %xmm1 -; X86-NEXT: vandpd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0 -; X86-NEXT: vorpd %xmm1, %xmm0, %xmm0 +; X86-NEXT: vminpd %xmm1, %xmm0, %xmm2 +; X86-NEXT: vandpd {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm3 +; X86-NEXT: vorpd %xmm2, %xmm3, %xmm2 +; X86-NEXT: vcmpunordpd %xmm0, %xmm0, %xmm0 +; X86-NEXT: vblendvpd %xmm0, %xmm1, %xmm2, %xmm0 ; X86-NEXT: retl %r = call <2 x double> @llvm.minimumnum.v2f64(<2 x double> %x, <2 x double> <double 0., double -0.>) ret <2 x double> %r @@ -1520,16 +1896,21 @@ define <4 x float> @test_fmaximumnum_v4f32_splat(<4 x float> %x, float %y) { ; SSE2-LABEL: test_fmaximumnum_v4f32_splat: ; SSE2: # %bb.0: ; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,0,0,0] -; SSE2-NEXT: movaps %xmm0, %xmm3 -; SSE2-NEXT: maxps %xmm1, %xmm3 -; SSE2-NEXT: cmpunordps %xmm1, %xmm1 ; SSE2-NEXT: movaps %xmm0, %xmm2 -; SSE2-NEXT: andps %xmm1, %xmm2 -; SSE2-NEXT: orps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 -; SSE2-NEXT: andps %xmm3, %xmm0 -; SSE2-NEXT: andnps %xmm0, %xmm1 -; SSE2-NEXT: orps %xmm1, %xmm2 -; SSE2-NEXT: movaps %xmm2, %xmm0 +; SSE2-NEXT: cmpunordps %xmm0, %xmm2 +; SSE2-NEXT: movaps %xmm1, %xmm3 +; SSE2-NEXT: andps %xmm2, %xmm3 +; SSE2-NEXT: movaps %xmm0, %xmm4 +; SSE2-NEXT: maxps %xmm1, %xmm4 +; SSE2-NEXT: movaps {{.*#+}} xmm5 = [NaN,NaN,NaN,NaN] +; SSE2-NEXT: orps %xmm0, %xmm5 +; SSE2-NEXT: andps %xmm4, %xmm5 +; SSE2-NEXT: andnps %xmm5, %xmm2 +; SSE2-NEXT: orps %xmm3, %xmm2 +; SSE2-NEXT: cmpunordps %xmm1, %xmm1 +; SSE2-NEXT: andps %xmm1, %xmm0 +; SSE2-NEXT: andnps %xmm2, %xmm1 +; SSE2-NEXT: orps %xmm1, %xmm0 ; SSE2-NEXT: retq ; ; AVX1-LABEL: test_fmaximumnum_v4f32_splat: @@ -1538,6 +1919,8 @@ define <4 x float> @test_fmaximumnum_v4f32_splat(<4 x float> %x, float %y) { ; AVX1-NEXT: vmaxps %xmm1, %xmm0, %xmm2 ; AVX1-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm3 ; AVX1-NEXT: vandps %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vcmpunordps %xmm0, %xmm0, %xmm3 +; AVX1-NEXT: vblendvps %xmm3, %xmm1, %xmm2, %xmm2 ; AVX1-NEXT: vcmpunordps %xmm1, %xmm1, %xmm1 ; AVX1-NEXT: vblendvps %xmm1, %xmm0, %xmm2, %xmm0 ; AVX1-NEXT: retq @@ -1549,6 +1932,8 @@ define <4 x float> @test_fmaximumnum_v4f32_splat(<4 x float> %x, float %y) { ; AVX512F-NEXT: vorps %xmm2, %xmm0, %xmm2 ; AVX512F-NEXT: vmaxps %xmm1, %xmm0, %xmm3 ; AVX512F-NEXT: vandps %xmm3, %xmm2, %xmm2 +; AVX512F-NEXT: vcmpunordps %xmm0, %xmm0, %xmm3 +; AVX512F-NEXT: vblendvps %xmm3, %xmm1, %xmm2, %xmm2 ; AVX512F-NEXT: vcmpunordps %xmm1, %xmm1, %xmm1 ; AVX512F-NEXT: vblendvps %xmm1, %xmm0, %xmm2, %xmm0 ; AVX512F-NEXT: retq @@ -1560,6 +1945,8 @@ define <4 x float> @test_fmaximumnum_v4f32_splat(<4 x float> %x, float %y) { ; AVX512DQ-NEXT: vorps %xmm2, %xmm0, %xmm2 ; AVX512DQ-NEXT: vmaxps %xmm1, %xmm0, %xmm3 ; AVX512DQ-NEXT: vandps %xmm3, %xmm2, %xmm2 +; AVX512DQ-NEXT: vcmpunordps %xmm0, %xmm0, %xmm3 +; AVX512DQ-NEXT: vblendvps %xmm3, %xmm1, %xmm2, %xmm2 ; AVX512DQ-NEXT: vcmpunordps %xmm1, %xmm1, %xmm1 ; AVX512DQ-NEXT: vblendvps %xmm1, %xmm0, %xmm2, %xmm0 ; AVX512DQ-NEXT: retq @@ -1569,6 +1956,8 @@ define <4 x float> @test_fmaximumnum_v4f32_splat(<4 x float> %x, float %y) { ; AVX512BF16-NEXT: vbroadcastss %xmm1, %xmm2 ; AVX512BF16-NEXT: vmaxps %xmm2, %xmm0, %xmm1 ; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm1 = xmm1 & (xmm0 | m32bcst) +; AVX512BF16-NEXT: vcmpunordps %xmm0, %xmm0, %k1 +; AVX512BF16-NEXT: vmovaps %xmm2, %xmm1 {%k1} ; AVX512BF16-NEXT: vcmpunordps %xmm2, %xmm2, %k1 ; AVX512BF16-NEXT: vmovaps %xmm0, %xmm1 {%k1} ; AVX512BF16-NEXT: vmovaps %xmm1, %xmm0 @@ -1586,6 +1975,8 @@ define <4 x float> @test_fmaximumnum_v4f32_splat(<4 x float> %x, float %y) { ; X86-NEXT: vmaxps %xmm1, %xmm0, %xmm2 ; X86-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm3 ; X86-NEXT: vandps %xmm2, %xmm3, %xmm2 +; X86-NEXT: vcmpunordps %xmm0, %xmm0, %xmm3 +; X86-NEXT: vblendvps %xmm3, %xmm1, %xmm2, %xmm2 ; X86-NEXT: vcmpunordps %xmm1, %xmm1, %xmm1 ; X86-NEXT: vblendvps %xmm1, %xmm0, %xmm2, %xmm0 ; X86-NEXT: retl @@ -1618,22 +2009,29 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind ; SSE2-NEXT: psrld $16, %xmm0 ; SSE2-NEXT: callq __extendhfsf2@PLT ; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [NaN,NaN,NaN,NaN] -; SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE2-NEXT: movdqa %xmm0, %xmm2 -; SSE2-NEXT: por %xmm1, %xmm2 +; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [NaN,NaN,NaN,NaN] ; SSE2-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE2-NEXT: movdqa %xmm0, %xmm1 +; SSE2-NEXT: por %xmm2, %xmm1 +; SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; SSE2-NEXT: callq __extendhfsf2@PLT ; SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload ; SSE2-NEXT: movaps %xmm3, %xmm1 ; SSE2-NEXT: maxss %xmm0, %xmm1 ; SSE2-NEXT: andps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload +; SSE2-NEXT: movaps %xmm3, %xmm2 +; SSE2-NEXT: movaps %xmm3, %xmm4 +; SSE2-NEXT: cmpunordss %xmm3, %xmm2 +; SSE2-NEXT: movaps %xmm2, %xmm3 +; SSE2-NEXT: andnps %xmm1, %xmm3 +; SSE2-NEXT: andps %xmm0, %xmm2 +; SSE2-NEXT: orps %xmm3, %xmm2 ; SSE2-NEXT: cmpunordss %xmm0, %xmm0 -; SSE2-NEXT: movaps %xmm0, %xmm2 -; SSE2-NEXT: andnps %xmm1, %xmm2 -; SSE2-NEXT: andps %xmm3, %xmm0 -; SSE2-NEXT: orps %xmm2, %xmm0 +; SSE2-NEXT: movaps %xmm0, %xmm1 +; SSE2-NEXT: andnps %xmm2, %xmm1 +; SSE2-NEXT: andps %xmm4, %xmm0 +; SSE2-NEXT: orps %xmm1, %xmm0 ; SSE2-NEXT: callq __truncsfhf2@PLT ; SSE2-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE2-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload @@ -1648,11 +2046,18 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind ; SSE2-NEXT: movaps %xmm3, %xmm1 ; SSE2-NEXT: maxss %xmm0, %xmm1 ; SSE2-NEXT: andps (%rsp), %xmm1 # 16-byte Folded Reload +; SSE2-NEXT: movaps %xmm3, %xmm2 +; SSE2-NEXT: movaps %xmm3, %xmm4 +; SSE2-NEXT: cmpunordss %xmm3, %xmm2 +; SSE2-NEXT: movaps %xmm2, %xmm3 +; SSE2-NEXT: andnps %xmm1, %xmm3 +; SSE2-NEXT: andps %xmm0, %xmm2 +; SSE2-NEXT: orps %xmm3, %xmm2 ; SSE2-NEXT: cmpunordss %xmm0, %xmm0 -; SSE2-NEXT: movaps %xmm0, %xmm2 -; SSE2-NEXT: andnps %xmm1, %xmm2 -; SSE2-NEXT: andps %xmm3, %xmm0 -; SSE2-NEXT: orps %xmm2, %xmm0 +; SSE2-NEXT: movaps %xmm0, %xmm1 +; SSE2-NEXT: andnps %xmm2, %xmm1 +; SSE2-NEXT: andps %xmm4, %xmm0 +; SSE2-NEXT: orps %xmm1, %xmm0 ; SSE2-NEXT: callq __truncsfhf2@PLT ; SSE2-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload @@ -1667,11 +2072,18 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind ; SSE2-NEXT: movaps %xmm3, %xmm1 ; SSE2-NEXT: maxss %xmm0, %xmm1 ; SSE2-NEXT: andps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload +; SSE2-NEXT: movaps %xmm3, %xmm2 +; SSE2-NEXT: movaps %xmm3, %xmm4 +; SSE2-NEXT: cmpunordss %xmm3, %xmm2 +; SSE2-NEXT: movaps %xmm2, %xmm3 +; SSE2-NEXT: andnps %xmm1, %xmm3 +; SSE2-NEXT: andps %xmm0, %xmm2 +; SSE2-NEXT: orps %xmm3, %xmm2 ; SSE2-NEXT: cmpunordss %xmm0, %xmm0 -; SSE2-NEXT: movaps %xmm0, %xmm2 -; SSE2-NEXT: andnps %xmm1, %xmm2 -; SSE2-NEXT: andps %xmm3, %xmm0 -; SSE2-NEXT: orps %xmm2, %xmm0 +; SSE2-NEXT: movaps %xmm0, %xmm1 +; SSE2-NEXT: andnps %xmm2, %xmm1 +; SSE2-NEXT: andps %xmm4, %xmm0 +; SSE2-NEXT: orps %xmm1, %xmm0 ; SSE2-NEXT: callq __truncsfhf2@PLT ; SSE2-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload @@ -1687,11 +2099,18 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind ; SSE2-NEXT: movaps %xmm3, %xmm1 ; SSE2-NEXT: maxss %xmm0, %xmm1 ; SSE2-NEXT: andps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload +; SSE2-NEXT: movaps %xmm3, %xmm2 +; SSE2-NEXT: movaps %xmm3, %xmm4 +; SSE2-NEXT: cmpunordss %xmm3, %xmm2 +; SSE2-NEXT: movaps %xmm2, %xmm3 +; SSE2-NEXT: andnps %xmm1, %xmm3 +; SSE2-NEXT: andps %xmm0, %xmm2 +; SSE2-NEXT: orps %xmm3, %xmm2 ; SSE2-NEXT: cmpunordss %xmm0, %xmm0 -; SSE2-NEXT: movaps %xmm0, %xmm2 -; SSE2-NEXT: andnps %xmm1, %xmm2 -; SSE2-NEXT: andps %xmm3, %xmm0 -; SSE2-NEXT: orps %xmm2, %xmm0 +; SSE2-NEXT: movaps %xmm0, %xmm1 +; SSE2-NEXT: andnps %xmm2, %xmm1 +; SSE2-NEXT: andps %xmm4, %xmm0 +; SSE2-NEXT: orps %xmm1, %xmm0 ; SSE2-NEXT: callq __truncsfhf2@PLT ; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload ; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] @@ -1725,53 +2144,61 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind ; AVX1-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX1-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; AVX1-NEXT: callq __extendhfsf2@PLT -; AVX1-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload -; AVX1-NEXT: vmaxss %xmm0, %xmm2, %xmm1 +; AVX1-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload +; AVX1-NEXT: vmaxss %xmm0, %xmm3, %xmm1 ; AVX1-NEXT: vandps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload +; AVX1-NEXT: vcmpunordss %xmm3, %xmm3, %xmm2 +; AVX1-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm1 ; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm0 -; AVX1-NEXT: vblendvps %xmm0, %xmm2, %xmm1, %xmm0 +; AVX1-NEXT: vblendvps %xmm0, %xmm3, %xmm1, %xmm0 ; AVX1-NEXT: callq __truncsfhf2@PLT ; AVX1-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX1-NEXT: vmovaps (%rsp), %xmm0 # 16-byte Reload ; AVX1-NEXT: callq __extendhfsf2@PLT ; AVX1-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX1-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 -; AVX1-NEXT: vmovaps %xmm1, (%rsp) # 16-byte Spill +; AVX1-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill ; AVX1-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; AVX1-NEXT: callq __extendhfsf2@PLT -; AVX1-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload -; AVX1-NEXT: vmaxss %xmm0, %xmm2, %xmm1 +; AVX1-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload +; AVX1-NEXT: vmaxss %xmm0, %xmm3, %xmm1 ; AVX1-NEXT: vandps (%rsp), %xmm1, %xmm1 # 16-byte Folded Reload +; AVX1-NEXT: vcmpunordss %xmm3, %xmm3, %xmm2 +; AVX1-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm1 ; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm0 -; AVX1-NEXT: vblendvps %xmm0, %xmm2, %xmm1, %xmm0 +; AVX1-NEXT: vblendvps %xmm0, %xmm3, %xmm1, %xmm0 ; AVX1-NEXT: callq __truncsfhf2@PLT ; AVX1-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX1-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; AVX1-NEXT: callq __extendhfsf2@PLT ; AVX1-NEXT: vmovaps %xmm0, (%rsp) # 16-byte Spill -; AVX1-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 -; AVX1-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX1-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX1-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; AVX1-NEXT: callq __extendhfsf2@PLT -; AVX1-NEXT: vmovaps (%rsp), %xmm2 # 16-byte Reload -; AVX1-NEXT: vmaxss %xmm0, %xmm2, %xmm1 +; AVX1-NEXT: vmovaps (%rsp), %xmm3 # 16-byte Reload +; AVX1-NEXT: vmaxss %xmm0, %xmm3, %xmm1 ; AVX1-NEXT: vandps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload +; AVX1-NEXT: vcmpunordss %xmm3, %xmm3, %xmm2 +; AVX1-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm1 ; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm0 -; AVX1-NEXT: vblendvps %xmm0, %xmm2, %xmm1, %xmm0 +; AVX1-NEXT: vblendvps %xmm0, %xmm3, %xmm1, %xmm0 ; AVX1-NEXT: callq __truncsfhf2@PLT ; AVX1-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX1-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; AVX1-NEXT: callq __extendhfsf2@PLT ; AVX1-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX1-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 -; AVX1-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX1-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX1-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; AVX1-NEXT: callq __extendhfsf2@PLT -; AVX1-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload -; AVX1-NEXT: vmaxss %xmm0, %xmm2, %xmm1 +; AVX1-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload +; AVX1-NEXT: vmaxss %xmm0, %xmm3, %xmm1 ; AVX1-NEXT: vandps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload +; AVX1-NEXT: vcmpunordss %xmm3, %xmm3, %xmm2 +; AVX1-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm1 ; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm0 -; AVX1-NEXT: vblendvps %xmm0, %xmm2, %xmm1, %xmm0 +; AVX1-NEXT: vblendvps %xmm0, %xmm3, %xmm1, %xmm0 ; AVX1-NEXT: callq __truncsfhf2@PLT ; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload ; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] @@ -1792,6 +2219,8 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind ; AVX512F-NEXT: vcvtph2ps %xmm5, %xmm5 ; AVX512F-NEXT: vmaxss %xmm5, %xmm3, %xmm6 ; AVX512F-NEXT: vandps %xmm6, %xmm4, %xmm4 +; AVX512F-NEXT: vcmpunordss %xmm3, %xmm3, %k1 +; AVX512F-NEXT: vmovss %xmm5, %xmm4, %xmm4 {%k1} ; AVX512F-NEXT: vcmpunordss %xmm5, %xmm5, %k1 ; AVX512F-NEXT: vmovss %xmm3, %xmm4, %xmm4 {%k1} ; AVX512F-NEXT: vcvtps2ph $4, %xmm4, %xmm3 @@ -1802,6 +2231,8 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind ; AVX512F-NEXT: vcvtph2ps %xmm6, %xmm6 ; AVX512F-NEXT: vmaxss %xmm6, %xmm4, %xmm7 ; AVX512F-NEXT: vandps %xmm7, %xmm5, %xmm5 +; AVX512F-NEXT: vcmpunordss %xmm4, %xmm4, %k1 +; AVX512F-NEXT: vmovss %xmm6, %xmm5, %xmm5 {%k1} ; AVX512F-NEXT: vcmpunordss %xmm6, %xmm6, %k1 ; AVX512F-NEXT: vmovss %xmm4, %xmm5, %xmm5 {%k1} ; AVX512F-NEXT: vcvtps2ph $4, %xmm5, %xmm4 @@ -1813,6 +2244,8 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind ; AVX512F-NEXT: vcvtph2ps %xmm6, %xmm6 ; AVX512F-NEXT: vmaxss %xmm6, %xmm4, %xmm7 ; AVX512F-NEXT: vandps %xmm7, %xmm5, %xmm5 +; AVX512F-NEXT: vcmpunordss %xmm4, %xmm4, %k1 +; AVX512F-NEXT: vmovss %xmm6, %xmm5, %xmm5 {%k1} ; AVX512F-NEXT: vcmpunordss %xmm6, %xmm6, %k1 ; AVX512F-NEXT: vmovss %xmm4, %xmm5, %xmm5 {%k1} ; AVX512F-NEXT: vcvtps2ph $4, %xmm5, %xmm4 @@ -1823,6 +2256,8 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind ; AVX512F-NEXT: vcvtph2ps %xmm7, %xmm7 ; AVX512F-NEXT: vmaxss %xmm7, %xmm5, %xmm8 ; AVX512F-NEXT: vandps %xmm6, %xmm8, %xmm6 +; AVX512F-NEXT: vcmpunordss %xmm5, %xmm5, %k1 +; AVX512F-NEXT: vmovss %xmm7, %xmm6, %xmm6 {%k1} ; AVX512F-NEXT: vcmpunordss %xmm7, %xmm7, %k1 ; AVX512F-NEXT: vmovss %xmm5, %xmm6, %xmm6 {%k1} ; AVX512F-NEXT: vcvtps2ph $4, %xmm6, %xmm5 @@ -1835,6 +2270,8 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind ; AVX512F-NEXT: vcvtph2ps %xmm6, %xmm6 ; AVX512F-NEXT: vmaxss %xmm6, %xmm4, %xmm7 ; AVX512F-NEXT: vandps %xmm7, %xmm5, %xmm5 +; AVX512F-NEXT: vcmpunordss %xmm4, %xmm4, %k1 +; AVX512F-NEXT: vmovss %xmm6, %xmm5, %xmm5 {%k1} ; AVX512F-NEXT: vcmpunordss %xmm6, %xmm6, %k1 ; AVX512F-NEXT: vmovss %xmm4, %xmm5, %xmm5 {%k1} ; AVX512F-NEXT: vcvtps2ph $4, %xmm5, %xmm4 @@ -1845,6 +2282,8 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind ; AVX512F-NEXT: vcvtph2ps %xmm7, %xmm7 ; AVX512F-NEXT: vmaxss %xmm7, %xmm5, %xmm8 ; AVX512F-NEXT: vandps %xmm6, %xmm8, %xmm6 +; AVX512F-NEXT: vcmpunordss %xmm5, %xmm5, %k1 +; AVX512F-NEXT: vmovss %xmm7, %xmm6, %xmm6 {%k1} ; AVX512F-NEXT: vcmpunordss %xmm7, %xmm7, %k1 ; AVX512F-NEXT: vmovss %xmm5, %xmm6, %xmm6 {%k1} ; AVX512F-NEXT: vcvtps2ph $4, %xmm6, %xmm5 @@ -1854,6 +2293,8 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind ; AVX512F-NEXT: vcvtph2ps %xmm1, %xmm7 ; AVX512F-NEXT: vmaxss %xmm7, %xmm5, %xmm8 ; AVX512F-NEXT: vandps %xmm6, %xmm8, %xmm6 +; AVX512F-NEXT: vcmpunordss %xmm5, %xmm5, %k1 +; AVX512F-NEXT: vmovss %xmm7, %xmm6, %xmm6 {%k1} ; AVX512F-NEXT: vcmpunordss %xmm7, %xmm7, %k1 ; AVX512F-NEXT: vmovss %xmm5, %xmm6, %xmm6 {%k1} ; AVX512F-NEXT: vcvtps2ph $4, %xmm6, %xmm5 @@ -1864,6 +2305,8 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind ; AVX512F-NEXT: vcvtph2ps %xmm1, %xmm1 ; AVX512F-NEXT: vmaxss %xmm1, %xmm0, %xmm6 ; AVX512F-NEXT: vandps %xmm6, %xmm2, %xmm2 +; AVX512F-NEXT: vcmpunordss %xmm0, %xmm0, %k1 +; AVX512F-NEXT: vmovss %xmm1, %xmm2, %xmm2 {%k1} ; AVX512F-NEXT: vcmpunordss %xmm1, %xmm1, %k1 ; AVX512F-NEXT: vmovss %xmm0, %xmm2, %xmm2 {%k1} ; AVX512F-NEXT: vcvtps2ph $4, %xmm2, %xmm0 @@ -1882,6 +2325,8 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind ; AVX512DQ-NEXT: vcvtph2ps %xmm5, %xmm5 ; AVX512DQ-NEXT: vmaxss %xmm5, %xmm3, %xmm6 ; AVX512DQ-NEXT: vandps %xmm6, %xmm4, %xmm4 +; AVX512DQ-NEXT: vcmpunordss %xmm3, %xmm3, %k1 +; AVX512DQ-NEXT: vmovss %xmm5, %xmm4, %xmm4 {%k1} ; AVX512DQ-NEXT: vcmpunordss %xmm5, %xmm5, %k1 ; AVX512DQ-NEXT: vmovss %xmm3, %xmm4, %xmm4 {%k1} ; AVX512DQ-NEXT: vcvtps2ph $4, %xmm4, %xmm3 @@ -1892,6 +2337,8 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind ; AVX512DQ-NEXT: vcvtph2ps %xmm6, %xmm6 ; AVX512DQ-NEXT: vmaxss %xmm6, %xmm4, %xmm7 ; AVX512DQ-NEXT: vandps %xmm7, %xmm5, %xmm5 +; AVX512DQ-NEXT: vcmpunordss %xmm4, %xmm4, %k1 +; AVX512DQ-NEXT: vmovss %xmm6, %xmm5, %xmm5 {%k1} ; AVX512DQ-NEXT: vcmpunordss %xmm6, %xmm6, %k1 ; AVX512DQ-NEXT: vmovss %xmm4, %xmm5, %xmm5 {%k1} ; AVX512DQ-NEXT: vcvtps2ph $4, %xmm5, %xmm4 @@ -1903,6 +2350,8 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind ; AVX512DQ-NEXT: vcvtph2ps %xmm6, %xmm6 ; AVX512DQ-NEXT: vmaxss %xmm6, %xmm4, %xmm7 ; AVX512DQ-NEXT: vandps %xmm7, %xmm5, %xmm5 +; AVX512DQ-NEXT: vcmpunordss %xmm4, %xmm4, %k1 +; AVX512DQ-NEXT: vmovss %xmm6, %xmm5, %xmm5 {%k1} ; AVX512DQ-NEXT: vcmpunordss %xmm6, %xmm6, %k1 ; AVX512DQ-NEXT: vmovss %xmm4, %xmm5, %xmm5 {%k1} ; AVX512DQ-NEXT: vcvtps2ph $4, %xmm5, %xmm4 @@ -1913,6 +2362,8 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind ; AVX512DQ-NEXT: vcvtph2ps %xmm7, %xmm7 ; AVX512DQ-NEXT: vmaxss %xmm7, %xmm5, %xmm8 ; AVX512DQ-NEXT: vandps %xmm6, %xmm8, %xmm6 +; AVX512DQ-NEXT: vcmpunordss %xmm5, %xmm5, %k1 +; AVX512DQ-NEXT: vmovss %xmm7, %xmm6, %xmm6 {%k1} ; AVX512DQ-NEXT: vcmpunordss %xmm7, %xmm7, %k1 ; AVX512DQ-NEXT: vmovss %xmm5, %xmm6, %xmm6 {%k1} ; AVX512DQ-NEXT: vcvtps2ph $4, %xmm6, %xmm5 @@ -1925,6 +2376,8 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind ; AVX512DQ-NEXT: vcvtph2ps %xmm6, %xmm6 ; AVX512DQ-NEXT: vmaxss %xmm6, %xmm4, %xmm7 ; AVX512DQ-NEXT: vandps %xmm7, %xmm5, %xmm5 +; AVX512DQ-NEXT: vcmpunordss %xmm4, %xmm4, %k1 +; AVX512DQ-NEXT: vmovss %xmm6, %xmm5, %xmm5 {%k1} ; AVX512DQ-NEXT: vcmpunordss %xmm6, %xmm6, %k1 ; AVX512DQ-NEXT: vmovss %xmm4, %xmm5, %xmm5 {%k1} ; AVX512DQ-NEXT: vcvtps2ph $4, %xmm5, %xmm4 @@ -1935,6 +2388,8 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind ; AVX512DQ-NEXT: vcvtph2ps %xmm7, %xmm7 ; AVX512DQ-NEXT: vmaxss %xmm7, %xmm5, %xmm8 ; AVX512DQ-NEXT: vandps %xmm6, %xmm8, %xmm6 +; AVX512DQ-NEXT: vcmpunordss %xmm5, %xmm5, %k1 +; AVX512DQ-NEXT: vmovss %xmm7, %xmm6, %xmm6 {%k1} ; AVX512DQ-NEXT: vcmpunordss %xmm7, %xmm7, %k1 ; AVX512DQ-NEXT: vmovss %xmm5, %xmm6, %xmm6 {%k1} ; AVX512DQ-NEXT: vcvtps2ph $4, %xmm6, %xmm5 @@ -1944,6 +2399,8 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind ; AVX512DQ-NEXT: vcvtph2ps %xmm1, %xmm7 ; AVX512DQ-NEXT: vmaxss %xmm7, %xmm5, %xmm8 ; AVX512DQ-NEXT: vandps %xmm6, %xmm8, %xmm6 +; AVX512DQ-NEXT: vcmpunordss %xmm5, %xmm5, %k1 +; AVX512DQ-NEXT: vmovss %xmm7, %xmm6, %xmm6 {%k1} ; AVX512DQ-NEXT: vcmpunordss %xmm7, %xmm7, %k1 ; AVX512DQ-NEXT: vmovss %xmm5, %xmm6, %xmm6 {%k1} ; AVX512DQ-NEXT: vcvtps2ph $4, %xmm6, %xmm5 @@ -1954,6 +2411,8 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind ; AVX512DQ-NEXT: vcvtph2ps %xmm1, %xmm1 ; AVX512DQ-NEXT: vmaxss %xmm1, %xmm0, %xmm6 ; AVX512DQ-NEXT: vandps %xmm6, %xmm2, %xmm2 +; AVX512DQ-NEXT: vcmpunordss %xmm0, %xmm0, %k1 +; AVX512DQ-NEXT: vmovss %xmm1, %xmm2, %xmm2 {%k1} ; AVX512DQ-NEXT: vcmpunordss %xmm1, %xmm1, %k1 ; AVX512DQ-NEXT: vmovss %xmm0, %xmm2, %xmm2 {%k1} ; AVX512DQ-NEXT: vcvtps2ph $4, %xmm2, %xmm0 @@ -1971,6 +2430,8 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind ; AVX512BF16-NEXT: vmaxss %xmm3, %xmm4, %xmm5 ; AVX512BF16-NEXT: vpbroadcastd {{.*#+}} xmm2 = [NaN,NaN,NaN,NaN] ; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm5 = xmm5 & (xmm4 | xmm2) +; AVX512BF16-NEXT: vcmpunordss %xmm4, %xmm4, %k1 +; AVX512BF16-NEXT: vmovss %xmm3, %xmm5, %xmm5 {%k1} ; AVX512BF16-NEXT: vcmpunordss %xmm3, %xmm3, %k1 ; AVX512BF16-NEXT: vmovss %xmm4, %xmm5, %xmm5 {%k1} ; AVX512BF16-NEXT: vcvtps2ph $4, %xmm5, %xmm3 @@ -1980,6 +2441,8 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind ; AVX512BF16-NEXT: vcvtph2ps %xmm5, %xmm5 ; AVX512BF16-NEXT: vmaxss %xmm4, %xmm5, %xmm6 ; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm6 = xmm6 & (xmm5 | xmm2) +; AVX512BF16-NEXT: vcmpunordss %xmm5, %xmm5, %k1 +; AVX512BF16-NEXT: vmovss %xmm4, %xmm6, %xmm6 {%k1} ; AVX512BF16-NEXT: vcmpunordss %xmm4, %xmm4, %k1 ; AVX512BF16-NEXT: vmovss %xmm5, %xmm6, %xmm6 {%k1} ; AVX512BF16-NEXT: vcvtps2ph $4, %xmm6, %xmm4 @@ -1990,6 +2453,8 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind ; AVX512BF16-NEXT: vcvtph2ps %xmm5, %xmm5 ; AVX512BF16-NEXT: vmaxss %xmm4, %xmm5, %xmm6 ; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm6 = xmm6 & (xmm5 | xmm2) +; AVX512BF16-NEXT: vcmpunordss %xmm5, %xmm5, %k1 +; AVX512BF16-NEXT: vmovss %xmm4, %xmm6, %xmm6 {%k1} ; AVX512BF16-NEXT: vcmpunordss %xmm4, %xmm4, %k1 ; AVX512BF16-NEXT: vmovss %xmm5, %xmm6, %xmm6 {%k1} ; AVX512BF16-NEXT: vcvtps2ph $4, %xmm6, %xmm4 @@ -1999,6 +2464,8 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind ; AVX512BF16-NEXT: vcvtph2ps %xmm6, %xmm6 ; AVX512BF16-NEXT: vmaxss %xmm5, %xmm6, %xmm7 ; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm7 = xmm7 & (xmm6 | xmm2) +; AVX512BF16-NEXT: vcmpunordss %xmm6, %xmm6, %k1 +; AVX512BF16-NEXT: vmovss %xmm5, %xmm7, %xmm7 {%k1} ; AVX512BF16-NEXT: vcmpunordss %xmm5, %xmm5, %k1 ; AVX512BF16-NEXT: vmovss %xmm6, %xmm7, %xmm7 {%k1} ; AVX512BF16-NEXT: vcvtps2ph $4, %xmm7, %xmm5 @@ -2010,6 +2477,8 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind ; AVX512BF16-NEXT: vcvtph2ps %xmm5, %xmm5 ; AVX512BF16-NEXT: vmaxss %xmm4, %xmm5, %xmm6 ; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm6 = xmm6 & (xmm5 | xmm2) +; AVX512BF16-NEXT: vcmpunordss %xmm5, %xmm5, %k1 +; AVX512BF16-NEXT: vmovss %xmm4, %xmm6, %xmm6 {%k1} ; AVX512BF16-NEXT: vcmpunordss %xmm4, %xmm4, %k1 ; AVX512BF16-NEXT: vmovss %xmm5, %xmm6, %xmm6 {%k1} ; AVX512BF16-NEXT: vcvtps2ph $4, %xmm6, %xmm4 @@ -2019,6 +2488,8 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind ; AVX512BF16-NEXT: vcvtph2ps %xmm6, %xmm6 ; AVX512BF16-NEXT: vmaxss %xmm5, %xmm6, %xmm7 ; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm7 = xmm7 & (xmm6 | xmm2) +; AVX512BF16-NEXT: vcmpunordss %xmm6, %xmm6, %k1 +; AVX512BF16-NEXT: vmovss %xmm5, %xmm7, %xmm7 {%k1} ; AVX512BF16-NEXT: vcmpunordss %xmm5, %xmm5, %k1 ; AVX512BF16-NEXT: vmovss %xmm6, %xmm7, %xmm7 {%k1} ; AVX512BF16-NEXT: vcvtps2ph $4, %xmm7, %xmm5 @@ -2027,6 +2498,8 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind ; AVX512BF16-NEXT: vcvtph2ps %xmm0, %xmm6 ; AVX512BF16-NEXT: vmaxss %xmm5, %xmm6, %xmm7 ; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm7 = xmm7 & (xmm6 | xmm2) +; AVX512BF16-NEXT: vcmpunordss %xmm6, %xmm6, %k1 +; AVX512BF16-NEXT: vmovss %xmm5, %xmm7, %xmm7 {%k1} ; AVX512BF16-NEXT: vcmpunordss %xmm5, %xmm5, %k1 ; AVX512BF16-NEXT: vmovss %xmm6, %xmm7, %xmm7 {%k1} ; AVX512BF16-NEXT: vcvtps2ph $4, %xmm7, %xmm5 @@ -2036,6 +2509,8 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind ; AVX512BF16-NEXT: vcvtph2ps %xmm0, %xmm0 ; AVX512BF16-NEXT: vmaxss %xmm1, %xmm0, %xmm6 ; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm6 = xmm6 & (xmm0 | xmm2) +; AVX512BF16-NEXT: vcmpunordss %xmm0, %xmm0, %k1 +; AVX512BF16-NEXT: vmovss %xmm1, %xmm6, %xmm6 {%k1} ; AVX512BF16-NEXT: vcmpunordss %xmm1, %xmm1, %k1 ; AVX512BF16-NEXT: vmovss %xmm0, %xmm6, %xmm6 {%k1} ; AVX512BF16-NEXT: vcvtps2ph $4, %xmm6, %xmm0 @@ -2086,6 +2561,8 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind ; X86-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero ; X86-NEXT: vmaxss %xmm2, %xmm0, %xmm3 ; X86-NEXT: vandps %xmm3, %xmm1, %xmm1 +; X86-NEXT: vcmpunordss %xmm0, %xmm0, %xmm3 +; X86-NEXT: vblendvps %xmm3, %xmm2, %xmm1, %xmm1 ; X86-NEXT: vcmpunordss %xmm2, %xmm2, %xmm2 ; X86-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0 ; X86-NEXT: vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill @@ -2100,6 +2577,8 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind ; X86-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero ; X86-NEXT: vmaxss %xmm2, %xmm0, %xmm3 ; X86-NEXT: vandps %xmm3, %xmm1, %xmm1 +; X86-NEXT: vcmpunordss %xmm0, %xmm0, %xmm3 +; X86-NEXT: vblendvps %xmm3, %xmm2, %xmm1, %xmm1 ; X86-NEXT: vcmpunordss %xmm2, %xmm2, %xmm2 ; X86-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0 ; X86-NEXT: vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill @@ -2130,6 +2609,8 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind ; X86-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero ; X86-NEXT: vmaxss %xmm2, %xmm0, %xmm3 ; X86-NEXT: vandps %xmm3, %xmm1, %xmm1 +; X86-NEXT: vcmpunordss %xmm0, %xmm0, %xmm3 +; X86-NEXT: vblendvps %xmm3, %xmm2, %xmm1, %xmm1 ; X86-NEXT: vcmpunordss %xmm2, %xmm2, %xmm2 ; X86-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0 ; X86-NEXT: vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill @@ -2144,6 +2625,8 @@ define <4 x half> @test_fmaximumnum_v4f16(<4 x half> %x, <4 x half> %y) nounwind ; X86-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero ; X86-NEXT: vmaxss %xmm2, %xmm0, %xmm3 ; X86-NEXT: vandps %xmm3, %xmm1, %xmm1 +; X86-NEXT: vcmpunordss %xmm0, %xmm0, %xmm3 +; X86-NEXT: vblendvps %xmm3, %xmm2, %xmm1, %xmm1 ; X86-NEXT: vcmpunordss %xmm2, %xmm2, %xmm2 ; X86-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0 ; X86-NEXT: vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill @@ -2168,20 +2651,27 @@ define bfloat @test_fmaximumnum_bf16(bfloat %x, bfloat %y) nounwind { ; SSE2-LABEL: test_fmaximumnum_bf16: ; SSE2: # %bb.0: ; SSE2-NEXT: pushq %rax -; SSE2-NEXT: pextrw $0, %xmm1, %eax -; SSE2-NEXT: pextrw $0, %xmm0, %ecx +; SSE2-NEXT: pextrw $0, %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm1, %ecx ; SSE2-NEXT: shll $16, %ecx -; SSE2-NEXT: movd %ecx, %xmm1 +; SSE2-NEXT: movd %ecx, %xmm0 ; SSE2-NEXT: shll $16, %eax -; SSE2-NEXT: movd %eax, %xmm0 +; SSE2-NEXT: movd %eax, %xmm1 ; SSE2-NEXT: movdqa %xmm1, %xmm2 -; SSE2-NEXT: maxss %xmm0, %xmm2 +; SSE2-NEXT: cmpunordss %xmm1, %xmm2 +; SSE2-NEXT: movaps %xmm2, %xmm3 +; SSE2-NEXT: andps %xmm0, %xmm3 +; SSE2-NEXT: movaps %xmm1, %xmm4 +; SSE2-NEXT: maxss %xmm0, %xmm4 +; SSE2-NEXT: movaps {{.*#+}} xmm5 = [NaN,NaN,NaN,NaN] +; SSE2-NEXT: orps %xmm1, %xmm5 +; SSE2-NEXT: andps %xmm4, %xmm5 +; SSE2-NEXT: andnps %xmm5, %xmm2 +; SSE2-NEXT: orps %xmm3, %xmm2 ; SSE2-NEXT: cmpunordss %xmm0, %xmm0 ; SSE2-NEXT: movaps %xmm0, %xmm3 -; SSE2-NEXT: andps %xmm1, %xmm3 -; SSE2-NEXT: orps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; SSE2-NEXT: andps %xmm2, %xmm1 -; SSE2-NEXT: andnps %xmm1, %xmm0 +; SSE2-NEXT: andnps %xmm2, %xmm3 +; SSE2-NEXT: andps %xmm1, %xmm0 ; SSE2-NEXT: orps %xmm3, %xmm0 ; SSE2-NEXT: callq __truncsfbf2@PLT ; SSE2-NEXT: popq %rax @@ -2199,6 +2689,8 @@ define bfloat @test_fmaximumnum_bf16(bfloat %x, bfloat %y) nounwind { ; AVX1-NEXT: vmaxss %xmm0, %xmm1, %xmm2 ; AVX1-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm3 ; AVX1-NEXT: vandps %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vcmpunordss %xmm1, %xmm1, %xmm3 +; AVX1-NEXT: vblendvps %xmm3, %xmm0, %xmm2, %xmm2 ; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm0 ; AVX1-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0 ; AVX1-NEXT: callq __truncsfbf2@PLT @@ -2218,6 +2710,8 @@ define bfloat @test_fmaximumnum_bf16(bfloat %x, bfloat %y) nounwind { ; AVX512F-NEXT: vbroadcastss {{.*#+}} xmm3 = [NaN,NaN,NaN,NaN] ; AVX512F-NEXT: vorps %xmm3, %xmm2, %xmm3 ; AVX512F-NEXT: vandps %xmm0, %xmm3, %xmm0 +; AVX512F-NEXT: vcmpunordss %xmm2, %xmm2, %k1 +; AVX512F-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1} ; AVX512F-NEXT: vcmpunordss %xmm1, %xmm1, %k1 ; AVX512F-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1} ; AVX512F-NEXT: callq __truncsfbf2@PLT @@ -2237,6 +2731,8 @@ define bfloat @test_fmaximumnum_bf16(bfloat %x, bfloat %y) nounwind { ; AVX512DQ-NEXT: vbroadcastss {{.*#+}} xmm3 = [NaN,NaN,NaN,NaN] ; AVX512DQ-NEXT: vorps %xmm3, %xmm2, %xmm3 ; AVX512DQ-NEXT: vandps %xmm0, %xmm3, %xmm0 +; AVX512DQ-NEXT: vcmpunordss %xmm2, %xmm2, %k1 +; AVX512DQ-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1} ; AVX512DQ-NEXT: vcmpunordss %xmm1, %xmm1, %k1 ; AVX512DQ-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1} ; AVX512DQ-NEXT: callq __truncsfbf2@PLT @@ -2253,6 +2749,8 @@ define bfloat @test_fmaximumnum_bf16(bfloat %x, bfloat %y) nounwind { ; AVX512BF16-NEXT: vmovd %eax, %xmm1 ; AVX512BF16-NEXT: vmaxss %xmm0, %xmm1, %xmm2 ; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm2 = xmm2 & (xmm1 | m32bcst) +; AVX512BF16-NEXT: vcmpunordss %xmm1, %xmm1, %k1 +; AVX512BF16-NEXT: vmovss %xmm0, %xmm2, %xmm2 {%k1} ; AVX512BF16-NEXT: vcmpunordss %xmm0, %xmm0, %k1 ; AVX512BF16-NEXT: vmovss %xmm1, %xmm2, %xmm2 {%k1} ; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm0 @@ -2282,6 +2780,8 @@ define bfloat @test_fmaximumnum_bf16(bfloat %x, bfloat %y) nounwind { ; X86-NEXT: vmaxss %xmm0, %xmm1, %xmm2 ; X86-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1, %xmm3 ; X86-NEXT: vandps %xmm2, %xmm3, %xmm2 +; X86-NEXT: vcmpunordss %xmm1, %xmm1, %xmm3 +; X86-NEXT: vblendvps %xmm3, %xmm0, %xmm2, %xmm2 ; X86-NEXT: vcmpunordss %xmm0, %xmm0, %xmm0 ; X86-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0 ; X86-NEXT: vmovss %xmm0, (%esp) @@ -2302,81 +2802,106 @@ define <4 x bfloat> @test_fmaximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) n ; SSE2-NEXT: pushq %r12 ; SSE2-NEXT: pushq %rbx ; SSE2-NEXT: subq $56, %rsp -; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: movdqa %xmm0, %xmm2 ; SSE2-NEXT: psrlq $48, %xmm2 ; SSE2-NEXT: pextrw $0, %xmm2, %ebx -; SSE2-NEXT: movdqa %xmm0, %xmm2 +; SSE2-NEXT: movdqa %xmm1, %xmm2 ; SSE2-NEXT: psrlq $48, %xmm2 ; SSE2-NEXT: pextrw $0, %xmm2, %ebp -; SSE2-NEXT: movdqa %xmm1, %xmm2 -; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,1],xmm1[1,1] -; SSE2-NEXT: pextrw $0, %xmm2, %r14d ; SSE2-NEXT: movdqa %xmm0, %xmm2 ; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,1],xmm0[1,1] +; SSE2-NEXT: pextrw $0, %xmm2, %r14d +; SSE2-NEXT: movdqa %xmm1, %xmm2 +; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,1],xmm1[1,1] ; SSE2-NEXT: pextrw $0, %xmm2, %r15d -; SSE2-NEXT: pextrw $0, %xmm1, %r12d -; SSE2-NEXT: pextrw $0, %xmm0, %r13d -; SSE2-NEXT: psrld $16, %xmm1 -; SSE2-NEXT: pextrw $0, %xmm1, %eax +; SSE2-NEXT: pextrw $0, %xmm0, %r12d +; SSE2-NEXT: pextrw $0, %xmm1, %r13d ; SSE2-NEXT: psrld $16, %xmm0 -; SSE2-NEXT: pextrw $0, %xmm0, %ecx +; SSE2-NEXT: pextrw $0, %xmm0, %eax +; SSE2-NEXT: psrld $16, %xmm1 +; SSE2-NEXT: pextrw $0, %xmm1, %ecx ; SSE2-NEXT: shll $16, %ecx -; SSE2-NEXT: movd %ecx, %xmm1 +; SSE2-NEXT: movd %ecx, %xmm0 ; SSE2-NEXT: shll $16, %eax -; SSE2-NEXT: movd %eax, %xmm0 +; SSE2-NEXT: movd %eax, %xmm1 ; SSE2-NEXT: movdqa %xmm1, %xmm2 -; SSE2-NEXT: maxss %xmm0, %xmm2 +; SSE2-NEXT: cmpunordss %xmm1, %xmm2 +; SSE2-NEXT: movaps %xmm2, %xmm3 +; SSE2-NEXT: andps %xmm0, %xmm3 +; SSE2-NEXT: movaps %xmm1, %xmm4 +; SSE2-NEXT: maxss %xmm0, %xmm4 ; SSE2-NEXT: cmpunordss %xmm0, %xmm0 -; SSE2-NEXT: movaps %xmm0, %xmm3 -; SSE2-NEXT: andps %xmm1, %xmm3 +; SSE2-NEXT: movaps %xmm0, %xmm5 +; SSE2-NEXT: andps %xmm1, %xmm0 ; SSE2-NEXT: orps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; SSE2-NEXT: andps %xmm2, %xmm1 -; SSE2-NEXT: andnps %xmm1, %xmm0 -; SSE2-NEXT: orps %xmm3, %xmm0 +; SSE2-NEXT: andps %xmm4, %xmm1 +; SSE2-NEXT: andnps %xmm1, %xmm2 +; SSE2-NEXT: orps %xmm3, %xmm2 +; SSE2-NEXT: andnps %xmm2, %xmm5 +; SSE2-NEXT: orps %xmm5, %xmm0 ; SSE2-NEXT: callq __truncsfbf2@PLT ; SSE2-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE2-NEXT: shll $16, %r13d -; SSE2-NEXT: movd %r13d, %xmm1 +; SSE2-NEXT: movd %r13d, %xmm0 ; SSE2-NEXT: shll $16, %r12d -; SSE2-NEXT: movd %r12d, %xmm0 +; SSE2-NEXT: movd %r12d, %xmm1 ; SSE2-NEXT: movdqa %xmm1, %xmm2 -; SSE2-NEXT: maxss %xmm0, %xmm2 +; SSE2-NEXT: cmpunordss %xmm1, %xmm2 +; SSE2-NEXT: movaps %xmm2, %xmm3 +; SSE2-NEXT: andps %xmm0, %xmm3 +; SSE2-NEXT: movaps %xmm1, %xmm4 +; SSE2-NEXT: maxss %xmm0, %xmm4 ; SSE2-NEXT: cmpunordss %xmm0, %xmm0 -; SSE2-NEXT: movaps %xmm0, %xmm3 -; SSE2-NEXT: andps %xmm1, %xmm3 +; SSE2-NEXT: movaps %xmm0, %xmm5 +; SSE2-NEXT: andps %xmm1, %xmm0 ; SSE2-NEXT: orps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; SSE2-NEXT: andps %xmm2, %xmm1 -; SSE2-NEXT: andnps %xmm1, %xmm0 -; SSE2-NEXT: orps %xmm3, %xmm0 +; SSE2-NEXT: andps %xmm4, %xmm1 +; SSE2-NEXT: andnps %xmm1, %xmm2 +; SSE2-NEXT: orps %xmm3, %xmm2 +; SSE2-NEXT: andnps %xmm2, %xmm5 +; SSE2-NEXT: orps %xmm5, %xmm0 ; SSE2-NEXT: callq __truncsfbf2@PLT ; SSE2-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE2-NEXT: shll $16, %r15d -; SSE2-NEXT: movd %r15d, %xmm1 +; SSE2-NEXT: movd %r15d, %xmm0 ; SSE2-NEXT: shll $16, %r14d -; SSE2-NEXT: movd %r14d, %xmm0 +; SSE2-NEXT: movd %r14d, %xmm1 ; SSE2-NEXT: movdqa %xmm1, %xmm2 -; SSE2-NEXT: maxss %xmm0, %xmm2 +; SSE2-NEXT: cmpunordss %xmm1, %xmm2 +; SSE2-NEXT: movaps %xmm2, %xmm3 +; SSE2-NEXT: andps %xmm0, %xmm3 +; SSE2-NEXT: movaps %xmm1, %xmm4 +; SSE2-NEXT: maxss %xmm0, %xmm4 ; SSE2-NEXT: cmpunordss %xmm0, %xmm0 -; SSE2-NEXT: movaps %xmm0, %xmm3 -; SSE2-NEXT: andps %xmm1, %xmm3 +; SSE2-NEXT: movaps %xmm0, %xmm5 +; SSE2-NEXT: andps %xmm1, %xmm0 ; SSE2-NEXT: orps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; SSE2-NEXT: andps %xmm2, %xmm1 -; SSE2-NEXT: andnps %xmm1, %xmm0 -; SSE2-NEXT: orps %xmm3, %xmm0 +; SSE2-NEXT: andps %xmm4, %xmm1 +; SSE2-NEXT: andnps %xmm1, %xmm2 +; SSE2-NEXT: orps %xmm3, %xmm2 +; SSE2-NEXT: andnps %xmm2, %xmm5 +; SSE2-NEXT: orps %xmm5, %xmm0 ; SSE2-NEXT: callq __truncsfbf2@PLT ; SSE2-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill ; SSE2-NEXT: shll $16, %ebp -; SSE2-NEXT: movd %ebp, %xmm1 +; SSE2-NEXT: movd %ebp, %xmm0 ; SSE2-NEXT: shll $16, %ebx -; SSE2-NEXT: movd %ebx, %xmm0 +; SSE2-NEXT: movd %ebx, %xmm1 ; SSE2-NEXT: movdqa %xmm1, %xmm2 -; SSE2-NEXT: maxss %xmm0, %xmm2 +; SSE2-NEXT: cmpunordss %xmm1, %xmm2 +; SSE2-NEXT: movaps %xmm2, %xmm3 +; SSE2-NEXT: andps %xmm0, %xmm3 +; SSE2-NEXT: movaps %xmm1, %xmm4 +; SSE2-NEXT: maxss %xmm0, %xmm4 +; SSE2-NEXT: movaps {{.*#+}} xmm5 = [NaN,NaN,NaN,NaN] +; SSE2-NEXT: orps %xmm1, %xmm5 +; SSE2-NEXT: andps %xmm4, %xmm5 +; SSE2-NEXT: andnps %xmm5, %xmm2 +; SSE2-NEXT: orps %xmm3, %xmm2 ; SSE2-NEXT: cmpunordss %xmm0, %xmm0 ; SSE2-NEXT: movaps %xmm0, %xmm3 -; SSE2-NEXT: andps %xmm1, %xmm3 -; SSE2-NEXT: orps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; SSE2-NEXT: andps %xmm2, %xmm1 -; SSE2-NEXT: andnps %xmm1, %xmm0 +; SSE2-NEXT: andnps %xmm2, %xmm3 +; SSE2-NEXT: andps %xmm1, %xmm0 ; SSE2-NEXT: orps %xmm3, %xmm0 ; SSE2-NEXT: callq __truncsfbf2@PLT ; SSE2-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload @@ -2424,6 +2949,8 @@ define <4 x bfloat> @test_fmaximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) n ; AVX1-NEXT: vmaxss %xmm0, %xmm1, %xmm2 ; AVX1-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm3 ; AVX1-NEXT: vandps %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vcmpunordss %xmm1, %xmm1, %xmm3 +; AVX1-NEXT: vblendvps %xmm3, %xmm0, %xmm2, %xmm2 ; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm0 ; AVX1-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0 ; AVX1-NEXT: callq __truncsfbf2@PLT @@ -2435,6 +2962,8 @@ define <4 x bfloat> @test_fmaximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) n ; AVX1-NEXT: vmaxss %xmm0, %xmm1, %xmm2 ; AVX1-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm3 ; AVX1-NEXT: vandps %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vcmpunordss %xmm1, %xmm1, %xmm3 +; AVX1-NEXT: vblendvps %xmm3, %xmm0, %xmm2, %xmm2 ; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm0 ; AVX1-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0 ; AVX1-NEXT: callq __truncsfbf2@PLT @@ -2446,6 +2975,8 @@ define <4 x bfloat> @test_fmaximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) n ; AVX1-NEXT: vmaxss %xmm0, %xmm1, %xmm2 ; AVX1-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm3 ; AVX1-NEXT: vandps %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vcmpunordss %xmm1, %xmm1, %xmm3 +; AVX1-NEXT: vblendvps %xmm3, %xmm0, %xmm2, %xmm2 ; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm0 ; AVX1-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0 ; AVX1-NEXT: callq __truncsfbf2@PLT @@ -2457,6 +2988,8 @@ define <4 x bfloat> @test_fmaximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) n ; AVX1-NEXT: vmaxss %xmm0, %xmm1, %xmm2 ; AVX1-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm3 ; AVX1-NEXT: vandps %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vcmpunordss %xmm1, %xmm1, %xmm3 +; AVX1-NEXT: vblendvps %xmm3, %xmm0, %xmm2, %xmm2 ; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm0 ; AVX1-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0 ; AVX1-NEXT: callq __truncsfbf2@PLT @@ -2504,6 +3037,8 @@ define <4 x bfloat> @test_fmaximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) n ; AVX512F-NEXT: vmovd %eax, %xmm2 ; AVX512F-NEXT: vmaxss %xmm2, %xmm1, %xmm3 ; AVX512F-NEXT: vpand %xmm3, %xmm0, %xmm0 +; AVX512F-NEXT: vcmpunordss %xmm1, %xmm1, %k1 +; AVX512F-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1} ; AVX512F-NEXT: vcmpunordss %xmm2, %xmm2, %k1 ; AVX512F-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1} ; AVX512F-NEXT: callq __truncsfbf2@PLT @@ -2516,6 +3051,8 @@ define <4 x bfloat> @test_fmaximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) n ; AVX512F-NEXT: vmovd %r13d, %xmm2 ; AVX512F-NEXT: vmaxss %xmm2, %xmm1, %xmm3 ; AVX512F-NEXT: vpand %xmm3, %xmm0, %xmm0 +; AVX512F-NEXT: vcmpunordss %xmm1, %xmm1, %k1 +; AVX512F-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1} ; AVX512F-NEXT: vcmpunordss %xmm2, %xmm2, %k1 ; AVX512F-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1} ; AVX512F-NEXT: callq __truncsfbf2@PLT @@ -2528,6 +3065,8 @@ define <4 x bfloat> @test_fmaximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) n ; AVX512F-NEXT: vmovd %r15d, %xmm2 ; AVX512F-NEXT: vmaxss %xmm2, %xmm1, %xmm3 ; AVX512F-NEXT: vpand %xmm3, %xmm0, %xmm0 +; AVX512F-NEXT: vcmpunordss %xmm1, %xmm1, %k1 +; AVX512F-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1} ; AVX512F-NEXT: vcmpunordss %xmm2, %xmm2, %k1 ; AVX512F-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1} ; AVX512F-NEXT: callq __truncsfbf2@PLT @@ -2540,6 +3079,8 @@ define <4 x bfloat> @test_fmaximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) n ; AVX512F-NEXT: vmovd %ebx, %xmm2 ; AVX512F-NEXT: vmaxss %xmm2, %xmm1, %xmm3 ; AVX512F-NEXT: vpand %xmm3, %xmm0, %xmm0 +; AVX512F-NEXT: vcmpunordss %xmm1, %xmm1, %k1 +; AVX512F-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1} ; AVX512F-NEXT: vcmpunordss %xmm2, %xmm2, %k1 ; AVX512F-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1} ; AVX512F-NEXT: callq __truncsfbf2@PLT @@ -2583,6 +3124,8 @@ define <4 x bfloat> @test_fmaximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) n ; AVX512DQ-NEXT: vmovd %eax, %xmm2 ; AVX512DQ-NEXT: vmaxss %xmm2, %xmm1, %xmm3 ; AVX512DQ-NEXT: vpand %xmm3, %xmm0, %xmm0 +; AVX512DQ-NEXT: vcmpunordss %xmm1, %xmm1, %k1 +; AVX512DQ-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1} ; AVX512DQ-NEXT: vcmpunordss %xmm2, %xmm2, %k1 ; AVX512DQ-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1} ; AVX512DQ-NEXT: callq __truncsfbf2@PLT @@ -2595,6 +3138,8 @@ define <4 x bfloat> @test_fmaximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) n ; AVX512DQ-NEXT: vmovd %r13d, %xmm2 ; AVX512DQ-NEXT: vmaxss %xmm2, %xmm1, %xmm3 ; AVX512DQ-NEXT: vpand %xmm3, %xmm0, %xmm0 +; AVX512DQ-NEXT: vcmpunordss %xmm1, %xmm1, %k1 +; AVX512DQ-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1} ; AVX512DQ-NEXT: vcmpunordss %xmm2, %xmm2, %k1 ; AVX512DQ-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1} ; AVX512DQ-NEXT: callq __truncsfbf2@PLT @@ -2607,6 +3152,8 @@ define <4 x bfloat> @test_fmaximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) n ; AVX512DQ-NEXT: vmovd %r15d, %xmm2 ; AVX512DQ-NEXT: vmaxss %xmm2, %xmm1, %xmm3 ; AVX512DQ-NEXT: vpand %xmm3, %xmm0, %xmm0 +; AVX512DQ-NEXT: vcmpunordss %xmm1, %xmm1, %k1 +; AVX512DQ-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1} ; AVX512DQ-NEXT: vcmpunordss %xmm2, %xmm2, %k1 ; AVX512DQ-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1} ; AVX512DQ-NEXT: callq __truncsfbf2@PLT @@ -2619,6 +3166,8 @@ define <4 x bfloat> @test_fmaximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) n ; AVX512DQ-NEXT: vmovd %ebx, %xmm2 ; AVX512DQ-NEXT: vmaxss %xmm2, %xmm1, %xmm3 ; AVX512DQ-NEXT: vpand %xmm3, %xmm0, %xmm0 +; AVX512DQ-NEXT: vcmpunordss %xmm1, %xmm1, %k1 +; AVX512DQ-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1} ; AVX512DQ-NEXT: vcmpunordss %xmm2, %xmm2, %k1 ; AVX512DQ-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1} ; AVX512DQ-NEXT: callq __truncsfbf2@PLT @@ -2872,6 +3421,8 @@ define <4 x bfloat> @test_fmaximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) n ; X86-NEXT: vbroadcastss {{.*#+}} xmm4 = [NaN,NaN,NaN,NaN] ; X86-NEXT: vorps %xmm4, %xmm0, %xmm3 ; X86-NEXT: vandps %xmm1, %xmm3, %xmm1 +; X86-NEXT: vcmpunordss %xmm0, %xmm0, %xmm3 +; X86-NEXT: vblendvps %xmm3, %xmm2, %xmm1, %xmm1 ; X86-NEXT: vcmpunordss %xmm2, %xmm2, %xmm2 ; X86-NEXT: vblendvps %xmm2, %xmm0, %xmm1, %xmm0 ; X86-NEXT: vmovss %xmm0, (%esp) @@ -2882,6 +3433,8 @@ define <4 x bfloat> @test_fmaximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) n ; X86-NEXT: vmaxss %xmm0, %xmm1, %xmm2 ; X86-NEXT: vorps %xmm4, %xmm1, %xmm3 ; X86-NEXT: vandps %xmm2, %xmm3, %xmm2 +; X86-NEXT: vcmpunordss %xmm1, %xmm1, %xmm3 +; X86-NEXT: vblendvps %xmm3, %xmm0, %xmm2, %xmm2 ; X86-NEXT: vcmpunordss %xmm0, %xmm0, %xmm0 ; X86-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0 ; X86-NEXT: vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill @@ -2896,6 +3449,8 @@ define <4 x bfloat> @test_fmaximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) n ; X86-NEXT: vmaxss %xmm0, %xmm1, %xmm2 ; X86-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1, %xmm3 ; X86-NEXT: vandps %xmm2, %xmm3, %xmm2 +; X86-NEXT: vcmpunordss %xmm1, %xmm1, %xmm3 +; X86-NEXT: vblendvps %xmm3, %xmm0, %xmm2, %xmm2 ; X86-NEXT: vcmpunordss %xmm0, %xmm0, %xmm0 ; X86-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0 ; X86-NEXT: vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill @@ -2910,6 +3465,8 @@ define <4 x bfloat> @test_fmaximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) n ; X86-NEXT: vmaxss %xmm0, %xmm1, %xmm2 ; X86-NEXT: vorps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1, %xmm3 ; X86-NEXT: vandps %xmm2, %xmm3, %xmm2 +; X86-NEXT: vcmpunordss %xmm1, %xmm1, %xmm3 +; X86-NEXT: vblendvps %xmm3, %xmm0, %xmm2, %xmm2 ; X86-NEXT: vcmpunordss %xmm0, %xmm0, %xmm0 ; X86-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0 ; X86-NEXT: vmovups %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill @@ -2938,20 +3495,27 @@ define bfloat @test_fminimumnum_bf16(bfloat %x, bfloat %y) nounwind { ; SSE2-LABEL: test_fminimumnum_bf16: ; SSE2: # %bb.0: ; SSE2-NEXT: pushq %rax -; SSE2-NEXT: pextrw $0, %xmm1, %eax -; SSE2-NEXT: pextrw $0, %xmm0, %ecx +; SSE2-NEXT: pextrw $0, %xmm0, %eax +; SSE2-NEXT: pextrw $0, %xmm1, %ecx ; SSE2-NEXT: shll $16, %ecx -; SSE2-NEXT: movd %ecx, %xmm1 +; SSE2-NEXT: movd %ecx, %xmm0 ; SSE2-NEXT: shll $16, %eax -; SSE2-NEXT: movd %eax, %xmm0 +; SSE2-NEXT: movd %eax, %xmm1 ; SSE2-NEXT: movdqa %xmm1, %xmm2 -; SSE2-NEXT: minss %xmm0, %xmm2 +; SSE2-NEXT: cmpunordss %xmm1, %xmm2 +; SSE2-NEXT: movaps %xmm2, %xmm3 +; SSE2-NEXT: andps %xmm0, %xmm3 +; SSE2-NEXT: movaps %xmm1, %xmm4 +; SSE2-NEXT: minss %xmm0, %xmm4 +; SSE2-NEXT: movaps {{.*#+}} xmm5 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0] +; SSE2-NEXT: andps %xmm1, %xmm5 +; SSE2-NEXT: orps %xmm4, %xmm5 +; SSE2-NEXT: andnps %xmm5, %xmm2 +; SSE2-NEXT: orps %xmm3, %xmm2 ; SSE2-NEXT: cmpunordss %xmm0, %xmm0 ; SSE2-NEXT: movaps %xmm0, %xmm3 -; SSE2-NEXT: andps %xmm1, %xmm3 -; SSE2-NEXT: andps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 -; SSE2-NEXT: orps %xmm2, %xmm1 -; SSE2-NEXT: andnps %xmm1, %xmm0 +; SSE2-NEXT: andnps %xmm2, %xmm3 +; SSE2-NEXT: andps %xmm1, %xmm0 ; SSE2-NEXT: orps %xmm3, %xmm0 ; SSE2-NEXT: callq __truncsfbf2@PLT ; SSE2-NEXT: popq %rax @@ -2969,6 +3533,8 @@ define bfloat @test_fminimumnum_bf16(bfloat %x, bfloat %y) nounwind { ; AVX1-NEXT: vminss %xmm0, %xmm1, %xmm2 ; AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm3 ; AVX1-NEXT: vorps %xmm2, %xmm3, %xmm2 +; AVX1-NEXT: vcmpunordss %xmm1, %xmm1, %xmm3 +; AVX1-NEXT: vblendvps %xmm3, %xmm0, %xmm2, %xmm2 ; AVX1-NEXT: vcmpunordss %xmm0, %xmm0, %xmm0 ; AVX1-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0 ; AVX1-NEXT: callq __truncsfbf2@PLT @@ -2988,6 +3554,8 @@ define bfloat @test_fminimumnum_bf16(bfloat %x, bfloat %y) nounwind { ; AVX512F-NEXT: vbroadcastss {{.*#+}} xmm3 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0] ; AVX512F-NEXT: vandps %xmm3, %xmm2, %xmm3 ; AVX512F-NEXT: vorps %xmm0, %xmm3, %xmm0 +; AVX512F-NEXT: vcmpunordss %xmm2, %xmm2, %k1 +; AVX512F-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1} ; AVX512F-NEXT: vcmpunordss %xmm1, %xmm1, %k1 ; AVX512F-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1} ; AVX512F-NEXT: callq __truncsfbf2@PLT @@ -3007,6 +3575,8 @@ define bfloat @test_fminimumnum_bf16(bfloat %x, bfloat %y) nounwind { ; AVX512DQ-NEXT: vbroadcastss {{.*#+}} xmm3 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0] ; AVX512DQ-NEXT: vandps %xmm3, %xmm2, %xmm3 ; AVX512DQ-NEXT: vorps %xmm0, %xmm3, %xmm0 +; AVX512DQ-NEXT: vcmpunordss %xmm2, %xmm2, %k1 +; AVX512DQ-NEXT: vmovss %xmm1, %xmm0, %xmm0 {%k1} ; AVX512DQ-NEXT: vcmpunordss %xmm1, %xmm1, %k1 ; AVX512DQ-NEXT: vmovss %xmm2, %xmm0, %xmm0 {%k1} ; AVX512DQ-NEXT: callq __truncsfbf2@PLT @@ -3023,6 +3593,8 @@ define bfloat @test_fminimumnum_bf16(bfloat %x, bfloat %y) nounwind { ; AVX512BF16-NEXT: vmovd %eax, %xmm1 ; AVX512BF16-NEXT: vminss %xmm0, %xmm1, %xmm2 ; AVX512BF16-NEXT: vpternlogd {{.*#+}} xmm2 = xmm2 | (xmm1 & m32bcst) +; AVX512BF16-NEXT: vcmpunordss %xmm1, %xmm1, %k1 +; AVX512BF16-NEXT: vmovss %xmm0, %xmm2, %xmm2 {%k1} ; AVX512BF16-NEXT: vcmpunordss %xmm0, %xmm0, %k1 ; AVX512BF16-NEXT: vmovss %xmm1, %xmm2, %xmm2 {%k1} ; AVX512BF16-NEXT: vcvtneps2bf16 %xmm2, %xmm0 @@ -3052,6 +3624,8 @@ define bfloat @test_fminimumnum_bf16(bfloat %x, bfloat %y) nounwind { ; X86-NEXT: vminss %xmm0, %xmm1, %xmm2 ; X86-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1, %xmm3 ; X86-NEXT: vorps %xmm2, %xmm3, %xmm2 +; X86-NEXT: vcmpunordss %xmm1, %xmm1, %xmm3 +; X86-NEXT: vblendvps %xmm3, %xmm0, %xmm2, %xmm2 ; X86-NEXT: vcmpunordss %xmm0, %xmm0, %xmm0 ; X86-NEXT: vblendvps %xmm0, %xmm1, %xmm2, %xmm0 ; X86-NEXT: vmovss %xmm0, (%esp) @@ -3172,47 +3746,47 @@ define fp128 @test_fmaximumnum_fp128(fp128 %x, fp128 %y) nounwind { ; SSE2-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload ; SSE2-NEXT: testl %eax, %eax ; SSE2-NEXT: movaps %xmm0, %xmm1 -; SSE2-NEXT: jne .LBB41_2 +; SSE2-NEXT: jne .LBB44_2 ; SSE2-NEXT: # %bb.1: # %start ; SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; SSE2-NEXT: .LBB41_2: # %start +; SSE2-NEXT: .LBB44_2: # %start ; SSE2-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE2-NEXT: movaps %xmm0, %xmm1 ; SSE2-NEXT: callq __unordtf2@PLT ; SSE2-NEXT: testl %eax, %eax ; SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; SSE2-NEXT: jne .LBB41_4 +; SSE2-NEXT: jne .LBB44_4 ; SSE2-NEXT: # %bb.3: # %start ; SSE2-NEXT: movaps (%rsp), %xmm1 # 16-byte Reload -; SSE2-NEXT: .LBB41_4: # %start +; SSE2-NEXT: .LBB44_4: # %start ; SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; SSE2-NEXT: movaps %xmm1, (%rsp) # 16-byte Spill ; SSE2-NEXT: callq __gttf2@PLT ; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; SSE2-NEXT: testl %eax, %eax ; SSE2-NEXT: movdqa %xmm0, %xmm1 -; SSE2-NEXT: jg .LBB41_6 +; SSE2-NEXT: jg .LBB44_6 ; SSE2-NEXT: # %bb.5: # %start ; SSE2-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload -; SSE2-NEXT: .LBB41_6: # %start +; SSE2-NEXT: .LBB44_6: # %start ; SSE2-NEXT: movdqa %xmm1, (%rsp) # 16-byte Spill ; SSE2-NEXT: callq __trunctfsf2@PLT ; SSE2-NEXT: movaps (%rsp), %xmm2 # 16-byte Reload ; SSE2-NEXT: movd %xmm0, %eax ; SSE2-NEXT: testl %eax, %eax -; SSE2-NEXT: je .LBB41_8 +; SSE2-NEXT: je .LBB44_8 ; SSE2-NEXT: # %bb.7: # %start ; SSE2-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE2-NEXT: .LBB41_8: # %start +; SSE2-NEXT: .LBB44_8: # %start ; SSE2-NEXT: pxor %xmm1, %xmm1 ; SSE2-NEXT: movaps %xmm2, %xmm0 ; SSE2-NEXT: callq __eqtf2@PLT ; SSE2-NEXT: testl %eax, %eax ; SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload -; SSE2-NEXT: je .LBB41_10 +; SSE2-NEXT: je .LBB44_10 ; SSE2-NEXT: # %bb.9: # %start ; SSE2-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload -; SSE2-NEXT: .LBB41_10: # %start +; SSE2-NEXT: .LBB44_10: # %start ; SSE2-NEXT: addq $40, %rsp ; SSE2-NEXT: retq ; @@ -3226,47 +3800,47 @@ define fp128 @test_fmaximumnum_fp128(fp128 %x, fp128 %y) nounwind { ; AVX-NEXT: vmovaps (%rsp), %xmm0 # 16-byte Reload ; AVX-NEXT: testl %eax, %eax ; AVX-NEXT: vmovaps %xmm0, %xmm1 -; AVX-NEXT: jne .LBB41_2 +; AVX-NEXT: jne .LBB44_2 ; AVX-NEXT: # %bb.1: # %start ; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; AVX-NEXT: .LBB41_2: # %start +; AVX-NEXT: .LBB44_2: # %start ; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vmovaps %xmm0, %xmm1 ; AVX-NEXT: callq __unordtf2@PLT ; AVX-NEXT: testl %eax, %eax ; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; AVX-NEXT: jne .LBB41_4 +; AVX-NEXT: jne .LBB44_4 ; AVX-NEXT: # %bb.3: # %start ; AVX-NEXT: vmovaps (%rsp), %xmm1 # 16-byte Reload -; AVX-NEXT: .LBB41_4: # %start +; AVX-NEXT: .LBB44_4: # %start ; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; AVX-NEXT: vmovaps %xmm1, (%rsp) # 16-byte Spill ; AVX-NEXT: callq __gttf2@PLT ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; AVX-NEXT: testl %eax, %eax ; AVX-NEXT: vmovdqa %xmm0, %xmm1 -; AVX-NEXT: jg .LBB41_6 +; AVX-NEXT: jg .LBB44_6 ; AVX-NEXT: # %bb.5: # %start ; AVX-NEXT: vmovdqa (%rsp), %xmm1 # 16-byte Reload -; AVX-NEXT: .LBB41_6: # %start +; AVX-NEXT: .LBB44_6: # %start ; AVX-NEXT: vmovdqa %xmm1, (%rsp) # 16-byte Spill ; AVX-NEXT: callq __trunctfsf2@PLT ; AVX-NEXT: vmovaps (%rsp), %xmm2 # 16-byte Reload ; AVX-NEXT: vmovd %xmm0, %eax ; AVX-NEXT: testl %eax, %eax -; AVX-NEXT: je .LBB41_8 +; AVX-NEXT: je .LBB44_8 ; AVX-NEXT: # %bb.7: # %start ; AVX-NEXT: vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: .LBB41_8: # %start +; AVX-NEXT: .LBB44_8: # %start ; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1 ; AVX-NEXT: vmovaps %xmm2, %xmm0 ; AVX-NEXT: callq __eqtf2@PLT ; AVX-NEXT: testl %eax, %eax ; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload -; AVX-NEXT: je .LBB41_10 +; AVX-NEXT: je .LBB44_10 ; AVX-NEXT: # %bb.9: # %start ; AVX-NEXT: vmovaps (%rsp), %xmm0 # 16-byte Reload -; AVX-NEXT: .LBB41_10: # %start +; AVX-NEXT: .LBB44_10: # %start ; AVX-NEXT: addq $40, %rsp ; AVX-NEXT: retq ; @@ -3280,47 +3854,47 @@ define fp128 @test_fmaximumnum_fp128(fp128 %x, fp128 %y) nounwind { ; AVX10_2-NEXT: vmovaps (%rsp), %xmm0 # 16-byte Reload ; AVX10_2-NEXT: testl %eax, %eax ; AVX10_2-NEXT: vmovaps %xmm0, %xmm1 -; AVX10_2-NEXT: jne .LBB41_2 +; AVX10_2-NEXT: jne .LBB44_2 ; AVX10_2-NEXT: # %bb.1: # %start ; AVX10_2-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; AVX10_2-NEXT: .LBB41_2: # %start +; AVX10_2-NEXT: .LBB44_2: # %start ; AVX10_2-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX10_2-NEXT: vmovaps %xmm0, %xmm1 ; AVX10_2-NEXT: callq __unordtf2@PLT ; AVX10_2-NEXT: testl %eax, %eax ; AVX10_2-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; AVX10_2-NEXT: jne .LBB41_4 +; AVX10_2-NEXT: jne .LBB44_4 ; AVX10_2-NEXT: # %bb.3: # %start ; AVX10_2-NEXT: vmovaps (%rsp), %xmm1 # 16-byte Reload -; AVX10_2-NEXT: .LBB41_4: # %start +; AVX10_2-NEXT: .LBB44_4: # %start ; AVX10_2-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; AVX10_2-NEXT: vmovaps %xmm1, (%rsp) # 16-byte Spill ; AVX10_2-NEXT: callq __gttf2@PLT ; AVX10_2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; AVX10_2-NEXT: testl %eax, %eax ; AVX10_2-NEXT: vmovdqa %xmm0, %xmm1 -; AVX10_2-NEXT: jg .LBB41_6 +; AVX10_2-NEXT: jg .LBB44_6 ; AVX10_2-NEXT: # %bb.5: # %start ; AVX10_2-NEXT: vmovdqa (%rsp), %xmm1 # 16-byte Reload -; AVX10_2-NEXT: .LBB41_6: # %start +; AVX10_2-NEXT: .LBB44_6: # %start ; AVX10_2-NEXT: vmovdqa %xmm1, (%rsp) # 16-byte Spill ; AVX10_2-NEXT: callq __trunctfsf2@PLT ; AVX10_2-NEXT: vmovaps (%rsp), %xmm2 # 16-byte Reload ; AVX10_2-NEXT: vmovd %xmm0, %eax ; AVX10_2-NEXT: testl %eax, %eax -; AVX10_2-NEXT: je .LBB41_8 +; AVX10_2-NEXT: je .LBB44_8 ; AVX10_2-NEXT: # %bb.7: # %start ; AVX10_2-NEXT: vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX10_2-NEXT: .LBB41_8: # %start +; AVX10_2-NEXT: .LBB44_8: # %start ; AVX10_2-NEXT: vpxor %xmm1, %xmm1, %xmm1 ; AVX10_2-NEXT: vmovaps %xmm2, %xmm0 ; AVX10_2-NEXT: callq __eqtf2@PLT ; AVX10_2-NEXT: testl %eax, %eax ; AVX10_2-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload -; AVX10_2-NEXT: je .LBB41_10 +; AVX10_2-NEXT: je .LBB44_10 ; AVX10_2-NEXT: # %bb.9: # %start ; AVX10_2-NEXT: vmovaps (%rsp), %xmm0 # 16-byte Reload -; AVX10_2-NEXT: .LBB41_10: # %start +; AVX10_2-NEXT: .LBB44_10: # %start ; AVX10_2-NEXT: addq $40, %rsp ; AVX10_2-NEXT: retq ; @@ -3362,47 +3936,47 @@ define fp128 @test_fminimumnum_fp128(fp128 %x, fp128 %y) nounwind { ; SSE2-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload ; SSE2-NEXT: testl %eax, %eax ; SSE2-NEXT: movaps %xmm0, %xmm1 -; SSE2-NEXT: jne .LBB42_2 +; SSE2-NEXT: jne .LBB45_2 ; SSE2-NEXT: # %bb.1: # %start ; SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; SSE2-NEXT: .LBB42_2: # %start +; SSE2-NEXT: .LBB45_2: # %start ; SSE2-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE2-NEXT: movaps %xmm0, %xmm1 ; SSE2-NEXT: callq __unordtf2@PLT ; SSE2-NEXT: testl %eax, %eax ; SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; SSE2-NEXT: jne .LBB42_4 +; SSE2-NEXT: jne .LBB45_4 ; SSE2-NEXT: # %bb.3: # %start ; SSE2-NEXT: movaps (%rsp), %xmm1 # 16-byte Reload -; SSE2-NEXT: .LBB42_4: # %start +; SSE2-NEXT: .LBB45_4: # %start ; SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; SSE2-NEXT: movaps %xmm1, (%rsp) # 16-byte Spill ; SSE2-NEXT: callq __lttf2@PLT ; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; SSE2-NEXT: testl %eax, %eax ; SSE2-NEXT: movdqa %xmm0, %xmm1 -; SSE2-NEXT: js .LBB42_6 +; SSE2-NEXT: js .LBB45_6 ; SSE2-NEXT: # %bb.5: # %start ; SSE2-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload -; SSE2-NEXT: .LBB42_6: # %start +; SSE2-NEXT: .LBB45_6: # %start ; SSE2-NEXT: movdqa %xmm1, (%rsp) # 16-byte Spill ; SSE2-NEXT: callq __trunctfsf2@PLT ; SSE2-NEXT: movaps (%rsp), %xmm2 # 16-byte Reload ; SSE2-NEXT: movd %xmm0, %eax ; SSE2-NEXT: negl %eax -; SSE2-NEXT: jo .LBB42_8 +; SSE2-NEXT: jo .LBB45_8 ; SSE2-NEXT: # %bb.7: # %start ; SSE2-NEXT: movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE2-NEXT: .LBB42_8: # %start +; SSE2-NEXT: .LBB45_8: # %start ; SSE2-NEXT: pxor %xmm1, %xmm1 ; SSE2-NEXT: movaps %xmm2, %xmm0 ; SSE2-NEXT: callq __eqtf2@PLT ; SSE2-NEXT: testl %eax, %eax ; SSE2-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload -; SSE2-NEXT: je .LBB42_10 +; SSE2-NEXT: je .LBB45_10 ; SSE2-NEXT: # %bb.9: # %start ; SSE2-NEXT: movaps (%rsp), %xmm0 # 16-byte Reload -; SSE2-NEXT: .LBB42_10: # %start +; SSE2-NEXT: .LBB45_10: # %start ; SSE2-NEXT: addq $40, %rsp ; SSE2-NEXT: retq ; @@ -3416,47 +3990,47 @@ define fp128 @test_fminimumnum_fp128(fp128 %x, fp128 %y) nounwind { ; AVX-NEXT: vmovaps (%rsp), %xmm0 # 16-byte Reload ; AVX-NEXT: testl %eax, %eax ; AVX-NEXT: vmovaps %xmm0, %xmm1 -; AVX-NEXT: jne .LBB42_2 +; AVX-NEXT: jne .LBB45_2 ; AVX-NEXT: # %bb.1: # %start ; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; AVX-NEXT: .LBB42_2: # %start +; AVX-NEXT: .LBB45_2: # %start ; AVX-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX-NEXT: vmovaps %xmm0, %xmm1 ; AVX-NEXT: callq __unordtf2@PLT ; AVX-NEXT: testl %eax, %eax ; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; AVX-NEXT: jne .LBB42_4 +; AVX-NEXT: jne .LBB45_4 ; AVX-NEXT: # %bb.3: # %start ; AVX-NEXT: vmovaps (%rsp), %xmm1 # 16-byte Reload -; AVX-NEXT: .LBB42_4: # %start +; AVX-NEXT: .LBB45_4: # %start ; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; AVX-NEXT: vmovaps %xmm1, (%rsp) # 16-byte Spill ; AVX-NEXT: callq __lttf2@PLT ; AVX-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; AVX-NEXT: testl %eax, %eax ; AVX-NEXT: vmovdqa %xmm0, %xmm1 -; AVX-NEXT: js .LBB42_6 +; AVX-NEXT: js .LBB45_6 ; AVX-NEXT: # %bb.5: # %start ; AVX-NEXT: vmovdqa (%rsp), %xmm1 # 16-byte Reload -; AVX-NEXT: .LBB42_6: # %start +; AVX-NEXT: .LBB45_6: # %start ; AVX-NEXT: vmovdqa %xmm1, (%rsp) # 16-byte Spill ; AVX-NEXT: callq __trunctfsf2@PLT ; AVX-NEXT: vmovaps (%rsp), %xmm2 # 16-byte Reload ; AVX-NEXT: vmovd %xmm0, %eax ; AVX-NEXT: negl %eax -; AVX-NEXT: jo .LBB42_8 +; AVX-NEXT: jo .LBB45_8 ; AVX-NEXT: # %bb.7: # %start ; AVX-NEXT: vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX-NEXT: .LBB42_8: # %start +; AVX-NEXT: .LBB45_8: # %start ; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1 ; AVX-NEXT: vmovaps %xmm2, %xmm0 ; AVX-NEXT: callq __eqtf2@PLT ; AVX-NEXT: testl %eax, %eax ; AVX-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload -; AVX-NEXT: je .LBB42_10 +; AVX-NEXT: je .LBB45_10 ; AVX-NEXT: # %bb.9: # %start ; AVX-NEXT: vmovaps (%rsp), %xmm0 # 16-byte Reload -; AVX-NEXT: .LBB42_10: # %start +; AVX-NEXT: .LBB45_10: # %start ; AVX-NEXT: addq $40, %rsp ; AVX-NEXT: retq ; @@ -3470,47 +4044,47 @@ define fp128 @test_fminimumnum_fp128(fp128 %x, fp128 %y) nounwind { ; AVX10_2-NEXT: vmovaps (%rsp), %xmm0 # 16-byte Reload ; AVX10_2-NEXT: testl %eax, %eax ; AVX10_2-NEXT: vmovaps %xmm0, %xmm1 -; AVX10_2-NEXT: jne .LBB42_2 +; AVX10_2-NEXT: jne .LBB45_2 ; AVX10_2-NEXT: # %bb.1: # %start ; AVX10_2-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; AVX10_2-NEXT: .LBB42_2: # %start +; AVX10_2-NEXT: .LBB45_2: # %start ; AVX10_2-NEXT: vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; AVX10_2-NEXT: vmovaps %xmm0, %xmm1 ; AVX10_2-NEXT: callq __unordtf2@PLT ; AVX10_2-NEXT: testl %eax, %eax ; AVX10_2-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; AVX10_2-NEXT: jne .LBB42_4 +; AVX10_2-NEXT: jne .LBB45_4 ; AVX10_2-NEXT: # %bb.3: # %start ; AVX10_2-NEXT: vmovaps (%rsp), %xmm1 # 16-byte Reload -; AVX10_2-NEXT: .LBB42_4: # %start +; AVX10_2-NEXT: .LBB45_4: # %start ; AVX10_2-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; AVX10_2-NEXT: vmovaps %xmm1, (%rsp) # 16-byte Spill ; AVX10_2-NEXT: callq __lttf2@PLT ; AVX10_2-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; AVX10_2-NEXT: testl %eax, %eax ; AVX10_2-NEXT: vmovdqa %xmm0, %xmm1 -; AVX10_2-NEXT: js .LBB42_6 +; AVX10_2-NEXT: js .LBB45_6 ; AVX10_2-NEXT: # %bb.5: # %start ; AVX10_2-NEXT: vmovdqa (%rsp), %xmm1 # 16-byte Reload -; AVX10_2-NEXT: .LBB42_6: # %start +; AVX10_2-NEXT: .LBB45_6: # %start ; AVX10_2-NEXT: vmovdqa %xmm1, (%rsp) # 16-byte Spill ; AVX10_2-NEXT: callq __trunctfsf2@PLT ; AVX10_2-NEXT: vmovaps (%rsp), %xmm2 # 16-byte Reload ; AVX10_2-NEXT: vmovd %xmm0, %eax ; AVX10_2-NEXT: negl %eax -; AVX10_2-NEXT: jo .LBB42_8 +; AVX10_2-NEXT: jo .LBB45_8 ; AVX10_2-NEXT: # %bb.7: # %start ; AVX10_2-NEXT: vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; AVX10_2-NEXT: .LBB42_8: # %start +; AVX10_2-NEXT: .LBB45_8: # %start ; AVX10_2-NEXT: vpxor %xmm1, %xmm1, %xmm1 ; AVX10_2-NEXT: vmovaps %xmm2, %xmm0 ; AVX10_2-NEXT: callq __eqtf2@PLT ; AVX10_2-NEXT: testl %eax, %eax ; AVX10_2-NEXT: vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload -; AVX10_2-NEXT: je .LBB42_10 +; AVX10_2-NEXT: je .LBB45_10 ; AVX10_2-NEXT: # %bb.9: # %start ; AVX10_2-NEXT: vmovaps (%rsp), %xmm0 # 16-byte Reload -; AVX10_2-NEXT: .LBB42_10: # %start +; AVX10_2-NEXT: .LBB45_10: # %start ; AVX10_2-NEXT: addq $40, %rsp ; AVX10_2-NEXT: retq ; _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
