https://gcc.gnu.org/g:f9eedd09c994e73405f6ac19acfda4ce866e9d22
commit r17-5304-gf9eedd09c994e73405f6ac19acfda4ce866e9d22 Author: Uros Bizjak <[email protected]> Date: Fri Oct 16 16:34:19 2026 +0200 i386: Use MULX for highpart-only multiplications [PR127900] For the testcase u64 mulh (u64 a, u64 b) { return ((unsigned __int128) a * b) >> 64; } GCC emits MUL, which clobbers both %rax and %rdx and the flags, even with -mbmi2, because the highpart-only multiplication patterns have no MULX alternative. MULX can write the high half to any register and doesn't clobber flags, so it avoids the extra register moves and frees %rax. Split the unsigned highpart multiplication patterns from the signed ones, since only the unsigned ones have a MULX form. Convert the unsigned patterns to define_insn_and_split with two alternatives: a BMI2 one that takes operand 1 in %rdx and is split after reload to MULX, and the existing MUL one. The splitters generate the existing *bmi2_umul<mode>3_highpart pattern, or the new *bmi2_umulsi3_highpart_zext pattern for the zero-extended SImode form. The signed patterns keep a single IMUL alternative, with the mnemonic written explicitly instead of through <sgnprefix>. Also use register_operand for operand 1 of the highpart patterns that used nonimmediate_operand, which makes the !(MEM_P (operands[1]) && MEM_P (operands[2])) conditions redundant, and remove them. Mark operand 1 of *bmi2_umul<mode>3_highpart as commutative. The second multiplication in g++.target/i386/pr113560.C needs only the high part and now uses MULX instead of MULQ, and the register allocator no longer needs to preserve %rax across it, so the return value increment becomes ADDQ instead of LEAQ. PR target/127900 gcc/ChangeLog: * config/i386/i386.md (*bmi2_umul<mode>3_highpart): Mark operand 1 as commutative. (<s>mul<mode>3_highpart): Split into... (umul<mode>3_highpart): ... a define_insn_and_split with a BMI2 alternative and ... (smul<mode>3_highpart): ... a define_insn. (*bmi2_umulsi3_highpart_zext): New pattern. (*<s>mulsi3_highpart_zext): Split into... (*umulsi3_highpart_zext): ... a define_insn_and_split with a BMI2 alternative and ... (*smulsi3_highpart_zext): ... a define_insn. (*<s>muldi3_highpart_1): Split into... (*umuldi3_highpart_1): ... a define_insn_and_split with a BMI2 alternative and ... (*smuldi3_highpart_1): ... a define_insn. Use register_operand for operand 1 and remove redundant MEM_P condition. (*<s>mulsi3_highpart_zext): Split into... (*umulsi3_highpart_1_zext): ... a define_insn_and_split with a BMI2 alternative and ... (*smulsi3_highpart_1_zext): ... a define_insn. Use register_operand for operand 1 and remove redundant MEM_P condition. (*<s>mulsi3_highpart_1): Split into... (*umulsi3_highpart_1): ... a define_insn_and_split with a BMI2 alternative and ... (*smulsi3_highpart_1): ... a define_insn. Use register_operand for operand 1 and remove redundant MEM_P condition. gcc/testsuite/ChangeLog: * gcc.target/i386/pr127900.c: New test. Diff: --- gcc/config/i386/i386.md | 245 +++++++++++++++++++++++++++---- gcc/testsuite/gcc.target/i386/pr127900.c | 14 ++ 2 files changed, 227 insertions(+), 32 deletions(-) diff --git a/gcc/config/i386/i386.md b/gcc/config/i386/i386.md index 4935e12cfc14..ce6c3e6bf823 100644 --- a/gcc/config/i386/i386.md +++ b/gcc/config/i386/i386.md @@ -11933,14 +11933,17 @@ (set (match_dup 5) (umul_highpart:DWIH (match_dup 2) (match_dup 3)))])]) +;; Highpart multiplication patterns + ;; BMI2 MULX highpart-only pattern. Uses MULX to get only the high part. ;; When both output operands are identical, MULX writes the high half to that ;; register, so no scratch register is needed. This avoids the mov from rdx ;; after mulq when only the high part is needed. + (define_insn "*bmi2_umul<mode>3_highpart" [(set (match_operand:DWIH 0 "register_operand" "=r") (umul_highpart:DWIH - (match_operand:DWIH 1 "register_operand" "d") + (match_operand:DWIH 1 "register_operand" "%d") (match_operand:DWIH 2 "nonimmediate_operand" "rm")))] "TARGET_BMI2" "mulx\t{%2, %0, %0|%0, %0, %2}" @@ -11948,16 +11951,45 @@ (set_attr "prefix" "vex") (set_attr "mode" "<MODE>")]) -;; Highpart multiplication patterns -(define_insn "<s>mul<mode>3_highpart" +(define_insn_and_split "umul<mode>3_highpart" + [(set (match_operand:DWIH 0 "register_operand" "=r,d") + (umul_highpart:DWIH + (match_operand:DWIH 1 "register_operand" "%d,a") + (match_operand:DWIH 2 "nonimmediate_operand" "rm,rm"))) + (clobber (match_scratch:DWIH 3 "=X,1")) + (clobber (reg:CC FLAGS_REG))] + "" + "@ + # + mul{<imodesuffix>}\t%2" + "TARGET_BMI2 && reload_completed + && REGNO (operands[1]) == DX_REG" + [(set (match_dup 0) + (umul_highpart:DWIH (match_dup 1) (match_dup 2)))] + "" + [(set_attr "isa" "bmi2,*") + (set_attr "type" "imulx,imul") + (set_attr "length_immediate" "*,0") + (set (attr "athlon_decode") + (cond [(eq_attr "alternative" "1") + (if_then_else (eq_attr "cpu" "athlon") + (const_string "vector") + (const_string "double"))] + (const_string "*"))) + (set_attr "amdfam10_decode" "*,double") + (set_attr "bdver1_decode" "*,direct") + (set_attr "prefix" "vex,orig") + (set_attr "mode" "<MODE>")]) + +(define_insn "smul<mode>3_highpart" [(set (match_operand:DWIH 0 "register_operand" "=d") - (any_mul_highpart:DWIH + (smul_highpart:DWIH (match_operand:DWIH 1 "register_operand" "%a") (match_operand:DWIH 2 "nonimmediate_operand" "rm"))) (clobber (match_scratch:DWIH 3 "=1")) (clobber (reg:CC FLAGS_REG))] "" - "<sgnprefix>mul{<imodesuffix>}\t%2" + "imul{<imodesuffix>}\t%2" [(set_attr "type" "imul") (set_attr "length_immediate" "0") (set (attr "athlon_decode") @@ -11968,16 +12000,60 @@ (set_attr "bdver1_decode" "direct") (set_attr "mode" "<MODE>")]) -(define_insn "*<s>mulsi3_highpart_zext" +(define_insn "*bmi2_umulsi3_highpart_zext" + [(set (match_operand:DI 0 "register_operand" "=r") + (zero_extend:DI + (umul_highpart:SI + (match_operand:SI 1 "register_operand" "%d") + (match_operand:SI 2 "nonimmediate_operand" "rm"))))] + "TARGET_64BIT && TARGET_BMI2" + "mulx\t{%2, %k0, %k0|%k0, %k0, %2}" + [(set_attr "type" "imulx") + (set_attr "prefix" "vex") + (set_attr "mode" "SI")]) + +(define_insn_and_split "*umulsi3_highpart_zext" + [(set (match_operand:DI 0 "register_operand" "=r,d") + (zero_extend:DI + (umul_highpart:SI + (match_operand:SI 1 "register_operand" "%d,a") + (match_operand:SI 2 "nonimmediate_operand" "rm,rm")))) + (clobber (match_scratch:SI 3 "=X,1")) + (clobber (reg:CC FLAGS_REG))] + "TARGET_64BIT" + "@ + # + mul{l}\t%2" + "&& TARGET_BMI2 && reload_completed + && REGNO (operands[1]) == DX_REG" + [(set (match_dup 0) + (zero_extend:DI + (umul_highpart:SI (match_dup 1) (match_dup 2))))] + "" + [(set_attr "isa" "bmi2,*") + (set_attr "type" "imulx,imul") + (set_attr "length_immediate" "*,0") + (set (attr "athlon_decode") + (cond [(eq_attr "alternative" "1") + (if_then_else (eq_attr "cpu" "athlon") + (const_string "vector") + (const_string "double"))] + (const_string "*"))) + (set_attr "amdfam10_decode" "*,double") + (set_attr "bdver1_decode" "*,direct") + (set_attr "prefix" "vex,orig") + (set_attr "mode" "SI")]) + +(define_insn "*smulsi3_highpart_zext" [(set (match_operand:DI 0 "register_operand" "=d") (zero_extend:DI - (any_mul_highpart:SI + (smul_highpart:SI (match_operand:SI 1 "register_operand" "%a") (match_operand:SI 2 "nonimmediate_operand" "rm")))) (clobber (match_scratch:SI 3 "=1")) (clobber (reg:CC FLAGS_REG))] "TARGET_64BIT" - "<sgnprefix>mul{l}\t%2" + "imul{l}\t%2" [(set_attr "type" "imul") (set_attr "length_immediate" "0") (set (attr "athlon_decode") @@ -11988,21 +12064,55 @@ (set_attr "bdver1_decode" "direct") (set_attr "mode" "SI")]) -(define_insn "*<s>muldi3_highpart_1" +(define_insn_and_split "*umuldi3_highpart_1" + [(set (match_operand:DI 0 "register_operand" "=r,d") + (truncate:DI + (lshiftrt:TI + (mult:TI + (zero_extend:TI + (match_operand:DI 1 "register_operand" "%d,a")) + (zero_extend:TI + (match_operand:DI 2 "nonimmediate_operand" "rm,rm"))) + (const_int 64)))) + (clobber (match_scratch:DI 3 "=X,1")) + (clobber (reg:CC FLAGS_REG))] + "TARGET_64BIT" + "@ + # + mul{q}\t%2" + "&& TARGET_BMI2 && reload_completed + && REGNO (operands[1]) == DX_REG" + [(set (match_dup 0) + (umul_highpart:DI (match_dup 1) (match_dup 2)))] + "" + [(set_attr "isa" "bmi2,*") + (set_attr "type" "imulx,imul") + (set_attr "length_immediate" "*,0") + (set (attr "athlon_decode") + (cond [(eq_attr "alternative" "1") + (if_then_else (eq_attr "cpu" "athlon") + (const_string "vector") + (const_string "double"))] + (const_string "*"))) + (set_attr "amdfam10_decode" "*,double") + (set_attr "bdver1_decode" "*,direct") + (set_attr "prefix" "vex,orig") + (set_attr "mode" "DI")]) + +(define_insn "*smuldi3_highpart_1" [(set (match_operand:DI 0 "register_operand" "=d") (truncate:DI (lshiftrt:TI (mult:TI - (any_extend:TI - (match_operand:DI 1 "nonimmediate_operand" "%a")) - (any_extend:TI + (sign_extend:TI + (match_operand:DI 1 "register_operand" "%a")) + (sign_extend:TI (match_operand:DI 2 "nonimmediate_operand" "rm"))) (const_int 64)))) (clobber (match_scratch:DI 3 "=1")) (clobber (reg:CC FLAGS_REG))] - "TARGET_64BIT - && !(MEM_P (operands[1]) && MEM_P (operands[2]))" - "<sgnprefix>mul{q}\t%2" + "TARGET_64BIT" + "imul{q}\t%2" [(set_attr "type" "imul") (set_attr "length_immediate" "0") (set (attr "athlon_decode") @@ -12013,20 +12123,56 @@ (set_attr "bdver1_decode" "direct") (set_attr "mode" "DI")]) -(define_insn "*<s>mulsi3_highpart_zext" +(define_insn_and_split "*umulsi3_highpart_1_zext" + [(set (match_operand:DI 0 "register_operand" "=r,d") + (zero_extend:DI + (truncate:SI + (lshiftrt:DI + (mult:DI (zero_extend:DI + (match_operand:SI 1 "register_operand" "%d,a")) + (zero_extend:DI + (match_operand:SI 2 "nonimmediate_operand" "rm,rm"))) + (const_int 32))))) + (clobber (match_scratch:SI 3 "=X,1")) + (clobber (reg:CC FLAGS_REG))] + "TARGET_64BIT" + "@ + # + mul{l}\t%2" + "&& TARGET_BMI2 && reload_completed + && REGNO (operands[1]) == DX_REG" + [(set (match_dup 0) + (zero_extend:DI + (umul_highpart:SI (match_dup 1) (match_dup 2))))] + "" + [(set_attr "isa" "bmi2,*") + (set_attr "type" "imulx,imul") + (set_attr "length_immediate" "*,0") + (set (attr "athlon_decode") + (cond [(eq_attr "alternative" "1") + (if_then_else (eq_attr "cpu" "athlon") + (const_string "vector") + (const_string "double"))] + (const_string "*"))) + (set_attr "amdfam10_decode" "*,double") + (set_attr "bdver1_decode" "*,direct") + (set_attr "prefix" "vex,orig") + (set_attr "mode" "SI")]) + +(define_insn "*smulsi3_highpart_1_zext" [(set (match_operand:DI 0 "register_operand" "=d") - (zero_extend:DI (truncate:SI - (lshiftrt:DI - (mult:DI (any_extend:DI - (match_operand:SI 1 "nonimmediate_operand" "%a")) - (any_extend:DI - (match_operand:SI 2 "nonimmediate_operand" "rm"))) - (const_int 32))))) + (zero_extend:DI + (truncate:SI + (lshiftrt:DI + (mult:DI (sign_extend:DI + (match_operand:SI 1 "register_operand" "%a")) + (sign_extend:DI + (match_operand:SI 2 "nonimmediate_operand" "rm"))) + (const_int 32))))) (clobber (match_scratch:SI 3 "=1")) (clobber (reg:CC FLAGS_REG))] - "TARGET_64BIT - && !(MEM_P (operands[1]) && MEM_P (operands[2]))" - "<sgnprefix>mul{l}\t%2" + "TARGET_64BIT" + "imul{l}\t%2" [(set_attr "type" "imul") (set_attr "length_immediate" "0") (set (attr "athlon_decode") @@ -12037,20 +12183,55 @@ (set_attr "bdver1_decode" "direct") (set_attr "mode" "SI")]) -(define_insn "*<s>mulsi3_highpart_1" +(define_insn_and_split "*umulsi3_highpart_1" + [(set (match_operand:SI 0 "register_operand" "=r,d") + (truncate:SI + (lshiftrt:DI + (mult:DI + (zero_extend:DI + (match_operand:SI 1 "register_operand" "%d,a")) + (zero_extend:DI + (match_operand:SI 2 "nonimmediate_operand" "rm,rm"))) + (const_int 32)))) + (clobber (match_scratch:SI 3 "=X,1")) + (clobber (reg:CC FLAGS_REG))] + "" + "@ + # + mul{l}\t%2" + "TARGET_BMI2 && reload_completed + && REGNO (operands[1]) == DX_REG" + [(set (match_dup 0) + (umul_highpart:SI (match_dup 1) (match_dup 2)))] + "" + [(set_attr "isa" "bmi2,*") + (set_attr "type" "imulx,imul") + (set_attr "length_immediate" "*,0") + (set (attr "athlon_decode") + (cond [(eq_attr "alternative" "1") + (if_then_else (eq_attr "cpu" "athlon") + (const_string "vector") + (const_string "double"))] + (const_string "*"))) + (set_attr "amdfam10_decode" "*,double") + (set_attr "bdver1_decode" "*,direct") + (set_attr "prefix" "vex,orig") + (set_attr "mode" "SI")]) + +(define_insn "*smulsi3_highpart_1" [(set (match_operand:SI 0 "register_operand" "=d") (truncate:SI (lshiftrt:DI (mult:DI - (any_extend:DI - (match_operand:SI 1 "nonimmediate_operand" "%a")) - (any_extend:DI + (sign_extend:DI + (match_operand:SI 1 "register_operand" "%a")) + (sign_extend:DI (match_operand:SI 2 "nonimmediate_operand" "rm"))) (const_int 32)))) (clobber (match_scratch:SI 3 "=1")) (clobber (reg:CC FLAGS_REG))] - "!(MEM_P (operands[1]) && MEM_P (operands[2]))" - "<sgnprefix>mul{l}\t%2" + "" + "imul{l}\t%2" [(set_attr "type" "imul") (set_attr "length_immediate" "0") (set (attr "athlon_decode") diff --git a/gcc/testsuite/gcc.target/i386/pr127900.c b/gcc/testsuite/gcc.target/i386/pr127900.c new file mode 100644 index 000000000000..1be93434c647 --- /dev/null +++ b/gcc/testsuite/gcc.target/i386/pr127900.c @@ -0,0 +1,14 @@ +/* PR target/127900 */ +/* { dg-do compile { target { ! ia32 } } } */ +/* { dg-options "-O2 -mbmi2" } */ + +typedef unsigned long long u64; + +u64 +mulh (u64 a, u64 b) +{ + return ((unsigned __int128) a * b) >> 64; +} + +/* { dg-final { scan-assembler "\tmulx" } } */ +/* { dg-final { scan-assembler-not "\tmulq" } } */
