Jeffshen opened a new issue, #70234: URL: https://github.com/apache/doris/issues/70234
### Search before asking - [x] I had searched in the [issues](https://github.com/apache/doris/issues?q=is%3Aissue) and found no similar issues. ### Description Since AMD and Intel latest CPU support AVX512 instructions, it's time to add support for avx512 in BE building. ### Solution diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index e4d9f08eccf..d4de7b5fd5a 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -101,6 +101,13 @@ message(STATUS "USE_UNWIND is ${USE_UNWIND}") message(STATUS "ENABLE_PCH is ${ENABLE_PCH}") message(STATUS "USE_AVX2 is ${USE_AVX2}") +# Enable the AVX512 instruction set on x86_64. +# Defaults to OFF so that the produced binary still runs on AVX2-only machines; +# enabling it makes AVX-512 a hard requirement at run time. +# Enable with -DUSE_AVX512=ON. +set(USE_AVX512 OFF CACHE BOOL "Enable AVX512 instruction set for x86_64") +message(STATUS "USE_AVX512 is ${USE_AVX512}") + # set CMAKE_BUILD_TYPE if (NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE RELEASE) @@ -388,6 +395,19 @@ if ("${CMAKE_BUILD_TARGET_ARCH}" STREQUAL "x86" OR "${CMAKE_BUILD_TARGET_ARCH}" add_definitions(-DUSE_AVX2) add_definitions(-DLIBDIVIDE_AVX2) endif() + if (USE_AVX512) + # -mavx512f also defines __AVX2__, so USE_AVX2=0 cannot be honored + # together with USE_AVX512=ON. + if (NOT USE_AVX2) + message(FATAL_ERROR "USE_AVX512 requires USE_AVX2=ON: -mavx512f implies AVX2.") + endif() + # Doris code only needs F (512-bit integer), BW (byte/word ops) and + # VL (EVEX-encoded 256-bit ops). DQ is required as well because these + # flags are also inherited by contrib/faiss, whose __AVX512F__ code + # paths use DQ intrinsics such as _mm512_inserti32x8. + add_compile_options(-mavx512f -mavx512bw -mavx512vl -mavx512dq) + add_definitions(-DUSE_AVX512) + endif() endif() diff --git a/build.sh b/build.sh index 37e13774f97..500ee0f9556 100755 --- a/build.sh +++ b/build.sh @@ -74,6 +74,8 @@ Usage: $0 <options> Environment variables: USE_AVX2 If the CPU does not support AVX2 instruction set, please set USE_AVX2=0. Default is ON. + USE_AVX512 Build Backend with the AVX512 instruction set (F/BW/VL/DQ). Implies AVX2. + The produced binary requires an AVX512 capable CPU. Default is OFF. ENABLE_DYNAMIC_ARCH If set ENABLE_DYNAMIC_ARCH=ON, it will enable dynamic CPU detection in OpenBLAS. Default is ON. Can also use --enable-dynamic-arch flag. ARM_MARCH Specify the ARM architecture instruction set. Default is armv8-a+crc. STRIP_DEBUG_INFO If set STRIP_DEBUG_INFO=ON, the debug information in the compiled binaries will be stored separately in the 'be/lib/debug_info' directory. Def ault is OFF. @@ -100,6 +102,7 @@ Usage: $0 <options> $0 --be-extension-ignore avro-scanner build be-java-extensions, choose which modules to ignore. Multiple modules separated by commas, like --be-extension-ignore avro-sc anner,hadoop-hudi-scanner USE_AVX2=0 $0 --be build Backend and not using AVX2 instruction. + USE_AVX512=ON $0 --be build Backend with AVX512 instructions (requires an AVX512 CPU). USE_AVX2=0 STRIP_DEBUG_INFO=ON $0 build all and not using AVX2 instruction, and strip the debug info for Backend ARM_MARCH=armv8-a+crc+simd $0 --be build Backend with specified ARM architecture instruction set $0 --be --disable-dynamic-arch build Backend with DYNAMIC_ARCH disabled in OpenBLAS @@ -460,6 +463,9 @@ fi if [[ -z "${USE_AVX2}" ]]; then USE_AVX2='ON' fi +if [[ -z "${USE_AVX512}" ]]; then + USE_AVX512='OFF' +fi if [[ -z "${ARM_MARCH}" ]]; then ARM_MARCH='armv8-a+crc' fi @@ -605,6 +611,7 @@ echo "Get params: CLEAN -- ${CLEAN} GLIBC_COMPATIBILITY -- ${GLIBC_COMPATIBILITY} USE_AVX2 -- ${USE_AVX2} + USE_AVX512 -- ${USE_AVX512} USE_LIBCPP -- ${USE_LIBCPP} USE_UNWIND -- ${USE_UNWIND} STRIP_DEBUG_INFO -- ${STRIP_DEBUG_INFO} @@ -738,6 +745,7 @@ if [[ "${BUILD_BE}" -eq 1 ]]; then -DENABLE_PCH="${ENABLE_PCH}" \ -DUSE_JEMALLOC="${USE_JEMALLOC}" \ -DUSE_AVX2="${USE_AVX2}" \ + -DUSE_AVX512="${USE_AVX512}" \ -DARM_MARCH="${ARM_MARCH}" \ -DGLIBC_COMPATIBILITY="${GLIBC_COMPATIBILITY}" \ -DEXTRA_CXX_FLAGS="${EXTRA_CXX_FLAGS}" \ diff --git a/run-be-ut.sh b/run-be-ut.sh index 50b76f6d6f1..edb4e819452 100755 --- a/run-be-ut.sh +++ b/run-be-ut.sh @@ -221,6 +221,10 @@ if [[ -z "${USE_AVX2}" ]]; then USE_AVX2='ON' fi +if [[ -z "${USE_AVX512}" ]]; then + USE_AVX512='OFF' +fi + if [[ -z "${ARM_MARCH}" ]]; then ARM_MARCH='armv8-a+crc' fi @@ -260,6 +264,7 @@ cd "${CMAKE_BUILD_DIR}" -DUSE_UNWIND="${USE_UNWIND}" \ -DUSE_JEMALLOC=OFF \ -DUSE_AVX2="${USE_AVX2}" \ + -DUSE_AVX512="${USE_AVX512}" \ -DARM_MARCH="${ARM_MARCH}" \ -DEXTRA_CXX_FLAGS="${EXTRA_CXX_FLAGS}" \ -DENABLE_CLANG_COVERAGE="${DENABLE_CLANG_COVERAGE}" \ ### Are you willing to submit PR? - [x] Yes I am willing to submit a PR! ### Code of Conduct - [x] I agree to follow this project's [Code of Conduct](https://www.apache.org/foundation/policies/conduct) -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
