This is an automated email from the ASF dual-hosted git repository.
HappenLee pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new bc86a22d6f7 [Fix](build) Manage datasketches-cpp in the BE CMake build
tree (#66511)
bc86a22d6f7 is described below
commit bc86a22d6f7f5e7570f84c6e518567b5a3e9a12b
Author: linrrarity <[email protected]>
AuthorDate: Fri Aug 7 15:51:48 2026 +0800
[Fix](build) Manage datasketches-cpp in the BE CMake build tree (#66511)
Related PR: #63143
Problem Summary:
The BE build previously configured and installed datasketches-cpp from
`contrib/datasketches-cpp` into `thirdparty/installed` before
configuring the BE itself.
Its CMake cache was stored under
`contrib/datasketches-cpp/build/Release`, which was **NOT** managed by
`clean_be()`. As a result, `build.sh --be --clean` could still fail
before reaching `clean_be()` when the cached compiler path belonged to
another workspace or no longer existed.
```text
install datasketches-cpp to thirdparty path before build be
Update datasketches-cpp submodule ...
/mnt/disk9/linzhenqi/d1/doris
Current commit ID of datasketches-cpp submodule:
de8553ba372e618382c2e7b44b0ffc9422b9458c, expected is
de8553ba372e618382c2e7b44b0ffc9422b9458c
-- The CXX compiler identification is unknown
CMake Error at CMakeLists.txt:25 (project):
The CMAKE_CXX_COMPILER:
/mnt/disk7/linzhenqi/dv/version-toolchain/ldb_toolchain_v28/bin/clang++
is not a full path to an existing compiler tool.
Tell CMake where to find the compiler by setting either the environment
variable "CXX" or the CMake cache entry CMAKE_CXX_COMPILER to the full
path
to the compiler, or to the compiler name if it is in the PATH.
-- Configuring incomplete, errors occurred!
```
The standalone installation also mixed a contrib dependency into the
thirdparty installation directory without using the thirdparty build
lifecycle.
### Release note
- Keep datasketches-cpp as a contrib submodule.
- Add datasketches-cpp to the BE CMake build tree with
`add_subdirectory`.
- Link `Exprs` against the upstream `DataSketches::HLL` interface
target.
- Mark the datasketches include directories as system headers so
upstream warnings are not promoted to errors by Doris's `-Werror`
settings.
- Use the upstream build-tree include path for `hll.hpp`.
- Remove the standalone CMake configure/install commands from
`build.sh`.
- Remove the now-unused `TP_INSTALLED_DIR` variable.
The datasketches CMake state is now stored under the BE build directory
and is removed together with the rest of the BE build artifacts by
`clean_be()`.
---
be/CMakeLists.txt | 10 ++++++++++
be/src/exprs/CMakeLists.txt | 4 +++-
.../aggregate_function_datasketches_hll_union_agg.h | 2 +-
.../exprs/aggregate/agg_datasketches_hll_union_agg_test.cpp | 2 +-
build-support/compile-bench/README.md | 5 -----
build.sh | 13 +------------
run-be-ut.sh | 7 -------
7 files changed, 16 insertions(+), 27 deletions(-)
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index aed51c2fb2e..2ad0127dcf9 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -240,6 +240,16 @@ SET(ZSTD_HOME "$ENV{DORIS_THIRDPARTY}/installed")
SET(ZSTD_INCLUDE_DIR "$ENV{DORIS_THIRDPARTY}/installed/include/zstd")
SET(CONTRIB_PATH "${PROJECT_SOURCE_DIR}/../contrib")
+set(BUILD_TESTS OFF)
+add_subdirectory(
+ ${CONTRIB_PATH}/datasketches-cpp
+ ${PROJECT_BINARY_DIR}/datasketches-cpp
+ EXCLUDE_FROM_ALL)
+target_include_directories(
+ hll SYSTEM INTERFACE
$<BUILD_INTERFACE:${CONTRIB_PATH}/datasketches-cpp/hll/include>)
+target_include_directories(
+ common SYSTEM INTERFACE
$<BUILD_INTERFACE:${CONTRIB_PATH}/datasketches-cpp/common/include>)
+
# Out of source build need to set the binary dir
add_subdirectory(${CONTRIB_PATH}/apache-orc ${PROJECT_BINARY_DIR}/apache-orc
EXCLUDE_FROM_ALL)
target_compile_options(orc PRIVATE -fno-omit-frame-pointer
-Wno-implicit-fallthrough -w)
diff --git a/be/src/exprs/CMakeLists.txt b/be/src/exprs/CMakeLists.txt
index d8995ccd8ad..62cc03b4c5c 100644
--- a/be/src/exprs/CMakeLists.txt
+++ b/be/src/exprs/CMakeLists.txt
@@ -32,7 +32,9 @@ set(SRC_FILES ${SRC_FILES}
add_library(Exprs STATIC ${SRC_FILES})
# function_array_distance uses faiss headers (platform_macros.h, distances.h),
# which are exported by ann_index via PUBLIC linkage with faiss.
-target_link_libraries(Exprs PRIVATE ann_index)
+target_link_libraries(Exprs
+ PRIVATE ann_index
+ PUBLIC DataSketches::HLL)
pch_reuse(Exprs)
diff --git
a/be/src/exprs/aggregate/aggregate_function_datasketches_hll_union_agg.h
b/be/src/exprs/aggregate/aggregate_function_datasketches_hll_union_agg.h
index ceaeecd0556..19d0a061814 100644
--- a/be/src/exprs/aggregate/aggregate_function_datasketches_hll_union_agg.h
+++ b/be/src/exprs/aggregate/aggregate_function_datasketches_hll_union_agg.h
@@ -18,9 +18,9 @@
#pragma once
#include <stddef.h>
-#include <DataSketches/hll.hpp>
#include <algorithm>
#include <boost/iterator/iterator_facade.hpp>
+#include <hll.hpp>
#include <memory>
#include <optional>
#include <type_traits>
diff --git a/be/test/exprs/aggregate/agg_datasketches_hll_union_agg_test.cpp
b/be/test/exprs/aggregate/agg_datasketches_hll_union_agg_test.cpp
index 7783ec89e2c..eeaeb2a45dc 100644
--- a/be/test/exprs/aggregate/agg_datasketches_hll_union_agg_test.cpp
+++ b/be/test/exprs/aggregate/agg_datasketches_hll_union_agg_test.cpp
@@ -17,7 +17,7 @@
#include <gtest/gtest.h>
-#include <DataSketches/hll.hpp>
+#include <hll.hpp>
#include "agent/be_exec_version_manager.h"
#include "common/config.h"
diff --git a/build-support/compile-bench/README.md
b/build-support/compile-bench/README.md
index eec937f5594..c1bdc8d38e6 100644
--- a/build-support/compile-bench/README.md
+++ b/build-support/compile-bench/README.md
@@ -83,11 +83,6 @@ Benchmark numbers are only comparable when every run does
the same cold work:
part of the real build being optimized. Override with `ENABLE_PCH=OFF` to
compare with/without PCH.
-Not isolated (by design, both are outside the BE compile and near-constant):
-the `contrib/datasketches-cpp` mini-build reuses its own build dir, and
-`--clean` is not required (passing it additionally rebuilds gensrc from
-scratch; the timed BE phases are unaffected).
-
## What the report contains
- **Phases**: gensrc, contrib submodules, datasketches install, cmake
diff --git a/build.sh b/build.sh
index ceaa09a0779..9e98fb2532e 100755
--- a/build.sh
+++ b/build.sh
@@ -34,7 +34,6 @@ if [[ -z "${DORIS_THIRDPARTY}" ]]; then
export DORIS_THIRDPARTY="${DORIS_HOME}/thirdparty"
fi
export TP_INCLUDE_DIR="${DORIS_THIRDPARTY}/installed/include"
-export TP_INSTALLED_DIR="${DORIS_THIRDPARTY}/installed"
export TP_LIB_DIR="${DORIS_THIRDPARTY}/installed/lib"
HADOOP_DEPS_NAME="hadoop-deps"
. "${DORIS_HOME}/env.sh"
@@ -857,19 +856,9 @@ FE_MODULES="$(
if [[ "${BUILD_BE}" -eq 1 ]]; then
if [[ "${COMPILE_BENCH}" -eq 1 ]]; then
- compile_bench_phase_begin "datasketches_install"
- fi
- echo "install datasketches-cpp to thirdparty path before build be"
- update_submodule "contrib/datasketches-cpp" "datasketches-cpp"
"https://github.com/apache/datasketches-cpp/archive/refs/heads/master.tar.gz";
- cd "${DORIS_HOME}/contrib/datasketches-cpp"
- "${CMAKE_CMD}" -S . -B build/Release -DCMAKE_BUILD_TYPE=Release
-DCMAKE_INSTALL_PREFIX=$TP_INSTALLED_DIR -DBUILD_TESTS=OFF
- "${CMAKE_CMD}" --build build/Release -t install
- cd "${DORIS_HOME}"
- if [[ "${COMPILE_BENCH}" -eq 1 ]]; then
- compile_bench_phase_end
compile_bench_phase_begin "contrib_submodules"
fi
-
+ update_submodule "contrib/datasketches-cpp" "datasketches-cpp"
"https://github.com/apache/datasketches-cpp/archive/refs/heads/master.tar.gz";
update_submodule "contrib/apache-orc" "apache-orc"
"https://github.com/apache/doris-thirdparty/archive/refs/heads/orc.tar.gz";
update_submodule "contrib/clucene" "clucene"
"https://github.com/apache/doris-thirdparty/archive/refs/heads/clucene.tar.gz";
update_submodule "contrib/openblas" "openblas"
"https://github.com/apache/doris-thirdparty/archive/refs/heads/openblas.tar.gz";
diff --git a/run-be-ut.sh b/run-be-ut.sh
index 8227976d7ce..3ff9cd573ee 100755
--- a/run-be-ut.sh
+++ b/run-be-ut.sh
@@ -45,7 +45,6 @@ if [[ -z "${DORIS_THIRDPARTY}" ]]; then
export DORIS_THIRDPARTY="${DORIS_HOME}/thirdparty"
fi
export TP_INCLUDE_DIR="${DORIS_THIRDPARTY}/installed/include"
-export TP_INSTALLED_DIR="${DORIS_THIRDPARTY}/installed"
export TP_LIB_DIR="${DORIS_THIRDPARTY}/installed/lib"
. "${DORIS_HOME}/env.sh"
@@ -251,13 +250,7 @@ update_submodule() {
fi
}
-echo "install datasketches-cpp to thirdparty path before build backend ut"
update_submodule "contrib/datasketches-cpp" "datasketches-cpp"
"https://github.com/apache/datasketches-cpp/archive/refs/heads/master.tar.gz";
-cd "${DORIS_HOME}/contrib/datasketches-cpp"
-"${CMAKE_CMD}" -S . -B build/Release -DCMAKE_BUILD_TYPE=Release
-DCMAKE_INSTALL_PREFIX=$TP_INSTALLED_DIR -DBUILD_TESTS=OFF
-"${CMAKE_CMD}" --build build/Release -t install
-cd "${DORIS_HOME}"
-
update_submodule "contrib/apache-orc" "apache-orc"
"https://github.com/apache/doris-thirdparty/archive/refs/heads/orc.tar.gz";
update_submodule "contrib/clucene" "clucene"
"https://github.com/apache/doris-thirdparty/archive/refs/heads/clucene.tar.gz";
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
