This is an automated email from the ASF dual-hosted git repository.
zhangstar333 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 5260a987eb9 [Enhancement](tools) Introduce new BE benchmark and remove
old one (#44181)
5260a987eb9 is described below
commit 5260a987eb96d6dd7ae158417a161eb58e4c7036
Author: zclllhhjj <[email protected]>
AuthorDate: Wed Nov 20 12:49:28 2024 +0800
[Enhancement](tools) Introduce new BE benchmark and remove old one (#44181)
### What problem does this PR solve?
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
the old be-benchmark has been not maintained for long time. now it's
time to drop it. in this pr we split google-benchmark as a individual
component rather than in be-ut.
replace https://github.com/apache/doris/pull/41180
To get closest to the actual test results, we compile the executable
with the entire BE binary (so that binary bloat, etc. is consistent with
the actual BE) and start it with `start_be.sh`, so that jemalloc, etc.
conditions are also consistent.
usage:
```bash
./build.sh --benchmark
......
be/bin/start_be.sh --benchmark
```
---
be/CMakeLists.txt | 61 +++---
be/benchmark/benchmark_main.cpp | 52 ++++++
be/src/runtime/memory/jemalloc_hook.cpp | 2 +-
be/src/service/CMakeLists.txt | 2 +-
be/test/CMakeLists.txt | 11 --
be/test/tools/benchmark_tool.cpp | 320 --------------------------------
bin/start_be.sh | 17 +-
build.sh | 21 +++
8 files changed, 128 insertions(+), 358 deletions(-)
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index 1d79048f965..d476af8e211 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -130,6 +130,8 @@ message(STATUS "THIRDPARTY_DIR is ${THIRDPARTY_DIR}")
option(MAKE_TEST "ON for make unit test or OFF for not" OFF)
message(STATUS "make test: ${MAKE_TEST}")
+option(BUILD_BENCHMARK "ON for make google benchmark or OFF for not" OFF)
+message(STATUS "make benchmark: ${BUILD_BENCHMARK}")
option(WITH_MYSQL "Support access MySQL" ON)
@@ -568,7 +570,7 @@ if (OS_MACOSX)
)
endif()
-if (MAKE_TEST)
+if (BUILD_BENCHMARK)
set(COMMON_THIRDPARTY
${COMMON_THIRDPARTY}
benchmark
@@ -708,6 +710,11 @@ if (MAKE_TEST)
endif()
endif ()
+# use this to avoid some runtime tracker. reuse BE_TEST symbol, no need
another.
+if (BUILD_BENCHMARK)
+ add_definitions(-DBE_TEST)
+endif()
+
get_directory_property(COMPILER_FLAGS COMPILE_OPTIONS)
get_directory_property(COMPILER_DEFINES COMPILE_DEFINITIONS)
message(STATUS "Compiler:
${CMAKE_CXX_COMPILER_ID}-${CMAKE_CXX_COMPILER_VERSION}")
@@ -754,7 +761,7 @@ add_subdirectory(${SRC_DIR}/http)
add_subdirectory(${SRC_DIR}/io)
add_subdirectory(${SRC_DIR}/olap)
add_subdirectory(${SRC_DIR}/runtime)
-add_subdirectory(${SRC_DIR}/service)
+add_subdirectory(${SRC_DIR}/service) # this include doris_be
add_subdirectory(${SRC_DIR}/udf)
add_subdirectory(${SRC_DIR}/cloud)
@@ -772,36 +779,44 @@ add_subdirectory(${SRC_DIR}/util)
add_subdirectory(${SRC_DIR}/vec)
add_subdirectory(${SRC_DIR}/pipeline)
+# this include doris_be_test
if (MAKE_TEST)
add_subdirectory(${TEST_DIR})
endif ()
add_subdirectory(${COMMON_SRC_DIR}/cpp ${BUILD_DIR}/src/common_cpp)
-# Install be
-install(DIRECTORY DESTINATION ${OUTPUT_DIR})
-install(DIRECTORY DESTINATION ${OUTPUT_DIR}/bin)
-install(DIRECTORY DESTINATION ${OUTPUT_DIR}/conf)
-
-install(FILES
- ${BASE_DIR}/../bin/start_be.sh
- ${BASE_DIR}/../bin/stop_be.sh
- ${BASE_DIR}/../tools/jeprof
- PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE
- GROUP_READ GROUP_WRITE GROUP_EXECUTE
- WORLD_READ WORLD_EXECUTE
- DESTINATION ${OUTPUT_DIR}/bin)
-
-install(FILES
- ${BASE_DIR}/../conf/be.conf
- ${BASE_DIR}/../conf/odbcinst.ini
- ${BASE_DIR}/../conf/asan_suppr.conf
- ${BASE_DIR}/../conf/lsan_suppr.conf
- DESTINATION ${OUTPUT_DIR}/conf)
+if(NOT BUILD_BENCHMARK)
+ # Install be
+ install(DIRECTORY DESTINATION ${OUTPUT_DIR})
+ install(DIRECTORY DESTINATION ${OUTPUT_DIR}/bin)
+ install(DIRECTORY DESTINATION ${OUTPUT_DIR}/conf)
+
+ install(FILES
+ ${BASE_DIR}/../bin/start_be.sh
+ ${BASE_DIR}/../bin/stop_be.sh
+ ${BASE_DIR}/../tools/jeprof
+ PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE
+ GROUP_READ GROUP_WRITE GROUP_EXECUTE
+ WORLD_READ WORLD_EXECUTE
+ DESTINATION ${OUTPUT_DIR}/bin)
+
+ install(FILES
+ ${BASE_DIR}/../conf/be.conf
+ ${BASE_DIR}/../conf/odbcinst.ini
+ ${BASE_DIR}/../conf/asan_suppr.conf
+ ${BASE_DIR}/../conf/lsan_suppr.conf
+ DESTINATION ${OUTPUT_DIR}/conf)
+endif()
get_property(dirs DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY
INCLUDE_DIRECTORIES)
foreach(dir ${dirs})
message(STATUS "dir='${dir}'")
endforeach()
-
+if (BUILD_BENCHMARK)
+ add_executable(benchmark_test ${BASE_DIR}/benchmark/benchmark_main.cpp)
+ target_link_libraries(benchmark_test ${DORIS_LINK_LIBS})
+ message(STATUS "Add benchmark to build")
+ install(TARGETS benchmark_test DESTINATION ${OUTPUT_DIR}/lib)
+endif()
\ No newline at end of file
diff --git a/be/benchmark/benchmark_main.cpp b/be/benchmark/benchmark_main.cpp
new file mode 100644
index 00000000000..cad6463e981
--- /dev/null
+++ b/be/benchmark/benchmark_main.cpp
@@ -0,0 +1,52 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <benchmark/benchmark.h>
+
+#include <string>
+
+#include "vec/columns/column_string.h"
+#include "vec/core/block.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_string.h"
+
+namespace doris::vectorized { // change if need
+
+static void Example1(benchmark::State& state) {
+ // init. dont time it.
+ state.PauseTiming();
+ Block block;
+ DataTypePtr str_type = std::make_shared<DataTypeString>();
+ std::vector<std::string> vals {100, "content"};
+ state.ResumeTiming();
+
+ // do test
+ for (auto _ : state) {
+ auto str_col = ColumnString::create();
+ for (auto& v : vals) {
+ str_col->insert_data(v.data(), v.size());
+ }
+ block.insert({std::move(str_col), str_type, "col"});
+ benchmark::DoNotOptimize(block); // mark the watched target
+ }
+}
+// could BENCHMARK many functions to compare them together.
+BENCHMARK(Example1);
+
+} // namespace doris::vectorized
+
+BENCHMARK_MAIN();
diff --git a/be/src/runtime/memory/jemalloc_hook.cpp
b/be/src/runtime/memory/jemalloc_hook.cpp
index 445d60d382c..dffc1344b71 100644
--- a/be/src/runtime/memory/jemalloc_hook.cpp
+++ b/be/src/runtime/memory/jemalloc_hook.cpp
@@ -60,7 +60,7 @@ void* doris_realloc(void* p, size_t size) __THROW {
return nullptr;
}
-#if USE_MEM_TRACKER
+#if defined(USE_MEM_TRACKER) && !defined(BE_TEST)
int64_t old_size = jemalloc_usable_size(p);
CONSUME_THREAD_MEM_TRACKER_BY_HOOK_WITH_FN(
[](size_t size, int64_t old_size) { return jenallocx(size, 0) -
old_size; }, size,
diff --git a/be/src/service/CMakeLists.txt b/be/src/service/CMakeLists.txt
index 4ce61134584..e44045dffce 100644
--- a/be/src/service/CMakeLists.txt
+++ b/be/src/service/CMakeLists.txt
@@ -28,7 +28,7 @@ add_library(Service STATIC ${SRC_FILES})
pch_reuse(Service)
-if (${MAKE_TEST} STREQUAL "OFF")
+if (${MAKE_TEST} STREQUAL "OFF" AND ${BUILD_BENCHMARK} STREQUAL "OFF")
add_executable(doris_be
doris_main.cpp
)
diff --git a/be/test/CMakeLists.txt b/be/test/CMakeLists.txt
index 5a37267a937..0060d304a85 100644
--- a/be/test/CMakeLists.txt
+++ b/be/test/CMakeLists.txt
@@ -113,14 +113,3 @@ if (OS_MACOSX AND ARCH_ARM)
COMMAND ${LLVM_STRIP} --strip-all $<TARGET_FILE:doris_be_test>
)
endif()
-
-if (BUILD_BENCHMARK_TOOL AND BUILD_BENCHMARK_TOOL STREQUAL "ON")
- add_executable(benchmark_tool
- tools/benchmark_tool.cpp
- testutil/test_util.cpp
- olap/tablet_schema_helper.cpp
- )
-
- target_link_libraries(benchmark_tool ${TEST_LINK_LIBS})
- set_target_properties(benchmark_tool PROPERTIES COMPILE_FLAGS
"-fno-access-control")
-endif()
diff --git a/be/test/tools/benchmark_tool.cpp b/be/test/tools/benchmark_tool.cpp
deleted file mode 100644
index 28c8f5c96f5..00000000000
--- a/be/test/tools/benchmark_tool.cpp
+++ /dev/null
@@ -1,320 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <benchmark/benchmark.h>
-#include <gflags/gflags.h>
-
-#include <algorithm>
-#include <chrono>
-#include <fstream>
-#include <functional>
-#include <iostream>
-#include <memory>
-#include <random>
-#include <sstream>
-#include <string>
-#include <vector>
-
-#include "common/compiler_util.h"
-#include "common/logging.h"
-#include "gutil/strings/split.h"
-#include "gutil/strings/substitute.h"
-#include "io/fs/file_system.h"
-#include "io/fs/file_writer.h"
-#include "io/fs/local_file_system.h"
-#include "olap/comparison_predicate.h"
-#include "olap/data_dir.h"
-#include "olap/in_list_predicate.h"
-#include "olap/olap_common.h"
-#include "olap/row_cursor.h"
-#include "olap/rowset/segment_v2/binary_dict_page.h"
-#include "olap/rowset/segment_v2/binary_plain_page.h"
-#include "olap/rowset/segment_v2/page_builder.h"
-#include "olap/rowset/segment_v2/page_decoder.h"
-#include "olap/rowset/segment_v2/segment_iterator.h"
-#include "olap/rowset/segment_v2/segment_writer.h"
-#include "olap/tablet_schema.h"
-#include "olap/tablet_schema_helper.h"
-#include "olap/types.h"
-#include "testutil/test_util.h"
-#include "util/debug_util.h"
-
-DEFINE_string(operation, "Custom",
- "valid operation: Custom, BinaryDictPageEncode,
BinaryDictPageDecode, SegmentScan, "
- "SegmentWrite, "
- "SegmentScanByFile, SegmentWriteByFile");
-DEFINE_string(input_file, "./sample.dat", "input file directory");
-DEFINE_string(column_type, "int,varchar", "valid type: int, char, varchar,
string");
-DEFINE_string(rows_number, "10000", "rows number");
-DEFINE_string(iterations, "10",
- "run times, this is set to 0 means the number of iterations is
automatically set ");
-
-const std::string kSegmentDir = "./segment_benchmark";
-
-std::string get_usage(const std::string& progname) {
- std::stringstream ss;
- ss << progname << " is the Doris BE benchmark tool.\n";
- ss << "Stop BE first before use this tool.\n";
-
- ss << "Usage:\n";
- ss << "./benchmark_tool --operation=Custom\n";
- ss << "./benchmark_tool --operation=BinaryDictPageEncode "
- "--rows_number=10000 --iterations=40\n";
- ss << "./benchmark_tool --operation=BinaryDictPageDecode "
- "--rows_number=10000 --iterations=40\n";
- ss << "./benchmark_tool --operation=SegmentScan --column_type=int,varchar "
- "--rows_number=10000 --iterations=0\n";
- ss << "./benchmark_tool --operation=SegmentWrite --column_type=int "
- "--rows_number=10000 --iterations=10\n";
- ss << "./benchmark_tool --operation=SegmentScanByFile
--input_file=./sample.dat "
- "--iterations=10\n";
- ss << "./benchmark_tool --operation=SegmentWriteByFile
--input_file=./sample.dat "
- "--iterations=10\n";
-
- ss << "Sampe data file format: \n"
- << "The first line defines Shcema\n"
- << "The rest of the content is DataSet\n"
- << "For example: \n"
- << "int,char,varchar\n"
- << "123,hello,world\n"
- << "321,good,bye\n";
- return ss.str();
-}
-
-namespace doris {
-class BaseBenchmark {
-public:
- BaseBenchmark(const std::string& name, int iterations) : _name(name),
_iterations(iterations) {}
- virtual ~BaseBenchmark() = default;
-
- void add_name(const std::string& str) { _name += str; }
-
- virtual void init() {}
- virtual void run() {}
-
- void register_bm() {
- auto bm = benchmark::RegisterBenchmark(_name.c_str(),
[&](benchmark::State& state) {
- //first turn will use more time
- this->init();
- this->run();
- for (auto _ : state) {
- state.PauseTiming();
- this->init();
- state.ResumeTiming();
- this->run();
- }
- });
- if (_iterations != 0) {
- bm->Iterations(_iterations);
- }
- bm->Unit(benchmark::kMillisecond);
- }
-
-private:
- std::string _name;
- int _iterations;
-};
-
-class BinaryDictPageBenchmark : public BaseBenchmark {
-public:
- BinaryDictPageBenchmark(const std::string& name, int iterations)
- : BaseBenchmark(name, iterations) {}
- virtual ~BinaryDictPageBenchmark() override {}
-
- virtual void init() override {}
- virtual void run() override {}
-
- void encode_pages(const std::vector<Slice>& contents) {
- PageBuilderOptions options;
- BinaryDictPageBuilder page_builder(options);
-
- results.clear();
- page_start_ids.clear();
- page_start_ids.push_back(0);
- for (size_t i = 0; i < contents.size(); i++) {
- const Slice* ptr = &contents[i];
- size_t add_num = 1;
- (void)page_builder.add(reinterpret_cast<const uint8_t*>(ptr),
&add_num);
- if (page_builder.is_page_full()) {
- OwnedSlice s = page_builder.finish();
- results.emplace_back(std::move(s));
- page_start_ids.push_back(i + 1);
- page_builder.reset();
- }
- }
- OwnedSlice s = page_builder.finish();
- results.emplace_back(std::move(s));
- page_start_ids.push_back(contents.size());
-
- (void)page_builder.get_dictionary_page(&dict_slice);
- }
-
- void decode_pages() {
- // TODO should rewrite this method by using vectorized next batch
method
- }
-
-private:
- std::vector<OwnedSlice> results;
- OwnedSlice dict_slice;
- std::vector<size_t> page_start_ids;
-};
-
-class BinaryDictPageEncodeBenchmark : public BinaryDictPageBenchmark {
-public:
- BinaryDictPageEncodeBenchmark(const std::string& name, int iterations, int
rows_number)
- : BinaryDictPageBenchmark(name + "/rows_number:" +
std::to_string(rows_number),
- iterations),
- _rows_number(rows_number) {}
- virtual ~BinaryDictPageEncodeBenchmark() override {}
-
- virtual void init() override {
- src_strings.clear();
- for (int i = 0; i < _rows_number; i++) {
- src_strings.emplace_back(rand_rng_string(rand_rng_int(1, 8)));
- }
-
- slices.clear();
- for (auto s : src_strings) {
- slices.emplace_back(s.c_str());
- }
- }
- virtual void run() override { encode_pages(slices); }
-
-private:
- std::vector<Slice> slices;
- std::vector<std::string> src_strings;
- int _rows_number;
-};
-
-class BinaryDictPageDecodeBenchmark : public BinaryDictPageBenchmark {
-public:
- BinaryDictPageDecodeBenchmark(const std::string& name, int iterations, int
rows_number)
- : BinaryDictPageBenchmark(name + "/rows_number:" +
std::to_string(rows_number),
- iterations),
- _rows_number(rows_number) {}
- virtual ~BinaryDictPageDecodeBenchmark() override {}
-
- virtual void init() override {
- src_strings.clear();
- for (int i = 0; i < _rows_number; i++) {
- src_strings.emplace_back(rand_rng_string(rand_rng_int(1, 8)));
- }
-
- slices.clear();
- for (auto s : src_strings) {
- slices.emplace_back(s.c_str());
- }
-
- encode_pages(slices);
- }
- virtual void run() override { decode_pages(); }
-
-private:
- std::vector<Slice> slices;
- std::vector<std::string> src_strings;
- int _rows_number;
-}; // namespace doris
-
-// This is sample custom test. User can write custom test code at
custom_init()&custom_run().
-// Call method: ./benchmark_tool --operation=Custom
-class CustomBenchmark : public BaseBenchmark {
-public:
- CustomBenchmark(const std::string& name, int iterations,
std::function<void()> init_func,
- std::function<void()> run_func)
- : BaseBenchmark(name, iterations), _init_func(init_func),
_run_func(run_func) {}
- virtual ~CustomBenchmark() override {}
-
- virtual void init() override { _init_func(); }
- virtual void run() override { _run_func(); }
-
-private:
- std::function<void()> _init_func;
- std::function<void()> _run_func;
-};
-void custom_init() {}
-void custom_run_plus() {
- int p = 100000;
- int q = 0;
- while (p--) {
- q++;
- if (UNLIKELY(q == 1024)) q = 0;
- }
-}
-void custom_run_mod() {
- int p = 100000;
- int q = 0;
- while (p--) {
- q++;
- if (q %= 1024) q = 0;
- }
-}
-
-class MultiBenchmark {
-public:
- MultiBenchmark() {}
- ~MultiBenchmark() {
- for (auto bm : benchmarks) {
- delete bm;
- }
- }
-
- void add_bm() {
- if (equal_ignore_case(FLAGS_operation, "Custom")) {
- benchmarks.emplace_back(
- new doris::CustomBenchmark("custom_run_plus",
std::stoi(FLAGS_iterations),
- doris::custom_init,
doris::custom_run_plus));
- benchmarks.emplace_back(
- new doris::CustomBenchmark("custom_run_mod",
std::stoi(FLAGS_iterations),
- doris::custom_init,
doris::custom_run_mod));
- } else if (equal_ignore_case(FLAGS_operation, "BinaryDictPageEncode"))
{
- benchmarks.emplace_back(new doris::BinaryDictPageEncodeBenchmark(
- FLAGS_operation, std::stoi(FLAGS_iterations),
std::stoi(FLAGS_rows_number)));
- } else if (equal_ignore_case(FLAGS_operation, "BinaryDictPageDecode"))
{
- benchmarks.emplace_back(new doris::BinaryDictPageDecodeBenchmark(
- FLAGS_operation, std::stoi(FLAGS_iterations),
std::stoi(FLAGS_rows_number)));
- } else {
- std::cout << "operation invalid!" << std::endl;
- }
- }
- void register_bm() {
- for (auto bm : benchmarks) {
- bm->register_bm();
- }
- }
-
-private:
- std::vector<doris::BaseBenchmark*> benchmarks;
-};
-
-} //namespace doris
-int main(int argc, char** argv) {
- std::string usage = get_usage(argv[0]);
- gflags::SetUsageMessage(usage);
- google::ParseCommandLineFlags(&argc, &argv, true);
-
- doris::StoragePageCache::create_global_cache(1 << 30, 10, 0);
-
- doris::MultiBenchmark multi_bm;
- multi_bm.add_bm();
- multi_bm.register_bm();
-
- benchmark::Initialize(&argc, argv);
- benchmark::RunSpecifiedBenchmarks();
- benchmark::Shutdown();
-
- return 0;
-}
diff --git a/bin/start_be.sh b/bin/start_be.sh
index 0ae0914d42e..5b7df5ead58 100755
--- a/bin/start_be.sh
+++ b/bin/start_be.sh
@@ -32,6 +32,7 @@ OPTS="$(getopt \
-l 'daemon' \
-l 'console' \
-l 'version' \
+ -l 'benchmark' \
-- "$@")"
eval set -- "${OPTS}"
@@ -39,6 +40,8 @@ eval set -- "${OPTS}"
RUN_DAEMON=0
RUN_CONSOLE=0
RUN_VERSION=0
+RUN_BENCHMARK=0
+
while true; do
case "$1" in
--daemon)
@@ -53,6 +56,10 @@ while true; do
RUN_VERSION=1
shift
;;
+ --benchmark)
+ RUN_BENCHMARK=1
+ shift
+ ;;
--)
shift
break
@@ -293,7 +300,7 @@ fi
pidfile="${PID_DIR}/be.pid"
-if [[ -f "${pidfile}" ]]; then
+if [[ -f "${pidfile}" && "${RUN_BENCHMARK}" -eq 0 ]]; then
if kill -0 "$(cat "${pidfile}")" >/dev/null 2>&1; then
echo "Backend is already running as process $(cat "${pidfile}"), stop
it first"
exit 1
@@ -419,7 +426,13 @@ else
export MALLOC_CONF="${JEMALLOC_CONF},prof_prefix:${JEMALLOC_PROF_PRFIX}"
fi
-if [[ "${RUN_DAEMON}" -eq 1 ]]; then
+if [[ "${RUN_BENCHMARK}" -eq 1 ]]; then
+ if [[ "$(uname -s)" == 'Darwin' ]]; then
+ env DYLD_LIBRARY_PATH="${DYLD_LIBRARY_PATH}" ${LIMIT:+${LIMIT}}
"${DORIS_HOME}/lib/benchmark_test"
+ else
+ ${LIMIT:+${LIMIT}} "${DORIS_HOME}/lib/benchmark_test"
+ fi
+elif [[ "${RUN_DAEMON}" -eq 1 ]]; then
if [[ "$(uname -s)" == 'Darwin' ]]; then
nohup env DYLD_LIBRARY_PATH="${DYLD_LIBRARY_PATH}" ${LIMIT:+${LIMIT}}
"${DORIS_HOME}/lib/doris_be" "$@" >>"${LOG_DIR}/be.out" 2>&1 </dev/null &
else
diff --git a/build.sh b/build.sh
index 5d70f523f8e..c90f6b14144 100755
--- a/build.sh
+++ b/build.sh
@@ -49,6 +49,7 @@ Usage: $0 <options>
--meta-tool build Backend meta tool. Default OFF.
--cloud build Cloud. Default OFF.
--index-tool build Backend inverted index tool. Default OFF.
+ --benchmark build Google Benchmark. Default OFF.
--broker build Broker. Default ON.
--spark-dpp build Spark DPP application. Default ON.
--hive-udf build Hive UDF library for Spark Load. Default ON.
@@ -64,12 +65,14 @@ Usage: $0 <options>
DISABLE_BE_JAVA_EXTENSIONS If set DISABLE_BE_JAVA_EXTENSIONS=ON, we will
do not build binary with java-udf,hudi-scanner,jdbc-scanner and so on Default
is OFF.
DISABLE_JAVA_CHECK_STYLE If set DISABLE_JAVA_CHECK_STYLE=ON, it will
skip style check of java code in FE.
DISABLE_BUILD_AZURE If set DISABLE_BUILD_AZURE=ON, it will not
build azure into BE.
+
Eg.
$0 build all
$0 --be build Backend
$0 --meta-tool build Backend meta tool
$0 --cloud build Cloud
$0 --index-tool build Backend inverted index tool
+ $0 --benchmark build Google Benchmark of Backend
$0 --fe --clean clean and build Frontend and Spark
Dpp application
$0 --fe --be --clean clean and build Frontend, Spark
Dpp application and Backend
$0 --spark-dpp build Spark DPP application alone
@@ -129,6 +132,7 @@ if ! OPTS="$(getopt \
-l 'broker' \
-l 'meta-tool' \
-l 'index-tool' \
+ -l 'benchmark' \
-l 'spark-dpp' \
-l 'hive-udf' \
-l 'be-java-extensions' \
@@ -151,6 +155,7 @@ BUILD_CLOUD=0
BUILD_BROKER=0
BUILD_META_TOOL='OFF'
BUILD_INDEX_TOOL='OFF'
+BUILD_BENCHMARK='OFF'
BUILD_SPARK_DPP=0
BUILD_BE_JAVA_EXTENSIONS=0
BUILD_HIVE_UDF=0
@@ -170,6 +175,7 @@ if [[ "$#" == 1 ]]; then
BUILD_BROKER=1
BUILD_META_TOOL='OFF'
BUILD_INDEX_TOOL='OFF'
+ BUILD_BENCHMARK='OFF'
BUILD_SPARK_DPP=1
BUILD_HIVE_UDF=1
BUILD_BE_JAVA_EXTENSIONS=1
@@ -205,6 +211,11 @@ else
BUILD_INDEX_TOOL='ON'
shift
;;
+ --benchmark)
+ BUILD_BENCHMARK='ON'
+ BUILD_BE=1 # go into BE cmake building, but benchmark instead of
doris_be
+ shift
+ ;;
--spark-dpp)
BUILD_SPARK_DPP=1
shift
@@ -446,6 +457,10 @@ if [[ -z "${ENABLE_CACHE_LOCK_DEBUG}" ]]; then
ENABLE_CACHE_LOCK_DEBUG='OFF'
fi
+if [[ -z "${BUILD_BENCHMARK}" ]]; then
+ BUILD_BENCHMARK='OFF'
+fi
+
if [[ -z "${RECORD_COMPILER_SWITCHES}" ]]; then
RECORD_COMPILER_SWITCHES='OFF'
fi
@@ -476,6 +491,7 @@ echo "Get params:
BUILD_BROKER -- ${BUILD_BROKER}
BUILD_META_TOOL -- ${BUILD_META_TOOL}
BUILD_INDEX_TOOL -- ${BUILD_INDEX_TOOL}
+ BUILD_BENCHMARK -- ${BUILD_BENCHMARK}
BUILD_SPARK_DPP -- ${BUILD_SPARK_DPP}
BUILD_BE_JAVA_EXTENSIONS -- ${BUILD_BE_JAVA_EXTENSIONS}
BUILD_HIVE_UDF -- ${BUILD_HIVE_UDF}
@@ -578,6 +594,7 @@ if [[ "${BUILD_BE}" -eq 1 ]]; then
-DENABLE_INJECTION_POINT="${ENABLE_INJECTION_POINT}" \
-DENABLE_CACHE_LOCK_DEBUG="${ENABLE_CACHE_LOCK_DEBUG}" \
-DMAKE_TEST=OFF \
+ -DBUILD_BENCHMARK="${BUILD_BENCHMARK}" \
-DBUILD_FS_BENCHMARK="${BUILD_FS_BENCHMARK}" \
${CMAKE_USE_CCACHE:+${CMAKE_USE_CCACHE}} \
-DWITH_MYSQL="${WITH_MYSQL}" \
@@ -797,6 +814,10 @@ EOF
cp -r -p "${DORIS_HOME}/be/output/lib/debug_info"
"${DORIS_OUTPUT}/be/lib"/
fi
+ if [[ "${BUILD_BENCHMARK}" = "ON" ]]; then
+ cp -r -p "${DORIS_HOME}/be/output/lib/benchmark_test"
"${DORIS_OUTPUT}/be/lib/"/
+ fi
+
if [[ "${BUILD_FS_BENCHMARK}" = "ON" ]]; then
cp -r -p "${DORIS_HOME}/bin/run-fs-benchmark.sh"
"${DORIS_OUTPUT}/be/bin/"/
fi
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
