This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 34f8ea966d8 branch-4.2: [fix](be) Route glibc allocation entry points
through jemalloc #69819 (#70153)
34f8ea966d8 is described below
commit 34f8ea966d8eb8ea924072d01bc62006ae466872
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Thu Nov 5 09:24:07 2026 +0800
branch-4.2: [fix](be) Route glibc allocation entry points through jemalloc
#69819 (#70153)
Cherry-picked from #69819
Co-authored-by: daidai <[email protected]>
---
be/src/runtime/memory/jemalloc_hook.cpp | 41 +++-
be/test/jemalloc_hook/CMakeLists.txt | 58 +++++
be/test/jemalloc_hook/jemalloc_hook_test.cpp | 300 +++++++++++++++++++++++
be/test/jemalloc_hook/jemalloc_hook_test_dso.cpp | 44 ++++
run-be-ut.sh | 32 ++-
5 files changed, 470 insertions(+), 5 deletions(-)
diff --git a/be/src/runtime/memory/jemalloc_hook.cpp
b/be/src/runtime/memory/jemalloc_hook.cpp
index 9cef99757bf..a70abd1f1e1 100644
--- a/be/src/runtime/memory/jemalloc_hook.cpp
+++ b/be/src/runtime/memory/jemalloc_hook.cpp
@@ -15,6 +15,11 @@
// specific language governing permissions and limitations
// under the License.
+#include <unistd.h>
+
+#include <cerrno>
+#include <limits>
+
#include "jemalloc/jemalloc.h"
#ifndef __THROW
@@ -35,7 +40,13 @@ void doris_free(void* p) __THROW {
}
void* doris_realloc(void* p, size_t size) __THROW {
- return jerealloc(p, size);
+ void* ptr = jerealloc(p, size);
+ // jemalloc 5.3.0 can reject an oversized request without setting errno.
+ // realloc(p, 0) may free p and return nullptr, so only nonzero sizes fail.
+ if (ptr == nullptr && size != 0) {
+ errno = ENOMEM;
+ }
+ return ptr;
}
void* doris_calloc(size_t n, size_t size) __THROW {
@@ -55,11 +66,21 @@ void* doris_aligned_alloc(size_t align, size_t size)
__THROW {
}
void* doris_valloc(size_t size) __THROW {
- return jevalloc(size);
+ // Unlike jemalloc's legacy valloc, aligned_alloc keeps zero-size results
+ // page-aligned and sets errno when an allocation fails.
+ return jealigned_alloc(getpagesize(), size);
}
void* doris_pvalloc(size_t size) __THROW {
- return jevalloc(size);
+ const size_t page_size = getpagesize();
+ // pvalloc rounds up to whole pages; SIZE_MAX must fail rather than wrap
+ // to a small allocation when adding the page-size adjustment.
+ if (size > std::numeric_limits<size_t>::max() - (page_size - 1)) {
+ errno = ENOMEM;
+ return nullptr;
+ }
+ const size_t rounded_size = (size + page_size - 1) / page_size * page_size;
+ return jealigned_alloc(page_size, rounded_size);
}
int doris_posix_memalign(void** r, size_t align, size_t size) __THROW {
@@ -77,9 +98,21 @@ void* realloc(void* p, size_t size) __THROW
ALIAS(doris_realloc);
void* calloc(size_t n, size_t size) __THROW ALIAS(doris_calloc);
void cfree(void* ptr) __THROW ALIAS(doris_cfree);
void* memalign(size_t align, size_t size) __THROW ALIAS(doris_memalign);
+#ifdef __GLIBC__
+// Keep glibc's exported allocation entry points paired with the public
jemalloc
+// hooks. For example, before glibc 2.25, dlopened modules allocate dynamic TLS
+// with __libc_memalign and release it through free.
+void* __libc_malloc(size_t size) __THROW ALIAS(doris_malloc);
+void __libc_free(void* ptr) __THROW ALIAS(doris_free);
+void* __libc_calloc(size_t n, size_t size) __THROW ALIAS(doris_calloc);
+void* __libc_realloc(void* ptr, size_t size) __THROW ALIAS(doris_realloc);
+void* __libc_memalign(size_t align, size_t size) __THROW ALIAS(doris_memalign);
+void* __libc_valloc(size_t size) __THROW ALIAS(doris_valloc);
+void* __libc_pvalloc(size_t size) __THROW ALIAS(doris_pvalloc);
+#endif
void* aligned_alloc(size_t align, size_t size) __THROW
ALIAS(doris_aligned_alloc);
void* valloc(size_t size) __THROW ALIAS(doris_valloc);
void* pvalloc(size_t size) __THROW ALIAS(doris_pvalloc);
int posix_memalign(void** r, size_t a, size_t s) __THROW
ALIAS(doris_posix_memalign);
size_t malloc_usable_size(void* ptr) __THROW ALIAS(doris_malloc_usable_size);
-}
\ No newline at end of file
+}
diff --git a/be/test/jemalloc_hook/CMakeLists.txt
b/be/test/jemalloc_hook/CMakeLists.txt
new file mode 100644
index 00000000000..201a4c68405
--- /dev/null
+++ b/be/test/jemalloc_hook/CMakeLists.txt
@@ -0,0 +1,58 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This allocator test must not use ASAN: its interceptors would hide missing
+# glibc allocation entry points. Keep it separate from doris_be_test, which
+# run-be-ut.sh builds with USE_JEMALLOC=OFF.
+cmake_minimum_required(VERSION 3.20)
+project(doris_jemalloc_hook_test LANGUAGES CXX)
+
+if (NOT CMAKE_SYSTEM_NAME STREQUAL "Linux")
+ message(FATAL_ERROR "jemalloc hook tests require Linux/glibc")
+endif()
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+get_filename_component(DORIS_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../.."
ABSOLUTE)
+set(THIRDPARTY_DIR "$ENV{DORIS_THIRDPARTY}/installed")
+find_package(Threads REQUIRED)
+
+foreach (LIB_NAME gtest gtest_main jemalloc_doris unwind lzma z)
+ find_library(${LIB_NAME}_LIBRARY NAMES ${LIB_NAME}
+ PATHS "${THIRDPARTY_DIR}/lib" "${THIRDPARTY_DIR}/lib64"
+ NO_DEFAULT_PATH REQUIRED)
+endforeach()
+
+add_library(jemalloc_hook_test_dso SHARED jemalloc_hook_test_dso.cpp)
+target_compile_options(jemalloc_hook_test_dso PRIVATE
-ftls-model=global-dynamic)
+
+add_executable(jemalloc_hook_test
+ jemalloc_hook_test.cpp
+ "${DORIS_SOURCE_DIR}/be/src/runtime/memory/jemalloc_hook.cpp")
+target_include_directories(jemalloc_hook_test PRIVATE
"${THIRDPARTY_DIR}/include")
+target_compile_definitions(jemalloc_hook_test PRIVATE
+ JEMALLOC_HOOK_TEST_DSO="$<TARGET_FILE:jemalloc_hook_test_dso>")
+# Do not enable ENABLE_EXPORTS: verify the hook also works with the production
+# executable's default symbol visibility.
+# libunwind's compressed .debug_frame reader needs zlib's uncompress.
+# Keep its compression archives after unwind in the static link order.
+target_link_libraries(jemalloc_hook_test PRIVATE
+ "${gtest_main_LIBRARY}" "${gtest_LIBRARY}" "${jemalloc_doris_LIBRARY}"
+ "${unwind_LIBRARY}" "${lzma_LIBRARY}" "${z_LIBRARY}"
+ Threads::Threads ${CMAKE_DL_LIBS})
+add_dependencies(jemalloc_hook_test jemalloc_hook_test_dso)
diff --git a/be/test/jemalloc_hook/jemalloc_hook_test.cpp
b/be/test/jemalloc_hook/jemalloc_hook_test.cpp
new file mode 100644
index 00000000000..10eacad09f7
--- /dev/null
+++ b/be/test/jemalloc_hook/jemalloc_hook_test.cpp
@@ -0,0 +1,300 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <dlfcn.h>
+#include <gtest/gtest.h>
+#include <malloc.h>
+#include <unistd.h>
+
+#include <array>
+#include <cerrno>
+#include <cstdint>
+#include <cstdlib>
+#include <cstring>
+#include <limits>
+#include <memory>
+#include <thread>
+
+extern "C" void* doris_memalign(size_t alignment, size_t size) noexcept;
+
+namespace doris {
+
+class JemallocHookTest : public testing::Test {
+protected:
+ using AllocationFunction = void* (*)(size_t);
+ using CallocFunction = void* (*)(size_t, size_t);
+ using FreeFunction = void (*)(void*);
+ using ReallocationFunction = void* (*)(void*, size_t);
+ using MemalignFunction = void* (*)(size_t, size_t);
+
+ void SetUp() override {
+ ASSERT_NO_FATAL_FAILURE(check_executable_interposition());
+ ASSERT_NO_FATAL_FAILURE(check_legacy_allocation_pair());
+ }
+
+ void check_executable_interposition() {
+ // Anchor the lookup to the linked production hook. A libc allocation
+ // paired with a libc free could otherwise pass without testing the
hook.
+ Dl_info hook_image = {};
+ ASSERT_NE(0, dladdr(reinterpret_cast<void*>(&doris_memalign),
&hook_image));
+ constexpr std::array<const char*, 9> symbol_names = {
+ "__libc_memalign", "malloc", "free",
+ "__libc_malloc", "__libc_free", "__libc_calloc",
+ "__libc_realloc", "__libc_valloc", "__libc_pvalloc"};
+ for (size_t i = 0; i < symbol_names.size(); ++i) {
+ _allocator_symbols[i] = dlsym(RTLD_DEFAULT, symbol_names[i]);
+ ASSERT_NE(nullptr, _allocator_symbols[i]) << symbol_names[i] << ":
" << dlerror();
+ Dl_info symbol_image = {};
+ ASSERT_NE(0, dladdr(_allocator_symbols[i], &symbol_image)) <<
symbol_names[i];
+ ASSERT_EQ(hook_image.dli_fbase, symbol_image.dli_fbase)
+ << symbol_names[i] << " resolved to " <<
symbol_image.dli_fname
+ << " instead of " << hook_image.dli_fname;
+ }
+ _memalign = reinterpret_cast<MemalignFunction>(_allocator_symbols[0]);
+ _libc_malloc =
reinterpret_cast<AllocationFunction>(_allocator_symbols[3]);
+ _libc_free = reinterpret_cast<FreeFunction>(_allocator_symbols[4]);
+ _libc_calloc = reinterpret_cast<CallocFunction>(_allocator_symbols[5]);
+ _libc_realloc =
reinterpret_cast<ReallocationFunction>(_allocator_symbols[6]);
+ _libc_valloc =
reinterpret_cast<AllocationFunction>(_allocator_symbols[7]);
+ _libc_pvalloc =
reinterpret_cast<AllocationFunction>(_allocator_symbols[8]);
+ }
+
+ void check_legacy_allocation_pair() {
+ // Exercise the legacy allocation/free pair even on newer glibc
versions
+ // whose loader no longer uses __libc_memalign for TLS. A subprocess
turns
+ // an allocator-mismatch crash into a test failure instead of aborting
UT.
+ ASSERT_EXIT(
+ {
+ void* ptr = _memalign(16, 32);
+ const int result = (ptr == nullptr);
+ std::free(ptr);
+ std::_Exit(result);
+ },
+ testing::ExitedWithCode(0), "");
+ }
+
+ std::array<void*, 9> _allocator_symbols = {};
+ MemalignFunction _memalign = nullptr;
+ AllocationFunction _libc_malloc = nullptr;
+ FreeFunction _libc_free = nullptr;
+ CallocFunction _libc_calloc = nullptr;
+ ReallocationFunction _libc_realloc = nullptr;
+ AllocationFunction _libc_valloc = nullptr;
+ AllocationFunction _libc_pvalloc = nullptr;
+};
+
+TEST_F(JemallocHookTest, LibcAllocationsCanBeFreedThroughPublicEntry) {
+ for (size_t size : {0, 17, 4097}) {
+ void* ptr = _libc_malloc(size);
+ ASSERT_NE(nullptr, ptr);
+ std::memset(ptr, 0x5a, size);
+ std::free(ptr);
+ }
+}
+
+TEST_F(JemallocHookTest, PublicAllocationsCanBeFreedThroughLibcEntry) {
+ _libc_free(nullptr);
+ for (size_t size : {0, 17, 4097}) {
+ std::unique_ptr<void, decltype(&std::free)> ptr(std::malloc(size),
&std::free);
+ ASSERT_NE(nullptr, ptr.get());
+ std::memset(ptr.get(), 0x5a, size);
+ errno = EDOM;
+ _libc_free(ptr.release());
+ const int free_errno = errno;
+ EXPECT_EQ(EDOM, free_errno);
+ }
+}
+
+TEST_F(JemallocHookTest, LibcCallocInitializesMemoryAndRejectsOverflow) {
+ for (size_t count : {0, 17}) {
+ constexpr size_t element_size = 13;
+ void* ptr = _libc_calloc(count, element_size);
+ ASSERT_NE(nullptr, ptr);
+ for (size_t i = 0; i < count * element_size; ++i) {
+ EXPECT_EQ(0, static_cast<unsigned char*>(ptr)[i]);
+ }
+ std::free(ptr);
+ }
+
+ errno = 0;
+ EXPECT_EQ(nullptr, _libc_calloc(std::numeric_limits<size_t>::max(), 2));
+ EXPECT_EQ(ENOMEM, errno);
+}
+
+TEST_F(JemallocHookTest, ReallocCanCrossLibcAndPublicEntryPoints) {
+ struct AllocationPair {
+ AllocationFunction allocate;
+ ReallocationFunction reallocate;
+ FreeFunction release;
+ };
+ const std::array<AllocationPair, 3> pairs = {{{&std::malloc,
_libc_realloc, &std::free},
+ {_libc_malloc,
&std::realloc, _libc_free},
+ {_libc_malloc,
_libc_realloc, &std::free}}};
+ for (const auto& pair : pairs) {
+ void* ptr = pair.allocate(17);
+ ASSERT_NE(nullptr, ptr);
+ std::memset(ptr, 0x5a, 17);
+ void* resized = pair.reallocate(ptr, 1031);
+ ASSERT_NE(nullptr, resized);
+ for (size_t i = 0; i < 17; ++i) {
+ EXPECT_EQ(0x5a, static_cast<unsigned char*>(resized)[i]);
+ }
+ std::memset(resized, 0x5a, 1031);
+ pair.release(resized);
+ }
+}
+
+TEST_F(JemallocHookTest, LibcReallocFailurePreservesTheOriginalAllocation) {
+ void* ptr = _libc_realloc(nullptr, 17);
+ ASSERT_NE(nullptr, ptr);
+ std::memset(ptr, 0x5a, 17);
+
+ errno = 0;
+ void* resized = _libc_realloc(ptr, std::numeric_limits<size_t>::max());
+ const int allocation_errno = errno;
+ ASSERT_EQ(nullptr, resized);
+ EXPECT_EQ(ENOMEM, allocation_errno);
+ for (size_t i = 0; i < 17; ++i) {
+ EXPECT_EQ(0x5a, static_cast<unsigned char*>(ptr)[i]);
+ }
+ std::free(ptr);
+}
+
+TEST_F(JemallocHookTest, LibcReallocZeroSizeIsNotReportedAsOutOfMemory) {
+ std::unique_ptr<void, decltype(&std::free)> ptr(std::malloc(17),
&std::free);
+ ASSERT_NE(nullptr, ptr.get());
+ errno = 0;
+ void* resized = _libc_realloc(ptr.release(), 0);
+ const int allocation_errno = errno;
+ EXPECT_NE(ENOMEM, allocation_errno);
+ std::free(resized);
+}
+
+TEST_F(JemallocHookTest, LibcPageAlignedAllocationsCanBeFreed) {
+ const long page_size = sysconf(_SC_PAGESIZE);
+ ASSERT_GT(page_size, 0);
+ for (auto allocate : {_libc_valloc, _libc_pvalloc}) {
+ for (size_t size : {size_t {0}, size_t {1},
static_cast<size_t>(page_size) + 1}) {
+ void* ptr = allocate(size);
+ ASSERT_NE(nullptr, ptr);
+ EXPECT_EQ(0, reinterpret_cast<uintptr_t>(ptr) % page_size) <<
"size: " << size;
+ std::memset(ptr, 0x5a, size);
+ std::free(ptr);
+ }
+ errno = 0;
+ void* ptr = allocate(std::numeric_limits<size_t>::max());
+ const int allocation_errno = errno;
+ EXPECT_EQ(nullptr, ptr);
+ EXPECT_EQ(ENOMEM, allocation_errno);
+ }
+}
+
+TEST_F(JemallocHookTest, LibcPvallocRoundsAllocationSizeToPages) {
+ const size_t page_size = getpagesize();
+ for (size_t size : {size_t {0}, size_t {1}, page_size + 1}) {
+ void* ptr = _libc_pvalloc(size);
+ ASSERT_NE(nullptr, ptr);
+ const size_t rounded_size = (size + page_size - 1) / page_size *
page_size;
+ ASSERT_GE(malloc_usable_size(ptr), rounded_size);
+ std::memset(ptr, 0x5a, rounded_size);
+ std::free(ptr);
+ }
+}
+
+TEST_F(JemallocHookTest, AlignedAllocationsCanBeFreed) {
+ for (size_t alignment : {1, 8, 16, 64, 4096}) {
+ for (size_t size : {0, 1, 17, 4097}) {
+ void* ptr = _memalign(alignment, size);
+ ASSERT_NE(nullptr, ptr);
+ EXPECT_EQ(0, reinterpret_cast<uintptr_t>(ptr) % alignment);
+ std::memset(ptr, 0x5a, size);
+ std::free(ptr);
+ }
+ }
+}
+
+TEST_F(JemallocHookTest, AllocationErrorsPreserveMemalignSemantics) {
+ errno = 0;
+ void* ptr = _memalign(3, 16);
+ EXPECT_EQ(nullptr, ptr);
+ EXPECT_EQ(EINVAL, errno);
+ std::free(ptr);
+
+ errno = 0;
+ ptr = _memalign(64, std::numeric_limits<size_t>::max());
+ EXPECT_EQ(nullptr, ptr);
+ EXPECT_EQ(ENOMEM, errno);
+ std::free(ptr);
+}
+
+TEST_F(JemallocHookTest, AllocationsCanBeFreedAfterAllocatingThreadsExit) {
+ std::array<void*, 64> pointers = {};
+ std::array<std::thread, 64> threads;
+ for (size_t i = 0; i < threads.size(); ++i) {
+ threads[i] = std::thread([&, i] { pointers[i] = _memalign(64, 1031);
});
+ }
+ for (auto& thread : threads) {
+ thread.join();
+ }
+ for (void* ptr : pointers) {
+ ASSERT_NE(nullptr, ptr);
+ std::memset(ptr, 0x5a, 1031);
+ std::free(ptr);
+ }
+}
+
+TEST_F(JemallocHookTest, LateLoadedDynamicTlsCanBeFreedAfterThreadsExit) {
+ std::unique_ptr<void, decltype(&dlclose)> handle(
+ dlopen(JEMALLOC_HOOK_TEST_DSO, RTLD_NOW | RTLD_LOCAL), &dlclose);
+ ASSERT_NE(nullptr, handle) << dlerror();
+ using GetAllocatorSymbols = void (*)(void**);
+ auto get_allocator_symbols = reinterpret_cast<GetAllocatorSymbols>(
+ dlsym(handle.get(), "jemalloc_hook_test_allocator_symbols"));
+ ASSERT_NE(nullptr, get_allocator_symbols) << dlerror();
+ std::array<void*, 9> allocator_symbols = {};
+ get_allocator_symbols(allocator_symbols.data());
+ // Check the DSO's actual relocations for every glibc allocation entry
point
+ // before exercising dynamic TLS.
+ ASSERT_EQ(_allocator_symbols, allocator_symbols);
+
+ using TouchTlsFunction = int (*)();
+ auto touch_tls =
+ reinterpret_cast<TouchTlsFunction>(dlsym(handle.get(),
"jemalloc_hook_test_touch_tls"));
+ ASSERT_NE(nullptr, touch_tls) << dlerror();
+
+ // Exceed glibc's default stack cache to exercise TLS deallocation instead
+ // of only keeping exited threads' stacks cached.
+ for (int round = 0; round < 8; ++round) {
+ std::array<int, 64> results = {};
+ std::array<std::thread, 64> threads;
+ for (size_t i = 0; i < threads.size(); ++i) {
+ threads[i] = std::thread([&, i] {
+ for (int call = 0; call < 8; ++call) {
+ results[i] = touch_tls();
+ }
+ });
+ }
+ for (auto& thread : threads) {
+ thread.join();
+ }
+ for (int result : results) {
+ EXPECT_EQ(8, result);
+ }
+ }
+}
+
+} // namespace doris
diff --git a/be/test/jemalloc_hook/jemalloc_hook_test_dso.cpp
b/be/test/jemalloc_hook/jemalloc_hook_test_dso.cpp
new file mode 100644
index 00000000000..21486d3d66e
--- /dev/null
+++ b/be/test/jemalloc_hook/jemalloc_hook_test_dso.cpp
@@ -0,0 +1,44 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cstdlib>
+
+extern "C" void* __libc_memalign(size_t alignment, size_t size) noexcept;
+extern "C" void* __libc_malloc(size_t size) noexcept;
+extern "C" void __libc_free(void* ptr) noexcept;
+extern "C" void* __libc_calloc(size_t count, size_t size) noexcept;
+extern "C" void* __libc_realloc(void* ptr, size_t size) noexcept;
+extern "C" void* __libc_valloc(size_t size) noexcept;
+extern "C" void* __libc_pvalloc(size_t size) noexcept;
+
+extern "C" void jemalloc_hook_test_allocator_symbols(void** symbols) {
+ symbols[0] = reinterpret_cast<void*>(&__libc_memalign);
+ symbols[1] = reinterpret_cast<void*>(&std::malloc);
+ symbols[2] = reinterpret_cast<void*>(&std::free);
+ symbols[3] = reinterpret_cast<void*>(&__libc_malloc);
+ symbols[4] = reinterpret_cast<void*>(&__libc_free);
+ symbols[5] = reinterpret_cast<void*>(&__libc_calloc);
+ symbols[6] = reinterpret_cast<void*>(&__libc_realloc);
+ symbols[7] = reinterpret_cast<void*>(&__libc_valloc);
+ symbols[8] = reinterpret_cast<void*>(&__libc_pvalloc);
+}
+
+// Access TLS from a late-loaded library, as happens with JNI native libraries.
+extern "C" int jemalloc_hook_test_touch_tls() {
+ static thread_local int tls_values[1024] = {};
+ return ++tls_values[1023];
+}
diff --git a/run-be-ut.sh b/run-be-ut.sh
index 915e877610e..b0b4926c6ca 100755
--- a/run-be-ut.sh
+++ b/run-be-ut.sh
@@ -22,6 +22,7 @@
# Optional options:
# --clean clean and build ut
# --run build and run all ut
+# --jemalloc-hook build standalone jemalloc hook tests without
sanitizers
# --run --filter=xx build and run specified ut
# --gdb debug with gdb, does not take effect if --run is not
specified
# --coverage generate coverage report, does not take effect if
--gdb is specified
@@ -117,6 +118,7 @@ Usage: $0 <options>
Optional options:
--clean clean and build ut
--run build and run all ut
+ --jemalloc-hook build standalone Linux jemalloc hook tests without
sanitizers
--run --filter=xx build and run specified ut
--run --gen_out generate expected check data for test
--run --gen_regression_case generate regression test cases
corrresponding to ut cases for ut cases that support it
@@ -145,7 +147,7 @@ Usage: $0 <options>
exit 1
}
-if ! OPTS="$(getopt -n "$0" -o vhj:f: -l
gen_out,gen_regression_case,coverage,benchmark,run,gdb,clean,filter: -- "$@")";
then
+if ! OPTS="$(getopt -n "$0" -o vhj:f: -l
gen_out,gen_regression_case,coverage,benchmark,run,gdb,clean,jemalloc-hook,filter:
-- "$@")"; then
usage
fi
@@ -153,6 +155,7 @@ eval set -- "${OPTS}"
CLEAN=0
RUN=0
+JEMALLOC_HOOK=0
GDB=0
DENABLE_CLANG_COVERAGE='OFF'
BUILD_AZURE='ON'
@@ -170,6 +173,10 @@ if [[ "$#" != 1 ]]; then
RUN=1
shift
;;
+ --jemalloc-hook)
+ JEMALLOC_HOOK=1
+ shift
+ ;;
--gdb)
GDB=1
shift
@@ -209,6 +216,29 @@ if [[ -z "${PARALLEL}" ]]; then
PARALLEL="$(($(nproc) / 5 + 1))"
fi
+# The regular BE UT uses ASAN and disables jemalloc. Exercise allocator
+# interposition in a separate native executable, before initializing BE's
+# unrelated submodules or configuring the sanitizer build.
+if [[ "${JEMALLOC_HOOK}" -eq 1 ]]; then
+ # These modes belong to the sanitizer BE UT, not the native allocator test.
+ if [[ "${GDB}" -eq 1 || "${DENABLE_CLANG_COVERAGE}" == 'ON' || -n
"${GEN_OUT}${GEN_REGRESSION_CASE}" ]]; then
+ echo "--jemalloc-hook supports --run, --filter, --clean and -j"
+ exit 1
+ fi
+ JEMALLOC_TEST_BUILD_DIR="${DORIS_HOME}/be/ut_build_JEMALLOC"
+ if [[ "${CLEAN}" -eq 1 ]]; then
+ rm -rf "${JEMALLOC_TEST_BUILD_DIR}"
+ fi
+ "${CMAKE_CMD}" -S "${DORIS_HOME}/be/test/jemalloc_hook" \
+ -B "${JEMALLOC_TEST_BUILD_DIR}" -G "${GENERATOR}"
+ "${CMAKE_CMD}" --build "${JEMALLOC_TEST_BUILD_DIR}" --parallel
"${PARALLEL}"
+ if [[ "${RUN}" -eq 1 ]]; then
+ "${JEMALLOC_TEST_BUILD_DIR}/jemalloc_hook_test"
"${FILTER:---gtest_filter=*}" \
+
"--gtest_output=xml:${JEMALLOC_TEST_BUILD_DIR}/gtest_output/jemalloc_hook_test.xml"
+ fi
+ exit 0
+fi
+
CMAKE_BUILD_TYPE="${BUILD_TYPE_UT:-ASAN}"
CMAKE_BUILD_TYPE="$(echo "${CMAKE_BUILD_TYPE}" | awk '{ print(toupper($0)) }')"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
