This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 7e2ebabd0 feat: add jemalloc as optional custom allocator (#1679)
7e2ebabd0 is described below
commit 7e2ebabd0a57cd73f764fe832a975673fc746830
Author: Matt Butrovich <[email protected]>
AuthorDate: Fri Apr 25 13:15:01 2025 -0400
feat: add jemalloc as optional custom allocator (#1679)
* Add jemalloc, snmalloc, and tcmalloc.
* Remove unnecessary check. rustc yelled at me if I tried to enable two.
* Adjust snmalloc following instructions.
* Remove tcmalloc since I can't get it to compile.
* Fix jemalloc loading with dlopen.
* Remove snmalloc.
---
native/Cargo.lock | 21 +++++++++++++++++++++
native/core/Cargo.toml | 8 +++++---
native/core/src/lib.rs | 10 +++++++++-
3 files changed, 35 insertions(+), 4 deletions(-)
diff --git a/native/Cargo.lock b/native/Cargo.lock
index 3c2b7c605..ce7dc7f7f 100644
--- a/native/Cargo.lock
+++ b/native/Cargo.lock
@@ -999,6 +999,7 @@ dependencies = [
"snap",
"tempfile",
"thiserror 2.0.12",
+ "tikv-jemallocator",
"tokio",
"url",
"zstd",
@@ -3862,6 +3863,26 @@ dependencies = [
"ordered-float",
]
+[[package]]
+name = "tikv-jemalloc-sys"
+version = "0.6.0+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "cd3c60906412afa9c2b5b5a48ca6a5abe5736aec9eb48ad05037a677e52e4e2d"
+dependencies = [
+ "cc",
+ "libc",
+]
+
+[[package]]
+name = "tikv-jemallocator"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "4cec5ff18518d81584f477e9bfdf957f5bb0979b0bac3af4ca30b5b3ae2d2865"
+dependencies = [
+ "libc",
+ "tikv-jemalloc-sys",
+]
+
[[package]]
name = "tiny-keccak"
version = "2.0.2"
diff --git a/native/core/Cargo.toml b/native/core/Cargo.toml
index fd8805794..e50226935 100644
--- a/native/core/Cargo.toml
+++ b/native/core/Cargo.toml
@@ -39,6 +39,7 @@ arrow = { workspace = true }
parquet = { workspace = true, default-features = false, features =
["experimental"] }
futures = { workspace = true }
mimalloc = { version = "*", default-features = false, optional = true }
+tikv-jemallocator = { version = "0.6.0", optional = true, features =
["disable_initial_exec_tls"] }
tokio = { version = "1", features = ["rt-multi-thread"] }
async-trait = { workspace = true }
log = "0.4"
@@ -51,13 +52,13 @@ snap = "1.1"
# we disable default features in lz4_flex to force the use of the faster
unsafe encoding and decoding implementation
lz4_flex = { version = "0.11.3", default-features = false }
zstd = "0.13.3"
-rand = { workspace = true}
+rand = { workspace = true }
num = { workspace = true }
bytes = { workspace = true }
tempfile = "3.8.0"
itertools = "0.14.0"
paste = "1.0.14"
-datafusion = { workspace = true }
+datafusion = { workspace = true }
once_cell = "1.18.0"
regex = { workspace = true }
crc32fast = "1.3.2"
@@ -80,7 +81,8 @@ datafusion-functions-nested = { version = "47.0.0" }
[features]
default = []
-hdfs=["datafusion-comet-objectstore-hdfs"]
+hdfs = ["datafusion-comet-objectstore-hdfs"]
+jemalloc = ["tikv-jemallocator"]
# exclude optional packages from cargo machete verifications
[package.metadata.cargo-machete]
diff --git a/native/core/src/lib.rs b/native/core/src/lib.rs
index 418075147..c6491eebe 100644
--- a/native/core/src/lib.rs
+++ b/native/core/src/lib.rs
@@ -36,9 +36,13 @@ use log4rs::{
encode::pattern::PatternEncoder,
Config,
};
+use once_cell::sync::OnceCell;
+
+#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
+use tikv_jemallocator::Jemalloc;
+
#[cfg(feature = "mimalloc")]
use mimalloc::MiMalloc;
-use once_cell::sync::OnceCell;
use errors::{try_unwrap_or_throw, CometError, CometResult};
@@ -50,6 +54,10 @@ pub mod execution;
mod jvm_bridge;
pub mod parquet;
+#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
+#[global_allocator]
+static GLOBAL: Jemalloc = Jemalloc;
+
#[cfg(feature = "mimalloc")]
#[global_allocator]
static GLOBAL: MiMalloc = MiMalloc;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
