This is an automated email from the ASF dual-hosted git repository.
lxy-9602 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-cpp.git
The following commit(s) were added to refs/heads/main by this push:
new c5e130d6 perf(read): warm up only raw bytes by default (#366)
c5e130d6 is described below
commit c5e130d6e01a10e885b47242072a4ca47ea4957d
Author: Yonghao Fang <[email protected]>
AuthorDate: Thu Sep 17 20:28:03 2026 +0800
perf(read): warm up only raw bytes by default (#366)
---
include/paimon/read_context.h | 2 +-
include/paimon/utils/prefetch_cache_config.h | 6 +++---
src/paimon/common/reader/data_file_reader_factory.h | 2 +-
src/paimon/core/operation/read_context.cpp | 4 ++--
src/paimon/core/operation/read_context_test.cpp | 8 ++++----
src/paimon/core/table/system/system_table_test.cpp | 2 +-
6 files changed, 12 insertions(+), 12 deletions(-)
diff --git a/include/paimon/read_context.h b/include/paimon/read_context.h
index 306e7350..e24a0e7d 100644
--- a/include/paimon/read_context.h
+++ b/include/paimon/read_context.h
@@ -362,7 +362,7 @@ class PAIMON_EXPORT ReadContextBuilder {
/// Warmup overlaps remote-storage latency with the read of the current
file. Higher levels hide
/// more latency but use more memory, and may warm files that a query
never reads (for example
/// when a LIMIT stops the scan early).
- /// @param level The warmup level to use (default: WarmupLevel::DECODED).
+ /// @param level The warmup level to use (default: WarmupLevel::RAW).
/// @return Reference to this builder for method chaining.
/// @note WarmupLevel::RAW warms the read-ahead cache, so it has no effect
and behaves like
/// WarmupLevel::NONE when the cache is off (see
SetReadAheadCacheEnabled()).
diff --git a/include/paimon/utils/prefetch_cache_config.h
b/include/paimon/utils/prefetch_cache_config.h
index d767fd92..10a82f1f 100644
--- a/include/paimon/utils/prefetch_cache_config.h
+++ b/include/paimon/utils/prefetch_cache_config.h
@@ -138,11 +138,11 @@ enum class PAIMON_EXPORT WarmupLevel {
/// Fetch only the next file's raw, still-compressed bytes into memory,
and leave the decoder
/// alone. Overlaps the remote fetch while keeping memory lower than
`DECODED`, because no
/// decoded batches are materialized ahead of the read. It fetches through
the read-ahead
- /// cache, so it falls back to `NONE` when that cache is disabled.
+ /// cache, so it falls back to `NONE` when that cache is disabled. This is
the default, as it
+ /// hides the remote fetch without committing memory to decoded batches.
RAW,
/// Fetch the raw bytes and start the background decode loop as well, so
decoded batches are
- /// ready before the file is read. Hides the most latency but uses the
most memory. This is the
- /// default.
+ /// ready before the file is read. Hides the most latency but uses the
most memory.
DECODED,
};
diff --git a/src/paimon/common/reader/data_file_reader_factory.h
b/src/paimon/common/reader/data_file_reader_factory.h
index 3be8f660..1b6f8cd1 100644
--- a/src/paimon/common/reader/data_file_reader_factory.h
+++ b/src/paimon/common/reader/data_file_reader_factory.h
@@ -56,7 +56,7 @@ struct DataFileReadOptions {
bool prefetch_io_metrics_enabled = false;
/// How far the prefetching reader prepares a file before it is read. It
only matters when
/// prefetching, because that reader is the one that can start work ahead
of the read.
- WarmupLevel warmup_level = WarmupLevel::DECODED;
+ WarmupLevel warmup_level = WarmupLevel::RAW;
};
/// Opens one data file as a `FileBatchReader`.
diff --git a/src/paimon/core/operation/read_context.cpp
b/src/paimon/core/operation/read_context.cpp
index 5626e8c2..ec6d65b8 100644
--- a/src/paimon/core/operation/read_context.cpp
+++ b/src/paimon/core/operation/read_context.cpp
@@ -115,7 +115,7 @@ class ReadContextBuilder::Impl {
realtime_context_.reset();
cache_config_ = CacheConfig();
cache_.reset();
- warmup_level_ = WarmupLevel::DECODED;
+ warmup_level_ = WarmupLevel::RAW;
}
private:
@@ -148,7 +148,7 @@ class ReadContextBuilder::Impl {
bool read_ahead_cache_enabled_ = true;
CacheConfig cache_config_;
std::shared_ptr<Cache> cache_;
- WarmupLevel warmup_level_ = WarmupLevel::DECODED;
+ WarmupLevel warmup_level_ = WarmupLevel::RAW;
};
ReadContextBuilder::ReadContextBuilder(const std::string& path)
diff --git a/src/paimon/core/operation/read_context_test.cpp
b/src/paimon/core/operation/read_context_test.cpp
index e1cdc3eb..62e09423 100644
--- a/src/paimon/core/operation/read_context_test.cpp
+++ b/src/paimon/core/operation/read_context_test.cpp
@@ -47,7 +47,7 @@ TEST(ReadContextTest, TestDefaultValue) {
ASSERT_FALSE(ctx->EnablePredicateFilter());
ASSERT_FALSE(ctx->EnablePrefetch());
ASSERT_TRUE(ctx->ReadAheadCacheEnabled());
- ASSERT_EQ(WarmupLevel::DECODED, ctx->GetWarmupLevel());
+ ASSERT_EQ(WarmupLevel::RAW, ctx->GetWarmupLevel());
ASSERT_EQ(600, ctx->GetPrefetchBatchCount());
ASSERT_EQ(3, ctx->GetPrefetchMaxParallelNum());
ASSERT_FALSE(ctx->EnableMultiThreadRowToBatch());
@@ -75,7 +75,7 @@ TEST(ReadContextTest, TestSetContent) {
builder.EnablePredicateFilter(true);
builder.EnablePrefetch(true);
builder.SetReadAheadCacheEnabled(false);
- builder.SetWarmupLevel(WarmupLevel::RAW);
+ builder.SetWarmupLevel(WarmupLevel::DECODED);
builder.SetPrefetchBatchCount(1200);
builder.SetPrefetchMaxParallelNum(6);
builder.EnableMultiThreadRowToBatch(true);
@@ -101,7 +101,7 @@ TEST(ReadContextTest, TestSetContent) {
ASSERT_TRUE(ctx->EnablePredicateFilter());
ASSERT_TRUE(ctx->EnablePrefetch());
ASSERT_FALSE(ctx->ReadAheadCacheEnabled());
- ASSERT_EQ(WarmupLevel::RAW, ctx->GetWarmupLevel());
+ ASSERT_EQ(WarmupLevel::DECODED, ctx->GetWarmupLevel());
ASSERT_EQ(1200, ctx->GetPrefetchBatchCount());
ASSERT_EQ(6, ctx->GetPrefetchMaxParallelNum());
ASSERT_TRUE(ctx->EnableMultiThreadRowToBatch());
@@ -136,7 +136,7 @@ TEST(ReadContextTest, TestSetWarmupLevel) {
ASSERT_OK_AND_ASSIGN(auto first_ctx, builder.Finish());
ASSERT_EQ(WarmupLevel::NONE, first_ctx->GetWarmupLevel());
ASSERT_OK_AND_ASSIGN(auto second_ctx, builder.Finish());
- ASSERT_EQ(WarmupLevel::DECODED, second_ctx->GetWarmupLevel());
+ ASSERT_EQ(WarmupLevel::RAW, second_ctx->GetWarmupLevel());
}
TEST(ReadContextTest, TestSetOptionsOverridesAddedOptions) {
diff --git a/src/paimon/core/table/system/system_table_test.cpp
b/src/paimon/core/table/system/system_table_test.cpp
index 00a538f3..15092fec 100644
--- a/src/paimon/core/table/system/system_table_test.cpp
+++ b/src/paimon/core/table/system/system_table_test.cpp
@@ -234,7 +234,7 @@ TEST(SystemTableTest,
TestReadOptimizedSystemTablePathParsing) {
// A system table builds a fresh ReadContext for the data table underneath it,
starting from the
// defaults, so a setting that is not copied across silently reverts: a caller
that asked for NONE
-// or RAW would get DECODED back. `$ro` has its own builder chain and
`$audit_log` and `$binlog`
+// or DECODED would get RAW back. `$ro` has its own builder chain and
`$audit_log` and `$binlog`
// share one, so all three are pinned here.
TEST(SystemTableTest, TestNewReadPropagatesWarmupLevel) {
std::map<std::string, std::string> options = {{Options::FILE_SYSTEM,
"local"},
