This is an automated email from the ASF dual-hosted git repository.
Gabriel39 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 20bb1bb4cab [fix](be) Keep Paimon JNI predicate compatibility (#65328)
20bb1bb4cab is described below
commit 20bb1bb4cabdd2904714fb0bd296d474a5013ac2
Author: Gabriel <[email protected]>
AuthorDate: Wed Jul 8 12:46:24 2026 +0800
[fix](be) Keep Paimon JNI predicate compatibility (#65328)
Format V2 Paimon JNI reader now falls back to legacy split-level
paimon_predicate when scan-level predicate is missing or empty. Added BE
unit coverage for scan priority, legacy fallback, and missing predicate
failure. Validation: build-support/clang-format.sh passed;
./run-be-ut.sh --run --filter=PaimonJniReaderTest* was attempted but
local CMake is blocked by incomplete thirdparty/installed dependencies
after missing Protobuf library.
---
be/src/format_v2/jni/paimon_jni_reader.cpp | 22 +++-
be/test/format_v2/jni/paimon_jni_reader_test.cpp | 127 +++++++++++++++++++++++
2 files changed, 145 insertions(+), 4 deletions(-)
diff --git a/be/src/format_v2/jni/paimon_jni_reader.cpp
b/be/src/format_v2/jni/paimon_jni_reader.cpp
index 3902093b7c8..ca1e602cdfe 100644
--- a/be/src/format_v2/jni/paimon_jni_reader.cpp
+++ b/be/src/format_v2/jni/paimon_jni_reader.cpp
@@ -32,6 +32,18 @@ constexpr std::string_view DORIS_ENABLE_JNI_IO_MANAGER =
"doris.enable_jni_io_ma
constexpr std::string_view DORIS_JNI_IO_MANAGER_TMP_DIR =
"doris.jni_io_manager.tmp_dir";
constexpr std::string_view PAIMON_JNI_SCANNER_IO_TMP_DIR =
"paimon_jni_scanner_io_tmp";
+const std::string* get_paimon_predicate(const TFileScanRangeParams*
scan_params,
+ const TPaimonFileDesc& paimon_params) {
+ if (scan_params != nullptr && scan_params->__isset.paimon_predicate &&
+ !scan_params->paimon_predicate.empty()) {
+ return &scan_params->paimon_predicate;
+ }
+ if (paimon_params.__isset.paimon_predicate &&
!paimon_params.paimon_predicate.empty()) {
+ return &paimon_params.paimon_predicate;
+ }
+ return nullptr;
+}
+
} // namespace
Status PaimonJniReader::validate_scan_range(const TFileRangeDesc& range) const
{
@@ -59,7 +71,7 @@ Status PaimonJniReader::validate_scan_range(const
TFileRangeDesc& range) const {
"missing serialized_table for paimon jni reader, possibly
caused by FE/BE "
"protocol mismatch");
}
- if (!_scan_params->__isset.paimon_predicate ||
_scan_params->paimon_predicate.empty()) {
+ if (get_paimon_predicate(_scan_params,
range.table_format_params.paimon_params) == nullptr) {
return Status::InternalError(
"missing paimon_predicate for paimon jni reader, possibly
caused by FE/BE "
"protocol mismatch");
@@ -77,8 +89,10 @@ Status
PaimonJniReader::build_scanner_params(std::map<std::string, std::string>*
params->clear();
const auto& paimon_params =
_current_range.table_format_params.paimon_params;
+ const auto* paimon_predicate = get_paimon_predicate(_scan_params,
paimon_params);
+ DORIS_CHECK(paimon_predicate != nullptr);
(*params)["paimon_split"] = paimon_params.paimon_split;
- (*params)["paimon_predicate"] = _scan_params->paimon_predicate;
+ (*params)["paimon_predicate"] = *paimon_predicate;
(*params)["serialized_table"] = _scan_params->serialized_table;
if (_scan_params->__isset.paimon_options &&
!_scan_params->paimon_options.empty()) {
@@ -106,8 +120,8 @@ Status
PaimonJniReader::build_scanner_params(std::map<std::string, std::string>*
}
}
// TODO: Remove legacy split-level paimon_predicate, paimon_options and
hadoop_conf from thrift
- // after all readers stop using them. Format V2 Paimon JNI consumes the
scan-level fields
- // planned by current FE and intentionally does not fall back to
deprecated split-level fields.
+ // after all readers stop using them. Predicate keeps the split-level
fallback for rolling
+ // upgrade compatibility with old FE paths that did not send scan-level
paimon_predicate.
return Status::OK();
}
diff --git a/be/test/format_v2/jni/paimon_jni_reader_test.cpp
b/be/test/format_v2/jni/paimon_jni_reader_test.cpp
new file mode 100644
index 00000000000..08fd333abd3
--- /dev/null
+++ b/be/test/format_v2/jni/paimon_jni_reader_test.cpp
@@ -0,0 +1,127 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "format_v2/jni/paimon_jni_reader.h"
+
+#include <gtest/gtest.h>
+
+#include <map>
+#include <string>
+#include <utility>
+
+#include "format_v2/table_reader.h"
+#include "gen_cpp/PlanNodes_types.h"
+
+namespace doris::format::paimon {
+namespace {
+
+TFileRangeDesc make_paimon_jni_range() {
+ TFileRangeDesc range;
+ TTableFormatFileDesc table_format_params;
+ table_format_params.__set_table_format_type("paimon");
+ TPaimonFileDesc paimon_params;
+ paimon_params.__set_reader_type(TPaimonReaderType::PAIMON_JNI);
+ paimon_params.__set_paimon_split("serialized-split");
+ table_format_params.__set_paimon_params(std::move(paimon_params));
+ range.__set_table_format_params(std::move(table_format_params));
+ return range;
+}
+
+TFileScanRangeParams make_scan_params() {
+ TFileScanRangeParams scan_params;
+ scan_params.__set_serialized_table("serialized-table");
+ return scan_params;
+}
+
+Status init_reader(PaimonJniReader* reader, TFileScanRangeParams* scan_params)
{
+ return reader->init({
+ .projected_columns = {},
+ .conjuncts = {},
+ .format = FileFormat::JNI,
+ .scan_params = scan_params,
+ .io_ctx = nullptr,
+ .runtime_state = nullptr,
+ .scanner_profile = nullptr,
+ });
+}
+
+Status build_params(PaimonJniReader* reader, const TFileRangeDesc& range,
+ std::map<std::string, std::string>* params) {
+ reader->_current_range = range;
+ return reader->build_scanner_params(params);
+}
+
+TEST(PaimonJniReaderTest, UsesScanLevelPredicateBeforeLegacySplitPredicate) {
+ auto range = make_paimon_jni_range();
+
range.table_format_params.paimon_params.__set_paimon_predicate("legacy-predicate");
+
+ auto scan_params = make_scan_params();
+ scan_params.__set_paimon_predicate("scan-predicate");
+
+ PaimonJniReader reader;
+ ASSERT_TRUE(init_reader(&reader, &scan_params).ok());
+ ASSERT_TRUE(reader.validate_scan_range(range).ok());
+
+ std::map<std::string, std::string> params;
+ ASSERT_TRUE(build_params(&reader, range, ¶ms).ok());
+ EXPECT_EQ(params["paimon_predicate"], "scan-predicate");
+}
+
+TEST(PaimonJniReaderTest,
FallsBackToLegacySplitPredicateWhenScanPredicateIsMissing) {
+ auto range = make_paimon_jni_range();
+
range.table_format_params.paimon_params.__set_paimon_predicate("legacy-predicate");
+
+ auto scan_params = make_scan_params();
+
+ PaimonJniReader reader;
+ ASSERT_TRUE(init_reader(&reader, &scan_params).ok());
+ ASSERT_TRUE(reader.validate_scan_range(range).ok());
+
+ std::map<std::string, std::string> params;
+ ASSERT_TRUE(build_params(&reader, range, ¶ms).ok());
+ EXPECT_EQ(params["paimon_predicate"], "legacy-predicate");
+}
+
+TEST(PaimonJniReaderTest,
FallsBackToLegacySplitPredicateWhenScanPredicateIsEmpty) {
+ auto range = make_paimon_jni_range();
+
range.table_format_params.paimon_params.__set_paimon_predicate("legacy-predicate");
+
+ auto scan_params = make_scan_params();
+ scan_params.__set_paimon_predicate("");
+
+ PaimonJniReader reader;
+ ASSERT_TRUE(init_reader(&reader, &scan_params).ok());
+ ASSERT_TRUE(reader.validate_scan_range(range).ok());
+
+ std::map<std::string, std::string> params;
+ ASSERT_TRUE(build_params(&reader, range, ¶ms).ok());
+ EXPECT_EQ(params["paimon_predicate"], "legacy-predicate");
+}
+
+TEST(PaimonJniReaderTest, RejectsMissingPredicateFromBothProtocolLocations) {
+ const auto range = make_paimon_jni_range();
+ auto scan_params = make_scan_params();
+
+ PaimonJniReader reader;
+ ASSERT_TRUE(init_reader(&reader, &scan_params).ok());
+ const auto status = reader.validate_scan_range(range);
+ EXPECT_FALSE(status.ok());
+ EXPECT_NE(status.to_string().find("missing paimon_predicate"),
std::string::npos);
+}
+
+} // namespace
+} // namespace doris::format::paimon
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
