This is an automated email from the ASF dual-hosted git repository.
liaoxin01 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 11fa8caee86 [fix](be) Fix strict load crash when direct slot mappings
are absent (#67633)
11fa8caee86 is described below
commit 11fa8caee86817ef51e2c8dbfde2dfbe80b7682c
Author: meiyi <[email protected]>
AuthorDate: Mon Sep 14 20:25:30 2026 +0800
[fix](be) Fix strict load crash when direct slot mappings are absent
(#67633)
Problem Summary: An expression-only load can legitimately omit the
optional destination-to-source direct slot map. FileScanner previously
skipped initializing its destination-ordered source descriptor vector in
that case. When an expression produced NULL under strict mode, output
conversion indexed the empty vector and could abort the BE.
```
*** SIGABRT unknown detail explain (@0x3f500000373) received by PID 883
(TID 2653 OR 0x7bd43699f700) from PID 883; stack trace: ***
0# doris::signal::(anonymous namespace)::FailureSignalHandler(int,
siginfo_t*, void*) at ../src/common/signal_handler.h:418
1# 0x00007FE3F5E55140 in /lib/x86_64-linux-gnu/libpthread.so.0
2# raise in /lib/x86_64-linux-gnu/libc.so.6
3# abort in /lib/x86_64-linux-gnu/libc.so.6
4# 0x000055E3C2D186A7 in /opt/apache-doris/be/lib/doris_be
5# std::vector<doris::SlotDescriptor*,
std::allocator<doris::SlotDescriptor*> >::operator[](unsigned long) at
/mnt/disk2/meiyi/soft/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/15/include/g++-v15/bits/stl_vector.h:1263
6# doris::FileScanner::_convert_to_output_block(doris::Block*) at
./be/src/exec/scan/file_scanner.cpp:810
7#
doris::FileScanner::_process_src_block_after_read_for_load(doris::Block*) at
./be/src/exec/scan/file_scanner.cpp:902
8# doris::FileScanner::_process_src_block_after_read(doris::Block*) at
./be/src/exec/scan/file_scanner.cpp:868
9# doris::FileScanner::_get_block_wrapped(doris::RuntimeState*,
doris::Block*, bool*) at ./be/src/exec/scan/file_scanner.cpp:595
10# doris::FileScanner::_get_block_impl(doris::RuntimeState*,
doris::Block*, bool*) at ./be/src/exec/scan/file_scanner.cpp:514
11# doris::Scanner::get_block(doris::RuntimeState*, doris::Block*, bool*)
at ./be/src/exec/scan/scanner.cpp:171
12# doris::Scanner::get_block_after_projects(doris::RuntimeState*,
doris::Block*, bool*) at ./be/src/exec/scan/scanner.cpp:138
13#
doris::ScannerScheduler::_scanner_scan(std::shared_ptr<doris::ScannerContext>,
std::shared_ptr<doris::ScanTask>) at
./be/src/exec/scan/scanner_scheduler.cpp:193
```
Initialize one source descriptor entry for every destination expression,
using nullptr when no direct mapping exists. This preserves destination
index alignment and the existing strict-mode and NOT NULL filtering
semantics.
---
be/src/exec/scan/file_scanner.cpp | 24 ++---
...st_stream_load_strict_mode_and_filter_ratio.out | 32 ++++++
...stream_load_strict_mode_and_filter_ratio.groovy | 116 ++++++++++++++++++++-
3 files changed, 157 insertions(+), 15 deletions(-)
diff --git a/be/src/exec/scan/file_scanner.cpp
b/be/src/exec/scan/file_scanner.cpp
index d9306a37f76..12564cfc79a 100644
--- a/be/src/exec/scan/file_scanner.cpp
+++ b/be/src/exec/scan/file_scanner.cpp
@@ -1839,7 +1839,6 @@ Status FileScanner::_init_expr_ctxes() {
if (_is_load) {
// follow desc expr map is only for load task.
- bool has_slot_id_map =
_params->__isset.dest_sid_to_src_sid_without_trans;
int idx = 0;
for (auto* slot_desc : _output_tuple_desc->slots()) {
auto it = _params->expr_of_dest_slot.find(slot_desc->id());
@@ -1857,20 +1856,17 @@ Status FileScanner::_init_expr_ctxes() {
_dest_vexpr_ctx.emplace_back(ctx);
_dest_slot_name_to_idx[slot_desc->col_name()] = idx++;
- if (has_slot_id_map) {
- auto it1 =
_params->dest_sid_to_src_sid_without_trans.find(slot_desc->id());
- if (it1 ==
std::end(_params->dest_sid_to_src_sid_without_trans)) {
- _src_slot_descs_order_by_dest.emplace_back(nullptr);
- } else {
- auto _src_slot_it = full_src_slot_map.find(it1->second);
- if (_src_slot_it == std::end(full_src_slot_map)) {
- return Status::InternalError("No src slot {} in src
slot descs",
- it1->second);
- }
-
_dest_slot_to_src_slot_index.emplace(_src_slot_descs_order_by_dest.size(),
-
full_src_index_map[_src_slot_it->first]);
-
_src_slot_descs_order_by_dest.emplace_back(_src_slot_it->second);
+ auto it1 =
_params->dest_sid_to_src_sid_without_trans.find(slot_desc->id());
+ if (it1 == std::end(_params->dest_sid_to_src_sid_without_trans)) {
+ _src_slot_descs_order_by_dest.emplace_back(nullptr);
+ } else {
+ auto _src_slot_it = full_src_slot_map.find(it1->second);
+ if (_src_slot_it == std::end(full_src_slot_map)) {
+ return Status::InternalError("No src slot {} in src slot
descs", it1->second);
}
+
_dest_slot_to_src_slot_index.emplace(_src_slot_descs_order_by_dest.size(),
+
full_src_index_map[_src_slot_it->first]);
+
_src_slot_descs_order_by_dest.emplace_back(_src_slot_it->second);
}
}
}
diff --git
a/regression-test/data/load_p0/stream_load/test_stream_load_strict_mode_and_filter_ratio.out
b/regression-test/data/load_p0/stream_load/test_stream_load_strict_mode_and_filter_ratio.out
index 13f0cc3e95d..fd086fd3a4a 100644
---
a/regression-test/data/load_p0/stream_load/test_stream_load_strict_mode_and_filter_ratio.out
+++
b/regression-test/data/load_p0/stream_load/test_stream_load_strict_mode_and_filter_ratio.out
@@ -135,3 +135,35 @@
70 a70
79 a79
+-- !sql_all_expr_strict --
+1234567890
+1234567890
+1234567893
+1234567894
+1234567895
+1234567896
+1234567897
+
+-- !sql_all_expr_strict_filter_ratio_exceeded --
+
+-- !sql_all_expr_nullable_strict --
+\N
+\N
+\N
+1234567890
+1234567890
+1234567893
+1234567894
+1234567895
+1234567896
+1234567897
+
+-- !sql_mixed_mapping_strict --
+\N 1234567890
+\N 1234567890
+\N 1234567893
+\N 1234567894
+\N 1234567895
+\N 1234567896
+\N 1234567897
+
diff --git
a/regression-test/suites/load_p0/stream_load/test_stream_load_strict_mode_and_filter_ratio.groovy
b/regression-test/suites/load_p0/stream_load/test_stream_load_strict_mode_and_filter_ratio.groovy
index fde86b368ea..b50da2794a0 100644
---
a/regression-test/suites/load_p0/stream_load/test_stream_load_strict_mode_and_filter_ratio.groovy
+++
b/regression-test/suites/load_p0/stream_load/test_stream_load_strict_mode_and_filter_ratio.groovy
@@ -624,4 +624,118 @@ suite("test_stream_load_strict_mode_and_filter_ratio",
"p0") {
}
}
qt_sql_string_exceed_len_strict1 "select * from
test_stream_load_strict_mode_and_filter_ratio order by 1"
-}
\ No newline at end of file
+
+ // Test strict-mode loading into a NOT NULL destination with only
expression mappings.
+ // Invalid casts produce NULL and must be filtered even when the direct
slot map is unset.
+ sql """ drop table if exists test_stream_load_strict_mode_and_filter_ratio
"""
+ sql """
+ create table test_stream_load_strict_mode_and_filter_ratio (
+ k00 DECIMALV3(10, 0) NOT NULL
+ ) properties ('replication_num' = '1');
+ """
+ streamLoad {
+ table "test_stream_load_strict_mode_and_filter_ratio"
+ file "test_not_number.csv"
+ set 'column_separator', '|'
+ set 'columns', 'src, k00=cast(src as bigint)'
+ set 'strict_mode', 'true'
+ set 'max_filter_ratio', '0.3'
+
+ check { result, exception, startTime, endTime ->
+ if (exception != null) {
+ throw exception
+ }
+ def json = parseJson(result)
+ assertEquals("success", json.Status.toLowerCase())
+ assertEquals(10, json.NumberTotalRows)
+ assertEquals(7, json.NumberLoadedRows)
+ assertEquals(3, json.NumberFilteredRows)
+ assertTrue(result.contains("ErrorURL"))
+ }
+ }
+ qt_sql_all_expr_strict "select * from
test_stream_load_strict_mode_and_filter_ratio order by 1"
+
+ // Exceeding the filter ratio must fail the load without committing any
rows.
+ sql """ truncate table test_stream_load_strict_mode_and_filter_ratio """
+ streamLoad {
+ table "test_stream_load_strict_mode_and_filter_ratio"
+ file "test_not_number.csv"
+ set 'column_separator', '|'
+ set 'columns', 'src, k00=cast(src as bigint)'
+ set 'strict_mode', 'true'
+ set 'max_filter_ratio', '0.2'
+
+ check { result, exception, startTime, endTime ->
+ if (exception != null) {
+ throw exception
+ }
+ def json = parseJson(result)
+ assertEquals("fail", json.Status.toLowerCase())
+ assertEquals(10, json.NumberTotalRows)
+ assertEquals(0, json.NumberLoadedRows)
+ assertEquals(3, json.NumberFilteredRows)
+ assertTrue(json.Message.contains("too many filtered rows"))
+ assertTrue(result.contains("ErrorURL"))
+ }
+ }
+ qt_sql_all_expr_strict_filter_ratio_exceeded "select * from
test_stream_load_strict_mode_and_filter_ratio order by 1"
+
+ // With no direct slot mapping, expression NULLs are retained by a
nullable destination.
+ sql """ drop table if exists test_stream_load_strict_mode_and_filter_ratio
"""
+ sql """
+ create table test_stream_load_strict_mode_and_filter_ratio (
+ k00 DECIMALV3(10, 0) NULL
+ ) properties ('replication_num' = '1');
+ """
+ streamLoad {
+ table "test_stream_load_strict_mode_and_filter_ratio"
+ file "test_not_number.csv"
+ set 'column_separator', '|'
+ set 'columns', 'src, k00=cast(src as bigint)'
+ set 'strict_mode', 'true'
+ set 'max_filter_ratio', '0'
+
+ check { result, exception, startTime, endTime ->
+ if (exception != null) {
+ throw exception
+ }
+ def json = parseJson(result)
+ assertEquals("success", json.Status.toLowerCase())
+ assertEquals(10, json.NumberTotalRows)
+ assertEquals(10, json.NumberLoadedRows)
+ assertEquals(0, json.NumberFilteredRows)
+ }
+ }
+ qt_sql_all_expr_nullable_strict "select * from
test_stream_load_strict_mode_and_filter_ratio order by 1"
+
+ // Put an expression before a direct column to check destination/source
index alignment.
+ // The expression is always NULL; strict mode only filters the three
invalid direct values.
+ sql """ drop table if exists test_stream_load_strict_mode_and_filter_ratio
"""
+ sql """
+ create table test_stream_load_strict_mode_and_filter_ratio (
+ k00 DECIMALV3(10, 0) NULL,
+ k01 DECIMALV3(10, 0) NULL
+ ) properties ('replication_num' = '1');
+ """
+ streamLoad {
+ table "test_stream_load_strict_mode_and_filter_ratio"
+ file "test_not_number.csv"
+ set 'column_separator', '|'
+ set 'columns', "k01, k00=cast(concat('invalid', k01) as bigint)"
+ set 'strict_mode', 'true'
+ set 'max_filter_ratio', '0.3'
+
+ check { result, exception, startTime, endTime ->
+ if (exception != null) {
+ throw exception
+ }
+ def json = parseJson(result)
+ assertEquals("success", json.Status.toLowerCase())
+ assertEquals(10, json.NumberTotalRows)
+ assertEquals(7, json.NumberLoadedRows)
+ assertEquals(3, json.NumberFilteredRows)
+ assertTrue(result.contains("ErrorURL"))
+ }
+ }
+ qt_sql_mixed_mapping_strict "select * from
test_stream_load_strict_mode_and_filter_ratio order by 1, 2"
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
