This is an automated email from the ASF dual-hosted git repository.
zhangstar333 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 6c86c91bb84 [bug](analytic) fix partition by column when it overflow
haven't convert to column64 (#65240)
6c86c91bb84 is described below
commit 6c86c91bb8451d2e804726f927ceb096b7bd1f1f
Author: zhangstar333 <[email protected]>
AuthorDate: Tue Jul 7 11:14:26 2026 +0800
[bug](analytic) fix partition by column when it overflow haven't convert to
column64 (#65240)
### What problem does this PR solve?
when insert into partition by column, will be call
insert_range_from_ignore_overflow function,
the dst column maybe overflow, so need add the
convert_column_if_overflow function to convert to column64。
---
be/src/core/column/column_string.cpp | 56 +++++++++++++---------
be/src/exec/operator/analytic_sink_operator.cpp | 8 ++++
be/test/core/column/column_string_test.cpp | 20 ++++++++
.../window_functions/test_column_boundary.out | 3 +-
4 files changed, 62 insertions(+), 25 deletions(-)
diff --git a/be/src/core/column/column_string.cpp
b/be/src/core/column/column_string.cpp
index a2e128d0d79..b8b7a6c451e 100644
--- a/be/src/core/column/column_string.cpp
+++ b/be/src/core/column/column_string.cpp
@@ -99,33 +99,43 @@ void ColumnStr<T>::insert_range_from_ignore_overflow(const
doris::IColumn& src,
return;
}
- const auto& src_concrete = assert_cast<const ColumnStr<T>&>(src);
- if (start + length > src_concrete.offsets.size()) {
- throw doris::Exception(doris::ErrorCode::INTERNAL_ERROR,
- "Parameter out of bound in "
-
"IColumnStr<T>::insert_range_from_ignore_overflow method.");
- }
+ auto do_insert = [&](const auto& src_concrete) {
+ const auto& src_offsets = src_concrete.get_offsets();
+ const auto& src_chars = src_concrete.get_chars();
+ if (start + length > src_offsets.size()) {
+ throw doris::Exception(doris::ErrorCode::INTERNAL_ERROR,
+ "Parameter out of bound in "
+
"IColumnStr<T>::insert_range_from_ignore_overflow method.");
+ }
- auto nested_offset = src_concrete.offset_at(start);
- auto nested_length = src_concrete.offsets[start + length - 1] -
nested_offset;
+ auto nested_offset = src_offsets[static_cast<ssize_t>(start) - 1];
+ auto nested_length = src_offsets[start + length - 1] - nested_offset;
- size_t old_chars_size = chars.size();
- chars.resize(old_chars_size + nested_length);
- memcpy(&chars[old_chars_size], &src_concrete.chars[nested_offset],
nested_length);
+ size_t old_chars_size = chars.size();
+ chars.resize(old_chars_size + nested_length);
+ memcpy(&chars[old_chars_size], &src_chars[nested_offset],
nested_length);
- if (start == 0 && offsets.empty()) {
- offsets.assign(src_concrete.offsets.begin(),
src_concrete.offsets.begin() + length);
- } else {
- size_t old_size = offsets.size();
- auto prev_max_offset = offsets.back(); /// -1th index is Ok, see
PaddedPODArray
- offsets.resize(old_size + length);
-
- for (size_t i = 0; i < length; ++i) {
- // unsinged integer overflow is well defined in C++,
- // so we don't need to check the overflow here.
- offsets[old_size + i] =
- src_concrete.offsets[start + i] - nested_offset +
prev_max_offset;
+ using OffsetsType = std::decay_t<decltype(src_offsets)>;
+ if (std::is_same_v<T, typename OffsetsType::value_type> && start == 0
&& offsets.empty()) {
+ offsets.assign(src_offsets.begin(), src_offsets.begin() + length);
+ } else {
+ size_t old_size = offsets.size();
+ auto prev_max_offset = offsets.back(); /// -1th index is Ok, see
PaddedPODArray
+ offsets.resize(old_size + length);
+
+ for (size_t i = 0; i < length; ++i) {
+ // unsinged integer overflow is well defined in C++,
+ // so we don't need to check the overflow here.
+ offsets[old_size + i] =
+ static_cast<T>(src_offsets[start + i] - nested_offset)
+ prev_max_offset;
+ }
}
+ };
+
+ if (src.is_column_string64()) {
+ do_insert(assert_cast<const ColumnStr<uint64_t>&>(src));
+ } else {
+ do_insert(assert_cast<const ColumnStr<uint32_t>&>(src));
}
#ifndef NDEBUG
diff --git a/be/src/exec/operator/analytic_sink_operator.cpp
b/be/src/exec/operator/analytic_sink_operator.cpp
index 534cf0b841c..f0d26ec627e 100644
--- a/be/src/exec/operator/analytic_sink_operator.cpp
+++ b/be/src/exec/operator/analytic_sink_operator.cpp
@@ -768,6 +768,12 @@ Status
AnalyticSinkOperatorX::_add_input_block(doris::RuntimeState* state, Block
local_state._input_block_first_row_positions.emplace_back(local_state._input_total_rows);
size_t block_rows = input_block->rows();
local_state._input_total_rows += block_rows;
+ auto convert_column_if_overflow = [](MutableColumnPtr& column) {
+ auto converted_column = column->convert_column_if_overflow();
+ if (converted_column.get() != column.get()) {
+ column = converted_column->assert_mutable();
+ }
+ };
// record origin columns, maybe be after this, could cast some column but
no need to output
auto column_to_keep = input_block->columns();
@@ -788,6 +794,7 @@ Status
AnalyticSinkOperatorX::_add_input_block(doris::RuntimeState* state, Block
RETURN_IF_ERROR(
_insert_range_column(input_block,
local_state._partition_by_eq_expr_ctxs[i],
local_state._partition_by_columns[i].get(), block_rows));
+ convert_column_if_overflow(local_state._partition_by_columns[i]);
}
}
{
@@ -796,6 +803,7 @@ Status
AnalyticSinkOperatorX::_add_input_block(doris::RuntimeState* state, Block
RETURN_IF_ERROR(_insert_range_column(input_block,
local_state._order_by_eq_expr_ctxs[i],
local_state._order_by_columns[i].get(),
block_rows));
+ convert_column_if_overflow(local_state._order_by_columns[i]);
}
}
{
diff --git a/be/test/core/column/column_string_test.cpp
b/be/test/core/column/column_string_test.cpp
index 37de6fe56e2..08ac03dcd99 100644
--- a/be/test/core/column/column_string_test.cpp
+++ b/be/test/core/column/column_string_test.cpp
@@ -190,6 +190,26 @@ TEST_F(ColumnStringTest, is_variable_length) {
ColumnString64::MutablePtr col64 = ColumnString64::create();
EXPECT_TRUE(col64->is_variable_length());
}
+
+TEST(ColumnStringStandaloneTest,
insert_range_from_ignore_overflow_to_string64_from_string32) {
+ auto src = ColumnString::create();
+ src->insert_data("a", 1);
+ src->insert_data("bc", 2);
+ src->insert_data("def", 3);
+
+ auto dst = ColumnString64::create();
+ dst->insert_range_from_ignore_overflow(*src, 0, src->size());
+ ASSERT_EQ(dst->size(), 3);
+ EXPECT_EQ(dst->get_data_at(0).to_string(), "a");
+ EXPECT_EQ(dst->get_data_at(1).to_string(), "bc");
+ EXPECT_EQ(dst->get_data_at(2).to_string(), "def");
+
+ dst->insert_range_from_ignore_overflow(*src, 1, 2);
+ ASSERT_EQ(dst->size(), 5);
+ EXPECT_EQ(dst->get_data_at(3).to_string(), "bc");
+ EXPECT_EQ(dst->get_data_at(4).to_string(), "def");
+}
+
TEST_F(ColumnStringTest, sanity_check) {
auto test_func = [](auto& col) {
auto& chars = col->get_chars();
diff --git
a/regression-test/data/query_p0/sql_functions/window_functions/test_column_boundary.out
b/regression-test/data/query_p0/sql_functions/window_functions/test_column_boundary.out
index 7a238f99935..33fb7867e55 100644
---
a/regression-test/data/query_p0/sql_functions/window_functions/test_column_boundary.out
+++
b/regression-test/data/query_p0/sql_functions/window_functions/test_column_boundary.out
@@ -3,5 +3,4 @@
200000000
-- !sql_2 --
-38135901019538738
-
+40000000000
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
