This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new d883850c484 branch-4.1: [fix](be) Fix array_range overflow and batch
integer output #67584 (#67689)
d883850c484 is described below
commit d883850c4840015b4e9fae94bd81dc43f39c1f08
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Wed Sep 9 22:55:34 2026 +0800
branch-4.1: [fix](be) Fix array_range overflow and batch integer output
#67584 (#67689)
Cherry-picked from #67584
Co-authored-by: HappenLee <[email protected]>
---
be/benchmark/benchmark_array_range.hpp | 97 ++++++++++
be/benchmark/benchmark_main.cpp | 1 +
.../exprs/function/array/function_array_range.cpp | 36 ++--
.../exprs/function/function_array_range_test.cpp | 197 +++++++++++++++++++++
4 files changed, 320 insertions(+), 11 deletions(-)
diff --git a/be/benchmark/benchmark_array_range.hpp
b/be/benchmark/benchmark_array_range.hpp
new file mode 100644
index 00000000000..fa2fc6a1289
--- /dev/null
+++ b/be/benchmark/benchmark_array_range.hpp
@@ -0,0 +1,97 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <benchmark/benchmark.h>
+
+#include <memory>
+
+#include "core/block/block.h"
+#include "core/column/column_vector.h"
+#include "core/data_type/data_type_array.h"
+#include "core/data_type/data_type_nullable.h"
+#include "core/data_type/data_type_number.h"
+#include "exprs/function/simple_function_factory.h"
+#include "exprs/function_context.h"
+
+namespace doris {
+
+static void BM_ArrayRange(benchmark::State& state) {
+ const size_t rows = state.range(0);
+ const auto length = static_cast<Int32>(state.range(1));
+ const auto step = static_cast<Int32>(state.range(2));
+ auto int_type = std::make_shared<DataTypeInt32>();
+ auto result_type =
make_nullable(std::make_shared<DataTypeArray>(make_nullable(int_type)));
+ Block block;
+ auto starts = ColumnInt32::create();
+ auto ends = ColumnInt32::create();
+ auto steps = ColumnInt32::create();
+ for (size_t row = 0; row < rows; ++row) {
+ const auto start = static_cast<Int32>(row % 17);
+ starts->insert_value(start);
+ ends->insert_value(start + length * step);
+ steps->insert_value(step);
+ }
+ block.insert({std::move(starts), int_type, "start"});
+ block.insert({std::move(ends), int_type, "end"});
+ block.insert({std::move(steps), int_type, "step"});
+ auto function = SimpleFunctionFactory::instance().get_function(
+ "array_range", block.get_columns_with_type_and_name(),
result_type);
+ auto context =
+ FunctionContext::create_context(nullptr, result_type, {int_type,
int_type, int_type});
+ auto status = function->open(context.get(),
FunctionContext::FRAGMENT_LOCAL);
+ if (!status.ok()) {
+ state.SkipWithError(status.to_string().c_str());
+ return;
+ }
+ status = function->open(context.get(), FunctionContext::THREAD_LOCAL);
+ if (!status.ok()) {
+ state.SkipWithError(status.to_string().c_str());
+ return;
+ }
+ block.insert({nullptr, result_type, "result"});
+ for (auto _ : state) {
+ status = function->execute(context.get(), block, {0, 1, 2}, 3, rows);
+ if (!status.ok()) {
+ state.SkipWithError(status.to_string().c_str());
+ break;
+ }
+ benchmark::DoNotOptimize(block.get_by_position(3).column);
+ benchmark::ClobberMemory();
+ }
+ status = function->close(context.get(), FunctionContext::THREAD_LOCAL);
+ if (!status.ok()) {
+ state.SkipWithError(status.to_string().c_str());
+ }
+ status = function->close(context.get(), FunctionContext::FRAGMENT_LOCAL);
+ if (!status.ok()) {
+ state.SkipWithError(status.to_string().c_str());
+ }
+ state.SetItemsProcessed(state.iterations() * rows * length);
+}
+
+BENCHMARK(BM_ArrayRange)
+ ->Args({4096, 0, 1})
+ ->Args({4096, 1, 1})
+ ->Args({4096, 16, 1})
+ ->Args({4096, 256, 1})
+ ->Args({4096, 1024, 1})
+ ->Args({4096, 256, 7})
+ ->Args({1, 1000000, 1});
+
+} // namespace doris
diff --git a/be/benchmark/benchmark_main.cpp b/be/benchmark/benchmark_main.cpp
index 775f5caffef..ad5f2720d0b 100644
--- a/be/benchmark/benchmark_main.cpp
+++ b/be/benchmark/benchmark_main.cpp
@@ -22,6 +22,7 @@
#include <iostream>
#include <vector>
+#include "benchmark_array_range.hpp"
#include "benchmark_arrow_validation.hpp"
#include "benchmark_bit_pack.hpp"
#include "benchmark_column_array_view.hpp"
diff --git a/be/src/exprs/function/array/function_array_range.cpp
b/be/src/exprs/function/array/function_array_range.cpp
index 0346e58b5f9..24d4de938bd 100644
--- a/be/src/exprs/function/array/function_array_range.cpp
+++ b/be/src/exprs/function/array/function_array_range.cpp
@@ -189,21 +189,31 @@ private:
dest_offsets.push_back(dest_offsets.back());
continue;
} else {
- if (idx < end_row && step_row > 0 &&
- ((static_cast<__int128_t>(end_row) -
static_cast<__int128_t>(idx) - 1) /
- static_cast<__int128_t>(step_row) +
- 1) > max_array_size_as_field) {
+ const Int64 distance = static_cast<Int64>(end_row) - idx;
+ if (distance <= 0) {
+ dest_offsets.push_back(dest_offsets.back());
+ continue;
+ }
+ if (distance <= step_row) {
+ nested_column.push_back(idx);
+ dest_offsets.push_back(dest_offsets.back() + 1);
+ continue;
+ }
+ const size_t array_size = (distance - 1) / step_row + 1;
+ if (array_size > max_array_size_as_field) {
return Status::InvalidArgument("Array size exceeds the
limit {}",
max_array_size_as_field);
}
- size_t offset = dest_offsets.back();
- while (idx < end[row]) {
- nested_column.push_back(idx);
- dest_nested_null_map.push_back(0);
- offset++;
- idx = idx + step_row;
+ const size_t offset = dest_offsets.back();
+ const size_t new_offset = offset + array_size;
+ nested_column.resize(new_offset);
+ auto* data = nested_column.data() + offset;
+ // The increment after the last element can exceed
INT32_MAX.
+ Int64 value = idx;
+ for (size_t i = 0; i < array_size; ++i, value += step_row)
{
+ data[i] = static_cast<Int32>(value);
}
- dest_offsets.push_back(offset);
+ dest_offsets.push_back(new_offset);
}
} else {
const auto& idx_0 = reinterpret_cast<const
DateV2Value<DateTimeV2ValueType>&>(idx);
@@ -238,6 +248,10 @@ private:
}
}
}
+ if constexpr (std::is_same_v<SourceDataType, Int32>) {
+ // Integer ranges contain no null elements; initialize the map
once for the block.
+ dest_nested_null_map.resize_fill(nested_column.size());
+ }
return Status::OK();
}
};
diff --git a/be/test/exprs/function/function_array_range_test.cpp
b/be/test/exprs/function/function_array_range_test.cpp
new file mode 100644
index 00000000000..d6026f84005
--- /dev/null
+++ b/be/test/exprs/function/function_array_range_test.cpp
@@ -0,0 +1,197 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+
+#include <array>
+#include <limits>
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "core/block/block.h"
+#include "core/column/column_array.h"
+#include "core/column/column_const.h"
+#include "core/data_type/data_type_array.h"
+#include "core/data_type/data_type_nullable.h"
+#include "core/data_type/data_type_number.h"
+#include "core/field.h"
+#include "exprs/function/simple_function_factory.h"
+#include "testutil/function_utils.h"
+
+namespace doris {
+namespace {
+
+using RangeRow = std::array<std::optional<Int32>, 3>;
+
+void check_range_result(const ColumnPtr& result,
+ const std::vector<std::optional<std::vector<Int32>>>&
expected) {
+ ASSERT_EQ(result->size(), expected.size());
+ for (size_t row = 0; row < expected.size(); ++row) {
+ SCOPED_TRACE(row);
+ if (!expected[row].has_value()) {
+ EXPECT_TRUE(result->is_null_at(row));
+ continue;
+ }
+ ASSERT_FALSE(result->is_null_at(row));
+ Field field;
+ result->get(row, field);
+ const auto& array = field.get<TYPE_ARRAY>();
+ ASSERT_EQ(array.size(), expected[row]->size());
+ for (size_t i = 0; i < array.size(); ++i) {
+ ASSERT_FALSE(array[i].is_null());
+ EXPECT_EQ(array[i].get<TYPE_INT>(), (*expected[row])[i]);
+ }
+ }
+}
+
+void check_range(const std::string& name, const std::vector<RangeRow>& rows,
+ const std::vector<std::optional<std::vector<Int32>>>&
expected,
+ unsigned const_mask = 0, size_t argument_count = 3,
+ bool expect_size_error = false) {
+ auto int_type = make_nullable(std::make_shared<DataTypeInt32>());
+ auto result_type =
make_nullable(std::make_shared<DataTypeArray>(int_type));
+ Block block;
+ ColumnNumbers arguments;
+ std::vector<DataTypePtr> arg_types;
+ for (size_t arg = 0; arg < argument_count; ++arg) {
+ auto column = int_type->create_column();
+ for (const auto& row : rows) {
+ if (row[arg].has_value()) {
+ column->insert(Field::create_field<TYPE_INT>(*row[arg]));
+ } else {
+ column->insert_default();
+ }
+ }
+ if (const_mask & (1U << arg)) {
+ column = ColumnConst::create(column->clone_resized(1),
rows.size());
+ }
+ block.insert({std::move(column), int_type, "arg"});
+ arguments.push_back(static_cast<uint32_t>(arg));
+ arg_types.push_back(int_type);
+ }
+ auto function = SimpleFunctionFactory::instance().get_function(
+ name, block.get_columns_with_type_and_name(), result_type);
+ ASSERT_NE(function, nullptr);
+ FunctionUtils utils(result_type, arg_types, false);
+ auto* context = utils.get_fn_ctx();
+ ASSERT_TRUE(function->open(context, FunctionContext::FRAGMENT_LOCAL).ok());
+ ASSERT_TRUE(function->open(context, FunctionContext::THREAD_LOCAL).ok());
+ block.insert({nullptr, result_type, "result"});
+ auto status = function->execute(context, block, arguments,
+ static_cast<uint32_t>(argument_count),
rows.size());
+ ASSERT_TRUE(function->close(context, FunctionContext::THREAD_LOCAL).ok());
+ ASSERT_TRUE(function->close(context,
FunctionContext::FRAGMENT_LOCAL).ok());
+ if (expect_size_error) {
+ ASSERT_FALSE(status.ok());
+ EXPECT_NE(status.to_string().find("Array size exceeds the limit"),
std::string::npos);
+ return;
+ }
+ ASSERT_TRUE(status.ok()) << status.to_string();
+ auto result =
block.get_by_position(argument_count).column->convert_to_full_column_if_const();
+ check_range_result(result, expected);
+}
+
+TEST(FunctionArrayRangeTest, OverflowAndOffsets) {
+ constexpr Int32 max = std::numeric_limits<Int32>::max();
+ const std::vector<RangeRow> rows = {{max, max, 2},
+ {max - 1, max, 2},
+ {1, 5, 2},
+ {max - 3, max, 2},
+ {1, max, max},
+ {2, 1, 1},
+ {std::nullopt, 5, 2},
+ {1, std::nullopt, 2},
+ {1, 5, std::nullopt},
+ {-1, 5, 2},
+ {1, -1, 2},
+ {1, 5, 0},
+ {1, 5, -1},
+ {0, 3, 1}};
+ const std::vector<std::optional<std::vector<Int32>>> expected = {
+ std::vector<Int32> {},
+ std::vector<Int32> {max - 1},
+ std::vector<Int32> {1, 3},
+ std::vector<Int32> {max - 3, max - 1},
+ std::vector<Int32> {1},
+ std::vector<Int32> {},
+ std::nullopt,
+ std::nullopt,
+ std::nullopt,
+ std::nullopt,
+ std::nullopt,
+ std::nullopt,
+ std::nullopt,
+ std::vector<Int32> {0, 1, 2}};
+ for (const auto* name : {"array_range", "sequence"}) {
+ check_range(name, rows, expected);
+ // Repeated rows exercise every mixture of constant and vector
arguments.
+ for (unsigned mask = 0; mask < 8; ++mask) {
+ check_range(
+ name, {{max - 3, max, 2}, {max - 3, max, 2}},
+ {std::vector<Int32> {max - 3, max - 1}, std::vector<Int32>
{max - 3, max - 1}},
+ mask);
+ }
+ }
+}
+
+TEST(FunctionArrayRangeTest, DefaultStartAndStep) {
+ for (const auto* name : {"array_range", "sequence"}) {
+ check_range(name, {{3, 0, 0}, {0, 0, 0}},
+ {std::vector<Int32> {0, 1, 2}, std::vector<Int32> {}}, 0,
1);
+ check_range(name, {{1, 4, 0}, {4, 4, 0}},
+ {std::vector<Int32> {1, 2, 3}, std::vector<Int32> {}}, 0,
2);
+ }
+}
+
+TEST(FunctionArrayRangeTest, GrowingArrays) {
+ std::vector<RangeRow> rows;
+ std::vector<std::optional<std::vector<Int32>>> expected;
+ for (Int32 row = 0; row < 32; ++row) {
+ const Int32 length = row * row * 3;
+ const Int32 step = row % 7 + 1;
+ rows.push_back({row, row + length * step, step});
+ std::vector<Int32> values;
+ for (Int32 i = 0; i < length; ++i) {
+ values.push_back(row + i * step);
+ }
+ expected.emplace_back(std::move(values));
+ rows.push_back({std::nullopt, 1, 1});
+ expected.emplace_back(std::nullopt);
+ rows.push_back({1, 1, 1});
+ expected.emplace_back(std::vector<Int32> {});
+ }
+ for (const auto* name : {"array_range", "sequence"}) {
+ check_range(name, rows, expected);
+ }
+}
+
+TEST(FunctionArrayRangeTest, ArraySizeLimit) {
+ const auto limit = static_cast<Int32>(max_array_size_as_field);
+ std::vector<Int32> expected(limit);
+ for (Int32 i = 0; i < limit; ++i) {
+ expected[i] = i * 2;
+ }
+ for (const auto* name : {"array_range", "sequence"}) {
+ check_range(name, {{0, limit * 2, 2}}, {expected});
+ check_range(name, {{0, limit * 2 + 1, 2}}, {}, 0, 3, true);
+ }
+}
+
+} // namespace
+} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
