This is an automated email from the ASF dual-hosted git repository.
HappenLee pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 593f6172e9d [fix](function) Skip the rows with a NULL argument in
array_range (#68620)
593f6172e9d is described below
commit 593f6172e9d05eda68124ca7eaafcb1316c636a4
Author: Chenyang Sun <[email protected]>
AuthorDate: Tue Sep 29 19:11:16 2026 +0800
[fix](function) Skip the rows with a NULL argument in array_range (#68620)
### What problem does this PR solve?
Issue Number: None
Related PR: #41739
Problem Summary:
A NULL argument of array_range, or its alias sequence, gives NULL for its
row. #41739 removed the
NULL handling of array_range and let the default NULL handling of the
function framework do it.
The default NULL handling runs the function on every row, including the
value under a NULL, and
sets NULL on the result only afterwards. The value under a NULL start can
be far from the end, so
the range is larger than the array size limit and the whole query fails:
CREATE TABLE t (id INT, s DATETIME(6) NULL, e DATETIME(6), p INT)
DUPLICATE KEY(id)
DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES ('replication_num' = '1');
INSERT INTO t VALUES (2, '2024-01-01 00:00:00.123456', '2024-01-01
00:00:03.123456', 1),
(3, NULL, '2024-01-02 00:00:00', 1);
SELECT id, sequence(s, e, INTERVAL p SECOND) FROM t ORDER BY id;
-- before: [INVALID_ARGUMENT]Array size exceeds the limit 1000000
-- after: 2 ["2024-01-01 00:00:00.123456", "2024-01-01
00:00:01.123456",
-- "2024-01-01 00:00:02.123456"]
-- 3 NULL
Each row alone gives the right result, because the default NULL handling
does not run the
function when an argument has only NULL. The integer version fails in the
same way, for example
array_range(s, e) with the rows (1, 3) and (NULL, 5000000).
As before #41739, turn off the default NULL handling and merge the null
maps of the arguments
into the null map that range_execute already checks, so a row with a NULL
argument is skipped
instead of built. The loops that build the ranges are not changed.
### Release note
Fix array_range and sequence failing with "Array size exceeds the limit"
when a row with a NULL
start is in the same block as other rows.
### Check List (For Author)
- Test: Regression test / Unit Test / Manual test
- Regression test: test_array_range_null_argument (new),
test_array_functions,
test_array_sort_lambda_comparator, test_timestamp_ns_functions,
test_timestamp_ns_function_argument_matrix
- BE unit test: FunctionArrayRangeTest.NullStartSkipsTheRange (new,
fails before the fix)
- Manual test: counted the instructions of array_range per block with
perf in a release
build. Inputs without NULL show no change beyond the noise, and the
rows with a NULL
argument are no longer built.
- Behavior changed: No
- Does this need documentation: No
Co-Authored-By: Claude Opus 5.5 <[email protected]>
---
.../exprs/function/array/function_array_range.cpp | 12 +++++
.../exprs/function/function_array_range_test.cpp | 12 +++++
.../test_array_range_null_argument.out | 15 +++++++
.../test_array_range_null_argument.groovy | 51 ++++++++++++++++++++++
4 files changed, 90 insertions(+)
diff --git a/be/src/exprs/function/array/function_array_range.cpp
b/be/src/exprs/function/array/function_array_range.cpp
index ab8bea14302..5380b4ffd03 100644
--- a/be/src/exprs/function/array/function_array_range.cpp
+++ b/be/src/exprs/function/array/function_array_range.cpp
@@ -41,6 +41,7 @@
#include "core/pod_array_fwd.h"
#include "core/types.h"
#include "core/value/vdatetime_value.h"
+#include "exec/common/util.hpp"
#include "exprs/aggregate/aggregate_function.h"
#include "exprs/function/function.h"
#include "exprs/function/function_date_or_datetime_computation.h"
@@ -78,6 +79,10 @@ public:
return make_nullable(res);
}
+ // range_execute skips the rows with a NULL argument. The default NULL
handling would build a
+ // range from the value under the NULL, which can be larger than the array
size limit.
+ bool use_default_implementation_for_nulls() const override { return false;
}
+
Status execute_impl(FunctionContext* context, Block& block, const
ColumnNumbers& arguments,
uint32_t result, size_t input_rows_count) const
override {
return Impl::execute_impl(context, block, arguments, result,
input_rows_count);
@@ -147,6 +152,13 @@ struct RangeImplUtil {
for (int i = 0; i < 3; ++i) {
argument_columns[i] =
block.get_by_position(arguments[i]).column->convert_to_full_column_if_const();
+ if (const auto* nullable =
check_and_get_column<ColumnNullable>(*argument_columns[i])) {
+ // Read the null map before replacing the column, because the
replacement can free
+ // the nullable column.
+ VectorizedUtils::update_null_map(args_null_map->get_data(),
+
nullable->get_null_map_data());
+ argument_columns[i] = nullable->get_nested_column_ptr();
+ }
}
auto start_column =
assert_cast<const
ColumnVector<SourceDataPType>*>(argument_columns[0].get());
diff --git a/be/test/exprs/function/function_array_range_test.cpp
b/be/test/exprs/function/function_array_range_test.cpp
index d6026f84005..c91d17918ca 100644
--- a/be/test/exprs/function/function_array_range_test.cpp
+++ b/be/test/exprs/function/function_array_range_test.cpp
@@ -193,5 +193,17 @@ TEST(FunctionArrayRangeTest, ArraySizeLimit) {
}
}
+TEST(FunctionArrayRangeTest, NullStartSkipsTheRange) {
+ // The value under a NULL start is 0. A range from 0 to this end is larger
than the limit, so
+ // the row must give NULL without building the range, even next to rows
that are not NULL.
+ const auto large_end = static_cast<Int32>(max_array_size_as_field) * 4;
+ for (const auto* name : {"array_range", "sequence"}) {
+ check_range(name, {{1, 3, 1}, {std::nullopt, large_end, 1}, {2, 6, 2}},
+ {std::vector<Int32> {1, 2}, std::nullopt,
std::vector<Int32> {2, 4}});
+ check_range(name, {{1, 3, 0}, {std::nullopt, large_end, 0}},
+ {std::vector<Int32> {1, 2}, std::nullopt}, 0, 2);
+ }
+}
+
} // namespace
} // namespace doris
diff --git
a/regression-test/data/query_p0/sql_functions/array_functions/test_array_range_null_argument.out
b/regression-test/data/query_p0/sql_functions/array_functions/test_array_range_null_argument.out
new file mode 100644
index 00000000000..89f0569b5f3
--- /dev/null
+++
b/regression-test/data/query_p0/sql_functions/array_functions/test_array_range_null_argument.out
@@ -0,0 +1,15 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !int_range --
+1 [1, 2, 3] [1, 2, 3] [1, 2, 3]
+2 \N \N \N
+3 \N \N \N
+4 [2, 3, 4, 5, 6, 7] \N \N
+5 [0, 1, 2] [0, 2] [0, 2]
+
+-- !datetime_range --
+1 ["2024-01-01 00:00:00.123456", "2024-01-01 00:00:01.123456",
"2024-01-01 00:00:02.123456"]
+2 \N
+3 \N
+4 ["2024-01-01 00:00:00.000000", "2024-01-01 00:00:01.000000"]
+5 \N
+
diff --git
a/regression-test/suites/query_p0/sql_functions/array_functions/test_array_range_null_argument.groovy
b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_range_null_argument.groovy
new file mode 100644
index 00000000000..6336e214313
--- /dev/null
+++
b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_range_null_argument.groovy
@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_array_range_null_argument") {
+ sql "DROP TABLE IF EXISTS test_array_range_null_argument"
+ sql """
+ CREATE TABLE test_array_range_null_argument (
+ id INT,
+ s INT NULL,
+ e INT NULL,
+ st INT NULL,
+ ds DATETIME(6) NULL,
+ de DATETIME(6) NULL
+ ) DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_num" = "1")
+ """
+ // The rows are read in one block. A row with a NULL argument gives NULL,
and must not build a
+ // range from the value under the NULL, which can be larger than the array
size limit.
+ sql """
+ INSERT INTO test_array_range_null_argument VALUES
+ (1, 1, 4, 1, '2024-01-01 00:00:00.123456', '2024-01-01
00:00:03.123456'),
+ (2, NULL, 5000000, 1, NULL, '2024-01-02 00:00:00'),
+ (3, 2, NULL, 1, '2024-01-01 00:00:00', NULL),
+ (4, 2, 8, NULL, '2024-01-01 00:00:00', '2024-01-01 00:00:02'),
+ (5, 0, 3, 2, NULL, NULL)
+ """
+
+ order_qt_int_range """
+ SELECT id, array_range(s, e), array_range(s, e, st), sequence(s, e, st)
+ FROM test_array_range_null_argument
+ """
+ order_qt_datetime_range """
+ SELECT id, sequence(ds, de, INTERVAL 1 SECOND)
+ FROM test_array_range_null_argument
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
