This is an automated email from the ASF dual-hosted git repository.
Mryange pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new d2cfcc1dcd4 [fix](function) Preserve NULL semantics in IP functions
(#68407)
d2cfcc1dcd4 is described below
commit d2cfcc1dcd4019fbae0d6ffbce16a7563bd9ecf4
Author: Mryange <[email protected]>
AuthorDate: Tue Sep 29 15:24:49 2026 +0800
[fix](function) Preserve NULL semantics in IP functions (#68407)
Nullable inputs to `ipv4_cidr_to_range`, `ipv6_cidr_to_range`, and
`cut_ipv6` can carry arbitrary physical payloads in NULL rows. The
functions previously read or validated those payloads, which could
produce errors instead of SQL NULL results. Handle nullable and constant
inputs with `ColumnView`, skip NULL rows before reading their values,
and propagate the combined null map to the result.
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
be/src/exprs/function/function_ip.h | 370 ++++++++++++++++------------
be/test/exprs/function/function_ip_test.cpp | 159 ++++++++++++
2 files changed, 374 insertions(+), 155 deletions(-)
diff --git a/be/src/exprs/function/function_ip.h
b/be/src/exprs/function/function_ip.h
index 359c84037b6..5e9de6be538 100644
--- a/be/src/exprs/function/function_ip.h
+++ b/be/src/exprs/function/function_ip.h
@@ -29,6 +29,7 @@
#include "core/block/column_with_type_and_name.h"
#include "core/column/column.h"
#include "core/column/column_const.h"
+#include "core/column/column_execute_util.h"
#include "core/column/column_nullable.h"
#include "core/column/column_string.h"
#include "core/column/column_struct.h"
@@ -792,73 +793,67 @@ public:
DataTypePtr get_return_type_impl(const DataTypes& arguments) const
override {
DataTypePtr element = std::make_shared<DataTypeIPv4>();
- return std::make_shared<DataTypeStruct>(DataTypes {element, element},
- Strings {"min", "max"});
+ DataTypePtr result = std::make_shared<DataTypeStruct>(DataTypes
{element, element},
+ Strings {"min",
"max"});
+ if (arguments[0]->is_nullable() || arguments[1]->is_nullable()) {
+ return make_nullable(result);
+ }
+ return result;
}
+ bool use_default_implementation_for_nulls() const override { return false;
}
+
Status execute_impl(FunctionContext* context, Block& block, const
ColumnNumbers& arguments,
uint32_t result, size_t input_rows_count) const
override {
- ColumnWithTypeAndName& ip_column = block.get_by_position(arguments[0]);
- ColumnWithTypeAndName& cidr_column =
block.get_by_position(arguments[1]);
-
- const auto& [ip_column_ptr, ip_col_const] =
unpack_if_const(ip_column.column);
- const auto& [cidr_column_ptr, cidr_col_const] =
unpack_if_const(cidr_column.column);
+ const auto& ip_argument = block.get_by_position(arguments[0]);
+ const auto& cidr_argument = block.get_by_position(arguments[1]);
+ const auto ip = ColumnView<TYPE_IPV4>::create(ip_argument.column);
+ const auto cidr =
ColumnView<TYPE_SMALLINT>::create(cidr_argument.column);
+ const bool result_nullable =
block.get_by_position(result).type->is_nullable();
- const auto* col_ip_column = assert_cast<const
ColumnIPv4*>(ip_column_ptr.get());
- const auto* col_cidr_column = assert_cast<const
ColumnInt16*>(cidr_column_ptr.get());
-
- const typename ColumnIPv4::Container& vec_ip_input =
col_ip_column->get_data();
- const ColumnInt16::Container& vec_cidr_input =
col_cidr_column->get_data();
auto col_lower_range_output = ColumnIPv4::create(input_rows_count, 0);
auto col_upper_range_output = ColumnIPv4::create(input_rows_count, 0);
+ auto& lower = col_lower_range_output->get_data();
+ auto& upper = col_upper_range_output->get_data();
+ ColumnUInt8::MutablePtr null_map;
+ if (result_nullable) {
+ null_map = ColumnUInt8::create(input_rows_count, 0);
+ execute_impl<true>(ip, cidr, input_rows_count, lower, upper,
&null_map->get_data());
+ } else {
+ execute_impl<false>(ip, cidr, input_rows_count, lower, upper,
nullptr);
+ }
- ColumnIPv4::Container& vec_lower_range_output =
col_lower_range_output->get_data();
- ColumnIPv4::Container& vec_upper_range_output =
col_upper_range_output->get_data();
+ ColumnPtr result_column = ColumnStruct::create(
+ Columns {std::move(col_lower_range_output),
std::move(col_upper_range_output)});
+ if (result_nullable) {
+ result_column = ColumnNullable::create(std::move(result_column),
std::move(null_map));
+ }
+ block.replace_by_position(result, std::move(result_column));
+ return Status::OK();
+ }
+private:
+ template <bool ResultNullable>
+ static void execute_impl(const ColumnView<TYPE_IPV4>& ip, const
ColumnView<TYPE_SMALLINT>& cidr,
+ size_t input_rows_count, ColumnIPv4::Container&
lower,
+ ColumnIPv4::Container& upper,
ColumnUInt8::Container* nulls) {
static constexpr UInt8 max_cidr_mask = IPV4_BINARY_LENGTH * 8;
-
- if (ip_col_const) {
- auto ip = vec_ip_input[0];
- for (size_t i = 0; i < input_rows_count; ++i) {
- auto cidr = vec_cidr_input[i];
- if (cidr < 0 || cidr > max_cidr_mask) {
- throw Exception(ErrorCode::INVALID_ARGUMENT, "Illegal cidr
value '{}'",
- std::to_string(cidr));
+ for (size_t i = 0; i < input_rows_count; ++i) {
+ if constexpr (ResultNullable) {
+ if (ip.is_null_at(i) || cidr.is_null_at(i)) {
+ (*nulls)[i] = 1;
+ continue;
}
- auto range = apply_cidr_mask(ip, cast_set<UInt8>(cidr));
- vec_lower_range_output[i] = range.first;
- vec_upper_range_output[i] = range.second;
}
- } else if (cidr_col_const) {
- auto cidr = vec_cidr_input[0];
- if (cidr < 0 || cidr > max_cidr_mask) {
+ const auto prefix = cidr.value_at(i);
+ if (prefix < 0 || prefix > max_cidr_mask) {
throw Exception(ErrorCode::INVALID_ARGUMENT, "Illegal cidr
value '{}'",
- std::to_string(cidr));
- }
- for (size_t i = 0; i < input_rows_count; ++i) {
- auto ip = vec_ip_input[i];
- auto range = apply_cidr_mask(ip, cast_set<UInt8>(cidr));
- vec_lower_range_output[i] = range.first;
- vec_upper_range_output[i] = range.second;
- }
- } else {
- for (size_t i = 0; i < input_rows_count; ++i) {
- auto ip = vec_ip_input[i];
- auto cidr = vec_cidr_input[i];
- if (cidr < 0 || cidr > max_cidr_mask) {
- throw Exception(ErrorCode::INVALID_ARGUMENT, "Illegal cidr
value '{}'",
- std::to_string(cidr));
- }
- auto range = apply_cidr_mask(ip, cast_set<UInt8>(cidr));
- vec_lower_range_output[i] = range.first;
- vec_upper_range_output[i] = range.second;
+ std::to_string(prefix));
}
+ const auto range = apply_cidr_mask(ip.value_at(i),
cast_set<UInt8>(prefix));
+ lower[i] = range.first;
+ upper[i] = range.second;
}
-
- block.replace_by_position(
- result, ColumnStruct::create(Columns
{std::move(col_lower_range_output),
-
std::move(col_upper_range_output)}));
- return Status::OK();
}
};
@@ -878,91 +873,132 @@ public:
DataTypePtr get_return_type_impl(const DataTypes& arguments) const
override {
DataTypePtr element = std::make_shared<DataTypeIPv6>();
- return std::make_shared<DataTypeStruct>(DataTypes {element, element},
- Strings {"min", "max"});
+ DataTypePtr result = std::make_shared<DataTypeStruct>(DataTypes
{element, element},
+ Strings {"min",
"max"});
+ if (arguments[0]->is_nullable() || arguments[1]->is_nullable()) {
+ return make_nullable(result);
+ }
+ return result;
}
+ bool use_default_implementation_for_nulls() const override { return false;
}
+
Status execute_impl(FunctionContext* context, Block& block, const
ColumnNumbers& arguments,
uint32_t result, size_t input_rows_count) const
override {
const auto& addr_column_with_type_and_name =
block.get_by_position(arguments[0]);
const auto& cidr_column_with_type_and_name =
block.get_by_position(arguments[1]);
- const auto& [addr_column, add_col_const] =
- unpack_if_const(addr_column_with_type_and_name.column);
- const auto& [cidr_column, col_const] =
- unpack_if_const(cidr_column_with_type_and_name.column);
+ const auto cidr =
ColumnView<TYPE_SMALLINT>::create(cidr_column_with_type_and_name.column);
+ const auto addr_type =
addr_column_with_type_and_name.type->get_primitive_type();
+ const bool result_nullable =
block.get_by_position(result).type->is_nullable();
- const auto* cidr_col = assert_cast<const
ColumnInt16*>(cidr_column.get());
- ColumnPtr col_res = nullptr;
-
- if (addr_column_with_type_and_name.type->get_primitive_type() ==
TYPE_IPV6) {
- const auto* ipv6_addr_column = assert_cast<const
ColumnIPv6*>(addr_column.get());
- col_res = execute_impl(*ipv6_addr_column, *cidr_col,
input_rows_count, add_col_const,
- col_const);
- } else if
(is_string_type(addr_column_with_type_and_name.type->get_primitive_type())) {
- ColumnPtr col_ipv6 =
-
convert_to_ipv6<IPConvertExceptionMode::Throw>(addr_column, nullptr);
- const auto* ipv6_addr_column = assert_cast<const
ColumnIPv6*>(col_ipv6.get());
- col_res = execute_impl(*ipv6_addr_column, *cidr_col,
input_rows_count, add_col_const,
- col_const);
+ auto col_res_lower_range = ColumnIPv6::create(input_rows_count, 0);
+ auto col_res_upper_range = ColumnIPv6::create(input_rows_count, 0);
+ auto& vec_res_lower_range = col_res_lower_range->get_data();
+ auto& vec_res_upper_range = col_res_upper_range->get_data();
+ ColumnUInt8::MutablePtr null_map;
+ if (result_nullable) {
+ null_map = ColumnUInt8::create(input_rows_count, 0);
+ }
+ if (addr_type == TYPE_IPV6) {
+ const auto addr =
ColumnView<TYPE_IPV6>::create(addr_column_with_type_and_name.column);
+ if (result_nullable) {
+ execute_impl<true>(addr, cidr, input_rows_count,
vec_res_lower_range,
+ vec_res_upper_range, &null_map->get_data());
+ } else {
+ execute_impl<false>(addr, cidr, input_rows_count,
vec_res_lower_range,
+ vec_res_upper_range, nullptr);
+ }
+ } else if (is_string_type(addr_type)) {
+ const auto addr =
+
ColumnView<TYPE_STRING>::create(addr_column_with_type_and_name.column);
+ if (result_nullable) {
+ execute_impl<true>(addr, cidr, input_rows_count,
vec_res_lower_range,
+ vec_res_upper_range, &null_map->get_data());
+ } else {
+ execute_impl<false>(addr, cidr, input_rows_count,
vec_res_lower_range,
+ vec_res_upper_range, nullptr);
+ }
} else {
return Status::RuntimeError(
"Illegal column {} of argument of function {}, Expected
IPv6 or String",
- addr_column->get_name(), get_name());
+ addr_column_with_type_and_name.column->get_name(),
get_name());
}
- block.replace_by_position(result, std::move(col_res));
+ ColumnPtr result_column = ColumnStruct::create(
+ Columns {std::move(col_res_lower_range),
std::move(col_res_upper_range)});
+ if (result_nullable) {
+ result_column = ColumnNullable::create(std::move(result_column),
std::move(null_map));
+ }
+ block.replace_by_position(result, std::move(result_column));
return Status::OK();
}
- static ColumnPtr execute_impl(const ColumnIPv6& from_column, const
ColumnInt16& cidr_column,
- size_t input_rows_count, bool is_addr_const
= false,
- bool is_cidr_const = false) {
- auto col_res_lower_range = ColumnIPv6::create(input_rows_count, 0);
- auto col_res_upper_range = ColumnIPv6::create(input_rows_count, 0);
- auto& vec_res_lower_range = col_res_lower_range->get_data();
- auto& vec_res_upper_range = col_res_upper_range->get_data();
-
+private:
+ template <bool ResultNullable>
+ static void execute_impl(const ColumnView<TYPE_IPV6>& addr,
+ const ColumnView<TYPE_SMALLINT>& cidr, size_t
input_rows_count,
+ ColumnIPv6::Container& lower,
ColumnIPv6::Container& upper,
+ ColumnUInt8::Container* nulls) {
static constexpr UInt8 max_cidr_mask = IPV6_BINARY_LENGTH * 8;
+ for (size_t i = 0; i < input_rows_count; ++i) {
+ if constexpr (ResultNullable) {
+ if (addr.is_null_at(i) || cidr.is_null_at(i)) {
+ (*nulls)[i] = 1;
+ continue;
+ }
+ }
+ const auto prefix = cidr.value_at(i);
+ if (prefix < 0 || prefix > max_cidr_mask) {
+ throw Exception(ErrorCode::INVALID_ARGUMENT, "Illegal cidr
value '{}'",
+ std::to_string(prefix));
+ }
+ const auto address = addr.value_at(i);
+ apply_cidr_mask(reinterpret_cast<const char*>(&address),
+ reinterpret_cast<char*>(&lower[i]),
reinterpret_cast<char*>(&upper[i]),
+ cast_set<UInt8>(prefix));
+ }
+ }
- if (is_addr_const) {
- for (size_t i = 0; i < input_rows_count; ++i) {
- auto cidr = cidr_column.get_int(i);
- if (cidr < 0 || cidr > max_cidr_mask) {
- throw Exception(ErrorCode::INVALID_ARGUMENT, "Illegal cidr
value '{}'",
- std::to_string(cidr));
+ template <bool ResultNullable>
+ static void execute_impl(const ColumnView<TYPE_STRING>& addr,
+ const ColumnView<TYPE_SMALLINT>& cidr, size_t
input_rows_count,
+ ColumnIPv6::Container& lower,
ColumnIPv6::Container& upper,
+ ColumnUInt8::Container* nulls) {
+ static constexpr UInt8 max_cidr_mask = IPV6_BINARY_LENGTH * 8;
+ for (size_t i = 0; i < input_rows_count; ++i) {
+ if constexpr (ResultNullable) {
+ if (addr.is_null_at(i) || cidr.is_null_at(i)) {
+ (*nulls)[i] = 1;
+ continue;
}
- apply_cidr_mask(from_column.get_data_at(0).data,
-
reinterpret_cast<char*>(&vec_res_lower_range[i]),
-
reinterpret_cast<char*>(&vec_res_upper_range[i]),
- cast_set<UInt8>(cidr));
}
- } else if (is_cidr_const) {
- auto cidr = cidr_column.get_int(0);
- if (cidr < 0 || cidr > max_cidr_mask) {
+ const auto prefix = cidr.value_at(i);
+ if (prefix < 0 || prefix > max_cidr_mask) {
throw Exception(ErrorCode::INVALID_ARGUMENT, "Illegal cidr
value '{}'",
- std::to_string(cidr));
+ std::to_string(prefix));
}
- for (size_t i = 0; i < input_rows_count; ++i) {
- apply_cidr_mask(from_column.get_data_at(i).data,
-
reinterpret_cast<char*>(&vec_res_lower_range[i]),
-
reinterpret_cast<char*>(&vec_res_upper_range[i]),
- cast_set<UInt8>(cidr));
+ const auto value = addr.value_at(i);
+ if (value.size == 0) {
+ throw Exception(ErrorCode::INVALID_ARGUMENT, "Invalid IPv6
value");
}
- } else {
- for (size_t i = 0; i < input_rows_count; ++i) {
- auto cidr = cidr_column.get_int(i);
- if (cidr < 0 || cidr > max_cidr_mask) {
- throw Exception(ErrorCode::INVALID_ARGUMENT, "Illegal cidr
value '{}'",
- std::to_string(cidr));
- }
- apply_cidr_mask(from_column.get_data_at(i).data,
-
reinterpret_cast<char*>(&vec_res_lower_range[i]),
-
reinterpret_cast<char*>(&vec_res_upper_range[i]),
- cast_set<UInt8>(cidr));
+ IPv6 address = 0;
+ Int64 parsed_ipv4 = 0;
+ bool parsed = false;
+ if (try_parse_ipv4(value.begin(), value.end(), parsed_ipv4)) {
+ map_ipv4_to_ipv6(static_cast<IPv4>(parsed_ipv4),
+ reinterpret_cast<UInt8*>(&address));
+ parsed = true;
+ } else {
+ parsed = parse_ipv6_whole(value.begin(), value.end(),
+ reinterpret_cast<UInt8*>(&address));
+ }
+ if (!parsed) {
+ throw Exception(ErrorCode::INVALID_ARGUMENT, "Invalid IPv6
value");
}
+ apply_cidr_mask(reinterpret_cast<const char*>(&address),
+ reinterpret_cast<char*>(&lower[i]),
reinterpret_cast<char*>(&upper[i]),
+ cast_set<UInt8>(prefix));
}
- return ColumnStruct::create(
- Columns {std::move(col_res_lower_range),
std::move(col_res_upper_range)});
}
};
@@ -1219,9 +1255,16 @@ public:
size_t get_number_of_arguments() const override { return 3; }
DataTypePtr get_return_type_impl(const DataTypes& arguments) const
override {
- return std::make_shared<DataTypeString>();
+ DataTypePtr result = std::make_shared<DataTypeString>();
+ if (arguments[0]->is_nullable() || arguments[1]->is_nullable() ||
+ arguments[2]->is_nullable()) {
+ return make_nullable(result);
+ }
+ return result;
}
+ bool use_default_implementation_for_nulls() const override { return false;
}
+
Status execute_impl(FunctionContext* context, Block& block, const
ColumnNumbers& arguments,
uint32_t result, size_t input_rows_count) const
override {
const auto& ipv6_column_with_type_and_name =
block.get_by_position(arguments[0]);
@@ -1230,73 +1273,90 @@ public:
const auto& bytes_to_cut_for_ipv4_column_with_type_and_name =
block.get_by_position(arguments[2]);
- const auto& [ipv6_column, ipv6_const] =
- unpack_if_const(ipv6_column_with_type_and_name.column);
- const auto& [bytes_to_cut_for_ipv6_column,
bytes_to_cut_for_ipv6_const] =
-
unpack_if_const(bytes_to_cut_for_ipv6_column_with_type_and_name.column);
- const auto& [bytes_to_cut_for_ipv4_column,
bytes_to_cut_for_ipv4_const] =
-
unpack_if_const(bytes_to_cut_for_ipv4_column_with_type_and_name.column);
-
- const auto* ipv6_addr_column = assert_cast<const
ColumnIPv6*>(ipv6_column.get());
- const auto* to_cut_for_ipv6_bytes_column =
- assert_cast<const
ColumnInt8*>(bytes_to_cut_for_ipv6_column.get());
- const auto* to_cut_for_ipv4_bytes_column =
- assert_cast<const
ColumnInt8*>(bytes_to_cut_for_ipv4_column.get());
-
- const auto& ipv6_addr_column_data = ipv6_addr_column->get_data();
- const auto& to_cut_for_ipv6_bytes_column_data =
to_cut_for_ipv6_bytes_column->get_data();
- const auto& to_cut_for_ipv4_bytes_column_data =
to_cut_for_ipv4_bytes_column->get_data();
+ const auto ipv6 =
ColumnView<TYPE_IPV6>::create(ipv6_column_with_type_and_name.column);
+ const auto bytes_to_cut_for_ipv6 = ColumnView<TYPE_TINYINT>::create(
+ bytes_to_cut_for_ipv6_column_with_type_and_name.column);
+ const auto bytes_to_cut_for_ipv4 = ColumnView<TYPE_TINYINT>::create(
+ bytes_to_cut_for_ipv4_column_with_type_and_name.column);
auto col_res = ColumnString::create();
ColumnString::Chars& chars_res = col_res->get_chars();
ColumnString::Offsets& offsets_res = col_res->get_offsets();
chars_res.resize(input_rows_count * (IPV6_MAX_TEXT_LENGTH + 1)); // +
1 for ending '\0'
offsets_res.resize(input_rows_count);
+ const bool result_nullable =
block.get_by_position(result).type->is_nullable();
+ ColumnUInt8::MutablePtr null_map;
+ if (result_nullable) {
+ null_map = ColumnUInt8::create(input_rows_count, 0);
+ }
auto* begin = reinterpret_cast<char*>(chars_res.data());
auto* pos = begin;
+ if (result_nullable) {
+ execute_impl<true>(
+ ipv6, bytes_to_cut_for_ipv6, bytes_to_cut_for_ipv4,
input_rows_count,
+ offsets_res, begin, pos,
bytes_to_cut_for_ipv6_column_with_type_and_name.type,
+ bytes_to_cut_for_ipv4_column_with_type_and_name.type,
&null_map->get_data());
+ } else {
+ execute_impl<false>(ipv6, bytes_to_cut_for_ipv6,
bytes_to_cut_for_ipv4,
+ input_rows_count, offsets_res, begin, pos,
+
bytes_to_cut_for_ipv6_column_with_type_and_name.type,
+
bytes_to_cut_for_ipv4_column_with_type_and_name.type, nullptr);
+ }
+
+ chars_res.resize(offsets_res[offsets_res.size() - 1]);
+
+ if (result_nullable) {
+ block.replace_by_position(
+ result, ColumnNullable::create(std::move(col_res),
std::move(null_map)));
+ } else {
+ block.replace_by_position(result, std::move(col_res));
+ }
+ return Status::OK();
+ }
+
+private:
+ template <bool ResultNullable>
+ void execute_impl(const ColumnView<TYPE_IPV6>& ipv6,
+ const ColumnView<TYPE_TINYINT>& bytes_to_cut_for_ipv6,
+ const ColumnView<TYPE_TINYINT>& bytes_to_cut_for_ipv4,
+ size_t input_rows_count, ColumnString::Offsets&
offsets_res, char* begin,
+ char*& pos, const DataTypePtr&
bytes_to_cut_for_ipv6_type,
+ const DataTypePtr& bytes_to_cut_for_ipv4_type,
+ ColumnUInt8::Container* nulls) const {
for (size_t i = 0; i < input_rows_count; ++i) {
- auto ipv6_idx = index_check_const(i, ipv6_const);
- auto bytes_to_cut_for_ipv6_idx = index_check_const(i,
bytes_to_cut_for_ipv6_const);
- auto bytes_to_cut_for_ipv4_idx = index_check_const(i,
bytes_to_cut_for_ipv4_const);
- // the current function logic is processed in big endian manner
- // But ipv6 in doris is stored in little-endian byte order
- // need transfer to big-endian byte order first, so we can't deal
this process in column
- auto val_128 = ipv6_addr_column_data[ipv6_idx];
+ if constexpr (ResultNullable) {
+ if (ipv6.is_null_at(i) || bytes_to_cut_for_ipv6.is_null_at(i)
||
+ bytes_to_cut_for_ipv4.is_null_at(i)) {
+ (*nulls)[i] = 1;
+ offsets_res[i] = cast_set<uint32_t>(pos - begin);
+ continue;
+ }
+ }
+ // Convert from Doris little-endian storage before formatting in
big-endian order.
+ auto val_128 = ipv6.value_at(i);
auto* address = reinterpret_cast<unsigned char*>(&val_128);
-
- Int8 bytes_to_cut_for_ipv6_count =
-
to_cut_for_ipv6_bytes_column_data[bytes_to_cut_for_ipv6_idx];
- Int8 bytes_to_cut_for_ipv4_count =
-
to_cut_for_ipv4_bytes_column_data[bytes_to_cut_for_ipv4_idx];
+ const Int8 bytes_to_cut_for_ipv6_count =
bytes_to_cut_for_ipv6.value_at(i);
+ const Int8 bytes_to_cut_for_ipv4_count =
bytes_to_cut_for_ipv4.value_at(i);
if (bytes_to_cut_for_ipv6_count > IPV6_BINARY_LENGTH) [[unlikely]]
{
throw Exception(ErrorCode::INVALID_ARGUMENT,
"Illegal value for argument 2 {} of function
{}",
-
bytes_to_cut_for_ipv6_column_with_type_and_name.type->get_name(),
- get_name());
+ bytes_to_cut_for_ipv6_type->get_name(),
get_name());
}
-
if (bytes_to_cut_for_ipv4_count > IPV6_BINARY_LENGTH) [[unlikely]]
{
throw Exception(ErrorCode::INVALID_ARGUMENT,
"Illegal value for argument 3 {} of function
{}",
-
bytes_to_cut_for_ipv4_column_with_type_and_name.type->get_name(),
- get_name());
+ bytes_to_cut_for_ipv4_type->get_name(),
get_name());
}
- UInt8 bytes_to_cut_count = is_ipv4_mapped(address) ?
bytes_to_cut_for_ipv4_count
- :
bytes_to_cut_for_ipv6_count;
+ const UInt8 bytes_to_cut_count = is_ipv4_mapped(address) ?
bytes_to_cut_for_ipv4_count
+ :
bytes_to_cut_for_ipv6_count;
cut_address(address, pos, bytes_to_cut_count);
offsets_res[i] = cast_set<uint32_t>(pos - begin);
}
-
- chars_res.resize(offsets_res[offsets_res.size() - 1]);
-
- block.replace_by_position(result, std::move(col_res));
- return Status::OK();
}
-private:
static bool is_ipv4_mapped(const UInt8* address) {
return (LittleEndian::Load64(address + 8) == 0) &&
((LittleEndian::Load64(address) & 0xFFFFFFFF00000000ULL) ==
0x0000FFFF00000000ULL);
diff --git a/be/test/exprs/function/function_ip_test.cpp
b/be/test/exprs/function/function_ip_test.cpp
index 29da803ffec..db68979c9b5 100644
--- a/be/test/exprs/function/function_ip_test.cpp
+++ b/be/test/exprs/function/function_ip_test.cpp
@@ -17,6 +17,8 @@
#include "exprs/function/function_ip.h"
+#include <initializer_list>
+
#include "core/column/column_const.h"
#include "core/data_type/data_type_ipv6.h"
#include "core/data_type/data_type_number.h"
@@ -28,6 +30,12 @@
namespace doris {
+static ColumnUInt8::MutablePtr make_null_map(std::initializer_list<UInt8>
values) {
+ auto null_map = ColumnUInt8::create();
+ null_map->get_data() = values;
+ return null_map;
+}
+
TEST(FunctionIpTest, IPAddressVariantTypeTest) {
IPAddressVariant ipv4_zero("0.0.0.0");
EXPECT_TRUE(ipv4_zero.is_v4());
@@ -294,6 +302,157 @@ TEST(FunctionIpTest, FunctionCutIPv6Test) {
check_function<DataTypeString, true>(func_name, input_types,
odd_bytes_data_set));
}
+TEST(FunctionIpTest, NullablePayloadsAreIgnoredByRangeAndCutFunctions) {
+ const auto ipv4_type = std::make_shared<DataTypeIPv4>();
+ const auto ipv6_type = std::make_shared<DataTypeIPv6>();
+ const auto int16_type = std::make_shared<DataTypeInt16>();
+ const auto int8_type = std::make_shared<DataTypeInt8>();
+ const auto string_type = std::make_shared<DataTypeString>();
+
+ {
+ auto ip = ColumnIPv4::create();
+ ip->get_data() = {0x7f000001, 0xffffffff, 0x7f000001};
+ auto cidr = ColumnInt16::create();
+ cidr->get_data() = {24, -16706, 24};
+ auto nullable_ip = ColumnNullable::create(std::move(ip),
make_null_map({0, 0, 1}));
+ auto nullable_cidr = ColumnNullable::create(std::move(cidr),
make_null_map({0, 1, 0}));
+ auto nullable_ipv4_type = make_nullable(ipv4_type);
+ auto nullable_int16_type = make_nullable(int16_type);
+ FunctionIPv4CIDRToRange function;
+ DataTypes argument_types {nullable_ipv4_type, nullable_int16_type};
+ auto result_type = function.get_return_type_impl(argument_types);
+ Block block;
+ block.insert({std::move(nullable_ip), nullable_ipv4_type, "ip"});
+ block.insert({std::move(nullable_cidr), nullable_int16_type, "cidr"});
+ block.insert({nullptr, result_type, "result"});
+
+ ASSERT_TRUE(function.execute_impl(nullptr, block, {0, 1}, 2, 3).ok());
+ const auto& result = assert_cast<const
ColumnNullable&>(*block.get_by_position(2).column);
+ EXPECT_FALSE(result.is_null_at(0));
+ EXPECT_TRUE(result.is_null_at(1));
+ EXPECT_TRUE(result.is_null_at(2));
+ }
+
+ {
+ auto addr = ColumnIPv6::create();
+ addr->get_data() = {static_cast<IPv6>(1), ~static_cast<IPv6>(0),
static_cast<IPv6>(1)};
+ auto cidr = ColumnInt16::create();
+ cidr->get_data() = {128, 8, -16706};
+ auto nullable_addr = ColumnNullable::create(std::move(addr),
make_null_map({0, 1, 0}));
+ auto nullable_cidr = ColumnNullable::create(std::move(cidr),
make_null_map({0, 0, 1}));
+ auto nullable_ipv6_type = make_nullable(ipv6_type);
+ auto nullable_int16_type = make_nullable(int16_type);
+ FunctionIPv6CIDRToRange function;
+ DataTypes argument_types {nullable_ipv6_type, nullable_int16_type};
+ auto result_type = function.get_return_type_impl(argument_types);
+ Block block;
+ block.insert({std::move(nullable_addr), nullable_ipv6_type, "addr"});
+ block.insert({std::move(nullable_cidr), nullable_int16_type, "cidr"});
+ block.insert({nullptr, result_type, "result"});
+
+ ASSERT_TRUE(function.execute_impl(nullptr, block, {0, 1}, 2, 3).ok());
+ const auto& result = assert_cast<const
ColumnNullable&>(*block.get_by_position(2).column);
+ EXPECT_FALSE(result.is_null_at(0));
+ EXPECT_TRUE(result.is_null_at(1));
+ EXPECT_TRUE(result.is_null_at(2));
+ }
+
+ {
+ auto addr = ColumnString::create();
+ addr->insert_data("2001:db8::1", 11);
+ addr->insert_data("", 0);
+ auto cidr = ColumnInt16::create();
+ cidr->get_data() = {64, 8};
+ auto nullable_addr = ColumnNullable::create(std::move(addr),
make_null_map({0, 1}));
+ auto nullable_addr_type = make_nullable(string_type);
+ FunctionIPv6CIDRToRange function;
+ DataTypes argument_types {nullable_addr_type, int16_type};
+ auto result_type = function.get_return_type_impl(argument_types);
+ Block block;
+ block.insert({std::move(nullable_addr), nullable_addr_type, "addr"});
+ block.insert({std::move(cidr), int16_type, "cidr"});
+ block.insert({nullptr, result_type, "result"});
+
+ ASSERT_TRUE(function.execute_impl(nullptr, block, {0, 1}, 2, 2).ok());
+ const auto& result = assert_cast<const
ColumnNullable&>(*block.get_by_position(2).column);
+ EXPECT_FALSE(result.is_null_at(0));
+ EXPECT_TRUE(result.is_null_at(1));
+ }
+
+ {
+ auto ip = ColumnIPv4::create(3, 0xffffffff);
+ auto nullable_ip = ColumnNullable::create(std::move(ip),
ColumnUInt8::create(3, 0));
+ auto const_cidr_data =
+ ColumnNullable::create(ColumnInt16::create(1, -16706),
ColumnUInt8::create(1, 1));
+ auto const_cidr = ColumnConst::create(std::move(const_cidr_data), 3);
+ auto nullable_ipv4_type = make_nullable(ipv4_type);
+ auto nullable_int16_type = make_nullable(int16_type);
+ FunctionIPv4CIDRToRange function;
+ DataTypes argument_types {nullable_ipv4_type, nullable_int16_type};
+ auto result_type = function.get_return_type_impl(argument_types);
+ Block block;
+ block.insert({std::move(nullable_ip), nullable_ipv4_type, "ip"});
+ block.insert({std::move(const_cidr), nullable_int16_type, "cidr"});
+ block.insert({nullptr, result_type, "result"});
+
+ ASSERT_TRUE(function.execute_impl(nullptr, block, {0, 1}, 2, 3).ok());
+ const auto& result = assert_cast<const
ColumnNullable&>(*block.get_by_position(2).column);
+ EXPECT_TRUE(result.is_null_at(0));
+ EXPECT_TRUE(result.is_null_at(1));
+ EXPECT_TRUE(result.is_null_at(2));
+ }
+
+ {
+ auto addr = ColumnIPv6::create();
+ addr->get_data() = {~static_cast<IPv6>(0), static_cast<IPv6>(1)};
+ auto ipv6_null_map = make_null_map({1, 0});
+ auto ipv6_column = ColumnNullable::create(std::move(addr),
std::move(ipv6_null_map));
+ auto ipv6_count = ColumnInt8::create();
+ ipv6_count->get_data() = {127, 0};
+ auto ipv4_count = ColumnInt8::create();
+ ipv4_count->get_data() = {0, 127};
+ auto ipv6_count_null_map = make_null_map({0, 0});
+ auto ipv4_count_null_map = make_null_map({0, 1});
+ auto nullable_ipv6 = make_nullable(ipv6_type);
+ auto nullable_int8 = make_nullable(int8_type);
+ auto nullable_ipv6_count =
+ ColumnNullable::create(std::move(ipv6_count),
std::move(ipv6_count_null_map));
+ auto nullable_ipv4_count =
+ ColumnNullable::create(std::move(ipv4_count),
std::move(ipv4_count_null_map));
+ FunctionCutIPv6 function;
+ DataTypes argument_types {nullable_ipv6, nullable_int8, nullable_int8};
+ auto result_type = function.get_return_type_impl(argument_types);
+ Block block;
+ block.insert({std::move(ipv6_column), nullable_ipv6, "addr"});
+ block.insert({std::move(nullable_ipv6_count), nullable_int8,
"ipv6_count"});
+ block.insert({std::move(nullable_ipv4_count), nullable_int8,
"ipv4_count"});
+ block.insert({nullptr, result_type, "result"});
+
+ ASSERT_TRUE(function.execute_impl(nullptr, block, {0, 1, 2}, 3,
2).ok());
+ const auto& result = assert_cast<const
ColumnNullable&>(*block.get_by_position(3).column);
+ EXPECT_TRUE(result.is_null_at(0));
+ EXPECT_TRUE(result.is_null_at(1));
+ }
+}
+
+TEST(FunctionIpTest, IPv6CIDRToRangeStillRejectsInvalidNonNullString) {
+ auto addr = ColumnString::create();
+ addr->insert_data("abc", 3);
+ auto nullable_addr_type =
make_nullable(std::make_shared<DataTypeString>());
+ auto nullable_addr = ColumnNullable::create(std::move(addr),
ColumnUInt8::create(1, 0));
+ auto cidr = ColumnInt16::create(1, 64);
+ auto cidr_type = std::make_shared<DataTypeInt16>();
+ FunctionIPv6CIDRToRange function;
+ DataTypes argument_types {nullable_addr_type, cidr_type};
+ auto result_type = function.get_return_type_impl(argument_types);
+ Block block;
+ block.insert({std::move(nullable_addr), nullable_addr_type, "addr"});
+ block.insert({std::move(cidr), cidr_type, "cidr"});
+ block.insert({nullptr, result_type, "result"});
+
+ EXPECT_THROW(static_cast<void>(function.execute_impl(nullptr, block, {0,
1}, 2, 1)), Exception);
+}
+
class MockIndexReader : public segment_v2::InvertedIndexReader {
public:
MockIndexReader(const TabletIndex& index_meta)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
