This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 82d44b4cc14 [fix](function) Remove duplicated keys of nested maps in
string casts (#68639)
82d44b4cc14 is described below
commit 82d44b4cc14575125714c21587452169ada1e145
Author: Chenyang Sun <[email protected]>
AuthorDate: Thu Oct 15 10:02:13 2026 +0800
[fix](function) Remove duplicated keys of nested maps in string casts
(#68639)
### What problem does this PR solve?
A string cast to MAP keeps the last value of a duplicated key, but the
maps inside an ARRAY, a STRUCT, or an ARRAY or STRUCT map value keep
both entries, so size() counts both and element_at() returns the first
value:
SELECT CAST('[{"a":1,"a":2}]' AS ARRAY<MAP<STRING, INT>>);
-- before: [{"a":1, "a":2}]
-- after: [{"a":2}]
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---------
Co-authored-by: Claude Opus 5.5 <[email protected]>
---
be/src/exprs/function/cast/cast_base.cpp | 28 +++++++++
be/src/exprs/function/cast/cast_to_map.h | 48 ++-------------
be/test/exprs/function/cast/cast_to_map_test.cpp | 50 ++++++++++++++++
.../test_cast_string_to_nested_map_dedup.out | 44 ++++++++++++++
.../test_cast_string_to_nested_map_dedup.groovy | 70 ++++++++++++++++++++++
5 files changed, 197 insertions(+), 43 deletions(-)
diff --git a/be/src/exprs/function/cast/cast_base.cpp
b/be/src/exprs/function/cast/cast_base.cpp
index 00f29e7a9b2..c62403baa4a 100644
--- a/be/src/exprs/function/cast/cast_base.cpp
+++ b/be/src/exprs/function/cast/cast_base.cpp
@@ -21,6 +21,9 @@
#include <cstdint>
#include <utility>
+#include "core/column/column_array.h"
+#include "core/column/column_map.h"
+#include "core/column/column_struct.h"
#include "util/jsonb_writer.h"
namespace doris::CastWrapper {
@@ -73,6 +76,29 @@ ChildNullMask build_child_null_mask(const
NullMap::value_type* parent_null_map,
.mask_holder = nullptr};
}
+// Remove the duplicated keys of every map in the column, also the maps inside
arrays, structs
+// and map values. The last value of a key wins, as in map().
+static Status deduplicate_map_keys(IColumn& column) {
+ if (auto* nullable = check_and_get_column<ColumnNullable>(column)) {
+ return deduplicate_map_keys(nullable->get_nested_column());
+ }
+ if (auto* array = check_and_get_column<ColumnArray>(column)) {
+ return deduplicate_map_keys(array->get_data());
+ }
+ if (auto* st = check_and_get_column<ColumnStruct>(column)) {
+ for (size_t i = 0; i < st->tuple_size(); ++i) {
+ RETURN_IF_ERROR(deduplicate_map_keys(st->get_column(i)));
+ }
+ return Status::OK();
+ }
+ if (auto* map = check_and_get_column<ColumnMap>(column)) {
+ // Values last: the ones dropped with a duplicated key are not visited.
+ RETURN_IF_ERROR(map->deduplicate_keys());
+ return deduplicate_map_keys(map->get_values());
+ }
+ return Status::OK();
+}
+
Status cast_from_generic_to_jsonb(FunctionContext* context, Block& block,
const ColumnNumbers& arguments, uint32_t
result,
size_t input_rows_count, const
NullMap::value_type* null_map) {
@@ -252,6 +278,7 @@ Status cast_from_string_to_complex_type(FunctionContext*
context, Block& block,
}
}
+ RETURN_IF_ERROR(deduplicate_map_keys(nested_column));
block.get_by_position(result).column = std::move(to_column);
return Status::OK();
}
@@ -286,6 +313,7 @@ Status
cast_from_string_to_complex_type_strict_mode(FunctionContext* context, Bl
nullable_col_to.get_null_map_data().push_back(0);
}
}
+ RETURN_IF_ERROR(deduplicate_map_keys(nested_column));
block.get_by_position(result).column = std::move(to_column);
return Status::OK();
}
diff --git a/be/src/exprs/function/cast/cast_to_map.h
b/be/src/exprs/function/cast/cast_to_map.h
index 4714d22805d..b811e4a21ce 100644
--- a/be/src/exprs/function/cast/cast_to_map.h
+++ b/be/src/exprs/function/cast/cast_to_map.h
@@ -23,53 +23,15 @@
#include "exprs/function/cast/cast_base.h"
namespace doris::CastWrapper {
-
-inline Status deduplicate_map_keys_in_result(Block& block, uint32_t result) {
- auto result_column_name = block.get_by_position(result).column->get_name();
- auto mutable_result_column =
IColumn::mutate(std::move(block.get_by_position(result).column));
-
- if (auto* nullable_column =
check_and_get_column<ColumnNullable>(*mutable_result_column)) {
- auto nested_column =
IColumn::mutate(nullable_column->get_nested_column_ptr());
- auto* map_column = check_and_get_column<ColumnMap>(*nested_column);
- if (!map_column) {
- return Status::RuntimeError("Illegal column {} for function CAST
AS MAP",
- result_column_name);
- }
-
- RETURN_IF_ERROR(map_column->deduplicate_keys(true));
- ColumnPtr nested_column_ptr = std::move(nested_column);
- nullable_column->change_nested_column(nested_column_ptr);
- } else {
- auto* map_column =
check_and_get_column<ColumnMap>(*mutable_result_column);
- if (!map_column) {
- return Status::RuntimeError("Illegal column {} for function CAST
AS MAP",
- result_column_name);
- }
-
- RETURN_IF_ERROR(map_column->deduplicate_keys(true));
- }
-
- block.get_by_position(result).column = std::move(mutable_result_column);
- return Status::OK();
-}
-
-inline WrapperType wrap_string_to_map_wrapper(WrapperType wrapper) {
- return [wrapper = std::move(wrapper)](FunctionContext* context, Block&
block,
- const ColumnNumbers& arguments,
uint32_t result,
- size_t input_rows_count,
- const NullMap::value_type* null_map
= nullptr) {
- RETURN_IF_ERROR(wrapper(context, block, arguments, result,
input_rows_count, null_map));
- return deduplicate_map_keys_in_result(block, result);
- };
-}
-
//TODO(Amory) . Need support more cast for key , value for map
WrapperType create_map_wrapper(FunctionContext* context, const DataTypePtr&
from_type,
const DataTypeMap& to_type) {
if (is_string_type(from_type->get_primitive_type())) {
- auto wrapper = context->enable_strict_mode() ?
cast_from_string_to_complex_type_strict_mode
- :
cast_from_string_to_complex_type;
- return wrap_string_to_map_wrapper(wrapper);
+ if (context->enable_strict_mode()) {
+ return cast_from_string_to_complex_type_strict_mode;
+ } else {
+ return cast_from_string_to_complex_type;
+ }
}
const auto* from = check_and_get_data_type<DataTypeMap>(from_type.get());
if (!from) {
diff --git a/be/test/exprs/function/cast/cast_to_map_test.cpp
b/be/test/exprs/function/cast/cast_to_map_test.cpp
index 77d0a9f15f0..7753dc55bf3 100644
--- a/be/test/exprs/function/cast/cast_to_map_test.cpp
+++ b/be/test/exprs/function/cast/cast_to_map_test.cpp
@@ -25,10 +25,12 @@
#include "core/column/column.h"
#include "core/column/column_map.h"
+#include "core/data_type/data_type_array.h"
#include "core/data_type/data_type_map.h"
#include "core/data_type/data_type_nullable.h"
#include "core/data_type/data_type_number.h"
#include "core/data_type/data_type_string.h"
+#include "core/data_type/data_type_struct.h"
#include "core/data_type/primitive_type.h"
#include "core/field.h"
#include "exprs/function/cast/cast_base.h"
@@ -276,4 +278,52 @@ TEST_F(FunctionCastTest,
test_cast_map_unchanged_key_with_null_row) {
EXPECT_EQ(to_type->to_string(*block.get_by_position(1).column, 1),
"{1:2}");
}
+// A duplicated key keeps its last value in every map of the result, also in
the maps inside
+// arrays, structs and map values, as it does in the top level map.
+TEST_F(FunctionCastTest, test_from_string_deduplicates_nested_map_keys) {
+ auto map_type =
std::make_shared<DataTypeNullable>(std::make_shared<DataTypeMap>(
+
std::make_shared<DataTypeNullable>(std::make_shared<DataTypeString>()),
+
std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt32>())));
+ auto string_type =
std::make_shared<DataTypeNullable>(std::make_shared<DataTypeString>());
+ auto array_of_map_type =
+
std::make_shared<DataTypeNullable>(std::make_shared<DataTypeArray>(map_type));
+
+ struct Case {
+ DataTypePtr to_type;
+ std::string from;
+ std::string expected;
+ };
+ std::vector<Case> cases = {
+ {map_type, R"({"a":1,"a":2})", R"({"a":2})"},
+ {array_of_map_type, R"([{"a":1,"a":2}])", R"([{"a":2}])"},
+ {std::make_shared<DataTypeNullable>(
+ std::make_shared<DataTypeStruct>(DataTypes {map_type},
Strings {"m"})),
+ R"({"m":{"a":1,"a":2}})", R"({"m":{"a":2}})"},
+ {std::make_shared<DataTypeNullable>(
+ std::make_shared<DataTypeMap>(string_type,
array_of_map_type)),
+ R"({"outer":[{"a":1,"a":2}]})", R"({"outer":[{"a":2}]})"},
+ {std::make_shared<DataTypeNullable>(
+ std::make_shared<DataTypeMap>(string_type,
array_of_map_type)),
+ R"({"outer":[{"a":1,"a":2}],"outer":[{"b":1,"b":2}]})",
R"({"outer":[{"b":2}]})"},
+ };
+
+ for (bool is_strict_mode : {false, true}) {
+ for (const auto& c : cases) {
+ auto ctx = create_context(is_strict_mode);
+ DataTypePtr from_type = std::make_shared<DataTypeString>();
+ auto fn = get_cast_wrapper(ctx.get(), from_type, c.to_type);
+ ASSERT_TRUE(fn != nullptr);
+
+ Block block = {
+ {ColumnHelper::create_column<DataTypeString>({c.from}),
from_type, "from"},
+ {nullptr, c.to_type, "to"},
+ };
+ ASSERT_TRUE(fn(ctx.get(), block, {0}, 1, block.rows(), nullptr));
+ EXPECT_EQ(c.to_type->to_string(*block.get_by_position(1).column,
0), c.expected)
+ << "from: " << c.from << ", to: " << c.to_type->get_name()
+ << ", strict mode: " << is_strict_mode;
+ }
+ }
+}
+
} // namespace doris
diff --git
a/regression-test/data/query_p0/sql_functions/cast_function/test_cast_string_to_nested_map_dedup.out
b/regression-test/data/query_p0/sql_functions/cast_function/test_cast_string_to_nested_map_dedup.out
new file mode 100644
index 00000000000..682df23194a
--- /dev/null
+++
b/regression-test/data/query_p0/sql_functions/cast_function/test_cast_string_to_nested_map_dedup.out
@@ -0,0 +1,44 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !top --
+{"a":2}
+
+-- !map_in_map --
+{"outer":{"a":2}}
+
+-- !array --
+[{"a":2}] 1 2
+
+-- !struct --
+{"m":{"a":2}} 1
+
+-- !array_in_map --
+{"outer":[{"a":2}]} 2
+
+-- !struct_in_map --
+{"outer":{"m":{"a":2}}}
+
+-- !top --
+{"a":2}
+
+-- !map_in_map --
+{"outer":{"a":2}}
+
+-- !array --
+[{"a":2}] 1 2
+
+-- !struct --
+{"m":{"a":2}} 1
+
+-- !array_in_map --
+{"outer":[{"a":2}]} 2
+
+-- !struct_in_map --
+{"outer":{"m":{"a":2}}}
+
+-- !column --
+1 [{"a":2}]
+2 [{"a":1}, {"b":3}]
+3 [{"a":1, "b":2}]
+4 []
+5 \N
+
diff --git
a/regression-test/suites/query_p0/sql_functions/cast_function/test_cast_string_to_nested_map_dedup.groovy
b/regression-test/suites/query_p0/sql_functions/cast_function/test_cast_string_to_nested_map_dedup.groovy
new file mode 100644
index 00000000000..cfc791e2a09
--- /dev/null
+++
b/regression-test/suites/query_p0/sql_functions/cast_function/test_cast_string_to_nested_map_dedup.groovy
@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_cast_string_to_nested_map_dedup") {
+ // A duplicated key keeps its last value in every map of a string cast,
also in the maps
+ // inside arrays, structs and map values, as it does in the top level map.
+ def check = {
+ qt_top """ SELECT CAST('{"a":1,"a":2}' AS MAP<STRING, INT>) """
+ qt_map_in_map """ SELECT CAST('{"outer":{"a":1,"a":2}}' AS MAP<STRING,
MAP<STRING, INT>>) """
+ qt_array """
+ SELECT CAST('[{"a":1,"a":2}]' AS ARRAY<MAP<STRING, INT>>) AS c,
+ size(element_at(CAST('[{"a":1,"a":2}]' AS ARRAY<MAP<STRING,
INT>>), 1)),
+ element_at(element_at(CAST('[{"a":1,"a":2}]' AS
ARRAY<MAP<STRING, INT>>), 1), 'a')
+ """
+ qt_struct """
+ SELECT CAST('{"m":{"a":1,"a":2}}' AS STRUCT<m:MAP<STRING, INT>>)
AS c,
+ size(struct_element(CAST('{"m":{"a":1,"a":2}}' AS
STRUCT<m:MAP<STRING, INT>>), 'm'))
+ """
+ qt_array_in_map """
+ SELECT CAST('{"outer":[{"a":1,"a":2}]}' AS MAP<STRING,
ARRAY<MAP<STRING, INT>>>) AS c,
+ element_at(element_at(element_at(
+ CAST('{"outer":[{"a":1,"a":2}]}' AS MAP<STRING,
ARRAY<MAP<STRING, INT>>>),
+ 'outer'), 1), 'a')
+ """
+ qt_struct_in_map """
+ SELECT CAST('{"outer":{"m":{"a":1,"a":2}}}' AS MAP<STRING,
STRUCT<m:MAP<STRING, INT>>>)
+ """
+ }
+
+ sql """ set enable_strict_cast = false """
+ check()
+ sql """ set enable_strict_cast = true """
+ check()
+ sql """ set enable_strict_cast = false """
+
+ sql """ DROP TABLE IF EXISTS test_cast_string_to_nested_map_dedup """
+ sql """
+ CREATE TABLE test_cast_string_to_nested_map_dedup (
+ id INT,
+ s STRING
+ ) DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES ("replication_num" = "1")
+ """
+ sql """
+ INSERT INTO test_cast_string_to_nested_map_dedup VALUES
+ (1, '[{"a":1,"a":2}]'),
+ (2, '[{"a":1}, {"b":1,"b":2,"b":3}]'),
+ (3, '[{"a":1,"b":2}]'),
+ (4, '[]'),
+ (5, NULL)
+ """
+ order_qt_column """
+ SELECT id, CAST(s AS ARRAY<MAP<STRING, INT>>) FROM
test_cast_string_to_nested_map_dedup
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
