This is an automated email from the ASF dual-hosted git repository.
taiyangli pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new bfab0b6a2a [GLUEN-9791][CH]Fix map key cannot be nullable (#9794)
bfab0b6a2a is described below
commit bfab0b6a2aa5c09fc357d7068a0d8944923c8329
Author: 李扬 <[email protected]>
AuthorDate: Thu Jun 19 10:57:59 2025 +0800
[GLUEN-9791][CH]Fix map key cannot be nullable (#9794)
* fix map key cannot be nullable
* fix fialed uts
* fix ut
---
.../execution/GlutenFunctionValidateSuite.scala | 5 ++
.../CommonScalarFunctionParser.cpp | 1 -
.../Parser/scalar_function_parser/map.cpp | 55 ++++++++++++++++++++++
3 files changed, 60 insertions(+), 1 deletion(-)
diff --git
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala
index 7ea5209e6d..10506cf9af 100644
---
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala
@@ -1364,4 +1364,9 @@ class GlutenFunctionValidateSuite extends
GlutenClickHouseWholeStageTransformerS
compareResultsAgainstVanillaSpark(sql, compareResult = true,
checkAggregateWithFilter)
}
}
+
+ test("Test map with nullable key") {
+ val sql = "select map(string_field1, int_field1) from json_test where
string_field1 is not null"
+ compareResultsAgainstVanillaSpark(sql, true, { _ => })
+ }
}
diff --git
a/cpp-ch/local-engine/Parser/scalar_function_parser/CommonScalarFunctionParser.cpp
b/cpp-ch/local-engine/Parser/scalar_function_parser/CommonScalarFunctionParser.cpp
index 841e51c00a..c1b7dcdb2e 100644
---
a/cpp-ch/local-engine/Parser/scalar_function_parser/CommonScalarFunctionParser.cpp
+++
b/cpp-ch/local-engine/Parser/scalar_function_parser/CommonScalarFunctionParser.cpp
@@ -169,7 +169,6 @@ REGISTER_COMMON_SCALAR_FUNCTION_PARSER(ArraysOverlap,
arrays_overlap, sparkArray
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(ArraysZip, arrays_zip,
arrayZipUnaligned);
// map functions
-REGISTER_COMMON_SCALAR_FUNCTION_PARSER(Map, map, map);
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(GetMapValue, get_map_value,
arrayElementOrNull);
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(MapKeys, map_keys, mapKeys);
REGISTER_COMMON_SCALAR_FUNCTION_PARSER(MapValues, map_values, mapValues);
diff --git a/cpp-ch/local-engine/Parser/scalar_function_parser/map.cpp
b/cpp-ch/local-engine/Parser/scalar_function_parser/map.cpp
new file mode 100644
index 0000000000..e9f25b6c06
--- /dev/null
+++ b/cpp-ch/local-engine/Parser/scalar_function_parser/map.cpp
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <Parser/FunctionParser.h>
+
+namespace DB
+{
+namespace ErrorCodes
+{
+ extern const int BAD_ARGUMENTS;
+ extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+}
+}
+
+namespace local_engine
+{
+
+class FunctionMap : public FunctionParser
+{
+public:
+ explicit FunctionMap(ParserContextPtr parser_context_) :
FunctionParser(parser_context_) { }
+ ~FunctionMap() override = default;
+
+ static constexpr auto name = "map";
+
+ String getName() const override { return name; }
+
+ const DB::ActionsDAG::Node * parse(const
substrait::Expression_ScalarFunction & substrait_func, DB::ActionsDAG &
actions_dag) const override
+ {
+ /// Parse map(arg1, arg2, ...) as map(assumeNotNull(arg1), arg2, ...)
+ auto parsed_args = parseFunctionArguments(substrait_func, actions_dag);
+ for (size_t i = 0; i < parsed_args.size(); i += 2)
+ parsed_args[i] = toFunctionNode(actions_dag, "assumeNotNull",
{parsed_args[i]});
+
+ auto map_node = toFunctionNode(actions_dag, "map", parsed_args);
+ return convertNodeTypeIfNeeded(substrait_func, map_node, actions_dag);
+ }
+};
+static FunctionParserRegister<FunctionMap> register_map;
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
