This is an automated email from the ASF dual-hosted git repository.
exmy pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 3817c1dab2 [GLUTEN-12834][CH] Fix nullable filtering after Expand with
constant columns (#12818)
3817c1dab2 is described below
commit 3817c1dab26477c62857823d87745562b8749b05
Author: exmy <[email protected]>
AuthorDate: Thu Aug 20 16:43:04 2026 +0800
[GLUTEN-12834][CH] Fix nullable filtering after Expand with constant
columns (#12818)
* [CH] Fix nullable filtering after Expand with constant columns
Materialize constant selection columns in regular and advanced Expand to
prevent stale constant filter state from leaking null rows. Preserve nullable
types for null literals and add regression coverage for both Expand paths.
* [CH] Simplify nullable handling and Expand test configuration
Use the ClickHouse makeNullable helper for null literals and configure the
lazy aggregate Expand tests only through SparkEnv, which is the actual
configuration source.
---
.../GlutenClickHouseTPCHSaltNullParquetSuite.scala | 76 +++++++++++++++++++++-
.../local-engine/Operator/AdvancedExpandStep.cpp | 2 +-
cpp-ch/local-engine/Operator/ExpandTransform.cpp | 2 +-
.../Parser/RelParsers/ExpandRelParser.cpp | 7 ++
4 files changed, 84 insertions(+), 3 deletions(-)
diff --git
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/tpch/GlutenClickHouseTPCHSaltNullParquetSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/tpch/GlutenClickHouseTPCHSaltNullParquetSuite.scala
index 4766c2fd24..f6277c214f 100644
---
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/tpch/GlutenClickHouseTPCHSaltNullParquetSuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/tpch/GlutenClickHouseTPCHSaltNullParquetSuite.scala
@@ -21,7 +21,7 @@ import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.execution._
import org.apache.gluten.execution.GlutenPlan
-import org.apache.spark.{SparkConf, SparkException}
+import org.apache.spark.{SparkConf, SparkEnv, SparkException}
import org.apache.spark.sql.DataFrame
import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, ConstantFolding,
NullPropagation}
import org.apache.spark.sql.execution.{ColumnarToRowExec, ReusedSubqueryExec,
SubqueryExec}
@@ -878,6 +878,80 @@ class GlutenClickHouseTPCHSaltNullParquetSuite
compareResultsAgainstVanillaSpark(sql, true, { _ => })
}
+ test("grouping sets preserves nullable columns across union") {
+ val sql =
+ """
+ |select msg_type, os, count(*) as cnt
+ |from (
+ | select 'file' as msg_type, 'Android' as os, id from range(10)
+ | union all
+ | select 'file' as msg_type, 'iOS' as os, id from range(10)
+ |) t
+ |group by grouping sets ((msg_type), (os))
+ |having msg_type is not null
+ |order by msg_type, os, cnt
+ |""".stripMargin
+ withSparkEnvConf(CHConfig.runtimeConfig("enable_lazy_aggregate_expand"),
"false") {
+ compareResultsAgainstVanillaSpark(
+ sql,
+ true,
+ {
+ df =>
+ val expands =
collectWithSubqueries(df.queryExecution.executedPlan) {
+ case e: ExpandExecTransformer
+ if !e.child.isInstanceOf[HashAggregateExecBaseTransformer] =>
+ e
+ }
+ assert(expands.size == 1)
+ }
+ )
+ }
+ }
+
+ test("lazy aggregate expand preserves nullable columns across union") {
+ val sql =
+ """
+ |select msg_type, os, count(*) as cnt
+ |from (
+ | select 'file' as msg_type, 'Android' as os, id from range(10)
+ | union all
+ | select 'file' as msg_type, 'iOS' as os, id from range(10)
+ |) t
+ |group by grouping sets ((msg_type), (os))
+ |having msg_type is not null
+ |order by msg_type, os, cnt
+ |""".stripMargin
+ withSparkEnvConf(CHConfig.runtimeConfig("enable_lazy_aggregate_expand"),
"true") {
+ compareResultsAgainstVanillaSpark(
+ sql,
+ true,
+ {
+ df =>
+ val expands =
collectWithSubqueries(df.queryExecution.executedPlan) {
+ case e: ExpandExecTransformer
+ if e.child.isInstanceOf[HashAggregateExecBaseTransformer] =>
+ e
+ }
+ assert(expands.size == 1)
+ }
+ )
+ }
+ }
+
+ private def withSparkEnvConf(key: String, value: String)(f: => Unit): Unit =
{
+ val sparkConf = SparkEnv.get.conf
+ val previousValue = sparkConf.getOption(key)
+ sparkConf.set(key, value)
+ try {
+ f
+ } finally {
+ previousValue match {
+ case Some(previous) => sparkConf.set(key, previous)
+ case None => sparkConf.remove(key)
+ }
+ }
+ }
+
test("expand with nullable type not match") {
val sql =
"""
diff --git a/cpp-ch/local-engine/Operator/AdvancedExpandStep.cpp
b/cpp-ch/local-engine/Operator/AdvancedExpandStep.cpp
index c7729c26c5..e81e39ce29 100644
--- a/cpp-ch/local-engine/Operator/AdvancedExpandStep.cpp
+++ b/cpp-ch/local-engine/Operator/AdvancedExpandStep.cpp
@@ -248,7 +248,7 @@ void AdvancedExpandTransform::expandInputChunk()
input_arg.column = input_column;
input_arg.type = input_header->getByPosition(index).type;
/// input_column maybe non-Nullable
- columns[col_i] = DB::castColumn(input_arg, type);
+ columns[col_i] = DB::castColumn(input_arg,
type)->convertToFullColumnIfConst();
}
else if (kind == EXPAND_FIELD_KIND_LITERAL)
{
diff --git a/cpp-ch/local-engine/Operator/ExpandTransform.cpp
b/cpp-ch/local-engine/Operator/ExpandTransform.cpp
index afc6596d55..e79076a762 100644
--- a/cpp-ch/local-engine/Operator/ExpandTransform.cpp
+++ b/cpp-ch/local-engine/Operator/ExpandTransform.cpp
@@ -114,7 +114,7 @@ void ExpandTransform::work()
input_arg.column = input_column;
input_arg.type = input_header.getByPosition(index).type;
/// input_column maybe non-Nullable
- columns[col_i] = DB::castColumn(input_arg, type);
+ columns[col_i] = DB::castColumn(input_arg,
type)->convertToFullColumnIfConst();
}
else if (kind == EXPAND_FIELD_KIND_LITERAL)
{
diff --git a/cpp-ch/local-engine/Parser/RelParsers/ExpandRelParser.cpp
b/cpp-ch/local-engine/Parser/RelParsers/ExpandRelParser.cpp
index 4bbdba6f90..3515060a22 100644
--- a/cpp-ch/local-engine/Parser/RelParsers/ExpandRelParser.cpp
+++ b/cpp-ch/local-engine/Parser/RelParsers/ExpandRelParser.cpp
@@ -20,6 +20,7 @@
#include <Columns/ColumnAggregateFunction.h>
#include <Core/Block.h>
#include <Core/ColumnWithTypeAndName.h>
+#include <DataTypes/DataTypeNullable.h>
#include <Operator/AdvancedExpandStep.h>
#include <Operator/ExpandStep.h>
#include <Parser/RelParsers/RelParser.h>
@@ -113,6 +114,12 @@ ExpandField ExpandRelParser::buildExpandField(const
DB::Block & header, const su
else if (project_expr.has_literal())
{
auto [type, field] = parseLiteral(project_expr.literal());
+ // A NULL literal is nullable even when the type carried by the
+ // Substrait literal does not explicitly encode nullability.
+ // Keep that information in the Expand output type so the
+ // generated column contains a real null map.
+ if (field.isNull() && type && !type->isNullable())
+ type = DB::makeNullable(type);
kinds.push_back(ExpandFieldKind::EXPAND_FIELD_KIND_LITERAL);
fields.push_back(field);
updateType(types[i], type);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
