surnaik commented on code in PR #13531:
URL: https://github.com/apache/gluten/pull/13531#discussion_r4571204479


##########
backends-velox/src/test/scala/org/apache/gluten/sql/GlutenBloomFilterFallbackSuite.scala:
##########
@@ -486,4 +487,236 @@ class GlutenBloomFilterFallbackSuite extends 
WholeStageTransformerSuite {
       }
     }
   }
+
+  // Gated to Spark 3.5+: Spark's BloomFilterAggregate only supports 
StringType starting in
+  // Spark 3.5 (SPARK-43726); on Spark 3.4 CheckAnalysis rejects 
bloom_filter_agg(string) with
+  // DATATYPE_MISMATCH.BLOOM_FILTER_WRONG_TYPE before Optimizer/Gluten rules 
run.
+  testWithMinSparkVersion(
+    "GLUTEN-13489: bloom_filter_agg on string column falls back to vanilla 
Spark",
+    "3.5") {
+    if 
(BackendsApiManager.getSettings.requireBloomFilterAggMightContainJointFallback())
 {
+      val table = "bloom_filter_str_test"
+      withTempView(table) {
+        Seq("a", "b", "c").toDF("s").createOrReplaceTempView(table)
+        val aggDf = spark.sql(s"SELECT bloom_filter_agg(s) AS bf FROM $table")
+        val bytes = aggDf.collect().head.getAs[Array[Byte]]("bf")
+        
assert(!aggDf.queryExecution.executedPlan.toString.contains("velox_bloom_filter_agg"))
+        val bf = BloomFilter.readFrom(new ByteArrayInputStream(bytes))
+        assert(bf.mightContainString("a"))
+        assert(!bf.mightContainString("z"))
+
+        val df = spark.sql(
+          s"""
+             |SELECT might_contain((SELECT bloom_filter_agg(s) FROM $table), 
xxhash64(s))
+             |FROM $table
+             |""".stripMargin)
+        checkAnswer(df, Seq(Row(false), Row(false), Row(false)))
+        val plan = df.queryExecution.executedPlan.toString
+        assert(!plan.contains("velox_might_contain"))
+        assert(!plan.contains("velox_bloom_filter_agg"))
+      }
+    }
+  }
+
+  test("GLUTEN-13489: bloom_filter_agg on nullable column filters out nulls 
and runs natively") {
+    if 
(BackendsApiManager.getSettings.requireBloomFilterAggMightContainJointFallback())
 {
+      val table = "bloom_filter_nullable_test"
+      withTempView(table) {
+        Seq[java.lang.Long](1L, null, 2L, 
3L).toDF("col").createOrReplaceTempView(table)
+        val df = spark.sql(
+          s"""
+             |SELECT col
+             |FROM $table
+             |WHERE might_contain((SELECT bloom_filter_agg(col) FROM $table), 
col)
+             |""".stripMargin)
+        checkAnswer(df, Seq(Row(1L), Row(2L), Row(3L)))
+        val nativeAggs = collectWithSubqueries(df.queryExecution.executedPlan) 
{
+          case h: HashAggregateExecBaseTransformer => h
+        }
+        assert(nativeAggs.size == 2, df.queryExecution.executedPlan.toString)
+
+        val filteredDf = spark.sql(
+          s"""
+             |SELECT col
+             |FROM $table
+             |WHERE might_contain(
+             |  (SELECT bloom_filter_agg(col) FILTER (WHERE col IS NULL OR col 
> 1L) FROM $table),
+             |  col)
+             |""".stripMargin)
+        checkAnswer(filteredDf, Seq(Row(2L), Row(3L)))
+        val filteredNativeAggs = 
collectWithSubqueries(filteredDf.queryExecution.executedPlan) {
+          case h: HashAggregateExecBaseTransformer => h
+        }
+        assert(filteredNativeAggs.size == 2, 
filteredDf.queryExecution.executedPlan.toString)
+      }
+
+      val allNullTable = "bloom_filter_all_null_test"
+      withTempView(allNullTable) {
+        Seq[java.lang.Long](null, 
null).toDF("col").createOrReplaceTempView(allNullTable)
+        val df = spark.sql(
+          s"SELECT might_contain((SELECT bloom_filter_agg(col) FROM 
$allNullTable), 1L)")
+        checkAnswer(df, Seq(Row(null)))

Review Comment:
   Good catch! In native execution, `BloomFilterAggAggregate` enforces 
`VELOX_USER_CHECK(!decoded.isNullAt(idx), "First argument of bloom_filter_agg 
cannot be null")` in `BloomFilterAggAggregate.cpp`, which is why the logical 
rule attaches `IsNotNull(child)` to `AggregateExpression.filter` when `child` 
is nullable, and `BloomFilterAggAggregate::extractValues` outputs `NULL` when 
no rows were accumulated (`!accumulator->initialized()`).
   
   However, when `VeloxBloomFilterAggregate` falls back to the JVM (e.g., 
`GlutenConfig.COLUMNAR_HASHAGG_ENABLED.key -> "false"`), 
`VeloxBloomFilterAggregate.eval` previously serialized the empty filter instead 
of returning `null` (unlike vanilla Spark's `BloomFilterAggregate.eval`, which 
returns `null` when `buffer.cardinality() == 0L`).
   
   In `5a21ff0ef`, I implemented `VeloxBloomFilter.cardinality()` and updated 
`VeloxBloomFilterAggregate.eval` to return `null` when `buffer.cardinality() == 
0L`, and added the test case with `GlutenConfig.COLUMNAR_HASHAGG_ENABLED.key -> 
"false"` on `allNullTable`.



##########
backends-velox/src/main/scala/org/apache/gluten/extension/BloomFilterMightContainJointRewriteRule.scala:
##########
@@ -49,12 +54,12 @@ case class BloomFilterMightContainJointRewriteRule(
           Expression,
           Int,
           Int) => TypedImperativeAggregate[T]): Expression = expr match {
-    case BloomFilterAggregate(
+    case bf @ BloomFilterAggregate(
           child,
           estimatedNumItemsExpression,
           numBitsExpression,
           mutableAggBufferOffset,
-          inputAggBufferOffset) =>
+          inputAggBufferOffset) if !isKeepVanilla(bf) =>

Review Comment:
   Updated in `5a21ff0ef`: the physical 
`BloomFilterMightContainJointRewriteRule` now requires `child: XxHash64` in 
`replaceBloomFilterAggregate` and `valueExpression: XxHash64` in 
`replaceMightContain` (matching `InjectRuntimeFilter`'s shape) in addition to 
`!isKeepVanilla(...)`, and I added test cases for both user-written 
`xxhash64(...)` calls and untagged non-`XxHash64` expressions.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to