philo-he commented on code in PR #13531:
URL: https://github.com/apache/gluten/pull/13531#discussion_r4557989296


##########
backends-velox/src/test/scala/org/apache/gluten/sql/GlutenBloomFilterFallbackSuite.scala:
##########
@@ -486,4 +487,236 @@ class GlutenBloomFilterFallbackSuite extends 
WholeStageTransformerSuite {
       }
     }
   }
+
+  // Gated to Spark 3.5+: Spark's BloomFilterAggregate only supports 
StringType starting in
+  // Spark 3.5 (SPARK-43726); on Spark 3.4 CheckAnalysis rejects 
bloom_filter_agg(string) with
+  // DATATYPE_MISMATCH.BLOOM_FILTER_WRONG_TYPE before Optimizer/Gluten rules 
run.
+  testWithMinSparkVersion(
+    "GLUTEN-13489: bloom_filter_agg on string column falls back to vanilla 
Spark",
+    "3.5") {
+    if 
(BackendsApiManager.getSettings.requireBloomFilterAggMightContainJointFallback())
 {
+      val table = "bloom_filter_str_test"
+      withTempView(table) {
+        Seq("a", "b", "c").toDF("s").createOrReplaceTempView(table)
+        val aggDf = spark.sql(s"SELECT bloom_filter_agg(s) AS bf FROM $table")
+        val bytes = aggDf.collect().head.getAs[Array[Byte]]("bf")
+        
assert(!aggDf.queryExecution.executedPlan.toString.contains("velox_bloom_filter_agg"))
+        val bf = BloomFilter.readFrom(new ByteArrayInputStream(bytes))
+        assert(bf.mightContainString("a"))
+        assert(!bf.mightContainString("z"))
+
+        val df = spark.sql(
+          s"""
+             |SELECT might_contain((SELECT bloom_filter_agg(s) FROM $table), 
xxhash64(s))
+             |FROM $table
+             |""".stripMargin)
+        checkAnswer(df, Seq(Row(false), Row(false), Row(false)))
+        val plan = df.queryExecution.executedPlan.toString
+        assert(!plan.contains("velox_might_contain"))
+        assert(!plan.contains("velox_bloom_filter_agg"))
+      }
+    }
+  }
+
+  test("GLUTEN-13489: bloom_filter_agg on nullable column filters out nulls 
and runs natively") {
+    if 
(BackendsApiManager.getSettings.requireBloomFilterAggMightContainJointFallback())
 {
+      val table = "bloom_filter_nullable_test"
+      withTempView(table) {
+        Seq[java.lang.Long](1L, null, 2L, 
3L).toDF("col").createOrReplaceTempView(table)
+        val df = spark.sql(
+          s"""
+             |SELECT col
+             |FROM $table
+             |WHERE might_contain((SELECT bloom_filter_agg(col) FROM $table), 
col)
+             |""".stripMargin)
+        checkAnswer(df, Seq(Row(1L), Row(2L), Row(3L)))
+        val nativeAggs = collectWithSubqueries(df.queryExecution.executedPlan) 
{
+          case h: HashAggregateExecBaseTransformer => h
+        }
+        assert(nativeAggs.size == 2, df.queryExecution.executedPlan.toString)
+
+        val filteredDf = spark.sql(
+          s"""
+             |SELECT col
+             |FROM $table
+             |WHERE might_contain(
+             |  (SELECT bloom_filter_agg(col) FILTER (WHERE col IS NULL OR col 
> 1L) FROM $table),
+             |  col)
+             |""".stripMargin)
+        checkAnswer(filteredDf, Seq(Row(2L), Row(3L)))
+        val filteredNativeAggs = 
collectWithSubqueries(filteredDf.queryExecution.executedPlan) {
+          case h: HashAggregateExecBaseTransformer => h
+        }
+        assert(filteredNativeAggs.size == 2, 
filteredDf.queryExecution.executedPlan.toString)
+      }
+
+      val allNullTable = "bloom_filter_all_null_test"
+      withTempView(allNullTable) {
+        Seq[java.lang.Long](null, 
null).toDF("col").createOrReplaceTempView(allNullTable)
+        val df = spark.sql(
+          s"SELECT might_contain((SELECT bloom_filter_agg(col) FROM 
$allNullTable), 1L)")
+        checkAnswer(df, Seq(Row(null)))

Review Comment:
   This test only covers the native path. When `VeloxBloomFilterAggregate` runs 
in the JVM instead (for example with hash-agg offload disabled, or after a 
whole-stage fallback), I think this query returns `false` instead of `NULL`.
   
   Why: when no value is added, Spark's `BloomFilterAggregate.eval` returns 
`null`, but `VeloxBloomFilterAggregate.eval` always returns the serialized 
filter. Here the new `IS NOT NULL` filter drops every row, so the JVM path 
returns an empty filter, and `might_contain` on an empty filter returns `false`.
   
   Could you add a case with `GlutenConfig.COLUMNAR_HASHAGG_ENABLED.key -> 
"false"`? If it returns `false`, `VeloxBloomFilterAggregate.eval` should return 
`null` when no value was added, as Spark does.



##########
backends-velox/src/main/scala/org/apache/gluten/extension/BloomFilterMightContainJointRewriteRule.scala:
##########
@@ -49,12 +54,12 @@ case class BloomFilterMightContainJointRewriteRule(
           Expression,
           Int,
           Int) => TypedImperativeAggregate[T]): Expression = expr match {
-    case BloomFilterAggregate(
+    case bf @ BloomFilterAggregate(
           child,
           estimatedNumItemsExpression,
           numBitsExpression,
           mutableAggBufferOffset,
-          inputAggBufferOffset) =>
+          inputAggBufferOffset) if !isKeepVanilla(bf) =>

Review Comment:
   This rule now depends on the `KEEP_VANILLA` tag set by the Logical rule: any 
bloom filter expression without the tag is still rewritten. `transform` and 
`withNewChildren` keep tags, but a rule that rebuilds the expression with 
`copy(...)` or its constructor drops them. I didn't find such a rule in Spark 
today. If one appears later, the #13489 failures come back silently; for 
example, a standalone `bloom_filter_agg` would return Velox-format bytes again.
   
   Could you make this rule safe even when a tag is missing? 
`InjectRuntimeFilter` always wraps both inputs in `XxHash64`: 
`bloom_filter_agg(xxhash64(key))` and `might_contain(..., xxhash64(key))`. If 
this rule only rewrote expressions of that shape, user-written calls would stay 
on vanilla Spark even if their tag were lost.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to