(parquet-java) branch master updated: GH-3639: Don't drop row groups for IN(..., null) when num_nulls is not set (#3640)

[gangwu]

This is an automated email from the ASF dual-hosted git repository.

wgtmac pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/parquet-java.git


The following commit(s) were added to refs/heads/master by this push:
     new 1e3338484 GH-3639: Don't drop row groups for IN(..., null) when 
num_nulls is not set (#3640)
1e3338484 is described below

commit 1e3338484eebe41d80d86e63da4428eb91c4d416
Author: Eduard Tudenhoefner <etudenhoef...@gmail.com>
AuthorDate: Wed Jul 8 15:38:43 2026 +0200

    GH-3639: Don't drop row groups for IN(..., null) when num_nulls is not set 
(#3640)
---
 .../filter2/statisticslevel/StatisticsFilter.java  |  5 +++
 .../statisticslevel/TestStatisticsFilter.java      | 40 ++++++++++++++++++++++
 2 files changed, 45 insertions(+)

diff --git 
a/parquet-hadoop/src/main/java/org/apache/parquet/filter2/statisticslevel/StatisticsFilter.java
 
b/parquet-hadoop/src/main/java/org/apache/parquet/filter2/statisticslevel/StatisticsFilter.java
index 2574c8ea3..d06ea825d 100644
--- 
a/parquet-hadoop/src/main/java/org/apache/parquet/filter2/statisticslevel/StatisticsFilter.java
+++ 
b/parquet-hadoop/src/main/java/org/apache/parquet/filter2/statisticslevel/StatisticsFilter.java
@@ -305,6 +305,11 @@ public class StatisticsFilter implements 
FilterPredicate.Visitor<Boolean> {
       } else {
         if (values.contains(null)) return BLOCK_MIGHT_MATCH;
       }
+    } else if (values.contains(null)) {
+      // the number of nulls is unknown, so this chunk might contain nulls 
that match the null
+      // literal in the IN set. we cannot fall through to the min/max check 
(which only considers
+      // the non-null values) or we might incorrectly drop a chunk containing 
matching null rows.
+      return BLOCK_MIGHT_MATCH;
     }
 
     // If any value in the IN set is NaN, be conservative
diff --git 
a/parquet-hadoop/src/test/java/org/apache/parquet/filter2/statisticslevel/TestStatisticsFilter.java
 
b/parquet-hadoop/src/test/java/org/apache/parquet/filter2/statisticslevel/TestStatisticsFilter.java
index e9756fa82..3cbccb13c 100644
--- 
a/parquet-hadoop/src/test/java/org/apache/parquet/filter2/statisticslevel/TestStatisticsFilter.java
+++ 
b/parquet-hadoop/src/test/java/org/apache/parquet/filter2/statisticslevel/TestStatisticsFilter.java
@@ -37,6 +37,7 @@ import static 
org.apache.parquet.filter2.predicate.FilterApi.or;
 import static org.apache.parquet.filter2.predicate.FilterApi.userDefined;
 import static 
org.apache.parquet.filter2.statisticslevel.StatisticsFilter.canDrop;
 import static org.apache.parquet.io.api.Binary.fromString;
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -389,6 +390,45 @@ public class TestStatisticsFilter {
         List.of(getIntColumnMeta(statsSomeNulls, 177L), 
getDoubleColumnMeta(doubleStats, 177L))));
   }
 
+  @Test
+  public void testInWithNullLiteralAndUnsetNumNulls() {
+    // Reproduces the bug where StatisticsFilter drops a row group for IN 
(..., null) when num_nulls
+    // is unset. min/max are present but the number of nulls is unknown, so we 
must not fall through
+    // to the min/max-only check (which only considers the non-null literals) 
and drop a chunk that
+    // may contain matching null rows
+    org.apache.parquet.column.statistics.Statistics<?> statsUnsetNulls =
+        org.apache.parquet.column.statistics.Statistics.getBuilderForReading(
+                Types.required(PrimitiveTypeName.INT32).named("test_int32"))
+            .withMin(BytesUtils.intToBytes(10))
+            .withMax(BytesUtils.intToBytes(100))
+            .build();
+    // min/max are available but num_nulls is not
+    assertThat(statsUnsetNulls.hasNonNullValue()).isTrue();
+    assertThat(statsUnsetNulls.isNumNullsSet()).isFalse();
+
+    List<ColumnChunkMetaData> metas =
+        List.of(getIntColumnMeta(statsUnsetNulls, 177L), 
getDoubleColumnMeta(doubleStats, 177L));
+
+    // IN (200, null) where 200 is outside [10, 100]. The chunk might contain 
null rows matching the
+    // null literal, so it must NOT be dropped
+    Set<Integer> valuesNullAndOutOfRange = new HashSet<>();
+    valuesNullAndOutOfRange.add(null);
+    valuesNullAndOutOfRange.add(200);
+    assertThat(canDrop(in(intColumn, valuesNullAndOutOfRange), 
metas)).isFalse();
+
+    // IN (200) without a null literal can still be dropped based on min/max 
even if num_nulls is
+    // unknown, confirming the fix does not over-broaden pruning.
+    Set<Integer> valuesOutOfRange = new HashSet<>();
+    valuesOutOfRange.add(200);
+    assertThat(canDrop(in(intColumn, valuesOutOfRange), metas)).isTrue();
+
+    // IN (50, null) where 50 is inside [10, 100] must also not be dropped.
+    Set<Integer> valuesNullAndInRange = new HashSet<>();
+    valuesNullAndInRange.add(null);
+    valuesNullAndInRange.add(50);
+    assertThat(canDrop(in(intColumn, valuesNullAndInRange), metas)).isFalse();
+  }
+
   @Test
   public void testContainsEqNonNull() {
     assertTrue(canDrop(contains(eq(intColumn, 9)), columnMetas));