(hive) branch master updated: HIVE-29365: Range predicate within the same histogram bucket leads to an estimate rowcount=1 (#6712)

[krisztiankasa]

This is an automated email from the ASF dual-hosted git repository.

kasakrisz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new cb2277209b2 HIVE-29365: Range predicate within the same histogram 
bucket leads to an estimate rowcount=1 (#6712)
cb2277209b2 is described below

commit cb2277209b242c45a2bcae38f49a7aaa9986b08d
Author: Thomas Rebele <thomasreb...@users.noreply.github.com>
AuthorDate: Tue Sep 1 14:09:09 2026 +0200

    HIVE-29365: Range predicate within the same histogram bucket leads to an 
estimate rowcount=1 (#6712)
---
 .../calcite/stats/FilterSelectivityEstimator.java    | 12 ++++++++----
 .../stats/TestFilterSelectivityEstimator.java        | 20 ++++++++++++++++++++
 2 files changed, 28 insertions(+), 4 deletions(-)

diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java
 
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java
index 40651032e5a..5506edde48f 100644
--- 
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java
@@ -937,12 +937,16 @@ private static Range<Float> 
convertRangeToClosedOpen(Range<Float> boundaries) {
    * @return the selectivity of "val1 &lt;= column &lt; val2"
    */
   static double rangedSelectivity(KllFloatsSketch kll, float val1, float val2) 
{
-    if (val1 >= val2) {
+    if (val1 >= val2 || val2 <= kll.getMinItem() || val1 > kll.getMaxItem()) {
       return 0;
     }
     float[] splitPoints = new float[] { val1, val2 };
     double[] boundaries = kll.getCDF(splitPoints, 
QuantileSearchCriteria.EXCLUSIVE);
-    return boundaries[1] - boundaries[0];
+    // due to the way the KLL sketch is constructed,
+    // it is not possible to differentiate selectivity values below the error
+    // (e.g., if the error is 2%, a real selectivity of 1.5% might be 
estimated as 0% by KLL)
+    double normalizedRankError = kll.getNormalizedRankError(false);
+    return Math.max(boundaries[1] - boundaries[0], normalizedRankError);
   }
 
   /**
@@ -1015,8 +1019,8 @@ public static double lessThanSelectivity(KllFloatsSketch 
kll, float value) {
    * @throws IllegalArgumentException if leftValue is equal to rightValue
    */
   public static double betweenSelectivity(KllFloatsSketch kll, float 
leftValue, float rightValue) {
-    // column >= leftValue AND column <= rightValue
-    if (rightValue < leftValue) {
+    // is it possible to fulfill the BETWEEN?
+    if (rightValue < leftValue || rightValue < kll.getMinItem() || leftValue > 
kll.getMaxItem()) {
       return 0;
     }
     if (Double.compare(leftValue, rightValue) == 0) {
diff --git 
a/ql/src/test/org/apache/hadoop/hive/ql/optimizer/calcite/stats/TestFilterSelectivityEstimator.java
 
b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/calcite/stats/TestFilterSelectivityEstimator.java
index 1736d257402..064bc464f6c 100644
--- 
a/ql/src/test/org/apache/hadoop/hive/ql/optimizer/calcite/stats/TestFilterSelectivityEstimator.java
+++ 
b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/calcite/stats/TestFilterSelectivityEstimator.java
@@ -130,6 +130,7 @@ public class TestFilterSelectivityEstimator {
   private static final KllFloatsSketch KLL3 = 
StatisticsTestUtils.createKll(VALUES3);
   private static final KllFloatsSketch KLL_TIME = 
StatisticsTestUtils.createKll(VALUES_TIME);
   private static final float DELTA = 1e-7f;
+  private static final double MIN_KLL_SELECTIVITY = 0.013294757464848584;
   private static final RexBuilder REX_BUILDER = new RexBuilder(new 
JavaTypeFactoryImpl(new HiveTypeSystemImpl()));
   private static final RelDataTypeFactory TYPE_FACTORY = 
REX_BUILDER.getTypeFactory();
 
@@ -272,6 +273,25 @@ public void testIsHistogramAvailableWhenEmptyArray() {
     Assert.assertFalse(isHistogramAvailable(colStatistics));
   }
 
+  /**
+   * Check the KLL resolution.
+   * <p>
+   * The resolution may not be lower than the minimum rank error [HIVE-29365].
+   * </p> */
+  @Test
+  public void testKllResolution() {
+    Assert.assertEquals(MIN_KLL_SELECTIVITY, betweenSelectivity(KLL, 1.20f, 
1.21f), DELTA);
+
+    
doReturn(Collections.singletonList(stats)).when(tableMock).getColStat(Collections.singletonList(0));
+    RexNode filter = REX_BUILDER.makeCall(SqlStdOperatorTable.AND,
+        REX_BUILDER.makeCall(SqlStdOperatorTable.GREATER_THAN, inputRef0, 
literalFloat(1.20f)),
+        REX_BUILDER.makeCall(SqlStdOperatorTable.LESS_THAN, inputRef0, 
literalFloat(1.21f)));
+    filter = simplify(filter);
+    Assert.assertEquals(SqlKind.SEARCH, filter.getKind());
+    FilterSelectivityEstimator estimator = new 
FilterSelectivityEstimator(scan, mq);
+    Assert.assertEquals(MIN_KLL_SELECTIVITY, 
estimator.estimateSelectivity(filter), DELTA);
+  }
+
   @Test
   public void testLessThanSelectivity() {
     Assert.assertEquals(0.6153846153846154, lessThanSelectivity(KLL, 3), 
DELTA);