This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new a19d0f788eb branch-4.1: [test](nereids) Replace flaky null stats case
with unit test (#65656)
a19d0f788eb is described below
commit a19d0f788eb9529e61991061e93058c7da15f42d
Author: shuke <[email protected]>
AuthorDate: Thu Jul 16 11:03:35 2026 +0800
branch-4.1: [test](nereids) Replace flaky null stats case with unit test
(#65656)
### What problem does this PR solve?
Related PR: #65445
Problem Summary:
Backport #65445 to branch-4.1.
The regression case `test_scale_num_nulls` depends on asynchronously
visible physical row counts. When the physical row count is still 0,
Nereids scales the selected-partition column statistics to 0 even though
the injected partition/table statistics are correct, making the case
flaky.
Replace the environment-dependent Groovy regression case with a
deterministic FE unit test that directly constructs the partition/table
statistics and verifies selected-partition scaling, including row count,
NDV, min/max, count, and numNulls.
### Coverage rationale
- This is a branch backport of merged PR #65445. The same planner-level
coverage concern was raised in [the upstream review
thread](https://github.com/apache/doris/pull/65445#discussion_r3556961223),
then explicitly resolved by a maintainer before the PR was approved and
merged.
- The independent behavior introduced with #62265 is selected-partition
statistics scaling. The replacement test invokes the real
`StatsCalculator.computeOlapScan()` path and verifies row count, NDV,
min/max, count, and `numNulls`. SQL predicate-to-partition selection
remains covered by `PruneOlapScanPartitionTest` on this release branch.
- The removed cluster case sampled the strict physical partition row
count immediately after insert/analyze. In Cloud P0 that value can
remain 0 for up to two minutes even when table-level column statistics
are correct. Retaining the case or waiting for that report would
preserve the flake or add up to two minutes to the suite.
- `SHOW COLUMN STATS` was only failure diagnostics, and memo text
formatting was not an independent assertion target of this case.
### Release note
None
### Check List (For Author)
- Test
- [ ] Regression test
- [x] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason
Local validation:
-
`StatsCalculatorTest#testComputeOlapScanScalesNumNullsForSelectedPartitions`:
passed (1 test)
- `StatsCalculatorTest`: passed (17 tests)
- `mvn -pl fe-core -am checkstyle:check`: passed (0 violations)
- Behavior changed:
- [x] No.
- [ ] Yes.
- Does this need documentation?
- [x] No.
- [ ] Yes.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
---
.../doris/nereids/stats/StatsCalculatorTest.java | 76 ++++++++++++++++++++++
.../suites/statistics/test_scale_num_nulls.groovy | 60 -----------------
2 files changed, 76 insertions(+), 60 deletions(-)
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java
index 075e3731977..357d2514fb8 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java
@@ -17,8 +17,11 @@
package org.apache.doris.nereids.stats;
+import org.apache.doris.analysis.IntLiteral;
import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.CascadesContext;
@@ -53,9 +56,11 @@ import org.apache.doris.nereids.util.MemoTestUtils;
import org.apache.doris.nereids.util.PlanConstructor;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.SessionVariable;
+import org.apache.doris.statistics.AnalysisManager;
import org.apache.doris.statistics.ColumnStatistic;
import org.apache.doris.statistics.ColumnStatisticBuilder;
import org.apache.doris.statistics.Statistics;
+import org.apache.doris.statistics.StatisticsCache;
import org.apache.doris.statistics.TableStatsMeta;
import com.google.common.collect.ImmutableList;
@@ -66,6 +71,8 @@ import mockit.Mock;
import mockit.MockUp;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
import java.io.IOException;
import java.util.ArrayList;
@@ -207,6 +214,75 @@ public class StatsCalculatorTest {
Assertions.assertNotNull(stats.columnStatistics().get(slot1));
}
+ @Test
+ public void testComputeOlapScanScalesNumNullsForSelectedPartitions() {
+ ConnectContext previousContext = ConnectContext.get();
+ ConnectContext connectContext = new ConnectContext();
+ connectContext.setThreadLocalInfo();
+
+ Env env = Mockito.mock(Env.class);
+ AnalysisManager analysisManager = Mockito.mock(AnalysisManager.class);
+ StatisticsCache statisticsCache = Mockito.mock(StatisticsCache.class);
+ StatisticsCache.OlapTableStatistics olapTableStatistics
+ = Mockito.mock(StatisticsCache.OlapTableStatistics.class);
+ OlapTable table = Mockito.mock(OlapTable.class);
+ LogicalOlapScan scan = Mockito.mock(LogicalOlapScan.class);
+ Partition selectedPartition = Mockito.mock(Partition.class);
+
+ long selectedPartitionId = 1L;
+ long baseIndexId = 10L;
+ Column column = new Column("val", PrimitiveType.INT);
+ SlotReference slot = new SlotReference(new ExprId(1), "val",
IntegerType.INSTANCE, true,
+ ImmutableList.of("test", "tbl"), table, column, table, column);
+ ColumnStatistic tableStatistic = new ColumnStatisticBuilder(12)
+ .setNdv(1)
+ .setMinValue(2)
+ .setMaxValue(2)
+ .setMinExpr(new IntLiteral(2))
+ .setMaxExpr(new IntLiteral(2))
+ .setNumNulls(3)
+ .build();
+
+ Mockito.when(env.getAnalysisManager()).thenReturn(analysisManager);
+ Mockito.when(env.getStatisticsCache()).thenReturn(statisticsCache);
+
Mockito.when(statisticsCache.getOlapTableStats(scan)).thenReturn(olapTableStatistics);
+ Mockito.when(olapTableStatistics.getColumnStatistics("val",
connectContext)).thenReturn(tableStatistic);
+ Mockito.when(scan.getTable()).thenReturn(table);
+ Mockito.when(scan.getSelectedIndexId()).thenReturn(baseIndexId);
+
Mockito.when(scan.getSelectedPartitionIds()).thenReturn(ImmutableList.of(selectedPartitionId));
+ Mockito.when(scan.getOutput()).thenReturn(ImmutableList.of(slot));
+ Mockito.when(scan.getVirtualColumns()).thenReturn(ImmutableList.of());
+ Mockito.when(table.getBaseIndexId()).thenReturn(baseIndexId);
+ Mockito.when(table.getRowCountForIndex(baseIndexId,
true)).thenReturn(12L);
+ Mockito.when(table.getRowCountForPartitionIndex(selectedPartitionId,
baseIndexId, true)).thenReturn(4L);
+ Mockito.when(table.getPartitionNum()).thenReturn(3);
+
Mockito.when(table.getPartition(selectedPartitionId)).thenReturn(selectedPartition);
+ Mockito.when(table.getQualifiedDbName()).thenReturn("test");
+ Mockito.when(selectedPartition.getName()).thenReturn("p1");
+
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+
+ Statistics statistics = new StatsCalculator((CascadesContext)
null).computeOlapScan(scan);
+ ColumnStatistic result = statistics.findColumnStatistics(slot);
+
+ Assertions.assertEquals(4, statistics.getRowCount(), 0.001);
+ Assertions.assertNotNull(result);
+ Assertions.assertEquals(1, result.ndv, 0.001);
+ Assertions.assertEquals(2, result.minValue, 0.001);
+ Assertions.assertEquals(2, result.maxValue, 0.001);
+ Assertions.assertEquals(4, result.count, 0.001);
+ Assertions.assertEquals(1, result.numNulls, 0.001);
+ Assertions.assertEquals("2", result.minExpr.getStringValue());
+ Assertions.assertEquals("2", result.maxExpr.getStringValue());
+ } finally {
+ ConnectContext.remove();
+ if (previousContext != null) {
+ previousContext.setThreadLocalInfo();
+ }
+ }
+ }
+
@Test
public void testLimit() {
List<String> qualifier = ImmutableList.of("test", "t");
diff --git a/regression-test/suites/statistics/test_scale_num_nulls.groovy
b/regression-test/suites/statistics/test_scale_num_nulls.groovy
deleted file mode 100644
index 043b072ca62..00000000000
--- a/regression-test/suites/statistics/test_scale_num_nulls.groovy
+++ /dev/null
@@ -1,60 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-suite("test_scale_num_nulls") {
- // For an OlapTable, when only a subset of partitions is selected,
- // the num_nulls value in column statistics needs to be scaled
proportionally.
- sql """
- drop table if exists ptable;
- CREATE TABLE `ptable` (
- `id` int NULL,
- `val` int NULL,
- `d` date NULL
- ) ENGINE=OLAP
- DUPLICATE KEY(`id`)
- PARTITION BY RANGE(`d`)
- (PARTITION p201701 VALUES [('2017-01-01'), ('2017-02-01')),
- PARTITION p201702 VALUES [('2017-02-01'), ('2017-03-01')),
- PARTITION p201703 VALUES [('2017-03-01'), ('2017-04-01')))
- DISTRIBUTED BY HASH(`id`) BUCKETS 16
- PROPERTIES (
- "replication_allocation" = "tag.location.default: 1",
- "min_load_replica_num" = "-1",
- "is_being_synced" = "false",
- "storage_medium" = "hdd",
- "storage_format" = "V2",
- "inverted_index_storage_format" = "V3",
- "light_schema_change" = "true",
- "disable_auto_compaction" = "false",
- "group_commit_interval_ms" = "10000",
- "group_commit_data_bytes" = "134217728"
- );
-
- insert into ptable values
- (1, null, '2017-01-01'), (11,2, '2017-01-01'), (111,2, '2017-01-01'),
(1111,2, '2017-01-01'),
- (2, null, '2017-02-01'), (22,2, '2017-02-01'), (222,2, '2017-02-01'),
(2222,2, '2017-02-01'),
- (3, null, '2017-03-01'), (33,2, '2017-03-01'), (333,2, '2017-03-01'),
(333,2, '2017-03-01');
-
- analyze table ptable with sync;
- """
- def colStats = sql "show column stats ptable";
- def memo = sql "explain memo plan select * from ptable where
d='2017-01-01'"
- // check numNulls=1.0000 for column val, which is scaled from 3 to 1
according to the partition pruning result.
- assertTrue(memo.toString().contains("val#1 -> ndv=1.0000,
min=2.000000(2), max=2.000000(2), count=4.0000, numNulls=1.0000"),
- "numNulls should be 1.0000, but is " + memo.toString() + "\n
column stats: \n" + colStats.toString());
-}
-
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
