This is an automated email from the ASF dual-hosted git repository.
lijibing pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 3e4b52937ea [improvement](statistics)Analyze table if it hasn't been
analyzed for longer than 1 day. (#38615)
3e4b52937ea is described below
commit 3e4b52937ea97aa9a9b59d9cf428f94e89a6394a
Author: Jibing-Li <[email protected]>
AuthorDate: Fri Aug 2 18:43:17 2024 +0800
[improvement](statistics)Analyze table if it hasn't been analyzed for
longer than 1 day. (#38615)
Need to reanalyze a table when it hasn't been analyzed for longer than 1
day.
Set enableMaterializedViewRewrite to false for statistics session.
---
.../main/java/org/apache/doris/common/Config.java | 7 ++++++
.../doris/statistics/util/StatisticsUtil.java | 7 ++++++
.../doris/statistics/util/StatisticsUtilTest.java | 28 ++++++++++++++++++----
3 files changed, 37 insertions(+), 5 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 7495b0db9ec..e67fd0a525c 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2730,6 +2730,13 @@ public class Config extends ConfigBase {
"SHould abort txn by checking conflick txn in schema change"})
public static boolean enable_abort_txn_by_checking_conflict_txn = true;
+ @ConfField(mutable = true, description = {
+ "内表自动收集时间间隔，当某一列上次收集时间距离当前时间大于该值，则会触发一次新的收集，0表示不会触发。",
+ "Columns that have not been collected within the specified
interval will trigger automatic analyze. "
+ + "0 means not trigger."
+ })
+ public static long auto_analyze_interval_seconds = 0;
+
//==========================================================================
// begin of cloud config
//==========================================================================
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
index 45b7454adff..7937040d323 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
@@ -224,6 +224,7 @@ public class StatisticsUtil {
sessionVariable.enablePushDownMinMaxOnUnique = true;
sessionVariable.enablePushDownStringMinMax = true;
sessionVariable.enableUniqueKeyPartialUpdate = false;
+ sessionVariable.enableMaterializedViewRewrite = false;
connectContext.setEnv(Env.getCurrentEnv());
connectContext.setDatabase(FeConstants.INTERNAL_DB_NAME);
connectContext.setQualifiedUser(UserIdentity.ROOT.getQualifiedUser());
@@ -1003,6 +1004,12 @@ public class StatisticsUtil {
if (columnStatsMeta == null) {
return true;
}
+ // Column hasn't been analyzed for longer than config interval.
+ if (Config.auto_analyze_interval_seconds > 0
+ && System.currentTimeMillis() - columnStatsMeta.updatedTime
+ > Config.auto_analyze_interval_seconds * 1000) {
+ return true;
+ }
// Partition table partition stats never been collected.
if (StatisticsUtil.enablePartitionAnalyze() &&
table.isPartitionedTable()
&& columnStatsMeta.partitionUpdateRows == null) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java
index 90abe3fca17..6dccd3af46e 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java
@@ -24,6 +24,7 @@ import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.Pair;
import org.apache.doris.datasource.CatalogIf;
@@ -231,10 +232,27 @@ class StatisticsUtilTest {
tableMeta.userInjected = false;
Assertions.assertTrue(StatisticsUtil.needAnalyzeColumn(table,
Pair.of("index", column.getName())));
+ // Test column hasn't been analyzed for longer than 1 day.
new MockUp<TableStatsMeta>() {
@Mock
public ColStatsMeta findColumnStatsMeta(String indexName, String
colName) {
- return new ColStatsMeta(0, null, null, null, 0, 0, 0, null);
+ return new ColStatsMeta(0, null, null, null, 0, 100, 0, null);
+ }
+ };
+ new MockUp<OlapTable>() {
+ @Mock
+ public long getRowCount() {
+ return 100;
+ }
+ };
+ Config.auto_analyze_interval_seconds = 60 * 60 * 24;
+ Assertions.assertTrue(StatisticsUtil.needAnalyzeColumn(table,
Pair.of("index", column.getName())));
+ Config.auto_analyze_interval_seconds = 0;
+
+ new MockUp<TableStatsMeta>() {
+ @Mock
+ public ColStatsMeta findColumnStatsMeta(String indexName, String
colName) {
+ return new ColStatsMeta(System.currentTimeMillis(), null,
null, null, 0, 0, 0, null);
}
};
@@ -282,7 +300,7 @@ class StatisticsUtilTest {
new MockUp<TableStatsMeta>() {
@Mock
public ColStatsMeta findColumnStatsMeta(String indexName, String
colName) {
- return new ColStatsMeta(0, null, null, null, 0, 100, 0, null);
+ return new ColStatsMeta(System.currentTimeMillis(), null,
null, null, 0, 100, 0, null);
}
};
tableMeta.partitionChanged.set(false);
@@ -292,7 +310,7 @@ class StatisticsUtilTest {
new MockUp<TableStatsMeta>() {
@Mock
public ColStatsMeta findColumnStatsMeta(String indexName, String
colName) {
- return new ColStatsMeta(0, null, null, null, 0, 0, 0, null);
+ return new ColStatsMeta(System.currentTimeMillis(), null,
null, null, 0, 0, 0, null);
}
};
tableMeta.partitionChanged.set(false);
@@ -308,7 +326,7 @@ class StatisticsUtilTest {
new MockUp<TableStatsMeta>() {
@Mock
public ColStatsMeta findColumnStatsMeta(String indexName, String
colName) {
- return new ColStatsMeta(0, null, null, null, 0, 500, 0, null);
+ return new ColStatsMeta(System.currentTimeMillis(), null,
null, null, 0, 500, 0, null);
}
};
tableMeta.partitionChanged.set(false);
@@ -324,7 +342,7 @@ class StatisticsUtilTest {
new MockUp<TableStatsMeta>() {
@Mock
public ColStatsMeta findColumnStatsMeta(String indexName, String
colName) {
- return new ColStatsMeta(0, null, null, null, 0, 100, 80, null);
+ return new ColStatsMeta(System.currentTimeMillis(), null,
null, null, 0, 100, 80, null);
}
};
tableMeta.partitionChanged.set(false);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
