This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new ce77b20e71 [fix](optimizer) Add table_statistics back (#24398)
ce77b20e71 is described below
commit ce77b20e713aafa9d37644e3b482ac8550fa2bb8
Author: AKIRA <[email protected]>
AuthorDate: Thu Sep 14 22:03:59 2023 +0900
[fix](optimizer) Add table_statistics back (#24398)
1. Add table_statistics back
2. fix 'col stats is null' exception
---
.../doris/catalog/InternalSchemaInitializer.java | 40 +++++++++++++++++++++-
.../java/org/apache/doris/qe/StmtExecutor.java | 12 ++++---
.../apache/doris/statistics/BaseAnalysisTask.java | 3 ++
.../apache/doris/statistics/ColumnStatistic.java | 6 ++--
.../doris/statistics/StatisticConstants.java | 2 +-
5 files changed, 55 insertions(+), 8 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java
index b7fdec73f0..fd42de00f7 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java
@@ -83,6 +83,7 @@ public class InternalSchemaInitializer extends Thread {
return;
}
Database database = op.get();
+ modifyTblReplicaCount(database, StatisticConstants.ANALYSIS_TBL_NAME);
modifyTblReplicaCount(database, StatisticConstants.STATISTIC_TBL_NAME);
modifyTblReplicaCount(database, StatisticConstants.HISTOGRAM_TBL_NAME);
}
@@ -125,6 +126,7 @@ public class InternalSchemaInitializer extends Thread {
}
private void createTbl() throws UserException {
+
Env.getCurrentEnv().getInternalCatalog().createTable(buildAnalysisTblStmt());
Env.getCurrentEnv().getInternalCatalog().createTable(buildStatisticsTblStmt());
Env.getCurrentEnv().getInternalCatalog().createTable(buildHistogramTblStmt());
}
@@ -143,6 +145,41 @@ public class InternalSchemaInitializer extends Thread {
}
}
+ @VisibleForTesting
+ public CreateTableStmt buildAnalysisTblStmt() throws UserException {
+ TableName tableName = new TableName("",
+ FeConstants.INTERNAL_DB_NAME,
StatisticConstants.ANALYSIS_TBL_NAME);
+ List<ColumnDef> columnDefs = new ArrayList<>();
+ columnDefs.add(new ColumnDef("id",
TypeDef.createVarchar(StatisticConstants.ID_LEN)));
+ columnDefs.add(new ColumnDef("catalog_id",
TypeDef.createVarchar(StatisticConstants.MAX_NAME_LEN)));
+ columnDefs.add(new ColumnDef("db_id",
TypeDef.createVarchar(StatisticConstants.MAX_NAME_LEN)));
+ columnDefs.add(new ColumnDef("tbl_id",
TypeDef.createVarchar(StatisticConstants.MAX_NAME_LEN)));
+ columnDefs.add(new ColumnDef("idx_id",
TypeDef.createVarchar(StatisticConstants.MAX_NAME_LEN)));
+ ColumnDef partId = new ColumnDef("part_id",
TypeDef.createVarchar(StatisticConstants.MAX_NAME_LEN));
+ partId.setAllowNull(true);
+ columnDefs.add(partId);
+ columnDefs.add(new ColumnDef("count",
TypeDef.create(PrimitiveType.BIGINT)));
+ columnDefs.add(new ColumnDef("last_analyze_time_in_ms",
TypeDef.create(PrimitiveType.BIGINT)));
+ columnDefs.add(new ColumnDef("update_time",
TypeDef.create(PrimitiveType.DATETIME)));
+ String engineName = "olap";
+ ArrayList<String> uniqueKeys = Lists.newArrayList("id", "catalog_id",
+ "db_id", "tbl_id", "idx_id", "part_id");
+ KeysDesc keysDesc = new KeysDesc(KeysType.UNIQUE_KEYS, uniqueKeys);
+ DistributionDesc distributionDesc = new HashDistributionDesc(
+ StatisticConstants.STATISTIC_TABLE_BUCKET_COUNT, uniqueKeys);
+ Map<String, String> properties = new HashMap<String, String>() {
+ {
+ put("replication_num", String.valueOf(
+ Math.max(1, Config.min_replication_num_per_tablet)));
+ }
+ };
+ CreateTableStmt createTableStmt = new CreateTableStmt(true, false,
+ tableName, columnDefs, engineName, keysDesc, null,
distributionDesc,
+ properties, null, "Doris internal statistics table, DO NOT
MODIFY IT", null);
+ StatisticsUtil.analyze(createTableStmt);
+ return createTableStmt;
+ }
+
@VisibleForTesting
public CreateTableStmt buildStatisticsTblStmt() throws UserException {
TableName tableName = new TableName("",
@@ -244,7 +281,8 @@ public class InternalSchemaInitializer extends Thread {
}
return false;
}
- return db.getTable(StatisticConstants.HISTOGRAM_TBL_NAME).isPresent();
+ return db.getTable(StatisticConstants.HISTOGRAM_TBL_NAME).isPresent()
+ &&
db.getTable(StatisticConstants.ANALYSIS_TBL_NAME).isPresent();
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
index cc1ed2dac2..7203acd577 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
@@ -2480,7 +2480,8 @@ public class StmtExecutor {
analyze(context.getSessionVariable().toThrift());
}
} catch (Exception e) {
- throw new RuntimeException("Failed to execute internal SQL. "
+ Util.getRootCauseMessage(e), e);
+ throw new RuntimeException("Failed to execute internal SQL. "
+ + Util.getRootCauseMessage(e) + " " +
originStmt.toString(), e);
}
planner.getFragments();
RowBatch batch;
@@ -2490,7 +2491,8 @@ public class StmtExecutor {
QeProcessorImpl.INSTANCE.registerQuery(context.queryId(),
new QeProcessorImpl.QueryInfo(context,
originStmt.originStmt, coord));
} catch (UserException e) {
- throw new RuntimeException("Failed to execute internal SQL. "
+ Util.getRootCauseMessage(e), e);
+ throw new RuntimeException("Failed to execute internal SQL. "
+ + " " + Util.getRootCauseMessage(e) +
originStmt.toString(), e);
}
Span queryScheduleSpan = context.getTracer()
@@ -2499,7 +2501,8 @@ public class StmtExecutor {
coord.exec();
} catch (Exception e) {
queryScheduleSpan.recordException(e);
- throw new RuntimeException("Failed to execute internal SQL. "
+ Util.getRootCauseMessage(e), e);
+ throw new RuntimeException("Failed to execute internal SQL. "
+ + Util.getRootCauseMessage(e) + " " +
originStmt.toString(), e);
} finally {
queryScheduleSpan.end();
}
@@ -2516,7 +2519,8 @@ public class StmtExecutor {
}
} catch (Exception e) {
fetchResultSpan.recordException(e);
- throw new RuntimeException("Failed to execute internal SQL. "
+ Util.getRootCauseMessage(e), e);
+ throw new RuntimeException("Failed to execute internal SQL. "
+ Util.getRootCauseMessage(e) + " "
+ + originStmt.toString(), e);
} finally {
fetchResultSpan.end();
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
index 5b5fcdd9c7..719df8769a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
@@ -178,6 +178,9 @@ public abstract class BaseAnalysisTask {
public abstract void doExecute() throws Exception;
protected void afterExecution() {
+ if (killed) {
+ return;
+ }
Env.getCurrentEnv().getStatisticsCache().syncLoadColStats(tbl.getId(),
-1, col.getName());
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
index 3f928ee72a..7986cb07a5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatistic.java
@@ -26,7 +26,6 @@ import org.apache.doris.common.DdlException;
import org.apache.doris.statistics.util.InternalQueryResult.ResultRow;
import org.apache.doris.statistics.util.StatisticsUtil;
-import com.google.common.base.Preconditions;
import com.google.gson.annotations.SerializedName;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -160,7 +159,10 @@ public class ColumnStatistic {
LOG.debug("Failed to deserialize column stats", t);
return ColumnStatistic.UNKNOWN;
}
- Preconditions.checkState(columnStatistic != null, "Column stats is
null");
+ // Means last analyze failed or interrupted for some reason.
+ if (columnStatistic == null) {
+ return ColumnStatistic.UNKNOWN;
+ }
columnStatistic.partitionIdToColStats.putAll(partitionIdToColStats);
return columnStatistic;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java
index 6ce033a251..a219483403 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java
@@ -77,7 +77,7 @@ public class StatisticConstants {
public static final int STATISTIC_INTERNAL_TABLE_REPLICA_NUM = 3;
- public static int ANALYZE_TASK_RETRY_TIMES = 5;
+ public static int ANALYZE_TASK_RETRY_TIMES = 3;
static {
STATISTICS_DB_BLACK_LIST.add(SystemInfoService.DEFAULT_CLUSTER
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
