This is an automated email from the ASF dual-hosted git repository.
englefly pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new c0d55b54c0 [Improvement](statistics) Support for statistics removing
and incremental collection (#18069)
c0d55b54c0 is described below
commit c0d55b54c047d0bab64ad67701858ad56748a22a
Author: ElvinWei <[email protected]>
AuthorDate: Thu Mar 30 10:40:43 2023 +0800
[Improvement](statistics) Support for statistics removing and incremental
collection (#18069)
* Support for removing statistics and incremental collection
* Fix syntax
---
fe/fe-core/src/main/cup/sql_parser.cup | 22 +++-
.../org/apache/doris/analysis/AnalyzeStmt.java | 10 +-
.../apache/doris/analysis/DropTableStatsStmt.java | 135 ++++++++++++++-------
.../doris/catalog/InternalSchemaInitializer.java | 17 ++-
.../main/java/org/apache/doris/qe/DdlExecutor.java | 3 +-
.../apache/doris/statistics/AnalysisManager.java | 16 +++
.../apache/doris/statistics/AnalysisTaskInfo.java | 2 +-
.../org/apache/doris/statistics/HistogramTask.java | 5 +-
.../apache/doris/statistics/OlapAnalysisTask.java | 3 +-
.../doris/statistics/StatisticsRepository.java | 69 +++++++++++
fe/fe-core/src/main/jflex/sql_scanner.flex | 1 +
11 files changed, 215 insertions(+), 68 deletions(-)
diff --git a/fe/fe-core/src/main/cup/sql_parser.cup
b/fe/fe-core/src/main/cup/sql_parser.cup
index c24f59b34e..e70d9b3494 100644
--- a/fe/fe-core/src/main/cup/sql_parser.cup
+++ b/fe/fe-core/src/main/cup/sql_parser.cup
@@ -398,6 +398,7 @@ terminal String
KW_IF,
KW_IMMEDIATE,
KW_IN,
+ KW_INCREMENTAL,
KW_INDEX,
KW_INDEXES,
KW_INFILE,
@@ -2791,17 +2792,26 @@ analyze_stmt ::=
{:
boolean is_whole_tbl = (cols == null);
boolean is_histogram = false;
- RESULT = new AnalyzeStmt(tbl, cols, partitionNames, properties,
is_whole_tbl, is_histogram);
+ boolean is_increment = false;
+ RESULT = new AnalyzeStmt(tbl, cols, partitionNames, properties,
is_whole_tbl, is_histogram, is_increment);
:}
- | KW_ANALYZE KW_TABLE table_name:tbl KW_UPDATE KW_HISTOGRAM KW_ON
ident_list:cols opt_partition_names:partitionNames opt_properties:properties
+ | KW_ANALYZE KW_INCREMENTAL KW_TABLE table_name:tbl opt_col_list:cols
opt_partition_names:partitionNames opt_properties:properties
{:
boolean is_whole_tbl = (cols == null);
+ boolean is_histogram = false;
+ boolean is_increment = true;
+ RESULT = new AnalyzeStmt(tbl, cols, partitionNames, properties,
is_whole_tbl, is_histogram, is_increment);
+ :}
+ | KW_ANALYZE KW_TABLE table_name:tbl KW_UPDATE KW_HISTOGRAM KW_ON
ident_list:cols opt_partition_names:partitionNames opt_properties:properties
+ {:
+ boolean is_whole_tbl = false;
boolean is_histogram = true;
- RESULT = new AnalyzeStmt(tbl, cols, partitionNames, properties,
is_whole_tbl, is_histogram);
+ boolean is_increment = false;
+ RESULT = new AnalyzeStmt(tbl, cols, partitionNames, properties,
is_whole_tbl, is_histogram, is_increment);
:}
| KW_ANALYZE KW_TABLE table_name:tbl KW_UPDATE KW_HISTOGRAM
{:
- RESULT = new AnalyzeStmt(tbl, null, null, new HashMap<>(), true, true);
+ RESULT = new AnalyzeStmt(tbl, null, null, new HashMap<>(), true, true,
false);
:}
;
@@ -2980,9 +2990,9 @@ drop_stmt ::=
RESULT = new DropPolicyStmt(PolicyTypeEnum.STORAGE, ifExists,
policyName, null, null);
:}
/* statistics */
- | KW_DROP KW_STATS opt_table_name:tbl opt_partition_names:partitionNames
+ | KW_DROP KW_STATS opt_table_name:tbl opt_col_list:cols
opt_partition_names:partitionNames
{:
- RESULT = new DropTableStatsStmt(tbl, partitionNames);
+ RESULT = new DropTableStatsStmt(tbl, partitionNames, cols);
:}
;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeStmt.java
index 73dfb14dcc..bdbbc6763f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeStmt.java
@@ -69,6 +69,7 @@ public class AnalyzeStmt extends DdlStmt {
public boolean isWholeTbl;
public boolean isHistogram;
+ public boolean isIncrement;
private final TableName tableName;
private final PartitionNames partitionNames;
@@ -84,13 +85,15 @@ public class AnalyzeStmt extends DdlStmt {
PartitionNames partitionNames,
Map<String, String> properties,
Boolean isWholeTbl,
- Boolean isHistogram) {
+ Boolean isHistogram,
+ Boolean isIncrement) {
this.tableName = tableName;
this.columnNames = columnNames;
this.partitionNames = partitionNames;
this.properties = properties;
this.isWholeTbl = isWholeTbl;
this.isHistogram = isHistogram;
+ this.isIncrement = isIncrement;
}
@Override
@@ -232,6 +235,11 @@ public class AnalyzeStmt extends DdlStmt {
StringBuilder sb = new StringBuilder();
sb.append("ANALYZE");
+ if (isIncrement) {
+ sb.append(" ");
+ sb.append("INCREMENTAL");
+ }
+
if (tableName != null) {
sb.append(" ");
sb.append(tableName.toSql());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropTableStatsStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropTableStatsStmt.java
index b7af4693de..f36da7e021 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropTableStatsStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropTableStatsStmt.java
@@ -18,25 +18,25 @@
package org.apache.doris.analysis;
import org.apache.doris.catalog.Database;
+import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.Table;
+import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.Util;
+import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
+import org.apache.commons.lang.StringUtils;
-import java.util.HashSet;
import java.util.List;
-import java.util.Map;
import java.util.Set;
+import java.util.stream.Collectors;
/**
* Manually drop statistics for tables or partitions.
@@ -48,20 +48,19 @@ import java.util.Set;
*/
public class DropTableStatsStmt extends DdlStmt {
private final TableName tableName;
- private final PartitionNames optPartitionNames;
+ private final PartitionNames partitionNames;
+ private final List<String> columnNames;
// after analyzed
- private final Map<Long, Set<String>> tblIdToPartition = Maps.newHashMap();
+ private long dbId;
+ private final Set<Long> tbIds = Sets.newHashSet();
+ private final Set<Long> partitionIds = Sets.newHashSet();
- public DropTableStatsStmt(TableName tableName, PartitionNames
optPartitionNames) {
+ public DropTableStatsStmt(TableName tableName,
+ PartitionNames partitionNames, List<String> columnNames) {
this.tableName = tableName;
- this.optPartitionNames = optPartitionNames;
- }
-
- public Map<Long, Set<String>> getTblIdToPartition() {
- Preconditions.checkArgument(isAnalyzed(),
- "The partition name must be obtained after the parsing is
complete");
- return tblIdToPartition;
+ this.partitionNames = partitionNames;
+ this.columnNames = columnNames;
}
@Override
@@ -69,50 +68,74 @@ public class DropTableStatsStmt extends DdlStmt {
super.analyze(analyzer);
if (tableName != null) {
- if (Strings.isNullOrEmpty(tableName.getDb())) {
- tableName.setDb(analyzer.getDefaultDb());
- }
-
tableName.analyze(analyzer);
- // check whether the deletion permission is granted
- checkAnalyzePriv(tableName.getDb(), tableName.getTbl());
+ String catalogName = tableName.getCtl();
+ String dbName = tableName.getDb();
+ String tblName = tableName.getTbl();
+ CatalogIf catalog = analyzer.getEnv().getCatalogMgr()
+ .getCatalogOrAnalysisException(catalogName);
+ DatabaseIf db = catalog.getDbOrAnalysisException(dbName);
+ TableIf table = db.getTableOrAnalysisException(tblName);
+
+ dbId = db.getId();
+ tbIds.add(table.getId());
// disallow external catalog
- Util.prohibitExternalCatalog(tableName.getCtl(),
this.getClass().getSimpleName());
+ Util.prohibitExternalCatalog(tableName.getCtl(),
+ this.getClass().getSimpleName());
+
+ // check permission
+ checkAnalyzePriv(db.getFullName(), table.getName());
+
+ // check partitionNames
+ if (partitionNames != null) {
+ partitionNames.analyze(analyzer);
+ partitionIds.addAll(partitionNames.getPartitionNames().stream()
+ .map(name -> table.getPartition(name).getId())
+ .collect(Collectors.toList()));
+ }
- Database db = analyzer.getEnv().getInternalCatalog()
- .getDbOrAnalysisException(tableName.getDb());
- long tableId =
db.getTableOrAnalysisException(tableName.getTbl()).getId();
-
- if (optPartitionNames == null) {
- tblIdToPartition.put(tableId, null);
- } else {
- optPartitionNames.analyze(analyzer);
- List<String> pNames = optPartitionNames.getPartitionNames();
- HashSet<String> partitionNames = Sets.newHashSet(pNames);
- tblIdToPartition.put(tableId, partitionNames);
+ // check columnNames
+ if (columnNames != null) {
+ for (String cName : columnNames) {
+ if (table.getColumn(cName) == null) {
+ ErrorReport.reportAnalysisException(
+ ErrorCode.ERR_WRONG_COLUMN_NAME,
+ "DROP",
+ ConnectContext.get().getQualifiedUser(),
+ ConnectContext.get().getRemoteIP(),
+ cName);
+ }
+ }
}
} else {
Database db = analyzer.getEnv().getInternalCatalog()
.getDbOrAnalysisException(analyzer.getDefaultDb());
- for (Table table : db.getTables()) {
+ List<Table> tables = db.getTables();
+ for (Table table : tables) {
checkAnalyzePriv(db.getFullName(), table.getName());
- tblIdToPartition.put(table.getId(), null);
}
+
+ dbId = db.getId();
+
tbIds.addAll(tables.stream().map(Table::getId).collect(Collectors.toList()));
}
}
- private void checkAnalyzePriv(String dbName, String tblName) throws
AnalysisException {
- if (!Env.getCurrentEnv().getAccessManager()
- .checkTblPriv(ConnectContext.get(), dbName, tblName,
PrivPredicate.DROP)) {
- ErrorReport.reportAnalysisException(
- ErrorCode.ERR_TABLEACCESS_DENIED_ERROR,
- "DROP",
- ConnectContext.get().getQualifiedUser(),
- ConnectContext.get().getRemoteIP(),
- dbName + "." + tblName);
- }
+ public long getDbId() {
+ return dbId;
+ }
+
+ public Set<Long> getTbIds() {
+ return tbIds;
+ }
+
+ public Set<Long> getPartitionIds() {
+ return partitionIds;
+ }
+
+ public Set<String> getColumnNames() {
+ return columnNames != null ? Sets.newHashSet(columnNames) :
Sets.newHashSet();
}
@Override
@@ -124,9 +147,15 @@ public class DropTableStatsStmt extends DdlStmt {
sb.append(tableName.toSql());
}
- if (optPartitionNames != null) {
+ if (columnNames != null) {
+ sb.append("(");
+ sb.append(StringUtils.join(columnNames, ","));
+ sb.append(")");
+ }
+
+ if (partitionNames != null) {
sb.append(" ");
- sb.append(optPartitionNames.toSql());
+ sb.append(partitionNames.toSql());
}
return sb.toString();
@@ -136,4 +165,16 @@ public class DropTableStatsStmt extends DdlStmt {
public String toString() {
return toSql();
}
+
+ private void checkAnalyzePriv(String dbName, String tblName) throws
AnalysisException {
+ if (!Env.getCurrentEnv().getAccessManager()
+ .checkTblPriv(ConnectContext.get(), dbName, tblName,
PrivPredicate.DROP)) {
+ ErrorReport.reportAnalysisException(
+ ErrorCode.ERR_TABLEACCESS_DENIED_ERROR,
+ "DROP",
+ ConnectContext.get().getQualifiedUser(),
+ ConnectContext.get().getRemoteIP(),
+ dbName + "." + tblName);
+ }
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java
index 23d0226d35..6feec883a7 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java
@@ -120,12 +120,11 @@ public class InternalSchemaInitializer extends Thread {
columnDefs.add(new ColumnDef("data_size_in_bytes",
TypeDef.create(PrimitiveType.BIGINT)));
columnDefs.add(new ColumnDef("update_time",
TypeDef.create(PrimitiveType.DATETIME)));
String engineName = "olap";
- KeysDesc keysDesc = new KeysDesc(KeysType.UNIQUE_KEYS,
- Lists.newArrayList("id"));
-
+ ArrayList<String> uniqueKeys = Lists.newArrayList("id", "catalog_id",
+ "db_id", "tbl_id", "idx_id", "col_id", "part_id");
+ KeysDesc keysDesc = new KeysDesc(KeysType.UNIQUE_KEYS, uniqueKeys);
DistributionDesc distributionDesc = new HashDistributionDesc(
- StatisticConstants.STATISTIC_TABLE_BUCKET_COUNT,
- Lists.newArrayList("id"));
+ StatisticConstants.STATISTIC_TABLE_BUCKET_COUNT, uniqueKeys);
Map<String, String> properties = new HashMap<String, String>() {
{
put("replication_num",
String.valueOf(Config.statistic_internal_table_replica_num));
@@ -154,11 +153,11 @@ public class InternalSchemaInitializer extends Thread {
columnDefs.add(new ColumnDef("buckets",
TypeDef.createVarchar(ScalarType.MAX_VARCHAR_LENGTH)));
columnDefs.add(new ColumnDef("update_time",
TypeDef.create(PrimitiveType.DATETIME)));
String engineName = "olap";
- KeysDesc keysDesc = new KeysDesc(KeysType.UNIQUE_KEYS,
- Lists.newArrayList("id"));
+ ArrayList<String> uniqueKeys = Lists.newArrayList("id", "catalog_id",
+ "db_id", "tbl_id", "idx_id", "col_id");
+ KeysDesc keysDesc = new KeysDesc(KeysType.UNIQUE_KEYS, uniqueKeys);
DistributionDesc distributionDesc = new HashDistributionDesc(
- StatisticConstants.STATISTIC_TABLE_BUCKET_COUNT,
- Lists.newArrayList("id"));
+ StatisticConstants.STATISTIC_TABLE_BUCKET_COUNT, uniqueKeys);
Map<String, String> properties = new HashMap<String, String>() {
{
put("replication_num",
String.valueOf(Config.statistic_internal_table_replica_num));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java
index 857f14e652..af82985471 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java
@@ -325,7 +325,8 @@ public class DdlExecutor {
} else if (ddlStmt instanceof CleanProfileStmt) {
ProfileManager.getInstance().cleanProfile();
} else if (ddlStmt instanceof DropTableStatsStmt) {
- // TODO: support later
+ DropTableStatsStmt stmt = (DropTableStatsStmt) ddlStmt;
+ StatisticsRepository.dropTableStatistics(stmt);
} else {
throw new DdlException("Unknown statement.");
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index bc8e944a52..00fab11d31 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -23,6 +23,7 @@ import org.apache.doris.analysis.TableName;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.MaterializedIndexMeta;
import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
@@ -35,6 +36,7 @@ import org.apache.doris.statistics.util.StatisticsUtil;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.text.StringSubstitutor;
import org.apache.log4j.LogManager;
@@ -48,6 +50,7 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import java.util.stream.Collectors;
public class AnalysisManager {
@@ -94,6 +97,19 @@ public class AnalysisManager {
Set<String> partitionNames = analyzeStmt.getPartitionNames();
Map<Long, AnalysisTaskInfo> analysisTaskInfos = new HashMap<>();
long jobId = Env.getCurrentEnv().getNextId();
+
+ // If the analysis is not incremental, need to delete existing
statistics.
+ // we cannot collect histograms incrementally and do not support it
+ if (!analyzeStmt.isIncrement && !analyzeStmt.isHistogram) {
+ long dbId = analyzeStmt.getDbId();
+ TableIf table = analyzeStmt.getTable();
+ Set<Long> tblIds = Sets.newHashSet(table.getId());
+ Set<Long> partIds = partitionNames.stream()
+ .map(p -> table.getPartition(p).getId())
+ .collect(Collectors.toSet());
+ StatisticsRepository.dropStatistics(dbId, tblIds, colNames,
partIds);
+ }
+
if (colNames != null) {
for (String colName : colNames) {
long taskId = Env.getCurrentEnv().getNextId();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskInfo.java
index 8c9acac24f..9f5f608229 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisTaskInfo.java
@@ -74,7 +74,7 @@ public class AnalysisTaskInfo {
public final AnalysisType analysisType;
// TODO: define constants or get them from configuration properties
- public final double sampleRate = 0.2;
+ public final double sampleRate = 1.0;
public final int maxBucketNum = 128;
public String message;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java
index 73cb2d1c3d..32f0f169c9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/HistogramTask.java
@@ -46,7 +46,7 @@ public class HistogramTask extends BaseAnalysisTask {
+ " ${idxId} AS idx_id, "
+ " '${colId}' AS col_id, "
+ " ${sampleRate} AS sample_rate, "
- + " HISTOGRAM(`${colName}`, 1, ${maxBucketNum}) AS buckets, "
+ + " HISTOGRAM(`${colName}`, ${maxBucketNum}) AS buckets, "
+ " NOW() AS create_time "
+ "FROM "
+ " `${dbName}`.`${tblName}`";
@@ -89,8 +89,9 @@ public class HistogramTask extends BaseAnalysisTask {
} else {
try {
tbl.readLock();
- String partNames = info.partitionNames.stream()
+ String partNames = partitionNames.stream()
.filter(x -> tbl.getPartition(x) != null)
+ .map(partName -> "`" + partName + "`")
.collect(Collectors.joining(","));
params.put("partName", partNames);
StringSubstitutor stringSubstitutor = new
StringSubstitutor(params);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
index a8ec452d95..927a78c14f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
@@ -80,7 +80,8 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
continue;
}
params.put("partId",
String.valueOf(tbl.getPartition(partName).getId()));
- params.put("partName", String.valueOf(partName));
+ // Avoid error when get the default partition
+ params.put("partName", "`" + partName + "`");
StringSubstitutor stringSubstitutor = new
StringSubstitutor(params);
partitionAnalysisSQLs.add(stringSubstitutor.replace(ANALYZE_PARTITION_SQL_TEMPLATE));
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
index be35e96f0a..8a5dab5b26 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
@@ -18,6 +18,7 @@
package org.apache.doris.statistics;
import org.apache.doris.analysis.AlterColumnStatsStmt;
+import org.apache.doris.analysis.DropTableStatsStmt;
import org.apache.doris.analysis.TableName;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
@@ -76,6 +77,12 @@ public class StatisticsRepository {
+ FULL_QUALIFIED_COLUMN_STATISTICS_NAME + " VALUES('${id}',
${catalogId}, ${dbId}, ${tblId}, '${idxId}',"
+ "'${colId}', ${partId}, ${count}, ${ndv}, ${nullCount},
'${min}', '${max}', ${dataSize}, NOW())";
+ private static final String DROP_TABLE_STATISTICS_TEMPLATE = "DELETE FROM
" + FeConstants.INTERNAL_DB_NAME
+ + "." + StatisticConstants.STATISTIC_TBL_NAME + " WHERE
${condition}";
+
+ private static final String DROP_TABLE_HISTOGRAM_TEMPLATE = "DELETE FROM "
+ FeConstants.INTERNAL_DB_NAME
+ + "." + StatisticConstants.HISTOGRAM_TBL_NAME + " WHERE
${condition}";
+
public static ColumnStatistic queryColumnStatisticsByName(long tableId,
String colName) {
ResultRow resultRow = queryColumnStatisticById(tableId, colName);
if (resultRow == null) {
@@ -131,6 +138,59 @@ public class StatisticsRepository {
return stringJoiner.toString();
}
+ public static void dropStatistics(Long dbId,
+ Set<Long> tblIds, Set<String> colNames, Set<Long> partIds) {
+ dropStatistics(dbId, tblIds, colNames, partIds, false);
+ }
+
+ public static void dropHistogram(Long dbId,
+ Set<Long> tblIds, Set<String> colNames, Set<Long> partIds) {
+ dropStatistics(dbId, tblIds, colNames, partIds, true);
+ }
+
+ private static void dropStatistics(Long dbId,
+ Set<Long> tblIds, Set<String> colNames, Set<Long> partIds, boolean
isHistogram) {
+ if (dbId <= 0) {
+ throw new IllegalArgumentException("Database id is not
specified.");
+ }
+
+ StringBuilder predicate = new StringBuilder();
+ predicate.append(String.format("db_id = '%d'", dbId));
+
+ if (!tblIds.isEmpty()) {
+ buildPredicate("tbl_id", tblIds, predicate);
+ }
+
+ if (!colNames.isEmpty()) {
+ buildPredicate("col_id", colNames, predicate);
+ }
+
+ if (!partIds.isEmpty() && !isHistogram) {
+ // Histogram is not collected and deleted by partition
+ buildPredicate("part_id", partIds, predicate);
+ }
+
+ Map<String, String> params = new HashMap<>();
+ params.put("condition", predicate.toString());
+ StringSubstitutor stringSubstitutor = new StringSubstitutor(params);
+
+ try {
+ String statement = isHistogram ?
stringSubstitutor.replace(DROP_TABLE_HISTOGRAM_TEMPLATE) :
+ stringSubstitutor.replace(DROP_TABLE_STATISTICS_TEMPLATE);
+ StatisticsUtil.execUpdate(statement);
+ } catch (Exception e) {
+ LOG.warn("Drop statistics failed", e);
+ }
+ }
+
+ private static <T> void buildPredicate(String fieldName, Set<T>
fieldValues, StringBuilder predicate) {
+ StringJoiner predicateBuilder = new StringJoiner(",", "(", ")");
+ fieldValues.stream().map(value -> String.format("'%s'", value))
+ .forEach(predicateBuilder::add);
+ String partPredicate = String.format(" AND %s IN %s", fieldName,
predicateBuilder);
+ predicate.append(partPredicate);
+ }
+
public static void createAnalysisTask(AnalysisTaskInfo analysisTaskInfo)
throws Exception {
Map<String, String> params = new HashMap<>();
params.put("jobId", String.valueOf(analysisTaskInfo.jobId));
@@ -210,4 +270,13 @@ public class StatisticsRepository {
Env.getCurrentEnv().getStatisticsCache()
.updateCache(objects.table.getId(), -1, colName, statistic);
}
+
+ public static void dropTableStatistics(DropTableStatsStmt
dropTableStatsStmt) {
+ Long dbId = dropTableStatsStmt.getDbId();
+ Set<Long> tbIds = dropTableStatsStmt.getTbIds();
+ Set<String> cols = dropTableStatsStmt.getColumnNames();
+ Set<Long> partIds = dropTableStatsStmt.getPartitionIds();
+ dropHistogram(dbId, tbIds, cols, partIds);
+ dropStatistics(dbId, tbIds, cols, partIds);
+ }
}
diff --git a/fe/fe-core/src/main/jflex/sql_scanner.flex
b/fe/fe-core/src/main/jflex/sql_scanner.flex
index 8bb82f7c7e..1afb791773 100644
--- a/fe/fe-core/src/main/jflex/sql_scanner.flex
+++ b/fe/fe-core/src/main/jflex/sql_scanner.flex
@@ -256,6 +256,7 @@ import org.apache.doris.qe.SqlModeHelper;
keywordMap.put("if", new Integer(SqlParserSymbols.KW_IF));
keywordMap.put("immediate", new
Integer(SqlParserSymbols.KW_IMMEDIATE));
keywordMap.put("in", new Integer(SqlParserSymbols.KW_IN));
+ keywordMap.put("incremental", new
Integer(SqlParserSymbols.KW_INCREMENTAL));
keywordMap.put("index", new Integer(SqlParserSymbols.KW_INDEX));
keywordMap.put("indexes", new Integer(SqlParserSymbols.KW_INDEXES));
keywordMap.put("infile", new Integer(SqlParserSymbols.KW_INFILE));
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
