This is an automated email from the ASF dual-hosted git repository.
englefly pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new b5c66cddab6 [feat](optimizer) Auto upgrade column_statistics from 1.2
#23853
b5c66cddab6 is described below
commit b5c66cddab652411df69385bced8e704eb8148d2
Author: AKIRA <[email protected]>
AuthorDate: Tue Sep 5 15:57:23 2023 +0800
[feat](optimizer) Auto upgrade column_statistics from 1.2 #23853
In Doris ver1.2 all the stats fields is not null, which might cause insert
failure in the ver2.0. Add logic to upgrade table schema automatically
---
.../doris/catalog/InternalSchemaInitializer.java | 50 +++++++---------------
1 file changed, 16 insertions(+), 34 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java
index 52b3005d796..b7fdec73f02 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java
@@ -21,6 +21,7 @@ import org.apache.doris.analysis.ColumnDef;
import org.apache.doris.analysis.CreateDbStmt;
import org.apache.doris.analysis.CreateTableStmt;
import org.apache.doris.analysis.DistributionDesc;
+import org.apache.doris.analysis.DropTableStmt;
import org.apache.doris.analysis.HashDistributionDesc;
import org.apache.doris.analysis.KeysDesc;
import org.apache.doris.analysis.TableName;
@@ -43,12 +44,10 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
-import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.stream.Collectors;
public class InternalSchemaInitializer extends Thread {
@@ -227,42 +226,25 @@ public class InternalSchemaInitializer extends Thread {
return false;
}
Database db = optionalDatabase.get();
- return db.getTable(StatisticConstants.STATISTIC_TBL_NAME).isPresent()
- &&
db.getTable(StatisticConstants.HISTOGRAM_TBL_NAME).isPresent();
- }
+ Optional<Table> optionalStatsTbl =
db.getTable(StatisticConstants.STATISTIC_TBL_NAME);
+ if (!optionalStatsTbl.isPresent()) {
+ return false;
+ }
- /**
- * Compare whether the current internal table schema meets expectations,
- * delete and rebuild if it does not meet the table schema.
- * TODO remove this code after the table structure is stable
- */
- private boolean isTableChanged(TableName tableName, List<ColumnDef>
columnDefs) {
- try {
- String catalogName =
Env.getCurrentEnv().getInternalCatalog().getName();
- String dbName = SystemInfoService.DEFAULT_CLUSTER + ":" +
tableName.getDb();
- TableIf table = StatisticsUtil.findTable(catalogName, dbName,
tableName.getTbl());
- List<Column> existColumns = table.getBaseSchema(false);
- existColumns.sort(Comparator.comparing(Column::getName));
- List<Column> columns = columnDefs.stream()
- .map(ColumnDef::toColumn)
- .sorted(Comparator.comparing(Column::getName))
- .collect(Collectors.toList());
- if (columns.size() != existColumns.size()) {
- return true;
- }
- for (int i = 0; i < columns.size(); i++) {
- Column c1 = columns.get(i);
- Column c2 = existColumns.get(i);
- if (!c1.getName().equals(c2.getName())
- || c1.getDataType() != c2.getDataType()) {
- return true;
- }
+ Table statsTbl = optionalStatsTbl.get();
+ Optional<Column> optionalColumn =
+ statsTbl.fullSchema.stream().filter(c ->
c.getName().equals("count")).findFirst();
+ if (!optionalColumn.isPresent() ||
!optionalColumn.get().isAllowNull()) {
+ try {
+ Env.getCurrentEnv().getInternalCatalog()
+ .dropTable(new DropTableStmt(true, new TableName(null,
+ StatisticConstants.DB_NAME,
StatisticConstants.STATISTIC_TBL_NAME), true));
+ } catch (Exception e) {
+ LOG.warn("Failed to drop outdated table", e);
}
return false;
- } catch (Throwable t) {
- LOG.warn("Failed to check table schema", t);
- return false;
}
+ return db.getTable(StatisticConstants.HISTOGRAM_TBL_NAME).isPresent();
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
