This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 729a08d6dd [fix](nereids) Fix analyze when encounter complex type
(#22310)
729a08d6dd is described below
commit 729a08d6dd909b087d5768280d7a0c426ca55cee
Author: AKIRA <[email protected]>
AuthorDate: Fri Jul 28 12:10:50 2023 +0800
[fix](nereids) Fix analyze when encounter complex type (#22310)
In the previous implementation, we couldn't handle the array type properly
---
.../java/org/apache/doris/analysis/AnalyzeTblStmt.java | 4 ++--
.../org/apache/doris/statistics/BaseAnalysisTask.java | 8 ++++----
.../org/apache/doris/statistics/util/StatisticsUtil.java | 15 +++++++++++++++
3 files changed, 21 insertions(+), 6 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java
index 03681bdb36..ddf2c417c4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeTblStmt.java
@@ -36,6 +36,7 @@ import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.statistics.AnalysisInfo.AnalysisType;
import org.apache.doris.statistics.ColumnStatistic;
+import org.apache.doris.statistics.util.StatisticsUtil;
import com.google.common.collect.Sets;
import org.apache.commons.lang3.StringUtils;
@@ -190,8 +191,7 @@ public class AnalyzeTblStmt extends AnalyzeStmt {
if (containsUnsupportedTytpe) {
if
(ConnectContext.get().getSessionVariable().ignoreColumnWithComplexType) {
columnNames = columnNames.stream()
- .filter(c ->
!ColumnStatistic.UNSUPPORTED_TYPE.contains(
- table.getColumn(c).getType()))
+ .filter(c ->
!StatisticsUtil.isUnsupportedType(table.getColumn(c).getType()))
.collect(Collectors.toList());
} else {
throw new AnalysisException(
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
index a1a52c0a11..2b40747550 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
@@ -26,8 +26,10 @@ import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.qe.StmtExecutor;
import org.apache.doris.statistics.AnalysisInfo.AnalysisMethod;
import org.apache.doris.statistics.AnalysisInfo.AnalysisType;
+import org.apache.doris.statistics.util.StatisticsUtil;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -152,10 +154,8 @@ public abstract class BaseAnalysisTask {
if (col == null) {
throw new RuntimeException(String.format("Column with name %s
not exists", info.tblName));
}
- if (isUnsupportedType(col.getType().getPrimitiveType())) {
- throw new RuntimeException(String.format("Column with type %s
is not supported",
- col.getType().toString()));
- }
+
Preconditions.checkArgument(!StatisticsUtil.isUnsupportedType(col.getType()),
+ String.format("Column with type %s is not supported",
col.getType().toString()));
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
index b118a7d02f..e8151bcf71 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
@@ -29,18 +29,22 @@ import org.apache.doris.analysis.StatementBase;
import org.apache.doris.analysis.StringLiteral;
import org.apache.doris.analysis.TableName;
import org.apache.doris.analysis.UserIdentity;
+import org.apache.doris.catalog.ArrayType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.HiveMetaStoreClientHelper;
import org.apache.doris.catalog.ListPartitionItem;
+import org.apache.doris.catalog.MapType;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.PartitionItem;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.catalog.StructType;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.Type;
+import org.apache.doris.catalog.VariantType;
import org.apache.doris.catalog.external.HMSExternalTable;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
@@ -644,4 +648,15 @@ public class StatisticsUtil {
columnStatisticBuilder.setNumNulls(columnStatisticBuilder.getNumNulls()
+ dataFile.nullValueCounts().get(colId));
}
+
+ public static boolean isUnsupportedType(Type type) {
+ if (ColumnStatistic.UNSUPPORTED_TYPE.contains(type)) {
+ return true;
+ }
+ return type instanceof ArrayType
+ || type instanceof StructType
+ || type instanceof MapType
+ || type instanceof VariantType;
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
