This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 41d4ed8367 [Improvement](multicatalog) support show_partitions for hms
catalog (#19242)
41d4ed8367 is described below
commit 41d4ed8367c5719d2d676d143b905c8a97cd10aa
Author: Yulei-Yang <[email protected]>
AuthorDate: Thu May 11 01:17:23 2023 +0800
[Improvement](multicatalog) support show_partitions for hms catalog (#19242)
* [Improvement](multicatalog) support show_partitions for hms catalog
* update according review advice
---
.../Show-Statements/SHOW-PARTITIONS.md | 10 +++-
.../Show-Statements/SHOW-PARTITIONS.md | 10 +++-
.../apache/doris/analysis/ShowPartitionsStmt.java | 59 ++++++++++++++++------
.../org/apache/doris/datasource/CatalogIf.java | 5 ++
.../java/org/apache/doris/qe/ShowExecutor.java | 33 ++++++++++--
5 files changed, 95 insertions(+), 22 deletions(-)
diff --git
a/docs/en/docs/sql-manual/sql-reference/Show-Statements/SHOW-PARTITIONS.md
b/docs/en/docs/sql-manual/sql-reference/Show-Statements/SHOW-PARTITIONS.md
index d382ea1173..738ce39a37 100644
--- a/docs/en/docs/sql-manual/sql-reference/Show-Statements/SHOW-PARTITIONS.md
+++ b/docs/en/docs/sql-manual/sql-reference/Show-Statements/SHOW-PARTITIONS.md
@@ -32,7 +32,7 @@ SHOW PARTITIONS
### Description
- This statement is used to display partition information
+ This statement is used to display partition information for tables in
Internal catalog or Hive Catalog
grammar:
@@ -42,9 +42,17 @@ grammar:
illustrate:
+When used in Internal catalog:
1. Support the filtering of PartitionId, PartitionName, State, Buckets,
ReplicationNum, LastConsistencyCheckTime and other columns
2. TEMPORARY specifies to list temporary partitions
+<version since="dev">
+
+when used in Hive Catalog:
+Will return all partitions' name. Support multilevel partition table
+
+</version>
+
### Example
1. Display all non-temporary partition information of the specified table
under the specified db
diff --git
a/docs/zh-CN/docs/sql-manual/sql-reference/Show-Statements/SHOW-PARTITIONS.md
b/docs/zh-CN/docs/sql-manual/sql-reference/Show-Statements/SHOW-PARTITIONS.md
index 50d39536f4..f9d6ddb567 100644
---
a/docs/zh-CN/docs/sql-manual/sql-reference/Show-Statements/SHOW-PARTITIONS.md
+++
b/docs/zh-CN/docs/sql-manual/sql-reference/Show-Statements/SHOW-PARTITIONS.md
@@ -32,7 +32,7 @@ SHOW PARTITIONS
### Description
-该语句用于展示分区信息
+该语句用于展示分区信息。支持 Internal catalog 和 Hive Catalog
语法：
@@ -42,9 +42,17 @@ SHOW PARTITIONS
说明:
+对于 Internal catalog：
1.
支持PartitionId,PartitionName,State,Buckets,ReplicationNum,LastConsistencyCheckTime等列的过滤
2. TEMPORARY指定列出临时分区
+<version since="dev">
+
+对于 Hive Catalog：
+支持返回所有分区，包括多级分区
+
+</version>
+
### Example
1. 展示指定db下指定表的所有非临时分区信息
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowPartitionsStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowPartitionsStmt.java
index 2d3b0a6fa0..d2cb8b1f0f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowPartitionsStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowPartitionsStmt.java
@@ -18,10 +18,11 @@
package org.apache.doris.analysis;
import org.apache.doris.catalog.Column;
-import org.apache.doris.catalog.Database;
+import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Table;
+import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ErrorCode;
@@ -32,7 +33,8 @@ import org.apache.doris.common.proc.ProcNodeInterface;
import org.apache.doris.common.proc.ProcResult;
import org.apache.doris.common.proc.ProcService;
import org.apache.doris.common.util.OrderByPair;
-import org.apache.doris.common.util.Util;
+import org.apache.doris.datasource.CatalogIf;
+import org.apache.doris.datasource.HMSExternalCatalog;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.ShowResultSetMetaData;
@@ -57,6 +59,7 @@ public class ShowPartitionsStmt extends ShowStmt {
private static final String FILTER_REPLICATION_NUM = "ReplicationNum";
private static final String FILTER_LAST_CONSISTENCY_CHECK_TIME =
"LastConsistencyCheckTime";
+ private CatalogIf catalog;
private TableName tableName;
private Expr whereClause;
private List<OrderByElement> orderByElements;
@@ -80,6 +83,14 @@ public class ShowPartitionsStmt extends ShowStmt {
this.isTempPartition = isTempPartition;
}
+ public CatalogIf getCatalog() {
+ return catalog;
+ }
+
+ public TableName getTableName() {
+ return tableName;
+ }
+
public List<OrderByPair> getOrderByPairs() {
return orderByPairs;
}
@@ -102,15 +113,19 @@ public class ShowPartitionsStmt extends ShowStmt {
// check access
String dbName = tableName.getDb();
String tblName = tableName.getTbl();
- if
(!Env.getCurrentEnv().getAccessManager().checkTblPriv(ConnectContext.get(),
dbName, tblName,
-
PrivPredicate.SHOW)) {
+ if
(!Env.getCurrentEnv().getAccessManager().checkTblPriv(ConnectContext.get(),
catalog.getName(), dbName,
+ tblName, PrivPredicate.SHOW)) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR,
"SHOW PARTITIONS",
ConnectContext.get().getQualifiedUser(),
ConnectContext.get().getRemoteIP(),
dbName + ": " + tblName);
}
- Database db =
Env.getCurrentInternalCatalog().getDbOrAnalysisException(dbName);
- Table table = db.getTableOrMetaException(tblName,
Table.TableType.OLAP);
+ if (!catalog.isInternalCatalog()) {
+ return;
+ }
+
+ DatabaseIf db = catalog.getDbOrAnalysisException(dbName);
+ TableIf table = db.getTableOrMetaException(tblName,
Table.TableType.OLAP);
table.readLock();
try {
// build proc path
@@ -137,8 +152,16 @@ public class ShowPartitionsStmt extends ShowStmt {
public void analyzeImpl(Analyzer analyzer) throws UserException {
super.analyze(analyzer);
tableName.analyze(analyzer);
- // disallow external catalog
- Util.prohibitExternalCatalog(tableName.getCtl(),
this.getClass().getSimpleName());
+ catalog =
Env.getCurrentEnv().getCatalogMgr().getCatalog(tableName.getCtl());
+ if (catalog == null) {
+
ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_NAME_FOR_CATALOG);
+ }
+
+ // disallow unsupported catalog
+ if (!(catalog.isInternalCatalog() || catalog instanceof
HMSExternalCatalog)) {
+ throw new AnalysisException(String.format("Catalog of type '%s' is
not allowed in ShowPartitionsStmt",
+ catalog.getType()));
+ }
// analyze where clause if not null
if (whereClause != null) {
@@ -222,15 +245,19 @@ public class ShowPartitionsStmt extends ShowStmt {
public ShowResultSetMetaData getMetaData() {
ShowResultSetMetaData.Builder builder =
ShowResultSetMetaData.builder();
- ProcResult result = null;
- try {
- result = node.fetchResult();
- } catch (AnalysisException e) {
- return builder.build();
- }
+ if (catalog.isInternalCatalog()) {
+ ProcResult result = null;
+ try {
+ result = node.fetchResult();
+ } catch (AnalysisException e) {
+ return builder.build();
+ }
- for (String col : result.getColumnNames()) {
- builder.addColumn(new Column(col, ScalarType.createVarchar(30)));
+ for (String col : result.getColumnNames()) {
+ builder.addColumn(new Column(col,
ScalarType.createVarchar(30)));
+ }
+ } else {
+ builder.addColumn(new Column("Partition",
ScalarType.createVarchar(60)));
}
return builder.build();
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
index 260c83c636..eac17865a2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
@@ -17,6 +17,7 @@
package org.apache.doris.datasource;
+
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
import org.apache.doris.common.AnalysisException;
@@ -50,6 +51,10 @@ public interface CatalogIf<T extends DatabaseIf> {
List<String> getDbNames();
+ default boolean isInternalCatalog() {
+ return this instanceof InternalCatalog;
+ }
+
// Will be used when querying the information_schema table
// Unable to get db for uninitialized catalog to avoid query timeout
default List<String> getDbNamesOrEmpty() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
index 06e9f22845..f1f52357fc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
@@ -171,6 +171,7 @@ import org.apache.doris.common.util.RuntimeProfile;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.CatalogIf;
+import org.apache.doris.datasource.HMSExternalCatalog;
import org.apache.doris.external.iceberg.IcebergTableCreationRecord;
import org.apache.doris.load.DeleteHandler;
import org.apache.doris.load.ExportJob;
@@ -1525,10 +1526,34 @@ public class ShowExecutor {
private void handleShowPartitions() throws AnalysisException {
ShowPartitionsStmt showStmt = (ShowPartitionsStmt) stmt;
- ProcNodeInterface procNodeI = showStmt.getNode();
- Preconditions.checkNotNull(procNodeI);
- List<List<String>> rows = ((PartitionsProcDir)
procNodeI).fetchResultByFilter(showStmt.getFilterMap(),
- showStmt.getOrderByPairs(),
showStmt.getLimitElement()).getRows();
+ if (showStmt.getCatalog().isInternalCatalog()) {
+ ProcNodeInterface procNodeI = showStmt.getNode();
+ Preconditions.checkNotNull(procNodeI);
+ List<List<String>> rows = ((PartitionsProcDir)
procNodeI).fetchResultByFilter(showStmt.getFilterMap(),
+ showStmt.getOrderByPairs(),
showStmt.getLimitElement()).getRows();
+ resultSet = new ShowResultSet(showStmt.getMetaData(), rows);
+ } else {
+ handleShowHMSTablePartitions(showStmt);
+ }
+ }
+
+ private void handleShowHMSTablePartitions(ShowPartitionsStmt showStmt) {
+ HMSExternalCatalog catalog = (HMSExternalCatalog)
(showStmt.getCatalog());
+ List<List<String>> rows = new ArrayList<>();
+ String dbName =
ClusterNamespace.getNameFromFullName(showStmt.getTableName().getDb());
+ List<String> partitionNames =
catalog.getClient().listPartitionNames(dbName,
+ showStmt.getTableName().getTbl());
+ for (String partition : partitionNames) {
+ List<String> list = new ArrayList<>();
+ list.add(partition);
+ rows.add(list);
+ }
+
+ // sort by partition name
+ rows.sort((x, y) -> {
+ return x.get(0).compareTo(y.get(0));
+ });
+
resultSet = new ShowResultSet(showStmt.getMetaData(), rows);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
