This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new c74c2a4f8e [fix](Metadata tvf) Metadata TVF supports read the
specified columns from Fe (#19110)
c74c2a4f8e is described below
commit c74c2a4f8eec67fef5b4c6e3e38645b1d0a66158
Author: Tiewei Fang <[email protected]>
AuthorDate: Sat Apr 29 00:06:08 2023 +0800
[fix](Metadata tvf) Metadata TVF supports read the specified columns from
Fe (#19110)
---
be/src/vec/exec/scan/vmeta_scanner.cpp | 7 ++++
.../catalog/external/JdbcExternalDatabase.java | 1 -
.../catalog/external/TestExternalDatabase.java | 1 -
.../tablefunction/BackendsTableValuedFunction.java | 34 +++++++++++++++
.../tablefunction/IcebergTableValuedFunction.java | 14 ++++++-
.../doris/tablefunction/MetadataGenerator.java | 49 +++++++++++++++++++---
.../ResourceGroupsTableValuedFunction.java | 11 +++++
gensrc/thrift/FrontendService.thrift | 1 +
.../table_valued_function/test_backends_tvf.groovy | 26 ++++++++++++
9 files changed, 136 insertions(+), 8 deletions(-)
diff --git a/be/src/vec/exec/scan/vmeta_scanner.cpp
b/be/src/vec/exec/scan/vmeta_scanner.cpp
index 647c4c7de6..db17ea9ac4 100644
--- a/be/src/vec/exec/scan/vmeta_scanner.cpp
+++ b/be/src/vec/exec/scan/vmeta_scanner.cpp
@@ -212,6 +212,13 @@ Status VMetaScanner::_fetch_metadata(const TMetaScanRange&
meta_scan_range) {
return Status::OK();
}
+ // set filter columns
+ std::vector<std::string> filter_columns;
+ for (const auto& slot : _tuple_desc->slots()) {
+ filter_columns.emplace_back(slot->col_name_lower_case());
+ }
+ request.metada_table_params.__set_columns_name(filter_columns);
+
// _state->execution_timeout() is seconds, change to milliseconds
int time_out = _state->execution_timeout() * 1000;
TNetworkAddress master_addr =
ExecEnv::GetInstance()->master_info()->network_address;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalDatabase.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalDatabase.java
index 4272d357e0..f04a389570 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalDatabase.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalDatabase.java
@@ -115,7 +115,6 @@ public class JdbcExternalDatabase extends
ExternalDatabase<JdbcExternalTable> im
initialized = true;
}
- // TODO(ftw): drew
@Override
public Set<String> getTableNamesWithLock() {
makeSureInitialized();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/TestExternalDatabase.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/TestExternalDatabase.java
index aece45d801..fe1852241d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/TestExternalDatabase.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/TestExternalDatabase.java
@@ -108,7 +108,6 @@ public class TestExternalDatabase extends
ExternalDatabase<TestExternalTable> im
initialized = true;
}
- // TODO(ftw): drew
@Override
public Set<String> getTableNamesWithLock() {
makeSureInitialized();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/BackendsTableValuedFunction.java
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/BackendsTableValuedFunction.java
index 864fcc8f80..fbf349c517 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/BackendsTableValuedFunction.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/BackendsTableValuedFunction.java
@@ -25,6 +25,7 @@ import org.apache.doris.thrift.TBackendsMetadataParams;
import org.apache.doris.thrift.TMetaScanRange;
import org.apache.doris.thrift.TMetadataType;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import java.util.List;
@@ -37,6 +38,39 @@ import java.util.Map;
public class BackendsTableValuedFunction extends MetadataTableValuedFunction {
public static final String NAME = "backends";
+ private static final ImmutableMap<String, Integer> COLUMN_TO_INDEX = new
ImmutableMap.Builder<String, Integer>()
+ .put("backendid", 0)
+ .put("cluster", 1)
+ .put("ip", 2)
+ .put("hostname", 3)
+ .put("heartbeatport", 4)
+ .put("beport", 5)
+ .put("httpport", 6)
+ .put("brpcport", 7)
+ .put("laststarttime", 8)
+ .put("lastheartbeat", 9)
+ .put("alive", 10)
+ .put("systemdecommissioned", 11)
+ .put("clusterdecommissioned", 12)
+ .put("tabletnum", 13)
+ .put("datausedcapacity", 14)
+ .put("availcapacity", 15)
+ .put("totalcapacity", 16)
+ .put("usedpct", 17)
+ .put("maxdiskusedpct", 18)
+ .put("remoteusedcapacity", 19)
+ .put("tag", 20)
+ .put("errmsg", 21)
+ .put("version", 22)
+ .put("status", 23)
+ .put("heartbeatfailurecounter", 24)
+ .put("noderole", 25)
+ .build();
+
+ public static Integer getColumnIndexFromColumnName(String columnName) {
+ return COLUMN_TO_INDEX.get(columnName.toLowerCase());
+ }
+
public BackendsTableValuedFunction(Map<String, String> params) throws
AnalysisException {
if (params.size() != 0) {
throw new AnalysisException("backends table-valued-function does
not support any params");
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/IcebergTableValuedFunction.java
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/IcebergTableValuedFunction.java
index 2be9d16915..54ceb04b03 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/IcebergTableValuedFunction.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/IcebergTableValuedFunction.java
@@ -31,6 +31,7 @@ import org.apache.doris.thrift.TIcebergQueryType;
import org.apache.doris.thrift.TMetaScanRange;
import org.apache.doris.thrift.TMetadataType;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
@@ -50,6 +51,18 @@ public class IcebergTableValuedFunction extends
MetadataTableValuedFunction {
private static final ImmutableSet<String> PROPERTIES_SET =
ImmutableSet.of(TABLE, QUERY_TYPE);
+ private static final ImmutableMap<String, Integer> COLUMN_TO_INDEX = new
ImmutableMap.Builder<String, Integer>()
+ .put("committed_at", 0)
+ .put("snapshot_id", 1)
+ .put("parent_id", 2)
+ .put("operation", 3)
+ .put("manifest_list", 4)
+ .build();
+
+ public static Integer getColumnIndexFromColumnName(String columnName) {
+ return COLUMN_TO_INDEX.get(columnName.toLowerCase());
+ }
+
private TIcebergQueryType queryType;
// here tableName represents the name of a table in Iceberg.
@@ -82,7 +95,6 @@ public class IcebergTableValuedFunction extends
MetadataTableValuedFunction {
this.icebergTableName.getDb() + ": " +
this.icebergTableName.getTbl());
}
try {
- // TODO(ftw): check here
this.queryType =
TIcebergQueryType.valueOf(queryTypeString.toUpperCase());
} catch (IllegalArgumentException e) {
throw new AnalysisException("Unsupported iceberg metadata query
type: " + queryType);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
index 1bfeca6612..f4dad26a99 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
@@ -33,6 +33,7 @@ import org.apache.doris.thrift.TFetchSchemaTableDataResult;
import org.apache.doris.thrift.TIcebergMetadataParams;
import org.apache.doris.thrift.TIcebergQueryType;
import org.apache.doris.thrift.TMetadataTableRequestParams;
+import org.apache.doris.thrift.TMetadataType;
import org.apache.doris.thrift.TRow;
import org.apache.doris.thrift.TStatus;
import org.apache.doris.thrift.TStatusCode;
@@ -63,17 +64,25 @@ public class MetadataGenerator {
if (!request.isSetMetadaTableParams()) {
return errorResult("Metadata table params is not set. ");
}
+ TFetchSchemaTableDataResult result;
+ TMetadataTableRequestParams params = request.getMetadaTableParams();
switch (request.getMetadaTableParams().getMetadataType()) {
case ICEBERG:
- return icebergMetadataResult(request.getMetadaTableParams());
+ result = icebergMetadataResult(params);
+ break;
case BACKENDS:
- return backendsMetadataResult(request.getMetadaTableParams());
+ result = backendsMetadataResult(params);
+ break;
case RESOURCE_GROUPS:
- return
resourceGroupsMetadataResult(request.getMetadaTableParams());
- default:
+ result = resourceGroupsMetadataResult(params);
break;
+ default:
+ return errorResult("Metadata table params is not set.");
}
- return errorResult("Metadata table params is not set. ");
+ if (result.getStatus().getStatusCode() == TStatusCode.OK) {
+ filterColumns(result, params.getColumnsName(),
params.getMetadataType());
+ }
+ return result;
}
@NotNull
@@ -119,6 +128,7 @@ public class MetadataGenerator {
}
trow.addToColumnValue(new
TCell().setStringVal(snapshot.operation()));
trow.addToColumnValue(new
TCell().setStringVal(snapshot.manifestListLocation()));
+
dataBatch.add(trow);
}
break;
@@ -232,6 +242,7 @@ public class MetadataGenerator {
// node role, show the value only when backend is alive.
trow.addToColumnValue(new TCell().setStringVal(backend.isAlive() ?
backend.getNodeRoleTag().value : ""));
+
dataBatch.add(trow);
}
@@ -265,6 +276,34 @@ public class MetadataGenerator {
return result;
}
+ private static void filterColumns(TFetchSchemaTableDataResult result,
+ List<String> columnNames, TMetadataType type) {
+ List<TRow> fullColumnsRow = result.getDataBatch();
+ List<TRow> filterColumnsRows = Lists.newArrayList();
+ for (TRow row : fullColumnsRow) {
+ TRow filterRow = new TRow();
+ for (String columnName : columnNames) {
+ Integer index = 0;
+ switch (type) {
+ case ICEBERG:
+ index =
IcebergTableValuedFunction.getColumnIndexFromColumnName(columnName);
+ break;
+ case BACKENDS:
+ index =
BackendsTableValuedFunction.getColumnIndexFromColumnName(columnName);
+ break;
+ case RESOURCE_GROUPS:
+ index =
ResourceGroupsTableValuedFunction.getColumnIndexFromColumnName(columnName);
+ break;
+ default:
+ break;
+ }
+ filterRow.addToColumnValue(row.getColumnValue().get(index));
+ }
+ filterColumnsRows.add(filterRow);
+ }
+ result.setDataBatch(filterColumnsRows);
+ }
+
private static org.apache.iceberg.Table getIcebergTable(HMSExternalCatalog
catalog, String db, String tbl)
throws MetaNotFoundException {
org.apache.iceberg.hive.HiveCatalog hiveCatalog = new
org.apache.iceberg.hive.HiveCatalog();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ResourceGroupsTableValuedFunction.java
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ResourceGroupsTableValuedFunction.java
index 171bf42bf1..11a1baee49 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ResourceGroupsTableValuedFunction.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ResourceGroupsTableValuedFunction.java
@@ -24,6 +24,7 @@ import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.thrift.TMetaScanRange;
import org.apache.doris.thrift.TMetadataType;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import java.util.List;
@@ -35,6 +36,16 @@ import java.util.Map;
*/
public class ResourceGroupsTableValuedFunction extends
MetadataTableValuedFunction {
public static final String NAME = "resource_groups";
+ private static final ImmutableMap<String, Integer> COLUMN_TO_INDEX = new
ImmutableMap.Builder<String, Integer>()
+ .put("id", 0)
+ .put("name", 1)
+ .put("item", 2)
+ .put("value", 3)
+ .build();
+
+ public static Integer getColumnIndexFromColumnName(String columnName) {
+ return COLUMN_TO_INDEX.get(columnName.toLowerCase());
+ }
public ResourceGroupsTableValuedFunction(Map<String, String> params)
throws AnalysisException {
if (params.size() != 0) {
diff --git a/gensrc/thrift/FrontendService.thrift
b/gensrc/thrift/FrontendService.thrift
index a88edd7680..5a049ef589 100644
--- a/gensrc/thrift/FrontendService.thrift
+++ b/gensrc/thrift/FrontendService.thrift
@@ -731,6 +731,7 @@ struct TMetadataTableRequestParams {
1: optional Types.TMetadataType metadata_type
2: optional PlanNodes.TIcebergMetadataParams iceberg_metadata_params
3: optional PlanNodes.TBackendsMetadataParams backends_metadata_params
+ 4: optional list<string> columns_name
}
struct TFetchSchemaTableDataRequest {
diff --git
a/regression-test/suites/correctness_p0/table_valued_function/test_backends_tvf.groovy
b/regression-test/suites/correctness_p0/table_valued_function/test_backends_tvf.groovy
index 19f524ee9c..3f95bcc04b 100644
---
a/regression-test/suites/correctness_p0/table_valued_function/test_backends_tvf.groovy
+++
b/regression-test/suites/correctness_p0/table_valued_function/test_backends_tvf.groovy
@@ -20,4 +20,30 @@ suite("test_backends_tvf") {
List<List<Object>> table = sql """ select * from backends(); """
assertTrue(table.size() > 0) // row should > 0
assertTrue(table[0].size == 26) // column should be 26
+
+ // filter columns
+ table = sql """ select BackendId, HostName, Alive, TotalCapacity, Version,
NodeRole from backends();"""
+ assertTrue(table.size() > 0) // row should > 0
+ assertTrue(table[0].size == 6) // column should be 26
+ assertEquals("true", table[0][2])
+
+ // case insensitive
+ table = sql """ select backendid, Hostname, alive, Totalcapacity, version,
nodeRole from backends();"""
+ assertTrue(table.size() > 0) // row should > 0
+ assertTrue(table[0].size == 6) // column should be 26
+ assertEquals("true", table[0][2])
+
+ // test aliase columns
+ table = sql """ select backendid as id, Hostname as name, alive, NodeRole
as r from backends();"""
+ assertTrue(table.size() > 0) // row should > 0
+ assertTrue(table[0].size == 4) // column should be 26
+ assertEquals("true", table[0][2])
+
+ // test changing position of columns
+ table = sql """ select Hostname as name, NodeRole as r, alive, ip from
backends();"""
+ assertTrue(table.size() > 0) // row should > 0
+ assertTrue(table[0].size == 4) // column should be 26
+ assertEquals("true", table[0][2])
+
+
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
