This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 293e8426056 [Fix](trino-connector) Fix the bug of query wrongs when
using trino-connector (#32100)
293e8426056 is described below
commit 293e84260560cb366edaf86d334377db9aa7d011
Author: Tiewei Fang <[email protected]>
AuthorDate: Wed Mar 13 10:30:42 2024 +0800
[Fix](trino-connector) Fix the bug of query wrongs when using
trino-connector (#32100)
---
.../format/table/trino_connector_jni_reader.cpp | 40 +++++++++++-----------
.../trinoconnector/TrinoConnectorJniScanner.java | 2 +-
.../source/TrinoConnectorScanNode.java | 11 +++---
gensrc/thrift/PlanNodes.thrift | 2 +-
4 files changed, 28 insertions(+), 27 deletions(-)
diff --git a/be/src/vec/exec/format/table/trino_connector_jni_reader.cpp
b/be/src/vec/exec/format/table/trino_connector_jni_reader.cpp
index bdd1c27e1ce..93e122ae0de 100644
--- a/be/src/vec/exec/format/table/trino_connector_jni_reader.cpp
+++ b/be/src/vec/exec/format/table/trino_connector_jni_reader.cpp
@@ -44,31 +44,31 @@ TrinoConnectorJniReader::TrinoConnectorJniReader(
RuntimeProfile* profile, const TFileRangeDesc& range)
: _file_slot_descs(file_slot_descs), _state(state), _profile(profile) {
std::vector<std::string> column_names;
- for (auto& desc : _file_slot_descs) {
+ for (const auto& desc : _file_slot_descs) {
std::string field = desc->col_name();
column_names.emplace_back(field);
}
- std::map<String, String> params;
- params["catalog_name"] =
range.table_format_params.trino_connector_params.catalog_name;
- params["db_name"] =
range.table_format_params.trino_connector_params.db_name;
- params["table_name"] =
range.table_format_params.trino_connector_params.table_name;
- params["trino_connector_split"] =
-
range.table_format_params.trino_connector_params.trino_connector_split;
- params["trino_connector_table_handle"] =
-
range.table_format_params.trino_connector_params.trino_connector_table_handle;
- params["trino_connector_column_handles"] =
-
range.table_format_params.trino_connector_params.trino_connector_column_handles;
- params["trino_connector_column_metadata"] =
-
range.table_format_params.trino_connector_params.trino_connector_column_metadata;
- params["trino_connector_column_names"] =
-
range.table_format_params.trino_connector_params.trino_connector_column_names;
- params["trino_connector_predicate"] =
-
range.table_format_params.trino_connector_params.trino_connector_predicate;
- params["trino_connector_trascation_handle"] =
-
range.table_format_params.trino_connector_params.trino_connector_trascation_handle;
+ std::map<String, String> params = {
+ {"catalog_name",
range.table_format_params.trino_connector_params.catalog_name},
+ {"db_name",
range.table_format_params.trino_connector_params.db_name},
+ {"table_name",
range.table_format_params.trino_connector_params.table_name},
+ {"trino_connector_split",
+
range.table_format_params.trino_connector_params.trino_connector_split},
+ {"trino_connector_table_handle",
+
range.table_format_params.trino_connector_params.trino_connector_table_handle},
+ {"trino_connector_column_handles",
+
range.table_format_params.trino_connector_params.trino_connector_column_handles},
+ {"trino_connector_column_metadata",
+
range.table_format_params.trino_connector_params.trino_connector_column_metadata},
+ {"trino_connector_predicate",
+
range.table_format_params.trino_connector_params.trino_connector_predicate},
+ {"trino_connector_trascation_handle",
+
range.table_format_params.trino_connector_params.trino_connector_trascation_handle},
+ {"required_fields", join(column_names, ",")}};
// Used to create trino connector options
- for (auto& kv :
range.table_format_params.trino_connector_params.trino_connector_options) {
+ for (const auto& kv :
+
range.table_format_params.trino_connector_params.trino_connector_options) {
params[TRINO_CONNECTOR_OPTION_PREFIX + kv.first] = kv.second;
}
_jni_connector = std::make_unique<JniConnector>(
diff --git
a/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorJniScanner.java
b/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorJniScanner.java
index 1daa57eee37..dae1149d6c7 100644
---
a/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorJniScanner.java
+++
b/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorJniScanner.java
@@ -126,7 +126,7 @@ public class TrinoConnectorJniScanner extends JniScanner {
public TrinoConnectorJniScanner(int batchSize, Map<String, String> params)
{
catalogNameString = params.get("catalog_name");
super.batchSize = batchSize;
- super.fields = params.get("trino_connector_column_names").split(",");
+ super.fields = params.get("required_fields").split(",");
super.predicates = new ScanPredicate[0];
connectorSplitString = params.get("trino_connector_split");
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/source/TrinoConnectorScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/source/TrinoConnectorScanNode.java
index 3c0d108b2cb..9e1282f349d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/source/TrinoConnectorScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/source/TrinoConnectorScanNode.java
@@ -84,7 +84,8 @@ import java.util.stream.Collectors;
public class TrinoConnectorScanNode extends FileQueryScanNode {
private static final int minScheduleSplitBatchSize = 10;
- private static TrinoConnectorSource source = null;
+ private TrinoConnectorSource source = null;
+ private ObjectMapperProvider objectMapperProvider;
// private static List<Predicate> predicates;
@@ -173,14 +174,11 @@ public class TrinoConnectorScanNode extends
FileQueryScanNode {
public void setTrinoConnectorParams(TFileRangeDesc rangeDesc,
TrinoConnectorSplit trinoConnectorSplit) {
// mock ObjectMapperProvider
- ObjectMapperProvider objectMapperProvider =
createObjectMapperProvider();
+ objectMapperProvider = createObjectMapperProvider();
// set TTrinoConnectorFileDesc
TTrinoConnectorFileDesc fileDesc = new TTrinoConnectorFileDesc();
fileDesc.setTrinoConnectorSplit(encodeObjectToString(trinoConnectorSplit.getSplit(),
objectMapperProvider));
- fileDesc.setTrinoConnectorColumnNames(
- source.getDesc().getSlots().stream().map(slot ->
slot.getColumn().getName())
- .collect(Collectors.joining(",")));
fileDesc.setCatalogName(source.getCatalog().getName());
fileDesc.setDbName(source.getTargetTable().getDbName());
fileDesc.setTrinoConnectorOptions(source.getCatalog().getTrinoConnectorProperties());
@@ -193,6 +191,9 @@ public class TrinoConnectorScanNode extends
FileQueryScanNode {
List<ColumnHandle> columnHandles = new ArrayList<>();
List<ColumnMetadata> columnMetadataList = new ArrayList<>();
for (SlotDescriptor slotDescriptor : source.getDesc().getSlots()) {
+ if (!slotDescriptor.isMaterialized()) {
+ continue;
+ }
String colName = slotDescriptor.getColumn().getName();
if (columnMetadataMap.containsKey(colName)) {
columnMetadataList.add(columnMetadataMap.get(colName));
diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift
index 51aa67aabf8..36761dd6170 100644
--- a/gensrc/thrift/PlanNodes.thrift
+++ b/gensrc/thrift/PlanNodes.thrift
@@ -324,7 +324,7 @@ struct TTrinoConnectorFileDesc {
5: optional string trino_connector_table_handle
6: optional string trino_connector_column_handles
7: optional string trino_connector_column_metadata
- 8: optional string trino_connector_column_names
+ 8: optional string trino_connector_column_names // not used
9: optional string trino_connector_split
10: optional string trino_connector_predicate
11: optional string trino_connector_trascation_handle
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
