This is an automated email from the ASF dual-hosted git repository. dataroaring pushed a commit to branch branch-3.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 9013cbcd1026f3364e59186ee0b2ad787b672ea0 Author: Tiewei Fang <[email protected]> AuthorDate: Mon May 27 10:12:06 2024 +0800 [Fix](trino-connector) Fix the properties' key for trino-connector (#35377) Previously, all property keys related to Trino were consistent with Trino. After modification, all Trino-related properties need to be prefixed with trino. --- .../format/table/trino_connector_jni_reader.cpp | 3 +-- .../trinoconnector/TrinoConnectorJniScanner.java | 8 +++--- .../constants/TrinoConnectorProperties.java | 2 +- .../TrinoConnectorExternalCatalog.java | 30 ++++++++++++++++------ .../source/TrinoConnectorScanNode.java | 2 +- .../hive/test_trino_different_parquet_types.groovy | 4 +-- .../hive/test_trino_hive_orc.groovy | 8 +++--- .../hive/test_trino_hive_other.groovy | 4 +-- .../hive/test_trino_hive_parquet.groovy | 4 +-- .../hive/test_trino_hive_schema_evolution.groovy | 4 +-- .../hive/test_trino_hive_serde_prop.groovy | 4 +-- .../hive/test_trino_hive_tablesample_p0.groovy | 4 +-- .../hive/test_trino_hive_tpch_sf1_orc.groovy | 4 +-- .../hive/test_trino_hive_tpch_sf1_parquet.groovy | 4 +-- .../test_trino_prepare_hive_data_in_case.groovy | 4 +-- 15 files changed, 50 insertions(+), 39 deletions(-) diff --git a/be/src/vec/exec/format/table/trino_connector_jni_reader.cpp b/be/src/vec/exec/format/table/trino_connector_jni_reader.cpp index c9b10e716ca..6fd86385197 100644 --- a/be/src/vec/exec/format/table/trino_connector_jni_reader.cpp +++ b/be/src/vec/exec/format/table/trino_connector_jni_reader.cpp @@ -36,8 +36,7 @@ class Block; namespace doris::vectorized { -const std::string TrinoConnectorJniReader::TRINO_CONNECTOR_OPTION_PREFIX = - "trino_connector_option_prefix."; +const std::string TrinoConnectorJniReader::TRINO_CONNECTOR_OPTION_PREFIX = "trino."; TrinoConnectorJniReader::TrinoConnectorJniReader( const std::vector<SlotDescriptor*>& file_slot_descs, RuntimeState* state, diff --git a/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorJniScanner.java b/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorJniScanner.java index a9d857693a6..bcc60337429 100644 --- a/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorJniScanner.java +++ b/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorJniScanner.java @@ -85,7 +85,7 @@ import java.util.stream.Collectors; public class TrinoConnectorJniScanner extends JniScanner { private static volatile int physicalProcessorCount = -1; private static final Logger LOG = LoggerFactory.getLogger(TrinoConnectorJniScanner.class); - private static final String TRINO_CONNECTOR_OPTION_PREFIX = "trino_connector_option_prefix."; + private static final String TRINO_CONNECTOR_PROPERTIES_PREFIX = "trino."; private final QueryIdGenerator queryIdGenerator = new QueryIdGenerator(); @@ -148,13 +148,11 @@ public class TrinoConnectorJniScanner extends JniScanner { trinoConnectorOptionParams = params.entrySet().stream() - .filter(kv -> kv.getKey().startsWith(TRINO_CONNECTOR_OPTION_PREFIX)) + .filter(kv -> kv.getKey().startsWith(TRINO_CONNECTOR_PROPERTIES_PREFIX)) .collect(Collectors - .toMap(kv1 -> kv1.getKey().substring(TRINO_CONNECTOR_OPTION_PREFIX.length()), + .toMap(kv1 -> kv1.getKey().substring(TRINO_CONNECTOR_PROPERTIES_PREFIX.length()), kv1 -> kv1.getValue())); catalogCreateTime = trinoConnectorOptionParams.remove("create_time"); - trinoConnectorOptionParams.remove("type"); - trinoConnectorOptionParams.remove("use_meta_cache"); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/TrinoConnectorProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/TrinoConnectorProperties.java index 704df6450ba..8892dcc7aa4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/TrinoConnectorProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/TrinoConnectorProperties.java @@ -18,5 +18,5 @@ package org.apache.doris.datasource.property.constants; public class TrinoConnectorProperties { - public static final String TRINO_CONNECTOR_NAME = "connector.name"; + public static final String TRINO_CONNECTOR_NAME = "trino.connector.name"; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalCatalog.java index bf146da1865..4f28407bf85 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalCatalog.java @@ -28,7 +28,6 @@ import org.apache.doris.trinoconnector.TrinoConnectorServicesProvider; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Maps; import com.google.common.util.concurrent.MoreExecutors; import io.airlift.node.NodeInfo; import io.opentelemetry.api.OpenTelemetry; @@ -84,6 +83,7 @@ import org.apache.logging.log4j.Logger; import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; @@ -95,6 +95,7 @@ import java.util.stream.Collectors; public class TrinoConnectorExternalCatalog extends ExternalCatalog { private static final Logger LOG = LogManager.getLogger(TrinoConnectorExternalCatalog.class); + private static final String TRINO_CONNECTOR_PROPERTIES_PREFIX = "trino."; private static final List<String> TRINO_CONNECTOR_REQUIRED_PROPERTIES = ImmutableList.of( TrinoConnectorProperties.TRINO_CONNECTOR_NAME @@ -104,12 +105,21 @@ public class TrinoConnectorExternalCatalog extends ExternalCatalog { private Connector connector; private ConnectorName connectorName; private Session trinoSession; + private ImmutableMap<String, String> trinoProperties; public TrinoConnectorExternalCatalog(long catalogId, String name, String resource, Map<String, String> props, String comment) { super(catalogId, name, Type.TRINO_CONNECTOR, comment); Objects.requireNonNull(name, "catalogName is null"); catalogProperty = new CatalogProperty(resource, props); + + // All properties obtained by this method are used by the trino-connector. + // We should not modify this map + trinoProperties = ImmutableMap.copyOf(catalogProperty.getProperties().entrySet().stream() + .filter(kv -> kv.getKey().startsWith(TRINO_CONNECTOR_PROPERTIES_PREFIX)) + .collect(Collectors + .toMap(kv1 -> kv1.getKey().substring(TRINO_CONNECTOR_PROPERTIES_PREFIX.length()), + kv1 -> kv1.getValue()))); } @Override @@ -185,11 +195,11 @@ public class TrinoConnectorExternalCatalog extends ExternalCatalog { private ConnectorServicesProvider createConnectorServicesProvider() { // 1. check and create ConnectorName - Map<String, String> trinoConnectorProperties = Maps.newHashMap(); - trinoConnectorProperties.putAll(catalogProperty.getProperties()); - trinoConnectorProperties.remove("create_time"); - trinoConnectorProperties.remove("type"); - trinoConnectorProperties.remove("use_meta_cache"); + if (!trinoProperties.containsKey("connector.name")) { + throw new RuntimeException("Can not find trino.connector.name property, please specify a connector name."); + } + Map<String, String> trinoConnectorProperties = new HashMap<>(); + trinoConnectorProperties.putAll(trinoProperties); String connectorNameString = trinoConnectorProperties.remove("connector.name"); Objects.requireNonNull(connectorNameString, "connectorName is null"); if (connectorNameString.indexOf('-') >= 0) { @@ -295,8 +305,12 @@ public class TrinoConnectorExternalCatalog extends ExternalCatalog { return Optional.empty(); } - public Map<String, String> getTrinoConnectorProperties() { - return catalogProperty.getProperties(); + // BE need create_time key + public Map<String, String> getTrinoConnectorPropertiesWithCreateTime() { + Map<String, String> trinoPropertiesWithCreateTime = new HashMap<>(); + trinoPropertiesWithCreateTime.putAll(trinoProperties); + trinoPropertiesWithCreateTime.put("create_time", catalogProperty.getProperties().get("create_time")); + return trinoPropertiesWithCreateTime; } public Connector getConnector() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/source/TrinoConnectorScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/source/TrinoConnectorScanNode.java index 2b09e30026c..66e50eda8b4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/source/TrinoConnectorScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/source/TrinoConnectorScanNode.java @@ -248,7 +248,7 @@ public class TrinoConnectorScanNode extends FileQueryScanNode { fileDesc.setTrinoConnectorSplit(encodeObjectToString(trinoConnectorSplit.getSplit(), objectMapperProvider)); fileDesc.setCatalogName(source.getCatalog().getName()); fileDesc.setDbName(source.getTargetTable().getDbName()); - fileDesc.setTrinoConnectorOptions(source.getCatalog().getTrinoConnectorProperties()); + fileDesc.setTrinoConnectorOptions(source.getCatalog().getTrinoConnectorPropertiesWithCreateTime()); fileDesc.setTableName(source.getTargetTable().getName()); fileDesc.setTrinoConnectorTableHandle(encodeObjectToString( source.getTrinoConnectorTableHandle(), objectMapperProvider)); diff --git a/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_different_parquet_types.groovy b/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_different_parquet_types.groovy index 63dca63c422..4a450b99901 100644 --- a/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_different_parquet_types.groovy +++ b/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_different_parquet_types.groovy @@ -190,8 +190,8 @@ suite("test_trino_different_parquet_types", "p0,external,hive,external_docker,ex sql """drop catalog if exists ${catalog_name}""" sql """create catalog if not exists ${catalog_name} properties ( "type"="trino-connector", - "connector.name"="hive", - 'hive.metastore.uri' = 'thrift://${externalEnvIp}:${hms_port}' + "trino.connector.name"="hive", + 'trino.hive.metastore.uri' = 'thrift://${externalEnvIp}:${hms_port}' );""" sql """use `${catalog_name}`.`default`""" diff --git a/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_orc.groovy b/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_orc.groovy index 602169b7975..89255cfbc56 100644 --- a/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_orc.groovy +++ b/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_orc.groovy @@ -97,8 +97,8 @@ suite("test_trino_hive_orc", "all_types,external,hive,external_docker,external_d sql """ create catalog if not exists ${catalog_name} properties ( "type"="trino-connector", - "connector.name"="hive", - 'hive.metastore.uri' = 'thrift://${externalEnvIp}:${hms_port}' + "trino.connector.name"="hive", + 'trino.hive.metastore.uri' = 'thrift://${externalEnvIp}:${hms_port}' ); """ @@ -116,8 +116,8 @@ suite("test_trino_hive_orc", "all_types,external,hive,external_docker,external_d sql """ create catalog if not exists ${catalog_name} properties ( "type"="trino-connector", - "connector.name"="hive", - 'hive.metastore.uri' = 'thrift://${externalEnvIp}:${hms_port}' + "trino.connector.name"="hive", + 'trino.hive.metastore.uri' = 'thrift://${externalEnvIp}:${hms_port}' ); """ sql """use `${catalog_name}`.`default`""" diff --git a/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_other.groovy b/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_other.groovy index 427951d0599..9d3430d1ad0 100644 --- a/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_other.groovy +++ b/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_other.groovy @@ -71,8 +71,8 @@ suite("test_trino_hive_other", "external,hive,external_docker,external_docker_hi sql """drop catalog if exists ${catalog_name}""" sql """create catalog if not exists ${catalog_name} properties ( "type"="trino-connector", - "connector.name"="hive", - 'hive.metastore.uri' = 'thrift://${externalEnvIp}:${hms_port}' + "trino.connector.name"="hive", + 'trino.hive.metastore.uri' = 'thrift://${externalEnvIp}:${hms_port}' );""" // test user's grants on external catalog diff --git a/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_parquet.groovy b/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_parquet.groovy index a060311e903..658b342dff0 100644 --- a/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_parquet.groovy +++ b/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_parquet.groovy @@ -191,8 +191,8 @@ suite("test_trino_hive_parquet", "p0,external,hive,external_docker,external_dock sql """ create catalog if not exists ${catalog_name} properties ( "type"="trino-connector", - "connector.name"="hive", - 'hive.metastore.uri' = 'thrift://${externalEnvIp}:${hms_port}' + "trino.connector.name"="hive", + 'trino.hive.metastore.uri' = 'thrift://${externalEnvIp}:${hms_port}' ); """ diff --git a/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_schema_evolution.groovy b/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_schema_evolution.groovy index a3662eb4b0c..1eb4a0f1b9e 100644 --- a/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_schema_evolution.groovy +++ b/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_schema_evolution.groovy @@ -74,8 +74,8 @@ suite("test_trino_hive_schema_evolution", "p0,external,hive,external_docker,exte sql """ create catalog if not exists ${catalog_name} properties ( "type"="trino-connector", - "connector.name"="hive", - 'hive.metastore.uri' = 'thrift://${externalEnvIp}:${hms_port}' + "trino.connector.name"="hive", + 'trino.hive.metastore.uri' = 'thrift://${externalEnvIp}:${hms_port}' ); """ sql """switch ${catalog_name}""" diff --git a/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_serde_prop.groovy b/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_serde_prop.groovy index c2caf784e45..8479c14fcbc 100644 --- a/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_serde_prop.groovy +++ b/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_serde_prop.groovy @@ -38,8 +38,8 @@ suite("test_trino_hive_serde_prop", "external_docker,hive,external_docker_hive,p sql """ create catalog if not exists ${catalog_name} properties ( "type"="trino-connector", - "connector.name"="hive", - 'hive.metastore.uri' = 'thrift://${externalEnvIp}:${hms_port}' + "trino.connector.name"="hive", + 'trino.hive.metastore.uri' = 'thrift://${externalEnvIp}:${hms_port}' ); """ diff --git a/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_tablesample_p0.groovy b/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_tablesample_p0.groovy index 83c358e05fc..be760d381df 100644 --- a/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_tablesample_p0.groovy +++ b/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_tablesample_p0.groovy @@ -38,8 +38,8 @@ suite("test_trino_hive_tablesample_p0", "all_types,p0,external,hive,external_doc sql """ create catalog if not exists ${catalog_name} properties ( "type"="trino-connector", - "connector.name"="hive", - 'hive.metastore.uri' = 'thrift://${externalEnvIp}:${hms_port}' + "trino.connector.name"="hive", + 'trino.hive.metastore.uri' = 'thrift://${externalEnvIp}:${hms_port}' ); """ sql """use `${catalog_name}`.`default`""" diff --git a/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_tpch_sf1_orc.groovy b/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_tpch_sf1_orc.groovy index 76769332f31..cafd9301753 100644 --- a/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_tpch_sf1_orc.groovy +++ b/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_tpch_sf1_orc.groovy @@ -864,8 +864,8 @@ order by sql """drop catalog if exists ${catalog_name}""" sql """create catalog if not exists ${catalog_name} properties ( "type"="trino-connector", - "connector.name"="hive", - 'hive.metastore.uri' = 'thrift://${externalEnvIp}:${hms_port}' + "trino.connector.name"="hive", + 'trino.hive.metastore.uri' = 'thrift://${externalEnvIp}:${hms_port}' );""" sql """switch ${catalog_name}""" sql """use `tpch1_orc`""" diff --git a/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_tpch_sf1_parquet.groovy b/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_tpch_sf1_parquet.groovy index e09782771fa..4034ac805ef 100644 --- a/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_tpch_sf1_parquet.groovy +++ b/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_hive_tpch_sf1_parquet.groovy @@ -865,8 +865,8 @@ order by sql """drop catalog if exists ${catalog_name}""" sql """create catalog if not exists ${catalog_name} properties ( "type"="trino-connector", - "connector.name"="hive", - 'hive.metastore.uri' = 'thrift://${externalEnvIp}:${hms_port}' + "trino.connector.name"="hive", + 'trino.hive.metastore.uri' = 'thrift://${externalEnvIp}:${hms_port}' );""" sql """switch ${catalog_name}""" diff --git a/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_prepare_hive_data_in_case.groovy b/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_prepare_hive_data_in_case.groovy index b465c2373d2..84049ac1db6 100644 --- a/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_prepare_hive_data_in_case.groovy +++ b/regression-test/suites/external_table_p0/trino_connector/hive/test_trino_prepare_hive_data_in_case.groovy @@ -48,8 +48,8 @@ suite("test_trino_prepare_hive_data_in_case", "p0,external,hive,external_docker, sql """ create catalog if not exists ${catalog_name} properties ( "type"="trino-connector", - "connector.name"="hive", - 'hive.metastore.uri' = 'thrift://${externalEnvIp}:${hms_port}' + "trino.connector.name"="hive", + 'trino.hive.metastore.uri' = 'thrift://${externalEnvIp}:${hms_port}' ); """ def values2 = sql """select count(*) from ${catalog_name}.`default`.${catalog_name};""" --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
