This is an automated email from the ASF dual-hosted git repository.
Gabriel39 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 8a3d7414198 [fix](fe) Support ClickHouse JDBC V2 metadata (#65709)
8a3d7414198 is described below
commit 8a3d7414198f71c83fbb837a0bf08b81390e6ee3
Author: Gabriel <[email protected]>
AuthorDate: Fri Jul 17 23:06:32 2026 +0800
[fix](fe) Support ClickHouse JDBC V2 metadata (#65709)
ClickHouse JDBC V2 exposes databases as schemas and
reports distributed tables as `REMOTE TABLE`. Doris inferred the
database metadata mode from the legacy `databaseterm` URL parameter and
filtered the vendor table type, so V2 catalogs could not discover
databases or distributed tables. This change uses JDBC metadata
capabilities to select catalog/schema mode and includes remote tables in
discovery in both JDBC client implementations.
---
.../clickhouse/init/03-create-table.sql | 10 +++-
.../jdbc/client/JdbcClickHouseConnectorClient.java | 38 +++++++-----
.../client/JdbcClickHouseConnectorClientTest.java | 70 ++++++++++++++++++++++
.../jdbc/client/JdbcClickHouseClient.java | 23 +++----
.../jdbc/client/JdbcClickHouseClientTest.java | 25 ++++++++
.../jdbc/test_clickhouse_jdbc_v2.out | 9 +++
.../jdbc/test_clickhouse_jdbc_v2.groovy | 43 +++++++++++++
7 files changed, 187 insertions(+), 31 deletions(-)
diff --git
a/docker/thirdparties/docker-compose/clickhouse/init/03-create-table.sql
b/docker/thirdparties/docker-compose/clickhouse/init/03-create-table.sql
index 039ddd1d369..d45b878d1ed 100644
--- a/docker/thirdparties/docker-compose/clickhouse/init/03-create-table.sql
+++ b/docker/thirdparties/docker-compose/clickhouse/init/03-create-table.sql
@@ -342,6 +342,14 @@ CREATE TABLE doris_test.extreme_test
) ENGINE = MergeTree()
ORDER BY id;
+CREATE TABLE doris_test.distributed_type AS doris_test.type
+ENGINE = Distributed(default, doris_test, type, rand());
+
+CREATE MATERIALIZED VIEW doris_test.materialized_view_type
+ENGINE = MergeTree
+ORDER BY k1
+AS SELECT * FROM doris_test.type;
+
CREATE TABLE doris_test.extreme_test_multi_block
(
id UInt64,
@@ -392,4 +400,4 @@ CREATE TABLE doris_test.extreme_test_multi_block
ipv6_col IPv6,
ipv6_nullable Nullable(IPv6)
) ENGINE = MergeTree()
-ORDER BY id;
\ No newline at end of file
+ORDER BY id;
diff --git
a/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/client/JdbcClickHouseConnectorClient.java
b/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/client/JdbcClickHouseConnectorClient.java
index b945bcb6c1c..c4cea29b4aa 100644
---
a/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/client/JdbcClickHouseConnectorClient.java
+++
b/fe/fe-connector/fe-connector-jdbc/src/main/java/org/apache/doris/connector/jdbc/client/JdbcClickHouseConnectorClient.java
@@ -133,7 +133,8 @@ public class JdbcClickHouseConnectorClient extends
JdbcConnectorClient {
@Override
protected String[] getTableTypes() {
- return new String[] {"TABLE", "VIEW", "SYSTEM TABLE"};
+ // ClickHouse JDBC V2 filters engines by these vendor-specific table
type names.
+ return new String[] {"TABLE", "VIEW", "SYSTEM TABLE", "REMOTE TABLE",
"MATERIALIZED VIEW"};
}
@Override
@@ -145,25 +146,32 @@ public class JdbcClickHouseConnectorClient extends
JdbcConnectorClient {
private boolean isNewClickHouseDriver(Connection conn) throws SQLException
{
if (isNewDriver == null) {
- String driverVersion = conn.getMetaData().getDriverVersion();
- isNewDriver = driverVersion != null &&
!driverVersion.startsWith("0.3")
- && !driverVersion.startsWith("0.4");
- if (!isNewDriver) {
- // Old driver uses schema mode (not catalog), matching old
JdbcClickHouseClient
- databaseTermIsCatalog = false;
- } else {
- // New driver checks the JDBC URL for databaseterm parameter
- databaseTermIsCatalog =
"catalog".equalsIgnoreCase(getDatabaseTermFromUrl());
- }
+ DatabaseMetaData meta = conn.getMetaData();
+ String driverVersion = meta.getDriverVersion();
+ isNewDriver = isNewClickHouseDriverVersion(driverVersion);
+ databaseTermIsCatalog = isDatabaseTermCatalog(
+ driverVersion, meta.supportsCatalogsInDataManipulation());
}
return isNewDriver;
}
- private String getDatabaseTermFromUrl() {
- if (jdbcUrl != null &&
jdbcUrl.toLowerCase().contains("databaseterm=schema")) {
- return "schema";
+ static boolean isNewClickHouseDriverVersion(String driverVersion) {
+ // Custom jars may omit or rewrite manifest versions; only a known
legacy version overrides metadata.
+ if (driverVersion == null || driverVersion.isEmpty()) {
+ return true;
}
- return "catalog";
+ try {
+ String[] versionParts = driverVersion.split("\\.");
+ int majorVersion = Integer.parseInt(versionParts[0]);
+ int minorVersion = Integer.parseInt(versionParts[1]);
+ return majorVersion > 0 || (majorVersion == 0 && minorVersion >=
5);
+ } catch (NumberFormatException | ArrayIndexOutOfBoundsException e) {
+ return true;
+ }
+ }
+
+ static boolean isDatabaseTermCatalog(String driverVersion, boolean
supportsCatalogs) {
+ return isNewClickHouseDriverVersion(driverVersion) && supportsCatalogs;
}
@Override
diff --git
a/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/client/JdbcClickHouseConnectorClientTest.java
b/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/client/JdbcClickHouseConnectorClientTest.java
new file mode 100644
index 00000000000..06124b24ba0
--- /dev/null
+++
b/fe/fe-connector/fe-connector-jdbc/src/test/java/org/apache/doris/connector/jdbc/client/JdbcClickHouseConnectorClientTest.java
@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.connector.jdbc.client;
+
+import org.apache.doris.connector.jdbc.JdbcDbType;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.Collections;
+
+public class JdbcClickHouseConnectorClientTest {
+
+ private JdbcClickHouseConnectorClient createClient() {
+ return new JdbcClickHouseConnectorClient(
+ "test_catalog",
+ JdbcDbType.CLICKHOUSE,
+ "jdbc:clickhouse://localhost:8123/default",
+ false,
+ Collections.emptyMap(),
+ Collections.emptyMap(),
+ false,
+ false);
+ }
+
+ @Test
+ void testDriverVersionDetection() {
+
Assertions.assertTrue(JdbcClickHouseConnectorClient.isNewClickHouseDriverVersion("0.9.8"));
+
Assertions.assertTrue(JdbcClickHouseConnectorClient.isNewClickHouseDriverVersion("0.7.1"));
+
Assertions.assertFalse(JdbcClickHouseConnectorClient.isNewClickHouseDriverVersion("0.4.2"));
+ }
+
+ @Test
+ void testDatabaseTermFollowsDriverMetadata() {
+
Assertions.assertFalse(JdbcClickHouseConnectorClient.isDatabaseTermCatalog("0.9.8",
false));
+
Assertions.assertTrue(JdbcClickHouseConnectorClient.isDatabaseTermCatalog("0.7.1",
true));
+
Assertions.assertFalse(JdbcClickHouseConnectorClient.isDatabaseTermCatalog("0.4.2",
true));
+ }
+
+ @Test
+ void testUnknownDriverVersionFollowsDriverMetadata() {
+ for (String driverVersion : new String[] {null, "", "custom-build"}) {
+
Assertions.assertTrue(JdbcClickHouseConnectorClient.isNewClickHouseDriverVersion(driverVersion));
+
Assertions.assertTrue(JdbcClickHouseConnectorClient.isDatabaseTermCatalog(driverVersion,
true));
+
Assertions.assertFalse(JdbcClickHouseConnectorClient.isDatabaseTermCatalog(driverVersion,
false));
+ }
+ }
+
+ @Test
+ void testClickHouseSpecificTableTypesAreVisible() {
+ Assertions.assertArrayEquals(
+ new String[] {"TABLE", "VIEW", "SYSTEM TABLE", "REMOTE TABLE",
"MATERIALIZED VIEW"},
+ createClient().getTableTypes());
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClient.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClient.java
index 4f340bebed4..fec623e88f2 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClient.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClient.java
@@ -39,12 +39,9 @@ public class JdbcClickHouseClient extends JdbcClient {
protected JdbcClickHouseClient(JdbcClientConfig jdbcClientConfig) {
super(jdbcClientConfig);
try (Connection conn = getConnection()) {
- String jdbcUrl = conn.getMetaData().getURL();
- if (!isNewClickHouseDriver(getJdbcDriverVersion())) {
- this.databaseTermIsCatalog = false;
- } else {
- this.databaseTermIsCatalog =
"catalog".equalsIgnoreCase(getDatabaseTermFromUrl(jdbcUrl));
- }
+ DatabaseMetaData databaseMetaData = conn.getMetaData();
+ this.databaseTermIsCatalog = isDatabaseTermCatalog(
+ databaseMetaData, databaseMetaData.getDriverVersion());
} catch (SQLException e) {
throw new JdbcClientException("Failed to initialize
JdbcClickHouseClient: %s", e.getMessage());
}
@@ -123,7 +120,8 @@ public class JdbcClickHouseClient extends JdbcClient {
@Override
protected String[] getTableTypes() {
- return new String[] {"TABLE", "VIEW", "SYSTEM TABLE"};
+ // ClickHouse JDBC V2 filters engines by these vendor-specific table
type names.
+ return new String[] {"TABLE", "VIEW", "SYSTEM TABLE", "REMOTE TABLE",
"MATERIALIZED VIEW"};
}
@Override
@@ -231,14 +229,9 @@ public class JdbcClickHouseClient extends JdbcClient {
}
}
- /**
- * Extract databaseterm parameters from the jdbc url.
- */
- private String getDatabaseTermFromUrl(String jdbcUrl) {
- if (jdbcUrl != null &&
jdbcUrl.toLowerCase().contains("databaseterm=schema")) {
- return "schema";
- }
- return "catalog";
+ static boolean isDatabaseTermCatalog(DatabaseMetaData databaseMetaData,
String driverVersion)
+ throws SQLException {
+ return isNewClickHouseDriver(driverVersion) &&
databaseMetaData.supportsCatalogsInDataManipulation();
}
/**
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClientTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClientTest.java
index 99e4aa62dd5..a76319d108f 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClientTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/client/JdbcClickHouseClientTest.java
@@ -19,11 +19,36 @@ package org.apache.doris.datasource.jdbc.client;
import org.junit.Assert;
import org.junit.Test;
+import org.mockito.Answers;
+import org.mockito.Mockito;
import java.lang.reflect.Method;
+import java.sql.DatabaseMetaData;
public class JdbcClickHouseClientTest {
+ @Test
+ public void testDatabaseTermFollowsDriverMetadata() throws Exception {
+ DatabaseMetaData databaseMetaData =
Mockito.mock(DatabaseMetaData.class);
+
+
Mockito.when(databaseMetaData.supportsCatalogsInDataManipulation()).thenReturn(false);
+
Assert.assertFalse(JdbcClickHouseClient.isDatabaseTermCatalog(databaseMetaData,
"0.9.8"));
+
+
Mockito.when(databaseMetaData.supportsCatalogsInDataManipulation()).thenReturn(true);
+
Assert.assertTrue(JdbcClickHouseClient.isDatabaseTermCatalog(databaseMetaData,
"0.7.1"));
+
+
Assert.assertFalse(JdbcClickHouseClient.isDatabaseTermCatalog(databaseMetaData,
"0.4.2"));
+ }
+
+ @Test
+ public void testClickHouseSpecificTableTypesAreVisible() {
+ JdbcClickHouseClient client = Mockito.mock(JdbcClickHouseClient.class,
Answers.CALLS_REAL_METHODS);
+
+ Assert.assertArrayEquals(
+ new String[] {"TABLE", "VIEW", "SYSTEM TABLE", "REMOTE TABLE",
"MATERIALIZED VIEW"},
+ client.getTableTypes());
+ }
+
@Test
public void testIsNewClickHouseDriver() {
try {
diff --git
a/regression-test/data/external_table_p0/jdbc/test_clickhouse_jdbc_v2.out
b/regression-test/data/external_table_p0/jdbc/test_clickhouse_jdbc_v2.out
new file mode 100644
index 00000000000..0ad208e4be8
--- /dev/null
+++ b/regression-test/data/external_table_p0/jdbc/test_clickhouse_jdbc_v2.out
@@ -0,0 +1,9 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !clickhouse_v2_schema --
+2
+
+-- !clickhouse_v2_distributed --
+2
+
+-- !clickhouse_v2_materialized_view --
+2
diff --git
a/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_v2.groovy
b/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_v2.groovy
new file mode 100644
index 00000000000..4d986201fbe
--- /dev/null
+++
b/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_v2.groovy
@@ -0,0 +1,43 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_clickhouse_jdbc_v2", "p0,external") {
+ String enabled = context.config.otherConfigs.get("enableJdbcTest")
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ String clickhousePort =
context.config.otherConfigs.get("clickhouse_22_port")
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String s3Endpoint = getS3Endpoint()
+ String bucket = getS3BucketName()
+ // Keep large drivers in the regression bucket so isolated workers do
not require public egress.
+ String driverUrl =
"https://${bucket}.${s3Endpoint}/regression/jdbc_driver/clickhouse-jdbc-0.9.8-all.jar";
+
+ sql """ drop catalog if exists clickhouse_v2_schema """
+ sql """ create catalog clickhouse_v2_schema properties(
+ "type"="jdbc",
+ "user"="default",
+ "password"="123456",
+ "jdbc_url" =
"jdbc:clickhouse://${externalEnvIp}:${clickhousePort}/doris_test?jdbc_schema_term=schema",
+ "driver_url" = "${driverUrl}",
+ "driver_class" = "com.clickhouse.jdbc.Driver"
+ );"""
+
+ order_qt_clickhouse_v2_schema """ select count(*) from
clickhouse_v2_schema.doris_test.type """
+ order_qt_clickhouse_v2_distributed """ select count(*) from
clickhouse_v2_schema.doris_test.distributed_type """
+ order_qt_clickhouse_v2_materialized_view """ select count(*) from
clickhouse_v2_schema.doris_test.materialized_view_type """
+ sql """ drop catalog clickhouse_v2_schema """
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
